Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include <algorithm>
12 : #include <list>
13 : #include <vector>
14 : #include <string>
15 : #include <securec.h>
16 : #include <hccl/hccl_types.h>
17 : // ltm指定config路径
18 : #include "common/src/config.h"
19 : #include "hccl/base.h"
20 : #include "param_check_pub.h"
21 : #include "remote_access.h"
22 : #include "../op_base/src/op_base.h"
23 : #include "hccl/hcom.h"
24 : #include "rank_consistentcy_checker.h"
25 : #include "profiling_manager_pub.h"
26 : #include "topoinfo_ranktableParser_pub.h"
27 : #include "stream_pub.h"
28 : #include "hcom_common.h"
29 : #include "comm_configer.h"
30 : #include "hcom_private_v2.h"
31 : #if (!defined(HCCD)) && (!defined(CCL_KERNEL_AICPU))
32 : #include "coll_comm_mgr.h"
33 : #endif
34 :
35 : #include "hcom_pub.h"
36 :
37 : using namespace std;
38 : using namespace hccl;
39 :
40 : // DEV_TYPE_V80 对应 DevType::DEV_TYPE_V80
41 : // DEV_TYPE_V51_310_P3 对应 DevType::DEV_TYPE_310P3
42 : // DEV_TYPE_V71 对应 DevType::DEV_TYPE_V71
43 : // DEV_TYPE_V51_310_P1 对应 DevType::DEV_TYPE_310P1
44 : // DEV_TYPE_V81 对应 DevType::DEV_TYPE_V81
45 : // DEV_TYPE_950 对应 DevType::DEV_TYPE_950
46 : // DEV_TYPE_960 对应 DevType::DEV_TYPE_960
47 : // DEV_TYPE_NOSOC 对应 DevType::DEV_TYPE_NOSOC
48 :
49 0 : DevType MakeEnumToDevType(int makeEnum)
50 : {
51 : // 正向映射：MAKE_ENUM到DevType
52 : static std::map<int, DevType> makeEnumToDevType
53 : = {{0, DevType::DEV_TYPE_910}, {1, DevType::DEV_TYPE_310P3}, {2, DevType::DEV_TYPE_910B},
54 : {3, DevType::DEV_TYPE_310P1}, {4, DevType::DEV_TYPE_910_93}, {5, DevType::DEV_TYPE_950},
55 0 : {6, DevType::DEV_TYPE_960}, {7, DevType::DEV_TYPE_NOSOC}};
56 :
57 0 : auto it = makeEnumToDevType.find(makeEnum);
58 0 : if (it != makeEnumToDevType.end()) {
59 0 : return it->second;
60 : } else {
61 0 : HCCL_WARNING("Invalid MAKE_ENUM value");
62 : }
63 0 : return DevType::DEV_TYPE_NOSOC;
64 : }
65 :
66 : using HcomCreateGroupCallback = HcclResult (*)(const std::string&, const std::vector<u32>&);
67 : using HcomCallBackGroupIsInit = bool (*)(HcomInfo&);
68 : using HcomDestroyGroupCallback = HcclResult (*)(const std::string&);
69 : using HcomDestroyCallback = HcclResult (*)(HcomInfo&);
70 : HcomCreateGroupCallback g_hcomCreateGroupCallback = nullptr;
71 : HcomCallBackGroupIsInit g_hcomCallBackGroupIsInit = nullptr;
72 : HcomDestroyGroupCallback g_hcomDestroyGroupCallback = nullptr;
73 : HcomDestroyCallback g_hcomDestroyCallback = nullptr;
74 :
75 : using HcomSetGroupTopoInfoPtr = HcclResult (*)(const char*, uint32_t);
76 : using HcomUnsetGroupTopoInfoPtr = void (*)(const char*);
77 : HcomSetGroupTopoInfoPtr g_hcomSetGroupTopoInfo = nullptr;
78 : HcomUnsetGroupTopoInfoPtr g_hcomUnsetGroupTopoInfo = nullptr;
79 :
80 : using HcomInfoCtx = struct HcomInfoCtxTag {
81 : HcomInfo hcomInfo;
82 : shared_ptr<RemoteAccess> remoteAccess;
83 : vector<MemRegisterAddr> remoteAddrInfos;
84 : HcomOpTagInfo opTagInfo;
85 : bool isUsed;
86 :
87 9306 : HcomInfoCtxTag() : remoteAccess(nullptr), remoteAddrInfos(0), isUsed(false) {}
88 : };
89 :
90 : // 梯度切分相关的全局变量
91 : namespace hccl {
92 : std::map<std::string, std::vector<u32>> g_segmentIdxMap;
93 : std::map<std::string, std::vector<float>> g_segmentSizeMap;
94 : std::mutex g_segmentIdxMapLock;
95 : std::mutex g_segmentSizeMapLock;
96 : std::mutex g_setTaskNumCalModeLock;
97 : } // namespace hccl
98 :
99 : std::mutex g_hcomInfoCtxMutex;
100 : HcomInfoCtx g_hcomInfoCtx[MAX_MODULE_DEVICE_NUM + 1];
101 :
102 : std::mutex g_backloggedGroupLock;
103 : std::map<std::string, std::vector<u32>> g_backloggedGroup; // 待创建的group
104 :
105 : std::mutex g_destroyDeviceLock;
106 : static std::mutex g_taskNumCalModeMutex;
107 :
108 : static bool g_isAutoTuneModeOpen = false;
109 : static bool g_notSupportSecAddrCopyWithOffset = false;
110 :
111 716 : HcomInfoCtx& HcomGetCurHcomCtx(void)
112 : {
113 716 : std::lock_guard<std::mutex> lock(g_hcomInfoCtxMutex);
114 716 : s32 deviceLogicId = INVALID_INT;
115 716 : if (hrtGetDevice(&deviceLogicId) == HCCL_SUCCESS && (static_cast<u32>(deviceLogicId) < MAX_MODULE_DEVICE_NUM)) {
116 716 : HCCL_INFO("[HcomGetCurHcomCtx] hrtGetDevice deviceLogicId[%d] ", deviceLogicId);
117 : /* 当前线程获取到deviceId, 如果是首次使用该deviceId的Ctx, 先判断之前是否已经配置过Ctx */
118 716 : if (!g_hcomInfoCtx[deviceLogicId].isUsed) {
119 7 : HCCL_INFO("[HcomGetCurHcomCtx] is no Used deviceLogicId[%d] ", deviceLogicId);
120 7 : if (g_hcomInfoCtx[MAX_MODULE_DEVICE_NUM].isUsed) {
121 0 : return g_hcomInfoCtx[MAX_MODULE_DEVICE_NUM];
122 : }
123 : }
124 716 : g_hcomInfoCtx[deviceLogicId].isUsed = true;
125 716 : return g_hcomInfoCtx[deviceLogicId];
126 : }
127 :
128 : /* 当前线程没有获取到deviceId, 查找是否有使用过的Ctx */
129 0 : for (u32 i = 0; i <= MAX_MODULE_DEVICE_NUM; i++) {
130 0 : if (g_hcomInfoCtx[i].isUsed) {
131 0 : HCCL_INFO("[HcomGetCurHcomCtx] no set device Used deviceLogicId[%u] ", i);
132 0 : return g_hcomInfoCtx[i];
133 : }
134 : }
135 :
136 : /* 当前线程没有获取到deviceId, 使用兜底Ctx */
137 0 : HCCL_INFO("[HcomGetCurHcomCtx] use cover bottom hcomInfoCtx");
138 0 : g_hcomInfoCtx[MAX_MODULE_DEVICE_NUM].isUsed = true;
139 0 : return g_hcomInfoCtx[MAX_MODULE_DEVICE_NUM];
140 716 : }
141 :
142 0 : HcomInfo& HcomGetCtxHomInfoById(u32 idx) { return g_hcomInfoCtx[idx].hcomInfo; }
143 :
144 716 : HcomInfo& HcomGetCtxHomInfo(void)
145 : {
146 716 : HcomInfoCtx& curHcomCtx = HcomGetCurHcomCtx();
147 716 : return curHcomCtx.hcomInfo;
148 : }
149 :
150 0 : HcomOpTagInfo& HcomGetCtxOpTagInfo(void)
151 : {
152 0 : HcomInfoCtx& curHcomCtx = HcomGetCurHcomCtx();
153 0 : return curHcomCtx.opTagInfo;
154 : }
155 :
156 0 : bool& HcomGetCtxAutoTuneMode(void) { return g_isAutoTuneModeOpen; }
157 :
158 235 : HcclResult HcomSetGroupTopoInfo(const char* group, uint32_t rankSize)
159 : {
160 235 : if (group == nullptr) {
161 0 : HCCL_ERROR("[Hcom][HcomSetGroupTopoInfo] group is null, please check");
162 0 : return HCCL_E_PTR;
163 : }
164 235 : if (g_hcomSetGroupTopoInfo == nullptr) {
165 235 : HCCL_INFO("[Hcom][HcomSetGroupTopoInfo] g_hcomSetGroupTopoInfo is null");
166 235 : HcomInfo& hcomInfo = HcomGetCtxHomInfo();
167 235 : std::lock_guard<std::mutex> lock(hcomInfo.groupRankNumMapLock);
168 : // 1. 单算子流程下只记录，后续不做处理。
169 : // 2. 图模式流程下，后续会通过HcomTopoInfoFuncInstall函数调用回调函数，进行GroupTopoInfo的设置。
170 470 : hcomInfo.groupRankNumMap[std::string(group)] = rankSize;
171 235 : HCCL_RUN_INFO("[Hcom][HcomSetGroupTopoInfo] store groupRankNumMap, group:%s, rankNum:%u", group, rankSize);
172 235 : return HCCL_SUCCESS;
173 235 : }
174 0 : return g_hcomSetGroupTopoInfo(group, rankSize);
175 : }
176 :
177 2 : HcclResult HcomInitCollComm([[maybe_unused]] uint32_t rank, void** commV2, [[maybe_unused]] HcclCommPtr& comm)
178 : {
179 2 : CHK_PTR_NULL(commV2);
180 1 : HCCL_INFO("[HcomInitCollComm] CollComm init start.");
181 : #if (!defined(HCCD)) && (!defined(CCL_KERNEL_AICPU))
182 1 : HcclUs startut = TIME_NOW();
183 1 : char commName[ROOTINFO_INDENTIFIER_MAX_LENGTH] = {};
184 1 : CHK_RET(HcclGetCommNameV2(*commV2, commName));
185 : // 获取cclbuffer
186 1 : uintptr_t cclBufferAddr{0};
187 1 : std::size_t cclBufferSize{0};
188 1 : HcclMemType cclBufferMemType{HcclMemType::HCCL_MEM_TYPE_DEVICE};
189 1 : CHK_RET(HcclGetCclBuffer(*commV2, cclBufferAddr, cclBufferSize, cclBufferMemType));
190 : HcclMem cclBuffer;
191 1 : cclBuffer.size = static_cast<uint64_t>(cclBufferSize);
192 1 : cclBuffer.type = cclBufferMemType;
193 1 : cclBuffer.addr = reinterpret_cast<void*>(cclBufferAddr);
194 1 : EXCEPTION_CATCH(comm = make_shared<hccl::hcclComm>(cclBufferSize, cclBufferSize, commName), return HCCL_E_PTR);
195 1 : void* rankGraph = nullptr;
196 1 : CHK_RET(HcclGetRankGraphV2(commV2, &rankGraph));
197 1 : constexpr HcclCommConfig* config = nullptr;
198 3 : CHK_RET(comm->InitCollComm(*commV2, rankGraph, rank, cclBuffer, commName, config));
199 1 : HCCL_RUN_INFO("[%s] success, take time [%lld]us.", __func__, DURATION_US(TIME_NOW() - startut));
200 : #endif
201 1 : return HCCL_SUCCESS;
202 : }
203 :
204 236 : void HcomUnSetGroupTopoInfo(const char* group)
205 : {
206 236 : if (g_hcomUnsetGroupTopoInfo == nullptr) {
207 236 : HCCL_INFO("[Hcom][HcomUnSetGroupTopoInfo] g_hcomUnsetGroupTopoInfo is null, can not unset");
208 236 : return;
209 : }
210 0 : if (group == nullptr) {
211 0 : HCCL_INFO("[Hcom][HcomUnSetGroupTopoInfo] group is null, can not unset");
212 0 : return;
213 : }
214 0 : g_hcomUnsetGroupTopoInfo(group);
215 0 : return;
216 : }
217 :
218 5 : HcclResult HcomGetCommHandleByGroup([[maybe_unused]] const char* group, [[maybe_unused]] HcclComm* commHandle)
219 : {
220 : #if (!defined(HCCD)) && (!defined(CCL_KERNEL_AICPU))
221 5 : CHK_PTR_NULL(commHandle);
222 4 : CHK_PTR_NULL(group);
223 :
224 3 : std::shared_ptr<hcclComm> hcclComm;
225 3 : s32 deviceLogicId = 0;
226 3 : HcclResult ret = HcclDeviceRefresh(deviceLogicId);
227 3 : CHK_PRT_RET(
228 : ret != HCCL_SUCCESS,
229 : HCCL_ERROR(
230 : "[HcomGetCommHandleByGroup]HcclDeviceRefresh failed, group[%s], deviceLogicId[%d], errNo[0x%016llx]", group,
231 : deviceLogicId, HCOM_ERROR_CODE(ret)),
232 : ret);
233 :
234 : // MC2单算子和动态图下发性能优化，优先查询返回
235 3 : s32 devId = HcclGetThreadDeviceId();
236 3 : HcclOpInfoCtx& opBaseHcom = CollCommMgr::GetInstance().LegacyGetHcclExistDeviceOpInfoCtx(devId);
237 3 : std::unique_lock<std::mutex> lock(opBaseHcom.opGroupMapMutex);
238 6 : auto iter = opBaseHcom.opGroup2CommMap.find(std::string(group));
239 3 : if (iter != opBaseHcom.opGroup2CommMap.end()) {
240 2 : hcclComm = iter->second;
241 2 : CHK_PRT_RET(
242 : hcclComm == nullptr,
243 : HCCL_ERROR(
244 : "[HcomGetCommHandleByGroup]opBaseHcom.comm is null, group[%s], deviceLogicId[%d]", group,
245 : deviceLogicId),
246 : HCCL_E_PTR);
247 2 : *commHandle = static_cast<HcclComm>(hcclComm.get());
248 2 : return HCCL_SUCCESS;
249 : }
250 1 : lock.unlock();
251 :
252 1 : ret = HcomGetCommByGroup(group, hcclComm);
253 1 : CHK_PRT_RET(
254 : ret != HCCL_SUCCESS,
255 : HCCL_ERROR(
256 : "[HcomGetCommHandleByGroup]HcomGetCommByGroup failed, group[%s], deviceLogicId[%d], errNo[0x%016llx]",
257 : group, deviceLogicId, HCOM_ERROR_CODE(ret)),
258 : ret);
259 0 : *commHandle = static_cast<HcclComm>(hcclComm.get());
260 : #endif
261 0 : return HCCL_SUCCESS;
262 3 : }
263 :
264 2 : HcclResult HcomGetCommByGroup(const char* group, std::shared_ptr<hccl::hcclComm>& hcclComm)
265 : {
266 2 : HcomInfo& hcomInfo = HcomGetCtxHomInfo();
267 2 : HcclResult ret = HcomCheckGroupName(group);
268 2 : CHK_PRT_RET(
269 : ret != HCCL_SUCCESS,
270 : HCCL_ERROR(
271 : "[Get][CommByGroup]HcomCheckGroupName failed, group[%s], errNo[0x%016llx]", group, HCOM_ERROR_CODE(ret)),
272 : ret);
273 2 : std::string strGroup = (group == nullptr) ? HCCL_WORLD_GROUP : group;
274 2 : if (strGroup == HCCL_WORLD_GROUP) {
275 0 : CHK_PRT_RET(hcomInfo.pComm == nullptr, HCCL_WARNING("[Get][CommByGroup]hcomInfo.pComm is null"), HCCL_E_PTR);
276 0 : hcclComm = hcomInfo.pComm;
277 : } else {
278 2 : std::unique_lock<std::mutex> groupParaLock(hcomInfo.groupParamsLock);
279 2 : auto iter = hcomInfo.hcomGroupMap.find(strGroup);
280 2 : if (iter == hcomInfo.hcomGroupMap.end()) {
281 2 : ret = HcclGetCommHandle(strGroup.c_str(), hcclComm);
282 2 : CHK_PRT_RET(
283 : ret != HCCL_SUCCESS,
284 : HCCL_WARNING(
285 : "[Get][CommByGroup]errNo[0x%016llx] group[%s]"
286 : "group is not exist",
287 : HCOM_ERROR_CODE(HCCL_E_NOT_FOUND), strGroup.c_str()),
288 : HCCL_E_NOT_FOUND);
289 : } else {
290 0 : hcclComm = (iter->second).pSubComm;
291 0 : CHK_PRT_RET(
292 : hcclComm == nullptr, HCCL_ERROR("[Get][CommByGroup] Get Comm is null, group[%s]", strGroup.c_str()),
293 : HCCL_E_PTR);
294 : }
295 1 : groupParaLock.unlock();
296 2 : }
297 :
298 1 : return HCCL_SUCCESS;
299 2 : }
300 :
301 0 : void HcomTopoInfoRegCallback(HcclResult (*p1)(const char*, uint32_t), void (*p2)(const char*))
302 : {
303 0 : g_hcomSetGroupTopoInfo = p1;
304 0 : g_hcomUnsetGroupTopoInfo = p2;
305 :
306 0 : HcomInfo& hcomInfo = HcomGetCtxHomInfo();
307 0 : std::unique_lock<std::mutex> lock(hcomInfo.groupRankNumMapLock);
308 0 : HCCL_RUN_INFO("[HcomTopoInfoRegCallback] size of groupRankNumMap is %zu", hcomInfo.groupRankNumMap.size());
309 0 : for (auto& item : hcomInfo.groupRankNumMap) {
310 0 : HCCL_RUN_INFO(
311 : "[HcomTopoInfoRegCallback] try to set topo info, group:%s, rankNum:%u", item.first.c_str(), item.second);
312 0 : HcclResult ret = HcomSetGroupTopoInfo(item.first.c_str(), item.second);
313 0 : if (ret != HCCL_SUCCESS) {
314 0 : HCCL_ERROR(
315 : "[HcomTopoInfoRegCallback][HcomSetGroupTopoInfo]Set Info failed. errNo[0x%016llx].",
316 : HCOM_ERROR_CODE(ret));
317 : }
318 : }
319 0 : }
320 :
321 : HcclResult HcomStoreBackloggedGroup(const std::string& group, const std::vector<u32>& groupRanks);
322 : HcclResult HcomQueryGroupRef(const char* group, u32& groupRef);
323 : HcclResult HcomDestroyBackloggedGroup(const std::string& group);
324 : HcclResult GetGroupRankInfo(const char* group, RankInfoType rankType, u32 inPara, u32* outPara);
325 :
326 0 : HcclResult GetRankList(u32 rankNum, const u32* rankIds, HcclGroupParams& params)
327 : {
328 0 : HcomInfo& hcomInfo = HcomGetCtxHomInfo();
329 0 : CHK_PTR_NULL(hcomInfo.pComm);
330 0 : std::vector<RankInfo_t> rankList;
331 0 : params.totalRanks = rankNum;
332 0 : params.worldRank = hcomInfo.params.rank;
333 0 : params.groupRank = INVALID_VALUE_RANKID;
334 0 : for (u32 i = 0; i < rankNum; i++) {
335 0 : params.groupRanks.push_back(rankIds[i]);
336 : }
337 :
338 0 : std::sort(params.groupRanks.begin(), params.groupRanks.end());
339 0 : if (params.groupRanks[rankNum - 1] >= hcomInfo.rankTable.rankNum) {
340 0 : HCCL_ERROR(
341 : "[get][RankList]errNo[0x%016llx] groupRanks[%u]:%u is invalid", HCOM_ERROR_CODE(HCCL_E_PARA), rankNum - 1,
342 : params.groupRanks[rankNum - 1]);
343 0 : return HCCL_E_PARA;
344 : }
345 0 : if (hcomInfo.rankTable.rankList.size() <= params.groupRanks[0]) {
346 0 : HCCL_ERROR(
347 : "[get][RankList]errNo[0x%016llx] groupRanks[0] is invalid:[%u]", HCOM_ERROR_CODE(HCCL_E_PARA),
348 : params.groupRanks[0]);
349 0 : return HCCL_E_PARA;
350 : }
351 : // groupRanks 个数已经校验非0
352 0 : std::string serverId = hcomInfo.rankTable.rankList[params.groupRanks[0]].serverId;
353 0 : u32 serverNum = 1; // severNum初始值应为1，代表groupId为0的serverId;
354 0 : RankInfo_t rankInfo;
355 0 : for (u32 i = 0; i < rankNum; i++) {
356 0 : rankInfo = hcomInfo.rankTable.rankList[params.groupRanks[i]];
357 : // 校验worldRankID
358 0 : if (rankInfo.rankId != params.groupRanks[i]) {
359 0 : HCCL_ERROR(
360 : "[get][RankList]errNo[0x%016llx] in rankList, worldRanks[%u] is invalid", HCOM_ERROR_CODE(HCCL_E_PARA),
361 : rankInfo.rankId);
362 0 : return HCCL_E_PARA;
363 : }
364 0 : if (params.groupRanks[i] == params.worldRank) {
365 0 : params.groupRank = i;
366 : }
367 0 : if (rankInfo.serverId != serverId) {
368 0 : serverNum++;
369 0 : serverId = rankInfo.serverId;
370 : }
371 0 : rankInfo.rankId = i; // 放入groupRankid
372 0 : rankList.push_back(rankInfo); // ranktable中的ranklist是以rankid的顺序排列的
373 0 : rankInfo.serverId = ""; // 释放前先指空字符串
374 : }
375 0 : params.serverNum = serverNum;
376 0 : bool isStandardCard = false;
377 0 : CHK_RET(hcomInfo.pComm->IsStandardCard(isStandardCard));
378 :
379 0 : if (!isStandardCard && hcomInfo.params.deviceType != DevType::DEV_TYPE_910B
380 0 : && hcomInfo.params.deviceType != DevType::DEV_TYPE_910_93) {
381 0 : CHK_RET(CheckRankTableConfigInfo(rankList, rankNum, serverNum));
382 : }
383 0 : return HCCL_SUCCESS;
384 0 : }
385 :
386 0 : HcclResult HcomCreateGroupImpl(const std::string& group, const std::vector<u32>& rankIds)
387 : {
388 0 : HcclUs startut = TIME_NOW();
389 0 : HcomInfo& hcomInfo = HcomGetCtxHomInfo();
390 0 : std::string rankId;
391 0 : for (u32 i = 0; i < rankIds.size(); i++) {
392 0 : if (i < rankIds.size() - 1) {
393 0 : rankId += to_string(rankIds[i]) + ',';
394 0 : } else if (i == rankIds.size() - 1) {
395 0 : rankId += to_string(rankIds[i]);
396 : }
397 : }
398 : /* 接口交互信息日志 */
399 0 : HCCL_RUN_INFO(
400 : "Entry-HcomCreateGroup:group[%s], rankNum[%u], rankIds[%s]", group.c_str(), rankIds.size(), rankId.c_str());
401 :
402 0 : if (hcomInfo.params.commWorkMode == HCCL_MODE_NORMAL) {
403 0 : CHK_PRT_RET(
404 : hcomInfo.pComm == nullptr,
405 : HCCL_ERROR("[Create][Group]hcomInfo.pComm is null, please check if the initialize process is called."),
406 : HCCL_E_PTR);
407 : } else {
408 0 : if (g_hcomCreateGroupCallback != nullptr) {
409 0 : return g_hcomCreateGroupCallback(group, rankIds);
410 : }
411 : }
412 :
413 0 : CHK_PRT_RET(
414 : hcomInfo.rankTable.rankList.empty(), HCCL_ERROR("[Create][Group]group[%s] rankList is empty", group.c_str()),
415 : HCCL_E_INTERNAL);
416 :
417 : /* 已经存在的group不允许再次创建 */
418 0 : std::unique_lock<std::mutex> groupParaLock(hcomInfo.groupParamsLock);
419 0 : if (hcomInfo.hcomGroupMap.find(group) != hcomInfo.hcomGroupMap.end()) {
420 0 : HCCL_ERROR(
421 : "[Create][Group]errNo[0x%016llx] group[%s] is already exist", HCOM_ERROR_CODE(HCCL_E_PARA), group.c_str());
422 0 : return HCCL_E_PARA;
423 : }
424 0 : groupParaLock.unlock();
425 :
426 0 : HcclGroupParams groupParamsTem;
427 0 : CHK_RET(GetRankList(rankIds.size(), rankIds.data(), groupParamsTem));
428 :
429 : // 如果是groupRank = INVALID_VALUE_RANKID，即本rank不参与create group
430 0 : if (groupParamsTem.groupRank == INVALID_VALUE_RANKID) {
431 0 : HCCL_ERROR(
432 : "[Create][Group]errNo[0x%016llx] confirm groupRank from worldRank[%u] error",
433 : HCOM_ERROR_CODE(HCCL_E_NOT_FOUND), hcomInfo.params.rank);
434 0 : return HCCL_E_NOT_FOUND;
435 : }
436 :
437 : /* 入参的正确性由HCCL确保 */
438 0 : CHK_RET(hcomInfo.pComm->CreateGroup(
439 : group, groupParamsTem.groupRank, hcomInfo.params.rank, groupParamsTem.groupRanks, groupParamsTem.pSubComm));
440 0 : CHK_SMART_PTR_NULL(groupParamsTem.pSubComm);
441 :
442 0 : groupParaLock.lock();
443 0 : hcomInfo.hcomGroupMap.insert(std::make_pair(group, groupParamsTem));
444 0 : groupParaLock.unlock();
445 :
446 0 : HCCL_RUN_INFO(
447 : "hcom create group[%s] success, take time [%lld]us", group.c_str(), DURATION_US(TIME_NOW() - startut));
448 0 : return HCCL_SUCCESS;
449 0 : }
450 :
451 0 : HcclResult HcomCreateGroup(const char* group, u32 rankNum, u32* rankIds)
452 : {
453 : /* 调优模式直接返回success */
454 0 : bool& isAutoTuneModeOpen = HcomGetCtxAutoTuneMode();
455 0 : if (isAutoTuneModeOpen) {
456 0 : return HCCL_SUCCESS;
457 : }
458 0 : RPT_INPUT_ERR(
459 : group == nullptr, "EI0003", std::vector<std::string>({"ccl_op", "value", "parameter", "expect"}),
460 : std::vector<std::string>({"HcomCreateGroup", "nullptr", "group", "non-null pointer"}));
461 0 : CHK_PTR_NULL(group);
462 0 : HcclResult ret = HcomCheckGroupName(group);
463 0 : RPT_INPUT_ERR(
464 : ret != HCCL_SUCCESS, "EI0003", std::vector<std::string>({"ccl_op", "value", "parameter", "value"}),
465 : std::vector<std::string>(
466 : {"HcomCreateGroup",
467 : {group, strnlen(group, GROUP_NAME_MAX_LEN + 1)},
468 : "group",
469 : "a non-empty string of length 1 to " + std::to_string(GROUP_NAME_MAX_LEN)
470 : + ", containing only alphanumeric characters and underscores"}));
471 0 : CHK_PRT_RET(
472 : ret != HCCL_SUCCESS,
473 : HCCL_ERROR(
474 : "[%s][%s]errNo[0x%016llx] group name is invalid", LOG_KEYWORDS_TASK_EXEC.c_str(),
475 : LOG_KEYWORDS_INVALID_ARGUMENT.c_str(), HCOM_ERROR_CODE(ret)),
476 : ret);
477 0 : CHK_PRT_RET(
478 : (!strncmp(group, HCCL_WORLD_GROUP, sizeof(HCCL_WORLD_GROUP))),
479 : HCCL_ERROR(
480 : "[%s][%s]create group isn't support world group", LOG_KEYWORDS_TASK_EXEC.c_str(),
481 : LOG_KEYWORDS_INVALID_ARGUMENT.c_str()),
482 : HCCL_E_PARA);
483 :
484 0 : RPT_INPUT_ERR(
485 : rankIds == nullptr, "EI0003", std::vector<std::string>({"ccl_op", "value", "parameter", "expect"}),
486 : std::vector<std::string>({"HcomCreateGroup", "nullptr", "rankIds", "non-null pointer"}));
487 0 : CHK_PTR_NULL(rankIds);
488 :
489 0 : if (rankNum == 0) {
490 0 : RPT_INPUT_ERR(
491 : true, "EI0003", std::vector<std::string>({"ccl_op", "value", "parameter", "expect"}),
492 : std::vector<std::string>(
493 : {"HcomCreateGroup", std::to_string(rankNum), "rankNum",
494 : "must be a positive integer (greater than 0)"}));
495 0 : HCCL_ERROR(
496 : "[%s][%s]errNo[0x%016llx] group[%s] rankNum[%u] is invalid", LOG_KEYWORDS_TASK_EXEC.c_str(),
497 : LOG_KEYWORDS_INVALID_ARGUMENT.c_str(), HCOM_ERROR_CODE(HCCL_E_PARA), group, rankNum);
498 0 : return HCCL_E_PARA;
499 : }
500 : // 入参合法性校验 END
501 0 : std::vector<u32> ranks(rankIds, rankIds + rankNum);
502 0 : HcomInfo& hcomInfo = HcomGetCtxHomInfo();
503 : #if (!defined(HCCD)) && (!defined(CCL_KERNEL_AICPU))
504 0 : HCCLV2_FUNC_RUN([&]() -> HcclResult {
505 : CHK_RET(HcomCreateGroupImplV2(group, rankNum, ranks));
506 : HcclGroupParams groupParams{};
507 : void* commV2 = nullptr;
508 : CHK_RET(HcomGetGroupParamsV2(group, static_cast<void*>(&groupParams), &commV2));
509 : Hccl::RankId rank = static_cast<Hccl::RankId>(groupParams.groupRank);
510 : CHK_RET(HcomInitCollComm(rank, &commV2, groupParams.pSubComm));
511 : CHK_PTR_NULL(groupParams.pSubComm);
512 : std::unique_lock<std::mutex> groupParaLock(hcomInfo.groupParamsLock);
513 : hcomInfo.hcomGroupMap.insert(std::make_pair(group, groupParams));
514 : groupParaLock.unlock();
515 : CHK_RET(HcomSetGroupTopoInfo(group, rankNum));
516 : HCCL_INFO("[HcomCreateGroup] create group[%s] success.", group);
517 : return HCCL_SUCCESS;
518 : }());
519 : #endif
520 0 : if (hcomInfo.pComm == nullptr
521 0 : && ((g_hcomCallBackGroupIsInit != nullptr) && (!(g_hcomCallBackGroupIsInit(hcomInfo))))) {
522 0 : CHK_RET(HcomStoreBackloggedGroup(group, ranks));
523 : } else {
524 0 : CHK_RET(HcomCreateGroupImpl(group, ranks));
525 0 : CHK_RET(HcomSetGroupTopoInfo(group, rankNum));
526 : }
527 0 : return HCCL_SUCCESS;
528 0 : }
529 :
530 0 : HcclResult DestroyFlag(const char* group, bool flag)
531 : {
532 0 : HcomInfo& hcomInfo = HcomGetCtxHomInfo();
533 :
534 0 : if (group == nullptr) {
535 0 : return HCCL_SUCCESS; // 全局通信域不需要查询flag
536 : }
537 0 : std::string strGroup = group;
538 0 : std::unique_lock<std::mutex> groupParaLock(hcomInfo.groupParamsLock);
539 0 : auto iter = hcomInfo.hcomGroupMap.find(strGroup);
540 0 : if (iter == hcomInfo.hcomGroupMap.end()) {
541 0 : HCCL_ERROR(
542 : "[Get][CommByGroup]errNo[0x%016llx] group[%s] group is not exist", HCOM_ERROR_CODE(HCCL_E_NOT_FOUND),
543 : strGroup.c_str());
544 0 : return HCCL_E_NOT_FOUND; // 不存在该服务器内相关dev的对应信息
545 : }
546 0 : iter->second.destroyFlag = flag;
547 0 : return HCCL_SUCCESS;
548 0 : }
549 :
550 0 : HcclResult QueryDestroyFlag(const char* group)
551 : {
552 0 : HcomInfo& hcomInfo = HcomGetCtxHomInfo();
553 0 : if (group == nullptr) {
554 0 : return HCCL_SUCCESS; // 全局通信域不需要查询flag
555 : }
556 0 : std::string strGroup = group;
557 0 : std::unique_lock<std::mutex> groupParaLock(hcomInfo.groupParamsLock);
558 0 : auto iter = hcomInfo.hcomGroupMap.find(strGroup);
559 0 : if (iter == hcomInfo.hcomGroupMap.end()) {
560 0 : HCCL_WARNING(
561 : "[Get][CommByGroup]errNo[0x%016llx] group[%s] group is not exist", HCOM_ERROR_CODE(HCCL_E_AGAIN),
562 : strGroup.c_str());
563 0 : return HCCL_E_AGAIN; // 不存在该服务器内相关dev的对应信息
564 : }
565 0 : if (iter->second.destroyFlag) {
566 0 : return HCCL_E_AGAIN;
567 : }
568 0 : return HCCL_SUCCESS;
569 0 : }
570 :
571 0 : HcclResult HcomDestroyGroupImpl(const std::string& group)
572 : {
573 : /* 调优模式直接返回success */
574 0 : bool& isAutoTuneModeOpen = HcomGetCtxAutoTuneMode();
575 0 : if (isAutoTuneModeOpen) {
576 0 : return HCCL_SUCCESS;
577 : }
578 0 : HcomInfo& hcomInfo = HcomGetCtxHomInfo();
579 0 : CHK_PRT_RET(
580 : hcomInfo.pComm == nullptr,
581 : HCCL_ERROR("[Destroy][Group]hcomInfo.pComm is null, "
582 : "please check if the initialize process is called."),
583 : HCCL_E_PTR);
584 0 : if (hcomInfo.params.commWorkMode != HCCL_MODE_NORMAL) {
585 0 : if (g_hcomDestroyGroupCallback != nullptr) {
586 0 : return g_hcomDestroyGroupCallback(group);
587 : }
588 : }
589 :
590 : /* 接口交互信息日志 */
591 0 : HCCL_RUN_INFO("Entry-HcomDestroyGroup:group[%s]", group.c_str());
592 0 : CHK_RET(DestroyFlag(group.c_str(), true));
593 0 : u32 ref = 0;
594 0 : CHK_RET(HcomQueryGroupRef(group.c_str(), ref));
595 0 : while (ref != 0) {
596 0 : std::shared_ptr<hccl::hcclComm> hcclComm = nullptr;
597 0 : CHK_RET(HcomGetCommByGroup(group.c_str(), hcclComm));
598 0 : SaluSleep(ONE_HUNDRED_MICROSECOND_OF_USLEEP);
599 0 : CHK_RET(HcomQueryGroupRef(group.c_str(), ref));
600 0 : }
601 :
602 0 : std::unique_lock<std::mutex> groupParaLock(hcomInfo.groupParamsLock);
603 0 : auto iter = hcomInfo.hcomGroupMap.find(group);
604 0 : if (iter == hcomInfo.hcomGroupMap.end()) {
605 0 : HCCL_ERROR(
606 : "[Destroy][Group]errNo[0x%016llx] group[%s] is not exist", HCOM_ERROR_CODE(HCCL_E_PARA), group.c_str());
607 0 : return HCCL_E_PARA;
608 : }
609 :
610 0 : CHK_RET(hcomInfo.pComm->DestroyGroup(group));
611 :
612 0 : (iter->second).groupRanks.clear(); // 清除该服务器内相关group的对应信息
613 :
614 0 : hcomInfo.hcomGroupMap.erase(group);
615 0 : groupParaLock.unlock();
616 :
617 0 : HCCL_RUN_INFO("hcom destroy group[%s] success.", group.c_str());
618 0 : return HCCL_SUCCESS;
619 0 : }
620 :
621 0 : HcclResult HcomDestroyGroup(const char* group)
622 : {
623 : /* 调优模式直接返回success */
624 0 : bool& isAutoTuneModeOpen = HcomGetCtxAutoTuneMode();
625 0 : if (isAutoTuneModeOpen) {
626 0 : return HCCL_SUCCESS;
627 : }
628 :
629 0 : RPT_INPUT_ERR(
630 : group == nullptr, "EI0003", std::vector<std::string>({"ccl_op", "value", "parameter", "expect"}),
631 : std::vector<std::string>({"HcomDestroyGroup", "nullptr", "group", "non-null pointer"}));
632 0 : CHK_PTR_NULL(group);
633 0 : CHK_RET(HcomCheckGroupName(group));
634 :
635 0 : if (!strncmp(group, HCCL_WORLD_GROUP, sizeof(HCCL_WORLD_GROUP))) {
636 0 : HCCL_ERROR(
637 : "[%s][%s]errNo[0x%016llx] destroy group is world group", LOG_KEYWORDS_TASK_EXEC.c_str(),
638 : LOG_KEYWORDS_INVALID_ARGUMENT.c_str(), HCOM_ERROR_CODE(HCCL_E_PARA));
639 0 : return HCCL_E_PARA;
640 : }
641 0 : HcomInfo& hcomInfo = HcomGetCtxHomInfo();
642 : #if (!defined(HCCD)) && (!defined(CCL_KERNEL_AICPU))
643 0 : HCCLV2_FUNC_RUN([&]() -> HcclResult {
644 : std::unique_lock<std::mutex> groupParaLock(hcomInfo.groupParamsLock);
645 : CHK_RET(HcomDestroyGroupImplV2(group));
646 : auto iter = hcomInfo.hcomGroupMap.find(group);
647 : if (iter == hcomInfo.hcomGroupMap.end()) {
648 : HCCL_ERROR("[Destroy][Group]errNo[0x%016llx] group[%s] is not exist", HCOM_ERROR_CODE(HCCL_E_PARA), group);
649 : return HCCL_E_PARA;
650 : }
651 : hcomInfo.hcomGroupMap.erase(group);
652 : groupParaLock.unlock();
653 : return HCCL_SUCCESS;
654 : }());
655 : #endif
656 :
657 0 : if (hcomInfo.pComm == nullptr
658 0 : && ((g_hcomCallBackGroupIsInit != nullptr) && (!(g_hcomCallBackGroupIsInit(hcomInfo))))) {
659 0 : CHK_RET(HcomDestroyBackloggedGroup(group));
660 : } else {
661 0 : CHK_RET(HcomDestroyGroupImpl(group));
662 : }
663 :
664 0 : HcomUnSetGroupTopoInfo(group);
665 :
666 0 : std::unique_lock<std::mutex> lock(g_backloggedGroupLock);
667 0 : if (g_backloggedGroup.find(group) != hcomInfo.backloggedGroup.end()) {
668 0 : g_backloggedGroup.erase(group);
669 0 : HCCL_INFO("hcom delete g_backlogged group[%s] success.", group);
670 : }
671 0 : return HCCL_SUCCESS;
672 0 : }
673 :
674 0 : HcclResult HcomFlushBackloggedGroups()
675 : {
676 0 : HCCL_INFO("HcomFlushBackloggedGroups");
677 0 : HcomInfo& hcomInfo = HcomGetCtxHomInfo();
678 0 : std::unique_lock<std::mutex> backGroupParaLock(g_backloggedGroupLock);
679 : using ITER = map<string, std::vector<u32>>::iterator;
680 0 : for (ITER iter = g_backloggedGroup.begin(); iter != g_backloggedGroup.end();) {
681 0 : HCCL_INFO("HcomFlushBackloggedGroups[%s], rank[%u]", iter->first.c_str(), hcomInfo.params.rank);
682 0 : if (std::count(iter->second.begin(), iter->second.end(), hcomInfo.params.rank)) {
683 0 : HCCL_INFO("HcomFlushBackloggedGroups[%s], rank[%u] success", iter->first.c_str(), hcomInfo.params.rank);
684 0 : hcomInfo.backloggedGroup.insert({iter->first, iter->second});
685 : }
686 0 : iter++;
687 : }
688 0 : backGroupParaLock.unlock();
689 :
690 0 : std::unique_lock<std::mutex> lock(hcomInfo.backloggedGroupLock);
691 0 : for (ITER iter = hcomInfo.backloggedGroup.begin(); iter != hcomInfo.backloggedGroup.end();) {
692 0 : CHK_RET(HcomCreateGroupImpl(iter->first, iter->second));
693 0 : hcomInfo.backloggedGroup.erase(iter++);
694 : }
695 0 : HCCL_INFO("HcomFlushBackloggedGroups success.");
696 0 : return HCCL_SUCCESS;
697 0 : }
698 :
699 0 : HcclResult HcomStoreBackloggedGroup(const std::string& group, const std::vector<u32>& groupRanks)
700 : {
701 0 : HcomInfo& hcomInfo = HcomGetCtxHomInfo();
702 : // 该线程最开始是否未设置deviceid并获取了ctx
703 0 : bool hcomUseDefaultCtx = (&hcomInfo == &(g_hcomInfoCtx[MAX_MODULE_DEVICE_NUM].hcomInfo));
704 0 : s32 deviceLogicId = INVALID_INT;
705 0 : if (hrtGetDevice(&deviceLogicId) != HCCL_SUCCESS || (static_cast<u32>(deviceLogicId) >= MAX_MODULE_DEVICE_NUM)
706 0 : || hcomUseDefaultCtx) {
707 0 : HCCL_INFO("[device not set]hcom store group[%s]", group.c_str());
708 0 : std::unique_lock<std::mutex> groupParaLock(g_backloggedGroupLock);
709 0 : if (g_backloggedGroup.find(group) != g_backloggedGroup.end()) {
710 0 : HCCL_INFO("[Store][BackloggedGroup]group[%s] is existed", group.c_str());
711 0 : if (g_backloggedGroup[group] == groupRanks) {
712 0 : HCCL_ERROR("[Store][BackloggedGroup]group[%s] has been created", group.c_str());
713 0 : return HCCL_E_PARA;
714 : }
715 0 : g_backloggedGroup[group] = groupRanks;
716 0 : HCCL_INFO("[Store][BackloggedGroup]group[%s] updated", group.c_str());
717 0 : return HCCL_SUCCESS;
718 : }
719 :
720 0 : g_backloggedGroup.insert({group, groupRanks});
721 0 : HCCL_INFO("[device not set]hcom store group[%s] success", group.c_str());
722 0 : return HCCL_SUCCESS;
723 0 : }
724 :
725 0 : std::unique_lock<std::mutex> groupParaLock(hcomInfo.groupParamsLock);
726 0 : if (hcomInfo.hcomGroupMap.find(group) != hcomInfo.hcomGroupMap.end()) {
727 0 : HCCL_ERROR("[Store][BackloggedGroup]group[%s] has been created", group.c_str());
728 0 : return HCCL_E_PARA;
729 : }
730 0 : groupParaLock.unlock();
731 :
732 0 : std::unique_lock<std::mutex> lock(hcomInfo.backloggedGroupLock);
733 0 : if (hcomInfo.backloggedGroup.find(group) != hcomInfo.backloggedGroup.end()) {
734 0 : HCCL_ERROR("[Store][BackloggedGroup]group[%s] is existed", group.c_str());
735 0 : return HCCL_E_PARA;
736 : } else {
737 0 : hcomInfo.backloggedGroup.insert({group, groupRanks});
738 : }
739 0 : HCCL_INFO("hcom store group[%s] success.", group.c_str());
740 0 : return HCCL_SUCCESS;
741 0 : }
742 :
743 0 : HcclResult HcomDestroyBackloggedGroup(const std::string& group)
744 : {
745 0 : HcomInfo& hcomInfo = HcomGetCtxHomInfo();
746 0 : std::unique_lock<std::mutex> lock(hcomInfo.backloggedGroupLock);
747 0 : if (hcomInfo.backloggedGroup.find(group) == hcomInfo.backloggedGroup.end()) {
748 0 : if (!hcomInfo.isHcomInit) {
749 0 : HCCL_WARNING(
750 : "[Destroy][BackloggedGroup]group[%s] is not existed, and hcom has not been inited yet", group.c_str());
751 0 : return HCCL_SUCCESS;
752 : } else {
753 0 : HCCL_ERROR("[Destroy][BackloggedGroup]group[%s] is not existed", group.c_str());
754 0 : return HCCL_E_PARA;
755 : }
756 : } else {
757 0 : hcomInfo.backloggedGroup.erase(group);
758 : }
759 0 : HCCL_INFO("hcom delete backlogged group[%s] success.", group.c_str());
760 0 : return HCCL_SUCCESS;
761 0 : }
762 :
763 0 : HcclResult HcomGetbackloggedByGroup(const char* group, std::vector<u32>& groupRanks, s32& groupSize)
764 : {
765 0 : CHK_RET(HcomCheckGroupName(group));
766 0 : std::string groupName = group;
767 :
768 0 : std::unique_lock<std::mutex> groupLock(g_backloggedGroupLock);
769 0 : auto it = g_backloggedGroup.find(groupName);
770 0 : if (it != g_backloggedGroup.end()) {
771 0 : groupRanks = it->second;
772 0 : groupSize = (it->second).size();
773 0 : HCCL_INFO("[device not set]get back logged group[%s], groupSize[%d]", groupName.c_str(), groupSize);
774 0 : return HCCL_SUCCESS;
775 : }
776 0 : groupLock.unlock();
777 0 : HcomInfo& hcomInfo = HcomGetCtxHomInfo();
778 0 : std::unique_lock<std::mutex> groupParaLock(hcomInfo.backloggedGroupLock);
779 0 : auto iter = hcomInfo.backloggedGroup.find(groupName);
780 0 : if (iter == hcomInfo.backloggedGroup.end()) {
781 0 : groupSize = 0;
782 0 : HCCL_DEBUG(
783 : "[Get][CommByGroup]errNo[0x%016llx] group[%s] is not exist", HCOM_ERROR_CODE(HCCL_E_NOT_FOUND), group);
784 0 : return HCCL_SUCCESS; // 不存在该服务器内相关dev的对应信息
785 : }
786 0 : groupRanks = iter->second;
787 0 : groupSize = (iter->second).size();
788 0 : HCCL_INFO("[device set]get back logged group[%s], groupSize[%d]", groupName.c_str(), groupSize);
789 0 : return HCCL_SUCCESS;
790 0 : }
791 :
792 0 : HcclResult HcomQueryGroupRef(const char* group, u32& groupRef)
793 : {
794 0 : HcomInfo& hcomInfo = HcomGetCtxHomInfo();
795 0 : if (group == nullptr) {
796 0 : return HCCL_SUCCESS; // 全局通信域不需要查询flag
797 : }
798 0 : std::string strGroup = group;
799 0 : std::unique_lock<std::mutex> groupParaLock(hcomInfo.groupParamsLock);
800 0 : auto iter = hcomInfo.hcomGroupMap.find(strGroup);
801 0 : if (iter == hcomInfo.hcomGroupMap.end()) {
802 0 : HCCL_WARNING(
803 : "[Get][CommByGroup]errNo[0x%016llx] group[%s] group is not exist", HCOM_ERROR_CODE(HCCL_E_AGAIN),
804 : strGroup.c_str());
805 0 : return HCCL_E_AGAIN; // 不存在该服务器内相关dev的对应信息
806 : }
807 0 : groupRef = iter->second.refCounter;
808 0 : return HCCL_SUCCESS;
809 0 : }
810 :
811 0 : HcclResult HcomGetWorldRankFromGroupRank(const char* group, u32 groupRank, u32* worldRank)
812 : {
813 0 : RPT_INPUT_ERR(
814 : worldRank == nullptr, "EI0003", std::vector<std::string>({"ccl_op", "value", "parameter", "expect"}),
815 : std::vector<std::string>({"HcomGetWorldRankFromGroupRank", "nullptr", "worldRank", "non-null pointer"}));
816 0 : CHK_PTR_NULL(worldRank);
817 0 : bool& isAutoTuneModeOpen = HcomGetCtxAutoTuneMode();
818 0 : if (isAutoTuneModeOpen) {
819 0 : *worldRank = 0;
820 0 : return HCCL_SUCCESS;
821 : }
822 0 : HcomInfo& hcomInfo = HcomGetCtxHomInfo();
823 :
824 0 : HcclResult ret = HcomCheckGroupName(group);
825 0 : RPT_INPUT_ERR(
826 : ret != HCCL_SUCCESS, "EI0003", std::vector<std::string>({"ccl_op", "value", "parameter", "expect"}),
827 : std::vector<std::string>(
828 : {"HcomGetWorldRankFromGroupRank",
829 : {group, strnlen(group, GROUP_NAME_MAX_LEN + 1)},
830 : "group",
831 : "a non-empty string of length 1 to " + std::to_string(GROUP_NAME_MAX_LEN)
832 : + ", containing only alphanumeric characters and underscores"}));
833 0 : CHK_PRT_RET(
834 : ret != HCCL_SUCCESS,
835 : HCCL_ERROR(
836 : "[%s][%s]errNo[0x%016llx] group name is invalid", LOG_KEYWORDS_TASK_EXEC.c_str(),
837 : LOG_KEYWORDS_INVALID_ARGUMENT.c_str(), HCOM_ERROR_CODE(ret)),
838 : ret);
839 : #if (!defined(HCCD)) && (!defined(CCL_KERNEL_AICPU))
840 0 : HCCLV2_FUNC_RUN(HcomGetWorldRankFromGroupRankV2(group, groupRank, worldRank));
841 : #endif
842 0 : if (groupRank >= hcomInfo.params.totalRanks) {
843 0 : HCCL_ERROR(
844 : "[Get][WorldRank]errNo[0x%016llx] groupRank[%u] is out of range[0-%u]", HCOM_ERROR_CODE(HCCL_E_PARA),
845 : groupRank, hcomInfo.params.totalRanks);
846 0 : return HCCL_E_PARA;
847 : }
848 0 : std::string strGroup = (group == nullptr) ? HCCL_WORLD_GROUP : group;
849 0 : CHK_RET(GetGroupRankInfo(strGroup.c_str(), RankInfoType::WORLD_RANK_ID_BY_GROUP, groupRank, worldRank));
850 0 : HCCL_INFO(
851 : "hcom get world rank success, group[%s], groupRank[%u], worldRank[%p]", strGroup.c_str(), groupRank, worldRank);
852 0 : return HCCL_SUCCESS;
853 0 : }
854 :
855 0 : HcclResult HcomGetGroupRankFromWorldRank(u32 worldRank, const char* group, u32* groupRank)
856 : {
857 0 : RPT_INPUT_ERR(
858 : groupRank == nullptr, "EI0003", std::vector<std::string>({"ccl_op", "value", "parameter", "expect"}),
859 : std::vector<std::string>({"HcomGetGroupRankFromWorldRank", "nullptr", "groupRank", "non-null pointer"}));
860 0 : CHK_PTR_NULL(groupRank);
861 0 : bool& isAutoTuneModeOpen = HcomGetCtxAutoTuneMode();
862 0 : if (isAutoTuneModeOpen) {
863 0 : *groupRank = 0;
864 0 : return HCCL_SUCCESS;
865 : }
866 0 : HcomInfo& hcomInfo = HcomGetCtxHomInfo();
867 :
868 0 : HcclResult ret = HcomCheckGroupName(group);
869 0 : RPT_INPUT_ERR(
870 : ret != HCCL_SUCCESS, "EI0003", std::vector<std::string>({"ccl_op", "value", "parameter", "expect"}),
871 : std::vector<std::string>(
872 : {"HcomGetGroupRankFromWorldRank",
873 : {group, strnlen(group, GROUP_NAME_MAX_LEN + 1)},
874 : "group",
875 : "a non-empty string of length 1 to " + std::to_string(GROUP_NAME_MAX_LEN)
876 : + ", containing only alphanumeric characters and underscores"}));
877 0 : CHK_PRT_RET(
878 : ret != HCCL_SUCCESS,
879 : HCCL_ERROR(
880 : "[%s][%s]errNo[0x%016llx] group name is invalid", LOG_KEYWORDS_TASK_EXEC.c_str(),
881 : LOG_KEYWORDS_INVALID_ARGUMENT.c_str(), HCOM_ERROR_CODE(ret)),
882 : ret);
883 : #if (!defined(HCCD)) && (!defined(CCL_KERNEL_AICPU))
884 0 : HCCLV2_FUNC_RUN(HcomGetGroupRankFromWorldRankV2(worldRank, group, groupRank));
885 : #endif
886 0 : std::string strGroup = (group == nullptr) ? HCCL_WORLD_GROUP : group;
887 0 : if (worldRank >= hcomInfo.params.totalRanks) {
888 0 : HCCL_ERROR(
889 : "[Get][GroupRank]errNo[0x%016llx] world[%u] rank is invalid", HCOM_ERROR_CODE(HCCL_E_PARA), worldRank);
890 0 : return HCCL_E_PARA;
891 : }
892 0 : CHK_RET(GetGroupRankInfo(strGroup.c_str(), RankInfoType::GROUP_RANK_ID_BY_WORLD, worldRank, groupRank));
893 0 : HCCL_INFO(
894 : "hcom get group rank success, group[%s], worldRank[%u], groupRank[%p]", strGroup.c_str(), worldRank, groupRank);
895 0 : return HCCL_SUCCESS;
896 0 : }
897 :
898 0 : bool HcomFindGroup(const char* group)
899 : {
900 0 : HcomInfo& hcomInfo = HcomGetCtxHomInfo();
901 : /* 已经存在的group不允许再次创建 */
902 0 : std::unique_lock<std::mutex> groupParaLock(hcomInfo.groupParamsLock);
903 0 : bool exists = !(hcomInfo.hcomGroupMap.find(group) == hcomInfo.hcomGroupMap.end());
904 0 : HCCL_INFO("[Find][Group] group[%s] is exist[%d]", group, exists);
905 0 : groupParaLock.unlock();
906 0 : return exists;
907 0 : }
908 :
909 0 : HcclResult GetWorldGroupRankInfo(RankInfoType rankType, u32 inPara, u32* outPara)
910 : {
911 0 : HcomInfo& hcomInfo = HcomGetCtxHomInfo();
912 0 : CHK_PTR_NULL(outPara);
913 0 : switch (rankType) {
914 0 : case RankInfoType::RANK_SIZE_IN_GROUP:
915 0 : *outPara = hcomInfo.params.totalRanks;
916 0 : break;
917 :
918 0 : case RankInfoType::RANK_ID_IN_GROUP:
919 0 : *outPara = hcomInfo.params.rank;
920 0 : break;
921 :
922 0 : case RankInfoType::WORLD_RANK_ID_BY_GROUP:
923 : case RankInfoType::GROUP_RANK_ID_BY_WORLD:
924 0 : *outPara = inPara;
925 0 : break;
926 0 : case RankInfoType::SERVER_NUM_IN_GROUP:
927 0 : *outPara = hcomInfo.rankTable.serverNum;
928 0 : break;
929 0 : default:
930 0 : HCCL_ERROR(
931 : "[Get][WorldGroupRankInfo]errNo[0x%016llx] invalid rankInfo type[%d]", HCOM_ERROR_CODE(HCCL_E_PARA),
932 : rankType);
933 0 : return HCCL_E_PARA;
934 : }
935 0 : return HCCL_SUCCESS;
936 : }
937 :
938 0 : HcclResult GetGroupRankInfo(const char* group, RankInfoType rankType, u32 inPara, u32* outPara)
939 : {
940 0 : CHK_PTR_NULL(outPara);
941 : // std::string strGroup = (group == nullptr) ? HCCL_WORLD_GROUP : group;
942 0 : if ((group == nullptr) || (strcmp(group, HCCL_WORLD_GROUP) == 0)) {
943 0 : CHK_RET(GetWorldGroupRankInfo(rankType, inPara, outPara));
944 0 : return HCCL_SUCCESS;
945 : }
946 0 : HcomInfo& hcomInfo = HcomGetCtxHomInfo();
947 :
948 0 : std::unique_lock<std::mutex> groupParaLock(hcomInfo.groupParamsLock);
949 0 : auto iter = hcomInfo.hcomGroupMap.find(group);
950 0 : if (iter == hcomInfo.hcomGroupMap.end()) {
951 0 : HCCL_ERROR(
952 : "[Get][GroupRankInfo]errNo[0x%016llx] group[%s] is not exist", HCOM_ERROR_CODE(HCCL_E_NOT_FOUND), group);
953 0 : return HCCL_E_NOT_FOUND; // 不存在该服务器内相关dev的对应信息
954 : }
955 : // group ranks判空
956 0 : CHK_PRT_RET(
957 : (iter->second).groupRanks.empty(),
958 : HCCL_ERROR(
959 : "[Get][GroupRankInfo]errNo[0x%016llx] group[%s]"
960 : "ranks is empty",
961 : HCOM_ERROR_CODE(HCCL_E_INTERNAL), group),
962 : HCCL_E_INTERNAL);
963 :
964 0 : switch (rankType) {
965 0 : case RankInfoType::RANK_SIZE_IN_GROUP:
966 0 : *outPara = (iter->second).totalRanks;
967 0 : return HCCL_SUCCESS;
968 :
969 0 : case RankInfoType::RANK_ID_IN_GROUP:
970 0 : *outPara = (iter->second).groupRank;
971 0 : return HCCL_SUCCESS;
972 :
973 0 : case RankInfoType::WORLD_RANK_ID_BY_GROUP:
974 0 : if (inPara >= (iter->second).totalRanks) {
975 0 : HCCL_ERROR(
976 : "[Get][GroupRankInfo]errNo[0x%016llx] group[%s] groupRank[%u] is invalid",
977 : HCOM_ERROR_CODE(HCCL_E_PARA), group, inPara);
978 0 : return HCCL_E_PARA;
979 : }
980 0 : *outPara = (iter->second).groupRanks[inPara];
981 0 : return HCCL_SUCCESS;
982 :
983 0 : case RankInfoType::GROUP_RANK_ID_BY_WORLD:
984 0 : for (u32 rank = 0; rank < (iter->second).totalRanks; rank++) {
985 0 : if (inPara == (iter->second).groupRanks[rank]) {
986 0 : *outPara = rank;
987 0 : return HCCL_SUCCESS;
988 : }
989 : }
990 0 : HCCL_ERROR(
991 : "[Get][GroupRankInfo]errNo[0x%016llx] invalid rankInfo type[%d]", HCOM_ERROR_CODE(HCCL_E_PARA),
992 : rankType);
993 0 : return HCCL_E_PARA;
994 0 : case RankInfoType::SERVER_NUM_IN_GROUP:
995 0 : *outPara = (iter->second).serverNum;
996 0 : return HCCL_SUCCESS;
997 0 : default:
998 0 : HCCL_ERROR(
999 : "[Get][GroupRankInfo]errNo[0x%016llx] invalid rankInfo type[%d]", HCOM_ERROR_CODE(HCCL_E_PARA),
1000 : rankType);
1001 0 : return HCCL_E_PARA;
1002 : }
1003 0 : }
1004 :
1005 0 : HcclResult HcomGetRankSize(const char* group, u32* rankSize)
1006 : {
1007 0 : RPT_INPUT_ERR(
1008 : rankSize == nullptr, "EI0003", std::vector<std::string>({"ccl_op", "value", "parameter", "expect"}),
1009 : std::vector<std::string>({"HcomGetRankSize", "nullptr", "rankSize", "non-null pointer"}));
1010 0 : CHK_PTR_NULL(rankSize);
1011 0 : bool& isAutoTuneModeOpen = HcomGetCtxAutoTuneMode();
1012 0 : if (isAutoTuneModeOpen) {
1013 0 : *rankSize = 1;
1014 0 : return HCCL_SUCCESS;
1015 : }
1016 :
1017 0 : HcclResult ret = HcomCheckGroupName(group);
1018 0 : RPT_INPUT_ERR(
1019 : ret != HCCL_SUCCESS, "EI0003", std::vector<std::string>({"ccl_op", "value", "parameter", "expect"}),
1020 : std::vector<std::string>(
1021 : {"HcomGetRankSize",
1022 : {group, strnlen(group, GROUP_NAME_MAX_LEN + 1)},
1023 : "group",
1024 : "a non-empty string of length 1 to " + std::to_string(GROUP_NAME_MAX_LEN)
1025 : + ", containing only alphanumeric characters and underscores"}));
1026 0 : CHK_PRT_RET(
1027 : ret != HCCL_SUCCESS,
1028 : HCCL_ERROR(
1029 : "[%s][%s]errNo[0x%016llx] group name is invalid", LOG_KEYWORDS_TASK_EXEC.c_str(),
1030 : LOG_KEYWORDS_INVALID_ARGUMENT.c_str(), HCOM_ERROR_CODE(ret)),
1031 : ret);
1032 :
1033 : #if (!defined(HCCD)) && (!defined(CCL_KERNEL_AICPU))
1034 0 : HCCLV2_FUNC_RUN(HcomGetRankSizeV2(group, rankSize));
1035 : #endif
1036 0 : std::shared_ptr<hccl::hcclComm> hcclComm;
1037 0 : if (group != nullptr && HcclGetCommHandle(group, hcclComm) == HCCL_SUCCESS) {
1038 0 : CHK_RET(hcclComm->GetRankSize(*rankSize));
1039 : } else {
1040 0 : std::string strGroup = (group == nullptr) ? HCCL_WORLD_GROUP : group;
1041 0 : ret = GetGroupRankInfo(strGroup.c_str(), RankInfoType::RANK_SIZE_IN_GROUP, 0, rankSize);
1042 0 : CHK_PRT_RET(
1043 : ret != HCCL_SUCCESS,
1044 : HCCL_ERROR(
1045 : "[Get][RankSize]errNo[0x%016llx] get group[%s] rank info error", HCOM_ERROR_CODE(ret),
1046 : strGroup.c_str()),
1047 : ret);
1048 0 : HCCL_INFO("hcom get rank size success, group[%s]", strGroup.c_str());
1049 0 : }
1050 0 : return HCCL_SUCCESS;
1051 0 : }
1052 :
1053 0 : HcclResult HcomDestroyOneDevice(HcomInfo& hcomInfo)
1054 : {
1055 0 : HcclUs startut = TIME_NOW();
1056 :
1057 : /* 接口交互信息日志 */
1058 0 : HCCL_RUN_INFO("Entry-HcomDestroy:void");
1059 :
1060 : // 模型运行结束后hcom destroy时，将CheckInfo信息清空
1061 0 : RankConsistentcyChecker::GetInstance().ClearCheckInfo();
1062 :
1063 : // group资源在word group资源销毁之前进行销毁
1064 0 : hcomInfo.params.commConnections.agentConnection = nullptr;
1065 0 : hcomInfo.params.commConnections.serverConnections.clear();
1066 0 : hcomInfo.hcomGroupMap.clear();
1067 0 : std::unique_lock<std::mutex> backloggedGroupLock(hcomInfo.backloggedGroupLock);
1068 0 : hcomInfo.backloggedGroup.clear();
1069 0 : backloggedGroupLock.unlock();
1070 :
1071 0 : hcomInfo.rankTable.nicNames.clear();
1072 0 : hcomInfo.rankTable.rankList.clear();
1073 0 : g_segmentIdxMap.clear();
1074 0 : g_segmentSizeMap.clear();
1075 0 : hcomInfo.params.profilingMode = HcomProfilingMode::PROFILING_CLOSE;
1076 0 : hcomInfo.params.profilingOption = "";
1077 0 : hcomInfo.isHcomInit = false;
1078 :
1079 0 : if (hcomInfo.params.deviceType != DevType::DEV_TYPE_NOSOC) {
1080 0 : ProfilingManagerPub::ClearStoragedProfilingInfo();
1081 : }
1082 :
1083 : /* 关键状态记录 */
1084 0 : HCCL_USER_CRITICAL_LOG(
1085 : "hcom destroy complete,take time [%lld]us, group[%s], rankNum[%u], rank[%u]", DURATION_US(TIME_NOW() - startut),
1086 : hcomInfo.params.identifier.c_str(), hcomInfo.rankTable.rankNum, hcomInfo.params.rank);
1087 :
1088 0 : return HCCL_SUCCESS;
1089 0 : }
1090 :
1091 0 : HcclResult HcomDestroy(void)
1092 : {
1093 : #if (!defined(HCCD)) && (!defined(CCL_KERNEL_AICPU))
1094 0 : HCCLV2_FUNC_RUN([&]() -> HcclResult {
1095 : std::unique_lock<std::mutex> lock(g_destroyDeviceLock);
1096 : CHK_RET(HcomDestroyV2());
1097 : for (u32 i = 0; i <= MAX_MODULE_DEVICE_NUM; i++) {
1098 : HcomInfo& hcomInfo = HcomGetCtxHomInfoById(i);
1099 : hcomInfo.pComm = nullptr;
1100 : hcomInfo.hcomGroupMap.clear();
1101 : }
1102 : return HCCL_SUCCESS;
1103 : }());
1104 : #endif
1105 0 : std::unique_lock<std::mutex> lock(g_destroyDeviceLock);
1106 0 : for (u32 i = 0; i <= MAX_MODULE_DEVICE_NUM; i++) {
1107 0 : HcomInfo& hcomInfo = HcomGetCtxHomInfoById(i);
1108 :
1109 0 : if (!hcomInfo.isHcomInit) {
1110 0 : if (hcomInfo.pComm != nullptr) {
1111 0 : hcomInfo.pComm = nullptr;
1112 : }
1113 0 : HCCL_INFO("[Destroy][Result]hcomInfo[%u].isHcomInit is false.", i);
1114 :
1115 : /* 接口交互信息日志 */
1116 0 : HCCL_INFO("Entry-HcomDestroy:void skip");
1117 0 : if (g_hcomDestroyCallback != nullptr) {
1118 0 : (void)g_hcomDestroyCallback(hcomInfo);
1119 : }
1120 0 : continue;
1121 : } else {
1122 0 : if (hcomInfo.pComm != nullptr) {
1123 0 : HcomUnSetGroupTopoInfo(hcomInfo.pComm->GetIdentifier().c_str());
1124 : }
1125 : }
1126 :
1127 0 : if (hcomInfo.pComm == nullptr
1128 0 : && ((g_hcomCallBackGroupIsInit != nullptr) && (!(g_hcomCallBackGroupIsInit(hcomInfo))))) {
1129 0 : HCCL_INFO("[Destroy][Result]hcomInfo[%u].pComm or pCommBase is nullptr.", i);
1130 0 : continue;
1131 : }
1132 :
1133 0 : if (hcomInfo.params.logicDevId != HOST_DEVICE_ID) {
1134 0 : u32 logicId = hcomInfo.params.logicDevId;
1135 0 : if (hcomInfo.rankTable.version.compare(HETEROG_CLUSTER_VERSION) == 0) {
1136 0 : CHK_RET(hrtGetDeviceIndexByPhyId(hcomInfo.params.logicDevId, logicId));
1137 : }
1138 0 : s32 deviceId = 0;
1139 0 : if (hrtGetDevice(&deviceId) != HCCL_SUCCESS) {
1140 0 : CHK_RET(hrtSetDevice(logicId));
1141 0 : HCCL_INFO("[HcomDestroy][SetDeviceId]logicDevId[%u]", logicId);
1142 : }
1143 : }
1144 :
1145 0 : HCCL_INFO("[Destroy][Result]hcomInfo[%u].pComm destroy.", i);
1146 0 : HcclResult ret = HcomDestroyOneDevice(hcomInfo);
1147 0 : if (ret == HCCL_SUCCESS) {
1148 0 : HCCL_INFO("[Destroy][Result]hcomInfo[%u].pComm HcomDestroyOneDevice success.", i);
1149 : } else {
1150 0 : HCCL_INFO("[Destroy][Result]hcomInfo[%u].pComm HcomDestroyOneDevice fail.", i);
1151 0 : return ret;
1152 : }
1153 :
1154 0 : if (g_hcomDestroyCallback != nullptr) {
1155 0 : (void)g_hcomDestroyCallback(hcomInfo);
1156 : }
1157 :
1158 0 : hcomInfo.pComm = nullptr;
1159 0 : hcomInfo.hcclCommTopoInfoDetectServer.clear();
1160 0 : hcomInfo.hcclCommTopoInfoDetectAgent.clear();
1161 : }
1162 0 : return HCCL_SUCCESS;
1163 0 : }
1164 :
1165 47 : void HcomGroupCallbackFuncInstall(
1166 : HcclResult (*p1)(const std::string&, const std::vector<u32>&), bool (*p2)(HcomInfo&),
1167 : HcclResult (*p3)(const std::string&), HcclResult (*p4)(HcomInfo&))
1168 : {
1169 47 : g_hcomCreateGroupCallback = p1;
1170 47 : g_hcomCallBackGroupIsInit = p2;
1171 47 : g_hcomDestroyGroupCallback = p3;
1172 47 : g_hcomDestroyCallback = p4;
1173 47 : }
1174 :
1175 0 : HcclResult HcomSetGradFusionByIndex(const char* group, u32 segmentNum, const u32* IdxList)
1176 : {
1177 0 : bool& isAutoTuneModeOpen = HcomGetCtxAutoTuneMode();
1178 0 : if (isAutoTuneModeOpen) {
1179 0 : return HCCL_SUCCESS;
1180 : }
1181 :
1182 0 : RPT_INPUT_ERR(
1183 : IdxList == nullptr, "EI0003", std::vector<std::string>({"ccl_op", "value", "parameter", "expect"}),
1184 : std::vector<std::string>({"HcomSetGradFusionByIndex", "nullptr", "IdxList", "non-null pointer"}));
1185 0 : CHK_PTR_NULL(IdxList);
1186 0 : bool bRet = segmentNum == 0;
1187 0 : RPT_INPUT_ERR(
1188 : bRet, "EI0003", std::vector<std::string>({"ccl_op", "value", "parameter", "expect"}),
1189 : std::vector<std::string>(
1190 : {"HcomSetGradFusionByIndex", std::to_string(0), "segmentNum",
1191 : "must be a positive integer (greater than 0)"}));
1192 0 : CHK_PRT_RET(
1193 : bRet,
1194 : HCCL_ERROR(
1195 : "[%s][%s]errNo[0x%016llx] set split IdxList length is zero", LOG_KEYWORDS_TASK_EXEC.c_str(),
1196 : LOG_KEYWORDS_INVALID_ARGUMENT.c_str(), HCOM_ERROR_CODE(HCCL_E_PARA)),
1197 : HCCL_E_PARA);
1198 0 : std::string strGroup = (group == nullptr) ? HCCL_WORLD_GROUP : group;
1199 0 : string idxList;
1200 0 : for (u32 i = 0; i < segmentNum; i++) {
1201 0 : if (i < segmentNum - 1) {
1202 0 : idxList += to_string(IdxList[i]) + ',';
1203 0 : } else if (i == segmentNum - 1) {
1204 0 : idxList += to_string(IdxList[i]);
1205 : }
1206 : }
1207 : /* 接口交互信息日志 */
1208 0 : HCCL_RUN_INFO(
1209 : "Entry-HcomSetGradFusionByIndex:group[%s], segmentNum[%u], IdxList[%s]", strGroup.c_str(), segmentNum,
1210 : idxList.c_str());
1211 :
1212 0 : CHK_RET(HcomCheckGroupName(strGroup.c_str()));
1213 :
1214 0 : std::vector<u32> tempList;
1215 :
1216 0 : for (u32 segidx = 0; segidx < segmentNum; segidx++) {
1217 0 : tempList.push_back(IdxList[segidx]);
1218 : }
1219 :
1220 0 : for (u32 i = 0; i < tempList.size() - 1; i++) {
1221 0 : if (tempList[i] >= tempList[i + 1]) {
1222 0 : HCCL_ERROR(
1223 : "[Set][GradFusionByIndex]errNo[0x%016llx] index list is not ascending", HCOM_ERROR_CODE(HCCL_E_PARA));
1224 0 : return HCCL_E_PARA;
1225 : }
1226 : }
1227 0 : std::unique_lock<std::mutex> segmentIdxMapLock(g_segmentIdxMapLock);
1228 0 : g_segmentIdxMap.insert(std::pair<std::string, std::vector<u32>>(strGroup, tempList));
1229 0 : segmentIdxMapLock.unlock();
1230 0 : return HCCL_SUCCESS;
1231 0 : }
1232 :
1233 0 : HcclResult HcomSetGradFusionBySize(const char* group, u32 segmentNum, const float* sizeList)
1234 : {
1235 0 : bool& isAutoTuneModeOpen = HcomGetCtxAutoTuneMode();
1236 0 : if (isAutoTuneModeOpen) {
1237 0 : return HCCL_SUCCESS;
1238 : }
1239 :
1240 0 : RPT_INPUT_ERR(
1241 : sizeList == nullptr, "EI0003", std::vector<std::string>({"ccl_op", "value", "parameter", "expect"}),
1242 : std::vector<std::string>({"HcomSetGradFusionBySize", "nullptr", "sizeList", "non-null pointer"}));
1243 0 : CHK_PTR_NULL(sizeList);
1244 0 : bool bRet = segmentNum == 0;
1245 0 : RPT_INPUT_ERR(
1246 : bRet, "EI0003", std::vector<std::string>({"ccl_op", "value", "parameter", "expect"}),
1247 : std::vector<std::string>(
1248 : {"HcomSetGradFusionBySize", std::to_string(0), "segmentNum",
1249 : "must be a positive integer (greater than 0)"}));
1250 0 : CHK_PRT_RET(
1251 : bRet,
1252 : HCCL_ERROR(
1253 : "[%s][%s]errNo[0x%016llx] set split sizeList length is zero", LOG_KEYWORDS_TASK_EXEC.c_str(),
1254 : LOG_KEYWORDS_INVALID_ARGUMENT.c_str(), HCOM_ERROR_CODE(HCCL_E_PARA)),
1255 : HCCL_E_PARA);
1256 0 : std::string strGroup = (group == nullptr) ? HCCL_WORLD_GROUP : group;
1257 0 : string strSizeList;
1258 0 : for (u32 i = 0; i < segmentNum; i++) {
1259 0 : if (i < segmentNum - 1) {
1260 0 : strSizeList += to_string(sizeList[i]) + ',';
1261 0 : } else if (i == segmentNum - 1) {
1262 0 : strSizeList += to_string(sizeList[i]);
1263 : }
1264 : }
1265 : /* 接口交互信息日志 */
1266 0 : HCCL_RUN_INFO(
1267 : "Entry-HcomSetGradFusionBySize:group[%s], segmentNum[%u], sizeList[%s]", strGroup.c_str(), segmentNum,
1268 : strSizeList.c_str());
1269 :
1270 0 : CHK_RET(HcomCheckGroupName(strGroup.c_str()));
1271 0 : std::vector<float> tempList;
1272 0 : float sizeTotal = 0;
1273 :
1274 0 : for (u32 sizeIdx = 0; sizeIdx < segmentNum; sizeIdx++) {
1275 0 : bRet = sizeList[sizeIdx] < 0;
1276 0 : CHK_PRT_RET(
1277 : bRet,
1278 : HCCL_ERROR(
1279 : "[Set][GradFusionBySize]errNo[0x%016llx] sizeList[%u] less than zero", HCOM_ERROR_CODE(HCCL_E_PARA),
1280 : sizeIdx),
1281 : HCCL_E_PARA);
1282 0 : tempList.push_back(sizeList[sizeIdx]);
1283 0 : sizeTotal += sizeList[sizeIdx];
1284 : }
1285 :
1286 0 : if (std::fabs(sizeTotal - 100) > 1e-6) { // 判断用户设置总百分比是否为100%
1287 0 : HCCL_ERROR("[Set][GradFusionBySize]errNo[0x%016llx] size list sum is not 100%%", HCOM_ERROR_CODE(HCCL_E_PARA));
1288 0 : return HCCL_E_PARA;
1289 : } else {
1290 0 : std::unique_lock<std::mutex> segmentSizeMapLock(g_segmentSizeMapLock);
1291 0 : g_segmentSizeMap.insert(std::pair<std::string, std::vector<float>>(strGroup, tempList));
1292 0 : segmentSizeMapLock.unlock();
1293 0 : return HCCL_SUCCESS;
1294 0 : }
1295 0 : }
1296 :
1297 0 : HcclResult HcomGenerateCommId(hccl::HcclCommParams& params)
1298 : {
1299 0 : s32 sRet = memset_s(params.id.internal, HCCL_ROOT_INFO_BYTES, 0, sizeof(params.id.internal));
1300 0 : CHK_PRT_RET(sRet != EOK, HCCL_ERROR("[GenerateCommId]memory set error. return[%d].", sRet), HCCL_E_PARA);
1301 :
1302 : HcclRootInfo uniqueId;
1303 0 : std::string group;
1304 0 : CHK_RET(hcclComm::GetUniqueId(&uniqueId));
1305 :
1306 0 : if (!params.isHeterogComm) {
1307 0 : group = "hccl_world_group";
1308 : } else {
1309 0 : group = "hccl_heterog_group";
1310 : }
1311 :
1312 0 : sRet = snprintf_s(
1313 0 : params.id.internal, HCCL_ROOT_INFO_BYTES, HCCL_ROOT_INFO_BYTES - 1, "%s%s%s", uniqueId.internal, "-",
1314 : group.c_str());
1315 0 : CHK_PRT_RET(
1316 : sRet == -1,
1317 : HCCL_ERROR("[GenerateCommId]errNo[0x%016llx] sal snprintf_s error", HCCL_ERROR_CODE(HCCL_E_INTERNAL)),
1318 : HCCL_E_INTERNAL);
1319 0 : HCCL_INFO("params.id.internal [%s]", params.id.internal);
1320 0 : return HCCL_SUCCESS;
1321 0 : }
1322 :
1323 0 : HcclResult InitHcomMiscInfo(hccl::HcclCommParams& params, const char* rankTable)
1324 : {
1325 0 : CHK_PTR_NULL(rankTable);
1326 :
1327 0 : RankConsistentcyChecker::GetInstance().SetCheckCannVersionSwitch(true); // 打开CANN软件版本校验开关
1328 :
1329 : // 记录版本信息
1330 0 : std::string curVersion = GetExternalInputCannVersion();
1331 0 : CHK_RET(RankConsistentcyChecker::GetInstance().RecordVerInfo(curVersion));
1332 0 : HcomInfo& hcomInfo = HcomGetCtxHomInfo();
1333 : // 计算rankTable的crc值并保存
1334 0 : HcclResult ret = HcomCalcCRC(params, rankTable);
1335 0 : CHK_PRT_RET(
1336 : ret != HCCL_SUCCESS,
1337 : HCCL_ERROR("[Init][OtherInfo]errNo[0x%016llx] calc ranktable crc error", HCCL_ERROR_CODE(HCCL_E_INTERNAL)),
1338 : HCCL_E_INTERNAL);
1339 : // 生成通信域标识符
1340 0 : ret = HcomGenerateCommId(hcomInfo.params);
1341 0 : CHK_PRT_RET(
1342 : ret != HCCL_SUCCESS,
1343 : HCCL_ERROR("[Init][OtherInfo]errNo[0x%016llx] generate CommId error", HCCL_ERROR_CODE(HCCL_E_INTERNAL)),
1344 : HCCL_E_INTERNAL);
1345 0 : return HCCL_SUCCESS;
1346 0 : }
1347 :
1348 0 : bool HcomCheckrtMemcpyAddrAsync(const std::string& group)
1349 : {
1350 0 : float counterVaule = 1.0f;
1351 :
1352 : // 偏移拷贝的二级指针
1353 0 : void* deviceMemSrcLevel2 = nullptr;
1354 0 : void* deviceMemDstLevel2 = nullptr;
1355 : // 偏移拷贝的一级指针
1356 0 : void* deviceMemSrc = nullptr;
1357 0 : void* deviceMemDst = nullptr;
1358 :
1359 0 : auto deleter = [&deviceMemSrcLevel2, &deviceMemDstLevel2, &deviceMemSrc, &deviceMemDst](void* dst) {
1360 0 : if (dst != nullptr) {
1361 0 : CHK_PRT(hrtFree(dst));
1362 0 : if (dst == deviceMemSrcLevel2) {
1363 0 : deviceMemSrcLevel2 = nullptr;
1364 0 : } else if (dst == deviceMemDstLevel2) {
1365 0 : deviceMemDstLevel2 = nullptr;
1366 0 : } else if (dst == deviceMemSrc) {
1367 0 : deviceMemSrc = nullptr;
1368 0 : } else if (dst == deviceMemDst) {
1369 0 : deviceMemDst = nullptr;
1370 : }
1371 : }
1372 0 : };
1373 :
1374 0 : CHK_RET(hrtMalloc(&deviceMemSrcLevel2, sizeof(void*)));
1375 0 : unique_ptr<void, decltype(deleter)> deviceMemSrcLevel2Unique(deviceMemSrcLevel2, deleter);
1376 0 : CHK_RET(hrtMalloc(&deviceMemDstLevel2, sizeof(void*)));
1377 0 : unique_ptr<void, decltype(deleter)> deviceMemDstLevel2Unique(deviceMemDstLevel2, deleter);
1378 0 : CHK_RET(hrtMalloc(&deviceMemSrc, sizeof(float)));
1379 0 : unique_ptr<void, decltype(deleter)> deviceMemSrcUnique(deviceMemSrc, deleter);
1380 0 : CHK_RET(hrtMalloc(&deviceMemDst, sizeof(float)));
1381 0 : unique_ptr<void, decltype(deleter)> deviceMemDstUnique(deviceMemDst, deleter);
1382 :
1383 0 : CHK_RET(hrtMemSyncCopy(
1384 : deviceMemDst, sizeof(float), &counterVaule, sizeof(float),
1385 : HcclRtMemcpyKind::HCCL_RT_MEMCPY_KIND_HOST_TO_DEVICE));
1386 0 : CHK_RET(hrtMemSyncCopy(
1387 : deviceMemSrc, sizeof(float), &counterVaule, sizeof(float),
1388 : HcclRtMemcpyKind::HCCL_RT_MEMCPY_KIND_HOST_TO_DEVICE));
1389 :
1390 0 : CHK_RET(hrtMemSyncCopy(
1391 : deviceMemDstLevel2, sizeof(void*), &deviceMemDst, sizeof(void*),
1392 : HcclRtMemcpyKind::HCCL_RT_MEMCPY_KIND_HOST_TO_DEVICE));
1393 0 : CHK_RET(hrtMemSyncCopy(
1394 : deviceMemSrcLevel2, sizeof(void*), &deviceMemSrc, sizeof(void*),
1395 : HcclRtMemcpyKind::HCCL_RT_MEMCPY_KIND_HOST_TO_DEVICE));
1396 :
1397 0 : u64 destMax = sizeof(s32);
1398 0 : u64 offset = 0;
1399 :
1400 0 : Stream stream(StreamType::STREAM_TYPE_ONLINE);
1401 0 : bool notSupportSecAddrCopyWithOffset = false;
1402 :
1403 : HcclResult ret
1404 0 : = hrtMemcpyAddrAsync(deviceMemDstLevel2, destMax, offset, deviceMemSrcLevel2, destMax, offset, stream.ptr());
1405 0 : if (ret == HCCL_E_NOT_SUPPORT) {
1406 0 : notSupportSecAddrCopyWithOffset = true;
1407 : } else {
1408 0 : CHK_RET(hcclStreamSynchronize(stream.ptr(), CommConfiger::GetInstance().GetCommConfigExecTimeOut(group)));
1409 : }
1410 :
1411 0 : g_notSupportSecAddrCopyWithOffset = notSupportSecAddrCopyWithOffset;
1412 :
1413 0 : return notSupportSecAddrCopyWithOffset;
1414 0 : }
1415 :
1416 1 : bool HcomGetSecAddrCopyFlag(const char* socVersion)
1417 : {
1418 1 : HCCL_INFO("[Hcom][HcomGetSecAddrCopyFlag] SecAddrCopyWithOffset flag is %d", g_notSupportSecAddrCopyWithOffset);
1419 : DevType devType;
1420 1 : std::string socVersionStr(socVersion);
1421 1 : CHK_RET(hrtGetDeviceTypeBySocVersion(socVersionStr, devType));
1422 :
1423 1 : return !g_notSupportSecAddrCopyWithOffset
1424 1 : && (devType == DevType::DEV_TYPE_910B || devType == DevType::DEV_TYPE_910);
1425 1 : }
1426 :
1427 0 : HcclResult HcomNormalInit(const char* rankTableM, const char* identify)
1428 : {
1429 0 : HcclResult ret = HCCL_SUCCESS;
1430 0 : bool& isAutoTuneModeOpen = HcomGetCtxAutoTuneMode();
1431 0 : isAutoTuneModeOpen = false;
1432 0 : HcomInfo& hcomInfo = HcomGetCtxHomInfo();
1433 :
1434 : /*--------------入参合法性检测---------------------*/
1435 0 : CHK_PTR_NULL(rankTableM);
1436 0 : CHK_PTR_NULL(identify);
1437 :
1438 : /* 防止重复调用初始化 */
1439 0 : CHK_PRT_RET(
1440 : (hcomInfo.pComm != nullptr),
1441 : HCCL_ERROR(
1442 : "[Init][Result]errNo[0x%016llx] identify[%s], "
1443 : "multiple initialization is not supported",
1444 : HCOM_ERROR_CODE(HCCL_E_UNAVAIL), identify),
1445 : HCCL_E_UNAVAIL);
1446 :
1447 : /* --------------初始化------------------------- */
1448 0 : bool errorFlag = false;
1449 0 : s32 logicDevId = 0;
1450 0 : hcomInfo.params.commWorkMode = WorkMode::HCCL_MODE_NORMAL;
1451 : do {
1452 0 : ret = InitHcomMiscInfo(hcomInfo.params, rankTableM);
1453 0 : CHK_PRT_BREAK(
1454 : ret != HCCL_SUCCESS, HCCL_ERROR("[Init][Result]errNo[0x%016llx] init other Info.", HCOM_ERROR_CODE(ret)),
1455 : errorFlag = true);
1456 :
1457 : DevType deviceType;
1458 0 : CHK_PRT_BREAK(hrtGetDevice(&logicDevId) != HCCL_SUCCESS, , errorFlag = true);
1459 0 : CHK_RET(hrtGetDeviceType(deviceType));
1460 : // 为适配12包，做此修改
1461 0 : g_notSupportSecAddrCopyWithOffset = HcomCheckrtMemcpyAddrAsync(identify);
1462 :
1463 0 : ret = CfgGetClusterInfo(
1464 0 : rankTableM, identify, hcomInfo.params, hcomInfo.rankTable, GetExternalInputInterSuperPodRetryEnable(),
1465 : deviceType);
1466 0 : CHK_PRT_BREAK(
1467 : ret != HCCL_SUCCESS,
1468 : HCCL_ERROR(
1469 : "[Init][Result]errNo[0x%016llx] cfg get ranktable[%p] info "
1470 : "error: identify[%s]",
1471 : HCOM_ERROR_CODE(ret), rankTableM, identify),
1472 : errorFlag = true);
1473 :
1474 0 : if (hcomInfo.rankTable.serverNum != SINGLE_SERVER_NUM
1475 0 : && (deviceType == DevType::DEV_TYPE_310P3 || deviceType == DevType::DEV_TYPE_310P1)) {
1476 0 : CHK_RET(InitExternalInputHeterog());
1477 : }
1478 :
1479 0 : hcomInfo.pComm.reset(new (std::nothrow) hccl::hcclComm(0, 0, HCCL_WORLD_GROUP));
1480 :
1481 0 : CHK_PRT_RET(
1482 : hcomInfo.pComm == nullptr,
1483 : HCCL_ERROR("[Init][Result]hcomInfo.pComm is null, "
1484 : "create failed"),
1485 : HCCL_E_PTR);
1486 0 : CommConfig commConfig(identify);
1487 0 : ret = hcomInfo.pComm->init(hcomInfo.params, commConfig, hcomInfo.rankTable);
1488 0 : CHK_PRT_BREAK(
1489 : ret != HCCL_SUCCESS, HCCL_ERROR("[Init][Result]errNo[0x%016llx] hcclComm init error", HCOM_ERROR_CODE(ret)),
1490 : errorFlag = true);
1491 :
1492 0 : ret = ShowRanktableConfigInfo(hcomInfo.cloudFlag, hcomInfo.params, hcomInfo.rankTable);
1493 0 : CHK_PRT_BREAK(
1494 : ret != HCCL_SUCCESS,
1495 : HCCL_ERROR("[Init][Result]errNo[0x%016llx] put ranktable info error", HCOM_ERROR_CODE(ret)),
1496 : errorFlag = true);
1497 0 : ret = InitWorkflowMode(HcclWorkflowMode::HCCL_WORKFLOW_MODE_OPS_KERNEL_INFO_LIB);
1498 0 : CHK_PRT_BREAK(
1499 : ret != HCCL_SUCCESS,
1500 : HCCL_ERROR("[Init][Result]errNo[0x%016llx] init work flow mode error", HCCL_ERROR_CODE(ret)),
1501 : errorFlag = true);
1502 :
1503 0 : ret = HcomFlushBackloggedGroups();
1504 0 : CHK_PRT_BREAK(
1505 : ret != HCCL_SUCCESS,
1506 : HCCL_ERROR("[Init][Result]errNo[0x%016llx] create backlogged group failed", HCOM_ERROR_CODE(ret)),
1507 : errorFlag = true);
1508 :
1509 0 : ret = HcomSetGroupTopoInfo(hcomInfo.pComm->GetIdentifier().c_str(), hcomInfo.rankTable.rankNum);
1510 0 : CHK_PRT_BREAK(
1511 : ret != HCCL_SUCCESS,
1512 : HCCL_ERROR(
1513 : "[Init][Result]errNo[0x%016llx] SetGroupTopoInfo error, "
1514 : "group[%s]",
1515 : HCOM_ERROR_CODE(ret), (hcomInfo.pComm->GetIdentifier().c_str())),
1516 : errorFlag = true);
1517 0 : } while (0);
1518 :
1519 0 : if (errorFlag) {
1520 0 : HCCL_ERROR(
1521 : "[Init][Result]hcom init failed, rankNum[%u], rank[%u], server[%s], device[%d], return[0x%016llx]",
1522 : hcomInfo.rankTable.rankNum, hcomInfo.params.rank, hcomInfo.params.serverId.c_str(), logicDevId,
1523 : HCOM_ERROR_CODE(ret));
1524 0 : (void)HcomDestroy();
1525 0 : return ret;
1526 : }
1527 0 : return HCCL_SUCCESS;
1528 : }
1529 :
1530 0 : HcclResult HcomCheckInitClusterInfo(const char* rankTableM, const char* identify)
1531 : {
1532 0 : HcclResult ret = HCCL_SUCCESS;
1533 : // rankTable合法性检测
1534 0 : u32 rankTableSize = 0;
1535 0 : ret = HcomCheckRankTable(rankTableM, rankTableSize);
1536 0 : CHK_PRT_RET(
1537 : ret != HCCL_SUCCESS,
1538 : HCCL_ERROR(
1539 : "[%s][%s]errNo[0x%016llx] input rankTable error", LOG_KEYWORDS_INIT_GROUP.c_str(),
1540 : LOG_KEYWORDS_INVALID_ARGUMENT.c_str(), HCOM_ERROR_CODE(ret)),
1541 : ret);
1542 :
1543 : // identify合法性检测
1544 0 : ret = HcomCheckIdentify(identify);
1545 0 : RPT_INPUT_ERR(
1546 : ret != HCCL_SUCCESS, "EI0003", std::vector<std::string>({"ccl_op", "value", "parameter", "expect"}),
1547 : std::vector<std::string>(
1548 : {"HcomInit", {identify, strnlen(identify, IDENTIFY_MAX_LEN + 1)}, "identify", "a valid node identifier"}));
1549 0 : CHK_PRT_RET(
1550 : ret != HCCL_SUCCESS,
1551 : HCCL_ERROR(
1552 : "[%s][%s]errNo[0x%016llx] identify parameter error", LOG_KEYWORDS_INIT_GROUP.c_str(),
1553 : LOG_KEYWORDS_INVALID_ARGUMENT.c_str(), HCOM_ERROR_CODE(ret)),
1554 : ret);
1555 0 : return ret;
1556 0 : }
1557 :
1558 0 : HcclResult HcomInitByFile(const char* rankTablePath, const char* identify)
1559 : {
1560 0 : HcomInfo& hcomInfo = HcomGetCtxHomInfo();
1561 :
1562 0 : CHK_PTR_NULL(rankTablePath);
1563 0 : CHK_PTR_NULL(identify);
1564 : #if (!defined(HCCD)) && (!defined(CCL_KERNEL_AICPU))
1565 0 : HCCLV2_FUNC_RUN([&]() -> HcclResult {
1566 : CHK_RET(HcomInitByFileV2(rankTablePath, identify));
1567 : u32 rankNum = 0;
1568 : CHK_RET(HcomGetRankSize(HCCL_WORLD_GROUP, &rankNum));
1569 : s32 myRank = std::atoi(identify);
1570 : Hccl::RankId rank = static_cast<Hccl::RankId>(myRank);
1571 : void* commV2 = nullptr;
1572 : CHK_RET(HcomGetCommV2(&commV2));
1573 : CHK_RET(HcomInitCollComm(rank, &commV2, hcomInfo.pComm));
1574 : CHK_RET(HcomSetGroupTopoInfo(HCCL_WORLD_GROUP, rankNum));
1575 : return HCCL_SUCCESS;
1576 : }());
1577 : #endif
1578 :
1579 0 : HcclUs startut = TIME_NOW();
1580 0 : HcclResult ret = HCCL_SUCCESS;
1581 :
1582 : // 读取rankTable文件到内存
1583 0 : std::string rankTableM;
1584 0 : std::string realFilePath;
1585 0 : ret = HcomLoadRanktableFile(rankTablePath, rankTableM, realFilePath);
1586 0 : CHK_PRT_RET(
1587 : ret != HCCL_SUCCESS,
1588 : HCCL_ERROR(
1589 : "[%s][%s]errNo[0x%016llx] rankTablePath[%s] identify[%s] load rankTable error.",
1590 : LOG_KEYWORDS_INIT_GROUP.c_str(), LOG_KEYWORDS_RANKTABLE_CONFIG.c_str(), HCCL_ERROR_CODE(HCCL_E_INTERNAL),
1591 : rankTablePath, identify),
1592 : HCCL_E_INTERNAL);
1593 0 : CHK_RET(HcomCheckInitClusterInfo(rankTableM.c_str(), identify));
1594 0 : HCCL_RUN_INFO("Entry-HcomInitByFile:rankTablePath[%s], identify[%s]", realFilePath.c_str(), identify);
1595 :
1596 0 : CHK_RET(InitExternalInput());
1597 0 : CHK_RET(InitEnvConfig());
1598 :
1599 : // 调用初始化接口
1600 0 : ret = HcomNormalInit(rankTableM.c_str(), identify);
1601 0 : CHK_PRT_RET(
1602 : ret != HCCL_SUCCESS,
1603 : HCCL_ERROR(
1604 : "[HcomInitByFile]errNo[0x%016llx] rankTablePath[%s] identify[%s] "
1605 : "hcom init failed.",
1606 : HCCL_ERROR_CODE(ret), realFilePath.c_str(), identify),
1607 : ret);
1608 0 : hcomInfo.isHcomInit = true;
1609 : /* 关键状态记录 */
1610 0 : HCCL_RUN_INFO(
1611 : "[HCCL_TRACE]hcom init by file success,take time [%lld]us, rankTablePath[%s], rankNum[%u], rank[%u],"
1612 : "server[%s], device[%d]",
1613 : DURATION_US(TIME_NOW() - startut), realFilePath.c_str(), hcomInfo.rankTable.rankNum, hcomInfo.params.rank,
1614 : hcomInfo.params.serverId.c_str(), hcomInfo.params.logicDevId);
1615 0 : return HCCL_SUCCESS;
1616 0 : }
1617 :
1618 3 : DevType HcomGetDeviceType()
1619 : {
1620 : DevType devType;
1621 3 : hrtGetDeviceType(devType);
1622 3 : if (devType == DevType::DEV_TYPE_950 || devType == DevType::DEV_TYPE_960) {
1623 0 : HcomGetDevTypeV2(devType);
1624 0 : HCCL_INFO("LaunchHcomKernel: devType is DEV_TYPE_950 or DEV_TYPE_960");
1625 0 : return MakeEnumToDevType(static_cast<int>(devType));
1626 : }
1627 :
1628 3 : HcomInfo& hcomInfo = HcomGetCtxHomInfo();
1629 3 : return hcomInfo.params.deviceType;
1630 : }
1631 :
1632 1 : HcclResult HcomCreateCommCCLbuffer(const char* group)
1633 : {
1634 1 : RPT_INPUT_ERR(
1635 : group == nullptr, "EI0003", std::vector<std::string>({"ccl_op", "value", "parameter", "expect"}),
1636 : std::vector<std::string>({"HcomGetDevType", "nullptr", "group", "non-null pointer"}));
1637 1 : CHK_PTR_NULL(group);
1638 :
1639 1 : HcclResult ret = HcomCheckGroupName(group);
1640 1 : RPT_INPUT_ERR(
1641 : ret != HCCL_SUCCESS, "EI0003", std::vector<std::string>({"ccl_op", "value", "parameter", "expect"}),
1642 : std::vector<std::string>(
1643 : {"HcomGetDevType",
1644 : {group, strnlen(group, GROUP_NAME_MAX_LEN + 1)},
1645 : "group",
1646 : "a non-empty string of length 1 to " + std::to_string(GROUP_NAME_MAX_LEN)
1647 : + ", containing only alphanumeric characters and underscores"}));
1648 1 : CHK_PRT_RET(
1649 : ret != HCCL_SUCCESS,
1650 : HCCL_ERROR("[Get][HcomGetDevType]errNo[0x%016llx] group name is invalid", HCOM_ERROR_CODE(ret)), ret);
1651 :
1652 1 : DevType devType = HcomGetDeviceType();
1653 1 : if (devType == DevType::DEV_TYPE_950 || devType == DevType::DEV_TYPE_960) {
1654 0 : HCCL_INFO("HcomCreateCommCclBufV2 start.");
1655 : #if (!defined(HCCD)) && (!defined(CCL_KERNEL_AICPU))
1656 0 : HCCLV2_FUNC_RUN(HcomCreateCommCclBufV2(group));
1657 : #endif
1658 0 : return HCCL_SUCCESS;
1659 : }
1660 :
1661 1 : std::shared_ptr<hccl::hcclComm> hcclComm;
1662 1 : CHK_RET(HcomGetCommByGroup(group, hcclComm));
1663 1 : CHK_RET(hcclComm->CreateCommCCLbuffer());
1664 1 : return HCCL_SUCCESS;
1665 1 : }
1666 :
1667 1 : HcclResult HcomGetInCCLbuffer(const char* group, void** buffer, u64* size)
1668 : {
1669 1 : RPT_INPUT_ERR(
1670 : group == nullptr, "EI0003", std::vector<std::string>({"ccl_op", "value", "parameter", "expect"}),
1671 : std::vector<std::string>({"HcomGetInCCLbuffer", "nullptr", "group", "non-null pointer"}));
1672 1 : CHK_PTR_NULL(group);
1673 1 : CHK_PTR_NULL(buffer);
1674 1 : CHK_PTR_NULL(size);
1675 :
1676 1 : HcclResult ret = HcomCheckGroupName(group);
1677 1 : RPT_INPUT_ERR(
1678 : ret != HCCL_SUCCESS, "EI0003", std::vector<std::string>({"ccl_op", "value", "parameter", "expect"}),
1679 : std::vector<std::string>(
1680 : {"HcomGetInCCLbuffer",
1681 : {group, strnlen(group, GROUP_NAME_MAX_LEN + 1)},
1682 : "group",
1683 : "a non-empty string of length 1 to " + std::to_string(GROUP_NAME_MAX_LEN)
1684 : + ", containing only alphanumeric characters and underscores"}));
1685 1 : CHK_PRT_RET(
1686 : ret != HCCL_SUCCESS,
1687 : HCCL_ERROR("[Get][HcomGetInCCLbuffer]errNo[0x%016llx] group name is invalid", HCOM_ERROR_CODE(ret)), ret);
1688 :
1689 1 : DevType devType = HcomGetDeviceType();
1690 1 : if (devType == DevType::DEV_TYPE_950 || devType == DevType::DEV_TYPE_960) {
1691 : #if (!defined(HCCD)) && (!defined(CCL_KERNEL_AICPU))
1692 0 : HCCLV2_FUNC_RUN(HcomGetInCclBufV2(group, *buffer, *size));
1693 : #endif
1694 0 : return HCCL_SUCCESS;
1695 : }
1696 :
1697 1 : std::shared_ptr<hccl::hcclComm> hcclComm;
1698 1 : CHK_RET(HcomGetCommByGroup(group, hcclComm));
1699 1 : CHK_RET(hcclComm->GetInCCLbuffer(*buffer, *size));
1700 1 : return HCCL_SUCCESS;
1701 1 : }
1702 :
1703 1 : HcclResult HcomGetOutCCLbuffer(const char* group, void** buffer, u64* size)
1704 : {
1705 1 : RPT_INPUT_ERR(
1706 : group == nullptr, "EI0003", std::vector<std::string>({"ccl_op", "value", "parameter", "expect"}),
1707 : std::vector<std::string>({"HcomGetOutCCLbuffer", "nullptr", "group", "non-null pointer"}));
1708 1 : CHK_PTR_NULL(group);
1709 1 : CHK_PTR_NULL(buffer);
1710 1 : CHK_PTR_NULL(size);
1711 :
1712 1 : HcclResult ret = HcomCheckGroupName(group);
1713 1 : RPT_INPUT_ERR(
1714 : ret != HCCL_SUCCESS, "EI0003", std::vector<std::string>({"ccl_op", "value", "parameter", "expect"}),
1715 : std::vector<std::string>(
1716 : {"HcomGetOutCCLbuffer",
1717 : {group, strnlen(group, GROUP_NAME_MAX_LEN + 1)},
1718 : "group",
1719 : "a non-empty string of length 1 to " + std::to_string(GROUP_NAME_MAX_LEN)
1720 : + ", containing only alphanumeric characters and underscores"}));
1721 1 : CHK_PRT_RET(
1722 : ret != HCCL_SUCCESS,
1723 : HCCL_ERROR("[Get][HcomGetOutCCLbuffer]errNo[0x%016llx] group name is invalid", HCOM_ERROR_CODE(ret)), ret);
1724 :
1725 1 : DevType devType = HcomGetDeviceType();
1726 1 : if (devType == DevType::DEV_TYPE_950 || devType == DevType::DEV_TYPE_960) {
1727 : #if (!defined(HCCD)) && (!defined(CCL_KERNEL_AICPU))
1728 0 : HCCLV2_FUNC_RUN(HcomGetOutCclBufV2(group, *buffer, *size));
1729 : #endif
1730 0 : return HCCL_SUCCESS;
1731 : }
1732 :
1733 1 : std::shared_ptr<hccl::hcclComm> hcclComm;
1734 1 : CHK_RET(HcomGetCommByGroup(group, hcclComm));
1735 1 : CHK_RET(hcclComm->GetOutCCLbuffer(*buffer, *size));
1736 1 : return HCCL_SUCCESS;
1737 1 : }
1738 :
1739 1 : HcclResult HcomGetAicpuOpStreamNotify(const char* group, HcclRtStream* opStream, u8 aicpuNotifyNum, void** aicpuNotify)
1740 : {
1741 1 : RPT_INPUT_ERR(
1742 : group == nullptr, "EI0003", std::vector<std::string>({"ccl_op", "value", "parameter", "expect"}),
1743 : std::vector<std::string>({"HcomGetDevType", "nullptr", "group", "non-null pointer"}));
1744 1 : CHK_PTR_NULL(group);
1745 1 : CHK_PTR_NULL(opStream);
1746 1 : CHK_PTR_NULL(aicpuNotify);
1747 :
1748 1 : HcclResult ret = HcomCheckGroupName(group);
1749 1 : RPT_INPUT_ERR(
1750 : ret != HCCL_SUCCESS, "EI0003", std::vector<std::string>({"ccl_op", "value", "parameter", "expect"}),
1751 : std::vector<std::string>(
1752 : {"HcomGetDevType",
1753 : {group, strnlen(group, GROUP_NAME_MAX_LEN + 1)},
1754 : "group",
1755 : "a non-empty string of length 1 to " + std::to_string(GROUP_NAME_MAX_LEN)
1756 : + ", containing only alphanumeric characters and underscores"}));
1757 1 : CHK_PRT_RET(
1758 : ret != HCCL_SUCCESS,
1759 : HCCL_ERROR("[Get][HcomGetDevType]errNo[0x%016llx] group name is invalid", HCOM_ERROR_CODE(ret)), ret);
1760 :
1761 1 : std::shared_ptr<hccl::hcclComm> hcclComm;
1762 1 : CHK_RET(HcomGetCommByGroup(group, hcclComm));
1763 1 : CHK_RET(hcclComm->GetAicpuOpStreamNotify(opStream, aicpuNotifyNum, aicpuNotify));
1764 1 : return HCCL_SUCCESS;
1765 1 : }
1766 :
1767 1 : HcclResult HcomMc2AiCpuStreamAllocAndGet(const char* group, u32 streamMode, rtStream_t* aiCpuStream)
1768 : {
1769 1 : RPT_INPUT_ERR(
1770 : group == nullptr, "EI0003", std::vector<std::string>({"ccl_op", "value", "parameter", "expect"}),
1771 : std::vector<std::string>({"HcomGetDevType", "nullptr", "group", "non-null pointer"}));
1772 1 : CHK_PTR_NULL(group);
1773 1 : CHK_PTR_NULL(aiCpuStream);
1774 :
1775 1 : HcclResult ret = HcomCheckGroupName(group);
1776 1 : RPT_INPUT_ERR(
1777 : ret != HCCL_SUCCESS, "EI0003", std::vector<std::string>({"ccl_op", "value", "parameter", "expect"}),
1778 : std::vector<std::string>(
1779 : {"HcomGetDevType",
1780 : {group, strnlen(group, GROUP_NAME_MAX_LEN + 1)},
1781 : "group",
1782 : "a non-empty string of length 1 to " + std::to_string(GROUP_NAME_MAX_LEN)
1783 : + ", containing only alphanumeric characters and underscores"}));
1784 1 : CHK_PRT_RET(
1785 : ret != HCCL_SUCCESS,
1786 : HCCL_ERROR("[Get][HcomGetDevType]errNo[0x%016llx] group name is invalid", HCOM_ERROR_CODE(ret)), ret);
1787 :
1788 : #if (!defined(HCCD)) && (!defined(CCL_KERNEL_AICPU))
1789 1 : HCCLV2_FUNC_RUN(HcomMc2AiCpuStreamAllocAndGetV2(group, streamMode, aiCpuStream));
1790 : #endif
1791 1 : std::shared_ptr<hccl::hcclComm> hcclComm;
1792 1 : ret = HcomGetCommByGroup(group, hcclComm);
1793 : // 兼容V2，获取通信域失败由外层判断，此处不报ERROR
1794 1 : CHK_PRT_RET(ret != HCCL_SUCCESS, HCCL_WARNING("[%s] HcomGetCommByGroup fail", __func__), ret);
1795 1 : CHK_RET(hcclComm->Mc2AiCpuStreamAllocAndGet(streamMode, *aiCpuStream));
1796 1 : return HCCL_SUCCESS;
1797 1 : }
|
