Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include <algorithm>
12 : #include <list>
13 : #include <vector>
14 : #include <string>
15 : #include <securec.h>
16 : #include <hccl/hccl_types.h>
17 : // ltm指定config路径
18 : #include "common/src/config.h"
19 : #include "hccl/base.h"
20 : #include "param_check_pub.h"
21 : #include "remote_access.h"
22 : #include "../op_base/src/op_base.h"
23 : #include "hccl/hcom.h"
24 : #include "rank_consistentcy_checker.h"
25 : #include "profiling_manager_pub.h"
26 : #include "topoinfo_ranktableParser_pub.h"
27 : #include "stream_pub.h"
28 : #include "hcom_common.h"
29 : #include "comm_configer.h"
30 : #include "hcom_private_v2.h"
31 :
32 : #include "hcom_pub.h"
33 :
34 : using namespace std;
35 : using namespace hccl;
36 :
37 :
38 : // DEV_TYPE_V80 对应 DevType::DEV_TYPE_V80
39 : // DEV_TYPE_V51_310_P3 对应 DevType::DEV_TYPE_310P3
40 : // DEV_TYPE_V71 对应 DevType::DEV_TYPE_V71
41 : // DEV_TYPE_V51_310_P1 对应 DevType::DEV_TYPE_310P1
42 : // DEV_TYPE_V81 对应 DevType::DEV_TYPE_V81
43 : // DEV_TYPE_950 对应 DevType::DEV_TYPE_950
44 : // DEV_TYPE_960 对应 DevType::DEV_TYPE_960
45 : // DEV_TYPE_NOSOC 对应 DevType::DEV_TYPE_NOSOC
46 :
47 0 : DevType MakeEnumToDevType(int makeEnum)
48 : {
49 : // 正向映射：MAKE_ENUM到DevType
50 : static std::map<int, DevType> makeEnumToDevType = {{0, DevType::DEV_TYPE_910},
51 : {1, DevType::DEV_TYPE_310P3},
52 : {2, DevType::DEV_TYPE_910B},
53 : {3, DevType::DEV_TYPE_310P1},
54 : {4, DevType::DEV_TYPE_910_93},
55 : {5, DevType::DEV_TYPE_950},
56 : {6, DevType::DEV_TYPE_960},
57 0 : {7, DevType::DEV_TYPE_NOSOC}};
58 :
59 0 : auto it = makeEnumToDevType.find(makeEnum);
60 0 : if (it != makeEnumToDevType.end()) {
61 0 : return it->second;
62 : } else {
63 0 : HCCL_WARNING("Invalid MAKE_ENUM value");
64 : }
65 0 : return DevType::DEV_TYPE_NOSOC;
66 : }
67 :
68 : using HcomCreateGroupCallback = HcclResult (*)(const std::string &, const std::vector<u32> &);
69 : using HcomCallBackGroupIsInit = bool (*)(HcomInfo &);
70 : using HcomDestroyGroupCallback = HcclResult (*)(const std::string &);
71 : using HcomDestroyCallback = HcclResult (*)(HcomInfo &);
72 : HcomCreateGroupCallback g_hcomCreateGroupCallback = nullptr;
73 : HcomCallBackGroupIsInit g_hcomCallBackGroupIsInit = nullptr;
74 : HcomDestroyGroupCallback g_hcomDestroyGroupCallback = nullptr;
75 : HcomDestroyCallback g_hcomDestroyCallback = nullptr;
76 :
77 : using HcomSetGroupTopoInfoPtr = HcclResult (*)(const char *, uint32_t);
78 : using HcomUnsetGroupTopoInfoPtr = void (*)(const char *);
79 : HcomSetGroupTopoInfoPtr g_hcomSetGroupTopoInfo = nullptr;
80 : HcomUnsetGroupTopoInfoPtr g_hcomUnsetGroupTopoInfo = nullptr;
81 :
82 : using HcomInfoCtx = struct HcomInfoCtxTag {
83 : HcomInfo hcomInfo;
84 : shared_ptr<RemoteAccess> remoteAccess;
85 : vector<MemRegisterAddr> remoteAddrInfos;
86 : HcomOpTagInfo opTagInfo;
87 : bool isUsed;
88 :
89 2706 : HcomInfoCtxTag()
90 8118 : : remoteAccess(nullptr), remoteAddrInfos(0), isUsed(false)
91 : {
92 2706 : }
93 : };
94 :
95 : // 梯度切分相关的全局变量
96 : namespace hccl {
97 : std::map<std::string, std::vector<u32>> g_segmentIdxMap;
98 : std::map<std::string, std::vector<float>> g_segmentSizeMap;
99 : std::mutex g_segmentIdxMapLock;
100 : std::mutex g_segmentSizeMapLock;
101 : std::mutex g_setTaskNumCalModeLock;
102 : }
103 :
104 :
105 : std::mutex g_hcomInfoCtxMutex;
106 : HcomInfoCtx g_hcomInfoCtx[MAX_MODULE_DEVICE_NUM + 1];
107 :
108 : std::mutex g_backloggedGroupLock;
109 : std::map<std::string, std::vector<u32>> g_backloggedGroup; // 待创建的group
110 :
111 : std::mutex g_destroyDeviceLock;
112 : static std::mutex g_taskNumCalModeMutex;
113 :
114 : static bool g_isAutoTuneModeOpen = false;
115 : static bool g_notSupportSecAddrCopyWithOffset = false;
116 :
117 713 : HcomInfoCtx& HcomGetCurHcomCtx(void)
118 : {
119 713 : std::lock_guard<std::mutex> lock(g_hcomInfoCtxMutex);
120 713 : s32 deviceLogicId = INVALID_INT;
121 713 : if (hrtGetDevice(&deviceLogicId) == HCCL_SUCCESS && (static_cast<u32>(deviceLogicId) < MAX_MODULE_DEVICE_NUM)) {
122 713 : HCCL_INFO("[HcomGetCurHcomCtx] hrtGetDevice deviceLogicId[%d] ", deviceLogicId);
123 : /* 当前线程获取到deviceId, 如果是首次使用该deviceId的Ctx, 先判断之前是否已经配置过Ctx */
124 713 : if (!g_hcomInfoCtx[deviceLogicId].isUsed) {
125 7 : HCCL_INFO("[HcomGetCurHcomCtx] is no Used deviceLogicId[%d] ", deviceLogicId);
126 7 : if (g_hcomInfoCtx[MAX_MODULE_DEVICE_NUM].isUsed) {
127 0 : return g_hcomInfoCtx[MAX_MODULE_DEVICE_NUM];
128 : }
129 : }
130 713 : g_hcomInfoCtx[deviceLogicId].isUsed = true;
131 713 : return g_hcomInfoCtx[deviceLogicId];
132 : }
133 :
134 : /* 当前线程没有获取到deviceId, 查找是否有使用过的Ctx */
135 0 : for (u32 i = 0; i <= MAX_MODULE_DEVICE_NUM; i++) {
136 0 : if (g_hcomInfoCtx[i].isUsed) {
137 0 : HCCL_INFO("[HcomGetCurHcomCtx] no set device Used deviceLogicId[%u] ", i);
138 0 : return g_hcomInfoCtx[i];
139 : }
140 : }
141 :
142 : /* 当前线程没有获取到deviceId, 使用兜底Ctx */
143 0 : HCCL_INFO("[HcomGetCurHcomCtx] use cover bottom hcomInfoCtx");
144 0 : g_hcomInfoCtx[MAX_MODULE_DEVICE_NUM].isUsed = true;
145 0 : return g_hcomInfoCtx[MAX_MODULE_DEVICE_NUM];
146 713 : }
147 :
148 0 : HcomInfo& HcomGetCtxHomInfoById(u32 idx)
149 : {
150 0 : return g_hcomInfoCtx[idx].hcomInfo;
151 : }
152 :
153 713 : HcomInfo& HcomGetCtxHomInfo(void)
154 : {
155 713 : HcomInfoCtx& curHcomCtx = HcomGetCurHcomCtx();
156 713 : return curHcomCtx.hcomInfo;
157 : }
158 :
159 0 : HcomOpTagInfo& HcomGetCtxOpTagInfo(void)
160 : {
161 0 : HcomInfoCtx& curHcomCtx = HcomGetCurHcomCtx();
162 0 : return curHcomCtx.opTagInfo;
163 : }
164 :
165 0 : bool& HcomGetCtxAutoTuneMode(void)
166 : {
167 0 : return g_isAutoTuneModeOpen;
168 : }
169 :
170 234 : HcclResult HcomSetGroupTopoInfo(const char *group, uint32_t rankSize)
171 : {
172 234 : if (group == nullptr) {
173 0 : HCCL_ERROR("[Hcom][HcomSetGroupTopoInfo] group is null, please check");
174 0 : return HCCL_E_PTR;
175 : }
176 234 : if (g_hcomSetGroupTopoInfo == nullptr) {
177 234 : HCCL_INFO("[Hcom][HcomSetGroupTopoInfo] g_hcomSetGroupTopoInfo is null");
178 234 : HcomInfo &hcomInfo = HcomGetCtxHomInfo();
179 234 : std::lock_guard<std::mutex> lock(hcomInfo.groupRankNumMapLock);
180 : // 1. 单算子流程下只记录，后续不做处理。
181 : // 2. 图模式流程下，后续会通过HcomTopoInfoFuncInstall函数调用回调函数，进行GroupTopoInfo的设置。
182 468 : hcomInfo.groupRankNumMap[std::string(group)] = rankSize;
183 234 : HCCL_RUN_INFO("[Hcom][HcomSetGroupTopoInfo] store groupRankNumMap, group:%s, rankNum:%u",
184 : group, rankSize);
185 234 : return HCCL_SUCCESS;
186 234 : }
187 0 : return g_hcomSetGroupTopoInfo(group, rankSize);
188 : }
189 :
190 2 : HcclResult HcomInitCollComm(uint32_t rank, void **commV2, HcclCommPtr &comm)
191 : {
192 2 : CHK_PTR_NULL(commV2);
193 1 : HCCL_INFO("[HcomInitCollComm] CollComm init start.");
194 : #if (!defined (HCCD)) && (!defined (CCL_KERNEL_AICPU))
195 1 : HcclUs startut = TIME_NOW();
196 1 : char commName[ROOTINFO_INDENTIFIER_MAX_LENGTH] = {};
197 1 : CHK_RET(HcclGetCommNameV2(*commV2, commName));
198 : //获取cclbuffer
199 1 : uintptr_t cclBufferAddr{0};
200 1 : std::size_t cclBufferSize{0};
201 1 : HcclMemType cclBufferMemType{HcclMemType::HCCL_MEM_TYPE_DEVICE};
202 1 : CHK_RET(HcclGetCclBuffer(*commV2, cclBufferAddr, cclBufferSize, cclBufferMemType));
203 : HcclMem cclBuffer;
204 1 : cclBuffer.size = static_cast<uint64_t>(cclBufferSize);
205 1 : cclBuffer.type = cclBufferMemType;
206 1 : cclBuffer.addr = reinterpret_cast<void*>(cclBufferAddr);
207 1 : EXCEPTION_CATCH(comm = make_shared<hccl::hcclComm>(cclBufferSize, cclBufferSize, commName), return HCCL_E_PTR);
208 1 : void *rankGraph = nullptr;
209 1 : CHK_RET(HcclGetRankGraphV2(commV2, &rankGraph));
210 1 : constexpr HcclCommConfig *config = nullptr;
211 3 : CHK_RET(comm->InitCollComm(*commV2, rankGraph, rank, cclBuffer, commName, config));
212 1 : HCCL_RUN_INFO("[%s] success, take time [%lld]us.", __func__, DURATION_US(TIME_NOW() - startut));
213 : #endif
214 1 : return HCCL_SUCCESS;
215 : }
216 :
217 234 : void HcomUnSetGroupTopoInfo(const char *group)
218 : {
219 234 : if (g_hcomUnsetGroupTopoInfo == nullptr) {
220 234 : HCCL_INFO("[Hcom][HcomUnSetGroupTopoInfo] g_hcomUnsetGroupTopoInfo is null, can not unset");
221 234 : return;
222 : }
223 0 : if (group == nullptr) {
224 0 : HCCL_INFO("[Hcom][HcomUnSetGroupTopoInfo] group is null, can not unset");
225 0 : return;
226 : }
227 0 : g_hcomUnsetGroupTopoInfo(group);
228 0 : return;
229 : }
230 :
231 5 : HcclResult HcomGetCommHandleByGroup(const char *group, HcclComm *commHandle)
232 : {
233 5 : CHK_PTR_NULL(commHandle);
234 4 : CHK_PTR_NULL(group);
235 :
236 3 : std::shared_ptr<hcclComm> hcclComm;
237 3 : s32 deviceLogicId = 0;
238 3 : HcclResult ret = HcclDeviceRefresh(deviceLogicId);
239 3 : CHK_PRT_RET(ret != HCCL_SUCCESS,
240 : HCCL_ERROR("[HcomGetCommHandleByGroup]HcclDeviceRefresh failed, group[%s], deviceLogicId[%d], errNo[0x%016llx]",
241 : group, deviceLogicId, HCOM_ERROR_CODE(ret)), ret);
242 :
243 : // MC2单算子和动态图下发性能优化，优先查询返回
244 3 : HcclOpInfoCtx &opBaseHcom = GetHcclExistDeviceOpInfoCtx();
245 3 : std::unique_lock<std::mutex> lock(opBaseHcom.opGroupMapMutex);
246 6 : auto iter = opBaseHcom.opGroup2CommMap.find(std::string(group));
247 3 : if (iter != opBaseHcom.opGroup2CommMap.end()) {
248 2 : hcclComm = iter->second;
249 2 : CHK_PRT_RET(hcclComm == nullptr,
250 : HCCL_ERROR("[HcomGetCommHandleByGroup]opBaseHcom.comm is null, group[%s], deviceLogicId[%d]",
251 : group, deviceLogicId), HCCL_E_PTR);
252 2 : *commHandle = static_cast<HcclComm>(hcclComm.get());
253 2 : return HCCL_SUCCESS;
254 : }
255 1 : lock.unlock();
256 :
257 1 : ret = HcomGetCommByGroup(group, hcclComm);
258 1 : CHK_PRT_RET(ret != HCCL_SUCCESS,
259 : HCCL_ERROR("[HcomGetCommHandleByGroup]HcomGetCommByGroup failed, group[%s], deviceLogicId[%d], errNo[0x%016llx]",
260 : group, deviceLogicId, HCOM_ERROR_CODE(ret)), ret);
261 0 : *commHandle = static_cast<HcclComm>(hcclComm.get());
262 0 : return HCCL_SUCCESS;
263 3 : }
264 :
265 2 : HcclResult HcomGetCommByGroup(const char *group, std::shared_ptr<hccl::hcclComm> &hcclComm)
266 : {
267 2 : HcomInfo &hcomInfo = HcomGetCtxHomInfo();
268 2 : HcclResult ret = HcomCheckGroupName(group);
269 2 : CHK_PRT_RET(ret != HCCL_SUCCESS,
270 : HCCL_ERROR("[Get][CommByGroup]HcomCheckGroupName failed, group[%s], errNo[0x%016llx]",
271 : group, HCOM_ERROR_CODE(ret)), ret);
272 2 : std::string strGroup = (group == nullptr) ? HCCL_WORLD_GROUP : group;
273 2 : if (strGroup == HCCL_WORLD_GROUP) {
274 0 : CHK_PRT_RET(hcomInfo.pComm == nullptr, HCCL_WARNING("[Get][CommByGroup]hcomInfo.pComm is null"), HCCL_E_PTR);
275 0 : hcclComm = hcomInfo.pComm;
276 : } else {
277 2 : std::unique_lock<std::mutex> groupParaLock(hcomInfo.groupParamsLock);
278 2 : auto iter = hcomInfo.hcomGroupMap.find(strGroup);
279 2 : if (iter == hcomInfo.hcomGroupMap.end()) {
280 2 : ret = HcclGetCommHandle(strGroup.c_str(), hcclComm);
281 2 : CHK_PRT_RET(ret != HCCL_SUCCESS, HCCL_WARNING("[Get][CommByGroup]errNo[0x%016llx] group[%s]" \
282 : "group is not exist", HCOM_ERROR_CODE(HCCL_E_NOT_FOUND), strGroup.c_str()), HCCL_E_NOT_FOUND);
283 : } else {
284 0 : hcclComm = (iter->second).pSubComm;
285 0 : CHK_PRT_RET(hcclComm == nullptr,
286 : HCCL_ERROR("[Get][CommByGroup] Get Comm is null, group[%s]", strGroup.c_str()), HCCL_E_PTR);
287 : }
288 1 : groupParaLock.unlock();
289 2 : }
290 :
291 1 : return HCCL_SUCCESS;
292 2 : }
293 :
294 0 : void HcomTopoInfoRegCallback(HcclResult (*p1)(const char *, uint32_t), void (*p2)(const char *))
295 : {
296 0 : g_hcomSetGroupTopoInfo = p1;
297 0 : g_hcomUnsetGroupTopoInfo = p2;
298 :
299 0 : HcomInfo &hcomInfo = HcomGetCtxHomInfo();
300 0 : std::unique_lock<std::mutex> lock(hcomInfo.groupRankNumMapLock);
301 0 : HCCL_RUN_INFO("[HcomTopoInfoRegCallback] size of groupRankNumMap is %zu", hcomInfo.groupRankNumMap.size());
302 0 : for (auto &item : hcomInfo.groupRankNumMap) {
303 0 : HCCL_RUN_INFO("[HcomTopoInfoRegCallback] try to set topo info, group:%s, rankNum:%u",
304 : item.first.c_str(), item.second);
305 0 : HcclResult ret = HcomSetGroupTopoInfo(item.first.c_str(), item.second);
306 0 : if (ret != HCCL_SUCCESS) {
307 0 : HCCL_ERROR("[HcomTopoInfoRegCallback][HcomSetGroupTopoInfo]Set Info failed. errNo[0x%016llx].",
308 : HCOM_ERROR_CODE(ret));
309 : }
310 : }
311 0 : }
312 :
313 : HcclResult HcomStoreBackloggedGroup(const std::string &group, const std::vector<u32> &groupRanks);
314 : HcclResult HcomQueryGroupRef(const char *group, u32 &groupRef);
315 : HcclResult HcomDestroyBackloggedGroup(const std::string &group);
316 : HcclResult GetGroupRankInfo(const char *group, RankInfoType rankType, u32 inPara, u32 *outPara);
317 :
318 0 : HcclResult GetRankList(u32 rankNum, const u32 *rankIds, HcclGroupParams ¶ms)
319 : {
320 0 : HcomInfo &hcomInfo = HcomGetCtxHomInfo();
321 0 : CHK_PTR_NULL(hcomInfo.pComm);
322 0 : std::vector<RankInfo_t> rankList;
323 0 : params.totalRanks = rankNum;
324 0 : params.worldRank = hcomInfo.params.rank;
325 0 : params.groupRank = INVALID_VALUE_RANKID;
326 0 : for (u32 i = 0; i < rankNum; i++) {
327 0 : params.groupRanks.push_back(rankIds[i]);
328 : }
329 :
330 0 : std::sort(params.groupRanks.begin(), params.groupRanks.end());
331 0 : if (params.groupRanks[rankNum - 1] >= hcomInfo.rankTable.rankNum) {
332 0 : HCCL_ERROR("[get][RankList]errNo[0x%016llx] groupRanks[%u]:%u is invalid", HCOM_ERROR_CODE(HCCL_E_PARA),
333 : rankNum - 1, params.groupRanks[rankNum - 1]);
334 0 : return HCCL_E_PARA;
335 : }
336 0 : if (hcomInfo.rankTable.rankList.size() <= params.groupRanks[0]) {
337 0 : HCCL_ERROR("[get][RankList]errNo[0x%016llx] groupRanks[0] is invalid:[%u]", HCOM_ERROR_CODE(HCCL_E_PARA),
338 : params.groupRanks[0]);
339 0 : return HCCL_E_PARA;
340 : }
341 : // groupRanks 个数已经校验非0
342 0 : std::string serverId = hcomInfo.rankTable.rankList[params.groupRanks[0]].serverId;
343 0 : u32 serverNum = 1; // severNum初始值应为1，代表groupId为0的serverId;
344 0 : RankInfo_t rankInfo;
345 0 : for (u32 i = 0; i < rankNum; i++) {
346 0 : rankInfo = hcomInfo.rankTable.rankList[params.groupRanks[i]];
347 : // 校验worldRankID
348 0 : if (rankInfo.rankId != params.groupRanks[i]) {
349 0 : HCCL_ERROR("[get][RankList]errNo[0x%016llx] in rankList, worldRanks[%u] is invalid",
350 : HCOM_ERROR_CODE(HCCL_E_PARA), rankInfo.rankId);
351 0 : return HCCL_E_PARA;
352 : }
353 0 : if (params.groupRanks[i] == params.worldRank) {
354 0 : params.groupRank = i;
355 : }
356 0 : if (rankInfo.serverId != serverId) {
357 0 : serverNum++;
358 0 : serverId = rankInfo.serverId;
359 : }
360 0 : rankInfo.rankId = i; // 放入groupRankid
361 0 : rankList.push_back(rankInfo); // ranktable中的ranklist是以rankid的顺序排列的
362 0 : rankInfo.serverId = ""; // 释放前先指空字符串
363 : }
364 0 : params.serverNum = serverNum;
365 0 : bool isStandardCard = false;
366 0 : CHK_RET(hcomInfo.pComm->IsStandardCard(isStandardCard));
367 :
368 0 : if (!isStandardCard && hcomInfo.params.deviceType != DevType::DEV_TYPE_910B &&
369 0 : hcomInfo.params.deviceType != DevType::DEV_TYPE_910_93) {
370 0 : CHK_RET(CheckRankTableConfigInfo(rankList, rankNum, serverNum));
371 : }
372 0 : return HCCL_SUCCESS;
373 0 : }
374 :
375 0 : HcclResult HcomCreateGroupImpl(const std::string &group, const std::vector<u32> &rankIds)
376 : {
377 0 : HcclUs startut = TIME_NOW();
378 0 : HcomInfo &hcomInfo = HcomGetCtxHomInfo();
379 0 : std::string rankId;
380 0 : for (u32 i = 0; i < rankIds.size(); i++) {
381 0 : if (i < rankIds.size() - 1) {
382 0 : rankId += to_string(rankIds[i]) + ',';
383 0 : } else if (i == rankIds.size() - 1) {
384 0 : rankId += to_string(rankIds[i]);
385 : }
386 : }
387 : /* 接口交互信息日志 */
388 0 : HCCL_RUN_INFO("Entry-HcomCreateGroup:group[%s], rankNum[%u], rankIds[%s]", group.c_str(), rankIds.size(),
389 : rankId.c_str());
390 :
391 0 : if (hcomInfo.params.commWorkMode == HCCL_MODE_NORMAL) {
392 0 : CHK_PRT_RET(hcomInfo.pComm == nullptr,
393 : HCCL_ERROR("[Create][Group]hcomInfo.pComm is null, please check if the initialize process is called."),
394 : HCCL_E_PTR);
395 : } else {
396 0 : if (g_hcomCreateGroupCallback != nullptr) {
397 0 : return g_hcomCreateGroupCallback(group, rankIds);
398 : }
399 : }
400 :
401 0 : CHK_PRT_RET(hcomInfo.rankTable.rankList.empty(),
402 : HCCL_ERROR("[Create][Group]group[%s] rankList is empty", group.c_str()), HCCL_E_INTERNAL);
403 :
404 : /* 已经存在的group不允许再次创建 */
405 0 : std::unique_lock<std::mutex> groupParaLock(hcomInfo.groupParamsLock);
406 0 : if (hcomInfo.hcomGroupMap.find(group) != hcomInfo.hcomGroupMap.end()) {
407 0 : HCCL_ERROR("[Create][Group]errNo[0x%016llx] group[%s] is already exist", HCOM_ERROR_CODE(HCCL_E_PARA),
408 : group.c_str());
409 0 : return HCCL_E_PARA;
410 : }
411 0 : groupParaLock.unlock();
412 :
413 0 : HcclGroupParams groupParamsTem;
414 0 : CHK_RET(GetRankList(rankIds.size(), rankIds.data(), groupParamsTem));
415 :
416 : // 如果是groupRank = INVALID_VALUE_RANKID，即本rank不参与create group
417 0 : if (groupParamsTem.groupRank == INVALID_VALUE_RANKID) {
418 0 : HCCL_ERROR("[Create][Group]errNo[0x%016llx] confirm groupRank from worldRank[%u] error",
419 : HCOM_ERROR_CODE(HCCL_E_NOT_FOUND), hcomInfo.params.rank);
420 0 : return HCCL_E_NOT_FOUND;
421 : }
422 :
423 : /* 入参的正确性由HCCL确保 */
424 0 : CHK_RET(hcomInfo.pComm->CreateGroup(
425 : group, groupParamsTem.groupRank, hcomInfo.params.rank, groupParamsTem.groupRanks, groupParamsTem.pSubComm));
426 0 : CHK_SMART_PTR_NULL(groupParamsTem.pSubComm);
427 :
428 0 : groupParaLock.lock();
429 0 : hcomInfo.hcomGroupMap.insert(std::make_pair(group, groupParamsTem));
430 0 : groupParaLock.unlock();
431 :
432 0 : HCCL_RUN_INFO("hcom create group[%s] success, take time [%lld]us",
433 : group.c_str(), DURATION_US(TIME_NOW() - startut));
434 0 : return HCCL_SUCCESS;
435 0 : }
436 :
437 0 : HcclResult HcomCreateGroup(const char *group, u32 rankNum, u32 *rankIds)
438 : {
439 : /* 调优模式直接返回success */
440 0 : bool &isAutoTuneModeOpen = HcomGetCtxAutoTuneMode();
441 0 : if (isAutoTuneModeOpen) {
442 0 : return HCCL_SUCCESS;
443 : }
444 0 : RPT_INPUT_ERR(group == nullptr, "EI0003", std::vector<std::string>({"ccl_op", "value", "parameter", "expect"}),\
445 : std::vector<std::string>({"HcomCreateGroup", "nullptr", "group", "non-null pointer"}));
446 0 : CHK_PTR_NULL(group);
447 0 : HcclResult ret = HcomCheckGroupName(group);
448 0 : RPT_INPUT_ERR(ret != HCCL_SUCCESS,
449 : "EI0003", std::vector<std::string>({ "ccl_op", "value", "parameter", "value" }),
450 : std::vector<std::string>({
451 : "HcomCreateGroup",
452 : { group, strnlen(group, GROUP_NAME_MAX_LEN + 1) },
453 : "group",
454 : "a non-empty string of length 1 to " + std::to_string(GROUP_NAME_MAX_LEN) +
455 : ", containing only alphanumeric characters and underscores"
456 : }
457 : ));
458 0 : CHK_PRT_RET(ret != HCCL_SUCCESS,
459 : HCCL_ERROR("[%s][%s]errNo[0x%016llx] group name is invalid",
460 : LOG_KEYWORDS_TASK_EXEC.c_str(),
461 : LOG_KEYWORDS_INVALID_ARGUMENT.c_str(),
462 : HCOM_ERROR_CODE(ret)),
463 : ret);
464 0 : CHK_PRT_RET((!strncmp(group, HCCL_WORLD_GROUP, sizeof(HCCL_WORLD_GROUP))),
465 : HCCL_ERROR("[%s][%s]create group isn't support world group",
466 : LOG_KEYWORDS_TASK_EXEC.c_str(),
467 : LOG_KEYWORDS_INVALID_ARGUMENT.c_str()),
468 : HCCL_E_PARA);
469 :
470 0 : RPT_INPUT_ERR(rankIds == nullptr, "EI0003", std::vector<std::string>({"ccl_op", "value", "parameter", "expect"}),\
471 : std::vector<std::string>({"HcomCreateGroup", "nullptr", "rankIds", "non-null pointer"}));
472 0 : CHK_PTR_NULL(rankIds);
473 :
474 0 : if (rankNum == 0) {
475 0 : RPT_INPUT_ERR(true, "EI0003", std::vector<std::string>({"ccl_op", "value", "parameter", "expect"}),
476 : std::vector<std::string>({
477 : "HcomCreateGroup",
478 : std::to_string(rankNum),
479 : "rankNum",
480 : "must be a positive integer (greater than 0)"
481 : }
482 : ));
483 0 : HCCL_ERROR("[%s][%s]errNo[0x%016llx] group[%s] rankNum[%u] is invalid",
484 : LOG_KEYWORDS_TASK_EXEC.c_str(), LOG_KEYWORDS_INVALID_ARGUMENT.c_str(), HCOM_ERROR_CODE(HCCL_E_PARA), group, rankNum);
485 0 : return HCCL_E_PARA;
486 : }
487 : // 入参合法性校验 END
488 0 : std::vector<u32> ranks(rankIds, rankIds + rankNum);
489 0 : HcomInfo &hcomInfo = HcomGetCtxHomInfo();
490 : #if (!defined (HCCD)) && (!defined (CCL_KERNEL_AICPU))
491 0 : HCCLV2_FUNC_RUN(
492 : [&]() -> HcclResult {
493 : CHK_RET(HcomCreateGroupImplV2(group, rankNum, ranks));
494 : HcclGroupParams groupParams{};
495 : void *commV2 = nullptr;
496 : CHK_RET(HcomGetGroupParamsV2(group, static_cast<void *>(&groupParams), &commV2));
497 : Hccl::RankId rank = static_cast<Hccl::RankId>(groupParams.groupRank);
498 : CHK_RET(HcomInitCollComm(rank, &commV2, groupParams.pSubComm));
499 : CHK_PTR_NULL(groupParams.pSubComm);
500 : std::unique_lock<std::mutex> groupParaLock(hcomInfo.groupParamsLock);
501 : hcomInfo.hcomGroupMap.insert(std::make_pair(group, groupParams));
502 : groupParaLock.unlock();
503 : CHK_RET(HcomSetGroupTopoInfo(group, rankNum));
504 : HCCL_INFO("[HcomCreateGroup] create group[%s] success.", group);
505 : return HCCL_SUCCESS;
506 : }());
507 : #endif
508 0 : if (hcomInfo.pComm == nullptr &&
509 0 : ((g_hcomCallBackGroupIsInit != nullptr) && (!(g_hcomCallBackGroupIsInit(hcomInfo))))) {
510 0 : CHK_RET(HcomStoreBackloggedGroup(group, ranks));
511 : } else {
512 0 : CHK_RET(HcomCreateGroupImpl(group, ranks));
513 0 : CHK_RET(HcomSetGroupTopoInfo(group, rankNum));
514 : }
515 0 : return HCCL_SUCCESS;
516 0 : }
517 :
518 0 : HcclResult DestroyFlag(const char *group, bool flag)
519 : {
520 0 : HcomInfo &hcomInfo = HcomGetCtxHomInfo();
521 :
522 0 : if (group == nullptr) {
523 0 : return HCCL_SUCCESS; // 全局通信域不需要查询flag
524 : }
525 0 : std::string strGroup = group;
526 0 : std::unique_lock<std::mutex> groupParaLock(hcomInfo.groupParamsLock);
527 0 : auto iter = hcomInfo.hcomGroupMap.find(strGroup);
528 0 : if (iter == hcomInfo.hcomGroupMap.end()) {
529 0 : HCCL_ERROR("[Get][CommByGroup]errNo[0x%016llx] group[%s] group is not exist",
530 : HCOM_ERROR_CODE(HCCL_E_NOT_FOUND), strGroup.c_str());
531 0 : return HCCL_E_NOT_FOUND; // 不存在该服务器内相关dev的对应信息
532 : }
533 0 : iter->second.destroyFlag = flag;
534 0 : return HCCL_SUCCESS;
535 0 : }
536 :
537 0 : HcclResult QueryDestroyFlag(const char *group)
538 : {
539 0 : HcomInfo &hcomInfo = HcomGetCtxHomInfo();
540 0 : if (group == nullptr) {
541 0 : return HCCL_SUCCESS; // 全局通信域不需要查询flag
542 : }
543 0 : std::string strGroup = group;
544 0 : std::unique_lock<std::mutex> groupParaLock(hcomInfo.groupParamsLock);
545 0 : auto iter = hcomInfo.hcomGroupMap.find(strGroup);
546 0 : if (iter == hcomInfo.hcomGroupMap.end()) {
547 0 : HCCL_WARNING("[Get][CommByGroup]errNo[0x%016llx] group[%s] group is not exist",
548 : HCOM_ERROR_CODE(HCCL_E_AGAIN), strGroup.c_str());
549 0 : return HCCL_E_AGAIN; // 不存在该服务器内相关dev的对应信息
550 : }
551 0 : if (iter->second.destroyFlag) {
552 0 : return HCCL_E_AGAIN;
553 : }
554 0 : return HCCL_SUCCESS;
555 0 : }
556 :
557 0 : HcclResult HcomDestroyGroupImpl(const std::string &group)
558 : {
559 : /* 调优模式直接返回success */
560 0 : bool &isAutoTuneModeOpen = HcomGetCtxAutoTuneMode();
561 0 : if (isAutoTuneModeOpen) {
562 0 : return HCCL_SUCCESS;
563 : }
564 0 : HcomInfo &hcomInfo = HcomGetCtxHomInfo();
565 0 : CHK_PRT_RET(hcomInfo.pComm == nullptr, HCCL_ERROR("[Destroy][Group]hcomInfo.pComm is null, "\
566 : "please check if the initialize process is called."), HCCL_E_PTR);
567 0 : if (hcomInfo.params.commWorkMode != HCCL_MODE_NORMAL) {
568 0 : if (g_hcomDestroyGroupCallback != nullptr) {
569 0 : return g_hcomDestroyGroupCallback(group);
570 : }
571 : }
572 :
573 : /* 接口交互信息日志 */
574 0 : HCCL_RUN_INFO("Entry-HcomDestroyGroup:group[%s]", group.c_str());
575 0 : CHK_RET(DestroyFlag(group.c_str(), true));
576 0 : u32 ref = 0;
577 0 : CHK_RET(HcomQueryGroupRef(group.c_str(), ref));
578 0 : while (ref != 0) {
579 0 : std::shared_ptr<hccl::hcclComm> hcclComm = nullptr;
580 0 : CHK_RET(HcomGetCommByGroup(group.c_str(), hcclComm));
581 0 : SaluSleep(ONE_HUNDRED_MICROSECOND_OF_USLEEP);
582 0 : CHK_RET(HcomQueryGroupRef(group.c_str(), ref));
583 0 : }
584 :
585 0 : std::unique_lock<std::mutex> groupParaLock(hcomInfo.groupParamsLock);
586 0 : auto iter = hcomInfo.hcomGroupMap.find(group);
587 0 : if (iter == hcomInfo.hcomGroupMap.end()) {
588 0 : HCCL_ERROR("[Destroy][Group]errNo[0x%016llx] group[%s] is not exist", HCOM_ERROR_CODE(HCCL_E_PARA),
589 : group.c_str());
590 0 : return HCCL_E_PARA;
591 : }
592 :
593 0 : CHK_RET(hcomInfo.pComm->DestroyGroup(group));
594 :
595 0 : (iter->second).groupRanks.clear(); // 清除该服务器内相关group的对应信息
596 :
597 0 : hcomInfo.hcomGroupMap.erase(group);
598 0 : groupParaLock.unlock();
599 :
600 0 : HCCL_RUN_INFO("hcom destroy group[%s] success.", group.c_str());
601 0 : return HCCL_SUCCESS;
602 0 : }
603 :
604 0 : HcclResult HcomDestroyGroup(const char *group)
605 : {
606 : /* 调优模式直接返回success */
607 0 : bool &isAutoTuneModeOpen = HcomGetCtxAutoTuneMode();
608 0 : if (isAutoTuneModeOpen) {
609 0 : return HCCL_SUCCESS;
610 : }
611 :
612 0 : RPT_INPUT_ERR(group == nullptr, "EI0003", std::vector<std::string>({"ccl_op", "value", "parameter", "expect"}),\
613 : std::vector<std::string>({"HcomDestroyGroup", "nullptr", "group", "non-null pointer"}));
614 0 : CHK_PTR_NULL(group);
615 0 : CHK_RET(HcomCheckGroupName(group));
616 :
617 0 : if (!strncmp(group, HCCL_WORLD_GROUP, sizeof(HCCL_WORLD_GROUP))) {
618 0 : HCCL_ERROR("[%s][%s]errNo[0x%016llx] destroy group is world group",
619 : LOG_KEYWORDS_TASK_EXEC.c_str(), LOG_KEYWORDS_INVALID_ARGUMENT.c_str(), HCOM_ERROR_CODE(HCCL_E_PARA));
620 0 : return HCCL_E_PARA;
621 : }
622 0 : HcomInfo &hcomInfo = HcomGetCtxHomInfo();
623 : #if (!defined (HCCD)) && (!defined (CCL_KERNEL_AICPU))
624 0 : HCCLV2_FUNC_RUN(
625 : [&]() -> HcclResult {
626 : std::unique_lock<std::mutex> groupParaLock(hcomInfo.groupParamsLock);
627 : CHK_RET(HcomDestroyGroupImplV2(group));
628 : auto iter = hcomInfo.hcomGroupMap.find(group);
629 : if (iter == hcomInfo.hcomGroupMap.end()) {
630 : HCCL_ERROR(
631 : "[Destroy][Group]errNo[0x%016llx] group[%s] is not exist", HCOM_ERROR_CODE(HCCL_E_PARA), group);
632 : return HCCL_E_PARA;
633 : }
634 : hcomInfo.hcomGroupMap.erase(group);
635 : groupParaLock.unlock();
636 : return HCCL_SUCCESS;
637 : }());
638 : #endif
639 :
640 0 : if (hcomInfo.pComm == nullptr &&
641 0 : ((g_hcomCallBackGroupIsInit != nullptr) && (!(g_hcomCallBackGroupIsInit(hcomInfo))))) {
642 0 : CHK_RET(HcomDestroyBackloggedGroup(group));
643 : } else {
644 0 : CHK_RET(HcomDestroyGroupImpl(group));
645 : }
646 :
647 0 : HcomUnSetGroupTopoInfo(group);
648 :
649 0 : std::unique_lock<std::mutex> lock(g_backloggedGroupLock);
650 0 : if (g_backloggedGroup.find(group) != hcomInfo.backloggedGroup.end()) {
651 0 : g_backloggedGroup.erase(group);
652 0 : HCCL_INFO("hcom delete g_backlogged group[%s] success.", group);
653 : }
654 0 : return HCCL_SUCCESS;
655 0 : }
656 :
657 0 : HcclResult HcomFlushBackloggedGroups()
658 : {
659 0 : HCCL_INFO("HcomFlushBackloggedGroups");
660 0 : HcomInfo &hcomInfo = HcomGetCtxHomInfo();
661 0 : std::unique_lock<std::mutex> backGroupParaLock(g_backloggedGroupLock);
662 : using ITER = map<string, std::vector<u32>>::iterator;
663 0 : for (ITER iter = g_backloggedGroup.begin(); iter != g_backloggedGroup.end();) {
664 0 : HCCL_INFO("HcomFlushBackloggedGroups[%s], rank[%u]", iter->first.c_str(), hcomInfo.params.rank);
665 0 : if (std::count(iter->second.begin(), iter->second.end(), hcomInfo.params.rank)) {
666 0 : HCCL_INFO("HcomFlushBackloggedGroups[%s], rank[%u] success", iter->first.c_str(), hcomInfo.params.rank);
667 0 : hcomInfo.backloggedGroup.insert({iter->first, iter->second});
668 : }
669 0 : iter++;
670 : }
671 0 : backGroupParaLock.unlock();
672 :
673 0 : std::unique_lock<std::mutex> lock(hcomInfo.backloggedGroupLock);
674 0 : for (ITER iter = hcomInfo.backloggedGroup.begin(); iter != hcomInfo.backloggedGroup.end();) {
675 0 : CHK_RET(HcomCreateGroupImpl(iter->first, iter->second));
676 0 : hcomInfo.backloggedGroup.erase(iter++);
677 : }
678 0 : HCCL_INFO("HcomFlushBackloggedGroups success.");
679 0 : return HCCL_SUCCESS;
680 0 : }
681 :
682 0 : HcclResult HcomStoreBackloggedGroup(const std::string &group, const std::vector<u32> &groupRanks)
683 : {
684 0 : HcomInfo &hcomInfo = HcomGetCtxHomInfo();
685 : // 该线程最开始是否未设置deviceid并获取了ctx
686 0 : bool hcomUseDefaultCtx = (&hcomInfo == &(g_hcomInfoCtx[MAX_MODULE_DEVICE_NUM].hcomInfo));
687 0 : s32 deviceLogicId = INVALID_INT;
688 0 : if (hrtGetDevice(&deviceLogicId) != HCCL_SUCCESS || (static_cast<u32>(deviceLogicId) >= MAX_MODULE_DEVICE_NUM) ||
689 : hcomUseDefaultCtx) {
690 0 : HCCL_INFO("[device not set]hcom store group[%s]", group.c_str());
691 0 : std::unique_lock<std::mutex> groupParaLock(g_backloggedGroupLock);
692 0 : if (g_backloggedGroup.find(group) != g_backloggedGroup.end()) {
693 0 : HCCL_INFO("[Store][BackloggedGroup]group[%s] is existed", group.c_str());
694 0 : if (g_backloggedGroup[group] == groupRanks) {
695 0 : HCCL_ERROR("[Store][BackloggedGroup]group[%s] has been created", group.c_str());
696 0 : return HCCL_E_PARA;
697 : }
698 0 : g_backloggedGroup[group] = groupRanks;
699 0 : HCCL_INFO("[Store][BackloggedGroup]group[%s] updated", group.c_str());
700 0 : return HCCL_SUCCESS;
701 : }
702 :
703 0 : g_backloggedGroup.insert({group, groupRanks});
704 0 : HCCL_INFO("[device not set]hcom store group[%s] success", group.c_str());
705 0 : return HCCL_SUCCESS;
706 0 : }
707 :
708 0 : std::unique_lock<std::mutex> groupParaLock(hcomInfo.groupParamsLock);
709 0 : if (hcomInfo.hcomGroupMap.find(group) != hcomInfo.hcomGroupMap.end()) {
710 0 : HCCL_ERROR("[Store][BackloggedGroup]group[%s] has been created", group.c_str());
711 0 : return HCCL_E_PARA;
712 : }
713 0 : groupParaLock.unlock();
714 :
715 0 : std::unique_lock<std::mutex> lock(hcomInfo.backloggedGroupLock);
716 0 : if (hcomInfo.backloggedGroup.find(group) != hcomInfo.backloggedGroup.end()) {
717 0 : HCCL_ERROR("[Store][BackloggedGroup]group[%s] is existed", group.c_str());
718 0 : return HCCL_E_PARA;
719 : } else {
720 0 : hcomInfo.backloggedGroup.insert({ group, groupRanks });
721 : }
722 0 : HCCL_INFO("hcom store group[%s] success.", group.c_str());
723 0 : return HCCL_SUCCESS;
724 0 : }
725 :
726 0 : HcclResult HcomDestroyBackloggedGroup(const std::string &group)
727 : {
728 0 : HcomInfo &hcomInfo = HcomGetCtxHomInfo();
729 0 : std::unique_lock<std::mutex> lock(hcomInfo.backloggedGroupLock);
730 0 : if (hcomInfo.backloggedGroup.find(group) == hcomInfo.backloggedGroup.end()) {
731 0 : if (!hcomInfo.isHcomInit) {
732 0 : HCCL_WARNING("[Destroy][BackloggedGroup]group[%s] is not existed, and hcom has not been inited yet",
733 : group.c_str());
734 0 : return HCCL_SUCCESS;
735 : } else {
736 0 : HCCL_ERROR("[Destroy][BackloggedGroup]group[%s] is not existed", group.c_str());
737 0 : return HCCL_E_PARA;
738 : }
739 : } else {
740 0 : hcomInfo.backloggedGroup.erase(group);
741 : }
742 0 : HCCL_INFO("hcom delete backlogged group[%s] success.", group.c_str());
743 0 : return HCCL_SUCCESS;
744 0 : }
745 :
746 0 : HcclResult HcomGetbackloggedByGroup(const char *group, std::vector<u32> &groupRanks, s32 &groupSize)
747 : {
748 0 : CHK_RET(HcomCheckGroupName(group));
749 0 : std::string groupName = group;
750 :
751 0 : std::unique_lock<std::mutex> groupLock(g_backloggedGroupLock);
752 0 : auto it = g_backloggedGroup.find(groupName);
753 0 : if (it != g_backloggedGroup.end()) {
754 0 : groupRanks = it->second;
755 0 : groupSize = (it->second).size();
756 0 : HCCL_INFO("[device not set]get back logged group[%s], groupSize[%d]", groupName.c_str(), groupSize);
757 0 : return HCCL_SUCCESS;
758 : }
759 0 : groupLock.unlock();
760 0 : HcomInfo &hcomInfo = HcomGetCtxHomInfo();
761 0 : std::unique_lock<std::mutex> groupParaLock(hcomInfo.backloggedGroupLock);
762 0 : auto iter = hcomInfo.backloggedGroup.find(groupName);
763 0 : if (iter == hcomInfo.backloggedGroup.end()) {
764 0 : groupSize = 0;
765 0 : HCCL_DEBUG("[Get][CommByGroup]errNo[0x%016llx] group[%s] is not exist",
766 : HCOM_ERROR_CODE(HCCL_E_NOT_FOUND), group);
767 0 : return HCCL_SUCCESS; // 不存在该服务器内相关dev的对应信息
768 : }
769 0 : groupRanks = iter->second;
770 0 : groupSize = (iter->second).size();
771 0 : HCCL_INFO("[device set]get back logged group[%s], groupSize[%d]", groupName.c_str(), groupSize);
772 0 : return HCCL_SUCCESS;
773 0 : }
774 :
775 0 : HcclResult HcomQueryGroupRef(const char *group, u32 &groupRef)
776 : {
777 0 : HcomInfo &hcomInfo = HcomGetCtxHomInfo();
778 0 : if (group == nullptr) {
779 0 : return HCCL_SUCCESS; // 全局通信域不需要查询flag
780 : }
781 0 : std::string strGroup = group;
782 0 : std::unique_lock<std::mutex> groupParaLock(hcomInfo.groupParamsLock);
783 0 : auto iter = hcomInfo.hcomGroupMap.find(strGroup);
784 0 : if (iter == hcomInfo.hcomGroupMap.end()) {
785 0 : HCCL_WARNING("[Get][CommByGroup]errNo[0x%016llx] group[%s] group is not exist",
786 : HCOM_ERROR_CODE(HCCL_E_AGAIN), strGroup.c_str());
787 0 : return HCCL_E_AGAIN; // 不存在该服务器内相关dev的对应信息
788 : }
789 0 : groupRef = iter->second.refCounter;
790 0 : return HCCL_SUCCESS;
791 0 : }
792 :
793 0 : HcclResult HcomGetWorldRankFromGroupRank(const char *group, u32 groupRank, u32 *worldRank)
794 : {
795 0 : RPT_INPUT_ERR(worldRank == nullptr, "EI0003", std::vector<std::string>({"ccl_op", "value", "parameter", "expect"}),\
796 : std::vector<std::string>({"HcomGetWorldRankFromGroupRank", "nullptr", "worldRank", "non-null pointer"}));
797 0 : CHK_PTR_NULL(worldRank);
798 0 : bool &isAutoTuneModeOpen = HcomGetCtxAutoTuneMode();
799 0 : if (isAutoTuneModeOpen) {
800 0 : *worldRank = 0;
801 0 : return HCCL_SUCCESS;
802 : }
803 0 : HcomInfo &hcomInfo = HcomGetCtxHomInfo();
804 :
805 0 : HcclResult ret = HcomCheckGroupName(group);
806 0 : RPT_INPUT_ERR(ret != HCCL_SUCCESS,
807 : "EI0003", std::vector<std::string>({"ccl_op", "value", "parameter", "expect"}),
808 : std::vector<std::string>({
809 : "HcomGetWorldRankFromGroupRank",
810 : { group, strnlen(group, GROUP_NAME_MAX_LEN + 1) },
811 : "group",
812 : "a non-empty string of length 1 to " + std::to_string(GROUP_NAME_MAX_LEN) +
813 : ", containing only alphanumeric characters and underscores"
814 : }));
815 0 : CHK_PRT_RET(ret != HCCL_SUCCESS,
816 : HCCL_ERROR("[%s][%s]errNo[0x%016llx] group name is invalid",
817 : LOG_KEYWORDS_TASK_EXEC.c_str(), LOG_KEYWORDS_INVALID_ARGUMENT.c_str(), HCOM_ERROR_CODE(ret)), ret);
818 : #if (!defined (HCCD)) && (!defined (CCL_KERNEL_AICPU))
819 0 : HCCLV2_FUNC_RUN(HcomGetWorldRankFromGroupRankV2(group, groupRank, worldRank));
820 : #endif
821 0 : if (groupRank >= hcomInfo.params.totalRanks) {
822 0 : HCCL_ERROR("[Get][WorldRank]errNo[0x%016llx] groupRank[%u] is out of range[0-%u]",
823 : HCOM_ERROR_CODE(HCCL_E_PARA), groupRank, hcomInfo.params.totalRanks);
824 0 : return HCCL_E_PARA;
825 : }
826 0 : std::string strGroup = (group == nullptr) ? HCCL_WORLD_GROUP : group;
827 0 : CHK_RET(GetGroupRankInfo(strGroup.c_str(), RankInfoType::WORLD_RANK_ID_BY_GROUP, groupRank, worldRank));
828 0 : HCCL_INFO("hcom get world rank success, group[%s], groupRank[%u], worldRank[%p]", strGroup.c_str(), groupRank,
829 : worldRank);
830 0 : return HCCL_SUCCESS;
831 0 : }
832 :
833 0 : HcclResult HcomGetGroupRankFromWorldRank(u32 worldRank, const char *group, u32 *groupRank)
834 : {
835 0 : RPT_INPUT_ERR(groupRank == nullptr, "EI0003", std::vector<std::string>({"ccl_op", "value", "parameter", "expect"}),\
836 : std::vector<std::string>({"HcomGetGroupRankFromWorldRank", "nullptr", "groupRank", "non-null pointer"}));
837 0 : CHK_PTR_NULL(groupRank);
838 0 : bool &isAutoTuneModeOpen = HcomGetCtxAutoTuneMode();
839 0 : if (isAutoTuneModeOpen) {
840 0 : *groupRank = 0;
841 0 : return HCCL_SUCCESS;
842 : }
843 0 : HcomInfo &hcomInfo = HcomGetCtxHomInfo();
844 :
845 0 : HcclResult ret = HcomCheckGroupName(group);
846 0 : RPT_INPUT_ERR(ret != HCCL_SUCCESS,
847 : "EI0003", std::vector<std::string>({"ccl_op", "value", "parameter", "expect"}),
848 : std::vector<std::string>({
849 : "HcomGetGroupRankFromWorldRank",
850 : { group, strnlen(group, GROUP_NAME_MAX_LEN + 1) },
851 : "group",
852 : "a non-empty string of length 1 to " + std::to_string(GROUP_NAME_MAX_LEN) +
853 : ", containing only alphanumeric characters and underscores"
854 : }));
855 0 : CHK_PRT_RET(ret != HCCL_SUCCESS,
856 : HCCL_ERROR("[%s][%s]errNo[0x%016llx] group name is invalid",
857 : LOG_KEYWORDS_TASK_EXEC.c_str(), LOG_KEYWORDS_INVALID_ARGUMENT.c_str(), HCOM_ERROR_CODE(ret)), ret);
858 : #if (!defined (HCCD)) && (!defined (CCL_KERNEL_AICPU))
859 0 : HCCLV2_FUNC_RUN(HcomGetGroupRankFromWorldRankV2(worldRank, group, groupRank));
860 : #endif
861 0 : std::string strGroup = (group == nullptr) ? HCCL_WORLD_GROUP : group;
862 0 : if (worldRank >= hcomInfo.params.totalRanks) {
863 0 : HCCL_ERROR("[Get][GroupRank]errNo[0x%016llx] world[%u] rank is invalid", HCOM_ERROR_CODE(HCCL_E_PARA),
864 : worldRank);
865 0 : return HCCL_E_PARA;
866 : }
867 0 : CHK_RET(GetGroupRankInfo(strGroup.c_str(), RankInfoType::GROUP_RANK_ID_BY_WORLD, worldRank, groupRank));
868 0 : HCCL_INFO("hcom get group rank success, group[%s], worldRank[%u], groupRank[%p]", strGroup.c_str(), worldRank,
869 : groupRank);
870 0 : return HCCL_SUCCESS;
871 0 : }
872 :
873 0 : bool HcomFindGroup(const char *group)
874 : {
875 0 : HcomInfo &hcomInfo = HcomGetCtxHomInfo();
876 : /* 已经存在的group不允许再次创建 */
877 0 : std::unique_lock<std::mutex> groupParaLock(hcomInfo.groupParamsLock);
878 0 : bool exists = !(hcomInfo.hcomGroupMap.find(group) == hcomInfo.hcomGroupMap.end());
879 0 : HCCL_INFO("[Find][Group] group[%s] is exist[%d]", group, exists);
880 0 : groupParaLock.unlock();
881 0 : return exists;
882 0 : }
883 :
884 0 : HcclResult GetWorldGroupRankInfo(RankInfoType rankType, u32 inPara, u32 *outPara)
885 : {
886 0 : HcomInfo &hcomInfo = HcomGetCtxHomInfo();
887 0 : CHK_PTR_NULL(outPara);
888 0 : switch (rankType) {
889 0 : case RankInfoType::RANK_SIZE_IN_GROUP:
890 0 : *outPara = hcomInfo.params.totalRanks;
891 0 : break;
892 :
893 0 : case RankInfoType::RANK_ID_IN_GROUP:
894 0 : *outPara = hcomInfo.params.rank;
895 0 : break;
896 :
897 0 : case RankInfoType::WORLD_RANK_ID_BY_GROUP:
898 : case RankInfoType::GROUP_RANK_ID_BY_WORLD:
899 0 : *outPara = inPara;
900 0 : break;
901 0 : case RankInfoType::SERVER_NUM_IN_GROUP:
902 0 : *outPara = hcomInfo.rankTable.serverNum;
903 0 : break;
904 0 : default:
905 0 : HCCL_ERROR("[Get][WorldGroupRankInfo]errNo[0x%016llx] invalid rankInfo type[%d]",
906 : HCOM_ERROR_CODE(HCCL_E_PARA), rankType);
907 0 : return HCCL_E_PARA;
908 : }
909 0 : return HCCL_SUCCESS;
910 : }
911 :
912 0 : HcclResult GetGroupRankInfo(const char *group, RankInfoType rankType, u32 inPara, u32 *outPara)
913 : {
914 0 : CHK_PTR_NULL(outPara);
915 : // std::string strGroup = (group == nullptr) ? HCCL_WORLD_GROUP : group;
916 0 : if ((group == nullptr) || (strcmp(group, HCCL_WORLD_GROUP) == 0)) {
917 0 : CHK_RET(GetWorldGroupRankInfo(rankType, inPara, outPara));
918 0 : return HCCL_SUCCESS;
919 : }
920 0 : HcomInfo &hcomInfo = HcomGetCtxHomInfo();
921 :
922 0 : std::unique_lock<std::mutex> groupParaLock(hcomInfo.groupParamsLock);
923 0 : auto iter = hcomInfo.hcomGroupMap.find(group);
924 0 : if (iter == hcomInfo.hcomGroupMap.end()) {
925 0 : HCCL_ERROR("[Get][GroupRankInfo]errNo[0x%016llx] group[%s] is not exist", HCOM_ERROR_CODE(HCCL_E_NOT_FOUND),
926 : group);
927 0 : return HCCL_E_NOT_FOUND; // 不存在该服务器内相关dev的对应信息
928 : }
929 : // group ranks判空
930 0 : CHK_PRT_RET((iter->second).groupRanks.empty(), HCCL_ERROR("[Get][GroupRankInfo]errNo[0x%016llx] group[%s]"
931 : "ranks is empty", HCOM_ERROR_CODE(HCCL_E_INTERNAL), group), HCCL_E_INTERNAL);
932 :
933 0 : switch (rankType) {
934 0 : case RankInfoType::RANK_SIZE_IN_GROUP:
935 0 : *outPara = (iter->second).totalRanks;
936 0 : return HCCL_SUCCESS;
937 :
938 0 : case RankInfoType::RANK_ID_IN_GROUP:
939 0 : *outPara = (iter->second).groupRank;
940 0 : return HCCL_SUCCESS;
941 :
942 0 : case RankInfoType::WORLD_RANK_ID_BY_GROUP:
943 0 : if (inPara >= (iter->second).totalRanks) {
944 0 : HCCL_ERROR("[Get][GroupRankInfo]errNo[0x%016llx] group[%s] groupRank[%u] is invalid",
945 : HCOM_ERROR_CODE(HCCL_E_PARA), group, inPara);
946 0 : return HCCL_E_PARA;
947 : }
948 0 : *outPara = (iter->second).groupRanks[inPara];
949 0 : return HCCL_SUCCESS;
950 :
951 0 : case RankInfoType::GROUP_RANK_ID_BY_WORLD:
952 0 : for (u32 rank = 0; rank < (iter->second).totalRanks; rank++) {
953 0 : if (inPara == (iter->second).groupRanks[rank]) {
954 0 : *outPara = rank;
955 0 : return HCCL_SUCCESS;
956 : }
957 : }
958 0 : HCCL_ERROR("[Get][GroupRankInfo]errNo[0x%016llx] invalid rankInfo type[%d]",
959 : HCOM_ERROR_CODE(HCCL_E_PARA), rankType);
960 0 : return HCCL_E_PARA;
961 0 : case RankInfoType::SERVER_NUM_IN_GROUP:
962 0 : *outPara = (iter->second).serverNum;
963 0 : return HCCL_SUCCESS;
964 0 : default:
965 0 : HCCL_ERROR("[Get][GroupRankInfo]errNo[0x%016llx] invalid rankInfo type[%d]",
966 : HCOM_ERROR_CODE(HCCL_E_PARA), rankType);
967 0 : return HCCL_E_PARA;
968 : }
969 0 : }
970 :
971 0 : HcclResult HcomGetRankSize(const char *group, u32 *rankSize)
972 : {
973 0 : RPT_INPUT_ERR(rankSize == nullptr, "EI0003", std::vector<std::string>({"ccl_op", "value", "parameter", "expect"}),\
974 : std::vector<std::string>({"HcomGetRankSize", "nullptr", "rankSize", "non-null pointer"}));
975 0 : CHK_PTR_NULL(rankSize);
976 0 : bool &isAutoTuneModeOpen = HcomGetCtxAutoTuneMode();
977 0 : if (isAutoTuneModeOpen) {
978 0 : *rankSize = 1;
979 0 : return HCCL_SUCCESS;
980 : }
981 :
982 0 : HcclResult ret = HcomCheckGroupName(group);
983 0 : RPT_INPUT_ERR(ret != HCCL_SUCCESS, "EI0003", std::vector<std::string>({"ccl_op", "value", "parameter", "expect"}),
984 : std::vector<std::string>({
985 : "HcomGetRankSize",
986 : { group, strnlen(group, GROUP_NAME_MAX_LEN + 1) },
987 : "group",
988 : "a non-empty string of length 1 to " + std::to_string(GROUP_NAME_MAX_LEN) +
989 : ", containing only alphanumeric characters and underscores"
990 : }));
991 0 : CHK_PRT_RET(ret != HCCL_SUCCESS,
992 : HCCL_ERROR("[%s][%s]errNo[0x%016llx] group name is invalid",
993 : LOG_KEYWORDS_TASK_EXEC.c_str(), LOG_KEYWORDS_INVALID_ARGUMENT.c_str(), HCOM_ERROR_CODE(ret)), ret);
994 :
995 : #if (!defined (HCCD)) && (!defined (CCL_KERNEL_AICPU))
996 0 : HCCLV2_FUNC_RUN(HcomGetRankSizeV2(group, rankSize));
997 : #endif
998 0 : std::shared_ptr<hccl::hcclComm> hcclComm;
999 0 : if (group != nullptr && HcclGetCommHandle(group, hcclComm) == HCCL_SUCCESS) {
1000 0 : CHK_RET(hcclComm->GetRankSize(*rankSize));
1001 : } else {
1002 0 : std::string strGroup = (group == nullptr) ? HCCL_WORLD_GROUP : group;
1003 0 : ret = GetGroupRankInfo(strGroup.c_str(), RankInfoType::RANK_SIZE_IN_GROUP, 0, rankSize);
1004 0 : CHK_PRT_RET(ret != HCCL_SUCCESS, HCCL_ERROR("[Get][RankSize]errNo[0x%016llx] get group[%s] rank info error",
1005 : HCOM_ERROR_CODE(ret), strGroup.c_str()), ret);
1006 0 : HCCL_INFO("hcom get rank size success, group[%s]", strGroup.c_str());
1007 0 : }
1008 0 : return HCCL_SUCCESS;
1009 0 : }
1010 :
1011 0 : HcclResult HcomDestroyOneDevice(HcomInfo &hcomInfo)
1012 : {
1013 0 : HcclUs startut = TIME_NOW();
1014 :
1015 : /* 接口交互信息日志 */
1016 0 : HCCL_RUN_INFO("Entry-HcomDestroy:void");
1017 :
1018 : // 模型运行结束后hcom destroy时，将CheckInfo信息清空
1019 0 : RankConsistentcyChecker::GetInstance().ClearCheckInfo();
1020 :
1021 : // group资源在word group资源销毁之前进行销毁
1022 0 : hcomInfo.params.commConnections.agentConnection = nullptr;
1023 0 : hcomInfo.params.commConnections.serverConnections.clear();
1024 0 : hcomInfo.hcomGroupMap.clear();
1025 0 : std::unique_lock<std::mutex> backloggedGroupLock(hcomInfo.backloggedGroupLock);
1026 0 : hcomInfo.backloggedGroup.clear();
1027 0 : backloggedGroupLock.unlock();
1028 :
1029 0 : hcomInfo.rankTable.nicNames.clear();
1030 0 : hcomInfo.rankTable.rankList.clear();
1031 0 : g_segmentIdxMap.clear();
1032 0 : g_segmentSizeMap.clear();
1033 0 : hcomInfo.params.profilingMode = HcomProfilingMode::PROFILING_CLOSE;
1034 0 : hcomInfo.params.profilingOption = "";
1035 0 : hcomInfo.isHcomInit = false;
1036 :
1037 0 : if (hcomInfo.params.deviceType != DevType::DEV_TYPE_NOSOC) {
1038 0 : ProfilingManagerPub::ClearStoragedProfilingInfo();
1039 : }
1040 :
1041 : /* 关键状态记录 */
1042 0 : HCCL_USER_CRITICAL_LOG("hcom destroy complete,take time [%lld]us, group[%s], rankNum[%u], rank[%u]",
1043 : DURATION_US(TIME_NOW() - startut), hcomInfo.params.identifier.c_str(), hcomInfo.rankTable.rankNum, hcomInfo.params.rank);
1044 :
1045 0 : return HCCL_SUCCESS;
1046 0 : }
1047 :
1048 0 : HcclResult HcomDestroy(void)
1049 : {
1050 : #if (!defined (HCCD)) && (!defined (CCL_KERNEL_AICPU))
1051 0 : HCCLV2_FUNC_RUN(
1052 : [&]() -> HcclResult {
1053 : std::unique_lock<std::mutex> lock(g_destroyDeviceLock);
1054 : CHK_RET(HcomDestroyV2());
1055 : for (u32 i = 0; i <= MAX_MODULE_DEVICE_NUM; i++) {
1056 : HcomInfo &hcomInfo = HcomGetCtxHomInfoById(i);
1057 : hcomInfo.pComm = nullptr;
1058 : hcomInfo.hcomGroupMap.clear();
1059 : }
1060 : return HCCL_SUCCESS;
1061 : }());
1062 : #endif
1063 0 : std::unique_lock<std::mutex> lock(g_destroyDeviceLock);
1064 0 : for (u32 i = 0; i <= MAX_MODULE_DEVICE_NUM; i++) {
1065 0 : HcomInfo &hcomInfo = HcomGetCtxHomInfoById(i);
1066 :
1067 0 : if (!hcomInfo.isHcomInit) {
1068 0 : if (hcomInfo.pComm != nullptr) {
1069 0 : hcomInfo.pComm = nullptr;
1070 : }
1071 0 : HCCL_INFO("[Destroy][Result]hcomInfo[%u].isHcomInit is false.", i);
1072 :
1073 : /* 接口交互信息日志 */
1074 0 : HCCL_INFO("Entry-HcomDestroy:void skip");
1075 0 : if (g_hcomDestroyCallback != nullptr) {
1076 0 : (void)g_hcomDestroyCallback(hcomInfo);
1077 : }
1078 0 : continue;
1079 : } else {
1080 0 : if (hcomInfo.pComm != nullptr) {
1081 0 : HcomUnSetGroupTopoInfo(hcomInfo.pComm->GetIdentifier().c_str());
1082 : }
1083 : }
1084 :
1085 0 : if (hcomInfo.pComm == nullptr &&
1086 0 : ((g_hcomCallBackGroupIsInit != nullptr) && (!(g_hcomCallBackGroupIsInit(hcomInfo))))) {
1087 0 : HCCL_INFO("[Destroy][Result]hcomInfo[%u].pComm or pCommBase is nullptr.", i);
1088 0 : continue;
1089 : }
1090 :
1091 0 : if (hcomInfo.params.logicDevId != HOST_DEVICE_ID) {
1092 0 : u32 logicId = hcomInfo.params.logicDevId;
1093 0 : if (hcomInfo.rankTable.version.compare(HETEROG_CLUSTER_VERSION) == 0) {
1094 0 : CHK_RET(hrtGetDeviceIndexByPhyId(hcomInfo.params.logicDevId, logicId));
1095 : }
1096 0 : s32 deviceId = 0;
1097 0 : if (hrtGetDevice(&deviceId) != HCCL_SUCCESS) {
1098 0 : CHK_RET(hrtSetDevice(logicId));
1099 0 : HCCL_INFO("[HcomDestroy][SetDeviceId]logicDevId[%u]", logicId);
1100 : }
1101 : }
1102 :
1103 0 : HCCL_INFO("[Destroy][Result]hcomInfo[%u].pComm destroy.", i);
1104 0 : HcclResult ret = HcomDestroyOneDevice(hcomInfo);
1105 0 : if (ret == HCCL_SUCCESS) {
1106 0 : HCCL_INFO("[Destroy][Result]hcomInfo[%u].pComm HcomDestroyOneDevice success.", i);
1107 : } else {
1108 0 : HCCL_INFO("[Destroy][Result]hcomInfo[%u].pComm HcomDestroyOneDevice fail.", i);
1109 0 : return ret;
1110 : }
1111 :
1112 0 : if (g_hcomDestroyCallback != nullptr) {
1113 0 : (void)g_hcomDestroyCallback(hcomInfo);
1114 : }
1115 :
1116 0 : hcomInfo.pComm = nullptr;
1117 0 : hcomInfo.hcclCommTopoInfoDetectServer.clear();
1118 0 : hcomInfo.hcclCommTopoInfoDetectAgent.clear();
1119 : }
1120 0 : return HCCL_SUCCESS;
1121 0 : }
1122 :
1123 41 : void HcomGroupCallbackFuncInstall(HcclResult (*p1)(const std::string &, const std::vector<u32> &),
1124 : bool (*p2)(HcomInfo &), HcclResult (*p3)(const std::string &), HcclResult (*p4)(HcomInfo &))
1125 : {
1126 41 : g_hcomCreateGroupCallback = p1;
1127 41 : g_hcomCallBackGroupIsInit = p2;
1128 41 : g_hcomDestroyGroupCallback = p3;
1129 41 : g_hcomDestroyCallback = p4;
1130 41 : }
1131 :
1132 0 : HcclResult HcomSetGradFusionByIndex(const char *group, u32 segmentNum, const u32 *IdxList)
1133 : {
1134 0 : bool &isAutoTuneModeOpen = HcomGetCtxAutoTuneMode();
1135 0 : if (isAutoTuneModeOpen) {
1136 0 : return HCCL_SUCCESS;
1137 : }
1138 :
1139 0 : RPT_INPUT_ERR(IdxList == nullptr,
1140 : "EI0003",
1141 : std::vector<std::string>({"ccl_op", "value", "parameter", "expect"}),
1142 : std::vector<std::string>({"HcomSetGradFusionByIndex", "nullptr", "IdxList", "non-null pointer"}));
1143 0 : CHK_PTR_NULL(IdxList);
1144 0 : bool bRet = segmentNum == 0;
1145 0 : RPT_INPUT_ERR(bRet,
1146 : "EI0003",
1147 : std::vector<std::string>({"ccl_op", "value", "parameter", "expect"}),
1148 : std::vector<std::string>(
1149 : {"HcomSetGradFusionByIndex", std::to_string(0), "segmentNum", "must be a positive integer (greater than 0)"}));
1150 0 : CHK_PRT_RET(bRet,
1151 : HCCL_ERROR("[%s][%s]errNo[0x%016llx] set split IdxList length is zero",
1152 : LOG_KEYWORDS_TASK_EXEC.c_str(),
1153 : LOG_KEYWORDS_INVALID_ARGUMENT.c_str(),
1154 : HCOM_ERROR_CODE(HCCL_E_PARA)),
1155 : HCCL_E_PARA);
1156 0 : std::string strGroup = (group == nullptr) ? HCCL_WORLD_GROUP : group;
1157 0 : string idxList;
1158 0 : for (u32 i = 0; i < segmentNum; i++) {
1159 0 : if (i < segmentNum - 1) {
1160 0 : idxList += to_string(IdxList[i]) + ',';
1161 0 : } else if (i == segmentNum - 1) {
1162 0 : idxList += to_string(IdxList[i]);
1163 : }
1164 : }
1165 : /* 接口交互信息日志 */
1166 0 : HCCL_RUN_INFO("Entry-HcomSetGradFusionByIndex:group[%s], segmentNum[%u], IdxList[%s]",
1167 : strGroup.c_str(), segmentNum, idxList.c_str());
1168 :
1169 0 : CHK_RET(HcomCheckGroupName(strGroup.c_str()));
1170 :
1171 0 : std::vector<u32> tempList;
1172 :
1173 0 : for (u32 segidx = 0; segidx < segmentNum; segidx++) {
1174 0 : tempList.push_back(IdxList[segidx]);
1175 : }
1176 :
1177 0 : for (u32 i = 0; i < tempList.size() - 1; i++) {
1178 0 : if (tempList[i] >= tempList[i + 1]) {
1179 0 : HCCL_ERROR("[Set][GradFusionByIndex]errNo[0x%016llx] index list is not ascending",
1180 : HCOM_ERROR_CODE(HCCL_E_PARA));
1181 0 : return HCCL_E_PARA;
1182 : }
1183 : }
1184 0 : std::unique_lock<std::mutex> segmentIdxMapLock(g_segmentIdxMapLock);
1185 0 : g_segmentIdxMap.insert(std::pair<std::string, std::vector<u32>>(strGroup, tempList));
1186 0 : segmentIdxMapLock.unlock();
1187 0 : return HCCL_SUCCESS;
1188 0 : }
1189 :
1190 0 : HcclResult HcomSetGradFusionBySize(const char *group, u32 segmentNum, const float *sizeList)
1191 : {
1192 0 : bool &isAutoTuneModeOpen = HcomGetCtxAutoTuneMode();
1193 0 : if (isAutoTuneModeOpen) {
1194 0 : return HCCL_SUCCESS;
1195 : }
1196 :
1197 0 : RPT_INPUT_ERR(sizeList == nullptr, "EI0003", std::vector<std::string>({"ccl_op", "value", "parameter", "expect"}),\
1198 : std::vector<std::string>({"HcomSetGradFusionBySize", "nullptr", "sizeList", "non-null pointer"}));
1199 0 : CHK_PTR_NULL(sizeList);
1200 0 : bool bRet = segmentNum == 0;
1201 0 : RPT_INPUT_ERR(bRet, "EI0003", std::vector<std::string>({"ccl_op", "value", "parameter", "expect"}),
1202 : std::vector<std::string>({
1203 : "HcomSetGradFusionBySize",
1204 : std::to_string(0),
1205 : "segmentNum",
1206 : "must be a positive integer (greater than 0)"
1207 : }
1208 : ));
1209 0 : CHK_PRT_RET(bRet, HCCL_ERROR("[%s][%s]errNo[0x%016llx] set split sizeList length is zero",
1210 : LOG_KEYWORDS_TASK_EXEC.c_str(), LOG_KEYWORDS_INVALID_ARGUMENT.c_str(), HCOM_ERROR_CODE(HCCL_E_PARA)), HCCL_E_PARA);
1211 0 : std::string strGroup = (group == nullptr) ? HCCL_WORLD_GROUP : group;
1212 0 : string strSizeList;
1213 0 : for (u32 i = 0; i < segmentNum; i++) {
1214 0 : if (i < segmentNum - 1) {
1215 0 : strSizeList += to_string(sizeList[i]) + ',';
1216 0 : } else if (i == segmentNum - 1) {
1217 0 : strSizeList += to_string(sizeList[i]);
1218 : }
1219 : }
1220 : /* 接口交互信息日志 */
1221 0 : HCCL_RUN_INFO("Entry-HcomSetGradFusionBySize:group[%s], segmentNum[%u], sizeList[%s]",
1222 : strGroup.c_str(), segmentNum, strSizeList.c_str());
1223 :
1224 0 : CHK_RET(HcomCheckGroupName(strGroup.c_str()));
1225 0 : std::vector<float> tempList;
1226 0 : float sizeTotal = 0;
1227 :
1228 0 : for (u32 sizeIdx = 0; sizeIdx < segmentNum; sizeIdx++) {
1229 0 : bRet = sizeList[sizeIdx] < 0;
1230 0 : CHK_PRT_RET(bRet, HCCL_ERROR("[Set][GradFusionBySize]errNo[0x%016llx] sizeList[%u] less than zero",
1231 : HCOM_ERROR_CODE(HCCL_E_PARA), sizeIdx), HCCL_E_PARA);
1232 0 : tempList.push_back(sizeList[sizeIdx]);
1233 0 : sizeTotal += sizeList[sizeIdx];
1234 : }
1235 :
1236 0 : if (std::fabs(sizeTotal - 100) > 1e-6) { // 判断用户设置总百分比是否为100%
1237 0 : HCCL_ERROR("[Set][GradFusionBySize]errNo[0x%016llx] size list sum is not 100%%", HCOM_ERROR_CODE(HCCL_E_PARA));
1238 0 : return HCCL_E_PARA;
1239 : } else {
1240 0 : std::unique_lock<std::mutex> segmentSizeMapLock(g_segmentSizeMapLock);
1241 0 : g_segmentSizeMap.insert(std::pair<std::string, std::vector<float>>(strGroup, tempList));
1242 0 : segmentSizeMapLock.unlock();
1243 0 : return HCCL_SUCCESS;
1244 0 : }
1245 0 : }
1246 :
1247 0 : HcclResult HcomGenerateCommId(hccl::HcclCommParams ¶ms)
1248 : {
1249 0 : s32 sRet = memset_s(params.id.internal, HCCL_ROOT_INFO_BYTES, 0, sizeof(params.id.internal));
1250 0 : CHK_PRT_RET(sRet != EOK, HCCL_ERROR("[GenerateCommId]memory set error. return[%d].", sRet), HCCL_E_PARA);
1251 :
1252 : HcclRootInfo uniqueId;
1253 0 : std::string group;
1254 0 : CHK_RET(hcclComm::GetUniqueId(&uniqueId));
1255 :
1256 0 : if (!params.isHeterogComm) {
1257 0 : group = "hccl_world_group";
1258 : } else {
1259 0 : group = "hccl_heterog_group";
1260 : }
1261 :
1262 0 : sRet = snprintf_s(params.id.internal, HCCL_ROOT_INFO_BYTES, HCCL_ROOT_INFO_BYTES - 1, "%s%s%s",
1263 : uniqueId.internal, "-", group.c_str());
1264 0 : CHK_PRT_RET(sRet == -1, HCCL_ERROR("[GenerateCommId]errNo[0x%016llx] sal snprintf_s error",
1265 : HCCL_ERROR_CODE(HCCL_E_INTERNAL)), HCCL_E_INTERNAL);
1266 0 : HCCL_INFO("params.id.internal [%s]", params.id.internal);
1267 0 : return HCCL_SUCCESS;
1268 0 : }
1269 :
1270 0 : HcclResult InitHcomMiscInfo(hccl::HcclCommParams ¶ms, const char *rankTable)
1271 : {
1272 0 : CHK_PTR_NULL(rankTable);
1273 :
1274 0 : RankConsistentcyChecker::GetInstance().SetCheckCannVersionSwitch(true); // 打开CANN软件版本校验开关
1275 :
1276 : // 记录版本信息
1277 0 : std::string curVersion = GetExternalInputCannVersion();
1278 0 : CHK_RET(RankConsistentcyChecker::GetInstance().RecordVerInfo(curVersion));
1279 0 : HcomInfo &hcomInfo = HcomGetCtxHomInfo();
1280 : // 计算rankTable的crc值并保存
1281 0 : HcclResult ret = HcomCalcCRC(params, rankTable);
1282 0 : CHK_PRT_RET(ret != HCCL_SUCCESS, HCCL_ERROR("[Init][OtherInfo]errNo[0x%016llx] calc ranktable crc error",
1283 : HCCL_ERROR_CODE(HCCL_E_INTERNAL)), HCCL_E_INTERNAL);
1284 : // 生成通信域标识符
1285 0 : ret = HcomGenerateCommId(hcomInfo.params);
1286 0 : CHK_PRT_RET(ret != HCCL_SUCCESS, HCCL_ERROR("[Init][OtherInfo]errNo[0x%016llx] generate CommId error",
1287 : HCCL_ERROR_CODE(HCCL_E_INTERNAL)), HCCL_E_INTERNAL);
1288 0 : return HCCL_SUCCESS;
1289 0 : }
1290 :
1291 0 : bool HcomCheckrtMemcpyAddrAsync(const std::string& group)
1292 : {
1293 0 : float counterVaule = 1.0f;
1294 :
1295 : // 偏移拷贝的二级指针
1296 0 : void *deviceMemSrcLevel2 = nullptr;
1297 0 : void *deviceMemDstLevel2 = nullptr;
1298 : // 偏移拷贝的一级指针
1299 0 : void *deviceMemSrc = nullptr;
1300 0 : void *deviceMemDst = nullptr;
1301 :
1302 0 : auto deleter = [&deviceMemSrcLevel2, &deviceMemDstLevel2, &deviceMemSrc, &deviceMemDst](void *dst) {
1303 0 : if (dst != nullptr) {
1304 0 : CHK_PRT(hrtFree(dst));
1305 0 : if (dst == deviceMemSrcLevel2) {
1306 0 : deviceMemSrcLevel2 = nullptr;
1307 0 : } else if (dst == deviceMemDstLevel2) {
1308 0 : deviceMemDstLevel2 = nullptr;
1309 0 : } else if (dst == deviceMemSrc) {
1310 0 : deviceMemSrc = nullptr;
1311 0 : } else if (dst == deviceMemDst) {
1312 0 : deviceMemDst = nullptr;
1313 : }
1314 : }
1315 0 : };
1316 :
1317 0 : CHK_RET(hrtMalloc(&deviceMemSrcLevel2, sizeof(void*)));
1318 0 : unique_ptr<void, decltype(deleter)> deviceMemSrcLevel2Unique(deviceMemSrcLevel2, deleter);
1319 0 : CHK_RET(hrtMalloc(&deviceMemDstLevel2, sizeof(void*)));
1320 0 : unique_ptr<void, decltype(deleter)> deviceMemDstLevel2Unique(deviceMemDstLevel2, deleter);
1321 0 : CHK_RET(hrtMalloc(&deviceMemSrc, sizeof(float)));
1322 0 : unique_ptr<void, decltype(deleter)> deviceMemSrcUnique(deviceMemSrc, deleter);
1323 0 : CHK_RET(hrtMalloc(&deviceMemDst, sizeof(float)));
1324 0 : unique_ptr<void, decltype(deleter)> deviceMemDstUnique(deviceMemDst, deleter);
1325 :
1326 0 : CHK_RET(hrtMemSyncCopy(deviceMemDst, sizeof(float), &counterVaule,
1327 : sizeof(float), HcclRtMemcpyKind::HCCL_RT_MEMCPY_KIND_HOST_TO_DEVICE));
1328 0 : CHK_RET(hrtMemSyncCopy(deviceMemSrc, sizeof(float), &counterVaule,
1329 : sizeof(float), HcclRtMemcpyKind::HCCL_RT_MEMCPY_KIND_HOST_TO_DEVICE));
1330 :
1331 0 : CHK_RET(hrtMemSyncCopy(deviceMemDstLevel2, sizeof(void*), &deviceMemDst,
1332 : sizeof(void*), HcclRtMemcpyKind::HCCL_RT_MEMCPY_KIND_HOST_TO_DEVICE));
1333 0 : CHK_RET(hrtMemSyncCopy(deviceMemSrcLevel2, sizeof(void*), &deviceMemSrc,
1334 : sizeof(void*), HcclRtMemcpyKind::HCCL_RT_MEMCPY_KIND_HOST_TO_DEVICE));
1335 :
1336 0 : u64 destMax = sizeof(s32);
1337 0 : u64 offset = 0;
1338 :
1339 0 : Stream stream(StreamType::STREAM_TYPE_ONLINE);
1340 0 : bool notSupportSecAddrCopyWithOffset = false;
1341 :
1342 0 : HcclResult ret = hrtMemcpyAddrAsync(deviceMemDstLevel2, destMax, offset, deviceMemSrcLevel2,
1343 : destMax, offset, stream.ptr());
1344 0 : if (ret == HCCL_E_NOT_SUPPORT) {
1345 0 : notSupportSecAddrCopyWithOffset = true;
1346 : } else {
1347 0 : CHK_RET(hcclStreamSynchronize(stream.ptr(), CommConfiger::GetInstance().GetCommConfigExecTimeOut(group)));
1348 : }
1349 :
1350 0 : g_notSupportSecAddrCopyWithOffset = notSupportSecAddrCopyWithOffset;
1351 :
1352 0 : return notSupportSecAddrCopyWithOffset;
1353 0 : }
1354 :
1355 1 : bool HcomGetSecAddrCopyFlag(const char *socVersion)
1356 : {
1357 1 : HCCL_INFO("[Hcom][HcomGetSecAddrCopyFlag] SecAddrCopyWithOffset flag is %d", g_notSupportSecAddrCopyWithOffset);
1358 : DevType devType;
1359 1 : std::string socVersionStr(socVersion);
1360 1 : CHK_RET(hrtGetDeviceTypeBySocVersion(socVersionStr, devType));
1361 :
1362 1 : return !g_notSupportSecAddrCopyWithOffset && (devType == DevType::DEV_TYPE_910B || devType == DevType::DEV_TYPE_910);
1363 1 : }
1364 :
1365 0 : HcclResult HcomNormalInit(const char *rankTableM, const char *identify)
1366 : {
1367 0 : HcclResult ret = HCCL_SUCCESS;
1368 0 : bool &isAutoTuneModeOpen = HcomGetCtxAutoTuneMode();
1369 0 : isAutoTuneModeOpen = false;
1370 0 : HcomInfo &hcomInfo = HcomGetCtxHomInfo();
1371 :
1372 : /*--------------入参合法性检测---------------------*/
1373 0 : CHK_PTR_NULL(rankTableM);
1374 0 : CHK_PTR_NULL(identify);
1375 :
1376 : /* 防止重复调用初始化 */
1377 0 : CHK_PRT_RET((hcomInfo.pComm != nullptr),
1378 : HCCL_ERROR("[Init][Result]errNo[0x%016llx] identify[%s], "\
1379 : "multiple initialization is not supported", HCOM_ERROR_CODE(HCCL_E_UNAVAIL), identify), HCCL_E_UNAVAIL);
1380 :
1381 : /* --------------初始化------------------------- */
1382 0 : bool errorFlag = false;
1383 0 : s32 logicDevId = 0;
1384 0 : hcomInfo.params.commWorkMode = WorkMode::HCCL_MODE_NORMAL;
1385 : do {
1386 0 : ret = InitHcomMiscInfo(hcomInfo.params, rankTableM);
1387 0 : CHK_PRT_BREAK(ret != HCCL_SUCCESS, HCCL_ERROR("[Init][Result]errNo[0x%016llx] init other Info.",
1388 : HCOM_ERROR_CODE(ret)), errorFlag = true);
1389 :
1390 : DevType deviceType;
1391 0 : CHK_PRT_BREAK(hrtGetDevice(&logicDevId) != HCCL_SUCCESS, , errorFlag = true);
1392 0 : CHK_RET(hrtGetDeviceType(deviceType));
1393 : // 为适配12包，做此修改
1394 0 : g_notSupportSecAddrCopyWithOffset = HcomCheckrtMemcpyAddrAsync(identify);
1395 :
1396 0 : ret = CfgGetClusterInfo(rankTableM, identify, hcomInfo.params, hcomInfo.rankTable,
1397 0 : GetExternalInputInterSuperPodRetryEnable(), deviceType);
1398 0 : CHK_PRT_BREAK(ret != HCCL_SUCCESS, HCCL_ERROR("[Init][Result]errNo[0x%016llx] cfg get ranktable[%p] info "\
1399 : "error: identify[%s]", HCOM_ERROR_CODE(ret), rankTableM, identify), errorFlag = true);
1400 :
1401 0 : if (hcomInfo.rankTable.serverNum != SINGLE_SERVER_NUM &&
1402 0 : (deviceType == DevType::DEV_TYPE_310P3 || deviceType == DevType::DEV_TYPE_310P1)) {
1403 0 : CHK_RET(InitExternalInputHeterog());
1404 : }
1405 :
1406 0 : hcomInfo.pComm.reset(new (std::nothrow) hccl::hcclComm(0, 0, HCCL_WORLD_GROUP));
1407 :
1408 0 : CHK_PRT_RET(hcomInfo.pComm == nullptr, HCCL_ERROR("[Init][Result]hcomInfo.pComm is null,\
1409 : create failed"), HCCL_E_PTR);
1410 0 : CommConfig commConfig(identify);
1411 0 : ret = hcomInfo.pComm->init(hcomInfo.params, commConfig, hcomInfo.rankTable);
1412 0 : CHK_PRT_BREAK(ret != HCCL_SUCCESS,
1413 : HCCL_ERROR("[Init][Result]errNo[0x%016llx] hcclComm init error",
1414 : HCOM_ERROR_CODE(ret)), errorFlag = true);
1415 :
1416 0 : ret = ShowRanktableConfigInfo(hcomInfo.cloudFlag, hcomInfo.params, hcomInfo.rankTable);
1417 0 : CHK_PRT_BREAK(ret != HCCL_SUCCESS, HCCL_ERROR("[Init][Result]errNo[0x%016llx] put ranktable info error",
1418 : HCOM_ERROR_CODE(ret)), errorFlag = true);
1419 0 : ret = InitWorkflowMode(HcclWorkflowMode::HCCL_WORKFLOW_MODE_OPS_KERNEL_INFO_LIB);
1420 0 : CHK_PRT_BREAK(ret != HCCL_SUCCESS, HCCL_ERROR("[Init][Result]errNo[0x%016llx] init work flow mode error",
1421 : HCCL_ERROR_CODE(ret)), errorFlag = true);
1422 :
1423 0 : ret = HcomFlushBackloggedGroups();
1424 0 : CHK_PRT_BREAK(ret != HCCL_SUCCESS, HCCL_ERROR("[Init][Result]errNo[0x%016llx] create backlogged group failed",
1425 : HCOM_ERROR_CODE(ret)), errorFlag = true);
1426 :
1427 0 : ret = HcomSetGroupTopoInfo(hcomInfo.pComm->GetIdentifier().c_str(), hcomInfo.rankTable.rankNum);
1428 0 : CHK_PRT_BREAK(ret != HCCL_SUCCESS, HCCL_ERROR("[Init][Result]errNo[0x%016llx] SetGroupTopoInfo error, "\
1429 : "group[%s]", HCOM_ERROR_CODE(ret), (hcomInfo.pComm->GetIdentifier().c_str())), errorFlag = true);
1430 0 : } while (0);
1431 :
1432 0 : if (errorFlag) {
1433 0 : HCCL_ERROR("[Init][Result]hcom init failed, rankNum[%u], rank[%u], server[%s], device[%d], return[0x%016llx]",
1434 : hcomInfo.rankTable.rankNum, hcomInfo.params.rank, hcomInfo.params.serverId.c_str(),
1435 : logicDevId, HCOM_ERROR_CODE(ret));
1436 0 : (void)HcomDestroy();
1437 0 : return ret;
1438 : }
1439 0 : return HCCL_SUCCESS;
1440 : }
1441 :
1442 0 : HcclResult HcomCheckInitClusterInfo(const char *rankTableM, const char *identify)
1443 : {
1444 0 : HcclResult ret = HCCL_SUCCESS;
1445 : // rankTable合法性检测
1446 0 : u32 rankTableSize = 0;
1447 0 : ret = HcomCheckRankTable(rankTableM, rankTableSize);
1448 0 : CHK_PRT_RET(ret != HCCL_SUCCESS,
1449 : HCCL_ERROR("[%s][%s]errNo[0x%016llx] input rankTable error",
1450 : LOG_KEYWORDS_INIT_GROUP.c_str(),
1451 : LOG_KEYWORDS_INVALID_ARGUMENT.c_str(),
1452 : HCOM_ERROR_CODE(ret)),
1453 : ret);
1454 :
1455 : // identify合法性检测
1456 0 : ret = HcomCheckIdentify(identify);
1457 0 : RPT_INPUT_ERR(ret != HCCL_SUCCESS,
1458 : "EI0003",
1459 : std::vector<std::string>({"ccl_op", "value", "parameter", "expect"}),
1460 : std::vector<std::string>(
1461 : {"HcomInit", {identify, strnlen(identify, IDENTIFY_MAX_LEN + 1)}, "identify", "a valid node identifier"}));
1462 0 : CHK_PRT_RET(ret != HCCL_SUCCESS,
1463 : HCCL_ERROR("[%s][%s]errNo[0x%016llx] identify parameter error",
1464 : LOG_KEYWORDS_INIT_GROUP.c_str(),
1465 : LOG_KEYWORDS_INVALID_ARGUMENT.c_str(),
1466 : HCOM_ERROR_CODE(ret)),
1467 : ret);
1468 0 : return ret;
1469 0 : }
1470 :
1471 0 : HcclResult HcomInitByFile(const char *rankTablePath, const char *identify)
1472 : {
1473 0 : HcomInfo &hcomInfo = HcomGetCtxHomInfo();
1474 :
1475 0 : CHK_PTR_NULL(rankTablePath);
1476 0 : CHK_PTR_NULL(identify);
1477 : #if (!defined (HCCD)) && (!defined (CCL_KERNEL_AICPU))
1478 0 : HCCLV2_FUNC_RUN(
1479 : [&]() -> HcclResult {
1480 : CHK_RET(HcomInitByFileV2(rankTablePath, identify));
1481 : u32 rankNum = 0;
1482 : CHK_RET(HcomGetRankSize(HCCL_WORLD_GROUP, &rankNum));
1483 : s32 myRank = std::atoi(identify);
1484 : Hccl::RankId rank = static_cast<Hccl::RankId>(myRank);
1485 : void *commV2 = nullptr;
1486 : CHK_RET(HcomGetCommV2(&commV2));
1487 : CHK_RET(HcomInitCollComm(rank, &commV2, hcomInfo.pComm));
1488 : CHK_RET(HcomSetGroupTopoInfo(HCCL_WORLD_GROUP, rankNum));
1489 : return HCCL_SUCCESS;
1490 : }());
1491 : #endif
1492 :
1493 0 : HcclUs startut = TIME_NOW();
1494 0 : HcclResult ret = HCCL_SUCCESS;
1495 :
1496 : // 读取rankTable文件到内存
1497 0 : std::string rankTableM;
1498 0 : std::string realFilePath;
1499 0 : ret = HcomLoadRanktableFile(rankTablePath, rankTableM, realFilePath);
1500 0 : CHK_PRT_RET(ret != HCCL_SUCCESS,
1501 : HCCL_ERROR("[%s][%s]errNo[0x%016llx] rankTablePath[%s] identify[%s] load rankTable error.",
1502 : LOG_KEYWORDS_INIT_GROUP.c_str(),
1503 : LOG_KEYWORDS_RANKTABLE_CONFIG.c_str(),
1504 : HCCL_ERROR_CODE(HCCL_E_INTERNAL),
1505 : rankTablePath,
1506 : identify),
1507 : HCCL_E_INTERNAL);
1508 0 : CHK_RET(HcomCheckInitClusterInfo(rankTableM.c_str(), identify));
1509 0 : HCCL_RUN_INFO("Entry-HcomInitByFile:rankTablePath[%s], identify[%s]", realFilePath.c_str(), identify);
1510 :
1511 0 : CHK_RET(InitExternalInput());
1512 0 : CHK_RET(InitEnvConfig());
1513 :
1514 : // 调用初始化接口
1515 0 : ret = HcomNormalInit(rankTableM.c_str(), identify);
1516 0 : CHK_PRT_RET(ret != HCCL_SUCCESS, HCCL_ERROR("[HcomInitByFile]errNo[0x%016llx] rankTablePath[%s] identify[%s] "
1517 : "hcom init failed.", HCCL_ERROR_CODE(ret), realFilePath.c_str(), identify), ret);
1518 0 : hcomInfo.isHcomInit = true;
1519 : /* 关键状态记录 */
1520 0 : HCCL_RUN_INFO("[HCCL_TRACE]hcom init by file success,take time [%lld]us, rankTablePath[%s], rankNum[%u], rank[%u],"\
1521 : "server[%s], device[%d]", DURATION_US(TIME_NOW() - startut), realFilePath.c_str(),
1522 : hcomInfo.rankTable.rankNum, hcomInfo.params.rank, hcomInfo.params.serverId.c_str(),
1523 : hcomInfo.params.logicDevId);
1524 0 : return HCCL_SUCCESS;
1525 0 : }
1526 :
1527 3 : DevType HcomGetDeviceType()
1528 : {
1529 : DevType devType;
1530 3 : hrtGetDeviceType(devType);
1531 3 : if(devType == DevType::DEV_TYPE_950 || devType == DevType::DEV_TYPE_960){
1532 0 : HcomGetDevTypeV2(devType);
1533 0 : HCCL_INFO("LaunchHcomKernel: devType is DEV_TYPE_950 or DEV_TYPE_960");
1534 0 : return MakeEnumToDevType(static_cast<int>(devType));
1535 : }
1536 :
1537 3 : HcomInfo &hcomInfo = HcomGetCtxHomInfo();
1538 3 : return hcomInfo.params.deviceType;
1539 : }
1540 :
1541 1 : HcclResult HcomCreateCommCCLbuffer(const char *group)
1542 : {
1543 1 : RPT_INPUT_ERR(group == nullptr, "EI0003", std::vector<std::string>({"ccl_op", "value", "parameter", "expect"}),\
1544 : std::vector<std::string>({"HcomGetDevType", "nullptr", "group", "non-null pointer"}));
1545 1 : CHK_PTR_NULL(group);
1546 :
1547 1 : HcclResult ret = HcomCheckGroupName(group);
1548 1 : RPT_INPUT_ERR(ret != HCCL_SUCCESS,
1549 : "EI0003", std::vector<std::string>({"ccl_op", "value", "parameter", "expect"}),
1550 : std::vector<std::string>({
1551 : "HcomGetDevType",
1552 : { group, strnlen(group, GROUP_NAME_MAX_LEN + 1) },
1553 : "group",
1554 : "a non-empty string of length 1 to " + std::to_string(GROUP_NAME_MAX_LEN) +
1555 : ", containing only alphanumeric characters and underscores"
1556 : }));
1557 1 : CHK_PRT_RET(ret != HCCL_SUCCESS,
1558 : HCCL_ERROR("[Get][HcomGetDevType]errNo[0x%016llx] group name is invalid", HCOM_ERROR_CODE(ret)), ret);
1559 :
1560 1 : DevType devType = HcomGetDeviceType();
1561 1 : if(devType == DevType::DEV_TYPE_950 || devType == DevType::DEV_TYPE_960){
1562 0 : HCCL_INFO("HcomCreateCommCclBufV2 start.");
1563 : #if (!defined (HCCD)) && (!defined (CCL_KERNEL_AICPU))
1564 0 : HCCLV2_FUNC_RUN(HcomCreateCommCclBufV2(group));
1565 : #endif
1566 0 : return HCCL_SUCCESS;
1567 : }
1568 :
1569 1 : std::shared_ptr<hccl::hcclComm> hcclComm;
1570 1 : CHK_RET(HcomGetCommByGroup(group, hcclComm));
1571 1 : CHK_RET(hcclComm->CreateCommCCLbuffer());
1572 1 : return HCCL_SUCCESS;
1573 1 : }
1574 :
1575 1 : HcclResult HcomGetInCCLbuffer(const char *group, void** buffer, u64 *size)
1576 : {
1577 1 : RPT_INPUT_ERR(group == nullptr, "EI0003", std::vector<std::string>({"ccl_op", "value", "parameter", "expect"}),\
1578 : std::vector<std::string>({"HcomGetInCCLbuffer", "nullptr", "group", "non-null pointer"}));
1579 1 : CHK_PTR_NULL(group);
1580 1 : CHK_PTR_NULL(buffer);
1581 1 : CHK_PTR_NULL(size);
1582 :
1583 1 : HcclResult ret = HcomCheckGroupName(group);
1584 1 : RPT_INPUT_ERR(ret != HCCL_SUCCESS,
1585 : "EI0003", std::vector<std::string>({"ccl_op", "value", "parameter", "expect"}),
1586 : std::vector<std::string>({
1587 : "HcomGetInCCLbuffer",
1588 : { group, strnlen(group, GROUP_NAME_MAX_LEN + 1) },
1589 : "group",
1590 : "a non-empty string of length 1 to " + std::to_string(GROUP_NAME_MAX_LEN) +
1591 : ", containing only alphanumeric characters and underscores"
1592 : }));
1593 1 : CHK_PRT_RET(ret != HCCL_SUCCESS,
1594 : HCCL_ERROR("[Get][HcomGetInCCLbuffer]errNo[0x%016llx] group name is invalid", HCOM_ERROR_CODE(ret)), ret);
1595 :
1596 1 : DevType devType = HcomGetDeviceType();
1597 1 : if(devType == DevType::DEV_TYPE_950 || devType == DevType::DEV_TYPE_960){
1598 : #if (!defined (HCCD)) && (!defined (CCL_KERNEL_AICPU))
1599 0 : HCCLV2_FUNC_RUN(HcomGetInCclBufV2(group, *buffer, *size));
1600 : #endif
1601 0 : return HCCL_SUCCESS;
1602 : }
1603 :
1604 1 : std::shared_ptr<hccl::hcclComm> hcclComm;
1605 1 : CHK_RET(HcomGetCommByGroup(group, hcclComm));
1606 1 : CHK_RET(hcclComm->GetInCCLbuffer(*buffer, *size));
1607 1 : return HCCL_SUCCESS;
1608 1 : }
1609 :
1610 1 : HcclResult HcomGetOutCCLbuffer(const char *group, void** buffer, u64 *size)
1611 : {
1612 1 : RPT_INPUT_ERR(group == nullptr, "EI0003", std::vector<std::string>({"ccl_op", "value", "parameter", "expect"}),\
1613 : std::vector<std::string>({"HcomGetOutCCLbuffer", "nullptr", "group", "non-null pointer"}));
1614 1 : CHK_PTR_NULL(group);
1615 1 : CHK_PTR_NULL(buffer);
1616 1 : CHK_PTR_NULL(size);
1617 :
1618 1 : HcclResult ret = HcomCheckGroupName(group);
1619 1 : RPT_INPUT_ERR(ret != HCCL_SUCCESS,
1620 : "EI0003", std::vector<std::string>({"ccl_op", "value", "parameter", "expect"}),
1621 : std::vector<std::string>({
1622 : "HcomGetOutCCLbuffer",
1623 : { group, strnlen(group, GROUP_NAME_MAX_LEN + 1) },
1624 : "group",
1625 : "a non-empty string of length 1 to " + std::to_string(GROUP_NAME_MAX_LEN) +
1626 : ", containing only alphanumeric characters and underscores"
1627 : }));
1628 1 : CHK_PRT_RET(ret != HCCL_SUCCESS,
1629 : HCCL_ERROR("[Get][HcomGetOutCCLbuffer]errNo[0x%016llx] group name is invalid", HCOM_ERROR_CODE(ret)), ret);
1630 :
1631 1 : DevType devType = HcomGetDeviceType();
1632 1 : if(devType == DevType::DEV_TYPE_950 || devType == DevType::DEV_TYPE_960){
1633 : #if (!defined (HCCD)) && (!defined (CCL_KERNEL_AICPU))
1634 0 : HCCLV2_FUNC_RUN(HcomGetOutCclBufV2(group, *buffer, *size));
1635 : #endif
1636 0 : return HCCL_SUCCESS;
1637 : }
1638 :
1639 1 : std::shared_ptr<hccl::hcclComm> hcclComm;
1640 1 : CHK_RET(HcomGetCommByGroup(group, hcclComm));
1641 1 : CHK_RET(hcclComm->GetOutCCLbuffer(*buffer, *size));
1642 1 : return HCCL_SUCCESS;
1643 1 : }
1644 :
1645 1 : HcclResult HcomGetAicpuOpStreamNotify(const char *group, HcclRtStream *opStream, u8 aicpuNotifyNum, void** aicpuNotify)
1646 : {
1647 1 : RPT_INPUT_ERR(group == nullptr, "EI0003", std::vector<std::string>({"ccl_op", "value", "parameter", "expect"}),\
1648 : std::vector<std::string>({"HcomGetDevType", "nullptr", "group", "non-null pointer"}));
1649 1 : CHK_PTR_NULL(group);
1650 1 : CHK_PTR_NULL(opStream);
1651 1 : CHK_PTR_NULL(aicpuNotify);
1652 :
1653 1 : HcclResult ret = HcomCheckGroupName(group);
1654 1 : RPT_INPUT_ERR(ret != HCCL_SUCCESS,
1655 : "EI0003", std::vector<std::string>({"ccl_op", "value", "parameter", "expect"}),
1656 : std::vector<std::string>({
1657 : "HcomGetDevType",
1658 : { group, strnlen(group, GROUP_NAME_MAX_LEN + 1) },
1659 : "group",
1660 : "a non-empty string of length 1 to " + std::to_string(GROUP_NAME_MAX_LEN) +
1661 : ", containing only alphanumeric characters and underscores"
1662 : }));
1663 1 : CHK_PRT_RET(ret != HCCL_SUCCESS,
1664 : HCCL_ERROR("[Get][HcomGetDevType]errNo[0x%016llx] group name is invalid", HCOM_ERROR_CODE(ret)), ret);
1665 :
1666 1 : std::shared_ptr<hccl::hcclComm> hcclComm;
1667 1 : CHK_RET(HcomGetCommByGroup(group, hcclComm));
1668 1 : CHK_RET(hcclComm->GetAicpuOpStreamNotify(opStream, aicpuNotifyNum, aicpuNotify));
1669 1 : return HCCL_SUCCESS;
1670 1 : }
1671 :
1672 1 : HcclResult HcomMc2AiCpuStreamAllocAndGet(const char *group, u32 streamMode, rtStream_t *aiCpuStream)
1673 : {
1674 1 : RPT_INPUT_ERR(group == nullptr, "EI0003", std::vector<std::string>({"ccl_op", "value", "parameter", "expect"}),\
1675 : std::vector<std::string>({"HcomGetDevType", "nullptr", "group", "non-null pointer"}));
1676 1 : CHK_PTR_NULL(group);
1677 1 : CHK_PTR_NULL(aiCpuStream);
1678 :
1679 1 : HcclResult ret = HcomCheckGroupName(group);
1680 1 : RPT_INPUT_ERR(ret != HCCL_SUCCESS,
1681 : "EI0003", std::vector<std::string>({"ccl_op", "value", "parameter", "expect"}),
1682 : std::vector<std::string>({
1683 : "HcomGetDevType",
1684 : { group, strnlen(group, GROUP_NAME_MAX_LEN + 1) },
1685 : "group",
1686 : "a non-empty string of length 1 to " + std::to_string(GROUP_NAME_MAX_LEN) +
1687 : ", containing only alphanumeric characters and underscores"
1688 : }));
1689 1 : CHK_PRT_RET(ret != HCCL_SUCCESS,
1690 : HCCL_ERROR("[Get][HcomGetDevType]errNo[0x%016llx] group name is invalid", HCOM_ERROR_CODE(ret)), ret);
1691 :
1692 : #if (!defined (HCCD)) && (!defined (CCL_KERNEL_AICPU))
1693 1 : HCCLV2_FUNC_RUN(HcomMc2AiCpuStreamAllocAndGetV2(group, streamMode, aiCpuStream));
1694 : #endif
1695 1 : std::shared_ptr<hccl::hcclComm> hcclComm;
1696 1 : ret = HcomGetCommByGroup(group, hcclComm);
1697 : // 兼容V2，获取通信域失败由外层判断，此处不报ERROR
1698 1 : CHK_PRT_RET(ret != HCCL_SUCCESS, HCCL_WARNING("[%s] HcomGetCommByGroup fail", __func__), ret);
1699 1 : CHK_RET(hcclComm->Mc2AiCpuStreamAllocAndGet(streamMode, *aiCpuStream));
1700 1 : return HCCL_SUCCESS;
1701 1 : }
|
