Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include <algorithm>
12 : #include <list>
13 : #include <vector>
14 : #include <string>
15 : #include <securec.h>
16 : #include <hccl/hccl_types.h>
17 : // ltm指定config路径
18 : #include "common/src/config.h"
19 : #include "hccl/base.h"
20 : #include "param_check_pub.h"
21 : #include "remote_access.h"
22 : #include "../op_base/src/op_base.h"
23 : #include "hccl/hcom.h"
24 : #include "rank_consistentcy_checker.h"
25 : #include "profiling_manager_pub.h"
26 : #include "topoinfo_ranktableParser_pub.h"
27 : #include "stream_pub.h"
28 : #include "hcom_common.h"
29 : #include "comm_configer.h"
30 : #include "hcom_private_v2.h"
31 : #if (!defined(HCCD)) && (!defined(CCL_KERNEL_AICPU))
32 : #include "coll_comm_mgr.h"
33 : #endif
34 :
35 : #include "hcom_pub.h"
36 :
37 : using namespace std;
38 : using namespace hccl;
39 :
40 : // DEV_TYPE_V80 对应 DevType::DEV_TYPE_V80
41 : // DEV_TYPE_V51_310_P3 对应 DevType::DEV_TYPE_310P3
42 : // DEV_TYPE_V71 对应 DevType::DEV_TYPE_V71
43 : // DEV_TYPE_V51_310_P1 对应 DevType::DEV_TYPE_310P1
44 : // DEV_TYPE_V81 对应 DevType::DEV_TYPE_V81
45 : // DEV_TYPE_950 对应 DevType::DEV_TYPE_950
46 : // DEV_TYPE_960 对应 DevType::DEV_TYPE_960
47 : // DEV_TYPE_NOSOC 对应 DevType::DEV_TYPE_NOSOC
48 :
49 0 : DevType MakeEnumToDevType(int makeEnum)
50 : {
51 : // 正向映射：MAKE_ENUM到DevType
52 : static std::map<int, DevType> makeEnumToDevType
53 : = {{0, DevType::DEV_TYPE_910}, {1, DevType::DEV_TYPE_310P3}, {2, DevType::DEV_TYPE_910B},
54 : {3, DevType::DEV_TYPE_310P1}, {4, DevType::DEV_TYPE_910_93}, {5, DevType::DEV_TYPE_950},
55 0 : {6, DevType::DEV_TYPE_960}, {7, DevType::DEV_TYPE_NOSOC}};
56 :
57 0 : auto it = makeEnumToDevType.find(makeEnum);
58 0 : if (it != makeEnumToDevType.end()) {
59 0 : return it->second;
60 : } else {
61 0 : HCCL_WARNING("Invalid MAKE_ENUM value");
62 : }
63 0 : return DevType::DEV_TYPE_NOSOC;
64 : }
65 :
66 : using HcomCreateGroupCallback = HcclResult (*)(const std::string&, const std::vector<u32>&);
67 : using HcomCallBackGroupIsInit = bool (*)(HcomInfo&);
68 : using HcomDestroyGroupCallback = HcclResult (*)(const std::string&);
69 : using HcomDestroyCallback = HcclResult (*)(HcomInfo&);
70 : HcomCreateGroupCallback g_hcomCreateGroupCallback = nullptr;
71 : HcomCallBackGroupIsInit g_hcomCallBackGroupIsInit = nullptr;
72 : HcomDestroyGroupCallback g_hcomDestroyGroupCallback = nullptr;
73 : HcomDestroyCallback g_hcomDestroyCallback = nullptr;
74 :
75 : using HcomSetGroupTopoInfoPtr = HcclResult (*)(const char*, uint32_t);
76 : using HcomUnsetGroupTopoInfoPtr = void (*)(const char*);
77 : HcomSetGroupTopoInfoPtr g_hcomSetGroupTopoInfo = nullptr;
78 : HcomUnsetGroupTopoInfoPtr g_hcomUnsetGroupTopoInfo = nullptr;
79 :
80 : using HcomInfoCtx = struct HcomInfoCtxTag {
81 : HcomInfo hcomInfo;
82 : shared_ptr<RemoteAccess> remoteAccess;
83 : vector<MemRegisterAddr> remoteAddrInfos;
84 : HcomOpTagInfo opTagInfo;
85 : bool isUsed;
86 :
87 10296 : HcomInfoCtxTag() : remoteAccess(nullptr), remoteAddrInfos(0), isUsed(false) {}
88 : };
89 :
90 : // 梯度切分相关的全局变量
91 : namespace hccl {
92 : std::map<std::string, std::vector<u32>> g_segmentIdxMap;
93 : std::map<std::string, std::vector<float>> g_segmentSizeMap;
94 : std::mutex g_segmentIdxMapLock;
95 : std::mutex g_segmentSizeMapLock;
96 : std::mutex g_setTaskNumCalModeLock;
97 : } // namespace hccl
98 :
99 : std::mutex g_hcomInfoCtxMutex;
100 : HcomInfoCtx g_hcomInfoCtx[MAX_MODULE_DEVICE_NUM + 1];
101 :
102 : std::mutex g_backloggedGroupLock;
103 : std::map<std::string, std::vector<u32>> g_backloggedGroup; // 待创建的group
104 :
105 : std::mutex g_destroyDeviceLock;
106 : static std::mutex g_taskNumCalModeMutex;
107 :
108 : static bool g_isAutoTuneModeOpen = false;
109 : static bool g_notSupportSecAddrCopyWithOffset = false;
110 :
111 716 : HcomInfoCtx& HcomGetCurHcomCtx(void)
112 : {
113 716 : std::lock_guard<std::mutex> lock(g_hcomInfoCtxMutex);
114 716 : s32 deviceLogicId = INVALID_INT;
115 716 : if (hrtGetDevice(&deviceLogicId) == HCCL_SUCCESS && (static_cast<u32>(deviceLogicId) < MAX_MODULE_DEVICE_NUM)) {
116 716 : HCCL_INFO("[HcomGetCurHcomCtx] hrtGetDevice deviceLogicId[%d] ", deviceLogicId);
117 : /* 当前线程获取到deviceId, 如果是首次使用该deviceId的Ctx, 先判断之前是否已经配置过Ctx */
118 716 : if (!g_hcomInfoCtx[deviceLogicId].isUsed) {
119 7 : HCCL_INFO("[HcomGetCurHcomCtx] is no Used deviceLogicId[%d] ", deviceLogicId);
120 7 : if (g_hcomInfoCtx[MAX_MODULE_DEVICE_NUM].isUsed) {
121 0 : return g_hcomInfoCtx[MAX_MODULE_DEVICE_NUM];
122 : }
123 : }
124 716 : g_hcomInfoCtx[deviceLogicId].isUsed = true;
125 716 : return g_hcomInfoCtx[deviceLogicId];
126 : }
127 :
128 : /* 当前线程没有获取到deviceId, 查找是否有使用过的Ctx */
129 0 : for (u32 i = 0; i <= MAX_MODULE_DEVICE_NUM; i++) {
130 0 : if (g_hcomInfoCtx[i].isUsed) {
131 0 : HCCL_INFO("[HcomGetCurHcomCtx] no set device Used deviceLogicId[%u] ", i);
132 0 : return g_hcomInfoCtx[i];
133 : }
134 : }
135 :
136 : /* 当前线程没有获取到deviceId, 使用兜底Ctx */
137 0 : HCCL_INFO("[HcomGetCurHcomCtx] use cover bottom hcomInfoCtx");
138 0 : g_hcomInfoCtx[MAX_MODULE_DEVICE_NUM].isUsed = true;
139 0 : return g_hcomInfoCtx[MAX_MODULE_DEVICE_NUM];
140 716 : }
141 :
142 0 : HcomInfo& HcomGetCtxHomInfoById(u32 idx) { return g_hcomInfoCtx[idx].hcomInfo; }
143 :
144 716 : HcomInfo& HcomGetCtxHomInfo(void)
145 : {
146 716 : HcomInfoCtx& curHcomCtx = HcomGetCurHcomCtx();
147 716 : return curHcomCtx.hcomInfo;
148 : }
149 :
150 0 : HcomOpTagInfo& HcomGetCtxOpTagInfo(void)
151 : {
152 0 : HcomInfoCtx& curHcomCtx = HcomGetCurHcomCtx();
153 0 : return curHcomCtx.opTagInfo;
154 : }
155 :
156 0 : bool& HcomGetCtxAutoTuneMode(void) { return g_isAutoTuneModeOpen; }
157 :
158 235 : HcclResult HcomSetGroupTopoInfo(const char* group, uint32_t rankSize)
159 : {
160 235 : if (group == nullptr) {
161 0 : HCCL_ERROR("[Hcom][HcomSetGroupTopoInfo] group is null, please check");
162 0 : return HCCL_E_PTR;
163 : }
164 235 : if (g_hcomSetGroupTopoInfo == nullptr) {
165 235 : HCCL_INFO("[Hcom][HcomSetGroupTopoInfo] g_hcomSetGroupTopoInfo is null");
166 235 : HcomInfo& hcomInfo = HcomGetCtxHomInfo();
167 235 : std::lock_guard<std::mutex> lock(hcomInfo.groupRankNumMapLock);
168 : // 1. 单算子流程下只记录，后续不做处理。
169 : // 2. 图模式流程下，后续会通过HcomTopoInfoFuncInstall函数调用回调函数，进行GroupTopoInfo的设置。
170 470 : hcomInfo.groupRankNumMap[std::string(group)] = rankSize;
171 235 : HCCL_RUN_INFO("[Hcom][HcomSetGroupTopoInfo] store groupRankNumMap, group:%s, rankNum:%u", group, rankSize);
172 235 : return HCCL_SUCCESS;
173 235 : }
174 0 : return g_hcomSetGroupTopoInfo(group, rankSize);
175 : }
176 :
177 2 : HcclResult HcomInitCollComm([[maybe_unused]] uint32_t rank, void** commV2, [[maybe_unused]] HcclCommPtr& comm)
178 : {
179 2 : CHK_PTR_NULL(commV2);
180 1 : HCCL_INFO("[HcomInitCollComm] CollComm init start.");
181 : #if (!defined(HCCD)) && (!defined(CCL_KERNEL_AICPU))
182 1 : HcclUs startut = TIME_NOW();
183 1 : char commName[ROOTINFO_INDENTIFIER_MAX_LENGTH] = {};
184 1 : CHK_RET(HcclGetCommNameV2(*commV2, commName));
185 : // 获取cclbuffer
186 1 : uintptr_t cclBufferAddr{0};
187 1 : std::size_t cclBufferSize{0};
188 1 : HcclMemType cclBufferMemType{HcclMemType::HCCL_MEM_TYPE_DEVICE};
189 1 : CHK_RET(HcclGetCclBuffer(*commV2, cclBufferAddr, cclBufferSize, cclBufferMemType));
190 : HcclMem cclBuffer;
191 1 : cclBuffer.size = static_cast<uint64_t>(cclBufferSize);
192 1 : cclBuffer.type = cclBufferMemType;
193 1 : cclBuffer.addr = reinterpret_cast<void*>(cclBufferAddr);
194 1 : EXCEPTION_CATCH(comm = make_shared<hccl::hcclComm>(cclBufferSize, cclBufferSize, commName), return HCCL_E_PTR);
195 1 : void* rankGraph = nullptr;
196 1 : CHK_RET(HcclGetRankGraphV2(commV2, &rankGraph));
197 1 : constexpr HcclCommConfig* config = nullptr;
198 3 : CHK_RET(comm->InitCollComm(*commV2, rankGraph, rank, cclBuffer, commName, config));
199 1 : HCCL_RUN_INFO("[%s] success, take time [%lld]us.", __func__, DURATION_US(TIME_NOW() - startut));
200 : #endif
201 1 : return HCCL_SUCCESS;
202 : }
203 :
204 236 : void HcomUnSetGroupTopoInfo(const char* group)
205 : {
206 236 : if (g_hcomUnsetGroupTopoInfo == nullptr) {
207 236 : HCCL_INFO("[Hcom][HcomUnSetGroupTopoInfo] g_hcomUnsetGroupTopoInfo is null, can not unset");
208 236 : return;
209 : }
210 0 : if (group == nullptr) {
211 0 : HCCL_INFO("[Hcom][HcomUnSetGroupTopoInfo] group is null, can not unset");
212 0 : return;
213 : }
214 0 : g_hcomUnsetGroupTopoInfo(group);
215 0 : return;
216 : }
217 :
218 5 : HcclResult HcomGetCommHandleByGroup([[maybe_unused]] const char* group, [[maybe_unused]] HcclComm* commHandle)
219 : {
220 : #if (!defined(HCCD)) && (!defined(CCL_KERNEL_AICPU))
221 5 : CHK_PTR_NULL(commHandle);
222 4 : CHK_PTR_NULL(group);
223 :
224 3 : std::shared_ptr<hcclComm> hcclComm;
225 3 : s32 deviceLogicId = 0;
226 3 : HcclResult ret = HcclDeviceRefresh(deviceLogicId);
227 3 : CHK_PRT_RET(
228 : ret != HCCL_SUCCESS,
229 : HCCL_ERROR(
230 : "[HcomGetCommHandleByGroup]HcclDeviceRefresh failed, group[%s], deviceLogicId[%d], errNo[0x%016llx]", group,
231 : deviceLogicId, HCOM_ERROR_CODE(ret)),
232 : ret);
233 :
234 : // MC2单算子和动态图下发性能优化，优先查询返回
235 3 : s32 devId = HcclGetThreadDeviceId();
236 3 : HcclOpInfoCtx& opBaseHcom = CollCommMgr::GetInstance().LegacyGetHcclExistDeviceOpInfoCtx(devId);
237 3 : std::unique_lock<std::mutex> lock(opBaseHcom.opGroupMapMutex);
238 6 : auto iter = opBaseHcom.opGroup2CommMap.find(std::string(group));
239 3 : if (iter != opBaseHcom.opGroup2CommMap.end()) {
240 2 : hcclComm = iter->second;
241 2 : CHK_PRT_RET(
242 : hcclComm == nullptr,
243 : HCCL_ERROR(
244 : "[HcomGetCommHandleByGroup]opBaseHcom.comm is null, group[%s], deviceLogicId[%d]", group,
245 : deviceLogicId),
246 : HCCL_E_PTR);
247 2 : *commHandle = static_cast<HcclComm>(hcclComm.get());
248 2 : return HCCL_SUCCESS;
249 : }
250 1 : lock.unlock();
251 :
252 1 : ret = HcomGetCommByGroup(group, hcclComm);
253 1 : CHK_PRT_RET(
254 : ret != HCCL_SUCCESS,
255 : HCCL_ERROR(
256 : "[HcomGetCommHandleByGroup]HcomGetCommByGroup failed, group[%s], deviceLogicId[%d], errNo[0x%016llx]",
257 : group, deviceLogicId, HCOM_ERROR_CODE(ret)),
258 : ret);
259 0 : *commHandle = static_cast<HcclComm>(hcclComm.get());
260 : #endif
261 0 : return HCCL_SUCCESS;
262 3 : }
263 :
264 2 : HcclResult HcomGetCommByGroup(const char* group, std::shared_ptr<hccl::hcclComm>& hcclComm)
265 : {
266 2 : HcomInfo& hcomInfo = HcomGetCtxHomInfo();
267 2 : HcclResult ret = HcomCheckGroupName(group);
268 2 : CHK_PRT_RET(
269 : ret != HCCL_SUCCESS,
270 : HCCL_ERROR(
271 : "[Get][CommByGroup]HcomCheckGroupName failed, group[%s], errNo[0x%016llx]", group, HCOM_ERROR_CODE(ret)),
272 : ret);
273 2 : std::string strGroup = (group == nullptr) ? HCCL_WORLD_GROUP : group;
274 2 : if (strGroup == HCCL_WORLD_GROUP) {
275 0 : CHK_PRT_RET(hcomInfo.pComm == nullptr, HCCL_WARNING("[Get][CommByGroup]hcomInfo.pComm is null"), HCCL_E_PTR);
276 0 : hcclComm = hcomInfo.pComm;
277 : } else {
278 2 : std::unique_lock<std::mutex> groupParaLock(hcomInfo.groupParamsLock);
279 2 : auto iter = hcomInfo.hcomGroupMap.find(strGroup);
280 2 : if (iter == hcomInfo.hcomGroupMap.end()) {
281 2 : ret = HcclGetCommHandle(strGroup.c_str(), hcclComm);
282 2 : CHK_PRT_RET(
283 : ret != HCCL_SUCCESS,
284 : HCCL_WARNING(
285 : "[Get][CommByGroup]errNo[0x%016llx] group[%s] "
286 : "does not exist",
287 : HCOM_ERROR_CODE(HCCL_E_NOT_FOUND), strGroup.c_str()),
288 : HCCL_E_NOT_FOUND);
289 : } else {
290 0 : hcclComm = (iter->second).pSubComm;
291 0 : CHK_PRT_RET(
292 : hcclComm == nullptr, HCCL_ERROR("[Get][CommByGroup] Get Comm is null, group[%s]", strGroup.c_str()),
293 : HCCL_E_PTR);
294 : }
295 1 : groupParaLock.unlock();
296 2 : }
297 :
298 1 : return HCCL_SUCCESS;
299 2 : }
300 :
301 0 : void HcomTopoInfoRegCallback(HcclResult (*p1)(const char*, uint32_t), void (*p2)(const char*))
302 : {
303 0 : g_hcomSetGroupTopoInfo = p1;
304 0 : g_hcomUnsetGroupTopoInfo = p2;
305 :
306 0 : HcomInfo& hcomInfo = HcomGetCtxHomInfo();
307 0 : std::unique_lock<std::mutex> lock(hcomInfo.groupRankNumMapLock);
308 0 : HCCL_RUN_INFO("[HcomTopoInfoRegCallback] size of groupRankNumMap is %zu", hcomInfo.groupRankNumMap.size());
309 0 : for (auto& item : hcomInfo.groupRankNumMap) {
310 0 : HCCL_RUN_INFO(
311 : "[HcomTopoInfoRegCallback] try to set topo info, group:%s, rankNum:%u", item.first.c_str(), item.second);
312 0 : HcclResult ret = HcomSetGroupTopoInfo(item.first.c_str(), item.second);
313 0 : if (ret != HCCL_SUCCESS) {
314 0 : HCCL_ERROR(
315 : "[HcomTopoInfoRegCallback][HcomSetGroupTopoInfo]Set Info failed. errNo[0x%016llx].",
316 : HCOM_ERROR_CODE(ret));
317 : }
318 : }
319 0 : }
320 :
321 : HcclResult HcomStoreBackloggedGroup(const std::string& group, const std::vector<u32>& groupRanks);
322 : HcclResult HcomQueryGroupRef(const char* group, u32& groupRef);
323 : HcclResult HcomDestroyBackloggedGroup(const std::string& group);
324 : HcclResult GetGroupRankInfo(const char* group, RankInfoType rankType, u32 inPara, u32* outPara);
325 :
326 0 : HcclResult GetRankList(u32 rankNum, const u32* rankIds, HcclGroupParams& params)
327 : {
328 0 : HcomInfo& hcomInfo = HcomGetCtxHomInfo();
329 0 : CHK_PTR_NULL(hcomInfo.pComm);
330 0 : std::vector<RankInfo_t> rankList;
331 0 : params.totalRanks = rankNum;
332 0 : params.worldRank = hcomInfo.params.rank;
333 0 : params.groupRank = INVALID_VALUE_RANKID;
334 0 : for (u32 i = 0; i < rankNum; i++) {
335 0 : params.groupRanks.push_back(rankIds[i]);
336 : }
337 :
338 0 : std::sort(params.groupRanks.begin(), params.groupRanks.end());
339 0 : if (params.groupRanks[rankNum - 1] >= hcomInfo.rankTable.rankNum) {
340 0 : HCCL_ERROR(
341 : "[get][RankList]errNo[0x%016llx] groupRanks[%u]:%u is invalid", HCOM_ERROR_CODE(HCCL_E_PARA), rankNum - 1,
342 : params.groupRanks[rankNum - 1]);
343 0 : return HCCL_E_PARA;
344 : }
345 0 : if (hcomInfo.rankTable.rankList.size() <= params.groupRanks[0]) {
346 0 : HCCL_ERROR(
347 : "[get][RankList]errNo[0x%016llx] groupRanks[0] is invalid:[%u]", HCOM_ERROR_CODE(HCCL_E_PARA),
348 : params.groupRanks[0]);
349 0 : return HCCL_E_PARA;
350 : }
351 : // groupRanks 个数已经校验非0
352 0 : std::string serverId = hcomInfo.rankTable.rankList[params.groupRanks[0]].serverId;
353 0 : u32 serverNum = 1; // severNum初始值应为1，代表groupId为0的serverId;
354 0 : RankInfo_t rankInfo;
355 0 : for (u32 i = 0; i < rankNum; i++) {
356 0 : rankInfo = hcomInfo.rankTable.rankList[params.groupRanks[i]];
357 : // 校验worldRankID
358 0 : if (rankInfo.rankId != params.groupRanks[i]) {
359 0 : HCCL_ERROR(
360 : "[get][RankList]errNo[0x%016llx] in rankList, worldRanks[%u] is invalid", HCOM_ERROR_CODE(HCCL_E_PARA),
361 : rankInfo.rankId);
362 0 : return HCCL_E_PARA;
363 : }
364 0 : if (params.groupRanks[i] == params.worldRank) {
365 0 : params.groupRank = i;
366 : }
367 0 : if (rankInfo.serverId != serverId) {
368 0 : serverNum++;
369 0 : serverId = rankInfo.serverId;
370 : }
371 0 : rankInfo.rankId = i; // 放入groupRankid
372 0 : rankList.push_back(rankInfo); // ranktable中的ranklist是以rankid的顺序排列的
373 0 : rankInfo.serverId = ""; // 释放前先指空字符串
374 : }
375 0 : params.serverNum = serverNum;
376 0 : bool isStandardCard = false;
377 0 : CHK_RET(hcomInfo.pComm->IsStandardCard(isStandardCard));
378 :
379 0 : if (!isStandardCard && hcomInfo.params.deviceType != DevType::DEV_TYPE_910B
380 0 : && hcomInfo.params.deviceType != DevType::DEV_TYPE_910_93) {
381 0 : CHK_RET(CheckRankTableConfigInfo(rankList, rankNum, serverNum));
382 : }
383 0 : return HCCL_SUCCESS;
384 0 : }
385 :
386 0 : HcclResult HcomCreateGroupImpl(const std::string& group, const std::vector<u32>& rankIds)
387 : {
388 0 : HcclUs startut = TIME_NOW();
389 0 : HcomInfo& hcomInfo = HcomGetCtxHomInfo();
390 0 : std::string rankId;
391 0 : for (u32 i = 0; i < rankIds.size(); i++) {
392 0 : if (i < rankIds.size() - 1) {
393 0 : rankId += to_string(rankIds[i]) + ',';
394 0 : } else if (i == rankIds.size() - 1) {
395 0 : rankId += to_string(rankIds[i]);
396 : }
397 : }
398 : /* 接口交互信息日志 */
399 0 : HCCL_RUN_INFO(
400 : "Entry-HcomCreateGroup:group[%s], rankNum[%u], rankIds[%s]", group.c_str(), rankIds.size(), rankId.c_str());
401 :
402 0 : if (hcomInfo.params.commWorkMode == HCCL_MODE_NORMAL) {
403 0 : CHK_PRT_RET(
404 : hcomInfo.pComm == nullptr,
405 : HCCL_ERROR("[Create][Group]hcomInfo.pComm is null, please check if the initialize process is called."),
406 : HCCL_E_PTR);
407 : } else {
408 0 : if (g_hcomCreateGroupCallback != nullptr) {
409 0 : return g_hcomCreateGroupCallback(group, rankIds);
410 : }
411 : }
412 :
413 0 : CHK_PRT_RET(
414 : hcomInfo.rankTable.rankList.empty(), HCCL_ERROR("[Create][Group]group[%s] rankList is empty", group.c_str()),
415 : HCCL_E_INTERNAL);
416 :
417 : /* 已经存在的group不允许再次创建 */
418 0 : std::unique_lock<std::mutex> groupParaLock(hcomInfo.groupParamsLock);
419 0 : if (hcomInfo.hcomGroupMap.find(group) != hcomInfo.hcomGroupMap.end()) {
420 0 : HCCL_ERROR(
421 : "[Create][Group]errNo[0x%016llx] group[%s] is already exist", HCOM_ERROR_CODE(HCCL_E_PARA), group.c_str());
422 0 : return HCCL_E_PARA;
423 : }
424 0 : groupParaLock.unlock();
425 :
426 0 : HcclGroupParams groupParamsTem;
427 0 : CHK_RET(GetRankList(rankIds.size(), rankIds.data(), groupParamsTem));
428 :
429 : // 如果是groupRank = INVALID_VALUE_RANKID，即本rank不参与create group
430 0 : if (groupParamsTem.groupRank == INVALID_VALUE_RANKID) {
431 0 : HCCL_ERROR(
432 : "[Create][Group]errNo[0x%016llx] confirm groupRank from worldRank[%u] error",
433 : HCOM_ERROR_CODE(HCCL_E_NOT_FOUND), hcomInfo.params.rank);
434 0 : return HCCL_E_NOT_FOUND;
435 : }
436 :
437 : /* 入参的正确性由HCCL确保 */
438 0 : CHK_RET(hcomInfo.pComm->CreateGroup(
439 : group, groupParamsTem.groupRank, hcomInfo.params.rank, groupParamsTem.groupRanks, groupParamsTem.pSubComm));
440 0 : CHK_SMART_PTR_NULL(groupParamsTem.pSubComm);
441 :
442 0 : groupParaLock.lock();
443 0 : hcomInfo.hcomGroupMap.insert(std::make_pair(group, groupParamsTem));
444 0 : groupParaLock.unlock();
445 :
446 0 : HCCL_RUN_INFO(
447 : "hcom create group[%s] success, take time [%lld]us", group.c_str(), DURATION_US(TIME_NOW() - startut));
448 0 : return HCCL_SUCCESS;
449 0 : }
450 :
451 0 : HcclResult HcomCreateGroup(const char* group, u32 rankNum, u32* rankIds)
452 : {
453 : /* 调优模式直接返回success */
454 0 : bool& isAutoTuneModeOpen = HcomGetCtxAutoTuneMode();
455 0 : if (isAutoTuneModeOpen) {
456 0 : return HCCL_SUCCESS;
457 : }
458 0 : RPT_INPUT_ERR(
459 : group == nullptr, "EI0003", std::vector<std::string>({"ccl_op", "value", "parameter", "expect"}),
460 : std::vector<std::string>({"HcomCreateGroup", "nullptr", "group", "non-null pointer"}));
461 0 : CHK_PTR_NULL(group);
462 0 : HcclResult ret = HcomCheckGroupName(group);
463 0 : RPT_INPUT_ERR(
464 : ret != HCCL_SUCCESS, "EI0003", std::vector<std::string>({"ccl_op", "value", "parameter", "value"}),
465 : std::vector<std::string>(
466 : {"HcomCreateGroup",
467 : {group, strnlen(group, GROUP_NAME_MAX_LEN + 1)},
468 : "group",
469 : "a non-empty string of length 1 to " + std::to_string(GROUP_NAME_MAX_LEN)
470 : + ", containing only alphanumeric characters and underscores"}));
471 0 : CHK_PRT_RET(
472 : ret != HCCL_SUCCESS,
473 : HCCL_ERROR(
474 : "[%s][%s]errNo[0x%016llx] group name is invalid", LOG_KEYWORDS_TASK_EXEC.c_str(),
475 : LOG_KEYWORDS_INVALID_ARGUMENT.c_str(), HCOM_ERROR_CODE(ret)),
476 : ret);
477 0 : CHK_PRT_RET(
478 : (strncmp(group, HCCL_WORLD_GROUP, sizeof(HCCL_WORLD_GROUP)) == 0),
479 : HCCL_ERROR(
480 : "[%s][%s]create group isn't support world group", LOG_KEYWORDS_TASK_EXEC.c_str(),
481 : LOG_KEYWORDS_INVALID_ARGUMENT.c_str()),
482 : HCCL_E_PARA);
483 :
484 0 : RPT_INPUT_ERR(
485 : rankIds == nullptr, "EI0003", std::vector<std::string>({"ccl_op", "value", "parameter", "expect"}),
486 : std::vector<std::string>({"HcomCreateGroup", "nullptr", "rankIds", "non-null pointer"}));
487 0 : CHK_PTR_NULL(rankIds);
488 :
489 0 : if (rankNum == 0) {
490 0 : RPT_INPUT_ERR(
491 : true, "EI0003", std::vector<std::string>({"ccl_op", "value", "parameter", "expect"}),
492 : std::vector<std::string>(
493 : {"HcomCreateGroup", std::to_string(rankNum), "rankNum",
494 : "must be a positive integer (greater than 0)"}));
495 0 : HCCL_ERROR(
496 : "[%s][%s]errNo[0x%016llx] group[%s] rankNum[%u] is invalid", LOG_KEYWORDS_TASK_EXEC.c_str(),
497 : LOG_KEYWORDS_INVALID_ARGUMENT.c_str(), HCOM_ERROR_CODE(HCCL_E_PARA), group, rankNum);
498 0 : return HCCL_E_PARA;
499 : }
500 : // 入参合法性校验 END
501 0 : std::vector<u32> ranks(rankIds, rankIds + rankNum);
502 0 : HcomInfo& hcomInfo = HcomGetCtxHomInfo();
503 : #if (!defined(HCCD)) && (!defined(CCL_KERNEL_AICPU))
504 0 : HCCLV2_FUNC_RUN([&]() -> HcclResult {
505 : CHK_RET(HcomCreateGroupImplV2(group, rankNum, ranks));
506 : HcclGroupParams groupParams{};
507 : void* commV2 = nullptr;
508 : CHK_RET(HcomGetGroupParamsV2(group, static_cast<void*>(&groupParams), &commV2));
509 : Hccl::RankId rank = static_cast<Hccl::RankId>(groupParams.groupRank);
510 : CHK_RET(HcomInitCollComm(rank, &commV2, groupParams.pSubComm));
511 : CHK_PTR_NULL(groupParams.pSubComm);
512 : std::unique_lock<std::mutex> groupParaLock(hcomInfo.groupParamsLock);
513 : hcomInfo.hcomGroupMap.insert(std::make_pair(group, groupParams));
514 : groupParaLock.unlock();
515 : CHK_RET(HcomSetGroupTopoInfo(group, rankNum));
516 : HCCL_INFO("[HcomCreateGroup] create group[%s] success.", group);
517 : return HCCL_SUCCESS;
518 : }());
519 : #endif
520 0 : if (hcomInfo.pComm == nullptr
521 0 : && ((g_hcomCallBackGroupIsInit != nullptr) && (!(g_hcomCallBackGroupIsInit(hcomInfo))))) {
522 0 : CHK_RET(HcomStoreBackloggedGroup(group, ranks));
523 : } else {
524 0 : CHK_RET(HcomCreateGroupImpl(group, ranks));
525 0 : CHK_RET(HcomSetGroupTopoInfo(group, rankNum));
526 : }
527 0 : return HCCL_SUCCESS;
528 0 : }
529 :
530 0 : HcclResult DestroyFlag(const char* group, bool flag)
531 : {
532 0 : HcomInfo& hcomInfo = HcomGetCtxHomInfo();
533 :
534 0 : if (group == nullptr) {
535 0 : return HCCL_SUCCESS; // 全局通信域不需要查询flag
536 : }
537 0 : std::string strGroup = group;
538 0 : std::unique_lock<std::mutex> groupParaLock(hcomInfo.groupParamsLock);
539 0 : auto iter = hcomInfo.hcomGroupMap.find(strGroup);
540 0 : if (iter == hcomInfo.hcomGroupMap.end()) {
541 0 : HCCL_ERROR(
542 : "[Get][CommByGroup]errNo[0x%016llx] group[%s] does not exist", HCOM_ERROR_CODE(HCCL_E_NOT_FOUND),
543 : strGroup.c_str());
544 0 : return HCCL_E_NOT_FOUND; // 不存在该服务器内相关dev的对应信息
545 : }
546 0 : iter->second.destroyFlag = flag;
547 0 : return HCCL_SUCCESS;
548 0 : }
549 :
550 0 : HcclResult QueryDestroyFlag(const char* group)
551 : {
552 0 : HcomInfo& hcomInfo = HcomGetCtxHomInfo();
553 0 : if (group == nullptr) {
554 0 : return HCCL_SUCCESS; // 全局通信域不需要查询flag
555 : }
556 0 : std::string strGroup = group;
557 0 : std::unique_lock<std::mutex> groupParaLock(hcomInfo.groupParamsLock);
558 0 : auto iter = hcomInfo.hcomGroupMap.find(strGroup);
559 0 : if (iter == hcomInfo.hcomGroupMap.end()) {
560 0 : HCCL_WARNING(
561 : "[Get][CommByGroup]errNo[0x%016llx] group[%s] does not exist", HCOM_ERROR_CODE(HCCL_E_AGAIN),
562 : strGroup.c_str());
563 0 : return HCCL_E_AGAIN; // 不存在该服务器内相关dev的对应信息
564 : }
565 0 : if (iter->second.destroyFlag) {
566 0 : return HCCL_E_AGAIN;
567 : }
568 0 : return HCCL_SUCCESS;
569 0 : }
570 :
571 0 : HcclResult HcomDestroyGroupImpl(const std::string& group)
572 : {
573 : /* 调优模式直接返回success */
574 0 : bool& isAutoTuneModeOpen = HcomGetCtxAutoTuneMode();
575 0 : if (isAutoTuneModeOpen) {
576 0 : return HCCL_SUCCESS;
577 : }
578 0 : HcomInfo& hcomInfo = HcomGetCtxHomInfo();
579 0 : CHK_PRT_RET(
580 : hcomInfo.pComm == nullptr,
581 : HCCL_ERROR("[Destroy][Group]hcomInfo.pComm is null, "
582 : "please check if the initialize process is called."),
583 : HCCL_E_PTR);
584 0 : if (hcomInfo.params.commWorkMode != HCCL_MODE_NORMAL) {
585 0 : if (g_hcomDestroyGroupCallback != nullptr) {
586 0 : return g_hcomDestroyGroupCallback(group);
587 : }
588 : }
589 :
590 : /* 接口交互信息日志 */
591 0 : HCCL_RUN_INFO("Entry-HcomDestroyGroup:group[%s]", group.c_str());
592 0 : CHK_RET(DestroyFlag(group.c_str(), true));
593 0 : u32 ref = 0;
594 0 : CHK_RET(HcomQueryGroupRef(group.c_str(), ref));
595 0 : while (ref != 0) {
596 0 : std::shared_ptr<hccl::hcclComm> hcclComm = nullptr;
597 0 : CHK_RET(HcomGetCommByGroup(group.c_str(), hcclComm));
598 0 : SaluSleep(ONE_HUNDRED_MICROSECOND_OF_USLEEP);
599 0 : CHK_RET(HcomQueryGroupRef(group.c_str(), ref));
600 0 : }
601 :
602 0 : std::unique_lock<std::mutex> groupParaLock(hcomInfo.groupParamsLock);
603 0 : auto iter = hcomInfo.hcomGroupMap.find(group);
604 0 : if (iter == hcomInfo.hcomGroupMap.end()) {
605 0 : HCCL_ERROR(
606 : "[Destroy][Group]errNo[0x%016llx] group[%s] does not exist", HCOM_ERROR_CODE(HCCL_E_PARA), group.c_str());
607 0 : return HCCL_E_PARA;
608 : }
609 :
610 0 : CHK_RET(hcomInfo.pComm->DestroyGroup(group));
611 :
612 0 : (iter->second).groupRanks.clear(); // 清除该服务器内相关group的对应信息
613 :
614 0 : hcomInfo.hcomGroupMap.erase(group);
615 0 : groupParaLock.unlock();
616 :
617 0 : HCCL_RUN_INFO("hcom destroy group[%s] success.", group.c_str());
618 0 : return HCCL_SUCCESS;
619 0 : }
620 :
621 0 : HcclResult HcomDestroyGroup(const char* group)
622 : {
623 : /* 调优模式直接返回success */
624 0 : bool& isAutoTuneModeOpen = HcomGetCtxAutoTuneMode();
625 0 : if (isAutoTuneModeOpen) {
626 0 : return HCCL_SUCCESS;
627 : }
628 :
629 0 : RPT_INPUT_ERR(
630 : group == nullptr, "EI0003", std::vector<std::string>({"ccl_op", "value", "parameter", "expect"}),
631 : std::vector<std::string>({"HcomDestroyGroup", "nullptr", "group", "non-null pointer"}));
632 0 : CHK_PTR_NULL(group);
633 0 : CHK_RET(HcomCheckGroupName(group));
634 :
635 0 : if (strncmp(group, HCCL_WORLD_GROUP, sizeof(HCCL_WORLD_GROUP)) == 0) {
636 0 : HCCL_ERROR(
637 : "[%s][%s]errNo[0x%016llx] destroy group is world group", LOG_KEYWORDS_TASK_EXEC.c_str(),
638 : LOG_KEYWORDS_INVALID_ARGUMENT.c_str(), HCOM_ERROR_CODE(HCCL_E_PARA));
639 0 : return HCCL_E_PARA;
640 : }
641 0 : HcomInfo& hcomInfo = HcomGetCtxHomInfo();
642 : #if (!defined(HCCD)) && (!defined(CCL_KERNEL_AICPU))
643 0 : HCCLV2_FUNC_RUN([&]() -> HcclResult {
644 : std::unique_lock<std::mutex> groupParaLock(hcomInfo.groupParamsLock);
645 : CHK_RET(HcomDestroyGroupImplV2(group));
646 : auto iter = hcomInfo.hcomGroupMap.find(group);
647 : if (iter == hcomInfo.hcomGroupMap.end()) {
648 : HCCL_ERROR(
649 : "[Destroy][Group]errNo[0x%016llx] group[%s] does not exist", HCOM_ERROR_CODE(HCCL_E_PARA), group);
650 : return HCCL_E_PARA;
651 : }
652 : hcomInfo.hcomGroupMap.erase(group);
653 : groupParaLock.unlock();
654 : return HCCL_SUCCESS;
655 : }());
656 : #endif
657 :
658 0 : if (hcomInfo.pComm == nullptr
659 0 : && ((g_hcomCallBackGroupIsInit != nullptr) && (!(g_hcomCallBackGroupIsInit(hcomInfo))))) {
660 0 : CHK_RET(HcomDestroyBackloggedGroup(group));
661 : } else {
662 0 : CHK_RET(HcomDestroyGroupImpl(group));
663 : }
664 :
665 0 : HcomUnSetGroupTopoInfo(group);
666 :
667 0 : std::unique_lock<std::mutex> lock(g_backloggedGroupLock);
668 0 : if (g_backloggedGroup.find(group) != hcomInfo.backloggedGroup.end()) {
669 0 : g_backloggedGroup.erase(group);
670 0 : HCCL_INFO("hcom delete g_backlogged group[%s] success.", group);
671 : }
672 0 : return HCCL_SUCCESS;
673 0 : }
674 :
675 0 : HcclResult HcomFlushBackloggedGroups()
676 : {
677 0 : HCCL_INFO("HcomFlushBackloggedGroups");
678 0 : HcomInfo& hcomInfo = HcomGetCtxHomInfo();
679 0 : std::unique_lock<std::mutex> backGroupParaLock(g_backloggedGroupLock);
680 : using ITER = map<string, std::vector<u32>>::iterator;
681 0 : for (ITER iter = g_backloggedGroup.begin(); iter != g_backloggedGroup.end();) {
682 0 : HCCL_INFO("HcomFlushBackloggedGroups[%s], rank[%u]", iter->first.c_str(), hcomInfo.params.rank);
683 0 : if (std::count(iter->second.begin(), iter->second.end(), hcomInfo.params.rank) > 0) {
684 0 : HCCL_INFO("HcomFlushBackloggedGroups[%s], rank[%u] success", iter->first.c_str(), hcomInfo.params.rank);
685 0 : hcomInfo.backloggedGroup.insert({iter->first, iter->second});
686 : }
687 0 : iter++;
688 : }
689 0 : backGroupParaLock.unlock();
690 :
691 0 : std::unique_lock<std::mutex> lock(hcomInfo.backloggedGroupLock);
692 0 : for (ITER iter = hcomInfo.backloggedGroup.begin(); iter != hcomInfo.backloggedGroup.end();) {
693 0 : CHK_RET(HcomCreateGroupImpl(iter->first, iter->second));
694 0 : hcomInfo.backloggedGroup.erase(iter++);
695 : }
696 0 : HCCL_INFO("HcomFlushBackloggedGroups success.");
697 0 : return HCCL_SUCCESS;
698 0 : }
699 :
700 0 : HcclResult HcomStoreBackloggedGroup(const std::string& group, const std::vector<u32>& groupRanks)
701 : {
702 0 : HcomInfo& hcomInfo = HcomGetCtxHomInfo();
703 : // 该线程最开始是否未设置deviceid并获取了ctx
704 0 : bool hcomUseDefaultCtx = (&hcomInfo == &(g_hcomInfoCtx[MAX_MODULE_DEVICE_NUM].hcomInfo));
705 0 : s32 deviceLogicId = INVALID_INT;
706 0 : if (hrtGetDevice(&deviceLogicId) != HCCL_SUCCESS || (static_cast<u32>(deviceLogicId) >= MAX_MODULE_DEVICE_NUM)
707 0 : || hcomUseDefaultCtx) {
708 0 : HCCL_INFO("[device not set]hcom store group[%s]", group.c_str());
709 0 : std::unique_lock<std::mutex> groupParaLock(g_backloggedGroupLock);
710 0 : if (g_backloggedGroup.find(group) != g_backloggedGroup.end()) {
711 0 : HCCL_INFO("[Store][BackloggedGroup]group[%s] already exists", group.c_str());
712 0 : if (g_backloggedGroup[group] == groupRanks) {
713 0 : HCCL_ERROR("[Store][BackloggedGroup]group[%s] has been created", group.c_str());
714 0 : return HCCL_E_PARA;
715 : }
716 0 : g_backloggedGroup[group] = groupRanks;
717 0 : HCCL_INFO("[Store][BackloggedGroup]group[%s] updated", group.c_str());
718 0 : return HCCL_SUCCESS;
719 : }
720 :
721 0 : g_backloggedGroup.insert({group, groupRanks});
722 0 : HCCL_INFO("[device not set]hcom store group[%s] success", group.c_str());
723 0 : return HCCL_SUCCESS;
724 0 : }
725 :
726 0 : std::unique_lock<std::mutex> groupParaLock(hcomInfo.groupParamsLock);
727 0 : if (hcomInfo.hcomGroupMap.find(group) != hcomInfo.hcomGroupMap.end()) {
728 0 : HCCL_ERROR("[Store][BackloggedGroup]group[%s] has been created", group.c_str());
729 0 : return HCCL_E_PARA;
730 : }
731 0 : groupParaLock.unlock();
732 :
733 0 : std::unique_lock<std::mutex> lock(hcomInfo.backloggedGroupLock);
734 0 : if (hcomInfo.backloggedGroup.find(group) != hcomInfo.backloggedGroup.end()) {
735 0 : HCCL_ERROR("[Store][BackloggedGroup]group[%s] already exists", group.c_str());
736 0 : return HCCL_E_PARA;
737 : } else {
738 0 : hcomInfo.backloggedGroup.insert({group, groupRanks});
739 : }
740 0 : HCCL_INFO("hcom store group[%s] success.", group.c_str());
741 0 : return HCCL_SUCCESS;
742 0 : }
743 :
744 0 : HcclResult HcomDestroyBackloggedGroup(const std::string& group)
745 : {
746 0 : HcomInfo& hcomInfo = HcomGetCtxHomInfo();
747 0 : std::unique_lock<std::mutex> lock(hcomInfo.backloggedGroupLock);
748 0 : if (hcomInfo.backloggedGroup.find(group) == hcomInfo.backloggedGroup.end()) {
749 0 : if (!hcomInfo.isHcomInit) {
750 0 : HCCL_WARNING(
751 : "[Destroy][BackloggedGroup]group[%s] does not exist, and hcom has not been inited yet", group.c_str());
752 0 : return HCCL_SUCCESS;
753 : } else {
754 0 : HCCL_ERROR("[Destroy][BackloggedGroup]group[%s] does not exist", group.c_str());
755 0 : return HCCL_E_PARA;
756 : }
757 : } else {
758 0 : hcomInfo.backloggedGroup.erase(group);
759 : }
760 0 : HCCL_INFO("hcom delete backlogged group[%s] success.", group.c_str());
761 0 : return HCCL_SUCCESS;
762 0 : }
763 :
764 0 : HcclResult HcomGetbackloggedByGroup(const char* group, std::vector<u32>& groupRanks, s32& groupSize)
765 : {
766 0 : CHK_RET(HcomCheckGroupName(group));
767 0 : std::string groupName = group;
768 :
769 0 : std::unique_lock<std::mutex> groupLock(g_backloggedGroupLock);
770 0 : auto it = g_backloggedGroup.find(groupName);
771 0 : if (it != g_backloggedGroup.end()) {
772 0 : groupRanks = it->second;
773 0 : groupSize = (it->second).size();
774 0 : HCCL_INFO("[device not set]get back logged group[%s], groupSize[%d]", groupName.c_str(), groupSize);
775 0 : return HCCL_SUCCESS;
776 : }
777 0 : groupLock.unlock();
778 0 : HcomInfo& hcomInfo = HcomGetCtxHomInfo();
779 0 : std::unique_lock<std::mutex> groupParaLock(hcomInfo.backloggedGroupLock);
780 0 : auto iter = hcomInfo.backloggedGroup.find(groupName);
781 0 : if (iter == hcomInfo.backloggedGroup.end()) {
782 0 : groupSize = 0;
783 0 : HCCL_DEBUG(
784 : "[Get][CommByGroup]errNo[0x%016llx] group[%s] does not exist", HCOM_ERROR_CODE(HCCL_E_NOT_FOUND), group);
785 0 : return HCCL_SUCCESS; // 不存在该服务器内相关dev的对应信息
786 : }
787 0 : groupRanks = iter->second;
788 0 : groupSize = (iter->second).size();
789 0 : HCCL_INFO("[device set]get back logged group[%s], groupSize[%d]", groupName.c_str(), groupSize);
790 0 : return HCCL_SUCCESS;
791 0 : }
792 :
793 0 : HcclResult HcomQueryGroupRef(const char* group, u32& groupRef)
794 : {
795 0 : HcomInfo& hcomInfo = HcomGetCtxHomInfo();
796 0 : if (group == nullptr) {
797 0 : return HCCL_SUCCESS; // 全局通信域不需要查询flag
798 : }
799 0 : std::string strGroup = group;
800 0 : std::unique_lock<std::mutex> groupParaLock(hcomInfo.groupParamsLock);
801 0 : auto iter = hcomInfo.hcomGroupMap.find(strGroup);
802 0 : if (iter == hcomInfo.hcomGroupMap.end()) {
803 0 : HCCL_WARNING(
804 : "[Get][CommByGroup]errNo[0x%016llx] group[%s] does not exist", HCOM_ERROR_CODE(HCCL_E_AGAIN),
805 : strGroup.c_str());
806 0 : return HCCL_E_AGAIN; // 不存在该服务器内相关dev的对应信息
807 : }
808 0 : groupRef = iter->second.refCounter;
809 0 : return HCCL_SUCCESS;
810 0 : }
811 :
812 0 : HcclResult HcomGetWorldRankFromGroupRank(const char* group, u32 groupRank, u32* worldRank)
813 : {
814 0 : RPT_INPUT_ERR(
815 : worldRank == nullptr, "EI0003", std::vector<std::string>({"ccl_op", "value", "parameter", "expect"}),
816 : std::vector<std::string>({"HcomGetWorldRankFromGroupRank", "nullptr", "worldRank", "non-null pointer"}));
817 0 : CHK_PTR_NULL(worldRank);
818 0 : bool& isAutoTuneModeOpen = HcomGetCtxAutoTuneMode();
819 0 : if (isAutoTuneModeOpen) {
820 0 : *worldRank = 0;
821 0 : return HCCL_SUCCESS;
822 : }
823 0 : HcomInfo& hcomInfo = HcomGetCtxHomInfo();
824 :
825 0 : HcclResult ret = HcomCheckGroupName(group);
826 0 : RPT_INPUT_ERR(
827 : ret != HCCL_SUCCESS, "EI0003", std::vector<std::string>({"ccl_op", "value", "parameter", "expect"}),
828 : std::vector<std::string>(
829 : {"HcomGetWorldRankFromGroupRank",
830 : {group, strnlen(group, GROUP_NAME_MAX_LEN + 1)},
831 : "group",
832 : "a non-empty string of length 1 to " + std::to_string(GROUP_NAME_MAX_LEN)
833 : + ", containing only alphanumeric characters and underscores"}));
834 0 : CHK_PRT_RET(
835 : ret != HCCL_SUCCESS,
836 : HCCL_ERROR(
837 : "[%s][%s]errNo[0x%016llx] group name is invalid", LOG_KEYWORDS_TASK_EXEC.c_str(),
838 : LOG_KEYWORDS_INVALID_ARGUMENT.c_str(), HCOM_ERROR_CODE(ret)),
839 : ret);
840 : #if (!defined(HCCD)) && (!defined(CCL_KERNEL_AICPU))
841 0 : HCCLV2_FUNC_RUN(HcomGetWorldRankFromGroupRankV2(group, groupRank, worldRank));
842 : #endif
843 0 : if (groupRank >= hcomInfo.params.totalRanks) {
844 0 : HCCL_ERROR(
845 : "[Get][WorldRank]errNo[0x%016llx] groupRank[%u] is out of range[0-%u]", HCOM_ERROR_CODE(HCCL_E_PARA),
846 : groupRank, hcomInfo.params.totalRanks);
847 0 : return HCCL_E_PARA;
848 : }
849 0 : std::string strGroup = (group == nullptr) ? HCCL_WORLD_GROUP : group;
850 0 : CHK_RET(GetGroupRankInfo(strGroup.c_str(), RankInfoType::WORLD_RANK_ID_BY_GROUP, groupRank, worldRank));
851 0 : HCCL_INFO(
852 : "hcom get world rank success, group[%s], groupRank[%u], worldRank[%p]", strGroup.c_str(), groupRank, worldRank);
853 0 : return HCCL_SUCCESS;
854 0 : }
855 :
856 0 : HcclResult HcomGetGroupRankFromWorldRank(u32 worldRank, const char* group, u32* groupRank)
857 : {
858 0 : RPT_INPUT_ERR(
859 : groupRank == nullptr, "EI0003", std::vector<std::string>({"ccl_op", "value", "parameter", "expect"}),
860 : std::vector<std::string>({"HcomGetGroupRankFromWorldRank", "nullptr", "groupRank", "non-null pointer"}));
861 0 : CHK_PTR_NULL(groupRank);
862 0 : bool& isAutoTuneModeOpen = HcomGetCtxAutoTuneMode();
863 0 : if (isAutoTuneModeOpen) {
864 0 : *groupRank = 0;
865 0 : return HCCL_SUCCESS;
866 : }
867 0 : HcomInfo& hcomInfo = HcomGetCtxHomInfo();
868 :
869 0 : HcclResult ret = HcomCheckGroupName(group);
870 0 : RPT_INPUT_ERR(
871 : ret != HCCL_SUCCESS, "EI0003", std::vector<std::string>({"ccl_op", "value", "parameter", "expect"}),
872 : std::vector<std::string>(
873 : {"HcomGetGroupRankFromWorldRank",
874 : {group, strnlen(group, GROUP_NAME_MAX_LEN + 1)},
875 : "group",
876 : "a non-empty string of length 1 to " + std::to_string(GROUP_NAME_MAX_LEN)
877 : + ", containing only alphanumeric characters and underscores"}));
878 0 : CHK_PRT_RET(
879 : ret != HCCL_SUCCESS,
880 : HCCL_ERROR(
881 : "[%s][%s]errNo[0x%016llx] group name is invalid", LOG_KEYWORDS_TASK_EXEC.c_str(),
882 : LOG_KEYWORDS_INVALID_ARGUMENT.c_str(), HCOM_ERROR_CODE(ret)),
883 : ret);
884 : #if (!defined(HCCD)) && (!defined(CCL_KERNEL_AICPU))
885 0 : HCCLV2_FUNC_RUN(HcomGetGroupRankFromWorldRankV2(worldRank, group, groupRank));
886 : #endif
887 0 : std::string strGroup = (group == nullptr) ? HCCL_WORLD_GROUP : group;
888 0 : if (worldRank >= hcomInfo.params.totalRanks) {
889 0 : HCCL_ERROR(
890 : "[Get][GroupRank]errNo[0x%016llx] world[%u] rank is invalid", HCOM_ERROR_CODE(HCCL_E_PARA), worldRank);
891 0 : return HCCL_E_PARA;
892 : }
893 0 : CHK_RET(GetGroupRankInfo(strGroup.c_str(), RankInfoType::GROUP_RANK_ID_BY_WORLD, worldRank, groupRank));
894 0 : HCCL_INFO(
895 : "hcom get group rank success, group[%s], worldRank[%u], groupRank[%p]", strGroup.c_str(), worldRank, groupRank);
896 0 : return HCCL_SUCCESS;
897 0 : }
898 :
899 0 : bool HcomFindGroup(const char* group)
900 : {
901 0 : HcomInfo& hcomInfo = HcomGetCtxHomInfo();
902 : /* 已经存在的group不允许再次创建 */
903 0 : std::unique_lock<std::mutex> groupParaLock(hcomInfo.groupParamsLock);
904 0 : bool exists = !(hcomInfo.hcomGroupMap.find(group) == hcomInfo.hcomGroupMap.end());
905 0 : HCCL_INFO("[Find][Group] group[%s] is exist[%d]", group, exists);
906 0 : groupParaLock.unlock();
907 0 : return exists;
908 0 : }
909 :
910 0 : HcclResult GetWorldGroupRankInfo(RankInfoType rankType, u32 inPara, u32* outPara)
911 : {
912 0 : HcomInfo& hcomInfo = HcomGetCtxHomInfo();
913 0 : CHK_PTR_NULL(outPara);
914 0 : switch (rankType) {
915 0 : case RankInfoType::RANK_SIZE_IN_GROUP:
916 0 : *outPara = hcomInfo.params.totalRanks;
917 0 : break;
918 :
919 0 : case RankInfoType::RANK_ID_IN_GROUP:
920 0 : *outPara = hcomInfo.params.rank;
921 0 : break;
922 :
923 0 : case RankInfoType::WORLD_RANK_ID_BY_GROUP:
924 : case RankInfoType::GROUP_RANK_ID_BY_WORLD:
925 0 : *outPara = inPara;
926 0 : break;
927 0 : case RankInfoType::SERVER_NUM_IN_GROUP:
928 0 : *outPara = hcomInfo.rankTable.serverNum;
929 0 : break;
930 0 : default:
931 0 : HCCL_ERROR(
932 : "[Get][WorldGroupRankInfo]errNo[0x%016llx] invalid rankInfo type[%d]", HCOM_ERROR_CODE(HCCL_E_PARA),
933 : rankType);
934 0 : return HCCL_E_PARA;
935 : }
936 0 : return HCCL_SUCCESS;
937 : }
938 :
939 0 : HcclResult GetGroupRankInfo(const char* group, RankInfoType rankType, u32 inPara, u32* outPara)
940 : {
941 0 : CHK_PTR_NULL(outPara);
942 : // std::string strGroup = (group == nullptr) ? HCCL_WORLD_GROUP : group;
943 0 : if ((group == nullptr) || (strcmp(group, HCCL_WORLD_GROUP) == 0)) {
944 0 : CHK_RET(GetWorldGroupRankInfo(rankType, inPara, outPara));
945 0 : return HCCL_SUCCESS;
946 : }
947 0 : HcomInfo& hcomInfo = HcomGetCtxHomInfo();
948 :
949 0 : std::unique_lock<std::mutex> groupParaLock(hcomInfo.groupParamsLock);
950 0 : auto iter = hcomInfo.hcomGroupMap.find(group);
951 0 : if (iter == hcomInfo.hcomGroupMap.end()) {
952 0 : HCCL_ERROR(
953 : "[Get][GroupRankInfo]errNo[0x%016llx] group[%s] is not exist", HCOM_ERROR_CODE(HCCL_E_NOT_FOUND), group);
954 0 : return HCCL_E_NOT_FOUND; // 不存在该服务器内相关dev的对应信息
955 : }
956 : // group ranks判空
957 0 : CHK_PRT_RET(
958 : (iter->second).groupRanks.empty(),
959 : HCCL_ERROR(
960 : "[Get][GroupRankInfo]errNo[0x%016llx] group[%s] "
961 : "ranks is empty",
962 : HCOM_ERROR_CODE(HCCL_E_INTERNAL), group),
963 : HCCL_E_INTERNAL);
964 :
965 0 : switch (rankType) {
966 0 : case RankInfoType::RANK_SIZE_IN_GROUP:
967 0 : *outPara = (iter->second).totalRanks;
968 0 : return HCCL_SUCCESS;
969 :
970 0 : case RankInfoType::RANK_ID_IN_GROUP:
971 0 : *outPara = (iter->second).groupRank;
972 0 : return HCCL_SUCCESS;
973 :
974 0 : case RankInfoType::WORLD_RANK_ID_BY_GROUP:
975 0 : if (inPara >= (iter->second).totalRanks) {
976 0 : HCCL_ERROR(
977 : "[Get][GroupRankInfo]errNo[0x%016llx] group[%s] groupRank[%u] is invalid",
978 : HCOM_ERROR_CODE(HCCL_E_PARA), group, inPara);
979 0 : return HCCL_E_PARA;
980 : }
981 0 : *outPara = (iter->second).groupRanks[inPara];
982 0 : return HCCL_SUCCESS;
983 :
984 0 : case RankInfoType::GROUP_RANK_ID_BY_WORLD:
985 0 : for (u32 rank = 0; rank < (iter->second).totalRanks; rank++) {
986 0 : if (inPara == (iter->second).groupRanks[rank]) {
987 0 : *outPara = rank;
988 0 : return HCCL_SUCCESS;
989 : }
990 : }
991 0 : HCCL_ERROR(
992 : "[Get][GroupRankInfo]errNo[0x%016llx] invalid rankInfo type[%d]", HCOM_ERROR_CODE(HCCL_E_PARA),
993 : rankType);
994 0 : return HCCL_E_PARA;
995 0 : case RankInfoType::SERVER_NUM_IN_GROUP:
996 0 : *outPara = (iter->second).serverNum;
997 0 : return HCCL_SUCCESS;
998 0 : default:
999 0 : HCCL_ERROR(
1000 : "[Get][GroupRankInfo]errNo[0x%016llx] invalid rankInfo type[%d]", HCOM_ERROR_CODE(HCCL_E_PARA),
1001 : rankType);
1002 0 : return HCCL_E_PARA;
1003 : }
1004 0 : }
1005 :
1006 0 : HcclResult HcomGetRankSize(const char* group, u32* rankSize)
1007 : {
1008 0 : RPT_INPUT_ERR(
1009 : rankSize == nullptr, "EI0003", std::vector<std::string>({"ccl_op", "value", "parameter", "expect"}),
1010 : std::vector<std::string>({"HcomGetRankSize", "nullptr", "rankSize", "non-null pointer"}));
1011 0 : CHK_PTR_NULL(rankSize);
1012 0 : bool& isAutoTuneModeOpen = HcomGetCtxAutoTuneMode();
1013 0 : if (isAutoTuneModeOpen) {
1014 0 : *rankSize = 1;
1015 0 : return HCCL_SUCCESS;
1016 : }
1017 :
1018 0 : HcclResult ret = HcomCheckGroupName(group);
1019 0 : RPT_INPUT_ERR(
1020 : ret != HCCL_SUCCESS, "EI0003", std::vector<std::string>({"ccl_op", "value", "parameter", "expect"}),
1021 : std::vector<std::string>(
1022 : {"HcomGetRankSize",
1023 : {group, strnlen(group, GROUP_NAME_MAX_LEN + 1)},
1024 : "group",
1025 : "a non-empty string of length 1 to " + std::to_string(GROUP_NAME_MAX_LEN)
1026 : + ", containing only alphanumeric characters and underscores"}));
1027 0 : CHK_PRT_RET(
1028 : ret != HCCL_SUCCESS,
1029 : HCCL_ERROR(
1030 : "[%s][%s]errNo[0x%016llx] group name is invalid", LOG_KEYWORDS_TASK_EXEC.c_str(),
1031 : LOG_KEYWORDS_INVALID_ARGUMENT.c_str(), HCOM_ERROR_CODE(ret)),
1032 : ret);
1033 :
1034 : #if (!defined(HCCD)) && (!defined(CCL_KERNEL_AICPU))
1035 0 : HCCLV2_FUNC_RUN(HcomGetRankSizeV2(group, rankSize));
1036 : #endif
1037 0 : std::shared_ptr<hccl::hcclComm> hcclComm;
1038 0 : if (group != nullptr && HcclGetCommHandle(group, hcclComm) == HCCL_SUCCESS) {
1039 0 : CHK_RET(hcclComm->GetRankSize(*rankSize));
1040 : } else {
1041 0 : std::string strGroup = (group == nullptr) ? HCCL_WORLD_GROUP : group;
1042 0 : ret = GetGroupRankInfo(strGroup.c_str(), RankInfoType::RANK_SIZE_IN_GROUP, 0, rankSize);
1043 0 : CHK_PRT_RET(
1044 : ret != HCCL_SUCCESS,
1045 : HCCL_ERROR(
1046 : "[Get][RankSize]errNo[0x%016llx] get group[%s] rank info error", HCOM_ERROR_CODE(ret),
1047 : strGroup.c_str()),
1048 : ret);
1049 0 : HCCL_INFO("hcom get rank size success, group[%s]", strGroup.c_str());
1050 0 : }
1051 0 : return HCCL_SUCCESS;
1052 0 : }
1053 :
1054 0 : HcclResult HcomDestroyOneDevice(HcomInfo& hcomInfo)
1055 : {
1056 0 : HcclUs startut = TIME_NOW();
1057 :
1058 : /* 接口交互信息日志 */
1059 0 : HCCL_RUN_INFO("Entry-HcomDestroy:void");
1060 :
1061 : // 模型运行结束后hcom destroy时，将CheckInfo信息清空
1062 0 : RankConsistentcyChecker::GetInstance().ClearCheckInfo();
1063 :
1064 : // group资源在word group资源销毁之前进行销毁
1065 0 : hcomInfo.params.commConnections.agentConnection = nullptr;
1066 0 : hcomInfo.params.commConnections.serverConnections.clear();
1067 0 : hcomInfo.hcomGroupMap.clear();
1068 0 : std::unique_lock<std::mutex> backloggedGroupLock(hcomInfo.backloggedGroupLock);
1069 0 : hcomInfo.backloggedGroup.clear();
1070 0 : backloggedGroupLock.unlock();
1071 :
1072 0 : hcomInfo.rankTable.nicNames.clear();
1073 0 : hcomInfo.rankTable.rankList.clear();
1074 0 : g_segmentIdxMap.clear();
1075 0 : g_segmentSizeMap.clear();
1076 0 : hcomInfo.params.profilingMode = HcomProfilingMode::PROFILING_CLOSE;
1077 0 : hcomInfo.params.profilingOption = "";
1078 0 : hcomInfo.isHcomInit = false;
1079 :
1080 0 : if (hcomInfo.params.deviceType != DevType::DEV_TYPE_NOSOC) {
1081 0 : ProfilingManagerPub::ClearStoragedProfilingInfo();
1082 : }
1083 :
1084 : /* 关键状态记录 */
1085 0 : HCCL_USER_CRITICAL_LOG(
1086 : "hcom destroy complete,take time [%lld]us, group[%s], rankNum[%u], rank[%u]", DURATION_US(TIME_NOW() - startut),
1087 : hcomInfo.params.identifier.c_str(), hcomInfo.rankTable.rankNum, hcomInfo.params.rank);
1088 :
1089 0 : return HCCL_SUCCESS;
1090 0 : }
1091 :
1092 0 : HcclResult HcomDestroy(void)
1093 : {
1094 : #if (!defined(HCCD)) && (!defined(CCL_KERNEL_AICPU))
1095 0 : HCCLV2_FUNC_RUN([&]() -> HcclResult {
1096 : std::unique_lock<std::mutex> lock(g_destroyDeviceLock);
1097 : CHK_RET(HcomDestroyV2());
1098 : for (u32 i = 0; i <= MAX_MODULE_DEVICE_NUM; i++) {
1099 : HcomInfo& hcomInfo = HcomGetCtxHomInfoById(i);
1100 : hcomInfo.pComm = nullptr;
1101 : hcomInfo.hcomGroupMap.clear();
1102 : }
1103 : return HCCL_SUCCESS;
1104 : }());
1105 : #endif
1106 0 : std::unique_lock<std::mutex> lock(g_destroyDeviceLock);
1107 0 : for (u32 i = 0; i <= MAX_MODULE_DEVICE_NUM; i++) {
1108 0 : HcomInfo& hcomInfo = HcomGetCtxHomInfoById(i);
1109 :
1110 0 : if (!hcomInfo.isHcomInit) {
1111 0 : if (hcomInfo.pComm != nullptr) {
1112 0 : hcomInfo.pComm = nullptr;
1113 : }
1114 0 : HCCL_INFO("[Destroy][Result]hcomInfo[%u].isHcomInit is false.", i);
1115 :
1116 : /* 接口交互信息日志 */
1117 0 : HCCL_INFO("Entry-HcomDestroy:void skip");
1118 0 : if (g_hcomDestroyCallback != nullptr) {
1119 0 : (void)g_hcomDestroyCallback(hcomInfo);
1120 : }
1121 0 : continue;
1122 : } else {
1123 0 : if (hcomInfo.pComm != nullptr) {
1124 0 : HcomUnSetGroupTopoInfo(hcomInfo.pComm->GetIdentifier().c_str());
1125 : }
1126 : }
1127 :
1128 0 : if (hcomInfo.pComm == nullptr
1129 0 : && ((g_hcomCallBackGroupIsInit != nullptr) && (!(g_hcomCallBackGroupIsInit(hcomInfo))))) {
1130 0 : HCCL_INFO("[Destroy][Result]hcomInfo[%u].pComm or pCommBase is nullptr.", i);
1131 0 : continue;
1132 : }
1133 :
1134 0 : if (hcomInfo.params.logicDevId != HOST_DEVICE_ID) {
1135 0 : u32 logicId = hcomInfo.params.logicDevId;
1136 0 : if (hcomInfo.rankTable.version.compare(HETEROG_CLUSTER_VERSION) == 0) {
1137 0 : CHK_RET(hrtGetDeviceIndexByPhyId(hcomInfo.params.logicDevId, logicId));
1138 : }
1139 0 : s32 deviceId = 0;
1140 0 : if (hrtGetDevice(&deviceId) != HCCL_SUCCESS) {
1141 0 : CHK_RET(hrtSetDevice(logicId));
1142 0 : HCCL_INFO("[HcomDestroy][SetDeviceId]logicDevId[%u]", logicId);
1143 : }
1144 : }
1145 :
1146 0 : HCCL_INFO("[Destroy][Result]hcomInfo[%u].pComm destroy.", i);
1147 0 : HcclResult ret = HcomDestroyOneDevice(hcomInfo);
1148 0 : if (ret == HCCL_SUCCESS) {
1149 0 : HCCL_INFO("[Destroy][Result]hcomInfo[%u].pComm HcomDestroyOneDevice success.", i);
1150 : } else {
1151 0 : HCCL_INFO("[Destroy][Result]hcomInfo[%u].pComm HcomDestroyOneDevice fail.", i);
1152 0 : return ret;
1153 : }
1154 :
1155 0 : if (g_hcomDestroyCallback != nullptr) {
1156 0 : (void)g_hcomDestroyCallback(hcomInfo);
1157 : }
1158 :
1159 0 : hcomInfo.pComm = nullptr;
1160 0 : hcomInfo.hcclCommTopoInfoDetectServer.clear();
1161 0 : hcomInfo.hcclCommTopoInfoDetectAgent.clear();
1162 : }
1163 0 : return HCCL_SUCCESS;
1164 0 : }
1165 :
1166 52 : void HcomGroupCallbackFuncInstall(
1167 : HcclResult (*p1)(const std::string&, const std::vector<u32>&), bool (*p2)(HcomInfo&),
1168 : HcclResult (*p3)(const std::string&), HcclResult (*p4)(HcomInfo&))
1169 : {
1170 52 : g_hcomCreateGroupCallback = p1;
1171 52 : g_hcomCallBackGroupIsInit = p2;
1172 52 : g_hcomDestroyGroupCallback = p3;
1173 52 : g_hcomDestroyCallback = p4;
1174 52 : }
1175 :
1176 0 : HcclResult HcomSetGradFusionByIndex(const char* group, u32 segmentNum, const u32* IdxList)
1177 : {
1178 0 : bool& isAutoTuneModeOpen = HcomGetCtxAutoTuneMode();
1179 0 : if (isAutoTuneModeOpen) {
1180 0 : return HCCL_SUCCESS;
1181 : }
1182 :
1183 0 : RPT_INPUT_ERR(
1184 : IdxList == nullptr, "EI0003", std::vector<std::string>({"ccl_op", "value", "parameter", "expect"}),
1185 : std::vector<std::string>({"HcomSetGradFusionByIndex", "nullptr", "IdxList", "non-null pointer"}));
1186 0 : CHK_PTR_NULL(IdxList);
1187 0 : bool bRet = segmentNum == 0;
1188 0 : RPT_INPUT_ERR(
1189 : bRet, "EI0003", std::vector<std::string>({"ccl_op", "value", "parameter", "expect"}),
1190 : std::vector<std::string>(
1191 : {"HcomSetGradFusionByIndex", std::to_string(0), "segmentNum",
1192 : "must be a positive integer (greater than 0)"}));
1193 0 : CHK_PRT_RET(
1194 : bRet,
1195 : HCCL_ERROR(
1196 : "[%s][%s]errNo[0x%016llx] set split IdxList length is zero", LOG_KEYWORDS_TASK_EXEC.c_str(),
1197 : LOG_KEYWORDS_INVALID_ARGUMENT.c_str(), HCOM_ERROR_CODE(HCCL_E_PARA)),
1198 : HCCL_E_PARA);
1199 0 : std::string strGroup = (group == nullptr) ? HCCL_WORLD_GROUP : group;
1200 0 : string idxList;
1201 0 : for (u32 i = 0; i < segmentNum; i++) {
1202 0 : if (i < segmentNum - 1) {
1203 0 : idxList += to_string(IdxList[i]) + ',';
1204 0 : } else if (i == segmentNum - 1) {
1205 0 : idxList += to_string(IdxList[i]);
1206 : }
1207 : }
1208 : /* 接口交互信息日志 */
1209 0 : HCCL_RUN_INFO(
1210 : "Entry-HcomSetGradFusionByIndex:group[%s], segmentNum[%u], IdxList[%s]", strGroup.c_str(), segmentNum,
1211 : idxList.c_str());
1212 :
1213 0 : CHK_RET(HcomCheckGroupName(strGroup.c_str()));
1214 :
1215 0 : std::vector<u32> tempList;
1216 :
1217 0 : for (u32 segidx = 0; segidx < segmentNum; segidx++) {
1218 0 : tempList.push_back(IdxList[segidx]);
1219 : }
1220 :
1221 0 : for (u32 i = 0; i < tempList.size() - 1; i++) {
1222 0 : if (tempList[i] >= tempList[i + 1]) {
1223 0 : HCCL_ERROR(
1224 : "[Set][GradFusionByIndex]errNo[0x%016llx] index list is not ascending", HCOM_ERROR_CODE(HCCL_E_PARA));
1225 0 : return HCCL_E_PARA;
1226 : }
1227 : }
1228 0 : std::unique_lock<std::mutex> segmentIdxMapLock(g_segmentIdxMapLock);
1229 0 : g_segmentIdxMap.insert(std::pair<std::string, std::vector<u32>>(strGroup, tempList));
1230 0 : segmentIdxMapLock.unlock();
1231 0 : return HCCL_SUCCESS;
1232 0 : }
1233 :
1234 0 : HcclResult HcomSetGradFusionBySize(const char* group, u32 segmentNum, const float* sizeList)
1235 : {
1236 0 : bool& isAutoTuneModeOpen = HcomGetCtxAutoTuneMode();
1237 0 : if (isAutoTuneModeOpen) {
1238 0 : return HCCL_SUCCESS;
1239 : }
1240 :
1241 0 : RPT_INPUT_ERR(
1242 : sizeList == nullptr, "EI0003", std::vector<std::string>({"ccl_op", "value", "parameter", "expect"}),
1243 : std::vector<std::string>({"HcomSetGradFusionBySize", "nullptr", "sizeList", "non-null pointer"}));
1244 0 : CHK_PTR_NULL(sizeList);
1245 0 : bool bRet = segmentNum == 0;
1246 0 : RPT_INPUT_ERR(
1247 : bRet, "EI0003", std::vector<std::string>({"ccl_op", "value", "parameter", "expect"}),
1248 : std::vector<std::string>(
1249 : {"HcomSetGradFusionBySize", std::to_string(0), "segmentNum",
1250 : "must be a positive integer (greater than 0)"}));
1251 0 : CHK_PRT_RET(
1252 : bRet,
1253 : HCCL_ERROR(
1254 : "[%s][%s]errNo[0x%016llx] set split sizeList length is zero", LOG_KEYWORDS_TASK_EXEC.c_str(),
1255 : LOG_KEYWORDS_INVALID_ARGUMENT.c_str(), HCOM_ERROR_CODE(HCCL_E_PARA)),
1256 : HCCL_E_PARA);
1257 0 : std::string strGroup = (group == nullptr) ? HCCL_WORLD_GROUP : group;
1258 0 : string strSizeList;
1259 0 : for (u32 i = 0; i < segmentNum; i++) {
1260 0 : if (i < segmentNum - 1) {
1261 0 : strSizeList += to_string(sizeList[i]) + ',';
1262 0 : } else if (i == segmentNum - 1) {
1263 0 : strSizeList += to_string(sizeList[i]);
1264 : }
1265 : }
1266 : /* 接口交互信息日志 */
1267 0 : HCCL_RUN_INFO(
1268 : "Entry-HcomSetGradFusionBySize:group[%s], segmentNum[%u], sizeList[%s]", strGroup.c_str(), segmentNum,
1269 : strSizeList.c_str());
1270 :
1271 0 : CHK_RET(HcomCheckGroupName(strGroup.c_str()));
1272 0 : std::vector<float> tempList;
1273 0 : float sizeTotal = 0;
1274 :
1275 0 : for (u32 sizeIdx = 0; sizeIdx < segmentNum; sizeIdx++) {
1276 0 : bRet = sizeList[sizeIdx] < 0;
1277 0 : CHK_PRT_RET(
1278 : bRet,
1279 : HCCL_ERROR(
1280 : "[Set][GradFusionBySize]errNo[0x%016llx] sizeList[%u] less than zero", HCOM_ERROR_CODE(HCCL_E_PARA),
1281 : sizeIdx),
1282 : HCCL_E_PARA);
1283 0 : tempList.push_back(sizeList[sizeIdx]);
1284 0 : sizeTotal += sizeList[sizeIdx];
1285 : }
1286 :
1287 0 : if (std::fabs(sizeTotal - 100) > 1e-6) { // 判断用户设置总百分比是否为100%
1288 0 : HCCL_ERROR("[Set][GradFusionBySize]errNo[0x%016llx] size list sum is not 100%%", HCOM_ERROR_CODE(HCCL_E_PARA));
1289 0 : return HCCL_E_PARA;
1290 : } else {
1291 0 : std::unique_lock<std::mutex> segmentSizeMapLock(g_segmentSizeMapLock);
1292 0 : g_segmentSizeMap.insert(std::pair<std::string, std::vector<float>>(strGroup, tempList));
1293 0 : segmentSizeMapLock.unlock();
1294 0 : return HCCL_SUCCESS;
1295 0 : }
1296 0 : }
1297 :
1298 0 : HcclResult HcomGenerateCommId(hccl::HcclCommParams& params)
1299 : {
1300 0 : s32 sRet = memset_s(params.id.internal, HCCL_ROOT_INFO_BYTES, 0, sizeof(params.id.internal));
1301 0 : CHK_PRT_RET(sRet != EOK, HCCL_ERROR("[GenerateCommId]memory set error. return[%d].", sRet), HCCL_E_PARA);
1302 :
1303 : HcclRootInfo uniqueId;
1304 0 : std::string group;
1305 0 : CHK_RET(hcclComm::GetUniqueId(&uniqueId));
1306 :
1307 0 : if (!params.isHeterogComm) {
1308 0 : group = "hccl_world_group";
1309 : } else {
1310 0 : group = "hccl_heterog_group";
1311 : }
1312 :
1313 0 : sRet = snprintf_s(
1314 0 : params.id.internal, HCCL_ROOT_INFO_BYTES, HCCL_ROOT_INFO_BYTES - 1, "%s%s%s", uniqueId.internal, "-",
1315 : group.c_str());
1316 0 : CHK_PRT_RET(
1317 : sRet == -1,
1318 : HCCL_ERROR("[GenerateCommId]errNo[0x%016llx] sal snprintf_s error", HCCL_ERROR_CODE(HCCL_E_INTERNAL)),
1319 : HCCL_E_INTERNAL);
1320 0 : HCCL_INFO("params.id.internal [%s]", params.id.internal);
1321 0 : return HCCL_SUCCESS;
1322 0 : }
1323 :
1324 0 : HcclResult InitHcomMiscInfo(hccl::HcclCommParams& params, const char* rankTable)
1325 : {
1326 0 : CHK_PTR_NULL(rankTable);
1327 :
1328 0 : RankConsistentcyChecker::GetInstance().SetCheckCannVersionSwitch(true); // 打开CANN软件版本校验开关
1329 :
1330 : // 记录版本信息
1331 0 : std::string curVersion = GetExternalInputCannVersion();
1332 0 : CHK_RET(RankConsistentcyChecker::GetInstance().RecordVerInfo(curVersion));
1333 0 : HcomInfo& hcomInfo = HcomGetCtxHomInfo();
1334 : // 计算rankTable的crc值并保存
1335 0 : HcclResult ret = HcomCalcCRC(params, rankTable);
1336 0 : CHK_PRT_RET(
1337 : ret != HCCL_SUCCESS,
1338 : HCCL_ERROR("[Init][OtherInfo]errNo[0x%016llx] calc ranktable crc error", HCCL_ERROR_CODE(HCCL_E_INTERNAL)),
1339 : HCCL_E_INTERNAL);
1340 : // 生成通信域标识符
1341 0 : ret = HcomGenerateCommId(hcomInfo.params);
1342 0 : CHK_PRT_RET(
1343 : ret != HCCL_SUCCESS,
1344 : HCCL_ERROR("[Init][OtherInfo]errNo[0x%016llx] generate CommId error", HCCL_ERROR_CODE(HCCL_E_INTERNAL)),
1345 : HCCL_E_INTERNAL);
1346 0 : return HCCL_SUCCESS;
1347 0 : }
1348 :
1349 0 : bool HcomCheckrtMemcpyAddrAsync(const std::string& group)
1350 : {
1351 0 : float counterVaule = 1.0f;
1352 :
1353 : // 偏移拷贝的二级指针
1354 0 : void* deviceMemSrcLevel2 = nullptr;
1355 0 : void* deviceMemDstLevel2 = nullptr;
1356 : // 偏移拷贝的一级指针
1357 0 : void* deviceMemSrc = nullptr;
1358 0 : void* deviceMemDst = nullptr;
1359 :
1360 0 : auto deleter = [&deviceMemSrcLevel2, &deviceMemDstLevel2, &deviceMemSrc, &deviceMemDst](void* dst) {
1361 0 : if (dst != nullptr) {
1362 0 : CHK_PRT(hrtFree(dst));
1363 0 : if (dst == deviceMemSrcLevel2) {
1364 0 : deviceMemSrcLevel2 = nullptr;
1365 0 : } else if (dst == deviceMemDstLevel2) {
1366 0 : deviceMemDstLevel2 = nullptr;
1367 0 : } else if (dst == deviceMemSrc) {
1368 0 : deviceMemSrc = nullptr;
1369 0 : } else if (dst == deviceMemDst) {
1370 0 : deviceMemDst = nullptr;
1371 : }
1372 : }
1373 0 : };
1374 :
1375 0 : CHK_RET(hrtMalloc(&deviceMemSrcLevel2, sizeof(void*)));
1376 0 : unique_ptr<void, decltype(deleter)> deviceMemSrcLevel2Unique(deviceMemSrcLevel2, deleter);
1377 0 : CHK_RET(hrtMalloc(&deviceMemDstLevel2, sizeof(void*)));
1378 0 : unique_ptr<void, decltype(deleter)> deviceMemDstLevel2Unique(deviceMemDstLevel2, deleter);
1379 0 : CHK_RET(hrtMalloc(&deviceMemSrc, sizeof(float)));
1380 0 : unique_ptr<void, decltype(deleter)> deviceMemSrcUnique(deviceMemSrc, deleter);
1381 0 : CHK_RET(hrtMalloc(&deviceMemDst, sizeof(float)));
1382 0 : unique_ptr<void, decltype(deleter)> deviceMemDstUnique(deviceMemDst, deleter);
1383 :
1384 0 : CHK_RET(hrtMemSyncCopy(
1385 : deviceMemDst, sizeof(float), &counterVaule, sizeof(float),
1386 : HcclRtMemcpyKind::HCCL_RT_MEMCPY_KIND_HOST_TO_DEVICE));
1387 0 : CHK_RET(hrtMemSyncCopy(
1388 : deviceMemSrc, sizeof(float), &counterVaule, sizeof(float),
1389 : HcclRtMemcpyKind::HCCL_RT_MEMCPY_KIND_HOST_TO_DEVICE));
1390 :
1391 0 : CHK_RET(hrtMemSyncCopy(
1392 : deviceMemDstLevel2, sizeof(void*), &deviceMemDst, sizeof(void*),
1393 : HcclRtMemcpyKind::HCCL_RT_MEMCPY_KIND_HOST_TO_DEVICE));
1394 0 : CHK_RET(hrtMemSyncCopy(
1395 : deviceMemSrcLevel2, sizeof(void*), &deviceMemSrc, sizeof(void*),
1396 : HcclRtMemcpyKind::HCCL_RT_MEMCPY_KIND_HOST_TO_DEVICE));
1397 :
1398 0 : u64 destMax = sizeof(s32);
1399 0 : u64 offset = 0;
1400 :
1401 0 : Stream stream(StreamType::STREAM_TYPE_ONLINE);
1402 0 : bool notSupportSecAddrCopyWithOffset = false;
1403 :
1404 : HcclResult ret
1405 0 : = hrtMemcpyAddrAsync(deviceMemDstLevel2, destMax, offset, deviceMemSrcLevel2, destMax, offset, stream.ptr());
1406 0 : if (ret == HCCL_E_NOT_SUPPORT) {
1407 0 : notSupportSecAddrCopyWithOffset = true;
1408 : } else {
1409 0 : CHK_RET(hcclStreamSynchronize(stream.ptr(), CommConfiger::GetInstance().GetCommConfigExecTimeOut(group)));
1410 : }
1411 :
1412 0 : g_notSupportSecAddrCopyWithOffset = notSupportSecAddrCopyWithOffset;
1413 :
1414 0 : return notSupportSecAddrCopyWithOffset;
1415 0 : }
1416 :
1417 1 : bool HcomGetSecAddrCopyFlag(const char* socVersion)
1418 : {
1419 1 : HCCL_INFO("[Hcom][HcomGetSecAddrCopyFlag] SecAddrCopyWithOffset flag is %d", g_notSupportSecAddrCopyWithOffset);
1420 : DevType devType;
1421 1 : std::string socVersionStr(socVersion);
1422 1 : CHK_RET(hrtGetDeviceTypeBySocVersion(socVersionStr, devType));
1423 :
1424 1 : return !g_notSupportSecAddrCopyWithOffset
1425 1 : && (devType == DevType::DEV_TYPE_910B || devType == DevType::DEV_TYPE_910);
1426 1 : }
1427 :
1428 0 : HcclResult HcomNormalInit(const char* rankTableM, const char* identify)
1429 : {
1430 0 : HcclResult ret = HCCL_SUCCESS;
1431 0 : bool& isAutoTuneModeOpen = HcomGetCtxAutoTuneMode();
1432 0 : isAutoTuneModeOpen = false;
1433 0 : HcomInfo& hcomInfo = HcomGetCtxHomInfo();
1434 :
1435 : /*--------------入参合法性检测---------------------*/
1436 0 : CHK_PTR_NULL(rankTableM);
1437 0 : CHK_PTR_NULL(identify);
1438 :
1439 : /* 防止重复调用初始化 */
1440 0 : CHK_PRT_RET(
1441 : (hcomInfo.pComm != nullptr),
1442 : HCCL_ERROR(
1443 : "[Init][Result]errNo[0x%016llx] identify[%s], "
1444 : "multiple initialization is not supported",
1445 : HCOM_ERROR_CODE(HCCL_E_UNAVAIL), identify),
1446 : HCCL_E_UNAVAIL);
1447 :
1448 : /* --------------初始化------------------------- */
1449 0 : bool errorFlag = false;
1450 0 : s32 logicDevId = 0;
1451 0 : hcomInfo.params.commWorkMode = WorkMode::HCCL_MODE_NORMAL;
1452 : do {
1453 0 : ret = InitHcomMiscInfo(hcomInfo.params, rankTableM);
1454 0 : CHK_PRT_BREAK(
1455 : ret != HCCL_SUCCESS, HCCL_ERROR("[Init][Result]errNo[0x%016llx] init other Info.", HCOM_ERROR_CODE(ret)),
1456 : errorFlag = true);
1457 :
1458 : DevType deviceType;
1459 0 : CHK_PRT_BREAK(hrtGetDevice(&logicDevId) != HCCL_SUCCESS, , errorFlag = true);
1460 0 : CHK_RET(hrtGetDeviceType(deviceType));
1461 : // 为适配12包，做此修改
1462 0 : g_notSupportSecAddrCopyWithOffset = HcomCheckrtMemcpyAddrAsync(identify);
1463 :
1464 0 : ret = CfgGetClusterInfo(
1465 0 : rankTableM, identify, hcomInfo.params, hcomInfo.rankTable, GetExternalInputInterSuperPodRetryEnable(),
1466 : deviceType);
1467 0 : CHK_PRT_BREAK(
1468 : ret != HCCL_SUCCESS,
1469 : HCCL_ERROR(
1470 : "[Init][Result]errNo[0x%016llx] cfg get ranktable[%p] info "
1471 : "error: identify[%s]",
1472 : HCOM_ERROR_CODE(ret), rankTableM, identify),
1473 : errorFlag = true);
1474 :
1475 0 : if (hcomInfo.rankTable.serverNum != SINGLE_SERVER_NUM
1476 0 : && (deviceType == DevType::DEV_TYPE_310P3 || deviceType == DevType::DEV_TYPE_310P1)) {
1477 0 : CHK_RET(InitExternalInputHeterog());
1478 : }
1479 :
1480 0 : hcomInfo.pComm.reset(new (std::nothrow) hccl::hcclComm(0, 0, HCCL_WORLD_GROUP));
1481 :
1482 0 : CHK_PRT_RET(
1483 : hcomInfo.pComm == nullptr,
1484 : HCCL_ERROR("[Init][Result]hcomInfo.pComm is null, "
1485 : "create failed"),
1486 : HCCL_E_PTR);
1487 0 : CommConfig commConfig(identify);
1488 0 : ret = hcomInfo.pComm->init(hcomInfo.params, commConfig, hcomInfo.rankTable);
1489 0 : CHK_PRT_BREAK(
1490 : ret != HCCL_SUCCESS, HCCL_ERROR("[Init][Result]errNo[0x%016llx] hcclComm init error", HCOM_ERROR_CODE(ret)),
1491 : errorFlag = true);
1492 :
1493 0 : ret = ShowRanktableConfigInfo(hcomInfo.cloudFlag, hcomInfo.params, hcomInfo.rankTable);
1494 0 : CHK_PRT_BREAK(
1495 : ret != HCCL_SUCCESS,
1496 : HCCL_ERROR("[Init][Result]errNo[0x%016llx] put ranktable info error", HCOM_ERROR_CODE(ret)),
1497 : errorFlag = true);
1498 0 : ret = InitWorkflowMode(HcclWorkflowMode::HCCL_WORKFLOW_MODE_OPS_KERNEL_INFO_LIB);
1499 0 : CHK_PRT_BREAK(
1500 : ret != HCCL_SUCCESS,
1501 : HCCL_ERROR("[Init][Result]errNo[0x%016llx] init work flow mode error", HCCL_ERROR_CODE(ret)),
1502 : errorFlag = true);
1503 :
1504 0 : ret = HcomFlushBackloggedGroups();
1505 0 : CHK_PRT_BREAK(
1506 : ret != HCCL_SUCCESS,
1507 : HCCL_ERROR("[Init][Result]errNo[0x%016llx] create backlogged group failed", HCOM_ERROR_CODE(ret)),
1508 : errorFlag = true);
1509 :
1510 0 : ret = HcomSetGroupTopoInfo(hcomInfo.pComm->GetIdentifier().c_str(), hcomInfo.rankTable.rankNum);
1511 0 : CHK_PRT_BREAK(
1512 : ret != HCCL_SUCCESS,
1513 : HCCL_ERROR(
1514 : "[Init][Result]errNo[0x%016llx] SetGroupTopoInfo error, "
1515 : "group[%s]",
1516 : HCOM_ERROR_CODE(ret), (hcomInfo.pComm->GetIdentifier().c_str())),
1517 : errorFlag = true);
1518 0 : } while (0);
1519 :
1520 0 : if (errorFlag) {
1521 0 : HCCL_ERROR(
1522 : "[Init][Result]hcom init failed, rankNum[%u], rank[%u], server[%s], device[%d], return[0x%016llx]",
1523 : hcomInfo.rankTable.rankNum, hcomInfo.params.rank, hcomInfo.params.serverId.c_str(), logicDevId,
1524 : HCOM_ERROR_CODE(ret));
1525 0 : (void)HcomDestroy();
1526 0 : return ret;
1527 : }
1528 0 : return HCCL_SUCCESS;
1529 : }
1530 :
1531 0 : HcclResult HcomCheckInitClusterInfo(const char* rankTableM, const char* identify)
1532 : {
1533 0 : HcclResult ret = HCCL_SUCCESS;
1534 : // rankTable合法性检测
1535 0 : u32 rankTableSize = 0;
1536 0 : ret = HcomCheckRankTable(rankTableM, rankTableSize);
1537 0 : CHK_PRT_RET(
1538 : ret != HCCL_SUCCESS,
1539 : HCCL_ERROR(
1540 : "[%s][%s]errNo[0x%016llx] input rankTable error", LOG_KEYWORDS_INIT_GROUP.c_str(),
1541 : LOG_KEYWORDS_INVALID_ARGUMENT.c_str(), HCOM_ERROR_CODE(ret)),
1542 : ret);
1543 :
1544 : // identify合法性检测
1545 0 : ret = HcomCheckIdentify(identify);
1546 0 : RPT_INPUT_ERR(
1547 : ret != HCCL_SUCCESS, "EI0003", std::vector<std::string>({"ccl_op", "value", "parameter", "expect"}),
1548 : std::vector<std::string>(
1549 : {"HcomInit", {identify, strnlen(identify, IDENTIFY_MAX_LEN + 1)}, "identify", "a valid node identifier"}));
1550 0 : CHK_PRT_RET(
1551 : ret != HCCL_SUCCESS,
1552 : HCCL_ERROR(
1553 : "[%s][%s]errNo[0x%016llx] identify parameter error", LOG_KEYWORDS_INIT_GROUP.c_str(),
1554 : LOG_KEYWORDS_INVALID_ARGUMENT.c_str(), HCOM_ERROR_CODE(ret)),
1555 : ret);
1556 0 : return ret;
1557 0 : }
1558 :
1559 0 : HcclResult HcomInitByFile(const char* rankTablePath, const char* identify)
1560 : {
1561 0 : HcomInfo& hcomInfo = HcomGetCtxHomInfo();
1562 :
1563 0 : CHK_PTR_NULL(rankTablePath);
1564 0 : CHK_PTR_NULL(identify);
1565 : #if (!defined(HCCD)) && (!defined(CCL_KERNEL_AICPU))
1566 0 : HCCLV2_FUNC_RUN([&]() -> HcclResult {
1567 : CHK_RET(HcomInitByFileV2(rankTablePath, identify));
1568 : u32 rankNum = 0;
1569 : CHK_RET(HcomGetRankSize(HCCL_WORLD_GROUP, &rankNum));
1570 : s32 myRank = std::atoi(identify);
1571 : Hccl::RankId rank = static_cast<Hccl::RankId>(myRank);
1572 : void* commV2 = nullptr;
1573 : CHK_RET(HcomGetCommV2(&commV2));
1574 : CHK_RET(HcomInitCollComm(rank, &commV2, hcomInfo.pComm));
1575 : CHK_RET(HcomSetGroupTopoInfo(HCCL_WORLD_GROUP, rankNum));
1576 : return HCCL_SUCCESS;
1577 : }());
1578 : #endif
1579 :
1580 0 : HcclUs startut = TIME_NOW();
1581 0 : HcclResult ret = HCCL_SUCCESS;
1582 :
1583 : // 读取rankTable文件到内存
1584 0 : std::string rankTableM;
1585 0 : std::string realFilePath;
1586 0 : ret = HcomLoadRanktableFile(rankTablePath, rankTableM, realFilePath);
1587 0 : CHK_PRT_RET(
1588 : ret != HCCL_SUCCESS,
1589 : HCCL_ERROR(
1590 : "[%s][%s]errNo[0x%016llx] rankTablePath[%s] identify[%s] load rankTable error.",
1591 : LOG_KEYWORDS_INIT_GROUP.c_str(), LOG_KEYWORDS_RANKTABLE_CONFIG.c_str(), HCCL_ERROR_CODE(HCCL_E_INTERNAL),
1592 : rankTablePath, identify),
1593 : HCCL_E_INTERNAL);
1594 0 : CHK_RET(HcomCheckInitClusterInfo(rankTableM.c_str(), identify));
1595 0 : HCCL_RUN_INFO("Entry-HcomInitByFile:rankTablePath[%s], identify[%s]", realFilePath.c_str(), identify);
1596 :
1597 0 : CHK_RET(InitExternalInput());
1598 0 : CHK_RET(InitEnvConfig());
1599 :
1600 : // 调用初始化接口
1601 0 : ret = HcomNormalInit(rankTableM.c_str(), identify);
1602 0 : CHK_PRT_RET(
1603 : ret != HCCL_SUCCESS,
1604 : HCCL_ERROR(
1605 : "[HcomInitByFile]errNo[0x%016llx] rankTablePath[%s] identify[%s] "
1606 : "hcom init failed.",
1607 : HCCL_ERROR_CODE(ret), realFilePath.c_str(), identify),
1608 : ret);
1609 0 : hcomInfo.isHcomInit = true;
1610 : /* 关键状态记录 */
1611 0 : HCCL_RUN_INFO(
1612 : "[HCCL_TRACE]hcom init by file success,take time [%lld]us, rankTablePath[%s], rankNum[%u], rank[%u],"
1613 : "server[%s], device[%d]",
1614 : DURATION_US(TIME_NOW() - startut), realFilePath.c_str(), hcomInfo.rankTable.rankNum, hcomInfo.params.rank,
1615 : hcomInfo.params.serverId.c_str(), hcomInfo.params.logicDevId);
1616 0 : return HCCL_SUCCESS;
1617 0 : }
1618 :
1619 3 : DevType HcomGetDeviceType()
1620 : {
1621 : DevType devType;
1622 3 : hrtGetDeviceType(devType);
1623 3 : if (devType == DevType::DEV_TYPE_950 || devType == DevType::DEV_TYPE_960) {
1624 0 : HcomGetDevTypeV2(devType);
1625 0 : HCCL_INFO("LaunchHcomKernel: devType is DEV_TYPE_950 or DEV_TYPE_960");
1626 0 : return MakeEnumToDevType(static_cast<int>(devType));
1627 : }
1628 :
1629 3 : HcomInfo& hcomInfo = HcomGetCtxHomInfo();
1630 3 : return hcomInfo.params.deviceType;
1631 : }
1632 :
1633 1 : HcclResult HcomCreateCommCCLbuffer(const char* group)
1634 : {
1635 1 : RPT_INPUT_ERR(
1636 : group == nullptr, "EI0003", std::vector<std::string>({"ccl_op", "value", "parameter", "expect"}),
1637 : std::vector<std::string>({"HcomGetDevType", "nullptr", "group", "non-null pointer"}));
1638 1 : CHK_PTR_NULL(group);
1639 :
1640 1 : HcclResult ret = HcomCheckGroupName(group);
1641 1 : RPT_INPUT_ERR(
1642 : ret != HCCL_SUCCESS, "EI0003", std::vector<std::string>({"ccl_op", "value", "parameter", "expect"}),
1643 : std::vector<std::string>(
1644 : {"HcomGetDevType",
1645 : {group, strnlen(group, GROUP_NAME_MAX_LEN + 1)},
1646 : "group",
1647 : "a non-empty string of length 1 to " + std::to_string(GROUP_NAME_MAX_LEN)
1648 : + ", containing only alphanumeric characters and underscores"}));
1649 1 : CHK_PRT_RET(
1650 : ret != HCCL_SUCCESS,
1651 : HCCL_ERROR("[Get][HcomGetDevType]errNo[0x%016llx] group name is invalid", HCOM_ERROR_CODE(ret)), ret);
1652 :
1653 1 : DevType devType = HcomGetDeviceType();
1654 1 : if (devType == DevType::DEV_TYPE_950 || devType == DevType::DEV_TYPE_960) {
1655 0 : HCCL_INFO("HcomCreateCommCclBufV2 start.");
1656 : #if (!defined(HCCD)) && (!defined(CCL_KERNEL_AICPU))
1657 0 : HCCLV2_FUNC_RUN(HcomCreateCommCclBufV2(group));
1658 : #endif
1659 0 : return HCCL_SUCCESS;
1660 : }
1661 :
1662 1 : std::shared_ptr<hccl::hcclComm> hcclComm;
1663 1 : CHK_RET(HcomGetCommByGroup(group, hcclComm));
1664 1 : CHK_RET(hcclComm->CreateCommCCLbuffer());
1665 1 : return HCCL_SUCCESS;
1666 1 : }
1667 :
1668 1 : HcclResult HcomGetInCCLbuffer(const char* group, void** buffer, u64* size)
1669 : {
1670 1 : RPT_INPUT_ERR(
1671 : group == nullptr, "EI0003", std::vector<std::string>({"ccl_op", "value", "parameter", "expect"}),
1672 : std::vector<std::string>({"HcomGetInCCLbuffer", "nullptr", "group", "non-null pointer"}));
1673 1 : CHK_PTR_NULL(group);
1674 1 : CHK_PTR_NULL(buffer);
1675 1 : CHK_PTR_NULL(size);
1676 :
1677 1 : HcclResult ret = HcomCheckGroupName(group);
1678 1 : RPT_INPUT_ERR(
1679 : ret != HCCL_SUCCESS, "EI0003", std::vector<std::string>({"ccl_op", "value", "parameter", "expect"}),
1680 : std::vector<std::string>(
1681 : {"HcomGetInCCLbuffer",
1682 : {group, strnlen(group, GROUP_NAME_MAX_LEN + 1)},
1683 : "group",
1684 : "a non-empty string of length 1 to " + std::to_string(GROUP_NAME_MAX_LEN)
1685 : + ", containing only alphanumeric characters and underscores"}));
1686 1 : CHK_PRT_RET(
1687 : ret != HCCL_SUCCESS,
1688 : HCCL_ERROR("[Get][HcomGetInCCLbuffer]errNo[0x%016llx] group name is invalid", HCOM_ERROR_CODE(ret)), ret);
1689 :
1690 1 : DevType devType = HcomGetDeviceType();
1691 1 : if (devType == DevType::DEV_TYPE_950 || devType == DevType::DEV_TYPE_960) {
1692 : #if (!defined(HCCD)) && (!defined(CCL_KERNEL_AICPU))
1693 0 : HCCLV2_FUNC_RUN(HcomGetInCclBufV2(group, *buffer, *size));
1694 : #endif
1695 0 : return HCCL_SUCCESS;
1696 : }
1697 :
1698 1 : std::shared_ptr<hccl::hcclComm> hcclComm;
1699 1 : CHK_RET(HcomGetCommByGroup(group, hcclComm));
1700 1 : CHK_RET(hcclComm->GetInCCLbuffer(*buffer, *size));
1701 1 : return HCCL_SUCCESS;
1702 1 : }
1703 :
1704 1 : HcclResult HcomGetOutCCLbuffer(const char* group, void** buffer, u64* size)
1705 : {
1706 1 : RPT_INPUT_ERR(
1707 : group == nullptr, "EI0003", std::vector<std::string>({"ccl_op", "value", "parameter", "expect"}),
1708 : std::vector<std::string>({"HcomGetOutCCLbuffer", "nullptr", "group", "non-null pointer"}));
1709 1 : CHK_PTR_NULL(group);
1710 1 : CHK_PTR_NULL(buffer);
1711 1 : CHK_PTR_NULL(size);
1712 :
1713 1 : HcclResult ret = HcomCheckGroupName(group);
1714 1 : RPT_INPUT_ERR(
1715 : ret != HCCL_SUCCESS, "EI0003", std::vector<std::string>({"ccl_op", "value", "parameter", "expect"}),
1716 : std::vector<std::string>(
1717 : {"HcomGetOutCCLbuffer",
1718 : {group, strnlen(group, GROUP_NAME_MAX_LEN + 1)},
1719 : "group",
1720 : "a non-empty string of length 1 to " + std::to_string(GROUP_NAME_MAX_LEN)
1721 : + ", containing only alphanumeric characters and underscores"}));
1722 1 : CHK_PRT_RET(
1723 : ret != HCCL_SUCCESS,
1724 : HCCL_ERROR("[Get][HcomGetOutCCLbuffer]errNo[0x%016llx] group name is invalid", HCOM_ERROR_CODE(ret)), ret);
1725 :
1726 1 : DevType devType = HcomGetDeviceType();
1727 1 : if (devType == DevType::DEV_TYPE_950 || devType == DevType::DEV_TYPE_960) {
1728 : #if (!defined(HCCD)) && (!defined(CCL_KERNEL_AICPU))
1729 0 : HCCLV2_FUNC_RUN(HcomGetOutCclBufV2(group, *buffer, *size));
1730 : #endif
1731 0 : return HCCL_SUCCESS;
1732 : }
1733 :
1734 1 : std::shared_ptr<hccl::hcclComm> hcclComm;
1735 1 : CHK_RET(HcomGetCommByGroup(group, hcclComm));
1736 1 : CHK_RET(hcclComm->GetOutCCLbuffer(*buffer, *size));
1737 1 : return HCCL_SUCCESS;
1738 1 : }
1739 :
1740 1 : HcclResult HcomGetAicpuOpStreamNotify(const char* group, HcclRtStream* opStream, u8 aicpuNotifyNum, void** aicpuNotify)
1741 : {
1742 1 : RPT_INPUT_ERR(
1743 : group == nullptr, "EI0003", std::vector<std::string>({"ccl_op", "value", "parameter", "expect"}),
1744 : std::vector<std::string>({"HcomGetDevType", "nullptr", "group", "non-null pointer"}));
1745 1 : CHK_PTR_NULL(group);
1746 1 : CHK_PTR_NULL(opStream);
1747 1 : CHK_PTR_NULL(aicpuNotify);
1748 :
1749 1 : HcclResult ret = HcomCheckGroupName(group);
1750 1 : RPT_INPUT_ERR(
1751 : ret != HCCL_SUCCESS, "EI0003", std::vector<std::string>({"ccl_op", "value", "parameter", "expect"}),
1752 : std::vector<std::string>(
1753 : {"HcomGetDevType",
1754 : {group, strnlen(group, GROUP_NAME_MAX_LEN + 1)},
1755 : "group",
1756 : "a non-empty string of length 1 to " + std::to_string(GROUP_NAME_MAX_LEN)
1757 : + ", containing only alphanumeric characters and underscores"}));
1758 1 : CHK_PRT_RET(
1759 : ret != HCCL_SUCCESS,
1760 : HCCL_ERROR("[Get][HcomGetDevType]errNo[0x%016llx] group name is invalid", HCOM_ERROR_CODE(ret)), ret);
1761 :
1762 1 : std::shared_ptr<hccl::hcclComm> hcclComm;
1763 1 : CHK_RET(HcomGetCommByGroup(group, hcclComm));
1764 1 : CHK_RET(hcclComm->GetAicpuOpStreamNotify(opStream, aicpuNotifyNum, aicpuNotify));
1765 1 : return HCCL_SUCCESS;
1766 1 : }
1767 :
1768 1 : HcclResult HcomMc2AiCpuStreamAllocAndGet(const char* group, u32 streamMode, rtStream_t* aiCpuStream)
1769 : {
1770 1 : RPT_INPUT_ERR(
1771 : group == nullptr, "EI0003", std::vector<std::string>({"ccl_op", "value", "parameter", "expect"}),
1772 : std::vector<std::string>({"HcomGetDevType", "nullptr", "group", "non-null pointer"}));
1773 1 : CHK_PTR_NULL(group);
1774 1 : CHK_PTR_NULL(aiCpuStream);
1775 :
1776 1 : HcclResult ret = HcomCheckGroupName(group);
1777 1 : RPT_INPUT_ERR(
1778 : ret != HCCL_SUCCESS, "EI0003", std::vector<std::string>({"ccl_op", "value", "parameter", "expect"}),
1779 : std::vector<std::string>(
1780 : {"HcomGetDevType",
1781 : {group, strnlen(group, GROUP_NAME_MAX_LEN + 1)},
1782 : "group",
1783 : "a non-empty string of length 1 to " + std::to_string(GROUP_NAME_MAX_LEN)
1784 : + ", containing only alphanumeric characters and underscores"}));
1785 1 : CHK_PRT_RET(
1786 : ret != HCCL_SUCCESS,
1787 : HCCL_ERROR("[Get][HcomGetDevType]errNo[0x%016llx] group name is invalid", HCOM_ERROR_CODE(ret)), ret);
1788 :
1789 : #if (!defined(HCCD)) && (!defined(CCL_KERNEL_AICPU))
1790 1 : HCCLV2_FUNC_RUN(HcomMc2AiCpuStreamAllocAndGetV2(group, streamMode, aiCpuStream));
1791 : #endif
1792 1 : std::shared_ptr<hccl::hcclComm> hcclComm;
1793 1 : ret = HcomGetCommByGroup(group, hcclComm);
1794 : // 兼容V2，获取通信域失败由外层判断，此处不报ERROR
1795 1 : CHK_PRT_RET(ret != HCCL_SUCCESS, HCCL_WARNING("[%s] HcomGetCommByGroup fail", __func__), ret);
1796 1 : CHK_RET(hcclComm->Mc2AiCpuStreamAllocAndGet(streamMode, *aiCpuStream));
1797 1 : return HCCL_SUCCESS;
1798 1 : }
|
