Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "hccl_communicator_attrs.h"
12 : #include "device_capacity.h"
13 : #include "common/src/config.h"
14 : #include "externalinput_pub.h"
15 : #include "env_config.h"
16 :
17 : using namespace std;
18 :
19 : namespace hccl {
20 0 : HcclResult HcclCommunicatorAttrs::Init(HcclCommParams& params, const RankTable_t& rankTable)
21 : {
22 0 : CHK_RET(InitCommParams(params));
23 0 : CHK_RET(InitRankInfo(rankTable));
24 0 : return HCCL_SUCCESS;
25 : }
26 :
27 500 : HcclResult HcclCommunicatorAttrs::Init(
28 : HcclCommParams& params, const RankTable_t& rankTable,
29 : const std::map<HcclCMDType, std::vector<HcclAlgoType>>& algoConfigMap)
30 : {
31 500 : algoConfigMap_ = algoConfigMap;
32 501 : CHK_RET(InitCommParams(params));
33 501 : CHK_RET(InitRankInfo(rankTable));
34 498 : return HCCL_SUCCESS;
35 : }
36 :
37 0 : HcclResult HcclCommunicatorAttrs::Init(
38 : HcclCommParams& params, const std::vector<RankInfo>& rankList, WorldGroupInfo& groupCommonData)
39 : {
40 0 : CHK_RET(InitCommParams(params));
41 0 : CHK_RET(InitRankInfoSubGroup(rankList, groupCommonData));
42 0 : return HCCL_SUCCESS;
43 : }
44 :
45 33 : HcclResult HcclCommunicatorAttrs::Init(
46 : HcclCommParams& params, const std::vector<RankInfo>& rankList, WorldGroupInfo& groupCommonData,
47 : const std::map<HcclCMDType, std::vector<HcclAlgoType>>& algoConfigMap)
48 : {
49 33 : algoConfigMap_ = algoConfigMap;
50 33 : CHK_RET(InitCommParams(params));
51 33 : CHK_RET(InitRankInfoSubGroup(rankList, groupCommonData));
52 33 : return HCCL_SUCCESS;
53 : }
54 :
55 530 : bool HcclCommunicatorAttrs::IsStandardCard()
56 : {
57 530 : if (Is310P3Common()) {
58 4 : HCCL_INFO("The current device just support this StandardCard case.");
59 4 : return true;
60 : }
61 526 : if (isDiffDeviceType_) {
62 0 : return false;
63 : }
64 : return (
65 526 : (pairLinkInfo_[static_cast<u32>(LinkTypeInServer::HCCS_TYPE)].size() == 0)
66 297 : && (pairLinkInfo_[static_cast<u32>(LinkTypeInServer::HCCS_SW_TYPE)].size() == 0)
67 821 : && (pairLinkInfo_[static_cast<u32>(LinkTypeInServer::SIO_TYPE)].size() == 0));
68 : }
69 :
70 530 : bool HcclCommunicatorAttrs::Is310PDuoCard()
71 : {
72 530 : return (Is310P3Common() && (pairLinkInfo_[static_cast<u32>(LinkTypeInServer::HCCS_TYPE)].size() == userRankSize_));
73 : }
74 :
75 4 : bool HcclCommunicatorAttrs::IsCommon310P3DUO(const std::vector<RankInfo_t>& rankList)
76 : {
77 4 : std::vector<u32> devIdList;
78 4 : std::vector<std::vector<u32>> checkDevList;
79 4 : checkDevList.resize(FACTOR_NUM_TWO);
80 :
81 20 : for (RankInfo_t rankInfo : rankList) {
82 16 : u32 curId = rankInfo.deviceInfo.devicePhyId;
83 16 : devIdList.push_back(curId);
84 16 : }
85 4 : if (devIdList.size() == DEVICE_PER_MODULE) {
86 0 : return true;
87 : }
88 4 : std::sort(devIdList.begin(), devIdList.end());
89 20 : for (u32 i = 0; i < devIdList.size(); i++) {
90 16 : if (devIdList[i] % FACTOR_NUM_TWO == 0) {
91 8 : checkDevList[0].push_back(devIdList[i]); // 主die
92 : } else {
93 8 : checkDevList[1].push_back(devIdList[i]); // 从die
94 : }
95 : }
96 4 : if (devIdList.size() == (DEVICE_PER_MODULE / FACTOR_NUM_TWO) && checkDevList[0].size() == checkDevList[1].size()) {
97 4 : return ((checkDevList[1][0] - checkDevList[0][0]) == 1) && ((checkDevList[1][1] - checkDevList[0][1]) != 0);
98 : } else {
99 0 : return false;
100 : }
101 : return false;
102 4 : }
103 :
104 1998 : bool HcclCommunicatorAttrs::CompareWithUserRank(const RankInfo& left, const RankInfo& right)
105 : {
106 1998 : return left.userRank < right.userRank;
107 : }
108 :
109 1496 : HcclResult HcclCommunicatorAttrs::CheckDeviceType(const DevType deviceType) const
110 : {
111 1496 : if ((deviceType >= DevType::DEV_TYPE_COUNT) || (deviceType < DevType::DEV_TYPE_910)) {
112 0 : HCCL_ERROR(
113 : "[Check][DeviceType]errNo[0x%016llx] device Type[%d] out of range[%d, %d]", HCCL_ERROR_CODE(HCCL_E_PARA),
114 : deviceType, DevType::DEV_TYPE_910, DevType::DEV_TYPE_NOSOC);
115 0 : return HCCL_E_PARA;
116 : }
117 1496 : HCCL_INFO("[HcclCommunicatorAttrs][CheckDeviceType] CheckDeviceType done");
118 1495 : return HCCL_SUCCESS;
119 : }
120 :
121 1496 : HcclResult HcclCommunicatorAttrs::GetNicInfo(
122 : const NICDeployment& nicDeploy, const u32 curRankIndex, const std::vector<RankInfo_t>& servRankList,
123 : RankInfo& rankInfo) const
124 : {
125 1496 : CHK_PRT_RET(
126 : servRankList.empty(),
127 : HCCL_ERROR("[Get][NicInfo]errNo[0x%016llx] server rank list is empty", HCCL_ERROR_CODE(HCCL_E_PARA)),
128 : HCCL_E_PARA);
129 1493 : CHK_PRT_RET(
130 : curRankIndex >= servRankList.size(),
131 : HCCL_ERROR(
132 : "[Get][NicInfo]rankindex[%u] invalid,rank list "
133 : "size is[%zu]",
134 : curRankIndex, servRankList.size()),
135 : HCCL_E_PARA);
136 1493 : rankInfo.nicDeploy = nicDeploy;
137 1493 : const RankInfo_t& curRankInfo = servRankList[curRankIndex];
138 1493 : if (nicDeploy == NICDeployment::NIC_DEPLOYMENT_HOST && curRankInfo.deviceInfo.deviceIp.size() == 0) {
139 : // 检查网卡个数
140 : // 网卡挂载位置在host时，按rank index从网卡列表中获取
141 0 : rankInfo.nicIp.push_back(curRankInfo.hostIp);
142 : } else {
143 1493 : CHK_PRT_RET(
144 : curRankInfo.deviceInfo.deviceIp.size() == 0,
145 : HCCL_ERROR("[Get][NicInfo]rankindex[%u] invalid,deviceIp is zero", curRankIndex), HCCL_E_PARA);
146 1495 : rankInfo.nicIp.push_back(curRankInfo.deviceInfo.deviceIp[0]);
147 1494 : if (curRankInfo.deviceInfo.backupDeviceIp.size() == 0) {
148 1494 : HcclIpAddress invalidAddr;
149 1495 : rankInfo.backupNicIp.push_back(invalidAddr);
150 1493 : } else {
151 0 : rankInfo.backupNicIp.push_back(curRankInfo.deviceInfo.backupDeviceIp[0]);
152 : }
153 1496 : rankInfo.deviceNicPort = curRankInfo.deviceInfo.port;
154 1496 : rankInfo.deviceVnicPort = curRankInfo.deviceInfo.vnicPort;
155 1496 : rankInfo.backupDevicePort = curRankInfo.deviceInfo.backupPort;
156 1496 : HCCL_INFO(
157 : "[Get][NicInfo]serverId[%s], serverIdx[%u], rankIndex[%u], nicIp[%s], backupNicIp[%s], "
158 : "deviceNicPort[%u], deviceVnicPort[%u], backupDevicePort[%u]",
159 : rankInfo.serverId.c_str(), rankInfo.serverIdx, curRankIndex, rankInfo.nicIp[0].GetReadableIP(),
160 : rankInfo.backupNicIp[0].GetReadableIP(), rankInfo.deviceNicPort, rankInfo.deviceVnicPort,
161 : rankInfo.backupDevicePort);
162 : }
163 1496 : HCCL_INFO("[HcclCommunicatorAttrs][GetNicInfo] GetNicInfo done");
164 1496 : return HCCL_SUCCESS;
165 : }
166 :
167 : // private
168 534 : HcclResult HcclCommunicatorAttrs::InitCommParams(HcclCommParams& params)
169 : {
170 534 : userRank_ = params.rank;
171 534 : realUserRank_ = params.userRank;
172 534 : userRankSize_ = params.totalRanks;
173 534 : deviceLogicId_ = params.logicDevId;
174 534 : deviceType_ = params.deviceType;
175 :
176 534 : identifier_ = params.identifier;
177 534 : collectiveId_ = params.id.internal;
178 534 : commWorkMode_ = params.commWorkMode;
179 534 : HCCL_DEBUG(
180 : "userRank_: %u realUserRank_: %u userRankSize_: %u deviceLogicId_: %u deviceType_: %u commWorkMode_: %u.",
181 : userRank_, realUserRank_, userRankSize_, deviceLogicId_, deviceType_, commWorkMode_);
182 534 : return HCCL_SUCCESS;
183 : }
184 :
185 501 : HcclResult HcclCommunicatorAttrs::SetServerId(const RankTable_t& rankTable)
186 : {
187 810 : for (u32 i = 0; i < rankTable.rankList.size(); i++) {
188 808 : if (rankTable.rankList[i].rankId == userRank_) {
189 499 : serverId_ = rankTable.rankList[i].serverId;
190 499 : superPodId_ = rankTable.rankList[i].superPodId;
191 499 : superDeviceId_ = rankTable.rankList[i].superDeviceId;
192 499 : break;
193 : }
194 : }
195 :
196 501 : if (serverId_.empty()) {
197 2 : HCCL_ERROR("[Set][ServerId]SetServerId fail");
198 2 : return HCCL_E_PARA;
199 : }
200 499 : HCCL_INFO("[HcclCommunicatorAttrs][SetServerId] SetServerId done");
201 499 : return HCCL_SUCCESS;
202 : }
203 :
204 532 : HcclResult HcclCommunicatorAttrs::SetServerNum(const std::vector<RankInfo_t>& ranks)
205 : {
206 532 : std::vector<std::string> serverIds;
207 2185 : for (u32 index = 0; index < ranks.size(); index++) {
208 1653 : std::vector<std::string>::iterator found = find(serverIds.begin(), serverIds.end(), ranks[index].serverId);
209 1653 : if (found == serverIds.end()) {
210 758 : serverIds.push_back(ranks[index].serverId);
211 : }
212 : }
213 532 : serverNum_ = serverIds.size();
214 532 : HCCL_INFO("[HcclCommunicatorAttrs][SetServerNum] SetServerNum done");
215 532 : return HCCL_SUCCESS;
216 532 : }
217 :
218 499 : HcclResult HcclCommunicatorAttrs::SetInnerServerAverageDevice(const RankTable_t& rankTable)
219 : {
220 499 : deviceNumPerServer_ = 0;
221 499 : bool isConnectedWithHCCS = true;
222 499 : LinkTypeInServer linkType = LinkTypeInServer::HCCS_TYPE;
223 1995 : for (u32 i = 0; i < rankTable.rankList.size(); i++) {
224 : // 同一server的标识IP 是一样的，所以可以以此推算出平均dev个数
225 1496 : if (rankTable.rankList[i].deviceInfo.devicePhyId == HOST_DEVICE_ID && isHaveCpuRank_ != true) {
226 0 : isHaveCpuRank_ = true;
227 : }
228 1496 : if (serverId_ == rankTable.rankList[i].serverId
229 1496 : && rankTable.rankList[i].deviceInfo.devicePhyId != HOST_DEVICE_ID) {
230 1224 : deviceNumPerServer_++;
231 : } else {
232 272 : continue;
233 : }
234 1224 : if (Is310PDevice()) {
235 0 : continue;
236 : }
237 1224 : CHK_RET(GetPairDeviceLinkType(rankTable, i, isConnectedWithHCCS, linkType));
238 : }
239 499 : if (deviceType_ == DevType::DEV_TYPE_910B && !isConnectedWithHCCS) {
240 0 : deviceNumPerAggregation_ = deviceNumPerServer_ / FACTOR_NUM_TWO;
241 : } else {
242 499 : deviceNumPerAggregation_ = deviceNumPerServer_;
243 : }
244 499 : return HCCL_SUCCESS;
245 : }
246 :
247 : // sub group适配获取server内设配数
248 33 : HcclResult HcclCommunicatorAttrs::SetInnerServerAverageDevice(const std::vector<RankInfo>& rankList)
249 : {
250 33 : deviceNumPerServer_ = 0;
251 33 : bool isConnectedWithHCCS = true;
252 33 : LinkTypeInServer linkType = LinkTypeInServer::HCCS_TYPE;
253 190 : for (u32 i = 0; i < rankList.size(); i++) {
254 : // 同一server的标识IP 是一样的，所以可以以此推算出平均dev个数
255 157 : if (serverId_ == rankList[i].serverId && rankList[i].devicePhyId != HOST_DEVICE_ID) {
256 118 : deviceNumPerServer_++;
257 : } else {
258 39 : continue;
259 : }
260 118 : if (Is310PDevice() || isHaveCpuRank_) {
261 : // 异构场景无需获取链路类型并校验
262 0 : continue;
263 : }
264 419 : for (u32 j = i + 1; j < rankList.size(); j++) {
265 301 : if (rankList[i].serverId == rankList[j].serverId) {
266 262 : CHK_RET(hrtGetPairDeviceLinkType(rankList[i].devicePhyId, rankList[j].devicePhyId, linkType));
267 : }
268 301 : if (linkType != LinkTypeInServer::HCCS_TYPE) {
269 0 : isConnectedWithHCCS = false;
270 : }
271 : }
272 : }
273 33 : if (deviceType_ == DevType::DEV_TYPE_910B && !isConnectedWithHCCS) {
274 0 : deviceNumPerAggregation_ = deviceNumPerServer_ / FACTOR_NUM_TWO;
275 : } else {
276 33 : deviceNumPerAggregation_ = deviceNumPerServer_;
277 : }
278 33 : return HCCL_SUCCESS;
279 : }
280 :
281 533 : HcclResult HcclCommunicatorAttrs::TransformRankInfoByServerId(
282 : const std::vector<RankInfo_t>& rankList, ServRankInfo& servRankInfo) const
283 : {
284 2186 : for (size_t index = 0; index < rankList.size(); ++index) {
285 1653 : const RankInfo_t& rankInfo = rankList[index];
286 1653 : std::string serverId = SalTrim(rankInfo.serverId);
287 1653 : ServRankInfo::iterator itr = servRankInfo.find(serverId);
288 1653 : if (itr != servRankInfo.end()) {
289 895 : itr->second.push_back(rankInfo);
290 : } else {
291 758 : std::vector<RankInfo_t> rankInfoList;
292 758 : rankInfoList.push_back(rankInfo);
293 758 : std::pair<std::string, std::vector<RankInfo_t>> rankInfoPair(serverId, rankInfoList);
294 758 : servRankInfo.insert(rankInfoPair);
295 758 : }
296 1653 : }
297 : // 每个server下的rank列表按 设备Id 从小到大的顺序排序
298 1290 : for (auto& iter : servRankInfo) {
299 757 : std::sort(iter.second.begin(), iter.second.end(), CompareWithDevicePhyId);
300 : }
301 533 : return HCCL_SUCCESS;
302 : }
303 :
304 1788 : bool HcclCommunicatorAttrs::CompareWithDevicePhyId(const RankInfo_t& left, const RankInfo_t& right)
305 : {
306 1788 : return left.deviceInfo.devicePhyId < right.deviceInfo.devicePhyId;
307 : }
308 :
309 534 : HcclResult HcclCommunicatorAttrs::SetModuleInfo(const std::vector<RankInfo_t>& rankList)
310 : {
311 534 : isDiffDeviceModule_ = IsDiffDeviceModule(rankList);
312 531 : multiModuleDiffDeviceNumMode_ = false;
313 531 : moduleNum_ = serverNum_;
314 :
315 531 : std::map<u32, std::vector<RankInfo_t>> moduleMap;
316 2202 : for (RankInfo_t rankInfo : rankList) {
317 1665 : u32 moduleIdx = INVALID_UINT;
318 1665 : CHK_RET(GetModuleIdx(rankInfo, moduleIdx)); // 这里不判断混合组网，只提取每个server实际的moduleidx
319 1665 : if (static_cast<s32>(rankInfo.deviceInfo.devicePhyId) == HOST_DEVICE_ID) {
320 0 : continue;
321 : }
322 1665 : auto iter = moduleMap.find(moduleIdx);
323 1662 : if (iter == moduleMap.end()) {
324 761 : std::vector<RankInfo_t> rankInfoList;
325 761 : rankInfoList.push_back(rankInfo);
326 762 : moduleMap.insert(std::make_pair(moduleIdx, rankInfoList));
327 760 : } else {
328 901 : iter->second.push_back(rankInfo);
329 : }
330 1664 : }
331 535 : if (moduleMap.size() == 0) {
332 0 : return HCCL_SUCCESS;
333 : }
334 :
335 534 : std::vector<u32> moduleDeviceNumVec;
336 :
337 534 : moduleNum_ = moduleMap.size();
338 534 : u32 preDeviceNum = moduleMap.begin()->second.size();
339 534 : u32 curDeviceNum = preDeviceNum;
340 534 : std::vector<u32> devicePhyIdInfoList;
341 1297 : for (auto& moduleInfo : moduleMap) {
342 763 : curDeviceNum = moduleInfo.second.size();
343 763 : if (curDeviceNum != preDeviceNum) {
344 17 : multiModuleDiffDeviceNumMode_ = true;
345 : }
346 :
347 763 : moduleDeviceNumVec.push_back(curDeviceNum);
348 :
349 763 : HCCL_INFO("module[%d] contains [%d]devices", moduleInfo.first, moduleInfo.second.size());
350 763 : devicePhyIdInfoList.clear();
351 2431 : for (auto& rankInfo : moduleInfo.second) {
352 1666 : devicePhyIdInfoList.push_back(rankInfo.deviceInfo.devicePhyId);
353 1661 : HCCL_INFO(
354 : "moduleIdx[%d] Info: rankId[%d], serverId[%s], serverIdx[%d], devicePhyId[%d]", moduleInfo.first,
355 : rankInfo.rankId, rankInfo.serverId.c_str(), rankInfo.serverIdx, rankInfo.deviceInfo.devicePhyId);
356 : }
357 763 : if (!CheckDoubleRingWithRohTopo(devicePhyIdInfoList)) {
358 479 : isARSDoubleRing_ = false;
359 479 : HCCL_DEBUG("SetModuleInfo isARSDoubleRing[%llu]", isARSDoubleRing_);
360 : }
361 : }
362 :
363 534 : if (isDiffDeviceType_) {
364 0 : gcdDeviceNumPerAggregation_ = CalGCD(moduleDeviceNumVec);
365 0 : multiModuleDiffDeviceNumMode_ = false;
366 0 : deviceNumPerAggregation_ = gcdDeviceNumPerAggregation_;
367 0 : useSuperPodMode_ = false;
368 0 : HCCL_INFO(
369 : "[HcclCommunicatorAttrs][SetModuleInfo]mix mode, set multiModuleDiffDeviceNumMode to false, "
370 : "gcdDeviceNumPerAggregation [%u] deviceNumPerAggregation [%u]",
371 : gcdDeviceNumPerAggregation_, deviceNumPerAggregation_);
372 : }
373 :
374 534 : HCCL_RUN_INFO("different module contains different numbers of cards:[%d]", multiModuleDiffDeviceNumMode_);
375 534 : HCCL_RUN_INFO("different module contains different type of cards:[%d]", isDiffDeviceType_);
376 534 : return HCCL_SUCCESS;
377 534 : }
378 :
379 532 : HcclResult HcclCommunicatorAttrs::SetSuperPodInfo(const std::vector<RankInfo_t>& rankList)
380 : {
381 : // 1.超节点数目 2.超节点间server数是否一致 3.
382 532 : superPodNum_ = 0;
383 532 : multiSuperPodDiffServerNumMode_ = false;
384 532 : multiSuperPodDiffDeviceNumMode_ = false;
385 532 : std::map<std::string, std::set<u32>> superPodToServerNum; // 记录每个超节点中的server数目
386 532 : std::map<std::string, std::vector<RankInfo_t>> superPodToDeviceNum; // 记录每个超节点中的设备(rank)数目
387 601 : for (RankInfo_t rankInfo : rankList) {
388 : // superPodId为空时, 返回超节点数量为0, 按照非超节点模式处理
389 589 : CHK_PRT_RET(
390 : rankInfo.superPodId.empty(),
391 : HCCL_DEBUG(
392 : "ranks[%u] superPodId[%s] is empty, set superPodNum to zero", rankInfo.rankId,
393 : rankInfo.superPodId.c_str()),
394 : HCCL_SUCCESS);
395 :
396 69 : superPodToServerNum[rankInfo.superPodId].insert(rankInfo.serverIdx);
397 69 : auto iter = superPodToDeviceNum.find(rankInfo.superPodId);
398 69 : if (iter == superPodToDeviceNum.end()) {
399 30 : std::vector<RankInfo_t> rankInfoList;
400 30 : rankInfoList.push_back(rankInfo);
401 30 : superPodToDeviceNum.insert(std::make_pair(rankInfo.superPodId, rankInfoList));
402 30 : } else {
403 39 : iter->second.push_back(rankInfo);
404 : }
405 589 : }
406 12 : superPodNum_ = superPodToServerNum.size();
407 12 : std::vector<u32> superPodServerNumVec;
408 12 : u32 preServerNum = superPodToServerNum.begin()->second.size();
409 12 : u32 curServerNum = preServerNum;
410 42 : for (auto superPodItem : superPodToServerNum) {
411 30 : curServerNum = superPodItem.second.size();
412 30 : if (curServerNum != preServerNum) {
413 11 : multiSuperPodDiffServerNumMode_ = true;
414 : }
415 30 : superPodServerNumVec.push_back(curServerNum);
416 30 : HCCL_INFO(
417 : "[Set][SuperPodInfo]SuperPod[%s] contains [%d]servers", superPodItem.first.c_str(),
418 : superPodItem.second.size());
419 30 : }
420 12 : HCCL_RUN_INFO(
421 : "[Set][SuperPodInfo]different surperPod contains different numbers of servers:[%d]",
422 : multiSuperPodDiffServerNumMode_);
423 :
424 42 : for (auto item : superPodToDeviceNum) {
425 30 : u32 curDeviceNum = item.second.size();
426 30 : if (curDeviceNum != superPodToDeviceNum.begin()->second.size()) {
427 11 : multiSuperPodDiffDeviceNumMode_ = true;
428 : }
429 30 : HCCL_INFO("[Set][SuperPodInfo]SuperPod[%s] contains [%d] devices", item.first.c_str(), item.second.size());
430 30 : }
431 :
432 : // 计算最大公约数，切分超节点
433 12 : if (!IsConfigAHCAlgo(algoConfigMap_) && !multiModuleDiffDeviceNumMode_ && multiSuperPodDiffServerNumMode_) {
434 8 : gcdServerNumPerSuperPod_ = CalGCD(superPodServerNumVec);
435 8 : multiSuperPodDiffServerNumMode_ = false; // 取公约数不存在server数不一致场景
436 8 : multiSuperPodDiffDeviceNumMode_ = false; // 切分后各超节点server数一致，device数也一致
437 8 : superPodNum_ = serverNum_ / gcdServerNumPerSuperPod_;
438 8 : HCCL_RUN_INFO(
439 : "[Set][SuperPodInfo] gcdServerNumPerSuperPod[%u] original superPodNum[%u] converted superPodNum[%u]",
440 : gcdServerNumPerSuperPod_, superPodToServerNum.size(), superPodNum_);
441 : }
442 :
443 12 : if (isDiffDeviceType_) {
444 0 : multiSuperPodDiffServerNumMode_ = false;
445 0 : HCCL_RUN_INFO("mix mode, set multiSuperPodDiffServerNumMode to false");
446 : }
447 12 : return HCCL_SUCCESS;
448 532 : }
449 :
450 : // 集群中存在910B A+X时，0-7卡: moduleIdx = 2 * serverIdx; 8-15卡: moduleIdx = 2 * serverIdx + 1
451 : // 集群中不存在910B A+X时，moduleIdx = serverIdx
452 1665 : HcclResult HcclCommunicatorAttrs::GetModuleIdx(const RankInfo_t& rankInfo, u32& moduleIdx)
453 : {
454 1665 : CHK_PRT_RET(
455 : rankInfo.serverIdx == INVALID_UINT,
456 : HCCL_ERROR("serverIdx is invalid:[%u], rankId:[%u]", rankInfo.serverIdx, rankInfo.rankId), HCCL_E_PARA);
457 1665 : CHK_PRT_RET(
458 : deviceType_ == DevType::DEV_TYPE_COUNT,
459 : HCCL_ERROR("deviceType_ is invalid:[%d], rankId:[%u]", deviceType_, rankInfo.rankId), HCCL_E_PARA);
460 1665 : u32 serverIdx = rankInfo.serverIdx;
461 1665 : if (GetRankInfoDevType(rankInfo) == DevType::DEV_TYPE_910B && isDiffDeviceModule_) {
462 17 : moduleIdx = serverIdx * FACTOR_NUM_TWO + rankInfo.deviceInfo.devicePhyId / DEVICE_PER_MODULE;
463 1648 : } else if (isDiffDeviceType_) {
464 0 : moduleIdx = serverIdx * FACTOR_NUM_TWO;
465 : } else {
466 1648 : moduleIdx = serverIdx;
467 : }
468 1665 : CHK_PRT_RET(
469 : moduleIdx == INVALID_UINT,
470 : HCCL_ERROR("GetModuleIdx failed. moduleIdx:[%d], rankId:[%u]", moduleIdx, rankInfo.rankId), HCCL_E_PARA);
471 1665 : return HCCL_SUCCESS;
472 : }
473 :
474 : // 用于标识集群中是否存在 910B A+X形态
475 534 : bool HcclCommunicatorAttrs::IsDiffDeviceModule(const std::vector<RankInfo_t>& rankList) const
476 : {
477 534 : bool minDevice = false;
478 534 : bool maxDevice = false;
479 534 : bool isDiffMeshAggregation = false;
480 534 : if (!isDiffDeviceType_ && (deviceType_ != DevType::DEV_TYPE_910B || rankList.size() == 0)) {
481 366 : HCCL_INFO("[IsDiffDeviceModule] deviceType_[%d], rankList.size[%u]", deviceType_, rankList.size());
482 366 : return false;
483 : }
484 :
485 975 : for (const RankInfo_t& rankInfo : rankList) {
486 809 : if (GetRankInfoDevType(rankInfo) == DevType::DEV_TYPE_910B && !isStandardCard_) {
487 661 : if (rankInfo.deviceInfo.devicePhyId < DEVICE_PER_MODULE) {
488 653 : minDevice = true;
489 : } else {
490 8 : maxDevice = true;
491 : }
492 : }
493 : }
494 165 : if (minDevice && maxDevice) {
495 3 : isDiffMeshAggregation = true;
496 : }
497 165 : return isDiffMeshAggregation;
498 : }
499 :
500 532 : HcclResult HcclCommunicatorAttrs::InitHccsPortNum()
501 : {
502 : DevType deviceType;
503 532 : CHK_RET(hrtGetDeviceType(deviceType));
504 532 : if (deviceType == DevType::DEV_TYPE_910_93) {
505 9 : CHK_RET(hrtGetHccsPortNum(deviceLogicId_, hccsPortNum_));
506 : }
507 532 : return HCCL_SUCCESS;
508 : }
509 :
510 496 : HcclResult HcclCommunicatorAttrs::SetRankInfoList(const RankTable_t& rankTable)
511 : {
512 : // 检查rank table入参正确性
513 496 : CHK_RET(CheckRankTable(rankTable, servRankInfo_));
514 : // 获取芯片类型
515 497 : DevType deviceType = DevType::DEV_TYPE_COUNT;
516 497 : CHK_RET(hrtGetDeviceType(deviceType));
517 :
518 : // 遍历rank table获取rank信息
519 497 : rankInfoList_.clear();
520 1184 : for (auto iter = servRankInfo_.begin(); iter != servRankInfo_.end(); ++iter) {
521 2180 : for (u32 index = 0; index < iter->second.size(); ++index) {
522 1494 : const RankInfo_t& orgRankInfo = iter->second[index];
523 : // 构建comm 使用的rank 信息
524 1494 : RankInfo rankInfo;
525 1496 : rankInfo.userRank = orgRankInfo.rankId;
526 1496 : rankInfo.worldRank = orgRankInfo.rankId;
527 :
528 1496 : rankInfo.deviceType = GetRankInfoDevType(orgRankInfo);
529 1496 : CHK_RET(CheckDeviceType(rankInfo.deviceType));
530 :
531 1495 : if (rankInfo.deviceType != DevType::DEV_TYPE_910B || rankInfo.deviceType != DevType::DEV_TYPE_910_93) {
532 : // 910B、910_93形态不做devicePhyId最大值的判断
533 1495 : CHK_RET(CheckDevPhyId(orgRankInfo.deviceInfo.devicePhyId));
534 : }
535 1495 : rankInfo.devicePhyId = orgRankInfo.deviceInfo.devicePhyId;
536 1495 : rankInfo.deviceNicPort = orgRankInfo.deviceInfo.port;
537 1495 : rankInfo.deviceVnicPort = orgRankInfo.deviceInfo.vnicPort;
538 :
539 1495 : rankInfo.serverId = orgRankInfo.serverId;
540 1496 : rankInfo.serverIdx = orgRankInfo.serverIdx;
541 1496 : rankInfo.hostIp = orgRankInfo.hostIp;
542 1496 : rankInfo.hostPort = orgRankInfo.hostPort;
543 1496 : rankInfo.localRank = orgRankInfo.localRank;
544 1496 : rankInfo.superDeviceId = orgRankInfo.superDeviceId;
545 1496 : if (gcdServerNumPerSuperPod_ > 0) {
546 49 : u32 gcdSuperPodIdx = rankInfo.serverIdx / gcdServerNumPerSuperPod_;
547 49 : rankInfo.superPodId = orgRankInfo.superPodId + "_" + std::to_string(gcdSuperPodIdx);
548 49 : rankInfo.superPodIdx = gcdSuperPodIdx;
549 49 : if (userRank_ == rankInfo.userRank) {
550 8 : HCCL_RUN_INFO(
551 : "[SetRankInfoList] userRank[%u] serverId[%s] serverIdx[%u] original superPodId[%s] "
552 : "superPodIdx[%u] converted superPodId[%s] superPodIdx[%u]",
553 : userRank_, rankInfo.serverId.c_str(), rankInfo.serverIdx, orgRankInfo.superPodId.c_str(),
554 : orgRankInfo.superPodIdx, rankInfo.superPodId.c_str(), rankInfo.superPodIdx);
555 : }
556 : } else {
557 1447 : rankInfo.superPodId = orgRankInfo.superPodId;
558 1447 : rankInfo.superPodIdx = orgRankInfo.superPodIdx;
559 : }
560 1496 : CHK_RET(GetNicInfo(orgRankInfo.deviceInfo.nicDeploy, index, iter->second, rankInfo));
561 1496 : rankInfo.nicIdx.assign(nicList_.begin(), nicList_.end());
562 1495 : rankInfoList_.push_back(rankInfo);
563 1495 : }
564 : }
565 : // 将rank id从小到大的顺序返回
566 499 : CHK_RET(SortRankInfoList());
567 498 : return HCCL_SUCCESS;
568 : }
569 :
570 504 : HcclResult HcclCommunicatorAttrs::CheckRankTable(const RankTable_t& rankTable, const ServRankInfo& servRankInfo)
571 : {
572 : // 检查网卡挂载位置
573 504 : if (CheckNicDeploy(rankTable.nicDeploy, deviceType_) != HCCL_SUCCESS) {
574 1 : HCCL_ERROR(
575 : "[Check][RankTable]errNo[0x%016llx] nicDeploy[%d] out of range[%d, %d]", HCCL_ERROR_CODE(HCCL_E_PARA),
576 : rankTable.nicDeploy, static_cast<int32_t>(NICDeployment::NIC_DEPLOYMENT_HOST),
577 : static_cast<int32_t>(NICDeployment::NIC_DEPLOYMENT_DEVICE));
578 1 : return HCCL_E_PARA;
579 : }
580 :
581 503 : if (Is310PDevice()) {
582 : // 异构场景无需检查server内device个数
583 0 : return HCCL_SUCCESS;
584 : }
585 :
586 503 : if (CheckSuperDeviceId(rankTable) != HCCL_SUCCESS) {
587 0 : HCCL_ERROR(
588 : "[Check][RankTable]errNo[0x%016llx] super_device_id is invalid in ranktable, "
589 : "ranktable config value: rankId[%u], superDeviceId[0x%x]",
590 : HCCL_ERROR_CODE(HCCL_E_PARA), userRank_, superDeviceId_);
591 0 : return HCCL_E_PARA;
592 : }
593 :
594 : // 检查服务器上的设备信息
595 503 : ServRankInfo::const_iterator iterBegin = servRankInfo.begin();
596 502 : u32 devNum = 0;
597 502 : CHK_RET(GetDevNum(iterBegin->second, devNum));
598 :
599 500 : bool multiServerDiffDeviceNumMode = false;
600 1193 : for (ServRankInfo::const_iterator iter = iterBegin; iter != servRankInfo.end(); ++iter) {
601 : // 检测每个服务器内的设备数是否相等，如果不相同即为多server不同卡模式
602 694 : u32 curServerDevNum = 0;
603 694 : CHK_RET(GetDevNum(iter->second, curServerDevNum));
604 693 : if (devNum != curServerDevNum) {
605 17 : HCCL_WARNING(
606 : "[Check][RankTable] devnum isn't same,(serverA:[%s],serverB:[%s])"
607 : "devNum(%u, %u)",
608 : iterBegin->first.c_str(), iter->first.c_str(), devNum, curServerDevNum);
609 17 : multiServerDiffDeviceNumMode = true;
610 : }
611 : }
612 :
613 : // 非多server不同卡模式下，判断实际设备数目和userRank_table中的记录一致
614 501 : if (multiServerDiffDeviceNumMode == false && rankTable.deviceNum != devNum * servRankInfo.size()) {
615 2 : HCCL_WARNING(
616 : "[Check][RankTable]errNo[0x%016llx] devnum isn't same, number in rankTable:[%u], actual:[%llu]",
617 : HCCL_ERROR_CODE(HCCL_E_PARA), rankTable.deviceNum, devNum * servRankInfo.size());
618 2 : return HCCL_E_PARA;
619 : }
620 :
621 : // 910模组：服务器内设备的数目必须是2的次幂,在此check(非模组形态无此限制不check)
622 : // 910B、910_93模组形态未定，服务器内设备的数目校验规则后续补充
623 710 : if (pairLinkInfo_[static_cast<u32>(LinkTypeInServer::HCCS_TYPE)].size() > 0 && devNum > HCCL_DEVICE_NUM_TWO
624 710 : && (deviceType_ != DevType::DEV_TYPE_910B && deviceType_ != DevType::DEV_TYPE_910_93 && !Is310P3Common())) {
625 30 : CHK_PRT_RET(
626 : CheckDevCount(devNum) != HCCL_SUCCESS,
627 : HCCL_ERROR("[Check][RankTable]errNo[0x%016llx] devnum is invalid in server.", HCCL_ERROR_CODE(HCCL_E_PARA)),
628 : HCCL_E_PARA);
629 : }
630 499 : return HCCL_SUCCESS;
631 : }
632 :
633 1496 : HcclResult HcclCommunicatorAttrs::CheckDevPhyId(const s32& devicePhyId) const
634 : {
635 1496 : if (devicePhyId > COMM_MAX_DEVICE_ID && devicePhyId != HOST_DEVICE_ID) {
636 0 : HCCL_ERROR(
637 : "[Check][DevPhyId]errNo[0x%016llx] devicePhyId[%d] out of range[-1, %d]", HCCL_ERROR_CODE(HCCL_E_PARA),
638 : devicePhyId, COMM_MAX_DEVICE_ID);
639 0 : return HCCL_E_PARA;
640 : }
641 1496 : return HCCL_SUCCESS;
642 : }
643 :
644 499 : HcclResult HcclCommunicatorAttrs::SortRankInfoList()
645 : {
646 : // 按rank id从小到大的顺序返回
647 499 : std::sort(rankInfoList_.begin(), rankInfoList_.end(), CompareWithUserRank);
648 :
649 1991 : for (u32 index = 0; index < rankInfoList_.size(); ++index) {
650 1492 : CHK_PRT_RET(
651 : (index != rankInfoList_[index].userRank),
652 : HCCL_ERROR(
653 : "[HcclCommunicatorAttrs][SortRankInfoList]errNo[0x%016llx] index[%u] != rankInfoList.userRank[%u]",
654 : HCCL_ERROR_CODE(HCCL_E_PARA), index, rankInfoList_[index].userRank),
655 : HCCL_E_PARA);
656 : }
657 498 : return HCCL_SUCCESS;
658 : }
659 :
660 504 : HcclResult HcclCommunicatorAttrs::CheckNicDeploy(NICDeployment nicDeploy, DevType deviceType) const
661 : {
662 : (void)deviceType;
663 504 : if (nicDeploy >= NICDeployment::NIC_DEPLOYMENT_RESERVED) {
664 1 : HCCL_ERROR(
665 : "[Check][NicDeploy]errNo[0x%016llx] nicDeploy[%u] out of range[%d, %d]", HCCL_ERROR_CODE(HCCL_E_PARA),
666 : nicDeploy, static_cast<int32_t>(NICDeployment::NIC_DEPLOYMENT_HOST),
667 : static_cast<int32_t>(NICDeployment::NIC_DEPLOYMENT_DEVICE));
668 1 : return HCCL_E_PARA;
669 : }
670 503 : return HCCL_SUCCESS;
671 : }
672 :
673 32 : HcclResult HcclCommunicatorAttrs::CheckDevCount(const u32 devNum)
674 : {
675 32 : if (devNum > HCCL_AISERVER_DEVICE_NUM) {
676 1 : HCCL_ERROR(
677 : "[Check][DevCount]errNo[0x%016llx] devNum[%u] out of range[%u, %u]", HCCL_ERROR_CODE(HCCL_E_PARA), devNum,
678 : 0, HCCL_AISERVER_DEVICE_NUM);
679 1 : return HCCL_E_PARA;
680 : }
681 : // 其他拓扑算法设备数目: 1 server: 1, 2, 4, 8
682 : // n server: 1*n, 2*n, 4*n, 8*n
683 31 : if (!Check2N(devNum)) {
684 14 : RPT_ENV_ERR(
685 : true, "EI0014", std::vector<std::string>({"value", "variable", "expect"}),
686 : std::vector<std::string>({std::to_string(devNum), "devNum", "to be 1, 2 or 4, or a multiple of 8"}));
687 1 : HCCL_ERROR(
688 : "[%s][%s]errNo[0x%016llx] devNum[%u] devNum must be divisible by 8, or equal to 1, 2 or 4",
689 : LOG_KEYWORDS_INIT_GROUP.c_str(), LOG_KEYWORDS_RANKTABLE_CHECK.c_str(), HCCL_ERROR_CODE(HCCL_E_PARA),
690 : devNum);
691 1 : return HCCL_E_PARA;
692 : }
693 30 : return HCCL_SUCCESS;
694 2 : }
695 :
696 32 : bool HcclCommunicatorAttrs::Check2N(u32 num) const
697 : {
698 32 : if (num < 1) {
699 0 : return false;
700 : } else {
701 32 : return ((num & (num - 1)) == 0);
702 : }
703 : }
704 :
705 498 : HcclResult HcclCommunicatorAttrs::SetLocalRankInfo()
706 : {
707 806 : for (u32 i = 0; i < rankInfoList_.size(); i++) {
708 806 : HCCL_DEBUG(
709 : " host ip: %s host port: %u dev phy id: %d.", rankInfoList_[i].hostIp.GetReadableAddress(),
710 : rankInfoList_[i].hostPort, rankInfoList_[i].devicePhyId);
711 806 : if (rankInfoList_[i].userRank == userRank_) {
712 498 : devicePhyId_ = rankInfoList_[i].devicePhyId;
713 498 : devIpAddr_ = rankInfoList_[i].nicIp;
714 498 : devBackupIpAddr_ = rankInfoList_[i].backupNicIp;
715 498 : devBackupPort_ = rankInfoList_[i].backupDevicePort;
716 498 : hostIp_ = rankInfoList_[i].hostIp;
717 498 : hostPort_ = rankInfoList_[i].hostPort;
718 498 : localRank_ = rankInfoList_[i].localRank;
719 498 : HCCL_DEBUG("localRank_[%u].", localRank_);
720 498 : break;
721 : }
722 : }
723 : // 在确定 servRankInfo_ 和 serverId_ 信息后，就完成初始判断
724 498 : HCCL_DEBUG("[HcclCommunicatorAttrs][Init]deviceType[%u].", deviceType_);
725 498 : if (static_cast<s32>(devicePhyId_) == HOST_DEVICE_ID) {
726 0 : HCCL_ERROR("[HcclCommunicatorAttrs][Init]not support cpu rank");
727 0 : return HCCL_E_NOT_SUPPORT;
728 : } else {
729 498 : HCCL_DEBUG("[HcclCommunicatorAttrs][Init]devicePhyId[%u] != HOST_DEVICE_ID", devicePhyId_);
730 498 : CHK_RET(hrtGetDevice(&deviceLogicId_));
731 : }
732 498 : return HCCL_SUCCESS;
733 : }
734 :
735 33 : HcclResult HcclCommunicatorAttrs::SetLocalRankInfoSubGroup(const std::vector<RankInfo>& rankList)
736 : {
737 33 : rankInfoList_.assign(rankList.begin(), rankList.end());
738 33 : for (u32 i = 0; i < rankInfoList_.size(); i++) {
739 33 : if (rankInfoList_[i].userRank == userRank_) {
740 33 : devIpAddr_ = rankInfoList_[i].nicIp;
741 33 : devBackupIpAddr_ = rankInfoList_[i].backupNicIp;
742 33 : devBackupPort_ = rankInfoList_[i].backupDevicePort;
743 33 : devicePhyId_ = rankInfoList_[i].devicePhyId;
744 33 : superPodId_ = rankInfoList_[i].superPodId;
745 33 : superDeviceId_ = rankInfoList_[i].superDeviceId;
746 33 : hostIp_ = rankInfoList_[i].hostIp;
747 33 : hostPort_ = rankInfoList_[i].hostPort;
748 33 : nicList_.assign(rankInfoList_[i].nicIdx.begin(), rankInfoList_[i].nicIdx.end());
749 33 : nicDeployment_ = rankInfoList_[i].nicDeploy;
750 33 : break;
751 : }
752 : }
753 33 : return HCCL_SUCCESS;
754 : }
755 :
756 524 : HcclResult HcclCommunicatorAttrs::CheckLocalRankInfo()
757 : {
758 2155 : for (u32 i = 0; i < rankInfoList_.size(); ++i) {
759 1631 : if (userRank_ == rankInfoList_[i].userRank) {
760 524 : CHK_PRT_RET(
761 : static_cast<s32>(devicePhyId_) != rankInfoList_[i].devicePhyId,
762 : HCCL_ERROR(
763 : "[Init][Para]errNo[0x%016llx] parameter check failed, "
764 : "userrank[%u] == rankInfoList.userrank[%u], phyid[%d] != rankInfoList.devid[%d]",
765 : HCCL_ERROR_CODE(HCCL_E_PARA), userRank_, rankInfoList_[i].userRank, static_cast<s32>(devicePhyId_),
766 : rankInfoList_[i].devicePhyId),
767 : HCCL_E_PARA);
768 : }
769 : }
770 524 : return HCCL_SUCCESS;
771 : }
772 :
773 91 : u32 HcclCommunicatorAttrs::CalMeshAggRankSize(int halfDevNum) const
774 : {
775 91 : u32 size = INVALID_VALUE_RANKSIZE;
776 182 : for (auto iter = servRankInfo_.begin(); iter != servRankInfo_.end(); ++iter) {
777 91 : u32 aggregationRankSize0 = 0;
778 91 : u32 aggregationRankSize1 = 0;
779 462 : for (u32 index = 0; index < iter->second.size(); ++index) {
780 371 : const RankInfo_t& orgRankInfo = iter->second[index];
781 371 : if (orgRankInfo.deviceInfo.devicePhyId < halfDevNum) {
782 275 : aggregationRankSize0++;
783 : } else {
784 96 : aggregationRankSize1++;
785 : }
786 : }
787 91 : u32 tmpsize = INVALID_VALUE_RANKSIZE;
788 91 : if ((aggregationRankSize0 != 0) && (aggregationRankSize1 != 0)) {
789 24 : tmpsize = aggregationRankSize0;
790 : } else {
791 67 : tmpsize = iter->second.size();
792 : }
793 91 : size = size > tmpsize ? tmpsize : size;
794 : }
795 91 : return size;
796 : }
797 :
798 524 : HcclResult HcclCommunicatorAttrs::SetMeshAggregationRankSize(u32 size)
799 : {
800 524 : HCCL_INFO("[Set][HcclCommunicatorAttrs][MeshAggregationRankSize]set MeshAggregationRankSize[%u].", size);
801 524 : meshAggregationRankSize_ = size;
802 524 : return HCCL_SUCCESS;
803 : }
804 :
805 524 : HcclResult HcclCommunicatorAttrs::CalAndSetMeshAggRankSize()
806 : {
807 524 : u32 size = INVALID_VALUE_RANKSIZE;
808 524 : if ((deviceType_ == DevType::DEV_TYPE_910B) && isDiffDeviceModule_) { // 910B 16p场景
809 0 : size = CalMeshAggRankSize(HCCL_DEVICE_NUM_EIGHT);
810 524 : } else if (deviceType_ == DevType::DEV_TYPE_910) {
811 322 : if (pairLinkInfo_[static_cast<u32>(LinkTypeInServer::HCCS_TYPE)].size() == 0) { // 标卡
812 231 : size = 1;
813 : } else { // 模组
814 91 : size = CalMeshAggRankSize(HCCL_DEVICE_NUM_FOUR);
815 : }
816 : } else { // 910B的8卡、310P 直接返回server内的size数量
817 202 : size = servRankInfo_.begin()->second.size();
818 : }
819 524 : CHK_RET(SetMeshAggregationRankSize(size));
820 524 : return HCCL_SUCCESS;
821 : }
822 :
823 34 : HcclResult HcclCommunicatorAttrs::SetWorldGroupInfo(
824 : std::unordered_map<std::string, std::map<u32, HcclIpAddress>>& phyIdNicInfoMap,
825 : std::vector<RankInfo>& worldRankInfoList, std::vector<u32>& nicRanksPort, std::vector<u32>& vnicRanksPort)
826 : {
827 34 : for (auto& ipInfo : phyIdNicInfoMap) {
828 0 : for (auto& devInfo : ipInfo.second) {
829 0 : rankDevicePhyIdNicInfoMap_[ipInfo.first][devInfo.first] = devInfo.second;
830 0 : HCCL_DEBUG(
831 : "phyIdNicInfoMap print hostIp[%s] devId[%u] devIp[%s]", ipInfo.first.c_str(), devInfo.first,
832 : devInfo.second.GetReadableAddress());
833 : }
834 : }
835 :
836 34 : for (auto& rankInfo : worldRankInfoList) {
837 0 : worldRankInfoList_.push_back(rankInfo);
838 : }
839 :
840 34 : for (auto& port : nicRanksPort) {
841 0 : nicRanksPort_.push_back(port);
842 0 : HCCL_DEBUG("nicRanksPort port[%u]", port);
843 : }
844 35 : for (auto& port : vnicRanksPort) {
845 1 : vnicRanksPort_.push_back(port);
846 1 : HCCL_DEBUG("vnicRanksPort port[%u]", port);
847 : }
848 34 : return HCCL_SUCCESS;
849 : }
850 :
851 : HcclResult
852 33 : HcclCommunicatorAttrs::TransformRankList(const std::vector<RankInfo>& rankListIn, std::vector<RankInfo_t>& rankListOut)
853 : {
854 190 : for (size_t index = 0; index < rankListIn.size(); ++index) {
855 157 : RankInfo_t rankInfoTmp;
856 157 : rankInfoTmp.serverId = rankListIn[index].serverId;
857 157 : rankInfoTmp.deviceInfo.devicePhyId = rankListIn[index].devicePhyId;
858 157 : rankInfoTmp.deviceInfo.deviceType = rankListIn[index].deviceType;
859 157 : rankInfoTmp.serverIdx = rankListIn[index].serverIdx;
860 157 : rankInfoTmp.rankId = rankListIn[index].userRank;
861 157 : rankInfoTmp.hostIp = rankListIn[index].hostIp;
862 157 : rankInfoTmp.hostPort = rankListIn[index].hostPort;
863 157 : rankInfoTmp.localRank = rankListIn[index].localRank;
864 157 : rankInfoTmp.superDeviceId = rankListIn[index].superDeviceId;
865 157 : rankInfoTmp.superPodId = rankListIn[index].superPodId;
866 157 : rankInfoTmp.superPodIdx = rankListIn[index].superPodIdx;
867 157 : rankListOut.push_back(rankInfoTmp);
868 157 : }
869 33 : return HCCL_SUCCESS;
870 : }
871 :
872 722 : bool HcclCommunicatorAttrs::IsEnableRoce()
873 : {
874 : // 910B单机两种使能roce场景：1、a+x同时使用两module 2.标卡
875 722 : bool roceSwitch = IsSupportEnableRoce();
876 722 : bool isInterServerVnic = false;
877 : // 910_93超节点内节点间走HCCS通信 && Vnic建链, 不需要使能NIC
878 722 : if (useSuperPodMode_ && superPodNum_ == 1 && GetExternalInputInterHccsDisable() == false) {
879 0 : isInterServerVnic = true;
880 : }
881 722 : bool ret = (interServer_ && !isInterServerVnic) || roceSwitch;
882 722 : HCCL_INFO(
883 : "IsEnableRoce ret: %d, interServer_: %d, isInterServerVnic: %d, roceSwitch: %d, "
884 : "isSingleMeshAggregation_: %u",
885 : ret, interServer_, isInterServerVnic, roceSwitch, isSingleMeshAggregation_);
886 722 : return ret;
887 : }
888 :
889 : // a+x mesh间需要同时保证ip有效和roce开关打开才能走rdma
890 6 : bool HcclCommunicatorAttrs::IsUsedRdmaLevel0AndIpInvalid()
891 : {
892 6 : u32 nicNum = devIpAddr_.size();
893 6 : bool ipInvalid = true;
894 12 : for (u32 i = 0; i < nicNum; i++) {
895 6 : if (devIpAddr_[i].IsInvalid()) {
896 0 : HCCL_INFO("[Init][Nic]nic num[%u] deviceip is invalid, total nicNum[%u]", i, nicNum);
897 0 : ipInvalid = false;
898 0 : continue;
899 : }
900 : }
901 : // 机间卡数不一致场景下，IP有效情况下就走RDMA
902 : // 机间卡数一致场景下，需环境变量ROCE打开(多机环境下未对IsEnableRoce开关进行控制)且IP有效情况下走RDMA
903 : return (
904 6 : (GetExternalInputIntraRoceSwitch() != 0 || multiModuleDiffDeviceNumMode_ || isDiffDeviceType_) && ipInvalid);
905 : }
906 :
907 722 : bool HcclCommunicatorAttrs::IsSupportEnableRoce()
908 : {
909 : // 910B单机两种使能roce场景：1、a+x同时使用两module 2.标卡
910 722 : bool roceSwitch = false;
911 722 : HCCL_INFO("[HcclCommunicator]IsSupportEnableRoce");
912 722 : if (isDiffDeviceType_) {
913 0 : roceSwitch = true;
914 722 : } else if (deviceType_ == DevType::DEV_TYPE_910B) {
915 672 : roceSwitch = ((GetExternalInputIntraRoceSwitch() != 0) && (!isSingleMeshAggregation_ || isStandardCard_))
916 336 : || multiModuleDiffDeviceNumMode_;
917 386 : } else if (deviceType_ == DevType::DEV_TYPE_910_93) {
918 240 : roceSwitch = multiSuperPodDiffServerNumMode_ || (multiModuleDiffDeviceNumMode_ && superPodNum_ > 1);
919 : } else { // 其他单机场景为了防止用户误用roce开关
920 146 : roceSwitch = isStandardCard_ ? GetExternalInputIntraRoceSwitch() : false;
921 : }
922 722 : return roceSwitch;
923 : }
924 :
925 1282 : void HcclCommunicatorAttrs::GetTopoAttr(HcclTopoAttr& topoAttr)
926 : {
927 1282 : topoAttr.serverNum = serverNum_;
928 1282 : topoAttr.superPodNum = superPodNum_;
929 1282 : topoAttr.moduleNum = moduleNum_;
930 1282 : topoAttr.deviceNumPerServer = deviceNumPerServer_;
931 1282 : topoAttr.deviceNumPerAggregation = deviceNumPerAggregation_;
932 1282 : topoAttr.multiModuleDiffDeviceNumMode = multiModuleDiffDeviceNumMode_;
933 1282 : topoAttr.multiSuperPodDiffServerNumMode = multiSuperPodDiffServerNumMode_;
934 1282 : topoAttr.multiSuperPodDiffDeviceNumMode = multiSuperPodDiffDeviceNumMode_;
935 1282 : topoAttr.meshAggregationRankSize = meshAggregationRankSize_;
936 1282 : topoAttr.isDiffDeviceModule = isDiffDeviceModule_;
937 1282 : topoAttr.isDiffDeviceType = isDiffDeviceType_;
938 1282 : topoAttr.gcdDeviceNumPerAggregation = gcdDeviceNumPerAggregation_;
939 1282 : topoAttr.isSingleMeshAggregation = isSingleMeshAggregation_;
940 1282 : topoAttr.isAllRankSamePlane = isAllRankSamePlane_;
941 1282 : topoAttr.userRank = userRank_;
942 1282 : topoAttr.realUserRank = realUserRank_;
943 1282 : topoAttr.userRankSize = userRankSize_;
944 1282 : topoAttr.devicePhyId = devicePhyId_;
945 1282 : topoAttr.useSuperPodMode = useSuperPodMode_;
946 1282 : topoAttr.deviceLogicId = deviceLogicId_;
947 1282 : topoAttr.deviceType = deviceType_;
948 1282 : topoAttr.isStandardCard = isStandardCard_;
949 1282 : topoAttr.is310PDuoCard = is310PDuoCard_;
950 1282 : topoAttr.isCommon310P3DUO = isCommon310P3DUO_;
951 1282 : topoAttr.hccsPortNum = hccsPortNum_;
952 1282 : topoAttr.nicList = nicList_;
953 1282 : topoAttr.pairLinkCounter = pairLinkCounter_;
954 1282 : topoAttr.pairLinkInfo = pairLinkInfo_;
955 1282 : topoAttr.rankInfoList = rankInfoList_;
956 1282 : topoAttr.isSupportRdmaLite = isSupportRdmaLite_;
957 1282 : topoAttr.isSupportHccsAndSio = isSupportHccsAndSio_;
958 1282 : topoAttr.localNicPort = GetLocalNicPort(NicType::DEVICE_NIC_TYPE);
959 1282 : topoAttr.isNeedInitNic = isNeedInitNic_;
960 1282 : topoAttr.isARSDoubleRing = isARSDoubleRing_;
961 1282 : }
962 :
963 524 : void HcclCommunicatorAttrs::GetAlgoAttr(HcclAlgoAttr& algoAttr)
964 : {
965 524 : algoAttr.isHaveCpuRank = isHaveCpuRank_;
966 524 : algoAttr.inlineReduceSwitchOn = inlineReduceSwitchOn_;
967 524 : algoAttr.isUsedRdmaLevel0 = isUsedRdmaLevel0_;
968 524 : HCCL_INFO("[CollectAlgoAttr]:isUsedRdmaLevel0:[%d]", isUsedRdmaLevel0_);
969 524 : algoAttr.isUsedInterHccsMode = isUsedInterHccsMode_;
970 524 : algoAttr.identifier = identifier_;
971 524 : algoAttr.collectiveId = collectiveId_;
972 524 : algoAttr.nicDeployment = nicDeployment_;
973 524 : algoAttr.commWorkMode = commWorkMode_;
974 524 : algoAttr.commAlgoConfig = algoConfigMap_;
975 524 : }
976 :
977 1282 : u32 HcclCommunicatorAttrs::GetLocalNicPort(NicType nicType)
978 : {
979 1282 : if (nicDeployment_ == NICDeployment::NIC_DEPLOYMENT_HOST) {
980 0 : return GetHostPort(devicePhyId_);
981 : }
982 : // isUseRankPort_在ranksPort初始化时一同配置：1. 异构场景 2. 开启device侧端口配置
983 : // groupRanksPort_为空说明此时处于全局通信域，要从ranksPort_取监听端口；否则取groupRanksPort_
984 1282 : if (nicType == NicType::HOST_NIC_TYPE) {
985 0 : return GetHostPort(devicePhyId_);
986 : }
987 1282 : if (nicType == NicType::VNIC_TYPE && GetExternalInputNpuPortSwitch()) {
988 : // vnic ports仅在开启device侧端口配置时单独配置
989 0 : std::vector<u32>& ranksPorts = groupVnicRanksPort_.empty() ? vnicRanksPort_ : groupVnicRanksPort_;
990 0 : return GetNicPort(devicePhyId_, ranksPorts, userRank_, isUseRankPort_);
991 : } else {
992 : // 1. 开启device侧端口配置时的nic port时使用ranksPorts
993 : // 2. 异构场景使用ranksPorts
994 : // 3. 其余场景场景isUseRankPort_应当为false，使用默认port
995 1282 : std::vector<u32>& ranksPorts = groupNicRanksPort_.empty() ? nicRanksPort_ : groupNicRanksPort_;
996 1282 : return GetNicPort(devicePhyId_, ranksPorts, userRank_, isUseRankPort_);
997 : }
998 : }
999 : } // namespace hccl
|
