Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "topo_info_extractor.h"
12 : #include "externalinput_pub.h"
13 : #include "device_capacity.h"
14 : #include "search_path.h"
15 : #include "comm_base_pub.h"
16 : #include "hccl_impl_pub.h"
17 :
18 :
19 : namespace hccl {
20 :
21 : constexpr u32 SERVER_RANK_SIZE = 8;
22 :
23 520 : TopoInfoExtractor::TopoInfoExtractor(HcclAlgoAttr &algoAttr, HcclTopoAttr &topoAttr, const TopoType topoType)
24 520 : : identifier_(algoAttr.identifier), userRank_(topoAttr.userRank), userRankSize_(topoAttr.userRankSize),
25 520 : topoType_(topoType), deviceType_(topoAttr.deviceType), rankVector_(topoAttr.rankInfoList),
26 1040 : meshAggregationRankSize_(topoAttr.meshAggregationRankSize), isUsedRdmaLevel0_(algoAttr.isUsedRdmaLevel0),
27 520 : isUsedInterHccsMode_(algoAttr.isUsedInterHccsMode), isDiffAggregation_(topoAttr.isDiffDeviceModule),
28 520 : isConfigAHC_(false),
29 520 : isConfigNULL_(false),
30 520 : multiModuleDiffDeviceNumMode_(topoAttr.multiModuleDiffDeviceNumMode),
31 520 : multiSuperPodDiffServerNumMode_(topoAttr.multiSuperPodDiffServerNumMode),
32 520 : multiSuperPodDiffDeviceNumMode_(topoAttr.multiSuperPodDiffDeviceNumMode),
33 520 : isDiffDeviceType_(topoAttr.isDiffDeviceType),
34 520 : gcdDeviceNumPerAggregation_(topoAttr.gcdDeviceNumPerAggregation),
35 1040 : CommPlaneSubGroupVector_(COMM_LEVEL_RESERVED),
36 2080 : CommPlaneVector_(COMM_LEVEL_RESERVED)
37 520 : { };
38 :
39 : #ifdef CCL_LLT
40 : // 为了适配老的LLT框架提供的构造函数
41 11 : TopoInfoExtractor::TopoInfoExtractor(std::string identifier, u32 userRank, u32 userRankSize, TopoType topoType,
42 : DevType deviceType, std::vector<RankInfo>& rankVector, u32 meshAggregationRankSize,
43 : bool isUsedRdmaLevel0, bool isUsedInterHccsMode, bool multiModuleDiffDeviceNumMode, bool multiSuperPodDiffServerNumMode,
44 11 : bool multiSuperPodDiffDeviceNumMode, bool isDiffDeviceType, u32 gcdDeviceNumPerAggregation)
45 11 : : identifier_(identifier), userRank_(userRank), userRankSize_(userRankSize), topoType_(topoType),
46 11 : deviceType_(deviceType), rankVector_(rankVector), meshAggregationRankSize_(meshAggregationRankSize),
47 11 : isUsedRdmaLevel0_(isUsedRdmaLevel0), isUsedInterHccsMode_(isUsedInterHccsMode), isDiffAggregation_(false),
48 11 : multiModuleDiffDeviceNumMode_(multiModuleDiffDeviceNumMode),
49 11 : multiSuperPodDiffServerNumMode_(multiSuperPodDiffServerNumMode),
50 11 : multiSuperPodDiffDeviceNumMode_(multiSuperPodDiffDeviceNumMode),
51 11 : isDiffDeviceType_(isDiffDeviceType), gcdDeviceNumPerAggregation_(gcdDeviceNumPerAggregation),
52 11 : isConfigAHC_(false),
53 11 : isConfigNULL_(false),
54 22 : CommPlaneVector_(COMM_LEVEL_RESERVED),
55 44 : CommPlaneSubGroupVector_(COMM_LEVEL_RESERVED)
56 11 : {};
57 : #endif
58 :
59 1060 : TopoInfoExtractor::~TopoInfoExtractor()
60 531 : {}
61 :
62 523 : HcclResult TopoInfoExtractor::Init(std::map<HcclCMDType, std::vector<HcclAlgoType>> &algoConfig)
63 : {
64 523 : HCCL_INFO(
65 : "factory init:collective id[%s], user rank[%u], user rank size[%u], topo type[%d], device Type[%d], "\
66 : "meshAggregationRankSize[%u]",
67 : identifier_.c_str(), userRank_, userRankSize_, topoType_, deviceType_, meshAggregationRankSize_);
68 :
69 : // 参数有效性校验
70 523 : CHK_RET(CheckInitInfo());
71 :
72 : // 初始化 AHC 相关信息
73 523 : InitAHCConfig(algoConfig);
74 :
75 : // 填充必要数据结构
76 523 : CHK_RET(SetRankInfo());
77 :
78 522 : if (IsGeneralServer() && GetRemoteIsHdc()) {
79 0 : HCCL_INFO("heterog ES ps factory init no need set topoInfo");
80 : } else {
81 : // 设置拓扑信息
82 522 : CHK_RET(SetTopologyInfo());
83 : // 根据拓扑类型以及芯片类型，校验两层拓扑(外层/内层)、单层拓扑的平面个数合法性
84 522 : CHK_RET(CheckPlaneInfo());
85 : }
86 :
87 522 : CHK_RET(SetRankMap());
88 522 : return HCCL_SUCCESS;
89 : }
90 :
91 522 : HcclResult TopoInfoExtractor::SetRankMap()
92 : {
93 : // 构建由UserRank到子通信域的映射
94 522 : subCommRank2UserRank_.resize(static_cast<u32>(COMM_LEVEL_RESERVED));
95 522 : userRank2subCommRank_.resize(static_cast<u32>(COMM_LEVEL_RESERVED));
96 522 : HCCL_DEBUG("[TopoInfoExtractor]SetRankMap begin");
97 :
98 8874 : for (u32 levelIndex = 0; levelIndex < CommPlaneVector_.size(); levelIndex++) {
99 8352 : u32 ringSize = CommPlaneVector_[levelIndex].size();
100 8352 : subCommRank2UserRank_[levelIndex].resize(ringSize);
101 8352 : userRank2subCommRank_[levelIndex].resize(ringSize);
102 14069 : for (u32 ringIndex = 0; ringIndex < ringSize; ringIndex++) {
103 5717 : u32 rankSize = CommPlaneVector_[levelIndex][ringIndex].size();
104 17274 : for (u32 rankIndex = 0; rankIndex < rankSize; rankIndex++) {
105 11557 : u32 userRank = CommPlaneVector_[levelIndex][ringIndex][rankIndex].userRank;
106 11557 : subCommRank2UserRank_[levelIndex][ringIndex][rankIndex] = userRank;
107 11557 : userRank2subCommRank_[levelIndex][ringIndex][userRank] = rankIndex;
108 : }
109 : }
110 : }
111 522 : return HCCL_SUCCESS;
112 : }
113 :
114 1053 : HcclResult TopoInfoExtractor::CheckInitInfo()
115 : {
116 1053 : if (rankVector_.size() == 0) {
117 0 : HCCL_ERROR("[Check][InitInfo]Not support the scenes: rank_vector size is zero");
118 0 : return HCCL_E_PARA; // 没有rank_table的场景直接报错
119 : }
120 :
121 : // 构造函数入参有效性检查:user_rank_size与user_rank_
122 1053 : if (userRankSize_ <= userRank_) {
123 0 : HCCL_ERROR("[Check][InitInfo]userRankSize_[%u] or userRank_[%u] is invalid.", userRankSize_, userRank_);
124 0 : return HCCL_E_PARA;
125 : }
126 :
127 1053 : if (userRankSize_ != rankVector_.size()) {
128 0 : HCCL_ERROR("[Check][InitInfo]userRankSize_[%u] is not equal to rank_vector size[%llu].", userRankSize_,\
129 : rankVector_.size());
130 0 : return HCCL_E_PARA;
131 : }
132 :
133 1053 : bool isParaInvalid = ((topoType_ == TopoType::TOPO_TYPE_RESERVED) || (deviceType_ >= DevType::DEV_TYPE_COUNT));
134 1053 : if (isParaInvalid) {
135 2 : HCCL_ERROR("[Check][InitInfo]Not support the scenes: TopoType[%d] or deviceType[%d] is invalid.",
136 : topoType_, deviceType_);
137 2 : return HCCL_E_PARA;
138 : }
139 :
140 : // 入参组合有效性检查:不支持4P_RING
141 1051 : if ((deviceType_ == DevType::DEV_TYPE_910 || deviceType_ == DevType::DEV_TYPE_910B ||
142 1051 : deviceType_ == DevType::DEV_TYPE_910_93) && (topoType_ == TopoType::TOPO_TYPE_4P_RING)) {
143 1 : HCCL_ERROR("[Check][InitInfo]Not support the scenes: TopoType[%d] with deviceType[%d] is invalid.", topoType_,
144 : deviceType_);
145 1 : return HCCL_E_PARA;
146 : }
147 :
148 1050 : return HCCL_SUCCESS;
149 : }
150 :
151 523 : HcclResult TopoInfoExtractor::SetRankInfo()
152 : {
153 838 : for (u32 index = 0; index < rankVector_.size(); index++) {
154 838 : if (userRank_ == rankVector_[index].userRank) {
155 523 : rankData_ = rankVector_[index];
156 523 : HCCL_INFO("[SetRankInfo]rankData_: userRank[%u], devicePhyId[%d], serverIdx[%u], superPodId[%s], superPodIdx[%u]",
157 : rankData_.userRank, rankData_.devicePhyId, rankData_.serverIdx, rankData_.superPodId.c_str(), rankData_.superPodIdx);
158 523 : break;
159 : }
160 : }
161 :
162 523 : std::set<u32> serverIdxs;
163 523 : std::set<u32> moduleIdxs;
164 2162 : for (u32 index = 0; index < rankVector_.size(); index++) {
165 : // 填充superPodRankMap_, 记录superPodId -> rankInfo
166 1639 : auto itSuperPod = superPodToRank_.find(rankVector_[index].superPodIdx);
167 1639 : if (itSuperPod != superPodToRank_.end()) {
168 1075 : itSuperPod->second.push_back(rankVector_[index]);
169 : } else {
170 564 : std::vector<RankInfo> rankVecTmp;
171 564 : rankVecTmp.push_back(rankVector_[index]);
172 564 : superPodToRank_.insert(std::make_pair(rankVector_[index].superPodIdx, rankVecTmp));
173 564 : }
174 :
175 1639 : u32 moduleIdx = 0;
176 1639 : CHK_RET(GetModuleIdx(rankVector_[index], moduleIdx));
177 1639 : moduleIdxs.insert(moduleIdx);
178 : // 填充serverRankMap_, 只记录本superPod下的serverIdx -> rankInfo
179 1639 : if (rankVector_[index].superPodId == rankData_.superPodId || isDiffDeviceType_) {
180 1588 : auto itServer = serverToRank_.find(moduleIdx);
181 1588 : if (itServer != serverToRank_.end()) { // 存在该服务器内相关rank的对应信息
182 896 : itServer->second.push_back(rankVector_[index]);
183 : } else { // 不存在则新增一条map记录
184 692 : std::vector<RankInfo> rankVecTmp;
185 692 : rankVecTmp.push_back(rankVector_[index]);
186 692 : serverToRank_.insert(std::make_pair(moduleIdx, rankVecTmp));
187 692 : }
188 : }
189 :
190 : // 填充 serverToRankMerge_, server 和 superPod 两层合并的通信域内所有 rank 信息
191 1639 : auto itServer = serverToRankMerge_.find(moduleIdx);
192 1639 : if (itServer != serverToRankMerge_.end()) { // 存在该服务器内相关rank的对应信息
193 898 : itServer->second.push_back(rankVector_[index]);
194 : } else { // 不存在则新增一条map记录
195 741 : std::vector<RankInfo> rankVecTmp;
196 741 : rankVecTmp.push_back(rankVector_[index]);
197 741 : serverToRankMerge_.insert(std::make_pair(moduleIdx, rankVecTmp));
198 741 : }
199 :
200 : // 同一个server内, 记录本rank和其他rank的链路
201 1639 : if (rankVector_[index].serverIdx == rankData_.serverIdx) {
202 1336 : LinkTypeInServer linkType = LinkTypeInServer::RESERVED_LINK_TYPE;
203 1336 : if (rankData_.devicePhyId != rankVector_[index].devicePhyId &&
204 813 : rankData_.devicePhyId != HOST_DEVICE_ID &&
205 2962 : rankVector_[index].devicePhyId != HOST_DEVICE_ID &&
206 813 : topoType_ != TopoType::TOPO_TYPE_HETEROG) {
207 813 : CHK_RET(hrtGetPairDeviceLinkType(rankData_.devicePhyId, rankVector_[index].devicePhyId, linkType));
208 : }
209 1336 : deviceLinkTypeMap_.insert(std::make_pair(rankVector_[index].devicePhyId, linkType));
210 : }
211 :
212 1639 : u32 serverIdx = 0;
213 1639 : CHK_RET(GetServerIdx(rankVector_[index], serverIdx));
214 1639 : serverIdxs.insert(serverIdx);
215 : }
216 :
217 523 : u32 rankNumPerAggregation = userRankSize_ / static_cast<u32>(moduleIdxs.size());
218 523 : if (isDiffDeviceType_) {
219 0 : rankNumPerAggregation = gcdDeviceNumPerAggregation_;
220 0 : HCCL_INFO("[SetRankInfo] isDiffDeviceType[%u] userRankSize[%u] moduleIdxs.size[%u] rankNumPerAggregation[%u]",
221 : isDiffDeviceType_, userRankSize_, moduleIdxs.size(), rankNumPerAggregation);
222 : }
223 :
224 : // 调整每个server内的user_rank排序(server内userRank从小到大,一定连续)
225 1215 : for (auto iterMap = serverToRank_.begin(); iterMap != serverToRank_.end(); iterMap++) {
226 692 : if (!(iterMap->second).empty()) {
227 692 : std::sort(iterMap->second.begin(), iterMap->second.end(), Ascending);
228 : }
229 : }
230 :
231 : // 调整每个superPod内的user_rank排序, 按照serverIdx从小到大、userRank从小到大排序
232 1087 : for (auto iterMap = superPodToRank_.begin(); iterMap != superPodToRank_.end(); iterMap++) {
233 564 : if (!(iterMap->second).empty()) {
234 564 : std::sort(iterMap->second.begin(), iterMap->second.end(), Ascending);
235 : }
236 : }
237 :
238 : // 调整多个 superPod 合并的 user_rank 排序，按照 serverIdx 从小到大、userRank从小到大排序
239 1264 : for (auto iterMap = serverToRankMerge_.begin(); iterMap!= serverToRankMerge_.end(); iterMap++) {
240 741 : if (!(iterMap->second).empty()) {
241 741 : std::sort(iterMap->second.begin(), iterMap->second.end(), Ascending);
242 : }
243 : }
244 :
245 1215 : for (auto it = serverToRank_.begin(); it != serverToRank_.end(); it++) {
246 692 : HCCL_DEBUG("[SetRankInfo][MIX_DEBUG] serverID[%u]", it->first);
247 2280 : for (auto index = it->second.begin(); index != it->second.end(); index++) {
248 1588 : HCCL_DEBUG("[SetRankInfo][MIX_DEBUG] userRank[%u], devicePhyId[%d], serverIdx[%u], superPodId[%s]",
249 : index->userRank, index->devicePhyId, index->serverIdx, index->superPodId.c_str());
250 : }
251 : }
252 :
253 1264 : for (auto it = serverToRankMerge_.begin(); it != serverToRankMerge_.end(); it++) {
254 741 : HCCL_DEBUG("[SetRankInfo][AHC_DEBUG] serverID[%u]", it->first);
255 2380 : for (auto index = it->second.begin(); index != it->second.end(); index++) {
256 1639 : HCCL_DEBUG("[SetRankInfo][AHC_DEBUG] userRank[%u], devicePhyId[%d], serverIdx[%u], superPodId[%s]",
257 : index->userRank, index->devicePhyId, index->serverIdx, index->superPodId.c_str());
258 : }
259 : }
260 523 : HCCL_DEBUG("[SetRankInfo][AHC_DEBUG] rankNumPerAggregation[%u] moduleIdxs.size()=[%u]",
261 : rankNumPerAggregation, moduleIdxs.size());
262 :
263 523 : ranksOneNode_ = { 0, 8, 4, 2, 1, 4, rankNumPerAggregation, 0, rankNumPerAggregation, rankNumPerAggregation};
264 :
265 : // 校验每个server内的设备个数与topo类型的组合是否正确
266 523 : if (topoType_ != TopoType::TOPO_TYPE_COMMON) {
267 485 : CHK_RET(CheckServerInfo());
268 : }
269 : // 校验每个superPod下的device数量相同
270 522 : CHK_RET(CheckSuperPodInfo());
271 :
272 522 : return HCCL_SUCCESS;
273 523 : }
274 :
275 522 : HcclResult TopoInfoExtractor::CheckSuperPodInfo()
276 : {
277 1085 : for (auto iter = superPodToRank_.begin(); iter != superPodToRank_.end(); iter++) {
278 563 : u32 devNum = superPodToRank_.begin()->second.size();
279 563 : u32 curDevNum = iter->second.size();
280 563 : if (devNum != curDevNum) {
281 0 : HCCL_INFO("[Check][SuperPodInfo]devNum[%u] in superPodIdx[%u] is inconsistent with "\
282 : "devNum[%u] in superPodIdx[%u].", devNum, superPodToRank_.begin(),
283 : curDevNum, iter->first);
284 : }
285 : }
286 522 : return HCCL_SUCCESS;
287 : }
288 :
289 485 : HcclResult TopoInfoExtractor::CheckServerInfo()
290 : {
291 : /*
292 : * HCOM模块：
293 : * 1、每个AI server之间的芯片个数必须一致，不一致则报错；
294 : * 2、每个AI server之间的芯片ID（device ID）必须相同（server0里面devID分别是0、1、4、5；server1->server127也必须是相同的）
295 : * ，不一致则报错；
296 : * HCCL API模块：
297 : * 3、校验rank_table传进来devID，与rt_get_device查询到的devID，是否相同，不一致则报错（针对当前设备）
298 : * 因此，上层模块已经校验过的不再重复，本函数仅用于校验每个server内的设备个数与topo类型的组合是否正确
299 : */
300 485 : u32 moduleIdx = 0;
301 485 : CHK_RET(GetModuleIdx(rankData_, moduleIdx));
302 485 : auto iterRank = serverToRank_.find(moduleIdx); // 查询本rank所在服务器
303 485 : bool check = (iterRank == serverToRank_.end());
304 485 : CHK_PRT_RET(check,
305 : HCCL_ERROR("[Check][ServerInfo]can't find serverId[%s] in rank map", rankData_.serverId.c_str()),
306 : HCCL_E_NOT_FOUND);
307 :
308 485 : HcclResult ret = HCCL_SUCCESS;
309 :
310 485 : switch (topoType_) {
311 124 : case TopoType::TOPO_TYPE_NP_MESH:
312 : case TopoType::TOPO_TYPE_4P_MESH:
313 : case TopoType::TOPO_TYPE_2P_MESH:
314 : case TopoType::TOPO_TYPE_1P_MESH: { // 4p_mesh场景下，支持server(4P+4P)和server(4P)+server(4P)，2p_mesh/1p_mesh同理
315 124 : ret = (((iterRank->second).size() == ranksOneNode_[static_cast<u32>(topoType_)]) ||
316 0 : ((iterRank->second).size() == 2 * ranksOneNode_[static_cast<u32>(topoType_)])) // 2表示8P满配走4PMESH算法
317 124 : ? HCCL_SUCCESS
318 : : HCCL_E_UNAVAIL;
319 124 : CHK_PRT_RET(ret != HCCL_SUCCESS,
320 : HCCL_ERROR("[Check][ServerInfo]check server info err:server rank size[%llu], expected "\
321 : "value[%u], topo type[%d]", (iterRank->second).size(), ranksOneNode_[static_cast<u32>(topoType_)],
322 : topoType_), HCCL_E_UNAVAIL);
323 124 : break;
324 : }
325 329 : case TopoType::TOPO_TYPE_NP_SINGLE_RING:
326 329 : ret = ((iterRank->second).size() == ranksOneNode_[static_cast<u32>(topoType_)]) ? HCCL_SUCCESS :
327 : HCCL_E_UNAVAIL;
328 329 : CHK_PRT_RET(ret != HCCL_SUCCESS,
329 : HCCL_ERROR("[Check][ServerInfo]check server info err:server rank size[%llu], expected "\
330 : "less than value[%u], topo type[%d]", (iterRank->second).size(),
331 : ranksOneNode_[static_cast<u32>(topoType_)], topoType_), HCCL_E_UNAVAIL);
332 329 : break;
333 0 : case TopoType::TOPO_TYPE_HETEROG:
334 : case TopoType::TOPO_TYPE_ES_MESH:
335 0 : break;
336 32 : default: { // 8P_RING or 4P_RING
337 32 : ret = ((iterRank->second).size() == ranksOneNode_[static_cast<u32>(topoType_)]) ? HCCL_SUCCESS :
338 : HCCL_E_UNAVAIL;
339 32 : CHK_PRT_RET(ret != HCCL_SUCCESS,
340 : HCCL_ERROR("[Check][ServerInfo]check server info err:server rank size[%llu], expected "\
341 : "value[%u], topo type[%d]", (iterRank->second).size(),
342 : ranksOneNode_[static_cast<u32>(topoType_)], topoType_), HCCL_E_UNAVAIL);
343 31 : break;
344 : }
345 : }
346 :
347 484 : HCCL_INFO("check server info:server rank size[%llu], expected value[%u], topo type[%d]",
348 : (iterRank->second).size(),
349 : ranksOneNode_[static_cast<u32>(topoType_)],
350 : topoType_);
351 484 : return ret;
352 : }
353 :
354 5851 : HcclResult TopoInfoExtractor::GetServerIdx(const RankInfo &rankInfo, u32 &serverIdx) const
355 : {
356 : // 通过ranktable指定集群信息场景，可以调整server在ranktable的排序(serverIdx）来指定server间通信的topo，从优化通信拓扑
357 : // rootInfo初始化场景，会自动收集集群信息，外部无法指定server的排序，可以无视serverIdx，使用serverID来代替
358 : // PS:返回的serverIdx，会影响rankMap_中server的排序，从而影响bridgeRank的选择，优化通信拓扑
359 5851 : CHK_PRT_RET((rankInfo.serverIdx == INVALID_UINT), HCCL_ERROR("server idx is invalid."), HCCL_E_INTERNAL);
360 5851 : serverIdx = rankInfo.serverIdx;
361 5851 : return HCCL_SUCCESS;
362 : }
363 :
364 522 : HcclResult TopoInfoExtractor::SetTopologyInfo()
365 : {
366 522 : CHK_RET(SetTopoDefaultInfo());
367 :
368 522 : CHK_RET(SetTopoInfoForLevel0());
369 522 : CHK_RET(SetTopoInfoForLevel1());
370 522 : CHK_RET(SetTopoInfoForLevel2());
371 522 : CHK_RET(SetTopoInfoForARS());
372 522 : CHK_RET(SetTopoInfoForCombineL1());
373 :
374 522 : CHK_RET(SetAHCSubGroupsAndAlgOption());
375 :
376 : // 是否支持按mesh划分通信拓扑
377 1042 : bool isSupportMeshTopo = meshAggregationRankSize_ > 0 &&
378 520 : userRankSize_ % meshAggregationRankSize_ == 0;
379 522 : if (isSupportMeshTopo) {
380 520 : CHK_RET(SetTopoInfoForMeshL0());
381 520 : CHK_RET(SetTopoInfoForMeshL1());
382 : } else {
383 2 : HCCL_INFO("[Set][TopologyInfo]topo is not support Mesh, meshAggregationRankSize_[%u], userRankSize_[%u]",
384 : meshAggregationRankSize_, userRankSize_);
385 : }
386 522 : CommPlaneVector_[COMM_COMBINE_ORDER].push_back(rankVector_);
387 522 : return HCCL_SUCCESS;
388 : }
389 :
390 522 : HcclResult TopoInfoExtractor::SetTopoDefaultInfo()
391 : {
392 : // 填充combined_rank_vector_:不区分board_type,只生成default单层拓扑
393 522 : std::vector<RankInfo> tmpCombinedVector;
394 :
395 522 : bool incrementFlag = true; // 节点间建链的两个deviceID必须相同(同一个网段平面)，server间需要特殊处理
396 : // 维护topo输出的信息
397 522 : std::string outLogInfo = "userRank/devicePhyId: ";
398 522 : RankInfo tempRankData;
399 :
400 : // 填充combined_rank_vector_的内层vector:combined场景只有一条固定的环
401 1189 : for (auto iterMap = serverToRank_.begin(); iterMap != serverToRank_.end(); iterMap++) {
402 691 : if (!(iterMap->second).empty()) {
403 : // TOPO_TYPE_COMMON为单环拓扑场景，不需要执行如下判断设置流程
404 691 : if (topoType_ != TopoType::TOPO_TYPE_COMMON) {
405 653 : if (((iterMap->second).size() == SERVER_RANK_SIZE) && !IsDiffDeviceModuleInServer()
406 653 : && (topoType_ != TopoType::TOPO_TYPE_HETEROG) && !Is310PDevice()) {
407 48 : CHK_RET(SetTopoDefaultInfoFor8P()); // 服务器内dev个数相同已在hcom层做过校验
408 24 : return HCCL_SUCCESS;
409 : }
410 : }
411 :
412 667 : if (incrementFlag) {
413 1740 : for (u32 incrementIndex = 0; incrementIndex < (iterMap->second).size(); incrementIndex++) {
414 1196 : u32 combinedUserRank = (iterMap->second)[incrementIndex].userRank;
415 1196 : bool checkError = (rankVector_.size() <= combinedUserRank);
416 1196 : CHK_PRT_RET(checkError, HCCL_ERROR("[Set][TopoDefaultInfo]combined userRank[%u] is bigger than "\
417 : "rank vector", combinedUserRank), HCCL_E_INTERNAL);
418 1196 : tempRankData = rankVector_[combinedUserRank];
419 1196 : outLogInfo.append(std::to_string(tempRankData.userRank));
420 1196 : outLogInfo.append("/");
421 1196 : outLogInfo.append(std::to_string(tempRankData.devicePhyId));
422 1196 : outLogInfo.append("; ");
423 1196 : tmpCombinedVector.push_back(tempRankData);
424 : }
425 :
426 544 : incrementFlag = false;
427 : } else {
428 321 : for (u32 decrementIndex = (iterMap->second).size(); decrementIndex > 0; decrementIndex--) {
429 198 : u32 combinedUserRank = (iterMap->second)[decrementIndex - 1].userRank;
430 198 : bool checkError = (rankVector_.size() <= combinedUserRank);
431 198 : CHK_PRT_RET(checkError, HCCL_ERROR("[Set][TopoDefaultInfo]combined userRank[%u] is bigger than "\
432 : "rank vector", combinedUserRank), HCCL_E_INTERNAL);
433 198 : tempRankData = rankVector_[combinedUserRank];
434 198 : outLogInfo.append(std::to_string(tempRankData.userRank));
435 198 : outLogInfo.append("/");
436 198 : outLogInfo.append(std::to_string(tempRankData.devicePhyId));
437 198 : outLogInfo.append("; ");
438 198 : tmpCombinedVector.push_back(tempRankData);
439 : }
440 :
441 123 : incrementFlag = true;
442 : }
443 : }
444 : }
445 498 : if (topoType_ == TopoType::TOPO_TYPE_COMMON) {
446 38 : std::sort(tmpCombinedVector.begin(), tmpCombinedVector.end(), CompareWithUserRankAscend);
447 : }
448 :
449 498 : CommPlaneVector_[COMM_COMBINE].push_back(tmpCombinedVector);
450 498 : return HCCL_SUCCESS;
451 522 : }
452 :
453 522 : HcclResult TopoInfoExtractor::CheckPlaneInfo()
454 : {
455 522 : bool isTopoComm = (topoType_ == TopoType::TOPO_TYPE_COMMON) && (CommPlaneVector_[COMM_COMBINE].size() != 1);
456 522 : CHK_PRT_RET(isTopoComm,
457 : HCCL_ERROR("[Check][PlaneInfo]topo type[%d] and combined plane nub[%llu] are not match",
458 : topoType_, CommPlaneVector_[COMM_COMBINE].size()), HCCL_E_INTERNAL);
459 :
460 546 : bool isTopo8pring = (topoType_ == TopoType::TOPO_TYPE_8P_RING) &&
461 48 : ((CommPlaneVector_[COMM_LEVEL0].size() != meshAggregationRankSize_) ||
462 24 : (ranksOneNode_[static_cast<u32>(topoType_)] != CommPlaneVector_[COMM_LEVEL1].size()));
463 522 : CHK_PRT_RET(isTopo8pring,
464 : HCCL_ERROR("[Check][PlaneInfo]topo type[%d], level0 plane nub[%llu], level1 plane nub[%llu], is not match",
465 : topoType_, CommPlaneVector_[COMM_LEVEL0].size(), CommPlaneVector_[COMM_LEVEL1].size()), HCCL_E_INTERNAL);
466 :
467 536 : bool isTopo2pring = (topoType_ == TopoType::TOPO_TYPE_NP_DOUBLE_RING) &&
468 14 : ((CommPlaneVector_[COMM_LEVEL0].size() != 2) || // 2表示一个节点内通信域里面是否只有2个ring
469 7 : (ranksOneNode_[static_cast<u32>(topoType_)] != CommPlaneVector_[COMM_LEVEL1].size()));
470 522 : CHK_PRT_RET(isTopo2pring,
471 : HCCL_ERROR("[Check][PlaneInfo]topo type[%d], level0 plane nub[%llu], level1 plane nub[%llu], is not match",
472 : topoType_, CommPlaneVector_[COMM_LEVEL0].size(), CommPlaneVector_[COMM_LEVEL1].size()), HCCL_E_INTERNAL);
473 :
474 522 : bool isTopo4pRing = (topoType_ == TopoType::TOPO_TYPE_4P_RING) &&
475 0 : ((CommPlaneVector_[COMM_LEVEL0].size() != 1) || // 1表示一个节点内通信域里面是否只有一个device
476 0 : (ranksOneNode_[static_cast<u32>(topoType_)] != CommPlaneVector_[COMM_LEVEL1].size()));
477 522 : CHK_PRT_RET(isTopo4pRing,
478 : HCCL_ERROR("[Check][PlaneInfo]topo type[%d], level0 plane nub[%llu], level1 plane nub[%llu], is not match",
479 : topoType_, CommPlaneVector_[COMM_LEVEL0].size(), CommPlaneVector_[COMM_LEVEL1].size()), HCCL_E_INTERNAL);
480 :
481 543 : bool isTopo4pMesh = (topoType_ == TopoType::TOPO_TYPE_4P_MESH) &&
482 42 : ((CommPlaneVector_[COMM_LEVEL0].size() != (ranksOneNode_[static_cast<u32>(topoType_)] - 1)) ||
483 21 : (ranksOneNode_[static_cast<u32>(topoType_)] != CommPlaneVector_[COMM_LEVEL1].size()));
484 522 : CHK_PRT_RET(isTopo4pMesh,
485 : HCCL_ERROR("[Check][PlaneInfo]topo type[%d], level0 plane nub[%llu], level1 plane nub[%llu], is not match",
486 : topoType_, CommPlaneVector_[COMM_LEVEL0].size(), CommPlaneVector_[COMM_LEVEL1].size()), HCCL_E_INTERNAL);
487 :
488 625 : bool isTopoNpMesh = (topoType_ == TopoType::TOPO_TYPE_NP_MESH) &&
489 206 : ((CommPlaneVector_[COMM_LEVEL0].size() != (ranksOneNode_[static_cast<u32>(topoType_)] - 1)) ||
490 103 : (ranksOneNode_[static_cast<u32>(topoType_)] != CommPlaneVector_[COMM_LEVEL1].size()));
491 522 : CHK_PRT_RET(isTopoNpMesh,
492 : HCCL_ERROR("[Check][PlaneInfo]topo type[%d], level0 plane nub[%llu], level1 plane nub[%llu], is not match",
493 : topoType_, CommPlaneVector_[COMM_LEVEL0].size(), CommPlaneVector_[COMM_LEVEL1].size()), HCCL_E_INTERNAL);
494 :
495 : // 1表示一个module里面是否只有一个device
496 522 : bool isTopo2pMesh = (topoType_ == TopoType::TOPO_TYPE_2P_MESH) &&
497 0 : ((CommPlaneVector_[COMM_LEVEL0].size() != 1) ||
498 0 : (ranksOneNode_[static_cast<u32>(topoType_)] != CommPlaneVector_[COMM_LEVEL1].size()));
499 522 : CHK_PRT_RET(isTopo2pMesh,
500 : HCCL_ERROR("[Check][PlaneInfo]topo type[%d], level0 plane nub[%llu], level1 plane nub[%llu], is not match",
501 : topoType_, CommPlaneVector_[COMM_LEVEL0].size(), CommPlaneVector_[COMM_LEVEL1].size()), HCCL_E_INTERNAL);
502 :
503 522 : bool isTopo1pMesh = (topoType_ == TopoType::TOPO_TYPE_1P_MESH) &&
504 0 : ((CommPlaneVector_[COMM_LEVEL0].size() != 1) || // 1表示一个节点内通信域里面是否只有一个device
505 0 : (ranksOneNode_[static_cast<u32>(topoType_)] != CommPlaneVector_[COMM_LEVEL1].size()));
506 522 : CHK_PRT_RET(isTopo1pMesh,
507 : HCCL_ERROR("[Check][PlaneInfo]topo type[%d], level0 plane nub[%llu], level1 plane nub[%llu], is not match",
508 : topoType_, CommPlaneVector_[COMM_LEVEL0].size(), CommPlaneVector_[COMM_LEVEL1].size()), HCCL_E_INTERNAL);
509 :
510 1180 : bool isTopoNpSingleRing = (topoType_ == TopoType::TOPO_TYPE_NP_SINGLE_RING) && (!IsDiffDeviceModuleInServer()) &&
511 329 : ((CommPlaneVector_[COMM_LEVEL0].size() != 1) ||
512 329 : (CommPlaneVector_[COMM_LEVEL1].size() != ranksOneNode_[static_cast<u32>(topoType_)]));
513 522 : CHK_PRT_RET(isTopoNpSingleRing,
514 : HCCL_ERROR("[Check][PlaneInfo]topo type[%d], level0 plane nub[%llu], level1 plane nub[%llu], is not match",
515 : topoType_, CommPlaneVector_[COMM_LEVEL0].size(), CommPlaneVector_[COMM_LEVEL1].size()), HCCL_E_INTERNAL);
516 :
517 522 : HCCL_RUN_INFO(
518 : "plane info:topo type[%d], device type[%d], COMM_COMBINE size[%llu], COMM_LEVEL0 size[%llu], COMM_LEVEL1 " \
519 : "size[%llu], COMM_LEVEL2 size[%llu], COMM_MESH_L0 size[%llu], COMM_MESH_L1 size[%llu]",
520 : topoType_, deviceType_, CommPlaneVector_[COMM_COMBINE].size(), CommPlaneVector_[COMM_LEVEL0].size(),
521 : CommPlaneVector_[COMM_LEVEL1].size(), CommPlaneVector_[COMM_LEVEL2].size(),
522 : CommPlaneVector_[COMM_MESH_L0].size(), CommPlaneVector_[COMM_MESH_L1].size());
523 :
524 522 : return HCCL_SUCCESS;
525 : }
526 :
527 522 : HcclResult TopoInfoExtractor::SetTopoInfoForLevel0()
528 : {
529 522 : u32 moduleIdx = 0;
530 522 : CHK_RET(GetModuleIdx(rankData_, moduleIdx));
531 522 : auto iterRank = serverToRank_.find(moduleIdx); // 查询本rank所在服务器
532 522 : bool check = (iterRank == serverToRank_.end());
533 522 : CHK_PRT_RET(check, HCCL_ERROR("[Set][TopoInfoForLevel0]can't find serverId[%s] in rank map",
534 : rankData_.serverId.c_str()), HCCL_E_NOT_FOUND);
535 : // 查询本rank所在服务器的rank数
536 522 : u32 ranksSize = (iterRank->second).size();
537 :
538 522 : multiLevel0Order_.clear();
539 : // 生成mockNicList
540 522 : std::vector<u32> mockNicList;
541 522 : mockNicList.reserve(ranksSize);
542 1856 : for (u32 startIndex = 0; startIndex < ranksSize; startIndex++) {
543 1334 : mockNicList.push_back(startIndex);
544 : }
545 :
546 522 : multiLevel0Order_ = GetRingsOrderByTopoType(ranksSize, topoType_, mockNicList);
547 :
548 522 : HCCL_DEBUG("[TopoInfoExtractor] The ring number is %zu, the rank size is %lu.", multiLevel0Order_.size(), ranksSize);
549 522 : if (multiLevel0Order_.size() == 1) {
550 491 : CHK_RET(SetSingleLevel0());
551 : } else { // 8p-ring/np ring 环场景
552 31 : u32 ringNum = multiLevel0Order_.size();
553 31 : CHK_RET(SetMultiLevel0(ringNum)); // 8P_RING场景下，外层拓扑中有四个环; 910_93场景中适配双环
554 : }
555 :
556 522 : AHCCommSubgroupInit(); // 准备 AHC COMM 场景下的分组
557 :
558 522 : return HCCL_SUCCESS;
559 522 : }
560 :
561 1044 : HcclResult TopoInfoExtractor::SetTopoInfoForLevel1(bool prepareAHC)
562 : {
563 1044 : std::map<u32, std::vector<RankInfo>> &serverToRank =
564 : (prepareAHC) ? serverToRankMerge_ : serverToRank_;
565 :
566 1044 : CommPlane commPlaneLevel1 = (prepareAHC) ? COMM_LEVEL1_AHC : COMM_LEVEL1;
567 :
568 1044 : u32 moduleIdx = 0;
569 1044 : CHK_RET(GetModuleIdx(rankData_, moduleIdx));
570 1044 : auto iterRank = serverToRank.find(moduleIdx); // 查询本rank所在服务器
571 1044 : bool check = (iterRank == serverToRank.end());
572 1044 : CHK_PRT_RET(check, HCCL_ERROR("[Set][TopoInfoForLevel1]can't find serverId[%s] in rank map",
573 : rankData_.serverId.c_str()), HCCL_E_NOT_FOUND);
574 :
575 : u32 ringSize;
576 1044 : if (topoType_ == TopoType::TOPO_TYPE_2P_MESH) { // 2P_MESH在任何情况下，内层拓扑平面始终为2
577 0 : ringSize = ranksOneNode_[static_cast<u32>(topoType_)];
578 1044 : } else if (topoType_ == TopoType::TOPO_TYPE_NP_SINGLE_RING) { // 标卡内层拓扑的环数
579 658 : ringSize = (iterRank->second).size();
580 : } else { // 其他场景下内层拓扑平面为每个module中的device数量
581 386 : ringSize = ranksOneNode_[static_cast<u32>(topoType_)];
582 : }
583 1044 : HCCL_INFO("[Set][TopoInfoForLevel1] topoType_[%u] ringSize[%u]",topoType_, ringSize);
584 :
585 : // 计算每个 level 环的超节点分组，每个环都一致，只计算一次
586 1044 : std::vector<std::string> idsForIndexingVector;
587 1044 : std::map<std::string, std::vector<u32>> aggregatedSubGroup;
588 1044 : bool calcGroupDone = false;
589 :
590 : // 内层拓扑的每层环
591 3252 : for (u32 ringIndex = 0; ringIndex < ringSize; ringIndex++) {
592 2208 : std::vector<RankInfo> tmpBridgeVector;
593 2208 : bool bridgeRankFlag = false;
594 2208 : std::string outLogInfo = ""; // 维护topo输出的信息
595 2208 : outLogInfo.append(" ringIndex: ");
596 2208 : outLogInfo.append(std::to_string(ringIndex));
597 2208 : outLogInfo.append(", ");
598 2208 : outLogInfo.append("userRank/serverId/devicePhyId/nicIp/isBridgeRank: ");
599 :
600 : // 2、填充bridge_rank_vector_的内层vector和is_bridge_vector_
601 2208 : u32 subGroupIndex = 0;
602 4859 : for (auto iterMap = serverToRank.begin(); iterMap != serverToRank.end(); iterMap++) {
603 2651 : if (!(iterMap->second).empty()) {
604 2651 : RankInfo tmpBridgePara;
605 2651 : u32 bridgeUserRank = (iterMap->second)[ringIndex].userRank;
606 2651 : u32 bridgeDevicePhyId = (iterMap->second)[ringIndex].devicePhyId;
607 2651 : std::vector<u32> bridgeNic((iterMap->second)[ringIndex].nicIdx);
608 2651 : bool checkError = (rankVector_.size() <= bridgeUserRank);
609 2651 : CHK_PRT_RET(checkError, HCCL_ERROR("[Set][TopoInfoForLevel1]bridge userRank[%u] is bigger than rank "\
610 : "vector", bridgeUserRank), HCCL_E_INTERNAL);
611 2651 : CHK_RET(SetBridgeLinkInfo(tmpBridgePara, bridgeUserRank));
612 2651 : tmpBridgeVector.push_back(tmpBridgePara);
613 2651 : std::vector<u32>::iterator iterNic = std::find(bridgeNic.begin(), bridgeNic.end(), bridgeDevicePhyId);
614 2651 : if ((bridgeNic.size() == 0) || (iterNic != bridgeNic.end())) {
615 2649 : if (bridgeUserRank == static_cast<u32>(userRank_)) { // 本rank是否为bridge_rank
616 968 : bridgeRankFlag = true;
617 : }
618 : }
619 :
620 2651 : outLogInfo.append(std::to_string(tmpBridgePara.userRank));
621 2651 : outLogInfo.append("/");
622 2651 : outLogInfo.append(tmpBridgePara.serverId);
623 2651 : outLogInfo.append("/");
624 2651 : outLogInfo.append(std::to_string(tmpBridgePara.devicePhyId));
625 2651 : outLogInfo.append("/");
626 2651 : outLogInfo.append(tmpBridgePara.nicIp[0].GetReadableAddress());
627 2651 : outLogInfo.append("/");
628 2651 : outLogInfo.append(std::to_string(bridgeRankFlag));
629 2651 : outLogInfo.append("; ");
630 :
631 : // 环内填充 aggregatedSubGroup superPodId-> subGroupIndex 用于生成分组信息
632 2651 : if (!calcGroupDone && (deviceType_ == DevType::DEV_TYPE_910_93 || deviceType_ == DevType::DEV_TYPE_910B)) {
633 539 : std::string IdForIndexing = (iterMap->second)[ringIndex].superPodId;
634 539 : auto itIndex = aggregatedSubGroup.find(IdForIndexing);
635 539 : if (itIndex != aggregatedSubGroup.end()) {
636 166 : itIndex->second.push_back(subGroupIndex);
637 : } else {
638 373 : idsForIndexingVector.push_back(IdForIndexing);
639 373 : std::vector<u32> subGroup;
640 373 : subGroup.push_back(subGroupIndex);
641 373 : aggregatedSubGroup.insert(std::make_pair(IdForIndexing, subGroup));
642 373 : }
643 539 : HCCL_INFO("[Set][TopoInfoForLevel1] calc subGroup superPodId[%s] subIndex[%u]",
644 : IdForIndexing.c_str(), subGroupIndex);
645 539 : }
646 :
647 2651 : subGroupIndex = subGroupIndex + 1;
648 2651 : }
649 : }
650 :
651 3266 : for (auto it = aggregatedSubGroup.begin(); it != aggregatedSubGroup.end(); it++) {
652 1058 : HCCL_DEBUG("[Set][TopoInfoForLevel1][AHC_DEBUG] superPodId[%s]", it->first.c_str());
653 2385 : for (auto index = it->second.begin(); index != it->second.end(); index++) {
654 1327 : HCCL_DEBUG("[Set][TopoInfoForLevel1][AHC_DEBUG] groupIndex[%u]", (*index));
655 : }
656 : }
657 :
658 4859 : for (auto it = tmpBridgeVector.begin(); it != tmpBridgeVector.end(); it++) {
659 2651 : HCCL_DEBUG("[Set][TopoInfoForLevel1][AHC_DEBUG] ringIndex[%u] tmpBridgevector userRank[%u]", ringIndex, it->userRank);
660 : }
661 :
662 : // 3、填充bridge_rank_vector_、isBridgeVector_
663 2208 : if (!prepareAHC) {
664 1104 : isBridgeVector_.push_back(bridgeRankFlag);
665 : }
666 2208 : CommPlaneVector_[commPlaneLevel1].push_back(tmpBridgeVector);
667 :
668 : // 4、填充当前 level 的通信域内分组信息（用于层次化算法）
669 2208 : if (!calcGroupDone) {
670 968 : std::vector<std::vector<u32>> SubGroups;
671 1341 : for (u32 i = 0; i < idsForIndexingVector.size(); i++) {
672 373 : std::string idForIndexing = idsForIndexingVector[i];
673 373 : SubGroups.push_back(aggregatedSubGroup[idForIndexing]);
674 373 : }
675 968 : CommPlaneSubGroupVector_[commPlaneLevel1].push_back(SubGroups);
676 968 : calcGroupDone = true;
677 968 : }
678 :
679 2208 : HCCL_INFO("SetTopoInfoForLevel1: topoRankInfo[%s]", outLogInfo.c_str());
680 2208 : }
681 :
682 1044 : HCCL_RUN_INFO("SetTopoInfoForLevel1: identifier[%s], userRank[%u], userRankSize[%u], plane size[%u]",
683 : identifier_.c_str(), userRank_, userRankSize_, CommPlaneVector_[commPlaneLevel1].size());
684 1044 : return HCCL_SUCCESS;
685 1044 : }
686 :
687 522 : HcclResult TopoInfoExtractor::SetTopoInfoForLevel2()
688 : {
689 33 : bool isLevel2Support = ((deviceType_ == DevType::DEV_TYPE_910_93) && multiModuleDiffDeviceNumMode_
690 555 : && !multiSuperPodDiffDeviceNumMode_);
691 : // 对称场景需要初始化多个平面，超节点内rank数一致也可切分平面；其他非对称场景 level1 和 level2 合并无需切分平面
692 522 : if (((!multiModuleDiffDeviceNumMode_ && !multiSuperPodDiffServerNumMode_) || isLevel2Support) && !isDiffDeviceType_) {
693 514 : HCCL_INFO("[Set][TopoInfoForLevel2] select origin proc");
694 :
695 : // 找到当前rank在本超节点内部的序号
696 514 : auto it = superPodToRank_.find(rankData_.superPodIdx);
697 514 : CHK_PRT_RET(it == superPodToRank_.end(),
698 : HCCL_ERROR("[Set][TopoInfoForLevel2]superPodIdx[%u] is not exist in superPodRankMap",
699 : rankData_.superPodIdx), HCCL_E_INTERNAL);
700 :
701 514 : u32 index = 0;
702 829 : for (; index < it->second.size(); ++index) {
703 829 : if (userRank_ == it->second[index].userRank) {
704 514 : break;
705 : }
706 : }
707 514 : CHK_PRT_RET(index >= it->second.size(),
708 : HCCL_ERROR("[Set][TopoInfoForLevel2]userRank_[%u] superPodId[%s] superPodIdx[%u] not exist in superPodRankMap",
709 : userRank_, rankData_.superPodId.c_str(), rankData_.superPodIdx), HCCL_E_INTERNAL);
710 :
711 514 : std::vector<RankInfo> tmpRankVec;
712 1069 : for (auto iterMap = superPodToRank_.begin(); iterMap != superPodToRank_.end(); iterMap++) {
713 555 : CHK_PRT_RET(iterMap->second.size() <= index,
714 : HCCL_ERROR("[Set][TopoInfoForLevel2]index[%u] is bigger than rank vector size[%u]",
715 : index, iterMap->second.size()), HCCL_E_INTERNAL);
716 :
717 555 : RankInfo& tempRankData = iterMap->second[index];
718 555 : tmpRankVec.push_back(tempRankData);
719 :
720 : // 维护topo输出的信息
721 555 : std::string outLogInfo = "userRank/devicePhyId/serverIdx/superPodId: ";
722 555 : outLogInfo.append(std::to_string(tempRankData.userRank));
723 555 : outLogInfo.append("/");
724 555 : outLogInfo.append(std::to_string(tempRankData.devicePhyId));
725 555 : outLogInfo.append("/");
726 555 : outLogInfo.append(std::to_string(tempRankData.serverIdx));
727 555 : outLogInfo.append("/");
728 555 : outLogInfo.append(tempRankData.superPodId);
729 555 : outLogInfo.append("; ");
730 555 : HCCL_INFO("SetTopoInfoForLevel2: topoRankInfo[%s]", outLogInfo.c_str());
731 555 : }
732 :
733 514 : CommPlaneVector_[COMM_LEVEL2].push_back(tmpRankVec);
734 514 : HCCL_RUN_INFO("SetTopoInfoForLevel2: identifier[%s], userRank[%u], userRankSize[%u], plane size[%u]",
735 : identifier_.c_str(), userRank_, userRankSize_, CommPlaneVector_[COMM_LEVEL2].size());
736 514 : }
737 :
738 522 : return HCCL_SUCCESS;
739 : }
740 :
741 520 : HcclResult TopoInfoExtractor::SetTopoInfoForMeshL0()
742 : {
743 : // 以MeshAggregation为粒度、MeshAggregation内各设备的mesh建链
744 520 : u32 rankSize = meshAggregationRankSize_;
745 520 : u32 userRankIndexBegin = userRank_ / meshAggregationRankSize_ * meshAggregationRankSize_;
746 520 : u32 userRankIndexEnd = userRankIndexBegin + meshAggregationRankSize_;
747 520 : std::vector<RankInfo> paraVector(rankSize);
748 520 : u32 rankIndex = 0;
749 520 : std::string outLogInfo = "userRank/devicePhyId: "; // 维护topo输出的信息
750 :
751 520 : CHK_PRT_RET(rankVector_.size() < userRankIndexEnd,
752 : HCCL_ERROR("[Set][TopoInfoForMeshL0]rankVector_ size[%u] should be greater than userRankIndexEnd[%u]",
753 : rankVector_.size(), userRankIndexEnd), HCCL_E_PARA);
754 :
755 1744 : for (u32 i = userRankIndexBegin; i < userRankIndexEnd; i ++) {
756 1224 : paraVector[rankIndex] = rankVector_[i];
757 1224 : outLogInfo.append(std::to_string(paraVector[rankIndex].userRank));
758 1224 : outLogInfo.append("/");
759 1224 : outLogInfo.append(std::to_string(paraVector[rankIndex].devicePhyId));
760 1224 : outLogInfo.append("; ");
761 1224 : rankIndex++;
762 : }
763 520 : CommPlaneVector_[COMM_MESH_L0].push_back(paraVector);
764 520 : return HCCL_SUCCESS;
765 520 : }
766 :
767 520 : HcclResult TopoInfoExtractor::SetTopoInfoForMeshL1()
768 : {
769 : // 以MeshAggregation为粒度、MeshAggregation间各平面的mesh建链
770 520 : u32 rankSize = userRankSize_ / meshAggregationRankSize_; // 1 = 7 / 4
771 520 : u32 planeID = userRank_ % meshAggregationRankSize_; // 0
772 520 : std::vector<RankInfo> paraVector(rankSize);
773 :
774 520 : CHK_PRT_RET(rankVector_.size() < userRankSize_,
775 : HCCL_ERROR("[Set][TopoInfoForMeshL1]rankVector_ size[%u] should be greater than userRankSize[%u]",
776 : rankVector_.size(), userRankSize_), HCCL_E_PARA);
777 :
778 1314 : for (u32 i = planeID; i < userRankSize_; i += meshAggregationRankSize_) {
779 794 : u32 rankIndex = i / meshAggregationRankSize_;
780 794 : paraVector[rankIndex] = rankVector_[i];
781 794 : std::string outLogInfo = "userRank/devicePhyId"; // 维护topo输出的信息
782 794 : outLogInfo.append(std::to_string(paraVector[rankIndex].userRank));
783 794 : outLogInfo.append("/");
784 794 : outLogInfo.append(std::to_string(paraVector[rankIndex].devicePhyId));
785 794 : outLogInfo.append("; ");
786 794 : HCCL_INFO("SetTopoInfoForMeshL1: topoRankInfo[%s]", outLogInfo.c_str());
787 794 : }
788 520 : CommPlaneVector_[COMM_MESH_L1].push_back(paraVector);
789 520 : return HCCL_SUCCESS;
790 520 : }
791 :
792 491 : HcclResult TopoInfoExtractor::SetSingleLevel0()
793 : {
794 : // 填充level0_rank_vector_，该函数处理场景非8P_RING算法
795 491 : std::vector<RankInfo> tmpLevel0Vector;
796 491 : u32 moduleIdx = 0;
797 491 : CHK_RET(GetModuleIdx(rankData_, moduleIdx));
798 491 : auto iterRank = serverToRank_.find(moduleIdx); // 查询本rank所在服务器
799 491 : bool check = (iterRank == serverToRank_.end());
800 491 : CHK_PRT_RET(check, HCCL_ERROR("[Set][SingleLevel0]can't find serverId[%s] in rank map", rankData_.serverId.c_str()),
801 : HCCL_E_NOT_FOUND);
802 :
803 491 : std::vector<s32> devicePhyIdVector;
804 1904 : for (u32 i = 0; i < rankVector_.size(); i++) {
805 1413 : devicePhyIdVector.push_back(rankVector_[i].devicePhyId);
806 : }
807 491 : s32 maxPhyId = *max_element(devicePhyIdVector.begin(), devicePhyIdVector.end());
808 : // 8P满配场景:4PMESH算法 + 8Pfullmesh + 16P仅使用左边module
809 515 : if (((iterRank->second).size() == DEVICE_PER_MODULE &&
810 515 : maxPhyId < DEVICE_PER_MODULE) &&
811 24 : (topoType_ == TopoType::TOPO_TYPE_4P_MESH || topoType_ == TopoType::TOPO_TYPE_NP_MESH)) {
812 0 : return SetSingleLevel0For8P(); // 服务器内dev个数相同已在hcom层做过校验
813 : }
814 :
815 : // 维护topo输出的信息
816 491 : std::string outLogInfo = "userRank/devicePhyId: ";
817 491 : RankInfo tempRankData;
818 : // 其他场景 + 16P使用右边module
819 1617 : for (u32 startIndex = 0; startIndex < (iterRank->second).size(); startIndex++) {
820 1126 : u32 level0StartRank = (iterRank->second)[startIndex].userRank;
821 1126 : bool checkError = (rankVector_.size() <= level0StartRank);
822 1126 : CHK_PRT_RET(checkError, HCCL_ERROR("[Set][SingleLevel0]level0 userRank[%u] is bigger than rank vector",
823 : level0StartRank), HCCL_E_INTERNAL);
824 1126 : tempRankData = rankVector_[level0StartRank];
825 :
826 1126 : outLogInfo.append(std::to_string(tempRankData.userRank));
827 1126 : outLogInfo.append("/");
828 1126 : outLogInfo.append(std::to_string(tempRankData.devicePhyId));
829 1126 : outLogInfo.append("; ");
830 1126 : tmpLevel0Vector.push_back(tempRankData);
831 : }
832 :
833 : // NPmesh或4Pmesh场景下，外层拓扑平面为device数量-1
834 491 : u32 level0Size = (topoType_ == TopoType::TOPO_TYPE_4P_MESH || topoType_ == TopoType::TOPO_TYPE_NP_MESH) ?
835 124 : (ranksOneNode_[static_cast<u32>(topoType_)] - 1) : 1;
836 :
837 1226 : for (u32 index = 0; index < level0Size; index++) {
838 735 : CommPlaneVector_[COMM_LEVEL0].push_back(tmpLevel0Vector);
839 : }
840 491 : HCCL_RUN_INFO("SetTopoInfoForLevel0: identifier[%s], userRank[%u], userRankSize[%u], topoRankInfo[%s]",
841 : identifier_.c_str(), userRank_, userRankSize_, outLogInfo.c_str());
842 491 : return HCCL_SUCCESS;
843 491 : }
844 :
845 0 : HcclResult TopoInfoExtractor::SetSingleLevel0For8P()
846 : {
847 : // 该函数处理场景:8P满配、非8P_RING算法(8P满配下走4PMESH)
848 0 : std::vector<RankInfo> tmpLevel0Vector;
849 0 : u32 moduleIdx = 0;
850 0 : CHK_RET(GetModuleIdx(rankData_, moduleIdx));
851 0 : auto iterRank = serverToRank_.find(moduleIdx); // 查询本rank所在服务器
852 0 : bool check = (iterRank == serverToRank_.end());
853 0 : CHK_PRT_RET(check, HCCL_ERROR("[Set][SingleLevel0For8P]can't find serverId[%s] in rank map",
854 : rankData_.serverId.c_str()), HCCL_E_NOT_FOUND);
855 :
856 0 : u32 startIndex = (rankData_.devicePhyId < static_cast<s32>(meshAggregationRankSize_)) ?
857 : 0 : meshAggregationRankSize_;
858 0 : u32 devcount = 0;
859 : // 维护topo输出的信息
860 0 : std::string outLogInfo = "userRank/devicePhyId: ";
861 0 : RankInfo tempRankData;
862 0 : while (devcount < meshAggregationRankSize_) {
863 0 : u32 level0StartRank = (iterRank->second)[startIndex].userRank;
864 0 : bool checkError = (rankVector_.size() <= level0StartRank);
865 0 : CHK_PRT_RET(checkError, HCCL_ERROR("[Set][SingleLevel0For8P]level0 userRank[%u] is bigger than rank vector",
866 : level0StartRank), HCCL_E_INTERNAL);
867 0 : tempRankData = rankVector_[level0StartRank];
868 :
869 0 : outLogInfo.append(std::to_string(tempRankData.userRank));
870 0 : outLogInfo.append("/");
871 0 : outLogInfo.append(std::to_string(tempRankData.devicePhyId));
872 0 : outLogInfo.append("; ");
873 0 : tmpLevel0Vector.push_back(tempRankData);
874 0 : startIndex++;
875 0 : devcount++;
876 : }
877 :
878 : // 4PMESH场景下，外层拓扑3个平面
879 0 : u32 level0Size = ranksOneNode_[static_cast<u32>(topoType_)] -1;
880 0 : for (u32 index = 0; index < level0Size; index++) {
881 0 : CommPlaneVector_[COMM_LEVEL0].push_back(tmpLevel0Vector);
882 : }
883 0 : HCCL_RUN_INFO("SetTopoInfoForLevel0: identifier[%s], userRank[%u], userRankSize[%u], topoRankInfo[%s]",
884 : identifier_.c_str(), userRank_, userRankSize_, outLogInfo.c_str());
885 0 : return HCCL_SUCCESS;
886 0 : }
887 :
888 : /*
889 : * *********************************************************************************
890 : * 用来标识集群中是否存在910B A+X形态
891 : * **********************************************************************************
892 : */
893 4565 : bool TopoInfoExtractor::IsDiffDeviceModuleInServer() const
894 : {
895 4565 : return deviceType_ == DevType::DEV_TYPE_910B && isDiffAggregation_;
896 : }
897 :
898 31 : HcclResult TopoInfoExtractor::SetMultiLevel0(u32 ringNum)
899 : {
900 31 : std::vector<u32> tmpLevel0Order;
901 31 : u32 moduleIdx = 0;
902 31 : RankInfo tempRankData;
903 31 : CHK_RET(GetModuleIdx(rankData_, moduleIdx));
904 31 : auto iterRank = serverToRank_.find(moduleIdx); // 查询本rank所在服务器
905 31 : bool check = (iterRank == serverToRank_.end());
906 31 : CHK_PRT_RET(check, HCCL_ERROR("[Set][MultiLevel0]can't find serverId[%s] in rank map", rankData_.serverId.c_str()),
907 : HCCL_E_NOT_FOUND);
908 :
909 : // 维护topo输出的信息
910 31 : std::string outLogInfo = "";
911 141 : for (u32 ringIndex = 0; ringIndex < ringNum; ringIndex++) {
912 110 : tmpLevel0Order = multiLevel0Order_[ringIndex]; // 获取每一个环的设备物理ID排序
913 110 : std::vector<RankInfo> tmpLevel0Vector;
914 110 : outLogInfo = "userRank/devicePhyId: ";
915 910 : for (u32 startIndex = 0; startIndex < (iterRank->second).size(); startIndex++) {
916 800 : u32 devIndex = tmpLevel0Order[startIndex];
917 800 : u32 level0RingUserank = (iterRank->second)[devIndex].userRank;
918 800 : bool checkError = (rankVector_.size() <= level0RingUserank);
919 800 : CHK_PRT_RET(checkError, HCCL_ERROR("[Set][MultiLevel0]level0 userRank[%u] is bigger than rank vector",
920 : level0RingUserank), HCCL_E_INTERNAL);
921 800 : tempRankData = rankVector_[level0RingUserank];
922 800 : outLogInfo.append(std::to_string(tempRankData.userRank));
923 800 : outLogInfo.append("/");
924 800 : outLogInfo.append(std::to_string(tempRankData.devicePhyId));
925 800 : outLogInfo.append("; ");
926 800 : tmpLevel0Vector.push_back(tempRankData);
927 : }
928 110 : HCCL_RUN_INFO("SetTopoInfoForLevel0: identifier[%s], userRank[%u], userRankSize[%u], topoRankInfo[%s]",
929 : identifier_.c_str(), userRank_, userRankSize_, outLogInfo.c_str());
930 110 : CommPlaneVector_[COMM_LEVEL0].push_back(tmpLevel0Vector);
931 110 : }
932 31 : return HCCL_SUCCESS;
933 31 : }
934 :
935 : // anypath创建通信域
936 0 : HcclResult TopoInfoExtractor::SetMultiLevel0AnyPath(std::vector<std::vector<u32> > multiOrder)
937 : {
938 0 : u32 ringNum = multiOrder.size();
939 0 : std::vector<u32> tmpLevel0Order;
940 0 : u32 moduleIdx = 0;
941 0 : CHK_RET(GetModuleIdx(rankData_, moduleIdx));
942 0 : auto iterRank = serverToRank_.find(moduleIdx); // 查询本rank所在服务器
943 0 : bool check = (iterRank == serverToRank_.end());
944 0 : CHK_PRT_RET(check, HCCL_ERROR("[Set][MultiLevel0]can't find serverId[%s] in rank map", rankData_.serverId.c_str()),
945 : HCCL_E_NOT_FOUND);
946 :
947 : // 维护topo输出的信息
948 0 : std::string outLogInfo = "";
949 0 : RankInfo tempRankData;
950 0 : for (u32 ringIndex = 0; ringIndex < ringNum; ringIndex++) {
951 0 : tmpLevel0Order = multiOrder[ringIndex]; // 获取每一个环的设备物理ID排序
952 0 : std::vector<RankInfo> tmpLevel0Vector;
953 0 : outLogInfo = "userRank/devicePhyId: ";
954 0 : for (u32 startIndex = 0; startIndex < (iterRank->second).size(); startIndex++) {
955 0 : u32 devIndex = tmpLevel0Order[startIndex];
956 0 : u32 level0RingUserank = (iterRank->second)[devIndex].userRank;
957 0 : bool checkError = (rankVector_.size() <= level0RingUserank);
958 0 : CHK_PRT_RET(checkError, HCCL_ERROR("[Set][MultiLevel0]level0 userRank[%u] is bigger than rank vector",
959 : level0RingUserank), HCCL_E_INTERNAL);
960 0 : tempRankData = rankVector_[level0RingUserank];
961 0 : outLogInfo.append(std::to_string(tempRankData.userRank));
962 0 : outLogInfo.append("/");
963 0 : outLogInfo.append(std::to_string(tempRankData.devicePhyId));
964 0 : outLogInfo.append("; ");
965 0 : tmpLevel0Vector.push_back(tempRankData);
966 : }
967 0 : HCCL_RUN_INFO("[AnyPath]SetTopoInfoForLevel0: identifier[%s], userRank[%u], userRankSize[%u], topoRankInfo[%s]",
968 : identifier_.c_str(), userRank_, userRankSize_, outLogInfo.c_str());
969 0 : CommPlaneVector_[COMM_LEVEL0_ANYPATH_SDMA].push_back(tmpLevel0Vector);
970 0 : CommPlaneVector_[COMM_LEVEL0_ANYPATH_RDMA].push_back(tmpLevel0Vector);
971 0 : }
972 :
973 0 : return HCCL_SUCCESS;
974 0 : }
975 :
976 : // 集群中存在910B A+X时，0-7卡: moduleIdx = 2 * serverIdx; 8-15卡: moduleIdx = 2 * serverIdx + 1
977 : // 集群中不存在910B A+X时，moduleIdx = serverIdx
978 4212 : HcclResult TopoInfoExtractor::GetModuleIdx(const RankInfo &rankInfo, u32 &moduleIdx)
979 : {
980 : // 获取moduleIdx，在16P同时使用左右两个module时，moduleIdx标识当前rank所在的module，其他场景下moduleIdx等同于serverIdx
981 4212 : u32 serverIdx = 0;
982 4212 : CHK_RET(GetServerIdx(rankInfo, serverIdx));
983 4212 : if (isDiffDeviceType_) {
984 0 : moduleIdx = rankInfo.userRank / gcdDeviceNumPerAggregation_;
985 0 : HCCL_DEBUG("[TopoInfoExtractor][GetModuleIdx]serverIdx [%u] devicePhyId[%u] userRank[%u] moduleIdx[%u] "
986 : "gcdDeviceNumPerAggregation[%u]", serverIdx, rankInfo.devicePhyId, rankInfo.userRank, moduleIdx,
987 : gcdDeviceNumPerAggregation_);
988 4212 : } else if (IsDiffDeviceModuleInServer()) {
989 0 : moduleIdx = serverIdx * FACTOR_NUM_TWO + rankInfo.devicePhyId / DEVICE_PER_MODULE;
990 : } else {
991 4212 : moduleIdx = serverIdx;
992 : }
993 4212 : return HCCL_SUCCESS;
994 : }
995 :
996 2651 : HcclResult TopoInfoExtractor::SetBridgeLinkInfo(RankInfo &bridgePara, u32 bridgeUserRank)
997 : {
998 2651 : bool checkSize = (rankVector_.size() <= bridgeUserRank);
999 2651 : CHK_PRT_RET(checkSize,
1000 : HCCL_ERROR("[Set][BridgeLinkInfo]bridge UserRank %u is bigger than rank vector", bridgeUserRank),
1001 : HCCL_E_INTERNAL);
1002 :
1003 2651 : bridgePara = rankVector_[bridgeUserRank];
1004 2651 : return HCCL_SUCCESS;
1005 : }
1006 :
1007 24 : HcclResult TopoInfoExtractor::SetTopoDefaultInfoFor8P()
1008 : {
1009 : // 填充combined_rank_vector_:不区分board_type,只生成default单层拓扑
1010 24 : std::vector<RankInfo> tmpCombinedVector;
1011 : // 服务器内排序固定为0, 2, 3, 1, 5, 7, 6, 4，挑选8P多环中适用于combined的一组服务器内排序
1012 72 : std::vector<u32> devOrder = { 0, 2, 3, 1, 5, 7, 6, 4 };
1013 : // 维护topo输出的信息
1014 24 : std::string outLogInfo = "userRank/devicePhyId: ";
1015 :
1016 : // 填充combined_rank_vector_的内层vector:combined场景只有一条固定的环
1017 48 : for (auto iterMap = serverToRank_.begin(); iterMap != serverToRank_.end(); iterMap++) {
1018 : /* 服务器内8P满配单环特殊适配逻辑 */
1019 216 : for (u32 index = 0; index < devOrder.size(); index++) {
1020 192 : u32 devIndex = devOrder[index];
1021 192 : u32 combinedUserRank = (iterMap->second)[devIndex].userRank;
1022 :
1023 192 : bool checkError = (rankVector_.size() <= combinedUserRank);
1024 192 : CHK_PRT_RET(checkError,
1025 : HCCL_ERROR("[Set][TopoDefaultInfoFor8P]combined userRank[%u] is bigger than rank vector",
1026 : combinedUserRank), HCCL_E_INTERNAL);
1027 :
1028 192 : RankInfo tmpCombinedPara = rankVector_[combinedUserRank];
1029 192 : outLogInfo.append(std::to_string(tmpCombinedPara.userRank));
1030 192 : outLogInfo.append("/");
1031 192 : outLogInfo.append(std::to_string(tmpCombinedPara.devicePhyId));
1032 192 : outLogInfo.append("; ");
1033 192 : tmpCombinedVector.push_back(tmpCombinedPara);
1034 192 : }
1035 : }
1036 :
1037 24 : CommPlaneVector_[COMM_COMBINE].push_back(tmpCombinedVector);
1038 24 : return HCCL_SUCCESS;
1039 24 : }
1040 :
1041 520 : HcclResult TopoInfoExtractor::GetCommPlaneRanks(std::vector<std::vector<std::vector<u32>>> &CommPlaneRanks)
1042 : {
1043 520 : CommPlaneRanks.resize(CommPlaneVector_.size());
1044 8840 : for (u32 level = 0; level < CommPlaneVector_.size(); level ++) {
1045 8320 : u32 ringSize = CommPlaneVector_[level].size();
1046 8320 : CommPlaneRanks[level].resize(ringSize);
1047 14029 : for (u32 ringIndex = 0 ; ringIndex < ringSize; ringIndex ++) {
1048 5709 : u32 rankSize = CommPlaneVector_[level][ringIndex].size();
1049 5709 : CommPlaneRanks[level][ringIndex].resize(rankSize);
1050 17222 : for (u32 rankIndex = 0 ; rankIndex < rankSize; rankIndex ++) {
1051 11513 : u32 userRank = CommPlaneVector_[level][ringIndex][rankIndex].userRank;
1052 11513 : CommPlaneRanks[level][ringIndex][rankIndex] = userRank;
1053 11513 : HCCL_DEBUG("GetCommPlaneRanks CommPlaneRanks[%u][%u][%u]=%u", level, ringIndex, rankIndex, userRank);
1054 : }
1055 : }
1056 : }
1057 520 : return HCCL_SUCCESS;
1058 : }
1059 :
1060 1047 : void TopoInfoExtractor::GetIsBridgeVector(std::vector<bool> &isBridgeVector)
1061 : {
1062 1047 : isBridgeVector = isBridgeVector_;
1063 1047 : return;
1064 : }
1065 :
1066 520 : HcclResult TopoInfoExtractor::GetIsUsedRdmaMap(std::unordered_map<u32, bool> &isUsedRdmaMap)
1067 : {
1068 2143 : for (const RankInfo &dstRank : rankVector_) {
1069 1623 : bool isInterSuperPod = false;
1070 1623 : bool isInterServer = false;
1071 1623 : uint32_t isConnectedWithPcie = 0;
1072 1623 : if (rankData_.superPodId != dstRank.superPodId) { // 跨超节点场景
1073 51 : isInterSuperPod = true;
1074 1572 : } else if (rankData_.serverIdx != dstRank.serverIdx) { // 不跨超节点, 跨server场景
1075 252 : isInterServer = true;
1076 : } else { // 同server, PCIE互连场景
1077 1320 : HCCL_DEBUG("[TopoInfoExtractor]GetIsUsedRdmaMap for interServer");
1078 1320 : auto it = deviceLinkTypeMap_.find(dstRank.devicePhyId);
1079 1320 : CHK_PRT_RET(it == deviceLinkTypeMap_.end(),
1080 : HCCL_ERROR("can't find devicePhyId[%d] in deviceLinkTypeMap_", dstRank.devicePhyId),
1081 : HCCL_E_NOT_FOUND);
1082 1320 : isConnectedWithPcie = (it->second == LinkTypeInServer::PXI_TYPE) ? true : false;
1083 : }
1084 : // 使能RDMA的场景: 1.跨超节点 2.跨server且不使能HCCS 3.PCIE连接且使能RDMA开关
1085 1572 : bool isUsedRdma = (isInterSuperPod) ||
1086 3195 : (isInterServer && !isUsedInterHccsMode_) || (isConnectedWithPcie && isUsedRdmaLevel0_);
1087 1623 : isUsedRdmaMap[dstRank.userRank] = isUsedRdma;
1088 1623 : HCCL_DEBUG("[GetIsUsedRdma]isUsedRdma[%u], isInterSuperPod[%u], isInterServer[%u], isUsedInterHccsMode_[%u], "\
1089 : "isConnectedWithPcie[%u], isUsedRdmaLevel0_[%u], dstRank[%u]", isUsedRdma, isInterSuperPod, isInterServer,
1090 : isUsedInterHccsMode_, isConnectedWithPcie, isUsedRdmaLevel0_, dstRank.userRank);
1091 : }
1092 520 : return HCCL_SUCCESS;
1093 : }
1094 :
1095 520 : HcclResult TopoInfoExtractor::GetRankVecInfo(std::vector<std::vector<std::vector<u32>>> &serverAndsuperPodToRank)
1096 : {
1097 520 : std::vector<std::vector<u32>> serverToRank;
1098 520 : std::vector<std::vector<u32>> superPodToRank;
1099 520 : serverToRank.clear();
1100 520 : superPodToRank.clear();
1101 520 : u32 firstIdx = 0;
1102 :
1103 520 : serverToRank.resize(serverToRank_.size());
1104 1209 : for (auto iterMap = serverToRank_.begin(); iterMap != serverToRank_.end(); iterMap++) {
1105 689 : serverToRank[firstIdx].resize((iterMap->second).size());
1106 689 : if (!(iterMap->second).empty()) {
1107 2261 : for (u32 i = 0; i < (iterMap->second).size(); i++) {
1108 1572 : serverToRank[firstIdx][i] = (iterMap->second)[i].userRank;
1109 : }
1110 : }
1111 689 : firstIdx++;
1112 : }
1113 :
1114 520 : u32 podFirstIdx = 0;
1115 520 : superPodToRank.resize(superPodToRank_.size());
1116 1081 : for (auto iterMap = superPodToRank_.begin(); iterMap != superPodToRank_.end(); iterMap++) {
1117 561 : if (!(iterMap->second).empty()) {
1118 561 : superPodToRank[podFirstIdx].resize((iterMap->second).size());
1119 2184 : for (u32 i = 0; i < (iterMap->second).size(); i++) {
1120 1623 : superPodToRank[podFirstIdx][i] = (iterMap->second)[i].userRank;
1121 1623 : HCCL_DEBUG("GetRankVecInfo superPodToRank[%u][%u]=%u", podFirstIdx, i, superPodToRank[podFirstIdx][i]);
1122 : }
1123 : }
1124 561 : podFirstIdx++;
1125 : }
1126 520 : serverAndsuperPodToRank.push_back(serverToRank);
1127 520 : serverAndsuperPodToRank.push_back(superPodToRank);
1128 520 : return HCCL_SUCCESS;
1129 520 : }
1130 522 : HcclResult TopoInfoExtractor::SetTopoInfoForCombineL1()
1131 : {
1132 522 : if (deviceType_ == DevType::DEV_TYPE_910_93) {
1133 : // 按照superPodIdx 划分得所有rank信息
1134 107 : for (auto iter = superPodToRank_.begin(); iter != superPodToRank_.end(); iter++) {
1135 74 : if (iter->first != rankData_.superPodIdx) {
1136 41 : continue; // 只在自己所在的超节点创建
1137 : }
1138 33 : CommPlaneVector_[COMM_COMBINE_L1].push_back(iter->second);
1139 33 : HCCL_DEBUG("[SetTopoInfoForARS]Superpod rankdSize[%u].", CommPlaneVector_[COMM_COMBINE_L1][0].size());
1140 : }
1141 : }
1142 522 : return HCCL_SUCCESS;
1143 : }
1144 :
1145 522 : HcclResult TopoInfoExtractor::SetTopoInfoForARS() // 针对ARS特性
1146 : {
1147 522 : if (deviceType_ == DevType::DEV_TYPE_910_93) {
1148 107 : for (auto iter = superPodToRank_.begin(); iter != superPodToRank_.end(); iter++) {
1149 74 : if (iter->first != rankData_.superPodIdx) {
1150 41 : continue; // 只在自己所在的超节点创建
1151 : }
1152 33 : CommPlaneVector_[COMM_ARS].push_back(iter->second);
1153 33 : HCCL_DEBUG("[SetTopoInfoForARS]Superpod rankdSize[%u].", CommPlaneVector_[COMM_ARS][0].size());
1154 : }
1155 : }
1156 522 : return HCCL_SUCCESS;
1157 : }
1158 :
1159 527 : void TopoInfoExtractor::GetCommPlaneVector(std::vector<std::vector<std::vector<RankInfo>>> &commPlaneVector)
1160 : {
1161 527 : commPlaneVector = CommPlaneVector_;
1162 527 : return;
1163 : }
1164 :
1165 523 : void TopoInfoExtractor::InitAHCConfig(std::map<HcclCMDType, std::vector<HcclAlgoType>> &algoConfig)
1166 : {
1167 54711 : for (u32 opType = 0; opType < static_cast<u32>(HcclCMDType::HCCL_CMD_MAX); opType++) {
1168 54190 : std::vector<HcclAlgoType> algoType = algoConfig[static_cast<HcclCMDType>(opType)];
1169 108378 : isConfigAHC_ = (algoType[HCCL_ALGO_LEVEL_1] == HcclAlgoType::HCCL_ALGO_TYPE_AHC ||
1170 54188 : algoType[HCCL_ALGO_LEVEL_1] == HcclAlgoType::HCCL_ALGO_TYPE_AHC_BROKE);
1171 54190 : if (isConfigAHC_) {
1172 2 : HCCL_INFO("[InitAHCConfig] set AHC alg, opType[%u]", opType);
1173 2 : break;
1174 : }
1175 54190 : }
1176 :
1177 52211 : for (u32 opType = 0; opType < static_cast<u32>(HcclCMDType::HCCL_CMD_MAX); opType++) { //没配置算法的情况下默认会走AHC嘛？ 给测试用
1178 51714 : std::vector<HcclAlgoType> algoType = algoConfig[static_cast<HcclCMDType>(opType)];
1179 51714 : isConfigNULL_ = algoType[HCCL_ALGO_LEVEL_0] == HcclAlgoType::HCCL_ALGO_TYPE_NULL;
1180 51714 : if (isConfigNULL_) {
1181 26 : HCCL_INFO("[InitAHCConfig] set NULL alg, opType[%u]", opType);
1182 26 : break;
1183 : }
1184 51714 : }
1185 523 : return;
1186 : }
1187 :
1188 522 : HcclResult TopoInfoExtractor::SetAHCSubGroupsAndAlgOption()
1189 : {
1190 : // 解析subgroup
1191 522 : CHK_RET(SetTopoInfoForLevel1(true));
1192 :
1193 : // 分组切分
1194 522 : CHK_RET(AHCSubGroupInit(COMM_LEVEL1_AHC, CommPlaneSubGroupVector_));
1195 522 : if (deviceType_ != DevType::DEV_TYPE_910_93) {
1196 489 : CHK_RET(AHCSubGroupInit(COMM_COMBINE, CommPlaneSubGroupVector_));
1197 : } else {
1198 33 : CHK_RET(AHCSubGroupInit(COMM_COMBINE_ORDER, CommPlaneSubGroupVector_));
1199 : }
1200 :
1201 : //算法配置初始化
1202 522 : CommAHCBaseInfo::InitConcAlgOption(ahcAlgOption_);
1203 522 : return HCCL_SUCCESS;
1204 : }
1205 :
1206 522 : void TopoInfoExtractor::AHCCommSubgroupInit()
1207 : {
1208 522 : if (deviceType_ != DevType::DEV_TYPE_910_93) {
1209 : // 用于910B AHC COMM_COMBINE 通信域分组场景测试
1210 489 : std::map<std::string, std::vector<u32>> serverIDGroup;
1211 489 : std::vector<std::string> serverIdsVector;
1212 1981 : for (u32 i = 0; i < rankVector_.size(); i++) {
1213 1492 : auto itServerID = serverIDGroup.find(rankVector_[i].serverId);
1214 1492 : if (itServerID != serverIDGroup.end()) {
1215 848 : itServerID->second.push_back(i);
1216 : } else {
1217 644 : serverIdsVector.push_back(rankVector_[i].serverId);
1218 644 : std::vector<u32> subGroup;
1219 644 : subGroup.push_back(i);
1220 644 : serverIDGroup.insert(std::make_pair(rankVector_[i].serverId, subGroup));
1221 644 : }
1222 : }
1223 489 : std::vector<std::vector<u32>> serverSubGroups;
1224 1133 : for (u32 i = 0; i < serverIdsVector.size(); i++) {
1225 644 : serverSubGroups.push_back(serverIDGroup[serverIdsVector[i]]);
1226 : }
1227 489 : CommPlaneSubGroupVector_[COMM_COMBINE].push_back(serverSubGroups);
1228 489 : } else {
1229 : // 用于910_93 AHC COMM_COMBINE_ORDER 通信域分组场景测试
1230 33 : std::vector<std::string> idsForIndexingVector;
1231 33 : std::map<std::string, std::vector<u32>> superPodIdGroup;
1232 178 : for (u32 i = 0; i < rankVector_.size(); i++) {
1233 145 : auto itSuperPodID = superPodIdGroup.find(rankVector_[i].superPodId);
1234 145 : if (itSuperPodID != superPodIdGroup.end()) {
1235 67 : itSuperPodID->second.push_back(i);
1236 : } else {
1237 78 : idsForIndexingVector.push_back(rankVector_[i].superPodId);
1238 78 : std::vector<u32> subGroup;
1239 78 : subGroup.push_back(i);
1240 78 : superPodIdGroup.insert(std::make_pair(rankVector_[i].superPodId, subGroup));
1241 78 : }
1242 : }
1243 33 : std::vector<std::vector<u32>> superPodSubGroups;
1244 111 : for (u32 i = 0; i < idsForIndexingVector.size(); i++) {
1245 78 : superPodSubGroups.push_back(superPodIdGroup[idsForIndexingVector[i]]);
1246 78 : HCCL_DEBUG("[SetTopoInfoForLevel0][AHC_DEBUG 910_93] superPodId[%s]", idsForIndexingVector[i].c_str());
1247 223 : for (u32 index = 0; index < superPodIdGroup[idsForIndexingVector[i]].size(); index++) {
1248 145 : HCCL_DEBUG("[SetTopoInfoForLevel0][AHC_DEBUG 910_93] groupIdx[%u]", superPodIdGroup[idsForIndexingVector[i]][index]);
1249 : }
1250 : }
1251 33 : CommPlaneSubGroupVector_[COMM_COMBINE_ORDER].push_back(superPodSubGroups);
1252 33 : }
1253 522 : return;
1254 : }
1255 :
1256 520 : void TopoInfoExtractor::GetCommPlaneSubGroupVector(std::vector<std::vector<std::vector<std::vector<u32>>>> &CommPlaneSubGroupVector)
1257 : {
1258 520 : CommPlaneSubGroupVector = CommPlaneSubGroupVector_;
1259 520 : return;
1260 : }
1261 :
1262 0 : void TopoInfoExtractor::SetAHCLevelAlgOption(AHCConcOpType ahcConcOpType,TemplateType templateType)
1263 : {
1264 0 : if (ahcAlgOption_.find(ahcConcOpType) == ahcAlgOption_.end()) {
1265 0 : ahcAlgOption_[ahcConcOpType] = templateType;
1266 : }
1267 0 : return;
1268 : }
1269 :
1270 520 : void TopoInfoExtractor::GetAHCAlgOption(std::map<AHCConcOpType, TemplateType> &ahcAlgOption)
1271 : {
1272 520 : ahcAlgOption = ahcAlgOption_;
1273 520 : return;
1274 : }
1275 :
1276 527 : void TopoInfoExtractor::GetRankData(RankInfo &rankData)
1277 : {
1278 527 : rankData = rankData_;
1279 527 : return;
1280 : }
1281 :
1282 527 : void TopoInfoExtractor::GetServerToRank(std::map<u32, std::vector<RankInfo>> &serverToRank)
1283 : {
1284 527 : serverToRank = serverToRank_;
1285 527 : return;
1286 : }
1287 :
1288 527 : void TopoInfoExtractor::GetSuperPodToRank(std::map<u32, std::vector<RankInfo>> &superPodToRank)
1289 : {
1290 527 : superPodToRank = superPodToRank_;
1291 527 : return;
1292 : }
1293 :
1294 527 : void TopoInfoExtractor::GetDeviceLinkTypeMap(std::map<s32, LinkTypeInServer> &deviceLinkTypeMap)
1295 : {
1296 527 : deviceLinkTypeMap = deviceLinkTypeMap_;
1297 527 : return;
1298 : }
1299 :
1300 1044 : HcclResult TopoInfoExtractor::AHCSubGroupInit(CommPlane algLevel, std::vector<std::vector<std::vector<std::vector<u32>>>> &CommPlaneSubGroupVector)
1301 : {
1302 1044 : std::vector<std::vector<std::vector<u32>>> &globalSubGroups = CommPlaneSubGroupVector[algLevel];
1303 :
1304 : // globalSubGroups 参数检查
1305 1044 : if ((globalSubGroups.size() == 0) || (globalSubGroups[0].size() == 0)) {
1306 358 : return HCCL_SUCCESS;
1307 : }
1308 :
1309 686 : CHK_RET(CommAHCBaseInfo::CheckGlobalGroups(globalSubGroups));
1310 :
1311 686 : u32 minSubGroupSize = globalSubGroups[0][0].size();
1312 686 : u32 maxSubGroupSize = globalSubGroups[0][0].size();
1313 686 : u32 subGroupSizeGCD = globalSubGroups[0][0].size();
1314 931 : for (u32 i = 1; i < globalSubGroups[0].size(); ++i) {
1315 245 : subGroupSizeGCD = std::__gcd(subGroupSizeGCD, static_cast<u32>(globalSubGroups[0][i].size()));
1316 245 : if (globalSubGroups[0][i].size() < minSubGroupSize) {
1317 0 : minSubGroupSize = globalSubGroups[0][i].size();
1318 : }
1319 245 : if (globalSubGroups[0][i].size() > maxSubGroupSize) {
1320 16 : maxSubGroupSize = globalSubGroups[0][i].size();
1321 : }
1322 : }
1323 :
1324 686 : u32 splitDivisor = minSubGroupSize;
1325 :
1326 686 : HCCL_DEBUG("[TopoInfoExtractor][AHCSubGroupInit] begin groupSplit algLevel = %u minSubGroupSize = %u maxSubGroupSize = %u SubGroupSizeGCD = %u \
1327 : splitDivisor = %u", algLevel, minSubGroupSize, maxSubGroupSize, subGroupSizeGCD, splitDivisor);
1328 :
1329 : // 切分分组逻辑, 满足整数倍切分，满足最大偏差大于等于最小分组数时切分
1330 686 : if ((subGroupSizeGCD == minSubGroupSize) || (maxSubGroupSize / splitDivisor > 1)) {
1331 : // 设置为 BROKE 类型
1332 686 : maxSubGroupSize = splitDivisor;
1333 : // 将所有的分组切分成 splitDivisor 粒度的 subGroup
1334 1665 : for (u32 i = 0; i < globalSubGroups[0].size(); ++i) {
1335 979 : if (globalSubGroups[0][i].size() / splitDivisor <= 1) {
1336 963 : continue;
1337 : }
1338 16 : std::vector<u32> originGroup = globalSubGroups[0][i];
1339 16 : globalSubGroups[0].erase(globalSubGroups[0].begin() + i);
1340 16 : u32 splitGroupsNum = (originGroup.size() / splitDivisor);
1341 48 : for (u32 j = 0; j < splitGroupsNum - 1; ++j) {
1342 64 : globalSubGroups[0].insert(globalSubGroups[0].begin() + i + j, std::vector<u32>(originGroup.begin() + j * splitDivisor,
1343 64 : originGroup.begin() + (j + 1) * splitDivisor));
1344 : }
1345 32 : globalSubGroups[0].insert(globalSubGroups[0].begin() + i + splitGroupsNum - 1, std::vector<u32>(originGroup.begin() + (splitGroupsNum - 1) * splitDivisor,
1346 32 : originGroup.begin() + originGroup.size()));
1347 16 : maxSubGroupSize = std::max(maxSubGroupSize, static_cast<u32>(originGroup.size()) - ((splitGroupsNum - 1) * splitDivisor));
1348 16 : i--;
1349 16 : }
1350 : }
1351 :
1352 686 : HCCL_DEBUG("[TopoInfoExtractor][AHCSubGroupInit] after groupSplit algLevel = %u minSubGroupSize = %u maxSubGroupSize = %u SubGroupSizeGCD = %u \
1353 : splitDivisor = %u", algLevel, minSubGroupSize, maxSubGroupSize, subGroupSizeGCD, splitDivisor);
1354 :
1355 686 : CHK_RET(CommAHCBaseInfo::CheckGlobalGroups(globalSubGroups));
1356 :
1357 686 : return HCCL_SUCCESS;
1358 : }
1359 :
1360 5748 : bool Ascending(const RankInfo &first, const RankInfo &second)
1361 : {
1362 5748 : if (first.serverIdx != second.serverIdx) {
1363 449 : return first.serverIdx < second.serverIdx;
1364 : } else {
1365 5299 : return first.userRank < second.userRank;
1366 : }
1367 : }
1368 :
1369 552 : bool CompareWithUserRankAscend(const RankInfo &left, const RankInfo &right)
1370 : {
1371 552 : return left.userRank < right.userRank;
1372 : }
1373 :
1374 : // 适配ROH平面网段隔离，奇数rank互通，偶数rank互通，奇偶不通
1375 1 : bool CheckSdmaWithRohTopo(const std::vector<u32> &nicList, std::vector<u32> &topoList)
1376 : {
1377 1 : std::vector<u32> tmpNicList(nicList);
1378 1 : SearchPath searchPath;
1379 1 : std::sort(tmpNicList.begin(), tmpNicList.end());
1380 1 : topoList = searchPath.Search(tmpNicList);
1381 1 : if (topoList.empty()) {
1382 0 : return false;
1383 : }
1384 1 : return true;
1385 1 : }
1386 :
1387 522 : std::vector<std::vector<u32>> GetRingsOrderByTopoType(u32 ranksSize, TopoType topoType, std::vector<u32> &nicList)
1388 : {
1389 522 : std::vector<std::vector<u32>> multiRingOrder;
1390 522 : if (topoType == TopoType::TOPO_TYPE_8P_RING) { // 4 ring 场景
1391 : // 每个环的排序是按照设备物理ID进行的
1392 48 : std::vector<u32> tmpLevel00 = { 0, 1, 2, 6, 5, 4, 7, 3 }; // 环0
1393 48 : std::vector<u32> tmpLevel01 = { 0, 3, 7, 4, 5, 6, 2, 1 }; // 环1
1394 48 : std::vector<u32> tmpLevel02 = { 0, 2, 3, 1, 5, 7, 6, 4 }; // 环2
1395 48 : std::vector<u32> tmpLevel03 = { 0, 4, 6, 7, 5, 1, 3, 2 }; // 环3
1396 :
1397 : // 填充8pring 多环的comm level0 四个环的顺序
1398 24 : multiRingOrder.push_back(tmpLevel00);
1399 24 : multiRingOrder.push_back(tmpLevel01);
1400 24 : multiRingOrder.push_back(tmpLevel02);
1401 24 : multiRingOrder.push_back(tmpLevel03);
1402 522 : } else if (topoType == TopoType::TOPO_TYPE_NP_DOUBLE_RING) { // 2 ring 场景
1403 7 : std::vector<u32> tmpLevel00; // 环0
1404 7 : std::vector<u32> tmpLevel01; // 环1
1405 7 : tmpLevel00 = nicList; // { 0, 1, 2, 3, 4, 5, 6, 7 };
1406 7 : tmpLevel01.reserve(ranksSize);
1407 7 : tmpLevel01.push_back(nicList[0]);
1408 7 : tmpLevel01.insert(tmpLevel01.end(), tmpLevel00.rbegin(), tmpLevel00.rend() - 1);
1409 7 : HCCL_INFO("[GetRingsOrderByTopoType] TopoType:TOPO_TYPE_NP_DOUBLE_RING");
1410 : // 填充 double ring 两环的comm level0的顺序
1411 7 : multiRingOrder.push_back(tmpLevel00);
1412 7 : multiRingOrder.push_back(tmpLevel01);
1413 7 : } else { // 1 ring 场景
1414 491 : std::vector<u32> tmpLevel00 = nicList; // 环0
1415 :
1416 : // 填充 single ring 单环的comm level0的顺序
1417 491 : multiRingOrder.push_back(tmpLevel00);
1418 491 : }
1419 : // 打印多个环
1420 522 : if (UNLIKELY(HcclCheckLogLevel(DLOG_DEBUG))) {
1421 1123 : for (size_t i = 0; i < multiRingOrder.size(); i++) {
1422 601 : auto ring = multiRingOrder[i];
1423 601 : std::ostringstream stringRepresentation;
1424 2527 : for (std::vector<uint32_t>::iterator it = ring.begin(); it != ring.end(); it++) {
1425 1926 : stringRepresentation << *it << " ";
1426 : }
1427 601 : std::string ringString = stringRepresentation.str();
1428 601 : const char *charRing = ringString.c_str();
1429 601 : HCCL_DEBUG("[GetRingsOrderByTopoType] The No.%zu ring: %s", i, charRing);
1430 601 : }
1431 : }
1432 522 : return multiRingOrder;
1433 0 : }
1434 :
1435 1 : std::vector<std::vector<u32>> GetRingsOrderForAnyPath(u32 ranksSize, TopoType topoType, std::vector<u32> &nicList)
1436 : {
1437 1 : std::vector<std::vector<u32>> multiRingOrder;
1438 1 : if (topoType == TopoType::TOPO_TYPE_NP_DOUBLE_RING) { // 2 ring 场景
1439 1 : std::vector<u32> tmpLevel00; // 环0
1440 1 : std::vector<u32> tmpLevel01; // 环1
1441 1 : std::vector<u32> rohLevel0;
1442 1 : if (CheckSdmaWithRohTopo(nicList, rohLevel0)) {
1443 1 : tmpLevel00 = rohLevel0; // 环0, 8卡 { 0, 1, 3, 2, 4, 5, 7, 6 };
1444 1 : tmpLevel01.reserve(ranksSize); // 环1, 8卡 { 0, 6, 7, 5, 4, 2, 3, 1 };
1445 1 : tmpLevel01.push_back(rohLevel0[0]);
1446 1 : tmpLevel01.insert(tmpLevel01.end(), rohLevel0.rbegin(), rohLevel0.rend() - 1);
1447 : } else {
1448 0 : tmpLevel00 = nicList; // { 0, 1, 2, 3, 4, 5, 6, 7 };
1449 0 : tmpLevel01.reserve(ranksSize);
1450 0 : tmpLevel01.push_back(nicList[0]);
1451 0 : tmpLevel01.insert(tmpLevel01.end(), tmpLevel00.rbegin(), tmpLevel00.rend() - 1);
1452 : }
1453 : // 填充 double ring 两环的comm level0的顺序
1454 1 : multiRingOrder.push_back(tmpLevel00);
1455 1 : multiRingOrder.push_back(tmpLevel01);
1456 1 : } else { // 1 ring 场景
1457 0 : std::vector<u32> tmpLevel00 = nicList; // 环0
1458 :
1459 : // 填充 single ring 单环的comm level0的顺序
1460 0 : multiRingOrder.push_back(tmpLevel00);
1461 0 : }
1462 : // 打印多个环
1463 1 : HCCL_DEBUG("[GetRingsOrderForAnyPath]print rings:");
1464 3 : for (size_t i = 0; i < multiRingOrder.size(); i++) {
1465 2 : auto ring = multiRingOrder[i];
1466 2 : std::ostringstream stringRepresentation;
1467 18 : for (std::vector<uint32_t>::iterator it = ring.begin(); it != ring.end(); it++) {
1468 16 : stringRepresentation << *it << " ";
1469 : }
1470 2 : std::string ringString = stringRepresentation.str();
1471 2 : const char *charRing = ringString.c_str();
1472 2 : HCCL_INFO("[GetRingsOrderByRdmaSdmaConcurrent] The No.%zu ring: %s.", i, charRing);
1473 2 : }
1474 1 : return multiRingOrder;
1475 0 : }
1476 :
1477 : }
|
