Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "topo_info_extractor.h"
12 : #include "externalinput_pub.h"
13 : #include "device_capacity.h"
14 : #include "search_path.h"
15 : #include "comm_base_pub.h"
16 : #include "hccl_impl_pub.h"
17 :
18 : namespace hccl {
19 :
20 : constexpr u32 SERVER_RANK_SIZE = 8;
21 :
22 523 : TopoInfoExtractor::TopoInfoExtractor(HcclAlgoAttr& algoAttr, HcclTopoAttr& topoAttr, const TopoType topoType)
23 523 : : identifier_(algoAttr.identifier),
24 523 : userRank_(topoAttr.userRank),
25 523 : userRankSize_(topoAttr.userRankSize),
26 523 : topoType_(topoType),
27 523 : deviceType_(topoAttr.deviceType),
28 523 : rankVector_(topoAttr.rankInfoList),
29 523 : meshAggregationRankSize_(topoAttr.meshAggregationRankSize),
30 523 : isUsedRdmaLevel0_(algoAttr.isUsedRdmaLevel0),
31 523 : isUsedInterHccsMode_(algoAttr.isUsedInterHccsMode),
32 523 : isDiffAggregation_(topoAttr.isDiffDeviceModule),
33 523 : isConfigAHC_(false),
34 523 : isConfigNULL_(false),
35 523 : multiModuleDiffDeviceNumMode_(topoAttr.multiModuleDiffDeviceNumMode),
36 523 : multiSuperPodDiffServerNumMode_(topoAttr.multiSuperPodDiffServerNumMode),
37 523 : multiSuperPodDiffDeviceNumMode_(topoAttr.multiSuperPodDiffDeviceNumMode),
38 523 : isDiffDeviceType_(topoAttr.isDiffDeviceType),
39 523 : gcdDeviceNumPerAggregation_(topoAttr.gcdDeviceNumPerAggregation),
40 1046 : CommPlaneSubGroupVector_(COMM_LEVEL_RESERVED),
41 2615 : CommPlaneVector_(COMM_LEVEL_RESERVED) {};
42 :
43 : #ifdef CCL_LLT
44 : // 为了适配老的LLT框架提供的构造函数
45 11 : TopoInfoExtractor::TopoInfoExtractor(
46 : std::string identifier, u32 userRank, u32 userRankSize, TopoType topoType, DevType deviceType,
47 : std::vector<RankInfo>& rankVector, u32 meshAggregationRankSize, bool isUsedRdmaLevel0, bool isUsedInterHccsMode,
48 : bool multiModuleDiffDeviceNumMode, bool multiSuperPodDiffServerNumMode, bool multiSuperPodDiffDeviceNumMode,
49 11 : bool isDiffDeviceType, u32 gcdDeviceNumPerAggregation)
50 11 : : identifier_(identifier),
51 11 : userRank_(userRank),
52 11 : userRankSize_(userRankSize),
53 11 : topoType_(topoType),
54 11 : deviceType_(deviceType),
55 11 : rankVector_(rankVector),
56 11 : meshAggregationRankSize_(meshAggregationRankSize),
57 11 : isUsedRdmaLevel0_(isUsedRdmaLevel0),
58 11 : isUsedInterHccsMode_(isUsedInterHccsMode),
59 11 : isDiffAggregation_(false),
60 11 : multiModuleDiffDeviceNumMode_(multiModuleDiffDeviceNumMode),
61 11 : multiSuperPodDiffServerNumMode_(multiSuperPodDiffServerNumMode),
62 11 : multiSuperPodDiffDeviceNumMode_(multiSuperPodDiffDeviceNumMode),
63 11 : isDiffDeviceType_(isDiffDeviceType),
64 11 : gcdDeviceNumPerAggregation_(gcdDeviceNumPerAggregation),
65 11 : isConfigAHC_(false),
66 11 : isConfigNULL_(false),
67 22 : CommPlaneVector_(COMM_LEVEL_RESERVED),
68 55 : CommPlaneSubGroupVector_(COMM_LEVEL_RESERVED) {};
69 : #endif
70 :
71 534 : TopoInfoExtractor::~TopoInfoExtractor() {}
72 :
73 526 : HcclResult TopoInfoExtractor::Init(std::map<HcclCMDType, std::vector<HcclAlgoType>>& algoConfig)
74 : {
75 526 : HCCL_INFO(
76 : "factory init:collective id[%s], user rank[%u], user rank size[%u], topo type[%d], device Type[%d], "
77 : "meshAggregationRankSize[%u]",
78 : identifier_.c_str(), userRank_, userRankSize_, topoType_, deviceType_, meshAggregationRankSize_);
79 :
80 : // 参数有效性校验
81 526 : CHK_RET(CheckInitInfo());
82 :
83 : // 初始化 AHC 相关信息
84 526 : InitAHCConfig(algoConfig);
85 :
86 : // 填充必要数据结构
87 526 : CHK_RET(SetRankInfo());
88 :
89 525 : if (IsGeneralServer() && GetRemoteIsHdc()) {
90 0 : HCCL_INFO("heterog ES ps factory init no need set topoInfo");
91 : } else {
92 : // 设置拓扑信息
93 525 : CHK_RET(SetTopologyInfo());
94 : // 根据拓扑类型以及芯片类型，校验两层拓扑(外层/内层)、单层拓扑的平面个数合法性
95 525 : CHK_RET(CheckPlaneInfo());
96 : }
97 :
98 525 : CHK_RET(SetRankMap());
99 525 : return HCCL_SUCCESS;
100 : }
101 :
102 525 : HcclResult TopoInfoExtractor::SetRankMap()
103 : {
104 : // 构建由UserRank到子通信域的映射
105 525 : subCommRank2UserRank_.resize(static_cast<u32>(COMM_LEVEL_RESERVED));
106 525 : userRank2subCommRank_.resize(static_cast<u32>(COMM_LEVEL_RESERVED));
107 525 : HCCL_DEBUG("[TopoInfoExtractor]SetRankMap begin");
108 :
109 8925 : for (u32 levelIndex = 0; levelIndex < CommPlaneVector_.size(); levelIndex++) {
110 8400 : u32 ringSize = CommPlaneVector_[levelIndex].size();
111 8400 : subCommRank2UserRank_[levelIndex].resize(ringSize);
112 8400 : userRank2subCommRank_[levelIndex].resize(ringSize);
113 14141 : for (u32 ringIndex = 0; ringIndex < ringSize; ringIndex++) {
114 5741 : u32 rankSize = CommPlaneVector_[levelIndex][ringIndex].size();
115 17332 : for (u32 rankIndex = 0; rankIndex < rankSize; rankIndex++) {
116 11591 : u32 userRank = CommPlaneVector_[levelIndex][ringIndex][rankIndex].userRank;
117 11591 : subCommRank2UserRank_[levelIndex][ringIndex][rankIndex] = userRank;
118 11591 : userRank2subCommRank_[levelIndex][ringIndex][userRank] = rankIndex;
119 : }
120 : }
121 : }
122 525 : return HCCL_SUCCESS;
123 : }
124 :
125 1059 : HcclResult TopoInfoExtractor::CheckInitInfo()
126 : {
127 1059 : if (rankVector_.size() == 0) {
128 0 : HCCL_ERROR("[Check][InitInfo]Not support the scenes: rank_vector size is zero");
129 0 : return HCCL_E_PARA; // 没有rank_table的场景直接报错
130 : }
131 :
132 : // 构造函数入参有效性检查:user_rank_size与user_rank_
133 1059 : if (userRankSize_ <= userRank_) {
134 0 : HCCL_ERROR("[Check][InitInfo]userRankSize_[%u] or userRank_[%u] is invalid.", userRankSize_, userRank_);
135 0 : return HCCL_E_PARA;
136 : }
137 :
138 1059 : if (userRankSize_ != rankVector_.size()) {
139 0 : HCCL_ERROR(
140 : "[Check][InitInfo]userRankSize_[%u] is not equal to rank_vector size[%llu].", userRankSize_,
141 : rankVector_.size());
142 0 : return HCCL_E_PARA;
143 : }
144 :
145 1059 : bool isParaInvalid = ((topoType_ == TopoType::TOPO_TYPE_RESERVED) || (deviceType_ >= DevType::DEV_TYPE_COUNT));
146 1059 : if (isParaInvalid) {
147 2 : HCCL_ERROR(
148 : "[Check][InitInfo]Not support the scenes: TopoType[%d] or deviceType[%d] is invalid.", topoType_,
149 : deviceType_);
150 2 : return HCCL_E_PARA;
151 : }
152 :
153 : // 入参组合有效性检查:不支持4P_RING
154 1057 : if ((deviceType_ == DevType::DEV_TYPE_910 || deviceType_ == DevType::DEV_TYPE_910B
155 75 : || deviceType_ == DevType::DEV_TYPE_910_93)
156 1049 : && (topoType_ == TopoType::TOPO_TYPE_4P_RING)) {
157 1 : HCCL_ERROR(
158 : "[Check][InitInfo]Not support the scenes: TopoType[%d] with deviceType[%d] is invalid.", topoType_,
159 : deviceType_);
160 1 : return HCCL_E_PARA;
161 : }
162 :
163 1056 : return HCCL_SUCCESS;
164 : }
165 :
166 526 : HcclResult TopoInfoExtractor::SetRankInfo()
167 : {
168 841 : for (u32 index = 0; index < rankVector_.size(); index++) {
169 841 : if (userRank_ == rankVector_[index].userRank) {
170 526 : rankData_ = rankVector_[index];
171 526 : HCCL_INFO(
172 : "[SetRankInfo]rankData_: userRank[%u], devicePhyId[%d], serverIdx[%u], superPodId[%s], superPodIdx[%u]",
173 : rankData_.userRank, rankData_.devicePhyId, rankData_.serverIdx, rankData_.superPodId.c_str(),
174 : rankData_.superPodIdx);
175 526 : break;
176 : }
177 : }
178 :
179 526 : std::set<u32> serverIdxs;
180 526 : std::set<u32> moduleIdxs;
181 2170 : for (u32 index = 0; index < rankVector_.size(); index++) {
182 : // 填充superPodRankMap_, 记录superPodId -> rankInfo
183 1644 : auto itSuperPod = superPodToRank_.find(rankVector_[index].superPodIdx);
184 1644 : if (itSuperPod != superPodToRank_.end()) {
185 1077 : itSuperPod->second.push_back(rankVector_[index]);
186 : } else {
187 567 : std::vector<RankInfo> rankVecTmp;
188 567 : rankVecTmp.push_back(rankVector_[index]);
189 567 : superPodToRank_.insert(std::make_pair(rankVector_[index].superPodIdx, rankVecTmp));
190 567 : }
191 :
192 1644 : u32 moduleIdx = 0;
193 1644 : CHK_RET(GetModuleIdx(rankVector_[index], moduleIdx));
194 1644 : moduleIdxs.insert(moduleIdx);
195 : // 填充serverRankMap_, 只记录本superPod下的serverIdx -> rankInfo
196 1644 : if (rankVector_[index].superPodId == rankData_.superPodId || isDiffDeviceType_) {
197 1593 : auto itServer = serverToRank_.find(moduleIdx);
198 1593 : if (itServer != serverToRank_.end()) { // 存在该服务器内相关rank的对应信息
199 896 : itServer->second.push_back(rankVector_[index]);
200 : } else { // 不存在则新增一条map记录
201 697 : std::vector<RankInfo> rankVecTmp;
202 697 : rankVecTmp.push_back(rankVector_[index]);
203 697 : serverToRank_.insert(std::make_pair(moduleIdx, rankVecTmp));
204 697 : }
205 : }
206 :
207 : // 填充 serverToRankMerge_, server 和 superPod 两层合并的通信域内所有 rank 信息
208 1644 : auto itServer = serverToRankMerge_.find(moduleIdx);
209 1644 : if (itServer != serverToRankMerge_.end()) { // 存在该服务器内相关rank的对应信息
210 898 : itServer->second.push_back(rankVector_[index]);
211 : } else { // 不存在则新增一条map记录
212 746 : std::vector<RankInfo> rankVecTmp;
213 746 : rankVecTmp.push_back(rankVector_[index]);
214 746 : serverToRankMerge_.insert(std::make_pair(moduleIdx, rankVecTmp));
215 746 : }
216 :
217 : // 同一个server内, 记录本rank和其他rank的链路
218 1644 : if (rankVector_[index].serverIdx == rankData_.serverIdx) {
219 1339 : LinkTypeInServer linkType = LinkTypeInServer::RESERVED_LINK_TYPE;
220 2152 : if (rankData_.devicePhyId != rankVector_[index].devicePhyId && rankData_.devicePhyId != HOST_DEVICE_ID
221 2152 : && rankVector_[index].devicePhyId != HOST_DEVICE_ID && topoType_ != TopoType::TOPO_TYPE_HETEROG) {
222 813 : CHK_RET(hrtGetPairDeviceLinkType(rankData_.devicePhyId, rankVector_[index].devicePhyId, linkType));
223 : }
224 1339 : deviceLinkTypeMap_.insert(std::make_pair(rankVector_[index].devicePhyId, linkType));
225 : }
226 :
227 1644 : u32 serverIdx = 0;
228 1644 : CHK_RET(GetServerIdx(rankVector_[index], serverIdx));
229 1644 : serverIdxs.insert(serverIdx);
230 : }
231 :
232 526 : u32 rankNumPerAggregation = userRankSize_ / static_cast<u32>(moduleIdxs.size());
233 526 : if (isDiffDeviceType_) {
234 0 : rankNumPerAggregation = gcdDeviceNumPerAggregation_;
235 0 : HCCL_INFO(
236 : "[SetRankInfo] isDiffDeviceType[%u] userRankSize[%u] moduleIdxs.size[%u] rankNumPerAggregation[%u]",
237 : isDiffDeviceType_, userRankSize_, moduleIdxs.size(), rankNumPerAggregation);
238 : }
239 :
240 : // 调整每个server内的user_rank排序(server内userRank从小到大,一定连续)
241 1223 : for (auto iterMap = serverToRank_.begin(); iterMap != serverToRank_.end(); iterMap++) {
242 697 : if (!(iterMap->second).empty()) {
243 697 : std::sort(iterMap->second.begin(), iterMap->second.end(), Ascending);
244 : }
245 : }
246 :
247 : // 调整每个superPod内的user_rank排序, 按照serverIdx从小到大、userRank从小到大排序
248 1093 : for (auto iterMap = superPodToRank_.begin(); iterMap != superPodToRank_.end(); iterMap++) {
249 567 : if (!(iterMap->second).empty()) {
250 567 : std::sort(iterMap->second.begin(), iterMap->second.end(), Ascending);
251 : }
252 : }
253 :
254 : // 调整多个 superPod 合并的 user_rank 排序，按照 serverIdx 从小到大、userRank从小到大排序
255 1272 : for (auto iterMap = serverToRankMerge_.begin(); iterMap != serverToRankMerge_.end(); iterMap++) {
256 746 : if (!(iterMap->second).empty()) {
257 746 : std::sort(iterMap->second.begin(), iterMap->second.end(), Ascending);
258 : }
259 : }
260 :
261 1223 : for (auto it = serverToRank_.begin(); it != serverToRank_.end(); it++) {
262 697 : HCCL_DEBUG("[SetRankInfo][MIX_DEBUG] serverID[%u]", it->first);
263 2290 : for (auto index = it->second.begin(); index != it->second.end(); index++) {
264 1593 : HCCL_DEBUG(
265 : "[SetRankInfo][MIX_DEBUG] userRank[%u], devicePhyId[%d], serverIdx[%u], superPodId[%s]",
266 : index->userRank, index->devicePhyId, index->serverIdx, index->superPodId.c_str());
267 : }
268 : }
269 :
270 1272 : for (auto it = serverToRankMerge_.begin(); it != serverToRankMerge_.end(); it++) {
271 746 : HCCL_DEBUG("[SetRankInfo][AHC_DEBUG] serverID[%u]", it->first);
272 2390 : for (auto index = it->second.begin(); index != it->second.end(); index++) {
273 1644 : HCCL_DEBUG(
274 : "[SetRankInfo][AHC_DEBUG] userRank[%u], devicePhyId[%d], serverIdx[%u], superPodId[%s]",
275 : index->userRank, index->devicePhyId, index->serverIdx, index->superPodId.c_str());
276 : }
277 : }
278 526 : HCCL_DEBUG(
279 : "[SetRankInfo][AHC_DEBUG] rankNumPerAggregation[%u] moduleIdxs.size()=[%u]", rankNumPerAggregation,
280 : moduleIdxs.size());
281 :
282 526 : ranksOneNode_ = {0, 8, 4, 2, 1, 4, rankNumPerAggregation, 0, rankNumPerAggregation, rankNumPerAggregation};
283 :
284 : // 校验每个server内的设备个数与topo类型的组合是否正确
285 526 : if (topoType_ != TopoType::TOPO_TYPE_COMMON) {
286 488 : CHK_RET(CheckServerInfo());
287 : }
288 : // 校验每个superPod下的device数量相同
289 525 : CHK_RET(CheckSuperPodInfo());
290 :
291 525 : return HCCL_SUCCESS;
292 526 : }
293 :
294 525 : HcclResult TopoInfoExtractor::CheckSuperPodInfo()
295 : {
296 1091 : for (auto iter = superPodToRank_.begin(); iter != superPodToRank_.end(); iter++) {
297 566 : u32 devNum = superPodToRank_.begin()->second.size();
298 566 : u32 curDevNum = iter->second.size();
299 566 : if (devNum != curDevNum) {
300 0 : HCCL_INFO(
301 : "[Check][SuperPodInfo]devNum[%u] in superPodIdx[%u] is inconsistent with "
302 : "devNum[%u] in superPodIdx[%u].",
303 : devNum, superPodToRank_.begin(), curDevNum, iter->first);
304 : }
305 : }
306 525 : return HCCL_SUCCESS;
307 : }
308 :
309 488 : HcclResult TopoInfoExtractor::CheckServerInfo()
310 : {
311 : /*
312 : * HCOM模块：
313 : * 1、每个AI server之间的芯片个数必须一致，不一致则报错；
314 : * 2、每个AI server之间的芯片ID（device
315 : * ID）必须相同（server0里面devID分别是0、1、4、5；server1->server127也必须是相同的） ，不一致则报错； HCCL
316 : * API模块： 3、校验rank_table传进来devID，与rt_get_device查询到的devID，是否相同，不一致则报错（针对当前设备）
317 : * 因此，上层模块已经校验过的不再重复，本函数仅用于校验每个server内的设备个数与topo类型的组合是否正确
318 : */
319 488 : u32 moduleIdx = 0;
320 488 : CHK_RET(GetModuleIdx(rankData_, moduleIdx));
321 488 : auto iterRank = serverToRank_.find(moduleIdx); // 查询本rank所在服务器
322 488 : bool check = (iterRank == serverToRank_.end());
323 488 : CHK_PRT_RET(
324 : check, HCCL_ERROR("[Check][ServerInfo]can't find serverId[%s] in rank map", rankData_.serverId.c_str()),
325 : HCCL_E_NOT_FOUND);
326 :
327 488 : HcclResult ret = HCCL_SUCCESS;
328 :
329 488 : switch (topoType_) {
330 124 : case TopoType::TOPO_TYPE_NP_MESH:
331 : case TopoType::TOPO_TYPE_4P_MESH:
332 : case TopoType::TOPO_TYPE_2P_MESH:
333 : case TopoType::
334 : TOPO_TYPE_1P_MESH: { // 4p_mesh场景下，支持server(4P+4P)和server(4P)+server(4P)，2p_mesh/1p_mesh同理
335 124 : ret = (((iterRank->second).size() == ranksOneNode_[static_cast<u32>(topoType_)])
336 0 : || ((iterRank->second).size()
337 0 : == 2 * ranksOneNode_[static_cast<u32>(topoType_)])) // 2表示8P满配走4PMESH算法
338 124 : ?
339 : HCCL_SUCCESS :
340 : HCCL_E_UNAVAIL;
341 124 : CHK_PRT_RET(
342 : ret != HCCL_SUCCESS,
343 : HCCL_ERROR(
344 : "[Check][ServerInfo]check server info err:server rank size[%llu], expected "
345 : "value[%u], topo type[%d]",
346 : (iterRank->second).size(), ranksOneNode_[static_cast<u32>(topoType_)], topoType_),
347 : HCCL_E_UNAVAIL);
348 124 : break;
349 : }
350 332 : case TopoType::TOPO_TYPE_NP_SINGLE_RING:
351 332 : ret = ((iterRank->second).size() == ranksOneNode_[static_cast<u32>(topoType_)]) ? HCCL_SUCCESS :
352 : HCCL_E_UNAVAIL;
353 332 : CHK_PRT_RET(
354 : ret != HCCL_SUCCESS,
355 : HCCL_ERROR(
356 : "[Check][ServerInfo]check server info err:server rank size[%llu], expected "
357 : "less than value[%u], topo type[%d]",
358 : (iterRank->second).size(), ranksOneNode_[static_cast<u32>(topoType_)], topoType_),
359 : HCCL_E_UNAVAIL);
360 332 : break;
361 0 : case TopoType::TOPO_TYPE_HETEROG:
362 : case TopoType::TOPO_TYPE_ES_MESH:
363 0 : break;
364 32 : default: { // 8P_RING or 4P_RING
365 32 : ret = ((iterRank->second).size() == ranksOneNode_[static_cast<u32>(topoType_)]) ? HCCL_SUCCESS :
366 : HCCL_E_UNAVAIL;
367 32 : CHK_PRT_RET(
368 : ret != HCCL_SUCCESS,
369 : HCCL_ERROR(
370 : "[Check][ServerInfo]check server info err:server rank size[%llu], expected "
371 : "value[%u], topo type[%d]",
372 : (iterRank->second).size(), ranksOneNode_[static_cast<u32>(topoType_)], topoType_),
373 : HCCL_E_UNAVAIL);
374 31 : break;
375 : }
376 : }
377 :
378 487 : HCCL_INFO(
379 : "check server info:server rank size[%llu], expected value[%u], topo type[%d]", (iterRank->second).size(),
380 : ranksOneNode_[static_cast<u32>(topoType_)], topoType_);
381 487 : return ret;
382 : }
383 :
384 5876 : HcclResult TopoInfoExtractor::GetServerIdx(const RankInfo& rankInfo, u32& serverIdx) const
385 : {
386 : // 通过ranktable指定集群信息场景，可以调整server在ranktable的排序(serverIdx）来指定server间通信的topo，从优化通信拓扑
387 : // rootInfo初始化场景，会自动收集集群信息，外部无法指定server的排序，可以无视serverIdx，使用serverID来代替
388 : // PS:返回的serverIdx，会影响rankMap_中server的排序，从而影响bridgeRank的选择，优化通信拓扑
389 5876 : CHK_PRT_RET((rankInfo.serverIdx == INVALID_UINT), HCCL_ERROR("server idx is invalid."), HCCL_E_INTERNAL);
390 5876 : serverIdx = rankInfo.serverIdx;
391 5876 : return HCCL_SUCCESS;
392 : }
393 :
394 525 : HcclResult TopoInfoExtractor::SetTopologyInfo()
395 : {
396 525 : CHK_RET(SetTopoDefaultInfo());
397 :
398 525 : CHK_RET(SetTopoInfoForLevel0());
399 525 : CHK_RET(SetTopoInfoForLevel1());
400 525 : CHK_RET(SetTopoInfoForLevel2());
401 525 : CHK_RET(SetTopoInfoForARS());
402 525 : CHK_RET(SetTopoInfoForCombineL1());
403 :
404 525 : CHK_RET(SetAHCSubGroupsAndAlgOption());
405 :
406 : // 是否支持按mesh划分通信拓扑
407 525 : bool isSupportMeshTopo = meshAggregationRankSize_ > 0 && userRankSize_ % meshAggregationRankSize_ == 0;
408 525 : if (isSupportMeshTopo) {
409 523 : CHK_RET(SetTopoInfoForMeshL0());
410 523 : CHK_RET(SetTopoInfoForMeshL1());
411 : } else {
412 2 : HCCL_INFO(
413 : "[Set][TopologyInfo]topo is not support Mesh, meshAggregationRankSize_[%u], userRankSize_[%u]",
414 : meshAggregationRankSize_, userRankSize_);
415 : }
416 525 : CommPlaneVector_[COMM_COMBINE_ORDER].push_back(rankVector_);
417 525 : return HCCL_SUCCESS;
418 : }
419 :
420 525 : HcclResult TopoInfoExtractor::SetTopoDefaultInfo()
421 : {
422 : // 填充combined_rank_vector_:不区分board_type,只生成default单层拓扑
423 525 : std::vector<RankInfo> tmpCombinedVector;
424 :
425 525 : bool incrementFlag = true; // 节点间建链的两个deviceID必须相同(同一个网段平面)，server间需要特殊处理
426 : // 维护topo输出的信息
427 525 : std::string outLogInfo = "userRank/devicePhyId: ";
428 525 : RankInfo tempRankData;
429 :
430 : // 填充combined_rank_vector_的内层vector:combined场景只有一条固定的环
431 1197 : for (auto iterMap = serverToRank_.begin(); iterMap != serverToRank_.end(); iterMap++) {
432 696 : if (!(iterMap->second).empty()) {
433 : // TOPO_TYPE_COMMON为单环拓扑场景，不需要执行如下判断设置流程
434 696 : if (topoType_ != TopoType::TOPO_TYPE_COMMON) {
435 658 : if (((iterMap->second).size() == SERVER_RANK_SIZE) && !IsDiffDeviceModuleInServer()
436 658 : && (topoType_ != TopoType::TOPO_TYPE_HETEROG) && !Is310PDevice()) {
437 48 : CHK_RET(SetTopoDefaultInfoFor8P()); // 服务器内dev个数相同已在hcom层做过校验
438 24 : return HCCL_SUCCESS;
439 : }
440 : }
441 :
442 672 : if (incrementFlag) {
443 1746 : for (u32 incrementIndex = 0; incrementIndex < (iterMap->second).size(); incrementIndex++) {
444 1199 : u32 combinedUserRank = (iterMap->second)[incrementIndex].userRank;
445 1199 : bool checkError = (rankVector_.size() <= combinedUserRank);
446 1199 : CHK_PRT_RET(
447 : checkError,
448 : HCCL_ERROR(
449 : "[Set][TopoDefaultInfo]combined userRank[%u] is bigger than "
450 : "rank vector",
451 : combinedUserRank),
452 : HCCL_E_INTERNAL);
453 1199 : tempRankData = rankVector_[combinedUserRank];
454 1199 : outLogInfo.append(std::to_string(tempRankData.userRank));
455 1199 : outLogInfo.append("/");
456 1199 : outLogInfo.append(std::to_string(tempRankData.devicePhyId));
457 1199 : outLogInfo.append("; ");
458 1199 : tmpCombinedVector.push_back(tempRankData);
459 : }
460 :
461 547 : incrementFlag = false;
462 : } else {
463 325 : for (u32 decrementIndex = (iterMap->second).size(); decrementIndex > 0; decrementIndex--) {
464 200 : u32 combinedUserRank = (iterMap->second)[decrementIndex - 1].userRank;
465 200 : bool checkError = (rankVector_.size() <= combinedUserRank);
466 200 : CHK_PRT_RET(
467 : checkError,
468 : HCCL_ERROR(
469 : "[Set][TopoDefaultInfo]combined userRank[%u] is bigger than "
470 : "rank vector",
471 : combinedUserRank),
472 : HCCL_E_INTERNAL);
473 200 : tempRankData = rankVector_[combinedUserRank];
474 200 : outLogInfo.append(std::to_string(tempRankData.userRank));
475 200 : outLogInfo.append("/");
476 200 : outLogInfo.append(std::to_string(tempRankData.devicePhyId));
477 200 : outLogInfo.append("; ");
478 200 : tmpCombinedVector.push_back(tempRankData);
479 : }
480 :
481 125 : incrementFlag = true;
482 : }
483 : }
484 : }
485 501 : if (topoType_ == TopoType::TOPO_TYPE_COMMON) {
486 38 : std::sort(tmpCombinedVector.begin(), tmpCombinedVector.end(), CompareWithUserRankAscend);
487 : }
488 :
489 501 : CommPlaneVector_[COMM_COMBINE].push_back(tmpCombinedVector);
490 501 : return HCCL_SUCCESS;
491 525 : }
492 :
493 525 : HcclResult TopoInfoExtractor::CheckPlaneInfo()
494 : {
495 525 : bool isTopoComm = (topoType_ == TopoType::TOPO_TYPE_COMMON) && (CommPlaneVector_[COMM_COMBINE].size() != 1);
496 525 : CHK_PRT_RET(
497 : isTopoComm,
498 : HCCL_ERROR(
499 : "[Check][PlaneInfo]topo type[%d] and combined plane nub[%llu] are not match", topoType_,
500 : CommPlaneVector_[COMM_COMBINE].size()),
501 : HCCL_E_INTERNAL);
502 :
503 525 : bool isTopo8pring = (topoType_ == TopoType::TOPO_TYPE_8P_RING)
504 549 : && ((CommPlaneVector_[COMM_LEVEL0].size() != meshAggregationRankSize_)
505 24 : || (ranksOneNode_[static_cast<u32>(topoType_)] != CommPlaneVector_[COMM_LEVEL1].size()));
506 525 : CHK_PRT_RET(
507 : isTopo8pring,
508 : HCCL_ERROR(
509 : "[Check][PlaneInfo]topo type[%d], level0 plane nub[%llu], level1 plane nub[%llu], is not match", topoType_,
510 : CommPlaneVector_[COMM_LEVEL0].size(), CommPlaneVector_[COMM_LEVEL1].size()),
511 : HCCL_E_INTERNAL);
512 :
513 525 : bool isTopo2pring = (topoType_ == TopoType::TOPO_TYPE_NP_DOUBLE_RING)
514 532 : && ((CommPlaneVector_[COMM_LEVEL0].size() != 2) || // 2表示一个节点内通信域里面是否只有2个ring
515 7 : (ranksOneNode_[static_cast<u32>(topoType_)] != CommPlaneVector_[COMM_LEVEL1].size()));
516 525 : CHK_PRT_RET(
517 : isTopo2pring,
518 : HCCL_ERROR(
519 : "[Check][PlaneInfo]topo type[%d], level0 plane nub[%llu], level1 plane nub[%llu], is not match", topoType_,
520 : CommPlaneVector_[COMM_LEVEL0].size(), CommPlaneVector_[COMM_LEVEL1].size()),
521 : HCCL_E_INTERNAL);
522 :
523 : bool isTopo4pRing
524 525 : = (topoType_ == TopoType::TOPO_TYPE_4P_RING)
525 525 : && ((CommPlaneVector_[COMM_LEVEL0].size() != 1) || // 1表示一个节点内通信域里面是否只有一个device
526 0 : (ranksOneNode_[static_cast<u32>(topoType_)] != CommPlaneVector_[COMM_LEVEL1].size()));
527 525 : CHK_PRT_RET(
528 : isTopo4pRing,
529 : HCCL_ERROR(
530 : "[Check][PlaneInfo]topo type[%d], level0 plane nub[%llu], level1 plane nub[%llu], is not match", topoType_,
531 : CommPlaneVector_[COMM_LEVEL0].size(), CommPlaneVector_[COMM_LEVEL1].size()),
532 : HCCL_E_INTERNAL);
533 :
534 525 : bool isTopo4pMesh = (topoType_ == TopoType::TOPO_TYPE_4P_MESH)
535 546 : && ((CommPlaneVector_[COMM_LEVEL0].size() != (ranksOneNode_[static_cast<u32>(topoType_)] - 1))
536 21 : || (ranksOneNode_[static_cast<u32>(topoType_)] != CommPlaneVector_[COMM_LEVEL1].size()));
537 525 : CHK_PRT_RET(
538 : isTopo4pMesh,
539 : HCCL_ERROR(
540 : "[Check][PlaneInfo]topo type[%d], level0 plane nub[%llu], level1 plane nub[%llu], is not match", topoType_,
541 : CommPlaneVector_[COMM_LEVEL0].size(), CommPlaneVector_[COMM_LEVEL1].size()),
542 : HCCL_E_INTERNAL);
543 :
544 525 : bool isTopoNpMesh = (topoType_ == TopoType::TOPO_TYPE_NP_MESH)
545 628 : && ((CommPlaneVector_[COMM_LEVEL0].size() != (ranksOneNode_[static_cast<u32>(topoType_)] - 1))
546 103 : || (ranksOneNode_[static_cast<u32>(topoType_)] != CommPlaneVector_[COMM_LEVEL1].size()));
547 525 : CHK_PRT_RET(
548 : isTopoNpMesh,
549 : HCCL_ERROR(
550 : "[Check][PlaneInfo]topo type[%d], level0 plane nub[%llu], level1 plane nub[%llu], is not match", topoType_,
551 : CommPlaneVector_[COMM_LEVEL0].size(), CommPlaneVector_[COMM_LEVEL1].size()),
552 : HCCL_E_INTERNAL);
553 :
554 : // 1表示一个module里面是否只有一个device
555 525 : bool isTopo2pMesh = (topoType_ == TopoType::TOPO_TYPE_2P_MESH)
556 525 : && ((CommPlaneVector_[COMM_LEVEL0].size() != 1)
557 0 : || (ranksOneNode_[static_cast<u32>(topoType_)] != CommPlaneVector_[COMM_LEVEL1].size()));
558 525 : CHK_PRT_RET(
559 : isTopo2pMesh,
560 : HCCL_ERROR(
561 : "[Check][PlaneInfo]topo type[%d], level0 plane nub[%llu], level1 plane nub[%llu], is not match", topoType_,
562 : CommPlaneVector_[COMM_LEVEL0].size(), CommPlaneVector_[COMM_LEVEL1].size()),
563 : HCCL_E_INTERNAL);
564 :
565 : bool isTopo1pMesh
566 525 : = (topoType_ == TopoType::TOPO_TYPE_1P_MESH)
567 525 : && ((CommPlaneVector_[COMM_LEVEL0].size() != 1) || // 1表示一个节点内通信域里面是否只有一个device
568 0 : (ranksOneNode_[static_cast<u32>(topoType_)] != CommPlaneVector_[COMM_LEVEL1].size()));
569 525 : CHK_PRT_RET(
570 : isTopo1pMesh,
571 : HCCL_ERROR(
572 : "[Check][PlaneInfo]topo type[%d], level0 plane nub[%llu], level1 plane nub[%llu], is not match", topoType_,
573 : CommPlaneVector_[COMM_LEVEL0].size(), CommPlaneVector_[COMM_LEVEL1].size()),
574 : HCCL_E_INTERNAL);
575 :
576 : bool isTopoNpSingleRing
577 332 : = (topoType_ == TopoType::TOPO_TYPE_NP_SINGLE_RING) && (!IsDiffDeviceModuleInServer())
578 1189 : && ((CommPlaneVector_[COMM_LEVEL0].size() != 1)
579 332 : || (CommPlaneVector_[COMM_LEVEL1].size() != ranksOneNode_[static_cast<u32>(topoType_)]));
580 525 : CHK_PRT_RET(
581 : isTopoNpSingleRing,
582 : HCCL_ERROR(
583 : "[Check][PlaneInfo]topo type[%d], level0 plane nub[%llu], level1 plane nub[%llu], is not match", topoType_,
584 : CommPlaneVector_[COMM_LEVEL0].size(), CommPlaneVector_[COMM_LEVEL1].size()),
585 : HCCL_E_INTERNAL);
586 :
587 525 : HCCL_RUN_INFO(
588 : "plane info:topo type[%d], device type[%d], COMM_COMBINE size[%llu], COMM_LEVEL0 size[%llu], COMM_LEVEL1 "
589 : "size[%llu], COMM_LEVEL2 size[%llu], COMM_MESH_L0 size[%llu], COMM_MESH_L1 size[%llu]",
590 : topoType_, deviceType_, CommPlaneVector_[COMM_COMBINE].size(), CommPlaneVector_[COMM_LEVEL0].size(),
591 : CommPlaneVector_[COMM_LEVEL1].size(), CommPlaneVector_[COMM_LEVEL2].size(),
592 : CommPlaneVector_[COMM_MESH_L0].size(), CommPlaneVector_[COMM_MESH_L1].size());
593 :
594 525 : return HCCL_SUCCESS;
595 : }
596 :
597 525 : HcclResult TopoInfoExtractor::SetTopoInfoForLevel0()
598 : {
599 525 : u32 moduleIdx = 0;
600 525 : CHK_RET(GetModuleIdx(rankData_, moduleIdx));
601 525 : auto iterRank = serverToRank_.find(moduleIdx); // 查询本rank所在服务器
602 525 : bool check = (iterRank == serverToRank_.end());
603 525 : CHK_PRT_RET(
604 : check, HCCL_ERROR("[Set][TopoInfoForLevel0]can't find serverId[%s] in rank map", rankData_.serverId.c_str()),
605 : HCCL_E_NOT_FOUND);
606 : // 查询本rank所在服务器的rank数
607 525 : u32 ranksSize = (iterRank->second).size();
608 :
609 525 : multiLevel0Order_.clear();
610 : // 生成mockNicList
611 525 : std::vector<u32> mockNicList;
612 525 : mockNicList.reserve(ranksSize);
613 1862 : for (u32 startIndex = 0; startIndex < ranksSize; startIndex++) {
614 1337 : mockNicList.push_back(startIndex);
615 : }
616 :
617 525 : multiLevel0Order_ = GetRingsOrderByTopoType(ranksSize, topoType_, mockNicList);
618 :
619 525 : HCCL_DEBUG(
620 : "[TopoInfoExtractor] The ring number is %zu, the rank size is %lu.", multiLevel0Order_.size(), ranksSize);
621 525 : if (multiLevel0Order_.size() == 1) {
622 494 : CHK_RET(SetSingleLevel0());
623 : } else { // 8p-ring/np ring 环场景
624 31 : u32 ringNum = multiLevel0Order_.size();
625 31 : CHK_RET(SetMultiLevel0(ringNum)); // 8P_RING场景下，外层拓扑中有四个环; 910_93场景中适配双环
626 : }
627 :
628 525 : AHCCommSubgroupInit(); // 准备 AHC COMM 场景下的分组
629 :
630 525 : return HCCL_SUCCESS;
631 525 : }
632 :
633 1050 : HcclResult TopoInfoExtractor::SetTopoInfoForLevel1(bool prepareAHC)
634 : {
635 1050 : std::map<u32, std::vector<RankInfo>>& serverToRank = (prepareAHC) ? serverToRankMerge_ : serverToRank_;
636 :
637 1050 : CommPlane commPlaneLevel1 = (prepareAHC) ? COMM_LEVEL1_AHC : COMM_LEVEL1;
638 :
639 1050 : u32 moduleIdx = 0;
640 1050 : CHK_RET(GetModuleIdx(rankData_, moduleIdx));
641 1050 : auto iterRank = serverToRank.find(moduleIdx); // 查询本rank所在服务器
642 1050 : bool check = (iterRank == serverToRank.end());
643 1050 : CHK_PRT_RET(
644 : check, HCCL_ERROR("[Set][TopoInfoForLevel1]can't find serverId[%s] in rank map", rankData_.serverId.c_str()),
645 : HCCL_E_NOT_FOUND);
646 :
647 : u32 ringSize;
648 1050 : if (topoType_ == TopoType::TOPO_TYPE_2P_MESH) { // 2P_MESH在任何情况下，内层拓扑平面始终为2
649 0 : ringSize = ranksOneNode_[static_cast<u32>(topoType_)];
650 1050 : } else if (topoType_ == TopoType::TOPO_TYPE_NP_SINGLE_RING) { // 标卡内层拓扑的环数
651 664 : ringSize = (iterRank->second).size();
652 : } else { // 其他场景下内层拓扑平面为每个module中的device数量
653 386 : ringSize = ranksOneNode_[static_cast<u32>(topoType_)];
654 : }
655 1050 : HCCL_INFO("[Set][TopoInfoForLevel1] topoType_[%u] ringSize[%u]", topoType_, ringSize);
656 :
657 : // 计算每个 level 环的超节点分组，每个环都一致，只计算一次
658 1050 : std::vector<std::string> idsForIndexingVector;
659 1050 : std::map<std::string, std::vector<u32>> aggregatedSubGroup;
660 1050 : bool calcGroupDone = false;
661 :
662 : // 内层拓扑的每层环
663 3264 : for (u32 ringIndex = 0; ringIndex < ringSize; ringIndex++) {
664 2214 : std::vector<RankInfo> tmpBridgeVector;
665 2214 : bool bridgeRankFlag = false;
666 2214 : std::string outLogInfo = ""; // 维护topo输出的信息
667 2214 : outLogInfo.append(" ringIndex: ");
668 2214 : outLogInfo.append(std::to_string(ringIndex));
669 2214 : outLogInfo.append(", ");
670 2214 : outLogInfo.append("userRank/serverId/devicePhyId/nicIp/isBridgeRank: ");
671 :
672 : // 2、填充bridge_rank_vector_的内层vector和is_bridge_vector_
673 2214 : u32 subGroupIndex = 0;
674 4875 : for (auto iterMap = serverToRank.begin(); iterMap != serverToRank.end(); iterMap++) {
675 2661 : if (!(iterMap->second).empty()) {
676 2661 : RankInfo tmpBridgePara;
677 2661 : u32 bridgeUserRank = (iterMap->second)[ringIndex].userRank;
678 2661 : u32 bridgeDevicePhyId = (iterMap->second)[ringIndex].devicePhyId;
679 2661 : std::vector<u32> bridgeNic((iterMap->second)[ringIndex].nicIdx);
680 2661 : bool checkError = (rankVector_.size() <= bridgeUserRank);
681 2661 : CHK_PRT_RET(
682 : checkError,
683 : HCCL_ERROR(
684 : "[Set][TopoInfoForLevel1]bridge userRank[%u] is bigger than rank "
685 : "vector",
686 : bridgeUserRank),
687 : HCCL_E_INTERNAL);
688 2661 : CHK_RET(SetBridgeLinkInfo(tmpBridgePara, bridgeUserRank));
689 2661 : tmpBridgeVector.push_back(tmpBridgePara);
690 2661 : std::vector<u32>::iterator iterNic = std::find(bridgeNic.begin(), bridgeNic.end(), bridgeDevicePhyId);
691 2661 : if ((bridgeNic.size() == 0) || (iterNic != bridgeNic.end())) {
692 2659 : if (bridgeUserRank == static_cast<u32>(userRank_)) { // 本rank是否为bridge_rank
693 974 : bridgeRankFlag = true;
694 : }
695 : }
696 :
697 2661 : outLogInfo.append(std::to_string(tmpBridgePara.userRank));
698 2661 : outLogInfo.append("/");
699 2661 : outLogInfo.append(tmpBridgePara.serverId);
700 2661 : outLogInfo.append("/");
701 2661 : outLogInfo.append(std::to_string(tmpBridgePara.devicePhyId));
702 2661 : outLogInfo.append("/");
703 2661 : outLogInfo.append(tmpBridgePara.nicIp[0].GetReadableAddress());
704 2661 : outLogInfo.append("/");
705 2661 : outLogInfo.append(std::to_string(bridgeRankFlag));
706 2661 : outLogInfo.append("; ");
707 :
708 : // 环内填充 aggregatedSubGroup superPodId-> subGroupIndex 用于生成分组信息
709 2661 : if (!calcGroupDone
710 1317 : && (deviceType_ == DevType::DEV_TYPE_910_93 || deviceType_ == DevType::DEV_TYPE_910B)) {
711 547 : std::string IdForIndexing = (iterMap->second)[ringIndex].superPodId;
712 547 : auto itIndex = aggregatedSubGroup.find(IdForIndexing);
713 547 : if (itIndex != aggregatedSubGroup.end()) {
714 170 : itIndex->second.push_back(subGroupIndex);
715 : } else {
716 377 : idsForIndexingVector.push_back(IdForIndexing);
717 377 : std::vector<u32> subGroup;
718 377 : subGroup.push_back(subGroupIndex);
719 377 : aggregatedSubGroup.insert(std::make_pair(IdForIndexing, subGroup));
720 377 : }
721 547 : HCCL_INFO(
722 : "[Set][TopoInfoForLevel1] calc subGroup superPodId[%s] subIndex[%u]", IdForIndexing.c_str(),
723 : subGroupIndex);
724 547 : }
725 :
726 2661 : subGroupIndex = subGroupIndex + 1;
727 2661 : }
728 : }
729 :
730 3276 : for (auto it = aggregatedSubGroup.begin(); it != aggregatedSubGroup.end(); it++) {
731 1062 : HCCL_DEBUG("[Set][TopoInfoForLevel1][AHC_DEBUG] superPodId[%s]", it->first.c_str());
732 2397 : for (auto index = it->second.begin(); index != it->second.end(); index++) {
733 1335 : HCCL_DEBUG("[Set][TopoInfoForLevel1][AHC_DEBUG] groupIndex[%u]", (*index));
734 : }
735 : }
736 :
737 4875 : for (auto it = tmpBridgeVector.begin(); it != tmpBridgeVector.end(); it++) {
738 2661 : HCCL_DEBUG(
739 : "[Set][TopoInfoForLevel1][AHC_DEBUG] ringIndex[%u] tmpBridgevector userRank[%u]", ringIndex,
740 : it->userRank);
741 : }
742 :
743 : // 3、填充bridge_rank_vector_、isBridgeVector_
744 2214 : if (!prepareAHC) {
745 1107 : isBridgeVector_.push_back(bridgeRankFlag);
746 : }
747 2214 : CommPlaneVector_[commPlaneLevel1].push_back(tmpBridgeVector);
748 :
749 : // 4、填充当前 level 的通信域内分组信息（用于层次化算法）
750 2214 : if (!calcGroupDone) {
751 974 : std::vector<std::vector<u32>> SubGroups;
752 1351 : for (u32 i = 0; i < idsForIndexingVector.size(); i++) {
753 377 : std::string idForIndexing = idsForIndexingVector[i];
754 377 : SubGroups.push_back(aggregatedSubGroup[idForIndexing]);
755 377 : }
756 974 : CommPlaneSubGroupVector_[commPlaneLevel1].push_back(SubGroups);
757 974 : calcGroupDone = true;
758 974 : }
759 :
760 2214 : HCCL_INFO("SetTopoInfoForLevel1: topoRankInfo[%s]", outLogInfo.c_str());
761 2214 : }
762 :
763 1050 : HCCL_RUN_INFO(
764 : "SetTopoInfoForLevel1: identifier[%s], userRank[%u], userRankSize[%u], plane size[%u]", identifier_.c_str(),
765 : userRank_, userRankSize_, CommPlaneVector_[commPlaneLevel1].size());
766 1050 : return HCCL_SUCCESS;
767 1050 : }
768 :
769 525 : HcclResult TopoInfoExtractor::SetTopoInfoForLevel2()
770 : {
771 525 : bool isLevel2Support
772 33 : = ((deviceType_ == DevType::DEV_TYPE_910_93) && multiModuleDiffDeviceNumMode_
773 558 : && !multiSuperPodDiffDeviceNumMode_);
774 : // 对称场景需要初始化多个平面，超节点内rank数一致也可切分平面；其他非对称场景 level1 和 level2 合并无需切分平面
775 525 : if (((!multiModuleDiffDeviceNumMode_ && !multiSuperPodDiffServerNumMode_) || isLevel2Support)
776 517 : && !isDiffDeviceType_) {
777 517 : HCCL_INFO("[Set][TopoInfoForLevel2] select origin proc");
778 :
779 : // 找到当前rank在本超节点内部的序号
780 517 : auto it = superPodToRank_.find(rankData_.superPodIdx);
781 517 : CHK_PRT_RET(
782 : it == superPodToRank_.end(),
783 : HCCL_ERROR(
784 : "[Set][TopoInfoForLevel2]superPodIdx[%u] is not exist in superPodRankMap", rankData_.superPodIdx),
785 : HCCL_E_INTERNAL);
786 :
787 517 : u32 index = 0;
788 832 : for (; index < it->second.size(); ++index) {
789 832 : if (userRank_ == it->second[index].userRank) {
790 517 : break;
791 : }
792 : }
793 517 : CHK_PRT_RET(
794 : index >= it->second.size(),
795 : HCCL_ERROR(
796 : "[Set][TopoInfoForLevel2]userRank_[%u] superPodId[%s] superPodIdx[%u] not exist in superPodRankMap",
797 : userRank_, rankData_.superPodId.c_str(), rankData_.superPodIdx),
798 : HCCL_E_INTERNAL);
799 :
800 517 : std::vector<RankInfo> tmpRankVec;
801 1075 : for (auto iterMap = superPodToRank_.begin(); iterMap != superPodToRank_.end(); iterMap++) {
802 558 : CHK_PRT_RET(
803 : iterMap->second.size() <= index,
804 : HCCL_ERROR(
805 : "[Set][TopoInfoForLevel2]index[%u] is bigger than rank vector size[%u]", index,
806 : iterMap->second.size()),
807 : HCCL_E_INTERNAL);
808 :
809 558 : RankInfo& tempRankData = iterMap->second[index];
810 558 : tmpRankVec.push_back(tempRankData);
811 :
812 : // 维护topo输出的信息
813 558 : std::string outLogInfo = "userRank/devicePhyId/serverIdx/superPodId: ";
814 558 : outLogInfo.append(std::to_string(tempRankData.userRank));
815 558 : outLogInfo.append("/");
816 558 : outLogInfo.append(std::to_string(tempRankData.devicePhyId));
817 558 : outLogInfo.append("/");
818 558 : outLogInfo.append(std::to_string(tempRankData.serverIdx));
819 558 : outLogInfo.append("/");
820 558 : outLogInfo.append(tempRankData.superPodId);
821 558 : outLogInfo.append("; ");
822 558 : HCCL_INFO("SetTopoInfoForLevel2: topoRankInfo[%s]", outLogInfo.c_str());
823 558 : }
824 :
825 517 : CommPlaneVector_[COMM_LEVEL2].push_back(tmpRankVec);
826 517 : HCCL_RUN_INFO(
827 : "SetTopoInfoForLevel2: identifier[%s], userRank[%u], userRankSize[%u], plane size[%u]", identifier_.c_str(),
828 : userRank_, userRankSize_, CommPlaneVector_[COMM_LEVEL2].size());
829 517 : }
830 :
831 525 : return HCCL_SUCCESS;
832 : }
833 :
834 523 : HcclResult TopoInfoExtractor::SetTopoInfoForMeshL0()
835 : {
836 : // 以MeshAggregation为粒度、MeshAggregation内各设备的mesh建链
837 523 : u32 rankSize = meshAggregationRankSize_;
838 523 : u32 userRankIndexBegin = userRank_ / meshAggregationRankSize_ * meshAggregationRankSize_;
839 523 : u32 userRankIndexEnd = userRankIndexBegin + meshAggregationRankSize_;
840 523 : std::vector<RankInfo> paraVector(rankSize);
841 523 : u32 rankIndex = 0;
842 523 : std::string outLogInfo = "userRank/devicePhyId: "; // 维护topo输出的信息
843 :
844 523 : CHK_PRT_RET(
845 : rankVector_.size() < userRankIndexEnd,
846 : HCCL_ERROR(
847 : "[Set][TopoInfoForMeshL0]rankVector_ size[%u] should be greater than userRankIndexEnd[%u]",
848 : rankVector_.size(), userRankIndexEnd),
849 : HCCL_E_PARA);
850 :
851 1750 : for (u32 i = userRankIndexBegin; i < userRankIndexEnd; i++) {
852 1227 : paraVector[rankIndex] = rankVector_[i];
853 1227 : outLogInfo.append(std::to_string(paraVector[rankIndex].userRank));
854 1227 : outLogInfo.append("/");
855 1227 : outLogInfo.append(std::to_string(paraVector[rankIndex].devicePhyId));
856 1227 : outLogInfo.append("; ");
857 1227 : rankIndex++;
858 : }
859 523 : CommPlaneVector_[COMM_MESH_L0].push_back(paraVector);
860 523 : return HCCL_SUCCESS;
861 523 : }
862 :
863 523 : HcclResult TopoInfoExtractor::SetTopoInfoForMeshL1()
864 : {
865 : // 以MeshAggregation为粒度、MeshAggregation间各平面的mesh建链
866 523 : u32 rankSize = userRankSize_ / meshAggregationRankSize_; // 1 = 7 / 4
867 523 : u32 planeID = userRank_ % meshAggregationRankSize_; // 0
868 523 : std::vector<RankInfo> paraVector(rankSize);
869 :
870 523 : CHK_PRT_RET(
871 : rankVector_.size() < userRankSize_,
872 : HCCL_ERROR(
873 : "[Set][TopoInfoForMeshL1]rankVector_ size[%u] should be greater than userRankSize[%u]", rankVector_.size(),
874 : userRankSize_),
875 : HCCL_E_PARA);
876 :
877 1322 : for (u32 i = planeID; i < userRankSize_; i += meshAggregationRankSize_) {
878 799 : u32 rankIndex = i / meshAggregationRankSize_;
879 799 : paraVector[rankIndex] = rankVector_[i];
880 799 : std::string outLogInfo = "userRank/devicePhyId"; // 维护topo输出的信息
881 799 : outLogInfo.append(std::to_string(paraVector[rankIndex].userRank));
882 799 : outLogInfo.append("/");
883 799 : outLogInfo.append(std::to_string(paraVector[rankIndex].devicePhyId));
884 799 : outLogInfo.append("; ");
885 799 : HCCL_INFO("SetTopoInfoForMeshL1: topoRankInfo[%s]", outLogInfo.c_str());
886 799 : }
887 523 : CommPlaneVector_[COMM_MESH_L1].push_back(paraVector);
888 523 : return HCCL_SUCCESS;
889 523 : }
890 :
891 494 : HcclResult TopoInfoExtractor::SetSingleLevel0()
892 : {
893 : // 填充level0_rank_vector_，该函数处理场景非8P_RING算法
894 494 : std::vector<RankInfo> tmpLevel0Vector;
895 494 : u32 moduleIdx = 0;
896 494 : CHK_RET(GetModuleIdx(rankData_, moduleIdx));
897 494 : auto iterRank = serverToRank_.find(moduleIdx); // 查询本rank所在服务器
898 494 : bool check = (iterRank == serverToRank_.end());
899 494 : CHK_PRT_RET(
900 : check, HCCL_ERROR("[Set][SingleLevel0]can't find serverId[%s] in rank map", rankData_.serverId.c_str()),
901 : HCCL_E_NOT_FOUND);
902 :
903 494 : std::vector<s32> devicePhyIdVector;
904 1912 : for (u32 i = 0; i < rankVector_.size(); i++) {
905 1418 : devicePhyIdVector.push_back(rankVector_[i].devicePhyId);
906 : }
907 494 : s32 maxPhyId = *max_element(devicePhyIdVector.begin(), devicePhyIdVector.end());
908 : // 8P满配场景:4PMESH算法 + 8Pfullmesh + 16P仅使用左边module
909 518 : if (((iterRank->second).size() == DEVICE_PER_MODULE && maxPhyId < DEVICE_PER_MODULE)
910 518 : && (topoType_ == TopoType::TOPO_TYPE_4P_MESH || topoType_ == TopoType::TOPO_TYPE_NP_MESH)) {
911 0 : return SetSingleLevel0For8P(); // 服务器内dev个数相同已在hcom层做过校验
912 : }
913 :
914 : // 维护topo输出的信息
915 494 : std::string outLogInfo = "userRank/devicePhyId: ";
916 494 : RankInfo tempRankData;
917 : // 其他场景 + 16P使用右边module
918 1623 : for (u32 startIndex = 0; startIndex < (iterRank->second).size(); startIndex++) {
919 1129 : u32 level0StartRank = (iterRank->second)[startIndex].userRank;
920 1129 : bool checkError = (rankVector_.size() <= level0StartRank);
921 1129 : CHK_PRT_RET(
922 : checkError,
923 : HCCL_ERROR("[Set][SingleLevel0]level0 userRank[%u] is bigger than rank vector", level0StartRank),
924 : HCCL_E_INTERNAL);
925 1129 : tempRankData = rankVector_[level0StartRank];
926 :
927 1129 : outLogInfo.append(std::to_string(tempRankData.userRank));
928 1129 : outLogInfo.append("/");
929 1129 : outLogInfo.append(std::to_string(tempRankData.devicePhyId));
930 1129 : outLogInfo.append("; ");
931 1129 : tmpLevel0Vector.push_back(tempRankData);
932 : }
933 :
934 : // NPmesh或4Pmesh场景下，外层拓扑平面为device数量-1
935 494 : u32 level0Size = (topoType_ == TopoType::TOPO_TYPE_4P_MESH || topoType_ == TopoType::TOPO_TYPE_NP_MESH) ?
936 124 : (ranksOneNode_[static_cast<u32>(topoType_)] - 1) :
937 494 : 1;
938 :
939 1232 : for (u32 index = 0; index < level0Size; index++) {
940 738 : CommPlaneVector_[COMM_LEVEL0].push_back(tmpLevel0Vector);
941 : }
942 494 : HCCL_RUN_INFO(
943 : "SetTopoInfoForLevel0: identifier[%s], userRank[%u], userRankSize[%u], topoRankInfo[%s]", identifier_.c_str(),
944 : userRank_, userRankSize_, outLogInfo.c_str());
945 494 : return HCCL_SUCCESS;
946 494 : }
947 :
948 0 : HcclResult TopoInfoExtractor::SetSingleLevel0For8P()
949 : {
950 : // 该函数处理场景:8P满配、非8P_RING算法(8P满配下走4PMESH)
951 0 : std::vector<RankInfo> tmpLevel0Vector;
952 0 : u32 moduleIdx = 0;
953 0 : CHK_RET(GetModuleIdx(rankData_, moduleIdx));
954 0 : auto iterRank = serverToRank_.find(moduleIdx); // 查询本rank所在服务器
955 0 : bool check = (iterRank == serverToRank_.end());
956 0 : CHK_PRT_RET(
957 : check, HCCL_ERROR("[Set][SingleLevel0For8P]can't find serverId[%s] in rank map", rankData_.serverId.c_str()),
958 : HCCL_E_NOT_FOUND);
959 :
960 0 : u32 startIndex
961 0 : = (rankData_.devicePhyId < static_cast<s32>(meshAggregationRankSize_)) ? 0 : meshAggregationRankSize_;
962 0 : u32 devcount = 0;
963 : // 维护topo输出的信息
964 0 : std::string outLogInfo = "userRank/devicePhyId: ";
965 0 : RankInfo tempRankData;
966 0 : while (devcount < meshAggregationRankSize_) {
967 0 : u32 level0StartRank = (iterRank->second)[startIndex].userRank;
968 0 : bool checkError = (rankVector_.size() <= level0StartRank);
969 0 : CHK_PRT_RET(
970 : checkError,
971 : HCCL_ERROR("[Set][SingleLevel0For8P]level0 userRank[%u] is bigger than rank vector", level0StartRank),
972 : HCCL_E_INTERNAL);
973 0 : tempRankData = rankVector_[level0StartRank];
974 :
975 0 : outLogInfo.append(std::to_string(tempRankData.userRank));
976 0 : outLogInfo.append("/");
977 0 : outLogInfo.append(std::to_string(tempRankData.devicePhyId));
978 0 : outLogInfo.append("; ");
979 0 : tmpLevel0Vector.push_back(tempRankData);
980 0 : startIndex++;
981 0 : devcount++;
982 : }
983 :
984 : // 4PMESH场景下，外层拓扑3个平面
985 0 : u32 level0Size = ranksOneNode_[static_cast<u32>(topoType_)] - 1;
986 0 : for (u32 index = 0; index < level0Size; index++) {
987 0 : CommPlaneVector_[COMM_LEVEL0].push_back(tmpLevel0Vector);
988 : }
989 0 : HCCL_RUN_INFO(
990 : "SetTopoInfoForLevel0: identifier[%s], userRank[%u], userRankSize[%u], topoRankInfo[%s]", identifier_.c_str(),
991 : userRank_, userRankSize_, outLogInfo.c_str());
992 0 : return HCCL_SUCCESS;
993 0 : }
994 :
995 : /*
996 : * *********************************************************************************
997 : * 用来标识集群中是否存在910B A+X形态
998 : * **********************************************************************************
999 : */
1000 4588 : bool TopoInfoExtractor::IsDiffDeviceModuleInServer() const
1001 : {
1002 4588 : return deviceType_ == DevType::DEV_TYPE_910B && isDiffAggregation_;
1003 : }
1004 :
1005 31 : HcclResult TopoInfoExtractor::SetMultiLevel0(u32 ringNum)
1006 : {
1007 31 : std::vector<u32> tmpLevel0Order;
1008 31 : u32 moduleIdx = 0;
1009 31 : RankInfo tempRankData;
1010 31 : CHK_RET(GetModuleIdx(rankData_, moduleIdx));
1011 31 : auto iterRank = serverToRank_.find(moduleIdx); // 查询本rank所在服务器
1012 31 : bool check = (iterRank == serverToRank_.end());
1013 31 : CHK_PRT_RET(
1014 : check, HCCL_ERROR("[Set][MultiLevel0]can't find serverId[%s] in rank map", rankData_.serverId.c_str()),
1015 : HCCL_E_NOT_FOUND);
1016 :
1017 : // 维护topo输出的信息
1018 31 : std::string outLogInfo = "";
1019 141 : for (u32 ringIndex = 0; ringIndex < ringNum; ringIndex++) {
1020 110 : tmpLevel0Order = multiLevel0Order_[ringIndex]; // 获取每一个环的设备物理ID排序
1021 110 : std::vector<RankInfo> tmpLevel0Vector;
1022 110 : outLogInfo = "userRank/devicePhyId: ";
1023 910 : for (u32 startIndex = 0; startIndex < (iterRank->second).size(); startIndex++) {
1024 800 : u32 devIndex = tmpLevel0Order[startIndex];
1025 800 : u32 level0RingUserank = (iterRank->second)[devIndex].userRank;
1026 800 : bool checkError = (rankVector_.size() <= level0RingUserank);
1027 800 : CHK_PRT_RET(
1028 : checkError,
1029 : HCCL_ERROR("[Set][MultiLevel0]level0 userRank[%u] is bigger than rank vector", level0RingUserank),
1030 : HCCL_E_INTERNAL);
1031 800 : tempRankData = rankVector_[level0RingUserank];
1032 800 : outLogInfo.append(std::to_string(tempRankData.userRank));
1033 800 : outLogInfo.append("/");
1034 800 : outLogInfo.append(std::to_string(tempRankData.devicePhyId));
1035 800 : outLogInfo.append("; ");
1036 800 : tmpLevel0Vector.push_back(tempRankData);
1037 : }
1038 110 : HCCL_RUN_INFO(
1039 : "SetTopoInfoForLevel0: identifier[%s], userRank[%u], userRankSize[%u], topoRankInfo[%s]",
1040 : identifier_.c_str(), userRank_, userRankSize_, outLogInfo.c_str());
1041 110 : CommPlaneVector_[COMM_LEVEL0].push_back(tmpLevel0Vector);
1042 110 : }
1043 31 : return HCCL_SUCCESS;
1044 31 : }
1045 :
1046 : // anypath创建通信域
1047 0 : HcclResult TopoInfoExtractor::SetMultiLevel0AnyPath(std::vector<std::vector<u32>> multiOrder)
1048 : {
1049 0 : u32 ringNum = multiOrder.size();
1050 0 : std::vector<u32> tmpLevel0Order;
1051 0 : u32 moduleIdx = 0;
1052 0 : CHK_RET(GetModuleIdx(rankData_, moduleIdx));
1053 0 : auto iterRank = serverToRank_.find(moduleIdx); // 查询本rank所在服务器
1054 0 : bool check = (iterRank == serverToRank_.end());
1055 0 : CHK_PRT_RET(
1056 : check, HCCL_ERROR("[Set][MultiLevel0]can't find serverId[%s] in rank map", rankData_.serverId.c_str()),
1057 : HCCL_E_NOT_FOUND);
1058 :
1059 : // 维护topo输出的信息
1060 0 : std::string outLogInfo = "";
1061 0 : RankInfo tempRankData;
1062 0 : for (u32 ringIndex = 0; ringIndex < ringNum; ringIndex++) {
1063 0 : tmpLevel0Order = multiOrder[ringIndex]; // 获取每一个环的设备物理ID排序
1064 0 : std::vector<RankInfo> tmpLevel0Vector;
1065 0 : outLogInfo = "userRank/devicePhyId: ";
1066 0 : for (u32 startIndex = 0; startIndex < (iterRank->second).size(); startIndex++) {
1067 0 : u32 devIndex = tmpLevel0Order[startIndex];
1068 0 : u32 level0RingUserank = (iterRank->second)[devIndex].userRank;
1069 0 : bool checkError = (rankVector_.size() <= level0RingUserank);
1070 0 : CHK_PRT_RET(
1071 : checkError,
1072 : HCCL_ERROR("[Set][MultiLevel0]level0 userRank[%u] is bigger than rank vector", level0RingUserank),
1073 : HCCL_E_INTERNAL);
1074 0 : tempRankData = rankVector_[level0RingUserank];
1075 0 : outLogInfo.append(std::to_string(tempRankData.userRank));
1076 0 : outLogInfo.append("/");
1077 0 : outLogInfo.append(std::to_string(tempRankData.devicePhyId));
1078 0 : outLogInfo.append("; ");
1079 0 : tmpLevel0Vector.push_back(tempRankData);
1080 : }
1081 0 : HCCL_RUN_INFO(
1082 : "[AnyPath]SetTopoInfoForLevel0: identifier[%s], userRank[%u], userRankSize[%u], topoRankInfo[%s]",
1083 : identifier_.c_str(), userRank_, userRankSize_, outLogInfo.c_str());
1084 0 : CommPlaneVector_[COMM_LEVEL0_ANYPATH_SDMA].push_back(tmpLevel0Vector);
1085 0 : CommPlaneVector_[COMM_LEVEL0_ANYPATH_RDMA].push_back(tmpLevel0Vector);
1086 0 : }
1087 :
1088 0 : return HCCL_SUCCESS;
1089 0 : }
1090 :
1091 : // 集群中存在910B A+X时，0-7卡: moduleIdx = 2 * serverIdx; 8-15卡: moduleIdx = 2 * serverIdx + 1
1092 : // 集群中不存在910B A+X时，moduleIdx = serverIdx
1093 4232 : HcclResult TopoInfoExtractor::GetModuleIdx(const RankInfo& rankInfo, u32& moduleIdx)
1094 : {
1095 : // 获取moduleIdx，在16P同时使用左右两个module时，moduleIdx标识当前rank所在的module，其他场景下moduleIdx等同于serverIdx
1096 4232 : u32 serverIdx = 0;
1097 4232 : CHK_RET(GetServerIdx(rankInfo, serverIdx));
1098 4232 : if (isDiffDeviceType_) {
1099 0 : moduleIdx = rankInfo.userRank / gcdDeviceNumPerAggregation_;
1100 0 : HCCL_DEBUG(
1101 : "[TopoInfoExtractor][GetModuleIdx]serverIdx [%u] devicePhyId[%u] userRank[%u] moduleIdx[%u] "
1102 : "gcdDeviceNumPerAggregation[%u]",
1103 : serverIdx, rankInfo.devicePhyId, rankInfo.userRank, moduleIdx, gcdDeviceNumPerAggregation_);
1104 4232 : } else if (IsDiffDeviceModuleInServer()) {
1105 0 : moduleIdx = serverIdx * FACTOR_NUM_TWO + rankInfo.devicePhyId / DEVICE_PER_MODULE;
1106 : } else {
1107 4232 : moduleIdx = serverIdx;
1108 : }
1109 4232 : return HCCL_SUCCESS;
1110 : }
1111 :
1112 2661 : HcclResult TopoInfoExtractor::SetBridgeLinkInfo(RankInfo& bridgePara, u32 bridgeUserRank)
1113 : {
1114 2661 : bool checkSize = (rankVector_.size() <= bridgeUserRank);
1115 2661 : CHK_PRT_RET(
1116 : checkSize, HCCL_ERROR("[Set][BridgeLinkInfo]bridge UserRank %u is bigger than rank vector", bridgeUserRank),
1117 : HCCL_E_INTERNAL);
1118 :
1119 2661 : bridgePara = rankVector_[bridgeUserRank];
1120 2661 : return HCCL_SUCCESS;
1121 : }
1122 :
1123 24 : HcclResult TopoInfoExtractor::SetTopoDefaultInfoFor8P()
1124 : {
1125 : // 填充combined_rank_vector_:不区分board_type,只生成default单层拓扑
1126 24 : std::vector<RankInfo> tmpCombinedVector;
1127 : // 服务器内排序固定为0, 2, 3, 1, 5, 7, 6, 4，挑选8P多环中适用于combined的一组服务器内排序
1128 72 : std::vector<u32> devOrder = {0, 2, 3, 1, 5, 7, 6, 4};
1129 : // 维护topo输出的信息
1130 24 : std::string outLogInfo = "userRank/devicePhyId: ";
1131 :
1132 : // 填充combined_rank_vector_的内层vector:combined场景只有一条固定的环
1133 48 : for (auto iterMap = serverToRank_.begin(); iterMap != serverToRank_.end(); iterMap++) {
1134 : /* 服务器内8P满配单环特殊适配逻辑 */
1135 216 : for (u32 index = 0; index < devOrder.size(); index++) {
1136 192 : u32 devIndex = devOrder[index];
1137 192 : u32 combinedUserRank = (iterMap->second)[devIndex].userRank;
1138 :
1139 192 : bool checkError = (rankVector_.size() <= combinedUserRank);
1140 192 : CHK_PRT_RET(
1141 : checkError,
1142 : HCCL_ERROR(
1143 : "[Set][TopoDefaultInfoFor8P]combined userRank[%u] is bigger than rank vector", combinedUserRank),
1144 : HCCL_E_INTERNAL);
1145 :
1146 192 : RankInfo tmpCombinedPara = rankVector_[combinedUserRank];
1147 192 : outLogInfo.append(std::to_string(tmpCombinedPara.userRank));
1148 192 : outLogInfo.append("/");
1149 192 : outLogInfo.append(std::to_string(tmpCombinedPara.devicePhyId));
1150 192 : outLogInfo.append("; ");
1151 192 : tmpCombinedVector.push_back(tmpCombinedPara);
1152 192 : }
1153 : }
1154 :
1155 24 : CommPlaneVector_[COMM_COMBINE].push_back(tmpCombinedVector);
1156 24 : return HCCL_SUCCESS;
1157 24 : }
1158 :
1159 523 : HcclResult TopoInfoExtractor::GetCommPlaneRanks(std::vector<std::vector<std::vector<u32>>>& CommPlaneRanks)
1160 : {
1161 523 : CommPlaneRanks.resize(CommPlaneVector_.size());
1162 8891 : for (u32 level = 0; level < CommPlaneVector_.size(); level++) {
1163 8368 : u32 ringSize = CommPlaneVector_[level].size();
1164 8368 : CommPlaneRanks[level].resize(ringSize);
1165 14101 : for (u32 ringIndex = 0; ringIndex < ringSize; ringIndex++) {
1166 5733 : u32 rankSize = CommPlaneVector_[level][ringIndex].size();
1167 5733 : CommPlaneRanks[level][ringIndex].resize(rankSize);
1168 17280 : for (u32 rankIndex = 0; rankIndex < rankSize; rankIndex++) {
1169 11547 : u32 userRank = CommPlaneVector_[level][ringIndex][rankIndex].userRank;
1170 11547 : CommPlaneRanks[level][ringIndex][rankIndex] = userRank;
1171 11547 : HCCL_DEBUG("GetCommPlaneRanks CommPlaneRanks[%u][%u][%u]=%u", level, ringIndex, rankIndex, userRank);
1172 : }
1173 : }
1174 : }
1175 523 : return HCCL_SUCCESS;
1176 : }
1177 :
1178 1053 : void TopoInfoExtractor::GetIsBridgeVector(std::vector<bool>& isBridgeVector)
1179 : {
1180 1053 : isBridgeVector = isBridgeVector_;
1181 1053 : return;
1182 : }
1183 :
1184 523 : HcclResult TopoInfoExtractor::GetIsUsedRdmaMap(std::unordered_map<u32, bool>& isUsedRdmaMap)
1185 : {
1186 2151 : for (const RankInfo& dstRank : rankVector_) {
1187 1628 : bool isInterSuperPod = false;
1188 1628 : bool isInterServer = false;
1189 1628 : uint32_t isConnectedWithPcie = 0;
1190 1628 : if (rankData_.superPodId != dstRank.superPodId) { // 跨超节点场景
1191 51 : isInterSuperPod = true;
1192 1577 : } else if (rankData_.serverIdx != dstRank.serverIdx) { // 不跨超节点, 跨server场景
1193 254 : isInterServer = true;
1194 : } else { // 同server, PCIE互连场景
1195 1323 : HCCL_DEBUG("[TopoInfoExtractor]GetIsUsedRdmaMap for interServer");
1196 1323 : auto it = deviceLinkTypeMap_.find(dstRank.devicePhyId);
1197 1323 : CHK_PRT_RET(
1198 : it == deviceLinkTypeMap_.end(),
1199 : HCCL_ERROR("can't find devicePhyId[%d] in deviceLinkTypeMap_", dstRank.devicePhyId), HCCL_E_NOT_FOUND);
1200 1323 : isConnectedWithPcie = (it->second == LinkTypeInServer::PXI_TYPE) ? true : false;
1201 : }
1202 : // 使能RDMA的场景: 1.跨超节点 2.跨server且不使能HCCS 3.PCIE连接且使能RDMA开关
1203 1577 : bool isUsedRdma = (isInterSuperPod) || (isInterServer && !isUsedInterHccsMode_)
1204 3205 : || (isConnectedWithPcie && isUsedRdmaLevel0_);
1205 1628 : isUsedRdmaMap[dstRank.userRank] = isUsedRdma;
1206 1628 : HCCL_DEBUG(
1207 : "[GetIsUsedRdma]isUsedRdma[%u], isInterSuperPod[%u], isInterServer[%u], isUsedInterHccsMode_[%u], "
1208 : "isConnectedWithPcie[%u], isUsedRdmaLevel0_[%u], dstRank[%u]",
1209 : isUsedRdma, isInterSuperPod, isInterServer, isUsedInterHccsMode_, isConnectedWithPcie, isUsedRdmaLevel0_,
1210 : dstRank.userRank);
1211 : }
1212 523 : return HCCL_SUCCESS;
1213 : }
1214 :
1215 523 : HcclResult TopoInfoExtractor::GetRankVecInfo(std::vector<std::vector<std::vector<u32>>>& serverAndsuperPodToRank)
1216 : {
1217 523 : std::vector<std::vector<u32>> serverToRank;
1218 523 : std::vector<std::vector<u32>> superPodToRank;
1219 523 : serverToRank.clear();
1220 523 : superPodToRank.clear();
1221 523 : u32 firstIdx = 0;
1222 :
1223 523 : serverToRank.resize(serverToRank_.size());
1224 1217 : for (auto iterMap = serverToRank_.begin(); iterMap != serverToRank_.end(); iterMap++) {
1225 694 : serverToRank[firstIdx].resize((iterMap->second).size());
1226 694 : if (!(iterMap->second).empty()) {
1227 2271 : for (u32 i = 0; i < (iterMap->second).size(); i++) {
1228 1577 : serverToRank[firstIdx][i] = (iterMap->second)[i].userRank;
1229 : }
1230 : }
1231 694 : firstIdx++;
1232 : }
1233 :
1234 523 : u32 podFirstIdx = 0;
1235 523 : superPodToRank.resize(superPodToRank_.size());
1236 1087 : for (auto iterMap = superPodToRank_.begin(); iterMap != superPodToRank_.end(); iterMap++) {
1237 564 : if (!(iterMap->second).empty()) {
1238 564 : superPodToRank[podFirstIdx].resize((iterMap->second).size());
1239 2192 : for (u32 i = 0; i < (iterMap->second).size(); i++) {
1240 1628 : superPodToRank[podFirstIdx][i] = (iterMap->second)[i].userRank;
1241 1628 : HCCL_DEBUG("GetRankVecInfo superPodToRank[%u][%u]=%u", podFirstIdx, i, superPodToRank[podFirstIdx][i]);
1242 : }
1243 : }
1244 564 : podFirstIdx++;
1245 : }
1246 523 : serverAndsuperPodToRank.push_back(serverToRank);
1247 523 : serverAndsuperPodToRank.push_back(superPodToRank);
1248 523 : return HCCL_SUCCESS;
1249 523 : }
1250 525 : HcclResult TopoInfoExtractor::SetTopoInfoForCombineL1()
1251 : {
1252 525 : if (deviceType_ == DevType::DEV_TYPE_910_93) {
1253 : // 按照superPodIdx 划分得所有rank信息
1254 107 : for (auto iter = superPodToRank_.begin(); iter != superPodToRank_.end(); iter++) {
1255 74 : if (iter->first != rankData_.superPodIdx) {
1256 41 : continue; // 只在自己所在的超节点创建
1257 : }
1258 33 : CommPlaneVector_[COMM_COMBINE_L1].push_back(iter->second);
1259 33 : HCCL_DEBUG("[SetTopoInfoForARS]Superpod rankdSize[%u].", CommPlaneVector_[COMM_COMBINE_L1][0].size());
1260 : }
1261 : }
1262 525 : return HCCL_SUCCESS;
1263 : }
1264 :
1265 525 : HcclResult TopoInfoExtractor::SetTopoInfoForARS() // 针对ARS特性
1266 : {
1267 525 : if (deviceType_ == DevType::DEV_TYPE_910_93) {
1268 107 : for (auto iter = superPodToRank_.begin(); iter != superPodToRank_.end(); iter++) {
1269 74 : if (iter->first != rankData_.superPodIdx) {
1270 41 : continue; // 只在自己所在的超节点创建
1271 : }
1272 33 : CommPlaneVector_[COMM_ARS].push_back(iter->second);
1273 33 : HCCL_DEBUG("[SetTopoInfoForARS]Superpod rankdSize[%u].", CommPlaneVector_[COMM_ARS][0].size());
1274 : }
1275 : }
1276 525 : return HCCL_SUCCESS;
1277 : }
1278 :
1279 530 : void TopoInfoExtractor::GetCommPlaneVector(std::vector<std::vector<std::vector<RankInfo>>>& commPlaneVector)
1280 : {
1281 530 : commPlaneVector = CommPlaneVector_;
1282 530 : return;
1283 : }
1284 :
1285 526 : void TopoInfoExtractor::InitAHCConfig(std::map<HcclCMDType, std::vector<HcclAlgoType>>& algoConfig)
1286 : {
1287 55026 : for (u32 opType = 0; opType < static_cast<u32>(HcclCMDType::HCCL_CMD_MAX); opType++) {
1288 54502 : std::vector<HcclAlgoType> algoType = algoConfig[static_cast<HcclCMDType>(opType)];
1289 : isConfigAHC_
1290 54502 : = (algoType[HCCL_ALGO_LEVEL_1] == HcclAlgoType::HCCL_ALGO_TYPE_AHC
1291 54502 : || algoType[HCCL_ALGO_LEVEL_1] == HcclAlgoType::HCCL_ALGO_TYPE_AHC_BROKE);
1292 54502 : if (isConfigAHC_) {
1293 2 : HCCL_INFO("[InitAHCConfig] set AHC alg, opType[%u]", opType);
1294 2 : break;
1295 : }
1296 54502 : }
1297 :
1298 52526 : for (u32 opType = 0; opType < static_cast<u32>(HcclCMDType::HCCL_CMD_MAX);
1299 : opType++) { // 没配置算法的情况下默认会走AHC嘛？ 给测试用
1300 52026 : std::vector<HcclAlgoType> algoType = algoConfig[static_cast<HcclCMDType>(opType)];
1301 52026 : isConfigNULL_ = algoType[HCCL_ALGO_LEVEL_0] == HcclAlgoType::HCCL_ALGO_TYPE_NULL;
1302 52026 : if (isConfigNULL_) {
1303 26 : HCCL_INFO("[InitAHCConfig] set NULL alg, opType[%u]", opType);
1304 26 : break;
1305 : }
1306 52026 : }
1307 526 : return;
1308 : }
1309 :
1310 525 : HcclResult TopoInfoExtractor::SetAHCSubGroupsAndAlgOption()
1311 : {
1312 : // 解析subgroup
1313 525 : CHK_RET(SetTopoInfoForLevel1(true));
1314 :
1315 : // 分组切分
1316 525 : CHK_RET(AHCSubGroupInit(COMM_LEVEL1_AHC, CommPlaneSubGroupVector_));
1317 525 : if (deviceType_ != DevType::DEV_TYPE_910_93) {
1318 492 : CHK_RET(AHCSubGroupInit(COMM_COMBINE, CommPlaneSubGroupVector_));
1319 : } else {
1320 33 : CHK_RET(AHCSubGroupInit(COMM_COMBINE_ORDER, CommPlaneSubGroupVector_));
1321 : }
1322 :
1323 : // 算法配置初始化
1324 525 : CommAHCBaseInfo::InitConcAlgOption(ahcAlgOption_);
1325 525 : return HCCL_SUCCESS;
1326 : }
1327 :
1328 525 : void TopoInfoExtractor::AHCCommSubgroupInit()
1329 : {
1330 525 : if (deviceType_ != DevType::DEV_TYPE_910_93) {
1331 : // 用于910B AHC COMM_COMBINE 通信域分组场景测试
1332 492 : std::map<std::string, std::vector<u32>> serverIDGroup;
1333 492 : std::vector<std::string> serverIdsVector;
1334 1989 : for (u32 i = 0; i < rankVector_.size(); i++) {
1335 1497 : auto itServerID = serverIDGroup.find(rankVector_[i].serverId);
1336 1497 : if (itServerID != serverIDGroup.end()) {
1337 848 : itServerID->second.push_back(i);
1338 : } else {
1339 649 : serverIdsVector.push_back(rankVector_[i].serverId);
1340 649 : std::vector<u32> subGroup;
1341 649 : subGroup.push_back(i);
1342 649 : serverIDGroup.insert(std::make_pair(rankVector_[i].serverId, subGroup));
1343 649 : }
1344 : }
1345 492 : std::vector<std::vector<u32>> serverSubGroups;
1346 1141 : for (u32 i = 0; i < serverIdsVector.size(); i++) {
1347 649 : serverSubGroups.push_back(serverIDGroup[serverIdsVector[i]]);
1348 : }
1349 492 : CommPlaneSubGroupVector_[COMM_COMBINE].push_back(serverSubGroups);
1350 492 : } else {
1351 : // 用于910_93 AHC COMM_COMBINE_ORDER 通信域分组场景测试
1352 33 : std::vector<std::string> idsForIndexingVector;
1353 33 : std::map<std::string, std::vector<u32>> superPodIdGroup;
1354 178 : for (u32 i = 0; i < rankVector_.size(); i++) {
1355 145 : auto itSuperPodID = superPodIdGroup.find(rankVector_[i].superPodId);
1356 145 : if (itSuperPodID != superPodIdGroup.end()) {
1357 67 : itSuperPodID->second.push_back(i);
1358 : } else {
1359 78 : idsForIndexingVector.push_back(rankVector_[i].superPodId);
1360 78 : std::vector<u32> subGroup;
1361 78 : subGroup.push_back(i);
1362 78 : superPodIdGroup.insert(std::make_pair(rankVector_[i].superPodId, subGroup));
1363 78 : }
1364 : }
1365 33 : std::vector<std::vector<u32>> superPodSubGroups;
1366 111 : for (u32 i = 0; i < idsForIndexingVector.size(); i++) {
1367 78 : superPodSubGroups.push_back(superPodIdGroup[idsForIndexingVector[i]]);
1368 78 : HCCL_DEBUG("[SetTopoInfoForLevel0][AHC_DEBUG 910_93] superPodId[%s]", idsForIndexingVector[i].c_str());
1369 223 : for (u32 index = 0; index < superPodIdGroup[idsForIndexingVector[i]].size(); index++) {
1370 145 : HCCL_DEBUG(
1371 : "[SetTopoInfoForLevel0][AHC_DEBUG 910_93] groupIdx[%u]",
1372 : superPodIdGroup[idsForIndexingVector[i]][index]);
1373 : }
1374 : }
1375 33 : CommPlaneSubGroupVector_[COMM_COMBINE_ORDER].push_back(superPodSubGroups);
1376 33 : }
1377 525 : return;
1378 : }
1379 :
1380 523 : void TopoInfoExtractor::GetCommPlaneSubGroupVector(
1381 : std::vector<std::vector<std::vector<std::vector<u32>>>>& CommPlaneSubGroupVector)
1382 : {
1383 523 : CommPlaneSubGroupVector = CommPlaneSubGroupVector_;
1384 523 : return;
1385 : }
1386 :
1387 0 : void TopoInfoExtractor::SetAHCLevelAlgOption(AHCConcOpType ahcConcOpType, TemplateType templateType)
1388 : {
1389 0 : if (ahcAlgOption_.find(ahcConcOpType) == ahcAlgOption_.end()) {
1390 0 : ahcAlgOption_[ahcConcOpType] = templateType;
1391 : }
1392 0 : return;
1393 : }
1394 :
1395 523 : void TopoInfoExtractor::GetAHCAlgOption(std::map<AHCConcOpType, TemplateType>& ahcAlgOption)
1396 : {
1397 523 : ahcAlgOption = ahcAlgOption_;
1398 523 : return;
1399 : }
1400 :
1401 530 : void TopoInfoExtractor::GetRankData(RankInfo& rankData)
1402 : {
1403 530 : rankData = rankData_;
1404 530 : return;
1405 : }
1406 :
1407 530 : void TopoInfoExtractor::GetServerToRank(std::map<u32, std::vector<RankInfo>>& serverToRank)
1408 : {
1409 530 : serverToRank = serverToRank_;
1410 530 : return;
1411 : }
1412 :
1413 530 : void TopoInfoExtractor::GetSuperPodToRank(std::map<u32, std::vector<RankInfo>>& superPodToRank)
1414 : {
1415 530 : superPodToRank = superPodToRank_;
1416 530 : return;
1417 : }
1418 :
1419 530 : void TopoInfoExtractor::GetDeviceLinkTypeMap(std::map<s32, LinkTypeInServer>& deviceLinkTypeMap)
1420 : {
1421 530 : deviceLinkTypeMap = deviceLinkTypeMap_;
1422 530 : return;
1423 : }
1424 :
1425 1050 : HcclResult TopoInfoExtractor::AHCSubGroupInit(
1426 : CommPlane algLevel, std::vector<std::vector<std::vector<std::vector<u32>>>>& CommPlaneSubGroupVector)
1427 : {
1428 1050 : std::vector<std::vector<std::vector<u32>>>& globalSubGroups = CommPlaneSubGroupVector[algLevel];
1429 :
1430 : // globalSubGroups 参数检查
1431 1050 : if ((globalSubGroups.size() == 0) || (globalSubGroups[0].size() == 0)) {
1432 359 : return HCCL_SUCCESS;
1433 : }
1434 :
1435 691 : CHK_RET(CommAHCBaseInfo::CheckGlobalGroups(globalSubGroups));
1436 :
1437 691 : u32 minSubGroupSize = globalSubGroups[0][0].size();
1438 691 : u32 maxSubGroupSize = globalSubGroups[0][0].size();
1439 691 : u32 subGroupSizeGCD = globalSubGroups[0][0].size();
1440 938 : for (u32 i = 1; i < globalSubGroups[0].size(); ++i) {
1441 247 : subGroupSizeGCD = std::__gcd(subGroupSizeGCD, static_cast<u32>(globalSubGroups[0][i].size()));
1442 247 : if (globalSubGroups[0][i].size() < minSubGroupSize) {
1443 0 : minSubGroupSize = globalSubGroups[0][i].size();
1444 : }
1445 247 : if (globalSubGroups[0][i].size() > maxSubGroupSize) {
1446 16 : maxSubGroupSize = globalSubGroups[0][i].size();
1447 : }
1448 : }
1449 :
1450 691 : u32 splitDivisor = minSubGroupSize;
1451 :
1452 691 : HCCL_DEBUG(
1453 : "[TopoInfoExtractor][AHCSubGroupInit] begin groupSplit algLevel = %u minSubGroupSize = %u maxSubGroupSize = %u "
1454 : "SubGroupSizeGCD = %u "
1455 : "splitDivisor = %u",
1456 : algLevel, minSubGroupSize, maxSubGroupSize, subGroupSizeGCD, splitDivisor);
1457 :
1458 : // 切分分组逻辑, 满足整数倍切分，满足最大偏差大于等于最小分组数时切分
1459 691 : if ((subGroupSizeGCD == minSubGroupSize) || (maxSubGroupSize / splitDivisor > 1)) {
1460 : // 设置为 BROKE 类型
1461 691 : maxSubGroupSize = splitDivisor;
1462 : // 将所有的分组切分成 splitDivisor 粒度的 subGroup
1463 1677 : for (u32 i = 0; i < globalSubGroups[0].size(); ++i) {
1464 986 : if (globalSubGroups[0][i].size() / splitDivisor <= 1) {
1465 970 : continue;
1466 : }
1467 16 : std::vector<u32> originGroup = globalSubGroups[0][i];
1468 16 : globalSubGroups[0].erase(globalSubGroups[0].begin() + i);
1469 16 : u32 splitGroupsNum = (originGroup.size() / splitDivisor);
1470 48 : for (u32 j = 0; j < splitGroupsNum - 1; ++j) {
1471 64 : globalSubGroups[0].insert(
1472 32 : globalSubGroups[0].begin() + i + j,
1473 32 : std::vector<u32>(
1474 64 : originGroup.begin() + j * splitDivisor, originGroup.begin() + (j + 1) * splitDivisor));
1475 : }
1476 32 : globalSubGroups[0].insert(
1477 16 : globalSubGroups[0].begin() + i + splitGroupsNum - 1,
1478 16 : std::vector<u32>(
1479 0 : originGroup.begin() + (splitGroupsNum - 1) * splitDivisor,
1480 32 : originGroup.begin() + originGroup.size()));
1481 32 : maxSubGroupSize = std::max(
1482 16 : maxSubGroupSize, static_cast<u32>(originGroup.size()) - ((splitGroupsNum - 1) * splitDivisor));
1483 16 : i--;
1484 16 : }
1485 : }
1486 :
1487 691 : HCCL_DEBUG(
1488 : "[TopoInfoExtractor][AHCSubGroupInit] after groupSplit algLevel = %u minSubGroupSize = %u maxSubGroupSize = %u "
1489 : "SubGroupSizeGCD = %u "
1490 : "splitDivisor = %u",
1491 : algLevel, minSubGroupSize, maxSubGroupSize, subGroupSizeGCD, splitDivisor);
1492 :
1493 691 : CHK_RET(CommAHCBaseInfo::CheckGlobalGroups(globalSubGroups));
1494 :
1495 691 : return HCCL_SUCCESS;
1496 : }
1497 :
1498 5752 : bool Ascending(const RankInfo& first, const RankInfo& second)
1499 : {
1500 5752 : if (first.serverIdx != second.serverIdx) {
1501 453 : return first.serverIdx < second.serverIdx;
1502 : } else {
1503 5299 : return first.userRank < second.userRank;
1504 : }
1505 : }
1506 :
1507 552 : bool CompareWithUserRankAscend(const RankInfo& left, const RankInfo& right) { return left.userRank < right.userRank; }
1508 :
1509 : // 适配ROH平面网段隔离，奇数rank互通，偶数rank互通，奇偶不通
1510 1 : bool CheckSdmaWithRohTopo(const std::vector<u32>& nicList, std::vector<u32>& topoList)
1511 : {
1512 1 : std::vector<u32> tmpNicList(nicList);
1513 1 : SearchPath searchPath;
1514 1 : std::sort(tmpNicList.begin(), tmpNicList.end());
1515 1 : topoList = searchPath.Search(tmpNicList);
1516 1 : if (topoList.empty()) {
1517 0 : return false;
1518 : }
1519 1 : return true;
1520 1 : }
1521 :
1522 525 : std::vector<std::vector<u32>> GetRingsOrderByTopoType(u32 ranksSize, TopoType topoType, std::vector<u32>& nicList)
1523 : {
1524 525 : std::vector<std::vector<u32>> multiRingOrder;
1525 525 : if (topoType == TopoType::TOPO_TYPE_8P_RING) { // 4 ring 场景
1526 : // 每个环的排序是按照设备物理ID进行的
1527 48 : std::vector<u32> tmpLevel00 = {0, 1, 2, 6, 5, 4, 7, 3}; // 环0
1528 48 : std::vector<u32> tmpLevel01 = {0, 3, 7, 4, 5, 6, 2, 1}; // 环1
1529 48 : std::vector<u32> tmpLevel02 = {0, 2, 3, 1, 5, 7, 6, 4}; // 环2
1530 48 : std::vector<u32> tmpLevel03 = {0, 4, 6, 7, 5, 1, 3, 2}; // 环3
1531 :
1532 : // 填充8pring 多环的comm level0 四个环的顺序
1533 24 : multiRingOrder.push_back(tmpLevel00);
1534 24 : multiRingOrder.push_back(tmpLevel01);
1535 24 : multiRingOrder.push_back(tmpLevel02);
1536 24 : multiRingOrder.push_back(tmpLevel03);
1537 525 : } else if (topoType == TopoType::TOPO_TYPE_NP_DOUBLE_RING) { // 2 ring 场景
1538 7 : std::vector<u32> tmpLevel00; // 环0
1539 7 : std::vector<u32> tmpLevel01; // 环1
1540 7 : tmpLevel00 = nicList; // { 0, 1, 2, 3, 4, 5, 6, 7 };
1541 7 : tmpLevel01.reserve(ranksSize);
1542 7 : tmpLevel01.push_back(nicList[0]);
1543 7 : tmpLevel01.insert(tmpLevel01.end(), tmpLevel00.rbegin(), tmpLevel00.rend() - 1);
1544 7 : HCCL_INFO("[GetRingsOrderByTopoType] TopoType:TOPO_TYPE_NP_DOUBLE_RING");
1545 : // 填充 double ring 两环的comm level0的顺序
1546 7 : multiRingOrder.push_back(tmpLevel00);
1547 7 : multiRingOrder.push_back(tmpLevel01);
1548 7 : } else { // 1 ring 场景
1549 494 : std::vector<u32> tmpLevel00 = nicList; // 环0
1550 :
1551 : // 填充 single ring 单环的comm level0的顺序
1552 494 : multiRingOrder.push_back(tmpLevel00);
1553 494 : }
1554 : // 打印多个环
1555 525 : if (UNLIKELY(HcclCheckLogLevel(DLOG_DEBUG))) {
1556 1129 : for (size_t i = 0; i < multiRingOrder.size(); i++) {
1557 604 : auto ring = multiRingOrder[i];
1558 604 : std::ostringstream stringRepresentation;
1559 2533 : for (std::vector<uint32_t>::iterator it = ring.begin(); it != ring.end(); it++) {
1560 1929 : stringRepresentation << *it << " ";
1561 : }
1562 604 : std::string ringString = stringRepresentation.str();
1563 604 : const char* charRing = ringString.c_str();
1564 604 : HCCL_DEBUG("[GetRingsOrderByTopoType] The No.%zu ring: %s", i, charRing);
1565 604 : }
1566 : }
1567 525 : return multiRingOrder;
1568 0 : }
1569 :
1570 1 : std::vector<std::vector<u32>> GetRingsOrderForAnyPath(u32 ranksSize, TopoType topoType, std::vector<u32>& nicList)
1571 : {
1572 1 : std::vector<std::vector<u32>> multiRingOrder;
1573 1 : if (topoType == TopoType::TOPO_TYPE_NP_DOUBLE_RING) { // 2 ring 场景
1574 1 : std::vector<u32> tmpLevel00; // 环0
1575 1 : std::vector<u32> tmpLevel01; // 环1
1576 1 : std::vector<u32> rohLevel0;
1577 1 : if (CheckSdmaWithRohTopo(nicList, rohLevel0)) {
1578 1 : tmpLevel00 = rohLevel0; // 环0, 8卡 { 0, 1, 3, 2, 4, 5, 7, 6 };
1579 1 : tmpLevel01.reserve(ranksSize); // 环1, 8卡 { 0, 6, 7, 5, 4, 2, 3, 1 };
1580 1 : tmpLevel01.push_back(rohLevel0[0]);
1581 1 : tmpLevel01.insert(tmpLevel01.end(), rohLevel0.rbegin(), rohLevel0.rend() - 1);
1582 : } else {
1583 0 : tmpLevel00 = nicList; // { 0, 1, 2, 3, 4, 5, 6, 7 };
1584 0 : tmpLevel01.reserve(ranksSize);
1585 0 : tmpLevel01.push_back(nicList[0]);
1586 0 : tmpLevel01.insert(tmpLevel01.end(), tmpLevel00.rbegin(), tmpLevel00.rend() - 1);
1587 : }
1588 : // 填充 double ring 两环的comm level0的顺序
1589 1 : multiRingOrder.push_back(tmpLevel00);
1590 1 : multiRingOrder.push_back(tmpLevel01);
1591 1 : } else { // 1 ring 场景
1592 0 : std::vector<u32> tmpLevel00 = nicList; // 环0
1593 :
1594 : // 填充 single ring 单环的comm level0的顺序
1595 0 : multiRingOrder.push_back(tmpLevel00);
1596 0 : }
1597 : // 打印多个环
1598 1 : HCCL_DEBUG("[GetRingsOrderForAnyPath]print rings:");
1599 3 : for (size_t i = 0; i < multiRingOrder.size(); i++) {
1600 2 : auto ring = multiRingOrder[i];
1601 2 : std::ostringstream stringRepresentation;
1602 18 : for (std::vector<uint32_t>::iterator it = ring.begin(); it != ring.end(); it++) {
1603 16 : stringRepresentation << *it << " ";
1604 : }
1605 2 : std::string ringString = stringRepresentation.str();
1606 2 : const char* charRing = ringString.c_str();
1607 2 : HCCL_INFO("[GetRingsOrderByRdmaSdmaConcurrent] The No.%zu ring: %s.", i, charRing);
1608 2 : }
1609 1 : return multiRingOrder;
1610 0 : }
1611 :
1612 : } // namespace hccl
|
