Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "topoinfo_parse.h"
12 : #include <string>
13 : #include <unordered_set>
14 : #include <algorithm>
15 : // ltm指定config路径
16 : #include "common/src/config.h"
17 :
18 : using namespace std;
19 :
20 : struct HcclAiServerValid4PRanksVectorHashFuc {
21 366 : std::size_t operator()(const std::vector<s32> key) const
22 : {
23 366 : size_t ret = 0;
24 1830 : for (auto it : key) {
25 1464 : ret ^= it;
26 : }
27 366 : return ret;
28 : }
29 : };
30 :
31 : // aiserver内连接信息rank合法选择
32 : const std::unordered_set<std::vector<s32>, HcclAiServerValid4PRanksVectorHashFuc> HCCL_AISERVER_VAILD_4P_RANKS = {
33 : {0, 1, 4, 5},
34 : {0, 2, 4, 6},
35 : {0, 3, 4, 7},
36 : {1, 2, 5, 6},
37 : {1, 3, 5, 7},
38 : {2, 3, 6, 7},
39 : {0, 1, 2, 3},
40 : {4, 5, 6, 7}
41 : };
42 :
43 : namespace hccl {
44 529 : TopoInfoParse::TopoInfoParse()
45 : {
46 529 : }
47 :
48 527 : TopoInfoParse::~TopoInfoParse()
49 : {
50 527 : }
51 :
52 496 : HcclResult TopoInfoParse::Init(const RankTable_t &rankTable, const std::string &serverId, const u32 deviceNumPerServer)
53 : {
54 496 : CHK_PRT_RET(deviceNumPerServer == 0, HCCL_ERROR("cur device num per server is 0"), HCCL_E_PARA);
55 496 : deviceNum_ = rankTable.deviceNum;
56 496 : serverNum_ = rankTable.serverNum;
57 496 : superPodNum_ = rankTable.superPodNum;
58 496 : serverId_ = serverId;
59 496 : nicDeploy_ = rankTable.nicDeploy;
60 496 : deviceNumPerServer_ = deviceNumPerServer;
61 496 : multiServerDiffDeviceNumMode_ = (serverNum_ * deviceNumPerServer_) == deviceNum_ ? false : true;
62 496 : CHK_RET(hrtGetDeviceType(deviceType_));
63 :
64 496 : DevType curDevType = rankTable.rankList.begin()->deviceInfo.deviceType;
65 1987 : for (auto rankInfo : rankTable.rankList) {
66 1491 : RankInfo curRankInfo;
67 1490 : curRankInfo.devicePhyId = rankInfo.deviceInfo.devicePhyId;
68 1490 : curRankInfo.deviceType = rankInfo.deviceInfo.deviceType;
69 1490 : curRankInfo.serverId = rankInfo.serverId;
70 1491 : curRankInfo.userRank = rankInfo.rankId;
71 1491 : curRankInfo.nicIp = rankInfo.deviceInfo.deviceIp;
72 1491 : curRankInfo.superDeviceId = rankInfo.superDeviceId;
73 1491 : curRankInfo.superPodId = rankInfo.superPodId;
74 1491 : rankList_.push_back(curRankInfo);
75 :
76 1490 : if (curDevType != rankInfo.deviceInfo.deviceType) {
77 0 : isDiffDeviceType_ = true;
78 : }
79 1490 : }
80 496 : return HCCL_SUCCESS;
81 : }
82 :
83 33 : HcclResult TopoInfoParse::Init(const std::vector<RankInfo> &rankList, const std::string &serverId,
84 : const u32 deviceNumPerServer)
85 : {
86 33 : CHK_PRT_RET(deviceNumPerServer == 0, HCCL_ERROR("cur device num per server is 0"), HCCL_E_PARA);
87 33 : rankList_ = rankList;
88 33 : serverId_ = serverId;
89 33 : deviceNumPerServer_ = deviceNumPerServer;
90 33 : CHK_RET(GetDevNum(rankList, deviceNum_));
91 33 : CHK_RET(GetServerNum(rankList, serverNum_));
92 33 : CHK_RET(GetSuperPodNum(rankList, superPodNum_));
93 33 : multiServerDiffDeviceNumMode_ = (serverNum_ * deviceNumPerServer_) == deviceNum_ ? false : true;
94 33 : CHK_RET(hrtGetDeviceType(deviceType_));
95 :
96 33 : DevType curDevType = rankList.begin()->deviceType;
97 190 : for (auto rankInfo : rankList) {
98 157 : if (curDevType != rankInfo.deviceType) {
99 0 : isDiffDeviceType_ = true;
100 : }
101 157 : }
102 33 : return HCCL_SUCCESS;
103 : }
104 :
105 528 : HcclResult TopoInfoParse::GetServerInnerLinkInfo(std::unordered_map<u32, u32> &pairLinkCounter,
106 : std::unordered_map<u32, std::unordered_map<int, std::vector<int>>> &pairLinkInfo)
107 : {
108 528 : std::vector<RankInfo> serverInnerInfo;
109 529 : CHK_RET(TransformRankInfoByServerId(serverInnerInfo));
110 :
111 528 : CHK_PRT_RET(serverInnerInfo.size() == 0,
112 : HCCL_ERROR("[Get][ServerInnerLinkInfo]server info input is empty, "
113 : "serverid[%s]",
114 : serverId_.c_str()),
115 : HCCL_E_PARA);
116 529 : pairLinkInfo.clear();
117 529 : pairLinkCounter[static_cast<u32>(LinkTypeInServer::HCCS_TYPE)] = 0;
118 528 : pairLinkCounter[static_cast<u32>(LinkTypeInServer::PXI_TYPE)] = 0;
119 528 : pairLinkCounter[static_cast<u32>(LinkTypeInServer::SIO_TYPE)] = 0;
120 529 : pairLinkCounter[static_cast<u32>(LinkTypeInServer::HCCS_SW_TYPE)] = 0;
121 1863 : for (auto &it_local : serverInnerInfo) {
122 7160 : for (auto &it_dest : serverInnerInfo) {
123 5820 : if (it_local.devicePhyId == it_dest.devicePhyId || it_local.devicePhyId == HOST_DEVICE_ID ||
124 4487 : it_dest.devicePhyId == HOST_DEVICE_ID) {
125 1333 : continue;
126 : }
127 : LinkTypeInServer linkType;
128 4487 : CHK_RET(hrtGetPairDeviceLinkType(it_local.devicePhyId, it_dest.devicePhyId, linkType));
129 4490 : pairLinkInfo[static_cast<u32>(linkType)][it_local.devicePhyId].push_back(it_dest.devicePhyId);
130 4493 : pairLinkCounter[static_cast<u32>(linkType)]++;
131 : }
132 : }
133 771 : for (auto it : pairLinkInfo) {
134 242 : HCCL_DEBUG("pair link information linkType[%u], size[%llu]", it.first, it.second.size());
135 242 : }
136 2644 : for (auto it : pairLinkCounter) {
137 2112 : HCCL_DEBUG("pair link counter information linkType[%u], size[%llu]", it.first, it.second);
138 : }
139 529 : return HCCL_SUCCESS;
140 529 : }
141 :
142 529 : HcclResult TopoInfoParse::TransformRankInfoByServerId(std::vector<hccl::RankInfo> &serverInnerInfo)
143 : {
144 : // 按server重新组织rank信息，便于后续校验及信息填写
145 2176 : for (auto tmpRankInfo : rankList_) {
146 1647 : if (tmpRankInfo.serverId == serverId_) {
147 1338 : serverInnerInfo.push_back(tmpRankInfo);
148 : }
149 1646 : }
150 : // 按设备Id从小到大的顺序排序
151 529 : std::sort(serverInnerInfo.begin(), serverInnerInfo.end(),
152 1620 : [](const RankInfo &left, const RankInfo &right) { return left.devicePhyId < right.devicePhyId; });
153 529 : return HCCL_SUCCESS;
154 : }
155 :
156 : // nicIdx不只是校验，还有修改
157 495 : HcclResult TopoInfoParse::ParseAndCheck(std::vector<u32> &nicIdx)
158 : {
159 495 : CHK_RET(CheckInterServerDeviceId());
160 495 : CHK_RET(CheckRankTableNicInfo(nicIdx));
161 495 : CHK_RET(CheckServerInnerRankInfo());
162 494 : return HCCL_SUCCESS;
163 : }
164 :
165 20 : HcclResult TopoInfoParse::Check()
166 : {
167 20 : CHK_RET(CheckInterServerDeviceId());
168 20 : CHK_RET(CheckServerInnerRankInfo());
169 20 : return HCCL_SUCCESS;
170 : }
171 :
172 : // server间device选取是否对称校验
173 515 : HcclResult TopoInfoParse::CheckInterServerDeviceId()
174 : {
175 515 : if (serverNum_ == 0) {
176 0 : HCCL_ERROR("[Check][DeviceId]errNo[0x%016llx] server num is zero", HCOM_ERROR_CODE(HCCL_E_PARA));
177 0 : return HCCL_E_PARA;
178 : }
179 515 : std::map<std::string, std::set<s32>> serverDeviceMapList;
180 2107 : for (auto it = rankList_.begin(); it != rankList_.end(); it++) {
181 1590 : std::string tmpServerId = it->serverId;
182 1592 : auto search = serverDeviceMapList.find(tmpServerId);
183 1588 : if (search != serverDeviceMapList.end()) {
184 890 : auto rs = serverDeviceMapList[tmpServerId].insert(it->devicePhyId);
185 887 : if (it->devicePhyId == HOST_DEVICE_ID) {
186 0 : continue;
187 : }
188 887 : if (!rs.second) {
189 0 : RPT_INPUT_ERR(true, "EI0014", std::vector<std::string>({ "value", "variable" ,"expect" }),
190 : std::vector<std::string>({ std::to_string(it->devicePhyId),
191 : " \"Device Id of server Id " + it->serverId + " \" ", "is unique" }));
192 0 : HCCL_ERROR("[%s][%s]errNo[0x%016llx] check ranklist[%u], device id repeat for one server",
193 : LOG_KEYWORDS_INIT_GROUP.c_str(), LOG_KEYWORDS_RANKTABLE_CHECK.c_str(), HCOM_ERROR_CODE(HCCL_E_PARA), it->userRank);
194 0 : return HCCL_E_PARA;
195 : }
196 : } else {
197 701 : std::set<s32> deviceSet;
198 698 : deviceSet.insert(it->devicePhyId);
199 699 : serverDeviceMapList.insert(std::pair<std::string, std::set<s32>>(tmpServerId, deviceSet));
200 698 : }
201 1588 : }
202 516 : if (serverDeviceMapList.size() == 0) {
203 0 : HCCL_ERROR("[Check][DeviceId]errNo[0x%016llx] for all ranklist, server num is zero",
204 : HCOM_ERROR_CODE(HCCL_E_PARA));
205 0 : return HCCL_E_PARA;
206 : }
207 516 : return HCCL_SUCCESS;
208 516 : }
209 :
210 495 : HcclResult TopoInfoParse::CheckAndAssignNicInfo(std::vector<u32> &nicIdx)
211 : {
212 495 : if (isDiffDeviceType_) {
213 : // 混合组网
214 0 : return HCCL_SUCCESS;
215 495 : } else if (serverNum_ == 1 || superPodNum_ == 1) {
216 395 : nicIdx.clear();
217 1460 : for (u32 index = 0; index < rankList_.size(); index++) {
218 1064 : if (serverId_ == rankList_[index].serverId && (rankList_[index].devicePhyId != HOST_DEVICE_ID)) {
219 1064 : nicIdx.push_back(static_cast<u32>(rankList_[index].devicePhyId));
220 : }
221 : }
222 394 : std::sort(nicIdx.begin(), nicIdx.end());
223 496 : } else if (deviceNumPerServer_ == HCCL_AISERVER_DEVICE_NUM && !multiServerDiffDeviceNumMode_) {
224 0 : CHK_PRT_RET(nicIdx.size() == 0, HCCL_ERROR("[CheckAndAssign][NicInfo]nic idx size is 0"), HCCL_E_PARA);
225 0 : CHK_PRT_RET(deviceNum_ == 0, HCCL_ERROR("[CheckAndAssign][NicInfo]device num is 0"), HCCL_E_PARA);
226 :
227 0 : bool bRet = deviceNum_ % nicIdx.size() != 0; // 在n台server中，校验网口总数是否为1n/2n/4n/8n
228 0 : CHK_PRT_RET(bRet, HCCL_ERROR("[CheckAndAssign][NicInfo]nic total num[%zu] error", nicIdx.size()), HCCL_E_PARA);
229 :
230 0 : if (nicIdx.size() != deviceNumPerServer_) {
231 : // 网口裁剪
232 : // 按server重新组织rank信息
233 0 : std::map<std::string, std::vector<u32> > severNicsMap;
234 0 : for (size_t index = 0; index < rankList_.size(); ++index) {
235 0 : std::string serverId = rankList_[index].serverId;
236 :
237 0 : if (rankList_[index].nicIp.size() == 0 || rankList_[index].nicIp[0].IsInvalid()) {
238 0 : continue;
239 : }
240 : // 以serverID为索引，将server下的ranks放入vector
241 0 : auto itr = severNicsMap.find(serverId);
242 0 : if (itr != severNicsMap.end()) {
243 0 : itr->second.push_back(rankList_[index].devicePhyId);
244 : } else {
245 0 : std::vector<u32> nicList;
246 0 : nicList.push_back(rankList_[index].devicePhyId);
247 0 : std::pair<std::string, std::vector<u32>> nicInfoPair(serverId, nicList);
248 0 : severNicsMap.insert(nicInfoPair);
249 0 : }
250 0 : }
251 :
252 : // 每个server下的nicList按设备Id从小到大的顺序排序
253 0 : for (auto &iter : severNicsMap) {
254 0 : std::sort(iter.second.begin(), iter.second.end());
255 0 : if (nicIdx != iter.second) {
256 0 : HCCL_ERROR("nic list should be the same between servers");
257 0 : return HCCL_E_PARA;
258 : }
259 : }
260 0 : } else {
261 : // 网口满配
262 0 : return HCCL_SUCCESS;
263 : }
264 : }
265 496 : return HCCL_SUCCESS;
266 : }
267 :
268 : // nicIdx做填充，nicIdx也是deviceId，deviceId做过的校验这里不再重复
269 494 : HcclResult TopoInfoParse::CheckRankTableNicInfo(std::vector<u32> &nicIdx)
270 : {
271 : // 在8P均使用的情况下校验nic的选择信息是否正确
272 494 : if (nicDeploy_ == NICDeployment::NIC_DEPLOYMENT_DEVICE) {
273 495 : CHK_RET(CheckAndAssignNicInfo(nicIdx));
274 0 : } else if (nicDeploy_ == NICDeployment::NIC_DEPLOYMENT_HOST &&
275 0 : serverNum_ == deviceNum_ / HCCL_AISERVER_DEVICE_NUM) {
276 0 : nicIdx.assign({ 0, 1, 2, 3, 4, 5, 6, 7 }); // 如果每个server8个rank且为host nic，则网口为满配
277 : }
278 495 : return HCCL_SUCCESS;
279 : }
280 :
281 : // 校验server内4p场景下deivce选取是否合法，2p与标卡场景重合
282 513 : HcclResult TopoInfoParse::CheckServerInnerRankInfo()
283 : {
284 : // 校验server内device选取
285 513 : std::vector<s32> serverInnerDeviceInfo;
286 2102 : for (u32 index = 0; index < rankList_.size(); index++) {
287 1586 : if (serverId_ == rankList_[index].serverId && rankList_[index].devicePhyId != HOST_DEVICE_ID) {
288 : /* 同一server的标识IP 是一样的，所以可以以此推算出平均dev个数 */
289 1315 : serverInnerDeviceInfo.push_back(rankList_[index].devicePhyId);
290 : }
291 : }
292 515 : std::sort(serverInnerDeviceInfo.begin(), serverInnerDeviceInfo.end());
293 :
294 515 : if (deviceType_ == DevType::DEV_TYPE_910) {
295 378 : if (deviceNumPerServer_ != HCCL_DEVICE_NUM_FOUR) {
296 356 : return HCCL_SUCCESS;
297 : }
298 22 : std::string selectedDevice = "selected devices:";
299 110 : for (auto devicePhyId : serverInnerDeviceInfo) {
300 88 : selectedDevice += std::to_string(devicePhyId);
301 88 : selectedDevice += " ";
302 : }
303 22 : if (HCCL_AISERVER_VAILD_4P_RANKS.find(serverInnerDeviceInfo) == HCCL_AISERVER_VAILD_4P_RANKS.end()) {
304 0 : std::string errormessage = "Value " + selectedDevice + " for rankTable "\
305 0 : "variable \"Device Id of server Id " + serverId_ + " \" is invalid, expected value is unique.";
306 0 : errormessage += selectedDevice;
307 0 : RPT_INPUT_ERR(true, "EI0014", std::vector<std::string>({ "value", "variable" ,"expect" }),
308 : std::vector<std::string>({ selectedDevice, " \"Device Id of server Id " + serverId_ + " \" ", "is unique" }));
309 0 : HCCL_ERROR("[%s][%s] %s", LOG_KEYWORDS_INIT_GROUP.c_str(), LOG_KEYWORDS_RANKTABLE_CHECK.c_str(), errormessage.c_str());
310 0 : return HCCL_E_PARA;
311 0 : }
312 22 : HCCL_DEBUG("%s", selectedDevice.c_str());
313 22 : }
314 :
315 159 : return HCCL_SUCCESS;
316 515 : }
317 :
318 527 : HcclResult TopoInfoParse::IsAllRankSamePlane(bool &isAllRankSamePlane)
319 : {
320 : // 只有1个rank，不考虑
321 527 : if (rankList_.size() == 1) {
322 205 : isAllRankSamePlane = true;
323 205 : return HCCL_SUCCESS;
324 : }
325 :
326 323 : auto isSameDevId = [&]()-> bool {
327 323 : s32 maxModuleDevNum = deviceType_ == DevType::DEV_TYPE_910B ? HCCL_DEVICE_NUM_EIGHT : MAX_MODULE_DEVICE_NUM;
328 482 : for (size_t index = 0; index < rankList_.size() - 1; ++index) {
329 417 : if (rankList_[index].devicePhyId % maxModuleDevNum != rankList_[index + 1].devicePhyId % maxModuleDevNum) {
330 258 : return false;
331 : }
332 : }
333 66 : return true;
334 323 : };
335 :
336 323 : isAllRankSamePlane = isSameDevId();
337 324 : HCCL_DEBUG("[TopoInfoParse]curr comm isAllRankSamePlane[%d]", isAllRankSamePlane);
338 324 : return HCCL_SUCCESS;
339 : }
340 :
341 527 : HcclResult TopoInfoParse::IsSingleMeshAggregation(bool &isSingleMeshAggregation)
342 : {
343 527 : if (deviceNumPerServer_ == deviceNum_) {
344 : // rank间都是hccs链接，则表明在同一个mesh cube中
345 423 : CHK_RET(IsAllRankConnectedWithHCCS(isSingleMeshAggregation));
346 : } else {
347 104 : isSingleMeshAggregation = false;
348 : }
349 527 : HCCL_DEBUG("[TopoInfoParse]curr comm isSingleMeshAggregation[%d]", isSingleMeshAggregation);
350 529 : return HCCL_SUCCESS;
351 : }
352 :
353 421 : HcclResult TopoInfoParse::IsAllRankConnectedWithHCCS(bool &isAllRankConnectedWithHCCS)
354 : {
355 1603 : for (u32 i = 0; i < rankList_.size(); i++) {
356 3339 : for (u32 j = i + 1; j < rankList_.size(); j++) {
357 2154 : LinkTypeInServer linkType = LinkTypeInServer::RESERVED_LINK_TYPE;
358 2154 : if (rankList_[i].devicePhyId != HOST_DEVICE_ID && rankList_[j].devicePhyId != HOST_DEVICE_ID) {
359 2155 : CHK_RET(hrtGetPairDeviceLinkType(rankList_[i].devicePhyId, rankList_[j].devicePhyId, linkType));
360 : }
361 2157 : if (linkType == LinkTypeInServer::PXI_TYPE) {
362 0 : isAllRankConnectedWithHCCS = false;
363 0 : return HCCL_SUCCESS;
364 : }
365 : }
366 : }
367 422 : isAllRankConnectedWithHCCS = true;
368 422 : return HCCL_SUCCESS;
369 : }
370 :
371 0 : HcclResult TopoInfoParse::GetDeviceNumInPerMeshAggregation(u32 devicePhyId, u32 &perAggregationNum)
372 : {
373 : // 每个rank本身加上和他通过hccs互联的rank表示当前server Aggregation中的rank数量
374 0 : perAggregationNum = 1;
375 0 : for (u32 i = 0; i < rankList_.size(); i++) {
376 0 : if (serverId_ == rankList_[i].serverId && rankList_[i].devicePhyId != static_cast<s32>(devicePhyId)) {
377 : LinkTypeInServer linkType;
378 0 : CHK_RET(hrtGetPairDeviceLinkType(devicePhyId, rankList_[i].devicePhyId, linkType));
379 0 : if (linkType == LinkTypeInServer::HCCS_TYPE) {
380 0 : perAggregationNum++;
381 : }
382 : }
383 : }
384 0 : HCCL_DEBUG("[TopoInfoParse][GetDeviceNumInPerMeshAggregation]serverId[%s] devicePhyId[%u] perAggregationNum[%u]",
385 : serverId_.c_str(), devicePhyId, perAggregationNum);
386 :
387 0 : return HCCL_SUCCESS;
388 : }
389 : }
|
