Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "topoinfo_parse.h"
12 : #include <string>
13 : #include <unordered_set>
14 : #include <algorithm>
15 : // ltm指定config路径
16 : #include "common/src/config.h"
17 :
18 : using namespace std;
19 :
20 : struct HcclAiServerValid4PRanksVectorHashFuc {
21 398 : std::size_t operator()(const std::vector<s32> key) const
22 : {
23 398 : size_t ret = 0;
24 1990 : for (auto it : key) {
25 1592 : ret ^= it;
26 : }
27 398 : return ret;
28 : }
29 : };
30 :
31 : // aiserver内连接信息rank合法选择
32 : const std::unordered_set<std::vector<s32>, HcclAiServerValid4PRanksVectorHashFuc> HCCL_AISERVER_VAILD_4P_RANKS
33 : = {{0, 1, 4, 5}, {0, 2, 4, 6}, {0, 3, 4, 7}, {1, 2, 5, 6}, {1, 3, 5, 7}, {2, 3, 6, 7}, {0, 1, 2, 3}, {4, 5, 6, 7}};
34 :
35 : namespace hccl {
36 532 : TopoInfoParse::TopoInfoParse() {}
37 :
38 532 : TopoInfoParse::~TopoInfoParse() {}
39 :
40 496 : HcclResult TopoInfoParse::Init(const RankTable_t& rankTable, const std::string& serverId, const u32 deviceNumPerServer)
41 : {
42 496 : CHK_PRT_RET(deviceNumPerServer == 0, HCCL_ERROR("cur device num per server is 0"), HCCL_E_PARA);
43 496 : deviceNum_ = rankTable.deviceNum;
44 496 : serverNum_ = rankTable.serverNum;
45 496 : superPodNum_ = rankTable.superPodNum;
46 496 : serverId_ = serverId;
47 499 : nicDeploy_ = rankTable.nicDeploy;
48 499 : deviceNumPerServer_ = deviceNumPerServer;
49 499 : multiServerDiffDeviceNumMode_ = (serverNum_ * deviceNumPerServer_) == deviceNum_ ? false : true;
50 499 : CHK_RET(hrtGetDeviceType(deviceType_));
51 :
52 499 : DevType curDevType = rankTable.rankList.begin()->deviceInfo.deviceType;
53 1995 : for (auto rankInfo : rankTable.rankList) {
54 1496 : RankInfo curRankInfo;
55 1496 : curRankInfo.devicePhyId = rankInfo.deviceInfo.devicePhyId;
56 1496 : curRankInfo.deviceType = rankInfo.deviceInfo.deviceType;
57 1496 : curRankInfo.serverId = rankInfo.serverId;
58 1496 : curRankInfo.userRank = rankInfo.rankId;
59 1496 : curRankInfo.nicIp = rankInfo.deviceInfo.deviceIp;
60 1493 : curRankInfo.superDeviceId = rankInfo.superDeviceId;
61 1493 : curRankInfo.superPodId = rankInfo.superPodId;
62 1493 : rankList_.push_back(curRankInfo);
63 :
64 1493 : if (curDevType != rankInfo.deviceInfo.deviceType) {
65 0 : isDiffDeviceType_ = true;
66 : }
67 1493 : }
68 499 : return HCCL_SUCCESS;
69 : }
70 :
71 : HcclResult
72 33 : TopoInfoParse::Init(const std::vector<RankInfo>& rankList, const std::string& serverId, const u32 deviceNumPerServer)
73 : {
74 33 : CHK_PRT_RET(deviceNumPerServer == 0, HCCL_ERROR("cur device num per server is 0"), HCCL_E_PARA);
75 33 : rankList_ = rankList;
76 33 : serverId_ = serverId;
77 33 : deviceNumPerServer_ = deviceNumPerServer;
78 33 : CHK_RET(GetDevNum(rankList, deviceNum_));
79 33 : CHK_RET(GetServerNum(rankList, serverNum_));
80 33 : CHK_RET(GetSuperPodNum(rankList, superPodNum_));
81 33 : multiServerDiffDeviceNumMode_ = (serverNum_ * deviceNumPerServer_) == deviceNum_ ? false : true;
82 33 : CHK_RET(hrtGetDeviceType(deviceType_));
83 :
84 33 : DevType curDevType = rankList.begin()->deviceType;
85 190 : for (auto rankInfo : rankList) {
86 157 : if (curDevType != rankInfo.deviceType) {
87 0 : isDiffDeviceType_ = true;
88 : }
89 157 : }
90 33 : return HCCL_SUCCESS;
91 : }
92 :
93 532 : HcclResult TopoInfoParse::GetServerInnerLinkInfo(
94 : std::unordered_map<u32, u32>& pairLinkCounter,
95 : std::unordered_map<u32, std::unordered_map<int, std::vector<int>>>& pairLinkInfo)
96 : {
97 532 : std::vector<RankInfo> serverInnerInfo;
98 532 : CHK_RET(TransformRankInfoByServerId(serverInnerInfo));
99 :
100 531 : CHK_PRT_RET(
101 : serverInnerInfo.size() == 0,
102 : HCCL_ERROR(
103 : "[Get][ServerInnerLinkInfo]server info input is empty, "
104 : "serverid[%s]",
105 : serverId_.c_str()),
106 : HCCL_E_PARA);
107 532 : pairLinkInfo.clear();
108 532 : pairLinkCounter[static_cast<u32>(LinkTypeInServer::HCCS_TYPE)] = 0;
109 532 : pairLinkCounter[static_cast<u32>(LinkTypeInServer::PXI_TYPE)] = 0;
110 531 : pairLinkCounter[static_cast<u32>(LinkTypeInServer::SIO_TYPE)] = 0;
111 530 : pairLinkCounter[static_cast<u32>(LinkTypeInServer::HCCS_SW_TYPE)] = 0;
112 1868 : for (auto& it_local : serverInnerInfo) {
113 7147 : for (auto& it_dest : serverInnerInfo) {
114 5808 : if (it_local.devicePhyId == it_dest.devicePhyId || it_local.devicePhyId == HOST_DEVICE_ID
115 4473 : || it_dest.devicePhyId == HOST_DEVICE_ID) {
116 1333 : continue;
117 : }
118 : LinkTypeInServer linkType;
119 4475 : CHK_RET(hrtGetPairDeviceLinkType(it_local.devicePhyId, it_dest.devicePhyId, linkType));
120 4493 : pairLinkInfo[static_cast<u32>(linkType)][it_local.devicePhyId].push_back(it_dest.devicePhyId);
121 4479 : pairLinkCounter[static_cast<u32>(linkType)]++;
122 : }
123 : }
124 770 : for (auto it : pairLinkInfo) {
125 243 : HCCL_DEBUG("pair link information linkType[%u], size[%llu]", it.first, it.second.size());
126 243 : }
127 2660 : for (auto it : pairLinkCounter) {
128 2120 : HCCL_DEBUG("pair link counter information linkType[%u], size[%llu]", it.first, it.second);
129 : }
130 531 : return HCCL_SUCCESS;
131 531 : }
132 :
133 532 : HcclResult TopoInfoParse::TransformRankInfoByServerId(std::vector<hccl::RankInfo>& serverInnerInfo)
134 : {
135 : // 按server重新组织rank信息，便于后续校验及信息填写
136 2185 : for (auto tmpRankInfo : rankList_) {
137 1653 : if (tmpRankInfo.serverId == serverId_) {
138 1342 : serverInnerInfo.push_back(tmpRankInfo);
139 : }
140 1653 : }
141 : // 按设备Id从小到大的顺序排序
142 532 : std::sort(serverInnerInfo.begin(), serverInnerInfo.end(), [](const RankInfo& left, const RankInfo& right) {
143 1610 : return left.devicePhyId < right.devicePhyId;
144 : });
145 531 : return HCCL_SUCCESS;
146 : }
147 :
148 : // nicIdx不只是校验，还有修改
149 499 : HcclResult TopoInfoParse::ParseAndCheck(std::vector<u32>& nicIdx)
150 : {
151 499 : CHK_RET(CheckInterServerDeviceId());
152 498 : CHK_RET(CheckRankTableNicInfo(nicIdx));
153 497 : CHK_RET(CheckServerInnerRankInfo());
154 498 : return HCCL_SUCCESS;
155 : }
156 :
157 20 : HcclResult TopoInfoParse::Check()
158 : {
159 20 : CHK_RET(CheckInterServerDeviceId());
160 20 : CHK_RET(CheckServerInnerRankInfo());
161 20 : return HCCL_SUCCESS;
162 : }
163 :
164 : // server间device选取是否对称校验
165 519 : HcclResult TopoInfoParse::CheckInterServerDeviceId()
166 : {
167 519 : if (serverNum_ == 0) {
168 0 : HCCL_ERROR("[Check][DeviceId]errNo[0x%016llx] server num is zero", HCOM_ERROR_CODE(HCCL_E_PARA));
169 0 : return HCCL_E_PARA;
170 : }
171 519 : std::map<std::string, std::set<s32>> serverDeviceMapList;
172 2116 : for (auto it = rankList_.begin(); it != rankList_.end(); it++) {
173 1593 : std::string tmpServerId = it->serverId;
174 1597 : auto search = serverDeviceMapList.find(tmpServerId);
175 1593 : if (search != serverDeviceMapList.end()) {
176 883 : auto rs = serverDeviceMapList[tmpServerId].insert(it->devicePhyId);
177 885 : if (it->devicePhyId == HOST_DEVICE_ID) {
178 0 : continue;
179 : }
180 885 : if (!rs.second) {
181 0 : RPT_INPUT_ERR(
182 : true, "EI0014", std::vector<std::string>({"value", "variable", "expect"}),
183 : std::vector<std::string>(
184 : {std::to_string(it->devicePhyId), " \"Device Id of server Id " + it->serverId + " \" ",
185 : "is unique"}));
186 0 : HCCL_ERROR(
187 : "[%s][%s]errNo[0x%016llx] check ranklist[%u], device id repeat for one server",
188 : LOG_KEYWORDS_INIT_GROUP.c_str(), LOG_KEYWORDS_RANKTABLE_CHECK.c_str(), HCOM_ERROR_CODE(HCCL_E_PARA),
189 : it->userRank);
190 0 : return HCCL_E_PARA;
191 : }
192 : } else {
193 706 : std::set<s32> deviceSet;
194 706 : deviceSet.insert(it->devicePhyId);
195 706 : serverDeviceMapList.insert(std::pair<std::string, std::set<s32>>(tmpServerId, deviceSet));
196 703 : }
197 1588 : }
198 519 : if (serverDeviceMapList.size() == 0) {
199 0 : HCCL_ERROR(
200 : "[Check][DeviceId]errNo[0x%016llx] for all ranklist, server num is zero", HCOM_ERROR_CODE(HCCL_E_PARA));
201 0 : return HCCL_E_PARA;
202 : }
203 519 : return HCCL_SUCCESS;
204 517 : }
205 :
206 499 : HcclResult TopoInfoParse::CheckAndAssignNicInfo(std::vector<u32>& nicIdx)
207 : {
208 499 : if (isDiffDeviceType_) {
209 : // 混合组网
210 0 : return HCCL_SUCCESS;
211 499 : } else if (serverNum_ == 1 || superPodNum_ == 1) {
212 397 : nicIdx.clear();
213 1455 : for (u32 index = 0; index < rankList_.size(); index++) {
214 1061 : if (serverId_ == rankList_[index].serverId && (rankList_[index].devicePhyId != HOST_DEVICE_ID)) {
215 1063 : nicIdx.push_back(static_cast<u32>(rankList_[index].devicePhyId));
216 : }
217 : }
218 394 : std::sort(nicIdx.begin(), nicIdx.end());
219 497 : } else if (deviceNumPerServer_ == HCCL_AISERVER_DEVICE_NUM && !multiServerDiffDeviceNumMode_) {
220 0 : CHK_PRT_RET(nicIdx.size() == 0, HCCL_ERROR("[CheckAndAssign][NicInfo]nic idx size is 0"), HCCL_E_PARA);
221 0 : CHK_PRT_RET(deviceNum_ == 0, HCCL_ERROR("[CheckAndAssign][NicInfo]device num is 0"), HCCL_E_PARA);
222 :
223 0 : bool bRet = deviceNum_ % nicIdx.size() != 0; // 在n台server中，校验网口总数是否为1n/2n/4n/8n
224 0 : CHK_PRT_RET(bRet, HCCL_ERROR("[CheckAndAssign][NicInfo]nic total num[%zu] error", nicIdx.size()), HCCL_E_PARA);
225 :
226 0 : if (nicIdx.size() != deviceNumPerServer_) {
227 : // 网口裁剪
228 : // 按server重新组织rank信息
229 0 : std::map<std::string, std::vector<u32>> severNicsMap;
230 0 : for (size_t index = 0; index < rankList_.size(); ++index) {
231 0 : std::string serverId = rankList_[index].serverId;
232 :
233 0 : if (rankList_[index].nicIp.size() == 0 || rankList_[index].nicIp[0].IsInvalid()) {
234 0 : continue;
235 : }
236 : // 以serverID为索引，将server下的ranks放入vector
237 0 : auto itr = severNicsMap.find(serverId);
238 0 : if (itr != severNicsMap.end()) {
239 0 : itr->second.push_back(rankList_[index].devicePhyId);
240 : } else {
241 0 : std::vector<u32> nicList;
242 0 : nicList.push_back(rankList_[index].devicePhyId);
243 0 : std::pair<std::string, std::vector<u32>> nicInfoPair(serverId, nicList);
244 0 : severNicsMap.insert(nicInfoPair);
245 0 : }
246 0 : }
247 :
248 : // 每个server下的nicList按设备Id从小到大的顺序排序
249 0 : for (auto& iter : severNicsMap) {
250 0 : std::sort(iter.second.begin(), iter.second.end());
251 0 : if (nicIdx != iter.second) {
252 0 : HCCL_ERROR("nic list should be the same between servers");
253 0 : return HCCL_E_PARA;
254 : }
255 : }
256 0 : } else {
257 : // 网口满配
258 0 : return HCCL_SUCCESS;
259 : }
260 : }
261 497 : return HCCL_SUCCESS;
262 : }
263 :
264 : // nicIdx做填充，nicIdx也是deviceId，deviceId做过的校验这里不再重复
265 499 : HcclResult TopoInfoParse::CheckRankTableNicInfo(std::vector<u32>& nicIdx)
266 : {
267 : // 在8P均使用的情况下校验nic的选择信息是否正确
268 499 : if (nicDeploy_ == NICDeployment::NIC_DEPLOYMENT_DEVICE) {
269 499 : CHK_RET(CheckAndAssignNicInfo(nicIdx));
270 0 : } else if (
271 0 : nicDeploy_ == NICDeployment::NIC_DEPLOYMENT_HOST && serverNum_ == deviceNum_ / HCCL_AISERVER_DEVICE_NUM) {
272 0 : nicIdx.assign({0, 1, 2, 3, 4, 5, 6, 7}); // 如果每个server8个rank且为host nic，则网口为满配
273 : }
274 497 : return HCCL_SUCCESS;
275 : }
276 :
277 : // 校验server内4p场景下deivce选取是否合法，2p与标卡场景重合
278 517 : HcclResult TopoInfoParse::CheckServerInnerRankInfo()
279 : {
280 : // 校验server内device选取
281 517 : std::vector<s32> serverInnerDeviceInfo;
282 2109 : for (u32 index = 0; index < rankList_.size(); index++) {
283 1592 : if (serverId_ == rankList_[index].serverId && rankList_[index].devicePhyId != HOST_DEVICE_ID) {
284 : /* 同一server的标识IP 是一样的，所以可以以此推算出平均dev个数 */
285 1321 : serverInnerDeviceInfo.push_back(rankList_[index].devicePhyId);
286 : }
287 : }
288 517 : std::sort(serverInnerDeviceInfo.begin(), serverInnerDeviceInfo.end());
289 :
290 518 : if (deviceType_ == DevType::DEV_TYPE_910) {
291 381 : if (deviceNumPerServer_ != HCCL_DEVICE_NUM_FOUR) {
292 359 : return HCCL_SUCCESS;
293 : }
294 22 : std::string selectedDevice = "selected devices:";
295 110 : for (auto devicePhyId : serverInnerDeviceInfo) {
296 88 : selectedDevice += std::to_string(devicePhyId);
297 88 : selectedDevice += " ";
298 : }
299 22 : if (HCCL_AISERVER_VAILD_4P_RANKS.find(serverInnerDeviceInfo) == HCCL_AISERVER_VAILD_4P_RANKS.end()) {
300 0 : std::string errormessage = "Value " + selectedDevice
301 0 : + " for rankTable "
302 0 : "variable \"Device Id of server Id "
303 0 : + serverId_ + " \" is invalid, expected value is unique.";
304 0 : errormessage += selectedDevice;
305 0 : RPT_INPUT_ERR(
306 : true, "EI0014", std::vector<std::string>({"value", "variable", "expect"}),
307 : std::vector<std::string>(
308 : {selectedDevice, " \"Device Id of server Id " + serverId_ + " \" ", "is unique"}));
309 0 : HCCL_ERROR(
310 : "[%s][%s] %s", LOG_KEYWORDS_INIT_GROUP.c_str(), LOG_KEYWORDS_RANKTABLE_CHECK.c_str(),
311 : errormessage.c_str());
312 0 : return HCCL_E_PARA;
313 0 : }
314 22 : HCCL_DEBUG("%s", selectedDevice.c_str());
315 22 : }
316 :
317 159 : return HCCL_SUCCESS;
318 518 : }
319 :
320 532 : HcclResult TopoInfoParse::IsAllRankSamePlane(bool& isAllRankSamePlane)
321 : {
322 : // 只有1个rank，不考虑
323 532 : if (rankList_.size() == 1) {
324 206 : isAllRankSamePlane = true;
325 206 : return HCCL_SUCCESS;
326 : }
327 :
328 326 : auto isSameDevId = [&]() -> bool {
329 326 : s32 maxModuleDevNum = deviceType_ == DevType::DEV_TYPE_910B ? HCCL_DEVICE_NUM_EIGHT : MAX_MODULE_DEVICE_NUM;
330 487 : for (size_t index = 0; index < rankList_.size() - 1; ++index) {
331 419 : if (rankList_[index].devicePhyId % maxModuleDevNum != rankList_[index + 1].devicePhyId % maxModuleDevNum) {
332 258 : return false;
333 : }
334 : }
335 68 : return true;
336 326 : };
337 :
338 326 : isAllRankSamePlane = isSameDevId();
339 326 : HCCL_DEBUG("[TopoInfoParse]curr comm isAllRankSamePlane[%d]", isAllRankSamePlane);
340 326 : return HCCL_SUCCESS;
341 : }
342 :
343 531 : HcclResult TopoInfoParse::IsSingleMeshAggregation(bool& isSingleMeshAggregation)
344 : {
345 531 : if (deviceNumPerServer_ == deviceNum_) {
346 : // rank间都是hccs链接，则表明在同一个mesh cube中
347 424 : CHK_RET(IsAllRankConnectedWithHCCS(isSingleMeshAggregation));
348 : } else {
349 107 : isSingleMeshAggregation = false;
350 : }
351 531 : HCCL_DEBUG("[TopoInfoParse]curr comm isSingleMeshAggregation[%d]", isSingleMeshAggregation);
352 532 : return HCCL_SUCCESS;
353 : }
354 :
355 424 : HcclResult TopoInfoParse::IsAllRankConnectedWithHCCS(bool& isAllRankConnectedWithHCCS)
356 : {
357 1607 : for (u32 i = 0; i < rankList_.size(); i++) {
358 3334 : for (u32 j = i + 1; j < rankList_.size(); j++) {
359 2148 : LinkTypeInServer linkType = LinkTypeInServer::RESERVED_LINK_TYPE;
360 2148 : if (rankList_[i].devicePhyId != HOST_DEVICE_ID && rankList_[j].devicePhyId != HOST_DEVICE_ID) {
361 2148 : CHK_RET(hrtGetPairDeviceLinkType(rankList_[i].devicePhyId, rankList_[j].devicePhyId, linkType));
362 : }
363 2153 : if (linkType == LinkTypeInServer::PXI_TYPE) {
364 0 : isAllRankConnectedWithHCCS = false;
365 0 : return HCCL_SUCCESS;
366 : }
367 : }
368 : }
369 426 : isAllRankConnectedWithHCCS = true;
370 426 : return HCCL_SUCCESS;
371 : }
372 :
373 0 : HcclResult TopoInfoParse::GetDeviceNumInPerMeshAggregation(u32 devicePhyId, u32& perAggregationNum)
374 : {
375 : // 每个rank本身加上和他通过hccs互联的rank表示当前server Aggregation中的rank数量
376 0 : perAggregationNum = 1;
377 0 : for (u32 i = 0; i < rankList_.size(); i++) {
378 0 : if (serverId_ == rankList_[i].serverId && rankList_[i].devicePhyId != static_cast<s32>(devicePhyId)) {
379 : LinkTypeInServer linkType;
380 0 : CHK_RET(hrtGetPairDeviceLinkType(devicePhyId, rankList_[i].devicePhyId, linkType));
381 0 : if (linkType == LinkTypeInServer::HCCS_TYPE) {
382 0 : perAggregationNum++;
383 : }
384 : }
385 : }
386 0 : HCCL_DEBUG(
387 : "[TopoInfoParse][GetDeviceNumInPerMeshAggregation]serverId[%s] devicePhyId[%u] perAggregationNum[%u]",
388 : serverId_.c_str(), devicePhyId, perAggregationNum);
389 :
390 0 : return HCCL_SUCCESS;
391 : }
392 : } // namespace hccl
|
