Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "config.h"
12 : #include <arpa/inet.h>
13 : #include <cctype>
14 : #include <fcntl.h>
15 : #include <securec.h>
16 : #include <sys/socket.h>
17 : #include <sys/types.h>
18 : #include <unistd.h>
19 : #include <map>
20 : #include <set>
21 : #include "topoinfo_ranktableParser_pub.h"
22 : #include "./topo/topoinfo_ranktableStandard.h"
23 : #include "./topo/topoinfo_ranktableConcise.h"
24 : #include "./topo/topoinfo_ranktableHeterog.h"
25 : #include "./topo/topoinfo_roletableParser.h"
26 : #include "comm.h"
27 :
28 : using namespace std;
29 : using namespace hccl;
30 :
31 : std::set<std::string> g_oneSidedIdentifierSet;
32 : std::mutex g_oneSidedIdentifierMutex;
33 :
34 388 : HcclResult CfgGetClusterInfo(
35 : const std::string& rankTableM, const std::string& identify, hccl::HcclCommParams& params,
36 : hccl::RankTable_t& rankTable, bool isInterSuperPodRetryEnable, DevType deviceType)
37 : {
38 388 : TopoInfoRanktableParser myTopoRanktable(rankTableM, identify);
39 388 : CHK_RET(myTopoRanktable.Init());
40 : // 获取rankTable版本
41 388 : CHK_RET(myTopoRanktable.GetRanktableVersion(rankTable.version));
42 : // 根据rankTable有没有版本信息属性和版本信息确定解析的方式
43 388 : std::unique_ptr<TopoInfoRanktableParser> pTopoRanktable = nullptr;
44 388 : if (rankTable.version.compare(HCCL_CLUSTER_VERSION) == 0
45 388 : || rankTable.version.compare(SUPERPOD_CLUSTER_VERSION) == 0) {
46 0 : pTopoRanktable.reset(new (std::nothrow) TopoinfoRanktableConcise(rankTableM, identify));
47 0 : pTopoRanktable->SetIsInterSuperPodRetryEnable(isInterSuperPodRetryEnable);
48 388 : } else if (rankTable.version.compare(HETEROG_CLUSTER_VERSION) == 0) {
49 0 : pTopoRanktable.reset(new (std::nothrow) TopoinfoRanktableHeterog(rankTableM, identify, deviceType));
50 388 : } else if (rankTable.version.compare("Standard") == 0) {
51 388 : pTopoRanktable.reset(new (std::nothrow) TopoinfoRanktableStandard(rankTableM, identify));
52 : } else {
53 0 : RPT_INPUT_ERR(
54 : true, "EI0014", std::vector<std::string>({"value", "variable", "expect"}),
55 : std::vector<std::string>({rankTable.version, "version", "a valid version number."}));
56 0 : HCCL_ERROR(
57 : "[%s][%s]version[%s] is not support", LOG_KEYWORDS_INIT_GROUP.c_str(), LOG_KEYWORDS_RANKTABLE_CHECK.c_str(),
58 : rankTable.version.c_str());
59 0 : return HCCL_E_NOT_SUPPORT;
60 : }
61 : // 检查指针是否为空
62 387 : CHK_SMART_PTR_NULL(pTopoRanktable);
63 : // 执行初始化，加载rankTable并进行解析
64 387 : CHK_RET(pTopoRanktable->Init());
65 : // 将解析到的内容保存到入参hcomInfo中
66 387 : HcclResult ret = pTopoRanktable->GetClusterInfo(params, rankTable);
67 387 : CHK_PRT_RET(
68 : ret != HCCL_SUCCESS, HCCL_ERROR("[Get][ClusterInfo]identify[%s],get cluterInfo info error", identify.c_str()),
69 : ret);
70 :
71 387 : CHK_PRT_RET((rankTable.serverNum == 0), HCCL_ERROR("[Get][ClusterInfo]serverNum is zero."), HCCL_E_PARA);
72 387 : CHK_RET(CheckRankListInfo(rankTable.rankList));
73 :
74 386 : if (rankTable.serverNum > 1) {
75 13 : CHK_RET(CheckRankIpFamily(rankTable.rankList));
76 : }
77 386 : if (rankTable.version.compare(HETEROG_CLUSTER_VERSION) == 0) {
78 : // 异构场景无需检查
79 0 : return HCCL_SUCCESS;
80 : } else {
81 386 : CHK_RET(CheckRankListBaseInfo(rankTable.deviceNum, rankTable.serverNum));
82 387 : CHK_RET(CheckDeviceNumValid(rankTable.rankList, rankTable.deviceNum, rankTable.serverNum, rankTable.version));
83 : }
84 386 : return HCCL_SUCCESS;
85 387 : }
86 :
87 0 : HcclResult CfgGetClusterInfoWithoutDev(
88 : const std::string& rankTableM, const std::string& identify, hccl::HcclCommParams& params,
89 : hccl::RankTable_t& rankTable, bool isInterSuperPodRetryEnable)
90 : {
91 0 : TopoInfoRanktableParser myTopoRanktable(rankTableM, identify);
92 0 : CHK_RET(myTopoRanktable.Init());
93 : // 获取rankTable版本
94 0 : CHK_RET(myTopoRanktable.GetRanktableVersion(rankTable.version));
95 : // 根据rankTable有没有版本信息属性和版本信息确定解析的方式
96 0 : std::unique_ptr<TopoInfoRanktableParser> pTopoRanktable = nullptr;
97 0 : if (rankTable.version.compare(HCCL_CLUSTER_VERSION) == 0) {
98 0 : pTopoRanktable.reset(new (std::nothrow) TopoinfoRanktableConcise(rankTableM, identify));
99 0 : pTopoRanktable->SetIsInterSuperPodRetryEnable(isInterSuperPodRetryEnable);
100 0 : } else if (rankTable.version.compare(HETEROG_CLUSTER_VERSION) == 0) {
101 0 : pTopoRanktable.reset(new (std::nothrow) TopoinfoRanktableHeterog(rankTableM, identify));
102 0 : } else if (rankTable.version.compare("Standard") == 0) {
103 0 : pTopoRanktable.reset(new (std::nothrow) TopoinfoRanktableStandard(rankTableM, identify));
104 : } else {
105 0 : HCCL_ERROR("[Get][RanktableVersion]version[%s] is not support", rankTable.version.c_str());
106 0 : return HCCL_E_NOT_SUPPORT;
107 : }
108 : // 检查指针是否为空
109 0 : CHK_SMART_PTR_NULL(pTopoRanktable);
110 : // 执行初始化，加载rankTable并进行解析
111 0 : CHK_RET(pTopoRanktable->Init());
112 : // 将解析到的内容保存到入参params、rankTable中
113 0 : HcclResult ret = pTopoRanktable->GetClusterInfo(params, rankTable);
114 0 : CHK_PRT_RET(
115 : ret != HCCL_SUCCESS, HCCL_ERROR("[Get][ClusterInfo]identify[%s],get cluterInfo info error", identify.c_str()),
116 : ret);
117 :
118 0 : CHK_RET(CheckRankListInfo(rankTable.rankList));
119 0 : CHK_RET(CheckDeviceNumValid(rankTable.rankList, rankTable.deviceNum, rankTable.serverNum, rankTable.version));
120 0 : return HCCL_SUCCESS;
121 0 : }
122 :
123 388 : HcclResult CheckRankId(const char* rankId)
124 : {
125 388 : CHK_PTR_NULL(rankId);
126 388 : string temp = rankId;
127 :
128 776 : for (u32 index = 0; index < temp.length(); index++) {
129 388 : if (!isdigit(temp[index])) {
130 0 : HCCL_ERROR("[Check][RankId]errNo[0x%016llx] check rankid is not digit", HCOM_ERROR_CODE(HCCL_E_PARA));
131 0 : return HCCL_E_PARA;
132 : }
133 : }
134 388 : return HCCL_SUCCESS;
135 388 : }
136 :
137 0 : HcclResult CheckRankTableConfigInfo(const std::vector<RankInfo_t>& rankList, u32 deviceNum, u32 serverNum)
138 : {
139 0 : if (rankList.size() != deviceNum) {
140 0 : HCCL_ERROR(
141 : "[Check][RankTableConfigInfo]errNo[0x%016llx] rankList size[%llu] neq deviceNum[%u]",
142 : HCOM_ERROR_CODE(HCCL_E_PARA), rankList.size(), deviceNum);
143 0 : return HCCL_E_PARA;
144 : }
145 0 : CHK_RET(CheckGroupRankList(rankList, deviceNum, serverNum));
146 0 : return HCCL_SUCCESS;
147 : }
148 :
149 234 : HcclResult ShowRanktableConfigInfo(const bool cloudFlag, hccl::HcclCommParams& params, hccl::RankTable_t& rankTable)
150 : {
151 234 : if (cloudFlag) {
152 0 : CHK_RET(DisplayCloudRankTableInfo(params, rankTable));
153 : } else {
154 234 : CHK_RET(DisplayRanktableInfo(params, rankTable));
155 : }
156 234 : return HCCL_SUCCESS;
157 : }
158 :
159 0 : HcclResult DisplayCloudRankTableInfo(hccl::HcclCommParams& params, hccl::RankTable_t& rankTable)
160 : {
161 0 : HCCL_DEBUG(
162 : "rank_table: \"Unique groupNum\":\"%u\", \"Unique deviceNum\":\"%u\", \"Unique serverNum\":\"%u\"",
163 : rankTable.groupNum, rankTable.deviceNum, rankTable.serverNum);
164 :
165 0 : HCCL_DEBUG("params: \"uniqueID\":\"%s\"", params.id.internal);
166 0 : return HCCL_SUCCESS;
167 : }
168 :
169 234 : HcclResult DisplayRanktableInfo(hccl::HcclCommParams& params, hccl::RankTable_t& rankTable)
170 : {
171 234 : std::string nicName = "\"para_plane_nic_name\":[";
172 :
173 234 : for (u32 i = 0; i < rankTable.nicNames.size(); i++) {
174 0 : if (i != 0) {
175 0 : nicName += ",";
176 : }
177 0 : std::string tmpString = rankTable.nicNames[i].c_str();
178 0 : nicName += "\"";
179 0 : }
180 234 : nicName += "],";
181 234 : HCCL_DEBUG(
182 : "rank_table: \"Unique deviceNum\":\"%u\", \"Unique serverNum\":\"%u\", \"para_plane_nic_location\""
183 : ":\"%u\", \"para_plane_nic_num\":\"%u\";%s",
184 : rankTable.deviceNum, rankTable.serverNum, rankTable.nicDeploy, rankTable.nicNum, nicName.c_str());
185 :
186 234 : HCCL_DEBUG("params: \"uniqueID\":\"%s\"", params.id.internal);
187 234 : return HCCL_SUCCESS;
188 234 : }
189 1 : HcclResult DisplayRanktableInfo(const hccl::RankTable_t& rankTable)
190 : {
191 1 : std::string nicName = "\"para_plane_nic_name\":[";
192 :
193 1 : for (u32 i = 0; i < rankTable.nicNames.size(); i++) {
194 0 : if (i != 0) {
195 0 : nicName += ",";
196 : }
197 0 : nicName += rankTable.nicNames[i];
198 : }
199 1 : nicName += "]";
200 :
201 1 : HCCL_INFO(
202 : "rank_table: \"Unique deviceNum\":\"%u\", \"Unique serverNum\":\"%u\", \"para_plane_nic_location\""
203 : ":\"%u\", \"para_plane_nic_num\":\"%u\";%s",
204 : rankTable.deviceNum, rankTable.serverNum, rankTable.nicDeploy, rankTable.nicNum, nicName.c_str());
205 :
206 1 : std::string deviceInfoStr = "\"device_information\":";
207 2 : for (u32 i = 0; i < rankTable.rankList.size(); i++) {
208 1 : deviceInfoStr += "{rankID[" + to_string(rankTable.rankList[i].rankId) + "],";
209 1 : deviceInfoStr += "serverId[" + rankTable.rankList[i].serverId + "],";
210 1 : deviceInfoStr += "deviceId[" + to_string(rankTable.rankList[i].deviceInfo.devicePhyId) + "],";
211 1 : deviceInfoStr += "deviceType[" + to_string(u32(rankTable.rankList[i].deviceInfo.deviceType)) + "],";
212 1 : string tmpString = rankTable.rankList[i].deviceInfo.deviceIp[0].GetReadableAddress();
213 1 : deviceInfoStr += "deviceIp[" + tmpString + "],";
214 1 : tmpString = rankTable.rankList[i].hostIp.GetReadableAddress();
215 1 : deviceInfoStr += "hostIp[" + tmpString + "]},";
216 1 : HCCL_INFO("%s", deviceInfoStr.c_str());
217 1 : deviceInfoStr.clear();
218 1 : }
219 :
220 1 : return HCCL_SUCCESS;
221 1 : }
222 :
223 1197 : HcclResult GetDevNum(const std::vector<RankInfo_t>& rankList, u32& devNum)
224 : {
225 1197 : devNum = 0;
226 3925 : for (auto& iter : rankList) {
227 2728 : if (iter.deviceInfo.devicePhyId != HOST_DEVICE_ID) {
228 2728 : devNum++;
229 : }
230 : }
231 1193 : return HCCL_SUCCESS;
232 : }
233 :
234 33 : HcclResult GetServerNum(const std::vector<RankInfo>& rankList, u32& serverNum)
235 : {
236 33 : serverNum = 0;
237 33 : std::set<u32> serverSet;
238 190 : for (auto& iter : rankList) {
239 157 : serverSet.insert(iter.serverIdx);
240 : }
241 33 : serverNum = serverSet.size();
242 33 : return HCCL_SUCCESS;
243 33 : }
244 :
245 33 : HcclResult GetDevNum(const std::vector<RankInfo>& rankList, u32& devNum)
246 : {
247 33 : devNum = 0;
248 190 : for (auto& iter : rankList) {
249 157 : if (iter.devicePhyId != HOST_DEVICE_ID) {
250 157 : devNum++;
251 : }
252 : }
253 33 : return HCCL_SUCCESS;
254 : }
255 :
256 : template <typename rankTable>
257 33 : HcclResult GetSuperPodNums(const std::vector<rankTable>& rankList, u32& superPodNum)
258 : {
259 33 : superPodNum = 0;
260 33 : std::set<std::string> superPodIds;
261 :
262 33 : for (u32 index = 0; index < rankList.size(); index++) {
263 : // superPodId为空时, 返回超节点数量为0, 按照非超节点模式处理
264 33 : CHK_PRT_RET(
265 : rankList[index].superPodId.empty(),
266 : HCCL_DEBUG(
267 : "ranks[%u] superPodId[%s] is empty, set superPodNum to zero", index,
268 : rankList[index].superPodId.c_str()),
269 : HCCL_SUCCESS);
270 :
271 0 : if (superPodIds.find(rankList[index].superPodId) == superPodIds.end()) {
272 0 : superPodIds.insert(rankList[index].superPodId);
273 : }
274 : }
275 0 : superPodNum = superPodIds.size();
276 0 : return HCCL_SUCCESS;
277 33 : }
278 :
279 0 : HcclResult GetSuperPodNum(const std::vector<RankInfo_t>& rankList, u32& superPodNum)
280 : {
281 0 : (void)GetSuperPodNums(rankList, superPodNum);
282 0 : return HCCL_SUCCESS;
283 : }
284 :
285 33 : HcclResult GetSuperPodNum(const std::vector<RankInfo>& rankList, u32& superPodNum)
286 : {
287 33 : (void)GetSuperPodNums(rankList, superPodNum);
288 33 : return HCCL_SUCCESS;
289 : }
290 :
291 0 : HcclResult CheckGroupRankList(const std::vector<RankInfo_t>& rankList, [[maybe_unused]] u32 deviceNum, u32 serverNum)
292 : {
293 0 : u32 realDevNum = 0;
294 0 : CHK_RET(GetDevNum(rankList, realDevNum));
295 0 : CHK_RET(CheckAverageDev(realDevNum, serverNum));
296 0 : CHK_RET(CheckRankListInfo(rankList, realDevNum, serverNum));
297 0 : return HCCL_SUCCESS;
298 : }
299 :
300 0 : HcclResult CheckDeviceId(const std::vector<RankInfo_t>& rankList, u32 deviceNum, u32 serverNum)
301 : // each server should has same device Id may not be continuous
302 : {
303 0 : if (serverNum == 0) {
304 0 : HCCL_ERROR("[Check][DeviceId]errNo[0x%016llx] server num is zero", HCOM_ERROR_CODE(HCCL_E_PARA));
305 0 : return HCCL_E_PARA;
306 : }
307 :
308 : (void)deviceNum;
309 0 : std::map<std::string, std::set<s32>> serverDeviceMapList;
310 0 : for (auto it = rankList.begin(); it != rankList.end(); it++) {
311 0 : if (it->deviceInfo.devicePhyId == HOST_DEVICE_ID) {
312 0 : continue;
313 : }
314 0 : std::string tmpServerId = it->serverId;
315 0 : auto search = serverDeviceMapList.find(tmpServerId);
316 0 : if (search != serverDeviceMapList.end()) {
317 0 : auto rs = serverDeviceMapList[tmpServerId].insert(it->deviceInfo.devicePhyId);
318 0 : if (!rs.second) {
319 0 : RPT_INPUT_ERR(
320 : true, "EI0014", std::vector<std::string>({"value", "variable", "expect"}),
321 : std::vector<std::string>(
322 : {std::to_string(it->deviceInfo.devicePhyId),
323 : +" \"Device Id of server Id " + tmpServerId + " \" ", "is unique"}));
324 0 : HCCL_ERROR(
325 : "[%s][%s]errNo[0x%016llx] check ranklist[%u], device id repeat for one server",
326 : LOG_KEYWORDS_INIT_GROUP.c_str(), LOG_KEYWORDS_RANKTABLE_CHECK.c_str(), HCOM_ERROR_CODE(HCCL_E_PARA),
327 : it->rankId);
328 0 : return HCCL_E_PARA;
329 : }
330 : } else {
331 0 : std::set<s32> deviceSet;
332 0 : deviceSet.insert(it->deviceInfo.devicePhyId);
333 0 : serverDeviceMapList.insert(std::pair<std::string, std::set<s32>>(tmpServerId, deviceSet));
334 0 : }
335 0 : }
336 0 : if (serverDeviceMapList.size() == 0) {
337 0 : HCCL_ERROR(
338 : "[Check][DeviceId]errNo[0x%016llx] for all ranklist, server num is zero", HCOM_ERROR_CODE(HCCL_E_PARA));
339 0 : return HCCL_E_PARA;
340 : }
341 0 : return HCCL_SUCCESS;
342 0 : }
343 :
344 : // 检查deviceNum、serverNum范围，rankList中rank id范围及是否升序连续分布
345 387 : HcclResult CheckRankListBaseInfo(u32 deviceNum, u32 serverNum)
346 : {
347 387 : HCCL_INFO("START CheckRankListBaseInfo");
348 387 : if (deviceNum == 0) {
349 0 : HCCL_ERROR("[Check][RankListBaseInfo]errNo[0x%016llx] device num is zero", HCOM_ERROR_CODE(HCCL_E_PARA));
350 0 : return HCCL_E_PARA;
351 : }
352 387 : if (serverNum == 0) {
353 0 : HCCL_ERROR("[Check][RankListBaseInfo]errNo[0x%016llx] server num is zero", HCOM_ERROR_CODE(HCCL_E_PARA));
354 0 : return HCCL_E_PARA;
355 : }
356 387 : return HCCL_SUCCESS;
357 : }
358 :
359 386 : HcclResult CheckRankListInfo(const std::vector<RankInfo_t>& rankList)
360 : {
361 : // rankList have been sorted when parsing rank table,
362 : // check the continuity of sorted rankList
363 386 : HCCL_INFO("START CheckRankListInfo");
364 1520 : for (u32 index = 0; index < rankList.size(); index++) {
365 1134 : if (rankList[index].rankId != index) {
366 0 : RPT_INPUT_ERR(
367 : true, "EI0014", std::vector<std::string>({"value", "variable", "expect"}),
368 : std::vector<std::string>(
369 : {std::to_string(rankList[index].rankId), "rank_id",
370 : "is less than the communication size " + std::to_string(rankList.size())
371 : + " and must be unique."}));
372 0 : HCCL_ERROR(
373 : "[%s][%s]errNo[0x%016llx] rankList[%u] rankId[%u] error", LOG_KEYWORDS_INIT_GROUP.c_str(),
374 : LOG_KEYWORDS_RANKTABLE_CHECK.c_str(), HCOM_ERROR_CODE(HCCL_E_PARA), index, rankList[index].rankId);
375 0 : return HCCL_E_PARA;
376 : }
377 : }
378 386 : return HCCL_SUCCESS;
379 0 : }
380 :
381 : // 校验rank ip family一致性
382 21 : HcclResult CheckRankIpFamily(const std::vector<RankInfo_t>& rankList)
383 : {
384 21 : HCCL_INFO("START CheckRankIpFamily");
385 21 : s32 hostFamily = 0;
386 21 : s32 deviceFamily = 0;
387 21 : std::string errormessage = "";
388 133 : for (u32 index = 0; index < rankList.size(); index++) {
389 116 : if (!rankList[index].hostIp.IsInvalid()) {
390 12 : if ((rankList[index].hostIp.GetFamily() != AF_INET) && (rankList[index].hostIp.GetFamily() != AF_INET6)) {
391 15 : RPT_INPUT_ERR(
392 : true, "EI0014", std::vector<std::string>({"value", "variable", "expect"}),
393 : std::vector<std::string>(
394 : {std::to_string(rankList[index].hostIp.GetFamily()),
395 : " \"Device Id of server Id " + rankList[index].serverId + "\" ", "is unique"}));
396 2 : errormessage = "Value " + std::to_string(rankList[index].hostIp.GetFamily())
397 1 : + " for rankTable variable \"Device Id of "
398 1 : "server Id "
399 2 : + rankList[index].serverId + "\" is invalid, expected value is unique.";
400 1 : HCCL_ERROR(
401 : "[%s][%s] %s", LOG_KEYWORDS_INIT_GROUP.c_str(), LOG_KEYWORDS_RANKTABLE_CHECK.c_str(),
402 : errormessage.c_str());
403 1 : return HCCL_E_PARA;
404 : }
405 :
406 11 : if (hostFamily != 0 && hostFamily != rankList[index].hostIp.GetFamily()) {
407 15 : RPT_INPUT_ERR(
408 : true, "EI0014", std::vector<std::string>({"value", "variable", "expect"}),
409 : std::vector<std::string>(
410 : {std::to_string(rankList[index].hostIp.GetFamily()),
411 : " \"Device Id of server Id " + rankList[index].serverId + "\" ", "is unique"}));
412 2 : errormessage = "Value " + std::to_string(rankList[index].hostIp.GetFamily())
413 1 : + " for rankTable variable \"Device Id of "
414 1 : "server Id "
415 2 : + rankList[index].serverId + "\" is invalid, expected value is unique.";
416 1 : HCCL_ERROR(
417 : "[%s][%s]%s", LOG_KEYWORDS_INIT_GROUP.c_str(), LOG_KEYWORDS_RANKTABLE_CHECK.c_str(),
418 : errormessage.c_str());
419 1 : return HCCL_E_PARA;
420 : }
421 :
422 10 : hostFamily = rankList[index].hostIp.GetFamily();
423 : }
424 :
425 : // device ip不存在时, 无需校验
426 114 : if (rankList[index].deviceInfo.deviceIp.empty()
427 222 : || ((rankList[index].deviceInfo.deviceIp.size() == 1)
428 108 : && rankList[index].deviceInfo.deviceIp[0].IsInvalid())) {
429 7 : continue;
430 : }
431 :
432 212 : for (auto& iter : rankList[index].deviceInfo.deviceIp) {
433 107 : if ((iter.GetFamily() != AF_INET) && (iter.GetFamily() != AF_INET6)) {
434 15 : RPT_INPUT_ERR(
435 : true, "EI0014", std::vector<std::string>({"value", "variable", "expect"}),
436 : std::vector<std::string>(
437 : {std::to_string(rankList[index].hostIp.GetFamily()),
438 : " \"Device Id of "
439 : "server Id "
440 : + rankList[index].serverId + " \" ",
441 : "is unique"}));
442 2 : errormessage = "Value " + std::to_string(rankList[index].hostIp.GetFamily())
443 1 : + " for rankTable variable \"Device Id of "
444 1 : "server Id "
445 2 : + rankList[index].serverId + " \" is invalid, expected value is unique.";
446 1 : HCCL_ERROR(
447 : "[%s][%s]%s", LOG_KEYWORDS_INIT_GROUP.c_str(), LOG_KEYWORDS_RANKTABLE_CHECK.c_str(),
448 : errormessage.c_str());
449 2 : return HCCL_E_PARA;
450 : }
451 106 : if (deviceFamily != 0 && deviceFamily != iter.GetFamily()) {
452 14 : RPT_ENV_ERR(
453 : true, "EI0001", std::vector<std::string>({"value", "env", "expect"}),
454 : std::vector<std::string>(
455 : {std::to_string(iter.GetFamily()), "RankIpFamily", std::to_string(deviceFamily)}));
456 1 : CHK_PRT_RET(
457 : true,
458 : HCCL_ERROR(
459 : "[%s][%s]rank[%u] device ip family[%d] is not same as others[%d].",
460 : LOG_KEYWORDS_INIT_GROUP.c_str(), LOG_KEYWORDS_ENV_CONFIG.c_str(), rankList[index].rankId,
461 : iter.GetFamily(), deviceFamily),
462 : HCCL_E_PARA);
463 : }
464 105 : deviceFamily = iter.GetFamily();
465 : }
466 : }
467 17 : return HCCL_SUCCESS;
468 32 : }
469 :
470 0 : HcclResult CheckRankListInfo(const std::vector<RankInfo_t>& rankList, u32 deviceNum, u32 serverNum)
471 : {
472 0 : CHK_RET(CheckRankListBaseInfo(deviceNum, serverNum));
473 0 : CHK_RET(CheckRankListInfo(rankList));
474 0 : CHK_RET(CheckDeviceNumValid(rankList, deviceNum, serverNum));
475 :
476 : // 校验每个serverID下的deviceID是否都在同一范围
477 0 : CHK_RET(CheckDeviceId(rankList, deviceNum, serverNum));
478 :
479 0 : return HCCL_SUCCESS;
480 : }
481 :
482 : // 检查rank list中每个server id下的device数是否相同
483 : HcclResult
484 387 : CheckDeviceNumValid(const std::vector<RankInfo_t>& rankList, u32 deviceNum, u32 serverNum, std::string version)
485 : {
486 387 : if (serverNum == 0) {
487 0 : HCCL_ERROR("[Check][DeviceNumValid]errNo[0x%016llx] server num is zero", HCOM_ERROR_CODE(HCCL_E_PARA));
488 0 : return HCCL_E_PARA;
489 : }
490 :
491 387 : if (version.compare(HETEROG_CLUSTER_VERSION) != 0) {
492 : DevType deviceType;
493 525 : CHK_RET(hrtGetDeviceType(deviceType));
494 : // 不对910B进行Server间卡数一致性的校验
495 387 : if (deviceType == DevType::DEV_TYPE_910B || deviceType == DevType::DEV_TYPE_910_93) {
496 138 : return HCCL_SUCCESS;
497 : }
498 : }
499 :
500 249 : std::map<std::string, u32> serverDeviceNumMapList;
501 689 : for (auto it = rankList.begin(); it != rankList.end(); it++) {
502 440 : if (it->deviceInfo.devicePhyId == HOST_DEVICE_ID) {
503 0 : continue;
504 : }
505 440 : std::string curServerId = it->serverId;
506 440 : auto search = serverDeviceNumMapList.find(curServerId);
507 440 : if (search != serverDeviceNumMapList.end()) {
508 191 : serverDeviceNumMapList[curServerId] = serverDeviceNumMapList[curServerId] + 1;
509 : } else {
510 249 : serverDeviceNumMapList.insert(std::pair<std::string, u32>(curServerId, 1));
511 : }
512 440 : }
513 498 : for (auto it = serverDeviceNumMapList.begin(); it != serverDeviceNumMapList.end(); it++) {
514 249 : if (it->second != (deviceNum / serverNum)) {
515 0 : RPT_INPUT_ERR(
516 : true, "EI0014", std::vector<std::string>({"value", "variable", "expect"}),
517 : std::vector<std::string>(
518 : {std::to_string(it->second),
519 : "\"DeviceNum of "
520 : "server Id "
521 : + it->first + "\" ",
522 : "is that number of devices under each server node is the same"}));
523 0 : HCCL_ERROR(
524 : "[%s][%s]errNo[0x%016llx] devices num of each server error", LOG_KEYWORDS_INIT_GROUP.c_str(),
525 : LOG_KEYWORDS_RANKTABLE_CHECK.c_str(), HCOM_ERROR_CODE(HCCL_E_PARA));
526 0 : return HCCL_E_PARA;
527 : }
528 : }
529 249 : return HCCL_SUCCESS;
530 249 : }
531 :
532 0 : HcclResult CheckPortValid(u32 port)
533 : {
534 0 : if (port > PORT_MAX) {
535 0 : HCCL_ERROR(
536 : "[Check][PortValid]errNo[0x%016llx] Port: [%u] not a valid port", HCOM_ERROR_CODE(HCCL_E_PARA), port);
537 0 : return HCCL_E_PARA;
538 : }
539 0 : return HCCL_SUCCESS;
540 : }
541 :
542 0 : HcclResult CheckRoleAndRankConsistent(const hccl::RoleTableInfo& roleTableInfo, const hccl::RankTable_t& rankTable)
543 : {
544 0 : u32 roleTableSize = roleTableInfo.servers.size() + roleTableInfo.clients.size();
545 0 : if (rankTable.rankNum < roleTableSize) {
546 0 : HCCL_ERROR(
547 : "[CheckRoleAndRankConsistent]errNo[0x%016llx] rank list size(%u): less than role size(%u)",
548 : HCOM_ERROR_CODE(HCCL_E_PARA), rankTable.rankNum, roleTableSize);
549 0 : return HCCL_E_PARA;
550 : }
551 :
552 0 : auto compareRoleAndRank = [&](RoleTableNodeInfo& role) -> HcclResult {
553 0 : bool isMatch = false;
554 0 : for (auto rank : rankTable.rankList) {
555 0 : if (rank.deviceInfo.devicePhyId == HOST_DEVICE_ID && role.ipAddr == rank.hostIp
556 0 : && role.port == rank.hostPort) {
557 0 : isMatch = true;
558 0 : break;
559 0 : } else if (
560 0 : rank.deviceInfo.devicePhyId != HOST_DEVICE_ID && role.ipAddr == rank.deviceInfo.deviceIp[0]
561 0 : && role.port == rank.deviceInfo.port) {
562 0 : isMatch = true;
563 0 : break;
564 : }
565 0 : }
566 0 : if (!isMatch) {
567 0 : HCCL_ERROR(
568 : "[CheckRoleAndRankConsistent]role node notequ rank, role.ipAddr[%s] role.port[%u]",
569 : role.ipAddr.GetReadableIP(), role.port);
570 0 : return HCCL_E_PARA;
571 : }
572 0 : return HCCL_SUCCESS;
573 0 : };
574 :
575 0 : for (auto role : roleTableInfo.servers) {
576 0 : CHK_RET(compareRoleAndRank(role));
577 0 : }
578 :
579 0 : for (auto role : roleTableInfo.clients) {
580 0 : CHK_RET(compareRoleAndRank(role));
581 0 : }
582 :
583 0 : return HCCL_SUCCESS;
584 : }
585 :
586 0 : HcclResult CfgGetRoleTableInfo(const std::string& rankTableM, RoleTableInfo& roleTableInfo)
587 : {
588 0 : TopoinfoRoletable myTopoRolektable(rankTableM);
589 0 : CHK_RET(myTopoRolektable.ParserRoleTable(roleTableInfo));
590 :
591 0 : return HCCL_SUCCESS;
592 0 : }
593 :
594 528 : void SetRetryEnable(
595 : DevType deviceType, const u32& superPodNum, const u32& serverNum, const u32& deviceNumPerAggregation,
596 : const bool& isDiffDeviceType, bool isAivMode, hccl::HcclIpAddress& serverIp, hccl::HcclIpAddress& localIp,
597 : bool& retryEnable, bool isInterServerRetry, bool isInterSuperPodRetry)
598 : {
599 528 : retryEnable = false;
600 528 : if (deviceType != DevType::DEV_TYPE_910_93 || isDiffDeviceType) {
601 489 : retryEnable = false;
602 39 : } else if (superPodNum > 1) { // L2重执行
603 12 : retryEnable = isInterSuperPodRetry || isInterServerRetry;
604 27 : } else if (serverNum > 1) { // L1重执行
605 16 : retryEnable = isInterServerRetry;
606 : }
607 :
608 528 : if (retryEnable && isAivMode) {
609 0 : retryEnable = false;
610 0 : HCCL_RUN_WARNING(
611 : "[Config][SetRetryEnable] AivMode[%d] and OpRetry configuration conflict. "
612 : "Enable Aiv, disable OpRetry",
613 : isAivMode);
614 : }
615 :
616 531 : if (retryEnable && (serverIp.IsInvalid() || localIp.IsInvalid())) {
617 0 : retryEnable = false;
618 0 : HCCL_RUN_WARNING(
619 : "[Config][SetRetryEnable] serverIp [%s] or localIp [%s] is invalid, disable OpRetry",
620 : serverIp.GetReadableIP(), localIp.GetReadableIP());
621 : }
622 :
623 531 : HCCL_INFO(
624 : "[Config][SetRetryEnable]deviceType[%d], superPodNum[%u], serverNum[%u], deviceNum[%u],"
625 : "isDiffDeviceType[%d], isAivMode[%d], retryEnable[%d].",
626 : deviceType, superPodNum, serverNum, deviceNumPerAggregation, isDiffDeviceType, isAivMode, retryEnable);
627 531 : }
628 :
629 291 : bool IsOneSidedIdentifier(const std::string& identifier)
630 : {
631 291 : return g_oneSidedIdentifierSet.find(identifier) != g_oneSidedIdentifierSet.end();
632 : }
633 :
634 0 : HcclResult AddOneSidedIdentifier(const std::string& identifier)
635 : {
636 0 : std::lock_guard<std::mutex> lock(g_oneSidedIdentifierMutex);
637 0 : g_oneSidedIdentifierSet.insert(identifier);
638 0 : return HCCL_SUCCESS;
639 0 : }
640 :
641 0 : HcclResult DeleteOneSidedIdentifier(const std::string& identifier)
642 : {
643 0 : std::lock_guard<std::mutex> lock(g_oneSidedIdentifierMutex);
644 0 : if (g_oneSidedIdentifierSet.find(identifier) == g_oneSidedIdentifierSet.end()) {
645 0 : HCCL_RUN_WARNING("[%s] identifier[%s] not found", __func__, identifier.c_str());
646 0 : return HCCL_E_NOT_FOUND;
647 : }
648 0 : g_oneSidedIdentifierSet.erase(identifier);
649 :
650 0 : return HCCL_SUCCESS;
651 0 : }
|
