Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "config.h"
12 : #include <arpa/inet.h>
13 : #include <cctype>
14 : #include <fcntl.h>
15 : #include <securec.h>
16 : #include <sys/socket.h>
17 : #include <sys/types.h>
18 : #include <unistd.h>
19 : #include<map>
20 : #include<set>
21 : #include "topoinfo_ranktableParser_pub.h"
22 : #include "./topo/topoinfo_ranktableStandard.h"
23 : #include "./topo/topoinfo_ranktableConcise.h"
24 : #include "./topo/topoinfo_ranktableHeterog.h"
25 : #include "./topo/topoinfo_roletableParser.h"
26 : #include "comm.h"
27 :
28 : using namespace std;
29 : using namespace hccl;
30 :
31 : std::set<std::string> g_oneSidedIdentifierSet;
32 : std::mutex g_oneSidedIdentifierMutex;
33 :
34 387 : HcclResult CfgGetClusterInfo(const std::string &rankTableM, const std::string &identify, hccl::HcclCommParams ¶ms,
35 : hccl::RankTable_t &rankTable, bool isInterSuperPodRetryEnable, DevType deviceType)
36 : {
37 387 : TopoInfoRanktableParser myTopoRanktable(rankTableM, identify);
38 387 : CHK_RET(myTopoRanktable.Init());
39 : // 获取rankTable版本
40 387 : CHK_RET(myTopoRanktable.GetRanktableVersion(rankTable.version));
41 : // 根据rankTable有没有版本信息属性和版本信息确定解析的方式
42 387 : std::unique_ptr<TopoInfoRanktableParser> pTopoRanktable = nullptr;
43 774 : if (rankTable.version.compare(HCCL_CLUSTER_VERSION) == 0 ||
44 387 : rankTable.version.compare(SUPERPOD_CLUSTER_VERSION) == 0) {
45 0 : pTopoRanktable.reset(new (std::nothrow) TopoinfoRanktableConcise(rankTableM, identify));
46 0 : pTopoRanktable->SetIsInterSuperPodRetryEnable(isInterSuperPodRetryEnable);
47 387 : } else if (rankTable.version.compare(HETEROG_CLUSTER_VERSION) == 0) {
48 0 : pTopoRanktable.reset(new (std::nothrow) TopoinfoRanktableHeterog(rankTableM, identify, deviceType));
49 387 : } else if (rankTable.version.compare("Standard") == 0) {
50 387 : pTopoRanktable.reset(new (std::nothrow) TopoinfoRanktableStandard(rankTableM, identify));
51 : } else {
52 0 : RPT_INPUT_ERR(true, "EI0014", std::vector<std::string>({ "value", "variable" ,"expect" }),
53 : std::vector<std::string>({rankTable.version, "version", "a valid version number." }));
54 0 : HCCL_ERROR("[%s][%s]version[%s] is not support",
55 : LOG_KEYWORDS_INIT_GROUP.c_str(), LOG_KEYWORDS_RANKTABLE_CHECK.c_str(),rankTable.version.c_str());
56 0 : return HCCL_E_NOT_SUPPORT;
57 : }
58 : // 检查指针是否为空
59 387 : CHK_SMART_PTR_NULL(pTopoRanktable);
60 : // 执行初始化，加载rankTable并进行解析
61 387 : CHK_RET(pTopoRanktable->Init());
62 : // 将解析到的内容保存到入参hcomInfo中
63 386 : HcclResult ret = pTopoRanktable->GetClusterInfo(params, rankTable);
64 386 : CHK_PRT_RET(ret != HCCL_SUCCESS, HCCL_ERROR("[Get][ClusterInfo]identify[%s],get cluterInfo info error",
65 : identify.c_str()), ret);
66 :
67 386 : CHK_PRT_RET((rankTable.serverNum == 0), HCCL_ERROR("[Get][ClusterInfo]serverNum is zero."), HCCL_E_PARA);
68 386 : CHK_RET(CheckRankListInfo(rankTable.rankList));
69 :
70 386 : if (rankTable.serverNum > 1) {
71 13 : CHK_RET(CheckRankIpFamily(rankTable.rankList));
72 : }
73 386 : if (rankTable.version.compare(HETEROG_CLUSTER_VERSION) == 0) {
74 : // 异构场景无需检查
75 0 : return HCCL_SUCCESS;
76 : } else {
77 386 : CHK_RET(CheckRankListBaseInfo(rankTable.deviceNum, rankTable.serverNum));
78 386 : CHK_RET(CheckDeviceNumValid(rankTable.rankList, rankTable.deviceNum,
79 : rankTable.serverNum, rankTable.version));
80 : }
81 386 : return HCCL_SUCCESS;
82 387 : }
83 :
84 0 : HcclResult CfgGetClusterInfoWithoutDev(const std::string &rankTableM, const std::string &identify,
85 : hccl::HcclCommParams ¶ms, hccl::RankTable_t &rankTable, bool isInterSuperPodRetryEnable)
86 : {
87 0 : TopoInfoRanktableParser myTopoRanktable(rankTableM, identify);
88 0 : CHK_RET(myTopoRanktable.Init());
89 : // 获取rankTable版本
90 0 : CHK_RET(myTopoRanktable.GetRanktableVersion(rankTable.version));
91 : // 根据rankTable有没有版本信息属性和版本信息确定解析的方式
92 0 : std::unique_ptr<TopoInfoRanktableParser> pTopoRanktable = nullptr;
93 0 : if (rankTable.version.compare(HCCL_CLUSTER_VERSION) == 0) {
94 0 : pTopoRanktable.reset(new (std::nothrow) TopoinfoRanktableConcise(rankTableM, identify));
95 0 : pTopoRanktable->SetIsInterSuperPodRetryEnable(isInterSuperPodRetryEnable);
96 0 : } else if (rankTable.version.compare(HETEROG_CLUSTER_VERSION) == 0) {
97 0 : pTopoRanktable.reset(new (std::nothrow) TopoinfoRanktableHeterog(rankTableM, identify));
98 0 : } else if (rankTable.version.compare("Standard") == 0) {
99 0 : pTopoRanktable.reset(new (std::nothrow) TopoinfoRanktableStandard(rankTableM, identify));
100 : } else {
101 0 : HCCL_ERROR("[Get][RanktableVersion]version[%s] is not support", rankTable.version.c_str());
102 0 : return HCCL_E_NOT_SUPPORT;
103 : }
104 : // 检查指针是否为空
105 0 : CHK_SMART_PTR_NULL(pTopoRanktable);
106 : // 执行初始化，加载rankTable并进行解析
107 0 : CHK_RET(pTopoRanktable->Init());
108 : // 将解析到的内容保存到入参params、rankTable中
109 0 : HcclResult ret = pTopoRanktable->GetClusterInfo(params, rankTable);
110 0 : CHK_PRT_RET(ret != HCCL_SUCCESS, HCCL_ERROR("[Get][ClusterInfo]identify[%s],get cluterInfo info error",
111 : identify.c_str()), ret);
112 :
113 0 : CHK_RET(CheckRankListInfo(rankTable.rankList));
114 0 : CHK_RET(CheckDeviceNumValid(rankTable.rankList, rankTable.deviceNum,
115 : rankTable.serverNum, rankTable.version));
116 0 : return HCCL_SUCCESS;
117 0 : }
118 :
119 386 : HcclResult CheckRankId(const char *rankId)
120 : {
121 386 : CHK_PTR_NULL(rankId);
122 386 : string temp = rankId;
123 :
124 772 : for (u32 index = 0; index < temp.length(); index++) {
125 386 : if (!isdigit(temp[index])) {
126 0 : HCCL_ERROR("[Check][RankId]errNo[0x%016llx] check rankid is not digit", HCOM_ERROR_CODE(HCCL_E_PARA));
127 0 : return HCCL_E_PARA;
128 : }
129 : }
130 386 : return HCCL_SUCCESS;
131 386 : }
132 :
133 0 : HcclResult CheckRankTableConfigInfo(const std::vector<RankInfo_t> &rankList, u32 deviceNum, u32 serverNum)
134 : {
135 0 : if (rankList.size() != deviceNum) {
136 0 : HCCL_ERROR("[Check][RankTableConfigInfo]errNo[0x%016llx] rankList size[%llu] neq deviceNum[%u]",
137 : HCOM_ERROR_CODE(HCCL_E_PARA), rankList.size(), deviceNum);
138 0 : return HCCL_E_PARA;
139 : }
140 0 : CHK_RET(CheckGroupRankList(rankList, deviceNum, serverNum));
141 0 : return HCCL_SUCCESS;
142 : }
143 :
144 233 : HcclResult ShowRanktableConfigInfo(const bool cloudFlag, hccl::HcclCommParams ¶ms,
145 : hccl::RankTable_t &rankTable)
146 : {
147 233 : if (cloudFlag) {
148 0 : CHK_RET(DisplayCloudRankTableInfo(params, rankTable));
149 : } else {
150 233 : CHK_RET(DisplayRanktableInfo(params, rankTable));
151 : }
152 233 : return HCCL_SUCCESS;
153 : }
154 :
155 0 : HcclResult DisplayCloudRankTableInfo(hccl::HcclCommParams ¶ms, hccl::RankTable_t &rankTable)
156 : {
157 0 : HCCL_DEBUG(
158 : "rank_table: \"Unique groupNum\":\"%u\", \"Unique deviceNum\":\"%u\", \"Unique serverNum\":\"%u\"",
159 : rankTable.groupNum, rankTable.deviceNum, rankTable.serverNum);
160 :
161 0 : HCCL_DEBUG("params: \"uniqueID\":\"%s\"", params.id.internal);
162 0 : return HCCL_SUCCESS;
163 : }
164 :
165 233 : HcclResult DisplayRanktableInfo(hccl::HcclCommParams ¶ms, hccl::RankTable_t &rankTable)
166 : {
167 233 : std::string nicName = "\"para_plane_nic_name\":[";
168 :
169 233 : for (u32 i = 0; i < rankTable.nicNames.size(); i++) {
170 0 : if (i != 0) {
171 0 : nicName += ",";
172 : }
173 0 : std::string tmpString = rankTable.nicNames[i].c_str();
174 0 : nicName += "\"";
175 0 : }
176 233 : nicName += "],";
177 233 : HCCL_DEBUG(
178 : "rank_table: \"Unique deviceNum\":\"%u\", \"Unique serverNum\":\"%u\", \"para_plane_nic_location\""\
179 : ":\"%u\", \"para_plane_nic_num\":\"%u\";%s",
180 : rankTable.deviceNum, rankTable.serverNum, rankTable.nicDeploy,
181 : rankTable.nicNum, nicName.c_str());
182 :
183 233 : HCCL_DEBUG("params: \"uniqueID\":\"%s\"", params.id.internal);
184 233 : return HCCL_SUCCESS;
185 233 : }
186 1 : HcclResult DisplayRanktableInfo(const hccl::RankTable_t &rankTable)
187 : {
188 1 : std::string nicName = "\"para_plane_nic_name\":[";
189 :
190 1 : for (u32 i = 0; i < rankTable.nicNames.size(); i++) {
191 0 : if (i != 0) {
192 0 : nicName += ",";
193 : }
194 0 : nicName += rankTable.nicNames[i];
195 : }
196 1 : nicName += "]";
197 :
198 1 : HCCL_INFO(
199 : "rank_table: \"Unique deviceNum\":\"%u\", \"Unique serverNum\":\"%u\", \"para_plane_nic_location\""\
200 : ":\"%u\", \"para_plane_nic_num\":\"%u\";%s",
201 : rankTable.deviceNum, rankTable.serverNum, rankTable.nicDeploy,
202 : rankTable.nicNum, nicName.c_str());
203 :
204 1 : std::string deviceInfoStr = "\"device_information\":";
205 2 : for (u32 i = 0; i < rankTable.rankList.size(); i++) {
206 1 : deviceInfoStr += "{rankID[" + to_string(rankTable.rankList[i].rankId) + "],";
207 1 : deviceInfoStr += "serverId[" + rankTable.rankList[i].serverId + "],";
208 1 : deviceInfoStr += "deviceId[" + to_string(rankTable.rankList[i].deviceInfo.devicePhyId) + "],";
209 1 : deviceInfoStr += "deviceType[" + to_string(u32(rankTable.rankList[i].deviceInfo.deviceType)) + "],";
210 1 : string tmpString = rankTable.rankList[i].deviceInfo.deviceIp[0].GetReadableAddress();
211 1 : deviceInfoStr += "deviceIp[" + tmpString + "],";
212 1 : tmpString = rankTable.rankList[i].hostIp.GetReadableAddress();
213 1 : deviceInfoStr += "hostIp[" + tmpString + "]},";
214 1 : HCCL_INFO("%s", deviceInfoStr.c_str());
215 1 : deviceInfoStr.clear();
216 1 : }
217 :
218 1 : return HCCL_SUCCESS;
219 1 : }
220 :
221 1189 : HcclResult GetDevNum(const std::vector<RankInfo_t> &rankList, u32 &devNum)
222 : {
223 1189 : devNum = 0;
224 3904 : for (auto &iter : rankList) {
225 2715 : if (iter.deviceInfo.devicePhyId != HOST_DEVICE_ID) {
226 2715 : devNum++;
227 : }
228 : }
229 1183 : return HCCL_SUCCESS;
230 : }
231 :
232 33 : HcclResult GetServerNum(const std::vector<RankInfo> &rankList, u32 &serverNum)
233 : {
234 33 : serverNum = 0;
235 33 : std::set<u32> serverSet;
236 190 : for (auto &iter : rankList) {
237 157 : serverSet.insert(iter.serverIdx);
238 : }
239 33 : serverNum = serverSet.size();
240 33 : return HCCL_SUCCESS;
241 33 : }
242 :
243 33 : HcclResult GetDevNum(const std::vector<RankInfo> &rankList, u32 &devNum)
244 : {
245 33 : devNum = 0;
246 190 : for (auto &iter : rankList) {
247 157 : if (iter.devicePhyId != HOST_DEVICE_ID) {
248 157 : devNum++;
249 : }
250 : }
251 33 : return HCCL_SUCCESS;
252 : }
253 :
254 : template <typename rankTable>
255 33 : HcclResult GetSuperPodNums(const std::vector<rankTable> &rankList, u32 &superPodNum)
256 : {
257 33 : superPodNum = 0;
258 33 : std::set<std::string> superPodIds;
259 :
260 33 : for (u32 index = 0; index < rankList.size(); index++) {
261 : // superPodId为空时, 返回超节点数量为0, 按照非超节点模式处理
262 33 : CHK_PRT_RET(rankList[index].superPodId.empty(),
263 : HCCL_DEBUG("ranks[%u] superPodId[%s] is empty, set superPodNum to zero", index,
264 : rankList[index].superPodId.c_str()),
265 : HCCL_SUCCESS);
266 :
267 0 : if (superPodIds.find(rankList[index].superPodId) == superPodIds.end()) {
268 0 : superPodIds.insert(rankList[index].superPodId);
269 : }
270 : }
271 0 : superPodNum = superPodIds.size();
272 0 : return HCCL_SUCCESS;
273 33 : }
274 :
275 0 : HcclResult GetSuperPodNum(const std::vector<RankInfo_t> &rankList, u32 &superPodNum)
276 : {
277 0 : (void)GetSuperPodNums(rankList, superPodNum);
278 0 : return HCCL_SUCCESS;
279 : }
280 :
281 33 : HcclResult GetSuperPodNum(const std::vector<RankInfo> &rankList, u32 &superPodNum)
282 : {
283 33 : (void)GetSuperPodNums(rankList, superPodNum);
284 33 : return HCCL_SUCCESS;
285 : }
286 :
287 0 : HcclResult CheckGroupRankList(const std::vector<RankInfo_t> &rankList, u32 deviceNum, u32 serverNum)
288 : {
289 0 : u32 realDevNum = 0;
290 0 : CHK_RET(GetDevNum(rankList, realDevNum));
291 0 : CHK_RET(CheckAverageDev(realDevNum, serverNum));
292 0 : CHK_RET(CheckRankListInfo(rankList, realDevNum, serverNum));
293 0 : return HCCL_SUCCESS;
294 : }
295 :
296 0 : HcclResult CheckDeviceId(const std::vector<RankInfo_t> &rankList, u32 deviceNum, u32 serverNum)
297 : // each server should has same device Id may not be continuous
298 : {
299 0 : if (serverNum == 0) {
300 0 : HCCL_ERROR("[Check][DeviceId]errNo[0x%016llx] server num is zero", HCOM_ERROR_CODE(HCCL_E_PARA));
301 0 : return HCCL_E_PARA;
302 : }
303 :
304 : (void)deviceNum;
305 0 : std::map<std::string, std::set<s32> > serverDeviceMapList;
306 0 : for (auto it = rankList.begin(); it != rankList.end(); it++) {
307 0 : if (it->deviceInfo.devicePhyId == HOST_DEVICE_ID) {
308 0 : continue;
309 : }
310 0 : std::string tmpServerId = it->serverId;
311 0 : auto search = serverDeviceMapList.find(tmpServerId);
312 0 : if (search != serverDeviceMapList.end()) {
313 0 : auto rs = serverDeviceMapList[tmpServerId].insert(it->deviceInfo.devicePhyId);
314 0 : if (!rs.second) {
315 0 : RPT_INPUT_ERR(true, "EI0014", std::vector<std::string>({ "value", "variable" ,"expect" }),
316 : std::vector<std::string>({ std::to_string(it->deviceInfo.devicePhyId),
317 : + " \"Device Id of server Id " + tmpServerId + " \" ", "is unique" }));
318 0 : HCCL_ERROR("[%s][%s]errNo[0x%016llx] check ranklist[%u], device id repeat for one server",
319 : LOG_KEYWORDS_INIT_GROUP.c_str(), LOG_KEYWORDS_RANKTABLE_CHECK.c_str(),
320 : HCOM_ERROR_CODE(HCCL_E_PARA), it->rankId);
321 0 : return HCCL_E_PARA;
322 : }
323 : } else {
324 0 : std::set<s32> deviceSet;
325 0 : deviceSet.insert(it->deviceInfo.devicePhyId);
326 0 : serverDeviceMapList.insert(std::pair<std::string, std::set<s32> >(tmpServerId, deviceSet));
327 0 : }
328 0 : }
329 0 : if (serverDeviceMapList.size() == 0) {
330 0 : HCCL_ERROR("[Check][DeviceId]errNo[0x%016llx] for all ranklist, server num is zero",
331 : HCOM_ERROR_CODE(HCCL_E_PARA));
332 0 : return HCCL_E_PARA;
333 : }
334 0 : return HCCL_SUCCESS;
335 0 : }
336 :
337 : // 检查deviceNum、serverNum范围，rankList中rank id范围及是否升序连续分布
338 386 : HcclResult CheckRankListBaseInfo(u32 deviceNum, u32 serverNum)
339 : {
340 386 : HCCL_INFO("START CheckRankListBaseInfo");
341 386 : if (deviceNum == 0) {
342 0 : HCCL_ERROR("[Check][RankListBaseInfo]errNo[0x%016llx] device num is zero", HCOM_ERROR_CODE(HCCL_E_PARA));
343 0 : return HCCL_E_PARA;
344 : }
345 386 : if (serverNum == 0) {
346 0 : HCCL_ERROR("[Check][RankListBaseInfo]errNo[0x%016llx] server num is zero", HCOM_ERROR_CODE(HCCL_E_PARA));
347 0 : return HCCL_E_PARA;
348 : }
349 386 : return HCCL_SUCCESS;
350 : }
351 :
352 386 : HcclResult CheckRankListInfo(const std::vector<RankInfo_t> &rankList)
353 : {
354 : // rankList have been sorted when parsing rank table,
355 : // check the continuity of sorted rankList
356 386 : HCCL_INFO("START CheckRankListInfo");
357 1522 : for (u32 index = 0; index < rankList.size(); index++) {
358 1136 : if (rankList[index].rankId != index) {
359 0 : RPT_INPUT_ERR(true, "EI0014", std::vector<std::string>({ "value", "variable" ,"expect" }),
360 : std::vector<std::string>({ std::to_string(rankList[index].rankId), "rank_id",
361 : "is less than the communication size " + std::to_string(rankList.size()) + " and must be unique." }));
362 0 : HCCL_ERROR("[%s][%s]errNo[0x%016llx] rankList[%u] rankId[%u] error",
363 : LOG_KEYWORDS_INIT_GROUP.c_str(), LOG_KEYWORDS_RANKTABLE_CHECK.c_str(), HCOM_ERROR_CODE(HCCL_E_PARA),
364 : index, rankList[index].rankId);
365 0 : return HCCL_E_PARA;
366 : }
367 : }
368 386 : return HCCL_SUCCESS;
369 0 : }
370 :
371 : // 校验rank ip family一致性
372 21 : HcclResult CheckRankIpFamily(const std::vector<RankInfo_t> &rankList)
373 : {
374 21 : HCCL_INFO("START CheckRankIpFamily");
375 21 : s32 hostFamily = 0;
376 21 : s32 deviceFamily = 0;
377 21 : std::string errormessage = "";
378 133 : for (u32 index = 0; index < rankList.size(); index++) {
379 116 : if (!rankList[index].hostIp.IsInvalid()) {
380 12 : if ((rankList[index].hostIp.GetFamily() != AF_INET) && (rankList[index].hostIp.GetFamily() != AF_INET6)) {
381 15 : RPT_INPUT_ERR(true,
382 : "EI0014",
383 : std::vector<std::string>({ "value", "variable" ,"expect" }),
384 : std::vector<std::string>({std::to_string(rankList[index].hostIp.GetFamily()),
385 : " \"Device Id of server Id " + rankList[index].serverId + "\" ", "is unique"}));
386 2 : errormessage = "Value " + std::to_string(rankList[index].hostIp.GetFamily()) + " for rankTable variable \"Device Id of "\
387 2 : "server Id " + rankList[index].serverId + "\" is invalid, expected value is unique.";
388 1 : HCCL_ERROR("[%s][%s] %s",
389 : LOG_KEYWORDS_INIT_GROUP.c_str(),
390 : LOG_KEYWORDS_RANKTABLE_CHECK.c_str(),
391 : errormessage.c_str());
392 1 : return HCCL_E_PARA;
393 : }
394 :
395 11 : if (hostFamily != 0 && hostFamily != rankList[index].hostIp.GetFamily()) {
396 15 : RPT_INPUT_ERR(true,
397 : "EI0014",
398 : std::vector<std::string>({ "value", "variable" ,"expect" }),
399 : std::vector<std::string>({std::to_string(rankList[index].hostIp.GetFamily()),
400 : " \"Device Id of server Id " + rankList[index].serverId + "\" ", "is unique"}));
401 2 : errormessage = "Value " + std::to_string(rankList[index].hostIp.GetFamily()) + " for rankTable variable \"Device Id of "\
402 2 : "server Id " + rankList[index].serverId + "\" is invalid, expected value is unique.";
403 1 : HCCL_ERROR("[%s][%s]%s",
404 : LOG_KEYWORDS_INIT_GROUP.c_str(),
405 : LOG_KEYWORDS_RANKTABLE_CHECK.c_str(),
406 : errormessage.c_str());
407 1 : return HCCL_E_PARA;
408 : }
409 :
410 10 : hostFamily = rankList[index].hostIp.GetFamily();
411 : }
412 :
413 : // device ip不存在时, 无需校验
414 330 : if (rankList[index].deviceInfo.deviceIp.empty() ||
415 216 : ((rankList[index].deviceInfo.deviceIp.size() == 1) && rankList[index].deviceInfo.deviceIp[0].IsInvalid())) {
416 7 : continue;
417 : }
418 :
419 212 : for (auto &iter : rankList[index].deviceInfo.deviceIp) {
420 107 : if ((iter.GetFamily() != AF_INET) && (iter.GetFamily() != AF_INET6)) {
421 15 : RPT_INPUT_ERR(true,
422 : "EI0014",
423 : std::vector<std::string>({ "value", "variable" ,"expect" }),
424 : std::vector<std::string>({std::to_string(rankList[index].hostIp.GetFamily()), " \"Device Id of "\
425 : "server Id " + rankList[index].serverId + " \" ", "is unique"}));
426 2 : errormessage = "Value " + std::to_string(rankList[index].hostIp.GetFamily()) + " for rankTable variable \"Device Id of "\
427 2 : "server Id " + rankList[index].serverId + " \" is invalid, expected value is unique.";
428 1 : HCCL_ERROR("[%s][%s]%s",
429 : LOG_KEYWORDS_INIT_GROUP.c_str(),
430 : LOG_KEYWORDS_RANKTABLE_CHECK.c_str(),
431 : errormessage.c_str());
432 2 : return HCCL_E_PARA;
433 : }
434 106 : if (deviceFamily != 0 && deviceFamily != iter.GetFamily()) {
435 14 : RPT_ENV_ERR(true, "EI0001", std::vector<std::string>({"value", "env", "expect"}),
436 : std::vector<std::string>({std::to_string(iter.GetFamily()), "RankIpFamily", std::to_string(deviceFamily)}));
437 1 : CHK_PRT_RET(true,
438 : HCCL_ERROR("[%s][%s]rank[%u] device ip family[%d] is not same as others[%d].",
439 : LOG_KEYWORDS_INIT_GROUP.c_str(),
440 : LOG_KEYWORDS_ENV_CONFIG.c_str(),
441 : rankList[index].rankId,
442 : iter.GetFamily(),
443 : deviceFamily),
444 : HCCL_E_PARA);
445 : }
446 105 : deviceFamily = iter.GetFamily();
447 : }
448 : }
449 17 : return HCCL_SUCCESS;
450 32 : }
451 :
452 0 : HcclResult CheckRankListInfo(const std::vector<RankInfo_t> &rankList, u32 deviceNum, u32 serverNum)
453 : {
454 0 : CHK_RET(CheckRankListBaseInfo(deviceNum, serverNum));
455 0 : CHK_RET(CheckRankListInfo(rankList));
456 0 : CHK_RET(CheckDeviceNumValid(rankList, deviceNum, serverNum));
457 :
458 : // 校验每个serverID下的deviceID是否都在同一范围
459 0 : CHK_RET(CheckDeviceId(rankList, deviceNum, serverNum));
460 :
461 0 : return HCCL_SUCCESS;
462 : }
463 :
464 : // 检查rank list中每个server id下的device数是否相同
465 386 : HcclResult CheckDeviceNumValid(const std::vector<RankInfo_t> &rankList, u32 deviceNum,
466 : u32 serverNum, std::string version)
467 : {
468 386 : if (serverNum == 0) {
469 0 : HCCL_ERROR("[Check][DeviceNumValid]errNo[0x%016llx] server num is zero", HCOM_ERROR_CODE(HCCL_E_PARA));
470 0 : return HCCL_E_PARA;
471 : }
472 :
473 386 : if (version.compare(HETEROG_CLUSTER_VERSION) != 0) {
474 : DevType deviceType;
475 524 : CHK_RET(hrtGetDeviceType(deviceType));
476 : // 不对910B进行Server间卡数一致性的校验
477 386 : if (deviceType == DevType::DEV_TYPE_910B || deviceType == DevType::DEV_TYPE_910_93) {
478 138 : return HCCL_SUCCESS;
479 : }
480 : }
481 :
482 248 : std::map<std::string, u32> serverDeviceNumMapList;
483 687 : for (auto it = rankList.begin(); it != rankList.end(); it++) {
484 439 : if (it->deviceInfo.devicePhyId == HOST_DEVICE_ID) {
485 0 : continue;
486 : }
487 439 : std::string curServerId = it->serverId;
488 439 : auto search = serverDeviceNumMapList.find(curServerId);
489 438 : if (search != serverDeviceNumMapList.end()) {
490 191 : serverDeviceNumMapList[curServerId] = serverDeviceNumMapList[curServerId] + 1;
491 : } else {
492 248 : serverDeviceNumMapList.insert(std::pair<std::string, u32>(curServerId, 1));
493 : }
494 439 : }
495 496 : for (auto it = serverDeviceNumMapList.begin(); it != serverDeviceNumMapList.end(); it++) {
496 248 : if (it->second != (deviceNum / serverNum)) {
497 0 : RPT_INPUT_ERR(true,
498 : "EI0014", std::vector<std::string>({ "value", "variable" ,"expect" }), std::vector<std::string>({ std::to_string(it->second), "\"DeviceNum of "\
499 : "server Id " + it->first + "\" ", "is that number of devices under each server node is the same" }));
500 0 : HCCL_ERROR("[%s][%s]errNo[0x%016llx] devices num of each server error",
501 : LOG_KEYWORDS_INIT_GROUP.c_str(), LOG_KEYWORDS_RANKTABLE_CHECK.c_str(), HCOM_ERROR_CODE(HCCL_E_PARA));
502 0 : return HCCL_E_PARA;
503 : }
504 : }
505 248 : return HCCL_SUCCESS;
506 248 : }
507 :
508 0 : HcclResult CheckPortValid(u32 port)
509 : {
510 0 : if (port > PORT_MAX) {
511 0 : HCCL_ERROR("[Check][PortValid]errNo[0x%016llx] Port: [%u] not a valid port",
512 : HCOM_ERROR_CODE(HCCL_E_PARA), port);
513 0 : return HCCL_E_PARA;
514 : }
515 0 : return HCCL_SUCCESS;
516 : }
517 :
518 0 : HcclResult CheckRoleAndRankConsistent(const hccl::RoleTableInfo &roleTableInfo,
519 : const hccl::RankTable_t &rankTable)
520 : {
521 0 : u32 roleTableSize = roleTableInfo.servers.size() + roleTableInfo.clients.size();
522 0 : if (rankTable.rankNum < roleTableSize) {
523 0 : HCCL_ERROR("[CheckRoleAndRankConsistent]errNo[0x%016llx] rank list size(%u): less than role size(%u)",
524 : HCOM_ERROR_CODE(HCCL_E_PARA), rankTable.rankNum, roleTableSize);
525 0 : return HCCL_E_PARA;
526 : }
527 :
528 0 : auto compareRoleAndRank = [&](RoleTableNodeInfo &role) -> HcclResult {
529 0 : bool isMatch = false;
530 0 : for (auto rank : rankTable.rankList) {
531 0 : if (rank.deviceInfo.devicePhyId == HOST_DEVICE_ID &&
532 0 : role.ipAddr == rank.hostIp && role.port == rank.hostPort) {
533 0 : isMatch = true;
534 0 : break;
535 0 : } else if (rank.deviceInfo.devicePhyId != HOST_DEVICE_ID && role.ipAddr == rank.deviceInfo.deviceIp[0] &&
536 0 : role.port == rank.deviceInfo.port) {
537 0 : isMatch = true;
538 0 : break;
539 : }
540 0 : }
541 0 : if (!isMatch) {
542 0 : HCCL_ERROR("[CheckRoleAndRankConsistent]role node notequ rank, role.ipAddr[%s] role.port[%u]",
543 : role.ipAddr.GetReadableIP(), role.port);
544 0 : return HCCL_E_PARA;
545 : }
546 0 : return HCCL_SUCCESS;
547 0 : };
548 :
549 0 : for (auto role : roleTableInfo.servers) {
550 0 : CHK_RET(compareRoleAndRank(role));
551 0 : }
552 :
553 0 : for (auto role : roleTableInfo.clients) {
554 0 : CHK_RET(compareRoleAndRank(role));
555 0 : }
556 :
557 0 : return HCCL_SUCCESS;
558 : }
559 :
560 0 : HcclResult CfgGetRoleTableInfo(const std::string &rankTableM, RoleTableInfo &roleTableInfo)
561 : {
562 0 : TopoinfoRoletable myTopoRolektable(rankTableM);
563 0 : CHK_RET(myTopoRolektable.ParserRoleTable(roleTableInfo));
564 :
565 0 : return HCCL_SUCCESS;
566 0 : }
567 :
568 528 : void SetRetryEnable(DevType deviceType, const u32 &superPodNum, const u32 &serverNum,
569 : const u32 &deviceNumPerAggregation, const bool &isDiffDeviceType, bool isAivMode,
570 : hccl::HcclIpAddress &serverIp, hccl::HcclIpAddress &localIp, bool &retryEnable,
571 : bool isInterServerRetry, bool isInterSuperPodRetry)
572 : {
573 528 : retryEnable = false;
574 528 : if (deviceType != DevType::DEV_TYPE_910_93 || isDiffDeviceType) {
575 489 : retryEnable = false;
576 39 : } else if (superPodNum > 1) { // L2重执行
577 12 : retryEnable = isInterSuperPodRetry || isInterServerRetry;
578 27 : } else if (serverNum > 1) { // L1重执行
579 16 : retryEnable = isInterServerRetry;
580 : }
581 :
582 528 : if (retryEnable && isAivMode) {
583 0 : retryEnable = false;
584 0 : HCCL_RUN_WARNING("[Config][SetRetryEnable] AivMode[%d] and OpRetry configuration conflict. "
585 : "Enable Aiv, disable OpRetry", isAivMode);
586 : }
587 :
588 527 : if (retryEnable && (serverIp.IsInvalid() || localIp.IsInvalid())) {
589 0 : retryEnable = false;
590 0 : HCCL_RUN_WARNING("[Config][SetRetryEnable] serverIp [%s] or localIp [%s] is invalid, disable OpRetry",
591 : serverIp.GetReadableIP(), localIp.GetReadableIP());
592 : }
593 :
594 527 : HCCL_INFO("[Config][SetRetryEnable]deviceType[%d], superPodNum[%u], serverNum[%u], deviceNum[%u],"
595 : "isDiffDeviceType[%d], isAivMode[%d], retryEnable[%d].",
596 : deviceType, superPodNum, serverNum, deviceNumPerAggregation, isDiffDeviceType, isAivMode, retryEnable);
597 528 : }
598 :
599 287 : bool IsOneSidedIdentifier(const std::string &identifier)
600 : {
601 287 : return g_oneSidedIdentifierSet.find(identifier) != g_oneSidedIdentifierSet.end();
602 : }
603 :
604 :
605 0 : HcclResult AddOneSidedIdentifier(const std::string &identifier)
606 : {
607 0 : std::lock_guard<std::mutex> lock(g_oneSidedIdentifierMutex);
608 0 : g_oneSidedIdentifierSet.insert(identifier);
609 0 : return HCCL_SUCCESS;
610 0 : }
611 :
612 0 : HcclResult DeleteOneSidedIdentifier(const std::string &identifier)
613 : {
614 0 : std::lock_guard<std::mutex> lock(g_oneSidedIdentifierMutex);
615 0 : if (g_oneSidedIdentifierSet.find(identifier) == g_oneSidedIdentifierSet.end()) {
616 0 : HCCL_RUN_WARNING("[%s] identifier[%s] not found", __func__, identifier.c_str());
617 0 : return HCCL_E_NOT_FOUND;
618 : }
619 0 : g_oneSidedIdentifierSet.erase(identifier);
620 :
621 0 : return HCCL_SUCCESS;
622 0 : }
|
