Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "topoinfo_ranktableConcise.h"
12 : #include <string>
13 : #include <fstream>
14 : #include <algorithm>
15 : #include <map>
16 : #include <vector>
17 : #include <unistd.h>
18 : #include <chrono>
19 : #include <iostream>
20 : #include <arpa/inet.h>
21 :
22 : #include "externalinput.h"
23 : #include "log.h"
24 : // ltm指定config路径
25 : #include "common/src/config.h"
26 : #include "workflow_pub.h"
27 : #include "device_capacity.h"
28 :
29 : using namespace std;
30 : using namespace hccl;
31 :
32 : constexpr u32 MAX_PORT_NUMBER = 65535;
33 : constexpr u32 HCCL_SOCKET_PORT_RANGE_AUTO = 0;
34 : constexpr u32 HCCL_DEVICE_PORT_DEFAULT = 16666;
35 : constexpr u32 HCCL_BACKUP_DEVICE_PORT_DEFAULT = 16667;
36 :
37 5 : TopoinfoRanktableConcise::TopoinfoRanktableConcise(const std::string &rankTableM, const std::string &identify)
38 : : TopoInfoRanktableParser(rankTableM, identify),
39 5 : isInterSuperPodRetryEnable_(GetExternalInputInterSuperPodRetryEnable())
40 : {
41 5 : }
42 :
43 5 : TopoinfoRanktableConcise::~TopoinfoRanktableConcise()
44 : {
45 5 : devIp2ObjIndex_.clear();
46 5 : }
47 :
48 0 : HcclResult TopoinfoRanktableConcise::Init()
49 : {
50 0 : CHK_RET(LoadRankTableString(rankTableFile_));
51 0 : CHK_RET(ParserClusterInfo(params_, rankTable_));
52 0 : return HCCL_SUCCESS;
53 : }
54 :
55 0 : HcclResult TopoinfoRanktableConcise::GetClusterInfo(RankTable_t &clusterInfo)
56 : {
57 0 : clusterInfo.nicDeploy = rankTable_.nicDeploy;
58 0 : clusterInfo.deviceNum = rankTable_.deviceNum;
59 0 : clusterInfo.serverNum = rankTable_.serverNum;
60 0 : clusterInfo.superPodNum = rankTable_.superPodNum;
61 0 : clusterInfo.rankNum = rankTable_.rankNum;
62 0 : clusterInfo.rankList = rankTable_.rankList;
63 0 : clusterInfo.serverList = rankTable_.serverList;
64 0 : return HCCL_SUCCESS;
65 : }
66 :
67 0 : HcclResult TopoinfoRanktableConcise::GetSelfClusterInfo(HcclCommParams ¶ms)
68 : {
69 : // 获取芯片类型信息
70 0 : params.deviceType = params_.deviceType;
71 0 : params.rank = params_.rank;
72 0 : params.userRank = params_.rank;
73 0 : params.logicDevId = params_.logicDevId;
74 0 : params.totalRanks = params_.totalRanks;
75 0 : params.serverId = params_.serverId;
76 0 : params.commPortConfig.devPortSwitchOn = params_.commPortConfig.devPortSwitchOn;
77 0 : return HCCL_SUCCESS;
78 : }
79 :
80 0 : HcclResult TopoinfoRanktableConcise::GetClusterInfo(hccl::HcclCommParams ¶ms,
81 : hccl::RankTable_t &rankTable)
82 : {
83 0 : CHK_RET(GetClusterInfo(rankTable));
84 0 : CHK_RET(GetSelfClusterInfo(params));
85 0 : return HCCL_SUCCESS;
86 : }
87 :
88 0 : HcclResult TopoinfoRanktableConcise::SetIsInterSuperPodRetryEnable(bool isRetryEnable)
89 : {
90 0 : isInterSuperPodRetryEnable_ = isRetryEnable;
91 0 : return HCCL_SUCCESS;
92 : }
93 :
94 0 : void TopoinfoRanktableConcise::DetectNicDepoly(RankTable_t &rankTable)
95 : {
96 : // 只有当hostIp有效而且deviceIp无效时，才使用HOST侧网卡部署，目前策略要求集群中所有卡的deploy
97 : // 形式一致，所以取ranklist[0]的方式即可
98 0 : auto isIpInvalid = [](const std::vector<HcclIpAddress> &deviceIp) -> bool {
99 0 : for (auto &ip : deviceIp) {
100 0 : if (!ip.IsInvalid()) { // 遍历vector中所有的IP，只要有任意一个IP有效就认为是有效的，那么返回false
101 0 : return false;
102 : }
103 : }
104 0 : return true;
105 : };
106 :
107 0 : rankTable.nicDeploy = (!rankTable.rankList.empty() &&
108 0 : !rankTable.rankList[0].hostIp.IsInvalid() && isIpInvalid(rankTable.rankList[0].deviceInfo.deviceIp)
109 0 : ) ? NICDeployment::NIC_DEPLOYMENT_HOST:
110 : NICDeployment::NIC_DEPLOYMENT_DEVICE;
111 0 : }
112 :
113 0 : HcclResult TopoinfoRanktableConcise::ParserClusterInfo(hccl::HcclCommParams ¶ms, hccl::RankTable_t &rankTable)
114 : {
115 0 : if (!IsTaskNumCalMode()) {
116 0 : CHK_RET(hrtGetDeviceType(params.deviceType));
117 : }
118 : // 获取ranktable info信息
119 0 : CHK_RET(GetRanktableInfo(rankTable));
120 0 : for (auto &rankInfo : rankTable.rankList) {
121 0 : HCCL_DEBUG("ParserClusterInfo serverId %s, rankId %u, superDeviceId 0x%x, superPodId %s",
122 : rankInfo.serverId.c_str(), rankInfo.rankId, rankInfo.superDeviceId, rankInfo.superPodId.c_str());
123 : }
124 :
125 0 : if (IsTaskNumCalMode()) {
126 0 : HCCL_INFO("[ParserClusterInfo] get task num cal mode.");
127 0 : return HCCL_SUCCESS;
128 : }
129 :
130 0 : std::sort(rankTable.rankList.begin(), rankTable.rankList.end(),
131 0 : [&](const RankInfo_t &a, const RankInfo_t &b) -> bool {return a.rankId < b.rankId;});
132 :
133 0 : u32 rankId = INVALID_VALUE_RANKID;
134 0 : if (IsAllDigit(identify_.c_str()) != HCCL_SUCCESS ||
135 0 : SalStrToULong(identify_, HCCL_BASE_DECIMAL, rankId) != HCCL_SUCCESS) {
136 0 : RPT_INPUT_ERR(true, "EI0003", std::vector<std::string>({"ccl_op", "value", "parameter", "expect"}),
137 : std::vector<std::string>({"HcclCommInitRootInfo", identify_.c_str(), "RANK_ID", "a non-negative decimal number"}));
138 0 : HCCL_ERROR("[%s][%s]errNo[0x%016llx] rank_id[%s] is invalid", LOG_KEYWORDS_INIT_GROUP.c_str(),
139 : LOG_KEYWORDS_INVALID_ARGUMENT.c_str(), HCOM_ERROR_CODE(HCCL_E_PARA), identify_.c_str());
140 0 : return HCCL_E_PARA;
141 : }
142 :
143 0 : if (rankId >= rankTable.rankList.size()) {
144 0 : RPT_ENV_ERR(true, "EI0014", std::vector<std::string>({ "value", "variable" ,"expect" }),
145 : std::vector<std::string>({std::to_string(rankId), "rank_id",
146 : std::to_string(rankTable.rankList[rankId].rankId) }));
147 0 : HCCL_ERROR("[%s][%s]rankId[%u] is invalid", LOG_KEYWORDS_INIT_GROUP.c_str(),
148 : LOG_KEYWORDS_RANKTABLE_CHECK.c_str(), rankId);
149 0 : return HCCL_E_PARA;
150 : }
151 :
152 0 : RPT_ENV_ERR(rankId != rankTable.rankList[rankId].rankId, "EI0014",
153 : std::vector<std::string>({"value", "variable" ,"expect"}), std::vector<std::string>({
154 : std::to_string(rankId), "rank_id", std::to_string(rankTable.rankList[rankId].rankId) }));
155 0 : CHK_PRT_RET(rankId != rankTable.rankList[rankId].rankId,
156 : HCCL_ERROR("[%s][%s]check rankList[%u] rankId[%u] failed", LOG_KEYWORDS_INIT_GROUP.c_str(),
157 : LOG_KEYWORDS_RANKTABLE_CHECK.c_str(), rankId, rankTable.rankList[rankId].rankId), HCCL_E_UNAVAIL);
158 :
159 0 : if (rankId < rankTable.rankList.size() &&
160 0 : rankTable.rankList[rankId].deviceInfo.nicDeploy == NICDeployment::NIC_DEPLOYMENT_HOST) {
161 0 : rankTable.nicDeploy = NICDeployment::NIC_DEPLOYMENT_HOST;
162 0 : SetHostUseDevNicFlag(false);
163 : } else {
164 0 : DetectNicDepoly(rankTable);
165 : }
166 0 : CHK_RET(CheckNicDeployConsistence(rankTable, rankTable.nicDeploy));
167 :
168 0 : u32 devId = rankTable.rankList[rankId].deviceInfo.devicePhyId;
169 0 : CHK_RET(hrtGetDevice(¶ms.logicDevId));
170 :
171 0 : u32 phyId = 0;
172 0 : CHK_RET(hrtGetDevicePhyIdByIndex(static_cast<u32>(params.logicDevId), phyId));
173 0 : RPT_ENV_ERR(phyId != static_cast<u32>(devId), "EI0014", std::vector<std::string>({"value", "variable" ,"expect"}),
174 : std::vector<std::string>({std::to_string(phyId), "devicePhyId",
175 : std::to_string(static_cast<u32>(devId)) }));
176 0 : CHK_PRT_RET(phyId != static_cast<u32>(devId),
177 : HCCL_ERROR("[%s][%s]ranktable config devId[%d],but local devId[%u]",
178 : LOG_KEYWORDS_INIT_GROUP.c_str(), LOG_KEYWORDS_RANKTABLE_CHECK.c_str(), devId, phyId), HCCL_E_UNAVAIL);
179 :
180 0 : params.rank = rankId;
181 0 : params.serverId = rankTable.rankList[rankId].serverId;
182 0 : params.totalRanks = rankTable.rankNum;
183 :
184 0 : return HCCL_SUCCESS;
185 0 : }
186 :
187 0 : HcclResult TopoinfoRanktableConcise::CheckNicDeployConsistence(RankTable_t &clusterInfo, NICDeployment deploy) const
188 : {
189 0 : CHK_PRT_RET(clusterInfo.rankList.size() == 0, HCCL_DEBUG("rank list size is 0, skip nic deeply check."),
190 : HCCL_SUCCESS);
191 :
192 0 : for (auto &it : clusterInfo.rankList) {
193 0 : CHK_PRT_RET(
194 : // 因为部分场景下，网卡部署在device侧，但是允许deviceIP无效，所以不检测该场景，只检测使用HOST但是hostIp无效
195 : (deploy == NICDeployment::NIC_DEPLOYMENT_HOST && it.hostIp.IsInvalid()),
196 : HCCL_ERROR("[Get][RanktableInfo]errNo"
197 : "[0x%016llx] hostIp config bettewn ranks is different.", HCOM_ERROR_CODE(HCCL_E_PARA)), HCCL_E_PARA);
198 : }
199 0 : return HCCL_SUCCESS;
200 : }
201 :
202 0 : HcclResult TopoinfoRanktableConcise::GetRanktableInfo(RankTable_t &clusterInfo)
203 : {
204 : // server_list
205 0 : CHK_RET(GetServerList(fileContent_, clusterInfo));
206 0 : CHK_RET(GetSuperPodList(fileContent_, clusterInfo));
207 0 : CHK_RET(GetDevNum(clusterInfo.rankList, clusterInfo.deviceNum));
208 0 : clusterInfo.rankNum = clusterInfo.rankList.size();
209 0 : return HCCL_SUCCESS;
210 : }
211 :
212 0 : HcclResult TopoinfoRanktableConcise::GetServerList(const nlohmann::json &obj, RankTable_t &clusterInfo)
213 : {
214 0 : clusterInfo.serverList.clear();
215 0 : nlohmann::json serverList;
216 0 : CHK_RET(GetJsonProperty(obj, "server_list", serverList, false));
217 0 : HCCL_DEBUG("[%s.json] -> server_list: size:[%zu]", fileName_.c_str(), serverList.size());
218 :
219 : // 获取serverCount并校验
220 0 : std::string serverCount;
221 0 : HcclResult ret = GetJsonProperty(obj, "server_count", serverCount, true);
222 0 : if (ret == HCCL_SUCCESS) {
223 0 : HCCL_DEBUG("[%s.json] -> group_count: [%s]", fileName_.c_str(), serverCount.c_str());
224 0 : CHK_RET(SalStrToULong(serverCount, HCCL_BASE_DECIMAL, clusterInfo.serverNum));
225 :
226 : // 校验serverCount
227 0 : if (serverList.size() != clusterInfo.serverNum) {
228 0 : RPT_INPUT_ERR(true, "EI0014", std::vector<std::string>({ "value", "variable" ,"expect" }),
229 : std::vector<std::string>({serverCount, "server_count",
230 : std::to_string(serverList.size())}));
231 0 : HCCL_ERROR("[%s][%s]errNo[0x%016llx] serverList size[%zu] neq server num[%u]",
232 : LOG_KEYWORDS_INIT_GROUP.c_str(), LOG_KEYWORDS_RANKTABLE_CHECK.c_str(), HCOM_ERROR_CODE(HCCL_E_PARA),
233 : serverList.size(), clusterInfo.serverNum);
234 0 : return HCCL_E_PARA;
235 : }
236 0 : } else if (ret == HCCL_E_NOT_FOUND) {
237 0 : clusterInfo.serverNum = serverList.size();
238 0 : HCCL_WARNING("[Get][ServerList]The 'server_count' in ranktable is not found, "\
239 : "set server_count to server_list size[%u]", serverList.size());
240 : } else {
241 0 : HCCL_ERROR("[Get][ServerList]get server_count error, ret[%d]", ret);
242 0 : return ret;
243 : }
244 0 : if (clusterInfo.serverNum == 0) {
245 0 : RPT_INPUT_ERR(true, "EI0014", std::vector<std::string>({ "value", "variable" ,"expect" }),
246 : std::vector<std::string>({ std::to_string(clusterInfo.serverNum), "server_count",
247 : "is a positive integer" }));
248 0 : HCCL_ERROR("[%s][%s]errNo[0x%016llx] server num is zero", LOG_KEYWORDS_INIT_GROUP.c_str(),
249 : LOG_KEYWORDS_RANKTABLE_CHECK.c_str(), HCOM_ERROR_CODE(HCCL_E_PARA));
250 0 : return HCCL_E_PARA;
251 : }
252 0 : HCCL_DEBUG("[Get][ServerList]serverNum is [%u]", clusterInfo.serverNum);
253 :
254 0 : for (u32 index = 0; index < serverList.size(); index++) {
255 : // get single server info
256 0 : CHK_RET(GetSingleServer(serverList, index, clusterInfo));
257 : }
258 :
259 0 : for (u32 index = 0; index < clusterInfo.rankList.size(); index++) {
260 0 : CHK_RET(VerifyBackupDeviceIpAndPort(clusterInfo.rankList, index));
261 : }
262 : // 获取device
263 0 : return HCCL_SUCCESS;
264 0 : }
265 :
266 :
267 1 : HcclResult TopoinfoRanktableConcise::GetSingleServer(const nlohmann::json &serverListObj, u32 objIndex,
268 : RankTable_t &clusterInfo)
269 : {
270 : HcclResult ret;
271 1 : std::string serverId;
272 1 : CHK_RET(GetJsonArrayMemberProperty(serverListObj, objIndex, "server_id", serverId, false));
273 1 : if (serverId.empty()) {
274 0 : HCCL_ERROR("[Get][JsonArrayMemberProperty]errNo[0x%016llx] serverId[%s] is empty",
275 : HCOM_ERROR_CODE(HCCL_E_PARA), serverId.c_str());
276 0 : return HCCL_E_PARA;
277 : }
278 :
279 1 : if (serverId.length() > SERVERID_MAX_LENGTH) {
280 1 : HCCL_ERROR("[Get][GetSingleServer]errNo[0x%016llx] serverId[%s] length[%u] is more than %u",
281 : HCOM_ERROR_CODE(HCCL_E_PARA), serverId.c_str(), serverId.length(), SERVERID_MAX_LENGTH);
282 1 : return HCCL_E_PARA;
283 : }
284 :
285 : // 将serverId添加到资源池,内部会进行IP地址校验，如果资源池中有serverId，则报错
286 0 : CHK_RET(CheckUniqueAndInsertPool(JsonUniqueInfoType::UNIQUE_INFO_TYPE_SERVER_ID, serverId,
287 : JsonCheckOpType::CHECK_OP_TYPE_INSERT));
288 0 : HCCL_DEBUG("server id[%u]:[%s]", objIndex, serverId.c_str());
289 :
290 : u32 serverIdx;
291 0 : GenerateServerIdx(serverId, serverIdx);
292 0 : HCCL_DEBUG("server id[%u]:[%s], serverIdx[%u]", objIndex, serverId.c_str(), serverIdx);
293 0 : std::string hostNicIp;
294 0 : HcclIpAddress hostIp;
295 0 : ret = GetJsonArrayMemberProperty(serverListObj, objIndex, "host_ip", hostNicIp, true);
296 0 : CHK_PRT_RET(ret != HCCL_SUCCESS && ret != HCCL_E_NOT_FOUND,
297 : HCCL_ERROR("[Get][SingleServer]get host ip error"), ret);
298 0 : HCCL_DEBUG("[%s.json] -> host_ip: [%s]. ret[%u]", fileName_.c_str(), hostNicIp.c_str(), ret);
299 0 : if (ret != HCCL_E_NOT_FOUND) {
300 0 : CHK_RET(ConvertIpAddress(hostNicIp, hostIp));
301 : }
302 :
303 : // 处理ranklist
304 0 : ret = GetDeviceList(serverListObj, objIndex, clusterInfo, serverId, serverIdx, hostIp);
305 0 : CHK_PRT_RET(ret != HCCL_SUCCESS, HCCL_ERROR("[Get][SingleServer]get dev list error:serverId[%s]",
306 : serverId.c_str()), ret);
307 :
308 0 : return HCCL_SUCCESS;
309 1 : }
310 :
311 0 : HcclResult TopoinfoRanktableConcise::GetDeviceList(const nlohmann::json &serverListObj, u32 objIndex,
312 : RankTable_t &clusterInfo, std::string &serverId, u32 &serverIdx, HcclIpAddress &hostIp)
313 : {
314 0 : HCCL_DEBUG("Get GetDeviceList[%u]: serverId[%s]", objIndex, serverId.c_str());
315 :
316 0 : nlohmann::json deviceList;
317 0 : CHK_RET(GetJsonArrayMemberProperty(serverListObj, objIndex, "device", deviceList, false));
318 :
319 0 : HCCL_DEBUG("[%s.json] -> device_list: size:%zu", fileName_.c_str(), deviceList.size());
320 :
321 0 : CHK_PRT_RET(deviceList.size() == 0, HCCL_ERROR("[Get][DeviceList]deviceList size is zero"), HCCL_E_PARA);
322 :
323 0 : for (u32 index = 0; index < deviceList.size(); index++) {
324 : // get single server info
325 0 : CHK_RET(GetSingleDevice(deviceList, index, clusterInfo, serverId, serverIdx, hostIp));
326 : }
327 :
328 : // 检查devip的数目是否一致
329 0 : u32 rankListSize = clusterInfo.rankList.size();
330 0 : CHK_PRT_RET(rankListSize == 0, HCCL_ERROR("[Get][DeviceList]get ranklist is zero"), HCCL_E_PARA);
331 :
332 0 : u32 checkDeviceIpSize = 0;
333 0 : for (u32 index = 0; index < clusterInfo.rankList.size(); index++) {
334 0 : if (index == 0) {
335 0 : checkDeviceIpSize = clusterInfo.rankList[0].deviceInfo.deviceIp.size();
336 : }
337 0 : if (clusterInfo.rankList[index].deviceInfo.deviceIp.size() != checkDeviceIpSize) {
338 0 : HCCL_ERROR("[Get][DeviceList]device[%u] size[%u] neq first device size[%u] error", index,
339 : clusterInfo.rankList[index].deviceInfo.deviceIp.size(), checkDeviceIpSize);
340 0 : return HCCL_E_PARA;
341 : }
342 : }
343 :
344 0 : return HCCL_SUCCESS;
345 0 : }
346 :
347 0 : HcclResult TopoinfoRanktableConcise::GetSingleNicInfo(const nlohmann::json &serverListObj, u32 objIndex,
348 : RankInfo_t &rankinfo)
349 : {
350 0 : std::string netPosition;
351 0 : rankinfo.deviceInfo.nicDeploy = NICDeployment::NIC_DEPLOYMENT_DEVICE;
352 0 : HcclResult ret = GetJsonArrayMemberProperty(serverListObj, objIndex, "net_position", netPosition, true);
353 0 : CHK_PRT_RET(ret != HCCL_SUCCESS && ret != HCCL_E_NOT_FOUND,
354 : HCCL_ERROR("[Get][GetSingleNicInfo]get net position error"), ret);
355 0 : HCCL_DEBUG("[%s.json] -> net_position: [%s]. ret[%u]", fileName_.c_str(), netPosition.c_str(), ret);
356 0 : if (ret != HCCL_E_NOT_FOUND) {
357 0 : if (netPosition == "host") { // 默认值为Device模式
358 0 : rankinfo.deviceInfo.nicDeploy = NICDeployment::NIC_DEPLOYMENT_HOST;
359 : } else {
360 0 : rankinfo.deviceInfo.nicDeploy = NICDeployment::NIC_DEPLOYMENT_DEVICE;
361 : }
362 : }
363 :
364 0 : std::string netProto;
365 0 : ret = GetJsonArrayMemberProperty(serverListObj, objIndex, "net_protocol", netProto, true);
366 0 : CHK_PRT_RET(ret != HCCL_SUCCESS && ret != HCCL_E_NOT_FOUND,
367 : HCCL_ERROR("[Get][GetSingleNicInfo]get net protocol error"), ret);
368 0 : HCCL_DEBUG("[%s.json] -> net_protocol: [%s]. ret[%u]", fileName_.c_str(), netProto.c_str(), ret);
369 0 : if (ret != HCCL_E_NOT_FOUND) {
370 0 : if (netProto == "rdma") { // 未配置则使用默认值，不处理，使用原有逻辑
371 0 : SetExternalInputProtocolType(ProtocolType::RDMA);
372 0 : rankinfo.deviceInfo.proto = u32(ProtocolType::RDMA);
373 0 : } else if (netProto == "tcp") {
374 0 : SetExternalInputProtocolType(ProtocolType::TCP);
375 0 : rankinfo.deviceInfo.proto = u32(ProtocolType::TCP);
376 : }
377 : }
378 :
379 0 : return HCCL_SUCCESS;
380 0 : }
381 :
382 0 : HcclResult TopoinfoRanktableConcise::GetSingleDevice(const nlohmann::json &deviceListObj, u32 objIndex,
383 : RankTable_t &clusterInfo, std::string &serverId, u32 &serverIdx, HcclIpAddress &hostIp)
384 : {
385 : // 获取rank_id
386 0 : std::string rankId;
387 0 : CHK_RET(GetJsonArrayMemberProperty(deviceListObj, objIndex, "rank_id", rankId, false));
388 0 : HCCL_DEBUG("[%s.json] -> rank_id: [%s]", fileName_.c_str(), rankId.c_str());
389 :
390 : // 获取device type
391 0 : DevType deviceType = DevType::DEV_TYPE_COUNT;
392 0 : CHK_RET(hrtGetDeviceType(deviceType));
393 :
394 : // 获取device_id
395 0 : std::string strDevid;
396 0 : CHK_RET(GetJsonArrayMemberProperty(deviceListObj, objIndex, "device_id", strDevid, false));
397 :
398 0 : u32 devicePhyId = 0;
399 0 : CHK_RET(SalStrToULong(strDevid, HCCL_BASE_DECIMAL, devicePhyId));
400 :
401 : u32 maxDeviceNum;
402 0 : CHK_RET(GetMaxDevNum(maxDeviceNum));
403 0 : if ((deviceType == DevType::DEV_TYPE_310P3 || deviceType == DevType::DEV_TYPE_910B ||
404 0 : deviceType == DevType::DEV_TYPE_910_93) && devicePhyId > (maxDeviceNum - 1)) {
405 : // deviceid in 0 ~ maxDeviceNum
406 0 : HCCL_ERROR("[Get][SingleDevice]errNo[0x%016llx] device_id[%u] more than maxDeviceNum[%u] invalid",
407 : HCOM_ERROR_CODE(HCCL_E_PARA), devicePhyId, maxDeviceNum);
408 0 : return HCCL_E_PARA;
409 0 : } else if ((deviceType != DevType::DEV_TYPE_310P3 && deviceType != DevType::DEV_TYPE_910B &&
410 0 : deviceType != DevType::DEV_TYPE_910_93) && devicePhyId > (HCCL_AISERVER_DEVICE_NUM - 1)) {
411 : // deviceid in 0 ~ 7
412 0 : HCCL_ERROR("[Get][SingleDevice]errNo[0x%016llx] device_id[%u] more than 7 is invalid",
413 : HCOM_ERROR_CODE(HCCL_E_PARA), devicePhyId);
414 0 : return HCCL_E_PARA;
415 : }
416 0 : HCCL_DEBUG("[%s.json] -> device_id: [%s]", fileName_.c_str(), strDevid.c_str());
417 :
418 0 : RankInfo_t rankinfo;
419 0 : rankinfo.serverId = serverId;
420 0 : rankinfo.serverIdx = serverIdx;
421 0 : rankinfo.hostIp = hostIp;
422 0 : rankinfo.deviceInfo.devicePhyId = devicePhyId;
423 :
424 0 : CHK_RET(GetSingleDeviceHostPort(deviceListObj, objIndex, rankinfo));
425 0 : CHK_RET(GetSingleDeviceIp(deviceListObj, objIndex, clusterInfo, rankinfo, deviceType, rankinfo.hostIp.IsInvalid()));
426 0 : CHK_RET(GetSingleDevicePort(deviceListObj, objIndex, rankinfo));
427 0 : CHK_RET(GetSingleBackupDeviceIp(deviceListObj, objIndex, rankinfo));
428 0 : CHK_RET(GetSingleNicInfo(deviceListObj, objIndex, rankinfo));
429 :
430 0 : if (SalStrToULong(rankId, HCCL_BASE_DECIMAL, rankinfo.rankId) != HCCL_SUCCESS) {
431 0 : RPT_INPUT_ERR(true, "EI0014", std::vector<std::string>({ "value", "variable" ,"expect" }),
432 : std::vector<std::string>({ rankId, "rank_id", "is a non-negative integer." }));
433 0 : HCCL_ERROR("[%s][%s]errNo[0x%016llx] rankid[%s] is invalid", LOG_KEYWORDS_INIT_GROUP.c_str(),
434 : LOG_KEYWORDS_RANKTABLE_CHECK.c_str(), HCOM_ERROR_CODE(HCCL_E_PARA), rankId.c_str());
435 0 : return HCCL_E_PARA;
436 : }
437 :
438 0 : rankinfo.podName = ""; // podname在新场景下置空
439 0 : rankId = "";
440 :
441 0 : string version;
442 0 : CHK_RET(GetRanktableVersion(version));
443 0 : if (version.compare(SUPERPOD_CLUSTER_VERSION) != 0) {
444 0 : clusterInfo.rankList.push_back(rankinfo);
445 0 : HCCL_DEBUG("[%s.json]->rankId[%u], serverId[%s], devicePhyId[%d]", fileName_.c_str(),
446 : rankinfo.rankId, rankinfo.serverId.c_str(), rankinfo.deviceInfo.devicePhyId);
447 :
448 0 : return HCCL_SUCCESS;
449 : }
450 :
451 0 : CHK_RET(GetSingleSuperDeviceId(deviceListObj, objIndex, clusterInfo, rankinfo));
452 0 : clusterInfo.rankList.push_back(rankinfo);
453 0 : return HCCL_SUCCESS;
454 0 : }
455 :
456 2 : HcclResult TopoinfoRanktableConcise::SplitString(const std::string& str, const std::string& strC,
457 : std::vector<std::string>& strVector) const
458 : {
459 2 : std::string::size_type startPos = 0;
460 2 : std::string::size_type foundPos = str.find(strC);
461 :
462 2 : while (foundPos != std::string::npos) {
463 0 : strVector.push_back(str.substr(startPos, foundPos - startPos));
464 0 : startPos = foundPos + strC.size();
465 0 : foundPos = str.find(strC, startPos);
466 : }
467 2 : if (startPos != str.length()) {
468 2 : strVector.push_back(str.substr(startPos));
469 : }
470 2 : return HCCL_SUCCESS;
471 : }
472 :
473 0 : HcclResult TopoinfoRanktableConcise::GetSingleDeviceIp(const nlohmann::json &deviceListObj, u32 objIndex,
474 : RankTable_t &clusterInfo, RankInfo_t &rankinfo, DevType deviceType, bool invalidHostIp)
475 : {
476 : // 获取device_ip （可能有多个）
477 0 : std::string deviceIp;
478 0 : HcclResult ret = GetJsonArrayMemberProperty(deviceListObj, objIndex, "device_ip", deviceIp, true);
479 : // 多机和走roce网卡ranktable必须有“device_ip”字段，单机可以没有
480 0 : if (clusterInfo.serverNum > 1 || (GetExternalInputIntraRoceSwitch() == 1)) {
481 : // 如果没有配置HostIp，那么deviceIp必须有效，否则出错
482 0 : bool isDeviceIpError = (ret != HCCL_SUCCESS && invalidHostIp) || (deviceType == DevType::DEV_TYPE_910B && deviceIp == "");
483 : // A3超节点内使用vnic建链，不需要device ip
484 0 : if (deviceType == DevType::DEV_TYPE_910_93 && clusterInfo.superPodNum <= 1 && GetExternalInputIntraRoceSwitch() == 0) {
485 0 : isDeviceIpError = false;
486 : }
487 0 : RPT_INPUT_ERR(isDeviceIpError, "EI0014", std::vector<std::string>({ "value", "variable" ,"expect" }),
488 : std::vector<std::string>({ deviceIp, "device_ip", "a valid IP address" }));
489 0 : CHK_PRT_RET(isDeviceIpError,
490 : HCCL_ERROR("[%s][%s]'device_ip' is not set correctly, " \
491 : "must be set when multi Server or HCCL_INTRA_ROCE_ENABLE enabled",
492 : LOG_KEYWORDS_INIT_GROUP.c_str(), LOG_KEYWORDS_RANKTABLE_CHECK.c_str()), HCCL_E_PARA);
493 0 : } else if (clusterInfo.serverNum == 1 && ret == HCCL_E_NOT_FOUND) {
494 0 : HcclIpAddress invalidAddr;
495 0 : rankinfo.deviceInfo.deviceIp.push_back(invalidAddr);
496 0 : HCCL_WARNING("[Get][SingleDeviceIp]'device_ip' in ranktable is not set!");
497 0 : return HCCL_SUCCESS;
498 0 : } else {
499 : // 如果没有配置HostIp，那么deviceIp必须有效，否则出错
500 0 : CHK_PRT_RET(ret != HCCL_SUCCESS && invalidHostIp,
501 : HCCL_ERROR("[Get][SingleDeviceIp]errNo[0x%016llx] 'device_ip' is not set correctly",
502 : HCOM_ERROR_CODE(HCCL_E_PARA)), ret);
503 : }
504 0 : HCCL_DEBUG("[%s.json] -> device_ip: [%s]", fileName_.c_str(), deviceIp.c_str());
505 :
506 : // 处理字符串device_ip
507 0 : std::vector<std::string> strDeviceIp;
508 0 : if (deviceIp != "") {
509 0 : CHK_RET(SplitString(deviceIp, ",", strDeviceIp));
510 :
511 0 : CHK_PRT_RET(strDeviceIp.size() == 0, HCCL_ERROR("[Get][SingleDeviceIp]in device:deviceip size is zero"),
512 : HCCL_E_PARA);
513 0 : for (u32 index = 0; index < strDeviceIp.size(); index++) {
514 0 : CHK_RET(CheckUniqueAndInsertPool(JsonUniqueInfoType::UNIQUE_INFO_TYPE_DEVICE_IP, strDeviceIp[index],
515 : JsonCheckOpType::CHECK_OP_TYPE_INSERT));
516 0 : HcclIpAddress ipAddr;
517 0 : CHK_RET(ConvertIpAddress(strDeviceIp[index], ipAddr));
518 0 : rankinfo.deviceInfo.deviceIp.push_back(ipAddr);
519 0 : devIp2ObjIndex_.emplace(ipAddr.GetReadableIP(), objIndex);
520 0 : }
521 : } else {
522 0 : HcclIpAddress invalidAddr;
523 0 : rankinfo.deviceInfo.deviceIp.push_back(invalidAddr);
524 0 : HCCL_WARNING("objIndex[%u],'device_ip' is not set", objIndex);
525 0 : }
526 :
527 0 : return HCCL_SUCCESS;
528 0 : }
529 :
530 4 : HcclResult TopoinfoRanktableConcise::GetSingleBackupDeviceIp(const nlohmann::json &deviceListObj, u32 objIndex,
531 : RankInfo_t &rankinfo)
532 : {
533 4 : if (params_.deviceType != DevType::DEV_TYPE_910_93 || !GetExternalInputHcclAicpuUnfold()
534 8 : || !isInterSuperPodRetryEnable_) {
535 0 : return HCCL_SUCCESS;
536 : }
537 : // 获取backup_device_ip（可能有多个）
538 4 : std::string backupDeviceIp;
539 4 : HcclResult ret = GetJsonArrayMemberProperty(deviceListObj, objIndex, "backup_device_ip", backupDeviceIp);
540 : // backup_device_ip字段未配置时通过warning日志提示
541 4 : if (ret == HCCL_E_NOT_FOUND) {
542 0 : HcclIpAddress invalidAddr;
543 0 : rankinfo.deviceInfo.backupDeviceIp.push_back(invalidAddr);
544 0 : HCCL_WARNING("[Get][SingleDeviceIp]'backup_device_ip' in ranktable is not set!");
545 0 : return HCCL_SUCCESS;
546 4 : } else if (ret == HCCL_E_PARA) {
547 2 : HCCL_ERROR("[Get][SingleDeviceIp]Get json array member property error");
548 2 : return HCCL_E_PARA;
549 : }
550 2 : HCCL_DEBUG("[%s.json] -> backup_device_ip: [%s]", fileName_.c_str(), backupDeviceIp.c_str());
551 :
552 : // 处理backup_device_ip字符串
553 2 : std::vector<std::string> strBackupDeviceIp;
554 2 : if (backupDeviceIp != "") {
555 4 : CHK_RET(SplitString(backupDeviceIp, ",", strBackupDeviceIp));
556 :
557 2 : CHK_PRT_RET(strBackupDeviceIp.size() == 0,
558 : HCCL_ERROR("[Get][SingleBackupDeviceIp]in device: deviceip size is zero"),
559 : HCCL_E_PARA);
560 3 : for (u32 index = 0; index < strBackupDeviceIp.size(); index++) {
561 2 : CHK_RET(CheckUniqueAndInsertPool(JsonUniqueInfoType::UNIQUE_INFO_TYPE_BACKUP_DEVICE_IP,
562 : strBackupDeviceIp[index], JsonCheckOpType::CHECK_OP_TYPE_INSERT));
563 1 : HcclIpAddress ipAddr;
564 1 : CHK_RET(ConvertIpAddress(strBackupDeviceIp[index], ipAddr));
565 1 : rankinfo.deviceInfo.backupDeviceIp.push_back(ipAddr);
566 1 : }
567 1 : CHK_RET(GetSingleBackupDevicePort(deviceListObj, objIndex, rankinfo));
568 1 : HCCL_INFO("[TopoinfoRanktableConcise][GetSingleBackupDeviceIp]devicePhyId[%u], backupDeviceIp[0]:[%s].",
569 : rankinfo.deviceInfo.devicePhyId, rankinfo.deviceInfo.backupDeviceIp[0].GetReadableIP());
570 : } else {
571 0 : HcclIpAddress invalidAddr;
572 0 : rankinfo.deviceInfo.backupDeviceIp.push_back(invalidAddr);
573 0 : HCCL_WARNING("objIndex[%u],'bakcup_device_ip' is not set", objIndex);
574 0 : }
575 1 : return HCCL_SUCCESS;
576 4 : }
577 :
578 0 : HcclResult TopoinfoRanktableConcise::GetSingleDeviceHostPort(const nlohmann::json &deviceListObj, u32 objIndex,
579 : RankInfo_t &rankinfo)
580 : {
581 0 : std::string hostPortStr;
582 0 : rankinfo.hostPort = HCCL_INVALID_PORT;
583 0 : HcclResult ret = GetJsonArrayMemberProperty(deviceListObj, objIndex, "host_port", hostPortStr, true);
584 0 : CHK_PRT_RET(ret != HCCL_SUCCESS && ret != HCCL_E_NOT_FOUND,
585 : HCCL_ERROR("[Get][SingleDeviceHostPort]get host port error, deviceIndex[%u]", objIndex), ret);
586 0 : HCCL_DEBUG("[%s.json] -> host_port: [%s]. ret[%u]", fileName_.c_str(), hostPortStr.c_str(), ret);
587 0 : if (ret != HCCL_E_NOT_FOUND) {
588 0 : CHK_RET(SalStrToULong(hostPortStr, HCCL_BASE_DECIMAL, rankinfo.hostPort));
589 0 : CHK_PRT_RET(rankinfo.hostPort == HCCL_SOCKET_PORT_RANGE_AUTO,
590 : HCCL_ERROR("[Get][SingleDeviceHostPort] deviceIndex[%u], please do not use the reserved port number[%u]",
591 : objIndex, HCCL_SOCKET_PORT_RANGE_AUTO), HCCL_E_PARA);
592 0 : CHK_PRT_RET(rankinfo.hostPort > MAX_PORT_NUMBER,
593 : HCCL_ERROR("[Get][SingleDeviceHostPort] deviceIndex[%u], port number[%u] exceed max port number[%u]",
594 : objIndex, rankinfo.hostPort, MAX_PORT_NUMBER), HCCL_E_PARA);
595 0 : HCCL_INFO("[TopoinfoRanktableConcise][GetSingleDeviceHostPort] deviceIndex[%u], devicePhyId[%u], "
596 : "get HOST port[%u].", objIndex, rankinfo.deviceInfo.devicePhyId, rankinfo.hostPort);
597 : } else {
598 0 : HCCL_INFO("[Get][SingleDeviceHostPort] deviceIndex[%u], 'host_port' in ranktable is not set. "
599 : "Multi-process may not be supported for op retry.", objIndex);
600 : }
601 0 : return HCCL_SUCCESS;
602 0 : }
603 :
604 2 : HcclResult TopoinfoRanktableConcise::GetSingleDevicePort(const nlohmann::json &deviceListObj, u32 objIndex,
605 : RankInfo_t &rankinfo)
606 : {
607 : // 获取device指定的port；如果用户未配置，则置为缺省值
608 2 : std::string strDevPort;
609 2 : HcclResult ret = GetJsonArrayMemberProperty(deviceListObj, objIndex, "device_port", strDevPort, true);
610 2 : CHK_PRT_RET(ret != HCCL_SUCCESS && ret != HCCL_E_NOT_FOUND,
611 : HCCL_ERROR("[Get][SingleDevicePort]Get json array member property error."), ret);
612 2 : if (ret == HCCL_E_NOT_FOUND) {
613 0 : rankinfo.deviceInfo.port = HCCL_INVALID_PORT;
614 0 : HCCL_INFO("[Get][SingleDevicePort]deviceIndex[%u], devicePhyId[%u], 'device_port' in ranktable is not set. "
615 : "Multi-process may not be supported for device nic.", objIndex, rankinfo.deviceInfo.devicePhyId);
616 0 : params_.commPortConfig.devPortSwitchOn = false; // 不启用用户指定的port作为device网卡通信的port
617 0 : return HCCL_SUCCESS;
618 : } else {
619 2 : CHK_RET(SalStrToULong(strDevPort, HCCL_BASE_DECIMAL, rankinfo.deviceInfo.port));
620 2 : CHK_PRT_RET(rankinfo.deviceInfo.port == HCCL_SOCKET_PORT_RANGE_AUTO,
621 : HCCL_ERROR("[Get][SingleDevicePort] deviceIndex[%u], devicePhyId[%u], "
622 : "please do not use the reserved port number [%u]. ",
623 : objIndex, rankinfo.deviceInfo.devicePhyId, HCCL_SOCKET_PORT_RANGE_AUTO), HCCL_E_PARA);
624 2 : CHK_PRT_RET(rankinfo.deviceInfo.port > MAX_PORT_NUMBER,
625 : HCCL_ERROR("[Get][SingleDevicePort] deviceIndex[%u], devicePhyId[%u], "
626 : "port number[%u] exceed max port number[%u]",
627 : objIndex, rankinfo.deviceInfo.devicePhyId, rankinfo.deviceInfo.port, MAX_PORT_NUMBER), HCCL_E_PARA);
628 2 : params_.commPortConfig.devPortSwitchOn = true; // 启用用户指定的port作为device网卡通信的port
629 : }
630 :
631 : // 获取device指定的vnic port；如果未配置，则与device_port相同（仅用于MasterInfo协商解析Ranktable）
632 2 : std::string strVnicPort;
633 2 : ret = GetJsonArrayMemberProperty(deviceListObj, objIndex, "device_vnic_port", strVnicPort, true);
634 2 : CHK_PRT_RET(ret != HCCL_SUCCESS && ret != HCCL_E_NOT_FOUND,
635 : HCCL_ERROR("[Get][SingleDevicePort]Get json array member property vnic error."), ret);
636 2 : if (ret == HCCL_E_NOT_FOUND) {
637 1 : rankinfo.deviceInfo.vnicPort = rankinfo.deviceInfo.port;
638 1 : HCCL_INFO("[Get][SingleDevicePort]deviceIndex[%u], devicePhyId[%u], "
639 : "'device_vnic_port' in ranktable is not set. ", objIndex, rankinfo.deviceInfo.devicePhyId);
640 : } else {
641 1 : CHK_RET(SalStrToULong(strVnicPort, HCCL_BASE_DECIMAL, rankinfo.deviceInfo.vnicPort));
642 1 : CHK_PRT_RET(rankinfo.deviceInfo.vnicPort == HCCL_SOCKET_PORT_RANGE_AUTO,
643 : HCCL_ERROR("[Get][SingleDevicePort] deviceIndex[%u], devicePhyId[%u], "
644 : "please do not use the reserved port number [%u] as vnic port number. ",
645 : objIndex, rankinfo.deviceInfo.devicePhyId, HCCL_SOCKET_PORT_RANGE_AUTO), HCCL_E_PARA);
646 1 : CHK_PRT_RET(rankinfo.deviceInfo.vnicPort > MAX_PORT_NUMBER,
647 : HCCL_ERROR("[Get][SingleDevicePort] deviceIndex[%u], devicePhyId[%u], "
648 : "vnic port number[%u] exceed max port number[%u]",
649 : objIndex, rankinfo.deviceInfo.devicePhyId, rankinfo.deviceInfo.vnicPort, MAX_PORT_NUMBER), HCCL_E_PARA);
650 : }
651 :
652 2 : HCCL_INFO("[TopoinfoRanktableConcise][GetSingleDevicePort] deviceIndex[%u], devicePhyId[%u], get device port[%u], "
653 : "device vnic port[%u].",
654 : objIndex, rankinfo.deviceInfo.devicePhyId, rankinfo.deviceInfo.port, rankinfo.deviceInfo.vnicPort);
655 2 : return HCCL_SUCCESS;
656 2 : }
657 :
658 1 : HcclResult TopoinfoRanktableConcise::GetSingleBackupDevicePort(const nlohmann::json &deviceListObj, u32 objIndex,
659 : RankInfo_t &rankinfo)
660 : {
661 : // 获取device指定的backup port；如果用户未配置，则置为缺省值
662 1 : std::string strBackupPort;
663 1 : HcclResult ret = GetJsonArrayMemberProperty(deviceListObj, objIndex, "backup_device_port", strBackupPort, true);
664 1 : CHK_PRT_RET(ret != HCCL_SUCCESS && ret != HCCL_E_NOT_FOUND,
665 : HCCL_ERROR("[Get][SingleBackupDevicePort]Get json array member property error."), ret);
666 1 : if (ret == HCCL_E_NOT_FOUND) {
667 1 : rankinfo.deviceInfo.backupPort = HCCL_INVALID_PORT;
668 1 : HCCL_INFO("[Get][SingleBackupDevicePort]deviceIndex[%u], devicePhyId[%u], "
669 : "'backup_device_port' in ranktable is not set. Multi-process may not be supported for backup nic.",
670 : objIndex, rankinfo.deviceInfo.devicePhyId);
671 1 : return HCCL_SUCCESS;
672 : }
673 :
674 0 : CHK_RET(SalStrToULong(strBackupPort, HCCL_BASE_DECIMAL, rankinfo.deviceInfo.backupPort));
675 0 : CHK_PRT_RET(rankinfo.deviceInfo.backupPort == HCCL_SOCKET_PORT_RANGE_AUTO,
676 : HCCL_ERROR("[Get][SingleBackupDevicePort] deviceIndex[%u], devicePhyId[%u], "
677 : "please do not use the reserved port number [%u]. ",
678 : objIndex, rankinfo.deviceInfo.devicePhyId, HCCL_SOCKET_PORT_RANGE_AUTO), HCCL_E_PARA);
679 0 : CHK_PRT_RET(rankinfo.deviceInfo.backupPort > MAX_PORT_NUMBER,
680 : HCCL_ERROR("[Get][SingleBackupDevicePort] deviceIndex[%u], devicePhyId[%u], "
681 : "port number[%u] exceed max port number[%u]",
682 : objIndex, rankinfo.deviceInfo.devicePhyId, rankinfo.deviceInfo.backupPort, MAX_PORT_NUMBER), HCCL_E_PARA);
683 :
684 0 : HCCL_INFO("[TopoinfoRanktableConcise][GetSingleBackupDevicePort] deviceIndex[%u], devicePhyId[%u], "
685 : "get backup device port[%u].", objIndex, rankinfo.deviceInfo.devicePhyId, rankinfo.deviceInfo.backupPort);
686 0 : return HCCL_SUCCESS;
687 1 : }
688 :
689 2 : HcclResult TopoinfoRanktableConcise::VerifyBackupDeviceIpAndPort(std::vector<RankInfo_t> &rankList, u32 devIndex)
690 : {
691 2 : if (params_.deviceType != DevType::DEV_TYPE_910_93 || !GetExternalInputHcclAicpuUnfold()
692 4 : || !isInterSuperPodRetryEnable_) {
693 0 : return HCCL_SUCCESS;
694 : }
695 2 : RankInfo_t &rankInfo = rankList[devIndex];
696 :
697 3 : for (auto &backupDevIp : rankInfo.deviceInfo.backupDeviceIp) {
698 1 : if (backupDevIp.IsInvalid()) {
699 : // 无效备用IP，无需校验
700 1 : continue;
701 : }
702 1 : string backupDevIpStr = backupDevIp.GetReadableIP();
703 1 : if (devIp2ObjIndex_.find(backupDevIpStr) == devIp2ObjIndex_.end()) {
704 1 : HCCL_RUN_WARNING("[Verify][BackupDeviceIp]"
705 : "Backup devIp[%s] for devicePhyId[%d] is not in this comm. "
706 : "The validation of this backup ip could not be verified! "
707 : "Please notice it might be an invalid backup ip!",
708 : backupDevIpStr.c_str(), devIndex);
709 1 : continue;
710 : }
711 :
712 0 : RankInfo_t &backupRankInfo = rankList[devIp2ObjIndex_[backupDevIpStr]];
713 :
714 0 : s32 backupDevPhyId = backupRankInfo.deviceInfo.devicePhyId;
715 0 : CHK_PRT_RET(backupDevPhyId == rankInfo.deviceInfo.devicePhyId,
716 : HCCL_ERROR("[Verify][BackupDeviceIp]"
717 : "PhyId[%d] for backup devIp[%s] is the same with self device[%u] phyId[%d]. "
718 : "Please do not use self ip as backup ip!",
719 : backupDevPhyId, backupDevIpStr.c_str(), devIndex, rankInfo.deviceInfo.devicePhyId),
720 : HCCL_E_PARA);
721 :
722 0 : LinkTypeInServer linkType = LinkTypeInServer::RESERVED_LINK_TYPE;
723 0 : CHK_RET(hrtGetPairDeviceLinkType(rankInfo.deviceInfo.devicePhyId, backupDevPhyId, linkType));
724 0 : CHK_PRT_RET(linkType != LinkTypeInServer::SIO_TYPE,
725 : HCCL_ERROR("[Verify][BackupDeviceIp]errNo[0x%016llx], device[%u], "
726 : "link between device phyId[%d] and backup device phyId[%d] is not sio link, backup device ip[%s]. "
727 : "Please check backup ip validation and whether it is on a pair device!",
728 : HCOM_ERROR_CODE(HCCL_E_PARA), devIndex, rankInfo.deviceInfo.devicePhyId,
729 : backupDevPhyId, backupDevIpStr.c_str()), HCCL_E_PARA);
730 :
731 : // 用于备用网卡的端口不可和用于主网卡的端口冲突
732 0 : CHK_PRT_RET(rankInfo.deviceInfo.backupPort != HCCL_INVALID_PORT
733 : && backupRankInfo.deviceInfo.port != HCCL_INVALID_PORT
734 : && rankInfo.deviceInfo.backupPort == backupRankInfo.deviceInfo.port,
735 : HCCL_ERROR("[Verify][BackupDevicePort] deviceIndex[%u], "
736 : "backup device port[%u] on devPhyId[%u] should not be the same with device port[%u] on devPhyId[%u].",
737 : devIndex, rankInfo.deviceInfo.backupPort, rankInfo.deviceInfo.devicePhyId, backupRankInfo.deviceInfo.port,
738 : backupRankInfo.deviceInfo.devicePhyId), HCCL_E_PARA);
739 1 : }
740 2 : return HCCL_SUCCESS;
741 : }
742 :
743 0 : HcclResult TopoinfoRanktableConcise::GetSingleSuperDeviceId(const nlohmann::json &deviceListObj, u32 objIndex,
744 : RankTable_t &clusterInfo, RankInfo_t &rankinfo)
745 : {
746 : // 获取super_device_id
747 : HcclResult ret;
748 0 : std::string strSuperDeviceId;
749 0 : ret = GetJsonArrayMemberProperty(deviceListObj, objIndex, "super_device_id", strSuperDeviceId, true);
750 :
751 0 : CHK_PRT_RET(ret == HCCL_E_NOT_FOUND,
752 : HCCL_WARNING("[Get][SingleSuperDeviceId]'super_device_id' is not found"), HCCL_SUCCESS);
753 :
754 0 : u32 superDeviceId = 0;
755 0 : ret = SalStrToULong(strSuperDeviceId, HCCL_BASE_DECIMAL, superDeviceId);
756 0 : if (ret != HCCL_SUCCESS) {
757 0 : RPT_INPUT_ERR(true, "EI0014", std::vector<std::string>({ "value", "variable" ,"expect" }),
758 : std::vector<std::string>({ strSuperDeviceId, "super_device_id",
759 : "a valid non-negative integer" }));
760 0 : HCCL_ERROR("[%s][%s]errNo[0x%016llx] super_device_id[%s] is invalid",
761 : LOG_KEYWORDS_INIT_GROUP.c_str(), LOG_KEYWORDS_RANKTABLE_CHECK.c_str(), HCOM_ERROR_CODE(HCCL_E_PARA),
762 : strSuperDeviceId.c_str());
763 0 : return ret;
764 : }
765 :
766 0 : rankinfo.superDeviceId = superDeviceId;
767 0 : HCCL_DEBUG("[%s.json] -> super_device_id: [%s]", fileName_.c_str(), strSuperDeviceId.c_str());
768 0 : return HCCL_SUCCESS;
769 0 : }
770 :
771 0 : HcclResult TopoinfoRanktableConcise::GetSuperPodList(const nlohmann::json &obj, RankTable_t &clusterInfo)
772 : {
773 0 : string version;
774 0 : CHK_RET(GetRanktableVersion(version));
775 0 : CHK_PRT_RET(version.compare(SUPERPOD_CLUSTER_VERSION) != 0,
776 : HCCL_INFO("[Get][SuperPodList]ranktable version[%s], do nothing.", version.c_str()), HCCL_SUCCESS);
777 :
778 0 : if (!IsTaskNumCalMode()) { // taskNum评估阶段，无法判断是否超节点模式
779 : // 环境配置非超节点时，不解析超节点信息, ranktable中记录的superPodId, superDeviceId, superPodNum均为默认值
780 0 : bool useSuperPodMode = false;
781 0 : CHK_RET(IsSuperPodMode(useSuperPodMode));
782 0 : CHK_PRT_RET(!useSuperPodMode, HCCL_INFO("[Get][SuperPodList]not super pod, do nothing"), HCCL_SUCCESS);
783 : }
784 :
785 : HcclResult ret;
786 0 : nlohmann::json superPodList;
787 0 : ret = GetJsonProperty(obj, "super_pod_list", superPodList, true);
788 0 : CHK_PRT_RET(ret == HCCL_E_NOT_FOUND,
789 : HCCL_WARNING("[Get][SuperPodList]'super_pod_list' is not found"), HCCL_SUCCESS);
790 0 : CHK_PRT_RET(ret != HCCL_SUCCESS && ret != HCCL_E_NOT_FOUND,
791 : HCCL_ERROR("[Get][SuperPodList]'super_pod_list' in ranktable is not set correctly, ret[%d]", ret), ret);
792 0 : HCCL_DEBUG("[%s.json]super_pod_list -> : size:[%zu]", fileName_.c_str(), superPodList.size());
793 :
794 0 : for (u32 index = 0; index < superPodList.size(); index++) {
795 0 : CHK_RET(GetSingleSuperPod(superPodList, index, clusterInfo));
796 : }
797 :
798 0 : clusterInfo.superPodNum = superPodList.size();
799 0 : CHK_RET(CheckSuperPodInfo(clusterInfo));
800 0 : return HCCL_SUCCESS;
801 0 : }
802 :
803 0 : HcclResult TopoinfoRanktableConcise::GetSingleSuperPod(const nlohmann::json &superPodList, u32 objIndex,
804 : RankTable_t &clusterInfo)
805 : {
806 0 : HcclResult ret = HCCL_SUCCESS;
807 0 : std::string superPodId;
808 0 : ret = GetJsonArrayMemberProperty(superPodList, objIndex, "super_pod_id", superPodId, true);
809 0 : if (ret != HCCL_SUCCESS || superPodId.empty()) {
810 0 : RPT_INPUT_ERR(true, "EI0014", std::vector<std::string>({ "value", "variable" ,"expect" }),
811 : std::vector<std::string>({ superPodId, "super_pod_id",
812 : "a valid non-empty string." }));
813 0 : HCCL_ERROR("[%s][%s]errNo[0x%016llx] super_pod_id[%s] is invalid", LOG_KEYWORDS_INIT_GROUP.c_str(),
814 : LOG_KEYWORDS_RANKTABLE_CHECK.c_str(), HCOM_ERROR_CODE(ret), superPodId.c_str());
815 0 : return ret;
816 : }
817 :
818 : // 将superPodId添加到资源池进行查重校验
819 0 : CHK_RET(CheckUniqueAndInsertPool(JsonUniqueInfoType::UNIQUE_INFO_TYPE_SUPER_POD_ID, superPodId,
820 : JsonCheckOpType::CHECK_OP_TYPE_INSERT));
821 0 : HCCL_DEBUG("superPod id[%u]:[%s]", objIndex, superPodId.c_str());
822 :
823 : // 处理ranklist
824 0 : ret = GetSuperPodServerList(superPodList, objIndex, clusterInfo, superPodId);
825 0 : CHK_PRT_RET(ret != HCCL_SUCCESS, HCCL_ERROR("[Get][SingleSuperPod]get server list error:superPodId[%s]",
826 : superPodId.c_str()), ret);
827 :
828 0 : return HCCL_SUCCESS;
829 0 : }
830 :
831 0 : HcclResult TopoinfoRanktableConcise::GetSuperPodServerList(const nlohmann::json &superPodList, u32 objIndex,
832 : RankTable_t &clusterInfo, std::string superPodId)
833 : {
834 0 : HCCL_DEBUG("GetSuperPodServerList[%u]: superPodId[%s]", objIndex, superPodId.c_str());
835 :
836 0 : nlohmann::json superPodServerList;
837 0 : CHK_RET(GetJsonArrayMemberProperty(superPodList, objIndex, "server_list", superPodServerList, false));
838 0 : for (u32 index = 0; index < superPodServerList.size(); index++) {
839 : // get single super pod server info
840 0 : CHK_RET(GetSingleSuperPodSever(superPodServerList, index, clusterInfo, superPodId));
841 : }
842 0 : return HCCL_SUCCESS;
843 0 : }
844 :
845 0 : HcclResult TopoinfoRanktableConcise::GetSingleSuperPodSever(const nlohmann::json &superPodServerList, u32 objIndex,
846 : RankTable_t &clusterInfo, std::string superPodId)
847 : {
848 0 : std::string serverId;
849 0 : HcclResult ret = HCCL_SUCCESS;
850 : do {
851 : // 获取server_id
852 0 : ret = GetJsonArrayMemberProperty(superPodServerList, objIndex, "server_id", serverId, true);
853 0 : CHK_PRT_BREAK(ret != HCCL_SUCCESS,
854 : HCCL_ERROR("[Get][SingleSuperPodSever]errNo[0x%016llx]server_id is not found or invalid",
855 : HCCL_ERROR_CODE(ret)),);
856 :
857 : // 将super_pod_list下的server_id在资源池中进行校验
858 0 : ret = CheckUniqueAndInsertPool(JsonUniqueInfoType::UNIQUE_INFO_TYPE_SERVER_ID, serverId,
859 0 : JsonCheckOpType::CHECK_OP_TYPE_FIND);
860 0 : CHK_PRT_BREAK(ret != HCCL_SUCCESS,
861 : HCCL_ERROR("[Get][SingleSuperPodSever]errNo[0x%016llx]server_id[%s] is not found in server_list",
862 : HCCL_ERROR_CODE(ret), serverId.c_str()),);
863 0 : HCCL_DEBUG("[%s.json]super_pod_list -> server_id: [%s]", fileName_.c_str(), serverId.c_str());
864 : } while (0);
865 :
866 : // server_id未找到, 或与server_list中的server_id不一致
867 0 : if (ret != HCCL_SUCCESS) {
868 0 : RPT_INPUT_ERR(true, "EI0014", std::vector<std::string>({ "value", "variable" ,"expect" }),
869 : std::vector<std::string>({ serverId, "server_id",
870 : "a valid non-empty string." }));
871 0 : return ret;
872 : }
873 :
874 0 : u32 superPodIdx = INVALID_UINT;
875 0 : GenerateSuperPodIdx(superPodId, superPodIdx);
876 :
877 0 : bool isFound = false;
878 0 : for (RankInfo_t& rankInfo : clusterInfo.rankList) {
879 0 : if (rankInfo.serverId == serverId) {
880 0 : rankInfo.superPodId = superPodId;
881 0 : rankInfo.superPodIdx = superPodIdx;
882 0 : isFound = true;
883 : }
884 : }
885 0 : CHK_PRT_RET(isFound == false,
886 : HCCL_ERROR("[Get][SingleSuperPodSever]server_id[%s] in super_pod_list is not in server_list",
887 : serverId.c_str()), HCCL_E_PARA);
888 :
889 0 : HCCL_DEBUG("[%s.json]super_pod_list -> server_id[%s], super_pod_id[%s]",
890 : fileName_.c_str(), serverId.c_str(), superPodId.c_str());
891 :
892 0 : return HCCL_SUCCESS;
893 0 : }
894 :
895 0 : HcclResult TopoinfoRanktableConcise::CheckSuperPodInfo(RankTable_t &clusterInfo) const
896 : {
897 0 : std::map<std::string, std::set<std::string>> superPodMap; // superPodId -> serverId
898 0 : std::map<std::string, std::set<u32>> superPodSdidMap; // super_pod_id -> superDeviceId
899 0 : for (RankInfo_t& rankInfo : clusterInfo.rankList) {
900 : // 超节点模式下, 校验superPodId和sdid值有效
901 0 : CHK_PRT_RET(rankInfo.superPodId.empty() || rankInfo.superDeviceId == INVALID_UINT,
902 : HCCL_ERROR("[Check][SuperPodInfo]superDeviceId[0x%x] or superPod[%s] in rank[%u] is invalid",
903 : rankInfo.superDeviceId, rankInfo.superPodId.c_str(), rankInfo.rankId), HCCL_E_PARA);
904 :
905 0 : auto it = superPodMap.find(rankInfo.superPodId);
906 0 : if (it == superPodMap.end()) {
907 0 : std::set<std::string> serverIdSet;
908 0 : serverIdSet.insert(rankInfo.serverId);
909 0 : superPodMap.insert({rankInfo.superPodId, serverIdSet});
910 0 : } else if (it->second.find(rankInfo.serverId) == it->second.end()) {
911 0 : it->second.insert(rankInfo.serverId);
912 : }
913 : // 用户忘记配置，superDeviceId等于无效值
914 0 : CHK_PRT_RET(rankInfo.superDeviceId == INVALID_UINT,
915 : HCCL_ERROR("[Check][SuperPodInfo]superDeviceId[0x%x] is invalid in rankId[%u], "
916 : "the configuration may be missing, please check the ranktable config!",
917 : rankInfo.superDeviceId, rankInfo.rankId), HCCL_E_PARA);
918 :
919 0 : auto iter = superPodSdidMap.find(rankInfo.superPodId);
920 0 : if (iter == superPodSdidMap.end()) {
921 0 : std::set<u32> superDeviceIdSet;
922 0 : superDeviceIdSet.insert(rankInfo.superDeviceId);
923 0 : superPodSdidMap.insert({rankInfo.superPodId, superDeviceIdSet});
924 0 : } else if (iter->second.find(rankInfo.superDeviceId) == iter->second.end()) {
925 0 : iter->second.insert(rankInfo.superDeviceId);
926 : } else {
927 : // 超节点内superDeviceId在超节点内唯一
928 0 : CHK_PRT_RET(iter->second.find(rankInfo.superDeviceId) != iter->second.end(),
929 : HCCL_ERROR("[Verify][SuperPodInfo]superDeviceId[0x%x] in superPod[%s]"
930 : "is already exist.",
931 : rankInfo.superDeviceId, iter->first.c_str()),
932 : HCCL_E_PARA);
933 : }
934 : }
935 :
936 0 : u32 serverNumTotal = 0;
937 0 : for (auto it = superPodMap.begin(); it != superPodMap.end(); ++it) {
938 0 : serverNumTotal += it->second.size();
939 : }
940 :
941 : // 校验super_pod_list和原有server_list的server数量一致
942 0 : CHK_PRT_RET(serverNumTotal != clusterInfo.serverNum,
943 : HCCL_ERROR("[Get][SuperPodList]serverNum[%u] in super_pod_list and serverNum[%u] in server_list "\
944 : "are inconsistent", serverNumTotal, clusterInfo.serverNum), HCCL_E_PARA);
945 0 : return HCCL_SUCCESS;
946 0 : }
|
