Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "topoinfo_ranktableConcise.h"
12 : #include <string>
13 : #include <fstream>
14 : #include <algorithm>
15 : #include <map>
16 : #include <vector>
17 : #include <unistd.h>
18 : #include <chrono>
19 : #include <iostream>
20 : #include <arpa/inet.h>
21 :
22 : #include "externalinput.h"
23 : #include "log.h"
24 : // ltm指定config路径
25 : #include "common/src/config.h"
26 : #include "workflow_pub.h"
27 : #include "device_capacity.h"
28 :
29 : using namespace std;
30 : using namespace hccl;
31 :
32 : constexpr u32 MAX_PORT_NUMBER = 65535;
33 : constexpr u32 HCCL_SOCKET_PORT_RANGE_AUTO = 0;
34 : constexpr u32 HCCL_DEVICE_PORT_DEFAULT = 16666;
35 : constexpr u32 HCCL_BACKUP_DEVICE_PORT_DEFAULT = 16667;
36 :
37 5 : TopoinfoRanktableConcise::TopoinfoRanktableConcise(const std::string& rankTableM, const std::string& identify)
38 : : TopoInfoRanktableParser(rankTableM, identify),
39 5 : isInterSuperPodRetryEnable_(GetExternalInputInterSuperPodRetryEnable())
40 5 : {}
41 :
42 5 : TopoinfoRanktableConcise::~TopoinfoRanktableConcise() { devIp2ObjIndex_.clear(); }
43 :
44 0 : HcclResult TopoinfoRanktableConcise::Init()
45 : {
46 0 : CHK_RET(LoadRankTableString(rankTableFile_));
47 0 : CHK_RET(ParserClusterInfo(params_, rankTable_));
48 0 : return HCCL_SUCCESS;
49 : }
50 :
51 0 : HcclResult TopoinfoRanktableConcise::GetClusterInfo(RankTable_t& clusterInfo)
52 : {
53 0 : clusterInfo.nicDeploy = rankTable_.nicDeploy;
54 0 : clusterInfo.deviceNum = rankTable_.deviceNum;
55 0 : clusterInfo.serverNum = rankTable_.serverNum;
56 0 : clusterInfo.superPodNum = rankTable_.superPodNum;
57 0 : clusterInfo.rankNum = rankTable_.rankNum;
58 0 : clusterInfo.rankList = rankTable_.rankList;
59 0 : clusterInfo.serverList = rankTable_.serverList;
60 0 : return HCCL_SUCCESS;
61 : }
62 :
63 0 : HcclResult TopoinfoRanktableConcise::GetSelfClusterInfo(HcclCommParams& params)
64 : {
65 : // 获取芯片类型信息
66 0 : params.deviceType = params_.deviceType;
67 0 : params.rank = params_.rank;
68 0 : params.userRank = params_.rank;
69 0 : params.logicDevId = params_.logicDevId;
70 0 : params.totalRanks = params_.totalRanks;
71 0 : params.serverId = params_.serverId;
72 0 : params.commPortConfig.devPortSwitchOn = params_.commPortConfig.devPortSwitchOn;
73 0 : return HCCL_SUCCESS;
74 : }
75 :
76 0 : HcclResult TopoinfoRanktableConcise::GetClusterInfo(hccl::HcclCommParams& params, hccl::RankTable_t& rankTable)
77 : {
78 0 : CHK_RET(GetClusterInfo(rankTable));
79 0 : CHK_RET(GetSelfClusterInfo(params));
80 0 : return HCCL_SUCCESS;
81 : }
82 :
83 0 : HcclResult TopoinfoRanktableConcise::SetIsInterSuperPodRetryEnable(bool isRetryEnable)
84 : {
85 0 : isInterSuperPodRetryEnable_ = isRetryEnable;
86 0 : return HCCL_SUCCESS;
87 : }
88 :
89 0 : void TopoinfoRanktableConcise::DetectNicDepoly(RankTable_t& rankTable)
90 : {
91 : // 只有当hostIp有效而且deviceIp无效时，才使用HOST侧网卡部署，目前策略要求集群中所有卡的deploy
92 : // 形式一致，所以取ranklist[0]的方式即可
93 0 : auto isIpInvalid = [](const std::vector<HcclIpAddress>& deviceIp) -> bool {
94 0 : for (auto& ip : deviceIp) {
95 0 : if (!ip.IsInvalid()) { // 遍历vector中所有的IP，只要有任意一个IP有效就认为是有效的，那么返回false
96 0 : return false;
97 : }
98 : }
99 0 : return true;
100 : };
101 :
102 0 : rankTable.nicDeploy = (!rankTable.rankList.empty() && !rankTable.rankList[0].hostIp.IsInvalid()
103 0 : && isIpInvalid(rankTable.rankList[0].deviceInfo.deviceIp)) ?
104 : NICDeployment::NIC_DEPLOYMENT_HOST :
105 : NICDeployment::NIC_DEPLOYMENT_DEVICE;
106 0 : }
107 :
108 0 : HcclResult TopoinfoRanktableConcise::ParserClusterInfo(hccl::HcclCommParams& params, hccl::RankTable_t& rankTable)
109 : {
110 0 : if (!IsTaskNumCalMode()) {
111 0 : CHK_RET(hrtGetDeviceType(params.deviceType));
112 : }
113 : // 获取ranktable info信息
114 0 : CHK_RET(GetRanktableInfo(rankTable));
115 0 : for (auto& rankInfo : rankTable.rankList) {
116 0 : HCCL_DEBUG(
117 : "ParserClusterInfo serverId %s, rankId %u, superDeviceId 0x%x, superPodId %s", rankInfo.serverId.c_str(),
118 : rankInfo.rankId, rankInfo.superDeviceId, rankInfo.superPodId.c_str());
119 : }
120 :
121 0 : if (IsTaskNumCalMode()) {
122 0 : HCCL_INFO("[ParserClusterInfo] get task num cal mode.");
123 0 : return HCCL_SUCCESS;
124 : }
125 :
126 0 : std::sort(
127 0 : rankTable.rankList.begin(), rankTable.rankList.end(), [&](const RankInfo_t& a, const RankInfo_t& b) -> bool {
128 0 : return a.rankId < b.rankId;
129 : });
130 :
131 0 : u32 rankId = INVALID_VALUE_RANKID;
132 0 : if (IsAllDigit(identify_.c_str()) != HCCL_SUCCESS
133 0 : || SalStrToULong(identify_, HCCL_BASE_DECIMAL, rankId) != HCCL_SUCCESS) {
134 0 : RPT_INPUT_ERR(
135 : true, "EI0003", std::vector<std::string>({"ccl_op", "value", "parameter", "expect"}),
136 : std::vector<std::string>(
137 : {"HcclCommInitRootInfo", identify_.c_str(), "RANK_ID", "a non-negative decimal number"}));
138 0 : HCCL_ERROR(
139 : "[%s][%s]errNo[0x%016llx] rank_id[%s] is invalid", LOG_KEYWORDS_INIT_GROUP.c_str(),
140 : LOG_KEYWORDS_INVALID_ARGUMENT.c_str(), HCOM_ERROR_CODE(HCCL_E_PARA), identify_.c_str());
141 0 : return HCCL_E_PARA;
142 : }
143 :
144 0 : if (rankId >= rankTable.rankList.size()) {
145 0 : RPT_ENV_ERR(
146 : true, "EI0014", std::vector<std::string>({"value", "variable", "expect"}),
147 : std::vector<std::string>(
148 : {std::to_string(rankId), "rank_id", std::to_string(rankTable.rankList[rankId].rankId)}));
149 0 : HCCL_ERROR(
150 : "[%s][%s]rankId[%u] is invalid", LOG_KEYWORDS_INIT_GROUP.c_str(), LOG_KEYWORDS_RANKTABLE_CHECK.c_str(),
151 : rankId);
152 0 : return HCCL_E_PARA;
153 : }
154 :
155 0 : RPT_ENV_ERR(
156 : rankId != rankTable.rankList[rankId].rankId, "EI0014",
157 : std::vector<std::string>({"value", "variable", "expect"}),
158 : std::vector<std::string>(
159 : {std::to_string(rankId), "rank_id", std::to_string(rankTable.rankList[rankId].rankId)}));
160 0 : CHK_PRT_RET(
161 : rankId != rankTable.rankList[rankId].rankId,
162 : HCCL_ERROR(
163 : "[%s][%s]check rankList[%u] rankId[%u] failed", LOG_KEYWORDS_INIT_GROUP.c_str(),
164 : LOG_KEYWORDS_RANKTABLE_CHECK.c_str(), rankId, rankTable.rankList[rankId].rankId),
165 : HCCL_E_UNAVAIL);
166 :
167 0 : if (rankId < rankTable.rankList.size()
168 0 : && rankTable.rankList[rankId].deviceInfo.nicDeploy == NICDeployment::NIC_DEPLOYMENT_HOST) {
169 0 : rankTable.nicDeploy = NICDeployment::NIC_DEPLOYMENT_HOST;
170 0 : SetHostUseDevNicFlag(false);
171 : } else {
172 0 : DetectNicDepoly(rankTable);
173 : }
174 0 : CHK_RET(CheckNicDeployConsistence(rankTable, rankTable.nicDeploy));
175 :
176 0 : u32 devId = rankTable.rankList[rankId].deviceInfo.devicePhyId;
177 0 : CHK_RET(hrtGetDevice(¶ms.logicDevId));
178 :
179 0 : u32 phyId = 0;
180 0 : CHK_RET(hrtGetDevicePhyIdByIndex(static_cast<u32>(params.logicDevId), phyId));
181 0 : RPT_ENV_ERR(
182 : phyId != static_cast<u32>(devId), "EI0014", std::vector<std::string>({"value", "variable", "expect"}),
183 : std::vector<std::string>({std::to_string(phyId), "devicePhyId", std::to_string(static_cast<u32>(devId))}));
184 0 : CHK_PRT_RET(
185 : phyId != static_cast<u32>(devId),
186 : HCCL_ERROR(
187 : "[%s][%s]ranktable config devId[%d],but local devId[%u]", LOG_KEYWORDS_INIT_GROUP.c_str(),
188 : LOG_KEYWORDS_RANKTABLE_CHECK.c_str(), devId, phyId),
189 : HCCL_E_UNAVAIL);
190 :
191 0 : params.rank = rankId;
192 0 : params.serverId = rankTable.rankList[rankId].serverId;
193 0 : params.totalRanks = rankTable.rankNum;
194 :
195 0 : return HCCL_SUCCESS;
196 0 : }
197 :
198 0 : HcclResult TopoinfoRanktableConcise::CheckNicDeployConsistence(RankTable_t& clusterInfo, NICDeployment deploy) const
199 : {
200 0 : CHK_PRT_RET(
201 : clusterInfo.rankList.size() == 0, HCCL_DEBUG("rank list size is 0, skip nic deeply check."), HCCL_SUCCESS);
202 :
203 0 : for (auto& it : clusterInfo.rankList) {
204 0 : CHK_PRT_RET(
205 : // 因为部分场景下，网卡部署在device侧，但是允许deviceIP无效，所以不检测该场景，只检测使用HOST但是hostIp无效
206 : (deploy == NICDeployment::NIC_DEPLOYMENT_HOST && it.hostIp.IsInvalid()),
207 : HCCL_ERROR(
208 : "[Get][RanktableInfo]errNo"
209 : "[0x%016llx] hostIp config bettewn ranks is different.",
210 : HCOM_ERROR_CODE(HCCL_E_PARA)),
211 : HCCL_E_PARA);
212 : }
213 0 : return HCCL_SUCCESS;
214 : }
215 :
216 0 : HcclResult TopoinfoRanktableConcise::GetRanktableInfo(RankTable_t& clusterInfo)
217 : {
218 : // server_list
219 0 : CHK_RET(GetServerList(fileContent_, clusterInfo));
220 0 : CHK_RET(GetSuperPodList(fileContent_, clusterInfo));
221 0 : CHK_RET(GetDevNum(clusterInfo.rankList, clusterInfo.deviceNum));
222 0 : clusterInfo.rankNum = clusterInfo.rankList.size();
223 0 : return HCCL_SUCCESS;
224 : }
225 :
226 0 : HcclResult TopoinfoRanktableConcise::GetServerList(const nlohmann::json& obj, RankTable_t& clusterInfo)
227 : {
228 0 : clusterInfo.serverList.clear();
229 0 : nlohmann::json serverList;
230 0 : CHK_RET(GetJsonProperty(obj, "server_list", serverList, false));
231 0 : HCCL_DEBUG("[%s.json] -> server_list: size:[%zu]", fileName_.c_str(), serverList.size());
232 :
233 : // 获取serverCount并校验
234 0 : std::string serverCount;
235 0 : HcclResult ret = GetJsonProperty(obj, "server_count", serverCount, true);
236 0 : if (ret == HCCL_SUCCESS) {
237 0 : HCCL_DEBUG("[%s.json] -> group_count: [%s]", fileName_.c_str(), serverCount.c_str());
238 0 : CHK_RET(SalStrToULong(serverCount, HCCL_BASE_DECIMAL, clusterInfo.serverNum));
239 :
240 : // 校验serverCount
241 0 : if (serverList.size() != clusterInfo.serverNum) {
242 0 : RPT_INPUT_ERR(
243 : true, "EI0014", std::vector<std::string>({"value", "variable", "expect"}),
244 : std::vector<std::string>({serverCount, "server_count", std::to_string(serverList.size())}));
245 0 : HCCL_ERROR(
246 : "[%s][%s]errNo[0x%016llx] serverList size[%zu] neq server num[%u]", LOG_KEYWORDS_INIT_GROUP.c_str(),
247 : LOG_KEYWORDS_RANKTABLE_CHECK.c_str(), HCOM_ERROR_CODE(HCCL_E_PARA), serverList.size(),
248 : clusterInfo.serverNum);
249 0 : return HCCL_E_PARA;
250 : }
251 0 : } else if (ret == HCCL_E_NOT_FOUND) {
252 0 : clusterInfo.serverNum = serverList.size();
253 0 : HCCL_WARNING(
254 : "[Get][ServerList]The 'server_count' in ranktable is not found, "
255 : "set server_count to server_list size[%u]",
256 : serverList.size());
257 : } else {
258 0 : HCCL_ERROR("[Get][ServerList]get server_count error, ret[%d]", ret);
259 0 : return ret;
260 : }
261 0 : if (clusterInfo.serverNum == 0) {
262 0 : RPT_INPUT_ERR(
263 : true, "EI0014", std::vector<std::string>({"value", "variable", "expect"}),
264 : std::vector<std::string>({std::to_string(clusterInfo.serverNum), "server_count", "is a positive integer"}));
265 0 : HCCL_ERROR(
266 : "[%s][%s]errNo[0x%016llx] server num is zero", LOG_KEYWORDS_INIT_GROUP.c_str(),
267 : LOG_KEYWORDS_RANKTABLE_CHECK.c_str(), HCOM_ERROR_CODE(HCCL_E_PARA));
268 0 : return HCCL_E_PARA;
269 : }
270 0 : HCCL_DEBUG("[Get][ServerList]serverNum is [%u]", clusterInfo.serverNum);
271 :
272 0 : for (u32 index = 0; index < serverList.size(); index++) {
273 : // get single server info
274 0 : CHK_RET(GetSingleServer(serverList, index, clusterInfo));
275 : }
276 :
277 0 : for (u32 index = 0; index < clusterInfo.rankList.size(); index++) {
278 0 : CHK_RET(VerifyBackupDeviceIpAndPort(clusterInfo.rankList, index));
279 : }
280 : // 获取device
281 0 : return HCCL_SUCCESS;
282 0 : }
283 :
284 : HcclResult
285 1 : TopoinfoRanktableConcise::GetSingleServer(const nlohmann::json& serverListObj, u32 objIndex, RankTable_t& clusterInfo)
286 : {
287 : HcclResult ret;
288 1 : std::string serverId;
289 1 : CHK_RET(GetJsonArrayMemberProperty(serverListObj, objIndex, "server_id", serverId, false));
290 1 : if (serverId.empty()) {
291 0 : HCCL_ERROR(
292 : "[Get][JsonArrayMemberProperty]errNo[0x%016llx] serverId[%s] is empty", HCOM_ERROR_CODE(HCCL_E_PARA),
293 : serverId.c_str());
294 0 : return HCCL_E_PARA;
295 : }
296 :
297 1 : if (serverId.length() > SERVERID_MAX_LENGTH) {
298 1 : HCCL_ERROR(
299 : "[Get][GetSingleServer]errNo[0x%016llx] serverId[%s] length[%u] is more than %u",
300 : HCOM_ERROR_CODE(HCCL_E_PARA), serverId.c_str(), serverId.length(), SERVERID_MAX_LENGTH);
301 1 : return HCCL_E_PARA;
302 : }
303 :
304 : // 将serverId添加到资源池,内部会进行IP地址校验，如果资源池中有serverId，则报错
305 0 : CHK_RET(CheckUniqueAndInsertPool(
306 : JsonUniqueInfoType::UNIQUE_INFO_TYPE_SERVER_ID, serverId, JsonCheckOpType::CHECK_OP_TYPE_INSERT));
307 0 : HCCL_DEBUG("server id[%u]:[%s]", objIndex, serverId.c_str());
308 :
309 : u32 serverIdx;
310 0 : GenerateServerIdx(serverId, serverIdx);
311 0 : HCCL_DEBUG("server id[%u]:[%s], serverIdx[%u]", objIndex, serverId.c_str(), serverIdx);
312 0 : std::string hostNicIp;
313 0 : HcclIpAddress hostIp;
314 0 : ret = GetJsonArrayMemberProperty(serverListObj, objIndex, "host_ip", hostNicIp, true);
315 0 : CHK_PRT_RET(
316 : ret != HCCL_SUCCESS && ret != HCCL_E_NOT_FOUND, HCCL_ERROR("[Get][SingleServer]get host ip error"), ret);
317 0 : HCCL_DEBUG("[%s.json] -> host_ip: [%s]. ret[%u]", fileName_.c_str(), hostNicIp.c_str(), ret);
318 0 : if (ret != HCCL_E_NOT_FOUND) {
319 0 : CHK_RET(ConvertIpAddress(hostNicIp, hostIp));
320 : }
321 :
322 : // 处理ranklist
323 0 : ret = GetDeviceList(serverListObj, objIndex, clusterInfo, serverId, serverIdx, hostIp);
324 0 : CHK_PRT_RET(
325 : ret != HCCL_SUCCESS, HCCL_ERROR("[Get][SingleServer]get dev list error:serverId[%s]", serverId.c_str()), ret);
326 :
327 0 : return HCCL_SUCCESS;
328 1 : }
329 :
330 0 : HcclResult TopoinfoRanktableConcise::GetDeviceList(
331 : const nlohmann::json& serverListObj, u32 objIndex, RankTable_t& clusterInfo, std::string& serverId, u32& serverIdx,
332 : HcclIpAddress& hostIp)
333 : {
334 0 : HCCL_DEBUG("Get GetDeviceList[%u]: serverId[%s]", objIndex, serverId.c_str());
335 :
336 0 : nlohmann::json deviceList;
337 0 : CHK_RET(GetJsonArrayMemberProperty(serverListObj, objIndex, "device", deviceList, false));
338 :
339 0 : HCCL_DEBUG("[%s.json] -> device_list: size:%zu", fileName_.c_str(), deviceList.size());
340 :
341 0 : CHK_PRT_RET(deviceList.size() == 0, HCCL_ERROR("[Get][DeviceList]deviceList size is zero"), HCCL_E_PARA);
342 :
343 0 : for (u32 index = 0; index < deviceList.size(); index++) {
344 : // get single server info
345 0 : CHK_RET(GetSingleDevice(deviceList, index, clusterInfo, serverId, serverIdx, hostIp));
346 : }
347 :
348 : // 检查devip的数目是否一致
349 0 : u32 rankListSize = clusterInfo.rankList.size();
350 0 : CHK_PRT_RET(rankListSize == 0, HCCL_ERROR("[Get][DeviceList]get ranklist is zero"), HCCL_E_PARA);
351 :
352 0 : u32 checkDeviceIpSize = 0;
353 0 : for (u32 index = 0; index < clusterInfo.rankList.size(); index++) {
354 0 : if (index == 0) {
355 0 : checkDeviceIpSize = clusterInfo.rankList[0].deviceInfo.deviceIp.size();
356 : }
357 0 : if (clusterInfo.rankList[index].deviceInfo.deviceIp.size() != checkDeviceIpSize) {
358 0 : HCCL_ERROR(
359 : "[Get][DeviceList]device[%u] size[%u] neq first device size[%u] error", index,
360 : clusterInfo.rankList[index].deviceInfo.deviceIp.size(), checkDeviceIpSize);
361 0 : return HCCL_E_PARA;
362 : }
363 : }
364 :
365 0 : return HCCL_SUCCESS;
366 0 : }
367 :
368 : HcclResult
369 0 : TopoinfoRanktableConcise::GetSingleNicInfo(const nlohmann::json& serverListObj, u32 objIndex, RankInfo_t& rankinfo)
370 : {
371 0 : std::string netPosition;
372 0 : rankinfo.deviceInfo.nicDeploy = NICDeployment::NIC_DEPLOYMENT_DEVICE;
373 0 : HcclResult ret = GetJsonArrayMemberProperty(serverListObj, objIndex, "net_position", netPosition, true);
374 0 : CHK_PRT_RET(
375 : ret != HCCL_SUCCESS && ret != HCCL_E_NOT_FOUND, HCCL_ERROR("[Get][GetSingleNicInfo]get net position error"),
376 : ret);
377 0 : HCCL_DEBUG("[%s.json] -> net_position: [%s]. ret[%u]", fileName_.c_str(), netPosition.c_str(), ret);
378 0 : if (ret != HCCL_E_NOT_FOUND) {
379 0 : if (netPosition == "host") { // 默认值为Device模式
380 0 : rankinfo.deviceInfo.nicDeploy = NICDeployment::NIC_DEPLOYMENT_HOST;
381 : } else {
382 0 : rankinfo.deviceInfo.nicDeploy = NICDeployment::NIC_DEPLOYMENT_DEVICE;
383 : }
384 : }
385 :
386 0 : std::string netProto;
387 0 : ret = GetJsonArrayMemberProperty(serverListObj, objIndex, "net_protocol", netProto, true);
388 0 : CHK_PRT_RET(
389 : ret != HCCL_SUCCESS && ret != HCCL_E_NOT_FOUND, HCCL_ERROR("[Get][GetSingleNicInfo]get net protocol error"),
390 : ret);
391 0 : HCCL_DEBUG("[%s.json] -> net_protocol: [%s]. ret[%u]", fileName_.c_str(), netProto.c_str(), ret);
392 0 : if (ret != HCCL_E_NOT_FOUND) {
393 0 : if (netProto == "rdma") { // 未配置则使用默认值，不处理，使用原有逻辑
394 0 : SetExternalInputProtocolType(ProtocolType::RDMA);
395 0 : rankinfo.deviceInfo.proto = u32(ProtocolType::RDMA);
396 0 : } else if (netProto == "tcp") {
397 0 : SetExternalInputProtocolType(ProtocolType::TCP);
398 0 : rankinfo.deviceInfo.proto = u32(ProtocolType::TCP);
399 : }
400 : }
401 :
402 0 : return HCCL_SUCCESS;
403 0 : }
404 :
405 0 : HcclResult TopoinfoRanktableConcise::GetSingleDevice(
406 : const nlohmann::json& deviceListObj, u32 objIndex, RankTable_t& clusterInfo, std::string& serverId, u32& serverIdx,
407 : HcclIpAddress& hostIp)
408 : {
409 : // 获取rank_id
410 0 : std::string rankId;
411 0 : CHK_RET(GetJsonArrayMemberProperty(deviceListObj, objIndex, "rank_id", rankId, false));
412 0 : HCCL_DEBUG("[%s.json] -> rank_id: [%s]", fileName_.c_str(), rankId.c_str());
413 :
414 : // 获取device type
415 0 : DevType deviceType = DevType::DEV_TYPE_COUNT;
416 0 : CHK_RET(hrtGetDeviceType(deviceType));
417 :
418 : // 获取device_id
419 0 : std::string strDevid;
420 0 : CHK_RET(GetJsonArrayMemberProperty(deviceListObj, objIndex, "device_id", strDevid, false));
421 :
422 0 : u32 devicePhyId = 0;
423 0 : CHK_RET(SalStrToULong(strDevid, HCCL_BASE_DECIMAL, devicePhyId));
424 :
425 : u32 maxDeviceNum;
426 0 : CHK_RET(GetMaxDevNum(maxDeviceNum));
427 0 : if ((deviceType == DevType::DEV_TYPE_310P3 || deviceType == DevType::DEV_TYPE_910B
428 0 : || deviceType == DevType::DEV_TYPE_910_93)
429 0 : && devicePhyId > (maxDeviceNum - 1)) {
430 : // deviceid in 0 ~ maxDeviceNum
431 0 : HCCL_ERROR(
432 : "[Get][SingleDevice]errNo[0x%016llx] device_id[%u] more than maxDeviceNum[%u] invalid",
433 : HCOM_ERROR_CODE(HCCL_E_PARA), devicePhyId, maxDeviceNum);
434 0 : return HCCL_E_PARA;
435 0 : } else if (
436 0 : (deviceType != DevType::DEV_TYPE_310P3 && deviceType != DevType::DEV_TYPE_910B
437 0 : && deviceType != DevType::DEV_TYPE_910_93)
438 0 : && devicePhyId > (HCCL_AISERVER_DEVICE_NUM - 1)) {
439 : // deviceid in 0 ~ 7
440 0 : HCCL_ERROR(
441 : "[Get][SingleDevice]errNo[0x%016llx] device_id[%u] more than 7 is invalid", HCOM_ERROR_CODE(HCCL_E_PARA),
442 : devicePhyId);
443 0 : return HCCL_E_PARA;
444 : }
445 0 : HCCL_DEBUG("[%s.json] -> device_id: [%s]", fileName_.c_str(), strDevid.c_str());
446 :
447 0 : RankInfo_t rankinfo;
448 0 : rankinfo.serverId = serverId;
449 0 : rankinfo.serverIdx = serverIdx;
450 0 : rankinfo.hostIp = hostIp;
451 0 : rankinfo.deviceInfo.devicePhyId = devicePhyId;
452 :
453 0 : CHK_RET(GetSingleDeviceHostPort(deviceListObj, objIndex, rankinfo));
454 0 : CHK_RET(GetSingleDeviceIp(deviceListObj, objIndex, clusterInfo, rankinfo, deviceType, rankinfo.hostIp.IsInvalid()));
455 0 : CHK_RET(GetSingleDevicePort(deviceListObj, objIndex, rankinfo));
456 0 : CHK_RET(GetSingleBackupDeviceIp(deviceListObj, objIndex, rankinfo));
457 0 : CHK_RET(GetSingleNicInfo(deviceListObj, objIndex, rankinfo));
458 :
459 0 : if (SalStrToULong(rankId, HCCL_BASE_DECIMAL, rankinfo.rankId) != HCCL_SUCCESS) {
460 0 : RPT_INPUT_ERR(
461 : true, "EI0014", std::vector<std::string>({"value", "variable", "expect"}),
462 : std::vector<std::string>({rankId, "rank_id", "is a non-negative integer."}));
463 0 : HCCL_ERROR(
464 : "[%s][%s]errNo[0x%016llx] rankid[%s] is invalid", LOG_KEYWORDS_INIT_GROUP.c_str(),
465 : LOG_KEYWORDS_RANKTABLE_CHECK.c_str(), HCOM_ERROR_CODE(HCCL_E_PARA), rankId.c_str());
466 0 : return HCCL_E_PARA;
467 : }
468 :
469 0 : rankinfo.podName = ""; // podname在新场景下置空
470 0 : rankId = "";
471 :
472 0 : string version;
473 0 : CHK_RET(GetRanktableVersion(version));
474 0 : if (version.compare(SUPERPOD_CLUSTER_VERSION) != 0) {
475 0 : clusterInfo.rankList.push_back(rankinfo);
476 0 : HCCL_DEBUG(
477 : "[%s.json]->rankId[%u], serverId[%s], devicePhyId[%d]", fileName_.c_str(), rankinfo.rankId,
478 : rankinfo.serverId.c_str(), rankinfo.deviceInfo.devicePhyId);
479 :
480 0 : return HCCL_SUCCESS;
481 : }
482 :
483 0 : CHK_RET(GetSingleSuperDeviceId(deviceListObj, objIndex, clusterInfo, rankinfo));
484 0 : clusterInfo.rankList.push_back(rankinfo);
485 0 : return HCCL_SUCCESS;
486 0 : }
487 :
488 2 : HcclResult TopoinfoRanktableConcise::SplitString(
489 : const std::string& str, const std::string& strC, std::vector<std::string>& strVector) const
490 : {
491 2 : std::string::size_type startPos = 0;
492 2 : std::string::size_type foundPos = str.find(strC);
493 :
494 2 : while (foundPos != std::string::npos) {
495 0 : strVector.push_back(str.substr(startPos, foundPos - startPos));
496 0 : startPos = foundPos + strC.size();
497 0 : foundPos = str.find(strC, startPos);
498 : }
499 2 : if (startPos != str.length()) {
500 2 : strVector.push_back(str.substr(startPos));
501 : }
502 2 : return HCCL_SUCCESS;
503 : }
504 :
505 0 : HcclResult TopoinfoRanktableConcise::GetSingleDeviceIp(
506 : const nlohmann::json& deviceListObj, u32 objIndex, RankTable_t& clusterInfo, RankInfo_t& rankinfo,
507 : DevType deviceType, bool invalidHostIp)
508 : {
509 : // 获取device_ip （可能有多个）
510 0 : std::string deviceIp;
511 0 : HcclResult ret = GetJsonArrayMemberProperty(deviceListObj, objIndex, "device_ip", deviceIp, true);
512 : // 多机和走roce网卡ranktable必须有“device_ip”字段，单机可以没有
513 0 : if (clusterInfo.serverNum > 1 || (GetExternalInputIntraRoceSwitch() == 1)) {
514 : // 如果没有配置HostIp，那么deviceIp必须有效，否则出错
515 : bool isDeviceIpError
516 0 : = (ret != HCCL_SUCCESS && invalidHostIp) || (deviceType == DevType::DEV_TYPE_910B && deviceIp == "");
517 : // A3超节点内使用vnic建链，不需要device ip
518 0 : if (deviceType == DevType::DEV_TYPE_910_93 && clusterInfo.superPodNum <= 1
519 0 : && GetExternalInputIntraRoceSwitch() == 0) {
520 0 : isDeviceIpError = false;
521 : }
522 0 : RPT_INPUT_ERR(
523 : isDeviceIpError, "EI0014", std::vector<std::string>({"value", "variable", "expect"}),
524 : std::vector<std::string>({deviceIp, "device_ip", "a valid IP address"}));
525 0 : CHK_PRT_RET(
526 : isDeviceIpError,
527 : HCCL_ERROR(
528 : "[%s][%s]'device_ip' is not set correctly, "
529 : "must be set when multi Server or HCCL_INTRA_ROCE_ENABLE enabled",
530 : LOG_KEYWORDS_INIT_GROUP.c_str(), LOG_KEYWORDS_RANKTABLE_CHECK.c_str()),
531 : HCCL_E_PARA);
532 0 : } else if (clusterInfo.serverNum == 1 && ret == HCCL_E_NOT_FOUND) {
533 0 : HcclIpAddress invalidAddr;
534 0 : rankinfo.deviceInfo.deviceIp.push_back(invalidAddr);
535 0 : HCCL_WARNING("[Get][SingleDeviceIp]'device_ip' in ranktable is not set!");
536 0 : return HCCL_SUCCESS;
537 0 : } else {
538 : // 如果没有配置HostIp，那么deviceIp必须有效，否则出错
539 0 : CHK_PRT_RET(
540 : ret != HCCL_SUCCESS && invalidHostIp,
541 : HCCL_ERROR(
542 : "[Get][SingleDeviceIp]errNo[0x%016llx] 'device_ip' is not set correctly", HCOM_ERROR_CODE(HCCL_E_PARA)),
543 : ret);
544 : }
545 0 : HCCL_DEBUG("[%s.json] -> device_ip: [%s]", fileName_.c_str(), deviceIp.c_str());
546 :
547 : // 处理字符串device_ip
548 0 : std::vector<std::string> strDeviceIp;
549 0 : if (deviceIp != "") {
550 0 : CHK_RET(SplitString(deviceIp, ",", strDeviceIp));
551 :
552 0 : CHK_PRT_RET(
553 : strDeviceIp.size() == 0, HCCL_ERROR("[Get][SingleDeviceIp]in device:deviceip size is zero"), HCCL_E_PARA);
554 0 : for (u32 index = 0; index < strDeviceIp.size(); index++) {
555 0 : CHK_RET(CheckUniqueAndInsertPool(
556 : JsonUniqueInfoType::UNIQUE_INFO_TYPE_DEVICE_IP, strDeviceIp[index],
557 : JsonCheckOpType::CHECK_OP_TYPE_INSERT));
558 0 : HcclIpAddress ipAddr;
559 0 : CHK_RET(ConvertIpAddress(strDeviceIp[index], ipAddr));
560 0 : rankinfo.deviceInfo.deviceIp.push_back(ipAddr);
561 0 : devIp2ObjIndex_.emplace(ipAddr.GetReadableIP(), objIndex);
562 0 : }
563 : } else {
564 0 : HcclIpAddress invalidAddr;
565 0 : rankinfo.deviceInfo.deviceIp.push_back(invalidAddr);
566 0 : HCCL_WARNING("objIndex[%u],'device_ip' is not set", objIndex);
567 0 : }
568 :
569 0 : return HCCL_SUCCESS;
570 0 : }
571 :
572 4 : HcclResult TopoinfoRanktableConcise::GetSingleBackupDeviceIp(
573 : const nlohmann::json& deviceListObj, u32 objIndex, RankInfo_t& rankinfo)
574 : {
575 4 : if (params_.deviceType != DevType::DEV_TYPE_910_93 || !GetExternalInputHcclAicpuUnfold()
576 8 : || !isInterSuperPodRetryEnable_) {
577 0 : return HCCL_SUCCESS;
578 : }
579 : // 获取backup_device_ip（可能有多个）
580 4 : std::string backupDeviceIp;
581 4 : HcclResult ret = GetJsonArrayMemberProperty(deviceListObj, objIndex, "backup_device_ip", backupDeviceIp);
582 : // backup_device_ip字段未配置时通过warning日志提示
583 4 : if (ret == HCCL_E_NOT_FOUND) {
584 0 : HcclIpAddress invalidAddr;
585 0 : rankinfo.deviceInfo.backupDeviceIp.push_back(invalidAddr);
586 0 : HCCL_WARNING("[Get][SingleDeviceIp]'backup_device_ip' in ranktable is not set!");
587 0 : return HCCL_SUCCESS;
588 4 : } else if (ret == HCCL_E_PARA) {
589 2 : HCCL_ERROR("[Get][SingleDeviceIp]Get json array member property error");
590 2 : return HCCL_E_PARA;
591 : }
592 2 : HCCL_DEBUG("[%s.json] -> backup_device_ip: [%s]", fileName_.c_str(), backupDeviceIp.c_str());
593 :
594 : // 处理backup_device_ip字符串
595 2 : std::vector<std::string> strBackupDeviceIp;
596 2 : if (backupDeviceIp != "") {
597 4 : CHK_RET(SplitString(backupDeviceIp, ",", strBackupDeviceIp));
598 :
599 2 : CHK_PRT_RET(
600 : strBackupDeviceIp.size() == 0, HCCL_ERROR("[Get][SingleBackupDeviceIp]in device: deviceip size is zero"),
601 : HCCL_E_PARA);
602 3 : for (u32 index = 0; index < strBackupDeviceIp.size(); index++) {
603 2 : CHK_RET(CheckUniqueAndInsertPool(
604 : JsonUniqueInfoType::UNIQUE_INFO_TYPE_BACKUP_DEVICE_IP, strBackupDeviceIp[index],
605 : JsonCheckOpType::CHECK_OP_TYPE_INSERT));
606 1 : HcclIpAddress ipAddr;
607 1 : CHK_RET(ConvertIpAddress(strBackupDeviceIp[index], ipAddr));
608 1 : rankinfo.deviceInfo.backupDeviceIp.push_back(ipAddr);
609 1 : }
610 1 : CHK_RET(GetSingleBackupDevicePort(deviceListObj, objIndex, rankinfo));
611 1 : HCCL_INFO(
612 : "[TopoinfoRanktableConcise][GetSingleBackupDeviceIp]devicePhyId[%u], backupDeviceIp[0]:[%s].",
613 : rankinfo.deviceInfo.devicePhyId, rankinfo.deviceInfo.backupDeviceIp[0].GetReadableIP());
614 : } else {
615 0 : HcclIpAddress invalidAddr;
616 0 : rankinfo.deviceInfo.backupDeviceIp.push_back(invalidAddr);
617 0 : HCCL_WARNING("objIndex[%u],'bakcup_device_ip' is not set", objIndex);
618 0 : }
619 1 : return HCCL_SUCCESS;
620 4 : }
621 :
622 0 : HcclResult TopoinfoRanktableConcise::GetSingleDeviceHostPort(
623 : const nlohmann::json& deviceListObj, u32 objIndex, RankInfo_t& rankinfo)
624 : {
625 0 : std::string hostPortStr;
626 0 : rankinfo.hostPort = HCCL_INVALID_PORT;
627 0 : HcclResult ret = GetJsonArrayMemberProperty(deviceListObj, objIndex, "host_port", hostPortStr, true);
628 0 : CHK_PRT_RET(
629 : ret != HCCL_SUCCESS && ret != HCCL_E_NOT_FOUND,
630 : HCCL_ERROR("[Get][SingleDeviceHostPort]get host port error, deviceIndex[%u]", objIndex), ret);
631 0 : HCCL_DEBUG("[%s.json] -> host_port: [%s]. ret[%u]", fileName_.c_str(), hostPortStr.c_str(), ret);
632 0 : if (ret != HCCL_E_NOT_FOUND) {
633 0 : CHK_RET(SalStrToULong(hostPortStr, HCCL_BASE_DECIMAL, rankinfo.hostPort));
634 0 : CHK_PRT_RET(
635 : rankinfo.hostPort == HCCL_SOCKET_PORT_RANGE_AUTO,
636 : HCCL_ERROR(
637 : "[Get][SingleDeviceHostPort] deviceIndex[%u], please do not use the reserved port number[%u]", objIndex,
638 : HCCL_SOCKET_PORT_RANGE_AUTO),
639 : HCCL_E_PARA);
640 0 : CHK_PRT_RET(
641 : rankinfo.hostPort > MAX_PORT_NUMBER,
642 : HCCL_ERROR(
643 : "[Get][SingleDeviceHostPort] deviceIndex[%u], port number[%u] exceed max port number[%u]", objIndex,
644 : rankinfo.hostPort, MAX_PORT_NUMBER),
645 : HCCL_E_PARA);
646 0 : HCCL_INFO(
647 : "[TopoinfoRanktableConcise][GetSingleDeviceHostPort] deviceIndex[%u], devicePhyId[%u], "
648 : "get HOST port[%u].",
649 : objIndex, rankinfo.deviceInfo.devicePhyId, rankinfo.hostPort);
650 : } else {
651 0 : HCCL_INFO(
652 : "[Get][SingleDeviceHostPort] deviceIndex[%u], 'host_port' in ranktable is not set. "
653 : "Multi-process may not be supported for op retry.",
654 : objIndex);
655 : }
656 0 : return HCCL_SUCCESS;
657 0 : }
658 :
659 : HcclResult
660 2 : TopoinfoRanktableConcise::GetSingleDevicePort(const nlohmann::json& deviceListObj, u32 objIndex, RankInfo_t& rankinfo)
661 : {
662 : // 获取device指定的port；如果用户未配置，则置为缺省值
663 2 : std::string strDevPort;
664 2 : HcclResult ret = GetJsonArrayMemberProperty(deviceListObj, objIndex, "device_port", strDevPort, true);
665 2 : CHK_PRT_RET(
666 : ret != HCCL_SUCCESS && ret != HCCL_E_NOT_FOUND,
667 : HCCL_ERROR("[Get][SingleDevicePort]Get json array member property error."), ret);
668 2 : if (ret == HCCL_E_NOT_FOUND) {
669 0 : rankinfo.deviceInfo.port = HCCL_INVALID_PORT;
670 0 : HCCL_INFO(
671 : "[Get][SingleDevicePort]deviceIndex[%u], devicePhyId[%u], 'device_port' in ranktable is not set. "
672 : "Multi-process may not be supported for device nic.",
673 : objIndex, rankinfo.deviceInfo.devicePhyId);
674 0 : params_.commPortConfig.devPortSwitchOn = false; // 不启用用户指定的port作为device网卡通信的port
675 0 : return HCCL_SUCCESS;
676 : } else {
677 2 : CHK_RET(SalStrToULong(strDevPort, HCCL_BASE_DECIMAL, rankinfo.deviceInfo.port));
678 2 : CHK_PRT_RET(
679 : rankinfo.deviceInfo.port == HCCL_SOCKET_PORT_RANGE_AUTO,
680 : HCCL_ERROR(
681 : "[Get][SingleDevicePort] deviceIndex[%u], devicePhyId[%u], "
682 : "please do not use the reserved port number [%u]. ",
683 : objIndex, rankinfo.deviceInfo.devicePhyId, HCCL_SOCKET_PORT_RANGE_AUTO),
684 : HCCL_E_PARA);
685 2 : CHK_PRT_RET(
686 : rankinfo.deviceInfo.port > MAX_PORT_NUMBER,
687 : HCCL_ERROR(
688 : "[Get][SingleDevicePort] deviceIndex[%u], devicePhyId[%u], "
689 : "port number[%u] exceed max port number[%u]",
690 : objIndex, rankinfo.deviceInfo.devicePhyId, rankinfo.deviceInfo.port, MAX_PORT_NUMBER),
691 : HCCL_E_PARA);
692 2 : params_.commPortConfig.devPortSwitchOn = true; // 启用用户指定的port作为device网卡通信的port
693 : }
694 :
695 : // 获取device指定的vnic port；如果未配置，则与device_port相同（仅用于MasterInfo协商解析Ranktable）
696 2 : std::string strVnicPort;
697 2 : ret = GetJsonArrayMemberProperty(deviceListObj, objIndex, "device_vnic_port", strVnicPort, true);
698 2 : CHK_PRT_RET(
699 : ret != HCCL_SUCCESS && ret != HCCL_E_NOT_FOUND,
700 : HCCL_ERROR("[Get][SingleDevicePort]Get json array member property vnic error."), ret);
701 2 : if (ret == HCCL_E_NOT_FOUND) {
702 1 : rankinfo.deviceInfo.vnicPort = rankinfo.deviceInfo.port;
703 1 : HCCL_INFO(
704 : "[Get][SingleDevicePort]deviceIndex[%u], devicePhyId[%u], "
705 : "'device_vnic_port' in ranktable is not set. ",
706 : objIndex, rankinfo.deviceInfo.devicePhyId);
707 : } else {
708 1 : CHK_RET(SalStrToULong(strVnicPort, HCCL_BASE_DECIMAL, rankinfo.deviceInfo.vnicPort));
709 1 : CHK_PRT_RET(
710 : rankinfo.deviceInfo.vnicPort == HCCL_SOCKET_PORT_RANGE_AUTO,
711 : HCCL_ERROR(
712 : "[Get][SingleDevicePort] deviceIndex[%u], devicePhyId[%u], "
713 : "please do not use the reserved port number [%u] as vnic port number. ",
714 : objIndex, rankinfo.deviceInfo.devicePhyId, HCCL_SOCKET_PORT_RANGE_AUTO),
715 : HCCL_E_PARA);
716 1 : CHK_PRT_RET(
717 : rankinfo.deviceInfo.vnicPort > MAX_PORT_NUMBER,
718 : HCCL_ERROR(
719 : "[Get][SingleDevicePort] deviceIndex[%u], devicePhyId[%u], "
720 : "vnic port number[%u] exceed max port number[%u]",
721 : objIndex, rankinfo.deviceInfo.devicePhyId, rankinfo.deviceInfo.vnicPort, MAX_PORT_NUMBER),
722 : HCCL_E_PARA);
723 : }
724 :
725 2 : HCCL_INFO(
726 : "[TopoinfoRanktableConcise][GetSingleDevicePort] deviceIndex[%u], devicePhyId[%u], get device port[%u], "
727 : "device vnic port[%u].",
728 : objIndex, rankinfo.deviceInfo.devicePhyId, rankinfo.deviceInfo.port, rankinfo.deviceInfo.vnicPort);
729 2 : return HCCL_SUCCESS;
730 2 : }
731 :
732 1 : HcclResult TopoinfoRanktableConcise::GetSingleBackupDevicePort(
733 : const nlohmann::json& deviceListObj, u32 objIndex, RankInfo_t& rankinfo)
734 : {
735 : // 获取device指定的backup port；如果用户未配置，则置为缺省值
736 1 : std::string strBackupPort;
737 1 : HcclResult ret = GetJsonArrayMemberProperty(deviceListObj, objIndex, "backup_device_port", strBackupPort, true);
738 1 : CHK_PRT_RET(
739 : ret != HCCL_SUCCESS && ret != HCCL_E_NOT_FOUND,
740 : HCCL_ERROR("[Get][SingleBackupDevicePort]Get json array member property error."), ret);
741 1 : if (ret == HCCL_E_NOT_FOUND) {
742 1 : rankinfo.deviceInfo.backupPort = HCCL_INVALID_PORT;
743 1 : HCCL_INFO(
744 : "[Get][SingleBackupDevicePort]deviceIndex[%u], devicePhyId[%u], "
745 : "'backup_device_port' in ranktable is not set. Multi-process may not be supported for backup nic.",
746 : objIndex, rankinfo.deviceInfo.devicePhyId);
747 1 : return HCCL_SUCCESS;
748 : }
749 :
750 0 : CHK_RET(SalStrToULong(strBackupPort, HCCL_BASE_DECIMAL, rankinfo.deviceInfo.backupPort));
751 0 : CHK_PRT_RET(
752 : rankinfo.deviceInfo.backupPort == HCCL_SOCKET_PORT_RANGE_AUTO,
753 : HCCL_ERROR(
754 : "[Get][SingleBackupDevicePort] deviceIndex[%u], devicePhyId[%u], "
755 : "please do not use the reserved port number [%u]. ",
756 : objIndex, rankinfo.deviceInfo.devicePhyId, HCCL_SOCKET_PORT_RANGE_AUTO),
757 : HCCL_E_PARA);
758 0 : CHK_PRT_RET(
759 : rankinfo.deviceInfo.backupPort > MAX_PORT_NUMBER,
760 : HCCL_ERROR(
761 : "[Get][SingleBackupDevicePort] deviceIndex[%u], devicePhyId[%u], "
762 : "port number[%u] exceed max port number[%u]",
763 : objIndex, rankinfo.deviceInfo.devicePhyId, rankinfo.deviceInfo.backupPort, MAX_PORT_NUMBER),
764 : HCCL_E_PARA);
765 :
766 0 : HCCL_INFO(
767 : "[TopoinfoRanktableConcise][GetSingleBackupDevicePort] deviceIndex[%u], devicePhyId[%u], "
768 : "get backup device port[%u].",
769 : objIndex, rankinfo.deviceInfo.devicePhyId, rankinfo.deviceInfo.backupPort);
770 0 : return HCCL_SUCCESS;
771 1 : }
772 :
773 2 : HcclResult TopoinfoRanktableConcise::VerifyBackupDeviceIpAndPort(std::vector<RankInfo_t>& rankList, u32 devIndex)
774 : {
775 2 : if (params_.deviceType != DevType::DEV_TYPE_910_93 || !GetExternalInputHcclAicpuUnfold()
776 4 : || !isInterSuperPodRetryEnable_) {
777 0 : return HCCL_SUCCESS;
778 : }
779 2 : RankInfo_t& rankInfo = rankList[devIndex];
780 :
781 3 : for (auto& backupDevIp : rankInfo.deviceInfo.backupDeviceIp) {
782 1 : if (backupDevIp.IsInvalid()) {
783 : // 无效备用IP，无需校验
784 1 : continue;
785 : }
786 1 : string backupDevIpStr = backupDevIp.GetReadableIP();
787 1 : if (devIp2ObjIndex_.find(backupDevIpStr) == devIp2ObjIndex_.end()) {
788 1 : HCCL_RUN_WARNING(
789 : "[Verify][BackupDeviceIp]"
790 : "Backup devIp[%s] for devicePhyId[%d] is not in this comm. "
791 : "The validation of this backup ip could not be verified! "
792 : "Please notice it might be an invalid backup ip!",
793 : backupDevIpStr.c_str(), devIndex);
794 1 : continue;
795 : }
796 :
797 0 : RankInfo_t& backupRankInfo = rankList[devIp2ObjIndex_[backupDevIpStr]];
798 :
799 0 : s32 backupDevPhyId = backupRankInfo.deviceInfo.devicePhyId;
800 0 : CHK_PRT_RET(
801 : backupDevPhyId == rankInfo.deviceInfo.devicePhyId,
802 : HCCL_ERROR(
803 : "[Verify][BackupDeviceIp]"
804 : "PhyId[%d] for backup devIp[%s] is the same with self device[%u] phyId[%d]. "
805 : "Please do not use self ip as backup ip!",
806 : backupDevPhyId, backupDevIpStr.c_str(), devIndex, rankInfo.deviceInfo.devicePhyId),
807 : HCCL_E_PARA);
808 :
809 0 : LinkTypeInServer linkType = LinkTypeInServer::RESERVED_LINK_TYPE;
810 0 : CHK_RET(hrtGetPairDeviceLinkType(rankInfo.deviceInfo.devicePhyId, backupDevPhyId, linkType));
811 0 : CHK_PRT_RET(
812 : linkType != LinkTypeInServer::SIO_TYPE,
813 : HCCL_ERROR(
814 : "[Verify][BackupDeviceIp]errNo[0x%016llx], device[%u], "
815 : "link between device phyId[%d] and backup device phyId[%d] is not sio link, backup device ip[%s]. "
816 : "Please check backup ip validation and whether it is on a pair device!",
817 : HCOM_ERROR_CODE(HCCL_E_PARA), devIndex, rankInfo.deviceInfo.devicePhyId, backupDevPhyId,
818 : backupDevIpStr.c_str()),
819 : HCCL_E_PARA);
820 :
821 : // 用于备用网卡的端口不可和用于主网卡的端口冲突
822 0 : CHK_PRT_RET(
823 : rankInfo.deviceInfo.backupPort != HCCL_INVALID_PORT && backupRankInfo.deviceInfo.port != HCCL_INVALID_PORT
824 : && rankInfo.deviceInfo.backupPort == backupRankInfo.deviceInfo.port,
825 : HCCL_ERROR(
826 : "[Verify][BackupDevicePort] deviceIndex[%u], "
827 : "backup device port[%u] on devPhyId[%u] should not be the same with device port[%u] on devPhyId[%u].",
828 : devIndex, rankInfo.deviceInfo.backupPort, rankInfo.deviceInfo.devicePhyId,
829 : backupRankInfo.deviceInfo.port, backupRankInfo.deviceInfo.devicePhyId),
830 : HCCL_E_PARA);
831 1 : }
832 2 : return HCCL_SUCCESS;
833 : }
834 :
835 0 : HcclResult TopoinfoRanktableConcise::GetSingleSuperDeviceId(
836 : const nlohmann::json& deviceListObj, u32 objIndex, [[maybe_unused]] RankTable_t& clusterInfo, RankInfo_t& rankinfo)
837 : {
838 : // 获取super_device_id
839 : HcclResult ret;
840 0 : std::string strSuperDeviceId;
841 0 : ret = GetJsonArrayMemberProperty(deviceListObj, objIndex, "super_device_id", strSuperDeviceId, true);
842 :
843 0 : CHK_PRT_RET(
844 : ret == HCCL_E_NOT_FOUND, HCCL_WARNING("[Get][SingleSuperDeviceId]'super_device_id' is not found"),
845 : HCCL_SUCCESS);
846 :
847 0 : u32 superDeviceId = 0;
848 0 : ret = SalStrToULong(strSuperDeviceId, HCCL_BASE_DECIMAL, superDeviceId);
849 0 : if (ret != HCCL_SUCCESS) {
850 0 : RPT_INPUT_ERR(
851 : true, "EI0014", std::vector<std::string>({"value", "variable", "expect"}),
852 : std::vector<std::string>({strSuperDeviceId, "super_device_id", "a valid non-negative integer"}));
853 0 : HCCL_ERROR(
854 : "[%s][%s]errNo[0x%016llx] super_device_id[%s] is invalid", LOG_KEYWORDS_INIT_GROUP.c_str(),
855 : LOG_KEYWORDS_RANKTABLE_CHECK.c_str(), HCOM_ERROR_CODE(HCCL_E_PARA), strSuperDeviceId.c_str());
856 0 : return ret;
857 : }
858 :
859 0 : rankinfo.superDeviceId = superDeviceId;
860 0 : HCCL_DEBUG("[%s.json] -> super_device_id: [%s]", fileName_.c_str(), strSuperDeviceId.c_str());
861 0 : return HCCL_SUCCESS;
862 0 : }
863 :
864 0 : HcclResult TopoinfoRanktableConcise::GetSuperPodList(const nlohmann::json& obj, RankTable_t& clusterInfo)
865 : {
866 0 : string version;
867 0 : CHK_RET(GetRanktableVersion(version));
868 0 : CHK_PRT_RET(
869 : version.compare(SUPERPOD_CLUSTER_VERSION) != 0,
870 : HCCL_INFO("[Get][SuperPodList]ranktable version[%s], do nothing.", version.c_str()), HCCL_SUCCESS);
871 :
872 0 : if (!IsTaskNumCalMode()) { // taskNum评估阶段，无法判断是否超节点模式
873 : // 环境配置非超节点时，不解析超节点信息, ranktable中记录的superPodId, superDeviceId, superPodNum均为默认值
874 0 : bool useSuperPodMode = false;
875 0 : CHK_RET(IsSuperPodMode(useSuperPodMode));
876 0 : CHK_PRT_RET(!useSuperPodMode, HCCL_INFO("[Get][SuperPodList]not super pod, do nothing"), HCCL_SUCCESS);
877 : }
878 :
879 : HcclResult ret;
880 0 : nlohmann::json superPodList;
881 0 : ret = GetJsonProperty(obj, "super_pod_list", superPodList, true);
882 0 : CHK_PRT_RET(
883 : ret == HCCL_E_NOT_FOUND, HCCL_WARNING("[Get][SuperPodList]'super_pod_list' is not found"), HCCL_SUCCESS);
884 0 : CHK_PRT_RET(
885 : ret != HCCL_SUCCESS && ret != HCCL_E_NOT_FOUND,
886 : HCCL_ERROR("[Get][SuperPodList]'super_pod_list' in ranktable is not set correctly, ret[%d]", ret), ret);
887 0 : HCCL_DEBUG("[%s.json]super_pod_list -> : size:[%zu]", fileName_.c_str(), superPodList.size());
888 :
889 0 : for (u32 index = 0; index < superPodList.size(); index++) {
890 0 : CHK_RET(GetSingleSuperPod(superPodList, index, clusterInfo));
891 : }
892 :
893 0 : clusterInfo.superPodNum = superPodList.size();
894 0 : CHK_RET(CheckSuperPodInfo(clusterInfo));
895 0 : return HCCL_SUCCESS;
896 0 : }
897 :
898 : HcclResult
899 0 : TopoinfoRanktableConcise::GetSingleSuperPod(const nlohmann::json& superPodList, u32 objIndex, RankTable_t& clusterInfo)
900 : {
901 0 : HcclResult ret = HCCL_SUCCESS;
902 0 : std::string superPodId;
903 0 : ret = GetJsonArrayMemberProperty(superPodList, objIndex, "super_pod_id", superPodId, true);
904 0 : if (ret != HCCL_SUCCESS || superPodId.empty()) {
905 0 : RPT_INPUT_ERR(
906 : true, "EI0014", std::vector<std::string>({"value", "variable", "expect"}),
907 : std::vector<std::string>({superPodId, "super_pod_id", "a valid non-empty string."}));
908 0 : HCCL_ERROR(
909 : "[%s][%s]errNo[0x%016llx] super_pod_id[%s] is invalid", LOG_KEYWORDS_INIT_GROUP.c_str(),
910 : LOG_KEYWORDS_RANKTABLE_CHECK.c_str(), HCOM_ERROR_CODE(ret), superPodId.c_str());
911 0 : return ret;
912 : }
913 :
914 : // 将superPodId添加到资源池进行查重校验
915 0 : CHK_RET(CheckUniqueAndInsertPool(
916 : JsonUniqueInfoType::UNIQUE_INFO_TYPE_SUPER_POD_ID, superPodId, JsonCheckOpType::CHECK_OP_TYPE_INSERT));
917 0 : HCCL_DEBUG("superPod id[%u]:[%s]", objIndex, superPodId.c_str());
918 :
919 : // 处理ranklist
920 0 : ret = GetSuperPodServerList(superPodList, objIndex, clusterInfo, superPodId);
921 0 : CHK_PRT_RET(
922 : ret != HCCL_SUCCESS,
923 : HCCL_ERROR("[Get][SingleSuperPod]get server list error:superPodId[%s]", superPodId.c_str()), ret);
924 :
925 0 : return HCCL_SUCCESS;
926 0 : }
927 :
928 0 : HcclResult TopoinfoRanktableConcise::GetSuperPodServerList(
929 : const nlohmann::json& superPodList, u32 objIndex, RankTable_t& clusterInfo, std::string superPodId)
930 : {
931 0 : HCCL_DEBUG("GetSuperPodServerList[%u]: superPodId[%s]", objIndex, superPodId.c_str());
932 :
933 0 : nlohmann::json superPodServerList;
934 0 : CHK_RET(GetJsonArrayMemberProperty(superPodList, objIndex, "server_list", superPodServerList, false));
935 0 : for (u32 index = 0; index < superPodServerList.size(); index++) {
936 : // get single super pod server info
937 0 : CHK_RET(GetSingleSuperPodSever(superPodServerList, index, clusterInfo, superPodId));
938 : }
939 0 : return HCCL_SUCCESS;
940 0 : }
941 :
942 0 : HcclResult TopoinfoRanktableConcise::GetSingleSuperPodSever(
943 : const nlohmann::json& superPodServerList, u32 objIndex, RankTable_t& clusterInfo, std::string superPodId)
944 : {
945 0 : std::string serverId;
946 0 : HcclResult ret = HCCL_SUCCESS;
947 : do {
948 : // 获取server_id
949 0 : ret = GetJsonArrayMemberProperty(superPodServerList, objIndex, "server_id", serverId, true);
950 0 : CHK_PRT_BREAK(
951 : ret != HCCL_SUCCESS,
952 : HCCL_ERROR(
953 : "[Get][SingleSuperPodSever]errNo[0x%016llx]server_id is not found or invalid", HCCL_ERROR_CODE(ret)), );
954 :
955 : // 将super_pod_list下的server_id在资源池中进行校验
956 0 : ret = CheckUniqueAndInsertPool(
957 0 : JsonUniqueInfoType::UNIQUE_INFO_TYPE_SERVER_ID, serverId, JsonCheckOpType::CHECK_OP_TYPE_FIND);
958 0 : CHK_PRT_BREAK(
959 : ret != HCCL_SUCCESS,
960 : HCCL_ERROR(
961 : "[Get][SingleSuperPodSever]errNo[0x%016llx]server_id[%s] is not found in server_list",
962 : HCCL_ERROR_CODE(ret), serverId.c_str()), );
963 0 : HCCL_DEBUG("[%s.json]super_pod_list -> server_id: [%s]", fileName_.c_str(), serverId.c_str());
964 : } while (0);
965 :
966 : // server_id未找到, 或与server_list中的server_id不一致
967 0 : if (ret != HCCL_SUCCESS) {
968 0 : RPT_INPUT_ERR(
969 : true, "EI0014", std::vector<std::string>({"value", "variable", "expect"}),
970 : std::vector<std::string>({serverId, "server_id", "a valid non-empty string."}));
971 0 : return ret;
972 : }
973 :
974 0 : u32 superPodIdx = INVALID_UINT;
975 0 : GenerateSuperPodIdx(superPodId, superPodIdx);
976 :
977 0 : bool isFound = false;
978 0 : for (RankInfo_t& rankInfo : clusterInfo.rankList) {
979 0 : if (rankInfo.serverId == serverId) {
980 0 : rankInfo.superPodId = superPodId;
981 0 : rankInfo.superPodIdx = superPodIdx;
982 0 : isFound = true;
983 : }
984 : }
985 0 : CHK_PRT_RET(
986 : isFound == false,
987 : HCCL_ERROR("[Get][SingleSuperPodSever]server_id[%s] in super_pod_list is not in server_list", serverId.c_str()),
988 : HCCL_E_PARA);
989 :
990 0 : HCCL_DEBUG(
991 : "[%s.json]super_pod_list -> server_id[%s], super_pod_id[%s]", fileName_.c_str(), serverId.c_str(),
992 : superPodId.c_str());
993 :
994 0 : return HCCL_SUCCESS;
995 0 : }
996 :
997 0 : HcclResult TopoinfoRanktableConcise::CheckSuperPodInfo(RankTable_t& clusterInfo) const
998 : {
999 0 : std::map<std::string, std::set<std::string>> superPodMap; // superPodId -> serverId
1000 0 : std::map<std::string, std::set<u32>> superPodSdidMap; // super_pod_id -> superDeviceId
1001 0 : for (RankInfo_t& rankInfo : clusterInfo.rankList) {
1002 : // 超节点模式下, 校验superPodId和sdid值有效
1003 0 : CHK_PRT_RET(
1004 : rankInfo.superPodId.empty() || rankInfo.superDeviceId == INVALID_UINT,
1005 : HCCL_ERROR(
1006 : "[Check][SuperPodInfo]superDeviceId[0x%x] or superPod[%s] in rank[%u] is invalid",
1007 : rankInfo.superDeviceId, rankInfo.superPodId.c_str(), rankInfo.rankId),
1008 : HCCL_E_PARA);
1009 :
1010 0 : auto it = superPodMap.find(rankInfo.superPodId);
1011 0 : if (it == superPodMap.end()) {
1012 0 : std::set<std::string> serverIdSet;
1013 0 : serverIdSet.insert(rankInfo.serverId);
1014 0 : superPodMap.insert({rankInfo.superPodId, serverIdSet});
1015 0 : } else if (it->second.find(rankInfo.serverId) == it->second.end()) {
1016 0 : it->second.insert(rankInfo.serverId);
1017 : }
1018 : // 用户忘记配置，superDeviceId等于无效值
1019 0 : CHK_PRT_RET(
1020 : rankInfo.superDeviceId == INVALID_UINT,
1021 : HCCL_ERROR(
1022 : "[Check][SuperPodInfo]superDeviceId[0x%x] is invalid in rankId[%u], "
1023 : "the configuration may be missing, please check the ranktable config!",
1024 : rankInfo.superDeviceId, rankInfo.rankId),
1025 : HCCL_E_PARA);
1026 :
1027 0 : auto iter = superPodSdidMap.find(rankInfo.superPodId);
1028 0 : if (iter == superPodSdidMap.end()) {
1029 0 : std::set<u32> superDeviceIdSet;
1030 0 : superDeviceIdSet.insert(rankInfo.superDeviceId);
1031 0 : superPodSdidMap.insert({rankInfo.superPodId, superDeviceIdSet});
1032 0 : } else if (iter->second.find(rankInfo.superDeviceId) == iter->second.end()) {
1033 0 : iter->second.insert(rankInfo.superDeviceId);
1034 : } else {
1035 : // 超节点内superDeviceId在超节点内唯一
1036 0 : CHK_PRT_RET(
1037 : iter->second.find(rankInfo.superDeviceId) != iter->second.end(),
1038 : HCCL_ERROR(
1039 : "[Verify][SuperPodInfo]superDeviceId[0x%x] in superPod[%s]"
1040 : "is already exist.",
1041 : rankInfo.superDeviceId, iter->first.c_str()),
1042 : HCCL_E_PARA);
1043 : }
1044 : }
1045 :
1046 0 : u32 serverNumTotal = 0;
1047 0 : for (auto it = superPodMap.begin(); it != superPodMap.end(); ++it) {
1048 0 : serverNumTotal += it->second.size();
1049 : }
1050 :
1051 : // 校验super_pod_list和原有server_list的server数量一致
1052 0 : CHK_PRT_RET(
1053 : serverNumTotal != clusterInfo.serverNum,
1054 : HCCL_ERROR(
1055 : "[Get][SuperPodList]serverNum[%u] in super_pod_list and serverNum[%u] in server_list "
1056 : "are inconsistent",
1057 : serverNumTotal, clusterInfo.serverNum),
1058 : HCCL_E_PARA);
1059 0 : return HCCL_SUCCESS;
1060 0 : }
|
