Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "topoinfo_ranktable_partition.h"
12 :
13 : #include <vector>
14 : #include <string>
15 : #include <unordered_map>
16 : #include <unordered_set>
17 :
18 : namespace hccl {
19 :
20 1 : TopoinfoRanktablePartition::TopoinfoRanktablePartition(
21 1 : hccl::HcclCommParams& globalParams, hccl::RankTable_t& globalRankTable)
22 1 : : globalParams_(globalParams),
23 1 : globalRankTable_(globalRankTable)
24 1 : {}
25 :
26 1 : TopoinfoRanktablePartition::~TopoinfoRanktablePartition() {}
27 :
28 1 : HcclResult TopoinfoRanktablePartition::GenerateSubRankTable(
29 : const uint32_t rankNum, const uint32_t* rankIds, hccl::RankTable_t& subRankTable)
30 : {
31 1 : subRankTable.nicDeploy = globalRankTable_.nicDeploy;
32 1 : std::unordered_map<uint32_t, size_t> rankInfoMap;
33 2 : for (size_t i = 0; i < globalRankTable_.rankList.size(); i++) {
34 1 : auto rankId = globalRankTable_.rankList[i].rankId;
35 1 : rankInfoMap[rankId] = i;
36 : }
37 1 : std::unordered_map<std::string, u32> serverIdMap;
38 1 : std::unordered_map<std::string, u32> superPodIdMap;
39 1 : std::unordered_set<uint32_t> rankIdSet;
40 1 : subRankTable.deviceNum = 0;
41 2 : for (size_t i = 0; i < rankNum; i++) {
42 1 : CHK_PTR_NULL(rankIds + i);
43 1 : uint32_t rankId = rankIds[i];
44 1 : CHK_PRT_RET(
45 : rankIdSet.find(rankId) != rankIdSet.end(),
46 : HCCL_ERROR(
47 : "[TopoinfoRanktablePartition][GenerateSubRankTable]errNo[0x%016llx], "
48 : "duplicated rankId[%u] in rankIds.",
49 : HCCL_ERROR_CODE(HCCL_E_PARA), rankId),
50 : HCCL_E_PARA);
51 :
52 1 : auto iter = rankInfoMap.find(rankId);
53 1 : CHK_PRT_RET(
54 : iter == rankInfoMap.end(),
55 : HCCL_ERROR(
56 : "[TopoinfoRanktablePartition][GenerateSubRankTable]errNo[0x%016llx], "
57 : "fail to find target rank[%u] in the global communicator.",
58 : HCCL_ERROR_CODE(HCCL_E_PARA), rankId),
59 : HCCL_E_PARA);
60 :
61 1 : hccl::RankInfo_t rankInfo = globalRankTable_.rankList[iter->second];
62 1 : serverIdMap.emplace(rankInfo.serverId, serverIdMap.size());
63 1 : superPodIdMap.emplace(rankInfo.superPodId, superPodIdMap.size());
64 :
65 1 : rankInfo.rankId = i;
66 1 : rankInfo.serverIdx = serverIdMap[rankInfo.serverId];
67 1 : rankInfo.superPodIdx = superPodIdMap[rankInfo.superPodId];
68 1 : subRankTable.rankList.emplace_back(rankInfo);
69 :
70 1 : if (rankInfo.deviceInfo.devicePhyId != HOST_DEVICE_ID) {
71 1 : subRankTable.deviceNum++;
72 : }
73 1 : HCCL_INFO(
74 : "[TopoinfoRanktablePartition][GenerateSubRankTable]"
75 : "Pick rank[%u] from global comm as rank[%u] in sub comm, "
76 : "severId[%s], serverIdx[%u], superPodId[%s], superDeviceId[%u], devicePhyId[%d].",
77 : rankId, i, rankInfo.serverId.c_str(), rankInfo.serverIdx, rankInfo.superPodId.c_str(),
78 : rankInfo.superDeviceId, rankInfo.deviceInfo.devicePhyId);
79 1 : }
80 1 : CHK_RET(GenerateSubSuperPodId(subRankTable));
81 1 : subRankTable.serverNum = serverIdMap.size();
82 1 : subRankTable.superPodNum = superPodIdMap.size();
83 1 : subRankTable.rankNum = rankNum;
84 1 : subRankTable.version = globalRankTable_.version;
85 1 : return HCCL_SUCCESS;
86 1 : }
87 :
88 1 : HcclResult TopoinfoRanktablePartition::GenerateSubSuperPodId(hccl::RankTable_t& subRankTable)
89 : {
90 1 : std::map<std::string, std::vector<RankInfo_t*>> podGroupClusters;
91 2 : for (auto& rankInfo : subRankTable.rankList) {
92 1 : podGroupClusters[rankInfo.originalSuperPodId].emplace_back(&rankInfo);
93 : }
94 1 : std::set<std::string> superPodIdSet;
95 1 : std::map<std::string, std::pair<u32, u32>> superPodIdRanges; // 记录每个逻辑超节点的rank id范围
96 2 : for (auto& subCluster : podGroupClusters) {
97 1 : auto& subClusterInfo = subCluster.second;
98 1 : if (subClusterInfo.size() <= 1) {
99 1 : continue;
100 : }
101 0 : u32 groupId = 0;
102 0 : superPodIdSet.insert(subCluster.first);
103 0 : RankInfo_t preRank = *(subClusterInfo[0]);
104 0 : superPodIdRanges[preRank.superPodId] = {preRank.rankId, preRank.rankId}; // 初始化范围
105 0 : for (u32 i = 1; i < subClusterInfo.size(); ++i) {
106 0 : RankInfo_t& curRank = *(subClusterInfo[i]);
107 : // 当前的curRank和上一个preRank的rankId不连续，分配新的逻辑超节点ID
108 0 : if (curRank.rankId != preRank.rankId + 1) {
109 0 : std::string newSuperPodId = curRank.originalSuperPodId + "_HCCLSPLIT_" + std::to_string(groupId);
110 0 : curRank.superPodId = newSuperPodId;
111 0 : groupId++;
112 0 : superPodIdRanges[curRank.superPodId] = {curRank.rankId, curRank.rankId}; // 初始化新的范围
113 0 : } else {
114 : // 同一个sub通信域两个rank原始逻辑超节点是一致的
115 : // rankId连续 上一个rank的superPodId可能已经重新分配，需要更新当前superPodId为上一个rank的
116 0 : curRank.superPodId = preRank.superPodId;
117 0 : superPodIdRanges[curRank.superPodId].second = curRank.rankId; // 更新最大rank id
118 : }
119 0 : superPodIdSet.insert(curRank.superPodId);
120 0 : preRank = curRank;
121 : }
122 0 : }
123 : // 打印每个逻辑超节点的rank id范围，只打印包含_HCCLSPLIT_的逻辑超节点
124 1 : for (const auto& entry : superPodIdRanges) {
125 0 : auto superPodId = entry.first;
126 0 : if (superPodId.find("_HCCLSPLIT_") != std::string::npos) {
127 0 : auto range = entry.second;
128 0 : HCCL_RUN_INFO(
129 : "[TopoinfoRanktablePartition][%s]Split superPod, ID[%s], rank range[%u, %u]", __func__,
130 : superPodId.c_str(), range.first, range.second);
131 : }
132 0 : }
133 1 : subRankTable.superPodNum = superPodIdSet.size();
134 1 : return HCCL_SUCCESS;
135 1 : }
136 :
137 1 : HcclResult TopoinfoRanktablePartition::GenerateSubParams(
138 : const hccl::RankTable_t& subRankTable, const uint32_t subCommRankId, hccl::HcclCommParams& subParams)
139 : {
140 1 : subParams.rank = subCommRankId;
141 1 : subParams.userRank = subRankTable.rankList[subCommRankId].rankId;
142 1 : subParams.totalRanks = subRankTable.rankList.size();
143 1 : subParams.logicDevId = globalParams_.logicDevId;
144 1 : subParams.serverId = subRankTable.rankList[subCommRankId].serverId;
145 1 : subParams.deviceType = globalParams_.deviceType;
146 1 : subParams.commPortConfig.devPortSwitchOn = globalParams_.commPortConfig.devPortSwitchOn;
147 1 : return HCCL_SUCCESS;
148 : }
149 :
150 1 : HcclResult TopoinfoRanktablePartition::GetRankTableStr(const hccl::RankTable_t& subRankTable, std::string& rankTableStr)
151 : {
152 1 : nlohmann::json basicJson;
153 1 : HcclResult ret = Struct2JsonRankTable(subRankTable, globalParams_.deviceType, basicJson);
154 1 : CHK_PRT_RET(ret != HCCL_SUCCESS, HCCL_RUN_WARNING("cluster info to json failed, ret[%d].", ret), HCCL_E_INTERNAL);
155 1 : rankTableStr = std::move(basicJson.dump());
156 1 : return HCCL_SUCCESS;
157 1 : }
158 :
159 1 : HcclResult TopoinfoRanktablePartition::TransformRankInfo(
160 : const RankTable_t& clusterInfo, nlohmann::json& perRankJson, u32 rankIndex)
161 : {
162 1 : auto rankInfo = clusterInfo.rankList[rankIndex];
163 1 : perRankJson[PROP_HOST_IP] = std::string(rankInfo.hostIp.GetReadableIP());
164 1 : perRankJson[PROP_DEV_ID] = std::to_string(rankInfo.deviceInfo.devicePhyId);
165 1 : perRankJson[PROP_DEV_NIC_PORT] = std::to_string(rankInfo.deviceInfo.port);
166 1 : perRankJson[PROP_DEV_VNIC_PORT] = std::to_string(rankInfo.deviceInfo.vnicPort);
167 1 : perRankJson[PROP_BACKUP_DEV_PORT] = std::to_string(rankInfo.deviceInfo.backupPort);
168 1 : perRankJson[PROP_RANK_ID] = std::to_string(rankInfo.rankId);
169 1 : perRankJson[PROP_SERVER_ID] = rankInfo.serverId;
170 1 : perRankJson[PROP_SUPER_POD_ID] = rankInfo.superPodId;
171 1 : perRankJson[PROP_SUPER_DEVICE_ID] = std::to_string(rankInfo.superDeviceId);
172 1 : if (rankInfo.deviceInfo.deviceIp.size() != 0 && !rankInfo.deviceInfo.deviceIp[0].IsInvalid()) {
173 2 : perRankJson[PROP_DEV_IP] = std::string(rankInfo.deviceInfo.deviceIp[0].GetReadableIP());
174 : }
175 1 : if (clusterInfo.nicDeploy == NICDeployment::NIC_DEPLOYMENT_DEVICE && rankInfo.deviceInfo.backupDeviceIp.size() != 0
176 2 : && !rankInfo.deviceInfo.backupDeviceIp[0].IsInvalid()) {
177 0 : perRankJson[PROP_BACKUP_DEV_IP] = std::string(rankInfo.deviceInfo.backupDeviceIp[0].GetReadableIP());
178 : }
179 1 : return HCCL_SUCCESS;
180 1 : }
181 :
182 1 : HcclResult TopoinfoRanktablePartition::TransformServerList(const RankTable_t& clusterInfo, nlohmann::json& rankListJson)
183 : {
184 2 : for (size_t i = 0; i < clusterInfo.rankList.size(); i++) {
185 1 : nlohmann::json perRankJson;
186 1 : CHK_RET(TransformRankInfo(clusterInfo, perRankJson, i));
187 1 : perRankJson[PROP_RANK_ID] = perRankJson;
188 1 : rankListJson.push_back(perRankJson);
189 1 : }
190 1 : return HCCL_SUCCESS;
191 : }
192 :
193 1 : HcclResult TopoinfoRanktablePartition::Struct2JsonRankTable(
194 : const RankTable_t& clusterInfo, const DevType deviceType, nlohmann::json& ClusterJson)
195 : {
196 1 : ClusterJson[PROP_SERVER_COUNT] = std::to_string(clusterInfo.serverNum);
197 1 : ClusterJson[PROP_SUPER_POD_NUM] = std::to_string(clusterInfo.superPodNum);
198 1 : ClusterJson[PROP_RANK_NUM] = std::to_string(clusterInfo.rankNum);
199 1 : ClusterJson[PROP_DEV_NUM] = std::to_string(clusterInfo.deviceNum);
200 :
201 1 : nlohmann::json rankListJson;
202 1 : CHK_RET(TransformServerList(clusterInfo, rankListJson));
203 1 : ClusterJson[PROP_RANK_LIST] = rankListJson;
204 :
205 1 : ClusterJson[PROP_STATUS] = "completed";
206 1 : if (!clusterInfo.version.empty()) {
207 1 : ClusterJson[PROP_VERSION] = clusterInfo.version;
208 : } else {
209 0 : ClusterJson[PROP_VERSION] = (deviceType == DevType::DEV_TYPE_910_93) ? "1.2" : "1.0";
210 : }
211 :
212 1 : return HCCL_SUCCESS;
213 1 : }
214 : } // namespace hccl
|
