Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "topoinfo_ranktable_partition.h"
12 :
13 : #include <vector>
14 : #include <string>
15 : #include <unordered_map>
16 : #include <unordered_set>
17 :
18 : namespace hccl {
19 :
20 1 : TopoinfoRanktablePartition::TopoinfoRanktablePartition(hccl::HcclCommParams &globalParams,
21 1 : hccl::RankTable_t &globalRankTable)
22 1 : : globalParams_(globalParams), globalRankTable_(globalRankTable)
23 : {
24 1 : }
25 :
26 1 : TopoinfoRanktablePartition::~TopoinfoRanktablePartition()
27 : {
28 1 : }
29 :
30 1 : HcclResult TopoinfoRanktablePartition::GenerateSubRankTable(const uint32_t rankNum, const uint32_t *rankIds,
31 : hccl::RankTable_t &subRankTable)
32 : {
33 1 : subRankTable.nicDeploy = globalRankTable_.nicDeploy;
34 1 : std::unordered_map<uint32_t, size_t> rankInfoMap;
35 2 : for (size_t i = 0; i < globalRankTable_.rankList.size(); i++) {
36 1 : auto rankId = globalRankTable_.rankList[i].rankId;
37 1 : rankInfoMap[rankId] = i;
38 : }
39 1 : std::unordered_map<std::string, u32> serverIdMap;
40 1 : std::unordered_map<std::string, u32> superPodIdMap;
41 1 : std::unordered_set<uint32_t> rankIdSet;
42 1 : subRankTable.deviceNum = 0;
43 2 : for (size_t i = 0; i < rankNum; i++) {
44 1 : CHK_PTR_NULL(rankIds + i);
45 1 : uint32_t rankId = rankIds[i];
46 1 : CHK_PRT_RET(
47 : rankIdSet.find(rankId) != rankIdSet.end(),
48 : HCCL_ERROR("[TopoinfoRanktablePartition][GenerateSubRankTable]errNo[0x%016llx], " \
49 : "duplicated rankId[%u] in rankIds.", HCCL_ERROR_CODE(HCCL_E_PARA), rankId),
50 : HCCL_E_PARA);
51 :
52 1 : auto iter = rankInfoMap.find(rankId);
53 1 : CHK_PRT_RET(
54 : iter == rankInfoMap.end(),
55 : HCCL_ERROR("[TopoinfoRanktablePartition][GenerateSubRankTable]errNo[0x%016llx], " \
56 : "fail to find target rank[%u] in the global communicator.", HCCL_ERROR_CODE(HCCL_E_PARA), rankId),
57 : HCCL_E_PARA);
58 :
59 1 : hccl::RankInfo_t rankInfo = globalRankTable_.rankList[iter->second];
60 1 : serverIdMap.emplace(rankInfo.serverId, serverIdMap.size());
61 1 : superPodIdMap.emplace(rankInfo.superPodId, superPodIdMap.size());
62 :
63 1 : rankInfo.rankId = i;
64 1 : rankInfo.serverIdx = serverIdMap[rankInfo.serverId];
65 1 : rankInfo.superPodIdx = superPodIdMap[rankInfo.superPodId];
66 1 : subRankTable.rankList.emplace_back(rankInfo);
67 :
68 1 : if (rankInfo.deviceInfo.devicePhyId != HOST_DEVICE_ID) {
69 1 : subRankTable.deviceNum++;
70 : }
71 1 : HCCL_INFO(
72 : "[TopoinfoRanktablePartition][GenerateSubRankTable]" \
73 : "Pick rank[%u] from global comm as rank[%u] in sub comm, " \
74 : "severId[%s], serverIdx[%u], superPodId[%s], superDeviceId[%u], devicePhyId[%d].",
75 : rankId, i, rankInfo.serverId.c_str(), rankInfo.serverIdx, rankInfo.superPodId.c_str(),
76 : rankInfo.superDeviceId, rankInfo.deviceInfo.devicePhyId);
77 1 : }
78 1 : CHK_RET(GenerateSubSuperPodId(subRankTable));
79 1 : subRankTable.serverNum = serverIdMap.size();
80 1 : subRankTable.superPodNum = superPodIdMap.size();
81 1 : subRankTable.rankNum = rankNum;
82 1 : subRankTable.version = globalRankTable_.version;
83 1 : return HCCL_SUCCESS;
84 1 : }
85 :
86 1 : HcclResult TopoinfoRanktablePartition::GenerateSubSuperPodId(hccl::RankTable_t &subRankTable)
87 : {
88 1 : std::map<std::string, std::vector<RankInfo_t*>> podGroupClusters;
89 2 : for (auto& rankInfo : subRankTable.rankList) {
90 1 : podGroupClusters[rankInfo.originalSuperPodId].emplace_back(&rankInfo);
91 : }
92 1 : std::set<std::string> superPodIdSet;
93 1 : std::map<std::string, std::pair<u32, u32>> superPodIdRanges; // 记录每个逻辑超节点的rank id范围
94 2 : for (auto& subCluster : podGroupClusters) {
95 1 : auto& subClusterInfo = subCluster.second;
96 1 : if (subClusterInfo.size() <= 1) {
97 1 : continue;
98 : }
99 0 : u32 groupId = 0;
100 0 : superPodIdSet.insert(subCluster.first);
101 0 : RankInfo_t preRank = *(subClusterInfo[0]);
102 0 : superPodIdRanges[preRank.superPodId] = {preRank.rankId, preRank.rankId}; // 初始化范围
103 0 : for (u32 i = 1; i < subClusterInfo.size(); ++i) {
104 0 : RankInfo_t& curRank = *(subClusterInfo[i]);
105 : // 当前的curRank和上一个preRank的rankId不连续，分配新的逻辑超节点ID
106 0 : if (curRank.rankId != preRank.rankId + 1) {
107 0 : std::string newSuperPodId = curRank.originalSuperPodId + "_HCCLSPLIT_" + std::to_string(groupId);
108 0 : curRank.superPodId = newSuperPodId;
109 0 : groupId++;
110 0 : superPodIdRanges[curRank.superPodId] = {curRank.rankId, curRank.rankId}; // 初始化新的范围
111 0 : } else {
112 : // 同一个sub通信域两个rank原始逻辑超节点是一致的
113 : // rankId连续 上一个rank的superPodId可能已经重新分配，需要更新当前superPodId为上一个rank的
114 0 : curRank.superPodId = preRank.superPodId;
115 0 : superPodIdRanges[curRank.superPodId].second = curRank.rankId; // 更新最大rank id
116 : }
117 0 : superPodIdSet.insert(curRank.superPodId);
118 0 : preRank = curRank;
119 : }
120 0 : }
121 : // 打印每个逻辑超节点的rank id范围，只打印包含_HCCLSPLIT_的逻辑超节点
122 1 : for (const auto& entry : superPodIdRanges) {
123 0 : auto superPodId = entry.first;
124 0 : if (superPodId.find("_HCCLSPLIT_") != std::string::npos) {
125 0 : auto range = entry.second;
126 0 : HCCL_RUN_INFO("[TopoinfoRanktablePartition][%s]Split superPod, ID[%s], rank range[%u, %u]", __func__,
127 : superPodId.c_str(), range.first, range.second);
128 : }
129 0 : }
130 1 : subRankTable.superPodNum = superPodIdSet.size();
131 1 : return HCCL_SUCCESS;
132 1 : }
133 :
134 1 : HcclResult TopoinfoRanktablePartition::GenerateSubParams(const hccl::RankTable_t &subRankTable,
135 : const uint32_t subCommRankId, hccl::HcclCommParams &subParams)
136 : {
137 1 : subParams.rank = subCommRankId;
138 1 : subParams.userRank = subRankTable.rankList[subCommRankId].rankId;
139 1 : subParams.totalRanks = subRankTable.rankList.size();
140 1 : subParams.logicDevId = globalParams_.logicDevId;
141 1 : subParams.serverId = subRankTable.rankList[subCommRankId].serverId;
142 1 : subParams.deviceType = globalParams_.deviceType;
143 1 : subParams.commPortConfig.devPortSwitchOn = globalParams_.commPortConfig.devPortSwitchOn;
144 1 : return HCCL_SUCCESS;
145 : }
146 :
147 1 : HcclResult TopoinfoRanktablePartition::GetRankTableStr(const hccl::RankTable_t &subRankTable, std::string &rankTableStr)
148 : {
149 1 : nlohmann::json basicJson;
150 1 : HcclResult ret = Struct2JsonRankTable(subRankTable, globalParams_.deviceType, basicJson);
151 1 : CHK_PRT_RET(ret != HCCL_SUCCESS, HCCL_RUN_WARNING("cluster info to json failed, ret[%d].", ret), HCCL_E_INTERNAL);
152 1 : rankTableStr = std::move(basicJson.dump());
153 1 : return HCCL_SUCCESS;
154 1 : }
155 :
156 1 : HcclResult TopoinfoRanktablePartition::TransformRankInfo(const RankTable_t &clusterInfo,
157 : nlohmann::json &perRankJson, u32 rankIndex)
158 : {
159 1 : auto rankInfo = clusterInfo.rankList[rankIndex];
160 1 : perRankJson[PROP_HOST_IP] = std::string(rankInfo.hostIp.GetReadableIP());
161 1 : perRankJson[PROP_DEV_ID] = std::to_string(rankInfo.deviceInfo.devicePhyId);
162 1 : perRankJson[PROP_DEV_NIC_PORT] = std::to_string(rankInfo.deviceInfo.port);
163 1 : perRankJson[PROP_DEV_VNIC_PORT] = std::to_string(rankInfo.deviceInfo.vnicPort);
164 1 : perRankJson[PROP_BACKUP_DEV_PORT] = std::to_string(rankInfo.deviceInfo.backupPort);
165 1 : perRankJson[PROP_RANK_ID] = std::to_string(rankInfo.rankId);
166 1 : perRankJson[PROP_SERVER_ID] = rankInfo.serverId;
167 1 : perRankJson[PROP_SUPER_POD_ID] = rankInfo.superPodId;
168 1 : perRankJson[PROP_SUPER_DEVICE_ID] = std::to_string(rankInfo.superDeviceId);
169 1 : if (rankInfo.deviceInfo.deviceIp.size() != 0 && !rankInfo.deviceInfo.deviceIp[0].IsInvalid()) {
170 2 : perRankJson[PROP_DEV_IP] = std::string(rankInfo.deviceInfo.deviceIp[0].GetReadableIP());
171 : }
172 1 : if (clusterInfo.nicDeploy == NICDeployment::NIC_DEPLOYMENT_DEVICE &&
173 2 : rankInfo.deviceInfo.backupDeviceIp.size() != 0 && !rankInfo.deviceInfo.backupDeviceIp[0].IsInvalid()) {
174 0 : perRankJson[PROP_BACKUP_DEV_IP] = std::string(rankInfo.deviceInfo.backupDeviceIp[0].GetReadableIP());
175 : }
176 1 : return HCCL_SUCCESS;
177 1 : }
178 :
179 1 : HcclResult TopoinfoRanktablePartition::TransformServerList(const RankTable_t &clusterInfo,
180 : nlohmann::json &rankListJson)
181 : {
182 2 : for (size_t i = 0; i < clusterInfo.rankList.size(); i++) {
183 1 : nlohmann::json perRankJson;
184 1 : CHK_RET(TransformRankInfo(clusterInfo, perRankJson, i));
185 1 : perRankJson[PROP_RANK_ID] = perRankJson;
186 1 : rankListJson.push_back(perRankJson);
187 1 : }
188 1 : return HCCL_SUCCESS;
189 : }
190 :
191 1 : HcclResult TopoinfoRanktablePartition::Struct2JsonRankTable(const RankTable_t &clusterInfo, const DevType deviceType,
192 : nlohmann::json& ClusterJson)
193 : {
194 1 : ClusterJson[PROP_SERVER_COUNT] = std::to_string(clusterInfo.serverNum);
195 1 : ClusterJson[PROP_SUPER_POD_NUM] = std::to_string(clusterInfo.superPodNum);
196 1 : ClusterJson[PROP_RANK_NUM] = std::to_string(clusterInfo.rankNum);
197 1 : ClusterJson[PROP_DEV_NUM] = std::to_string(clusterInfo.deviceNum);
198 :
199 1 : nlohmann::json rankListJson;
200 1 : CHK_RET(TransformServerList(clusterInfo, rankListJson));
201 1 : ClusterJson[PROP_RANK_LIST] = rankListJson;
202 :
203 1 : ClusterJson[PROP_STATUS] = "completed";
204 1 : if (!clusterInfo.version.empty()) {
205 1 : ClusterJson[PROP_VERSION] = clusterInfo.version;
206 : } else {
207 0 : ClusterJson[PROP_VERSION] = (deviceType == DevType::DEV_TYPE_910_93) ? "1.2" : "1.0";
208 : }
209 :
210 1 : return HCCL_SUCCESS;
211 1 : }
212 : } // namespace hccl
|
