Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include <string>
12 : #include "rank_graph.h"
13 : #include "externalinput_pub.h"
14 : #include "comm_base_pub.h"
15 :
16 : namespace hccl {
17 :
18 : // 根据 rankId 获取 rank 信息
19 6 : const RankInfo_t* RankGraphV1::FindRank(uint32_t rankId) const {
20 6 : auto it = rankIndex_.find(rankId);
21 6 : if (it == rankIndex_.end()) {
22 1 : return nullptr;
23 : }
24 5 : return &(it->second.rankInfo);
25 : }
26 :
27 488 : HcclResult RankGraphV1::DevTypeToCommProtocol(DevType &type, CommProtocol &protocol) const
28 : {
29 488 : CHK_RET(hrtGetDeviceType(type));
30 488 : switch (type) {
31 488 : case DevType::DEV_TYPE_910B:
32 : case DevType::DEV_TYPE_910_93:
33 : case DevType::DEV_TYPE_910:
34 488 : protocol = CommProtocol::COMM_PROTOCOL_ROCE;
35 488 : break;
36 0 : case DevType::DEV_TYPE_310P1:
37 : case DevType::DEV_TYPE_310P3:
38 0 : protocol = CommProtocol::COMM_PROTOCOL_PCIE;
39 0 : break;
40 0 : case DevType::DEV_TYPE_NOSOC:
41 0 : protocol = CommProtocol::COMM_PROTOCOL_PCIE;
42 0 : break;
43 0 : case DevType::DEV_TYPE_950:
44 : case DevType::DEV_TYPE_960:
45 : // 待扩展UB的协议，当前先不支持
46 0 : protocol = CommProtocol::COMM_PROTOCOL_RESERVED;
47 0 : break;
48 0 : default:
49 0 : HCCL_ERROR("[RankGraphV1] Unknown comm devType: %d", type);
50 0 : return HCCL_E_PARA;
51 : }
52 488 : return HCCL_SUCCESS;
53 : }
54 :
55 1469 : HcclResult RankGraphV1::BuildRankGraphInfo(const RankInfo_t &rankItem,
56 : const CommProtocol &protocol, RankGraphInfo &outInfo) const
57 : {
58 1469 : HCCL_INFO("[RankGraphV1][%s] rankId[%u] serverId[%s] serverIdx[%u] superDeviceId[%u] superPodId[%s] "
59 : "devicePhyId[%d]", __func__, rankItem.rankId, rankItem.serverId.c_str(), rankItem.serverIdx,
60 : rankItem.superDeviceId, rankItem.superPodId.c_str(), rankItem.deviceInfo.devicePhyId);
61 1469 : outInfo.rankInfo = rankItem;
62 1469 : std::vector<HcclIpAddress> addrs = rankItem.deviceInfo.deviceIp;
63 2977 : for (const auto &addr : addrs) {
64 : EndpointDesc point;
65 1508 : CHK_RET(static_cast<HcclResult>(EndpointDescInit(&point, 1)));
66 :
67 : // 初始化ROCE协议的基础点位信息
68 1508 : if (addr.IsIPv6()) {
69 0 : point.commAddr.type = COMM_ADDR_TYPE_IP_V6;
70 0 : point.commAddr.addr6 = addr.GetBinaryAddress().addr6;
71 : } else {
72 1508 : point.commAddr.type = COMM_ADDR_TYPE_IP_V4;
73 1508 : point.commAddr.addr = addr.GetBinaryAddress().addr;
74 : }
75 1508 : point.protocol = protocol;
76 1508 : if (rankItem.deviceInfo.nicDeploy == NICDeployment::NIC_DEPLOYMENT_HOST) {
77 0 : point.loc.locType = ENDPOINT_LOC_TYPE_HOST;
78 : } else {
79 1508 : point.loc.locType = ENDPOINT_LOC_TYPE_DEVICE;
80 : }
81 1508 : point.loc.device.devPhyId = rankItem.deviceInfo.devicePhyId;
82 1508 : point.loc.device.superDevId = rankItem.superDeviceId;
83 1508 : point.loc.device.serverIdx = rankItem.serverIdx;
84 1508 : point.loc.device.superPodIdx = rankItem.superPodIdx;
85 : // ROCE协议
86 1508 : outInfo.endPoints.push_back(std::move(point));
87 :
88 : // HCCS 协议
89 1508 : point.loc.locType = ENDPOINT_LOC_TYPE_DEVICE;
90 1508 : if (devType_ == DevType::DEV_TYPE_910B || devType_ == DevType::DEV_TYPE_910_93 ||
91 811 : devType_ == DevType::DEV_TYPE_310P1 || devType_ == DevType::DEV_TYPE_310P3) {
92 697 : EndpointDesc hccsPoint = point;
93 697 : hccsPoint.protocol = COMM_PROTOCOL_HCCS;
94 697 : hccsPoint.commAddr.type = COMM_ADDR_TYPE_ID;
95 697 : outInfo.endPoints.push_back(std::move(hccsPoint));
96 : }
97 :
98 : // PCIE 协议
99 1508 : if (devType_ == DevType::DEV_TYPE_910B || devType_ == DevType::DEV_TYPE_310P1 ||
100 844 : devType_ == DevType::DEV_TYPE_310P3) {
101 664 : EndpointDesc pciePoint = point;
102 664 : pciePoint.protocol = COMM_PROTOCOL_PCIE;
103 664 : pciePoint.commAddr.type = COMM_ADDR_TYPE_ID;
104 664 : outInfo.endPoints.push_back(std::move(pciePoint));
105 : }
106 : }
107 1469 : return HCCL_SUCCESS;
108 1469 : }
109 :
110 488 : HcclResult RankGraphV1::Init(const RankTable_t &rankTable, const HcclTopoAttr &topoAttr)
111 : {
112 488 : rankTable_ = rankTable;
113 488 : topoAttr_ = topoAttr;
114 488 : rankIndex_.clear();
115 488 : rankPairInfo_.clear();
116 488 : HCCL_INFO("[RankGraphV1][%s] rankNum[%zu]", __func__, rankTable_.rankList.size());
117 488 : CommProtocol protocol = CommProtocol::COMM_PROTOCOL_RESERVED;
118 488 : CHK_RET(DevTypeToCommProtocol(devType_, protocol));
119 : // 解析 rankTable，建立 rankId -> RankGraphInfo 映射
120 1957 : for (const auto& r : rankTable.rankList) {
121 1469 : RankGraphInfo info;
122 1469 : CHK_RET(BuildRankGraphInfo(r, protocol, info));
123 1469 : rankIndex_[r.rankId] = std::move(info);
124 1469 : }
125 488 : rankGraph_ = rankTable_.rankList;
126 488 : CHK_RET(InitRankInfo());
127 488 : CHK_RET(InitNetLayer());
128 488 : CHK_RET(InitHeterogMode());
129 488 : HCCL_INFO("[RankGraphV1][%s] Init success", __func__);
130 488 : return HCCL_SUCCESS;
131 : }
132 :
133 32 : HcclResult RankGraphV1::Init(const HcclTopoAttr &topoAttr)
134 : {
135 32 : topoAttr_ = topoAttr;
136 32 : rankIndex_.clear();
137 32 : rankPairInfo_.clear();
138 32 : HCCL_INFO("[RankGraphV1][%s] rankNum[%zu]", __func__, rankTable_.rankList.size());
139 32 : CHK_RET(InitRankInfo());
140 32 : CHK_RET(InitNetLayer());
141 32 : CHK_RET(InitHeterogMode());
142 32 : return HCCL_SUCCESS;
143 : }
144 :
145 0 : bool RankGraphV1::IsRoceInSameServer(uint32_t netLayer, const RankInfo_t &srcInfo, const RankInfo_t &dstInfo)
146 : {
147 : // 910B单机两种使能RoCE场景：1.A+X 两卡分别在两个MESH 2.标卡
148 0 : uint32_t srcPhyId = srcInfo.deviceInfo.devicePhyId;
149 0 : uint32_t dstPhyId = dstInfo.deviceInfo.devicePhyId;
150 0 : uint32_t intraRoceSwitch = GetExternalInputIntraRoceSwitch();
151 0 : HCCL_INFO("[%s] netLayer[%u], devType[%d], srcPhyId[%u], dstPhyId[%u], isStandardCard[%d], isDiffDeviceModule[%d] "
152 : "IntraRoceSwitch[%u]", __func__, netLayer, devType_, srcPhyId, dstPhyId, topoAttr_.isStandardCard,
153 : topoAttr_.isDiffDeviceModule, intraRoceSwitch);
154 0 : const uint32_t deviceMeshDivider = DEVICE_PER_MODULE;
155 0 : if (netLayer == HCCL_NETLAYER_1 && devType_ == DevType::DEV_TYPE_910B) {
156 0 : bool isSrcInLowerMesh = srcPhyId < deviceMeshDivider;
157 0 : bool isDstInLowerMesh = dstPhyId < deviceMeshDivider;
158 0 : bool isSrcInUpperMesh = srcPhyId >= deviceMeshDivider;
159 0 : bool isDstInUpperMesh = dstPhyId >= deviceMeshDivider;
160 :
161 : // 判定是否为跨MESH（一卡在低区、一卡在高区，匹配A+X跨MESH场景）
162 0 : bool isCrossMesh = (isSrcInLowerMesh || isDstInLowerMesh) && (isSrcInUpperMesh || isDstInUpperMesh);
163 : // 跨MESH或标卡直接满足
164 0 : bool isMeetRoceCondition = (isCrossMesh && topoAttr_.isDiffDeviceModule) || topoAttr_.isStandardCard;
165 0 : return isMeetRoceCondition && intraRoceSwitch == 1;
166 : }
167 :
168 : // 非910B的NETLAYER_1场景：仅标卡满足条件时取外部配置，否则返回false
169 0 : return topoAttr_.isStandardCard && intraRoceSwitch == 1 && netLayer == HCCL_NETLAYER_1;
170 : }
171 :
172 2 : CommProtocol RankGraphV1::GetCommProtocolInSameServer(const RankInfo_t &srcInfo, const RankInfo_t &dstInfo)
173 : {
174 : // 310P间链路为PCIE或HCCS
175 2 : LinkTypeInServer linkType = LinkTypeInServer::RESERVED_LINK_TYPE;
176 2 : hrtGetPairDeviceLinkType(srcInfo.deviceInfo.devicePhyId, dstInfo.deviceInfo.devicePhyId, linkType);
177 2 : HCCL_INFO("[RankGraphV1][%s] ranks[%u,%u] intra-server linkType[%d]", __func__,
178 : srcInfo.rankId, dstInfo.rankId, linkType);
179 2 : if (linkType == LinkTypeInServer::HCCS_TYPE || linkType == LinkTypeInServer::HCCS_SW_TYPE) {
180 2 : return CommProtocol::COMM_PROTOCOL_HCCS;
181 0 : } else if (linkType == LinkTypeInServer::SIO_TYPE) {
182 0 : return CommProtocol::COMM_PROTOCOL_SIO;
183 0 : } else if (linkType == LinkTypeInServer::PXI_TYPE) {
184 0 : bool isDiffDeviceModule = (topoAttr_.isDiffDeviceModule && devType_ == DevType::DEV_TYPE_910B);
185 0 : bool isRankModEqual = (srcInfo.rankId % DEVICE_PER_MODULE == dstInfo.rankId % DEVICE_PER_MODULE);
186 0 : bool isMeetPxiCondition = (!isDiffDeviceModule) || (isDiffDeviceModule && isRankModEqual);
187 0 : return isMeetPxiCondition ? CommProtocol::COMM_PROTOCOL_PCIE : CommProtocol::COMM_PROTOCOL_RESERVED;
188 : }
189 0 : return CommProtocol::COMM_PROTOCOL_RESERVED;
190 : }
191 :
192 0 : CommProtocol RankGraphV1::GetCommProtocolBetweenServers(const RankInfo_t &srcInfo, const RankInfo_t &dstInfo) const
193 : {
194 : // srcInfo与dstInfo一定是相同数据类型
195 0 : if (devType_ == DevType::DEV_TYPE_310P3 || devType_ == DevType::DEV_TYPE_310P1) {
196 0 : return CommProtocol::COMM_PROTOCOL_PCIE;
197 : }
198 0 : if (devType_ == DevType::DEV_TYPE_910B) {
199 0 : return CommProtocol::COMM_PROTOCOL_ROCE;
200 : }
201 0 : HCCL_DEBUG("[%s] srcInfo.superPodId %s dstInfo.superPodId %s", __func__, srcInfo.superPodId.c_str(), dstInfo.superPodId.c_str());
202 0 : if (devType_ == DevType::DEV_TYPE_910_93) {
203 : // 超节点内链路为HCCS
204 0 : if (!srcInfo.superPodId.empty() && srcInfo.superPodId == dstInfo.superPodId) {
205 0 : return CommProtocol::COMM_PROTOCOL_HCCS;
206 : }
207 : }
208 0 : return CommProtocol::COMM_PROTOCOL_RESERVED;
209 : }
210 :
211 2 : CommProtocol RankGraphV1::GetCommProtocolFromRankInfo(const RankInfo_t &srcInfo, const RankInfo_t &dstInfo,
212 : uint32_t netLayer)
213 : {
214 2 : if (srcInfo.deviceInfo.deviceType != dstInfo.deviceInfo.deviceType) {
215 0 : HCCL_ERROR("[RankGraphV1][%s] srcType[%d] != dstType[%d]", __func__,
216 : srcInfo.deviceInfo.deviceType, dstInfo.deviceInfo.deviceType);
217 0 : return CommProtocol::COMM_PROTOCOL_RESERVED;
218 : }
219 : // 首先判断是否在同一机内
220 2 : if (srcInfo.serverIdx == dstInfo.serverIdx) {
221 2 : if (netLayer == HCCL_NETLAYER_0) {
222 2 : return GetCommProtocolInSameServer(srcInfo, dstInfo);
223 : // 超节点有HCCL_NETLAYER_1及以上的情况，为HCCS链路，或者同卡不同DIE
224 0 : } else if (netLayer == HCCL_NETLAYER_1 && devType_ == DevType::DEV_TYPE_910_93 &&
225 0 : (srcInfo.superPodId == dstInfo.superPodId ||
226 0 : GetCommProtocolInSameServer(srcInfo, dstInfo) == CommProtocol::COMM_PROTOCOL_SIO)) {
227 0 : return CommProtocol::COMM_PROTOCOL_HCCS;
228 0 : } else if (IsRoceInSameServer(netLayer, srcInfo, dstInfo)) {
229 0 : return CommProtocol::COMM_PROTOCOL_ROCE;
230 : } else {
231 : // 接了交换机才会有HCCL_NETLAYER_1及以上的情况，当前无法判断是否连接交换机，接了交换机走RDMA
232 0 : return CommProtocol::COMM_PROTOCOL_RESERVED;
233 : }
234 : }
235 0 : if (srcInfo.serverIdx != dstInfo.serverIdx) {
236 0 : if (netLayer == HCCL_NETLAYER_0) {
237 0 : HCCL_INFO("[RankGraphV1][%s] ranks[%u,%u] not in same server", __func__, srcInfo.rankId, dstInfo.rankId);
238 0 : return CommProtocol::COMM_PROTOCOL_RESERVED;
239 : }
240 0 : if (netLayer == HCCL_NETLAYER_1) {
241 0 : HCCL_INFO("[RankGraphV1][%s] ranks[%u,%u] inter-server but same superPod[%s]", __func__,
242 : srcInfo.rankId, dstInfo.rankId, srcInfo.superPodId.c_str());
243 0 : return GetCommProtocolBetweenServers(srcInfo, dstInfo);
244 : // 跨超走ROCE
245 0 : } else if (!srcInfo.superPodId.empty() && srcInfo.superPodId != dstInfo.superPodId &&
246 : netLayer == HCCL_NETLAYER_2) {
247 0 : HCCL_INFO("[RankGraphV1][%s] ranks[%u,%u] inter-superPod use ROCE", __func__,
248 : srcInfo.rankId, dstInfo.rankId);
249 0 : return CommProtocol::COMM_PROTOCOL_ROCE;
250 : }
251 : }
252 0 : return CommProtocol::COMM_PROTOCOL_RESERVED;
253 : }
254 :
255 4 : bool RankGraphV1::NeedIgnoreEndPoints(CommProtocol srcProtocol, CommProtocol dstProtocol, CommProtocol linkProtocol) const
256 : {
257 4 : if (srcProtocol != dstProtocol) {
258 1 : return true;
259 : } else {
260 : // 两个hccs endpoints间可能是SIO链路
261 : // A + X 两个mesh间是PCIE链路, 310DUO卡两个DIE间链路是HCCS，主次DIE间是PCIE
262 3 : if (srcProtocol == COMM_PROTOCOL_HCCS && dstProtocol == COMM_PROTOCOL_HCCS
263 2 : && linkProtocol == COMM_PROTOCOL_SIO) {
264 1 : return false;
265 2 : } else if (dstProtocol != linkProtocol) {
266 0 : return true;
267 : }
268 : }
269 2 : return false;
270 : }
271 :
272 2 : void RankGraphV1::PrintLinksInfo(CommLink &link) const
273 : {
274 : // 打印CommLink 头部基础信息
275 2 : HCCL_INFO("[RankGraphV1][%s] link.header.version[%u] magicWord[0x%08x] size[%u] reserved[%u]", __func__,
276 : link.header.version, link.header.magicWord, link.header.size, link.header.reserved);
277 :
278 : // 打印【源端】srcEndpointDesc 完整信息
279 2 : HCCL_INFO("[RankGraphV1][%s] srcProtocol[%d] srcCommAddrType[%d] srcLocType[%d] srcDevPhyId[%u] "
280 : "srcSuperDevId[%u] srcServerIdx[%u] srcSuperPodIdx[%u]", __func__,
281 : link.srcEndpointDesc.protocol,
282 : link.srcEndpointDesc.commAddr.type,
283 : link.srcEndpointDesc.loc.locType,
284 : link.srcEndpointDesc.loc.device.devPhyId,
285 : link.srcEndpointDesc.loc.device.superDevId,
286 : link.srcEndpointDesc.loc.device.serverIdx,
287 : link.srcEndpointDesc.loc.device.superPodIdx);
288 :
289 : // 打印【目的端】dstEndpointDesc 完整信息
290 2 : HCCL_INFO("[RankGraphV1][%s] dstProtocol[%d] dstCommAddrType[%d] dstLocType[%d] dstDevPhyId[%u] "
291 : "dstSuperDevId[%u] dstServerIdx[%u] dstSuperPodIdx[%u]", __func__,
292 : link.dstEndpointDesc.protocol,
293 : link.dstEndpointDesc.commAddr.type,
294 : link.dstEndpointDesc.loc.locType,
295 : link.dstEndpointDesc.loc.device.devPhyId,
296 : link.dstEndpointDesc.loc.device.superDevId,
297 : link.dstEndpointDesc.loc.device.serverIdx,
298 : link.dstEndpointDesc.loc.device.superPodIdx);
299 :
300 : // 打印【链路属性】linkAttr 信息
301 2 : HCCL_INFO("[RankGraphV1][%s] linkProtocol[%d] hop[%u]", __func__, link.linkAttr.linkProtocol, link.linkAttr.hop);
302 2 : }
303 :
304 3 : HcclResult RankGraphV1::GetLinks(uint32_t netLayer, uint32_t srcRank, uint32_t dstRank,
305 : CommLink **linkList, uint32_t *listSize)
306 : {
307 5 : if (rankIndex_.find(srcRank) == rankIndex_.end() || rankIndex_.find(dstRank) == rankIndex_.end() ||
308 5 : FindRank(srcRank) == nullptr || FindRank(dstRank) == nullptr) {
309 1 : HCCL_ERROR("[RankGraphV1][%s] srcRank[%u] or dstRank[%u] is not existed in rankTable",
310 : __func__, srcRank, dstRank);
311 1 : return HCCL_E_PARA;
312 : }
313 :
314 2 : if (netLayer > HCCL_NETLAYER_2) {
315 1 : HCCL_ERROR("[RankGraphV1][%s] srcRank[%u] and dstRank[%u] do not have netLayer[%u]",
316 : __func__, srcRank, dstRank, netLayer);
317 1 : return HCCL_E_PARA;
318 : }
319 1 : auto &srcEndpointDescs = rankIndex_[srcRank].endPoints;
320 1 : auto &dstEndpointDescs = rankIndex_[dstRank].endPoints;
321 :
322 1 : const RankInfo_t &srcInfo = rankIndex_[srcRank].rankInfo;
323 1 : const RankInfo_t &dstInfo = rankIndex_[dstRank].rankInfo;
324 1 : CommProtocol protocol = COMM_PROTOCOL_RESERVED;
325 1 : protocol = GetCommProtocolFromRankInfo(srcInfo, dstInfo, netLayer);
326 1 : if (protocol == COMM_PROTOCOL_RESERVED) {
327 0 : HCCL_WARNING("[RankGraphV1][%s] no links between srcRank[%u] dstRank[%u]", __func__, srcRank, dstRank);
328 0 : *linkList = nullptr;
329 0 : *listSize = 0;
330 0 : return HCCL_SUCCESS;
331 : }
332 :
333 : // 1. 查询是否有缓存CommLink信息
334 1 : auto key = std::make_tuple(netLayer, srcRank, dstRank);
335 1 : auto it = rankPairInfo_.find(key);
336 1 : if (it == rankPairInfo_.end()) {
337 : // 没有则创建
338 1 : HCCL_INFO("[RankGraphV1][%s] no cached links, build new srcRank[%u] dstRank[%u]", __func__, srcRank, dstRank);
339 1 : std::vector<CommLink> links;
340 2 : for (size_t i = 0; i < srcEndpointDescs.size(); i++) {
341 2 : for (size_t j = 0; j < dstEndpointDescs.size(); j++) {
342 1 : if (NeedIgnoreEndPoints(srcEndpointDescs[i].protocol, dstEndpointDescs[j].protocol, protocol)) {
343 0 : continue;
344 : }
345 : CommLink link;
346 1 : CHK_RET(CommLinkInit(&link, 1));
347 :
348 1 : link.srcEndpointDesc = srcEndpointDescs[i];
349 1 : link.srcEndpointDesc.protocol = protocol;
350 1 : link.dstEndpointDesc = dstEndpointDescs[j];
351 1 : link.dstEndpointDesc.protocol = protocol;
352 1 : link.linkAttr.linkProtocol = protocol;
353 1 : PrintLinksInfo(link);
354 1 : links.push_back(std::move(link));
355 : }
356 : }
357 1 : it = rankPairInfo_.emplace(std::make_tuple(netLayer, srcRank, dstRank), std::move(links)).first;
358 1 : }
359 1 : HCCL_INFO("[RankGraphV1][%s] links, netLayer[%u] srcRank[%u] dstRank[%u] protocol[%d]", __func__,
360 : netLayer, srcRank, dstRank, protocol);
361 :
362 1 : auto &links = it->second;
363 1 : *listSize = static_cast<uint32_t>(links.size());
364 1 : if (links.empty()) {
365 0 : *linkList = nullptr;
366 0 : HCCL_ERROR("[RankGraphV1][%s] links empty for srcRank[%u] dstRank[%u]", __func__, srcRank, dstRank);
367 : } else {
368 1 : *linkList = links.data(); // 连续数组首地址
369 1 : HCCL_INFO("[RankGraphV1][%s] srcRank[%u] dstRank[%u] linkList[%p] linkNum[%u]", __func__, srcRank, dstRank, *linkList, *listSize);
370 : }
371 :
372 1 : return HCCL_SUCCESS;
373 : }
374 :
375 520 : HcclResult RankGraphV1::InitHeterogMode() {
376 520 : if (topoAttr_.rankInfoList.empty()) {
377 0 : HCCL_ERROR("[RankGraphV1][%s] invalid para. rankInfoList is empty", __func__);
378 0 : return HCCL_E_INTERNAL;
379 : }
380 :
381 520 : std::set<DevType> devTypes;
382 2143 : for (u32 index = 0; index < topoAttr_.rankInfoList.size(); index++) {
383 1623 : devTypes.insert(topoAttr_.rankInfoList[index].deviceType);
384 : }
385 :
386 : // 只包含一种芯片的同构组网
387 520 : if (devTypes.size() == 1) {
388 520 : heterogMode_ = HcclHeterogMode::HCCL_HETEROG_MODE_HOMOGENEOUS;
389 520 : return HCCL_SUCCESS;
390 : }
391 :
392 : // 包含两种芯片的异构混合组网
393 0 : constexpr uint32_t MIX_CHIPS = 2;
394 0 : if (devTypes.size() == MIX_CHIPS && devTypes.find(DevType::DEV_TYPE_910B) != devTypes.end() && devTypes.find(DevType::DEV_TYPE_910_93) != devTypes.end()) {
395 0 : heterogMode_ = HcclHeterogMode::HCCL_HETEROG_MODE_MIX_A2_A3;
396 0 : return HCCL_SUCCESS;
397 : }
398 :
399 0 : std::string devStr;
400 0 : for (auto itSet = devTypes.begin(); itSet !=devTypes.end(); itSet++) {
401 0 : if (itSet != devTypes.begin()) {
402 0 : devStr +=", ";
403 : }
404 0 : devStr += std::to_string(static_cast<int>(*itSet));
405 : }
406 0 : HCCL_ERROR("[RankGraphV1][%s] Unknown mode[%d], devtypes[%s]", __func__, HcclHeterogMode::HCCL_HETEROG_MODE_INVALID, devStr.c_str());
407 0 : return HCCL_E_INTERNAL;
408 520 : }
409 :
410 1 : HcclResult RankGraphV1::GetHeterogMode(HcclHeterogMode *mode) const
411 : {
412 1 : *mode = heterogMode_;
413 1 : return HCCL_SUCCESS;
414 : }
415 :
416 2 : HcclResult RankGraphV1::GetNetLayers(uint32_t **netLayers, uint32_t *netLayerNum)
417 : {
418 2 : if (netLayer_.empty()) {
419 1 : HCCL_ERROR("[RankGraphV1][%s] invalid para. netLayer is empty", __func__);
420 1 : return HCCL_E_INTERNAL;
421 : }
422 1 : *netLayers = netLayer_.data();
423 1 : *netLayerNum = netLayer_.size();
424 1 : return HCCL_SUCCESS;
425 : }
426 :
427 525 : HcclResult RankGraphV1::GetInstTopoTypeByNetLayer(uint32_t netLayer, CommTopo *topoType)
428 : {
429 525 : if (netLayer >= netLayer_.size()) {
430 1 : HCCL_ERROR("[RankGraphV1][%s] invalid para. netlayer[%u]", __func__, netLayer);
431 1 : return HCCL_E_PARA;
432 : }
433 524 : DevType deviceType = topoAttr_.deviceType;
434 524 : if (deviceType == DevType::DEV_TYPE_910_93) {
435 34 : if (netLayer == static_cast<uint32_t>(HcclNetLayerlevel::HCCL_NetLayer_L0)) {
436 34 : *topoType = CommTopo::COMM_TOPO_910_93;
437 0 : } else if ((netLayer == static_cast<uint32_t>(HcclNetLayerlevel::HCCL_NetLayer_L1) ||
438 : (netLayer == static_cast<uint32_t>(HcclNetLayerlevel::HCCL_NetLayer_L2)))) {
439 0 : *topoType = CommTopo::COMM_TOPO_CLOS;
440 : }
441 490 : } else if (deviceType == DevType::DEV_TYPE_910B || deviceType == DevType::DEV_TYPE_910) {
442 485 : if (netLayer == static_cast<uint32_t>(HcclNetLayerlevel::HCCL_NetLayer_L0)) {
443 484 : *topoType = CommTopo::COMM_TOPO_1DMESH;
444 1 : } else if (netLayer == static_cast<uint32_t>(HcclNetLayerlevel::HCCL_NetLayer_L1)) {
445 1 : *topoType = CommTopo::COMM_TOPO_CLOS;
446 : }
447 5 : } else if (deviceType == DevType::DEV_TYPE_310P3) {
448 5 : if (netLayer == static_cast<uint32_t>(HcclNetLayerlevel::HCCL_NetLayer_L0)) {
449 5 : *topoType = CommTopo::COMM_TOPO_310P;
450 : }
451 : }
452 524 : return HCCL_SUCCESS;
453 : }
454 :
455 2 : HcclResult RankGraphV1::GetInstSizeByNetLayer(uint32_t netLayer, uint32_t *rankNum)
456 : {
457 2 : if (netLayer >= netLayer_.size()) {
458 1 : HCCL_ERROR("[RankGraphV1][%s] invalid para. netlayer[%u]", __func__, netLayer);
459 1 : return HCCL_E_PARA;
460 : }
461 :
462 1 : if (rankList_.find(netLayer) == rankList_.end()) {
463 1 : HCCL_ERROR("[RankGraphV1][%s] failed to find rankList map. netlayer[%u]", __func__, netLayer);
464 1 : return HCCL_E_INTERNAL;
465 : }
466 0 : *rankNum = rankList_[netLayer].size();
467 :
468 0 : return HCCL_SUCCESS;
469 : }
470 :
471 2 : HcclResult RankGraphV1::GetInstRanksByNetLayer(uint32_t netLayer, uint32_t **rankList, uint32_t *rankNum)
472 : {
473 2 : if (netLayer >= netLayer_.size()) {
474 1 : HCCL_ERROR("[RankGraphV1][%s] invalid para. netlayer[%u]", __func__, netLayer);
475 1 : return HCCL_E_PARA;
476 : }
477 :
478 1 : if (rankList_.find(netLayer) == rankList_.end()) {
479 1 : HCCL_ERROR("[RankGraphV1][%s] failed to find rankList map. netlayer[%u]", __func__, netLayer);
480 1 : return HCCL_E_INTERNAL;
481 : }
482 0 : *rankNum = rankList_[netLayer].size();
483 0 : *rankList = rankList_[netLayer].data();
484 :
485 0 : return HCCL_SUCCESS;
486 : }
487 :
488 2 : HcclResult RankGraphV1::GetInstSizeListByNetLayer(uint32_t netLayer, uint32_t **instSizeList, uint32_t *listSize)
489 : {
490 2 : if (netLayer >= netLayer_.size()) {
491 1 : HCCL_ERROR("[RankGraphV1][%s] invalid para. netlayer[%u]", __func__, netLayer);
492 1 : return HCCL_E_PARA;
493 : }
494 :
495 1 : if (rankSizeList_.find(netLayer) == rankSizeList_.end()) {
496 1 : HCCL_ERROR("[RankGraphV1][%s] failed to find rankSizeList map. netlayer[%u]", __func__, netLayer);
497 1 : return HCCL_E_INTERNAL;
498 : }
499 0 : *instSizeList = rankSizeList_[netLayer].data();
500 0 : *listSize = rankSizeList_[netLayer].size();
501 :
502 0 : return HCCL_SUCCESS;
503 : }
504 :
505 1 : HcclResult RankGraphV1::GetTopoInstsByLayer(uint32_t netLayer, uint32_t **topoInsts, uint32_t *topoInstNum)
506 : {
507 1 : if (netLayer >= netLayer_.size()) {
508 0 : return HCCL_E_PARA;
509 : }
510 1 : if (rankSizeList_.find(netLayer) == rankSizeList_.end()) {
511 0 : return HCCL_E_INTERNAL;
512 : }
513 :
514 1 : uint32_t instNum = rankSizeList_[netLayer].size();
515 1 : *topoInstNum = instNum;
516 :
517 1 : static std::vector<uint32_t> sTopoInstList;
518 1 : sTopoInstList.clear();
519 1 : sTopoInstList.resize(instNum);
520 5 : for (uint32_t i = 0; i < instNum; ++i) {
521 4 : sTopoInstList[i] = i;
522 : }
523 :
524 1 : *topoInsts = sTopoInstList.data();
525 :
526 1 : return HCCL_SUCCESS;
527 : }
528 :
529 1 : HcclResult RankGraphV1::GetTopoType(uint32_t netLayer, CommTopo *topoType)
530 : {
531 1 : if (netLayer >= netLayer_.size()) {
532 0 : return HCCL_E_PARA;
533 : }
534 1 : return GetInstTopoTypeByNetLayer(netLayer, topoType);
535 : }
536 :
537 1 : HcclResult RankGraphV1::GetRanksByTopoInst(uint32_t netLayer, uint32_t topoInstId, uint32_t **ranks, uint32_t *rankNum)
538 : {
539 1 : if (netLayer >= netLayer_.size()) {
540 0 : return HCCL_E_PARA;
541 : }
542 1 : if (devType_ != DevType::DEV_TYPE_910B) {
543 0 : return HCCL_E_NOT_SUPPORT;
544 : }
545 1 : auto rankListIt = rankList_.find(netLayer);
546 1 : auto rankSizeListIt = rankSizeList_.find(netLayer);
547 1 : if (rankListIt != rankList_.end() && rankSizeListIt != rankSizeList_.end()) {
548 1 : if (topoInstId >= rankSizeListIt->second.size()) {
549 0 : return HCCL_E_PARA;
550 : }
551 : }
552 :
553 1 : *ranks = rankListIt->second.data();
554 1 : if (topoInstId < rankSizeListIt->second.size()) {
555 1 : *rankNum = rankSizeListIt->second[topoInstId];
556 : } else {
557 0 : *rankNum = 0;
558 : }
559 :
560 1 : return HCCL_SUCCESS;
561 : }
562 :
563 3 : std::vector<const RankInfo_t *> RankGraphV1::GetRanksInTopoInst(uint32_t netLayer, uint32_t topoInstId)
564 : {
565 3 : std::vector<const RankInfo_t *> ranks;
566 :
567 3 : if (netLayer >= netLayer_.size()) {
568 0 : return ranks;
569 : }
570 3 : if (netLayer == static_cast<uint32_t>(HcclNetLayerlevel::HCCL_NetLayer_L0)) {
571 2 : auto it = serverToRank_.find(topoInstId);
572 2 : if (it == serverToRank_.end()) {
573 2 : return ranks;
574 : }
575 0 : for (const auto &rankInfo : it->second) {
576 0 : for (const auto &graphInfo : rankGraph_) {
577 0 : if (graphInfo.rankId == rankInfo.userRank) {
578 0 : ranks.push_back(&graphInfo);
579 : }
580 : }
581 : }
582 : } else {
583 1 : auto listIt = rankList_.find(netLayer);
584 1 : auto sizeListIt = rankSizeList_.find(netLayer);
585 1 : if (listIt != rankList_.end() && sizeListIt != rankSizeList_.end()) {
586 1 : if (topoInstId >= sizeListIt->second.size()) {
587 0 : return ranks;
588 : }
589 : }
590 :
591 3 : for (uint32_t userRank : listIt->second) {
592 3 : for (const auto &rankInfo : rankGraph_) {
593 3 : if (rankInfo.rankId == userRank) {
594 2 : ranks.push_back(&rankInfo);
595 2 : break;
596 : }
597 : }
598 : }
599 : }
600 1 : return ranks;
601 0 : }
602 :
603 1 : std::set<CommProtocol> RankGraphV1::GetProtocolsByConnections(uint32_t netLayer,
604 : const std::vector<const RankInfo_t *> &topoInstRanks)
605 : {
606 1 : std::set<CommProtocol> protocols;
607 :
608 1 : const RankInfo_t *srcRankInfo = nullptr;
609 1 : for (const auto &rankInfo : rankGraph_) {
610 1 : if (rankInfo.rankId == rankData_.userRank) {
611 1 : srcRankInfo = &rankInfo;
612 1 : break;
613 : }
614 : }
615 1 : if (srcRankInfo == nullptr) {
616 0 : return protocols;
617 : }
618 :
619 2 : for (const RankInfo_t *dstRankInfo : topoInstRanks) {
620 1 : if (dstRankInfo->rankId == rankData_.userRank) {
621 0 : continue;
622 : }
623 :
624 1 : CommProtocol protocol = GetCommProtocolFromRankInfo(*srcRankInfo, *dstRankInfo, netLayer);
625 1 : if (protocol != COMM_PROTOCOL_RESERVED) {
626 1 : protocols.insert(protocol);
627 : }
628 : }
629 :
630 1 : return protocols;
631 0 : }
632 :
633 3 : HcclResult RankGraphV1::GetEndpointNum(uint32_t netLayer, uint32_t topoInstId, uint32_t *num)
634 : {
635 3 : if (netLayer >= netLayer_.size()) {
636 1 : HCCL_ERROR("[RankGraphV1][%s] invalid para. netlayer[%u]", __func__, netLayer);
637 1 : return HCCL_E_PARA;
638 : }
639 2 : if (rankIndex_.empty()) {
640 1 : HCCL_ERROR("[RankGraphV1][%s] rankIndex is empty", __func__);
641 1 : return HCCL_E_INTERNAL;
642 : }
643 1 : std::vector<const RankInfo_t *> topoInstRanks = GetRanksInTopoInst(netLayer, topoInstId);
644 1 : if (topoInstRanks.empty()) {
645 1 : *num = 0;
646 1 : return HCCL_SUCCESS;
647 : }
648 :
649 0 : std::set<CommProtocol> protocols = GetProtocolsByConnections(netLayer, topoInstRanks);
650 0 : if (protocols.empty()) {
651 0 : HCCL_INFO("[RankGraphV1][%s] no protocols found for netlayer [%u] topoInstId[%u]", __func__, netLayer, topoInstId);
652 : }
653 :
654 0 : const RankGraphInfo *currentRankInfo = nullptr;
655 0 : for (auto &pair : rankIndex_) {
656 0 : if (pair.second.rankInfo.rankId == rankData_.userRank) {
657 0 : currentRankInfo = &pair.second;
658 0 : break;
659 : }
660 : }
661 0 : if (currentRankInfo == nullptr) {
662 0 : return HCCL_E_INTERNAL;
663 : }
664 :
665 0 : uint32_t count = 0;
666 0 : for (const auto &endpoint : currentRankInfo->endPoints) {
667 0 : if (protocols.find(endpoint.protocol) != protocols.end()) {
668 0 : count++;
669 : }
670 : }
671 0 : *num = count;
672 :
673 0 : return HCCL_SUCCESS;
674 1 : }
675 :
676 5 : HcclResult RankGraphV1::GetEndpointDesc(uint32_t netLayer, uint32_t topoInstId, uint32_t *descNum, EndpointDesc *endpointDesc)
677 : {
678 5 : if (netLayer >= netLayer_.size()) {
679 1 : HCCL_ERROR("[RankGraphV1][%s] invalid para. netlayer[%u]", __func__, netLayer);
680 1 : return HCCL_E_PARA;
681 : }
682 4 : if (descNum == nullptr || endpointDesc == nullptr) {
683 2 : HCCL_ERROR("[RankGraphV1][%s] invalid para. null ptr", __func__);
684 2 : return HCCL_E_PARA;
685 : }
686 2 : if (rankIndex_.empty()) {
687 1 : HCCL_ERROR("[RankGraphV1][%s] rankIndex is empty", __func__);
688 1 : return HCCL_E_INTERNAL;
689 : }
690 1 : std::vector<const RankInfo_t *> topoInstRanks = GetRanksInTopoInst(netLayer, topoInstId);
691 1 : if (topoInstRanks.empty()) {
692 1 : *descNum = 0;
693 1 : return HCCL_SUCCESS;
694 : }
695 :
696 0 : std::set<CommProtocol> protocols = GetProtocolsByConnections(netLayer, topoInstRanks);
697 0 : if (protocols.empty()) {
698 0 : HCCL_INFO("[RankGraphV1][GetEndpointDesc] no protocols found, netlayer [%u] topoInstId[%u]", netLayer, topoInstId);
699 : }
700 :
701 0 : const RankGraphInfo *currRankInfo = nullptr;
702 0 : for (auto &pair : rankIndex_) {
703 0 : if (pair.second.rankInfo.rankId == rankData_.userRank) {
704 0 : currRankInfo = &pair.second;
705 0 : break;
706 : }
707 : }
708 0 : if (currRankInfo == nullptr) {
709 0 : return HCCL_E_INTERNAL;
710 : }
711 :
712 0 : uint32_t count = 0;
713 0 : for (const auto &endpoint : currRankInfo->endPoints) {
714 0 : if (protocols.find(endpoint.protocol) != protocols.end()) {
715 0 : if (count >= *descNum) {
716 0 : return HCCL_E_PARA;
717 : }
718 0 : endpointDesc[count] = endpoint;
719 0 : count++;
720 : }
721 : }
722 0 : *descNum = count;
723 :
724 0 : return HCCL_SUCCESS;
725 1 : }
726 :
727 5 : const EndpointDesc* RankGraphV1::MatchEndpointByAddr(
728 : const RankGraphInfo &rankGraphInfo, const EndpointDesc *endPointDesc) const
729 : {
730 7 : for (const auto &endpoint : rankGraphInfo.endPoints) {
731 5 : if (endpoint.commAddr.type != endPointDesc->commAddr.type ||
732 5 : endpoint.protocol != endPointDesc->protocol) {
733 0 : continue;
734 : }
735 5 : bool matched = false;
736 5 : switch (endpoint.commAddr.type) {
737 2 : case COMM_ADDR_TYPE_IP_V4:
738 2 : matched = (memcmp(&endpoint.commAddr.addr, &endPointDesc->commAddr.addr,
739 : sizeof(endpoint.commAddr.addr)) == 0);
740 2 : break;
741 1 : case COMM_ADDR_TYPE_IP_V6:
742 1 : matched = (memcmp(&endpoint.commAddr.addr6, &endPointDesc->commAddr.addr6,
743 : sizeof(endpoint.commAddr.addr6)) == 0);
744 1 : break;
745 1 : case COMM_ADDR_TYPE_ID:
746 1 : matched = true;
747 1 : break;
748 1 : default:
749 1 : break;
750 : }
751 5 : if (matched) {
752 3 : return &endpoint;
753 : }
754 : }
755 2 : return nullptr;
756 : }
757 :
758 7 : HcclResult RankGraphV1::FillAttr(EndpointAttr endpointAttr, const EndpointDesc *foundEndpoint, uint32_t infoLen, void *info) const
759 : {
760 7 : switch (endpointAttr) {
761 2 : case ENDPOINT_ATTR_BW_COEFF: {
762 2 : CHK_PRT_RET(infoLen != sizeof(EndpointAttrBwCoeff),
763 : HCCL_ERROR("[RankGraphV1::GetEndpointInfo] Size mismatch for ENDPOINT_ATTR_BW_COEFF: expected %zu, actual %u",
764 : sizeof(EndpointAttrBwCoeff), infoLen), HCCL_E_PARA);
765 1 : *(static_cast<EndpointAttrBwCoeff*>(info)) = 1;
766 1 : break;
767 : }
768 2 : case ENDPOINT_ATTR_DIE_ID: {
769 2 : CHK_PRT_RET(infoLen != sizeof(EndpointAttrDieId),
770 : HCCL_ERROR("[RankGraphV1::GetEndpointInfo] Size mismatch for ENDPOINT_ATTR_DIE_ID: expected %zu, actual %u",
771 : sizeof(EndpointAttrDieId), infoLen), HCCL_E_PARA);
772 1 : *(static_cast<EndpointAttrDieId*>(info)) = foundEndpoint->loc.device.superDevId;
773 1 : break;
774 : }
775 2 : case ENDPOINT_ATTR_LOCATION: {
776 2 : CHK_PRT_RET(infoLen != sizeof(EndpointAttrLocation),
777 : HCCL_ERROR("[RankGraphV1::GetEndpointInfo] Size mismatch for ENDPOINT_ATTR_LOCATION: expected %zu, actual %u",
778 : sizeof(EndpointAttrLocation), infoLen), HCCL_E_PARA);
779 1 : *(static_cast<EndpointAttrLocation*>(info)) = foundEndpoint->loc.locType;
780 1 : break;
781 : }
782 1 : default: {
783 1 : HCCL_ERROR("[RankGraphV1::GetEndpointInfo] Invalid endpointAttr[%d]", endpointAttr);
784 1 : return HCCL_E_PARA;
785 : }
786 : }
787 3 : return HCCL_SUCCESS;
788 : }
789 :
790 5 : HcclResult RankGraphV1::GetEndpointInfo(uint32_t rankId, const EndpointDesc *endPointDesc, EndpointAttr endpointAttr,
791 : uint32_t infoLen, void *info)
792 : {
793 5 : if (endPointDesc == nullptr || info == nullptr) {
794 2 : HCCL_ERROR("[RankGraphV1::GetEndpointInfo] Invalid parameter, null pointer");
795 2 : return HCCL_E_PTR;
796 : }
797 :
798 3 : if (rankIndex_.empty()) {
799 1 : HCCL_ERROR("[RankGraphV1::GetEndpointInfo] rankIndex is empty");
800 1 : return HCCL_E_INTERNAL;
801 : }
802 :
803 2 : auto rankIt = rankIndex_.find(rankId);
804 2 : if (rankIt == rankIndex_.end()) {
805 1 : HCCL_ERROR("[RankGraphV1::GetEndpointInfo] rankId[%u] not found in rankIndex", rankId);
806 1 : return HCCL_E_NOT_FOUND;
807 : }
808 :
809 1 : const RankGraphInfo &rankGraphInfo = rankIt->second;
810 1 : const EndpointDesc *foundEndpoint = MatchEndpointByAddr(rankGraphInfo, endPointDesc);
811 1 : if (foundEndpoint == nullptr) {
812 0 : HCCL_ERROR("[RankGraphV1::GetEndpointInfo] No matching endpoint found for rankId[%u]", rankId);
813 0 : return HCCL_E_NOT_FOUND;
814 : }
815 :
816 1 : return FillAttr(endpointAttr, foundEndpoint, infoLen, info);
817 : }
818 :
819 2 : HcclResult RankGraphV1::GetRankSize(uint32_t *rankSize)
820 : {
821 2 : CHK_PTR_NULL(rankSize);
822 1 : *rankSize = rankGraph_.size();
823 1 : return HCCL_SUCCESS;
824 : }
825 :
826 3 : HcclResult RankGraphV1::GetDevicePort(const uint32_t rank, uint32_t *devPort)
827 : {
828 3 : CHK_PTR_NULL(devPort);
829 2 : const RankInfo_t *rankInfo = FindRank(rank);
830 2 : if (rankInfo == nullptr) {
831 1 : HCCL_ERROR("[RankGraphV1][%s] rank[%u] not found", __func__, rank);
832 1 : return HCCL_E_PARA;
833 : }
834 1 : *devPort = rankInfo->deviceInfo.port;
835 1 : return HCCL_SUCCESS;
836 : }
837 :
838 3904 : bool RankGraphSort(const RankInfo &first, const RankInfo &second)
839 : {
840 3904 : if (first.serverIdx != second.serverIdx) {
841 449 : return first.serverIdx < second.serverIdx;
842 : } else {
843 3455 : return first.userRank < second.userRank;
844 : }
845 : }
846 :
847 520 : HcclResult RankGraphV1::InitGraphRankInfo()
848 : {
849 1989 : for (u32 index = 0; index < rankGraph_.size(); index++) {
850 1469 : struct GraphRankInfo graphRankInfo = {};
851 1469 : graphRankInfo.rankId = rankGraph_[index].rankId;
852 1469 : graphRankInfo.localRank = rankGraph_[index].localRank;
853 1469 : graphRankInfo.serverId = rankGraph_[index].serverId;
854 1469 : graphRankInfo.serverIdx = rankGraph_[index].serverIdx;
855 1469 : graphRankInfo.superDeviceId = rankGraph_[index].superDeviceId;
856 1469 : graphRankInfo.superPodId = rankGraph_[index].superPodId;
857 1469 : graphRankInfo.superPodIdx = rankGraph_[index].superPodIdx;
858 1469 : graphRankInfo.hostPort = rankGraph_[index].hostPort;
859 1469 : graphRankInfo.nodeId = rankGraph_[index].nodeId;
860 1469 : graphRankInfo.itemId = rankGraph_[index].itemId;
861 1469 : graphRankInfo.deviceInfo.devicePhyId = rankGraph_[index].deviceInfo.devicePhyId;
862 1469 : graphRankInfo.deviceInfo.deviceType = rankGraph_[index].deviceInfo.deviceType;
863 1469 : graphRankInfo.deviceInfo.port = rankGraph_[index].deviceInfo.port;
864 1469 : graphRankInfo.deviceInfo.vnicPort = rankGraph_[index].deviceInfo.vnicPort;
865 1469 : graphRankInfo.deviceInfo.backupPort = rankGraph_[index].deviceInfo.backupPort;
866 1469 : graphRankInfo.bindDeviceId = rankGraph_[index].bindDeviceId;
867 1469 : graphRankInfo.originalSuperPodId = rankGraph_[index].originalSuperPodId;
868 :
869 1469 : graphRankInfo_.push_back(graphRankInfo);
870 1469 : }
871 :
872 520 : return HCCL_SUCCESS;
873 : }
874 :
875 1 : HcclResult RankGraphV1::GetRankGraphInfo(GraphType type, void **graph, uint32_t *len)
876 : {
877 1 : switch (type) {
878 0 : case RANK_GRAPH_910_93: {
879 0 : *graph = graphRankInfo_.data();
880 0 : *len = graphRankInfo_.size() * sizeof(GraphRankInfo);
881 0 : break;
882 : }
883 1 : default: {
884 1 : HCCL_ERROR("[RankGraphV1][%s]Graph type[%d] is invalid", __func__, type);
885 1 : return HCCL_E_NOT_SUPPORT;
886 : }
887 : }
888 0 : return HCCL_SUCCESS;
889 : }
890 :
891 0 : HcclResult RankGraphV1::GetDeviceId(uint32_t rankId, uint32_t *deviceId)
892 : {
893 0 : return HCCL_SUCCESS;
894 : }
895 :
896 520 : HcclResult RankGraphV1::InitRankInfo()
897 : {
898 520 : auto& rankInfoList = topoAttr_.rankInfoList;
899 828 : for (u32 index = 0; index < rankInfoList.size(); index++) {
900 828 : if (topoAttr_.userRank == rankInfoList[index].userRank) {
901 520 : rankData_ = rankInfoList[index];
902 520 : break;
903 : }
904 : }
905 520 : CHK_RET(InitServerRankInfo());
906 520 : CHK_RET(InitSuperPodRankInfo());
907 520 : CHK_RET(InitGraphRankInfo());
908 520 : return HCCL_SUCCESS;
909 : }
910 :
911 520 : HcclResult RankGraphV1::InitServerRankInfo()
912 : {
913 520 : u32 serverIdx = 0;
914 520 : auto& rankInfoList = topoAttr_.rankInfoList;
915 2143 : for (u32 index = 0; index < rankInfoList.size(); index++) {
916 1623 : serverIdx = rankInfoList[index].serverIdx;
917 1623 : auto itServer = serverToRank_.find(serverIdx);
918 1623 : if (itServer != serverToRank_.end()) {
919 885 : itServer->second.push_back(rankInfoList[index]);
920 : } else {
921 738 : std::vector<RankInfo> rankVecTmp;
922 738 : rankVecTmp.push_back(rankInfoList[index]);
923 738 : serverToRank_.insert(std::make_pair(serverIdx, rankVecTmp));
924 738 : }
925 : }
926 : // 调整每个server内的user_rank排序(server内userRank从小到大,一定连续)
927 1258 : for (auto iterMap = serverToRank_.begin(); iterMap != serverToRank_.end(); iterMap++) {
928 738 : if (!(iterMap->second).empty()) {
929 738 : std::sort(iterMap->second.begin(), iterMap->second.end(), RankGraphSort);
930 : }
931 : }
932 520 : serverIdx = rankData_.serverIdx;
933 520 : auto rankVec = serverToRank_.find(serverIdx);
934 520 : if (rankVec != serverToRank_.end()) {
935 520 : std::string rankIdListServer;
936 1840 : for (auto iter : serverToRank_[serverIdx]) {
937 1320 : rankIdListServer += std::to_string(iter.userRank) + " ";
938 1320 : }
939 520 : HCCL_INFO("[RankGraphV1][%s] devtype[%d], curRank[%u], serverToRanklist[%s]", __func__,
940 : topoAttr_.deviceType, rankData_.userRank, rankIdListServer.c_str());
941 520 : }
942 520 : return HCCL_SUCCESS;
943 : }
944 :
945 520 : HcclResult RankGraphV1::InitSuperPodRankInfo()
946 : {
947 520 : auto& rankInfoList = topoAttr_.rankInfoList;
948 2143 : for (u32 index = 0; index < rankInfoList.size(); index++) {
949 : // 填充superPodRankMap_, 记录superPodId -> rankInfo
950 1623 : HCCL_DEBUG("[RankGraphV1][%s] superPodIdx[%u],superPodId[%s]", __func__,
951 : rankInfoList[index].superPodIdx, rankInfoList[index].superPodId.c_str());
952 1623 : auto itSuperPod = superPodToRank_.find(rankInfoList[index].superPodIdx);
953 1623 : if (itSuperPod != superPodToRank_.end()) {
954 1062 : itSuperPod->second.push_back(rankInfoList[index]);
955 : } else {
956 561 : std::vector<RankInfo> rankVecTmp;
957 561 : rankVecTmp.push_back(rankInfoList[index]);
958 561 : superPodToRank_.insert(std::make_pair(rankInfoList[index].superPodIdx, rankVecTmp));
959 561 : }
960 : }
961 :
962 : // 调整每个superPod内的user_rank排序, 按照serverIdx从小到大、userRank从小到大排序
963 1081 : for (auto iterMap = superPodToRank_.begin(); iterMap != superPodToRank_.end(); iterMap++) {
964 561 : if (!(iterMap->second).empty()) {
965 561 : std::sort(iterMap->second.begin(), iterMap->second.end(), RankGraphSort);
966 : }
967 : }
968 :
969 520 : if (superPodToRank_.find(rankData_.superPodIdx) != superPodToRank_.end()) {
970 520 : std::string rankIdListPod;
971 2102 : for (auto iter : superPodToRank_[rankData_.superPodIdx]) {
972 1582 : rankIdListPod += std::to_string(iter.userRank) + " ";
973 1582 : }
974 520 : HCCL_INFO("[RankGraphV1][%s] curRank[%u], curSuperPod[%s] superPodToRanklist[%s]",
975 : __func__, rankData_.userRank, rankData_.superPodId.c_str(), rankIdListPod.c_str());
976 520 : }
977 520 : return HCCL_SUCCESS;
978 : }
979 :
980 520 : HcclResult RankGraphV1::InitNetLayer()
981 : {
982 520 : netLayer_.clear();
983 520 : netLayer_.push_back(static_cast<uint32_t>(HcclNetLayerlevel::HCCL_NetLayer_L0));
984 :
985 520 : u32 serverIdx = rankData_.serverIdx;
986 520 : auto rankVec = serverToRank_.find(serverIdx);
987 520 : if (rankVec == serverToRank_.end()) {
988 0 : HCCL_ERROR("[RankGraphV1][%s] find serverToRank failed, serverIdx[%u]", __func__, serverIdx);
989 0 : return HCCL_E_INTERNAL;
990 : }
991 520 : std::vector<u32> rankListTmp;
992 1840 : for (auto iter : serverToRank_[serverIdx]) {
993 1320 : rankListTmp.push_back(iter.userRank);
994 1320 : }
995 520 : rankList_.insert({static_cast<uint32_t>(HcclNetLayerlevel::HCCL_NetLayer_L0), rankListTmp});
996 :
997 520 : std::vector<u32> rankSizeListTmp;
998 1258 : for (auto iter : serverToRank_) {
999 738 : rankSizeListTmp.push_back(iter.second.size());
1000 738 : }
1001 520 : rankSizeList_.insert({static_cast<uint32_t>(HcclNetLayerlevel::HCCL_NetLayer_L0), rankSizeListTmp});
1002 :
1003 520 : DevType deviceType = topoAttr_.deviceType;
1004 520 : if (serverToRank_.size() > 1) {
1005 100 : netLayer_.push_back(static_cast<uint32_t>(HcclNetLayerlevel::HCCL_NetLayer_L1));
1006 100 : if (deviceType == DevType::DEV_TYPE_910B || deviceType == DevType::DEV_TYPE_910) {
1007 78 : std::vector<u32> rankListTmp1;
1008 311 : for (auto& pair : serverToRank_) {
1009 580 : for (auto iter : pair.second) {
1010 347 : rankListTmp1.push_back(iter.userRank);
1011 347 : }
1012 : }
1013 78 : rankList_.insert({static_cast<uint32_t>(HcclNetLayerlevel::HCCL_NetLayer_L1), rankListTmp1});
1014 156 : rankSizeList_.insert({static_cast<uint32_t>(HcclNetLayerlevel::HCCL_NetLayer_L1), {topoAttr_.userRankSize}});
1015 100 : } else if (deviceType == DevType::DEV_TYPE_910_93) {
1016 22 : auto it = superPodToRank_.find(rankData_.superPodIdx);
1017 22 : if (it == superPodToRank_.end()) {
1018 0 : HCCL_ERROR("[RankGraphV1][%s] find superPodToRank_ failed, superPodIdx[%u]", __func__, rankData_.superPodIdx);
1019 0 : return HCCL_E_INTERNAL;
1020 : }
1021 22 : std::vector<u32> rankListTmp1;
1022 88 : for (auto iter : superPodToRank_[rankData_.superPodIdx]) {
1023 66 : rankListTmp1.push_back(iter.userRank);
1024 66 : }
1025 22 : std::vector<u32> rankSizeListTmp1;
1026 85 : for (auto iter : superPodToRank_) {
1027 63 : rankSizeListTmp1.push_back(iter.second.size());
1028 63 : }
1029 22 : rankList_.insert({static_cast<uint32_t>(HcclNetLayerlevel::HCCL_NetLayer_L1), rankListTmp1});
1030 22 : rankSizeList_.insert({static_cast<uint32_t>(HcclNetLayerlevel::HCCL_NetLayer_L1), rankSizeListTmp1});
1031 22 : }
1032 : }
1033 :
1034 520 : if (deviceType == DevType::DEV_TYPE_910_93 && superPodToRank_.size() > 1) {
1035 8 : netLayer_.push_back(static_cast<uint32_t>(HcclNetLayerlevel::HCCL_NetLayer_L2));
1036 8 : std::vector<u32> rankListTmp2;
1037 57 : for (const auto& pair : superPodToRank_) {
1038 98 : for (auto iter : pair.second) {
1039 49 : rankListTmp2.push_back(iter.userRank);
1040 49 : }
1041 : }
1042 8 : rankList_.insert({static_cast<uint32_t>(HcclNetLayerlevel::HCCL_NetLayer_L2), rankListTmp2});
1043 16 : rankSizeList_.insert({static_cast<uint32_t>(HcclNetLayerlevel::HCCL_NetLayer_L2), {topoAttr_.userRankSize}});
1044 8 : }
1045 520 : return HCCL_SUCCESS;
1046 520 : }
1047 : };
|
