Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include <string>
12 : #include "rank_graph.h"
13 : #include "externalinput_pub.h"
14 : #include "comm_base_pub.h"
15 :
16 : namespace hccl {
17 :
18 : // 根据 rankId 获取 rank 信息
19 6 : const RankInfo_t* RankGraphV1::FindRank(uint32_t rankId) const
20 : {
21 6 : auto it = rankIndex_.find(rankId);
22 6 : if (it == rankIndex_.end()) {
23 1 : return nullptr;
24 : }
25 5 : return &(it->second.rankInfo);
26 : }
27 :
28 491 : HcclResult RankGraphV1::DevTypeToCommProtocol(DevType& type, CommProtocol& protocol) const
29 : {
30 491 : CHK_RET(hrtGetDeviceType(type));
31 491 : switch (type) {
32 491 : case DevType::DEV_TYPE_910B:
33 : case DevType::DEV_TYPE_910_93:
34 : case DevType::DEV_TYPE_910:
35 491 : protocol = CommProtocol::COMM_PROTOCOL_ROCE;
36 491 : break;
37 0 : case DevType::DEV_TYPE_310P1:
38 : case DevType::DEV_TYPE_310P3:
39 0 : protocol = CommProtocol::COMM_PROTOCOL_PCIE;
40 0 : break;
41 0 : case DevType::DEV_TYPE_NOSOC:
42 0 : protocol = CommProtocol::COMM_PROTOCOL_PCIE;
43 0 : break;
44 0 : case DevType::DEV_TYPE_950:
45 : case DevType::DEV_TYPE_960:
46 : // 待扩展UB的协议，当前先不支持
47 0 : protocol = CommProtocol::COMM_PROTOCOL_RESERVED;
48 0 : break;
49 0 : default:
50 0 : HCCL_ERROR("[RankGraphV1] Unknown comm devType: %d", type);
51 0 : return HCCL_E_PARA;
52 : }
53 491 : return HCCL_SUCCESS;
54 : }
55 :
56 : HcclResult
57 1474 : RankGraphV1::BuildRankGraphInfo(const RankInfo_t& rankItem, const CommProtocol& protocol, RankGraphInfo& outInfo) const
58 : {
59 1474 : HCCL_INFO(
60 : "[RankGraphV1][%s] rankId[%u] serverId[%s] serverIdx[%u] superDeviceId[%u] superPodId[%s] "
61 : "devicePhyId[%d]",
62 : __func__, rankItem.rankId, rankItem.serverId.c_str(), rankItem.serverIdx, rankItem.superDeviceId,
63 : rankItem.superPodId.c_str(), rankItem.deviceInfo.devicePhyId);
64 1474 : outInfo.rankInfo = rankItem;
65 1474 : std::vector<HcclIpAddress> addrs = rankItem.deviceInfo.deviceIp;
66 2987 : for (const auto& addr : addrs) {
67 : EndpointDesc point;
68 1513 : CHK_RET(static_cast<HcclResult>(EndpointDescInit(&point, 1)));
69 :
70 : // 初始化ROCE协议的基础点位信息
71 1513 : if (addr.IsIPv6()) {
72 0 : point.commAddr.type = COMM_ADDR_TYPE_IP_V6;
73 0 : point.commAddr.addr6 = addr.GetBinaryAddress().addr6;
74 : } else {
75 1513 : point.commAddr.type = COMM_ADDR_TYPE_IP_V4;
76 1513 : point.commAddr.addr = addr.GetBinaryAddress().addr;
77 : }
78 1513 : point.protocol = protocol;
79 1513 : if (rankItem.deviceInfo.nicDeploy == NICDeployment::NIC_DEPLOYMENT_HOST) {
80 0 : point.loc.locType = ENDPOINT_LOC_TYPE_HOST;
81 : } else {
82 1513 : point.loc.locType = ENDPOINT_LOC_TYPE_DEVICE;
83 : }
84 1513 : point.loc.device.devPhyId = rankItem.deviceInfo.devicePhyId;
85 1513 : point.loc.device.superDevId = rankItem.superDeviceId;
86 1513 : point.loc.device.serverIdx = rankItem.serverIdx;
87 1513 : point.loc.device.superPodIdx = rankItem.superPodIdx;
88 : // ROCE协议
89 1513 : outInfo.endPoints.push_back(std::move(point));
90 :
91 : // HCCS 协议
92 1513 : point.loc.locType = ENDPOINT_LOC_TYPE_DEVICE;
93 1513 : if (devType_ == DevType::DEV_TYPE_910B || devType_ == DevType::DEV_TYPE_910_93
94 816 : || devType_ == DevType::DEV_TYPE_310P1 || devType_ == DevType::DEV_TYPE_310P3) {
95 697 : EndpointDesc hccsPoint = point;
96 697 : hccsPoint.protocol = COMM_PROTOCOL_HCCS;
97 697 : hccsPoint.commAddr.type = COMM_ADDR_TYPE_ID;
98 697 : outInfo.endPoints.push_back(std::move(hccsPoint));
99 : }
100 :
101 : // PCIE 协议
102 1513 : if (devType_ == DevType::DEV_TYPE_910B || devType_ == DevType::DEV_TYPE_310P1
103 849 : || devType_ == DevType::DEV_TYPE_310P3) {
104 664 : EndpointDesc pciePoint = point;
105 664 : pciePoint.protocol = COMM_PROTOCOL_PCIE;
106 664 : pciePoint.commAddr.type = COMM_ADDR_TYPE_ID;
107 664 : outInfo.endPoints.push_back(std::move(pciePoint));
108 : }
109 : }
110 1474 : return HCCL_SUCCESS;
111 1474 : }
112 :
113 491 : HcclResult RankGraphV1::Init(const RankTable_t& rankTable, const HcclTopoAttr& topoAttr)
114 : {
115 491 : rankTable_ = rankTable;
116 491 : topoAttr_ = topoAttr;
117 491 : rankIndex_.clear();
118 491 : rankPairInfo_.clear();
119 491 : HCCL_INFO("[RankGraphV1][%s] rankNum[%zu]", __func__, rankTable_.rankList.size());
120 491 : CommProtocol protocol = CommProtocol::COMM_PROTOCOL_RESERVED;
121 491 : CHK_RET(DevTypeToCommProtocol(devType_, protocol));
122 : // 解析 rankTable，建立 rankId -> RankGraphInfo 映射
123 1965 : for (const auto& r : rankTable.rankList) {
124 1474 : RankGraphInfo info;
125 1474 : CHK_RET(BuildRankGraphInfo(r, protocol, info));
126 1474 : rankIndex_[r.rankId] = std::move(info);
127 1474 : }
128 491 : rankGraph_ = rankTable_.rankList;
129 491 : CHK_RET(InitRankInfo());
130 491 : CHK_RET(InitNetLayer());
131 491 : CHK_RET(InitHeterogMode());
132 491 : HCCL_INFO("[RankGraphV1][%s] Init success", __func__);
133 491 : return HCCL_SUCCESS;
134 : }
135 :
136 32 : HcclResult RankGraphV1::Init(const HcclTopoAttr& topoAttr)
137 : {
138 32 : topoAttr_ = topoAttr;
139 32 : rankIndex_.clear();
140 32 : rankPairInfo_.clear();
141 32 : HCCL_INFO("[RankGraphV1][%s] rankNum[%zu]", __func__, rankTable_.rankList.size());
142 32 : CHK_RET(InitRankInfo());
143 32 : CHK_RET(InitNetLayer());
144 32 : CHK_RET(InitHeterogMode());
145 32 : return HCCL_SUCCESS;
146 : }
147 :
148 0 : bool RankGraphV1::IsRoceInSameServer(uint32_t netLayer, const RankInfo_t& srcInfo, const RankInfo_t& dstInfo)
149 : {
150 : // 910B单机两种使能RoCE场景：1.A+X 两卡分别在两个MESH 2.标卡
151 0 : uint32_t srcPhyId = srcInfo.deviceInfo.devicePhyId;
152 0 : uint32_t dstPhyId = dstInfo.deviceInfo.devicePhyId;
153 0 : uint32_t intraRoceSwitch = GetExternalInputIntraRoceSwitch();
154 0 : HCCL_INFO(
155 : "[%s] netLayer[%u], devType[%d], srcPhyId[%u], dstPhyId[%u], isStandardCard[%d], isDiffDeviceModule[%d] "
156 : "IntraRoceSwitch[%u]",
157 : __func__, netLayer, devType_, srcPhyId, dstPhyId, topoAttr_.isStandardCard, topoAttr_.isDiffDeviceModule,
158 : intraRoceSwitch);
159 0 : const uint32_t deviceMeshDivider = DEVICE_PER_MODULE;
160 0 : if (netLayer == HCCL_NETLAYER_1 && devType_ == DevType::DEV_TYPE_910B) {
161 0 : bool isSrcInLowerMesh = srcPhyId < deviceMeshDivider;
162 0 : bool isDstInLowerMesh = dstPhyId < deviceMeshDivider;
163 0 : bool isSrcInUpperMesh = srcPhyId >= deviceMeshDivider;
164 0 : bool isDstInUpperMesh = dstPhyId >= deviceMeshDivider;
165 :
166 : // 判定是否为跨MESH（一卡在低区、一卡在高区，匹配A+X跨MESH场景）
167 0 : bool isCrossMesh = (isSrcInLowerMesh || isDstInLowerMesh) && (isSrcInUpperMesh || isDstInUpperMesh);
168 : // 跨MESH或标卡直接满足
169 0 : bool isMeetRoceCondition = (isCrossMesh && topoAttr_.isDiffDeviceModule) || topoAttr_.isStandardCard;
170 0 : return isMeetRoceCondition && intraRoceSwitch == 1;
171 : }
172 :
173 : // 非910B的NETLAYER_1场景：仅标卡满足条件时取外部配置，否则返回false
174 0 : return topoAttr_.isStandardCard && intraRoceSwitch == 1 && netLayer == HCCL_NETLAYER_1;
175 : }
176 :
177 2 : CommProtocol RankGraphV1::GetCommProtocolInSameServer(const RankInfo_t& srcInfo, const RankInfo_t& dstInfo)
178 : {
179 : // 310P间链路为PCIE或HCCS
180 2 : LinkTypeInServer linkType = LinkTypeInServer::RESERVED_LINK_TYPE;
181 2 : hrtGetPairDeviceLinkType(srcInfo.deviceInfo.devicePhyId, dstInfo.deviceInfo.devicePhyId, linkType);
182 2 : HCCL_INFO(
183 : "[RankGraphV1][%s] ranks[%u,%u] intra-server linkType[%d]", __func__, srcInfo.rankId, dstInfo.rankId, linkType);
184 2 : if (linkType == LinkTypeInServer::HCCS_TYPE || linkType == LinkTypeInServer::HCCS_SW_TYPE) {
185 2 : return CommProtocol::COMM_PROTOCOL_HCCS;
186 0 : } else if (linkType == LinkTypeInServer::SIO_TYPE) {
187 0 : return CommProtocol::COMM_PROTOCOL_SIO;
188 0 : } else if (linkType == LinkTypeInServer::PXI_TYPE) {
189 0 : bool isDiffDeviceModule = (topoAttr_.isDiffDeviceModule && devType_ == DevType::DEV_TYPE_910B);
190 0 : bool isRankModEqual = (srcInfo.rankId % DEVICE_PER_MODULE == dstInfo.rankId % DEVICE_PER_MODULE);
191 0 : bool isMeetPxiCondition = (!isDiffDeviceModule) || (isDiffDeviceModule && isRankModEqual);
192 0 : return isMeetPxiCondition ? CommProtocol::COMM_PROTOCOL_PCIE : CommProtocol::COMM_PROTOCOL_RESERVED;
193 : }
194 0 : return CommProtocol::COMM_PROTOCOL_RESERVED;
195 : }
196 :
197 0 : CommProtocol RankGraphV1::GetCommProtocolBetweenServers(const RankInfo_t& srcInfo, const RankInfo_t& dstInfo) const
198 : {
199 : // srcInfo与dstInfo一定是相同数据类型
200 0 : if (devType_ == DevType::DEV_TYPE_310P3 || devType_ == DevType::DEV_TYPE_310P1) {
201 0 : return CommProtocol::COMM_PROTOCOL_PCIE;
202 : }
203 0 : if (devType_ == DevType::DEV_TYPE_910B) {
204 0 : return CommProtocol::COMM_PROTOCOL_ROCE;
205 : }
206 0 : HCCL_DEBUG(
207 : "[%s] srcInfo.superPodId %s dstInfo.superPodId %s", __func__, srcInfo.superPodId.c_str(),
208 : dstInfo.superPodId.c_str());
209 0 : if (devType_ == DevType::DEV_TYPE_910_93) {
210 : // 超节点内链路为HCCS
211 0 : if (!srcInfo.superPodId.empty() && srcInfo.superPodId == dstInfo.superPodId) {
212 0 : return CommProtocol::COMM_PROTOCOL_HCCS;
213 : }
214 : }
215 0 : return CommProtocol::COMM_PROTOCOL_RESERVED;
216 : }
217 :
218 : CommProtocol
219 2 : RankGraphV1::GetCommProtocolFromRankInfo(const RankInfo_t& srcInfo, const RankInfo_t& dstInfo, uint32_t netLayer)
220 : {
221 2 : if (srcInfo.deviceInfo.deviceType != dstInfo.deviceInfo.deviceType) {
222 0 : HCCL_ERROR(
223 : "[RankGraphV1][%s] srcType[%d] != dstType[%d]", __func__, srcInfo.deviceInfo.deviceType,
224 : dstInfo.deviceInfo.deviceType);
225 0 : return CommProtocol::COMM_PROTOCOL_RESERVED;
226 : }
227 : // 首先判断是否在同一机内
228 2 : if (srcInfo.serverIdx == dstInfo.serverIdx) {
229 2 : if (netLayer == HCCL_NETLAYER_0) {
230 2 : return GetCommProtocolInSameServer(srcInfo, dstInfo);
231 : // 超节点有HCCL_NETLAYER_1及以上的情况，为HCCS链路，或者同卡不同DIE
232 0 : } else if (
233 0 : netLayer == HCCL_NETLAYER_1 && devType_ == DevType::DEV_TYPE_910_93
234 0 : && (srcInfo.superPodId == dstInfo.superPodId
235 0 : || GetCommProtocolInSameServer(srcInfo, dstInfo) == CommProtocol::COMM_PROTOCOL_SIO)) {
236 0 : return CommProtocol::COMM_PROTOCOL_HCCS;
237 0 : } else if (IsRoceInSameServer(netLayer, srcInfo, dstInfo)) {
238 0 : return CommProtocol::COMM_PROTOCOL_ROCE;
239 : } else {
240 : // 接了交换机才会有HCCL_NETLAYER_1及以上的情况，当前无法判断是否连接交换机，接了交换机走RDMA
241 0 : return CommProtocol::COMM_PROTOCOL_RESERVED;
242 : }
243 : }
244 0 : if (srcInfo.serverIdx != dstInfo.serverIdx) {
245 0 : if (netLayer == HCCL_NETLAYER_0) {
246 0 : HCCL_INFO("[RankGraphV1][%s] ranks[%u,%u] not in same server", __func__, srcInfo.rankId, dstInfo.rankId);
247 0 : return CommProtocol::COMM_PROTOCOL_RESERVED;
248 : }
249 0 : if (netLayer == HCCL_NETLAYER_1) {
250 0 : HCCL_INFO(
251 : "[RankGraphV1][%s] ranks[%u,%u] inter-server but same superPod[%s]", __func__, srcInfo.rankId,
252 : dstInfo.rankId, srcInfo.superPodId.c_str());
253 0 : return GetCommProtocolBetweenServers(srcInfo, dstInfo);
254 : // 跨超走ROCE
255 0 : } else if (
256 0 : !srcInfo.superPodId.empty() && srcInfo.superPodId != dstInfo.superPodId && netLayer == HCCL_NETLAYER_2) {
257 0 : HCCL_INFO(
258 : "[RankGraphV1][%s] ranks[%u,%u] inter-superPod use ROCE", __func__, srcInfo.rankId, dstInfo.rankId);
259 0 : return CommProtocol::COMM_PROTOCOL_ROCE;
260 : }
261 : }
262 0 : return CommProtocol::COMM_PROTOCOL_RESERVED;
263 : }
264 :
265 4 : bool RankGraphV1::NeedIgnoreEndPoints(
266 : CommProtocol srcProtocol, CommProtocol dstProtocol, CommProtocol linkProtocol) const
267 : {
268 4 : if (srcProtocol != dstProtocol) {
269 1 : return true;
270 : } else {
271 : // 两个hccs endpoints间可能是SIO链路
272 : // A + X 两个mesh间是PCIE链路, 310DUO卡两个DIE间链路是HCCS，主次DIE间是PCIE
273 3 : if (srcProtocol == COMM_PROTOCOL_HCCS && dstProtocol == COMM_PROTOCOL_HCCS
274 2 : && linkProtocol == COMM_PROTOCOL_SIO) {
275 1 : return false;
276 2 : } else if (dstProtocol != linkProtocol) {
277 0 : return true;
278 : }
279 : }
280 2 : return false;
281 : }
282 :
283 2 : void RankGraphV1::PrintLinksInfo(CommLink& link) const
284 : {
285 : // 打印CommLink 头部基础信息
286 2 : HCCL_INFO(
287 : "[RankGraphV1][%s] link.header.version[%u] magicWord[0x%08x] size[%u] reserved[%u]", __func__,
288 : link.header.version, link.header.magicWord, link.header.size, link.header.reserved);
289 :
290 : // 打印【源端】srcEndpointDesc 完整信息
291 2 : HCCL_INFO(
292 : "[RankGraphV1][%s] srcProtocol[%d] srcCommAddrType[%d] srcLocType[%d] srcDevPhyId[%u] "
293 : "srcSuperDevId[%u] srcServerIdx[%u] srcSuperPodIdx[%u]",
294 : __func__, link.srcEndpointDesc.protocol, link.srcEndpointDesc.commAddr.type, link.srcEndpointDesc.loc.locType,
295 : link.srcEndpointDesc.loc.device.devPhyId, link.srcEndpointDesc.loc.device.superDevId,
296 : link.srcEndpointDesc.loc.device.serverIdx, link.srcEndpointDesc.loc.device.superPodIdx);
297 :
298 : // 打印【目的端】dstEndpointDesc 完整信息
299 2 : HCCL_INFO(
300 : "[RankGraphV1][%s] dstProtocol[%d] dstCommAddrType[%d] dstLocType[%d] dstDevPhyId[%u] "
301 : "dstSuperDevId[%u] dstServerIdx[%u] dstSuperPodIdx[%u]",
302 : __func__, link.dstEndpointDesc.protocol, link.dstEndpointDesc.commAddr.type, link.dstEndpointDesc.loc.locType,
303 : link.dstEndpointDesc.loc.device.devPhyId, link.dstEndpointDesc.loc.device.superDevId,
304 : link.dstEndpointDesc.loc.device.serverIdx, link.dstEndpointDesc.loc.device.superPodIdx);
305 :
306 : // 打印【链路属性】linkAttr 信息
307 2 : HCCL_INFO("[RankGraphV1][%s] linkProtocol[%d] hop[%u]", __func__, link.linkAttr.linkProtocol, link.linkAttr.hop);
308 2 : }
309 :
310 : HcclResult
311 3 : RankGraphV1::GetLinks(uint32_t netLayer, uint32_t srcRank, uint32_t dstRank, CommLink** linkList, uint32_t* listSize)
312 : {
313 5 : if (rankIndex_.find(srcRank) == rankIndex_.end() || rankIndex_.find(dstRank) == rankIndex_.end()
314 5 : || FindRank(srcRank) == nullptr || FindRank(dstRank) == nullptr) {
315 1 : HCCL_ERROR(
316 : "[RankGraphV1][%s] srcRank[%u] or dstRank[%u] is not existed in rankTable", __func__, srcRank, dstRank);
317 1 : return HCCL_E_PARA;
318 : }
319 :
320 2 : if (netLayer > HCCL_NETLAYER_2) {
321 1 : HCCL_ERROR(
322 : "[RankGraphV1][%s] srcRank[%u] and dstRank[%u] do not have netLayer[%u]", __func__, srcRank, dstRank,
323 : netLayer);
324 1 : return HCCL_E_PARA;
325 : }
326 1 : auto& srcEndpointDescs = rankIndex_[srcRank].endPoints;
327 1 : auto& dstEndpointDescs = rankIndex_[dstRank].endPoints;
328 :
329 1 : const RankInfo_t& srcInfo = rankIndex_[srcRank].rankInfo;
330 1 : const RankInfo_t& dstInfo = rankIndex_[dstRank].rankInfo;
331 1 : CommProtocol protocol = COMM_PROTOCOL_RESERVED;
332 1 : protocol = GetCommProtocolFromRankInfo(srcInfo, dstInfo, netLayer);
333 1 : if (protocol == COMM_PROTOCOL_RESERVED) {
334 0 : HCCL_WARNING("[RankGraphV1][%s] no links between srcRank[%u] dstRank[%u]", __func__, srcRank, dstRank);
335 0 : *linkList = nullptr;
336 0 : *listSize = 0;
337 0 : return HCCL_SUCCESS;
338 : }
339 :
340 : // 1. 查询是否有缓存CommLink信息
341 1 : auto key = std::make_tuple(netLayer, srcRank, dstRank);
342 1 : auto it = rankPairInfo_.find(key);
343 1 : if (it == rankPairInfo_.end()) {
344 : // 没有则创建
345 1 : HCCL_INFO("[RankGraphV1][%s] no cached links, build new srcRank[%u] dstRank[%u]", __func__, srcRank, dstRank);
346 1 : std::vector<CommLink> links;
347 2 : for (size_t i = 0; i < srcEndpointDescs.size(); i++) {
348 2 : for (size_t j = 0; j < dstEndpointDescs.size(); j++) {
349 1 : if (NeedIgnoreEndPoints(srcEndpointDescs[i].protocol, dstEndpointDescs[j].protocol, protocol)) {
350 0 : continue;
351 : }
352 : CommLink link;
353 1 : CHK_RET(CommLinkInit(&link, 1));
354 :
355 1 : link.srcEndpointDesc = srcEndpointDescs[i];
356 1 : link.srcEndpointDesc.protocol = protocol;
357 1 : link.dstEndpointDesc = dstEndpointDescs[j];
358 1 : link.dstEndpointDesc.protocol = protocol;
359 1 : link.linkAttr.linkProtocol = protocol;
360 1 : PrintLinksInfo(link);
361 1 : links.push_back(std::move(link));
362 : }
363 : }
364 1 : it = rankPairInfo_.emplace(std::make_tuple(netLayer, srcRank, dstRank), std::move(links)).first;
365 1 : }
366 1 : HCCL_INFO(
367 : "[RankGraphV1][%s] links, netLayer[%u] srcRank[%u] dstRank[%u] protocol[%d]", __func__, netLayer, srcRank,
368 : dstRank, protocol);
369 :
370 1 : auto& links = it->second;
371 1 : *listSize = static_cast<uint32_t>(links.size());
372 1 : if (links.empty()) {
373 0 : *linkList = nullptr;
374 0 : HCCL_ERROR("[RankGraphV1][%s] links empty for srcRank[%u] dstRank[%u]", __func__, srcRank, dstRank);
375 : } else {
376 1 : *linkList = links.data(); // 连续数组首地址
377 1 : HCCL_INFO(
378 : "[RankGraphV1][%s] srcRank[%u] dstRank[%u] linkList[%p] linkNum[%u]", __func__, srcRank, dstRank, *linkList,
379 : *listSize);
380 : }
381 :
382 1 : return HCCL_SUCCESS;
383 : }
384 :
385 523 : HcclResult RankGraphV1::InitHeterogMode()
386 : {
387 523 : if (topoAttr_.rankInfoList.empty()) {
388 0 : HCCL_ERROR("[RankGraphV1][%s] invalid para. rankInfoList is empty", __func__);
389 0 : return HCCL_E_INTERNAL;
390 : }
391 :
392 523 : std::set<DevType> devTypes;
393 2151 : for (u32 index = 0; index < topoAttr_.rankInfoList.size(); index++) {
394 1628 : devTypes.insert(topoAttr_.rankInfoList[index].deviceType);
395 : }
396 :
397 : // 只包含一种芯片的同构组网
398 523 : if (devTypes.size() == 1) {
399 523 : heterogMode_ = HcclHeterogMode::HCCL_HETEROG_MODE_HOMOGENEOUS;
400 523 : return HCCL_SUCCESS;
401 : }
402 :
403 : // 包含两种芯片的异构混合组网
404 0 : constexpr uint32_t MIX_CHIPS = 2;
405 0 : if (devTypes.size() == MIX_CHIPS && devTypes.find(DevType::DEV_TYPE_910B) != devTypes.end()
406 0 : && devTypes.find(DevType::DEV_TYPE_910_93) != devTypes.end()) {
407 0 : heterogMode_ = HcclHeterogMode::HCCL_HETEROG_MODE_MIX_A2_A3;
408 0 : return HCCL_SUCCESS;
409 : }
410 :
411 0 : std::string devStr;
412 0 : for (auto itSet = devTypes.begin(); itSet != devTypes.end(); itSet++) {
413 0 : if (itSet != devTypes.begin()) {
414 0 : devStr += ", ";
415 : }
416 0 : devStr += std::to_string(static_cast<int>(*itSet));
417 : }
418 0 : HCCL_ERROR(
419 : "[RankGraphV1][%s] Unknown mode[%d], devtypes[%s]", __func__, HcclHeterogMode::HCCL_HETEROG_MODE_INVALID,
420 : devStr.c_str());
421 0 : return HCCL_E_INTERNAL;
422 523 : }
423 :
424 1 : HcclResult RankGraphV1::GetHeterogMode(HcclHeterogMode* mode) const
425 : {
426 1 : *mode = heterogMode_;
427 1 : return HCCL_SUCCESS;
428 : }
429 :
430 2 : HcclResult RankGraphV1::GetNetLayers(uint32_t** netLayers, uint32_t* netLayerNum)
431 : {
432 2 : if (netLayer_.empty()) {
433 1 : HCCL_ERROR("[RankGraphV1][%s] invalid para. netLayer is empty", __func__);
434 1 : return HCCL_E_INTERNAL;
435 : }
436 1 : *netLayers = netLayer_.data();
437 1 : *netLayerNum = netLayer_.size();
438 1 : return HCCL_SUCCESS;
439 : }
440 :
441 528 : HcclResult RankGraphV1::GetInstTopoTypeByNetLayer(uint32_t netLayer, CommTopo* topoType)
442 : {
443 528 : if (netLayer >= netLayer_.size()) {
444 1 : HCCL_ERROR("[RankGraphV1][%s] invalid para. netlayer[%u]", __func__, netLayer);
445 1 : return HCCL_E_PARA;
446 : }
447 527 : DevType deviceType = topoAttr_.deviceType;
448 527 : if (deviceType == DevType::DEV_TYPE_910_93) {
449 34 : if (netLayer == static_cast<uint32_t>(HcclNetLayerlevel::HCCL_NetLayer_L0)) {
450 34 : *topoType = CommTopo::COMM_TOPO_910_93;
451 0 : } else if ((netLayer == static_cast<uint32_t>(HcclNetLayerlevel::HCCL_NetLayer_L1)
452 0 : || (netLayer == static_cast<uint32_t>(HcclNetLayerlevel::HCCL_NetLayer_L2)))) {
453 0 : *topoType = CommTopo::COMM_TOPO_CLOS;
454 : }
455 493 : } else if (deviceType == DevType::DEV_TYPE_910B || deviceType == DevType::DEV_TYPE_910) {
456 488 : if (netLayer == static_cast<uint32_t>(HcclNetLayerlevel::HCCL_NetLayer_L0)) {
457 487 : *topoType = CommTopo::COMM_TOPO_1DMESH;
458 1 : } else if (netLayer == static_cast<uint32_t>(HcclNetLayerlevel::HCCL_NetLayer_L1)) {
459 1 : *topoType = CommTopo::COMM_TOPO_CLOS;
460 : }
461 5 : } else if (deviceType == DevType::DEV_TYPE_310P3) {
462 5 : if (netLayer == static_cast<uint32_t>(HcclNetLayerlevel::HCCL_NetLayer_L0)) {
463 5 : *topoType = CommTopo::COMM_TOPO_310P;
464 : }
465 : }
466 527 : return HCCL_SUCCESS;
467 : }
468 :
469 2 : HcclResult RankGraphV1::GetInstSizeByNetLayer(uint32_t netLayer, uint32_t* rankNum)
470 : {
471 2 : if (netLayer >= netLayer_.size()) {
472 1 : HCCL_ERROR("[RankGraphV1][%s] invalid para. netlayer[%u]", __func__, netLayer);
473 1 : return HCCL_E_PARA;
474 : }
475 :
476 1 : if (rankList_.find(netLayer) == rankList_.end()) {
477 1 : HCCL_ERROR("[RankGraphV1][%s] failed to find rankList map. netlayer[%u]", __func__, netLayer);
478 1 : return HCCL_E_INTERNAL;
479 : }
480 0 : *rankNum = rankList_[netLayer].size();
481 :
482 0 : return HCCL_SUCCESS;
483 : }
484 :
485 2 : HcclResult RankGraphV1::GetInstRanksByNetLayer(uint32_t netLayer, uint32_t** rankList, uint32_t* rankNum)
486 : {
487 2 : if (netLayer >= netLayer_.size()) {
488 1 : HCCL_ERROR("[RankGraphV1][%s] invalid para. netlayer[%u]", __func__, netLayer);
489 1 : return HCCL_E_PARA;
490 : }
491 :
492 1 : if (rankList_.find(netLayer) == rankList_.end()) {
493 1 : HCCL_ERROR("[RankGraphV1][%s] failed to find rankList map. netlayer[%u]", __func__, netLayer);
494 1 : return HCCL_E_INTERNAL;
495 : }
496 0 : *rankNum = rankList_[netLayer].size();
497 0 : *rankList = rankList_[netLayer].data();
498 :
499 0 : return HCCL_SUCCESS;
500 : }
501 :
502 2 : HcclResult RankGraphV1::GetInstSizeListByNetLayer(uint32_t netLayer, uint32_t** instSizeList, uint32_t* listSize)
503 : {
504 2 : if (netLayer >= netLayer_.size()) {
505 1 : HCCL_ERROR("[RankGraphV1][%s] invalid para. netlayer[%u]", __func__, netLayer);
506 1 : return HCCL_E_PARA;
507 : }
508 :
509 1 : if (rankSizeList_.find(netLayer) == rankSizeList_.end()) {
510 1 : HCCL_ERROR("[RankGraphV1][%s] failed to find rankSizeList map. netlayer[%u]", __func__, netLayer);
511 1 : return HCCL_E_INTERNAL;
512 : }
513 0 : *instSizeList = rankSizeList_[netLayer].data();
514 0 : *listSize = rankSizeList_[netLayer].size();
515 :
516 0 : return HCCL_SUCCESS;
517 : }
518 :
519 1 : HcclResult RankGraphV1::GetTopoInstsByLayer(uint32_t netLayer, uint32_t** topoInsts, uint32_t* topoInstNum)
520 : {
521 1 : if (netLayer >= netLayer_.size()) {
522 0 : return HCCL_E_PARA;
523 : }
524 1 : if (rankSizeList_.find(netLayer) == rankSizeList_.end()) {
525 0 : return HCCL_E_INTERNAL;
526 : }
527 :
528 1 : uint32_t instNum = rankSizeList_[netLayer].size();
529 1 : *topoInstNum = instNum;
530 :
531 1 : static std::vector<uint32_t> sTopoInstList;
532 1 : sTopoInstList.clear();
533 1 : sTopoInstList.resize(instNum);
534 5 : for (uint32_t i = 0; i < instNum; ++i) {
535 4 : sTopoInstList[i] = i;
536 : }
537 :
538 1 : *topoInsts = sTopoInstList.data();
539 :
540 1 : return HCCL_SUCCESS;
541 : }
542 :
543 1 : HcclResult RankGraphV1::GetTopoType(uint32_t netLayer, CommTopo* topoType)
544 : {
545 1 : if (netLayer >= netLayer_.size()) {
546 0 : return HCCL_E_PARA;
547 : }
548 1 : return GetInstTopoTypeByNetLayer(netLayer, topoType);
549 : }
550 :
551 1 : HcclResult RankGraphV1::GetRanksByTopoInst(uint32_t netLayer, uint32_t topoInstId, uint32_t** ranks, uint32_t* rankNum)
552 : {
553 1 : if (netLayer >= netLayer_.size()) {
554 0 : return HCCL_E_PARA;
555 : }
556 1 : if (devType_ != DevType::DEV_TYPE_910B) {
557 0 : return HCCL_E_NOT_SUPPORT;
558 : }
559 1 : auto rankListIt = rankList_.find(netLayer);
560 1 : auto rankSizeListIt = rankSizeList_.find(netLayer);
561 1 : if (rankListIt != rankList_.end() && rankSizeListIt != rankSizeList_.end()) {
562 1 : if (topoInstId >= rankSizeListIt->second.size()) {
563 0 : return HCCL_E_PARA;
564 : }
565 : }
566 :
567 1 : *ranks = rankListIt->second.data();
568 1 : if (topoInstId < rankSizeListIt->second.size()) {
569 1 : *rankNum = rankSizeListIt->second[topoInstId];
570 : } else {
571 0 : *rankNum = 0;
572 : }
573 :
574 1 : return HCCL_SUCCESS;
575 : }
576 :
577 3 : std::vector<const RankInfo_t*> RankGraphV1::GetRanksInTopoInst(uint32_t netLayer, uint32_t topoInstId)
578 : {
579 3 : std::vector<const RankInfo_t*> ranks;
580 :
581 3 : if (netLayer >= netLayer_.size()) {
582 0 : return ranks;
583 : }
584 3 : if (netLayer == static_cast<uint32_t>(HcclNetLayerlevel::HCCL_NetLayer_L0)) {
585 2 : auto it = serverToRank_.find(topoInstId);
586 2 : if (it == serverToRank_.end()) {
587 2 : return ranks;
588 : }
589 0 : for (const auto& rankInfo : it->second) {
590 0 : for (const auto& graphInfo : rankGraph_) {
591 0 : if (graphInfo.rankId == rankInfo.userRank) {
592 0 : ranks.push_back(&graphInfo);
593 : }
594 : }
595 : }
596 : } else {
597 1 : auto listIt = rankList_.find(netLayer);
598 1 : auto sizeListIt = rankSizeList_.find(netLayer);
599 1 : if (listIt != rankList_.end() && sizeListIt != rankSizeList_.end()) {
600 1 : if (topoInstId >= sizeListIt->second.size()) {
601 0 : return ranks;
602 : }
603 : }
604 :
605 3 : for (uint32_t userRank : listIt->second) {
606 3 : for (const auto& rankInfo : rankGraph_) {
607 3 : if (rankInfo.rankId == userRank) {
608 2 : ranks.push_back(&rankInfo);
609 2 : break;
610 : }
611 : }
612 : }
613 : }
614 1 : return ranks;
615 0 : }
616 :
617 : std::set<CommProtocol>
618 1 : RankGraphV1::GetProtocolsByConnections(uint32_t netLayer, const std::vector<const RankInfo_t*>& topoInstRanks)
619 : {
620 1 : std::set<CommProtocol> protocols;
621 :
622 1 : const RankInfo_t* srcRankInfo = nullptr;
623 1 : for (const auto& rankInfo : rankGraph_) {
624 1 : if (rankInfo.rankId == rankData_.userRank) {
625 1 : srcRankInfo = &rankInfo;
626 1 : break;
627 : }
628 : }
629 1 : if (srcRankInfo == nullptr) {
630 0 : return protocols;
631 : }
632 :
633 2 : for (const RankInfo_t* dstRankInfo : topoInstRanks) {
634 1 : if (dstRankInfo->rankId == rankData_.userRank) {
635 0 : continue;
636 : }
637 :
638 1 : CommProtocol protocol = GetCommProtocolFromRankInfo(*srcRankInfo, *dstRankInfo, netLayer);
639 1 : if (protocol != COMM_PROTOCOL_RESERVED) {
640 1 : protocols.insert(protocol);
641 : }
642 : }
643 :
644 1 : return protocols;
645 0 : }
646 :
647 3 : HcclResult RankGraphV1::GetEndpointNum(uint32_t netLayer, uint32_t topoInstId, uint32_t* num)
648 : {
649 3 : if (netLayer >= netLayer_.size()) {
650 1 : HCCL_ERROR("[RankGraphV1][%s] invalid para. netlayer[%u]", __func__, netLayer);
651 1 : return HCCL_E_PARA;
652 : }
653 2 : if (rankIndex_.empty()) {
654 1 : HCCL_ERROR("[RankGraphV1][%s] rankIndex is empty", __func__);
655 1 : return HCCL_E_INTERNAL;
656 : }
657 1 : std::vector<const RankInfo_t*> topoInstRanks = GetRanksInTopoInst(netLayer, topoInstId);
658 1 : if (topoInstRanks.empty()) {
659 1 : *num = 0;
660 1 : return HCCL_SUCCESS;
661 : }
662 :
663 0 : std::set<CommProtocol> protocols = GetProtocolsByConnections(netLayer, topoInstRanks);
664 0 : if (protocols.empty()) {
665 0 : HCCL_INFO(
666 : "[RankGraphV1][%s] no protocols found for netlayer [%u] topoInstId[%u]", __func__, netLayer, topoInstId);
667 : }
668 :
669 0 : const RankGraphInfo* currentRankInfo = nullptr;
670 0 : for (auto& pair : rankIndex_) {
671 0 : if (pair.second.rankInfo.rankId == rankData_.userRank) {
672 0 : currentRankInfo = &pair.second;
673 0 : break;
674 : }
675 : }
676 0 : if (currentRankInfo == nullptr) {
677 0 : return HCCL_E_INTERNAL;
678 : }
679 :
680 0 : uint32_t count = 0;
681 0 : for (const auto& endpoint : currentRankInfo->endPoints) {
682 0 : if (protocols.find(endpoint.protocol) != protocols.end()) {
683 0 : count++;
684 : }
685 : }
686 0 : *num = count;
687 :
688 0 : return HCCL_SUCCESS;
689 1 : }
690 :
691 : HcclResult
692 5 : RankGraphV1::GetEndpointDesc(uint32_t netLayer, uint32_t topoInstId, uint32_t* descNum, EndpointDesc* endpointDesc)
693 : {
694 5 : if (netLayer >= netLayer_.size()) {
695 1 : HCCL_ERROR("[RankGraphV1][%s] invalid para. netlayer[%u]", __func__, netLayer);
696 1 : return HCCL_E_PARA;
697 : }
698 4 : if (descNum == nullptr || endpointDesc == nullptr) {
699 2 : HCCL_ERROR("[RankGraphV1][%s] invalid para. null ptr", __func__);
700 2 : return HCCL_E_PARA;
701 : }
702 2 : if (rankIndex_.empty()) {
703 1 : HCCL_ERROR("[RankGraphV1][%s] rankIndex is empty", __func__);
704 1 : return HCCL_E_INTERNAL;
705 : }
706 1 : std::vector<const RankInfo_t*> topoInstRanks = GetRanksInTopoInst(netLayer, topoInstId);
707 1 : if (topoInstRanks.empty()) {
708 1 : *descNum = 0;
709 1 : return HCCL_SUCCESS;
710 : }
711 :
712 0 : std::set<CommProtocol> protocols = GetProtocolsByConnections(netLayer, topoInstRanks);
713 0 : if (protocols.empty()) {
714 0 : HCCL_INFO(
715 : "[RankGraphV1][GetEndpointDesc] no protocols found, netlayer [%u] topoInstId[%u]", netLayer, topoInstId);
716 : }
717 :
718 0 : const RankGraphInfo* currRankInfo = nullptr;
719 0 : for (auto& pair : rankIndex_) {
720 0 : if (pair.second.rankInfo.rankId == rankData_.userRank) {
721 0 : currRankInfo = &pair.second;
722 0 : break;
723 : }
724 : }
725 0 : if (currRankInfo == nullptr) {
726 0 : return HCCL_E_INTERNAL;
727 : }
728 :
729 0 : uint32_t count = 0;
730 0 : for (const auto& endpoint : currRankInfo->endPoints) {
731 0 : if (protocols.find(endpoint.protocol) != protocols.end()) {
732 0 : if (count >= *descNum) {
733 0 : return HCCL_E_PARA;
734 : }
735 0 : endpointDesc[count] = endpoint;
736 0 : count++;
737 : }
738 : }
739 0 : *descNum = count;
740 :
741 0 : return HCCL_SUCCESS;
742 1 : }
743 :
744 : const EndpointDesc*
745 5 : RankGraphV1::MatchEndpointByAddr(const RankGraphInfo& rankGraphInfo, const EndpointDesc* endPointDesc) const
746 : {
747 7 : for (const auto& endpoint : rankGraphInfo.endPoints) {
748 5 : if (endpoint.commAddr.type != endPointDesc->commAddr.type || endpoint.protocol != endPointDesc->protocol) {
749 0 : continue;
750 : }
751 5 : bool matched = false;
752 5 : switch (endpoint.commAddr.type) {
753 2 : case COMM_ADDR_TYPE_IP_V4:
754 : matched
755 2 : = (memcmp(&endpoint.commAddr.addr, &endPointDesc->commAddr.addr, sizeof(endpoint.commAddr.addr))
756 : == 0);
757 2 : break;
758 1 : case COMM_ADDR_TYPE_IP_V6:
759 : matched
760 1 : = (memcmp(&endpoint.commAddr.addr6, &endPointDesc->commAddr.addr6, sizeof(endpoint.commAddr.addr6))
761 : == 0);
762 1 : break;
763 1 : case COMM_ADDR_TYPE_ID:
764 1 : matched = true;
765 1 : break;
766 1 : default:
767 1 : break;
768 : }
769 5 : if (matched) {
770 3 : return &endpoint;
771 : }
772 : }
773 2 : return nullptr;
774 : }
775 :
776 : HcclResult
777 7 : RankGraphV1::FillAttr(EndpointAttr endpointAttr, const EndpointDesc* foundEndpoint, uint32_t infoLen, void* info) const
778 : {
779 7 : switch (endpointAttr) {
780 2 : case ENDPOINT_ATTR_BW_COEFF: {
781 2 : CHK_PRT_RET(
782 : infoLen != sizeof(EndpointAttrBwCoeff),
783 : HCCL_ERROR(
784 : "[RankGraphV1::GetEndpointInfo] Size mismatch for ENDPOINT_ATTR_BW_COEFF: expected %zu, actual %u",
785 : sizeof(EndpointAttrBwCoeff), infoLen),
786 : HCCL_E_PARA);
787 1 : *(static_cast<EndpointAttrBwCoeff*>(info)) = 1;
788 1 : break;
789 : }
790 2 : case ENDPOINT_ATTR_DIE_ID: {
791 2 : CHK_PRT_RET(
792 : infoLen != sizeof(EndpointAttrDieId),
793 : HCCL_ERROR(
794 : "[RankGraphV1::GetEndpointInfo] Size mismatch for ENDPOINT_ATTR_DIE_ID: expected %zu, actual %u",
795 : sizeof(EndpointAttrDieId), infoLen),
796 : HCCL_E_PARA);
797 1 : *(static_cast<EndpointAttrDieId*>(info)) = foundEndpoint->loc.device.superDevId;
798 1 : break;
799 : }
800 2 : case ENDPOINT_ATTR_LOCATION: {
801 2 : CHK_PRT_RET(
802 : infoLen != sizeof(EndpointAttrLocation),
803 : HCCL_ERROR(
804 : "[RankGraphV1::GetEndpointInfo] Size mismatch for ENDPOINT_ATTR_LOCATION: expected %zu, actual %u",
805 : sizeof(EndpointAttrLocation), infoLen),
806 : HCCL_E_PARA);
807 1 : *(static_cast<EndpointAttrLocation*>(info)) = foundEndpoint->loc.locType;
808 1 : break;
809 : }
810 1 : default: {
811 1 : HCCL_ERROR("[RankGraphV1::GetEndpointInfo] Invalid endpointAttr[%d]", endpointAttr);
812 1 : return HCCL_E_PARA;
813 : }
814 : }
815 3 : return HCCL_SUCCESS;
816 : }
817 :
818 5 : HcclResult RankGraphV1::GetEndpointInfo(
819 : uint32_t rankId, const EndpointDesc* endPointDesc, EndpointAttr endpointAttr, uint32_t infoLen, void* info)
820 : {
821 5 : if (endPointDesc == nullptr || info == nullptr) {
822 2 : HCCL_ERROR("[RankGraphV1::GetEndpointInfo] Invalid parameter, null pointer");
823 2 : return HCCL_E_PTR;
824 : }
825 :
826 3 : if (rankIndex_.empty()) {
827 1 : HCCL_ERROR("[RankGraphV1::GetEndpointInfo] rankIndex is empty");
828 1 : return HCCL_E_INTERNAL;
829 : }
830 :
831 2 : auto rankIt = rankIndex_.find(rankId);
832 2 : if (rankIt == rankIndex_.end()) {
833 1 : HCCL_ERROR("[RankGraphV1::GetEndpointInfo] rankId[%u] not found in rankIndex", rankId);
834 1 : return HCCL_E_NOT_FOUND;
835 : }
836 :
837 1 : const RankGraphInfo& rankGraphInfo = rankIt->second;
838 1 : const EndpointDesc* foundEndpoint = MatchEndpointByAddr(rankGraphInfo, endPointDesc);
839 1 : if (foundEndpoint == nullptr) {
840 0 : HCCL_ERROR("[RankGraphV1::GetEndpointInfo] No matching endpoint found for rankId[%u]", rankId);
841 0 : return HCCL_E_NOT_FOUND;
842 : }
843 :
844 1 : return FillAttr(endpointAttr, foundEndpoint, infoLen, info);
845 : }
846 :
847 2 : HcclResult RankGraphV1::GetRankSize(uint32_t* rankSize)
848 : {
849 2 : CHK_PTR_NULL(rankSize);
850 1 : *rankSize = rankGraph_.size();
851 1 : return HCCL_SUCCESS;
852 : }
853 :
854 3 : HcclResult RankGraphV1::GetDevicePort(const uint32_t rank, uint32_t* devPort)
855 : {
856 3 : CHK_PTR_NULL(devPort);
857 2 : const RankInfo_t* rankInfo = FindRank(rank);
858 2 : if (rankInfo == nullptr) {
859 1 : HCCL_ERROR("[RankGraphV1][%s] rank[%u] not found", __func__, rank);
860 1 : return HCCL_E_PARA;
861 : }
862 1 : *devPort = rankInfo->deviceInfo.port;
863 1 : return HCCL_SUCCESS;
864 : }
865 :
866 3908 : bool RankGraphSort(const RankInfo& first, const RankInfo& second)
867 : {
868 3908 : if (first.serverIdx != second.serverIdx) {
869 453 : return first.serverIdx < second.serverIdx;
870 : } else {
871 3455 : return first.userRank < second.userRank;
872 : }
873 : }
874 :
875 523 : HcclResult RankGraphV1::InitGraphRankInfo()
876 : {
877 1997 : for (u32 index = 0; index < rankGraph_.size(); index++) {
878 1474 : struct GraphRankInfo graphRankInfo = {};
879 1474 : graphRankInfo.rankId = rankGraph_[index].rankId;
880 1474 : graphRankInfo.localRank = rankGraph_[index].localRank;
881 1474 : graphRankInfo.serverId = rankGraph_[index].serverId;
882 1474 : graphRankInfo.serverIdx = rankGraph_[index].serverIdx;
883 1474 : graphRankInfo.superDeviceId = rankGraph_[index].superDeviceId;
884 1474 : graphRankInfo.superPodId = rankGraph_[index].superPodId;
885 1474 : graphRankInfo.superPodIdx = rankGraph_[index].superPodIdx;
886 1474 : graphRankInfo.hostPort = rankGraph_[index].hostPort;
887 1474 : graphRankInfo.nodeId = rankGraph_[index].nodeId;
888 1474 : graphRankInfo.itemId = rankGraph_[index].itemId;
889 1474 : graphRankInfo.deviceInfo.devicePhyId = rankGraph_[index].deviceInfo.devicePhyId;
890 1474 : graphRankInfo.deviceInfo.deviceType = rankGraph_[index].deviceInfo.deviceType;
891 1474 : graphRankInfo.deviceInfo.port = rankGraph_[index].deviceInfo.port;
892 1474 : graphRankInfo.deviceInfo.vnicPort = rankGraph_[index].deviceInfo.vnicPort;
893 1474 : graphRankInfo.deviceInfo.backupPort = rankGraph_[index].deviceInfo.backupPort;
894 1474 : graphRankInfo.bindDeviceId = rankGraph_[index].bindDeviceId;
895 1474 : graphRankInfo.originalSuperPodId = rankGraph_[index].originalSuperPodId;
896 :
897 1474 : graphRankInfo_.push_back(graphRankInfo);
898 1474 : }
899 :
900 523 : return HCCL_SUCCESS;
901 : }
902 :
903 1 : HcclResult RankGraphV1::GetRankGraphInfo(GraphType type, void** graph, uint32_t* len)
904 : {
905 1 : switch (type) {
906 0 : case RANK_GRAPH_910_93: {
907 0 : *graph = graphRankInfo_.data();
908 0 : *len = graphRankInfo_.size() * sizeof(GraphRankInfo);
909 0 : break;
910 : }
911 1 : default: {
912 1 : HCCL_ERROR("[RankGraphV1][%s]Graph type[%d] is invalid", __func__, type);
913 1 : return HCCL_E_NOT_SUPPORT;
914 : }
915 : }
916 0 : return HCCL_SUCCESS;
917 : }
918 :
919 0 : HcclResult RankGraphV1::GetDeviceId([[maybe_unused]] uint32_t rankId, [[maybe_unused]] uint32_t* deviceId)
920 : {
921 0 : return HCCL_SUCCESS;
922 : }
923 :
924 523 : HcclResult RankGraphV1::InitRankInfo()
925 : {
926 523 : auto& rankInfoList = topoAttr_.rankInfoList;
927 831 : for (u32 index = 0; index < rankInfoList.size(); index++) {
928 831 : if (topoAttr_.userRank == rankInfoList[index].userRank) {
929 523 : rankData_ = rankInfoList[index];
930 523 : break;
931 : }
932 : }
933 523 : CHK_RET(InitServerRankInfo());
934 523 : CHK_RET(InitSuperPodRankInfo());
935 523 : CHK_RET(InitGraphRankInfo());
936 523 : return HCCL_SUCCESS;
937 : }
938 :
939 523 : HcclResult RankGraphV1::InitServerRankInfo()
940 : {
941 523 : u32 serverIdx = 0;
942 523 : auto& rankInfoList = topoAttr_.rankInfoList;
943 2151 : for (u32 index = 0; index < rankInfoList.size(); index++) {
944 1628 : serverIdx = rankInfoList[index].serverIdx;
945 1628 : auto itServer = serverToRank_.find(serverIdx);
946 1628 : if (itServer != serverToRank_.end()) {
947 885 : itServer->second.push_back(rankInfoList[index]);
948 : } else {
949 743 : std::vector<RankInfo> rankVecTmp;
950 743 : rankVecTmp.push_back(rankInfoList[index]);
951 743 : serverToRank_.insert(std::make_pair(serverIdx, rankVecTmp));
952 743 : }
953 : }
954 : // 调整每个server内的user_rank排序(server内userRank从小到大,一定连续)
955 1266 : for (auto iterMap = serverToRank_.begin(); iterMap != serverToRank_.end(); iterMap++) {
956 743 : if (!(iterMap->second).empty()) {
957 743 : std::sort(iterMap->second.begin(), iterMap->second.end(), RankGraphSort);
958 : }
959 : }
960 523 : serverIdx = rankData_.serverIdx;
961 523 : auto rankVec = serverToRank_.find(serverIdx);
962 523 : if (rankVec != serverToRank_.end()) {
963 523 : std::string rankIdListServer;
964 1846 : for (auto iter : serverToRank_[serverIdx]) {
965 1323 : rankIdListServer += std::to_string(iter.userRank) + " ";
966 1323 : }
967 523 : HCCL_INFO(
968 : "[RankGraphV1][%s] devtype[%d], curRank[%u], serverToRanklist[%s]", __func__, topoAttr_.deviceType,
969 : rankData_.userRank, rankIdListServer.c_str());
970 523 : }
971 523 : return HCCL_SUCCESS;
972 : }
973 :
974 523 : HcclResult RankGraphV1::InitSuperPodRankInfo()
975 : {
976 523 : auto& rankInfoList = topoAttr_.rankInfoList;
977 2151 : for (u32 index = 0; index < rankInfoList.size(); index++) {
978 : // 填充superPodRankMap_, 记录superPodId -> rankInfo
979 1628 : HCCL_DEBUG(
980 : "[RankGraphV1][%s] superPodIdx[%u],superPodId[%s]", __func__, rankInfoList[index].superPodIdx,
981 : rankInfoList[index].superPodId.c_str());
982 1628 : auto itSuperPod = superPodToRank_.find(rankInfoList[index].superPodIdx);
983 1628 : if (itSuperPod != superPodToRank_.end()) {
984 1064 : itSuperPod->second.push_back(rankInfoList[index]);
985 : } else {
986 564 : std::vector<RankInfo> rankVecTmp;
987 564 : rankVecTmp.push_back(rankInfoList[index]);
988 564 : superPodToRank_.insert(std::make_pair(rankInfoList[index].superPodIdx, rankVecTmp));
989 564 : }
990 : }
991 :
992 : // 调整每个superPod内的user_rank排序, 按照serverIdx从小到大、userRank从小到大排序
993 1087 : for (auto iterMap = superPodToRank_.begin(); iterMap != superPodToRank_.end(); iterMap++) {
994 564 : if (!(iterMap->second).empty()) {
995 564 : std::sort(iterMap->second.begin(), iterMap->second.end(), RankGraphSort);
996 : }
997 : }
998 :
999 523 : if (superPodToRank_.find(rankData_.superPodIdx) != superPodToRank_.end()) {
1000 523 : std::string rankIdListPod;
1001 2110 : for (auto iter : superPodToRank_[rankData_.superPodIdx]) {
1002 1587 : rankIdListPod += std::to_string(iter.userRank) + " ";
1003 1587 : }
1004 523 : HCCL_INFO(
1005 : "[RankGraphV1][%s] curRank[%u], curSuperPod[%s] superPodToRanklist[%s]", __func__, rankData_.userRank,
1006 : rankData_.superPodId.c_str(), rankIdListPod.c_str());
1007 523 : }
1008 523 : return HCCL_SUCCESS;
1009 : }
1010 :
1011 523 : HcclResult RankGraphV1::InitNetLayer()
1012 : {
1013 523 : netLayer_.clear();
1014 523 : netLayer_.push_back(static_cast<uint32_t>(HcclNetLayerlevel::HCCL_NetLayer_L0));
1015 :
1016 523 : u32 serverIdx = rankData_.serverIdx;
1017 523 : auto rankVec = serverToRank_.find(serverIdx);
1018 523 : if (rankVec == serverToRank_.end()) {
1019 0 : HCCL_ERROR("[RankGraphV1][%s] find serverToRank failed, serverIdx[%u]", __func__, serverIdx);
1020 0 : return HCCL_E_INTERNAL;
1021 : }
1022 523 : std::vector<u32> rankListTmp;
1023 1846 : for (auto iter : serverToRank_[serverIdx]) {
1024 1323 : rankListTmp.push_back(iter.userRank);
1025 1323 : }
1026 523 : rankList_.insert({static_cast<uint32_t>(HcclNetLayerlevel::HCCL_NetLayer_L0), rankListTmp});
1027 :
1028 523 : std::vector<u32> rankSizeListTmp;
1029 1266 : for (auto iter : serverToRank_) {
1030 743 : rankSizeListTmp.push_back(iter.second.size());
1031 743 : }
1032 523 : rankSizeList_.insert({static_cast<uint32_t>(HcclNetLayerlevel::HCCL_NetLayer_L0), rankSizeListTmp});
1033 :
1034 523 : DevType deviceType = topoAttr_.deviceType;
1035 523 : if (serverToRank_.size() > 1) {
1036 102 : netLayer_.push_back(static_cast<uint32_t>(HcclNetLayerlevel::HCCL_NetLayer_L1));
1037 102 : if (deviceType == DevType::DEV_TYPE_910B || deviceType == DevType::DEV_TYPE_910) {
1038 80 : std::vector<u32> rankListTmp1;
1039 317 : for (auto& pair : serverToRank_) {
1040 588 : for (auto iter : pair.second) {
1041 351 : rankListTmp1.push_back(iter.userRank);
1042 351 : }
1043 : }
1044 80 : rankList_.insert({static_cast<uint32_t>(HcclNetLayerlevel::HCCL_NetLayer_L1), rankListTmp1});
1045 80 : rankSizeList_.insert(
1046 80 : {static_cast<uint32_t>(HcclNetLayerlevel::HCCL_NetLayer_L1), {topoAttr_.userRankSize}});
1047 102 : } else if (deviceType == DevType::DEV_TYPE_910_93) {
1048 22 : auto it = superPodToRank_.find(rankData_.superPodIdx);
1049 22 : if (it == superPodToRank_.end()) {
1050 0 : HCCL_ERROR(
1051 : "[RankGraphV1][%s] find superPodToRank_ failed, superPodIdx[%u]", __func__, rankData_.superPodIdx);
1052 0 : return HCCL_E_INTERNAL;
1053 : }
1054 22 : std::vector<u32> rankListTmp1;
1055 88 : for (auto iter : superPodToRank_[rankData_.superPodIdx]) {
1056 66 : rankListTmp1.push_back(iter.userRank);
1057 66 : }
1058 22 : std::vector<u32> rankSizeListTmp1;
1059 85 : for (auto iter : superPodToRank_) {
1060 63 : rankSizeListTmp1.push_back(iter.second.size());
1061 63 : }
1062 22 : rankList_.insert({static_cast<uint32_t>(HcclNetLayerlevel::HCCL_NetLayer_L1), rankListTmp1});
1063 22 : rankSizeList_.insert({static_cast<uint32_t>(HcclNetLayerlevel::HCCL_NetLayer_L1), rankSizeListTmp1});
1064 22 : }
1065 : }
1066 :
1067 523 : if (deviceType == DevType::DEV_TYPE_910_93 && superPodToRank_.size() > 1) {
1068 8 : netLayer_.push_back(static_cast<uint32_t>(HcclNetLayerlevel::HCCL_NetLayer_L2));
1069 8 : std::vector<u32> rankListTmp2;
1070 57 : for (const auto& pair : superPodToRank_) {
1071 98 : for (auto iter : pair.second) {
1072 49 : rankListTmp2.push_back(iter.userRank);
1073 49 : }
1074 : }
1075 8 : rankList_.insert({static_cast<uint32_t>(HcclNetLayerlevel::HCCL_NetLayer_L2), rankListTmp2});
1076 16 : rankSizeList_.insert({static_cast<uint32_t>(HcclNetLayerlevel::HCCL_NetLayer_L2), {topoAttr_.userRankSize}});
1077 8 : }
1078 523 : return HCCL_SUCCESS;
1079 523 : }
1080 : }; // namespace hccl
|
