Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include <string>
12 : #include "rank_graph.h"
13 : #include "externalinput_pub.h"
14 : #include "comm_base_pub.h"
15 :
16 : namespace hccl {
17 :
18 : // 根据 rankId 获取 rank 信息
19 6 : const RankInfo_t* RankGraphV1::FindRank(uint32_t rankId) const
20 : {
21 6 : auto it = rankIndex_.find(rankId);
22 6 : if (it == rankIndex_.end()) {
23 1 : return nullptr;
24 : }
25 5 : return &(it->second.rankInfo);
26 : }
27 :
28 491 : HcclResult RankGraphV1::DevTypeToCommProtocol(DevType& type, CommProtocol& protocol) const
29 : {
30 491 : CHK_RET(hrtGetDeviceType(type));
31 491 : switch (type) {
32 491 : case DevType::DEV_TYPE_910B:
33 : case DevType::DEV_TYPE_910_93:
34 : case DevType::DEV_TYPE_910:
35 491 : protocol = CommProtocol::COMM_PROTOCOL_ROCE;
36 491 : break;
37 0 : case DevType::DEV_TYPE_310P1:
38 : case DevType::DEV_TYPE_310P3:
39 0 : protocol = CommProtocol::COMM_PROTOCOL_PCIE;
40 0 : break;
41 0 : case DevType::DEV_TYPE_NOSOC:
42 0 : protocol = CommProtocol::COMM_PROTOCOL_PCIE;
43 0 : break;
44 0 : case DevType::DEV_TYPE_950:
45 : case DevType::DEV_TYPE_960:
46 : // 待扩展UB的协议，当前先不支持
47 0 : protocol = CommProtocol::COMM_PROTOCOL_RESERVED;
48 0 : break;
49 0 : default:
50 0 : HCCL_ERROR("[RankGraphV1] Unknown comm devType: %d", type);
51 0 : return HCCL_E_PARA;
52 : }
53 491 : return HCCL_SUCCESS;
54 : }
55 :
56 : HcclResult
57 1474 : RankGraphV1::BuildRankGraphInfo(const RankInfo_t& rankItem, const CommProtocol& protocol, RankGraphInfo& outInfo) const
58 : {
59 1474 : HCCL_INFO(
60 : "[RankGraphV1][%s] rankId[%u] serverId[%s] serverIdx[%u] superDeviceId[%u] superPodId[%s] "
61 : "devicePhyId[%d]",
62 : __func__, rankItem.rankId, rankItem.serverId.c_str(), rankItem.serverIdx, rankItem.superDeviceId,
63 : rankItem.superPodId.c_str(), rankItem.deviceInfo.devicePhyId);
64 1474 : outInfo.rankInfo = rankItem;
65 1474 : std::vector<HcclIpAddress> addrs = rankItem.deviceInfo.deviceIp;
66 2987 : for (const auto& addr : addrs) {
67 : EndpointDesc point;
68 1513 : CHK_RET(static_cast<HcclResult>(EndpointDescInit(&point, 1)));
69 :
70 : // 初始化ROCE协议的基础点位信息
71 1513 : if (addr.IsIPv6()) {
72 0 : point.commAddr.type = COMM_ADDR_TYPE_IP_V6;
73 0 : point.commAddr.addr6 = addr.GetBinaryAddress().addr6;
74 : } else {
75 1513 : point.commAddr.type = COMM_ADDR_TYPE_IP_V4;
76 1513 : point.commAddr.addr = addr.GetBinaryAddress().addr;
77 : }
78 1513 : point.protocol = protocol;
79 1513 : if (rankItem.deviceInfo.nicDeploy == NICDeployment::NIC_DEPLOYMENT_HOST) {
80 0 : point.loc.locType = ENDPOINT_LOC_TYPE_HOST;
81 : } else {
82 1513 : point.loc.locType = ENDPOINT_LOC_TYPE_DEVICE;
83 : }
84 1513 : point.loc.device.devPhyId = rankItem.deviceInfo.devicePhyId;
85 1513 : point.loc.device.superDevId = rankItem.superDeviceId;
86 1513 : point.loc.device.serverIdx = rankItem.serverIdx;
87 1513 : point.loc.device.superPodIdx = rankItem.superPodIdx;
88 : // ROCE协议
89 1513 : outInfo.endPoints.push_back(std::move(point));
90 :
91 : // HCCS 协议
92 1513 : point.loc.locType = ENDPOINT_LOC_TYPE_DEVICE;
93 1513 : if (devType_ == DevType::DEV_TYPE_910B || devType_ == DevType::DEV_TYPE_910_93
94 816 : || devType_ == DevType::DEV_TYPE_310P1 || devType_ == DevType::DEV_TYPE_310P3) {
95 697 : EndpointDesc hccsPoint = point;
96 697 : hccsPoint.protocol = COMM_PROTOCOL_HCCS;
97 697 : hccsPoint.commAddr.type = COMM_ADDR_TYPE_ID;
98 697 : outInfo.endPoints.push_back(std::move(hccsPoint));
99 : }
100 :
101 : // PCIE 协议
102 1513 : if (devType_ == DevType::DEV_TYPE_910B || devType_ == DevType::DEV_TYPE_310P1
103 849 : || devType_ == DevType::DEV_TYPE_310P3) {
104 664 : EndpointDesc pciePoint = point;
105 664 : pciePoint.protocol = COMM_PROTOCOL_PCIE;
106 664 : pciePoint.commAddr.type = COMM_ADDR_TYPE_ID;
107 664 : outInfo.endPoints.push_back(std::move(pciePoint));
108 : }
109 : }
110 1474 : return HCCL_SUCCESS;
111 1474 : }
112 :
113 491 : HcclResult RankGraphV1::Init(const RankTable_t& rankTable, const HcclTopoAttr& topoAttr)
114 : {
115 491 : rankTable_ = rankTable;
116 491 : topoAttr_ = topoAttr;
117 491 : rankIndex_.clear();
118 491 : rankPairInfo_.clear();
119 491 : HCCL_INFO("[RankGraphV1][%s] rankNum[%zu]", __func__, rankTable_.rankList.size());
120 491 : CommProtocol protocol = CommProtocol::COMM_PROTOCOL_RESERVED;
121 491 : CHK_RET(DevTypeToCommProtocol(devType_, protocol));
122 : // 解析 rankTable，建立 rankId -> RankGraphInfo 映射
123 1965 : for (const auto& r : rankTable.rankList) {
124 1474 : RankGraphInfo info;
125 1474 : CHK_RET(BuildRankGraphInfo(r, protocol, info));
126 1474 : rankIndex_[r.rankId] = std::move(info);
127 1474 : }
128 491 : rankGraph_ = rankTable_.rankList;
129 491 : CHK_RET(InitRankInfo());
130 491 : CHK_RET(InitNetLayer());
131 491 : CHK_RET(InitHeterogMode());
132 491 : HCCL_INFO("[RankGraphV1][%s] Init success", __func__);
133 491 : return HCCL_SUCCESS;
134 : }
135 :
136 32 : HcclResult RankGraphV1::Init(const HcclTopoAttr& topoAttr)
137 : {
138 32 : topoAttr_ = topoAttr;
139 32 : rankIndex_.clear();
140 32 : rankPairInfo_.clear();
141 32 : HCCL_INFO("[RankGraphV1][%s] rankNum[%zu]", __func__, rankTable_.rankList.size());
142 32 : CHK_RET(InitRankInfo());
143 32 : CHK_RET(InitNetLayer());
144 32 : CHK_RET(InitHeterogMode());
145 32 : return HCCL_SUCCESS;
146 : }
147 :
148 0 : bool RankGraphV1::IsRoceInSameServer(uint32_t netLayer, const RankInfo_t& srcInfo, const RankInfo_t& dstInfo)
149 : {
150 : // 910B单机两种使能RoCE场景：1.A+X 两卡分别在两个MESH 2.标卡
151 0 : uint32_t srcPhyId = srcInfo.deviceInfo.devicePhyId;
152 0 : uint32_t dstPhyId = dstInfo.deviceInfo.devicePhyId;
153 0 : uint32_t intraRoceSwitch = GetExternalInputIntraRoceSwitch();
154 0 : HCCL_INFO(
155 : "[%s] netLayer[%u], devType[%d], srcPhyId[%u], dstPhyId[%u], isStandardCard[%d], isDiffDeviceModule[%d] "
156 : "IntraRoceSwitch[%u]",
157 : __func__, netLayer, devType_, srcPhyId, dstPhyId, topoAttr_.isStandardCard, topoAttr_.isDiffDeviceModule,
158 : intraRoceSwitch);
159 0 : const uint32_t deviceMeshDivider = DEVICE_PER_MODULE;
160 0 : if (netLayer == HCCL_NETLAYER_1 && devType_ == DevType::DEV_TYPE_910B) {
161 0 : bool isSrcInLowerMesh = srcPhyId < deviceMeshDivider;
162 0 : bool isDstInLowerMesh = dstPhyId < deviceMeshDivider;
163 0 : bool isSrcInUpperMesh = srcPhyId >= deviceMeshDivider;
164 0 : bool isDstInUpperMesh = dstPhyId >= deviceMeshDivider;
165 :
166 : // 判定是否为跨MESH（一卡在低区、一卡在高区，匹配A+X跨MESH场景）
167 0 : bool isCrossMesh = (isSrcInLowerMesh || isDstInLowerMesh) && (isSrcInUpperMesh || isDstInUpperMesh);
168 : // 跨MESH或标卡直接满足
169 0 : bool isMeetRoceCondition = (isCrossMesh && topoAttr_.isDiffDeviceModule) || topoAttr_.isStandardCard;
170 0 : return isMeetRoceCondition && intraRoceSwitch == 1;
171 : }
172 :
173 : // 非910B的NETLAYER_1场景：仅标卡满足条件时取外部配置，否则返回false
174 0 : return topoAttr_.isStandardCard && intraRoceSwitch == 1 && netLayer == HCCL_NETLAYER_1;
175 : }
176 :
177 2 : CommProtocol RankGraphV1::GetCommProtocolInSameServer(const RankInfo_t& srcInfo, const RankInfo_t& dstInfo)
178 : {
179 : // 310P间链路为PCIE或HCCS
180 2 : LinkTypeInServer linkType = LinkTypeInServer::RESERVED_LINK_TYPE;
181 2 : hrtGetPairDeviceLinkType(srcInfo.deviceInfo.devicePhyId, dstInfo.deviceInfo.devicePhyId, linkType);
182 2 : HCCL_INFO(
183 : "[RankGraphV1][%s] ranks[%u,%u] intra-server linkType[%d]", __func__, srcInfo.rankId, dstInfo.rankId, linkType);
184 2 : if (linkType == LinkTypeInServer::HCCS_TYPE || linkType == LinkTypeInServer::HCCS_SW_TYPE) {
185 2 : return CommProtocol::COMM_PROTOCOL_HCCS;
186 0 : } else if (linkType == LinkTypeInServer::SIO_TYPE) {
187 0 : return CommProtocol::COMM_PROTOCOL_SIO;
188 0 : } else if (linkType == LinkTypeInServer::PXI_TYPE) {
189 0 : bool isDiffDeviceModule = (topoAttr_.isDiffDeviceModule && devType_ == DevType::DEV_TYPE_910B);
190 0 : bool isRankModEqual = (srcInfo.rankId % DEVICE_PER_MODULE == dstInfo.rankId % DEVICE_PER_MODULE);
191 0 : bool isMeetPxiCondition = (!isDiffDeviceModule) || (isDiffDeviceModule && isRankModEqual);
192 0 : return isMeetPxiCondition ? CommProtocol::COMM_PROTOCOL_PCIE : CommProtocol::COMM_PROTOCOL_RESERVED;
193 : }
194 0 : return CommProtocol::COMM_PROTOCOL_RESERVED;
195 : }
196 :
197 0 : CommProtocol RankGraphV1::GetCommProtocolBetweenServers(const RankInfo_t& srcInfo, const RankInfo_t& dstInfo) const
198 : {
199 : // srcInfo与dstInfo一定是相同数据类型
200 0 : if (devType_ == DevType::DEV_TYPE_310P3 || devType_ == DevType::DEV_TYPE_310P1) {
201 0 : return CommProtocol::COMM_PROTOCOL_PCIE;
202 : }
203 0 : if (devType_ == DevType::DEV_TYPE_910B) {
204 0 : return CommProtocol::COMM_PROTOCOL_ROCE;
205 : }
206 0 : HCCL_DEBUG(
207 : "[%s] srcInfo.superPodId %s dstInfo.superPodId %s", __func__, srcInfo.superPodId.c_str(),
208 : dstInfo.superPodId.c_str());
209 0 : if (devType_ == DevType::DEV_TYPE_910_93) {
210 : // 超节点内链路为HCCS
211 0 : if (!srcInfo.superPodId.empty() && srcInfo.superPodId == dstInfo.superPodId) {
212 0 : return CommProtocol::COMM_PROTOCOL_HCCS;
213 : }
214 : }
215 0 : return CommProtocol::COMM_PROTOCOL_RESERVED;
216 : }
217 :
218 : CommProtocol
219 2 : RankGraphV1::GetCommProtocolFromRankInfo(const RankInfo_t& srcInfo, const RankInfo_t& dstInfo, uint32_t netLayer)
220 : {
221 2 : if (srcInfo.deviceInfo.deviceType != dstInfo.deviceInfo.deviceType) {
222 0 : HCCL_ERROR(
223 : "[RankGraphV1][%s] srcType[%d] != dstType[%d]", __func__, srcInfo.deviceInfo.deviceType,
224 : dstInfo.deviceInfo.deviceType);
225 0 : return CommProtocol::COMM_PROTOCOL_RESERVED;
226 : }
227 : // 首先判断是否在同一机内
228 2 : if (srcInfo.serverIdx == dstInfo.serverIdx) {
229 2 : if (netLayer == HCCL_NETLAYER_0) {
230 2 : return GetCommProtocolInSameServer(srcInfo, dstInfo);
231 : // 超节点有HCCL_NETLAYER_1及以上的情况，为HCCS链路，或者同卡不同DIE
232 0 : } else if (
233 0 : netLayer == HCCL_NETLAYER_1 && devType_ == DevType::DEV_TYPE_910_93
234 0 : && (srcInfo.superPodId == dstInfo.superPodId
235 0 : || GetCommProtocolInSameServer(srcInfo, dstInfo) == CommProtocol::COMM_PROTOCOL_SIO)) {
236 0 : return CommProtocol::COMM_PROTOCOL_HCCS;
237 0 : } else if (IsRoceInSameServer(netLayer, srcInfo, dstInfo)) {
238 0 : return CommProtocol::COMM_PROTOCOL_ROCE;
239 : } else {
240 : // 接了交换机才会有HCCL_NETLAYER_1及以上的情况，当前无法判断是否连接交换机，接了交换机走RDMA
241 0 : return CommProtocol::COMM_PROTOCOL_RESERVED;
242 : }
243 : }
244 0 : if (srcInfo.serverIdx != dstInfo.serverIdx) {
245 0 : if (netLayer == HCCL_NETLAYER_0) {
246 0 : HCCL_INFO("[RankGraphV1][%s] ranks[%u,%u] not in same server", __func__, srcInfo.rankId, dstInfo.rankId);
247 0 : return CommProtocol::COMM_PROTOCOL_RESERVED;
248 : }
249 0 : if (netLayer == HCCL_NETLAYER_1) {
250 0 : HCCL_INFO(
251 : "[RankGraphV1][%s] ranks[%u,%u] inter-server but same superPod[%s]", __func__, srcInfo.rankId,
252 : dstInfo.rankId, srcInfo.superPodId.c_str());
253 0 : return GetCommProtocolBetweenServers(srcInfo, dstInfo);
254 : // 跨超走ROCE
255 0 : } else if (
256 0 : !srcInfo.superPodId.empty() && srcInfo.superPodId != dstInfo.superPodId && netLayer == HCCL_NETLAYER_2) {
257 0 : HCCL_INFO(
258 : "[RankGraphV1][%s] ranks[%u,%u] inter-superPod use ROCE", __func__, srcInfo.rankId, dstInfo.rankId);
259 0 : return CommProtocol::COMM_PROTOCOL_ROCE;
260 : }
261 : }
262 0 : return CommProtocol::COMM_PROTOCOL_RESERVED;
263 : }
264 :
265 4 : bool RankGraphV1::NeedIgnoreEndPoints(
266 : CommProtocol srcProtocol, CommProtocol dstProtocol, CommProtocol linkProtocol) const
267 : {
268 4 : if (srcProtocol != dstProtocol) {
269 1 : return true;
270 : } else {
271 : // 两个hccs endpoints间可能是SIO链路
272 : // A + X 两个mesh间是PCIE链路, 310DUO卡两个DIE间链路是HCCS，主次DIE间是PCIE
273 3 : if (srcProtocol == COMM_PROTOCOL_HCCS && dstProtocol == COMM_PROTOCOL_HCCS
274 2 : && linkProtocol == COMM_PROTOCOL_SIO) {
275 1 : return false;
276 2 : } else if (dstProtocol != linkProtocol) {
277 0 : return true;
278 : }
279 : }
280 2 : return false;
281 : }
282 :
283 2 : void RankGraphV1::PrintLinksInfo(CommLink& link) const
284 : {
285 : // 打印CommLink 头部基础信息
286 2 : HCCL_INFO(
287 : "[RankGraphV1][%s] link.header.version[%u] magicWord[0x%08x] size[%u] reserved[%u]", __func__,
288 : link.header.version, link.header.magicWord, link.header.size, link.header.reserved);
289 :
290 : // 打印【源端】srcEndpointDesc 完整信息
291 2 : HCCL_INFO(
292 : "[RankGraphV1][%s] srcProtocol[%d] srcCommAddrType[%d] srcLocType[%d] srcDevPhyId[%u] "
293 : "srcSuperDevId[%u] srcServerIdx[%u] srcSuperPodIdx[%u]",
294 : __func__, link.srcEndpointDesc.protocol, link.srcEndpointDesc.commAddr.type, link.srcEndpointDesc.loc.locType,
295 : link.srcEndpointDesc.loc.device.devPhyId, link.srcEndpointDesc.loc.device.superDevId,
296 : link.srcEndpointDesc.loc.device.serverIdx, link.srcEndpointDesc.loc.device.superPodIdx);
297 :
298 : // 打印【目的端】dstEndpointDesc 完整信息
299 2 : HCCL_INFO(
300 : "[RankGraphV1][%s] dstProtocol[%d] dstCommAddrType[%d] dstLocType[%d] dstDevPhyId[%u] "
301 : "dstSuperDevId[%u] dstServerIdx[%u] dstSuperPodIdx[%u]",
302 : __func__, link.dstEndpointDesc.protocol, link.dstEndpointDesc.commAddr.type, link.dstEndpointDesc.loc.locType,
303 : link.dstEndpointDesc.loc.device.devPhyId, link.dstEndpointDesc.loc.device.superDevId,
304 : link.dstEndpointDesc.loc.device.serverIdx, link.dstEndpointDesc.loc.device.superPodIdx);
305 :
306 : // 打印【链路属性】linkAttr 信息
307 2 : HCCL_INFO("[RankGraphV1][%s] linkProtocol[%d] hop[%u]", __func__, link.linkAttr.linkProtocol, link.linkAttr.hop);
308 2 : }
309 :
310 : HcclResult
311 3 : RankGraphV1::GetLinks(uint32_t netLayer, uint32_t srcRank, uint32_t dstRank, CommLink** linkList, uint32_t* listSize)
312 : {
313 5 : if (rankIndex_.find(srcRank) == rankIndex_.end() || rankIndex_.find(dstRank) == rankIndex_.end()
314 5 : || FindRank(srcRank) == nullptr || FindRank(dstRank) == nullptr) {
315 1 : HCCL_ERROR(
316 : "[RankGraphV1][%s] srcRank[%u] or dstRank[%u] is not existed in rankTable", __func__, srcRank, dstRank);
317 1 : return HCCL_E_PARA;
318 : }
319 :
320 2 : if (netLayer > HCCL_NETLAYER_2) {
321 1 : HCCL_ERROR(
322 : "[RankGraphV1][%s] srcRank[%u] and dstRank[%u] do not have netLayer[%u]", __func__, srcRank, dstRank,
323 : netLayer);
324 1 : return HCCL_E_PARA;
325 : }
326 1 : auto& srcEndpointDescs = rankIndex_[srcRank].endPoints;
327 1 : auto& dstEndpointDescs = rankIndex_[dstRank].endPoints;
328 :
329 1 : const RankInfo_t& srcInfo = rankIndex_[srcRank].rankInfo;
330 1 : const RankInfo_t& dstInfo = rankIndex_[dstRank].rankInfo;
331 1 : CommProtocol protocol = COMM_PROTOCOL_RESERVED;
332 1 : protocol = GetCommProtocolFromRankInfo(srcInfo, dstInfo, netLayer);
333 1 : if (protocol == COMM_PROTOCOL_RESERVED) {
334 0 : HCCL_WARNING("[RankGraphV1][%s] no links between srcRank[%u] dstRank[%u]", __func__, srcRank, dstRank);
335 0 : *linkList = nullptr;
336 0 : *listSize = 0;
337 0 : return HCCL_SUCCESS;
338 : }
339 :
340 : // 1. 查询是否有缓存CommLink信息
341 1 : auto key = std::make_tuple(netLayer, srcRank, dstRank);
342 1 : auto it = rankPairInfo_.find(key);
343 1 : if (it == rankPairInfo_.end()) {
344 : // 没有则创建
345 1 : HCCL_INFO("[RankGraphV1][%s] no cached links, build new srcRank[%u] dstRank[%u]", __func__, srcRank, dstRank);
346 1 : std::vector<CommLink> links;
347 2 : for (size_t i = 0; i < srcEndpointDescs.size(); i++) {
348 2 : for (size_t j = 0; j < dstEndpointDescs.size(); j++) {
349 1 : if (NeedIgnoreEndPoints(srcEndpointDescs[i].protocol, dstEndpointDescs[j].protocol, protocol)) {
350 0 : continue;
351 : }
352 : CommLink link;
353 1 : CHK_RET(CommLinkInit(&link, 1));
354 :
355 1 : link.srcEndpointDesc = srcEndpointDescs[i];
356 1 : link.srcEndpointDesc.protocol = protocol;
357 1 : link.dstEndpointDesc = dstEndpointDescs[j];
358 1 : link.dstEndpointDesc.protocol = protocol;
359 1 : link.linkAttr.linkProtocol = protocol;
360 1 : PrintLinksInfo(link);
361 1 : links.push_back(std::move(link));
362 : }
363 : }
364 1 : it = rankPairInfo_.emplace(std::make_tuple(netLayer, srcRank, dstRank), std::move(links)).first;
365 1 : }
366 1 : HCCL_INFO(
367 : "[RankGraphV1][%s] links, netLayer[%u] srcRank[%u] dstRank[%u] protocol[%d]", __func__, netLayer, srcRank,
368 : dstRank, protocol);
369 :
370 1 : auto& links = it->second;
371 1 : *listSize = static_cast<uint32_t>(links.size());
372 1 : if (links.empty()) {
373 0 : *linkList = nullptr;
374 0 : HCCL_ERROR("[RankGraphV1][%s] links empty for srcRank[%u] dstRank[%u]", __func__, srcRank, dstRank);
375 : } else {
376 1 : *linkList = links.data(); // 连续数组首地址
377 1 : HCCL_INFO(
378 : "[RankGraphV1][%s] srcRank[%u] dstRank[%u] linkList[%p] linkNum[%u]", __func__, srcRank, dstRank, *linkList,
379 : *listSize);
380 : }
381 :
382 1 : return HCCL_SUCCESS;
383 : }
384 :
385 523 : HcclResult RankGraphV1::InitHeterogMode()
386 : {
387 523 : if (topoAttr_.rankInfoList.empty()) {
388 0 : HCCL_ERROR("[RankGraphV1][%s] invalid para. rankInfoList is empty", __func__);
389 0 : return HCCL_E_INTERNAL;
390 : }
391 :
392 523 : std::set<DevType> devTypes;
393 2151 : for (u32 index = 0; index < topoAttr_.rankInfoList.size(); index++) {
394 1628 : devTypes.insert(topoAttr_.rankInfoList[index].deviceType);
395 : }
396 :
397 : // 只包含一种芯片的同构组网
398 523 : if (devTypes.size() == 1) {
399 523 : heterogMode_ = HcclHeterogMode::HCCL_HETEROG_MODE_HOMOGENEOUS;
400 523 : return HCCL_SUCCESS;
401 : }
402 :
403 : // 包含两种芯片的异构混合组网
404 0 : constexpr uint32_t MIX_CHIPS = 2;
405 0 : if (devTypes.size() == MIX_CHIPS && devTypes.find(DevType::DEV_TYPE_910B) != devTypes.end()
406 0 : && devTypes.find(DevType::DEV_TYPE_910_93) != devTypes.end()) {
407 0 : heterogMode_ = HcclHeterogMode::HCCL_HETEROG_MODE_MIX_A2_A3;
408 0 : return HCCL_SUCCESS;
409 : }
410 :
411 0 : std::string devStr;
412 0 : for (auto itSet = devTypes.begin(); itSet != devTypes.end(); itSet++) {
413 0 : if (itSet != devTypes.begin()) {
414 0 : devStr += ", ";
415 : }
416 0 : devStr += std::to_string(static_cast<int>(*itSet));
417 : }
418 0 : HCCL_ERROR(
419 : "[RankGraphV1][%s] Unknown mode[%d], devtypes[%s]", __func__, HcclHeterogMode::HCCL_HETEROG_MODE_INVALID,
420 : devStr.c_str());
421 0 : return HCCL_E_INTERNAL;
422 523 : }
423 :
424 1 : HcclResult RankGraphV1::GetHeterogMode(HcclHeterogMode* mode) const
425 : {
426 1 : *mode = heterogMode_;
427 1 : return HCCL_SUCCESS;
428 : }
429 :
430 2 : HcclResult RankGraphV1::GetNetLayers(uint32_t** netLayers, uint32_t* netLayerNum)
431 : {
432 2 : if (netLayer_.empty()) {
433 1 : HCCL_ERROR("[RankGraphV1][%s] invalid para. netLayer is empty", __func__);
434 1 : return HCCL_E_INTERNAL;
435 : }
436 1 : *netLayers = netLayer_.data();
437 1 : *netLayerNum = netLayer_.size();
438 1 : return HCCL_SUCCESS;
439 : }
440 :
441 528 : HcclResult RankGraphV1::GetInstTopoTypeByNetLayer(uint32_t netLayer, CommTopo* topoType)
442 : {
443 528 : if (netLayer >= netLayer_.size()) {
444 1 : HCCL_ERROR("[RankGraphV1][%s] invalid para. netlayer[%u]", __func__, netLayer);
445 1 : return HCCL_E_PARA;
446 : }
447 527 : DevType deviceType = topoAttr_.deviceType;
448 527 : if (deviceType == DevType::DEV_TYPE_910_93) {
449 34 : if (netLayer == static_cast<uint32_t>(HcclNetLayerlevel::HCCL_NetLayer_L0)) {
450 34 : *topoType = CommTopo::COMM_TOPO_910_93;
451 0 : } else if ((netLayer == static_cast<uint32_t>(HcclNetLayerlevel::HCCL_NetLayer_L1)
452 0 : || (netLayer == static_cast<uint32_t>(HcclNetLayerlevel::HCCL_NetLayer_L2)))) {
453 0 : *topoType = CommTopo::COMM_TOPO_CLOS;
454 : }
455 493 : } else if (deviceType == DevType::DEV_TYPE_910B || deviceType == DevType::DEV_TYPE_910) {
456 488 : if (netLayer == static_cast<uint32_t>(HcclNetLayerlevel::HCCL_NetLayer_L0)) {
457 487 : *topoType = CommTopo::COMM_TOPO_1DMESH;
458 1 : } else if (netLayer == static_cast<uint32_t>(HcclNetLayerlevel::HCCL_NetLayer_L1)) {
459 1 : *topoType = CommTopo::COMM_TOPO_CLOS;
460 : }
461 5 : } else if (deviceType == DevType::DEV_TYPE_310P3) {
462 5 : if (netLayer == static_cast<uint32_t>(HcclNetLayerlevel::HCCL_NetLayer_L0)) {
463 5 : *topoType = CommTopo::COMM_TOPO_310P;
464 : }
465 : }
466 527 : return HCCL_SUCCESS;
467 : }
468 :
469 2 : HcclResult RankGraphV1::GetInstSizeByNetLayer(uint32_t netLayer, uint32_t* rankNum)
470 : {
471 2 : if (netLayer >= netLayer_.size()) {
472 1 : HCCL_ERROR("[RankGraphV1][%s] invalid para. netlayer[%u]", __func__, netLayer);
473 1 : return HCCL_E_PARA;
474 : }
475 :
476 1 : if (rankList_.find(netLayer) == rankList_.end()) {
477 1 : HCCL_ERROR("[RankGraphV1][%s] failed to find rankList map. netlayer[%u]", __func__, netLayer);
478 1 : return HCCL_E_INTERNAL;
479 : }
480 0 : *rankNum = rankList_[netLayer].size();
481 :
482 0 : return HCCL_SUCCESS;
483 : }
484 :
485 2 : HcclResult RankGraphV1::GetInstRanksByNetLayer(uint32_t netLayer, uint32_t** rankList, uint32_t* rankNum)
486 : {
487 2 : if (netLayer >= netLayer_.size()) {
488 1 : HCCL_ERROR("[RankGraphV1][%s] invalid para. netlayer[%u]", __func__, netLayer);
489 1 : return HCCL_E_PARA;
490 : }
491 :
492 1 : if (rankList_.find(netLayer) == rankList_.end()) {
493 1 : HCCL_ERROR("[RankGraphV1][%s] failed to find rankList map. netlayer[%u]", __func__, netLayer);
494 1 : return HCCL_E_INTERNAL;
495 : }
496 0 : *rankNum = rankList_[netLayer].size();
497 0 : *rankList = rankList_[netLayer].data();
498 :
499 0 : return HCCL_SUCCESS;
500 : }
501 :
502 2 : HcclResult RankGraphV1::GetInstSizeListByNetLayer(uint32_t netLayer, uint32_t** instSizeList, uint32_t* listSize)
503 : {
504 2 : if (netLayer >= netLayer_.size()) {
505 1 : HCCL_ERROR("[RankGraphV1][%s] invalid para. netlayer[%u]", __func__, netLayer);
506 1 : return HCCL_E_PARA;
507 : }
508 :
509 1 : if (rankSizeList_.find(netLayer) == rankSizeList_.end()) {
510 1 : HCCL_ERROR("[RankGraphV1][%s] failed to find rankSizeList map. netlayer[%u]", __func__, netLayer);
511 1 : return HCCL_E_INTERNAL;
512 : }
513 0 : *instSizeList = rankSizeList_[netLayer].data();
514 0 : *listSize = rankSizeList_[netLayer].size();
515 :
516 0 : return HCCL_SUCCESS;
517 : }
518 :
519 1 : HcclResult RankGraphV1::GetTopoInstsByLayer(uint32_t netLayer, uint32_t** topoInsts, uint32_t* topoInstNum)
520 : {
521 1 : if (netLayer >= netLayer_.size()) {
522 0 : return HCCL_E_PARA;
523 : }
524 1 : if (rankSizeList_.find(netLayer) == rankSizeList_.end()) {
525 0 : return HCCL_E_INTERNAL;
526 : }
527 :
528 1 : uint32_t instNum = rankSizeList_[netLayer].size();
529 1 : *topoInstNum = instNum;
530 :
531 1 : topoInstsVec_.clear();
532 1 : topoInstsVec_.resize(instNum);
533 5 : for (uint32_t i = 0; i < instNum; ++i) {
534 4 : topoInstsVec_[i] = i;
535 : }
536 :
537 1 : *topoInsts = topoInstsVec_.data();
538 :
539 1 : return HCCL_SUCCESS;
540 : }
541 :
542 1 : HcclResult RankGraphV1::GetTopoType(uint32_t netLayer, CommTopo* topoType)
543 : {
544 1 : if (netLayer >= netLayer_.size()) {
545 0 : return HCCL_E_PARA;
546 : }
547 1 : return GetInstTopoTypeByNetLayer(netLayer, topoType);
548 : }
549 :
550 1 : HcclResult RankGraphV1::GetRanksByTopoInst(uint32_t netLayer, uint32_t topoInstId, uint32_t** ranks, uint32_t* rankNum)
551 : {
552 1 : if (netLayer >= netLayer_.size()) {
553 0 : return HCCL_E_PARA;
554 : }
555 1 : if (devType_ != DevType::DEV_TYPE_910B) {
556 0 : return HCCL_E_NOT_SUPPORT;
557 : }
558 1 : auto rankListIt = rankList_.find(netLayer);
559 1 : auto rankSizeListIt = rankSizeList_.find(netLayer);
560 1 : if (rankListIt != rankList_.end() && rankSizeListIt != rankSizeList_.end()) {
561 1 : if (topoInstId >= rankSizeListIt->second.size()) {
562 0 : return HCCL_E_PARA;
563 : }
564 : }
565 :
566 1 : *ranks = rankListIt->second.data();
567 1 : if (topoInstId < rankSizeListIt->second.size()) {
568 1 : *rankNum = rankSizeListIt->second[topoInstId];
569 : } else {
570 0 : *rankNum = 0;
571 : }
572 :
573 1 : return HCCL_SUCCESS;
574 : }
575 :
576 3 : std::vector<const RankInfo_t*> RankGraphV1::GetRanksInTopoInst(uint32_t netLayer, uint32_t topoInstId)
577 : {
578 3 : std::vector<const RankInfo_t*> ranks;
579 :
580 3 : if (netLayer >= netLayer_.size()) {
581 0 : return ranks;
582 : }
583 3 : if (netLayer == static_cast<uint32_t>(HcclNetLayerlevel::HCCL_NetLayer_L0)) {
584 2 : auto it = serverToRank_.find(topoInstId);
585 2 : if (it == serverToRank_.end()) {
586 2 : return ranks;
587 : }
588 0 : for (const auto& rankInfo : it->second) {
589 0 : for (const auto& graphInfo : rankGraph_) {
590 0 : if (graphInfo.rankId == rankInfo.userRank) {
591 0 : ranks.push_back(&graphInfo);
592 : }
593 : }
594 : }
595 : } else {
596 1 : auto listIt = rankList_.find(netLayer);
597 1 : auto sizeListIt = rankSizeList_.find(netLayer);
598 1 : if (listIt != rankList_.end() && sizeListIt != rankSizeList_.end()) {
599 1 : if (topoInstId >= sizeListIt->second.size()) {
600 0 : return ranks;
601 : }
602 : }
603 :
604 3 : for (uint32_t userRank : listIt->second) {
605 3 : for (const auto& rankInfo : rankGraph_) {
606 3 : if (rankInfo.rankId == userRank) {
607 2 : ranks.push_back(&rankInfo);
608 2 : break;
609 : }
610 : }
611 : }
612 : }
613 1 : return ranks;
614 0 : }
615 :
616 : std::set<CommProtocol>
617 1 : RankGraphV1::GetProtocolsByConnections(uint32_t netLayer, const std::vector<const RankInfo_t*>& topoInstRanks)
618 : {
619 1 : std::set<CommProtocol> protocols;
620 :
621 1 : const RankInfo_t* srcRankInfo = nullptr;
622 1 : for (const auto& rankInfo : rankGraph_) {
623 1 : if (rankInfo.rankId == rankData_.userRank) {
624 1 : srcRankInfo = &rankInfo;
625 1 : break;
626 : }
627 : }
628 1 : if (srcRankInfo == nullptr) {
629 0 : return protocols;
630 : }
631 :
632 2 : for (const RankInfo_t* dstRankInfo : topoInstRanks) {
633 1 : if (dstRankInfo->rankId == rankData_.userRank) {
634 0 : continue;
635 : }
636 :
637 1 : CommProtocol protocol = GetCommProtocolFromRankInfo(*srcRankInfo, *dstRankInfo, netLayer);
638 1 : if (protocol != COMM_PROTOCOL_RESERVED) {
639 1 : protocols.insert(protocol);
640 : }
641 : }
642 :
643 1 : return protocols;
644 0 : }
645 :
646 3 : HcclResult RankGraphV1::GetEndpointNum(uint32_t netLayer, uint32_t topoInstId, uint32_t* num)
647 : {
648 3 : if (netLayer >= netLayer_.size()) {
649 1 : HCCL_ERROR("[RankGraphV1][%s] invalid para. netlayer[%u]", __func__, netLayer);
650 1 : return HCCL_E_PARA;
651 : }
652 2 : if (rankIndex_.empty()) {
653 1 : HCCL_ERROR("[RankGraphV1][%s] rankIndex is empty", __func__);
654 1 : return HCCL_E_INTERNAL;
655 : }
656 1 : std::vector<const RankInfo_t*> topoInstRanks = GetRanksInTopoInst(netLayer, topoInstId);
657 1 : if (topoInstRanks.empty()) {
658 1 : *num = 0;
659 1 : return HCCL_SUCCESS;
660 : }
661 :
662 0 : std::set<CommProtocol> protocols = GetProtocolsByConnections(netLayer, topoInstRanks);
663 0 : if (protocols.empty()) {
664 0 : HCCL_INFO(
665 : "[RankGraphV1][%s] no protocols found for netlayer [%u] topoInstId[%u]", __func__, netLayer, topoInstId);
666 : }
667 :
668 0 : const RankGraphInfo* currentRankInfo = nullptr;
669 0 : for (auto& pair : rankIndex_) {
670 0 : if (pair.second.rankInfo.rankId == rankData_.userRank) {
671 0 : currentRankInfo = &pair.second;
672 0 : break;
673 : }
674 : }
675 0 : if (currentRankInfo == nullptr) {
676 0 : return HCCL_E_INTERNAL;
677 : }
678 :
679 0 : uint32_t count = 0;
680 0 : for (const auto& endpoint : currentRankInfo->endPoints) {
681 0 : if (protocols.find(endpoint.protocol) != protocols.end()) {
682 0 : count++;
683 : }
684 : }
685 0 : *num = count;
686 :
687 0 : return HCCL_SUCCESS;
688 1 : }
689 :
690 : HcclResult
691 5 : RankGraphV1::GetEndpointDesc(uint32_t netLayer, uint32_t topoInstId, uint32_t* descNum, EndpointDesc* endpointDesc)
692 : {
693 5 : if (netLayer >= netLayer_.size()) {
694 1 : HCCL_ERROR("[RankGraphV1][%s] invalid para. netlayer[%u]", __func__, netLayer);
695 1 : return HCCL_E_PARA;
696 : }
697 4 : if (descNum == nullptr || endpointDesc == nullptr) {
698 2 : HCCL_ERROR("[RankGraphV1][%s] invalid para. null ptr", __func__);
699 2 : return HCCL_E_PARA;
700 : }
701 2 : if (rankIndex_.empty()) {
702 1 : HCCL_ERROR("[RankGraphV1][%s] rankIndex is empty", __func__);
703 1 : return HCCL_E_INTERNAL;
704 : }
705 1 : std::vector<const RankInfo_t*> topoInstRanks = GetRanksInTopoInst(netLayer, topoInstId);
706 1 : if (topoInstRanks.empty()) {
707 1 : *descNum = 0;
708 1 : return HCCL_SUCCESS;
709 : }
710 :
711 0 : std::set<CommProtocol> protocols = GetProtocolsByConnections(netLayer, topoInstRanks);
712 0 : if (protocols.empty()) {
713 0 : HCCL_INFO(
714 : "[RankGraphV1][GetEndpointDesc] no protocols found, netlayer [%u] topoInstId[%u]", netLayer, topoInstId);
715 : }
716 :
717 0 : const RankGraphInfo* currRankInfo = nullptr;
718 0 : for (auto& pair : rankIndex_) {
719 0 : if (pair.second.rankInfo.rankId == rankData_.userRank) {
720 0 : currRankInfo = &pair.second;
721 0 : break;
722 : }
723 : }
724 0 : if (currRankInfo == nullptr) {
725 0 : return HCCL_E_INTERNAL;
726 : }
727 :
728 0 : uint32_t count = 0;
729 0 : for (const auto& endpoint : currRankInfo->endPoints) {
730 0 : if (protocols.find(endpoint.protocol) != protocols.end()) {
731 0 : if (count >= *descNum) {
732 0 : return HCCL_E_PARA;
733 : }
734 0 : endpointDesc[count] = endpoint;
735 0 : count++;
736 : }
737 : }
738 0 : *descNum = count;
739 :
740 0 : return HCCL_SUCCESS;
741 1 : }
742 :
743 : const EndpointDesc*
744 5 : RankGraphV1::MatchEndpointByAddr(const RankGraphInfo& rankGraphInfo, const EndpointDesc* endPointDesc) const
745 : {
746 7 : for (const auto& endpoint : rankGraphInfo.endPoints) {
747 5 : if (endpoint.commAddr.type != endPointDesc->commAddr.type || endpoint.protocol != endPointDesc->protocol) {
748 0 : continue;
749 : }
750 5 : bool matched = false;
751 5 : switch (endpoint.commAddr.type) {
752 2 : case COMM_ADDR_TYPE_IP_V4:
753 : matched
754 2 : = (memcmp(&endpoint.commAddr.addr, &endPointDesc->commAddr.addr, sizeof(endpoint.commAddr.addr))
755 : == 0);
756 2 : break;
757 1 : case COMM_ADDR_TYPE_IP_V6:
758 : matched
759 1 : = (memcmp(&endpoint.commAddr.addr6, &endPointDesc->commAddr.addr6, sizeof(endpoint.commAddr.addr6))
760 : == 0);
761 1 : break;
762 1 : case COMM_ADDR_TYPE_ID:
763 1 : matched = true;
764 1 : break;
765 1 : default:
766 1 : break;
767 : }
768 5 : if (matched) {
769 3 : return &endpoint;
770 : }
771 : }
772 2 : return nullptr;
773 : }
774 :
775 : HcclResult
776 7 : RankGraphV1::FillAttr(EndpointAttr endpointAttr, const EndpointDesc* foundEndpoint, uint32_t infoLen, void* info) const
777 : {
778 7 : switch (endpointAttr) {
779 2 : case ENDPOINT_ATTR_BW_COEFF: {
780 2 : CHK_PRT_RET(
781 : infoLen != sizeof(EndpointAttrBwCoeff),
782 : HCCL_ERROR(
783 : "[RankGraphV1::GetEndpointInfo] Size mismatch for ENDPOINT_ATTR_BW_COEFF: expected %zu, actual %u",
784 : sizeof(EndpointAttrBwCoeff), infoLen),
785 : HCCL_E_PARA);
786 1 : *(static_cast<EndpointAttrBwCoeff*>(info)) = 1;
787 1 : break;
788 : }
789 2 : case ENDPOINT_ATTR_DIE_ID: {
790 2 : CHK_PRT_RET(
791 : infoLen != sizeof(EndpointAttrDieId),
792 : HCCL_ERROR(
793 : "[RankGraphV1::GetEndpointInfo] Size mismatch for ENDPOINT_ATTR_DIE_ID: expected %zu, actual %u",
794 : sizeof(EndpointAttrDieId), infoLen),
795 : HCCL_E_PARA);
796 1 : *(static_cast<EndpointAttrDieId*>(info)) = foundEndpoint->loc.device.superDevId;
797 1 : break;
798 : }
799 2 : case ENDPOINT_ATTR_LOCATION: {
800 2 : CHK_PRT_RET(
801 : infoLen != sizeof(EndpointAttrLocation),
802 : HCCL_ERROR(
803 : "[RankGraphV1::GetEndpointInfo] Size mismatch for ENDPOINT_ATTR_LOCATION: expected %zu, actual %u",
804 : sizeof(EndpointAttrLocation), infoLen),
805 : HCCL_E_PARA);
806 1 : *(static_cast<EndpointAttrLocation*>(info)) = foundEndpoint->loc.locType;
807 1 : break;
808 : }
809 1 : default: {
810 1 : HCCL_ERROR("[RankGraphV1::GetEndpointInfo] Invalid endpointAttr[%d]", endpointAttr);
811 1 : return HCCL_E_PARA;
812 : }
813 : }
814 3 : return HCCL_SUCCESS;
815 : }
816 :
817 5 : HcclResult RankGraphV1::GetEndpointInfo(
818 : uint32_t rankId, const EndpointDesc* endPointDesc, EndpointAttr endpointAttr, uint32_t infoLen, void* info)
819 : {
820 5 : if (endPointDesc == nullptr || info == nullptr) {
821 2 : HCCL_ERROR("[RankGraphV1::GetEndpointInfo] Invalid parameter, null pointer");
822 2 : return HCCL_E_PTR;
823 : }
824 :
825 3 : if (rankIndex_.empty()) {
826 1 : HCCL_ERROR("[RankGraphV1::GetEndpointInfo] rankIndex is empty");
827 1 : return HCCL_E_INTERNAL;
828 : }
829 :
830 2 : auto rankIt = rankIndex_.find(rankId);
831 2 : if (rankIt == rankIndex_.end()) {
832 1 : HCCL_ERROR("[RankGraphV1::GetEndpointInfo] rankId[%u] not found in rankIndex", rankId);
833 1 : return HCCL_E_NOT_FOUND;
834 : }
835 :
836 1 : const RankGraphInfo& rankGraphInfo = rankIt->second;
837 1 : const EndpointDesc* foundEndpoint = MatchEndpointByAddr(rankGraphInfo, endPointDesc);
838 1 : if (foundEndpoint == nullptr) {
839 0 : HCCL_ERROR("[RankGraphV1::GetEndpointInfo] No matching endpoint found for rankId[%u]", rankId);
840 0 : return HCCL_E_NOT_FOUND;
841 : }
842 :
843 1 : return FillAttr(endpointAttr, foundEndpoint, infoLen, info);
844 : }
845 :
846 2 : HcclResult RankGraphV1::GetRankSize(uint32_t* rankSize)
847 : {
848 2 : CHK_PTR_NULL(rankSize);
849 1 : *rankSize = rankGraph_.size();
850 1 : return HCCL_SUCCESS;
851 : }
852 :
853 3 : HcclResult RankGraphV1::GetDevicePort(const uint32_t rank, uint32_t* devPort)
854 : {
855 3 : CHK_PTR_NULL(devPort);
856 2 : const RankInfo_t* rankInfo = FindRank(rank);
857 2 : if (rankInfo == nullptr) {
858 1 : HCCL_ERROR("[RankGraphV1][%s] rank[%u] not found", __func__, rank);
859 1 : return HCCL_E_PARA;
860 : }
861 1 : *devPort = rankInfo->deviceInfo.port;
862 1 : return HCCL_SUCCESS;
863 : }
864 :
865 3908 : bool RankGraphSort(const RankInfo& first, const RankInfo& second)
866 : {
867 3908 : if (first.serverIdx != second.serverIdx) {
868 453 : return first.serverIdx < second.serverIdx;
869 : } else {
870 3455 : return first.userRank < second.userRank;
871 : }
872 : }
873 :
874 523 : HcclResult RankGraphV1::InitGraphRankInfo()
875 : {
876 1997 : for (u32 index = 0; index < rankGraph_.size(); index++) {
877 1474 : struct GraphRankInfo graphRankInfo = {};
878 1474 : graphRankInfo.rankId = rankGraph_[index].rankId;
879 1474 : graphRankInfo.localRank = rankGraph_[index].localRank;
880 1474 : graphRankInfo.serverId = rankGraph_[index].serverId;
881 1474 : graphRankInfo.serverIdx = rankGraph_[index].serverIdx;
882 1474 : graphRankInfo.superDeviceId = rankGraph_[index].superDeviceId;
883 1474 : graphRankInfo.superPodId = rankGraph_[index].superPodId;
884 1474 : graphRankInfo.superPodIdx = rankGraph_[index].superPodIdx;
885 1474 : graphRankInfo.hostPort = rankGraph_[index].hostPort;
886 1474 : graphRankInfo.nodeId = rankGraph_[index].nodeId;
887 1474 : graphRankInfo.itemId = rankGraph_[index].itemId;
888 1474 : graphRankInfo.deviceInfo.devicePhyId = rankGraph_[index].deviceInfo.devicePhyId;
889 1474 : graphRankInfo.deviceInfo.deviceType = rankGraph_[index].deviceInfo.deviceType;
890 1474 : graphRankInfo.deviceInfo.port = rankGraph_[index].deviceInfo.port;
891 1474 : graphRankInfo.deviceInfo.vnicPort = rankGraph_[index].deviceInfo.vnicPort;
892 1474 : graphRankInfo.deviceInfo.backupPort = rankGraph_[index].deviceInfo.backupPort;
893 1474 : graphRankInfo.bindDeviceId = rankGraph_[index].bindDeviceId;
894 1474 : graphRankInfo.originalSuperPodId = rankGraph_[index].originalSuperPodId;
895 :
896 1474 : graphRankInfo_.push_back(graphRankInfo);
897 1474 : }
898 :
899 523 : return HCCL_SUCCESS;
900 : }
901 :
902 1 : HcclResult RankGraphV1::GetRankGraphInfo(GraphType type, void** graph, uint32_t* len)
903 : {
904 1 : switch (type) {
905 0 : case RANK_GRAPH_910_93: {
906 0 : *graph = graphRankInfo_.data();
907 0 : *len = graphRankInfo_.size() * sizeof(GraphRankInfo);
908 0 : break;
909 : }
910 1 : default: {
911 1 : HCCL_ERROR("[RankGraphV1][%s]Graph type[%d] is invalid", __func__, type);
912 1 : return HCCL_E_NOT_SUPPORT;
913 : }
914 : }
915 0 : return HCCL_SUCCESS;
916 : }
917 :
918 0 : HcclResult RankGraphV1::GetDeviceId([[maybe_unused]] uint32_t rankId, [[maybe_unused]] uint32_t* deviceId)
919 : {
920 0 : return HCCL_SUCCESS;
921 : }
922 :
923 523 : HcclResult RankGraphV1::InitRankInfo()
924 : {
925 523 : auto& rankInfoList = topoAttr_.rankInfoList;
926 831 : for (u32 index = 0; index < rankInfoList.size(); index++) {
927 831 : if (topoAttr_.userRank == rankInfoList[index].userRank) {
928 523 : rankData_ = rankInfoList[index];
929 523 : break;
930 : }
931 : }
932 523 : CHK_RET(InitServerRankInfo());
933 523 : CHK_RET(InitSuperPodRankInfo());
934 523 : CHK_RET(InitGraphRankInfo());
935 523 : return HCCL_SUCCESS;
936 : }
937 :
938 523 : HcclResult RankGraphV1::InitServerRankInfo()
939 : {
940 523 : u32 serverIdx = 0;
941 523 : auto& rankInfoList = topoAttr_.rankInfoList;
942 2151 : for (u32 index = 0; index < rankInfoList.size(); index++) {
943 1628 : serverIdx = rankInfoList[index].serverIdx;
944 1628 : auto itServer = serverToRank_.find(serverIdx);
945 1628 : if (itServer != serverToRank_.end()) {
946 885 : itServer->second.push_back(rankInfoList[index]);
947 : } else {
948 743 : std::vector<RankInfo> rankVecTmp;
949 743 : rankVecTmp.push_back(rankInfoList[index]);
950 743 : serverToRank_.insert(std::make_pair(serverIdx, rankVecTmp));
951 743 : }
952 : }
953 : // 调整每个server内的user_rank排序(server内userRank从小到大,一定连续)
954 1266 : for (auto iterMap = serverToRank_.begin(); iterMap != serverToRank_.end(); iterMap++) {
955 743 : if (!(iterMap->second).empty()) {
956 743 : std::sort(iterMap->second.begin(), iterMap->second.end(), RankGraphSort);
957 : }
958 : }
959 523 : serverIdx = rankData_.serverIdx;
960 523 : auto rankVec = serverToRank_.find(serverIdx);
961 523 : if (rankVec != serverToRank_.end()) {
962 523 : std::string rankIdListServer;
963 1846 : for (auto iter : serverToRank_[serverIdx]) {
964 1323 : rankIdListServer += std::to_string(iter.userRank) + " ";
965 1323 : }
966 523 : HCCL_INFO(
967 : "[RankGraphV1][%s] devtype[%d], curRank[%u], serverToRanklist[%s]", __func__, topoAttr_.deviceType,
968 : rankData_.userRank, rankIdListServer.c_str());
969 523 : }
970 523 : return HCCL_SUCCESS;
971 : }
972 :
973 523 : HcclResult RankGraphV1::InitSuperPodRankInfo()
974 : {
975 523 : auto& rankInfoList = topoAttr_.rankInfoList;
976 2151 : for (u32 index = 0; index < rankInfoList.size(); index++) {
977 : // 填充superPodRankMap_, 记录superPodId -> rankInfo
978 1628 : HCCL_DEBUG(
979 : "[RankGraphV1][%s] superPodIdx[%u],superPodId[%s]", __func__, rankInfoList[index].superPodIdx,
980 : rankInfoList[index].superPodId.c_str());
981 1628 : auto itSuperPod = superPodToRank_.find(rankInfoList[index].superPodIdx);
982 1628 : if (itSuperPod != superPodToRank_.end()) {
983 1064 : itSuperPod->second.push_back(rankInfoList[index]);
984 : } else {
985 564 : std::vector<RankInfo> rankVecTmp;
986 564 : rankVecTmp.push_back(rankInfoList[index]);
987 564 : superPodToRank_.insert(std::make_pair(rankInfoList[index].superPodIdx, rankVecTmp));
988 564 : }
989 : }
990 :
991 : // 调整每个superPod内的user_rank排序, 按照serverIdx从小到大、userRank从小到大排序
992 1087 : for (auto iterMap = superPodToRank_.begin(); iterMap != superPodToRank_.end(); iterMap++) {
993 564 : if (!(iterMap->second).empty()) {
994 564 : std::sort(iterMap->second.begin(), iterMap->second.end(), RankGraphSort);
995 : }
996 : }
997 :
998 523 : if (superPodToRank_.find(rankData_.superPodIdx) != superPodToRank_.end()) {
999 523 : std::string rankIdListPod;
1000 2110 : for (auto iter : superPodToRank_[rankData_.superPodIdx]) {
1001 1587 : rankIdListPod += std::to_string(iter.userRank) + " ";
1002 1587 : }
1003 523 : HCCL_INFO(
1004 : "[RankGraphV1][%s] curRank[%u], curSuperPod[%s] superPodToRanklist[%s]", __func__, rankData_.userRank,
1005 : rankData_.superPodId.c_str(), rankIdListPod.c_str());
1006 523 : }
1007 523 : return HCCL_SUCCESS;
1008 : }
1009 :
1010 523 : HcclResult RankGraphV1::InitNetLayer()
1011 : {
1012 523 : netLayer_.clear();
1013 523 : netLayer_.push_back(static_cast<uint32_t>(HcclNetLayerlevel::HCCL_NetLayer_L0));
1014 :
1015 523 : u32 serverIdx = rankData_.serverIdx;
1016 523 : auto rankVec = serverToRank_.find(serverIdx);
1017 523 : if (rankVec == serverToRank_.end()) {
1018 0 : HCCL_ERROR("[RankGraphV1][%s] find serverToRank failed, serverIdx[%u]", __func__, serverIdx);
1019 0 : return HCCL_E_INTERNAL;
1020 : }
1021 523 : std::vector<u32> rankListTmp;
1022 1846 : for (auto iter : serverToRank_[serverIdx]) {
1023 1323 : rankListTmp.push_back(iter.userRank);
1024 1323 : }
1025 523 : rankList_.insert({static_cast<uint32_t>(HcclNetLayerlevel::HCCL_NetLayer_L0), rankListTmp});
1026 :
1027 523 : std::vector<u32> rankSizeListTmp;
1028 1266 : for (auto iter : serverToRank_) {
1029 743 : rankSizeListTmp.push_back(iter.second.size());
1030 743 : }
1031 523 : rankSizeList_.insert({static_cast<uint32_t>(HcclNetLayerlevel::HCCL_NetLayer_L0), rankSizeListTmp});
1032 :
1033 523 : DevType deviceType = topoAttr_.deviceType;
1034 523 : if (serverToRank_.size() > 1) {
1035 102 : netLayer_.push_back(static_cast<uint32_t>(HcclNetLayerlevel::HCCL_NetLayer_L1));
1036 102 : if (deviceType == DevType::DEV_TYPE_910B || deviceType == DevType::DEV_TYPE_910) {
1037 80 : std::vector<u32> rankListTmp1;
1038 317 : for (auto& pair : serverToRank_) {
1039 588 : for (auto iter : pair.second) {
1040 351 : rankListTmp1.push_back(iter.userRank);
1041 351 : }
1042 : }
1043 80 : rankList_.insert({static_cast<uint32_t>(HcclNetLayerlevel::HCCL_NetLayer_L1), rankListTmp1});
1044 80 : rankSizeList_.insert(
1045 80 : {static_cast<uint32_t>(HcclNetLayerlevel::HCCL_NetLayer_L1), {topoAttr_.userRankSize}});
1046 102 : } else if (deviceType == DevType::DEV_TYPE_910_93) {
1047 22 : auto it = superPodToRank_.find(rankData_.superPodIdx);
1048 22 : if (it == superPodToRank_.end()) {
1049 0 : HCCL_ERROR(
1050 : "[RankGraphV1][%s] find superPodToRank_ failed, superPodIdx[%u]", __func__, rankData_.superPodIdx);
1051 0 : return HCCL_E_INTERNAL;
1052 : }
1053 22 : std::vector<u32> rankListTmp1;
1054 88 : for (auto iter : superPodToRank_[rankData_.superPodIdx]) {
1055 66 : rankListTmp1.push_back(iter.userRank);
1056 66 : }
1057 22 : std::vector<u32> rankSizeListTmp1;
1058 85 : for (auto iter : superPodToRank_) {
1059 63 : rankSizeListTmp1.push_back(iter.second.size());
1060 63 : }
1061 22 : rankList_.insert({static_cast<uint32_t>(HcclNetLayerlevel::HCCL_NetLayer_L1), rankListTmp1});
1062 22 : rankSizeList_.insert({static_cast<uint32_t>(HcclNetLayerlevel::HCCL_NetLayer_L1), rankSizeListTmp1});
1063 22 : }
1064 : }
1065 :
1066 523 : if (deviceType == DevType::DEV_TYPE_910_93 && superPodToRank_.size() > 1) {
1067 8 : netLayer_.push_back(static_cast<uint32_t>(HcclNetLayerlevel::HCCL_NetLayer_L2));
1068 8 : std::vector<u32> rankListTmp2;
1069 57 : for (const auto& pair : superPodToRank_) {
1070 98 : for (auto iter : pair.second) {
1071 49 : rankListTmp2.push_back(iter.userRank);
1072 49 : }
1073 : }
1074 8 : rankList_.insert({static_cast<uint32_t>(HcclNetLayerlevel::HCCL_NetLayer_L2), rankListTmp2});
1075 16 : rankSizeList_.insert({static_cast<uint32_t>(HcclNetLayerlevel::HCCL_NetLayer_L2), {topoAttr_.userRankSize}});
1076 8 : }
1077 523 : return HCCL_SUCCESS;
1078 523 : }
1079 : }; // namespace hccl
|
