Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include <unordered_map>
12 : #include "virtual_topo.h"
13 : #include "string_util.h"
14 : #include "binary_stream.h"
15 : #include "exception_util.h"
16 : #include "topo_common_types.h"
17 : #include "null_ptr_exception.h"
18 : #include "internal_exception.h"
19 : #include "not_support_exception.h"
20 : #include "rank_gph.h"
21 :
22 : namespace Hccl {
23 :
24 : using namespace std;
25 :
26 594 : void RankGraph::AddPeer(const shared_ptr<NetInstance::Peer> &peer)
27 : {
28 594 : if (!initFlag_) {
29 593 : if (peer == nullptr) {
30 1 : THROW<NullPtrException>("[RankGraph][AddPeer] peer is nullptr.");
31 : }
32 :
33 592 : RankId rankId = peer->GetRankId();
34 592 : peers_[rankId] = peer;
35 : } else {
36 1 : THROW<InternalException>("RankGraph AddPeer fail, rankGraph has been initialized, please check.");
37 : }
38 592 : }
39 :
40 228 : void RankGraph::AddNetInstance(const shared_ptr<NetInstance> &netInst)
41 : {
42 228 : if (netInst == nullptr) {
43 2 : THROW<NullPtrException>(StringFormat("[RankGraph][AddNetInstance] netInst is nullptr"));
44 : }
45 227 : if (!initFlag_) {
46 : // 外部调用保证netInst非空
47 : // 处理添加的netInst中的rank在virtualTopo中没有的情况
48 564 : for (const auto rankId : netInst->GetRankIds()) {
49 339 : if (!HasRank(rankId)) {
50 1 : THROW<InvalidParamsException>(
51 3 : StringFormat("[RankGraph][AddNetInstance] Non-innerRank[%d] exists netInst.", rankId));
52 : }
53 226 : }
54 :
55 : // 添加netInst, 若重复添加打印日志
56 225 : auto res = netInsts_[netInst->GetNetLayer()].emplace(netInst->GetNetInstId(), netInst);
57 225 : if (!res.second) {
58 0 : HCCL_WARNING("[RankGraph][AddNetInstance] netLayer[%u] netInstId[%s] is existed.", netInst->GetNetLayer(), netInst->GetNetInstId().c_str());
59 : }
60 : } else {
61 1 : THROW<InternalException>("RankGraph AddNetInstance fail, rankGraph has been initialized, please check.");
62 : }
63 225 : }
64 :
65 142 : void RankGraph::InitInnerRanks()
66 : {
67 : // 只支持在创建好InnerGroup之后调用, 否则抛异
68 142 : auto innerInstance = GetNetInstanceByRankId(0, myRank_);
69 142 : if (innerInstance == nullptr) {
70 1 : THROW<NullPtrException>(
71 3 : StringFormat("[RankGraph][SetInnerRanks] myRank[%d] netLayer[0] netInst is not existed.", myRank_));
72 : }
73 :
74 141 : innerRanks_ = innerInstance->GetRankIds();
75 141 : }
76 :
77 29 : void RankGraph::InitFinish()
78 : {
79 29 : initFlag_ = true;
80 29 : }
81 :
82 2180 : bool RankGraph::HasRank(RankId rankId) const
83 : {
84 2180 : if (peers_.find(rankId) == peers_.end()) {
85 18 : HCCL_DEBUG("[RankGraph][HasRank] rankId[%d] is not existed", rankId);
86 6 : return false;
87 : }
88 2174 : return true;
89 : }
90 :
91 322 : u32 RankGraph::GetRankSize() const
92 : {
93 322 : return peers_.size();
94 : }
95 :
96 7 : u32 RankGraph::GetInnerRankSize() const
97 : {
98 7 : return innerRanks_.size();
99 : }
100 :
101 84 : RankId RankGraph::GetMyRank() const
102 : {
103 84 : return myRank_;
104 : }
105 :
106 758 : LocalId RankGraph::GetLocalId(RankId rankId) const
107 : {
108 758 : if (!HasRank(rankId)) {
109 0 : THROW<InvalidParamsException>(StringFormat("[RankGraph][GetLocalId] rankId[%d] is not existed.", rankId));
110 : }
111 :
112 758 : return peers_.at(rankId)->GetLocalId();
113 : }
114 :
115 10 : LocalId RankGraph::GetReplacedLocalId(RankId rankId) const
116 : {
117 10 : if (!HasRank(rankId)) {
118 0 : THROW<InvalidParamsException>(StringFormat("[RankGraph][GetLocalId] rankId[%d] is not existed.", rankId));
119 : }
120 :
121 10 : return peers_.at(rankId)->GetReplacedLocalId();
122 : }
123 :
124 :
125 94 : set<u32> RankGraph::GetLevels(RankId rankId) const
126 : {
127 94 : if (!HasRank(rankId)) {
128 0 : THROW<InvalidParamsException>(StringFormat("[RankGraph][GetLevels] rankId[%d] is not existed.", rankId));
129 : }
130 :
131 94 : return peers_.at(rankId)->GetLevels();
132 : }
133 :
134 2 : u32 RankGraph::GetLevelNum() const
135 : {
136 2 : u32 validLevelNum{0};
137 18 : for (const auto &netInst : netInsts_) {
138 16 : if (!netInst.empty()) {
139 2 : validLevelNum++;
140 : }
141 : }
142 4 : HCCL_INFO("[RankGraph][%s] validLevelNum[%u]", __func__, validLevelNum);
143 2 : return validLevelNum;
144 : }
145 :
146 :
147 5 : const NetInstance *RankGraph::GetNetInstanceByNetInstId(u32 netLayer, const string &netInstId) const
148 : {
149 : // 不存在netInst, 则返回空
150 5 : if (netLayer >= netInsts_.size() || netInsts_.at(netLayer).count(netInstId) == 0) {
151 0 : HCCL_WARNING("[RankGraph][GetNetInstance] NetInstance netLayer[%u] netInstId[%s] is not existed.", netLayer,
152 : netInstId.c_str());
153 0 : return nullptr;
154 : }
155 5 : return netInsts_.at(netLayer).at(netInstId).get();
156 : }
157 :
158 12 : NetInstance *RankGraph::GetNetInstanceByNetInstId(u32 netLayer, const std::string &netInstId)
159 : {
160 : // 不存在netInst, 则返回空
161 12 : if (netLayer >= netInsts_.size() || netInsts_.at(netLayer).count(netInstId) == 0) {
162 0 : HCCL_WARNING("[RankGraph][GetNetInstance] NetInstance netLayer[%u] netInstId[%s] is not existed.", netLayer,
163 : netInstId.c_str());
164 0 : return nullptr;
165 : }
166 12 : return netInsts_.at(netLayer).at(netInstId).get();
167 : }
168 :
169 188 : const NetInstance *RankGraph::GetNetInstanceByRankId(u32 netLayer, RankId rankId) const
170 : {
171 : // 不存在netInst, 则返回空
172 188 : if (!HasRank(rankId)) {
173 3 : HCCL_WARNING("[RankGraph][GetNetInstance] NetInstance rankId[%d] netLayer[%u] is not existed.", rankId, netLayer);
174 1 : return nullptr;
175 : }
176 187 : return peers_.at(rankId)->GetNetInstance(netLayer);
177 : }
178 :
179 255 : NetInstance *RankGraph::GetNetInstanceByRankId(u32 netLayer, RankId rankId)
180 : {
181 255 : if (!HasRank(rankId)) {
182 3 : HCCL_WARNING("[RankGraph][GetNetInstance] NetInstance rankId[%d] netLayer[%u] is not existed.", rankId, netLayer);
183 1 : return nullptr;
184 : }
185 254 : const NetInstance *constInstance = peers_.at(rankId)->GetNetInstance(netLayer);
186 254 : if (constInstance == nullptr) {
187 0 : THROW<NullPtrException>(StringFormat("[RankGraph][GetGroup]GetNetInstance(rankId, netLayer) is nullptr"));
188 : }
189 254 : string netInstId = constInstance->GetNetInstId();
190 254 : return netInsts_.at(netLayer).at(netInstId).get();
191 254 : }
192 :
193 528 : const shared_ptr<NetInstance::Peer> RankGraph::GetPeer(RankId rankId) const
194 : {
195 528 : if (!HasRank(rankId)) {
196 6 : HCCL_WARNING("[RankGraph][GetPeer] rankId[%d] is not existed.", rankId);
197 2 : return nullptr;
198 : }
199 :
200 526 : return peers_.at(rankId);
201 : }
202 :
203 1919 : vector<NetInstance::Path> RankGraph::GetPaths(u32 netLayer, RankId sRankId, RankId dRankId) const
204 : {
205 : // 若sRank和dRank均不在innerGroup里，则返回空
206 1919 : if (innerRanks_.count(sRankId) == 0 && innerRanks_.count(dRankId) == 0) {
207 5430 : HCCL_WARNING("[RankGraph][GetPaths] sRankId[%d] and dRankId[%d] are not in innerRanks", sRankId, dRankId);
208 1810 : return {};
209 : }
210 :
211 109 : vector<NetInstance::Path> paths;
212 109 : auto netInst = GetNetInstanceByRankId(netLayer, sRankId);
213 109 : if (netInst == nullptr) {
214 96 : HCCL_WARNING("[RankGraph][GetPaths] netLayer[%u] sRankId[%d] netInst is not existed.", netLayer, sRankId);
215 32 : return {};
216 : }
217 77 : if (!netInst->HasNode(NetInstance::Peer::GenerateNodeId(dRankId))) {
218 3 : HCCL_WARNING("[RankGraph][GetPaths] netLayer[%u] sRankId[%d] netInst has no dRankId[%d].", netLayer, sRankId,
219 : dRankId);
220 1 : return {};
221 : }
222 :
223 76 : paths = netInst->GetPaths(sRankId, dRankId);
224 76 : if (paths.size() == 0) {
225 24 : HCCL_WARNING("[RankGraph][GetPaths] netLayer[%u] sRankId[%d] dRankId[%d] netInst has no path.", netLayer,
226 : sRankId, dRankId);
227 8 : return paths;
228 : }
229 :
230 202 : HCCL_DEBUG("[RankGraph][GetPaths] netLayer[%u] sRankId[%d] dRankId[%d] pathsize[%u].", netLayer, sRankId, dRankId,
231 : paths.size());
232 68 : return paths;
233 109 : }
234 :
235 3 : u32 RankGraph::GetLayerRanks(const u32 netLayer) const
236 : {
237 3 : u32 layerRankSize = 0;
238 3 : if(netInsts_.at(netLayer).size() == 0){
239 0 : HCCL_WARNING("[RankGraph][GetLayerRanks] RankGraph has no netInstance on netLayer %u", netLayer);
240 0 : return 0;
241 : }
242 8 : for (const auto& netInst : netInsts_.at(netLayer)) {
243 5 : layerRankSize += netInst.second->GetRankSize();
244 : }
245 3 : return layerRankSize;
246 : }
247 :
248 6 : void RankGraph::GetLocalInstRanks(const u32 netLayer, vector<u32> &rankList, u32 &rankNum) const
249 : {
250 6 : const NetInstance *netInstance = GetNetInstanceByRankId(netLayer, myRank_);
251 6 : if (netInstance == nullptr) {
252 2 : THROW<NullPtrException>(StringFormat("[RankGraph][GetLocalInstRanks] myRank %u has no netInstance on layer %u", myRank_, netLayer));
253 : }
254 5 : set<RankId> rankSet = netInstance->GetRankIds();
255 5 : rankList.clear();
256 15 : for (const RankId &rank : rankSet) {
257 10 : rankList.push_back(static_cast<u32>(rank));
258 : }
259 5 : rankNum = rankSet.size();
260 5 : }
261 :
262 7 : u32 RankGraph::GetLocalInstSize(const u32 netLayer) const
263 : {
264 7 : const NetInstance *netInstance = GetNetInstanceByRankId(netLayer, myRank_);
265 7 : if (netInstance == nullptr) {
266 0 : THROW<NullPtrException>(StringFormat("[RankGraph][GetLocalInstSize] myRank %u has no netInstance on layer %u", myRank_, netLayer));
267 : }
268 7 : return netInstance->GetRankSize();
269 : }
270 :
271 26 : const NetType RankGraph::GetNetType(const u32 netLayer) const
272 : {
273 26 : const NetInstance *netInstance = GetNetInstanceByRankId(netLayer, myRank_);
274 26 : if (netInstance == nullptr) {
275 4 : THROW<NullPtrException>(StringFormat("[RankGraph][GetLocalInstSize] myRank %u has no netInstance on layer %u", myRank_, netLayer));
276 : }
277 22 : return netInstance->GetNetType();
278 : }
279 :
280 6 : HcclResult RankGraph::GetNetInstanceList(const u32 netLayer, vector<u32> &instSizeList, u32 &listSize) const
281 : {
282 6 : instSizeList.clear();
283 6 : listSize = 0;
284 6 : if(netInsts_.at(netLayer).size() == 0){
285 3 : HCCL_WARNING("[RankGraph][GetNetInstanceList] RankGraph has no net instance on layer %u", netLayer);
286 1 : return HCCL_E_PARA;
287 : }
288 12 : for (const auto& netInst : netInsts_.at(netLayer)) {
289 7 : instSizeList.push_back(netInst.second->GetRankSize());
290 : }
291 5 : listSize = instSizeList.size();
292 5 : return HCCL_SUCCESS;
293 : }
294 :
295 2 : void RankGraph::GetTopoInstsByLayer(const u32 netLayer, std::vector<u32>& topoInsts, u32& topoInstNum) const
296 : {
297 2 : auto* netInstance = GetNetInstanceByRankId(netLayer, myRank_);
298 2 : netInstance->GetTopoInstsByLayer(topoInsts, topoInstNum);
299 2 : }
300 :
301 3 : HcclResult RankGraph::GetTopoType(const u32 netLayer, const u32 topoInstId, TopoType &topoType) const
302 : {
303 3 : auto *netInstance = GetNetInstanceByRankId(netLayer, myRank_);
304 3 : CHK_PRT_RET(netInstance == nullptr,
305 : HCCL_ERROR("[RankGraph::GetTopoType] netInstance is nullptr, myRank[%d], netLayer[%u], "
306 : "topoInstId[%u]", myRank_, netLayer, topoInstId),
307 : HCCL_E_PTR);
308 :
309 3 : auto ret = netInstance->GetTopoType(topoInstId, topoType);
310 3 : CHK_PRT_RET(ret != HCCL_SUCCESS,
311 : HCCL_ERROR("[%s] Failed to GetTopoType, myRank[%d], netLayer[%u], netInstId[%s], topoInstId[%u], "
312 : "ret[%d]", __func__, myRank_, netLayer, netInstance->GetNetInstId().c_str(), topoInstId,
313 : ret),
314 : ret);
315 2 : return HCCL_SUCCESS;
316 : }
317 :
318 3 : HcclResult RankGraph::GetRanksByTopoInst(
319 : const u32 netLayer, const u32 topoInstId, std::vector<u32> &ranks, u32 &rankNum) const
320 : {
321 3 : auto *netInstance = GetNetInstanceByRankId(netLayer, myRank_);
322 3 : CHK_PRT_RET(netInstance == nullptr,
323 : HCCL_ERROR("[RankGraph::GetRanksByTopoInst] netInstance is nullptr, myRank[%d], netLayer[%u], "
324 : "topoInstId[%u]", myRank_, netLayer, topoInstId),
325 : HCCL_E_PTR);
326 :
327 3 : auto ret = netInstance->GetRanksByTopoInst(topoInstId, ranks, rankNum);
328 3 : CHK_PRT_RET(ret != HCCL_SUCCESS,
329 : HCCL_ERROR("[%s] Failed to GetRanksByTopoInst, myRank[%d], netLayer[%u], netInstId[%s], "
330 : "topoInstId[%u], ret[%d]", __func__, myRank_, netLayer,
331 : netInstance->GetNetInstId().c_str(), topoInstId, ret),
332 : ret);
333 2 : return HCCL_SUCCESS;
334 : }
335 :
336 8 : HcclResult RankGraph::GetEndpointNum(uint32_t layer, uint32_t topoInstId, uint32_t* num) const
337 : {
338 8 : auto peer = GetPeer(myRank_);
339 8 : if (peer == nullptr) {
340 3 : HCCL_ERROR("[RankGraph::GetEndpointNum] Peer is nullptr at netLayer [%u]", layer);
341 1 : return HCCL_E_PTR;
342 : }
343 7 : auto ifacesVec = peer->GetIfacesByLayer(layer);
344 7 : uint32_t sum = 0;
345 20 : for (auto& iface : ifacesVec) {
346 13 : if (iface->GetTopoInstId() == topoInstId) {
347 11 : sum += iface->GetLinkProtocols().size();
348 : }
349 : }
350 7 : *num = sum;
351 7 : return HCCL_SUCCESS;
352 8 : }
353 :
354 115 : HcclResult GetCommAddr(CommAddr &commAddr, const IpAddress &ipAddr)
355 : {
356 115 : s32 family = ipAddr.GetFamily();
357 115 : if (family == AF_INET) {
358 115 : string addr = ipAddr.GetIpStr();
359 115 : if (ipAddr.IsEID(addr)) {
360 0 : commAddr.type = COMM_ADDR_TYPE_EID;
361 0 : const auto &eid = ipAddr.GetEid();
362 0 : for (u32 i = 0; i < URMA_EID_LEN && i < sizeof(commAddr.eid); i++) {
363 0 : commAddr.eid[i] = eid.raw[i];
364 : }
365 : } else {
366 115 : commAddr.type = COMM_ADDR_TYPE_IP_V4;
367 115 : commAddr.addr = ipAddr.GetBinaryAddress().addr;
368 : }
369 115 : } else if (family == AF_INET6) {
370 0 : commAddr.type = COMM_ADDR_TYPE_IP_V6;
371 0 : commAddr.addr6 = ipAddr.GetBinaryAddress().addr6;
372 : } else {
373 0 : HCCL_ERROR("invalid commAddrType");
374 0 : return HCCL_E_INTERNAL;
375 : }
376 115 : return HCCL_SUCCESS;
377 : }
378 :
379 91 : EndpointLocType AddrPositionToEndpointLoc(AddrPosition pos) {
380 91 : switch (pos) {
381 6 : case AddrPosition::HOST: return ENDPOINT_LOC_TYPE_HOST;
382 85 : case AddrPosition::DEVICE: return ENDPOINT_LOC_TYPE_DEVICE;
383 0 : default: return ENDPOINT_LOC_TYPE_RESERVED;
384 : }
385 : }
386 :
387 6 : HcclResult RankGraph::GetEndpointDesc(uint32_t layer, uint32_t topoInstId, uint32_t* descNum, EndpointDesc* endpointDesc) const
388 : {
389 6 : auto peer = GetPeer(myRank_);
390 9 : CHK_PTR_NULL(peer);
391 :
392 5 : auto ifacesVec = peer->GetIfacesByLayer(layer);
393 5 : uint32_t count = 0;
394 :
395 : // 找到 topoInstId 匹配的 iface
396 15 : for (const auto& iface : ifacesVec) {
397 10 : if (iface->GetTopoInstId() != topoInstId) {
398 2 : continue;
399 : }
400 :
401 : // 对该 iface，从 endpointToIfaceMap_ 中找出所有匹配的 EndpointDesc
402 : // 一个 iface 可能对应多个 protocol（即多个 EndpointDesc）
403 8 : const auto& endpointMap = peer->GetEndpointToIfaceMap();
404 24 : for (const auto& entry : endpointMap) {
405 16 : std::pair<CommAddr, CommProtocol> endpoint = entry.first;
406 16 : const std::shared_ptr<NetInstance::ConnInterface>& mappedIface = entry.second;
407 :
408 16 : if (mappedIface != iface) {
409 10 : continue;
410 : }
411 :
412 : // 检查输出缓冲区是否足够
413 6 : if (count >= *descNum) {
414 0 : HCCL_ERROR("[RankGraph::GetEndpointDesc] endpointDesc array too small: "
415 : "need %u, given %u", count + 1, *descNum);
416 0 : return HCCL_E_PARA;
417 : }
418 :
419 6 : endpointDesc[count].commAddr = endpoint.first;
420 6 : endpointDesc[count].protocol = endpoint.second;
421 6 : endpointDesc[count].loc.locType = AddrPositionToEndpointLoc(iface->GetPos());
422 :
423 18 : HCCL_INFO("[RankGraph::GetEndpointDesc] local type is %d, protocol %d", endpointDesc[count].loc.locType,
424 : endpointDesc[count].protocol);
425 6 : count++;
426 : }
427 8 : }
428 :
429 5 : *descNum = count;
430 5 : return HCCL_SUCCESS;
431 6 : }
432 :
433 3 : HcclResult RankGraph::GetEndpointInfo(uint32_t rankId,
434 : const EndpointDesc *endpointDesc,
435 : EndpointAttr endpointAttr,
436 : uint32_t infoLen,
437 : void *info) const
438 : {
439 3 : if (endpointDesc == nullptr || info == nullptr) {
440 3 : HCCL_ERROR("[GetEndpointInfo] Invalid parameter");
441 1 : return HCCL_E_PTR;
442 : }
443 2 : const std::shared_ptr<NetInstance::Peer> peer = GetPeer(rankId);
444 2 : if (peer == nullptr) {
445 0 : HCCL_ERROR("[RankGraph::GetEndpointInfo] Peer is nullptr for rankId [%u]", rankId);
446 0 : return HCCL_E_PTR;
447 : }
448 :
449 : // 查找接口
450 2 : auto key = std::make_pair(endpointDesc->commAddr, endpointDesc->protocol);
451 2 : const auto& endpointToIfaceMap = peer->GetEndpointToIfaceMap();
452 2 : auto it = endpointToIfaceMap.find(key);
453 2 : if (it == endpointToIfaceMap.end()) {
454 0 : HCCL_ERROR("[GetEndpointInfo] No matching interface found");
455 0 : return HCCL_E_NOT_FOUND;
456 : }
457 2 : const auto& iface = it->second;
458 : // 填充信息
459 2 : switch (endpointAttr) {
460 2 : case ENDPOINT_ATTR_BW_COEFF: {
461 2 : if (infoLen != sizeof(EndpointAttrBwCoeff)) {
462 0 : HCCL_ERROR("[GetEndpointInfo] Size mismatch: expected %zu, actual %u",
463 : sizeof(EndpointAttrBwCoeff), infoLen);
464 0 : return HCCL_E_PARA;
465 : }
466 2 : *(static_cast<EndpointAttrBwCoeff*>(info)) = iface->GetPorts().size();
467 2 : break;
468 : }
469 0 : case ENDPOINT_ATTR_DIE_ID: {
470 0 : if (infoLen != sizeof(EndpointAttrDieId)) {
471 0 : HCCL_ERROR("[GetEndpointInfo] Size mismatch: expected %zu, actual %u",
472 : sizeof(EndpointAttrDieId), infoLen);
473 0 : return HCCL_E_PARA;
474 : }
475 0 : *(static_cast<EndpointAttrDieId*>(info)) = iface->GetLocalDieId();
476 0 : HCCL_INFO("GetEndpointInfo rankId[%u] iface[%s]", rankId, iface->Describe().c_str());
477 0 : break;
478 : }
479 0 : case ENDPOINT_ATTR_LOCATION: {
480 0 : if (infoLen != sizeof(EndpointAttrLocation)) {
481 0 : HCCL_ERROR("[GetEndpointInfo] Size mismatch: expected %zu, actual %u",
482 : sizeof(EndpointAttrLocation), infoLen);
483 0 : return HCCL_E_PARA;
484 : }
485 0 : *(static_cast<EndpointAttrLocation*>(info)) = iface->GetPos();
486 0 : break;
487 : }
488 0 : default: {
489 0 : HCCL_ERROR("[GetEndpointInfo] Invalid endpointAttr [%d]", endpointAttr);
490 0 : return HCCL_E_PARA;
491 : }
492 : }
493 2 : return HCCL_SUCCESS;
494 2 : }
495 :
496 4 : bool RankGraph::IsSymmetric(const u32 netLayer) const
497 : {
498 4 : if(netInsts_.at(netLayer).size() == 0){
499 2 : THROW<NullPtrException>(StringFormat("[RankGraph][IsSymmetric] RankGraph no netInstance on net layer %u", netLayer));
500 : }
501 3 : std::unordered_set<u32> rankSize;
502 8 : for (const auto& netInst : netInsts_.at(netLayer)) {
503 5 : rankSize.insert(netInst.second->GetRankSize());
504 : }
505 6 : return rankSize.size() == 1;
506 3 : }
507 :
508 143 : std::shared_ptr<NetInstance> GetOrCreateNetInstance(u32 netLayer, const string &netInstId, NetType type,
509 : Level2Id2NetInst &netInsts, RankGraph *rankGraph)
510 : {
511 143 : std::shared_ptr<NetInstance> netInstance;
512 :
513 : // 若netLayer和netInstId对应netInstance没创建则创建
514 143 : if (netInsts[netLayer].count(netInstId) == 0) {
515 45 : if (type == NetType::TOPO_FILE_DESC) {
516 25 : netInstance = std::make_shared<InnerNetInstance>(netLayer, netInstId);
517 20 : } else if (type == NetType::CLOS) {
518 20 : netInstance = std::make_shared<ClosNetInstance>(netLayer, netInstId);
519 : }
520 45 : netInsts[netLayer][netInstId] = netInstance;
521 : // netInstance添加到virtualTopo
522 45 : rankGraph->AddNetInstance(netInstance);
523 :
524 135 : HCCL_DEBUG("[CreateNetInstance] create netInstance success, netLayer[%u] netInstId[%s] type[%s].", netLayer, netInstId.c_str(),
525 : type.Describe().c_str());
526 : } else {
527 : // 若netInstance存在, type不一致则报错
528 98 : NetType curType = netInsts[netLayer][netInstId]->GetNetType();
529 98 : if (curType != type) {
530 0 : HCCL_WARNING("[CreateNetInstance]FabType [%s] and [%s] no match", curType.Describe().c_str(),
531 : type.Describe().c_str());
532 0 : return nullptr;
533 : }
534 : // 若netInstance存在, type一致则直接获取
535 98 : netInstance = netInsts[netLayer][netInstId];
536 : }
537 143 : return netInstance;
538 143 : }
539 :
540 56 : void GetNewNodeInfo(u32 layer, RankId newRankId, const NetInstance::Link &oldLink, shared_ptr<NetInstance> &newNetInstance,
541 : RankId2PeerMap &tmpPeers, shared_ptr<NetInstance::Node> &newNode,
542 : shared_ptr<NetInstance::ConnInterface> &newIface, bool isSource)
543 : {
544 56 : shared_ptr<NetInstance::Node> oldNode;
545 56 : shared_ptr<NetInstance::ConnInterface> oldIface;
546 56 : if (isSource) {
547 28 : oldNode = oldLink.GetSourceNode();
548 28 : oldIface = oldLink.GetSourceIface();
549 : } else {
550 28 : oldNode = oldLink.GetTargetNode();
551 28 : oldIface = oldLink.GetTargetIface();
552 : }
553 :
554 56 : NetInstance::Node::NodeType type = oldNode->GetType();
555 56 : if (type == NetInstance::Node::NodeType::PEER) {
556 32 : newIface = oldIface;
557 32 : newNode = tmpPeers.at(newRankId);
558 32 : tmpPeers.at(newRankId)->AddConnInterface(layer, newIface);
559 24 : } else if (type == NetInstance::Node::NodeType::FABRIC) {
560 24 : newIface = nullptr;
561 24 : newNode = oldNode;
562 24 : if (!newNetInstance->HasNode(newNode->GetNodeId())) {
563 1 : newNetInstance->AddNode(newNode);
564 : }
565 : } else {
566 0 : THROW<NotSupportException>(
567 0 : StringFormat("[CreateSubNetInstances][GetNewNodeInfo] newRankId[%d] oldLink Node isSource[%d] type[%s] "
568 : " is not supported.",
569 0 : newRankId, isSource, type.Describe().c_str()));
570 : }
571 56 : }
572 :
573 33 : bool NeedUpdateTopoInstForSubGraph(const NetInstance *oldNetInstance, u32 topoInstId, RankId parentMyRank)
574 : {
575 33 : auto topoInstIter = oldNetInstance->topoInsts_.find(topoInstId);
576 33 : if (topoInstIter == oldNetInstance->topoInsts_.end()) {
577 0 : HCCL_DEBUG("[SubRankGraph][NeedUpdateTopoInstForSubGraph] skip topoInstId[%u], parentMyRank[%d], "
578 : "not found in parent netInstId[%s]",
579 : topoInstId, parentMyRank, oldNetInstance->GetNetInstId().c_str());
580 0 : return false;
581 : }
582 33 : if (topoInstIter->second == nullptr) {
583 3 : HCCL_WARNING("[SubRankGraph][NeedUpdateTopoInstForSubGraph] skip topoInstId[%u], parentMyRank[%d], "
584 : "topoInst is null in parent netInstId[%s]",
585 : topoInstId, parentMyRank, oldNetInstance->GetNetInstId().c_str());
586 1 : return false;
587 : }
588 32 : return topoInstIter->second->ranks.count(parentMyRank) > 0;
589 : }
590 :
591 28 : void AddNewLink(u32 layer, const NetInstance::Link &oldLink, RankId srcNewRankId, RankId dstNewRankId,
592 : shared_ptr<NetInstance> &newNetInstance, RankId2PeerMap &tmpPeers,
593 : const NetInstance *oldNetInstance, RankId parentMyRank)
594 : {
595 : // 不添加绕路link
596 28 : if (oldLink.GetHop() > 1 && oldLink.GetType() != LinkType::PEER2NET) {
597 0 : return;
598 : }
599 :
600 28 : shared_ptr<NetInstance::ConnInterface> newSourceIface;
601 28 : shared_ptr<NetInstance::ConnInterface> newTargetIface;
602 28 : shared_ptr<NetInstance::Node> newSourceNode;
603 28 : shared_ptr<NetInstance::Node> newTargetNode;
604 : // oldLink有fabicNode需要先addFabricNode
605 : // SourceNode
606 28 : GetNewNodeInfo(layer, srcNewRankId, oldLink, newNetInstance, tmpPeers, newSourceNode, newSourceIface, true);
607 : // TargetNode
608 28 : GetNewNodeInfo(layer, dstNewRankId, oldLink, newNetInstance, tmpPeers, newTargetNode, newTargetIface, false);
609 : // link
610 : shared_ptr<NetInstance::Link> link
611 28 : = make_shared<NetInstance::Link>(newSourceNode, newTargetNode, newSourceIface, newTargetIface, oldLink.GetType(),
612 56 : oldLink.GetLinkProtocols(), oldLink.GetLinkDirection(), oldLink.GetHop());
613 :
614 28 : newNetInstance->AddLink(link);
615 28 : if (newSourceIface != nullptr) {
616 16 : u32 sourceTopoInstId = newSourceIface->GetTopoInstId();
617 16 : TopoType sourceTopoType = newSourceIface->GetTopoType();
618 16 : if (NeedUpdateTopoInstForSubGraph(oldNetInstance, sourceTopoInstId, parentMyRank)) {
619 14 : newNetInstance->UpdateTopoInst(sourceTopoInstId, sourceTopoType, srcNewRankId);
620 : }
621 : }
622 28 : if (newTargetIface != nullptr) {
623 16 : u32 targetTopoInstId = newTargetIface->GetTopoInstId();
624 16 : TopoType targetTopoType = newTargetIface->GetTopoType();
625 16 : if (NeedUpdateTopoInstForSubGraph(oldNetInstance, targetTopoInstId, parentMyRank)) {
626 14 : newNetInstance->UpdateTopoInst(targetTopoInstId, targetTopoType, dstNewRankId);
627 : }
628 : }
629 :
630 83 : for (const auto&pair: newNetInstance->topoInsts_){
631 55 : uint32_t topoInstId = pair.first;
632 55 : if(pair.second==nullptr){
633 0 : THROW<NullPtrException>(StringFormat("[SubRankGraph][AddNewLink] topoInstId %u has no TopoInst", topoInstId));
634 : }
635 55 : auto topoType = pair.second->topoType;
636 55 : if (UNLIKELY(HcclCheckLogLevel(DLOG_DEBUG))) {
637 165 : HCCL_DEBUG("[SubRankGraph] topoInstId[%u] topoType[%d]", topoInstId, topoType);
638 : }
639 : }
640 84 : HCCL_DEBUG("[RankGraph][AddNewLink] srcNewRankId[%d] dstNewRankId[%d] newLink[%s]", srcNewRankId, dstNewRankId,
641 : link->Describe().c_str());
642 28 : }
643 :
644 5 : void AddGroupLinks(const vector<RankId> &rankIds, const NetInstance *oldNetInstance, shared_ptr<NetInstance> &newNetInstance,
645 : RankId2PeerMap &tmpPeers, RankId parentMyRank)
646 : {
647 5 : set<RankId> newRankIds = newNetInstance->GetRankIds();
648 5 : u32 layer = newNetInstance->GetNetLayer();
649 5 : if (oldNetInstance == nullptr) {
650 0 : THROW<NullPtrException>(StringFormat("[AddGroupLinks]oldNetInstance is nullptr"));
651 : }
652 5 : if (newRankIds.size() == 1) {
653 : // 子通信域单卡场景直接返回1DMESH
654 4 : RankId singleId = *newRankIds.begin();
655 4 : newNetInstance->UpdateTopoInst(0, TopoType::MESH_1D, singleId);
656 4 : return;
657 : }
658 5 : for (RankId srcRankId : newRankIds) {
659 20 : for (RankId dstRankId : newRankIds) {
660 16 : if (srcRankId == dstRankId) {
661 4 : continue;
662 : }
663 : // 对oldNetInstance中的每一条Link, 创建新的Link添加到newNetInstance
664 12 : vector<NetInstance::Path> oldPaths = oldNetInstance->GetPaths(rankIds[srcRankId], rankIds[dstRankId]);
665 28 : for (auto &oldPath : oldPaths) {
666 44 : for (auto &oldLink : oldPath.links) {
667 28 : AddNewLink(layer, oldLink, srcRankId, dstRankId, newNetInstance, tmpPeers, oldNetInstance,
668 : parentMyRank);
669 : }
670 : }
671 12 : }
672 : }
673 5 : }
674 :
675 3 : void RankGraph::AddSubPeers(const std::vector<RankId> &rankIds, RankGraph *subRankGraph, RankId2PeerMap &peers) const
676 : {
677 : // 遍历rankIds将索引作为子虚拟拓扑的rankId构造subPeer并添加到subRankGraph
678 3 : s32 rankSize = rankIds.size();
679 9 : for (RankId subRankId = 0; subRankId < rankSize; ++subRankId) {
680 6 : RankId rankId = rankIds[subRankId];
681 6 : shared_ptr<NetInstance::Peer> oldPeer = GetPeer(rankId);
682 6 : LocalId localId = oldPeer->GetLocalId();
683 6 : LocalId replacedLocalId = oldPeer->GetReplacedLocalId();
684 6 : DeviceId deviceId = oldPeer->GetDeviceId();
685 6 : u32 devicePort = oldPeer->GetDevicePort();
686 6 : u32 hostPort = oldPeer->GetHostPort();
687 6 : shared_ptr<NetInstance::Peer> subPeer = make_shared<NetInstance::Peer>(subRankId, localId, replacedLocalId, deviceId, devicePort, hostPort);
688 6 : subRankGraph->AddPeer(subPeer);
689 6 : peers.emplace(subRankId, subPeer);
690 6 : const auto& oldEndpointMap = oldPeer->GetEndpointToIfaceMap();
691 6 : for (const auto& entry : oldEndpointMap) {
692 0 : subPeer->SetEndpointToIface(entry.first.first, entry.first.second, entry.second);
693 0 : HCCL_DEBUG("[SubRankGraph][AddSubPeers] endpointToIfaceMap: protocol[%d] for subRankId[%d]",
694 : entry.first.second, subRankId);
695 : }
696 18 : HCCL_DEBUG("[RankGraph][AddSubPeers] oldRankId[%d] subPeer[%s] add success.", rankId,
697 : subPeer->Describe().c_str());
698 6 : }
699 3 : }
700 :
701 :
702 3 : RankId GetSubRankId(const vector<RankId> &rankIds, RankId rank)
703 : {
704 : RankId subRank;
705 :
706 : // rankIds中查找rank, 数组索引即为subMyRank
707 3 : auto iter = find(begin(rankIds), end(rankIds), rank);
708 3 : if (iter != end(rankIds)) {
709 3 : subRank = distance(begin(rankIds), iter);
710 : } else {
711 0 : THROW<InvalidParamsException>(
712 0 : StringFormat("[RankGraph][CreateSubVirtTopo] rankIds has no rank[%d].", rank));
713 : }
714 :
715 9 : HCCL_DEBUG("[GetSubRank] rank[%d] subRank[%d].", rank, subRank);
716 3 : return subRank;
717 : }
718 :
719 : /**
720 : * 1. 创建子NetInstance
721 : * 2. peer添加对应netInstance
722 : */
723 3 : void RankGraph::CreateSubNetInstances(const std::vector<RankId> rankIds, Level2Id2NetInst &subNetInstances,
724 : RankId2PeerMap &peers, RankGraph *subRankGraph) const
725 : {
726 : // 遍历rankIds, 获取每个rankId所在的oldNetInstance, 创建subNetInstance
727 3 : RankId rankSize = rankIds.size();
728 9 : for (RankId subRankId = 0; subRankId < rankSize; ++subRankId) {
729 6 : set<u32> curLevels = GetLevels(rankIds[subRankId]);
730 14 : for (u32 netLayer : curLevels) {
731 8 : const NetInstance *oldNetInstance = GetNetInstanceByRankId(netLayer, rankIds[subRankId]);
732 8 : if (oldNetInstance == nullptr) {
733 0 : THROW<NullPtrException>(
734 0 : StringFormat("[RankGraph][CreateSubNetInstances] oldNetInstance is nullptr"));
735 : }
736 : // 创建subNetInstance (根据oldNetInstance.netLayer,id,type)
737 8 : NetType netType = oldNetInstance->GetNetType();
738 8 : string netInstId = oldNetInstance->GetNetInstId();
739 8 : shared_ptr<NetInstance> subNetInstance = GetOrCreateNetInstance(netLayer, netInstId, netType, subNetInstances, subRankGraph);
740 8 : if (subNetInstance == nullptr) {
741 0 : THROW<NullPtrException>(
742 0 : StringFormat("[RankGraph][CreateSubNetInstances] subNetInstance is nullptr"));
743 : }
744 :
745 : // subNetInstance Add RankId and subPeer
746 8 : shared_ptr<NetInstance::Peer> subPeer = peers.at(subRankId);
747 8 : subNetInstance->AddRankId(subRankId);
748 8 : subNetInstance->AddNode(subPeer);
749 :
750 : // subPeer Add subNetInstance
751 8 : subPeer->AddNetInstance(subNetInstance);
752 24 : HCCL_DEBUG("[RankGraph][CreateSubNetInstances] subNetInstance subRankId[%d] subType[%s] subNetInstId[%s]",
753 : subRankId, netType.Describe().c_str(), netInstId.c_str());
754 8 : }
755 6 : }
756 3 : }
757 :
758 3 : void RankGraph::AddSubLinks(const std::vector<RankId> &rankIds, RankId2PeerMap &peers, Level2Id2NetInst &subNetInsts,
759 : RankId parentMyRank) const
760 : {
761 : // 遍历subNetInstances，对每一个NetInstance插入Links
762 27 : for (u32 netLayer = 0; netLayer < subNetInsts.size(); ++netLayer) {
763 29 : for (auto &curNetInstance : subNetInsts[netLayer]) {
764 5 : const NetInstance *oldNetInstance = GetNetInstanceByNetInstId(netLayer, curNetInstance.first);
765 5 : AddGroupLinks(rankIds, oldNetInstance, curNetInstance.second, peers, parentMyRank);
766 : }
767 : }
768 3 : }
769 :
770 3 : unique_ptr<RankGraph> RankGraph::CreateSubRankGraph(const std::vector<u32> &rankIds) const
771 : {
772 : // 参数类型转换
773 3 : vector<RankId> subRankIds;
774 3 : for_each(rankIds.begin(), rankIds.end(), [&](u32 rankId) {
775 6 : subRankIds.emplace_back(static_cast<RankId>(rankId));
776 6 : });
777 :
778 : // 参数检查, 若rankIds中存在当前virtualTopo不存在的rankId, 抛异
779 9 : for (const auto rankId : subRankIds) {
780 6 : if (!HasRank(rankId)) {
781 0 : THROW<InvalidParamsException>(
782 0 : StringFormat("[RankGraph][CreateSubVirtTopo] rankId[%d] is not existed.", rankId));
783 : }
784 : }
785 :
786 : // step1: 创建subRankGraph
787 3 : RankId subMyRankId = GetSubRankId(subRankIds, myRank_);
788 3 : unique_ptr<RankGraph> subRankGraph = make_unique<RankGraph>(subMyRankId);
789 :
790 : // step2: subRankGraph添加subPeers
791 3 : RankId2PeerMap peers; // 保存Peer指针以便后续执行Add操作
792 3 : AddSubPeers(subRankIds, subRankGraph.get(), peers);
793 :
794 : // step3: 构造subNetInstances, NetInstance添加RankId和Peer, Peer添加NetInstance
795 3 : Level2Id2NetInst subNetInstances(MAX_NET_LAYER); // 保存NetInstance指针以便后续执行Add操作
796 3 : CreateSubNetInstances(subRankIds, subNetInstances, peers, subRankGraph.get());
797 :
798 : // step4: subNetInstances添加Links, Peer添加ConnIfaces
799 3 : AddSubLinks(subRankIds, peers, subNetInstances, myRank_);
800 :
801 : // step5: 设置innerRanks
802 3 : subRankGraph->InitInnerRanks();
803 : // step6: 构造完成
804 3 : subRankGraph->InitFinish();
805 :
806 9 : HCCL_INFO("[subRankGraph] Build success!");
807 3 : subRankGraph->Dump();
808 3 : return subRankGraph;
809 3 : }
810 :
811 5 : std::vector<char> RankGraph::GetPackedData(const std::vector<std::pair<u32, RankId>> &netLayerRankPairs) const
812 : {
813 5 : std::vector<u32> numVec;
814 5 : std::vector<LinkData> links;
815 5 : u32 netLayerRankPairsNum = netLayerRankPairs.size();
816 :
817 15 : HCCL_DEBUG("netLayerRankPairs Num=%u", netLayerRankPairsNum);
818 :
819 9 : for (const auto &it : netLayerRankPairs) {
820 4 : auto paths = GetPaths(it.first, myRank_, it.second);
821 4 : numVec.push_back(paths.size());
822 12 : HCCL_DEBUG("RankGraph::GetPackedData: netLayer=%u, srcRank=%u, dstRank=%u", it.first, myRank_, it.second);
823 6 : for (const auto &path : paths) {
824 2 : links.emplace_back(path);
825 6 : HCCL_DEBUG("RankGraph::GetPackedData: %s", links.back().Describe().c_str());
826 : }
827 4 : }
828 5 : if (links.empty()) {
829 12 : HCCL_WARNING("[RankGraph][GetPackedData]connected links is empty");
830 : }
831 :
832 5 : std::vector<char> result;
833 5 : BinaryStream binaryStream;
834 5 : u32 linkSize = links.size();
835 5 : binaryStream << netLayerRankPairsNum;
836 5 : binaryStream << linkSize;
837 15 : HCCL_DEBUG("netLayerRankPairsNum=%u, linkSize=%u", netLayerRankPairsNum, linkSize);
838 5 : u32 idx = 0;
839 9 : for (const auto &it : netLayerRankPairs) {
840 4 : binaryStream << it.first;
841 4 : binaryStream << it.second;
842 4 : binaryStream << numVec[idx];
843 12 : HCCL_DEBUG("netLayer=%u, RankId=%u, num=%u", it.first, it.second, numVec[idx]);
844 4 : idx++;
845 : }
846 :
847 7 : for (const auto &link : links) {
848 2 : binaryStream << link.GetUniqueId();
849 : }
850 5 : binaryStream.Dump(result);
851 :
852 5 : return result;
853 5 : }
854 :
855 70 : string RankIds2Str(const set<RankId> &rankIds)
856 : {
857 70 : stringstream ranks;
858 288 : for (auto it = rankIds.begin(); it != rankIds.end(); ++it) {
859 218 : if (it != rankIds.begin()) {
860 148 : ranks << ", ";
861 : }
862 218 : ranks << *it;
863 : }
864 140 : return ranks.str();
865 70 : }
866 :
867 26 : void RankGraph::Dump() const
868 : {
869 78 : HCCL_DEBUG("RankGraph Dump:");
870 78 : HCCL_DEBUG("myRank: %d", myRank_);
871 78 : HCCL_DEBUG("innerRanks: [%s]", RankIds2Str(innerRanks_).c_str());
872 78 : HCCL_DEBUG("peers:");
873 102 : for (const auto& peer : peers_) {
874 228 : HCCL_DEBUG("%s", peer.second->Describe().c_str());
875 : }
876 78 : HCCL_DEBUG("netInsts:");
877 234 : for (uint32_t i = 0; i < netInsts_.size(); ++i) {
878 252 : for (auto& netInst : netInsts_[i]) {
879 132 : HCCL_DEBUG("[netLayer=%u, netInstId=%s]", i, netInst.first.c_str());
880 132 : HCCL_DEBUG("%s", netInst.second->Describe().c_str());
881 132 : HCCL_DEBUG("rankIds: [%s]", RankIds2Str(netInst.second->GetRankIds()).c_str());
882 132 : HCCL_DEBUG("peers:");
883 186 : for (const auto& peerMapIt : netInst.second->GetPeers()) {
884 426 : HCCL_DEBUG("%s", peerMapIt.second->Describe().c_str());
885 : }
886 132 : HCCL_DEBUG("fabrics:");
887 113 : for (const auto& fabric : netInst.second->GetFabrics()) {
888 207 : HCCL_DEBUG("%s", fabric->Describe().c_str());
889 : }
890 44 : set<NodeId> nodeIds{};
891 132 : HCCL_DEBUG("Graph Nodes:");
892 44 : netInst.second->GetGraph().TraverseNode([&](shared_ptr<NetInstance::Node> node) {
893 211 : nodeIds.insert(node->GetNodeId());
894 633 : HCCL_DEBUG("%s", node->Describe().c_str());
895 211 : });
896 132 : HCCL_DEBUG("Graph Links:");
897 255 : for (NodeId nodeId : nodeIds) {
898 211 : netInst.second->GetGraph().TraverseEdge(nodeId, [&](shared_ptr<NetInstance::Link> link) {
899 1404 : HCCL_DEBUG("%s", link->Describe().c_str());
900 468 : });
901 : }
902 44 : }
903 : }
904 26 : }
905 101 : CommProtocol LinkProtocolToCommProtocol(const LinkProtocol &linkProtocol)
906 : {
907 101 : constexpr std::pair<LinkProtocol, CommProtocol> protocolPairs[] = {
908 : {LinkProtocol::UB_CTP, COMM_PROTOCOL_UBC_CTP},
909 : {LinkProtocol::UB_TP, COMM_PROTOCOL_UBC_TP},
910 : {LinkProtocol::ROCE, COMM_PROTOCOL_ROCE},
911 : {LinkProtocol::HCCS, COMM_PROTOCOL_HCCS},
912 : {LinkProtocol::PCIE, COMM_PROTOCOL_PCIE},
913 : {LinkProtocol::UB_MEM, COMM_PROTOCOL_UB_MEM},
914 : {LinkProtocol::UBOE, COMM_PROTOCOL_UBOE},
915 : {LinkProtocol::UBG, COMM_PROTOCOL_UBG}};
916 :
917 105 : for (const auto &p : protocolPairs) {
918 105 : if (p.first == linkProtocol) {
919 101 : return p.second;
920 : }
921 : }
922 :
923 0 : return COMM_PROTOCOL_RESERVED;
924 : }
925 : } // namespace Hccl
|
