Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include <unordered_map>
12 : #include "virtual_topo.h"
13 : #include "string_util.h"
14 : #include "binary_stream.h"
15 : #include "exception_util.h"
16 : #include "topo_common_types.h"
17 : #include "null_ptr_exception.h"
18 : #include "internal_exception.h"
19 : #include "not_support_exception.h"
20 : #include "rank_gph.h"
21 :
22 : namespace Hccl {
23 :
24 : using namespace std;
25 :
26 671 : void RankGraph::AddPeer(const shared_ptr<NetInstance::Peer>& peer)
27 : {
28 671 : if (!initFlag_) {
29 670 : if (peer == nullptr) {
30 1 : THROW<NullPtrException>("[RankGraph][AddPeer] peer is nullptr.");
31 : }
32 :
33 669 : RankId rankId = peer->GetRankId();
34 669 : peers_[rankId] = peer;
35 : } else {
36 1 : THROW<InternalException>("RankGraph AddPeer fail, rankGraph has been initialized, please check.");
37 : }
38 669 : }
39 :
40 269 : void RankGraph::AddNetInstance(const shared_ptr<NetInstance>& netInst)
41 : {
42 269 : if (netInst == nullptr) {
43 2 : THROW<NullPtrException>(StringFormat("[RankGraph][AddNetInstance] netInst is nullptr"));
44 : }
45 268 : if (!initFlag_) {
46 : // 外部调用保证netInst非空
47 : // 处理添加的netInst中的rank在virtualTopo中没有的情况
48 702 : for (const auto rankId : netInst->GetRankIds()) {
49 436 : if (!HasRank(rankId)) {
50 1 : THROW<InvalidParamsException>(
51 3 : StringFormat("[RankGraph][AddNetInstance] Non-innerRank[%d] exists netInst.", rankId));
52 : }
53 267 : }
54 :
55 : // 添加netInst, 若重复添加打印日志
56 266 : auto res = netInsts_[netInst->GetNetLayer()].emplace(netInst->GetNetInstId(), netInst);
57 266 : if (!res.second) {
58 0 : HCCL_WARNING(
59 : "[RankGraph][AddNetInstance] netLayer[%u] netInstId[%s] is existed.", netInst->GetNetLayer(),
60 : netInst->GetNetInstId().c_str());
61 : }
62 : } else {
63 1 : THROW<InternalException>("RankGraph AddNetInstance fail, rankGraph has been initialized, please check.");
64 : }
65 266 : }
66 :
67 185 : void RankGraph::InitInnerRanks()
68 : {
69 : // 只支持在创建好InnerGroup之后调用, 否则抛异
70 185 : auto innerInstance = GetNetInstanceByRankId(0, myRank_);
71 185 : if (innerInstance == nullptr) {
72 1 : THROW<NullPtrException>(
73 3 : StringFormat("[RankGraph][SetInnerRanks] myRank[%d] netLayer[0] netInst is not existed.", myRank_));
74 : }
75 :
76 184 : innerRanks_ = innerInstance->GetRankIds();
77 184 : }
78 :
79 24 : void RankGraph::InitFinish() { initFlag_ = true; }
80 :
81 2015 : bool RankGraph::HasRank(RankId rankId) const
82 : {
83 2015 : if (peers_.find(rankId) == peers_.end()) {
84 6 : HCCL_DEBUG("[RankGraph][HasRank] rankId[%d] is not existed", rankId);
85 6 : return false;
86 : }
87 2009 : return true;
88 : }
89 :
90 502 : u32 RankGraph::GetRankSize() const { return peers_.size(); }
91 :
92 7 : u32 RankGraph::GetInnerRankSize() const { return innerRanks_.size(); }
93 :
94 75 : RankId RankGraph::GetMyRank() const { return myRank_; }
95 :
96 532 : LocalId RankGraph::GetLocalId(RankId rankId) const
97 : {
98 532 : if (!HasRank(rankId)) {
99 0 : THROW<InvalidParamsException>(StringFormat("[RankGraph][GetLocalId] rankId[%d] is not existed.", rankId));
100 : }
101 :
102 532 : return peers_.at(rankId)->GetLocalId();
103 : }
104 :
105 10 : LocalId RankGraph::GetReplacedLocalId(RankId rankId) const
106 : {
107 10 : if (!HasRank(rankId)) {
108 0 : THROW<InvalidParamsException>(StringFormat("[RankGraph][GetLocalId] rankId[%d] is not existed.", rankId));
109 : }
110 :
111 10 : return peers_.at(rankId)->GetReplacedLocalId();
112 : }
113 :
114 89 : set<u32> RankGraph::GetLevels(RankId rankId) const
115 : {
116 89 : if (!HasRank(rankId)) {
117 0 : THROW<InvalidParamsException>(StringFormat("[RankGraph][GetLevels] rankId[%d] is not existed.", rankId));
118 : }
119 :
120 89 : return peers_.at(rankId)->GetLevels();
121 : }
122 :
123 2 : u32 RankGraph::GetLevelNum() const
124 : {
125 2 : u32 validLevelNum{0};
126 18 : for (const auto& netInst : netInsts_) {
127 16 : if (!netInst.empty()) {
128 2 : validLevelNum++;
129 : }
130 : }
131 4 : HCCL_INFO("[RankGraph][%s] validLevelNum[%u]", __func__, validLevelNum);
132 2 : return validLevelNum;
133 : }
134 :
135 8 : const NetInstance* RankGraph::GetNetInstanceByNetInstId(u32 netLayer, const string& netInstId) const
136 : {
137 : // 不存在netInst, 则返回空
138 8 : if (netLayer >= netInsts_.size() || netInsts_.at(netLayer).count(netInstId) == 0) {
139 3 : HCCL_WARNING(
140 : "[RankGraph][GetNetInstance] NetInstance netLayer[%u] netInstId[%s] is not existed.", netLayer,
141 : netInstId.c_str());
142 3 : return nullptr;
143 : }
144 5 : return netInsts_.at(netLayer).at(netInstId).get();
145 : }
146 :
147 5 : NetInstance* RankGraph::GetNetInstanceByNetInstId(u32 netLayer, const std::string& netInstId)
148 : {
149 : // 不存在netInst, 则返回空
150 5 : if (netLayer >= netInsts_.size() || netInsts_.at(netLayer).count(netInstId) == 0) {
151 3 : HCCL_WARNING(
152 : "[RankGraph][GetNetInstance] NetInstance netLayer[%u] netInstId[%s] is not existed.", netLayer,
153 : netInstId.c_str());
154 3 : return nullptr;
155 : }
156 2 : return netInsts_.at(netLayer).at(netInstId).get();
157 : }
158 :
159 154 : const NetInstance* RankGraph::GetNetInstanceByRankId(u32 netLayer, RankId rankId) const
160 : {
161 : // 不存在netInst, 则返回空
162 154 : if (!HasRank(rankId)) {
163 1 : HCCL_WARNING(
164 : "[RankGraph][GetNetInstance] NetInstance rankId[%d] netLayer[%u] is not existed.", rankId, netLayer);
165 1 : return nullptr;
166 : }
167 153 : return peers_.at(rankId)->GetNetInstance(netLayer);
168 : }
169 :
170 279 : NetInstance* RankGraph::GetNetInstanceByRankId(u32 netLayer, RankId rankId)
171 : {
172 279 : if (!HasRank(rankId)) {
173 1 : HCCL_WARNING(
174 : "[RankGraph][GetNetInstance] NetInstance rankId[%d] netLayer[%u] is not existed.", rankId, netLayer);
175 1 : return nullptr;
176 : }
177 278 : const NetInstance* constInstance = peers_.at(rankId)->GetNetInstance(netLayer);
178 278 : if (constInstance == nullptr) {
179 0 : THROW<NullPtrException>(StringFormat("[RankGraph][GetGroup]GetNetInstance(rankId, netLayer) is nullptr"));
180 : }
181 278 : string netInstId = constInstance->GetNetInstId();
182 278 : return netInsts_.at(netLayer).at(netInstId).get();
183 278 : }
184 :
185 507 : const shared_ptr<NetInstance::Peer> RankGraph::GetPeer(RankId rankId) const
186 : {
187 507 : if (!HasRank(rankId)) {
188 2 : HCCL_WARNING("[RankGraph][GetPeer] rankId[%d] is not existed.", rankId);
189 2 : return nullptr;
190 : }
191 :
192 505 : return peers_.at(rankId);
193 : }
194 :
195 1885 : vector<NetInstance::Path> RankGraph::GetPaths(u32 netLayer, RankId sRankId, RankId dRankId) const
196 : {
197 : // 若sRank和dRank均不在innerGroup里，则返回空
198 1885 : if (innerRanks_.count(sRankId) == 0 && innerRanks_.count(dRankId) == 0) {
199 5430 : HCCL_WARNING("[RankGraph][GetPaths] sRankId[%d] and dRankId[%d] are not in innerRanks", sRankId, dRankId);
200 1810 : return {};
201 : }
202 :
203 75 : vector<NetInstance::Path> paths;
204 75 : auto netInst = GetNetInstanceByRankId(netLayer, sRankId);
205 75 : if (netInst == nullptr) {
206 52 : HCCL_WARNING("[RankGraph][GetPaths] netLayer[%u] sRankId[%d] netInst is not existed.", netLayer, sRankId);
207 18 : return {};
208 : }
209 57 : if (!netInst->HasNode(NetInstance::Peer::GenerateNodeId(dRankId))) {
210 1 : HCCL_WARNING(
211 : "[RankGraph][GetPaths] netLayer[%u] sRankId[%d] netInst has no dRankId[%d].", netLayer, sRankId, dRankId);
212 1 : return {};
213 : }
214 :
215 56 : paths = netInst->GetPaths(sRankId, dRankId);
216 56 : if (paths.size() == 0) {
217 12 : HCCL_WARNING(
218 : "[RankGraph][GetPaths] netLayer[%u] sRankId[%d] dRankId[%d] netInst has no path.", netLayer, sRankId,
219 : dRankId);
220 4 : return paths;
221 : }
222 :
223 144 : HCCL_DEBUG(
224 : "[RankGraph][GetPaths] netLayer[%u] sRankId[%d] dRankId[%d] pathsize[%u].", netLayer, sRankId, dRankId,
225 : paths.size());
226 52 : return paths;
227 75 : }
228 :
229 3 : u32 RankGraph::GetLayerRanks(const u32 netLayer) const
230 : {
231 3 : u32 layerRankSize = 0;
232 3 : if (netInsts_.at(netLayer).size() == 0) {
233 0 : HCCL_WARNING("[RankGraph][GetLayerRanks] RankGraph has no netInstance on netLayer %u", netLayer);
234 0 : return 0;
235 : }
236 8 : for (const auto& netInst : netInsts_.at(netLayer)) {
237 5 : layerRankSize += netInst.second->GetRankSize();
238 : }
239 3 : return layerRankSize;
240 : }
241 :
242 6 : void RankGraph::GetLocalInstRanks(const u32 netLayer, vector<u32>& rankList, u32& rankNum) const
243 : {
244 6 : const NetInstance* netInstance = GetNetInstanceByRankId(netLayer, myRank_);
245 6 : if (netInstance == nullptr) {
246 1 : THROW<NullPtrException>(
247 3 : StringFormat("[RankGraph][GetLocalInstRanks] myRank %u has no netInstance on layer %u", myRank_, netLayer));
248 : }
249 5 : set<RankId> rankSet = netInstance->GetRankIds();
250 5 : rankList.clear();
251 15 : for (const RankId& rank : rankSet) {
252 10 : rankList.push_back(static_cast<u32>(rank));
253 : }
254 5 : rankNum = rankSet.size();
255 5 : }
256 :
257 7 : u32 RankGraph::GetLocalInstSize(const u32 netLayer) const
258 : {
259 7 : const NetInstance* netInstance = GetNetInstanceByRankId(netLayer, myRank_);
260 7 : if (netInstance == nullptr) {
261 0 : THROW<NullPtrException>(
262 0 : StringFormat("[RankGraph][GetLocalInstSize] myRank %u has no netInstance on layer %u", myRank_, netLayer));
263 : }
264 7 : return netInstance->GetRankSize();
265 : }
266 :
267 26 : const NetType RankGraph::GetNetType(const u32 netLayer) const
268 : {
269 26 : const NetInstance* netInstance = GetNetInstanceByRankId(netLayer, myRank_);
270 26 : if (netInstance == nullptr) {
271 4 : THROW<NullPtrException>(
272 8 : StringFormat("[RankGraph][GetLocalInstSize] myRank %u has no netInstance on layer %u", myRank_, netLayer));
273 : }
274 22 : return netInstance->GetNetType();
275 : }
276 :
277 6 : HcclResult RankGraph::GetNetInstanceList(const u32 netLayer, vector<u32>& instSizeList, u32& listSize) const
278 : {
279 6 : instSizeList.clear();
280 6 : listSize = 0;
281 6 : if (netInsts_.at(netLayer).size() == 0) {
282 3 : HCCL_WARNING("[RankGraph][GetNetInstanceList] RankGraph has no net instance on layer %u", netLayer);
283 1 : return HCCL_E_PARA;
284 : }
285 12 : for (const auto& netInst : netInsts_.at(netLayer)) {
286 7 : instSizeList.push_back(netInst.second->GetRankSize());
287 : }
288 5 : listSize = instSizeList.size();
289 5 : return HCCL_SUCCESS;
290 : }
291 :
292 2 : void RankGraph::GetTopoInstsByLayer(const u32 netLayer, std::vector<u32>& topoInsts, u32& topoInstNum) const
293 : {
294 2 : auto* netInstance = GetNetInstanceByRankId(netLayer, myRank_);
295 2 : netInstance->GetTopoInstsByLayer(topoInsts, topoInstNum);
296 2 : }
297 :
298 3 : HcclResult RankGraph::GetTopoType(const u32 netLayer, const u32 topoInstId, TopoType& topoType) const
299 : {
300 3 : auto* netInstance = GetNetInstanceByRankId(netLayer, myRank_);
301 3 : CHK_PRT_RET(
302 : netInstance == nullptr,
303 : HCCL_ERROR(
304 : "[RankGraph::GetTopoType] netInstance is nullptr, myRank[%d], netLayer[%u], "
305 : "topoInstId[%u]",
306 : myRank_, netLayer, topoInstId),
307 : HCCL_E_PTR);
308 :
309 3 : auto ret = netInstance->GetTopoType(topoInstId, topoType);
310 3 : CHK_PRT_RET(
311 : ret != HCCL_SUCCESS,
312 : HCCL_ERROR(
313 : "[%s] Failed to GetTopoType, myRank[%d], netLayer[%u], netInstId[%s], topoInstId[%u], "
314 : "ret[%d]",
315 : __func__, myRank_, netLayer, netInstance->GetNetInstId().c_str(), topoInstId, ret),
316 : ret);
317 2 : return HCCL_SUCCESS;
318 : }
319 :
320 : HcclResult
321 3 : RankGraph::GetRanksByTopoInst(const u32 netLayer, const u32 topoInstId, std::vector<u32>& ranks, u32& rankNum) const
322 : {
323 3 : auto* netInstance = GetNetInstanceByRankId(netLayer, myRank_);
324 3 : CHK_PRT_RET(
325 : netInstance == nullptr,
326 : HCCL_ERROR(
327 : "[RankGraph::GetRanksByTopoInst] netInstance is nullptr, myRank[%d], netLayer[%u], "
328 : "topoInstId[%u]",
329 : myRank_, netLayer, topoInstId),
330 : HCCL_E_PTR);
331 :
332 3 : auto ret = netInstance->GetRanksByTopoInst(topoInstId, ranks, rankNum);
333 3 : CHK_PRT_RET(
334 : ret != HCCL_SUCCESS,
335 : HCCL_ERROR(
336 : "[%s] Failed to GetRanksByTopoInst, myRank[%d], netLayer[%u], netInstId[%s], "
337 : "topoInstId[%u], ret[%d]",
338 : __func__, myRank_, netLayer, netInstance->GetNetInstId().c_str(), topoInstId, ret),
339 : ret);
340 2 : return HCCL_SUCCESS;
341 : }
342 :
343 8 : HcclResult RankGraph::GetEndpointNum(uint32_t layer, uint32_t topoInstId, uint32_t* num) const
344 : {
345 8 : auto peer = GetPeer(myRank_);
346 8 : if (peer == nullptr) {
347 1 : HCCL_ERROR("[RankGraph::GetEndpointNum] Peer is nullptr at netLayer [%u]", layer);
348 1 : return HCCL_E_PTR;
349 : }
350 7 : auto ifacesVec = peer->GetIfacesByLayer(layer);
351 7 : uint32_t sum = 0;
352 20 : for (auto& iface : ifacesVec) {
353 13 : if (iface->GetTopoInstId() == topoInstId) {
354 11 : sum += iface->GetLinkProtocols().size();
355 : }
356 : }
357 7 : *num = sum;
358 7 : return HCCL_SUCCESS;
359 8 : }
360 :
361 89 : HcclResult GetCommAddr(CommAddr& commAddr, const IpAddress& ipAddr)
362 : {
363 89 : s32 family = ipAddr.GetFamily();
364 89 : if (family == AF_INET) {
365 89 : string addr = ipAddr.GetIpStr();
366 89 : if (ipAddr.IsEID(addr)) {
367 0 : commAddr.type = COMM_ADDR_TYPE_EID;
368 0 : const auto& eid = ipAddr.GetEid();
369 0 : for (u32 i = 0; i < URMA_EID_LEN && i < sizeof(commAddr.eid); i++) {
370 0 : commAddr.eid[i] = eid.raw[i];
371 : }
372 : } else {
373 89 : commAddr.type = COMM_ADDR_TYPE_IP_V4;
374 89 : commAddr.addr = ipAddr.GetBinaryAddress().addr;
375 : }
376 89 : } else if (family == AF_INET6) {
377 0 : commAddr.type = COMM_ADDR_TYPE_IP_V6;
378 0 : commAddr.addr6 = ipAddr.GetBinaryAddress().addr6;
379 : } else {
380 0 : HCCL_ERROR("invalid commAddrType");
381 0 : return HCCL_E_INTERNAL;
382 : }
383 89 : return HCCL_SUCCESS;
384 : }
385 :
386 65 : EndpointLocType AddrPositionToEndpointLoc(AddrPosition pos)
387 : {
388 65 : switch (pos) {
389 5 : case AddrPosition::HOST:
390 5 : return ENDPOINT_LOC_TYPE_HOST;
391 60 : case AddrPosition::DEVICE:
392 60 : return ENDPOINT_LOC_TYPE_DEVICE;
393 0 : default:
394 0 : return ENDPOINT_LOC_TYPE_RESERVED;
395 : }
396 : }
397 :
398 : HcclResult
399 6 : RankGraph::GetEndpointDesc(uint32_t layer, uint32_t topoInstId, uint32_t* descNum, EndpointDesc* endpointDesc) const
400 : {
401 6 : auto peer = GetPeer(myRank_);
402 6 : CHK_PTR_NULL(peer);
403 :
404 5 : auto ifacesVec = peer->GetIfacesByLayer(layer);
405 5 : uint32_t count = 0;
406 :
407 : // 找到 topoInstId 匹配的 iface
408 15 : for (const auto& iface : ifacesVec) {
409 10 : if (iface->GetTopoInstId() != topoInstId) {
410 2 : continue;
411 : }
412 :
413 : // 对该 iface，从 endpointToIfaceMap_ 中找出所有匹配的 EndpointDesc
414 : // 一个 iface 可能对应多个 protocol（即多个 EndpointDesc）
415 8 : const auto& endpointMap = peer->GetEndpointToIfaceMap();
416 24 : for (const auto& entry : endpointMap) {
417 16 : std::pair<CommAddr, CommProtocol> endpoint = entry.first;
418 16 : const std::shared_ptr<NetInstance::ConnInterface>& mappedIface = entry.second;
419 :
420 16 : if (mappedIface != iface) {
421 10 : continue;
422 : }
423 :
424 : // 检查输出缓冲区是否足够
425 6 : if (count >= *descNum) {
426 0 : HCCL_ERROR(
427 : "[RankGraph::GetEndpointDesc] endpointDesc array too small: "
428 : "need %u, given %u",
429 : count + 1, *descNum);
430 0 : return HCCL_E_PARA;
431 : }
432 :
433 6 : endpointDesc[count].commAddr = endpoint.first;
434 6 : endpointDesc[count].protocol = endpoint.second;
435 6 : endpointDesc[count].loc.locType = AddrPositionToEndpointLoc(iface->GetPos());
436 :
437 18 : HCCL_INFO(
438 : "[RankGraph::GetEndpointDesc] local type is %d, protocol %d", endpointDesc[count].loc.locType,
439 : endpointDesc[count].protocol);
440 6 : count++;
441 : }
442 8 : }
443 :
444 5 : *descNum = count;
445 5 : return HCCL_SUCCESS;
446 6 : }
447 :
448 3 : HcclResult RankGraph::GetEndpointInfo(
449 : uint32_t rankId, const EndpointDesc* endpointDesc, EndpointAttr endpointAttr, uint32_t infoLen, void* info) const
450 : {
451 3 : if (endpointDesc == nullptr || info == nullptr) {
452 3 : HCCL_ERROR("[GetEndpointInfo] Invalid parameter");
453 1 : return HCCL_E_PTR;
454 : }
455 2 : const std::shared_ptr<NetInstance::Peer> peer = GetPeer(rankId);
456 2 : if (peer == nullptr) {
457 0 : HCCL_ERROR("[RankGraph::GetEndpointInfo] Peer is nullptr for rankId [%u]", rankId);
458 0 : return HCCL_E_PTR;
459 : }
460 :
461 : // 查找接口
462 2 : auto key = std::make_pair(endpointDesc->commAddr, endpointDesc->protocol);
463 2 : const auto& endpointToIfaceMap = peer->GetEndpointToIfaceMap();
464 2 : auto it = endpointToIfaceMap.find(key);
465 2 : if (it == endpointToIfaceMap.end()) {
466 0 : HCCL_ERROR("[GetEndpointInfo] No matching interface found");
467 0 : return HCCL_E_NOT_FOUND;
468 : }
469 2 : const auto& iface = it->second;
470 : // 填充信息
471 2 : switch (endpointAttr) {
472 2 : case ENDPOINT_ATTR_BW_COEFF: {
473 2 : if (infoLen != sizeof(EndpointAttrBwCoeff)) {
474 0 : HCCL_ERROR(
475 : "[GetEndpointInfo] Size mismatch: expected %zu, actual %u", sizeof(EndpointAttrBwCoeff), infoLen);
476 0 : return HCCL_E_PARA;
477 : }
478 2 : *(static_cast<EndpointAttrBwCoeff*>(info)) = iface->GetPorts().size();
479 2 : break;
480 : }
481 0 : case ENDPOINT_ATTR_DIE_ID: {
482 0 : if (infoLen != sizeof(EndpointAttrDieId)) {
483 0 : HCCL_ERROR(
484 : "[GetEndpointInfo] Size mismatch: expected %zu, actual %u", sizeof(EndpointAttrDieId), infoLen);
485 0 : return HCCL_E_PARA;
486 : }
487 0 : *(static_cast<EndpointAttrDieId*>(info)) = iface->GetLocalDieId();
488 0 : HCCL_INFO("GetEndpointInfo rankId[%u] iface[%s]", rankId, iface->Describe().c_str());
489 0 : break;
490 : }
491 0 : case ENDPOINT_ATTR_LOCATION: {
492 0 : if (infoLen != sizeof(EndpointAttrLocation)) {
493 0 : HCCL_ERROR(
494 : "[GetEndpointInfo] Size mismatch: expected %zu, actual %u", sizeof(EndpointAttrLocation), infoLen);
495 0 : return HCCL_E_PARA;
496 : }
497 0 : *(static_cast<EndpointAttrLocation*>(info)) = iface->GetPos();
498 0 : break;
499 : }
500 0 : default: {
501 0 : HCCL_ERROR("[GetEndpointInfo] Invalid endpointAttr [%d]", endpointAttr);
502 0 : return HCCL_E_PARA;
503 : }
504 : }
505 2 : return HCCL_SUCCESS;
506 2 : }
507 :
508 4 : bool RankGraph::IsSymmetric(const u32 netLayer) const
509 : {
510 4 : if (netInsts_.at(netLayer).size() == 0) {
511 1 : THROW<NullPtrException>(
512 3 : StringFormat("[RankGraph][IsSymmetric] RankGraph no netInstance on net layer %u", netLayer));
513 : }
514 3 : std::unordered_set<u32> rankSize;
515 8 : for (const auto& netInst : netInsts_.at(netLayer)) {
516 5 : rankSize.insert(netInst.second->GetRankSize());
517 : }
518 6 : return rankSize.size() == 1;
519 3 : }
520 :
521 106 : std::shared_ptr<NetInstance> GetOrCreateNetInstance(
522 : u32 netLayer, const string& netInstId, NetType type, Level2Id2NetInst& netInsts, RankGraph* rankGraph)
523 : {
524 106 : std::shared_ptr<NetInstance> netInstance;
525 :
526 : // 若netLayer和netInstId对应netInstance没创建则创建
527 106 : if (netInsts[netLayer].count(netInstId) == 0) {
528 35 : if (type == NetType::TOPO_FILE_DESC) {
529 20 : netInstance = std::make_shared<InnerNetInstance>(netLayer, netInstId);
530 15 : } else if (type == NetType::CLOS) {
531 15 : netInstance = std::make_shared<ClosNetInstance>(netLayer, netInstId);
532 : }
533 35 : netInsts[netLayer][netInstId] = netInstance;
534 : // netInstance添加到virtualTopo
535 35 : rankGraph->AddNetInstance(netInstance);
536 :
537 83 : HCCL_DEBUG(
538 : "[CreateNetInstance] create netInstance success, netLayer[%u] netInstId[%s] type[%s].", netLayer,
539 : netInstId.c_str(), type.Describe().c_str());
540 : } else {
541 : // 若netInstance存在, type不一致则报错
542 71 : NetType curType = netInsts[netLayer][netInstId]->GetNetType();
543 71 : if (curType != type) {
544 0 : HCCL_WARNING(
545 : "[CreateNetInstance]FabType [%s] and [%s] no match", curType.Describe().c_str(),
546 : type.Describe().c_str());
547 0 : return nullptr;
548 : }
549 : // 若netInstance存在, type一致则直接获取
550 71 : netInstance = netInsts[netLayer][netInstId];
551 : }
552 106 : return netInstance;
553 106 : }
554 :
555 56 : void GetNewNodeInfo(
556 : u32 layer, RankId newRankId, const NetInstance::Link& oldLink, shared_ptr<NetInstance>& newNetInstance,
557 : RankId2PeerMap& tmpPeers, shared_ptr<NetInstance::Node>& newNode, shared_ptr<NetInstance::ConnInterface>& newIface,
558 : bool isSource)
559 : {
560 56 : shared_ptr<NetInstance::Node> oldNode;
561 56 : shared_ptr<NetInstance::ConnInterface> oldIface;
562 56 : if (isSource) {
563 28 : oldNode = oldLink.GetSourceNode();
564 28 : oldIface = oldLink.GetSourceIface();
565 : } else {
566 28 : oldNode = oldLink.GetTargetNode();
567 28 : oldIface = oldLink.GetTargetIface();
568 : }
569 :
570 56 : NetInstance::Node::NodeType type = oldNode->GetType();
571 56 : if (type == NetInstance::Node::NodeType::PEER) {
572 32 : newIface = oldIface;
573 32 : newNode = tmpPeers.at(newRankId);
574 32 : tmpPeers.at(newRankId)->AddConnInterface(layer, newIface);
575 24 : } else if (type == NetInstance::Node::NodeType::FABRIC) {
576 24 : newIface = nullptr;
577 24 : newNode = oldNode;
578 24 : if (!newNetInstance->HasNode(newNode->GetNodeId())) {
579 1 : newNetInstance->AddNode(newNode);
580 : }
581 : } else {
582 0 : THROW<NotSupportException>(StringFormat(
583 : "[CreateSubNetInstances][GetNewNodeInfo] newRankId[%d] oldLink Node isSource[%d] type[%s] "
584 : "is not supported.",
585 0 : newRankId, isSource, type.Describe().c_str()));
586 : }
587 56 : }
588 :
589 33 : bool NeedUpdateTopoInstForSubGraph(const NetInstance* oldNetInstance, u32 topoInstId, RankId parentMyRank)
590 : {
591 33 : auto topoInstIter = oldNetInstance->topoInsts_.find(topoInstId);
592 33 : if (topoInstIter == oldNetInstance->topoInsts_.end()) {
593 0 : HCCL_DEBUG(
594 : "[SubRankGraph][NeedUpdateTopoInstForSubGraph] skip topoInstId[%u], parentMyRank[%d], "
595 : "not found in parent netInstId[%s]",
596 : topoInstId, parentMyRank, oldNetInstance->GetNetInstId().c_str());
597 0 : return false;
598 : }
599 33 : if (topoInstIter->second == nullptr) {
600 1 : HCCL_WARNING(
601 : "[SubRankGraph][NeedUpdateTopoInstForSubGraph] skip topoInstId[%u], parentMyRank[%u], "
602 : "topoInst is null in parent netInstId[%s]",
603 : topoInstId, parentMyRank, oldNetInstance->GetNetInstId().c_str());
604 1 : return false;
605 : }
606 32 : return topoInstIter->second->ranks.count(parentMyRank) > 0;
607 : }
608 :
609 28 : void AddNewLink(
610 : u32 layer, const NetInstance::Link& oldLink, RankId srcNewRankId, RankId dstNewRankId,
611 : shared_ptr<NetInstance>& newNetInstance, RankId2PeerMap& tmpPeers, const NetInstance* oldNetInstance,
612 : RankId parentMyRank)
613 : {
614 : // 不添加绕路link
615 28 : if (oldLink.GetHop() > 1 && oldLink.GetType() != LinkType::PEER2NET) {
616 0 : return;
617 : }
618 :
619 28 : shared_ptr<NetInstance::ConnInterface> newSourceIface;
620 28 : shared_ptr<NetInstance::ConnInterface> newTargetIface;
621 28 : shared_ptr<NetInstance::Node> newSourceNode;
622 28 : shared_ptr<NetInstance::Node> newTargetNode;
623 : // oldLink有fabicNode需要先addFabricNode
624 : // SourceNode
625 28 : GetNewNodeInfo(layer, srcNewRankId, oldLink, newNetInstance, tmpPeers, newSourceNode, newSourceIface, true);
626 : // TargetNode
627 28 : GetNewNodeInfo(layer, dstNewRankId, oldLink, newNetInstance, tmpPeers, newTargetNode, newTargetIface, false);
628 : // link
629 : shared_ptr<NetInstance::Link> link = make_shared<NetInstance::Link>(
630 56 : newSourceNode, newTargetNode, newSourceIface, newTargetIface, oldLink.GetType(), oldLink.GetLinkProtocols(),
631 56 : oldLink.GetLinkDirection(), oldLink.GetHop());
632 :
633 28 : newNetInstance->AddLink(link);
634 28 : if (newSourceIface != nullptr) {
635 16 : u32 sourceTopoInstId = newSourceIface->GetTopoInstId();
636 16 : TopoType sourceTopoType = newSourceIface->GetTopoType();
637 16 : if (NeedUpdateTopoInstForSubGraph(oldNetInstance, sourceTopoInstId, parentMyRank)) {
638 14 : newNetInstance->UpdateTopoInst(sourceTopoInstId, sourceTopoType, srcNewRankId);
639 : }
640 : }
641 28 : if (newTargetIface != nullptr) {
642 16 : u32 targetTopoInstId = newTargetIface->GetTopoInstId();
643 16 : TopoType targetTopoType = newTargetIface->GetTopoType();
644 16 : if (NeedUpdateTopoInstForSubGraph(oldNetInstance, targetTopoInstId, parentMyRank)) {
645 14 : newNetInstance->UpdateTopoInst(targetTopoInstId, targetTopoType, dstNewRankId);
646 : }
647 : }
648 :
649 83 : for (const auto& pair : newNetInstance->topoInsts_) {
650 55 : uint32_t topoInstId = pair.first;
651 55 : if (pair.second == nullptr) {
652 0 : THROW<NullPtrException>(
653 0 : StringFormat("[SubRankGraph][AddNewLink] topoInstId %u has no TopoInst", topoInstId));
654 : }
655 55 : auto topoType = pair.second->topoType;
656 55 : if (UNLIKELY(HcclCheckLogLevel(DLOG_DEBUG))) {
657 55 : HCCL_DEBUG("[SubRankGraph] topoInstId[%u] topoType[%d]", topoInstId, topoType);
658 : }
659 : }
660 28 : HCCL_DEBUG(
661 : "[RankGraph][AddNewLink] srcNewRankId[%d] dstNewRankId[%d] newLink[%s]", srcNewRankId, dstNewRankId,
662 : link->Describe().c_str());
663 28 : }
664 :
665 5 : void AddGroupLinks(
666 : const vector<RankId>& rankIds, const NetInstance* oldNetInstance, shared_ptr<NetInstance>& newNetInstance,
667 : RankId2PeerMap& tmpPeers, RankId parentMyRank)
668 : {
669 5 : set<RankId> newRankIds = newNetInstance->GetRankIds();
670 5 : u32 layer = newNetInstance->GetNetLayer();
671 5 : if (oldNetInstance == nullptr) {
672 0 : THROW<NullPtrException>(StringFormat("[AddGroupLinks]oldNetInstance is nullptr"));
673 : }
674 5 : if (newRankIds.size() == 1) {
675 : // 子通信域单卡场景直接返回1DMESH
676 4 : RankId singleId = *newRankIds.begin();
677 4 : newNetInstance->UpdateTopoInst(0, TopoType::MESH_1D, singleId);
678 4 : return;
679 : }
680 5 : for (RankId srcRankId : newRankIds) {
681 20 : for (RankId dstRankId : newRankIds) {
682 16 : if (srcRankId == dstRankId) {
683 4 : continue;
684 : }
685 : // 对oldNetInstance中的每一条Link, 创建新的Link添加到newNetInstance
686 12 : vector<NetInstance::Path> oldPaths = oldNetInstance->GetPaths(rankIds[srcRankId], rankIds[dstRankId]);
687 28 : for (auto& oldPath : oldPaths) {
688 44 : for (auto& oldLink : oldPath.links) {
689 28 : AddNewLink(
690 : layer, oldLink, srcRankId, dstRankId, newNetInstance, tmpPeers, oldNetInstance, parentMyRank);
691 : }
692 : }
693 12 : }
694 : }
695 5 : }
696 :
697 3 : void RankGraph::AddSubPeers(const std::vector<RankId>& rankIds, RankGraph* subRankGraph, RankId2PeerMap& peers) const
698 : {
699 : // 遍历rankIds将索引作为子虚拟拓扑的rankId构造subPeer并添加到subRankGraph
700 3 : s32 rankSize = rankIds.size();
701 9 : for (RankId subRankId = 0; subRankId < rankSize; ++subRankId) {
702 6 : RankId rankId = rankIds[subRankId];
703 6 : shared_ptr<NetInstance::Peer> oldPeer = GetPeer(rankId);
704 6 : LocalId localId = oldPeer->GetLocalId();
705 6 : LocalId replacedLocalId = oldPeer->GetReplacedLocalId();
706 6 : DeviceId deviceId = oldPeer->GetDeviceId();
707 6 : u32 devicePort = oldPeer->GetDevicePort();
708 6 : u32 hostPort = oldPeer->GetHostPort();
709 : shared_ptr<NetInstance::Peer> subPeer
710 6 : = make_shared<NetInstance::Peer>(subRankId, localId, replacedLocalId, deviceId, devicePort, hostPort);
711 6 : subRankGraph->AddPeer(subPeer);
712 6 : peers.emplace(subRankId, subPeer);
713 6 : const auto& oldEndpointMap = oldPeer->GetEndpointToIfaceMap();
714 6 : for (const auto& entry : oldEndpointMap) {
715 0 : subPeer->SetEndpointToIface(entry.first.first, entry.first.second, entry.second);
716 0 : HCCL_DEBUG(
717 : "[SubRankGraph][AddSubPeers] endpointToIfaceMap: protocol[%d] for subRankId[%d]", entry.first.second,
718 : subRankId);
719 : }
720 6 : HCCL_DEBUG(
721 : "[RankGraph][AddSubPeers] oldRankId[%d] subPeer[%s] add success.", rankId, subPeer->Describe().c_str());
722 6 : }
723 3 : }
724 :
725 3 : RankId GetSubRankId(const vector<RankId>& rankIds, RankId rank)
726 : {
727 : RankId subRank;
728 :
729 : // rankIds中查找rank, 数组索引即为subMyRank
730 3 : auto iter = find(begin(rankIds), end(rankIds), rank);
731 3 : if (iter != end(rankIds)) {
732 3 : subRank = distance(begin(rankIds), iter);
733 : } else {
734 0 : THROW<InvalidParamsException>(StringFormat("[RankGraph][CreateSubVirtTopo] rankIds has no rank[%d].", rank));
735 : }
736 :
737 3 : HCCL_DEBUG("[GetSubRank] rank[%d] subRank[%d].", rank, subRank);
738 3 : return subRank;
739 : }
740 :
741 : /**
742 : * 1. 创建子NetInstance
743 : * 2. peer添加对应netInstance
744 : */
745 3 : void RankGraph::CreateSubNetInstances(
746 : const std::vector<RankId> rankIds, Level2Id2NetInst& subNetInstances, RankId2PeerMap& peers,
747 : RankGraph* subRankGraph) const
748 : {
749 : // 遍历rankIds, 获取每个rankId所在的oldNetInstance, 创建subNetInstance
750 3 : RankId rankSize = rankIds.size();
751 9 : for (RankId subRankId = 0; subRankId < rankSize; ++subRankId) {
752 6 : set<u32> curLevels = GetLevels(rankIds[subRankId]);
753 14 : for (u32 netLayer : curLevels) {
754 8 : const NetInstance* oldNetInstance = GetNetInstanceByRankId(netLayer, rankIds[subRankId]);
755 8 : if (oldNetInstance == nullptr) {
756 0 : THROW<NullPtrException>(StringFormat("[RankGraph][CreateSubNetInstances] oldNetInstance is nullptr"));
757 : }
758 : // 创建subNetInstance (根据oldNetInstance.netLayer,id,type)
759 8 : NetType netType = oldNetInstance->GetNetType();
760 8 : string netInstId = oldNetInstance->GetNetInstId();
761 : shared_ptr<NetInstance> subNetInstance
762 8 : = GetOrCreateNetInstance(netLayer, netInstId, netType, subNetInstances, subRankGraph);
763 8 : if (subNetInstance == nullptr) {
764 0 : THROW<NullPtrException>(StringFormat("[RankGraph][CreateSubNetInstances] subNetInstance is nullptr"));
765 : }
766 :
767 : // subNetInstance Add RankId and subPeer
768 8 : shared_ptr<NetInstance::Peer> subPeer = peers.at(subRankId);
769 8 : subNetInstance->AddRankId(subRankId);
770 8 : subNetInstance->AddNode(subPeer);
771 :
772 : // subPeer Add subNetInstance
773 8 : subPeer->AddNetInstance(subNetInstance);
774 8 : HCCL_DEBUG(
775 : "[RankGraph][CreateSubNetInstances] subNetInstance subRankId[%d] subType[%s] subNetInstId[%s]",
776 : subRankId, netType.Describe().c_str(), netInstId.c_str());
777 8 : }
778 6 : }
779 3 : }
780 :
781 3 : void RankGraph::AddSubLinks(
782 : const std::vector<RankId>& rankIds, RankId2PeerMap& peers, Level2Id2NetInst& subNetInsts, RankId parentMyRank) const
783 : {
784 : // 遍历subNetInstances，对每一个NetInstance插入Links
785 27 : for (u32 netLayer = 0; netLayer < subNetInsts.size(); ++netLayer) {
786 29 : for (auto& curNetInstance : subNetInsts[netLayer]) {
787 5 : const NetInstance* oldNetInstance = GetNetInstanceByNetInstId(netLayer, curNetInstance.first);
788 5 : AddGroupLinks(rankIds, oldNetInstance, curNetInstance.second, peers, parentMyRank);
789 : }
790 : }
791 3 : }
792 :
793 3 : unique_ptr<RankGraph> RankGraph::CreateSubRankGraph(const std::vector<u32>& rankIds) const
794 : {
795 : // 参数类型转换
796 3 : vector<RankId> subRankIds;
797 3 : for_each(rankIds.begin(), rankIds.end(), [&](u32 rankId) {
798 6 : subRankIds.emplace_back(static_cast<RankId>(rankId));
799 6 : });
800 :
801 : // 参数检查, 若rankIds中存在当前virtualTopo不存在的rankId, 抛异
802 9 : for (const auto rankId : subRankIds) {
803 6 : if (!HasRank(rankId)) {
804 0 : THROW<InvalidParamsException>(
805 0 : StringFormat("[RankGraph][CreateSubVirtTopo] rankId[%d] is not existed.", rankId));
806 : }
807 : }
808 :
809 : // step1: 创建subRankGraph
810 3 : RankId subMyRankId = GetSubRankId(subRankIds, myRank_);
811 3 : unique_ptr<RankGraph> subRankGraph = make_unique<RankGraph>(subMyRankId);
812 :
813 : // step2: subRankGraph添加subPeers
814 3 : RankId2PeerMap peers; // 保存Peer指针以便后续执行Add操作
815 3 : AddSubPeers(subRankIds, subRankGraph.get(), peers);
816 :
817 : // step3: 构造subNetInstances, NetInstance添加RankId和Peer, Peer添加NetInstance
818 3 : Level2Id2NetInst subNetInstances(MAX_NET_LAYER); // 保存NetInstance指针以便后续执行Add操作
819 3 : CreateSubNetInstances(subRankIds, subNetInstances, peers, subRankGraph.get());
820 :
821 : // step4: subNetInstances添加Links, Peer添加ConnIfaces
822 3 : AddSubLinks(subRankIds, peers, subNetInstances, myRank_);
823 :
824 : // step5: 设置innerRanks
825 3 : subRankGraph->InitInnerRanks();
826 : // step6: 构造完成
827 3 : subRankGraph->InitFinish();
828 :
829 3 : HCCL_INFO("[subRankGraph] Build success!");
830 3 : subRankGraph->Dump();
831 3 : return subRankGraph;
832 3 : }
833 :
834 5 : std::vector<char> RankGraph::GetPackedData(const std::vector<std::pair<u32, RankId>>& netLayerRankPairs) const
835 : {
836 5 : std::vector<u32> numVec;
837 5 : std::vector<LinkData> links;
838 5 : u32 netLayerRankPairsNum = netLayerRankPairs.size();
839 :
840 15 : HCCL_DEBUG("netLayerRankPairs Num=%u", netLayerRankPairsNum);
841 :
842 9 : for (const auto& it : netLayerRankPairs) {
843 4 : auto paths = GetPaths(it.first, myRank_, it.second);
844 4 : numVec.push_back(paths.size());
845 12 : HCCL_DEBUG("RankGraph::GetPackedData: netLayer=%u, srcRank=%u, dstRank=%u", it.first, myRank_, it.second);
846 6 : for (const auto& path : paths) {
847 2 : links.emplace_back(path);
848 6 : HCCL_DEBUG("RankGraph::GetPackedData: %s", links.back().Describe().c_str());
849 : }
850 4 : }
851 5 : if (links.empty()) {
852 12 : HCCL_WARNING("[RankGraph][GetPackedData]connected links is empty");
853 : }
854 :
855 5 : std::vector<char> result;
856 5 : BinaryStream binaryStream;
857 5 : u32 linkSize = links.size();
858 5 : binaryStream << netLayerRankPairsNum;
859 5 : binaryStream << linkSize;
860 15 : HCCL_DEBUG("netLayerRankPairsNum=%u, linkSize=%u", netLayerRankPairsNum, linkSize);
861 5 : u32 idx = 0;
862 9 : for (const auto& it : netLayerRankPairs) {
863 4 : binaryStream << it.first;
864 4 : binaryStream << it.second;
865 4 : binaryStream << numVec[idx];
866 12 : HCCL_DEBUG("netLayer=%u, RankId=%u, num=%u", it.first, it.second, numVec[idx]);
867 4 : idx++;
868 : }
869 :
870 7 : for (const auto& link : links) {
871 2 : binaryStream << link.GetUniqueId();
872 : }
873 5 : binaryStream.Dump(result);
874 :
875 5 : return result;
876 5 : }
877 :
878 58 : string RankIds2Str(const set<RankId>& rankIds)
879 : {
880 58 : stringstream ranks;
881 221 : for (auto it = rankIds.begin(); it != rankIds.end(); ++it) {
882 163 : if (it != rankIds.begin()) {
883 106 : ranks << ", ";
884 : }
885 163 : ranks << *it;
886 : }
887 116 : return ranks.str();
888 58 : }
889 :
890 22 : void RankGraph::Dump() const
891 : {
892 50 : HCCL_DEBUG("RankGraph Dump:");
893 50 : HCCL_DEBUG("myRank: %d", myRank_);
894 50 : HCCL_DEBUG("innerRanks: [%s]", RankIds2Str(innerRanks_).c_str());
895 50 : HCCL_DEBUG("peers:");
896 79 : for (const auto& peer : peers_) {
897 141 : HCCL_DEBUG("%s", peer.second->Describe().c_str());
898 : }
899 50 : HCCL_DEBUG("netInsts:");
900 198 : for (uint32_t i = 0; i < netInsts_.size(); ++i) {
901 212 : for (auto& netInst : netInsts_[i]) {
902 84 : HCCL_DEBUG("[netLayer=%u, netInstId=%s]", i, netInst.first.c_str());
903 84 : HCCL_DEBUG("%s", netInst.second->Describe().c_str());
904 84 : HCCL_DEBUG("rankIds: [%s]", RankIds2Str(netInst.second->GetRankIds()).c_str());
905 84 : HCCL_DEBUG("peers:");
906 143 : for (const auto& peerMapIt : netInst.second->GetPeers()) {
907 271 : HCCL_DEBUG("%s", peerMapIt.second->Describe().c_str());
908 : }
909 84 : HCCL_DEBUG("fabrics:");
910 84 : for (const auto& fabric : netInst.second->GetFabrics()) {
911 132 : HCCL_DEBUG("%s", fabric->Describe().c_str());
912 : }
913 36 : set<NodeId> nodeIds{};
914 84 : HCCL_DEBUG("Graph Nodes:");
915 36 : netInst.second->GetGraph().TraverseNode([&](shared_ptr<NetInstance::Node> node) {
916 155 : nodeIds.insert(node->GetNodeId());
917 403 : HCCL_DEBUG("%s", node->Describe().c_str());
918 155 : });
919 84 : HCCL_DEBUG("Graph Links:");
920 191 : for (NodeId nodeId : nodeIds) {
921 155 : netInst.second->GetGraph().TraverseEdge(nodeId, [&](shared_ptr<NetInstance::Link> link) {
922 796 : HCCL_DEBUG("%s", link->Describe().c_str());
923 296 : });
924 : }
925 36 : }
926 : }
927 22 : }
928 75 : CommProtocol LinkProtocolToCommProtocol(const LinkProtocol& linkProtocol)
929 : {
930 75 : constexpr std::pair<LinkProtocol, CommProtocol> protocolPairs[]
931 : = {{LinkProtocol::UB_CTP, COMM_PROTOCOL_UB_CTP}, {LinkProtocol::UB_TP, COMM_PROTOCOL_UBC_TP},
932 : {LinkProtocol::ROCE, COMM_PROTOCOL_ROCE}, {LinkProtocol::HCCS, COMM_PROTOCOL_HCCS},
933 : {LinkProtocol::PCIE, COMM_PROTOCOL_PCIE}, {LinkProtocol::UB_MEM, COMM_PROTOCOL_UB_MEM},
934 : {LinkProtocol::UBOE, COMM_PROTOCOL_UBOE}, {LinkProtocol::UB_RTP, COMM_PROTOCOL_UB_RTP}};
935 :
936 75 : for (const auto& p : protocolPairs) {
937 75 : if (p.first == linkProtocol) {
938 75 : return p.second;
939 : }
940 : }
941 :
942 0 : return COMM_PROTOCOL_RESERVED;
943 : }
944 : } // namespace Hccl
|
