Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "../rank_graph_builder/updater_for_64_plus_1.h"
12 : #include "topo_common_types.h"
13 : #include "exception_util.h"
14 : #include "invalid_params_exception.h"
15 :
16 : namespace Hccl {
17 :
18 : using namespace std;
19 :
20 54 : void UpdaterFor64Plus1::SaveReplaceInfo(const NewRankInfo& rank)
21 : {
22 54 : if (rank.localId != BACKUP_LOCAL_ID) {
23 51 : return;
24 : }
25 3 : string netInstId = "";
26 5 : for (const auto& netLayerInfo : rank.rankLevelInfos) {
27 2 : if (netLayerInfo.netLayer == 0) {
28 2 : netInstId = netLayerInfo.netInstId;
29 : }
30 : }
31 3 : if (netInstId.empty()) {
32 2 : THROW<InvalidParamsException>(StringFormat("[UpdaterFor64Plus1::Init] "
33 1 : "Replaced rank[%d] has empty R0Id", rank.rankId));
34 : }
35 2 : if (replaceInfo.find(netInstId) != replaceInfo.end()) {
36 0 : THROW<InvalidParamsException>(StringFormat("[UpdaterFor64Plus1::Init] "
37 : "R0Group[%s] has more than one backups", netInstId.c_str()));
38 : }
39 2 : replaceInfo[netInstId] = make_pair(rank.localId, rank.replacedLocalId);
40 3 : }
41 :
42 19 : void UpdaterFor64Plus1::UpdateRankGraph(RankGraph* rankGraph, const RankTableInfo *rankTable) const
43 : {
44 19 : if (rankGraph == nullptr) {
45 0 : THROW<NullPtrException>(StringFormat("[UpdaterFor64Plus1][%s] rankGraph is nullptr", __func__));
46 : }
47 19 : if (rankTable == nullptr) {
48 0 : THROW<NullPtrException>(StringFormat("[UpdaterFor64Plus1][%s] rankTable is nullptr", __func__));
49 : }
50 20 : for (const auto& it : replaceInfo) {
51 1 : const auto& netInstId = it.first;
52 1 : s32 localId = it.second.first;
53 1 : s32 replacedLocalId = it.second.second;
54 1 : NetInstance* netInstance = rankGraph->GetNetInstanceByNetInstId(0, netInstId);
55 1 : if (netInstance == nullptr) {
56 0 : HCCL_WARNING("[UpdaterFor64Plus1][%s] netInstance netlayer[0] netInstanceId[%s] not exist", __func__, netInstId.c_str());
57 0 : continue;
58 0 : }
59 1 : UpdateNetInstance(netInstance, localId, replacedLocalId, rankTable);
60 : }
61 19 : }
62 :
63 1 : void UpdaterFor64Plus1::UpdateNetInstance(NetInstance* netInstance, LocalId localId, LocalId replacedLocalId, const RankTableInfo *rankTable) const
64 : {
65 1 : if (netInstance == nullptr) {
66 0 : THROW<NullPtrException>(StringFormat("[UpdaterFor64Plus1][%s] netInstance is nullptr", __func__));
67 : }
68 1 : auto phyTopoGraph = PhyTopo::GetInstance()->GetTopoGraph(0);
69 1 : if (phyTopoGraph == nullptr) {
70 0 : THROW<NullPtrException>(StringFormat("[UpdaterFor64Plus1][%s] phyTopoGraph is nullptr", __func__));
71 : }
72 3 : HCCL_DEBUG("[UpdaterFor64Plus1][%s] Updating NetInstance[%s]: localId[%u]->replacedId[%u]",
73 : __func__, netInstance->GetNetInstId().c_str(), localId, replacedLocalId);
74 :
75 : // 找到与故障D直连的D
76 1 : shared_ptr<NetInstance::Peer> backupPeer; // local[64]
77 1 : vector<shared_ptr<NetInstance::Peer>> backupLinkedPeers;
78 5 : for (const auto& it : netInstance->GetPeers()) {
79 4 : auto peer = it.second;
80 4 : if (peer->GetLocalId() == BACKUP_LOCAL_ID) {
81 1 : backupPeer = peer;
82 1 : continue;
83 : }
84 3 : if (!IsSameX(peer->GetLocalId(), replacedLocalId) && !IsSameY(peer->GetLocalId(), replacedLocalId)) {
85 1 : continue;
86 : }
87 2 : if (!phyTopoGraph->HasEdge(PhyTopo::Peer::GetId(peer->GetLocalId()), PhyTopo::Peer::GetId(replacedLocalId))) {
88 0 : continue;
89 : }
90 2 : backupLinkedPeers.emplace_back(peer);
91 4 : }
92 :
93 : // 添加与故障D直连的D到备份D的Link
94 3 : for (const auto& peer : backupLinkedPeers) {
95 2 : AddPeer2BackupLinks(peer, backupPeer, replacedLocalId, netInstance, rankTable);
96 : }
97 1 : }
98 :
99 2 : void UpdaterFor64Plus1::AddPeer2BackupLinks(shared_ptr<NetInstance::Peer> peer,
100 : shared_ptr<NetInstance::Peer> backupPeer, LocalId replacedLocalId,
101 : NetInstance* netInstance, const RankTableInfo* rankTable) const
102 : {
103 2 : auto phyTopoGraph = PhyTopo::GetInstance()->GetTopoGraph(0);
104 :
105 2 : std::unordered_map<u64, u64> fabricIds;
106 : auto peer2AllPlaneEdges =
107 2 : phyTopoGraph->GetEdges(PhyTopo::Peer::GetId(backupPeer->GetLocalId()), PhyTopo::Fabric::GetId());
108 10 : for (auto edge : peer2AllPlaneEdges) {
109 8 : auto topoInstId = edge->GetTopoInstId();
110 8 : auto fabricId = static_cast<u64>(topoInstId) | (static_cast<u64>(1) << 32);
111 8 : fabricIds[topoInstId] = fabricId;
112 8 : }
113 :
114 2 : auto idx = GetLinkIndex(peer->GetLocalId(), replacedLocalId);
115 2 : auto backupPlaneId = idx.first;
116 2 : auto backupLinkIdx = idx.second;
117 6 : HCCL_DEBUG("[UpdaterFor64Plus1][%s] Peer{rankId[%d], localId[%u]} will use BackupPlane[%u] addr[%u]", __func__,
118 : peer->GetRankId(), peer->GetLocalId(), backupPlaneId, backupLinkIdx);
119 :
120 : // 先获取phyTopoGraph中备份面和备份D的连接，因为只有一个fabric，所以会获取到全量16条备份面和备份D的连接
121 : // Edges中可能包含多个连接，但是在phytopo中保存为一条连接，内部有多个连接的端口
122 : // 拿到物理边后，根据backupPlaneId匹配，选择对应的一条物理边
123 2 : std::shared_ptr<PhyTopo::Link> backD2PlaneEdges = GetPeer2PlaneEdges(backupPlaneId, backupPeer, phyTopoGraph);
124 2 : std::shared_ptr<PhyTopo::Link> peer2PlaneEdges = GetPeer2PlaneEdges(backupPlaneId, peer, phyTopoGraph);
125 :
126 : // 取得端口列表，根据backupLinkIdx去选择对应的端口
127 2 : std::set<std::string> backD2PlanePorts = backD2PlaneEdges->GetSourceIFace()->GetPorts();
128 2 : std::set<std::string> peer2PlanePorts = peer2PlaneEdges->GetSourceIFace()->GetPorts();
129 :
130 : // 校验端口集合大小是否符合预期
131 2 : if (backD2PlanePorts.size() != BACKUP_TO_PLANE_ADDR_NUM) {
132 0 : THROW<InvalidParamsException>("[UpdaterFor64Plus1][%s] Backup to BackupPlane[%u] port num error", __func__,
133 : backupPlaneId);
134 : }
135 2 : if (peer2PlanePorts.size() != 1) {
136 0 : THROW<InvalidParamsException>("[UpdaterFor64Plus1][%s] Peer[%u] to BackupPlane[%u] port num error", __func__,
137 : peer->GetLocalId(), backupPlaneId);
138 : }
139 :
140 : // 从端口集合中取出对应逻辑位置的端口
141 2 : std::string backD2PlanePort = GetPortFromSet(backD2PlanePorts, backupLinkIdx);
142 2 : std::string peer2PlanePort = GetPortFromSet(peer2PlanePorts, 0);
143 :
144 : // 匹配了对应的端口后，去ranktableinfo中查对应端口的地址信息
145 : // todo 加一下peer->GetPortAddrMapLayer0()是否能找到port对应的地址。
146 2 : IpAddress backD2PlaneAddr = backupPeer->GetPortAddrMapLayer0()[backD2PlanePort][0];
147 2 : IpAddress peer2PlaneAddr = peer->GetPortAddrMapLayer0()[peer2PlanePort][0];
148 :
149 : // 组装成NetInstance的conninterface，加入peer和backupPeer
150 2 : if (backD2PlaneEdges->GetSourceIFace() == nullptr || peer2PlaneEdges->GetSourceIFace() == nullptr) {
151 0 : THROW<InvalidParamsException>("[UpdaterFor64Plus1][%s] source ConnInterface is nullptr", __func__);
152 : }
153 2 : std::set<string> peer2PlanePortSet;
154 2 : peer2PlanePortSet.insert(peer2PlanePort);
155 2 : std::set<string> backD2PlanePortSet;
156 2 : backD2PlanePortSet.insert(backD2PlanePort);
157 :
158 : shared_ptr<NetInstance::ConnInterface> backupIface = make_shared<NetInstance::ConnInterface>(
159 4 : backD2PlaneAddr, backD2PlanePortSet, backD2PlaneEdges->GetSourceIFace()->GetPos(), LinkType::PEER2PEER,
160 6 : backD2PlaneEdges->GetLinkProtocols(), backD2PlaneEdges->GetTopoType(), backD2PlaneEdges->GetTopoInstId());
161 : shared_ptr<NetInstance::ConnInterface> peerIface = make_shared<NetInstance::ConnInterface>(
162 4 : peer2PlaneAddr, peer2PlanePortSet, peer2PlaneEdges->GetSourceIFace()->GetPos(), LinkType::PEER2PEER,
163 6 : peer2PlaneEdges->GetLinkProtocols(), backD2PlaneEdges->GetTopoType(), backD2PlaneEdges->GetTopoInstId());
164 :
165 2 : backupPeer->AddConnInterface(0, backupIface);
166 2 : peer->AddConnInterface(0, peerIface);
167 :
168 : shared_ptr<NetInstance::Link> peer2Backup = make_shared<NetInstance::Link>(
169 2 : peer, backupPeer, peerIface, backupIface, LinkType::PEER2PEER, backD2PlaneEdges->GetLinkProtocols());
170 2 : netInstance->AddLink(peer2Backup);
171 : shared_ptr<NetInstance::Link> backup2Peer = make_shared<NetInstance::Link>(
172 2 : backupPeer, peer, backupIface, peerIface, LinkType::PEER2PEER, backD2PlaneEdges->GetLinkProtocols());
173 2 : netInstance->AddLink(backup2Peer);
174 :
175 : // peer2peer建立后删除graph中DB到所选planeId的对应的peer2net链路
176 : // 直接删除备份d和fabric的链接保证GetLinks接口只能获取到peer2db的一条peer2peer
177 : // 删除peer到fabric用到的peer2net
178 10 : for (auto id : fabricIds) {
179 8 : netInstance->DeleteLink(backupPeer->GetNodeId(), id.second);
180 : }
181 2 : }
182 :
183 4 : std::shared_ptr<PhyTopo::Link> UpdaterFor64Plus1::GetPeer2PlaneEdges(u32 backupPlaneId, shared_ptr<NetInstance::Peer> peer,
184 : std::shared_ptr<Graph<PhyTopo::Node, PhyTopo::Link>> phyTopoGraph) const{
185 4 : std::shared_ptr<PhyTopo::Link> peer2PlaneEdges = nullptr;
186 : auto peer2AllPlaneEdges = phyTopoGraph->GetEdges(
187 4 : PhyTopo::Peer::GetId(peer->GetLocalId()), PhyTopo::Fabric::GetId());
188 4 : if (peer2AllPlaneEdges.size() != BACKUP_PLANE_NUM) {
189 0 : THROW<InvalidParamsException>("[UpdaterFor64Plus1][%s] BackupPlane num error",
190 : __func__);
191 : }
192 8 : for (auto &edges : peer2AllPlaneEdges) {
193 8 : if (edges->GetTopoInstId() == backupPlaneId) {
194 4 : peer2PlaneEdges = edges;
195 4 : break;
196 : }
197 : }
198 4 : if (peer2PlaneEdges == nullptr) {
199 0 : THROW<NullPtrException>(StringFormat("[UpdaterFor64Plus1][%s] peer2PlaneEdges is nullptr", __func__));
200 : }
201 4 : return peer2PlaneEdges;
202 4 : }
203 :
204 7 : std::string UpdaterFor64Plus1::GetPortFromSet(std::set<string> &ports, u32 linkIdx) const
205 : {
206 7 : std::string peer2PlanePort = "";
207 7 : if (linkIdx >= ports.size()) {
208 1 : THROW<InvalidParamsException>("[UpdaterFor64Plus1][%s] BackupPlane port num error",
209 : __func__);
210 : }
211 9 : for (auto &port : ports) {
212 9 : if (linkIdx == 0) {
213 6 : peer2PlanePort = port;
214 6 : break;
215 : }
216 3 : linkIdx--;
217 : }
218 6 : return peer2PlanePort;
219 1 : }
220 :
221 10 : bool UpdaterFor64Plus1::IsSameX(LocalId srcLocalId, LocalId dstLocalId) const
222 : {
223 : // 两个localId对应的D是否在同一个X轴
224 10 : return (srcLocalId / DEVICE_NUM_PER_AXIS) == (dstLocalId / DEVICE_NUM_PER_AXIS);
225 : }
226 :
227 6 : bool UpdaterFor64Plus1::IsSameY(LocalId srcLocalId, LocalId dstLocalId) const
228 : {
229 : // 两个localId对应的D是否在同一个Y轴
230 6 : return (srcLocalId % DEVICE_NUM_PER_AXIS) == (dstLocalId % DEVICE_NUM_PER_AXIS);
231 : }
232 :
233 9 : pair<u32, u32> UpdaterFor64Plus1::GetLinkIndex(LocalId localId, LocalId replacedLocalId) const
234 : {
235 9 : if (localId >= BACKUP_LOCAL_ID || replacedLocalId >= BACKUP_LOCAL_ID || localId == replacedLocalId) {
236 2 : THROW<InvalidParamsException>("[UpdaterFor64Plus1][%s] localId[%u] or replacedId[%u] invalid",
237 : __func__, localId, replacedLocalId);
238 : return {};
239 : }
240 7 : if (IsSameX(localId, replacedLocalId)) {
241 3 : auto idx = localId % DEVICE_NUM_PER_AXIS;
242 3 : if (idx < DEVICE_HALF_NUM_PER_AXIS) {
243 2 : return make_pair(0, idx); // X轴左边4个正常D走备份面0
244 : } else {
245 1 : return make_pair(1, idx % DEVICE_HALF_NUM_PER_AXIS); // X轴右边4个正常D走备份面1
246 : }
247 4 : } else if (IsSameY(localId, replacedLocalId)) {
248 3 : auto idx = localId / DEVICE_NUM_PER_AXIS;
249 3 : if (idx < DEVICE_HALF_NUM_PER_AXIS) {
250 2 : return make_pair(2, idx); // Y轴上边4个正常D走备份面2
251 : } else {
252 1 : return make_pair(3, idx % DEVICE_HALF_NUM_PER_AXIS); // Y轴下边4个正常D走备份面3
253 : }
254 : } else {
255 1 : THROW<InvalidParamsException>("[UpdaterFor64Plus1][%s] localId[%u] and replacedId[%u] are not at same line",
256 : __func__, localId, replacedLocalId);
257 : return {};
258 : }
259 : }
260 :
261 : } // namespace Hccl
|
