Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include <algorithm>
12 :
13 : #include "../rank_graph_builder/updater_for_64_plus_1.h"
14 : #include "topo_common_types.h"
15 : #include "exception_util.h"
16 : #include "invalid_params_exception.h"
17 :
18 : namespace Hccl {
19 :
20 : using namespace std;
21 :
22 : namespace {
23 :
24 6 : bool HasLayer0Port(const shared_ptr<NetInstance::Peer>& peer, const set<string>& ports)
25 : {
26 6 : if (peer == nullptr || ports.empty()) {
27 0 : return false;
28 : }
29 6 : IpAddress addr;
30 6 : return any_of(ports.begin(), ports.end(), [&peer, &addr](const string& port) {
31 6 : return peer->TryGetLayer0Address(port, addr);
32 6 : });
33 : }
34 :
35 2 : bool HasLayer0Peer2PeerLink(
36 : const shared_ptr<Graph<PhyTopo::Node, PhyTopo::Link>>& phyTopoGraph, const shared_ptr<NetInstance::Peer>& peer,
37 : LocalId replacedLocalId)
38 : {
39 2 : if (phyTopoGraph == nullptr || peer == nullptr) {
40 0 : return false;
41 : }
42 2 : bool matched = false;
43 4 : phyTopoGraph->TraverseEdge(
44 : PhyTopo::Peer::GetId(peer->GetLocalId()), PhyTopo::Peer::GetId(replacedLocalId),
45 2 : [&peer, &matched](const shared_ptr<PhyTopo::Link>& link) {
46 2 : if (link == nullptr || link->GetType() != LinkType::PEER2PEER || link->GetSourceIFace() == nullptr) {
47 0 : return;
48 : }
49 2 : matched = matched || HasLayer0Port(peer, link->GetSourceIFace()->GetPorts());
50 : });
51 2 : return matched;
52 : }
53 :
54 : } // namespace
55 :
56 66 : void UpdaterFor64Plus1::SaveReplaceInfo(const NewRankInfo& rank)
57 : {
58 66 : if (rank.localId != BACKUP_LOCAL_ID) {
59 63 : return;
60 : }
61 3 : string netInstId = "";
62 5 : for (const auto& netLayerInfo : rank.rankLevelInfos) {
63 2 : if (netLayerInfo.netLayer == 0) {
64 2 : netInstId = netLayerInfo.netInstId;
65 : }
66 : }
67 3 : if (netInstId.empty()) {
68 2 : THROW<InvalidParamsException>(StringFormat(
69 : "[UpdaterFor64Plus1::Init] "
70 : "Replaced rank[%d] has empty R0Id",
71 1 : rank.rankId));
72 : }
73 2 : if (replaceInfo.find(netInstId) != replaceInfo.end()) {
74 0 : THROW<InvalidParamsException>(StringFormat(
75 : "[UpdaterFor64Plus1::Init] "
76 : "R0Group[%s] has more than one backups",
77 : netInstId.c_str()));
78 : }
79 2 : replaceInfo[netInstId] = make_pair(rank.localId, rank.replacedLocalId);
80 3 : }
81 :
82 26 : void UpdaterFor64Plus1::UpdateRankGraph(RankGraph* rankGraph, const RankTableInfo* rankTable) const
83 : {
84 26 : if (rankGraph == nullptr) {
85 0 : THROW<NullPtrException>(StringFormat("[UpdaterFor64Plus1][%s] rankGraph is nullptr", __func__));
86 : }
87 26 : if (rankTable == nullptr) {
88 0 : THROW<NullPtrException>(StringFormat("[UpdaterFor64Plus1][%s] rankTable is nullptr", __func__));
89 : }
90 27 : for (const auto& it : replaceInfo) {
91 1 : const auto& netInstId = it.first;
92 1 : s32 localId = it.second.first;
93 1 : s32 replacedLocalId = it.second.second;
94 1 : NetInstance* netInstance = rankGraph->GetNetInstanceByNetInstId(0, netInstId);
95 1 : if (netInstance == nullptr) {
96 0 : HCCL_WARNING(
97 : "[UpdaterFor64Plus1][%s] netInstance netlayer[0] netInstanceId[%s] not exist", __func__,
98 : netInstId.c_str());
99 0 : continue;
100 0 : }
101 1 : UpdateNetInstance(netInstance, localId, replacedLocalId, rankTable);
102 : }
103 26 : }
104 :
105 1 : void UpdaterFor64Plus1::UpdateNetInstance(
106 : NetInstance* netInstance, LocalId localId, LocalId replacedLocalId, const RankTableInfo* rankTable) const
107 : {
108 1 : if (netInstance == nullptr) {
109 0 : THROW<NullPtrException>(StringFormat("[UpdaterFor64Plus1][%s] netInstance is nullptr", __func__));
110 : }
111 1 : auto phyTopoGraph = PhyTopo::GetInstance()->GetTopoGraph();
112 1 : if (phyTopoGraph == nullptr) {
113 0 : THROW<NullPtrException>(StringFormat("[UpdaterFor64Plus1][%s] phyTopoGraph is nullptr", __func__));
114 : }
115 3 : HCCL_DEBUG(
116 : "[UpdaterFor64Plus1][%s] Updating NetInstance[%s]: localId[%u]->replacedId[%u]", __func__,
117 : netInstance->GetNetInstId().c_str(), localId, replacedLocalId);
118 :
119 : // 找到与故障D直连的D
120 1 : shared_ptr<NetInstance::Peer> backupPeer; // local[64]
121 1 : vector<shared_ptr<NetInstance::Peer>> backupLinkedPeers;
122 5 : for (const auto& it : netInstance->GetPeers()) {
123 4 : auto peer = it.second;
124 4 : if (peer->GetLocalId() == BACKUP_LOCAL_ID) {
125 1 : backupPeer = peer;
126 1 : continue;
127 : }
128 3 : if (!IsSameX(peer->GetLocalId(), replacedLocalId) && !IsSameY(peer->GetLocalId(), replacedLocalId)) {
129 1 : continue;
130 : }
131 2 : if (!HasLayer0Peer2PeerLink(phyTopoGraph, peer, replacedLocalId)) {
132 0 : continue;
133 : }
134 2 : backupLinkedPeers.emplace_back(peer);
135 4 : }
136 :
137 : // 添加与故障D直连的D到备份D的Link
138 3 : for (const auto& peer : backupLinkedPeers) {
139 2 : AddPeer2BackupLinks(peer, backupPeer, replacedLocalId, netInstance, rankTable);
140 : }
141 1 : }
142 :
143 2 : void UpdaterFor64Plus1::AddPeer2BackupLinks(
144 : shared_ptr<NetInstance::Peer> peer, shared_ptr<NetInstance::Peer> backupPeer, LocalId replacedLocalId,
145 : NetInstance* netInstance, [[maybe_unused]] const RankTableInfo* rankTable) const
146 : {
147 2 : auto phyTopoGraph = PhyTopo::GetInstance()->GetTopoGraph();
148 :
149 2 : auto idx = GetLinkIndex(peer->GetLocalId(), replacedLocalId);
150 2 : auto backupPlaneId = idx.first;
151 2 : auto backupLinkIdx = idx.second;
152 6 : HCCL_DEBUG(
153 : "[UpdaterFor64Plus1][%s] Peer{rankId[%d], localId[%u]} will use BackupPlane[%u] addr[%u]", __func__,
154 : peer->GetRankId(), peer->GetLocalId(), backupPlaneId, backupLinkIdx);
155 :
156 : // 先获取phyTopoGraph中备份面和备份D的连接，因为只有一个fabric，所以会获取到全量16条备份面和备份D的连接
157 : // Edges中可能包含多个连接，但是在phytopo中保存为一条连接，内部有多个连接的端口
158 : // 拿到物理边后，根据backupPlaneId匹配，选择对应的一条物理边
159 : std::shared_ptr<PhyTopo::Link> backD2PlaneEdges
160 2 : = GetPeer2PlaneEdges(backupPlaneId, backupPeer, phyTopoGraph, BACKUP_TO_PLANE_ADDR_NUM);
161 : std::shared_ptr<PhyTopo::Link> peer2PlaneEdges
162 2 : = GetPeer2PlaneEdges(backupPlaneId, peer, phyTopoGraph, 1, backD2PlaneEdges->GetLinkProtocols());
163 :
164 : // 取得端口列表，根据backupLinkIdx去选择对应的端口
165 2 : std::set<std::string> backD2PlanePorts = backD2PlaneEdges->GetSourceIFace()->GetPorts();
166 2 : std::set<std::string> peer2PlanePorts = peer2PlaneEdges->GetSourceIFace()->GetPorts();
167 :
168 : // 校验端口集合大小是否符合预期
169 2 : if (backD2PlanePorts.size() != BACKUP_TO_PLANE_ADDR_NUM) {
170 0 : THROW<InvalidParamsException>(
171 : "[UpdaterFor64Plus1][%s] Backup to BackupPlane[%u] port num error", __func__, backupPlaneId);
172 : }
173 2 : if (peer2PlanePorts.size() != 1) {
174 0 : THROW<InvalidParamsException>(
175 : "[UpdaterFor64Plus1][%s] Peer[%u] to BackupPlane[%u] port num error", __func__, peer->GetLocalId(),
176 : backupPlaneId);
177 : }
178 :
179 : // 从端口集合中取出对应逻辑位置的端口
180 2 : std::string backD2PlanePort = GetPortFromSet(backD2PlanePorts, backupLinkIdx);
181 2 : std::string peer2PlanePort = GetPortFromSet(peer2PlanePorts, 0);
182 :
183 : // 最终选中的端口必须能在 RankTable layer 0 中找到地址。
184 2 : IpAddress backD2PlaneAddr;
185 2 : IpAddress peer2PlaneAddr;
186 4 : if (backupPeer == nullptr || peer == nullptr || !backupPeer->TryGetLayer0Address(backD2PlanePort, backD2PlaneAddr)
187 4 : || !peer->TryGetLayer0Address(peer2PlanePort, peer2PlaneAddr)) {
188 0 : THROW<InvalidParamsException>(
189 0 : StringFormat("[UpdaterFor64Plus1][%s] selected port does not belong to layer0", __func__));
190 : }
191 :
192 : // 组装成NetInstance的conninterface，加入peer和backupPeer
193 2 : if (backD2PlaneEdges->GetSourceIFace() == nullptr || peer2PlaneEdges->GetSourceIFace() == nullptr) {
194 0 : THROW<InvalidParamsException>("[UpdaterFor64Plus1][%s] source ConnInterface is nullptr", __func__);
195 : }
196 2 : std::set<string> peer2PlanePortSet;
197 2 : peer2PlanePortSet.insert(peer2PlanePort);
198 2 : std::set<string> backD2PlanePortSet;
199 2 : backD2PlanePortSet.insert(backD2PlanePort);
200 :
201 : shared_ptr<NetInstance::ConnInterface> backupIface = make_shared<NetInstance::ConnInterface>(
202 4 : backD2PlaneAddr, backD2PlanePortSet, backD2PlaneEdges->GetSourceIFace()->GetPos(), LinkType::PEER2PEER,
203 6 : backD2PlaneEdges->GetLinkProtocols(), backD2PlaneEdges->GetTopoType(), backD2PlaneEdges->GetTopoInstId());
204 : shared_ptr<NetInstance::ConnInterface> peerIface = make_shared<NetInstance::ConnInterface>(
205 4 : peer2PlaneAddr, peer2PlanePortSet, peer2PlaneEdges->GetSourceIFace()->GetPos(), LinkType::PEER2PEER,
206 6 : peer2PlaneEdges->GetLinkProtocols(), backD2PlaneEdges->GetTopoType(), backD2PlaneEdges->GetTopoInstId());
207 :
208 2 : backupPeer->AddConnInterface(0, backupIface);
209 2 : peer->AddConnInterface(0, peerIface);
210 :
211 : shared_ptr<NetInstance::Link> peer2Backup = make_shared<NetInstance::Link>(
212 2 : peer, backupPeer, peerIface, backupIface, LinkType::PEER2PEER, backD2PlaneEdges->GetLinkProtocols());
213 2 : netInstance->AddLink(peer2Backup);
214 : shared_ptr<NetInstance::Link> backup2Peer = make_shared<NetInstance::Link>(
215 2 : backupPeer, peer, backupIface, peerIface, LinkType::PEER2PEER, backD2PlaneEdges->GetLinkProtocols());
216 2 : netInstance->AddLink(backup2Peer);
217 :
218 : // 删除备份 D 到所有 Fabric 的链路；Fabric 由 planeId 生成，不能用 topoInstId 推导节点 ID。
219 10 : for (const auto& fabric : netInstance->GetFabrics()) {
220 8 : if (fabric == nullptr) {
221 0 : continue;
222 : }
223 8 : netInstance->DeleteLink(backupPeer->GetNodeId(), fabric->GetNodeId());
224 : }
225 2 : }
226 :
227 4 : std::shared_ptr<PhyTopo::Link> UpdaterFor64Plus1::GetPeer2PlaneEdges(
228 : u32 backupPlaneId, shared_ptr<NetInstance::Peer> peer,
229 : std::shared_ptr<Graph<PhyTopo::Node, PhyTopo::Link>> phyTopoGraph, u32 expectedPortNum,
230 : const std::set<LinkProtocol>& expectedProtocols) const
231 : {
232 4 : std::shared_ptr<PhyTopo::Link> peer2PlaneEdges = nullptr;
233 : auto peer2AllPlaneEdges
234 4 : = phyTopoGraph->GetEdges(PhyTopo::Peer::GetId(peer->GetLocalId()), PhyTopo::Fabric::GetId());
235 4 : std::set<u32> backupPlaneIds;
236 20 : for (const auto& edge : peer2AllPlaneEdges) {
237 : // 按物理属性统计合法备份面，避免 RankTable 仅提供部分备份端口时漏计。
238 32 : if (edge == nullptr || edge->GetType() != LinkType::PEER2NET || edge->GetSourceIFace() == nullptr
239 32 : || edge->GetSourceIFace()->GetPorts().size() != expectedPortNum
240 32 : || (!expectedProtocols.empty() && edge->GetLinkProtocols() != expectedProtocols)
241 32 : || edge->GetTopoInstId() >= BACKUP_PLANE_NUM) {
242 0 : continue;
243 : }
244 16 : backupPlaneIds.insert(edge->GetTopoInstId());
245 : // 真正选中的边必须命中 RankTable layer 0 端口。
246 16 : if (edge->GetTopoInstId() == backupPlaneId && HasLayer0Port(peer, edge->GetSourceIFace()->GetPorts())) {
247 4 : if (peer2PlaneEdges != nullptr) {
248 0 : THROW<InvalidParamsException>(
249 : "[UpdaterFor64Plus1][%s] BackupPlane[%u] is ambiguous", __func__, backupPlaneId);
250 : }
251 4 : peer2PlaneEdges = edge;
252 : }
253 : }
254 4 : if (backupPlaneIds.size() != BACKUP_PLANE_NUM) {
255 0 : THROW<InvalidParamsException>("[UpdaterFor64Plus1][%s] BackupPlane num error", __func__);
256 : }
257 4 : if (peer2PlaneEdges == nullptr) {
258 0 : THROW<NullPtrException>(StringFormat("[UpdaterFor64Plus1][%s] peer2PlaneEdges is nullptr", __func__));
259 : }
260 4 : return peer2PlaneEdges;
261 4 : }
262 :
263 7 : std::string UpdaterFor64Plus1::GetPortFromSet(std::set<string>& ports, u32 linkIdx) const
264 : {
265 7 : std::string peer2PlanePort = "";
266 7 : if (linkIdx >= ports.size()) {
267 1 : THROW<InvalidParamsException>("[UpdaterFor64Plus1][%s] BackupPlane port num error", __func__);
268 : }
269 9 : for (auto& port : ports) {
270 9 : if (linkIdx == 0) {
271 6 : peer2PlanePort = port;
272 6 : break;
273 : }
274 3 : linkIdx--;
275 : }
276 6 : return peer2PlanePort;
277 1 : }
278 :
279 10 : bool UpdaterFor64Plus1::IsSameX(LocalId srcLocalId, LocalId dstLocalId) const
280 : {
281 : // 两个localId对应的D是否在同一个X轴
282 10 : return (srcLocalId / DEVICE_NUM_PER_AXIS) == (dstLocalId / DEVICE_NUM_PER_AXIS);
283 : }
284 :
285 6 : bool UpdaterFor64Plus1::IsSameY(LocalId srcLocalId, LocalId dstLocalId) const
286 : {
287 : // 两个localId对应的D是否在同一个Y轴
288 6 : return (srcLocalId % DEVICE_NUM_PER_AXIS) == (dstLocalId % DEVICE_NUM_PER_AXIS);
289 : }
290 :
291 9 : pair<u32, u32> UpdaterFor64Plus1::GetLinkIndex(LocalId localId, LocalId replacedLocalId) const
292 : {
293 9 : if (localId >= BACKUP_LOCAL_ID || replacedLocalId >= BACKUP_LOCAL_ID || localId == replacedLocalId) {
294 2 : THROW<InvalidParamsException>(
295 : "[UpdaterFor64Plus1][%s] localId[%u] or replacedId[%u] invalid", __func__, localId, replacedLocalId);
296 : return {};
297 : }
298 7 : if (IsSameX(localId, replacedLocalId)) {
299 3 : auto idx = localId % DEVICE_NUM_PER_AXIS;
300 3 : if (idx < DEVICE_HALF_NUM_PER_AXIS) {
301 2 : return make_pair(0, idx); // X轴左边4个正常D走备份面0
302 : } else {
303 1 : return make_pair(1, idx % DEVICE_HALF_NUM_PER_AXIS); // X轴右边4个正常D走备份面1
304 : }
305 4 : } else if (IsSameY(localId, replacedLocalId)) {
306 3 : auto idx = localId / DEVICE_NUM_PER_AXIS;
307 3 : if (idx < DEVICE_HALF_NUM_PER_AXIS) {
308 2 : return make_pair(2, idx); // Y轴上边4个正常D走备份面2
309 : } else {
310 1 : return make_pair(3, idx % DEVICE_HALF_NUM_PER_AXIS); // Y轴下边4个正常D走备份面3
311 : }
312 : } else {
313 1 : THROW<InvalidParamsException>(
314 : "[UpdaterFor64Plus1][%s] localId[%u] and replacedId[%u] are not at same line", __func__, localId,
315 : replacedLocalId);
316 : return {};
317 : }
318 : }
319 :
320 : } // namespace Hccl
|
