Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "topo_match_concurr_mesh_nhr.h"
12 :
13 : namespace Hccl {
14 0 : TopoMatchConcurrMeshNHR::TopoMatchConcurrMeshNHR(const RankId vRank, const u32 rankSize, const RankGraph *rankGraph,
15 0 : const DevType devType)
16 0 : : TopoMatchBase(vRank, rankSize, rankGraph, devType)
17 : {
18 0 : }
19 :
20 0 : TopoMatchConcurrMeshNHR::~TopoMatchConcurrMeshNHR()
21 : {
22 0 : }
23 :
24 0 : HcclResult TopoMatchConcurrMeshNHR::MatchTopo(std::vector<std::vector<std::vector<RankId>>> &vTopo,
25 : std::vector<std::vector<RankId>> &virtRanks,
26 : std::vector<std::map<RankId, u32>> &virtRankMap)
27 : {
28 : // 获取并校验当前通信层数
29 0 : std::set<u32> levelSet = rankGraph_->GetLevels(myRank_);
30 0 : CHK_PRT_RET((levelSet.size() < COMM_LEVEL_SIZE_2),
31 : HCCL_ERROR("[CollAlgFactory] [TopoMatchConcurrMeshNHR] Rank [%d], Invalid virtual topo. levelSet[%u]",
32 : myRank_, levelSet.size()),
33 : HcclResult::HCCL_E_PARA);
34 :
35 0 : rankGraph_->Dump();
36 :
37 : // 获取 level0 Pod Topo 信息
38 0 : const NetInstance* netInstance = rankGraph_->GetNetInstanceByRankId(0, myRank_);
39 0 : if (netInstance == nullptr) {
40 0 : HCCL_ERROR("TopoMatchConcurrMeshNHR::MatchTopo netInstance is nullptr");
41 0 : return HcclResult::HCCL_E_PTR;
42 : }
43 0 : const u32 rankSizeLevel0 = netInstance->GetRankSize();
44 0 : std::set<RankId> rankSetLevel0 = netInstance->GetRankIds();
45 :
46 : // 校验 level0 Pod Topo 是否符合 m x n 要求
47 0 : CHK_RET(CalcRankOnSamePlaneOfR0(rankOnSameBoardVector_, rankOnSameSlotVector_, numRanksPerBoard_));
48 0 : const auto minmaxPair = std::minmax_element(numRanksPerBoard_.begin(), numRanksPerBoard_.end());
49 0 : u32 minNumRankPerBoard = *minmaxPair.first;
50 0 : u32 maxNumRankPerBoard = *minmaxPair.second;
51 0 : CHK_PRT_RET((minNumRankPerBoard != maxNumRankPerBoard),
52 : HCCL_ERROR("[CollAlgFactory] [TopoMatchConcurrMeshNHR] Rank [%d], Invalid virtual topo for "
53 : "multi-dimensional concurrent mesh, min numRanksPerBoard_[%u], max numRanksPerBoard_[%u].",
54 : myRank_, minNumRankPerBoard, maxNumRankPerBoard),
55 : HcclResult::HCCL_E_PARA);
56 :
57 0 : CHK_PRT_RET(((rankSize_ == 1) || (numRanksPerBoard_[0] * numRanksPerBoard_.size() != rankSizeLevel0)),
58 : HCCL_ERROR("[CollAlgFactory] [TopoMatchConcurrMeshNHR] Rank [%d], Invalid virtual topo for "
59 : "multi-dimensional concurrent mesh algorithm with rankSize [%u], rankSizeLevel0 [%u], "
60 : "ranksPerBoard [%u], ranksPerSlot [%u].", myRank_, rankSize_, rankSizeLevel0,
61 : numRanksPerBoard_[0], numRanksPerBoard_.size()),
62 : HcclResult::HCCL_E_PARA);
63 :
64 : // 在全局视角下计算 level0 sub-communicator size
65 0 : std::vector<u32> instSizeList; // rank num on each pod
66 : u32 listSize; // pod num
67 0 : rankGraph_->GetNetInstanceList(0, instSizeList, listSize); // global view
68 0 : HCCL_DEBUG("[CollAlgFactory] [TopoMatchConcurrMeshNHR] Rank [%d], instSizeList[%u]=[%s]",
69 : myRank_, instSizeList.size(), PrintVector<u32>(instSizeList).c_str());
70 :
71 0 : if (!CheckSymmetric(instSizeList)) {
72 : // 非对称情形
73 0 : HCCL_DEBUG("[CollAlgFactory] [TopoMatchConcurrMeshNHR] Rank [%d], Asymmetric MatchTopo.", myRank_);
74 0 : u32 gcdInstSize = GcdMultiple(instSizeList);
75 :
76 : // 计算 level 0 子通信域
77 0 : CHK_RET(GenerateLevel0(rankSetLevel0, gcdInstSize, myRank_, vTopo, virtRanks));
78 :
79 : // 计算 level 1 子通信域
80 0 : const NetInstance* netInstanceL1 = rankGraph_->GetNetInstanceByRankId(1, myRank_);
81 0 : if (netInstanceL1 == nullptr) {
82 0 : HCCL_ERROR("TopoMatchConcurrMeshNHR::MatchTopo netInstanceL1 is nullptr");
83 0 : return HcclResult::HCCL_E_PTR;
84 : }
85 0 : std::set<RankId> rankSetLevel1 = netInstanceL1->GetRankIds();
86 :
87 0 : CHK_RET(GenerateLevel1(rankSetLevel1, gcdInstSize, myRank_, vTopo, virtRanks));
88 :
89 0 : HCCL_DEBUG("[CollAlgFactory] [TopoMatchConcurrMeshNHR] Rank [%d], level0 & level1 virtRanks=[%s] ",
90 : myRank_, PrintMatrix<RankId>(virtRanks).c_str());
91 0 : HCCL_DEBUG("[CollAlgFactory] [TopoMatchConcurrMeshNHR] Rank [%d], level0 & level1 vTopo=[%s] ",
92 : myRank_, PrintTensor<RankId>(vTopo).c_str());
93 0 : } else {
94 : // 对称情形
95 0 : HCCL_DEBUG("[CollAlgFactory] [TopoMatchConcurrMeshNHR] Rank [%d], Symmetric MatchTopo.", myRank_);
96 :
97 0 : u32 myLocalId = rankGraph_->GetLocalId(myRank_);
98 0 : rankOnSameBoard_ = rankOnSameBoardVector_[myLocalId / RANK_SIZE_EIGHT];
99 0 : rankOnSameSlot_ = rankOnSameSlotVector_[myLocalId % RANK_SIZE_EIGHT];
100 0 : if ((rankOnSameBoard_.size() == 1) || (rankOnSameSlot_.size() == 1)) {
101 0 : HCCL_DEBUG("[CollAlgFactory] [TopoMatchConcurrMeshNHR] Rank [%d], virtual topo with rankSize [%u], ranksPerBoard "
102 : "[%u], ranksPerSlot [%u]. 1-D Mesh algorithm should be adopted.",
103 : myRank_, rankSize_, rankOnSameBoard_.size(), rankOnSameSlot_.size());
104 : }
105 :
106 0 : sort(rankOnSameBoard_.begin(), rankOnSameBoard_.end());
107 0 : sort(rankOnSameSlot_.begin(), rankOnSameSlot_.end());
108 :
109 0 : Matrix<RankId> rankOnSamePod;
110 0 : rankOnSamePod.push_back(rankOnSameBoard_);
111 0 : rankOnSamePod.push_back(rankOnSameSlot_);
112 0 : vTopo.push_back(rankOnSamePod);
113 0 : virtRanks.push_back({rankSetLevel0.cbegin(), rankSetLevel0.cend()});
114 :
115 : // 获取 level1 Topo 信息
116 0 : const NetInstance* netInstanceL1 = rankGraph_->GetNetInstanceByRankId(1, myRank_);
117 0 : if (netInstanceL1 == nullptr) {
118 0 : HCCL_ERROR("TopoMatchConcurrMeshNHR::MatchTopo netInstanceL1 is nullptr");
119 0 : return HcclResult::HCCL_E_PTR;
120 : }
121 0 : std::set<RankId> rankSetLevel1 = netInstanceL1->GetRankIds();
122 :
123 : // 它要求 rankId 是连续的, 每隔 rankSizeLevel0 取一个
124 0 : CHK_RET(GenerateLevel1(rankSetLevel1, rankSizeLevel0, myRank_, vTopo, virtRanks));
125 :
126 0 : HCCL_DEBUG("[CollAlgFactory] [TopoMatchConcurrMeshNHR] Rank [%d], level0 & level1 virtRanks=[%s] ",
127 : myRank_, PrintMatrix<RankId>(virtRanks).c_str());
128 0 : HCCL_DEBUG("[CollAlgFactory] [TopoMatchConcurrMeshNHR] Rank [%d], level0 & level1 vTopo=[%s] ",
129 : myRank_, PrintTensor<RankId>(vTopo).c_str());
130 0 : }
131 :
132 : // 子通信域计算完毕, 生成 virtRankMap
133 0 : CHK_PRT_RET(
134 : GenVirtRankMappingMultiLevel(virtRanks, virtRankMap) != HcclResult::HCCL_SUCCESS,
135 : HCCL_ERROR("[CollAlgFactory] [TopoMatchConcurrMeshNHR] Rank [%d], Fail to generate virtRankMapping.", myRank_),
136 : HcclResult::HCCL_E_INTERNAL);
137 :
138 0 : return HcclResult::HCCL_SUCCESS;
139 0 : }
140 :
141 0 : HcclResult TopoMatchConcurrMeshNHR::GenerateLevel0(
142 : const std::set<RankId> &rankSetLevel0, u32 gcdInstSize, RankId rankId,
143 : std::vector<std::vector<std::vector<RankId>>> &vTopo,
144 : std::vector<std::vector<RankId>> &virtRanks)
145 : {
146 : // 获取 my pod size, 计算 level0 subcommunicator size
147 : (void) rankId;
148 0 : u32 dim0Size = numRanksPerBoard_.at(0);
149 0 : u32 gcdDim0Size = GcdTwo(gcdInstSize, dim0Size);
150 0 : u32 gcdDim1Size = gcdInstSize / gcdDim0Size;
151 :
152 : // 维数校验 (2D)
153 0 : CHK_PRT_RET((gcdDim0Size == 1) || (gcdDim1Size == 1), // 1D case
154 : HCCL_ERROR("[CollAlgFactory] [TopoMatchConcurrMeshNHR] Rank [%d], Pod Topo Shape != 2D."
155 : "gcdDim0Size [%u], gcdDim1Size [%u]",
156 : myRank_, gcdDim0Size, gcdDim1Size),
157 : HcclResult::HCCL_E_PARA);
158 :
159 : // 判断 level 0 topo 是否满足 rankId 连续限制
160 0 : bool isRankIdContinue = true;
161 0 : if (gcdDim0Size < dim0Size && gcdDim1Size > 1) {
162 0 : isRankIdContinue = false;
163 : }
164 :
165 0 : if (!isRankIdContinue) {
166 0 : HCCL_ERROR("RankId [%d]. Checker does not support nhr + nhr algorithm validation.", myRank_);
167 0 : RankId minRankId = *rankSetLevel0.cbegin();
168 0 : RankId relRankId = myRank_ - minRankId;
169 0 : u32 groupId = relRankId / gcdInstSize;
170 :
171 0 : const u32 startOffset = groupId * gcdInstSize;
172 0 : const u32 totalSize = rankSetLevel0.size();
173 0 : auto startIt = rankSetLevel0.cbegin();
174 : std::advance(startIt, startOffset);
175 :
176 0 : const u32 elementsToTake = std::min(gcdInstSize, totalSize - startOffset);
177 0 : auto endIt = startIt;
178 : std::advance(endIt, elementsToTake);
179 :
180 0 : std::vector<RankId> level0Ranks = std::vector<RankId>(startIt, endIt);
181 0 : vTopo.push_back({level0Ranks});
182 0 : virtRanks.push_back(level0Ranks);
183 0 : return HcclResult::HCCL_SUCCESS;
184 0 : }
185 :
186 0 : std::vector<RankId> rankVecLevel0 = std::vector<RankId>(rankSetLevel0.cbegin(), rankSetLevel0.cend());
187 0 : Matrix<RankId> rankMatLevel0 = {};
188 0 : auto startIt = rankVecLevel0.cbegin();
189 0 : auto endIt = rankVecLevel0.cend();
190 0 : while (startIt != endIt) {
191 0 : auto currentEndIt = startIt;
192 0 : std::advance(currentEndIt, std::min(dim0Size, static_cast<u32>(std::distance(startIt, endIt))));
193 0 : rankMatLevel0.emplace_back(startIt, currentEndIt);
194 0 : startIt = currentEndIt;
195 : }
196 :
197 0 : std::vector<RankId> level0Ranks;
198 0 : Matrix<RankId> level0Topo;
199 0 : CHK_RET(FindLevel0Block(rankMatLevel0, gcdDim1Size, gcdDim0Size, myRank_, level0Ranks, level0Topo));
200 :
201 0 : CHK_PRT_RET((level0Topo[0].size() != gcdDim0Size || level0Topo[1].size() != gcdDim1Size),
202 : HCCL_ERROR("[CollAlgFactory] [TopoMatchConcurrMeshNHR] Rank [%d], Invalid level0Topo size[%u][%u]",
203 : myRank_, level0Topo[0].size(), level0Topo[1].size()),
204 : HcclResult::HCCL_E_PARA);
205 0 : vTopo.push_back(level0Topo);
206 0 : virtRanks.push_back(level0Ranks);
207 :
208 0 : return HcclResult::HCCL_SUCCESS;
209 0 : }
210 :
211 0 : HcclResult TopoMatchConcurrMeshNHR::FindLevel0Block(const Matrix<RankId>& podTopo, u32 subDim0Size, u32 subDim1Size,
212 : u32 myRank, std::vector<RankId>& subRankList,
213 : Matrix<RankId>& subRankTopo) const
214 : {
215 0 : CHK_PRT_RET((podTopo.empty() || podTopo[0].empty() || subDim0Size == 0 || subDim1Size == 0),
216 : HCCL_ERROR("[CollAlgFactory] [TopoMatchConcurrMeshNHR] Rank [%d], empty podTopo or invalid subDim.",
217 : myRank_),
218 : HcclResult::HCCL_E_PARA);
219 0 : const u32 totalRows = podTopo.size();
220 0 : const u32 totalCols = podTopo[0].size();
221 0 : CHK_PRT_RET((totalRows % subDim0Size != 0 || totalCols % subDim1Size != 0),
222 : HCCL_ERROR("[CollAlgFactory] [TopoMatchConcurrMeshNHR] Rank [%d], invalid subDim.", myRank_),
223 : HcclResult::HCCL_E_PARA);
224 :
225 0 : u32 targetRow = -1;
226 0 : u32 targetCol = -1;
227 0 : CHK_RET(FindMyRankLocation(podTopo, myRank, targetRow, targetCol)); // 查找 myRank 在 podTopo 中坐标
228 :
229 : // 计算 podTopo 块信息
230 0 : const u32 blockRow = targetRow / subDim0Size;
231 0 : const u32 blockCol = targetCol / subDim1Size;
232 :
233 : // 计算目标子块编号和起始索引
234 0 : const u32 startRow = blockRow * subDim0Size;
235 0 : const u32 startCol = blockCol * subDim1Size;
236 :
237 0 : CHK_RET(ExtractLevel0Block(podTopo, startRow, startCol, targetRow, targetCol,
238 : subDim0Size, subDim1Size, subRankList, subRankTopo));
239 :
240 0 : return HcclResult::HCCL_SUCCESS;
241 : }
242 :
243 0 : HcclResult TopoMatchConcurrMeshNHR::FindMyRankLocation(const Matrix<RankId>& podTopo, u32 myRank, u32 &row, u32 &col) const
244 : {
245 0 : row = -1;
246 0 : col = -1; // 初始化为非法值
247 0 : for (u32 r = 0; r < podTopo.size(); r++) {
248 0 : auto it = std::find(podTopo[r].cbegin(), podTopo[r].cend(), myRank);
249 0 : if (it != podTopo[r].cend()) {
250 0 : row = r;
251 0 : col = std::distance(podTopo[r].cbegin(), it);
252 0 : return HcclResult::HCCL_SUCCESS;
253 : }
254 : }
255 0 : return HcclResult::HCCL_E_PARA;
256 : }
257 :
258 0 : HcclResult TopoMatchConcurrMeshNHR::ExtractLevel0Block(const Matrix<RankId>& podTopo, u32 startRow, u32 startCol,
259 : u32 targetRow, u32 targetCol,
260 : u32 subDim0Size, u32 subDim1Size,
261 : std::vector<RankId>& subRankList,
262 : Matrix<RankId>& subRankTopo) const
263 : {
264 0 : subRankList.clear();
265 0 : subRankList.reserve(subDim0Size * subDim1Size);
266 0 : subRankTopo.clear();
267 :
268 0 : const u32 targetRelRow = targetRow - startRow; // myRank 在子块中的相对行 = 绝对行 - 子块起始行
269 0 : const u32 targetRelCol = targetCol - startCol; // myRank 在子块中的相对列 = 绝对列 - 子块起始列
270 :
271 : // 校验相对行&列在 podTopo 范围内
272 0 : const u32 totalRows = podTopo.size();
273 0 : const u32 totalCols = podTopo[0].size();
274 0 : CHK_PRT_RET(targetRelRow >= totalRows,
275 : HCCL_ERROR("[CollAlgFactory] [TopoMatchConcurrMeshNHR] Rank [%d], Invalid targetRelRow[%u], "
276 : "totalRows[%u]", myRank_, targetRelRow, totalRows),
277 : HcclResult::HCCL_E_PARA);
278 0 : CHK_PRT_RET(targetRelCol >= totalCols,
279 : HCCL_ERROR("[CollAlgFactory] [TopoMatchConcurrMeshNHR] Rank [%d], Invalid targetRelCol[%u], "
280 : "totalCols[%u]", myRank_, targetRelCol, totalCols),
281 : HcclResult::HCCL_E_PARA);
282 :
283 0 : std::vector<RankId> targetRowVec;
284 0 : std::vector<RankId> targetColVec;
285 0 : targetRowVec.reserve(subDim1Size);
286 0 : targetColVec.reserve(subDim0Size);
287 :
288 0 : auto matRowStartIt = podTopo.cbegin() + startRow; // 使用迭代器算术获取起始行迭代器
289 :
290 : // 循环 subDim0Size 次，每次处理一行
291 0 : for (u32 r = 0; r < subDim0Size; r++) {
292 0 : const auto& sourceRow = *(matRowStartIt + r); // 获取当前行的常量引用
293 : // 计算该行的列坐标起始
294 0 : auto startIt = sourceRow.cbegin() + startCol;
295 0 : auto endIt = sourceRow.cbegin() + startCol + subDim1Size;
296 :
297 0 : subRankList.insert(subRankList.end(), startIt, endIt);
298 :
299 0 : if (r == targetRelRow) {
300 0 : targetRowVec.insert(targetRowVec.end(), startIt, endIt);
301 : }
302 :
303 0 : auto colIt = sourceRow.cbegin() + startCol + targetRelCol;
304 0 : targetColVec.push_back(*colIt);
305 : }
306 :
307 0 : std::sort(targetRowVec.begin(), targetRowVec.end());
308 0 : std::sort(targetColVec.begin(), targetColVec.end());
309 :
310 0 : subRankTopo.emplace_back(std::move(targetRowVec));
311 0 : subRankTopo.emplace_back(std::move(targetColVec));
312 :
313 0 : return HcclResult::HCCL_SUCCESS;
314 0 : }
315 :
316 0 : bool TopoMatchConcurrMeshNHR::CheckSymmetric(std::vector<u32>& values) const
317 : {
318 0 : const auto minmaxPair = std::minmax_element(values.begin(), values.end());
319 0 : u32 minValue = *minmaxPair.first;
320 0 : u32 maxValue = *minmaxPair.second;
321 :
322 0 : return (minValue == maxValue);
323 : }
324 : } // namespace Hccl
|
