Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include <cmath>
12 :
13 : #include "log.h"
14 :
15 : #include "coll_alg_registry.h"
16 : #include "all_gather_seq_executor.h"
17 :
18 : namespace Hccl {
19 : template <typename AlgTopoMatch, typename AlgTemp0, typename AlgTemp1>
20 0 : AllGatherSeqExecutor<AlgTopoMatch, AlgTemp0, AlgTemp1>::AllGatherSeqExecutor() : CollAlgBase()
21 0 : {}
22 :
23 : template <typename AlgTopoMatch, typename AlgTemp0, typename AlgTemp1>
24 0 : AllGatherSeqExecutor<AlgTopoMatch, AlgTemp0, AlgTemp1>::~AllGatherSeqExecutor()
25 0 : {}
26 :
27 : // dataSize_ as input
28 : template <typename AlgTopoMatch, typename AlgTemp0, typename AlgTemp1>
29 0 : HcclResult AllGatherSeqExecutor<AlgTopoMatch, AlgTemp0, AlgTemp1>::CalcResOffload(
30 : const RankGraph* rankGraph, const u64& dataSize, CollOffloadOpResReq& resReq)
31 : {
32 : (void)dataSize;
33 0 : resReq.requiredScratchMemSize = 0;
34 :
35 : // Topo Match
36 0 : AlgTopoMatch topoMatch(myRank_, rankSize_, rankGraph, devType_);
37 0 : CHK_RET(topoMatch.MatchTopo(vTopo_, virtRanks_, virtRankMap_));
38 0 : HCCL_INFO("[CollAlgFactory] Rank[%d], [%s].", myRank_, topoMatch.Describe().c_str());
39 :
40 : // instantiate templates
41 0 : auto virtRankMapIter = virtRankMap_.begin();
42 0 : auto vTopoIter = vTopo_.begin();
43 0 : tempRankSizes_.push_back((*virtRankMapIter).size());
44 :
45 0 : AlgTemp0 tempAlg0(myRank_, tempRankSizes_[0], (*vTopoIter), (*virtRankMapIter));
46 :
47 0 : virtRankMapIter++;
48 0 : vTopoIter++;
49 0 : tempRankSizes_.push_back((*virtRankMapIter).size());
50 :
51 0 : AlgTemp1 tempAlg1(myRank_, tempRankSizes_[1], (*vTopoIter), (*virtRankMapIter));
52 :
53 : // calculate required primQues
54 0 : AlgTempResReq tempResReq0;
55 0 : CHK_RET(tempAlg0.CalcRes(tempResReq0));
56 0 : AlgTempResReq tempResReq1;
57 0 : CHK_RET(tempAlg1.CalcRes(tempResReq1));
58 :
59 0 : resReq.requiredSubQueNum = std::max(tempResReq0.queNum, tempResReq1.queNum) - 1;
60 0 : return HcclResult::HCCL_SUCCESS;
61 0 : }
62 :
63 : // dataSize_ as input
64 : template <typename AlgTopoMatch, typename AlgTemp0, typename AlgTemp1>
65 0 : HcclResult AllGatherSeqExecutor<AlgTopoMatch, AlgTemp0, AlgTemp1>::GenPrimQues(
66 : const RankGraph* rankGraph, const CollAlgOperator& op, const CollAlgParams& params, PrimQuePtr primQue)
67 : {
68 : // init and check params
69 0 : CHK_RET(Init(op, params, primQue));
70 :
71 : // Topo Match
72 0 : AlgTopoMatch topoMatch(myRank_, rankSize_, rankGraph, devType_);
73 0 : CHK_RET(topoMatch.MatchTopo(vTopo_, virtRanks_, virtRankMap_));
74 :
75 0 : auto virtRankMapIter = virtRankMap_.begin();
76 0 : tempRankSizes_.push_back((*virtRankMapIter).size());
77 0 : virtRankMapIter++;
78 0 : tempRankSizes_.push_back((*virtRankMapIter).size());
79 :
80 : // instantiate templates
81 0 : AlgTemp0 tempAlg0(myRank_, tempRankSizes_[0], vTopo_[0], virtRankMap_[0]);
82 0 : tempAlg0.SetDmaMode(dmaMode_);
83 0 : AlgTemp1 tempAlg1(myRank_, tempRankSizes_[1], vTopo_[1], virtRankMap_[1]);
84 0 : tempAlg1.SetDmaMode(dmaMode_);
85 :
86 : // calculate required primQues and prepare queue
87 0 : AlgTempResReq tempResReq0;
88 0 : CHK_RET(tempAlg0.CalcRes(tempResReq0));
89 :
90 0 : std::vector<PrimQuePtr> requiredQue0;
91 0 : CHK_RET(InitQueue(tempResReq0.queNum, requiredQue0));
92 0 : HCCL_INFO(
93 : "[CollAlgFactory] Rank[%d], allGather template 0 [%s]: requiredQue Num [%u].", myRank_,
94 : tempAlg0.Describe().c_str(), tempResReq0.queNum);
95 0 : tempRequiredQues_.push_back(requiredQue0);
96 :
97 0 : ResLinks tempLinks0;
98 0 : CHK_RET(PrepResLinks(myRank_, rankGraph, linkPriority_, tempResReq0.links, tempLinks0));
99 0 : tempResLinks_.push_back(tempLinks0);
100 :
101 0 : AlgTempResReq tempResReq1;
102 0 : CHK_RET(tempAlg1.CalcRes(tempResReq1));
103 :
104 0 : std::vector<PrimQuePtr> requiredQue1;
105 0 : CHK_RET(InitQueue(tempResReq1.queNum, requiredQue1));
106 0 : HCCL_INFO(
107 : "[CollAlgFactory] Rank[%d], allGather template 1 [%s]: requiredQue Num [%u].", myRank_,
108 : tempAlg1.Describe().c_str(), tempResReq1.queNum);
109 0 : tempRequiredQues_.push_back(requiredQue1);
110 :
111 0 : ResLinks tempLinks1;
112 0 : CHK_RET(PrepResLinks(myRank_, rankGraph, linkPriority_, tempResReq1.links, tempLinks1));
113 0 : tempResLinks_.push_back(tempLinks1);
114 :
115 0 : u32 dataSizePerVolume = DataTypeSizeGet(dataType_);
116 0 : dataSize_ = dataCount_ * dataSizePerVolume;
117 :
118 0 : if (opMode_ == OpMode::OFFLOAD) {
119 0 : HCCL_INFO("[CollAlgFactory] Rank[%d], Generating Primitive Queues in OFFLOAD Mode for Host.", myRank_);
120 0 : CHK_RET(GenPrimQues4Offload(tempAlg0, tempAlg1));
121 : } else { // OPBASE
122 0 : HCCL_INFO("[CollAlgFactory] Rank[%d], Generating Primitive Queues in OPBASE Mode for Host.", myRank_);
123 0 : CHK_RET(GenPrimQues4Opbase(dataSizePerVolume, tempAlg0, tempAlg1));
124 : }
125 :
126 0 : return HcclResult::HCCL_SUCCESS;
127 0 : }
128 :
129 : template <typename AlgTopoMatch, typename AlgTemp0, typename AlgTemp1>
130 : HcclResult
131 0 : AllGatherSeqExecutor<AlgTopoMatch, AlgTemp0, AlgTemp1>::CalcRes(const RankGraph* rankGraph, CollAlgResReq& algResReq)
132 : {
133 : // Topo Match
134 0 : AlgTopoMatch topoMatch(myRank_, rankSize_, rankGraph, devType_);
135 0 : CHK_RET(topoMatch.MatchTopo(vTopo_, virtRanks_, virtRankMap_));
136 0 : algResReq.topoInfo.UpdateMultiLevelTopo(virtRanks_, virtRankMap_, vTopo_);
137 :
138 : // instantiate templates
139 0 : AlgTemp0 tempAlg0(myRank_, virtRankMap_[0].size(), vTopo_[0], virtRankMap_[0]);
140 0 : tempAlg0.SetDmaMode(dmaMode_);
141 0 : AlgTemp1 tempAlg1(myRank_, virtRankMap_[1].size(), vTopo_[1], virtRankMap_[1]);
142 0 : tempAlg1.SetDmaMode(dmaMode_);
143 :
144 : // calculate required resources
145 0 : AlgTempResReq tempResReq0;
146 0 : CHK_RET(tempAlg0.CalcRes(tempResReq0));
147 0 : AlgTempResReq tempResReq1;
148 0 : CHK_RET(tempAlg1.CalcRes(tempResReq1));
149 :
150 0 : algResReq.primQueueNum = std::max(tempResReq0.queNum, tempResReq1.queNum);
151 :
152 0 : LinkReq linkReqSeq = GetSeqLinksUnion(tempResReq0.links, tempResReq1.links);
153 0 : CHK_RET(CalcResLinks(myRank_, rankGraph, linkPriority_, linkReqSeq, algResReq.links));
154 :
155 0 : return HcclResult::HCCL_SUCCESS;
156 0 : }
157 :
158 : template <typename AlgTopoMatch, typename AlgTemp0, typename AlgTemp1>
159 0 : HcclResult AllGatherSeqExecutor<AlgTopoMatch, AlgTemp0, AlgTemp1>::GenPrimQuesAIC(
160 : const AlgTopoInfo& topoInfo, const CollAlgOperator& op, const CollAlgParams& params, ConnectedLinkMgr* linkMgr,
161 : PrimQuePtr primQue)
162 : {
163 : // init and check params
164 0 : CHK_RET(Init(op, params, primQue));
165 :
166 : // Topo Match
167 0 : vTopo_ = topoInfo.vTopo;
168 0 : virtRanks_ = topoInfo.virtRanks;
169 0 : virtRankMap_ = topoInfo.virtRankMap;
170 :
171 0 : tempRankSizes_.push_back((virtRankMap_[0]).size());
172 0 : tempRankSizes_.push_back((virtRankMap_[1]).size());
173 :
174 : // instantiate templates
175 0 : AlgTemp0 tempAlg0(myRank_, tempRankSizes_[0], vTopo_[0], virtRankMap_[0]);
176 0 : tempAlg0.SetDmaMode(dmaMode_);
177 0 : AlgTemp1 tempAlg1(myRank_, tempRankSizes_[1], vTopo_[1], virtRankMap_[1]);
178 0 : tempAlg1.SetDmaMode(dmaMode_);
179 :
180 : // calculate required primQues and prepare queue
181 0 : AlgTempResReq tempResReq0;
182 0 : CHK_RET(tempAlg0.CalcRes(tempResReq0));
183 :
184 0 : AlgTempResReq tempResReq1;
185 0 : CHK_RET(tempAlg1.CalcRes(tempResReq1));
186 :
187 0 : std::vector<PrimQuePtr> requiredQue0;
188 0 : CHK_RET(InitQueue(tempResReq0.queNum, requiredQue0));
189 0 : HCCL_INFO(
190 : "[CollAlgFactory] Rank[%d], allGather template 0 [%s]: requiredQue Num [%u].", myRank_,
191 : tempAlg0.Describe().c_str(), tempResReq0.queNum);
192 0 : tempRequiredQues_.push_back(requiredQue0);
193 :
194 0 : std::vector<PrimQuePtr> requiredQue1;
195 0 : CHK_RET(InitQueue(tempResReq1.queNum, requiredQue1));
196 0 : HCCL_INFO(
197 : "[CollAlgFactory] Rank[%d], allGather template 1 [%s]: requiredQue Num [%u].", myRank_,
198 : tempAlg1.Describe().c_str(), tempResReq1.queNum);
199 0 : tempRequiredQues_.push_back(requiredQue1);
200 :
201 0 : ResLinks tempLinks0;
202 0 : CHK_RET(PrepResLinks(myRank_, tempResReq0.links, linkMgr, tempLinks0));
203 0 : tempResLinks_.push_back(tempLinks0);
204 :
205 0 : ResLinks tempLinks1;
206 0 : CHK_RET(PrepResLinks(myRank_, tempResReq1.links, linkMgr, tempLinks1));
207 0 : tempResLinks_.push_back(tempLinks1);
208 :
209 0 : u32 dataSizePerVolume = DataTypeSizeGet(dataType_);
210 0 : dataSize_ = dataCount_ * dataSizePerVolume;
211 :
212 0 : if (opMode_ == OpMode::OFFLOAD) {
213 0 : HCCL_INFO("[CollAlgFactory] Rank[%d], Generating Primitive Queues in OFFLOAD Mode for AICPU.", myRank_);
214 0 : CHK_RET(GenPrimQues4Offload(tempAlg0, tempAlg1));
215 : } else { // OPBASE
216 0 : HCCL_INFO("[CollAlgFactory] Rank[%d], Generating Primitive Queues in OPBASE Mode for AICPU.", myRank_);
217 0 : CHK_RET(GenPrimQues4Opbase(dataSizePerVolume, tempAlg0, tempAlg1));
218 : }
219 :
220 0 : return HcclResult::HCCL_SUCCESS;
221 0 : }
222 :
223 : template <typename AlgTopoMatch, typename AlgTemp0, typename AlgTemp1>
224 0 : HcclResult AllGatherSeqExecutor<AlgTopoMatch, AlgTemp0, AlgTemp1>::GenPrimQues4Offload(
225 : AlgTemplateBase& tempAlg0, AlgTemplateBase& tempAlg1)
226 : {
227 0 : RankSliceInfo sliceInfoVec0;
228 0 : AllignInfo allignInfo = {enableAllign_, allignSize_, dataType_};
229 0 : CHK_RET(tempAlg0.CalcSliceInfo(allignInfo, dataSize_, sliceInfoVec0));
230 :
231 0 : u64 outcomeSize = dataSize_ * tempRankSizes_[0];
232 0 : u32 outDataIdx = virtRankMap_[1][myRank_];
233 :
234 0 : BuffInfo buffInfo;
235 0 : buffInfo.inBuffType = BufferType::INPUT;
236 0 : buffInfo.outBuffType = BufferType::OUTPUT;
237 0 : buffInfo.inBuffBaseOff = 0;
238 0 : buffInfo.outBuffBaseOff = outDataIdx * outcomeSize;
239 :
240 0 : TempFuncs tempFuncs;
241 0 : tempFuncs.opMode = opMode_;
242 0 : tempFuncs.enableCounterNotify = IsEnableCounterNotify();
243 0 : tempFuncs.forAlgSeqComb = false;
244 :
245 0 : CHK_RET(tempAlg0.GenPrimQue(tempFuncs, sliceInfoVec0, buffInfo, tempResLinks_[0], tempRequiredQues_[0]));
246 :
247 : // level 1
248 0 : RankSliceInfo sliceInfoVec1;
249 0 : CHK_RET(tempAlg1.CalcSliceInfo(allignInfo, outcomeSize, sliceInfoVec1));
250 :
251 0 : buffInfo.outBuffBaseOff = 0;
252 :
253 0 : tempFuncs.forAlgSeqComb = true;
254 :
255 0 : CHK_RET(tempAlg1.GenPrimQue(tempFuncs, sliceInfoVec1, buffInfo, tempResLinks_[1], tempRequiredQues_[1]));
256 :
257 0 : return HcclResult::HCCL_SUCCESS;
258 0 : }
259 :
260 : template <typename AlgTopoMatch, typename AlgTemp0, typename AlgTemp1>
261 0 : HcclResult AllGatherSeqExecutor<AlgTopoMatch, AlgTemp0, AlgTemp1>::GenPrimQues4Opbase(
262 : const u32 dataSizePerVolume, AlgTemplateBase& tempAlg0, AlgTemplateBase& tempAlg1)
263 : {
264 0 : CHK_PRT_RET(
265 : dataSizePerVolume == 0,
266 : HCCL_ERROR("[CollAlgFactory] Rank [%d], Invalid dataSizePerVolume [%u].", myRank_, dataSizePerVolume),
267 : HcclResult::HCCL_E_INTERNAL);
268 :
269 0 : u32 scratchInputSize = static_cast<int>(
270 0 : (rankSize_ % dataSizePerVolume == 0) ?
271 0 : floor(maxTmpMemSize_ / rankSize_) :
272 0 : floor(maxTmpMemSize_ / (rankSize_ * dataSizePerVolume)) * dataSizePerVolume);
273 :
274 0 : CHK_PRT_RET(
275 : scratchInputSize == 0,
276 : HCCL_ERROR("[CollAlgFactory] Rank [%d], Invalid input maxTmpMemSize [%u].", myRank_, maxTmpMemSize_),
277 : HcclResult::HCCL_E_PARA);
278 :
279 0 : BuffInfo buffInfo;
280 0 : buffInfo.outBuffType = BufferType::SCRATCH;
281 :
282 0 : u32 sendRecvTimes = (dataSize_ / scratchInputSize) + ((dataSize_ % scratchInputSize) == 0 ? 0 : 1);
283 0 : HCCL_INFO("[CollAlgFactory] Rank [%d], sendRecvTimes [%u].", myRank_, sendRecvTimes);
284 :
285 0 : for (u32 idx = 0; idx < sendRecvTimes; idx++) {
286 : // datasize of level 0
287 0 : u64 currDataSize = (idx == (sendRecvTimes - 1)) ? (dataSize_ - idx * scratchInputSize) : scratchInputSize;
288 :
289 : // expected outcome of level 0 allgather
290 0 : u64 outcomeSize = currDataSize * tempRankSizes_[0];
291 0 : u32 outDataIdx = virtRankMap_[1][myRank_];
292 :
293 : // level 0
294 0 : RankSliceInfo sliceInfoVec0;
295 0 : AllignInfo allignInfo = {enableAllign_, allignSize_, dataType_};
296 0 : CHK_RET(tempAlg0.CalcSliceInfo(allignInfo, currDataSize, sliceInfoVec0));
297 :
298 0 : buffInfo.outBuffBaseOff = outDataIdx * outcomeSize;
299 :
300 0 : TempFuncs tempFuncs;
301 0 : tempFuncs.opMode = opMode_;
302 0 : tempFuncs.enableCounterNotify = IsEnableCounterNotify();
303 0 : tempFuncs.isForepart = true; // Usr Buff to CCL Buff required
304 0 : tempFuncs.isBottom = false; // CCL Buff to Usr Buff required
305 :
306 0 : UsrData usrData;
307 0 : DataSlice usrInSlice = DataSlice(BufferType::INPUT, idx * scratchInputSize, currDataSize);
308 0 : DataSlice scratchInSlice = DataSlice(
309 0 : BufferType::SCRATCH, virtRankMap_[0][myRank_] * currDataSize + outDataIdx * outcomeSize, currDataSize);
310 0 : usrData.usrInSlices.push_back(usrInSlice);
311 0 : usrData.scratchInSlices.push_back(scratchInSlice);
312 :
313 0 : tempFuncs.usrData = usrData;
314 :
315 0 : CHK_RET(tempAlg0.GenPrimQue(tempFuncs, sliceInfoVec0, buffInfo, tempResLinks_[0], tempRequiredQues_[0]));
316 :
317 : // level 1
318 0 : RankSliceInfo sliceInfoVec1;
319 0 : CHK_RET(tempAlg1.CalcSliceInfo(allignInfo, outcomeSize, sliceInfoVec1));
320 :
321 0 : buffInfo.outBuffBaseOff = 0;
322 :
323 0 : tempFuncs.isForepart = false; // Usr Buff to CCL Buff required
324 0 : tempFuncs.isBottom = true; // CCL Buff to Usr Buff required
325 :
326 0 : for (u32 rankIdx = 0; rankIdx < rankSize_; rankIdx++) {
327 0 : DataSlice scratchOutSlice = DataSlice(BufferType::SCRATCH, rankIdx * currDataSize, currDataSize);
328 0 : DataSlice usrOutSlice
329 0 : = DataSlice(BufferType::OUTPUT, rankIdx * dataSize_ + idx * scratchInputSize, currDataSize);
330 0 : tempFuncs.usrData.scratchOutSlices.push_back(scratchOutSlice);
331 0 : tempFuncs.usrData.usrOutSlices.push_back(usrOutSlice);
332 : }
333 :
334 0 : CHK_RET(tempAlg1.GenPrimQue(tempFuncs, sliceInfoVec1, buffInfo, tempResLinks_[1], tempRequiredQues_[1]));
335 : }
336 :
337 0 : return HcclResult::HCCL_SUCCESS;
338 : }
339 :
340 : REGISTER_IMPL_BY_TWO_TEMPS(
341 : OpType::ALLGATHER, AllGatherSeqMeshRing, AllGatherSeqExecutor, TopoMatchMeshRing, TempAllGatherMesh,
342 : TempAllGatherRing);
343 : } // namespace Hccl
|
