Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include <cmath>
12 :
13 : #include "log.h"
14 :
15 : #include "coll_alg_registry.h"
16 : #include "all_gather_seq_executor.h"
17 :
18 : namespace Hccl {
19 : template <typename AlgTopoMatch, typename AlgTemp0, typename AlgTemp1>
20 0 : AllGatherSeqExecutor<AlgTopoMatch, AlgTemp0, AlgTemp1>::AllGatherSeqExecutor() : CollAlgBase()
21 : {
22 0 : }
23 :
24 : template <typename AlgTopoMatch, typename AlgTemp0, typename AlgTemp1>
25 0 : AllGatherSeqExecutor<AlgTopoMatch, AlgTemp0, AlgTemp1>::~AllGatherSeqExecutor()
26 : {
27 0 : }
28 :
29 : // dataSize_ as input
30 : template <typename AlgTopoMatch, typename AlgTemp0, typename AlgTemp1>
31 0 : HcclResult AllGatherSeqExecutor<AlgTopoMatch, AlgTemp0, AlgTemp1>::CalcResOffload(const RankGraph *rankGraph,
32 : const u64 &dataSize,
33 : CollOffloadOpResReq &resReq)
34 : {
35 : (void)dataSize;
36 0 : resReq.requiredScratchMemSize = 0;
37 :
38 : // Topo Match
39 0 : AlgTopoMatch topoMatch(myRank_, rankSize_, rankGraph, devType_);
40 0 : CHK_RET(topoMatch.MatchTopo(vTopo_, virtRanks_, virtRankMap_));
41 0 : HCCL_INFO("[CollAlgFactory] Rank[%d], [%s].", myRank_, topoMatch.Describe().c_str());
42 :
43 : // instantiate templates
44 0 : auto virtRankMapIter = virtRankMap_.begin();
45 0 : auto vTopoIter = vTopo_.begin();
46 0 : tempRankSizes_.push_back((*virtRankMapIter).size());
47 :
48 0 : AlgTemp0 tempAlg0(myRank_, tempRankSizes_[0], (*vTopoIter), (*virtRankMapIter));
49 :
50 0 : virtRankMapIter++;
51 0 : vTopoIter++;
52 0 : tempRankSizes_.push_back((*virtRankMapIter).size());
53 :
54 0 : AlgTemp1 tempAlg1(myRank_, tempRankSizes_[1], (*vTopoIter), (*virtRankMapIter));
55 :
56 : // calculate required primQues
57 0 : AlgTempResReq tempResReq0;
58 0 : CHK_RET(tempAlg0.CalcRes(tempResReq0));
59 0 : AlgTempResReq tempResReq1;
60 0 : CHK_RET(tempAlg1.CalcRes(tempResReq1));
61 :
62 0 : resReq.requiredSubQueNum = std::max(tempResReq0.queNum, tempResReq1.queNum) - 1;
63 0 : return HcclResult::HCCL_SUCCESS;
64 0 : }
65 :
66 : // dataSize_ as input
67 : template <typename AlgTopoMatch, typename AlgTemp0, typename AlgTemp1>
68 0 : HcclResult AllGatherSeqExecutor<AlgTopoMatch, AlgTemp0, AlgTemp1>::GenPrimQues(const RankGraph *rankGraph,
69 : const CollAlgOperator &op,
70 : const CollAlgParams ¶ms,
71 : PrimQuePtr primQue)
72 : {
73 : // init and check params
74 0 : CHK_RET(Init(op, params, primQue));
75 :
76 : // Topo Match
77 0 : AlgTopoMatch topoMatch(myRank_, rankSize_, rankGraph, devType_);
78 0 : CHK_RET(topoMatch.MatchTopo(vTopo_, virtRanks_, virtRankMap_));
79 :
80 0 : auto virtRankMapIter = virtRankMap_.begin();
81 0 : tempRankSizes_.push_back((*virtRankMapIter).size());
82 0 : virtRankMapIter++;
83 0 : tempRankSizes_.push_back((*virtRankMapIter).size());
84 :
85 : // instantiate templates
86 0 : AlgTemp0 tempAlg0(myRank_, tempRankSizes_[0], vTopo_[0], virtRankMap_[0]);
87 0 : tempAlg0.SetDmaMode(dmaMode_);
88 0 : AlgTemp1 tempAlg1(myRank_, tempRankSizes_[1], vTopo_[1], virtRankMap_[1]);
89 0 : tempAlg1.SetDmaMode(dmaMode_);
90 :
91 : // calculate required primQues and prepare queue
92 0 : AlgTempResReq tempResReq0;
93 0 : CHK_RET(tempAlg0.CalcRes(tempResReq0));
94 :
95 0 : std::vector<PrimQuePtr> requiredQue0;
96 0 : CHK_RET(InitQueue(tempResReq0.queNum, requiredQue0));
97 0 : HCCL_INFO("[CollAlgFactory] Rank[%d], allGather template 0 [%s]: requiredQue Num [%u].", myRank_,
98 : tempAlg0.Describe().c_str(), tempResReq0.queNum);
99 0 : tempRequiredQues_.push_back(requiredQue0);
100 :
101 0 : ResLinks tempLinks0;
102 0 : CHK_RET(PrepResLinks(myRank_, rankGraph, linkPriority_, tempResReq0.links, tempLinks0));
103 0 : tempResLinks_.push_back(tempLinks0);
104 :
105 0 : AlgTempResReq tempResReq1;
106 0 : CHK_RET(tempAlg1.CalcRes(tempResReq1));
107 :
108 0 : std::vector<PrimQuePtr> requiredQue1;
109 0 : CHK_RET(InitQueue(tempResReq1.queNum, requiredQue1));
110 0 : HCCL_INFO("[CollAlgFactory] Rank[%d], allGather template 1 [%s]: requiredQue Num [%u].", myRank_,
111 : tempAlg1.Describe().c_str(), tempResReq1.queNum);
112 0 : tempRequiredQues_.push_back(requiredQue1);
113 :
114 0 : ResLinks tempLinks1;
115 0 : CHK_RET(PrepResLinks(myRank_, rankGraph, linkPriority_, tempResReq1.links, tempLinks1));
116 0 : tempResLinks_.push_back(tempLinks1);
117 :
118 0 : u32 dataSizePerVolume = DataTypeSizeGet(dataType_);
119 0 : dataSize_ = dataCount_ * dataSizePerVolume;
120 :
121 0 : if (opMode_ == OpMode::OFFLOAD) {
122 0 : HCCL_INFO("[CollAlgFactory] Rank[%d], Generating Primitive Queues in OFFLOAD Mode for Host.", myRank_);
123 0 : CHK_RET(GenPrimQues4Offload(tempAlg0, tempAlg1));
124 : } else { // OPBASE
125 0 : HCCL_INFO("[CollAlgFactory] Rank[%d], Generating Primitive Queues in OPBASE Mode for Host.", myRank_);
126 0 : CHK_RET(GenPrimQues4Opbase(dataSizePerVolume, tempAlg0, tempAlg1));
127 : }
128 :
129 0 : return HcclResult::HCCL_SUCCESS;
130 0 : }
131 :
132 : template <typename AlgTopoMatch, typename AlgTemp0, typename AlgTemp1>
133 0 : HcclResult AllGatherSeqExecutor<AlgTopoMatch, AlgTemp0, AlgTemp1>::CalcRes(const RankGraph *rankGraph,
134 : CollAlgResReq &algResReq)
135 : {
136 : // Topo Match
137 0 : AlgTopoMatch topoMatch(myRank_, rankSize_, rankGraph, devType_);
138 0 : CHK_RET(topoMatch.MatchTopo(vTopo_, virtRanks_, virtRankMap_));
139 0 : algResReq.topoInfo.UpdateMultiLevelTopo(virtRanks_, virtRankMap_, vTopo_);
140 :
141 : // instantiate templates
142 0 : AlgTemp0 tempAlg0(myRank_, virtRankMap_[0].size(), vTopo_[0], virtRankMap_[0]);
143 0 : tempAlg0.SetDmaMode(dmaMode_);
144 0 : AlgTemp1 tempAlg1(myRank_, virtRankMap_[1].size(), vTopo_[1], virtRankMap_[1]);
145 0 : tempAlg1.SetDmaMode(dmaMode_);
146 :
147 : // calculate required resources
148 0 : AlgTempResReq tempResReq0;
149 0 : CHK_RET(tempAlg0.CalcRes(tempResReq0));
150 0 : AlgTempResReq tempResReq1;
151 0 : CHK_RET(tempAlg1.CalcRes(tempResReq1));
152 :
153 0 : algResReq.primQueueNum = std::max(tempResReq0.queNum, tempResReq1.queNum);
154 :
155 0 : LinkReq linkReqSeq = GetSeqLinksUnion(tempResReq0.links, tempResReq1.links);
156 0 : CHK_RET(CalcResLinks(myRank_, rankGraph, linkPriority_, linkReqSeq, algResReq.links));
157 :
158 0 : return HcclResult::HCCL_SUCCESS;
159 0 : }
160 :
161 : template <typename AlgTopoMatch, typename AlgTemp0, typename AlgTemp1>
162 0 : HcclResult AllGatherSeqExecutor<AlgTopoMatch, AlgTemp0, AlgTemp1>::GenPrimQuesAIC(const AlgTopoInfo &topoInfo,
163 : const CollAlgOperator &op,
164 : const CollAlgParams ¶ms,
165 : ConnectedLinkMgr *linkMgr,
166 : PrimQuePtr primQue)
167 : {
168 : // init and check params
169 0 : CHK_RET(Init(op, params, primQue));
170 :
171 : // Topo Match
172 0 : vTopo_ = topoInfo.vTopo;
173 0 : virtRanks_ = topoInfo.virtRanks;
174 0 : virtRankMap_ = topoInfo.virtRankMap;
175 :
176 0 : tempRankSizes_.push_back((virtRankMap_[0]).size());
177 0 : tempRankSizes_.push_back((virtRankMap_[1]).size());
178 :
179 : // instantiate templates
180 0 : AlgTemp0 tempAlg0(myRank_, tempRankSizes_[0], vTopo_[0], virtRankMap_[0]);
181 0 : tempAlg0.SetDmaMode(dmaMode_);
182 0 : AlgTemp1 tempAlg1(myRank_, tempRankSizes_[1], vTopo_[1], virtRankMap_[1]);
183 0 : tempAlg1.SetDmaMode(dmaMode_);
184 :
185 : // calculate required primQues and prepare queue
186 0 : AlgTempResReq tempResReq0;
187 0 : CHK_RET(tempAlg0.CalcRes(tempResReq0));
188 :
189 0 : AlgTempResReq tempResReq1;
190 0 : CHK_RET(tempAlg1.CalcRes(tempResReq1));
191 :
192 0 : std::vector<PrimQuePtr> requiredQue0;
193 0 : CHK_RET(InitQueue(tempResReq0.queNum, requiredQue0));
194 0 : HCCL_INFO("[CollAlgFactory] Rank[%d], allGather template 0 [%s]: requiredQue Num [%u].", myRank_,
195 : tempAlg0.Describe().c_str(), tempResReq0.queNum);
196 0 : tempRequiredQues_.push_back(requiredQue0);
197 :
198 0 : std::vector<PrimQuePtr> requiredQue1;
199 0 : CHK_RET(InitQueue(tempResReq1.queNum, requiredQue1));
200 0 : HCCL_INFO("[CollAlgFactory] Rank[%d], allGather template 1 [%s]: requiredQue Num [%u].", myRank_,
201 : tempAlg1.Describe().c_str(), tempResReq1.queNum);
202 0 : tempRequiredQues_.push_back(requiredQue1);
203 :
204 0 : ResLinks tempLinks0;
205 0 : CHK_RET(PrepResLinks(myRank_, tempResReq0.links, linkMgr, tempLinks0));
206 0 : tempResLinks_.push_back(tempLinks0);
207 :
208 0 : ResLinks tempLinks1;
209 0 : CHK_RET(PrepResLinks(myRank_, tempResReq1.links, linkMgr, tempLinks1));
210 0 : tempResLinks_.push_back(tempLinks1);
211 :
212 0 : u32 dataSizePerVolume = DataTypeSizeGet(dataType_);
213 0 : dataSize_ = dataCount_ * dataSizePerVolume;
214 :
215 0 : if (opMode_ == OpMode::OFFLOAD) {
216 0 : HCCL_INFO("[CollAlgFactory] Rank[%d], Generating Primitive Queues in OFFLOAD Mode for AICPU.", myRank_);
217 0 : CHK_RET(GenPrimQues4Offload(tempAlg0, tempAlg1));
218 : } else { // OPBASE
219 0 : HCCL_INFO("[CollAlgFactory] Rank[%d], Generating Primitive Queues in OPBASE Mode for AICPU.", myRank_);
220 0 : CHK_RET(GenPrimQues4Opbase(dataSizePerVolume, tempAlg0, tempAlg1));
221 : }
222 :
223 0 : return HcclResult::HCCL_SUCCESS;
224 0 : }
225 :
226 : template <typename AlgTopoMatch, typename AlgTemp0, typename AlgTemp1>
227 0 : HcclResult AllGatherSeqExecutor<AlgTopoMatch, AlgTemp0, AlgTemp1>::GenPrimQues4Offload(AlgTemplateBase &tempAlg0,
228 : AlgTemplateBase &tempAlg1)
229 : {
230 0 : RankSliceInfo sliceInfoVec0;
231 0 : AllignInfo allignInfo = {enableAllign_, allignSize_, dataType_};
232 0 : CHK_RET(tempAlg0.CalcSliceInfo(allignInfo, dataSize_, sliceInfoVec0));
233 :
234 0 : u64 outcomeSize = dataSize_ * tempRankSizes_[0];
235 0 : u32 outDataIdx = virtRankMap_[1][myRank_];
236 :
237 0 : BuffInfo buffInfo;
238 0 : buffInfo.inBuffType = BufferType::INPUT;
239 0 : buffInfo.outBuffType = BufferType::OUTPUT;
240 0 : buffInfo.inBuffBaseOff = 0;
241 0 : buffInfo.outBuffBaseOff = outDataIdx * outcomeSize;
242 :
243 0 : TempFuncs tempFuncs;
244 0 : tempFuncs.opMode = opMode_;
245 0 : tempFuncs.enableCounterNotify = IsEnableCounterNotify();
246 0 : tempFuncs.forAlgSeqComb = false;
247 :
248 0 : CHK_RET(tempAlg0.GenPrimQue(tempFuncs, sliceInfoVec0, buffInfo, tempResLinks_[0], tempRequiredQues_[0]));
249 :
250 : // level 1
251 0 : RankSliceInfo sliceInfoVec1;
252 0 : CHK_RET(tempAlg1.CalcSliceInfo(allignInfo, outcomeSize, sliceInfoVec1));
253 :
254 0 : buffInfo.outBuffBaseOff = 0;
255 :
256 0 : tempFuncs.forAlgSeqComb = true;
257 :
258 0 : CHK_RET(tempAlg1.GenPrimQue(tempFuncs, sliceInfoVec1, buffInfo, tempResLinks_[1], tempRequiredQues_[1]));
259 :
260 0 : return HcclResult::HCCL_SUCCESS;
261 0 : }
262 :
263 : template <typename AlgTopoMatch, typename AlgTemp0, typename AlgTemp1>
264 0 : HcclResult AllGatherSeqExecutor<AlgTopoMatch, AlgTemp0, AlgTemp1>::GenPrimQues4Opbase(const u32 dataSizePerVolume,
265 : AlgTemplateBase &tempAlg0,
266 : AlgTemplateBase &tempAlg1)
267 : {
268 0 : CHK_PRT_RET(dataSizePerVolume == 0,
269 : HCCL_ERROR("[CollAlgFactory] Rank [%d], Invalid dataSizePerVolume [%u].", myRank_, dataSizePerVolume),
270 : HcclResult::HCCL_E_INTERNAL);
271 :
272 0 : u32 scratchInputSize
273 0 : = static_cast<int>((rankSize_ % dataSizePerVolume == 0)
274 0 : ? floor(maxTmpMemSize_ / rankSize_)
275 0 : : floor(maxTmpMemSize_ / (rankSize_ * dataSizePerVolume)) * dataSizePerVolume);
276 :
277 0 : CHK_PRT_RET(scratchInputSize == 0,
278 : HCCL_ERROR("[CollAlgFactory] Rank [%d], Invalid input maxTmpMemSize [%u].", myRank_, maxTmpMemSize_),
279 : HcclResult::HCCL_E_PARA);
280 :
281 0 : BuffInfo buffInfo;
282 0 : buffInfo.outBuffType = BufferType::SCRATCH;
283 :
284 0 : u32 sendRecvTimes = (dataSize_ / scratchInputSize) + ((dataSize_ % scratchInputSize) == 0 ? 0 : 1);
285 0 : HCCL_INFO("[CollAlgFactory] Rank [%d], sendRecvTimes [%u].", myRank_, sendRecvTimes);
286 :
287 0 : for (u32 idx = 0; idx < sendRecvTimes; idx++) {
288 : // datasize of level 0
289 0 : u64 currDataSize = (idx == (sendRecvTimes - 1)) ? (dataSize_ - idx * scratchInputSize) : scratchInputSize;
290 :
291 : // expected outcome of level 0 allgather
292 0 : u64 outcomeSize = currDataSize * tempRankSizes_[0];
293 0 : u32 outDataIdx = virtRankMap_[1][myRank_];
294 :
295 : // level 0
296 0 : RankSliceInfo sliceInfoVec0;
297 0 : AllignInfo allignInfo = {enableAllign_, allignSize_, dataType_};
298 0 : CHK_RET(tempAlg0.CalcSliceInfo(allignInfo, currDataSize, sliceInfoVec0));
299 :
300 0 : buffInfo.outBuffBaseOff = outDataIdx * outcomeSize;
301 :
302 0 : TempFuncs tempFuncs;
303 0 : tempFuncs.opMode = opMode_;
304 0 : tempFuncs.enableCounterNotify = IsEnableCounterNotify();
305 0 : tempFuncs.isForepart = true; // Usr Buff to CCL Buff required
306 0 : tempFuncs.isBottom = false; // CCL Buff to Usr Buff required
307 :
308 0 : UsrData usrData;
309 0 : DataSlice usrInSlice = DataSlice(BufferType::INPUT, idx * scratchInputSize, currDataSize);
310 0 : DataSlice scratchInSlice = DataSlice(
311 0 : BufferType::SCRATCH, virtRankMap_[0][myRank_] * currDataSize + outDataIdx * outcomeSize, currDataSize);
312 0 : usrData.usrInSlices.push_back(usrInSlice);
313 0 : usrData.scratchInSlices.push_back(scratchInSlice);
314 :
315 0 : tempFuncs.usrData = usrData;
316 :
317 0 : CHK_RET(tempAlg0.GenPrimQue(tempFuncs, sliceInfoVec0, buffInfo, tempResLinks_[0], tempRequiredQues_[0]));
318 :
319 : // level 1
320 0 : RankSliceInfo sliceInfoVec1;
321 0 : CHK_RET(tempAlg1.CalcSliceInfo(allignInfo, outcomeSize, sliceInfoVec1));
322 :
323 0 : buffInfo.outBuffBaseOff = 0;
324 :
325 0 : tempFuncs.isForepart = false; // Usr Buff to CCL Buff required
326 0 : tempFuncs.isBottom = true; // CCL Buff to Usr Buff required
327 :
328 0 : for (u32 rankIdx = 0; rankIdx < rankSize_; rankIdx++) {
329 0 : DataSlice scratchOutSlice = DataSlice(BufferType::SCRATCH, rankIdx * currDataSize, currDataSize);
330 0 : DataSlice usrOutSlice
331 0 : = DataSlice(BufferType::OUTPUT, rankIdx * dataSize_ + idx * scratchInputSize, currDataSize);
332 0 : tempFuncs.usrData.scratchOutSlices.push_back(scratchOutSlice);
333 0 : tempFuncs.usrData.usrOutSlices.push_back(usrOutSlice);
334 : }
335 :
336 0 : CHK_RET(tempAlg1.GenPrimQue(tempFuncs, sliceInfoVec1, buffInfo, tempResLinks_[1], tempRequiredQues_[1]));
337 : }
338 :
339 0 : return HcclResult::HCCL_SUCCESS;
340 : }
341 :
342 : REGISTER_IMPL_BY_TWO_TEMPS(OpType::ALLGATHER, AllGatherSeqMeshRing, AllGatherSeqExecutor, TopoMatchMeshRing,
343 : TempAllGatherMesh, TempAllGatherRing);
344 : } // namespace Hccl
|
