Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include <cmath>
12 :
13 : #include "ins_all_gather_sole_executor.h"
14 :
15 : #include "log.h"
16 : #include "ins_coll_alg_registry.h"
17 :
18 : #ifndef CCL_KERNEL_AICPU
19 : #include "ccu_temp_all_gather_mesh_1D.h"
20 : #include "ccu_temp_all_gather_mesh_1D_detour.h"
21 : #include "ccu_temp_all_gather_mesh_1D_mem2mem.h"
22 : #include "ccu_temp_all_gather_mesh_2D.h"
23 : #endif
24 :
25 : #include "topo_match_mesh.h"
26 : #include "topo_match_concurr_mesh.h"
27 :
28 : namespace Hccl {
29 : template <typename AlgTopoMatch, typename InsAlgTemplate>
30 0 : InsAllGatherSoleExecutor<AlgTopoMatch, InsAlgTemplate>::InsAllGatherSoleExecutor() : InsCollAlgBase()
31 : {
32 0 : }
33 :
34 : template <typename AlgTopoMatch, typename InsAlgTemplate>
35 0 : InsAllGatherSoleExecutor<AlgTopoMatch, InsAlgTemplate>::~InsAllGatherSoleExecutor()
36 : {
37 0 : }
38 :
39 : template <typename AlgTopoMatch, typename InsAlgTemplate>
40 0 : HcclResult InsAllGatherSoleExecutor<AlgTopoMatch, InsAlgTemplate>::CalcResOffload(const RankGraph *rankGraph,
41 : const u64 &dataSize,
42 : CollOffloadOpResReq &resReq)
43 : {
44 : (void)dataSize;
45 0 : resReq.requiredScratchMemSize = 0;
46 : // Topo Match
47 0 : AlgTopoMatch topoMatch(myRank_, rankSize_, rankGraph, devType_);
48 0 : CHK_RET(topoMatch.MatchTopo(vTopo_, virtRanks_, virtRankMap_));
49 :
50 : // instantiate a template
51 0 : InsAlgTemplate tempAlg(myRank_, rankSize_, vTopo_, virtRankMap_);
52 :
53 : // calculate required insQueues and prepare queue
54 0 : AlgTempResReq tempResReq;
55 0 : if (enableDetour_) {
56 0 : CHK_RET(tempAlg.CalcResDetour(rankGraph, tempResReq));
57 : } else {
58 0 : CHK_RET(tempAlg.CalcRes(tempResReq));
59 : }
60 :
61 0 : resReq.requiredSubQueNum = tempResReq.streamNum - 1;
62 :
63 0 : return HcclResult::HCCL_SUCCESS;
64 0 : }
65 :
66 : // dataSize_ as input
67 : template <typename AlgTopoMatch, typename InsAlgTemplate>
68 0 : HcclResult InsAllGatherSoleExecutor<AlgTopoMatch, InsAlgTemplate>::Orchestrate(const RankGraph *rankGraph,
69 : const CollAlgOperator &op,
70 : const CollAlgParams ¶ms,
71 : InsQuePtr insQue)
72 : {
73 : // init and check params
74 0 : CHK_RET(Init(op, params, insQue));
75 :
76 : // Topo Match
77 0 : AlgTopoMatch topoMatch(myRank_, rankSize_, rankGraph, devType_);
78 0 : CHK_RET(topoMatch.MatchTopo(vTopo_, virtRanks_, virtRankMap_));
79 0 : HCCL_DEBUG("[InsCollAlgFactory] Rank[%d], [%s].", myRank_, topoMatch.Describe().c_str());
80 :
81 : // instantiate a template
82 0 : InsAlgTemplate tempAlg(myRank_, rankSize_, vTopo_, virtRankMap_);
83 0 : tempAlg.SetDmaMode(dmaMode_);
84 0 : tempAlg.SetCollOp(op); // CCU template需要传递op信息
85 :
86 : // calculate required insQues and prepare queue
87 0 : AlgTempResReq tempResReq;
88 0 : if (enableDetour_) {
89 0 : tempAlg.SetDataType(dataType_);
90 0 : CHK_RET(tempAlg.CalcResDetour(rankGraph, tempResReq));
91 : } else {
92 0 : CHK_RET(tempAlg.CalcRes(tempResReq));
93 : }
94 :
95 0 : CHK_RET(InitQueue(tempResReq.queNum, requiredQue_));
96 0 : HCCL_DEBUG("[InsCollAlgFactory] [InsAllGatherSoleExecutor] Rank[%d], template [%s], requiredQue Num [%u].", myRank_,
97 : tempAlg.Describe().c_str(), tempResReq.queNum);
98 :
99 0 : CHK_RET(PrepResLinks(myRank_, rankGraph, linkPriority_, tempResReq.links, tempResLinks_));
100 :
101 0 : u32 dataSizePerVolume = DataTypeSizeGet(dataType_);
102 0 : dataSize_ = dataCount_ * dataSizePerVolume;
103 :
104 0 : if (opMode_ == OpMode::OFFLOAD) {
105 0 : HCCL_DEBUG("[InsCollAlgFactory] [InsAllGatherSoleExecutor] Rank[%d], Generating Instruction Queues in OFFLOAD Mode for HOST.", myRank_);
106 0 : CHK_RET(GenInsQues4Offload(tempAlg));
107 : } else { // OPBASE
108 0 : HCCL_DEBUG("[InsCollAlgFactory] [InsAllGatherSoleExecutor] Rank[%d], Generating Instruction Queues in OPBASE Mode for HOST.", myRank_);
109 0 : CHK_RET(GenInsQues4Opbase(tempAlg));
110 : }
111 :
112 0 : return HcclResult::HCCL_SUCCESS;
113 0 : }
114 :
115 : template <typename AlgTopoMatch, typename InsAlgTemplate>
116 0 : HcclResult InsAllGatherSoleExecutor<AlgTopoMatch, InsAlgTemplate>::CalcRes(const RankGraph *rankGraph,
117 : CollAlgResReq &algResReq)
118 : {
119 : // Topo Match
120 0 : AlgTopoMatch topoMatch(myRank_, rankSize_, rankGraph, devType_);
121 0 : CHK_RET(topoMatch.MatchTopo(vTopo_, virtRanks_, virtRankMap_));
122 0 : algResReq.topoInfo.UpdateSingleLevelTopo(virtRanks_, virtRankMap_, vTopo_);
123 :
124 : // instantiate a template
125 0 : InsAlgTemplate tempAlg(myRank_, rankSize_, vTopo_, virtRankMap_);
126 :
127 : // calculate required insQues and prepare queue
128 0 : AlgTempResReq tempResReq;
129 0 : if (enableDetour_) {
130 0 : HCCL_DEBUG("Algorithm: Allgather. [%s] Rank[%d], CalcRes with detouring enabled.", __func__, myRank_);
131 0 : CHK_RET(tempAlg.CalcResDetour(rankGraph, tempResReq));
132 : } else {
133 0 : HCCL_DEBUG("Algorithm: Allgather. [%s] Rank[%d], CalcRes with detouring disabled.", __func__, myRank_);
134 0 : CHK_RET(tempAlg.CalcRes(tempResReq));
135 : }
136 0 : CHK_RET(CalcLinkInfo(myRank_, rankGraph, tempResReq.links, algResReq.levelRankPairs));
137 0 : algResReq.primQueueNum= tempResReq.streamNum;
138 0 : algResReq.queueNotifys = tempResReq.queNotifys;
139 0 : algResReq.localWaitGroupCntNotify = tempResReq.localWaitGroupCntNotify;
140 0 : algResReq.localBcastPostCntNotify = tempResReq.localBcastPostCntNotify;
141 0 : HCCL_DEBUG("[%s] Rank[%d], requiredQueNum [%u].", __func__, myRank_, algResReq.primQueueNum);
142 0 : CHK_RET(CalcResLinks(myRank_, rankGraph, linkPriority_, tempResReq.links, algResReq.links));
143 :
144 0 : return HcclResult::HCCL_SUCCESS;
145 0 : }
146 :
147 : // 算子执行aicpu接口
148 : template <typename AlgTopoMatch, typename InsAlgTemplate>
149 0 : HcclResult InsAllGatherSoleExecutor<AlgTopoMatch, InsAlgTemplate>::Orchestrate(const AlgTopoInfo &topoInfo,
150 : const CollAlgOperator &op,
151 : const CollAlgParams ¶ms,
152 : ConnectedLinkMgr *linkMgr,
153 : InsQuePtr insQue)
154 : {
155 : // init and check params
156 0 : CHK_RET(Init(op, params, insQue));
157 :
158 : // instantiate a template
159 0 : InsAlgTemplate tempAlg(myRank_, rankSize_, topoInfo.vTopo[0], topoInfo.virtRankMap[0]);
160 :
161 0 : tempAlg.SetDmaMode(dmaMode_);
162 0 : tempAlg.SetCollOp(op); // CCU template需要传递op信息
163 0 : virtRankMap_ = topoInfo.virtRankMap[0];
164 :
165 0 : HCCL_DEBUG("[InsCollAlgFactory] Rank[%d], Init insAlgTemplate with rankSize [%u] and dmaMode [%s].", myRank_,
166 : rankSize_, dmaMode_.Describe().c_str());
167 :
168 : // calculate required insQues and prepare queue
169 0 : AlgTempResReq tempResReq;
170 0 : if (enableDetour_) {
171 0 : tempAlg.SetDataType(dataType_);
172 0 : CHK_RET(tempAlg.CalcResDetour(linkMgr, tempResReq));
173 : } else {
174 0 : CHK_RET(tempAlg.CalcRes(tempResReq));
175 : }
176 :
177 0 : CHK_RET(InitQueue(tempResReq.queNum, requiredQue_));
178 0 : HCCL_DEBUG("[InsCollAlgFactory] Rank[%d], template [%s], requiredQue Num [%u] for AICPU.", myRank_,
179 : tempAlg.Describe().c_str(), tempResReq.queNum);
180 :
181 0 : CHK_RET(PrepResLinks(myRank_, tempResReq.links, linkMgr, tempResLinks_));
182 :
183 0 : u32 dataSizePerVolume = DataTypeSizeGet(dataType_);
184 0 : dataSize_ = dataCount_ * dataSizePerVolume;
185 :
186 0 : if (opMode_ == OpMode::OFFLOAD) {
187 0 : HCCL_DEBUG("[InsCollAlgFactory] Rank[%d], Generating Instruction Queues in OFFLOAD Mode for AICPU.", myRank_);
188 0 : CHK_RET(GenInsQues4Offload(tempAlg));
189 : } else { // OPBASE
190 0 : HCCL_DEBUG("[InsCollAlgFactory] Rank[%d], Generating Instruction Queues in OPBASE Mode for AICPU.", myRank_);
191 0 : CHK_RET(GenInsQues4Opbase(tempAlg));
192 : }
193 :
194 0 : return HcclResult::HCCL_SUCCESS;
195 0 : }
196 :
197 : template <typename AlgTopoMatch, typename InsAlgTemplate>
198 0 : HcclResult InsAllGatherSoleExecutor<AlgTopoMatch, InsAlgTemplate>::GenInsQues4Offload(InsAlgTemplate &tempAlg)
199 : {
200 0 : u32 dataSizePerVolume = DataTypeSizeGet(dataType_);
201 0 : CHK_PRT_RET(dataSizePerVolume == 0,
202 : HCCL_ERROR("[CollAlgFactory] Rank [%d], Invalid dataSizePerVolume [%u].", myRank_, dataSizePerVolume),
203 : HcclResult::HCCL_E_INTERNAL);
204 0 : CHK_PRT_RET(rankSize_ == 0, HCCL_ERROR("[CollAlgFactory] RankSize is zero!"), HcclResult::HCCL_E_PARA);
205 0 : u64 transportBoundDataSize = UB_MAX_DATA_SIZE; // algTemplate->CalcLoopMaxCount();
206 0 : BuffInfo buffInfo;
207 0 : buffInfo.inBuffType = BufferType::INPUT;
208 0 : buffInfo.outBuffType = BufferType::OUTPUT;
209 0 : buffInfo.inBuffBaseOff = 0;
210 0 : buffInfo.outBuffBaseOff = 0;
211 :
212 0 : HCCL_DEBUG("[InsCollAlgFactory] Rank[%d], input buffer type [%s], output buffer type [%s], input buffer base "
213 : "offset [%u], output buffer base offset [%u].",
214 : myRank_, buffInfo.inBuffType.Describe().c_str(), buffInfo.outBuffType.Describe().c_str(),
215 : buffInfo.inBuffBaseOff, buffInfo.outBuffBaseOff);
216 :
217 0 : u64 sendRecvTimes = (dataSize_ / transportBoundDataSize) + ((dataSize_ % transportBoundDataSize) == 0 ? 0 : 1);
218 0 : HCCL_DEBUG("[CollAlgFactory] Rank [%d], sendRecvTimes [%u].", myRank_, sendRecvTimes);
219 0 : for (u64 idx = 0; idx < sendRecvTimes; idx++) {
220 0 : u64 currDataSize = (idx == (sendRecvTimes - 1)) ? (dataSize_ - idx * transportBoundDataSize) : transportBoundDataSize; // 判断是否为最后一轮
221 0 : RankSliceInfo sliceInfoVec;
222 0 : AllignInfo allignInfo = {enableAllign_, allignSize_, dataType_};
223 0 : CHK_RET(tempAlg.CalcSliceInfo(allignInfo, currDataSize, sliceInfoVec));
224 0 : HCCL_DEBUG("[InsCollAlgFactory] Rank[%d], done calculating slice information.", myRank_);
225 :
226 0 : TempFuncs tempFuncs;
227 0 : tempFuncs.opMode = opMode_;
228 0 : tempFuncs.enableCounterNotify = IsEnableCounterNotifyByDevType(myRank_, devType_);
229 0 : tempFuncs.isForepart = true;
230 0 : tempFuncs.isBottom = true;
231 :
232 0 : UsrData usrData;
233 0 : DataSlice usrInSlice = DataSlice(BufferType::INPUT, idx * transportBoundDataSize, currDataSize);
234 0 : DataSlice scratchInSlice = DataSlice(BufferType::SCRATCH, virtRankMap_[myRank_] * currDataSize, currDataSize);
235 0 : usrData.usrInSlices.push_back(usrInSlice);
236 0 : usrData.scratchInSlices.push_back(scratchInSlice);
237 :
238 0 : for (u64 rankIdx = 0; rankIdx < rankSize_; rankIdx++) {
239 0 : DataSlice scratchOutSlice
240 0 : = DataSlice(BufferType::SCRATCH, virtRankMap_[rankIdx] * currDataSize, currDataSize);
241 0 : DataSlice usrOutSlice = DataSlice(
242 0 : BufferType::OUTPUT, virtRankMap_[rankIdx] * dataSize_ + idx * transportBoundDataSize, currDataSize);
243 0 : usrData.scratchOutSlices.push_back(scratchOutSlice);
244 0 : usrData.usrOutSlices.push_back(usrOutSlice);
245 : }
246 :
247 0 : tempFuncs.usrData = usrData;
248 0 : CHK_RET(tempAlg.Run(tempFuncs, sliceInfoVec, buffInfo, tempResLinks_, requiredQue_));
249 : }
250 :
251 0 : return HcclResult::HCCL_SUCCESS;
252 : }
253 :
254 : template <typename AlgTopoMatch, typename InsAlgTemplate>
255 0 : HcclResult InsAllGatherSoleExecutor<AlgTopoMatch, InsAlgTemplate>::GenInsQues4Opbase(InsAlgTemplate &tempAlg)
256 : {
257 0 : HCCL_DEBUG("[CollAlgFactory] AlgTemplate is [%s]", tempAlg.Describe().c_str());
258 0 : u32 dataSizePerVolume = DataTypeSizeGet(dataType_);
259 0 : CHK_PRT_RET(dataSizePerVolume == 0,
260 : HCCL_ERROR("[CollAlgFactory] Rank [%d], Invalid dataSizePerVolume [%u].", myRank_, dataSizePerVolume),
261 : HcclResult::HCCL_E_INTERNAL);
262 :
263 0 : CHK_PRT_RET(rankSize_ == 0, HCCL_ERROR("[CollAlgFactory] RankSize is zero!"), HcclResult::HCCL_E_PARA);
264 0 : u64 scratchInputMemSize =
265 0 : static_cast<u64>(floor(maxTmpMemSize_ / (rankSize_ * dataSizePerVolume)) * dataSizePerVolume);
266 0 : u64 transportBoundDataSize = UB_MAX_DATA_SIZE;
267 0 : scratchInputMemSize = min(scratchInputMemSize, transportBoundDataSize);
268 0 : HCCL_INFO("[InsCollAlgFactory] [InsAllGatherSoleExecutor] maxTmpMemSize_ [%u]", maxTmpMemSize_);
269 0 : CHK_PRT_RET(scratchInputMemSize == 0,
270 : HCCL_ERROR("[CollAlgFactory] Rank [%d], Invalid input maxTmpMemSize [%u].", myRank_, maxTmpMemSize_),
271 : HcclResult::HCCL_E_PARA);
272 :
273 0 : BuffInfo buffInfo;
274 0 : buffInfo.inBuffType = BufferType::SCRATCH;
275 0 : buffInfo.outBuffType = BufferType::SCRATCH;
276 0 : buffInfo.inBuffBaseOff = 0;
277 0 : buffInfo.outBuffBaseOff = 0;
278 :
279 0 : u64 sendRecvTimes = (dataSize_ / scratchInputMemSize) + ((dataSize_ % scratchInputMemSize) == 0 ? 0 : 1);
280 0 : HCCL_DEBUG("[InsCollAlgFactory] [InsAllGatherSoleExecutor] Rank [%d], sendRecvTimes [%u].", myRank_, sendRecvTimes);
281 :
282 0 : for (u64 idx = 0; idx < sendRecvTimes; idx++) {
283 0 : u64 currDataSize = (idx == (sendRecvTimes - 1)) ? (dataSize_ - idx * scratchInputMemSize) : scratchInputMemSize;
284 :
285 0 : RankSliceInfo sliceInfoVec;
286 0 : AllignInfo allignInfo = {enableAllign_, allignSize_, dataType_};
287 0 : CHK_RET(tempAlg.CalcSliceInfo(allignInfo, currDataSize, sliceInfoVec));
288 :
289 0 : TempFuncs tempFuncs;
290 0 : tempFuncs.opMode = opMode_;
291 0 : tempFuncs.enableCounterNotify = IsEnableCounterNotifyByDevType(myRank_, devType_);
292 0 : tempFuncs.isForepart = true; // Usr Buff to CCL Buff required
293 0 : tempFuncs.isBottom = true; // CCL Buff to Usr Buff required
294 :
295 0 : UsrData usrData;
296 0 : DataSlice usrInSlice = DataSlice(BufferType::INPUT, idx * scratchInputMemSize, currDataSize);
297 0 : DataSlice scratchInSlice = DataSlice(BufferType::SCRATCH, virtRankMap_[myRank_] * currDataSize, currDataSize);
298 0 : usrData.usrInSlices.push_back(usrInSlice);
299 0 : usrData.scratchInSlices.push_back(scratchInSlice);
300 :
301 0 : for (u64 rankIdx = 0; rankIdx < rankSize_; rankIdx++) {
302 0 : DataSlice scratchOutSlice
303 0 : = DataSlice(BufferType::SCRATCH, virtRankMap_[rankIdx] * currDataSize, currDataSize);
304 0 : DataSlice usrOutSlice = DataSlice(
305 0 : BufferType::OUTPUT, virtRankMap_[rankIdx] * dataSize_ + idx * scratchInputMemSize, currDataSize);
306 0 : usrData.scratchOutSlices.push_back(scratchOutSlice);
307 0 : usrData.usrOutSlices.push_back(usrOutSlice);
308 : }
309 :
310 0 : tempFuncs.usrData = usrData;
311 0 : CHK_RET(tempAlg.Run(tempFuncs, sliceInfoVec, buffInfo, tempResLinks_, requiredQue_));
312 : }
313 :
314 0 : return HcclResult::HCCL_SUCCESS;
315 : }
316 :
317 :
318 : #ifndef CCL_KERNEL_AICPU
319 : INS_REGISTER_IMPL_BY_TEMP(OpType::ALLGATHER, CcuAllGatherMesh1D, InsAllGatherSoleExecutor, TopoMatchMesh,
320 : CcuTempAllGatherMesh1D);
321 : INS_REGISTER_IMPL_BY_TEMP(OpType::ALLGATHER, CcuAllGatherMeshDetour1D, InsAllGatherSoleExecutor, TopoMatchMesh,
322 : CcuTempAllGatherMeshDetour1D);
323 : INS_REGISTER_IMPL_BY_TEMP(OpType::ALLGATHER, CcuAllGatherMeshMem2Mem1D, InsAllGatherSoleExecutor, TopoMatchMesh,
324 : CcuTempAllGatherMeshMem2Mem1D);
325 : INS_REGISTER_IMPL_BY_TEMP(OpType::ALLGATHER, CcuAllGatherMesh2D, InsAllGatherSoleExecutor, TopoMatchConcurrMesh,
326 : CcuTempAllGatherMesh2D);
327 : #endif
328 : } // namespace Hccl
|
