Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include <cmath>
12 :
13 : #include "ins_all_gather_sole_executor.h"
14 :
15 : #include "log.h"
16 : #include "ins_coll_alg_registry.h"
17 :
18 : #ifndef CCL_KERNEL_AICPU
19 : #include "ccu_temp_all_gather_mesh_1D.h"
20 : #include "ccu_temp_all_gather_mesh_1D_detour.h"
21 : #include "ccu_temp_all_gather_mesh_1D_mem2mem.h"
22 : #include "ccu_temp_all_gather_mesh_2D.h"
23 : #endif
24 :
25 : #include "topo_match_mesh.h"
26 : #include "topo_match_concurr_mesh.h"
27 :
28 : namespace Hccl {
29 : template <typename AlgTopoMatch, typename InsAlgTemplate>
30 0 : InsAllGatherSoleExecutor<AlgTopoMatch, InsAlgTemplate>::InsAllGatherSoleExecutor() : InsCollAlgBase()
31 0 : {}
32 :
33 : template <typename AlgTopoMatch, typename InsAlgTemplate>
34 0 : InsAllGatherSoleExecutor<AlgTopoMatch, InsAlgTemplate>::~InsAllGatherSoleExecutor()
35 0 : {}
36 :
37 : template <typename AlgTopoMatch, typename InsAlgTemplate>
38 0 : HcclResult InsAllGatherSoleExecutor<AlgTopoMatch, InsAlgTemplate>::CalcResOffload(
39 : const RankGraph* rankGraph, const u64& dataSize, CollOffloadOpResReq& resReq)
40 : {
41 : (void)dataSize;
42 0 : resReq.requiredScratchMemSize = 0;
43 : // Topo Match
44 0 : AlgTopoMatch topoMatch(myRank_, rankSize_, rankGraph, devType_);
45 0 : CHK_RET(topoMatch.MatchTopo(vTopo_, virtRanks_, virtRankMap_));
46 :
47 : // instantiate a template
48 0 : InsAlgTemplate tempAlg(myRank_, rankSize_, vTopo_, virtRankMap_);
49 :
50 : // calculate required insQueues and prepare queue
51 0 : AlgTempResReq tempResReq;
52 0 : if (enableDetour_) {
53 0 : CHK_RET(tempAlg.CalcResDetour(rankGraph, tempResReq));
54 : } else {
55 0 : CHK_RET(tempAlg.CalcRes(tempResReq));
56 : }
57 :
58 0 : resReq.requiredSubQueNum = tempResReq.streamNum - 1;
59 :
60 0 : return HcclResult::HCCL_SUCCESS;
61 0 : }
62 :
63 : // dataSize_ as input
64 : template <typename AlgTopoMatch, typename InsAlgTemplate>
65 0 : HcclResult InsAllGatherSoleExecutor<AlgTopoMatch, InsAlgTemplate>::Orchestrate(
66 : const RankGraph* rankGraph, const CollAlgOperator& op, const CollAlgParams& params, InsQuePtr insQue)
67 : {
68 : // init and check params
69 0 : CHK_RET(Init(op, params, insQue));
70 :
71 : // Topo Match
72 0 : AlgTopoMatch topoMatch(myRank_, rankSize_, rankGraph, devType_);
73 0 : CHK_RET(topoMatch.MatchTopo(vTopo_, virtRanks_, virtRankMap_));
74 0 : HCCL_DEBUG("[InsCollAlgFactory] Rank[%d], [%s].", myRank_, topoMatch.Describe().c_str());
75 :
76 : // instantiate a template
77 0 : InsAlgTemplate tempAlg(myRank_, rankSize_, vTopo_, virtRankMap_);
78 0 : tempAlg.SetDmaMode(dmaMode_);
79 0 : tempAlg.SetCollOp(op); // CCU template需要传递op信息
80 :
81 : // calculate required insQues and prepare queue
82 0 : AlgTempResReq tempResReq;
83 0 : if (enableDetour_) {
84 0 : tempAlg.SetDataType(dataType_);
85 0 : CHK_RET(tempAlg.CalcResDetour(rankGraph, tempResReq));
86 : } else {
87 0 : CHK_RET(tempAlg.CalcRes(tempResReq));
88 : }
89 :
90 0 : CHK_RET(InitQueue(tempResReq.queNum, requiredQue_));
91 0 : HCCL_DEBUG(
92 : "[InsCollAlgFactory] [InsAllGatherSoleExecutor] Rank[%d], template [%s], requiredQue Num [%u].", myRank_,
93 : tempAlg.Describe().c_str(), tempResReq.queNum);
94 :
95 0 : CHK_RET(PrepResLinks(myRank_, rankGraph, linkPriority_, tempResReq.links, tempResLinks_));
96 :
97 0 : u32 dataSizePerVolume = DataTypeSizeGet(dataType_);
98 0 : dataSize_ = dataCount_ * dataSizePerVolume;
99 :
100 0 : if (opMode_ == OpMode::OFFLOAD) {
101 0 : HCCL_DEBUG(
102 : "[InsCollAlgFactory] [InsAllGatherSoleExecutor] Rank[%d], Generating Instruction Queues in OFFLOAD Mode "
103 : "for HOST.",
104 : myRank_);
105 0 : CHK_RET(GenInsQues4Offload(tempAlg));
106 : } else { // OPBASE
107 0 : HCCL_DEBUG(
108 : "[InsCollAlgFactory] [InsAllGatherSoleExecutor] Rank[%d], Generating Instruction Queues in OPBASE Mode for "
109 : "HOST.",
110 : myRank_);
111 0 : CHK_RET(GenInsQues4Opbase(tempAlg));
112 : }
113 :
114 0 : return HcclResult::HCCL_SUCCESS;
115 0 : }
116 :
117 : template <typename AlgTopoMatch, typename InsAlgTemplate>
118 : HcclResult
119 0 : InsAllGatherSoleExecutor<AlgTopoMatch, InsAlgTemplate>::CalcRes(const RankGraph* rankGraph, CollAlgResReq& algResReq)
120 : {
121 : // Topo Match
122 0 : AlgTopoMatch topoMatch(myRank_, rankSize_, rankGraph, devType_);
123 0 : CHK_RET(topoMatch.MatchTopo(vTopo_, virtRanks_, virtRankMap_));
124 0 : algResReq.topoInfo.UpdateSingleLevelTopo(virtRanks_, virtRankMap_, vTopo_);
125 :
126 : // instantiate a template
127 0 : InsAlgTemplate tempAlg(myRank_, rankSize_, vTopo_, virtRankMap_);
128 :
129 : // calculate required insQues and prepare queue
130 0 : AlgTempResReq tempResReq;
131 0 : if (enableDetour_) {
132 0 : HCCL_DEBUG("Algorithm: Allgather. [%s] Rank[%d], CalcRes with detouring enabled.", __func__, myRank_);
133 0 : CHK_RET(tempAlg.CalcResDetour(rankGraph, tempResReq));
134 : } else {
135 0 : HCCL_DEBUG("Algorithm: Allgather. [%s] Rank[%d], CalcRes with detouring disabled.", __func__, myRank_);
136 0 : CHK_RET(tempAlg.CalcRes(tempResReq));
137 : }
138 0 : CHK_RET(CalcLinkInfo(myRank_, rankGraph, tempResReq.links, algResReq.levelRankPairs));
139 0 : algResReq.primQueueNum = tempResReq.streamNum;
140 0 : algResReq.queueNotifys = tempResReq.queNotifys;
141 0 : algResReq.localWaitGroupCntNotify = tempResReq.localWaitGroupCntNotify;
142 0 : algResReq.localBcastPostCntNotify = tempResReq.localBcastPostCntNotify;
143 0 : HCCL_DEBUG("[%s] Rank[%d], requiredQueNum [%u].", __func__, myRank_, algResReq.primQueueNum);
144 0 : CHK_RET(CalcResLinks(myRank_, rankGraph, linkPriority_, tempResReq.links, algResReq.links));
145 :
146 0 : return HcclResult::HCCL_SUCCESS;
147 0 : }
148 :
149 : // 算子执行aicpu接口
150 : template <typename AlgTopoMatch, typename InsAlgTemplate>
151 0 : HcclResult InsAllGatherSoleExecutor<AlgTopoMatch, InsAlgTemplate>::Orchestrate(
152 : const AlgTopoInfo& topoInfo, const CollAlgOperator& op, const CollAlgParams& params, ConnectedLinkMgr* linkMgr,
153 : InsQuePtr insQue)
154 : {
155 : // init and check params
156 0 : CHK_RET(Init(op, params, insQue));
157 :
158 : // instantiate a template
159 0 : InsAlgTemplate tempAlg(myRank_, rankSize_, topoInfo.vTopo[0], topoInfo.virtRankMap[0]);
160 :
161 0 : tempAlg.SetDmaMode(dmaMode_);
162 0 : tempAlg.SetCollOp(op); // CCU template需要传递op信息
163 0 : virtRankMap_ = topoInfo.virtRankMap[0];
164 :
165 0 : HCCL_DEBUG(
166 : "[InsCollAlgFactory] Rank[%d], Init insAlgTemplate with rankSize [%u] and dmaMode [%s].", myRank_, rankSize_,
167 : dmaMode_.Describe().c_str());
168 :
169 : // calculate required insQues and prepare queue
170 0 : AlgTempResReq tempResReq;
171 0 : if (enableDetour_) {
172 0 : tempAlg.SetDataType(dataType_);
173 0 : CHK_RET(tempAlg.CalcResDetour(linkMgr, tempResReq));
174 : } else {
175 0 : CHK_RET(tempAlg.CalcRes(tempResReq));
176 : }
177 :
178 0 : CHK_RET(InitQueue(tempResReq.queNum, requiredQue_));
179 0 : HCCL_DEBUG(
180 : "[InsCollAlgFactory] Rank[%d], template [%s], requiredQue Num [%u] for AICPU.", myRank_,
181 : tempAlg.Describe().c_str(), tempResReq.queNum);
182 :
183 0 : CHK_RET(PrepResLinks(myRank_, tempResReq.links, linkMgr, tempResLinks_));
184 :
185 0 : u32 dataSizePerVolume = DataTypeSizeGet(dataType_);
186 0 : dataSize_ = dataCount_ * dataSizePerVolume;
187 :
188 0 : if (opMode_ == OpMode::OFFLOAD) {
189 0 : HCCL_DEBUG("[InsCollAlgFactory] Rank[%d], Generating Instruction Queues in OFFLOAD Mode for AICPU.", myRank_);
190 0 : CHK_RET(GenInsQues4Offload(tempAlg));
191 : } else { // OPBASE
192 0 : HCCL_DEBUG("[InsCollAlgFactory] Rank[%d], Generating Instruction Queues in OPBASE Mode for AICPU.", myRank_);
193 0 : CHK_RET(GenInsQues4Opbase(tempAlg));
194 : }
195 :
196 0 : return HcclResult::HCCL_SUCCESS;
197 0 : }
198 :
199 : template <typename AlgTopoMatch, typename InsAlgTemplate>
200 0 : HcclResult InsAllGatherSoleExecutor<AlgTopoMatch, InsAlgTemplate>::GenInsQues4Offload(InsAlgTemplate& tempAlg)
201 : {
202 0 : u32 dataSizePerVolume = DataTypeSizeGet(dataType_);
203 0 : CHK_PRT_RET(
204 : dataSizePerVolume == 0,
205 : HCCL_ERROR("[CollAlgFactory] Rank [%d], Invalid dataSizePerVolume [%u].", myRank_, dataSizePerVolume),
206 : HcclResult::HCCL_E_INTERNAL);
207 0 : CHK_PRT_RET(rankSize_ == 0, HCCL_ERROR("[CollAlgFactory] RankSize is zero!"), HcclResult::HCCL_E_PARA);
208 0 : u64 transportBoundDataSize = UB_MAX_DATA_SIZE; // algTemplate->CalcLoopMaxCount();
209 0 : BuffInfo buffInfo;
210 0 : buffInfo.inBuffType = BufferType::INPUT;
211 0 : buffInfo.outBuffType = BufferType::OUTPUT;
212 0 : buffInfo.inBuffBaseOff = 0;
213 0 : buffInfo.outBuffBaseOff = 0;
214 :
215 0 : HCCL_DEBUG(
216 : "[InsCollAlgFactory] Rank[%d], input buffer type [%s], output buffer type [%s], input buffer base "
217 : "offset [%u], output buffer base offset [%u].",
218 : myRank_, buffInfo.inBuffType.Describe().c_str(), buffInfo.outBuffType.Describe().c_str(),
219 : buffInfo.inBuffBaseOff, buffInfo.outBuffBaseOff);
220 :
221 0 : u64 sendRecvTimes = (dataSize_ / transportBoundDataSize) + ((dataSize_ % transportBoundDataSize) == 0 ? 0 : 1);
222 0 : HCCL_DEBUG("[CollAlgFactory] Rank [%d], sendRecvTimes [%u].", myRank_, sendRecvTimes);
223 0 : for (u64 idx = 0; idx < sendRecvTimes; idx++) {
224 0 : u64 currDataSize = (idx == (sendRecvTimes - 1)) ? (dataSize_ - idx * transportBoundDataSize) :
225 : transportBoundDataSize; // 判断是否为最后一轮
226 0 : RankSliceInfo sliceInfoVec;
227 0 : AllignInfo allignInfo = {enableAllign_, allignSize_, dataType_};
228 0 : CHK_RET(tempAlg.CalcSliceInfo(allignInfo, currDataSize, sliceInfoVec));
229 0 : HCCL_DEBUG("[InsCollAlgFactory] Rank[%d], done calculating slice information.", myRank_);
230 :
231 0 : TempFuncs tempFuncs;
232 0 : tempFuncs.opMode = opMode_;
233 0 : tempFuncs.enableCounterNotify = IsEnableCounterNotifyByDevType(myRank_, devType_);
234 0 : tempFuncs.isForepart = true;
235 0 : tempFuncs.isBottom = true;
236 :
237 0 : UsrData usrData;
238 0 : DataSlice usrInSlice = DataSlice(BufferType::INPUT, idx * transportBoundDataSize, currDataSize);
239 0 : DataSlice scratchInSlice = DataSlice(BufferType::SCRATCH, virtRankMap_[myRank_] * currDataSize, currDataSize);
240 0 : usrData.usrInSlices.push_back(usrInSlice);
241 0 : usrData.scratchInSlices.push_back(scratchInSlice);
242 :
243 0 : for (u64 rankIdx = 0; rankIdx < rankSize_; rankIdx++) {
244 0 : DataSlice scratchOutSlice
245 0 : = DataSlice(BufferType::SCRATCH, virtRankMap_[rankIdx] * currDataSize, currDataSize);
246 0 : DataSlice usrOutSlice = DataSlice(
247 0 : BufferType::OUTPUT, virtRankMap_[rankIdx] * dataSize_ + idx * transportBoundDataSize, currDataSize);
248 0 : usrData.scratchOutSlices.push_back(scratchOutSlice);
249 0 : usrData.usrOutSlices.push_back(usrOutSlice);
250 : }
251 :
252 0 : tempFuncs.usrData = usrData;
253 0 : CHK_RET(tempAlg.Run(tempFuncs, sliceInfoVec, buffInfo, tempResLinks_, requiredQue_));
254 : }
255 :
256 0 : return HcclResult::HCCL_SUCCESS;
257 : }
258 :
259 : template <typename AlgTopoMatch, typename InsAlgTemplate>
260 0 : HcclResult InsAllGatherSoleExecutor<AlgTopoMatch, InsAlgTemplate>::GenInsQues4Opbase(InsAlgTemplate& tempAlg)
261 : {
262 0 : HCCL_DEBUG("[CollAlgFactory] AlgTemplate is [%s]", tempAlg.Describe().c_str());
263 0 : u32 dataSizePerVolume = DataTypeSizeGet(dataType_);
264 0 : CHK_PRT_RET(
265 : dataSizePerVolume == 0,
266 : HCCL_ERROR("[CollAlgFactory] Rank [%d], Invalid dataSizePerVolume [%u].", myRank_, dataSizePerVolume),
267 : HcclResult::HCCL_E_INTERNAL);
268 :
269 0 : CHK_PRT_RET(rankSize_ == 0, HCCL_ERROR("[CollAlgFactory] RankSize is zero!"), HcclResult::HCCL_E_PARA);
270 0 : u64 scratchInputMemSize
271 0 : = static_cast<u64>(floor(maxTmpMemSize_ / (rankSize_ * dataSizePerVolume)) * dataSizePerVolume);
272 0 : u64 transportBoundDataSize = UB_MAX_DATA_SIZE;
273 0 : scratchInputMemSize = min(scratchInputMemSize, transportBoundDataSize);
274 0 : HCCL_INFO("[InsCollAlgFactory] [InsAllGatherSoleExecutor] maxTmpMemSize_ [%u]", maxTmpMemSize_);
275 0 : CHK_PRT_RET(
276 : scratchInputMemSize == 0,
277 : HCCL_ERROR("[CollAlgFactory] Rank [%d], Invalid input maxTmpMemSize [%u].", myRank_, maxTmpMemSize_),
278 : HcclResult::HCCL_E_PARA);
279 :
280 0 : BuffInfo buffInfo;
281 0 : buffInfo.inBuffType = BufferType::SCRATCH;
282 0 : buffInfo.outBuffType = BufferType::SCRATCH;
283 0 : buffInfo.inBuffBaseOff = 0;
284 0 : buffInfo.outBuffBaseOff = 0;
285 :
286 0 : u64 sendRecvTimes = (dataSize_ / scratchInputMemSize) + ((dataSize_ % scratchInputMemSize) == 0 ? 0 : 1);
287 0 : HCCL_DEBUG("[InsCollAlgFactory] [InsAllGatherSoleExecutor] Rank [%d], sendRecvTimes [%u].", myRank_, sendRecvTimes);
288 :
289 0 : for (u64 idx = 0; idx < sendRecvTimes; idx++) {
290 0 : u64 currDataSize = (idx == (sendRecvTimes - 1)) ? (dataSize_ - idx * scratchInputMemSize) : scratchInputMemSize;
291 :
292 0 : RankSliceInfo sliceInfoVec;
293 0 : AllignInfo allignInfo = {enableAllign_, allignSize_, dataType_};
294 0 : CHK_RET(tempAlg.CalcSliceInfo(allignInfo, currDataSize, sliceInfoVec));
295 :
296 0 : TempFuncs tempFuncs;
297 0 : tempFuncs.opMode = opMode_;
298 0 : tempFuncs.enableCounterNotify = IsEnableCounterNotifyByDevType(myRank_, devType_);
299 0 : tempFuncs.isForepart = true; // Usr Buff to CCL Buff required
300 0 : tempFuncs.isBottom = true; // CCL Buff to Usr Buff required
301 :
302 0 : UsrData usrData;
303 0 : DataSlice usrInSlice = DataSlice(BufferType::INPUT, idx * scratchInputMemSize, currDataSize);
304 0 : DataSlice scratchInSlice = DataSlice(BufferType::SCRATCH, virtRankMap_[myRank_] * currDataSize, currDataSize);
305 0 : usrData.usrInSlices.push_back(usrInSlice);
306 0 : usrData.scratchInSlices.push_back(scratchInSlice);
307 :
308 0 : for (u64 rankIdx = 0; rankIdx < rankSize_; rankIdx++) {
309 0 : DataSlice scratchOutSlice
310 0 : = DataSlice(BufferType::SCRATCH, virtRankMap_[rankIdx] * currDataSize, currDataSize);
311 0 : DataSlice usrOutSlice = DataSlice(
312 0 : BufferType::OUTPUT, virtRankMap_[rankIdx] * dataSize_ + idx * scratchInputMemSize, currDataSize);
313 0 : usrData.scratchOutSlices.push_back(scratchOutSlice);
314 0 : usrData.usrOutSlices.push_back(usrOutSlice);
315 : }
316 :
317 0 : tempFuncs.usrData = usrData;
318 0 : CHK_RET(tempAlg.Run(tempFuncs, sliceInfoVec, buffInfo, tempResLinks_, requiredQue_));
319 : }
320 :
321 0 : return HcclResult::HCCL_SUCCESS;
322 : }
323 :
324 : #ifndef CCL_KERNEL_AICPU
325 : INS_REGISTER_IMPL_BY_TEMP(
326 : OpType::ALLGATHER, CcuAllGatherMesh1D, InsAllGatherSoleExecutor, TopoMatchMesh, CcuTempAllGatherMesh1D);
327 : INS_REGISTER_IMPL_BY_TEMP(
328 : OpType::ALLGATHER, CcuAllGatherMeshDetour1D, InsAllGatherSoleExecutor, TopoMatchMesh, CcuTempAllGatherMeshDetour1D);
329 : INS_REGISTER_IMPL_BY_TEMP(
330 : OpType::ALLGATHER, CcuAllGatherMeshMem2Mem1D, InsAllGatherSoleExecutor, TopoMatchMesh,
331 : CcuTempAllGatherMeshMem2Mem1D);
332 : INS_REGISTER_IMPL_BY_TEMP(
333 : OpType::ALLGATHER, CcuAllGatherMesh2D, InsAllGatherSoleExecutor, TopoMatchConcurrMesh, CcuTempAllGatherMesh2D);
334 : #endif
335 : } // namespace Hccl
|
