Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "log.h"
12 : #include "ins_coll_alg_registry.h"
13 : #include "topo_match_concurr_mesh.h"
14 : #include "topo_match_mesh.h"
15 : #include "topo_match_nhr.h"
16 :
17 : #include "ins_temp_all_reduce_mesh_1D_two_shot.h"
18 : #include "ins_temp_all_reduce_mesh_2D_two_shot.h"
19 : #include "ins_temp_all_reduce_mesh_1D_one_shot.h"
20 : #include "ins_temp_all_reduce_mesh_1D_two_shot_mesh_chunk.h"
21 : #include "ins_temp_all_reduce_nhr.h"
22 : #include "ins_v2_all_reduce_sole_executor.h"
23 : #ifndef CCL_KERNEL_AICPU
24 : #include "aiv_temp_all_reduce_mesh_1D_oneshot.h"
25 : #include "aiv_temp_all_reduce_mesh_1D_twoshot.h"
26 : #include "ccu_temp_all_reduce_nhr_1D_mem2mem.h"
27 : #include "ccu_temp_all_reduce_mesh_1D_mem2mem.h"
28 : #endif
29 : #include "ins_temp_all_reduce_aicpu_reduce.h"
30 : #include "ins_temp_all_reduce_aicpu_reduce_mesh_2D.h"
31 :
32 : namespace Hccl {
33 : constexpr u64 MAX_OFFLOAD_SCRATCH_SIZE = 200 * 1024 * 1024; // 200M
34 :
35 : template <typename AlgTopoMatch, typename InsAlgTemplate>
36 0 : InsV2AllReduceSoleExecutor<AlgTopoMatch, InsAlgTemplate>::InsV2AllReduceSoleExecutor() : InsCollAlgBase()
37 0 : {}
38 :
39 : template <typename AlgTopoMatch, typename InsAlgTemplate>
40 0 : InsV2AllReduceSoleExecutor<AlgTopoMatch, InsAlgTemplate>::~InsV2AllReduceSoleExecutor()
41 0 : {}
42 :
43 : template <typename AlgTopoMatch, typename InsAlgTemplate>
44 0 : HcclResult InsV2AllReduceSoleExecutor<AlgTopoMatch, InsAlgTemplate>::InitCommInfo(const RankGraph* rankGraph)
45 : {
46 0 : AlgTopoMatch topoMatch(myRank_, rankSize_, rankGraph, devType_);
47 0 : CHK_RET(topoMatch.MatchTopo(vTopo_, virtRanks_, virtRankMap_));
48 0 : return HcclResult::HCCL_SUCCESS;
49 0 : }
50 :
51 : template <typename AlgTopoMatch, typename InsAlgTemplate>
52 0 : HcclResult InsV2AllReduceSoleExecutor<AlgTopoMatch, InsAlgTemplate>::InitCommInfo(const AlgTopoInfo& topoInfo)
53 : {
54 0 : CHK_PRT_RET(
55 : topoInfo.vTopo.size() == 0 || topoInfo.virtRankMap.size() == 0 || topoInfo.virtRanks.size() == 0,
56 : HCCL_ERROR("[InsV2AllReduceSoleExecutor][InitCommInfo] topoInfo vector member size is 0 !"), HCCL_E_INTERNAL);
57 0 : vTopo_ = topoInfo.vTopo[0]; // 本通信域内的通信平面
58 0 : virtRankMap_ = topoInfo.virtRankMap[0]; // 本通信域内的 rank 映射表
59 0 : virtRanks_ = topoInfo.virtRanks[0]; // 本通信域内的 rank 集合
60 0 : return HcclResult::HCCL_SUCCESS;
61 : }
62 :
63 : template <typename AlgTopoMatch, typename InsAlgTemplate>
64 0 : HcclResult InsV2AllReduceSoleExecutor<AlgTopoMatch, InsAlgTemplate>::CreateTemplates(
65 : std::shared_ptr<InsAlgTemplate>& algTemplatePtr)
66 : {
67 0 : algTemplatePtr = std::make_shared<InsAlgTemplate>(myRank_, rankSize_, vTopo_, virtRankMap_);
68 0 : CHK_PTR_NULL(algTemplatePtr); // 检查是否成功分配内存
69 0 : algTemplatePtr->SetDmaMode(dmaMode_);
70 0 : algTemplatePtr->InitReduceInfo(redOp_, dataType_);
71 0 : algTemplatePtr->SetCollOp(op_);
72 0 : return HcclResult::HCCL_SUCCESS;
73 : }
74 :
75 : template <typename AlgTopoMatch, typename InsAlgTemplate>
76 0 : HcclResult InsV2AllReduceSoleExecutor<AlgTopoMatch, InsAlgTemplate>::GetTemplateResRequest(
77 : const RankGraph* rankGraph, std::shared_ptr<InsAlgTemplate>& algTemplate, AlgTempResReq& tempResReq) const
78 : {
79 0 : if (enableDetour_) {
80 0 : HCCL_DEBUG("[%s] Rank[%d]. CalcRes with detouring enabled.", __func__, myRank_);
81 0 : CHK_RET(algTemplate->CalcResDetour(rankGraph, tempResReq));
82 : } else {
83 0 : HCCL_DEBUG("[%s] Rank[%d]. CalcRes with detouring disabled.", __func__, myRank_);
84 0 : CHK_RET(algTemplate->CalcRes(tempResReq));
85 : }
86 0 : return HcclResult::HCCL_SUCCESS;
87 : }
88 :
89 : template <typename AlgTopoMatch, typename InsAlgTemplate>
90 0 : HcclResult InsV2AllReduceSoleExecutor<AlgTopoMatch, InsAlgTemplate>::GetTemplateResRequest(
91 : ConnectedLinkMgr* linkMgr, std::shared_ptr<InsAlgTemplate>& algTemplate, AlgTempResReq& tempResReq) const
92 : {
93 0 : if (enableDetour_) {
94 0 : HCCL_DEBUG("[InsV2AllReduceSoleExecutor] Rank[%d], CalcRes with detouring enabled.", myRank_);
95 0 : CHK_RET(algTemplate->CalcResDetour(linkMgr, tempResReq));
96 : } else {
97 0 : HCCL_DEBUG("[InsV2AllReduceSoleExecutor] Rank[%d], CalcRes with detouring disabled.", myRank_);
98 0 : CHK_RET(algTemplate->CalcRes(tempResReq));
99 : }
100 0 : return HcclResult::HCCL_SUCCESS;
101 : }
102 :
103 : template <typename AlgTopoMatch, typename InsAlgTemplate>
104 : HcclResult
105 0 : InsV2AllReduceSoleExecutor<AlgTopoMatch, InsAlgTemplate>::CalcRes(const RankGraph* rankGraph, CollAlgResReq& algResReq)
106 : {
107 : // Topo Match
108 0 : CHK_RET(InitCommInfo(rankGraph));
109 :
110 0 : std::shared_ptr<InsAlgTemplate> algTemplate = nullptr;
111 0 : CHK_RET(CreateTemplates(algTemplate));
112 :
113 : // 通过判断哪层通信域能有到所有remoteRank的path，判断当前算法跑在哪一层
114 0 : std::map<u32, u32> rank2PathNumMap;
115 0 : HCCL_INFO("[InsV2AllReduceSoleExecutor] CalcRes SetPathNumMap");
116 0 : CHK_RET(SetPathNumMapByRankGraphMultiLevel(rankGraph, virtRanks_, myRank_, rank2PathNumMap));
117 0 : algTemplate->setPathNumMap(rank2PathNumMap);
118 0 : AlgTempResReq tempResReq;
119 0 : CHK_RET(GetTemplateResRequest(rankGraph, algTemplate, tempResReq));
120 :
121 0 : CHK_RET(CalcLinkInfo(myRank_, rankGraph, tempResReq.links, algResReq.levelRankPairs));
122 0 : algResReq.topoInfo.UpdateSingleLevelTopo(virtRanks_, virtRankMap_, vTopo_);
123 0 : algResReq.primQueueNum = tempResReq.streamNum;
124 0 : algResReq.localWaitGroupCntNotify = tempResReq.localWaitGroupCntNotify;
125 0 : algResReq.localBcastPostCntNotify = tempResReq.localBcastPostCntNotify;
126 0 : algResReq.queueNotifys = tempResReq.queNotifys;
127 0 : HCCL_DEBUG("[%s] Rank[%d], requiredQueNum [%u].", __func__, myRank_, algResReq.primQueueNum);
128 0 : CHK_RET(CalcResLinks(myRank_, rankGraph, linkPriority_, tempResReq.links, algResReq.links));
129 :
130 0 : return HcclResult::HCCL_SUCCESS;
131 0 : }
132 :
133 : template <typename AlgTopoMatch, typename InsAlgTemplate>
134 0 : HcclResult InsV2AllReduceSoleExecutor<AlgTopoMatch, InsAlgTemplate>::CalcResOffload(
135 : const RankGraph* rankGraph, const u64& dataSize, CollOffloadOpResReq& resReq)
136 : {
137 : (void)dataSize;
138 :
139 : // Topo Match
140 0 : CHK_RET(InitCommInfo(rankGraph));
141 :
142 0 : std::shared_ptr<InsAlgTemplate> algTemplate = nullptr;
143 0 : CHK_RET(CreateTemplates(algTemplate));
144 :
145 : // 通过判断哪层通信域能有到所有remoteRank的path，判断当前算法跑在哪一层
146 0 : std::map<u32, u32> rank2PathNumMap;
147 0 : HCCL_INFO("[InsV2AllReduceSoleExecutor] CalcResOffload SetPathNumMap");
148 0 : CHK_RET(SetPathNumMapByRankGraphMultiLevel(rankGraph, virtRanks_, myRank_, rank2PathNumMap));
149 0 : algTemplate->setPathNumMap(rank2PathNumMap);
150 0 : AlgTempResReq tempResReq;
151 0 : CHK_RET(GetTemplateResRequest(rankGraph, algTemplate, tempResReq));
152 0 : u64 transportBoundDataSize = UB_MAX_DATA_SIZE;
153 0 : resReq.requiredScratchMemSize = transportBoundDataSize;
154 0 : resReq.requiredSubQueNum = tempResReq.streamNum - 1;
155 :
156 0 : return HcclResult::HCCL_SUCCESS;
157 0 : }
158 :
159 : template <typename AlgTopoMatch, typename InsAlgTemplate>
160 : HcclResult
161 0 : InsV2AllReduceSoleExecutor<AlgTopoMatch, InsAlgTemplate>::CalNumBlocks(u32& numBlocks, u64 dataSize, u32 numBlocksLimit)
162 : {
163 0 : std::shared_ptr<InsAlgTemplate> algTemplate = nullptr;
164 0 : CHK_RET(CreateTemplates(algTemplate));
165 0 : algTemplate->CalNumBlocks(numBlocks, dataSize, numBlocksLimit);
166 0 : return HcclResult::HCCL_SUCCESS;
167 0 : }
168 : // HOST 侧算法入口
169 : template <typename AlgTopoMatch, typename InsAlgTemplate>
170 0 : HcclResult InsV2AllReduceSoleExecutor<AlgTopoMatch, InsAlgTemplate>::Orchestrate(
171 : const RankGraph* rankGraph, const CollAlgOperator& op, const CollAlgParams& params, InsQuePtr insQue)
172 : {
173 0 : HCCL_INFO("[InsV2AllReduceSoleExecutor][Orchestrate] Orchestrate HOST Start.");
174 0 : CHK_RET(Init(op, params, insQue));
175 0 : CHK_RET(InitCommInfo(rankGraph));
176 :
177 0 : std::shared_ptr<InsAlgTemplate> algTemplate = nullptr;
178 0 : CHK_RET(CreateTemplates(algTemplate));
179 :
180 0 : AlgTempResReq tempResReq;
181 0 : CHK_RET(GetTemplateResRequest(rankGraph, algTemplate, tempResReq));
182 :
183 0 : HCCL_DEBUG(
184 : "[InsV2AllReduceSoleExecutor][Orchestrate] Rank[%d], template [%s], requiredQue Num [%u].", myRank_,
185 : algTemplate->Describe().c_str(), tempResReq.queNum);
186 0 : CHK_RET(InitQueue(tempResReq.queNum, tempInsQue_));
187 0 : CHK_RET(PrepResLinks(myRank_, rankGraph, linkPriority_, tempResReq.links, tempResLinks_));
188 0 : CHK_RET(OrchestrateLoop(algTemplate));
189 0 : return HcclResult::HCCL_SUCCESS;
190 0 : }
191 :
192 : // AICPU 侧算法入口
193 : template <typename AlgTopoMatch, typename InsAlgTemplate>
194 0 : HcclResult InsV2AllReduceSoleExecutor<AlgTopoMatch, InsAlgTemplate>::Orchestrate(
195 : const AlgTopoInfo& topoInfo, const CollAlgOperator& op, const CollAlgParams& params, ConnectedLinkMgr* linkMgr,
196 : InsQuePtr insQue)
197 : {
198 0 : HCCL_INFO("[InsV2AllReduceSoleExecutor][Orchestrate] Orchestrate AICPU Start.");
199 0 : CHK_RET(Init(op, params, insQue));
200 0 : CHK_RET(InitCommInfo(topoInfo));
201 :
202 0 : std::shared_ptr<InsAlgTemplate> algTemplate = nullptr;
203 0 : CHK_RET(CreateTemplates(algTemplate));
204 :
205 0 : std::map<u32, u32> rank2PathNumMap;
206 0 : CHK_RET(SetPathNumMapByLinkMgrMultiLevel(linkMgr, virtRanks_, myRank_, rank2PathNumMap));
207 0 : algTemplate->setPathNumMap(rank2PathNumMap);
208 :
209 0 : AlgTempResReq tempResReq;
210 0 : CHK_RET(GetTemplateResRequest(linkMgr, algTemplate, tempResReq));
211 :
212 0 : HCCL_DEBUG(
213 : "[InsV2AllReduceSoleExecutor][Orchestrate] Rank[%d], template [%s], requiredQue Num [%u].", myRank_,
214 : algTemplate->Describe().c_str(), tempResReq.queNum);
215 0 : CHK_RET(InitQueue(tempResReq.queNum, tempInsQue_));
216 0 : CHK_RET(PrepResLinks(myRank_, tempResReq.links, linkMgr, tempResLinks_));
217 0 : CHK_RET(OrchestrateLoop(algTemplate));
218 0 : return HcclResult::HCCL_SUCCESS;
219 0 : }
220 :
221 : // 切分数据并调用 template
222 : template <typename AlgTopoMatch, typename InsAlgTemplate>
223 : HcclResult
224 0 : InsV2AllReduceSoleExecutor<AlgTopoMatch, InsAlgTemplate>::OrchestrateLoop(std::shared_ptr<InsAlgTemplate> algTemplate)
225 : {
226 0 : HCCL_INFO("[InsV2AllReduceSoleExecutor][OrchestrateOpbase] Start, template[%s]", algTemplate->Describe().c_str());
227 0 : u32 dataSizePerVolume = DataTypeSizeGet(dataType_);
228 0 : dataSize_ = dataCount_ * dataSizePerVolume;
229 :
230 0 : TemplateDataParams tempAlgParams;
231 0 : tempAlgParams.repeatNum = 1; // 不需要重复
232 0 : tempAlgParams.buffInfo.inBuffType = BufferType::INPUT;
233 0 : tempAlgParams.buffInfo.outBuffType = BufferType::OUTPUT;
234 0 : tempAlgParams.buffInfo.scratBuffType = BufferType::SCRATCH;
235 0 : tempAlgParams.inputRepeatStride = 0;
236 0 : tempAlgParams.outputRepeatStride = 0;
237 :
238 0 : TempFuncs tempFuncs;
239 0 : tempFuncs.opMode = opMode_;
240 : // template 中2D未适配
241 0 : tempFuncs.enableCounterNotify = false;
242 0 : tempFuncs.isForepart = true;
243 0 : tempFuncs.isBottom = true;
244 :
245 0 : u64 maxDataSizePerLoop = 0;
246 0 : u64 transportBoundDataSize = UB_MAX_DATA_SIZE; // algTemplate->CalcLoopMaxCount();
247 0 : u32 templateScratchMultiplier = algTemplate->CalcScratchMultiple(BufferType::INPUT, BufferType::OUTPUT);
248 0 : if (templateScratchMultiplier != 0) {
249 : // maxTmpMemSize_大小的buffer当前template仍然可以用，只是传入的待处理数据只有scratchBoundDataSize大小
250 0 : u64 scratchBoundDataSize = maxTmpMemSize_ / templateScratchMultiplier;
251 0 : maxDataSizePerLoop = std::min(transportBoundDataSize, scratchBoundDataSize);
252 : } else {
253 0 : maxDataSizePerLoop = transportBoundDataSize;
254 : }
255 :
256 : // 如果有template融合可以多种取整策略融合: maxDataSizePerLoop / (MN2*dataSizePerVolume)*MN2;
257 0 : u64 maxDataCountPerLoop = maxDataSizePerLoop / dataSizePerVolume;
258 0 : HCCL_INFO(
259 : "[InsV2AllReduceSoleExecutor][OrchestrateOpbase] maxDataCountPerLoop[%llu], maxDataSizePerLoop[%llu], "
260 : "transportBoundDataSize[%llu], templateScratchMultiplier[%llu]",
261 : maxDataCountPerLoop, maxDataSizePerLoop, transportBoundDataSize, templateScratchMultiplier);
262 0 : CHK_PRT_RET(
263 : maxDataCountPerLoop == 0,
264 : HCCL_ERROR(
265 : "[InsV2AllReduceSoleExecutor][OrchestrateOpbase] maxDataCountPerLoop is 0, scratch buffer "
266 : "size:%u,maxDataSizePerLoop:%llu, dataSize:%llu",
267 : maxTmpMemSize_, maxDataSizePerLoop, dataSize_),
268 : HCCL_E_INTERNAL);
269 :
270 0 : u64 processedDataCount = 0;
271 0 : u64 loopTimes = dataCount_ / maxDataCountPerLoop + static_cast<u32>(dataCount_ % maxDataCountPerLoop != 0);
272 0 : for (u64 loop = 0; loop < loopTimes; loop++) {
273 0 : u64 currDataCount = (loop == loopTimes - 1) ? dataCount_ - processedDataCount : maxDataCountPerLoop;
274 :
275 0 : tempAlgParams.buffInfo.inBuffBaseOff = processedDataCount * dataSizePerVolume;
276 0 : tempAlgParams.buffInfo.outBuffBaseOff = processedDataCount * dataSizePerVolume;
277 0 : tempAlgParams.buffInfo.scratchBuffBaseOff = 0;
278 0 : tempAlgParams.buffInfo.scratchBuffSize = maxTmpMemSize_;
279 0 : tempAlgParams.sliceSize = currDataCount * dataSizePerVolume;
280 0 : tempAlgParams.tailSize = tempAlgParams.sliceSize;
281 0 : tempAlgParams.inputSliceStride = 0; // 输入数据仅有 1 个 slice, 不需要 stride
282 0 : tempAlgParams.outputSliceStride = 0; // 每张卡的数据间隔为算子输入大小
283 :
284 0 : CHK_RET(algTemplate->GenExtIns(tempFuncs, tempAlgParams, tempResLinks_, tempInsQue_));
285 0 : processedDataCount += currDataCount;
286 : }
287 :
288 0 : return HcclResult::HCCL_SUCCESS;
289 0 : }
290 :
291 : INS_REGISTER_IMPL_BY_TEMP(
292 : OpType::ALLREDUCE, InsAllReduceMesh1DOneShot, InsV2AllReduceSoleExecutor, TopoMatchMesh,
293 : InsTempAllReduceMesh1DOneShot);
294 : INS_REGISTER_IMPL_BY_TEMP(
295 : OpType::ALLREDUCE, InsAllReduceMesh2DTwoShot, InsV2AllReduceSoleExecutor, TopoMatchConcurrMesh,
296 : InsTempAllReduceMesh2DTwoShot);
297 : INS_REGISTER_IMPL_BY_TEMP(
298 : OpType::ALLREDUCE, InsAllReduceMesh1DTwoShot, InsV2AllReduceSoleExecutor, TopoMatchMesh,
299 : InsTempAllReduceMesh1DTwoShot);
300 : INS_REGISTER_IMPL_BY_TEMP(
301 : OpType::ALLREDUCE, InsAllReduceMesh1DTwoShotMeshChunk, InsV2AllReduceSoleExecutor, TopoMatchMesh,
302 : InsTempAllReduceMesh1DTwoShotMeshChunk);
303 : INS_REGISTER_IMPL_BY_TEMP(
304 : OpType::ALLREDUCE, InsAllReduceNHR, InsV2AllReduceSoleExecutor, TopoMatchNHR, InsTempAllReduceNHR);
305 : #ifndef CCL_KERNEL_AICPU
306 : INS_REGISTER_IMPL_BY_TEMP(
307 : OpType::ALLREDUCE, AivAllReduceMesh1DOneShot, InsV2AllReduceSoleExecutor, TopoMatchMesh,
308 : AivTempAllReduceMesh1DOneShot);
309 : INS_REGISTER_IMPL_BY_TEMP(
310 : OpType::ALLREDUCE, AivAllReduceMesh1DTwoShot, InsV2AllReduceSoleExecutor, TopoMatchMesh,
311 : AivTempAllReduceMesh1DTwoShot);
312 : INS_REGISTER_IMPL_BY_TEMP(
313 : OpType::ALLREDUCE, CcuAllReduceNHR1D, InsV2AllReduceSoleExecutor, TopoMatchMesh, CcuTempAllReduceNHRMem2Mem1D);
314 : INS_REGISTER_IMPL_BY_TEMP(
315 : OpType::ALLREDUCE, CcuAllReduceMeshMem2Mem1D, InsV2AllReduceSoleExecutor, TopoMatchMesh,
316 : CcuTempAllReduceMeshMem2Mem1D);
317 : #endif
318 : INS_REGISTER_IMPL_BY_TEMP(
319 : OpType::ALLREDUCE, InsAllReduceAicpuReduce, InsV2AllReduceSoleExecutor, TopoMatchMesh, InsTempAllReduceAicpuReduce);
320 : INS_REGISTER_IMPL_BY_TEMP(
321 : OpType::ALLREDUCE, InsAllReduceAicpuReduceMesh2D, InsV2AllReduceSoleExecutor, TopoMatchConcurrMesh,
322 : InsTempAllReduceAicpuReduceMesh2D);
323 :
324 : } // namespace Hccl
|
