Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "ins_v2_all_gather_sole_executor.h"
12 :
13 : #include "log.h"
14 :
15 : #include "ins_coll_alg_registry.h"
16 :
17 : #include "topo_match_mesh.h"
18 : #include "topo_match_nhr.h"
19 : #include "topo_match_concurr_mesh.h"
20 :
21 : #include "ins_temp_all_gather_mesh.h"
22 : #include "ins_temp_all_gather_mesh_2D.h"
23 : #include "ins_temp_all_gather_nhr.h"
24 : #ifndef CCL_KERNEL_AICPU
25 : #include "ccu_temp_all_gather_mesh_1D_mem2mem_with_stride.h"
26 : #include "ccu_temp_all_gather_nhr_1D_mem2mem.h"
27 : #include "ccu_temp_all_gather_mesh_2D_mem2mem.h"
28 : #include "aiv_temp_all_gather_mesh_1D.h"
29 : #include "ccu_temp_all_gather_mesh_1D_2die.h"
30 : #endif
31 :
32 : namespace Hccl {
33 : constexpr u64 MAX_OFFLOAD_SCRATCH_SIZE = 200 * 1024 * 1024; // 200M
34 :
35 : template <typename AlgTopoMatch, typename InsAlgTemplate>
36 0 : InsV2AllGatherSoleExecutor<AlgTopoMatch, InsAlgTemplate>::InsV2AllGatherSoleExecutor() : InsCollAlgBase()
37 0 : {}
38 :
39 : template <typename AlgTopoMatch, typename InsAlgTemplate>
40 0 : InsV2AllGatherSoleExecutor<AlgTopoMatch, InsAlgTemplate>::~InsV2AllGatherSoleExecutor()
41 0 : {}
42 :
43 : template <typename AlgTopoMatch, typename InsAlgTemplate>
44 0 : HcclResult InsV2AllGatherSoleExecutor<AlgTopoMatch, InsAlgTemplate>::InitCommInfo(const RankGraph* rankGraph)
45 : {
46 0 : AlgTopoMatch topoMatch(myRank_, rankSize_, rankGraph, devType_);
47 0 : CHK_RET(topoMatch.MatchTopo(vTopo_, virtRanks_, virtRankMap_));
48 0 : return HcclResult::HCCL_SUCCESS;
49 0 : }
50 :
51 : template <typename AlgTopoMatch, typename InsAlgTemplate>
52 0 : HcclResult InsV2AllGatherSoleExecutor<AlgTopoMatch, InsAlgTemplate>::InitCommInfo(const AlgTopoInfo& topoInfo)
53 : {
54 0 : if (topoInfo.vTopo.size() < 1) {
55 0 : return HcclResult::HCCL_E_INTERNAL;
56 : }
57 0 : vTopo_ = topoInfo.vTopo[0]; // 本通信域内的通信平面
58 0 : virtRankMap_ = topoInfo.virtRankMap[0]; // 本通信域内的 rank 映射表
59 0 : virtRanks_ = topoInfo.virtRanks[0]; // 本通信域内的 rank 集合
60 0 : return HcclResult::HCCL_SUCCESS;
61 : }
62 :
63 : template <typename AlgTopoMatch, typename InsAlgTemplate>
64 0 : HcclResult InsV2AllGatherSoleExecutor<AlgTopoMatch, InsAlgTemplate>::CreateTemplates(
65 : std::shared_ptr<InsAlgTemplate>& algTemplatePtr)
66 : {
67 0 : HCCL_DEBUG("[InsV2AllGatherSoleExecutor][CreateTemplates]");
68 0 : algTemplatePtr = std::make_shared<InsAlgTemplate>(myRank_, rankSize_, vTopo_, virtRankMap_);
69 0 : CHK_PTR_NULL(algTemplatePtr); // 检查是否成功分配内存
70 0 : algTemplatePtr->SetDmaMode(dmaMode_);
71 0 : algTemplatePtr->SetDataType(dataType_);
72 0 : algTemplatePtr->SetCollOp(op_);
73 0 : return HcclResult::HCCL_SUCCESS;
74 : }
75 :
76 : template <typename AlgTopoMatch, typename InsAlgTemplate>
77 0 : HcclResult InsV2AllGatherSoleExecutor<AlgTopoMatch, InsAlgTemplate>::GetTemplateResRequest(
78 : const RankGraph* rankGraph, std::shared_ptr<InsAlgTemplate>& algTemplate, AlgTempResReq& tempResReq) const
79 : {
80 0 : if (enableDetour_) {
81 0 : HCCL_DEBUG("[InsV2AllGatherSoleExecutor] [%s] Rank[%d], CalcRes with detouring enabled.", __func__, myRank_);
82 0 : CHK_RET(algTemplate->CalcResDetour(rankGraph, tempResReq));
83 : } else {
84 0 : HCCL_DEBUG("[InsV2AllGatherSoleExecutor] [%s] Rank[%d], CalcRes with detouring disabled.", __func__, myRank_);
85 0 : CHK_RET(algTemplate->CalcRes(tempResReq));
86 : }
87 0 : return HcclResult::HCCL_SUCCESS;
88 : }
89 :
90 : template <typename AlgTopoMatch, typename InsAlgTemplate>
91 0 : HcclResult InsV2AllGatherSoleExecutor<AlgTopoMatch, InsAlgTemplate>::GetTemplateResRequest(
92 : ConnectedLinkMgr* linkMgr, std::shared_ptr<InsAlgTemplate>& algTemplate, AlgTempResReq& tempResReq) const
93 : {
94 0 : if (enableDetour_) {
95 0 : HCCL_DEBUG("[InsV2AllGatherSoleExecutor] [%s] Rank[%d], CalcRes with detouring enabled.", __func__, myRank_);
96 0 : CHK_RET(algTemplate->CalcResDetour(linkMgr, tempResReq));
97 : } else {
98 0 : HCCL_DEBUG("[InsV2AllGatherSoleExecutor] [%s] Rank[%d], CalcRes with detouring disabled.", __func__, myRank_);
99 0 : CHK_RET(algTemplate->CalcRes(tempResReq));
100 : }
101 0 : return HcclResult::HCCL_SUCCESS;
102 : }
103 :
104 : template <typename AlgTopoMatch, typename InsAlgTemplate>
105 : HcclResult
106 0 : InsV2AllGatherSoleExecutor<AlgTopoMatch, InsAlgTemplate>::CalNumBlocks(u32& numBlocks, u64 dataSize, u32 numBlocksLimit)
107 : {
108 0 : std::shared_ptr<InsAlgTemplate> algTemplate = nullptr;
109 0 : CHK_RET(CreateTemplates(algTemplate));
110 0 : CHK_RET(algTemplate->CalNumBlocks(numBlocks, dataSize, numBlocksLimit));
111 0 : return HcclResult::HCCL_SUCCESS;
112 0 : }
113 :
114 : // HOST 侧算法入口
115 : template <typename AlgTopoMatch, typename InsAlgTemplate>
116 0 : HcclResult InsV2AllGatherSoleExecutor<AlgTopoMatch, InsAlgTemplate>::Orchestrate(
117 : const RankGraph* rankGraph, const CollAlgOperator& op, const CollAlgParams& params, InsQuePtr insQue)
118 : {
119 0 : HCCL_DEBUG("[InsV2AllGatherSoleExecutor][Orchestrate] Orchestrate HOST Start");
120 0 : CHK_RET(Init(op, params, insQue));
121 0 : CHK_RET(InitCommInfo(rankGraph));
122 :
123 0 : std::shared_ptr<InsAlgTemplate> algTemplate = nullptr;
124 0 : CHK_RET(CreateTemplates(algTemplate));
125 :
126 0 : AlgTempResReq tempResReq;
127 0 : CHK_RET(GetTemplateResRequest(rankGraph, algTemplate, tempResReq));
128 :
129 0 : HCCL_DEBUG(
130 : "[InsV2AllGatherSoleExecutor][Orchestrate] Rank[%d], template [%s], requiredQue Num [%u].", myRank_,
131 : algTemplate->Describe().c_str(), tempResReq.queNum);
132 0 : CHK_RET(InitQueue(tempResReq.queNum, tempInsQue_));
133 0 : CHK_RET(PrepResLinks(myRank_, rankGraph, linkPriority_, tempResReq.links, tempResLinks_));
134 0 : CHK_RET(OrchestrateLoop(algTemplate));
135 0 : return HcclResult::HCCL_SUCCESS;
136 0 : }
137 :
138 : // AICPU 侧算法入口
139 : template <typename AlgTopoMatch, typename InsAlgTemplate>
140 0 : HcclResult InsV2AllGatherSoleExecutor<AlgTopoMatch, InsAlgTemplate>::Orchestrate(
141 : const AlgTopoInfo& topoInfo, const CollAlgOperator& op, const CollAlgParams& params, ConnectedLinkMgr* linkMgr,
142 : InsQuePtr insQue)
143 : {
144 0 : HCCL_DEBUG("[InsV2AllGatherSoleExecutor][Orchestrate] Orchestrate AICPU Start");
145 0 : CHK_RET(Init(op, params, insQue));
146 0 : dataType_ = op.dataType;
147 :
148 0 : CHK_RET(InitCommInfo(topoInfo));
149 :
150 0 : std::shared_ptr<InsAlgTemplate> algTemplate = nullptr;
151 0 : CHK_RET(CreateTemplates(algTemplate));
152 0 : algTemplate->SetDataType(dataType_);
153 0 : std::map<u32, u32> rank2PathNumMap;
154 0 : CHK_RET(SetPathNumMapByLinkMgrMultiLevel(linkMgr, virtRanks_, myRank_, rank2PathNumMap));
155 0 : algTemplate->setPathNumMap(rank2PathNumMap);
156 :
157 0 : AlgTempResReq tempResReq;
158 0 : CHK_RET(GetTemplateResRequest(linkMgr, algTemplate, tempResReq));
159 :
160 0 : HCCL_DEBUG(
161 : "[InsV2AllGatherSoleExecutor][Orchestrate] Rank[%d], template [%s], requiredQue Num [%u].", myRank_,
162 : algTemplate->Describe().c_str(), tempResReq.queNum);
163 0 : CHK_RET(InitQueue(tempResReq.queNum, tempInsQue_));
164 0 : CHK_RET(PrepResLinks(myRank_, tempResReq.links, linkMgr, tempResLinks_));
165 0 : CHK_RET(OrchestrateLoop(algTemplate));
166 0 : return HcclResult::HCCL_SUCCESS;
167 0 : }
168 :
169 : // 切分数据并调用 template
170 : template <typename AlgTopoMatch, typename InsAlgTemplate>
171 : HcclResult
172 0 : InsV2AllGatherSoleExecutor<AlgTopoMatch, InsAlgTemplate>::OrchestrateLoop(std::shared_ptr<InsAlgTemplate> algTemplate)
173 : {
174 0 : HCCL_INFO("[InsV2AllGatherSoleExecutor][OrchestrateOpbase] Start, template[%s]", algTemplate->Describe().c_str());
175 0 : u32 dataSizePerVolume = DataTypeSizeGet(dataType_);
176 0 : dataSize_ = dataCount_ * dataSizePerVolume;
177 :
178 0 : TemplateDataParams tempAlgParams;
179 0 : tempAlgParams.inputRepeatStride = 0;
180 0 : tempAlgParams.outputRepeatStride = 0;
181 0 : tempAlgParams.buffInfo.inBuffType = BufferType::INPUT;
182 0 : tempAlgParams.buffInfo.outBuffType = BufferType::OUTPUT;
183 0 : tempAlgParams.buffInfo.scratBuffType = BufferType::SCRATCH;
184 0 : tempAlgParams.repeatNum = 1; // 不需要重复
185 :
186 0 : TempFuncs tempFuncs;
187 0 : tempFuncs.isForepart = true;
188 0 : tempFuncs.isBottom = true;
189 0 : tempFuncs.opMode = opMode_;
190 0 : tempFuncs.enableCounterNotify = IsEnableCounterNotify();
191 :
192 0 : u64 maxDataSizePerLoop = 0;
193 0 : u64 transportBoundDataSize = UB_MAX_DATA_SIZE; // algTemplate->CalcLoopMaxCount();
194 :
195 0 : u32 templateScratchMultiplier = algTemplate->CalcScratchMultiple(BufferType::INPUT, BufferType::OUTPUT);
196 0 : if (templateScratchMultiplier != 0) {
197 0 : u64 scratchBoundDataSize = maxTmpMemSize_ / templateScratchMultiplier;
198 0 : maxDataSizePerLoop = std::min(transportBoundDataSize, scratchBoundDataSize);
199 : } else {
200 0 : maxDataSizePerLoop = transportBoundDataSize;
201 : }
202 0 : u64 maxDataCountPerLoop = maxDataSizePerLoop / dataTypeSize_;
203 0 : HCCL_INFO(
204 : "[InsV2AllGatherSoleExecutor][OrchestrateOpbase] maxDataCountPerLoop[%llu], maxDataSizePerLoop[%llu], "
205 : "transportBoundDataSize[%llu], templateScratchMultiplier[%llu]",
206 : maxDataCountPerLoop, maxDataSizePerLoop, transportBoundDataSize, templateScratchMultiplier);
207 0 : CHK_PRT_RET(
208 : maxDataCountPerLoop == 0,
209 : HCCL_ERROR("[InsV2AllGatherSoleExecutor][OrchestrateOpbase] maxDataCountPerLoop is 0"), HCCL_E_INTERNAL);
210 :
211 0 : u64 processedDataCount = 0; // 已经处理的数据count
212 0 : u64 loopTimes = dataCount_ / maxDataCountPerLoop + static_cast<u64>(dataCount_ % maxDataCountPerLoop != 0);
213 0 : for (u64 loop = 0; loop < loopTimes; loop++) {
214 0 : u64 currDataCount = (loop == loopTimes - 1) ? dataCount_ - processedDataCount : maxDataCountPerLoop;
215 :
216 0 : tempAlgParams.buffInfo.inBuffBaseOff = processedDataCount * dataTypeSize_;
217 0 : tempAlgParams.buffInfo.outBuffBaseOff = processedDataCount * dataTypeSize_;
218 0 : tempAlgParams.buffInfo.scratchBuffBaseOff = 0;
219 0 : tempAlgParams.sliceSize = currDataCount * dataTypeSize_;
220 0 : tempAlgParams.tailSize = tempAlgParams.sliceSize;
221 0 : tempAlgParams.inputSliceStride = 0; // 输入数据仅有 1 个 slice, 不需要 stride
222 0 : tempAlgParams.outputSliceStride = dataSize_; // 每张卡的数据间隔为算子输入大小
223 :
224 0 : CHK_RET(algTemplate->GenExtIns(tempFuncs, tempAlgParams, tempResLinks_, tempInsQue_));
225 0 : processedDataCount += currDataCount;
226 : }
227 :
228 0 : return HcclResult::HCCL_SUCCESS;
229 0 : }
230 :
231 : template <typename AlgTopoMatch, typename InsAlgTemplate>
232 : HcclResult
233 0 : InsV2AllGatherSoleExecutor<AlgTopoMatch, InsAlgTemplate>::CalcRes(const RankGraph* rankGraph, CollAlgResReq& algResReq)
234 : {
235 : // Topo Match
236 0 : CHK_RET(InitCommInfo(rankGraph));
237 :
238 : // instantiate a template
239 0 : InsAlgTemplate tempAlg(myRank_, rankSize_, vTopo_, virtRankMap_);
240 :
241 0 : std::shared_ptr<InsAlgTemplate> algTemplate = nullptr;
242 0 : CHK_RET(CreateTemplates(algTemplate));
243 :
244 : // 通过判断哪层通信域能有到所有remoteRank的path，判断当前算法跑在哪一层
245 0 : std::map<u32, u32> rank2PathNumMap;
246 0 : HCCL_INFO("[InsV2AllGatherSoleExecutor] CalcRes SetPathNumMap");
247 0 : CHK_RET(SetPathNumMapByRankGraphMultiLevel(rankGraph, virtRanks_, myRank_, rank2PathNumMap));
248 0 : algTemplate->setPathNumMap(rank2PathNumMap);
249 0 : AlgTempResReq tempResReq;
250 0 : CHK_RET(GetTemplateResRequest(rankGraph, algTemplate, tempResReq));
251 :
252 0 : CHK_RET(CalcLinkInfo(myRank_, rankGraph, tempResReq.links, algResReq.levelRankPairs));
253 0 : algResReq.topoInfo.UpdateSingleLevelTopo(virtRanks_, virtRankMap_, vTopo_);
254 0 : algResReq.queueNotifys = tempResReq.queNotifys;
255 0 : algResReq.primQueueNum = tempResReq.streamNum;
256 0 : algResReq.localWaitGroupCntNotify = tempResReq.localWaitGroupCntNotify;
257 0 : algResReq.localBcastPostCntNotify = tempResReq.localBcastPostCntNotify;
258 0 : HCCL_DEBUG(
259 : "[InsV2AllGatherSoleExecutor] [%s] Rank[%d], requiredQueNum [%u].", __func__, myRank_, algResReq.primQueueNum);
260 0 : CHK_RET(CalcResLinks(myRank_, rankGraph, linkPriority_, tempResReq.links, algResReq.links));
261 :
262 0 : return HcclResult::HCCL_SUCCESS;
263 0 : }
264 :
265 : template <typename AlgTopoMatch, typename InsAlgTemplate>
266 0 : HcclResult InsV2AllGatherSoleExecutor<AlgTopoMatch, InsAlgTemplate>::CalcResOffload(
267 : const RankGraph* rankGraph, const u64& dataSize, CollOffloadOpResReq& resReq)
268 : {
269 : // Topo Match
270 : (void)dataSize;
271 0 : CHK_RET(InitCommInfo(rankGraph));
272 :
273 0 : std::shared_ptr<InsAlgTemplate> algTemplate = nullptr;
274 0 : CHK_RET(CreateTemplates(algTemplate));
275 :
276 : // 通过判断哪层通信域能有到所有remoteRank的path，判断当前算法跑在哪一层
277 0 : std::map<u32, u32> rank2PathNumMap;
278 0 : HCCL_INFO("[InsV2AllGatherSoleExecutor] CalcResOffload SetPathNumMap");
279 0 : CHK_RET(SetPathNumMapByRankGraphMultiLevel(rankGraph, virtRanks_, myRank_, rank2PathNumMap));
280 0 : algTemplate->setPathNumMap(rank2PathNumMap);
281 0 : AlgTempResReq tempResReq;
282 0 : CHK_RET(GetTemplateResRequest(rankGraph, algTemplate, tempResReq));
283 0 : resReq.requiredScratchMemSize = UB_MAX_DATA_SIZE;
284 0 : resReq.requiredSubQueNum = tempResReq.streamNum - 1;
285 :
286 0 : return HcclResult::HCCL_SUCCESS;
287 0 : }
288 :
289 : INS_REGISTER_IMPL_BY_TEMP(
290 : OpType::ALLGATHER, InsAllGatherMesh, InsV2AllGatherSoleExecutor, TopoMatchMesh, InsTempAllGatherMesh1D);
291 : INS_REGISTER_IMPL_BY_TEMP(
292 : OpType::ALLGATHER, InsAllGatherMesh2D, InsV2AllGatherSoleExecutor, TopoMatchConcurrMesh, InsTempAllGatherMesh2D);
293 : INS_REGISTER_IMPL_BY_TEMP(
294 : OpType::ALLGATHER, InsAllGatherNHR, InsV2AllGatherSoleExecutor, TopoMatchNHR, InsTempAllGatherNHR);
295 : #ifndef CCL_KERNEL_AICPU
296 : INS_REGISTER_IMPL_BY_TEMP(
297 : OpType::ALLGATHER, AivAllGatherMesh1D, InsV2AllGatherSoleExecutor, TopoMatchMesh, AivTempAllGatherMesh1D);
298 : INS_REGISTER_IMPL_BY_TEMP(
299 : OpType::ALLGATHER, CcuAllGatherMesh1DMem2MemWithStride, InsV2AllGatherSoleExecutor, TopoMatchMesh,
300 : CcuTempAllGatherMesh1DMem2MemWithStride);
301 : INS_REGISTER_IMPL_BY_TEMP(
302 : OpType::ALLGATHER, CcuAllGatherNHR1D, InsV2AllGatherSoleExecutor, TopoMatchMesh, CcuTempAllGatherNHRMem2Mem1D);
303 : INS_REGISTER_IMPL_BY_TEMP(
304 : OpType::ALLGATHER, CcuAllGatherMeshMem2Mem2D, InsV2AllGatherSoleExecutor, TopoMatchConcurrMesh,
305 : CcuTempAllGatherMeshMem2Mem2D);
306 : INS_REGISTER_IMPL_BY_TEMP(
307 : OpType::ALLGATHER, CcuAllGatherMesh1D2Die, InsV2AllGatherSoleExecutor, TopoMatchMesh, CcuTempAllGatherMesh1D2Die);
308 : #endif
309 : } // namespace Hccl
|
