Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "coll_all_reduce_mesh_executor.h"
12 :
13 : namespace hccl {
14 :
15 1 : CollAllReduceMeshExecutor::CollAllReduceMeshExecutor(
16 1 : const HcclDispatcher dispatcher, std::unique_ptr<TopoMatcher>& topoMatcher)
17 1 : : CollAllReduceExecutor(dispatcher, topoMatcher)
18 : {
19 4 : DMAReduceFlag_ = false;
20 4 : }
21 :
22 4 : void CollAllReduceMeshExecutor::ParseParam(const OpParam& param)
23 : {
24 4 : tag_ = param.tag;
25 : bool isInlineReduce
26 4 : = IsSupportSDMAReduce(param.inputPtr, param.outputPtr, param.DataDes.dataType, param.reduceType);
27 8 : meshSinglePlane_ = (topoAttr_.deviceType == DevType::DEV_TYPE_910B)
28 4 : && topoMatcher_->GetExternalInputHcclDeterministic() == DETERMINISTIC_DISABLE && isInlineReduce
29 8 : && (workflowMode_ != HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE);
30 4 : aicpuUnfoldMode_ = param.aicpuUnfoldMode;
31 4 : }
32 :
33 4 : HcclResult CollAllReduceMeshExecutor::CalcStreamNum(u32& streamNum)
34 : {
35 4 : u32 totalStreamNum = topoAttr_.deviceNumPerAggregation > 1U ? topoAttr_.deviceNumPerAggregation - 1U : 1U;
36 4 : streamNum = totalStreamNum - 1U;
37 4 : HCCL_INFO("[CollAllReduceMeshExecutor][CalcStreamNum] tag[%s] streamNum[%u]", tag_.c_str(), streamNum);
38 4 : return HCCL_SUCCESS;
39 : }
40 :
41 4 : HcclResult CollAllReduceMeshExecutor::CalcCommInfo(std::vector<LevelNSubCommTransport>& opTransport)
42 : {
43 4 : TransportMemType inputType = TransportMemType::RESERVED;
44 4 : TransportMemType outputType = TransportMemType::RESERVED;
45 4 : CHK_RET(CalcTransportMemType(inputType, outputType));
46 4 : CHK_RET(CalcLevel0CommInfo(inputType, outputType, opTransport));
47 4 : CHK_RET(CalcLevel1CommInfo(inputType, outputType, opTransport));
48 4 : return HCCL_SUCCESS;
49 : }
50 :
51 4 : HcclResult CollAllReduceMeshExecutor::CalcTransportMemType(TransportMemType& inputType, TransportMemType& outputType)
52 : {
53 4 : if (workflowMode_ == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE) {
54 0 : inputType = TransportMemType::CCL_INPUT;
55 0 : outputType = TransportMemType::CCL_OUTPUT;
56 : } else {
57 4 : inputType = TransportMemType::PARAM_INPUT;
58 4 : outputType = TransportMemType::PARAM_OUTPUT;
59 : }
60 4 : HCCL_INFO(
61 : "[CollAllReduceMeshExecutor][CalcTransportMemType] tag[%s] inputType[%d], outputType[%d]", tag_.c_str(),
62 : inputType, outputType);
63 4 : return HCCL_SUCCESS;
64 : }
65 :
66 4 : HcclResult CollAllReduceMeshExecutor::CalcLevel0CommInfo(
67 : TransportMemType inputType, TransportMemType outputType, std::vector<LevelNSubCommTransport>& opTransport)
68 : {
69 4 : CommParaInfo commParaLevel0(COMM_LEVEL0, CommType::COMM_TAG_MESH);
70 4 : commParaLevel0.meshSinglePlane = meshSinglePlane_;
71 4 : CHK_RET(CalcCommPlaneInfo(tag_, commParaLevel0, opTransport[COMM_LEVEL0], inputType, outputType));
72 4 : return HCCL_SUCCESS;
73 4 : }
74 :
75 0 : bool CollAllReduceMeshExecutor::IsHugeData(const u64 curSize)
76 : {
77 0 : bool hugeData = curSize / topoAttr_.deviceNumPerAggregation / HCCL_INTERNODE_MAX_DATA_RATE > RDMA_SEND_MAX_SIZE
78 0 : || curSize > SDMA_SEND_MAX_SIZE;
79 0 : return hugeData;
80 : }
81 :
82 0 : bool CollAllReduceMeshExecutor::IsSmallData([[maybe_unused]] const u64 totalSize, const u64 curSize)
83 : {
84 0 : bool smallData = IsAllReduceSmallData(curSize);
85 0 : return smallData;
86 : }
87 :
88 0 : HcclResult CollAllReduceMeshExecutor::KernelRun(const OpParam& param, ExecMem& execMem)
89 : {
90 0 : HCCL_CONFIG_INFO(HCCL_ALG, "[CollAllReduceMeshExecutor][KernelRun] userRank[%u] starts.", topoAttr_.userRank);
91 0 : u32 perDataSize = SIZE_TABLE[param.DataDes.dataType];
92 :
93 0 : std::vector<Slice> dataSegsSlice; // 数据分成ranksize份，每份的起始偏移和大小
94 0 : std::vector<std::vector<Slice>> multiStreamSlice; // 每个stream使用的数据基于用户buffer的偏移
95 0 : std::unique_ptr<AlgTemplateBase> level1TempAlg;
96 0 : std::unique_ptr<AlgTemplateBase> level0TempAlg;
97 :
98 0 : CHK_RET(CheckCommSize(COMM_LEVEL0, COMM_INDEX_0 + 1));
99 0 : SubCommInfo level0CommInfo = GetSubCommInfo(COMM_LEVEL0, COMM_INDEX_0);
100 :
101 0 : u32 sliceNum = level0CommInfo.localRankSize;
102 : // 根据数据量算每个环上数据的偏移和大小
103 0 : CHK_RET(AlgTemplateBase::PrepareSliceData(execMem.count, perDataSize, sliceNum, 0, dataSegsSlice));
104 : // mesh算法stream数量为server内rank数减1
105 :
106 0 : CHK_RET(ActiveSlaveStreams(param.stream));
107 :
108 0 : if (topoMatcher_->GetExternalInputHcclDeterministic() == DETERMINISTIC_DISABLE
109 0 : && (param.DataDes.dataType != HCCL_DATA_TYPE_INT64)
110 0 : && (topoAttr_.deviceType == DevType::DEV_TYPE_910B && param.reduceType != HCCL_REDUCE_PROD)) {
111 0 : CHK_RET(MultiStreamReduceScatterMeshAtomic(
112 : param.tag, execMem.inputMem, execMem.outputMem, execMem.count, param.DataDes.dataType, param.reduceType,
113 : dataSegsSlice, const_cast<Stream&>(param.stream), COMM_LEVEL0));
114 : } else {
115 0 : CHK_RET(AlgTemplateBase::PrepareSliceMeshStreams(dataSegsSlice, sliceNum - 1, multiStreamSlice));
116 0 : CHK_RET(MultiStreamReduceScatterMesh(
117 : param.tag, execMem.inputMem, execMem.outputMem, execMem.count, param.DataDes.dataType, param.reduceType,
118 : multiStreamSlice, const_cast<Stream&>(param.stream), COMM_LEVEL0));
119 : }
120 :
121 0 : HCCL_INFO("AllReduce meshhd stage0 run success.");
122 :
123 : /* 内层topo:all_reduce */
124 : /* 外层所有rank均参与内层的allReduce计算，所以此处对rank不作限制，但是每个rank需找到自己所在的内层通信域 */
125 0 : u32 commIndex = level0CommInfo.localRank;
126 0 : CHK_PRT_RET(
127 : commIndex >= dataSegsSlice.size(),
128 : HCCL_ERROR(
129 : "[CollAllReduceMeshExecutor][Run]commIndex[%u] >= dataSegsSlice size[%zu]", commIndex,
130 : dataSegsSlice.size()),
131 : HCCL_E_INTERNAL);
132 :
133 0 : CHK_RET(CheckCommSize(COMM_LEVEL1, commIndex + 1));
134 0 : SubCommInfo level1CommInfo = GetSubCommInfo(COMM_LEVEL1, commIndex);
135 :
136 0 : DeviceMem allreduceInput = execMem.inputMem.range(dataSegsSlice[commIndex].offset, dataSegsSlice[commIndex].size);
137 0 : CHK_SMART_PTR_NULL(allreduceInput);
138 0 : DeviceMem allreduceOutput = execMem.outputMem.range(dataSegsSlice[commIndex].offset, dataSegsSlice[commIndex].size);
139 0 : CHK_SMART_PTR_NULL(allreduceOutput);
140 :
141 0 : u64 reduceAttr = GetReduceAttr(execMem.inputMem, execMem.outputMem, param.DataDes.dataType, param.reduceType);
142 :
143 0 : if (algType_.algoLevel1 == AlgTypeLevel1::ALG_LEVEL1_RING) {
144 : level1TempAlg
145 0 : = AlgTemplateRegistry::Instance().GetAlgTemplate(TemplateType::TEMPLATE_ALL_REDUCE_RING, dispatcher_);
146 0 : HCCL_INFO("AllReduce mesh: using ring algo inter-server.");
147 0 : CHK_SMART_PTR_NULL(level1TempAlg);
148 0 : CHK_RET(level1TempAlg->Prepare(reduceAttr));
149 0 : } else if (algType_.algoLevel1 == AlgTypeLevel1::ALG_LEVEL1_NHR) {
150 0 : u64 curSize = execMem.count * perDataSize; // 单位 byte
151 0 : HCCL_DEBUG(
152 : "AllReduce mesh: curSize[%llu] deviceNumPerAggregation[%u] commLevel0Size[%u]", curSize,
153 : topoAttr_.deviceNumPerAggregation, level0CommInfo.localRankSize);
154 0 : if (curSize / topoAttr_.deviceNumPerAggregation <= NHR_ALLREDUCE_SMALL_SIZE) {
155 0 : level1TempAlg = AlgTemplateRegistry::Instance().GetAlgTemplate(
156 0 : TemplateType::TEMPLATE_ALL_REDUCE_NHR_ONESHOT, dispatcher_);
157 : } else {
158 : level1TempAlg
159 0 : = AlgTemplateRegistry::Instance().GetAlgTemplate(TemplateType::TEMPLATE_ALL_REDUCE_NHR, dispatcher_);
160 : }
161 0 : HCCL_INFO("AllReduce mesh: using nhr algo inter-server.");
162 0 : CHK_SMART_PTR_NULL(level1TempAlg);
163 0 : CHK_RET(level1TempAlg->Prepare(reduceAttr));
164 0 : level1TempAlg->CloseBarrier();
165 0 : } else if (algType_.algoLevel1 == AlgTypeLevel1::ALG_LEVEL1_NHR_V1) {
166 : level1TempAlg
167 0 : = AlgTemplateRegistry::Instance().GetAlgTemplate(TemplateType::TEMPLATE_ALL_REDUCE_NHR_V1, dispatcher_);
168 0 : HCCL_INFO("AllReduce mesh: using nhr_v1 algo inter-server.");
169 0 : CHK_SMART_PTR_NULL(level1TempAlg);
170 0 : CHK_RET(level1TempAlg->Prepare(reduceAttr));
171 0 : } else if (algType_.algoLevel1 == AlgTypeLevel1::ALG_LEVEL1_NB) {
172 : level1TempAlg
173 0 : = AlgTemplateRegistry::Instance().GetAlgTemplate(TemplateType::TEMPLATE_ALL_REDUCE_NB, dispatcher_);
174 0 : HCCL_INFO("AllReduce mesh: using nb algo inter-server.");
175 0 : CHK_SMART_PTR_NULL(level1TempAlg);
176 0 : CHK_RET(level1TempAlg->Prepare(reduceAttr));
177 : } else {
178 0 : level1TempAlg = AlgTemplateRegistry::Instance().GetAlgTemplate(
179 0 : TemplateType::TEMPLATE_ALL_REDUCE_RECURSIVE_HALVING_DOUBLING, dispatcher_);
180 0 : HCCL_INFO("AllReduce mesh: using Recursive halving-doubling algo inter-server.");
181 0 : CHK_SMART_PTR_NULL(level1TempAlg);
182 0 : CHK_RET(level1TempAlg->Prepare(reduceAttr));
183 : }
184 0 : CHK_SMART_PTR_NULL(level1TempAlg);
185 :
186 0 : u32 rankSize = level1CommInfo.localRankSize;
187 : // 节点间的hd 使用环0来记录
188 :
189 0 : u64 hdCount = dataSegsSlice[commIndex].size / perDataSize;
190 0 : CHK_RET(level1TempAlg->Prepare(
191 : allreduceInput, allreduceOutput, allreduceOutput, hdCount, param.DataDes.dataType, param.stream,
192 : param.reduceType, LEVEL0_BRIDGE_RANK_ID, std::vector<Slice>(0), dataSegsSlice[commIndex].offset));
193 :
194 0 : CHK_RET(level1TempAlg->RegisterProfiler(
195 : (rankSize << PROF_RANKSIZE_OFFSET_OF_PLANEID) + level1CommInfo.localRank, PROF_STAGE_1, HCCL_EXEC_STEP_NOT_SET,
196 : param.stream));
197 :
198 0 : CHK_RET(RunTemplate(level1TempAlg, level1CommInfo));
199 :
200 0 : HCCL_INFO("AllReduce meshhd stage1 run success.");
201 :
202 : /* 外层topo:all_gather */
203 :
204 0 : if (topoAttr_.deviceType == DevType::DEV_TYPE_910B) {
205 0 : level0TempAlg = AlgTemplateRegistry::Instance().GetAlgTemplate(
206 0 : TemplateType::TEMPLATE_ALL_GATHER_MESH_ATOMIC, dispatcher_);
207 : } else {
208 : level0TempAlg
209 0 : = AlgTemplateRegistry::Instance().GetAlgTemplate(TemplateType::TEMPLATE_ALL_GATHER_MESH, dispatcher_);
210 : }
211 :
212 0 : CHK_SMART_PTR_NULL(level0TempAlg);
213 0 : CHK_RET(level0TempAlg->Prepare(
214 : algResResp_->slaveStreams, algResResp_->notifiesMain, algResResp_->notifiesAux, topoAttr_.userRank, nullptr,
215 : level0CommInfo.localRank, level0CommInfo.localRankSize));
216 :
217 : /* 节点内执行器 stage2 */
218 : {
219 0 : u32 rankSize = level0CommInfo.localRankSize;
220 0 : CHK_RET(level0TempAlg->Prepare(
221 : execMem.outputMem, execMem.outputMem, execMem.inputMem, execMem.count, param.DataDes.dataType, param.stream,
222 : param.reduceType, LEVEL0_BRIDGE_RANK_ID, dataSegsSlice, 0));
223 :
224 0 : CHK_RET(level0TempAlg->RegisterProfiler(
225 : (rankSize << PROF_RANKSIZE_OFFSET_OF_PLANEID) + level0CommInfo.localRank, PROF_STAGE_2,
226 : HCCL_EXEC_STEP_NOT_SET, param.stream));
227 :
228 0 : CHK_RET(RunTemplate(level0TempAlg, level0CommInfo));
229 : }
230 :
231 0 : HCCL_INFO("AllReduce meshhd stage2 run success");
232 0 : return HCCL_SUCCESS;
233 0 : }
234 :
235 0 : HcclResult CollAllReduceMeshExecutor::Getlevel1CommRank(SubCommInfo& level1CommInfo)
236 : {
237 0 : if (CheckCommSize(COMM_LEVEL0, COMM_INDEX_0 + 1) != HCCL_SUCCESS) {
238 0 : return HCCL_E_UNAVAIL;
239 : }
240 0 : SubCommInfo level0CommInfo = GetSubCommInfo(COMM_LEVEL0, COMM_INDEX_0);
241 0 : u32 ringNum
242 0 : = (topoType_ == TopoType::TOPO_TYPE_8P_RING) ? LEVEL0_PLANE_NUM_IN_8PRING : LEVEL0_PLANE_NUM_IN_NPRING_SINGLE;
243 0 : u32 commIndex = (ringNum == LEVEL0_PLANE_NUM_IN_8PRING) ? topoAttr_.devicePhyId : level0CommInfo.localRank;
244 :
245 0 : if (CheckCommSize(COMM_LEVEL1, commIndex + 1) != HCCL_SUCCESS) {
246 0 : return HCCL_E_UNAVAIL;
247 : }
248 0 : level1CommInfo = GetSubCommInfo(COMM_LEVEL1, commIndex);
249 :
250 0 : return HCCL_SUCCESS;
251 0 : }
252 :
253 0 : HcclResult CollAllReduceMeshExecutor::SelectTempAlg(std::unique_ptr<AlgTemplateBase>& level1TempAlg, u32 level1RankSize)
254 : {
255 0 : if (level1RankSize > 1) {
256 0 : if (algType_.algoLevel1 == AlgTypeLevel1::ALG_LEVEL1_RING) {
257 : level1TempAlg
258 0 : = AlgTemplateRegistry::Instance().GetAlgTemplate(TemplateType::TEMPLATE_ALL_REDUCE_RING, dispatcher_);
259 0 : HCCL_INFO("AllReduce mesh: using ring algo inter-server.");
260 0 : } else if (algType_.algoLevel1 == AlgTypeLevel1::ALG_LEVEL1_NHR) {
261 : level1TempAlg
262 0 : = AlgTemplateRegistry::Instance().GetAlgTemplate(TemplateType::TEMPLATE_ALL_REDUCE_NHR, dispatcher_);
263 0 : } else if (algType_.algoLevel1 == AlgTypeLevel1::ALG_LEVEL1_NHR_V1) {
264 : level1TempAlg
265 0 : = AlgTemplateRegistry::Instance().GetAlgTemplate(TemplateType::TEMPLATE_ALL_REDUCE_NHR_V1, dispatcher_);
266 0 : } else if (algType_.algoLevel1 == AlgTypeLevel1::ALG_LEVEL1_AHC) {
267 : level1TempAlg
268 0 : = AlgTemplateRegistry::Instance().GetAlgTemplate(TemplateType::TEMPLATE_ALL_REDUCE_AHC, dispatcher_);
269 0 : HCCL_INFO("AllReduce mesh: using ahc algo inter-server.");
270 0 : } else if (algType_.algoLevel1 == AlgTypeLevel1::ALG_LEVEL1_AHC_BROKE) {
271 0 : level1TempAlg = AlgTemplateRegistry::Instance().GetAlgTemplate(
272 0 : TemplateType::TEMPLATE_ALL_REDUCE_AHC_BROKE, dispatcher_);
273 0 : HCCL_INFO("AllReduce mesh: using ahc-broke algo inter-server.");
274 0 : } else if (algType_.algoLevel1 == AlgTypeLevel1::ALG_LEVEL1_NB) {
275 : level1TempAlg
276 0 : = AlgTemplateRegistry::Instance().GetAlgTemplate(TemplateType::TEMPLATE_ALL_REDUCE_NB, dispatcher_);
277 0 : HCCL_INFO("AllReduce mesh: using nb algo inter-server.");
278 : } else {
279 0 : level1TempAlg = AlgTemplateRegistry::Instance().GetAlgTemplate(
280 0 : TemplateType::TEMPLATE_ALL_REDUCE_RECURSIVE_HALVING_DOUBLING, dispatcher_);
281 0 : HCCL_INFO("AllReduce mesh: using Recursive halving-doubling algo inter-server.");
282 : }
283 0 : CHK_SMART_PTR_NULL(level1TempAlg);
284 0 : return HCCL_SUCCESS;
285 : }
286 0 : return HCCL_E_UNAVAIL;
287 : }
288 : REGISTER_EXEC("AllReduceMeshExecutor", AllReduceMesh, CollAllReduceMeshExecutor);
289 :
290 : } // namespace hccl
|
