Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "coll_reduce_scatter_mesh_graph_executor.h"
12 :
13 : namespace hccl {
14 :
15 8 : CollReduceScatterMeshGraphExecutor::CollReduceScatterMeshGraphExecutor(
16 8 : const HcclDispatcher dispatcher, std::unique_ptr<TopoMatcher>& topoMatcher)
17 8 : : CollReduceScatterExecutor(dispatcher, topoMatcher)
18 : {
19 8 : DMAReduceFlag_ = false;
20 8 : }
21 :
22 8 : void CollReduceScatterMeshGraphExecutor::ParseParam(const OpParam& param)
23 : {
24 8 : tag_ = param.tag;
25 :
26 : // 910B 图模式非确定计算，inlineReduce使能，MESH拓扑场景下，创建一个mesh平面
27 : bool isInlineReduce
28 8 : = IsSupportSDMAReduce(param.inputPtr, param.outputPtr, param.DataDes.dataType, param.reduceType);
29 16 : meshSinglePlane_ = (topoAttr_.deviceType == DevType::DEV_TYPE_910B)
30 8 : && topoMatcher_->GetExternalInputHcclDeterministic() == DETERMINISTIC_DISABLE && isInlineReduce
31 16 : && (workflowMode_ != HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE);
32 :
33 : // 是否需要scratch memory
34 8 : scratchMemFlag_ = true;
35 :
36 : // 记录图模式总数据量
37 8 : totalSize_ = topoAttr_.userRankSize * param.DataDes.count * SIZE_TABLE[param.DataDes.dataType];
38 8 : aicpuUnfoldMode_ = param.aicpuUnfoldMode;
39 8 : }
40 :
41 8 : HcclResult CollReduceScatterMeshGraphExecutor::CalcScratchMemSize(u64& scratchMemSize)
42 : {
43 8 : if (scratchMemFlag_) {
44 8 : scratchMemSize = totalSize_;
45 : } else {
46 0 : scratchMemSize = 0U;
47 : }
48 8 : HCCL_INFO(
49 : "[CollReduceScatterMeshGraphExecutor][CalcScratchMemSize] tag[%s] scratchMemSize[%llu]", tag_.c_str(),
50 : scratchMemSize);
51 8 : return HCCL_SUCCESS;
52 : }
53 :
54 8 : HcclResult CollReduceScatterMeshGraphExecutor::CalcStreamNum(u32& streamNum)
55 : {
56 8 : u32 totalStreamNum = topoAttr_.deviceNumPerAggregation > 1U ? topoAttr_.deviceNumPerAggregation - 1U : 1U;
57 8 : streamNum = totalStreamNum - 1U;
58 8 : HCCL_INFO("[CollReduceScatterMeshGraphExecutor][CalcStreamNum] tag[%s] streamNum[%u]", tag_.c_str(), streamNum);
59 8 : return HCCL_SUCCESS;
60 : }
61 :
62 8 : HcclResult CollReduceScatterMeshGraphExecutor::CalcCommInfo(std::vector<LevelNSubCommTransport>& opTransport)
63 : {
64 8 : TransportMemType inputType = TransportMemType::RESERVED;
65 8 : TransportMemType outputType = TransportMemType::RESERVED;
66 8 : CHK_RET(CalcTransportMemType(inputType, outputType));
67 8 : CHK_RET(CalcLevel0CommInfo(inputType, outputType, opTransport));
68 8 : CHK_RET(CalcLevel1CommInfo(inputType, outputType, opTransport));
69 8 : return HCCL_SUCCESS;
70 : }
71 :
72 : HcclResult
73 8 : CollReduceScatterMeshGraphExecutor::CalcTransportMemType(TransportMemType& inputType, TransportMemType& outputType)
74 : {
75 8 : inputType = TransportMemType::SCRATCH;
76 8 : outputType = TransportMemType::PARAM_INPUT;
77 :
78 8 : HCCL_INFO(
79 : "[CollReduceScatterMeshGraphExecutor][CalcTransportMemType] tag[%s] inputType[%d], outputType[%d]",
80 : tag_.c_str(), inputType, outputType);
81 8 : return HCCL_SUCCESS;
82 : }
83 :
84 8 : HcclResult CollReduceScatterMeshGraphExecutor::CalcLevel0CommInfo(
85 : TransportMemType inputType, TransportMemType outputType, std::vector<LevelNSubCommTransport>& opTransport)
86 : {
87 8 : CommParaInfo commParaLevel0(COMM_LEVEL0, CommType::COMM_TAG_MESH);
88 8 : commParaLevel0.meshSinglePlane = meshSinglePlane_;
89 8 : CHK_RET(CalcCommPlaneInfo(tag_, commParaLevel0, opTransport[COMM_LEVEL0], inputType, outputType));
90 8 : return HCCL_SUCCESS;
91 8 : }
92 :
93 0 : bool CollReduceScatterMeshGraphExecutor::IsHugeData(const u64 curSize, [[maybe_unused]] OpParam* param)
94 : {
95 0 : bool hugeData = (curSize * topoAttr_.userRankSize / HCCL_INTERNODE_MAX_DATA_RATE > RDMA_SEND_MAX_SIZE)
96 0 : || (curSize > SDMA_SEND_MAX_SIZE);
97 0 : return hugeData;
98 : }
99 :
100 0 : HcclResult CollReduceScatterMeshGraphExecutor::KernelRun(const OpParam& param, ExecMem& execMem)
101 : {
102 0 : HCCL_CONFIG_INFO(
103 : HCCL_ALG, "[CollReduceScatterMeshGraphExecutor][KernelRun] userRank[%u] starts.", topoAttr_.userRank);
104 :
105 0 : u32 perDataSize = SIZE_TABLE[param.DataDes.dataType];
106 0 : u64 singleRankDataSize = execMem.count * perDataSize;
107 :
108 0 : CHK_RET(CheckCommSize(COMM_LEVEL0, COMM_INDEX_0 + 1));
109 0 : SubCommInfo level0CommInfo = GetSubCommInfo(COMM_LEVEL0, COMM_INDEX_0);
110 0 : u32 commIndex = level0CommInfo.localRank; // 找到rank所在的节点间平面
111 0 : u32 level0RankSize = level0CommInfo.localRankSize;
112 0 : CHK_RET(CheckCommSize(COMM_LEVEL1, commIndex + 1));
113 0 : SubCommInfo level1CommInfo = GetSubCommInfo(COMM_LEVEL1, commIndex);
114 0 : u32 serverIndex = level1CommInfo.localRank;
115 0 : u32 level1RankSize = level1CommInfo.localRankSize;
116 0 : CHK_RET(ActiveSlaveStreams(param.stream));
117 :
118 : /* ******************第1步: input to scratch *******************************/
119 0 : HCCL_INFO(
120 : "[CollReduceScatterMeshGraphExecutor][KernelRun] userRank[%u], level0RankSize[%u], level1RankSize[%u]",
121 : topoAttr_.userRank, level0RankSize, level1RankSize);
122 0 : for (u32 inputSliceId = 0; inputSliceId < topoAttr_.userRankSize; inputSliceId++) {
123 0 : u32 dstServerId = inputSliceId / topoAttr_.deviceNumPerAggregation;
124 0 : u32 dstLocalRank = inputSliceId % topoAttr_.deviceNumPerAggregation;
125 0 : u32 dstSliceId = dstLocalRank * topoAttr_.moduleNum + dstServerId;
126 :
127 0 : u64 srcInputOffset = inputSliceId * singleRankDataSize;
128 0 : u64 dstScratchOffset = dstSliceId * singleRankDataSize;
129 :
130 0 : DeviceMem srcInputMem = execMem.inputMem.range(srcInputOffset, singleRankDataSize);
131 0 : CHK_SMART_PTR_NULL(srcInputMem);
132 0 : DeviceMem dstScratchMem = execMem.scratchMem.range(dstScratchOffset, singleRankDataSize);
133 0 : CHK_SMART_PTR_NULL(dstScratchMem);
134 :
135 0 : HcclResult ret = HcclD2DMemcpyAsync(dispatcher_, dstScratchMem, srcInputMem, const_cast<Stream&>(param.stream));
136 0 : CHK_PRT_RET(
137 : ret != HCCL_SUCCESS,
138 : HCCL_ERROR(
139 : "[CollReduceScatterMeshGraphExecutor][KernelRun] rank[%u] slice[%u] to slice[%u] failed",
140 : topoAttr_.userRank, inputSliceId, dstSliceId),
141 : ret);
142 0 : }
143 :
144 : /* ******************第2步: intranode *******************************/
145 0 : u32 sliceNum = level0CommInfo.localRankSize;
146 : // 根据数据量算每个环上数据的偏移和大小，把做完hd的slice均分成RankSize份
147 0 : std::vector<Slice> dataSegsSlice;
148 0 : u32 level0ReduceCount = execMem.count * level1RankSize;
149 0 : CHK_RET(PrepareReduceScatterSliceData(level0ReduceCount, perDataSize, sliceNum, dataSegsSlice));
150 :
151 : // 每个server分配的slice大小
152 0 : u64 serverSliceSize = execMem.inputMem.size();
153 : // 每个服务器对应的偏移
154 0 : u64 serverSliceOffset = 0;
155 :
156 0 : HCCL_DEBUG(
157 : "inputMem.size=%llu, level0CommInfo.localRankSize=%u, serverSliceSize=%llu, serverSliceOffset=%llu "
158 : "commIndex=%u level1CommInfo.localRank=%u",
159 : execMem.inputMem.size(), level0CommInfo.localRankSize, serverSliceSize, serverSliceOffset, commIndex,
160 : level1CommInfo.localRank);
161 :
162 0 : DeviceMem reduceScatterMeshInput = execMem.scratchMem.range(serverSliceOffset, serverSliceSize);
163 0 : CHK_SMART_PTR_NULL(reduceScatterMeshInput);
164 0 : DeviceMem reduceScatterMeshOutput = execMem.inputMem.range(serverSliceOffset, serverSliceSize);
165 0 : CHK_SMART_PTR_NULL(reduceScatterMeshOutput);
166 :
167 0 : HcomCollOpInfo* opInfoPtr = nullptr;
168 :
169 0 : if (topoMatcher_->GetExternalInputHcclDeterministic() == DETERMINISTIC_DISABLE
170 0 : && (param.DataDes.dataType != HCCL_DATA_TYPE_INT64)
171 0 : && (topoAttr_.deviceType == DevType::DEV_TYPE_910B && param.reduceType != HCCL_REDUCE_PROD)) {
172 0 : CHK_RET(MultiStreamReduceScatterMeshAtomic(
173 : param.tag, reduceScatterMeshInput, reduceScatterMeshOutput, // 非确定性
174 : level0ReduceCount, param.DataDes.dataType, param.reduceType, dataSegsSlice,
175 : const_cast<Stream&>(param.stream), COMM_LEVEL0, serverSliceOffset, opInfoPtr));
176 : } else {
177 0 : std::vector<std::vector<Slice>> multiStreamSlice; // 每个stream使用的数据基于用户buffer的偏移
178 : // mesh算法stream数量为rank数减1
179 0 : CHK_RET(AlgTemplateBase::PrepareSliceMeshStreams(dataSegsSlice, sliceNum - 1, multiStreamSlice));
180 0 : CHK_RET(MultiStreamReduceScatterMesh(
181 : param.tag, reduceScatterMeshInput, reduceScatterMeshOutput, // 确定性
182 : level0ReduceCount, param.DataDes.dataType, param.reduceType, multiStreamSlice,
183 : const_cast<Stream&>(param.stream), COMM_LEVEL0, serverSliceOffset));
184 0 : }
185 :
186 : /* ******************第3步: internode *******************************/
187 :
188 0 : if (level1RankSize > 1) {
189 0 : u64 reduceAttr = GetReduceAttr(execMem.inputMem, execMem.outputMem, param.DataDes.dataType, param.reduceType);
190 0 : std::unique_ptr<AlgTemplateBase> level1TempAlg;
191 0 : u64 ringCount = execMem.count;
192 0 : u64 level1SliceSize = execMem.inputMem.size() / level0RankSize;
193 : // 每个服务器对应的偏移
194 0 : u64 level1SliceOffset = commIndex * level1SliceSize;
195 0 : DeviceMem level1ReduceScatterInput = execMem.scratchMem.range(level1SliceOffset, level1SliceSize);
196 0 : CHK_SMART_PTR_NULL(level1ReduceScatterInput);
197 0 : DeviceMem level1ReduceScatterScratch = execMem.inputMem.range(level1SliceOffset, level1SliceSize);
198 0 : CHK_SMART_PTR_NULL(level1ReduceScatterScratch);
199 0 : HCCL_INFO(
200 : "[CollReduceScatterMeshGraphExecutor][KernelRun] rank[%u] level 1 sliceSize[%llu] sliceOffset[%llu]",
201 : topoAttr_.userRank, level1SliceSize, level1SliceOffset);
202 0 : if (algType_.algoLevel1 == AlgTypeLevel1::ALG_LEVEL1_RING) {
203 0 : level1TempAlg = AlgTemplateRegistry::Instance().GetAlgTemplate(
204 0 : TemplateType::TEMPLATE_REDUCESCATTER_RING, dispatcher_);
205 0 : CHK_SMART_PTR_NULL(level1TempAlg);
206 0 : CHK_RET(level1TempAlg->Prepare(reduceAttr));
207 0 : HCCL_INFO("ReduceScatter mesh: using ring algo inter-server.");
208 0 : CHK_RET(level1TempAlg->Prepare(
209 : level1ReduceScatterInput, level1ReduceScatterInput, level1ReduceScatterScratch, ringCount,
210 : param.DataDes.dataType, param.stream, param.reduceType, LEVEL0_BRIDGE_RANK_ID, std::vector<Slice>(0),
211 : level1SliceOffset));
212 0 : } else if (algType_.algoLevel1 == AlgTypeLevel1::ALG_LEVEL1_NHR) {
213 : level1TempAlg
214 0 : = AlgTemplateRegistry::Instance().GetAlgTemplate(TemplateType::TEMPLATE_REDUCESCATTER_NHR, dispatcher_);
215 0 : HCCL_INFO("ReduceScatter mesh: using nhr algo inter-server.");
216 0 : CHK_SMART_PTR_NULL(level1TempAlg);
217 0 : CHK_RET(level1TempAlg->Prepare(reduceAttr, false));
218 0 : CHK_RET(level1TempAlg->Prepare(
219 : level1ReduceScatterInput, level1ReduceScatterInput, level1ReduceScatterScratch, ringCount,
220 : param.DataDes.dataType, param.stream, param.reduceType, LEVEL0_BRIDGE_RANK_ID, std::vector<Slice>(0),
221 : level1SliceOffset));
222 0 : level1TempAlg->CloseBarrier();
223 0 : } else if (algType_.algoLevel1 == AlgTypeLevel1::ALG_LEVEL1_NHR_V1) {
224 0 : level1TempAlg = AlgTemplateRegistry::Instance().GetAlgTemplate(
225 0 : TemplateType::TEMPLATE_REDUCESCATTER_NHR_V1, dispatcher_);
226 0 : HCCL_INFO("ReduceScatter mesh: using nhr_v1 algo inter-server.");
227 0 : CHK_SMART_PTR_NULL(level1TempAlg);
228 0 : CHK_RET(level1TempAlg->Prepare(reduceAttr));
229 0 : CHK_RET(level1TempAlg->Prepare(
230 : level1ReduceScatterInput, level1ReduceScatterInput, level1ReduceScatterScratch, ringCount,
231 : param.DataDes.dataType, param.stream, param.reduceType, LEVEL0_BRIDGE_RANK_ID, std::vector<Slice>(0),
232 : level1SliceOffset));
233 0 : } else if (algType_.algoLevel1 == AlgTypeLevel1::ALG_LEVEL1_NB) {
234 : level1TempAlg
235 0 : = AlgTemplateRegistry::Instance().GetAlgTemplate(TemplateType::TEMPLATE_REDUCESCATTER_NB, dispatcher_);
236 0 : HCCL_INFO("ReduceScatter mesh: using nonuniform-bruck algo inter-server.");
237 0 : CHK_SMART_PTR_NULL(level1TempAlg);
238 0 : CHK_RET(level1TempAlg->Prepare(reduceAttr));
239 0 : CHK_RET(level1TempAlg->Prepare(
240 : level1ReduceScatterInput, level1ReduceScatterInput, level1ReduceScatterScratch, ringCount,
241 : param.DataDes.dataType, param.stream, param.reduceType, LEVEL0_BRIDGE_RANK_ID, std::vector<Slice>(0),
242 : level1SliceOffset));
243 : } else {
244 0 : level1TempAlg = AlgTemplateRegistry::Instance().GetAlgTemplate(
245 0 : TemplateType::TEMPLATE_REDUCESCATTER_RECURSIVE_HD, dispatcher_);
246 0 : CHK_SMART_PTR_NULL(level1TempAlg);
247 0 : CHK_RET(level1TempAlg->Prepare(reduceAttr));
248 0 : HCCL_INFO(
249 : "ReduceScatter mesh: algo is [%s] using halving-doubling algo inter-server.",
250 : (HCCL_ALGO_LEVEL1_MAP.at(algType_.algoLevel1)).c_str());
251 0 : u64 inputDataCount = level1SliceSize / perDataSize; // count是output的数据个数
252 0 : CHK_RET(level1TempAlg->Prepare(
253 : level1ReduceScatterInput, level1ReduceScatterInput, level1ReduceScatterScratch, inputDataCount,
254 : param.DataDes.dataType, param.stream, param.reduceType, LEVEL0_BRIDGE_RANK_ID, std::vector<Slice>(0),
255 : level1SliceOffset));
256 : }
257 0 : CHK_RET(level1TempAlg->RegisterProfiler(
258 : (level1RankSize << PROF_RANKSIZE_OFFSET_OF_PLANEID) + level1CommInfo.localRank, PROF_STAGE_1,
259 : HCCL_EXEC_STEP_NOT_SET, param.stream));
260 0 : CHK_RET(RunTemplate(level1TempAlg, level1CommInfo));
261 0 : }
262 :
263 : /* *******************第4步: 节点内reducescatter ******************************************/
264 0 : u32 transposeRankId = commIndex * level1RankSize + serverIndex;
265 0 : u64 finalOffset = transposeRankId * singleRankDataSize;
266 0 : DeviceMem srcScratchMem = execMem.scratchMem.range(finalOffset, singleRankDataSize);
267 0 : CHK_SMART_PTR_NULL(srcScratchMem);
268 : HcclResult ret
269 0 : = HcclD2DMemcpyAsync(dispatcher_, execMem.outputMem, srcScratchMem, const_cast<Stream&>(param.stream));
270 0 : CHK_PRT_RET(
271 : ret != HCCL_SUCCESS,
272 : HCCL_ERROR(
273 : "[CollReduceScatterMeshGraphExecutor][KernelRun] rank[%u] memcpy failed, offset[%llu], size[%llu]",
274 : topoAttr_.userRank, finalOffset, singleRankDataSize),
275 : ret);
276 :
277 0 : return HCCL_SUCCESS;
278 0 : }
279 :
280 : REGISTER_EXEC("ReduceScatterMeshGraphExecutor", ReduceScatterMeshGraph, CollReduceScatterMeshGraphExecutor);
281 : } // namespace hccl
|
