Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "coll_all_gather_mesh_graph_executor.h"
12 :
13 : namespace hccl {
14 1 : CollAllGatherMeshGraphExecutor::CollAllGatherMeshGraphExecutor(
15 1 : const HcclDispatcher dispatcher, std::unique_ptr<TopoMatcher>& topoMatcher)
16 1 : : CollAllGatherExecutor(dispatcher, topoMatcher)
17 : {
18 4 : DMAReduceFlag_ = false;
19 4 : }
20 :
21 4 : HcclResult CollAllGatherMeshGraphExecutor::CalcStreamNum(u32& streamNum)
22 : {
23 4 : u32 totalStreamNum = topoAttr_.deviceNumPerAggregation > 1U ? topoAttr_.deviceNumPerAggregation - 1U : 1U;
24 4 : streamNum = totalStreamNum - 1U;
25 4 : HCCL_INFO("[CollAllGatherMeshGraphExecutor][CalcStreamNum] tag[%s] streamNum[%u]", tag_.c_str(), streamNum);
26 4 : return HCCL_SUCCESS;
27 : }
28 :
29 4 : HcclResult CollAllGatherMeshGraphExecutor::CalcCommInfo(std::vector<LevelNSubCommTransport>& opTransport)
30 : {
31 4 : TransportMemType inputType = TransportMemType::RESERVED;
32 4 : TransportMemType outputType = TransportMemType::RESERVED;
33 4 : CHK_RET(CalcTransportMemType(inputType, outputType));
34 4 : CHK_RET(CalcLevel0CommInfo(inputType, outputType, opTransport));
35 4 : CHK_RET(CalcLevel1CommInfo(inputType, outputType, opTransport));
36 4 : return HCCL_SUCCESS;
37 : }
38 :
39 : HcclResult
40 4 : CollAllGatherMeshGraphExecutor::CalcTransportMemType(TransportMemType& inputType, TransportMemType& outputType)
41 : {
42 4 : inputType = TransportMemType::PARAM_INPUT;
43 4 : outputType = TransportMemType::PARAM_OUTPUT;
44 4 : HCCL_INFO(
45 : "[CollAllGatherMeshGraphExecutor][CalcTransportMemType] tag[%s] inputType[%d], outputType[%d]", tag_.c_str(),
46 : inputType, outputType);
47 4 : return HCCL_SUCCESS;
48 : }
49 :
50 4 : HcclResult CollAllGatherMeshGraphExecutor::CalcLevel0CommInfo(
51 : TransportMemType inputType, TransportMemType outputType, std::vector<LevelNSubCommTransport>& opTransport)
52 : {
53 4 : CommParaInfo commParaLevel0(COMM_LEVEL0, CommType::COMM_TAG_MESH);
54 8 : commParaLevel0.meshSinglePlane = (topoAttr_.deviceType == DevType::DEV_TYPE_910B)
55 4 : && topoMatcher_->GetExternalInputHcclDeterministic() == DETERMINISTIC_DISABLE
56 8 : && (workflowMode_ != HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE);
57 4 : CHK_RET(CalcCommPlaneInfo(tag_, commParaLevel0, opTransport[COMM_LEVEL0], inputType, outputType));
58 4 : return HCCL_SUCCESS;
59 4 : }
60 :
61 0 : HcclResult CollAllGatherMeshGraphExecutor::KernelRun(const OpParam& param, ExecMem& execMem)
62 : {
63 0 : HCCL_CONFIG_INFO(HCCL_ALG, "[CollAllGatherMeshGraphExecutor][KernelRun]AllGather start");
64 :
65 : // 获取子通信域信息
66 0 : u32 perDataSize = SIZE_TABLE[param.DataDes.dataType];
67 0 : CHK_RET(CheckCommSize(COMM_LEVEL0, COMM_INDEX_0 + 1));
68 0 : SubCommInfo level0CommInfo = GetSubCommInfo(COMM_LEVEL0, COMM_INDEX_0);
69 0 : u32 level0RankSize = level0CommInfo.localRankSize;
70 0 : u32 commIndex = level0CommInfo.localRank;
71 0 : CHK_RET(CheckCommSize(COMM_LEVEL1, commIndex + 1));
72 0 : SubCommInfo level1CommInfo = GetSubCommInfo(COMM_LEVEL1, commIndex);
73 0 : u32 level1RankSize = level1CommInfo.localRankSize;
74 0 : u32 serverIndex = level1CommInfo.localRank;
75 0 : u64 inputMemSize = execMem.inputMem.size();
76 0 : CHK_RET(ActiveSlaveStreams(param.stream));
77 :
78 : // 第一步，将数据从input内存拷贝到output内存的对应位置
79 0 : u32 transposeRankId = commIndex * level1RankSize + serverIndex;
80 0 : u64 initOffset = transposeRankId * inputMemSize;
81 0 : DeviceMem dstMem = execMem.outputMem.range(initOffset, inputMemSize);
82 0 : CHK_SMART_PTR_NULL(dstMem);
83 :
84 0 : HcclResult ret = HcclD2DMemcpyAsync(dispatcher_, dstMem, execMem.inputMem, const_cast<Stream&>(param.stream));
85 0 : CHK_PRT_RET(
86 : ret != HCCL_SUCCESS,
87 : HCCL_ERROR(
88 : "[CollAllGatherMeshGraphExecutor][KernelRun]AllGather 4PmeshHD memcpy Failed, Offset[%llu], Size[%llu].",
89 : initOffset, inputMemSize),
90 : ret);
91 :
92 : // 第二步，各个AI Server 间 AllGather
93 0 : u64 inputDataCount = inputMemSize / perDataSize;
94 0 : u64 level1Offset = commIndex * level1RankSize * inputMemSize;
95 0 : HCCL_INFO(
96 : "[CollAllGatherMeshGraphExecutor][KernelRun] userRank[%u] commIndex[%u] level1Offset[%llu] level0RankSize[%u] "
97 : "level1RankSize[%u] outputMemSize[%llu] count[%u]",
98 : topoAttr_.userRank, commIndex, level1Offset, level0RankSize, level1RankSize, execMem.outputMem.size(),
99 : execMem.count);
100 0 : DeviceMem level1OutputMem = execMem.outputMem.range(level1Offset, inputMemSize * level1RankSize);
101 0 : CHK_SMART_PTR_NULL(level1OutputMem);
102 :
103 0 : std::unique_ptr<AlgTemplateBase> level1TempAlg;
104 0 : if (algType_.algoLevel1 == AlgTypeLevel1::ALG_LEVEL1_RING
105 0 : || (topoAttr_.isDiffDeviceModule && topoAttr_.serverNum == 1)) {
106 : // 1-单server-SDMA
107 : level1TempAlg
108 0 : = AlgTemplateRegistry::Instance().GetAlgTemplate(TemplateType::TEMPLATE_ALL_GATHER_RING, dispatcher_);
109 0 : HCCL_INFO("AllGather mesh: using ring algo inter-server.");
110 0 : } else if (algType_.algoLevel1 == AlgTypeLevel1::ALG_LEVEL1_NHR) {
111 : level1TempAlg
112 0 : = AlgTemplateRegistry::Instance().GetAlgTemplate(TemplateType::TEMPLATE_ALL_GATHER_NHR, dispatcher_);
113 0 : HCCL_INFO("AllGather mesh: using nhr algo inter-server.");
114 0 : } else if (algType_.algoLevel1 == AlgTypeLevel1::ALG_LEVEL1_NHR_V1) {
115 : level1TempAlg
116 0 : = AlgTemplateRegistry::Instance().GetAlgTemplate(TemplateType::TEMPLATE_ALL_GATHER_NHRV1, dispatcher_);
117 0 : HCCL_INFO("AllGather mesh: using nhr_v1 algo inter-server.");
118 0 : } else if (algType_.algoLevel1 == AlgTypeLevel1::ALG_LEVEL1_NB) {
119 : level1TempAlg
120 0 : = AlgTemplateRegistry::Instance().GetAlgTemplate(TemplateType::TEMPLATE_ALL_GATHER_NB, dispatcher_);
121 0 : HCCL_INFO("AllGather mesh: using nonuniform-bruck algo inter-server.");
122 : } else {
123 0 : level1TempAlg = AlgTemplateRegistry::Instance().GetAlgTemplate(
124 0 : TemplateType::TEMPLATE_ALL_GATHER_RECURSIVE_HALVING_DOUBLING, dispatcher_);
125 0 : HCCL_INFO(
126 : "AllGather mesh: algo is [%s] using halving-doubling algo inter-server.",
127 : (HCCL_ALGO_LEVEL1_MAP.at(algType_.algoLevel1)).c_str());
128 : }
129 0 : CHK_SMART_PTR_NULL(level1TempAlg);
130 : // 此处虽然带入inputMem作为scratch mem, 但inputMem 不能被使用
131 0 : CHK_RET(level1TempAlg->Prepare(
132 : level1OutputMem, level1OutputMem, execMem.inputMem, inputDataCount, param.DataDes.dataType, param.stream,
133 : HcclReduceOp::HCCL_REDUCE_RESERVED, INVALID_VALUE_RANKID, std::vector<Slice>(COMM_INDEX_0), level1Offset));
134 :
135 0 : u32 rankSize = level1CommInfo.localRankSize;
136 0 : CHK_RET(level1TempAlg->RegisterProfiler(
137 : (rankSize << PROF_RANKSIZE_OFFSET_OF_PLANEID) + serverIndex, PROF_STAGE_1, HCCL_EXEC_STEP_NOT_SET,
138 : param.stream));
139 :
140 0 : CHK_RET(RunTemplate(level1TempAlg, level1CommInfo));
141 0 : HCCL_INFO("[CollAllGatherMeshGraphExecutor][KernelRun] AllGather level1 run success");
142 :
143 : // 第3步 机内AllGather
144 0 : std::vector<Slice> dataSegsSlice; // 数据分成ranksize份，每份的起始偏移和大小
145 0 : std::vector<std::vector<Slice>> multiStreamSlice; // 每个stream使用的数据基于用户buffer的偏移
146 0 : u32 sliceNum = level0RankSize;
147 0 : CHK_RET(PrepareAllgatherSlice(sliceNum, inputMemSize * level1RankSize, dataSegsSlice));
148 : // mesh算法stream数量为server内rank数减1
149 0 : CHK_RET(AlgTemplateBase::PrepareSliceMeshStreams(dataSegsSlice, sliceNum - 1, multiStreamSlice));
150 :
151 0 : std::unique_ptr<AlgTemplateBase> level0TempAlg;
152 0 : if (topoAttr_.deviceType == DevType::DEV_TYPE_910B) {
153 0 : level0TempAlg = AlgTemplateRegistry::Instance().GetAlgTemplate(
154 0 : TemplateType::TEMPLATE_ALL_GATHER_MESH_ATOMIC, dispatcher_);
155 : } else {
156 0 : HCCL_ERROR(
157 : "[CollAllGatherMeshGraphExecutor][KernelRun] current device type [%u] not supported.",
158 : topoAttr_.deviceType);
159 0 : return HCCL_E_NOT_SUPPORT;
160 : }
161 0 : CHK_SMART_PTR_NULL(level0TempAlg);
162 0 : CHK_RET(level0TempAlg->Prepare(
163 : algResResp_->slaveStreams, algResResp_->notifiesMain, algResResp_->notifiesAux, topoAttr_.userRank, nullptr,
164 : commIndex, level0RankSize));
165 0 : CHK_RET(level0TempAlg->Prepare(
166 : execMem.outputMem, execMem.outputMem, execMem.inputMem, execMem.count * level0RankSize * level1RankSize,
167 : param.DataDes.dataType, param.stream, HCCL_REDUCE_RESERVED, LEVEL0_BRIDGE_RANK_ID, dataSegsSlice, 0));
168 0 : rankSize = level0RankSize;
169 0 : CHK_RET(level0TempAlg->RegisterProfiler(
170 : (rankSize << PROF_RANKSIZE_OFFSET_OF_PLANEID) + commIndex, PROF_STAGE_2, HCCL_EXEC_STEP_NOT_SET, param.stream));
171 0 : CHK_RET(RunTemplate(level0TempAlg, level0CommInfo));
172 0 : HCCL_INFO("[CollAllGatherMeshGraphExecutor][KernelRun] AllGather level0 run success");
173 :
174 : // 第4步 内存重排
175 0 : std::vector<bool> reorderedDone(topoAttr_.userRankSize, false);
176 0 : for (u32 scratchSliceIndex = 0; scratchSliceIndex < topoAttr_.userRankSize; scratchSliceIndex++) {
177 0 : if (reorderedDone[scratchSliceIndex]) {
178 0 : continue;
179 : }
180 :
181 : // 当前位置应该放的数据实际所在位置
182 0 : u32 srcServerId = scratchSliceIndex / level0RankSize;
183 0 : u32 srcLocalRank = scratchSliceIndex % level0RankSize;
184 0 : u32 srcSliceRankId = srcLocalRank * level1RankSize + srcServerId;
185 0 : if (srcSliceRankId == scratchSliceIndex) {
186 0 : reorderedDone[scratchSliceIndex] = true;
187 0 : continue;
188 : }
189 :
190 0 : u32 dstSliceRankId = scratchSliceIndex;
191 : // 将当前数据拷贝到input暂存
192 0 : u64 srcOutputOffset = scratchSliceIndex * inputMemSize;
193 0 : u64 dstOutputOffset = dstSliceRankId * inputMemSize;
194 0 : DeviceMem currentDataMem = execMem.outputMem.range(dstOutputOffset, inputMemSize);
195 0 : CHK_SMART_PTR_NULL(currentDataMem);
196 0 : ret = HcclD2DMemcpyAsync(dispatcher_, execMem.inputMem, currentDataMem, const_cast<Stream&>(param.stream));
197 0 : CHK_PRT_RET(
198 : ret != HCCL_SUCCESS,
199 : HCCL_ERROR(
200 : "[CollAllGatherMeshGraphExecutor][KernelRun] slice[%u] to input mem temp failed", scratchSliceIndex),
201 : ret);
202 : // 环遍历到当前位置退出循环
203 0 : u32 loopCount = 0;
204 0 : while (srcSliceRankId != scratchSliceIndex) {
205 0 : HCCL_INFO(
206 : "[CollAllGatherMeshGraphExecutor][KernelRun] slice[%u] to slice[%u]", srcSliceRankId, dstSliceRankId);
207 0 : srcOutputOffset = srcSliceRankId * inputMemSize;
208 0 : dstOutputOffset = dstSliceRankId * inputMemSize;
209 0 : DeviceMem srcOutputMem = execMem.outputMem.range(srcOutputOffset, inputMemSize);
210 0 : CHK_SMART_PTR_NULL(srcOutputMem);
211 0 : DeviceMem dstOutputMem = execMem.outputMem.range(dstOutputOffset, inputMemSize);
212 0 : CHK_SMART_PTR_NULL(dstOutputMem);
213 0 : ret = HcclD2DMemcpyAsync(dispatcher_, dstOutputMem, srcOutputMem, const_cast<Stream&>(param.stream));
214 0 : CHK_PRT_RET(
215 : ret != HCCL_SUCCESS,
216 : HCCL_ERROR(
217 : "[CollAllGatherMeshGraphExecutor][KernelRun] slice[%u] to slice[%u] failed", srcSliceRankId,
218 : dstSliceRankId),
219 : ret);
220 0 : reorderedDone[dstSliceRankId] = true;
221 0 : dstSliceRankId = srcSliceRankId;
222 0 : srcServerId = srcSliceRankId / level0RankSize;
223 0 : srcLocalRank = srcSliceRankId % level0RankSize;
224 0 : srcSliceRankId = srcLocalRank * level1RankSize + srcServerId;
225 :
226 0 : loopCount++;
227 0 : if (loopCount > topoAttr_.userRankSize) {
228 0 : HCCL_ERROR("[CollAllGatherMeshGraphExecutor][KernelRun] ERROR: loop exceeds user rank size");
229 0 : return HCCL_E_INTERNAL;
230 : }
231 0 : }
232 0 : dstOutputOffset = dstSliceRankId * inputMemSize;
233 0 : DeviceMem ringEndMem = execMem.outputMem.range(dstOutputOffset, inputMemSize);
234 0 : CHK_SMART_PTR_NULL(ringEndMem);
235 0 : ret = HcclD2DMemcpyAsync(dispatcher_, ringEndMem, execMem.inputMem, const_cast<Stream&>(param.stream));
236 0 : CHK_PRT_RET(
237 : ret != HCCL_SUCCESS,
238 : HCCL_ERROR("[CollAllGatherMeshGraphExecutor][KernelRun] temp input to slice[%u] failed", dstSliceRankId),
239 : ret);
240 0 : reorderedDone[dstSliceRankId] = true;
241 0 : }
242 0 : return HCCL_SUCCESS;
243 0 : }
244 0 : HcclResult CollAllGatherMeshGraphExecutor::Getlevel1CommRank(SubCommInfo& level1CommInfo)
245 : {
246 0 : if (CheckCommSize(COMM_LEVEL0, COMM_INDEX_0 + 1) != HCCL_SUCCESS) {
247 0 : return HCCL_E_UNAVAIL;
248 : }
249 0 : SubCommInfo level0CommInfo = GetSubCommInfo(COMM_LEVEL0, COMM_INDEX_0);
250 0 : u32 ringNum
251 0 : = (topoType_ == TopoType::TOPO_TYPE_8P_RING) ? LEVEL0_PLANE_NUM_IN_8PRING : LEVEL0_PLANE_NUM_IN_NPRING_SINGLE;
252 0 : u32 commIndex = (ringNum == LEVEL0_PLANE_NUM_IN_8PRING) ? topoAttr_.devicePhyId : level0CommInfo.localRank;
253 :
254 0 : if (CheckCommSize(COMM_LEVEL1, commIndex + 1) != HCCL_SUCCESS) {
255 0 : return HCCL_E_UNAVAIL;
256 : }
257 0 : level1CommInfo = GetSubCommInfo(COMM_LEVEL1, commIndex);
258 :
259 0 : return HCCL_SUCCESS;
260 0 : }
261 :
262 : HcclResult
263 0 : CollAllGatherMeshGraphExecutor::SelectTempAlg(std::unique_ptr<AlgTemplateBase>& level1TempAlg, u32 level1RankSize)
264 : {
265 0 : if (level1RankSize > 1) {
266 0 : if (algType_.algoLevel1 == AlgTypeLevel1::ALG_LEVEL1_RING
267 0 : || (topoAttr_.isDiffDeviceModule && topoAttr_.serverNum == 1)) {
268 : // 1-单server-SDMA
269 : level1TempAlg
270 0 : = AlgTemplateRegistry::Instance().GetAlgTemplate(TemplateType::TEMPLATE_ALL_GATHER_RING, dispatcher_);
271 0 : HCCL_INFO("AllGather mesh: using ring algo inter-server.");
272 0 : } else if (algType_.algoLevel1 == AlgTypeLevel1::ALG_LEVEL1_NHR) {
273 : level1TempAlg
274 0 : = AlgTemplateRegistry::Instance().GetAlgTemplate(TemplateType::TEMPLATE_ALL_GATHER_NHR, dispatcher_);
275 0 : HCCL_INFO("AllGather mesh: using nhr algo inter-server.");
276 0 : } else if (algType_.algoLevel1 == AlgTypeLevel1::ALG_LEVEL1_NHR_V1) {
277 : level1TempAlg
278 0 : = AlgTemplateRegistry::Instance().GetAlgTemplate(TemplateType::TEMPLATE_ALL_GATHER_NHRV1, dispatcher_);
279 0 : HCCL_INFO("AllGather mesh: using nhr_v1 algo inter-server.");
280 0 : } else if (algType_.algoLevel1 == AlgTypeLevel1::ALG_LEVEL1_NB) {
281 : level1TempAlg
282 0 : = AlgTemplateRegistry::Instance().GetAlgTemplate(TemplateType::TEMPLATE_ALL_GATHER_NB, dispatcher_);
283 0 : HCCL_INFO("AllGather mesh: using nonuniform-bruck algo inter-server.");
284 : } else {
285 0 : level1TempAlg = AlgTemplateRegistry::Instance().GetAlgTemplate(
286 0 : TemplateType::TEMPLATE_ALL_GATHER_RECURSIVE_HALVING_DOUBLING, dispatcher_);
287 0 : HCCL_INFO(
288 : "AllGather mesh: algo is [%s] using halving-doubling algo inter-server.",
289 : (HCCL_ALGO_LEVEL1_MAP.at(algType_.algoLevel1)).c_str());
290 : }
291 0 : CHK_SMART_PTR_NULL(level1TempAlg);
292 0 : return HCCL_SUCCESS;
293 : }
294 0 : return HCCL_E_UNAVAIL;
295 : }
296 : REGISTER_EXEC("AllGatherMeshGraphExecutor", AllGatherMeshGraph, CollAllGatherMeshGraphExecutor);
297 : } // namespace hccl
|
