Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "coll_all_gather_mesh_executor.h"
12 :
13 : namespace hccl {
14 0 : CollAllGatherMeshExecutor::CollAllGatherMeshExecutor(
15 0 : const HcclDispatcher dispatcher, std::unique_ptr<TopoMatcher>& topoMatcher)
16 0 : : CollAllGatherExecutor(dispatcher, topoMatcher)
17 : {
18 0 : DMAReduceFlag_ = false;
19 0 : }
20 :
21 0 : HcclResult CollAllGatherMeshExecutor::CalcStreamNum(u32& streamNum)
22 : {
23 0 : u32 totalStreamNum = topoAttr_.deviceNumPerAggregation > 1U ? topoAttr_.deviceNumPerAggregation - 1U : 1U;
24 0 : streamNum = totalStreamNum - 1U;
25 0 : HCCL_INFO("[CollAllGatherMeshExecutor][CalcStreamNum] tag[%s] streamNum[%u].", tag_.c_str(), streamNum);
26 0 : return HCCL_SUCCESS;
27 : }
28 :
29 0 : HcclResult CollAllGatherMeshExecutor::CalcCommInfo(std::vector<LevelNSubCommTransport>& opTransport)
30 : {
31 0 : TransportMemType inputType = TransportMemType::RESERVED;
32 0 : TransportMemType outputType = TransportMemType::RESERVED;
33 0 : CHK_RET(CalcTransportMemType(inputType, outputType));
34 0 : CHK_RET(CalcLevel0CommInfo(inputType, outputType, opTransport));
35 0 : CHK_RET(CalcLevel1CommInfo(inputType, outputType, opTransport));
36 0 : return HCCL_SUCCESS;
37 : }
38 :
39 0 : HcclResult CollAllGatherMeshExecutor::CalcTransportMemType(TransportMemType& inputType, TransportMemType& outputType)
40 : {
41 0 : if (workflowMode_ == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE) {
42 0 : inputType = TransportMemType::CCL_INPUT;
43 0 : outputType = TransportMemType::CCL_OUTPUT;
44 : } else {
45 0 : inputType = TransportMemType::PARAM_INPUT;
46 0 : outputType = TransportMemType::PARAM_OUTPUT;
47 : }
48 0 : HCCL_INFO(
49 : "[CollAllGatherMeshExecutor][CalcTransportMemType] tag[%s] inputType[%d], outputType[%d].", tag_.c_str(),
50 : inputType, outputType);
51 0 : return HCCL_SUCCESS;
52 : }
53 :
54 0 : HcclResult CollAllGatherMeshExecutor::CalcLevel0CommInfo(
55 : TransportMemType inputType, TransportMemType outputType, std::vector<LevelNSubCommTransport>& opTransport)
56 : {
57 0 : CommParaInfo commParaLevel0(COMM_LEVEL0, CommType::COMM_TAG_MESH);
58 0 : commParaLevel0.meshSinglePlane = (topoAttr_.deviceType == DevType::DEV_TYPE_910B)
59 0 : && topoMatcher_->GetExternalInputHcclDeterministic() == DETERMINISTIC_DISABLE
60 0 : && (workflowMode_ != HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE);
61 0 : CHK_RET(CalcCommPlaneInfo(tag_, commParaLevel0, opTransport[COMM_LEVEL0], inputType, outputType));
62 0 : return HCCL_SUCCESS;
63 0 : }
64 :
65 0 : u64 CollAllGatherMeshExecutor::CalcLoopMaxCount(const u64 cclBuffSize, const u32 unitSize)
66 : {
67 0 : u64 maxCountPerLoop = cclBuffSize / topoAttr_.userRankSize / HCCL_MIN_SLICE_ALIGN * HCCL_MIN_SLICE_ALIGN / unitSize;
68 0 : return maxCountPerLoop;
69 : }
70 :
71 0 : HcclResult CollAllGatherMeshExecutor::KernelRun(const OpParam& param, ExecMem& execMem)
72 : {
73 0 : HCCL_CONFIG_INFO(HCCL_ALG, "[CollAllGatherMeshExecutor][KernelRun]AllGather mesh start");
74 0 : u32 perDataSize = SIZE_TABLE[param.DataDes.dataType];
75 :
76 : // 获取子通信域信息
77 0 : CHK_RET(CheckCommSize(COMM_LEVEL0, COMM_INDEX_0 + 1));
78 0 : SubCommInfo level0CommInfo = GetSubCommInfo(COMM_LEVEL0, COMM_INDEX_0);
79 0 : u32 level0RankSize = level0CommInfo.localRankSize;
80 0 : u32 commIndex = level0CommInfo.localRank;
81 0 : CHK_RET(CheckCommSize(COMM_LEVEL1, commIndex + 1));
82 0 : SubCommInfo level1CommInfo = GetSubCommInfo(COMM_LEVEL1, commIndex);
83 0 : u32 serverIndex = level1CommInfo.localRank;
84 :
85 0 : HCCL_DEBUG("[CollAllGatherMeshExecutor][KernelRun]serverIndex is %u, commIndex is %u", serverIndex, commIndex);
86 0 : u64 inputMemSize = execMem.inputMem.size();
87 0 : u64 baseOffset = serverIndex * inputMemSize * level0RankSize;
88 0 : u64 level0Offset = commIndex * inputMemSize;
89 0 : DeviceMem dstMem = execMem.outputMem.range(baseOffset + level0Offset, inputMemSize);
90 0 : CHK_SMART_PTR_NULL(dstMem);
91 : // 第一步，将数据从input内存拷贝到output内存的对应位置
92 0 : HcclResult ret = HcclD2DMemcpyAsync(dispatcher_, dstMem, execMem.inputMem, const_cast<Stream&>(param.stream));
93 0 : CHK_PRT_RET(
94 : ret != HCCL_SUCCESS,
95 : HCCL_ERROR(
96 : "[CollAllGatherMeshExecutor][KernelRun]AllGather 4PmeshHD memcpy Failed, Offset[%llu], Size[%llu].",
97 : baseOffset + level0Offset, inputMemSize),
98 : ret);
99 :
100 : // 第二步，各个AI Server 内 multi stream mesh AllGather
101 0 : std::vector<Slice> dataSegsSlice; // 数据分成ranksize份，每份的起始偏移和大小
102 0 : std::vector<std::vector<Slice>> multiStreamSlice; // 每个stream使用的数据基于用户buffer的偏移
103 0 : u32 sliceNum = level0RankSize;
104 0 : CHK_RET(PrepareAllgatherSlice(sliceNum, inputMemSize, dataSegsSlice));
105 :
106 : // mesh算法stream数量为server内rank数减1
107 0 : CHK_RET(AlgTemplateBase::PrepareSliceMeshStreams(dataSegsSlice, sliceNum - 1, multiStreamSlice));
108 :
109 0 : CHK_RET(ActiveSlaveStreams(param.stream));
110 :
111 : // 抽取当前用于多环AllGather 的output内存数据
112 0 : DeviceMem currentOutputMem = execMem.outputMem.range(baseOffset, inputMemSize * level0RankSize);
113 0 : CHK_SMART_PTR_NULL(currentOutputMem);
114 :
115 0 : std::unique_ptr<AlgTemplateBase> level0TempAlg;
116 0 : if (topoAttr_.deviceType == DevType::DEV_TYPE_910B) {
117 0 : level0TempAlg = AlgTemplateRegistry::Instance().GetAlgTemplate(
118 0 : TemplateType::TEMPLATE_ALL_GATHER_MESH_ATOMIC, dispatcher_);
119 : } else {
120 : level0TempAlg
121 0 : = AlgTemplateRegistry::Instance().GetAlgTemplate(TemplateType::TEMPLATE_ALL_GATHER_MESH, dispatcher_);
122 : }
123 0 : CHK_SMART_PTR_NULL(level0TempAlg);
124 0 : CHK_RET(level0TempAlg->Prepare(
125 : algResResp_->slaveStreams, algResResp_->notifiesMain, algResResp_->notifiesAux, topoAttr_.userRank, nullptr,
126 : commIndex, level0RankSize));
127 0 : CHK_RET(level0TempAlg->Prepare(
128 : currentOutputMem, currentOutputMem, execMem.inputMem, execMem.count * level0RankSize, param.DataDes.dataType,
129 : param.stream, HCCL_REDUCE_RESERVED, LEVEL0_BRIDGE_RANK_ID, dataSegsSlice, baseOffset));
130 0 : u32 rankSize = level0RankSize;
131 0 : CHK_RET(level0TempAlg->RegisterProfiler(
132 : (rankSize << PROF_RANKSIZE_OFFSET_OF_PLANEID) + commIndex, PROF_STAGE_1, HCCL_EXEC_STEP_NOT_SET, param.stream));
133 0 : CHK_RET(RunTemplate(level0TempAlg, level0CommInfo));
134 0 : HCCL_INFO("AllGather mesh HD level0 run success");
135 :
136 : // 第三步， AI server 间 recursive halving doubling AllGather
137 0 : u64 hdSize = inputMemSize * level0RankSize;
138 0 : u64 hdCount = hdSize / perDataSize;
139 :
140 0 : std::unique_ptr<AlgTemplateBase> level1TempAlg;
141 0 : if (algType_.algoLevel1 == AlgTypeLevel1::ALG_LEVEL1_RING
142 0 : || (topoAttr_.isDiffDeviceModule && topoAttr_.serverNum == 1)) {
143 : // 1-单server-SDMA
144 : level1TempAlg
145 0 : = AlgTemplateRegistry::Instance().GetAlgTemplate(TemplateType::TEMPLATE_ALL_GATHER_RING, dispatcher_);
146 0 : HCCL_INFO("AllGather mesh: using ring algo inter-server.");
147 0 : } else if (algType_.algoLevel1 == AlgTypeLevel1::ALG_LEVEL1_NHR) {
148 : level1TempAlg
149 0 : = AlgTemplateRegistry::Instance().GetAlgTemplate(TemplateType::TEMPLATE_ALL_GATHER_NHR, dispatcher_);
150 0 : HCCL_INFO("AllGather mesh: using nhr algo inter-server.");
151 0 : } else if (algType_.algoLevel1 == AlgTypeLevel1::ALG_LEVEL1_NHR_V1) {
152 : level1TempAlg
153 0 : = AlgTemplateRegistry::Instance().GetAlgTemplate(TemplateType::TEMPLATE_ALL_GATHER_NHRV1, dispatcher_);
154 0 : HCCL_INFO("AllGather mesh: using nhr_v1 algo inter-server.");
155 0 : } else if (algType_.algoLevel1 == AlgTypeLevel1::ALG_LEVEL1_NB) {
156 : level1TempAlg
157 0 : = AlgTemplateRegistry::Instance().GetAlgTemplate(TemplateType::TEMPLATE_ALL_GATHER_NB, dispatcher_);
158 0 : HCCL_INFO("AllGather mesh: using nonuniform-bruck algo inter-server.");
159 : } else {
160 0 : level1TempAlg = AlgTemplateRegistry::Instance().GetAlgTemplate(
161 0 : TemplateType::TEMPLATE_ALL_GATHER_RECURSIVE_HALVING_DOUBLING, dispatcher_);
162 0 : HCCL_INFO("AllGather mesh: using halving-doubling algo inter-server.");
163 : }
164 0 : CHK_SMART_PTR_NULL(level1TempAlg);
165 :
166 : // 此处虽然带入inputMem作为scratch mem, 但inputMem 不能被使用
167 0 : CHK_RET(level1TempAlg->Prepare(
168 : execMem.outputMem, execMem.outputMem, execMem.inputMem, hdCount, param.DataDes.dataType, param.stream,
169 : HcclReduceOp::HCCL_REDUCE_RESERVED, INVALID_VALUE_RANKID, std::vector<Slice>(COMM_INDEX_0), 0));
170 :
171 0 : rankSize = level1CommInfo.localRankSize;
172 0 : CHK_RET(level1TempAlg->RegisterProfiler(
173 : (rankSize << PROF_RANKSIZE_OFFSET_OF_PLANEID) + serverIndex, PROF_STAGE_2, HCCL_EXEC_STEP_NOT_SET,
174 : param.stream));
175 :
176 0 : CHK_RET(RunTemplate(level1TempAlg, level1CommInfo));
177 0 : HCCL_INFO("AllGather mesh HD level1 run success");
178 0 : return HCCL_SUCCESS;
179 0 : }
180 0 : HcclResult CollAllGatherMeshExecutor::Getlevel1CommRank(SubCommInfo& level1CommInfo)
181 : {
182 0 : if (CheckCommSize(COMM_LEVEL0, COMM_INDEX_0 + 1) != HCCL_SUCCESS) {
183 0 : return HCCL_E_UNAVAIL;
184 : }
185 0 : SubCommInfo level0CommInfo = GetSubCommInfo(COMM_LEVEL0, COMM_INDEX_0);
186 0 : u32 ringNum
187 0 : = (topoType_ == TopoType::TOPO_TYPE_8P_RING) ? LEVEL0_PLANE_NUM_IN_8PRING : LEVEL0_PLANE_NUM_IN_NPRING_SINGLE;
188 0 : u32 commIndex = (ringNum == LEVEL0_PLANE_NUM_IN_8PRING) ? topoAttr_.devicePhyId : level0CommInfo.localRank;
189 :
190 0 : if (CheckCommSize(COMM_LEVEL1, commIndex + 1) != HCCL_SUCCESS) {
191 0 : return HCCL_E_UNAVAIL;
192 : }
193 0 : level1CommInfo = GetSubCommInfo(COMM_LEVEL1, commIndex);
194 :
195 0 : return HCCL_SUCCESS;
196 0 : }
197 :
198 0 : HcclResult CollAllGatherMeshExecutor::SelectTempAlg(std::unique_ptr<AlgTemplateBase>& level1TempAlg, u32 level1RankSize)
199 : {
200 0 : if (level1RankSize > 1) {
201 0 : if (algType_.algoLevel1 == AlgTypeLevel1::ALG_LEVEL1_RING
202 0 : || (topoAttr_.isDiffDeviceModule && topoAttr_.serverNum == 1)) {
203 : // 1-单server-SDMA
204 : level1TempAlg
205 0 : = AlgTemplateRegistry::Instance().GetAlgTemplate(TemplateType::TEMPLATE_ALL_GATHER_RING, dispatcher_);
206 0 : HCCL_INFO("AllGather mesh: using ring algo inter-server.");
207 0 : } else if (algType_.algoLevel1 == AlgTypeLevel1::ALG_LEVEL1_NHR) {
208 : level1TempAlg
209 0 : = AlgTemplateRegistry::Instance().GetAlgTemplate(TemplateType::TEMPLATE_ALL_GATHER_NHR, dispatcher_);
210 0 : HCCL_INFO("AllGather mesh: using nhr algo inter-server.");
211 0 : } else if (algType_.algoLevel1 == AlgTypeLevel1::ALG_LEVEL1_NHR_V1) {
212 : level1TempAlg
213 0 : = AlgTemplateRegistry::Instance().GetAlgTemplate(TemplateType::TEMPLATE_ALL_GATHER_NHRV1, dispatcher_);
214 0 : HCCL_INFO("AllGather mesh: using nhr_v1 algo inter-server.");
215 0 : } else if (algType_.algoLevel1 == AlgTypeLevel1::ALG_LEVEL1_NB) {
216 : level1TempAlg
217 0 : = AlgTemplateRegistry::Instance().GetAlgTemplate(TemplateType::TEMPLATE_ALL_GATHER_NB, dispatcher_);
218 0 : HCCL_INFO("AllGather mesh: using nonuniform-bruck algo inter-server.");
219 : } else {
220 0 : level1TempAlg = AlgTemplateRegistry::Instance().GetAlgTemplate(
221 0 : TemplateType::TEMPLATE_ALL_GATHER_RECURSIVE_HALVING_DOUBLING, dispatcher_);
222 0 : HCCL_INFO("AllGather mesh: using halving-doubling algo inter-server.");
223 : }
224 0 : CHK_SMART_PTR_NULL(level1TempAlg);
225 0 : return HCCL_SUCCESS;
226 : }
227 0 : return HCCL_E_UNAVAIL;
228 : }
229 : REGISTER_EXEC("AllGatherMeshExecutor", AllGatherMesh, CollAllGatherMeshExecutor);
230 : } // namespace hccl
|
