Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "coll_all_to_all_staged_aiv_rdma_executor.h"
12 :
13 : namespace hccl {
14 : constexpr u32 A_X_SIZE = 16;
15 :
16 0 : CollRunAlltoAllStagedAivRdmaExecutor::CollRunAlltoAllStagedAivRdmaExecutor(const HcclDispatcher dispatcher,
17 0 : std::unique_ptr<TopoMatcher> &topoMatcher)
18 0 : : CollAlltoAllExecutor(dispatcher, topoMatcher)
19 : {
20 0 : desc_.isAivMode = true;
21 0 : }
22 :
23 0 : HcclResult CollRunAlltoAllStagedAivRdmaExecutor::Orchestrate(OpParam& param, AlgResourceResponse& algRes)
24 : {
25 0 : HcclUs startut = TIME_NOW();
26 0 : tag_ = param.tag;
27 0 : algResResp_ = &algRes;
28 :
29 0 : ExecMem execMem;
30 0 : execMem.count = 0;
31 0 : execMem.inputPtr = param.inputPtr;
32 0 : execMem.outputPtr = param.outputPtr;
33 :
34 : // alltoall aiv暂不支持图模式
35 0 : execMem.inputMem = algRes.cclInputMem;
36 0 : execMem.outputMem = algRes.cclOutputMem;
37 0 : execMem.scratchMem = algRes.aivInputMem;
38 0 : HcclResult ret = KernelRun(param, execMem);
39 0 : CHK_PRT_RET(ret != HCCL_SUCCESS,
40 : HCCL_ERROR("[CollRunAlltoAllStagedAivRdmaExecutor][Orchestrate]errNo[0x%016llx]executor run failed",
41 : HCCL_ERROR_CODE(ret)), ret);
42 :
43 0 : HCCL_INFO("[CollRunAlltoAllStagedAivRdmaExecutor]tag[%s], orchestrate success, take time [%lld]us.",
44 : param.tag.c_str(), DURATION_US(TIME_NOW() - startut));
45 :
46 0 : return HCCL_SUCCESS;
47 0 : }
48 :
49 0 : HcclResult CollRunAlltoAllStagedAivRdmaExecutor::CalcStreamNum(u32& streamNum)
50 : {
51 0 : streamNum = 0; // AIV通信不需要申请从流
52 :
53 0 : HCCL_INFO("[CollRunAlltoAllStagedAivRdmaExecutor][CalcStreamNum] tag[%s] streamNum[%u]", tag_.c_str(), streamNum);
54 0 : return HCCL_SUCCESS;
55 : }
56 :
57 0 : HcclResult CollRunAlltoAllStagedAivRdmaExecutor::CalcScratchMemSize(u64& scratchMemSize)
58 : {
59 0 : scratchMemSize = 0U; // AIV模式不需要scratch内存，直接在cclbuffer上进行内存重排
60 0 : HCCL_INFO("[CollRunAlltoAllStagedAivRdmaExecutor][CalcScratchMemSize]tag[%s] scratchMemSize_ is [%llu]",
61 : tag_.c_str(), scratchMemSize);
62 0 : return HCCL_SUCCESS;
63 : }
64 :
65 0 : HcclResult CollRunAlltoAllStagedAivRdmaExecutor::CalcLevel0CommInfo(TransportMemType inputType,
66 : TransportMemType outputType, std::vector<LevelNSubCommTransport>& opTransport)
67 : {
68 0 : CommParaInfo commParaLevel0(COMM_MESH_L0, CommType::COMM_TAG_MESH);
69 0 : commParaLevel0.meshSinglePlane = true;
70 0 : CHK_RET(CalcCommPlaneInfo(tag_, commParaLevel0, opTransport[COMM_MESH_L0], inputType, outputType));
71 0 : return HCCL_SUCCESS;
72 0 : }
73 :
74 0 : HcclResult CollRunAlltoAllStagedAivRdmaExecutor::CalcLevel1CommInfo(TransportMemType inputType,
75 : TransportMemType outputType, std::vector<LevelNSubCommTransport>& opTransport)
76 : {
77 0 : CommParaInfo commParaInfo(COMM_MESH_L1, CommType::COMM_TAG_MESH);
78 0 : CHK_RET(CalcCommPlaneInfo(tag_, commParaInfo, opTransport[COMM_MESH_L1], inputType, outputType));
79 0 : return HCCL_SUCCESS;
80 0 : }
81 :
82 0 : HcclResult CollRunAlltoAllStagedAivRdmaExecutor::CalcCommInfo(std::vector<LevelNSubCommTransport>& opTransport)
83 : {
84 : // aiv阶段使用cclin做数据搬移，使用aivout做标记
85 0 : CHK_RET(CalcLevel0CommInfo(TransportMemType::CCL_INPUT, TransportMemType::AIV_INPUT, opTransport));
86 0 : CHK_RET(CalcLevel1CommInfo(TransportMemType::CCL_INPUT, TransportMemType::CCL_OUTPUT, opTransport));
87 :
88 0 : HCCL_DEBUG("[CollRunAlltoAllStagedAivRdmaExecutor][CalcCommInfo] ends");
89 0 : return HCCL_SUCCESS;
90 : }
91 :
92 0 : HcclResult CollRunAlltoAllStagedAivRdmaExecutor::CalNumBlocks(u32& numBlocks, u32 rankSize, u64 dataSize, HcclCMDType cmdType)
93 : {
94 0 : numBlocks = rankSize; // 默认情况使用rankSize个AIV
95 0 : u32 bestNumBlocks = numBlocks;
96 :
97 0 : CHK_PRT_RET(numBlocks_ < numBlocks,
98 : HCCL_WARNING("[CollRunAlltoAllStagedAivRdmaExecutor][CalNumBlocks]aivCore[%u] is invalid, at least need [%u].",
99 : numBlocks_, numBlocks), HCCL_E_PARA);
100 :
101 0 : HCCL_INFO("[CollRunAlltoAllStagedAivRdmaExecutor][CalNumBlocks] numBlocks is set to [%u], limit[%u], recommanded[%u]",
102 : numBlocks, numBlocks_, bestNumBlocks);
103 0 : return HCCL_SUCCESS;
104 : }
105 :
106 : // run aiv kernel
107 0 : HcclResult CollRunAlltoAllStagedAivRdmaExecutor::RunAlltoAllStaged1InAIV(const OpParam ¶m, ExecMem &execMem) {
108 : void* dataBuffers[MAX_RANK_SIZE];
109 : void* flagBuffers[MAX_RANK_SIZE];
110 :
111 0 : u32 serverNum = innerCommInfo_.localRankSize;
112 0 : u64 sendCount = *(static_cast<const u64 *>(param.All2AllDataDes.sendCountMatrix));
113 :
114 0 : CHK_RET(PrepareAivBuffers(execMem.inputMem, execMem.scratchMem, dataBuffers, flagBuffers));
115 :
116 0 : AivOpArgs opArgs {
117 0 : HcclCMDType::HCCL_CMD_ALLTOALL, execMem.inputPtr, execMem.outputPtr, sendCount,
118 0 : param.All2AllDataDes.sendType, HCCL_REDUCE_RESERVED, 0, true
119 0 : };
120 : AivTopoArgs topoArgs {
121 : outerCommInfo_.localRank, outerCommInfo_.localRankSize,
122 0 : topoAttr_.isDiffDeviceModule ? topoAttr_.devicePhyId : A_X_SIZE, 0, serverNum
123 0 : };
124 0 : topoArgs.identify = algoAttr_.identifier;
125 : u32 numBlocks;
126 0 : CHK_PRT_RET(CalNumBlocks(numBlocks, outerCommInfo_.localRankSize) != HCCL_SUCCESS,
127 : HCCL_ERROR("[%s] CalNumBlocks failed", __func__),
128 : HCCL_E_PARA);
129 0 : numBlocks_ = numBlocks;
130 : AivResourceArgs resourceArgs {
131 0 : param.tag, param.stream.ptr(), dataBuffers, flagBuffers, execMem.inputMem.size(), numBlocks_, param.aivTag
132 0 : };
133 0 : AivAlgArgs algArgs {0};
134 0 : algArgs.execTimeOut = topoMatcher_->GetExecTimeOutConfig();
135 0 : algArgs.execTimeOutSet = true;
136 0 : struct AivProfilingInfo aivProfilingInfo;
137 0 : aivProfilingInfo.counter = opCounter_;
138 0 : HCCL_DEBUG("[CollRunAlltoAllStagedAivRdmaExecutor]RunAlltoAllStaged1InAIV for numBlocks is %u", numBlocks_);
139 0 : CHK_RET(ExecuteKernelLaunch(opArgs, topoArgs, resourceArgs, algArgs, aivProfilingInfo));
140 0 : return HCCL_SUCCESS;
141 0 : }
142 :
143 0 : HcclResult CollRunAlltoAllStagedAivRdmaExecutor::RunAlltoAllStaged2(const OpParam ¶m, ExecMem &execMem)
144 : {
145 0 : std::map<u32, std::list<OneSendRecvAddrInfo>> sendAddrInfosInter;
146 0 : std::map<u32, std::list<OneSendRecvAddrInfo>> recvAddrInfosInter;
147 :
148 0 : CalcInterMeshAggregationAlltoAllMemInfo(param, sendAddrInfosInter, recvAddrInfosInter);
149 :
150 0 : HcclOpMetaInfoDef opMeta = HcclOpMetaInfo::GetOneForAllToAll(CopyPattern::ZCOPY, algResResp_->paramInputMem.size(),
151 : false, true);
152 0 : CHK_RET(InitTask(dispatcher_, const_cast<Stream&>(param.stream), opMeta.isEnableCache, opMeta.GetCacheKey()));
153 :
154 0 : std::unique_ptr<AlgTemplateBase> alltoallInner = AlgTemplateRegistry::Instance().GetAlgTemplate(
155 0 : TemplateType::TEMPLATE_ALL_2_ALL_V_STAGED_PAIRWISE, dispatcher_);
156 0 : CHK_SMART_PTR_NULL(alltoallInner);
157 :
158 0 : CHK_RET(alltoallInner->Prepare(execMem.inputMem, execMem.outputMem, sendAddrInfosInter, recvAddrInfosInter,
159 : true, const_cast<Stream&>(param.stream)));
160 :
161 0 : CHK_RET(RunAlltoAllVTemplateStaged(alltoallInner, innerCommInfo_));
162 0 : return HCCL_SUCCESS;
163 0 : }
164 :
165 0 : void CollRunAlltoAllStagedAivRdmaExecutor::CalcInterMeshAggregationAlltoAllMemInfo(const OpParam ¶m,
166 : std::map<u32, std::list<OneSendRecvAddrInfo>> &sendAddrInfosInter,
167 : std::map<u32, std::list<OneSendRecvAddrInfo>> &recvAddrInfosInter)
168 : {
169 0 : u64 sendCount = *(static_cast<const u64 *>(param.All2AllDataDes.sendCountMatrix));
170 : // serverLength表示每个rank给每个server需要发送的数据总量
171 0 : u64 serverSendLength = outerCommInfo_.localRankSize * sendCount * sendDataSize_;
172 0 : u64 serverRecvLength = outerCommInfo_.localRankSize * sendCount * recvDataSize_;
173 : // 数据中转时每个rank按server顺序存储对应的中转数据，使用userRankOffset表示根据当前rank所在server计算对端的偏移
174 0 : for (u32 i = 0; i < innerCommInfo_.localRankSize; i++) {
175 : OneSendRecvAddrInfo sendAddrInfo;
176 0 : sendAddrInfo.localOffset = i * serverSendLength;
177 0 : sendAddrInfo.remoteOffset = innerCommInfo_.localRank * serverSendLength;
178 0 : sendAddrInfo.localLength = serverSendLength;
179 0 : sendAddrInfo.remoteLength = serverSendLength;
180 0 : sendAddrInfosInter[i].push_back(sendAddrInfo);
181 :
182 : OneSendRecvAddrInfo recvAddrInfo;
183 0 : recvAddrInfo.localOffset = i * serverRecvLength;
184 0 : recvAddrInfo.remoteOffset = innerCommInfo_.localRank * serverRecvLength;
185 0 : recvAddrInfo.localLength = serverRecvLength;
186 0 : recvAddrInfo.remoteLength = serverRecvLength;
187 0 : recvAddrInfosInter[i].push_back(recvAddrInfo);
188 : }
189 0 : return ;
190 : }
191 :
192 0 : HcclResult CollRunAlltoAllStagedAivRdmaExecutor::KernelRun(const OpParam ¶m, ExecMem &execMem)
193 : {
194 0 : HCCL_CONFIG_INFO(HCCL_ALG, "[CollRunAlltoAllStagedAivRdmaExecutor][KernelRun] AllToAll staged starts");
195 0 : CHK_PRT_RET(topoAttr_.userRankSize % topoAttr_.meshAggregationRankSize != 0,
196 : HCCL_ERROR("userRankSize[%u] is not an Integer multiple of MeshAggregation Dev Num[%u]",
197 : topoAttr_.userRankSize, topoAttr_.meshAggregationRankSize), HCCL_E_PARA);
198 :
199 0 : CHK_RET(SalGetDataTypeSize(param.All2AllDataDes.sendType, sendDataSize_));
200 0 : CHK_RET(SalGetDataTypeSize(param.All2AllDataDes.recvType, recvDataSize_));
201 :
202 0 : CHK_RET(CheckCommSize(COMM_MESH_L0, COMM_INDEX_0 + 1));
203 0 : outerCommInfo_ = GetSubCommInfo(COMM_MESH_L0, COMM_INDEX_0);
204 :
205 0 : CHK_RET(CheckCommSize(COMM_MESH_L1, COMM_INDEX_0 + 1));
206 0 : innerCommInfo_ = GetSubCommInfo(COMM_MESH_L1, COMM_INDEX_0);
207 :
208 : // step1：每个server内通过aiv进行数据交换，将中转数据存储在ipc_buffer中
209 0 : CHK_RET(RunAlltoAllStaged1InAIV(param, execMem));
210 0 : HCCL_INFO("[CollRunAlltoAllStagedAivRdmaExecutor][kernelRun] stage0 run aiv in level0 success!");
211 :
212 : // step2：server间通过rdma进行数据交换，将中转数据分发到各个rank的ccl_out中
213 0 : CHK_RET(RunAlltoAllStaged2(param, execMem));
214 0 : HCCL_INFO("[CollRunAlltoAllStagedAivRdmaExecutor][kernelRun] stage1 run rdma in level1 success!");
215 :
216 : // 每个rank上将数据从ccl_out中搬运到用户输出buffer中
217 0 : DeviceMem srcMem = (execMem.outputMem).range(0, algResResp_->paramOutputMem.size());
218 0 : CHK_RET(HcclD2DMemcpyAsync(dispatcher_, algResResp_->paramOutputMem, srcMem, const_cast<Stream&>(param.stream)));
219 :
220 0 : CHK_RET(LaunchTask(dispatcher_, const_cast<Stream&>(param.stream)));
221 0 : HCCL_INFO("[CollRunAlltoAllStagedAivRdmaExecutor][kernelRun] AllToAll staged ends");
222 0 : return HCCL_SUCCESS;
223 0 : }
224 :
225 0 : HcclResult CollRunAlltoAllStagedAivRdmaExecutor::PrepareAivBuffers(DeviceMem &inputMem, DeviceMem &outputMem, void **dataBuffers,
226 : void **flagBuffers)
227 : {
228 0 : void *tmpCCLBufferData = nullptr;
229 0 : void *tmpCCLBufferFlag = nullptr;
230 0 : for (u32 i = 0; i < outerCommInfo_.localRankSize; i++) {
231 0 : if (i != outerCommInfo_.localRank) {
232 0 : if (outerCommInfo_.links[i] != nullptr) {
233 0 : CHK_RET(outerCommInfo_.links[i]->GetRemoteMem(UserMemType::INPUT_MEM, &(tmpCCLBufferData)));
234 0 : CHK_RET(outerCommInfo_.links[i]->GetRemoteMem(UserMemType::OUTPUT_MEM, &(tmpCCLBufferFlag)));
235 : // cclbuffer后32K数据作为数据标志位
236 0 : dataBuffers[i] = static_cast<u8 *>(tmpCCLBufferData);
237 0 : flagBuffers[i] = static_cast<u8 *>(tmpCCLBufferFlag) + HCCL_MID_COUNT_32_MB;
238 : }
239 : } else {
240 0 : dataBuffers[i] = static_cast<u8 *>(inputMem.ptr());
241 0 : flagBuffers[i] = static_cast<u8 *>(outputMem.ptr()) + HCCL_MID_COUNT_32_MB;
242 : }
243 : }
244 0 : return HCCL_SUCCESS;
245 : }
246 :
247 : REGISTER_EXEC("AlltoAllStagedAIVRdmaExecutor", AlltoAllStagedAIVRdma, CollRunAlltoAllStagedAivRdmaExecutor);
248 : } // namespace hccl
|
