Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "ins_temp_gather_nhr.h"
12 : #include "ins_temp_all_gather_nhr.h"
13 : #include "alg_data_trans_wrapper.h"
14 : #include "dev_mode.h"
15 : #include "log.h"
16 :
17 :
18 : namespace Hccl {
19 0 : InsTempGatherNHR::InsTempGatherNHR(const RankId virtualRank, const u32 tempRankSize,
20 : const std::vector<std::vector<RankId>> &tempVTopo,
21 0 : const std::map<RankId, u32> &tempVirtRankMap)
22 0 : : InsAlgTemplateBase(virtualRank, tempRankSize, tempVTopo, tempVirtRankMap)
23 : {
24 0 : }
25 :
26 0 : InsTempGatherNHR::~InsTempGatherNHR()
27 : {
28 0 : }
29 :
30 : // NHR 算法需要的资源计算
31 0 : HcclResult InsTempGatherNHR::CalcRes(AlgTempResReq &tempResReq)
32 : {
33 : // NHR 需要的 que Num 为 1
34 0 : tempResReq.queNum = 1;
35 0 : tempResReq.streamNum = tempResReq.queNum;
36 0 : tempResReq.queNotifys = CreateMasterSlaveQueNotifiesRequest(tempResReq.queNum);
37 0 : CHK_PRT_RET(CalcResLinksNHR(myRank_, tempRankSize_, tempVTopo_, tempResReq) != HcclResult::HCCL_SUCCESS,
38 : HCCL_ERROR("[CollAlgFactory] [InsTempGatherNHR] Rank [%d], resLinks calculation error!", myRank_),
39 : HcclResult::HCCL_E_INTERNAL);
40 :
41 0 : return HcclResult::HCCL_SUCCESS;
42 : }
43 :
44 0 : uint64_t InsTempGatherNHR::GetExpandedMode() const
45 : {
46 0 : return DeviceMode::AICPU;
47 : }
48 :
49 : /*
50 : 按照mesh的方式计算SliceInfo，例如N张卡，就是N份slice
51 : */
52 0 : HcclResult InsTempGatherNHR::CalcSliceInfo(const AllignInfo &allignInfo, const u64 dataSize, RankSliceInfo &sliceInfoVec)
53 : {
54 : // 一般情况下，nhr的temp是单级的, Gather nhr的dataSize为output大小
55 0 : CHK_PRT_RET(tempVTopo_.size() != 1,
56 : HCCL_ERROR("[CollAlgFactory] [InsTempGatherNHR], tempVtopo size is [%zu] one stage NHR only support one template.", tempVTopo_.size()),
57 : HcclResult::HCCL_E_INTERNAL);
58 :
59 0 : std::vector<SliceInfo> tmp(tempVTopo_.size());
60 0 : sliceInfoVec.resize(tempRankSize_, tmp);
61 :
62 0 : CHK_RET(CalcRsAgSliceInfoNHR(myRank_, tempRankSize_, allignInfo, dataSize, sliceInfoVec));
63 :
64 0 : return HcclResult::HCCL_SUCCESS;
65 0 : }
66 :
67 0 : HcclResult InsTempGatherNHR::PreCopy(const TempFuncs &tempFuncs, const RankSliceInfo &sliceInfoVec,
68 : std::vector<InsQuePtr> &tempInsQues)
69 : {
70 : // 前拷贝数据量
71 0 : u64 preCopyDataSize = sliceInfoVec[tempVirtRankMap_[myRank_]][0].size;
72 : // 前拷贝数据在 main buffer 上的偏移
73 0 : u64 preCopyOffset = sliceInfoVec[tempVirtRankMap_[myRank_]][0].offset + mainBufferBaseOffset_;
74 :
75 0 : HCCL_INFO("[InsTempGatherNHR][PreCopy] mainBufferType[%d], preCopyOffset[%llu], preCopyDataSize[%llu]",
76 : mainBufferType_, preCopyOffset, preCopyDataSize);
77 0 : if (preCopyDataSize == 0) {
78 0 : HCCL_INFO("[InsTempGatherNHR][PreCopy] preCopyDataSize is 0, no need copy");
79 0 : return HcclResult::HCCL_SUCCESS;
80 : }
81 0 : if (tempFuncs.isForepart && opMode_ == OpMode::OPBASE) {
82 : // 单算子模式下，第一个算子，直接使用 user data 拷贝
83 0 : HCCL_INFO("[InsTempGatherNHR][PreCopy] Opbase Forepart, copy base on user data");
84 0 : CHK_RET(LocalCopySlices(tempInsQues[0], tempFuncs.usrData.usrInSlices,
85 : tempFuncs.usrData.scratchInSlices));
86 0 : } else if (tempFuncs.isForepart && opMode_ == OpMode::OFFLOAD) {
87 : // 图模式下，第一个算子，从 inBuff 拷贝到 scratchBuff
88 0 : if (buffInfo_.inBuffType == mainBufferType_ && buffInfo_.inBuffBaseOff == preCopyOffset) {
89 : // 如果 inBuff 就是 mainBuffer，不需要拷贝
90 0 : HCCL_INFO("[InsTempGatherNHR][PreCopy] isForepart inBuff is same as scratchBuff, no need pre copy");
91 : } else {
92 0 : HCCL_INFO("[InsTempGatherNHR][PreCopy] isForepart, copy from inBuff to scratchBuff");
93 0 : DataSlice srcSlice = DataSlice(buffInfo_.inBuffType, buffInfo_.inBuffBaseOff, preCopyDataSize);
94 0 : DataSlice dstSlice = DataSlice(mainBufferType_, preCopyOffset, preCopyDataSize);
95 0 : CHK_RET(LocalCopy(tempInsQues[0], srcSlice, dstSlice));
96 : }
97 0 : } else if (tempFuncs.forAlgSeqComb) {
98 : // 作为融合算子的一部分，需要将 inBuff 拷贝到 scratchbuff
99 0 : if (buffInfo_.inBuffType == mainBufferType_ && buffInfo_.inBuffBaseOff == preCopyOffset) {
100 : // 如果 inBuff 就是 mainBuffer，不需要拷贝
101 0 : HCCL_INFO("[InsTempGatherNHR][PreCopy] forAlgSeqComb inBuff is same as scratchBuff, no need pre copy");
102 : } else {
103 0 : HCCL_INFO("[InsTempGatherNHR][PreCopy] forAlgSeqComb, copy from inBuff to scratchBuff");
104 0 : DataSlice srcSlice = DataSlice(buffInfo_.inBuffType, buffInfo_.inBuffBaseOff, preCopyDataSize);
105 0 : DataSlice dstSlice = DataSlice(mainBufferType_, preCopyOffset, preCopyDataSize);
106 0 : CHK_RET(LocalCopy(tempInsQues[0], srcSlice, dstSlice));
107 : }
108 : }
109 0 : return HcclResult::HCCL_SUCCESS;
110 : }
111 :
112 0 : HcclResult InsTempGatherNHR::PostCopy(const TempFuncs &tempFuncs, const RankSliceInfo &sliceInfoVec,
113 : std::vector<InsQuePtr> &tempInsQues)
114 : {
115 0 : if (u32(myRank_) != root_) { // 非root节点不需要后拷贝
116 0 : return HcclResult::HCCL_SUCCESS;
117 : }
118 :
119 : // 后拷贝数据量
120 0 : u64 postCopyDataSize = sliceInfoVec.back().back().size + sliceInfoVec.back().back().offset;
121 : // 后拷贝数据在 main buffer 上的偏移
122 0 : u64 postCopyOffset = mainBufferBaseOffset_;
123 0 : HCCL_INFO("[InsTempGatherNHR][PostCopy] mainBufferType[%d], postCopyOffset[%llu], postCopyDataSize[%llu]",
124 : mainBufferType_, postCopyOffset, postCopyDataSize);
125 :
126 : // 通信后数据全部在 scratch 上，如果是单算子模式, 并且是最后一步算子，需要将数据从 scratch 拷贝到 userOut
127 0 : if (tempFuncs.isBottom && opMode_ == OpMode::OPBASE) {
128 0 : HCCL_INFO("[InsTempGatherNHR][PostCopy] Opbase && isBottom, copy from outBuff to userOut");
129 0 : CHK_RET(LocalCopySlices(tempInsQues[0], tempFuncs.usrData.scratchOutSlices, tempFuncs.usrData.usrOutSlices));
130 : } else {
131 0 : if (buffInfo_.outBuffType == mainBufferType_ && buffInfo_.outBuffBaseOff == postCopyOffset) {
132 : // 如果 mainBuffer 就是 OutBuffer，不需要拷贝
133 0 : HCCL_INFO("[InsTempGatherNHR][PostCopy] outBuff is same as scratchBuff, no need post copy");
134 : } else {
135 0 : HCCL_INFO("[InsTempGatherNHR][PostCopy] , copy from scratchBuff to outBuff");
136 0 : DataSlice srcSlice = DataSlice(mainBufferType_, postCopyOffset, postCopyDataSize);
137 0 : DataSlice dstSlice = DataSlice(buffInfo_.outBuffType, buffInfo_.outBuffBaseOff, postCopyDataSize);
138 0 : CHK_RET(LocalCopy(tempInsQues[0], srcSlice, dstSlice));
139 : }
140 :
141 : // 图模式或者单算子模式下非第一个算子，需要将数据从 scratch 拷贝到 outBuff
142 0 : HCCL_INFO("[InsTempGatherNHR][PreCopy] not first op, not seq comb, copy from inBuff to outBuff");
143 0 : DataSlice srcSlice = DataSlice(mainBufferType_, postCopyOffset, postCopyDataSize);
144 0 : DataSlice dstSlice = DataSlice(buffInfo_.outBuffType, buffInfo_.outBuffBaseOff, postCopyDataSize);
145 0 : CHK_RET(LocalCopy(tempInsQues[0], srcSlice, dstSlice));
146 : }
147 0 : return HcclResult::HCCL_SUCCESS;
148 : }
149 :
150 0 : HcclResult InsTempGatherNHR::Run(const TempFuncs &tempFuncs, const RankSliceInfo &sliceInfoVec,
151 : const BuffInfo &buffInfo, const ResLinks &tempLinks,
152 : std::vector<InsQuePtr> &tempInsQues)
153 : {
154 0 : HCCL_INFO("[InsTempGatherNHR] Run start");
155 : // 初始化参数
156 0 : opMode_ = tempFuncs.opMode;
157 0 : enableCounterNotify_ = tempFuncs.enableCounterNotify;
158 0 : buffInfo_ = buffInfo;
159 0 : queNum_ = tempVTopo_.size();
160 0 : CHK_PRT_RET(queNum_ != tempInsQues.size(),
161 : HCCL_ERROR("[CollAlgFactory] [InsTempGatherNHR] Rank [%d], requiredQue Error.", myRank_),
162 : HcclResult::HCCL_E_INTERNAL);
163 :
164 : // Gather NHR 算子在通信过程中需要使用 scratch, 主要的通信过程也是在scratch上进行的
165 0 : mainBufferType_ = buffInfo_.scratBuffType;
166 0 : mainBufferBaseOffset_ = buffInfo_.scratchBuffBaseOff;
167 :
168 0 : HCCL_INFO("[InsTempGatherNHR Run]RankID:[%d], root:[%u], isForepart:[%d], isBottom:[%d]", myRank_, root_, tempFuncs.isForepart, tempFuncs.isBottom);
169 :
170 : // 前拷贝
171 0 : PreCopy(tempFuncs, sliceInfoVec, tempInsQues);
172 :
173 0 : std::vector<AicpuNHRStepInfo> nhrSteps;
174 0 : GetGatherStepInfo(nhrSteps);
175 :
176 0 : for (auto &nhrstep : nhrSteps) {
177 0 : CHK_PRT_RET(BatchTxRx(nhrstep, tempLinks, tempInsQues[0], sliceInfoVec),
178 : HCCL_ERROR("[InsTempGatherNHR] BatchTxRx failed"),
179 : HcclResult::HCCL_E_INTERNAL);
180 : }
181 :
182 : // 后拷贝
183 0 : PostCopy(tempFuncs, sliceInfoVec, tempInsQues);
184 0 : return HcclResult::HCCL_SUCCESS;
185 0 : }
186 :
187 : // Send multiple DataSlices
188 0 : HcclResult InsTempGatherNHR::BatchTxRx(AicpuNHRStepInfo &stepInfo, const ResLinks &tempLinks, InsQuePtr &queue,
189 : const RankSliceInfo &sliceInfoVec)
190 : {
191 : // 只有Tx,使用send指令
192 0 : if (stepInfo.txSliceIdxs.size() > 0 && stepInfo.rxSliceIdxs.size() == 0) {
193 0 : BatchSend(stepInfo, tempLinks, queue, sliceInfoVec, mainBufferType_, mainBufferBaseOffset_);
194 : }
195 : // 只有Rx，使用recv指令
196 0 : else if (stepInfo.txSliceIdxs.size() == 0 && stepInfo.rxSliceIdxs.size() > 0) {
197 0 : BatchRecv(stepInfo, tempLinks, queue, sliceInfoVec, mainBufferType_, mainBufferBaseOffset_);
198 : }
199 : // 既有Tx又有Rx，使用SendRecv指令
200 0 : else if (stepInfo.txSliceIdxs.size() > 0 && stepInfo.rxSliceIdxs.size() > 0) {
201 0 : BatchSR(stepInfo, tempLinks, queue, sliceInfoVec, mainBufferType_, mainBufferBaseOffset_);
202 : }
203 0 : return HcclResult::HCCL_SUCCESS;
204 : }
205 :
206 0 : HcclResult InsTempGatherNHR::BatchSend(AicpuNHRStepInfo &stepInfo, const ResLinks &tempLinks, InsQuePtr &queue,
207 : const RankSliceInfo &sliceInfoVec, BufferType memType, u32 memOffset) const
208 : {
209 0 : CHK_PRT_RET(tempLinks.count(stepInfo.toRank) == 0 ,
210 : HCCL_ERROR("[InsTempGatherNHR][BatchSend] rank [%u] not found in links map", stepInfo.toRank),
211 : HcclResult::HCCL_E_INTERNAL);
212 :
213 0 : const LinkData &linkSend = tempLinks.at(stepInfo.toRank)[0];
214 0 : std::vector<DataSlice> txSlices;
215 0 : for (u32 i = 0; i < stepInfo.txSliceIdxs.size(); i++) {
216 0 : u32 txId = stepInfo.txSliceIdxs[i];
217 0 : DataSlice txSrcDstSlice = DataSlice( memType, memOffset + sliceInfoVec[txId][0].offset, sliceInfoVec[txId][0].size );
218 0 : txSlices.push_back(txSrcDstSlice);
219 : }
220 0 : SlicesList txSlicesList(txSlices, txSlices);
221 0 : DataInfo sendData(linkSend, txSlicesList);
222 0 : CHK_PRT_RET(Send(sendData, queue), HCCL_ERROR("[InsTempGatherNHR] BatchSend failed"),
223 : HcclResult::HCCL_E_INTERNAL);
224 0 : return HcclResult::HCCL_SUCCESS;
225 0 : }
226 :
227 0 : HcclResult InsTempGatherNHR::BatchRecv(AicpuNHRStepInfo &stepInfo, const ResLinks &tempLinks, InsQuePtr &queue,
228 : const RankSliceInfo &sliceInfoVec, BufferType memType, u32 memOffset) const
229 : {
230 0 : CHK_PRT_RET(tempLinks.count(stepInfo.fromRank) == 0 ,
231 : HCCL_ERROR("[InsTempGatherNHR][BatchRecv] rank [%u] not found in links map", stepInfo.fromRank),
232 : HcclResult::HCCL_E_INTERNAL);
233 :
234 0 : const LinkData &linkRecv = tempLinks.at(stepInfo.fromRank)[0];
235 0 : std::vector<DataSlice> rxSlices;
236 0 : for (u32 i = 0; i < stepInfo.rxSliceIdxs.size(); i++) {
237 0 : u32 rxId = stepInfo.rxSliceIdxs[i];
238 0 : DataSlice rxSrcDstSlice = DataSlice( memType, memOffset + sliceInfoVec[rxId][0].offset, sliceInfoVec[rxId][0].size );
239 0 : rxSlices.push_back(rxSrcDstSlice);
240 : }
241 0 : SlicesList rxSlicesList(rxSlices, rxSlices);
242 0 : DataInfo recvData(linkRecv, rxSlicesList);
243 0 : CHK_PRT_RET(Recv(recvData, queue), HCCL_ERROR("[InsTempGatherNHR] BatchTxRx Recv failed"),
244 : HcclResult::HCCL_E_INTERNAL);
245 0 : return HcclResult::HCCL_SUCCESS;
246 0 : }
247 :
248 0 : HcclResult InsTempGatherNHR::BatchSR(AicpuNHRStepInfo &stepInfo, const ResLinks &tempLinks, InsQuePtr &queue,
249 : const RankSliceInfo &sliceInfoVec, BufferType memType, u32 memOffset)const
250 : {
251 0 : CHK_PRT_RET(tempLinks.count(stepInfo.toRank) == 0 ,
252 : HCCL_ERROR("[InsTempGatherNHR][BatchSR] rank [%u] not found in links map", stepInfo.toRank),
253 : HcclResult::HCCL_E_INTERNAL);
254 0 : const LinkData &linkSend = tempLinks.at(stepInfo.toRank)[0];
255 0 : CHK_PRT_RET(tempLinks.count(stepInfo.fromRank) == 0 ,
256 : HCCL_ERROR("[InsTempGatherNHR][BatchSR] rank [%u] not found in links map", stepInfo.fromRank),
257 : HcclResult::HCCL_E_INTERNAL);
258 0 : const LinkData &linkRecv = tempLinks.at(stepInfo.fromRank)[0];
259 0 : TxRxLinks linkSendRecv = {linkSend, linkRecv};
260 :
261 0 : std::vector<DataSlice> txSlices;
262 0 : for (u32 i = 0; i < stepInfo.txSliceIdxs.size(); i++) {
263 0 : u32 txId = stepInfo.txSliceIdxs[i];
264 0 : DataSlice txSrcDstSlice = DataSlice( memType, memOffset + sliceInfoVec[txId][0].offset, sliceInfoVec[txId][0].size );
265 0 : txSlices.push_back(txSrcDstSlice);
266 : }
267 0 : SlicesList txSlicesList(txSlices, txSlices);
268 0 : std::vector<DataSlice> rxSlices;
269 0 : for (u32 i = 0; i < stepInfo.rxSliceIdxs.size(); i++) {
270 0 : u32 rxId = stepInfo.rxSliceIdxs[i];
271 0 : DataSlice rxSrcDstSlice = DataSlice( memType, memOffset + sliceInfoVec[rxId][0].offset, sliceInfoVec[rxId][0].size );
272 0 : rxSlices.push_back(rxSrcDstSlice);
273 : }
274 0 : SlicesList rxSlicesList(rxSlices, rxSlices);
275 0 : TxRxSlicesList txRxSlicesList(txSlicesList, rxSlicesList);
276 0 : SendRecvInfo sendRecvInfo(linkSendRecv, txRxSlicesList);
277 0 : CHK_PRT_RET(SendRecv(sendRecvInfo, queue, 0, true, dmaMode_), HCCL_ERROR("[InsTempGatherNHR] BatchTxRx SendRecv failed"),
278 : HcclResult::HCCL_E_INTERNAL);
279 0 : return HcclResult::HCCL_SUCCESS;
280 0 : }
281 :
282 : // NHR每步的算法描述原理函数
283 0 : HcclResult InsTempGatherNHR::GetScatterStepInfo(u32 step, u32 nSteps, AicpuNHRStepInfo &stepInfo) const
284 : {
285 0 : u32 rankSize = tempRankSize_;
286 0 : stepInfo.txSliceIdxs.clear();
287 0 : stepInfo.rxSliceIdxs.clear();
288 0 : stepInfo.nSlices = 0;
289 0 : stepInfo.toRank = rankSize;
290 0 : stepInfo.fromRank = rankSize;
291 0 : stepInfo.step = step;
292 0 : stepInfo.myRank = myRank_;
293 :
294 0 : u32 deltaRoot = (root_ + rankSize - myRank_) % rankSize;
295 0 : u32 deltaRankPair = 1 << step;
296 :
297 : // 数据份数和数据编号增量
298 0 : u32 nSlices = (rankSize - 1 + (1 << step)) / (1 << (step + 1));
299 0 : u32 deltaSliceIndex = 1 << (step + 1);
300 :
301 : // 判断是否是2的幂
302 0 : u32 nRanks = 0; // 本步需要进行收/发的rank数
303 0 : bool isPerfect = (rankSize & (rankSize - 1)) == 0;
304 0 : if (!isPerfect && step == nSteps - 1) {
305 0 : nRanks = rankSize - deltaRankPair;
306 : } else {
307 0 : nRanks = deltaRankPair;
308 : }
309 :
310 0 : if (deltaRoot < nRanks) { // 需要发
311 0 : u32 sendTo = (myRank_ + rankSize - deltaRankPair) % rankSize;
312 0 : u32 txSliceIdx = sendTo;
313 0 : for (u32 i = 0; i < nSlices; i++) {
314 0 : u32 targetTxSliceIdx = txSliceIdx;
315 0 : stepInfo.txSliceIdxs.push_back(targetTxSliceIdx);
316 0 : txSliceIdx = (txSliceIdx + rankSize - deltaSliceIndex) % rankSize;
317 : }
318 :
319 0 : stepInfo.toRank = sendTo;
320 0 : stepInfo.nSlices = nSlices;
321 0 : } else if (deltaRoot >= deltaRankPair && deltaRoot < nRanks + deltaRankPair) { // 需要收
322 0 : u32 recvFrom = (myRank_ + deltaRankPair) % rankSize;
323 0 : u32 rxSliceIdx = myRank_;
324 0 : for (u32 i = 0; i < nSlices; i++) {
325 0 : u32 targetRxSliceIdx = rxSliceIdx;
326 0 : stepInfo.rxSliceIdxs.push_back(targetRxSliceIdx);
327 0 : rxSliceIdx = (rxSliceIdx + rankSize - deltaSliceIndex) % rankSize;
328 : }
329 :
330 0 : stepInfo.fromRank = recvFrom;
331 0 : stepInfo.nSlices = nSlices;
332 : }
333 0 : return HcclResult::HCCL_SUCCESS;
334 : }
335 :
336 0 : HcclResult InsTempGatherNHR::GetGatherStepInfo(std::vector<AicpuNHRStepInfo> &nhrSteps) const
337 : {
338 0 : nhrSteps.clear();
339 0 : u32 nSteps = GetNHRStepNum(tempRankSize_);
340 0 : nhrSteps.resize(nSteps);
341 0 : for (u32 step = 0; step < nSteps; step++) {
342 : // 复用 Scatter NHR 的计算方法,只不过要把步骤顺序和收发端反过来
343 0 : u32 stepIdx = nSteps - step - 1;
344 0 : GetScatterStepInfo(step, nSteps, nhrSteps[stepIdx]);
345 0 : nhrSteps[stepIdx].step = step;
346 0 : u32 tmp = nhrSteps[stepIdx].toRank;
347 0 : nhrSteps[stepIdx].toRank = nhrSteps[stepIdx].fromRank;
348 0 : nhrSteps[stepIdx].fromRank = tmp;
349 0 : nhrSteps[stepIdx].txSliceIdxs.swap(nhrSteps[stepIdx].rxSliceIdxs);
350 : }
351 0 : return HcclResult::HCCL_SUCCESS;
352 : }
353 :
354 :
355 : } // namespace Hccl
|
