Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "ins_temp_gather_nhr.h"
12 : #include "ins_temp_all_gather_nhr.h"
13 : #include "alg_data_trans_wrapper.h"
14 : #include "dev_mode.h"
15 : #include "log.h"
16 :
17 : namespace Hccl {
18 0 : InsTempGatherNHR::InsTempGatherNHR(
19 : const RankId virtualRank, const u32 tempRankSize, const std::vector<std::vector<RankId>>& tempVTopo,
20 0 : const std::map<RankId, u32>& tempVirtRankMap)
21 0 : : InsAlgTemplateBase(virtualRank, tempRankSize, tempVTopo, tempVirtRankMap)
22 0 : {}
23 :
24 0 : InsTempGatherNHR::~InsTempGatherNHR() {}
25 :
26 : // NHR 算法需要的资源计算
27 0 : HcclResult InsTempGatherNHR::CalcRes(AlgTempResReq& tempResReq)
28 : {
29 : // NHR 需要的 que Num 为 1
30 0 : tempResReq.queNum = 1;
31 0 : tempResReq.streamNum = tempResReq.queNum;
32 0 : tempResReq.queNotifys = CreateMasterSlaveQueNotifiesRequest(tempResReq.queNum);
33 0 : CHK_PRT_RET(
34 : CalcResLinksNHR(myRank_, tempRankSize_, tempVTopo_, tempResReq) != HcclResult::HCCL_SUCCESS,
35 : HCCL_ERROR("[CollAlgFactory] [InsTempGatherNHR] Rank [%d], resLinks calculation error!", myRank_),
36 : HcclResult::HCCL_E_INTERNAL);
37 :
38 0 : return HcclResult::HCCL_SUCCESS;
39 : }
40 :
41 0 : uint64_t InsTempGatherNHR::GetExpandedMode() const { return DeviceMode::AICPU; }
42 :
43 : /*
44 : 按照mesh的方式计算SliceInfo，例如N张卡，就是N份slice
45 : */
46 : HcclResult
47 0 : InsTempGatherNHR::CalcSliceInfo(const AllignInfo& allignInfo, const u64 dataSize, RankSliceInfo& sliceInfoVec)
48 : {
49 : // 一般情况下，nhr的temp是单级的, Gather nhr的dataSize为output大小
50 0 : CHK_PRT_RET(
51 : tempVTopo_.size() != 1,
52 : HCCL_ERROR(
53 : "[CollAlgFactory] [InsTempGatherNHR], tempVtopo size is [%zu] one stage NHR only support one template.",
54 : tempVTopo_.size()),
55 : HcclResult::HCCL_E_INTERNAL);
56 :
57 0 : std::vector<SliceInfo> tmp(tempVTopo_.size());
58 0 : sliceInfoVec.resize(tempRankSize_, tmp);
59 :
60 0 : CHK_RET(CalcRsAgSliceInfoNHR(myRank_, tempRankSize_, allignInfo, dataSize, sliceInfoVec));
61 :
62 0 : return HcclResult::HCCL_SUCCESS;
63 0 : }
64 :
65 0 : HcclResult InsTempGatherNHR::PreCopy(
66 : const TempFuncs& tempFuncs, const RankSliceInfo& sliceInfoVec, std::vector<InsQuePtr>& tempInsQues)
67 : {
68 : // 前拷贝数据量
69 0 : u64 preCopyDataSize = sliceInfoVec[tempVirtRankMap_[myRank_]][0].size;
70 : // 前拷贝数据在 main buffer 上的偏移
71 0 : u64 preCopyOffset = sliceInfoVec[tempVirtRankMap_[myRank_]][0].offset + mainBufferBaseOffset_;
72 :
73 0 : HCCL_INFO(
74 : "[InsTempGatherNHR][PreCopy] mainBufferType[%d], preCopyOffset[%llu], preCopyDataSize[%llu]", mainBufferType_,
75 : preCopyOffset, preCopyDataSize);
76 0 : if (preCopyDataSize == 0) {
77 0 : HCCL_INFO("[InsTempGatherNHR][PreCopy] preCopyDataSize is 0, no need copy");
78 0 : return HcclResult::HCCL_SUCCESS;
79 : }
80 0 : if (tempFuncs.isForepart && opMode_ == OpMode::OPBASE) {
81 : // 单算子模式下，第一个算子，直接使用 user data 拷贝
82 0 : HCCL_INFO("[InsTempGatherNHR][PreCopy] Opbase Forepart, copy base on user data");
83 0 : CHK_RET(LocalCopySlices(tempInsQues[0], tempFuncs.usrData.usrInSlices, tempFuncs.usrData.scratchInSlices));
84 0 : } else if (tempFuncs.isForepart && opMode_ == OpMode::OFFLOAD) {
85 : // 图模式下，第一个算子，从 inBuff 拷贝到 scratchBuff
86 0 : if (buffInfo_.inBuffType == mainBufferType_ && buffInfo_.inBuffBaseOff == preCopyOffset) {
87 : // 如果 inBuff 就是 mainBuffer，不需要拷贝
88 0 : HCCL_INFO("[InsTempGatherNHR][PreCopy] isForepart inBuff is same as scratchBuff, no need pre copy");
89 : } else {
90 0 : HCCL_INFO("[InsTempGatherNHR][PreCopy] isForepart, copy from inBuff to scratchBuff");
91 0 : DataSlice srcSlice = DataSlice(buffInfo_.inBuffType, buffInfo_.inBuffBaseOff, preCopyDataSize);
92 0 : DataSlice dstSlice = DataSlice(mainBufferType_, preCopyOffset, preCopyDataSize);
93 0 : CHK_RET(LocalCopy(tempInsQues[0], srcSlice, dstSlice));
94 : }
95 0 : } else if (tempFuncs.forAlgSeqComb) {
96 : // 作为融合算子的一部分，需要将 inBuff 拷贝到 scratchbuff
97 0 : if (buffInfo_.inBuffType == mainBufferType_ && buffInfo_.inBuffBaseOff == preCopyOffset) {
98 : // 如果 inBuff 就是 mainBuffer，不需要拷贝
99 0 : HCCL_INFO("[InsTempGatherNHR][PreCopy] forAlgSeqComb inBuff is same as scratchBuff, no need pre copy");
100 : } else {
101 0 : HCCL_INFO("[InsTempGatherNHR][PreCopy] forAlgSeqComb, copy from inBuff to scratchBuff");
102 0 : DataSlice srcSlice = DataSlice(buffInfo_.inBuffType, buffInfo_.inBuffBaseOff, preCopyDataSize);
103 0 : DataSlice dstSlice = DataSlice(mainBufferType_, preCopyOffset, preCopyDataSize);
104 0 : CHK_RET(LocalCopy(tempInsQues[0], srcSlice, dstSlice));
105 : }
106 : }
107 0 : return HcclResult::HCCL_SUCCESS;
108 : }
109 :
110 0 : HcclResult InsTempGatherNHR::PostCopy(
111 : const TempFuncs& tempFuncs, const RankSliceInfo& sliceInfoVec, std::vector<InsQuePtr>& tempInsQues)
112 : {
113 0 : if (u32(myRank_) != root_) { // 非root节点不需要后拷贝
114 0 : return HcclResult::HCCL_SUCCESS;
115 : }
116 :
117 : // 后拷贝数据量
118 0 : u64 postCopyDataSize = sliceInfoVec.back().back().size + sliceInfoVec.back().back().offset;
119 : // 后拷贝数据在 main buffer 上的偏移
120 0 : u64 postCopyOffset = mainBufferBaseOffset_;
121 0 : HCCL_INFO(
122 : "[InsTempGatherNHR][PostCopy] mainBufferType[%d], postCopyOffset[%llu], postCopyDataSize[%llu]",
123 : mainBufferType_, postCopyOffset, postCopyDataSize);
124 :
125 : // 通信后数据全部在 scratch 上，如果是单算子模式, 并且是最后一步算子，需要将数据从 scratch 拷贝到 userOut
126 0 : if (tempFuncs.isBottom && opMode_ == OpMode::OPBASE) {
127 0 : HCCL_INFO("[InsTempGatherNHR][PostCopy] Opbase && isBottom, copy from outBuff to userOut");
128 0 : CHK_RET(LocalCopySlices(tempInsQues[0], tempFuncs.usrData.scratchOutSlices, tempFuncs.usrData.usrOutSlices));
129 : } else {
130 0 : if (buffInfo_.outBuffType == mainBufferType_ && buffInfo_.outBuffBaseOff == postCopyOffset) {
131 : // 如果 mainBuffer 就是 OutBuffer，不需要拷贝
132 0 : HCCL_INFO("[InsTempGatherNHR][PostCopy] outBuff is same as scratchBuff, no need post copy");
133 : } else {
134 0 : HCCL_INFO("[InsTempGatherNHR][PostCopy] , copy from scratchBuff to outBuff");
135 0 : DataSlice srcSlice = DataSlice(mainBufferType_, postCopyOffset, postCopyDataSize);
136 0 : DataSlice dstSlice = DataSlice(buffInfo_.outBuffType, buffInfo_.outBuffBaseOff, postCopyDataSize);
137 0 : CHK_RET(LocalCopy(tempInsQues[0], srcSlice, dstSlice));
138 : }
139 :
140 : // 图模式或者单算子模式下非第一个算子，需要将数据从 scratch 拷贝到 outBuff
141 0 : HCCL_INFO("[InsTempGatherNHR][PreCopy] not first op, not seq comb, copy from inBuff to outBuff");
142 0 : DataSlice srcSlice = DataSlice(mainBufferType_, postCopyOffset, postCopyDataSize);
143 0 : DataSlice dstSlice = DataSlice(buffInfo_.outBuffType, buffInfo_.outBuffBaseOff, postCopyDataSize);
144 0 : CHK_RET(LocalCopy(tempInsQues[0], srcSlice, dstSlice));
145 : }
146 0 : return HcclResult::HCCL_SUCCESS;
147 : }
148 :
149 0 : HcclResult InsTempGatherNHR::Run(
150 : const TempFuncs& tempFuncs, const RankSliceInfo& sliceInfoVec, const BuffInfo& buffInfo, const ResLinks& tempLinks,
151 : std::vector<InsQuePtr>& tempInsQues)
152 : {
153 0 : HCCL_INFO("[InsTempGatherNHR] Run start");
154 : // 初始化参数
155 0 : opMode_ = tempFuncs.opMode;
156 0 : enableCounterNotify_ = tempFuncs.enableCounterNotify;
157 0 : buffInfo_ = buffInfo;
158 0 : queNum_ = tempVTopo_.size();
159 0 : CHK_PRT_RET(
160 : queNum_ != tempInsQues.size(),
161 : HCCL_ERROR("[CollAlgFactory] [InsTempGatherNHR] Rank [%d], requiredQue Error.", myRank_),
162 : HcclResult::HCCL_E_INTERNAL);
163 :
164 : // Gather NHR 算子在通信过程中需要使用 scratch, 主要的通信过程也是在scratch上进行的
165 0 : mainBufferType_ = buffInfo_.scratBuffType;
166 0 : mainBufferBaseOffset_ = buffInfo_.scratchBuffBaseOff;
167 :
168 0 : HCCL_INFO(
169 : "[InsTempGatherNHR Run]RankID:[%d], root:[%u], isForepart:[%d], isBottom:[%d]", myRank_, root_,
170 : tempFuncs.isForepart, tempFuncs.isBottom);
171 :
172 : // 前拷贝
173 0 : PreCopy(tempFuncs, sliceInfoVec, tempInsQues);
174 :
175 0 : std::vector<AicpuNHRStepInfo> nhrSteps;
176 0 : GetGatherStepInfo(nhrSteps);
177 :
178 0 : for (auto& nhrstep : nhrSteps) {
179 0 : CHK_PRT_RET(
180 : BatchTxRx(nhrstep, tempLinks, tempInsQues[0], sliceInfoVec),
181 : HCCL_ERROR("[InsTempGatherNHR] BatchTxRx failed"), HcclResult::HCCL_E_INTERNAL);
182 : }
183 :
184 : // 后拷贝
185 0 : PostCopy(tempFuncs, sliceInfoVec, tempInsQues);
186 0 : return HcclResult::HCCL_SUCCESS;
187 0 : }
188 :
189 : // Send multiple DataSlices
190 0 : HcclResult InsTempGatherNHR::BatchTxRx(
191 : AicpuNHRStepInfo& stepInfo, const ResLinks& tempLinks, InsQuePtr& queue, const RankSliceInfo& sliceInfoVec)
192 : {
193 : // 只有Tx,使用send指令
194 0 : if (stepInfo.txSliceIdxs.size() > 0 && stepInfo.rxSliceIdxs.size() == 0) {
195 0 : BatchSend(stepInfo, tempLinks, queue, sliceInfoVec, mainBufferType_, mainBufferBaseOffset_);
196 : }
197 : // 只有Rx，使用recv指令
198 0 : else if (stepInfo.txSliceIdxs.size() == 0 && stepInfo.rxSliceIdxs.size() > 0) {
199 0 : BatchRecv(stepInfo, tempLinks, queue, sliceInfoVec, mainBufferType_, mainBufferBaseOffset_);
200 : }
201 : // 既有Tx又有Rx，使用SendRecv指令
202 0 : else if (stepInfo.txSliceIdxs.size() > 0 && stepInfo.rxSliceIdxs.size() > 0) {
203 0 : BatchSR(stepInfo, tempLinks, queue, sliceInfoVec, mainBufferType_, mainBufferBaseOffset_);
204 : }
205 0 : return HcclResult::HCCL_SUCCESS;
206 : }
207 :
208 0 : HcclResult InsTempGatherNHR::BatchSend(
209 : AicpuNHRStepInfo& stepInfo, const ResLinks& tempLinks, InsQuePtr& queue, const RankSliceInfo& sliceInfoVec,
210 : BufferType memType, u32 memOffset) const
211 : {
212 0 : CHK_PRT_RET(
213 : tempLinks.count(stepInfo.toRank) == 0,
214 : HCCL_ERROR("[InsTempGatherNHR][BatchSend] rank [%u] not found in links map", stepInfo.toRank),
215 : HcclResult::HCCL_E_INTERNAL);
216 :
217 0 : const LinkData& linkSend = tempLinks.at(stepInfo.toRank)[0];
218 0 : std::vector<DataSlice> txSlices;
219 0 : for (u32 i = 0; i < stepInfo.txSliceIdxs.size(); i++) {
220 0 : u32 txId = stepInfo.txSliceIdxs[i];
221 : DataSlice txSrcDstSlice
222 0 : = DataSlice(memType, memOffset + sliceInfoVec[txId][0].offset, sliceInfoVec[txId][0].size);
223 0 : txSlices.push_back(txSrcDstSlice);
224 : }
225 0 : SlicesList txSlicesList(txSlices, txSlices);
226 0 : DataInfo sendData(linkSend, txSlicesList);
227 0 : CHK_PRT_RET(Send(sendData, queue), HCCL_ERROR("[InsTempGatherNHR] BatchSend failed"), HcclResult::HCCL_E_INTERNAL);
228 0 : return HcclResult::HCCL_SUCCESS;
229 0 : }
230 :
231 0 : HcclResult InsTempGatherNHR::BatchRecv(
232 : AicpuNHRStepInfo& stepInfo, const ResLinks& tempLinks, InsQuePtr& queue, const RankSliceInfo& sliceInfoVec,
233 : BufferType memType, u32 memOffset) const
234 : {
235 0 : CHK_PRT_RET(
236 : tempLinks.count(stepInfo.fromRank) == 0,
237 : HCCL_ERROR("[InsTempGatherNHR][BatchRecv] rank [%u] not found in links map", stepInfo.fromRank),
238 : HcclResult::HCCL_E_INTERNAL);
239 :
240 0 : const LinkData& linkRecv = tempLinks.at(stepInfo.fromRank)[0];
241 0 : std::vector<DataSlice> rxSlices;
242 0 : for (u32 i = 0; i < stepInfo.rxSliceIdxs.size(); i++) {
243 0 : u32 rxId = stepInfo.rxSliceIdxs[i];
244 : DataSlice rxSrcDstSlice
245 0 : = DataSlice(memType, memOffset + sliceInfoVec[rxId][0].offset, sliceInfoVec[rxId][0].size);
246 0 : rxSlices.push_back(rxSrcDstSlice);
247 : }
248 0 : SlicesList rxSlicesList(rxSlices, rxSlices);
249 0 : DataInfo recvData(linkRecv, rxSlicesList);
250 0 : CHK_PRT_RET(
251 : Recv(recvData, queue), HCCL_ERROR("[InsTempGatherNHR] BatchTxRx Recv failed"), HcclResult::HCCL_E_INTERNAL);
252 0 : return HcclResult::HCCL_SUCCESS;
253 0 : }
254 :
255 0 : HcclResult InsTempGatherNHR::BatchSR(
256 : AicpuNHRStepInfo& stepInfo, const ResLinks& tempLinks, InsQuePtr& queue, const RankSliceInfo& sliceInfoVec,
257 : BufferType memType, u32 memOffset) const
258 : {
259 0 : CHK_PRT_RET(
260 : tempLinks.count(stepInfo.toRank) == 0,
261 : HCCL_ERROR("[InsTempGatherNHR][BatchSR] rank [%u] not found in links map", stepInfo.toRank),
262 : HcclResult::HCCL_E_INTERNAL);
263 0 : const LinkData& linkSend = tempLinks.at(stepInfo.toRank)[0];
264 0 : CHK_PRT_RET(
265 : tempLinks.count(stepInfo.fromRank) == 0,
266 : HCCL_ERROR("[InsTempGatherNHR][BatchSR] rank [%u] not found in links map", stepInfo.fromRank),
267 : HcclResult::HCCL_E_INTERNAL);
268 0 : const LinkData& linkRecv = tempLinks.at(stepInfo.fromRank)[0];
269 0 : TxRxLinks linkSendRecv = {linkSend, linkRecv};
270 :
271 0 : std::vector<DataSlice> txSlices;
272 0 : for (u32 i = 0; i < stepInfo.txSliceIdxs.size(); i++) {
273 0 : u32 txId = stepInfo.txSliceIdxs[i];
274 : DataSlice txSrcDstSlice
275 0 : = DataSlice(memType, memOffset + sliceInfoVec[txId][0].offset, sliceInfoVec[txId][0].size);
276 0 : txSlices.push_back(txSrcDstSlice);
277 : }
278 0 : SlicesList txSlicesList(txSlices, txSlices);
279 0 : std::vector<DataSlice> rxSlices;
280 0 : for (u32 i = 0; i < stepInfo.rxSliceIdxs.size(); i++) {
281 0 : u32 rxId = stepInfo.rxSliceIdxs[i];
282 : DataSlice rxSrcDstSlice
283 0 : = DataSlice(memType, memOffset + sliceInfoVec[rxId][0].offset, sliceInfoVec[rxId][0].size);
284 0 : rxSlices.push_back(rxSrcDstSlice);
285 : }
286 0 : SlicesList rxSlicesList(rxSlices, rxSlices);
287 0 : TxRxSlicesList txRxSlicesList(txSlicesList, rxSlicesList);
288 0 : SendRecvInfo sendRecvInfo(linkSendRecv, txRxSlicesList);
289 0 : CHK_PRT_RET(
290 : SendRecv(sendRecvInfo, queue, 0, true, dmaMode_), HCCL_ERROR("[InsTempGatherNHR] BatchTxRx SendRecv failed"),
291 : HcclResult::HCCL_E_INTERNAL);
292 0 : return HcclResult::HCCL_SUCCESS;
293 0 : }
294 :
295 : // NHR每步的算法描述原理函数
296 0 : HcclResult InsTempGatherNHR::GetScatterStepInfo(u32 step, u32 nSteps, AicpuNHRStepInfo& stepInfo) const
297 : {
298 0 : u32 rankSize = tempRankSize_;
299 0 : stepInfo.txSliceIdxs.clear();
300 0 : stepInfo.rxSliceIdxs.clear();
301 0 : stepInfo.nSlices = 0;
302 0 : stepInfo.toRank = rankSize;
303 0 : stepInfo.fromRank = rankSize;
304 0 : stepInfo.step = step;
305 0 : stepInfo.myRank = myRank_;
306 :
307 0 : u32 deltaRoot = (root_ + rankSize - myRank_) % rankSize;
308 0 : u32 deltaRankPair = 1 << step;
309 :
310 : // 数据份数和数据编号增量
311 0 : u32 nSlices = (rankSize - 1 + (1 << step)) / (1 << (step + 1));
312 0 : u32 deltaSliceIndex = 1 << (step + 1);
313 :
314 : // 判断是否是2的幂
315 0 : u32 nRanks = 0; // 本步需要进行收/发的rank数
316 0 : bool isPerfect = (rankSize & (rankSize - 1)) == 0;
317 0 : if (!isPerfect && step == nSteps - 1) {
318 0 : nRanks = rankSize - deltaRankPair;
319 : } else {
320 0 : nRanks = deltaRankPair;
321 : }
322 :
323 0 : if (deltaRoot < nRanks) { // 需要发
324 0 : u32 sendTo = (myRank_ + rankSize - deltaRankPair) % rankSize;
325 0 : u32 txSliceIdx = sendTo;
326 0 : for (u32 i = 0; i < nSlices; i++) {
327 0 : u32 targetTxSliceIdx = txSliceIdx;
328 0 : stepInfo.txSliceIdxs.push_back(targetTxSliceIdx);
329 0 : txSliceIdx = (txSliceIdx + rankSize - deltaSliceIndex) % rankSize;
330 : }
331 :
332 0 : stepInfo.toRank = sendTo;
333 0 : stepInfo.nSlices = nSlices;
334 0 : } else if (deltaRoot >= deltaRankPair && deltaRoot < nRanks + deltaRankPair) { // 需要收
335 0 : u32 recvFrom = (myRank_ + deltaRankPair) % rankSize;
336 0 : u32 rxSliceIdx = myRank_;
337 0 : for (u32 i = 0; i < nSlices; i++) {
338 0 : u32 targetRxSliceIdx = rxSliceIdx;
339 0 : stepInfo.rxSliceIdxs.push_back(targetRxSliceIdx);
340 0 : rxSliceIdx = (rxSliceIdx + rankSize - deltaSliceIndex) % rankSize;
341 : }
342 :
343 0 : stepInfo.fromRank = recvFrom;
344 0 : stepInfo.nSlices = nSlices;
345 : }
346 0 : return HcclResult::HCCL_SUCCESS;
347 : }
348 :
349 0 : HcclResult InsTempGatherNHR::GetGatherStepInfo(std::vector<AicpuNHRStepInfo>& nhrSteps) const
350 : {
351 0 : nhrSteps.clear();
352 0 : u32 nSteps = GetNHRStepNum(tempRankSize_);
353 0 : nhrSteps.resize(nSteps);
354 0 : for (u32 step = 0; step < nSteps; step++) {
355 : // 复用 Scatter NHR 的计算方法,只不过要把步骤顺序和收发端反过来
356 0 : u32 stepIdx = nSteps - step - 1;
357 0 : GetScatterStepInfo(step, nSteps, nhrSteps[stepIdx]);
358 0 : nhrSteps[stepIdx].step = step;
359 0 : u32 tmp = nhrSteps[stepIdx].toRank;
360 0 : nhrSteps[stepIdx].toRank = nhrSteps[stepIdx].fromRank;
361 0 : nhrSteps[stepIdx].fromRank = tmp;
362 0 : nhrSteps[stepIdx].txSliceIdxs.swap(nhrSteps[stepIdx].rxSliceIdxs);
363 : }
364 0 : return HcclResult::HCCL_SUCCESS;
365 : }
366 :
367 : } // namespace Hccl
|
