Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include <numeric>
12 :
13 : #include "log.h"
14 :
15 : #include "temp_reduce_scatter_mesh.h"
16 :
17 : namespace Hccl {
18 0 : TempReduceScatterMesh::TempReduceScatterMesh(const RankId virtualRank, const u32 tempRankSize,
19 : const std::vector<std::vector<RankId>> &tempVTopo,
20 0 : const std::map<RankId, u32> &tempVirtRankMap)
21 0 : : AlgTemplateBase(virtualRank, tempRankSize, tempVTopo, tempVirtRankMap)
22 : {
23 0 : }
24 :
25 0 : TempReduceScatterMesh::~TempReduceScatterMesh()
26 : {
27 0 : }
28 :
29 0 : HcclResult TempReduceScatterMesh::CalcRes(const bool forAllReduce, AlgTempResReq &tempResReq,
30 : u32 &requiredScratchMultiplier)
31 : {
32 : (void)forAllReduce;
33 0 : tempResReq.queNum = tempVTopo_[0].size() - 1;
34 0 : requiredScratchMultiplier = tempRankSize_;
35 :
36 0 : CHK_RET(CalcResLinks(tempResReq));
37 0 : return HcclResult::HCCL_SUCCESS;
38 : }
39 :
40 0 : HcclResult TempReduceScatterMesh::CalcResDetour(const bool forAllReduce, const RankGraph *rankGraph,
41 : AlgTempResReq &tempResReq, u32 &requiredScratchMultiplier)
42 : {
43 : (void)forAllReduce;
44 :
45 : u32 myAlgRank;
46 0 : CHK_RET(GetAlgRank(myRank_, tempVTopo_[0], myAlgRank));
47 :
48 0 : linkNumBtwPeers_ = GetLinkNum(rankGraph, myRank_, tempVTopo_[0][(myAlgRank + 1) % tempRankSize_]);
49 0 : if (linkNumBtwPeers_ == 1) {
50 0 : HCCL_INFO(
51 : "[CollAlgFactory] [TempReduceScatterMesh] [WARNING] Rank [%d], linkNum between rank [%d] and rank [%d] "
52 : "equals 1, not able to detour",
53 : myRank_, myRank_, tempVTopo_[0][(myAlgRank + 1) % tempRankSize_]);
54 0 : enableDetour_ = false;
55 : } else {
56 0 : enableDetour_ = true;
57 : }
58 :
59 0 : queNumPerNeighbor_ = (linkNumBtwPeers_ + 1) >> 1;
60 0 : tempResReq.queNum = (tempVTopo_[0].size() - 1) * queNumPerNeighbor_;
61 :
62 0 : requiredScratchMultiplier = tempRankSize_;
63 :
64 0 : CHK_RET(CalcResLinks(tempResReq));
65 0 : return HcclResult::HCCL_SUCCESS;
66 : }
67 :
68 0 : HcclResult TempReduceScatterMesh::CalcResDetour(const bool forAllReduce, ConnectedLinkMgr *linkMgr,
69 : AlgTempResReq &tempResReq, u32 &requiredScratchMultiplier)
70 : {
71 : (void)forAllReduce;
72 :
73 : u32 myAlgRank;
74 0 : CHK_RET(GetAlgRank(myRank_, tempVTopo_[0], myAlgRank));
75 :
76 0 : linkNumBtwPeers_ = (linkMgr->GetLinks(tempVTopo_[0][(myAlgRank + 1) % tempRankSize_])).size();
77 :
78 0 : enableDetour_ = (linkNumBtwPeers_ == 1) ? false : true;
79 :
80 0 : queNumPerNeighbor_ = (linkNumBtwPeers_ + 1) >> 1;
81 0 : tempResReq.queNum = (tempVTopo_[0].size() - 1) * queNumPerNeighbor_;
82 :
83 0 : requiredScratchMultiplier = tempRankSize_;
84 :
85 0 : CHK_RET(CalcResLinks(tempResReq));
86 0 : return HcclResult::HCCL_SUCCESS;
87 : }
88 :
89 0 : HcclResult TempReduceScatterMesh::CalcResLinks(AlgTempResReq &tempResReq)
90 : {
91 : u32 myAlgRank;
92 0 : CHK_RET(GetAlgRank(myRank_, tempVTopo_[0], myAlgRank));
93 :
94 0 : for (u32 queIdx = 0; queIdx < tempVTopo_[0].size() - 1; queIdx++) {
95 : // find neighbors -> virtualRank
96 0 : RankId neighborRank = tempVTopo_[0][(myAlgRank + 1 + queIdx) % tempRankSize_];
97 :
98 : // LinkNum
99 0 : tempResReq.links[neighborRank] = linkNumBtwPeers_;
100 : }
101 :
102 0 : return HcclResult::HCCL_SUCCESS;
103 : }
104 :
105 : /*
106 : dataSize / (rankSize) --> chunkSize
107 : dataSize / (rankSize * queNum) --> sliceSize
108 :
109 : SliceInfoVecforRing: [1st chunk: [1st Slice, 2nd Slice, ...], 2nd chunk: [1st Slice, 2nd Slice, ...], ...]
110 : */
111 0 : HcclResult TempReduceScatterMesh::CalcSliceInfo(const AllignInfo &allignInfo, const bool forAllReduce,
112 : const u64 dataSize, RankSliceInfo &sliceInfoVec)
113 : {
114 0 : std::vector<SliceInfo> tmp(1);
115 0 : sliceInfoVec.resize(tempRankSize_, tmp);
116 :
117 0 : if (forAllReduce) {
118 : // for allreduce, dataSize = total dataSize
119 0 : CHK_RET(CalcSliceInfoAllReduce(allignInfo, dataSize, sliceInfoVec));
120 : } else {
121 : // for reduce scatter, dataSize = chunkSize
122 0 : CHK_RET(CalcRsAgSliceInfoMesh(myRank_, tempRankSize_, allignInfo, dataSize, sliceInfoVec));
123 : }
124 0 : return HcclResult::HCCL_SUCCESS;
125 0 : }
126 :
127 0 : HcclResult TempReduceScatterMesh::CalcSliceInfoAllReduce(const AllignInfo &allignInfo, const u64 dataSize,
128 : RankSliceInfo &sliceInfoVec) const
129 : {
130 : u64 unitAllignSize;
131 0 : CHK_RET(GetUnitAllignSize(allignInfo, unitAllignSize));
132 :
133 0 : u64 chunkSize = RoundUp(dataSize, (tempRankSize_ * unitAllignSize)) * unitAllignSize;
134 :
135 0 : u64 accumOff = 0;
136 0 : for (u32 rankIdx = 0; rankIdx < tempRankSize_; rankIdx++) {
137 0 : u64 currChunkSize = ((dataSize - accumOff) > chunkSize) ? chunkSize : (dataSize - accumOff);
138 0 : SliceInfo slice = {accumOff, currChunkSize};
139 0 : sliceInfoVec[rankIdx][0] = slice;
140 0 : accumOff += currChunkSize;
141 : }
142 :
143 0 : CHK_PRT_RET((sliceInfoVec[tempRankSize_ - 1][0].offset + sliceInfoVec[tempRankSize_ - 1][0].size != dataSize),
144 : HCCL_ERROR("[CollAlgFactory] [TempReduceScatterMesh] Rank [%d], SliceInfo calculation error!", myRank_),
145 : HcclResult::HCCL_E_INTERNAL);
146 :
147 0 : return HcclResult::HCCL_SUCCESS;
148 : }
149 :
150 0 : HcclResult TempReduceScatterMesh::GenPrimQue(const TempFuncs &tempFuncs, const RankSliceInfo &sliceInfoVec,
151 : const BuffInfo &buffInfo, const ResLinks &tempLinks,
152 : std::vector<PrimQuePtr> &tempPrimQues)
153 : {
154 0 : opMode_ = tempFuncs.opMode;
155 0 : enableCounterNotify_ = tempFuncs.enableCounterNotify;
156 0 : buffInfo_ = buffInfo;
157 :
158 0 : auto linkIter = tempLinks.begin();
159 0 : linkNumBtwPeers_ = linkIter->second.size();
160 0 : HCCL_INFO("[CollAlgFactory] [TempReduceScatterMesh] Rank [%d], linkNumBtwPeers equals to [%u].", myRank_,
161 : linkNumBtwPeers_);
162 0 : queNumPerNeighbor_ = (linkNumBtwPeers_ + 1) >> 1;
163 0 : HCCL_INFO("[CollAlgFactory] [TempReduceScatterMesh] Rank [%d], queNumPerNeighbor equals to [%u].", myRank_,
164 : queNumPerNeighbor_);
165 0 : enableDetour_ = (linkNumBtwPeers_ == 1) ? false : true;
166 :
167 0 : majorQueNum_ = tempVTopo_[0].size() - 1;
168 0 : CHK_PRT_RET(majorQueNum_ * queNumPerNeighbor_ != tempPrimQues.size(),
169 : HCCL_ERROR("[CollAlgFactory] [TempReduceScatterMesh] Rank [%d], requiredQueNum [%u] not equals to "
170 : "templateQueNum [%u].",
171 : myRank_, majorQueNum_ * queNumPerNeighbor_, tempPrimQues.size()),
172 : HcclResult::HCCL_E_INTERNAL);
173 :
174 : // queue arrangement
175 0 : std::vector<PrimQuePtr> mainPrimQues;
176 0 : for (u32 queIdx = 0; queIdx < majorQueNum_; queIdx++) {
177 0 : mainPrimQues.push_back(tempPrimQues[queIdx * queNumPerNeighbor_]);
178 : }
179 :
180 : // LocalCopy: from input to scratch In Buffer for OPBASE
181 0 : if ((opMode_ == OpMode::OPBASE) && tempFuncs.isForepart) {
182 0 : CHK_RET(PreCopyOpbase(tempFuncs.usrData, mainPrimQues));
183 : }
184 :
185 : // semaphore sync
186 0 : if (majorQueNum_ > 1) {
187 0 : CHK_RET(PreSyncInterQueues(mainPrimQues));
188 : }
189 :
190 : // locate myRank in tempVTopo -> algRank
191 : u32 myAlgRank;
192 0 : CHK_RET(GetAlgRank(myRank_, tempVTopo_[0], myAlgRank));
193 :
194 : // run Mesh
195 0 : CHK_PRT_RET(
196 : RunMesh(myAlgRank, tempVTopo_[0], sliceInfoVec, tempLinks, tempPrimQues) != HcclResult::HCCL_SUCCESS,
197 : HCCL_ERROR("[CollAlgFactory] [TempReduceScatterMesh] Rank [%d], unable to run the mesh algorithm.", myRank_),
198 : HcclResult::HCCL_E_INTERNAL);
199 :
200 : // semaphore sync
201 0 : if (majorQueNum_ > 1) {
202 0 : CHK_RET(PostSyncInterQueues(mainPrimQues));
203 : }
204 :
205 : // LocalCopy for standalone reducescatter in Offload Mode
206 0 : if ((opMode_ == OpMode::OFFLOAD) && !tempFuncs.forAllReduce && !tempFuncs.forAlgSeqComb) {
207 0 : CHK_RET(PostCopyOffload(sliceInfoVec, mainPrimQues));
208 : }
209 :
210 : // LocalCopy from scratch to output for Opbase
211 0 : if (tempFuncs.isBottom && !tempFuncs.forAllReduce) {
212 0 : CHK_RET(PostCopyOpbase(tempFuncs.usrData, mainPrimQues));
213 : }
214 :
215 0 : return HcclResult::HCCL_SUCCESS;
216 0 : }
217 :
218 0 : HcclResult TempReduceScatterMesh::RunMesh(const u32 myAlgRank, const std::vector<RankId> &vTopo,
219 : const RankSliceInfo &sliceInfoVec, const ResLinks &tempLinks,
220 : std::vector<PrimQuePtr> &tempPrimQues)
221 : {
222 0 : for (u32 queIdx = 0; queIdx < vTopo.size() - 1; queIdx++) {
223 : // find neighbors -> virtualRank
224 0 : RankId neighborRank = vTopo[(myAlgRank + 1 + queIdx) % tempRankSize_];
225 :
226 0 : u32 recvChunkIdx = tempVirtRankMap_[myRank_];
227 0 : u32 sendChunkIdx = tempVirtRankMap_[neighborRank];
228 :
229 : // queue assignment
230 0 : if (enableDetour_) {
231 0 : std::vector<PrimQuePtr> detourPrimQues;
232 0 : for (u32 detourIdx = 0; detourIdx < queNumPerNeighbor_; detourIdx++) {
233 0 : detourPrimQues.push_back(tempPrimQues[queIdx * queNumPerNeighbor_ + detourIdx]);
234 : }
235 0 : HCCL_INFO("[CollAlgFactory] [TempReduceScatterMesh] Rank [%d], Run Mesh with Detour.", myRank_);
236 0 : CHK_RET(RunIndividualPeerDetour(neighborRank, sliceInfoVec[sendChunkIdx][0], sliceInfoVec[recvChunkIdx][0],
237 : tempLinks, detourPrimQues));
238 0 : } else {
239 0 : PrimQuePtr currQue = tempPrimQues[queIdx];
240 0 : LinkData neighborLinkData = tempLinks.at(neighborRank)[0];
241 0 : HCCL_INFO("[CollAlgFactory] [TempReduceScatterMesh] Rank [%d], Run Mesh without Detour.", myRank_);
242 0 : CHK_RET(RunIndividualPeer(neighborRank, neighborLinkData, sliceInfoVec[sendChunkIdx][0],
243 : sliceInfoVec[recvChunkIdx][0], currQue));
244 0 : }
245 : }
246 :
247 0 : return HcclResult::HCCL_SUCCESS;
248 : }
249 :
250 0 : HcclResult TempReduceScatterMesh::RunIndividualPeerDetour(const RankId neighborRank, const SliceInfo &sendReduceSlice,
251 : const SliceInfo &recvReduceSlice, const ResLinks &tempLinks,
252 : std::vector<PrimQuePtr> &detourPrimQues)
253 : {
254 0 : CHK_RET(PreSyncInterQueues(detourPrimQues));
255 :
256 0 : u32 dataSizePerVolume = DataTypeSizeGet(dataType_);
257 0 : u64 unitRecvReduceSize = RoundUp(recvReduceSlice.size, queNumPerNeighbor_ * dataSizePerVolume) * dataSizePerVolume;
258 0 : u64 resRecvReduceSize = recvReduceSlice.size;
259 0 : u64 currRecvReduceOff = recvReduceSlice.offset;
260 :
261 0 : u64 unitSendReduceSize = RoundUp(sendReduceSlice.size, queNumPerNeighbor_ * dataSizePerVolume) * dataSizePerVolume;
262 0 : u64 resSendReduceSize = sendReduceSlice.size;
263 0 : u64 currSendReduceOff = sendReduceSlice.offset;
264 :
265 0 : std::vector<std::vector<LinkDataIterator>> sendRecvRedLinks;
266 0 : CHK_RET(GetSendRecvRedLinks(neighborRank, tempLinks, sendRecvRedLinks));
267 :
268 0 : for (u32 detourIdx = 0; detourIdx < queNumPerNeighbor_; detourIdx++) {
269 0 : u64 currSendReduceSize = resSendReduceSize > unitSendReduceSize ? unitSendReduceSize : resSendReduceSize;
270 0 : u64 currRecvReduceSize = resRecvReduceSize > unitRecvReduceSize ? unitRecvReduceSize : resRecvReduceSize;
271 0 : SliceInfo currSendReduceSlice = {currSendReduceOff, currSendReduceSize};
272 0 : SliceInfo currRecvReduceSlice = {currRecvReduceOff, currRecvReduceSize};
273 :
274 : std::unique_ptr<PrimGroup> primGroup
275 0 : = RunSendRecvReduce(neighborRank, (*sendRecvRedLinks[detourIdx][0]), (*sendRecvRedLinks[detourIdx][1]),
276 0 : currSendReduceSlice, currRecvReduceSlice);
277 0 : detourPrimQues[detourIdx]->Append(std::move(primGroup));
278 :
279 0 : resRecvReduceSize -= currRecvReduceSize;
280 0 : resSendReduceSize -= currSendReduceSize;
281 0 : currRecvReduceOff += currRecvReduceSize;
282 0 : currSendReduceOff += currSendReduceSize;
283 0 : }
284 :
285 0 : CHK_RET(PostSyncInterQueues(detourPrimQues));
286 0 : return HcclResult::HCCL_SUCCESS;
287 0 : }
288 :
289 0 : HcclResult TempReduceScatterMesh::GetSendRecvRedLinks(const RankId neighborRank, const ResLinks &tempLinks,
290 : std::vector<std::vector<LinkDataIterator>> &sendRecvLinks) const
291 : {
292 0 : CHK_PRT_RET(
293 : ((queNumPerNeighbor_ != NUM_TWO) || (tempRankSize_ != NUM_TWO)),
294 : HCCL_ERROR(
295 : "[CollAlgFactory] [TempReduceScatterMesh] Rank [%d], detouring is supported only in 2P Mesh in 4P topo.",
296 : myRank_),
297 : HcclResult::HCCL_E_INTERNAL);
298 :
299 0 : std::vector<LinkDataIterator> tmpLinks(NUM_TWO);
300 0 : sendRecvLinks.resize(queNumPerNeighbor_, tmpLinks);
301 :
302 0 : CHK_PRT_RET(
303 : GetDetourSendRecvLinksIn4P(myRank_, neighborRank, tempLinks, sendRecvLinks),
304 : HCCL_ERROR("[InsCollAlgFactory] [TempReduceScatterMesh] Rank [%d], get send recv links in 2P Mesh in 4P topo.",
305 : myRank_),
306 : HcclResult::HCCL_E_INTERNAL);
307 :
308 0 : return HcclResult::HCCL_SUCCESS;
309 0 : }
310 :
311 0 : HcclResult TempReduceScatterMesh::RunIndividualPeer(const RankId neighborRank, const LinkData &neighborLinkData,
312 : const SliceInfo &sendReduceSlice, const SliceInfo &recvReduceSlice,
313 : PrimQuePtr currQue) const
314 : {
315 : std::unique_ptr<PrimGroup> primGroup
316 0 : = RunSendRecvReduce(neighborRank, neighborLinkData, neighborLinkData, sendReduceSlice, recvReduceSlice);
317 0 : currQue->Append(std::move(primGroup));
318 0 : return HcclResult::HCCL_SUCCESS;
319 0 : }
320 :
321 0 : std::unique_ptr<PrimGroup> TempReduceScatterMesh::RunSendRecvReduce(const RankId neighborRank,
322 : const LinkData &sendLinkData,
323 : const LinkData &recvLinkData,
324 : const SliceInfo &currSendReduceSlice,
325 : const SliceInfo &currRecvReduceSlice) const
326 : {
327 : // PrimGroup
328 0 : std::unique_ptr<PrimGroup> primGroup = std::make_unique<PrimGroup>();
329 :
330 : // RecvReduce
331 0 : u64 recvOffset = currRecvReduceSlice.offset;
332 0 : u64 recvSize = currRecvReduceSlice.size;
333 0 : DataSlice recvRemSlice = DataSlice(buffInfo_.inBuffType, recvOffset + buffInfo_.inBuffBaseOff, recvSize);
334 0 : DataSlice recvLocSrcSlice = DataSlice(buffInfo_.scratBuffType, recvOffset + buffInfo_.scratchBuffBaseOff, recvSize);
335 0 : DataSlice recvLocDstSlice = DataSlice(buffInfo_.inBuffType, recvOffset + buffInfo_.inBuffBaseOff, recvSize);
336 0 : std::unique_ptr<Primitive> primRecvReduce = std::make_unique<PrimRecvReduce>(
337 0 : neighborRank, recvLinkData, recvRemSlice, recvLocSrcSlice, recvLocDstSlice, dataType_, redOp_, dmaMode_);
338 :
339 0 : primGroup->Append(std::move(primRecvReduce));
340 :
341 : // SendReduce
342 0 : u64 sendOffset = currSendReduceSlice.offset;
343 0 : u64 sendSize = currSendReduceSlice.size;
344 0 : DataSlice sendLocSlice = DataSlice(buffInfo_.inBuffType, sendOffset + buffInfo_.inBuffBaseOff, sendSize);
345 0 : DataSlice sendRemSrcSlice = DataSlice(buffInfo_.scratBuffType, sendOffset + buffInfo_.scratchBuffBaseOff, sendSize);
346 0 : DataSlice sendRemDstSlice = DataSlice(buffInfo_.inBuffType, sendOffset + buffInfo_.inBuffBaseOff, sendSize);
347 0 : std::unique_ptr<Primitive> primSendReduce = std::make_unique<PrimSendReduce>(
348 0 : neighborRank, sendLinkData, sendLocSlice, sendRemSrcSlice, sendRemDstSlice, dataType_, redOp_, dmaMode_);
349 :
350 0 : primGroup->Append(std::move(primSendReduce));
351 :
352 0 : return primGroup;
353 0 : }
354 :
355 0 : HcclResult TempReduceScatterMesh::PostCopyOffload(const RankSliceInfo &sliceInfoVec,
356 : std::vector<PrimQuePtr> &tempPrimQues)
357 : {
358 0 : u64 srcOffset = sliceInfoVec[tempVirtRankMap_[myRank_]][0].offset;
359 0 : u64 srcSize = sliceInfoVec[tempVirtRankMap_[myRank_]][0].size;
360 0 : u64 dstOffset = 0;
361 :
362 0 : DataSlice srcSlice = DataSlice(buffInfo_.inBuffType, srcOffset + buffInfo_.inBuffBaseOff, srcSize);
363 0 : DataSlice dstSlice = DataSlice(buffInfo_.outBuffType, dstOffset + buffInfo_.outBuffBaseOff, srcSize);
364 0 : std::unique_ptr<Primitive> primLocalCopy = std::make_unique<PrimLocalCopy>(srcSlice, dstSlice);
365 0 : tempPrimQues[0]->Append(std::move(primLocalCopy));
366 :
367 0 : return HcclResult::HCCL_SUCCESS;
368 0 : }
369 : } // namespace Hccl
|
