Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include <numeric>
12 :
13 : #include "log.h"
14 :
15 : #include "temp_reduce_scatter_mesh.h"
16 :
17 : namespace Hccl {
18 0 : TempReduceScatterMesh::TempReduceScatterMesh(
19 : const RankId virtualRank, const u32 tempRankSize, const std::vector<std::vector<RankId>>& tempVTopo,
20 0 : const std::map<RankId, u32>& tempVirtRankMap)
21 0 : : AlgTemplateBase(virtualRank, tempRankSize, tempVTopo, tempVirtRankMap)
22 0 : {}
23 :
24 0 : TempReduceScatterMesh::~TempReduceScatterMesh() {}
25 :
26 : HcclResult
27 0 : TempReduceScatterMesh::CalcRes(const bool forAllReduce, AlgTempResReq& tempResReq, u32& requiredScratchMultiplier)
28 : {
29 : (void)forAllReduce;
30 0 : tempResReq.queNum = tempVTopo_[0].size() - 1;
31 0 : requiredScratchMultiplier = tempRankSize_;
32 :
33 0 : CHK_RET(CalcResLinks(tempResReq));
34 0 : return HcclResult::HCCL_SUCCESS;
35 : }
36 :
37 0 : HcclResult TempReduceScatterMesh::CalcResDetour(
38 : const bool forAllReduce, const RankGraph* rankGraph, AlgTempResReq& tempResReq, u32& requiredScratchMultiplier)
39 : {
40 : (void)forAllReduce;
41 :
42 : u32 myAlgRank;
43 0 : CHK_RET(GetAlgRank(myRank_, tempVTopo_[0], myAlgRank));
44 :
45 0 : linkNumBtwPeers_ = GetLinkNum(rankGraph, myRank_, tempVTopo_[0][(myAlgRank + 1) % tempRankSize_]);
46 0 : if (linkNumBtwPeers_ == 1) {
47 0 : HCCL_INFO(
48 : "[CollAlgFactory] [TempReduceScatterMesh] [WARNING] Rank [%d], linkNum between rank [%d] and rank [%d] "
49 : "equals 1, not able to detour",
50 : myRank_, myRank_, tempVTopo_[0][(myAlgRank + 1) % tempRankSize_]);
51 0 : enableDetour_ = false;
52 : } else {
53 0 : enableDetour_ = true;
54 : }
55 :
56 0 : queNumPerNeighbor_ = (linkNumBtwPeers_ + 1) >> 1;
57 0 : tempResReq.queNum = (tempVTopo_[0].size() - 1) * queNumPerNeighbor_;
58 :
59 0 : requiredScratchMultiplier = tempRankSize_;
60 :
61 0 : CHK_RET(CalcResLinks(tempResReq));
62 0 : return HcclResult::HCCL_SUCCESS;
63 : }
64 :
65 0 : HcclResult TempReduceScatterMesh::CalcResDetour(
66 : const bool forAllReduce, ConnectedLinkMgr* linkMgr, AlgTempResReq& tempResReq, u32& requiredScratchMultiplier)
67 : {
68 : (void)forAllReduce;
69 :
70 : u32 myAlgRank;
71 0 : CHK_RET(GetAlgRank(myRank_, tempVTopo_[0], myAlgRank));
72 :
73 0 : linkNumBtwPeers_ = (linkMgr->GetLinks(tempVTopo_[0][(myAlgRank + 1) % tempRankSize_])).size();
74 :
75 0 : enableDetour_ = (linkNumBtwPeers_ == 1) ? false : true;
76 :
77 0 : queNumPerNeighbor_ = (linkNumBtwPeers_ + 1) >> 1;
78 0 : tempResReq.queNum = (tempVTopo_[0].size() - 1) * queNumPerNeighbor_;
79 :
80 0 : requiredScratchMultiplier = tempRankSize_;
81 :
82 0 : CHK_RET(CalcResLinks(tempResReq));
83 0 : return HcclResult::HCCL_SUCCESS;
84 : }
85 :
86 0 : HcclResult TempReduceScatterMesh::CalcResLinks(AlgTempResReq& tempResReq)
87 : {
88 : u32 myAlgRank;
89 0 : CHK_RET(GetAlgRank(myRank_, tempVTopo_[0], myAlgRank));
90 :
91 0 : for (u32 queIdx = 0; queIdx < tempVTopo_[0].size() - 1; queIdx++) {
92 : // find neighbors -> virtualRank
93 0 : RankId neighborRank = tempVTopo_[0][(myAlgRank + 1 + queIdx) % tempRankSize_];
94 :
95 : // LinkNum
96 0 : tempResReq.links[neighborRank] = linkNumBtwPeers_;
97 : }
98 :
99 0 : return HcclResult::HCCL_SUCCESS;
100 : }
101 :
102 : /*
103 : dataSize / (rankSize) --> chunkSize
104 : dataSize / (rankSize * queNum) --> sliceSize
105 :
106 : SliceInfoVecforRing: [1st chunk: [1st Slice, 2nd Slice, ...], 2nd chunk: [1st Slice, 2nd Slice, ...], ...]
107 : */
108 0 : HcclResult TempReduceScatterMesh::CalcSliceInfo(
109 : const AllignInfo& allignInfo, const bool forAllReduce, const u64 dataSize, RankSliceInfo& sliceInfoVec)
110 : {
111 0 : std::vector<SliceInfo> tmp(1);
112 0 : sliceInfoVec.resize(tempRankSize_, tmp);
113 :
114 0 : if (forAllReduce) {
115 : // for allreduce, dataSize = total dataSize
116 0 : CHK_RET(CalcSliceInfoAllReduce(allignInfo, dataSize, sliceInfoVec));
117 : } else {
118 : // for reduce scatter, dataSize = chunkSize
119 0 : CHK_RET(CalcRsAgSliceInfoMesh(myRank_, tempRankSize_, allignInfo, dataSize, sliceInfoVec));
120 : }
121 0 : return HcclResult::HCCL_SUCCESS;
122 0 : }
123 :
124 0 : HcclResult TempReduceScatterMesh::CalcSliceInfoAllReduce(
125 : const AllignInfo& allignInfo, const u64 dataSize, RankSliceInfo& sliceInfoVec) const
126 : {
127 : u64 unitAllignSize;
128 0 : CHK_RET(GetUnitAllignSize(allignInfo, unitAllignSize));
129 :
130 0 : u64 chunkSize = RoundUp(dataSize, (tempRankSize_ * unitAllignSize)) * unitAllignSize;
131 :
132 0 : u64 accumOff = 0;
133 0 : for (u32 rankIdx = 0; rankIdx < tempRankSize_; rankIdx++) {
134 0 : u64 currChunkSize = ((dataSize - accumOff) > chunkSize) ? chunkSize : (dataSize - accumOff);
135 0 : SliceInfo slice = {accumOff, currChunkSize};
136 0 : sliceInfoVec[rankIdx][0] = slice;
137 0 : accumOff += currChunkSize;
138 : }
139 :
140 0 : CHK_PRT_RET(
141 : (sliceInfoVec[tempRankSize_ - 1][0].offset + sliceInfoVec[tempRankSize_ - 1][0].size != dataSize),
142 : HCCL_ERROR("[CollAlgFactory] [TempReduceScatterMesh] Rank [%d], SliceInfo calculation error!", myRank_),
143 : HcclResult::HCCL_E_INTERNAL);
144 :
145 0 : return HcclResult::HCCL_SUCCESS;
146 : }
147 :
148 0 : HcclResult TempReduceScatterMesh::GenPrimQue(
149 : const TempFuncs& tempFuncs, const RankSliceInfo& sliceInfoVec, const BuffInfo& buffInfo, const ResLinks& tempLinks,
150 : std::vector<PrimQuePtr>& tempPrimQues)
151 : {
152 0 : opMode_ = tempFuncs.opMode;
153 0 : enableCounterNotify_ = tempFuncs.enableCounterNotify;
154 0 : buffInfo_ = buffInfo;
155 :
156 0 : auto linkIter = tempLinks.begin();
157 0 : linkNumBtwPeers_ = linkIter->second.size();
158 0 : HCCL_INFO(
159 : "[CollAlgFactory] [TempReduceScatterMesh] Rank [%d], linkNumBtwPeers equals to [%u].", myRank_,
160 : linkNumBtwPeers_);
161 0 : queNumPerNeighbor_ = (linkNumBtwPeers_ + 1) >> 1;
162 0 : HCCL_INFO(
163 : "[CollAlgFactory] [TempReduceScatterMesh] Rank [%d], queNumPerNeighbor equals to [%u].", myRank_,
164 : queNumPerNeighbor_);
165 0 : enableDetour_ = (linkNumBtwPeers_ == 1) ? false : true;
166 :
167 0 : majorQueNum_ = tempVTopo_[0].size() - 1;
168 0 : CHK_PRT_RET(
169 : majorQueNum_ * queNumPerNeighbor_ != tempPrimQues.size(),
170 : HCCL_ERROR(
171 : "[CollAlgFactory] [TempReduceScatterMesh] Rank [%d], requiredQueNum [%u] not equals to "
172 : "templateQueNum [%u].",
173 : myRank_, majorQueNum_ * queNumPerNeighbor_, tempPrimQues.size()),
174 : HcclResult::HCCL_E_INTERNAL);
175 :
176 : // queue arrangement
177 0 : std::vector<PrimQuePtr> mainPrimQues;
178 0 : for (u32 queIdx = 0; queIdx < majorQueNum_; queIdx++) {
179 0 : mainPrimQues.push_back(tempPrimQues[queIdx * queNumPerNeighbor_]);
180 : }
181 :
182 : // LocalCopy: from input to scratch In Buffer for OPBASE
183 0 : if ((opMode_ == OpMode::OPBASE) && tempFuncs.isForepart) {
184 0 : CHK_RET(PreCopyOpbase(tempFuncs.usrData, mainPrimQues));
185 : }
186 :
187 : // semaphore sync
188 0 : if (majorQueNum_ > 1) {
189 0 : CHK_RET(PreSyncInterQueues(mainPrimQues));
190 : }
191 :
192 : // locate myRank in tempVTopo -> algRank
193 : u32 myAlgRank;
194 0 : CHK_RET(GetAlgRank(myRank_, tempVTopo_[0], myAlgRank));
195 :
196 : // run Mesh
197 0 : CHK_PRT_RET(
198 : RunMesh(myAlgRank, tempVTopo_[0], sliceInfoVec, tempLinks, tempPrimQues) != HcclResult::HCCL_SUCCESS,
199 : HCCL_ERROR("[CollAlgFactory] [TempReduceScatterMesh] Rank [%d], unable to run the mesh algorithm.", myRank_),
200 : HcclResult::HCCL_E_INTERNAL);
201 :
202 : // semaphore sync
203 0 : if (majorQueNum_ > 1) {
204 0 : CHK_RET(PostSyncInterQueues(mainPrimQues));
205 : }
206 :
207 : // LocalCopy for standalone reducescatter in Offload Mode
208 0 : if ((opMode_ == OpMode::OFFLOAD) && !tempFuncs.forAllReduce && !tempFuncs.forAlgSeqComb) {
209 0 : CHK_RET(PostCopyOffload(sliceInfoVec, mainPrimQues));
210 : }
211 :
212 : // LocalCopy from scratch to output for Opbase
213 0 : if (tempFuncs.isBottom && !tempFuncs.forAllReduce) {
214 0 : CHK_RET(PostCopyOpbase(tempFuncs.usrData, mainPrimQues));
215 : }
216 :
217 0 : return HcclResult::HCCL_SUCCESS;
218 0 : }
219 :
220 0 : HcclResult TempReduceScatterMesh::RunMesh(
221 : const u32 myAlgRank, const std::vector<RankId>& vTopo, const RankSliceInfo& sliceInfoVec, const ResLinks& tempLinks,
222 : std::vector<PrimQuePtr>& tempPrimQues)
223 : {
224 0 : for (u32 queIdx = 0; queIdx < vTopo.size() - 1; queIdx++) {
225 : // find neighbors -> virtualRank
226 0 : RankId neighborRank = vTopo[(myAlgRank + 1 + queIdx) % tempRankSize_];
227 :
228 0 : u32 recvChunkIdx = tempVirtRankMap_[myRank_];
229 0 : u32 sendChunkIdx = tempVirtRankMap_[neighborRank];
230 :
231 : // queue assignment
232 0 : if (enableDetour_) {
233 0 : std::vector<PrimQuePtr> detourPrimQues;
234 0 : for (u32 detourIdx = 0; detourIdx < queNumPerNeighbor_; detourIdx++) {
235 0 : detourPrimQues.push_back(tempPrimQues[queIdx * queNumPerNeighbor_ + detourIdx]);
236 : }
237 0 : HCCL_INFO("[CollAlgFactory] [TempReduceScatterMesh] Rank [%d], Run Mesh with Detour.", myRank_);
238 0 : CHK_RET(RunIndividualPeerDetour(
239 : neighborRank, sliceInfoVec[sendChunkIdx][0], sliceInfoVec[recvChunkIdx][0], tempLinks, detourPrimQues));
240 0 : } else {
241 0 : PrimQuePtr currQue = tempPrimQues[queIdx];
242 0 : LinkData neighborLinkData = tempLinks.at(neighborRank)[0];
243 0 : HCCL_INFO("[CollAlgFactory] [TempReduceScatterMesh] Rank [%d], Run Mesh without Detour.", myRank_);
244 0 : CHK_RET(RunIndividualPeer(
245 : neighborRank, neighborLinkData, sliceInfoVec[sendChunkIdx][0], sliceInfoVec[recvChunkIdx][0], currQue));
246 0 : }
247 : }
248 :
249 0 : return HcclResult::HCCL_SUCCESS;
250 : }
251 :
252 0 : HcclResult TempReduceScatterMesh::RunIndividualPeerDetour(
253 : const RankId neighborRank, const SliceInfo& sendReduceSlice, const SliceInfo& recvReduceSlice,
254 : const ResLinks& tempLinks, std::vector<PrimQuePtr>& detourPrimQues)
255 : {
256 0 : CHK_RET(PreSyncInterQueues(detourPrimQues));
257 :
258 0 : u32 dataSizePerVolume = DataTypeSizeGet(dataType_);
259 0 : u64 unitRecvReduceSize = RoundUp(recvReduceSlice.size, queNumPerNeighbor_ * dataSizePerVolume) * dataSizePerVolume;
260 0 : u64 resRecvReduceSize = recvReduceSlice.size;
261 0 : u64 currRecvReduceOff = recvReduceSlice.offset;
262 :
263 0 : u64 unitSendReduceSize = RoundUp(sendReduceSlice.size, queNumPerNeighbor_ * dataSizePerVolume) * dataSizePerVolume;
264 0 : u64 resSendReduceSize = sendReduceSlice.size;
265 0 : u64 currSendReduceOff = sendReduceSlice.offset;
266 :
267 0 : std::vector<std::vector<LinkDataIterator>> sendRecvRedLinks;
268 0 : CHK_RET(GetSendRecvRedLinks(neighborRank, tempLinks, sendRecvRedLinks));
269 :
270 0 : for (u32 detourIdx = 0; detourIdx < queNumPerNeighbor_; detourIdx++) {
271 0 : u64 currSendReduceSize = resSendReduceSize > unitSendReduceSize ? unitSendReduceSize : resSendReduceSize;
272 0 : u64 currRecvReduceSize = resRecvReduceSize > unitRecvReduceSize ? unitRecvReduceSize : resRecvReduceSize;
273 0 : SliceInfo currSendReduceSlice = {currSendReduceOff, currSendReduceSize};
274 0 : SliceInfo currRecvReduceSlice = {currRecvReduceOff, currRecvReduceSize};
275 :
276 : std::unique_ptr<PrimGroup> primGroup = RunSendRecvReduce(
277 0 : neighborRank, (*sendRecvRedLinks[detourIdx][0]), (*sendRecvRedLinks[detourIdx][1]), currSendReduceSlice,
278 0 : currRecvReduceSlice);
279 0 : detourPrimQues[detourIdx]->Append(std::move(primGroup));
280 :
281 0 : resRecvReduceSize -= currRecvReduceSize;
282 0 : resSendReduceSize -= currSendReduceSize;
283 0 : currRecvReduceOff += currRecvReduceSize;
284 0 : currSendReduceOff += currSendReduceSize;
285 0 : }
286 :
287 0 : CHK_RET(PostSyncInterQueues(detourPrimQues));
288 0 : return HcclResult::HCCL_SUCCESS;
289 0 : }
290 :
291 0 : HcclResult TempReduceScatterMesh::GetSendRecvRedLinks(
292 : const RankId neighborRank, const ResLinks& tempLinks,
293 : std::vector<std::vector<LinkDataIterator>>& sendRecvLinks) const
294 : {
295 0 : CHK_PRT_RET(
296 : ((queNumPerNeighbor_ != NUM_TWO) || (tempRankSize_ != NUM_TWO)),
297 : HCCL_ERROR(
298 : "[CollAlgFactory] [TempReduceScatterMesh] Rank [%d], detouring is supported only in 2P Mesh in 4P topo.",
299 : myRank_),
300 : HcclResult::HCCL_E_INTERNAL);
301 :
302 0 : std::vector<LinkDataIterator> tmpLinks(NUM_TWO);
303 0 : sendRecvLinks.resize(queNumPerNeighbor_, tmpLinks);
304 :
305 0 : CHK_PRT_RET(
306 : GetDetourSendRecvLinksIn4P(myRank_, neighborRank, tempLinks, sendRecvLinks),
307 : HCCL_ERROR(
308 : "[InsCollAlgFactory] [TempReduceScatterMesh] Rank [%d], get send recv links in 2P Mesh in 4P topo.",
309 : myRank_),
310 : HcclResult::HCCL_E_INTERNAL);
311 :
312 0 : return HcclResult::HCCL_SUCCESS;
313 0 : }
314 :
315 0 : HcclResult TempReduceScatterMesh::RunIndividualPeer(
316 : const RankId neighborRank, const LinkData& neighborLinkData, const SliceInfo& sendReduceSlice,
317 : const SliceInfo& recvReduceSlice, PrimQuePtr currQue) const
318 : {
319 : std::unique_ptr<PrimGroup> primGroup
320 0 : = RunSendRecvReduce(neighborRank, neighborLinkData, neighborLinkData, sendReduceSlice, recvReduceSlice);
321 0 : currQue->Append(std::move(primGroup));
322 0 : return HcclResult::HCCL_SUCCESS;
323 0 : }
324 :
325 0 : std::unique_ptr<PrimGroup> TempReduceScatterMesh::RunSendRecvReduce(
326 : const RankId neighborRank, const LinkData& sendLinkData, const LinkData& recvLinkData,
327 : const SliceInfo& currSendReduceSlice, const SliceInfo& currRecvReduceSlice) const
328 : {
329 : // PrimGroup
330 0 : std::unique_ptr<PrimGroup> primGroup = std::make_unique<PrimGroup>();
331 :
332 : // RecvReduce
333 0 : u64 recvOffset = currRecvReduceSlice.offset;
334 0 : u64 recvSize = currRecvReduceSlice.size;
335 0 : DataSlice recvRemSlice = DataSlice(buffInfo_.inBuffType, recvOffset + buffInfo_.inBuffBaseOff, recvSize);
336 0 : DataSlice recvLocSrcSlice = DataSlice(buffInfo_.scratBuffType, recvOffset + buffInfo_.scratchBuffBaseOff, recvSize);
337 0 : DataSlice recvLocDstSlice = DataSlice(buffInfo_.inBuffType, recvOffset + buffInfo_.inBuffBaseOff, recvSize);
338 0 : std::unique_ptr<Primitive> primRecvReduce = std::make_unique<PrimRecvReduce>(
339 0 : neighborRank, recvLinkData, recvRemSlice, recvLocSrcSlice, recvLocDstSlice, dataType_, redOp_, dmaMode_);
340 :
341 0 : primGroup->Append(std::move(primRecvReduce));
342 :
343 : // SendReduce
344 0 : u64 sendOffset = currSendReduceSlice.offset;
345 0 : u64 sendSize = currSendReduceSlice.size;
346 0 : DataSlice sendLocSlice = DataSlice(buffInfo_.inBuffType, sendOffset + buffInfo_.inBuffBaseOff, sendSize);
347 0 : DataSlice sendRemSrcSlice = DataSlice(buffInfo_.scratBuffType, sendOffset + buffInfo_.scratchBuffBaseOff, sendSize);
348 0 : DataSlice sendRemDstSlice = DataSlice(buffInfo_.inBuffType, sendOffset + buffInfo_.inBuffBaseOff, sendSize);
349 0 : std::unique_ptr<Primitive> primSendReduce = std::make_unique<PrimSendReduce>(
350 0 : neighborRank, sendLinkData, sendLocSlice, sendRemSrcSlice, sendRemDstSlice, dataType_, redOp_, dmaMode_);
351 :
352 0 : primGroup->Append(std::move(primSendReduce));
353 :
354 0 : return primGroup;
355 0 : }
356 :
357 : HcclResult
358 0 : TempReduceScatterMesh::PostCopyOffload(const RankSliceInfo& sliceInfoVec, std::vector<PrimQuePtr>& tempPrimQues)
359 : {
360 0 : u64 srcOffset = sliceInfoVec[tempVirtRankMap_[myRank_]][0].offset;
361 0 : u64 srcSize = sliceInfoVec[tempVirtRankMap_[myRank_]][0].size;
362 0 : u64 dstOffset = 0;
363 :
364 0 : DataSlice srcSlice = DataSlice(buffInfo_.inBuffType, srcOffset + buffInfo_.inBuffBaseOff, srcSize);
365 0 : DataSlice dstSlice = DataSlice(buffInfo_.outBuffType, dstOffset + buffInfo_.outBuffBaseOff, srcSize);
366 0 : std::unique_ptr<Primitive> primLocalCopy = std::make_unique<PrimLocalCopy>(srcSlice, dstSlice);
367 0 : tempPrimQues[0]->Append(std::move(primLocalCopy));
368 :
369 0 : return HcclResult::HCCL_SUCCESS;
370 0 : }
371 : } // namespace Hccl
|
