Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "log.h"
12 :
13 : #include "temp_all_gather_mesh.h"
14 :
15 : namespace Hccl {
16 0 : TempAllGatherMesh::TempAllGatherMesh(const RankId virtualRank, const u32 tempRankSize,
17 : const std::vector<std::vector<RankId>> &tempVTopo,
18 0 : const std::map<RankId, u32> &tempVirtRankMap)
19 0 : : AlgTemplateBase(virtualRank, tempRankSize, tempVTopo, tempVirtRankMap)
20 : {
21 0 : }
22 :
23 0 : TempAllGatherMesh::~TempAllGatherMesh()
24 : {
25 0 : }
26 :
27 0 : HcclResult TempAllGatherMesh::CalcRes(AlgTempResReq &tempResReq)
28 : {
29 0 : tempResReq.queNum = tempVTopo_[0].size() - 1;
30 :
31 0 : CHK_RET(CalcResLinksMesh(myRank_, tempRankSize_, tempVTopo_, linkNumBtwPeers_, tempResReq));
32 0 : return HcclResult::HCCL_SUCCESS;
33 : }
34 :
35 0 : HcclResult TempAllGatherMesh::CalcResDetour(const RankGraph *rankGraph, AlgTempResReq &tempResReq)
36 : {
37 : u32 myAlgRank;
38 0 : CHK_RET(GetAlgRank(myRank_, tempVTopo_[0], myAlgRank));
39 :
40 0 : linkNumBtwPeers_ = GetLinkNum(rankGraph, myRank_, tempVTopo_[0][(myAlgRank + 1) % tempRankSize_]);
41 0 : if (linkNumBtwPeers_ == 1) {
42 0 : HCCL_INFO("[CollAlgFactory] [TempAllGatherMesh] [WARNING] Rank [%d], linkNum between rank [%d] and rank [%d] "
43 : "equals 1, not able to detour",
44 : myRank_, myRank_, tempVTopo_[0][(myAlgRank + 1) % tempRankSize_]);
45 0 : enableDetour_ = false;
46 : } else {
47 0 : enableDetour_ = true;
48 : }
49 :
50 0 : queNumPerNeighbor_ = (linkNumBtwPeers_ + 1) >> 1;
51 0 : tempResReq.queNum = (tempVTopo_[0].size() - 1) * queNumPerNeighbor_;
52 :
53 0 : CHK_RET(CalcResLinksMesh(myRank_, tempRankSize_, tempVTopo_, linkNumBtwPeers_, tempResReq));
54 0 : return HcclResult::HCCL_SUCCESS;
55 : }
56 :
57 0 : HcclResult TempAllGatherMesh::CalcResDetour(ConnectedLinkMgr *linkMgr, AlgTempResReq &tempResReq)
58 : {
59 : u32 myAlgRank;
60 0 : CHK_RET(GetAlgRank(myRank_, tempVTopo_[0], myAlgRank));
61 :
62 0 : linkNumBtwPeers_ = (linkMgr->GetLinks(tempVTopo_[0][(myAlgRank + 1) % tempRankSize_])).size();
63 :
64 0 : enableDetour_ = (linkNumBtwPeers_ == 1) ? false : true;
65 :
66 0 : queNumPerNeighbor_ = (linkNumBtwPeers_ + 1) >> 1;
67 0 : tempResReq.queNum = (tempVTopo_[0].size() - 1) * queNumPerNeighbor_;
68 :
69 0 : CHK_RET(CalcResLinksMesh(myRank_, tempRankSize_, tempVTopo_, linkNumBtwPeers_, tempResReq));
70 0 : return HcclResult::HCCL_SUCCESS;
71 : }
72 :
73 : /*
74 : dataSize / (rankSize) --> chunkSize
75 : dataSize / (rankSize * dimNum) --> sliceSize
76 :
77 : SliceInfoVecforConcurrMesh: [1st chunk: [1st Slice, 2nd Slice], 2nd chunk: [1st Slice, 2nd Slice], ...]
78 : */
79 0 : HcclResult TempAllGatherMesh::CalcSliceInfo(const AllignInfo &allignInfo, const u64 dataSize,
80 : RankSliceInfo &sliceInfoVec)
81 : {
82 0 : std::vector<SliceInfo> tmp(1);
83 0 : sliceInfoVec.resize(tempRankSize_, tmp);
84 :
85 0 : CHK_RET(CalcRsAgSliceInfoMesh(myRank_, tempRankSize_, allignInfo, dataSize, sliceInfoVec));
86 :
87 0 : return HcclResult::HCCL_SUCCESS;
88 0 : }
89 :
90 0 : HcclResult TempAllGatherMesh::GenPrimQue(const TempFuncs &tempFuncs, const RankSliceInfo &sliceInfoVec,
91 : const BuffInfo &buffInfo, const ResLinks &tempLinks,
92 : std::vector<PrimQuePtr> &tempPrimQues)
93 : {
94 0 : opMode_ = tempFuncs.opMode;
95 0 : enableCounterNotify_ = tempFuncs.enableCounterNotify;
96 0 : buffInfo_ = buffInfo;
97 :
98 0 : auto linkIter = tempLinks.begin();
99 0 : linkNumBtwPeers_ = linkIter->second.size();
100 0 : HCCL_INFO("[CollAlgFactory] [TempAllGatherMesh] Rank [%d], linkNumBtwPeers equals to [%u].", myRank_,
101 : linkNumBtwPeers_);
102 0 : queNumPerNeighbor_ = (linkNumBtwPeers_ + 1) >> 1;
103 0 : HCCL_INFO("[CollAlgFactory] [TempAllGatherMesh] Rank [%d], queNumPerNeighbor equals to [%u].", myRank_,
104 : queNumPerNeighbor_);
105 0 : enableDetour_ = (linkNumBtwPeers_ == 1) ? false : true;
106 :
107 0 : majorQueNum_ = tempVTopo_[0].size() - 1;
108 0 : CHK_PRT_RET(
109 : majorQueNum_ * queNumPerNeighbor_ != tempPrimQues.size(),
110 : HCCL_ERROR(
111 : "[CollAlgFactory] [TempAllGatherMesh] Rank [%d], requiredQueNum [%u] not equals to templateQueNum [%u].",
112 : myRank_, majorQueNum_ * queNumPerNeighbor_, tempPrimQues.size()),
113 : HcclResult::HCCL_E_INTERNAL);
114 :
115 : // queue arrangement
116 0 : std::vector<PrimQuePtr> mainPrimQues;
117 0 : for (u32 queIdx = 0; queIdx < majorQueNum_; queIdx++) {
118 0 : mainPrimQues.push_back(tempPrimQues[queIdx * queNumPerNeighbor_]);
119 : }
120 :
121 : // Local Copy from Input to Scratch Buffer for OPBASE
122 0 : if ((opMode_ == OpMode::OPBASE) && tempFuncs.isForepart && !tempFuncs.forAllReduce) {
123 0 : CHK_RET(PreCopyOpbase(tempFuncs.usrData, mainPrimQues));
124 : }
125 :
126 : // Local Copy from Input to Output Buffer for OFFLOAD
127 0 : if ((opMode_ == OpMode::OFFLOAD) && (!tempFuncs.forAlgSeqComb)) {
128 0 : CHK_RET(PreCopyOffload(sliceInfoVec, tempFuncs.forAllReduce, mainPrimQues));
129 : }
130 :
131 : // semaphore sync
132 0 : if (majorQueNum_ > 1) {
133 0 : CHK_RET(PreSyncInterQueues(mainPrimQues));
134 : }
135 :
136 : // locate myRank in tempVTopo -> algRank
137 : u32 myAlgRank;
138 0 : CHK_RET(GetAlgRank(myRank_, tempVTopo_[0], myAlgRank));
139 :
140 : // run Mesh
141 0 : CHK_PRT_RET(
142 : RunMesh(myAlgRank, tempVTopo_[0], sliceInfoVec, tempLinks, tempPrimQues) != HcclResult::HCCL_SUCCESS,
143 : HCCL_ERROR("[CollAlgFactory] [TempAllGatherMesh] Rank [%d], unable to run the mesh algorithm.", myRank_),
144 : HcclResult::HCCL_E_INTERNAL);
145 :
146 : // semaphore sync
147 0 : if (majorQueNum_ > 1) {
148 0 : CHK_RET(PostSyncInterQueues(mainPrimQues));
149 : }
150 :
151 : // LocalCopy: from scratch to output for opbase
152 0 : if ((opMode_ == OpMode::OPBASE) && tempFuncs.isBottom) {
153 0 : CHK_RET(PostCopyOpbase(tempFuncs.usrData, mainPrimQues));
154 : }
155 :
156 0 : return HcclResult::HCCL_SUCCESS;
157 0 : }
158 :
159 0 : HcclResult TempAllGatherMesh::RunMesh(const u32 myAlgRank, const std::vector<RankId> &vTopo,
160 : const RankSliceInfo &sliceInfoVec, const ResLinks &tempLinks,
161 : std::vector<PrimQuePtr> &tempPrimQues)
162 : {
163 0 : for (u32 queIdx = 0; queIdx < vTopo.size() - 1; queIdx++) {
164 : // find neighbors -> virtualRank
165 0 : RankId neighborRank = vTopo[(myAlgRank + 1 + queIdx) % tempRankSize_];
166 :
167 0 : u32 recvChunkIdx = tempVirtRankMap_[neighborRank];
168 0 : u32 sendChunkIdx = tempVirtRankMap_[myRank_];
169 :
170 : // queue assignment
171 0 : if (enableDetour_) {
172 0 : std::vector<PrimQuePtr> detourPrimQues;
173 0 : for (u32 detourIdx = 0; detourIdx < queNumPerNeighbor_; detourIdx++) {
174 0 : detourPrimQues.push_back(tempPrimQues[queIdx * queNumPerNeighbor_ + detourIdx]);
175 : }
176 :
177 0 : CHK_RET(RunIndividualPeerDetour(neighborRank, sliceInfoVec[sendChunkIdx][0], sliceInfoVec[recvChunkIdx][0],
178 : tempLinks, detourPrimQues));
179 0 : } else {
180 0 : PrimQuePtr currQue = tempPrimQues[queIdx];
181 0 : LinkData neighborLinkData = tempLinks.at(neighborRank)[0];
182 0 : CHK_RET(RunIndividualPeer(neighborRank, neighborLinkData, sliceInfoVec[sendChunkIdx][0],
183 : sliceInfoVec[recvChunkIdx][0], currQue));
184 0 : }
185 : }
186 :
187 0 : return HcclResult::HCCL_SUCCESS;
188 : }
189 :
190 0 : HcclResult TempAllGatherMesh::RunIndividualPeerDetour(const RankId neighborRank, const SliceInfo &sendSlice,
191 : const SliceInfo &recvSlice, const ResLinks &tempLinks,
192 : std::vector<PrimQuePtr> &detourPrimQues)
193 : {
194 0 : CHK_RET(PreSyncInterQueues(detourPrimQues));
195 :
196 0 : u32 dataSizePerVolume = DataTypeSizeGet(dataType_);
197 0 : u64 unitRecvSize = RoundUp(recvSlice.size, queNumPerNeighbor_ * dataSizePerVolume) * dataSizePerVolume;
198 0 : u64 resRecvSize = recvSlice.size;
199 0 : u64 currRecvOff = recvSlice.offset + buffInfo_.outBuffBaseOff;
200 :
201 0 : u64 unitSendSize = RoundUp(sendSlice.size, queNumPerNeighbor_ * dataSizePerVolume) * dataSizePerVolume;
202 0 : u64 resSendSize = sendSlice.size;
203 0 : u64 currSendOff = sendSlice.offset + buffInfo_.outBuffBaseOff;
204 :
205 0 : std::vector<std::vector<LinkDataIterator>> sendRecvLinks;
206 0 : CHK_RET(GetSendRecvLinks(neighborRank, tempLinks, sendRecvLinks));
207 :
208 0 : for (u32 detourIdx = 0; detourIdx < queNumPerNeighbor_; detourIdx++) {
209 0 : u64 currRecvSize = resRecvSize > unitRecvSize ? unitRecvSize : resRecvSize;
210 0 : u64 currSendSize = resSendSize > unitSendSize ? unitSendSize : resSendSize;
211 0 : SliceInfo currSendSlice = {currSendOff, currSendSize};
212 0 : SliceInfo currRecvSlice = {currRecvOff, currRecvSize};
213 :
214 : std::unique_ptr<PrimGroup> primGroup = RunSendRecv(
215 0 : neighborRank, (*sendRecvLinks[detourIdx][0]), (*sendRecvLinks[detourIdx][1]), currSendSlice, currRecvSlice);
216 :
217 0 : detourPrimQues[detourIdx]->Append(std::move(primGroup));
218 :
219 0 : resRecvSize -= currRecvSize;
220 0 : resSendSize -= currSendSize;
221 0 : currRecvOff += currRecvSize;
222 0 : currSendOff += currSendSize;
223 0 : }
224 :
225 0 : CHK_RET(PostSyncInterQueues(detourPrimQues));
226 0 : return HcclResult::HCCL_SUCCESS;
227 0 : }
228 :
229 0 : HcclResult TempAllGatherMesh::GetSendRecvLinks(const RankId neighborRank, const ResLinks &tempLinks,
230 : std::vector<std::vector<LinkDataIterator>> &sendRecvLinks) const
231 : {
232 0 : CHK_PRT_RET(
233 : ((queNumPerNeighbor_ != NUM_TWO) || (tempRankSize_ != NUM_TWO)),
234 : HCCL_ERROR("[CollAlgFactory] [TempAllGatherMesh] Rank [%d], detouring is supported only in 2P Mesh in 4P topo.",
235 : myRank_),
236 : HcclResult::HCCL_E_INTERNAL);
237 :
238 0 : std::vector<LinkDataIterator> tmpLinks(NUM_TWO);
239 0 : sendRecvLinks.resize(queNumPerNeighbor_, tmpLinks);
240 :
241 0 : CHK_PRT_RET(
242 : GetDetourSendRecvLinksIn4P(myRank_, neighborRank, tempLinks, sendRecvLinks),
243 : HCCL_ERROR("[InsCollAlgFactory] [TempAllGatherMesh] Rank [%d], get send recv links in 2P Mesh in 4P topo.",
244 : myRank_),
245 : HcclResult::HCCL_E_INTERNAL);
246 0 : return HcclResult::HCCL_SUCCESS;
247 0 : }
248 :
249 0 : HcclResult TempAllGatherMesh::RunIndividualPeer(const RankId neighborRank, const LinkData &neighborLinkData,
250 : const SliceInfo &sendSlice, const SliceInfo &recvSlice,
251 : PrimQuePtr currQue)
252 : {
253 0 : SliceInfo currSendSlice = {sendSlice.offset + buffInfo_.outBuffBaseOff, sendSlice.size};
254 0 : SliceInfo currRecvSlice = {recvSlice.offset + buffInfo_.outBuffBaseOff, recvSlice.size};
255 :
256 : std::unique_ptr<PrimGroup> primGroup
257 0 : = RunSendRecv(neighborRank, neighborLinkData, neighborLinkData, currSendSlice, currRecvSlice);
258 0 : currQue->Append(std::move(primGroup));
259 0 : return HcclResult::HCCL_SUCCESS;
260 0 : }
261 :
262 0 : std::unique_ptr<PrimGroup> TempAllGatherMesh::RunSendRecv(const RankId neighborRank, const LinkData &sendLinkData,
263 : const LinkData &recvLinkData, const SliceInfo &currSendSlice,
264 : const SliceInfo &currRecvSlice) const
265 : {
266 : // PrimGroup
267 0 : std::unique_ptr<PrimGroup> primGroup = std::make_unique<PrimGroup>();
268 :
269 : // Recv
270 0 : DataSlice recvRemSlice = DataSlice(buffInfo_.outBuffType, currRecvSlice.offset, currRecvSlice.size);
271 0 : DataSlice recvLocSlice = DataSlice(buffInfo_.outBuffType, currRecvSlice.offset, currRecvSlice.size);
272 : std::unique_ptr<Primitive> primRecv
273 0 : = std::make_unique<PrimRecv>(neighborRank, recvLinkData, recvLocSlice, recvRemSlice, dmaMode_);
274 :
275 0 : primGroup->Append(std::move(primRecv));
276 :
277 : // Send
278 0 : DataSlice sendLocSlice = DataSlice(buffInfo_.outBuffType, currSendSlice.offset, currSendSlice.size);
279 0 : DataSlice sendRemSlice = DataSlice(buffInfo_.outBuffType, currSendSlice.offset, currSendSlice.size);
280 : std::unique_ptr<Primitive> primSend
281 0 : = std::make_unique<PrimSend>(neighborRank, sendLinkData, sendLocSlice, sendRemSlice, dmaMode_);
282 :
283 0 : primGroup->Append(std::move(primSend));
284 :
285 0 : return primGroup;
286 0 : }
287 :
288 0 : HcclResult TempAllGatherMesh::PreCopyOffload(const RankSliceInfo &sliceInfoVec, const bool forAllReduce,
289 : std::vector<PrimQuePtr> &tempPrimQues)
290 : {
291 0 : u64 srcOffset = forAllReduce ? sliceInfoVec[tempVirtRankMap_[myRank_]][0].offset : 0;
292 0 : u64 srcSize = sliceInfoVec[tempVirtRankMap_[myRank_]][0].size;
293 0 : u64 dstOffset = sliceInfoVec[tempVirtRankMap_[myRank_]][0].offset;
294 0 : DataSlice srcSlice = DataSlice(buffInfo_.inBuffType, srcOffset + buffInfo_.inBuffBaseOff, srcSize);
295 0 : DataSlice dstSlice = DataSlice(buffInfo_.outBuffType, dstOffset + buffInfo_.outBuffBaseOff, srcSize);
296 0 : std::unique_ptr<Primitive> primLocalCopy = std::make_unique<PrimLocalCopy>(srcSlice, dstSlice);
297 0 : tempPrimQues[0]->Append(std::move(primLocalCopy));
298 :
299 0 : return HcclResult::HCCL_SUCCESS;
300 0 : }
301 : } // namespace Hccl
|
