Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "log.h"
12 :
13 : #include "temp_all_gather_mesh.h"
14 :
15 : namespace Hccl {
16 0 : TempAllGatherMesh::TempAllGatherMesh(
17 : const RankId virtualRank, const u32 tempRankSize, const std::vector<std::vector<RankId>>& tempVTopo,
18 0 : const std::map<RankId, u32>& tempVirtRankMap)
19 0 : : AlgTemplateBase(virtualRank, tempRankSize, tempVTopo, tempVirtRankMap)
20 0 : {}
21 :
22 0 : TempAllGatherMesh::~TempAllGatherMesh() {}
23 :
24 0 : HcclResult TempAllGatherMesh::CalcRes(AlgTempResReq& tempResReq)
25 : {
26 0 : tempResReq.queNum = tempVTopo_[0].size() - 1;
27 :
28 0 : CHK_RET(CalcResLinksMesh(myRank_, tempRankSize_, tempVTopo_, linkNumBtwPeers_, tempResReq));
29 0 : return HcclResult::HCCL_SUCCESS;
30 : }
31 :
32 0 : HcclResult TempAllGatherMesh::CalcResDetour(const RankGraph* rankGraph, AlgTempResReq& tempResReq)
33 : {
34 : u32 myAlgRank;
35 0 : CHK_RET(GetAlgRank(myRank_, tempVTopo_[0], myAlgRank));
36 :
37 0 : linkNumBtwPeers_ = GetLinkNum(rankGraph, myRank_, tempVTopo_[0][(myAlgRank + 1) % tempRankSize_]);
38 0 : if (linkNumBtwPeers_ == 1) {
39 0 : HCCL_INFO(
40 : "[CollAlgFactory] [TempAllGatherMesh] [WARNING] Rank [%d], linkNum between rank [%d] and rank [%d] "
41 : "equals 1, not able to detour",
42 : myRank_, myRank_, tempVTopo_[0][(myAlgRank + 1) % tempRankSize_]);
43 0 : enableDetour_ = false;
44 : } else {
45 0 : enableDetour_ = true;
46 : }
47 :
48 0 : queNumPerNeighbor_ = (linkNumBtwPeers_ + 1) >> 1;
49 0 : tempResReq.queNum = (tempVTopo_[0].size() - 1) * queNumPerNeighbor_;
50 :
51 0 : CHK_RET(CalcResLinksMesh(myRank_, tempRankSize_, tempVTopo_, linkNumBtwPeers_, tempResReq));
52 0 : return HcclResult::HCCL_SUCCESS;
53 : }
54 :
55 0 : HcclResult TempAllGatherMesh::CalcResDetour(ConnectedLinkMgr* linkMgr, AlgTempResReq& tempResReq)
56 : {
57 : u32 myAlgRank;
58 0 : CHK_RET(GetAlgRank(myRank_, tempVTopo_[0], myAlgRank));
59 :
60 0 : linkNumBtwPeers_ = (linkMgr->GetLinks(tempVTopo_[0][(myAlgRank + 1) % tempRankSize_])).size();
61 :
62 0 : enableDetour_ = (linkNumBtwPeers_ == 1) ? false : true;
63 :
64 0 : queNumPerNeighbor_ = (linkNumBtwPeers_ + 1) >> 1;
65 0 : tempResReq.queNum = (tempVTopo_[0].size() - 1) * queNumPerNeighbor_;
66 :
67 0 : CHK_RET(CalcResLinksMesh(myRank_, tempRankSize_, tempVTopo_, linkNumBtwPeers_, tempResReq));
68 0 : return HcclResult::HCCL_SUCCESS;
69 : }
70 :
71 : /*
72 : dataSize / (rankSize) --> chunkSize
73 : dataSize / (rankSize * dimNum) --> sliceSize
74 :
75 : SliceInfoVecforConcurrMesh: [1st chunk: [1st Slice, 2nd Slice], 2nd chunk: [1st Slice, 2nd Slice], ...]
76 : */
77 : HcclResult
78 0 : TempAllGatherMesh::CalcSliceInfo(const AllignInfo& allignInfo, const u64 dataSize, RankSliceInfo& sliceInfoVec)
79 : {
80 0 : std::vector<SliceInfo> tmp(1);
81 0 : sliceInfoVec.resize(tempRankSize_, tmp);
82 :
83 0 : CHK_RET(CalcRsAgSliceInfoMesh(myRank_, tempRankSize_, allignInfo, dataSize, sliceInfoVec));
84 :
85 0 : return HcclResult::HCCL_SUCCESS;
86 0 : }
87 :
88 0 : HcclResult TempAllGatherMesh::GenPrimQue(
89 : const TempFuncs& tempFuncs, const RankSliceInfo& sliceInfoVec, const BuffInfo& buffInfo, const ResLinks& tempLinks,
90 : std::vector<PrimQuePtr>& tempPrimQues)
91 : {
92 0 : opMode_ = tempFuncs.opMode;
93 0 : enableCounterNotify_ = tempFuncs.enableCounterNotify;
94 0 : buffInfo_ = buffInfo;
95 :
96 0 : auto linkIter = tempLinks.begin();
97 0 : linkNumBtwPeers_ = linkIter->second.size();
98 0 : HCCL_INFO(
99 : "[CollAlgFactory] [TempAllGatherMesh] Rank [%d], linkNumBtwPeers equals to [%u].", myRank_, linkNumBtwPeers_);
100 0 : queNumPerNeighbor_ = (linkNumBtwPeers_ + 1) >> 1;
101 0 : HCCL_INFO(
102 : "[CollAlgFactory] [TempAllGatherMesh] Rank [%d], queNumPerNeighbor equals to [%u].", myRank_,
103 : queNumPerNeighbor_);
104 0 : enableDetour_ = (linkNumBtwPeers_ == 1) ? false : true;
105 :
106 0 : majorQueNum_ = tempVTopo_[0].size() - 1;
107 0 : CHK_PRT_RET(
108 : majorQueNum_ * queNumPerNeighbor_ != tempPrimQues.size(),
109 : HCCL_ERROR(
110 : "[CollAlgFactory] [TempAllGatherMesh] Rank [%d], requiredQueNum [%u] not equals to templateQueNum [%u].",
111 : myRank_, majorQueNum_ * queNumPerNeighbor_, tempPrimQues.size()),
112 : HcclResult::HCCL_E_INTERNAL);
113 :
114 : // queue arrangement
115 0 : std::vector<PrimQuePtr> mainPrimQues;
116 0 : for (u32 queIdx = 0; queIdx < majorQueNum_; queIdx++) {
117 0 : mainPrimQues.push_back(tempPrimQues[queIdx * queNumPerNeighbor_]);
118 : }
119 :
120 : // Local Copy from Input to Scratch Buffer for OPBASE
121 0 : if ((opMode_ == OpMode::OPBASE) && tempFuncs.isForepart && !tempFuncs.forAllReduce) {
122 0 : CHK_RET(PreCopyOpbase(tempFuncs.usrData, mainPrimQues));
123 : }
124 :
125 : // Local Copy from Input to Output Buffer for OFFLOAD
126 0 : if ((opMode_ == OpMode::OFFLOAD) && (!tempFuncs.forAlgSeqComb)) {
127 0 : CHK_RET(PreCopyOffload(sliceInfoVec, tempFuncs.forAllReduce, mainPrimQues));
128 : }
129 :
130 : // semaphore sync
131 0 : if (majorQueNum_ > 1) {
132 0 : CHK_RET(PreSyncInterQueues(mainPrimQues));
133 : }
134 :
135 : // locate myRank in tempVTopo -> algRank
136 : u32 myAlgRank;
137 0 : CHK_RET(GetAlgRank(myRank_, tempVTopo_[0], myAlgRank));
138 :
139 : // run Mesh
140 0 : CHK_PRT_RET(
141 : RunMesh(myAlgRank, tempVTopo_[0], sliceInfoVec, tempLinks, tempPrimQues) != HcclResult::HCCL_SUCCESS,
142 : HCCL_ERROR("[CollAlgFactory] [TempAllGatherMesh] Rank [%d], unable to run the mesh algorithm.", myRank_),
143 : HcclResult::HCCL_E_INTERNAL);
144 :
145 : // semaphore sync
146 0 : if (majorQueNum_ > 1) {
147 0 : CHK_RET(PostSyncInterQueues(mainPrimQues));
148 : }
149 :
150 : // LocalCopy: from scratch to output for opbase
151 0 : if ((opMode_ == OpMode::OPBASE) && tempFuncs.isBottom) {
152 0 : CHK_RET(PostCopyOpbase(tempFuncs.usrData, mainPrimQues));
153 : }
154 :
155 0 : return HcclResult::HCCL_SUCCESS;
156 0 : }
157 :
158 0 : HcclResult TempAllGatherMesh::RunMesh(
159 : const u32 myAlgRank, const std::vector<RankId>& vTopo, const RankSliceInfo& sliceInfoVec, const ResLinks& tempLinks,
160 : std::vector<PrimQuePtr>& tempPrimQues)
161 : {
162 0 : for (u32 queIdx = 0; queIdx < vTopo.size() - 1; queIdx++) {
163 : // find neighbors -> virtualRank
164 0 : RankId neighborRank = vTopo[(myAlgRank + 1 + queIdx) % tempRankSize_];
165 :
166 0 : u32 recvChunkIdx = tempVirtRankMap_[neighborRank];
167 0 : u32 sendChunkIdx = tempVirtRankMap_[myRank_];
168 :
169 : // queue assignment
170 0 : if (enableDetour_) {
171 0 : std::vector<PrimQuePtr> detourPrimQues;
172 0 : for (u32 detourIdx = 0; detourIdx < queNumPerNeighbor_; detourIdx++) {
173 0 : detourPrimQues.push_back(tempPrimQues[queIdx * queNumPerNeighbor_ + detourIdx]);
174 : }
175 :
176 0 : CHK_RET(RunIndividualPeerDetour(
177 : neighborRank, sliceInfoVec[sendChunkIdx][0], sliceInfoVec[recvChunkIdx][0], tempLinks, detourPrimQues));
178 0 : } else {
179 0 : PrimQuePtr currQue = tempPrimQues[queIdx];
180 0 : LinkData neighborLinkData = tempLinks.at(neighborRank)[0];
181 0 : CHK_RET(RunIndividualPeer(
182 : neighborRank, neighborLinkData, sliceInfoVec[sendChunkIdx][0], sliceInfoVec[recvChunkIdx][0], currQue));
183 0 : }
184 : }
185 :
186 0 : return HcclResult::HCCL_SUCCESS;
187 : }
188 :
189 0 : HcclResult TempAllGatherMesh::RunIndividualPeerDetour(
190 : const RankId neighborRank, const SliceInfo& sendSlice, const SliceInfo& recvSlice, const ResLinks& tempLinks,
191 : std::vector<PrimQuePtr>& detourPrimQues)
192 : {
193 0 : CHK_RET(PreSyncInterQueues(detourPrimQues));
194 :
195 0 : u32 dataSizePerVolume = DataTypeSizeGet(dataType_);
196 0 : u64 unitRecvSize = RoundUp(recvSlice.size, queNumPerNeighbor_ * dataSizePerVolume) * dataSizePerVolume;
197 0 : u64 resRecvSize = recvSlice.size;
198 0 : u64 currRecvOff = recvSlice.offset + buffInfo_.outBuffBaseOff;
199 :
200 0 : u64 unitSendSize = RoundUp(sendSlice.size, queNumPerNeighbor_ * dataSizePerVolume) * dataSizePerVolume;
201 0 : u64 resSendSize = sendSlice.size;
202 0 : u64 currSendOff = sendSlice.offset + buffInfo_.outBuffBaseOff;
203 :
204 0 : std::vector<std::vector<LinkDataIterator>> sendRecvLinks;
205 0 : CHK_RET(GetSendRecvLinks(neighborRank, tempLinks, sendRecvLinks));
206 :
207 0 : for (u32 detourIdx = 0; detourIdx < queNumPerNeighbor_; detourIdx++) {
208 0 : u64 currRecvSize = resRecvSize > unitRecvSize ? unitRecvSize : resRecvSize;
209 0 : u64 currSendSize = resSendSize > unitSendSize ? unitSendSize : resSendSize;
210 0 : SliceInfo currSendSlice = {currSendOff, currSendSize};
211 0 : SliceInfo currRecvSlice = {currRecvOff, currRecvSize};
212 :
213 : std::unique_ptr<PrimGroup> primGroup = RunSendRecv(
214 0 : neighborRank, (*sendRecvLinks[detourIdx][0]), (*sendRecvLinks[detourIdx][1]), currSendSlice, currRecvSlice);
215 :
216 0 : detourPrimQues[detourIdx]->Append(std::move(primGroup));
217 :
218 0 : resRecvSize -= currRecvSize;
219 0 : resSendSize -= currSendSize;
220 0 : currRecvOff += currRecvSize;
221 0 : currSendOff += currSendSize;
222 0 : }
223 :
224 0 : CHK_RET(PostSyncInterQueues(detourPrimQues));
225 0 : return HcclResult::HCCL_SUCCESS;
226 0 : }
227 :
228 0 : HcclResult TempAllGatherMesh::GetSendRecvLinks(
229 : const RankId neighborRank, const ResLinks& tempLinks,
230 : std::vector<std::vector<LinkDataIterator>>& sendRecvLinks) const
231 : {
232 0 : CHK_PRT_RET(
233 : ((queNumPerNeighbor_ != NUM_TWO) || (tempRankSize_ != NUM_TWO)),
234 : HCCL_ERROR(
235 : "[CollAlgFactory] [TempAllGatherMesh] Rank [%d], detouring is supported only in 2P Mesh in 4P topo.",
236 : myRank_),
237 : HcclResult::HCCL_E_INTERNAL);
238 :
239 0 : std::vector<LinkDataIterator> tmpLinks(NUM_TWO);
240 0 : sendRecvLinks.resize(queNumPerNeighbor_, tmpLinks);
241 :
242 0 : CHK_PRT_RET(
243 : GetDetourSendRecvLinksIn4P(myRank_, neighborRank, tempLinks, sendRecvLinks),
244 : HCCL_ERROR(
245 : "[InsCollAlgFactory] [TempAllGatherMesh] Rank [%d], get send recv links in 2P Mesh in 4P topo.", myRank_),
246 : HcclResult::HCCL_E_INTERNAL);
247 0 : return HcclResult::HCCL_SUCCESS;
248 0 : }
249 :
250 0 : HcclResult TempAllGatherMesh::RunIndividualPeer(
251 : const RankId neighborRank, const LinkData& neighborLinkData, const SliceInfo& sendSlice, const SliceInfo& recvSlice,
252 : PrimQuePtr currQue)
253 : {
254 0 : SliceInfo currSendSlice = {sendSlice.offset + buffInfo_.outBuffBaseOff, sendSlice.size};
255 0 : SliceInfo currRecvSlice = {recvSlice.offset + buffInfo_.outBuffBaseOff, recvSlice.size};
256 :
257 : std::unique_ptr<PrimGroup> primGroup
258 0 : = RunSendRecv(neighborRank, neighborLinkData, neighborLinkData, currSendSlice, currRecvSlice);
259 0 : currQue->Append(std::move(primGroup));
260 0 : return HcclResult::HCCL_SUCCESS;
261 0 : }
262 :
263 0 : std::unique_ptr<PrimGroup> TempAllGatherMesh::RunSendRecv(
264 : const RankId neighborRank, const LinkData& sendLinkData, const LinkData& recvLinkData,
265 : const SliceInfo& currSendSlice, const SliceInfo& currRecvSlice) const
266 : {
267 : // PrimGroup
268 0 : std::unique_ptr<PrimGroup> primGroup = std::make_unique<PrimGroup>();
269 :
270 : // Recv
271 0 : DataSlice recvRemSlice = DataSlice(buffInfo_.outBuffType, currRecvSlice.offset, currRecvSlice.size);
272 0 : DataSlice recvLocSlice = DataSlice(buffInfo_.outBuffType, currRecvSlice.offset, currRecvSlice.size);
273 : std::unique_ptr<Primitive> primRecv
274 0 : = std::make_unique<PrimRecv>(neighborRank, recvLinkData, recvLocSlice, recvRemSlice, dmaMode_);
275 :
276 0 : primGroup->Append(std::move(primRecv));
277 :
278 : // Send
279 0 : DataSlice sendLocSlice = DataSlice(buffInfo_.outBuffType, currSendSlice.offset, currSendSlice.size);
280 0 : DataSlice sendRemSlice = DataSlice(buffInfo_.outBuffType, currSendSlice.offset, currSendSlice.size);
281 : std::unique_ptr<Primitive> primSend
282 0 : = std::make_unique<PrimSend>(neighborRank, sendLinkData, sendLocSlice, sendRemSlice, dmaMode_);
283 :
284 0 : primGroup->Append(std::move(primSend));
285 :
286 0 : return primGroup;
287 0 : }
288 :
289 0 : HcclResult TempAllGatherMesh::PreCopyOffload(
290 : const RankSliceInfo& sliceInfoVec, const bool forAllReduce, std::vector<PrimQuePtr>& tempPrimQues)
291 : {
292 0 : u64 srcOffset = forAllReduce ? sliceInfoVec[tempVirtRankMap_[myRank_]][0].offset : 0;
293 0 : u64 srcSize = sliceInfoVec[tempVirtRankMap_[myRank_]][0].size;
294 0 : u64 dstOffset = sliceInfoVec[tempVirtRankMap_[myRank_]][0].offset;
295 0 : DataSlice srcSlice = DataSlice(buffInfo_.inBuffType, srcOffset + buffInfo_.inBuffBaseOff, srcSize);
296 0 : DataSlice dstSlice = DataSlice(buffInfo_.outBuffType, dstOffset + buffInfo_.outBuffBaseOff, srcSize);
297 0 : std::unique_ptr<Primitive> primLocalCopy = std::make_unique<PrimLocalCopy>(srcSlice, dstSlice);
298 0 : tempPrimQues[0]->Append(std::move(primLocalCopy));
299 :
300 0 : return HcclResult::HCCL_SUCCESS;
301 0 : }
302 : } // namespace Hccl
|
