Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "log.h"
12 :
13 : #include "alg_data_trans_wrapper.h"
14 : #include "ins_temp_all_gather_mesh.h"
15 :
16 : namespace Hccl {
17 0 : InsTempAllGatherMesh1D::InsTempAllGatherMesh1D(
18 : const RankId virtualRank, const u32 tempRankSize, const std::vector<std::vector<RankId>>& tempVTopo,
19 0 : const std::map<RankId, u32>& tempVirtRankMap)
20 0 : : InsAlgTemplateBase(virtualRank, tempRankSize, tempVTopo, tempVirtRankMap)
21 0 : {}
22 :
23 0 : InsTempAllGatherMesh1D::~InsTempAllGatherMesh1D() {}
24 :
25 0 : HcclResult InsTempAllGatherMesh1D::CalcRes(AlgTempResReq& tempResReq)
26 : {
27 0 : HCCL_DEBUG("[InsTempAllGatherMesh1D] Enter CalcRes");
28 0 : CHK_RET(CalcResLinksMesh(myRank_, tempRankSize_, tempVTopo_, linkNumBtwPeers_, tempResReq));
29 :
30 0 : auto& linkReq = tempResReq.links;
31 0 : u32 pathNum = 0;
32 0 : for (auto resReqIter = linkReq.begin(); resReqIter != linkReq.end(); resReqIter++) {
33 0 : auto remoteRank = resReqIter->first;
34 0 : if (rank2PathNumMap_.find(remoteRank) == rank2PathNumMap_.end() || rank2PathNumMap_[remoteRank] == 0) {
35 0 : HCCL_ERROR("[InsTempAllGatherMesh1D] No path to remoteRank[%d]", remoteRank);
36 0 : return HcclResult::HCCL_E_INTERNAL;
37 : }
38 0 : if (pathNum == 0) {
39 0 : pathNum = rank2PathNumMap_[remoteRank];
40 0 : } else if (rank2PathNumMap_[remoteRank] != pathNum) {
41 0 : HCCL_ERROR(
42 : "[InsTempAllGatherMesh1D] Inconsistency pathNum to remoteRanks, Previous consistent "
43 : "pathNum=[%u], mismatched "
44 : "remoteRank=[%d], pathNum=[%u]",
45 : pathNum, remoteRank, rank2PathNumMap_[remoteRank]);
46 0 : return HcclResult::HCCL_E_INTERNAL;
47 : }
48 0 : resReqIter->second = pathNum;
49 : }
50 :
51 0 : tempResReq.queNum = tempVTopo_[0].size() * pathNum;
52 0 : HCCL_INFO("[InsTempAllGatherMesh1D] tempResReq.queNum = %u", tempResReq.queNum);
53 0 : tempResReq.streamNum = tempResReq.queNum;
54 0 : tempResReq.queNotifys = CreateMasterSlaveQueNotifiesRequest(tempResReq.queNum);
55 0 : HCCL_DEBUG("[InsTempAllGatherMesh1D] CalcRes queNotifys size[%zu]", tempResReq.queNotifys.size());
56 :
57 0 : QId centerQ = 0;
58 0 : tempResReq.localWaitGroupCntNotify.emplace_back(centerQ, 0);
59 0 : tempResReq.localBcastPostCntNotify.emplace_back(centerQ, 0);
60 :
61 0 : HCCL_DEBUG("[InsTempAllGatherMesh1D] CalcRes done");
62 0 : return HcclResult::HCCL_SUCCESS;
63 : }
64 :
65 : HcclResult
66 0 : InsTempAllGatherMesh1D::CalcSliceInfo(const AllignInfo& allignInfo, const u64 dataSize, RankSliceInfo& sliceInfoVec)
67 : {
68 0 : std::vector<SliceInfo> tmp(1);
69 0 : sliceInfoVec.resize(tempRankSize_, tmp);
70 :
71 0 : CHK_RET(CalcRsAgSliceInfoMesh(myRank_, tempRankSize_, allignInfo, dataSize, sliceInfoVec));
72 :
73 0 : return HcclResult::HCCL_SUCCESS;
74 0 : }
75 :
76 0 : HcclResult InsTempAllGatherMesh1D::GenExtIns(
77 : const TempFuncs& tempFuncs, const TemplateDataParams& tempAlgParams, const ResLinks& tempLinks,
78 : std::vector<InsQuePtr>& tempInsQues)
79 : {
80 0 : HCCL_INFO("[InsTempAllGatherMesh1D] RunAllGather start");
81 :
82 0 : opMode_ = tempFuncs.opMode;
83 0 : tempAlgParams_ = tempAlgParams;
84 0 : tempLinks_ = tempLinks;
85 :
86 0 : uint32_t linkNum = tempLinks.begin()->second.size();
87 : // 流的数量不能少于linkNum
88 0 : CHK_PRT_RET(
89 : linkNum > tempInsQues.size(),
90 : HCCL_ERROR("[CollAlgFactory] [InsTempAllGatherMesh] Rank [%d], requiredQue Error.", myRank_),
91 : HcclResult::HCCL_E_INTERNAL);
92 0 : std::vector<float> dataSplitRate(linkNum);
93 0 : CHK_RET(CalcDataSplitRateForLinks(tempLinks.begin()->second, dataSplitRate));
94 0 : queNumPerNeighbor_ = linkNum;
95 0 : std::vector<InsQuePtr> localInsQues;
96 0 : localInsQues.push_back(tempInsQues[0]);
97 0 : localInsQues.push_back(tempInsQues[tempInsQues.size() - 1]);
98 :
99 0 : CHK_RET(LocalCopyToScratch(tempInsQues[0]));
100 : // 如果输入和输出内存是同一片地址则必须先拷贝完成再运算Mesh，否则的话提升性能同步并行
101 0 : if (tempAlgParams.buffInfo.inBuffType == tempAlgParams.buffInfo.outBuffType) {
102 : // 因为后续RunMesh会写Output，如果inBuffType和outBuffType先同步后拷贝会导致改写
103 0 : CHK_RET(LocalCopyToUsrOut(tempInsQues[0]));
104 0 : CHK_RET(PreSyncInterQueues(tempInsQues));
105 : } else {
106 0 : CHK_RET(PreSyncInterQueues(tempInsQues));
107 0 : CHK_RET(LocalCopyToUsrOut(tempInsQues[0]));
108 : }
109 :
110 : // locate myRank in tempVTopo -> algRank
111 : u32 myAlgRank;
112 0 : CHK_RET(GetAlgRank(myRank_, tempVTopo_[0], myAlgRank));
113 : // run Mesh 使用第1至rankSize条queue
114 0 : CHK_PRT_RET(
115 : RunMesh(myAlgRank, tempVTopo_[0], tempInsQues) != HcclResult::HCCL_SUCCESS,
116 : HCCL_ERROR("[InsTempAllGatherMesh1D] Rank [%d], unable to run mesh algorithm.", myRank_),
117 : HcclResult::HCCL_E_INTERNAL);
118 : // semaphore sync
119 0 : CHK_RET(PostSyncInterQueues(tempInsQues));
120 :
121 0 : return HcclResult::HCCL_SUCCESS;
122 0 : }
123 :
124 0 : HcclResult InsTempAllGatherMesh1D::LocalCopyToUsrOut(InsQuePtr tempInsQue)
125 : {
126 : u32 myAlgRank;
127 0 : CHK_RET(GetAlgRank(myRank_, tempVTopo_[0], myAlgRank));
128 : // 做个保护，tailSize填写为0就认为尾块是正常块
129 0 : u64 tailSize = (tempAlgParams_.tailSize == 0) ? tempAlgParams_.sliceSize : tempAlgParams_.tailSize;
130 0 : u64 sliceSize = (myAlgRank == tempRankSize_ - 1) ? tailSize : tempAlgParams_.sliceSize;
131 0 : for (u32 rpt = 0; rpt < tempAlgParams_.repeatNum; ++rpt) {
132 0 : const u64 inBaseOff = tempAlgParams_.buffInfo.inBuffBaseOff + rpt * tempAlgParams_.inputRepeatStride;
133 0 : const u64 outBaseOff = tempAlgParams_.buffInfo.outBuffBaseOff + rpt * tempAlgParams_.outputRepeatStride;
134 :
135 0 : const u64 inOff = tempAlgParams_.inputSliceStride * myAlgRank + inBaseOff;
136 0 : const u64 outOff = tempAlgParams_.outputSliceStride * myAlgRank + outBaseOff;
137 0 : if (tempAlgParams_.buffInfo.inBuffType == tempAlgParams_.buffInfo.outBuffType && inOff == outOff) {
138 0 : continue;
139 : }
140 : // OPBASE模式下scratch中已有备份数据，从scratch读取避免INPUT src/dst重叠导致数据损坏
141 0 : if (opMode_ == OpMode::OPBASE && tempAlgParams_.buffInfo.inBuffType == tempAlgParams_.buffInfo.outBuffType) {
142 0 : const u64 scratchRepeatStride = tempAlgParams_.sliceSize * (tempRankSize_ - 1) + tailSize;
143 0 : const u64 scratchOff = tempAlgParams_.buffInfo.scratchBuffBaseOff + rpt * scratchRepeatStride
144 0 : + tempAlgParams_.sliceSize * myAlgRank;
145 0 : DataSlice src(tempAlgParams_.buffInfo.scratBuffType, scratchOff, sliceSize);
146 0 : DataSlice dst(tempAlgParams_.buffInfo.outBuffType, outOff, sliceSize);
147 0 : HCCL_INFO("[InsTempAllGatherMesh1D] scratch:%s -> out:%s", src.Describe().c_str(), dst.Describe().c_str());
148 0 : auto ins = std::make_unique<InsLocalCopy>(src, dst);
149 0 : tempInsQue->Append(std::move(ins));
150 0 : } else {
151 0 : DataSlice src(tempAlgParams_.buffInfo.inBuffType, inOff, sliceSize);
152 0 : DataSlice dst(tempAlgParams_.buffInfo.outBuffType, outOff, sliceSize);
153 0 : HCCL_INFO("[InsTempAllGatherMesh1D] in:%s -> out:%s", src.Describe().c_str(), dst.Describe().c_str());
154 0 : auto ins = std::make_unique<InsLocalCopy>(src, dst);
155 0 : tempInsQue->Append(std::move(ins));
156 0 : }
157 : }
158 0 : return HcclResult::HCCL_SUCCESS;
159 : }
160 :
161 0 : HcclResult InsTempAllGatherMesh1D::LocalCopyToScratch(InsQuePtr tempInsQue)
162 : {
163 : u32 myAlgRank;
164 0 : CHK_RET(GetAlgRank(myRank_, tempVTopo_[0], myAlgRank));
165 : // 做个保护，tailSize填写为0就认为尾块是正常块
166 0 : u64 tailSize = (tempAlgParams_.tailSize == 0) ? tempAlgParams_.sliceSize : tempAlgParams_.tailSize;
167 0 : u64 sliceSize = (myAlgRank == tempRankSize_ - 1) ? tailSize : tempAlgParams_.sliceSize;
168 :
169 0 : if (opMode_ == OpMode::OPBASE) {
170 0 : for (u32 rpt = 0; rpt < tempAlgParams_.repeatNum; ++rpt) {
171 0 : const u64 scratchRepeatStride = tempAlgParams_.sliceSize * (tempRankSize_ - 1) + tailSize;
172 0 : const u64 inBaseOff = tempAlgParams_.buffInfo.inBuffBaseOff + rpt * tempAlgParams_.inputRepeatStride;
173 0 : const u64 outBaseOff = tempAlgParams_.buffInfo.scratchBuffBaseOff + rpt * scratchRepeatStride;
174 0 : const u64 inOff = tempAlgParams_.inputSliceStride * myAlgRank + inBaseOff;
175 0 : const u64 outOff = tempAlgParams_.sliceSize * myAlgRank + outBaseOff;
176 :
177 0 : DataSlice src(tempAlgParams_.buffInfo.inBuffType, inOff, sliceSize);
178 0 : DataSlice dst(tempAlgParams_.buffInfo.scratBuffType, outOff, sliceSize);
179 0 : HCCL_INFO("[InsTempAllGatherMesh1D] in:%s -> scratch:%s", src.Describe().c_str(), dst.Describe().c_str());
180 :
181 0 : auto ins = std::make_unique<InsLocalCopy>(src, dst);
182 0 : tempInsQue->Append(std::move(ins));
183 0 : }
184 : }
185 0 : return HcclResult::HCCL_SUCCESS;
186 : }
187 0 : HcclResult InsTempAllGatherMesh1D::SingleRunMesh(
188 : const u32 myAlgRank, u32 connectedRank, std::vector<InsQuePtr>& tempInsQues, u32& queIdx)
189 : {
190 0 : u32 connectedAlgRank = 0;
191 0 : CHK_RET(GetAlgRank(connectedRank, tempVTopo_[0], connectedAlgRank));
192 : // 做个保护，tailSize填写为0就认为尾块是正常块
193 0 : tempAlgParams_.tailSize = (tempAlgParams_.tailSize == 0) ? tempAlgParams_.sliceSize : tempAlgParams_.tailSize;
194 0 : std::vector<LinkData>& neighborLinkDatas = tempLinks_.at(connectedRank);
195 0 : u32 linkNum = rank2PathNumMap_.at(connectedRank);
196 0 : if (linkNum != neighborLinkDatas.size()) {
197 0 : HCCL_ERROR("InsTempAllGatherMesh1D::RunMesh linkNum != neighborLinkDatas.size()");
198 0 : return HcclResult::HCCL_E_INTERNAL;
199 : }
200 0 : std::vector<float> dataSplitRate(linkNum);
201 0 : CHK_RET(CalcDataSplitRateForLinks(neighborLinkDatas, dataSplitRate));
202 0 : for (u32 j = 0; j < linkNum; j++) {
203 0 : CHK_PRT_RET(
204 : queIdx >= tempInsQues.size(),
205 : HCCL_ERROR("[SingleRunMesh] queIdx=%u, tempInsQues.size=%zu", queIdx, tempInsQues.size()),
206 : HcclResult::HCCL_E_INTERNAL);
207 0 : LinkData& neighborLinkData = neighborLinkDatas[j];
208 0 : std::vector<DataSlice> txSrcSlices, txDstSlices, rxSrcSlices, rxDstSlices;
209 0 : PrepareLinkSlices(
210 : myAlgRank, connectedAlgRank, dataSplitRate, j, txSrcSlices, txDstSlices, rxSrcSlices, rxDstSlices);
211 0 : TxRxSlicesList sendRecvSlicesList({txSrcSlices, txDstSlices}, {rxSrcSlices, rxDstSlices});
212 0 : TxRxLinks sendRecvLinks(neighborLinkData, neighborLinkData);
213 0 : SendRecvInfo sendRecvInfo(sendRecvLinks, sendRecvSlicesList);
214 0 : InsQuePtr curInsQues = tempInsQues.at(++queIdx);
215 0 : CHK_PRT_RET(
216 : SendRecv(sendRecvInfo, curInsQues, 0, true, DmaMode::GET),
217 : HCCL_ERROR("[InsTempAllGatherMesh1D] sendrecv failed (connectedRank=%d)", connectedRank),
218 : HcclResult::HCCL_E_INTERNAL);
219 0 : }
220 0 : return HcclResult::HCCL_SUCCESS;
221 0 : }
222 :
223 0 : HcclResult InsTempAllGatherMesh1D::RunMesh(
224 : const u32 myAlgRank, const std::vector<RankId>& vTopo, std::vector<InsQuePtr>& tempInsQues)
225 : {
226 0 : u32 queIdx = 0;
227 0 : for (u32 rankIdx = 0; rankIdx < vTopo.size() - 1; rankIdx++) {
228 0 : RankId connectedRank = vTopo[(myAlgRank + 1 + rankIdx) % vTopo.size()];
229 0 : auto it = tempLinks_.find(connectedRank);
230 0 : if (it == tempLinks_.end()) {
231 0 : HCCL_ERROR("[InsTempAllGatherMesh1D] connectedRank does not exist");
232 0 : return HcclResult::HCCL_E_PARA;
233 : }
234 0 : CHK_PRT_RET(
235 : tempLinks_.at(connectedRank).empty(),
236 : HCCL_ERROR(
237 : "[InsTempAllGatherMesh1D] connectedRank=%d, tempLinks_.size=%zu", connectedRank, tempLinks_.size()),
238 : HcclResult::HCCL_E_INTERNAL);
239 : // SingleRunMesh函数中会自动移动curInsQues队列下标
240 0 : CHK_PRT_RET(
241 : SingleRunMesh(myAlgRank, connectedRank, tempInsQues, queIdx),
242 : HCCL_ERROR("[SingleRunMesh] myAlgRank=%u, connectedRank=%d", myAlgRank, connectedRank),
243 : HcclResult::HCCL_E_INTERNAL);
244 : }
245 0 : return HcclResult::HCCL_SUCCESS;
246 : }
247 :
248 0 : void InsTempAllGatherMesh1D::PrepareLinkSlices(
249 : const u32 myAlgRank, const u32 connectedAlgRank, const std::vector<float>& dataSplitRate, const u32 linkIndex,
250 : std::vector<DataSlice>& txSrcSlices, std::vector<DataSlice>& txDstSlices, std::vector<DataSlice>& rxSrcSlices,
251 : std::vector<DataSlice>& rxDstSlices)
252 : {
253 : BufferType writeType
254 0 : = (opMode_ == OpMode::OPBASE) ? tempAlgParams_.buffInfo.scratBuffType : tempAlgParams_.buffInfo.inBuffType;
255 0 : for (u32 rpt = 0; rpt < tempAlgParams_.repeatNum; ++rpt) {
256 0 : const u64 inBaseOff = tempAlgParams_.buffInfo.inBuffBaseOff + rpt * tempAlgParams_.inputRepeatStride;
257 0 : const u64 outBaseOff = tempAlgParams_.buffInfo.outBuffBaseOff + rpt * tempAlgParams_.outputRepeatStride;
258 0 : const u64 scratchRepeatStride = tempAlgParams_.sliceSize * (tempRankSize_ - 1) + tempAlgParams_.tailSize;
259 0 : const u64 scratchBase = tempAlgParams_.buffInfo.scratchBuffBaseOff + rpt * scratchRepeatStride;
260 0 : u64 txInOffset = tempAlgParams_.inputSliceStride * myAlgRank + inBaseOff;
261 0 : u64 txOutOffset = tempAlgParams_.outputSliceStride * myAlgRank + outBaseOff;
262 0 : u64 txScratchOffset = scratchBase + tempAlgParams_.sliceSize * myAlgRank;
263 0 : u64 txDstOffset = (opMode_ == OpMode::OPBASE) ? txScratchOffset : txOutOffset;
264 0 : u64 rxInOffset = tempAlgParams_.inputSliceStride * connectedAlgRank + inBaseOff;
265 0 : u64 rxOutOffset = tempAlgParams_.outputSliceStride * connectedAlgRank + outBaseOff;
266 0 : u64 rxScratchOffset = scratchBase + tempAlgParams_.sliceSize * connectedAlgRank;
267 0 : u64 rxSrcOffset = (opMode_ == OpMode::OPBASE) ? rxScratchOffset : rxInOffset;
268 0 : u64 sendSliceSize = (myAlgRank == tempRankSize_ - 1) ? tempAlgParams_.tailSize : tempAlgParams_.sliceSize;
269 0 : u64 recvSliceSize
270 0 : = (connectedAlgRank == tempRankSize_ - 1) ? tempAlgParams_.tailSize : tempAlgParams_.sliceSize;
271 0 : DataSlice txSrcSlice(tempAlgParams_.buffInfo.inBuffType, txInOffset, sendSliceSize);
272 0 : DataSlice txDstSlice(writeType, txDstOffset, sendSliceSize);
273 0 : DataSlice rxSrcSlice(writeType, rxSrcOffset, recvSliceSize);
274 0 : DataSlice rxDstSlice(tempAlgParams_.buffInfo.outBuffType, rxOutOffset, recvSliceSize);
275 0 : txSrcSlices.push_back(CalcDataSliceForLinks(txSrcSlice, dataSplitRate, linkIndex, dataType_));
276 0 : txDstSlices.push_back(CalcDataSliceForLinks(txDstSlice, dataSplitRate, linkIndex, dataType_));
277 0 : rxSrcSlices.push_back(CalcDataSliceForLinks(rxSrcSlice, dataSplitRate, linkIndex, dataType_));
278 0 : rxDstSlices.push_back(CalcDataSliceForLinks(rxDstSlice, dataSplitRate, linkIndex, dataType_));
279 : }
280 0 : return;
281 : }
282 :
283 : } // namespace Hccl
|
