Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "log.h"
12 :
13 : #include "alg_data_trans_wrapper.h"
14 : #include "ins_temp_all_gather_mesh.h"
15 :
16 : namespace Hccl {
17 0 : InsTempAllGatherMesh1D::InsTempAllGatherMesh1D(const RankId virtualRank, const u32 tempRankSize,
18 0 : const std::vector<std::vector<RankId>> &tempVTopo, const std::map<RankId, u32> &tempVirtRankMap)
19 0 : : InsAlgTemplateBase(virtualRank, tempRankSize, tempVTopo, tempVirtRankMap)
20 : {
21 0 : }
22 :
23 0 : InsTempAllGatherMesh1D::~InsTempAllGatherMesh1D()
24 : {
25 0 : }
26 :
27 0 : HcclResult InsTempAllGatherMesh1D::CalcRes(AlgTempResReq &tempResReq)
28 : {
29 0 : HCCL_DEBUG("[InsTempAllGatherMesh1D] Enter CalcRes");
30 0 : CHK_RET(CalcResLinksMesh(myRank_, tempRankSize_, tempVTopo_, linkNumBtwPeers_, tempResReq));
31 :
32 0 : auto &linkReq = tempResReq.links;
33 0 : u32 pathNum = 0;
34 0 : for (auto resReqIter = linkReq.begin(); resReqIter != linkReq.end(); resReqIter++) {
35 0 : auto remoteRank = resReqIter->first;
36 0 : if (rank2PathNumMap_.find(remoteRank) == rank2PathNumMap_.end() || rank2PathNumMap_[remoteRank] == 0) {
37 0 : HCCL_ERROR("[InsTempAllGatherMesh1D] No path to remoteRank[%d]", remoteRank);
38 0 : return HcclResult::HCCL_E_INTERNAL;
39 : }
40 0 : if (pathNum == 0) {
41 0 : pathNum = rank2PathNumMap_[remoteRank];
42 0 : } else if (rank2PathNumMap_[remoteRank] != pathNum) {
43 0 : HCCL_ERROR("[InsTempAllGatherMesh1D] Inconsistency pathNum to remoteRanks, Previous consistent "
44 : "pathNum=[%u], mismatched "
45 : "remoteRank=[%d], pathNum=[%u]",
46 : pathNum, remoteRank, rank2PathNumMap_[remoteRank]);
47 0 : return HcclResult::HCCL_E_INTERNAL;
48 : }
49 0 : resReqIter->second = pathNum;
50 : }
51 :
52 0 : tempResReq.queNum = tempVTopo_[0].size() * pathNum;
53 0 : HCCL_INFO("[InsTempAllGatherMesh1D] tempResReq.queNum = %u", tempResReq.queNum);
54 0 : tempResReq.streamNum = tempResReq.queNum;
55 0 : tempResReq.queNotifys = CreateMasterSlaveQueNotifiesRequest(tempResReq.queNum);
56 0 : HCCL_DEBUG("[InsTempAllGatherMesh1D] CalcRes queNotifys size[%zu]", tempResReq.queNotifys.size());
57 :
58 0 : QId centerQ = 0;
59 0 : tempResReq.localWaitGroupCntNotify.emplace_back(centerQ, 0);
60 0 : tempResReq.localBcastPostCntNotify.emplace_back(centerQ, 0);
61 :
62 0 : HCCL_DEBUG("[InsTempAllGatherMesh1D] CalcRes done");
63 0 : return HcclResult::HCCL_SUCCESS;
64 : }
65 :
66 0 : HcclResult InsTempAllGatherMesh1D::CalcSliceInfo(
67 : const AllignInfo &allignInfo, const u64 dataSize, RankSliceInfo &sliceInfoVec)
68 : {
69 0 : std::vector<SliceInfo> tmp(1);
70 0 : sliceInfoVec.resize(tempRankSize_, tmp);
71 :
72 0 : CHK_RET(CalcRsAgSliceInfoMesh(myRank_, tempRankSize_, allignInfo, dataSize, sliceInfoVec));
73 :
74 0 : return HcclResult::HCCL_SUCCESS;
75 0 : }
76 :
77 0 : HcclResult InsTempAllGatherMesh1D::GenExtIns(const TempFuncs &tempFuncs, const TemplateDataParams &tempAlgParams,
78 : const ResLinks &tempLinks, std::vector<InsQuePtr> &tempInsQues)
79 : {
80 0 : HCCL_INFO("[InsTempAllGatherMesh1D] RunAllGather start");
81 :
82 0 : opMode_ = tempFuncs.opMode;
83 0 : tempAlgParams_ = tempAlgParams;
84 0 : tempLinks_ = tempLinks;
85 :
86 0 : uint32_t linkNum = tempLinks.begin()->second.size();
87 : // 流的数量不能少于linkNum
88 0 : CHK_PRT_RET(linkNum > tempInsQues.size(),
89 : HCCL_ERROR("[CollAlgFactory] [InsTempAllGatherMesh] Rank [%d], requiredQue Error.", myRank_),
90 : HcclResult::HCCL_E_INTERNAL);
91 0 : std::vector<float> dataSplitRate(linkNum);
92 0 : CHK_RET(CalcDataSplitRateForLinks(tempLinks.begin()->second, dataSplitRate));
93 0 : queNumPerNeighbor_ = linkNum;
94 0 : std::vector<InsQuePtr> localInsQues;
95 0 : localInsQues.push_back(tempInsQues[0]);
96 0 : localInsQues.push_back(tempInsQues[tempInsQues.size() - 1]);
97 :
98 0 : CHK_RET(LocalCopyToScratch(tempInsQues[0]));
99 : // 如果输入和输出内存是同一片地址则必须先拷贝完成再运算Mesh，否则的话提升性能同步并行
100 0 : if (tempAlgParams.buffInfo.inBuffType == tempAlgParams.buffInfo.outBuffType) {
101 : // 因为后续RunMesh会写Output，如果inBuffType和outBuffType先同步后拷贝会导致改写
102 0 : CHK_RET(LocalCopyToUsrOut(tempInsQues[0]));
103 0 : CHK_RET(PreSyncInterQueues(tempInsQues));
104 : } else {
105 0 : CHK_RET(PreSyncInterQueues(tempInsQues));
106 0 : CHK_RET(LocalCopyToUsrOut(tempInsQues[0]));
107 : }
108 :
109 : // locate myRank in tempVTopo -> algRank
110 : u32 myAlgRank;
111 0 : CHK_RET(GetAlgRank(myRank_, tempVTopo_[0], myAlgRank));
112 : // run Mesh 使用第1至rankSize条queue
113 0 : CHK_PRT_RET(RunMesh(myAlgRank, tempVTopo_[0], tempInsQues) != HcclResult::HCCL_SUCCESS,
114 : HCCL_ERROR("[InsTempAllGatherMesh1D] Rank [%d], unable to run mesh algorithm.", myRank_),
115 : HcclResult::HCCL_E_INTERNAL);
116 : // semaphore sync
117 0 : CHK_RET(PostSyncInterQueues(tempInsQues));
118 :
119 0 : return HcclResult::HCCL_SUCCESS;
120 0 : }
121 :
122 0 : HcclResult InsTempAllGatherMesh1D::LocalCopyToUsrOut(InsQuePtr tempInsQue)
123 : {
124 : u32 myAlgRank;
125 0 : CHK_RET(GetAlgRank(myRank_, tempVTopo_[0], myAlgRank));
126 : // 做个保护，tailSize填写为0就认为尾块是正常块
127 0 : u64 tailSize = (tempAlgParams_.tailSize == 0) ? tempAlgParams_.sliceSize : tempAlgParams_.tailSize;
128 0 : u64 sliceSize = (myAlgRank == tempRankSize_ - 1) ? tailSize : tempAlgParams_.sliceSize;
129 0 : for (u32 rpt = 0; rpt < tempAlgParams_.repeatNum; ++rpt) {
130 0 : const u64 inBaseOff = tempAlgParams_.buffInfo.inBuffBaseOff + rpt * tempAlgParams_.inputRepeatStride;
131 0 : const u64 outBaseOff = tempAlgParams_.buffInfo.outBuffBaseOff + rpt * tempAlgParams_.outputRepeatStride;
132 :
133 0 : const u64 inOff = tempAlgParams_.inputSliceStride * myAlgRank + inBaseOff;
134 0 : const u64 outOff = tempAlgParams_.outputSliceStride * myAlgRank + outBaseOff;
135 0 : if (tempAlgParams_.buffInfo.inBuffType == tempAlgParams_.buffInfo.outBuffType && inOff == outOff) {
136 0 : continue;
137 : }
138 : // OPBASE模式下scratch中已有备份数据，从scratch读取避免INPUT src/dst重叠导致数据损坏
139 0 : if (opMode_ == OpMode::OPBASE &&
140 0 : tempAlgParams_.buffInfo.inBuffType == tempAlgParams_.buffInfo.outBuffType) {
141 0 : const u64 scratchRepeatStride =
142 0 : tempAlgParams_.sliceSize * (tempRankSize_ - 1) + tailSize;
143 0 : const u64 scratchOff = tempAlgParams_.buffInfo.scratchBuffBaseOff + rpt * scratchRepeatStride +
144 0 : tempAlgParams_.sliceSize * myAlgRank;
145 0 : DataSlice src(tempAlgParams_.buffInfo.scratBuffType, scratchOff, sliceSize);
146 0 : DataSlice dst(tempAlgParams_.buffInfo.outBuffType, outOff, sliceSize);
147 0 : HCCL_INFO("[InsTempAllGatherMesh1D] scratch:%s -> out:%s",
148 : src.Describe().c_str(), dst.Describe().c_str());
149 0 : auto ins = std::make_unique<InsLocalCopy>(src, dst);
150 0 : tempInsQue->Append(std::move(ins));
151 0 : } else {
152 0 : DataSlice src(tempAlgParams_.buffInfo.inBuffType, inOff, sliceSize);
153 0 : DataSlice dst(tempAlgParams_.buffInfo.outBuffType, outOff, sliceSize);
154 0 : HCCL_INFO("[InsTempAllGatherMesh1D] in:%s -> out:%s", src.Describe().c_str(), dst.Describe().c_str());
155 0 : auto ins = std::make_unique<InsLocalCopy>(src, dst);
156 0 : tempInsQue->Append(std::move(ins));
157 0 : }
158 : }
159 0 : return HcclResult::HCCL_SUCCESS;
160 : }
161 :
162 0 : HcclResult InsTempAllGatherMesh1D::LocalCopyToScratch(InsQuePtr tempInsQue)
163 : {
164 : u32 myAlgRank;
165 0 : CHK_RET(GetAlgRank(myRank_, tempVTopo_[0], myAlgRank));
166 : // 做个保护，tailSize填写为0就认为尾块是正常块
167 0 : u64 tailSize = (tempAlgParams_.tailSize == 0) ? tempAlgParams_.sliceSize : tempAlgParams_.tailSize;
168 0 : u64 sliceSize = (myAlgRank == tempRankSize_ - 1) ? tailSize : tempAlgParams_.sliceSize;
169 :
170 0 : if (opMode_ == OpMode::OPBASE) {
171 0 : for (u32 rpt = 0; rpt < tempAlgParams_.repeatNum; ++rpt) {
172 0 : const u64 scratchRepeatStride = tempAlgParams_.sliceSize * (tempRankSize_ - 1) + tailSize;
173 0 : const u64 inBaseOff = tempAlgParams_.buffInfo.inBuffBaseOff + rpt * tempAlgParams_.inputRepeatStride;
174 0 : const u64 outBaseOff = tempAlgParams_.buffInfo.scratchBuffBaseOff + rpt * scratchRepeatStride;
175 0 : const u64 inOff = tempAlgParams_.inputSliceStride * myAlgRank + inBaseOff;
176 0 : const u64 outOff = tempAlgParams_.sliceSize * myAlgRank + outBaseOff;
177 :
178 0 : DataSlice src(tempAlgParams_.buffInfo.inBuffType, inOff, sliceSize);
179 0 : DataSlice dst(tempAlgParams_.buffInfo.scratBuffType, outOff, sliceSize);
180 0 : HCCL_INFO("[InsTempAllGatherMesh1D] in:%s -> scratch:%s", src.Describe().c_str(), dst.Describe().c_str());
181 :
182 0 : auto ins = std::make_unique<InsLocalCopy>(src, dst);
183 0 : tempInsQue->Append(std::move(ins));
184 0 : }
185 : }
186 0 : return HcclResult::HCCL_SUCCESS;
187 : }
188 0 : HcclResult InsTempAllGatherMesh1D::SingleRunMesh(
189 : const u32 myAlgRank, u32 connectedRank, std::vector<InsQuePtr> &tempInsQues, u32 &queIdx)
190 : {
191 0 : u32 connectedAlgRank = 0;
192 0 : CHK_RET(GetAlgRank(connectedRank, tempVTopo_[0], connectedAlgRank));
193 : // 做个保护，tailSize填写为0就认为尾块是正常块
194 0 : tempAlgParams_.tailSize = (tempAlgParams_.tailSize == 0) ? tempAlgParams_.sliceSize : tempAlgParams_.tailSize;
195 0 : std::vector<LinkData> &neighborLinkDatas = tempLinks_.at(connectedRank);
196 0 : u32 linkNum = rank2PathNumMap_.at(connectedRank);
197 0 : if (linkNum != neighborLinkDatas.size()) {
198 0 : HCCL_ERROR("InsTempAllGatherMesh1D::RunMesh linkNum != neighborLinkDatas.size()");
199 0 : return HcclResult::HCCL_E_INTERNAL;
200 : }
201 0 : std::vector<float> dataSplitRate(linkNum);
202 0 : CHK_RET(CalcDataSplitRateForLinks(neighborLinkDatas, dataSplitRate));
203 0 : for (u32 j = 0; j < linkNum; j++) {
204 0 : CHK_PRT_RET(queIdx >= tempInsQues.size(),
205 : HCCL_ERROR("[SingleRunMesh] queIdx=%u, tempInsQues.size=%zu", queIdx, tempInsQues.size()),
206 : HcclResult::HCCL_E_INTERNAL);
207 0 : LinkData &neighborLinkData = neighborLinkDatas[j];
208 0 : std::vector<DataSlice> txSrcSlices, txDstSlices, rxSrcSlices, rxDstSlices;
209 0 : PrepareLinkSlices(
210 : myAlgRank, connectedAlgRank, dataSplitRate, j, txSrcSlices, txDstSlices, rxSrcSlices, rxDstSlices);
211 0 : TxRxSlicesList sendRecvSlicesList({txSrcSlices, txDstSlices}, {rxSrcSlices, rxDstSlices});
212 0 : TxRxLinks sendRecvLinks(neighborLinkData, neighborLinkData);
213 0 : SendRecvInfo sendRecvInfo(sendRecvLinks, sendRecvSlicesList);
214 0 : InsQuePtr curInsQues = tempInsQues.at(++queIdx);
215 0 : CHK_PRT_RET(SendRecv(sendRecvInfo, curInsQues, 0, true, DmaMode::GET),
216 : HCCL_ERROR("[InsTempAllGatherMesh1D] sendrecv failed (connectedRank=%d)", connectedRank),
217 : HcclResult::HCCL_E_INTERNAL);
218 0 : }
219 0 : return HcclResult::HCCL_SUCCESS;
220 0 : }
221 :
222 0 : HcclResult InsTempAllGatherMesh1D::RunMesh(
223 : const u32 myAlgRank, const std::vector<RankId> &vTopo, std::vector<InsQuePtr> &tempInsQues)
224 : {
225 0 : u32 queIdx = 0;
226 0 : for (u32 rankIdx = 0; rankIdx < vTopo.size() - 1; rankIdx++) {
227 0 : RankId connectedRank = vTopo[(myAlgRank + 1 + rankIdx) % vTopo.size()];
228 0 : auto it = tempLinks_.find(connectedRank);
229 0 : if (it == tempLinks_.end()) {
230 0 : HCCL_ERROR("[InsTempAllGatherMesh1D] connectedRank does not exist");
231 0 : return HcclResult::HCCL_E_PARA;
232 : }
233 0 : CHK_PRT_RET(tempLinks_.at(connectedRank).empty(),
234 : HCCL_ERROR(
235 : "[InsTempAllGatherMesh1D] connectedRank=%d, tempLinks_.size=%zu", connectedRank, tempLinks_.size()),
236 : HcclResult::HCCL_E_INTERNAL);
237 : // SingleRunMesh函数中会自动移动curInsQues队列下标
238 0 : CHK_PRT_RET(SingleRunMesh(myAlgRank, connectedRank, tempInsQues, queIdx),
239 : HCCL_ERROR("[SingleRunMesh] myAlgRank=%u, connectedRank=%d", myAlgRank, connectedRank),
240 : HcclResult::HCCL_E_INTERNAL);
241 : }
242 0 : return HcclResult::HCCL_SUCCESS;
243 : }
244 :
245 0 : void InsTempAllGatherMesh1D::PrepareLinkSlices(const u32 myAlgRank, const u32 connectedAlgRank,
246 : const std::vector<float> &dataSplitRate, const u32 linkIndex, std::vector<DataSlice> &txSrcSlices,
247 : std::vector<DataSlice> &txDstSlices, std::vector<DataSlice> &rxSrcSlices, std::vector<DataSlice> &rxDstSlices)
248 : {
249 : BufferType writeType
250 0 : = (opMode_ == OpMode::OPBASE) ? tempAlgParams_.buffInfo.scratBuffType : tempAlgParams_.buffInfo.inBuffType;
251 0 : for (u32 rpt = 0; rpt < tempAlgParams_.repeatNum; ++rpt) {
252 0 : const u64 inBaseOff = tempAlgParams_.buffInfo.inBuffBaseOff + rpt * tempAlgParams_.inputRepeatStride;
253 0 : const u64 outBaseOff = tempAlgParams_.buffInfo.outBuffBaseOff + rpt * tempAlgParams_.outputRepeatStride;
254 0 : const u64 scratchRepeatStride = tempAlgParams_.sliceSize * (tempRankSize_ - 1) + tempAlgParams_.tailSize;
255 0 : const u64 scratchBase = tempAlgParams_.buffInfo.scratchBuffBaseOff + rpt * scratchRepeatStride;
256 0 : u64 txInOffset = tempAlgParams_.inputSliceStride * myAlgRank + inBaseOff;
257 0 : u64 txOutOffset = tempAlgParams_.outputSliceStride * myAlgRank + outBaseOff;
258 0 : u64 txScratchOffset = scratchBase + tempAlgParams_.sliceSize * myAlgRank;
259 0 : u64 txDstOffset = (opMode_ == OpMode::OPBASE) ? txScratchOffset : txOutOffset;
260 0 : u64 rxInOffset = tempAlgParams_.inputSliceStride * connectedAlgRank + inBaseOff;
261 0 : u64 rxOutOffset = tempAlgParams_.outputSliceStride * connectedAlgRank + outBaseOff;
262 0 : u64 rxScratchOffset = scratchBase + tempAlgParams_.sliceSize * connectedAlgRank;
263 0 : u64 rxSrcOffset = (opMode_ == OpMode::OPBASE) ? rxScratchOffset : rxInOffset;
264 0 : u64 sendSliceSize = (myAlgRank == tempRankSize_ - 1) ? tempAlgParams_.tailSize : tempAlgParams_.sliceSize;
265 0 : u64 recvSliceSize
266 0 : = (connectedAlgRank == tempRankSize_ - 1) ? tempAlgParams_.tailSize : tempAlgParams_.sliceSize;
267 0 : DataSlice txSrcSlice(tempAlgParams_.buffInfo.inBuffType, txInOffset, sendSliceSize);
268 0 : DataSlice txDstSlice(writeType, txDstOffset, sendSliceSize);
269 0 : DataSlice rxSrcSlice(writeType, rxSrcOffset, recvSliceSize);
270 0 : DataSlice rxDstSlice(tempAlgParams_.buffInfo.outBuffType, rxOutOffset, recvSliceSize);
271 0 : txSrcSlices.push_back(CalcDataSliceForLinks(txSrcSlice, dataSplitRate, linkIndex, dataType_));
272 0 : txDstSlices.push_back(CalcDataSliceForLinks(txDstSlice, dataSplitRate, linkIndex, dataType_));
273 0 : rxSrcSlices.push_back(CalcDataSliceForLinks(rxSrcSlice, dataSplitRate, linkIndex, dataType_));
274 0 : rxDstSlices.push_back(CalcDataSliceForLinks(rxDstSlice, dataSplitRate, linkIndex, dataType_));
275 : }
276 0 : return;
277 : }
278 :
279 : } // namespace Hccl
|
