Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "log.h"
12 : #include "alg_data_trans_wrapper.h"
13 : #include "ins_temp_all_reduce_mesh_2D_two_shot.h"
14 :
15 : namespace Hccl {
16 0 : InsTempAllReduceMesh2DTwoShot::InsTempAllReduceMesh2DTwoShot(const RankId virtualRank, const u32 tempRankSize,
17 0 : const std::vector<std::vector<RankId>> &tempVTopo, const std::map<RankId, u32> &tempVirtRankMap)
18 0 : : InsAlgTemplateBase(virtualRank, tempRankSize, tempVTopo, tempVirtRankMap)
19 : {
20 0 : HCCL_INFO("[InsTempAllReduceMesh2DTwoShot] Init.");
21 0 : }
22 :
23 0 : InsTempAllReduceMesh2DTwoShot::~InsTempAllReduceMesh2DTwoShot()
24 : {
25 0 : HCCL_INFO("[InsTempAllReduceMesh2DTwoShot] exit.");
26 0 : }
27 :
28 : /*
29 : * Desc: 计算资源需求
30 : * return: tempResReq: 资源计算结果存储，包括notify信息，links信息等
31 : * return: HcclResult
32 : */
33 0 : HcclResult InsTempAllReduceMesh2DTwoShot::CalcRes(AlgTempResReq &tempResReq)
34 : {
35 : // 1D Mesh 需要的 que Num 为 ranksize
36 0 : tempResReq.queNum = tempVTopo_[0].size() + tempVTopo_[1].size();
37 0 : tempResReq.streamNum = tempResReq.queNum;
38 0 : tempResReq.queNotifys = CreateQueNotifiesRequest(tempResReq.queNum, 1, 0, tempVTopo_[0].size());
39 :
40 0 : QId centerQ = 0;
41 0 : tempResReq.localWaitGroupCntNotify.emplace_back(centerQ, 0);
42 0 : tempResReq.localBcastPostCntNotify.emplace_back(centerQ, 0);
43 :
44 : uint32_t myAlgRank;
45 0 : for (u32 dim = 0; dim < tempVTopo_.size(); dim++) {
46 0 : CHK_RET(GetAlgRank(myRank_, tempVTopo_[dim], myAlgRank));
47 0 : for (u32 queIdx = 0; queIdx < tempVTopo_[dim].size() - 1; queIdx++) {
48 0 : u32 neighborAlgRank = (myAlgRank + 1 + queIdx) % (tempVTopo_[dim].size());
49 0 : RankId neighborRank = tempVTopo_[dim][neighborAlgRank];
50 0 : HCCL_INFO("InsTempAllReduceMesh2DTwoShot::CalcRes Rank[%d], Dim[%u], NeighborRank[%d].", myRank_,
51 : dim, neighborRank);
52 : // LinkNum
53 0 : tempResReq.links[neighborRank] = 1;
54 : }
55 : }
56 0 : HCCL_INFO("InsTempAllReduceMesh2DTwoShot::CalcRes done");
57 0 : return HcclResult::HCCL_SUCCESS;
58 : }
59 :
60 0 : std::vector<std::tuple<QId, QId, u32>> InsTempAllReduceMesh2DTwoShot::CreateQueNotifiesRequest(
61 : u32 queueNum, u32 pairNum, QId masterIdX, QId masterIdY) const
62 : {
63 0 : std::vector<std::tuple<QId, QId, u32>> notifyRequests;
64 0 : HCCL_DEBUG("[Create][MasterSlaveQueNotifiesRequest] queueNum[%u], pairNum[%u]", queueNum, pairNum);
65 0 : if (queueNum == 0 || pairNum == 0) {
66 0 : HCCL_INFO("[Create][MasterSlaveQueNotifiesRequest] queueNum or pairNum is zero, "
67 : "return empty notifyRequests");
68 0 : return notifyRequests;
69 : };
70 :
71 0 : u32 slaveNum = queueNum - 1;
72 0 : HCCL_INFO("[Create][MasterSlaveQueNotifiesRequest] slavNum[%u]", slaveNum);
73 0 : if (slaveNum < 1 || pairNum < 1) {
74 0 : return notifyRequests;
75 : }
76 :
77 0 : notifyRequests.reserve((slaveNum + queueNum - masterIdY - 1) * pairNum);
78 : // masterX(master0)跟所有的stream有同步关系
79 0 : for (QId q = 0; q < queueNum; q++) {
80 0 : if (q == masterIdX) {
81 0 : continue;
82 : }
83 0 : for (u32 i = 0; i < pairNum; i++) {
84 0 : notifyRequests.emplace_back(std::make_tuple(masterIdX, q, i));
85 0 : notifyRequests.emplace_back(std::make_tuple(q, masterIdX, i));
86 : }
87 : }
88 :
89 0 : for (QId q = masterIdY + 1; q < queueNum; q++) {
90 0 : for (u32 i = 0; i < pairNum; i++) {
91 0 : notifyRequests.emplace_back(std::make_tuple(masterIdY, q, i));
92 0 : notifyRequests.emplace_back(std::make_tuple(q, masterIdY, i));
93 : }
94 : }
95 0 : return notifyRequests;
96 0 : }
97 :
98 : /*
99 : * Desc: 返回当前rank能处理的数据量和scratch buffer之间的比例关系
100 : * param: input: 输入数据位置
101 : * param: output 输出数据位置
102 : */
103 0 : u32 InsTempAllReduceMesh2DTwoShot::CalcScratchMultiple(BufferType input, BufferType output) const
104 : {
105 : // scratchbuffer如果能够通过ranksize规整：buffersize%2*ranksize_M*ranksize_N=0，则这里只需要返回1，最大化利用scratchbuffer
106 :
107 : // 否则返回2，使用1倍的buffer保证能缓存所有其他rank发来的数据，理论上数据被分成2*M*N块，假设有尾块，每个数据块的大小(inputcount/（2*M*N)+1）
108 : // 总共需要(inputCount/（2*M*N)+1）*（2*M*N）=[inputCount+2*M*N]*elembytesize,
109 : // 而预留的buffersize=inputcount*elembytesize，
110 : // 所以如果2*M*N>inputcount(预留)则缓存buffer仍然不够，但是由于最小的scratchbuffersize=1M，而2*M*N很难大于1M/2(一半数据一半缓存)，
111 : // 所以返回2的时候要判断(2*M*N+inputcount)*elembytesize>scratchbufferSize(即预留缓存buffer+输入数据占用的buffer)；2*M*N是常量，只需要增加buffersize解决
112 : (void)input;
113 : (void)output;
114 0 : u32 multiple = 2;
115 0 : return multiple;
116 : }
117 :
118 0 : HcclResult InsTempAllReduceMesh2DTwoShot::BuildSlice(
119 : const std::vector<RankId>& rankInfo, const u64 dataSize, const u64 chunkSize, RankSliceInfo &sliceInfoVec) const
120 : {
121 0 : std::vector<SliceInfo> tmp(1);
122 0 : sliceInfoVec.resize(rankInfo.size(), tmp);
123 :
124 0 : u64 accumOff = 0;
125 0 : for (u32 rankIdx = 0; rankIdx < rankInfo.size(); rankIdx++) {
126 0 : u64 currChunkSize = ((dataSize - accumOff) > chunkSize) ? chunkSize : (dataSize - accumOff);
127 0 : SliceInfo slice = {accumOff, currChunkSize};
128 0 : sliceInfoVec[rankIdx][0]=slice;
129 0 : accumOff += currChunkSize;
130 : }
131 0 : return HcclResult::HCCL_SUCCESS;
132 0 : }
133 :
134 : /*
135 : * Desc: GenExtIns 算子执行入口
136 : * param: tempAlgParams: slice和stride信息
137 : * param: tempFuncs: 辅助信息包括userIn/OutSlices, opMode等标记信息
138 : * param: tempLinks: 当前rank通信链接信息
139 : * param: tempInsQues: 通信队列
140 : * return: HcclResult
141 : */
142 0 : HcclResult InsTempAllReduceMesh2DTwoShot::GenExtIns(const TempFuncs &tempFuncs, const TemplateDataParams &tempAlgParams,
143 : const ResLinks &tempLinks, std::vector<InsQuePtr> &tempInsQues)
144 : {
145 0 : InitInnerParams(tempFuncs, tempAlgParams, tempLinks, tempInsQues);
146 : // step1: reducescatter, X轴划分为M个块，每个块大小N*chunksize, Y轴划分为N个块，每个块M*chucksize
147 0 : CHK_RET(PreSyncQues(tempInsQues, 0));
148 0 : CHK_RET(PostSyncQues(tempInsQues, 0));
149 0 : SubStageArgs bufferInfo = {tempAlgParams.buffInfo.inBuffType, tempAlgParams.buffInfo.scratBuffType,
150 0 : tempAlgParams.buffInfo.inBuffBaseOff, 0};
151 0 : CHK_RET(RunReduceScatter(bufferInfo, XsliceInfoVec_, tempLinks, XtempInsQues_, tempVTopo_[0]));
152 :
153 0 : if (YDataSize_ != 0) {
154 0 : bufferInfo = {tempAlgParams.buffInfo.inBuffType, tempAlgParams.buffInfo.scratBuffType,
155 0 : tempAlgParams.buffInfo.inBuffBaseOff + M_ * N_ * chunkSize_, M_ * N_ * chunkSize_};
156 0 : CHK_RET(RunReduceScatter(bufferInfo, YsliceInfoVec_, tempLinks, YtempInsQues_, tempVTopo_[1]));
157 : }
158 :
159 : // step2: 换轴reducescatter
160 0 : CHK_RET(PreSyncQues(tempInsQues, 0));
161 0 : CHK_RET(PostSyncQues(tempInsQues, 0));
162 0 : if (XDataSizeS2_ != 0) {
163 0 : bufferInfo = {tempAlgParams.buffInfo.scratBuffType, tempAlgParams.buffInfo.scratBuffType, M_ * N_ * chunkSize_,
164 0 : M_ * N_ * chunkSize_ + M_ * chunkSize_};
165 0 : CHK_RET(RunReduceScatter(bufferInfo, XsliceInfoVecS2_, tempLinks, XtempInsQues_, tempVTopo_[0]));
166 : }
167 0 : if (YDataSizeS2_ != 0) {
168 0 : bufferInfo = {tempAlgParams.buffInfo.scratBuffType, tempAlgParams.buffInfo.scratBuffType, 0, N_ * chunkSize_};
169 0 : CHK_RET(RunReduceScatter(bufferInfo, YsliceInfoVecS2_, tempLinks, YtempInsQues_, tempVTopo_[1]));
170 : }
171 :
172 : // step3: allgather
173 0 : CHK_RET(PreSyncQues(tempInsQues, 0));
174 0 : CHK_RET(PostSyncQues(tempInsQues, 0));
175 0 : if (XDataSizeS2_ != 0) { // X轴allgather
176 0 : bufferInfo = {tempAlgParams.buffInfo.scratBuffType, tempAlgParams.buffInfo.scratBuffType,
177 0 : M_ * N_ * chunkSize_ + M_ * chunkSize_, M_ * N_ * chunkSize_};
178 0 : CHK_RET(RunAllgather(bufferInfo, XsliceInfoVecS2_, tempLinks, XtempInsQues_, tempVTopo_[0]));
179 : }
180 0 : if (YDataSizeS2_ != 0) { // Y轴allgather
181 0 : bufferInfo = {tempAlgParams.buffInfo.scratBuffType, tempAlgParams.buffInfo.scratBuffType, N_ * chunkSize_, 0};
182 0 : CHK_RET(RunAllgather(bufferInfo, YsliceInfoVecS2_, tempLinks, YtempInsQues_, tempVTopo_[1]));
183 : }
184 :
185 : // step4: 换轴allgather
186 0 : CHK_RET(PreSyncQues(tempInsQues, 0));
187 0 : CHK_RET(PostSyncQues(tempInsQues, 0));
188 0 : bufferInfo = {tempAlgParams.buffInfo.scratBuffType, tempAlgParams.buffInfo.outBuffType, 0,
189 0 : tempAlgParams.buffInfo.outBuffBaseOff};
190 0 : CHK_RET(RunAllgather(bufferInfo, XsliceInfoVec_, tempLinks, XtempInsQues_, tempVTopo_[0]));
191 0 : if (YDataSize_ != 0) {
192 0 : bufferInfo = {tempAlgParams.buffInfo.scratBuffType, tempAlgParams.buffInfo.outBuffType, M_ * N_ * chunkSize_,
193 0 : tempAlgParams.buffInfo.outBuffBaseOff + M_ * N_ * chunkSize_};
194 0 : CHK_RET(RunAllgather(bufferInfo, YsliceInfoVec_, tempLinks, YtempInsQues_, tempVTopo_[1]));
195 : }
196 0 : CHK_RET(PreSyncQues(tempInsQues, 0));
197 0 : CHK_RET(PostSyncQues(tempInsQues, 0));
198 0 : return HcclResult::HCCL_SUCCESS;
199 : }
200 :
201 0 : HcclResult InsTempAllReduceMesh2DTwoShot::InitInnerParams(const TempFuncs &tempFuncs,
202 : const TemplateDataParams &tempAlgParams, const ResLinks &tempLinks, std::vector<InsQuePtr> &tempInsQues)
203 : {
204 : (void)tempLinks;
205 0 : HCCL_INFO("[InsTempAllReduceMesh2DTwoShot] start.");
206 0 : opMode_ = tempFuncs.opMode;
207 0 : enableCounterNotify_ = tempFuncs.enableCounterNotify;
208 :
209 0 : M_ = tempVTopo_[0].size();
210 0 : N_ = tempVTopo_[1].size();
211 0 : queNum_ = M_ + N_;
212 0 : CHK_PRT_RET(queNum_ != tempInsQues.size(),
213 : HCCL_ERROR("[InsTempAllReduceMesh2DTwoShot] Rank [%d], queNum_:[%u], tempInsQues size:[%u],requiredQue Error.",
214 : myRank_,
215 : queNum_,
216 : tempInsQues.size()),
217 : HcclResult::HCCL_E_INTERNAL);
218 :
219 0 : u32 dataSizePerVolume = DataTypeSizeGet(dataType_); // 均分为2MN块
220 0 : u32 times = 2;
221 0 : chunkSize_ = RoundUp(tempAlgParams.sliceSize, (M_ * N_ * times * dataSizePerVolume)) * dataSizePerVolume;
222 0 : CHK_PRT_RET((chunkSize_ * M_ * N_ * times) > tempAlgParams.buffInfo.scratchBuffSize,
223 : HCCL_ERROR("[InsTempAllReduceMesh2DTwoShot]Rank [%d], Input size:[%llu], BfSize:[%llu] Insufficient buffer!",
224 : myRank_,
225 : tempAlgParams.sliceSize,
226 : tempAlgParams.buffInfo.scratchBuffSize),
227 : HcclResult::HCCL_E_INTERNAL);
228 0 : XtempInsQues_ = std::vector<InsQuePtr>(tempInsQues.begin(), tempInsQues.begin() + M_);
229 0 : YtempInsQues_ = std::vector<InsQuePtr>(tempInsQues.begin() + M_, tempInsQues.end());
230 :
231 0 : CHK_RET(GetAlgRank(myRank_, tempVTopo_[0], XAlgrankId_));
232 0 : CHK_RET(GetAlgRank(myRank_, tempVTopo_[1], YAlgrankId_));
233 :
234 : // for step1
235 0 : XDataSize_ = tempAlgParams.sliceSize >= M_ * N_ * chunkSize_ ? M_ * N_ * chunkSize_ : tempAlgParams.sliceSize;
236 0 : YDataSize_ = tempAlgParams.sliceSize - XDataSize_;
237 0 : BuildSlice(tempVTopo_[0], XDataSize_, N_ * chunkSize_, XsliceInfoVec_);
238 0 : BuildSlice(tempVTopo_[1], YDataSize_, M_ * chunkSize_, YsliceInfoVec_);
239 :
240 : // for step2
241 0 : YDataSizeS2_ = XsliceInfoVec_[XAlgrankId_][0].size; // 找到前一步切分时本rank负责的数据块大小
242 0 : XDataSizeS2_ = YsliceInfoVec_[YAlgrankId_][0].size;
243 0 : BuildSlice(tempVTopo_[1], YDataSizeS2_, chunkSize_, YsliceInfoVecS2_);
244 0 : BuildSlice(tempVTopo_[0], XDataSizeS2_, chunkSize_, XsliceInfoVecS2_);
245 0 : return HcclResult::HCCL_SUCCESS;
246 : }
247 :
248 : /*
249 : * Desc: 2D Mesh twoshot AllReduce: Scatter+reduce
250 : * param: sliceInfoVec: 每个rank的数据切片信息
251 : * param: tempLinks: 当前rank通信链接信息
252 : * param: tempInsQues: 通信队列
253 : * param: tempFuncs: 辅助信息包括userIn/OutSlices, opMode等标记信息
254 : * return: HcclResult
255 : */
256 0 : HcclResult InsTempAllReduceMesh2DTwoShot::RunReduceScatter(SubStageArgs& subparams,
257 : const RankSliceInfo &sliceInfoVec, const ResLinks &tempLinks, std::vector<InsQuePtr> &tempInsQues,
258 : const std::vector<RankId> &rankInfo) const
259 : {
260 : u32 myAlgrankId;
261 0 : CHK_RET(GetAlgRank(myRank_, rankInfo, myAlgrankId));
262 :
263 0 : CHK_RET(PreSyncQues(tempInsQues, 0));
264 : // scatter
265 0 : for (u32 rankId = 0; rankId < rankInfo.size(); rankId++) {// 写模式
266 : DataSlice ssrcSlice = DataSlice(subparams.inType,
267 0 : sliceInfoVec[rankId][0].offset + subparams.inbaseOff, sliceInfoVec[rankId][0].size);
268 : DataSlice sdestSlice = DataSlice(subparams.outType,
269 0 : myAlgrankId * sliceInfoVec[rankId][0].size + subparams.outbaesOff, sliceInfoVec[rankId][0].size);
270 0 : if (rankId == myAlgrankId) {
271 0 : if (sliceInfoVec[rankId][0].size != 0) {// 如果是本地rank，直接拷贝到scratch对应位置
272 0 : CHK_PRT_RET(LocalCopy(tempInsQues[rankId], ssrcSlice, sdestSlice),
273 : HCCL_ERROR(
274 : "[InsTempAllReduceMesh2DTwoShot][RunReduceScatter] RunAllReduce scatter LocalCopy failed"),
275 : HcclResult::HCCL_E_INTERNAL);
276 : }
277 : } else {
278 0 : const std::vector<LinkData> &linkSendRecv = tempLinks.at(rankInfo[rankId]);
279 : // 发送, 未过滤size为0的情况
280 0 : std::vector<DataSlice> sendSrcSlices{ssrcSlice};
281 0 : std::vector<DataSlice> sendDestSlices{sdestSlice};
282 : // 接收，未过滤size为0的情况
283 : DataSlice rsrcSlice = DataSlice(subparams.inType,
284 0 : sliceInfoVec[myAlgrankId][0].offset + subparams.inbaseOff, sliceInfoVec[myAlgrankId][0].size);
285 : DataSlice rdestSlice = DataSlice(subparams.outType,
286 0 : rankId * sliceInfoVec[myAlgrankId][0].size + subparams.outbaesOff, sliceInfoVec[myAlgrankId][0].size);
287 0 : std::vector<DataSlice> recvSrcSlices{rsrcSlice};
288 0 : std::vector<DataSlice> recvDestSlices{rdestSlice};
289 0 : TxRxLinks sendRecvLinks(linkSendRecv[0], linkSendRecv[0]);
290 0 : TxRxSlicesList sendRecvSlicesList({sendSrcSlices, sendDestSlices}, {recvSrcSlices, recvDestSlices});
291 0 : SendRecvInfo sendRecvInfo(sendRecvLinks, sendRecvSlicesList);
292 0 : CHK_PRT_RET(SendRecv(sendRecvInfo, tempInsQues[rankId], 0, true, DmaMode::PUT),
293 : HCCL_ERROR("[InsTempAllReduceMesh2DTwoShot][RunReduceScatter] RunAllReduce scatter failed"),
294 : HcclResult::HCCL_E_INTERNAL);
295 0 : }
296 : }
297 0 : CHK_RET(PostSyncQues(tempInsQues, 0)); // 从流同步,等待所有并发的send和copy完成
298 0 : if (sliceInfoVec[myAlgrankId][0].size != 0) { // local reduce, 计算结果都放在最开始的位置
299 0 : DataSlice ldestSlice = DataSlice(subparams.outType, subparams.outbaesOff, sliceInfoVec[myAlgrankId][0].size);
300 0 : for (u32 rankId = 1; rankId < rankInfo.size(); rankId++) {
301 : DataSlice lsrcSlice = DataSlice(subparams.outType,
302 0 : rankId * sliceInfoVec[myAlgrankId][0].size + subparams.outbaesOff, sliceInfoVec[myAlgrankId][0].size);
303 : // 所有reduce操作在同一个insque中才能保序；
304 0 : CHK_PRT_RET(LocalReduce(tempInsQues[0], lsrcSlice, ldestSlice, dataType_, redOp_),
305 : HCCL_ERROR("[InsTempAllReduceMesh2DTwoShot]LocalReduce failed"), HcclResult::HCCL_E_INTERNAL);
306 : }
307 : }
308 0 : return HcclResult::HCCL_SUCCESS;
309 : }
310 :
311 : /*
312 : * Desc: 2D Mesh twoshot AllReduce: Allgather
313 : * param: sliceInfoVec: 每个rank的数据切片信息
314 : * param: tempLinks: 当前rank通信链接信息
315 : * param: tempInsQues: 通信队列
316 : * param: tempFuncs: 辅助信息包括userIn/OutSlices, opMode等标记信息
317 : * return: HcclResult
318 : */
319 0 : HcclResult InsTempAllReduceMesh2DTwoShot::RunAllgather(SubStageArgs& subparams,
320 : const RankSliceInfo &sliceInfoVec, const ResLinks &tempLinks, std::vector<InsQuePtr> &tempInsQues,
321 : const std::vector<RankId> &rankInfo) const
322 : {
323 : u32 myAlgrankId;
324 0 : CHK_RET(GetAlgRank(myRank_, rankInfo, myAlgrankId));
325 :
326 : // sync:前同步
327 0 : CHK_RET(PreSyncQues(tempInsQues, 0));
328 :
329 : // allgather
330 0 : for (u32 rankId = 0; rankId < rankInfo.size(); rankId++) {
331 0 : DataSlice rsrcSlice = DataSlice(subparams.inType, subparams.inbaseOff, sliceInfoVec[rankId][0].size);
332 : DataSlice rdestSlice = DataSlice(
333 0 : subparams.outType, sliceInfoVec[rankId][0].offset + subparams.outbaesOff, sliceInfoVec[rankId][0].size);
334 0 : if (u32(myAlgrankId) == rankId) {
335 0 : if (sliceInfoVec[rankId][0].size != 0) {
336 : // copy本端计算的结果到user output
337 0 : CHK_PRT_RET(LocalCopy(tempInsQues[rankId], rsrcSlice, rdestSlice),
338 : HCCL_ERROR("[InsTempAllReduceMesh2DTwoShot][RunAllgather] RunAllReduce AllGather "
339 : "LocalCopy failed"),
340 : HcclResult::HCCL_E_INTERNAL);
341 : }
342 : } else {
343 0 : const std::vector<LinkData> &linkSendRecv = tempLinks.at(rankInfo[rankId]);
344 : // 接收, 未过滤size为0的情况
345 0 : std::vector<DataSlice> recvSrcSlices{rsrcSlice};
346 0 : std::vector<DataSlice> recvDestSlices{rdestSlice};
347 :
348 : // 发送，未过滤size为0的情况
349 0 : DataSlice ssrcSlice = DataSlice(subparams.inType, subparams.inbaseOff, sliceInfoVec[myAlgrankId][0].size);
350 : DataSlice sdestSlice = DataSlice(subparams.outType,
351 0 : sliceInfoVec[myAlgrankId][0].offset + subparams.outbaesOff,
352 0 : sliceInfoVec[myAlgrankId][0].size);
353 0 : std::vector<DataSlice> sendSrcSlices{ssrcSlice};
354 0 : std::vector<DataSlice> sendDestSlices{sdestSlice};
355 :
356 0 : TxRxLinks sendRecvLinks(linkSendRecv[0], linkSendRecv[0]);
357 0 : TxRxSlicesList sendRecvSlicesList({sendSrcSlices, sendDestSlices}, {recvSrcSlices, recvDestSlices});
358 :
359 0 : SendRecvInfo sendRecvInfo(sendRecvLinks, sendRecvSlicesList);
360 0 : CHK_PRT_RET(SendRecv(sendRecvInfo, tempInsQues[rankId], 0, true, DmaMode::GET),
361 : HCCL_ERROR("[InsTempAllReduceMesh2DTwoShot][RunAllgather] RunAllReduce AllGather failed"),
362 : HcclResult::HCCL_E_INTERNAL);
363 0 : }
364 : }
365 0 : CHK_RET(PostSyncQues(tempInsQues, 0));
366 0 : return HcclResult::HCCL_SUCCESS;
367 : }
368 :
369 : } // namespace Hccl
|
