Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "ccu_context_all_reduce_mesh1d_mem2mem.h"
12 : #include "ccu_instruction_all_reduce_mesh1d_mem2mem.h"
13 : #include "ccu_assist.h"
14 : namespace Hccl {
15 : constexpr int INPUT_XN_ID = 0;
16 : constexpr int OUTPUT_XN_ID = 1;
17 : constexpr int SCRATCH_XN_ID = 2;
18 : constexpr int TOKEN_XN_ID = 3;
19 : constexpr int CKE_IDX_0 = 0;
20 : constexpr int CKE_IDX_1 = 1;
21 : constexpr int CKE_IDX_2 = 2;
22 : constexpr int CKE_IDX_3 = 3;
23 :
24 : using CurrentCtxArg = CcuCtxArgAllReduceMeshMem2Mem1D;
25 : using CurrentTaskArg = CcuTaskArgAllReduceMeshMem2Mem1D;
26 :
27 0 : CcuContextAllReduceMeshMem2Mem1D::CcuContextAllReduceMeshMem2Mem1D(
28 0 : const CcuCtxArg &arg, const std::vector<CcuTransport *> &transports, const CcuTransportGroup &group)
29 0 : : CcuContextAlgBase(arg, transports, group)
30 : {
31 0 : HCCL_DEBUG("[CcuContextAllReduceMeshMem2Mem1D] Enter Constructor.");
32 0 : const CurrentCtxArg *ctxArg = dynamic_cast<const CurrentCtxArg *>(&arg);
33 0 : if (ctxArg == nullptr) {
34 0 : THROW<NullPtrException>(StringFormat("CcuContextAllReduceMeshMem2Mem1D::ctxArg ptr is null"));
35 : }
36 0 : dataType_ = ctxArg->op_.dataType;
37 0 : outputDataType_ = ctxArg->op_.outputDataType;
38 0 : rankId_ = ctxArg->rankId_;
39 0 : rankSize_ = ctxArg->dimSize_[0];
40 0 : reduceOp_ = ctxArg->op_.reduceOp;
41 0 : if (outputDataType_ == DataType::INVALID) {
42 0 : outputDataType_ = dataType_;
43 0 : HCCL_INFO("[CcuContextAllReduceMeshMem2Mem1D] outputDataType is [INVALID], set outputDataType to[%s]",
44 : outputDataType_.Describe().c_str());
45 : }
46 0 : CHK_PRT_THROW(
47 : ctxArg->dimSize_[0] == 0,
48 : HCCL_ERROR("[CcuContextAllReduceMeshMem2Mem1D] ctxArg->dimSize_[0] is zero"),
49 : InvalidParamsException, "ctxArg->dimSize_[0] is invalid");
50 0 : if (ctxArg->dimSize_.size() > 0) {
51 0 : rankSize_ = ctxArg->dimSize_[0];
52 : }
53 0 : HCCL_INFO("[CcuContextAllReduceMeshMem2Mem1D] Init, CtxArgs are rankId[%u], rankSize_[%u], dataType[%s], "
54 : "outputDataType[%s], reduceOp[%s]",
55 : rankId_, rankSize_, dataType_.Describe().c_str(), outputDataType_.Describe().c_str(),
56 : reduceOp_.Describe().c_str());
57 0 : }
58 :
59 0 : void CcuContextAllReduceMeshMem2Mem1D::InitResource()
60 : {
61 0 : if (transports.size() == 0) {
62 0 : THROW<NullPtrException>(StringFormat("CcuContextAllReduceMeshMem2Mem1D transports is empty"));
63 : }
64 0 : HCCL_INFO("[CcuContextAllReduceMeshMem2Mem1D]transports.size: [%u]", transports.size());
65 0 : uint16_t transportIdx = 0;
66 : // 按照rank号从小到大遍历transports，遇到本rank就填充本地资源，否则依次取远端资源，要求给框架返回的Link同样是按顺序排列的
67 0 : for (uint64_t peerId = 0; peerId < rankSize_; peerId++) {
68 0 : if (peerId == rankId_) {
69 0 : input_.push_back(CreateVariable());
70 0 : output_.push_back(CreateVariable());
71 0 : scratch_.push_back(CreateVariable());
72 0 : token_.push_back(CreateVariable());
73 : } else {
74 0 : HCCL_DEBUG("[CcuContextAllReduceMeshMem2Mem1D] MyRank[%u], PeerId[%llu], TransportId[%u]",
75 : rankId_, peerId, transportIdx);
76 0 : CHK_PRT_THROW(
77 : transports[transportIdx] == nullptr,
78 : HCCL_ERROR("[CcuContextAllReduceMeshMem2Mem1D] [InitResource] transports[%u] is nullptr",
79 : transportIdx),
80 : NullPtrException, "transport is null");
81 0 : input_.push_back(CreateVariable((*transports[transportIdx]), INPUT_XN_ID));
82 0 : output_.push_back(CreateVariable((*transports[transportIdx]), OUTPUT_XN_ID));
83 0 : scratch_.push_back(CreateVariable((*transports[transportIdx]), SCRATCH_XN_ID));
84 0 : token_.push_back(CreateVariable((*transports[transportIdx]), TOKEN_XN_ID));
85 0 : transportIdx++;
86 : }
87 : }
88 0 : currentRankSliceInputOffset_ = CreateVariable();
89 0 : currentRankSliceOutputOffset_ = CreateVariable();
90 0 : normalSliceSize_ = CreateVariable();
91 0 : lastSliceSize_ = CreateVariable();
92 0 : mySliceSize_ = CreateVariable();
93 0 : sliceOffset_ = CreateVariable();
94 0 : isInputOutputEqual_ = CreateVariable();
95 0 : locMask_ = CreateMaskSignal();
96 0 : srcMem_ = CreateMemory();
97 0 : dstMem_ = CreateMemory();
98 0 : reduceScatterSrc_.reserve(rankSize_);
99 0 : for (uint32_t rankIdx = 0; rankIdx < rankSize_; rankIdx++) {
100 0 : reduceScatterSrc_.push_back(CreateMemory());
101 : }
102 0 : reduceScatterDst_.reserve(rankSize_);
103 0 : for (uint32_t rankIdx = 0; rankIdx < rankSize_; rankIdx++) {
104 0 : reduceScatterDst_.push_back(CreateMemory());
105 : }
106 0 : sliceSize_ = CreateVariable();
107 0 : selfBit_ = 1 << rankId_;
108 0 : allBit_ = ((1 << rankSize_) - 1) & (~(1 << rankId_)); // rankId_位为0，其他位都为1
109 0 : localGoSize_ = CreateGroupOpSize();
110 0 : return;
111 : }
112 :
113 0 : std::string CcuContextAllReduceMeshMem2Mem1D::GetLoopBlockTag(std::string loopType, int32_t index)
114 : {
115 0 : return loopType + LOOP_BLOCK_TAG + std::to_string(index);
116 : }
117 :
118 0 : void CcuContextAllReduceMeshMem2Mem1D::CreateReduceLoop(uint32_t size, DataType dataType, DataType outputDataType,
119 : ReduceOp opType)
120 : {
121 0 : constexpr uint32_t LOOP_NUM = 16;
122 0 : AllocGoResource(LOOP_NUM);
123 :
124 0 : std::string loopType = CcuRep::GetReduceTypeStr(dataType, opType);
125 0 : if (registeredLoop.find(loopType) != registeredLoop.end()) {
126 0 : return;
127 : }
128 :
129 0 : uint32_t expansionNum = CcuRep::GetReduceExpansionNum(opType, dataType, outputDataType);
130 0 : uint32_t usedBufNum = size > expansionNum ? size : expansionNum;
131 :
132 0 : for (int32_t index = 0; index < 2; index++) { // 需要实例化2个Loop
133 0 : CcuRep::Memory src = CreateMemory();
134 0 : CcuRep::Memory dst = CreateMemory();
135 0 : std::vector<CcuRep::Memory> scratch;
136 0 : for (uint32_t i = 0; i < size; i++) {
137 0 : scratch.emplace_back(CreateMemory());
138 : }
139 0 : CcuRep::Variable lenForExpansion = CreateVariable();
140 0 : CcuRep::Variable len = CreateVariable();
141 0 : CcuRep::LoopBlock lb(this, GetLoopBlockTag(loopType, index));
142 0 : lb(dst, src, scratch, len, lenForExpansion);
143 :
144 0 : std::vector<CcuRep::CcuBuffer> bufs = {moRes.ccuBuffer.begin() + index * moConfig.msInterleave,
145 0 : moRes.ccuBuffer.begin() + index * moConfig.msInterleave + usedBufNum};
146 0 : CcuRep::MaskSignal sem = moRes.maskSignal[index];
147 :
148 0 : for (uint32_t i = 0; i < size; i++) {
149 0 : if (i == rankId_) {
150 0 : LocalCopy(bufs[i], src, len, sem, 1 << i);
151 : } else {
152 0 : LocalCopy(bufs[i], scratch[i], len, sem, 1 << i);
153 : }
154 : }
155 0 : LocalWait(sem, (1 << size) - 1);
156 :
157 0 : if (size > 1) {
158 0 : LocalReduce(bufs, size, dataType, outputDataType, opType, sem, len);
159 0 : LocalWait(sem);
160 : }
161 :
162 0 : LocalCopy(dst, bufs[0], lenForExpansion, sem);
163 0 : LocalWait(sem);
164 0 : }
165 :
166 0 : registeredLoop.insert(loopType);
167 0 : }
168 :
169 0 : void CcuContextAllReduceMeshMem2Mem1D::ReduceLoopGroup(CcuRep::Memory outDstOrg, CcuRep::Memory srcOrg,
170 : std::vector<CcuRep::Memory> &scratchOrg, GroupOpSize goSize, DataType dataType, DataType outputDataType,
171 : ReduceOp opType)
172 : {
173 0 : const uint32_t size = scratchOrg.size();
174 :
175 0 : CcuRep::Memory dst = CreateMemory();
176 0 : CcuRep::Memory src = CreateMemory();
177 0 : std::vector<CcuRep::Memory> scratch;
178 0 : dst = outDstOrg;
179 0 : src = srcOrg;
180 0 : for (uint32_t idx = 0; idx < size; idx++) {
181 0 : scratch.push_back(CreateMemory());
182 0 : scratch[idx] = scratchOrg[idx];
183 : }
184 :
185 0 : CreateReduceLoop(size, dataType, outputDataType, opType);
186 :
187 0 : std::string loopType = CcuRep::GetReduceTypeStr(dataType, opType);
188 0 : uint32_t expansionNum = CcuRep::GetReduceExpansionNum(opType, dataType, outputDataType);
189 0 : CcuRep::Variable sliceSizeExpansion = CreateVariable();
190 :
191 0 : if (expansionNum != 1) {
192 0 : CcuRep::Variable tmp = CreateVariable();
193 0 : tmp = CcuRep::GetExpansionParam(expansionNum);
194 0 : dst.token += tmp;
195 0 : }
196 :
197 : // m部分
198 0 : CCU_IF(goSize.loopParam != 0) // goSize1
199 : {
200 0 : CcuRep::Variable loopParam = CreateVariable();
201 0 : CcuRep::Variable sliceSize = CreateVariable();
202 0 : loopParam = CcuRep::GetLoopParam(0, moConfig.memSlice * moConfig.loopCount, 0);
203 0 : loopParam += goSize.loopParam;
204 0 : sliceSizeExpansion = moConfig.memSlice * expansionNum;
205 0 : sliceSize = moConfig.memSlice;
206 :
207 0 : auto lc = Loop(GetLoopBlockTag(loopType, 0))(dst, src, scratch, sliceSize, sliceSizeExpansion);
208 :
209 0 : CcuRep::Variable paraCfg = CreateVariable();
210 0 : CcuRep::Variable offsetCfg = CreateVariable();
211 0 : paraCfg = CcuRep::GetParallelParam(moConfig.loopCount - 1, 0, 1);
212 0 : offsetCfg = CcuRep::GetOffsetParam(moConfig.memSlice, moConfig.msInterleave, 1);
213 :
214 0 : LoopGroup({lc}, {loopParam}, paraCfg, offsetCfg);
215 0 : }
216 :
217 0 : CCU_IF(goSize.parallelParam != 0) // goSize2
218 : {
219 : // p部分，加m的偏移
220 0 : for (uint32_t i = 0; i < size; i++) {
221 0 : scratch[i].addr += goSize.addrOffset;
222 : }
223 0 : src.addr += goSize.addrOffset; // goSize0
224 0 : for (uint32_t i = 0; i < expansionNum; i++) {
225 0 : dst.addr += goSize.addrOffset;
226 : }
227 :
228 0 : sliceSizeExpansion = 0;
229 0 : for (uint32_t i = 0; i < expansionNum; i++) {
230 0 : sliceSizeExpansion += goSize.residual; // goSize3
231 : }
232 :
233 0 : auto lc0 = Loop(GetLoopBlockTag(loopType, 0))(dst, src, scratch, goSize.residual, sliceSizeExpansion);
234 :
235 : // n部分，再加p的偏移
236 0 : for (uint32_t i = 0; i < size; i++) {
237 0 : scratch[i].addr += goSize.residual;
238 : }
239 0 : src.addr += goSize.residual;
240 0 : for (uint32_t i = 0; i < expansionNum; i++) {
241 0 : dst.addr += goSize.residual;
242 : }
243 :
244 0 : CcuRep::Variable sliceSize = CreateVariable();
245 0 : sliceSize = moConfig.memSlice;
246 0 : sliceSizeExpansion = moConfig.memSlice * expansionNum;
247 :
248 0 : auto lc1 = Loop(GetLoopBlockTag(loopType, 1))(dst, src, scratch, sliceSize, sliceSizeExpansion);
249 :
250 0 : CcuRep::Variable loopCfg0 = CreateVariable();
251 0 : CcuRep::Variable loopCfg1 = CreateVariable();
252 0 : CcuRep::Variable offsetCfg = CreateVariable();
253 0 : loopCfg0 = CcuRep::GetLoopParam(0, 0, 1);
254 0 : loopCfg1 = CcuRep::GetLoopParam(0, 0, 1);
255 0 : offsetCfg = CcuRep::GetOffsetParam(moConfig.memSlice, moConfig.msInterleave, 1);
256 :
257 0 : LoopGroup({lc0, lc1}, {loopCfg0, loopCfg1}, goSize.parallelParam, offsetCfg);
258 0 : }
259 0 : }
260 :
261 0 : void CcuContextAllReduceMeshMem2Mem1D::LoadArgs()
262 : {
263 0 : Load(input_[rankId_]);
264 0 : Load(output_[rankId_]);
265 0 : Load(token_[rankId_]);
266 0 : Load(scratch_[rankId_]);
267 0 : Load(currentRankSliceInputOffset_);
268 0 : Load(currentRankSliceOutputOffset_);
269 0 : Load(normalSliceSize_);
270 0 : Load(lastSliceSize_);
271 0 : Load(mySliceSize_);
272 0 : Load(sliceOffset_);
273 0 : Load(isInputOutputEqual_);
274 0 : Load(localGoSize_);
275 0 : return;
276 : }
277 :
278 0 : void CcuContextAllReduceMeshMem2Mem1D::PreSync()
279 : {
280 : // 互换内存信息
281 0 : for (auto t : transports) {
282 0 : HCCL_INFO("[CcuContextAllReduceMeshMem2Mem1D] AllReduceMeshMem2Mem1D LocalPost begin");
283 : // 交换起始地址
284 0 : WriteVariableWithSignal(*t, input_[rankId_], INPUT_XN_ID, CKE_IDX_1, selfBit_);
285 0 : WriteVariableWithSignal(*t, output_[rankId_], OUTPUT_XN_ID, CKE_IDX_2, selfBit_);
286 0 : WriteVariableWithSignal(*t, token_[rankId_], TOKEN_XN_ID, CKE_IDX_3, selfBit_);
287 : }
288 0 : GroupWait(*transportGroup, CKE_IDX_1, allBit_);
289 0 : GroupWait(*transportGroup, CKE_IDX_2, allBit_);
290 0 : GroupWait(*transportGroup, CKE_IDX_3, allBit_);
291 0 : HCCL_INFO("[CcuContextAllReduceMeshMem2Mem1D] AllReduceMeshMem2Mem1D wait all end");
292 0 : return;
293 : }
294 :
295 0 : void CcuContextAllReduceMeshMem2Mem1D::PostSync()
296 : {
297 0 : for (auto &t : transports) {
298 0 : RemotePost(*t, CKE_IDX_0, selfBit_);
299 : }
300 0 : GroupWait(*transportGroup, CKE_IDX_0, allBit_);
301 0 : HCCL_INFO("[CcuContextAllReduceMeshMem2Mem1D] AllReduceMeshMem2Mem1D AllReduce groupwait end");
302 0 : }
303 :
304 0 : void CcuContextAllReduceMeshMem2Mem1D::BcastLocToRmt(const CcuRep::Variable &srcAddr,
305 : const std::vector<CcuRep::Variable> &dstAddr)
306 : {
307 0 : CHK_PRT_THROW(
308 : dstAddr.size() != transports.size() + 1,
309 : HCCL_ERROR("[ReduceRmtToLoc] srcAddr.size[%zu] != transports size[%zu] + 1", dstAddr.size(), transports.size()),
310 : InvalidParamsException, "Invalid srcAddr size");
311 :
312 0 : srcMem_.addr = srcAddr;
313 0 : srcMem_.addr += sliceOffset_;
314 0 : srcMem_.token = token_[rankId_];
315 :
316 0 : uint32_t transportIdx = 0;
317 0 : for (uint32_t rmtId = 0; rmtId < dstAddr.size(); rmtId++) {
318 0 : if (rmtId == rankId_) {
319 0 : continue;
320 : }
321 0 : dstMem_.addr = dstAddr[rmtId];
322 0 : dstMem_.addr += sliceOffset_;
323 0 : dstMem_.token = token_[rmtId];
324 :
325 0 : Write(*transports[transportIdx], dstMem_, srcMem_, sliceSize_, locMask_, 1 << rmtId);
326 0 : transportIdx++;
327 : }
328 0 : LocalWait(locMask_, allBit_);
329 0 : }
330 :
331 0 : void CcuContextAllReduceMeshMem2Mem1D::ReduceRmtToLoc(const std::vector<CcuRep::Variable> &srcAddr,
332 : const CcuRep::Variable &dstAddr)
333 : {
334 0 : CHK_PRT_THROW(
335 : srcAddr.size() != transports.size() + 1,
336 : HCCL_ERROR("[ReduceRmtToLoc] srcAddr.size[%zu] != transports size[%zu] +1", srcAddr.size(), transports.size()),
337 : InvalidParamsException, "Invalid srcAddr size");
338 :
339 0 : dstMem_.addr = dstAddr;
340 0 : dstMem_.addr += sliceOffset_;
341 0 : dstMem_.token = token_[rankId_];
342 :
343 0 : CcuRep::Variable scratchOffset = CreateVariable();
344 0 : scratchOffset = 0;
345 0 : for (uint32_t rankIdx = 0; rankIdx < rankSize_; rankIdx++) {
346 0 : reduceScatterSrc_[rankIdx].addr = srcAddr[rankIdx];
347 0 : reduceScatterSrc_[rankIdx].addr += sliceOffset_;
348 0 : reduceScatterSrc_[rankIdx].token = token_[rankIdx];
349 :
350 0 : reduceScatterDst_[rankIdx].addr = scratch_[rankId_];
351 0 : reduceScatterDst_[rankIdx].addr += scratchOffset;
352 0 : scratchOffset += normalSliceSize_;
353 0 : reduceScatterDst_[rankIdx].token = token_[rankId_];
354 : }
355 :
356 0 : uint32_t transportId = 0;
357 0 : for (uint32_t rankIdx = 0; rankIdx < rankSize_; rankIdx++) {
358 0 : if (rankIdx == rankId_) {
359 0 : LocalPost(locMask_, 1 << rankIdx);
360 : } else {
361 0 : Read(*transports[transportId], reduceScatterDst_[rankIdx], reduceScatterSrc_[rankIdx], sliceSize_, locMask_,
362 0 : 1 << rankIdx);
363 0 : transportId++;
364 : }
365 : }
366 0 : LocalWait(locMask_, (1 << rankSize_) - 1);
367 0 : ReduceLoopGroup(dstMem_, reduceScatterSrc_[rankId_], reduceScatterDst_, localGoSize_, dataType_, outputDataType_, reduceOp_);
368 0 : }
369 :
370 0 : void CcuContextAllReduceMeshMem2Mem1D::DoRepeatAllReduce()
371 : {
372 0 : if (rankId_ != rankSize_ - 1) {
373 0 : sliceSize_ = normalSliceSize_;
374 : } else {
375 0 : sliceSize_ = lastSliceSize_;
376 : }
377 0 : ReduceRmtToLoc(input_, output_[rankId_]);
378 0 : BcastLocToRmt(output_[rankId_], output_);
379 0 : }
380 :
381 0 : void CcuContextAllReduceMeshMem2Mem1D::Algorithm()
382 : {
383 0 : HCCL_INFO("[CcuContextAllReduceMeshMem2Mem1D] AllReduceMeshMem2Mem1D run");
384 0 : InitResource();
385 0 : LoadArgs();
386 0 : PreSync();
387 :
388 0 : CCU_IF(mySliceSize_ != 0)
389 : {
390 0 : DoRepeatAllReduce();
391 0 : }
392 0 : PostSync();
393 0 : HCCL_INFO("[CcuContextAllReduceMeshMem2Mem1D] AllReduceMeshMem2Mem1D end");
394 0 : return;
395 : }
396 :
397 0 : std::vector<uint64_t> CcuContextAllReduceMeshMem2Mem1D::GeneArgs(const CcuTaskArg &arg)
398 : {
399 0 : const CurrentTaskArg *taskArg = dynamic_cast<const CurrentTaskArg *>(&arg);
400 : // 空指针校验
401 0 : if (taskArg == nullptr) {
402 0 : THROW<NullPtrException>(StringFormat("CcuContextAllReduceMeshMem2Mem1D::taskArg ptr is null"));
403 : }
404 0 : uint64_t inputAddr = taskArg->inputAddr_;
405 0 : uint64_t outputAddr = taskArg->outputAddr_;
406 0 : uint64_t tokenInfo = taskArg->token_;
407 0 : uint64_t scratchAddr = taskArg->scratchAddr_;
408 0 : uint64_t currentRankSliceInputOffset = taskArg->inputSliceStride_ * rankId_;
409 0 : uint64_t currentRankSliceOutputOffset = taskArg->outputSliceStride_ * rankId_;
410 0 : uint64_t normalSliceSize = taskArg->normalSliceSize_;
411 0 : uint64_t lastSliceSize = taskArg->lastSliceSize_;
412 0 : uint64_t mySliceSize = taskArg->mySliceSize_;
413 0 : uint64_t sliceOffset = taskArg->normalSliceSize_ * rankId_;
414 0 : uint64_t isInputOutputEqual = taskArg->isInputOutputEqual_;
415 :
416 : std::vector<uint64_t> taskArgs = {
417 : inputAddr,
418 : outputAddr,
419 : tokenInfo,
420 : scratchAddr,
421 : currentRankSliceInputOffset,
422 : currentRankSliceOutputOffset,
423 : normalSliceSize,
424 : lastSliceSize,
425 : mySliceSize,
426 : sliceOffset,
427 : isInputOutputEqual,
428 0 : };
429 :
430 0 : auto normalGoSize = CalGoSize(normalSliceSize);
431 0 : auto lastGoSize = CalGoSize(lastSliceSize);
432 :
433 0 : if (rankId_ != rankSize_ - 1 ) {
434 0 : taskArgs.insert(taskArgs.end(), normalGoSize.begin(), normalGoSize.end());
435 : } else {
436 0 : taskArgs.insert(taskArgs.end(), lastGoSize.begin(), lastGoSize.end());
437 : }
438 :
439 0 : HCCL_INFO("[CcuContextAllReduce1DMesh] TaskArgs: inputAddr[%llu], outputAddr[%llu], scratchAddr[%llu], "
440 : "currentRankSliceInputOffset[%llu], currentRankSliceOutputOffset[%llu], normalSliceSize[%llu], "
441 : "lastSliceSize[%llu], mySliceSize[%llu], sliceOffset[%llu], isInputOutputEqual[%llu]",
442 : inputAddr, outputAddr, scratchAddr, currentRankSliceInputOffset, currentRankSliceOutputOffset,
443 : normalSliceSize, lastSliceSize, mySliceSize, sliceOffset, isInputOutputEqual);
444 :
445 0 : return taskArgs;
446 0 : }
447 : } // namespace Hccl
|
