Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "ccu_context_all_reduce_mesh1d_mem2mem.h"
12 : #include "ccu_instruction_all_reduce_mesh1d_mem2mem.h"
13 : #include "ccu_assist.h"
14 : namespace Hccl {
15 : constexpr int INPUT_XN_ID = 0;
16 : constexpr int OUTPUT_XN_ID = 1;
17 : constexpr int SCRATCH_XN_ID = 2;
18 : constexpr int TOKEN_XN_ID = 3;
19 : constexpr int CKE_IDX_0 = 0;
20 : constexpr int CKE_IDX_1 = 1;
21 : constexpr int CKE_IDX_2 = 2;
22 : constexpr int CKE_IDX_3 = 3;
23 :
24 : using CurrentCtxArg = CcuCtxArgAllReduceMeshMem2Mem1D;
25 : using CurrentTaskArg = CcuTaskArgAllReduceMeshMem2Mem1D;
26 :
27 0 : CcuContextAllReduceMeshMem2Mem1D::CcuContextAllReduceMeshMem2Mem1D(
28 0 : const CcuCtxArg& arg, const std::vector<CcuTransport*>& transports, const CcuTransportGroup& group)
29 0 : : CcuContextAlgBase(arg, transports, group)
30 : {
31 0 : HCCL_DEBUG("[CcuContextAllReduceMeshMem2Mem1D] Enter Constructor.");
32 0 : const CurrentCtxArg* ctxArg = dynamic_cast<const CurrentCtxArg*>(&arg);
33 0 : if (ctxArg == nullptr) {
34 0 : THROW<NullPtrException>(StringFormat("CcuContextAllReduceMeshMem2Mem1D::ctxArg ptr is null"));
35 : }
36 0 : dataType_ = ctxArg->op_.dataType;
37 0 : outputDataType_ = ctxArg->op_.outputDataType;
38 0 : rankId_ = ctxArg->rankId_;
39 0 : rankSize_ = ctxArg->dimSize_[0];
40 0 : reduceOp_ = ctxArg->op_.reduceOp;
41 0 : if (outputDataType_ == DataType::INVALID) {
42 0 : outputDataType_ = dataType_;
43 0 : HCCL_INFO(
44 : "[CcuContextAllReduceMeshMem2Mem1D] outputDataType is [INVALID], set outputDataType to[%s]",
45 : outputDataType_.Describe().c_str());
46 : }
47 0 : CHK_PRT_THROW(
48 : ctxArg->dimSize_[0] == 0, HCCL_ERROR("[CcuContextAllReduceMeshMem2Mem1D] ctxArg->dimSize_[0] is zero"),
49 : InvalidParamsException, "ctxArg->dimSize_[0] is invalid");
50 0 : if (ctxArg->dimSize_.size() > 0) {
51 0 : rankSize_ = ctxArg->dimSize_[0];
52 : }
53 0 : HCCL_INFO(
54 : "[CcuContextAllReduceMeshMem2Mem1D] Init, CtxArgs are rankId[%u], rankSize_[%llu], dataType[%s], "
55 : "outputDataType[%s], reduceOp[%s]",
56 : rankId_, rankSize_, dataType_.Describe().c_str(), outputDataType_.Describe().c_str(),
57 : reduceOp_.Describe().c_str());
58 0 : }
59 :
60 0 : void CcuContextAllReduceMeshMem2Mem1D::InitResource()
61 : {
62 0 : if (transports.size() == 0) {
63 0 : THROW<NullPtrException>(StringFormat("CcuContextAllReduceMeshMem2Mem1D transports is empty"));
64 : }
65 0 : HCCL_INFO("[CcuContextAllReduceMeshMem2Mem1D]transports.size: [%zu]", transports.size());
66 0 : uint16_t transportIdx = 0;
67 : // 按照rank号从小到大遍历transports，遇到本rank就填充本地资源，否则依次取远端资源，要求给框架返回的Link同样是按顺序排列的
68 0 : for (uint64_t peerId = 0; peerId < rankSize_; peerId++) {
69 0 : if (peerId == rankId_) {
70 0 : input_.push_back(CreateVariable());
71 0 : output_.push_back(CreateVariable());
72 0 : scratch_.push_back(CreateVariable());
73 0 : token_.push_back(CreateVariable());
74 : } else {
75 0 : HCCL_DEBUG(
76 : "[CcuContextAllReduceMeshMem2Mem1D] MyRank[%u], PeerId[%llu], TransportId[%u]", rankId_, peerId,
77 : transportIdx);
78 0 : CHK_PRT_THROW(
79 : transports[transportIdx] == nullptr,
80 : HCCL_ERROR("[CcuContextAllReduceMeshMem2Mem1D] [InitResource] transports[%u] is nullptr", transportIdx),
81 : NullPtrException, "transport is null");
82 0 : input_.push_back(CreateVariable((*transports[transportIdx]), INPUT_XN_ID));
83 0 : output_.push_back(CreateVariable((*transports[transportIdx]), OUTPUT_XN_ID));
84 0 : scratch_.push_back(CreateVariable((*transports[transportIdx]), SCRATCH_XN_ID));
85 0 : token_.push_back(CreateVariable((*transports[transportIdx]), TOKEN_XN_ID));
86 0 : transportIdx++;
87 : }
88 : }
89 0 : currentRankSliceInputOffset_ = CreateVariable();
90 0 : currentRankSliceOutputOffset_ = CreateVariable();
91 0 : normalSliceSize_ = CreateVariable();
92 0 : lastSliceSize_ = CreateVariable();
93 0 : mySliceSize_ = CreateVariable();
94 0 : sliceOffset_ = CreateVariable();
95 0 : isInputOutputEqual_ = CreateVariable();
96 0 : locMask_ = CreateMaskSignal();
97 0 : srcMem_ = CreateMemory();
98 0 : dstMem_ = CreateMemory();
99 0 : reduceScatterSrc_.reserve(rankSize_);
100 0 : for (uint32_t rankIdx = 0; rankIdx < rankSize_; rankIdx++) {
101 0 : reduceScatterSrc_.push_back(CreateMemory());
102 : }
103 0 : reduceScatterDst_.reserve(rankSize_);
104 0 : for (uint32_t rankIdx = 0; rankIdx < rankSize_; rankIdx++) {
105 0 : reduceScatterDst_.push_back(CreateMemory());
106 : }
107 0 : sliceSize_ = CreateVariable();
108 0 : selfBit_ = 1 << rankId_;
109 0 : allBit_ = ((1 << rankSize_) - 1) & (~(1 << rankId_)); // rankId_位为0，其他位都为1
110 0 : localGoSize_ = CreateGroupOpSize();
111 0 : return;
112 : }
113 :
114 0 : std::string CcuContextAllReduceMeshMem2Mem1D::GetLoopBlockTag(std::string loopType, int32_t index)
115 : {
116 0 : return loopType + LOOP_BLOCK_TAG + std::to_string(index);
117 : }
118 :
119 0 : void CcuContextAllReduceMeshMem2Mem1D::CreateReduceLoop(
120 : uint32_t size, DataType dataType, DataType outputDataType, ReduceOp opType)
121 : {
122 0 : constexpr uint32_t LOOP_NUM = 16;
123 0 : AllocGoResource(LOOP_NUM);
124 :
125 0 : std::string loopType = CcuRep::GetReduceTypeStr(dataType, opType);
126 0 : if (registeredLoop.find(loopType) != registeredLoop.end()) {
127 0 : return;
128 : }
129 :
130 0 : uint32_t expansionNum = CcuRep::GetReduceExpansionNum(opType, dataType, outputDataType);
131 0 : uint32_t usedBufNum = size > expansionNum ? size : expansionNum;
132 :
133 0 : for (int32_t index = 0; index < 2; index++) { // 需要实例化2个Loop
134 0 : CcuRep::Memory src = CreateMemory();
135 0 : CcuRep::Memory dst = CreateMemory();
136 0 : std::vector<CcuRep::Memory> scratch;
137 0 : for (uint32_t i = 0; i < size; i++) {
138 0 : scratch.emplace_back(CreateMemory());
139 : }
140 0 : CcuRep::Variable lenForExpansion = CreateVariable();
141 0 : CcuRep::Variable len = CreateVariable();
142 0 : CcuRep::LoopBlock lb(this, GetLoopBlockTag(loopType, index));
143 0 : lb(dst, src, scratch, len, lenForExpansion);
144 :
145 : std::vector<CcuRep::CcuBuffer> bufs
146 0 : = {moRes.ccuBuffer.begin() + index * moConfig.msInterleave,
147 0 : moRes.ccuBuffer.begin() + index * moConfig.msInterleave + usedBufNum};
148 0 : CcuRep::MaskSignal sem = moRes.maskSignal[index];
149 :
150 0 : for (uint32_t i = 0; i < size; i++) {
151 0 : if (i == rankId_) {
152 0 : LocalCopy(bufs[i], src, len, sem, 1 << i);
153 : } else {
154 0 : LocalCopy(bufs[i], scratch[i], len, sem, 1 << i);
155 : }
156 : }
157 0 : LocalWait(sem, (1 << size) - 1);
158 :
159 0 : if (size > 1) {
160 0 : LocalReduce(bufs, size, dataType, outputDataType, opType, sem, len);
161 0 : LocalWait(sem);
162 : }
163 :
164 0 : LocalCopy(dst, bufs[0], lenForExpansion, sem);
165 0 : LocalWait(sem);
166 0 : }
167 :
168 0 : registeredLoop.insert(loopType);
169 0 : }
170 :
171 0 : void CcuContextAllReduceMeshMem2Mem1D::ReduceLoopGroup(
172 : CcuRep::Memory outDstOrg, CcuRep::Memory srcOrg, std::vector<CcuRep::Memory>& scratchOrg, GroupOpSize goSize,
173 : DataType dataType, DataType outputDataType, ReduceOp opType)
174 : {
175 0 : const uint32_t size = scratchOrg.size();
176 :
177 0 : CcuRep::Memory dst = CreateMemory();
178 0 : CcuRep::Memory src = CreateMemory();
179 0 : std::vector<CcuRep::Memory> scratch;
180 0 : dst = outDstOrg;
181 0 : src = srcOrg;
182 0 : for (uint32_t idx = 0; idx < size; idx++) {
183 0 : scratch.push_back(CreateMemory());
184 0 : scratch[idx] = scratchOrg[idx];
185 : }
186 :
187 0 : CreateReduceLoop(size, dataType, outputDataType, opType);
188 :
189 0 : std::string loopType = CcuRep::GetReduceTypeStr(dataType, opType);
190 0 : uint32_t expansionNum = CcuRep::GetReduceExpansionNum(opType, dataType, outputDataType);
191 0 : CcuRep::Variable sliceSizeExpansion = CreateVariable();
192 :
193 0 : if (expansionNum != 1) {
194 0 : CcuRep::Variable tmp = CreateVariable();
195 0 : tmp = CcuRep::GetExpansionParam(expansionNum);
196 0 : dst.token += tmp;
197 0 : }
198 :
199 : // m部分
200 0 : CCU_IF(goSize.loopParam != 0) // goSize1
201 : {
202 0 : CcuRep::Variable loopParam = CreateVariable();
203 0 : CcuRep::Variable sliceSize = CreateVariable();
204 0 : loopParam = CcuRep::GetLoopParam(0, moConfig.memSlice * moConfig.loopCount, 0);
205 0 : loopParam += goSize.loopParam;
206 0 : sliceSizeExpansion = moConfig.memSlice * expansionNum;
207 0 : sliceSize = moConfig.memSlice;
208 :
209 0 : auto lc = Loop(GetLoopBlockTag(loopType, 0))(dst, src, scratch, sliceSize, sliceSizeExpansion);
210 :
211 0 : CcuRep::Variable paraCfg = CreateVariable();
212 0 : CcuRep::Variable offsetCfg = CreateVariable();
213 0 : paraCfg = CcuRep::GetParallelParam(moConfig.loopCount - 1, 0, 1);
214 0 : offsetCfg = CcuRep::GetOffsetParam(moConfig.memSlice, moConfig.msInterleave, 1);
215 :
216 0 : LoopGroup({lc}, {loopParam}, paraCfg, offsetCfg);
217 0 : }
218 :
219 0 : CCU_IF(goSize.parallelParam != 0) // goSize2
220 : {
221 : // p部分，加m的偏移
222 0 : for (uint32_t i = 0; i < size; i++) {
223 0 : scratch[i].addr += goSize.addrOffset;
224 : }
225 0 : src.addr += goSize.addrOffset; // goSize0
226 0 : for (uint32_t i = 0; i < expansionNum; i++) {
227 0 : dst.addr += goSize.addrOffset;
228 : }
229 :
230 0 : sliceSizeExpansion = 0;
231 0 : for (uint32_t i = 0; i < expansionNum; i++) {
232 0 : sliceSizeExpansion += goSize.residual; // goSize3
233 : }
234 :
235 0 : auto lc0 = Loop(GetLoopBlockTag(loopType, 0))(dst, src, scratch, goSize.residual, sliceSizeExpansion);
236 :
237 : // n部分，再加p的偏移
238 0 : for (uint32_t i = 0; i < size; i++) {
239 0 : scratch[i].addr += goSize.residual;
240 : }
241 0 : src.addr += goSize.residual;
242 0 : for (uint32_t i = 0; i < expansionNum; i++) {
243 0 : dst.addr += goSize.residual;
244 : }
245 :
246 0 : CcuRep::Variable sliceSize = CreateVariable();
247 0 : sliceSize = moConfig.memSlice;
248 0 : sliceSizeExpansion = moConfig.memSlice * expansionNum;
249 :
250 0 : auto lc1 = Loop(GetLoopBlockTag(loopType, 1))(dst, src, scratch, sliceSize, sliceSizeExpansion);
251 :
252 0 : CcuRep::Variable loopCfg0 = CreateVariable();
253 0 : CcuRep::Variable loopCfg1 = CreateVariable();
254 0 : CcuRep::Variable offsetCfg = CreateVariable();
255 0 : loopCfg0 = CcuRep::GetLoopParam(0, 0, 1);
256 0 : loopCfg1 = CcuRep::GetLoopParam(0, 0, 1);
257 0 : offsetCfg = CcuRep::GetOffsetParam(moConfig.memSlice, moConfig.msInterleave, 1);
258 :
259 0 : LoopGroup({lc0, lc1}, {loopCfg0, loopCfg1}, goSize.parallelParam, offsetCfg);
260 0 : }
261 0 : }
262 :
263 0 : void CcuContextAllReduceMeshMem2Mem1D::LoadArgs()
264 : {
265 0 : Load(input_[rankId_]);
266 0 : Load(output_[rankId_]);
267 0 : Load(token_[rankId_]);
268 0 : Load(scratch_[rankId_]);
269 0 : Load(currentRankSliceInputOffset_);
270 0 : Load(currentRankSliceOutputOffset_);
271 0 : Load(normalSliceSize_);
272 0 : Load(lastSliceSize_);
273 0 : Load(mySliceSize_);
274 0 : Load(sliceOffset_);
275 0 : Load(isInputOutputEqual_);
276 0 : Load(localGoSize_);
277 0 : return;
278 : }
279 :
280 0 : void CcuContextAllReduceMeshMem2Mem1D::PreSync()
281 : {
282 : // 互换内存信息
283 0 : for (auto t : transports) {
284 0 : HCCL_INFO("[CcuContextAllReduceMeshMem2Mem1D] AllReduceMeshMem2Mem1D LocalPost begin");
285 : // 交换起始地址
286 0 : WriteVariableWithSignal(*t, input_[rankId_], INPUT_XN_ID, CKE_IDX_1, selfBit_);
287 0 : WriteVariableWithSignal(*t, output_[rankId_], OUTPUT_XN_ID, CKE_IDX_2, selfBit_);
288 0 : WriteVariableWithSignal(*t, token_[rankId_], TOKEN_XN_ID, CKE_IDX_3, selfBit_);
289 : }
290 0 : GroupWait(*transportGroup, CKE_IDX_1, allBit_);
291 0 : GroupWait(*transportGroup, CKE_IDX_2, allBit_);
292 0 : GroupWait(*transportGroup, CKE_IDX_3, allBit_);
293 0 : HCCL_INFO("[CcuContextAllReduceMeshMem2Mem1D] AllReduceMeshMem2Mem1D wait all end");
294 0 : return;
295 : }
296 :
297 0 : void CcuContextAllReduceMeshMem2Mem1D::PostSync()
298 : {
299 0 : for (auto& t : transports) {
300 0 : RemotePost(*t, CKE_IDX_0, selfBit_);
301 : }
302 0 : GroupWait(*transportGroup, CKE_IDX_0, allBit_);
303 0 : HCCL_INFO("[CcuContextAllReduceMeshMem2Mem1D] AllReduceMeshMem2Mem1D AllReduce groupwait end");
304 0 : }
305 :
306 0 : void CcuContextAllReduceMeshMem2Mem1D::BcastLocToRmt(
307 : const CcuRep::Variable& srcAddr, const std::vector<CcuRep::Variable>& dstAddr)
308 : {
309 0 : CHK_PRT_THROW(
310 : dstAddr.size() != transports.size() + 1,
311 : HCCL_ERROR("[ReduceRmtToLoc] srcAddr.size[%zu] != transports size[%zu] + 1", dstAddr.size(), transports.size()),
312 : InvalidParamsException, "Invalid srcAddr size");
313 :
314 0 : srcMem_.addr = srcAddr;
315 0 : srcMem_.addr += sliceOffset_;
316 0 : srcMem_.token = token_[rankId_];
317 :
318 0 : uint32_t transportIdx = 0;
319 0 : for (uint32_t rmtId = 0; rmtId < dstAddr.size(); rmtId++) {
320 0 : if (rmtId == rankId_) {
321 0 : continue;
322 : }
323 0 : dstMem_.addr = dstAddr[rmtId];
324 0 : dstMem_.addr += sliceOffset_;
325 0 : dstMem_.token = token_[rmtId];
326 :
327 0 : Write(*transports[transportIdx], dstMem_, srcMem_, sliceSize_, locMask_, 1 << rmtId);
328 0 : transportIdx++;
329 : }
330 0 : LocalWait(locMask_, allBit_);
331 0 : }
332 :
333 0 : void CcuContextAllReduceMeshMem2Mem1D::ReduceRmtToLoc(
334 : const std::vector<CcuRep::Variable>& srcAddr, const CcuRep::Variable& dstAddr)
335 : {
336 0 : CHK_PRT_THROW(
337 : srcAddr.size() != transports.size() + 1,
338 : HCCL_ERROR("[ReduceRmtToLoc] srcAddr.size[%zu] != transports size[%zu] +1", srcAddr.size(), transports.size()),
339 : InvalidParamsException, "Invalid srcAddr size");
340 :
341 0 : dstMem_.addr = dstAddr;
342 0 : dstMem_.addr += sliceOffset_;
343 0 : dstMem_.token = token_[rankId_];
344 :
345 0 : CcuRep::Variable scratchOffset = CreateVariable();
346 0 : scratchOffset = 0;
347 0 : for (uint32_t rankIdx = 0; rankIdx < rankSize_; rankIdx++) {
348 0 : reduceScatterSrc_[rankIdx].addr = srcAddr[rankIdx];
349 0 : reduceScatterSrc_[rankIdx].addr += sliceOffset_;
350 0 : reduceScatterSrc_[rankIdx].token = token_[rankIdx];
351 :
352 0 : reduceScatterDst_[rankIdx].addr = scratch_[rankId_];
353 0 : reduceScatterDst_[rankIdx].addr += scratchOffset;
354 0 : scratchOffset += normalSliceSize_;
355 0 : reduceScatterDst_[rankIdx].token = token_[rankId_];
356 : }
357 :
358 0 : uint32_t transportId = 0;
359 0 : for (uint32_t rankIdx = 0; rankIdx < rankSize_; rankIdx++) {
360 0 : if (rankIdx == rankId_) {
361 0 : LocalPost(locMask_, 1 << rankIdx);
362 : } else {
363 0 : Read(
364 0 : *transports[transportId], reduceScatterDst_[rankIdx], reduceScatterSrc_[rankIdx], sliceSize_, locMask_,
365 0 : 1 << rankIdx);
366 0 : transportId++;
367 : }
368 : }
369 0 : LocalWait(locMask_, (1 << rankSize_) - 1);
370 0 : ReduceLoopGroup(
371 0 : dstMem_, reduceScatterSrc_[rankId_], reduceScatterDst_, localGoSize_, dataType_, outputDataType_, reduceOp_);
372 0 : }
373 :
374 0 : void CcuContextAllReduceMeshMem2Mem1D::DoRepeatAllReduce()
375 : {
376 0 : if (rankId_ != rankSize_ - 1) {
377 0 : sliceSize_ = normalSliceSize_;
378 : } else {
379 0 : sliceSize_ = lastSliceSize_;
380 : }
381 0 : ReduceRmtToLoc(input_, output_[rankId_]);
382 0 : BcastLocToRmt(output_[rankId_], output_);
383 0 : }
384 :
385 0 : void CcuContextAllReduceMeshMem2Mem1D::Algorithm()
386 : {
387 0 : HCCL_INFO("[CcuContextAllReduceMeshMem2Mem1D] AllReduceMeshMem2Mem1D run");
388 0 : InitResource();
389 0 : LoadArgs();
390 0 : PreSync();
391 :
392 0 : CCU_IF(mySliceSize_ != 0) { DoRepeatAllReduce(); }
393 0 : PostSync();
394 0 : HCCL_INFO("[CcuContextAllReduceMeshMem2Mem1D] AllReduceMeshMem2Mem1D end");
395 0 : return;
396 : }
397 :
398 0 : std::vector<uint64_t> CcuContextAllReduceMeshMem2Mem1D::GeneArgs(const CcuTaskArg& arg)
399 : {
400 0 : const CurrentTaskArg* taskArg = dynamic_cast<const CurrentTaskArg*>(&arg);
401 : // 空指针校验
402 0 : if (taskArg == nullptr) {
403 0 : THROW<NullPtrException>(StringFormat("CcuContextAllReduceMeshMem2Mem1D::taskArg ptr is null"));
404 : }
405 0 : uint64_t inputAddr = taskArg->inputAddr_;
406 0 : uint64_t outputAddr = taskArg->outputAddr_;
407 0 : uint64_t tokenInfo = taskArg->token_;
408 0 : uint64_t scratchAddr = taskArg->scratchAddr_;
409 0 : uint64_t currentRankSliceInputOffset = taskArg->inputSliceStride_ * rankId_;
410 0 : uint64_t currentRankSliceOutputOffset = taskArg->outputSliceStride_ * rankId_;
411 0 : uint64_t normalSliceSize = taskArg->normalSliceSize_;
412 0 : uint64_t lastSliceSize = taskArg->lastSliceSize_;
413 0 : uint64_t mySliceSize = taskArg->mySliceSize_;
414 0 : uint64_t sliceOffset = taskArg->normalSliceSize_ * rankId_;
415 0 : uint64_t isInputOutputEqual = taskArg->isInputOutputEqual_;
416 :
417 : std::vector<uint64_t> taskArgs = {
418 : inputAddr,
419 : outputAddr,
420 : tokenInfo,
421 : scratchAddr,
422 : currentRankSliceInputOffset,
423 : currentRankSliceOutputOffset,
424 : normalSliceSize,
425 : lastSliceSize,
426 : mySliceSize,
427 : sliceOffset,
428 : isInputOutputEqual,
429 0 : };
430 :
431 0 : auto normalGoSize = CalGoSize(normalSliceSize);
432 0 : auto lastGoSize = CalGoSize(lastSliceSize);
433 :
434 0 : if (rankId_ != rankSize_ - 1) {
435 0 : taskArgs.insert(taskArgs.end(), normalGoSize.begin(), normalGoSize.end());
436 : } else {
437 0 : taskArgs.insert(taskArgs.end(), lastGoSize.begin(), lastGoSize.end());
438 : }
439 :
440 0 : HCCL_INFO(
441 : "[CcuContextAllReduce1DMesh] TaskArgs: inputAddr[%llu], outputAddr[%llu], scratchAddr[%llu], "
442 : "currentRankSliceInputOffset[%llu], currentRankSliceOutputOffset[%llu], normalSliceSize[%llu], "
443 : "lastSliceSize[%llu], mySliceSize[%llu], sliceOffset[%llu], isInputOutputEqual[%llu]",
444 : inputAddr, outputAddr, scratchAddr, currentRankSliceInputOffset, currentRankSliceOutputOffset, normalSliceSize,
445 : lastSliceSize, mySliceSize, sliceOffset, isInputOutputEqual);
446 :
447 0 : return taskArgs;
448 0 : }
449 : } // namespace Hccl
|
