Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "log.h"
12 : #include "ins_coll_alg_registry.h"
13 : #include "topo_match_nhr.h"
14 : #include "topo_match_mesh.h"
15 : #include "topo_match_concurr_mesh.h"
16 : #include "ins_temp_reduce_scatter_mesh_1D.h"
17 : #include "ins_temp_reduce_scatter_mesh_1D_meshchunk.h"
18 : #include "ins_temp_reduce_scatter_mesh_2D.h"
19 : #include "ins_temp_reduce_scatter_nhr.h"
20 : #ifndef CCL_KERNEL_AICPU
21 : #include "aiv_temp_reduce_scatter_mesh_1D.h"
22 : #include "ccu_temp_reduce_scatter_nhr_1D_mem2mem.h"
23 : #include "ccu_temp_reduce_scatter_mesh_1D_2die.h"
24 : #endif
25 : #include "ins_v2_reduce_scatter_sole_executor.h"
26 : #include "ins_temp_reduce_scatter_aicpu_reduce.h"
27 : #include "ins_temp_reduce_scatter_aicpu_reduce_mesh_2D.h"
28 : #include "ccu_temp_reduce_scatter_mesh_1D_mem2mem.h"
29 :
30 : namespace Hccl {
31 : constexpr u64 MAX_OFFLOAD_SCRATCH_SIZE = 200 * 1024 * 1024; // 200M
32 : template <typename AlgTopoMatch, typename InsAlgTemplate>
33 0 : InsV2ReduceScatterSoleExecutor<AlgTopoMatch, InsAlgTemplate>::InsV2ReduceScatterSoleExecutor() : InsCollAlgBase()
34 0 : {}
35 :
36 : template <typename AlgTopoMatch, typename InsAlgTemplate>
37 0 : InsV2ReduceScatterSoleExecutor<AlgTopoMatch, InsAlgTemplate>::~InsV2ReduceScatterSoleExecutor()
38 0 : {}
39 :
40 : template <typename AlgTopoMatch, typename InsAlgTemplate>
41 0 : HcclResult InsV2ReduceScatterSoleExecutor<AlgTopoMatch, InsAlgTemplate>::InitCommInfo(const RankGraph* rankGraph)
42 : {
43 0 : AlgTopoMatch topoMatch(myRank_, rankSize_, rankGraph, devType_);
44 0 : CHK_RET(topoMatch.MatchTopo(vTopo_, virtRanks_, virtRankMap_));
45 0 : return HcclResult::HCCL_SUCCESS;
46 0 : }
47 :
48 : template <typename AlgTopoMatch, typename InsAlgTemplate>
49 0 : HcclResult InsV2ReduceScatterSoleExecutor<AlgTopoMatch, InsAlgTemplate>::InitCommInfo(const AlgTopoInfo& topoInfo)
50 : {
51 0 : CHK_PRT_RET(
52 : topoInfo.vTopo.empty(), HCCL_ERROR("[InsV2ReduceScatterSoleExecutor][InitCommInfo] vTopo size is invalid"),
53 : HCCL_E_PARA);
54 0 : CHK_PRT_RET(
55 : topoInfo.virtRankMap.empty(),
56 : HCCL_ERROR("[InsV2ReduceScatterSoleExecutor][InitCommInfo] virtRankMap size is invalid"), HCCL_E_PARA);
57 0 : CHK_PRT_RET(
58 : topoInfo.virtRanks.empty(),
59 : HCCL_ERROR("[InsV2ReduceScatterSoleExecutor][InitCommInfo] virtRanks size is invalid"), HCCL_E_PARA);
60 0 : vTopo_ = topoInfo.vTopo[0]; // 本通信域内的通信平面
61 0 : virtRankMap_ = topoInfo.virtRankMap[0]; // 本通信域内的 rank 映射表
62 0 : virtRanks_ = topoInfo.virtRanks[0]; // 本通信域内的 rank 集合
63 0 : return HcclResult::HCCL_SUCCESS;
64 : }
65 :
66 : template <typename AlgTopoMatch, typename InsAlgTemplate>
67 0 : HcclResult InsV2ReduceScatterSoleExecutor<AlgTopoMatch, InsAlgTemplate>::CreateTemplates(
68 : std::shared_ptr<InsAlgTemplate>& algTemplatePtr)
69 : {
70 0 : HCCL_DEBUG("[InsV2ReduceScatterSoleExecutor][CreateTemplates]");
71 0 : algTemplatePtr = std::make_shared<InsAlgTemplate>(myRank_, rankSize_, vTopo_, virtRankMap_);
72 0 : CHK_PTR_NULL(algTemplatePtr);
73 0 : algTemplatePtr->SetDmaMode(dmaMode_);
74 0 : algTemplatePtr->SetDataType(dataType_);
75 0 : algTemplatePtr->SetCollOp(op_);
76 0 : algTemplatePtr->InitReduceInfo(redOp_, dataType_);
77 0 : return HcclResult::HCCL_SUCCESS;
78 : }
79 :
80 : template <typename AlgTopoMatch, typename InsAlgTemplate>
81 0 : HcclResult InsV2ReduceScatterSoleExecutor<AlgTopoMatch, InsAlgTemplate>::CalNumBlocks(
82 : u32& numBlocks, u64 dataSize, u32 numBlocksLimit)
83 : {
84 0 : std::shared_ptr<InsAlgTemplate> algTemplate = nullptr;
85 0 : CHK_RET(CreateTemplates(algTemplate));
86 0 : CHK_RET(algTemplate->CalNumBlocks(numBlocks, dataSize, numBlocksLimit));
87 0 : return HcclResult::HCCL_SUCCESS;
88 0 : }
89 :
90 : // HOST 侧算法入口，将对应的instruction添加到指令队列中
91 : template <typename AlgTopoMatch, typename InsAlgTemplate>
92 0 : HcclResult InsV2ReduceScatterSoleExecutor<AlgTopoMatch, InsAlgTemplate>::Orchestrate(
93 : const RankGraph* rankGraph, const CollAlgOperator& op, const CollAlgParams& params, InsQuePtr insQue)
94 : {
95 0 : HCCL_INFO("[InsV2ReduceScatterSoleExecutor][Orchestrate] Orchestrate host Start");
96 0 : CHK_RET(Init(op, params, insQue));
97 0 : CHK_RET(InitCommInfo(rankGraph));
98 0 : u32 dataSizePerVolume = DataTypeSizeGet(dataType_);
99 0 : dataSize_ = dataCount_ * dataSizePerVolume;
100 0 : std::shared_ptr<InsAlgTemplate> algTemplate = nullptr;
101 0 : CHK_RET(CreateTemplates(algTemplate));
102 :
103 0 : AlgTempResReq tempResReq;
104 0 : if (enableDetour_) {
105 0 : HCCL_DEBUG(
106 : "[InsV2ReduceScatterSoleExecutor][Orchestrate] [%s] Rank[%d], CalcRes with detouring enabled.", __func__,
107 : myRank_);
108 0 : CHK_RET(algTemplate->CalcResDetour(rankGraph, tempResReq));
109 : } else {
110 0 : HCCL_DEBUG(
111 : "[InsV2ReduceScatterSoleExecutor][Orchestrate] [%s] Rank[%d], CalcRes with detouring disabled.", __func__,
112 : myRank_);
113 0 : CHK_RET(algTemplate->CalcRes(tempResReq));
114 : }
115 :
116 0 : HCCL_DEBUG(
117 : "[InsV2ReduceScatterSoleExecutor][Orchestrate] Rank[%d], template [%s], requiredQue Num [%u].", myRank_,
118 : algTemplate->Describe().c_str(), tempResReq.queNum);
119 0 : CHK_RET(InitQueue(tempResReq.queNum, tempInsQue_));
120 0 : CHK_RET(PrepResLinks(myRank_, rankGraph, linkPriority_, tempResReq.links, tempResLinks_));
121 0 : CHK_RET(OrchestrateLoop(algTemplate));
122 0 : return HcclResult::HCCL_SUCCESS;
123 0 : }
124 :
125 : template <typename AlgTopoMatch, typename InsAlgTemplate>
126 0 : HcclResult InsV2ReduceScatterSoleExecutor<AlgTopoMatch, InsAlgTemplate>::Orchestrate(
127 : const AlgTopoInfo& topoInfo, const CollAlgOperator& op, const CollAlgParams& params, ConnectedLinkMgr* linkMgr,
128 : InsQuePtr insQue)
129 : {
130 0 : HCCL_INFO("[InsV2ReduceScatterSoleExecutor][Orchestrate] Orchestrate AICPU Start");
131 0 : CHK_RET(Init(op, params, insQue));
132 0 : CHK_RET(InitCommInfo(topoInfo));
133 0 : vTopo_ = topoInfo.vTopo[0]; // 本通信域内的通信平面
134 0 : virtRankMap_ = topoInfo.virtRankMap[0]; // 本通信域内的 rank 映射表
135 0 : virtRanks_ = topoInfo.virtRanks[0]; // 本通信域内的 rank 集合
136 0 : u32 dataSizePerVolume = DataTypeSizeGet(dataType_);
137 0 : dataSize_ = dataCount_ * dataSizePerVolume;
138 0 : std::shared_ptr<InsAlgTemplate> algTemplate = nullptr;
139 0 : CHK_RET(CreateTemplates(algTemplate));
140 :
141 0 : std::map<u32, u32> rank2PathNumMap;
142 0 : CHK_RET(SetPathNumMapByLinkMgrMultiLevel(linkMgr, virtRanks_, myRank_, rank2PathNumMap));
143 0 : algTemplate->setPathNumMap(rank2PathNumMap);
144 :
145 0 : AlgTempResReq tempResReq;
146 0 : if (enableDetour_) {
147 0 : CHK_RET(algTemplate->CalcResDetour(linkMgr, tempResReq));
148 : } else {
149 0 : CHK_RET(algTemplate->CalcRes(tempResReq));
150 : }
151 0 : CHK_RET(InitQueue(tempResReq.queNum, tempInsQue_));
152 0 : CHK_RET(PrepResLinks(myRank_, tempResReq.links, linkMgr, tempResLinks_));
153 0 : CHK_RET(OrchestrateLoop(algTemplate));
154 0 : return HcclResult::HCCL_SUCCESS;
155 0 : }
156 :
157 : // 单算子模式资源计算接口
158 : template <typename AlgTopoMatch, typename InsAlgTemplate>
159 0 : HcclResult InsV2ReduceScatterSoleExecutor<AlgTopoMatch, InsAlgTemplate>::OrchestrateLoop(
160 : std::shared_ptr<InsAlgTemplate> algTemplate)
161 : {
162 0 : HCCL_INFO(
163 : "[InsV2ReduceScatterSoleExecutor][OrchestrateOpbase] Start, template[%s]", algTemplate->Describe().c_str());
164 :
165 0 : TemplateDataParams tempAlgParams;
166 0 : tempAlgParams.buffInfo.inBuffType = BufferType::INPUT;
167 0 : tempAlgParams.buffInfo.outBuffType = BufferType::OUTPUT;
168 0 : tempAlgParams.buffInfo.scratBuffType = BufferType::SCRATCH;
169 :
170 0 : u64 maxDataSizePerLoop = 0;
171 0 : u64 transportBoundDataSize = UB_MAX_DATA_SIZE;
172 0 : HCCL_INFO("[InsV2ReduceScatterSoleExecutor]maxTmpMemSize_ [%u]", maxTmpMemSize_);
173 0 : u32 templateScratchMultiplier
174 0 : = algTemplate->CalcScratchMultiple(tempAlgParams.buffInfo.inBuffType, tempAlgParams.buffInfo.outBuffType);
175 0 : if (templateScratchMultiplier != 0) {
176 : // reduce 时 保障scratch上偏移按照dataTypeSize_对齐
177 0 : u64 scratchBoundDataSize = (maxTmpMemSize_ / templateScratchMultiplier) / dataTypeSize_ * dataTypeSize_;
178 0 : maxDataSizePerLoop = min(transportBoundDataSize, scratchBoundDataSize);
179 : } else {
180 0 : maxDataSizePerLoop = transportBoundDataSize;
181 : }
182 0 : u64 maxDataCountPerLoop = maxDataSizePerLoop / dataTypeSize_; // 单次循环处理的数据量大小，同时会处理两片数据
183 0 : HCCL_INFO(
184 : "[InsV2ReduceScatterSoleExecutor][OrchestrateOpbase] maxDataCountPerLoop[%llu], maxDataSizePerLoop[%llu], "
185 : "transportBoundDataSize[%llu], templateScratchMultiplier[%llu]",
186 : maxDataCountPerLoop, maxDataSizePerLoop, transportBoundDataSize, templateScratchMultiplier);
187 0 : CHK_PRT_RET(
188 : maxDataCountPerLoop == 0,
189 : HCCL_ERROR("[InsV2ReduceScatterSoleExecutor][OrchestrateOpbase] maxDataCountPerLoop is 0"), HCCL_E_INTERNAL);
190 : // 这里处理的数据量，是单次循环所处理的总数据量，包括两个数据片，每一半stream处理一个数据片
191 0 : TempFuncs tempFuncs;
192 0 : tempFuncs.isForepart = true;
193 0 : tempFuncs.opMode = opMode_;
194 0 : tempFuncs.enableCounterNotify = IsEnableCounterNotify();
195 0 : tempFuncs.isBottom = true;
196 : // maxDataCountPerLoop是一次循环所处理的一片数据量大小
197 0 : u64 processedDataCount = 0;
198 0 : u64 allDataCountPerLoop = maxDataCountPerLoop;
199 : #ifdef CCL_KERNEL_AICPU
200 : if (vTopo_.size() > 1) { // aicpu mesh 2d
201 : allDataCountPerLoop = maxDataCountPerLoop * 2;
202 : }
203 : #endif
204 0 : u64 loopTimes = dataCount_ / allDataCountPerLoop + static_cast<u64>(dataCount_ % allDataCountPerLoop != 0);
205 0 : HCCL_INFO(
206 : "[InsV2ReduceScatterSoleExecutor]allDataCountPerLoop [%u],dataCount_ [%u],loopTimes [%u]", allDataCountPerLoop,
207 : dataCount_, loopTimes);
208 0 : for (u64 loop = 0; loop < loopTimes; loop++) {
209 0 : u64 currDataCount = (loop == loopTimes - 1) ? dataCount_ - processedDataCount : allDataCountPerLoop;
210 0 : tempAlgParams.buffInfo.inBuffBaseOff = processedDataCount * dataTypeSize_;
211 0 : tempAlgParams.buffInfo.outBuffBaseOff = processedDataCount * dataTypeSize_;
212 0 : tempAlgParams.buffInfo.scratchBuffBaseOff = 0;
213 :
214 0 : tempAlgParams.sliceSize = currDataCount * dataTypeSize_; // 这里是单次循环处理的两片数据的大小
215 0 : tempAlgParams.tailSize = tempAlgParams.sliceSize;
216 : // 这里的stride当成传统意义上的sreide间隔
217 0 : tempAlgParams.inputSliceStride = dataSize_; // 如果是输入，偏移是算子的output datasize
218 0 : tempAlgParams.outputSliceStride = maxDataSizePerLoop; // 如果是scratchbuffer，偏移是单次循环所处理的最大数据量
219 0 : HCCL_INFO(
220 : "[InsV2ReduceScatterSoleExecutor] loop [%u] tempAlgParams.inputSliceStride "
221 : "[%u],tempAlgParams.outputSliceStride [%u] tempAlgParams.sliceSize [%u]",
222 : loop, tempAlgParams.inputSliceStride, tempAlgParams.outputSliceStride, tempAlgParams.sliceSize);
223 0 : HCCL_INFO(
224 : "[InsV2ReduceScatterSoleExecutor] loop [%u] tempAlgParams.buffInfo.inBuffBaseOff "
225 : "[%u],tempAlgParams.buffInfo.outBuffBaseOff [%u]",
226 : loop, tempAlgParams.buffInfo.inBuffBaseOff, tempAlgParams.buffInfo.outBuffBaseOff);
227 : // 不需要重复
228 0 : tempAlgParams.repeatNum = 1;
229 0 : tempAlgParams.inputRepeatStride = 0;
230 0 : tempAlgParams.outputRepeatStride = 0;
231 :
232 0 : CHK_RET(algTemplate->GenExtIns(tempFuncs, tempAlgParams, tempResLinks_, tempInsQue_));
233 0 : processedDataCount += currDataCount;
234 : }
235 :
236 0 : return HcclResult::HCCL_SUCCESS;
237 0 : }
238 :
239 : template <typename AlgTopoMatch, typename InsAlgTemplate>
240 0 : HcclResult InsV2ReduceScatterSoleExecutor<AlgTopoMatch, InsAlgTemplate>::CalcRes(
241 : const RankGraph* rankGraph, CollAlgResReq& algResReq)
242 : {
243 0 : AlgTopoMatch topoMatch(myRank_, rankSize_, rankGraph, devType_);
244 0 : CHK_RET(topoMatch.MatchTopo(vTopo_, virtRanks_, virtRankMap_));
245 0 : algResReq.topoInfo.UpdateSingleLevelTopo(virtRanks_, virtRankMap_, vTopo_);
246 :
247 0 : InsAlgTemplate tempAlg(myRank_, rankSize_, vTopo_, virtRankMap_);
248 :
249 : // 通过判断哪层通信域能有到所有remoteRank的path，判断当前算法跑在哪一层
250 0 : std::map<u32, u32> rank2PathNumMap;
251 0 : HCCL_INFO("[InsV2ReduceScatterSoleExecutor] CalcRes SetPathNumMap");
252 0 : CHK_RET(SetPathNumMapByRankGraphMultiLevel(rankGraph, virtRanks_, myRank_, rank2PathNumMap));
253 0 : tempAlg.setPathNumMap(rank2PathNumMap);
254 :
255 0 : AlgTempResReq tempResReq;
256 0 : if (enableDetour_) {
257 0 : HCCL_DEBUG(
258 : "[InsV2ReduceScatterSoleExecutor][CalcRes] [%s] Rank[%d], CalcRes with detouring enabled.", __func__,
259 : myRank_);
260 0 : CHK_RET(tempAlg.CalcResDetour(rankGraph, tempResReq));
261 : } else {
262 0 : HCCL_DEBUG(
263 : "[InsV2ReduceScatterSoleExecutor][CalcRes] [%s] Rank[%d], CalcRes with detouring disabled.", __func__,
264 : myRank_);
265 0 : CHK_RET(tempAlg.CalcRes(tempResReq));
266 : }
267 0 : CHK_RET(CalcLinkInfo(myRank_, rankGraph, tempResReq.links, algResReq.levelRankPairs));
268 0 : algResReq.primQueueNum = tempResReq.streamNum;
269 0 : algResReq.queueNotifys = tempResReq.queNotifys;
270 0 : algResReq.localWaitGroupCntNotify = tempResReq.localWaitGroupCntNotify;
271 0 : algResReq.localBcastPostCntNotify = tempResReq.localBcastPostCntNotify;
272 0 : HCCL_DEBUG("[%s] Rank[%d], requiredQueNum [%u].", __func__, myRank_, algResReq.primQueueNum);
273 0 : CHK_RET(CalcResLinks(myRank_, rankGraph, linkPriority_, tempResReq.links, algResReq.links));
274 :
275 0 : return HcclResult::HCCL_SUCCESS;
276 0 : }
277 :
278 : template <typename AlgTopoMatch, typename InsAlgTemplate>
279 0 : HcclResult InsV2ReduceScatterSoleExecutor<AlgTopoMatch, InsAlgTemplate>::CalcResOffload(
280 : const RankGraph* rankGraph, const u64& dataSize, CollOffloadOpResReq& resReq)
281 : {
282 : (void)dataSize;
283 :
284 : // Topo Match
285 0 : CHK_RET(InitCommInfo(rankGraph));
286 :
287 0 : std::shared_ptr<InsAlgTemplate> algTemplate = nullptr;
288 0 : CHK_RET(CreateTemplates(algTemplate));
289 :
290 : // 通过判断哪层通信域能有到所有remoteRank的path，判断当前算法跑在哪一层
291 0 : std::map<u32, u32> rank2PathNumMap;
292 0 : HCCL_INFO("[InsV2ReduceScatterSoleExecutor] CalcResOffload SetPathNumMap");
293 0 : CHK_RET(SetPathNumMapByRankGraphMultiLevel(rankGraph, virtRanks_, myRank_, rank2PathNumMap));
294 0 : algTemplate->setPathNumMap(rank2PathNumMap);
295 :
296 0 : AlgTempResReq tempResReq;
297 0 : if (enableDetour_) {
298 0 : HCCL_DEBUG(
299 : "[InsV2ReduceScatterSoleExecutor][CalcResOffload] [%s] Rank[%d], CalcRes with detouring enabled.", __func__,
300 : myRank_);
301 0 : CHK_RET(algTemplate->CalcResDetour(rankGraph, tempResReq));
302 : } else {
303 0 : HCCL_DEBUG(
304 : "[InsV2ReduceScatterSoleExecutor][CalcResOffload] [%s] Rank[%d], CalcRes with detouring disabled.",
305 : __func__, myRank_);
306 0 : CHK_RET(algTemplate->CalcRes(tempResReq));
307 : }
308 0 : resReq.requiredScratchMemSize = UB_MAX_DATA_SIZE;
309 0 : resReq.requiredSubQueNum = tempResReq.streamNum - 1;
310 :
311 0 : return HcclResult::HCCL_SUCCESS;
312 0 : }
313 :
314 : INS_REGISTER_IMPL_BY_TEMP(
315 : OpType::REDUCESCATTER, InsReduceScatterMesh1D, InsV2ReduceScatterSoleExecutor, TopoMatchMesh,
316 : InsTempReduceScatterMesh1D);
317 : INS_REGISTER_IMPL_BY_TEMP(
318 : OpType::REDUCESCATTER, InsReduceScatterMesh1DMeshChunk, InsV2ReduceScatterSoleExecutor, TopoMatchMesh,
319 : InsTempReduceScatterMesh1DMeshChunk);
320 : INS_REGISTER_IMPL_BY_TEMP(
321 : OpType::REDUCESCATTER, InsReduceScatterNHR, InsV2ReduceScatterSoleExecutor, TopoMatchNHR, InsTempReduceScatterNHR);
322 : INS_REGISTER_IMPL_BY_TEMP(
323 : OpType::REDUCESCATTER, InsReduceScatterMesh2D, InsV2ReduceScatterSoleExecutor, TopoMatchConcurrMesh,
324 : InsTempReduceScatterMesh2D);
325 : INS_REGISTER_IMPL_BY_TEMP(
326 : OpType::REDUCESCATTER, InsReduceScatterAicpuReduce, InsV2ReduceScatterSoleExecutor, TopoMatchMesh,
327 : InsTempReduceScatterAicpuReduce);
328 : INS_REGISTER_IMPL_BY_TEMP(
329 : OpType::REDUCESCATTER, InsReduceScatterAicpuReduceMesh2D, InsV2ReduceScatterSoleExecutor, TopoMatchConcurrMesh,
330 : InsTempReduceScatterAicpuReduceMesh2D);
331 : #ifndef CCL_KERNEL_AICPU
332 : INS_REGISTER_IMPL_BY_TEMP(
333 : OpType::REDUCESCATTER, AivReduceScatterMesh1D, InsV2ReduceScatterSoleExecutor, TopoMatchMesh,
334 : AivTempReduceScatterMesh1D);
335 : INS_REGISTER_IMPL_BY_TEMP(
336 : OpType::REDUCESCATTER, CcuReduceScatterMeshMem2Mem1D, InsV2ReduceScatterSoleExecutor, TopoMatchMesh,
337 : CcuTempReduceScatterMeshMem2Mem1D);
338 : INS_REGISTER_IMPL_BY_TEMP(
339 : OpType::REDUCESCATTER, CcuReduceScatterNHR1DMem2Mem, InsV2ReduceScatterSoleExecutor, TopoMatchMesh,
340 : CcuTempReduceScatterNHR1DMem2Mem);
341 : INS_REGISTER_IMPL_BY_TEMP(
342 : OpType::REDUCESCATTER, CcuReduceScatterMesh1D2Die, InsV2ReduceScatterSoleExecutor, TopoMatchMesh,
343 : CcuTempReduceScatterMesh1D2Die);
344 : #endif
345 : } // namespace Hccl
|
