Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "alg_template_register.h"
12 : #include "reduce_scatter_recursive_hd.h"
13 :
14 : namespace hccl {
15 5 : ReduceScatterRecursiveHalvingDoubling::ReduceScatterRecursiveHalvingDoubling(const HcclDispatcher dispatcher)
16 5 : : RecursiveHalvingDoublingBase(dispatcher)
17 5 : {}
18 :
19 10 : ReduceScatterRecursiveHalvingDoubling::~ReduceScatterRecursiveHalvingDoubling() {}
20 :
21 5 : HcclResult ReduceScatterRecursiveHalvingDoubling::Prepare(u64 reduceAttrBitMap, HcomCollOpInfo* opInfo)
22 : {
23 : (void)opInfo;
24 5 : reduceAttr = reduceAttrBitMap;
25 5 : return HCCL_SUCCESS;
26 : }
27 :
28 : // reducescatter recursiveHD 入口函数
29 0 : HcclResult ReduceScatterRecursiveHalvingDoubling::RunAsync(
30 : const u32 rank, const u32 rankSize, const std::vector<std::shared_ptr<Transport>>& links)
31 : {
32 0 : CHK_SMART_PTR_NULL(dispatcher_);
33 0 : CHK_PTR_NULL(stream_.ptr());
34 0 : HCCL_INFO(
35 : "run: rank[%u] totalrank[%u] inputMem[%p] outputMem[%p] count[%llu]", rank, rankSize, inputMem_.ptr(),
36 : outputMem_.ptr(), count_);
37 0 : if (!outputMem_ || !inputMem_) {
38 0 : HCCL_ERROR(
39 : "[ReduceScatterRecursiveHalvingDoubling][RunAsync]rank[%u] run_async inputmem or outputmem is null", rank);
40 0 : return HCCL_E_PTR;
41 : }
42 :
43 0 : HcclResult ret = HCCL_SUCCESS;
44 :
45 0 : if (rankSize == 1) {
46 0 : if (inputMem_ != outputMem_) {
47 0 : ret = HcclD2DMemcpyAsync(dispatcher_, outputMem_, inputMem_, stream_);
48 : }
49 0 : return ret;
50 : }
51 :
52 : // 创建reducer & sender
53 0 : senderInfo_.reset(new (std::nothrow) Sender(dataType_, reductionOp_, reduceAttr));
54 0 : CHK_SMART_PTR_NULL(senderInfo_);
55 :
56 0 : reducerInfo_.reset(new (std::nothrow) Reducer(dataType_, reductionOp_, reduceAttr));
57 0 : CHK_SMART_PTR_NULL(reducerInfo_);
58 :
59 0 : bool bRetSize = (links.size() < rankSize);
60 0 : CHK_PRT_RET(
61 : bRetSize,
62 : HCCL_ERROR(
63 : "[ReduceScatterRecursiveHalvingDoubling][RunAsync]rank[%u] linksize[%llu] is "
64 : "error",
65 : rank, links.size()),
66 : HCCL_E_INTERNAL);
67 :
68 0 : ret = CalcPartOneSizeAndBlockSize(rankSize);
69 0 : CHK_PRT_RET(
70 : ret != HCCL_SUCCESS,
71 : HCCL_ERROR(
72 : "[ReduceScatterRecursiveHalvingDoubling][RunAsync]calculate part1size[%u] "
73 : "and blocksize[%u] Failed! rankSize[%u]",
74 : part1Size_, blockSize_, rankSize),
75 : ret);
76 :
77 0 : HCCL_DEBUG("rank[%u] calculate par1size[%u] blocksize[%u] ranksize[%u]", rank, part1Size_, blockSize_, rankSize);
78 :
79 0 : CHK_RET(ReduceInPartOne(rank, links));
80 :
81 0 : CHK_RET(CalculateSlices(dataBytes_, rankSize));
82 :
83 0 : CHK_RET(ReduceScatterInBlock(rank, rankSize, links));
84 :
85 0 : CHK_RET(ScatterInPartOne(rank, rankSize, links));
86 :
87 0 : HCCL_INFO("ReduceScatterRecursiveHalvingDoubling finished: rank[%u]", rank);
88 0 : return HCCL_SUCCESS;
89 : }
90 :
91 0 : HcclResult ReduceScatterRecursiveHalvingDoubling::CalculateSlices(u64 dataBytes, const u32 rankSize) const
92 : {
93 0 : CHK_PRT_RET((blockSize_ == 0), HCCL_ERROR("[Calculate][Slices]calculate_slices para error"), HCCL_E_INTERNAL);
94 :
95 0 : slices_.resize(blockSize_);
96 0 : u64 bytesPerSlice = dataBytes / rankSize; // input大小 / server数 = 服务器内rank数*count (4p mesh以4*count为粒度)
97 0 : u32 i = 0;
98 0 : u32 halfPart1Size = (part1Size_ / 2); // 除2计算一半part1的大小
99 :
100 : /* 先给属于part1的block rank分配slice。每个rank有两份数据 */
101 0 : while (i < halfPart1Size) {
102 0 : slices_[i].size = 2 * bytesPerSlice; // 乘2计算2倍数据大小
103 0 : slices_[i].offset = i * 2 * bytesPerSlice; // 乘2计算2倍数据大小
104 0 : i++;
105 : }
106 :
107 : /* 再给剩余的block rank分配slice。每个rank有一份数据 */
108 0 : while (i < blockSize_) {
109 0 : slices_[i].size = bytesPerSlice;
110 0 : slices_[i].offset = (i * bytesPerSlice) + (halfPart1Size * bytesPerSlice);
111 0 : i++;
112 : }
113 :
114 0 : return HCCL_SUCCESS;
115 : }
116 :
117 0 : HcclResult ReduceScatterRecursiveHalvingDoubling::ReduceInPartOne(u32 rank, const std::vector<LINK>& links)
118 : {
119 0 : if (rank < part1Size_ && rank % 2 == 0) { // 模2判断奇偶性，rank属于第一部分，并且为偶数rank
120 0 : u32 peerRank = rank + 1;
121 0 : HCCL_DEBUG(
122 : "rank[%u] outputmem receives from peerrank[%u] inputmem, offset[%llu], size[%llu]", rank, peerRank,
123 : baseOffset_, scratchMem_.size());
124 0 : if (peerRank < links.size()) {
125 0 : CHK_SMART_PTR_NULL(links[peerRank]);
126 0 : HcclResult ret = links[peerRank]->TxAck(stream_);
127 0 : CHK_PRT_RET(
128 : ret != HCCL_SUCCESS,
129 : HCCL_ERROR("[Reduce][InPartOneToEven]rank[%u] tx ack from peerank[%u] failed", rank, peerRank), ret);
130 0 : ret = links[peerRank]->RxAck(stream_);
131 0 : CHK_PRT_RET(
132 : ret != HCCL_SUCCESS,
133 : HCCL_ERROR("[Reduce][InPartOneToEven]rank[%u] rx ack from peerank[%u] failed", rank, peerRank), ret);
134 : // 接收数据到本端的input
135 0 : HCCL_DEBUG("send mem[%p] size[%llu] to peerank[%u]", scratchMem_.ptr(), scratchMem_.size(), peerRank);
136 0 : ret = links[peerRank]->TxAsync(UserMemType::INPUT_MEM, baseOffset_, scratchMem_.ptr(), 0, stream_);
137 0 : CHK_PRT_RET(
138 : ret != HCCL_SUCCESS,
139 : HCCL_ERROR(
140 : "[Reduce][InPartOneToEven]TxAsync: tx async size[%llu] "
141 : "failed",
142 : 0),
143 : ret);
144 0 : CHK_RET(reducerInfo_->run(
145 : dispatcher_, links[peerRank], baseOffset_, inputMem_, inputMem_, scratchMem_, stream_));
146 0 : ret = links[peerRank]->RxWaitDone(stream_);
147 0 : CHK_PRT_RET(ret != HCCL_SUCCESS, HCCL_ERROR("[Reduce][InPartOne]RxWaitDone failed"), ret);
148 : }
149 0 : } else if (rank < part1Size_ && rank % 2 == 1) { // 向上一个rank的input发数据 2
150 0 : u32 peerRank = rank - 1;
151 0 : if (peerRank < links.size()) {
152 0 : CHK_SMART_PTR_NULL(links[peerRank]);
153 0 : HcclResult ret = links[peerRank]->TxAck(stream_);
154 0 : CHK_PRT_RET(
155 : ret != HCCL_SUCCESS,
156 : HCCL_ERROR("[Reduce][InPartOneToEven]rank[%u] tx ack from peerank[%u] failed", rank, peerRank), ret);
157 0 : ret = links[peerRank]->RxAck(stream_);
158 0 : CHK_PRT_RET(
159 : ret != HCCL_SUCCESS,
160 : HCCL_ERROR("[Reduce][InPartOneToEven]rank[%u] rx ack from peerank[%u] failed", rank, peerRank), ret);
161 : // 发送到对端的input
162 0 : HCCL_DEBUG(
163 : "rank[%u] sends inputMem[%p] to peerrank[%u] offset[%llu], size[%llu]", rank, inputMem_.ptr(), peerRank,
164 : baseOffset_, inputMem_.size());
165 0 : ret = senderInfo_->run(links[peerRank], baseOffset_, inputMem_, stream_);
166 0 : CHK_PRT_RET(
167 : ret != HCCL_SUCCESS, HCCL_ERROR("[Reduce][InPartOne]tx sync to peerank[%u] failed", peerRank), ret);
168 0 : ret = links[peerRank]->RxAsync(UserMemType::OUTPUT_MEM, baseOffset_, inputMem_.ptr(), 0, stream_);
169 0 : CHK_PRT_RET(
170 : ret != HCCL_SUCCESS,
171 : HCCL_ERROR("[AlgTemplateBase][ExecuteTxSync]ExecuteTxSync: rx async size[%llu] failed", 0), ret);
172 0 : ret = links[peerRank]->DataReceivedAck(stream_);
173 0 : CHK_PRT_RET(
174 : ret != HCCL_SUCCESS,
175 : HCCL_ERROR("[AlgTemplateBase][ExecuteTxSync]ExecuteTxSync: data received ack failed"), ret);
176 0 : ret = links[peerRank]->TxWaitDone(stream_);
177 0 : CHK_PRT_RET(ret != HCCL_SUCCESS, HCCL_ERROR("[Reduce][InPartOne]TxWaitDone failed"), ret);
178 : }
179 : }
180 :
181 0 : return HCCL_SUCCESS;
182 : }
183 :
184 : HcclResult
185 0 : ReduceScatterRecursiveHalvingDoubling::ReduceScatterInBlock(u32 rank, u32 rankSize, const std::vector<LINK>& links)
186 : {
187 0 : u32 rankInBlock = 0;
188 0 : if (rank < part1Size_ && (rank % 2) == 1) { // rank号对2求余，rank为奇数
189 0 : return HCCL_SUCCESS;
190 0 : } else if (rank < part1Size_ && (rank % 2) == 0) { // rank对2求余，rank为偶数
191 0 : rankInBlock = rank / 2; // 直接除以2即为本rank的在block内的排序
192 : } else {
193 0 : rankInBlock = rank - part1Size_ / 2; // 通过rank减去part1除2的大小即不处于第一部分的block内rank号
194 : }
195 :
196 : std::unique_ptr<AlgTemplateBase> executor
197 0 : = AlgTemplateRegistry::Instance().GetAlgTemplate(TemplateType::TEMPLATE_REDUCESCATTER_HD, dispatcher_);
198 0 : CHK_SMART_PTR_NULL(executor);
199 0 : CHK_RET(executor->Prepare(
200 : inputMem_, inputMem_, scratchMem_, count_, dataType_, stream_, reductionOp_, root_, slices_, baseOffset_,
201 : blockSize_, reduceAttr, UserMemType::INPUT_MEM, UserMemType::OUTPUT_MEM));
202 :
203 0 : CHK_RET(executor->RegisterProfiler(profilerInput_.planeID, profilerInput_.stage, profilerInput_.step, stream_));
204 :
205 0 : std::vector<LINK> subLinks;
206 0 : CHK_RET(BuildSubLinks(links, subLinks, rankSize));
207 :
208 0 : CHK_PRT_RET(
209 : subLinks.size() == 0,
210 : HCCL_ERROR(
211 : "[ReduceScatterRecursiveHalvingDoubling][ReduceScatterInBlock]rank[%u] "
212 : "build sub links failed",
213 : rank),
214 : HCCL_E_PARA);
215 0 : CHK_RET(executor->RunAsync(rankInBlock, blockSize_, subLinks));
216 0 : return HCCL_SUCCESS;
217 0 : }
218 :
219 : HcclResult
220 0 : ReduceScatterRecursiveHalvingDoubling::ScatterInPartOne(u32 rank, u32 rankSize, const std::vector<LINK>& links)
221 : {
222 0 : u32 bytesPerData = DataUnitSize(dataType_);
223 0 : u64 dataBytes = count_ * bytesPerData;
224 0 : u64 bytesPerSlice = dataBytes / rankSize;
225 :
226 0 : if (rank < part1Size_ && rank % 2 == 0) { // 模2计算奇偶性，偶数rank把自己第二份数据给下一个奇数rank
227 0 : u32 peerRank = rank + 1;
228 0 : if (peerRank < links.size()) {
229 0 : CHK_SMART_PTR_NULL(links[peerRank]);
230 0 : HcclResult ret = links[peerRank]->TxAck(stream_);
231 0 : CHK_PRT_RET(
232 : ret != HCCL_SUCCESS,
233 : HCCL_ERROR("[Scatter][InPartOneToEven]rank[%u] tx ack from peerank[%u] failed", rank, peerRank), ret);
234 0 : ret = links[peerRank]->RxAck(stream_);
235 0 : CHK_PRT_RET(
236 : ret != HCCL_SUCCESS,
237 : HCCL_ERROR("[Scatter][InPartOneToEven]rank[%u] rx ack from peerank[%u] failed", rank, peerRank), ret);
238 : // 发送到对端的input
239 0 : HCCL_DEBUG(
240 : "rank[%u] sends inputmem[%p] to peerrank[%u] Offset[%llu], Size[%llu]", rank, inputMem_.ptr(), peerRank,
241 : baseOffset_, inputMem_.size());
242 :
243 0 : u64 offset = peerRank * bytesPerSlice; // 计算对端rank的slice偏移
244 0 : void* srcAddr = reinterpret_cast<s8*>(inputMem_.ptr()) + offset;
245 0 : ret = ExecuteTxSync(
246 0 : links[peerRank], UserMemType::INPUT_MEM, offset + baseOffset_, srcAddr, bytesPerSlice, stream_);
247 0 : CHK_PRT_RET(
248 : ret != HCCL_SUCCESS, HCCL_ERROR("[Scatter][InPartOne]tx sync to peerank[%u] failed", peerRank), ret);
249 0 : ret = links[peerRank]->TxWaitDone(stream_);
250 0 : CHK_PRT_RET(ret != HCCL_SUCCESS, HCCL_ERROR("[Reduce][InPartOne]TxWaitDone failed"), ret);
251 : }
252 0 : } else if (rank < part1Size_ && rank % 2 == 1) { // 模2计算奇偶性，奇数rank接收偶数rank发过来下半份的数据
253 0 : u32 peerRank = rank - 1;
254 0 : if (peerRank < links.size()) {
255 0 : CHK_SMART_PTR_NULL(links[peerRank]);
256 0 : HcclResult ret = links[peerRank]->TxAck(stream_);
257 0 : CHK_PRT_RET(
258 : ret != HCCL_SUCCESS,
259 : HCCL_ERROR("[Scatter][InPartOneToEven]rank[%u] tx ack from peerank[%u] failed", rank, peerRank), ret);
260 0 : ret = links[peerRank]->RxAck(stream_);
261 0 : CHK_PRT_RET(
262 : ret != HCCL_SUCCESS,
263 : HCCL_ERROR("[Scatter][InPartOneToEven]rank[%u] rx ack from peerank[%u] failed", rank, peerRank), ret);
264 : // 接收数据到本端的 inputMem_
265 0 : HCCL_DEBUG("rx mem[%p] size[%llu] from peerank[%u]", inputMem_.ptr(), inputMem_.size(), peerRank);
266 :
267 0 : u64 offset = rank * bytesPerSlice; // 本rank slice偏移
268 0 : void* dstAddr = reinterpret_cast<s8*>(inputMem_.ptr()) + offset;
269 0 : ret = ExecuteRxSync(
270 0 : links[peerRank], UserMemType::INPUT_MEM, offset + baseOffset_, dstAddr, bytesPerSlice, stream_);
271 0 : CHK_PRT_RET(
272 : ret != HCCL_SUCCESS, HCCL_ERROR("[Scatter][InPartOne]rx sync from peerank[%u] failed", peerRank), ret);
273 0 : ret = links[peerRank]->RxWaitDone(stream_);
274 0 : CHK_PRT_RET(ret != HCCL_SUCCESS, HCCL_ERROR("[Reduce][InPartOne]RxWaitDone failed"), ret);
275 : }
276 : }
277 :
278 0 : return HCCL_SUCCESS;
279 : }
280 :
281 0 : HcclResult ReduceScatterRecursiveHalvingDoubling::GetNslbAdjInfo(
282 : const u32 rank, const u32 rankSize, const std::vector<LINK>& links, AdjInfo& nslbAdjInfo)
283 : {
284 0 : u32 nslbRound = 0;
285 0 : u32 base = 1;
286 0 : const u32 minExponent = 1;
287 0 : while ((base << nslbRound) <= rankSize) {
288 0 : nslbRound++;
289 : }
290 0 : if (nslbRound >= minExponent) {
291 0 : nslbRound = nslbRound - minExponent;
292 : }
293 0 : u32 nslbBlockSize = base << nslbRound;
294 : // 获取第一部分：rank数减block数乘2
295 0 : u32 nslbPart1Size = (rankSize - nslbBlockSize) * 2;
296 : // 2的次幂场景下处理流程
297 0 : if (nslbPart1Size == 0) {
298 0 : u32 stepNum = 0;
299 0 : while ((rankSize >> (stepNum + 1)) != 0) {
300 0 : stepNum++;
301 : }
302 0 : for (u32 step = 0; step < stepNum; step++) {
303 0 : u32 peerRankBitmask = 1 << (stepNum - step - 1);
304 0 : u32 peerRank = rank ^ peerRankBitmask;
305 0 : NslbDpAdjInfo adjInfoStep = {};
306 0 : u32 remoteuserRank = links[peerRank]->GetRemoteRank();
307 0 : adjInfoStep.dstLocalRankId = remoteuserRank;
308 0 : adjInfoStep.phaseId = step + 1;
309 0 : adjInfoStep.rev = 0;
310 0 : nslbAdjInfo.nsAdjInfo.push_back(adjInfoStep);
311 : }
312 0 : nslbAdjInfo.dstRankNum = stepNum;
313 0 : return HCCL_SUCCESS;
314 : }
315 : // 非2的次幂场景下，被合并部分的奇数rank处理流程
316 0 : if (rank < nslbPart1Size && rank % NSLBDP_REDUCE_SCATTER_MOLD2 == 1) {
317 0 : u32 peerRank = rank - 1;
318 0 : if (peerRank < links.size()) {
319 0 : NslbDpAdjInfo adjInfoStep = {};
320 0 : adjInfoStep.dstLocalRankId = links[peerRank]->GetRemoteRank();
321 0 : adjInfoStep.phaseId = 1;
322 0 : adjInfoStep.rev = 0;
323 0 : HCCL_INFO("AllGatherHDR-nslb: peerRank[%u]", peerRank);
324 0 : nslbAdjInfo.nsAdjInfo.push_back(adjInfoStep);
325 0 : nslbAdjInfo.dstRankNum = 1;
326 : }
327 0 : return HCCL_SUCCESS;
328 : }
329 : // 针对合并后映射成2的次幂场景处理
330 0 : u32 rankInBlock = 0;
331 0 : if (rank < nslbPart1Size && (rank % NSLBDP_REDUCE_SCATTER_MOLD2) == 0) {
332 0 : rankInBlock = rank / NSLBDP_REDUCE_SCATTER_MOLD2; // 直接除以2即为本rank的在block内的排序
333 : } else {
334 0 : rankInBlock = rank
335 : - nslbPart1Size
336 0 : / NSLBDP_REDUCE_SCATTER_MOLD2; // 通过rank减去part1除2的大小即不处于第一部分的block内rank号
337 : }
338 0 : std::vector<LINK> subLinks;
339 0 : std::vector<LINK>::const_iterator iter = links.begin();
340 0 : subLinks.resize(nslbBlockSize);
341 0 : for (u32 i = 0; i < rankSize; i++) {
342 0 : if (i < nslbPart1Size
343 0 : && (i % NSLBDP_REDUCE_SCATTER_MOLD2) == 1) { // 模2余1代表当前rank在part1的奇数位置上，不参与block内的建链
344 0 : continue;
345 0 : } else if (
346 0 : i < nslbPart1Size && (i % NSLBDP_REDUCE_SCATTER_MOLD2) == 0) { // 模2余0代表当前rank在part1的偶数位置上
347 0 : std::vector<LINK>::const_iterator niter = std::next(iter, i);
348 0 : if (niter != links.end()) {
349 0 : subLinks[i / NSLBDP_REDUCE_SCATTER_MOLD2] = *niter;
350 : }
351 0 : } else {
352 0 : std::vector<LINK>::const_iterator niter = std::next(iter, i);
353 0 : if (niter != links.end()) {
354 0 : subLinks[i - nslbPart1Size / NSLBDP_REDUCE_SCATTER_MOLD2] = *niter;
355 : }
356 : }
357 : }
358 0 : u32 stepNum = 0;
359 0 : while ((rankSize >> (stepNum + 1)) != 0) {
360 0 : stepNum++;
361 : }
362 : // 映射完成后针对以新的通信域进行邻接表获取
363 0 : u32 begin = 1;
364 0 : for (u32 step = 0; step < stepNum; step++) {
365 0 : u32 peerRankBitmask = 1 << (stepNum - step - 1);
366 0 : u32 peerRank = rankInBlock ^ peerRankBitmask;
367 0 : if (subLinks[peerRank] == nullptr) {
368 0 : continue;
369 : }
370 0 : NslbDpAdjInfo adjInfoStep = {};
371 0 : u32 remoteuserRank = subLinks[peerRank]->GetRemoteRank();
372 0 : adjInfoStep.dstLocalRankId = remoteuserRank;
373 0 : adjInfoStep.phaseId = step + begin + 1;
374 0 : adjInfoStep.rev = 0;
375 0 : nslbAdjInfo.nsAdjInfo.push_back(adjInfoStep);
376 : }
377 0 : nslbAdjInfo.dstRankNum = nslbAdjInfo.nsAdjInfo.size();
378 :
379 0 : if (nslbAdjInfo.nsAdjInfo.size() == 0) {
380 0 : return HCCL_SUCCESS;
381 : }
382 : // 上面处理完成后，紧接着处理合并部分的偶数rank同步到奇数rank增加phaseId
383 0 : if (rank < nslbPart1Size && rank % NSLBDP_REDUCE_SCATTER_MOLD2 == 0) {
384 0 : u32 peerRank = rank + 1;
385 0 : uint16_t phaseSize = nslbAdjInfo.nsAdjInfo.size();
386 0 : if (peerRank < links.size()) {
387 0 : NslbDpAdjInfo adjInfoStep = {};
388 0 : adjInfoStep.dstLocalRankId = links[peerRank]->GetRemoteRank();
389 0 : adjInfoStep.phaseId = nslbAdjInfo.nsAdjInfo[phaseSize - 1].phaseId + 1;
390 0 : adjInfoStep.rev = 0;
391 0 : HCCL_INFO("Scatter-nslb: peerRank[%u]", peerRank);
392 0 : nslbAdjInfo.nsAdjInfo.push_back(adjInfoStep);
393 0 : nslbAdjInfo.dstRankNum = nslbAdjInfo.nsAdjInfo.size();
394 : }
395 0 : return HCCL_SUCCESS;
396 : }
397 0 : return HCCL_SUCCESS;
398 0 : }
399 : REGISTER_TEMPLATE(TemplateType::TEMPLATE_REDUCESCATTER_RECURSIVE_HD, ReduceScatterRecursiveHalvingDoubling);
400 : } // namespace hccl
|
