Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "reduce_scatter_operator.h"
12 : #include "device_capacity.h"
13 : #include "coll_alg_utils.h"
14 : #include "hccl_aiv.h"
15 : #include "coll_alg_op_registry.h"
16 : #include <algorithm>
17 :
18 : constexpr u32 MODULE_NUM_FOUR = 4;
19 : // 保序规约仅支持多于2卡场景(2卡时收发对端唯一, 无规约顺序差异)
20 : constexpr u32 DETERMINISTIC_MIN_USER_RANK_SIZE = 2;
21 : constexpr u32 HCCL_310P_DATA_SIZE_MID_COUNT = 320 * 1024;
22 : constexpr u32 HCCL_310P_DATA_SIZE_SMALL_COUNT = 1024;
23 : constexpr u32 HCCL_310P_SLIM_RING_MAX_SIZE = 8;
24 :
25 : // Pipeline并行比串行更优的总数据量临界点（基于910_93 2 SuperPod, 1 Server, 12 Rank 性能基线测试，
26 : // 数据量低于此值时调度开销超过流水收益）
27 : constexpr u64 HCCL_PIPELINE_TOTAL_DATA_SIZE_THRESHOLD = 608 * 1024 * 1024;
28 :
29 : namespace hccl {
30 :
31 30 : ReduceScatterOperator::ReduceScatterOperator(
32 : AlgConfigurator* algConfigurator, CCLBufferManager& cclBufferManager, HcclDispatcher dispatcher,
33 30 : std::unique_ptr<TopoMatcher>& topoMatcher)
34 30 : : CollAlgOperator(algConfigurator, cclBufferManager, dispatcher, topoMatcher, HcclCMDType::HCCL_CMD_REDUCE_SCATTER)
35 31 : {}
36 :
37 74 : ReduceScatterOperator::~ReduceScatterOperator() {}
38 :
39 1 : HcclResult ReduceScatterOperator::SelectAlg(
40 : const std::string& tag, const OpParam& param, std::string& algName, std::string& newTag)
41 : {
42 1 : ResourceLimit limit;
43 2 : return SelectAlg(tag, param, algName, newTag, limit);
44 : }
45 :
46 33 : HcclResult ReduceScatterOperator::SelectAlg(
47 : const std::string& tag, const OpParam& param, std::string& algName, std::string& newTag, const ResourceLimit& limit)
48 : {
49 33 : if (userRankSize_ == 1) {
50 1 : algName = "ReduceScatterSingleExecutor";
51 1 : return HCCL_SUCCESS;
52 : }
53 : HcclResult ret;
54 32 : if (isDiffDeviceType_) {
55 0 : ret = SelectAlgforMix(param, algName);
56 32 : } else if (deviceType_ == DevType::DEV_TYPE_310P3) {
57 0 : ret = SelectAlgfor310P3(param, algName);
58 32 : } else if (deviceType_ == DevType::DEV_TYPE_910) {
59 0 : ret = SelectAlgfor910A(param, algName);
60 32 : } else if (deviceType_ == DevType::DEV_TYPE_910B) {
61 31 : ret = SelectAlgfor910B(param, algName);
62 1 : } else if (deviceType_ == DevType::DEV_TYPE_910_93) {
63 4 : ret = SelectAlgfor91093(param, algName, limit);
64 : } else {
65 0 : HCCL_ERROR("[SelectAlg] device type[%d] is out of range for selector.", deviceType_);
66 0 : return HCCL_E_NOT_SUPPORT;
67 : }
68 35 : CHK_PRT_RET(
69 : ret != HCCL_SUCCESS,
70 : HCCL_ERROR("[ReduceScatterSelector][SelectAlg]tag[%s], ReduceScatter failed, return[%d]", tag.c_str(), ret),
71 : ret);
72 :
73 35 : if (GetWorkflowMode() == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OPS_KERNEL_INFO_LIB) {
74 13 : newTag = tag;
75 : } else {
76 21 : if (deviceType_ == DevType::DEV_TYPE_310P3) {
77 0 : newTag = tag + algName;
78 : } else {
79 21 : auto level1Iter = HCCL_ALGO_LEVEL1_NAME_MAP.find(algType_.algoLevel1);
80 19 : CHK_PRT_RET(
81 : level1Iter == HCCL_ALGO_LEVEL1_NAME_MAP.end(),
82 : HCCL_ERROR("level1: algType1[%u] is invalid.", algType_.algoLevel1), HCCL_E_INTERNAL);
83 19 : newTag = tag + level1Iter->second + algName;
84 : }
85 :
86 21 : bool isInlineReduce = IsSupportSDMAReduce(
87 22 : cclBufferManager_.GetInCCLbuffer().ptr(), cclBufferManager_.GetOutCCLbuffer().ptr(), param.DataDes.dataType,
88 22 : param.reduceType);
89 21 : bool isRdmaReduce = IsSupportRDMAReduce(param.DataDes.dataType, param.reduceType);
90 21 : const std::string REDUCE_SCATTER_NO_INLINE = "_no_inline";
91 33 : newTag = (isInlineReduce && isRdmaReduce) ? newTag : newTag + REDUCE_SCATTER_NO_INLINE;
92 23 : }
93 36 : if (algName == "ReduceScatterARSFor91093Executor") {
94 0 : u32 ringSize = CalcOptimalIntraRingsize(
95 0 : param.DataDes.count, param.DataDes.dataType, HcclCMDType::HCCL_CMD_REDUCE_SCATTER);
96 0 : newTag += std::to_string(ringSize);
97 : }
98 34 : newTag += (param.aicpuUnfoldMode ? "_device" : "_host");
99 34 : return ret;
100 : }
101 :
102 0 : HcclResult ReduceScatterOperator::SelectAlgforMix(const OpParam& param, std::string& algName)
103 : {
104 : (void)param;
105 :
106 : // 混合组网场景不支持规约保序
107 0 : if (IsNeedStrictMode(param)) {
108 0 : HCCL_ERROR("[ReduceScatterOperator][SelectAlgforMix] not support DETERMINISTIC_STRICT mode.");
109 0 : return HCCL_E_NOT_SUPPORT;
110 : }
111 :
112 0 : if (gcdDeviceNumPerAggregation_ > 1) {
113 0 : algType_.algoLevel1 = AlgTypeLevel1::ALG_LEVEL1_NHR;
114 0 : HCCL_WARNING("[ReduceScatterOperator][SelectAlgforMix] only support NHR in AlgoLevel1 yet, "
115 : "default is algType=NHR.");
116 0 : algName = "ReduceScatterMixExecutor";
117 : } else {
118 0 : algType_.algoLevel1 = AlgTypeLevel1::ALG_LEVEL1_RING;
119 : ;
120 0 : HCCL_WARNING("[ReduceScatterOperator][SelectAlgforMix] only support ring in AlgoComm yet, "
121 : "default is algType=ring.");
122 0 : algName = "ReduceScatterComm";
123 : }
124 :
125 0 : HCCL_INFO("[SelectAlgforMix] ReduceScatter SelectAlgforMix is algName [%s].", algName.c_str());
126 0 : return HCCL_SUCCESS;
127 : }
128 :
129 0 : HcclResult ReduceScatterOperator::SelectAlgfor310P3(const OpParam& param, std::string& algName)
130 : {
131 0 : if (HCCL_310P_DATA_SIZE_SMALL_COUNT < param.DataDes.count && param.DataDes.count <= HCCL_310P_DATA_SIZE_MID_COUNT
132 0 : && userRankSize_ <= HCCL_310P_SLIM_RING_MAX_SIZE) {
133 0 : algName = "ReduceScatterSlimRing";
134 : } else {
135 0 : algName = "ReduceScatterRing";
136 : }
137 :
138 0 : HCCL_INFO(
139 : "[SelectAlgfor310P3] ReduceScatter SelectAlgfor310P3 is algName [%s] DataDesCount [%llu]", algName.c_str(),
140 : param.DataDes.count);
141 0 : return HCCL_SUCCESS;
142 : }
143 :
144 0 : HcclResult ReduceScatterOperator::SelectAlgfor910A(const OpParam& param, std::string& algName)
145 : {
146 0 : bool isMeshTopo = topoType_ == TopoType::TOPO_TYPE_4P_MESH || topoType_ == TopoType::TOPO_TYPE_2P_MESH;
147 0 : bool isRingTopo = topoType_ == TopoType::TOPO_TYPE_NP_SINGLE_RING || topoType_ == TopoType::TOPO_TYPE_8P_RING;
148 :
149 0 : const u32 RANK_SIZE_FOUR = 4;
150 0 : const u32 RANK_SIZE_EIGHT = 8;
151 0 : bool isOpbase = GetWorkflowMode() == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE;
152 0 : u64 dataSize = SIZE_TABLE[param.DataDes.dataType] * param.DataDes.count;
153 0 : if (isOpbase && serverNum_ == 1 && dataSize <= HCCL_SMALL_COUNT_256_KB
154 0 : && (userRankSize_ == RANK_SIZE_FOUR || userRankSize_ == RANK_SIZE_EIGHT)) {
155 0 : algType_.algoLevel1 = AlgTypeLevel1::ALG_LEVEL1_HD;
156 0 : algName = "ReduceScatterComm";
157 0 : } else if (isMeshTopo) {
158 0 : algName = "ReduceScatterMeshExecutor";
159 0 : } else if (isRingTopo) {
160 0 : algName = "ReduceScatterRingExecutor";
161 : } else {
162 0 : algName = "ReduceScatterComm";
163 : }
164 0 : HCCL_INFO("[SelectAlgfor910A] ReduceScatter SelectAlgfor910A is algName [%s].", algName.c_str());
165 0 : return HCCL_SUCCESS;
166 : }
167 :
168 28 : HcclResult ReduceScatterOperator::SelectAlgfor910B(const OpParam& param, std::string& algName)
169 : {
170 28 : u32 unitSize = SIZE_TABLE[param.DataDes.dataType];
171 :
172 28 : bool isOnlyAiv = topoMatcher_->GetIsOnlyAivConfig();
173 0 : bool isMeshTopo = topoType_ == TopoType::TOPO_TYPE_NP_MESH || topoType_ == TopoType::TOPO_TYPE_4P_MESH
174 29 : || topoType_ == TopoType::TOPO_TYPE_2P_MESH || topoType_ == TopoType::TOPO_TYPE_1P_MESH;
175 29 : bool isRingTopo = topoType_ == TopoType::TOPO_TYPE_NP_SINGLE_RING;
176 :
177 29 : u64 dataSize = param.DataDes.count * unitSize; // 单位：字节
178 29 : u64 cclBufferSize = cclBufferManager_.GetInCCLbufferSize() / userRankSize_;
179 :
180 31 : void* commInputPtr = nullptr;
181 31 : void* commOutputPtr = nullptr;
182 31 : u64 commInputSize = 0;
183 31 : u64 commOutputSize = 0;
184 :
185 31 : CHK_RET(cclBufferManager_.GetInCCLbuffer(commInputPtr, commInputSize));
186 30 : CHK_RET(cclBufferManager_.GetOutCCLbuffer(commOutputPtr, commOutputSize));
187 30 : bool isServNumPowOfTwo = (serverNum_ > 0) && ((serverNum_ & (serverNum_ - 1)) == 0);
188 30 : bool isOpbase = (GetWorkflowMode() == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE);
189 30 : bool isInlineReduce = IsSupportSDMAReduce(
190 31 : cclBufferManager_.GetInCCLbuffer().ptr(), cclBufferManager_.GetOutCCLbuffer().ptr(), param.DataDes.dataType,
191 29 : param.reduceType);
192 :
193 29 : if (topoMatcher_->GetDeterministicConfig() == DETERMINISTIC_STRICT
194 30 : && userRankSize_ > DETERMINISTIC_MIN_USER_RANK_SIZE) {
195 0 : if (multiModuleDiffDeviceNumMode_) {
196 : // 保序规约场景（多batch一致），当前不支持A2非对称场景
197 0 : HCCL_ERROR(
198 : "[SelectAlgfor910B] reduce order preservation only support"
199 : " Symmetry(multiModuleDiffDeviceNumMode_[%d]).",
200 : multiModuleDiffDeviceNumMode_);
201 0 : return HCCL_E_NOT_SUPPORT;
202 : }
203 0 : if (param.DataDes.dataType == HCCL_DATA_TYPE_FP16 || param.DataDes.dataType == HCCL_DATA_TYPE_FP32
204 0 : || param.DataDes.dataType == HCCL_DATA_TYPE_BFP16) {
205 : // 只有浮点数存在多batch不一致的可能，整数天然一致
206 0 : if (param.aicpuUnfoldMode || topoMatcher_->GetAivModeConfig()) {
207 : // AIV / AICPU场景，规约保序优先级更高
208 0 : HCCL_WARNING(
209 : "[SelectAlgfor910B]aicpuMode[%d], AivModeConfig[%d], "
210 : "the AIV/AICPU mode does not support when the reduce order preservation is enabled.",
211 : param.aicpuUnfoldMode, topoMatcher_->GetAivModeConfig());
212 : }
213 0 : algName = "ReduceScatterOrderPreservedExecutor";
214 0 : HCCL_INFO("[SelectAlgfor910B] ReduceScatterSelectAlgfor910B is algName [%s].", algName.c_str());
215 0 : return HCCL_SUCCESS;
216 : }
217 : }
218 :
219 : // 暂只支持单算子模式
220 30 : bool isCCLBufferGE16M
221 30 : = isOpbase && (commInputSize >= HCCL_MID_COUNT_16_MB && commOutputSize >= HCCL_MID_COUNT_16_MB);
222 :
223 0 : bool isSupportAivRdmaCount = !isSingleMeshAggregation_ && !multiModuleDiffDeviceNumMode_ && isMeshTopo
224 30 : && (((isServNumPowOfTwo || dataSize <= HCCL_SMALL_COUNT_128_KB)
225 0 : && dataSize * userRankSize_ <= HCCL_MID_COUNT_16_MB && isCCLBufferGE16M
226 0 : && dataSize <= HCCL_SMALL_COUNT_256_KB)
227 0 : || isOnlyAiv);
228 :
229 30 : bool isSupportAivDeter = isSingleMeshAggregation_ && serverNum_ == 1
230 30 : && (topoMatcher_->GetDeterministicConfig() == DETERMINISTIC_ENABLE)
231 58 : && ((dataSize * userRankSize_ <= HCCL_SMALL_COUNT_8_MB) || isOnlyAiv);
232 :
233 : bool isAivMode
234 28 : = topoMatcher_->GetAivModeConfig() && IsSupportAIVReduce(param.DataDes.dataType, param.reduceType)
235 0 : && (topoMatcher_->GetDeterministicConfig() == DETERMINISTIC_DISABLE || isSupportAivDeter)
236 27 : && ((isSingleMeshAggregation_ && (dataSize <= AIV_BIG_SIZE || isOnlyAiv)) || isSupportAivRdmaCount);
237 27 : if (isAivMode) {
238 0 : if (isSupportAivDeter) {
239 0 : if (dataSize * userRankSize_ <= HCCL_SMALL_COUNT_8_MB) {
240 0 : algName = "ReduceScatterAivDeterSmallExecutor";
241 : } else {
242 0 : algName = "ReduceScatterAivDeterExecutor";
243 : }
244 0 : HCCL_INFO("[SelectAlgfor910B] ReduceScatter SelectAlgfor910B is algName [%s].", algName.c_str());
245 0 : return HCCL_SUCCESS;
246 : }
247 0 : if (isSupportAivRdmaCount) {
248 0 : algName = "ReduceScatterAivRdmaExecutor";
249 0 : } else if (
250 0 : workflowMode_ == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE && dataSize <= AIV_REDUCE_SCATTER_MID_SIZE) {
251 0 : algName = "ReduceScatterMeshAivSmallCountExecutor";
252 : } else {
253 0 : algName = "ReduceScatterMeshAivExecutor";
254 : }
255 0 : HCCL_INFO("[SelectAlgfor910BAIV] ReduceScatterSelectAlgfor910B is algName [%s].", algName.c_str());
256 0 : return HCCL_SUCCESS;
257 : }
258 :
259 27 : if (GetWorkflowMode() == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE) {
260 17 : bool isRdmaReduce = IsSupportRDMAReduce(param.DataDes.dataType, param.reduceType);
261 :
262 17 : std::string algTypeLevel1Tag;
263 17 : CHK_RET(AutoSelectAlgTypeLevel1(
264 : HcclCMDType::HCCL_CMD_REDUCE_SCATTER, dataSize, cclBufferSize, algTypeLevel1Tag, isInlineReduce,
265 : isRdmaReduce));
266 20 : if (GetExternalInputHcclEnableEntryLog() && param.opBaseAtraceInfo != nullptr) {
267 0 : CHK_RET(param.opBaseAtraceInfo->SavealgtypeTraceInfo(algTypeLevel1Tag, param.tag));
268 : }
269 20 : }
270 :
271 : // AHC 算法选择逻辑
272 30 : if (((algType_.algoLevel1 == AlgTypeLevel1::ALG_LEVEL1_AHC)
273 30 : || (algType_.algoLevel1 == AlgTypeLevel1::ALG_LEVEL1_AHC_BROKE))) {
274 0 : CHK_RET(SelectAlgforAHC(dataSize, AHCOpType::AHC_OP_TYPE_REDUCE_SCATTER));
275 : }
276 :
277 : // pipeline算法task数量多，如果超出FFTS子图限制，则重定向到HD算法
278 30 : if (algType_.algoLevel1 == AlgTypeLevel1::ALG_LEVEL1_PIPELINE) {
279 0 : u32 contextNum = CalcContextNumForPipeline(HcclCMDType::HCCL_CMD_REDUCE_SCATTER);
280 0 : if (contextNum > HCCL_FFTS_CAPACITY) {
281 0 : algType_.algoLevel1 = AlgTypeLevel1::ALG_LEVEL1_HD;
282 0 : HCCL_WARNING(
283 : "[ReduceScatterOperator][SelectAlgfor910B] context num[%u] is out of capacity of FFTS+ "
284 : "graph[%u], reset algorithm to HD.",
285 : contextNum, HCCL_FFTS_CAPACITY);
286 : }
287 : }
288 :
289 30 : if (isMeshTopo) {
290 30 : if (topoMatcher_->GetDeterministicConfig() == DETERMINISTIC_ENABLE
291 9 : && GetWorkflowMode() == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE
292 37 : && algType_.algoLevel1 == AlgTypeLevel1::ALG_LEVEL1_PIPELINE && deviceNumPerAggregation_ > DEVICE_TWO) {
293 0 : algName = "ReduceScatterDeterPipelineExecutor";
294 28 : } else if (GetWorkflowMode() == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE) {
295 : bool enableSmallCountDeterministicAlgo
296 18 : = !isSingleMeshAggregation_
297 18 : && IsSupportSDMAReduce(
298 0 : cclBufferManager_.GetInCCLbuffer().ptr(), cclBufferManager_.GetOutCCLbuffer().ptr(),
299 0 : param.DataDes.dataType, param.reduceType);
300 17 : if (SingleMeshInlineReduce(
301 18 : cclBufferManager_.GetInCCLbuffer().ptr(), cclBufferManager_.GetOutCCLbuffer().ptr(),
302 18 : param.DataDes.dataType, param.reduceType)) {
303 10 : if (topoMatcher_->GetDeterministicConfig() != DETERMINISTIC_DISABLE) {
304 2 : algName = "ReduceScatterDeterExecutor";
305 : } else {
306 8 : algName = "ReduceScatterMeshDmaEliminationExecutor";
307 : }
308 8 : } else if (
309 8 : topoMatcher_->GetDeterministicConfig() == DETERMINISTIC_DISABLE
310 4 : && algType_.algoLevel1 == AlgTypeLevel1::ALG_LEVEL1_PIPELINE
311 12 : && IsMultiMeshInlineReduce(
312 0 : cclBufferManager_.GetInCCLbuffer().ptr(), cclBufferManager_.GetOutCCLbuffer().ptr(),
313 0 : param.DataDes.dataType, param.reduceType)) {
314 0 : algName = "ReduceScatterMeshOpbasePipelineExecutor";
315 8 : } else if (
316 : enableSmallCountDeterministicAlgo
317 8 : && ((dataSize <= HCCL_SMALL_COUNT_512_KB
318 0 : && topoMatcher_->GetDeterministicConfig() == DETERMINISTIC_ENABLE)
319 0 : || dataSize * userRankSize_ < HCCL_SMALL_COUNT_512_KB)) {
320 0 : algName = "ReduceScatterMeshOpbaseSmallCountDeterministicExecutor";
321 : }
322 : } else {
323 10 : if (SingleMeshInlineReduce(param.inputPtr, param.outputPtr, param.DataDes.dataType, param.reduceType)) {
324 10 : if (topoMatcher_->GetDeterministicConfig() != DETERMINISTIC_DISABLE
325 10 : && deviceNumPerAggregation_ > DEVICE_TWO) {
326 3 : algName = "ReduceScatterDeterExecutor";
327 : } else {
328 7 : if (dataSize <= HCCL_SMALL_COUNT_1_MB
329 7 : && workflowMode_ == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OPS_KERNEL_INFO_LIB) {
330 7 : algName = "ReduceScatterMeshGraphExecutor";
331 : } else {
332 0 : algName = "ReduceScatterMeshExecutor";
333 : }
334 : }
335 : }
336 : }
337 32 : if (algName.empty()) {
338 0 : if (workflowMode_ == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OPS_KERNEL_INFO_LIB && moduleNum_ > 1
339 0 : && deviceNumPerAggregation_ > 1 && topoMatcher_->GetDeterministicConfig() == DETERMINISTIC_DISABLE
340 0 : && IsMultiMeshInlineReduce(param.inputPtr, param.outputPtr, param.DataDes.dataType, param.reduceType)
341 8 : && (dataSize > HCCL_SMALL_COUNT_1_MB || moduleNum_ <= MODULE_NUM_FOUR
342 0 : || algType_.algoLevel1 == AlgTypeLevel1::ALG_LEVEL1_PIPELINE)) {
343 0 : algName = "ReduceScatterMeshGraphPipelineExecutor";
344 8 : } else if (
345 8 : workflowMode_ == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE || dataSize > HCCL_SMALL_COUNT_1_MB) {
346 8 : algName = "ReduceScatterMeshExecutor";
347 0 : } else if (workflowMode_ == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OPS_KERNEL_INFO_LIB) {
348 0 : algName = "ReduceScatterMeshGraphExecutor";
349 : }
350 : }
351 0 : } else if (isRingTopo) {
352 0 : algName = "ReduceScatterRingExecutor";
353 : } else {
354 0 : algName = "ReduceScatterComm";
355 : }
356 : // 如果配置了aiv only,但是实际没有选择aiv算法,需要通过DFX打印出具体原因
357 31 : if (isOnlyAiv && !isAivMode) {
358 0 : HCCL_ERROR("The current conditions do not meet the aiv only execution criteria because:");
359 0 : CHK_PRT_RET(
360 : !IsSupportAIVReduce(param.DataDes.dataType, param.reduceType),
361 : HCCL_ERROR(
362 : "current data type[%s] or reduceType[%s] not supported, "
363 : "data type support range:[int8, int16, int32, float16, float32, bfloat16] reduce type support "
364 : "range:[sum, max, min]",
365 : GetDataTypeEnumStr(param.DataDes.dataType).c_str(), GetReduceOpEnumStr(param.reduceType).c_str()),
366 : HCCL_E_NOT_SUPPORT);
367 :
368 0 : CHK_PRT_RET(
369 : !isSupportAivDeter,
370 : HCCL_ERROR(
371 : "is not support aiv deter.isSingleMeshAggregation_[%d] isOpbase[%d] "
372 : "deterministic config[%u] dataSize[%llu], serverNum_[%u]",
373 : isSingleMeshAggregation_, isOpbase, topoMatcher_->GetDeterministicConfig(), dataSize, serverNum_),
374 : HCCL_E_NOT_SUPPORT);
375 :
376 0 : CHK_PRT_RET(
377 : !isSingleMeshAggregation_ && multiModuleDiffDeviceNumMode_,
378 : HCCL_ERROR(
379 : "The number of cards between servers in a multi-server setup must be consistent. "
380 : "isSingleMeshAggregation_[%d] multiModuleDiffDeviceNumMode_[%d]",
381 : isSingleMeshAggregation_, multiModuleDiffDeviceNumMode_),
382 : HCCL_E_NOT_SUPPORT);
383 :
384 0 : CHK_PRT_RET(!isMeshTopo, HCCL_ERROR("current topo type[%d] not supported", topoType_), HCCL_E_NOT_SUPPORT);
385 :
386 0 : CHK_PRT_RET(!isServNumPowOfTwo, HCCL_ERROR("server num[%u] is pow of two.", serverNum_), HCCL_E_NOT_SUPPORT);
387 :
388 0 : CHK_PRT_RET(
389 : !isCCLBufferGE16M,
390 : HCCL_ERROR(
391 : "current isOpbase[%d] or commInputSize[%llu] or commOutputSize[%llu] not supported", isOpbase,
392 : commInputSize, commOutputSize),
393 : HCCL_E_NOT_SUPPORT);
394 0 : HCCL_ERROR(
395 : "isSingleMeshAggregation_[%d] multiModuleDiffDeviceNumMode_[%d] dataSize[%llu]", isSingleMeshAggregation_,
396 : multiModuleDiffDeviceNumMode_, dataSize);
397 0 : return HCCL_E_NOT_SUPPORT;
398 : }
399 31 : HCCL_INFO(
400 : "[SelectAlgfor910B] ReduceScatter SelectAlgfor910B is algName [%s], current mode is [%u].", algName.c_str(),
401 : workflowMode_);
402 31 : return HCCL_SUCCESS;
403 : }
404 :
405 : HcclResult
406 4 : ReduceScatterOperator::SelectAlgfor91093(const OpParam& param, std::string& algName, const ResourceLimit& limit)
407 : {
408 4 : u32 unitSize = SIZE_TABLE[param.DataDes.dataType];
409 4 : u64 dataSize = param.DataDes.count * unitSize; // 单位：字节
410 4 : if (dataSize >= cclBufferManager_.GetInCCLbufferSize()) {
411 4 : HCCL_WARNING(
412 : "The current inCCLbufferSize is [%llu] bytes, change the HCCL_BUFFSIZE environment variable "
413 : "to be greater than the current data volume[%llu] bytes to improve the performance of the 91093 "
414 : "environment.",
415 : cclBufferManager_.GetInCCLbufferSize(), dataSize);
416 : }
417 4 : bool isOnlyAiv = topoMatcher_->GetIsOnlyAivConfig();
418 4 : bool isOpbase = workflowMode_ == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE;
419 :
420 0 : bool isAivCrossNode = superPodNum_ == 1 && serverNum_ > 1 && !GetExternalInputInterHccsDisable()
421 4 : && ((
422 0 : (userRankSize_ <= ONE_EIGHTH_MAX_NUM_BLOCKS
423 0 : && dataSize <= AIV_REDUCE_SCATTER_A3_SMALL_RANKSIZE_ENTRY_SIZE)
424 0 : || (userRankSize_ <= ONE_THIRD_MAX_NUM_BLOCKS
425 0 : && dataSize <= AIV_REDUCE_SCATTER_A3_MID_RANKSIZE_ENTRY_SIZE)
426 0 : || (dataSize <= AIV_REDUCE_SCATTER_A3_LARGE_RANKSIZE_ENTRY_SIZE) || isOnlyAiv));
427 :
428 8 : bool isAivSingleNode = serverNum_ == 1
429 4 : && ((isOpbase && (dataSize <= AIV_REDUCE_SCATTER_A3_ENTRY_SIZE || isOnlyAiv))
430 0 : || (!isOpbase && (dataSize <= AIV_REDUCE_SCATTER_A3_GRAPH_ENTRY_SIZE || isOnlyAiv)));
431 :
432 : // A3 AIV 确定性 超节点内(单机与跨机) 支持单算子与图模式 限制单卡数据量8MB
433 0 : bool isSupportAivDeter = (superPodNum_ == 1) && topoMatcher_->GetAivModeConfig()
434 0 : && IsSupportAIVReduce(param.DataDes.dataType, param.reduceType)
435 0 : && (topoMatcher_->GetDeterministicConfig() != DETERMINISTIC_DISABLE)
436 0 : && ((dataSize * userRankSize_ < HCCL_SMALL_COUNT_8_MB) || isOnlyAiv) && (!retryEnable_)
437 4 : && userRankSize_ > 1 && !multiModuleDiffDeviceNumMode_;
438 :
439 4 : bool isAivMode = topoMatcher_->GetAivModeConfig() && IsSupportAIVReduce(param.DataDes.dataType, param.reduceType)
440 0 : && (isAivSingleNode || isAivCrossNode)
441 0 : && (topoMatcher_->GetDeterministicConfig() == DETERMINISTIC_DISABLE) && (!retryEnable_)
442 4 : && !multiModuleDiffDeviceNumMode_;
443 :
444 4 : if (isSupportAivDeter) {
445 0 : algName = "ReduceScatterMeshAivFor91093Executor";
446 0 : HCCL_INFO("[SelectAlgfor91093] reduce_scatter SelectAlgfor91093 algName [%s]", algName.c_str());
447 0 : return HCCL_SUCCESS;
448 : }
449 :
450 4 : if (IsNeedStrictMode(param)) {
451 0 : CHK_PRT_RET(
452 : !CheckStrictCondition(param),
453 : HCCL_ERROR("[ReduceScatterOperator][SelectAlgfor91093] not support DETERMINISTIC_STRICT mode."),
454 : HCCL_E_NOT_SUPPORT);
455 :
456 0 : algName = "ReduceScatterOrderPreservedFor91093Executor";
457 0 : HCCL_INFO("[SelectAlgfor91093] reduce_scatter SelectAlgfor91093 algName [%s].", algName.c_str());
458 0 : return HCCL_SUCCESS;
459 : }
460 :
461 4 : if (isAivMode) {
462 0 : if (isAivCrossNode) {
463 0 : algName = "ReduceScatterMeshAivFor91093Executor";
464 0 : } else if (
465 0 : (isOpbase && dataSize <= AIV_REDUCE_SCATTER_SMALL_SIZE)
466 0 : || (!isOpbase
467 0 : && dataSize <= std::min(limit.aivCoreLimit / userRankSize_, NUM_BLOCKS_FACTOR_FOUR)
468 0 : * AIV_REDUCE_SCATTER_BIG_SIZE)) {
469 0 : algName = "ReduceScatterMeshAivSmallCountExecutor";
470 : } else {
471 0 : algName = "ReduceScatterMeshAivExecutor";
472 : }
473 0 : HCCL_INFO("[SelectAlgfor91093] ReduceScatter SelectAlgfor91093 is algName [%s]", algName.c_str());
474 0 : return HCCL_SUCCESS;
475 : }
476 : // ARS 算法选择
477 4 : bool isARSAlgo = multiModuleDiffDeviceNumMode_ && !multiSuperPodDiffDeviceNumMode_;
478 4 : if (isARSAlgo) {
479 0 : if (!(algType_.algoLevel1 == AlgTypeLevel1::ALG_LEVEL1_NB
480 0 : || algType_.algoLevel1 == AlgTypeLevel1::ALG_LEVEL1_RING)) {
481 0 : algType_.algoLevel1 = AlgTypeLevel1::ALG_LEVEL1_NHR;
482 0 : HCCL_WARNING("[ReduceScatterOperator][SelectAlgfor91093] ARS only support NHR or RING in AlgoLevel1 "
483 : "yet, default is NHR.");
484 : }
485 : }
486 : // AHC 算法选择逻辑
487 8 : bool isAHCAlgo = (algType_.algoLevel1 == AlgTypeLevel1::ALG_LEVEL1_AHC)
488 4 : || (algType_.algoLevel1 == AlgTypeLevel1::ALG_LEVEL1_AHC_BROKE);
489 4 : if (isAHCAlgo) {
490 0 : CHK_RET(SelectAlgforAHC(dataSize, AHCOpType::AHC_OP_TYPE_REDUCE_SCATTER));
491 : }
492 :
493 : bool isSupportInlineReduce
494 4 : = IsSupportSDMAReduce(param.inputPtr, param.outputPtr, param.DataDes.dataType, param.reduceType);
495 4 : bool isPowOfTwo = ((userRankSize_ - 1) & userRankSize_) == 0;
496 4 : void* commInputPtr = nullptr;
497 4 : u64 commInputSize = 0;
498 4 : s32 HCCS_PORT_NUM_910_93_7 = 7;
499 4 : u64 smallCountSingleServerThreshold
500 4 : = (hccsPortNum_ == HCCS_PORT_NUM_910_93_7) ? HCCL_SMALL_COUNT_512_KB : HCCL_SMALL_COUNT_1_MB;
501 4 : u64 smallCountMultiServerThreshold
502 4 : = (hccsPortNum_ == HCCS_PORT_NUM_910_93_7) ? HCCL_SMALL_COUNT_1_MB : HCCL_SMALL_COUNT_2_MB;
503 4 : CHK_RET(cclBufferManager_.GetInCCLbuffer(commInputPtr, commInputSize));
504 4 : u64 maxPipelineBlockSize = 0;
505 4 : if (userRankSize_ != 0) {
506 : maxPipelineBlockSize
507 4 : = commInputSize / userRankSize_ / HCCL_DEVICE_NUM_TWO / HCCL_MIN_SLICE_ALIGN * HCCL_MIN_SLICE_ALIGN;
508 : }
509 4 : bool isSupportPipelineFor91093 = (maxPipelineBlockSize >= HCCL_SMALL_COUNT_4_MB)
510 4 : && (dataSize * userRankSize_ > HCCL_PIPELINE_TOTAL_DATA_SIZE_THRESHOLD);
511 4 : HCCL_INFO(
512 : "[ReduceScatterOperator][SelectAlgfor91093] dataSize[%llu] commInputSize[%llu] "
513 : "userRankSize[%u] maxPipelineBlockSize[%llu] isSupportPipelineFor91093[%d]",
514 : dataSize, commInputSize, userRankSize_, maxPipelineBlockSize, isSupportPipelineFor91093);
515 3 : bool dmaReduceLimit = (workflowMode_ == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE) && isPowOfTwo
516 7 : && ((commInputSize * HCCL_DEVICE_NUM_TWO
517 3 : < param.DataDes.count * SIZE_TABLE[param.DataDes.dataType] * userRankSize_)
518 0 : || retryEnable_);
519 : bool smallCountOptimSingleServer
520 4 : = (!retryEnable_) && (serverNum_ == 1)
521 0 : && ((workflowMode_ == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE)
522 0 : || (workflowMode_ != HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE && !param.aicpuUnfoldMode))
523 0 : && isSupportInlineReduce && (deviceNumPerAggregation_ > HCCL_DEVICE_NUM_TWO)
524 0 : && (param.DataDes.count * SIZE_TABLE[param.DataDes.dataType] <= smallCountSingleServerThreshold)
525 8 : && !GetExternalInputInterHccsDisable() && !dmaReduceLimit;
526 3 : bool smallCountOptimMultiServer = isSupportInlineReduce && (deviceNumPerAggregation_ > HCCL_DEVICE_NUM_TWO)
527 0 : && (serverNum_ != 1) && (superPodNum_ == 1) && !dmaReduceLimit
528 7 : && !GetExternalInputInterHccsDisable();
529 1 : bool isHccsPlusSio = userRankSize_ == 2 && pairLinkCounter_[static_cast<u32>(LinkTypeInServer::SIO_TYPE)] == 2
530 5 : && pairLinkCounter_[static_cast<u32>(LinkTypeInServer::HCCS_TYPE)] == 0;
531 : bool useHostComm
532 4 : = !isSupportInlineReduce
533 5 : && ((serverNum_ != 1 && superPodNum_ == 1 && !GetExternalInputInterHccsDisable())
534 1 : || ((superPodNum_ > 1 || GetExternalInputInterHccsDisable()) && !retryEnable_
535 0 : && ((isPowOfTwo && param.DataDes.count * SIZE_TABLE[param.DataDes.dataType] <= HCCL_SMALL_COUNT_4_MB)
536 0 : || (!isPowOfTwo
537 0 : && param.DataDes.count * SIZE_TABLE[param.DataDes.dataType] <= HCCL_SMALL_COUNT_2_MB))));
538 4 : bool smallCountOptimMultiPod = false;
539 :
540 4 : isHccsPlusSio = false;
541 4 : if (isHccsPlusSio && isSupportHccsAndSio_) {
542 0 : algName = "ReduceScatterHccsSioExecutor";
543 4 : } else if (multiModuleDiffDeviceNumMode_ && multiSuperPodDiffDeviceNumMode_) {
544 0 : algName = "ReduceScatterComm";
545 4 : } else if (multiModuleDiffDeviceNumMode_ && !multiSuperPodDiffDeviceNumMode_) {
546 0 : algName = "ReduceScatterARSFor91093Executor";
547 4 : } else if (
548 4 : smallCountOptimMultiPod || useHostComm
549 4 : || (smallCountOptimMultiServer && !isPowOfTwo
550 0 : && (param.DataDes.count * SIZE_TABLE[param.DataDes.dataType] <= HCCL_SMALL_COUNT_256_KB))) {
551 0 : algName = "ReduceScatterComm";
552 0 : algType_.algoLevel1 = AlgTypeLevel1::ALG_LEVEL1_HD;
553 4 : } else if (
554 : smallCountOptimSingleServer
555 4 : || (smallCountOptimMultiServer && isPowOfTwo
556 0 : && (param.DataDes.count * SIZE_TABLE[param.DataDes.dataType] * serverNum_
557 : <= smallCountMultiServerThreshold))) {
558 0 : algName = "ReduceScatterDeterExecutor";
559 4 : } else if (
560 3 : isSupportInlineReduce && (param.supportSymmetricMemory || param.supportZeroCopy)
561 0 : && // isSupportInlineReduce：不申请scratch ==> 不支持非InlineReduce
562 0 : (topoType_ == TopoType::TOPO_TYPE_NP_DOUBLE_RING
563 0 : || param.DataDes.count * unitSize * deviceNumPerAggregation_ > HCCL_MID_COUNT_16_MB)) {
564 0 : const u32 SEVER_NUM_FOUR = 4;
565 0 : constexpr u64 RING_EXCHANGE_PIPELINE_DATA_SIZE_MIN = 2 * 1024 * 1024;
566 : HcclAlgoType configAlgTypeLevel2
567 0 : = topoMatcher_->GetAlgoConfig(HcclCMDType::HCCL_CMD_REDUCE_SCATTER)[HCCL_ALGO_LEVEL_2];
568 0 : if ((superPodNum_ > 1) && (userRankSize_ / superPodNum_ > 1)
569 0 : && ((configAlgTypeLevel2 == HcclAlgoType::HCCL_ALGO_TYPE_PIPELINE)
570 0 : || ((configAlgTypeLevel2 == HcclAlgoType::HCCL_ALGO_TYPE_DEFAULT)
571 0 : && (dataSize >= RING_EXCHANGE_PIPELINE_DATA_SIZE_MIN)))) {
572 : // 单算子, 超节点数大于1， 每个超节点的rank数大于1
573 0 : algName = "ReduceScatterRingZerocopyExchangePipelineExecutor"; // 连续数据通信+数据交换+Pipeline
574 0 : algType_.algoLevel2 = AlgTypeLevel2::ALG_LEVEL2_PIPELINE;
575 0 : } else if (serverNum_ < SEVER_NUM_FOUR || isAHCAlgo) {
576 0 : algName = "ReduceScatterRingZerocopyExecutor"; // 非连续数据通信（限制Server数，避免数据切太碎）
577 : } else {
578 0 : algName = "ReduceScatterRingZerocopyExchangeExecutor"; // 连续数据通信+数据交换（AHC不支持）
579 : }
580 4 : } else if (
581 3 : isOpbase && superPodNum_ > 1 && !isAHCAlgo && !multiSuperPodDiffDeviceNumMode_ && isSupportInlineReduce
582 0 : && (topoType_ == TopoType::TOPO_TYPE_NP_SINGLE_RING || topoType_ == TopoType::TOPO_TYPE_NP_DOUBLE_RING)
583 0 : && isSupportPipelineFor91093) {
584 0 : algName = "ReduceScatterPipelineFor91093Executor";
585 : } else {
586 4 : if (topoType_ == TopoType::TOPO_TYPE_NP_SINGLE_RING) {
587 2 : algName = "ReduceScatterRingFor91093Executor";
588 2 : } else if (topoType_ == TopoType::TOPO_TYPE_NP_DOUBLE_RING) {
589 2 : if (IsSupportUnifiedMarch(param, topoType_, serverNum_, superPodNum_)) {
590 0 : algName = "ReduceScatterSemiRingExecutor";
591 : } else {
592 2 : algName = "ReduceScatterFastDoubleRingFor91093Executor";
593 : }
594 : } else {
595 0 : algName = "ReduceScatterComm";
596 : }
597 : }
598 :
599 1 : if (!(algType_.algoLevel1 == AlgTypeLevel1::ALG_LEVEL1_RING || algType_.algoLevel1 == AlgTypeLevel1::ALG_LEVEL1_NB
600 1 : || (algType_.algoLevel0 == AlgTypeLevel0::ALG_LEVEL0_WHOLE_RING
601 0 : && algType_.algoLevel1 == AlgTypeLevel1::ALG_LEVEL1_WHOLE_RING)
602 1 : || algType_.algoLevel1 == AlgTypeLevel1::ALG_LEVEL1_AHC
603 1 : || algType_.algoLevel1 == AlgTypeLevel1::ALG_LEVEL1_AHC_BROKE)
604 5 : && (algName != "ReduceScatterComm" && algType_.algoLevel1 == AlgTypeLevel1::ALG_LEVEL1_HD)) {
605 : // 910_93超节点只支持server间ring,NB和NHR，默认需继续使用NHR
606 0 : algType_.algoLevel1 = AlgTypeLevel1::ALG_LEVEL1_NHR;
607 0 : HCCL_WARNING("[ReduceScatterOperator][SelectAlgfor91093] only support ring, NB AHC and NHR in AlgoLevel1 yet, "
608 : "default is algType=NHR.");
609 : }
610 : // 如果配置了aiv only,但是实际没有选择aiv算法,需要通过DFX打印出具体原因
611 4 : if (isOnlyAiv && !isAivMode && !isSupportAivDeter) {
612 0 : HCCL_ERROR("The current conditions do not meet the aiv only execution criteria because:");
613 0 : CHK_PRT_RET(
614 : !IsSupportAIVReduce(param.DataDes.dataType, param.reduceType),
615 : HCCL_ERROR(
616 : "current data type[%s] or reduceType[%s] not supported, "
617 : "data type support range:[int8, int16, int32, float16, float32, bfloat16] reduce type support "
618 : "range:[sum, max, min]",
619 : GetDataTypeEnumStr(param.DataDes.dataType).c_str(), GetReduceOpEnumStr(param.reduceType).c_str()),
620 : HCCL_E_NOT_SUPPORT);
621 :
622 0 : CHK_PRT_RET(retryEnable_, HCCL_ERROR("retryEnable [%d] not supported", retryEnable_), HCCL_E_NOT_SUPPORT);
623 :
624 0 : CHK_PRT_RET(
625 : superPodNum_ != 1, HCCL_ERROR("multi superpod [%u] not supported", superPodNum_), HCCL_E_NOT_SUPPORT);
626 :
627 0 : CHK_PRT_RET(
628 : multiModuleDiffDeviceNumMode_,
629 : HCCL_ERROR("multiModuleDiffDeviceNumMode [%d] not supported", multiModuleDiffDeviceNumMode_),
630 : HCCL_E_NOT_SUPPORT);
631 0 : return HCCL_E_NOT_SUPPORT;
632 : }
633 4 : HCCL_INFO("[SelectAlgfor91093] ReduceScatter SelectAlgfor91093 is algName [%s]", algName.c_str());
634 :
635 4 : HCCL_INFO(
636 : "[SelectAlgfor91093] isOpbase[%d] superPodNum_[%u] isAHCAlgo[%d] multiSuperPodDiffDeviceNumMode_[%d] "
637 : "isSupportInlineReduce[%d] topoType_[%d] dataSize[%llu]",
638 : isOpbase, superPodNum_, isAHCAlgo, multiSuperPodDiffDeviceNumMode_, isSupportInlineReduce, topoType_, dataSize);
639 4 : return HCCL_SUCCESS;
640 : }
641 :
642 : REGISTER_OP(HcclCMDType::HCCL_CMD_REDUCE_SCATTER, ReduceScatter, ReduceScatterOperator);
643 :
644 : } // namespace hccl
|
