Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "reduce_scatter_operator.h"
12 : #include "device_capacity.h"
13 : #include "coll_alg_utils.h"
14 : #include "hccl_aiv.h"
15 : #include "coll_alg_op_registry.h"
16 : #include <algorithm>
17 :
18 : constexpr u32 MODULE_NUM_FOUR = 4;
19 : constexpr u32 HCCL_310P_DATA_SIZE_MID_COUNT = 320 * 1024;
20 : constexpr u32 HCCL_310P_DATA_SIZE_SMALL_COUNT = 1024;
21 : constexpr u32 HCCL_310P_SLIM_RING_MAX_SIZE = 8;
22 :
23 : // Pipeline并行比串行更优的总数据量临界点（基于910_93 2 SuperPod, 1 Server, 12 Rank 性能基线测试，
24 : // 数据量低于此值时调度开销超过流水收益）
25 : constexpr u64 HCCL_PIPELINE_TOTAL_DATA_SIZE_THRESHOLD = 608 * 1024 * 1024;
26 :
27 : namespace hccl {
28 :
29 32 : ReduceScatterOperator::ReduceScatterOperator(
30 : AlgConfigurator* algConfigurator, CCLBufferManager& cclBufferManager, HcclDispatcher dispatcher,
31 32 : std::unique_ptr<TopoMatcher>& topoMatcher)
32 32 : : CollAlgOperator(algConfigurator, cclBufferManager, dispatcher, topoMatcher, HcclCMDType::HCCL_CMD_REDUCE_SCATTER)
33 33 : {}
34 :
35 74 : ReduceScatterOperator::~ReduceScatterOperator() {}
36 :
37 1 : HcclResult ReduceScatterOperator::SelectAlg(
38 : const std::string& tag, const OpParam& param, std::string& algName, std::string& newTag)
39 : {
40 1 : ResourceLimit limit;
41 2 : return SelectAlg(tag, param, algName, newTag, limit);
42 : }
43 :
44 35 : HcclResult ReduceScatterOperator::SelectAlg(
45 : const std::string& tag, const OpParam& param, std::string& algName, std::string& newTag, const ResourceLimit& limit)
46 : {
47 35 : if (userRankSize_ == 1) {
48 1 : algName = "ReduceScatterSingleExecutor";
49 1 : return HCCL_SUCCESS;
50 : }
51 : HcclResult ret;
52 34 : if (isDiffDeviceType_) {
53 0 : ret = SelectAlgforMix(param, algName);
54 34 : } else if (deviceType_ == DevType::DEV_TYPE_310P3) {
55 0 : ret = SelectAlgfor310P3(param, algName);
56 34 : } else if (deviceType_ == DevType::DEV_TYPE_910) {
57 0 : ret = SelectAlgfor910A(param, algName);
58 34 : } else if (deviceType_ == DevType::DEV_TYPE_910B) {
59 28 : ret = SelectAlgfor910B(param, algName);
60 6 : } else if (deviceType_ == DevType::DEV_TYPE_910_93) {
61 4 : ret = SelectAlgfor91093(param, algName, limit);
62 : } else {
63 2 : HCCL_ERROR("[SelectAlg] device type[%d] is out of range for selector.", deviceType_);
64 0 : return HCCL_E_NOT_SUPPORT;
65 : }
66 36 : CHK_PRT_RET(
67 : ret != HCCL_SUCCESS,
68 : HCCL_ERROR("[ReduceScatterSelector][SelectAlg]tag[%s], ReduceScatter failed, return[%d]", tag.c_str(), ret),
69 : ret);
70 :
71 36 : if (GetWorkflowMode() == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OPS_KERNEL_INFO_LIB) {
72 13 : newTag = tag;
73 : } else {
74 23 : if (deviceType_ == DevType::DEV_TYPE_310P3) {
75 0 : newTag = tag + algName;
76 : } else {
77 23 : auto level1Iter = HCCL_ALGO_LEVEL1_NAME_MAP.find(algType_.algoLevel1);
78 19 : CHK_PRT_RET(
79 : level1Iter == HCCL_ALGO_LEVEL1_NAME_MAP.end(),
80 : HCCL_ERROR("level1: algType1[%u] is invalid.", algType_.algoLevel1), HCCL_E_INTERNAL);
81 19 : newTag = tag + level1Iter->second + algName;
82 : }
83 :
84 20 : bool isInlineReduce = IsSupportSDMAReduce(
85 21 : cclBufferManager_.GetInCCLbuffer().ptr(), cclBufferManager_.GetOutCCLbuffer().ptr(), param.DataDes.dataType,
86 21 : param.reduceType);
87 20 : bool isRdmaReduce = IsSupportRDMAReduce(param.DataDes.dataType, param.reduceType);
88 20 : const std::string REDUCE_SCATTER_NO_INLINE = "_no_inline";
89 34 : newTag = (isInlineReduce && isRdmaReduce) ? newTag : newTag + REDUCE_SCATTER_NO_INLINE;
90 23 : }
91 36 : if (algName == "ReduceScatterARSFor91093Executor") {
92 0 : u32 ringSize = CalcOptimalIntraRingsize(
93 0 : param.DataDes.count, param.DataDes.dataType, HcclCMDType::HCCL_CMD_REDUCE_SCATTER);
94 0 : newTag += std::to_string(ringSize);
95 : }
96 36 : newTag += (param.aicpuUnfoldMode ? "_device" : "_host");
97 36 : return ret;
98 : }
99 :
100 0 : HcclResult ReduceScatterOperator::SelectAlgforMix(const OpParam& param, std::string& algName)
101 : {
102 : (void)param;
103 :
104 : // 混合组网场景不支持规约保序
105 0 : if (IsNeedStrictMode(param)) {
106 0 : HCCL_ERROR("[ReduceScatterOperator][SelectAlgforMix] not support DETERMINISTIC_STRICT mode.");
107 0 : return HCCL_E_NOT_SUPPORT;
108 : }
109 :
110 0 : if (gcdDeviceNumPerAggregation_ > 1) {
111 0 : algType_.algoLevel1 = AlgTypeLevel1::ALG_LEVEL1_NHR;
112 0 : HCCL_WARNING("[ReduceScatterOperator][SelectAlgforMix] only support NHR in AlgoLevel1 yet, "
113 : "default is algType=NHR.");
114 0 : algName = "ReduceScatterMixExecutor";
115 : } else {
116 0 : algType_.algoLevel1 = AlgTypeLevel1::ALG_LEVEL1_RING;
117 : ;
118 0 : HCCL_WARNING("[ReduceScatterOperator][SelectAlgforMix] only support ring in AlgoComm yet, "
119 : "default is algType=ring.");
120 0 : algName = "ReduceScatterComm";
121 : }
122 :
123 0 : HCCL_INFO("[SelectAlgforMix] ReduceScatter SelectAlgforMix is algName [%s].", algName.c_str());
124 0 : return HCCL_SUCCESS;
125 : }
126 :
127 0 : HcclResult ReduceScatterOperator::SelectAlgfor310P3(const OpParam& param, std::string& algName)
128 : {
129 0 : if (HCCL_310P_DATA_SIZE_SMALL_COUNT < param.DataDes.count && param.DataDes.count <= HCCL_310P_DATA_SIZE_MID_COUNT
130 0 : && userRankSize_ <= HCCL_310P_SLIM_RING_MAX_SIZE) {
131 0 : algName = "ReduceScatterSlimRing";
132 : } else {
133 0 : algName = "ReduceScatterRing";
134 : }
135 :
136 0 : HCCL_INFO(
137 : "[SelectAlgfor310P3] ReduceScatter SelectAlgfor310P3 is algName [%s] DataDesCount [%llu]", algName.c_str(),
138 : param.DataDes.count);
139 0 : return HCCL_SUCCESS;
140 : }
141 :
142 0 : HcclResult ReduceScatterOperator::SelectAlgfor910A(const OpParam& param, std::string& algName)
143 : {
144 0 : bool isMeshTopo = topoType_ == TopoType::TOPO_TYPE_4P_MESH || topoType_ == TopoType::TOPO_TYPE_2P_MESH;
145 0 : bool isRingTopo = topoType_ == TopoType::TOPO_TYPE_NP_SINGLE_RING || topoType_ == TopoType::TOPO_TYPE_8P_RING;
146 :
147 0 : const u32 RANK_SIZE_FOUR = 4;
148 0 : const u32 RANK_SIZE_EIGHT = 8;
149 0 : bool isOpbase = GetWorkflowMode() == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE;
150 0 : u64 dataSize = SIZE_TABLE[param.DataDes.dataType] * param.DataDes.count;
151 0 : if (isOpbase && serverNum_ == 1 && dataSize <= HCCL_SMALL_COUNT_256_KB
152 0 : && (userRankSize_ == RANK_SIZE_FOUR || userRankSize_ == RANK_SIZE_EIGHT)) {
153 0 : algType_.algoLevel1 = AlgTypeLevel1::ALG_LEVEL1_HD;
154 0 : algName = "ReduceScatterComm";
155 0 : } else if (isMeshTopo) {
156 0 : algName = "ReduceScatterMeshExecutor";
157 0 : } else if (isRingTopo) {
158 0 : algName = "ReduceScatterRingExecutor";
159 : } else {
160 0 : algName = "ReduceScatterComm";
161 : }
162 0 : HCCL_INFO("[SelectAlgfor910A] ReduceScatter SelectAlgfor910A is algName [%s].", algName.c_str());
163 0 : return HCCL_SUCCESS;
164 : }
165 :
166 27 : HcclResult ReduceScatterOperator::SelectAlgfor910B(const OpParam& param, std::string& algName)
167 : {
168 27 : u32 unitSize = SIZE_TABLE[param.DataDes.dataType];
169 :
170 27 : bool isOnlyAiv = topoMatcher_->GetIsOnlyAivConfig();
171 0 : bool isMeshTopo = topoType_ == TopoType::TOPO_TYPE_NP_MESH || topoType_ == TopoType::TOPO_TYPE_4P_MESH
172 30 : || topoType_ == TopoType::TOPO_TYPE_2P_MESH || topoType_ == TopoType::TOPO_TYPE_1P_MESH;
173 30 : bool isRingTopo = topoType_ == TopoType::TOPO_TYPE_NP_SINGLE_RING;
174 :
175 30 : u64 dataSize = param.DataDes.count * unitSize; // 单位：字节
176 30 : u64 cclBufferSize = cclBufferManager_.GetInCCLbufferSize() / userRankSize_;
177 :
178 29 : void* commInputPtr = nullptr;
179 29 : void* commOutputPtr = nullptr;
180 29 : u64 commInputSize = 0;
181 29 : u64 commOutputSize = 0;
182 :
183 29 : CHK_RET(cclBufferManager_.GetInCCLbuffer(commInputPtr, commInputSize));
184 28 : CHK_RET(cclBufferManager_.GetOutCCLbuffer(commOutputPtr, commOutputSize));
185 28 : bool isServNumPowOfTwo = (serverNum_ > 0) && ((serverNum_ & (serverNum_ - 1)) == 0);
186 28 : bool isOpbase = (GetWorkflowMode() == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE);
187 28 : bool isInlineReduce = IsSupportSDMAReduce(
188 29 : cclBufferManager_.GetInCCLbuffer().ptr(), cclBufferManager_.GetOutCCLbuffer().ptr(), param.DataDes.dataType,
189 27 : param.reduceType);
190 :
191 30 : if (topoMatcher_->GetDeterministicConfig() == DETERMINISTIC_STRICT && userRankSize_ > 2) {
192 0 : if (multiModuleDiffDeviceNumMode_) {
193 : // 保序规约场景（多batch一致），当前不支持A2非对称场景
194 0 : HCCL_ERROR(
195 : "[SelectAlgfor910B] reduce order preservation only support"
196 : " Symmetry(multiModuleDiffDeviceNumMode_[%d]).",
197 : multiModuleDiffDeviceNumMode_);
198 0 : return HCCL_E_NOT_SUPPORT;
199 : }
200 0 : if (param.DataDes.dataType == HCCL_DATA_TYPE_FP16 || param.DataDes.dataType == HCCL_DATA_TYPE_FP32
201 0 : || param.DataDes.dataType == HCCL_DATA_TYPE_BFP16) {
202 : // 只有浮点数存在多batch不一致的可能，整数天然一致
203 1 : if (param.aicpuUnfoldMode || topoMatcher_->GetAivModeConfig()) {
204 : // AIV / AICPU场景，规约保序优先级更高
205 0 : HCCL_WARNING(
206 : "[SelectAlgfor910B]aicpuMode[%d], AivModeConfig[%d], "
207 : "the AIV/AICPU mode does not support when the reduce order preservation is enabled.",
208 : param.aicpuUnfoldMode, topoMatcher_->GetAivModeConfig());
209 : }
210 0 : algName = "ReduceScatterOrderPreservedExecutor";
211 0 : HCCL_INFO("[SelectAlgfor910B] ReduceScatterSelectAlgfor910B is algName [%s].", algName.c_str());
212 0 : return HCCL_SUCCESS;
213 : }
214 : }
215 :
216 : // 暂只支持单算子模式
217 30 : bool isCCLBufferGE16M
218 30 : = isOpbase && (commInputSize >= HCCL_MID_COUNT_16_MB && commOutputSize >= HCCL_MID_COUNT_16_MB);
219 :
220 0 : bool isSupportAivRdmaCount = !isSingleMeshAggregation_ && !multiModuleDiffDeviceNumMode_ && isMeshTopo
221 30 : && (((isServNumPowOfTwo || dataSize <= HCCL_SMALL_COUNT_128_KB)
222 0 : && dataSize * userRankSize_ <= HCCL_MID_COUNT_16_MB && isCCLBufferGE16M
223 0 : && dataSize <= HCCL_SMALL_COUNT_256_KB)
224 0 : || isOnlyAiv);
225 :
226 31 : bool isSupportAivDeter = isSingleMeshAggregation_ && serverNum_ == 1
227 31 : && (topoMatcher_->GetDeterministicConfig() == DETERMINISTIC_ENABLE)
228 59 : && ((dataSize * userRankSize_ <= HCCL_SMALL_COUNT_8_MB) || isOnlyAiv);
229 :
230 : bool isAivMode
231 28 : = topoMatcher_->GetAivModeConfig() && IsSupportAIVReduce(param.DataDes.dataType, param.reduceType)
232 0 : && (topoMatcher_->GetDeterministicConfig() == DETERMINISTIC_DISABLE || isSupportAivDeter)
233 27 : && ((isSingleMeshAggregation_ && (dataSize <= AIV_BIG_SIZE || isOnlyAiv)) || isSupportAivRdmaCount);
234 27 : if (isAivMode) {
235 0 : if (isSupportAivDeter) {
236 0 : if (dataSize * userRankSize_ <= HCCL_SMALL_COUNT_8_MB) {
237 0 : algName = "ReduceScatterAivDeterSmallExecutor";
238 : } else {
239 0 : algName = "ReduceScatterAivDeterExecutor";
240 : }
241 0 : HCCL_INFO("[SelectAlgfor910B] ReduceScatter SelectAlgfor910B is algName [%s].", algName.c_str());
242 0 : return HCCL_SUCCESS;
243 : }
244 0 : if (isSupportAivRdmaCount) {
245 0 : algName = "ReduceScatterAivRdmaExecutor";
246 0 : } else if (
247 0 : workflowMode_ == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE && dataSize <= AIV_REDUCE_SCATTER_MID_SIZE) {
248 0 : algName = "ReduceScatterMeshAivSmallCountExecutor";
249 : } else {
250 0 : algName = "ReduceScatterMeshAivExecutor";
251 : }
252 0 : HCCL_INFO("[SelectAlgfor910BAIV] ReduceScatterSelectAlgfor910B is algName [%s].", algName.c_str());
253 0 : return HCCL_SUCCESS;
254 : }
255 :
256 27 : if (GetWorkflowMode() == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE) {
257 15 : bool isRdmaReduce = IsSupportRDMAReduce(param.DataDes.dataType, param.reduceType);
258 :
259 16 : std::string algTypeLevel1Tag;
260 16 : CHK_RET(AutoSelectAlgTypeLevel1(
261 : HcclCMDType::HCCL_CMD_REDUCE_SCATTER, dataSize, cclBufferSize, algTypeLevel1Tag, isInlineReduce,
262 : isRdmaReduce));
263 20 : if (GetExternalInputHcclEnableEntryLog() && param.opBaseAtraceInfo != nullptr) {
264 0 : CHK_RET(param.opBaseAtraceInfo->SavealgtypeTraceInfo(algTypeLevel1Tag, param.tag));
265 : }
266 20 : }
267 :
268 : // AHC 算法选择逻辑
269 32 : if (((algType_.algoLevel1 == AlgTypeLevel1::ALG_LEVEL1_AHC)
270 32 : || (algType_.algoLevel1 == AlgTypeLevel1::ALG_LEVEL1_AHC_BROKE))) {
271 0 : CHK_RET(SelectAlgforAHC(dataSize, AHCOpType::AHC_OP_TYPE_REDUCE_SCATTER));
272 : }
273 :
274 : // pipeline算法task数量多，如果超出FFTS子图限制，则重定向到HD算法
275 32 : if (algType_.algoLevel1 == AlgTypeLevel1::ALG_LEVEL1_PIPELINE) {
276 0 : u32 contextNum = CalcContextNumForPipeline(HcclCMDType::HCCL_CMD_REDUCE_SCATTER);
277 0 : if (contextNum > HCCL_FFTS_CAPACITY) {
278 0 : algType_.algoLevel1 = AlgTypeLevel1::ALG_LEVEL1_HD;
279 0 : HCCL_WARNING(
280 : "[ReduceScatterOperator][SelectAlgfor910B] context num[%u] is out of capacity of FFTS+ "
281 : "graph[%u], reset algorithm to HD.",
282 : contextNum, HCCL_FFTS_CAPACITY);
283 : }
284 : }
285 :
286 32 : if (isMeshTopo) {
287 32 : if (topoMatcher_->GetDeterministicConfig() == DETERMINISTIC_ENABLE
288 10 : && GetWorkflowMode() == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE
289 40 : && algType_.algoLevel1 == AlgTypeLevel1::ALG_LEVEL1_PIPELINE && deviceNumPerAggregation_ > DEVICE_TWO) {
290 0 : algName = "ReduceScatterDeterPipelineExecutor";
291 30 : } else if (GetWorkflowMode() == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE) {
292 : bool enableSmallCountDeterministicAlgo
293 18 : = !isSingleMeshAggregation_
294 18 : && IsSupportSDMAReduce(
295 0 : cclBufferManager_.GetInCCLbuffer().ptr(), cclBufferManager_.GetOutCCLbuffer().ptr(),
296 0 : param.DataDes.dataType, param.reduceType);
297 17 : if (SingleMeshInlineReduce(
298 18 : cclBufferManager_.GetInCCLbuffer().ptr(), cclBufferManager_.GetOutCCLbuffer().ptr(),
299 18 : param.DataDes.dataType, param.reduceType)) {
300 12 : if (topoMatcher_->GetDeterministicConfig() != DETERMINISTIC_DISABLE) {
301 2 : algName = "ReduceScatterDeterExecutor";
302 : } else {
303 8 : algName = "ReduceScatterMeshDmaEliminationExecutor";
304 : }
305 8 : } else if (
306 8 : topoMatcher_->GetDeterministicConfig() == DETERMINISTIC_DISABLE
307 4 : && algType_.algoLevel1 == AlgTypeLevel1::ALG_LEVEL1_PIPELINE
308 12 : && IsMultiMeshInlineReduce(
309 0 : cclBufferManager_.GetInCCLbuffer().ptr(), cclBufferManager_.GetOutCCLbuffer().ptr(),
310 0 : param.DataDes.dataType, param.reduceType)) {
311 0 : algName = "ReduceScatterMeshOpbasePipelineExecutor";
312 8 : } else if (
313 : enableSmallCountDeterministicAlgo
314 8 : && ((dataSize <= HCCL_SMALL_COUNT_512_KB
315 0 : && topoMatcher_->GetDeterministicConfig() == DETERMINISTIC_ENABLE)
316 0 : || dataSize * userRankSize_ < HCCL_SMALL_COUNT_512_KB)) {
317 0 : algName = "ReduceScatterMeshOpbaseSmallCountDeterministicExecutor";
318 : }
319 : } else {
320 12 : if (SingleMeshInlineReduce(param.inputPtr, param.outputPtr, param.DataDes.dataType, param.reduceType)) {
321 12 : if (topoMatcher_->GetDeterministicConfig() != DETERMINISTIC_DISABLE
322 12 : && deviceNumPerAggregation_ > DEVICE_TWO) {
323 4 : algName = "ReduceScatterDeterExecutor";
324 : } else {
325 8 : if (dataSize <= HCCL_SMALL_COUNT_1_MB
326 8 : && workflowMode_ == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OPS_KERNEL_INFO_LIB) {
327 8 : algName = "ReduceScatterMeshGraphExecutor";
328 : } else {
329 0 : algName = "ReduceScatterMeshExecutor";
330 : }
331 : }
332 : }
333 : }
334 32 : if (algName.empty()) {
335 0 : if (workflowMode_ == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OPS_KERNEL_INFO_LIB && moduleNum_ > 1
336 0 : && deviceNumPerAggregation_ > 1 && topoMatcher_->GetDeterministicConfig() == DETERMINISTIC_DISABLE
337 0 : && IsMultiMeshInlineReduce(param.inputPtr, param.outputPtr, param.DataDes.dataType, param.reduceType)
338 8 : && (dataSize > HCCL_SMALL_COUNT_1_MB || moduleNum_ <= MODULE_NUM_FOUR
339 0 : || algType_.algoLevel1 == AlgTypeLevel1::ALG_LEVEL1_PIPELINE)) {
340 0 : algName = "ReduceScatterMeshGraphPipelineExecutor";
341 8 : } else if (
342 8 : workflowMode_ == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE || dataSize > HCCL_SMALL_COUNT_1_MB) {
343 8 : algName = "ReduceScatterMeshExecutor";
344 0 : } else if (workflowMode_ == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OPS_KERNEL_INFO_LIB) {
345 0 : algName = "ReduceScatterMeshGraphExecutor";
346 : }
347 : }
348 0 : } else if (isRingTopo) {
349 0 : algName = "ReduceScatterRingExecutor";
350 : } else {
351 0 : algName = "ReduceScatterComm";
352 : }
353 : // 如果配置了aiv only,但是实际没有选择aiv算法,需要通过DFX打印出具体原因
354 31 : if (isOnlyAiv && !isAivMode) {
355 0 : HCCL_ERROR("The current conditions do not meet the aiv only execution criteria because:");
356 0 : CHK_PRT_RET(
357 : !IsSupportAIVReduce(param.DataDes.dataType, param.reduceType),
358 : HCCL_ERROR(
359 : "current data type[%s] or reduceType[%s] not supported, "
360 : "data type support range:[int8, int16, int32, float16, float32, bfloat16] reduce type support "
361 : "range:[sum, max, min]",
362 : GetDataTypeEnumStr(param.DataDes.dataType).c_str(), GetReduceOpEnumStr(param.reduceType).c_str()),
363 : HCCL_E_NOT_SUPPORT);
364 :
365 0 : CHK_PRT_RET(
366 : !isSupportAivDeter,
367 : HCCL_ERROR(
368 : "is not support aiv deter.isSingleMeshAggregation_[%d] isOpbase[%d] "
369 : "deterministic config[%u] dataSize[%llu], serverNum_[%u]",
370 : isSingleMeshAggregation_, isOpbase, topoMatcher_->GetDeterministicConfig(), dataSize, serverNum_),
371 : HCCL_E_NOT_SUPPORT);
372 :
373 0 : CHK_PRT_RET(
374 : !isSingleMeshAggregation_ && multiModuleDiffDeviceNumMode_,
375 : HCCL_ERROR(
376 : "The number of cards between servers in a multi-server setup must be consistent. "
377 : "isSingleMeshAggregation_[%d] multiModuleDiffDeviceNumMode_[%d]",
378 : isSingleMeshAggregation_, multiModuleDiffDeviceNumMode_),
379 : HCCL_E_NOT_SUPPORT);
380 :
381 0 : CHK_PRT_RET(!isMeshTopo, HCCL_ERROR("current topo type[%d] not supported", topoType_), HCCL_E_NOT_SUPPORT);
382 :
383 0 : CHK_PRT_RET(!isServNumPowOfTwo, HCCL_ERROR("server num[%u] is pow of two.", serverNum_), HCCL_E_NOT_SUPPORT);
384 :
385 0 : CHK_PRT_RET(
386 : !isCCLBufferGE16M,
387 : HCCL_ERROR(
388 : "current isOpbase[%d] or commInputSize[%llu] or commOutputSize[%llu] not supported", isOpbase,
389 : commInputSize, commOutputSize),
390 : HCCL_E_NOT_SUPPORT);
391 0 : HCCL_ERROR(
392 : "isSingleMeshAggregation_[%d] multiModuleDiffDeviceNumMode_[%d] dataSize[%llu]", isSingleMeshAggregation_,
393 : multiModuleDiffDeviceNumMode_, dataSize);
394 0 : return HCCL_E_NOT_SUPPORT;
395 : }
396 31 : HCCL_INFO(
397 : "[SelectAlgfor910B] ReduceScatter SelectAlgfor910B is algName [%s], current mode is [%u].", algName.c_str(),
398 : workflowMode_);
399 32 : return HCCL_SUCCESS;
400 : }
401 :
402 : HcclResult
403 4 : ReduceScatterOperator::SelectAlgfor91093(const OpParam& param, std::string& algName, const ResourceLimit& limit)
404 : {
405 4 : u32 unitSize = SIZE_TABLE[param.DataDes.dataType];
406 4 : u64 dataSize = param.DataDes.count * unitSize; // 单位：字节
407 4 : if (dataSize >= cclBufferManager_.GetInCCLbufferSize()) {
408 4 : HCCL_WARNING(
409 : "The current inCCLbufferSize is [%llu] bytes, change the HCCL_BUFFSIZE environment variable "
410 : "to be greater than the current data volume[%llu] bytes to improve the performance of the 91093 "
411 : "environment.",
412 : cclBufferManager_.GetInCCLbufferSize(), dataSize);
413 : }
414 4 : bool isOnlyAiv = topoMatcher_->GetIsOnlyAivConfig();
415 4 : bool isOpbase = workflowMode_ == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE;
416 :
417 0 : bool isAivCrossNode = superPodNum_ == 1 && serverNum_ > 1 && !GetExternalInputInterHccsDisable()
418 4 : && ((
419 0 : (userRankSize_ <= ONE_EIGHTH_MAX_NUM_BLOCKS
420 0 : && dataSize <= AIV_REDUCE_SCATTER_A3_SMALL_RANKSIZE_ENTRY_SIZE)
421 0 : || (userRankSize_ <= ONE_THIRD_MAX_NUM_BLOCKS
422 0 : && dataSize <= AIV_REDUCE_SCATTER_A3_MID_RANKSIZE_ENTRY_SIZE)
423 0 : || (dataSize <= AIV_REDUCE_SCATTER_A3_LARGE_RANKSIZE_ENTRY_SIZE) || isOnlyAiv));
424 :
425 8 : bool isAivSingleNode = serverNum_ == 1
426 4 : && ((isOpbase && (dataSize <= AIV_REDUCE_SCATTER_A3_ENTRY_SIZE || isOnlyAiv))
427 0 : || (!isOpbase && (dataSize <= AIV_REDUCE_SCATTER_A3_GRAPH_ENTRY_SIZE || isOnlyAiv)));
428 :
429 : // A3 AIV 确定性 超节点内(单机与跨机) 支持单算子与图模式 限制单卡数据量8MB
430 0 : bool isSupportAivDeter = (superPodNum_ == 1) && topoMatcher_->GetAivModeConfig()
431 0 : && IsSupportAIVReduce(param.DataDes.dataType, param.reduceType)
432 0 : && (topoMatcher_->GetDeterministicConfig() != DETERMINISTIC_DISABLE)
433 0 : && ((dataSize * userRankSize_ < HCCL_SMALL_COUNT_8_MB) || isOnlyAiv) && (!retryEnable_)
434 4 : && userRankSize_ > 1 && !multiModuleDiffDeviceNumMode_;
435 :
436 4 : bool isAivMode = topoMatcher_->GetAivModeConfig() && IsSupportAIVReduce(param.DataDes.dataType, param.reduceType)
437 0 : && (isAivSingleNode || isAivCrossNode)
438 0 : && (topoMatcher_->GetDeterministicConfig() == DETERMINISTIC_DISABLE) && (!retryEnable_)
439 4 : && !multiModuleDiffDeviceNumMode_;
440 :
441 4 : if (isSupportAivDeter) {
442 0 : algName = "ReduceScatterMeshAivFor91093Executor";
443 0 : HCCL_INFO("[SelectAlgfor91093] reduce_scatter SelectAlgfor91093 algName [%s]", algName.c_str());
444 0 : return HCCL_SUCCESS;
445 : }
446 :
447 4 : if (IsNeedStrictMode(param)) {
448 0 : CHK_PRT_RET(
449 : !CheckStrictCondition(param),
450 : HCCL_ERROR("[ReduceScatterOperator][SelectAlgfor91093] not support DETERMINISTIC_STRICT mode."),
451 : HCCL_E_NOT_SUPPORT);
452 :
453 0 : algName = "ReduceScatterOrderPreservedFor91093Executor";
454 0 : HCCL_INFO("[SelectAlgfor91093] reduce_scatter SelectAlgfor91093 algName [%s].", algName.c_str());
455 0 : return HCCL_SUCCESS;
456 : }
457 :
458 4 : if (isAivMode) {
459 0 : if (isAivCrossNode) {
460 0 : algName = "ReduceScatterMeshAivFor91093Executor";
461 0 : } else if (
462 0 : (isOpbase && dataSize <= AIV_REDUCE_SCATTER_SMALL_SIZE)
463 0 : || (!isOpbase
464 0 : && dataSize <= std::min(limit.aivCoreLimit / userRankSize_, NUM_BLOCKS_FACTOR_FOUR)
465 0 : * AIV_REDUCE_SCATTER_BIG_SIZE)) {
466 0 : algName = "ReduceScatterMeshAivSmallCountExecutor";
467 : } else {
468 0 : algName = "ReduceScatterMeshAivExecutor";
469 : }
470 0 : HCCL_INFO("[SelectAlgfor91093] ReduceScatter SelectAlgfor91093 is algName [%s]", algName.c_str());
471 0 : return HCCL_SUCCESS;
472 : }
473 : // ARS 算法选择
474 4 : bool isARSAlgo = multiModuleDiffDeviceNumMode_ && !multiSuperPodDiffDeviceNumMode_;
475 4 : if (isARSAlgo) {
476 0 : if (!(algType_.algoLevel1 == AlgTypeLevel1::ALG_LEVEL1_NB
477 0 : || algType_.algoLevel1 == AlgTypeLevel1::ALG_LEVEL1_RING)) {
478 0 : algType_.algoLevel1 = AlgTypeLevel1::ALG_LEVEL1_NHR;
479 0 : HCCL_WARNING("[ReduceScatterOperator][SelectAlgfor91093] ARS only support NHR or RING in AlgoLevel1 "
480 : "yet, default is NHR.");
481 : }
482 : }
483 : // AHC 算法选择逻辑
484 8 : bool isAHCAlgo = (algType_.algoLevel1 == AlgTypeLevel1::ALG_LEVEL1_AHC)
485 4 : || (algType_.algoLevel1 == AlgTypeLevel1::ALG_LEVEL1_AHC_BROKE);
486 4 : if (isAHCAlgo) {
487 0 : CHK_RET(SelectAlgforAHC(dataSize, AHCOpType::AHC_OP_TYPE_REDUCE_SCATTER));
488 : }
489 :
490 : bool isSupportInlineReduce
491 4 : = IsSupportSDMAReduce(param.inputPtr, param.outputPtr, param.DataDes.dataType, param.reduceType);
492 4 : bool isPowOfTwo = ((userRankSize_ - 1) & userRankSize_) == 0;
493 4 : void* commInputPtr = nullptr;
494 4 : u64 commInputSize = 0;
495 4 : s32 HCCS_PORT_NUM_910_93_7 = 7;
496 4 : u64 smallCountSingleServerThreshold
497 4 : = (hccsPortNum_ == HCCS_PORT_NUM_910_93_7) ? HCCL_SMALL_COUNT_512_KB : HCCL_SMALL_COUNT_1_MB;
498 4 : u64 smallCountMultiServerThreshold
499 4 : = (hccsPortNum_ == HCCS_PORT_NUM_910_93_7) ? HCCL_SMALL_COUNT_1_MB : HCCL_SMALL_COUNT_2_MB;
500 4 : CHK_RET(cclBufferManager_.GetInCCLbuffer(commInputPtr, commInputSize));
501 4 : u64 maxPipelineBlockSize = 0;
502 4 : if (userRankSize_ != 0) {
503 : maxPipelineBlockSize
504 4 : = commInputSize / userRankSize_ / HCCL_DEVICE_NUM_TWO / HCCL_MIN_SLICE_ALIGN * HCCL_MIN_SLICE_ALIGN;
505 : }
506 4 : bool isSupportPipelineFor91093 = (maxPipelineBlockSize >= HCCL_SMALL_COUNT_4_MB)
507 4 : && (dataSize * userRankSize_ > HCCL_PIPELINE_TOTAL_DATA_SIZE_THRESHOLD);
508 4 : HCCL_INFO(
509 : "[ReduceScatterOperator][SelectAlgfor91093] dataSize[%llu] commInputSize[%llu] "
510 : "userRankSize[%u] maxPipelineBlockSize[%llu] isSupportPipelineFor91093[%d]",
511 : dataSize, commInputSize, userRankSize_, maxPipelineBlockSize, isSupportPipelineFor91093);
512 3 : bool dmaReduceLimit = (workflowMode_ == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE) && isPowOfTwo
513 7 : && ((commInputSize * HCCL_DEVICE_NUM_TWO
514 3 : < param.DataDes.count * SIZE_TABLE[param.DataDes.dataType] * userRankSize_)
515 0 : || retryEnable_);
516 : bool smallCountOptimSingleServer
517 4 : = (!retryEnable_) && (serverNum_ == 1)
518 0 : && ((workflowMode_ == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE)
519 0 : || (workflowMode_ != HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE && !param.aicpuUnfoldMode))
520 0 : && isSupportInlineReduce && (deviceNumPerAggregation_ > HCCL_DEVICE_NUM_TWO)
521 0 : && (param.DataDes.count * SIZE_TABLE[param.DataDes.dataType] <= smallCountSingleServerThreshold)
522 8 : && !GetExternalInputInterHccsDisable() && !dmaReduceLimit;
523 3 : bool smallCountOptimMultiServer = isSupportInlineReduce && (deviceNumPerAggregation_ > HCCL_DEVICE_NUM_TWO)
524 0 : && (serverNum_ != 1) && (superPodNum_ == 1) && !dmaReduceLimit
525 7 : && !GetExternalInputInterHccsDisable();
526 1 : bool isHccsPlusSio = userRankSize_ == 2 && pairLinkCounter_[static_cast<u32>(LinkTypeInServer::SIO_TYPE)] == 2
527 5 : && pairLinkCounter_[static_cast<u32>(LinkTypeInServer::HCCS_TYPE)] == 0;
528 : bool useHostComm
529 4 : = !isSupportInlineReduce
530 5 : && ((serverNum_ != 1 && superPodNum_ == 1 && !GetExternalInputInterHccsDisable())
531 1 : || ((superPodNum_ > 1 || GetExternalInputInterHccsDisable()) && !retryEnable_
532 0 : && ((isPowOfTwo && param.DataDes.count * SIZE_TABLE[param.DataDes.dataType] <= HCCL_SMALL_COUNT_4_MB)
533 0 : || (!isPowOfTwo
534 0 : && param.DataDes.count * SIZE_TABLE[param.DataDes.dataType] <= HCCL_SMALL_COUNT_2_MB))));
535 4 : bool smallCountOptimMultiPod = false;
536 :
537 4 : isHccsPlusSio = false;
538 4 : if (isHccsPlusSio && isSupportHccsAndSio_) {
539 0 : algName = "ReduceScatterHccsSioExecutor";
540 4 : } else if (multiModuleDiffDeviceNumMode_ && multiSuperPodDiffDeviceNumMode_) {
541 0 : algName = "ReduceScatterComm";
542 4 : } else if (multiModuleDiffDeviceNumMode_ && !multiSuperPodDiffDeviceNumMode_) {
543 0 : algName = "ReduceScatterARSFor91093Executor";
544 4 : } else if (
545 4 : smallCountOptimMultiPod || useHostComm
546 4 : || (smallCountOptimMultiServer && !isPowOfTwo
547 0 : && (param.DataDes.count * SIZE_TABLE[param.DataDes.dataType] <= HCCL_SMALL_COUNT_256_KB))) {
548 0 : algName = "ReduceScatterComm";
549 0 : algType_.algoLevel1 = AlgTypeLevel1::ALG_LEVEL1_HD;
550 4 : } else if (
551 : smallCountOptimSingleServer
552 4 : || (smallCountOptimMultiServer && isPowOfTwo
553 0 : && (param.DataDes.count * SIZE_TABLE[param.DataDes.dataType] * serverNum_
554 : <= smallCountMultiServerThreshold))) {
555 0 : algName = "ReduceScatterDeterExecutor";
556 4 : } else if (
557 3 : isSupportInlineReduce && (param.supportSymmetricMemory || param.supportZeroCopy)
558 0 : && // isSupportInlineReduce：不申请scratch ==> 不支持非InlineReduce
559 0 : (topoType_ == TopoType::TOPO_TYPE_NP_DOUBLE_RING
560 0 : || param.DataDes.count * unitSize * deviceNumPerAggregation_ > HCCL_MID_COUNT_16_MB)) {
561 0 : const u32 SEVER_NUM_FOUR = 4;
562 0 : constexpr u64 RING_EXCHANGE_PIPELINE_DATA_SIZE_MIN = 2 * 1024 * 1024;
563 : HcclAlgoType configAlgTypeLevel2
564 0 : = topoMatcher_->GetAlgoConfig(HcclCMDType::HCCL_CMD_REDUCE_SCATTER)[HCCL_ALGO_LEVEL_2];
565 0 : if ((superPodNum_ > 1) && (userRankSize_ / superPodNum_ > 1)
566 0 : && ((configAlgTypeLevel2 == HcclAlgoType::HCCL_ALGO_TYPE_PIPELINE)
567 0 : || ((configAlgTypeLevel2 == HcclAlgoType::HCCL_ALGO_TYPE_DEFAULT)
568 0 : && (dataSize >= RING_EXCHANGE_PIPELINE_DATA_SIZE_MIN)))) {
569 : // 单算子, 超节点数大于1， 每个超节点的rank数大于1
570 0 : algName = "ReduceScatterRingZerocopyExchangePipelineExecutor"; // 连续数据通信+数据交换+Pipeline
571 0 : algType_.algoLevel2 = AlgTypeLevel2::ALG_LEVEL2_PIPELINE;
572 0 : } else if (serverNum_ < SEVER_NUM_FOUR || isAHCAlgo) {
573 0 : algName = "ReduceScatterRingZerocopyExecutor"; // 非连续数据通信（限制Server数，避免数据切太碎）
574 : } else {
575 0 : algName = "ReduceScatterRingZerocopyExchangeExecutor"; // 连续数据通信+数据交换（AHC不支持）
576 : }
577 4 : } else if (
578 3 : isOpbase && superPodNum_ > 1 && !isAHCAlgo && !multiSuperPodDiffDeviceNumMode_ && isSupportInlineReduce
579 0 : && (topoType_ == TopoType::TOPO_TYPE_NP_SINGLE_RING || topoType_ == TopoType::TOPO_TYPE_NP_DOUBLE_RING)
580 0 : && isSupportPipelineFor91093) {
581 0 : algName = "ReduceScatterPipelineFor91093Executor";
582 : } else {
583 4 : if (topoType_ == TopoType::TOPO_TYPE_NP_SINGLE_RING) {
584 2 : algName = "ReduceScatterRingFor91093Executor";
585 2 : } else if (topoType_ == TopoType::TOPO_TYPE_NP_DOUBLE_RING) {
586 2 : if (IsSupportUnifiedMarch(param, topoType_, serverNum_, superPodNum_)) {
587 0 : algName = "ReduceScatterSemiRingExecutor";
588 : } else {
589 2 : algName = "ReduceScatterFastDoubleRingFor91093Executor";
590 : }
591 : } else {
592 0 : algName = "ReduceScatterComm";
593 : }
594 : }
595 :
596 1 : if (!(algType_.algoLevel1 == AlgTypeLevel1::ALG_LEVEL1_RING || algType_.algoLevel1 == AlgTypeLevel1::ALG_LEVEL1_NB
597 1 : || (algType_.algoLevel0 == AlgTypeLevel0::ALG_LEVEL0_WHOLE_RING
598 0 : && algType_.algoLevel1 == AlgTypeLevel1::ALG_LEVEL1_WHOLE_RING)
599 1 : || algType_.algoLevel1 == AlgTypeLevel1::ALG_LEVEL1_AHC
600 1 : || algType_.algoLevel1 == AlgTypeLevel1::ALG_LEVEL1_AHC_BROKE)
601 5 : && (algName != "ReduceScatterComm" && algType_.algoLevel1 == AlgTypeLevel1::ALG_LEVEL1_HD)) {
602 : // 910_93超节点只支持server间ring,NB和NHR，默认需继续使用NHR
603 0 : algType_.algoLevel1 = AlgTypeLevel1::ALG_LEVEL1_NHR;
604 0 : HCCL_WARNING("[ReduceScatterOperator][SelectAlgfor91093] only support ring, NB AHC and NHR in AlgoLevel1 yet, "
605 : "default is algType=NHR.");
606 : }
607 : // 如果配置了aiv only,但是实际没有选择aiv算法,需要通过DFX打印出具体原因
608 4 : if (isOnlyAiv && !isAivMode && !isSupportAivDeter) {
609 0 : HCCL_ERROR("The current conditions do not meet the aiv only execution criteria because:");
610 0 : CHK_PRT_RET(
611 : !IsSupportAIVReduce(param.DataDes.dataType, param.reduceType),
612 : HCCL_ERROR(
613 : "current data type[%s] or reduceType[%s] not supported, "
614 : "data type support range:[int8, int16, int32, float16, float32, bfloat16] reduce type support "
615 : "range:[sum, max, min]",
616 : GetDataTypeEnumStr(param.DataDes.dataType).c_str(), GetReduceOpEnumStr(param.reduceType).c_str()),
617 : HCCL_E_NOT_SUPPORT);
618 :
619 0 : CHK_PRT_RET(retryEnable_, HCCL_ERROR("retryEnable [%d] not supported", retryEnable_), HCCL_E_NOT_SUPPORT);
620 :
621 0 : CHK_PRT_RET(
622 : superPodNum_ != 1, HCCL_ERROR("multi superpod [%u] not supported", superPodNum_), HCCL_E_NOT_SUPPORT);
623 :
624 0 : CHK_PRT_RET(
625 : multiModuleDiffDeviceNumMode_,
626 : HCCL_ERROR("multiModuleDiffDeviceNumMode [%d] not supported", multiModuleDiffDeviceNumMode_),
627 : HCCL_E_NOT_SUPPORT);
628 0 : return HCCL_E_NOT_SUPPORT;
629 : }
630 4 : HCCL_INFO("[SelectAlgfor91093] ReduceScatter SelectAlgfor91093 is algName [%s]", algName.c_str());
631 :
632 4 : HCCL_INFO(
633 : "[SelectAlgfor91093] isOpbase[%d] superPodNum_[%u] isAHCAlgo[%d] multiSuperPodDiffDeviceNumMode_[%d] "
634 : "isSupportInlineReduce[%d] topoType_[%d] dataSize[%llu]",
635 : isOpbase, superPodNum_, isAHCAlgo, multiSuperPodDiffDeviceNumMode_, isSupportInlineReduce, topoType_, dataSize);
636 4 : return HCCL_SUCCESS;
637 : }
638 :
639 : REGISTER_OP(HcclCMDType::HCCL_CMD_REDUCE_SCATTER, ReduceScatter, ReduceScatterOperator);
640 :
641 : } // namespace hccl
|
