Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "reduce_scatter_operator.h"
12 : #include "device_capacity.h"
13 : #include "coll_alg_utils.h"
14 : #include "hccl_aiv.h"
15 : #include "coll_alg_op_registry.h"
16 : #include <algorithm>
17 :
18 : constexpr u32 MODULE_NUM_FOUR = 4;
19 : constexpr u32 HCCL_310P_DATA_SIZE_MID_COUNT = 320 * 1024;
20 : constexpr u32 HCCL_310P_DATA_SIZE_SMALL_COUNT = 1024;
21 : constexpr u32 HCCL_310P_SLIM_RING_MAX_SIZE = 8;
22 :
23 : // Pipeline并行比串行更优的总数据量临界点（基于910_93 2 SuperPod, 1 Server, 12 Rank 性能基线测试，
24 : // 数据量低于此值时调度开销超过流水收益）
25 : constexpr u64 HCCL_PIPELINE_TOTAL_DATA_SIZE_THRESHOLD = 608 * 1024 * 1024;
26 :
27 : namespace hccl {
28 :
29 31 : ReduceScatterOperator::ReduceScatterOperator(AlgConfigurator* algConfigurator, CCLBufferManager &cclBufferManager,
30 31 : HcclDispatcher dispatcher, std::unique_ptr<TopoMatcher> &topoMatcher) :
31 31 : CollAlgOperator(algConfigurator, cclBufferManager, dispatcher, topoMatcher, HcclCMDType::HCCL_CMD_REDUCE_SCATTER)
32 : {
33 30 : }
34 :
35 74 : ReduceScatterOperator::~ReduceScatterOperator()
36 : {
37 74 : }
38 :
39 1 : HcclResult ReduceScatterOperator::SelectAlg(const std::string& tag, const OpParam& param, std::string& algName,
40 : std::string& newTag)
41 : {
42 1 : ResourceLimit limit;
43 2 : return SelectAlg(tag, param, algName, newTag, limit);
44 : }
45 :
46 35 : HcclResult ReduceScatterOperator::SelectAlg(const std::string& tag, const OpParam& param, std::string& algName,
47 : std::string& newTag, const ResourceLimit &limit)
48 : {
49 35 : if (userRankSize_ == 1) {
50 1 : algName = "ReduceScatterSingleExecutor";
51 1 : return HCCL_SUCCESS;
52 : }
53 : HcclResult ret;
54 34 : if (isDiffDeviceType_) {
55 0 : ret = SelectAlgforMix(param, algName);
56 34 : } else if (deviceType_ == DevType::DEV_TYPE_310P3) {
57 0 : ret = SelectAlgfor310P3(param, algName);
58 34 : } else if (deviceType_ == DevType::DEV_TYPE_910) {
59 0 : ret = SelectAlgfor910A(param, algName);
60 34 : } else if (deviceType_ == DevType::DEV_TYPE_910B) {
61 29 : ret = SelectAlgfor910B(param, algName);
62 5 : } else if (deviceType_ == DevType::DEV_TYPE_910_93) {
63 4 : ret = SelectAlgfor91093(param, algName, limit);
64 : } else {
65 1 : HCCL_ERROR("[SelectAlg] device type[%d] is out of range for selector.", deviceType_);
66 0 : return HCCL_E_NOT_SUPPORT;
67 : }
68 36 : CHK_PRT_RET(ret != HCCL_SUCCESS,
69 : HCCL_ERROR("[ReduceScatterSelector][SelectAlg]tag[%s], ReduceScatter failed, return[%d]",
70 : tag.c_str(), ret), ret);
71 :
72 36 : if (GetWorkflowMode() == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OPS_KERNEL_INFO_LIB) {
73 13 : newTag = tag;
74 : } else {
75 23 : if (deviceType_ == DevType::DEV_TYPE_310P3) {
76 0 : newTag = tag + algName;
77 : } else {
78 23 : auto level1Iter = HCCL_ALGO_LEVEL1_NAME_MAP.find(algType_.algoLevel1);
79 18 : CHK_PRT_RET(level1Iter == HCCL_ALGO_LEVEL1_NAME_MAP.end(), HCCL_ERROR("level1: algType1[%u] is invalid.",
80 : algType_.algoLevel1), HCCL_E_INTERNAL);
81 18 : newTag = tag + level1Iter->second + algName;
82 : }
83 :
84 22 : bool isInlineReduce = IsSupportSDMAReduce(cclBufferManager_.GetInCCLbuffer().ptr(),
85 22 : cclBufferManager_.GetOutCCLbuffer().ptr(), param.DataDes.dataType, param.reduceType);
86 21 : bool isRdmaReduce = IsSupportRDMAReduce(param.DataDes.dataType, param.reduceType);
87 21 : const std::string REDUCE_SCATTER_NO_INLINE = "_no_inline";
88 35 : newTag = (isInlineReduce && isRdmaReduce) ? newTag : newTag + REDUCE_SCATTER_NO_INLINE;
89 23 : }
90 36 : if (algName == "ReduceScatterARSFor91093Executor") {
91 0 : u32 ringSize = CalcOptimalIntraRingsize(param.DataDes.count, param.DataDes.dataType, HcclCMDType::HCCL_CMD_REDUCE_SCATTER);
92 0 : newTag += std::to_string(ringSize);
93 : }
94 35 : newTag += (param.aicpuUnfoldMode ? "_device" : "_host");
95 35 : return ret;
96 : }
97 :
98 0 : HcclResult ReduceScatterOperator::SelectAlgforMix(const OpParam& param, std::string& algName)
99 : {
100 : (void) param;
101 :
102 : // 混合组网场景不支持规约保序
103 0 : if (IsNeedStrictMode(param)) {
104 0 : HCCL_ERROR("[ReduceScatterOperator][SelectAlgforMix] not support DETERMINISTIC_STRICT mode.");
105 0 : return HCCL_E_NOT_SUPPORT;
106 : }
107 :
108 0 : if (gcdDeviceNumPerAggregation_ > 1) {
109 0 : algType_.algoLevel1 = AlgTypeLevel1::ALG_LEVEL1_NHR;
110 0 : HCCL_WARNING("[ReduceScatterOperator][SelectAlgforMix] only support NHR in AlgoLevel1 yet, "\
111 : "default is algType=NHR.");
112 0 : algName = "ReduceScatterMixExecutor";
113 : } else {
114 0 : algType_.algoLevel1 = AlgTypeLevel1::ALG_LEVEL1_RING;;
115 0 : HCCL_WARNING("[ReduceScatterOperator][SelectAlgforMix] only support ring in AlgoComm yet, "\
116 : "default is algType=ring.");
117 0 : algName = "ReduceScatterComm";
118 : }
119 :
120 0 : HCCL_INFO("[SelectAlgforMix] ReduceScatter SelectAlgforMix is algName [%s].", algName.c_str());
121 0 : return HCCL_SUCCESS;
122 : }
123 :
124 0 : HcclResult ReduceScatterOperator::SelectAlgfor310P3(const OpParam& param, std::string& algName)
125 : {
126 0 : if(HCCL_310P_DATA_SIZE_SMALL_COUNT< param.DataDes.count &¶m.DataDes.count <= HCCL_310P_DATA_SIZE_MID_COUNT && userRankSize_ <= HCCL_310P_SLIM_RING_MAX_SIZE){
127 0 : algName = "ReduceScatterSlimRing";
128 : }
129 : else {
130 0 : algName = "ReduceScatterRing";
131 : }
132 :
133 0 : HCCL_INFO("[SelectAlgfor310P3] ReduceScatter SelectAlgfor310P3 is algName [%s] DataDesCount [%llu]", algName.c_str(), param.DataDes.count);
134 0 : return HCCL_SUCCESS;
135 : }
136 :
137 0 : HcclResult ReduceScatterOperator::SelectAlgfor910A(const OpParam& param, std::string& algName)
138 : {
139 0 : bool isMeshTopo = topoType_ == TopoType::TOPO_TYPE_4P_MESH || topoType_ == TopoType::TOPO_TYPE_2P_MESH;
140 0 : bool isRingTopo = topoType_ == TopoType::TOPO_TYPE_NP_SINGLE_RING || topoType_ == TopoType::TOPO_TYPE_8P_RING;
141 :
142 0 : const u32 RANK_SIZE_FOUR = 4;
143 0 : const u32 RANK_SIZE_EIGHT = 8;
144 0 : bool isOpbase = GetWorkflowMode() == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE;
145 0 : u64 dataSize = SIZE_TABLE[param.DataDes.dataType] * param.DataDes.count;
146 0 : if (isOpbase && serverNum_ == 1 && dataSize <= HCCL_SMALL_COUNT_256_KB
147 0 : && (userRankSize_ == RANK_SIZE_FOUR || userRankSize_ == RANK_SIZE_EIGHT)) {
148 0 : algType_.algoLevel1 = AlgTypeLevel1::ALG_LEVEL1_HD;
149 0 : algName = "ReduceScatterComm";
150 0 : } else if (isMeshTopo) {
151 0 : algName = "ReduceScatterMeshExecutor";
152 0 : } else if (isRingTopo) {
153 0 : algName = "ReduceScatterRingExecutor";
154 : } else {
155 0 : algName = "ReduceScatterComm";
156 : }
157 0 : HCCL_INFO("[SelectAlgfor910A] ReduceScatter SelectAlgfor910A is algName [%s].", algName.c_str());
158 0 : return HCCL_SUCCESS;
159 : }
160 :
161 27 : HcclResult ReduceScatterOperator::SelectAlgfor910B(const OpParam& param, std::string& algName)
162 : {
163 27 : u32 unitSize = SIZE_TABLE[param.DataDes.dataType];
164 :
165 27 : bool isOnlyAiv = topoMatcher_->GetIsOnlyAivConfig();
166 0 : bool isMeshTopo = topoType_ == TopoType::TOPO_TYPE_NP_MESH || topoType_ == TopoType::TOPO_TYPE_4P_MESH ||
167 29 : topoType_ == TopoType::TOPO_TYPE_2P_MESH || topoType_ == TopoType::TOPO_TYPE_1P_MESH;
168 29 : bool isRingTopo = topoType_ == TopoType::TOPO_TYPE_NP_SINGLE_RING;
169 :
170 29 : u64 dataSize = param.DataDes.count * unitSize; // 单位：字节
171 29 : u64 cclBufferSize = cclBufferManager_.GetInCCLbufferSize() / userRankSize_;
172 :
173 29 : void *commInputPtr = nullptr;
174 29 : void *commOutputPtr = nullptr;
175 29 : u64 commInputSize = 0;
176 29 : u64 commOutputSize = 0;
177 :
178 29 : CHK_RET(cclBufferManager_.GetInCCLbuffer(commInputPtr, commInputSize));
179 27 : CHK_RET(cclBufferManager_.GetOutCCLbuffer(commOutputPtr, commOutputSize));
180 28 : bool isServNumPowOfTwo = (serverNum_ > 0) && ((serverNum_ & (serverNum_ - 1)) == 0);
181 28 : bool isOpbase = (GetWorkflowMode() == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE);
182 28 : bool isInlineReduce = IsSupportSDMAReduce(cclBufferManager_.GetInCCLbuffer().ptr(),
183 26 : cclBufferManager_.GetOutCCLbuffer().ptr(), param.DataDes.dataType, param.reduceType);
184 :
185 28 : if (topoMatcher_->GetDeterministicConfig() == DETERMINISTIC_STRICT) {
186 0 : if (multiModuleDiffDeviceNumMode_) {
187 : // 保序规约场景（多batch一致），当前不支持A2非对称场景
188 0 : HCCL_ERROR("[SelectAlgfor910B] reduce order preservation only support"
189 : " Symmetry(multiModuleDiffDeviceNumMode_[%d]).", multiModuleDiffDeviceNumMode_);
190 0 : return HCCL_E_NOT_SUPPORT;
191 : }
192 0 : if (param.DataDes.dataType == HCCL_DATA_TYPE_FP16 || param.DataDes.dataType == HCCL_DATA_TYPE_FP32
193 0 : || param.DataDes.dataType == HCCL_DATA_TYPE_BFP16) {
194 : // 只有浮点数存在多batch不一致的可能，整数天然一致
195 0 : if (param.aicpuUnfoldMode || topoMatcher_->GetAivModeConfig()) {
196 : // AIV / AICPU场景，规约保序优先级更高
197 0 : HCCL_WARNING("[SelectAlgfor910B]aicpuMode[%d], AivModeConfig[%d], "
198 : "the AIV/AICPU mode does not support when the reduce order preservation is enabled.",
199 : param.aicpuUnfoldMode, topoMatcher_->GetAivModeConfig());
200 : }
201 0 : algName = "ReduceScatterOrderPreservedExecutor";
202 0 : HCCL_INFO("[SelectAlgfor910B] ReduceScatterSelectAlgfor910B is algName [%s].", algName.c_str());
203 0 : return HCCL_SUCCESS;
204 : }
205 : }
206 :
207 : // 暂只支持单算子模式
208 46 : bool isCCLBufferGE16M = isOpbase &&
209 18 : (commInputSize >= HCCL_MID_COUNT_16_MB && commOutputSize >= HCCL_MID_COUNT_16_MB);
210 :
211 56 : bool isSupportAivRdmaCount = !isSingleMeshAggregation_
212 0 : && !multiModuleDiffDeviceNumMode_
213 0 : && isMeshTopo
214 28 : && (((isServNumPowOfTwo || dataSize <= HCCL_SMALL_COUNT_128_KB)
215 0 : && dataSize * userRankSize_ <= HCCL_MID_COUNT_16_MB
216 0 : && isCCLBufferGE16M
217 0 : && dataSize <= HCCL_SMALL_COUNT_256_KB) || isOnlyAiv);
218 :
219 28 : bool isSupportAivDeter = isSingleMeshAggregation_
220 27 : && serverNum_ == 1
221 27 : && (topoMatcher_->GetDeterministicConfig() == DETERMINISTIC_ENABLE)
222 53 : && ((dataSize * userRankSize_ <= HCCL_SMALL_COUNT_8_MB) || isOnlyAiv);
223 :
224 26 : bool isAivMode = topoMatcher_->GetAivModeConfig()
225 0 : && IsSupportAIVReduce(param.DataDes.dataType, param.reduceType)
226 0 : && (topoMatcher_->GetDeterministicConfig() == DETERMINISTIC_DISABLE || isSupportAivDeter )
227 25 : && ((isSingleMeshAggregation_ && (dataSize <= AIV_BIG_SIZE || isOnlyAiv)) || isSupportAivRdmaCount);
228 25 : if (isAivMode) {
229 0 : if (isSupportAivDeter) {
230 0 : if (dataSize * userRankSize_ <= HCCL_SMALL_COUNT_8_MB){
231 0 : algName = "ReduceScatterAivDeterSmallExecutor";
232 : } else {
233 0 : algName = "ReduceScatterAivDeterExecutor";
234 : }
235 0 : HCCL_INFO("[SelectAlgfor910B] ReduceScatter SelectAlgfor910B is algName [%s].", algName.c_str());
236 0 : return HCCL_SUCCESS;
237 : }
238 0 : if (isSupportAivRdmaCount) {
239 0 : algName = "ReduceScatterAivRdmaExecutor";
240 0 : } else if (workflowMode_ == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE && dataSize <= AIV_REDUCE_SCATTER_MID_SIZE) {
241 0 : algName = "ReduceScatterMeshAivSmallCountExecutor";
242 : } else {
243 0 : algName = "ReduceScatterMeshAivExecutor";
244 : }
245 0 : HCCL_INFO("[SelectAlgfor910BAIV] ReduceScatterSelectAlgfor910B is algName [%s].", algName.c_str());
246 0 : return HCCL_SUCCESS;
247 : }
248 :
249 25 : if (GetWorkflowMode() == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE) {
250 15 : bool isRdmaReduce = IsSupportRDMAReduce(param.DataDes.dataType, param.reduceType);
251 :
252 18 : std::string algTypeLevel1Tag;
253 18 : CHK_RET(AutoSelectAlgTypeLevel1(HcclCMDType::HCCL_CMD_REDUCE_SCATTER, dataSize, cclBufferSize, algTypeLevel1Tag,
254 : isInlineReduce, isRdmaReduce));
255 20 : if (GetExternalInputHcclEnableEntryLog() && param.opBaseAtraceInfo != nullptr) {
256 0 : CHK_RET(param.opBaseAtraceInfo->SavealgtypeTraceInfo(algTypeLevel1Tag, param.tag));
257 : }
258 18 : }
259 :
260 : // AHC 算法选择逻辑
261 28 : if (((algType_.algoLevel1 == AlgTypeLevel1::ALG_LEVEL1_AHC) ||
262 28 : (algType_.algoLevel1 == AlgTypeLevel1::ALG_LEVEL1_AHC_BROKE))) {
263 0 : CHK_RET(SelectAlgforAHC(dataSize, AHCOpType::AHC_OP_TYPE_REDUCE_SCATTER));
264 : }
265 :
266 : // pipeline算法task数量多，如果超出FFTS子图限制，则重定向到HD算法
267 28 : if (algType_.algoLevel1 == AlgTypeLevel1::ALG_LEVEL1_PIPELINE) {
268 0 : u32 contextNum = CalcContextNumForPipeline(HcclCMDType::HCCL_CMD_REDUCE_SCATTER);
269 0 : if (contextNum > HCCL_FFTS_CAPACITY) {
270 0 : algType_.algoLevel1 = AlgTypeLevel1::ALG_LEVEL1_HD;
271 0 : HCCL_WARNING("[ReduceScatterOperator][SelectAlgfor910B] context num[%u] is out of capacity of FFTS+ "\
272 : "graph[%u], reset algorithm to HD.", contextNum, HCCL_FFTS_CAPACITY);
273 : }
274 : }
275 :
276 30 : if (isMeshTopo) {
277 30 : if (topoMatcher_->GetDeterministicConfig() == DETERMINISTIC_ENABLE
278 8 : && GetWorkflowMode() == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE
279 5 : && algType_.algoLevel1 == AlgTypeLevel1::ALG_LEVEL1_PIPELINE
280 34 : && deviceNumPerAggregation_ > DEVICE_TWO) {
281 0 : algName = "ReduceScatterDeterPipelineExecutor";
282 26 : } else if (GetWorkflowMode() == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE) {
283 17 : bool enableSmallCountDeterministicAlgo = !isSingleMeshAggregation_ &&
284 0 : IsSupportSDMAReduce(cclBufferManager_.GetInCCLbuffer().ptr(),
285 0 : cclBufferManager_.GetOutCCLbuffer().ptr(), param.DataDes.dataType, param.reduceType);
286 17 : if (SingleMeshInlineReduce(cclBufferManager_.GetInCCLbuffer().ptr(),
287 17 : cclBufferManager_.GetOutCCLbuffer().ptr(), param.DataDes.dataType, param.reduceType)) {
288 12 : if (topoMatcher_->GetDeterministicConfig() != DETERMINISTIC_DISABLE) {
289 1 : algName = "ReduceScatterDeterExecutor";
290 : } else {
291 8 : algName = "ReduceScatterMeshDmaEliminationExecutor";
292 : }
293 6 : } else if (topoMatcher_->GetDeterministicConfig() == DETERMINISTIC_DISABLE &&
294 6 : algType_.algoLevel1 == AlgTypeLevel1::ALG_LEVEL1_PIPELINE &&
295 0 : IsMultiMeshInlineReduce(cclBufferManager_.GetInCCLbuffer().ptr(),
296 0 : cclBufferManager_.GetOutCCLbuffer().ptr(), param.DataDes.dataType, param.reduceType)) {
297 0 : algName = "ReduceScatterMeshOpbasePipelineExecutor";
298 6 : } else if (enableSmallCountDeterministicAlgo && ((dataSize <= HCCL_SMALL_COUNT_512_KB &&
299 0 : topoMatcher_->GetDeterministicConfig() == DETERMINISTIC_ENABLE) ||
300 0 : dataSize * userRankSize_< HCCL_SMALL_COUNT_512_KB)) {
301 0 : algName = "ReduceScatterMeshOpbaseSmallCountDeterministicExecutor";
302 : }
303 : } else {
304 9 : if (SingleMeshInlineReduce(param.inputPtr, param.outputPtr, param.DataDes.dataType, param.reduceType)) {
305 12 : if (topoMatcher_->GetDeterministicConfig() != DETERMINISTIC_DISABLE &&
306 3 : deviceNumPerAggregation_ > DEVICE_TWO) {
307 3 : algName = "ReduceScatterDeterExecutor";
308 : } else {
309 6 : if (dataSize <= HCCL_SMALL_COUNT_1_MB &&
310 6 : workflowMode_ == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OPS_KERNEL_INFO_LIB) {
311 6 : algName = "ReduceScatterMeshGraphExecutor";
312 : } else {
313 0 : algName = "ReduceScatterMeshExecutor";
314 : }
315 : }
316 : }
317 : }
318 29 : if (algName.empty()) {
319 0 : if (workflowMode_ == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OPS_KERNEL_INFO_LIB && moduleNum_ > 1 &&
320 0 : deviceNumPerAggregation_ > 1 && topoMatcher_->GetDeterministicConfig() == DETERMINISTIC_DISABLE &&
321 6 : IsMultiMeshInlineReduce(param.inputPtr, param.outputPtr, param.DataDes.dataType, param.reduceType) &&
322 0 : (dataSize > HCCL_SMALL_COUNT_1_MB || moduleNum_ <= MODULE_NUM_FOUR ||
323 0 : algType_.algoLevel1 == AlgTypeLevel1::ALG_LEVEL1_PIPELINE)) {
324 0 : algName = "ReduceScatterMeshGraphPipelineExecutor";
325 6 : } else if (workflowMode_ == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE ||
326 : dataSize > HCCL_SMALL_COUNT_1_MB) {
327 6 : algName = "ReduceScatterMeshExecutor";
328 0 : } else if (workflowMode_ == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OPS_KERNEL_INFO_LIB) {
329 0 : algName = "ReduceScatterMeshGraphExecutor";
330 : }
331 : }
332 0 : } else if (isRingTopo) {
333 0 : algName = "ReduceScatterRingExecutor";
334 : } else {
335 0 : algName = "ReduceScatterComm";
336 : }
337 : // 如果配置了aiv only,但是实际没有选择aiv算法,需要通过DFX打印出具体原因
338 32 : if (isOnlyAiv && !isAivMode) {
339 0 : HCCL_ERROR("The current conditions do not meet the aiv only execution criteria because:");
340 0 : CHK_PRT_RET(!IsSupportAIVReduce(param.DataDes.dataType, param.reduceType), HCCL_ERROR("current data type[%s] or reduceType[%s] not supported, "\
341 : "data type support range:[int8, int16, int32, float16, float32, bfloat16] reduce type support range:[sum, max, min]",
342 : GetDataTypeEnumStr(param.DataDes.dataType).c_str(), GetReduceOpEnumStr(param.reduceType).c_str()), HCCL_E_NOT_SUPPORT);
343 :
344 0 : CHK_PRT_RET(!isSupportAivDeter, HCCL_ERROR("is not support aiv deter.isSingleMeshAggregation_[%d] isOpbase[%d] "\
345 : "deterministic config[%u] dataSize[%llu], serverNum_[%u]",
346 : isSingleMeshAggregation_, isOpbase, topoMatcher_->GetDeterministicConfig(), dataSize, serverNum_), HCCL_E_NOT_SUPPORT);
347 :
348 0 : CHK_PRT_RET(!isSingleMeshAggregation_ && multiModuleDiffDeviceNumMode_,
349 : HCCL_ERROR("The number of cards between servers in a multi-server setup must be consistent. "\
350 : "isSingleMeshAggregation_[%d] multiModuleDiffDeviceNumMode_[%d]",
351 : isSingleMeshAggregation_, multiModuleDiffDeviceNumMode_), HCCL_E_NOT_SUPPORT);
352 :
353 0 : CHK_PRT_RET(!isMeshTopo, HCCL_ERROR("current topo type[%d] not supported", topoType_), HCCL_E_NOT_SUPPORT);
354 :
355 0 : CHK_PRT_RET(!isServNumPowOfTwo, HCCL_ERROR("server num[%u] is pow of two.", serverNum_), HCCL_E_NOT_SUPPORT);
356 :
357 0 : CHK_PRT_RET(!isCCLBufferGE16M, HCCL_ERROR("current isOpbase[%d] or commInputSize[%llu] or commOutputSize[%llu] not supported",
358 : isOpbase, commInputSize, commOutputSize), HCCL_E_NOT_SUPPORT);
359 0 : HCCL_ERROR("isSingleMeshAggregation_[%d] multiModuleDiffDeviceNumMode_[%d] dataSize[%llu]",
360 : isSingleMeshAggregation_, multiModuleDiffDeviceNumMode_, dataSize);
361 0 : return HCCL_E_NOT_SUPPORT;
362 : }
363 32 : HCCL_INFO("[SelectAlgfor910B] ReduceScatter SelectAlgfor910B is algName [%s], current mode is [%u].", algName.c_str(), workflowMode_);
364 32 : return HCCL_SUCCESS;
365 : }
366 :
367 4 : HcclResult ReduceScatterOperator::SelectAlgfor91093(const OpParam& param, std::string& algName, const ResourceLimit &limit)
368 : {
369 4 : u32 unitSize = SIZE_TABLE[param.DataDes.dataType];
370 4 : u64 dataSize = param.DataDes.count * unitSize; // 单位：字节
371 4 : if (dataSize >= cclBufferManager_.GetInCCLbufferSize()) {
372 4 : HCCL_WARNING("The current inCCLbufferSize is [%llu] bytes, change the HCCL_BUFFSIZE environment variable "\
373 : "to be greater than the current data volume[%llu] bytes to improve the performance of the 91093 environment.",
374 : cclBufferManager_.GetInCCLbufferSize(), dataSize);
375 : }
376 4 : bool isOnlyAiv = topoMatcher_->GetIsOnlyAivConfig();
377 4 : bool isOpbase = workflowMode_ == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE;
378 :
379 4 : bool isAivCrossNode = superPodNum_ == 1
380 0 : && serverNum_ > 1
381 0 : && !GetExternalInputInterHccsDisable()
382 4 : && (
383 0 : ((userRankSize_ <= ONE_EIGHTH_MAX_NUM_BLOCKS && dataSize <= AIV_REDUCE_SCATTER_A3_SMALL_RANKSIZE_ENTRY_SIZE) ||
384 0 : (userRankSize_ <= ONE_THIRD_MAX_NUM_BLOCKS && dataSize <= AIV_REDUCE_SCATTER_A3_MID_RANKSIZE_ENTRY_SIZE) ||
385 0 : (dataSize <= AIV_REDUCE_SCATTER_A3_LARGE_RANKSIZE_ENTRY_SIZE) || isOnlyAiv)
386 4 : );
387 :
388 8 : bool isAivSingleNode = serverNum_ == 1
389 4 : && (
390 0 : (isOpbase && (dataSize <= AIV_REDUCE_SCATTER_A3_ENTRY_SIZE || isOnlyAiv)) ||
391 0 : (!isOpbase && (dataSize <= AIV_REDUCE_SCATTER_A3_GRAPH_ENTRY_SIZE || isOnlyAiv))
392 : );
393 :
394 : // A3 AIV 确定性 超节点内(单机与跨机) 支持单算子与图模式 限制单卡数据量8MB
395 4 : bool isSupportAivDeter = (superPodNum_ == 1)
396 0 : && topoMatcher_->GetAivModeConfig()
397 0 : && IsSupportAIVReduce(param.DataDes.dataType, param.reduceType)
398 0 : && (topoMatcher_->GetDeterministicConfig() != DETERMINISTIC_DISABLE)
399 0 : && ((dataSize * userRankSize_ < HCCL_SMALL_COUNT_8_MB) || isOnlyAiv)
400 0 : && (!retryEnable_)
401 0 : && userRankSize_ > 1
402 4 : && !multiModuleDiffDeviceNumMode_;
403 :
404 4 : bool isAivMode = topoMatcher_->GetAivModeConfig()
405 0 : && IsSupportAIVReduce(param.DataDes.dataType, param.reduceType)
406 0 : && ( isAivSingleNode || isAivCrossNode )
407 0 : && (topoMatcher_->GetDeterministicConfig() == DETERMINISTIC_DISABLE)
408 0 : && (!retryEnable_)
409 4 : && !multiModuleDiffDeviceNumMode_;
410 :
411 4 : if (isSupportAivDeter) {
412 0 : algName = "ReduceScatterMeshAivFor91093Executor";
413 0 : HCCL_INFO("[SelectAlgfor91093] reduce_scatter SelectAlgfor91093 algName [%s]", algName.c_str());
414 0 : return HCCL_SUCCESS;
415 : }
416 :
417 4 : if (IsNeedStrictMode(param)) {
418 0 : CHK_PRT_RET(!CheckStrictCondition(param),
419 : HCCL_ERROR("[ReduceScatterOperator][SelectAlgfor91093] not support DETERMINISTIC_STRICT mode."),
420 : HCCL_E_NOT_SUPPORT);
421 :
422 0 : algName = "ReduceScatterOrderPreservedFor91093Executor";
423 0 : HCCL_INFO("[SelectAlgfor91093] reduce_scatter SelectAlgfor91093 algName [%s].", algName.c_str());
424 0 : return HCCL_SUCCESS;
425 : }
426 :
427 4 : if (isAivMode) {
428 0 : if (isAivCrossNode) {
429 0 : algName = "ReduceScatterMeshAivFor91093Executor";
430 0 : } else if ((isOpbase && dataSize <= AIV_REDUCE_SCATTER_SMALL_SIZE)
431 0 : || (!isOpbase && dataSize <= std::min(limit.aivCoreLimit / userRankSize_, NUM_BLOCKS_FACTOR_FOUR)
432 0 : * AIV_REDUCE_SCATTER_BIG_SIZE)) {
433 0 : algName = "ReduceScatterMeshAivSmallCountExecutor";
434 : } else {
435 0 : algName = "ReduceScatterMeshAivExecutor";
436 : }
437 0 : HCCL_INFO("[SelectAlgfor91093] ReduceScatter SelectAlgfor91093 is algName [%s]", algName.c_str());
438 0 : return HCCL_SUCCESS;
439 : }
440 : // ARS 算法选择
441 4 : bool isARSAlgo = multiModuleDiffDeviceNumMode_ && !multiSuperPodDiffDeviceNumMode_;
442 4 : if (isARSAlgo) {
443 0 : if (!(algType_.algoLevel1 == AlgTypeLevel1::ALG_LEVEL1_NB || algType_.algoLevel1 == AlgTypeLevel1::ALG_LEVEL1_RING)) {
444 0 : algType_.algoLevel1 = AlgTypeLevel1::ALG_LEVEL1_NHR;
445 0 : HCCL_WARNING("[ReduceScatterOperator][SelectAlgfor91093] ARS only support NHR or RING in AlgoLevel1 "\
446 : "yet, default is NHR.");
447 : }
448 : }
449 : // AHC 算法选择逻辑
450 4 : bool isAHCAlgo = (algType_.algoLevel1 == AlgTypeLevel1::ALG_LEVEL1_AHC) || (algType_.algoLevel1 == AlgTypeLevel1::ALG_LEVEL1_AHC_BROKE);
451 4 : if (isAHCAlgo) {
452 0 : CHK_RET(SelectAlgforAHC(dataSize, AHCOpType::AHC_OP_TYPE_REDUCE_SCATTER));
453 : }
454 :
455 4 : bool isSupportInlineReduce = IsSupportSDMAReduce(param.inputPtr, param.outputPtr, param.DataDes.dataType, param.reduceType);
456 4 : bool isPowOfTwo = ((userRankSize_ - 1) & userRankSize_) == 0;
457 4 : void *commInputPtr = nullptr;
458 4 : u64 commInputSize = 0;
459 4 : s32 HCCS_PORT_NUM_910_93_7 = 7;
460 4 : u64 smallCountSingleServerThreshold = (hccsPortNum_ == HCCS_PORT_NUM_910_93_7) ? HCCL_SMALL_COUNT_512_KB : HCCL_SMALL_COUNT_1_MB;
461 4 : u64 smallCountMultiServerThreshold = (hccsPortNum_ == HCCS_PORT_NUM_910_93_7) ? HCCL_SMALL_COUNT_1_MB : HCCL_SMALL_COUNT_2_MB;
462 4 : CHK_RET(cclBufferManager_.GetInCCLbuffer(commInputPtr, commInputSize));
463 4 : u64 maxPipelineBlockSize = 0;
464 4 : if (userRankSize_ != 0) {
465 4 : maxPipelineBlockSize = commInputSize / userRankSize_ / HCCL_DEVICE_NUM_TWO /
466 4 : HCCL_MIN_SLICE_ALIGN * HCCL_MIN_SLICE_ALIGN;
467 : }
468 4 : bool isSupportPipelineFor91093 = (maxPipelineBlockSize >= HCCL_SMALL_COUNT_4_MB) &&
469 0 : (dataSize * userRankSize_ > HCCL_PIPELINE_TOTAL_DATA_SIZE_THRESHOLD);
470 4 : HCCL_INFO("[ReduceScatterOperator][SelectAlgfor91093] dataSize[%llu] commInputSize[%llu] "
471 : "userRankSize[%u] maxPipelineBlockSize[%llu] isSupportPipelineFor91093[%d]",
472 : dataSize, commInputSize, userRankSize_, maxPipelineBlockSize, isSupportPipelineFor91093);
473 7 : bool dmaReduceLimit = (workflowMode_ == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE) && isPowOfTwo &&
474 3 : ((commInputSize * HCCL_DEVICE_NUM_TWO < param.DataDes.count * SIZE_TABLE[param.DataDes.dataType] * userRankSize_) ||
475 0 : retryEnable_);
476 : bool smallCountOptimSingleServer =
477 8 : (!retryEnable_) &&
478 4 : (serverNum_ == 1) &&
479 0 : ((workflowMode_ == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE) ||
480 0 : (workflowMode_ != HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE && !param.aicpuUnfoldMode)) &&
481 0 : isSupportInlineReduce &&
482 0 : (deviceNumPerAggregation_ > HCCL_DEVICE_NUM_TWO) &&
483 0 : (param.DataDes.count * SIZE_TABLE[param.DataDes.dataType] <= smallCountSingleServerThreshold) &&
484 8 : !GetExternalInputInterHccsDisable() && !dmaReduceLimit;
485 : bool smallCountOptimMultiServer =
486 3 : isSupportInlineReduce &&
487 3 : (deviceNumPerAggregation_ > HCCL_DEVICE_NUM_TWO) && (serverNum_ != 1) && (superPodNum_ == 1) &&
488 7 : !dmaReduceLimit && !GetExternalInputInterHccsDisable();
489 4 : bool isHccsPlusSio = userRankSize_ == 2 && pairLinkCounter_[static_cast<u32>(LinkTypeInServer::SIO_TYPE)] == 2 &&
490 0 : pairLinkCounter_[static_cast<u32>(LinkTypeInServer::HCCS_TYPE)] == 0;
491 5 : bool useHostComm = !isSupportInlineReduce && ((serverNum_ != 1 && superPodNum_ == 1 && !GetExternalInputInterHccsDisable())
492 1 : || ((superPodNum_ > 1 || GetExternalInputInterHccsDisable()) && !retryEnable_
493 0 : && ((isPowOfTwo && param.DataDes.count * SIZE_TABLE[param.DataDes.dataType] <= HCCL_SMALL_COUNT_4_MB)
494 0 : || (!isPowOfTwo && param.DataDes.count * SIZE_TABLE[param.DataDes.dataType] <= HCCL_SMALL_COUNT_2_MB))));
495 4 : bool smallCountOptimMultiPod = false;
496 :
497 4 : isHccsPlusSio = false;
498 4 : if (isHccsPlusSio && isSupportHccsAndSio_) {
499 0 : algName = "ReduceScatterHccsSioExecutor";
500 4 : } else if (multiModuleDiffDeviceNumMode_ && multiSuperPodDiffDeviceNumMode_) {
501 0 : algName = "ReduceScatterComm";
502 4 : } else if (multiModuleDiffDeviceNumMode_ && !multiSuperPodDiffDeviceNumMode_) {
503 0 : algName = "ReduceScatterARSFor91093Executor";
504 4 : } else if (smallCountOptimMultiPod || useHostComm || (smallCountOptimMultiServer && !isPowOfTwo &&
505 0 : (param.DataDes.count * SIZE_TABLE[param.DataDes.dataType] <= HCCL_SMALL_COUNT_256_KB))) {
506 0 : algName = "ReduceScatterComm";
507 0 : algType_.algoLevel1 = AlgTypeLevel1::ALG_LEVEL1_HD;
508 4 : } else if (smallCountOptimSingleServer ||
509 0 : (smallCountOptimMultiServer && isPowOfTwo &&
510 0 : (param.DataDes.count * SIZE_TABLE[param.DataDes.dataType] * serverNum_ <= smallCountMultiServerThreshold))) {
511 0 : algName = "ReduceScatterDeterExecutor";
512 4 : } else if (isSupportInlineReduce && (param.supportSymmetricMemory || param.supportZeroCopy) && // isSupportInlineReduce：不申请scratch ==> 不支持非InlineReduce
513 0 : (topoType_ == TopoType::TOPO_TYPE_NP_DOUBLE_RING || param.DataDes.count * unitSize * deviceNumPerAggregation_ > HCCL_MID_COUNT_16_MB)) {
514 0 : const u32 SEVER_NUM_FOUR = 4;
515 0 : constexpr u64 RING_EXCHANGE_PIPELINE_DATA_SIZE_MIN = 2 * 1024 * 1024;
516 0 : HcclAlgoType configAlgTypeLevel2 = topoMatcher_->GetAlgoConfig(HcclCMDType::HCCL_CMD_REDUCE_SCATTER)[HCCL_ALGO_LEVEL_2];
517 0 : if ((superPodNum_ > 1) && (userRankSize_ / superPodNum_ > 1) &&
518 0 : ((configAlgTypeLevel2 == HcclAlgoType::HCCL_ALGO_TYPE_PIPELINE) ||
519 0 : ((configAlgTypeLevel2 == HcclAlgoType::HCCL_ALGO_TYPE_DEFAULT) && (dataSize >= RING_EXCHANGE_PIPELINE_DATA_SIZE_MIN)))) {
520 : // 单算子, 超节点数大于1， 每个超节点的rank数大于1
521 0 : algName = "ReduceScatterRingZerocopyExchangePipelineExecutor"; // 连续数据通信+数据交换+Pipeline
522 0 : algType_.algoLevel2 = AlgTypeLevel2::ALG_LEVEL2_PIPELINE;
523 0 : } else if (serverNum_ < SEVER_NUM_FOUR || isAHCAlgo) {
524 0 : algName = "ReduceScatterRingZerocopyExecutor"; // 非连续数据通信（限制Server数，避免数据切太碎）
525 : } else {
526 0 : algName = "ReduceScatterRingZerocopyExchangeExecutor"; // 连续数据通信+数据交换（AHC不支持）
527 : }
528 4 : } else if (isOpbase && superPodNum_ > 1 &&
529 0 : !isAHCAlgo &&
530 0 : !multiSuperPodDiffDeviceNumMode_ &&
531 0 : isSupportInlineReduce &&
532 0 : (topoType_ == TopoType::TOPO_TYPE_NP_SINGLE_RING ||
533 0 : topoType_ == TopoType::TOPO_TYPE_NP_DOUBLE_RING) &&
534 : isSupportPipelineFor91093) {
535 0 : algName = "ReduceScatterPipelineFor91093Executor";
536 : } else {
537 4 : if (topoType_ == TopoType::TOPO_TYPE_NP_SINGLE_RING) {
538 2 : algName = "ReduceScatterRingFor91093Executor";
539 2 : } else if (topoType_ == TopoType::TOPO_TYPE_NP_DOUBLE_RING) {
540 2 : if (IsSupportUnifiedMarch(param, topoType_, serverNum_, superPodNum_)) {
541 0 : algName = "ReduceScatterSemiRingExecutor";
542 : } else {
543 2 : algName = "ReduceScatterFastDoubleRingFor91093Executor";
544 : }
545 : } else {
546 0 : algName = "ReduceScatterComm";
547 : }
548 : }
549 :
550 1 : if (!(algType_.algoLevel1 == AlgTypeLevel1::ALG_LEVEL1_RING || algType_.algoLevel1 == AlgTypeLevel1::ALG_LEVEL1_NB ||
551 1 : (algType_.algoLevel0 == AlgTypeLevel0::ALG_LEVEL0_WHOLE_RING && algType_.algoLevel1 == AlgTypeLevel1::ALG_LEVEL1_WHOLE_RING) ||
552 1 : algType_.algoLevel1 == AlgTypeLevel1::ALG_LEVEL1_AHC || algType_.algoLevel1 == AlgTypeLevel1::ALG_LEVEL1_AHC_BROKE)
553 5 : && (algName != "ReduceScatterComm" && algType_.algoLevel1 == AlgTypeLevel1::ALG_LEVEL1_HD)) {
554 : // 910_93超节点只支持server间ring,NB和NHR，默认需继续使用NHR
555 0 : algType_.algoLevel1 = AlgTypeLevel1::ALG_LEVEL1_NHR;
556 0 : HCCL_WARNING("[ReduceScatterOperator][SelectAlgfor91093] only support ring, NB AHC and NHR in AlgoLevel1 yet, "\
557 : "default is algType=NHR.");
558 : }
559 : // 如果配置了aiv only,但是实际没有选择aiv算法,需要通过DFX打印出具体原因
560 4 : if (isOnlyAiv && !isAivMode && !isSupportAivDeter) {
561 0 : HCCL_ERROR("The current conditions do not meet the aiv only execution criteria because:");
562 0 : CHK_PRT_RET(!IsSupportAIVReduce(param.DataDes.dataType, param.reduceType), HCCL_ERROR("current data type[%s] or reduceType[%s] not supported, "\
563 : "data type support range:[int8, int16, int32, float16, float32, bfloat16] reduce type support range:[sum, max, min]",
564 : GetDataTypeEnumStr(param.DataDes.dataType).c_str(), GetReduceOpEnumStr(param.reduceType).c_str()), HCCL_E_NOT_SUPPORT);
565 :
566 0 : CHK_PRT_RET(retryEnable_, HCCL_ERROR("retryEnable [%d] not supported", retryEnable_), HCCL_E_NOT_SUPPORT);
567 :
568 0 : CHK_PRT_RET(superPodNum_ != 1, HCCL_ERROR("multi superpod [%u] not supported", superPodNum_), HCCL_E_NOT_SUPPORT);
569 :
570 0 : CHK_PRT_RET(multiModuleDiffDeviceNumMode_, HCCL_ERROR("multiModuleDiffDeviceNumMode [%d] not supported", multiModuleDiffDeviceNumMode_), HCCL_E_NOT_SUPPORT);
571 0 : return HCCL_E_NOT_SUPPORT;
572 : }
573 4 : HCCL_INFO("[SelectAlgfor91093] ReduceScatter SelectAlgfor91093 is algName [%s]", algName.c_str());
574 :
575 4 : HCCL_INFO("[SelectAlgfor91093] isOpbase[%d] superPodNum_[%u] isAHCAlgo[%d] multiSuperPodDiffDeviceNumMode_[%d] "
576 : "isSupportInlineReduce[%d] topoType_[%d] dataSize[%llu]",
577 : isOpbase, superPodNum_, isAHCAlgo, multiSuperPodDiffDeviceNumMode_,
578 : isSupportInlineReduce, topoType_, dataSize);
579 4 : return HCCL_SUCCESS;
580 : }
581 :
582 : REGISTER_OP(HcclCMDType::HCCL_CMD_REDUCE_SCATTER, ReduceScatter, ReduceScatterOperator);
583 :
584 : }
|
