Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include <cmath>
12 : #include <algorithm>
13 : #include "device_capacity.h"
14 : #include "coll_executor_base.h"
15 : #include "coll_alg_exec_registry.h"
16 : #include "coll_alg_operator.h"
17 :
18 : namespace hccl {
19 : using namespace std;
20 : constexpr float GB2B = 1024 * 1024 * 1024;
21 : constexpr float SECOND2MICROSECOND = 1000000;
22 : constexpr float RHD_FACTOR_TWO = 2.0;
23 : constexpr float RHD_FACTOR_ONE = 1.0;
24 : constexpr float DOUBLE_SUB_HCCLCMD = 2.0; // The hcclCMD can be considered as combination of two hcclCMDs.
25 : constexpr float COPY_TIME_IN_RHD = 1.0;
26 : constexpr double NHR_FACTOR_TWO = 2.0;
27 : constexpr double NHR_FACTOR_THREE = 3.0;
28 : constexpr double NHR_FACTOR_FOUR = 4.0;
29 : constexpr double NHR_SUB_TWO = 2.0;
30 : constexpr float LATENCY = 60; // 静态时延 60 us;
31 : constexpr u64 PIPELINE_MIN_SIZE = 32 * 1024; // 当数据量大于等于32KB时，reduce_scatter和all_gather使能pipeline模式
32 : constexpr u64 PIPELINE_ALLREDUCE_MIN_SIZE = 1024 * 1024; // 当数据量大于等于1MB时，allreduce使能pipeline模式
33 : constexpr u64 PIPELINE_MIN_SIZE_NO_LITE = 2 * 1024 * 1024; // 如不支持RDMALite，当数据量大于等于2MB时，使能pipeline模式
34 : constexpr u64 HCCL_FFTS_CAPACITY = 65535; // FFTS+子图最大容量
35 : constexpr u32 AHC_MIN_SUBGROUP_SPLIT_DIVISOR = 2;
36 : constexpr u32 AHC_LEVEL0_GROUP_SIZE_THRESHOLD = 3;
37 : constexpr u32 SERVER_COUNT_THRESHOLD_FOR_MULTI_DETER_PIPELINE = 2;
38 : constexpr u32 MIN_STRICT_RANK_NUM = 3;
39 :
40 89 : CollAlgOperator::CollAlgOperator(
41 : AlgConfigurator* algConfigurator, CCLBufferManager& cclBufferManager, HcclDispatcher dispatcher,
42 89 : std::unique_ptr<TopoMatcher>& topoMatcher, HcclCMDType opType)
43 90 : : algConfigurator_(algConfigurator),
44 90 : cclBufferManager_(cclBufferManager),
45 88 : dispatcher_(dispatcher),
46 88 : topoMatcher_(topoMatcher),
47 596 : workflowMode_(GetWorkflowMode())
48 : {
49 84 : SetTopoAttr(algConfigurator_);
50 91 : SetAlgoAttr(algConfigurator_);
51 94 : algConfigurator->GetAlgTypeDirect(algType_, opType);
52 87 : algConfigurator->GetAlgoLevel1DefaultSwitch(isAlgoLevel1Default_, opType);
53 86 : algConfigurator->GetTopoType(topoType_);
54 86 : }
55 :
56 0 : HcclResult CollAlgOperator::SelectAlg(
57 : [[maybe_unused]] const std::string& tag, [[maybe_unused]] const OpParam& param,
58 : [[maybe_unused]] std::string& algName, [[maybe_unused]] std::string& newTag)
59 : {
60 0 : return HCCL_SUCCESS;
61 : }
62 :
63 36 : HcclResult CollAlgOperator::SelectAlg(
64 : const std::string& tag, const OpParam& param, std::string& algName, std::string& newTag,
65 : [[maybe_unused]] const ResourceLimit& limit)
66 : {
67 36 : return SelectAlg(tag, param, algName, newTag);
68 : }
69 :
70 1 : HcclResult CollAlgOperator::GetAivExecParam(
71 : std::string& algName, const OpParam& param, AlgResourceResponse& algRes, AivSuperKernelArgs& args)
72 : {
73 1 : if (executor_.get() == nullptr) {
74 0 : executor_ = CollAlgExecRegistry::Instance().GetAlgExec(algName, dispatcher_, topoMatcher_);
75 0 : CHK_PRT_RET(
76 : executor_.get() == nullptr,
77 : HCCL_ERROR("[CollAlgOperator][GetAivExecParam]Fail to find executor for algName[%s]", algName.c_str()),
78 : HCCL_E_PARA);
79 : }
80 1 : return executor_->GetAivExecParam(param, algRes, args);
81 : }
82 :
83 : HcclResult
84 1 : CollAlgOperator::CalNumBlocks(std::string& algName, const OpParam& param, u32& numBlocks, int32_t aivCoreLimit)
85 : {
86 1 : if (executor_.get() == nullptr) {
87 0 : executor_ = CollAlgExecRegistry::Instance().GetAlgExec(algName, dispatcher_, topoMatcher_);
88 0 : CHK_PRT_RET(
89 : executor_.get() == nullptr,
90 : HCCL_ERROR("[CollAlgOperator][CalNumBlocks]Fail to find executor for algName[%s]", algName.c_str()),
91 : HCCL_E_PARA);
92 0 : CHK_RET(SetExecutorAttr(param));
93 : }
94 :
95 1 : if (aivCoreLimit != 0) {
96 1 : CHK_RET(executor_->SetNumBlocks(aivCoreLimit));
97 : }
98 :
99 1 : if (param.opType == HcclCMDType::HCCL_CMD_ALLTOALL) {
100 0 : return executor_->CalNumBlocks(
101 0 : numBlocks, userRankSize_, param.All2AllDataDes.sendCount * SIZE_TABLE[param.All2AllDataDes.sendType],
102 0 : param.opType);
103 1 : } else if (
104 1 : param.opType == HcclCMDType::HCCL_CMD_ALLREDUCE || param.opType == HcclCMDType::HCCL_CMD_REDUCE_SCATTER
105 0 : || param.opType == HcclCMDType::HCCL_CMD_ALLGATHER || param.opType == HcclCMDType::HCCL_CMD_BROADCAST) {
106 1 : return executor_->CalNumBlocks(
107 1 : numBlocks, userRankSize_, param.DataDes.count * SIZE_TABLE[param.DataDes.dataType], param.opType);
108 : } else {
109 0 : return executor_->CalNumBlocks(numBlocks, userRankSize_);
110 : }
111 : return HCCL_SUCCESS;
112 : }
113 :
114 0 : HcclResult CollAlgOperator::GetOpExpansionStr(const OpParam& param, const AlgDesc& algDesc, std::string& opExpansionStr)
115 : {
116 0 : if (algDesc.isAivMode) {
117 0 : opExpansionStr = "AIV";
118 0 : } else if (param.aicpuUnfoldMode) {
119 0 : opExpansionStr = "AI_CPU";
120 0 : } else if (static_cast<bool>(topoMatcher_->GetExternalInputHcclEnableFfts())) {
121 0 : opExpansionStr = "HOST";
122 : } else {
123 0 : opExpansionStr = "HOST_TS";
124 : }
125 0 : return HCCL_SUCCESS;
126 : }
127 :
128 77 : HcclResult CollAlgOperator::SelectAlg(
129 : const std::string& tag, const OpParam& param, const ResourceLimit& limit, std::string& algName, AlgDesc& algDesc,
130 : std::string& newTag)
131 : {
132 77 : bool isOnlyAiv = topoMatcher_->GetIsOnlyAivConfig();
133 75 : bool supportOnlyAiv
134 67 : = (param.opType == HcclCMDType::HCCL_CMD_ALLGATHER || param.opType == HcclCMDType::HCCL_CMD_REDUCE_SCATTER
135 33 : || param.opType == HcclCMDType::HCCL_CMD_ALLTOALLV || param.opType == HcclCMDType::HCCL_CMD_ALLTOALLVC
136 142 : || param.opType == HcclCMDType::HCCL_CMD_ALLTOALL || param.opType == HcclCMDType::HCCL_CMD_ALLREDUCE);
137 75 : CHK_PRT_RET(
138 : isOnlyAiv && !supportOnlyAiv,
139 : HCCL_ERROR(
140 : "[CollAlgOperator][SelectAlg] opType[%s] currently do not support aivonly",
141 : GetCMDTypeEnumStr(param.opType).c_str()),
142 : HCCL_E_NOT_SUPPORT);
143 75 : CHK_PRT_RET(
144 : isOnlyAiv && userRankSize_ == 1 && supportOnlyAiv,
145 : HCCL_ERROR("[CollAlgOperator][SelectAlg] aivonly not support, please ensure rankNum is greater than one"),
146 : HCCL_E_NOT_SUPPORT);
147 :
148 : // 兼容老接口
149 75 : if (limit.ifLimit) {
150 75 : CHK_RET(SelectAlg(tag, param, algName, newTag, limit));
151 : } else {
152 0 : CHK_RET(SelectAlg(tag, param, algName, newTag));
153 : }
154 :
155 : // 非AIV算法提前返回, 采用兜底Executor
156 78 : if (algName.empty()) {
157 0 : executor_ = CollAlgExecRegistry::Instance().GetAlgExec("SendExecutor", dispatcher_, topoMatcher_);
158 0 : CHK_PRT_RET(
159 : executor_.get() == nullptr,
160 : HCCL_ERROR("[CollAlgOperator][SelectAlg]Fail to find executor for algName[DefaultExecutor]"), HCCL_E_PARA);
161 : } else {
162 : // 校验控核
163 79 : if (limit.ifLimit && deviceType_ == DevType::DEV_TYPE_910_93 && topoMatcher_->GetAivModeConfig()) {
164 0 : CHK_RET(SelectAlgFor91093WithCoreLimit(param, limit, algName));
165 : }
166 :
167 : // 从对应executor获取算法描述
168 79 : if (executor_.get() == nullptr) {
169 75 : executor_ = CollAlgExecRegistry::Instance().GetAlgExec(algName, dispatcher_, topoMatcher_);
170 75 : CHK_PRT_RET(
171 : executor_.get() == nullptr,
172 : HCCL_ERROR("[CollAlgOperator][SelectAlg]Fail to find executor for algName[%s]", algName.c_str()),
173 : HCCL_E_PARA);
174 75 : CHK_RET(SetExecutorAttr(param));
175 : }
176 : }
177 :
178 75 : bool isLastSelect = algDesc.isLastSelect;
179 75 : algDesc = executor_->GetAlgDesc();
180 : // 打印维测日志
181 73 : if (UNLIKELY(GetDebugConfig() & HCCL_ALG) && isLastSelect) {
182 : // 获取展开模式，转换成字符串
183 0 : std::string opExpansionStr;
184 0 : CHK_RET(GetOpExpansionStr(param, algDesc, opExpansionStr));
185 : // 尝试获取确定性属性（如果Executor有声明自己是否为确定性）
186 0 : std::string appendStr = "";
187 0 : if (algDesc.deterministic >= 0) {
188 0 : appendStr += "deterministic[" + std::to_string(algDesc.deterministic) + "]";
189 : }
190 : // 打印关键维测内容
191 0 : bool isOpBase = GetWorkflowMode() == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE;
192 0 : HCCL_CONFIG_INFO(
193 : HCCL_ALG,
194 : "[%s] newTag[%s] algName[%s] userRank[%u] topoType[%d] algType[%s] "
195 : "userRankSize[%u] level0Size[%u] moduleNum_[%u] level2Size[%u] ",
196 : __func__, newTag.c_str(), algName.c_str(), userRank_, topoType_, AlgTypeToStr(algDesc.algType).c_str(),
197 : userRankSize_, deviceNumPerAggregation_, moduleNum_, superPodNum_);
198 0 : HCCL_CONFIG_INFO(
199 : HCCL_ALG,
200 : "[%s] newTag[%s] "
201 : "opExpansionMode[%s] isZeroCopy[%u] retryEnable[%u] isOpBase[%u] isCapture[%u] aivCoreLimit[%u] %s.",
202 : __func__, newTag.c_str(), opExpansionStr.c_str(), algDesc.isZeroCopy, retryEnable_, isOpBase,
203 : param.isCapture, limit.aivCoreLimit, appendStr.c_str());
204 0 : }
205 76 : return HCCL_SUCCESS;
206 : }
207 :
208 : HcclResult
209 0 : CollAlgOperator::SelectAlgFor91093WithCoreLimit(const OpParam& param, const ResourceLimit& limit, std::string& algName)
210 : {
211 0 : if (executor_.get() == nullptr) {
212 0 : executor_ = CollAlgExecRegistry::Instance().GetAlgExec(algName, dispatcher_, topoMatcher_);
213 0 : CHK_PRT_RET(
214 : executor_.get() == nullptr,
215 : HCCL_ERROR(
216 : "[CollAlgOperator][SelectAlgFor91093WithCoreLimit]Fail to find executor for algName[%s]",
217 : algName.c_str()),
218 : HCCL_E_PARA);
219 0 : CHK_RET(SetExecutorAttr(param));
220 : }
221 :
222 0 : CHK_RET(SetNumBlocks(limit.aivCoreLimit));
223 :
224 0 : std::string reSelName;
225 0 : switch (param.opType) {
226 0 : case HcclCMDType::HCCL_CMD_ALLREDUCE:
227 0 : reSelName = "AllReduceMeshAivFor91093Executor";
228 0 : break;
229 0 : case HcclCMDType::HCCL_CMD_ALLGATHER:
230 0 : reSelName = "AllGatherMeshAivFor91093Executor";
231 0 : break;
232 0 : case HcclCMDType::HCCL_CMD_REDUCE_SCATTER:
233 0 : reSelName = "ReduceScatterMeshAivFor91093Executor";
234 0 : break;
235 0 : case HcclCMDType::HCCL_CMD_ALLTOALLV:
236 : case HcclCMDType::HCCL_CMD_ALLTOALL:
237 : case HcclCMDType::HCCL_CMD_ALLTOALLVC:
238 0 : reSelName = "AlltoAllMeshAivFor91093Executor";
239 0 : break;
240 0 : default:
241 0 : break;
242 : }
243 :
244 : u32 numBlocks;
245 0 : HcclResult ret = CalNumBlocks(algName, param, numBlocks);
246 0 : if (ret != HCCL_SUCCESS) {
247 0 : CHK_PRT_RET(
248 : reSelName.empty() || reSelName == algName,
249 : HCCL_ERROR(
250 : "[CollAlgOperator][SelectAlgFor91093WithCoreLimit]Fail to check CalNumBlocks for algName[%s]",
251 : algName.c_str()),
252 : HCCL_E_PARA);
253 :
254 0 : algName = reSelName;
255 0 : executor_ = nullptr;
256 0 : HCCL_INFO("[CollAlgOperator][SelectAlgFor91093WithCoreLimit]Re select to algName[%s]", reSelName.c_str());
257 : }
258 :
259 0 : return HCCL_SUCCESS;
260 0 : }
261 :
262 : HcclResult
263 77 : CollAlgOperator::CalcResRequest(const std::string& algName, const OpParam& param, AlgResourceRequest& resourceRequest)
264 : {
265 77 : if (executor_.get() == nullptr) {
266 0 : executor_ = CollAlgExecRegistry::Instance().GetAlgExec(algName, dispatcher_, topoMatcher_);
267 0 : CHK_PRT_RET(
268 : executor_.get() == nullptr,
269 : HCCL_ERROR("[CollAlgOperator][CalcResRequest]Fail to find executor for algName[%s]", algName.c_str()),
270 : HCCL_E_PARA);
271 0 : CHK_RET(SetExecutorAttr(param));
272 : }
273 76 : return executor_->CalcResRequest(param, resourceRequest);
274 : }
275 :
276 24 : HcclResult CollAlgOperator::Orchestrate(const std::string& algName, OpParam& param, AlgResourceResponse& algResource)
277 : {
278 24 : HCCL_INFO("[CollAlgOperator][Orchestrate]algName[%s]", algName.c_str());
279 24 : if (executor_.get() == nullptr) {
280 0 : executor_ = CollAlgExecRegistry::Instance().GetAlgExec(algName, dispatcher_, topoMatcher_);
281 0 : CHK_PRT_RET(
282 : executor_.get() == nullptr,
283 : HCCL_ERROR("[CollAlgOperator][Orchestrate]Fail to find executor for algName[%s]", algName.c_str()),
284 : HCCL_E_PARA);
285 0 : CHK_RET(SetExecutorAttr(param));
286 : }
287 24 : executor_->SetAivClearEnable(aivClearEnable_);
288 24 : executor_->SetAlgOpContext(algOpContext_);
289 24 : executor_->SetOpCounter(opCounter_);
290 24 : return executor_->Orchestrate(param, algResource);
291 : }
292 :
293 0 : HcclResult CollAlgOperator::GetAdjInfo(
294 : const std::string& algName, OpParam& param, AlgResourceResponse& algResource, AdjInfo& nslbAdjInfo)
295 : {
296 0 : if (executor_.get() == nullptr) {
297 0 : executor_ = CollAlgExecRegistry::Instance().GetAlgExec(algName, dispatcher_, topoMatcher_);
298 0 : CHK_PRT_RET(
299 : executor_.get() == nullptr,
300 : HCCL_ERROR("[CollAlgOperator][Orchestrate]Fail to find executor for algName[%s]", algName.c_str()),
301 : HCCL_E_PARA);
302 0 : CHK_RET(SetExecutorAttr(param));
303 : }
304 :
305 0 : return executor_->GetAdjInfo(algResource, nslbAdjInfo);
306 : }
307 :
308 25 : HcclResult CollAlgOperator::PrepareCommInfoToDevice(const std::string& algName, AlgResourceResponse& algResource)
309 : {
310 25 : if (executor_.get() == nullptr) {
311 0 : executor_ = CollAlgExecRegistry::Instance().GetAlgExec(algName, dispatcher_, topoMatcher_);
312 0 : CHK_PRT_RET(
313 : executor_.get() == nullptr,
314 : HCCL_ERROR(
315 : "[CollAlgOperator][PrepareCommInfoToDevice]Fail to find executor for algName[%s]", algName.c_str()),
316 : HCCL_E_PARA);
317 : }
318 25 : return executor_->PrepareCommInfoToDevice(algResource);
319 : }
320 :
321 0 : HcclResult CollAlgOperator::CalcIncreLinkRequest(
322 : const std::string& algName, const OpParam& param, std::set<u32>& ranksHasLinked,
323 : AlgResourceRequest& resourceRequest, bool& needIncreLink)
324 : {
325 0 : if (executor_.get() == nullptr) {
326 0 : executor_ = CollAlgExecRegistry::Instance().GetAlgExec(algName, dispatcher_, topoMatcher_);
327 0 : CHK_PRT_RET(
328 : executor_.get() == nullptr,
329 : HCCL_ERROR("[CollAlgOperator][CalcIncreLinkRequest]Fail to find executor for algName[%s]", algName.c_str()),
330 : HCCL_E_PARA);
331 : }
332 0 : return executor_->CalcIncreLinkRequest(param, ranksHasLinked, resourceRequest, needIncreLink);
333 : }
334 :
335 83 : void CollAlgOperator::SetTopoAttr(AlgConfigurator* algConfigurator)
336 : {
337 83 : const HcclTopoAttr& topoAttr = algConfigurator->GetTopoAttr();
338 87 : serverNum_ = topoAttr.serverNum;
339 87 : moduleNum_ = topoAttr.moduleNum;
340 87 : superPodNum_ = topoAttr.superPodNum;
341 87 : deviceNumPerServer_ = topoAttr.deviceNumPerServer;
342 87 : deviceNumPerAggregation_ = topoAttr.deviceNumPerAggregation;
343 87 : multiModuleDiffDeviceNumMode_ = topoAttr.multiModuleDiffDeviceNumMode;
344 87 : multiSuperPodDiffServerNumMode_ = topoAttr.multiSuperPodDiffServerNumMode;
345 87 : multiSuperPodDiffDeviceNumMode_ = topoAttr.multiSuperPodDiffDeviceNumMode;
346 87 : isDiffDeviceType_ = topoAttr.isDiffDeviceType;
347 87 : gcdDeviceNumPerAggregation_ = topoAttr.gcdDeviceNumPerAggregation;
348 :
349 87 : meshAggregationRankSize_ = topoAttr.meshAggregationRankSize;
350 87 : isDiffDeviceModule_ = topoAttr.isDiffDeviceModule;
351 87 : isSingleMeshAggregation_ = topoAttr.isSingleMeshAggregation;
352 87 : isAllRankSamePlane_ = topoAttr.isAllRankSamePlane;
353 87 : is310PDuoCard_ = topoAttr.is310PDuoCard;
354 87 : isCommon310P3DUO_ = topoAttr.isCommon310P3DUO;
355 87 : hccsPortNum_ = topoAttr.hccsPortNum;
356 :
357 87 : userRank_ = topoAttr.userRank;
358 87 : realUserRank_ = topoAttr.realUserRank;
359 87 : userRankSize_ = topoAttr.userRankSize;
360 :
361 87 : devicePhyId_ = topoAttr.devicePhyId;
362 87 : deviceLogicId_ = topoAttr.deviceLogicId;
363 87 : deviceType_ = topoAttr.deviceType;
364 :
365 87 : nicList_ = topoAttr.nicList;
366 83 : pairLinkCounter_ = topoAttr.pairLinkCounter;
367 91 : isSupportRdmaLite_ = topoAttr.isSupportRdmaLite;
368 91 : isSupportHccsAndSio_ = topoAttr.isSupportHccsAndSio;
369 91 : useSuperPodMode_ = topoAttr.useSuperPodMode;
370 91 : isARSDoubleRing_ = topoAttr.isARSDoubleRing;
371 91 : return;
372 : }
373 :
374 90 : void CollAlgOperator::SetAlgoAttr(AlgConfigurator* algConfigurator)
375 : {
376 90 : const HcclAlgoAttr& algoAttr = algConfigurator->GetAlgoAttr();
377 91 : isHaveCpuRank_ = algoAttr.isHaveCpuRank;
378 91 : inlineReduceSwitchOn_ = algoAttr.inlineReduceSwitchOn;
379 91 : identifier_ = algoAttr.identifier;
380 94 : return;
381 : }
382 :
383 70 : HcclResult CollAlgOperator::SetExecutorAttr(const OpParam& param)
384 : {
385 70 : CHK_RET(executor_->SetAlgType(algType_));
386 70 : CHK_RET(executor_->SetCCLInBuffer(cclBufferManager_.GetInCCLbufferSize()));
387 :
388 69 : if (param.opType == HcclCMDType::HCCL_CMD_REDUCE_SCATTER) {
389 33 : bool isSupportSDMAReduce = false;
390 33 : if (GetWorkflowMode() != HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE) {
391 : isSupportSDMAReduce
392 12 : = IsSupportSDMAReduce(param.inputPtr, param.outputPtr, param.DataDes.dataType, param.reduceType);
393 : } else {
394 19 : isSupportSDMAReduce = IsSupportSDMAReduce(
395 19 : cclBufferManager_.GetInCCLbuffer().ptr(), cclBufferManager_.GetOutCCLbuffer().ptr(),
396 21 : param.DataDes.dataType, param.reduceType);
397 : }
398 31 : CHK_RET(executor_->SetIsSupportSDMAReduce(isSupportSDMAReduce));
399 : }
400 69 : return HCCL_SUCCESS;
401 : }
402 :
403 0 : std::string CollAlgOperator::GenerateNewTagByAlgTypeLevel1(std::string tag, std::string algTypeLevel1Tag) const
404 : {
405 0 : if (algTypeLevel1Tag == "") {
406 0 : return tag;
407 : } else {
408 0 : return tag + "_" + algTypeLevel1Tag;
409 : }
410 : }
411 :
412 0 : HcclResult CollAlgOperator::AppendTag(const AlgTypeLevel1& algTypeLevel1, std::string& tag)
413 : {
414 0 : switch (algTypeLevel1) {
415 0 : case AlgTypeLevel1::ALG_LEVEL1_RING:
416 0 : tag = "ALG_LEVEL1_RING";
417 0 : break;
418 0 : case AlgTypeLevel1::ALG_LEVEL1_HD:
419 0 : tag = "ALG_LEVEL1_HD";
420 0 : break;
421 0 : case AlgTypeLevel1::ALG_LEVEL1_NHR:
422 0 : tag = "ALG_LEVEL1_NHR";
423 0 : break;
424 0 : case AlgTypeLevel1::ALG_LEVEL1_PIPELINE:
425 0 : tag = "ALG_LEVEL1_PIPELINE";
426 0 : break;
427 0 : default:
428 0 : HCCL_WARNING("[CollAlgOperator][AppendTag] The algTypeLevel1 %d is not supported.", algTypeLevel1);
429 0 : break;
430 : }
431 0 : return HCCL_SUCCESS;
432 : }
433 :
434 15 : HcclResult CollAlgOperator::AutoSelectAlgTypeLevel1(
435 : HcclCMDType hcclCMDType, u64 countSize, u64 cclBufferSize, std::string& algTypeLevel1Tag, bool isInlineReduce,
436 : bool isRdmaReduce, bool isAivMode)
437 : {
438 15 : if (isSingleMeshAggregation_) {
439 18 : HCCL_INFO(
440 : "[AutoSelectAlgTypeLevel1] there are %u server(%u module) in level1, no need to choose algo.", serverNum_,
441 : moduleNum_);
442 20 : return HCCL_SUCCESS;
443 : }
444 :
445 : // auto algo selection process
446 0 : if (isAlgoLevel1Default_) {
447 : // parse algType_ and get algTypeLevel1 and algTypeLevel0
448 0 : auto originalAlgTypeLevel0 = algType_.algoLevel0;
449 : // set algTypeLevel1
450 : AlgTypeLevel1 algTypeLevel1;
451 0 : CHK_RET(GetDefaultAlgoLevel1V2(
452 : hcclCMDType, countSize, cclBufferSize, algTypeLevel1, isInlineReduce, isRdmaReduce, isAivMode));
453 0 : auto iter = HCCL_ALGO_LEVEL1_NAME_MAP.find(algTypeLevel1);
454 0 : CHK_PRT_RET(
455 : iter == HCCL_ALGO_LEVEL1_NAME_MAP.end(),
456 : HCCL_ERROR("[AutoSelectAlgTypeLevel1] level1: algType[%u] is invalid.", algTypeLevel1), HCCL_E_INTERNAL);
457 0 : HCCL_INFO(
458 : "[AutoSelectAlgTypeLevel1] there are %u server(%u module) in level1, using %s algo", serverNum_, moduleNum_,
459 : iter->second.c_str());
460 0 : algType_.algoLevel0 = originalAlgTypeLevel0;
461 0 : algType_.algoLevel1 = algTypeLevel1;
462 : // tag 增加所选的算法
463 0 : AppendTag(algTypeLevel1, algTypeLevel1Tag);
464 : }
465 0 : return HCCL_SUCCESS;
466 : }
467 :
468 0 : HcclResult CollAlgOperator::SelectAlgoForComm(
469 : HcclCMDType hcclCMDType, float delay, u64 curSize, float bandWidth, AlgTypeLevel1& algType)
470 : {
471 : // 从map中查找对应的计算函数
472 0 : auto it = selectFuncMap_.find(hcclCMDType);
473 0 : if (it == selectFuncMap_.end()) {
474 0 : HCCL_ERROR("[Get][AlgTypeLevel1] The hcclCMDType %d is not supported.", hcclCMDType);
475 0 : return HCCL_E_NOT_SUPPORT;
476 : }
477 0 : return (it->second)(delay, curSize, bandWidth, algType);
478 : }
479 :
480 : // 保守估计Pipeline算法所需context数量
481 0 : u32 CollAlgOperator::CalcContextNumForPipeline(HcclCMDType hcclCMDType)
482 : {
483 : bool isDeterPipeline
484 0 : = topoMatcher_->GetDeterministicConfig() == DETERMINISTIC_ENABLE
485 0 : && (hcclCMDType == HcclCMDType::HCCL_CMD_ALLREDUCE || hcclCMDType == HcclCMDType::HCCL_CMD_REDUCE_SCATTER);
486 0 : const u32 stepNum = moduleNum_; // 通信步数
487 0 : const u32 hccsContextNumPerStep = 5 * (deviceNumPerAggregation_ - 1); // SDMA跨片每步所需context数
488 0 : const u32 roceContextNumPerStep = 7; // RDMA每步所需context数
489 0 : const u32 copyContextNumPerStep = 1; // SDMA片内每步所需context数
490 0 : const u32 localReduceNumPerStep = isDeterPipeline ? (deviceNumPerAggregation_ - 1) : 0;
491 0 : const u32 contextNumPerStep
492 0 : = hccsContextNumPerStep + roceContextNumPerStep + copyContextNumPerStep + localReduceNumPerStep; // 小计
493 0 : const u32 barrierContextNum = 4; // 通信结束时barrier操作所需context数
494 :
495 0 : switch (hcclCMDType) {
496 0 : case HcclCMDType::HCCL_CMD_ALLREDUCE: // fall-through
497 : case HcclCMDType::HCCL_CMD_REDUCE_SCATTER: // fall-through
498 : case HcclCMDType::HCCL_CMD_ALLGATHER:
499 : case HcclCMDType::HCCL_CMD_ALLGATHER_V: {
500 0 : const u32 copyContextNum = 1; // 通信首尾所需context数量
501 0 : u32 contextNum = stepNum * contextNumPerStep + barrierContextNum + copyContextNum;
502 0 : if (hcclCMDType == HcclCMDType::HCCL_CMD_ALLREDUCE) {
503 0 : contextNum += contextNum;
504 : }
505 0 : if (isDeterPipeline) {
506 0 : contextNum += stepNum - 1; // 最后的local reduce
507 : }
508 0 : return contextNum;
509 : }
510 0 : case HcclCMDType::HCCL_CMD_ALLTOALLV: // fall-through
511 : case HcclCMDType::HCCL_CMD_ALLTOALLVC: // fall-through
512 : case HcclCMDType::HCCL_CMD_ALLTOALL: {
513 0 : const u32 copyContextNum = 1 + moduleNum_; // 通信首尾所需context数量
514 0 : return stepNum * contextNumPerStep + barrierContextNum + copyContextNum;
515 : }
516 0 : default:
517 0 : return 0;
518 : }
519 : }
520 :
521 0 : HcclResult CollAlgOperator::GetDefaultAlgoLevel1V2(
522 : HcclCMDType hcclCMDType, u64 curSize, u64 cclBufferSize, AlgTypeLevel1& algType, bool isInlineReduce,
523 : bool isRdmaReduce, bool isAivMode)
524 : {
525 : // pipeline mode is deployed,where there is multi-sever multi-device(insever) now,
526 : // since RDMA is not reduced by normal serial orchestration of tasks.
527 : // So pipeline mode is more dominant than normal serial orchestration now.
528 0 : auto originalAlgTypeLevel0 = algType_.algoLevel0;
529 : bool disdeterniminsticWithInlineReduce
530 0 : = isInlineReduce && isRdmaReduce && topoMatcher_->GetDeterministicConfig() == DETERMINISTIC_DISABLE;
531 : bool deterniminsticWithInlineReduce
532 0 : = isInlineReduce && isRdmaReduce && topoMatcher_->GetDeterministicConfig() == DETERMINISTIC_ENABLE;
533 :
534 : // 对于不支持Rdma Lite的场景，下发性能较差，RS和AG需要一个很大的数据量（AR的一半）才能掩盖下发时间
535 0 : u64 pipelineMinSize = (isSupportRdmaLite_) ? (PIPELINE_MIN_SIZE) : (PIPELINE_MIN_SIZE_NO_LITE);
536 0 : if (((hcclCMDType == HcclCMDType::HCCL_CMD_REDUCE_SCATTER && disdeterniminsticWithInlineReduce)
537 0 : || hcclCMDType == HcclCMDType::HCCL_CMD_ALLGATHER || hcclCMDType == HcclCMDType::HCCL_CMD_ALLGATHER_V)
538 0 : && deviceNumPerAggregation_ != 1 && curSize >= pipelineMinSize && IsAlgTypeLevel0Mesh(originalAlgTypeLevel0)
539 0 : && CalcContextNumForPipeline(hcclCMDType) <= HCCL_FFTS_CAPACITY) {
540 0 : algType = AlgTypeLevel1::ALG_LEVEL1_PIPELINE;
541 0 : return HCCL_SUCCESS;
542 : }
543 0 : if (hcclCMDType == HcclCMDType::HCCL_CMD_REDUCE_SCATTER && deterniminsticWithInlineReduce
544 0 : && deviceNumPerAggregation_ > 1 && curSize >= pipelineMinSize && IsAlgTypeLevel0Mesh(originalAlgTypeLevel0)
545 0 : && CalcContextNumForPipeline(hcclCMDType) <= HCCL_FFTS_CAPACITY && moduleNum_ > 1
546 0 : && curSize >= HCCL_SMALL_COUNT_256_KB) {
547 0 : algType = AlgTypeLevel1::ALG_LEVEL1_PIPELINE;
548 0 : return HCCL_SUCCESS;
549 : }
550 :
551 : // 对于不支持Rdma Lite的场景，下发性能较差，AllReduce需要一个较大的数据量才能掩盖下发时间
552 0 : pipelineMinSize = (isSupportRdmaLite_) ? (PIPELINE_ALLREDUCE_MIN_SIZE) : (PIPELINE_MIN_SIZE_NO_LITE);
553 0 : if (hcclCMDType == HcclCMDType::HCCL_CMD_ALLREDUCE) {
554 : // 计算每个slice的大小
555 0 : u64 allreduceCurSize = 0;
556 0 : allreduceCurSize = curSize / (moduleNum_ * deviceNumPerAggregation_);
557 0 : if (disdeterniminsticWithInlineReduce && deviceNumPerAggregation_ != 1 && allreduceCurSize >= pipelineMinSize
558 0 : && !isAivMode && IsAlgTypeLevel0Mesh(originalAlgTypeLevel0)
559 0 : && CalcContextNumForPipeline(hcclCMDType) <= HCCL_FFTS_CAPACITY) {
560 0 : algType = AlgTypeLevel1::ALG_LEVEL1_PIPELINE;
561 0 : return HCCL_SUCCESS;
562 : }
563 0 : if (deterniminsticWithInlineReduce && deviceNumPerAggregation_ > 1 && allreduceCurSize >= HCCL_SMALL_COUNT_1_MB
564 0 : && !isAivMode && IsAlgTypeLevel0Mesh(originalAlgTypeLevel0)
565 0 : && CalcContextNumForPipeline(hcclCMDType) <= HCCL_FFTS_CAPACITY) {
566 0 : algType = AlgTypeLevel1::ALG_LEVEL1_PIPELINE;
567 0 : return HCCL_SUCCESS;
568 : }
569 : }
570 0 : u64 dataSizePerLoop = curSize > cclBufferSize ? cclBufferSize : curSize;
571 0 : float delay = LATENCY; // 静态时延 60 us;
572 : float bandWidth;
573 0 : CHK_RET(GetBandWidthPerNPU(1, userRankSize_, deviceNumPerAggregation_, bandWidth)); // 单位：GB/s
574 0 : bandWidth = bandWidth * GB2B; // 单位：B/s
575 0 : CHK_RET(SelectAlgoForComm(hcclCMDType, delay, dataSizePerLoop, bandWidth, algType));
576 0 : return HCCL_SUCCESS;
577 : }
578 :
579 : HcclResult
580 0 : CollAlgOperator::SelectAlgoTypeForReduceScatter(float delay, u64 recvCurSize, float bandWidth, AlgTypeLevel1& algType)
581 : {
582 0 : auto steps = moduleNum_ - 1;
583 : // theoretical time cost of Ring
584 0 : double ringCost
585 0 : = static_cast<double>(steps) * delay
586 0 : + static_cast<double>(steps) / moduleNum_ * recvCurSize * userRankSize_ / bandWidth * SECOND2MICROSECOND;
587 :
588 : // theoretical time cost of NHR
589 0 : double nhrCost = ceil(log2(moduleNum_)) * delay
590 0 : + static_cast<double>(moduleNum_ - 1) / moduleNum_ * recvCurSize * userRankSize_ / bandWidth
591 0 : * SECOND2MICROSECOND;
592 :
593 : // compare costs between NHR and Ring, if same cost, Ring > NHR > HD
594 0 : algType = (nhrCost < ringCost) ? AlgTypeLevel1::ALG_LEVEL1_NHR : AlgTypeLevel1::ALG_LEVEL1_RING;
595 0 : double interMinCost = min(nhrCost, ringCost);
596 :
597 : // theoretical time cost of HD/RHD
598 0 : double hdCost = 0.0;
599 0 : if ((moduleNum_ & (moduleNum_ - 1)) == 0) {
600 : // theoretical time cost of HD
601 : hdCost
602 0 : = log2(moduleNum_) * delay
603 0 : + static_cast<double>(steps) / moduleNum_ * recvCurSize * userRankSize_ / bandWidth * SECOND2MICROSECOND;
604 : } else {
605 : // theoretical time cost of Recursive HD,
606 : // the (RHD_FACTOR_TWO * moduleNum_ - RHD_FACTOR_ONE) / moduleNum_ is equal to 1 + (moduleNum_ -1) / moduleNum_
607 0 : hdCost = ceil(log2(moduleNum_)) * delay
608 0 : + static_cast<double>(RHD_FACTOR_TWO * moduleNum_ - RHD_FACTOR_ONE) / moduleNum_ * recvCurSize
609 0 : * userRankSize_ / bandWidth * SECOND2MICROSECOND;
610 : }
611 :
612 : // compare cost among NHR, HD and Ring
613 0 : algType = (hdCost < interMinCost) ? AlgTypeLevel1::ALG_LEVEL1_HD : algType;
614 0 : return HCCL_SUCCESS;
615 : }
616 :
617 : HcclResult
618 0 : CollAlgOperator::SelectAlgoTypeForAllGather(float delay, u64 sendCurSize, float bandWidth, AlgTypeLevel1& algType)
619 : {
620 0 : auto steps = moduleNum_ - 1;
621 : // theoretical time cost of Ring
622 0 : double ringCost
623 0 : = static_cast<double>(steps) * delay
624 0 : + static_cast<double>(steps) / moduleNum_ * sendCurSize * userRankSize_ / bandWidth * SECOND2MICROSECOND;
625 :
626 : // theoretical time cost of NHR
627 0 : double nhrCost = ceil(log2(moduleNum_)) * delay
628 0 : + static_cast<double>(moduleNum_ - 1) / moduleNum_ * sendCurSize * userRankSize_ / bandWidth
629 0 : * SECOND2MICROSECOND;
630 :
631 : // compare costs between NHR and Ring, if same cost, Ring > NHR > HD
632 0 : algType = (nhrCost < ringCost) ? AlgTypeLevel1::ALG_LEVEL1_NHR : AlgTypeLevel1::ALG_LEVEL1_RING;
633 0 : double interMinCost = min(nhrCost, ringCost);
634 :
635 : // theoretical time cost of HD/RHD
636 0 : double hdCost = 0.0;
637 0 : if ((moduleNum_ & (moduleNum_ - 1)) == 0) {
638 : // theoretical time cost of HD
639 : hdCost
640 0 : = log2(moduleNum_) * delay
641 0 : + static_cast<double>(steps) / moduleNum_ * sendCurSize * userRankSize_ / bandWidth * SECOND2MICROSECOND;
642 : } else {
643 : // theoretical time cost of Recursive HD
644 : // the (RHD_FACTOR_TWO * moduleNum_ - RHD_FACTOR_ONE) / moduleNum_ is equal to 1 + (moduleNum_ -1) / moduleNum_
645 0 : hdCost = ceil(log2(moduleNum_)) * delay
646 0 : + static_cast<double>(RHD_FACTOR_TWO * moduleNum_ - RHD_FACTOR_ONE) / moduleNum_ * sendCurSize
647 0 : * userRankSize_ / bandWidth * SECOND2MICROSECOND;
648 : }
649 :
650 : // compare cost among NHR, HD and Ring
651 0 : algType = (hdCost < interMinCost) ? AlgTypeLevel1::ALG_LEVEL1_HD : algType;
652 0 : return HCCL_SUCCESS;
653 : }
654 :
655 : HcclResult
656 0 : CollAlgOperator::SelectAlgoTypeForAllGatherV(float delay, u64 sendCurSize, float bandWidth, AlgTypeLevel1& algType)
657 : {
658 0 : auto steps = moduleNum_ - 1;
659 : // theoretical time cost of Ring
660 0 : double ringCost
661 0 : = static_cast<double>(steps) * delay
662 0 : + static_cast<double>(steps) / moduleNum_ * sendCurSize * userRankSize_ / bandWidth * SECOND2MICROSECOND;
663 :
664 0 : HCCL_DEBUG("[%s] CollAlgOperator for SelectAlgoTypeForAllGatherV", __func__);
665 : // theoretical time cost of NHR
666 0 : double nhrCost = ceil(log2(moduleNum_)) * delay
667 0 : + static_cast<double>(moduleNum_ - 1) / moduleNum_ * sendCurSize * userRankSize_ / bandWidth
668 0 : * SECOND2MICROSECOND;
669 :
670 : // compare costs between NHR and Ring, if same cost, Ring > NHR > HD
671 0 : algType = (nhrCost < ringCost) ? AlgTypeLevel1::ALG_LEVEL1_NHR : AlgTypeLevel1::ALG_LEVEL1_RING;
672 :
673 0 : return HCCL_SUCCESS;
674 : }
675 :
676 : HcclResult
677 0 : CollAlgOperator::SelectAlgoTypeForGather(float delay, u64 sendCurSize, float bandWidth, AlgTypeLevel1& algType)
678 : {
679 0 : auto steps = moduleNum_ - 1;
680 : // theoretical time cost of Ring
681 0 : double ringCost
682 0 : = static_cast<double>(steps) * delay
683 0 : + static_cast<double>(steps) / moduleNum_ * sendCurSize * userRankSize_ / bandWidth * SECOND2MICROSECOND;
684 0 : double hdCost = 0.0;
685 0 : if ((moduleNum_ & (moduleNum_ - 1)) == 0) {
686 : // theoretical time cost of HD
687 : hdCost
688 0 : = log2(moduleNum_) * delay
689 0 : + static_cast<double>(steps) / moduleNum_ * sendCurSize * userRankSize_ / bandWidth * SECOND2MICROSECOND;
690 : } else {
691 : // theoretical time cost of Recursive HD
692 : // the (RHD_FACTOR_TWO * moduleNum_ - RHD_FACTOR_ONE) / moduleNum_ is equal to 1 + (moduleNum_ -1) / moduleNum_
693 0 : hdCost = ceil(log2(moduleNum_)) * delay
694 0 : + static_cast<double>(RHD_FACTOR_TWO * moduleNum_ - RHD_FACTOR_ONE) / moduleNum_ * sendCurSize
695 0 : * userRankSize_ / bandWidth * SECOND2MICROSECOND;
696 : }
697 0 : algType = (hdCost < ringCost) ? AlgTypeLevel1::ALG_LEVEL1_HD : AlgTypeLevel1::ALG_LEVEL1_RING;
698 0 : return HCCL_SUCCESS;
699 : }
700 :
701 : HcclResult
702 0 : CollAlgOperator::SelectAlgoTypeForAllReduce(float delay, u64 curSize, float bandWidth, AlgTypeLevel1& algType)
703 : {
704 0 : auto steps = moduleNum_ - 1;
705 : // theoretical time cost of Ring
706 0 : double ringCost = DOUBLE_SUB_HCCLCMD * static_cast<double>(steps) * delay
707 0 : + DOUBLE_SUB_HCCLCMD * static_cast<double>(steps) / moduleNum_ * curSize
708 0 : / deviceNumPerAggregation_ / bandWidth * SECOND2MICROSECOND;
709 :
710 : // theoretical time cost of NHR
711 0 : double nhrCost = NHR_FACTOR_TWO * ceil(log2(moduleNum_)) * delay
712 0 : + NHR_FACTOR_TWO * static_cast<double>(moduleNum_ - 1) / moduleNum_ * curSize
713 0 : / deviceNumPerAggregation_ / bandWidth * SECOND2MICROSECOND;
714 :
715 : // compare costs between NHR and Ring, if same cost, Ring > NHR > HD
716 0 : algType = (nhrCost < ringCost) ? AlgTypeLevel1::ALG_LEVEL1_NHR : AlgTypeLevel1::ALG_LEVEL1_RING;
717 0 : double interMinCost = min(nhrCost, ringCost);
718 :
719 : // theoretical time cost of HD/RHD
720 0 : double hdCost = 0.0;
721 0 : if ((moduleNum_ & (moduleNum_ - 1)) == 0) {
722 : // theoretical time cost of HD
723 0 : hdCost = DOUBLE_SUB_HCCLCMD * log2(moduleNum_) * delay
724 0 : + DOUBLE_SUB_HCCLCMD * static_cast<double>(steps) / moduleNum_ * curSize / deviceNumPerAggregation_
725 0 : / bandWidth * SECOND2MICROSECOND;
726 : } else {
727 : // theoretical time cost of Recursive HD
728 : // the (RHD_FACTOR_TWO * moduleNum_ - RHD_FACTOR_ONE) / moduleNum_ is equal to 1 + (moduleNum_ -1) / moduleNum_
729 0 : hdCost = DOUBLE_SUB_HCCLCMD * ceil(log2(moduleNum_)) * delay
730 0 : + DOUBLE_SUB_HCCLCMD * static_cast<double>(RHD_FACTOR_TWO * moduleNum_ - RHD_FACTOR_ONE) / moduleNum_
731 0 : * curSize / deviceNumPerAggregation_ / bandWidth * SECOND2MICROSECOND;
732 : }
733 :
734 : // compare cost among NHR, HD and Ring
735 0 : algType = (hdCost < interMinCost) ? AlgTypeLevel1::ALG_LEVEL1_HD : algType;
736 0 : return HCCL_SUCCESS;
737 : }
738 :
739 : HcclResult
740 0 : CollAlgOperator::SelectAlgoTypeForBroadcast(float delay, u64 curSize, float bandWidth, AlgTypeLevel1& algType)
741 : {
742 0 : auto steps = moduleNum_ - 1;
743 : // theoretical time cost of Ring
744 0 : double ringCost = DOUBLE_SUB_HCCLCMD * static_cast<double>(steps) * delay
745 0 : + DOUBLE_SUB_HCCLCMD * static_cast<double>(steps) / moduleNum_ * curSize
746 0 : / deviceNumPerAggregation_ / bandWidth * SECOND2MICROSECOND;
747 0 : double hdCost = 0.0;
748 0 : if ((moduleNum_ & (moduleNum_ - 1)) == 0) {
749 : // theoretical time cost of HD
750 0 : hdCost = DOUBLE_SUB_HCCLCMD * log2(moduleNum_) * delay
751 0 : + DOUBLE_SUB_HCCLCMD * static_cast<double>(steps) / moduleNum_ * curSize / deviceNumPerAggregation_
752 0 : / bandWidth * SECOND2MICROSECOND;
753 : } else {
754 : // theoretical time cost of Recursive HD
755 : // rhd-broadcast = scatter + allgather + copy
756 0 : hdCost = (COPY_TIME_IN_RHD + DOUBLE_SUB_HCCLCMD * floor(log2(moduleNum_))) * delay
757 0 : + (COPY_TIME_IN_RHD + DOUBLE_SUB_HCCLCMD * static_cast<double>(steps) / moduleNum_) * curSize
758 0 : / deviceNumPerAggregation_ / bandWidth * SECOND2MICROSECOND;
759 : }
760 0 : algType = (hdCost < ringCost) ? AlgTypeLevel1::ALG_LEVEL1_HD : AlgTypeLevel1::ALG_LEVEL1_RING;
761 0 : return HCCL_SUCCESS;
762 : }
763 :
764 0 : HcclResult CollAlgOperator::SelectAlgoTypeForReduce(float delay, u64 curSize, float bandWidth, AlgTypeLevel1& algType)
765 : {
766 0 : HCCL_DEBUG("[CollAlgOperator]SelectAlgoTypeForReduce start");
767 0 : auto steps = moduleNum_ - 1;
768 : // theoretical time cost of Ring
769 0 : double ringCost = DOUBLE_SUB_HCCLCMD * static_cast<double>(steps) * delay
770 0 : + DOUBLE_SUB_HCCLCMD * static_cast<double>(steps) / moduleNum_ * curSize
771 0 : / deviceNumPerAggregation_ / bandWidth * SECOND2MICROSECOND;
772 0 : double hdCost = 0.0;
773 0 : if ((moduleNum_ & (moduleNum_ - 1)) == 0) {
774 : // theoretical time cost of HD
775 0 : hdCost = DOUBLE_SUB_HCCLCMD * log2(moduleNum_) * delay
776 0 : + DOUBLE_SUB_HCCLCMD * static_cast<double>(steps) / moduleNum_ * curSize / deviceNumPerAggregation_
777 0 : / bandWidth * SECOND2MICROSECOND;
778 : } else {
779 : // theoretical time cost of Recursive HD
780 : // rhd-broadcast = reducescatter + gather + copy
781 0 : hdCost = (COPY_TIME_IN_RHD + DOUBLE_SUB_HCCLCMD * floor(log2(moduleNum_))) * delay
782 0 : + (COPY_TIME_IN_RHD + DOUBLE_SUB_HCCLCMD * static_cast<double>(steps) / moduleNum_) * curSize
783 0 : / deviceNumPerAggregation_ / bandWidth * SECOND2MICROSECOND;
784 : }
785 0 : algType = (hdCost < ringCost) ? AlgTypeLevel1::ALG_LEVEL1_HD : AlgTypeLevel1::ALG_LEVEL1_RING;
786 0 : return HCCL_SUCCESS;
787 : }
788 :
789 24 : AlgType CollAlgOperator::GetAlgType() { return algType_; }
790 :
791 2 : bool CollAlgOperator::Is2U2PInfer()
792 : {
793 : return (
794 0 : (deviceNumPerAggregation_ == HCCL_DEVICE_NUM_TWO) && (serverNum_ == 1)
795 0 : && (deviceType_ == DevType::DEV_TYPE_910B) && (meshAggregationRankSize_ == HCCL_DEVICE_NUM_TWO)
796 2 : && (pairLinkCounter_[static_cast<u32>(LinkTypeInServer::HCCS_TYPE)] == 0));
797 : }
798 :
799 33 : bool CollAlgOperator::Is910BSingleMesh()
800 : {
801 1 : bool isMeshTopo = topoType_ == TopoType::TOPO_TYPE_NP_MESH || topoType_ == TopoType::TOPO_TYPE_4P_MESH
802 34 : || topoType_ == TopoType::TOPO_TYPE_2P_MESH || topoType_ == TopoType::TOPO_TYPE_1P_MESH;
803 :
804 : bool isSingleMesh
805 33 : = (deviceType_ == DevType::DEV_TYPE_910B) && (isMeshTopo || Is2U2PInfer()) && (userRankSize_ != 1);
806 33 : return isSingleMesh;
807 : }
808 :
809 1 : bool CollAlgOperator::NeedCreateSingleMeshPlane(const bool isInlineReduce)
810 : {
811 : // 910B 图模式非确定计算，inlineReduce使能，MESH拓扑场景下，创建一个mesh平面
812 1 : bool meshSinglePlane = Is910BSingleMesh() && topoMatcher_->GetDeterministicConfig() == DETERMINISTIC_DISABLE
813 1 : && isInlineReduce && (GetWorkflowMode() != HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE);
814 :
815 1 : return meshSinglePlane;
816 : }
817 :
818 32 : bool CollAlgOperator::SingleMeshInlineReduce(void* inputPtr, void* outputPtr, HcclDataType dataType, HcclReduceOp op)
819 : {
820 32 : bool isInlineReduce = IsSupportSDMAReduce(inputPtr, outputPtr, dataType, op);
821 30 : bool singleMeshInlineReduce = Is910BSingleMesh() && isInlineReduce && isSingleMeshAggregation_;
822 32 : return singleMeshInlineReduce;
823 : }
824 :
825 4 : bool CollAlgOperator::IsMultiMeshInlineReduce(void* inputPtr, void* outputPtr, HcclDataType dataType, HcclReduceOp op)
826 : {
827 0 : bool isMeshTopo = topoType_ == TopoType::TOPO_TYPE_NP_MESH || topoType_ == TopoType::TOPO_TYPE_4P_MESH
828 4 : || topoType_ == TopoType::TOPO_TYPE_2P_MESH || topoType_ == TopoType::TOPO_TYPE_1P_MESH;
829 :
830 4 : bool isInlineReduce = IsSupportSDMAReduce(inputPtr, outputPtr, dataType, op);
831 1 : bool isRdmaReduce = IsSupportRDMAReduce(dataType, op);
832 1 : bool multiMeshInlineReduce = (deviceType_ == DevType::DEV_TYPE_910B) && isMeshTopo && isInlineReduce && isRdmaReduce
833 2 : && (!isSingleMeshAggregation_);
834 1 : return multiMeshInlineReduce;
835 : }
836 :
837 0 : void CollAlgOperator::SetLegacyHcclImpl(std::unique_ptr<hcclImpl>& impl)
838 : {
839 0 : hcclImpl_ = impl.get();
840 0 : return;
841 : }
842 :
843 7 : HcclResult CollAlgOperator::SetRetryEnable(bool retryEnable)
844 : {
845 7 : retryEnable_ = retryEnable;
846 7 : return HCCL_SUCCESS;
847 : }
848 :
849 7 : HcclResult CollAlgOperator::SetAivClearEnable(bool aivClearEnable)
850 : {
851 7 : aivClearEnable_ = aivClearEnable;
852 7 : return HCCL_SUCCESS;
853 : }
854 :
855 0 : HcclResult CollAlgOperator::SetAlgOpContext(AlgOpContext algOpContext)
856 : {
857 0 : algOpContext_ = algOpContext;
858 0 : return HCCL_SUCCESS;
859 : }
860 :
861 29 : bool CollAlgOperator::SupportRetryWithInplaceCheck(
862 : const HcclCMDType& opType, OpParam& param, std::string& algName, u8& isInplaceStatus,
863 : InplaceSupportRetryStatus& inPlaceSupportRetryStatus)
864 : {
865 : // 不支持inplace的通信算子重执行
866 29 : if (IsHcclOpInplace(opType, param, userRank_, userRankSize_, isInplaceStatus)) {
867 13 : void* commInputPtr = nullptr;
868 13 : u64 commInputSize = 0;
869 21 : CHK_RET(cclBufferManager_.GetInCCLbuffer(commInputPtr, commInputSize));
870 13 : if (!FitRetryConditionforInPlaceOp(
871 13 : opType, param, algName, commInputSize, userRankSize_, retryEnable_, inPlaceSupportRetryStatus)) {
872 8 : HCCL_DEBUG(
873 : "[CollAlgOperator][OpRetry][AICPU]hccl aicpu can not retry, opType[%s], inputPtr[%p], "
874 : "outputPtr[%p].",
875 : GetCMDTypeEnumStr(opType).c_str(), param.inputPtr, param.outputPtr);
876 8 : return false;
877 : }
878 : }
879 : // true 存在两种情况：
880 : // 1. 非inplace场景
881 : // 2. 是inplace但同时符合retry条件的场景
882 21 : return true;
883 : }
884 :
885 21 : HcclResult CollAlgOperator::GetNumBlocks(u32& numBlocks)
886 : {
887 21 : CHK_SMART_PTR_NULL(executor_);
888 21 : return executor_->GetNumBlocks(numBlocks);
889 : }
890 :
891 5 : HcclResult CollAlgOperator::SetNumBlocks(const u32& numBlocks)
892 : {
893 5 : CHK_SMART_PTR_NULL(executor_);
894 5 : return executor_->SetNumBlocks(numBlocks);
895 : }
896 :
897 2 : HcclResult CollAlgOperator::GetCache(HcclCacheInfo& cacheInfo)
898 : {
899 2 : CHK_SMART_PTR_NULL(executor_);
900 2 : return executor_->GetCache(cacheInfo);
901 : }
902 :
903 24 : HcclResult CollAlgOperator::SetOpCounter(const OpCounterInfo& opCounter)
904 : {
905 24 : opCounter_ = opCounter;
906 24 : return HCCL_SUCCESS;
907 : }
908 :
909 0 : HcclResult CollAlgOperator::SetRmaInfo(void* rmaInfo)
910 : {
911 0 : CHK_SMART_PTR_NULL(executor_);
912 0 : CHK_PTR_NULL(rmaInfo);
913 0 : return executor_->SetRmaInfo(rmaInfo);
914 : }
915 :
916 9 : HcclResult CollAlgOperator::SelectAlgforAHC(u64 dataSize, AHCOpType ahcOpType)
917 : {
918 9 : if (multiModuleDiffDeviceNumMode_) {
919 0 : return HCCL_SUCCESS;
920 : }
921 :
922 9 : bool isAHCWholeConfig
923 9 : = (algType_.algoLevel0 == AlgTypeLevel0::ALG_LEVEL0_RESERVED
924 9 : && (algType_.algoLevel1 == AlgTypeLevel1::ALG_LEVEL1_AHC
925 0 : || algType_.algoLevel1 == AlgTypeLevel1::ALG_LEVEL1_AHC_BROKE));
926 :
927 9 : CommPlane ahcSubGroupLevel = COMM_LEVEL1_AHC;
928 9 : if (isAHCWholeConfig) {
929 0 : if (deviceType_ != DevType::DEV_TYPE_910_93) {
930 0 : ahcSubGroupLevel = COMM_COMBINE;
931 : } else {
932 0 : ahcSubGroupLevel = COMM_COMBINE_ORDER;
933 : }
934 9 : } else if (deviceType_ != DevType::DEV_TYPE_910_93) {
935 0 : HCCL_DEBUG("[AHCAlgSelect] hccl algorithm: 910B not support level1 ahc, return ERROR.");
936 0 : return HCCL_E_PARA;
937 : }
938 :
939 9 : HCCL_INFO(
940 : "[SelectAlgforAHC] ahcOpType[%u] isAHCWholeConfig[%u] AHClevel[%u] algType_[%u] deviceType_[%u]", ahcOpType,
941 : isAHCWholeConfig, ahcSubGroupLevel, algType_.algoLevel1, deviceType_);
942 :
943 : AlgTypeLevel1 algTypeLevel1;
944 :
945 9 : std::vector<std::vector<std::vector<u32>>> globalSubGroups;
946 9 : std::map<AHCConcOpType, TemplateType> ahcAlgOption;
947 9 : CHK_RET(topoMatcher_->GetGlobalSubGroups(ahcSubGroupLevel, globalSubGroups));
948 9 : topoMatcher_->GetAHCAlgOption(ahcAlgOption);
949 :
950 9 : AHCAlgSelectParam ahcAlgSelectParam;
951 9 : ahcAlgSelectParam.opType = ahcOpType;
952 9 : ahcAlgSelectParam.dataSize = dataSize;
953 :
954 : // AHC 封装算法选择逻辑
955 9 : CHK_RET(AHCAlgSelect(algTypeLevel1, globalSubGroups, ahcAlgOption, ahcAlgSelectParam));
956 :
957 9 : topoMatcher_->SetAHCAlgOption(ahcAlgOption);
958 :
959 9 : auto iter = HCCL_ALGO_LEVEL1_NAME_MAP.find(algTypeLevel1);
960 9 : CHK_PRT_RET(
961 : iter == HCCL_ALGO_LEVEL1_NAME_MAP.end(),
962 : HCCL_ERROR("[AHCAlgSelect] level1: algType_[%u] is invalid.", algTypeLevel1), HCCL_E_INTERNAL);
963 :
964 : // 支持 AHC 自适应调节为 BROKE 类型
965 9 : if (algType_.algoLevel1 != algTypeLevel1 && algType_.algoLevel1 == AlgTypeLevel1::ALG_LEVEL1_AHC) {
966 8 : algType_.algoLevel1 = algTypeLevel1;
967 : }
968 :
969 9 : HCCL_INFO(
970 : "[AHCAlgSelect] hccl algorithm: there are %u server(%u module) in level1, using %s algo", serverNum_,
971 : moduleNum_, iter->second.c_str());
972 :
973 9 : return HCCL_SUCCESS;
974 9 : }
975 :
976 9 : HcclResult CollAlgOperator::AHCAlgSelect(
977 : AlgTypeLevel1& algType, std::vector<std::vector<std::vector<u32>>>& globalSubGroups,
978 : std::map<AHCConcOpType, TemplateType>& ahcAlgOption, AHCAlgSelectParam& ahcAlgSelectParam)
979 : {
980 : // globalSubGroups 参数检查
981 9 : CHK_RET(CommAHCBaseInfo::CheckGlobalGroups(globalSubGroups));
982 :
983 9 : bool isAHCType = false;
984 9 : u32 minSubGroupSize = globalSubGroups[0][0].size();
985 9 : u32 maxSubGroupSize = globalSubGroups[0][0].size();
986 45 : for (u32 i = 1; i < globalSubGroups[0].size(); ++i) {
987 36 : if (globalSubGroups[0][i].size() < minSubGroupSize) {
988 0 : minSubGroupSize = globalSubGroups[0][i].size();
989 : }
990 36 : if (globalSubGroups[0][i].size() > maxSubGroupSize) {
991 0 : maxSubGroupSize = globalSubGroups[0][i].size();
992 : }
993 : }
994 54 : for (u32 i = 0; i < globalSubGroups[0].size(); ++i) {
995 45 : if (globalSubGroups[0][i].size() != minSubGroupSize) {
996 0 : isAHCType = true;
997 0 : break;
998 : }
999 : }
1000 :
1001 : // 多平面 reduce scatter 和 all gather 算子，强制写死成BROKE类型
1002 9 : if (deviceNumPerServer_ != 1 && ahcAlgSelectParam.opType != AHCOpType::AHC_OP_TYPE_ALLREDUCE) {
1003 0 : isAHCType = false;
1004 : }
1005 :
1006 : // add AHC Conc Type logic here, modify init Type depend on the input para
1007 9 : CHK_RET(AHCAlgOptionSelect(algType, globalSubGroups, ahcAlgOption, ahcAlgSelectParam));
1008 :
1009 9 : if (ahcAlgSelectParam.enableAlgAutoSelect == false) { // 关闭算法自适应功能时，默认设置AHC算法
1010 0 : algType = AlgTypeLevel1::ALG_LEVEL1_AHC;
1011 0 : return HCCL_SUCCESS;
1012 : }
1013 :
1014 9 : if (isAHCType) {
1015 0 : algType = AlgTypeLevel1::ALG_LEVEL1_AHC; // 设置为 AHC 类型
1016 : } else {
1017 9 : algType = AlgTypeLevel1::ALG_LEVEL1_AHC_BROKE; // 设置为 BROKE 类型
1018 : }
1019 :
1020 9 : HCCL_DEBUG(
1021 : "[AHCAlgSelect] end minSubGroupSize = %u maxSubGroupSize = %u isAHCType = %u", minSubGroupSize, maxSubGroupSize,
1022 : isAHCType);
1023 :
1024 9 : return HCCL_SUCCESS;
1025 : }
1026 :
1027 11 : HcclResult CollAlgOperator::AHCAlgOptionSelect(
1028 : const AlgTypeLevel1& algType, std::vector<std::vector<std::vector<u32>>>& globalSubGroups,
1029 : std::map<AHCConcOpType, TemplateType>& ahcAlgOption, const AHCAlgSelectParam& ahcAlgSelectParam)
1030 : {
1031 : (void)algType;
1032 : (void)ahcAlgSelectParam;
1033 11 : AHCConcOpType ahcConcOpType;
1034 : // 一层组间拼接时，分组数大于设定阈值则修改默认算法为NHR
1035 11 : if (globalSubGroups[0].size() <= AHC_LEVEL0_GROUP_SIZE_THRESHOLD) {
1036 4 : HCCL_DEBUG("[AHCAlgSelect] conc inter select type RING ");
1037 4 : ahcConcOpType = {AHCLevel::AHC_LEVEL_0, ConcType::CONC_INTER, AHCOpType::AHC_OP_TYPE_REDUCE_SCATTER};
1038 4 : ahcAlgOption[ahcConcOpType] = TemplateType::TEMPLATE_REDUCESCATTER_RING;
1039 :
1040 4 : ahcConcOpType = {AHCLevel::AHC_LEVEL_0, ConcType::CONC_INTER, AHCOpType::AHC_OP_TYPE_ALLREDUCE};
1041 4 : ahcAlgOption[ahcConcOpType] = TemplateType::TEMPLATE_ALL_REDUCE_RING;
1042 :
1043 4 : ahcConcOpType = {AHCLevel::AHC_LEVEL_0, ConcType::CONC_INTER, AHCOpType::AHC_OP_TYPE_ALLGATHER};
1044 4 : ahcAlgOption[ahcConcOpType] = TemplateType::TEMPLATE_ALL_GATHER_RING;
1045 : } else {
1046 7 : HCCL_DEBUG("[AHCAlgSelect] conc inter select type NHR ");
1047 7 : ahcConcOpType = {AHCLevel::AHC_LEVEL_0, ConcType::CONC_INTER, AHCOpType::AHC_OP_TYPE_REDUCE_SCATTER};
1048 7 : ahcAlgOption[ahcConcOpType] = TemplateType::TEMPLATE_REDUCESCATTER_NHR;
1049 :
1050 7 : ahcConcOpType = {AHCLevel::AHC_LEVEL_0, ConcType::CONC_INTER, AHCOpType::AHC_OP_TYPE_ALLREDUCE};
1051 7 : ahcAlgOption[ahcConcOpType] = TemplateType::TEMPLATE_ALL_REDUCE_NHR;
1052 :
1053 7 : ahcConcOpType = {AHCLevel::AHC_LEVEL_0, ConcType::CONC_INTER, AHCOpType::AHC_OP_TYPE_ALLGATHER};
1054 7 : ahcAlgOption[ahcConcOpType] = TemplateType::TEMPLATE_ALL_GATHER_NHR;
1055 : }
1056 11 : return HCCL_SUCCESS;
1057 : }
1058 :
1059 0 : u32 CollAlgOperator::CalcOptimalIntraRingsize(u64 count, HcclDataType dataType, HcclCMDType opType)
1060 : {
1061 0 : if (!topoMatcher_->GetARSFlag())
1062 0 : return 0;
1063 :
1064 0 : u32 level0RankSize = topoMatcher_->GetCommPlaneRanks(COMM_LEVEL0)[0].size();
1065 0 : u32 rankSizeInSuperPod = topoMatcher_->GetCommPlaneRanks(COMM_ARS)[0].size();
1066 0 : u32 perDataSize = 0;
1067 0 : CHK_RET(SalGetDataTypeSize(dataType, perDataSize));
1068 : // 不支持 ARS 或环内卡数不是 2 的倍数
1069 0 : u32 level0RingSize = 1;
1070 0 : if (!isARSDoubleRing_ || (level0RankSize % FACTOR_TWO != 0)) {
1071 0 : HCCL_INFO(
1072 : "not Support ARS doubleRing, level0RingSize:[%u], level0RankSize[%u].", level0RingSize, level0RankSize);
1073 0 : return level0RingSize;
1074 : }
1075 : // --- 1. 带宽 & 基本参数 ---
1076 : float bwHCCS, bwHBM, bwSIO;
1077 0 : constexpr u32 level0 = 0;
1078 0 : constexpr u32 level2 = 2;
1079 0 : constexpr u32 level3 = 3;
1080 0 : CHK_RET(GetBandWidthPerNPU(level0, userRankSize_, deviceNumPerAggregation_, bwHCCS));
1081 0 : CHK_RET(GetBandWidthPerNPU(level2, userRankSize_, deviceNumPerAggregation_, bwHBM));
1082 0 : CHK_RET(GetBandWidthPerNPU(level3, userRankSize_, deviceNumPerAggregation_, bwSIO));
1083 0 : float latency = BASE_COMM_LATENCY / MULTIPLIER_MS2US; // ms
1084 : // --- 2. 数据总量 (GB) ---
1085 0 : float baseSizeGB = static_cast<double>(count) * perDataSize / GB2B;
1086 0 : float totalSize = baseSizeGB;
1087 0 : HCCL_INFO(
1088 : "CalcOptimalIntraRingsize: count[%u], totalSize:[%lf]GB, perDataSize[%u].", count, totalSize, perDataSize);
1089 0 : if (opType == HcclCMDType::HCCL_CMD_REDUCE_SCATTER || opType == HcclCMDType::HCCL_CMD_ALLGATHER) {
1090 0 : totalSize *= rankSizeInSuperPod;
1091 : }
1092 : // --- 3. 枚举可能的环大小 ---
1093 0 : std::vector<u32> factors;
1094 0 : for (u32 i = 1; i <= rankSizeInSuperPod / i; ++i) {
1095 0 : if (rankSizeInSuperPod % i == 0) {
1096 0 : if (i != rankSizeInSuperPod / i) {
1097 0 : factors.push_back(rankSizeInSuperPod / i);
1098 : }
1099 0 : factors.push_back(i);
1100 : }
1101 : }
1102 0 : std::sort(factors.begin(), factors.end());
1103 : // --- 4. 计算最优带宽 ---
1104 0 : double maxBwARS = 0.0;
1105 0 : for (u32 N1 : factors) {
1106 : // 传输时延 (ms)
1107 : double latencyIntra;
1108 0 : if (N1 == FACTOR_TWO) {
1109 0 : latencyIntra = totalSize * MULTIPLIER_S2MS / FACTOR_TWO / bwSIO;
1110 0 : } else if ((N1 % FACTOR_TWO == 0) && (N1 > FACTOR_TWO)) {
1111 0 : latencyIntra = (N1 - 1) * totalSize * MULTIPLIER_S2MS / N1 / bwHCCS / FACTOR_TWO;
1112 : } else {
1113 0 : latencyIntra = (N1 - 1) * totalSize * MULTIPLIER_S2MS / N1 / bwHCCS;
1114 : }
1115 0 : u32 N2 = rankSizeInSuperPod / N1;
1116 : // 静态时延 (ms)
1117 0 : double interStep = (algType_.algoLevel1 == AlgTypeLevel1::ALG_LEVEL1_RING) ? (N2 - 1) : log2(N2);
1118 0 : double latencyStep = (interStep + (N1 - 1)) * latency;
1119 0 : double latencyInter = (N2 - 1) * totalSize * MULTIPLIER_S2MS / N1 / N2 / bwHCCS;
1120 : // HBM 拷贝时延 (ms)
1121 0 : double latencyCopy = totalSize * MULTIPLIER_S2MS / bwHBM;
1122 0 : u8 mul = (HcclCMDType::HCCL_CMD_ALLREDUCE == opType) ? FACTOR_TWO : 1;
1123 0 : double timeCost = mul * (latencyStep + latencyIntra + latencyInter) + latencyCopy;
1124 0 : double bwARS = totalSize / timeCost; // GB/ms
1125 0 : if (bwARS > maxBwARS) {
1126 0 : level0RingSize = N1;
1127 0 : maxBwARS = bwARS;
1128 : }
1129 : }
1130 0 : HCCL_INFO("level0RingSize:[%u], level0RankSize[%u], totalSize:[%lf]GB", level0RingSize, level0RankSize, totalSize);
1131 0 : return level0RingSize;
1132 0 : }
1133 :
1134 20 : bool CollAlgOperator::IsNeedStrictMode(const OpParam& param)
1135 : {
1136 : bool isStrictMode
1137 20 : = (topoMatcher_->GetDeterministicConfig() == DETERMINISTIC_STRICT)
1138 0 : && (param.DataDes.dataType == HCCL_DATA_TYPE_FP16 || param.DataDes.dataType == HCCL_DATA_TYPE_FP32
1139 0 : || param.DataDes.dataType == HCCL_DATA_TYPE_BFP16 || param.DataDes.dataType == HCCL_DATA_TYPE_FP64)
1140 0 : && (param.reduceType == HCCL_REDUCE_SUM || param.reduceType == HCCL_REDUCE_PROD)
1141 20 : && userRankSize_ >= MIN_STRICT_RANK_NUM;
1142 :
1143 20 : return isStrictMode;
1144 : }
1145 :
1146 0 : bool CollAlgOperator::CheckStrictCondition(const OpParam& param) const
1147 : {
1148 0 : CHK_PRT_RET(
1149 : multiModuleDiffDeviceNumMode_ || multiSuperPodDiffDeviceNumMode_ || multiSuperPodDiffServerNumMode_,
1150 : HCCL_ERROR("[CollAlgOperator][CheckStrictCondition] DETERMINISTIC_STRICT mode not support asymmetrical topo."),
1151 : false);
1152 :
1153 0 : CHK_PRT_RET(
1154 : param.reduceType == HCCL_REDUCE_PROD,
1155 : HCCL_ERROR("[CollAlgOperator][CheckStrictCondition] DETERMINISTIC_STRICT mode not support PROD."), false);
1156 :
1157 0 : CHK_PRT_RET(
1158 : param.DataDes.dataType == HCCL_DATA_TYPE_FP64,
1159 : HCCL_ERROR("[CollAlgOperator][CheckStrictCondition] DETERMINISTIC_STRICT mode not support FP64."), false);
1160 :
1161 0 : CHK_PRT_RET(
1162 : GetExternalInputInterHccsDisable(),
1163 : HCCL_ERROR("[CollAlgOperator][CheckStrictCondition] DETERMINISTIC_STRICT mode not support HCCS disable."),
1164 : false);
1165 :
1166 0 : return true;
1167 : }
1168 :
1169 : } // namespace hccl
|
