Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "externalinput_pub.h"
12 : #include "externalinput.h"
13 : #include "comm_config_pub.h"
14 : #include "adapter_error_manager_pub.h"
15 : #include "adapter_rts_common.h"
16 : #include "alg_env_config.h"
17 :
18 : namespace hccl {
19 712 : CommConfig::CommConfig(const std::string& commName)
20 712 : : bufferSize_(GetExternalInputCCLBuffSize()),
21 715 : deterministic_(GetExternalInputHcclDeterministicV2()),
22 714 : commName_(commName),
23 714 : aivMode_(GetExternalInputHcclAivMode()),
24 715 : aicpuUnfold_(GetExternalInputHcclAicpuUnfold()),
25 715 : trafficClass_(HCCL_COMM_TRAFFIC_CLASS_CONFIG_NOT_SET),
26 715 : serviceLevel_(HCCL_COMM_SERVICE_LEVEL_CONFIG_NOT_SET),
27 715 : worldRankID_(0),
28 715 : jobID_(0),
29 715 : aclGraphZeroCopyEnable_(0),
30 715 : onlyAivMode_(false),
31 715 : execTimeOut_(GetInternalExecTimeOut()),
32 715 : execTimeOutSetByConfig_(false),
33 715 : retryMaxCnt_(GetExternalInputRetryMaxCnt()),
34 715 : retryHoldTime_(GetExternalInputRetryHoldTime()),
35 715 : retryIntervalTime_(GetExternalInputRetryIntervalTime()),
36 1430 : bufferName_(""),
37 712 : hcclQos_(HCCL_COMM_QOS_CONFIG_NOT_SET),
38 712 : symmetricMemoryStride_(HCCL_DEFAULT_SYMMETRIC_MEMORY_STRIDE),
39 2140 : sqDepth_(HCCL_COMM_SQ_DEPTH_CONFIG_NOT_SET)
40 : {
41 712 : InitAlgoConfig();
42 715 : InitRetryEnable();
43 715 : }
44 :
45 2023 : CommConfig::CommConfig()
46 2023 : : bufferSize_(GetExternalInputCCLBuffSize()),
47 2024 : deterministic_(GetExternalInputHcclDeterministicV2()),
48 2024 : aivMode_(GetExternalInputHcclAivMode()),
49 2024 : aicpuUnfold_(GetExternalInputHcclAicpuUnfold()),
50 2024 : trafficClass_(HCCL_COMM_TRAFFIC_CLASS_CONFIG_NOT_SET),
51 2024 : serviceLevel_(HCCL_COMM_SERVICE_LEVEL_CONFIG_NOT_SET),
52 2024 : worldRankID_(0),
53 2024 : jobID_(0),
54 2024 : aclGraphZeroCopyEnable_(0),
55 2024 : onlyAivMode_(false),
56 2024 : execTimeOut_(GetInternalExecTimeOut()),
57 2024 : execTimeOutSetByConfig_(false),
58 2024 : retryMaxCnt_(GetExternalInputRetryMaxCnt()),
59 2024 : retryHoldTime_(GetExternalInputRetryHoldTime()),
60 2024 : retryIntervalTime_(GetExternalInputRetryIntervalTime()),
61 4048 : bufferName_(""),
62 2023 : hcclQos_(HCCL_COMM_QOS_CONFIG_NOT_SET),
63 2023 : symmetricMemoryStride_(HCCL_DEFAULT_SYMMETRIC_MEMORY_STRIDE),
64 6070 : sqDepth_(HCCL_COMM_SQ_DEPTH_CONFIG_NOT_SET)
65 : {
66 2023 : InitAlgoConfig();
67 2024 : InitRetryEnable();
68 2024 : }
69 :
70 3395 : CommConfig::~CommConfig() {}
71 :
72 2738 : void CommConfig::InitAlgoConfig()
73 : {
74 287275 : for (u32 opType = 0; opType < static_cast<u32>(HcclCMDType::HCCL_CMD_MAX); opType++) {
75 284847 : algoConfig_[static_cast<HcclCMDType>(opType)]
76 569021 : = GetExternalInputHcclAlgoConfig(static_cast<HcclCMDType>(opType));
77 : }
78 2737 : }
79 :
80 2739 : void CommConfig::InitRetryEnable()
81 : {
82 2739 : retryEnable_[HCCL_RETRY_ENABLE_LEVEL_0] = GetExternalInputIntraServerRetryEnable();
83 2739 : retryEnable_[HCCL_RETRY_ENABLE_LEVEL_1] = GetExternalInputInterServerRetryEnable();
84 2739 : retryEnable_[HCCL_RETRY_ENABLE_LEVEL_2] = GetExternalInputInterSuperPodRetryEnable();
85 2739 : }
86 :
87 12 : HcclResult CommConfig::Load(const HcclCommConfig* userConfig)
88 : {
89 : // 检查是否为空
90 12 : CHK_PTR_NULL(userConfig);
91 :
92 : // 读取结构体的size
93 11 : size_t configSize = *(reinterpret_cast<const size_t*>(userConfig));
94 11 : HCCL_INFO("[Load] config size[%llu]", configSize);
95 :
96 11 : const size_t maxConfigSize = sizeof(CommConfigHandle);
97 11 : if (configSize > maxConfigSize) {
98 0 : HCCL_WARNING(
99 : "[Load] configSize[%llu] is larger than sizeof(CommConfigHandle)[%llu]", configSize, maxConfigSize);
100 0 : configSize = maxConfigSize;
101 11 : } else if (configSize < maxConfigSize) {
102 0 : HCCL_WARNING("[Load] configSize[%llu] is less than sizeof(CommConfigHandle)[%llu]", configSize, maxConfigSize);
103 : }
104 :
105 : // 根据size读取结构体
106 11 : CommConfigHandle configHandle;
107 11 : s32 sRet = memcpy_s(&configHandle, maxConfigSize, userConfig, configSize);
108 11 : CHK_PRT_RET(
109 : sRet != EOK,
110 : HCCL_ERROR(
111 : "[Load] memcpy comm config fail. errorno[%d] "
112 : "params:destMaxSize[%u], count[%u]",
113 : sRet, maxConfigSize, configSize),
114 : HCCL_E_MEMORY);
115 :
116 : // 检查Magic word是否合法
117 11 : CHK_RET(CheckMagicWord(configHandle));
118 :
119 : // 根据版本号读取配置，检查配置参数合法性
120 11 : CHK_RET(SetConfigByVersion(configHandle));
121 :
122 10 : HCCL_RUN_INFO(
123 : "[Load] comm config info of [%s]: configSize[%llu], version[%u], opExpansionMode[%u]", commName_.c_str(),
124 : configHandle.info.configSize, configHandle.info.version, configHandle.opExpansionMode);
125 10 : HCCL_RUN_INFO(
126 : "[Load] comm config of [%s]: bufferSize[%llu], deterministic[%u], trafficClass[%u], serviceLevel[%u]"
127 : ", execTimeOut[%u]s, bufferName[%s], hcclQos[%u], symmetricMemoryStride[%llu], aclGraphZeroCopyEnable[%u]",
128 : commName_.c_str(), bufferSize_, deterministic_, trafficClass_, serviceLevel_, execTimeOut_, bufferName_.c_str(),
129 : hcclQos_, symmetricMemoryStride_, aclGraphZeroCopyEnable_);
130 10 : return HCCL_SUCCESS;
131 : }
132 :
133 11 : HcclResult CommConfig::CheckMagicWord(const CommConfigHandle& config)
134 : {
135 11 : if (config.info.magicWord != COMM_CONFIG_MAGIC_WORD) {
136 0 : RPT_INPUT_ERR(
137 : true, "EI0003", std::vector<std::string>({"ccl_op", "value", "parameter", "expect"}),
138 : std::vector<std::string>(
139 : {"HcclCommInitRootInfoConfig", std::to_string(config.info.magicWord), "magic word",
140 : "The magic word must be initialized with the result of HcclCommConfigInit()"}));
141 0 : HCCL_ERROR(
142 : "[CheckMagicWord] Invalid magic word[0x%x]. Please make sure the config has been initialized by "
143 : "HcclCommConfigInit().",
144 : config.info.magicWord);
145 0 : return HCCL_E_PARA;
146 : }
147 :
148 11 : return HCCL_SUCCESS;
149 0 : }
150 :
151 26 : HcclResult CommConfig::SetConfigByVersion(const CommConfigHandle& config)
152 : {
153 26 : if (config.info.version > CommConfigVersion::COMM_CONFIG_VERSION_ELEVEN) {
154 : // 传入的config的版本高于当前版本，警告不支持的配置项将被忽略
155 0 : HCCL_WARNING(
156 : "[SetConfigByVersion] The version of provided config[%u] is higher than the current version[%u], "
157 : "unsupported configuration will be ignored.",
158 : config.info.version, CommConfigVersion::COMM_CONFIG_VERSION_ELEVEN);
159 26 : } else if (config.info.version < CommConfigVersion::COMM_CONFIG_VERSION_ELEVEN) {
160 : // 传入的config的版本低于当前版本，警告高版本支持的配置项将被忽略
161 16 : HCCL_WARNING(
162 : "[SetConfigByVersion] The version of provided config[%u] is lower than the current version[%u], "
163 : "configurations supported by later versions will be ignored.",
164 : config.info.version, CommConfigVersion::COMM_CONFIG_VERSION_ELEVEN);
165 : }
166 :
167 26 : if (config.info.version >= CommConfigVersion::COMM_CONFIG_VERSION_ONE) {
168 : // 版本大于等于1，设置CCL buffer、确定性计算配置
169 25 : CHK_RET(SetConfigBufferSize(config));
170 24 : CHK_RET(SetConfigDeterministic(config));
171 : }
172 :
173 25 : if (config.info.version >= CommConfigVersion::COMM_CONFIG_VERSION_TWO) {
174 : // 版本大于等于2，设置通信域名称
175 21 : CHK_RET(SetConfigCommName(config));
176 : }
177 :
178 25 : if (config.info.version >= CommConfigVersion::COMM_CONFIG_VERSION_THREE) {
179 : // 版本大于等于3，设置Udi
180 21 : CHK_RET(SetConfigUdi(config));
181 : }
182 :
183 25 : if (config.info.version >= CommConfigVersion::COMM_CONFIG_VERSION_FOUR) {
184 : // 版本大于等于4，设置Aiv、Aicpu
185 21 : CHK_RET(SetConfigOpExpansionMode(config));
186 : }
187 :
188 25 : if (config.info.version >= CommConfigVersion::COMM_CONFIG_VERSION_FIVE) {
189 : // 版本大于等于5，支持配置TC，SL
190 10 : trafficClass_ = config.trafficClass;
191 10 : serviceLevel_ = config.serviceLevel;
192 : }
193 :
194 25 : if (config.info.version >= CommConfigVersion::COMM_CONFIG_VERSION_SIX) {
195 : // 版本大于等于6
196 10 : worldRankID_ = config.worldRankID;
197 10 : jobID_ = config.jobID;
198 : }
199 :
200 25 : if (config.info.version >= CommConfigVersion::COMM_CONFIG_VERSION_SEVEN) {
201 : // 版本大于等于7，支持配置AclGraph使能/去使能
202 10 : RPT_INPUT_ERR(
203 : config.aclGraphZeroCopyEnable > 1, "EI0003",
204 : std::vector<std::string>({"ccl_op", "value", "parameter", "expect"}),
205 : std::vector<std::string>(
206 : {"HcclCommInitRootInfoConfig", std::to_string(config.aclGraphZeroCopyEnable), "aclGraphZeroCopy",
207 : "0 or 1."}));
208 10 : CHK_PRT_RET(
209 : config.aclGraphZeroCopyEnable > 1,
210 : HCCL_ERROR(
211 : "[CommConfig][SetConfigByVersion] aclGraphZeroCopyEnable value=[%u] invalid. support 0 or 1",
212 : config.aclGraphZeroCopyEnable),
213 : HCCL_E_PARA);
214 10 : aclGraphZeroCopyEnable_ = config.aclGraphZeroCopyEnable;
215 : }
216 :
217 25 : if (config.info.version >= CommConfigVersion::COMM_CONFIG_VERSION_EIGHT) {
218 : // 版本大于等于8，支持配置execTimeOut
219 10 : CHK_RET(SetConfigExecTimeout(config));
220 : // 支持配置HcclAlgo
221 10 : CHK_RET(SetConfigHcclAlgo(config));
222 : // 解析重执行设置
223 10 : CHK_RET(SetConfigHcclRetryEnable(config));
224 10 : CHK_RET(SetConfigHcclRetryParams(config));
225 : }
226 :
227 25 : if (config.info.version >= CommConfigVersion::COMM_CONFIG_VERSION_NINE) {
228 : // 版本大于等于9
229 10 : CHK_RET(SetConfigBufferName(config));
230 : }
231 :
232 25 : if (config.info.version >= CommConfigVersion::COMM_CONFIG_VERSION_TEN) {
233 : // 版本大于等于10,支持配置通信域级别的AI CPU SDMA QOS
234 10 : hcclQos_ = config.hcclQos;
235 : // 版本大于等于10，支持配置对称内存每个rank的预留VA大小
236 10 : symmetricMemoryStride_ = config.symmetricMemoryStride;
237 : }
238 :
239 25 : if (config.info.version >= CommConfigVersion::COMM_CONFIG_VERSION_ELEVEN) {
240 : // 版本大于等于11，支持配置通信域级别的sqDepth
241 9 : sqDepth_ = config.sqDepth;
242 : }
243 25 : HCCL_INFO("NSLBDP-VERSION config.info.version = [%u] .", config.info.version);
244 25 : return HCCL_SUCCESS;
245 0 : }
246 :
247 25 : HcclResult CommConfig::SetConfigBufferSize(const CommConfigHandle& config)
248 : {
249 25 : if (config.bufferSize == HCCL_COMM_BUFFSIZE_CONFIG_NOT_SET) {
250 : // 默认跟随环境变量配置
251 4 : HCCL_INFO(
252 : "[SetConfigByVersion] The hcclBufferSize is not configured, use the env config [%u](Bytes) as default.",
253 : bufferSize_);
254 21 : } else if (config.bufferSize < HCCL_CCL_COMM_BUFFER_MIN) {
255 17 : RPT_INPUT_ERR(
256 : true, "EI0003", std::vector<std::string>({"ccl_op", "value", "parameter", "expect"}),
257 : std::vector<std::string>(
258 : {"HcclCommInitRootInfoConfig", std::to_string(config.bufferSize), "hcclBufferSize",
259 : "should be equal to or greater than 1(MB)."}));
260 1 : HCCL_ERROR(
261 : "[%s][%s] The configuration of hcclBufferSize[%u(MB)] is invalid, which should be "
262 : "greater than %u(MB).",
263 : LOG_KEYWORDS_TASK_EXEC.c_str(), LOG_KEYWORDS_INVALID_ARGUMENT.c_str(), config.bufferSize,
264 : HCCL_CCL_COMM_BUFFER_MIN);
265 1 : return HCCL_E_PARA;
266 : } else {
267 : // 使用config配置
268 20 : bufferSize_ = static_cast<u64>(config.bufferSize) * HCCL_CCL_COMM_FIXED_CALC_BUFFER_SIZE; // MByte 转 Byte
269 : }
270 24 : return HCCL_SUCCESS;
271 2 : }
272 :
273 24 : HcclResult CommConfig::SetConfigDeterministic(const CommConfigHandle& config)
274 : {
275 24 : if (config.deterministic == HCCL_COMM_DETERMINISTIC_CONFIG_NOT_SET) {
276 : // 默认跟随环境变量配置
277 10 : HCCL_INFO(
278 : "[SetConfigByVersion] The hcclDeterministic is not configured, use the env config [%u] as default.",
279 : deterministic_);
280 14 : } else if (config.deterministic > DETERMINISTIC_STRICT) {
281 0 : RPT_INPUT_ERR(
282 : true, "EI0003", std::vector<std::string>({"ccl_op", "value", "parameter", "expect"}),
283 : std::vector<std::string>(
284 : {"HcclCommInitRootInfoConfig", std::to_string(config.deterministic), "hcclDeterministic",
285 : "should be 0(disable) , 1(enable) or 2(strict)."}));
286 0 : HCCL_ERROR(
287 : "[%s][%s] The configuration of hcclDeterministic[%u] is invalid, "
288 : "which should be 0(disable) , 1(enable) or 2(strict).",
289 : LOG_KEYWORDS_TASK_EXEC.c_str(), LOG_KEYWORDS_INVALID_ARGUMENT.c_str(), config.deterministic);
290 0 : return HCCL_E_PARA;
291 : } else {
292 14 : if (config.deterministic == DETERMINISTIC_STRICT) {
293 : DevType deviceType;
294 1 : CHK_RET(hrtGetDeviceType(deviceType));
295 1 : if (deviceType != DevType::DEV_TYPE_910B) {
296 0 : RPT_INPUT_ERR(
297 : true, "EI0003", std::vector<std::string>({"ccl_op", "value", "parameter", "expect"}),
298 : std::vector<std::string>(
299 : {"HcclCommInitRootInfoConfig", std::to_string(config.deterministic), "hcclDeterministic",
300 : "set to 2(strict), only support A2."}));
301 0 : HCCL_ERROR(
302 : "[%s][%s] The configuration of hcclDeterministic[%u] is set to "
303 : "2(strict), and only support A2",
304 : LOG_KEYWORDS_TASK_EXEC.c_str(), LOG_KEYWORDS_INVALID_ARGUMENT.c_str(), config.deterministic);
305 0 : return HCCL_E_PARA;
306 : }
307 : }
308 14 : deterministic_ = static_cast<u8>(config.deterministic); // 前面已保证数值不超过UINT8_MAX，直接进行类型转换
309 : }
310 24 : return HCCL_SUCCESS;
311 0 : }
312 :
313 21 : HcclResult CommConfig::SetConfigCommName(const CommConfigHandle& config)
314 : {
315 21 : if (config.commName[0] != '\0') {
316 12 : auto commNameLength = strlen(config.commName);
317 12 : commNameLength = commNameLength < COMM_NAME_MAX_LENGTH ? commNameLength : COMM_NAME_MAX_LENGTH;
318 24 : commName_ = std::string(config.commName, commNameLength);
319 : }
320 21 : return HCCL_SUCCESS;
321 : }
322 :
323 21 : HcclResult CommConfig::SetConfigUdi(const CommConfigHandle& config)
324 : {
325 21 : if (config.udi[0] == '\0') {
326 10 : udi_ = "Unspecified";
327 10 : return HCCL_SUCCESS;
328 : }
329 11 : auto udiLength = strlen(config.udi);
330 11 : udiLength = udiLength < COMM_NAME_MAX_LENGTH ? udiLength : COMM_NAME_MAX_LENGTH;
331 11 : udi_ = std::string(config.udi, udiLength);
332 11 : return HCCL_SUCCESS;
333 : }
334 :
335 10 : HcclResult CommConfig::SetConfigBufferName(const CommConfigHandle& config)
336 : {
337 10 : if (config.bufferName[0] != '\0') {
338 0 : auto bufferNameLength = strlen(config.bufferName);
339 0 : bufferNameLength = bufferNameLength < BUFFER_NAME_MAX_LENGTH ? bufferNameLength : BUFFER_NAME_MAX_LENGTH;
340 0 : bufferName_ = std::string(config.bufferName, bufferNameLength);
341 : }
342 10 : return HCCL_SUCCESS;
343 : }
344 :
345 23 : HcclResult CommConfig::SetConfigOpExpansionMode(const CommConfigHandle& config)
346 : {
347 23 : switch (config.opExpansionMode) {
348 11 : case COMM_CONFIG_OPEXPANSION_DEFAULT:
349 11 : HCCL_INFO(
350 : "CommConfig is set to 0(default), aicpuUnfold_ is [%d] and aivMode_ is [%d].", aicpuUnfold_, aivMode_);
351 11 : break;
352 2 : case COMM_CONFIG_OPEXPANSION_HOST:
353 2 : aivMode_ = false;
354 2 : aicpuUnfold_ = false;
355 2 : HCCL_INFO(
356 : "CommConfig is set to 1(host), aicpuUnfold_ is [%d] and aivMode_ is [%d].", aicpuUnfold_, aivMode_);
357 2 : break;
358 3 : case COMM_CONFIG_OPEXPANSION_AICPU:
359 : // 目前只有A3和300I支持Aicpu展开
360 : DevType deviceType;
361 3 : CHK_RET(hrtGetDeviceType(deviceType));
362 3 : if (deviceType == DevType::DEV_TYPE_910_93 || deviceType == DevType::DEV_TYPE_910B) {
363 3 : aicpuUnfold_ = true;
364 3 : aivMode_ = false;
365 3 : HCCL_INFO(
366 : "CommConfig is set to 2(aicpuUnfold_), aicpuUnfold_ is [%d] and aivMode_ is [%d].", aicpuUnfold_,
367 : aivMode_);
368 : } else {
369 0 : HCCL_WARNING(
370 : "Only A3 and 300I support aicpu unfold, set aicpuUnfold_ to [%d] and aivMode_ to [%d].",
371 : aicpuUnfold_, aivMode_);
372 : }
373 3 : break;
374 4 : case COMM_CONFIG_OPEXPANSION_AIV:
375 4 : aivMode_ = true;
376 4 : HCCL_INFO(
377 : "CommConfig is set to 3(aivMode), aicpuUnfold_ is [%d] and aivMode_ is [%d].", aicpuUnfold_, aivMode_);
378 4 : break;
379 2 : case COMM_CONFIG_OPEXPANSION_ONLY_AIV:
380 2 : onlyAivMode_ = true;
381 2 : HCCL_INFO(
382 : "CommConfig is set to 4(aivOnly), aicpuUnfold_ is [%d] and aivMode_ is [%d] onlyAivMode_ is[%d].",
383 : aicpuUnfold_, aivMode_, onlyAivMode_);
384 2 : break;
385 1 : default:
386 : // 目前opExpansionMode的合法值为[0,4]，值不合法时回退为环境变量配置
387 1 : HCCL_WARNING(
388 : "Current version not support opExpansionMode[%u], set aicpuUnfold_ to [%d] and aivMode_ to [%d].",
389 : config.opExpansionMode, aicpuUnfold_, aivMode_);
390 1 : break;
391 : }
392 :
393 23 : return HCCL_SUCCESS;
394 : }
395 :
396 11 : HcclResult CommConfig::SetConfigExecTimeout(const CommConfigHandle& config)
397 : {
398 11 : if (config.execTimeOut == COMM_EXECTIMEOUT_CONFIG_NOT_SET) {
399 : // 默认跟随环境变量
400 11 : HCCL_INFO(
401 : "[SetConfigByVersion] The hcclExecTimeOut is not configured, use the env config [%u s] as default.",
402 : execTimeOut_);
403 : } else {
404 0 : s32 execTimeOut = config.execTimeOut;
405 :
406 : DevType deviceType;
407 0 : CHK_RET(hrtGetDeviceType(deviceType)); // 910A和910B/C要分开
408 0 : if (deviceType == DevType::DEV_TYPE_910_93 || deviceType == DevType::DEV_TYPE_910B) {
409 : // 910B和910_93算子超时时间范围0s-2147483647s,其中0代表永不超时
410 0 : if ((execTimeOut < 0) || (execTimeOut > HCCL_EXEC_TIME_OUT_S_910_93)) {
411 0 : HCCL_WARNING(
412 : "[SetConfigByVersion][SetConfigExecTimeout] The configuration of ComConfigHcclExecTimeOut[%d]s is "
413 : "invalid, "
414 : "which should be a number greater than or equal to 0s and less "
415 : "than or equal to 2147483647s",
416 : execTimeOut);
417 0 : return HCCL_SUCCESS;
418 : }
419 : } else {
420 : // 非910B和910_93算子超时时间范围1s-17340s
421 0 : if ((execTimeOut <= 0) || (execTimeOut > HCCL_EXEC_TIME_OUT_S)) {
422 0 : HCCL_WARNING(
423 : "[SetConfigByVersion] The configuration of ComConfigHcclExecTimeOut[%d]s is invalid, "
424 : "which should be a number greater than 0s and less "
425 : "than or equal to 17340s",
426 : execTimeOut);
427 0 : return HCCL_SUCCESS;
428 : }
429 :
430 0 : s32 intPart = execTimeOut / HCCL_INTEVAL_EXEC_TIME_OUT_S;
431 0 : intPart = (intPart == 0) ? 1 : intPart;
432 0 : execTimeOut = intPart * HCCL_INTEVAL_EXEC_TIME_OUT_S;
433 : }
434 :
435 0 : double timeout = execTimeOut;
436 0 : execTimeOut_ = static_cast<s32>(std::ceil(timeout));
437 0 : execTimeOutSetByConfig_ = true;
438 0 : HCCL_INFO("[SetConfigByVersion] HCCL_EXEC_TIMEOUT set by config to [%d]s", execTimeOut);
439 : }
440 :
441 11 : return HCCL_SUCCESS;
442 : }
443 :
444 10 : HcclResult CommConfig::SetConfigHcclAlgo(const CommConfigHandle& config)
445 : {
446 10 : if (config.hcclAlgo[0] == '\0') {
447 10 : return HCCL_SUCCESS;
448 : }
449 :
450 0 : auto hcclAlgoLength = strlen(config.hcclAlgo);
451 0 : hcclAlgoLength = hcclAlgoLength < COMM_ALGO_MAX_LENGTH ? hcclAlgoLength : COMM_ALGO_MAX_LENGTH;
452 0 : std::string algoConfig = std::string(config.hcclAlgo, hcclAlgoLength);
453 :
454 0 : algoConfig.erase(std::remove(algoConfig.begin(), algoConfig.end(), ' '), algoConfig.end());
455 0 : if (algoConfig.empty()) {
456 0 : HCCL_WARNING("[SetConfigHcclAlgo]hccl algo config is empty, HCCL use externalinput algo selection.");
457 0 : return HCCL_SUCCESS;
458 : }
459 0 : std::vector<std::string> algoPerOptype;
460 0 : CHK_RET(SplitHcclOpType(algoConfig, algoPerOptype));
461 0 : bool anyCommonConfig = false;
462 0 : bool anySpecificConfig = false;
463 0 : CHK_RET(CheckAlgoConfigValid(algoPerOptype, anyCommonConfig, anySpecificConfig));
464 0 : if (anyCommonConfig) {
465 0 : std::vector<HcclAlgoType> algType;
466 0 : CHK_RET(ParseAlgoString("all op type", algoPerOptype[0], algType));
467 0 : for (u32 opType = 0; opType < static_cast<u32>(HcclCMDType::HCCL_CMD_MAX); opType++) {
468 0 : algoConfig_[static_cast<HcclCMDType>(opType)] = algType;
469 : }
470 0 : } else {
471 0 : CHK_RET(SetSpecificAlgTypeConfig(algoPerOptype));
472 : }
473 0 : HCCL_RUN_INFO("HCCL_ALGO set by HcclCommConfig to [%s]", algoConfig.c_str());
474 :
475 0 : return HCCL_SUCCESS;
476 0 : }
477 :
478 10 : HcclResult CommConfig::SetConfigHcclRetryEnable(const CommConfigHandle& config)
479 : {
480 10 : if (config.hcclRetryEnable[0] == '\0') {
481 10 : return HCCL_SUCCESS;
482 : }
483 0 : auto retryEnableLength = strlen(config.hcclRetryEnable);
484 : retryEnableLength
485 0 : = retryEnableLength < COMM_RETRY_ENABLE_MAX_LENGTH ? retryEnableLength : COMM_RETRY_ENABLE_MAX_LENGTH;
486 0 : std::string retryConfig = std::string(config.hcclRetryEnable, retryEnableLength);
487 : // 去除空格
488 0 : retryConfig.erase(std::remove(retryConfig.begin(), retryConfig.end(), ' '), retryConfig.end());
489 0 : if (retryConfig.empty()) {
490 0 : HCCL_WARNING(
491 : "[%s] Hccl retry config is empty. The retryEnable of all levels is"
492 : "set by environment variable.",
493 : __func__);
494 0 : return HCCL_SUCCESS;
495 : }
496 0 : std::vector<std::string> retryEnables;
497 0 : HcclResult ret = SplitRetryEnable(retryConfig, retryEnables);
498 0 : CHK_PRT_RET(
499 : ret != HCCL_SUCCESS,
500 : HCCL_WARNING(
501 : "[CommConfig][SetConfigHcclRetryEnable] Hccl retry config[%s] is invalid. "
502 : "expect: L1:0, L2:0",
503 : retryConfig.c_str()),
504 : ret);
505 0 : CHK_RET(SetConfigRetryEnable(retryEnables));
506 0 : HCCL_RUN_INFO("HCCL_OP_RETRY_ENABLE set by commconfig to [%s].", retryConfig.c_str());
507 0 : return HCCL_SUCCESS;
508 0 : }
509 :
510 0 : HcclResult CommConfig::SplitRetryEnable(const std::string& retryConfig, std::vector<std::string>& retryEnables)
511 : {
512 0 : std::string remainRetryConfig;
513 0 : std::size_t found = retryConfig.find(",");
514 0 : if ((found == 0) || (found == (retryConfig.length() - 1))) {
515 0 : HCCL_ERROR("[SplitRetryEnable] retry config is invalid.");
516 0 : return HCCL_E_PARA;
517 0 : } else if (found != std::string::npos) {
518 0 : remainRetryConfig = retryConfig.substr(found + 1);
519 : } else {
520 : // 最后一组配置,剩余的字符串为空
521 0 : remainRetryConfig = "";
522 : }
523 0 : retryEnables.push_back(retryConfig.substr(0, found));
524 :
525 0 : if (retryEnables.size() > HCCL_RETRY_ENABLE_LEVEL_NUM) {
526 0 : HCCL_ERROR(
527 : "[SplitRetryEnable] retryEnable config is invalid. retryEnable level is more than %u.",
528 : HCCL_RETRY_ENABLE_LEVEL_NUM);
529 0 : return HCCL_E_PARA;
530 : }
531 0 : if (!remainRetryConfig.empty()) {
532 0 : CHK_RET(SplitRetryEnable(remainRetryConfig, retryEnables));
533 : }
534 0 : return HCCL_SUCCESS;
535 0 : }
536 :
537 0 : HcclResult CommConfig::SetConfigRetryEnable(const std::vector<std::string>& retryEnables)
538 : {
539 : const std::map<std::string, u32> hcclRetryLevelMap
540 0 : = {{"L0", HCCL_RETRY_ENABLE_LEVEL_0}, {"L1", HCCL_RETRY_ENABLE_LEVEL_1}, {"L2", HCCL_RETRY_ENABLE_LEVEL_2}};
541 :
542 0 : std::map<std::string, u32> countHcclRetryLevelMap = {{"L0", 0}, {"L1", 0}, {"L2", 0}};
543 :
544 0 : const std::map<std::string, bool> hcclRetryEnableMap = {{"0", false}, {"1", true}};
545 0 : for (const auto& retryEnableLevel : retryEnables) {
546 0 : u32 level = 0;
547 0 : bool retryEnable = false;
548 0 : std::size_t found = retryEnableLevel.find(":");
549 0 : if ((found == 0) || (found == (retryEnableLevel.length() - 1))) {
550 0 : HCCL_INFO("[SetRetryEnable] Hccl retryEnableLevel is invalid.");
551 0 : return HCCL_SUCCESS;
552 : }
553 0 : std::string orginalLevel = retryEnableLevel.substr(0, found);
554 0 : std::string orginalRetryEnable = retryEnableLevel.substr(found + 1);
555 0 : if (orginalLevel == "L0") {
556 0 : HCCL_RUN_WARNING("[SetConfigRetryEnable] L0 config does not take effect");
557 : }
558 : // 检查是否存在重复配置level
559 0 : auto iterCountRetryLevel = countHcclRetryLevelMap.find(orginalLevel);
560 0 : if (iterCountRetryLevel == countHcclRetryLevelMap.end()) {
561 0 : HCCL_RUN_WARNING(
562 : "[SetRetryEnable] Retry config is invalid, level %s is not supported.", orginalLevel.c_str());
563 0 : return HCCL_SUCCESS;
564 : }
565 0 : if (countHcclRetryLevelMap[orginalLevel] == 1) {
566 0 : HCCL_RUN_WARNING(
567 : "[SetRetryEnable] Retry config level[%s] is repeated, expect: L1:0, L2:0", orginalLevel.c_str());
568 0 : return HCCL_SUCCESS;
569 : }
570 0 : countHcclRetryLevelMap[orginalLevel] += 1;
571 : // 获取level和对应的retryEnable，并赋值给g_externalInput.hcclRetryConfig
572 0 : auto iterRetryLevel = hcclRetryLevelMap.find(orginalLevel);
573 0 : if (iterRetryLevel == hcclRetryLevelMap.end()) {
574 0 : HCCL_RUN_WARNING(
575 : "[SetRetryEnable] Retry config is invalid, level %s is not supported.", orginalLevel.c_str());
576 0 : return HCCL_SUCCESS;
577 : }
578 0 : auto iterRetryEnable = hcclRetryEnableMap.find(orginalRetryEnable);
579 0 : if (iterRetryEnable == hcclRetryEnableMap.end()) {
580 0 : HCCL_RUN_WARNING(
581 : "[SetRetryEnable] Retry config is invalid, retryEnable %s is not supported.",
582 : orginalRetryEnable.c_str());
583 0 : return HCCL_SUCCESS;
584 : }
585 0 : level = iterRetryLevel->second;
586 0 : retryEnable = iterRetryEnable->second;
587 0 : retryEnable_[level] = retryEnable;
588 0 : }
589 0 : return HCCL_SUCCESS;
590 0 : }
591 :
592 10 : HcclResult CommConfig::SetConfigHcclRetryParams(const CommConfigHandle& config)
593 : {
594 10 : if (config.hcclRetryParams[0] == '\0') {
595 10 : return HCCL_SUCCESS;
596 : }
597 0 : auto retryParamsLength = strlen(config.hcclRetryParams);
598 : retryParamsLength
599 0 : = retryParamsLength < COMM_RETRY_PARAMS_MAX_LENGTH ? retryParamsLength : COMM_RETRY_PARAMS_MAX_LENGTH;
600 0 : std::string retryParams = std::string(config.hcclRetryParams, retryParamsLength);
601 0 : u32 maxcnt = 0;
602 0 : u32 holdtime = 0;
603 0 : u32 intervaltime = 0;
604 0 : int ret = 0;
605 0 : ret = sscanf_s(retryParams.c_str(), "MaxCnt:%u, HoldTime:%u, IntervalTime:%u", &maxcnt, &holdtime, &intervaltime);
606 : /* 三个参数全部解析成功，返回值为3，否则不等于3 */
607 0 : if ((ret != 3) || (maxcnt > HCCL_RETRY_MAXCNT_MAX) || (maxcnt < HCCL_RETRY_MAXCNT_MIN)
608 0 : || (holdtime > HCCL_RETRY_HLOD_TIME_MAX) || (intervaltime > HCCL_RETRY_INTERVAL_MAX)) {
609 0 : HCCL_ERROR(
610 : "[SetConfigHcclRetryParams]fail, HCCL_OP_RETRY_PARAMS: %s is invalid, format must be: "
611 : "MaxCnt:cnt, HoldTime:time, IntervalTime:time, cnt range is [1, 10], time range is [0, 60000]ms.",
612 : retryParams.c_str());
613 0 : return HCCL_E_PARA;
614 : }
615 0 : retryMaxCnt_ = maxcnt;
616 0 : retryHoldTime_ = holdtime;
617 0 : retryIntervalTime_ = intervaltime;
618 :
619 0 : HCCL_RUN_INFO(
620 : "[SetConfigHcclRetryParams]HCCL_OP_RETRY_PARAMS is set, "
621 : "MaxCnt is [%u], HoldTime is [%u]ms, IntervalTime is [%u]ms.",
622 : maxcnt, holdtime, intervaltime);
623 0 : return HCCL_SUCCESS;
624 0 : }
625 :
626 0 : HcclResult CommConfig::SetSpecificAlgTypeConfig(std::vector<std::string>& algos)
627 : {
628 0 : for (std::string& algConfig : algos) {
629 0 : std::size_t found = algConfig.find("=");
630 0 : std::string opStringName = algConfig.substr(0, found);
631 0 : if (opStringName == "others") {
632 0 : std::vector<HcclAlgoType> algType;
633 0 : std::string remainAlgoConfig = algConfig.substr(found + 1);
634 0 : CHK_RET(ParseAlgoString("others op type", remainAlgoConfig, algType));
635 0 : for (u32 opType = 0; opType < static_cast<u32>(HcclCMDType::HCCL_CMD_MAX); opType++) {
636 0 : algoConfig_[static_cast<HcclCMDType>(opType)] = algType;
637 : }
638 0 : }
639 0 : }
640 : std::map<std::string, HcclCMDType> hcclOpTypeMap = {
641 0 : {"broadcast", HcclCMDType::HCCL_CMD_BROADCAST},
642 0 : {"allreduce", HcclCMDType::HCCL_CMD_ALLREDUCE},
643 0 : {"reduce", HcclCMDType::HCCL_CMD_REDUCE},
644 0 : {"send", HcclCMDType::HCCL_CMD_SEND},
645 0 : {"receive", HcclCMDType::HCCL_CMD_RECEIVE},
646 0 : {"allgather", HcclCMDType::HCCL_CMD_ALLGATHER},
647 0 : {"reducescatter", HcclCMDType::HCCL_CMD_REDUCE_SCATTER},
648 0 : {"alltoall", HcclCMDType::HCCL_CMD_ALLTOALL},
649 0 : {"gather", HcclCMDType::HCCL_CMD_GATHER},
650 0 : {"scatter", HcclCMDType::HCCL_CMD_SCATTER},
651 0 : {"sendrecv", HcclCMDType::HCCL_CMD_BATCH_SEND_RECV},
652 0 : };
653 0 : for (std::string& algConfig : algos) {
654 0 : std::size_t found = algConfig.find("=");
655 0 : std::string opStringName = algConfig.substr(0, found);
656 0 : if (hcclOpTypeMap.find(opStringName) != hcclOpTypeMap.end()) {
657 0 : HcclCMDType optype = hcclOpTypeMap[opStringName];
658 0 : std::string remainAlgoConfig = algConfig.substr(found + 1);
659 0 : std::vector<HcclAlgoType> algType;
660 0 : CHK_RET(ParseAlgoString(opStringName, remainAlgoConfig, algType));
661 0 : if (algType[0] == HcclAlgoType::HCCL_ALGO_TYPE_NULL) {
662 0 : HCCL_WARNING("[SetSpecificAlgType] specific config level0 not support null type.");
663 0 : return HCCL_SUCCESS;
664 : }
665 0 : algoConfig_[optype] = algType;
666 0 : } else {
667 0 : HCCL_WARNING(
668 : "[SetSpecificAlgType] specific config optype[%s] is invalid, please check", opStringName.c_str());
669 0 : return HCCL_SUCCESS;
670 : }
671 0 : }
672 0 : algoConfig_[HcclCMDType::HCCL_CMD_ALLTOALLV] = algoConfig_[HcclCMDType::HCCL_CMD_ALLTOALL];
673 0 : algoConfig_[HcclCMDType::HCCL_CMD_ALLTOALLVC] = algoConfig_[HcclCMDType::HCCL_CMD_ALLTOALL];
674 0 : return HCCL_SUCCESS;
675 0 : }
676 :
677 176 : HcclResult CommConfig::SetConfigTrafficClass(u32 trafficClass)
678 : {
679 176 : trafficClass_ = trafficClass;
680 176 : return HCCL_SUCCESS;
681 : }
682 :
683 173 : HcclResult CommConfig::SetConfigServiceLevel(u32 serviceLevel)
684 : {
685 173 : serviceLevel_ = serviceLevel;
686 173 : return HCCL_SUCCESS;
687 : }
688 :
689 184 : HcclResult CommConfig::SetConfigHcclQos(u32 hcclQos)
690 : {
691 184 : hcclQos_ = hcclQos;
692 184 : return HCCL_SUCCESS;
693 : }
694 :
695 3 : HcclResult CommConfig::SetConfigHcclAlgoStr(const std::string& hcclAlgo)
696 : {
697 3 : hcclAlgoStr_ = hcclAlgo;
698 3 : return HCCL_SUCCESS;
699 : }
700 :
701 1 : const std::string& CommConfig::GetConfigHcclAlgoStr() const { return hcclAlgoStr_; }
702 :
703 179 : HcclResult CommConfig::SetConfigSqDepth(u32 sqDepth)
704 : {
705 179 : sqDepth_ = sqDepth;
706 179 : return HCCL_SUCCESS;
707 : }
708 :
709 0 : HcclResult CommConfig::SetConfigExecTimeOut(s32 execTimeOut)
710 : {
711 0 : execTimeOut_ = execTimeOut;
712 0 : return HCCL_SUCCESS;
713 : }
714 :
715 878 : u64 CommConfig::GetConfigBufferSize() const { return bufferSize_; }
716 :
717 243 : u8 CommConfig::GetConfigDeterministic() const { return deterministic_; }
718 :
719 735 : const std::string& CommConfig::GetConfigCommName() const { return commName_; }
720 :
721 5 : const std::string& CommConfig::GetConfigUdi() const { return udi_; }
722 :
723 769 : bool CommConfig::GetConfigAivMode() const { return aivMode_; }
724 :
725 766 : bool CommConfig::GetConfigIsOnlyAivMode() const { return onlyAivMode_; }
726 :
727 2217 : bool CommConfig::GetConfigAicpuUnfold() const { return aicpuUnfold_; }
728 :
729 257 : u32 CommConfig::GetConfigTrafficClass() const { return trafficClass_; }
730 :
731 256 : u32 CommConfig::GetConfigServiceLevel() const { return serviceLevel_; }
732 :
733 0 : u32 CommConfig::GetConfigWorldRankID() const { return worldRankID_; }
734 :
735 235 : u64 CommConfig::GetConfigJobID() const { return jobID_; }
736 :
737 78 : u8 CommConfig::GetConfigAclGraphZeroCopyEnable() const { return aclGraphZeroCopyEnable_; }
738 :
739 1287 : s32 CommConfig::GetConfigExecTimeOut() const { return execTimeOut_; }
740 :
741 1 : bool CommConfig::GetConfigExecTimeOutSet() const { return execTimeOutSetByConfig_; }
742 :
743 150 : std::vector<HcclAlgoType> CommConfig::GetConfigHcclAlgo(HcclCMDType opType) { return algoConfig_[opType]; }
744 :
745 769 : const std::map<HcclCMDType, std::vector<HcclAlgoType>>& CommConfig::GetConfigHcclAlgoMap() const { return algoConfig_; }
746 :
747 0 : bool CommConfig::GetConfigIntraServerRetryEnable() const { return retryEnable_[HCCL_RETRY_ENABLE_LEVEL_0]; }
748 :
749 532 : bool CommConfig::GetConfigInterServerRetryEnable() const { return retryEnable_[HCCL_RETRY_ENABLE_LEVEL_1]; }
750 :
751 767 : bool CommConfig::GetConfigInterSuperPodRetryEnable() const { return retryEnable_[HCCL_RETRY_ENABLE_LEVEL_2]; }
752 :
753 0 : u32 CommConfig::GetConfigRetryMaxCnt() const { return retryMaxCnt_; }
754 :
755 2 : u32 CommConfig::GetConfigRetryHoldTime() const { return retryHoldTime_; }
756 :
757 2 : u32 CommConfig::GetConfigRetryIntervalTime() const { return retryIntervalTime_; }
758 :
759 236 : const std::string& CommConfig::GetConfigBufferName() const { return bufferName_; }
760 :
761 260 : u32 CommConfig::GetConfigHcclQos() const
762 : {
763 260 : HCCL_INFO("[GetConfigHcclQos] hcclQos = %u", hcclQos_);
764 260 : return hcclQos_;
765 : }
766 :
767 28 : u64 CommConfig::GetConfigSymmetricMemoryStride() const { return symmetricMemoryStride_; }
768 :
769 22 : u32 CommConfig::GetConfigSqDepth() const { return sqDepth_; }
770 : } // namespace hccl
|
