Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "externalinput_pub.h"
12 : #include "externalinput.h"
13 : #include "comm_config_pub.h"
14 : #include "adapter_error_manager_pub.h"
15 : #include "adapter_rts_common.h"
16 : #include "alg_env_config.h"
17 :
18 : namespace hccl {
19 634 : CommConfig::CommConfig(const std::string &commName)
20 634 : : bufferSize_(GetExternalInputCCLBuffSize()),
21 637 : deterministic_(GetExternalInputHcclDeterministicV2()),
22 636 : commName_(commName),
23 638 : aivMode_(GetExternalInputHcclAivMode()),
24 638 : aicpuUnfold_(GetExternalInputHcclAicpuUnfold()),
25 635 : trafficClass_(HCCL_COMM_TRAFFIC_CLASS_CONFIG_NOT_SET),
26 635 : serviceLevel_(HCCL_COMM_SERVICE_LEVEL_CONFIG_NOT_SET),
27 635 : worldRankID_(0),
28 635 : jobID_(0),
29 635 : aclGraphZeroCopyEnable_(0),
30 635 : onlyAivMode_(false),
31 635 : execTimeOut_(GetInternalExecTimeOut()),
32 636 : execTimeOutSetByConfig_(false),
33 637 : retryMaxCnt_(GetExternalInputRetryMaxCnt()),
34 637 : retryHoldTime_(GetExternalInputRetryHoldTime()),
35 637 : retryIntervalTime_(GetExternalInputRetryIntervalTime()),
36 1270 : bufferName_(""),
37 635 : hcclQos_(HCCL_COMM_QOS_CONFIG_NOT_SET),
38 1274 : symmetricMemoryStride_(HCCL_DEFAULT_SYMMETRIC_MEMORY_STRIDE)
39 : {
40 635 : InitAlgoConfig();
41 638 : InitRetryEnable();
42 638 : }
43 :
44 2012 : CommConfig::CommConfig()
45 2012 : : bufferSize_(GetExternalInputCCLBuffSize()),
46 2013 : deterministic_(GetExternalInputHcclDeterministicV2()),
47 2013 : aivMode_(GetExternalInputHcclAivMode()),
48 2013 : aicpuUnfold_(GetExternalInputHcclAicpuUnfold()),
49 2013 : trafficClass_(HCCL_COMM_TRAFFIC_CLASS_CONFIG_NOT_SET),
50 2013 : serviceLevel_(HCCL_COMM_SERVICE_LEVEL_CONFIG_NOT_SET),
51 2013 : worldRankID_(0),
52 2013 : jobID_(0),
53 2013 : aclGraphZeroCopyEnable_(0),
54 2013 : onlyAivMode_(false),
55 2013 : execTimeOut_(GetInternalExecTimeOut()),
56 2011 : execTimeOutSetByConfig_(false),
57 2012 : retryMaxCnt_(GetExternalInputRetryMaxCnt()),
58 2013 : retryHoldTime_(GetExternalInputRetryHoldTime()),
59 2011 : retryIntervalTime_(GetExternalInputRetryIntervalTime()),
60 4022 : bufferName_(""),
61 2013 : hcclQos_(HCCL_COMM_QOS_CONFIG_NOT_SET),
62 4024 : symmetricMemoryStride_(HCCL_DEFAULT_SYMMETRIC_MEMORY_STRIDE)
63 : {
64 2013 : InitAlgoConfig();
65 2013 : InitRetryEnable();
66 2013 : }
67 :
68 3244 : CommConfig::~CommConfig() {}
69 :
70 2648 : void CommConfig::InitAlgoConfig()
71 : {
72 278089 : for (u32 opType = 0; opType < static_cast<u32>(HcclCMDType::HCCL_CMD_MAX); opType++) {
73 275439 : algoConfig_[static_cast<HcclCMDType>(opType)] = GetExternalInputHcclAlgoConfig(static_cast<HcclCMDType>(opType));
74 : }
75 2650 : }
76 :
77 2651 : void CommConfig::InitRetryEnable()
78 : {
79 2651 : retryEnable_[HCCL_RETRY_ENABLE_LEVEL_0] = GetExternalInputIntraServerRetryEnable();
80 2651 : retryEnable_[HCCL_RETRY_ENABLE_LEVEL_1] = GetExternalInputInterServerRetryEnable();
81 2651 : retryEnable_[HCCL_RETRY_ENABLE_LEVEL_2] = GetExternalInputInterSuperPodRetryEnable();
82 2651 : }
83 :
84 11 : HcclResult CommConfig::Load(const HcclCommConfig *userConfig)
85 : {
86 : // 检查是否为空
87 11 : CHK_PTR_NULL(userConfig);
88 :
89 : // 读取结构体的size
90 10 : size_t configSize = *(reinterpret_cast<const size_t *>(userConfig));
91 10 : HCCL_INFO("[Load] config size[%llu]", configSize);
92 :
93 10 : const size_t maxConfigSize = sizeof(CommConfigHandle);
94 10 : if (configSize > maxConfigSize) {
95 0 : HCCL_WARNING("[Load] configSize[%llu] is larger than sizeof(CommConfigHandle)[%llu]",
96 : configSize, maxConfigSize);
97 0 : configSize = maxConfigSize;
98 10 : } else if (configSize < maxConfigSize) {
99 0 : HCCL_WARNING("[Load] configSize[%llu] is less than sizeof(CommConfigHandle)[%llu]",
100 : configSize, maxConfigSize);
101 : }
102 :
103 : // 根据size读取结构体
104 10 : CommConfigHandle configHandle;
105 10 : s32 sRet = memcpy_s(&configHandle, maxConfigSize, userConfig, configSize);
106 10 : CHK_PRT_RET(sRet != EOK,
107 : HCCL_ERROR("[Load] memcpy comm config fail. errorno[%d] "
108 : "params:destMaxSize[%u], count[%u]",
109 : sRet, maxConfigSize, configSize),
110 : HCCL_E_MEMORY);
111 :
112 : // 检查Magic word是否合法
113 10 : CHK_RET(CheckMagicWord(configHandle));
114 :
115 : // 根据版本号读取配置，检查配置参数合法性
116 10 : CHK_RET(SetConfigByVersion(configHandle));
117 :
118 9 : HCCL_RUN_INFO("[Load] comm config info of [%s]: configSize[%llu], version[%u], opExpansionMode[%u]", commName_.c_str(),
119 : configHandle.info.configSize, configHandle.info.version, configHandle.opExpansionMode);
120 9 : HCCL_RUN_INFO("[Load] comm config of [%s]: bufferSize[%llu], deterministic[%u], trafficClass[%u], serviceLevel[%u]"
121 : ", execTimeOut[%u]s, bufferName[%s], hcclQos[%u], symmetricMemoryStride[%llu], aclGraphZeroCopyEnable[%u]",
122 : commName_.c_str(), bufferSize_, deterministic_, trafficClass_, serviceLevel_, execTimeOut_, bufferName_.c_str(), hcclQos_, symmetricMemoryStride_, aclGraphZeroCopyEnable_);
123 9 : return HCCL_SUCCESS;
124 : }
125 :
126 10 : HcclResult CommConfig::CheckMagicWord(const CommConfigHandle &config)
127 : {
128 10 : if (config.info.magicWord != COMM_CONFIG_MAGIC_WORD) {
129 0 : RPT_INPUT_ERR(true,
130 : "EI0003",
131 : std::vector<std::string>({"ccl_op", "value", "parameter", "expect"}),
132 : std::vector<std::string>({"HcclCommInitRootInfoConfig",
133 : std::to_string(config.info.magicWord),
134 : "magic word",
135 : "The magic word must be initialized with the result of HcclCommConfigInit()"}));
136 0 : HCCL_ERROR("[CheckMagicWord] Invalid magic word[0x%x]. Please make sure the config has been initialized by "
137 : "HcclCommConfigInit().",
138 : config.info.magicWord);
139 0 : return HCCL_E_PARA;
140 : }
141 :
142 10 : return HCCL_SUCCESS;
143 0 : }
144 :
145 25 : HcclResult CommConfig::SetConfigByVersion(const CommConfigHandle &config)
146 : {
147 25 : if (config.info.version > CommConfigVersion::COMM_CONFIG_VERSION_TEN) {
148 : // 传入的config的版本高于当前版本，警告不支持的配置项将被忽略
149 0 : HCCL_WARNING("[SetConfigByVersion] The version of provided config[%u] is higher than the current version[%u], "
150 : "unsupported configuration will be ignored.",
151 : config.info.version,
152 : CommConfigVersion::COMM_CONFIG_VERSION_TEN);
153 25 : } else if (config.info.version < CommConfigVersion::COMM_CONFIG_VERSION_TEN) {
154 : // 传入的config的版本低于当前版本，警告高版本支持的配置项将被忽略
155 15 : HCCL_WARNING("[SetConfigByVersion] The version of provided config[%u] is lower than the current version[%u], "
156 : "configurations supported by later versions will be ignored.",
157 : config.info.version,
158 : CommConfigVersion::COMM_CONFIG_VERSION_TEN);
159 : }
160 :
161 25 : if (config.info.version >= CommConfigVersion::COMM_CONFIG_VERSION_ONE) {
162 : // 版本大于等于1，设置CCL buffer、确定性计算配置
163 24 : CHK_RET(SetConfigBufferSize(config));
164 23 : CHK_RET(SetConfigDeterministic(config));
165 : }
166 :
167 24 : if (config.info.version >= CommConfigVersion::COMM_CONFIG_VERSION_TWO) {
168 : // 版本大于等于2，设置通信域名称
169 20 : CHK_RET(SetConfigCommName(config));
170 : }
171 :
172 24 : if (config.info.version >= CommConfigVersion::COMM_CONFIG_VERSION_THREE) {
173 : // 版本大于等于3，设置Udi
174 20 : CHK_RET(SetConfigUdi(config));
175 : }
176 :
177 24 : if (config.info.version >= CommConfigVersion::COMM_CONFIG_VERSION_FOUR) {
178 : // 版本大于等于4，设置Aiv、Aicpu
179 20 : CHK_RET(SetConfigOpExpansionMode(config));
180 : }
181 :
182 24 : if (config.info.version >= CommConfigVersion::COMM_CONFIG_VERSION_FIVE) {
183 : // 版本大于等于5，支持配置TC，SL
184 9 : trafficClass_ = config.trafficClass;
185 9 : serviceLevel_ = config.serviceLevel;
186 : }
187 :
188 24 : if (config.info.version >= CommConfigVersion::COMM_CONFIG_VERSION_SIX) {
189 : // 版本大于等于6
190 9 : worldRankID_ = config.worldRankID;
191 9 : jobID_ = config.jobID;
192 : }
193 :
194 24 : if (config.info.version >= CommConfigVersion::COMM_CONFIG_VERSION_SEVEN) {
195 : // 版本大于等于7，支持配置AclGraph使能/去使能
196 9 : RPT_INPUT_ERR(config.aclGraphZeroCopyEnable > 1,
197 : "EI0003",
198 : std::vector<std::string>({"ccl_op", "value", "parameter", "expect"}),
199 : std::vector<std::string>({"HcclCommInitRootInfoConfig",
200 : std::to_string(config.aclGraphZeroCopyEnable),
201 : "aclGraphZeroCopy",
202 : "0 or 1."}));
203 9 : CHK_PRT_RET(config.aclGraphZeroCopyEnable > 1,
204 : HCCL_ERROR("[CommConfig][SetConfigByVersion] aclGraphZeroCopyEnable value=[%u] invalid. support 0 or 1", config.aclGraphZeroCopyEnable),
205 : HCCL_E_PARA);
206 9 : aclGraphZeroCopyEnable_ = config.aclGraphZeroCopyEnable;
207 : }
208 :
209 24 : if (config.info.version >= CommConfigVersion::COMM_CONFIG_VERSION_EIGHT) {
210 : // 版本大于等于8，支持配置execTimeOut
211 9 : CHK_RET(SetConfigExecTimeout(config));
212 : // 支持配置HcclAlgo
213 9 : CHK_RET(SetConfigHcclAlgo(config));
214 : // 解析重执行设置
215 9 : CHK_RET(SetConfigHcclRetryEnable(config));
216 9 : CHK_RET(SetConfigHcclRetryParams(config));
217 : }
218 :
219 24 : if (config.info.version >= CommConfigVersion::COMM_CONFIG_VERSION_NINE) {
220 : // 版本大于等于9
221 9 : CHK_RET(SetConfigBufferName(config));
222 : }
223 :
224 24 : if (config.info.version >= CommConfigVersion::COMM_CONFIG_VERSION_TEN) {
225 : // 版本大于等于10,支持配置通信域级别的AI CPU SDMA QOS
226 9 : hcclQos_ = config.hcclQos;
227 : // 版本大于等于10，支持配置对称内存每个rank的预留VA大小
228 9 : symmetricMemoryStride_ = config.symmetricMemoryStride;
229 : }
230 24 : HCCL_INFO("NSLBDP-VERSION config.info.version = [%u] .", config.info.version);
231 24 : return HCCL_SUCCESS;
232 0 : }
233 :
234 24 : HcclResult CommConfig::SetConfigBufferSize(const CommConfigHandle &config)
235 : {
236 24 : if (config.bufferSize == HCCL_COMM_BUFFSIZE_CONFIG_NOT_SET) {
237 : // 默认跟随环境变量配置
238 3 : HCCL_INFO("[SetConfigByVersion] The hcclBufferSize is not configured, use the env config [%u](Bytes) as default.",
239 : bufferSize_);
240 21 : } else if (config.bufferSize < HCCL_CCL_COMM_BUFFER_MIN) {
241 17 : RPT_INPUT_ERR(true,
242 : "EI0003",
243 : std::vector<std::string>({"ccl_op", "value", "parameter", "expect"}),
244 : std::vector<std::string>({"HcclCommInitRootInfoConfig",
245 : std::to_string(config.bufferSize),
246 : "hcclBufferSize",
247 : "should be equal to or greater than 1(MB)."}));
248 1 : HCCL_ERROR("[%s][%s] The configuration of hcclBufferSize[%u(MB)] is invalid, which should be "
249 : "greater than %u(MB).",
250 : LOG_KEYWORDS_TASK_EXEC.c_str(),
251 : LOG_KEYWORDS_INVALID_ARGUMENT.c_str(),
252 : config.bufferSize,
253 : HCCL_CCL_COMM_BUFFER_MIN);
254 1 : return HCCL_E_PARA;
255 : } else {
256 : // 使用config配置
257 20 : bufferSize_ = static_cast<u64>(config.bufferSize) * HCCL_CCL_COMM_FIXED_CALC_BUFFER_SIZE; // MByte 转 Byte
258 : }
259 23 : return HCCL_SUCCESS;
260 2 : }
261 :
262 23 : HcclResult CommConfig::SetConfigDeterministic(const CommConfigHandle &config)
263 : {
264 23 : if (config.deterministic == HCCL_COMM_DETERMINISTIC_CONFIG_NOT_SET) {
265 : // 默认跟随环境变量配置
266 9 : HCCL_INFO("[SetConfigByVersion] The hcclDeterministic is not configured, use the env config [%u] as default.",
267 : deterministic_);
268 14 : } else if (config.deterministic > DETERMINISTIC_STRICT) {
269 0 : RPT_INPUT_ERR(true,
270 : "EI0003",
271 : std::vector<std::string>({"ccl_op", "value", "parameter", "expect"}),
272 : std::vector<std::string>({"HcclCommInitRootInfoConfig",
273 : std::to_string(config.deterministic),
274 : "hcclDeterministic",
275 : "should be 0(disable) , 1(enable) or 2(strict)."}));
276 0 : HCCL_ERROR("[%s][%s] The configuration of hcclDeterministic[%u] is invalid, "
277 : "which should be 0(disable) , 1(enable) or 2(strict).",
278 : LOG_KEYWORDS_TASK_EXEC.c_str(),
279 : LOG_KEYWORDS_INVALID_ARGUMENT.c_str(),
280 : config.deterministic);
281 0 : return HCCL_E_PARA;
282 : } else {
283 14 : if (config.deterministic == DETERMINISTIC_STRICT) {
284 : DevType deviceType;
285 1 : CHK_RET(hrtGetDeviceType(deviceType));
286 1 : if (deviceType != DevType::DEV_TYPE_910B) {
287 0 : RPT_INPUT_ERR(true,
288 : "EI0003",
289 : std::vector<std::string>({"ccl_op", "value", "parameter", "expect"}),
290 : std::vector<std::string>({"HcclCommInitRootInfoConfig",
291 : std::to_string(config.deterministic),
292 : "hcclDeterministic",
293 : "set to 2(strict), only support A2."}));
294 0 : HCCL_ERROR("[%s][%s] The configuration of hcclDeterministic[%u] is set to "
295 : "2(strict), and only support A2",
296 : LOG_KEYWORDS_TASK_EXEC.c_str(),
297 : LOG_KEYWORDS_INVALID_ARGUMENT.c_str(),
298 : config.deterministic);
299 0 : return HCCL_E_PARA;
300 : }
301 : }
302 14 : deterministic_ = static_cast<u8>(config.deterministic); // 前面已保证数值不超过UINT8_MAX，直接进行类型转换
303 : }
304 23 : return HCCL_SUCCESS;
305 0 : }
306 :
307 20 : HcclResult CommConfig::SetConfigCommName(const CommConfigHandle &config)
308 : {
309 20 : if (config.commName[0] != '\0') {
310 12 : auto commNameLength = strlen(config.commName);
311 12 : commNameLength = commNameLength < COMM_NAME_MAX_LENGTH? commNameLength : COMM_NAME_MAX_LENGTH;
312 24 : commName_ = std::string(config.commName, commNameLength);
313 : }
314 20 : return HCCL_SUCCESS;
315 : }
316 :
317 20 : HcclResult CommConfig::SetConfigUdi(const CommConfigHandle &config)
318 : {
319 20 : if (config.udi[0] == '\0') {
320 9 : udi_ = "Unspecified";
321 9 : return HCCL_SUCCESS;
322 : }
323 11 : auto udiLength = strlen(config.udi);
324 11 : udiLength = udiLength < COMM_NAME_MAX_LENGTH ? udiLength : COMM_NAME_MAX_LENGTH;
325 11 : udi_ = std::string(config.udi, udiLength);
326 11 : return HCCL_SUCCESS;
327 : }
328 :
329 9 : HcclResult CommConfig::SetConfigBufferName(const CommConfigHandle &config)
330 : {
331 9 : if (config.bufferName[0] != '\0') {
332 0 : auto bufferNameLength = strlen(config.bufferName);
333 0 : bufferNameLength = bufferNameLength < BUFFER_NAME_MAX_LENGTH ? bufferNameLength : BUFFER_NAME_MAX_LENGTH;
334 0 : bufferName_ = std::string(config.bufferName, bufferNameLength);
335 : }
336 9 : return HCCL_SUCCESS;
337 : }
338 :
339 22 : HcclResult CommConfig::SetConfigOpExpansionMode(const CommConfigHandle &config)
340 : {
341 22 : switch (config.opExpansionMode) {
342 10 : case COMM_CONFIG_OPEXPANSION_DEFAULT:
343 10 : HCCL_INFO("CommConfig is set to 0(default), aicpuUnfold_ is [%d] and aivMode_ is [%d].", aicpuUnfold_, aivMode_);
344 10 : break;
345 2 : case COMM_CONFIG_OPEXPANSION_HOST:
346 2 : aivMode_ = false;
347 2 : aicpuUnfold_ = false;
348 2 : HCCL_INFO("CommConfig is set to 1(host), aicpuUnfold_ is [%d] and aivMode_ is [%d].", aicpuUnfold_, aivMode_);
349 2 : break;
350 3 : case COMM_CONFIG_OPEXPANSION_AICPU:
351 : // 目前只有A3和300I支持Aicpu展开
352 : DevType deviceType;
353 3 : CHK_RET(hrtGetDeviceType(deviceType));
354 3 : if (deviceType == DevType::DEV_TYPE_910_93 || deviceType == DevType::DEV_TYPE_910B) {
355 3 : aicpuUnfold_ = true;
356 3 : aivMode_ = false;
357 3 : HCCL_INFO("CommConfig is set to 2(aicpuUnfold_), aicpuUnfold_ is [%d] and aivMode_ is [%d].", aicpuUnfold_, aivMode_);
358 : } else {
359 0 : HCCL_WARNING("Only A3 and 300I support aicpu unfold, set aicpuUnfold_ to [%d] and aivMode_ to [%d].", aicpuUnfold_, aivMode_);
360 : }
361 3 : break;
362 4 : case COMM_CONFIG_OPEXPANSION_AIV:
363 4 : aivMode_ = true;
364 4 : HCCL_INFO("CommConfig is set to 3(aivMode), aicpuUnfold_ is [%d] and aivMode_ is [%d].", aicpuUnfold_, aivMode_);
365 4 : break;
366 2 : case COMM_CONFIG_OPEXPANSION_ONLY_AIV:
367 2 : onlyAivMode_ = true;
368 2 : HCCL_INFO("CommConfig is set to 4(aivOnly), aicpuUnfold_ is [%d] and aivMode_ is [%d] onlyAivMode_ is[%d].", aicpuUnfold_, aivMode_, onlyAivMode_);
369 2 : break;
370 1 : default:
371 : // 目前opExpansionMode的合法值为[0,4]，值不合法时回退为环境变量配置
372 1 : HCCL_WARNING("Current version not support opExpansionMode[%u], set aicpuUnfold_ to [%d] and aivMode_ to [%d].", config.opExpansionMode, aicpuUnfold_, aivMode_);
373 1 : break;
374 : }
375 :
376 22 : return HCCL_SUCCESS;
377 : }
378 :
379 9 : HcclResult CommConfig::SetConfigExecTimeout(const CommConfigHandle &config)
380 : {
381 9 : if (config.execTimeOut == COMM_EXECTIMEOUT_CONFIG_NOT_SET) {
382 : // 默认跟随环境变量
383 9 : HCCL_INFO("[SetConfigByVersion] The hcclExecTimeOut is not configured, use the env config [%u] as default.", execTimeOut_);
384 : } else {
385 0 : s32 execTimeOut = config.execTimeOut;
386 :
387 : DevType deviceType;
388 0 : CHK_RET(hrtGetDeviceType(deviceType)); // 910A和910B/C要分开
389 0 : if (deviceType == DevType::DEV_TYPE_910_93 || deviceType == DevType::DEV_TYPE_910B) {
390 : // 910B和910_93算子超时时间范围0s-2147483647s,其中0代表永不超时
391 0 : if ((execTimeOut < 0) || (execTimeOut > HCCL_EXEC_TIME_OUT_S_910_93)) {
392 0 : HCCL_WARNING("[SetConfigByVersion][SetConfigExecTimeout] The configuration of ComConfigHcclExecTimeOut[%d]s is invalid, "\
393 : "which should be a number greater than or equal to 0s and less "\
394 : "than or equal to 2147483647s", execTimeOut);
395 0 : return HCCL_SUCCESS;
396 : }
397 : } else {
398 : // 非910B和910_93算子超时时间范围1s-17340s
399 0 : if ((execTimeOut <= 0) || (execTimeOut > HCCL_EXEC_TIME_OUT_S)) {
400 0 : HCCL_WARNING("[SetConfigByVersion] The configuration of ComConfigHcclExecTimeOut[%d]s is invalid, "\
401 : "which should be a number greater than 0s and less "\
402 : "than or equal to 17340s", execTimeOut);
403 0 : return HCCL_SUCCESS;
404 : }
405 :
406 0 : s32 intPart = execTimeOut / HCCL_INTEVAL_EXEC_TIME_OUT_S;
407 0 : intPart = (intPart == 0) ? 1 : intPart;
408 0 : execTimeOut = intPart * HCCL_INTEVAL_EXEC_TIME_OUT_S;
409 : }
410 :
411 0 : double timeout = execTimeOut;
412 0 : execTimeOut_ = static_cast<s32>(std::ceil(timeout));
413 0 : execTimeOutSetByConfig_ = true;
414 0 : HCCL_INFO("[SetConfigByVersion] HCCL_EXEC_TIMEOUT set by config to [%d]s", execTimeOut);
415 : }
416 :
417 9 : return HCCL_SUCCESS;
418 : }
419 :
420 9 : HcclResult CommConfig::SetConfigHcclAlgo(const CommConfigHandle &config)
421 : {
422 9 : if (config.hcclAlgo[0] == '\0') {
423 9 : return HCCL_SUCCESS;
424 : }
425 :
426 0 : auto hcclAlgoLength = strlen(config.hcclAlgo);
427 0 : hcclAlgoLength = hcclAlgoLength < COMM_ALGO_MAX_LENGTH ? hcclAlgoLength : COMM_ALGO_MAX_LENGTH;
428 0 : std::string algoConfig = std::string(config.hcclAlgo, hcclAlgoLength);
429 :
430 0 : algoConfig.erase(std::remove(algoConfig.begin(), algoConfig.end(), ' '), algoConfig.end());
431 0 : if (algoConfig.empty()) {
432 0 : HCCL_WARNING("[SetConfigHcclAlgo]hccl algo config is empty, HCCL use externalinput algo selection.");
433 0 : return HCCL_SUCCESS;
434 : }
435 0 : std::vector<std::string> algoPerOptype;
436 0 : CHK_RET(SplitHcclOpType(algoConfig, algoPerOptype));
437 0 : bool anyCommonConfig = false;
438 0 : bool anySpecificConfig = false;
439 0 : CHK_RET(CheckAlgoConfigValid(algoPerOptype, anyCommonConfig, anySpecificConfig));
440 0 : if (anyCommonConfig) {
441 0 : std::vector<HcclAlgoType> algType;
442 0 : CHK_RET(ParseAlgoString("all op type", algoPerOptype[0], algType));
443 0 : for (u32 opType = 0; opType < static_cast<u32>(HcclCMDType::HCCL_CMD_MAX); opType++) {
444 0 : algoConfig_[static_cast<HcclCMDType>(opType)] = algType;
445 : }
446 0 : } else {
447 0 : CHK_RET(SetSpecificAlgTypeConfig(algoPerOptype));
448 : }
449 0 : HCCL_RUN_INFO("HCCL_ALGO set by HcclCommConfig to [%s]", algoConfig.c_str());
450 :
451 0 : return HCCL_SUCCESS;
452 0 : }
453 :
454 9 : HcclResult CommConfig::SetConfigHcclRetryEnable(const CommConfigHandle &config)
455 : {
456 9 : if (config.hcclRetryEnable[0] == '\0') {
457 9 : return HCCL_SUCCESS;
458 : }
459 0 : auto retryEnableLength = strlen(config.hcclRetryEnable);
460 0 : retryEnableLength = retryEnableLength < COMM_RETRY_ENABLE_MAX_LENGTH ?
461 : retryEnableLength : COMM_RETRY_ENABLE_MAX_LENGTH;
462 0 : std::string retryConfig = std::string(config.hcclRetryEnable, retryEnableLength);
463 : // 去除空格
464 0 : retryConfig.erase(std::remove(retryConfig.begin(), retryConfig.end(), ' '), retryConfig.end());
465 0 : if (retryConfig.empty()) {
466 0 : HCCL_WARNING("[%s] Hccl retry config is empty. The retryEnable of all levels is" \
467 : "set by environment variable.", __func__);
468 0 : return HCCL_SUCCESS;
469 : }
470 0 : std::vector<std::string> retryEnables;
471 0 : HcclResult ret = SplitRetryEnable(retryConfig, retryEnables);
472 0 : CHK_PRT_RET(ret != HCCL_SUCCESS, HCCL_WARNING("[CommConfig][SetConfigHcclRetryEnable] Hccl retry config[%s] is invalid. "
473 : "expect: L1:0, L2:0", retryConfig.c_str()), ret);
474 0 : CHK_RET(SetConfigRetryEnable(retryEnables));
475 0 : HCCL_RUN_INFO("HCCL_OP_RETRY_ENABLE set by commconfig to [%s].", retryConfig.c_str());
476 0 : return HCCL_SUCCESS;
477 0 : }
478 :
479 0 : HcclResult CommConfig::SplitRetryEnable(const std::string &retryConfig, std::vector<std::string> &retryEnables)
480 : {
481 0 : std::string remainRetryConfig;
482 0 : std::size_t found = retryConfig.find(",");
483 0 : if ((found == 0) || (found == (retryConfig.length() - 1))) {
484 0 : HCCL_ERROR("[SplitRetryEnable] retry config is invalid.");
485 0 : return HCCL_E_PARA;
486 0 : } else if (found != std::string::npos) {
487 0 : remainRetryConfig = retryConfig.substr(found + 1);
488 : } else {
489 : // 最后一组配置,剩余的字符串为空
490 0 : remainRetryConfig = "";
491 : }
492 0 : retryEnables.push_back(retryConfig.substr(0, found));
493 :
494 0 : if (retryEnables.size() > HCCL_RETRY_ENABLE_LEVEL_NUM) {
495 0 : HCCL_ERROR("[SplitRetryEnable] retryEnable config is invalid. retryEnable level is more than %u.",
496 : HCCL_RETRY_ENABLE_LEVEL_NUM);
497 0 : return HCCL_E_PARA;
498 : }
499 0 : if (!remainRetryConfig.empty()) {
500 0 : CHK_RET(SplitRetryEnable(remainRetryConfig, retryEnables));
501 : }
502 0 : return HCCL_SUCCESS;
503 0 : }
504 :
505 0 : HcclResult CommConfig::SetConfigRetryEnable(const std::vector<std::string> &retryEnables)
506 : {
507 : const std::map<std::string, u32> hcclRetryLevelMap = {
508 : {"L0", HCCL_RETRY_ENABLE_LEVEL_0}, {"L1", HCCL_RETRY_ENABLE_LEVEL_1}, {"L2", HCCL_RETRY_ENABLE_LEVEL_2}
509 0 : };
510 :
511 0 : std::map<std::string, u32> countHcclRetryLevelMap = {{"L0", 0}, {"L1", 0}, {"L2", 0}};
512 :
513 0 : const std::map<std::string, bool> hcclRetryEnableMap = {{"0", false}, {"1", true}};
514 0 : for (const auto& retryEnableLevel : retryEnables) {
515 0 : u32 level = 0;
516 0 : bool retryEnable = false;
517 0 : std::size_t found = retryEnableLevel.find(":");
518 0 : if ((found == 0) || (found == (retryEnableLevel.length() - 1))) {
519 0 : HCCL_INFO("[SetRetryEnable] Hccl retryEnableLevel is invalid.");
520 0 : return HCCL_SUCCESS;
521 : }
522 0 : std::string orginalLevel = retryEnableLevel.substr(0, found);
523 0 : std::string orginalRetryEnable = retryEnableLevel.substr(found + 1);
524 0 : if (orginalLevel == "L0") {
525 0 : HCCL_RUN_WARNING("[SetConfigRetryEnable] L0 config does not take effect");
526 : }
527 : // 检查是否存在重复配置level
528 0 : auto iterCountRetryLevel = countHcclRetryLevelMap.find(orginalLevel);
529 0 : if (iterCountRetryLevel == countHcclRetryLevelMap.end()) {
530 0 : HCCL_RUN_WARNING("[SetRetryEnable] Retry config is invalid, level %s is not supported.",
531 : orginalLevel.c_str());
532 0 : return HCCL_SUCCESS;
533 : }
534 0 : if (countHcclRetryLevelMap[orginalLevel] == 1) {
535 0 : HCCL_RUN_WARNING("[SetRetryEnable] Retry config level[%s] is repeated, expect: L1:0, L2:0",
536 : orginalLevel.c_str());
537 0 : return HCCL_SUCCESS;
538 : }
539 0 : countHcclRetryLevelMap[orginalLevel] += 1;
540 : // 获取level和对应的retryEnable，并赋值给g_externalInput.hcclRetryConfig
541 0 : auto iterRetryLevel = hcclRetryLevelMap.find(orginalLevel);
542 0 : if (iterRetryLevel == hcclRetryLevelMap.end()) {
543 0 : HCCL_RUN_WARNING("[SetRetryEnable] Retry config is invalid, level %s is not supported.",
544 : orginalLevel.c_str());
545 0 : return HCCL_SUCCESS;
546 : }
547 0 : auto iterRetryEnable = hcclRetryEnableMap.find(orginalRetryEnable);
548 0 : if (iterRetryEnable == hcclRetryEnableMap.end()) {
549 0 : HCCL_RUN_WARNING("[SetRetryEnable] Retry config is invalid, retryEnable %s is not supported.",
550 : orginalRetryEnable.c_str());
551 0 : return HCCL_SUCCESS;
552 : }
553 0 : level = iterRetryLevel->second;
554 0 : retryEnable = iterRetryEnable->second;
555 0 : retryEnable_[level] = retryEnable;
556 0 : }
557 0 : return HCCL_SUCCESS;
558 0 : }
559 :
560 9 : HcclResult CommConfig::SetConfigHcclRetryParams(const CommConfigHandle &config)
561 : {
562 9 : if (config.hcclRetryParams[0] == '\0') {
563 9 : return HCCL_SUCCESS;
564 : }
565 0 : auto retryParamsLength = strlen(config.hcclRetryParams);
566 0 : retryParamsLength = retryParamsLength < COMM_RETRY_PARAMS_MAX_LENGTH ?
567 : retryParamsLength : COMM_RETRY_PARAMS_MAX_LENGTH;
568 0 : std::string retryParams = std::string(config.hcclRetryParams, retryParamsLength);
569 0 : u32 maxcnt = 0;
570 0 : u32 holdtime = 0;
571 0 : u32 intervaltime = 0;
572 0 : int ret = 0;
573 0 : ret = sscanf_s(retryParams.c_str(), "MaxCnt:%u, HoldTime:%u, IntervalTime:%u",
574 : &maxcnt, &holdtime, &intervaltime);
575 : /* 三个参数全部解析成功，返回值为3，否则不等于3 */
576 0 : if ((ret != 3) || (maxcnt > HCCL_RETRY_MAXCNT_MAX) || (maxcnt < HCCL_RETRY_MAXCNT_MIN)
577 0 : || (holdtime > HCCL_RETRY_HLOD_TIME_MAX) || (intervaltime > HCCL_RETRY_INTERVAL_MAX)) {
578 0 : HCCL_ERROR("[SetConfigHcclRetryParams]fail, HCCL_OP_RETRY_PARAMS: %s is invalid, format must be: "\
579 : "MaxCnt:cnt, HoldTime:time, IntervalTime:time, cnt range is [1, 10], time range is [0, 60000]ms.",
580 : retryParams.c_str());
581 0 : return HCCL_E_PARA;
582 : }
583 0 : retryMaxCnt_ = maxcnt;
584 0 : retryHoldTime_ = holdtime;
585 0 : retryIntervalTime_ = intervaltime;
586 :
587 0 : HCCL_RUN_INFO("[SetConfigHcclRetryParams]HCCL_OP_RETRY_PARAMS is set, " \
588 : "MaxCnt is [%u], HoldTime is [%u]ms, IntervalTime is [%u]ms.",
589 : maxcnt, holdtime, intervaltime);
590 0 : return HCCL_SUCCESS;
591 0 : }
592 :
593 0 : HcclResult CommConfig::SetSpecificAlgTypeConfig(std::vector<std::string> &algos)
594 : {
595 0 : for (std::string& algConfig : algos) {
596 0 : std::size_t found = algConfig.find("=");
597 0 : std::string opStringName = algConfig.substr(0, found);
598 0 : if (opStringName == "others") {
599 0 : std::vector<HcclAlgoType> algType;
600 0 : std::string remainAlgoConfig = algConfig.substr(found + 1);
601 0 : CHK_RET(ParseAlgoString("others op type", remainAlgoConfig, algType));
602 0 : for (u32 opType = 0; opType < static_cast<u32>(HcclCMDType::HCCL_CMD_MAX); opType++) {
603 0 : algoConfig_[static_cast<HcclCMDType>(opType)] = algType;
604 : }
605 0 : }
606 0 : }
607 : std::map<std::string, HcclCMDType> hcclOpTypeMap = {
608 0 : {"broadcast", HcclCMDType::HCCL_CMD_BROADCAST},
609 0 : {"allreduce", HcclCMDType::HCCL_CMD_ALLREDUCE},
610 0 : {"reduce", HcclCMDType::HCCL_CMD_REDUCE},
611 0 : {"send", HcclCMDType::HCCL_CMD_SEND},
612 0 : {"receive", HcclCMDType::HCCL_CMD_RECEIVE},
613 0 : {"allgather", HcclCMDType::HCCL_CMD_ALLGATHER},
614 0 : {"reducescatter", HcclCMDType::HCCL_CMD_REDUCE_SCATTER},
615 0 : {"alltoall", HcclCMDType::HCCL_CMD_ALLTOALL},
616 0 : {"gather", HcclCMDType::HCCL_CMD_GATHER},
617 0 : {"scatter", HcclCMDType::HCCL_CMD_SCATTER},
618 0 : {"sendrecv", HcclCMDType::HCCL_CMD_BATCH_SEND_RECV},
619 0 : };
620 0 : for (std::string& algConfig : algos) {
621 0 : std::size_t found = algConfig.find("=");
622 0 : std::string opStringName = algConfig.substr(0, found);
623 0 : if (hcclOpTypeMap.find(opStringName) != hcclOpTypeMap.end()) {
624 0 : HcclCMDType optype = hcclOpTypeMap[opStringName];
625 0 : std::string remainAlgoConfig = algConfig.substr(found + 1);
626 0 : std::vector<HcclAlgoType> algType;
627 0 : CHK_RET(ParseAlgoString(opStringName, remainAlgoConfig, algType));
628 0 : if (algType[0] == HcclAlgoType::HCCL_ALGO_TYPE_NULL) {
629 0 : HCCL_WARNING("[SetSpecificAlgType] specific config level0 not support null type.");
630 0 : return HCCL_SUCCESS;
631 : }
632 0 : algoConfig_[optype] = algType;
633 0 : } else {
634 0 : HCCL_WARNING("[SetSpecificAlgType] specific config optype[%s] is invalid, please check",
635 : opStringName.c_str());
636 0 : return HCCL_SUCCESS;
637 : }
638 0 : }
639 0 : algoConfig_[HcclCMDType::HCCL_CMD_ALLTOALLV] =
640 0 : algoConfig_[HcclCMDType::HCCL_CMD_ALLTOALL];
641 0 : algoConfig_[HcclCMDType::HCCL_CMD_ALLTOALLVC] =
642 0 : algoConfig_[HcclCMDType::HCCL_CMD_ALLTOALL];
643 0 : return HCCL_SUCCESS;
644 0 : }
645 :
646 123 : HcclResult CommConfig::SetConfigTrafficClass(u32 trafficClass)
647 : {
648 123 : trafficClass_ = trafficClass;
649 123 : return HCCL_SUCCESS;
650 : }
651 :
652 120 : HcclResult CommConfig::SetConfigServiceLevel(u32 serviceLevel)
653 : {
654 120 : serviceLevel_ = serviceLevel;
655 120 : return HCCL_SUCCESS;
656 : }
657 :
658 130 : HcclResult CommConfig::SetConfigHcclQos(u32 hcclQos)
659 : {
660 130 : hcclQos_ = hcclQos;
661 130 : return HCCL_SUCCESS;
662 : }
663 :
664 0 : HcclResult CommConfig::SetConfigExecTimeOut(s32 execTimeOut)
665 : {
666 0 : execTimeOut_ = execTimeOut;
667 0 : return HCCL_SUCCESS;
668 : }
669 :
670 820 : u64 CommConfig::GetConfigBufferSize() const
671 : {
672 820 : return bufferSize_;
673 : }
674 :
675 242 : u8 CommConfig::GetConfigDeterministic() const
676 : {
677 242 : return deterministic_;
678 : }
679 :
680 732 : const std::string& CommConfig::GetConfigCommName() const
681 : {
682 732 : return commName_;
683 : }
684 :
685 5 : const std::string& CommConfig::GetConfigUdi() const
686 : {
687 5 : return udi_;
688 : }
689 :
690 764 : bool CommConfig::GetConfigAivMode() const
691 : {
692 764 : return aivMode_;
693 : }
694 :
695 763 : bool CommConfig::GetConfigIsOnlyAivMode() const
696 : {
697 763 : return onlyAivMode_;
698 : }
699 :
700 2198 : bool CommConfig::GetConfigAicpuUnfold() const
701 : {
702 2198 : return aicpuUnfold_;
703 : }
704 :
705 256 : u32 CommConfig::GetConfigTrafficClass() const
706 : {
707 256 : return trafficClass_;
708 : }
709 :
710 255 : u32 CommConfig::GetConfigServiceLevel() const
711 : {
712 255 : return serviceLevel_;
713 : }
714 :
715 0 : u32 CommConfig::GetConfigWorldRankID() const
716 : {
717 0 : return worldRankID_;
718 : }
719 :
720 234 : u64 CommConfig::GetConfigJobID() const
721 : {
722 234 : return jobID_;
723 : }
724 :
725 77 : u8 CommConfig::GetConfigAclGraphZeroCopyEnable() const
726 : {
727 77 : return aclGraphZeroCopyEnable_;
728 : }
729 :
730 1280 : s32 CommConfig::GetConfigExecTimeOut() const
731 : {
732 1280 : return execTimeOut_;
733 : }
734 :
735 1 : bool CommConfig::GetConfigExecTimeOutSet() const
736 : {
737 1 : return execTimeOutSetByConfig_;
738 : }
739 :
740 148 : std::vector<HcclAlgoType> CommConfig::GetConfigHcclAlgo(HcclCMDType opType)
741 : {
742 148 : return algoConfig_[opType];
743 : }
744 :
745 765 : const std::map<HcclCMDType, std::vector<HcclAlgoType>>& CommConfig::GetConfigHcclAlgoMap() const
746 : {
747 765 : return algoConfig_;
748 : }
749 :
750 0 : bool CommConfig::GetConfigIntraServerRetryEnable() const
751 : {
752 0 : return retryEnable_[HCCL_RETRY_ENABLE_LEVEL_0];
753 : }
754 :
755 528 : bool CommConfig::GetConfigInterServerRetryEnable() const
756 : {
757 528 : return retryEnable_[HCCL_RETRY_ENABLE_LEVEL_1];
758 : }
759 :
760 763 : bool CommConfig::GetConfigInterSuperPodRetryEnable() const
761 : {
762 763 : return retryEnable_[HCCL_RETRY_ENABLE_LEVEL_2];
763 : }
764 :
765 0 : u32 CommConfig::GetConfigRetryMaxCnt() const
766 : {
767 0 : return retryMaxCnt_;
768 : }
769 :
770 0 : u32 CommConfig::GetConfigRetryHoldTime() const
771 : {
772 0 : return retryHoldTime_;
773 : }
774 :
775 0 : u32 CommConfig::GetConfigRetryIntervalTime() const
776 : {
777 0 : return retryIntervalTime_;
778 : }
779 :
780 235 : const std::string& CommConfig::GetConfigBufferName() const
781 : {
782 235 : return bufferName_;
783 : }
784 :
785 253 : u32 CommConfig::GetConfigHcclQos() const
786 : {
787 253 : HCCL_INFO("[GetConfigHcclQos] hcclQos = %u", hcclQos_);
788 253 : return hcclQos_;
789 : }
790 :
791 28 : u64 CommConfig::GetConfigSymmetricMemoryStride() const
792 : {
793 28 : return symmetricMemoryStride_;
794 : }
795 : }
|
