Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "calc_crc.h"
12 : #include "rank_consistentcy_checker.h"
13 : #include "env_config.h"
14 :
15 : namespace hccl {
16 :
17 590 : RankConsistentcyChecker::RankConsistentcyChecker()
18 590 : : cannVersion_{0},
19 590 : cannVerCheckSwitch_(false),
20 590 : cannVerInfoRecordFlag_(false),
21 590 : configFileExist_(false)
22 590 : {}
23 :
24 590 : RankConsistentcyChecker::~RankConsistentcyChecker() = default;
25 :
26 1929 : RankConsistentcyChecker& RankConsistentcyChecker::GetInstance(s32 deviceLogicId)
27 : {
28 2514 : static RankConsistentcyChecker instance[MAX_MODULE_DEVICE_NUM];
29 1929 : if (deviceLogicId == HOST_DEVICE_ID) {
30 0 : HCCL_INFO("[GetInstance] deviceLogicId[-1] is HOST_DEVICE_ID");
31 0 : return instance[0];
32 : }
33 1929 : hrtGetDeviceRefresh(&deviceLogicId);
34 1932 : HCCL_INFO("[GetInstance] get deviceLogicId[%d]", deviceLogicId);
35 1932 : CHK_PRT_RET(
36 : (static_cast<u32>(deviceLogicId) >= MAX_MODULE_DEVICE_NUM || deviceLogicId < 0),
37 : HCCL_WARNING("[R]deviceLogicId[%d] is invalid", deviceLogicId), instance[0]);
38 :
39 1930 : return instance[deviceLogicId];
40 : }
41 :
42 : // gather
43 0 : HcclResult RankConsistentcyChecker::RecordOpPara(
44 : HcclCMDType opCMD, const std::string& tag, u64 count, HcclDataType dataType, u32 root, u64 inCclBufferSize,
45 : u64 outCclBufferSize, const char* group, u32 crc, u32 aivCoreLimit)
46 : {
47 0 : return RecordOpPara(
48 : opCMD, tag, count, dataType, HCCL_REDUCE_RESERVED, root, 0, 0, 0, inCclBufferSize, outCclBufferSize, group, crc,
49 0 : aivCoreLimit);
50 : }
51 :
52 : // reduce
53 77 : HcclResult RankConsistentcyChecker::RecordOpPara(
54 : HcclCMDType opCMD, const std::string& tag, u64 count, HcclDataType dataType, HcclReduceOp op, u32 root,
55 : u64 inCclBufferSize, u64 outCclBufferSize, const char* group, u32 crc, [[maybe_unused]] u8 deterministic,
56 : u32 aivCoreLimit)
57 : {
58 77 : return RecordOpPara(
59 77 : opCMD, tag, count, dataType, op, root, 0, 0, 0, inCclBufferSize, outCclBufferSize, group, crc, aivCoreLimit);
60 : }
61 :
62 : // send && receive
63 0 : HcclResult RankConsistentcyChecker::RecordOpPara(
64 : HcclCMDType opCMD, const std::string& tag, u64 count, HcclDataType dataType, u32 rank, u32 srTag, u32 selfRank,
65 : u64 inCclBufferSize, u64 outCclBufferSize, const char* group, u32 crc)
66 : {
67 0 : return RecordOpPara(
68 : opCMD, tag, count, dataType, HCCL_REDUCE_RESERVED, 0, rank, srTag, selfRank, inCclBufferSize, outCclBufferSize,
69 0 : group, crc);
70 : }
71 :
72 : // batchsendrecv
73 0 : HcclResult RankConsistentcyChecker::RecordOpPara(
74 : HcclCMDType opCMD, const std::string& tag, u64 inCclBufferSize, u64 outCclBufferSize, const char* group, u32 crc)
75 : {
76 0 : return RecordOpPara(
77 : opCMD, tag, 0, HCCL_DATA_TYPE_RESERVED, HCCL_REDUCE_RESERVED, 0, 0, 0, 0, inCclBufferSize, outCclBufferSize,
78 0 : group, crc);
79 : }
80 :
81 : // reduce scatter v && AllGather v
82 0 : HcclResult RankConsistentcyChecker::RecordOpPara(
83 : HcclCMDType opCMD, const std::string& tag, const void* counts, const void* displs, const u32 rankSize,
84 : HcclDataType dataType, HcclReduceOp op, u64 inCclBufferSize, u64 outCclBufferSize, const char* group, u32 crc,
85 : [[maybe_unused]] u8 deterministic, u32 aivCoreLimit)
86 : {
87 0 : CHK_RET(RecordOpPara(
88 : opCMD, tag, 0, dataType, op, 0, 0, 0, 0, inCclBufferSize, outCclBufferSize, group, crc, aivCoreLimit));
89 0 : CHK_RET(RecordVaringOpPara(tag, counts, displs, rankSize));
90 0 : return HCCL_SUCCESS;
91 : }
92 :
93 0 : HcclResult RankConsistentcyChecker::RecordVaringOpPara(
94 : const std::string& tag, const void* counts, const void* displs, const u32 rankSize)
95 : {
96 : u32 countsCrc;
97 0 : CHK_RET(CalcRawDataCrc(static_cast<const char_t*>(counts), rankSize * sizeof(u64), countsCrc));
98 0 : crcRecords_[tag][HcclCrcRecordType::HCCL_CRC_RECORD_VARING_COUNTS] = countsCrc;
99 :
100 : u32 displsCrc;
101 0 : CHK_RET(CalcRawDataCrc(static_cast<const char_t*>(displs), rankSize * sizeof(u64), displsCrc));
102 0 : crcRecords_[tag][HcclCrcRecordType::HCCL_CRC_RECORD_VARING_DISPLACEMENTS] = displsCrc;
103 0 : return HCCL_SUCCESS;
104 : }
105 :
106 24 : HcclResult RankConsistentcyChecker::DelOpPara(const std::string& tag)
107 : {
108 24 : std::lock_guard<std::mutex> lock(mutex_);
109 24 : CHK_PRT_RET(
110 : !cmdInfoMap_.erase(tag),
111 : HCCL_ERROR(
112 : "[RankConsistentcyChecker][DelOpPara]CMD info for tag[%s] does not exist, delete fail.", tag.c_str()),
113 : HCCL_E_INTERNAL);
114 24 : CHK_PRT_RET(
115 : !infoFlagCmdMap_.erase(tag),
116 : HCCL_ERROR(
117 : "[RankConsistentcyChecker][DelOpPara]CMD info flag cmd for tag[%s] does not exist, delete fail.",
118 : tag.c_str()),
119 : HCCL_E_INTERNAL);
120 24 : crcRecords_.erase(tag);
121 24 : return HCCL_SUCCESS;
122 24 : }
123 :
124 236 : HcclResult RankConsistentcyChecker::RecordVerInfo(const std::string& versionInfo)
125 : {
126 236 : std::lock_guard<std::mutex> lock(mutex_);
127 : // Only record once, in case of multiple calls while other process is reading CANN version information
128 : // in CompareFrame func and get the intermediate state.
129 236 : if (cannVerInfoRecordFlag_) {
130 231 : HCCL_INFO("[RankConsistentcyChecker][RecordVerInfo]Cann version information has been recorded.");
131 231 : return HCCL_SUCCESS;
132 : }
133 :
134 5 : u32 strLen = versionInfo.length();
135 5 : s32 sRet = memset_s(cannVersion_, MAX_CANN_VERSION_LEN + 1, 0, MAX_CANN_VERSION_LEN + 1);
136 5 : CHK_PRT_RET(
137 : sRet != EOK,
138 : HCCL_WARNING(
139 : "[RankConsistentcyChecker][RecordVerInfo]memory set 0 fail for version str "
140 : "array. return[%d].",
141 : sRet),
142 : HCCL_SUCCESS);
143 :
144 5 : CHK_PRT_RET(strLen == 0, HCCL_WARNING("[Record][CannVersion] version information str is empty."), HCCL_SUCCESS);
145 :
146 5 : CHK_PRT_RET(
147 : strLen >= MAX_CANN_VERSION_LEN,
148 : HCCL_WARNING("[Record][CannVersion]"
149 : "length of version information str is too long."),
150 : HCCL_SUCCESS);
151 5 : sRet = strncpy_s(cannVersion_, MAX_CANN_VERSION_LEN + 1, versionInfo.c_str(), strLen);
152 5 : CHK_PRT_RET(
153 : sRet != EOK, HCCL_WARNING("[Record][CannVersion] call strncpy_s failed, return [%d].", sRet), HCCL_SUCCESS);
154 :
155 5 : cannVerInfoRecordFlag_ = true;
156 5 : return HCCL_SUCCESS;
157 236 : }
158 :
159 1055 : u64 RankConsistentcyChecker::GetRankConsistentDataLength() { return sizeof(HcclCheckInfo); }
160 :
161 0 : void RankConsistentcyChecker::RecordProtocolType(ProtocolType protocolType)
162 : {
163 0 : HCCL_INFO("[RankConsistentcyChecker][RecordProtocolType]protocolType set to [%d].", static_cast<s32>(protocolType));
164 0 : protocolType_ = protocolType;
165 0 : return;
166 : }
167 :
168 4 : HcclResult RankConsistentcyChecker::GetCheckFrame(u8* destBuf, u64 maxDestBuf, const std::string& tag)
169 : {
170 4 : CHK_PTR_NULL(destBuf);
171 : // 要发送的校验帧
172 4 : HcclCheckInfo checkInfo;
173 4 : u64 checkInfoLen = sizeof(checkInfo);
174 4 : HcclResult ret = GenerateCheckFrame(checkInfo, tag);
175 4 : checkInfo.cmdInfo.selfRank = 0; // 自身的group rank 不做校验,置0
176 4 : CHK_PRT_RET(
177 : ret != HCCL_SUCCESS,
178 : HCCL_ERROR(
179 : "[RankConsistentcyChecker][GetCheckFrame]generate check frame fail. "
180 : "return[%d]",
181 : ret),
182 : ret);
183 :
184 4 : s32 sret = memcpy_s(destBuf, maxDestBuf, &checkInfo, checkInfoLen);
185 4 : CHK_PRT_RET(
186 : sret != EOK,
187 : HCCL_ERROR(
188 : "[RankConsistentcyChecker][GetCheckFrame]frame len[%llu] is bigger than "
189 : "dest buffer len[%llu].",
190 : checkInfoLen, maxDestBuf),
191 : HCCL_E_INTERNAL);
192 4 : return HCCL_SUCCESS;
193 : }
194 :
195 0 : HcclResult RankConsistentcyChecker::CheckFrameRecv(const u8* recvBuf, u32 recvBufLen, const std::string& tag)
196 : {
197 : #ifndef HCCD
198 0 : if ((GetExternalInconsistentCheckSwitch() == InconsistentCheckMode::OFF)
199 0 : || (GetExternalInconsistentCheckSwitch() == InconsistentCheckMode::FIRST
200 0 : && inconsistentCheckFirstDone_ == true)) {
201 0 : return HCCL_SUCCESS;
202 : }
203 : #endif
204 0 : CHK_PTR_NULL(recvBuf);
205 0 : CHK_PRT_RET(
206 : recvBufLen == 0 || recvBufLen > MAX_FRAME_LEN,
207 : HCCL_ERROR(
208 : "[RankConsistentcyChecker][CheckFrameRecv] errNo[0x%016llx] recvBufLen is wrong.",
209 : HCCL_ERROR_CODE(HCCL_E_INTERNAL)),
210 : HCCL_E_INTERNAL);
211 :
212 0 : CHK_PRT_RET(
213 : recvBufLen < sizeof(HcclCheckInfo),
214 : HCCL_ERROR(
215 : "[RankConsistentcyChecker][CheckFrameRecv] errNo[0x%016llx] recvBufLen[%u]is less than "
216 : "check info[%zu].",
217 : HCCL_ERROR_CODE(HCCL_E_INTERNAL), recvBufLen, sizeof(HcclCheckInfo)),
218 : HCCL_E_PARA);
219 :
220 0 : HcclCheckInfo checkInfoRecv;
221 : // 对固定长度的全局数组变量，结构体变量进行初始化和拷贝，可以不用检查初始化安全函数返回值
222 0 : (void)memset_s(&checkInfoRecv, sizeof(HcclCheckInfo), 0, sizeof(HcclCheckInfo));
223 0 : (void)memcpy_s(&checkInfoRecv, sizeof(HcclCheckInfo), recvBuf, sizeof(HcclCheckInfo));
224 :
225 0 : HcclCheckInfo checkInfo;
226 0 : CHK_RET(GenerateCheckFrame(checkInfo, tag));
227 0 : if (checkInfo.cmdInfo.cmdType == HcclCMDType::HCCL_CMD_SEND) {
228 0 : checkInfo.cmdInfo.cmdType = HcclCMDType::HCCL_CMD_RECEIVE;
229 0 : checkInfo.cmdInfo.rank = checkInfo.cmdInfo.selfRank;
230 0 : } else if (checkInfo.cmdInfo.cmdType == HcclCMDType::HCCL_CMD_RECEIVE) {
231 0 : checkInfo.cmdInfo.cmdType = HcclCMDType::HCCL_CMD_SEND;
232 0 : checkInfo.cmdInfo.rank = checkInfo.cmdInfo.selfRank;
233 : }
234 :
235 0 : checkInfo.cmdInfo.selfRank = 0; // 自身的子group rank 不做校验
236 0 : if (CompareFrame(checkInfo, checkInfoRecv)) {
237 0 : return HCCL_E_INTERNAL;
238 : }
239 :
240 0 : HCCL_INFO(
241 : "[RankConsistentcyChecker][CheckFrameRecv] check success, len of frame[%u], len of check data[%zu].",
242 : recvBufLen, sizeof(checkInfo));
243 0 : return HCCL_SUCCESS;
244 : }
245 :
246 62 : void RankConsistentcyChecker::ClearCheckInfo()
247 : {
248 62 : configFileExist_ = false;
249 62 : cannVerInfoRecordFlag_ = false;
250 62 : ClearCrcInfo();
251 : {
252 62 : std::lock_guard<std::mutex> lock(mutex_);
253 62 : cmdInfoMap_.clear();
254 62 : infoFlagCmdMap_.clear();
255 62 : }
256 : // 相关规范的例外场景，对固定数组的memset_s可以不判断返回值
257 62 : (void)memset_s(cannVersion_, MAX_CANN_VERSION_LEN + 1, 0, MAX_CANN_VERSION_LEN + 1);
258 62 : protocolType_ = ProtocolType::RESERVED;
259 62 : return;
260 : }
261 :
262 236 : HcclResult RankConsistentcyChecker::CalcStringCrc(const char* str, u32& crc)
263 : {
264 : // 计算字符串CRC
265 236 : HcclResult ret = CalcCrc::HcclCalcCrc(str, strlen(str), crc);
266 236 : CHK_PRT_RET(
267 : ret != HCCL_SUCCESS,
268 : HCCL_ERROR(
269 : "[RankConsistentcyChecker][CalcStringCrc] errNo[0x%016llx] calc string crc error",
270 : HCCL_ERROR_CODE(HCCL_E_INTERNAL)),
271 : HCCL_E_INTERNAL);
272 :
273 236 : HCCL_DEBUG("[RankConsistentcyChecker][CalcStringCrc] result crc[%u].", crc);
274 236 : return HCCL_SUCCESS;
275 : }
276 :
277 0 : HcclResult RankConsistentcyChecker::CalcRawDataCrc(const void* ptr, u64 length, u32& crc)
278 : {
279 : // 计算内存数据块CRC
280 0 : HcclResult ret = CalcCrc::HcclCalcCrc(static_cast<const char*>(ptr), length, crc);
281 0 : CHK_PRT_RET(
282 : ret != HCCL_SUCCESS,
283 : HCCL_ERROR(
284 : "[RankConsistentcyChecker][CalcRawDataCrc] errNo[0x%016llx] calc string crc error",
285 : HCCL_ERROR_CODE(HCCL_E_INTERNAL)),
286 : HCCL_E_INTERNAL);
287 :
288 0 : HCCL_DEBUG("[RankConsistentcyChecker][CalcRawDataCrc] result crc[%u].", crc);
289 0 : return HCCL_SUCCESS;
290 : }
291 :
292 240 : void RankConsistentcyChecker::SetCheckCannVersionSwitch(const bool cannVerCheckSwitch)
293 : {
294 240 : cannVerCheckSwitch_ = cannVerCheckSwitch;
295 240 : return;
296 : }
297 :
298 : // private
299 77 : HcclResult RankConsistentcyChecker::RecordOpPara(
300 : HcclCMDType opCMD, const std::string& tag, u64 count, HcclDataType dataType, HcclReduceOp op, u32 root, u32 rank,
301 : u32 srTag, u32 selfRank, u64 inCclBufferSize, u64 outCclBufferSize, const char* group, u32 crc, u8 deterministic,
302 : u32 aivCoreLimit)
303 : {
304 77 : HcclCMDInfo cmdInfo;
305 : // 相关规范的例外场景，对固定数组的memset_s可以不判断返回值
306 77 : (void)memset_s(&cmdInfo, sizeof(cmdInfo), 0, sizeof(cmdInfo));
307 :
308 77 : cmdInfo.cmdType = opCMD;
309 77 : s32 sRet = strncpy_s(cmdInfo.tag, TAG_MAX_LEN + 1, tag.c_str(), tag.length());
310 77 : CHK_PRT_RET(
311 : sRet != EOK,
312 : HCCL_ERROR(
313 : "[RankConsistentcyChecker][RecordOpPara]errNo[0x%016llx] strlen[%u] of tag is "
314 : "longer than buffer[%u].",
315 : HCCL_ERROR_CODE(HCCL_E_PARA), tag.length(), TAG_MAX_LEN),
316 : HCCL_E_PARA);
317 :
318 77 : cmdInfo.count = count;
319 77 : cmdInfo.dataType = dataType;
320 :
321 77 : std::string strGroup = (group == nullptr) ? HCCL_WORLD_GROUP : group;
322 77 : sRet = strncpy_s(cmdInfo.group, GROUP_NAME_MAX_LEN + 1, strGroup.c_str(), strGroup.length());
323 77 : CHK_PRT_RET(
324 : sRet != EOK,
325 : HCCL_ERROR(
326 : "[RankConsistentcyChecker][RecordOpPara]errNo[0x%016llx] strlen[%u] group is "
327 : "longer than buffer[%u].",
328 : HCCL_ERROR_CODE(HCCL_E_PARA), strGroup.length(), GROUP_NAME_MAX_LEN),
329 : HCCL_E_PARA);
330 :
331 77 : cmdInfo.op = op;
332 77 : cmdInfo.root = root;
333 77 : cmdInfo.rank = rank;
334 77 : cmdInfo.srTag = srTag;
335 77 : cmdInfo.selfRank = selfRank;
336 77 : cmdInfo.inCclBufferSize = inCclBufferSize;
337 77 : cmdInfo.outCclBufferSize = outCclBufferSize;
338 77 : cmdInfo.aivCoreLimit = aivCoreLimit;
339 77 : cmdInfo.deterministic = deterministic;
340 :
341 77 : std::lock_guard<std::mutex> lock(mutex_);
342 77 : cmdInfoMap_[tag] = cmdInfo;
343 75 : infoFlagCmdMap_[tag] = true;
344 76 : crcRecords_[tag][HcclCrcRecordType::HCCL_CRC_RECORD_RANKTABLE] = crc;
345 :
346 69 : return HCCL_SUCCESS;
347 69 : }
348 :
349 0 : HcclResult RankConsistentcyChecker::GetOpParaByTag(const std::string& tag, HcclCMDInfo& CMDInfoOutput)
350 : {
351 0 : auto getResult = cmdInfoMap_.find(tag);
352 0 : CHK_PRT_RET(
353 : getResult == cmdInfoMap_.end(),
354 : HCCL_ERROR(
355 : "[RankConsistentcyChecker][GetOpParaByTag]There is not any CMD information for tag[%s]", tag.c_str()),
356 : HCCL_E_INTERNAL);
357 0 : CMDInfoOutput = getResult->second;
358 0 : return HCCL_SUCCESS;
359 : }
360 :
361 0 : HcclResult RankConsistentcyChecker::GetCrcByTag(const std::string& tag, HcclCRCInfo& crcInfo)
362 : {
363 0 : const auto recordsIter = crcRecords_.find(tag);
364 0 : CHK_PRT_RET(
365 : recordsIter == crcRecords_.end(),
366 : HCCL_ERROR("[RankConsistentcyChecker][GetCrcByTag]There is not any CRC information for tag[%s]", tag.c_str()),
367 : HCCL_E_INTERNAL);
368 0 : crcInfo.crcNum = 0;
369 0 : const auto& tagRecords = recordsIter->second;
370 0 : for (const auto& record : tagRecords) {
371 0 : crcInfo.crcArray[crcInfo.crcNum++] = record.second;
372 0 : HCCL_DEBUG("[RankConsistentcyChecker][GetCrcByTag]Append crc[%u] for tag[%s].", record.second, tag.c_str());
373 : }
374 :
375 0 : HCCL_INFO(
376 : "[RankConsistentcyChecker][GetCrcByTag]After adding crc for tag[%s], crcNum set to [%u].", tag.c_str(),
377 : crcInfo.crcNum);
378 0 : return HCCL_SUCCESS;
379 : }
380 :
381 3 : HcclResult RankConsistentcyChecker::GenerateCheckFrame(HcclCheckInfo& checkInfo, const std::string& tag)
382 : {
383 : // 初始化用于发送的BUFFER
384 : // 对入参的结构体变量指向的内存进行初始化时，使用了变量的结构体类型大小进行初始化，
385 : // 如果指针不为空，可以不检查初始化安全函数的返回值
386 3 : u64 checkInfoLen = sizeof(HcclCheckInfo);
387 3 : (void)memset_s(&checkInfo, checkInfoLen, 0, checkInfoLen);
388 :
389 : // 添加CRC字段到校验帧
390 4 : u32 crcLen = crcTable_.size();
391 4 : checkInfo.crcInfoGlobal.configFileExist_ = configFileExist_;
392 4 : if (crcLen != 0) {
393 0 : CHK_PRT_RET(
394 : crcLen > MAX_CRC_LEN,
395 : HCCL_ERROR("[RankConsistentcyChecker][GenerateCheckFrame]crc num[%u] is too big.", crcLen),
396 : HCCL_E_INTERNAL);
397 0 : checkInfo.crcInfoGlobal.crcNum = crcLen;
398 0 : CHK_RET(GetCrc(crcLen, &checkInfo.crcInfoGlobal.crcArray[0]));
399 : }
400 : // 添加CMD参数信息到校验帧
401 : {
402 4 : std::lock_guard<std::mutex> lock(mutex_);
403 4 : auto getResult = infoFlagCmdMap_.find(tag);
404 4 : if (getResult != infoFlagCmdMap_.end()) {
405 0 : CHK_PRT_RET(
406 : GetOpParaByTag(tag, checkInfo.cmdInfo) != HCCL_SUCCESS,
407 : HCCL_ERROR("[RankConsistentcyChecker][GenerateCheckFrame]get Op para by tag[%s] fail.", tag.c_str()),
408 : HCCL_E_INTERNAL);
409 0 : checkInfo.crcInfoOp.configFileExist_ = configFileExist_;
410 : // 添加CRC字段到校验帧
411 0 : CHK_PRT_RET(
412 : GetCrcByTag(tag, checkInfo.crcInfoOp) != HCCL_SUCCESS,
413 : HCCL_ERROR(
414 : "[RankConsistentcyChecker][GenerateCheckFrame]get ranktable crc by tag[%s] fail.", tag.c_str()),
415 : HCCL_E_INTERNAL);
416 : }
417 4 : }
418 : // 添加HCCL版本信息到校验帧
419 4 : if (cannVerInfoRecordFlag_) {
420 0 : HCCL_DEBUG("[RankConsistentcyChecker][GenerateCheckFrame] CANN version information is [%s].", cannVersion_);
421 0 : s32 sret = memcpy_s(checkInfo.version, MAX_CANN_VERSION_LEN + 1, cannVersion_, strlen(cannVersion_));
422 0 : CHK_PRT_RET(
423 : sret != EOK,
424 : HCCL_ERROR(
425 : "[RankConsistentcyChecker][GenerateCheckFrame] memcpy CANN version information failed, "
426 : "errorno [%d].",
427 : sret),
428 : HCCL_E_MEMORY);
429 : }
430 : // 添加拉远通信传输类型校验
431 : // 910* 不会配置isTcpMode，因此910*在此处的待校验值是一致的
432 4 : checkInfo.protocolType = protocolType_;
433 4 : HCCL_INFO("[RankConsistentcyChecker][GenerateCheckFrame] loc protocolType is [%d].", checkInfo.protocolType);
434 :
435 4 : return HCCL_SUCCESS;
436 : }
437 :
438 29 : bool RankConsistentcyChecker::CompareSection(const char* pRawData, const char* recvBuf, u32 len)
439 : {
440 11741 : for (u32 i = 0; i < len; i++) {
441 11715 : if (*(pRawData + i) != *(recvBuf + i)) {
442 3 : return false;
443 : }
444 : }
445 26 : return true;
446 : }
447 :
448 18 : bool RankConsistentcyChecker::CompareCrcInfo(
449 : const HcclCMDInfo& hcclCMDInfo, HcclCRCInfo& crcInfo, HcclCRCInfo& crcInfoRecv)
450 : {
451 18 : bool bIsDiff = false;
452 : // 检校验整体是否一致
453 18 : if (!CompareSection(
454 : reinterpret_cast<char_t*>(&crcInfo), reinterpret_cast<char_t*>(&crcInfoRecv), sizeof(crcInfo))) {
455 2 : bIsDiff = true;
456 : // 检查每种CRC类型是否一致
457 4 : for (auto i = 0U; i < crcInfo.crcNum; ++i) {
458 2 : if (crcInfo.crcArray[i] != crcInfoRecv.crcArray[i]) {
459 2 : ReportCrcCheckFailed(
460 : hcclCMDInfo, static_cast<HcclCrcRecordType>(i), crcInfo.crcArray[i], crcInfoRecv.crcArray[i]);
461 : }
462 : }
463 : }
464 18 : return bIsDiff;
465 : }
466 :
467 0 : void RankConsistentcyChecker::ReportCmdInfoCheckFailed(
468 : const HcclCMDInfo& hcclCMDInfo, const std::string& paraName, const std::string& localPara,
469 : const std::string& remotePara) const
470 : {
471 0 : ReportCommonError(hcclCMDInfo, paraName, localPara, remotePara, "CMD information");
472 0 : }
473 :
474 3 : void RankConsistentcyChecker::ReportCmdInfoCheckFailed(
475 : const HcclCMDInfo& hcclCMDInfo, const std::string& paraName, uint32_t localPara, uint32_t remotePara)
476 : {
477 3 : ReportCommonError(hcclCMDInfo, paraName, std::to_string(localPara), std::to_string(remotePara), "CMD information");
478 3 : }
479 :
480 2 : void RankConsistentcyChecker::ReportCrcCheckFailed(
481 : const HcclCMDInfo& hcclCMDInfo, HcclCrcRecordType crcType, const uint32_t localCrc, const uint32_t remoteCrc) const
482 : {
483 2 : const auto crcTypeStr = GetCRCTypeEnumStr(crcType);
484 2 : ReportCommonError(hcclCMDInfo, crcTypeStr, std::to_string(localCrc), std::to_string(remoteCrc), "CRC check");
485 2 : }
486 :
487 5 : void RankConsistentcyChecker::ReportCommonError(
488 : const HcclCMDInfo& hcclCMDInfo, const std::string& paraName, const std::string& localParaStr,
489 : const std::string& remoteParaStr, const std::string& errorMsg) const
490 : {
491 5 : std::string opInfo = "Unknown";
492 15 : for (const auto& pair : HCCL_OPTYPE_NAME_MAP) {
493 15 : if (pair.second == hcclCMDInfo.cmdType) {
494 5 : opInfo = std::string(pair.first);
495 5 : break;
496 : }
497 : }
498 100 : RPT_INPUT_ERR(
499 : true, "EI0005", std::vector<std::string>({"ccl_op", "group", "para_name", "local_para", "remote_para"}),
500 : std::vector<std::string>({opInfo, hcclCMDInfo.group, paraName, localParaStr, remoteParaStr}));
501 5 : HCCL_ERROR(
502 : "[%s][%s]%s %s check fail. local[%s], remote[%s]", LOG_KEYWORDS_INIT_CHANNEL.c_str(),
503 : LOG_KEYWORDS_PARAMETER_CONFLICT.c_str(), errorMsg.c_str(), paraName.c_str(), localParaStr.c_str(),
504 : remoteParaStr.c_str());
505 15 : }
506 :
507 1 : void RankConsistentcyChecker::CompareCmdInfo(HcclCheckInfo& checkInfo, HcclCheckInfo& checkInfoRecv)
508 : {
509 1 : auto localInfo = &checkInfo.cmdInfo;
510 1 : auto remoteInfo = &checkInfoRecv.cmdInfo;
511 :
512 1 : if (!CompareSection(localInfo->tag, remoteInfo->tag, TAG_MAX_LEN + 1)) {
513 0 : ReportCmdInfoCheckFailed(*localInfo, "tag", localInfo->tag, remoteInfo->tag);
514 : }
515 :
516 1 : if (localInfo->cmdType != remoteInfo->cmdType) {
517 2 : ReportCmdInfoCheckFailed(
518 1 : *localInfo, "cmdType", static_cast<uint32_t>(localInfo->cmdType),
519 1 : static_cast<uint32_t>(remoteInfo->cmdType));
520 : }
521 :
522 1 : if (localInfo->count != remoteInfo->count) {
523 3 : ReportCmdInfoCheckFailed(*localInfo, "count", localInfo->count, remoteInfo->count);
524 : }
525 :
526 1 : if (localInfo->dataType != remoteInfo->dataType) {
527 2 : ReportCmdInfoCheckFailed(
528 1 : *localInfo, "dataType", static_cast<uint32_t>(localInfo->dataType),
529 1 : static_cast<uint32_t>(remoteInfo->dataType));
530 : }
531 :
532 1 : if (localInfo->op != remoteInfo->op) {
533 0 : ReportCmdInfoCheckFailed(
534 0 : *localInfo, "op", static_cast<uint32_t>(localInfo->op), static_cast<uint32_t>(remoteInfo->op));
535 : }
536 :
537 1 : if (!CompareSection(localInfo->group, remoteInfo->group, GROUP_NAME_MAX_LEN + 1)) {
538 0 : ReportCmdInfoCheckFailed(*localInfo, "group", localInfo->group, remoteInfo->group);
539 : }
540 :
541 1 : if (localInfo->root != remoteInfo->root) {
542 0 : ReportCmdInfoCheckFailed(*localInfo, "root", localInfo->root, remoteInfo->root);
543 : }
544 :
545 1 : if (localInfo->rank != remoteInfo->rank) {
546 0 : ReportCmdInfoCheckFailed(*localInfo, "rank", localInfo->rank, remoteInfo->rank);
547 : }
548 :
549 1 : if (localInfo->srTag != remoteInfo->srTag) {
550 0 : ReportCmdInfoCheckFailed(*localInfo, "srTag", localInfo->srTag, remoteInfo->srTag);
551 : }
552 :
553 1 : if (localInfo->inCclBufferSize != remoteInfo->inCclBufferSize) {
554 0 : ReportCmdInfoCheckFailed(
555 0 : *localInfo, "inCclBufferSize", localInfo->inCclBufferSize, remoteInfo->inCclBufferSize);
556 : }
557 :
558 1 : if (localInfo->outCclBufferSize != remoteInfo->outCclBufferSize) {
559 0 : ReportCmdInfoCheckFailed(
560 0 : *localInfo, "outCclBufferSize", localInfo->outCclBufferSize, remoteInfo->outCclBufferSize);
561 : }
562 :
563 1 : if (localInfo->aivCoreLimit != remoteInfo->aivCoreLimit) {
564 0 : ReportCmdInfoCheckFailed(*localInfo, "aivCoreLimit", localInfo->aivCoreLimit, remoteInfo->aivCoreLimit);
565 : }
566 :
567 1 : if (localInfo->deterministic != remoteInfo->deterministic) {
568 0 : ReportCmdInfoCheckFailed(*localInfo, "deterministic", localInfo->deterministic, remoteInfo->deterministic);
569 : }
570 :
571 1 : return;
572 : }
573 :
574 18 : std::vector<std::string_view> Split(const std::string& str, char delimiter)
575 : {
576 18 : std::vector<std::string_view> parts;
577 18 : std::string_view sv(str);
578 18 : size_t start = 0;
579 238 : for (size_t i = 0; i <= sv.size(); ++i) {
580 220 : if (i == sv.size() || sv[i] == delimiter) {
581 33 : parts.push_back(sv.substr(start, i - start));
582 33 : start = i + 1;
583 : }
584 : }
585 18 : return parts;
586 0 : }
587 :
588 9 : std::vector<std::string> GetVersionErrMessage(const std::string& localVersion, const std::string& remoteVersion)
589 : {
590 9 : std::vector<std::string> versionErrMessage;
591 9 : auto localVersionParts = Split(localVersion, '_');
592 9 : auto remoteVersionParts = Split(remoteVersion, '_');
593 18 : std::string localHcommVersion = localVersionParts.size() > 0 ? std::string(localVersionParts[0]) : "empty";
594 20 : std::string localHcclVersion = localVersionParts.size() > 1 ? std::string(localVersionParts[1]) : "empty";
595 18 : std::string remoteHcommVersion = remoteVersionParts.size() > 0 ? std::string(remoteVersionParts[0]) : "empty";
596 20 : std::string remoteHcclVersion = remoteVersionParts.size() > 1 ? std::string(remoteVersionParts[1]) : "empty";
597 :
598 9 : bool hcommDiff = (localHcommVersion != remoteHcommVersion);
599 9 : bool hcclDiff = (localHcclVersion != remoteHcclVersion);
600 9 : if (hcommDiff && !hcclDiff) {
601 : // 将版本号拆分为hcomm版本号和hccl版本号, 构造ErrMessage
602 : // The %s versions are inconsistent. The local %s, while the remote %s.
603 : versionErrMessage
604 25 : = {"Toolkit (cann-hcomm)", "version is " + localHcommVersion, "version is " + remoteHcommVersion};
605 4 : } else if (!hcommDiff && hcclDiff) {
606 5 : versionErrMessage = {"ops (cann-hccl)", "version is " + localHcclVersion, "version is " + remoteHcclVersion};
607 3 : } else if (hcommDiff && hcclDiff) {
608 : versionErrMessage
609 8 : = {"Toolkit (cann-hcomm) and ops (cann-hccl)",
610 4 : "Toolkit (cann-hcomm) version is " + localHcommVersion + " and ops (cann-hccl) version is "
611 : + localHcclVersion,
612 4 : "Toolkit (cann-hcomm) version is " + remoteHcommVersion + " and ops (cann-hccl) version is "
613 10 : + remoteHcclVersion};
614 : } else {
615 : // 拆分版本号字符串失败, 构造为原版的ErrMessage
616 : // The HCCL versions are inconsistent. The local version is %s, while the remote version is %s.
617 1 : HCCL_ERROR("[RankConsistentcyChecker][GetVersionErrMessage]Failed to split the version string.");
618 5 : versionErrMessage = {"HCCL", "version is " + localVersion, "version is " + remoteVersion};
619 : }
620 9 : return versionErrMessage;
621 31 : }
622 :
623 8 : void RankConsistentcyChecker::CompareVersion(const HcclCheckInfo& local, const HcclCheckInfo& remote, bool& isDiff)
624 : {
625 16 : std::string localCannVersion = local.version;
626 8 : std::string remoteCannVersion = remote.version;
627 8 : if (localCannVersion.empty() || remoteCannVersion.empty()) { // cann版本信息读取失败，返回告警
628 1 : HCCL_WARNING(
629 : "[RankConsistentcyChecker][CompareFrame] CANN version str is empty. local_version %s, "
630 : "remote_version %s.",
631 : local.version, remote.version);
632 7 : } else if (localCannVersion != remoteCannVersion) { // cann版本信息读取成功，且版本不一致
633 5 : const char* ascendHomePath = std::getenv("ASCEND_HOME_PATH");
634 : std::string versionFilePath
635 10 : = (ascendHomePath != nullptr) ? std::string(ascendHomePath) + "/share/info/hccl/version.info" : "unknown";
636 5 : auto versionErrMessage = GetVersionErrMessage(localCannVersion, remoteCannVersion);
637 : // GetVersionErrMessage函数内部已保证versionErrMessage.size() == 3
638 70 : RPT_INPUT_ERR(
639 : true, "EI0008", std::vector<std::string>({"inconsistent_package", "local_version", "remote_version"}),
640 : std::vector<std::string>(
641 : {versionErrMessage[0], versionErrMessage[1] + " (version file: " + versionFilePath + ")",
642 : versionErrMessage[2]}));
643 5 : HCCL_ERROR(
644 : "[%s][%s] errNo[0x%016llx] Inconsistent %s Versions. The local %s, while the remote %s.",
645 : LOG_KEYWORDS_INIT_CHANNEL.c_str(), LOG_KEYWORDS_VERSION_CONFLICT.c_str(), HCCL_ERROR_CODE(HCCL_E_INTERNAL),
646 : versionErrMessage[0].c_str(), versionErrMessage[1].c_str(), versionErrMessage[2].c_str());
647 5 : isDiff = true;
648 5 : }
649 18 : }
650 :
651 9 : bool RankConsistentcyChecker::CompareFrame(HcclCheckInfo& checkInfo, HcclCheckInfo& checkInfoRecv)
652 : {
653 9 : bool bIsDiff = false;
654 9 : if (CompareCrcInfo(checkInfo.cmdInfo, checkInfo.crcInfoGlobal, checkInfoRecv.crcInfoGlobal)) {
655 1 : HCCL_ERROR(
656 : "[RankConsistentcyChecker][CompareFrame]errNo[0x%016llx] CRC check fail, please check the "
657 : "rankTable file and hccl_config file.",
658 : HCCL_ERROR_CODE(HCCL_E_INTERNAL));
659 1 : bIsDiff = true;
660 : }
661 9 : if (CompareCrcInfo(checkInfo.cmdInfo, checkInfo.crcInfoOp, checkInfoRecv.crcInfoOp)) {
662 1 : HCCL_ERROR(
663 : "[RankConsistentcyChecker][CompareFrame]errNo[0x%016llx] Op CRC check fail, please check the op"
664 : " parameters, rankTable file and hccl_config file.",
665 : HCCL_ERROR_CODE(HCCL_E_INTERNAL));
666 1 : bIsDiff = true;
667 : }
668 9 : if (!CompareSection(
669 9 : reinterpret_cast<char_t*>(&checkInfo.cmdInfo), reinterpret_cast<char_t*>(&checkInfoRecv.cmdInfo),
670 : sizeof(checkInfo.cmdInfo))) {
671 1 : CompareCmdInfo(checkInfo, checkInfoRecv);
672 1 : HCCL_ERROR(
673 : "[RankConsistentcyChecker][CompareFrame]errNo[0x%016llx] CMD check fail", HCCL_ERROR_CODE(HCCL_E_INTERNAL));
674 1 : bIsDiff = true;
675 : }
676 9 : HCCL_INFO(
677 : "loc protocolType is [%d], rem protocolType is [%d].", checkInfo.protocolType, checkInfoRecv.protocolType);
678 9 : if (checkInfo.protocolType != checkInfoRecv.protocolType) {
679 1 : HCCL_ERROR(
680 : "[RankConsistentcyChecker][CompareFrame]errNo[0x%016llx] ProtocolType check fail",
681 : HCCL_ERROR_CODE(HCCL_E_INTERNAL));
682 1 : bIsDiff = true;
683 : }
684 :
685 : // Cann版本校验，只在集合通信场景校验CANN版本
686 9 : if (cannVerCheckSwitch_) {
687 8 : CompareVersion(checkInfo, checkInfoRecv, bIsDiff);
688 : }
689 : #ifndef HCCD
690 9 : if (GetExternalInconsistentCheckSwitch() == InconsistentCheckMode::FIRST) {
691 9 : inconsistentCheckFirstDone_ = true;
692 : }
693 : #endif
694 9 : return bIsDiff;
695 : }
696 :
697 9 : HcclResult RankConsistentcyChecker::AddCrc(const u32 crcValue)
698 : {
699 9 : HCCL_DEBUG("crcValue[%u].", crcValue);
700 9 : crcTable_.push_back(crcValue);
701 9 : HCCL_DEBUG("num[%llu].", crcTable_.size());
702 9 : return HCCL_SUCCESS;
703 : }
704 :
705 63 : HcclResult RankConsistentcyChecker::ClearCrcInfo(void)
706 : {
707 63 : this->crcTable_.clear();
708 63 : if (this->crcTable_.size() != 0) {
709 0 : HCCL_ERROR("[Clear][CrcInfo]errNo[0x%016llx] clear crcTable_ is failed", HCCL_ERROR_CODE(HCCL_E_INTERNAL));
710 0 : return HCCL_E_INTERNAL;
711 : }
712 63 : return HCCL_SUCCESS;
713 : }
714 :
715 1 : HcclResult RankConsistentcyChecker::GetCrc(u32 num, u32* crcAddr)
716 : {
717 1 : CHK_PTR_NULL(crcAddr);
718 1 : HCCL_DEBUG("num[%u], crc[%u].", num, *crcAddr);
719 :
720 1 : if (num == 0) {
721 0 : HCCL_ERROR("[Get][Crc]errNo[0x%016llx] In get crc the value of num is 0", HCCL_ERROR_CODE(HCCL_E_PARA));
722 0 : return HCCL_E_PARA;
723 : }
724 :
725 1 : if (num != crcTable_.size()) {
726 0 : HCCL_ERROR(
727 : "[Get][Crc]errNo[0x%016llx] num error inputNum[%u], localNum[%llu]", HCCL_ERROR_CODE(HCCL_E_INTERNAL), num,
728 : crcTable_.size());
729 0 : return HCCL_E_INTERNAL;
730 : }
731 :
732 4 : for (u32 i = 0; i < num; i++) {
733 3 : crcAddr[i] = crcTable_[i];
734 : }
735 1 : return HCCL_SUCCESS;
736 : }
737 : } // namespace hccl
|
