Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "calc_crc.h"
12 : #include "rank_consistentcy_checker.h"
13 : #include "env_config.h"
14 :
15 : namespace hccl {
16 :
17 1180 : RankConsistentcyChecker::RankConsistentcyChecker() : cannVersion_{0}, cannVerCheckSwitch_(false),
18 590 : cannVerInfoRecordFlag_(false), configFileExist_(false)
19 : {
20 590 : }
21 :
22 590 : RankConsistentcyChecker::~RankConsistentcyChecker() = default;
23 :
24 1923 : RankConsistentcyChecker& RankConsistentcyChecker::GetInstance(s32 deviceLogicId)
25 : {
26 2508 : static RankConsistentcyChecker instance[MAX_MODULE_DEVICE_NUM];
27 1923 : if (deviceLogicId == HOST_DEVICE_ID) {
28 0 : HCCL_INFO("[GetInstance] deviceLogicId[-1] is HOST_DEVICE_ID");
29 0 : return instance[0];
30 : }
31 1923 : hrtGetDeviceRefresh(&deviceLogicId);
32 1923 : HCCL_INFO("[GetInstance] get deviceLogicId[%d]", deviceLogicId);
33 1923 : CHK_PRT_RET((static_cast<u32>(deviceLogicId) >= MAX_MODULE_DEVICE_NUM || deviceLogicId < 0),
34 : HCCL_WARNING("[R]deviceLogicId[%d] is invalid", deviceLogicId), instance[0]);
35 :
36 1921 : return instance[deviceLogicId];
37 : }
38 :
39 : // gather
40 0 : HcclResult RankConsistentcyChecker::RecordOpPara(HcclCMDType opCMD, const std::string &tag, u64 count,
41 : HcclDataType dataType, u32 root, u64 inCclBufferSize, u64 outCclBufferSize, const char *group, u32 crc,
42 : u32 aivCoreLimit)
43 : {
44 0 : return RecordOpPara(opCMD, tag, count, dataType, HCCL_REDUCE_RESERVED, root, 0, 0, 0, inCclBufferSize,
45 0 : outCclBufferSize, group, crc, aivCoreLimit);
46 : }
47 :
48 : // reduce
49 77 : HcclResult RankConsistentcyChecker::RecordOpPara(HcclCMDType opCMD, const std::string &tag, u64 count,
50 : HcclDataType dataType, HcclReduceOp op, u32 root, u64 inCclBufferSize, u64 outCclBufferSize,
51 : const char *group, u32 crc, u8 deterministic, u32 aivCoreLimit)
52 : {
53 77 : return RecordOpPara(opCMD, tag, count, dataType, op, root, 0, 0, 0, inCclBufferSize, outCclBufferSize, group, crc, aivCoreLimit);
54 : }
55 :
56 : // send && receive
57 0 : HcclResult RankConsistentcyChecker::RecordOpPara(HcclCMDType opCMD, const std::string &tag, u64 count,
58 : HcclDataType dataType, u32 rank, u32 srTag, u32 selfRank, u64 inCclBufferSize, u64 outCclBufferSize,
59 : const char *group, u32 crc)
60 : {
61 0 : return RecordOpPara(opCMD, tag, count, dataType, HCCL_REDUCE_RESERVED, 0, rank, srTag, selfRank,
62 0 : inCclBufferSize, outCclBufferSize, group, crc);
63 : }
64 :
65 : // batchsendrecv
66 0 : HcclResult RankConsistentcyChecker::RecordOpPara(HcclCMDType opCMD, const std::string &tag,
67 : u64 inCclBufferSize, u64 outCclBufferSize, const char *group, u32 crc)
68 : {
69 0 : return RecordOpPara(opCMD, tag, 0, HCCL_DATA_TYPE_RESERVED, HCCL_REDUCE_RESERVED, 0, 0, 0, 0, inCclBufferSize,
70 0 : outCclBufferSize, group, crc);
71 : }
72 :
73 : // reduce scatter v && AllGather v
74 0 : HcclResult RankConsistentcyChecker::RecordOpPara(HcclCMDType opCMD, const std::string &tag,
75 : const void *counts, const void *displs, const u32 rankSize,
76 : HcclDataType dataType, HcclReduceOp op, u64 inCclBufferSize, u64 outCclBufferSize, const char *group, u32 crc, u8 deterministic,
77 : u32 aivCoreLimit)
78 : {
79 0 : CHK_RET(RecordOpPara(opCMD, tag, 0, dataType, op, 0, 0, 0, 0, inCclBufferSize,
80 : outCclBufferSize, group, crc, aivCoreLimit));
81 0 : CHK_RET(RecordVaringOpPara(tag, counts, displs, rankSize));
82 0 : return HCCL_SUCCESS;
83 : }
84 :
85 0 : HcclResult RankConsistentcyChecker::RecordVaringOpPara(const std::string &tag, const void *counts, const void *displs,
86 : const u32 rankSize)
87 : {
88 : u32 countsCrc;
89 0 : CHK_RET(CalcRawDataCrc(static_cast<const char_t*>(counts), rankSize * sizeof(u64), countsCrc));
90 0 : crcRecords_[tag][HcclCrcRecordType::HCCL_CRC_RECORD_VARING_COUNTS] = countsCrc;
91 :
92 : u32 displsCrc;
93 0 : CHK_RET(CalcRawDataCrc(static_cast<const char_t*>(displs), rankSize * sizeof(u64), displsCrc));
94 0 : crcRecords_[tag][HcclCrcRecordType::HCCL_CRC_RECORD_VARING_DISPLACEMENTS] = displsCrc;
95 0 : return HCCL_SUCCESS;
96 : }
97 :
98 24 : HcclResult RankConsistentcyChecker::DelOpPara(const std::string &tag)
99 : {
100 24 : std::lock_guard<std::mutex> lock(mutex_);
101 24 : CHK_PRT_RET(!cmdInfoMap_.erase(tag),
102 : HCCL_ERROR("[RankConsistentcyChecker][DelOpPara]CMD info for tag[%s] does not exist, delete fail.",
103 : tag.c_str()), HCCL_E_INTERNAL);
104 24 : CHK_PRT_RET(!infoFlagCmdMap_.erase(tag),
105 : HCCL_ERROR("[RankConsistentcyChecker][DelOpPara]CMD info flag cmd for tag[%s] does not exist, delete fail.",
106 : tag.c_str()), HCCL_E_INTERNAL);
107 24 : crcRecords_.erase(tag);
108 24 : return HCCL_SUCCESS;
109 24 : }
110 :
111 235 : HcclResult RankConsistentcyChecker::RecordVerInfo(const std::string &versionInfo)
112 : {
113 235 : std::lock_guard<std::mutex> lock(mutex_);
114 : // Only record once, in case of multiple calls while other process is reading CANN version information
115 : // in CompareFrame func and get the intermediate state.
116 235 : if (cannVerInfoRecordFlag_) {
117 230 : HCCL_INFO("[RankConsistentcyChecker][RecordVerInfo]Cann version information has been recorded.");
118 230 : return HCCL_SUCCESS;
119 : }
120 :
121 5 : u32 strLen = versionInfo.length();
122 5 : s32 sRet = memset_s(cannVersion_, MAX_CANN_VERSION_LEN + 1, 0, MAX_CANN_VERSION_LEN + 1);
123 5 : CHK_PRT_RET(sRet != EOK, HCCL_WARNING("[RankConsistentcyChecker][RecordVerInfo]memory set 0 fail for version str "
124 : "array. return[%d].", sRet), HCCL_SUCCESS);
125 :
126 5 : CHK_PRT_RET(strLen == 0, HCCL_WARNING("[Record][CannVersion] version information str is empty."),
127 : HCCL_SUCCESS);
128 :
129 5 : CHK_PRT_RET(strLen >= MAX_CANN_VERSION_LEN, HCCL_WARNING("[Record][CannVersion]"
130 : "length of version information str is too long."), HCCL_SUCCESS);
131 5 : sRet = strncpy_s(cannVersion_, MAX_CANN_VERSION_LEN + 1, versionInfo.c_str(), strLen);
132 5 : CHK_PRT_RET(sRet != EOK, HCCL_WARNING("[Record][CannVersion] call strncpy_s failed, return [%d].", sRet),
133 : HCCL_SUCCESS);
134 :
135 5 : cannVerInfoRecordFlag_ = true;
136 5 : return HCCL_SUCCESS;
137 235 : }
138 :
139 1049 : u64 RankConsistentcyChecker::GetRankConsistentDataLength()
140 : {
141 1049 : return sizeof(HcclCheckInfo);
142 : }
143 :
144 0 : void RankConsistentcyChecker::RecordProtocolType(ProtocolType protocolType)
145 : {
146 0 : HCCL_INFO("[RankConsistentcyChecker][RecordProtocolType]protocolType set to [%d].",
147 : static_cast<s32>(protocolType));
148 0 : protocolType_ = protocolType;
149 0 : return;
150 : }
151 :
152 4 : HcclResult RankConsistentcyChecker::GetCheckFrame(u8 *destBuf, u64 maxDestBuf, const std::string &tag)
153 : {
154 4 : CHK_PTR_NULL(destBuf);
155 : // 要发送的校验帧
156 4 : HcclCheckInfo checkInfo;
157 4 : u64 checkInfoLen = sizeof(checkInfo);
158 4 : HcclResult ret = GenerateCheckFrame(checkInfo, tag);
159 4 : checkInfo.cmdInfo.selfRank = 0; // 自身的group rank 不做校验,置0
160 4 : CHK_PRT_RET(ret != HCCL_SUCCESS, HCCL_ERROR("[RankConsistentcyChecker][GetCheckFrame]generate check frame fail. "
161 : "return[%d]", ret), ret);
162 :
163 4 : s32 sret = memcpy_s(destBuf, maxDestBuf, &checkInfo, checkInfoLen);
164 4 : CHK_PRT_RET(sret != EOK, HCCL_ERROR("[RankConsistentcyChecker][GetCheckFrame]frame len[%llu] is bigger than "
165 : "dest buffer len[%llu].", checkInfoLen, maxDestBuf), HCCL_E_INTERNAL);
166 4 : return HCCL_SUCCESS;
167 : }
168 :
169 0 : HcclResult RankConsistentcyChecker::CheckFrameRecv(const u8 *recvBuf, u32 recvBufLen, const std::string &tag)
170 : {
171 : #ifndef HCCD
172 0 : if ((GetExternalInconsistentCheckSwitch() == InconsistentCheckMode::OFF) ||
173 0 : (GetExternalInconsistentCheckSwitch() == InconsistentCheckMode::FIRST && inconsistentCheckFirstDone_ == true)) {
174 0 : return HCCL_SUCCESS;
175 : }
176 : #endif
177 0 : CHK_PTR_NULL(recvBuf);
178 0 : CHK_PRT_RET(recvBufLen == 0 || recvBufLen > MAX_FRAME_LEN,
179 : HCCL_ERROR("[RankConsistentcyChecker][CheckFrameRecv] errNo[0x%016llx] recvBufLen is wrong.",
180 : HCCL_ERROR_CODE(HCCL_E_INTERNAL)), HCCL_E_INTERNAL);
181 :
182 0 : CHK_PRT_RET(recvBufLen < sizeof(HcclCheckInfo),
183 : HCCL_ERROR("[RankConsistentcyChecker][CheckFrameRecv] errNo[0x%016llx] recvBufLen[%u]is less than "
184 : "check info[%zu].", HCCL_ERROR_CODE(HCCL_E_INTERNAL), recvBufLen, sizeof(HcclCheckInfo)), HCCL_E_PARA);
185 :
186 0 : HcclCheckInfo checkInfoRecv;
187 : // 对固定长度的全局数组变量，结构体变量进行初始化和拷贝，可以不用检查初始化安全函数返回值
188 0 : (void)memset_s(&checkInfoRecv, sizeof(HcclCheckInfo), 0, sizeof(HcclCheckInfo));
189 0 : (void)memcpy_s(&checkInfoRecv, sizeof(HcclCheckInfo), recvBuf, sizeof(HcclCheckInfo));
190 :
191 0 : HcclCheckInfo checkInfo;
192 0 : CHK_RET(GenerateCheckFrame(checkInfo, tag));
193 0 : if (checkInfo.cmdInfo.cmdType == HcclCMDType::HCCL_CMD_SEND) {
194 0 : checkInfo.cmdInfo.cmdType = HcclCMDType::HCCL_CMD_RECEIVE;
195 0 : checkInfo.cmdInfo.rank = checkInfo.cmdInfo.selfRank;
196 0 : } else if (checkInfo.cmdInfo.cmdType == HcclCMDType::HCCL_CMD_RECEIVE) {
197 0 : checkInfo.cmdInfo.cmdType = HcclCMDType::HCCL_CMD_SEND;
198 0 : checkInfo.cmdInfo.rank = checkInfo.cmdInfo.selfRank;
199 : }
200 :
201 0 : checkInfo.cmdInfo.selfRank = 0; // 自身的子group rank 不做校验
202 0 : if (CompareFrame(checkInfo, checkInfoRecv)) {
203 0 : return HCCL_E_INTERNAL;
204 : }
205 :
206 0 : HCCL_INFO("[RankConsistentcyChecker][CheckFrameRecv] check success, len of frame[%u], len of check data[%zu].",
207 : recvBufLen, sizeof(checkInfo));
208 0 : return HCCL_SUCCESS;
209 : }
210 :
211 62 : void RankConsistentcyChecker::ClearCheckInfo()
212 : {
213 62 : configFileExist_ = false;
214 62 : cannVerInfoRecordFlag_ = false;
215 62 : ClearCrcInfo();
216 : {
217 62 : std::lock_guard<std::mutex> lock(mutex_);
218 62 : cmdInfoMap_.clear();
219 62 : infoFlagCmdMap_.clear();
220 62 : }
221 : // 相关规范的例外场景，对固定数组的memset_s可以不判断返回值
222 62 : (void)memset_s(cannVersion_, MAX_CANN_VERSION_LEN + 1, 0, MAX_CANN_VERSION_LEN + 1);
223 62 : protocolType_ = ProtocolType::RESERVED;
224 62 : return;
225 : }
226 :
227 235 : HcclResult RankConsistentcyChecker::CalcStringCrc(const char *str, u32 &crc)
228 : {
229 : // 计算字符串CRC
230 235 : HcclResult ret = CalcCrc::HcclCalcCrc(str, strlen(str), crc);
231 235 : CHK_PRT_RET(ret != HCCL_SUCCESS,
232 : HCCL_ERROR("[RankConsistentcyChecker][CalcStringCrc] errNo[0x%016llx] calc string crc error",
233 : HCCL_ERROR_CODE(HCCL_E_INTERNAL)), HCCL_E_INTERNAL);
234 :
235 235 : HCCL_DEBUG("[RankConsistentcyChecker][CalcStringCrc] result crc[%u].", crc);
236 235 : return HCCL_SUCCESS;
237 : }
238 :
239 0 : HcclResult RankConsistentcyChecker::CalcRawDataCrc(const void *ptr, u64 length, u32 &crc)
240 : {
241 : // 计算内存数据块CRC
242 0 : HcclResult ret = CalcCrc::HcclCalcCrc(static_cast<const char*>(ptr), length, crc);
243 0 : CHK_PRT_RET(ret != HCCL_SUCCESS,
244 : HCCL_ERROR("[RankConsistentcyChecker][CalcRawDataCrc] errNo[0x%016llx] calc string crc error",
245 : HCCL_ERROR_CODE(HCCL_E_INTERNAL)), HCCL_E_INTERNAL);
246 :
247 0 : HCCL_DEBUG("[RankConsistentcyChecker][CalcRawDataCrc] result crc[%u].", crc);
248 0 : return HCCL_SUCCESS;
249 : }
250 :
251 239 : void RankConsistentcyChecker::SetCheckCannVersionSwitch(const bool cannVerCheckSwitch)
252 : {
253 239 : cannVerCheckSwitch_ = cannVerCheckSwitch;
254 239 : return;
255 : }
256 :
257 : // private
258 77 : HcclResult RankConsistentcyChecker::RecordOpPara(HcclCMDType opCMD, const std::string &tag, u64 count,
259 : HcclDataType dataType, HcclReduceOp op, u32 root, u32 rank, u32 srTag, u32 selfRank, u64 inCclBufferSize,
260 : u64 outCclBufferSize, const char *group, u32 crc, u8 deterministic, u32 aivCoreLimit)
261 : {
262 77 : HcclCMDInfo cmdInfo;
263 : // 相关规范的例外场景，对固定数组的memset_s可以不判断返回值
264 77 : (void)memset_s(&cmdInfo, sizeof(cmdInfo), 0, sizeof(cmdInfo));
265 :
266 77 : cmdInfo.cmdType = opCMD;
267 77 : s32 sRet = strncpy_s(cmdInfo.tag, TAG_MAX_LEN + 1, tag.c_str(), tag.length());
268 77 : CHK_PRT_RET(sRet != EOK, HCCL_ERROR("[RankConsistentcyChecker][RecordOpPara]errNo[0x%016llx] strlen[%u] of tag is "
269 : "longer than buffer[%u].", HCCL_ERROR_CODE(HCCL_E_PARA), tag.length(), TAG_MAX_LEN), HCCL_E_PARA);
270 :
271 77 : cmdInfo.count = count;
272 77 : cmdInfo.dataType = dataType;
273 :
274 77 : std::string strGroup = (group == nullptr) ? HCCL_WORLD_GROUP : group;
275 77 : sRet = strncpy_s(cmdInfo.group, GROUP_NAME_MAX_LEN + 1, strGroup.c_str(), strGroup.length());
276 77 : CHK_PRT_RET(sRet != EOK, HCCL_ERROR("[RankConsistentcyChecker][RecordOpPara]errNo[0x%016llx] strlen[%u] group is "
277 : "longer than buffer[%u].", HCCL_ERROR_CODE(HCCL_E_PARA), strGroup.length(), GROUP_NAME_MAX_LEN), HCCL_E_PARA);
278 :
279 77 : cmdInfo.op = op;
280 77 : cmdInfo.root = root;
281 77 : cmdInfo.rank = rank;
282 77 : cmdInfo.srTag = srTag;
283 77 : cmdInfo.selfRank = selfRank;
284 77 : cmdInfo.inCclBufferSize = inCclBufferSize;
285 77 : cmdInfo.outCclBufferSize = outCclBufferSize;
286 77 : cmdInfo.aivCoreLimit = aivCoreLimit;
287 77 : cmdInfo.deterministic = deterministic;
288 :
289 77 : std::lock_guard<std::mutex> lock(mutex_);
290 77 : cmdInfoMap_[tag] = cmdInfo;
291 76 : infoFlagCmdMap_[tag] = true;
292 76 : crcRecords_[tag][HcclCrcRecordType::HCCL_CRC_RECORD_RANKTABLE] = crc;
293 :
294 76 : return HCCL_SUCCESS;
295 76 : }
296 :
297 0 : HcclResult RankConsistentcyChecker::GetOpParaByTag(const std::string &tag, HcclCMDInfo &CMDInfoOutput)
298 : {
299 0 : auto getResult = cmdInfoMap_.find(tag);
300 0 : CHK_PRT_RET(getResult == cmdInfoMap_.end(),
301 : HCCL_ERROR("[RankConsistentcyChecker][GetOpParaByTag]There is not any CMD information for tag[%s]",
302 : tag.c_str()), HCCL_E_INTERNAL);
303 0 : CMDInfoOutput = getResult->second;
304 0 : return HCCL_SUCCESS;
305 : }
306 :
307 0 : HcclResult RankConsistentcyChecker::GetCrcByTag(const std::string &tag, HcclCRCInfo &crcInfo)
308 : {
309 0 : const auto recordsIter = crcRecords_.find(tag);
310 0 : CHK_PRT_RET(recordsIter == crcRecords_.end(),
311 : HCCL_ERROR("[RankConsistentcyChecker][GetCrcByTag]There is not any CRC information for tag[%s]",
312 : tag.c_str()), HCCL_E_INTERNAL);
313 0 : crcInfo.crcNum = 0;
314 0 : const auto &tagRecords = recordsIter->second;
315 0 : for (const auto &record : tagRecords) {
316 0 : crcInfo.crcArray[crcInfo.crcNum++] = record.second;
317 0 : HCCL_DEBUG("[RankConsistentcyChecker][GetCrcByTag]Append crc[%u] for tag[%s].", record.second, tag.c_str());
318 : }
319 :
320 0 : HCCL_INFO("[RankConsistentcyChecker][GetCrcByTag]After adding crc for tag[%s], crcNum set to [%u].",
321 : tag.c_str(), crcInfo.crcNum);
322 0 : return HCCL_SUCCESS;
323 : }
324 :
325 3 : HcclResult RankConsistentcyChecker::GenerateCheckFrame(HcclCheckInfo &checkInfo, const std::string &tag)
326 : {
327 : // 初始化用于发送的BUFFER
328 : // 对入参的结构体变量指向的内存进行初始化时，使用了变量的结构体类型大小进行初始化，
329 : // 如果指针不为空，可以不检查初始化安全函数的返回值
330 3 : u64 checkInfoLen = sizeof(HcclCheckInfo);
331 3 : (void)memset_s(&checkInfo, checkInfoLen, 0, checkInfoLen);
332 :
333 : // 添加CRC字段到校验帧
334 4 : u32 crcLen = crcTable_.size();
335 4 : checkInfo.crcInfoGlobal.configFileExist_ = configFileExist_;
336 4 : if (crcLen != 0) {
337 0 : CHK_PRT_RET(crcLen > MAX_CRC_LEN,
338 : HCCL_ERROR("[RankConsistentcyChecker][GenerateCheckFrame]crc num[%u] is too big.", crcLen),
339 : HCCL_E_INTERNAL);
340 0 : checkInfo.crcInfoGlobal.crcNum = crcLen;
341 0 : CHK_RET(GetCrc(crcLen, &checkInfo.crcInfoGlobal.crcArray[0]));
342 : }
343 : // 添加CMD参数信息到校验帧
344 : {
345 4 : std::lock_guard<std::mutex> lock(mutex_);
346 4 : auto getResult = infoFlagCmdMap_.find(tag);
347 4 : if (getResult != infoFlagCmdMap_.end()) {
348 0 : CHK_PRT_RET(GetOpParaByTag(tag, checkInfo.cmdInfo) != HCCL_SUCCESS,
349 : HCCL_ERROR("[RankConsistentcyChecker][GenerateCheckFrame]get Op para by tag[%s] fail.", tag.c_str()),
350 : HCCL_E_INTERNAL);
351 0 : checkInfo.crcInfoOp.configFileExist_ = configFileExist_;
352 : // 添加CRC字段到校验帧
353 0 : CHK_PRT_RET(GetCrcByTag(tag, checkInfo.crcInfoOp) != HCCL_SUCCESS,
354 : HCCL_ERROR("[RankConsistentcyChecker][GenerateCheckFrame]get ranktable crc by tag[%s] fail.", tag.c_str()),
355 : HCCL_E_INTERNAL);
356 : }
357 4 : }
358 : // 添加HCCL版本信息到校验帧
359 4 : if (cannVerInfoRecordFlag_) {
360 0 : HCCL_DEBUG("[RankConsistentcyChecker][GenerateCheckFrame] CANN version information is [%s].", cannVersion_);
361 0 : s32 sret = memcpy_s(checkInfo.version, MAX_CANN_VERSION_LEN + 1, cannVersion_, strlen(cannVersion_));
362 0 : CHK_PRT_RET(sret != EOK,
363 : HCCL_ERROR("[RankConsistentcyChecker][GenerateCheckFrame] memcpy CANN version information failed, "
364 : "errorno [%d].", sret), HCCL_E_MEMORY);
365 : }
366 : // 添加拉远通信传输类型校验
367 : // 910* 不会配置isTcpMode，因此910*在此处的待校验值是一致的
368 4 : checkInfo.protocolType = protocolType_;
369 4 : HCCL_INFO("[RankConsistentcyChecker][GenerateCheckFrame] loc protocolType is [%d].", checkInfo.protocolType);
370 :
371 4 : return HCCL_SUCCESS;
372 : }
373 :
374 29 : bool RankConsistentcyChecker::CompareSection(const char *pRawData, const char *recvBuf, u32 len)
375 : {
376 11741 : for (u32 i = 0; i < len; i++) {
377 11715 : if (*(pRawData + i) != *(recvBuf + i)) {
378 3 : return false;
379 : }
380 : }
381 26 : return true;
382 : }
383 :
384 18 : bool RankConsistentcyChecker::CompareCrcInfo(const HcclCMDInfo &hcclCMDInfo, HcclCRCInfo &crcInfo, HcclCRCInfo &crcInfoRecv)
385 : {
386 18 : bool bIsDiff = false;
387 : // 检校验整体是否一致
388 18 : if (!CompareSection(reinterpret_cast<char_t *>(&crcInfo), reinterpret_cast<char_t *>(&crcInfoRecv), sizeof(crcInfo))) {
389 2 : bIsDiff = true;
390 : // 检查每种CRC类型是否一致
391 4 : for (auto i = 0U; i < crcInfo.crcNum; ++i) {
392 2 : if (crcInfo.crcArray[i] != crcInfoRecv.crcArray[i]) {
393 2 : ReportCrcCheckFailed(hcclCMDInfo, static_cast<HcclCrcRecordType>(i), crcInfo.crcArray[i],
394 : crcInfoRecv.crcArray[i]);
395 : }
396 : }
397 : }
398 18 : return bIsDiff;
399 : }
400 :
401 0 : void RankConsistentcyChecker::ReportCmdInfoCheckFailed(const HcclCMDInfo &hcclCMDInfo, const std::string ¶Name,
402 : const std::string &localPara, const std::string &remotePara) const
403 : {
404 0 : ReportCommonError(hcclCMDInfo, paraName, localPara, remotePara, "CMD information" );
405 0 : }
406 :
407 3 : void RankConsistentcyChecker::ReportCmdInfoCheckFailed(const HcclCMDInfo &hcclCMDInfo, const std::string ¶Name,
408 : uint32_t localPara, uint32_t remotePara)
409 : {
410 3 : ReportCommonError(hcclCMDInfo, paraName, std::to_string(localPara), std::to_string(remotePara), "CMD information");
411 3 : }
412 :
413 2 : void RankConsistentcyChecker::ReportCrcCheckFailed(const HcclCMDInfo &hcclCMDInfo, HcclCrcRecordType crcType,
414 : const uint32_t localCrc, const uint32_t remoteCrc) const
415 : {
416 2 : const auto crcTypeStr = GetCRCTypeEnumStr(crcType);
417 2 : ReportCommonError(hcclCMDInfo, crcTypeStr, std::to_string(localCrc), std::to_string(remoteCrc), "CRC check");
418 2 : }
419 :
420 5 : void RankConsistentcyChecker::ReportCommonError(const HcclCMDInfo &hcclCMDInfo, const std::string ¶Name,
421 : const std::string &localParaStr, const std::string &remoteParaStr, const std::string &errorMsg) const {
422 5 : std::string opInfo = "Unknown";
423 15 : for (const auto& pair : HCCL_OPTYPE_NAME_MAP) {
424 15 : if (pair.second == hcclCMDInfo.cmdType) {
425 5 : opInfo = std::string(pair.first);
426 5 : break;
427 : }
428 : }
429 100 : RPT_INPUT_ERR(true,
430 : "EI0005",
431 : std::vector<std::string>({"ccl_op", "group", "para_name", "local_para", "remote_para"}),
432 : std::vector<std::string>({opInfo, hcclCMDInfo.group, paraName, localParaStr, remoteParaStr}));
433 5 : HCCL_ERROR("[%s][%s]%s %s check fail. local[%s], remote[%s]",
434 : LOG_KEYWORDS_INIT_CHANNEL.c_str(),
435 : LOG_KEYWORDS_PARAMETER_CONFLICT.c_str(),
436 : errorMsg.c_str(),
437 : paraName.c_str(),
438 : localParaStr.c_str(),
439 : remoteParaStr.c_str());
440 15 : }
441 :
442 1 : void RankConsistentcyChecker::CompareCmdInfo(HcclCheckInfo &checkInfo, HcclCheckInfo &checkInfoRecv)
443 : {
444 1 : auto localInfo = &checkInfo.cmdInfo;
445 1 : auto remoteInfo = &checkInfoRecv.cmdInfo;
446 :
447 1 : if (!CompareSection(localInfo->tag, remoteInfo->tag, TAG_MAX_LEN + 1)) {
448 0 : ReportCmdInfoCheckFailed(*localInfo, "tag", localInfo->tag, remoteInfo->tag);
449 : }
450 :
451 1 : if (localInfo->cmdType != remoteInfo->cmdType) {
452 2 : ReportCmdInfoCheckFailed(*localInfo, "cmdType",
453 1 : static_cast<uint32_t>(localInfo->cmdType), static_cast<uint32_t>(remoteInfo->cmdType));
454 : }
455 :
456 1 : if (localInfo->count != remoteInfo->count) {
457 3 : ReportCmdInfoCheckFailed(*localInfo, "count", localInfo->count, remoteInfo->count);
458 : }
459 :
460 1 : if (localInfo->dataType != remoteInfo->dataType) {
461 2 : ReportCmdInfoCheckFailed(*localInfo, "dataType",
462 1 : static_cast<uint32_t>(localInfo->dataType), static_cast<uint32_t>(remoteInfo->dataType));
463 : }
464 :
465 1 : if (localInfo->op != remoteInfo->op) {
466 0 : ReportCmdInfoCheckFailed(*localInfo, "op", static_cast<uint32_t>(localInfo->op), static_cast<uint32_t>(remoteInfo->op));
467 : }
468 :
469 1 : if (!CompareSection(localInfo->group, remoteInfo->group, GROUP_NAME_MAX_LEN + 1)) {
470 0 : ReportCmdInfoCheckFailed(*localInfo, "group", localInfo->group, remoteInfo->group);
471 : }
472 :
473 1 : if (localInfo->root != remoteInfo->root) {
474 0 : ReportCmdInfoCheckFailed(*localInfo, "root", localInfo->root, remoteInfo->root);
475 : }
476 :
477 1 : if (localInfo->rank != remoteInfo->rank) {
478 0 : ReportCmdInfoCheckFailed(*localInfo, "rank", localInfo->rank, remoteInfo->rank);
479 : }
480 :
481 1 : if (localInfo->srTag != remoteInfo->srTag) {
482 0 : ReportCmdInfoCheckFailed(*localInfo, "srTag", localInfo->srTag, remoteInfo->srTag);
483 : }
484 :
485 1 : if (localInfo->inCclBufferSize != remoteInfo->inCclBufferSize) {
486 0 : ReportCmdInfoCheckFailed(*localInfo, "inCclBufferSize", localInfo->inCclBufferSize,
487 0 : remoteInfo->inCclBufferSize);
488 : }
489 :
490 1 : if (localInfo->outCclBufferSize != remoteInfo->outCclBufferSize) {
491 0 : ReportCmdInfoCheckFailed(*localInfo, "outCclBufferSize", localInfo->outCclBufferSize,
492 0 : remoteInfo->outCclBufferSize);
493 : }
494 :
495 1 : if (localInfo->aivCoreLimit != remoteInfo->aivCoreLimit) {
496 0 : ReportCmdInfoCheckFailed(*localInfo, "aivCoreLimit", localInfo->aivCoreLimit,
497 : remoteInfo->aivCoreLimit);
498 : }
499 :
500 1 : if (localInfo->deterministic != remoteInfo->deterministic) {
501 0 : ReportCmdInfoCheckFailed(*localInfo, "deterministic", localInfo->deterministic,
502 0 : remoteInfo->deterministic);
503 : }
504 :
505 1 : return;
506 : }
507 :
508 18 : std::vector<std::string_view> Split(const std::string &str, char delimiter)
509 : {
510 18 : std::vector<std::string_view> parts;
511 18 : std::string_view sv(str);
512 18 : size_t start = 0;
513 238 : for (size_t i = 0; i <= sv.size(); ++i) {
514 220 : if (i == sv.size() || sv[i] == delimiter) {
515 33 : parts.push_back(sv.substr(start, i - start));
516 33 : start = i + 1;
517 : }
518 : }
519 18 : return parts;
520 0 : }
521 :
522 9 : std::vector<std::string> GetVersionErrMessage(const std::string &localVersion, const std::string &remoteVersion)
523 : {
524 9 : std::vector<std::string> versionErrMessage;
525 9 : auto localVersionParts = Split(localVersion, '_');
526 9 : auto remoteVersionParts = Split(remoteVersion, '_');
527 18 : std::string localHcommVersion = localVersionParts.size() > 0 ? std::string(localVersionParts[0]) : "empty";
528 20 : std::string localHcclVersion = localVersionParts.size() > 1 ? std::string(localVersionParts[1]) : "empty";
529 18 : std::string remoteHcommVersion = remoteVersionParts.size() > 0 ? std::string(remoteVersionParts[0]) : "empty";
530 20 : std::string remoteHcclVersion = remoteVersionParts.size() > 1 ? std::string(remoteVersionParts[1]) : "empty";
531 :
532 9 : bool hcommDiff = (localHcommVersion != remoteHcommVersion);
533 9 : bool hcclDiff = (localHcclVersion != remoteHcclVersion);
534 9 : if (hcommDiff && !hcclDiff) {
535 : // 将版本号拆分为hcomm版本号和hccl版本号, 构造ErrMessage
536 : // The %s versions are inconsistent. The local %s, while the remote %s.
537 20 : versionErrMessage =
538 25 : {"Toolkit (cann-hcomm)", "version is " + localHcommVersion, "version is " + remoteHcommVersion};
539 4 : } else if (!hcommDiff && hcclDiff) {
540 5 : versionErrMessage = {"ops (cann-hccl)", "version is " + localHcclVersion, "version is " + remoteHcclVersion};
541 3 : } else if (hcommDiff && hcclDiff) {
542 8 : versionErrMessage = {
543 : "Toolkit (cann-hcomm) and ops (cann-hccl)",
544 2 : "Toolkit (cann-hcomm) version is " + localHcommVersion +
545 2 : " and ops (cann-hccl) version is " + localHcclVersion,
546 2 : "Toolkit (cann-hcomm) version is " + remoteHcommVersion +
547 2 : " and ops (cann-hccl) version is " + remoteHcclVersion
548 10 : };
549 : } else {
550 : // 拆分版本号字符串失败, 构造为原版的ErrMessage
551 : // The HCCL versions are inconsistent. The local version is %s, while the remote version is %s.
552 1 : HCCL_ERROR("[RankConsistentcyChecker][GetVersionErrMessage]Failed to split the version string.");
553 5 : versionErrMessage = {"HCCL", "version is " + localVersion, "version is " + remoteVersion};
554 : }
555 9 : return versionErrMessage;
556 31 : }
557 :
558 8 : void RankConsistentcyChecker::CompareVersion(const HcclCheckInfo &local, const HcclCheckInfo &remote, bool &isDiff)
559 : {
560 16 : std::string localCannVersion = local.version;
561 8 : std::string remoteCannVersion = remote.version;
562 8 : if (localCannVersion.empty() || remoteCannVersion.empty()) { // cann版本信息读取失败，返回告警
563 1 : HCCL_WARNING("[RankConsistentcyChecker][CompareFrame] CANN version str is empty. local_version %s, "
564 : "remote_version %s.", local.version, remote.version);
565 7 : } else if (localCannVersion != remoteCannVersion) { // cann版本信息读取成功，且版本不一致
566 5 : const char *ascendHomePath = std::getenv("ASCEND_HOME_PATH");
567 : std::string versionFilePath =
568 10 : (ascendHomePath != nullptr) ? std::string(ascendHomePath) + "/share/info/hccl/version.info" : "unknown";
569 5 : auto versionErrMessage = GetVersionErrMessage(localCannVersion, remoteCannVersion);
570 : // GetVersionErrMessage函数内部已保证versionErrMessage.size() == 3
571 70 : RPT_INPUT_ERR(true, "EI0008",
572 : std::vector<std::string>({"inconsistent_package", "local_version", "remote_version"}),
573 : std::vector<std::string>({versionErrMessage[0],
574 : versionErrMessage[1] + " (version file: " + versionFilePath + ")", versionErrMessage[2]}));
575 5 : HCCL_ERROR("[%s][%s] errNo[0x%016llx] Inconsistent %s Versions. The local %s, while the remote %s.",
576 : LOG_KEYWORDS_INIT_CHANNEL.c_str(), LOG_KEYWORDS_VERSION_CONFLICT.c_str(),
577 : HCCL_ERROR_CODE(HCCL_E_INTERNAL), versionErrMessage[0].c_str(),
578 : versionErrMessage[1].c_str(), versionErrMessage[2].c_str());
579 5 : isDiff = true;
580 5 : }
581 18 : }
582 :
583 9 : bool RankConsistentcyChecker::CompareFrame(HcclCheckInfo &checkInfo, HcclCheckInfo &checkInfoRecv)
584 : {
585 9 : bool bIsDiff = false;
586 9 : if (CompareCrcInfo(checkInfo.cmdInfo, checkInfo.crcInfoGlobal, checkInfoRecv.crcInfoGlobal)) {
587 1 : HCCL_ERROR("[RankConsistentcyChecker][CompareFrame]errNo[0x%016llx] CRC check fail, please check the "
588 : "rankTable file and hccl_config file.", HCCL_ERROR_CODE(HCCL_E_INTERNAL));
589 1 : bIsDiff = true;
590 : }
591 9 : if (CompareCrcInfo(checkInfo.cmdInfo, checkInfo.crcInfoOp, checkInfoRecv.crcInfoOp)) {
592 1 : HCCL_ERROR("[RankConsistentcyChecker][CompareFrame]errNo[0x%016llx] Op CRC check fail, please check the op"
593 : " parameters, rankTable file and hccl_config file.", HCCL_ERROR_CODE(HCCL_E_INTERNAL));
594 1 : bIsDiff = true;
595 : }
596 9 : if (!CompareSection(reinterpret_cast<char_t *>(&checkInfo.cmdInfo),
597 9 : reinterpret_cast<char_t *>(&checkInfoRecv.cmdInfo), sizeof(checkInfo.cmdInfo))) {
598 1 : CompareCmdInfo(checkInfo, checkInfoRecv);
599 1 : HCCL_ERROR("[RankConsistentcyChecker][CompareFrame]errNo[0x%016llx] CMD check fail",
600 : HCCL_ERROR_CODE(HCCL_E_INTERNAL));
601 1 : bIsDiff = true;
602 : }
603 9 : HCCL_INFO("loc protocolType is [%d], rem protocolType is [%d].",
604 : checkInfo.protocolType, checkInfoRecv.protocolType);
605 9 : if (checkInfo.protocolType != checkInfoRecv.protocolType) {
606 1 : HCCL_ERROR("[RankConsistentcyChecker][CompareFrame]errNo[0x%016llx] ProtocolType check fail",
607 : HCCL_ERROR_CODE(HCCL_E_INTERNAL));
608 1 : bIsDiff = true;
609 : }
610 :
611 : // Cann版本校验，只在集合通信场景校验CANN版本
612 9 : if (cannVerCheckSwitch_) {
613 8 : CompareVersion(checkInfo, checkInfoRecv, bIsDiff);
614 : }
615 : #ifndef HCCD
616 9 : if (GetExternalInconsistentCheckSwitch() == InconsistentCheckMode::FIRST) {
617 9 : inconsistentCheckFirstDone_ = true;
618 : }
619 : #endif
620 9 : return bIsDiff;
621 : }
622 :
623 9 : HcclResult RankConsistentcyChecker::AddCrc(const u32 crcValue)
624 : {
625 9 : HCCL_DEBUG("crcValue[%u].", crcValue);
626 9 : crcTable_.push_back(crcValue);
627 9 : HCCL_DEBUG("num[%llu].", crcTable_.size());
628 9 : return HCCL_SUCCESS;
629 : }
630 :
631 63 : HcclResult RankConsistentcyChecker::ClearCrcInfo(void)
632 : {
633 63 : this->crcTable_.clear();
634 63 : if (this->crcTable_.size() != 0) {
635 0 : HCCL_ERROR("[Clear][CrcInfo]errNo[0x%016llx] clear crcTable_ is failed", HCCL_ERROR_CODE(HCCL_E_INTERNAL));
636 0 : return HCCL_E_INTERNAL;
637 : }
638 63 : return HCCL_SUCCESS;
639 : }
640 :
641 1 : HcclResult RankConsistentcyChecker::GetCrc(u32 num, u32 *crcAddr)
642 : {
643 1 : CHK_PTR_NULL(crcAddr);
644 1 : HCCL_DEBUG("num[%u], crc[%u].", num, *crcAddr);
645 :
646 1 : if (num == 0) {
647 0 : HCCL_ERROR("[Get][Crc]errNo[0x%016llx] In get crc the value of num is 0", HCCL_ERROR_CODE(HCCL_E_PARA));
648 0 : return HCCL_E_PARA;
649 : }
650 :
651 1 : if (num != crcTable_.size()) {
652 0 : HCCL_ERROR("[Get][Crc]errNo[0x%016llx] num error inputNum[%u], localNum[%llu]",
653 : HCCL_ERROR_CODE(HCCL_E_INTERNAL), num, crcTable_.size());
654 0 : return HCCL_E_INTERNAL;
655 : }
656 :
657 4 : for (u32 i = 0; i < num; i++) {
658 3 : crcAddr[i] = crcTable_[i];
659 : }
660 1 : return HCCL_SUCCESS;
661 : }
662 : }
|
