Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "calc_crc.h"
12 : #include "rank_consistentcy_checker.h"
13 : #include "env_config.h"
14 :
15 : namespace hccl {
16 :
17 1178 : RankConsistentcyChecker::RankConsistentcyChecker() : cannVersion_{0}, cannVerCheckSwitch_(false),
18 589 : cannVerInfoRecordFlag_(false), configFileExist_(false)
19 : {
20 589 : }
21 :
22 589 : RankConsistentcyChecker::~RankConsistentcyChecker() = default;
23 :
24 1920 : RankConsistentcyChecker& RankConsistentcyChecker::GetInstance(s32 deviceLogicId)
25 : {
26 2505 : static RankConsistentcyChecker instance[MAX_MODULE_DEVICE_NUM];
27 1920 : if (deviceLogicId == HOST_DEVICE_ID) {
28 0 : HCCL_INFO("[GetInstance] deviceLogicId[-1] is HOST_DEVICE_ID");
29 0 : return instance[0];
30 : }
31 1920 : hrtGetDeviceRefresh(&deviceLogicId);
32 1923 : HCCL_INFO("[GetInstance] get deviceLogicId[%d]", deviceLogicId);
33 1923 : CHK_PRT_RET((static_cast<u32>(deviceLogicId) >= MAX_MODULE_DEVICE_NUM || deviceLogicId < 0),
34 : HCCL_WARNING("[R]deviceLogicId[%d] is invalid", deviceLogicId), instance[0]);
35 :
36 1921 : return instance[deviceLogicId];
37 : }
38 :
39 : // gather
40 0 : HcclResult RankConsistentcyChecker::RecordOpPara(HcclCMDType opCMD, const std::string &tag, u64 count,
41 : HcclDataType dataType, u32 root, u64 inCclBufferSize, u64 outCclBufferSize, const char *group, u32 crc,
42 : u32 aivCoreLimit)
43 : {
44 0 : return RecordOpPara(opCMD, tag, count, dataType, HCCL_REDUCE_RESERVED, root, 0, 0, 0, inCclBufferSize,
45 0 : outCclBufferSize, group, crc, aivCoreLimit);
46 : }
47 :
48 : // reduce
49 77 : HcclResult RankConsistentcyChecker::RecordOpPara(HcclCMDType opCMD, const std::string &tag, u64 count,
50 : HcclDataType dataType, HcclReduceOp op, u32 root, u64 inCclBufferSize, u64 outCclBufferSize,
51 : const char *group, u32 crc, u8 deterministic, u32 aivCoreLimit)
52 : {
53 77 : return RecordOpPara(opCMD, tag, count, dataType, op, root, 0, 0, 0, inCclBufferSize, outCclBufferSize, group, crc, aivCoreLimit);
54 : }
55 :
56 : // send && receive
57 0 : HcclResult RankConsistentcyChecker::RecordOpPara(HcclCMDType opCMD, const std::string &tag, u64 count,
58 : HcclDataType dataType, u32 rank, u32 srTag, u32 selfRank, u64 inCclBufferSize, u64 outCclBufferSize,
59 : const char *group, u32 crc)
60 : {
61 0 : return RecordOpPara(opCMD, tag, count, dataType, HCCL_REDUCE_RESERVED, 0, rank, srTag, selfRank,
62 0 : inCclBufferSize, outCclBufferSize, group, crc);
63 : }
64 :
65 : // batchsendrecv
66 0 : HcclResult RankConsistentcyChecker::RecordOpPara(HcclCMDType opCMD, const std::string &tag,
67 : u64 inCclBufferSize, u64 outCclBufferSize, const char *group, u32 crc)
68 : {
69 0 : return RecordOpPara(opCMD, tag, 0, HCCL_DATA_TYPE_RESERVED, HCCL_REDUCE_RESERVED, 0, 0, 0, 0, inCclBufferSize,
70 0 : outCclBufferSize, group, crc);
71 : }
72 :
73 : // reduce scatter v && AllGather v
74 0 : HcclResult RankConsistentcyChecker::RecordOpPara(HcclCMDType opCMD, const std::string &tag,
75 : const void *counts, const void *displs, const u32 rankSize,
76 : HcclDataType dataType, HcclReduceOp op, u64 inCclBufferSize, u64 outCclBufferSize, const char *group, u32 crc, u8 deterministic,
77 : u32 aivCoreLimit)
78 : {
79 0 : CHK_RET(RecordOpPara(opCMD, tag, 0, dataType, op, 0, 0, 0, 0, inCclBufferSize,
80 : outCclBufferSize, group, crc, aivCoreLimit));
81 0 : CHK_RET(RecordVaringOpPara(tag, counts, displs, rankSize));
82 0 : return HCCL_SUCCESS;
83 : }
84 :
85 0 : HcclResult RankConsistentcyChecker::RecordVaringOpPara(const std::string &tag, const void *counts, const void *displs,
86 : const u32 rankSize)
87 : {
88 : u32 countsCrc;
89 0 : CHK_RET(CalcRawDataCrc(static_cast<const char_t*>(counts), rankSize * sizeof(u64), countsCrc));
90 0 : crcRecords_[tag][HcclCrcRecordType::HCCL_CRC_RECORD_VARING_COUNTS] = countsCrc;
91 :
92 : u32 displsCrc;
93 0 : CHK_RET(CalcRawDataCrc(static_cast<const char_t*>(displs), rankSize * sizeof(u64), displsCrc));
94 0 : crcRecords_[tag][HcclCrcRecordType::HCCL_CRC_RECORD_VARING_DISPLACEMENTS] = displsCrc;
95 0 : return HCCL_SUCCESS;
96 : }
97 :
98 24 : HcclResult RankConsistentcyChecker::DelOpPara(const std::string &tag)
99 : {
100 24 : std::lock_guard<std::mutex> lock(mutex_);
101 24 : CHK_PRT_RET(!cmdInfoMap_.erase(tag),
102 : HCCL_ERROR("[RankConsistentcyChecker][DelOpPara]CMD info for tag[%s] does not exist, delete fail.",
103 : tag.c_str()), HCCL_E_INTERNAL);
104 24 : CHK_PRT_RET(!infoFlagCmdMap_.erase(tag),
105 : HCCL_ERROR("[RankConsistentcyChecker][DelOpPara]CMD info flag cmd for tag[%s] does not exist, delete fail.",
106 : tag.c_str()), HCCL_E_INTERNAL);
107 24 : crcRecords_.erase(tag);
108 24 : return HCCL_SUCCESS;
109 24 : }
110 :
111 235 : HcclResult RankConsistentcyChecker::RecordVerInfo(const std::string &versionInfo)
112 : {
113 235 : std::lock_guard<std::mutex> lock(mutex_);
114 : // Only record once, in case of multiple calls while other process is reading CANN version information
115 : // in CompareFrame func and get the intermediate state.
116 235 : if (cannVerInfoRecordFlag_) {
117 230 : HCCL_INFO("[RankConsistentcyChecker][RecordVerInfo]Cann version information has been recorded.");
118 230 : return HCCL_SUCCESS;
119 : }
120 :
121 5 : u32 strLen = versionInfo.length();
122 5 : s32 sRet = memset_s(cannVersion_, MAX_CANN_VERSION_LEN + 1, 0, MAX_CANN_VERSION_LEN + 1);
123 5 : CHK_PRT_RET(sRet != EOK, HCCL_WARNING("[RankConsistentcyChecker][RecordVerInfo]memory set 0 fail for version str "
124 : "array. return[%d].", sRet), HCCL_SUCCESS);
125 :
126 5 : CHK_PRT_RET(strLen == 0, HCCL_WARNING("[Record][CannVersion] version information str is empty."),
127 : HCCL_SUCCESS);
128 :
129 5 : CHK_PRT_RET(strLen >= MAX_CANN_VERSION_LEN, HCCL_WARNING("[Record][CannVersion]"
130 : "length of version information str is too long."), HCCL_SUCCESS);
131 5 : sRet = strncpy_s(cannVersion_, MAX_CANN_VERSION_LEN + 1, versionInfo.c_str(), strLen);
132 5 : CHK_PRT_RET(sRet != EOK, HCCL_WARNING("[Record][CannVersion] call strncpy_s failed, return [%d].", sRet),
133 : HCCL_SUCCESS);
134 :
135 5 : cannVerInfoRecordFlag_ = true;
136 5 : return HCCL_SUCCESS;
137 235 : }
138 :
139 1049 : u64 RankConsistentcyChecker::GetRankConsistentDataLength()
140 : {
141 1049 : return sizeof(HcclCheckInfo);
142 : }
143 :
144 0 : void RankConsistentcyChecker::RecordProtocolType(ProtocolType protocolType)
145 : {
146 0 : HCCL_INFO("[RankConsistentcyChecker][RecordProtocolType]protocolType set to [%d].",
147 : static_cast<s32>(protocolType));
148 0 : protocolType_ = protocolType;
149 0 : return;
150 : }
151 :
152 4 : HcclResult RankConsistentcyChecker::GetCheckFrame(u8 *destBuf, u64 maxDestBuf, const std::string &tag)
153 : {
154 4 : CHK_PTR_NULL(destBuf);
155 : // 要发送的校验帧
156 4 : HcclCheckInfo checkInfo;
157 4 : u64 checkInfoLen = sizeof(checkInfo);
158 4 : HcclResult ret = GenerateCheckFrame(checkInfo, tag);
159 4 : checkInfo.cmdInfo.selfRank = 0; // 自身的group rank 不做校验,置0
160 4 : CHK_PRT_RET(ret != HCCL_SUCCESS, HCCL_ERROR("[RankConsistentcyChecker][GetCheckFrame]generate check frame fail. "
161 : "return[%d]", ret), ret);
162 :
163 4 : s32 sret = memcpy_s(destBuf, maxDestBuf, &checkInfo, checkInfoLen);
164 4 : CHK_PRT_RET(sret != EOK, HCCL_ERROR("[RankConsistentcyChecker][GetCheckFrame]frame len[%llu] is bigger than "
165 : "dest buffer len[%llu].", checkInfoLen, maxDestBuf), HCCL_E_INTERNAL);
166 4 : return HCCL_SUCCESS;
167 : }
168 :
169 0 : HcclResult RankConsistentcyChecker::CheckFrameRecv(const u8 *recvBuf, u32 recvBufLen, const std::string &tag)
170 : {
171 : #ifndef HCCD
172 0 : if ((GetExternalInconsistentCheckSwitch() == InconsistentCheckMode::OFF) ||
173 0 : (GetExternalInconsistentCheckSwitch() == InconsistentCheckMode::FIRST && inconsistentCheckFirstDone_ == true)) {
174 0 : return HCCL_SUCCESS;
175 : }
176 : #endif
177 0 : CHK_PTR_NULL(recvBuf);
178 0 : CHK_PRT_RET(recvBufLen == 0 || recvBufLen > MAX_FRAME_LEN,
179 : HCCL_ERROR("[RankConsistentcyChecker][CheckFrameRecv] errNo[0x%016llx] recvBufLen is wrong.",
180 : HCCL_ERROR_CODE(HCCL_E_INTERNAL)), HCCL_E_INTERNAL);
181 :
182 0 : CHK_PRT_RET(recvBufLen < sizeof(HcclCheckInfo),
183 : HCCL_ERROR("[RankConsistentcyChecker][CheckFrameRecv] errNo[0x%016llx] recvBufLen[%u]is less than "
184 : "check info[%zu].", HCCL_ERROR_CODE(HCCL_E_INTERNAL), recvBufLen, sizeof(HcclCheckInfo)), HCCL_E_PARA);
185 :
186 0 : HcclCheckInfo checkInfoRecv;
187 : // 对固定长度的全局数组变量，结构体变量进行初始化和拷贝，可以不用检查初始化安全函数返回值
188 0 : (void)memset_s(&checkInfoRecv, sizeof(HcclCheckInfo), 0, sizeof(HcclCheckInfo));
189 0 : (void)memcpy_s(&checkInfoRecv, sizeof(HcclCheckInfo), recvBuf, sizeof(HcclCheckInfo));
190 :
191 0 : HcclCheckInfo checkInfo;
192 0 : CHK_RET(GenerateCheckFrame(checkInfo, tag));
193 0 : if (checkInfo.cmdInfo.cmdType == HcclCMDType::HCCL_CMD_SEND) {
194 0 : checkInfo.cmdInfo.cmdType = HcclCMDType::HCCL_CMD_RECEIVE;
195 0 : checkInfo.cmdInfo.rank = checkInfo.cmdInfo.selfRank;
196 0 : } else if (checkInfo.cmdInfo.cmdType == HcclCMDType::HCCL_CMD_RECEIVE) {
197 0 : checkInfo.cmdInfo.cmdType = HcclCMDType::HCCL_CMD_SEND;
198 0 : checkInfo.cmdInfo.rank = checkInfo.cmdInfo.selfRank;
199 : }
200 :
201 0 : checkInfo.cmdInfo.selfRank = 0; // 自身的子group rank 不做校验
202 0 : if (CompareFrame(checkInfo, checkInfoRecv)) {
203 0 : return HCCL_E_INTERNAL;
204 : }
205 :
206 0 : HCCL_INFO("[RankConsistentcyChecker][CheckFrameRecv] check success, len of frame[%u], len of check data[%zu].",
207 : recvBufLen, sizeof(checkInfo));
208 0 : return HCCL_SUCCESS;
209 : }
210 :
211 62 : void RankConsistentcyChecker::ClearCheckInfo()
212 : {
213 62 : configFileExist_ = false;
214 62 : cannVerInfoRecordFlag_ = false;
215 62 : ClearCrcInfo();
216 : {
217 62 : std::lock_guard<std::mutex> lock(mutex_);
218 62 : cmdInfoMap_.clear();
219 62 : infoFlagCmdMap_.clear();
220 62 : }
221 : // 相关规范的例外场景，对固定数组的memset_s可以不判断返回值
222 62 : (void)memset_s(cannVersion_, MAX_CANN_VERSION_LEN + 1, 0, MAX_CANN_VERSION_LEN + 1);
223 62 : protocolType_ = ProtocolType::RESERVED;
224 62 : return;
225 : }
226 :
227 235 : HcclResult RankConsistentcyChecker::CalcStringCrc(const char *str, u32 &crc)
228 : {
229 : // 计算字符串CRC
230 235 : HcclResult ret = CalcCrc::HcclCalcCrc(str, strlen(str), crc);
231 235 : CHK_PRT_RET(ret != HCCL_SUCCESS,
232 : HCCL_ERROR("[RankConsistentcyChecker][CalcStringCrc] errNo[0x%016llx] calc string crc error",
233 : HCCL_ERROR_CODE(HCCL_E_INTERNAL)), HCCL_E_INTERNAL);
234 :
235 235 : HCCL_DEBUG("[RankConsistentcyChecker][CalcStringCrc] result crc[%u].", crc);
236 235 : return HCCL_SUCCESS;
237 : }
238 :
239 0 : HcclResult RankConsistentcyChecker::CalcRawDataCrc(const void *ptr, u64 length, u32 &crc)
240 : {
241 : // 计算内存数据块CRC
242 0 : HcclResult ret = CalcCrc::HcclCalcCrc(static_cast<const char*>(ptr), length, crc);
243 0 : CHK_PRT_RET(ret != HCCL_SUCCESS,
244 : HCCL_ERROR("[RankConsistentcyChecker][CalcRawDataCrc] errNo[0x%016llx] calc string crc error",
245 : HCCL_ERROR_CODE(HCCL_E_INTERNAL)), HCCL_E_INTERNAL);
246 :
247 0 : HCCL_DEBUG("[RankConsistentcyChecker][CalcRawDataCrc] result crc[%u].", crc);
248 0 : return HCCL_SUCCESS;
249 : }
250 :
251 237 : void RankConsistentcyChecker::SetCheckCannVersionSwitch(const bool cannVerCheckSwitch)
252 : {
253 237 : cannVerCheckSwitch_ = cannVerCheckSwitch;
254 237 : return;
255 : }
256 :
257 : // private
258 77 : HcclResult RankConsistentcyChecker::RecordOpPara(HcclCMDType opCMD, const std::string &tag, u64 count,
259 : HcclDataType dataType, HcclReduceOp op, u32 root, u32 rank, u32 srTag, u32 selfRank, u64 inCclBufferSize,
260 : u64 outCclBufferSize, const char *group, u32 crc, u8 deterministic, u32 aivCoreLimit)
261 : {
262 77 : HcclCMDInfo cmdInfo;
263 : // 相关规范的例外场景，对固定数组的memset_s可以不判断返回值
264 77 : (void)memset_s(&cmdInfo, sizeof(cmdInfo), 0, sizeof(cmdInfo));
265 :
266 77 : cmdInfo.cmdType = opCMD;
267 77 : s32 sRet = strncpy_s(cmdInfo.tag, TAG_MAX_LEN + 1, tag.c_str(), tag.length());
268 77 : CHK_PRT_RET(sRet != EOK, HCCL_ERROR("[RankConsistentcyChecker][RecordOpPara]errNo[0x%016llx] strlen[%u] of tag is "
269 : "longer than buffer[%u].", HCCL_ERROR_CODE(HCCL_E_PARA), tag.length(), TAG_MAX_LEN), HCCL_E_PARA);
270 :
271 77 : cmdInfo.count = count;
272 77 : cmdInfo.dataType = dataType;
273 :
274 77 : std::string strGroup = (group == nullptr) ? HCCL_WORLD_GROUP : group;
275 76 : sRet = strncpy_s(cmdInfo.group, GROUP_NAME_MAX_LEN + 1, strGroup.c_str(), strGroup.length());
276 76 : CHK_PRT_RET(sRet != EOK, HCCL_ERROR("[RankConsistentcyChecker][RecordOpPara]errNo[0x%016llx] strlen[%u] group is "
277 : "longer than buffer[%u].", HCCL_ERROR_CODE(HCCL_E_PARA), strGroup.length(), GROUP_NAME_MAX_LEN), HCCL_E_PARA);
278 :
279 76 : cmdInfo.op = op;
280 76 : cmdInfo.root = root;
281 76 : cmdInfo.rank = rank;
282 76 : cmdInfo.srTag = srTag;
283 76 : cmdInfo.selfRank = selfRank;
284 76 : cmdInfo.inCclBufferSize = inCclBufferSize;
285 76 : cmdInfo.outCclBufferSize = outCclBufferSize;
286 76 : cmdInfo.aivCoreLimit = aivCoreLimit;
287 76 : cmdInfo.deterministic = deterministic;
288 :
289 76 : std::lock_guard<std::mutex> lock(mutex_);
290 76 : cmdInfoMap_[tag] = cmdInfo;
291 75 : infoFlagCmdMap_[tag] = true;
292 73 : crcRecords_[tag][HcclCrcRecordType::HCCL_CRC_RECORD_RANKTABLE] = crc;
293 :
294 72 : return HCCL_SUCCESS;
295 72 : }
296 :
297 0 : HcclResult RankConsistentcyChecker::GetOpParaByTag(const std::string &tag, HcclCMDInfo &CMDInfoOutput)
298 : {
299 0 : auto getResult = cmdInfoMap_.find(tag);
300 0 : CHK_PRT_RET(getResult == cmdInfoMap_.end(),
301 : HCCL_ERROR("[RankConsistentcyChecker][GetOpParaByTag]There is not any CMD information for tag[%s]",
302 : tag.c_str()), HCCL_E_INTERNAL);
303 0 : CMDInfoOutput = getResult->second;
304 0 : return HCCL_SUCCESS;
305 : }
306 :
307 0 : HcclResult RankConsistentcyChecker::GetCrcByTag(const std::string &tag, HcclCRCInfo &crcInfo)
308 : {
309 0 : const auto recordsIter = crcRecords_.find(tag);
310 0 : CHK_PRT_RET(recordsIter == crcRecords_.end(),
311 : HCCL_ERROR("[RankConsistentcyChecker][GetCrcByTag]There is not any CRC information for tag[%s]",
312 : tag.c_str()), HCCL_E_INTERNAL);
313 0 : crcInfo.crcNum = 0;
314 0 : const auto &tagRecords = recordsIter->second;
315 0 : for (const auto &record : tagRecords) {
316 0 : crcInfo.crcArray[crcInfo.crcNum++] = record.second;
317 0 : HCCL_DEBUG("[RankConsistentcyChecker][GetCrcByTag]Append crc[%u] for tag[%s].", record.second, tag.c_str());
318 : }
319 :
320 0 : HCCL_INFO("[RankConsistentcyChecker][GetCrcByTag]After adding crc for tag[%s], crcNum set to [%u].",
321 : tag.c_str(), crcInfo.crcNum);
322 0 : return HCCL_SUCCESS;
323 : }
324 :
325 3 : HcclResult RankConsistentcyChecker::GenerateCheckFrame(HcclCheckInfo &checkInfo, const std::string &tag)
326 : {
327 : // 初始化用于发送的BUFFER
328 : // 对入参的结构体变量指向的内存进行初始化时，使用了变量的结构体类型大小进行初始化，
329 : // 如果指针不为空，可以不检查初始化安全函数的返回值
330 3 : u64 checkInfoLen = sizeof(HcclCheckInfo);
331 3 : (void)memset_s(&checkInfo, checkInfoLen, 0, checkInfoLen);
332 :
333 : // 添加CRC字段到校验帧
334 4 : u32 crcLen = crcTable_.size();
335 4 : checkInfo.crcInfoGlobal.configFileExist_ = configFileExist_;
336 4 : if (crcLen != 0) {
337 0 : CHK_PRT_RET(crcLen > MAX_CRC_LEN,
338 : HCCL_ERROR("[RankConsistentcyChecker][GenerateCheckFrame]crc num[%u] is too big.", crcLen),
339 : HCCL_E_INTERNAL);
340 0 : checkInfo.crcInfoGlobal.crcNum = crcLen;
341 0 : CHK_RET(GetCrc(crcLen, &checkInfo.crcInfoGlobal.crcArray[0]));
342 : }
343 : // 添加CMD参数信息到校验帧
344 : {
345 4 : std::lock_guard<std::mutex> lock(mutex_);
346 4 : auto getResult = infoFlagCmdMap_.find(tag);
347 4 : if (getResult != infoFlagCmdMap_.end()) {
348 0 : CHK_PRT_RET(GetOpParaByTag(tag, checkInfo.cmdInfo) != HCCL_SUCCESS,
349 : HCCL_ERROR("[RankConsistentcyChecker][GenerateCheckFrame]get Op para by tag[%s] fail.", tag.c_str()),
350 : HCCL_E_INTERNAL);
351 0 : checkInfo.crcInfoOp.configFileExist_ = configFileExist_;
352 : // 添加CRC字段到校验帧
353 0 : CHK_PRT_RET(GetCrcByTag(tag, checkInfo.crcInfoOp) != HCCL_SUCCESS,
354 : HCCL_ERROR("[RankConsistentcyChecker][GenerateCheckFrame]get ranktable crc by tag[%s] fail.", tag.c_str()),
355 : HCCL_E_INTERNAL);
356 : }
357 4 : }
358 : // 添加HCCL版本信息到校验帧
359 4 : if (cannVerInfoRecordFlag_) {
360 0 : HCCL_DEBUG("[RankConsistentcyChecker][GenerateCheckFrame] CANN version information is [%s].", cannVersion_);
361 0 : s32 sret = memcpy_s(checkInfo.version, MAX_CANN_VERSION_LEN + 1, cannVersion_, strlen(cannVersion_));
362 0 : CHK_PRT_RET(sret != EOK,
363 : HCCL_ERROR("[RankConsistentcyChecker][GenerateCheckFrame] memcpy CANN version information failed, "
364 : "errorno [%d].", sret), HCCL_E_MEMORY);
365 : }
366 : // 添加拉远通信传输类型校验
367 : // 910* 不会配置isTcpMode，因此910*在此处的待校验值是一致的
368 4 : checkInfo.protocolType = protocolType_;
369 4 : HCCL_INFO("[RankConsistentcyChecker][GenerateCheckFrame] loc protocolType is [%d].", checkInfo.protocolType);
370 :
371 4 : return HCCL_SUCCESS;
372 : }
373 :
374 14 : bool RankConsistentcyChecker::CompareSection(const char *pRawData, const char *recvBuf, u32 len)
375 : {
376 4606 : for (u32 i = 0; i < len; i++) {
377 4595 : if (*(pRawData + i) != *(recvBuf + i)) {
378 3 : return false;
379 : }
380 : }
381 11 : return true;
382 : }
383 :
384 8 : bool RankConsistentcyChecker::CompareCrcInfo(const HcclCMDInfo &hcclCMDInfo, HcclCRCInfo &crcInfo, HcclCRCInfo &crcInfoRecv)
385 : {
386 8 : bool bIsDiff = false;
387 : // 检校验整体是否一致
388 8 : if (!CompareSection(reinterpret_cast<char_t *>(&crcInfo), reinterpret_cast<char_t *>(&crcInfoRecv), sizeof(crcInfo))) {
389 2 : bIsDiff = true;
390 : // 检查每种CRC类型是否一致
391 4 : for (auto i = 0U; i < crcInfo.crcNum; ++i) {
392 2 : if (crcInfo.crcArray[i] != crcInfoRecv.crcArray[i]) {
393 2 : ReportCrcCheckFailed(hcclCMDInfo, static_cast<HcclCrcRecordType>(i), crcInfo.crcArray[i],
394 : crcInfoRecv.crcArray[i]);
395 : }
396 : }
397 : }
398 8 : return bIsDiff;
399 : }
400 :
401 0 : void RankConsistentcyChecker::ReportCmdInfoCheckFailed(const HcclCMDInfo &hcclCMDInfo, const std::string ¶Name,
402 : const std::string &localPara, const std::string &remotePara) const
403 : {
404 0 : ReportCommonError(hcclCMDInfo, paraName, localPara, remotePara, "CMD information" );
405 0 : }
406 :
407 3 : void RankConsistentcyChecker::ReportCmdInfoCheckFailed(const HcclCMDInfo &hcclCMDInfo, const std::string ¶Name,
408 : uint32_t localPara, uint32_t remotePara)
409 : {
410 3 : ReportCommonError(hcclCMDInfo, paraName, std::to_string(localPara), std::to_string(remotePara), "CMD information");
411 3 : }
412 :
413 2 : void RankConsistentcyChecker::ReportCrcCheckFailed(const HcclCMDInfo &hcclCMDInfo, HcclCrcRecordType crcType,
414 : const uint32_t localCrc, const uint32_t remoteCrc) const
415 : {
416 2 : const auto crcTypeStr = GetCRCTypeEnumStr(crcType);
417 2 : ReportCommonError(hcclCMDInfo, crcTypeStr, std::to_string(localCrc), std::to_string(remoteCrc), "CRC check");
418 2 : }
419 :
420 5 : void RankConsistentcyChecker::ReportCommonError(const HcclCMDInfo &hcclCMDInfo, const std::string ¶Name,
421 : const std::string &localParaStr, const std::string &remoteParaStr, const std::string &errorMsg) const {
422 5 : std::string opInfo = "Unknown";
423 15 : for (const auto& pair : HCCL_OPTYPE_NAME_MAP) {
424 15 : if (pair.second == hcclCMDInfo.cmdType) {
425 5 : opInfo = std::string(pair.first);
426 5 : break;
427 : }
428 : }
429 100 : RPT_INPUT_ERR(true,
430 : "EI0005",
431 : std::vector<std::string>({"ccl_op", "group", "para_name", "local_para", "remote_para"}),
432 : std::vector<std::string>({opInfo, hcclCMDInfo.group, paraName, localParaStr, remoteParaStr}));
433 5 : HCCL_ERROR("[%s][%s]%s %s check fail. local[%s], remote[%s]",
434 : LOG_KEYWORDS_INIT_CHANNEL.c_str(),
435 : LOG_KEYWORDS_PARAMETER_CONFLICT.c_str(),
436 : errorMsg.c_str(),
437 : paraName.c_str(),
438 : localParaStr.c_str(),
439 : remoteParaStr.c_str());
440 15 : }
441 :
442 1 : void RankConsistentcyChecker::CompareCmdInfo(HcclCheckInfo &checkInfo, HcclCheckInfo &checkInfoRecv)
443 : {
444 1 : auto localInfo = &checkInfo.cmdInfo;
445 1 : auto remoteInfo = &checkInfoRecv.cmdInfo;
446 :
447 1 : if (!CompareSection(localInfo->tag, remoteInfo->tag, TAG_MAX_LEN + 1)) {
448 0 : ReportCmdInfoCheckFailed(*localInfo, "tag", localInfo->tag, remoteInfo->tag);
449 : }
450 :
451 1 : if (localInfo->cmdType != remoteInfo->cmdType) {
452 2 : ReportCmdInfoCheckFailed(*localInfo, "cmdType",
453 1 : static_cast<uint32_t>(localInfo->cmdType), static_cast<uint32_t>(remoteInfo->cmdType));
454 : }
455 :
456 1 : if (localInfo->count != remoteInfo->count) {
457 3 : ReportCmdInfoCheckFailed(*localInfo, "count", localInfo->count, remoteInfo->count);
458 : }
459 :
460 1 : if (localInfo->dataType != remoteInfo->dataType) {
461 2 : ReportCmdInfoCheckFailed(*localInfo, "dataType",
462 1 : static_cast<uint32_t>(localInfo->dataType), static_cast<uint32_t>(remoteInfo->dataType));
463 : }
464 :
465 1 : if (localInfo->op != remoteInfo->op) {
466 0 : ReportCmdInfoCheckFailed(*localInfo, "op", static_cast<uint32_t>(localInfo->op), static_cast<uint32_t>(remoteInfo->op));
467 : }
468 :
469 1 : if (!CompareSection(localInfo->group, remoteInfo->group, GROUP_NAME_MAX_LEN + 1)) {
470 0 : ReportCmdInfoCheckFailed(*localInfo, "group", localInfo->group, remoteInfo->group);
471 : }
472 :
473 1 : if (localInfo->root != remoteInfo->root) {
474 0 : ReportCmdInfoCheckFailed(*localInfo, "root", localInfo->root, remoteInfo->root);
475 : }
476 :
477 1 : if (localInfo->rank != remoteInfo->rank) {
478 0 : ReportCmdInfoCheckFailed(*localInfo, "rank", localInfo->rank, remoteInfo->rank);
479 : }
480 :
481 1 : if (localInfo->srTag != remoteInfo->srTag) {
482 0 : ReportCmdInfoCheckFailed(*localInfo, "srTag", localInfo->srTag, remoteInfo->srTag);
483 : }
484 :
485 1 : if (localInfo->inCclBufferSize != remoteInfo->inCclBufferSize) {
486 0 : ReportCmdInfoCheckFailed(*localInfo, "inCclBufferSize", localInfo->inCclBufferSize,
487 0 : remoteInfo->inCclBufferSize);
488 : }
489 :
490 1 : if (localInfo->outCclBufferSize != remoteInfo->outCclBufferSize) {
491 0 : ReportCmdInfoCheckFailed(*localInfo, "outCclBufferSize", localInfo->outCclBufferSize,
492 0 : remoteInfo->outCclBufferSize);
493 : }
494 :
495 1 : if (localInfo->aivCoreLimit != remoteInfo->aivCoreLimit) {
496 0 : ReportCmdInfoCheckFailed(*localInfo, "aivCoreLimit", localInfo->aivCoreLimit,
497 : remoteInfo->aivCoreLimit);
498 : }
499 :
500 1 : if (localInfo->deterministic != remoteInfo->deterministic) {
501 0 : ReportCmdInfoCheckFailed(*localInfo, "deterministic", localInfo->deterministic,
502 0 : remoteInfo->deterministic);
503 : }
504 :
505 1 : return;
506 : }
507 :
508 4 : bool RankConsistentcyChecker::CompareFrame(HcclCheckInfo &checkInfo, HcclCheckInfo &checkInfoRecv)
509 : {
510 4 : bool bIsDiff = false;
511 4 : if (CompareCrcInfo(checkInfo.cmdInfo, checkInfo.crcInfoGlobal, checkInfoRecv.crcInfoGlobal)) {
512 1 : HCCL_ERROR("[RankConsistentcyChecker][CompareFrame]errNo[0x%016llx] CRC check fail, please check the "
513 : "rankTable file and hccl_config file.", HCCL_ERROR_CODE(HCCL_E_INTERNAL));
514 1 : bIsDiff = true;
515 : }
516 4 : if (CompareCrcInfo(checkInfo.cmdInfo, checkInfo.crcInfoOp, checkInfoRecv.crcInfoOp)) {
517 1 : HCCL_ERROR("[RankConsistentcyChecker][CompareFrame]errNo[0x%016llx] Op CRC check fail, please check the op"
518 : " parameters, rankTable file and hccl_config file.", HCCL_ERROR_CODE(HCCL_E_INTERNAL));
519 1 : bIsDiff = true;
520 : }
521 4 : if (!CompareSection(reinterpret_cast<char_t *>(&checkInfo.cmdInfo),
522 4 : reinterpret_cast<char_t *>(&checkInfoRecv.cmdInfo), sizeof(checkInfo.cmdInfo))) {
523 1 : CompareCmdInfo(checkInfo, checkInfoRecv);
524 1 : HCCL_ERROR("[RankConsistentcyChecker][CompareFrame]errNo[0x%016llx] CMD check fail",
525 : HCCL_ERROR_CODE(HCCL_E_INTERNAL));
526 1 : bIsDiff = true;
527 : }
528 4 : HCCL_INFO("loc protocolType is [%d], rem protocolType is [%d].",
529 : checkInfo.protocolType, checkInfoRecv.protocolType);
530 4 : if (checkInfo.protocolType != checkInfoRecv.protocolType) {
531 1 : HCCL_ERROR("[RankConsistentcyChecker][CompareFrame]errNo[0x%016llx] ProtocolType check fail",
532 : HCCL_ERROR_CODE(HCCL_E_INTERNAL));
533 1 : bIsDiff = true;
534 : }
535 :
536 : // Cann版本校验，只在集合通信场景校验CANN版本
537 4 : if (cannVerCheckSwitch_) {
538 6 : std::string localCannVersion = checkInfo.version;
539 3 : std::string remoteCannVersion = checkInfoRecv.version;
540 3 : if (localCannVersion.empty() || remoteCannVersion.empty()) { // cann版本信息读取失败，返回告警
541 1 : HCCL_WARNING("[RankConsistentcyChecker][CompareFrame] CANN version str is empty. local_version %s, "
542 : "remote_version %s.", checkInfo.version, checkInfoRecv.version);
543 2 : } else if (localCannVersion != remoteCannVersion) { // cann版本信息读取成功，且版本不一致
544 1 : const char *ascendHomePath = std::getenv("ASCEND_HOME_PATH");
545 2 : std::string versionFilePath = (ascendHomePath != nullptr) ? std::string(ascendHomePath) + "/share/info/hccl/version.info" : "unknown";
546 11 : RPT_INPUT_ERR(true, "EI0008", std::vector<std::string>({"local_version", "remote_version"}),
547 : std::vector<std::string>({localCannVersion+ " (version file: " + versionFilePath + ")", remoteCannVersion}));
548 1 : HCCL_ERROR("[%s][%s] errNo[0x%016llx] Inconsistent HCCL Versions. local_version %s, remote_version %s.",
549 : LOG_KEYWORDS_INIT_CHANNEL.c_str(), LOG_KEYWORDS_VERSION_CONFLICT.c_str(),
550 : HCCL_ERROR_CODE(HCCL_E_INTERNAL), checkInfo.version, checkInfoRecv.version);
551 1 : bIsDiff = true;
552 1 : }
553 3 : }
554 : #ifndef HCCD
555 4 : if (GetExternalInconsistentCheckSwitch() == InconsistentCheckMode::FIRST) {
556 4 : inconsistentCheckFirstDone_ = true;
557 : }
558 : #endif
559 4 : return bIsDiff;
560 2 : }
561 :
562 9 : HcclResult RankConsistentcyChecker::AddCrc(const u32 crcValue)
563 : {
564 9 : HCCL_DEBUG("crcValue[%u].", crcValue);
565 9 : crcTable_.push_back(crcValue);
566 9 : HCCL_DEBUG("num[%llu].", crcTable_.size());
567 9 : return HCCL_SUCCESS;
568 : }
569 :
570 63 : HcclResult RankConsistentcyChecker::ClearCrcInfo(void)
571 : {
572 63 : this->crcTable_.clear();
573 63 : if (this->crcTable_.size() != 0) {
574 0 : HCCL_ERROR("[Clear][CrcInfo]errNo[0x%016llx] clear crcTable_ is failed", HCCL_ERROR_CODE(HCCL_E_INTERNAL));
575 0 : return HCCL_E_INTERNAL;
576 : }
577 63 : return HCCL_SUCCESS;
578 : }
579 :
580 1 : HcclResult RankConsistentcyChecker::GetCrc(u32 num, u32 *crcAddr)
581 : {
582 1 : CHK_PTR_NULL(crcAddr);
583 1 : HCCL_DEBUG("num[%u], crc[%u].", num, *crcAddr);
584 :
585 1 : if (num == 0) {
586 0 : HCCL_ERROR("[Get][Crc]errNo[0x%016llx] In get crc the value of num is 0", HCCL_ERROR_CODE(HCCL_E_PARA));
587 0 : return HCCL_E_PARA;
588 : }
589 :
590 1 : if (num != crcTable_.size()) {
591 0 : HCCL_ERROR("[Get][Crc]errNo[0x%016llx] num error inputNum[%u], localNum[%llu]",
592 : HCCL_ERROR_CODE(HCCL_E_INTERNAL), num, crcTable_.size());
593 0 : return HCCL_E_INTERNAL;
594 : }
595 :
596 4 : for (u32 i = 0; i < num; i++) {
597 3 : crcAddr[i] = crcTable_[i];
598 : }
599 1 : return HCCL_SUCCESS;
600 : }
601 : }
|
