Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "calc_crc.h"
12 : #include "rank_consistentcy_checker.h"
13 : #include "env_config.h"
14 :
15 : namespace hccl {
16 :
17 590 : RankConsistentcyChecker::RankConsistentcyChecker()
18 590 : : cannVersion_{0},
19 590 : cannVerCheckSwitch_(false),
20 590 : cannVerInfoRecordFlag_(false),
21 590 : configFileExist_(false)
22 590 : {}
23 :
24 590 : RankConsistentcyChecker::~RankConsistentcyChecker() = default;
25 :
26 1927 : RankConsistentcyChecker& RankConsistentcyChecker::GetInstance(s32 deviceLogicId)
27 : {
28 2512 : static RankConsistentcyChecker instance[MAX_MODULE_DEVICE_NUM];
29 1929 : if (deviceLogicId == HOST_DEVICE_ID) {
30 0 : HCCL_INFO("[GetInstance] deviceLogicId[-1] is HOST_DEVICE_ID");
31 0 : return instance[0];
32 : }
33 1929 : hrtGetDeviceRefresh(&deviceLogicId);
34 1930 : HCCL_INFO("[GetInstance] get deviceLogicId[%d]", deviceLogicId);
35 1931 : CHK_PRT_RET(
36 : (static_cast<u32>(deviceLogicId) >= MAX_MODULE_DEVICE_NUM || deviceLogicId < 0),
37 : HCCL_WARNING("[R]deviceLogicId[%d] is invalid", deviceLogicId), instance[0]);
38 :
39 1929 : return instance[deviceLogicId];
40 : }
41 :
42 : // gather
43 0 : HcclResult RankConsistentcyChecker::RecordOpPara(
44 : HcclCMDType opCMD, const std::string& tag, u64 count, HcclDataType dataType, u32 root, u64 inCclBufferSize,
45 : u64 outCclBufferSize, const char* group, u32 crc, u32 aivCoreLimit)
46 : {
47 0 : return RecordOpPara(
48 : opCMD, tag, count, dataType, HCCL_REDUCE_RESERVED, root, 0, 0, 0, inCclBufferSize, outCclBufferSize, group, crc,
49 0 : aivCoreLimit);
50 : }
51 :
52 : // reduce
53 76 : HcclResult RankConsistentcyChecker::RecordOpPara(
54 : HcclCMDType opCMD, const std::string& tag, u64 count, HcclDataType dataType, HcclReduceOp op, u32 root,
55 : u64 inCclBufferSize, u64 outCclBufferSize, const char* group, u32 crc, u8 deterministic, u32 aivCoreLimit)
56 : {
57 76 : return RecordOpPara(
58 76 : opCMD, tag, count, dataType, op, root, 0, 0, 0, inCclBufferSize, outCclBufferSize, group, crc, aivCoreLimit);
59 : }
60 :
61 : // send && receive
62 0 : HcclResult RankConsistentcyChecker::RecordOpPara(
63 : HcclCMDType opCMD, const std::string& tag, u64 count, HcclDataType dataType, u32 rank, u32 srTag, u32 selfRank,
64 : u64 inCclBufferSize, u64 outCclBufferSize, const char* group, u32 crc)
65 : {
66 0 : return RecordOpPara(
67 : opCMD, tag, count, dataType, HCCL_REDUCE_RESERVED, 0, rank, srTag, selfRank, inCclBufferSize, outCclBufferSize,
68 0 : group, crc);
69 : }
70 :
71 : // batchsendrecv
72 0 : HcclResult RankConsistentcyChecker::RecordOpPara(
73 : HcclCMDType opCMD, const std::string& tag, u64 inCclBufferSize, u64 outCclBufferSize, const char* group, u32 crc)
74 : {
75 0 : return RecordOpPara(
76 : opCMD, tag, 0, HCCL_DATA_TYPE_RESERVED, HCCL_REDUCE_RESERVED, 0, 0, 0, 0, inCclBufferSize, outCclBufferSize,
77 0 : group, crc);
78 : }
79 :
80 : // reduce scatter v && AllGather v
81 0 : HcclResult RankConsistentcyChecker::RecordOpPara(
82 : HcclCMDType opCMD, const std::string& tag, const void* counts, const void* displs, const u32 rankSize,
83 : HcclDataType dataType, HcclReduceOp op, u64 inCclBufferSize, u64 outCclBufferSize, const char* group, u32 crc,
84 : u8 deterministic, u32 aivCoreLimit)
85 : {
86 0 : CHK_RET(RecordOpPara(
87 : opCMD, tag, 0, dataType, op, 0, 0, 0, 0, inCclBufferSize, outCclBufferSize, group, crc, aivCoreLimit));
88 0 : CHK_RET(RecordVaringOpPara(tag, counts, displs, rankSize));
89 0 : return HCCL_SUCCESS;
90 : }
91 :
92 0 : HcclResult RankConsistentcyChecker::RecordVaringOpPara(
93 : const std::string& tag, const void* counts, const void* displs, const u32 rankSize)
94 : {
95 : u32 countsCrc;
96 0 : CHK_RET(CalcRawDataCrc(static_cast<const char_t*>(counts), rankSize * sizeof(u64), countsCrc));
97 0 : crcRecords_[tag][HcclCrcRecordType::HCCL_CRC_RECORD_VARING_COUNTS] = countsCrc;
98 :
99 : u32 displsCrc;
100 0 : CHK_RET(CalcRawDataCrc(static_cast<const char_t*>(displs), rankSize * sizeof(u64), displsCrc));
101 0 : crcRecords_[tag][HcclCrcRecordType::HCCL_CRC_RECORD_VARING_DISPLACEMENTS] = displsCrc;
102 0 : return HCCL_SUCCESS;
103 : }
104 :
105 24 : HcclResult RankConsistentcyChecker::DelOpPara(const std::string& tag)
106 : {
107 24 : std::lock_guard<std::mutex> lock(mutex_);
108 24 : CHK_PRT_RET(
109 : !cmdInfoMap_.erase(tag),
110 : HCCL_ERROR(
111 : "[RankConsistentcyChecker][DelOpPara]CMD info for tag[%s] does not exist, delete fail.", tag.c_str()),
112 : HCCL_E_INTERNAL);
113 24 : CHK_PRT_RET(
114 : !infoFlagCmdMap_.erase(tag),
115 : HCCL_ERROR(
116 : "[RankConsistentcyChecker][DelOpPara]CMD info flag cmd for tag[%s] does not exist, delete fail.",
117 : tag.c_str()),
118 : HCCL_E_INTERNAL);
119 24 : crcRecords_.erase(tag);
120 24 : return HCCL_SUCCESS;
121 24 : }
122 :
123 236 : HcclResult RankConsistentcyChecker::RecordVerInfo(const std::string& versionInfo)
124 : {
125 236 : std::lock_guard<std::mutex> lock(mutex_);
126 : // Only record once, in case of multiple calls while other process is reading CANN version information
127 : // in CompareFrame func and get the intermediate state.
128 236 : if (cannVerInfoRecordFlag_) {
129 231 : HCCL_INFO("[RankConsistentcyChecker][RecordVerInfo]Cann version information has been recorded.");
130 231 : return HCCL_SUCCESS;
131 : }
132 :
133 5 : u32 strLen = versionInfo.length();
134 5 : s32 sRet = memset_s(cannVersion_, MAX_CANN_VERSION_LEN + 1, 0, MAX_CANN_VERSION_LEN + 1);
135 5 : CHK_PRT_RET(
136 : sRet != EOK,
137 : HCCL_WARNING(
138 : "[RankConsistentcyChecker][RecordVerInfo]memory set 0 fail for version str "
139 : "array. return[%d].",
140 : sRet),
141 : HCCL_SUCCESS);
142 :
143 5 : CHK_PRT_RET(strLen == 0, HCCL_WARNING("[Record][CannVersion] version information str is empty."), HCCL_SUCCESS);
144 :
145 5 : CHK_PRT_RET(
146 : strLen >= MAX_CANN_VERSION_LEN,
147 : HCCL_WARNING("[Record][CannVersion]"
148 : "length of version information str is too long."),
149 : HCCL_SUCCESS);
150 5 : sRet = strncpy_s(cannVersion_, MAX_CANN_VERSION_LEN + 1, versionInfo.c_str(), strLen);
151 5 : CHK_PRT_RET(
152 : sRet != EOK, HCCL_WARNING("[Record][CannVersion] call strncpy_s failed, return [%d].", sRet), HCCL_SUCCESS);
153 :
154 5 : cannVerInfoRecordFlag_ = true;
155 5 : return HCCL_SUCCESS;
156 236 : }
157 :
158 1055 : u64 RankConsistentcyChecker::GetRankConsistentDataLength() { return sizeof(HcclCheckInfo); }
159 :
160 0 : void RankConsistentcyChecker::RecordProtocolType(ProtocolType protocolType)
161 : {
162 0 : HCCL_INFO("[RankConsistentcyChecker][RecordProtocolType]protocolType set to [%d].", static_cast<s32>(protocolType));
163 0 : protocolType_ = protocolType;
164 0 : return;
165 : }
166 :
167 4 : HcclResult RankConsistentcyChecker::GetCheckFrame(u8* destBuf, u64 maxDestBuf, const std::string& tag)
168 : {
169 4 : CHK_PTR_NULL(destBuf);
170 : // 要发送的校验帧
171 4 : HcclCheckInfo checkInfo;
172 4 : u64 checkInfoLen = sizeof(checkInfo);
173 4 : HcclResult ret = GenerateCheckFrame(checkInfo, tag);
174 4 : checkInfo.cmdInfo.selfRank = 0; // 自身的group rank 不做校验,置0
175 4 : CHK_PRT_RET(
176 : ret != HCCL_SUCCESS,
177 : HCCL_ERROR(
178 : "[RankConsistentcyChecker][GetCheckFrame]generate check frame fail. "
179 : "return[%d]",
180 : ret),
181 : ret);
182 :
183 4 : s32 sret = memcpy_s(destBuf, maxDestBuf, &checkInfo, checkInfoLen);
184 4 : CHK_PRT_RET(
185 : sret != EOK,
186 : HCCL_ERROR(
187 : "[RankConsistentcyChecker][GetCheckFrame]frame len[%llu] is bigger than "
188 : "dest buffer len[%llu].",
189 : checkInfoLen, maxDestBuf),
190 : HCCL_E_INTERNAL);
191 4 : return HCCL_SUCCESS;
192 : }
193 :
194 0 : HcclResult RankConsistentcyChecker::CheckFrameRecv(const u8* recvBuf, u32 recvBufLen, const std::string& tag)
195 : {
196 : #ifndef HCCD
197 0 : if ((GetExternalInconsistentCheckSwitch() == InconsistentCheckMode::OFF)
198 0 : || (GetExternalInconsistentCheckSwitch() == InconsistentCheckMode::FIRST
199 0 : && inconsistentCheckFirstDone_ == true)) {
200 0 : return HCCL_SUCCESS;
201 : }
202 : #endif
203 0 : CHK_PTR_NULL(recvBuf);
204 0 : CHK_PRT_RET(
205 : recvBufLen == 0 || recvBufLen > MAX_FRAME_LEN,
206 : HCCL_ERROR(
207 : "[RankConsistentcyChecker][CheckFrameRecv] errNo[0x%016llx] recvBufLen is wrong.",
208 : HCCL_ERROR_CODE(HCCL_E_INTERNAL)),
209 : HCCL_E_INTERNAL);
210 :
211 0 : CHK_PRT_RET(
212 : recvBufLen < sizeof(HcclCheckInfo),
213 : HCCL_ERROR(
214 : "[RankConsistentcyChecker][CheckFrameRecv] errNo[0x%016llx] recvBufLen[%u]is less than "
215 : "check info[%zu].",
216 : HCCL_ERROR_CODE(HCCL_E_INTERNAL), recvBufLen, sizeof(HcclCheckInfo)),
217 : HCCL_E_PARA);
218 :
219 0 : HcclCheckInfo checkInfoRecv;
220 : // 对固定长度的全局数组变量，结构体变量进行初始化和拷贝，可以不用检查初始化安全函数返回值
221 0 : (void)memset_s(&checkInfoRecv, sizeof(HcclCheckInfo), 0, sizeof(HcclCheckInfo));
222 0 : (void)memcpy_s(&checkInfoRecv, sizeof(HcclCheckInfo), recvBuf, sizeof(HcclCheckInfo));
223 :
224 0 : HcclCheckInfo checkInfo;
225 0 : CHK_RET(GenerateCheckFrame(checkInfo, tag));
226 0 : if (checkInfo.cmdInfo.cmdType == HcclCMDType::HCCL_CMD_SEND) {
227 0 : checkInfo.cmdInfo.cmdType = HcclCMDType::HCCL_CMD_RECEIVE;
228 0 : checkInfo.cmdInfo.rank = checkInfo.cmdInfo.selfRank;
229 0 : } else if (checkInfo.cmdInfo.cmdType == HcclCMDType::HCCL_CMD_RECEIVE) {
230 0 : checkInfo.cmdInfo.cmdType = HcclCMDType::HCCL_CMD_SEND;
231 0 : checkInfo.cmdInfo.rank = checkInfo.cmdInfo.selfRank;
232 : }
233 :
234 0 : checkInfo.cmdInfo.selfRank = 0; // 自身的子group rank 不做校验
235 0 : if (CompareFrame(checkInfo, checkInfoRecv)) {
236 0 : return HCCL_E_INTERNAL;
237 : }
238 :
239 0 : HCCL_INFO(
240 : "[RankConsistentcyChecker][CheckFrameRecv] check success, len of frame[%u], len of check data[%zu].",
241 : recvBufLen, sizeof(checkInfo));
242 0 : return HCCL_SUCCESS;
243 : }
244 :
245 62 : void RankConsistentcyChecker::ClearCheckInfo()
246 : {
247 62 : configFileExist_ = false;
248 62 : cannVerInfoRecordFlag_ = false;
249 62 : ClearCrcInfo();
250 : {
251 62 : std::lock_guard<std::mutex> lock(mutex_);
252 62 : cmdInfoMap_.clear();
253 62 : infoFlagCmdMap_.clear();
254 62 : }
255 : // 相关规范的例外场景，对固定数组的memset_s可以不判断返回值
256 62 : (void)memset_s(cannVersion_, MAX_CANN_VERSION_LEN + 1, 0, MAX_CANN_VERSION_LEN + 1);
257 62 : protocolType_ = ProtocolType::RESERVED;
258 62 : return;
259 : }
260 :
261 236 : HcclResult RankConsistentcyChecker::CalcStringCrc(const char* str, u32& crc)
262 : {
263 : // 计算字符串CRC
264 236 : HcclResult ret = CalcCrc::HcclCalcCrc(str, strlen(str), crc);
265 236 : CHK_PRT_RET(
266 : ret != HCCL_SUCCESS,
267 : HCCL_ERROR(
268 : "[RankConsistentcyChecker][CalcStringCrc] errNo[0x%016llx] calc string crc error",
269 : HCCL_ERROR_CODE(HCCL_E_INTERNAL)),
270 : HCCL_E_INTERNAL);
271 :
272 236 : HCCL_DEBUG("[RankConsistentcyChecker][CalcStringCrc] result crc[%u].", crc);
273 236 : return HCCL_SUCCESS;
274 : }
275 :
276 0 : HcclResult RankConsistentcyChecker::CalcRawDataCrc(const void* ptr, u64 length, u32& crc)
277 : {
278 : // 计算内存数据块CRC
279 0 : HcclResult ret = CalcCrc::HcclCalcCrc(static_cast<const char*>(ptr), length, crc);
280 0 : CHK_PRT_RET(
281 : ret != HCCL_SUCCESS,
282 : HCCL_ERROR(
283 : "[RankConsistentcyChecker][CalcRawDataCrc] errNo[0x%016llx] calc string crc error",
284 : HCCL_ERROR_CODE(HCCL_E_INTERNAL)),
285 : HCCL_E_INTERNAL);
286 :
287 0 : HCCL_DEBUG("[RankConsistentcyChecker][CalcRawDataCrc] result crc[%u].", crc);
288 0 : return HCCL_SUCCESS;
289 : }
290 :
291 240 : void RankConsistentcyChecker::SetCheckCannVersionSwitch(const bool cannVerCheckSwitch)
292 : {
293 240 : cannVerCheckSwitch_ = cannVerCheckSwitch;
294 240 : return;
295 : }
296 :
297 : // private
298 77 : HcclResult RankConsistentcyChecker::RecordOpPara(
299 : HcclCMDType opCMD, const std::string& tag, u64 count, HcclDataType dataType, HcclReduceOp op, u32 root, u32 rank,
300 : u32 srTag, u32 selfRank, u64 inCclBufferSize, u64 outCclBufferSize, const char* group, u32 crc, u8 deterministic,
301 : u32 aivCoreLimit)
302 : {
303 77 : HcclCMDInfo cmdInfo;
304 : // 相关规范的例外场景，对固定数组的memset_s可以不判断返回值
305 77 : (void)memset_s(&cmdInfo, sizeof(cmdInfo), 0, sizeof(cmdInfo));
306 :
307 77 : cmdInfo.cmdType = opCMD;
308 77 : s32 sRet = strncpy_s(cmdInfo.tag, TAG_MAX_LEN + 1, tag.c_str(), tag.length());
309 77 : CHK_PRT_RET(
310 : sRet != EOK,
311 : HCCL_ERROR(
312 : "[RankConsistentcyChecker][RecordOpPara]errNo[0x%016llx] strlen[%u] of tag is "
313 : "longer than buffer[%u].",
314 : HCCL_ERROR_CODE(HCCL_E_PARA), tag.length(), TAG_MAX_LEN),
315 : HCCL_E_PARA);
316 :
317 77 : cmdInfo.count = count;
318 77 : cmdInfo.dataType = dataType;
319 :
320 77 : std::string strGroup = (group == nullptr) ? HCCL_WORLD_GROUP : group;
321 74 : sRet = strncpy_s(cmdInfo.group, GROUP_NAME_MAX_LEN + 1, strGroup.c_str(), strGroup.length());
322 74 : CHK_PRT_RET(
323 : sRet != EOK,
324 : HCCL_ERROR(
325 : "[RankConsistentcyChecker][RecordOpPara]errNo[0x%016llx] strlen[%u] group is "
326 : "longer than buffer[%u].",
327 : HCCL_ERROR_CODE(HCCL_E_PARA), strGroup.length(), GROUP_NAME_MAX_LEN),
328 : HCCL_E_PARA);
329 :
330 74 : cmdInfo.op = op;
331 74 : cmdInfo.root = root;
332 74 : cmdInfo.rank = rank;
333 74 : cmdInfo.srTag = srTag;
334 74 : cmdInfo.selfRank = selfRank;
335 74 : cmdInfo.inCclBufferSize = inCclBufferSize;
336 74 : cmdInfo.outCclBufferSize = outCclBufferSize;
337 74 : cmdInfo.aivCoreLimit = aivCoreLimit;
338 74 : cmdInfo.deterministic = deterministic;
339 :
340 74 : std::lock_guard<std::mutex> lock(mutex_);
341 77 : cmdInfoMap_[tag] = cmdInfo;
342 74 : infoFlagCmdMap_[tag] = true;
343 74 : crcRecords_[tag][HcclCrcRecordType::HCCL_CRC_RECORD_RANKTABLE] = crc;
344 :
345 69 : return HCCL_SUCCESS;
346 69 : }
347 :
348 0 : HcclResult RankConsistentcyChecker::GetOpParaByTag(const std::string& tag, HcclCMDInfo& CMDInfoOutput)
349 : {
350 0 : auto getResult = cmdInfoMap_.find(tag);
351 0 : CHK_PRT_RET(
352 : getResult == cmdInfoMap_.end(),
353 : HCCL_ERROR(
354 : "[RankConsistentcyChecker][GetOpParaByTag]There is not any CMD information for tag[%s]", tag.c_str()),
355 : HCCL_E_INTERNAL);
356 0 : CMDInfoOutput = getResult->second;
357 0 : return HCCL_SUCCESS;
358 : }
359 :
360 0 : HcclResult RankConsistentcyChecker::GetCrcByTag(const std::string& tag, HcclCRCInfo& crcInfo)
361 : {
362 0 : const auto recordsIter = crcRecords_.find(tag);
363 0 : CHK_PRT_RET(
364 : recordsIter == crcRecords_.end(),
365 : HCCL_ERROR("[RankConsistentcyChecker][GetCrcByTag]There is not any CRC information for tag[%s]", tag.c_str()),
366 : HCCL_E_INTERNAL);
367 0 : crcInfo.crcNum = 0;
368 0 : const auto& tagRecords = recordsIter->second;
369 0 : for (const auto& record : tagRecords) {
370 0 : crcInfo.crcArray[crcInfo.crcNum++] = record.second;
371 0 : HCCL_DEBUG("[RankConsistentcyChecker][GetCrcByTag]Append crc[%u] for tag[%s].", record.second, tag.c_str());
372 : }
373 :
374 0 : HCCL_INFO(
375 : "[RankConsistentcyChecker][GetCrcByTag]After adding crc for tag[%s], crcNum set to [%u].", tag.c_str(),
376 : crcInfo.crcNum);
377 0 : return HCCL_SUCCESS;
378 : }
379 :
380 3 : HcclResult RankConsistentcyChecker::GenerateCheckFrame(HcclCheckInfo& checkInfo, const std::string& tag)
381 : {
382 : // 初始化用于发送的BUFFER
383 : // 对入参的结构体变量指向的内存进行初始化时，使用了变量的结构体类型大小进行初始化，
384 : // 如果指针不为空，可以不检查初始化安全函数的返回值
385 3 : u64 checkInfoLen = sizeof(HcclCheckInfo);
386 3 : (void)memset_s(&checkInfo, checkInfoLen, 0, checkInfoLen);
387 :
388 : // 添加CRC字段到校验帧
389 4 : u32 crcLen = crcTable_.size();
390 4 : checkInfo.crcInfoGlobal.configFileExist_ = configFileExist_;
391 4 : if (crcLen != 0) {
392 0 : CHK_PRT_RET(
393 : crcLen > MAX_CRC_LEN,
394 : HCCL_ERROR("[RankConsistentcyChecker][GenerateCheckFrame]crc num[%u] is too big.", crcLen),
395 : HCCL_E_INTERNAL);
396 0 : checkInfo.crcInfoGlobal.crcNum = crcLen;
397 0 : CHK_RET(GetCrc(crcLen, &checkInfo.crcInfoGlobal.crcArray[0]));
398 : }
399 : // 添加CMD参数信息到校验帧
400 : {
401 4 : std::lock_guard<std::mutex> lock(mutex_);
402 4 : auto getResult = infoFlagCmdMap_.find(tag);
403 4 : if (getResult != infoFlagCmdMap_.end()) {
404 0 : CHK_PRT_RET(
405 : GetOpParaByTag(tag, checkInfo.cmdInfo) != HCCL_SUCCESS,
406 : HCCL_ERROR("[RankConsistentcyChecker][GenerateCheckFrame]get Op para by tag[%s] fail.", tag.c_str()),
407 : HCCL_E_INTERNAL);
408 0 : checkInfo.crcInfoOp.configFileExist_ = configFileExist_;
409 : // 添加CRC字段到校验帧
410 0 : CHK_PRT_RET(
411 : GetCrcByTag(tag, checkInfo.crcInfoOp) != HCCL_SUCCESS,
412 : HCCL_ERROR(
413 : "[RankConsistentcyChecker][GenerateCheckFrame]get ranktable crc by tag[%s] fail.", tag.c_str()),
414 : HCCL_E_INTERNAL);
415 : }
416 4 : }
417 : // 添加HCCL版本信息到校验帧
418 4 : if (cannVerInfoRecordFlag_) {
419 0 : HCCL_DEBUG("[RankConsistentcyChecker][GenerateCheckFrame] CANN version information is [%s].", cannVersion_);
420 0 : s32 sret = memcpy_s(checkInfo.version, MAX_CANN_VERSION_LEN + 1, cannVersion_, strlen(cannVersion_));
421 0 : CHK_PRT_RET(
422 : sret != EOK,
423 : HCCL_ERROR(
424 : "[RankConsistentcyChecker][GenerateCheckFrame] memcpy CANN version information failed, "
425 : "errorno [%d].",
426 : sret),
427 : HCCL_E_MEMORY);
428 : }
429 : // 添加拉远通信传输类型校验
430 : // 910* 不会配置isTcpMode，因此910*在此处的待校验值是一致的
431 4 : checkInfo.protocolType = protocolType_;
432 4 : HCCL_INFO("[RankConsistentcyChecker][GenerateCheckFrame] loc protocolType is [%d].", checkInfo.protocolType);
433 :
434 4 : return HCCL_SUCCESS;
435 : }
436 :
437 29 : bool RankConsistentcyChecker::CompareSection(const char* pRawData, const char* recvBuf, u32 len)
438 : {
439 11741 : for (u32 i = 0; i < len; i++) {
440 11715 : if (*(pRawData + i) != *(recvBuf + i)) {
441 3 : return false;
442 : }
443 : }
444 26 : return true;
445 : }
446 :
447 18 : bool RankConsistentcyChecker::CompareCrcInfo(
448 : const HcclCMDInfo& hcclCMDInfo, HcclCRCInfo& crcInfo, HcclCRCInfo& crcInfoRecv)
449 : {
450 18 : bool bIsDiff = false;
451 : // 检校验整体是否一致
452 18 : if (!CompareSection(
453 : reinterpret_cast<char_t*>(&crcInfo), reinterpret_cast<char_t*>(&crcInfoRecv), sizeof(crcInfo))) {
454 2 : bIsDiff = true;
455 : // 检查每种CRC类型是否一致
456 4 : for (auto i = 0U; i < crcInfo.crcNum; ++i) {
457 2 : if (crcInfo.crcArray[i] != crcInfoRecv.crcArray[i]) {
458 2 : ReportCrcCheckFailed(
459 : hcclCMDInfo, static_cast<HcclCrcRecordType>(i), crcInfo.crcArray[i], crcInfoRecv.crcArray[i]);
460 : }
461 : }
462 : }
463 18 : return bIsDiff;
464 : }
465 :
466 0 : void RankConsistentcyChecker::ReportCmdInfoCheckFailed(
467 : const HcclCMDInfo& hcclCMDInfo, const std::string& paraName, const std::string& localPara,
468 : const std::string& remotePara) const
469 : {
470 0 : ReportCommonError(hcclCMDInfo, paraName, localPara, remotePara, "CMD information");
471 0 : }
472 :
473 3 : void RankConsistentcyChecker::ReportCmdInfoCheckFailed(
474 : const HcclCMDInfo& hcclCMDInfo, const std::string& paraName, uint32_t localPara, uint32_t remotePara)
475 : {
476 3 : ReportCommonError(hcclCMDInfo, paraName, std::to_string(localPara), std::to_string(remotePara), "CMD information");
477 3 : }
478 :
479 2 : void RankConsistentcyChecker::ReportCrcCheckFailed(
480 : const HcclCMDInfo& hcclCMDInfo, HcclCrcRecordType crcType, const uint32_t localCrc, const uint32_t remoteCrc) const
481 : {
482 2 : const auto crcTypeStr = GetCRCTypeEnumStr(crcType);
483 2 : ReportCommonError(hcclCMDInfo, crcTypeStr, std::to_string(localCrc), std::to_string(remoteCrc), "CRC check");
484 2 : }
485 :
486 5 : void RankConsistentcyChecker::ReportCommonError(
487 : const HcclCMDInfo& hcclCMDInfo, const std::string& paraName, const std::string& localParaStr,
488 : const std::string& remoteParaStr, const std::string& errorMsg) const
489 : {
490 5 : std::string opInfo = "Unknown";
491 15 : for (const auto& pair : HCCL_OPTYPE_NAME_MAP) {
492 15 : if (pair.second == hcclCMDInfo.cmdType) {
493 5 : opInfo = std::string(pair.first);
494 5 : break;
495 : }
496 : }
497 100 : RPT_INPUT_ERR(
498 : true, "EI0005", std::vector<std::string>({"ccl_op", "group", "para_name", "local_para", "remote_para"}),
499 : std::vector<std::string>({opInfo, hcclCMDInfo.group, paraName, localParaStr, remoteParaStr}));
500 5 : HCCL_ERROR(
501 : "[%s][%s]%s %s check fail. local[%s], remote[%s]", LOG_KEYWORDS_INIT_CHANNEL.c_str(),
502 : LOG_KEYWORDS_PARAMETER_CONFLICT.c_str(), errorMsg.c_str(), paraName.c_str(), localParaStr.c_str(),
503 : remoteParaStr.c_str());
504 15 : }
505 :
506 1 : void RankConsistentcyChecker::CompareCmdInfo(HcclCheckInfo& checkInfo, HcclCheckInfo& checkInfoRecv)
507 : {
508 1 : auto localInfo = &checkInfo.cmdInfo;
509 1 : auto remoteInfo = &checkInfoRecv.cmdInfo;
510 :
511 1 : if (!CompareSection(localInfo->tag, remoteInfo->tag, TAG_MAX_LEN + 1)) {
512 0 : ReportCmdInfoCheckFailed(*localInfo, "tag", localInfo->tag, remoteInfo->tag);
513 : }
514 :
515 1 : if (localInfo->cmdType != remoteInfo->cmdType) {
516 2 : ReportCmdInfoCheckFailed(
517 1 : *localInfo, "cmdType", static_cast<uint32_t>(localInfo->cmdType),
518 1 : static_cast<uint32_t>(remoteInfo->cmdType));
519 : }
520 :
521 1 : if (localInfo->count != remoteInfo->count) {
522 3 : ReportCmdInfoCheckFailed(*localInfo, "count", localInfo->count, remoteInfo->count);
523 : }
524 :
525 1 : if (localInfo->dataType != remoteInfo->dataType) {
526 2 : ReportCmdInfoCheckFailed(
527 1 : *localInfo, "dataType", static_cast<uint32_t>(localInfo->dataType),
528 1 : static_cast<uint32_t>(remoteInfo->dataType));
529 : }
530 :
531 1 : if (localInfo->op != remoteInfo->op) {
532 0 : ReportCmdInfoCheckFailed(
533 0 : *localInfo, "op", static_cast<uint32_t>(localInfo->op), static_cast<uint32_t>(remoteInfo->op));
534 : }
535 :
536 1 : if (!CompareSection(localInfo->group, remoteInfo->group, GROUP_NAME_MAX_LEN + 1)) {
537 0 : ReportCmdInfoCheckFailed(*localInfo, "group", localInfo->group, remoteInfo->group);
538 : }
539 :
540 1 : if (localInfo->root != remoteInfo->root) {
541 0 : ReportCmdInfoCheckFailed(*localInfo, "root", localInfo->root, remoteInfo->root);
542 : }
543 :
544 1 : if (localInfo->rank != remoteInfo->rank) {
545 0 : ReportCmdInfoCheckFailed(*localInfo, "rank", localInfo->rank, remoteInfo->rank);
546 : }
547 :
548 1 : if (localInfo->srTag != remoteInfo->srTag) {
549 0 : ReportCmdInfoCheckFailed(*localInfo, "srTag", localInfo->srTag, remoteInfo->srTag);
550 : }
551 :
552 1 : if (localInfo->inCclBufferSize != remoteInfo->inCclBufferSize) {
553 0 : ReportCmdInfoCheckFailed(
554 0 : *localInfo, "inCclBufferSize", localInfo->inCclBufferSize, remoteInfo->inCclBufferSize);
555 : }
556 :
557 1 : if (localInfo->outCclBufferSize != remoteInfo->outCclBufferSize) {
558 0 : ReportCmdInfoCheckFailed(
559 0 : *localInfo, "outCclBufferSize", localInfo->outCclBufferSize, remoteInfo->outCclBufferSize);
560 : }
561 :
562 1 : if (localInfo->aivCoreLimit != remoteInfo->aivCoreLimit) {
563 0 : ReportCmdInfoCheckFailed(*localInfo, "aivCoreLimit", localInfo->aivCoreLimit, remoteInfo->aivCoreLimit);
564 : }
565 :
566 1 : if (localInfo->deterministic != remoteInfo->deterministic) {
567 0 : ReportCmdInfoCheckFailed(*localInfo, "deterministic", localInfo->deterministic, remoteInfo->deterministic);
568 : }
569 :
570 1 : return;
571 : }
572 :
573 18 : std::vector<std::string_view> Split(const std::string& str, char delimiter)
574 : {
575 18 : std::vector<std::string_view> parts;
576 18 : std::string_view sv(str);
577 18 : size_t start = 0;
578 238 : for (size_t i = 0; i <= sv.size(); ++i) {
579 220 : if (i == sv.size() || sv[i] == delimiter) {
580 33 : parts.push_back(sv.substr(start, i - start));
581 33 : start = i + 1;
582 : }
583 : }
584 18 : return parts;
585 0 : }
586 :
587 9 : std::vector<std::string> GetVersionErrMessage(const std::string& localVersion, const std::string& remoteVersion)
588 : {
589 9 : std::vector<std::string> versionErrMessage;
590 9 : auto localVersionParts = Split(localVersion, '_');
591 9 : auto remoteVersionParts = Split(remoteVersion, '_');
592 18 : std::string localHcommVersion = localVersionParts.size() > 0 ? std::string(localVersionParts[0]) : "empty";
593 20 : std::string localHcclVersion = localVersionParts.size() > 1 ? std::string(localVersionParts[1]) : "empty";
594 18 : std::string remoteHcommVersion = remoteVersionParts.size() > 0 ? std::string(remoteVersionParts[0]) : "empty";
595 20 : std::string remoteHcclVersion = remoteVersionParts.size() > 1 ? std::string(remoteVersionParts[1]) : "empty";
596 :
597 9 : bool hcommDiff = (localHcommVersion != remoteHcommVersion);
598 9 : bool hcclDiff = (localHcclVersion != remoteHcclVersion);
599 9 : if (hcommDiff && !hcclDiff) {
600 : // 将版本号拆分为hcomm版本号和hccl版本号, 构造ErrMessage
601 : // The %s versions are inconsistent. The local %s, while the remote %s.
602 : versionErrMessage
603 25 : = {"Toolkit (cann-hcomm)", "version is " + localHcommVersion, "version is " + remoteHcommVersion};
604 4 : } else if (!hcommDiff && hcclDiff) {
605 5 : versionErrMessage = {"ops (cann-hccl)", "version is " + localHcclVersion, "version is " + remoteHcclVersion};
606 3 : } else if (hcommDiff && hcclDiff) {
607 : versionErrMessage
608 8 : = {"Toolkit (cann-hcomm) and ops (cann-hccl)",
609 4 : "Toolkit (cann-hcomm) version is " + localHcommVersion + " and ops (cann-hccl) version is "
610 : + localHcclVersion,
611 4 : "Toolkit (cann-hcomm) version is " + remoteHcommVersion + " and ops (cann-hccl) version is "
612 10 : + remoteHcclVersion};
613 : } else {
614 : // 拆分版本号字符串失败, 构造为原版的ErrMessage
615 : // The HCCL versions are inconsistent. The local version is %s, while the remote version is %s.
616 1 : HCCL_ERROR("[RankConsistentcyChecker][GetVersionErrMessage]Failed to split the version string.");
617 5 : versionErrMessage = {"HCCL", "version is " + localVersion, "version is " + remoteVersion};
618 : }
619 9 : return versionErrMessage;
620 31 : }
621 :
622 8 : void RankConsistentcyChecker::CompareVersion(const HcclCheckInfo& local, const HcclCheckInfo& remote, bool& isDiff)
623 : {
624 16 : std::string localCannVersion = local.version;
625 8 : std::string remoteCannVersion = remote.version;
626 8 : if (localCannVersion.empty() || remoteCannVersion.empty()) { // cann版本信息读取失败，返回告警
627 1 : HCCL_WARNING(
628 : "[RankConsistentcyChecker][CompareFrame] CANN version str is empty. local_version %s, "
629 : "remote_version %s.",
630 : local.version, remote.version);
631 7 : } else if (localCannVersion != remoteCannVersion) { // cann版本信息读取成功，且版本不一致
632 5 : const char* ascendHomePath = std::getenv("ASCEND_HOME_PATH");
633 : std::string versionFilePath
634 10 : = (ascendHomePath != nullptr) ? std::string(ascendHomePath) + "/share/info/hccl/version.info" : "unknown";
635 5 : auto versionErrMessage = GetVersionErrMessage(localCannVersion, remoteCannVersion);
636 : // GetVersionErrMessage函数内部已保证versionErrMessage.size() == 3
637 70 : RPT_INPUT_ERR(
638 : true, "EI0008", std::vector<std::string>({"inconsistent_package", "local_version", "remote_version"}),
639 : std::vector<std::string>(
640 : {versionErrMessage[0], versionErrMessage[1] + " (version file: " + versionFilePath + ")",
641 : versionErrMessage[2]}));
642 5 : HCCL_ERROR(
643 : "[%s][%s] errNo[0x%016llx] Inconsistent %s Versions. The local %s, while the remote %s.",
644 : LOG_KEYWORDS_INIT_CHANNEL.c_str(), LOG_KEYWORDS_VERSION_CONFLICT.c_str(), HCCL_ERROR_CODE(HCCL_E_INTERNAL),
645 : versionErrMessage[0].c_str(), versionErrMessage[1].c_str(), versionErrMessage[2].c_str());
646 5 : isDiff = true;
647 5 : }
648 18 : }
649 :
650 9 : bool RankConsistentcyChecker::CompareFrame(HcclCheckInfo& checkInfo, HcclCheckInfo& checkInfoRecv)
651 : {
652 9 : bool bIsDiff = false;
653 9 : if (CompareCrcInfo(checkInfo.cmdInfo, checkInfo.crcInfoGlobal, checkInfoRecv.crcInfoGlobal)) {
654 1 : HCCL_ERROR(
655 : "[RankConsistentcyChecker][CompareFrame]errNo[0x%016llx] CRC check fail, please check the "
656 : "rankTable file and hccl_config file.",
657 : HCCL_ERROR_CODE(HCCL_E_INTERNAL));
658 1 : bIsDiff = true;
659 : }
660 9 : if (CompareCrcInfo(checkInfo.cmdInfo, checkInfo.crcInfoOp, checkInfoRecv.crcInfoOp)) {
661 1 : HCCL_ERROR(
662 : "[RankConsistentcyChecker][CompareFrame]errNo[0x%016llx] Op CRC check fail, please check the op"
663 : " parameters, rankTable file and hccl_config file.",
664 : HCCL_ERROR_CODE(HCCL_E_INTERNAL));
665 1 : bIsDiff = true;
666 : }
667 9 : if (!CompareSection(
668 9 : reinterpret_cast<char_t*>(&checkInfo.cmdInfo), reinterpret_cast<char_t*>(&checkInfoRecv.cmdInfo),
669 : sizeof(checkInfo.cmdInfo))) {
670 1 : CompareCmdInfo(checkInfo, checkInfoRecv);
671 1 : HCCL_ERROR(
672 : "[RankConsistentcyChecker][CompareFrame]errNo[0x%016llx] CMD check fail", HCCL_ERROR_CODE(HCCL_E_INTERNAL));
673 1 : bIsDiff = true;
674 : }
675 9 : HCCL_INFO(
676 : "loc protocolType is [%d], rem protocolType is [%d].", checkInfo.protocolType, checkInfoRecv.protocolType);
677 9 : if (checkInfo.protocolType != checkInfoRecv.protocolType) {
678 1 : HCCL_ERROR(
679 : "[RankConsistentcyChecker][CompareFrame]errNo[0x%016llx] ProtocolType check fail",
680 : HCCL_ERROR_CODE(HCCL_E_INTERNAL));
681 1 : bIsDiff = true;
682 : }
683 :
684 : // Cann版本校验，只在集合通信场景校验CANN版本
685 9 : if (cannVerCheckSwitch_) {
686 8 : CompareVersion(checkInfo, checkInfoRecv, bIsDiff);
687 : }
688 : #ifndef HCCD
689 9 : if (GetExternalInconsistentCheckSwitch() == InconsistentCheckMode::FIRST) {
690 9 : inconsistentCheckFirstDone_ = true;
691 : }
692 : #endif
693 9 : return bIsDiff;
694 : }
695 :
696 9 : HcclResult RankConsistentcyChecker::AddCrc(const u32 crcValue)
697 : {
698 9 : HCCL_DEBUG("crcValue[%u].", crcValue);
699 9 : crcTable_.push_back(crcValue);
700 9 : HCCL_DEBUG("num[%llu].", crcTable_.size());
701 9 : return HCCL_SUCCESS;
702 : }
703 :
704 63 : HcclResult RankConsistentcyChecker::ClearCrcInfo(void)
705 : {
706 63 : this->crcTable_.clear();
707 63 : if (this->crcTable_.size() != 0) {
708 0 : HCCL_ERROR("[Clear][CrcInfo]errNo[0x%016llx] clear crcTable_ is failed", HCCL_ERROR_CODE(HCCL_E_INTERNAL));
709 0 : return HCCL_E_INTERNAL;
710 : }
711 63 : return HCCL_SUCCESS;
712 : }
713 :
714 1 : HcclResult RankConsistentcyChecker::GetCrc(u32 num, u32* crcAddr)
715 : {
716 1 : CHK_PTR_NULL(crcAddr);
717 1 : HCCL_DEBUG("num[%u], crc[%u].", num, *crcAddr);
718 :
719 1 : if (num == 0) {
720 0 : HCCL_ERROR("[Get][Crc]errNo[0x%016llx] In get crc the value of num is 0", HCCL_ERROR_CODE(HCCL_E_PARA));
721 0 : return HCCL_E_PARA;
722 : }
723 :
724 1 : if (num != crcTable_.size()) {
725 0 : HCCL_ERROR(
726 : "[Get][Crc]errNo[0x%016llx] num error inputNum[%u], localNum[%llu]", HCCL_ERROR_CODE(HCCL_E_INTERNAL), num,
727 : crcTable_.size());
728 0 : return HCCL_E_INTERNAL;
729 : }
730 :
731 4 : for (u32 i = 0; i < num; i++) {
732 3 : crcAddr[i] = crcTable_[i];
733 : }
734 1 : return HCCL_SUCCESS;
735 : }
736 : } // namespace hccl
|
