Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "calc_crc.h"
12 : #include "rank_consistentcy_checker.h"
13 : #include "env_config.h"
14 :
15 : namespace hccl {
16 :
17 590 : RankConsistentcyChecker::RankConsistentcyChecker()
18 590 : : cannVersion_{0},
19 590 : cannVerCheckSwitch_(false),
20 590 : cannVerInfoRecordFlag_(false),
21 590 : configFileExist_(false)
22 590 : {}
23 :
24 590 : RankConsistentcyChecker::~RankConsistentcyChecker() = default;
25 :
26 1926 : RankConsistentcyChecker& RankConsistentcyChecker::GetInstance(s32 deviceLogicId)
27 : {
28 2511 : static RankConsistentcyChecker instance[MAX_MODULE_DEVICE_NUM];
29 1926 : if (deviceLogicId == HOST_DEVICE_ID) {
30 0 : HCCL_INFO("[GetInstance] deviceLogicId[-1] is HOST_DEVICE_ID");
31 0 : return instance[0];
32 : }
33 1926 : hrtGetDeviceRefresh(&deviceLogicId);
34 1929 : HCCL_INFO("[GetInstance] get deviceLogicId[%d]", deviceLogicId);
35 1931 : CHK_PRT_RET(
36 : (static_cast<u32>(deviceLogicId) >= MAX_MODULE_DEVICE_NUM || deviceLogicId < 0),
37 : HCCL_WARNING("[R]deviceLogicId[%d] is invalid", deviceLogicId), instance[0]);
38 :
39 1929 : return instance[deviceLogicId];
40 : }
41 :
42 : // gather
43 0 : HcclResult RankConsistentcyChecker::RecordOpPara(
44 : HcclCMDType opCMD, const std::string& tag, u64 count, HcclDataType dataType, u32 root, u64 inCclBufferSize,
45 : u64 outCclBufferSize, const char* group, u32 crc, u32 aivCoreLimit)
46 : {
47 0 : return RecordOpPara(
48 : opCMD, tag, count, dataType, HCCL_REDUCE_RESERVED, root, 0, 0, 0, inCclBufferSize, outCclBufferSize, group, crc,
49 0 : aivCoreLimit);
50 : }
51 :
52 : // reduce
53 77 : HcclResult RankConsistentcyChecker::RecordOpPara(
54 : HcclCMDType opCMD, const std::string& tag, u64 count, HcclDataType dataType, HcclReduceOp op, u32 root,
55 : u64 inCclBufferSize, u64 outCclBufferSize, const char* group, u32 crc, [[maybe_unused]] u8 deterministic,
56 : u32 aivCoreLimit)
57 : {
58 77 : return RecordOpPara(
59 76 : opCMD, tag, count, dataType, op, root, 0, 0, 0, inCclBufferSize, outCclBufferSize, group, crc, aivCoreLimit);
60 : }
61 :
62 : // send && receive
63 0 : HcclResult RankConsistentcyChecker::RecordOpPara(
64 : HcclCMDType opCMD, const std::string& tag, u64 count, HcclDataType dataType, u32 rank, u32 srTag, u32 selfRank,
65 : u64 inCclBufferSize, u64 outCclBufferSize, const char* group, u32 crc)
66 : {
67 0 : return RecordOpPara(
68 : opCMD, tag, count, dataType, HCCL_REDUCE_RESERVED, 0, rank, srTag, selfRank, inCclBufferSize, outCclBufferSize,
69 0 : group, crc);
70 : }
71 :
72 : // batchsendrecv
73 0 : HcclResult RankConsistentcyChecker::RecordOpPara(
74 : HcclCMDType opCMD, const std::string& tag, u64 inCclBufferSize, u64 outCclBufferSize, const char* group, u32 crc)
75 : {
76 0 : return RecordOpPara(
77 : opCMD, tag, 0, HCCL_DATA_TYPE_RESERVED, HCCL_REDUCE_RESERVED, 0, 0, 0, 0, inCclBufferSize, outCclBufferSize,
78 0 : group, crc);
79 : }
80 :
81 : // reduce scatter v && AllGather v
82 0 : HcclResult RankConsistentcyChecker::RecordOpPara(
83 : HcclCMDType opCMD, const std::string& tag, const void* counts, const void* displs, const u32 rankSize,
84 : HcclDataType dataType, HcclReduceOp op, u64 inCclBufferSize, u64 outCclBufferSize, const char* group, u32 crc,
85 : [[maybe_unused]] u8 deterministic, u32 aivCoreLimit)
86 : {
87 0 : CHK_RET(RecordOpPara(
88 : opCMD, tag, 0, dataType, op, 0, 0, 0, 0, inCclBufferSize, outCclBufferSize, group, crc, aivCoreLimit));
89 0 : CHK_RET(RecordVaringOpPara(tag, counts, displs, rankSize));
90 0 : return HCCL_SUCCESS;
91 : }
92 :
93 0 : HcclResult RankConsistentcyChecker::RecordVaringOpPara(
94 : const std::string& tag, const void* counts, const void* displs, const u32 rankSize)
95 : {
96 : u32 countsCrc;
97 0 : CHK_RET(CalcRawDataCrc(static_cast<const char_t*>(counts), rankSize * sizeof(u64), countsCrc));
98 0 : crcRecords_[tag][HcclCrcRecordType::HCCL_CRC_RECORD_VARING_COUNTS] = countsCrc;
99 :
100 : u32 displsCrc;
101 0 : CHK_RET(CalcRawDataCrc(static_cast<const char_t*>(displs), rankSize * sizeof(u64), displsCrc));
102 0 : crcRecords_[tag][HcclCrcRecordType::HCCL_CRC_RECORD_VARING_DISPLACEMENTS] = displsCrc;
103 0 : return HCCL_SUCCESS;
104 : }
105 :
106 24 : HcclResult RankConsistentcyChecker::DelOpPara(const std::string& tag)
107 : {
108 24 : std::lock_guard<std::mutex> lock(mutex_);
109 24 : CHK_PRT_RET(
110 : !cmdInfoMap_.erase(tag),
111 : HCCL_ERROR(
112 : "[RankConsistentcyChecker][DelOpPara]CMD info for tag[%s] does not exist, delete fail.", tag.c_str()),
113 : HCCL_E_INTERNAL);
114 24 : CHK_PRT_RET(
115 : !infoFlagCmdMap_.erase(tag),
116 : HCCL_ERROR(
117 : "[RankConsistentcyChecker][DelOpPara]CMD info flag cmd for tag[%s] does not exist, delete fail.",
118 : tag.c_str()),
119 : HCCL_E_INTERNAL);
120 24 : crcRecords_.erase(tag);
121 24 : return HCCL_SUCCESS;
122 24 : }
123 :
124 236 : HcclResult RankConsistentcyChecker::RecordVerInfo(const std::string& versionInfo)
125 : {
126 236 : std::lock_guard<std::mutex> lock(mutex_);
127 : // Only record once, in case of multiple calls while other process is reading CANN version information
128 : // in CompareFrame func and get the intermediate state.
129 236 : if (cannVerInfoRecordFlag_) {
130 231 : HCCL_INFO("[RankConsistentcyChecker][RecordVerInfo]Cann version information has been recorded.");
131 231 : return HCCL_SUCCESS;
132 : }
133 :
134 5 : u32 strLen = versionInfo.length();
135 5 : s32 sRet = memset_s(cannVersion_, MAX_CANN_VERSION_LEN + 1, 0, MAX_CANN_VERSION_LEN + 1);
136 5 : CHK_PRT_RET(
137 : sRet != EOK,
138 : HCCL_WARNING(
139 : "[RankConsistentcyChecker][RecordVerInfo]memory set 0 fail for version str "
140 : "array. return[%d].",
141 : sRet),
142 : HCCL_SUCCESS);
143 :
144 5 : CHK_PRT_RET(strLen == 0, HCCL_WARNING("[Record][CannVersion] version information str is empty."), HCCL_SUCCESS);
145 :
146 5 : CHK_PRT_RET(
147 : strLen >= MAX_CANN_VERSION_LEN,
148 : HCCL_WARNING("[Record][CannVersion]"
149 : "length of version information str is too long."),
150 : HCCL_SUCCESS);
151 5 : sRet = strncpy_s(cannVersion_, MAX_CANN_VERSION_LEN + 1, versionInfo.c_str(), strLen);
152 5 : CHK_PRT_RET(
153 : sRet != EOK, HCCL_WARNING("[Record][CannVersion] call strncpy_s failed, return [%d].", sRet), HCCL_SUCCESS);
154 :
155 5 : cannVerInfoRecordFlag_ = true;
156 5 : return HCCL_SUCCESS;
157 236 : }
158 :
159 1055 : u64 RankConsistentcyChecker::GetRankConsistentDataLength() { return sizeof(HcclCheckInfo); }
160 :
161 0 : void RankConsistentcyChecker::RecordProtocolType(ProtocolType protocolType)
162 : {
163 0 : HCCL_INFO("[RankConsistentcyChecker][RecordProtocolType]protocolType set to [%d].", static_cast<s32>(protocolType));
164 0 : protocolType_ = protocolType;
165 0 : return;
166 : }
167 :
168 4 : HcclResult RankConsistentcyChecker::GetCheckFrame(u8* destBuf, u64 maxDestBuf, const std::string& tag)
169 : {
170 4 : CHK_PTR_NULL(destBuf);
171 : // 要发送的校验帧
172 4 : HcclCheckInfo checkInfo;
173 4 : u64 checkInfoLen = sizeof(checkInfo);
174 4 : HcclResult ret = GenerateCheckFrame(checkInfo, tag);
175 4 : checkInfo.cmdInfo.selfRank = 0; // 自身的group rank 不做校验,置0
176 4 : CHK_PRT_RET(
177 : ret != HCCL_SUCCESS,
178 : HCCL_ERROR(
179 : "[RankConsistentcyChecker][GetCheckFrame]generate check frame fail. "
180 : "return[%d]",
181 : ret),
182 : ret);
183 :
184 4 : s32 sret = memcpy_s(destBuf, maxDestBuf, &checkInfo, checkInfoLen);
185 4 : CHK_PRT_RET(
186 : sret != EOK,
187 : HCCL_ERROR(
188 : "[RankConsistentcyChecker][GetCheckFrame]frame len[%llu] is bigger than "
189 : "dest buffer len[%llu].",
190 : checkInfoLen, maxDestBuf),
191 : HCCL_E_INTERNAL);
192 4 : return HCCL_SUCCESS;
193 : }
194 :
195 0 : HcclResult RankConsistentcyChecker::CheckFrameRecv(const u8* recvBuf, u32 recvBufLen, const std::string& tag)
196 : {
197 : #ifndef HCCD
198 0 : if ((GetExternalInconsistentCheckSwitch() == InconsistentCheckMode::OFF)
199 0 : || (GetExternalInconsistentCheckSwitch() == InconsistentCheckMode::FIRST
200 0 : && inconsistentCheckFirstDone_ == true)) {
201 0 : return HCCL_SUCCESS;
202 : }
203 : #endif
204 0 : CHK_PTR_NULL(recvBuf);
205 0 : CHK_PRT_RET(
206 : recvBufLen == 0 || recvBufLen > MAX_FRAME_LEN,
207 : HCCL_ERROR(
208 : "[RankConsistentcyChecker][CheckFrameRecv] errNo[0x%016llx] recvBufLen[%u] is invalid, "
209 : "valid range is (0, %u].",
210 : HCCL_ERROR_CODE(HCCL_E_INTERNAL), recvBufLen, MAX_FRAME_LEN),
211 : HCCL_E_INTERNAL);
212 :
213 0 : CHK_PRT_RET(
214 : recvBufLen < sizeof(HcclCheckInfo),
215 : HCCL_ERROR(
216 : "[RankConsistentcyChecker][CheckFrameRecv] errNo[0x%016llx] recvBufLen[%u]is less than "
217 : "check info[%zu].",
218 : HCCL_ERROR_CODE(HCCL_E_INTERNAL), recvBufLen, sizeof(HcclCheckInfo)),
219 : HCCL_E_PARA);
220 :
221 0 : HcclCheckInfo checkInfoRecv;
222 : // 对固定长度的全局数组变量，结构体变量进行初始化和拷贝，可以不用检查初始化安全函数返回值
223 0 : (void)memset_s(&checkInfoRecv, sizeof(HcclCheckInfo), 0, sizeof(HcclCheckInfo));
224 0 : (void)memcpy_s(&checkInfoRecv, sizeof(HcclCheckInfo), recvBuf, sizeof(HcclCheckInfo));
225 :
226 0 : HcclCheckInfo checkInfo;
227 0 : CHK_RET(GenerateCheckFrame(checkInfo, tag));
228 0 : if (checkInfo.cmdInfo.cmdType == HcclCMDType::HCCL_CMD_SEND) {
229 0 : checkInfo.cmdInfo.cmdType = HcclCMDType::HCCL_CMD_RECEIVE;
230 0 : checkInfo.cmdInfo.rank = checkInfo.cmdInfo.selfRank;
231 0 : } else if (checkInfo.cmdInfo.cmdType == HcclCMDType::HCCL_CMD_RECEIVE) {
232 0 : checkInfo.cmdInfo.cmdType = HcclCMDType::HCCL_CMD_SEND;
233 0 : checkInfo.cmdInfo.rank = checkInfo.cmdInfo.selfRank;
234 : }
235 :
236 0 : checkInfo.cmdInfo.selfRank = 0; // 自身的子group rank 不做校验
237 0 : if (CompareFrame(checkInfo, checkInfoRecv)) {
238 0 : return HCCL_E_INTERNAL;
239 : }
240 :
241 0 : HCCL_INFO(
242 : "[RankConsistentcyChecker][CheckFrameRecv] check success, len of frame[%u], len of check data[%zu].",
243 : recvBufLen, sizeof(checkInfo));
244 0 : return HCCL_SUCCESS;
245 : }
246 :
247 62 : void RankConsistentcyChecker::ClearCheckInfo()
248 : {
249 62 : configFileExist_ = false;
250 62 : cannVerInfoRecordFlag_ = false;
251 62 : ClearCrcInfo();
252 : {
253 62 : std::lock_guard<std::mutex> lock(mutex_);
254 62 : cmdInfoMap_.clear();
255 62 : infoFlagCmdMap_.clear();
256 62 : }
257 : // 相关规范的例外场景，对固定数组的memset_s可以不判断返回值
258 62 : (void)memset_s(cannVersion_, MAX_CANN_VERSION_LEN + 1, 0, MAX_CANN_VERSION_LEN + 1);
259 62 : protocolType_ = ProtocolType::RESERVED;
260 62 : return;
261 : }
262 :
263 236 : HcclResult RankConsistentcyChecker::CalcStringCrc(const char* str, u32& crc)
264 : {
265 : // 计算字符串CRC
266 236 : HcclResult ret = CalcCrc::HcclCalcCrc(str, strlen(str), crc);
267 236 : CHK_PRT_RET(
268 : ret != HCCL_SUCCESS,
269 : HCCL_ERROR(
270 : "[RankConsistentcyChecker][CalcStringCrc] errNo[0x%016llx] calc string crc error",
271 : HCCL_ERROR_CODE(HCCL_E_INTERNAL)),
272 : HCCL_E_INTERNAL);
273 :
274 236 : HCCL_DEBUG("[RankConsistentcyChecker][CalcStringCrc] result crc[%u].", crc);
275 236 : return HCCL_SUCCESS;
276 : }
277 :
278 0 : HcclResult RankConsistentcyChecker::CalcRawDataCrc(const void* ptr, u64 length, u32& crc)
279 : {
280 : // 计算内存数据块CRC
281 0 : HcclResult ret = CalcCrc::HcclCalcCrc(static_cast<const char*>(ptr), length, crc);
282 0 : CHK_PRT_RET(
283 : ret != HCCL_SUCCESS,
284 : HCCL_ERROR(
285 : "[RankConsistentcyChecker][CalcRawDataCrc] errNo[0x%016llx] calc string crc error",
286 : HCCL_ERROR_CODE(HCCL_E_INTERNAL)),
287 : HCCL_E_INTERNAL);
288 :
289 0 : HCCL_DEBUG("[RankConsistentcyChecker][CalcRawDataCrc] result crc[%u].", crc);
290 0 : return HCCL_SUCCESS;
291 : }
292 :
293 240 : void RankConsistentcyChecker::SetCheckCannVersionSwitch(const bool cannVerCheckSwitch)
294 : {
295 240 : cannVerCheckSwitch_ = cannVerCheckSwitch;
296 240 : return;
297 : }
298 :
299 : // private
300 77 : HcclResult RankConsistentcyChecker::RecordOpPara(
301 : HcclCMDType opCMD, const std::string& tag, u64 count, HcclDataType dataType, HcclReduceOp op, u32 root, u32 rank,
302 : u32 srTag, u32 selfRank, u64 inCclBufferSize, u64 outCclBufferSize, const char* group, u32 crc, u8 deterministic,
303 : u32 aivCoreLimit)
304 : {
305 77 : HcclCMDInfo cmdInfo;
306 : // 相关规范的例外场景，对固定数组的memset_s可以不判断返回值
307 77 : (void)memset_s(&cmdInfo, sizeof(cmdInfo), 0, sizeof(cmdInfo));
308 :
309 77 : cmdInfo.cmdType = opCMD;
310 77 : s32 sRet = strncpy_s(cmdInfo.tag, TAG_MAX_LEN + 1, tag.c_str(), tag.length());
311 77 : CHK_PRT_RET(
312 : sRet != EOK,
313 : HCCL_ERROR(
314 : "[RankConsistentcyChecker][RecordOpPara]errNo[0x%016llx] strlen[%u] of tag is "
315 : "longer than buffer[%u].",
316 : HCCL_ERROR_CODE(HCCL_E_PARA), tag.length(), TAG_MAX_LEN),
317 : HCCL_E_PARA);
318 :
319 77 : cmdInfo.count = count;
320 77 : cmdInfo.dataType = dataType;
321 :
322 77 : std::string strGroup = (group == nullptr) ? HCCL_WORLD_GROUP : group;
323 75 : sRet = strncpy_s(cmdInfo.group, GROUP_NAME_MAX_LEN + 1, strGroup.c_str(), strGroup.length());
324 74 : CHK_PRT_RET(
325 : sRet != EOK,
326 : HCCL_ERROR(
327 : "[RankConsistentcyChecker][RecordOpPara]errNo[0x%016llx] strlen[%u] group is "
328 : "longer than buffer[%u].",
329 : HCCL_ERROR_CODE(HCCL_E_PARA), strGroup.length(), GROUP_NAME_MAX_LEN),
330 : HCCL_E_PARA);
331 :
332 74 : cmdInfo.op = op;
333 74 : cmdInfo.root = root;
334 74 : cmdInfo.rank = rank;
335 74 : cmdInfo.srTag = srTag;
336 74 : cmdInfo.selfRank = selfRank;
337 74 : cmdInfo.inCclBufferSize = inCclBufferSize;
338 74 : cmdInfo.outCclBufferSize = outCclBufferSize;
339 74 : cmdInfo.aivCoreLimit = aivCoreLimit;
340 74 : cmdInfo.deterministic = deterministic;
341 :
342 74 : std::lock_guard<std::mutex> lock(mutex_);
343 76 : cmdInfoMap_[tag] = cmdInfo;
344 73 : infoFlagCmdMap_[tag] = true;
345 69 : crcRecords_[tag][HcclCrcRecordType::HCCL_CRC_RECORD_RANKTABLE] = crc;
346 :
347 62 : return HCCL_SUCCESS;
348 62 : }
349 :
350 0 : HcclResult RankConsistentcyChecker::GetOpParaByTag(const std::string& tag, HcclCMDInfo& CMDInfoOutput)
351 : {
352 0 : auto getResult = cmdInfoMap_.find(tag);
353 0 : CHK_PRT_RET(
354 : getResult == cmdInfoMap_.end(),
355 : HCCL_ERROR(
356 : "[RankConsistentcyChecker][GetOpParaByTag]There is not any CMD information for tag[%s]", tag.c_str()),
357 : HCCL_E_INTERNAL);
358 0 : CMDInfoOutput = getResult->second;
359 0 : return HCCL_SUCCESS;
360 : }
361 :
362 0 : HcclResult RankConsistentcyChecker::GetCrcByTag(const std::string& tag, HcclCRCInfo& crcInfo)
363 : {
364 0 : const auto recordsIter = crcRecords_.find(tag);
365 0 : CHK_PRT_RET(
366 : recordsIter == crcRecords_.end(),
367 : HCCL_ERROR("[RankConsistentcyChecker][GetCrcByTag]There is not any CRC information for tag[%s]", tag.c_str()),
368 : HCCL_E_INTERNAL);
369 0 : crcInfo.crcNum = 0;
370 0 : const auto& tagRecords = recordsIter->second;
371 0 : for (const auto& record : tagRecords) {
372 0 : crcInfo.crcArray[crcInfo.crcNum++] = record.second;
373 0 : HCCL_DEBUG("[RankConsistentcyChecker][GetCrcByTag]Append crc[%u] for tag[%s].", record.second, tag.c_str());
374 : }
375 :
376 0 : HCCL_INFO(
377 : "[RankConsistentcyChecker][GetCrcByTag]After adding crc for tag[%s], crcNum set to [%u].", tag.c_str(),
378 : crcInfo.crcNum);
379 0 : return HCCL_SUCCESS;
380 : }
381 :
382 3 : HcclResult RankConsistentcyChecker::GenerateCheckFrame(HcclCheckInfo& checkInfo, const std::string& tag)
383 : {
384 : // 初始化用于发送的BUFFER
385 : // 对入参的结构体变量指向的内存进行初始化时，使用了变量的结构体类型大小进行初始化，
386 : // 如果指针不为空，可以不检查初始化安全函数的返回值
387 3 : u64 checkInfoLen = sizeof(HcclCheckInfo);
388 3 : (void)memset_s(&checkInfo, checkInfoLen, 0, checkInfoLen);
389 :
390 : // 添加CRC字段到校验帧
391 4 : u32 crcLen = crcTable_.size();
392 4 : checkInfo.crcInfoGlobal.configFileExist_ = configFileExist_;
393 4 : if (crcLen != 0) {
394 0 : CHK_PRT_RET(
395 : crcLen > MAX_CRC_LEN,
396 : HCCL_ERROR("[RankConsistentcyChecker][GenerateCheckFrame]crc num[%u] is too big.", crcLen),
397 : HCCL_E_INTERNAL);
398 0 : checkInfo.crcInfoGlobal.crcNum = crcLen;
399 0 : CHK_RET(GetCrc(crcLen, &checkInfo.crcInfoGlobal.crcArray[0]));
400 : }
401 : // 添加CMD参数信息到校验帧
402 : {
403 4 : std::lock_guard<std::mutex> lock(mutex_);
404 4 : auto getResult = infoFlagCmdMap_.find(tag);
405 4 : if (getResult != infoFlagCmdMap_.end()) {
406 0 : CHK_PRT_RET(
407 : GetOpParaByTag(tag, checkInfo.cmdInfo) != HCCL_SUCCESS,
408 : HCCL_ERROR("[RankConsistentcyChecker][GenerateCheckFrame]get Op para by tag[%s] fail.", tag.c_str()),
409 : HCCL_E_INTERNAL);
410 0 : checkInfo.crcInfoOp.configFileExist_ = configFileExist_;
411 : // 添加CRC字段到校验帧
412 0 : CHK_PRT_RET(
413 : GetCrcByTag(tag, checkInfo.crcInfoOp) != HCCL_SUCCESS,
414 : HCCL_ERROR(
415 : "[RankConsistentcyChecker][GenerateCheckFrame]get ranktable crc by tag[%s] fail.", tag.c_str()),
416 : HCCL_E_INTERNAL);
417 : }
418 4 : }
419 : // 添加HCCL版本信息到校验帧
420 4 : if (cannVerInfoRecordFlag_) {
421 0 : HCCL_DEBUG("[RankConsistentcyChecker][GenerateCheckFrame] CANN version information is [%s].", cannVersion_);
422 0 : s32 sret = memcpy_s(checkInfo.version, MAX_CANN_VERSION_LEN + 1, cannVersion_, strlen(cannVersion_));
423 0 : CHK_PRT_RET(
424 : sret != EOK,
425 : HCCL_ERROR(
426 : "[RankConsistentcyChecker][GenerateCheckFrame] memcpy CANN version information failed, "
427 : "errorno [%d].",
428 : sret),
429 : HCCL_E_MEMORY);
430 : }
431 : // 添加拉远通信传输类型校验
432 : // 910* 不会配置isTcpMode，因此910*在此处的待校验值是一致的
433 4 : checkInfo.protocolType = protocolType_;
434 4 : HCCL_INFO("[RankConsistentcyChecker][GenerateCheckFrame] loc protocolType is [%d].", checkInfo.protocolType);
435 :
436 4 : return HCCL_SUCCESS;
437 : }
438 :
439 29 : bool RankConsistentcyChecker::CompareSection(const char* pRawData, const char* recvBuf, u32 len)
440 : {
441 11741 : for (u32 i = 0; i < len; i++) {
442 11715 : if (*(pRawData + i) != *(recvBuf + i)) {
443 3 : return false;
444 : }
445 : }
446 26 : return true;
447 : }
448 :
449 18 : bool RankConsistentcyChecker::CompareCrcInfo(
450 : const HcclCMDInfo& hcclCMDInfo, HcclCRCInfo& crcInfo, HcclCRCInfo& crcInfoRecv)
451 : {
452 18 : bool bIsDiff = false;
453 : // 检校验整体是否一致
454 18 : if (!CompareSection(
455 : reinterpret_cast<char_t*>(&crcInfo), reinterpret_cast<char_t*>(&crcInfoRecv), sizeof(crcInfo))) {
456 2 : bIsDiff = true;
457 : // 检查每种CRC类型是否一致
458 4 : for (auto i = 0U; i < crcInfo.crcNum; ++i) {
459 2 : if (crcInfo.crcArray[i] != crcInfoRecv.crcArray[i]) {
460 2 : ReportCrcCheckFailed(
461 : hcclCMDInfo, static_cast<HcclCrcRecordType>(i), crcInfo.crcArray[i], crcInfoRecv.crcArray[i]);
462 : }
463 : }
464 : }
465 18 : return bIsDiff;
466 : }
467 :
468 0 : void RankConsistentcyChecker::ReportCmdInfoCheckFailed(
469 : const HcclCMDInfo& hcclCMDInfo, const std::string& paraName, const std::string& localPara,
470 : const std::string& remotePara) const
471 : {
472 0 : ReportCommonError(hcclCMDInfo, paraName, localPara, remotePara, "CMD information");
473 0 : }
474 :
475 3 : void RankConsistentcyChecker::ReportCmdInfoCheckFailed(
476 : const HcclCMDInfo& hcclCMDInfo, const std::string& paraName, uint32_t localPara, uint32_t remotePara)
477 : {
478 3 : ReportCommonError(hcclCMDInfo, paraName, std::to_string(localPara), std::to_string(remotePara), "CMD information");
479 3 : }
480 :
481 2 : void RankConsistentcyChecker::ReportCrcCheckFailed(
482 : const HcclCMDInfo& hcclCMDInfo, HcclCrcRecordType crcType, const uint32_t localCrc, const uint32_t remoteCrc) const
483 : {
484 2 : const auto crcTypeStr = GetCRCTypeEnumStr(crcType);
485 2 : ReportCommonError(hcclCMDInfo, crcTypeStr, std::to_string(localCrc), std::to_string(remoteCrc), "CRC check");
486 2 : }
487 :
488 5 : void RankConsistentcyChecker::ReportCommonError(
489 : const HcclCMDInfo& hcclCMDInfo, const std::string& paraName, const std::string& localParaStr,
490 : const std::string& remoteParaStr, const std::string& errorMsg) const
491 : {
492 5 : std::string opInfo = "Unknown";
493 15 : for (const auto& pair : HCCL_OPTYPE_NAME_MAP) {
494 15 : if (pair.second == hcclCMDInfo.cmdType) {
495 5 : opInfo = std::string(pair.first);
496 5 : break;
497 : }
498 : }
499 100 : RPT_INPUT_ERR(
500 : true, "EI0005", std::vector<std::string>({"ccl_op", "group", "para_name", "local_para", "remote_para"}),
501 : std::vector<std::string>({opInfo, hcclCMDInfo.group, paraName, localParaStr, remoteParaStr}));
502 5 : HCCL_ERROR(
503 : "[%s][%s]%s %s check fail. local[%s], remote[%s]", LOG_KEYWORDS_INIT_CHANNEL.c_str(),
504 : LOG_KEYWORDS_PARAMETER_CONFLICT.c_str(), errorMsg.c_str(), paraName.c_str(), localParaStr.c_str(),
505 : remoteParaStr.c_str());
506 15 : }
507 :
508 1 : void RankConsistentcyChecker::CompareCmdInfo(HcclCheckInfo& checkInfo, HcclCheckInfo& checkInfoRecv)
509 : {
510 1 : auto localInfo = &checkInfo.cmdInfo;
511 1 : auto remoteInfo = &checkInfoRecv.cmdInfo;
512 :
513 1 : if (!CompareSection(localInfo->tag, remoteInfo->tag, TAG_MAX_LEN + 1)) {
514 0 : ReportCmdInfoCheckFailed(*localInfo, "tag", localInfo->tag, remoteInfo->tag);
515 : }
516 :
517 1 : if (localInfo->cmdType != remoteInfo->cmdType) {
518 2 : ReportCmdInfoCheckFailed(
519 1 : *localInfo, "cmdType", static_cast<uint32_t>(localInfo->cmdType),
520 1 : static_cast<uint32_t>(remoteInfo->cmdType));
521 : }
522 :
523 1 : if (localInfo->count != remoteInfo->count) {
524 3 : ReportCmdInfoCheckFailed(*localInfo, "count", localInfo->count, remoteInfo->count);
525 : }
526 :
527 1 : if (localInfo->dataType != remoteInfo->dataType) {
528 2 : ReportCmdInfoCheckFailed(
529 1 : *localInfo, "dataType", static_cast<uint32_t>(localInfo->dataType),
530 1 : static_cast<uint32_t>(remoteInfo->dataType));
531 : }
532 :
533 1 : if (localInfo->op != remoteInfo->op) {
534 0 : ReportCmdInfoCheckFailed(
535 0 : *localInfo, "op", static_cast<uint32_t>(localInfo->op), static_cast<uint32_t>(remoteInfo->op));
536 : }
537 :
538 1 : if (!CompareSection(localInfo->group, remoteInfo->group, GROUP_NAME_MAX_LEN + 1)) {
539 0 : ReportCmdInfoCheckFailed(*localInfo, "group", localInfo->group, remoteInfo->group);
540 : }
541 :
542 1 : if (localInfo->root != remoteInfo->root) {
543 0 : ReportCmdInfoCheckFailed(*localInfo, "root", localInfo->root, remoteInfo->root);
544 : }
545 :
546 1 : if (localInfo->rank != remoteInfo->rank) {
547 0 : ReportCmdInfoCheckFailed(*localInfo, "rank", localInfo->rank, remoteInfo->rank);
548 : }
549 :
550 1 : if (localInfo->srTag != remoteInfo->srTag) {
551 0 : ReportCmdInfoCheckFailed(*localInfo, "srTag", localInfo->srTag, remoteInfo->srTag);
552 : }
553 :
554 1 : if (localInfo->inCclBufferSize != remoteInfo->inCclBufferSize) {
555 0 : ReportCmdInfoCheckFailed(
556 0 : *localInfo, "inCclBufferSize", localInfo->inCclBufferSize, remoteInfo->inCclBufferSize);
557 : }
558 :
559 1 : if (localInfo->outCclBufferSize != remoteInfo->outCclBufferSize) {
560 0 : ReportCmdInfoCheckFailed(
561 0 : *localInfo, "outCclBufferSize", localInfo->outCclBufferSize, remoteInfo->outCclBufferSize);
562 : }
563 :
564 1 : if (localInfo->aivCoreLimit != remoteInfo->aivCoreLimit) {
565 0 : ReportCmdInfoCheckFailed(*localInfo, "aivCoreLimit", localInfo->aivCoreLimit, remoteInfo->aivCoreLimit);
566 : }
567 :
568 1 : if (localInfo->deterministic != remoteInfo->deterministic) {
569 0 : ReportCmdInfoCheckFailed(*localInfo, "deterministic", localInfo->deterministic, remoteInfo->deterministic);
570 : }
571 :
572 1 : return;
573 : }
574 :
575 18 : std::vector<std::string_view> Split(const std::string& str, char delimiter)
576 : {
577 18 : std::vector<std::string_view> parts;
578 18 : std::string_view sv(str);
579 18 : size_t start = 0;
580 238 : for (size_t i = 0; i <= sv.size(); ++i) {
581 220 : if (i == sv.size() || sv[i] == delimiter) {
582 33 : parts.push_back(sv.substr(start, i - start));
583 33 : start = i + 1;
584 : }
585 : }
586 18 : return parts;
587 0 : }
588 :
589 9 : std::vector<std::string> GetVersionErrMessage(const std::string& localVersion, const std::string& remoteVersion)
590 : {
591 9 : std::vector<std::string> versionErrMessage;
592 9 : auto localVersionParts = Split(localVersion, '_');
593 9 : auto remoteVersionParts = Split(remoteVersion, '_');
594 18 : std::string localHcommVersion = localVersionParts.size() > 0 ? std::string(localVersionParts[0]) : "empty";
595 20 : std::string localHcclVersion = localVersionParts.size() > 1 ? std::string(localVersionParts[1]) : "empty";
596 18 : std::string remoteHcommVersion = remoteVersionParts.size() > 0 ? std::string(remoteVersionParts[0]) : "empty";
597 20 : std::string remoteHcclVersion = remoteVersionParts.size() > 1 ? std::string(remoteVersionParts[1]) : "empty";
598 :
599 9 : bool hcommDiff = (localHcommVersion != remoteHcommVersion);
600 9 : bool hcclDiff = (localHcclVersion != remoteHcclVersion);
601 9 : if (hcommDiff && !hcclDiff) {
602 : // 将版本号拆分为hcomm版本号和hccl版本号, 构造ErrMessage
603 : // The %s versions are inconsistent. The local %s, while the remote %s.
604 : versionErrMessage
605 25 : = {"Toolkit (cann-hcomm)", "version is " + localHcommVersion, "version is " + remoteHcommVersion};
606 4 : } else if (!hcommDiff && hcclDiff) {
607 5 : versionErrMessage = {"ops (cann-hccl)", "version is " + localHcclVersion, "version is " + remoteHcclVersion};
608 3 : } else if (hcommDiff && hcclDiff) {
609 : versionErrMessage
610 8 : = {"Toolkit (cann-hcomm) and ops (cann-hccl)",
611 4 : "Toolkit (cann-hcomm) version is " + localHcommVersion + " and ops (cann-hccl) version is "
612 : + localHcclVersion,
613 4 : "Toolkit (cann-hcomm) version is " + remoteHcommVersion + " and ops (cann-hccl) version is "
614 10 : + remoteHcclVersion};
615 : } else {
616 : // 拆分版本号字符串失败, 构造为原版的ErrMessage
617 : // The HCCL versions are inconsistent. The local version is %s, while the remote version is %s.
618 1 : HCCL_ERROR("[RankConsistentcyChecker][GetVersionErrMessage]Failed to split the version string.");
619 5 : versionErrMessage = {"HCCL", "version is " + localVersion, "version is " + remoteVersion};
620 : }
621 9 : return versionErrMessage;
622 31 : }
623 :
624 8 : void RankConsistentcyChecker::CompareVersion(const HcclCheckInfo& local, const HcclCheckInfo& remote, bool& isDiff)
625 : {
626 16 : std::string localCannVersion = local.version;
627 8 : std::string remoteCannVersion = remote.version;
628 8 : if (localCannVersion.empty() || remoteCannVersion.empty()) { // cann版本信息读取失败，返回告警
629 1 : HCCL_WARNING(
630 : "[RankConsistentcyChecker][CompareFrame] CANN version str is empty. local_version %s, "
631 : "remote_version %s.",
632 : local.version, remote.version);
633 7 : } else if (localCannVersion != remoteCannVersion) { // cann版本信息读取成功，且版本不一致
634 5 : const char* ascendHomePath = std::getenv("ASCEND_HOME_PATH");
635 : std::string versionFilePath
636 10 : = (ascendHomePath != nullptr) ? std::string(ascendHomePath) + "/share/info/hccl/version.info" : "unknown";
637 5 : auto versionErrMessage = GetVersionErrMessage(localCannVersion, remoteCannVersion);
638 : // GetVersionErrMessage函数内部已保证versionErrMessage.size() == 3
639 70 : RPT_INPUT_ERR(
640 : true, "EI0008", std::vector<std::string>({"inconsistent_package", "local_version", "remote_version"}),
641 : std::vector<std::string>(
642 : {versionErrMessage[0], versionErrMessage[1] + " (version file: " + versionFilePath + ")",
643 : versionErrMessage[2]}));
644 5 : HCCL_ERROR(
645 : "[%s][%s] errNo[0x%016llx] Inconsistent %s Versions. The local %s, while the remote %s.",
646 : LOG_KEYWORDS_INIT_CHANNEL.c_str(), LOG_KEYWORDS_VERSION_CONFLICT.c_str(), HCCL_ERROR_CODE(HCCL_E_INTERNAL),
647 : versionErrMessage[0].c_str(), versionErrMessage[1].c_str(), versionErrMessage[2].c_str());
648 5 : isDiff = true;
649 5 : }
650 18 : }
651 :
652 9 : bool RankConsistentcyChecker::CompareFrame(HcclCheckInfo& checkInfo, HcclCheckInfo& checkInfoRecv)
653 : {
654 9 : bool bIsDiff = false;
655 9 : if (CompareCrcInfo(checkInfo.cmdInfo, checkInfo.crcInfoGlobal, checkInfoRecv.crcInfoGlobal)) {
656 1 : HCCL_ERROR(
657 : "[RankConsistentcyChecker][CompareFrame]errNo[0x%016llx] CRC check fail, please check the "
658 : "rankTable file and hccl_config file.",
659 : HCCL_ERROR_CODE(HCCL_E_INTERNAL));
660 1 : bIsDiff = true;
661 : }
662 9 : if (CompareCrcInfo(checkInfo.cmdInfo, checkInfo.crcInfoOp, checkInfoRecv.crcInfoOp)) {
663 1 : HCCL_ERROR(
664 : "[RankConsistentcyChecker][CompareFrame]errNo[0x%016llx] Op CRC check fail, please check the op"
665 : " parameters, rankTable file and hccl_config file.",
666 : HCCL_ERROR_CODE(HCCL_E_INTERNAL));
667 1 : bIsDiff = true;
668 : }
669 9 : if (!CompareSection(
670 9 : reinterpret_cast<char_t*>(&checkInfo.cmdInfo), reinterpret_cast<char_t*>(&checkInfoRecv.cmdInfo),
671 : sizeof(checkInfo.cmdInfo))) {
672 1 : CompareCmdInfo(checkInfo, checkInfoRecv);
673 1 : HCCL_ERROR(
674 : "[RankConsistentcyChecker][CompareFrame]errNo[0x%016llx] CMD check fail", HCCL_ERROR_CODE(HCCL_E_INTERNAL));
675 1 : bIsDiff = true;
676 : }
677 9 : HCCL_INFO(
678 : "loc protocolType is [%d], rem protocolType is [%d].", checkInfo.protocolType, checkInfoRecv.protocolType);
679 9 : if (checkInfo.protocolType != checkInfoRecv.protocolType) {
680 1 : HCCL_ERROR(
681 : "[RankConsistentcyChecker][CompareFrame]errNo[0x%016llx] ProtocolType check fail",
682 : HCCL_ERROR_CODE(HCCL_E_INTERNAL));
683 1 : bIsDiff = true;
684 : }
685 :
686 : // Cann版本校验，只在集合通信场景校验CANN版本
687 9 : if (cannVerCheckSwitch_) {
688 8 : CompareVersion(checkInfo, checkInfoRecv, bIsDiff);
689 : }
690 : #ifndef HCCD
691 9 : if (GetExternalInconsistentCheckSwitch() == InconsistentCheckMode::FIRST) {
692 9 : inconsistentCheckFirstDone_ = true;
693 : }
694 : #endif
695 9 : return bIsDiff;
696 : }
697 :
698 9 : HcclResult RankConsistentcyChecker::AddCrc(const u32 crcValue)
699 : {
700 9 : HCCL_DEBUG("crcValue[%u].", crcValue);
701 9 : crcTable_.push_back(crcValue);
702 9 : HCCL_DEBUG("num[%llu].", crcTable_.size());
703 9 : return HCCL_SUCCESS;
704 : }
705 :
706 63 : HcclResult RankConsistentcyChecker::ClearCrcInfo(void)
707 : {
708 63 : this->crcTable_.clear();
709 63 : if (this->crcTable_.size() != 0) {
710 0 : HCCL_ERROR("[Clear][CrcInfo]errNo[0x%016llx] clear crcTable_ is failed", HCCL_ERROR_CODE(HCCL_E_INTERNAL));
711 0 : return HCCL_E_INTERNAL;
712 : }
713 63 : return HCCL_SUCCESS;
714 : }
715 :
716 1 : HcclResult RankConsistentcyChecker::GetCrc(u32 num, u32* crcAddr)
717 : {
718 1 : CHK_PTR_NULL(crcAddr);
719 1 : HCCL_DEBUG("num[%u], crc[%u].", num, *crcAddr);
720 :
721 1 : if (num == 0) {
722 0 : HCCL_ERROR("[Get][Crc]errNo[0x%016llx] In get crc the value of num is 0", HCCL_ERROR_CODE(HCCL_E_PARA));
723 0 : return HCCL_E_PARA;
724 : }
725 :
726 1 : if (num != crcTable_.size()) {
727 0 : HCCL_ERROR(
728 : "[Get][Crc]errNo[0x%016llx] num error inputNum[%u], localNum[%llu]", HCCL_ERROR_CODE(HCCL_E_INTERNAL), num,
729 : crcTable_.size());
730 0 : return HCCL_E_INTERNAL;
731 : }
732 :
733 4 : for (u32 i = 0; i < num; i++) {
734 3 : crcAddr[i] = crcTable_[i];
735 : }
736 1 : return HCCL_SUCCESS;
737 : }
738 : } // namespace hccl
|
