Line data Source code
1 : /**
2 : * Copyright (c) 2026 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "rank_consistency_checker_v2.h"
12 : #include "adapter_pub.h"
13 : #include "adapter_error_manager_pub.h"
14 : #include "checkcrc.h"
15 : using namespace Hccl;
16 :
17 : namespace hccl {
18 130 : RankConsistencyCheckerV2::~RankConsistencyCheckerV2() = default;
19 :
20 15 : RankConsistencyCheckerV2& RankConsistencyCheckerV2::GetInstance(const s32& deviceLogicId)
21 : {
22 145 : static RankConsistencyCheckerV2 instance[MAX_MODULE_DEVICE_NUM];
23 15 : HCCL_INFO("[RankConsistencyCheckerV2][GetInstance] get deviceLogicId[%d]", deviceLogicId);
24 15 : CHK_PRT_RET(
25 : (static_cast<u32>(deviceLogicId) >= MAX_MODULE_DEVICE_NUM || deviceLogicId < 0),
26 : HCCL_WARNING("[RankConsistencyCheckerV2::RankConsistencyCheckerV2]deviceLogicId[%d] is invalid", deviceLogicId),
27 : instance[0]);
28 15 : return instance[deviceLogicId];
29 : }
30 :
31 2 : HcclResult RankConsistencyCheckerV2::RecordRankTableCrcV2(u32 crc)
32 : {
33 2 : rankTableCrcV2_.clear();
34 4 : rankTableCrcV2_.emplace_back(std::string("ranktable_content"), crc);
35 2 : HCCL_INFO("[RankConsistencyCheckerV2::RecordRankTableCrcV2] ranktable crc[%u] recorded.", crc);
36 2 : return HCCL_SUCCESS;
37 : }
38 :
39 7 : HcclResult RankConsistencyCheckerV2::RecordCannVersionV2(const std::string& version)
40 : {
41 7 : u32 strLen = version.length();
42 7 : s32 sRet = memset_s(cannVersion_, CANN_VERSION_MAX_LEN + 1, 0, CANN_VERSION_MAX_LEN + 1);
43 7 : CHK_PRT_RET(
44 : sRet != EOK,
45 : HCCL_WARNING(
46 : "[RankConsistencyCheckerV2::RecordCannVersionV2]memory set 0 fail for version str "
47 : "array. return[%d].",
48 : sRet),
49 : HCCL_SUCCESS);
50 :
51 7 : CHK_PRT_RET(
52 : strLen == 0, HCCL_WARNING("[RankConsistencyCheckerV2::RecordCannVersionV2] version information str is empty."),
53 : HCCL_SUCCESS);
54 :
55 7 : CHK_PRT_RET(
56 : strLen >= CANN_VERSION_MAX_LEN + 1,
57 : HCCL_WARNING("[RankConsistencyCheckerV2::RecordCannVersionV2]"
58 : "length of version information str is too long."),
59 : HCCL_SUCCESS);
60 7 : sRet = strncpy_s(cannVersion_, CANN_VERSION_MAX_LEN + 1, version.c_str(), strLen);
61 7 : CHK_PRT_RET(
62 : sRet != EOK,
63 : HCCL_WARNING("[RankConsistencyCheckerV2::RecordCannVersionV2] call strncpy_s failed, return [%d].", sRet),
64 : HCCL_SUCCESS);
65 7 : HCCL_INFO("[RankConsistencyCheckerV2::RecordCannVersionV2] version[%s] recorded.", version.c_str());
66 7 : return HCCL_SUCCESS;
67 : }
68 :
69 8 : HcclResult RankConsistencyCheckerV2::GenerateCheckFrameV2(CheckFrameV2& localFrame)
70 : {
71 8 : s32 sRet = memset_s(&localFrame, sizeof(CheckFrameV2), 0, sizeof(CheckFrameV2));
72 8 : CHK_PRT_RET(
73 : sRet != EOK, HCCL_ERROR("[RankConsistencyCheckerV2::GenerateCheckFrameV2] memset failed."), HCCL_E_INTERNAL);
74 :
75 : // 填充ranktable CRC
76 8 : localFrame.rankTableCrcNum = std::min(static_cast<u32>(rankTableCrcV2_.size()), MAX_CRC_LEN_V2);
77 14 : for (u32 i = 0; i < localFrame.rankTableCrcNum; i++) {
78 6 : localFrame.rankTableCrcArray[i] = rankTableCrcV2_[i].crc;
79 : }
80 :
81 : // 填充CANN版本
82 8 : sRet = memcpy_s(localFrame.version, CANN_VERSION_MAX_LEN + 1, cannVersion_, CANN_VERSION_MAX_LEN + 1);
83 8 : CHK_PRT_RET(
84 : sRet != EOK, HCCL_ERROR("[RankConsistencyCheckerV2::GenerateCheckFrameV2] memcpy version failed."),
85 : HCCL_E_INTERNAL);
86 :
87 8 : HCCL_INFO("[RankConsistencyCheckerV2::GenerateCheckFrameV2] success");
88 8 : return HCCL_SUCCESS;
89 : }
90 :
91 4 : HcclResult RankConsistencyCheckerV2::CompareCheckFrameV2(const CheckFrameV2& local, const CheckFrameV2& remote)
92 : {
93 : // 默认值是false表示相同，只有不同才会赋值true代表校验参数不一致，后续比对方法不会再赋值false
94 4 : bool isDiff = false;
95 : // dfx日志处理在下面函数的内部，外部不处理日志
96 : // 比对ranktable CRC
97 : isDiff
98 8 : = isDiff || CompareCrcArrayV2(local.rankTableCrcArray, remote.rankTableCrcArray, rankTableCrcV2_, "ranktable");
99 :
100 4 : CompareVersionV2(local, remote, isDiff);
101 8 : return isDiff ? HCCL_E_INTERNAL : HCCL_SUCCESS;
102 : }
103 :
104 2 : u64 RankConsistencyCheckerV2::GetCheckFrameLengthV2() const { return sizeof(CheckFrameV2); }
105 :
106 4 : void RankConsistencyCheckerV2::SetInconsistentCheckFirstDone(bool inconsistentCheckFirstDone)
107 : {
108 4 : inconsistentCheckFirstDone_ = inconsistentCheckFirstDone;
109 4 : }
110 :
111 2 : bool RankConsistencyCheckerV2::GetInconsistentCheckFirstDone() const { return inconsistentCheckFirstDone_; }
112 :
113 : // private
114 4 : bool RankConsistencyCheckerV2::CompareCrcArrayV2(
115 : const u32* localArray, const u32* remoteArray, const std::vector<CrcEntryV2>& nameSource,
116 : const std::string& categoryLabel) const
117 : {
118 4 : bool isDiff = false;
119 4 : u32 count = std::min(static_cast<u32>(nameSource.size()), MAX_CRC_LEN_V2);
120 8 : for (u32 i = 0; i < count; i++) {
121 4 : if (localArray[i] != remoteArray[i]) {
122 1 : const std::string& name = nameSource[i].name;
123 1 : HCCL_ERROR(
124 : "[RankConsistencyCheckerV2::CompareCheckFrameV2] %s mismatch: [%s], local[%u], remote[%u].",
125 : categoryLabel.c_str(), name.c_str(), localArray[i], remoteArray[i]);
126 20 : RPT_INPUT_ERR(
127 : true, "EI0005", std::vector<std::string>({"ccl_op", "group", "para_name", "local_para", "remote_para"}),
128 : std::vector<std::string>(
129 : {"N/A", "N/A", name, std::to_string(localArray[i]), std::to_string(remoteArray[i])}));
130 1 : isDiff = true;
131 : }
132 : }
133 :
134 4 : return isDiff;
135 2 : }
136 :
137 0 : std::vector<std::string_view> Split(const std::string& str, char delimiter)
138 : {
139 0 : std::vector<std::string_view> parts;
140 0 : std::string_view sv(str);
141 0 : size_t start = 0;
142 0 : for (size_t i = 0; i <= sv.size(); ++i) {
143 0 : if (i == sv.size() || sv[i] == delimiter) {
144 0 : parts.push_back(sv.substr(start, i - start));
145 0 : start = i + 1;
146 : }
147 : }
148 0 : return parts;
149 0 : }
150 :
151 0 : std::vector<std::string> GetVersionErrMessage(const std::string& localVersion, const std::string& remoteVersion)
152 : {
153 0 : std::vector<std::string> versionErrMessage;
154 0 : auto localVersionParts = Split(localVersion, '_');
155 0 : auto remoteVersionParts = Split(remoteVersion, '_');
156 0 : std::string localHcommVersion = localVersionParts.size() > 0 ? std::string(localVersionParts[0]) : "empty";
157 0 : std::string localHcclVersion = localVersionParts.size() > 1 ? std::string(localVersionParts[1]) : "empty";
158 0 : std::string remoteHcommVersion = remoteVersionParts.size() > 0 ? std::string(remoteVersionParts[0]) : "empty";
159 0 : std::string remoteHcclVersion = remoteVersionParts.size() > 1 ? std::string(remoteVersionParts[1]) : "empty";
160 :
161 0 : bool hcommDiff = (localHcommVersion != remoteHcommVersion);
162 0 : bool hcclDiff = (localHcclVersion != remoteHcclVersion);
163 0 : if (hcommDiff && !hcclDiff) {
164 : // 将版本号拆分为hcomm版本号和hccl版本号, 构造ErrMessage
165 : // The %s versions are inconsistent. The local %s, while the remote %s.
166 : versionErrMessage
167 0 : = {"Toolkit (cann-hcomm)", "version is " + localHcommVersion, "version is " + remoteHcommVersion};
168 0 : } else if (!hcommDiff && hcclDiff) {
169 0 : versionErrMessage = {"ops (cann-hccl)", "version is " + localHcclVersion, "version is " + remoteHcclVersion};
170 0 : } else if (hcommDiff && hcclDiff) {
171 : versionErrMessage
172 0 : = {"Toolkit (cann-hcomm) and ops (cann-hccl)",
173 0 : "Toolkit (cann-hcomm) version is " + localHcommVersion + " and ops (cann-hccl) version is "
174 : + localHcclVersion,
175 0 : "Toolkit (cann-hcomm) version is " + remoteHcommVersion + " and ops (cann-hccl) version is "
176 0 : + remoteHcclVersion};
177 : } else {
178 : // 拆分版本号字符串失败, 构造为原版的ErrMessage
179 : // The HCCL versions are inconsistent. The local version is %s, while the remote version is %s.
180 0 : HCCL_ERROR("[RankConsistentcyChecker][GetVersionErrMessage]Failed to split the version string.");
181 0 : versionErrMessage = {"HCCL", "version is " + localVersion, "version is " + remoteVersion};
182 : }
183 0 : return versionErrMessage;
184 0 : }
185 :
186 : HcclResult
187 8 : RankConsistencyCheckerV2::CompareVersionV2(const CheckFrameV2& local, const CheckFrameV2& remote, bool& isDiff) const
188 : {
189 16 : std::string localVer(local.version);
190 8 : std::string remoteVer(remote.version);
191 8 : if (localVer.empty() || remoteVer.empty()) {
192 1 : HCCL_WARNING(
193 : "[RankConsistencyCheckerV2::CompareVersionV2] CANN version str is empty. local[%s], remote[%s].",
194 : localVer.c_str(), remoteVer.c_str());
195 1 : return HCCL_SUCCESS;
196 7 : } else if (localVer != remoteVer) {
197 4 : auto versionErrMessage = GetVersionErrMessage(localVer, remoteVer);
198 : // GetVersionErrMessage函数内部已保证versionErrMessage.size() == 3
199 52 : RPT_INPUT_ERR(
200 : true, "EI0008", std::vector<std::string>({"inconsistent_package", "local_version", "remote_version"}),
201 : std::vector<std::string>({versionErrMessage[0], versionErrMessage[1], versionErrMessage[2]}));
202 4 : HCCL_ERROR(
203 : "[RankConsistencyCheckerV2::CompareVersionV2] CANN version mismatch: local[%s], remote[%s].",
204 : localVer.c_str(), remoteVer.c_str());
205 4 : isDiff = true;
206 4 : }
207 7 : return HCCL_SUCCESS;
208 12 : }
209 :
210 0 : HcclResult RankConsistencyCheckerV2::CalcRawDataCrc(const void* ptr, u64 length, u32& crc) const
211 : {
212 : // 计算内存数据块CRC
213 0 : Hccl::CheckCrc checkCrc;
214 0 : HcclResult ret = checkCrc.Calc32Crc(static_cast<const char*>(ptr), length, &crc);
215 0 : CHK_PRT_RET(
216 : ret != HCCL_SUCCESS,
217 : HCCL_ERROR(
218 : "[RankConsistencyCheckerV2::CalcRawDataCrc] errNo[0x%016llx] calc string crc error",
219 : HCCL_ERROR_CODE(HCCL_E_INTERNAL)),
220 : HCCL_E_INTERNAL);
221 0 : HCCL_DEBUG("[RankConsistencyCheckerV2::CalcRawDataCrc] result crc[%u].", crc);
222 0 : return HCCL_SUCCESS;
223 0 : }
224 : } // namespace hccl
|
