Line data Source code
1 : /**
2 : * Copyright (c) 2026 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 : #include "rank_consistency_checker_v2.h"
11 : #include "adapter_pub.h"
12 : #include "adapter_error_manager_pub.h"
13 : #include "checkcrc.h"
14 : using namespace Hccl;
15 :
16 : namespace hccl {
17 130 : RankConsistencyCheckerV2::~RankConsistencyCheckerV2() = default;
18 :
19 15 : RankConsistencyCheckerV2& RankConsistencyCheckerV2::GetInstance(const s32& deviceLogicId)
20 : {
21 145 : static RankConsistencyCheckerV2 instance[MAX_MODULE_DEVICE_NUM];
22 15 : HCCL_INFO("[RankConsistencyCheckerV2][GetInstance] get deviceLogicId[%d]", deviceLogicId);
23 15 : CHK_PRT_RET(
24 : (static_cast<u32>(deviceLogicId) >= MAX_MODULE_DEVICE_NUM || deviceLogicId < 0),
25 : HCCL_WARNING("[RankConsistencyCheckerV2::RankConsistencyCheckerV2]deviceLogicId[%d] is invalid", deviceLogicId),
26 : instance[0]);
27 15 : return instance[deviceLogicId];
28 : }
29 :
30 2 : HcclResult RankConsistencyCheckerV2::RecordRankTableCrcV2(u32 crc)
31 : {
32 2 : rankTableCrcV2_.clear();
33 4 : rankTableCrcV2_.emplace_back(std::string("ranktable_content"), crc);
34 2 : HCCL_INFO("[RankConsistencyCheckerV2::RecordRankTableCrcV2] ranktable crc[%u] recorded.", crc);
35 2 : return HCCL_SUCCESS;
36 : }
37 :
38 7 : HcclResult RankConsistencyCheckerV2::RecordCannVersionV2(const std::string& version)
39 : {
40 7 : u32 strLen = version.length();
41 7 : s32 sRet = memset_s(cannVersion_, CANN_VERSION_MAX_LEN + 1, 0, CANN_VERSION_MAX_LEN + 1);
42 7 : CHK_PRT_RET(
43 : sRet != EOK,
44 : HCCL_WARNING(
45 : "[RankConsistencyCheckerV2::RecordCannVersionV2]memory set 0 fail for version str "
46 : "array. return[%d].",
47 : sRet),
48 : HCCL_SUCCESS);
49 :
50 7 : CHK_PRT_RET(
51 : strLen == 0, HCCL_WARNING("[RankConsistencyCheckerV2::RecordCannVersionV2] version information str is empty."),
52 : HCCL_SUCCESS);
53 :
54 7 : CHK_PRT_RET(
55 : strLen >= CANN_VERSION_MAX_LEN + 1,
56 : HCCL_WARNING("[RankConsistencyCheckerV2::RecordCannVersionV2]"
57 : "length of version information str is too long."),
58 : HCCL_SUCCESS);
59 7 : sRet = strncpy_s(cannVersion_, CANN_VERSION_MAX_LEN + 1, version.c_str(), strLen);
60 7 : CHK_PRT_RET(
61 : sRet != EOK,
62 : HCCL_WARNING("[RankConsistencyCheckerV2::RecordCannVersionV2] call strncpy_s failed, return [%d].", sRet),
63 : HCCL_SUCCESS);
64 7 : HCCL_INFO("[RankConsistencyCheckerV2::RecordCannVersionV2] version[%s] recorded.", version.c_str());
65 7 : return HCCL_SUCCESS;
66 : }
67 :
68 8 : HcclResult RankConsistencyCheckerV2::GenerateCheckFrameV2(CheckFrameV2& localFrame)
69 : {
70 8 : s32 sRet = memset_s(&localFrame, sizeof(CheckFrameV2), 0, sizeof(CheckFrameV2));
71 8 : CHK_PRT_RET(
72 : sRet != EOK, HCCL_ERROR("[RankConsistencyCheckerV2::GenerateCheckFrameV2] memset failed."), HCCL_E_INTERNAL);
73 :
74 : // 填充ranktable CRC
75 8 : localFrame.rankTableCrcNum = std::min(static_cast<u32>(rankTableCrcV2_.size()), MAX_CRC_LEN_V2);
76 14 : for (u32 i = 0; i < localFrame.rankTableCrcNum; i++) {
77 6 : localFrame.rankTableCrcArray[i] = rankTableCrcV2_[i].crc;
78 : }
79 :
80 : // 填充CANN版本
81 8 : sRet = memcpy_s(localFrame.version, CANN_VERSION_MAX_LEN + 1, cannVersion_, CANN_VERSION_MAX_LEN + 1);
82 8 : CHK_PRT_RET(
83 : sRet != EOK, HCCL_ERROR("[RankConsistencyCheckerV2::GenerateCheckFrameV2] memcpy version failed."),
84 : HCCL_E_INTERNAL);
85 :
86 8 : HCCL_INFO("[RankConsistencyCheckerV2::GenerateCheckFrameV2] success");
87 8 : return HCCL_SUCCESS;
88 : }
89 :
90 4 : HcclResult RankConsistencyCheckerV2::CompareCheckFrameV2(const CheckFrameV2& local, const CheckFrameV2& remote)
91 : {
92 : // 默认值是false表示相同，只有不同才会赋值true代表校验参数不一致，后续比对方法不会再赋值false
93 4 : bool isDiff = false;
94 : // dfx日志处理在下面函数的内部，外部不处理日志
95 : // 比对ranktable CRC
96 : isDiff
97 8 : = isDiff || CompareCrcArrayV2(local.rankTableCrcArray, remote.rankTableCrcArray, rankTableCrcV2_, "ranktable");
98 :
99 4 : CompareVersionV2(local, remote, isDiff);
100 8 : return isDiff ? HCCL_E_INTERNAL : HCCL_SUCCESS;
101 : }
102 :
103 2 : u64 RankConsistencyCheckerV2::GetCheckFrameLengthV2() const { return sizeof(CheckFrameV2); }
104 :
105 4 : void RankConsistencyCheckerV2::SetInconsistentCheckFirstDone(bool inconsistentCheckFirstDone)
106 : {
107 4 : inconsistentCheckFirstDone_ = inconsistentCheckFirstDone;
108 4 : }
109 :
110 2 : bool RankConsistencyCheckerV2::GetInconsistentCheckFirstDone() const { return inconsistentCheckFirstDone_; }
111 :
112 : // private
113 4 : bool RankConsistencyCheckerV2::CompareCrcArrayV2(
114 : const u32* localArray, const u32* remoteArray, const std::vector<CrcEntryV2>& nameSource,
115 : const std::string& categoryLabel) const
116 : {
117 4 : bool isDiff = false;
118 4 : u32 count = std::min(static_cast<u32>(nameSource.size()), MAX_CRC_LEN_V2);
119 8 : for (u32 i = 0; i < count; i++) {
120 4 : if (localArray[i] != remoteArray[i]) {
121 1 : const std::string& name = nameSource[i].name;
122 1 : HCCL_ERROR(
123 : "[RankConsistencyCheckerV2::CompareCheckFrameV2] %s mismatch: [%s], local[%u], remote[%u].",
124 : categoryLabel.c_str(), name.c_str(), localArray[i], remoteArray[i]);
125 20 : RPT_INPUT_ERR(
126 : true, "EI0005", std::vector<std::string>({"ccl_op", "group", "para_name", "local_para", "remote_para"}),
127 : std::vector<std::string>(
128 : {"N/A", "N/A", name, std::to_string(localArray[i]), std::to_string(remoteArray[i])}));
129 1 : isDiff = true;
130 : }
131 : }
132 :
133 4 : return isDiff;
134 2 : }
135 :
136 0 : std::vector<std::string_view> Split(const std::string& str, char delimiter)
137 : {
138 0 : std::vector<std::string_view> parts;
139 0 : std::string_view sv(str);
140 0 : size_t start = 0;
141 0 : for (size_t i = 0; i <= sv.size(); ++i) {
142 0 : if (i == sv.size() || sv[i] == delimiter) {
143 0 : parts.push_back(sv.substr(start, i - start));
144 0 : start = i + 1;
145 : }
146 : }
147 0 : return parts;
148 0 : }
149 :
150 0 : std::vector<std::string> GetVersionErrMessage(const std::string& localVersion, const std::string& remoteVersion)
151 : {
152 0 : std::vector<std::string> versionErrMessage;
153 0 : auto localVersionParts = Split(localVersion, '_');
154 0 : auto remoteVersionParts = Split(remoteVersion, '_');
155 0 : std::string localHcommVersion = localVersionParts.size() > 0 ? std::string(localVersionParts[0]) : "empty";
156 0 : std::string localHcclVersion = localVersionParts.size() > 1 ? std::string(localVersionParts[1]) : "empty";
157 0 : std::string remoteHcommVersion = remoteVersionParts.size() > 0 ? std::string(remoteVersionParts[0]) : "empty";
158 0 : std::string remoteHcclVersion = remoteVersionParts.size() > 1 ? std::string(remoteVersionParts[1]) : "empty";
159 :
160 0 : bool hcommDiff = (localHcommVersion != remoteHcommVersion);
161 0 : bool hcclDiff = (localHcclVersion != remoteHcclVersion);
162 0 : if (hcommDiff && !hcclDiff) {
163 : // 将版本号拆分为hcomm版本号和hccl版本号, 构造ErrMessage
164 : // The %s versions are inconsistent. The local %s, while the remote %s.
165 : versionErrMessage
166 0 : = {"Toolkit (cann-hcomm)", "version is " + localHcommVersion, "version is " + remoteHcommVersion};
167 0 : } else if (!hcommDiff && hcclDiff) {
168 0 : versionErrMessage = {"ops (cann-hccl)", "version is " + localHcclVersion, "version is " + remoteHcclVersion};
169 0 : } else if (hcommDiff && hcclDiff) {
170 : versionErrMessage
171 0 : = {"Toolkit (cann-hcomm) and ops (cann-hccl)",
172 0 : "Toolkit (cann-hcomm) version is " + localHcommVersion + " and ops (cann-hccl) version is "
173 : + localHcclVersion,
174 0 : "Toolkit (cann-hcomm) version is " + remoteHcommVersion + " and ops (cann-hccl) version is "
175 0 : + remoteHcclVersion};
176 : } else {
177 : // 拆分版本号字符串失败, 构造为原版的ErrMessage
178 : // The HCCL versions are inconsistent. The local version is %s, while the remote version is %s.
179 0 : HCCL_ERROR("[RankConsistentcyChecker][GetVersionErrMessage]Failed to split the version string.");
180 0 : versionErrMessage = {"HCCL", "version is " + localVersion, "version is " + remoteVersion};
181 : }
182 0 : return versionErrMessage;
183 0 : }
184 :
185 : HcclResult
186 8 : RankConsistencyCheckerV2::CompareVersionV2(const CheckFrameV2& local, const CheckFrameV2& remote, bool& isDiff) const
187 : {
188 16 : std::string localVer(local.version);
189 8 : std::string remoteVer(remote.version);
190 8 : if (localVer.empty() || remoteVer.empty()) {
191 1 : HCCL_WARNING(
192 : "[RankConsistencyCheckerV2::CompareVersionV2] CANN version str is empty. local[%s], remote[%s].",
193 : localVer.c_str(), remoteVer.c_str());
194 1 : return HCCL_SUCCESS;
195 7 : } else if (localVer != remoteVer) {
196 4 : auto versionErrMessage = GetVersionErrMessage(localVer, remoteVer);
197 : // GetVersionErrMessage函数内部已保证versionErrMessage.size() == 3
198 52 : RPT_INPUT_ERR(
199 : true, "EI0008", std::vector<std::string>({"inconsistent_package", "local_version", "remote_version"}),
200 : std::vector<std::string>({versionErrMessage[0], versionErrMessage[1], versionErrMessage[2]}));
201 4 : HCCL_ERROR(
202 : "[RankConsistencyCheckerV2::CompareVersionV2] CANN version mismatch: local[%s], remote[%s].",
203 : localVer.c_str(), remoteVer.c_str());
204 4 : isDiff = true;
205 4 : }
206 7 : return HCCL_SUCCESS;
207 12 : }
208 :
209 0 : HcclResult RankConsistencyCheckerV2::CalcRawDataCrc(const void* ptr, u64 length, u32& crc) const
210 : {
211 : // 计算内存数据块CRC
212 0 : Hccl::CheckCrc checkCrc;
213 0 : HcclResult ret = checkCrc.Calc32Crc(static_cast<const char*>(ptr), length, &crc);
214 0 : CHK_PRT_RET(
215 : ret != HCCL_SUCCESS,
216 : HCCL_ERROR(
217 : "[RankConsistencyCheckerV2::CalcRawDataCrc] errNo[0x%016llx] calc string crc error",
218 : HCCL_ERROR_CODE(HCCL_E_INTERNAL)),
219 : HCCL_E_INTERNAL);
220 0 : HCCL_DEBUG("[RankConsistencyCheckerV2::CalcRawDataCrc] result crc[%u].", crc);
221 0 : return HCCL_SUCCESS;
222 0 : }
223 : } // namespace hccl
|
