Line data Source code
1 : /**
2 : * Copyright (c) 2026 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 : #include "rank_consistency_checker_v2.h"
11 : #include "adapter_pub.h"
12 : #include "adapter_error_manager_pub.h"
13 : #include "checkcrc.h"
14 : using namespace Hccl;
15 :
16 : namespace hccl {
17 130 : RankConsistencyCheckerV2::~RankConsistencyCheckerV2() = default;
18 :
19 15 : RankConsistencyCheckerV2& RankConsistencyCheckerV2::GetInstance(const s32 &deviceLogicId)
20 : {
21 145 : static RankConsistencyCheckerV2 instance[MAX_MODULE_DEVICE_NUM];
22 15 : HCCL_INFO("[RankConsistencyCheckerV2][GetInstance] get deviceLogicId[%d]", deviceLogicId);
23 15 : CHK_PRT_RET((static_cast<u32>(deviceLogicId) >= MAX_MODULE_DEVICE_NUM || deviceLogicId < 0),
24 : HCCL_WARNING("[RankConsistencyCheckerV2::RankConsistencyCheckerV2]deviceLogicId[%d] is invalid", deviceLogicId), instance[0]);
25 15 : return instance[deviceLogicId];
26 : }
27 :
28 2 : HcclResult RankConsistencyCheckerV2::RecordRankTableCrcV2(u32 crc)
29 : {
30 2 : rankTableCrcV2_.clear();
31 4 : rankTableCrcV2_.emplace_back(std::string("ranktable_content"), crc);
32 2 : HCCL_INFO("[RankConsistencyCheckerV2::RecordRankTableCrcV2] ranktable crc[%u] recorded.", crc);
33 2 : return HCCL_SUCCESS;
34 : }
35 :
36 7 : HcclResult RankConsistencyCheckerV2::RecordCannVersionV2(const std::string &version)
37 : {
38 7 : u32 strLen = version.length();
39 7 : s32 sRet = memset_s(cannVersion_, CANN_VERSION_MAX_LEN + 1, 0, CANN_VERSION_MAX_LEN + 1);
40 7 : CHK_PRT_RET(sRet != EOK, HCCL_WARNING("[RankConsistencyCheckerV2::RecordCannVersionV2]memory set 0 fail for version str "
41 : "array. return[%d].", sRet), HCCL_SUCCESS);
42 :
43 7 : CHK_PRT_RET(strLen == 0, HCCL_WARNING("[RankConsistencyCheckerV2::RecordCannVersionV2] version information str is empty."),
44 : HCCL_SUCCESS);
45 :
46 7 : CHK_PRT_RET(strLen >= CANN_VERSION_MAX_LEN + 1, HCCL_WARNING("[RankConsistencyCheckerV2::RecordCannVersionV2]"
47 : "length of version information str is too long."), HCCL_SUCCESS);
48 7 : sRet = strncpy_s(cannVersion_, CANN_VERSION_MAX_LEN + 1, version.c_str(), strLen);
49 7 : CHK_PRT_RET(sRet != EOK, HCCL_WARNING("[RankConsistencyCheckerV2::RecordCannVersionV2] call strncpy_s failed, return [%d].", sRet),
50 : HCCL_SUCCESS);
51 7 : HCCL_INFO("[RankConsistencyCheckerV2::RecordCannVersionV2] version[%s] recorded.", version.c_str());
52 7 : return HCCL_SUCCESS;
53 : }
54 :
55 8 : HcclResult RankConsistencyCheckerV2::GenerateCheckFrameV2(CheckFrameV2 &localFrame)
56 : {
57 8 : s32 sRet = memset_s(&localFrame, sizeof(CheckFrameV2), 0, sizeof(CheckFrameV2));
58 8 : CHK_PRT_RET(sRet != EOK,
59 : HCCL_ERROR("[RankConsistencyCheckerV2::GenerateCheckFrameV2] memset failed."), HCCL_E_INTERNAL);
60 :
61 : // 填充ranktable CRC
62 8 : localFrame.rankTableCrcNum = std::min(static_cast<u32>(rankTableCrcV2_.size()), MAX_CRC_LEN_V2);
63 14 : for (u32 i = 0; i < localFrame.rankTableCrcNum; i++) {
64 6 : localFrame.rankTableCrcArray[i] = rankTableCrcV2_[i].crc;
65 : }
66 :
67 : // 填充CANN版本
68 16 : sRet = memcpy_s(localFrame.version, CANN_VERSION_MAX_LEN + 1,
69 8 : cannVersion_, CANN_VERSION_MAX_LEN + 1);
70 8 : CHK_PRT_RET(sRet != EOK,
71 : HCCL_ERROR("[RankConsistencyCheckerV2::GenerateCheckFrameV2] memcpy version failed."), HCCL_E_INTERNAL);
72 :
73 8 : HCCL_INFO("[RankConsistencyCheckerV2::GenerateCheckFrameV2] success");
74 8 : return HCCL_SUCCESS;
75 : }
76 :
77 4 : HcclResult RankConsistencyCheckerV2::CompareCheckFrameV2(const CheckFrameV2 &local,
78 : const CheckFrameV2 &remote)
79 : {
80 : // 默认值是false表示相同，只有不同才会赋值true代表校验参数不一致，后续比对方法不会再赋值false
81 4 : bool isDiff = false;
82 : // dfx日志处理在下面函数的内部，外部不处理日志
83 : // 比对ranktable CRC
84 12 : isDiff = isDiff || CompareCrcArrayV2(
85 4 : local.rankTableCrcArray, remote.rankTableCrcArray,
86 4 : rankTableCrcV2_, "ranktable");
87 :
88 4 : CompareVersionV2(local, remote, isDiff);
89 8 : return isDiff ? HCCL_E_INTERNAL : HCCL_SUCCESS;
90 : }
91 :
92 2 : u64 RankConsistencyCheckerV2::GetCheckFrameLengthV2() const
93 : {
94 2 : return sizeof(CheckFrameV2);
95 : }
96 :
97 4 : void RankConsistencyCheckerV2::SetInconsistentCheckFirstDone(bool inconsistentCheckFirstDone)
98 : {
99 4 : inconsistentCheckFirstDone_ = inconsistentCheckFirstDone;
100 4 : }
101 :
102 2 : bool RankConsistencyCheckerV2::GetInconsistentCheckFirstDone() const
103 : {
104 2 : return inconsistentCheckFirstDone_;
105 : }
106 :
107 : // private
108 4 : bool RankConsistencyCheckerV2::CompareCrcArrayV2(const u32 *localArray,
109 : const u32 *remoteArray,
110 : const std::vector<CrcEntryV2> &nameSource,
111 : const std::string &categoryLabel) const
112 : {
113 4 : bool isDiff = false;
114 4 : u32 count = std::min(static_cast<u32>(nameSource.size()), MAX_CRC_LEN_V2);
115 8 : for (u32 i = 0; i < count; i++) {
116 4 : if (localArray[i] != remoteArray[i]) {
117 1 : const std::string &name = nameSource[i].name;
118 1 : HCCL_ERROR("[RankConsistencyCheckerV2::CompareCheckFrameV2] %s mismatch: [%s], local[%u], remote[%u].",
119 : categoryLabel.c_str(), name.c_str(), localArray[i], remoteArray[i]);
120 20 : RPT_INPUT_ERR(true, "EI0005",
121 : std::vector<std::string>({"ccl_op", "group", "para_name", "local_para", "remote_para"}),
122 : std::vector<std::string>({"N/A", "N/A", name, std::to_string(localArray[i]), std::to_string(remoteArray[i])}));
123 1 : isDiff = true;
124 : }
125 : }
126 :
127 4 : return isDiff;
128 2 : }
129 :
130 0 : std::vector<std::string_view> Split(const std::string &str, char delimiter)
131 : {
132 0 : std::vector<std::string_view> parts;
133 0 : std::string_view sv(str);
134 0 : size_t start = 0;
135 0 : for (size_t i = 0; i <= sv.size(); ++i) {
136 0 : if (i == sv.size() || sv[i] == delimiter) {
137 0 : parts.push_back(sv.substr(start, i - start));
138 0 : start = i + 1;
139 : }
140 : }
141 0 : return parts;
142 0 : }
143 :
144 0 : std::vector<std::string> GetVersionErrMessage(const std::string &localVersion, const std::string &remoteVersion)
145 : {
146 0 : std::vector<std::string> versionErrMessage;
147 0 : auto localVersionParts = Split(localVersion, '_');
148 0 : auto remoteVersionParts = Split(remoteVersion, '_');
149 0 : std::string localHcommVersion = localVersionParts.size() > 0 ? std::string(localVersionParts[0]) : "empty";
150 0 : std::string localHcclVersion = localVersionParts.size() > 1 ? std::string(localVersionParts[1]) : "empty";
151 0 : std::string remoteHcommVersion = remoteVersionParts.size() > 0 ? std::string(remoteVersionParts[0]) : "empty";
152 0 : std::string remoteHcclVersion = remoteVersionParts.size() > 1 ? std::string(remoteVersionParts[1]) : "empty";
153 :
154 0 : bool hcommDiff = (localHcommVersion != remoteHcommVersion);
155 0 : bool hcclDiff = (localHcclVersion != remoteHcclVersion);
156 0 : if (hcommDiff && !hcclDiff) {
157 : // 将版本号拆分为hcomm版本号和hccl版本号, 构造ErrMessage
158 : // The %s versions are inconsistent. The local %s, while the remote %s.
159 0 : versionErrMessage =
160 0 : {"Toolkit (cann-hcomm)", "version is " + localHcommVersion, "version is " + remoteHcommVersion};
161 0 : } else if (!hcommDiff && hcclDiff) {
162 0 : versionErrMessage = {"ops (cann-hccl)", "version is " + localHcclVersion, "version is " + remoteHcclVersion};
163 0 : } else if (hcommDiff && hcclDiff) {
164 0 : versionErrMessage = {
165 : "Toolkit (cann-hcomm) and ops (cann-hccl)",
166 0 : "Toolkit (cann-hcomm) version is " + localHcommVersion +
167 0 : " and ops (cann-hccl) version is " + localHcclVersion,
168 0 : "Toolkit (cann-hcomm) version is " + remoteHcommVersion +
169 0 : " and ops (cann-hccl) version is " + remoteHcclVersion
170 0 : };
171 : } else {
172 : // 拆分版本号字符串失败, 构造为原版的ErrMessage
173 : // The HCCL versions are inconsistent. The local version is %s, while the remote version is %s.
174 0 : HCCL_ERROR("[RankConsistentcyChecker][GetVersionErrMessage]Failed to split the version string.");
175 0 : versionErrMessage = {"HCCL", "version is " + localVersion, "version is " + remoteVersion};
176 : }
177 0 : return versionErrMessage;
178 0 : }
179 :
180 8 : HcclResult RankConsistencyCheckerV2::CompareVersionV2(const CheckFrameV2 &local, const CheckFrameV2 &remote, bool &isDiff) const
181 : {
182 16 : std::string localVer(local.version);
183 8 : std::string remoteVer(remote.version);
184 8 : if (localVer.empty() || remoteVer.empty()) {
185 1 : HCCL_WARNING("[RankConsistencyCheckerV2::CompareVersionV2] CANN version str is empty. local[%s], remote[%s].",
186 : localVer.c_str(), remoteVer.c_str());
187 1 : return HCCL_SUCCESS;
188 7 : } else if (localVer != remoteVer) {
189 4 : auto versionErrMessage = GetVersionErrMessage(localVer, remoteVer);
190 : // GetVersionErrMessage函数内部已保证versionErrMessage.size() == 3
191 52 : RPT_INPUT_ERR(true, "EI0008",
192 : std::vector<std::string>({"inconsistent_package", "local_version", "remote_version"}),
193 : std::vector<std::string>({versionErrMessage[0], versionErrMessage[1], versionErrMessage[2]}));
194 4 : HCCL_ERROR("[RankConsistencyCheckerV2::CompareVersionV2] CANN version mismatch: local[%s], remote[%s].",
195 : localVer.c_str(), remoteVer.c_str());
196 4 : isDiff = true;
197 4 : }
198 7 : return HCCL_SUCCESS;
199 12 : }
200 :
201 0 : HcclResult RankConsistencyCheckerV2::CalcRawDataCrc(const void *ptr, u64 length, u32 &crc) const
202 : {
203 : // 计算内存数据块CRC
204 0 : Hccl::CheckCrc checkCrc;
205 0 : HcclResult ret = checkCrc.Calc32Crc(static_cast<const char*>(ptr), length, &crc);
206 0 : CHK_PRT_RET(ret != HCCL_SUCCESS,
207 : HCCL_ERROR("[RankConsistencyCheckerV2::CalcRawDataCrc] errNo[0x%016llx] calc string crc error",
208 : HCCL_ERROR_CODE(HCCL_E_INTERNAL)), HCCL_E_INTERNAL);
209 0 : HCCL_DEBUG("[RankConsistencyCheckerV2::CalcRawDataCrc] result crc[%u].", crc);
210 0 : return HCCL_SUCCESS;
211 0 : }
212 : }
213 :
|
