Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #ifndef RANK_CONSISTENTCY_CHECKER_H
12 : #define RANK_CONSISTENTCY_CHECKER_H
13 :
14 : #include <map>
15 : #include <cstdint>
16 : #include <string>
17 : #include <vector>
18 : #include <memory>
19 : #include <mutex>
20 : #include <hccl/hccl_types.h>
21 :
22 : #include "hccl_common.h"
23 : #include "common.h"
24 : #include "externalinput_pub.h"
25 :
26 : namespace hccl {
27 : constexpr u32 DEFAULT_CRC = 0xFFFFFFFF; // CRC默认值
28 : constexpr u32 MAX_CANN_VERSION_LEN = 50; // CANN版本校验
29 : constexpr u32 MAX_CRC_LEN = 128; // 最大CRC个数128（CRC最大直径长度：128*sizeof（u32））
30 :
31 : using HcclCMDInfo = struct TagHcclCMDInfo {
32 : HcclCMDType cmdType{HcclCMDType::HCCL_CMD_INVALID};
33 : char tag[TAG_MAX_LEN + 1] = {0};
34 : u64 count{0};
35 : HcclDataType dataType{HCCL_DATA_TYPE_RESERVED};
36 : HcclReduceOp op{HCCL_REDUCE_RESERVED};
37 : char group[GROUP_NAME_MAX_LEN + 1] = {0};
38 : u32 root{0};
39 : u32 rank{0};
40 : u32 srTag{0};
41 : u32 selfRank{0};
42 : u64 inCclBufferSize{0};
43 : u64 outCclBufferSize{0};
44 : u32 aivCoreLimit{0};
45 : u8 deterministic{0};
46 : };
47 :
48 : using HcclCRCInfo = struct TagHcclCRCInfo {
49 : u32 configFileExist_ = 0;
50 : u32 crcNum = 0;
51 : u32 crcArray[MAX_CRC_LEN] = {0};
52 : };
53 :
54 : using HcclCheckInfo = struct TagHcclCheckInfo {
55 : HcclCRCInfo crcInfoGlobal;
56 : HcclCRCInfo crcInfoOp;
57 : HcclCMDInfo cmdInfo;
58 : ProtocolType protocolType = ProtocolType::RESERVED;
59 : char version[MAX_CANN_VERSION_LEN + 1] = {0};
60 : };
61 :
62 : enum class HcclCrcRecordType {
63 : HCCL_CRC_RECORD_RANKTABLE = 0,
64 : HCCL_CRC_RECORD_VARING_COUNTS = 1,
65 : HCCL_CRC_RECORD_VARING_DISPLACEMENTS = 2,
66 : };
67 :
68 : const std::map<HcclCrcRecordType, std::string> HCCL_CRC_RECORD_TYPE_STR_MAP{
69 : {HcclCrcRecordType::HCCL_CRC_RECORD_RANKTABLE, "ranktable"},
70 : {HcclCrcRecordType::HCCL_CRC_RECORD_VARING_COUNTS, "varing_counts"},
71 : {HcclCrcRecordType::HCCL_CRC_RECORD_VARING_DISPLACEMENTS, "varing_displacements"},
72 : };
73 :
74 2 : inline std::string GetCRCTypeEnumStr(HcclCrcRecordType crcType)
75 : {
76 2 : const auto iter = HCCL_CRC_RECORD_TYPE_STR_MAP.find(crcType);
77 2 : if (iter == HCCL_CRC_RECORD_TYPE_STR_MAP.end()) {
78 0 : return "Invalid HcclCrcRecordType";
79 : } else {
80 2 : return iter->second;
81 : }
82 : }
83 :
84 : class RankConsistentcyChecker {
85 : public:
86 : ~RankConsistentcyChecker();
87 :
88 : static RankConsistentcyChecker& GetInstance(s32 deviceLogicId = 0xFF);
89 :
90 : // gather
91 : HcclResult RecordOpPara(HcclCMDType opCMD, const std::string &tag, u64 count, HcclDataType dataType, u32 root,
92 : u64 inCclBufferSize, u64 outCclBufferSize, const char *group = nullptr, u32 crc = DEFAULT_CRC,
93 : u32 aivCoreLimit = 0);
94 : // all reduce && all gather && broadcast && scatter && reduce && alltoall alltoallv alltoallvc
95 : HcclResult RecordOpPara(HcclCMDType opCMD, const std::string &tag, u64 count, HcclDataType dataType,
96 : HcclReduceOp op, u32 root, u64 inCclBufferSize, u64 outCclBufferSize, const char *group = nullptr,
97 : u32 crc = DEFAULT_CRC, u8 deterministic = 0, u32 aivCoreLimit = 0);
98 : // send && receive
99 : HcclResult RecordOpPara(HcclCMDType opCMD, const std::string &tag, u64 count, HcclDataType dataType, u32 rank,
100 : u32 srTag, u32 selfRank, u64 inCclBufferSize, u64 outCclBufferSize, const char *group,
101 : u32 crc = DEFAULT_CRC);
102 : // batchsendrecv
103 : HcclResult RecordOpPara(HcclCMDType opCMD, const std::string &tag, u64 inCclBufferSize, u64 outCclBufferSize,
104 : const char *group = nullptr, u32 crc = DEFAULT_CRC);
105 : // reduce scatter v && AllGather v
106 : HcclResult RecordOpPara(HcclCMDType opCMD, const std::string &tag,
107 : const void* counts, const void *displs, const u32 rankSize, HcclDataType dataType, HcclReduceOp op,
108 : u64 inCclBufferSize, u64 outCclBufferSize, const char *group = nullptr, u32 crc = DEFAULT_CRC, u8 deterministic = 0,
109 : u32 aivCoreLimit = 0);
110 :
111 : HcclResult DelOpPara(const std::string &tag);
112 :
113 : HcclResult RecordVerInfo(const std::string &versionInfo);
114 :
115 : u64 GetRankConsistentDataLength();
116 :
117 : void RecordProtocolType(ProtocolType protocolType);
118 :
119 : HcclResult GetCheckFrame(u8 *destBuf, u64 maxDestBuf, const std::string &tag);
120 :
121 : HcclResult CheckFrameRecv(const u8 *recvBuf, u32 recvBufLen, const std::string &tag);
122 :
123 : void ClearCheckInfo();
124 :
125 : HcclResult CalcStringCrc(const char *str, u32 &crc);
126 :
127 : void SetCheckCannVersionSwitch(const bool cannVerCheckSwitch);
128 :
129 : private:
130 : explicit RankConsistentcyChecker();
131 : // all of that
132 : HcclResult RecordOpPara(HcclCMDType opCMD, const std::string &tag, u64 count, HcclDataType dataType,
133 : HcclReduceOp op, u32 root, u32 rank, u32 srTag, u32 selfRank, u64 inCclBufferSize, u64 outCclBufferSize,
134 : const char *group, u32 crc, u8 deterministic = 0, u32 aivCoreLimit = 0);
135 : // for reduce_scatter_v and all_gatherv
136 : HcclResult RecordVaringOpPara(const std::string &tag, const void *counts, const void *displs, const u32 rankSize);
137 : // get CMDinfo by tag
138 : HcclResult GetOpParaByTag(const std::string &tag, HcclCMDInfo &CMDInfoOutput);
139 : HcclResult GetCrcByTag(const std::string &tag, HcclCRCInfo &crcInfo);
140 : HcclResult GenerateCheckFrame(HcclCheckInfo &checkInfo, const std::string &tag);
141 : void CompareVersion(const HcclCheckInfo &local, const HcclCheckInfo &remote, bool &isDiff);
142 : bool CompareFrame(HcclCheckInfo &checkInfo, HcclCheckInfo &checkInfoRecv);
143 : bool CompareCrcInfo(const HcclCMDInfo &hcclCMDInfo, HcclCRCInfo &crcInfo, HcclCRCInfo &crcInfoRecv);
144 : void ReportCmdInfoCheckFailed(const HcclCMDInfo &hcclCMDInfo, const std::string ¶Name,
145 : const std::string &localPara, const std::string &remotePara) const;
146 : void ReportCmdInfoCheckFailed(const HcclCMDInfo &hcclCMDInfo, const std::string ¶Name,
147 : uint32_t localPara, uint32_t remotePara);
148 : void ReportCrcCheckFailed(const HcclCMDInfo &hcclCMDInfo, HcclCrcRecordType crcType, const uint32_t localCrc,
149 : const uint32_t remoteCrc) const; // 打印CRC校验失败信息
150 : void ReportCommonError(const HcclCMDInfo &hcclCMDInfo, const std::string ¶Name,
151 : const std::string &localParaStr, const std::string &remoteParaStr, const std::string &errorMsg) const;
152 : void CompareCmdInfo(HcclCheckInfo &checkInfo, HcclCheckInfo &checkInfoRecv);
153 : bool CompareSection(const char *pRawData, const char *recvBuf, u32 len);
154 : HcclResult AddCrc(const u32 crcValue);
155 : HcclResult ClearCrcInfo(void);
156 : HcclResult GetCrc(u32 num, u32 *crcAddr);
157 : HcclResult CalcRawDataCrc(const void *ptr, u64 length, u32 &crc);
158 :
159 : // 要校验的内容
160 : std::unordered_map<std::string, HcclCMDInfo> cmdInfoMap_;
161 : std::unordered_map<std::string, std::map<HcclCrcRecordType, u32>> crcRecords_; // CRC校验码记录
162 : // cann 版本号
163 : char cannVersion_[MAX_CANN_VERSION_LEN + 1];
164 : // cann 版本校验开关
165 : bool cannVerCheckSwitch_;
166 : // 算法标志是否已经更新标志，防止发送空数据
167 : bool cannVerInfoRecordFlag_;
168 : // CMD是否已经更新标志，防止发送空数据，要校验内容的信息的更新情况（false：未更新）
169 : std::unordered_map<std::string, bool> infoFlagCmdMap_;
170 : // config文件是否存在，1表示存在，0表示不存在
171 : bool configFileExist_;
172 : ProtocolType protocolType_ = ProtocolType::RESERVED;
173 : std::vector<u32> crcTable_;
174 : std::mutex mutex_;
175 : bool inconsistentCheckFirstDone_ = false; // first模式下是否已完成首次校验
176 : };
177 : }
178 : #endif // RANK_CONSISTENTCY_CHECKER_H
|
