Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #ifndef RANK_CONSISTENTCY_CHECKER_H
12 : #define RANK_CONSISTENTCY_CHECKER_H
13 :
14 : #include <map>
15 : #include <cstdint>
16 : #include <string>
17 : #include <vector>
18 : #include <memory>
19 : #include <mutex>
20 : #include <hccl/hccl_types.h>
21 :
22 : #include "hccl_common.h"
23 : #include "common.h"
24 : #include "externalinput_pub.h"
25 :
26 : namespace hccl {
27 : constexpr u32 DEFAULT_CRC = 0xFFFFFFFF; // CRC默认值
28 : constexpr u32 MAX_CANN_VERSION_LEN = 50; // CANN版本校验
29 : constexpr u32 MAX_CRC_LEN = 128; // 最大CRC个数128（CRC最大直径长度：128*sizeof（u32））
30 :
31 : using HcclCMDInfo = struct TagHcclCMDInfo {
32 : HcclCMDType cmdType{HcclCMDType::HCCL_CMD_INVALID};
33 : char tag[TAG_MAX_LEN + 1] = {0};
34 : u64 count{0};
35 : HcclDataType dataType{HCCL_DATA_TYPE_RESERVED};
36 : HcclReduceOp op{HCCL_REDUCE_RESERVED};
37 : char group[GROUP_NAME_MAX_LEN + 1] = {0};
38 : u32 root{0};
39 : u32 rank{0};
40 : u32 srTag{0};
41 : u32 selfRank{0};
42 : u64 inCclBufferSize{0};
43 : u64 outCclBufferSize{0};
44 : u32 aivCoreLimit{0};
45 : u8 deterministic{0};
46 : };
47 :
48 : using HcclCRCInfo = struct TagHcclCRCInfo {
49 : u32 configFileExist_ = 0;
50 : u32 crcNum = 0;
51 : u32 crcArray[MAX_CRC_LEN] = {0};
52 : };
53 :
54 : using HcclCheckInfo = struct TagHcclCheckInfo {
55 : HcclCRCInfo crcInfoGlobal;
56 : HcclCRCInfo crcInfoOp;
57 : HcclCMDInfo cmdInfo;
58 : ProtocolType protocolType = ProtocolType::RESERVED;
59 : char version[MAX_CANN_VERSION_LEN + 1] = {0};
60 : };
61 :
62 : enum class HcclCrcRecordType {
63 : HCCL_CRC_RECORD_RANKTABLE = 0,
64 : HCCL_CRC_RECORD_VARING_COUNTS = 1,
65 : HCCL_CRC_RECORD_VARING_DISPLACEMENTS = 2,
66 : };
67 :
68 : const std::map<HcclCrcRecordType, std::string> HCCL_CRC_RECORD_TYPE_STR_MAP{
69 : {HcclCrcRecordType::HCCL_CRC_RECORD_RANKTABLE, "ranktable"},
70 : {HcclCrcRecordType::HCCL_CRC_RECORD_VARING_COUNTS, "varing_counts"},
71 : {HcclCrcRecordType::HCCL_CRC_RECORD_VARING_DISPLACEMENTS, "varing_displacements"},
72 : };
73 :
74 2 : inline std::string GetCRCTypeEnumStr(HcclCrcRecordType crcType)
75 : {
76 2 : const auto iter = HCCL_CRC_RECORD_TYPE_STR_MAP.find(crcType);
77 2 : if (iter == HCCL_CRC_RECORD_TYPE_STR_MAP.end()) {
78 0 : return "Invalid HcclCrcRecordType";
79 : } else {
80 2 : return iter->second;
81 : }
82 : }
83 :
84 : class RankConsistentcyChecker {
85 : public:
86 : ~RankConsistentcyChecker();
87 :
88 : static RankConsistentcyChecker& GetInstance(s32 deviceLogicId = 0xFF);
89 :
90 : // gather
91 : HcclResult RecordOpPara(
92 : HcclCMDType opCMD, const std::string& tag, u64 count, HcclDataType dataType, u32 root, u64 inCclBufferSize,
93 : u64 outCclBufferSize, const char* group = nullptr, u32 crc = DEFAULT_CRC, u32 aivCoreLimit = 0);
94 : // all reduce && all gather && broadcast && scatter && reduce && alltoall alltoallv alltoallvc
95 : HcclResult RecordOpPara(
96 : HcclCMDType opCMD, const std::string& tag, u64 count, HcclDataType dataType, HcclReduceOp op, u32 root,
97 : u64 inCclBufferSize, u64 outCclBufferSize, const char* group = nullptr, u32 crc = DEFAULT_CRC,
98 : u8 deterministic = 0, u32 aivCoreLimit = 0);
99 : // send && receive
100 : HcclResult RecordOpPara(
101 : HcclCMDType opCMD, const std::string& tag, u64 count, HcclDataType dataType, u32 rank, u32 srTag, u32 selfRank,
102 : u64 inCclBufferSize, u64 outCclBufferSize, const char* group, u32 crc = DEFAULT_CRC);
103 : // batchsendrecv
104 : HcclResult RecordOpPara(
105 : HcclCMDType opCMD, const std::string& tag, u64 inCclBufferSize, u64 outCclBufferSize,
106 : const char* group = nullptr, u32 crc = DEFAULT_CRC);
107 : // reduce scatter v && AllGather v
108 : HcclResult RecordOpPara(
109 : HcclCMDType opCMD, const std::string& tag, const void* counts, const void* displs, const u32 rankSize,
110 : HcclDataType dataType, HcclReduceOp op, u64 inCclBufferSize, u64 outCclBufferSize, const char* group = nullptr,
111 : u32 crc = DEFAULT_CRC, u8 deterministic = 0, u32 aivCoreLimit = 0);
112 :
113 : HcclResult DelOpPara(const std::string& tag);
114 :
115 : HcclResult RecordVerInfo(const std::string& versionInfo);
116 :
117 : u64 GetRankConsistentDataLength();
118 :
119 : void RecordProtocolType(ProtocolType protocolType);
120 :
121 : HcclResult GetCheckFrame(u8* destBuf, u64 maxDestBuf, const std::string& tag);
122 :
123 : HcclResult CheckFrameRecv(const u8* recvBuf, u32 recvBufLen, const std::string& tag);
124 :
125 : void ClearCheckInfo();
126 :
127 : HcclResult CalcStringCrc(const char* str, u32& crc);
128 :
129 : void SetCheckCannVersionSwitch(const bool cannVerCheckSwitch);
130 :
131 : private:
132 : explicit RankConsistentcyChecker();
133 : // all of that
134 : HcclResult RecordOpPara(
135 : HcclCMDType opCMD, const std::string& tag, u64 count, HcclDataType dataType, HcclReduceOp op, u32 root,
136 : u32 rank, u32 srTag, u32 selfRank, u64 inCclBufferSize, u64 outCclBufferSize, const char* group, u32 crc,
137 : u8 deterministic = 0, u32 aivCoreLimit = 0);
138 : // for reduce_scatter_v and all_gatherv
139 : HcclResult RecordVaringOpPara(const std::string& tag, const void* counts, const void* displs, const u32 rankSize);
140 : // get CMDinfo by tag
141 : HcclResult GetOpParaByTag(const std::string& tag, HcclCMDInfo& CMDInfoOutput);
142 : HcclResult GetCrcByTag(const std::string& tag, HcclCRCInfo& crcInfo);
143 : HcclResult GenerateCheckFrame(HcclCheckInfo& checkInfo, const std::string& tag);
144 : void CompareVersion(const HcclCheckInfo& local, const HcclCheckInfo& remote, bool& isDiff);
145 : bool CompareFrame(HcclCheckInfo& checkInfo, HcclCheckInfo& checkInfoRecv);
146 : bool CompareCrcInfo(const HcclCMDInfo& hcclCMDInfo, HcclCRCInfo& crcInfo, HcclCRCInfo& crcInfoRecv);
147 : void ReportCmdInfoCheckFailed(
148 : const HcclCMDInfo& hcclCMDInfo, const std::string& paraName, const std::string& localPara,
149 : const std::string& remotePara) const;
150 : void ReportCmdInfoCheckFailed(
151 : const HcclCMDInfo& hcclCMDInfo, const std::string& paraName, uint32_t localPara, uint32_t remotePara);
152 : void ReportCrcCheckFailed(
153 : const HcclCMDInfo& hcclCMDInfo, HcclCrcRecordType crcType, const uint32_t localCrc,
154 : const uint32_t remoteCrc) const; // 打印CRC校验失败信息
155 : void ReportCommonError(
156 : const HcclCMDInfo& hcclCMDInfo, const std::string& paraName, const std::string& localParaStr,
157 : const std::string& remoteParaStr, const std::string& errorMsg) const;
158 : void CompareCmdInfo(HcclCheckInfo& checkInfo, HcclCheckInfo& checkInfoRecv);
159 : bool CompareSection(const char* pRawData, const char* recvBuf, u32 len);
160 : HcclResult AddCrc(const u32 crcValue);
161 : HcclResult ClearCrcInfo(void);
162 : HcclResult GetCrc(u32 num, u32* crcAddr);
163 : HcclResult CalcRawDataCrc(const void* ptr, u64 length, u32& crc);
164 :
165 : // 要校验的内容
166 : std::unordered_map<std::string, HcclCMDInfo> cmdInfoMap_;
167 : std::unordered_map<std::string, std::map<HcclCrcRecordType, u32>> crcRecords_; // CRC校验码记录
168 : // cann 版本号
169 : char cannVersion_[MAX_CANN_VERSION_LEN + 1];
170 : // cann 版本校验开关
171 : bool cannVerCheckSwitch_;
172 : // 算法标志是否已经更新标志，防止发送空数据
173 : bool cannVerInfoRecordFlag_;
174 : // CMD是否已经更新标志，防止发送空数据，要校验内容的信息的更新情况（false：未更新）
175 : std::unordered_map<std::string, bool> infoFlagCmdMap_;
176 : // config文件是否存在，1表示存在，0表示不存在
177 : bool configFileExist_;
178 : ProtocolType protocolType_ = ProtocolType::RESERVED;
179 : std::vector<u32> crcTable_;
180 : std::mutex mutex_;
181 : bool inconsistentCheckFirstDone_ = false; // first模式下是否已完成首次校验
182 : };
183 : } // namespace hccl
184 : #endif // RANK_CONSISTENTCY_CHECKER_H
|
