Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #ifndef HCCL_HEARTBEAT_H
12 : #define HCCL_HEARTBEAT_H
13 :
14 : #include <thread>
15 : #include <map>
16 : #include <deque>
17 : #include <mutex>
18 :
19 : #include "hccl/hccl_types.h"
20 : #include "log.h"
21 : #include "reference_map.h"
22 : #include "ring_buffer.h"
23 : #include "common.h"
24 : #include "sal_pub.h"
25 : #include "hccl_socket_manager.h"
26 : #include "transport_pub.h"
27 : #include "topoinfo_struct.h"
28 : #include "comm_config_pub.h"
29 : namespace hccl {
30 : using RankId = u32;
31 : constexpr u32 BROADCAST_INTERVAL = 50; // 背景线程执行周期为50 ms
32 : constexpr u32 BROADCAST_INTERVAL_WITH_CHECK = 25; // 背景线程执行周期为25 ms
33 : constexpr u32 STUCK_INTERVAL = 300000; // 5min监控一次,默认 300000 ms
34 : constexpr u32 STUCK_COUNT = STUCK_INTERVAL / BROADCAST_INTERVAL;
35 : constexpr u32 OPINFO_SEND_NUM_BY_TAG = 500; // 一次心跳帧发送的算子信息个数
36 : constexpr u32 OPINFO_TAG_QUEUE_NUM = 10; // 一次心跳帧发送的算子信息个数
37 : constexpr u32 HEARTBEAT_INTERVAL = 1000; // 心跳帧发送周期为1000 ms
38 : constexpr u32 HEARTBEAT_COUNT = HEARTBEAT_INTERVAL / BROADCAST_INTERVAL; // 心跳帧发送间隔数
39 : constexpr u32 BASE_NUMBER = 2;
40 : constexpr u32 RETRY_CQE_ARRAY_SIZE = 128; // 重执行时获取的CQE数组的最大数量，最大128
41 : constexpr u32 JITTER_TIME = 300; // 关键事件允许的误差事件范围±300s。误差来源：EVENT和NOTIFY差异、传播耗时、计时误差
42 : constexpr u32 EVENT_MAX_CNT = 5000; // 防止内存泄漏，同时不能太短，防止正常事件被冲掉
43 : constexpr u32 THROUND_MILS = 1000; // 1000ms
44 : constexpr u32 OPINFO_QUEUE_MAX_SIZE = 131072; // 算子下发校验队列最大算子个数，防止内存占用
45 : constexpr u32 MAX_SENDBUFF_SIZE = 3072; // SendBuff[dst] 最大数量
46 : constexpr u32 SR_TAG_MAP_MAX_NUM = 65536;
47 : constexpr u32 HBFRAME_SEND_LOOP_MAX_NUM = 120;
48 : constexpr s32 HCCL_STUCK_DETECT_TIME_MIN = 60; // 卡住检测最短时间
49 : constexpr s32 HCCL_STUCK_DETECT_TIME_BASE = 3; // 卡住检测时间为execTime/3
50 : constexpr s32 HCCL_LOST_THRESHOLD = 30; // 心跳丢失阈值为30s
51 :
52 4 : using UIDType = struct HcclHeartBeatUid {
53 : char id[512] = {0}; // ip[IP_ADDRESS_BUFFER_LEN] + ifname[MAX_INTERFACE_NAME_LEN] + devid 最大不超过512字节
54 455 : bool operator == (const HcclHeartBeatUid &that) const
55 : {
56 1547 : return std::string(this->id) == std::string(that.id);
57 : }
58 9 : bool operator != (const HcclHeartBeatUid &that) const
59 : {
60 36 : return std::string(this->id) != std::string(that.id);
61 : }
62 427 : bool operator < (const HcclHeartBeatUid &that) const
63 : {
64 1708 : return std::string(this->id) < std::string(that.id);
65 : }
66 : };
67 : }
68 :
69 : namespace std {
70 : template <> class hash<hccl::HcclHeartBeatUid> {
71 : public:
72 632 : size_t operator () (const hccl::HcclHeartBeatUid &uid) const
73 : {
74 1137 : return hash<string>()(string(uid.id));
75 : }
76 : };
77 : }
78 :
79 : namespace hccl {
80 : constexpr u8 HAS_CONN = 1;
81 : constexpr u8 NO_CONN = 0;
82 : constexpr u32 TIME_FROM_1900 = 1900;
83 :
84 : enum class HeartBeatStatus {
85 : HEARTBEAT_OK,
86 : HEARTBEAT_LOST,
87 : HEARTBEAT_NOTIFY,
88 : HEARTBEAT_CQE_ERR,
89 : HEARTBEAT_OPRETRY_NOT_SUPPORT,
90 : HEARTBEAT_STUCK,
91 : HEARTBEAT_INCONSISTENT
92 : };
93 : const std::map<HeartBeatStatus, std::string> HEARTBEAT_STATUS_STR_MAP{
94 : {HeartBeatStatus::HEARTBEAT_OK, "OK"},
95 : {HeartBeatStatus::HEARTBEAT_LOST, "LOST"},
96 : {HeartBeatStatus::HEARTBEAT_NOTIFY, "NOTIFY"},
97 : {HeartBeatStatus::HEARTBEAT_CQE_ERR, "ERROR CQE"},
98 : {HeartBeatStatus::HEARTBEAT_OPRETRY_NOT_SUPPORT, "OPRETRY NOT SUPPORT"},
99 : {HeartBeatStatus::HEARTBEAT_STUCK, "STUCK"},
100 : {HeartBeatStatus::HEARTBEAT_INCONSISTENT, "INCONSISTENT"}
101 : };
102 5 : inline std::string GetHeartBeatStatusStr(HeartBeatStatus status)
103 : {
104 5 : auto iter = HEARTBEAT_STATUS_STR_MAP.find(status);
105 5 : if (iter == HEARTBEAT_STATUS_STR_MAP.end()) {
106 0 : return "Unknown";
107 : } else {
108 5 : return iter->second;
109 : }
110 : }
111 :
112 : struct CounterStat {
113 : std::pair<int32_t, int32_t> oldCounter{0, 0};
114 : std::pair<int32_t, int32_t> newCounter{0, 0};
115 : std::uint64_t issueCnt = 0;
116 : bool isNeedDetect = false;
117 : bool isFirst = true;
118 : std::uint64_t couterPrintInter = STUCK_COUNT;
119 9 : CounterStat() {};
120 : };
121 :
122 : enum class InconsistentType{
123 : NO_INCONSISTENT = 0,
124 : OPTYPE_INCONSISTENT = 1,
125 : DATATYPE_INCONSISTENT = 2,
126 : REDUCETYPE_INCONSISTENT = 3,
127 : ROOT_INCONSISTENT = 4,
128 : COUNT_INCONSISTENT = 5
129 : };
130 :
131 : const std::map<InconsistentType, std::string> OP_INCONSISTENT_STR_MAP{
132 : {InconsistentType::NO_INCONSISTENT, "no exist op inconsistent"},
133 : {InconsistentType::OPTYPE_INCONSISTENT, "op type inconsistent"},
134 : {InconsistentType::DATATYPE_INCONSISTENT, "op data type inconsistent"},
135 : {InconsistentType::REDUCETYPE_INCONSISTENT, "op reduce type inconsistent"},
136 : {InconsistentType::ROOT_INCONSISTENT, "op root inconsistent"},
137 : {InconsistentType::COUNT_INCONSISTENT, "op count inconsistent"}
138 : };
139 :
140 0 : inline std::string GetInconsistentTypeStr(InconsistentType status)
141 : {
142 0 : auto iter = OP_INCONSISTENT_STR_MAP.find(status);
143 0 : if (iter == OP_INCONSISTENT_STR_MAP.end()) {
144 0 : return "Unknown";
145 : } else {
146 0 : return iter->second;
147 : }
148 : };
149 :
150 : struct OpInfoDesc {
151 : HcclCMDType opType = HcclCMDType::HCCL_CMD_INVALID;
152 : HcclDataType dataType = HcclDataType::HCCL_DATA_TYPE_RESERVED;
153 : HcclReduceOp reduceOp = HcclReduceOp::HCCL_REDUCE_RESERVED;
154 : uint32_t root = 0;
155 : uint64_t count = 0;
156 : uint64_t index = 0;
157 : bool isValid = false;
158 : };
159 :
160 : struct OpInconsistentInfo {
161 : InconsistentType inconsistentType;
162 : std::string localInfo;
163 : std::string remoteInfo;
164 : OpInfoDesc opInfoDesc;
165 0 : OpInconsistentInfo(InconsistentType inconsistentType, const std::string &localInfo, const std::string &remoteInfo, const OpInfoDesc &opInfoDesc)
166 0 : : inconsistentType(inconsistentType), localInfo(localInfo), remoteInfo(remoteInfo), opInfoDesc(opInfoDesc)
167 0 : {}
168 : };
169 :
170 : struct OpInfoTagQueue {
171 : OpInfoDesc opInfoList[OPINFO_SEND_NUM_BY_TAG];
172 : char identifier[ROOTINFO_INDENTIFIER_MAX_LENGTH] = {};
173 : u32 opInfoNum = 0;
174 : };
175 :
176 : struct OpInfoTagQueueFrame {
177 : OpInfoTagQueue opInfoTagQueue[OPINFO_TAG_QUEUE_NUM];
178 : };
179 :
180 : struct HeartBeatFrame {
181 : UIDType src;
182 : UIDType dst;
183 : UIDType crimer;
184 : UIDType informer;
185 : HeartBeatStatus status = HeartBeatStatus::HEARTBEAT_OK;
186 : HcclUs TOARelative; // time of arrival (Relative)
187 : HcclSystemTime TOASystem; // time of arrival (System)
188 2 : HeartBeatFrame() {}
189 5 : HeartBeatFrame(UIDType &crimer, UIDType &informer, HeartBeatStatus status, HcclUs TOARelativeIn,
190 : HcclSystemTime TOASystemIn)
191 10 : : crimer(crimer), informer(informer), status(status), TOARelative(TOARelativeIn),
192 5 : TOASystem(TOASystemIn)
193 5 : {}
194 7 : HeartBeatFrame(UIDType &src, UIDType &dst, UIDType &crimer, UIDType &informer, HeartBeatStatus status)
195 7 : : src(src), dst(dst), crimer(crimer), informer(informer), status(status)
196 7 : {}
197 : };
198 :
199 : struct HeartBeatFrameWithOpCheck {
200 : UIDType src;
201 : UIDType dst;
202 : UIDType crimer;
203 : UIDType informer;
204 : HeartBeatStatus status = HeartBeatStatus::HEARTBEAT_OK;
205 : HcclUs TOARelative; // time of arrival (Relative)
206 : HcclSystemTime TOASystem; // time of arrival (System)
207 : OpInfoTagQueueFrame opInfoTagQueueFrame;
208 2 : HeartBeatFrameWithOpCheck() {}
209 : HeartBeatFrameWithOpCheck(UIDType &crimer, UIDType &informer, HeartBeatStatus status, HcclUs TOARelativeIn,
210 : HcclSystemTime TOASystemIn)
211 : : crimer(crimer), informer(informer), status(status), TOARelative(TOARelativeIn),
212 : TOASystem(TOASystemIn)
213 : {}
214 0 : HeartBeatFrameWithOpCheck(UIDType &src, UIDType &dst, UIDType &crimer, UIDType &informer, HeartBeatStatus status)
215 0 : : src(src), dst(dst), crimer(crimer), informer(informer), status(status)
216 0 : {}
217 : };
218 :
219 : struct ConnInfo {
220 : std::shared_ptr<HcclSocket> socket = nullptr;
221 : std::queue<HeartBeatFrame> sendBuffer;
222 : std::queue<HeartBeatFrameWithOpCheck> sendBufferWithOpCheck;
223 : u32 restSize = 0;
224 : RingBuffer recvBuffer;
225 : u32 lostNum = 0;
226 : u32 lostReportCnt = 0;
227 : bool newConn = false;
228 : std::vector<SocketWlistInfo> wlistInfosVec;
229 18 : ConnInfo() {}
230 8 : ConnInfo(bool newConn, std::shared_ptr<HcclSocket> &socket)
231 8 : : socket(socket), newConn(newConn)
232 8 : {}
233 : };
234 :
235 : struct LinkInfo {
236 : std::string identifier;
237 : RankId localRank;
238 : std::string localServerId;
239 : s32 localDevicePhyId;
240 : RankId remoteRank;
241 : std::string remoteServerId;
242 : s32 remoteDevicePhyId;
243 11 : LinkInfo() {}
244 0 : LinkInfo(std::string &identifier, RankId localRank, std::string &localServerId, s32 localDevicePhyId,
245 : RankId remoteRank, std::string &remoteServerId, s32 remoteDevicePhyId)
246 0 : : identifier(identifier), localRank(localRank), localServerId(localServerId), localDevicePhyId(localDevicePhyId),
247 0 : remoteRank(remoteRank), remoteServerId(remoteServerId), remoteDevicePhyId(remoteDevicePhyId)
248 0 : {}
249 : };
250 :
251 2 : using ErrCqeInfo = struct TagErrCqeInfo {
252 : CqeInfo cqeInfo;
253 : LinkInfo linkInfo;
254 : u32 qpn;
255 5 : TagErrCqeInfo() {}
256 4 : TagErrCqeInfo(CqeInfo &cqeInfo, LinkInfo &linkInfo, u32 qpn)
257 4 : : cqeInfo(cqeInfo), linkInfo(linkInfo), qpn(qpn)
258 4 : {}
259 7 : bool operator<(const TagErrCqeInfo& other) const {
260 7 : return qpn < other.qpn;
261 : }
262 : };
263 :
264 : class Heartbeat {
265 : public:
266 : static Heartbeat& GetInstance(s32 deviceLogicID);
267 : HcclResult RegisterToHeartBeat(u32 userRank, DevType devType, std::vector<RankInfo> &rankInfoList, const u32 port,
268 : const bool isNeedNic, const std::string &commIdentifier, bool useSuperPodMode, bool isUsedRdmaLevel0,
269 : bool retryEnable = false, bool backupEnable = false);
270 : HcclResult RegisterToHeartBeat(u32 userRank, DevType devType, std::vector<RankInfo> &rankInfoList, const u32 port,
271 : const bool isNeedNic, u32 peerRankId, const std::string &commIdentifier, const std::string& tag,
272 : bool useSuperPodMode, bool isUsedRdmaLevel0, bool retryEnable = false, bool backupEnable = false);
273 : HcclResult AddOpInfoToHeartBeat(const std::string &identifier, const OpInfoDesc &opInfo, const std::string &newTag);
274 : HcclResult DeleteOpInfoToHeartBeat(const std::string &identifier, const std::string &newTag);
275 : HcclResult UnRegisterRanks(const std::string& group = HCCL_WORLD_GROUP);
276 : // 集合通信，解开注册
277 : void UnRegisterToHeartBeat(DevType devType, const std::string &commIdentifier);
278 : // 非点对点通信，解开注册
279 : void UnRegisterToHeartBeat(DevType devType, const std::string &commIdentifier, const std::string &tag);
280 : HcclResult CheckErrorCqe(const std::string &identifier, HcclResult &result);
281 : HcclResult CheckOpInconsistentError(const std::string &identifier, HcclResult &result);
282 : HcclResult SetRankPortInfo(bool isUseRankPort, std::vector<u32> &nicRanksPorts, std::vector<u32> &vnicRanksPorts,
283 : bool devPortSwitchOn);
284 : std::vector<std::string> GetErrStatusVec(const std::string& group = HCCL_WORLD_GROUP);
285 : HcclResult GetQpnErr(const std::string &identifier, std::set<std::tuple<u32, u32, u32>> &qpErrSet);
286 : HcclResult BroadcastCqeErr(const std::string &identifier);
287 : HcclResult ClearAllCqeErr(const std::string &identifier);
288 : HcclResult ClearCqeErr(const std::string &identifier, u32 remoteRank, u32 qpn = 0);
289 : void SetOpretryErr();
290 : void GetIpQueue();
291 : bool IsPaused() const;
292 : bool IsResumed() const;
293 :
294 : private:
295 845 : Heartbeat() = default;
296 : ~Heartbeat();
297 : HcclResult Init(const RankInfo& locRank, const bool useSuperPodMode, const bool isNeedNic, const u32 port,
298 : const std::string& group = HCCL_WORLD_GROUP);
299 : HcclResult DeInit();
300 : HcclResult RegisterRanks(DevType devType, const RankInfo& locRank, std::vector<RankInfo>& rankInfos, const u32 port,
301 : const bool isNeedNic, const std::string& group = HCCL_WORLD_GROUP, bool useSuperPodMode = false,
302 : bool isUsedRdma = false);
303 : std::string GetConnTag(HcclSocketRole role, UIDType &rem);
304 : HcclResult GetConnInfo(RankInfo& remRank, bool useSuperPodMode, HcclSocketRole role, HcclSocketType type,
305 : std::map<UIDType, ConnInfo>& needConnectRank);
306 : template <typename T> HcclResult GetSamePlaneConnInfo(HcclSocketType type, std::vector<std::pair<T, u32>>& connVec,
307 : T& locId, std::vector<RankInfo>& rankInfos, std::map<UIDType, ConnInfo>& needConnectRank,
308 : bool useSuperPodMode, u32 worldRank);
309 : HcclResult GetConnectRank(const RankInfo& locRank, std::vector<RankInfo>& rankInfos, std::map<UIDType,
310 : ConnInfo>& needConnectRank, bool useSuperPodMode, bool isUsedRdma = false);
311 : UIDType GetUId(const RankInfo& rankInfo) const;
312 : std::string FormatUId(const UIDType& uid) const;
313 : HcclResult SendFrame(UIDType &dst, UIDType &crimer, UIDType &informer, HeartBeatStatus status);
314 : HcclResult SendFrameWithOpCheck(UIDType &dst, UIDType &crimer, UIDType &informer, HeartBeatStatus status,
315 : const OpInfoTagQueueFrame &opInfoTagQueueFrame);
316 : HcclResult RecvFrame(UIDType &src);
317 : HcclResult RecvFrameWithOpCheck(UIDType &src);
318 : HcclResult ParseFrame(HeartBeatFrame& bf, UIDType &src);
319 : HcclResult ParseFrameWithOpCheck(HeartBeatFrameWithOpCheck &bf, UIDType &src);
320 : void SetStatus(UIDType &crimer, UIDType &informer, HeartBeatStatus status, bool needBroadcast = true);
321 : void HeartbeatStatusMonitor();
322 : void ProcessExceptionEvent();
323 : void ProcessCqeErrInfo();
324 : void DelErrorSocket();
325 : bool IsKeyEvent(HeartBeatFrame &event, HcclUs curTime, const std::string& group = HCCL_WORLD_GROUP);
326 : void MakeErrMsg(std::queue<HeartBeatFrame> &keyEvents, std::vector<std::string> &errStatusVec);
327 : std::vector<std::string> PrintEvents(std::map<HeartBeatStatus, std::queue<HeartBeatFrame>> &keyEvents);
328 : void StuckDetection(uint64_t &cnt, CounterStat &counterStat);
329 : void InitStuckDetection(CounterStat &counterStat);
330 : void PrintAndBroadCastErrorCqe(const ErrCqeInfo &info);
331 : void SaveQpnForOpRetry(const ErrCqeInfo &info);
332 : void OpRetryCQEHandle(const HcclNetDevCtx netDevCtx);
333 : bool GetRetryEnable(const ErrCqeInfo &info);
334 : HcclResult ClearRetryEnableMapItem(const std::string &identifier);
335 : void ProcessCqeErrInfoByNetDevCtx(const HcclIpAddress &nicIp);
336 : bool IsEnableBackupLink();
337 : void RegisterRetryInfo(const std::string &commIdentifier, bool retryEnable, bool backupEnable);
338 : HcclResult InitNic(const NicType nicType, const s32 devicePhyId, const s32 deviceLogicId,
339 : const hccl::HcclIpAddress ip, const u32 port, const bool isBackUp = false);
340 : HcclResult InitDeviceNic(const RankInfo &locRank, bool isNeedNic, u32 port);
341 : HcclResult InitHostNic(const RankInfo &locRank, bool isNeedNic, u32 port);
342 : u32 GetPort(HcclSocketType type, u32 remoteUserRank, u32 remoteDeviceId);
343 : u32 GetHostPort(s32 devicePhyId);
344 : HcclResult PrepareConnect(ConnInfo &info);
345 : void CreateLinkWithRemote(std::string group, UIDType rem, ConnInfo needConnectRank);
346 : void CreateHBLinksAsync();
347 : void AddOpInfo(const std::string &identifier, const OpInfoDesc &opInfo, const std::string ¶mTag);
348 : void GetOneOpInfo(std::string &tag, OpInfoDesc &opInfo);
349 : void GetSendOpInfoList(OpInfoTagQueueFrame &opInfoTagQueueFrame);
350 : void SaveOpInfo(const OpInfoTagQueueFrame &opInfoTagQueueFrame, UIDType &src);
351 : HcclResult CheckIsSameOp(const OpInfoDesc &localOpInfo, const OpInfoDesc &remoteOpInfo, InconsistentType &status);
352 : void CheckRecvOpInfoList();
353 : void RegisterSROpIdentifier(const std::string &identifier, const std::string ¶mTag);
354 : void AddInconsistentOpRecord(const std::string &identifier, const OpInfoDesc &localOpInfo, InconsistentType status,
355 : const std::string &localInfo, const std::string &remoteInfo);
356 : void CheckSnapshotStatus();
357 : struct Status {
358 : HeartBeatStatus status = HeartBeatStatus::HEARTBEAT_OK;
359 : UIDType informer;
360 : bool needBroadcast = false;
361 63 : Status() {}
362 : };
363 :
364 : enum class HBLinkStatus {
365 : HEARTBEAT_LINK_NOT_START,
366 : HEARTBEAT_LINK_BUILDING,
367 : HEARTBEAT_LINK_COMPLETED
368 : };
369 :
370 : HcclIpAddress vnicIp_;
371 : HcclIpAddress nicIp_;
372 : HcclIpAddress backupNicIp_;
373 : u32 devicePhyId_;
374 : u32 deviceBackUpPhyId_;
375 : u32 superDeviceId_;
376 : NICDeployment nicDeploy_;
377 : UIDType uid_;
378 : bool initialized_ = false;
379 : u32 lostThreshold_ = 0;
380 : bool isDeInit_ = false;
381 : bool startSendRecvTask_ = false;
382 : std::map<std::string, std::queue<std::pair<UIDType, ConnInfo>>> hbLinkConnInfo_{};
383 : std::mutex hbLinkConnInfoMtx_;
384 : std::map<std::string, std::map<UIDType, u8>> groupMap_;
385 : ReferenceMap<UIDType, ConnInfo> rankId2SocketMap_;
386 : ReferenceMap<UIDType, Status> rankId2StatusMap_;
387 : std::map<UIDType, HBLinkStatus> rankId2LinkStatusMap_;
388 : std::map<UIDType, std::unique_ptr<std::thread>> linkThreadMap_{};
389 : std::atomic<bool> linkThreadRunning_{false};
390 : std::atomic<u32> linkThreadCount_{0};
391 : std::unique_ptr<std::thread> sendRecvThread_;
392 : std::queue<HeartBeatFrame> errStatusQueue_;
393 : std::queue<UIDType> errRankQueue_;
394 : std::mutex ProcessLock_;
395 : u32 deviceLogicId_;
396 : u32 deviceBackupLogicId_;
397 : std::map<std::string, std::set<ErrCqeInfo>> remoteIpMap;
398 : std::set<u32> qpnDissociativeSet;
399 : std::mutex remoteIpMutex_;
400 : bool isUseRankPort_{ false };
401 : bool devPortSwitchOn_{ false };
402 : std::vector<u32> nicRanksPorts_;
403 : std::vector<u32> vnicRanksPorts_;
404 : std::vector<UIDType> errorSocket_;
405 : std::map<std::string, std::map<u32, std::set<ErrCqeInfo>>> rankMapForRetryAgent;
406 : std::mutex qpnMapMutexForRetry_;
407 : std::map<std::string, bool> retryEnableTable_;
408 : std::mutex retryEnableMutex_;
409 : std::set<std::string> backupEnableTable_;
410 : std::mutex backupEnableMutex_;
411 : std::mutex ctxMapMutex_;
412 : std::map<HcclIpAddress, HcclNetDevCtx> netDevCtxMap_;
413 : std::map<HcclIpAddress, std::shared_ptr<HcclSocket>> listenSocketMap_;
414 : s32 stuckDetectTime_;
415 : std::mutex opInfoQueueMutex_;
416 : std::deque<std::pair<std::string, OpInfoDesc>> opInfoQueue_;
417 : std::deque<std::pair<std::string, OpInfoDesc>> opInfoQueueForSend_;
418 : std::unordered_map<std::string, u64> opInfoIndexMap_;
419 : std::mutex opInfoMapMutex_;
420 : std::unordered_map<std::string, std::map<u64, OpInfoDesc>> opInfoMap_;
421 : std::list<std::tuple<OpInfoDesc, std::string, UIDType>> recvOpInfoList_;
422 : std::mutex inconsistentOpMutex_;
423 : std::map<std::string, OpInconsistentInfo> inconsistentOpMap_;
424 : std::mutex srTagMutex_;
425 : std::map<std::string, std::string> srTagMap_;//SR算子tag->identifier映射
426 : bool isPaused_ { false }; // heartbeat need to be paused when snapshot
427 : };
428 : } // namespace hccl
429 :
430 : #endif // HCCL_HEARTBEAT_H
|
