Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #ifndef HCCL_RETRY_SERVER_H
12 : #define HCCL_RETRY_SERVER_H
13 : #include <unordered_set>
14 : #include "opretry_base.h"
15 :
16 : namespace hccl {
17 :
18 : HcclResult CreateOpRetryServerByState(RetryState state, RetryContext* retryCtx);
19 :
20 : // server状态机 正常运行状态转移表
21 : const std::map<RetryState, RetryState> RETRY_SERVER_STATE_TRANSFER_LABEL {
22 : {RETRY_STATE_SERVER_RUNNING, RETRY_STATE_CMD_STOP_AICPU},
23 : {RETRY_STATE_CMD_STOP_AICPU, RETRY_STATE_WAIT_AICPU_STOPED},
24 : {RETRY_STATE_WAIT_AICPU_STOPED, RETRY_STATE_CMD_STOP_STREAM},
25 : {RETRY_STATE_CMD_STOP_STREAM, RETRY_STATE_WAIT_STREAM_STOPED},
26 : {RETRY_STATE_WAIT_STREAM_STOPED, RETRY_STATE_CMD_CLEAR_STREAM},
27 : {RETRY_STATE_CMD_CLEAR_STREAM, RETRY_STATE_WAIT_STREAM_CLEARED},
28 : {RETRY_STATE_WAIT_STREAM_CLEARED, RETRY_STATE_CMD_STOP_TRANSPORT},
29 : {RETRY_STATE_CMD_STOP_TRANSPORT, RETRY_STATE_WAIT_STOP_TRANSPORT},
30 : {RETRY_STATE_WAIT_STOP_TRANSPORT, RETRY_STATE_CMD_CHECK},
31 : {RETRY_STATE_CMD_CHECK, RETRY_STATE_WAIT_CHECK_INFO},
32 : {RETRY_STATE_WAIT_CHECK_INFO, RETRY_STATE_CHECK_OP},
33 : {RETRY_STATE_CHECK_OP, RETRY_STATE_CMD_CHECK_LINK},
34 : {RETRY_STATE_CMD_CHECK_LINK, RETRY_STATE_WAIT_LINK_CHECKED},
35 : {RETRY_STATE_WAIT_LINK_CHECKED, RETRY_STATE_CHECK_ALL_LINK},
36 : {RETRY_STATE_CHECK_ALL_LINK, RETRY_STATE_CMD_RESUME_TRANSPORT},
37 : {RETRY_STATE_CMD_RESUME_TRANSPORT, RETRY_STATE_WAIT_RESUME_TRANSPORT},
38 : {RETRY_STATE_WAIT_RESUME_TRANSPORT, RETRY_STATE_CMD_RESET_NOTIFY},
39 : {RETRY_STATE_CMD_RESET_NOTIFY, RETRY_STATE_WAIT_NOTIFY_RESETED},
40 : {RETRY_STATE_WAIT_NOTIFY_RESETED, RETRY_STATE_CMD_CAN_RETRY},
41 : {RETRY_STATE_CMD_CAN_RETRY, RETRY_STATE_WAIT_CAN_RETRY},
42 : {RETRY_STATE_WAIT_CAN_RETRY, RETRY_STATE_SERVER_RUNNING},
43 : {RETRY_STATE_SERVER_RETRY_FAIL, RETRY_STATE_SERVER_RUNNING}
44 : };
45 :
46 : // server状态机 IssueCmd状态对应的command
47 : const std::map<RetryState, RetryCommand> RETRY_SERVER_STATE_TO_CMD_LABEL {
48 : {RETRY_STATE_CMD_CHECK_LINK, RETRY_CMD_CHECK_LINK},
49 : {RETRY_STATE_CMD_STOP_AICPU, RETRY_CMD_STOP_AICPU},
50 : {RETRY_STATE_CMD_STOP_STREAM, RETRY_CMD_STOP_STREAM},
51 : {RETRY_STATE_CMD_CLEAR_STREAM, RETRY_CMD_CLEAR_STREAM},
52 : {RETRY_STATE_CMD_STOP_TRANSPORT, RETRY_CMD_STOP_TRANSPORT},
53 : {RETRY_STATE_CMD_RESET_NOTIFY, RETRY_CMD_RESET_NOTIFY},
54 : {RETRY_STATE_CMD_RESUME_TRANSPORT, RETRY_CMD_RESUME_TRANSPORT},
55 : {RETRY_STATE_CMD_CHECK, RETRY_CMD_CHECK_OPNAME},
56 : {RETRY_STATE_CMD_CAN_RETRY, RETRY_CMD_CAN_RETRY}
57 : };
58 :
59 : // RETRY_STATE_SERVER_RETRY_FAIL 重执行异常状态处理
60 3 : class OpRetryServerBase : public OpRetryBase {
61 : public:
62 : HcclResult ProcessError(RetryContext* retryCtx) override;
63 : };
64 :
65 : // RETRY_STATE_SERVER_RUNNING
66 1 : class OpRetryServerRunning : public OpRetryServerBase {
67 : public:
68 : HcclResult ProcessEvent(RetryContext* retryCtx) override;
69 : HcclResult ParaseErrorCode(RetryContext* retryCtx, HcclAgentRetryInfo &agentInfo, RetryState &nextState);
70 : protected:
71 : std::map<u32, std::chrono::steady_clock::time_point> lastRecvTimes_;
72 : std::unordered_set<u32> disableAgent_; // 记录已经关闭的对端, 不再轮询, 避免刷屏
73 : };
74 :
75 : // server处理错误rank状态机
76 1 : class OpRetryServerHandleError : public OpRetryServerRunning {
77 : public:
78 : HcclResult ProcessEvent(RetryContext* retryCtx) override;
79 : private:
80 : HcclResult SetNeedRetryServerRank(RetryContext* retryCtx, const HcclOpIdentifier &opId);
81 : };
82 :
83 : // 公共状态-向agent状态机发送命令
84 : class OpRetryServerIssueCmd : public OpRetryServerBase {
85 : public:
86 : HcclResult ProcessEvent(RetryContext* retryCtx) override;
87 : };
88 :
89 : // 公共状态-等待agent状态机回复
90 : class OpRetryServerWaitResp : public OpRetryServerBase {
91 : public:
92 : HcclResult ProcessEvent(RetryContext* retryCtx) override;
93 : private:
94 : // 接收到对端重执行失败的信息后，打印当前接收到的Agent节点信息
95 : void PrintAgentInfoAfterFail(std::map<u32, HcclAgentRetryInfo> &serverSockets, std::set<u32> &recvVaild, HcclAgentRetryInfo &agentRetryInfo);
96 : };
97 :
98 1 : class OpRetryServerCheckOp : public OpRetryServerBase {
99 : public:
100 : HcclResult ProcessEvent(RetryContext* retryCtx) override;
101 : };
102 :
103 : class OpRetryServerCheckAllLink : public OpRetryServerBase {
104 : public:
105 : HcclResult ProcessEvent(RetryContext* retryCtx) override;
106 : };
107 :
108 : class OpRetryServerIssueChangeLinkAndResume : public OpRetryServerBase {
109 : public:
110 : HcclResult ProcessEvent(RetryContext* retryCtx) override;
111 : };
112 :
113 : class OpRetryServerWaitLinkInfo : public OpRetryServerBase {
114 : public:
115 : HcclResult ProcessEvent(RetryContext* retryCtx) override;
116 : };
117 :
118 : class OpRetryServerRetryFail : public OpRetryServerBase {
119 : public:
120 : HcclResult ProcessEvent(RetryContext* retryCtx) override;
121 : };
122 :
123 : // 强制终止NPU后的状态，等待通信域恢复
124 : class OpRetryServerWaitResume : public OpRetryServerRunning {
125 : public:
126 : HcclResult ProcessEvent(RetryContext* retryCtx) override;
127 : };
128 :
129 : class SwitchNicServerCheckAllSwitchRanks : public OpRetryServerBase {
130 : public:
131 : HcclResult ProcessEvent(RetryContext* retryCtx) override;
132 : private:
133 : bool CompareSwitchRankList(const u32* firstSwitchRankList, const u32* switchRankList, const u32 switchRankNum);
134 : bool CompareUseBackupLists(const bool* firstArray, const bool* secondArray, const u32 switchRankNum);
135 : bool CheckRemotePorts(const u32 rankId, const ActiveSwitchInfo &switchRankInfo);
136 : HcclResult CollectSingleAgentActiveSwitchInfo(RetryContext *retryCtx, const u32 rankId,
137 : HcclAgentRetryInfo &agentInfo);
138 : HcclResult CollectAgentActiveSwitchInfo(RetryContext *retryCtx);
139 : HcclResult CheckAgentActiveSwitchInfo(RetryContext *retryCtx);
140 : };
141 :
142 1 : class ResumeServerCheckAllLink : public OpRetryServerBase {
143 : public:
144 : HcclResult ProcessEvent(RetryContext* retryCtx) override;
145 : private:
146 : HcclResult WaitAgentCheckLinkResult(RetryContext* retryCtx);
147 : HcclResult CheckAllLink(RetryContext* retryCtx, RetryState &nextState);
148 : };
149 :
150 : class ResumeServerChangeLink : public OpRetryServerBase {
151 : public:
152 : HcclResult ProcessEvent(RetryContext* retryCtx) override;
153 : private:
154 : HcclResult CmdAgentChangeLink(RetryContext* retryCtx);
155 : HcclResult WaitAllChangeLinkResult(RetryContext* retryCtx, RetryState &nextState);
156 : };
157 : }
158 : #endif
|
