Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #ifndef HCCL_OP_RETRY_PUB_H
12 : #define HCCL_OP_RETRY_PUB_H
13 :
14 : #include <functional>
15 : #include "hccl/base.h"
16 : #include "hccl_common.h"
17 : #include "stream_pub.h"
18 : #include "hccl_socket.h"
19 : #include "hdc_pub.h"
20 : #include "notify_pool.h"
21 : #include "aicpu_operator_pub.h"
22 :
23 : namespace hccl {
24 :
25 : constexpr u32 OPRETRY_DFX_IPINFO_LENGTH = 256;
26 : constexpr u32 CONNECT_REMOTE_DEFAULT = 1;
27 : constexpr u32 CONNECT_REMOTE_BACKUP = 2;
28 : constexpr u32 ACTIVE_SWITCH_TIMES = 2;
29 :
30 : typedef enum {
31 : // server状态
32 : RETRY_STATE_SERVER_RUNNING = 0,
33 : RETRY_STATE_CMD_CHECK_LINK,
34 : RETRY_STATE_WAIT_LINK_CHECKED,
35 : RETRY_STATE_CHECK_ALL_LINK,
36 : RETRY_STATE_CMD_CHANGE_LINK,
37 : RETRY_STATE_WAIT_LINK_CHANGED,
38 : RETRY_STETA_HANDLE_ALL_ERR,
39 : RETRY_STATE_CMD_STOP_AICPU, // 发送StopAicpu命令
40 : RETRY_STATE_WAIT_AICPU_STOPED, // 等待Aicpu停止
41 : RETRY_STATE_CMD_STOP_STREAM,
42 : RETRY_STATE_WAIT_STREAM_STOPED,
43 : RETRY_STATE_CMD_CLEAR_STREAM,
44 : RETRY_STATE_WAIT_STREAM_CLEARED,
45 : RETRY_STATE_CMD_STOP_TRANSPORT,
46 : RETRY_STATE_WAIT_STOP_TRANSPORT,
47 : RETRY_STATE_CMD_RESET_NOTIFY,
48 : RETRY_STATE_WAIT_NOTIFY_RESETED,
49 : RETRY_STATE_CMD_RESUME_TRANSPORT,
50 : RETRY_STATE_WAIT_RESUME_TRANSPORT,
51 : RETRY_STATE_CMD_CHECK,
52 : RETRY_STATE_WAIT_CHECK_INFO,
53 : RETRY_STATE_CHECK_OP,
54 : RETRY_STATE_CMD_CAN_RETRY,
55 : RETRY_STATE_WAIT_CAN_RETRY,
56 : RETRY_STATE_SERVER_RETRY_FAIL,
57 : RETRY_STATE_CMD_PLAN_SWITCH_NIC,
58 : RETRY_STATE_SERVER_WAIT_RESUME,
59 : RETRY_RESUME_STATE_SERVER_CHECK_LINK,
60 : RETRY_RESUME_STATE_SERVER_CHANGE_LINK,
61 :
62 : // agent状态
63 : RETRY_STATE_AGENT_RUNNING,
64 : RETRY_STATE_RESP_AICPU_ERR,
65 : RETRY_STATE_RESP_LINK_CHECKED,
66 : RETRY_STATE_WAIT_CMD_STOP_AICPU,
67 : RETRY_STATE_POLL_AICPU_STOPED,
68 : RETRY_STATE_RESP_AICPU_STOPED,
69 : RETRY_STATE_WAIT_CMD_STOP_STREAM,
70 : RETRY_STATE_POLL_STREAM_STOPED,
71 : RETRY_STATE_RESP_STREAM_STOPED,
72 : RETRY_STATE_WAIT_CMD_CLEAR_STREAM,
73 : RETRY_STATE_RESP_STREAM_CLEARED,
74 : RETRY_STATE_RESP_LINK_CHANGED,
75 : RETRY_STATE_WAIT_CHANGE_LINK_INFO,
76 : RETRY_STATE_POLL_AICPU_CHANGED,
77 : RETRY_STATE_WAIT_CMD_STOP_TRANSPORT,
78 : RETRY_STATE_RESP_STOP_TRANSPORT,
79 : RETRY_STATE_WAIT_CMD_RESET_NOTIFY,
80 : RETRY_STATE_RESP_NOTIFY_RESETED,
81 : RETRY_STATE_WAIT_CMD_CHECK_LINK,
82 : RETRY_STATE_WAIT_CMD_RESUME_TRANSPORT,
83 : RETRY_STATE_RESP_RESUME_TRANSPORT,
84 : RETRY_STATE_WAIT_CMD_CHECK,
85 : RETRY_STATE_RESP_CHECK_INFO,
86 : RETRY_STATE_WAIT_CMD_CAN_RETRY,
87 : RETRY_STATE_POLL_AICPU_RETRYEND,
88 : RETRY_STATE_RESP_AICPU_RETRYEND,
89 : RETRY_STATE_RESP_RUNNING_ERR,
90 : RETRY_STATE_WAIT_CMD_RETRY_FAIL,
91 : RETRY_STATE_AGENT_RETRY_FAIL,
92 : RETRY_STATE_SEND_SWITCH_INFO,
93 : RETRY_STATE_WAIT_CMD_SEND_AICPU,
94 : RETRY_STATE_AGENT_WAIT_RESUME,
95 : RETRY_RESUME_STATE_AGENT_CHECK_LINK,
96 : RETRY_RESUME_STATE_AGENT_CHANGE_LINK,
97 :
98 : RETRY_STATE_RESERVED,
99 : } RetryState;
100 :
101 : const std::map<RetryState, std::string> RETRY_STATE_STR_MAP{
102 : // server状态
103 : {RETRY_STATE_SERVER_RUNNING, "RETRY_STATE_SERVER_RUNNING"},
104 : {RETRY_STATE_CMD_CHECK_LINK, "RETRY_STATE_CMD_CHECK_LINK"},
105 : {RETRY_STATE_CMD_CHANGE_LINK, "RETRY_STATE_CMD_CHANGE_LINK"},
106 : {RETRY_STATE_WAIT_LINK_CHANGED, "RETRY_STATE_WAIT_LINK_CHANGED"},
107 : {RETRY_STATE_WAIT_LINK_CHECKED, "RETRY_STATE_WAIT_LINK_CHECKED"},
108 : {RETRY_STATE_CHECK_ALL_LINK, "RETRY_STATE_CHECK_ALL_LINK"},
109 : {RETRY_STETA_HANDLE_ALL_ERR, "RETRY_STETA_HANDLE_ALL_ERR"},
110 : {RETRY_STATE_CMD_STOP_AICPU, "RETRY_STATE_CMD_STOP_AICPU"},
111 : {RETRY_STATE_WAIT_AICPU_STOPED, "RETRY_STATE_WAIT_AICPU_STOPED"},
112 : {RETRY_STATE_CMD_STOP_STREAM, "RETRY_STATE_CMD_STOP_STREAM"},
113 : {RETRY_STATE_WAIT_STREAM_STOPED, "RETRY_STATE_WAIT_STREAM_STOPED"},
114 : {RETRY_STATE_CMD_CLEAR_STREAM, "RETRY_STATE_CMD_CLEAR_STREAM"},
115 : {RETRY_STATE_WAIT_STREAM_CLEARED, "RETRY_STATE_WAIT_STREAM_CLEARED"},
116 : {RETRY_STATE_CMD_RESET_NOTIFY, "RETRY_STATE_CMD_RESET_NOTIFY"},
117 : {RETRY_STATE_CMD_STOP_TRANSPORT, "RETRY_STATE_CMD_STOP_TRANSPORT"},
118 : {RETRY_STATE_WAIT_STOP_TRANSPORT, "RETRY_STATE_WAIT_STOP_TRANSPORT"},
119 : {RETRY_STATE_WAIT_NOTIFY_RESETED, "RETRY_STATE_WAIT_NOTIFY_RESETED"},
120 : {RETRY_STATE_WAIT_CMD_CHECK_LINK, "RETRY_STATE_WAIT_CMD_CHECK_LINK"},
121 : {RETRY_STATE_CMD_RESUME_TRANSPORT, "RETRY_STATE_CMD_RESUME_TRANSPORT"},
122 : {RETRY_STATE_WAIT_RESUME_TRANSPORT, "RETRY_STATE_WAIT_RESUME_TRANSPORT"},
123 : {RETRY_STATE_CMD_CHECK, "RETRY_STATE_CMD_CHECK"},
124 : {RETRY_STATE_WAIT_CHECK_INFO, "RETRY_STATE_WAIT_CHECK_INFO"},
125 : {RETRY_STATE_CHECK_OP, "RETRY_STATE_CHECK_OP"},
126 : {RETRY_STATE_CMD_CAN_RETRY, "RETRY_STATE_CMD_CAN_RETRY"},
127 : {RETRY_STATE_WAIT_CAN_RETRY, "RETRY_STATE_WAIT_CAN_RETRY"},
128 : {RETRY_STATE_SERVER_RETRY_FAIL, "RETRY_STATE_SERVER_RETRY_FAIL"},
129 : {RETRY_STATE_CMD_PLAN_SWITCH_NIC, "RETRY_STATE_CMD_PLAN_SWITCH_NIC"},
130 : {RETRY_STATE_SERVER_WAIT_RESUME, "RETRY_STATE_SERVER_WAIT_RESUME"},
131 : {RETRY_RESUME_STATE_SERVER_CHECK_LINK, "RETRY_RESUME_STATE_SERVER_CHECK_LINK"},
132 : {RETRY_RESUME_STATE_SERVER_CHANGE_LINK, "RETRY_RESUME_STATE_SERVER_CHANGE_LINK"},
133 :
134 : // agent状态
135 : {RETRY_STATE_AGENT_RUNNING, "RETRY_STATE_AGENT_RUNNING"},
136 : {RETRY_STATE_RESP_AICPU_ERR, "RETRY_STATE_RESP_AICPU_ERR"},
137 : {RETRY_STATE_RESP_LINK_CHECKED, "RETRY_STATE_RESP_LINK_CHECKED"},
138 : {RETRY_STATE_RESP_LINK_CHANGED, "RETRY_STATE_RESP_LINK_CHANGED"},
139 : {RETRY_STATE_POLL_AICPU_CHANGED, "RETRY_STATE_POLL_AICPU_CHANGED"},
140 : {RETRY_STATE_WAIT_CMD_STOP_AICPU, "RETRY_STATE_WAIT_CMD_STOP_AICPU"},
141 : {RETRY_STATE_POLL_AICPU_STOPED, "RETRY_STATE_POLL_AICPU_STOPED"},
142 : {RETRY_STATE_RESP_AICPU_STOPED, "RETRY_STATE_RESP_AICPU_STOPED"},
143 : {RETRY_STATE_WAIT_CMD_STOP_STREAM, "RETRY_STATE_WAIT_CMD_STOP_STREAM"},
144 : {RETRY_STATE_POLL_STREAM_STOPED, "RETRY_STATE_POLL_STREAM_STOPED"},
145 : {RETRY_STATE_RESP_STREAM_STOPED, "RETRY_STATE_RESP_STREAM_STOPED"},
146 : {RETRY_STATE_WAIT_CMD_CLEAR_STREAM, "RETRY_STATE_WAIT_CMD_CLEAR_STREAM"},
147 : {RETRY_STATE_WAIT_CHANGE_LINK_INFO, "RETRY_STATE_WAIT_CHANGE_LINK_INFO"},
148 : {RETRY_STATE_RESP_STREAM_CLEARED, "RETRY_STATE_RESP_STREAM_CLEARED"},
149 : {RETRY_STATE_WAIT_CMD_STOP_TRANSPORT, "RETRY_STATE_WAIT_CMD_STOP_TRANSPORT"},
150 : {RETRY_STATE_RESP_STOP_TRANSPORT, "RETRY_STATE_RESP_STOP_TRANSPORT"},
151 : {RETRY_STATE_WAIT_CMD_RESET_NOTIFY, "RETRY_STATE_WAIT_CMD_RESET_NOTIFY"},
152 : {RETRY_STATE_RESP_NOTIFY_RESETED, "RETRY_STATE_RESP_NOTIFY_RESETED"},
153 : {RETRY_STATE_WAIT_CMD_RESUME_TRANSPORT, "RETRY_STATE_WAIT_CMD_RESUME_TRANSPORT"},
154 : {RETRY_STATE_RESP_RESUME_TRANSPORT, "RETRY_STATE_RESP_RESUME_TRANSPORT"},
155 : {RETRY_STATE_WAIT_CMD_CHECK, "RETRY_STATE_WAIT_CMD_CHECK"},
156 : {RETRY_STATE_RESP_CHECK_INFO, "RETRY_STATE_RESP_CHECK_INFO"},
157 : {RETRY_STATE_WAIT_CMD_CAN_RETRY, "RETRY_STATE_WAIT_CMD_CAN_RETRY"},
158 : {RETRY_STATE_POLL_AICPU_RETRYEND, "RETRY_STATE_POLL_AICPU_RETRYEND"},
159 : {RETRY_STATE_RESP_AICPU_RETRYEND, "RETRY_STATE_RESP_AICPU_RETRYEND"},
160 : {RETRY_STATE_RESP_RUNNING_ERR, "RETRY_STATE_RESP_RUNNING_ERR"},
161 : {RETRY_STATE_WAIT_CMD_RETRY_FAIL, "RETRY_STATE_WAIT_CMD_RETRY_FAIL"},
162 : {RETRY_STATE_AGENT_RETRY_FAIL, "RETRY_STATE_AGENT_RETRY_FAIL"},
163 : {RETRY_STATE_SEND_SWITCH_INFO, "RETRY_STATE_SEND_SWITCH_INFO"},
164 : {RETRY_STATE_WAIT_CMD_SEND_AICPU, "RETRY_STATE_WAIT_CMD_SEND_AICPU"},
165 : {RETRY_STATE_AGENT_WAIT_RESUME, "RETRY_STATE_AGENT_WAIT_RESUME"},
166 : {RETRY_RESUME_STATE_AGENT_CHECK_LINK, "RETRY_RESUME_STATE_AGENT_CHECK_LINK"},
167 : {RETRY_RESUME_STATE_AGENT_CHANGE_LINK, "RETRY_RESUME_STATE_AGENT_CHANGE_LINK"},
168 :
169 : {RETRY_STATE_RESERVED, "RETRY_STATE_RESERVED"}};
170 :
171 : typedef enum {
172 : RETRY_CMD_RUNNING = 0, // 正常运行
173 : RETRY_CMD_CHECK_LINK,
174 : RETRY_CMD_STOP_AICPU,
175 : RETRY_CMD_STOP_STREAM,
176 : RETRY_CMD_CLEAR_STREAM,
177 : RETRY_CMD_STOP_TRANSPORT,
178 : RETRY_CMD_RESET_NOTIFY,
179 : RETRY_CMD_RESUME_TRANSPORT,
180 : RETRY_CMD_CHECK_OPNAME,
181 : RETRY_CMD_CAN_RETRY,
182 : RETRY_CMD_RETRY_FAIL,
183 : RETRY_CMD_NOTIFY_SWITCH_SUC,
184 : RETRY_CMD_NOTIFY_SWITCH_FAIL,
185 : RETRY_CMD_RESERVED,
186 : RETRY_CMD_RETRY_CONSTRAINT_FAIL, // 当前需要上报故障的重执行约束：inplace约束、算子不一致
187 : RESUME_CMD_CHECK_LINK,
188 : RESUME_CMD_RUNNING,
189 : } RetryCommand;
190 :
191 : const std::map<RetryCommand, std::string> RETRY_COMMAND_STR_MAP{
192 : {RETRY_CMD_RUNNING, "RETRY_CMD_RUNNING"},
193 : {RETRY_CMD_CHECK_LINK, "RETRY_CMD_CHECK_LINK"},
194 : {RETRY_CMD_STOP_AICPU, "RETRY_CMD_STOP_AICPU"},
195 : {RETRY_CMD_STOP_STREAM, "RETRY_CMD_STOP_STREAM"},
196 : {RETRY_CMD_CLEAR_STREAM, "RETRY_CMD_CLEAR_STREAM"},
197 : {RETRY_CMD_STOP_TRANSPORT, "RETRY_CMD_STOP_TRANSPORT"},
198 : {RETRY_CMD_RESET_NOTIFY, "RETRY_CMD_RESET_NOTIFY"},
199 : {RETRY_CMD_RESUME_TRANSPORT, "RETRY_CMD_RESUME_TRANSPORT"},
200 : {RETRY_CMD_CHECK_OPNAME, "RETRY_CMD_CHECK_OPNAME"},
201 : {RETRY_CMD_CAN_RETRY, "RETRY_CMD_CAN_RETRY"},
202 : {RETRY_CMD_RETRY_FAIL, "RETRY_CMD_RETRY_FAIL"},
203 : {RETRY_CMD_NOTIFY_SWITCH_SUC, "RETRY_CMD_NOTIFY_SWITCH_SUC"},
204 : {RETRY_CMD_NOTIFY_SWITCH_FAIL, "RETRY_CMD_NOTIFY_SWITCH_FAIL"},
205 : {RETRY_CMD_RESERVED, "RETRY_CMD_RESERVED"},
206 : {RETRY_CMD_RETRY_CONSTRAINT_FAIL, "RETRY_CMD_RETRY_CONSTRAINT_FAIL"},
207 : {RESUME_CMD_CHECK_LINK, "RESUME_CMD_CHECK_LINK"},
208 : {RESUME_CMD_RUNNING, "RESUME_CMD_RUNNING"},
209 : };
210 :
211 : // server状态机 WaitResp状态对应的agent状态
212 : const std::map<RetryState, RetryState> RETRY_SERVER_WAIT_AGENT_STATE_LABEL{
213 : {RETRY_STATE_WAIT_LINK_CHECKED, RETRY_STATE_RESP_LINK_CHECKED},
214 : {RETRY_STATE_WAIT_AICPU_STOPED, RETRY_STATE_RESP_AICPU_STOPED},
215 : {RETRY_STATE_WAIT_STREAM_STOPED, RETRY_STATE_RESP_STREAM_STOPED},
216 : {RETRY_STATE_WAIT_STREAM_CLEARED, RETRY_STATE_RESP_STREAM_CLEARED},
217 : {RETRY_STATE_WAIT_LINK_CHANGED, RETRY_STATE_RESP_LINK_CHANGED},
218 : {RETRY_STATE_WAIT_STOP_TRANSPORT, RETRY_STATE_RESP_STOP_TRANSPORT},
219 : {RETRY_STATE_WAIT_NOTIFY_RESETED, RETRY_STATE_RESP_NOTIFY_RESETED},
220 : {RETRY_STATE_WAIT_RESUME_TRANSPORT, RETRY_STATE_RESP_RESUME_TRANSPORT},
221 : {RETRY_STATE_WAIT_CHECK_INFO, RETRY_STATE_RESP_CHECK_INFO},
222 : {RETRY_STATE_WAIT_CAN_RETRY, RETRY_STATE_RESP_AICPU_RETRYEND},
223 : };
224 : struct OpRetryServerInfo {
225 : HcclIpAddress hostIP; // root节点的hostIP
226 : u32 hostPort; // root节点的hostPort
227 : s32 devId; // devicePhyId
228 : };
229 3 : struct OpRetryAgentInfo {
230 : u32 userRank; // 本group中的userrank
231 : s32 deviceLogicId;
232 : HcclIpAddress hostIP; // 当前rank对应的hostIP
233 : HcclIpAddress deviceIP; // 当前rank对应的deviceIP
234 : HcclNetDevCtx netDevCtx;
235 : HcclNetDevCtx backUpNetDevCtx;
236 : };
237 :
238 : typedef enum {
239 : RETRY_INFO_CMD = 1,
240 : RETRY_COMMAND_INFO_CMD,
241 : LINK_PORT_STATUS_CMD,
242 : ACTIVE_SWITCH_INFO_CMD,
243 : } RetryStructCommand;
244 :
245 4 : struct RetryInfo {
246 : u32 cmd = RETRY_INFO_CMD;
247 : u32 rankId = 0;
248 : RetryState retryState = RETRY_STATE_RESERVED; // 重执行状态机当前状态
249 : bool linkState = true; // 预留, link状态
250 : KfcExecStatus opInfo;
251 : bool isChangeLinkFlag = false; // 当前是否为借轨
252 : char dfxIpInfo[OPRETRY_DFX_IPINFO_LENGTH] = {0}; // 重执行状态机维测信息（deviceIP + hostIP）
253 : bool isNeedReportOpRetryErr = false; // 针对重执行算子不一致和inplace场景，上报故障
254 : };
255 2 : struct RetryCommandInfo {
256 : u32 cmd = RETRY_COMMAND_INFO_CMD;
257 : RetryCommand command;
258 : HcclOpIdentifier opId;
259 : };
260 :
261 : /* 重执行agent状态机使用 */
262 : using HcclOpStreamRes = std::map<std::string, std::vector<Stream>>;
263 : using OpRetryResetNotifyCallback = std::function<HcclResult(bool, s64)>;
264 : using OpRetrySetTransportStatusCallback = std::function<HcclResult(
265 : const HcclOpIdentifier&, bool, const std::map<u32, bool>&, const std::map<u32, bool>&, bool)>;
266 : using OpRetryGetSwitchRanksCallback = std::function<HcclResult(u32*, bool*, u32&, u8*, u32&, bool&, bool&)>;
267 : using OpRetrySetTransportResumeStatusCallBack
268 : = std::function<HcclResult(const std::map<u32, bool>&, const std::map<u32, bool>&, bool, bool)>;
269 : } // namespace hccl
270 : #endif
|
