Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #ifndef HCCL_OP_RETRY_PUB_H
12 : #define HCCL_OP_RETRY_PUB_H
13 :
14 : #include <functional>
15 : #include "hccl/base.h"
16 : #include "hccl_common.h"
17 : #include "stream_pub.h"
18 : #include "hccl_socket.h"
19 : #include "hdc_pub.h"
20 : #include "notify_pool.h"
21 : #include "aicpu_operator_pub.h"
22 :
23 : namespace hccl {
24 :
25 : constexpr u32 OPRETRY_DFX_IPINFO_LENGTH = 256;
26 : constexpr u32 CONNECT_REMOTE_DEFAULT = 1;
27 : constexpr u32 CONNECT_REMOTE_BACKUP = 2;
28 : constexpr u32 ACTIVE_SWITCH_TIMES = 2;
29 :
30 : typedef enum {
31 : // server状态
32 : RETRY_STATE_SERVER_RUNNING = 0,
33 : RETRY_STATE_CMD_CHECK_LINK,
34 : RETRY_STATE_WAIT_LINK_CHECKED,
35 : RETRY_STATE_CHECK_ALL_LINK,
36 : RETRY_STATE_CMD_CHANGE_LINK,
37 : RETRY_STATE_WAIT_LINK_CHANGED,
38 : RETRY_STETA_HANDLE_ALL_ERR,
39 : RETRY_STATE_CMD_STOP_AICPU, // 发送StopAicpu命令
40 : RETRY_STATE_WAIT_AICPU_STOPED, // 等待Aicpu停止
41 : RETRY_STATE_CMD_STOP_STREAM,
42 : RETRY_STATE_WAIT_STREAM_STOPED,
43 : RETRY_STATE_CMD_CLEAR_STREAM,
44 : RETRY_STATE_WAIT_STREAM_CLEARED,
45 : RETRY_STATE_CMD_STOP_TRANSPORT,
46 : RETRY_STATE_WAIT_STOP_TRANSPORT,
47 : RETRY_STATE_CMD_RESET_NOTIFY,
48 : RETRY_STATE_WAIT_NOTIFY_RESETED,
49 : RETRY_STATE_CMD_RESUME_TRANSPORT,
50 : RETRY_STATE_WAIT_RESUME_TRANSPORT,
51 : RETRY_STATE_CMD_CHECK,
52 : RETRY_STATE_WAIT_CHECK_INFO,
53 : RETRY_STATE_CHECK_OP,
54 : RETRY_STATE_CMD_CAN_RETRY,
55 : RETRY_STATE_WAIT_CAN_RETRY,
56 : RETRY_STATE_SERVER_RETRY_FAIL,
57 : RETRY_STATE_CMD_PLAN_SWITCH_NIC,
58 : RETRY_STATE_SERVER_WAIT_RESUME,
59 : RETRY_RESUME_STATE_SERVER_CHECK_LINK,
60 : RETRY_RESUME_STATE_SERVER_CHANGE_LINK,
61 :
62 : // agent状态
63 : RETRY_STATE_AGENT_RUNNING,
64 : RETRY_STATE_RESP_AICPU_ERR,
65 : RETRY_STATE_RESP_LINK_CHECKED,
66 : RETRY_STATE_WAIT_CMD_STOP_AICPU,
67 : RETRY_STATE_POLL_AICPU_STOPED,
68 : RETRY_STATE_RESP_AICPU_STOPED,
69 : RETRY_STATE_WAIT_CMD_STOP_STREAM,
70 : RETRY_STATE_POLL_STREAM_STOPED,
71 : RETRY_STATE_RESP_STREAM_STOPED,
72 : RETRY_STATE_WAIT_CMD_CLEAR_STREAM,
73 : RETRY_STATE_RESP_STREAM_CLEARED,
74 : RETRY_STATE_RESP_LINK_CHANGED,
75 : RETRY_STATE_WAIT_CHANGE_LINK_INFO,
76 : RETRY_STATE_POLL_AICPU_CHANGED,
77 : RETRY_STATE_WAIT_CMD_STOP_TRANSPORT,
78 : RETRY_STATE_RESP_STOP_TRANSPORT,
79 : RETRY_STATE_WAIT_CMD_RESET_NOTIFY,
80 : RETRY_STATE_RESP_NOTIFY_RESETED,
81 : RETRY_STATE_WAIT_CMD_CHECK_LINK,
82 : RETRY_STATE_WAIT_CMD_RESUME_TRANSPORT,
83 : RETRY_STATE_RESP_RESUME_TRANSPORT,
84 : RETRY_STATE_WAIT_CMD_CHECK,
85 : RETRY_STATE_RESP_CHECK_INFO,
86 : RETRY_STATE_WAIT_CMD_CAN_RETRY,
87 : RETRY_STATE_POLL_AICPU_RETRYEND,
88 : RETRY_STATE_RESP_AICPU_RETRYEND,
89 : RETRY_STATE_RESP_RUNNING_ERR,
90 : RETRY_STATE_WAIT_CMD_RETRY_FAIL,
91 : RETRY_STATE_AGENT_RETRY_FAIL,
92 : RETRY_STATE_SEND_SWITCH_INFO,
93 : RETRY_STATE_WAIT_CMD_SEND_AICPU,
94 : RETRY_STATE_AGENT_WAIT_RESUME,
95 : RETRY_RESUME_STATE_AGENT_CHECK_LINK,
96 : RETRY_RESUME_STATE_AGENT_CHANGE_LINK,
97 :
98 : RETRY_STATE_RESERVED,
99 : } RetryState;
100 :
101 : const std::map<RetryState, std::string> RETRY_STATE_STR_MAP {
102 : // server状态
103 : {RETRY_STATE_SERVER_RUNNING, "RETRY_STATE_SERVER_RUNNING"},
104 : {RETRY_STATE_CMD_CHECK_LINK, "RETRY_STATE_CMD_CHECK_LINK"},
105 : {RETRY_STATE_CMD_CHANGE_LINK, "RETRY_STATE_CMD_CHANGE_LINK"},
106 : {RETRY_STATE_WAIT_LINK_CHANGED, "RETRY_STATE_WAIT_LINK_CHANGED"},
107 : {RETRY_STATE_WAIT_LINK_CHECKED, "RETRY_STATE_WAIT_LINK_CHECKED"},
108 : {RETRY_STATE_CHECK_ALL_LINK, "RETRY_STATE_CHECK_ALL_LINK"},
109 : {RETRY_STETA_HANDLE_ALL_ERR, "RETRY_STETA_HANDLE_ALL_ERR"},
110 : {RETRY_STATE_CMD_STOP_AICPU, "RETRY_STATE_CMD_STOP_AICPU"},
111 : {RETRY_STATE_WAIT_AICPU_STOPED, "RETRY_STATE_WAIT_AICPU_STOPED"},
112 : {RETRY_STATE_CMD_STOP_STREAM, "RETRY_STATE_CMD_STOP_STREAM"},
113 : {RETRY_STATE_WAIT_STREAM_STOPED, "RETRY_STATE_WAIT_STREAM_STOPED"},
114 : {RETRY_STATE_CMD_CLEAR_STREAM, "RETRY_STATE_CMD_CLEAR_STREAM"},
115 : {RETRY_STATE_WAIT_STREAM_CLEARED, "RETRY_STATE_WAIT_STREAM_CLEARED"},
116 : {RETRY_STATE_CMD_RESET_NOTIFY, "RETRY_STATE_CMD_RESET_NOTIFY"},
117 : {RETRY_STATE_CMD_STOP_TRANSPORT, "RETRY_STATE_CMD_STOP_TRANSPORT"},
118 : {RETRY_STATE_WAIT_STOP_TRANSPORT, "RETRY_STATE_WAIT_STOP_TRANSPORT"},
119 : {RETRY_STATE_WAIT_NOTIFY_RESETED, "RETRY_STATE_WAIT_NOTIFY_RESETED"},
120 : {RETRY_STATE_WAIT_CMD_CHECK_LINK, "RETRY_STATE_WAIT_CMD_CHECK_LINK"},
121 : {RETRY_STATE_CMD_RESUME_TRANSPORT, "RETRY_STATE_CMD_RESUME_TRANSPORT"},
122 : {RETRY_STATE_WAIT_RESUME_TRANSPORT, "RETRY_STATE_WAIT_RESUME_TRANSPORT"},
123 : {RETRY_STATE_CMD_CHECK, "RETRY_STATE_CMD_CHECK"},
124 : {RETRY_STATE_WAIT_CHECK_INFO, "RETRY_STATE_WAIT_CHECK_INFO"},
125 : {RETRY_STATE_CHECK_OP, "RETRY_STATE_CHECK_OP"},
126 : {RETRY_STATE_CMD_CAN_RETRY, "RETRY_STATE_CMD_CAN_RETRY"},
127 : {RETRY_STATE_WAIT_CAN_RETRY, "RETRY_STATE_WAIT_CAN_RETRY"},
128 : {RETRY_STATE_SERVER_RETRY_FAIL, "RETRY_STATE_SERVER_RETRY_FAIL"},
129 : {RETRY_STATE_CMD_PLAN_SWITCH_NIC, "RETRY_STATE_CMD_PLAN_SWITCH_NIC"},
130 : {RETRY_STATE_SERVER_WAIT_RESUME, "RETRY_STATE_SERVER_WAIT_RESUME"},
131 : {RETRY_RESUME_STATE_SERVER_CHECK_LINK, "RETRY_RESUME_STATE_SERVER_CHECK_LINK"},
132 : {RETRY_RESUME_STATE_SERVER_CHANGE_LINK, "RETRY_RESUME_STATE_SERVER_CHANGE_LINK"},
133 :
134 : // agent状态
135 : {RETRY_STATE_AGENT_RUNNING, "RETRY_STATE_AGENT_RUNNING"},
136 : {RETRY_STATE_RESP_AICPU_ERR, "RETRY_STATE_RESP_AICPU_ERR"},
137 : {RETRY_STATE_RESP_LINK_CHECKED, "RETRY_STATE_RESP_LINK_CHECKED"},
138 : {RETRY_STATE_RESP_LINK_CHANGED, "RETRY_STATE_RESP_LINK_CHANGED"},
139 : {RETRY_STATE_POLL_AICPU_CHANGED, "RETRY_STATE_POLL_AICPU_CHANGED"},
140 : {RETRY_STATE_WAIT_CMD_STOP_AICPU, "RETRY_STATE_WAIT_CMD_STOP_AICPU"},
141 : {RETRY_STATE_POLL_AICPU_STOPED, "RETRY_STATE_POLL_AICPU_STOPED"},
142 : {RETRY_STATE_RESP_AICPU_STOPED, "RETRY_STATE_RESP_AICPU_STOPED"},
143 : {RETRY_STATE_WAIT_CMD_STOP_STREAM, "RETRY_STATE_WAIT_CMD_STOP_STREAM"},
144 : {RETRY_STATE_POLL_STREAM_STOPED, "RETRY_STATE_POLL_STREAM_STOPED"},
145 : {RETRY_STATE_RESP_STREAM_STOPED, "RETRY_STATE_RESP_STREAM_STOPED"},
146 : {RETRY_STATE_WAIT_CMD_CLEAR_STREAM, "RETRY_STATE_WAIT_CMD_CLEAR_STREAM"},
147 : {RETRY_STATE_WAIT_CHANGE_LINK_INFO, "RETRY_STATE_WAIT_CHANGE_LINK_INFO"},
148 : {RETRY_STATE_RESP_STREAM_CLEARED, "RETRY_STATE_RESP_STREAM_CLEARED"},
149 : {RETRY_STATE_WAIT_CMD_STOP_TRANSPORT, "RETRY_STATE_WAIT_CMD_STOP_TRANSPORT"},
150 : {RETRY_STATE_RESP_STOP_TRANSPORT, "RETRY_STATE_RESP_STOP_TRANSPORT"},
151 : {RETRY_STATE_WAIT_CMD_RESET_NOTIFY, "RETRY_STATE_WAIT_CMD_RESET_NOTIFY"},
152 : {RETRY_STATE_RESP_NOTIFY_RESETED, "RETRY_STATE_RESP_NOTIFY_RESETED"},
153 : {RETRY_STATE_WAIT_CMD_RESUME_TRANSPORT, "RETRY_STATE_WAIT_CMD_RESUME_TRANSPORT"},
154 : {RETRY_STATE_RESP_RESUME_TRANSPORT, "RETRY_STATE_RESP_RESUME_TRANSPORT"},
155 : {RETRY_STATE_WAIT_CMD_CHECK, "RETRY_STATE_WAIT_CMD_CHECK"},
156 : {RETRY_STATE_RESP_CHECK_INFO, "RETRY_STATE_RESP_CHECK_INFO"},
157 : {RETRY_STATE_WAIT_CMD_CAN_RETRY, "RETRY_STATE_WAIT_CMD_CAN_RETRY"},
158 : {RETRY_STATE_POLL_AICPU_RETRYEND, "RETRY_STATE_POLL_AICPU_RETRYEND"},
159 : {RETRY_STATE_RESP_AICPU_RETRYEND, "RETRY_STATE_RESP_AICPU_RETRYEND"},
160 : {RETRY_STATE_RESP_RUNNING_ERR, "RETRY_STATE_RESP_RUNNING_ERR"},
161 : {RETRY_STATE_WAIT_CMD_RETRY_FAIL, "RETRY_STATE_WAIT_CMD_RETRY_FAIL"},
162 : {RETRY_STATE_AGENT_RETRY_FAIL, "RETRY_STATE_AGENT_RETRY_FAIL"},
163 : {RETRY_STATE_SEND_SWITCH_INFO, "RETRY_STATE_SEND_SWITCH_INFO"},
164 : {RETRY_STATE_WAIT_CMD_SEND_AICPU, "RETRY_STATE_WAIT_CMD_SEND_AICPU"},
165 : {RETRY_STATE_AGENT_WAIT_RESUME, "RETRY_STATE_AGENT_WAIT_RESUME"},
166 : {RETRY_RESUME_STATE_AGENT_CHECK_LINK, "RETRY_RESUME_STATE_AGENT_CHECK_LINK"},
167 : {RETRY_RESUME_STATE_AGENT_CHANGE_LINK, "RETRY_RESUME_STATE_AGENT_CHANGE_LINK"},
168 :
169 : {RETRY_STATE_RESERVED, "RETRY_STATE_RESERVED"}
170 : };
171 :
172 : typedef enum {
173 : RETRY_CMD_RUNNING = 0, // 正常运行
174 : RETRY_CMD_CHECK_LINK,
175 : RETRY_CMD_STOP_AICPU,
176 : RETRY_CMD_STOP_STREAM,
177 : RETRY_CMD_CLEAR_STREAM,
178 : RETRY_CMD_STOP_TRANSPORT,
179 : RETRY_CMD_RESET_NOTIFY,
180 : RETRY_CMD_RESUME_TRANSPORT,
181 : RETRY_CMD_CHECK_OPNAME,
182 : RETRY_CMD_CAN_RETRY,
183 : RETRY_CMD_RETRY_FAIL,
184 : RETRY_CMD_NOTIFY_SWITCH_SUC,
185 : RETRY_CMD_NOTIFY_SWITCH_FAIL,
186 : RETRY_CMD_RESERVED,
187 : RETRY_CMD_RETRY_CONSTRAINT_FAIL, // 当前需要上报故障的重执行约束：inplace约束、算子不一致
188 : RESUME_CMD_CHECK_LINK,
189 : RESUME_CMD_RUNNING,
190 : } RetryCommand;
191 :
192 : const std::map<RetryCommand, std::string> RETRY_COMMAND_STR_MAP {
193 : {RETRY_CMD_RUNNING, "RETRY_CMD_RUNNING"},
194 : {RETRY_CMD_CHECK_LINK, "RETRY_CMD_CHECK_LINK"},
195 : {RETRY_CMD_STOP_AICPU, "RETRY_CMD_STOP_AICPU"},
196 : {RETRY_CMD_STOP_STREAM, "RETRY_CMD_STOP_STREAM"},
197 : {RETRY_CMD_CLEAR_STREAM, "RETRY_CMD_CLEAR_STREAM"},
198 : {RETRY_CMD_STOP_TRANSPORT, "RETRY_CMD_STOP_TRANSPORT"},
199 : {RETRY_CMD_RESET_NOTIFY, "RETRY_CMD_RESET_NOTIFY"},
200 : {RETRY_CMD_RESUME_TRANSPORT, "RETRY_CMD_RESUME_TRANSPORT"},
201 : {RETRY_CMD_CHECK_OPNAME, "RETRY_CMD_CHECK_OPNAME"},
202 : {RETRY_CMD_CAN_RETRY, "RETRY_CMD_CAN_RETRY"},
203 : {RETRY_CMD_RETRY_FAIL, "RETRY_CMD_RETRY_FAIL"},
204 : {RETRY_CMD_NOTIFY_SWITCH_SUC, "RETRY_CMD_NOTIFY_SWITCH_SUC"},
205 : {RETRY_CMD_NOTIFY_SWITCH_FAIL, "RETRY_CMD_NOTIFY_SWITCH_FAIL"},
206 : {RETRY_CMD_RESERVED, "RETRY_CMD_RESERVED"},
207 : {RETRY_CMD_RETRY_CONSTRAINT_FAIL, "RETRY_CMD_RETRY_CONSTRAINT_FAIL"},
208 : {RESUME_CMD_CHECK_LINK, "RESUME_CMD_CHECK_LINK"},
209 : {RESUME_CMD_RUNNING, "RESUME_CMD_RUNNING"},
210 : };
211 :
212 : // server状态机 WaitResp状态对应的agent状态
213 : const std::map<RetryState, RetryState> RETRY_SERVER_WAIT_AGENT_STATE_LABEL {
214 : {RETRY_STATE_WAIT_LINK_CHECKED, RETRY_STATE_RESP_LINK_CHECKED},
215 : {RETRY_STATE_WAIT_AICPU_STOPED, RETRY_STATE_RESP_AICPU_STOPED},
216 : {RETRY_STATE_WAIT_STREAM_STOPED, RETRY_STATE_RESP_STREAM_STOPED},
217 : {RETRY_STATE_WAIT_STREAM_CLEARED, RETRY_STATE_RESP_STREAM_CLEARED},
218 : {RETRY_STATE_WAIT_LINK_CHANGED, RETRY_STATE_RESP_LINK_CHANGED},
219 : {RETRY_STATE_WAIT_STOP_TRANSPORT, RETRY_STATE_RESP_STOP_TRANSPORT},
220 : {RETRY_STATE_WAIT_NOTIFY_RESETED, RETRY_STATE_RESP_NOTIFY_RESETED},
221 : {RETRY_STATE_WAIT_RESUME_TRANSPORT, RETRY_STATE_RESP_RESUME_TRANSPORT},
222 : {RETRY_STATE_WAIT_CHECK_INFO, RETRY_STATE_RESP_CHECK_INFO},
223 : {RETRY_STATE_WAIT_CAN_RETRY, RETRY_STATE_RESP_AICPU_RETRYEND},
224 : };
225 : struct OpRetryServerInfo {
226 : HcclIpAddress hostIP; // root节点的hostIP
227 : u32 hostPort; // root节点的hostPort
228 : s32 devId; // devicePhyId
229 : };
230 3 : struct OpRetryAgentInfo {
231 : u32 userRank; // 本group中的userrank
232 : s32 deviceLogicId;
233 : HcclIpAddress hostIP; // 当前rank对应的hostIP
234 : HcclIpAddress deviceIP; // 当前rank对应的deviceIP
235 : HcclNetDevCtx netDevCtx;
236 : HcclNetDevCtx backUpNetDevCtx;
237 : };
238 :
239 : typedef enum {
240 : RETRY_INFO_CMD = 1,
241 : RETRY_COMMAND_INFO_CMD,
242 : LINK_PORT_STATUS_CMD,
243 : ACTIVE_SWITCH_INFO_CMD,
244 : } RetryStructCommand;
245 :
246 4 : struct RetryInfo {
247 : u32 cmd = RETRY_INFO_CMD;
248 : u32 rankId = 0;
249 : RetryState retryState = RETRY_STATE_RESERVED; // 重执行状态机当前状态
250 : bool linkState = true; // 预留, link状态
251 : KfcExecStatus opInfo;
252 : bool isChangeLinkFlag = false; // 当前是否为借轨
253 : char dfxIpInfo[OPRETRY_DFX_IPINFO_LENGTH] = {0}; // 重执行状态机维测信息（deviceIP + hostIP）
254 : bool isNeedReportOpRetryErr = false; // 针对重执行算子不一致和inplace场景，上报故障
255 : };
256 2 : struct RetryCommandInfo{
257 : u32 cmd = RETRY_COMMAND_INFO_CMD;
258 : RetryCommand command;
259 : HcclOpIdentifier opId;
260 : };
261 :
262 : /* 重执行agent状态机使用 */
263 : using HcclOpStreamRes = std::map<std::string, std::vector<Stream> >;
264 : using OpRetryResetNotifyCallback = std::function<HcclResult(bool, s64)>;
265 : using OpRetrySetTransportStatusCallback = std::function<HcclResult(const HcclOpIdentifier &, bool,
266 : const std::map<u32, bool> &, const std::map<u32, bool> &, bool)>;
267 : using OpRetryGetSwitchRanksCallback = std::function<HcclResult(u32 *, bool*, u32 &, u8 *, u32 &, bool &, bool &)>;
268 : using OpRetrySetTransportResumeStatusCallBack = std::function<HcclResult(const std::map<u32, bool> &, const std::map<u32, bool> &, bool, bool)>;
269 : }
270 : #endif
|
