Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "aicpu_kfc_retry_process.h"
12 :
13 : #include "aicpu_hccl_def.h"
14 : #include "framework/aicpu_communicator.h"
15 :
16 : using namespace hccl;
17 :
18 : ANONYMOUS_NAMESPACE_BEGIN
19 8 : HcclResult MC2OpExecFsmStoppingProcess(HcclCommAicpu &comm, HcclOpExecFSM &state, KfcError &errorCode)
20 : {
21 8 : KfcCommand cmd = KfcCommand::kNone;
22 8 : auto ret = comm.BackGroundGetCmd(cmd);
23 8 : if (ret != HCCL_SUCCESS) {
24 1 : HCCL_ERROR("MC2 restart GetOpExecCtrlCmd failed, ret:%u", ret);
25 1 : errorCode = KfcError::kExec;
26 1 : state = HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR;
27 1 : return ret;
28 : }
29 :
30 7 : if (cmd == KfcCommand::kExit) {
31 1 : HCCL_ERROR("MC2 restart aicpu exec fsm stop by exit cmd.");
32 1 : errorCode = KfcError::kExit;
33 1 : state = HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR;
34 6 : } else if (cmd == KfcCommand::kStopExec) {
35 2 : HCCL_DEBUG("MC2 restart MC2 aicpu get stop exec cmd.");
36 2 : state = HcclOpExecFSM::HCCL_OP_EXEC_FSM_STOPPED;
37 4 : } else if (cmd == KfcCommand::kStopLaunch) {
38 : // do nothing
39 3 : } else if ((cmd == KfcCommand::kNone) || (cmd == KfcCommand::kRetry)) {
40 : // do nothing
41 : } else {
42 1 : HCCL_ERROR("MC2 restart GetOpExecCtrlCmd failed, invalid cmd[%u]", cmd);
43 1 : errorCode = KfcError::kExec;
44 1 : state = HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR;
45 : }
46 7 : return HCCL_SUCCESS;
47 : }
48 :
49 4 : HcclResult MC2OpExecFsmStoppedProcess(HcclCommAicpu &comm, HcclOpExecFSM &state, KfcError &errorCode)
50 : {
51 4 : KfcCommand cmd = KfcCommand::kNone;
52 4 : auto ret = comm.BackGroundGetCmd(cmd);
53 4 : if (ret != HCCL_SUCCESS) {
54 1 : HCCL_ERROR("MC2 restart GetOpExecCtrlCmd failed, ret:%u", ret);
55 1 : errorCode = KfcError::kExec;
56 1 : state = HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR;
57 1 : return ret;
58 : }
59 :
60 3 : if (cmd == KfcCommand::kExit) {
61 1 : HCCL_ERROR("MC2 restart hccl aicpu exec fsm stop by exit cmd.");
62 1 : errorCode = KfcError::kExit;
63 1 : state = HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR;
64 : } else {
65 2 : errorCode = KfcError::kNone;
66 2 : state = HcclOpExecFSM::HCCL_OP_EXEC_FSM_WAIT_RETRY;
67 : }
68 3 : return HCCL_SUCCESS;
69 : }
70 :
71 7 : HcclResult MC2OpExecFsmWaitRetryProcess(HcclCommAicpu &comm, HcclOpExecFSM &state, KfcError &errorCode,
72 : bool linkChanged)
73 : {
74 7 : KfcCommand cmd = KfcCommand::kNone;
75 7 : auto ret = comm.BackGroundGetCmd(cmd);
76 7 : if (ret != HCCL_SUCCESS) {
77 1 : HCCL_ERROR("MC2 restart GetOpExecCtrlCmd failed, ret:%u", ret);
78 1 : errorCode = KfcError::kExec;
79 1 : state = HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR;
80 1 : return ret;
81 : }
82 :
83 6 : if (cmd == KfcCommand::kRetry) {
84 2 : HCCL_DEBUG("MC2 restart aicpu recv retry cmd from host.");
85 2 : comm.GetDfxExtendInfo()->pollStatus = PollStatus::kDefault;
86 2 : comm.GetDfxExtendInfo()->cqeStatus = dfx::CqeStatus::kDefault;
87 2 : comm.ResetOpRetryException(HcclCMDType::HCCL_CMD_INVALID);
88 2 : state = HcclOpExecFSM::HCCL_OP_EXEC_FSM_RETRY;
89 4 : } else if (cmd == KfcCommand::kChangeLink && !linkChanged) {
90 2 : HCCL_DEBUG("MC2 restart aicpu recv change link cmd");
91 2 : state = HcclOpExecFSM::HCCL_OP_EXEC_FSM_CHANGE_LINK;
92 2 : } else if (cmd == KfcCommand::kExit) {
93 1 : HCCL_ERROR("MC2 restart aicpu recv exit cmd from host.");
94 1 : errorCode = KfcError::kExit;
95 1 : state = HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR;
96 : }
97 6 : return HCCL_SUCCESS;
98 : }
99 : ANONYMOUS_NAMESPACE_END
100 :
101 7 : HcclResult AicpuKfcRetryProcess::RetryProcess(HcclCommAicpu &comm, RestartParam &restartParam, uint32_t idx)
102 : {
103 7 : HcclResult ret = HCCL_SUCCESS;
104 7 : auto waitStopExecCmdTimeoutMs = comm.HcclGetWaitStopExecCmdTimeout();
105 7 : auto waitStopExecCmdTimeout = std::chrono::milliseconds(waitStopExecCmdTimeoutMs);
106 7 : auto waitRetryCmdTimeoutMs = comm.HcclGetWaitRetryCmdTimeout(restartParam.restartCnt);
107 7 : auto waitRetryCmdTimeout = std::chrono::milliseconds(waitRetryCmdTimeoutMs);
108 :
109 7 : switch (restartParam.fsmState[idx]) {
110 1 : case HcclOpExecFSM::HCCL_OP_EXEC_FSM_WAIT_END:
111 1 : HCCL_INFO("MC2 restart state HCCL_OP_EXEC_FSM_WAIT_END");
112 1 : restartParam.errorCode[idx] = KfcError::kSdma;
113 1 : restartParam.fsmState[idx] = HcclOpExecFSM::HCCL_OP_EXEC_FSM_STOPPING;
114 2 : ret = comm.UpdateOpExecStatus(restartParam.fsmState[idx], KfcStatus::kStoplaunch,
115 1 : restartParam.errorCode[idx], restartParam.restartCnt); // 上报sdma异常
116 1 : if (restartParam.fsmState[idx] == HcclOpExecFSM::HCCL_OP_EXEC_FSM_STOPPING) {
117 1 : restartParam.startTime[idx] = std::chrono::steady_clock::now();
118 : }
119 1 : break;
120 1 : case HcclOpExecFSM::HCCL_OP_EXEC_FSM_STOPPING:
121 1 : if ((std::chrono::steady_clock::now() - restartParam.startTime[idx]) >= waitStopExecCmdTimeout) {
122 0 : HCCL_ERROR("MC2 restart aicpu wait stop exec timeout[%u ms].",
123 : comm.HcclGetWaitStopExecCmdTimeout());
124 0 : restartParam.errorCode[idx] = KfcError::kTimeout;
125 0 : restartParam.fsmState[idx] = HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR;
126 : } else {
127 1 : ret = MC2OpExecFsmStoppingProcess(comm, restartParam.fsmState[idx], restartParam.errorCode[idx]);
128 : }
129 1 : break;
130 1 : case HcclOpExecFSM::HCCL_OP_EXEC_FSM_STOPPED:
131 1 : HCCL_INFO("MC2 restart state HCCL_OP_EXEC_FSM_STOPPED");
132 1 : ret = MC2OpExecFsmStoppedProcess(comm, restartParam.fsmState[idx], restartParam.errorCode[idx]);
133 1 : if (restartParam.fsmState[idx] == HcclOpExecFSM::HCCL_OP_EXEC_FSM_WAIT_RETRY) {
134 1 : CHK_RET(comm.UpdateOpExecStatus(restartParam.fsmState[idx], KfcStatus::kStopExec,
135 : restartParam.errorCode[idx], restartParam.restartCnt));
136 1 : restartParam.startTime[idx] = std::chrono::steady_clock::now();
137 : }
138 1 : break;
139 1 : case HcclOpExecFSM::HCCL_OP_EXEC_FSM_CHANGE_LINK:
140 1 : HCCL_INFO("MC2 restart state HCCL_OP_EXEC_FSM_CHANGE_LINK");
141 : // MC2重执行，清空所有rdma链接
142 1 : comm.CleanAllRoceResource();
143 1 : restartParam.errorCode[idx] = KfcError::kNone;
144 1 : ret = comm.UpdateOpExecStatus(restartParam.fsmState[idx], KfcStatus::kChanged, restartParam.errorCode[idx],
145 : restartParam.restartCnt);
146 1 : restartParam.linkChanged[idx] = true;
147 1 : restartParam.fsmState[idx] = HcclOpExecFSM::HCCL_OP_EXEC_FSM_WAIT_RETRY;
148 1 : break;
149 2 : case HcclOpExecFSM::HCCL_OP_EXEC_FSM_WAIT_RETRY:
150 2 : if ((std::chrono::steady_clock::now() - restartParam.startTime[idx]) >= waitRetryCmdTimeout) {
151 0 : HCCL_ERROR("MC2 restart aicpu wait retry timeout[%u ms].", waitRetryCmdTimeoutMs);
152 0 : restartParam.errorCode[idx] = KfcError::kTimeout;
153 0 : restartParam.fsmState[idx] = HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR;
154 : } else {
155 2 : ret = MC2OpExecFsmWaitRetryProcess(comm, restartParam.fsmState[idx], restartParam.errorCode[idx],
156 2 : restartParam.linkChanged[idx]);
157 : }
158 2 : break;
159 0 : case HcclOpExecFSM::HCCL_OP_EXEC_FSM_RETRY:
160 0 : HCCL_INFO("MC2 restart state HCCL_OP_EXEC_FSM_RETRY");
161 0 : restartParam.consultationResult[idx] = true;
162 0 : restartParam.errorCode[idx] = KfcError::kNone;
163 0 : comm.UpdateOpExecStatus(restartParam.fsmState[idx], KfcStatus::kEnd, restartParam.errorCode[idx],
164 : restartParam.restartCnt);
165 0 : return HCCL_SUCCESS;
166 1 : case HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR:
167 : default: {
168 1 : HCCL_ERROR("MC2 restart aicpu restart process error.");
169 1 : comm.UpdateOpExecStatus(restartParam.fsmState[idx], KfcStatus::kError, restartParam.errorCode[idx],
170 : restartParam.restartCnt);
171 1 : comm.GetDfxExtendInfo()->kfcStatus = DfxKfcStatus::kOneFinished;
172 1 : return (ret == HCCL_SUCCESS) ? HCCL_E_INTERNAL : ret;
173 : }
174 : }
175 6 : return ret;
176 : }
|
