Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "aicpu_kfc_retry_process.h"
12 :
13 : #include "aicpu_hccl_def.h"
14 : #include "framework/aicpu_communicator.h"
15 :
16 : using namespace hccl;
17 :
18 : ANONYMOUS_NAMESPACE_BEGIN
19 8 : HcclResult MC2OpExecFsmStoppingProcess(HcclCommAicpu& comm, HcclOpExecFSM& state, KfcError& errorCode)
20 : {
21 8 : KfcCommand cmd = KfcCommand::kNone;
22 8 : auto ret = comm.BackGroundGetCmd(cmd);
23 8 : if (ret != HCCL_SUCCESS) {
24 1 : HCCL_ERROR("MC2 restart GetOpExecCtrlCmd failed, ret:%u", ret);
25 1 : errorCode = KfcError::kExec;
26 1 : state = HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR;
27 1 : return ret;
28 : }
29 :
30 7 : if (cmd == KfcCommand::kExit) {
31 1 : HCCL_ERROR("MC2 restart aicpu exec fsm stop by exit cmd.");
32 1 : errorCode = KfcError::kExit;
33 1 : state = HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR;
34 6 : } else if (cmd == KfcCommand::kStopExec) {
35 2 : HCCL_DEBUG("MC2 restart MC2 aicpu get stop exec cmd.");
36 2 : state = HcclOpExecFSM::HCCL_OP_EXEC_FSM_STOPPED;
37 4 : } else if (cmd == KfcCommand::kStopLaunch) {
38 : // do nothing
39 3 : } else if ((cmd == KfcCommand::kNone) || (cmd == KfcCommand::kRetry)) {
40 : // do nothing
41 : } else {
42 1 : HCCL_ERROR("MC2 restart GetOpExecCtrlCmd failed, invalid cmd[%u]", cmd);
43 1 : errorCode = KfcError::kExec;
44 1 : state = HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR;
45 : }
46 7 : return HCCL_SUCCESS;
47 : }
48 :
49 4 : HcclResult MC2OpExecFsmStoppedProcess(HcclCommAicpu& comm, HcclOpExecFSM& state, KfcError& errorCode)
50 : {
51 4 : KfcCommand cmd = KfcCommand::kNone;
52 4 : auto ret = comm.BackGroundGetCmd(cmd);
53 4 : if (ret != HCCL_SUCCESS) {
54 1 : HCCL_ERROR("MC2 restart GetOpExecCtrlCmd failed, ret:%u", ret);
55 1 : errorCode = KfcError::kExec;
56 1 : state = HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR;
57 1 : return ret;
58 : }
59 :
60 3 : if (cmd == KfcCommand::kExit) {
61 1 : HCCL_ERROR("MC2 restart hccl aicpu exec fsm stop by exit cmd.");
62 1 : errorCode = KfcError::kExit;
63 1 : state = HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR;
64 : } else {
65 2 : errorCode = KfcError::kNone;
66 2 : state = HcclOpExecFSM::HCCL_OP_EXEC_FSM_WAIT_RETRY;
67 : }
68 3 : return HCCL_SUCCESS;
69 : }
70 :
71 : HcclResult
72 7 : MC2OpExecFsmWaitRetryProcess(HcclCommAicpu& comm, HcclOpExecFSM& state, KfcError& errorCode, bool linkChanged)
73 : {
74 7 : KfcCommand cmd = KfcCommand::kNone;
75 7 : auto ret = comm.BackGroundGetCmd(cmd);
76 7 : if (ret != HCCL_SUCCESS) {
77 1 : HCCL_ERROR("MC2 restart GetOpExecCtrlCmd failed, ret:%u", ret);
78 1 : errorCode = KfcError::kExec;
79 1 : state = HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR;
80 1 : return ret;
81 : }
82 :
83 6 : if (cmd == KfcCommand::kRetry) {
84 2 : HCCL_DEBUG("MC2 restart aicpu recv retry cmd from host.");
85 2 : comm.GetDfxExtendInfo()->pollStatus = PollStatus::kDefault;
86 2 : comm.GetDfxExtendInfo()->cqeStatus = dfx::CqeStatus::kDefault;
87 2 : comm.ResetOpRetryException(HcclCMDType::HCCL_CMD_INVALID);
88 2 : state = HcclOpExecFSM::HCCL_OP_EXEC_FSM_RETRY;
89 4 : } else if (cmd == KfcCommand::kChangeLink && !linkChanged) {
90 2 : HCCL_DEBUG("MC2 restart aicpu recv change link cmd");
91 2 : state = HcclOpExecFSM::HCCL_OP_EXEC_FSM_CHANGE_LINK;
92 2 : } else if (cmd == KfcCommand::kExit) {
93 1 : HCCL_ERROR("MC2 restart aicpu recv exit cmd from host.");
94 1 : errorCode = KfcError::kExit;
95 1 : state = HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR;
96 : }
97 6 : return HCCL_SUCCESS;
98 : }
99 : ANONYMOUS_NAMESPACE_END
100 :
101 7 : HcclResult AicpuKfcRetryProcess::RetryProcess(HcclCommAicpu& comm, RestartParam& restartParam, uint32_t idx)
102 : {
103 7 : HcclResult ret = HCCL_SUCCESS;
104 7 : auto waitStopExecCmdTimeoutMs = comm.HcclGetWaitStopExecCmdTimeout();
105 7 : auto waitStopExecCmdTimeout = std::chrono::milliseconds(waitStopExecCmdTimeoutMs);
106 7 : auto waitRetryCmdTimeoutMs = comm.HcclGetWaitRetryCmdTimeout(restartParam.restartCnt);
107 7 : auto waitRetryCmdTimeout = std::chrono::milliseconds(waitRetryCmdTimeoutMs);
108 :
109 7 : switch (restartParam.fsmState[idx]) {
110 1 : case HcclOpExecFSM::HCCL_OP_EXEC_FSM_WAIT_END:
111 1 : HCCL_INFO("MC2 restart state HCCL_OP_EXEC_FSM_WAIT_END");
112 1 : restartParam.errorCode[idx] = KfcError::kSdma;
113 1 : restartParam.fsmState[idx] = HcclOpExecFSM::HCCL_OP_EXEC_FSM_STOPPING;
114 2 : ret = comm.UpdateOpExecStatus(
115 1 : restartParam.fsmState[idx], KfcStatus::kStoplaunch, restartParam.errorCode[idx],
116 : restartParam.restartCnt); // 上报sdma异常
117 1 : if (restartParam.fsmState[idx] == HcclOpExecFSM::HCCL_OP_EXEC_FSM_STOPPING) {
118 1 : restartParam.startTime[idx] = std::chrono::steady_clock::now();
119 : }
120 1 : break;
121 1 : case HcclOpExecFSM::HCCL_OP_EXEC_FSM_STOPPING:
122 1 : if ((std::chrono::steady_clock::now() - restartParam.startTime[idx]) >= waitStopExecCmdTimeout) {
123 0 : HCCL_ERROR("MC2 restart aicpu wait stop exec timeout[%u ms].", comm.HcclGetWaitStopExecCmdTimeout());
124 0 : restartParam.errorCode[idx] = KfcError::kTimeout;
125 0 : restartParam.fsmState[idx] = HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR;
126 : } else {
127 1 : ret = MC2OpExecFsmStoppingProcess(comm, restartParam.fsmState[idx], restartParam.errorCode[idx]);
128 : }
129 1 : break;
130 1 : case HcclOpExecFSM::HCCL_OP_EXEC_FSM_STOPPED:
131 1 : HCCL_INFO("MC2 restart state HCCL_OP_EXEC_FSM_STOPPED");
132 1 : ret = MC2OpExecFsmStoppedProcess(comm, restartParam.fsmState[idx], restartParam.errorCode[idx]);
133 1 : if (restartParam.fsmState[idx] == HcclOpExecFSM::HCCL_OP_EXEC_FSM_WAIT_RETRY) {
134 1 : CHK_RET(comm.UpdateOpExecStatus(
135 : restartParam.fsmState[idx], KfcStatus::kStopExec, restartParam.errorCode[idx],
136 : restartParam.restartCnt));
137 1 : restartParam.startTime[idx] = std::chrono::steady_clock::now();
138 : }
139 1 : break;
140 1 : case HcclOpExecFSM::HCCL_OP_EXEC_FSM_CHANGE_LINK:
141 1 : HCCL_INFO("MC2 restart state HCCL_OP_EXEC_FSM_CHANGE_LINK");
142 : // MC2重执行，清空所有rdma链接
143 1 : comm.CleanAllRoceResource();
144 1 : restartParam.errorCode[idx] = KfcError::kNone;
145 2 : ret = comm.UpdateOpExecStatus(
146 1 : restartParam.fsmState[idx], KfcStatus::kChanged, restartParam.errorCode[idx], restartParam.restartCnt);
147 1 : restartParam.linkChanged[idx] = true;
148 1 : restartParam.fsmState[idx] = HcclOpExecFSM::HCCL_OP_EXEC_FSM_WAIT_RETRY;
149 1 : break;
150 2 : case HcclOpExecFSM::HCCL_OP_EXEC_FSM_WAIT_RETRY:
151 2 : if ((std::chrono::steady_clock::now() - restartParam.startTime[idx]) >= waitRetryCmdTimeout) {
152 0 : HCCL_ERROR("MC2 restart aicpu wait retry timeout[%u ms].", waitRetryCmdTimeoutMs);
153 0 : restartParam.errorCode[idx] = KfcError::kTimeout;
154 0 : restartParam.fsmState[idx] = HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR;
155 : } else {
156 2 : ret = MC2OpExecFsmWaitRetryProcess(
157 2 : comm, restartParam.fsmState[idx], restartParam.errorCode[idx], restartParam.linkChanged[idx]);
158 : }
159 2 : break;
160 0 : case HcclOpExecFSM::HCCL_OP_EXEC_FSM_RETRY:
161 0 : HCCL_INFO("MC2 restart state HCCL_OP_EXEC_FSM_RETRY");
162 0 : restartParam.consultationResult[idx] = true;
163 0 : restartParam.errorCode[idx] = KfcError::kNone;
164 0 : comm.UpdateOpExecStatus(
165 0 : restartParam.fsmState[idx], KfcStatus::kEnd, restartParam.errorCode[idx], restartParam.restartCnt);
166 0 : return HCCL_SUCCESS;
167 1 : case HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR:
168 : default: {
169 1 : HCCL_ERROR("MC2 restart aicpu restart process error.");
170 1 : comm.UpdateOpExecStatus(
171 1 : restartParam.fsmState[idx], KfcStatus::kError, restartParam.errorCode[idx], restartParam.restartCnt);
172 1 : comm.GetDfxExtendInfo()->kfcStatus = DfxKfcStatus::kOneFinished;
173 1 : return (ret == HCCL_SUCCESS) ? HCCL_E_INTERNAL : ret;
174 : }
175 : }
176 6 : return ret;
177 : }
|
