Line data    Source code

       1              : /**
       2              :  * Copyright (c) 2025 Huawei Technologies Co., Ltd.
       3              :  * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
       4              :  * CANN Open Software License Agreement Version 2.0 (the "License").
       5              :  * Please refer to the License for details. You may not use this file except in compliance with the License.
       6              :  * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
       7              :  * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
       8              :  * See LICENSE in the root of the software repository for the full text of the License.
       9              :  */
      10              : 
      11              : #ifndef HCCL_RETRY_SERVER_H
      12              : #define HCCL_RETRY_SERVER_H
      13              : #include <unordered_set>
      14              : #include "opretry_base.h"
      15              : 
      16              : namespace hccl {
      17              : 
      18              : HcclResult CreateOpRetryServerByState(RetryState state, RetryContext* retryCtx);
      19              : 
      20              : // server状态机 正常运行状态转移表
      21              : const std::map<RetryState, RetryState> RETRY_SERVER_STATE_TRANSFER_LABEL{
      22              :     {RETRY_STATE_SERVER_RUNNING, RETRY_STATE_CMD_STOP_AICPU},
      23              :     {RETRY_STATE_CMD_STOP_AICPU, RETRY_STATE_WAIT_AICPU_STOPED},
      24              :     {RETRY_STATE_WAIT_AICPU_STOPED, RETRY_STATE_CMD_STOP_STREAM},
      25              :     {RETRY_STATE_CMD_STOP_STREAM, RETRY_STATE_WAIT_STREAM_STOPED},
      26              :     {RETRY_STATE_WAIT_STREAM_STOPED, RETRY_STATE_CMD_CLEAR_STREAM},
      27              :     {RETRY_STATE_CMD_CLEAR_STREAM, RETRY_STATE_WAIT_STREAM_CLEARED},
      28              :     {RETRY_STATE_WAIT_STREAM_CLEARED, RETRY_STATE_CMD_STOP_TRANSPORT},
      29              :     {RETRY_STATE_CMD_STOP_TRANSPORT, RETRY_STATE_WAIT_STOP_TRANSPORT},
      30              :     {RETRY_STATE_WAIT_STOP_TRANSPORT, RETRY_STATE_CMD_CHECK},
      31              :     {RETRY_STATE_CMD_CHECK, RETRY_STATE_WAIT_CHECK_INFO},
      32              :     {RETRY_STATE_WAIT_CHECK_INFO, RETRY_STATE_CHECK_OP},
      33              :     {RETRY_STATE_CHECK_OP, RETRY_STATE_CMD_CHECK_LINK},
      34              :     {RETRY_STATE_CMD_CHECK_LINK, RETRY_STATE_WAIT_LINK_CHECKED},
      35              :     {RETRY_STATE_WAIT_LINK_CHECKED, RETRY_STATE_CHECK_ALL_LINK},
      36              :     {RETRY_STATE_CHECK_ALL_LINK, RETRY_STATE_CMD_RESUME_TRANSPORT},
      37              :     {RETRY_STATE_CMD_RESUME_TRANSPORT, RETRY_STATE_WAIT_RESUME_TRANSPORT},
      38              :     {RETRY_STATE_WAIT_RESUME_TRANSPORT, RETRY_STATE_CMD_RESET_NOTIFY},
      39              :     {RETRY_STATE_CMD_RESET_NOTIFY, RETRY_STATE_WAIT_NOTIFY_RESETED},
      40              :     {RETRY_STATE_WAIT_NOTIFY_RESETED, RETRY_STATE_CMD_CAN_RETRY},
      41              :     {RETRY_STATE_CMD_CAN_RETRY, RETRY_STATE_WAIT_CAN_RETRY},
      42              :     {RETRY_STATE_WAIT_CAN_RETRY, RETRY_STATE_SERVER_RUNNING},
      43              :     {RETRY_STATE_SERVER_RETRY_FAIL, RETRY_STATE_SERVER_RUNNING}};
      44              : 
      45              : // server状态机 IssueCmd状态对应的command
      46              : const std::map<RetryState, RetryCommand> RETRY_SERVER_STATE_TO_CMD_LABEL{
      47              :     {RETRY_STATE_CMD_CHECK_LINK, RETRY_CMD_CHECK_LINK},
      48              :     {RETRY_STATE_CMD_STOP_AICPU, RETRY_CMD_STOP_AICPU},
      49              :     {RETRY_STATE_CMD_STOP_STREAM, RETRY_CMD_STOP_STREAM},
      50              :     {RETRY_STATE_CMD_CLEAR_STREAM, RETRY_CMD_CLEAR_STREAM},
      51              :     {RETRY_STATE_CMD_STOP_TRANSPORT, RETRY_CMD_STOP_TRANSPORT},
      52              :     {RETRY_STATE_CMD_RESET_NOTIFY, RETRY_CMD_RESET_NOTIFY},
      53              :     {RETRY_STATE_CMD_RESUME_TRANSPORT, RETRY_CMD_RESUME_TRANSPORT},
      54              :     {RETRY_STATE_CMD_CHECK, RETRY_CMD_CHECK_OPNAME},
      55              :     {RETRY_STATE_CMD_CAN_RETRY, RETRY_CMD_CAN_RETRY}};
      56              : 
      57              : // RETRY_STATE_SERVER_RETRY_FAIL 重执行异常状态处理
      58            3 : class OpRetryServerBase : public OpRetryBase {
      59              : public:
      60              :     HcclResult ProcessError(RetryContext* retryCtx) override;
      61              : };
      62              : 
      63              : // RETRY_STATE_SERVER_RUNNING
      64            1 : class OpRetryServerRunning : public OpRetryServerBase {
      65              : public:
      66              :     HcclResult ProcessEvent(RetryContext* retryCtx) override;
      67              :     HcclResult ParaseErrorCode(RetryContext* retryCtx, HcclAgentRetryInfo& agentInfo, RetryState& nextState);
      68              : 
      69              : protected:
      70              :     std::map<u32, std::chrono::steady_clock::time_point> lastRecvTimes_;
      71              :     std::unordered_set<u32> disableAgent_; // 记录已经关闭的对端, 不再轮询, 避免刷屏
      72              : };
      73              : 
      74              : // server处理错误rank状态机
      75            1 : class OpRetryServerHandleError : public OpRetryServerRunning {
      76              : public:
      77              :     HcclResult ProcessEvent(RetryContext* retryCtx) override;
      78              : 
      79              : private:
      80              :     HcclResult SetNeedRetryServerRank(RetryContext* retryCtx, const HcclOpIdentifier& opId);
      81              : };
      82              : 
      83              : // 公共状态-向agent状态机发送命令
      84              : class OpRetryServerIssueCmd : public OpRetryServerBase {
      85              : public:
      86              :     HcclResult ProcessEvent(RetryContext* retryCtx) override;
      87              : };
      88              : 
      89              : // 公共状态-等待agent状态机回复
      90              : class OpRetryServerWaitResp : public OpRetryServerBase {
      91              : public:
      92              :     HcclResult ProcessEvent(RetryContext* retryCtx) override;
      93              : 
      94              : private:
      95              :     // 接收到对端重执行失败的信息后，打印当前接收到的Agent节点信息
      96              :     void PrintAgentInfoAfterFail(
      97              :         std::map<u32, HcclAgentRetryInfo>& serverSockets, std::set<u32>& recvVaild, HcclAgentRetryInfo& agentRetryInfo);
      98              : };
      99              : 
     100            1 : class OpRetryServerCheckOp : public OpRetryServerBase {
     101              : public:
     102              :     HcclResult ProcessEvent(RetryContext* retryCtx) override;
     103              : };
     104              : 
     105              : class OpRetryServerCheckAllLink : public OpRetryServerBase {
     106              : public:
     107              :     HcclResult ProcessEvent(RetryContext* retryCtx) override;
     108              : };
     109              : 
     110              : class OpRetryServerIssueChangeLinkAndResume : public OpRetryServerBase {
     111              : public:
     112              :     HcclResult ProcessEvent(RetryContext* retryCtx) override;
     113              : };
     114              : 
     115              : class OpRetryServerWaitLinkInfo : public OpRetryServerBase {
     116              : public:
     117              :     HcclResult ProcessEvent(RetryContext* retryCtx) override;
     118              : };
     119              : 
     120              : class OpRetryServerRetryFail : public OpRetryServerBase {
     121              : public:
     122              :     HcclResult ProcessEvent(RetryContext* retryCtx) override;
     123              : };
     124              : 
     125              : // 强制终止NPU后的状态，等待通信域恢复
     126              : class OpRetryServerWaitResume : public OpRetryServerRunning {
     127              : public:
     128              :     HcclResult ProcessEvent(RetryContext* retryCtx) override;
     129              : };
     130              : 
     131              : class SwitchNicServerCheckAllSwitchRanks : public OpRetryServerBase {
     132              : public:
     133              :     HcclResult ProcessEvent(RetryContext* retryCtx) override;
     134              : 
     135              : private:
     136              :     bool CompareSwitchRankList(const u32* firstSwitchRankList, const u32* switchRankList, const u32 switchRankNum);
     137              :     bool CompareUseBackupLists(const bool* firstArray, const bool* secondArray, const u32 switchRankNum);
     138              :     bool CheckRemotePorts(const u32 rankId, const ActiveSwitchInfo& switchRankInfo);
     139              :     HcclResult
     140              :     CollectSingleAgentActiveSwitchInfo(RetryContext* retryCtx, const u32 rankId, HcclAgentRetryInfo& agentInfo);
     141              :     HcclResult CollectAgentActiveSwitchInfo(RetryContext* retryCtx);
     142              :     HcclResult CheckAgentActiveSwitchInfo(RetryContext* retryCtx);
     143              : };
     144              : 
     145            1 : class ResumeServerCheckAllLink : public OpRetryServerBase {
     146              : public:
     147              :     HcclResult ProcessEvent(RetryContext* retryCtx) override;
     148              : 
     149              : private:
     150              :     HcclResult WaitAgentCheckLinkResult(RetryContext* retryCtx);
     151              :     HcclResult CheckAllLink(RetryContext* retryCtx, RetryState& nextState);
     152              : };
     153              : 
     154              : class ResumeServerChangeLink : public OpRetryServerBase {
     155              : public:
     156              :     HcclResult ProcessEvent(RetryContext* retryCtx) override;
     157              : 
     158              : private:
     159              :     HcclResult CmdAgentChangeLink(RetryContext* retryCtx);
     160              :     HcclResult WaitAllChangeLinkResult(RetryContext* retryCtx, RetryState& nextState);
     161              : };
     162              : } // namespace hccl
     163              : #endif