Line data    Source code

       1              : /**
       2              :  * Copyright (c) 2025 Huawei Technologies Co., Ltd.
       3              :  * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
       4              :  * CANN Open Software License Agreement Version 2.0 (the "License").
       5              :  * Please refer to the License for details. You may not use this file except in compliance with the License.
       6              :  * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
       7              :  * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
       8              :  * See LICENSE in the root of the software repository for the full text of the License.
       9              :  */
      10              : 
      11              : #ifndef HCCL_RETRY_SERVER_H
      12              : #define HCCL_RETRY_SERVER_H
      13              : #include <unordered_set>
      14              : #include "opretry_base.h"
      15              : 
      16              : namespace hccl {
      17              : 
      18              : HcclResult CreateOpRetryServerByState(RetryState state, RetryContext* retryCtx);
      19              : 
      20              : // server状态机 正常运行状态转移表
      21              : const std::map<RetryState, RetryState> RETRY_SERVER_STATE_TRANSFER_LABEL{
      22              :     {RETRY_STATE_SERVER_RUNNING, RETRY_STATE_CMD_STOP_AICPU},
      23              :     {RETRY_STATE_CMD_STOP_AICPU, RETRY_STATE_WAIT_AICPU_STOPED},
      24              :     {RETRY_STATE_WAIT_AICPU_STOPED, RETRY_STATE_CMD_STOP_STREAM},
      25              :     {RETRY_STATE_CMD_STOP_STREAM, RETRY_STATE_WAIT_STREAM_STOPED},
      26              :     {RETRY_STATE_WAIT_STREAM_STOPED, RETRY_STATE_CMD_CLEAR_STREAM},
      27              :     {RETRY_STATE_CMD_CLEAR_STREAM, RETRY_STATE_WAIT_STREAM_CLEARED},
      28              :     {RETRY_STATE_WAIT_STREAM_CLEARED, RETRY_STATE_CMD_STOP_TRANSPORT},
      29              :     {RETRY_STATE_CMD_STOP_TRANSPORT, RETRY_STATE_WAIT_STOP_TRANSPORT},
      30              :     {RETRY_STATE_WAIT_STOP_TRANSPORT, RETRY_STATE_CMD_CHECK},
      31              :     {RETRY_STATE_CMD_CHECK, RETRY_STATE_WAIT_CHECK_INFO},
      32              :     {RETRY_STATE_WAIT_CHECK_INFO, RETRY_STATE_CHECK_OP},
      33              :     {RETRY_STATE_CHECK_OP, RETRY_STATE_CMD_CHECK_LINK},
      34              :     {RETRY_STATE_CMD_CHECK_LINK, RETRY_STATE_WAIT_LINK_CHECKED},
      35              :     {RETRY_STATE_WAIT_LINK_CHECKED, RETRY_STATE_CHECK_ALL_LINK},
      36              :     {RETRY_STATE_CHECK_ALL_LINK, RETRY_STATE_CMD_RESUME_TRANSPORT},
      37              :     {RETRY_STATE_CMD_RESUME_TRANSPORT, RETRY_STATE_WAIT_RESUME_TRANSPORT},
      38              :     {RETRY_STATE_WAIT_RESUME_TRANSPORT, RETRY_STATE_CMD_RESET_NOTIFY},
      39              :     {RETRY_STATE_CMD_RESET_NOTIFY, RETRY_STATE_WAIT_NOTIFY_RESETED},
      40              :     {RETRY_STATE_WAIT_NOTIFY_RESETED, RETRY_STATE_CMD_CAN_RETRY},
      41              :     {RETRY_STATE_CMD_CAN_RETRY, RETRY_STATE_WAIT_CAN_RETRY},
      42              :     {RETRY_STATE_WAIT_CAN_RETRY, RETRY_STATE_SERVER_RUNNING},
      43              :     {RETRY_STATE_SERVER_RETRY_FAIL, RETRY_STATE_SERVER_RUNNING}};
      44              : 
      45              : // server状态机 IssueCmd状态对应的command
      46              : const std::map<RetryState, RetryCommand> RETRY_SERVER_STATE_TO_CMD_LABEL{
      47              :     {RETRY_STATE_CMD_CHECK_LINK, RETRY_CMD_CHECK_LINK},
      48              :     {RETRY_STATE_CMD_STOP_AICPU, RETRY_CMD_STOP_AICPU},
      49              :     {RETRY_STATE_CMD_STOP_STREAM, RETRY_CMD_STOP_STREAM},
      50              :     {RETRY_STATE_CMD_CLEAR_STREAM, RETRY_CMD_CLEAR_STREAM},
      51              :     {RETRY_STATE_CMD_STOP_TRANSPORT, RETRY_CMD_STOP_TRANSPORT},
      52              :     {RETRY_STATE_CMD_RESET_NOTIFY, RETRY_CMD_RESET_NOTIFY},
      53              :     {RETRY_STATE_CMD_RESUME_TRANSPORT, RETRY_CMD_RESUME_TRANSPORT},
      54              :     {RETRY_STATE_CMD_CHECK, RETRY_CMD_CHECK_OPNAME},
      55              :     {RETRY_STATE_CMD_CAN_RETRY, RETRY_CMD_CAN_RETRY}};
      56              : 
      57              : // RETRY_STATE_SERVER_RETRY_FAIL 重执行异常状态处理
      58            3 : class OpRetryServerBase : public OpRetryBase {
      59              : public:
      60              :     HcclResult ProcessError(RetryContext* retryCtx) override;
      61              : };
      62              : 
      63              : // RETRY_STATE_SERVER_RUNNING
      64            1 : class OpRetryServerRunning : public OpRetryServerBase {
      65              : public:
      66              :     HcclResult ProcessEvent(RetryContext* retryCtx) override;
      67              :     HcclResult ParaseErrorCode(RetryContext* retryCtx, HcclAgentRetryInfo& agentInfo, RetryState& nextState);
      68              : 
      69              : protected:
      70              :     std::map<u32, std::chrono::steady_clock::time_point> lastRecvTimes_;
      71              :     std::unordered_set<u32> disableAgent_; // 记录已经关闭的对端, 不再轮询, 避免刷屏
      72              : };
      73              : 
      74              : // server处理错误rank状态机
      75            1 : class OpRetryServerHandleError : public OpRetryServerRunning {
      76              : public:
      77              :     HcclResult ProcessEvent(RetryContext* retryCtx) override;
      78              : 
      79              : private:
      80              :     HcclResult SetNeedRetryServerRank(RetryContext* retryCtx, const HcclOpIdentifier& opId);
      81              : };
      82              : 
      83              : // 公共状态-向agent状态机发送命令
      84              : class OpRetryServerIssueCmd : public OpRetryServerBase {
      85              : public:
      86              :     HcclResult ProcessEvent(RetryContext* retryCtx) override;
      87              : };
      88              : 
      89              : // 公共状态-等待agent状态机回复
      90              : class OpRetryServerWaitResp : public OpRetryServerBase {
      91              : public:
      92              :     HcclResult ProcessEvent(RetryContext* retryCtx) override;
      93              : 
      94              : private:
      95              :     // 接收到对端重执行失败的信息后，打印当前接收到的Agent节点信息
      96              :     void PrintAgentInfoAfterFail(
      97              :         std::map<u32, HcclAgentRetryInfo>& serverSockets, std::set<u32>& recvVaild,
      98              :         HcclAgentRetryInfo& agentRetryInfo) const;
      99              : };
     100              : 
     101            1 : class OpRetryServerCheckOp : public OpRetryServerBase {
     102              : public:
     103              :     HcclResult ProcessEvent(RetryContext* retryCtx) override;
     104              : };
     105              : 
     106              : class OpRetryServerCheckAllLink : public OpRetryServerBase {
     107              : public:
     108              :     HcclResult ProcessEvent(RetryContext* retryCtx) override;
     109              : };
     110              : 
     111              : class OpRetryServerIssueChangeLinkAndResume : public OpRetryServerBase {
     112              : public:
     113              :     HcclResult ProcessEvent(RetryContext* retryCtx) override;
     114              : };
     115              : 
     116              : class OpRetryServerWaitLinkInfo : public OpRetryServerBase {
     117              : public:
     118              :     HcclResult ProcessEvent(RetryContext* retryCtx) override;
     119              : };
     120              : 
     121              : class OpRetryServerRetryFail : public OpRetryServerBase {
     122              : public:
     123              :     HcclResult ProcessEvent(RetryContext* retryCtx) override;
     124              : };
     125              : 
     126              : // 强制终止NPU后的状态，等待通信域恢复
     127              : class OpRetryServerWaitResume : public OpRetryServerRunning {
     128              : public:
     129              :     HcclResult ProcessEvent(RetryContext* retryCtx) override;
     130              : };
     131              : 
     132              : class SwitchNicServerCheckAllSwitchRanks : public OpRetryServerBase {
     133              : public:
     134              :     HcclResult ProcessEvent(RetryContext* retryCtx) override;
     135              : 
     136              : private:
     137              :     bool
     138              :     CompareSwitchRankList(const u32* firstSwitchRankList, const u32* switchRankList, const u32 switchRankNum) const;
     139              :     bool CompareUseBackupLists(const bool* firstArray, const bool* secondArray, const u32 switchRankNum) const;
     140              :     bool CheckRemotePorts(const u32 rankId, const ActiveSwitchInfo& switchRankInfo);
     141              :     HcclResult
     142              :     CollectSingleAgentActiveSwitchInfo(RetryContext* retryCtx, const u32 rankId, HcclAgentRetryInfo& agentInfo);
     143              :     HcclResult CollectAgentActiveSwitchInfo(RetryContext* retryCtx);
     144              :     HcclResult CheckAgentActiveSwitchInfo(RetryContext* retryCtx);
     145              : };
     146              : 
     147            1 : class ResumeServerCheckAllLink : public OpRetryServerBase {
     148              : public:
     149              :     HcclResult ProcessEvent(RetryContext* retryCtx) override;
     150              : 
     151              : private:
     152              :     HcclResult WaitAgentCheckLinkResult(RetryContext* retryCtx);
     153              :     HcclResult CheckAllLink(RetryContext* retryCtx, RetryState& nextState);
     154              : };
     155              : 
     156              : class ResumeServerChangeLink : public OpRetryServerBase {
     157              : public:
     158              :     HcclResult ProcessEvent(RetryContext* retryCtx) override;
     159              : 
     160              : private:
     161              :     HcclResult CmdAgentChangeLink(RetryContext* retryCtx);
     162              :     HcclResult WaitAllChangeLinkResult(RetryContext* retryCtx, RetryState& nextState);
     163              : };
     164              : } // namespace hccl
     165              : #endif