Line data    Source code

       1              : /**
       2              :  * Copyright (c) 2026 Huawei Technologies Co., Ltd.
       3              :  * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
       4              :  * CANN Open Software License Agreement Version 2.0 (the "License").
       5              :  * Please refer to the License for details. You may not use this file except in compliance with the License.
       6              :  * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
       7              :  * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
       8              :  * See LICENSE in the root of the software repository for the full text of the License.
       9              :  */
      10              : #include "ns_recovery_func_lite.h"
      11              : #include "kfc.h"
      12              : #include "sal_pub.h"
      13              : #include "ns_recovery_lite.h"
      14              : #include "aicpu_indop_process.h"
      15              : 
      16              : namespace hccl {
      17            8 : NsRecoveryFuncLite &NsRecoveryFuncLite::GetInstance()
      18              : {
      19            8 :     static NsRecoveryFuncLite func;
      20            8 :     return func;
      21              : }
      22              : 
      23            2 : void NsRecoveryFuncLite::Call()
      24              : {
      25            2 :     std::shared_lock<std::shared_mutex> rwlock(AicpuIndopProcess::AicpuGetCommMutex());
      26              : 
      27            2 :     std::vector<std::pair<std::string, CollCommAicpuMgr *>> aicpuCommInfo;
      28            2 :     auto ret = AicpuIndopProcess::AicpuGetCommAll(aicpuCommInfo);
      29            2 :     if (ret != HCCL_SUCCESS) {
      30            0 :         HCCL_ERROR("[NsRecovery][BackGround] AicpuGetCommAll failed, errNo[0x%016llx]", ret);
      31            0 :         return;
      32              :     }
      33            3 :     for (auto &commInfo : aicpuCommInfo) {
      34            1 :         CollCommAicpu* deviceComm = commInfo.second->GetCollCommAicpu();
      35            1 :         if (deviceComm->GetCommmStatus() == HcclCommStatus::HCCL_COMM_STATUS_INVALID) {
      36            1 :             continue;
      37              :         }
      38            0 :         HandleStopLaunch(deviceComm);
      39            0 :         HandleClean(deviceComm);
      40              :     }
      41            2 : }
      42              : 
      43            0 : void NsRecoveryFuncLite::HandleStopLaunch(CollCommAicpu *deviceComm) const
      44              : {
      45            0 :     if (deviceComm->GetCommmStatus() == HcclCommStatus::HCCL_COMM_STATUS_SUSPENDING) {
      46            0 :         return;
      47              :     }
      48              : 
      49            0 :     Hccl::KfcCommand cmd = deviceComm->GetNsRecoveryLitePtr()->BackGroundGetCmd();
      50            0 :     if (cmd != Hccl::KfcCommand::NS_STOP_LAUNCH) {
      51            0 :         return;
      52              :     }
      53            0 :     HCCL_INFO("[NsRecovery][BackGround] received KfcCommand[NS_STOP_LAUNCH]");
      54            0 :     deviceComm->GetNsRecoveryLitePtr()->SetNeedClean(true);
      55            0 :     deviceComm->SetCommmStatus(HcclCommStatus::HCCL_COMM_STATUS_SUSPENDING);
      56            0 :     deviceComm->GetNsRecoveryLitePtr()->BackGroundSetStatus(Hccl::KfcStatus::STOP_LAUNCH_DONE);
      57            0 :     HCCL_INFO("[NsRecovery][BackGround] commId[%s] send KfcStatus[STOP_LAUNCH_DONE]", deviceComm->GetIdentifier().c_str());
      58              : }
      59              : 
      60            0 : void NsRecoveryFuncLite::HandleClean(CollCommAicpu *deviceComm)
      61              : {
      62            0 :     if (!deviceComm->GetNsRecoveryLitePtr()->IsNeedClean()) {
      63            0 :         return;
      64              :     }
      65            0 :     Hccl::KfcCommand cmd = deviceComm->GetNsRecoveryLitePtr()->BackGroundGetCmd();
      66            0 :     if (cmd != Hccl::KfcCommand::NS_CLEAN) {
      67            0 :         return;
      68              :     }
      69            0 :     HCCL_INFO("[NsRecovery][BackGround] received KfcCommand[NS_CLEAN]");
      70            0 :     StreamClean(deviceComm);
      71            0 :     deviceComm->Clean();
      72            0 :     deviceComm->GetNsRecoveryLitePtr()->SetNeedClean(false);
      73            0 :     deviceComm->GetNsRecoveryLitePtr()->BackGroundSetStatus(Hccl::KfcStatus::CLEAN_DONE);
      74            0 :     HCCL_INFO("[NsRecovery][BackGround] commId[%s] send KfcStatus[CLEAN_DONE]", deviceComm->GetIdentifier().c_str());
      75              : }
      76              : 
      77            0 : void NsRecoveryFuncLite::StreamClean(CollCommAicpu *deviceComm)
      78              : {
      79              :     // 查询停流是否完成
      80            0 :     u32 localDevId{0};
      81            0 :     auto ret = drvGetLocalDevIDByHostDevID(deviceComm->GetTopoInfo().devicePhyId, &localDevId);
      82            0 :     if (ret != DRV_ERROR_NONE) {
      83            0 :         HCCL_ERROR("NsRecoveryFuncLite::%s call drvGetLocalDevIDByHostDevID failed, devPhyId %u, ret %d", __func__, 
      84              :             deviceComm->GetTopoInfo().devicePhyId, ret);
      85            0 :         return;
      86              :     }
      87            0 :     if (DeviceQuery(localDevId, APP_ABORT_STAUTS::APP_ABORT_KILL_FINISH, 0U) != HCCL_SUCCESS) {
      88            0 :         deviceComm->GetNsRecoveryLitePtr()->BackGroundSetStatus(Hccl::KfcStatus::ERROR, Hccl::KfcErrType::EXEC);
      89            0 :         HCCL_ERROR("[NsRecovery][BackGround] Stream Stop failed");
      90            0 :         return;
      91              :     }
      92              : 
      93              :     // 通过thread获得streamlite信息，清理资源
      94            0 :     std::vector<std::shared_ptr<hccl::Thread>> threads = deviceComm->GetAllThread();
      95            0 :     for (auto &thread : threads) {
      96            0 :         Hccl::StreamLite *streamLitePtr = reinterpret_cast<Hccl::StreamLite *>(thread->GetStreamLitePtr());
      97            0 :         streamLitePtr->GetRtsq()->Reset();
      98              :     }
      99            0 :     HCCL_INFO("[NsRecovery][BackGround] commId[%s] streamClean success.", deviceComm->GetIdentifier().c_str());
     100            0 : }
     101              : 
     102              : constexpr u64 NSEC_PER_SEC = 1000000000U;
     103            9 : inline u64 GetCurCpuTimestamp()
     104              : {
     105            9 :     struct timespec timestamp{0, 0};
     106            9 :     (void)clock_gettime(CLOCK_MONOTONIC_RAW, &timestamp);
     107            9 :     return static_cast<u64>((timestamp.tv_sec * NSEC_PER_SEC) + (timestamp.tv_nsec));
     108              : }
     109              : 
     110              : constexpr u32 FIVE_MILLISECOND_OF_USLEEP = 5000U;
     111              : 
     112            6 : HcclResult NsRecoveryFuncLite::DeviceQuery(const uint32_t devId, const uint32_t step, const uint64_t timeout)
     113              : {
     114              :     uint32_t status;
     115              :     uint64_t endTime;
     116            6 :     const uint64_t startTime = GetCurCpuTimestamp();
     117            6 :     bool flag = true;
     118            8 :     while (flag) {
     119            8 :         ts_ctrl_msg_body_t queryIn = {};
     120            8 :         ts_ctrl_msg_body_t queryAck = {};
     121            8 :         size_t ackCount = sizeof(ts_ctrl_msg_body_t);
     122            8 :         queryIn.type = OPERATION_TYPE::OP_QUERY_ABORT_STATUS;
     123            8 :         queryIn.u.query_task_info.choice = APP_ABORT_STS_QUERY_CHOICE::APP_ABORT_STS_QUERY_BY_PID;
     124              :         struct tsdrv_ctrl_msg para;
     125            8 :         para.tsid = 0;
     126            8 :         para.msg_len = sizeof(ts_ctrl_msg_body_t);
     127            8 :         para.msg = static_cast<void*>(&queryIn);
     128            8 :         const drvError_t ret = halTsdrvCtl(devId, TSDRV_CTL_CMD_CTRL_MSG,
     129              :             static_cast<void*>(&para), sizeof(tsdrv_ctrl_msg), static_cast<void*>(&queryAck), &ackCount);
     130            8 :         if ((ret != DRV_ERROR_NONE) || (ackCount != sizeof(ts_ctrl_msg_body_t))) {
     131            2 :             HCCL_ERROR("halTsdrvCtl failed. ret = %d", ret);
     132            3 :             return HcclResult::HCCL_E_DRV;
     133              :         }
     134              : 
     135            6 :         status = queryAck.u.query_task_ack_info.status;
     136            6 :         if (status >= step) {
     137            3 :             flag = false;
     138            3 :             break;
     139              :         }
     140            3 :         endTime = GetCurCpuTimestamp();
     141            3 :         if ((timeout != 0U) && ((endTime - startTime) > timeout)) {
     142            1 :             HCCL_ERROR("[DeviceQuery]kill query timeout.");
     143            1 :             return HcclResult::HCCL_E_TIMEOUT;
     144              :         }
     145            2 :         SaluSleep(FIVE_MILLISECOND_OF_USLEEP);
     146              :     }
     147            3 :     return HcclResult::HCCL_SUCCESS;
     148              : }
     149              : 
     150              : }