Line data    Source code

       1              : /**
       2              :  * Copyright (c) 2025 Huawei Technologies Co., Ltd.
       3              :  * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
       4              :  * CANN Open Software License Agreement Version 2.0 (the "License").
       5              :  * Please refer to the License for details. You may not use this file except in compliance with the License.
       6              :  * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
       7              :  * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
       8              :  * See LICENSE in the root of the software repository for the full text of the License.
       9              :  */
      10              : 
      11              : #ifndef HCCL_COMM_TASKEXCEPTION_LIFT_H
      12              : #define HCCL_COMM_TASKEXCEPTION_LIFT_H
      13              : 
      14              : #include "daemon_func.h"
      15              : #include "res_pub.h"
      16              : #include "coll_comm_aicpu.h"
      17              : #include "aicpu_hccl_sqcq.h"
      18              : #include "error_message_v2.h"
      19              : #include "coll_comm_aicpu_mgr.h"
      20              : 
      21              : namespace Hccl {
      22              : template <typename T, uint32_t CAPACITY>
      23              : class DfxCircularQueue;
      24              : }
      25              : 
      26              : namespace hcomm {
      27              : 
      28              : class HcclCommTaskExceptionLite : public Hccl::DaemonFunc {
      29              : public:
      30              :     static HcclCommTaskExceptionLite& GetInstance();
      31              :     void Init(u32 devId);
      32              :     void Call() override;
      33              :     HcclResult PrintAllCommTaskException(); // 打印所有通信域所有流的task信息
      34              :     HcclResult PrintCommTaskException(CollCommAicpu* aicpuComm);
      35              : 
      36              : private:
      37            3 :     HcclCommTaskExceptionLite() = default;
      38            3 :     ~HcclCommTaskExceptionLite() override = default;
      39              : 
      40              :     // 检测流上的异常cqe并进行打印和上报
      41              :     HcclResult HandleExceptionCqe();
      42              :     HcclResult GetThreadCqe(hccl::Thread* thread, rtLogicCqReport_t& cqeException, CqeStatus& cqeStatus);
      43              :     HcclResult ProcessCqe(
      44              :         CollCommAicpu* aicpuComm, const rtLogicCqReport_t& exceptionInfo, const CqeStatus& cqeStatus,
      45              :         const std::vector<std::pair<std::string, CollCommAicpu*>>& aicpuCommInfo);
      46              : 
      47              :     // errMsg上报到host
      48              :     HcclResult ReportErrMsg(CollCommAicpu* aicpuComm, const rtLogicCqReport_t& exceptionInfo);
      49              :     HcclResult GenerateErrorMessageReport(
      50              :         CollCommAicpu* aicpuComm, const Hccl::DfxTaskInfo& taskInfo, const rtLogicCqReport_t& exceptionInfo,
      51              :         Hccl::ErrorMessageReport& errMsgInfo);
      52              :     void GenerateTaskErrMsg(
      53              :         const Hccl::DfxTaskInfo& taskInfo, Hccl::ErrorMessageReport& errMsgInfo,
      54              :         const rtLogicCqReport_t& exceptionInfo);
      55              :     void FillNotifyErrMsg(const Hccl::DfxTaskInfo& taskInfo, Hccl::ErrorMessageReport& errMsgInfo);
      56              :     void FillReduceErrMsg(
      57              :         const Hccl::DfxTaskInfo& taskInfo, Hccl::ErrorMessageReport& errMsgInfo,
      58              :         const rtLogicCqReport_t& exceptionInfo);
      59              :     void FillDmaErrMsg(
      60              :         const Hccl::DfxTaskInfo& taskInfo, Hccl::ErrorMessageReport& errMsgInfo,
      61              :         const rtLogicCqReport_t& exceptionInfo);
      62              :     void FillSdmaErrMsg(const Hccl::DfxTaskInfo& taskInfo, Hccl::ErrorMessageReport& errMsgInfo);
      63              :     void FillUbErrMsg(
      64              :         const Hccl::DfxTaskInfo& taskInfo, Hccl::ErrorMessageReport& errMsgInfo,
      65              :         const rtLogicCqReport_t& exceptionInfo);
      66              :     void FillReduceInlineErrMsg(const Hccl::DfxTaskInfo& taskInfo, Hccl::ErrorMessageReport& errMsgInfo);
      67              :     HcclResult SendTaskExceptionByMBox(const u32 notifyId, const u32 tsId, const rtLogicCqReport_t& exceptionInfo);
      68              :     uint16_t SwitchUBCqeErrCodeToTsErrCode(u32 cqeErrCode);
      69              :     uint16_t SwitchSdmaCqeErrCodeToTsErrCode(u32 cqeErrCode);
      70              : 
      71              :     // 打印流上的task信息的方法函数
      72              :     HcclResult PrintTaskExceptionBySqeId(CollCommAicpu* aicpuComm, u32 sqId, u32 sqeId);
      73              :     HcclResult PrintTaskContextInfo(CollCommAicpu* aicpuComm, u32 sqId, u32 taskId);
      74              :     HcclResult
      75              :     CollectTaskContext(CollCommAicpu* aicpuComm, u32 sqId, u32 taskId, std::vector<Hccl::DfxTaskInfo*>& taskContext);
      76              :     void PrintEid(const Hccl::DfxTaskInfo& taskInfo);
      77              :     std::string GetGroupInfo(CollCommAicpu* aicpuComm);
      78              :     u32 GetSqeId(uint16_t taskId, uint16_t streamId);
      79              : 
      80              :     Hccl::DfxTaskInfo* FindDfxTaskInfo(CollCommAicpu* aicpuComm, u32 sqId, u32 sqeId);
      81              :     Hccl::DfxCircularQueue<Hccl::DfxTaskInfo, Hccl::DFX_TASK_INFO_QUEUE_CAPACITY>*
      82              :     GetTaskQueueBySqId(CollCommAicpu* aicpuComm, u32 sqId);
      83              :     void GetEidFromChannelHandle(const Hccl::DfxTaskInfo& taskInfo, Hccl::Eid& locEid, Hccl::Eid& rmtEid);
      84              :     u32 GetRemoteRankId(const Hccl::DfxTaskInfo& taskInfo);
      85              :     std::string GetConciseTaskName(const Hccl::DfxTaskInfo& taskInfo);
      86              :     std::string GetNotifyInfo(const Hccl::DfxTaskInfo& taskInfo);
      87              :     void GetNotifyIdFromSqe(u64 sqeAddr, u32& notifyId);
      88              :     u32 GetOpIndex(const Hccl::DfxTaskInfo* taskInfo);
      89              :     void PrintOpDataInfo(const Hccl::DfxTaskInfo* taskInfo);
      90              :     // dpu相关
      91              :     HcclResult HandleDpuTaskexception(CollCommAicpu* aicpuComm);
      92              :     HcclResult IsHandleDpuStop(uint8_t* taskexceptionVa, bool& isStop);
      93              : 
      94              : private:
      95              :     bool stopCall_{false};
      96              :     u32 devId_{INVALID_UINT};
      97              :     std::unordered_map<u32, u32> threadsPrinted_;
      98              : };
      99              : 
     100              : } // namespace hcomm
     101              : 
     102              : #endif