Line data    Source code

       1              : /**
       2              :  * Copyright (c) 2025 Huawei Technologies Co., Ltd.
       3              :  * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
       4              :  * CANN Open Software License Agreement Version 2.0 (the "License").
       5              :  * Please refer to the License for details. You may not use this file except in compliance with the License.
       6              :  * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
       7              :  * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
       8              :  * See LICENSE in the root of the software repository for the full text of the License.
       9              :  */
      10              : #ifndef HCCL_COMM_TASKEXCEPTION_LIFT_H
      11              : #define HCCL_COMM_TASKEXCEPTION_LIFT_H
      12              : 
      13              : #include "daemon_func.h"
      14              : #include "res_pub.h"
      15              : #include "coll_comm_aicpu.h"
      16              : #include "aicpu_hccl_sqcq.h"
      17              : #include "error_message_v2.h"
      18              : #include "coll_comm_aicpu_mgr.h"
      19              : 
      20              : namespace Hccl {
      21              : template <typename T, uint32_t CAPACITY>
      22              : class DfxCircularQueue;
      23              : }
      24              : 
      25              : namespace hcomm {
      26              : 
      27              : class HcclCommTaskExceptionLite : public Hccl::DaemonFunc {
      28              : public:
      29              :     static HcclCommTaskExceptionLite& GetInstance();
      30              :     void Init(u32 devId);
      31              :     void Call() override;
      32              :     HcclResult PrintAllCommTaskException(); // 打印所有通信域所有流的task信息
      33              :     HcclResult PrintCommTaskException(CollCommAicpu* aicpuComm);
      34              : 
      35              : private:
      36            3 :     HcclCommTaskExceptionLite() = default;
      37            3 :     ~HcclCommTaskExceptionLite() override = default;
      38              : 
      39              :     // 检测流上的异常cqe并进行打印和上报
      40              :     HcclResult HandleExceptionCqe();
      41              :     HcclResult GetThreadCqe(hccl::Thread* thread, rtLogicCqReport_t& cqeException, CqeStatus& cqeStatus);
      42              :     HcclResult ProcessCqe(
      43              :         CollCommAicpu* aicpuComm, const rtLogicCqReport_t& exceptionInfo, const CqeStatus& cqeStatus,
      44              :         const std::vector<std::pair<std::string, CollCommAicpu*>>& aicpuCommInfo);
      45              : 
      46              :     // errMsg上报到host
      47              :     HcclResult ReportErrMsg(CollCommAicpu* aicpuComm, const rtLogicCqReport_t& exceptionInfo);
      48              :     HcclResult GenerateErrorMessageReport(
      49              :         CollCommAicpu* aicpuComm, const Hccl::DfxTaskInfo& taskInfo, const rtLogicCqReport_t& exceptionInfo,
      50              :         Hccl::ErrorMessageReport& errMsgInfo);
      51              :     void GenerateTaskErrMsg(
      52              :         const Hccl::DfxTaskInfo& taskInfo, Hccl::ErrorMessageReport& errMsgInfo,
      53              :         const rtLogicCqReport_t& exceptionInfo);
      54              :     void FillNotifyErrMsg(const Hccl::DfxTaskInfo& taskInfo, Hccl::ErrorMessageReport& errMsgInfo);
      55              :     void FillReduceErrMsg(
      56              :         const Hccl::DfxTaskInfo& taskInfo, Hccl::ErrorMessageReport& errMsgInfo,
      57              :         const rtLogicCqReport_t& exceptionInfo);
      58              :     void FillDmaErrMsg(
      59              :         const Hccl::DfxTaskInfo& taskInfo, Hccl::ErrorMessageReport& errMsgInfo,
      60              :         const rtLogicCqReport_t& exceptionInfo);
      61              :     void FillSdmaErrMsg(const Hccl::DfxTaskInfo& taskInfo, Hccl::ErrorMessageReport& errMsgInfo);
      62              :     void FillUbErrMsg(
      63              :         const Hccl::DfxTaskInfo& taskInfo, Hccl::ErrorMessageReport& errMsgInfo,
      64              :         const rtLogicCqReport_t& exceptionInfo);
      65              :     void FillReduceInlineErrMsg(const Hccl::DfxTaskInfo& taskInfo, Hccl::ErrorMessageReport& errMsgInfo);
      66              :     HcclResult SendTaskExceptionByMBox(const u32 notifyId, const u32 tsId, const rtLogicCqReport_t& exceptionInfo);
      67              :     uint16_t SwitchUBCqeErrCodeToTsErrCode(u32 cqeErrCode);
      68              :     uint16_t SwitchSdmaCqeErrCodeToTsErrCode(u32 cqeErrCode);
      69              : 
      70              :     // 打印流上的task信息的方法函数
      71              :     HcclResult PrintTaskExceptionBySqeId(CollCommAicpu* aicpuComm, u32 sqId, u32 sqeId);
      72              :     HcclResult PrintTaskContextInfo(CollCommAicpu* aicpuComm, u32 sqId, u32 taskId);
      73              :     HcclResult
      74              :     CollectTaskContext(CollCommAicpu* aicpuComm, u32 sqId, u32 taskId, std::vector<Hccl::DfxTaskInfo*>& taskContext);
      75              :     void PrintEid(const Hccl::DfxTaskInfo& taskInfo);
      76              :     std::string GetGroupInfo(CollCommAicpu* aicpuComm);
      77              :     u32 GetSqeId(uint16_t taskId, uint16_t streamId);
      78              : 
      79              :     Hccl::DfxTaskInfo* FindDfxTaskInfo(CollCommAicpu* aicpuComm, u32 sqId, u32 sqeId);
      80              :     Hccl::DfxCircularQueue<Hccl::DfxTaskInfo, Hccl::DFX_TASK_INFO_QUEUE_CAPACITY>*
      81              :     GetTaskQueueBySqId(CollCommAicpu* aicpuComm, u32 sqId);
      82              :     void GetEidFromChannelHandle(const Hccl::DfxTaskInfo& taskInfo, Hccl::Eid& locEid, Hccl::Eid& rmtEid);
      83              :     u32 GetRemoteRankId(const Hccl::DfxTaskInfo& taskInfo);
      84              :     std::string GetConciseTaskName(const Hccl::DfxTaskInfo& taskInfo);
      85              :     std::string GetNotifyInfo(const Hccl::DfxTaskInfo& taskInfo);
      86              :     void GetNotifyIdFromSqe(u64 sqeAddr, u32& notifyId);
      87              :     u32 GetOpIndex(const Hccl::DfxTaskInfo* taskInfo);
      88              :     void PrintOpDataInfo(const Hccl::DfxTaskInfo* taskInfo);
      89              :     // dpu相关
      90              :     HcclResult HandleDpuTaskexception(CollCommAicpu* aicpuComm);
      91              :     HcclResult IsHandleDpuStop(uint8_t* taskexceptionVa, bool& isStop);
      92              : 
      93              : private:
      94              :     bool stopCall_{false};
      95              :     u32 devId_{INVALID_UINT};
      96              :     std::unordered_map<u32, u32> threadsPrinted_;
      97              : };
      98              : 
      99              : } // namespace hcomm
     100              : 
     101              : #endif