Line data    Source code

       1              : /**
       2              :  * Copyright (c) 2025 Huawei Technologies Co., Ltd.
       3              :  * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
       4              :  * CANN Open Software License Agreement Version 2.0 (the "License").
       5              :  * Please refer to the License for details. You may not use this file except in compliance with the License.
       6              :  * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
       7              :  * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
       8              :  * See LICENSE in the root of the software repository for the full text of the License.
       9              :  */
      10              : #ifndef HCCL_COMM_TASKEXCEPTION_LIFT_H
      11              : #define HCCL_COMM_TASKEXCEPTION_LIFT_H
      12              : 
      13              : #include "daemon_func.h"
      14              : #include "mirror_task_manager.h"
      15              : #include "coll_comm_aicpu.h"
      16              : #include "aicpu_hccl_sqcq.h"
      17              : #include "error_message_v2.h"
      18              : #include "aicpu_indop_process.h"
      19              : 
      20              : namespace hcomm {
      21              : 
      22              : class HcclCommTaskExceptionLite : public Hccl::DaemonFunc {
      23              : public:
      24              :     static HcclCommTaskExceptionLite &GetInstance();
      25              :     void Init(u32 devId);
      26              :     void Call() override;
      27              :     HcclResult PrintAllCommTaskException(); // 打印所有通信域所有流的task信息
      28              :     HcclResult PrintCommTaskException(CollCommAicpu *aicpuComm);
      29              : 
      30              : private:
      31            1 :     HcclCommTaskExceptionLite() = default;
      32            1 :     ~HcclCommTaskExceptionLite() override = default;
      33              : 
      34              :     // 检测流上的异常cqe并进行打印和上报
      35              :     HcclResult HandleExceptionCqe();
      36              :     HcclResult GetThreadCqe(hccl::Thread* thread, rtLogicCqReport_t &cqeException, CqeStatus &cqeStatus);
      37              :     HcclResult ProcessCqe(CollCommAicpu *aicpuComm, const rtLogicCqReport_t &exceptionInfo, const CqeStatus &cqeStatus,
      38              :         const std::vector<std::pair<std::string, CollCommAicpuMgr *>> &aicpuCommInfo);
      39              : 
      40              :     // errMsg上报到host
      41              :     HcclResult ReportErrMsg(CollCommAicpu *aicpuComm, const rtLogicCqReport_t &exceptionInfo);
      42              :     HcclResult GenerateErrorMessageReport(CollCommAicpu *aicpuComm, const Hccl::TaskInfo& taskInfo,
      43              :         const rtLogicCqReport_t &exceptionInfo, Hccl::ErrorMessageReport &errMsgInfo);
      44              :     void GenerateTaskErrMsg(const Hccl::TaskInfo& taskInfo, Hccl::ErrorMessageReport &errMsgInfo,
      45              :         const rtLogicCqReport_t &exceptionInfo);
      46              :     void FillNotifyErrMsg(const Hccl::TaskInfo& taskInfo, Hccl::ErrorMessageReport &errMsgInfo);
      47              :     void FillReduceErrMsg(const Hccl::TaskInfo& taskInfo, Hccl::ErrorMessageReport &errMsgInfo,
      48              :         const rtLogicCqReport_t &exceptionInfo);
      49              :     void FillDmaErrMsg(const Hccl::TaskInfo& taskInfo, Hccl::ErrorMessageReport &errMsgInfo,
      50              :         const rtLogicCqReport_t &exceptionInfo);
      51              :     void FillSdmaErrMsg(const Hccl::TaskInfo& taskInfo, Hccl::ErrorMessageReport &errMsgInfo);
      52              :     void FillUbErrMsg(const Hccl::TaskInfo& taskInfo, Hccl::ErrorMessageReport &errMsgInfo,
      53              :         const rtLogicCqReport_t &exceptionInfo);
      54              :     void FillReduceInlineErrMsg(const Hccl::TaskInfo& taskInfo, Hccl::ErrorMessageReport &errMsgInfo);
      55              :     HcclResult SendTaskExceptionByMBox(const u32 notifyId, const u32 tsId, const rtLogicCqReport_t &exceptionInfo);
      56              :     uint16_t SwitchUBCqeErrCodeToTsErrCode(u32 cqeErrCode);
      57              :     uint16_t SwitchSdmaCqeErrCodeToTsErrCode(u32 cqeErrCode);
      58              : 
      59              :     // 打印流上的task信息的方法函数
      60              :     HcclResult PrintTaskExceptionBySqeId(CollCommAicpu *aicpuComm, u32 sqId, u32 sqeId);
      61              :     HcclResult PrintTaskContextInfo(CollCommAicpu *aicpuComm, u32 sqId, u32 taskId);
      62              :     HcclResult CollectTaskContext(CollCommAicpu *aicpuComm, u32 sqId, u32 taskId,
      63              :         std::vector<Hccl::TaskInfo*> &taskContext);
      64              :     void PrintEid(const Hccl::TaskInfo& taskInfo);
      65              :     std::string GetGroupInfo(CollCommAicpu *aicpuComm);
      66              :     u32 GetSqeId(uint16_t taskId, uint16_t streamId);
      67              : 
      68              :     // dpu相关
      69              :     HcclResult HandleDpuTaskexception(CollCommAicpu *aicpuComm);
      70              :     HcclResult IsHandleDpuStop(uint8_t *taskexceptionVa, bool &isStop);
      71              : 
      72              : private:
      73              :     bool stopCall_{false}; // 避免taskException失败后刷屏
      74              :     u32 devId_{INVALID_UINT};
      75              :     Hccl::MirrorTaskManager* mirrorTaskManager_{nullptr};  // 使用原始指针，不管理生命周期
      76              :     std::unordered_map<u32, u32> threadsPrinted_; // sqId -> sqeId, 记录已经打印过taskException的流信息
      77              : };
      78              : 
      79              : }
      80              : 
      81              : #endif