Line data    Source code

       1              : /**
       2              :  * Copyright (c) 2025 Huawei Technologies Co., Ltd.
       3              :  * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
       4              :  * CANN Open Software License Agreement Version 2.0 (the "License").
       5              :  * Please refer to the License for details. You may not use this file except in compliance with the License.
       6              :  * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
       7              :  * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
       8              :  * See LICENSE in the root of the software repository for the full text of the License.
       9              :  */
      10              : #ifndef HCCL_COMM_TASKEXCEPTION_H
      11              : #define HCCL_COMM_TASKEXCEPTION_H
      12              : 
      13              : #include <array>
      14              : #include <map>
      15              : #include "types.h"
      16              : #include "hccl_types.h"
      17              : #include "orion_adapter_rts.h"
      18              : #include "global_mirror_tasks.h"
      19              : #include "error_message_v2.h"
      20              : #include "orion_adapter_hccp.h"
      21              : #include "rdma_handle_manager.h"
      22              : #include "coll_comm.h"
      23              : 
      24              : namespace hcomm {
      25              : using RdmaHandle = void*;
      26              : 
      27              : using GetAicpuCqeErrInfoCallBackHcomm = void (*)(
      28              :     u32 RemoteLocalId, u32 LocDeviceId, uint16_t status, std::string LocalEid, std::string RemoteEid,
      29              :     std::string RemoteInsId); // 获取远端rankId的回调函数类型
      30              : void RegisterGetAicpuCqeErrInfoCallBackHcomm(GetAicpuCqeErrInfoCallBackHcomm p1); // 注册获取远端rankId的回调函数
      31              : 
      32              : using AicpuGetErrStatusVecCallBack = std::vector<std::string> (*)(s32 deviceLogicID);
      33              : void RegisterAicpuGetErrStatusVecCallBack(AicpuGetErrStatusVecCallBack p1);
      34              : 
      35              : class TaskExceptionHost {
      36              : public:
      37              :     static TaskExceptionHost* GetInstance(s32 deviceLogicID);
      38         2990 :     TaskExceptionHost() = default;
      39              :     ~TaskExceptionHost();
      40              : 
      41              :     HcclResult Register(u64 commHandle);
      42              :     HcclResult UnRegister(u64 commHandle);
      43              :     static void ProcessCallback(rtExceptionInfo_t* exceptionInfo);
      44              : 
      45              : private:
      46              :     void Process(rtExceptionInfo_t* exceptionInfo);
      47              :     void HandleAicpuErrorReport(
      48              :         rtExceptionInfo_t* exceptionInfo, const Hccl::ErrorMessageReport& errorMessage,
      49              :         const Hccl::TaskInfo& taskInfo) const;
      50              :     void HandleHostErrorReport(rtExceptionInfo_t* exceptionInfo, const Hccl::TaskInfo& taskInfo) const;
      51              :     void ReportErrorMsg(
      52              :         const Hccl::TaskInfo& exceptionTaskInfo, const std::string& groupRankContent,
      53              :         const Hccl::ErrorMessageReport& errorMessage, rtExceptionInfo_t* exceptionInfo) const;
      54              : 
      55              :     std::string GetGroupRankInfo(const Hccl::TaskInfo& taskInfo) const;
      56              :     void ProcessException(rtExceptionInfo_t* exceptionInfo, const Hccl::TaskInfo& taskInfo);
      57              :     void PrintTaskContextInfo(uint32_t deviceId, uint32_t streamId, uint32_t taskId) const;
      58              :     void PrintUbDfxInfo(rtExceptionInfo_t* exceptionInfo, const Hccl::ErrorMessageReport& errorMessage) const;
      59              :     void PrintGroupErrorMessage(
      60              :         const Hccl::ErrorMessageReport& errorMessage, const Hccl::TaskInfo& exceptionTaskInfo,
      61              :         std::string& groupRankContent, std::string& stageErrInfo) const;
      62              :     void PrintOpDataErrorMessage(
      63              :         u32 deviceId, const Hccl::ErrorMessageReport& errorMessage, std::string& stageErrInfo) const;
      64              :     HcclResult PrintUbRegisters(s32 devLogicId, const RdmaHandle rdmaHandle) const;
      65              :     void ClusterMoniterGetAicpuCqeErrInfo(
      66              :         u32 remoteLocalId, u32 locDeviceId, uint16_t status, std::string localEid, std::string remoteEid,
      67              :         std::string remoteInsId) const;
      68              :     void GetAicpuCqeErrInfo(
      69              :         rtExceptionInfo_t* exceptionInfo, const Hccl::ErrorMessageReport& errorMessage,
      70              :         const Hccl::TaskInfo& taskInfo) const;
      71              :     void GetAicpuCqeErrRemoteLocalIdByRankId(hccl::CollComm* collComm, uint32_t rankid, u32& remoteLocalId) const;
      72              :     void GetAicpuCqeErrNetInstanceByRankId(hccl::CollComm* collComm, uint32_t rankid, std::string& netInstanceId) const;
      73              :     bool ProcessDpuException(const rtExceptionInfo_t* exceptionInfo) const;
      74              : 
      75              : private:
      76              :     std::mutex taskExceptionMutex_;
      77              :     std::unordered_set<u64> CommRegisterMap_;
      78              :     bool hasAicpuReport_{false};
      79              : };
      80              : } // namespace hcomm
      81              : 
      82              : #endif