Line data    Source code

       1              : /**
       2              :  * Copyright (c) 2025 Huawei Technologies Co., Ltd.
       3              :  * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
       4              :  * CANN Open Software License Agreement Version 2.0 (the "License").
       5              :  * Please refer to the License for details. You may not use this file except in compliance with the License.
       6              :  * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
       7              :  * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
       8              :  * See LICENSE in the root of the software repository for the full text of the License.
       9              :  */
      10              : 
      11              : #ifndef HCCL_COMM_TASKEXCEPTION_H
      12              : #define HCCL_COMM_TASKEXCEPTION_H
      13              : 
      14              : #include <array>
      15              : #include <map>
      16              : #include "types.h"
      17              : #include "hccl_types.h"
      18              : #include "orion_adapter_rts.h"
      19              : #include "global_mirror_tasks.h"
      20              : #include "error_message_v2.h"
      21              : #include "orion_adapter_hccp.h"
      22              : #include "rdma_handle_manager.h"
      23              : #include "coll_comm.h"
      24              : 
      25              : namespace hcomm {
      26              : using RdmaHandle = void*;
      27              : 
      28              : using GetAicpuCqeErrInfoCallBackHcomm = void (*)(
      29              :     u32 RemoteLocalId, u32 LocDeviceId, uint16_t status, std::string LocalEid, std::string RemoteEid,
      30              :     std::string RemoteInsId); // 获取远端rankId的回调函数类型
      31              : void RegisterGetAicpuCqeErrInfoCallBackHcomm(GetAicpuCqeErrInfoCallBackHcomm p1); // 注册获取远端rankId的回调函数
      32              : 
      33              : using AicpuGetErrStatusVecCallBack = std::vector<std::string> (*)(s32 deviceLogicID);
      34              : void RegisterAicpuGetErrStatusVecCallBack(AicpuGetErrStatusVecCallBack p1);
      35              : 
      36              : class TaskExceptionHost {
      37              : public:
      38              :     static TaskExceptionHost* GetInstance(s32 deviceLogicID);
      39         3055 :     TaskExceptionHost() = default;
      40              :     ~TaskExceptionHost();
      41              : 
      42              :     HcclResult Register(u64 commHandle);
      43              :     HcclResult UnRegister(u64 commHandle);
      44              :     static void ProcessCallback(rtExceptionInfo_t* exceptionInfo);
      45              : 
      46              : private:
      47              :     void Process(rtExceptionInfo_t* exceptionInfo);
      48              :     void HandleAicpuErrorReport(
      49              :         rtExceptionInfo_t* exceptionInfo, const Hccl::ErrorMessageReport& errorMessage,
      50              :         const Hccl::TaskInfo& taskInfo) const;
      51              :     void HandleHostErrorReport(rtExceptionInfo_t* exceptionInfo, const Hccl::TaskInfo& taskInfo) const;
      52              :     void ReportErrorMsg(
      53              :         const Hccl::TaskInfo& exceptionTaskInfo, const std::string& groupRankContent,
      54              :         const Hccl::ErrorMessageReport& errorMessage, rtExceptionInfo_t* exceptionInfo) const;
      55              : 
      56              :     std::string GetGroupRankInfo(const Hccl::TaskInfo& taskInfo) const;
      57              :     void ProcessException(rtExceptionInfo_t* exceptionInfo, const Hccl::TaskInfo& taskInfo);
      58              :     void PrintTaskContextInfo(uint32_t deviceId, uint32_t streamId, uint32_t taskId) const;
      59              :     void PrintUbDfxInfo(rtExceptionInfo_t* exceptionInfo, const Hccl::ErrorMessageReport& errorMessage) const;
      60              :     void PrintGroupErrorMessage(
      61              :         const Hccl::ErrorMessageReport& errorMessage, const Hccl::TaskInfo& exceptionTaskInfo,
      62              :         std::string& groupRankContent, std::string& stageErrInfo) const;
      63              :     void PrintOpDataErrorMessage(
      64              :         u32 deviceId, const Hccl::ErrorMessageReport& errorMessage, std::string& stageErrInfo) const;
      65              :     HcclResult PrintUbRegisters(s32 devLogicId, const RdmaHandle rdmaHandle) const;
      66              :     void ClusterMoniterGetAicpuCqeErrInfo(
      67              :         u32 remoteLocalId, u32 locDeviceId, uint16_t status, std::string localEid, std::string remoteEid,
      68              :         std::string remoteInsId) const;
      69              :     void GetAicpuCqeErrInfo(
      70              :         rtExceptionInfo_t* exceptionInfo, const Hccl::ErrorMessageReport& errorMessage,
      71              :         const Hccl::TaskInfo& taskInfo) const;
      72              :     void GetAicpuCqeErrRemoteLocalIdByRankId(hccl::CollComm* collComm, uint32_t rankid, u32& remoteLocalId) const;
      73              :     void GetAicpuCqeErrNetInstanceByRankId(hccl::CollComm* collComm, uint32_t rankid, std::string& netInstanceId) const;
      74              :     bool ProcessDpuException(const rtExceptionInfo_t* exceptionInfo) const;
      75              : 
      76              : private:
      77              :     std::mutex taskExceptionMutex_;
      78              :     std::unordered_set<u64> CommRegisterMap_;
      79              :     bool hasAicpuReport_{false};
      80              : };
      81              : } // namespace hcomm
      82              : 
      83              : #endif