Line data    Source code

       1              : /**
       2              :  * Copyright (c) 2025 Huawei Technologies Co., Ltd.
       3              :  * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
       4              :  * CANN Open Software License Agreement Version 2.0 (the "License").
       5              :  * Please refer to the License for details. You may not use this file except in compliance with the License.
       6              :  * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
       7              :  * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
       8              :  * See LICENSE in the root of the software repository for the full text of the License.
       9              :  */
      10              : 
      11              : #ifndef HCCL_COMM_TASKEXCEPTION_H
      12              : #define HCCL_COMM_TASKEXCEPTION_H
      13              : 
      14              : #include <array>
      15              : #include <map>
      16              : #include "types.h"
      17              : #include "hccl_types.h"
      18              : #include "orion_adapter_rts.h"
      19              : #include "global_mirror_tasks.h"
      20              : #include "error_message_v2.h"
      21              : #include "orion_adapter_hccp.h"
      22              : #include "rdma_handle_manager.h"
      23              : #include "coll_comm.h"
      24              : 
      25              : namespace hcomm {
      26              : using RdmaHandle = void*;
      27              : 
      28              : using GetAicpuCqeErrInfoCallBackHcomm = void (*)(
      29              :     u32 RemoteLocalId, u32 LocDeviceId, uint16_t status, std::string LocalEid, std::string RemoteEid,
      30              :     std::string RemoteInsId); // 获取远端rankId的回调函数类型
      31              : void RegisterGetAicpuCqeErrInfoCallBackHcomm(GetAicpuCqeErrInfoCallBackHcomm p1); // 注册获取远端rankId的回调函数
      32              : 
      33              : using AicpuGetErrStatusVecCallBack = std::vector<std::string> (*)(s32 deviceLogicID);
      34              : void RegisterAicpuGetErrStatusVecCallBack(AicpuGetErrStatusVecCallBack p1);
      35              : 
      36              : class TaskExceptionHost {
      37              : public:
      38              :     static TaskExceptionHost* GetInstance(s32 deviceLogicID);
      39         3380 :     TaskExceptionHost() = default;
      40              :     ~TaskExceptionHost();
      41              : 
      42              :     HcclResult Register(u64 commHandle);
      43              :     HcclResult UnRegister(u64 commHandle);
      44              :     static void ProcessCallback(rtExceptionInfo_t* exceptionInfo);
      45              : 
      46              : private:
      47              :     void Process(rtExceptionInfo_t* exceptionInfo);
      48              :     void HandleAicpuErrorReport(
      49              :         rtExceptionInfo_t* exceptionInfo, const Hccl::ErrorMessageReport& errorMessage,
      50              :         const Hccl::TaskInfo& taskInfo) const;
      51              :     void HandleHostErrorReport(rtExceptionInfo_t* exceptionInfo, const Hccl::TaskInfo& taskInfo) const;
      52              :     void ReportErrorMsg(
      53              :         const Hccl::TaskInfo& exceptionTaskInfo, const std::string& groupRankContent,
      54              :         const Hccl::ErrorMessageReport& errorMessage, rtExceptionInfo_t* exceptionInfo) const;
      55              :     void ReportEI0018Error(const Hccl::TaskInfo& exceptionTaskInfo, const Hccl::ErrorMessageReport& errorMessage) const;
      56              :     bool ShouldReportError() const;
      57              : 
      58              :     std::string GetGroupRankInfo(const Hccl::TaskInfo& taskInfo) const;
      59              :     void ProcessException(rtExceptionInfo_t* exceptionInfo, const Hccl::TaskInfo& taskInfo);
      60              :     void PrintTaskContextInfo(uint32_t deviceId, uint32_t streamId, uint32_t taskId) const;
      61              :     void PrintUbDfxInfo(rtExceptionInfo_t* exceptionInfo, const Hccl::ErrorMessageReport& errorMessage) const;
      62              :     void PrintGroupErrorMessage(
      63              :         const Hccl::ErrorMessageReport& errorMessage, const Hccl::TaskInfo& exceptionTaskInfo,
      64              :         std::string& groupRankContent, std::string& stageErrInfo) const;
      65              :     void PrintOpDataErrorMessage(
      66              :         u32 deviceId, const Hccl::ErrorMessageReport& errorMessage, std::string& stageErrInfo) const;
      67              :     HcclResult PrintUbRegisters(s32 devLogicId, const RdmaHandle rdmaHandle) const;
      68              :     void ClusterMoniterGetAicpuCqeErrInfo(
      69              :         u32 remoteLocalId, u32 locDeviceId, uint16_t status, std::string localEid, std::string remoteEid,
      70              :         std::string remoteInsId) const;
      71              :     void GetAicpuCqeErrInfo(
      72              :         rtExceptionInfo_t* exceptionInfo, const Hccl::ErrorMessageReport& errorMessage,
      73              :         const Hccl::TaskInfo& taskInfo) const;
      74              :     void GetAicpuCqeErrRemoteLocalIdByRankId(hccl::CollComm* collComm, uint32_t rankid, u32& remoteLocalId) const;
      75              :     void GetAicpuCqeErrNetInstanceByRankId(hccl::CollComm* collComm, uint32_t rankid, std::string& netInstanceId) const;
      76              :     bool ProcessDpuException(const rtExceptionInfo_t* exceptionInfo) const;
      77              : 
      78              : private:
      79              :     std::mutex taskExceptionMutex_;
      80              :     std::unordered_set<u64> CommRegisterMap_;
      81              :     bool hasAicpuReport_{false};
      82              :     mutable std::mutex ei0002ReportedMutex_;
      83              :     mutable bool ei0002Reported_{false};
      84              : };
      85              : } // namespace hcomm
      86              : 
      87              : #endif