Line data    Source code

       1              : /**
       2              :  * Copyright (c) 2025 Huawei Technologies Co., Ltd.
       3              :  * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
       4              :  * CANN Open Software License Agreement Version 2.0 (the "License").
       5              :  * Please refer to the License for details. You may not use this file except in compliance with the License.
       6              :  * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
       7              :  * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
       8              :  * See LICENSE in the root of the software repository for the full text of the License.
       9              :  */
      10              : #ifndef HCCL_COMM_TASKEXCEPTION_H
      11              : #define HCCL_COMM_TASKEXCEPTION_H
      12              : 
      13              : #include <array>
      14              : #include <map>
      15              : #include "types.h"
      16              : #include "hccl_types.h"
      17              : #include "orion_adapter_rts.h"
      18              : #include "global_mirror_tasks.h"
      19              : #include "error_message_v2.h"
      20              : #include "orion_adapter_hccp.h"
      21              : #include "rdma_handle_manager.h"
      22              : #include "coll_comm.h"
      23              : 
      24              : namespace hcomm {
      25              : using RdmaHandle = void*;
      26              : 
      27              : using GetAicpuCqeErrInfoCallBackHcomm = void (*)(u32 RemoteLocalId, u32 LocDeviceId, uint16_t status, std::string LocalEid, std::string RemoteEid, std::string RemoteInsId); // 获取远端rankId的回调函数类型
      28              : void RegisterGetAicpuCqeErrInfoCallBackHcomm(GetAicpuCqeErrInfoCallBackHcomm p1); // 注册获取远端rankId的回调函数   
      29              : 
      30              : using AicpuGetErrStatusVecCallBack = std::vector<std::string> (*)(s32 deviceLogicID);
      31              : void RegisterAicpuGetErrStatusVecCallBack(AicpuGetErrStatusVecCallBack p1);
      32              : 
      33              : struct DpuTaskexceptionParams {
      34              :     HcclResult ret;
      35              :     uint32_t   devId; // todo 好像无用
      36              :     char       commId[COMM_NAME_MAX_LENGTH]; // todo好像无用
      37              : };
      38              : 
      39              : class TaskExceptionHost {
      40              : public:
      41              :     static TaskExceptionHost* GetInstance(s32 deviceLogicID);
      42         2795 :     TaskExceptionHost() = default;
      43              :     ~TaskExceptionHost();
      44              : 
      45              :     HcclResult Register(u64 commHandle);
      46              :     HcclResult UnRegister(u64 commHandle);
      47              :     static void ProcessCallback(rtExceptionInfo_t *exceptionInfo);
      48              : 
      49              : private:
      50              :     void Process(rtExceptionInfo_t *exceptionInfo);
      51              :     void HandleAicpuErrorReport(rtExceptionInfo_t *exceptionInfo, const Hccl::ErrorMessageReport &errorMessage, const Hccl::TaskInfo &taskInfo) const;
      52              :     void HandleHostErrorReport(rtExceptionInfo_t *exceptionInfo, const Hccl::TaskInfo &taskInfo) const;
      53              :     void ReportErrorMsg(const Hccl::TaskInfo &exceptionTaskInfo, const std::string &groupRankContent,
      54              :         const Hccl::ErrorMessageReport &errorMessage, rtExceptionInfo_t *exceptionInfo) const;
      55              : 
      56              :     std::string GetGroupRankInfo(const Hccl::TaskInfo& taskInfo) const;
      57              :     void ProcessException(rtExceptionInfo_t* exceptionInfo, const Hccl::TaskInfo& taskInfo);
      58              :     void PrintTaskContextInfo(uint32_t deviceId, uint32_t streamId, uint32_t taskId) const;
      59              :     void PrintUbDfxInfo(rtExceptionInfo_t *exceptionInfo, const Hccl::ErrorMessageReport &errorMessage) const;
      60              :     void PrintGroupErrorMessage(const Hccl::ErrorMessageReport &errorMessage, const Hccl::TaskInfo &exceptionTaskInfo, std::string &groupRankContent, std::string &stageErrInfo) const;
      61              :     void PrintOpDataErrorMessage(u32 deviceId, const Hccl::ErrorMessageReport &errorMessage, std::string &stageErrInfo) const;
      62              :     HcclResult PrintUbRegisters(s32 devLogicId, const RdmaHandle rdmaHandle) const;
      63              :     void ClusterMoniterGetAicpuCqeErrInfo(u32 remoteLocalId, u32 locDeviceId, uint16_t status, std::string localEid, std::string remoteEid, std::string remoteInsId) const;
      64              :     void GetAicpuCqeErrInfo(rtExceptionInfo_t* exceptionInfo, const Hccl::ErrorMessageReport &errorMessage, const Hccl::TaskInfo& taskInfo) const;
      65              :     void GetAicpuCqeErrRemoteLocalIdByRankId(hccl::CollComm* collComm, uint32_t rankid, u32 &remoteLocalId) const;
      66              :     void GetAicpuCqeErrNetInstanceByRankId(hccl::CollComm* collComm, uint32_t rankid, std::string &netInstanceId) const;
      67              :     bool ProcessDpuException(const rtExceptionInfo_t* exceptionInfo) const;
      68              : 
      69              : private:
      70              :     std::mutex taskExceptionMutex_;
      71              :     std::unordered_set<u64> CommRegisterMap_;
      72              :     bool hasAicpuReport_{false};
      73              : };
      74              : } // namespace hccl
      75              : 
      76              : #endif