Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 : #ifndef CCU_TASKEXCEPTION_H
11 : #define CCU_TASKEXCEPTION_H
12 :
13 : #include <array>
14 : #include "global_mirror_tasks.h"
15 : #include "ccu_error_info.h"
16 : #include "rank_pair.h"
17 : #include "ccu_error_info_v1.h"
18 : #include "ccu_error_info_v2.h"
19 : #include "ccu_rep_base_v1.h"
20 : #include "ccu_rep_context_v1.h"
21 : #include "ccu_device_pub.h"
22 : #include "ccu_jetty_.h"
23 : #include "coll_comm.h"
24 :
25 : namespace hcomm {
26 : class CcuUrmaChannel;
27 : using RdmaHandle = void*;
28 :
29 : using GetCcuCqeErrInfoCallBackHcomm = void (*)(
30 : u32 RemoteLocalId, u32 locDeviceId, uint16_t status, std::string LocalEid, std::string RemoteEid,
31 : std::string RemoteInsId); // 获取远端rankId的回调函数类型
32 : void RegisterGetCcuCqeErrInfoCallBackHcomm(GetCcuCqeErrInfoCallBackHcomm p1); // 注册获取远端rankId的回调函数
33 :
34 : using CcuGetErrStatusVecCallBack = std::vector<std::string> (*)(s32 deviceLogicID);
35 : void RegisterCcuGetErrStatusVecCallBack(CcuGetErrStatusVecCallBack callback);
36 :
37 : class CcuTaskException {
38 : public:
39 : CcuTaskException() = default;
40 : ~CcuTaskException() = default;
41 : static void ProcessCcuException(const rtExceptionInfo_t* exceptionInfo, const Hccl::TaskInfo& taskInfo);
42 :
43 : private:
44 : static HcclResult InitChannelMap(s32 deviceId, u64 ccuKernelHandle);
45 : static std::string GetGroupRankInfo(const Hccl::TaskInfo& taskInfo);
46 :
47 : static HcclResult PrintUbRegisters(s32 devLogicId, RdmaHandle rdmaHandle);
48 : static HcclResult
49 : PrintCcuUbRegisters(const std::vector<CcuErrorInfo>& errorInfos, s32 devLogicId, const Hccl::TaskInfo& taskInfo);
50 : static HcclResult
51 : GetCcuJettys(const CcuErrorInfo& errorInfo, std::pair<CcuChannelInfo, std::vector<CcuJetty*>>& ctx);
52 : static uint16_t GetChannleIdByCcuErrorInfo(const CcuErrorInfo& errorInfo);
53 :
54 : static void PrintCcuErrorInfo(uint32_t deviceId, uint16_t status, const Hccl::TaskInfo& taskInfo);
55 : static void
56 : PrintCcuErrorLog(const std::vector<CcuErrorInfo>& errorInfos, const Hccl::TaskInfo& taskInfo, u32 deviceId);
57 :
58 : // 获取信息的公用接口
59 : static HcclResult GetChannelIdByHandle(const ChannelHandle& channel, uint32_t& channelId);
60 : static HcclResult
61 : GetSignalIdByHandle(const ChannelHandle& channel, uint16_t semIdx, bool isRmtSig, uint32_t& signalId);
62 : static HcclResult GetVariableIdByHandle(const ChannelHandle& channel, uint16_t varIdx, uint32_t& varId);
63 : static HcclResult GetCcuUrmaChannel(const ChannelHandle& channel, CcuUrmaChannel*& channelImpl);
64 :
65 : // 获取ErrorMsg
66 : static std::string
67 : GetCcuErrorMsgByType(const CcuErrorInfo& ccuErrorInfo, const Hccl::TaskInfo& taskInfo, u32 deviceId);
68 : static std::string
69 : GetCcuErrorMsgLoop(const CcuErrorInfo& ccuErrorInfo, const Hccl::TaskInfo& taskInfo, u32 deviceId);
70 : static std::string GetCcuErrorMsgMission(const CcuErrorInfo& ccuErrorInfo);
71 : static std::string GetCcuErrorMsgDefault(const CcuErrorInfo& ccuErrorInfo);
72 : static std::string
73 : GetCcuErrorMsgLoopGroup(const CcuErrorInfo& ccuErrorInfo, const Hccl::TaskInfo& taskInfo, u32 deviceId);
74 : static std::string
75 : GetCcuErrorMsgLocPostSem(const CcuErrorInfo& ccuErrorInfo, const Hccl::TaskInfo& taskInfo, u32 deviceId);
76 : static std::string
77 : GetCcuErrorMsgLocWaitEvent(const CcuErrorInfo& ccuErrorInfo, const Hccl::TaskInfo& taskInfo, u32 deviceId);
78 : static std::string
79 : GetCcuErrorMsgLocWaitNotify(const CcuErrorInfo& ccuErrorInfo, const Hccl::TaskInfo& taskInfo, u32 deviceId);
80 : static std::string
81 : GetCcuErrorMsgRemPostSem(const CcuErrorInfo& ccuErrorInfo, const Hccl::TaskInfo& taskInfo, u32 deviceId);
82 : static std::string
83 : GetCcuErrorMsgRemWaitSem(const CcuErrorInfo& ccuErrorInfo, const Hccl::TaskInfo& taskInfo, u32 deviceId);
84 : static std::string
85 : GetCcuErrorMsgRemPostVar(const CcuErrorInfo& ccuErrorInfo, const Hccl::TaskInfo& taskInfo, u32 deviceId);
86 : static std::string
87 : GetCcuErrorMsgPostSharedSem(const CcuErrorInfo& ccuErrorInfo, const Hccl::TaskInfo& taskInfo, u32 deviceId);
88 : static std::string
89 : GetCcuErrorMsgRead(const CcuErrorInfo& ccuErrorInfo, const Hccl::TaskInfo& taskInfo, u32 deviceId);
90 : static std::string
91 : GetCcuErrorMsgWrite(const CcuErrorInfo& ccuErrorInfo, const Hccl::TaskInfo& taskInfo, u32 deviceId);
92 : static std::string
93 : GetCcuErrorMsgLocalCpy(const CcuErrorInfo& ccuErrorInfo, const Hccl::TaskInfo& taskInfo, u32 deviceId);
94 : static std::string
95 : GetCcuErrorMsgLocalReduce(const CcuErrorInfo& ccuErrorInfo, const Hccl::TaskInfo& taskInfo, u32 deviceId);
96 : static std::string
97 : GetCcuErrorMsgBufRead(const CcuErrorInfo& ccuErrorInfo, const Hccl::TaskInfo& taskInfo, u32 deviceId);
98 : static std::string
99 : GetCcuErrorMsgBufWrite(const CcuErrorInfo& ccuErrorInfo, const Hccl::TaskInfo& taskInfo, u32 deviceId);
100 : static std::string
101 : GetCcuErrorMsgBufLocRead(const CcuErrorInfo& ccuErrorInfo, const Hccl::TaskInfo& taskInfo, u32 deviceId);
102 : static std::string
103 : GetCcuErrorMsgBufLocWrite(const CcuErrorInfo& ccuErrorInfo, const Hccl::TaskInfo& taskInfo, u32 deviceId);
104 : static std::string
105 : GetCcuErrorMsgBufReduce(const CcuErrorInfo& ccuErrorInfo, const Hccl::TaskInfo& taskInfo, u32 deviceId);
106 :
107 : static HcclResult GetCcuChannelHandleById(u16 channelId, u64& channelHandle);
108 : static RankId GetRankIdByChannelId(uint16_t channelId, const Hccl::TaskInfo& taskInfo, u32 deviceId);
109 : static std::pair<Hccl::IpAddress, Hccl::IpAddress>
110 : GetAddrPairByChannelId(uint16_t channelId, const Hccl::TaskInfo& taskInfo, u32 deviceId);
111 : static std::string GetCcuLenErrorMsg(const uint64_t len);
112 : static HcclResult GenErrorInfoLoopGroup(
113 : const ErrorInfoBase& baseInfo, std::shared_ptr<CcuRep::CcuRepBase> repBase, CcuRep::CcuRepContext& ctx,
114 : std::vector<CcuErrorInfo>& errorInfo);
115 :
116 : // 生成Error Info
117 : static void GenErrorInfoByRepType(
118 : const ErrorInfoBase& baseInfo, std::shared_ptr<CcuRep::CcuRepBase> repBase,
119 : std::vector<CcuErrorInfo>& errorInfo);
120 : static void GenErrorInfoLocRecordEvent(
121 : const ErrorInfoBase& baseInfo, std::shared_ptr<CcuRep::CcuRepBase> repBase,
122 : std::vector<CcuErrorInfo>& errorInfo);
123 : static void GenErrorInfoLocWaitNotify(
124 : const ErrorInfoBase& baseInfo, std::shared_ptr<CcuRep::CcuRepBase> repBase,
125 : std::vector<CcuErrorInfo>& errorInfo);
126 : static void GenErrorInfoLocWaitEvent(
127 : const ErrorInfoBase& baseInfo, std::shared_ptr<CcuRep::CcuRepBase> repBase,
128 : std::vector<CcuErrorInfo>& errorInfo);
129 : static void GenErrorInfoRemPostSem(
130 : const ErrorInfoBase& baseInfo, std::shared_ptr<CcuRep::CcuRepBase> repBase,
131 : std::vector<CcuErrorInfo>& errorInfo);
132 : static void GenErrorInfoRemWaitSem(
133 : const ErrorInfoBase& baseInfo, std::shared_ptr<CcuRep::CcuRepBase> repBase,
134 : std::vector<CcuErrorInfo>& errorInfo);
135 : static void GenErrorInfoRemPostVar(
136 : const ErrorInfoBase& baseInfo, std::shared_ptr<CcuRep::CcuRepBase> repBase,
137 : std::vector<CcuErrorInfo>& errorInfo);
138 : static void GenErrorInfoPostSharedSem(
139 : const ErrorInfoBase& baseInfo, std::shared_ptr<CcuRep::CcuRepBase> repBase,
140 : std::vector<CcuErrorInfo>& errorInfo);
141 : static void GenErrorInfoRead(
142 : const ErrorInfoBase& baseInfo, std::shared_ptr<CcuRep::CcuRepBase> repBase,
143 : std::vector<CcuErrorInfo>& errorInfo);
144 : static void GenErrorInfoWrite(
145 : const ErrorInfoBase& baseInfo, std::shared_ptr<CcuRep::CcuRepBase> repBase,
146 : std::vector<CcuErrorInfo>& errorInfo);
147 : static void GenErrorInfoLocalCpy(
148 : const ErrorInfoBase& baseInfo, std::shared_ptr<CcuRep::CcuRepBase> repBase,
149 : std::vector<CcuErrorInfo>& errorInfo);
150 : static void GenErrorInfoLocalReduce(
151 : const ErrorInfoBase& baseInfo, std::shared_ptr<CcuRep::CcuRepBase> repBase,
152 : std::vector<CcuErrorInfo>& errorInfo);
153 : static void GenErrorInfoBufRead(
154 : const ErrorInfoBase& baseInfo, std::shared_ptr<CcuRep::CcuRepBase> repBase,
155 : std::vector<CcuErrorInfo>& errorInfo);
156 : static void GenErrorInfoBufWrite(
157 : const ErrorInfoBase& baseInfo, std::shared_ptr<CcuRep::CcuRepBase> repBase,
158 : std::vector<CcuErrorInfo>& errorInfo);
159 : static void GenErrorInfoBufLocRead(
160 : const ErrorInfoBase& baseInfo, std::shared_ptr<CcuRep::CcuRepBase> repBase,
161 : std::vector<CcuErrorInfo>& errorInfo);
162 : static void GenErrorInfoBufLocWrite(
163 : const ErrorInfoBase& baseInfo, std::shared_ptr<CcuRep::CcuRepBase> repBase,
164 : std::vector<CcuErrorInfo>& errorInfo);
165 : static void GenErrorInfoBufReduce(
166 : const ErrorInfoBase& baseInfo, std::shared_ptr<CcuRep::CcuRepBase> repBase,
167 : std::vector<CcuErrorInfo>& errorInfo);
168 : static void GenErrorInfoDefault(
169 : const ErrorInfoBase& baseInfo, std::shared_ptr<CcuRep::CcuRepBase> repBase,
170 : std::vector<CcuErrorInfo>& errorInfo);
171 :
172 : static uint64_t GetCcuXnValue(int32_t deviceId, uint32_t dieId, uint32_t xnId);
173 : static HcclResult
174 : GenErrorInfoLoop(const ErrorInfoBase& baseInfo, CcuRep::CcuRepContext& ctx, std::vector<CcuErrorInfo>& errorInfo);
175 : static void GenStatusInfo(const ErrorInfoBase& baseInfo, std::vector<CcuErrorInfo>& errorInfo);
176 : static HcclResult GetCcuMissionContextRaw(
177 : int32_t deviceId, uint32_t dieId, uint32_t missionId, uint8_t* buf, size_t bufLen, size_t& copiedLen);
178 : static HcclResult GetCcuLoopContextRaw(
179 : int32_t deviceId, uint32_t dieId, uint32_t loopCtxId, uint8_t* buf, size_t bufLen, size_t& copiedLen);
180 : static HcclResult GetCcuErrorMsg(
181 : int32_t deviceId, uint16_t missionStatus, const Hccl::ParaCcu& ccuTaskParam,
182 : std::vector<CcuErrorInfo>& errorInfo);
183 : static void PrintPanicLogInfo(const uint8_t* panicLog);
184 : static uint16_t GetCcuCKEValue(int32_t deviceId, uint32_t dieId, uint32_t ckeId);
185 :
186 : static uint64_t GetCcuGSAValue(int32_t deviceId, uint32_t dieId, uint32_t gsaId);
187 5 : static uint16_t GetMSIdPerDie(uint16_t msId) { return msId & 0x7fff; }
188 : static void GetCcuCqeErrorInfo(
189 : const CcuErrorInfo& ccuErrorInfo, const Hccl::TaskInfo& taskInfo, u32 locDeviceId, uint8_t missionStatus);
190 : static void GetCcuCqeErrRemoteLocalIdByRankId(hccl::CollComm* collComm, uint32_t rankid, u32& remoteLocalId);
191 : static void GetCcuCqeErrNetInstanceByRankId(hccl::CollComm* collComm, uint32_t rankid, std::string& netInstanceId);
192 : static void ClusterMoniterGetCcuCqeErrInfo(
193 : u32 RemoteDeviceId, u32 locDeviceId, uint16_t status, std::string LocalEid, std::string RemoteEid,
194 : std::string RemoteInsId);
195 : };
196 : } // namespace hcomm
197 :
198 : #endif
|
