Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #ifndef CCU_TASKEXCEPTION_H
12 : #define CCU_TASKEXCEPTION_H
13 :
14 : #include <array>
15 : #include "global_mirror_tasks.h"
16 : #include "ccu_error_info.h"
17 : #include "rank_pair.h"
18 : #include "ccu_error_info_v1.h"
19 : #include "ccu_error_info_v2.h"
20 : #include "ccu_rep_base_v1.h"
21 : #include "ccu_rep_context_v1.h"
22 : #include "ccu_device_pub.h"
23 : #include "ccu_jetty_.h"
24 : #include "coll_comm.h"
25 :
26 : namespace hcomm {
27 : class CcuUrmaChannel;
28 : using RdmaHandle = void*;
29 :
30 : using GetCcuCqeErrInfoCallBackHcomm = void (*)(
31 : u32 RemoteLocalId, u32 locDeviceId, uint16_t status, std::string LocalEid, std::string RemoteEid,
32 : std::string RemoteInsId); // 获取远端rankId的回调函数类型
33 : void RegisterGetCcuCqeErrInfoCallBackHcomm(GetCcuCqeErrInfoCallBackHcomm p1); // 注册获取远端rankId的回调函数
34 :
35 : using CcuGetErrStatusVecCallBack = std::vector<std::string> (*)(s32 deviceLogicID);
36 : void RegisterCcuGetErrStatusVecCallBack(CcuGetErrStatusVecCallBack callback);
37 :
38 : class CcuTaskException {
39 : public:
40 : CcuTaskException() = default;
41 : ~CcuTaskException() = default;
42 : static void ProcessCcuException(const rtExceptionInfo_t* exceptionInfo, const Hccl::TaskInfo& taskInfo);
43 :
44 : private:
45 : static HcclResult InitChannelMap(s32 deviceId, u64 ccuKernelHandle);
46 : static std::string GetGroupRankInfo(const Hccl::TaskInfo& taskInfo);
47 :
48 : static HcclResult PrintUbRegisters(s32 devLogicId, RdmaHandle rdmaHandle);
49 : static HcclResult
50 : PrintCcuUbRegisters(const std::vector<CcuErrorInfo>& errorInfos, s32 devLogicId, const Hccl::TaskInfo& taskInfo);
51 : static HcclResult
52 : GetCcuJettys(const CcuErrorInfo& errorInfo, std::pair<CcuChannelInfo, std::vector<CcuJetty*>>& ctx);
53 : static uint16_t GetChannleIdByCcuErrorInfo(const CcuErrorInfo& errorInfo);
54 :
55 : static void PrintCcuErrorInfo(uint32_t deviceId, uint16_t status, const Hccl::TaskInfo& taskInfo);
56 : static void
57 : PrintCcuErrorLog(const std::vector<CcuErrorInfo>& errorInfos, const Hccl::TaskInfo& taskInfo, u32 deviceId);
58 :
59 : // 获取信息的公用接口
60 : static HcclResult GetChannelIdByHandle(const ChannelHandle& channel, uint32_t& channelId);
61 : static HcclResult
62 : GetSignalIdByHandle(const ChannelHandle& channel, uint16_t semIdx, bool isRmtSig, uint32_t& signalId);
63 : static HcclResult GetVariableIdByHandle(const ChannelHandle& channel, uint16_t varIdx, uint32_t& varId);
64 : static HcclResult GetCcuUrmaChannel(const ChannelHandle& channel, CcuUrmaChannel*& channelImpl);
65 :
66 : // 获取ErrorMsg
67 : static std::string
68 : GetCcuErrorMsgByType(const CcuErrorInfo& ccuErrorInfo, const Hccl::TaskInfo& taskInfo, u32 deviceId);
69 : static std::string
70 : GetCcuErrorMsgLoop(const CcuErrorInfo& ccuErrorInfo, const Hccl::TaskInfo& taskInfo, u32 deviceId);
71 : static std::string GetCcuErrorMsgMission(const CcuErrorInfo& ccuErrorInfo);
72 : static std::string GetCcuErrorMsgDefault(const CcuErrorInfo& ccuErrorInfo);
73 : static std::string
74 : GetCcuErrorMsgLoopGroup(const CcuErrorInfo& ccuErrorInfo, const Hccl::TaskInfo& taskInfo, u32 deviceId);
75 : static std::string
76 : GetCcuErrorMsgLocPostSem(const CcuErrorInfo& ccuErrorInfo, const Hccl::TaskInfo& taskInfo, u32 deviceId);
77 : static std::string
78 : GetCcuErrorMsgLocWaitEvent(const CcuErrorInfo& ccuErrorInfo, const Hccl::TaskInfo& taskInfo, u32 deviceId);
79 : static std::string
80 : GetCcuErrorMsgLocWaitNotify(const CcuErrorInfo& ccuErrorInfo, const Hccl::TaskInfo& taskInfo, u32 deviceId);
81 : static std::string
82 : GetCcuErrorMsgRemPostSem(const CcuErrorInfo& ccuErrorInfo, const Hccl::TaskInfo& taskInfo, u32 deviceId);
83 : static std::string
84 : GetCcuErrorMsgRemWaitSem(const CcuErrorInfo& ccuErrorInfo, const Hccl::TaskInfo& taskInfo, u32 deviceId);
85 : static std::string
86 : GetCcuErrorMsgRemPostVar(const CcuErrorInfo& ccuErrorInfo, const Hccl::TaskInfo& taskInfo, u32 deviceId);
87 : static std::string
88 : GetCcuErrorMsgPostSharedSem(const CcuErrorInfo& ccuErrorInfo, const Hccl::TaskInfo& taskInfo, u32 deviceId);
89 : static std::string
90 : GetCcuErrorMsgRead(const CcuErrorInfo& ccuErrorInfo, const Hccl::TaskInfo& taskInfo, u32 deviceId);
91 : static std::string
92 : GetCcuErrorMsgWrite(const CcuErrorInfo& ccuErrorInfo, const Hccl::TaskInfo& taskInfo, u32 deviceId);
93 : static std::string
94 : GetCcuErrorMsgLocalCpy(const CcuErrorInfo& ccuErrorInfo, const Hccl::TaskInfo& taskInfo, u32 deviceId);
95 : static std::string
96 : GetCcuErrorMsgLocalReduce(const CcuErrorInfo& ccuErrorInfo, const Hccl::TaskInfo& taskInfo, u32 deviceId);
97 : static std::string
98 : GetCcuErrorMsgBufRead(const CcuErrorInfo& ccuErrorInfo, const Hccl::TaskInfo& taskInfo, u32 deviceId);
99 : static std::string
100 : GetCcuErrorMsgBufWrite(const CcuErrorInfo& ccuErrorInfo, const Hccl::TaskInfo& taskInfo, u32 deviceId);
101 : static std::string
102 : GetCcuErrorMsgBufLocRead(const CcuErrorInfo& ccuErrorInfo, const Hccl::TaskInfo& taskInfo, u32 deviceId);
103 : static std::string
104 : GetCcuErrorMsgBufLocWrite(const CcuErrorInfo& ccuErrorInfo, const Hccl::TaskInfo& taskInfo, u32 deviceId);
105 : static std::string
106 : GetCcuErrorMsgBufReduce(const CcuErrorInfo& ccuErrorInfo, const Hccl::TaskInfo& taskInfo, u32 deviceId);
107 :
108 : static HcclResult GetCcuChannelHandleById(u16 channelId, u64& channelHandle);
109 : static RankId GetRankIdByChannelId(uint16_t channelId, const Hccl::TaskInfo& taskInfo, u32 deviceId);
110 : static std::pair<Hccl::IpAddress, Hccl::IpAddress>
111 : GetAddrPairByChannelId(uint16_t channelId, const Hccl::TaskInfo& taskInfo, u32 deviceId);
112 : static std::string GetCcuLenErrorMsg(const uint64_t len);
113 : static HcclResult GenErrorInfoLoopGroup(
114 : const ErrorInfoBase& baseInfo, std::shared_ptr<CcuRep::CcuRepBase> repBase, CcuRep::CcuRepContext& ctx,
115 : std::vector<CcuErrorInfo>& errorInfo);
116 :
117 : // 生成Error Info
118 : static void GenErrorInfoByRepType(
119 : const ErrorInfoBase& baseInfo, std::shared_ptr<CcuRep::CcuRepBase> repBase,
120 : std::vector<CcuErrorInfo>& errorInfo);
121 : static void GenErrorInfoLocRecordEvent(
122 : const ErrorInfoBase& baseInfo, std::shared_ptr<CcuRep::CcuRepBase> repBase,
123 : std::vector<CcuErrorInfo>& errorInfo);
124 : static void GenErrorInfoLocWaitNotify(
125 : const ErrorInfoBase& baseInfo, std::shared_ptr<CcuRep::CcuRepBase> repBase,
126 : std::vector<CcuErrorInfo>& errorInfo);
127 : static void GenErrorInfoLocWaitEvent(
128 : const ErrorInfoBase& baseInfo, std::shared_ptr<CcuRep::CcuRepBase> repBase,
129 : std::vector<CcuErrorInfo>& errorInfo);
130 : static void GenErrorInfoRemPostSem(
131 : const ErrorInfoBase& baseInfo, std::shared_ptr<CcuRep::CcuRepBase> repBase,
132 : std::vector<CcuErrorInfo>& errorInfo);
133 : static void GenErrorInfoRemWaitSem(
134 : const ErrorInfoBase& baseInfo, std::shared_ptr<CcuRep::CcuRepBase> repBase,
135 : std::vector<CcuErrorInfo>& errorInfo);
136 : static void GenErrorInfoRemPostVar(
137 : const ErrorInfoBase& baseInfo, std::shared_ptr<CcuRep::CcuRepBase> repBase,
138 : std::vector<CcuErrorInfo>& errorInfo);
139 : static void GenErrorInfoPostSharedSem(
140 : const ErrorInfoBase& baseInfo, std::shared_ptr<CcuRep::CcuRepBase> repBase,
141 : std::vector<CcuErrorInfo>& errorInfo);
142 : static void GenErrorInfoRead(
143 : const ErrorInfoBase& baseInfo, std::shared_ptr<CcuRep::CcuRepBase> repBase,
144 : std::vector<CcuErrorInfo>& errorInfo);
145 : static void GenErrorInfoWrite(
146 : const ErrorInfoBase& baseInfo, std::shared_ptr<CcuRep::CcuRepBase> repBase,
147 : std::vector<CcuErrorInfo>& errorInfo);
148 : static void GenErrorInfoLocalCpy(
149 : const ErrorInfoBase& baseInfo, std::shared_ptr<CcuRep::CcuRepBase> repBase,
150 : std::vector<CcuErrorInfo>& errorInfo);
151 : static void GenErrorInfoLocalReduce(
152 : const ErrorInfoBase& baseInfo, std::shared_ptr<CcuRep::CcuRepBase> repBase,
153 : std::vector<CcuErrorInfo>& errorInfo);
154 : static void GenErrorInfoBufRead(
155 : const ErrorInfoBase& baseInfo, std::shared_ptr<CcuRep::CcuRepBase> repBase,
156 : std::vector<CcuErrorInfo>& errorInfo);
157 : static void GenErrorInfoBufWrite(
158 : const ErrorInfoBase& baseInfo, std::shared_ptr<CcuRep::CcuRepBase> repBase,
159 : std::vector<CcuErrorInfo>& errorInfo);
160 : static void GenErrorInfoBufLocRead(
161 : const ErrorInfoBase& baseInfo, std::shared_ptr<CcuRep::CcuRepBase> repBase,
162 : std::vector<CcuErrorInfo>& errorInfo);
163 : static void GenErrorInfoBufLocWrite(
164 : const ErrorInfoBase& baseInfo, std::shared_ptr<CcuRep::CcuRepBase> repBase,
165 : std::vector<CcuErrorInfo>& errorInfo);
166 : static void GenErrorInfoBufReduce(
167 : const ErrorInfoBase& baseInfo, std::shared_ptr<CcuRep::CcuRepBase> repBase,
168 : std::vector<CcuErrorInfo>& errorInfo);
169 : static void GenErrorInfoDefault(
170 : const ErrorInfoBase& baseInfo, std::shared_ptr<CcuRep::CcuRepBase> repBase,
171 : std::vector<CcuErrorInfo>& errorInfo);
172 :
173 : static uint64_t GetCcuXnValue(int32_t deviceId, uint32_t dieId, uint32_t xnId);
174 : static HcclResult
175 : GenErrorInfoLoop(const ErrorInfoBase& baseInfo, CcuRep::CcuRepContext& ctx, std::vector<CcuErrorInfo>& errorInfo);
176 : static void GenStatusInfo(const ErrorInfoBase& baseInfo, std::vector<CcuErrorInfo>& errorInfo);
177 : static HcclResult GetCcuMissionContextRaw(
178 : int32_t deviceId, uint32_t dieId, uint32_t missionId, uint8_t* buf, size_t bufLen, size_t& copiedLen);
179 : static HcclResult GetCcuLoopContextRaw(
180 : int32_t deviceId, uint32_t dieId, uint32_t loopCtxId, uint8_t* buf, size_t bufLen, size_t& copiedLen);
181 : static HcclResult GetCcuErrorMsg(
182 : int32_t deviceId, uint16_t missionStatus, const Hccl::ParaCcu& ccuTaskParam,
183 : std::vector<CcuErrorInfo>& errorInfo);
184 : static void PrintPanicLogInfo(const uint8_t* panicLog);
185 : static uint16_t GetCcuCKEValue(int32_t deviceId, uint32_t dieId, uint32_t ckeId);
186 :
187 : static uint64_t GetCcuGSAValue(int32_t deviceId, uint32_t dieId, uint32_t gsaId);
188 5 : static uint16_t GetMSIdPerDie(uint16_t msId) { return msId & 0x7fff; }
189 : static void GetCcuCqeErrorInfo(
190 : const CcuErrorInfo& ccuErrorInfo, const Hccl::TaskInfo& taskInfo, u32 locDeviceId, uint8_t missionStatus);
191 : static void GetCcuCqeErrRemoteLocalIdByRankId(hccl::CollComm* collComm, uint32_t rankid, u32& remoteLocalId);
192 : static void GetCcuCqeErrNetInstanceByRankId(hccl::CollComm* collComm, uint32_t rankid, std::string& netInstanceId);
193 : static void ClusterMoniterGetCcuCqeErrInfo(
194 : u32 RemoteDeviceId, u32 locDeviceId, uint16_t status, std::string LocalEid, std::string RemoteEid,
195 : std::string RemoteInsId);
196 : };
197 : } // namespace hcomm
198 :
199 : #endif
|
