Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #ifndef HCCL_TASK_EXCEPTION_HANDLER_PUB_H
12 : #define HCCL_TASK_EXCEPTION_HANDLER_PUB_H
13 : #include <map>
14 : #include <queue>
15 : #include <mutex>
16 : #include <array>
17 : #include <list>
18 : #include <functional>
19 : #include "rt_external.h"
20 : #include "acl/acl_rt.h"
21 : #include "profiler_base_pub.h"
22 : #include "aicpu_operator_pub.h"
23 : namespace hccl {
24 17 : inline void PrintBaseErrorLog(const std::string& stageErrInfo, const std::string& baseInfo)
25 : {
26 17 : HCCL_ERROR("%sTask run failed, base information is %s", stageErrInfo.c_str(), baseInfo.c_str());
27 17 : }
28 :
29 14 : inline void PrintParaErrorLog(const std::string& stageErrInfo, const std::string& paraInfoStr, const std::string& tag)
30 : {
31 14 : HCCL_ERROR(
32 : "%sTask run failed, para information is %s, tag[%s].", stageErrInfo.c_str(), paraInfoStr.c_str(), tag.c_str());
33 14 : }
34 :
35 8 : inline void PrintOpDataErrorLog(const std::string& stageErrInfo, const std::string& opDataContent)
36 : {
37 8 : HCCL_ERROR("%sTask run failed, opData information is %s", stageErrInfo.c_str(), opDataContent.c_str());
38 8 : }
39 :
40 : inline void
41 12 : PrintGroupErrorLog(const std::string& stageErrInfo, const std::string& groupRankContent, const std::string& tag)
42 : {
43 12 : HCCL_ERROR(
44 : "%sTask run failed, groupRank information is %s, tag[%s].", stageErrInfo.c_str(), groupRankContent.c_str(),
45 : tag.c_str());
46 12 : }
47 :
48 3 : inline void PrintContextErrorLog(const std::string& stageErrInfo, const std::string& ctxBaseInfo)
49 : {
50 3 : HCCL_ERROR("%sTask run failed, context base information is %s", stageErrInfo.c_str(), ctxBaseInfo.c_str());
51 3 : }
52 :
53 : struct ParaDMA {
54 : const void* src;
55 : const void* dst;
56 : std::size_t size;
57 : u64 notifyID;
58 : LinkType linkType;
59 : u32 remoteUserRank;
60 : };
61 :
62 : struct ParaReduce {
63 : const void* src;
64 : const void* dst;
65 : std::size_t size;
66 : HcclReduceOp op;
67 : HcclDataType dataType;
68 : LinkType linkType;
69 : u32 remoteUserRank;
70 : };
71 : struct ParaNotify {
72 : u64 notifyID;
73 : s32 stage; // 用于标识stream间同步时所在的stage， 非用于stream同步的默认为-1
74 : u32 remoteUserRank;
75 : };
76 : struct ParaAiv {
77 : HcclCMDType cmdType;
78 : u32 tag;
79 : u64 size;
80 : u32 numBlocks;
81 : u32 rankSize;
82 : s32 aivRdmaStep;
83 : void* flagMem;
84 : u32 rank;
85 : bool isOpbase;
86 : };
87 : struct TaskInfo {
88 : u32 streamID;
89 : u32 taskID;
90 : std::string tag;
91 : TaskType taskType;
92 : bool isAlgInfo;
93 : AlgType algType;
94 : u32 index;
95 : union {
96 : ParaDMA DMA; // taskType = SDMA/RDMA使用, 包括rtRDMASend写notify
97 : ParaReduce Reduce; // taskType = inline/CCE Reduce使用
98 : ParaNotify Notify; // taskType = Noitfy Record/Wait使用
99 : ParaAiv Aiv; // taskType = Aiv 使用
100 : } taskPara;
101 : TaskInfo(
102 : u32& streamID, u32& taskID, std::string& tag, TaskType& taskType, AlgType& algType, u32& index,
103 : const TaskParaDMA& para);
104 : TaskInfo(
105 : u32& streamID, u32& taskID, std::string& tag, TaskType& taskType, AlgType& algType, u32& index,
106 : const TaskParaReduce& para);
107 : TaskInfo(
108 : u32& streamID, u32& taskID, std::string& tag, TaskType& taskType, AlgType& algType, u32& index,
109 : const TaskParaNotify& para);
110 : TaskInfo(u32& streamID, u32& taskID, std::string& tag, const TaskParaAiv& para);
111 : std::string GetBaseInfoStr(); // 防止tag字符串过长，base信息和para信息分开打印
112 : std::string GetParaInfoStr();
113 : std::string GetParaDMA();
114 : std::string GetParaReduce();
115 : std::string GetParaNotify();
116 : std::string GetParaAiv();
117 : std::string GetRankInfo();
118 : std::string GetNotifyInfo();
119 : u32 GetRemoteUserRank();
120 : };
121 : struct FFTSOpInfo {
122 : u32 streamID;
123 : u32 taskID;
124 : std::shared_ptr<char> tag;
125 : AlgType algType;
126 : u32 index;
127 : std::shared_ptr<char> descBuf = nullptr;
128 : size_t descBufLen = 0;
129 : std::string GetBaseInfoStr();
130 : };
131 :
132 : struct CtxInfo {
133 : TaskType taskType;
134 : AlgType algType;
135 : u32 index;
136 : union {
137 : ParaDMA DMA; // taskType = SDMA/RDMA使用, 包括rtRDMASend写notify
138 : ParaReduce Reduce; // taskType = inline/CCE Reduce使用
139 : ParaNotify Notify; // taskType = Noitfy Record/Wait使用
140 : } ctxPara;
141 : CtxInfo(TaskType& taskType, const TaskParaDMA& para);
142 : CtxInfo(TaskType& taskType, const TaskParaReduce& para);
143 : CtxInfo(TaskType& taskType, const TaskParaNotify& para);
144 3 : CtxInfo() = default;
145 : std::string GetCtxBaseInfoStr(); // 防止tag字符串过长，base信息和para信息分开打印
146 : std::string GetCtxParaInfoStr();
147 : std::string GetCtxParaDMA();
148 : std::string GetCtxParaReduce();
149 : std::string GetCtxParaNotify();
150 : std::string GetCtxRankInfo();
151 : std::string GetCtxNotifyInfo();
152 : u32 GetCtxRemoteUserRank();
153 : };
154 : class TaskExceptionHandler : public ProfilerBase {
155 : public:
156 : explicit TaskExceptionHandler(u32 deviceLogicId);
157 : ~TaskExceptionHandler() override;
158 : static HcclResult Init();
159 : static HcclResult DeInit();
160 : HcclResult Save(u32& streamID, u32& taskID, TaskType& taskType, const TaskParaDMA& para) override;
161 : HcclResult Save(u32& streamID, u32& taskID, TaskType& taskType, const TaskParaReduce& para) override;
162 : HcclResult Save(u32& streamID, u32& taskID, TaskType& taskType, const TaskParaNotify& para) override;
163 : HcclResult Save(u32 streamID, u32 taskID, const TaskParaAiv& para) override;
164 : HcclResult Save(u32& streamID, u32& taskID, const void* descBuf = nullptr, size_t descBufLen = 0) override;
165 : HcclResult SaveToLog(const TaskParaHost& paraHost) override;
166 : HcclResult
167 : Save(u32 captureStreamID, u32 streamID, u32 taskID, TaskType& taskType, const TaskParaDMA& para) override;
168 : HcclResult
169 : Save(u32 captureStreamID, u32 streamID, u32 taskID, TaskType& taskType, const TaskParaReduce& para) override;
170 : HcclResult
171 : Save(u32 captureStreamID, u32 streamID, u32 taskID, TaskType& taskType, const TaskParaNotify& para) override;
172 : HcclResult
173 : Save(u32 captureStreamID, u32 streamID, u32 taskID, const void* descBuf = nullptr, size_t descBufLen = 0) override;
174 : HcclResult Save(u32 captureStreamID, u32 streamID, u32 taskID, const TaskParaAiv& para) override;
175 : static void Callback(rtExceptionInfo* exceptionInfo);
176 : HcclResult Run(const StepData& stepData) override;
177 : HcclResult Flush() override;
178 :
179 : protected:
180 : private:
181 : HcclResult InsertTaskMap(u32& streamID, TaskInfo& tmpTaskInfo) const;
182 : HcclResult InsertOpMap(u32& streamID, u32& taskID, std::string& tag, AlgType& algType, u32& index) const;
183 : HcclResult InsertOpCtxInfo(
184 : u32& streamID, u32& taskID, std::string& tag, AlgType& algType, u32& index, const void* descBuf,
185 : size_t descBufLen) const;
186 : HcclResult InsertRankInfo(std::string& tag) const;
187 : HcclResult InsertOpData(std::string& tag) const;
188 : static void PrintTaskContextInfo(
189 : const std::shared_ptr<std::vector<CtxInfo>>& taskList, u32 contextId, std::string& stageErrInfo);
190 : static void PrintTaskContextInfo(const std::shared_ptr<std::deque<TaskInfo>>& taskQue, std::string& stageErrInfo);
191 : static void PrintTaskAivBuffer(const std::shared_ptr<std::deque<TaskInfo>>& taskQue);
192 : static void PrintTaskAivInfo(const std::shared_ptr<std::deque<TaskInfo>>& taskQue);
193 : static HcclResult PrintCommAivInfo();
194 : static void ParseTaskSyncFlag(s32* flagMem, u32 flagMemSize, u32 rankSize, u32 rank, u32 index);
195 : static std::string SerializeSyncFlag(s32* buf, u32 num, u32 interval);
196 : static void PrintOpDataInfo(OpDataInfo& opDataInfo, bool isFftsPlus, std::string& stageErrInfo);
197 : static void TimeStruct2Str(struct timeval& tv, std::string& opDataContent);
198 : static bool DealExceptionOp(rtExceptionInfo* exceptionInfo);
199 : static bool DealExceptionTask(rtExceptionInfo* exceptionInfo);
200 : static bool DealExceptionCtx(rtExceptionInfo* exceptionInfo);
201 : static bool DealExceptionOpData(
202 : rtExceptionInfo* exceptionInfo, std::string& tag, bool isFftsPlus, u32 index, std::string& stageErrInfo);
203 : static bool DealExceptionGroupRank(
204 : rtExceptionInfo* exceptionInfo, std::string& tag, bool isFftsPlus, std::string& groupRankContentInfo,
205 : std::string& stageErrInfo);
206 : static bool FindAndValidateContext(rtExceptionInfo* exceptionInfo);
207 : static bool ProcessContext(
208 : rtExceptionInfo* exceptionInfo, std::string& stageErrInfo, FFTSOpInfo& fftsOpInfo, CtxInfo& exceptionCtxInfo);
209 : static void PrintFftsCtxInfo(FFTSOpInfo& fftsOpInfo);
210 : static void PrintAicpuErrorMessage(rtExceptionInfo* exceptionInfo, bool& isExistAicpuError);
211 : static void PrintGroupErrorMessage(
212 : ErrorMessageReport& errorMessage, TaskInfo& exceptionTaskInfo, std::string& groupRankContent,
213 : std::string& stageErrInfo);
214 : static void PrintOpDataErrorMessage(u32 deviceId, ErrorMessageReport& errorMessage, std::string& stageErrInfo);
215 : static std::array<std::map<int, std::shared_ptr<std::deque<TaskInfo>>>, MAX_MODULE_DEVICE_NUM> taskMap;
216 : static std::array<std::mutex, MAX_MODULE_DEVICE_NUM> taskMapMutex;
217 : static std::array<std::map<int, std::shared_ptr<std::deque<FFTSOpInfo>>>, MAX_MODULE_DEVICE_NUM> opMap;
218 : static std::array<std::mutex, MAX_MODULE_DEVICE_NUM> opMapMutex;
219 : static std::array<
220 : std::map<
221 : int,
222 : std::shared_ptr<std::deque<std::pair<std::shared_ptr<FFTSOpInfo>, std::shared_ptr<std::vector<CtxInfo>>>>>>,
223 : MAX_MODULE_DEVICE_NUM>
224 : opCtxInfo;
225 : static std::array<std::mutex, MAX_MODULE_DEVICE_NUM> opCtxInfoMutex;
226 : static std::array<std::vector<CtxInfo>, MAX_MODULE_DEVICE_NUM> ctxInfoArray;
227 : static std::array<std::mutex, MAX_MODULE_DEVICE_NUM> ctxInfoVectorMutex;
228 : static std::array<
229 : std::map<const std::string, std::pair<const std::string, std::shared_ptr<GroupRankInfo>>>,
230 : MAX_MODULE_DEVICE_NUM>
231 : groupRankMap;
232 : static std::array<std::mutex, MAX_MODULE_DEVICE_NUM> groupRankMapMutex;
233 : static std::array<std::map<const std::string, std::shared_ptr<std::queue<OpDataInfo>>>, MAX_MODULE_DEVICE_NUM>
234 : tagOpDataMap;
235 : static std::array<std::mutex, MAX_MODULE_DEVICE_NUM> tagOpDataMapMutex;
236 : static std::array<std::map<const std::string, std::string>, MAX_MODULE_DEVICE_NUM> groupUdiMap;
237 : static std::array<std::mutex, MAX_MODULE_DEVICE_NUM> groupUdiMapMutex;
238 : static std::atomic<int> communicatorCount_;
239 : static std::atomic<bool> errMsgFlag_; // errorMsg只上报一次, 控制上报频次
240 : };
241 :
242 : using GetErrStatusVecCallBack = std::vector<std::string> (*)(s32 deviceLogicID, const std::string& group);
243 : using GetAicpuTaskExceptionCallBack = std::function<ErrorMessageReport()>;
244 : #ifdef __cplusplus
245 : extern "C" {
246 : #endif // __cplusplus
247 : extern void RegisterGetErrStatusVecCallBack(GetErrStatusVecCallBack);
248 : extern void RegisterGetAicpuTaskExceptionCallBack(s32 streamId, u32 deviceLogicId, GetAicpuTaskExceptionCallBack p1);
249 : extern void UnregisterGetAicpuTaskExceptionCallBack(s32 streamId, u32 deviceLogicId);
250 : #ifdef __cplusplus
251 : }
252 : #endif // __cplusplus
253 : } // namespace hccl
254 :
255 : #endif
|
