Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #ifndef HCCL_TASK_EXCEPTION_HANDLER_PUB_H
12 : #define HCCL_TASK_EXCEPTION_HANDLER_PUB_H
13 : #include <map>
14 : #include <queue>
15 : #include <mutex>
16 : #include <array>
17 : #include <list>
18 : #include <functional>
19 : #include "rt_external.h"
20 : #include "acl/acl_rt.h"
21 : #include "profiler_base_pub.h"
22 : #include "aicpu_operator_pub.h"
23 : namespace hccl {
24 17 : inline void PrintBaseErrorLog(const std::string &stageErrInfo, const std::string &baseInfo)
25 : {
26 17 : HCCL_ERROR("%sTask run failed, base information is %s", stageErrInfo.c_str(), baseInfo.c_str());
27 17 : }
28 :
29 14 : inline void PrintParaErrorLog(const std::string &stageErrInfo, const std::string ¶InfoStr, const std::string &tag)
30 : {
31 14 : HCCL_ERROR("%sTask run failed, para information is %s, tag[%s].", stageErrInfo.c_str(), paraInfoStr.c_str(), tag.c_str());
32 14 : }
33 :
34 8 : inline void PrintOpDataErrorLog(const std::string &stageErrInfo, const std::string &opDataContent)
35 : {
36 8 : HCCL_ERROR("%sTask run failed, opData information is %s", stageErrInfo.c_str(), opDataContent.c_str());
37 8 : }
38 :
39 12 : inline void PrintGroupErrorLog(const std::string &stageErrInfo, const std::string &groupRankContent, const std::string &tag)
40 : {
41 12 : HCCL_ERROR("%sTask run failed, groupRank information is %s, tag[%s].", stageErrInfo.c_str(), groupRankContent.c_str(), tag.c_str());
42 12 : }
43 :
44 3 : inline void PrintContextErrorLog(const std::string &stageErrInfo, const std::string &ctxBaseInfo)
45 : {
46 3 : HCCL_ERROR("%sTask run failed, context base information is %s", stageErrInfo.c_str(), ctxBaseInfo.c_str());
47 3 : }
48 :
49 : struct ParaDMA {
50 : const void *src;
51 : const void *dst;
52 : std::size_t size;
53 : u64 notifyID;
54 : LinkType linkType;
55 : u32 remoteUserRank;
56 : };
57 :
58 : struct ParaReduce {
59 : const void *src;
60 : const void *dst;
61 : std::size_t size;
62 : HcclReduceOp op;
63 : HcclDataType dataType;
64 : LinkType linkType;
65 : u32 remoteUserRank;
66 : };
67 : struct ParaNotify {
68 : u64 notifyID;
69 : s32 stage; // 用于标识stream间同步时所在的stage， 非用于stream同步的默认为-1
70 : u32 remoteUserRank;
71 : };
72 : struct ParaAiv{
73 : HcclCMDType cmdType;
74 : u32 tag;
75 : u64 size;
76 : u32 numBlocks;
77 : u32 rankSize;
78 : s32 aivRdmaStep;
79 : void* flagMem;
80 : u32 rank;
81 : bool isOpbase;
82 : };
83 : struct TaskInfo {
84 : u32 streamID;
85 : u32 taskID;
86 : std::string tag;
87 : TaskType taskType;
88 : bool isAlgInfo;
89 : AlgType algType;
90 : u32 index;
91 : union {
92 : ParaDMA DMA; // taskType = SDMA/RDMA使用, 包括rtRDMASend写notify
93 : ParaReduce Reduce; // taskType = inline/CCE Reduce使用
94 : ParaNotify Notify; // taskType = Noitfy Record/Wait使用
95 : ParaAiv Aiv; // taskType = Aiv 使用
96 : }taskPara;
97 : TaskInfo(u32 &streamID, u32 &taskID, std::string &tag, TaskType &taskType, AlgType &algType, u32 &index,
98 : const TaskParaDMA ¶);
99 : TaskInfo(u32 &streamID, u32 &taskID, std::string &tag, TaskType &taskType, AlgType &algType, u32 &index,
100 : const TaskParaReduce ¶);
101 : TaskInfo(u32 &streamID, u32 &taskID, std::string &tag, TaskType &taskType, AlgType &algType, u32 &index,
102 : const TaskParaNotify ¶);
103 : TaskInfo(u32 &streamID, u32 &taskID, std::string &tag, const TaskParaAiv& para);
104 : std::string GetBaseInfoStr(); // 防止tag字符串过长，base信息和para信息分开打印
105 : std::string GetParaInfoStr();
106 : std::string GetParaDMA();
107 : std::string GetParaReduce();
108 : std::string GetParaNotify();
109 : std::string GetParaAiv();
110 : std::string GetRankInfo();
111 : std::string GetNotifyInfo();
112 : u32 GetRemoteUserRank();
113 : };
114 : struct FFTSOpInfo {
115 : u32 streamID;
116 : u32 taskID;
117 : std::shared_ptr<char> tag;
118 : AlgType algType;
119 : u32 index;
120 : std::shared_ptr<char> descBuf = nullptr;
121 : size_t descBufLen = 0;
122 : std::string GetBaseInfoStr();
123 : };
124 :
125 : struct CtxInfo {
126 : TaskType taskType;
127 : AlgType algType;
128 : u32 index;
129 : union {
130 : ParaDMA DMA; // taskType = SDMA/RDMA使用, 包括rtRDMASend写notify
131 : ParaReduce Reduce; // taskType = inline/CCE Reduce使用
132 : ParaNotify Notify; // taskType = Noitfy Record/Wait使用
133 : } ctxPara;
134 : CtxInfo(TaskType &taskType, const TaskParaDMA ¶);
135 : CtxInfo(TaskType &taskType, const TaskParaReduce ¶);
136 : CtxInfo(TaskType &taskType, const TaskParaNotify ¶);
137 3 : CtxInfo() = default;
138 : std::string GetCtxBaseInfoStr(); // 防止tag字符串过长，base信息和para信息分开打印
139 : std::string GetCtxParaInfoStr();
140 : std::string GetCtxParaDMA();
141 : std::string GetCtxParaReduce();
142 : std::string GetCtxParaNotify();
143 : std::string GetCtxRankInfo();
144 : std::string GetCtxNotifyInfo();
145 : u32 GetCtxRemoteUserRank();
146 : };
147 : class TaskExceptionHandler : public ProfilerBase {
148 : public:
149 : explicit TaskExceptionHandler(u32 deviceLogicId);
150 : ~TaskExceptionHandler() override;
151 : static HcclResult Init();
152 : static HcclResult DeInit();
153 : HcclResult Save(u32 &streamID, u32 &taskID, TaskType &taskType, const TaskParaDMA ¶) override;
154 : HcclResult Save(u32 &streamID, u32 &taskID, TaskType &taskType, const TaskParaReduce ¶) override;
155 : HcclResult Save(u32 &streamID, u32 &taskID, TaskType &taskType, const TaskParaNotify ¶) override;
156 : HcclResult Save(u32 streamID, u32 taskID, const TaskParaAiv ¶) override;
157 : HcclResult Save(u32 &streamID, u32 &taskID, const void *descBuf = nullptr, size_t descBufLen = 0) override;
158 : HcclResult SaveToLog(const TaskParaHost ¶Host) override;
159 : HcclResult Save(u32 captureStreamID, u32 streamID, u32 taskID, TaskType &taskType, const TaskParaDMA ¶) override;
160 : HcclResult Save(u32 captureStreamID, u32 streamID, u32 taskID, TaskType &taskType, const TaskParaReduce ¶) override;
161 : HcclResult Save(u32 captureStreamID, u32 streamID, u32 taskID, TaskType &taskType, const TaskParaNotify ¶) override;
162 : HcclResult Save(u32 captureStreamID, u32 streamID, u32 taskID, const void *descBuf = nullptr, size_t descBufLen = 0) override;
163 : HcclResult Save(u32 captureStreamID, u32 streamID, u32 taskID, const TaskParaAiv ¶) override;
164 : static void Callback(rtExceptionInfo *exceptionInfo);
165 : HcclResult Run(const StepData &stepData) override;
166 : HcclResult Flush() override;
167 : protected:
168 : private:
169 : HcclResult InsertTaskMap(u32 &streamID, TaskInfo &tmpTaskInfo) const;
170 : HcclResult InsertOpMap(u32 &streamID, u32 &taskID, std::string &tag, AlgType &algType, u32 &index) const;
171 : HcclResult InsertOpCtxInfo(u32 &streamID, u32 &taskID, std::string &tag, AlgType &algType,
172 : u32 &index, const void *descBuf, size_t descBufLen) const;
173 : HcclResult InsertRankInfo(std::string &tag) const;
174 : HcclResult InsertOpData(std::string &tag) const;
175 : static void PrintTaskContextInfo(const std::shared_ptr<std::vector<CtxInfo>> &taskList, u32 contextId, std::string &stageErrInfo);
176 : static void PrintTaskContextInfo(const std::shared_ptr<std::deque<TaskInfo>> &taskQue, std::string &stageErrInfo);
177 : static void PrintTaskAivBuffer(const std::shared_ptr<std::deque<TaskInfo>> &taskQue);
178 : static void PrintTaskAivInfo(const std::shared_ptr<std::deque<TaskInfo>> &taskQue);
179 : static HcclResult PrintCommAivInfo();
180 : static void ParseTaskSyncFlag(s32 *flagMem, u32 flagMemSize, u32 rankSize, u32 rank, u32 index);
181 : static std::string SerializeSyncFlag(s32 *buf, u32 num, u32 interval);
182 : static void PrintOpDataInfo(OpDataInfo &opDataInfo, bool isFftsPlus, std::string &stageErrInfo);
183 : static void TimeStruct2Str(struct timeval &tv, std::string &opDataContent);
184 : static bool DealExceptionOp(rtExceptionInfo *exceptionInfo);
185 : static bool DealExceptionTask(rtExceptionInfo *exceptionInfo);
186 : static bool DealExceptionCtx(rtExceptionInfo *exceptionInfo);
187 : static bool DealExceptionOpData(rtExceptionInfo *exceptionInfo, std::string &tag, bool isFftsPlus,
188 : u32 index, std::string &stageErrInfo);
189 : static bool DealExceptionGroupRank(rtExceptionInfo *exceptionInfo, std::string &tag, bool isFftsPlus,
190 : std::string &groupRankContentInfo, std::string &stageErrInfo);
191 : static bool FindAndValidateContext(rtExceptionInfo *exceptionInfo);
192 : static bool ProcessContext(rtExceptionInfo *exceptionInfo, std::string &stageErrInfo,
193 : FFTSOpInfo &fftsOpInfo, CtxInfo &exceptionCtxInfo);
194 : static void PrintFftsCtxInfo(FFTSOpInfo &fftsOpInfo);
195 : static void PrintAicpuErrorMessage(rtExceptionInfo *exceptionInfo, bool &isExistAicpuError);
196 : static void PrintGroupErrorMessage(ErrorMessageReport &errorMessage, TaskInfo &exceptionTaskInfo,
197 : std::string &groupRankContent, std::string &stageErrInfo);
198 : static void PrintOpDataErrorMessage(u32 deviceId, ErrorMessageReport &errorMessage, std::string &stageErrInfo);
199 : static std::array<std::map<int, std::shared_ptr<std::deque<TaskInfo>>>, \
200 : MAX_MODULE_DEVICE_NUM> taskMap;
201 : static std::array<std::mutex, MAX_MODULE_DEVICE_NUM> taskMapMutex;
202 : static std::array<std::map<int, std::shared_ptr<std::deque<FFTSOpInfo>>>, MAX_MODULE_DEVICE_NUM> opMap;
203 : static std::array<std::mutex, MAX_MODULE_DEVICE_NUM> opMapMutex;
204 : static std::array<std::map<int, std::shared_ptr<std::deque<std::pair<std::shared_ptr<FFTSOpInfo>, \
205 : std::shared_ptr<std::vector<CtxInfo>>>>>>, MAX_MODULE_DEVICE_NUM> opCtxInfo;
206 : static std::array<std::mutex, MAX_MODULE_DEVICE_NUM> opCtxInfoMutex;
207 : static std::array<std::vector<CtxInfo>, MAX_MODULE_DEVICE_NUM> ctxInfoArray;
208 : static std::array<std::mutex, MAX_MODULE_DEVICE_NUM> ctxInfoVectorMutex;
209 : static std::array<std::map<const std::string, std::pair<const std::string, std::shared_ptr<GroupRankInfo>>>, \
210 : MAX_MODULE_DEVICE_NUM> groupRankMap;
211 : static std::array<std::mutex, MAX_MODULE_DEVICE_NUM> groupRankMapMutex;
212 : static std::array<std::map<const std::string, std::shared_ptr<std::queue<OpDataInfo>>>, \
213 : MAX_MODULE_DEVICE_NUM> tagOpDataMap;
214 : static std::array<std::mutex, MAX_MODULE_DEVICE_NUM> tagOpDataMapMutex;
215 : static std::array<std::map<const std::string, std::string>, MAX_MODULE_DEVICE_NUM> groupUdiMap;
216 : static std::array<std::mutex, MAX_MODULE_DEVICE_NUM> groupUdiMapMutex;
217 : static std::atomic<int> communicatorCount_;
218 : static std::atomic<bool> errMsgFlag_; // errorMsg只上报一次, 控制上报频次
219 : };
220 :
221 : using GetErrStatusVecCallBack = std::vector<std::string> (*)(s32 deviceLogicID, const std::string& group);
222 : using GetAicpuTaskExceptionCallBack = std::function<ErrorMessageReport()>;
223 : #ifdef __cplusplus
224 : extern "C" {
225 : #endif // __cplusplus
226 : extern void RegisterGetErrStatusVecCallBack(GetErrStatusVecCallBack);
227 : extern void RegisterGetAicpuTaskExceptionCallBack(s32 streamId, u32 deviceLogicId, GetAicpuTaskExceptionCallBack p1);
228 : extern void UnregisterGetAicpuTaskExceptionCallBack(s32 streamId, u32 deviceLogicId);
229 : #ifdef __cplusplus
230 : }
231 : #endif // __cplusplus
232 : }
233 :
234 : #endif
|
