Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 : #include <set>
11 : #include <cctype>
12 : #include <cstring>
13 : #include <algorithm>
14 : #include "log/adx_log.h"
15 : #include "path.h"
16 : #include "dump_manager.h"
17 : #include "dump_config_converter.h"
18 : #include "sys_utils.h"
19 : #include "dump_args.h"
20 : #include "dump_args_callback.h"
21 : #include "dump_file.h"
22 : #include "exception_info_common.h"
23 : #include "kernel_symbol_locator.h"
24 : #include "kernel_info_collector.h"
25 : #include "exception_dumper.h"
26 :
27 : namespace Adx {
28 : namespace {
29 : constexpr char DEFAULT_DUMP_PATH[] = "./";
30 : constexpr char EXTRA_DUMP_PATH[] = "/extra-info/data-dump/";
31 : constexpr size_t MAX_DUMP_OP_NUM = (2048U * 2048U) / 20U;
32 :
33 12 : static bool IsSafeKernelDisplayName(const char *name)
34 : {
35 77 : for (size_t i = 0; i < MAX_KERNELNAME_LEN && name[i] != '\0'; ++i) {
36 69 : char c = name[i];
37 69 : if (c == '/' || c == '\\' || c < 0x20 || c == 0x7F) {
38 4 : return false;
39 : }
40 : }
41 16 : return std::string(name).find("..") == std::string::npos;
42 : }
43 :
44 7 : bool ValidateKernelName(const ExceptionDumpInfo &info)
45 : {
46 7 : if (strnlen(info.kernelName, MAX_KERNELNAME_LEN) == MAX_KERNELNAME_LEN ||
47 7 : strnlen(info.kernelDisplayName, MAX_KERNELNAME_LEN) == MAX_KERNELNAME_LEN) {
48 0 : return false;
49 : }
50 :
51 7 : if (!IsSafeKernelDisplayName(info.kernelName) || !IsSafeKernelDisplayName(info.kernelDisplayName)) {
52 4 : return false;
53 : }
54 3 : return true;
55 : }
56 :
57 9 : bool IsValidExceptionDumpMode(ExceptionDumpMode dumpMode)
58 : {
59 8 : return dumpMode == ExceptionDumpMode::DUMP_MODE_NONE ||
60 17 : dumpMode == ExceptionDumpMode::DUMP_MODE_OVERWRITE ||
61 9 : dumpMode == ExceptionDumpMode::DUMP_MODE_ADDITIONAL;
62 : }
63 : } // namespace
64 :
65 : uint64_t *g_dynamicChunk = nullptr;
66 : uint64_t *g_staticChunk = nullptr;
67 :
68 48 : ExceptionDumper::~ExceptionDumper()
69 : {
70 48 : if (g_dynamicChunk != nullptr) {
71 18 : delete[] g_dynamicChunk;
72 18 : g_dynamicChunk = nullptr;
73 : }
74 :
75 48 : if (g_staticChunk != nullptr) {
76 18 : delete[] g_staticChunk;
77 18 : g_staticChunk = nullptr;
78 : }
79 48 : KernelSymbolLocator::ClearCache();
80 48 : destructionFlag_ = true;
81 48 : }
82 :
83 57 : bool ExceptionDumper::InitArgsExceptionMemory() const
84 : {
85 57 : if (g_dynamicChunk == nullptr) {
86 7149942 : g_dynamicChunk = new (std::nothrow) uint64_t[DYNAMIC_RING_CHUNK_SIZE + DFX_MAX_TENSOR_NUM + RESERVE_SPACE]();
87 18 : if (g_dynamicChunk == nullptr) {
88 0 : return false;
89 : }
90 : }
91 :
92 57 : if (g_staticChunk == nullptr) {
93 2431350 : g_staticChunk = new (std::nothrow) uint64_t[STATIC_RING_CHUNK_SIZE + DFX_MAX_TENSOR_NUM + RESERVE_SPACE]();
94 18 : if (g_staticChunk == nullptr) {
95 0 : return false;
96 : }
97 : }
98 57 : return true;
99 : }
100 :
101 123 : bool ExceptionDumper::IsRepeatEnableException(DumpType type, const DumpConfig &dumpConfig)
102 : {
103 123 : if (type == DumpType::ARGS_EXCEPTION || type == DumpType:: EXCEPTION ||
104 : type == DumpType::AIC_ERR_DETAIL_DUMP) {
105 56 : std::string dumpStatus = dumpConfig.dumpStatus;
106 56 : std::transform(dumpStatus.begin(), dumpStatus.end(), dumpStatus.begin(), ::tolower);
107 56 : if (dumpStatus != "on") {
108 11 : return false;
109 : }
110 45 : return IsEnabledExceptionDump();
111 56 : }
112 67 : return false;
113 : }
114 :
115 125 : bool ExceptionDumper::IsEnabledExceptionDump() const
116 : {
117 125 : return coredumpStatus_ || exceptionStatus_ || argsExceptionStatus_;
118 : }
119 :
120 77 : int32_t ExceptionDumper::ExceptionDumperInit(DumpType dumpType, const DumpConfig &dumpConfig)
121 : {
122 77 : bool status = false;
123 77 : if (!setting_.InitDumpStatus(dumpConfig.dumpStatus, status)) {
124 1 : IDE_LOGE("The value of dumpStatus: %s is invalid.", dumpConfig.dumpStatus.c_str());
125 1 : return ADUMP_FAILED;
126 : }
127 :
128 : // status = false means turn off, flag = false means not start, print warning when not start but want to turn off
129 76 : if (dumpType == DumpType::EXCEPTION) {
130 13 : IDE_CTRL_VALUE_WARN(status || exceptionStatus_, return ADUMP_SUCCESS, "dump type %d not start.", dumpType);
131 12 : exceptionStatus_ = status;
132 63 : } else if (dumpType == DumpType::ARGS_EXCEPTION) {
133 57 : IDE_CTRL_VALUE_WARN(status || argsExceptionStatus_, return ADUMP_SUCCESS, "dump type %d not start.", dumpType);
134 53 : IDE_CTRL_VALUE_FAILED(InitArgsExceptionMemory(), return ADUMP_FAILED, "Init args exception memory failed.");
135 53 : if (status && !argsExceptionStatus_) {
136 47 : IDE_CTRL_VALUE_WARN(LoadTensorPluginLib() == ADUMP_SUCCESS, return ADUMP_FAILED,
137 : "Load tersor custom plugin failed.");
138 : }
139 53 : argsExceptionStatus_ = status;
140 : } else {
141 : // detail exception dump can not off
142 6 : if (status == false) {
143 : static bool havePrint = false;
144 2 : if (!havePrint) {
145 1 : IDE_RUN_LOGI("Can not turn off detail exception dump.");
146 1 : havePrint = true;
147 : }
148 2 : IDE_LOGW("Can not turn off detail exception dump.");
149 2 : return ADUMP_SUCCESS;
150 : }
151 4 : IDE_CTRL_VALUE_FAILED(InitArgsExceptionMemory(), return ADUMP_FAILED, "Init args exception memory failed.");
152 4 : coredumpStatus_ = true;
153 : }
154 :
155 69 : SetDumpPath(dumpConfig.dumpPath);
156 69 : return ADUMP_SUCCESS;
157 : }
158 :
159 61 : std::string ExceptionDumper::GetDumpSceneName() const
160 : {
161 61 : if (coredumpStatus_) {
162 3 : return ADUMP_DUMP_EXCEPTION_AIC_ERR_DETAIL;
163 58 : } else if (exceptionStatus_) {
164 1 : return ADUMP_DUMP_EXCEPTION_AIC_ERR_NORM;
165 57 : } else if (argsExceptionStatus_) {
166 57 : return ADUMP_DUMP_EXCEPTION_AIC_ERR_BRIEF;
167 : } else {
168 0 : return "unknown";
169 : }
170 : }
171 :
172 82 : int32_t ExceptionDumper::DumpException(const rtExceptionInfo &exception)
173 : {
174 82 : IDE_CTRL_VALUE_WARN(!destructionFlag_, return ADUMP_FAILED, "ExceptionDumper has been destructed.");
175 82 : IDE_CTRL_VALUE_WARN(ExceptionInfoCommon::IsSupportExceptionDump(exception),
176 : return ADUMP_FAILED, "Exception is not need to dump.");
177 68 : IDE_CTRL_VALUE_WARN(IsEnabledExceptionDump(), return ADUMP_FAILED,
178 : "Not enable exception dump.");
179 66 : std::string dumpPath = CreateDeviceDumpPath(exception.deviceid);
180 66 : if (dumpPath.empty()) {
181 5 : return ADUMP_FAILED;
182 : }
183 :
184 61 : const std::string dumpScene = GetDumpSceneName();
185 61 : const std::string exceptionType = ExceptionInfoCommon::GetExceptionTaskTypeName(exception);
186 61 : const std::string kernelName = ExceptionInfoCommon::GetExceptionKernelName(exception);
187 61 : IDE_LOGE("[Dump][Exception] Begin to dump exception. dumpScene=%s, deviceId=%u, streamId=%u, taskId=%u, "
188 : "exceptionType=%d(%s), kernelName=%s.", dumpScene.c_str(), exception.deviceid, exception.streamid,
189 : exception.taskid, static_cast<int32_t>(exception.expandInfo.type), exceptionType.c_str(),
190 : kernelName.c_str());
191 :
192 61 : if (coredumpStatus_) {
193 3 : return DumpDetailException(exception, dumpPath);
194 58 : } else if (exceptionStatus_) {
195 1 : return DumpNormalException(exception, dumpPath);
196 : } else {
197 57 : return DumpArgsException(exception, dumpPath);
198 : }
199 66 : }
200 :
201 69 : void ExceptionDumper::SetDumpPath(const std::string &dumpPath)
202 : {
203 86 : dumpPath_ = dumpPath.empty() ? DEFAULT_DUMP_PATH : dumpPath;
204 69 : IDE_LOGI("Update exception dump path: %s", dumpPath_.c_str());
205 69 : }
206 :
207 4 : void ExceptionDumper::AddDumpOperator(const OperatorInfo &opInfo)
208 : {
209 4 : OperatorInfoV2 dumpOperator = {};
210 4 : DumpManager::Instance().ConvertOperatorInfo(opInfo, dumpOperator);
211 4 : AddDumpOperatorV2(dumpOperator);
212 4 : }
213 :
214 118 : void ExceptionDumper::AddDumpOperatorV2(const OperatorInfoV2 &opInfo)
215 : {
216 118 : const std::lock_guard<std::mutex> lock(mutex_);
217 118 : if (opInfo.agingFlag) {
218 6 : agingOperators_.emplace_back(opInfo);
219 6 : if (agingOperators_.size() > MAX_DUMP_OP_NUM) {
220 0 : agingOperators_.pop_front();
221 : }
222 : } else {
223 112 : uint32_t maxStreamCount = 0;
224 112 : uint32_t maxTaskCount = 0;
225 112 : rtGetMaxStreamAndTask(0, &maxStreamCount, &maxTaskCount);
226 112 : auto &taskDeque = residentOperators_[opInfo.deviceId][opInfo.streamId];
227 112 : taskDeque.emplace_back(opInfo);
228 112 : if (taskDeque.size() > maxTaskCount) {
229 2 : taskDeque.pop_front();
230 : }
231 : }
232 118 : IDE_LOGI("Dump operator size, aging: %zu, resident: %zu", agingOperators_.size(), residentOperators_.size());
233 118 : }
234 :
235 9 : int32_t ExceptionDumper::DelDumpOperator(uint32_t deviceId, uint32_t streamId)
236 : {
237 9 : const std::lock_guard<std::mutex> lock(mutex_);
238 9 : auto it = residentOperators_.find(deviceId);
239 9 : if (it != residentOperators_.end()) {
240 8 : it->second.erase(streamId);
241 8 : if (it->second.empty()) {
242 8 : residentOperators_.erase(deviceId);
243 : }
244 : }
245 9 : return ADUMP_SUCCESS;
246 9 : }
247 :
248 96 : std::string ExceptionDumper::CreateDumpPath(Path &dumpPath) const
249 : {
250 116 : IDE_CTRL_VALUE_FAILED(dumpPath.CreateDirectory(true), return "", "Create path[%s] failed",
251 : dumpPath.GetCString());
252 94 : IDE_CTRL_VALUE_FAILED(dumpPath.RealPath(), return "", "Get RealPath of [%s] failed, strerr=%s",
253 : dumpPath.GetCString(), strerror(errno));
254 82 : return dumpPath.GetString();
255 : }
256 :
257 90 : std::string ExceptionDumper::CreateDeviceDumpPath(uint32_t deviceId) const
258 : {
259 : // Create device dump path when exception call-backed
260 90 : std::string dumpPath = dumpPath_.empty() ? DEFAULT_DUMP_PATH : dumpPath_;
261 90 : Path deviceDumpPath(dumpPath);
262 90 : deviceDumpPath.Append(EXTRA_DUMP_PATH).Append(std::to_string(deviceId));
263 180 : return CreateDumpPath(deviceDumpPath);
264 90 : }
265 :
266 24 : int32_t ExceptionDumper::GetExceptionDumpPath(std::string &path)
267 : {
268 24 : int32_t deviceId = 0;
269 24 : const rtError_t rtRet = rtGetDevice(&deviceId);
270 24 : IDE_CTRL_VALUE_FAILED(rtRet == RT_ERROR_NONE, return ADUMP_FAILED,
271 : "rtGetDevice failed when get the exception root dump path, ret=%d.", static_cast<int32_t>(rtRet));
272 :
273 24 : path = CreateDeviceDumpPath(static_cast<uint32_t>(deviceId));
274 24 : IDE_CTRL_VALUE_FAILED(!path.empty(), return ADUMP_FAILED, "exception dump root path is not ready.");
275 18 : return ADUMP_SUCCESS;
276 : }
277 :
278 15 : int32_t ExceptionDumper::SaveExceptionInfo(const std::string &fileName, const std::string &userTag,
279 : const std::vector<TensorInfo> &tensors)
280 : {
281 15 : std::string rootPath;
282 15 : IDE_CTRL_VALUE_FAILED(GetExceptionDumpPath(rootPath) == ADUMP_SUCCESS, return ADUMP_FAILED,
283 : "[SaveExceptionInfo] get exception dump root path failed.");
284 :
285 12 : std::string realFilePath;
286 12 : IDE_CTRL_VALUE_FAILED(Path::BuildFullPathUnderRoot(rootPath, fileName, realFilePath),
287 : return ADUMP_FAILED, "[SaveExceptionInfo] build full file path failed.");
288 :
289 12 : realFilePath += ".custom." + SysUtils::GetCurrentTimeWithMillisecond();
290 :
291 12 : int32_t devId = 0;
292 12 : const rtError_t rtRet = rtGetDevice(&devId);
293 12 : IDE_CTRL_VALUE_FAILED(rtRet == RT_ERROR_NONE, return ADUMP_FAILED,
294 : "[SaveExceptionInfo] rtGetDevice failed, ret=%d.", static_cast<int32_t>(rtRet));
295 :
296 12 : DumpFile dumpFile(static_cast<uint32_t>(devId), realFilePath);
297 12 : dumpFile.SetHeader(Path(realFilePath).GetFileName());
298 12 : if (!userTag.empty()) {
299 18 : dumpFile.SetOpAttr("user_tag", userTag);
300 : }
301 :
302 12 : std::vector<std::string> record;
303 12 : dumpFile.SetTensors(tensors, record);
304 :
305 12 : IDE_CTRL_VALUE_FAILED(dumpFile.Dump(record) == ADUMP_SUCCESS, return ADUMP_FAILED,
306 : "[SaveExceptionInfo] dump exception info to file failed, file: %s", realFilePath.c_str());
307 :
308 9 : (void)mmChmod(realFilePath.c_str(), M_IRUSR);
309 9 : IDE_LOGE("[Dump][Exception] dump custom exception to file, file: %s", realFilePath.c_str());
310 9 : return ADUMP_SUCCESS;
311 15 : }
312 :
313 6 : std::string ExceptionDumper::CreateExtraDumpPath()
314 : {
315 : // Create extra dump path for tensor custom dump data
316 6 : std::string dumpPath = dumpPath_.empty() ? DEFAULT_DUMP_PATH : dumpPath_;
317 6 : Path extraDumpPath(dumpPath);
318 6 : extraDumpPath.Append(EXTRA_DUMP_PATH);
319 6 : extraDumpPath_ = CreateDumpPath(extraDumpPath);
320 12 : return extraDumpPath_;
321 6 : }
322 :
323 7 : const char* ExceptionDumper::GetExtraDumpCPath() const
324 : {
325 7 : return extraDumpPath_.empty() ? nullptr : extraDumpPath_.c_str();
326 : }
327 :
328 0 : bool ExceptionDumper::FindExceptionOperator(const rtExceptionInfo &exception, DumpOperator &excOp)
329 : {
330 0 : OpIdentity excOpIdentiy(exception.deviceid, exception.taskid, exception.streamid);
331 0 : if (exception.expandInfo.type == RT_EXCEPTION_FFTS_PLUS) {
332 0 : uint32_t contextId = static_cast<uint32_t>(exception.expandInfo.u.fftsPlusInfo.contextId);
333 0 : uint32_t threadId = static_cast<uint32_t>(exception.expandInfo.u.fftsPlusInfo.threadId);
334 0 : IDE_LOGI("ffts+ op context id: %u, thread id: %u.", contextId, threadId);
335 0 : excOpIdentiy.contextId = contextId;
336 : }
337 :
338 0 : const std::lock_guard<std::mutex> lock(mutex_);
339 0 : IDE_LOGI("Dump op size, aging:%zu, resident:%zu, target: %s", agingOperators_.size(), residentOperators_.size(),
340 : excOpIdentiy.GetString().c_str());
341 0 : for (const auto &op : agingOperators_) {
342 0 : if (op.IsBelongTo(excOpIdentiy)) {
343 0 : IDE_LOGI("Find exception op in aging list: %s", excOpIdentiy.GetString().c_str());
344 0 : excOp = op;
345 0 : return true;
346 : }
347 : }
348 :
349 0 : if (residentOperators_.find(excOpIdentiy.deviceId) != residentOperators_.end()) {
350 0 : auto &streamMap = residentOperators_[excOpIdentiy.deviceId];
351 0 : if (streamMap.find(excOpIdentiy.streamId) != streamMap.end()) {
352 0 : auto &taskDeque = streamMap[excOpIdentiy.streamId];
353 0 : for (const auto &op : taskDeque) {
354 0 : if (op.IsBelongTo(excOpIdentiy)) {
355 0 : IDE_LOGI("Find exception op in resident list: %s", excOpIdentiy.GetString().c_str());
356 0 : excOp = op;
357 0 : return true;
358 : }
359 : }
360 : }
361 : }
362 :
363 0 : IDE_LOGW("Not find dump operator, target: %s", excOpIdentiy.GetString().c_str());
364 0 : return false;
365 0 : }
366 :
367 48 : void ExceptionDumper::DumpHostKernelBinBeforeSymbolize(const rtExceptionInfo &exception,
368 : const std::string &dumpPath) const
369 : {
370 48 : rtExceptionArgsInfo_t exceptionArgsInfo{};
371 48 : int32_t ret = ExceptionInfoCommon::GetExceptionInfo(exception, exceptionArgsInfo);
372 48 : IDE_CTRL_VALUE_WARN(ret == ADUMP_SUCCESS, return,
373 : "Get exception args info failed, skip early dump host kernel bin.");
374 :
375 48 : KernelInfoCollector collector;
376 48 : collector.LoadKernelInfo(exceptionArgsInfo);
377 48 : ret = collector.LoadKernelBinBuffer();
378 48 : IDE_CTRL_VALUE_WARN(ret == ADUMP_SUCCESS, return,
379 : "Load kernel bin buffer failed, skip early dump host kernel bin.");
380 :
381 : // 提前同步落 _host.o；DumpHostKernelBin 幂等，后置慢搜索路径已存在则会跳过。
382 48 : (void)collector.DumpHostKernelBin(dumpPath);
383 48 : }
384 :
385 1 : int32_t ExceptionDumper::DumpNormalException(const rtExceptionInfo &exception, const std::string &dumpPath)
386 : {
387 1 : IDE_CTRL_VALUE_WARN(ExceptionInfoCommon::IsSupportDefaultExceptionDump(exception),
388 : return ADUMP_FAILED, "Exception is not support default dump.");
389 0 : DumpHostKernelBinBeforeSymbolize(exception, dumpPath);
390 0 : KernelSymbolLocator::DumpErrorSymbols(exception);
391 0 : DumpOperator excOp;
392 0 : bool find = FindExceptionOperator(exception, excOp);
393 0 : if (!find) {
394 0 : return ADUMP_SUCCESS;
395 : }
396 :
397 0 : rtExceptionArgsInfo_t exceptionArgsInfo{};
398 0 : if (ExceptionInfoCommon::GetExceptionInfo(exception, exceptionArgsInfo) != ADUMP_SUCCESS) {
399 0 : IDE_LOGW("Get exception args info failed.");
400 0 : return ADUMP_FAILED;
401 : }
402 0 : (void)excOp.RefreshAddrs(exceptionArgsInfo);
403 0 : (void)excOp.LogExceptionInfo(exceptionArgsInfo);
404 0 : (void)excOp.CopyOpKernelFile();
405 :
406 0 : const int32_t ret = excOp.DumpException(exception.deviceid, dumpPath);
407 0 : if (ret != ADUMP_SUCCESS) {
408 0 : return ADUMP_FAILED;
409 : }
410 0 : return ADUMP_SUCCESS;
411 0 : }
412 :
413 49 : int32_t ExceptionDumper::DumpArgsExceptionDefault(const rtExceptionInfo &exception, const std::string &dumpPath)
414 : {
415 49 : IDE_CTRL_VALUE_WARN(ExceptionInfoCommon::IsSupportDefaultExceptionDump(exception),
416 : return ADUMP_FAILED, "Exception is not support default dump.");
417 48 : DumpHostKernelBinBeforeSymbolize(exception, dumpPath);
418 48 : KernelSymbolLocator::DumpErrorSymbols(exception);
419 48 : DumpArgs args;
420 48 : if (args.LoadArgsExceptionInfo(exception) != ADUMP_SUCCESS) {
421 30 : return ADUMP_FAILED;
422 : }
423 :
424 18 : if (args.DumpArgsExceptionInfo(exception.deviceid, dumpPath) != ADUMP_SUCCESS) {
425 0 : return ADUMP_FAILED;
426 : }
427 18 : return ADUMP_SUCCESS;
428 48 : }
429 :
430 56 : int32_t ExceptionDumper::DumpArgsExceptionInner(const rtExceptionInfo &exception, const std::string &dumpPath)
431 : {
432 56 : ExceptionDumpMode dumpMode = ExceptionDumpMode::DUMP_MODE_NONE;
433 56 : std::vector<ExceptionDumpInfo> dumpInfos;
434 56 : int32_t invokeRet = InvokeCallbacks(exception, dumpInfos, dumpMode);
435 56 : if (invokeRet != ADUMP_SUCCESS) {
436 0 : return invokeRet;
437 : }
438 56 : if (dumpMode == ExceptionDumpMode::DUMP_MODE_OVERWRITE) {
439 5 : if (dumpInfos.empty()) {
440 3 : IDE_LOGE("OVERWRITE mode with empty callback data after validation, reject.");
441 3 : return ADUMP_FAILED;
442 : }
443 2 : IDE_LOGI("DumpMode of callbacks is OVERWRITE, skip default dump, dump callback data only.");
444 2 : DumpCallbackData(exception, dumpInfos, dumpPath);
445 2 : return ADUMP_SUCCESS;
446 : }
447 :
448 51 : if (dumpMode == ExceptionDumpMode::DUMP_MODE_ADDITIONAL) {
449 2 : if (!dumpInfos.empty()) {
450 1 : IDE_LOGI("DumpMode of callbacks is ADDITIONAL, dump callback data and default.");
451 1 : DumpCallbackData(exception, dumpInfos, dumpPath);
452 : }
453 : }
454 :
455 51 : return DumpArgsExceptionDefault(exception, dumpPath);
456 56 : }
457 :
458 56 : int32_t ExceptionDumper::InvokeCallbacks(const rtExceptionInfo &exception,
459 : std::vector<ExceptionDumpInfo> &dumpInfos,
460 : ExceptionDumpMode &dumpMode)
461 : {
462 56 : dumpMode = ExceptionDumpMode::DUMP_MODE_NONE;
463 56 : dumpInfos.clear();
464 :
465 56 : std::lock_guard<std::recursive_mutex> lock(callbackMutex_);
466 56 : if (callbacks_.empty()) {
467 47 : IDE_LOGI("No exception callbacks registered.");
468 47 : return ADUMP_SUCCESS;
469 : }
470 :
471 9 : ExceptionRegInfo exceptionRegInfo{0, nullptr};
472 9 : int32_t ret = ExceptionInfoCommon::GetExceptionRegInfo(exception, exceptionRegInfo);
473 9 : IDE_CTRL_VALUE_WARN(ret == ADUMP_SUCCESS, return ADUMP_FAILED,
474 : "GetExceptionRegInfo failed, ret=%d, coreNum=%u.", ret, exceptionRegInfo.coreNum);
475 :
476 9 : uint32_t maxDumpSize = exceptionRegInfo.coreNum == 0 ? 1 : exceptionRegInfo.coreNum;
477 9 : IDE_LOGI("Exception callbacks size=%zu, maxDumpSize=%u.", callbacks_.size(), maxDumpSize);
478 :
479 18 : for (auto &callback : callbacks_) {
480 9 : std::vector<ExceptionDumpInfo> cbDumpInfos(maxDumpSize);
481 :
482 9 : uint32_t cbDumpRealSize = 0;
483 9 : ExceptionDumpMode cbDumpMode = ExceptionDumpMode::DUMP_MODE_NONE;
484 :
485 9 : uint32_t ret = callback(reinterpret_cast<void*>(const_cast<rtExceptionInfo*>(&exception)), cbDumpInfos.data(),
486 : maxDumpSize, &cbDumpRealSize, &cbDumpMode);
487 9 : if (ret != ADUMP_SUCCESS) {
488 0 : IDE_LOGW("Callback execute failed. ret=%u", ret);
489 0 : continue;
490 : }
491 9 : if (!IsValidExceptionDumpMode(cbDumpMode)) {
492 1 : IDE_LOGW("Callback dumpMode invalid. dumpMode=%u", static_cast<uint32_t>(cbDumpMode));
493 1 : continue;
494 : }
495 8 : if (cbDumpRealSize == 0 || cbDumpRealSize > maxDumpSize) {
496 1 : IDE_LOGW("Callback dumpRealSize invalid. maxDumpSize=%u, dumpRealSize=%u", maxDumpSize, cbDumpRealSize);
497 1 : continue;
498 : }
499 :
500 7 : IDE_LOGI("Callback dumpRealSize=%u, dumpMode=%d.", cbDumpRealSize, cbDumpMode);
501 14 : for (uint32_t i = 0; i < cbDumpRealSize; ++i) {
502 7 : if (!ValidateKernelName(cbDumpInfos[i])) {
503 4 : IDE_LOGW("Invalid kernelName/kernelDisplayName in callback data, skip. index=%u", i);
504 4 : continue;
505 : }
506 3 : dumpInfos.push_back(cbDumpInfos[i]);
507 : }
508 :
509 : // 多回调聚合时取优先级最高者：ADDITIONAL > OVERWRITE > NONE
510 7 : if (static_cast<uint32_t>(cbDumpMode) > static_cast<uint32_t>(dumpMode)) {
511 7 : dumpMode = cbDumpMode;
512 : }
513 9 : }
514 :
515 9 : IDE_LOGI("Callback final dumpMode=%d, dumpInfo total size=%zu.", dumpMode, dumpInfos.size());
516 9 : return ADUMP_SUCCESS;
517 56 : }
518 :
519 1 : void ExceptionDumper::DumpCallbackData(const rtExceptionInfo &exception,
520 : const std::vector<ExceptionDumpInfo> &dumpInfos,
521 : const std::string &dumpPath)
522 : {
523 2 : for (const auto &info : dumpInfos) {
524 1 : IDE_LOGE("[Dump][Exception] Begin to dump callback exception. coreType=%u, coreId=%u, "
525 : "argAddr=%p, argSize=%u, binHandle=%p, extraTensorNum=%u, kernelName=%s.",
526 : info.coreType, info.coreId, info.argAddr, info.argSize, info.bin, info.extraTensorNum, info.kernelName);
527 1 : IDE_LOGE("[Dump][Exception] Callback exception. kernelDisplayName=%s.", info.kernelDisplayName);
528 1 : DumpArgsCallback argsCallback(exception, info, dumpPath);
529 1 : int32_t ret = argsCallback.DumpDfxArgs();
530 1 : if (ret != ADUMP_SUCCESS) {
531 0 : IDE_LOGW("Dump callback args failed.");
532 : }
533 :
534 1 : ret = argsCallback.DumpExtraTensors();
535 1 : if (ret != ADUMP_SUCCESS) {
536 0 : IDE_LOGW("Dump callback extra tensor failed.");
537 : }
538 :
539 1 : ret = argsCallback.Dump();
540 1 : if (ret != ADUMP_SUCCESS) {
541 0 : IDE_LOGW("Dump callback data to file failed.");
542 : }
543 :
544 1 : ret = argsCallback.DumpKernelBin();
545 1 : if (ret != ADUMP_SUCCESS) {
546 0 : IDE_LOGW("Dump callback kernel bin failed.");
547 : }
548 :
549 1 : ret = argsCallback.DumpKernelErrorSymbols();
550 1 : if (ret != ADUMP_SUCCESS) {
551 0 : IDE_LOGW("Dump callback kernel error symbols failed.");
552 : }
553 :
554 1 : IDE_LOGI("Dump callback data finished, kernelName=%s.", info.kernelName);
555 1 : }
556 1 : KernelSymbolLocator::ClearCache();
557 1 : IDE_LOGI("Dump all callback data success.");
558 1 : }
559 :
560 5 : void ExceptionDumper::ExceptionModeDowngrade()
561 : {
562 5 : coredumpEnableComplete_ = false;
563 5 : }
564 :
565 0 : void ExceptionDumper::Exit() const
566 : {
567 0 : IDE_LOGE("The process exits.");
568 0 : AdxLogFlush();
569 : #ifndef __ADUMP_LLT
570 : _exit(EXIT_FAILURE);
571 : #endif
572 0 : }
573 :
574 : #ifdef __ADUMP_LLT
575 644 : void ExceptionDumper::Reset()
576 : {
577 644 : const std::lock_guard<std::mutex> lock(mutex_);
578 644 : agingOperators_.clear();
579 644 : residentOperators_.clear();
580 644 : exceptionStatus_ = false;
581 644 : argsExceptionStatus_ = false;
582 644 : coredumpStatus_ = false;
583 644 : coredumpEnableComplete_ = true;
584 644 : }
585 : #endif
586 :
587 32 : int32_t ExceptionDumper::RegisterExceptionDumpCallback(ExceptionDumpCallback callback)
588 : {
589 32 : if (callback == nullptr) {
590 7 : IDE_LOGE("Register callback is nullptr.");
591 7 : return ADUMP_INPUT_FAILED;
592 : }
593 :
594 25 : const std::lock_guard<std::recursive_mutex> lock(callbackMutex_);
595 25 : for (auto &cb : callbacks_) {
596 4 : if (cb == callback) {
597 4 : IDE_LOGW("Callback already registered.");
598 4 : return ADUMP_SUCCESS;
599 : }
600 : }
601 21 : callbacks_.push_back(callback);
602 21 : IDE_LOGI("Callback registered, total=%zu.", callbacks_.size());
603 21 : return ADUMP_SUCCESS;
604 25 : }
605 :
606 21 : int32_t ExceptionDumper::UnregisterExceptionDumpCallback(ExceptionDumpCallback callback)
607 : {
608 21 : if (callback == nullptr) {
609 7 : IDE_LOGE("Unregister callback is nullptr.");
610 7 : return ADUMP_INPUT_FAILED;
611 : }
612 :
613 14 : const std::lock_guard<std::recursive_mutex> lock(callbackMutex_);
614 14 : for (auto it = callbacks_.begin(); it != callbacks_.end(); ++it) {
615 10 : if (*it == callback) {
616 10 : callbacks_.erase(it);
617 10 : IDE_LOGI("Callback unregistered, total=%zu.", callbacks_.size());
618 10 : return ADUMP_SUCCESS;
619 : }
620 : }
621 4 : IDE_LOGW("Callback not found.");
622 4 : return ADUMP_SUCCESS;
623 14 : }
624 :
625 : } // namespace Adx
|
