Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 : #include <set>
11 : #include <cctype>
12 : #include <cstring>
13 : #include <algorithm>
14 : #include "log/adx_log.h"
15 : #include "path.h"
16 : #include "dump_manager.h"
17 : #include "dump_config_converter.h"
18 : #include "sys_utils.h"
19 : #include "dump_args.h"
20 : #include "dump_args_callback.h"
21 : #include "dump_file.h"
22 : #include "exception_info_common.h"
23 : #include "kernel_symbol_locator.h"
24 : #include "kernel_info_collector.h"
25 : #include "exception_dumper.h"
26 :
27 : namespace Adx {
28 : namespace {
29 : constexpr char DEFAULT_DUMP_PATH[] = "./";
30 : constexpr char EXTRA_DUMP_PATH[] = "/extra-info/data-dump/";
31 : constexpr size_t MAX_DUMP_OP_NUM = (2048U * 2048U) / 20U;
32 :
33 12 : static bool IsSafeKernelDisplayName(const char *name)
34 : {
35 77 : for (size_t i = 0; i < MAX_KERNELNAME_LEN && name[i] != '\0'; ++i) {
36 69 : char c = name[i];
37 69 : if (c == '/' || c == '\\' || c < 0x20 || c == 0x7F) {
38 4 : return false;
39 : }
40 : }
41 16 : return std::string(name).find("..") == std::string::npos;
42 : }
43 :
44 7 : bool ValidateKernelName(const ExceptionDumpInfo &info)
45 : {
46 7 : if (strnlen(info.kernelName, MAX_KERNELNAME_LEN) == MAX_KERNELNAME_LEN ||
47 7 : strnlen(info.kernelDisplayName, MAX_KERNELNAME_LEN) == MAX_KERNELNAME_LEN) {
48 0 : return false;
49 : }
50 :
51 7 : if (!IsSafeKernelDisplayName(info.kernelName) || !IsSafeKernelDisplayName(info.kernelDisplayName)) {
52 4 : return false;
53 : }
54 3 : return true;
55 : }
56 :
57 9 : bool IsValidExceptionDumpMode(ExceptionDumpMode dumpMode)
58 : {
59 8 : return dumpMode == ExceptionDumpMode::DUMP_MODE_NONE ||
60 17 : dumpMode == ExceptionDumpMode::DUMP_MODE_OVERWRITE ||
61 9 : dumpMode == ExceptionDumpMode::DUMP_MODE_ADDITIONAL;
62 : }
63 : } // namespace
64 :
65 : uint64_t *g_dynamicChunk = nullptr;
66 : uint64_t *g_staticChunk = nullptr;
67 :
68 48 : ExceptionDumper::~ExceptionDumper()
69 : {
70 48 : if (g_dynamicChunk != nullptr) {
71 18 : delete[] g_dynamicChunk;
72 18 : g_dynamicChunk = nullptr;
73 : }
74 :
75 48 : if (g_staticChunk != nullptr) {
76 18 : delete[] g_staticChunk;
77 18 : g_staticChunk = nullptr;
78 : }
79 48 : KernelSymbolLocator::ClearCache();
80 48 : destructionFlag_ = true;
81 48 : }
82 :
83 57 : bool ExceptionDumper::InitArgsExceptionMemory() const
84 : {
85 57 : if (g_dynamicChunk == nullptr) {
86 7149942 : g_dynamicChunk = new (std::nothrow) uint64_t[DYNAMIC_RING_CHUNK_SIZE + DFX_MAX_TENSOR_NUM + RESERVE_SPACE]();
87 18 : if (g_dynamicChunk == nullptr) {
88 0 : return false;
89 : }
90 : }
91 :
92 57 : if (g_staticChunk == nullptr) {
93 2431350 : g_staticChunk = new (std::nothrow) uint64_t[STATIC_RING_CHUNK_SIZE + DFX_MAX_TENSOR_NUM + RESERVE_SPACE]();
94 18 : if (g_staticChunk == nullptr) {
95 0 : return false;
96 : }
97 : }
98 57 : return true;
99 : }
100 :
101 123 : bool ExceptionDumper::IsRepeatEnableException(DumpType type, const DumpConfig &dumpConfig)
102 : {
103 123 : if (type == DumpType::ARGS_EXCEPTION || type == DumpType:: EXCEPTION ||
104 : type == DumpType::AIC_ERR_DETAIL_DUMP) {
105 56 : std::string dumpStatus = dumpConfig.dumpStatus;
106 56 : std::transform(dumpStatus.begin(), dumpStatus.end(), dumpStatus.begin(), ::tolower);
107 56 : if (dumpStatus != "on") {
108 11 : return false;
109 : }
110 45 : return GetCoredumpStatus() || GetExceptionStatus() || GetArgsExceptionStatus();
111 56 : }
112 67 : return false;
113 : }
114 :
115 77 : int32_t ExceptionDumper::ExceptionDumperInit(DumpType dumpType, const DumpConfig &dumpConfig)
116 : {
117 77 : bool status = false;
118 77 : if (!setting_.InitDumpStatus(dumpConfig.dumpStatus, status)) {
119 1 : IDE_LOGE("The value of dumpStatus: %s is invalid.", dumpConfig.dumpStatus.c_str());
120 1 : return ADUMP_FAILED;
121 : }
122 :
123 : // status = false means turn off, flag = false means not start, print warning when not start but want to turn off
124 76 : if (dumpType == DumpType::EXCEPTION) {
125 13 : IDE_CTRL_VALUE_WARN(status || exceptionStatus_, return ADUMP_SUCCESS, "dump type %d not start.", dumpType);
126 12 : exceptionStatus_ = status;
127 63 : } else if (dumpType == DumpType::ARGS_EXCEPTION) {
128 57 : IDE_CTRL_VALUE_WARN(status || argsExceptionStatus_, return ADUMP_SUCCESS, "dump type %d not start.", dumpType);
129 53 : IDE_CTRL_VALUE_FAILED(InitArgsExceptionMemory(), return ADUMP_FAILED, "Init args exception memory failed.");
130 53 : if (status && !argsExceptionStatus_) {
131 47 : IDE_CTRL_VALUE_WARN(LoadTensorPluginLib() == ADUMP_SUCCESS, return ADUMP_FAILED,
132 : "Load tersor custom plugin failed.");
133 : }
134 53 : argsExceptionStatus_ = status;
135 : } else {
136 : // detail exception dump can not off
137 6 : if (status == false) {
138 : static bool havePrint = false;
139 2 : if (!havePrint) {
140 1 : IDE_RUN_LOGI("Can not turn off detail exception dump.");
141 1 : havePrint = true;
142 : }
143 2 : IDE_LOGW("Can not turn off detail exception dump.");
144 2 : return ADUMP_SUCCESS;
145 : }
146 4 : IDE_CTRL_VALUE_FAILED(InitArgsExceptionMemory(), return ADUMP_FAILED, "Init args exception memory failed.");
147 4 : coredumpStatus_ = true;
148 : }
149 :
150 69 : SetDumpPath(dumpConfig.dumpPath);
151 69 : return ADUMP_SUCCESS;
152 : }
153 :
154 61 : std::string ExceptionDumper::GetDumpSceneName() const
155 : {
156 61 : if (coredumpStatus_) {
157 3 : return ADUMP_DUMP_EXCEPTION_AIC_ERR_DETAIL;
158 58 : } else if (exceptionStatus_) {
159 1 : return ADUMP_DUMP_EXCEPTION_AIC_ERR_NORM;
160 57 : } else if (argsExceptionStatus_) {
161 57 : return ADUMP_DUMP_EXCEPTION_AIC_ERR_BRIEF;
162 : } else {
163 0 : return "unknown";
164 : }
165 : }
166 :
167 82 : int32_t ExceptionDumper::DumpException(const rtExceptionInfo &exception)
168 : {
169 82 : IDE_CTRL_VALUE_WARN(!destructionFlag_, return ADUMP_FAILED, "ExceptionDumper has been destructed.");
170 82 : IDE_CTRL_VALUE_WARN(ExceptionInfoCommon::IsSupportExceptionDump(exception),
171 : return ADUMP_FAILED, "Exception is not need to dump.");
172 68 : IDE_CTRL_VALUE_WARN(coredumpStatus_ || exceptionStatus_ || argsExceptionStatus_, return ADUMP_FAILED,
173 : "Not enable exception dump.");
174 66 : std::string dumpPath = CreateDeviceDumpPath(exception.deviceid);
175 66 : if (dumpPath.empty()) {
176 5 : return ADUMP_FAILED;
177 : }
178 :
179 61 : const std::string dumpScene = GetDumpSceneName();
180 61 : const std::string exceptionType = ExceptionInfoCommon::GetExceptionTaskTypeName(exception);
181 61 : const std::string kernelName = ExceptionInfoCommon::GetExceptionKernelName(exception);
182 61 : IDE_LOGE("[Dump][Exception] Begin to dump exception. dumpScene=%s, deviceId=%u, streamId=%u, taskId=%u, "
183 : "exceptionType=%d(%s), kernelName=%s.", dumpScene.c_str(), exception.deviceid, exception.streamid,
184 : exception.taskid, static_cast<int32_t>(exception.expandInfo.type), exceptionType.c_str(),
185 : kernelName.c_str());
186 :
187 61 : if (coredumpStatus_) {
188 3 : return DumpDetailException(exception, dumpPath);
189 58 : } else if (exceptionStatus_) {
190 1 : return DumpNormalException(exception, dumpPath);
191 : } else {
192 57 : return DumpArgsException(exception, dumpPath);
193 : }
194 66 : }
195 :
196 69 : void ExceptionDumper::SetDumpPath(const std::string &dumpPath)
197 : {
198 86 : dumpPath_ = dumpPath.empty() ? DEFAULT_DUMP_PATH : dumpPath;
199 69 : IDE_LOGI("Update exception dump path: %s", dumpPath_.c_str());
200 69 : }
201 :
202 4 : void ExceptionDumper::AddDumpOperator(const OperatorInfo &opInfo)
203 : {
204 4 : OperatorInfoV2 dumpOperator = {};
205 4 : DumpManager::Instance().ConvertOperatorInfo(opInfo, dumpOperator);
206 4 : AddDumpOperatorV2(dumpOperator);
207 4 : }
208 :
209 118 : void ExceptionDumper::AddDumpOperatorV2(const OperatorInfoV2 &opInfo)
210 : {
211 118 : const std::lock_guard<std::mutex> lock(mutex_);
212 118 : if (opInfo.agingFlag) {
213 6 : agingOperators_.emplace_back(opInfo);
214 6 : if (agingOperators_.size() > MAX_DUMP_OP_NUM) {
215 0 : agingOperators_.pop_front();
216 : }
217 : } else {
218 112 : uint32_t maxStreamCount = 0;
219 112 : uint32_t maxTaskCount = 0;
220 112 : rtGetMaxStreamAndTask(0, &maxStreamCount, &maxTaskCount);
221 112 : auto &taskDeque = residentOperators_[opInfo.deviceId][opInfo.streamId];
222 112 : taskDeque.emplace_back(opInfo);
223 112 : if (taskDeque.size() > maxTaskCount) {
224 2 : taskDeque.pop_front();
225 : }
226 : }
227 118 : IDE_LOGI("Dump operator size, aging: %zu, resident: %zu", agingOperators_.size(), residentOperators_.size());
228 118 : }
229 :
230 9 : int32_t ExceptionDumper::DelDumpOperator(uint32_t deviceId, uint32_t streamId)
231 : {
232 9 : const std::lock_guard<std::mutex> lock(mutex_);
233 9 : auto it = residentOperators_.find(deviceId);
234 9 : if (it != residentOperators_.end()) {
235 8 : it->second.erase(streamId);
236 8 : if (it->second.empty()) {
237 8 : residentOperators_.erase(deviceId);
238 : }
239 : }
240 9 : return ADUMP_SUCCESS;
241 9 : }
242 :
243 72 : std::string ExceptionDumper::CreateDumpPath(Path &dumpPath) const
244 : {
245 80 : IDE_CTRL_VALUE_FAILED(dumpPath.CreateDirectory(true), return "", "Create path[%s] failed",
246 : dumpPath.GetCString());
247 76 : IDE_CTRL_VALUE_FAILED(dumpPath.RealPath(), return "", "Get RealPath of [%s] failed, strerr=%s",
248 : dumpPath.GetCString(), strerror(errno));
249 64 : return dumpPath.GetString();
250 : }
251 :
252 66 : std::string ExceptionDumper::CreateDeviceDumpPath(uint32_t deviceId) const
253 : {
254 : // Create device dump path when exception call-backed
255 66 : std::string dumpPath = dumpPath_.empty() ? DEFAULT_DUMP_PATH : dumpPath_;
256 66 : Path deviceDumpPath(dumpPath);
257 66 : deviceDumpPath.Append(EXTRA_DUMP_PATH).Append(std::to_string(deviceId));
258 132 : return CreateDumpPath(deviceDumpPath);
259 66 : }
260 :
261 6 : std::string ExceptionDumper::CreateExtraDumpPath()
262 : {
263 : // Create extra dump path for tensor custom dump data
264 6 : std::string dumpPath = dumpPath_.empty() ? DEFAULT_DUMP_PATH : dumpPath_;
265 6 : Path extraDumpPath(dumpPath);
266 6 : extraDumpPath.Append(EXTRA_DUMP_PATH);
267 6 : extraDumpPath_ = CreateDumpPath(extraDumpPath);
268 12 : return extraDumpPath_;
269 6 : }
270 :
271 7 : const char* ExceptionDumper::GetExtraDumpCPath() const
272 : {
273 7 : return extraDumpPath_.empty() ? nullptr : extraDumpPath_.c_str();
274 : }
275 :
276 0 : bool ExceptionDumper::FindExceptionOperator(const rtExceptionInfo &exception, DumpOperator &excOp)
277 : {
278 0 : OpIdentity excOpIdentiy(exception.deviceid, exception.taskid, exception.streamid);
279 0 : if (exception.expandInfo.type == RT_EXCEPTION_FFTS_PLUS) {
280 0 : uint32_t contextId = static_cast<uint32_t>(exception.expandInfo.u.fftsPlusInfo.contextId);
281 0 : uint32_t threadId = static_cast<uint32_t>(exception.expandInfo.u.fftsPlusInfo.threadId);
282 0 : IDE_LOGI("ffts+ op context id: %u, thread id: %u.", contextId, threadId);
283 0 : excOpIdentiy.contextId = contextId;
284 : }
285 :
286 0 : const std::lock_guard<std::mutex> lock(mutex_);
287 0 : IDE_LOGI("Dump op size, aging:%zu, resident:%zu, target: %s", agingOperators_.size(), residentOperators_.size(),
288 : excOpIdentiy.GetString().c_str());
289 0 : for (const auto &op : agingOperators_) {
290 0 : if (op.IsBelongTo(excOpIdentiy)) {
291 0 : IDE_LOGI("Find exception op in aging list: %s", excOpIdentiy.GetString().c_str());
292 0 : excOp = op;
293 0 : return true;
294 : }
295 : }
296 :
297 0 : if (residentOperators_.find(excOpIdentiy.deviceId) != residentOperators_.end()) {
298 0 : auto &streamMap = residentOperators_[excOpIdentiy.deviceId];
299 0 : if (streamMap.find(excOpIdentiy.streamId) != streamMap.end()) {
300 0 : auto &taskDeque = streamMap[excOpIdentiy.streamId];
301 0 : for (const auto &op : taskDeque) {
302 0 : if (op.IsBelongTo(excOpIdentiy)) {
303 0 : IDE_LOGI("Find exception op in resident list: %s", excOpIdentiy.GetString().c_str());
304 0 : excOp = op;
305 0 : return true;
306 : }
307 : }
308 : }
309 : }
310 :
311 0 : IDE_LOGW("Not find dump operator, target: %s", excOpIdentiy.GetString().c_str());
312 0 : return false;
313 0 : }
314 :
315 1 : int32_t ExceptionDumper::DumpNormalException(const rtExceptionInfo &exception, const std::string &dumpPath)
316 : {
317 1 : IDE_CTRL_VALUE_WARN(ExceptionInfoCommon::IsSupportDefaultExceptionDump(exception),
318 : return ADUMP_FAILED, "Exception is not support default dump.");
319 0 : KernelSymbolLocator::DumpErrorSymbols(exception);
320 0 : DumpOperator excOp;
321 0 : bool find = FindExceptionOperator(exception, excOp);
322 0 : if (!find) {
323 0 : return ADUMP_SUCCESS;
324 : }
325 :
326 0 : rtExceptionArgsInfo_t exceptionArgsInfo{};
327 0 : if (ExceptionInfoCommon::GetExceptionInfo(exception, exceptionArgsInfo) != ADUMP_SUCCESS) {
328 0 : IDE_LOGW("Get exception args info failed.");
329 0 : return ADUMP_FAILED;
330 : }
331 0 : (void)excOp.RefreshAddrs(exceptionArgsInfo);
332 0 : (void)excOp.LogExceptionInfo(exceptionArgsInfo);
333 0 : (void)excOp.CopyOpKernelFile();
334 :
335 0 : const int32_t ret = excOp.DumpException(exception.deviceid, dumpPath);
336 0 : if (ret != ADUMP_SUCCESS) {
337 0 : return ADUMP_FAILED;
338 : }
339 0 : return ADUMP_SUCCESS;
340 0 : }
341 :
342 49 : int32_t ExceptionDumper::DumpArgsExceptionDefault(const rtExceptionInfo &exception, const std::string &dumpPath)
343 : {
344 49 : IDE_CTRL_VALUE_WARN(ExceptionInfoCommon::IsSupportDefaultExceptionDump(exception),
345 : return ADUMP_FAILED, "Exception is not support default dump.");
346 48 : KernelSymbolLocator::DumpErrorSymbols(exception);
347 48 : DumpArgs args;
348 48 : if (args.LoadArgsExceptionInfo(exception) != ADUMP_SUCCESS) {
349 30 : return ADUMP_FAILED;
350 : }
351 :
352 18 : if (args.DumpArgsExceptionInfo(exception.deviceid, dumpPath) != ADUMP_SUCCESS) {
353 0 : return ADUMP_FAILED;
354 : }
355 18 : return ADUMP_SUCCESS;
356 48 : }
357 :
358 56 : int32_t ExceptionDumper::DumpArgsExceptionInner(const rtExceptionInfo &exception, const std::string &dumpPath)
359 : {
360 56 : ExceptionDumpMode dumpMode = ExceptionDumpMode::DUMP_MODE_NONE;
361 56 : std::vector<ExceptionDumpInfo> dumpInfos;
362 56 : int32_t invokeRet = InvokeCallbacks(exception, dumpInfos, dumpMode);
363 56 : if (invokeRet != ADUMP_SUCCESS) {
364 0 : return invokeRet;
365 : }
366 56 : if (dumpMode == ExceptionDumpMode::DUMP_MODE_OVERWRITE) {
367 5 : if (dumpInfos.empty()) {
368 3 : IDE_LOGE("OVERWRITE mode with empty callback data after validation, reject.");
369 3 : return ADUMP_FAILED;
370 : }
371 2 : IDE_LOGI("DumpMode of callbacks is OVERWRITE, skip default dump, dump callback data only.");
372 2 : DumpCallbackData(exception, dumpInfos, dumpPath);
373 2 : return ADUMP_SUCCESS;
374 : }
375 :
376 51 : if (dumpMode == ExceptionDumpMode::DUMP_MODE_ADDITIONAL) {
377 2 : if (!dumpInfos.empty()) {
378 1 : IDE_LOGI("DumpMode of callbacks is ADDITIONAL, dump callback data and default.");
379 1 : DumpCallbackData(exception, dumpInfos, dumpPath);
380 : }
381 : }
382 :
383 51 : return DumpArgsExceptionDefault(exception, dumpPath);
384 56 : }
385 :
386 56 : int32_t ExceptionDumper::InvokeCallbacks(const rtExceptionInfo &exception,
387 : std::vector<ExceptionDumpInfo> &dumpInfos,
388 : ExceptionDumpMode &dumpMode)
389 : {
390 56 : dumpMode = ExceptionDumpMode::DUMP_MODE_NONE;
391 56 : dumpInfos.clear();
392 :
393 56 : std::lock_guard<std::recursive_mutex> lock(callbackMutex_);
394 56 : if (callbacks_.empty()) {
395 47 : IDE_LOGI("No exception callbacks registered.");
396 47 : return ADUMP_SUCCESS;
397 : }
398 :
399 9 : ExceptionRegInfo exceptionRegInfo{0, nullptr};
400 9 : int32_t ret = ExceptionInfoCommon::GetExceptionRegInfo(exception, exceptionRegInfo);
401 9 : IDE_CTRL_VALUE_WARN(ret == ADUMP_SUCCESS, return ADUMP_FAILED,
402 : "GetExceptionRegInfo failed, ret=%d, coreNum=%u.", ret, exceptionRegInfo.coreNum);
403 :
404 9 : uint32_t maxDumpSize = exceptionRegInfo.coreNum == 0 ? 1 : exceptionRegInfo.coreNum;
405 9 : IDE_LOGI("Exception callbacks size=%zu, maxDumpSize=%u.", callbacks_.size(), maxDumpSize);
406 :
407 18 : for (auto &callback : callbacks_) {
408 9 : std::vector<ExceptionDumpInfo> cbDumpInfos(maxDumpSize);
409 :
410 9 : uint32_t cbDumpRealSize = 0;
411 9 : ExceptionDumpMode cbDumpMode = ExceptionDumpMode::DUMP_MODE_NONE;
412 :
413 9 : uint32_t ret = callback(reinterpret_cast<void*>(const_cast<rtExceptionInfo*>(&exception)), cbDumpInfos.data(),
414 : maxDumpSize, &cbDumpRealSize, &cbDumpMode);
415 9 : if (ret != ADUMP_SUCCESS) {
416 0 : IDE_LOGW("Callback execute failed. ret=%u", ret);
417 0 : continue;
418 : }
419 9 : if (!IsValidExceptionDumpMode(cbDumpMode)) {
420 1 : IDE_LOGW("Callback dumpMode invalid. dumpMode=%u", static_cast<uint32_t>(cbDumpMode));
421 1 : continue;
422 : }
423 8 : if (cbDumpRealSize == 0 || cbDumpRealSize > maxDumpSize) {
424 1 : IDE_LOGW("Callback dumpRealSize invalid. maxDumpSize=%u, dumpRealSize=%u", maxDumpSize, cbDumpRealSize);
425 1 : continue;
426 : }
427 :
428 7 : IDE_LOGI("Callback dumpRealSize=%u, dumpMode=%d.", cbDumpRealSize, cbDumpMode);
429 14 : for (uint32_t i = 0; i < cbDumpRealSize; ++i) {
430 7 : if (!ValidateKernelName(cbDumpInfos[i])) {
431 4 : IDE_LOGW("Invalid kernelName/kernelDisplayName in callback data, skip. index=%u", i);
432 4 : continue;
433 : }
434 3 : dumpInfos.push_back(cbDumpInfos[i]);
435 : }
436 :
437 : // 多回调聚合时取优先级最高者：ADDITIONAL > OVERWRITE > NONE
438 7 : if (static_cast<uint32_t>(cbDumpMode) > static_cast<uint32_t>(dumpMode)) {
439 7 : dumpMode = cbDumpMode;
440 : }
441 9 : }
442 :
443 9 : IDE_LOGI("Callback final dumpMode=%d, dumpInfo total size=%zu.", dumpMode, dumpInfos.size());
444 9 : return ADUMP_SUCCESS;
445 56 : }
446 :
447 1 : void ExceptionDumper::DumpCallbackData(const rtExceptionInfo &exception,
448 : const std::vector<ExceptionDumpInfo> &dumpInfos,
449 : const std::string &dumpPath)
450 : {
451 2 : for (const auto &info : dumpInfos) {
452 1 : IDE_LOGE("[Dump][Exception] Begin to dump callback exception. coreType=%u, coreId=%u, "
453 : "argAddr=%p, argSize=%u, binHandle=%p, extraTensorNum=%u, kernelName=%s.",
454 : info.coreType, info.coreId, info.argAddr, info.argSize, info.bin, info.extraTensorNum, info.kernelName);
455 1 : IDE_LOGE("[Dump][Exception] Callback exception. kernelDisplayName=%s.", info.kernelDisplayName);
456 1 : DumpArgsCallback argsCallback(exception, info, dumpPath);
457 1 : int32_t ret = argsCallback.DumpDfxArgs();
458 1 : if (ret != ADUMP_SUCCESS) {
459 0 : IDE_LOGW("Dump callback args failed.");
460 : }
461 :
462 1 : ret = argsCallback.DumpExtraTensors();
463 1 : if (ret != ADUMP_SUCCESS) {
464 0 : IDE_LOGW("Dump callback extra tensor failed.");
465 : }
466 :
467 1 : ret = argsCallback.Dump();
468 1 : if (ret != ADUMP_SUCCESS) {
469 0 : IDE_LOGW("Dump callback data to file failed.");
470 : }
471 :
472 1 : ret = argsCallback.DumpKernelBin();
473 1 : if (ret != ADUMP_SUCCESS) {
474 0 : IDE_LOGW("Dump callback kernel bin failed.");
475 : }
476 :
477 1 : ret = argsCallback.DumpKernelErrorSymbols();
478 1 : if (ret != ADUMP_SUCCESS) {
479 0 : IDE_LOGW("Dump callback kernel error symbols failed.");
480 : }
481 :
482 1 : IDE_LOGI("Dump callback data finished, kernelName=%s.", info.kernelName);
483 1 : }
484 1 : KernelSymbolLocator::ClearCache();
485 1 : IDE_LOGI("Dump all callback data success.");
486 1 : }
487 :
488 5 : void ExceptionDumper::ExceptionModeDowngrade()
489 : {
490 5 : coredumpEnableComplete_ = false;
491 5 : }
492 :
493 0 : void ExceptionDumper::Exit() const
494 : {
495 0 : IDE_LOGE("The process exits.");
496 0 : AdxLogFlush();
497 : #ifndef __ADUMP_LLT
498 : _exit(EXIT_FAILURE);
499 : #endif
500 0 : }
501 :
502 : #ifdef __ADUMP_LLT
503 571 : void ExceptionDumper::Reset()
504 : {
505 571 : const std::lock_guard<std::mutex> lock(mutex_);
506 571 : agingOperators_.clear();
507 571 : residentOperators_.clear();
508 571 : exceptionStatus_ = false;
509 571 : argsExceptionStatus_ = false;
510 571 : coredumpStatus_ = false;
511 571 : coredumpEnableComplete_ = true;
512 571 : }
513 : #endif
514 :
515 32 : int32_t ExceptionDumper::RegisterExceptionDumpCallback(ExceptionDumpCallback callback)
516 : {
517 32 : if (callback == nullptr) {
518 7 : IDE_LOGE("Register callback is nullptr.");
519 7 : return ADUMP_INPUT_FAILED;
520 : }
521 :
522 25 : const std::lock_guard<std::recursive_mutex> lock(callbackMutex_);
523 25 : for (auto &cb : callbacks_) {
524 4 : if (cb == callback) {
525 4 : IDE_LOGW("Callback already registered.");
526 4 : return ADUMP_SUCCESS;
527 : }
528 : }
529 21 : callbacks_.push_back(callback);
530 21 : IDE_LOGI("Callback registered, total=%zu.", callbacks_.size());
531 21 : return ADUMP_SUCCESS;
532 25 : }
533 :
534 21 : int32_t ExceptionDumper::UnregisterExceptionDumpCallback(ExceptionDumpCallback callback)
535 : {
536 21 : if (callback == nullptr) {
537 7 : IDE_LOGE("Unregister callback is nullptr.");
538 7 : return ADUMP_INPUT_FAILED;
539 : }
540 :
541 14 : const std::lock_guard<std::recursive_mutex> lock(callbackMutex_);
542 14 : for (auto it = callbacks_.begin(); it != callbacks_.end(); ++it) {
543 10 : if (*it == callback) {
544 10 : callbacks_.erase(it);
545 10 : IDE_LOGI("Callback unregistered, total=%zu.", callbacks_.size());
546 10 : return ADUMP_SUCCESS;
547 : }
548 : }
549 4 : IDE_LOGW("Callback not found.");
550 4 : return ADUMP_SUCCESS;
551 14 : }
552 :
553 : } // namespace Adx
|
