Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 : #include <set>
11 : #include <cctype>
12 : #include <cstring>
13 : #include <algorithm>
14 : #include "log/adx_log.h"
15 : #include "path.h"
16 : #include "dump_manager.h"
17 : #include "dump_config_converter.h"
18 : #include "sys_utils.h"
19 : #include "dump_args.h"
20 : #include "dump_args_callback.h"
21 : #include "dump_file.h"
22 : #include "exception_info_common.h"
23 : #include "kernel_symbol_locator.h"
24 : #include "kernel_info_collector.h"
25 : #include "kernel_source_symbolizer.h"
26 : #include "exception_dumper.h"
27 :
28 : namespace Adx {
29 : namespace {
30 : constexpr char DEFAULT_DUMP_PATH[] = "./";
31 : constexpr char EXTRA_DUMP_PATH[] = "/extra-info/data-dump/";
32 : constexpr size_t MAX_DUMP_OP_NUM = (2048U * 2048U) / 20U;
33 :
34 24 : static bool IsSafeKernelDisplayName(const char* name)
35 : {
36 188 : for (size_t i = 0; i < MAX_KERNELNAME_LEN && name[i] != '\0'; ++i) {
37 168 : char c = name[i];
38 168 : if (c == '/' || c == '\\' || c < 0x20 || c == 0x7F) {
39 4 : return false;
40 : }
41 : }
42 40 : return std::string(name).find("..") == std::string::npos;
43 : }
44 :
45 13 : bool ValidateKernelName(const ExceptionDumpInfo& info)
46 : {
47 13 : if (strnlen(info.kernelName, MAX_KERNELNAME_LEN) == MAX_KERNELNAME_LEN ||
48 13 : strnlen(info.kernelDisplayName, MAX_KERNELNAME_LEN) == MAX_KERNELNAME_LEN) {
49 0 : return false;
50 : }
51 :
52 13 : if (!IsSafeKernelDisplayName(info.kernelName) || !IsSafeKernelDisplayName(info.kernelDisplayName)) {
53 4 : return false;
54 : }
55 9 : return true;
56 : }
57 :
58 19 : bool IsValidExceptionDumpMode(ExceptionDumpMode dumpMode)
59 : {
60 19 : return dumpMode == ExceptionDumpMode::DUMP_MODE_NONE || dumpMode == ExceptionDumpMode::DUMP_MODE_OVERWRITE ||
61 19 : dumpMode == ExceptionDumpMode::DUMP_MODE_ADDITIONAL;
62 : }
63 : } // namespace
64 :
65 : uint64_t* g_dynamicChunk = nullptr;
66 : uint64_t* g_staticChunk = nullptr;
67 :
68 58 : ExceptionDumper::~ExceptionDumper()
69 : {
70 58 : if (g_dynamicChunk != nullptr) {
71 19 : delete[] g_dynamicChunk;
72 19 : g_dynamicChunk = nullptr;
73 : }
74 :
75 58 : if (g_staticChunk != nullptr) {
76 19 : delete[] g_staticChunk;
77 19 : g_staticChunk = nullptr;
78 : }
79 58 : KernelSymbolLocator::ClearCache();
80 58 : destructionFlag_ = true;
81 58 : }
82 :
83 58 : bool ExceptionDumper::InitArgsExceptionMemory() const
84 : {
85 58 : if (g_dynamicChunk == nullptr) {
86 7547161 : g_dynamicChunk = new (std::nothrow) uint64_t[DYNAMIC_RING_CHUNK_SIZE + DFX_MAX_TENSOR_NUM + RESERVE_SPACE]();
87 19 : if (g_dynamicChunk == nullptr) {
88 0 : return false;
89 : }
90 : }
91 :
92 58 : if (g_staticChunk == nullptr) {
93 2566425 : g_staticChunk = new (std::nothrow) uint64_t[STATIC_RING_CHUNK_SIZE + DFX_MAX_TENSOR_NUM + RESERVE_SPACE]();
94 19 : if (g_staticChunk == nullptr) {
95 0 : return false;
96 : }
97 : }
98 58 : return true;
99 : }
100 :
101 123 : bool ExceptionDumper::IsRepeatEnableException(DumpType type, const DumpConfig& dumpConfig)
102 : {
103 123 : if (type == DumpType::ARGS_EXCEPTION || type == DumpType::EXCEPTION || type == DumpType::AIC_ERR_DETAIL_DUMP) {
104 56 : std::string dumpStatus = dumpConfig.dumpStatus;
105 56 : std::transform(dumpStatus.begin(), dumpStatus.end(), dumpStatus.begin(), ::tolower);
106 56 : if (dumpStatus != "on") {
107 11 : return false;
108 : }
109 45 : return IsEnabledExceptionDump();
110 56 : }
111 67 : return false;
112 : }
113 :
114 135 : bool ExceptionDumper::IsEnabledExceptionDump() const
115 : {
116 135 : return coredumpStatus_ || exceptionStatus_ || argsExceptionStatus_;
117 : }
118 :
119 87 : int32_t ExceptionDumper::ExceptionDumperInit(DumpType dumpType, const DumpConfig& dumpConfig)
120 : {
121 87 : bool status = false;
122 87 : if (!setting_.InitDumpStatus(dumpConfig.dumpStatus, status)) {
123 1 : IDE_LOGE("The value of dumpStatus: %s is invalid.", dumpConfig.dumpStatus.c_str());
124 1 : return ADUMP_FAILED;
125 : }
126 :
127 : // status = false means turn off, flag = false means not start, print warning when not start but want to turn off
128 86 : if (dumpType == DumpType::EXCEPTION) {
129 22 : IDE_CTRL_VALUE_WARN(status || exceptionStatus_, return ADUMP_SUCCESS, "dump type %d not start.", dumpType);
130 21 : exceptionStatus_ = status;
131 64 : } else if (dumpType == DumpType::ARGS_EXCEPTION) {
132 58 : IDE_CTRL_VALUE_WARN(status || argsExceptionStatus_, return ADUMP_SUCCESS, "dump type %d not start.", dumpType);
133 54 : IDE_CTRL_VALUE_FAILED(InitArgsExceptionMemory(), return ADUMP_FAILED, "Init args exception memory failed.");
134 54 : if (status && !argsExceptionStatus_) {
135 48 : IDE_CTRL_VALUE_WARN(
136 : LoadTensorPluginLib() == ADUMP_SUCCESS, return ADUMP_FAILED, "Load tensor custom plugin failed.");
137 : }
138 54 : argsExceptionStatus_ = status;
139 : } else {
140 : // detail exception dump can not off
141 6 : if (status == false) {
142 : static bool havePrint = false;
143 2 : if (!havePrint) {
144 1 : IDE_RUN_LOGI("Can not turn off detail exception dump.");
145 1 : havePrint = true;
146 : }
147 2 : IDE_LOGW("Can not turn off detail exception dump.");
148 2 : return ADUMP_SUCCESS;
149 : }
150 4 : IDE_CTRL_VALUE_FAILED(InitArgsExceptionMemory(), return ADUMP_FAILED, "Init args exception memory failed.");
151 4 : coredumpStatus_ = true;
152 : }
153 :
154 79 : SetDumpPath(dumpConfig.dumpPath);
155 79 : return ADUMP_SUCCESS;
156 : }
157 :
158 71 : std::string ExceptionDumper::GetDumpSceneName() const
159 : {
160 71 : if (coredumpStatus_) {
161 3 : return ADUMP_DUMP_EXCEPTION_AIC_ERR_DETAIL;
162 68 : } else if (exceptionStatus_) {
163 10 : return ADUMP_DUMP_EXCEPTION_AIC_ERR_NORM;
164 58 : } else if (argsExceptionStatus_) {
165 58 : return ADUMP_DUMP_EXCEPTION_AIC_ERR_BRIEF;
166 : } else {
167 0 : return "unknown";
168 : }
169 : }
170 :
171 92 : int32_t ExceptionDumper::DumpException(const rtExceptionInfo& exception)
172 : {
173 92 : IDE_CTRL_VALUE_WARN(!destructionFlag_, return ADUMP_FAILED, "ExceptionDumper has been destructed.");
174 92 : IDE_CTRL_VALUE_WARN(
175 : ExceptionInfoCommon::IsSupportExceptionDump(exception), return ADUMP_FAILED, "Exception is not need to dump.");
176 78 : IDE_CTRL_VALUE_WARN(IsEnabledExceptionDump(), return ADUMP_FAILED, "Not enable exception dump.");
177 76 : std::string dumpPath = CreateDeviceDumpPath(exception.deviceid);
178 76 : if (dumpPath.empty()) {
179 5 : return ADUMP_FAILED;
180 : }
181 :
182 71 : const std::string dumpScene = GetDumpSceneName();
183 71 : const std::string exceptionType = ExceptionInfoCommon::GetExceptionTaskTypeName(exception);
184 71 : const std::string kernelName = ExceptionInfoCommon::GetExceptionKernelName(exception);
185 71 : IDE_LOGE(
186 : "[Dump][Exception] Begin to dump exception. dumpScene=%s, deviceId=%u, streamId=%u, taskId=%u, "
187 : "exceptionType=%d(%s), kernelName=%s.",
188 : dumpScene.c_str(), exception.deviceid, exception.streamid, exception.taskid,
189 : static_cast<int32_t>(exception.expandInfo.type), exceptionType.c_str(), kernelName.c_str());
190 :
191 71 : if (coredumpStatus_) {
192 3 : return DumpDetailException(exception, dumpPath);
193 68 : } else if (exceptionStatus_) {
194 10 : return DumpNormalException(exception, dumpPath);
195 : } else {
196 58 : return DumpArgsException(exception, dumpPath);
197 : }
198 76 : }
199 :
200 79 : void ExceptionDumper::SetDumpPath(const std::string& dumpPath)
201 : {
202 96 : dumpPath_ = dumpPath.empty() ? DEFAULT_DUMP_PATH : dumpPath;
203 79 : IDE_LOGI("Update exception dump path: %s", dumpPath_.c_str());
204 79 : }
205 :
206 4 : void ExceptionDumper::AddDumpOperator(const OperatorInfo& opInfo)
207 : {
208 4 : OperatorInfoV2 dumpOperator = {};
209 4 : DumpManager::Instance().ConvertOperatorInfo(opInfo, dumpOperator);
210 4 : AddDumpOperatorV2(dumpOperator);
211 4 : }
212 :
213 118 : void ExceptionDumper::AddDumpOperatorV2(const OperatorInfoV2& opInfo)
214 : {
215 118 : const std::lock_guard<std::mutex> lock(mutex_);
216 118 : if (opInfo.agingFlag) {
217 6 : agingOperators_.emplace_back(opInfo);
218 6 : if (agingOperators_.size() > MAX_DUMP_OP_NUM) {
219 0 : agingOperators_.pop_front();
220 : }
221 : } else {
222 112 : uint32_t maxStreamCount = 0;
223 112 : uint32_t maxTaskCount = 0;
224 112 : rtGetMaxStreamAndTask(0, &maxStreamCount, &maxTaskCount);
225 112 : auto& taskDeque = residentOperators_[opInfo.deviceId][opInfo.streamId];
226 112 : taskDeque.emplace_back(opInfo);
227 112 : if (taskDeque.size() > maxTaskCount) {
228 2 : taskDeque.pop_front();
229 : }
230 : }
231 118 : IDE_LOGI("Dump operator size, aging: %zu, resident: %zu", agingOperators_.size(), residentOperators_.size());
232 118 : }
233 :
234 9 : int32_t ExceptionDumper::DelDumpOperator(uint32_t deviceId, uint32_t streamId)
235 : {
236 9 : const std::lock_guard<std::mutex> lock(mutex_);
237 9 : auto it = residentOperators_.find(deviceId);
238 9 : if (it != residentOperators_.end()) {
239 8 : it->second.erase(streamId);
240 8 : if (it->second.empty()) {
241 8 : residentOperators_.erase(deviceId);
242 : }
243 : }
244 9 : return ADUMP_SUCCESS;
245 9 : }
246 :
247 106 : std::string ExceptionDumper::CreateDumpPath(Path& dumpPath) const
248 : {
249 126 : IDE_CTRL_VALUE_FAILED(dumpPath.CreateDirectory(true), return "", "Create path[%s] failed", dumpPath.GetCString());
250 104 : IDE_CTRL_VALUE_FAILED(
251 : dumpPath.RealPath(), return "", "Get RealPath of [%s] failed, strerror=%s", dumpPath.GetCString(),
252 : strerror(errno));
253 92 : return dumpPath.GetString();
254 : }
255 :
256 100 : std::string ExceptionDumper::CreateDeviceDumpPath(uint32_t deviceId) const
257 : {
258 : // Create device dump path when exception call-backed
259 100 : std::string dumpPath = dumpPath_.empty() ? DEFAULT_DUMP_PATH : dumpPath_;
260 100 : Path deviceDumpPath(dumpPath);
261 100 : deviceDumpPath.Append(EXTRA_DUMP_PATH).Append(std::to_string(deviceId));
262 200 : return CreateDumpPath(deviceDumpPath);
263 100 : }
264 :
265 24 : int32_t ExceptionDumper::GetExceptionDumpPath(std::string& path)
266 : {
267 24 : int32_t deviceId = 0;
268 24 : const rtError_t rtRet = rtGetDevice(&deviceId);
269 24 : IDE_CTRL_VALUE_FAILED(
270 : rtRet == RT_ERROR_NONE, return ADUMP_FAILED,
271 : "rtGetDevice failed when get the exception root dump path, ret=%d.", static_cast<int32_t>(rtRet));
272 :
273 24 : path = CreateDeviceDumpPath(static_cast<uint32_t>(deviceId));
274 24 : IDE_CTRL_VALUE_FAILED(!path.empty(), return ADUMP_FAILED, "exception dump root path is not ready.");
275 18 : return ADUMP_SUCCESS;
276 : }
277 :
278 15 : int32_t ExceptionDumper::SaveExceptionInfo(
279 : const std::string& fileName, const std::string& userTag, const std::vector<TensorInfo>& tensors)
280 : {
281 15 : std::string rootPath;
282 15 : IDE_CTRL_VALUE_FAILED(
283 : GetExceptionDumpPath(rootPath) == ADUMP_SUCCESS, return ADUMP_FAILED,
284 : "[SaveExceptionInfo] get exception dump root path failed.");
285 :
286 12 : std::string realFilePath;
287 12 : IDE_CTRL_VALUE_FAILED(
288 : Path::BuildFullPathUnderRoot(rootPath, fileName, realFilePath), return ADUMP_FAILED,
289 : "[SaveExceptionInfo] build full file path failed.");
290 :
291 12 : realFilePath += ".custom." + SysUtils::GetCurrentTimeWithMillisecond();
292 :
293 12 : int32_t devId = 0;
294 12 : const rtError_t rtRet = rtGetDevice(&devId);
295 12 : IDE_CTRL_VALUE_FAILED(
296 : rtRet == RT_ERROR_NONE, return ADUMP_FAILED, "[SaveExceptionInfo] rtGetDevice failed, ret=%d.",
297 : static_cast<int32_t>(rtRet));
298 :
299 12 : DumpFile dumpFile(static_cast<uint32_t>(devId), realFilePath);
300 12 : dumpFile.SetHeader(Path(realFilePath).GetFileName());
301 12 : if (!userTag.empty()) {
302 18 : dumpFile.SetOpAttr("user_tag", userTag);
303 : }
304 :
305 12 : std::vector<std::string> record;
306 12 : dumpFile.SetTensors(tensors, record);
307 :
308 12 : IDE_CTRL_VALUE_FAILED(
309 : dumpFile.Dump(record) == ADUMP_SUCCESS, return ADUMP_FAILED,
310 : "[SaveExceptionInfo] dump exception info to file failed, file: %s", realFilePath.c_str());
311 :
312 9 : (void)mmChmod(realFilePath.c_str(), M_IRUSR);
313 9 : IDE_LOGE("[Dump][Exception] dump custom exception to file, file: %s", realFilePath.c_str());
314 9 : return ADUMP_SUCCESS;
315 15 : }
316 :
317 6 : std::string ExceptionDumper::CreateExtraDumpPath()
318 : {
319 : // Create extra dump path for tensor custom dump data
320 6 : std::string dumpPath = dumpPath_.empty() ? DEFAULT_DUMP_PATH : dumpPath_;
321 6 : Path extraDumpPath(dumpPath);
322 6 : extraDumpPath.Append(EXTRA_DUMP_PATH);
323 6 : extraDumpPath_ = CreateDumpPath(extraDumpPath);
324 12 : return extraDumpPath_;
325 6 : }
326 :
327 7 : const char* ExceptionDumper::GetExtraDumpCPath() const
328 : {
329 7 : return extraDumpPath_.empty() ? nullptr : extraDumpPath_.c_str();
330 : }
331 :
332 0 : bool ExceptionDumper::FindExceptionOperator(const rtExceptionInfo& exception, DumpOperator& excOp)
333 : {
334 0 : OpIdentity excOpIdentiy(exception.deviceid, exception.taskid, exception.streamid);
335 0 : if (exception.expandInfo.type == RT_EXCEPTION_FFTS_PLUS) {
336 0 : uint32_t contextId = static_cast<uint32_t>(exception.expandInfo.u.fftsPlusInfo.contextId);
337 0 : uint32_t threadId = static_cast<uint32_t>(exception.expandInfo.u.fftsPlusInfo.threadId);
338 0 : IDE_LOGI("ffts+ op context id: %u, thread id: %u.", contextId, threadId);
339 0 : excOpIdentiy.contextId = contextId;
340 : }
341 :
342 0 : const std::lock_guard<std::mutex> lock(mutex_);
343 0 : IDE_LOGI(
344 : "Dump op size, aging:%zu, resident:%zu, target: %s", agingOperators_.size(), residentOperators_.size(),
345 : excOpIdentiy.GetString().c_str());
346 0 : for (const auto& op : agingOperators_) {
347 0 : if (op.IsBelongTo(excOpIdentiy)) {
348 0 : IDE_LOGI("Find exception op in aging list: %s", excOpIdentiy.GetString().c_str());
349 0 : excOp = op;
350 0 : return true;
351 : }
352 : }
353 :
354 0 : if (residentOperators_.find(excOpIdentiy.deviceId) != residentOperators_.end()) {
355 0 : auto& streamMap = residentOperators_[excOpIdentiy.deviceId];
356 0 : if (streamMap.find(excOpIdentiy.streamId) != streamMap.end()) {
357 0 : auto& taskDeque = streamMap[excOpIdentiy.streamId];
358 0 : for (const auto& op : taskDeque) {
359 0 : if (op.IsBelongTo(excOpIdentiy)) {
360 0 : IDE_LOGI("Find exception op in resident list: %s", excOpIdentiy.GetString().c_str());
361 0 : excOp = op;
362 0 : return true;
363 : }
364 : }
365 : }
366 : }
367 :
368 0 : IDE_LOGW("Not find dump operator, target: %s", excOpIdentiy.GetString().c_str());
369 0 : return false;
370 0 : }
371 :
372 49 : void ExceptionDumper::DumpHostKernelBinBeforeSymbolize(
373 : const rtExceptionInfo& exception, const std::string& dumpPath) const
374 : {
375 49 : rtExceptionArgsInfo_t exceptionArgsInfo{};
376 49 : int32_t ret = ExceptionInfoCommon::GetExceptionInfo(exception, exceptionArgsInfo);
377 49 : IDE_CTRL_VALUE_WARN(
378 : ret == ADUMP_SUCCESS, return, "Get exception args info failed, skip early dump host kernel bin.");
379 :
380 49 : KernelInfoCollector collector;
381 49 : collector.LoadKernelInfo(exceptionArgsInfo);
382 49 : ret = collector.LoadKernelBinBuffer();
383 49 : IDE_CTRL_VALUE_WARN(
384 : ret == ADUMP_SUCCESS, return, "Load kernel bin buffer failed, skip early dump host kernel bin.");
385 :
386 : // 提前无条件同步落 _host.o（只依赖 kernel bin buffer，不依赖 ELF 符号解析），
387 : // 保证符号表损坏等场景下 _host.o 仍能落盘供事后分析；DumpHostKernelBin 幂等，
388 : // 后续 symbolize/慢搜索路径复用已落盘文件不会重复写。
389 49 : std::string hostOPath;
390 49 : (void)collector.DumpHostKernelBin(dumpPath, hostOPath);
391 49 : }
392 :
393 10 : int32_t ExceptionDumper::DumpNormalException(const rtExceptionInfo& exception, const std::string& dumpPath)
394 : {
395 10 : return DumpExceptionWithCallbacks(exception, dumpPath, &ExceptionDumper::DumpNormalExceptionDefault);
396 : }
397 :
398 1 : int32_t ExceptionDumper::DumpNormalExceptionDefault(const rtExceptionInfo& exception, const std::string& dumpPath)
399 : {
400 1 : IDE_CTRL_VALUE_WARN(
401 : ExceptionInfoCommon::IsSupportDefaultExceptionDump(exception), return ADUMP_FAILED,
402 : "Exception is not support default dump.");
403 : // 先无条件提前落 _host.o，再符号化（symbolize 复用已落盘文件，不重复落盘）。
404 0 : DumpHostKernelBinBeforeSymbolize(exception, dumpPath);
405 0 : KernelSymbolLocator::DumpErrorSymbols(exception, dumpPath);
406 0 : DumpOperator excOp;
407 0 : bool find = FindExceptionOperator(exception, excOp);
408 0 : if (!find) {
409 0 : return ADUMP_SUCCESS;
410 : }
411 :
412 0 : rtExceptionArgsInfo_t exceptionArgsInfo{};
413 0 : if (ExceptionInfoCommon::GetExceptionInfo(exception, exceptionArgsInfo) != ADUMP_SUCCESS) {
414 0 : IDE_LOGW("Get exception args info failed.");
415 0 : return ADUMP_FAILED;
416 : }
417 0 : (void)excOp.RefreshAddrs(exceptionArgsInfo);
418 0 : (void)excOp.LogExceptionInfo(exceptionArgsInfo);
419 0 : (void)excOp.CopyOpKernelFile();
420 :
421 0 : const int32_t ret = excOp.DumpException(exception.deviceid, dumpPath);
422 0 : if (ret != ADUMP_SUCCESS) {
423 0 : return ADUMP_FAILED;
424 : }
425 0 : return ADUMP_SUCCESS;
426 0 : }
427 :
428 50 : int32_t ExceptionDumper::DumpArgsExceptionDefault(const rtExceptionInfo& exception, const std::string& dumpPath)
429 : {
430 50 : IDE_CTRL_VALUE_WARN(
431 : ExceptionInfoCommon::IsSupportDefaultExceptionDump(exception), return ADUMP_FAILED,
432 : "Exception is not support default dump.");
433 : // 先无条件提前落 _host.o，再符号化（symbolize 复用已落盘文件，不重复落盘）。
434 49 : DumpHostKernelBinBeforeSymbolize(exception, dumpPath);
435 49 : KernelSymbolLocator::DumpErrorSymbols(exception, dumpPath);
436 49 : DumpArgs args;
437 49 : if (args.LoadArgsExceptionInfo(exception) != ADUMP_SUCCESS) {
438 31 : return ADUMP_FAILED;
439 : }
440 :
441 18 : if (args.DumpArgsExceptionInfo(exception.deviceid, dumpPath) != ADUMP_SUCCESS) {
442 0 : return ADUMP_FAILED;
443 : }
444 18 : return ADUMP_SUCCESS;
445 49 : }
446 :
447 67 : int32_t ExceptionDumper::DumpExceptionWithCallbacks(
448 : const rtExceptionInfo& exception, const std::string& dumpPath,
449 : int32_t (ExceptionDumper::*defaultDump)(const rtExceptionInfo&, const std::string&))
450 : {
451 67 : ExceptionDumpMode dumpMode = ExceptionDumpMode::DUMP_MODE_NONE;
452 67 : std::vector<ExceptionDumpInfo> dumpInfos;
453 67 : int32_t invokeRet = InvokeCallbacks(exception, dumpInfos, dumpMode);
454 67 : if (invokeRet != ADUMP_SUCCESS) {
455 1 : return invokeRet;
456 : }
457 66 : if (dumpMode == ExceptionDumpMode::DUMP_MODE_OVERWRITE) {
458 7 : if (dumpInfos.empty()) {
459 3 : IDE_LOGE("OVERWRITE mode with empty callback data after validation, reject.");
460 3 : return ADUMP_FAILED;
461 : }
462 4 : IDE_LOGI("DumpMode of callbacks is OVERWRITE, skip default dump, dump callback data only.");
463 4 : DumpCallbackData(exception, dumpInfos, dumpPath);
464 4 : return ADUMP_SUCCESS;
465 : }
466 :
467 59 : if (dumpMode == ExceptionDumpMode::DUMP_MODE_ADDITIONAL && !dumpInfos.empty()) {
468 4 : IDE_LOGI("DumpMode of callbacks is ADDITIONAL, dump callback data and default.");
469 4 : DumpCallbackData(exception, dumpInfos, dumpPath);
470 : }
471 :
472 59 : return (this->*defaultDump)(exception, dumpPath);
473 67 : }
474 :
475 57 : int32_t ExceptionDumper::DumpArgsExceptionInner(const rtExceptionInfo& exception, const std::string& dumpPath)
476 : {
477 57 : return DumpExceptionWithCallbacks(exception, dumpPath, &ExceptionDumper::DumpArgsExceptionDefault);
478 : }
479 :
480 67 : int32_t ExceptionDumper::InvokeCallbacks(
481 : const rtExceptionInfo& exception, std::vector<ExceptionDumpInfo>& dumpInfos, ExceptionDumpMode& dumpMode)
482 : {
483 67 : dumpMode = ExceptionDumpMode::DUMP_MODE_NONE;
484 67 : dumpInfos.clear();
485 :
486 67 : std::lock_guard<std::recursive_mutex> lock(callbackMutex_);
487 67 : if (callbacks_.empty()) {
488 50 : IDE_LOGI("No exception callbacks registered.");
489 50 : return ADUMP_SUCCESS;
490 : }
491 :
492 17 : ExceptionRegInfo exceptionRegInfo{0, nullptr};
493 17 : int32_t ret = ExceptionInfoCommon::GetExceptionRegInfo(exception, exceptionRegInfo);
494 17 : IDE_CTRL_VALUE_WARN(
495 : ret == ADUMP_SUCCESS, return ADUMP_FAILED, "GetExceptionRegInfo failed, ret=%d, coreNum=%u.", ret,
496 : exceptionRegInfo.coreNum);
497 :
498 16 : uint32_t maxDumpSize = exceptionRegInfo.coreNum == 0 ? 1 : exceptionRegInfo.coreNum;
499 16 : IDE_LOGI("Exception callbacks size=%zu, maxDumpSize=%u.", callbacks_.size(), maxDumpSize);
500 :
501 70 : for (auto& callback : callbacks_) {
502 19 : std::vector<ExceptionDumpInfo> cbDumpInfos(maxDumpSize);
503 :
504 19 : uint32_t cbDumpRealSize = 0;
505 19 : ExceptionDumpMode cbDumpMode = ExceptionDumpMode::DUMP_MODE_NONE;
506 :
507 19 : uint32_t ret = callback(
508 : reinterpret_cast<void*>(const_cast<rtExceptionInfo*>(&exception)), cbDumpInfos.data(), maxDumpSize,
509 : &cbDumpRealSize, &cbDumpMode);
510 19 : if (ret != ADUMP_SUCCESS) {
511 0 : IDE_LOGW("Callback execute failed. ret=%u", ret);
512 0 : continue;
513 : }
514 19 : if (!IsValidExceptionDumpMode(cbDumpMode)) {
515 1 : IDE_LOGW("Callback dumpMode invalid. dumpMode=%u", static_cast<uint32_t>(cbDumpMode));
516 1 : continue;
517 : }
518 18 : if (cbDumpRealSize == 0 || cbDumpRealSize > maxDumpSize) {
519 5 : IDE_LOGW("Callback dumpRealSize invalid. maxDumpSize=%u, dumpRealSize=%u", maxDumpSize, cbDumpRealSize);
520 5 : continue;
521 : }
522 :
523 13 : IDE_LOGI("Callback dumpRealSize=%u, dumpMode=%d.", cbDumpRealSize, cbDumpMode);
524 26 : for (uint32_t i = 0; i < cbDumpRealSize; ++i) {
525 13 : if (!ValidateKernelName(cbDumpInfos[i])) {
526 4 : IDE_LOGW("Invalid kernelName/kernelDisplayName in callback data, skip. index=%u", i);
527 4 : continue;
528 : }
529 9 : dumpInfos.push_back(cbDumpInfos[i]);
530 : }
531 :
532 : // 多回调聚合时取优先级最高者：ADDITIONAL > OVERWRITE > NONE
533 13 : if (static_cast<uint32_t>(cbDumpMode) > static_cast<uint32_t>(dumpMode)) {
534 13 : dumpMode = cbDumpMode;
535 : }
536 19 : }
537 :
538 16 : IDE_LOGI("Callback final dumpMode=%d, dumpInfo total size=%zu.", dumpMode, dumpInfos.size());
539 16 : return ADUMP_SUCCESS;
540 67 : }
541 :
542 1 : void ExceptionDumper::DumpCallbackData(
543 : const rtExceptionInfo& exception, const std::vector<ExceptionDumpInfo>& dumpInfos, const std::string& dumpPath)
544 : {
545 : // 收集每个回调 info(单核) 的定位结果，循环结束后按 (.o, 偏移) 聚类打印。
546 1 : std::vector<ErrorLocation> allLocations;
547 3 : for (const auto& info : dumpInfos) {
548 1 : IDE_LOGE(
549 : "[Dump][Exception] Begin to dump callback exception. coreType=%u, coreId=%u, "
550 : "argAddr=%p, argSize=%u, binHandle=%p, extraTensorNum=%u, kernelName=%s.",
551 : info.coreType, info.coreId, info.argAddr, info.argSize, info.bin, info.extraTensorNum, info.kernelName);
552 1 : IDE_LOGE("[Dump][Exception] Callback exception. kernelDisplayName=%s.", info.kernelDisplayName);
553 1 : DumpArgsCallback argsCallback(exception, info, dumpPath);
554 1 : int32_t ret = argsCallback.DumpDfxArgs();
555 1 : if (ret != ADUMP_SUCCESS) {
556 0 : IDE_LOGW("Dump callback args failed.");
557 : }
558 :
559 1 : ret = argsCallback.DumpExtraTensors();
560 1 : if (ret != ADUMP_SUCCESS) {
561 0 : IDE_LOGW("Dump callback extra tensor failed.");
562 : }
563 :
564 1 : ret = argsCallback.Dump();
565 1 : if (ret != ADUMP_SUCCESS) {
566 0 : IDE_LOGW("Dump callback data to file failed.");
567 : }
568 :
569 : // 先落 _host.o(同步)，DumpKernelErrorSymbols 内定位偏移后对其批量 symbolize。
570 1 : ret = argsCallback.DumpKernelBin();
571 1 : if (ret != ADUMP_SUCCESS) {
572 0 : IDE_LOGW("Dump callback kernel bin failed.");
573 : }
574 :
575 1 : ErrorLocation location;
576 1 : ret = argsCallback.DumpKernelErrorSymbols(location);
577 1 : if (ret != ADUMP_SUCCESS) {
578 0 : IDE_LOGW("Dump callback kernel error symbols failed.");
579 : } else {
580 1 : allLocations.push_back(location);
581 : }
582 :
583 1 : IDE_LOGI("Dump callback data finished, kernelName=%s.", info.kernelName);
584 1 : }
585 : // 各 core 的源码解析已在 DumpKernelErrorSymbols → 批量 symbolize 内完成并回填，
586 : // 此处只在工具可用时统一打印聚类汇总（工具不可用时汇总无源码信息、无增量价值）。
587 1 : if (KernelSourceSymbolizer::IsAvailable()) {
588 0 : KernelSymbolLocator::PrintClassificationSummary(allLocations);
589 : }
590 1 : KernelSymbolLocator::ClearCache();
591 1 : IDE_LOGI("Dump all callback data success.");
592 1 : }
593 :
594 5 : void ExceptionDumper::ExceptionModeDowngrade() { coredumpEnableComplete_ = false; }
595 :
596 0 : void ExceptionDumper::Exit() const
597 : {
598 0 : IDE_LOGE("The process exits.");
599 0 : AdxLogFlush();
600 : #ifndef __ADUMP_LLT
601 : _exit(EXIT_FAILURE);
602 : #endif
603 0 : }
604 :
605 : #ifdef __ADUMP_LLT
606 654 : void ExceptionDumper::Reset()
607 : {
608 654 : const std::lock_guard<std::mutex> lock(mutex_);
609 654 : agingOperators_.clear();
610 654 : residentOperators_.clear();
611 654 : exceptionStatus_ = false;
612 654 : argsExceptionStatus_ = false;
613 654 : coredumpStatus_ = false;
614 654 : coredumpEnableComplete_ = true;
615 654 : }
616 : #endif
617 :
618 44 : int32_t ExceptionDumper::RegisterExceptionDumpCallback(ExceptionDumpCallback callback)
619 : {
620 44 : if (callback == nullptr) {
621 7 : IDE_LOGE("Register callback is nullptr.");
622 7 : return ADUMP_INPUT_FAILED;
623 : }
624 :
625 37 : const std::lock_guard<std::recursive_mutex> lock(callbackMutex_);
626 77 : for (auto& cb : callbacks_) {
627 8 : if (cb == callback) {
628 5 : IDE_LOGW("Callback already registered.");
629 5 : return ADUMP_SUCCESS;
630 : }
631 : }
632 32 : callbacks_.push_back(callback);
633 32 : IDE_LOGI("Callback registered, total=%zu.", callbacks_.size());
634 32 : return ADUMP_SUCCESS;
635 37 : }
636 :
637 21 : int32_t ExceptionDumper::UnregisterExceptionDumpCallback(ExceptionDumpCallback callback)
638 : {
639 21 : if (callback == nullptr) {
640 7 : IDE_LOGE("Unregister callback is nullptr.");
641 7 : return ADUMP_INPUT_FAILED;
642 : }
643 :
644 14 : const std::lock_guard<std::recursive_mutex> lock(callbackMutex_);
645 28 : for (auto it = callbacks_.begin(); it != callbacks_.end(); ++it) {
646 10 : if (*it == callback) {
647 20 : callbacks_.erase(it);
648 10 : IDE_LOGI("Callback unregistered, total=%zu.", callbacks_.size());
649 10 : return ADUMP_SUCCESS;
650 : }
651 : }
652 4 : IDE_LOGW("Callback not found.");
653 4 : return ADUMP_SUCCESS;
654 14 : }
655 :
656 : } // namespace Adx
|
