Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 : #include <set>
11 : #include <cctype>
12 : #include <cstring>
13 : #include <algorithm>
14 : #include "log/adx_log.h"
15 : #include "path.h"
16 : #include "dump_manager.h"
17 : #include "dump_config_converter.h"
18 : #include "sys_utils.h"
19 : #include "dump_args.h"
20 : #include "dump_args_callback.h"
21 : #include "dump_file.h"
22 : #include "exception_info_common.h"
23 : #include "kernel_symbol_locator.h"
24 : #include "kernel_info_collector.h"
25 : #include "kernel_source_symbolizer.h"
26 : #include "exception_dumper.h"
27 :
28 : namespace Adx {
29 : namespace {
30 : constexpr char DEFAULT_DUMP_PATH[] = "./";
31 : constexpr char EXTRA_DUMP_PATH[] = "/extra-info/data-dump/";
32 : constexpr size_t MAX_DUMP_OP_NUM = (2048U * 2048U) / 20U;
33 :
34 12 : static bool IsSafeKernelDisplayName(const char *name)
35 : {
36 77 : for (size_t i = 0; i < MAX_KERNELNAME_LEN && name[i] != '\0'; ++i) {
37 69 : char c = name[i];
38 69 : if (c == '/' || c == '\\' || c < 0x20 || c == 0x7F) {
39 4 : return false;
40 : }
41 : }
42 16 : return std::string(name).find("..") == std::string::npos;
43 : }
44 :
45 7 : bool ValidateKernelName(const ExceptionDumpInfo &info)
46 : {
47 7 : if (strnlen(info.kernelName, MAX_KERNELNAME_LEN) == MAX_KERNELNAME_LEN ||
48 7 : strnlen(info.kernelDisplayName, MAX_KERNELNAME_LEN) == MAX_KERNELNAME_LEN) {
49 0 : return false;
50 : }
51 :
52 7 : if (!IsSafeKernelDisplayName(info.kernelName) || !IsSafeKernelDisplayName(info.kernelDisplayName)) {
53 4 : return false;
54 : }
55 3 : return true;
56 : }
57 :
58 9 : bool IsValidExceptionDumpMode(ExceptionDumpMode dumpMode)
59 : {
60 8 : return dumpMode == ExceptionDumpMode::DUMP_MODE_NONE ||
61 17 : dumpMode == ExceptionDumpMode::DUMP_MODE_OVERWRITE ||
62 9 : dumpMode == ExceptionDumpMode::DUMP_MODE_ADDITIONAL;
63 : }
64 : } // namespace
65 :
66 : uint64_t *g_dynamicChunk = nullptr;
67 : uint64_t *g_staticChunk = nullptr;
68 :
69 49 : ExceptionDumper::~ExceptionDumper()
70 : {
71 49 : if (g_dynamicChunk != nullptr) {
72 19 : delete[] g_dynamicChunk;
73 19 : g_dynamicChunk = nullptr;
74 : }
75 :
76 49 : if (g_staticChunk != nullptr) {
77 19 : delete[] g_staticChunk;
78 19 : g_staticChunk = nullptr;
79 : }
80 49 : KernelSymbolLocator::ClearCache();
81 49 : destructionFlag_ = true;
82 49 : }
83 :
84 58 : bool ExceptionDumper::InitArgsExceptionMemory() const
85 : {
86 58 : if (g_dynamicChunk == nullptr) {
87 7547161 : g_dynamicChunk = new (std::nothrow) uint64_t[DYNAMIC_RING_CHUNK_SIZE + DFX_MAX_TENSOR_NUM + RESERVE_SPACE]();
88 19 : if (g_dynamicChunk == nullptr) {
89 0 : return false;
90 : }
91 : }
92 :
93 58 : if (g_staticChunk == nullptr) {
94 2566425 : g_staticChunk = new (std::nothrow) uint64_t[STATIC_RING_CHUNK_SIZE + DFX_MAX_TENSOR_NUM + RESERVE_SPACE]();
95 19 : if (g_staticChunk == nullptr) {
96 0 : return false;
97 : }
98 : }
99 58 : return true;
100 : }
101 :
102 123 : bool ExceptionDumper::IsRepeatEnableException(DumpType type, const DumpConfig &dumpConfig)
103 : {
104 123 : if (type == DumpType::ARGS_EXCEPTION || type == DumpType:: EXCEPTION ||
105 : type == DumpType::AIC_ERR_DETAIL_DUMP) {
106 56 : std::string dumpStatus = dumpConfig.dumpStatus;
107 56 : std::transform(dumpStatus.begin(), dumpStatus.end(), dumpStatus.begin(), ::tolower);
108 56 : if (dumpStatus != "on") {
109 11 : return false;
110 : }
111 45 : return IsEnabledExceptionDump();
112 56 : }
113 67 : return false;
114 : }
115 :
116 126 : bool ExceptionDumper::IsEnabledExceptionDump() const
117 : {
118 126 : return coredumpStatus_ || exceptionStatus_ || argsExceptionStatus_;
119 : }
120 :
121 78 : int32_t ExceptionDumper::ExceptionDumperInit(DumpType dumpType, const DumpConfig &dumpConfig)
122 : {
123 78 : bool status = false;
124 78 : if (!setting_.InitDumpStatus(dumpConfig.dumpStatus, status)) {
125 1 : IDE_LOGE("The value of dumpStatus: %s is invalid.", dumpConfig.dumpStatus.c_str());
126 1 : return ADUMP_FAILED;
127 : }
128 :
129 : // status = false means turn off, flag = false means not start, print warning when not start but want to turn off
130 77 : if (dumpType == DumpType::EXCEPTION) {
131 13 : IDE_CTRL_VALUE_WARN(status || exceptionStatus_, return ADUMP_SUCCESS, "dump type %d not start.", dumpType);
132 12 : exceptionStatus_ = status;
133 64 : } else if (dumpType == DumpType::ARGS_EXCEPTION) {
134 58 : IDE_CTRL_VALUE_WARN(status || argsExceptionStatus_, return ADUMP_SUCCESS, "dump type %d not start.", dumpType);
135 54 : IDE_CTRL_VALUE_FAILED(InitArgsExceptionMemory(), return ADUMP_FAILED, "Init args exception memory failed.");
136 54 : if (status && !argsExceptionStatus_) {
137 48 : IDE_CTRL_VALUE_WARN(LoadTensorPluginLib() == ADUMP_SUCCESS, return ADUMP_FAILED,
138 : "Load tersor custom plugin failed.");
139 : }
140 54 : argsExceptionStatus_ = status;
141 : } else {
142 : // detail exception dump can not off
143 6 : if (status == false) {
144 : static bool havePrint = false;
145 2 : if (!havePrint) {
146 1 : IDE_RUN_LOGI("Can not turn off detail exception dump.");
147 1 : havePrint = true;
148 : }
149 2 : IDE_LOGW("Can not turn off detail exception dump.");
150 2 : return ADUMP_SUCCESS;
151 : }
152 4 : IDE_CTRL_VALUE_FAILED(InitArgsExceptionMemory(), return ADUMP_FAILED, "Init args exception memory failed.");
153 4 : coredumpStatus_ = true;
154 : }
155 :
156 70 : SetDumpPath(dumpConfig.dumpPath);
157 70 : return ADUMP_SUCCESS;
158 : }
159 :
160 62 : std::string ExceptionDumper::GetDumpSceneName() const
161 : {
162 62 : if (coredumpStatus_) {
163 3 : return ADUMP_DUMP_EXCEPTION_AIC_ERR_DETAIL;
164 59 : } else if (exceptionStatus_) {
165 1 : return ADUMP_DUMP_EXCEPTION_AIC_ERR_NORM;
166 58 : } else if (argsExceptionStatus_) {
167 58 : return ADUMP_DUMP_EXCEPTION_AIC_ERR_BRIEF;
168 : } else {
169 0 : return "unknown";
170 : }
171 : }
172 :
173 83 : int32_t ExceptionDumper::DumpException(const rtExceptionInfo &exception)
174 : {
175 83 : IDE_CTRL_VALUE_WARN(!destructionFlag_, return ADUMP_FAILED, "ExceptionDumper has been destructed.");
176 83 : IDE_CTRL_VALUE_WARN(ExceptionInfoCommon::IsSupportExceptionDump(exception),
177 : return ADUMP_FAILED, "Exception is not need to dump.");
178 69 : IDE_CTRL_VALUE_WARN(IsEnabledExceptionDump(), return ADUMP_FAILED,
179 : "Not enable exception dump.");
180 67 : std::string dumpPath = CreateDeviceDumpPath(exception.deviceid);
181 67 : if (dumpPath.empty()) {
182 5 : return ADUMP_FAILED;
183 : }
184 :
185 62 : const std::string dumpScene = GetDumpSceneName();
186 62 : const std::string exceptionType = ExceptionInfoCommon::GetExceptionTaskTypeName(exception);
187 62 : const std::string kernelName = ExceptionInfoCommon::GetExceptionKernelName(exception);
188 62 : IDE_LOGE("[Dump][Exception] Begin to dump exception. dumpScene=%s, deviceId=%u, streamId=%u, taskId=%u, "
189 : "exceptionType=%d(%s), kernelName=%s.", dumpScene.c_str(), exception.deviceid, exception.streamid,
190 : exception.taskid, static_cast<int32_t>(exception.expandInfo.type), exceptionType.c_str(),
191 : kernelName.c_str());
192 :
193 62 : if (coredumpStatus_) {
194 3 : return DumpDetailException(exception, dumpPath);
195 59 : } else if (exceptionStatus_) {
196 1 : return DumpNormalException(exception, dumpPath);
197 : } else {
198 58 : return DumpArgsException(exception, dumpPath);
199 : }
200 67 : }
201 :
202 70 : void ExceptionDumper::SetDumpPath(const std::string &dumpPath)
203 : {
204 87 : dumpPath_ = dumpPath.empty() ? DEFAULT_DUMP_PATH : dumpPath;
205 70 : IDE_LOGI("Update exception dump path: %s", dumpPath_.c_str());
206 70 : }
207 :
208 4 : void ExceptionDumper::AddDumpOperator(const OperatorInfo &opInfo)
209 : {
210 4 : OperatorInfoV2 dumpOperator = {};
211 4 : DumpManager::Instance().ConvertOperatorInfo(opInfo, dumpOperator);
212 4 : AddDumpOperatorV2(dumpOperator);
213 4 : }
214 :
215 118 : void ExceptionDumper::AddDumpOperatorV2(const OperatorInfoV2 &opInfo)
216 : {
217 118 : const std::lock_guard<std::mutex> lock(mutex_);
218 118 : if (opInfo.agingFlag) {
219 6 : agingOperators_.emplace_back(opInfo);
220 6 : if (agingOperators_.size() > MAX_DUMP_OP_NUM) {
221 0 : agingOperators_.pop_front();
222 : }
223 : } else {
224 112 : uint32_t maxStreamCount = 0;
225 112 : uint32_t maxTaskCount = 0;
226 112 : rtGetMaxStreamAndTask(0, &maxStreamCount, &maxTaskCount);
227 112 : auto &taskDeque = residentOperators_[opInfo.deviceId][opInfo.streamId];
228 112 : taskDeque.emplace_back(opInfo);
229 112 : if (taskDeque.size() > maxTaskCount) {
230 2 : taskDeque.pop_front();
231 : }
232 : }
233 118 : IDE_LOGI("Dump operator size, aging: %zu, resident: %zu", agingOperators_.size(), residentOperators_.size());
234 118 : }
235 :
236 9 : int32_t ExceptionDumper::DelDumpOperator(uint32_t deviceId, uint32_t streamId)
237 : {
238 9 : const std::lock_guard<std::mutex> lock(mutex_);
239 9 : auto it = residentOperators_.find(deviceId);
240 9 : if (it != residentOperators_.end()) {
241 8 : it->second.erase(streamId);
242 8 : if (it->second.empty()) {
243 8 : residentOperators_.erase(deviceId);
244 : }
245 : }
246 9 : return ADUMP_SUCCESS;
247 9 : }
248 :
249 97 : std::string ExceptionDumper::CreateDumpPath(Path &dumpPath) const
250 : {
251 117 : IDE_CTRL_VALUE_FAILED(dumpPath.CreateDirectory(true), return "", "Create path[%s] failed",
252 : dumpPath.GetCString());
253 95 : IDE_CTRL_VALUE_FAILED(dumpPath.RealPath(), return "", "Get RealPath of [%s] failed, strerr=%s",
254 : dumpPath.GetCString(), strerror(errno));
255 83 : return dumpPath.GetString();
256 : }
257 :
258 91 : std::string ExceptionDumper::CreateDeviceDumpPath(uint32_t deviceId) const
259 : {
260 : // Create device dump path when exception call-backed
261 91 : std::string dumpPath = dumpPath_.empty() ? DEFAULT_DUMP_PATH : dumpPath_;
262 91 : Path deviceDumpPath(dumpPath);
263 91 : deviceDumpPath.Append(EXTRA_DUMP_PATH).Append(std::to_string(deviceId));
264 182 : return CreateDumpPath(deviceDumpPath);
265 91 : }
266 :
267 24 : int32_t ExceptionDumper::GetExceptionDumpPath(std::string &path)
268 : {
269 24 : int32_t deviceId = 0;
270 24 : const rtError_t rtRet = rtGetDevice(&deviceId);
271 24 : IDE_CTRL_VALUE_FAILED(rtRet == RT_ERROR_NONE, return ADUMP_FAILED,
272 : "rtGetDevice failed when get the exception root dump path, ret=%d.", static_cast<int32_t>(rtRet));
273 :
274 24 : path = CreateDeviceDumpPath(static_cast<uint32_t>(deviceId));
275 24 : IDE_CTRL_VALUE_FAILED(!path.empty(), return ADUMP_FAILED, "exception dump root path is not ready.");
276 18 : return ADUMP_SUCCESS;
277 : }
278 :
279 15 : int32_t ExceptionDumper::SaveExceptionInfo(const std::string &fileName, const std::string &userTag,
280 : const std::vector<TensorInfo> &tensors)
281 : {
282 15 : std::string rootPath;
283 15 : IDE_CTRL_VALUE_FAILED(GetExceptionDumpPath(rootPath) == ADUMP_SUCCESS, return ADUMP_FAILED,
284 : "[SaveExceptionInfo] get exception dump root path failed.");
285 :
286 12 : std::string realFilePath;
287 12 : IDE_CTRL_VALUE_FAILED(Path::BuildFullPathUnderRoot(rootPath, fileName, realFilePath),
288 : return ADUMP_FAILED, "[SaveExceptionInfo] build full file path failed.");
289 :
290 12 : realFilePath += ".custom." + SysUtils::GetCurrentTimeWithMillisecond();
291 :
292 12 : int32_t devId = 0;
293 12 : const rtError_t rtRet = rtGetDevice(&devId);
294 12 : IDE_CTRL_VALUE_FAILED(rtRet == RT_ERROR_NONE, return ADUMP_FAILED,
295 : "[SaveExceptionInfo] rtGetDevice failed, ret=%d.", static_cast<int32_t>(rtRet));
296 :
297 12 : DumpFile dumpFile(static_cast<uint32_t>(devId), realFilePath);
298 12 : dumpFile.SetHeader(Path(realFilePath).GetFileName());
299 12 : if (!userTag.empty()) {
300 18 : dumpFile.SetOpAttr("user_tag", userTag);
301 : }
302 :
303 12 : std::vector<std::string> record;
304 12 : dumpFile.SetTensors(tensors, record);
305 :
306 12 : IDE_CTRL_VALUE_FAILED(dumpFile.Dump(record) == ADUMP_SUCCESS, return ADUMP_FAILED,
307 : "[SaveExceptionInfo] dump exception info to file failed, file: %s", realFilePath.c_str());
308 :
309 9 : (void)mmChmod(realFilePath.c_str(), M_IRUSR);
310 9 : IDE_LOGE("[Dump][Exception] dump custom exception to file, file: %s", realFilePath.c_str());
311 9 : return ADUMP_SUCCESS;
312 15 : }
313 :
314 6 : std::string ExceptionDumper::CreateExtraDumpPath()
315 : {
316 : // Create extra dump path for tensor custom dump data
317 6 : std::string dumpPath = dumpPath_.empty() ? DEFAULT_DUMP_PATH : dumpPath_;
318 6 : Path extraDumpPath(dumpPath);
319 6 : extraDumpPath.Append(EXTRA_DUMP_PATH);
320 6 : extraDumpPath_ = CreateDumpPath(extraDumpPath);
321 12 : return extraDumpPath_;
322 6 : }
323 :
324 7 : const char* ExceptionDumper::GetExtraDumpCPath() const
325 : {
326 7 : return extraDumpPath_.empty() ? nullptr : extraDumpPath_.c_str();
327 : }
328 :
329 0 : bool ExceptionDumper::FindExceptionOperator(const rtExceptionInfo &exception, DumpOperator &excOp)
330 : {
331 0 : OpIdentity excOpIdentiy(exception.deviceid, exception.taskid, exception.streamid);
332 0 : if (exception.expandInfo.type == RT_EXCEPTION_FFTS_PLUS) {
333 0 : uint32_t contextId = static_cast<uint32_t>(exception.expandInfo.u.fftsPlusInfo.contextId);
334 0 : uint32_t threadId = static_cast<uint32_t>(exception.expandInfo.u.fftsPlusInfo.threadId);
335 0 : IDE_LOGI("ffts+ op context id: %u, thread id: %u.", contextId, threadId);
336 0 : excOpIdentiy.contextId = contextId;
337 : }
338 :
339 0 : const std::lock_guard<std::mutex> lock(mutex_);
340 0 : IDE_LOGI("Dump op size, aging:%zu, resident:%zu, target: %s", agingOperators_.size(), residentOperators_.size(),
341 : excOpIdentiy.GetString().c_str());
342 0 : for (const auto &op : agingOperators_) {
343 0 : if (op.IsBelongTo(excOpIdentiy)) {
344 0 : IDE_LOGI("Find exception op in aging list: %s", excOpIdentiy.GetString().c_str());
345 0 : excOp = op;
346 0 : return true;
347 : }
348 : }
349 :
350 0 : if (residentOperators_.find(excOpIdentiy.deviceId) != residentOperators_.end()) {
351 0 : auto &streamMap = residentOperators_[excOpIdentiy.deviceId];
352 0 : if (streamMap.find(excOpIdentiy.streamId) != streamMap.end()) {
353 0 : auto &taskDeque = streamMap[excOpIdentiy.streamId];
354 0 : for (const auto &op : taskDeque) {
355 0 : if (op.IsBelongTo(excOpIdentiy)) {
356 0 : IDE_LOGI("Find exception op in resident list: %s", excOpIdentiy.GetString().c_str());
357 0 : excOp = op;
358 0 : return true;
359 : }
360 : }
361 : }
362 : }
363 :
364 0 : IDE_LOGW("Not find dump operator, target: %s", excOpIdentiy.GetString().c_str());
365 0 : return false;
366 0 : }
367 :
368 49 : void ExceptionDumper::DumpHostKernelBinBeforeSymbolize(const rtExceptionInfo &exception,
369 : const std::string &dumpPath) const
370 : {
371 49 : rtExceptionArgsInfo_t exceptionArgsInfo{};
372 49 : int32_t ret = ExceptionInfoCommon::GetExceptionInfo(exception, exceptionArgsInfo);
373 49 : IDE_CTRL_VALUE_WARN(ret == ADUMP_SUCCESS, return,
374 : "Get exception args info failed, skip early dump host kernel bin.");
375 :
376 49 : KernelInfoCollector collector;
377 49 : collector.LoadKernelInfo(exceptionArgsInfo);
378 49 : ret = collector.LoadKernelBinBuffer();
379 49 : IDE_CTRL_VALUE_WARN(ret == ADUMP_SUCCESS, return,
380 : "Load kernel bin buffer failed, skip early dump host kernel bin.");
381 :
382 : // 提前无条件同步落 _host.o（只依赖 kernel bin buffer，不依赖 ELF 符号解析），
383 : // 保证符号表损坏等场景下 _host.o 仍能落盘供事后分析；DumpHostKernelBin 幂等，
384 : // 后续 symbolize/慢搜索路径复用已落盘文件不会重复写。
385 49 : std::string hostOPath;
386 49 : (void)collector.DumpHostKernelBin(dumpPath, hostOPath);
387 49 : }
388 :
389 1 : int32_t ExceptionDumper::DumpNormalException(const rtExceptionInfo &exception, const std::string &dumpPath)
390 : {
391 1 : IDE_CTRL_VALUE_WARN(ExceptionInfoCommon::IsSupportDefaultExceptionDump(exception),
392 : return ADUMP_FAILED, "Exception is not support default dump.");
393 : // 先无条件提前落 _host.o，再符号化（symbolize 复用已落盘文件，不重复落盘）。
394 0 : DumpHostKernelBinBeforeSymbolize(exception, dumpPath);
395 0 : KernelSymbolLocator::DumpErrorSymbols(exception, dumpPath);
396 0 : DumpOperator excOp;
397 0 : bool find = FindExceptionOperator(exception, excOp);
398 0 : if (!find) {
399 0 : return ADUMP_SUCCESS;
400 : }
401 :
402 0 : rtExceptionArgsInfo_t exceptionArgsInfo{};
403 0 : if (ExceptionInfoCommon::GetExceptionInfo(exception, exceptionArgsInfo) != ADUMP_SUCCESS) {
404 0 : IDE_LOGW("Get exception args info failed.");
405 0 : return ADUMP_FAILED;
406 : }
407 0 : (void)excOp.RefreshAddrs(exceptionArgsInfo);
408 0 : (void)excOp.LogExceptionInfo(exceptionArgsInfo);
409 0 : (void)excOp.CopyOpKernelFile();
410 :
411 0 : const int32_t ret = excOp.DumpException(exception.deviceid, dumpPath);
412 0 : if (ret != ADUMP_SUCCESS) {
413 0 : return ADUMP_FAILED;
414 : }
415 0 : return ADUMP_SUCCESS;
416 0 : }
417 :
418 50 : int32_t ExceptionDumper::DumpArgsExceptionDefault(const rtExceptionInfo &exception, const std::string &dumpPath)
419 : {
420 50 : IDE_CTRL_VALUE_WARN(ExceptionInfoCommon::IsSupportDefaultExceptionDump(exception),
421 : return ADUMP_FAILED, "Exception is not support default dump.");
422 : // 先无条件提前落 _host.o，再符号化（symbolize 复用已落盘文件，不重复落盘）。
423 49 : DumpHostKernelBinBeforeSymbolize(exception, dumpPath);
424 49 : KernelSymbolLocator::DumpErrorSymbols(exception, dumpPath);
425 49 : DumpArgs args;
426 49 : if (args.LoadArgsExceptionInfo(exception) != ADUMP_SUCCESS) {
427 31 : return ADUMP_FAILED;
428 : }
429 :
430 18 : if (args.DumpArgsExceptionInfo(exception.deviceid, dumpPath) != ADUMP_SUCCESS) {
431 0 : return ADUMP_FAILED;
432 : }
433 18 : return ADUMP_SUCCESS;
434 49 : }
435 :
436 57 : int32_t ExceptionDumper::DumpArgsExceptionInner(const rtExceptionInfo &exception, const std::string &dumpPath)
437 : {
438 57 : ExceptionDumpMode dumpMode = ExceptionDumpMode::DUMP_MODE_NONE;
439 57 : std::vector<ExceptionDumpInfo> dumpInfos;
440 57 : int32_t invokeRet = InvokeCallbacks(exception, dumpInfos, dumpMode);
441 57 : if (invokeRet != ADUMP_SUCCESS) {
442 0 : return invokeRet;
443 : }
444 57 : if (dumpMode == ExceptionDumpMode::DUMP_MODE_OVERWRITE) {
445 5 : if (dumpInfos.empty()) {
446 3 : IDE_LOGE("OVERWRITE mode with empty callback data after validation, reject.");
447 3 : return ADUMP_FAILED;
448 : }
449 2 : IDE_LOGI("DumpMode of callbacks is OVERWRITE, skip default dump, dump callback data only.");
450 2 : DumpCallbackData(exception, dumpInfos, dumpPath);
451 2 : return ADUMP_SUCCESS;
452 : }
453 :
454 52 : if (dumpMode == ExceptionDumpMode::DUMP_MODE_ADDITIONAL) {
455 2 : if (!dumpInfos.empty()) {
456 1 : IDE_LOGI("DumpMode of callbacks is ADDITIONAL, dump callback data and default.");
457 1 : DumpCallbackData(exception, dumpInfos, dumpPath);
458 : }
459 : }
460 :
461 52 : return DumpArgsExceptionDefault(exception, dumpPath);
462 57 : }
463 :
464 57 : int32_t ExceptionDumper::InvokeCallbacks(const rtExceptionInfo &exception,
465 : std::vector<ExceptionDumpInfo> &dumpInfos,
466 : ExceptionDumpMode &dumpMode)
467 : {
468 57 : dumpMode = ExceptionDumpMode::DUMP_MODE_NONE;
469 57 : dumpInfos.clear();
470 :
471 57 : std::lock_guard<std::recursive_mutex> lock(callbackMutex_);
472 57 : if (callbacks_.empty()) {
473 48 : IDE_LOGI("No exception callbacks registered.");
474 48 : return ADUMP_SUCCESS;
475 : }
476 :
477 9 : ExceptionRegInfo exceptionRegInfo{0, nullptr};
478 9 : int32_t ret = ExceptionInfoCommon::GetExceptionRegInfo(exception, exceptionRegInfo);
479 9 : IDE_CTRL_VALUE_WARN(ret == ADUMP_SUCCESS, return ADUMP_FAILED,
480 : "GetExceptionRegInfo failed, ret=%d, coreNum=%u.", ret, exceptionRegInfo.coreNum);
481 :
482 9 : uint32_t maxDumpSize = exceptionRegInfo.coreNum == 0 ? 1 : exceptionRegInfo.coreNum;
483 9 : IDE_LOGI("Exception callbacks size=%zu, maxDumpSize=%u.", callbacks_.size(), maxDumpSize);
484 :
485 18 : for (auto &callback : callbacks_) {
486 9 : std::vector<ExceptionDumpInfo> cbDumpInfos(maxDumpSize);
487 :
488 9 : uint32_t cbDumpRealSize = 0;
489 9 : ExceptionDumpMode cbDumpMode = ExceptionDumpMode::DUMP_MODE_NONE;
490 :
491 9 : uint32_t ret = callback(reinterpret_cast<void*>(const_cast<rtExceptionInfo*>(&exception)), cbDumpInfos.data(),
492 : maxDumpSize, &cbDumpRealSize, &cbDumpMode);
493 9 : if (ret != ADUMP_SUCCESS) {
494 0 : IDE_LOGW("Callback execute failed. ret=%u", ret);
495 0 : continue;
496 : }
497 9 : if (!IsValidExceptionDumpMode(cbDumpMode)) {
498 1 : IDE_LOGW("Callback dumpMode invalid. dumpMode=%u", static_cast<uint32_t>(cbDumpMode));
499 1 : continue;
500 : }
501 8 : if (cbDumpRealSize == 0 || cbDumpRealSize > maxDumpSize) {
502 1 : IDE_LOGW("Callback dumpRealSize invalid. maxDumpSize=%u, dumpRealSize=%u", maxDumpSize, cbDumpRealSize);
503 1 : continue;
504 : }
505 :
506 7 : IDE_LOGI("Callback dumpRealSize=%u, dumpMode=%d.", cbDumpRealSize, cbDumpMode);
507 14 : for (uint32_t i = 0; i < cbDumpRealSize; ++i) {
508 7 : if (!ValidateKernelName(cbDumpInfos[i])) {
509 4 : IDE_LOGW("Invalid kernelName/kernelDisplayName in callback data, skip. index=%u", i);
510 4 : continue;
511 : }
512 3 : dumpInfos.push_back(cbDumpInfos[i]);
513 : }
514 :
515 : // 多回调聚合时取优先级最高者：ADDITIONAL > OVERWRITE > NONE
516 7 : if (static_cast<uint32_t>(cbDumpMode) > static_cast<uint32_t>(dumpMode)) {
517 7 : dumpMode = cbDumpMode;
518 : }
519 9 : }
520 :
521 9 : IDE_LOGI("Callback final dumpMode=%d, dumpInfo total size=%zu.", dumpMode, dumpInfos.size());
522 9 : return ADUMP_SUCCESS;
523 57 : }
524 :
525 1 : void ExceptionDumper::DumpCallbackData(const rtExceptionInfo &exception,
526 : const std::vector<ExceptionDumpInfo> &dumpInfos,
527 : const std::string &dumpPath)
528 : {
529 : // 收集每个回调 info(单核) 的定位结果，循环结束后按 (.o, 偏移) 聚类打印。
530 1 : std::vector<ErrorLocation> allLocations;
531 2 : for (const auto &info : dumpInfos) {
532 1 : IDE_LOGE("[Dump][Exception] Begin to dump callback exception. coreType=%u, coreId=%u, "
533 : "argAddr=%p, argSize=%u, binHandle=%p, extraTensorNum=%u, kernelName=%s.",
534 : info.coreType, info.coreId, info.argAddr, info.argSize, info.bin, info.extraTensorNum, info.kernelName);
535 1 : IDE_LOGE("[Dump][Exception] Callback exception. kernelDisplayName=%s.", info.kernelDisplayName);
536 1 : DumpArgsCallback argsCallback(exception, info, dumpPath);
537 1 : int32_t ret = argsCallback.DumpDfxArgs();
538 1 : if (ret != ADUMP_SUCCESS) {
539 0 : IDE_LOGW("Dump callback args failed.");
540 : }
541 :
542 1 : ret = argsCallback.DumpExtraTensors();
543 1 : if (ret != ADUMP_SUCCESS) {
544 0 : IDE_LOGW("Dump callback extra tensor failed.");
545 : }
546 :
547 1 : ret = argsCallback.Dump();
548 1 : if (ret != ADUMP_SUCCESS) {
549 0 : IDE_LOGW("Dump callback data to file failed.");
550 : }
551 :
552 : // 先落 _host.o(同步)，DumpKernelErrorSymbols 内定位偏移后对其批量 symbolize。
553 1 : ret = argsCallback.DumpKernelBin();
554 1 : if (ret != ADUMP_SUCCESS) {
555 0 : IDE_LOGW("Dump callback kernel bin failed.");
556 : }
557 :
558 1 : ErrorLocation location;
559 1 : ret = argsCallback.DumpKernelErrorSymbols(location);
560 1 : if (ret != ADUMP_SUCCESS) {
561 0 : IDE_LOGW("Dump callback kernel error symbols failed.");
562 : } else {
563 1 : allLocations.push_back(location);
564 : }
565 :
566 1 : IDE_LOGI("Dump callback data finished, kernelName=%s.", info.kernelName);
567 1 : }
568 : // 各 core 的源码解析已在 DumpKernelErrorSymbols → 批量 symbolize 内完成并回填，
569 : // 此处只在工具可用时统一打印聚类汇总（工具不可用时汇总无源码信息、无增量价值）。
570 1 : if (KernelSourceSymbolizer::IsAvailable()) {
571 0 : KernelSymbolLocator::PrintClassificationSummary(allLocations);
572 : }
573 1 : KernelSymbolLocator::ClearCache();
574 1 : IDE_LOGI("Dump all callback data success.");
575 1 : }
576 :
577 5 : void ExceptionDumper::ExceptionModeDowngrade()
578 : {
579 5 : coredumpEnableComplete_ = false;
580 5 : }
581 :
582 0 : void ExceptionDumper::Exit() const
583 : {
584 0 : IDE_LOGE("The process exits.");
585 0 : AdxLogFlush();
586 : #ifndef __ADUMP_LLT
587 : _exit(EXIT_FAILURE);
588 : #endif
589 0 : }
590 :
591 : #ifdef __ADUMP_LLT
592 645 : void ExceptionDumper::Reset()
593 : {
594 645 : const std::lock_guard<std::mutex> lock(mutex_);
595 645 : agingOperators_.clear();
596 645 : residentOperators_.clear();
597 645 : exceptionStatus_ = false;
598 645 : argsExceptionStatus_ = false;
599 645 : coredumpStatus_ = false;
600 645 : coredumpEnableComplete_ = true;
601 645 : }
602 : #endif
603 :
604 32 : int32_t ExceptionDumper::RegisterExceptionDumpCallback(ExceptionDumpCallback callback)
605 : {
606 32 : if (callback == nullptr) {
607 7 : IDE_LOGE("Register callback is nullptr.");
608 7 : return ADUMP_INPUT_FAILED;
609 : }
610 :
611 25 : const std::lock_guard<std::recursive_mutex> lock(callbackMutex_);
612 25 : for (auto &cb : callbacks_) {
613 4 : if (cb == callback) {
614 4 : IDE_LOGW("Callback already registered.");
615 4 : return ADUMP_SUCCESS;
616 : }
617 : }
618 21 : callbacks_.push_back(callback);
619 21 : IDE_LOGI("Callback registered, total=%zu.", callbacks_.size());
620 21 : return ADUMP_SUCCESS;
621 25 : }
622 :
623 21 : int32_t ExceptionDumper::UnregisterExceptionDumpCallback(ExceptionDumpCallback callback)
624 : {
625 21 : if (callback == nullptr) {
626 7 : IDE_LOGE("Unregister callback is nullptr.");
627 7 : return ADUMP_INPUT_FAILED;
628 : }
629 :
630 14 : const std::lock_guard<std::recursive_mutex> lock(callbackMutex_);
631 14 : for (auto it = callbacks_.begin(); it != callbacks_.end(); ++it) {
632 10 : if (*it == callback) {
633 10 : callbacks_.erase(it);
634 10 : IDE_LOGI("Callback unregistered, total=%zu.", callbacks_.size());
635 10 : return ADUMP_SUCCESS;
636 : }
637 : }
638 4 : IDE_LOGW("Callback not found.");
639 4 : return ADUMP_SUCCESS;
640 14 : }
641 :
642 : } // namespace Adx
|
