Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 : #include "dump_manager.h"
11 : #include <thread>
12 : #include <cctype>
13 : #include <cinttypes>
14 : #include <map>
15 : #include <algorithm>
16 : #include <cerrno>
17 : #include <sstream>
18 : #include "str_utils.h"
19 : #include "lib_path.h"
20 : #include "file_utils.h"
21 : #include "adump_platform_manager.h"
22 : #include "log/adx_log.h"
23 : #include "runtime/context.h"
24 : #include "runtime/config.h"
25 : #include "rts/rts_snapshot.h"
26 : #include "error_codes/rt_error_codes.h"
27 : #include "adump_dsmi.h"
28 : #include "common_utils.h"
29 : #include "exception_info_common.h"
30 : #include "dump_config_converter.h"
31 : #include "adump_api.h"
32 : #include "adump_error_manager.h"
33 : #include "operator_dumper.h"
34 : #include "kernel_dfx_dumper.h"
35 : #include "adx_dump_record.h"
36 : #include "common/file.h"
37 : #include "common/path.h"
38 : #include "sys_utils.h"
39 : #include "dump_file.h"
40 : #include "adx_datadump_server.h"
41 :
42 : namespace Adx {
43 : constexpr char EXCEPTION_CB_MODULE[] = "AdumpException";
44 : constexpr char COREDUMP_CB_MODULE[] = "AdumpCoredump";
45 :
46 : std::vector<std::shared_ptr<OperatorPreliminary>> DumpManager::operatorMap_;
47 :
48 0 : static void ExceptionCallback(rtExceptionInfo* const exception)
49 : {
50 0 : IDE_RUN_LOGI("An exception callback message is received.");
51 0 : if (exception != nullptr) {
52 0 : (void)DumpManager::Instance().DumpExceptionInfo(*exception);
53 0 : AdxLogFlush();
54 : }
55 0 : }
56 :
57 6 : static void NotifyCoredumpCallback(uint32_t devId, bool isOpen)
58 : {
59 6 : static std::map<uint32_t, uint32_t> setDeviceRecord;
60 : static std::mutex coredumpMtx;
61 6 : std::lock_guard<std::mutex> lk(coredumpMtx);
62 6 : auto it = setDeviceRecord.find(devId);
63 6 : if (!isOpen) {
64 2 : if (it != setDeviceRecord.end()) {
65 2 : it->second--;
66 2 : if (it->second == 0) {
67 1 : setDeviceRecord.erase(it);
68 1 : IDE_LOGI("Device %u has been removed from the effective list.", devId);
69 : }
70 : }
71 2 : return;
72 : }
73 4 : if (it != setDeviceRecord.end()) {
74 2 : it->second++;
75 2 : return;
76 : }
77 2 : rtError_t ret = rtDebugSetDumpMode(RT_DEBUG_DUMP_ON_EXCEPTION);
78 2 : if (ret != RT_ERROR_NONE) {
79 0 : IDE_RUN_LOGI("detail exception dump mode not support, switch to lite exception dump, ret:%d", ret);
80 0 : DumpManager::Instance().ExceptionModeDowngrade();
81 : }
82 2 : IDE_LOGI("Device %u has been added to the effective list.", devId);
83 2 : setDeviceRecord[devId] = 1;
84 6 : }
85 :
86 0 : static uint32_t DumpSnapShotLockPreCallback(int32_t deviceId, void* args)
87 : {
88 : UNUSED(deviceId);
89 : UNUSED(args);
90 0 : return DumpManager::Instance().StopDataDumpServer() ? 0U : 1U;
91 : }
92 :
93 2072 : DumpManager& DumpManager::Instance()
94 : {
95 2072 : static DumpManager instance;
96 2072 : return instance;
97 : }
98 :
99 3 : DumpManager::DumpManager()
100 : {
101 : try {
102 : // 1. 通过环境变量使能Exception Dump
103 3 : EnableExceptionDumpWithEnv();
104 : // 2. 通过环境变量使能Kernel Dfx Dump
105 3 : KernelDfxDumper::Instance();
106 0 : } catch (...) {
107 0 : IDE_LOGW("Enable dump function with env variable failed!");
108 0 : }
109 3 : }
110 :
111 258 : bool DumpManager::StartDataDumpServer()
112 : {
113 : // 注册快照回调：快照备份前关闭Data Dump Server
114 258 : RegisterSnapShotCallback();
115 : #if !defined(ADUMP_SOC_HOST) || ADUMP_SOC_HOST == 1
116 : // 如果没有启动Data Dump Server，启动Data Dump Server
117 258 : int32_t dumpNum = AdxDumpRecord::Instance().GetDumpInitNum();
118 258 : if (dumpNum == 0) {
119 39 : return AdxDataDumpServerInit() == ADUMP_SUCCESS;
120 : }
121 : #endif
122 219 : return true;
123 : }
124 :
125 39 : bool DumpManager::StopDataDumpServer()
126 : {
127 : #if !defined(ADUMP_SOC_HOST) || ADUMP_SOC_HOST == 1
128 39 : int32_t dumpNum = AdxDumpRecord::Instance().GetDumpInitNum();
129 78 : while (dumpNum > 0) {
130 39 : IDE_CTRL_VALUE_FAILED(
131 : AdxDataDumpServerUnInit() == ADUMP_SUCCESS, return false, "Stop data dump server failed!");
132 39 : dumpNum = AdxDumpRecord::Instance().GetDumpInitNum();
133 : }
134 : #endif
135 39 : return true;
136 : }
137 :
138 267 : void DumpManager::RegisterSnapShotCallback()
139 : {
140 267 : if (!snapCbkRegistered_) {
141 15 : rtError_t ret = rtSnapShotCallbackRegister(RT_SNAPSHOT_LOCK_PRE, DumpSnapShotLockPreCallback, nullptr);
142 15 : if (ret == ACL_ERROR_RT_FEATURE_NOT_SUPPORT) {
143 3 : IDE_LOGI("RTS does not support snapshot feature. ret=%d", ret);
144 3 : return;
145 : }
146 12 : IDE_CTRL_VALUE_WARN(
147 : ret == ACL_SUCCESS, return, "Register DumpSnapShotLockPreCallback to RTS failed. ret=%d", ret);
148 9 : snapCbkRegistered_ = true;
149 9 : IDE_LOGI("Register DumpSnapShotLockPreCallback success.");
150 : }
151 : }
152 :
153 3 : void DumpManager::EnableExceptionDumpWithEnv()
154 : {
155 3 : DumpConfig config;
156 : DumpType dumpType;
157 3 : if (DumpConfigConverter::EnableExceptionDumpWithEnv(config, dumpType)) {
158 0 : if (SetDumpConfig(dumpType, config) != ADUMP_SUCCESS) {
159 0 : IDE_LOGW("Enable exception dump failed. dumpType: %d", dumpType);
160 : } else {
161 0 : isEnvExceptionDump_ = true;
162 : }
163 : }
164 3 : }
165 :
166 258 : bool DumpManager::AdjustOpExecuteTimeOut()
167 : {
168 258 : if (opTimeoutModified_) {
169 195 : IDE_LOGI("Op execute timeout has been adjusted for data dump, skip.");
170 195 : return true;
171 : }
172 63 : uint32_t curTimeout = 0U;
173 63 : rtError_t ret = rtGetOpExecuteTimeoutV2(&curTimeout);
174 63 : IDE_CTRL_VALUE_FAILED(
175 : ret == RT_ERROR_NONE, return false, "Get op execute timeout failed, skip adjust for data dump. ret=%d", ret);
176 60 : if (curTimeout >= OP_EXECUTE_TIMEOUT_FOR_DATADUMP_MS) {
177 6 : IDE_LOGI(
178 : "Current op execute timeout %ums is no less than %ums, no need to adjust for data dump.", curTimeout,
179 : OP_EXECUTE_TIMEOUT_FOR_DATADUMP_MS);
180 6 : return true;
181 : }
182 54 : ret = rtSetOpExecuteTimeOutWithMs(OP_EXECUTE_TIMEOUT_FOR_DATADUMP_MS);
183 54 : IDE_CTRL_VALUE_FAILED(
184 : ret == RT_ERROR_NONE, return false, "Set op execute timeout to %ums failed for data dump. ret=%d",
185 : OP_EXECUTE_TIMEOUT_FOR_DATADUMP_MS, ret);
186 51 : originOpExecuteTimeOut_ = curTimeout;
187 51 : opTimeoutModified_ = true;
188 51 : IDE_LOGI(
189 : "Adjust op execute timeout from %ums to %ums for data dump.", curTimeout, OP_EXECUTE_TIMEOUT_FOR_DATADUMP_MS);
190 51 : return true;
191 : }
192 :
193 36 : bool DumpManager::RestoreOpExecuteTimeOut()
194 : {
195 36 : if (!opTimeoutModified_) {
196 9 : return true;
197 : }
198 27 : rtError_t ret = rtSetOpExecuteTimeOutWithMs(originOpExecuteTimeOut_);
199 : // 恢复失败时保持 opTimeoutModified_=true，以便下次关闭 data dump 时重试恢复
200 27 : IDE_CTRL_VALUE_FAILED(
201 : ret == RT_ERROR_NONE, return false, "Restore op execute timeout to %ums failed. ret=%d",
202 : originOpExecuteTimeOut_, ret);
203 24 : IDE_LOGI("Restore op execute timeout to %ums after data dump disabled.", originOpExecuteTimeOut_);
204 24 : opTimeoutModified_ = false;
205 24 : originOpExecuteTimeOut_ = 0U;
206 24 : return true;
207 : }
208 :
209 0 : void DumpManager::KFCResourceInit()
210 : {
211 : #if !defined(ADUMP_SOC_HOST) || ADUMP_SOC_HOST == 1
212 0 : if (isKFCInit_) {
213 0 : IDE_LOGD("KFC resources have been initialized on all devices.");
214 0 : return;
215 : }
216 0 : std::vector<uint32_t> devList = AdumpDsmi::DrvGetDeviceList();
217 :
218 0 : for (uint32_t& deviceId : devList) {
219 0 : IDE_LOGI("Start to initialize KFC resources on device %u.", deviceId);
220 0 : SharedPtr<OperatorPreliminary> opIniter = MakeSharedInstance<OperatorPreliminary>(GetDumpSetting(), deviceId);
221 0 : IDE_CTRL_VALUE_FAILED_NODO(
222 : opIniter != nullptr && opIniter->OperatorInit() == ADUMP_SUCCESS, return,
223 : "Failed to execute the resource initialization task on device %u.", deviceId);
224 0 : DumpManager::operatorMap_.emplace_back(std::move(opIniter));
225 0 : IDE_LOGI("KFC executed on the device %u successfully.", deviceId);
226 0 : }
227 : #endif
228 0 : isKFCInit_ = true;
229 0 : }
230 :
231 54 : int32_t DumpManager::ExceptionConfig(DumpType dumpType, const DumpConfig& dumpConfig)
232 : {
233 54 : if (exceptionDumper_.IsRepeatEnableException(dumpType, dumpConfig)) {
234 3 : IDE_LOGW(
235 : "Exception dump has been enabled, not support enable exception dump[%s] again",
236 : DumpConfigConverter::DumpTypeToStr(dumpType).c_str());
237 3 : return ADUMP_SUCCESS;
238 : }
239 :
240 51 : if (dumpType == DumpType::AIC_ERR_DETAIL_DUMP && !CheckCoredumpSupportedPlatform()) {
241 3 : IDE_LOGE("Current platform is not support coredump mode.");
242 3 : return ADUMP_FAILED;
243 : }
244 :
245 48 : IDE_RUN_LOGI(
246 : "Set %s[%d] dump setting, status: %s, dump switch: %llu", DumpConfigConverter::DumpTypeToStr(dumpType).c_str(),
247 : dumpType, dumpConfig.dumpStatus.c_str(), dumpConfig.dumpSwitch);
248 48 : if (exceptionDumper_.ExceptionDumperInit(dumpType, dumpConfig) != ADUMP_SUCCESS) {
249 1 : IDE_LOGW("Failed to initialize the exception dump.");
250 1 : return ADUMP_SUCCESS;
251 : }
252 47 : dumpSetting_.InitDumpSwitch(dumpConfig.dumpSwitch & DUMP_SWITCH_MASK);
253 47 : IDE_CTRL_VALUE_WARN(
254 : RegsiterExceptionCallback(), return ADUMP_SUCCESS,
255 : "Failed to register the args exception dump callback function.");
256 46 : if (exceptionDumper_.GetCoredumpStatus()) { // 如果启动了coredump模式
257 1 : IDE_CTRL_VALUE_FAILED(
258 : rtRegDeviceStateCallbackEx(COREDUMP_CB_MODULE, &NotifyCoredumpCallback, DEV_CB_POS_BACK) == RT_ERROR_NONE,
259 : return ADUMP_FAILED, "Failed to register the coredump callback function to rtSetDevice.");
260 : }
261 46 : return ADUMP_SUCCESS;
262 : }
263 :
264 399 : int32_t DumpManager::SetDumpConfig(DumpType dumpType, const DumpConfig& dumpConfig)
265 : {
266 399 : std::lock_guard<std::mutex> lk(resourceMtx_);
267 399 : if (dumpType == DumpType::EXCEPTION || dumpType == DumpType::ARGS_EXCEPTION ||
268 : dumpType == DumpType::AIC_ERR_DETAIL_DUMP) {
269 54 : return ExceptionConfig(dumpType, dumpConfig);
270 : }
271 345 : auto ret = dumpSetting_.Init(dumpType, dumpConfig);
272 345 : if (ret != ADUMP_SUCCESS) {
273 21 : return ret;
274 : }
275 :
276 : // 启动Data Dump Server
277 324 : if (dumpConfig.dumpStatus != ADUMP_DUMP_STATUS_SWITCH_OFF) {
278 258 : IDE_CTRL_VALUE_FAILED(
279 : StartDataDumpServer(), return ADUMP_FAILED, "Start data dump server failed! dumpType=%s[%d]",
280 : DumpConfigConverter::DumpTypeToStr(dumpType).c_str(), dumpType);
281 : // 后续失败不恢复超时时间，不影响功能。
282 258 : IDE_CTRL_VALUE_FAILED(AdjustOpExecuteTimeOut(), return ADUMP_FAILED, "Adjust op execute timeout failed.");
283 : }
284 :
285 318 : if (CheckBinValidation() && (isKFCInit_ == false)) {
286 0 : auto kfcBind = std::bind(&DumpManager::KFCResourceInit, this);
287 0 : std::thread kfcThread(kfcBind);
288 0 : kfcThread.join();
289 0 : if (!isKFCInit_) {
290 0 : IDE_LOGE("SetDumpConfig failed due to kfc resource initialization error.");
291 0 : DumpManager::operatorMap_.clear();
292 0 : return ADUMP_FAILED;
293 : }
294 0 : }
295 :
296 318 : ret = OperatorDumper(dumpSetting_).UpdateDevMemCache();
297 318 : IDE_CTRL_VALUE_FAILED(
298 : ret == ADUMP_SUCCESS, return ADUMP_FAILED, "Update device memory cache for data dump failed! dumpType=%s[%d]",
299 : DumpConfigConverter::DumpTypeToStr(dumpType).c_str(), dumpType);
300 :
301 318 : IDE_RUN_LOGI(
302 : "Set %s[%d] dump setting, status: %s, mode: %s, data: %s, dump switch: %llu, path:%s, dump stats:%s.",
303 : DumpConfigConverter::DumpTypeToStr(dumpType).c_str(), dumpType, dumpConfig.dumpStatus.c_str(),
304 : dumpConfig.dumpMode.c_str(), dumpConfig.dumpData.c_str(), dumpConfig.dumpSwitch, dumpConfig.dumpPath.c_str(),
305 : StrUtils::ToString(dumpConfig.dumpStatsItem).c_str());
306 318 : return ADUMP_SUCCESS;
307 399 : }
308 :
309 111 : int32_t DumpManager::SetDumpConfig(const char* dumpConfigData, size_t dumpConfigSize, const char* dumpConfigPath)
310 : {
311 111 : std::lock_guard<std::mutex> lk(resourceMtx2_);
312 111 : if ((dumpConfigData == nullptr) || (dumpConfigSize == 0U) || (dumpConfigPath == nullptr)) {
313 24 : IDE_LOGE("Set dump config failed. Config data is null or empty.");
314 24 : return ADUMP_FAILED;
315 : }
316 87 : DumpConfig dumpConfig;
317 87 : DumpDfxConfig dumpDfxConfig;
318 : DumpType dumpType;
319 87 : bool needDump = true;
320 87 : DumpConfigConverter converter{dumpConfigData, dumpConfigSize, dumpConfigPath};
321 87 : int32_t ret = converter.Convert(dumpType, dumpConfig, needDump, dumpDfxConfig);
322 87 : if (ret != ADUMP_SUCCESS) {
323 12 : IDE_LOGE("Parse dump config from memory[%s] failed.", dumpConfigData);
324 12 : return ADUMP_INPUT_FAILED;
325 : }
326 :
327 : // 开启KernelDataDump
328 75 : ret = KernelDfxDumper::Instance().EnableDfxDumper(dumpDfxConfig);
329 75 : IDE_CTRL_VALUE_FAILED(ret == ADUMP_SUCCESS, return ret, "Enable kernel dfx dump failed.");
330 :
331 75 : if (!needDump) {
332 9 : return ADUMP_SUCCESS;
333 : }
334 :
335 : // 已开启exception dump：不支持重复使能，不更新配置缓存，不重复回调注册模块组件
336 66 : if (exceptionDumper_.IsRepeatEnableException(dumpType, dumpConfig)) {
337 0 : IDE_LOGW(
338 : "Exception dump has been enabled, not support enable exception dump[%s] again",
339 : DumpConfigConverter::DumpTypeToStr(dumpType).c_str());
340 0 : return ADUMP_SUCCESS;
341 : }
342 :
343 66 : (void)dumpConfigInfo_.assign(dumpConfigData, dumpConfigSize);
344 66 : IDE_LOGI("Dump config info set: addr=%p, size=%zu", dumpConfigInfo_.data(), dumpConfigInfo_.size());
345 66 : ret = SetDumpConfig(dumpType, dumpConfig);
346 : // 同步触发callback事件
347 93 : for (auto& item : enableCallbackFunc_) {
348 27 : IDE_LOGI("SetDumpConfig HandleDumpEvent start for module [%zu]", item.first);
349 27 : HandleDumpEvent(item.first, DumpEnableAction::ENABLE);
350 : }
351 66 : IDE_CTRL_VALUE_FAILED(ret == ADUMP_SUCCESS, return ret, "Set dump config failed.");
352 60 : (void)openedDump_.insert(dumpType);
353 60 : return ADUMP_SUCCESS;
354 111 : }
355 :
356 42 : int32_t DumpManager::UnSetDumpConfig()
357 : {
358 42 : std::lock_guard<std::mutex> lk(resourceMtx2_);
359 42 : DumpConfig config;
360 42 : config.dumpStatus = ADUMP_DUMP_STATUS_SWITCH_OFF;
361 42 : config.dumpSwitch = 0;
362 66 : for (const auto dumpType : openedDump_) {
363 30 : if (IsEnableDump(dumpType)) {
364 30 : const auto ret = SetDumpConfig(dumpType, config);
365 30 : IDE_CTRL_VALUE_FAILED(
366 : ret == ADUMP_SUCCESS, return ADUMP_FAILED, "[Set][Dump]Set dump off failed! dumpType=%s[%d], ret=%d",
367 : DumpConfigConverter::DumpTypeToStr(dumpType).c_str(), dumpType, ret);
368 24 : IDE_LOGI(
369 : "[Set][Dump]Set dump off successfully, dumpType=%s[%d]",
370 : DumpConfigConverter::DumpTypeToStr(dumpType).c_str(), dumpType);
371 : }
372 : }
373 36 : openedDump_.clear();
374 : // 同步触发callback事件
375 81 : for (auto& item : disableCallbackFunc_) {
376 45 : IDE_LOGI("UnSetDumpConfig start for module [%zu]", item.first);
377 45 : HandleDumpEvent(item.first, DumpEnableAction::DISABLE);
378 : }
379 36 : dumpConfigInfo_.clear();
380 36 : IDE_LOGI("Dump config info cleared.");
381 :
382 : // 等待所有 dump 操作完成并清理资源
383 36 : DumpResourceSafeMap::Instance().waitAndClear();
384 : // 释放device资源
385 36 : OperatorDumper::FreeDevMemCache();
386 : // 停止data dump server
387 36 : IDE_CTRL_VALUE_FAILED(StopDataDumpServer(), return ADUMP_FAILED, "Stop data dump server failed!");
388 : // 恢复算子超时时间
389 36 : IDE_CTRL_VALUE_FAILED(RestoreOpExecuteTimeOut(), return ADUMP_FAILED, "Restore op execute timeout failed!");
390 :
391 33 : return ADUMP_SUCCESS;
392 42 : }
393 :
394 321 : std::vector<std::string> DumpManager::GetBinNames() const
395 : {
396 321 : auto plat = PlatformReflection<DataDumpInterface>::CreatePlatform(dumpSetting_.GetPlatformType());
397 321 : if (plat == nullptr) {
398 321 : return {};
399 : }
400 0 : return plat->GetKfcBinNames();
401 321 : }
402 :
403 321 : bool DumpManager::CheckBinValidation()
404 : {
405 321 : const std::vector<std::string> opNames = GetBinNames();
406 321 : if ((dumpSetting_.GetDumpData().compare(DUMP_STATS_DATA) != 0) || opNames.empty()) {
407 321 : IDE_LOGI("CheckBinValidation result is false");
408 321 : return false;
409 : }
410 0 : for (const auto& opName : opNames) {
411 0 : const std::string opPath = LibPath::Instance().GetTargetPath(opName);
412 0 : IDE_CTRL_VALUE_FAILED(!opPath.empty(), continue, "Received an empty path for file %s.", opName.c_str());
413 0 : if (FileUtils::IsFileExist(opPath)) {
414 0 : IDE_LOGI("CheckBinValidation result is true with file %s", opName.c_str());
415 0 : return true;
416 : }
417 0 : }
418 0 : IDE_LOGI("CheckBinValidation result is false");
419 0 : return false;
420 321 : }
421 :
422 161 : bool DumpManager::IsEnableDump(DumpType dumpType)
423 : {
424 161 : std::lock_guard<std::mutex> lk(resourceMtx_);
425 161 : if (dumpType == DumpType::ARGS_EXCEPTION) {
426 38 : return exceptionDumper_.GetArgsExceptionStatus() || exceptionDumper_.GetCoredumpStatus();
427 123 : } else if (dumpType == DumpType::OPERATOR) {
428 93 : return dumpSetting_.GetDumpStatus();
429 30 : } else if (dumpType == DumpType::OP_OVERFLOW) {
430 21 : return dumpSetting_.GetDumpDebugStatus();
431 9 : } else if (dumpType == DumpType::EXCEPTION) {
432 3 : return exceptionDumper_.GetExceptionStatus();
433 6 : } else if (dumpType == DumpType::AIC_ERR_DETAIL_DUMP) {
434 3 : return exceptionDumper_.GetCoredumpStatus();
435 : } else {
436 3 : IDE_LOGW("Dump type is not support.");
437 : }
438 :
439 3 : return false;
440 161 : }
441 :
442 39 : int32_t DumpManager::GetInputOutputTensors(
443 : const std::string& opType, const std::string& opName, const std::vector<TensorInfoV2>& tensors,
444 : std::vector<DumpTensor>& inputTensors, std::vector<DumpTensor>& outputTensors)
445 : {
446 114 : for (const auto& tensorInfo : tensors) {
447 36 : if (tensorInfo.tensorAddr == nullptr || tensorInfo.tensorSize == 0) {
448 6 : IDE_LOGW(
449 : "Tensor of op=%s[%s] is empty, addr=%p, size=%zu, skip it.", opName.c_str(), opType.c_str(),
450 : tensorInfo.tensorAddr, tensorInfo.tensorSize);
451 6 : continue;
452 : }
453 :
454 30 : if (tensorInfo.placement != TensorPlacement::kOnDeviceHbm) {
455 6 : IDE_LOGW("Tensor of op=%s[%s] is not on device, skip it.", opName.c_str(), opType.c_str());
456 6 : continue;
457 : }
458 :
459 24 : if (tensorInfo.type == TensorType::INPUT) {
460 21 : inputTensors.emplace_back(tensorInfo);
461 3 : } else if (tensorInfo.type == TensorType::OUTPUT) {
462 3 : outputTensors.emplace_back(tensorInfo);
463 : }
464 : }
465 39 : return ADUMP_SUCCESS;
466 : }
467 :
468 24 : bool DumpManager::IsEnableDumpOperatorWithCapture(
469 : const std::string& opType, const std::string& opName, aclrtStream stream)
470 : {
471 24 : rtStreamCaptureStatus status = RT_STREAM_CAPTURE_STATUS_MAX;
472 24 : rtModel_t* captureMdl = nullptr;
473 24 : int32_t ret = rtStreamGetCaptureInfo(stream, &status, captureMdl);
474 24 : if (ret != ACL_SUCCESS) {
475 6 : IDE_LOGW("Get stream capture info error: %d, will switch to common dump", ret);
476 6 : return false;
477 : }
478 18 : IDE_LOGI("%s[%s] : stream capture status: %d", opName.c_str(), opType.c_str(), status);
479 18 : return status == RT_STREAM_CAPTURE_STATUS_ACTIVE;
480 : }
481 :
482 27 : int32_t DumpManager::DumpOperatorWithCfg(
483 : const std::string& opType, const std::string& opName, const std::vector<TensorInfo>& tensors, aclrtStream stream,
484 : const DumpCfg& dumpCfg)
485 : {
486 27 : std::lock_guard<std::mutex> lk(resourceMtx_);
487 27 : if (!dumpSetting_.GetDumpStatusEx() && !dumpSetting_.GetDumpDebugStatus()) {
488 3 : IDE_LOGW("Operator or overflow dump is not enable, can't dump data.");
489 3 : return ADUMP_SUCCESS;
490 : }
491 :
492 24 : bool isInvalid = dumpCfg.numAttrs != 0UL && dumpCfg.attrs == nullptr;
493 24 : IDE_CTRL_VALUE_FAILED(
494 : !isInvalid, return ADUMP_FAILED, "The dump cfg attrs is null pointer! op=%s[%s].", opName.c_str(),
495 : opType.c_str());
496 :
497 21 : std::vector<DumpTensor> inputTensors;
498 21 : std::vector<DumpTensor> outputTensors;
499 : int32_t ret =
500 21 : GetInputOutputTensors(opType, opName, ConvertTensorInfoToDumpTensorV2(tensors), inputTensors, outputTensors);
501 21 : IDE_CTRL_VALUE_FAILED(
502 : ret == ADUMP_SUCCESS, return ADUMP_FAILED, "Get input and output tensors failed! op=%s[%s].", opName.c_str(),
503 : opType.c_str());
504 21 : IDE_CTRL_VALUE_WARN(
505 : !inputTensors.empty() || !outputTensors.empty(), return ADUMP_SUCCESS, "No tensor need to dump. op=%s[%s].",
506 : opName.c_str(), opType.c_str());
507 :
508 6 : if (IsEnableDumpOperatorWithCapture(opType, opName, stream)) {
509 3 : if (dumpSetting_.GetDumpDebugStatus() || dumpSetting_.IsDumpDataStats()) {
510 0 : IDE_LOGI("overflow or stats is not allow in capture stream");
511 0 : return ADUMP_SUCCESS;
512 : }
513 3 : return DumpOperatorWithCapture(opType, opName, inputTensors, outputTensors, stream);
514 : }
515 :
516 3 : OperatorDumper opDumper(opType, opName);
517 3 : ret = opDumper.SetDumpSetting(dumpSetting_)
518 3 : .RuntimeStream(stream)
519 3 : .InputDumpTensor(inputTensors)
520 3 : .OutputDumpTensor(outputTensors)
521 3 : .LaunchWithCfg(dumpCfg);
522 3 : IDE_CTRL_VALUE_FAILED(
523 : ret == ADUMP_SUCCESS, return ret, "Launch dump operator with dump cfg failed! op=%s[%s].", opName.c_str(),
524 : opType.c_str());
525 0 : return ADUMP_SUCCESS;
526 27 : }
527 :
528 6 : int32_t DumpManager::DumpOperator(
529 : const std::string& opType, const std::string& opName, const std::vector<TensorInfo>& tensors, aclrtStream stream)
530 : {
531 6 : return DumpOperatorV2(opType, opName, ConvertTensorInfoToDumpTensorV2(tensors), stream);
532 : }
533 :
534 24 : int32_t DumpManager::DumpOperatorV2(
535 : const std::string& opType, const std::string& opName, const std::vector<TensorInfoV2>& tensors, aclrtStream stream)
536 : {
537 24 : std::lock_guard<std::mutex> lk(resourceMtx_);
538 24 : if (!dumpSetting_.GetDumpStatus() && !dumpSetting_.GetDumpDebugStatus()) {
539 6 : IDE_LOGW("Operator or overflow dump is not enable, can't dump.");
540 6 : return ADUMP_SUCCESS;
541 : }
542 :
543 18 : std::vector<DumpTensor> inputTensors;
544 18 : std::vector<DumpTensor> outputTensors;
545 18 : int32_t ret = GetInputOutputTensors(opType, opName, tensors, inputTensors, outputTensors);
546 18 : IDE_CTRL_VALUE_FAILED(
547 : ret == ADUMP_SUCCESS, return ADUMP_FAILED, "Get input and output tensors failed! opName: %s, opType: %s",
548 : opName.c_str(), opType.c_str());
549 :
550 18 : if (IsEnableDumpOperatorWithCapture(opType, opName, stream)) {
551 12 : if (dumpSetting_.GetDumpDebugStatus() || dumpSetting_.IsDumpDataStats()) {
552 9 : IDE_LOGI("overflow or stats is not allow in capture stream");
553 9 : return ADUMP_SUCCESS;
554 : }
555 3 : return DumpOperatorWithCapture(opType, opName, inputTensors, outputTensors, stream);
556 : }
557 :
558 6 : OperatorDumper opDumper(opType, opName);
559 6 : ret = opDumper.SetDumpSetting(dumpSetting_)
560 6 : .RuntimeStream(stream)
561 6 : .InputDumpTensor(inputTensors)
562 6 : .OutputDumpTensor(outputTensors)
563 6 : .Launch();
564 6 : IDE_CTRL_VALUE_FAILED(
565 : ret == ADUMP_SUCCESS, return ret, "Launch dump operator failed! op=%s[%s].", opName.c_str(), opType.c_str());
566 3 : return ADUMP_SUCCESS;
567 24 : }
568 :
569 9 : int32_t DumpManager::DumpOperatorWithCapture(
570 : const std::string& opType, const std::string& opName, const std::vector<DumpTensor>& inputTensors,
571 : const std::vector<DumpTensor>& outputTensors, aclrtStream mainStream)
572 : {
573 9 : if (mainStream == nullptr) {
574 9 : IDE_LOGE("mainStream is nullptr.");
575 9 : return ADUMP_FAILED;
576 : }
577 :
578 0 : if (!isCaptureDumpServerInit_) {
579 0 : IDE_CTRL_VALUE_FAILED(StartDataDumpServer(), return ADUMP_FAILED, "Start data dump server failed!");
580 0 : isCaptureDumpServerInit_ = true;
581 : }
582 :
583 0 : uint32_t streamId = 0;
584 0 : uint32_t taskId = 0;
585 0 : uint32_t deviceId = 0;
586 0 : std::string dumpPath;
587 0 : int32_t ret = CollectStreamContextInfo(mainStream, opName, opType, streamId, taskId, deviceId, dumpPath);
588 0 : if (ret != ADUMP_SUCCESS) {
589 0 : IDE_LOGE("%s(%s) collect stream context info failed.", opName.c_str(), opType.c_str());
590 0 : return ret;
591 : }
592 :
593 0 : std::string mainStreamKey = std::to_string(streamId) + "_" + std::to_string(taskId);
594 0 : DumpInfoParams params = {
595 0 : mainStreamKey, inputTensors, outputTensors, opType, opName, streamId, taskId, deviceId, 0, 0, dumpPath};
596 0 : ret = GetDumpInfoFromMap(params);
597 0 : std::shared_ptr<DumpStreamInfo> dumpInfoPtr = DumpResourceSafeMap::Instance().get(mainStreamKey);
598 0 : if (ret != ADUMP_SUCCESS || dumpInfoPtr == nullptr) {
599 0 : IDE_LOGE("%s(%s) get dump info failed.", opName.c_str(), opType.c_str());
600 0 : return ADUMP_FAILED;
601 : }
602 0 : IDE_LOGI("%s(%s) dump data : create DumpStreamInfo success", opName.c_str(), opType.c_str());
603 :
604 0 : ret = SetupAsyncDump(dumpInfoPtr, opName, opType, mainStream);
605 0 : if (ret != ADUMP_SUCCESS) {
606 0 : return ret;
607 : }
608 0 : IDE_LOGI(
609 : "%s(%s) set main stream %u, dump stream %u, callback function success", opName.c_str(), opType.c_str(),
610 : dumpInfoPtr->streamId, dumpInfoPtr->dumpStmId);
611 :
612 0 : return ADUMP_SUCCESS;
613 0 : }
614 :
615 4 : void DumpManager::AddExceptionOp(const OperatorInfo& opInfo) { exceptionDumper_.AddDumpOperator(opInfo); }
616 :
617 4 : void DumpManager::AddExceptionOpV2(const OperatorInfoV2& opInfo) { exceptionDumper_.AddDumpOperatorV2(opInfo); }
618 :
619 4 : void DumpManager::ConvertOperatorInfo(const OperatorInfo& opInfo, OperatorInfoV2& operatorInfoV2) const
620 : {
621 4 : operatorInfoV2.agingFlag = opInfo.agingFlag;
622 4 : operatorInfoV2.taskId = opInfo.taskId;
623 4 : operatorInfoV2.streamId = opInfo.streamId;
624 4 : operatorInfoV2.deviceId = opInfo.deviceId;
625 4 : operatorInfoV2.contextId = opInfo.contextId;
626 4 : operatorInfoV2.opType = opInfo.opType;
627 4 : operatorInfoV2.opName = opInfo.opName;
628 4 : operatorInfoV2.tensorInfos = ConvertTensorInfoToDumpTensorV2(opInfo.tensorInfos);
629 4 : operatorInfoV2.deviceInfos = opInfo.deviceInfos;
630 4 : operatorInfoV2.additionalInfo = opInfo.additionalInfo;
631 4 : }
632 :
633 31 : std::vector<TensorInfoV2> DumpManager::ConvertTensorInfoToDumpTensorV2(const std::vector<TensorInfo>& tensorInfos) const
634 : {
635 31 : std::vector<TensorInfoV2> tensors;
636 31 : tensors.reserve(tensorInfos.size());
637 86 : for (const auto& tensorInfo : tensorInfos) {
638 24 : TensorInfoV2 tensor = {};
639 24 : ConvertTensorInfo(tensorInfo, tensor);
640 24 : tensors.emplace_back(tensor);
641 24 : }
642 31 : return tensors;
643 0 : }
644 :
645 96 : void DumpManager::ConvertTensorInfo(const TensorInfo& tensorInfo, TensorInfoV2& tensor) const
646 : {
647 96 : tensor.dataType = tensorInfo.dataType;
648 96 : tensor.format = tensorInfo.format;
649 96 : tensor.placement = tensorInfo.placement;
650 96 : tensor.tensorAddr = tensorInfo.tensorAddr;
651 96 : tensor.tensorSize = tensorInfo.tensorSize;
652 96 : tensor.type = tensorInfo.type;
653 96 : tensor.addrType = tensorInfo.addrType;
654 96 : tensor.argsOffSet = tensorInfo.argsOffSet;
655 96 : std::vector<int64_t> shape = tensorInfo.shape;
656 267 : for (auto dim : shape) {
657 75 : tensor.shape.emplace_back(static_cast<uint64_t>(dim));
658 : }
659 96 : std::vector<int64_t> originShape = tensorInfo.originShape;
660 267 : for (auto dim : originShape) {
661 75 : tensor.originShape.emplace_back(static_cast<uint64_t>(dim));
662 : }
663 96 : }
664 :
665 6 : int32_t DumpManager::DelExceptionOp(uint32_t deviceId, uint32_t streamId)
666 : {
667 6 : return exceptionDumper_.DelDumpOperator(deviceId, streamId);
668 : }
669 :
670 52 : int32_t DumpManager::DumpExceptionInfo(const rtExceptionInfo& exception)
671 : {
672 52 : return exceptionDumper_.DumpException(exception);
673 : }
674 :
675 17 : uint64_t DumpManager::AdumpGetDumpSwitch()
676 : {
677 17 : std::lock_guard<std::mutex> lk(resourceMtx_);
678 34 : return dumpSetting_.GetDumpSwitch();
679 17 : }
680 :
681 47 : bool DumpManager::RegsiterExceptionCallback()
682 : {
683 47 : if (!registered_ && rtRegTaskFailCallbackByModule(EXCEPTION_CB_MODULE, ExceptionCallback) == RT_ERROR_NONE) {
684 37 : registered_ = true;
685 : }
686 47 : IDE_LOGI("Register exception callback, registered: %d", static_cast<int32_t>(registered_));
687 47 : return registered_;
688 : }
689 :
690 24 : DumpSetting DumpManager::GetDumpSetting() const { return dumpSetting_; }
691 :
692 3 : void DumpManager::ExceptionModeDowngrade() { exceptionDumper_.ExceptionModeDowngrade(); }
693 :
694 60 : bool DumpManager::IsEnabledExceptionDump()
695 : {
696 60 : std::lock_guard<std::mutex> lk(resourceMtx_);
697 120 : return exceptionDumper_.IsEnabledExceptionDump();
698 60 : }
699 :
700 30 : int32_t DumpManager::RegisterCallback(uint32_t moduleId, AdumpCallback enableFunc, AdumpCallback disableFunc)
701 : {
702 30 : if (enableFunc == nullptr) {
703 6 : IDE_LOGE("Register callback failed: enableFunc is null for module %u", moduleId);
704 6 : return ADUMP_FAILED;
705 : }
706 24 : if (disableFunc == nullptr) {
707 6 : IDE_LOGE("Register callback failed: disableFunc is null for module %u", moduleId);
708 6 : return ADUMP_FAILED;
709 : }
710 18 : std::lock_guard<std::mutex> lk(resourceMtx2_);
711 18 : enableCallbackFunc_[moduleId] = enableFunc;
712 18 : disableCallbackFunc_[moduleId] = disableFunc;
713 18 : IDE_LOGI("Registered callback for module %u", moduleId);
714 18 : return HandleDumpEvent(moduleId, DumpEnableAction::AUTO);
715 18 : }
716 :
717 33 : int32_t DumpManager::StartDumpArgs(const std::string& dumpPath)
718 : {
719 33 : uint64_t dumpSwitch = 0;
720 : {
721 33 : std::lock_guard<std::mutex> lk(resourceMtx_);
722 33 : dumpSwitch = dumpSetting_.GetDumpSwitch();
723 33 : if ((dumpSwitch & OP_INFO_RECORD_DUMP) == OP_INFO_RECORD_DUMP) {
724 135 : REPORT_EP0008_API_CALL_SEQUENCE(FUNC_NAME_ACL_OP_START_DUMP_ARGS, ADUMP_REASON_API_CALLED_REPEATEDLY);
725 9 : return -1;
726 : }
727 :
728 24 : Adx::Path path(dumpPath);
729 24 : if (path.Empty()) {
730 69 : REPORT_EP0006_INVALID_ARGUMENT(
731 : FUNC_NAME_ACL_OP_START_DUMP_ARGS, dumpPath, FUNC_ACL_OP_START_DUMP_ARGS_PARAM_PATH,
732 : ADUMP_REASON_PARAM_PATH_EMPTY);
733 3 : return -1;
734 : }
735 21 : if (!path.Exist()) {
736 18 : if (!path.CreateDirectory(true)) {
737 3 : std::string reason = StrUtils::Format(ADUMP_REASON_PARAM_PATH_CREATE_DIR_ERROR, strerror(errno));
738 66 : REPORT_EP0006_INVALID_ARGUMENT(
739 : FUNC_NAME_ACL_OP_START_DUMP_ARGS, dumpPath, FUNC_ACL_OP_START_DUMP_ARGS_PARAM_PATH, reason);
740 3 : return -1;
741 3 : }
742 : }
743 18 : if (!path.IsDirectory()) {
744 69 : REPORT_EP0006_INVALID_ARGUMENT(
745 : FUNC_NAME_ACL_OP_START_DUMP_ARGS, dumpPath, FUNC_ACL_OP_START_DUMP_ARGS_PARAM_PATH,
746 : ADUMP_REASON_PARAM_PATH_NOT_DIRECTORY);
747 3 : return -1;
748 : }
749 15 : constexpr uint32_t accessMode = static_cast<uint32_t>(M_R_OK) | static_cast<uint32_t>(M_W_OK);
750 15 : if (!path.Asccess(accessMode)) {
751 69 : REPORT_EP0006_INVALID_ARGUMENT(
752 : FUNC_NAME_ACL_OP_START_DUMP_ARGS, dumpPath, FUNC_ACL_OP_START_DUMP_ARGS_PARAM_PATH,
753 : ADUMP_REASON_PATH_NO_PERMISSION);
754 3 : return -1;
755 : }
756 :
757 12 : dumpSwitch |= OP_INFO_RECORD_DUMP;
758 12 : dumpSetting_.InitDumpSwitch(dumpSwitch);
759 12 : opInfoRecordPath_ = path.GetString();
760 45 : }
761 33 : for (auto& item : enableCallbackFunc_) {
762 21 : item.second(dumpSwitch, dumpConfigInfo_.data(), dumpConfigInfo_.size());
763 : }
764 12 : IDE_RUN_LOGI("OpInfoRecord start success!");
765 12 : return 0;
766 42 : }
767 :
768 15 : int32_t DumpManager::StopDumpArgs()
769 : {
770 15 : uint64_t dumpSwitch = 0;
771 : {
772 15 : std::lock_guard<std::mutex> lk(resourceMtx_);
773 15 : dumpSwitch = dumpSetting_.GetDumpSwitch();
774 15 : if ((dumpSwitch & OP_INFO_RECORD_DUMP) != OP_INFO_RECORD_DUMP) {
775 6 : return 0;
776 : }
777 9 : IDE_RUN_LOGI("OpInfoRecord Stop Entry!");
778 9 : dumpSwitch &= ~OP_INFO_RECORD_DUMP;
779 9 : dumpSetting_.InitDumpSwitch(dumpSwitch);
780 15 : }
781 21 : for (auto& item : disableCallbackFunc_) {
782 12 : item.second(dumpSwitch, dumpConfigInfo_.data(), dumpConfigInfo_.size());
783 : }
784 9 : IDE_RUN_LOGI("OpInfoRecord success!");
785 9 : return 0;
786 : }
787 :
788 6 : const char* DumpManager::GetExtraExceptionDumpPath()
789 : {
790 6 : std::lock_guard<std::mutex> lk(resourceMtx_);
791 6 : exceptionDumper_.CreateExtraDumpPath();
792 12 : return exceptionDumper_.GetExtraDumpCPath();
793 6 : }
794 :
795 36 : const char* DumpManager::GetDataDumpPath()
796 : {
797 36 : std::lock_guard<std::mutex> lk(resourceMtx_);
798 72 : return dumpSetting_.GetDumpCPath();
799 36 : }
800 :
801 9 : int32_t DumpManager::GetExceptionDumpPath(std::string& path)
802 : {
803 9 : std::lock_guard<std::mutex> lk(resourceMtx_);
804 18 : return exceptionDumper_.GetExceptionDumpPath(path);
805 9 : }
806 :
807 15 : int32_t DumpManager::SaveExceptionInfo(
808 : const std::string& fileName, const std::string& userTag, const std::vector<TensorInfo>& tensors)
809 : {
810 15 : std::lock_guard<std::mutex> lk(resourceMtx_);
811 30 : return exceptionDumper_.SaveExceptionInfo(fileName, userTag, tensors);
812 15 : }
813 :
814 27 : int32_t DumpManager::SaveFile(const char* data, size_t dataLen, const char* fileName, SaveType type)
815 : {
816 27 : std::string canonicalPath;
817 54 : if (!Adx::Path::BuildFullPathUnderRoot(opInfoRecordPath_, fileName, canonicalPath)) {
818 12 : IDE_LOGE("invalid fileName[%s], may escape root path.", fileName);
819 12 : return -1;
820 : }
821 15 : int32_t openFlag = 0;
822 15 : if (type == SaveType::OVERWRITE) {
823 9 : openFlag = O_CREAT | O_WRONLY | O_TRUNC;
824 : } else {
825 6 : openFlag = O_CREAT | O_WRONLY | O_APPEND;
826 : }
827 15 : File file(canonicalPath, openFlag);
828 :
829 15 : if (file.IsFileOpen() != 0) {
830 6 : IDE_LOGE("open file[%s] failed!", fileName);
831 6 : return -1;
832 : }
833 9 : int64_t ret = file.Write(data, dataLen);
834 9 : IDE_CTRL_VALUE_FAILED(ret >= 0, return -1, "Save file %s failed!", fileName);
835 :
836 9 : IDE_LOGI("DumpJsonToFile %s success!", fileName);
837 9 : return 0;
838 27 : }
839 :
840 48 : int32_t DumpManager::CallbackEnvExceptionDumpEvent(AdumpCallback callbackFunc)
841 : {
842 48 : if (isEnvExceptionDump_) {
843 0 : IDE_LOGI("Callback module when exception dump enabled with env.");
844 0 : if (exceptionDumper_.GetArgsExceptionStatus() || exceptionDumper_.GetCoredumpStatus()) {
845 0 : return callbackFunc(DUMP_SWITCH_L0_MACK, nullptr, 0);
846 0 : } else if (exceptionDumper_.GetExceptionStatus()) {
847 0 : return callbackFunc(DUMP_SWITCH_L1_MACK, nullptr, 0);
848 : }
849 : }
850 48 : return ADUMP_SUCCESS;
851 : }
852 :
853 : // DUMP 配置变化时，触发dump事件，同步回调用户接口
854 96 : int32_t DumpManager::HandleDumpEvent(uint32_t moduleId, DumpEnableAction action)
855 : {
856 96 : const uint64_t dumpSwitch = dumpSetting_.GetDumpSwitch();
857 96 : auto callbackFunc = disableCallbackFunc_[moduleId];
858 96 : if (action == DumpEnableAction::ENABLE) {
859 27 : callbackFunc = enableCallbackFunc_[moduleId];
860 : // 回调环境变量开启exception dump
861 27 : (void)CallbackEnvExceptionDumpEvent(callbackFunc);
862 : }
863 96 : if (action == DumpEnableAction::AUTO) {
864 : // 回调环境变量开启exception dump
865 18 : (void)CallbackEnvExceptionDumpEvent(enableCallbackFunc_[moduleId]);
866 18 : if (dumpConfigInfo_.data() == nullptr || dumpConfigInfo_.size() == 0U) {
867 3 : IDE_LOGW("Config data is null or empty. Not trigger HandleDumpEvent.");
868 3 : return ADUMP_SUCCESS;
869 : }
870 15 : if (dumpSwitch > 0U) {
871 15 : callbackFunc = enableCallbackFunc_[moduleId];
872 : }
873 : }
874 :
875 93 : if (!callbackFunc) {
876 15 : IDE_LOGE("No registered callback for module %u", moduleId);
877 15 : return ADUMP_FAILED;
878 : }
879 :
880 78 : IDE_LOGI("HandleDumpEvent callbackFunc start for module [%zu]", moduleId);
881 78 : IDE_LOGI("HandleDumpEvent callbackFunc switch [%" PRIu64 "]", dumpSwitch);
882 78 : IDE_LOGI(
883 : "HandleDumpEvent callbackFunc Dump config info: addr=%p, size=%zu", dumpConfigInfo_.data(),
884 : dumpConfigInfo_.size());
885 78 : int32_t result = callbackFunc(dumpSwitch, dumpConfigInfo_.data(), dumpConfigInfo_.size());
886 78 : IDE_LOGI("callbackFunc returned: %d", result);
887 78 : return result;
888 : }
889 :
890 : #ifdef __ADUMP_LLT
891 654 : void DumpManager::Reset()
892 : {
893 654 : registered_ = false;
894 654 : exceptionDumper_.Reset();
895 654 : }
896 :
897 9 : bool DumpManager::GetKFCInitStatus() { return isKFCInit_; }
898 :
899 6 : void DumpManager::SetKFCInitStatus(bool status) { isKFCInit_ = status; }
900 : #endif
901 :
902 18 : int32_t DumpManager::RegisterExceptionDumpCallback(ExceptionDumpCallback callback)
903 : {
904 18 : return exceptionDumper_.RegisterExceptionDumpCallback(callback);
905 : }
906 :
907 18 : int32_t DumpManager::UnregisterExceptionDumpCallback(ExceptionDumpCallback callback)
908 : {
909 18 : return exceptionDumper_.UnregisterExceptionDumpCallback(callback);
910 : }
911 :
912 : } // namespace Adx
|
