Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 : #include "dump_manager.h"
11 : #include <thread>
12 : #include <cctype>
13 : #include <cinttypes>
14 : #include <map>
15 : #include <algorithm>
16 : #include <cerrno>
17 : #include <sstream>
18 : #include "str_utils.h"
19 : #include "lib_path.h"
20 : #include "file_utils.h"
21 : #include "adump_platform_manager.h"
22 : #include "log/adx_log.h"
23 : #include "runtime/context.h"
24 : #include "runtime/config.h"
25 : #include "rts/rts_snapshot.h"
26 : #include "error_codes/rt_error_codes.h"
27 : #include "adump_dsmi.h"
28 : #include "common_utils.h"
29 : #include "exception_info_common.h"
30 : #include "dump_config_converter.h"
31 : #include "adump_api.h"
32 : #include "adump_error_manager.h"
33 : #include "operator_dumper.h"
34 : #include "kernel_dfx_dumper.h"
35 : #include "adx_dump_record.h"
36 : #include "common/file.h"
37 : #include "common/path.h"
38 : #include "sys_utils.h"
39 : #include "dump_file.h"
40 : #include "adx_datadump_server.h"
41 :
42 : namespace Adx {
43 : constexpr char EXCEPTION_CB_MODULE[] = "AdumpException";
44 : constexpr char COREDUMP_CB_MODULE[] = "AdumpCoredump";
45 :
46 : std::vector<std::shared_ptr<OperatorPreliminary>> DumpManager::operatorMap_;
47 :
48 0 : static void ExceptionCallback(rtExceptionInfo* const exception)
49 : {
50 0 : IDE_RUN_LOGI("An exception callback message is received.");
51 0 : if (exception != nullptr) {
52 0 : (void)DumpManager::Instance().DumpExceptionInfo(*exception);
53 0 : AdxLogFlush();
54 : }
55 0 : }
56 :
57 6 : static void NotifyCoredumpCallback(uint32_t devId, bool isOpen)
58 : {
59 6 : static std::map<uint32_t, uint32_t> setDeviceRecord;
60 : static std::mutex coredumpMtx;
61 6 : std::lock_guard<std::mutex> lk(coredumpMtx);
62 6 : auto it = setDeviceRecord.find(devId);
63 6 : if (!isOpen) {
64 2 : if (it != setDeviceRecord.end()) {
65 2 : it->second--;
66 2 : if (it->second == 0) {
67 1 : setDeviceRecord.erase(it);
68 1 : IDE_LOGI("Device %u has been removed from the effective list.", devId);
69 : }
70 : }
71 2 : return;
72 : }
73 4 : if (it != setDeviceRecord.end()) {
74 2 : it->second++;
75 2 : return;
76 : }
77 2 : rtError_t ret = rtDebugSetDumpMode(RT_DEBUG_DUMP_ON_EXCEPTION);
78 2 : if (ret != RT_ERROR_NONE) {
79 0 : IDE_RUN_LOGI("detail exception dump mode not support, switch to lite exception dump, ret:%d", ret);
80 0 : DumpManager::Instance().ExceptionModeDowngrade();
81 : }
82 2 : IDE_LOGI("Device %u has been added to the effective list.", devId);
83 2 : setDeviceRecord[devId] = 1;
84 6 : }
85 :
86 0 : static uint32_t DumpSnapShotLockPreCallback(int32_t deviceId, void* args)
87 : {
88 : UNUSED(deviceId);
89 : UNUSED(args);
90 0 : return DumpManager::Instance().StopDataDumpServer() ? 0U : 1U;
91 : }
92 :
93 2062 : DumpManager& DumpManager::Instance()
94 : {
95 2062 : static DumpManager instance;
96 2062 : return instance;
97 : }
98 :
99 3 : DumpManager::DumpManager()
100 : {
101 : try {
102 : // 1. 通过环境变量使能Exception Dump
103 3 : EnableExceptionDumpWithEnv();
104 : // 2. 通过环境变量使能Kernel Dfx Dump
105 3 : KernelDfxDumper::Instance();
106 0 : } catch (...) {
107 0 : IDE_LOGW("Enable dump function with env variable failed!");
108 0 : }
109 3 : }
110 :
111 258 : bool DumpManager::StartDataDumpServer()
112 : {
113 : // 注册快照回调：快照备份前关闭Data Dump Server
114 258 : RegisterSnapShotCallback();
115 : #if !defined(ADUMP_SOC_HOST) || ADUMP_SOC_HOST == 1
116 : // 如果没有启动Data Dump Server，启动Data Dump Server
117 258 : int32_t dumpNum = AdxDumpRecord::Instance().GetDumpInitNum();
118 258 : if (dumpNum == 0) {
119 39 : return AdxDataDumpServerInit() == ADUMP_SUCCESS;
120 : }
121 : #endif
122 219 : return true;
123 : }
124 :
125 39 : bool DumpManager::StopDataDumpServer()
126 : {
127 : #if !defined(ADUMP_SOC_HOST) || ADUMP_SOC_HOST == 1
128 39 : int32_t dumpNum = AdxDumpRecord::Instance().GetDumpInitNum();
129 78 : while (dumpNum > 0) {
130 39 : IDE_CTRL_VALUE_FAILED(
131 : AdxDataDumpServerUnInit() == ADUMP_SUCCESS, return false, "Stop data dump server failed!");
132 39 : dumpNum = AdxDumpRecord::Instance().GetDumpInitNum();
133 : }
134 : #endif
135 39 : return true;
136 : }
137 :
138 267 : void DumpManager::RegisterSnapShotCallback()
139 : {
140 267 : if (!snapCbkRegistered_) {
141 15 : rtError_t ret = rtSnapShotCallbackRegister(RT_SNAPSHOT_LOCK_PRE, DumpSnapShotLockPreCallback, nullptr);
142 15 : if (ret == ACL_ERROR_RT_FEATURE_NOT_SUPPORT) {
143 3 : IDE_LOGI("RTS does not support snapshot feature. ret=%d", ret);
144 3 : return;
145 : }
146 12 : IDE_CTRL_VALUE_WARN(
147 : ret == ACL_SUCCESS, return, "Register DumpSnapShotLockPreCallback to RTS failed. ret=%d", ret);
148 9 : snapCbkRegistered_ = true;
149 9 : IDE_LOGI("Register DumpSnapShotLockPreCallback success.");
150 : }
151 : }
152 :
153 3 : void DumpManager::EnableExceptionDumpWithEnv()
154 : {
155 3 : DumpConfig config;
156 : DumpType dumpType;
157 3 : if (DumpConfigConverter::EnableExceptionDumpWithEnv(config, dumpType)) {
158 0 : if (SetDumpConfig(dumpType, config) != ADUMP_SUCCESS) {
159 0 : IDE_LOGW("Enable exception dump failed. dumpType: %d", dumpType);
160 : } else {
161 0 : isEnvExceptionDump_ = true;
162 : }
163 : }
164 3 : }
165 :
166 258 : bool DumpManager::AdjustOpExecuteTimeOut()
167 : {
168 258 : if (opTimeoutModified_) {
169 195 : IDE_LOGI("Op execute timeout has been adjusted for data dump, skip.");
170 195 : return true;
171 : }
172 63 : uint32_t curTimeout = 0U;
173 63 : rtError_t ret = rtGetOpExecuteTimeoutV2(&curTimeout);
174 63 : IDE_CTRL_VALUE_FAILED(ret == RT_ERROR_NONE, return false,
175 : "Get op execute timeout failed, skip adjust for data dump. ret=%d", ret);
176 60 : if (curTimeout >= OP_EXECUTE_TIMEOUT_FOR_DATADUMP_MS) {
177 6 : IDE_LOGI("Current op execute timeout %ums is no less than %ums, no need to adjust for data dump.",
178 : curTimeout, OP_EXECUTE_TIMEOUT_FOR_DATADUMP_MS);
179 6 : return true;
180 : }
181 54 : ret = rtSetOpExecuteTimeOutWithMs(OP_EXECUTE_TIMEOUT_FOR_DATADUMP_MS);
182 54 : IDE_CTRL_VALUE_FAILED(ret == RT_ERROR_NONE, return false,
183 : "Set op execute timeout to %ums failed for data dump. ret=%d", OP_EXECUTE_TIMEOUT_FOR_DATADUMP_MS, ret);
184 51 : originOpExecuteTimeOut_ = curTimeout;
185 51 : opTimeoutModified_ = true;
186 51 : IDE_LOGI("Adjust op execute timeout from %ums to %ums for data dump.",
187 : curTimeout, OP_EXECUTE_TIMEOUT_FOR_DATADUMP_MS);
188 51 : return true;
189 : }
190 :
191 36 : bool DumpManager::RestoreOpExecuteTimeOut()
192 : {
193 36 : if (!opTimeoutModified_) {
194 9 : return true;
195 : }
196 27 : rtError_t ret = rtSetOpExecuteTimeOutWithMs(originOpExecuteTimeOut_);
197 : // 恢复失败时保持 opTimeoutModified_=true，以便下次关闭 data dump 时重试恢复
198 27 : IDE_CTRL_VALUE_FAILED(ret == RT_ERROR_NONE, return false,
199 : "Restore op execute timeout to %ums failed. ret=%d", originOpExecuteTimeOut_, ret);
200 24 : IDE_LOGI("Restore op execute timeout to %ums after data dump disabled.", originOpExecuteTimeOut_);
201 24 : opTimeoutModified_ = false;
202 24 : originOpExecuteTimeOut_ = 0U;
203 24 : return true;
204 : }
205 :
206 0 : void DumpManager::KFCResourceInit()
207 : {
208 : #if !defined(ADUMP_SOC_HOST) || ADUMP_SOC_HOST == 1
209 0 : if (isKFCInit_) {
210 0 : IDE_LOGD("KFC resources have been initialized on all devices.");
211 0 : return;
212 : }
213 0 : std::vector<uint32_t> devList = AdumpDsmi::DrvGetDeviceList();
214 :
215 0 : for (uint32_t& deviceId : devList) {
216 0 : IDE_LOGI("Start to initialize KFC resources on device %u.", deviceId);
217 0 : SharedPtr<OperatorPreliminary> opIniter = MakeSharedInstance<OperatorPreliminary>(GetDumpSetting(), deviceId);
218 0 : IDE_CTRL_VALUE_FAILED_NODO(
219 : opIniter != nullptr && opIniter->OperatorInit() == ADUMP_SUCCESS, return,
220 : "Failed to execute the resource initialization task on device %u.", deviceId);
221 0 : DumpManager::operatorMap_.emplace_back(std::move(opIniter));
222 0 : IDE_LOGI("KFC executed on the device %u successfully.", deviceId);
223 0 : }
224 : #endif
225 0 : isKFCInit_ = true;
226 0 : }
227 :
228 54 : int32_t DumpManager::ExceptionConfig(DumpType dumpType, const DumpConfig& dumpConfig)
229 : {
230 54 : if (exceptionDumper_.IsRepeatEnableException(dumpType, dumpConfig)) {
231 3 : IDE_LOGW(
232 : "Exception dump has been enabled, not support enable exception dump[%s] again",
233 : DumpConfigConverter::DumpTypeToStr(dumpType).c_str());
234 3 : return ADUMP_SUCCESS;
235 : }
236 :
237 51 : if (dumpType == DumpType::AIC_ERR_DETAIL_DUMP && !CheckCoredumpSupportedPlatform()) {
238 3 : IDE_LOGE("Current platform is not support coredump mode.");
239 3 : return ADUMP_FAILED;
240 : }
241 :
242 48 : IDE_RUN_LOGI(
243 : "Set %s[%d] dump setting, status: %s, dump switch: %llu", DumpConfigConverter::DumpTypeToStr(dumpType).c_str(),
244 : dumpType, dumpConfig.dumpStatus.c_str(), dumpConfig.dumpSwitch);
245 48 : if (exceptionDumper_.ExceptionDumperInit(dumpType, dumpConfig) != ADUMP_SUCCESS) {
246 1 : IDE_LOGW("Failed to initialize the exception dump.");
247 1 : return ADUMP_SUCCESS;
248 : }
249 47 : dumpSetting_.InitDumpSwitch(dumpConfig.dumpSwitch & DUMP_SWITCH_MASK);
250 47 : IDE_CTRL_VALUE_WARN(
251 : RegsiterExceptionCallback(), return ADUMP_SUCCESS,
252 : "Failed to register the args exception dump callback function.");
253 46 : if (exceptionDumper_.GetCoredumpStatus()) { // 如果启动了coredump模式
254 1 : IDE_CTRL_VALUE_FAILED(
255 : rtRegDeviceStateCallbackEx(COREDUMP_CB_MODULE, &NotifyCoredumpCallback, DEV_CB_POS_BACK) == RT_ERROR_NONE,
256 : return ADUMP_FAILED, "Failed to register the coredump callback function to rtSetDevice.");
257 : }
258 46 : return ADUMP_SUCCESS;
259 : }
260 :
261 399 : int32_t DumpManager::SetDumpConfig(DumpType dumpType, const DumpConfig& dumpConfig)
262 : {
263 399 : std::lock_guard<std::mutex> lk(resourceMtx_);
264 399 : if (dumpType == DumpType::EXCEPTION || dumpType == DumpType::ARGS_EXCEPTION ||
265 : dumpType == DumpType::AIC_ERR_DETAIL_DUMP) {
266 54 : return ExceptionConfig(dumpType, dumpConfig);
267 : }
268 345 : auto ret = dumpSetting_.Init(dumpType, dumpConfig);
269 345 : if (ret != ADUMP_SUCCESS) {
270 21 : return ret;
271 : }
272 :
273 : // 启动Data Dump Server
274 324 : if (dumpConfig.dumpStatus != ADUMP_DUMP_STATUS_SWITCH_OFF) {
275 258 : IDE_CTRL_VALUE_FAILED(StartDataDumpServer(), return ADUMP_FAILED,
276 : "Start data dump server failed! dumpType=%s[%d]",
277 : DumpConfigConverter::DumpTypeToStr(dumpType).c_str(), dumpType);
278 : // 后续失败不恢复超时时间，不影响功能。
279 258 : IDE_CTRL_VALUE_FAILED(AdjustOpExecuteTimeOut(), return ADUMP_FAILED, "Adjust op execute timeout failed.");
280 : }
281 :
282 318 : if (CheckBinValidation() && (isKFCInit_ == false)) {
283 0 : auto kfcBind = std::bind(&DumpManager::KFCResourceInit, this);
284 0 : std::thread kfcThread(kfcBind);
285 0 : kfcThread.join();
286 0 : if (!isKFCInit_) {
287 0 : IDE_LOGE("SetDumpConfig failed due to kfc resource initialization error.");
288 0 : DumpManager::operatorMap_.clear();
289 0 : return ADUMP_FAILED;
290 : }
291 0 : }
292 :
293 318 : ret = OperatorDumper(dumpSetting_).UpdateDevMemCache();
294 318 : IDE_CTRL_VALUE_FAILED(ret == ADUMP_SUCCESS, return ADUMP_FAILED,
295 : "Update device memery cache for data dump failed! dumpType=%s[%d]",
296 : DumpConfigConverter::DumpTypeToStr(dumpType).c_str(), dumpType);
297 :
298 318 : IDE_RUN_LOGI(
299 : "Set %s[%d] dump setting, status: %s, mode: %s, data: %s, dump switch: %llu, path:%s, dump stats:%s.",
300 : DumpConfigConverter::DumpTypeToStr(dumpType).c_str(), dumpType, dumpConfig.dumpStatus.c_str(),
301 : dumpConfig.dumpMode.c_str(), dumpConfig.dumpData.c_str(), dumpConfig.dumpSwitch, dumpConfig.dumpPath.c_str(),
302 : StrUtils::ToString(dumpConfig.dumpStatsItem).c_str());
303 318 : return ADUMP_SUCCESS;
304 399 : }
305 :
306 111 : int32_t DumpManager::SetDumpConfig(const char *dumpConfigData, size_t dumpConfigSize, const char *dumpConfigPath)
307 : {
308 111 : std::lock_guard<std::mutex> lk(resourceMtx2_);
309 111 : if ((dumpConfigData == nullptr) || (dumpConfigSize == 0U) || (dumpConfigPath == nullptr)) {
310 24 : IDE_LOGE("Set dump config failed. Config data is null or empty.");
311 24 : return ADUMP_FAILED;
312 : }
313 87 : DumpConfig dumpConfig;
314 87 : DumpDfxConfig dumpDfxConfig;
315 : DumpType dumpType;
316 87 : bool needDump = true;
317 87 : DumpConfigConverter converter{dumpConfigData, dumpConfigSize, dumpConfigPath};
318 87 : int32_t ret = converter.Convert(dumpType, dumpConfig, needDump, dumpDfxConfig);
319 87 : if (ret != ADUMP_SUCCESS) {
320 12 : IDE_LOGE("Parse dump config from memory[%s] failed.", dumpConfigData);
321 12 : return ADUMP_INPUT_FAILED;
322 : }
323 :
324 : // 开启KernelDataDump
325 75 : ret = KernelDfxDumper::Instance().EnableDfxDumper(dumpDfxConfig);
326 75 : IDE_CTRL_VALUE_FAILED(ret == ADUMP_SUCCESS, return ret, "Enable kernel dfx dump failed.");
327 :
328 75 : if (!needDump) {
329 9 : return ADUMP_SUCCESS;
330 : }
331 :
332 : // 已开启exception dump：不支持重复使能，不更新配置缓存，不重复回调注册模块组件
333 66 : if (exceptionDumper_.IsRepeatEnableException(dumpType, dumpConfig)) {
334 0 : IDE_LOGW(
335 : "Exception dump has been enabled, not support enable exception dump[%s] again",
336 : DumpConfigConverter::DumpTypeToStr(dumpType).c_str());
337 0 : return ADUMP_SUCCESS;
338 : }
339 :
340 66 : (void)dumpConfigInfo_.assign(dumpConfigData, dumpConfigSize);
341 66 : IDE_LOGI("Dump config info set: addr=%p, size=%zu", dumpConfigInfo_.data(), dumpConfigInfo_.size());
342 66 : ret = SetDumpConfig(dumpType, dumpConfig);
343 : // 同步触发callback事件
344 93 : for (auto& item : enableCallbackFunc_) {
345 27 : IDE_LOGI("SetDumpConfig HandleDumpEvent start for module [%zu]", item.first);
346 27 : HandleDumpEvent(item.first, DumpEnableAction::ENABLE);
347 : }
348 66 : IDE_CTRL_VALUE_FAILED(ret == ADUMP_SUCCESS, return ret, "Set dump config failed.");
349 60 : (void)openedDump_.insert(dumpType);
350 60 : return ADUMP_SUCCESS;
351 111 : }
352 :
353 42 : int32_t DumpManager::UnSetDumpConfig()
354 : {
355 42 : std::lock_guard<std::mutex> lk(resourceMtx2_);
356 42 : DumpConfig config;
357 42 : config.dumpStatus = ADUMP_DUMP_STATUS_SWITCH_OFF;
358 42 : config.dumpSwitch = 0;
359 66 : for (const auto dumpType : openedDump_) {
360 30 : if (IsEnableDump(dumpType)) {
361 30 : const auto ret = SetDumpConfig(dumpType, config);
362 30 : IDE_CTRL_VALUE_FAILED(
363 : ret == ADUMP_SUCCESS, return ADUMP_FAILED, "[Set][Dump]Set dump off failed! dumpType=%s[%d], ret=%d",
364 : DumpConfigConverter::DumpTypeToStr(dumpType).c_str(), dumpType, ret);
365 24 : IDE_LOGI(
366 : "[Set][Dump]Set dump off successfully, dumpType=%s[%d]",
367 : DumpConfigConverter::DumpTypeToStr(dumpType).c_str(), dumpType);
368 : }
369 : }
370 36 : openedDump_.clear();
371 : // 同步触发callback事件
372 81 : for (auto& item : disableCallbackFunc_) {
373 45 : IDE_LOGI("UnSetDumpConfig start for module [%zu]", item.first);
374 45 : HandleDumpEvent(item.first, DumpEnableAction::DISABLE);
375 : }
376 36 : dumpConfigInfo_.clear();
377 36 : IDE_LOGI("Dump config info cleared.");
378 :
379 : // 等待所有 dump 操作完成并清理资源
380 36 : DumpResourceSafeMap::Instance().waitAndClear();
381 : // 释放device资源
382 36 : OperatorDumper::FreeDevMemCache();
383 : // 停止data dump server
384 36 : IDE_CTRL_VALUE_FAILED(StopDataDumpServer(), return ADUMP_FAILED, "Stop data dump server failed!");
385 : // 恢复算子超时时间
386 36 : IDE_CTRL_VALUE_FAILED(RestoreOpExecuteTimeOut(), return ADUMP_FAILED, "Restore op execute timeout failed!");
387 :
388 33 : return ADUMP_SUCCESS;
389 42 : }
390 :
391 321 : std::string DumpManager::GetBinName() const
392 : {
393 321 : auto plat = PlatformReflection<DataDumpInterface>::CreatePlatform(dumpSetting_.GetPlatformType());
394 321 : if (plat == nullptr) {
395 642 : return "";
396 : }
397 0 : return plat->GetKfcBinName();
398 321 : }
399 :
400 321 : bool DumpManager::CheckBinValidation()
401 : {
402 321 : const std::string opName = GetBinName();
403 321 : if ((dumpSetting_.GetDumpData().compare(DUMP_STATS_DATA) != 0) || opName.empty()) {
404 321 : return false;
405 : }
406 0 : const std::string opPath = LibPath::Instance().GetTargetPath(opName);
407 0 : IDE_CTRL_VALUE_FAILED(!opPath.empty(), return false, "Received an empty path for file %s.", opName.c_str());
408 0 : bool isExist = FileUtils::IsFileExist(opPath);
409 0 : IDE_LOGI("CheckBinValidation result is %s", isExist ? "true" : "false");
410 0 : return isExist;
411 321 : }
412 :
413 161 : bool DumpManager::IsEnableDump(DumpType dumpType)
414 : {
415 161 : std::lock_guard<std::mutex> lk(resourceMtx_);
416 161 : if (dumpType == DumpType::ARGS_EXCEPTION) {
417 38 : return exceptionDumper_.GetArgsExceptionStatus() || exceptionDumper_.GetCoredumpStatus();
418 123 : } else if (dumpType == DumpType::OPERATOR) {
419 93 : return dumpSetting_.GetDumpStatus();
420 30 : } else if (dumpType == DumpType::OP_OVERFLOW) {
421 21 : return dumpSetting_.GetDumpDebugStatus();
422 9 : } else if (dumpType == DumpType::EXCEPTION) {
423 3 : return exceptionDumper_.GetExceptionStatus();
424 6 : } else if (dumpType == DumpType::AIC_ERR_DETAIL_DUMP) {
425 3 : return exceptionDumper_.GetCoredumpStatus();
426 : } else {
427 3 : IDE_LOGW("Dump type is not support.");
428 : }
429 :
430 3 : return false;
431 161 : }
432 :
433 39 : int32_t DumpManager::GetInputOutputTensors(
434 : const std::string& opType, const std::string& opName, const std::vector<TensorInfoV2>& tensors,
435 : std::vector<DumpTensor>& inputTensors, std::vector<DumpTensor>& outputTensors)
436 : {
437 75 : for (const auto& tensorInfo : tensors) {
438 36 : if (tensorInfo.tensorAddr == nullptr || tensorInfo.tensorSize == 0) {
439 6 : IDE_LOGW("Tensor of op=%s[%s] is empty, addr=%p, size=%zu, skip it.",
440 : opName.c_str(), opType.c_str(), tensorInfo.tensorAddr, tensorInfo.tensorSize);
441 6 : continue;
442 : }
443 :
444 30 : if (tensorInfo.placement != TensorPlacement::kOnDeviceHbm) {
445 6 : IDE_LOGW("Tensor of op=%s[%s] is not on device, skip it.", opName.c_str(), opType.c_str());
446 6 : continue;
447 : }
448 :
449 24 : if (tensorInfo.type == TensorType::INPUT) {
450 21 : inputTensors.emplace_back(tensorInfo);
451 3 : } else if (tensorInfo.type == TensorType::OUTPUT) {
452 3 : outputTensors.emplace_back(tensorInfo);
453 : }
454 : }
455 39 : return ADUMP_SUCCESS;
456 : }
457 :
458 24 : bool DumpManager::IsEnableDumpOperatorWithCapture(const std::string& opType, const std::string& opName,
459 : aclrtStream stream)
460 : {
461 24 : rtStreamCaptureStatus status = RT_STREAM_CAPTURE_STATUS_MAX;
462 24 : rtModel_t* captureMdl = nullptr;
463 24 : int32_t ret = rtStreamGetCaptureInfo(stream, &status, captureMdl);
464 24 : if (ret != ACL_SUCCESS) {
465 6 : IDE_LOGW("Get stream capture info error: %d, will switch to common dump", ret);
466 6 : return false;
467 : }
468 18 : IDE_LOGI("%s[%s] : stream capture status: %d", opName.c_str(), opType.c_str(), status);
469 18 : return status == RT_STREAM_CAPTURE_STATUS_ACTIVE;
470 : }
471 :
472 27 : int32_t DumpManager::DumpOperatorWithCfg(const std::string &opType, const std::string &opName,
473 : const std::vector<TensorInfo> &tensors, aclrtStream stream, const DumpCfg &dumpCfg)
474 : {
475 27 : std::lock_guard<std::mutex> lk(resourceMtx_);
476 27 : if (!dumpSetting_.GetDumpStatusEx() && !dumpSetting_.GetDumpDebugStatus()) {
477 3 : IDE_LOGW("Operator or overflow dump is not enable, can't dump data.");
478 3 : return ADUMP_SUCCESS;
479 : }
480 :
481 24 : bool isInvalid = dumpCfg.numAttrs != 0UL && dumpCfg.attrs == nullptr;
482 24 : IDE_CTRL_VALUE_FAILED(!isInvalid, return ADUMP_FAILED,
483 : "The dump cfg attrs is null pointer! op=%s[%s].", opName.c_str(), opType.c_str());
484 :
485 21 : std::vector<DumpTensor> inputTensors;
486 21 : std::vector<DumpTensor> outputTensors;
487 21 : int32_t ret = GetInputOutputTensors(opType, opName, ConvertTensorInfoToDumpTensorV2(tensors),
488 : inputTensors, outputTensors);
489 21 : IDE_CTRL_VALUE_FAILED(ret == ADUMP_SUCCESS, return ADUMP_FAILED,
490 : "Get input and output tensors failed! op=%s[%s].", opName.c_str(), opType.c_str());
491 21 : IDE_CTRL_VALUE_WARN(!inputTensors.empty() || !outputTensors.empty(), return ADUMP_SUCCESS,
492 : "No tensor need to dump. op=%s[%s].", opName.c_str(), opType.c_str());
493 :
494 6 : if (IsEnableDumpOperatorWithCapture(opType, opName, stream)) {
495 3 : if (dumpSetting_.GetDumpDebugStatus() || dumpSetting_.IsDumpDataStats()) {
496 0 : IDE_LOGI("overflow or stats is not allow in capture stream");
497 0 : return ADUMP_SUCCESS;
498 : }
499 3 : return DumpOperatorWithCapture(opType, opName, inputTensors, outputTensors, stream);
500 : }
501 :
502 3 : OperatorDumper opDumper(opType, opName);
503 3 : ret = opDumper.SetDumpSetting(dumpSetting_)
504 3 : .RuntimeStream(stream)
505 3 : .InputDumpTensor(inputTensors)
506 3 : .OutputDumpTensor(outputTensors)
507 3 : .LaunchWithCfg(dumpCfg);
508 3 : IDE_CTRL_VALUE_FAILED(ret == ADUMP_SUCCESS, return ret,
509 : "Launch dump operator with dump cfg failed! op=%s[%s].", opName.c_str(), opType.c_str());
510 0 : return ADUMP_SUCCESS;
511 27 : }
512 :
513 6 : int32_t DumpManager::DumpOperator(
514 : const std::string& opType, const std::string& opName, const std::vector<TensorInfo>& tensors, aclrtStream stream)
515 : {
516 6 : return DumpOperatorV2(opType, opName, ConvertTensorInfoToDumpTensorV2(tensors), stream);
517 : }
518 :
519 24 : int32_t DumpManager::DumpOperatorV2(
520 : const std::string& opType, const std::string& opName, const std::vector<TensorInfoV2>& tensors, aclrtStream stream)
521 : {
522 24 : std::lock_guard<std::mutex> lk(resourceMtx_);
523 24 : if (!dumpSetting_.GetDumpStatus() && !dumpSetting_.GetDumpDebugStatus()) {
524 6 : IDE_LOGW("Operator or overflow dump is not enable, can't dump.");
525 6 : return ADUMP_SUCCESS;
526 : }
527 :
528 18 : std::vector<DumpTensor> inputTensors;
529 18 : std::vector<DumpTensor> outputTensors;
530 18 : int32_t ret = GetInputOutputTensors(opType, opName, tensors, inputTensors, outputTensors);
531 18 : IDE_CTRL_VALUE_FAILED(ret == ADUMP_SUCCESS, return ADUMP_FAILED,
532 : "Get input and output tensors failed! opName: %s, opType: %s", opName.c_str(), opType.c_str());
533 :
534 18 : if (IsEnableDumpOperatorWithCapture(opType, opName, stream)) {
535 12 : if (dumpSetting_.GetDumpDebugStatus() || dumpSetting_.IsDumpDataStats()) {
536 9 : IDE_LOGI("overflow or stats is not allow in capture stream");
537 9 : return ADUMP_SUCCESS;
538 : }
539 3 : return DumpOperatorWithCapture(opType, opName, inputTensors, outputTensors, stream);
540 : }
541 :
542 6 : OperatorDumper opDumper(opType, opName);
543 6 : ret = opDumper.SetDumpSetting(dumpSetting_)
544 6 : .RuntimeStream(stream)
545 6 : .InputDumpTensor(inputTensors)
546 6 : .OutputDumpTensor(outputTensors)
547 6 : .Launch();
548 6 : IDE_CTRL_VALUE_FAILED(ret == ADUMP_SUCCESS, return ret,
549 : "Launch dump operator failed! op=%s[%s].", opName.c_str(), opType.c_str());
550 3 : return ADUMP_SUCCESS;
551 24 : }
552 :
553 9 : int32_t DumpManager::DumpOperatorWithCapture(
554 : const std::string& opType, const std::string& opName, const std::vector<DumpTensor>& inputTensors,
555 : const std::vector<DumpTensor>& outputTensors, aclrtStream mainStream)
556 : {
557 9 : if (mainStream == nullptr) {
558 9 : IDE_LOGE("mainStream is nullptr.");
559 9 : return ADUMP_FAILED;
560 : }
561 :
562 0 : if (!isCaptureDumpServerInit_) {
563 0 : IDE_CTRL_VALUE_FAILED(StartDataDumpServer(), return ADUMP_FAILED, "Start data dump server failed!");
564 0 : isCaptureDumpServerInit_ = true;
565 : }
566 :
567 0 : uint32_t streamId = 0;
568 0 : uint32_t taskId = 0;
569 0 : uint32_t deviceId = 0;
570 0 : std::string dumpPath;
571 0 : int32_t ret = CollectStreamContextInfo(mainStream, opName, opType, streamId, taskId, deviceId, dumpPath);
572 0 : if (ret != ADUMP_SUCCESS) {
573 0 : IDE_LOGE("%s(%s) collect stream context info failed.", opName.c_str(), opType.c_str());
574 0 : return ret;
575 : }
576 :
577 0 : std::string mainStreamKey = std::to_string(streamId) + "_" + std::to_string(taskId);
578 0 : DumpInfoParams params = {
579 0 : mainStreamKey, inputTensors, outputTensors, opType, opName, streamId, taskId, deviceId, 0, 0, dumpPath};
580 0 : ret = GetDumpInfoFromMap(params);
581 0 : std::shared_ptr<DumpStreamInfo> dumpInfoPtr = DumpResourceSafeMap::Instance().get(mainStreamKey);
582 0 : if (ret != ADUMP_SUCCESS || dumpInfoPtr == nullptr) {
583 0 : IDE_LOGE("%s(%s) get dump info failed.", opName.c_str(), opType.c_str());
584 0 : return ADUMP_FAILED;
585 : }
586 0 : IDE_LOGI("%s(%s) dump data : create DumpStreamInfo success", opName.c_str(), opType.c_str());
587 :
588 0 : ret = SetupAsyncDump(dumpInfoPtr, opName, opType, mainStream);
589 0 : if (ret != ADUMP_SUCCESS) {
590 0 : return ret;
591 : }
592 0 : IDE_LOGI("%s(%s) set main stream %u, dump stream %u, callback function success",
593 : opName.c_str(), opType.c_str(), dumpInfoPtr->streamId, dumpInfoPtr->dumpStmId);
594 :
595 0 : return ADUMP_SUCCESS;
596 0 : }
597 :
598 4 : void DumpManager::AddExceptionOp(const OperatorInfo& opInfo)
599 : {
600 4 : exceptionDumper_.AddDumpOperator(opInfo);
601 4 : }
602 :
603 4 : void DumpManager::AddExceptionOpV2(const OperatorInfoV2& opInfo)
604 : {
605 4 : exceptionDumper_.AddDumpOperatorV2(opInfo);
606 4 : }
607 :
608 4 : void DumpManager::ConvertOperatorInfo(const OperatorInfo& opInfo, OperatorInfoV2& operatorInfoV2) const
609 : {
610 4 : operatorInfoV2.agingFlag = opInfo.agingFlag;
611 4 : operatorInfoV2.taskId = opInfo.taskId;
612 4 : operatorInfoV2.streamId = opInfo.streamId;
613 4 : operatorInfoV2.deviceId = opInfo.deviceId;
614 4 : operatorInfoV2.contextId = opInfo.contextId;
615 4 : operatorInfoV2.opType = opInfo.opType;
616 4 : operatorInfoV2.opName = opInfo.opName;
617 4 : operatorInfoV2.tensorInfos = ConvertTensorInfoToDumpTensorV2(opInfo.tensorInfos);
618 4 : operatorInfoV2.deviceInfos = opInfo.deviceInfos;
619 4 : operatorInfoV2.additionalInfo = opInfo.additionalInfo;
620 4 : }
621 :
622 31 : std::vector<TensorInfoV2> DumpManager::ConvertTensorInfoToDumpTensorV2(const std::vector<TensorInfo>& tensorInfos) const
623 : {
624 31 : std::vector<TensorInfoV2> tensors;
625 31 : tensors.reserve(tensorInfos.size());
626 55 : for (const auto& tensorInfo : tensorInfos) {
627 24 : TensorInfoV2 tensor = {};
628 24 : ConvertTensorInfo(tensorInfo, tensor);
629 24 : tensors.emplace_back(tensor);
630 24 : }
631 31 : return tensors;
632 0 : }
633 :
634 96 : void DumpManager::ConvertTensorInfo(const TensorInfo& tensorInfo, TensorInfoV2& tensor) const
635 : {
636 96 : tensor.dataType = tensorInfo.dataType;
637 96 : tensor.format = tensorInfo.format;
638 96 : tensor.placement = tensorInfo.placement;
639 96 : tensor.tensorAddr = tensorInfo.tensorAddr;
640 96 : tensor.tensorSize = tensorInfo.tensorSize;
641 96 : tensor.type = tensorInfo.type;
642 96 : tensor.addrType = tensorInfo.addrType;
643 96 : tensor.argsOffSet = tensorInfo.argsOffSet;
644 96 : std::vector<int64_t> shape = tensorInfo.shape;
645 171 : for (auto dim : shape) {
646 75 : tensor.shape.emplace_back(static_cast<uint64_t>(dim));
647 : }
648 96 : std::vector<int64_t> originShape = tensorInfo.originShape;
649 171 : for (auto dim : originShape) {
650 75 : tensor.originShape.emplace_back(static_cast<uint64_t>(dim));
651 : }
652 96 : }
653 :
654 6 : int32_t DumpManager::DelExceptionOp(uint32_t deviceId, uint32_t streamId)
655 : {
656 6 : return exceptionDumper_.DelDumpOperator(deviceId, streamId);
657 : }
658 :
659 52 : int32_t DumpManager::DumpExceptionInfo(const rtExceptionInfo& exception)
660 : {
661 52 : return exceptionDumper_.DumpException(exception);
662 : }
663 :
664 17 : uint64_t DumpManager::AdumpGetDumpSwitch()
665 : {
666 17 : std::lock_guard<std::mutex> lk(resourceMtx_);
667 34 : return dumpSetting_.GetDumpSwitch();
668 17 : }
669 :
670 47 : bool DumpManager::RegsiterExceptionCallback()
671 : {
672 47 : if (!registered_ && rtRegTaskFailCallbackByModule(EXCEPTION_CB_MODULE, ExceptionCallback) == RT_ERROR_NONE) {
673 37 : registered_ = true;
674 : }
675 47 : IDE_LOGI("Register exception callback, registered: %d", static_cast<int32_t>(registered_));
676 47 : return registered_;
677 : }
678 :
679 24 : DumpSetting DumpManager::GetDumpSetting() const
680 : {
681 24 : return dumpSetting_;
682 : }
683 :
684 3 : void DumpManager::ExceptionModeDowngrade()
685 : {
686 3 : exceptionDumper_.ExceptionModeDowngrade();
687 3 : }
688 :
689 60 : bool DumpManager::IsEnabledExceptionDump()
690 : {
691 60 : std::lock_guard<std::mutex> lk(resourceMtx_);
692 120 : return exceptionDumper_.IsEnabledExceptionDump();
693 60 : }
694 :
695 30 : int32_t DumpManager::RegisterCallback(uint32_t moduleId, AdumpCallback enableFunc, AdumpCallback disableFunc)
696 : {
697 30 : if (enableFunc == nullptr) {
698 6 : IDE_LOGE("Register callback failed: enableFunc is null for module %u", moduleId);
699 6 : return ADUMP_FAILED;
700 : }
701 24 : if (disableFunc == nullptr) {
702 6 : IDE_LOGE("Register callback failed: disableFunc is null for module %u", moduleId);
703 6 : return ADUMP_FAILED;
704 : }
705 18 : std::lock_guard<std::mutex> lk(resourceMtx2_);
706 18 : enableCallbackFunc_[moduleId] = enableFunc;
707 18 : disableCallbackFunc_[moduleId] = disableFunc;
708 18 : IDE_LOGI("Registered callback for module %u", moduleId);
709 18 : return HandleDumpEvent(moduleId, DumpEnableAction::AUTO);
710 18 : }
711 :
712 33 : int32_t DumpManager::StartDumpArgs(const std::string& dumpPath)
713 : {
714 33 : uint64_t dumpSwitch = 0;
715 : {
716 33 : std::lock_guard<std::mutex> lk(resourceMtx_);
717 33 : dumpSwitch = dumpSetting_.GetDumpSwitch();
718 33 : if ((dumpSwitch & OP_INFO_RECORD_DUMP) == OP_INFO_RECORD_DUMP) {
719 144 : REPORT_EP0008_API_CALL_SEQUENCE(
720 : FUNC_NAME_ACL_OP_START_DUMP_ARGS, ADUMP_REASON_API_CALLED_REPEATEDLY);
721 9 : return -1;
722 : }
723 :
724 24 : Adx::Path path(dumpPath);
725 24 : if (path.Empty()) {
726 72 : REPORT_EP0006_INVALID_ARGUMENT(
727 : FUNC_NAME_ACL_OP_START_DUMP_ARGS, dumpPath, FUNC_ACL_OP_START_DUMP_ARGS_PARAM_PATH,
728 : ADUMP_REASON_PARAM_PATH_EMPTY);
729 3 : return -1;
730 : }
731 21 : if (!path.Exist()) {
732 18 : if (!path.CreateDirectory(true)) {
733 3 : std::string reason = StrUtils::Format(ADUMP_REASON_PARAM_PATH_CREATE_DIR_ERROR, strerror(errno));
734 69 : REPORT_EP0006_INVALID_ARGUMENT(
735 : FUNC_NAME_ACL_OP_START_DUMP_ARGS, dumpPath, FUNC_ACL_OP_START_DUMP_ARGS_PARAM_PATH, reason);
736 3 : return -1;
737 3 : }
738 : }
739 18 : if (!path.IsDirectory()) {
740 72 : REPORT_EP0006_INVALID_ARGUMENT(
741 : FUNC_NAME_ACL_OP_START_DUMP_ARGS, dumpPath, FUNC_ACL_OP_START_DUMP_ARGS_PARAM_PATH,
742 : ADUMP_REASON_PARAM_PATH_NOT_DIRECTORY);
743 3 : return -1;
744 : }
745 15 : constexpr uint32_t accessMode = static_cast<uint32_t>(M_R_OK) | static_cast<uint32_t>(M_W_OK);
746 15 : if (!path.Asccess(accessMode)) {
747 72 : REPORT_EP0006_INVALID_ARGUMENT(
748 : FUNC_NAME_ACL_OP_START_DUMP_ARGS, dumpPath, FUNC_ACL_OP_START_DUMP_ARGS_PARAM_PATH,
749 : ADUMP_REASON_PATH_NO_PERMISSION);
750 3 : return -1;
751 : }
752 :
753 12 : dumpSwitch |= OP_INFO_RECORD_DUMP;
754 12 : dumpSetting_.InitDumpSwitch(dumpSwitch);
755 12 : opInfoRecordPath_ = path.GetString();
756 45 : }
757 33 : for (auto& item : enableCallbackFunc_) {
758 21 : item.second(dumpSwitch, dumpConfigInfo_.data(), dumpConfigInfo_.size());
759 : }
760 12 : IDE_RUN_LOGI("OpInfoRecord start success!");
761 12 : return 0;
762 42 : }
763 :
764 15 : int32_t DumpManager::StopDumpArgs()
765 : {
766 15 : uint64_t dumpSwitch = 0;
767 : {
768 15 : std::lock_guard<std::mutex> lk(resourceMtx_);
769 15 : dumpSwitch = dumpSetting_.GetDumpSwitch();
770 15 : if ((dumpSwitch & OP_INFO_RECORD_DUMP) != OP_INFO_RECORD_DUMP) {
771 6 : return 0;
772 : }
773 9 : IDE_RUN_LOGI("OpInfoRecord Stop Entry!");
774 9 : dumpSwitch &= ~OP_INFO_RECORD_DUMP;
775 9 : dumpSetting_.InitDumpSwitch(dumpSwitch);
776 15 : }
777 21 : for (auto& item : disableCallbackFunc_) {
778 12 : item.second(dumpSwitch, dumpConfigInfo_.data(), dumpConfigInfo_.size());
779 : }
780 9 : IDE_RUN_LOGI("OpInfoRecord success!");
781 9 : return 0;
782 : }
783 :
784 6 : const char* DumpManager::GetExtraExceptionDumpPath()
785 : {
786 6 : std::lock_guard<std::mutex> lk(resourceMtx_);
787 6 : exceptionDumper_.CreateExtraDumpPath();
788 12 : return exceptionDumper_.GetExtraDumpCPath();
789 6 : }
790 :
791 36 : const char* DumpManager::GetDataDumpPath()
792 : {
793 36 : std::lock_guard<std::mutex> lk(resourceMtx_);
794 72 : return dumpSetting_.GetDumpCPath();
795 36 : }
796 :
797 9 : int32_t DumpManager::GetExceptionDumpPath(std::string &path)
798 : {
799 9 : std::lock_guard<std::mutex> lk(resourceMtx_);
800 18 : return exceptionDumper_.GetExceptionDumpPath(path);
801 9 : }
802 :
803 15 : int32_t DumpManager::SaveExceptionInfo(const std::string& fileName, const std::string& userTag,
804 : const std::vector<TensorInfo>& tensors)
805 : {
806 15 : std::lock_guard<std::mutex> lk(resourceMtx_);
807 30 : return exceptionDumper_.SaveExceptionInfo(fileName, userTag, tensors);
808 15 : }
809 :
810 27 : int32_t DumpManager::SaveFile(const char* data, size_t dataLen, const char* fileName, SaveType type)
811 : {
812 27 : std::string canonicalPath;
813 54 : if (!Adx::Path::BuildFullPathUnderRoot(opInfoRecordPath_, fileName, canonicalPath)) {
814 12 : IDE_LOGE("invalid fileName[%s], may escape root path.", fileName);
815 12 : return -1;
816 : }
817 15 : int32_t openFlag = 0;
818 15 : if (type == SaveType::OVERWRITE) {
819 9 : openFlag = O_CREAT | O_WRONLY | O_TRUNC;
820 : } else {
821 6 : openFlag = O_CREAT | O_WRONLY | O_APPEND;
822 : }
823 15 : File file(canonicalPath, openFlag);
824 :
825 15 : if (file.IsFileOpen() != 0) {
826 6 : IDE_LOGE("open file[%s] failed!", fileName);
827 6 : return -1;
828 : }
829 9 : int64_t ret = file.Write(data, dataLen);
830 9 : IDE_CTRL_VALUE_FAILED(ret >= 0, return -1, "Save file %s failed!", fileName);
831 :
832 9 : IDE_LOGI("DumpJsonToFile %s success!", fileName);
833 9 : return 0;
834 27 : }
835 :
836 48 : int32_t DumpManager::CallbackEnvExceptionDumpEvent(AdumpCallback callbackFunc)
837 : {
838 48 : if (isEnvExceptionDump_) {
839 0 : IDE_LOGI("Callback module when exception dump enabled with env.");
840 0 : if (exceptionDumper_.GetArgsExceptionStatus() || exceptionDumper_.GetCoredumpStatus()) {
841 0 : return callbackFunc(DUMP_SWITCH_L0_MACK, nullptr, 0);
842 0 : } else if (exceptionDumper_.GetExceptionStatus()) {
843 0 : return callbackFunc(DUMP_SWITCH_L1_MACK, nullptr, 0);
844 : }
845 : }
846 48 : return ADUMP_SUCCESS;
847 : }
848 :
849 : // DUMP 配置变化时，触发dump事件，同步回调用户接口
850 96 : int32_t DumpManager::HandleDumpEvent(uint32_t moduleId, DumpEnableAction action)
851 : {
852 96 : const uint64_t dumpSwitch = dumpSetting_.GetDumpSwitch();
853 96 : auto callbackFunc = disableCallbackFunc_[moduleId];
854 96 : if (action == DumpEnableAction::ENABLE) {
855 27 : callbackFunc = enableCallbackFunc_[moduleId];
856 : // 回调环境变量开启exception dump
857 27 : (void)CallbackEnvExceptionDumpEvent(callbackFunc);
858 : }
859 96 : if (action == DumpEnableAction::AUTO) {
860 : // 回调环境变量开启exception dump
861 18 : (void)CallbackEnvExceptionDumpEvent(enableCallbackFunc_[moduleId]);
862 18 : if (dumpConfigInfo_.data() == nullptr || dumpConfigInfo_.size() == 0U) {
863 3 : IDE_LOGW("Config data is null or empty. Not trigger HandleDumpEvent.");
864 3 : return ADUMP_SUCCESS;
865 : }
866 15 : if (dumpSwitch > 0U) {
867 15 : callbackFunc = enableCallbackFunc_[moduleId];
868 : }
869 : }
870 :
871 93 : if (!callbackFunc) {
872 15 : IDE_LOGE("No registered callback for module %u", moduleId);
873 15 : return ADUMP_FAILED;
874 : }
875 :
876 78 : IDE_LOGI("HandleDumpEvent callbackFunc start for module [%zu]", moduleId);
877 78 : IDE_LOGI("HandleDumpEvent callbackFunc switch [%" PRIu64 "]", dumpSwitch);
878 78 : IDE_LOGI(
879 : "HandleDumpEvent callbackFunc Dump config info: addr=%p, size=%zu", dumpConfigInfo_.data(),
880 : dumpConfigInfo_.size());
881 78 : int32_t result = callbackFunc(dumpSwitch, dumpConfigInfo_.data(), dumpConfigInfo_.size());
882 78 : IDE_LOGI("callbackFunc returned: %d", result);
883 78 : return result;
884 : }
885 :
886 : #ifdef __ADUMP_LLT
887 644 : void DumpManager::Reset()
888 : {
889 644 : registered_ = false;
890 644 : exceptionDumper_.Reset();
891 644 : }
892 :
893 9 : bool DumpManager::GetKFCInitStatus()
894 : {
895 9 : return isKFCInit_;
896 : }
897 :
898 6 : void DumpManager::SetKFCInitStatus(bool status)
899 : {
900 6 : isKFCInit_ = status;
901 6 : }
902 : #endif
903 :
904 18 : int32_t DumpManager::RegisterExceptionDumpCallback(ExceptionDumpCallback callback)
905 : {
906 18 : return exceptionDumper_.RegisterExceptionDumpCallback(callback);
907 : }
908 :
909 18 : int32_t DumpManager::UnregisterExceptionDumpCallback(ExceptionDumpCallback callback)
910 : {
911 18 : return exceptionDumper_.UnregisterExceptionDumpCallback(callback);
912 : }
913 :
914 : } // namespace Adx
|
