Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 : #include "dump_manager.h"
11 : #include <thread>
12 : #include <cctype>
13 : #include <cinttypes>
14 : #include <map>
15 : #include <algorithm>
16 : #include <cerrno>
17 : #include <sstream>
18 : #include "str_utils.h"
19 : #include "lib_path.h"
20 : #include "file_utils.h"
21 : #include "adump_platform_manager.h"
22 : #include "log/adx_log.h"
23 : #include "runtime/context.h"
24 : #include "runtime/config.h"
25 : #include "rts/rts_snapshot.h"
26 : #include "error_codes/rt_error_codes.h"
27 : #include "adump_dsmi.h"
28 : #include "common_utils.h"
29 : #include "exception_info_common.h"
30 : #include "dump_config_converter.h"
31 : #include "adump_api.h"
32 : #include "adump_error_manager.h"
33 : #include "operator_dumper.h"
34 : #include "kernel_dfx_dumper.h"
35 : #include "adx_dump_record.h"
36 : #include "common/file.h"
37 : #include "adx_datadump_server.h"
38 :
39 : namespace Adx {
40 : constexpr char EXCEPTION_CB_MODULE[] = "AdumpException";
41 : constexpr char COREDUMP_CB_MODULE[] = "AdumpCoredump";
42 :
43 : std::vector<std::shared_ptr<OperatorPreliminary>> DumpManager::operatorMap_;
44 :
45 0 : static void ExceptionCallback(rtExceptionInfo* const exception)
46 : {
47 0 : IDE_RUN_LOGI("An exception callback message is received.");
48 0 : if (exception != nullptr) {
49 0 : (void)DumpManager::Instance().DumpExceptionInfo(*exception);
50 0 : AdxLogFlush();
51 : }
52 0 : }
53 :
54 6 : static void NotifyCoredumpCallback(uint32_t devId, bool isOpen)
55 : {
56 6 : static std::map<uint32_t, uint32_t> setDeviceRecord;
57 : static std::mutex coredumpMtx;
58 6 : std::lock_guard<std::mutex> lk(coredumpMtx);
59 6 : auto it = setDeviceRecord.find(devId);
60 6 : if (!isOpen) {
61 2 : if (it != setDeviceRecord.end()) {
62 2 : it->second--;
63 2 : if (it->second == 0) {
64 1 : setDeviceRecord.erase(it);
65 1 : IDE_LOGI("Device %u has been removed from the effective list.", devId);
66 : }
67 : }
68 2 : return;
69 : }
70 4 : if (it != setDeviceRecord.end()) {
71 2 : it->second++;
72 2 : return;
73 : }
74 2 : rtError_t ret = rtDebugSetDumpMode(RT_DEBUG_DUMP_ON_EXCEPTION);
75 2 : if (ret != RT_ERROR_NONE) {
76 0 : IDE_RUN_LOGI("detail exception dump mode not support, switch to lite exception dump, ret:%d", ret);
77 0 : DumpManager::Instance().ExceptionModeDowngrade();
78 : }
79 2 : IDE_LOGI("Device %u has been added to the effective list.", devId);
80 2 : setDeviceRecord[devId] = 1;
81 6 : }
82 :
83 0 : static uint32_t DumpSnapShotLockPreCallback(int32_t deviceId, void* args)
84 : {
85 : UNUSED(deviceId);
86 : UNUSED(args);
87 0 : return DumpManager::Instance().StopDataDumpServer() ? 0U : 1U;
88 : }
89 :
90 1887 : DumpManager& DumpManager::Instance()
91 : {
92 1887 : static DumpManager instance;
93 1887 : return instance;
94 : }
95 :
96 3 : DumpManager::DumpManager()
97 : {
98 : try {
99 : // 1. 通过环境变量使能Exception Dump
100 3 : EnableExceptionDumpWithEnv();
101 : // 2. 通过环境变量使能Kernel Dfx Dump
102 3 : KernelDfxDumper::Instance();
103 0 : } catch (...) {
104 0 : IDE_LOGW("Enable dump function with env variable failed!");
105 0 : }
106 3 : }
107 :
108 258 : bool DumpManager::StartDataDumpServer()
109 : {
110 : // 注册快照回调：快照备份前关闭Data Dump Server
111 258 : RegisterSnapShotCallback();
112 : #if !defined(ADUMP_SOC_HOST) || ADUMP_SOC_HOST == 1
113 : // 如果没有启动Data Dump Server，启动Data Dump Server
114 258 : int32_t dumpNum = AdxDumpRecord::Instance().GetDumpInitNum();
115 258 : if (dumpNum == 0) {
116 39 : return AdxDataDumpServerInit() == ADUMP_SUCCESS;
117 : }
118 : #endif
119 219 : return true;
120 : }
121 :
122 39 : bool DumpManager::StopDataDumpServer()
123 : {
124 : #if !defined(ADUMP_SOC_HOST) || ADUMP_SOC_HOST == 1
125 39 : int32_t dumpNum = AdxDumpRecord::Instance().GetDumpInitNum();
126 78 : while (dumpNum > 0) {
127 39 : IDE_CTRL_VALUE_FAILED(
128 : AdxDataDumpServerUnInit() == ADUMP_SUCCESS, return false, "Stop data dump server failed!");
129 39 : dumpNum = AdxDumpRecord::Instance().GetDumpInitNum();
130 : }
131 : #endif
132 39 : return true;
133 : }
134 :
135 267 : void DumpManager::RegisterSnapShotCallback()
136 : {
137 267 : if (!snapCbkRegistered_) {
138 15 : rtError_t ret = rtSnapShotCallbackRegister(RT_SNAPSHOT_LOCK_PRE, DumpSnapShotLockPreCallback, nullptr);
139 15 : if (ret == ACL_ERROR_RT_FEATURE_NOT_SUPPORT) {
140 3 : IDE_LOGI("RTS does not support snapshot feature. ret=%d", ret);
141 3 : return;
142 : }
143 12 : IDE_CTRL_VALUE_WARN(
144 : ret == ACL_SUCCESS, return, "Register DumpSnapShotLockPreCallback to RTS failed. ret=%d", ret);
145 9 : snapCbkRegistered_ = true;
146 9 : IDE_LOGI("Register DumpSnapShotLockPreCallback success.");
147 : }
148 : }
149 :
150 3 : void DumpManager::EnableExceptionDumpWithEnv()
151 : {
152 3 : DumpConfig config;
153 : DumpType dumpType;
154 3 : if (DumpConfigConverter::EnableExceptionDumpWithEnv(config, dumpType)) {
155 0 : if (SetDumpConfig(dumpType, config) != ADUMP_SUCCESS) {
156 0 : IDE_LOGW("Enable exception dump failed. dumpType: %d", dumpType);
157 : } else {
158 0 : isEnvExceptionDump_ = true;
159 : }
160 : }
161 3 : }
162 :
163 258 : bool DumpManager::AdjustOpExecuteTimeOut()
164 : {
165 258 : if (opTimeoutModified_) {
166 195 : IDE_LOGI("Op execute timeout has been adjusted for data dump, skip.");
167 195 : return true;
168 : }
169 63 : uint32_t curTimeout = 0U;
170 63 : rtError_t ret = rtGetOpExecuteTimeoutV2(&curTimeout);
171 63 : IDE_CTRL_VALUE_FAILED(ret == RT_ERROR_NONE, return false,
172 : "Get op execute timeout failed, skip adjust for data dump. ret=%d", ret);
173 60 : if (curTimeout >= OP_EXECUTE_TIMEOUT_FOR_DATADUMP_MS) {
174 6 : IDE_LOGI("Current op execute timeout %ums is no less than %ums, no need to adjust for data dump.",
175 : curTimeout, OP_EXECUTE_TIMEOUT_FOR_DATADUMP_MS);
176 6 : return true;
177 : }
178 54 : ret = rtSetOpExecuteTimeOutWithMs(OP_EXECUTE_TIMEOUT_FOR_DATADUMP_MS);
179 54 : IDE_CTRL_VALUE_FAILED(ret == RT_ERROR_NONE, return false,
180 : "Set op execute timeout to %ums failed for data dump. ret=%d", OP_EXECUTE_TIMEOUT_FOR_DATADUMP_MS, ret);
181 51 : originOpExecuteTimeOut_ = curTimeout;
182 51 : opTimeoutModified_ = true;
183 51 : IDE_LOGI("Adjust op execute timeout from %ums to %ums for data dump.",
184 : curTimeout, OP_EXECUTE_TIMEOUT_FOR_DATADUMP_MS);
185 51 : return true;
186 : }
187 :
188 36 : bool DumpManager::RestoreOpExecuteTimeOut()
189 : {
190 36 : if (!opTimeoutModified_) {
191 9 : return true;
192 : }
193 27 : rtError_t ret = rtSetOpExecuteTimeOutWithMs(originOpExecuteTimeOut_);
194 : // 恢复失败时保持 opTimeoutModified_=true，以便下次关闭 data dump 时重试恢复
195 27 : IDE_CTRL_VALUE_FAILED(ret == RT_ERROR_NONE, return false,
196 : "Restore op execute timeout to %ums failed. ret=%d", originOpExecuteTimeOut_, ret);
197 24 : IDE_LOGI("Restore op execute timeout to %ums after data dump disabled.", originOpExecuteTimeOut_);
198 24 : opTimeoutModified_ = false;
199 24 : originOpExecuteTimeOut_ = 0U;
200 24 : return true;
201 : }
202 :
203 0 : void DumpManager::KFCResourceInit()
204 : {
205 : #if !defined(ADUMP_SOC_HOST) || ADUMP_SOC_HOST == 1
206 0 : if (isKFCInit_) {
207 0 : IDE_LOGD("KFC resources have been initialized on all devices.");
208 0 : return;
209 : }
210 0 : std::vector<uint32_t> devList = AdumpDsmi::DrvGetDeviceList();
211 :
212 0 : for (uint32_t& deviceId : devList) {
213 0 : IDE_LOGI("Start to initialize KFC resources on device %u.", deviceId);
214 0 : SharedPtr<OperatorPreliminary> opIniter = MakeSharedInstance<OperatorPreliminary>(GetDumpSetting(), deviceId);
215 0 : IDE_CTRL_VALUE_FAILED_NODO(
216 : opIniter != nullptr && opIniter->OperatorInit() == ADUMP_SUCCESS, return,
217 : "Failed to execute the resource initialization task on device %u.", deviceId);
218 0 : DumpManager::operatorMap_.emplace_back(std::move(opIniter));
219 0 : IDE_LOGI("KFC executed on the device %u successfully.", deviceId);
220 0 : }
221 : #endif
222 0 : isKFCInit_ = true;
223 0 : }
224 :
225 54 : int32_t DumpManager::ExceptionConfig(DumpType dumpType, const DumpConfig& dumpConfig)
226 : {
227 54 : if (exceptionDumper_.IsRepeatEnableException(dumpType, dumpConfig)) {
228 3 : IDE_LOGW(
229 : "Exception dump has been enabled, not support enable exception dump[%s] again",
230 : DumpConfigConverter::DumpTypeToStr(dumpType).c_str());
231 3 : return ADUMP_SUCCESS;
232 : }
233 :
234 51 : if (dumpType == DumpType::AIC_ERR_DETAIL_DUMP && !CheckCoredumpSupportedPlatform()) {
235 3 : IDE_LOGE("Current platform is not support coredump mode.");
236 3 : return ADUMP_FAILED;
237 : }
238 :
239 48 : IDE_RUN_LOGI(
240 : "Set %s[%d] dump setting, status: %s, dump switch: %llu", DumpConfigConverter::DumpTypeToStr(dumpType).c_str(),
241 : dumpType, dumpConfig.dumpStatus.c_str(), dumpConfig.dumpSwitch);
242 48 : if (exceptionDumper_.ExceptionDumperInit(dumpType, dumpConfig) != ADUMP_SUCCESS) {
243 1 : IDE_LOGW("Failed to initialize the exception dump.");
244 1 : return ADUMP_SUCCESS;
245 : }
246 47 : dumpSetting_.InitDumpSwitch(dumpConfig.dumpSwitch & DUMP_SWITCH_MASK);
247 47 : IDE_CTRL_VALUE_WARN(
248 : RegsiterExceptionCallback(), return ADUMP_SUCCESS,
249 : "Failed to register the args exception dump callback function.");
250 46 : if (exceptionDumper_.GetCoredumpStatus()) { // 如果启动了coredump模式
251 1 : IDE_CTRL_VALUE_FAILED(
252 : rtRegDeviceStateCallbackEx(COREDUMP_CB_MODULE, &NotifyCoredumpCallback, DEV_CB_POS_BACK) == RT_ERROR_NONE,
253 : return ADUMP_FAILED, "Failed to register the coredump callback function to rtSetDevice.");
254 : }
255 46 : return ADUMP_SUCCESS;
256 : }
257 :
258 399 : int32_t DumpManager::SetDumpConfig(DumpType dumpType, const DumpConfig& dumpConfig)
259 : {
260 399 : std::lock_guard<std::mutex> lk(resourceMtx_);
261 399 : if (dumpType == DumpType::EXCEPTION || dumpType == DumpType::ARGS_EXCEPTION ||
262 : dumpType == DumpType::AIC_ERR_DETAIL_DUMP) {
263 54 : return ExceptionConfig(dumpType, dumpConfig);
264 : }
265 345 : auto ret = dumpSetting_.Init(dumpType, dumpConfig);
266 345 : if (ret != ADUMP_SUCCESS) {
267 21 : return ret;
268 : }
269 :
270 : // 启动Data Dump Server
271 324 : if (dumpConfig.dumpStatus != ADUMP_DUMP_STATUS_SWITCH_OFF) {
272 258 : IDE_CTRL_VALUE_FAILED(StartDataDumpServer(), return ADUMP_FAILED,
273 : "Start data dump server failed! dumpType=%s[%d]",
274 : DumpConfigConverter::DumpTypeToStr(dumpType).c_str(), dumpType);
275 : // 后续失败不恢复超时时间，不影响功能。
276 258 : IDE_CTRL_VALUE_FAILED(AdjustOpExecuteTimeOut(), return ADUMP_FAILED, "Adjust op execute timeout failed.");
277 : }
278 :
279 318 : if (CheckBinValidation() && (isKFCInit_ == false)) {
280 0 : auto kfcBind = std::bind(&DumpManager::KFCResourceInit, this);
281 0 : std::thread kfcThread(kfcBind);
282 0 : kfcThread.join();
283 0 : if (!isKFCInit_) {
284 0 : IDE_LOGE("SetDumpConfig failed due to kfc resource initialization error.");
285 0 : DumpManager::operatorMap_.clear();
286 0 : return ADUMP_FAILED;
287 : }
288 0 : }
289 :
290 318 : ret = OperatorDumper(dumpSetting_).UpdateDevMemCache();
291 318 : IDE_CTRL_VALUE_FAILED(ret == ADUMP_SUCCESS, return ADUMP_FAILED,
292 : "Update device memery cache for data dump failed! dumpType=%s[%d]",
293 : DumpConfigConverter::DumpTypeToStr(dumpType).c_str(), dumpType);
294 :
295 318 : IDE_RUN_LOGI(
296 : "Set %s[%d] dump setting, status: %s, mode: %s, data: %s, dump switch: %llu, path:%s, dump stats:%s.",
297 : DumpConfigConverter::DumpTypeToStr(dumpType).c_str(), dumpType, dumpConfig.dumpStatus.c_str(),
298 : dumpConfig.dumpMode.c_str(), dumpConfig.dumpData.c_str(), dumpConfig.dumpSwitch, dumpConfig.dumpPath.c_str(),
299 : StrUtils::ToString(dumpConfig.dumpStatsItem).c_str());
300 318 : return ADUMP_SUCCESS;
301 399 : }
302 :
303 111 : int32_t DumpManager::SetDumpConfig(const char *dumpConfigData, size_t dumpConfigSize, const char *dumpConfigPath)
304 : {
305 111 : std::lock_guard<std::mutex> lk(resourceMtx2_);
306 111 : if ((dumpConfigData == nullptr) || (dumpConfigSize == 0U) || (dumpConfigPath == nullptr)) {
307 24 : IDE_LOGE("Set dump config failed. Config data is null or empty.");
308 24 : return ADUMP_FAILED;
309 : }
310 87 : DumpConfig dumpConfig;
311 87 : DumpDfxConfig dumpDfxConfig;
312 : DumpType dumpType;
313 87 : bool needDump = true;
314 87 : DumpConfigConverter converter{dumpConfigData, dumpConfigSize, dumpConfigPath};
315 87 : int32_t ret = converter.Convert(dumpType, dumpConfig, needDump, dumpDfxConfig);
316 87 : if (ret != ADUMP_SUCCESS) {
317 12 : IDE_LOGE("Parse dump config from memory[%s] failed.", dumpConfigData);
318 12 : return ADUMP_INPUT_FAILED;
319 : }
320 :
321 : // 开启KernelDataDump
322 75 : ret = KernelDfxDumper::Instance().EnableDfxDumper(dumpDfxConfig);
323 75 : IDE_CTRL_VALUE_FAILED(ret == ADUMP_SUCCESS, return ret, "Enable kernel dfx dump failed.");
324 :
325 75 : if (!needDump) {
326 9 : return ADUMP_SUCCESS;
327 : }
328 :
329 : // 已开启exception dump：不支持重复使能，不更新配置缓存，不重复回调注册模块组件
330 66 : if (exceptionDumper_.IsRepeatEnableException(dumpType, dumpConfig)) {
331 0 : IDE_LOGW(
332 : "Exception dump has been enabled, not support enable exception dump[%s] again",
333 : DumpConfigConverter::DumpTypeToStr(dumpType).c_str());
334 0 : return ADUMP_SUCCESS;
335 : }
336 :
337 66 : (void)dumpConfigInfo_.assign(dumpConfigData, dumpConfigSize);
338 66 : IDE_LOGI("Dump config info set: addr=%p, size=%zu", dumpConfigInfo_.data(), dumpConfigInfo_.size());
339 66 : ret = SetDumpConfig(dumpType, dumpConfig);
340 : // 同步触发callback事件
341 93 : for (auto& item : enableCallbackFunc_) {
342 27 : IDE_LOGI("SetDumpConfig HandleDumpEvent start for module [%zu]", item.first);
343 27 : HandleDumpEvent(item.first, DumpEnableAction::ENABLE);
344 : }
345 66 : IDE_CTRL_VALUE_FAILED(ret == ADUMP_SUCCESS, return ret, "Set dump config failed.");
346 60 : (void)openedDump_.insert(dumpType);
347 60 : return ADUMP_SUCCESS;
348 111 : }
349 :
350 42 : int32_t DumpManager::UnSetDumpConfig()
351 : {
352 42 : std::lock_guard<std::mutex> lk(resourceMtx2_);
353 42 : DumpConfig config;
354 42 : config.dumpStatus = ADUMP_DUMP_STATUS_SWITCH_OFF;
355 42 : config.dumpSwitch = 0;
356 66 : for (const auto dumpType : openedDump_) {
357 30 : if (IsEnableDump(dumpType)) {
358 30 : const auto ret = SetDumpConfig(dumpType, config);
359 30 : IDE_CTRL_VALUE_FAILED(
360 : ret == ADUMP_SUCCESS, return ADUMP_FAILED, "[Set][Dump]Set dump off failed! dumpType=%s[%d], ret=%d",
361 : DumpConfigConverter::DumpTypeToStr(dumpType).c_str(), dumpType, ret);
362 24 : IDE_LOGI(
363 : "[Set][Dump]Set dump off successfully, dumpType=%s[%d]",
364 : DumpConfigConverter::DumpTypeToStr(dumpType).c_str(), dumpType);
365 : }
366 : }
367 36 : openedDump_.clear();
368 : // 同步触发callback事件
369 81 : for (auto& item : disableCallbackFunc_) {
370 45 : IDE_LOGI("UnSetDumpConfig start for module [%zu]", item.first);
371 45 : HandleDumpEvent(item.first, DumpEnableAction::DISABLE);
372 : }
373 36 : dumpConfigInfo_.clear();
374 36 : IDE_LOGI("Dump config info cleared.");
375 :
376 : // 等待所有 dump 操作完成并清理资源
377 36 : DumpResourceSafeMap::Instance().waitAndClear();
378 : // 释放device资源
379 36 : OperatorDumper::FreeDevMemCache();
380 : // 停止data dump server
381 36 : IDE_CTRL_VALUE_FAILED(StopDataDumpServer(), return ADUMP_FAILED, "Stop data dump server failed!");
382 : // 恢复算子超时时间
383 36 : IDE_CTRL_VALUE_FAILED(RestoreOpExecuteTimeOut(), return ADUMP_FAILED, "Restore op execute timeout failed!");
384 :
385 33 : return ADUMP_SUCCESS;
386 42 : }
387 :
388 321 : std::string DumpManager::GetBinName() const
389 : {
390 321 : auto plat = PlatformReflection<DataDumpInterface>::CreatePlatform(dumpSetting_.GetPlatformType());
391 321 : if (plat == nullptr) {
392 642 : return "";
393 : }
394 0 : return plat->GetKfcBinName();
395 321 : }
396 :
397 321 : bool DumpManager::CheckBinValidation()
398 : {
399 321 : const std::string opName = GetBinName();
400 321 : if ((dumpSetting_.GetDumpData().compare(DUMP_STATS_DATA) != 0) || opName.empty()) {
401 321 : return false;
402 : }
403 0 : const std::string opPath = LibPath::Instance().GetTargetPath(opName);
404 0 : IDE_CTRL_VALUE_FAILED(!opPath.empty(), return false, "Received an empty path for file %s.", opName.c_str());
405 0 : bool isExist = FileUtils::IsFileExist(opPath);
406 0 : IDE_LOGI("CheckBinValidation result is %s", isExist ? "true" : "false");
407 0 : return isExist;
408 321 : }
409 :
410 161 : bool DumpManager::IsEnableDump(DumpType dumpType)
411 : {
412 161 : std::lock_guard<std::mutex> lk(resourceMtx_);
413 161 : if (dumpType == DumpType::ARGS_EXCEPTION) {
414 38 : return exceptionDumper_.GetArgsExceptionStatus() || exceptionDumper_.GetCoredumpStatus();
415 123 : } else if (dumpType == DumpType::OPERATOR) {
416 93 : return dumpSetting_.GetDumpStatus();
417 30 : } else if (dumpType == DumpType::OP_OVERFLOW) {
418 21 : return dumpSetting_.GetDumpDebugStatus();
419 9 : } else if (dumpType == DumpType::EXCEPTION) {
420 3 : return exceptionDumper_.GetExceptionStatus();
421 6 : } else if (dumpType == DumpType::AIC_ERR_DETAIL_DUMP) {
422 3 : return exceptionDumper_.GetCoredumpStatus();
423 : } else {
424 3 : IDE_LOGW("Dump type is not support.");
425 : }
426 :
427 3 : return false;
428 161 : }
429 :
430 39 : int32_t DumpManager::GetInputOutputTensors(
431 : const std::string& opType, const std::string& opName, const std::vector<TensorInfoV2>& tensors,
432 : std::vector<DumpTensor>& inputTensors, std::vector<DumpTensor>& outputTensors)
433 : {
434 75 : for (const auto& tensorInfo : tensors) {
435 36 : if (tensorInfo.tensorAddr == nullptr || tensorInfo.tensorSize == 0) {
436 6 : IDE_LOGW("Tensor of op=%s[%s] is empty, addr=%p, size=%zu, skip it.",
437 : opName.c_str(), opType.c_str(), tensorInfo.tensorAddr, tensorInfo.tensorSize);
438 6 : continue;
439 : }
440 :
441 30 : if (tensorInfo.placement != TensorPlacement::kOnDeviceHbm) {
442 6 : IDE_LOGW("Tensor of op=%s[%s] is not on device, skip it.", opName.c_str(), opType.c_str());
443 6 : continue;
444 : }
445 :
446 24 : if (tensorInfo.type == TensorType::INPUT) {
447 21 : inputTensors.emplace_back(tensorInfo);
448 3 : } else if (tensorInfo.type == TensorType::OUTPUT) {
449 3 : outputTensors.emplace_back(tensorInfo);
450 : }
451 : }
452 39 : return ADUMP_SUCCESS;
453 : }
454 :
455 24 : bool DumpManager::IsEnableDumpOperatorWithCapture(const std::string& opType, const std::string& opName,
456 : aclrtStream stream)
457 : {
458 24 : rtStreamCaptureStatus status = RT_STREAM_CAPTURE_STATUS_MAX;
459 24 : rtModel_t* captureMdl = nullptr;
460 24 : int32_t ret = rtStreamGetCaptureInfo(stream, &status, captureMdl);
461 24 : if (ret != ACL_SUCCESS) {
462 6 : IDE_LOGW("Get stream capture info error: %d, will switch to common dump", ret);
463 6 : return false;
464 : }
465 18 : IDE_LOGI("%s[%s] : stream capture status: %d", opName.c_str(), opType.c_str(), status);
466 18 : return status == RT_STREAM_CAPTURE_STATUS_ACTIVE;
467 : }
468 :
469 27 : int32_t DumpManager::DumpOperatorWithCfg(const std::string &opType, const std::string &opName,
470 : const std::vector<TensorInfo> &tensors, aclrtStream stream, const DumpCfg &dumpCfg)
471 : {
472 27 : std::lock_guard<std::mutex> lk(resourceMtx_);
473 27 : if (!dumpSetting_.GetDumpStatusEx() && !dumpSetting_.GetDumpDebugStatus()) {
474 3 : IDE_LOGW("Operator or overflow dump is not enable, can't dump data.");
475 3 : return ADUMP_SUCCESS;
476 : }
477 :
478 24 : bool isInvalid = dumpCfg.numAttrs != 0UL && dumpCfg.attrs == nullptr;
479 24 : IDE_CTRL_VALUE_FAILED(!isInvalid, return ADUMP_FAILED,
480 : "The dump cfg attrs is null pointer! op=%s[%s].", opName.c_str(), opType.c_str());
481 :
482 21 : std::vector<DumpTensor> inputTensors;
483 21 : std::vector<DumpTensor> outputTensors;
484 21 : int32_t ret = GetInputOutputTensors(opType, opName, ConvertTensorInfoToDumpTensorV2(tensors),
485 : inputTensors, outputTensors);
486 21 : IDE_CTRL_VALUE_FAILED(ret == ADUMP_SUCCESS, return ADUMP_FAILED,
487 : "Get input and output tensors failed! op=%s[%s].", opName.c_str(), opType.c_str());
488 21 : IDE_CTRL_VALUE_WARN(!inputTensors.empty() || !outputTensors.empty(), return ADUMP_SUCCESS,
489 : "No tensor need to dump. op=%s[%s].", opName.c_str(), opType.c_str());
490 :
491 6 : if (IsEnableDumpOperatorWithCapture(opType, opName, stream)) {
492 3 : if (dumpSetting_.GetDumpDebugStatus() || dumpSetting_.IsDumpDataStats()) {
493 0 : IDE_LOGI("overflow or stats is not allow in capture stream");
494 0 : return ADUMP_SUCCESS;
495 : }
496 3 : return DumpOperatorWithCapture(opType, opName, inputTensors, outputTensors, stream);
497 : }
498 :
499 3 : OperatorDumper opDumper(opType, opName);
500 3 : ret = opDumper.SetDumpSetting(dumpSetting_)
501 3 : .RuntimeStream(stream)
502 3 : .InputDumpTensor(inputTensors)
503 3 : .OutputDumpTensor(outputTensors)
504 3 : .LaunchWithCfg(dumpCfg);
505 3 : IDE_CTRL_VALUE_FAILED(ret == ADUMP_SUCCESS, return ret,
506 : "Launch dump operator with dump cfg failed! op=%s[%s].", opName.c_str(), opType.c_str());
507 0 : return ADUMP_SUCCESS;
508 27 : }
509 :
510 6 : int32_t DumpManager::DumpOperator(
511 : const std::string& opType, const std::string& opName, const std::vector<TensorInfo>& tensors, aclrtStream stream)
512 : {
513 6 : return DumpOperatorV2(opType, opName, ConvertTensorInfoToDumpTensorV2(tensors), stream);
514 : }
515 :
516 24 : int32_t DumpManager::DumpOperatorV2(
517 : const std::string& opType, const std::string& opName, const std::vector<TensorInfoV2>& tensors, aclrtStream stream)
518 : {
519 24 : std::lock_guard<std::mutex> lk(resourceMtx_);
520 24 : if (!dumpSetting_.GetDumpStatus() && !dumpSetting_.GetDumpDebugStatus()) {
521 6 : IDE_LOGW("Operator or overflow dump is not enable, can't dump.");
522 6 : return ADUMP_SUCCESS;
523 : }
524 :
525 18 : std::vector<DumpTensor> inputTensors;
526 18 : std::vector<DumpTensor> outputTensors;
527 18 : int32_t ret = GetInputOutputTensors(opType, opName, tensors, inputTensors, outputTensors);
528 18 : IDE_CTRL_VALUE_FAILED(ret == ADUMP_SUCCESS, return ADUMP_FAILED,
529 : "Get input and output tensors failed! opName: %s, opType: %s", opName.c_str(), opType.c_str());
530 :
531 18 : if (IsEnableDumpOperatorWithCapture(opType, opName, stream)) {
532 12 : if (dumpSetting_.GetDumpDebugStatus() || dumpSetting_.IsDumpDataStats()) {
533 9 : IDE_LOGI("overflow or stats is not allow in capture stream");
534 9 : return ADUMP_SUCCESS;
535 : }
536 3 : return DumpOperatorWithCapture(opType, opName, inputTensors, outputTensors, stream);
537 : }
538 :
539 6 : OperatorDumper opDumper(opType, opName);
540 6 : ret = opDumper.SetDumpSetting(dumpSetting_)
541 6 : .RuntimeStream(stream)
542 6 : .InputDumpTensor(inputTensors)
543 6 : .OutputDumpTensor(outputTensors)
544 6 : .Launch();
545 6 : IDE_CTRL_VALUE_FAILED(ret == ADUMP_SUCCESS, return ret,
546 : "Launch dump operator failed! op=%s[%s].", opName.c_str(), opType.c_str());
547 3 : return ADUMP_SUCCESS;
548 24 : }
549 :
550 9 : int32_t DumpManager::DumpOperatorWithCapture(
551 : const std::string& opType, const std::string& opName, const std::vector<DumpTensor>& inputTensors,
552 : const std::vector<DumpTensor>& outputTensors, aclrtStream mainStream)
553 : {
554 9 : if (mainStream == nullptr) {
555 9 : IDE_LOGE("mainStream is nullptr.");
556 9 : return ADUMP_FAILED;
557 : }
558 :
559 0 : if (!isCaptureDumpServerInit_) {
560 0 : IDE_CTRL_VALUE_FAILED(StartDataDumpServer(), return ADUMP_FAILED, "Start data dump server failed!");
561 0 : isCaptureDumpServerInit_ = true;
562 : }
563 :
564 0 : uint32_t streamId = 0;
565 0 : uint32_t taskId = 0;
566 0 : uint32_t deviceId = 0;
567 0 : std::string dumpPath;
568 0 : int32_t ret = CollectStreamContextInfo(mainStream, opName, opType, streamId, taskId, deviceId, dumpPath);
569 0 : if (ret != ADUMP_SUCCESS) {
570 0 : IDE_LOGE("%s(%s) collect stream context info failed.", opName.c_str(), opType.c_str());
571 0 : return ret;
572 : }
573 :
574 0 : std::string mainStreamKey = std::to_string(streamId) + "_" + std::to_string(taskId);
575 0 : DumpInfoParams params = {
576 0 : mainStreamKey, inputTensors, outputTensors, opType, opName, streamId, taskId, deviceId, 0, 0, dumpPath};
577 0 : ret = GetDumpInfoFromMap(params);
578 0 : std::shared_ptr<DumpStreamInfo> dumpInfoPtr = DumpResourceSafeMap::Instance().get(mainStreamKey);
579 0 : if (ret != ADUMP_SUCCESS || dumpInfoPtr == nullptr) {
580 0 : IDE_LOGE("%s(%s) get dump info failed.", opName.c_str(), opType.c_str());
581 0 : return ADUMP_FAILED;
582 : }
583 0 : IDE_LOGI("%s(%s) dump data : create DumpStreamInfo success", opName.c_str(), opType.c_str());
584 :
585 0 : ret = SetupAsyncDump(dumpInfoPtr, opName, opType, mainStream);
586 0 : if (ret != ADUMP_SUCCESS) {
587 0 : return ret;
588 : }
589 0 : IDE_LOGI("%s(%s) set main stream %u, dump stream %u, callback function success",
590 : opName.c_str(), opType.c_str(), dumpInfoPtr->streamId, dumpInfoPtr->dumpStmId);
591 :
592 0 : return ADUMP_SUCCESS;
593 0 : }
594 :
595 4 : void DumpManager::AddExceptionOp(const OperatorInfo& opInfo)
596 : {
597 4 : exceptionDumper_.AddDumpOperator(opInfo);
598 4 : }
599 :
600 4 : void DumpManager::AddExceptionOpV2(const OperatorInfoV2& opInfo)
601 : {
602 4 : exceptionDumper_.AddDumpOperatorV2(opInfo);
603 4 : }
604 :
605 4 : void DumpManager::ConvertOperatorInfo(const OperatorInfo& opInfo, OperatorInfoV2& operatorInfoV2) const
606 : {
607 4 : operatorInfoV2.agingFlag = opInfo.agingFlag;
608 4 : operatorInfoV2.taskId = opInfo.taskId;
609 4 : operatorInfoV2.streamId = opInfo.streamId;
610 4 : operatorInfoV2.deviceId = opInfo.deviceId;
611 4 : operatorInfoV2.contextId = opInfo.contextId;
612 4 : operatorInfoV2.opType = opInfo.opType;
613 4 : operatorInfoV2.opName = opInfo.opName;
614 4 : operatorInfoV2.tensorInfos = ConvertTensorInfoToDumpTensorV2(opInfo.tensorInfos);
615 4 : operatorInfoV2.deviceInfos = opInfo.deviceInfos;
616 4 : operatorInfoV2.additionalInfo = opInfo.additionalInfo;
617 4 : }
618 :
619 31 : std::vector<TensorInfoV2> DumpManager::ConvertTensorInfoToDumpTensorV2(const std::vector<TensorInfo>& tensorInfos) const
620 : {
621 31 : std::vector<TensorInfoV2> tensors;
622 31 : tensors.reserve(tensorInfos.size());
623 55 : for (const auto& tensorInfo : tensorInfos) {
624 24 : TensorInfoV2 tensor = {};
625 24 : ConvertTensorInfo(tensorInfo, tensor);
626 24 : tensors.emplace_back(tensor);
627 24 : }
628 31 : return tensors;
629 0 : }
630 :
631 96 : void DumpManager::ConvertTensorInfo(const TensorInfo& tensorInfo, TensorInfoV2& tensor) const
632 : {
633 96 : tensor.dataType = tensorInfo.dataType;
634 96 : tensor.format = tensorInfo.format;
635 96 : tensor.placement = tensorInfo.placement;
636 96 : tensor.tensorAddr = tensorInfo.tensorAddr;
637 96 : tensor.tensorSize = tensorInfo.tensorSize;
638 96 : tensor.type = tensorInfo.type;
639 96 : tensor.addrType = tensorInfo.addrType;
640 96 : tensor.argsOffSet = tensorInfo.argsOffSet;
641 96 : std::vector<int64_t> shape = tensorInfo.shape;
642 171 : for (auto dim : shape) {
643 75 : tensor.shape.emplace_back(static_cast<uint64_t>(dim));
644 : }
645 96 : std::vector<int64_t> originShape = tensorInfo.originShape;
646 171 : for (auto dim : originShape) {
647 75 : tensor.originShape.emplace_back(static_cast<uint64_t>(dim));
648 : }
649 96 : }
650 :
651 6 : int32_t DumpManager::DelExceptionOp(uint32_t deviceId, uint32_t streamId)
652 : {
653 6 : return exceptionDumper_.DelDumpOperator(deviceId, streamId);
654 : }
655 :
656 52 : int32_t DumpManager::DumpExceptionInfo(const rtExceptionInfo& exception)
657 : {
658 52 : return exceptionDumper_.DumpException(exception);
659 : }
660 :
661 17 : uint64_t DumpManager::AdumpGetDumpSwitch()
662 : {
663 17 : std::lock_guard<std::mutex> lk(resourceMtx_);
664 34 : return dumpSetting_.GetDumpSwitch();
665 17 : }
666 :
667 47 : bool DumpManager::RegsiterExceptionCallback()
668 : {
669 47 : if (!registered_ && rtRegTaskFailCallbackByModule(EXCEPTION_CB_MODULE, ExceptionCallback) == RT_ERROR_NONE) {
670 37 : registered_ = true;
671 : }
672 47 : IDE_LOGI("Register exception callback, registered: %d", static_cast<int32_t>(registered_));
673 47 : return registered_;
674 : }
675 :
676 24 : DumpSetting DumpManager::GetDumpSetting() const
677 : {
678 24 : return dumpSetting_;
679 : }
680 :
681 3 : void DumpManager::ExceptionModeDowngrade()
682 : {
683 3 : exceptionDumper_.ExceptionModeDowngrade();
684 3 : }
685 :
686 30 : int32_t DumpManager::RegisterCallback(uint32_t moduleId, AdumpCallback enableFunc, AdumpCallback disableFunc)
687 : {
688 30 : if (enableFunc == nullptr) {
689 6 : IDE_LOGE("Register callback failed: enableFunc is null for module %u", moduleId);
690 6 : return ADUMP_FAILED;
691 : }
692 24 : if (disableFunc == nullptr) {
693 6 : IDE_LOGE("Register callback failed: disableFunc is null for module %u", moduleId);
694 6 : return ADUMP_FAILED;
695 : }
696 18 : std::lock_guard<std::mutex> lk(resourceMtx2_);
697 18 : enableCallbackFunc_[moduleId] = enableFunc;
698 18 : disableCallbackFunc_[moduleId] = disableFunc;
699 18 : IDE_LOGI("Registered callback for module %u", moduleId);
700 18 : return HandleDumpEvent(moduleId, DumpEnableAction::AUTO);
701 18 : }
702 :
703 33 : int32_t DumpManager::StartDumpArgs(const std::string& dumpPath)
704 : {
705 33 : uint64_t dumpSwitch = 0;
706 : {
707 33 : std::lock_guard<std::mutex> lk(resourceMtx_);
708 33 : dumpSwitch = dumpSetting_.GetDumpSwitch();
709 33 : if ((dumpSwitch & OP_INFO_RECORD_DUMP) == OP_INFO_RECORD_DUMP) {
710 162 : REPORT_EP0008_API_CALL_SEQUENCE(
711 : FUNC_NAME_ACL_OP_START_DUMP_ARGS, ADUMP_REASON_API_CALLED_REPEATEDLY);
712 9 : return -1;
713 : }
714 :
715 24 : Adx::Path path(dumpPath);
716 24 : if (path.Empty()) {
717 72 : REPORT_EP0006_INVALID_ARGUMENT(
718 : FUNC_NAME_ACL_OP_START_DUMP_ARGS, dumpPath, FUNC_ACL_OP_START_DUMP_ARGS_PARAM_PATH,
719 : ADUMP_REASON_PARAM_PATH_EMPTY);
720 3 : return -1;
721 : }
722 21 : if (!path.Exist()) {
723 15 : if (!path.CreateDirectory(true)) {
724 3 : std::string reason = StrUtils::Format(ADUMP_REASON_PARAM_PATH_CREATE_DIR_ERROR, strerror(errno));
725 69 : REPORT_EP0006_INVALID_ARGUMENT(
726 : FUNC_NAME_ACL_OP_START_DUMP_ARGS, dumpPath, FUNC_ACL_OP_START_DUMP_ARGS_PARAM_PATH, reason);
727 3 : return -1;
728 3 : }
729 : }
730 18 : if (!path.IsDirectory()) {
731 72 : REPORT_EP0006_INVALID_ARGUMENT(
732 : FUNC_NAME_ACL_OP_START_DUMP_ARGS, dumpPath, FUNC_ACL_OP_START_DUMP_ARGS_PARAM_PATH,
733 : ADUMP_REASON_PARAM_PATH_NOT_DIRECTORY);
734 3 : return -1;
735 : }
736 15 : constexpr uint32_t accessMode = static_cast<uint32_t>(M_R_OK) | static_cast<uint32_t>(M_W_OK);
737 15 : if (!path.Asccess(accessMode)) {
738 72 : REPORT_EP0006_INVALID_ARGUMENT(
739 : FUNC_NAME_ACL_OP_START_DUMP_ARGS, dumpPath, FUNC_ACL_OP_START_DUMP_ARGS_PARAM_PATH,
740 : ADUMP_REASON_PATH_NO_PERMISSION);
741 3 : return -1;
742 : }
743 :
744 12 : dumpSwitch |= OP_INFO_RECORD_DUMP;
745 12 : dumpSetting_.InitDumpSwitch(dumpSwitch);
746 12 : opInfoRecordPath_ = path.GetString();
747 45 : }
748 33 : for (auto& item : enableCallbackFunc_) {
749 21 : item.second(dumpSwitch, dumpConfigInfo_.data(), dumpConfigInfo_.size());
750 : }
751 12 : IDE_RUN_LOGI("OpInfoRecord start success!");
752 12 : return 0;
753 24 : }
754 :
755 15 : int32_t DumpManager::StopDumpArgs()
756 : {
757 15 : uint64_t dumpSwitch = 0;
758 : {
759 15 : std::lock_guard<std::mutex> lk(resourceMtx_);
760 15 : dumpSwitch = dumpSetting_.GetDumpSwitch();
761 15 : if ((dumpSwitch & OP_INFO_RECORD_DUMP) != OP_INFO_RECORD_DUMP) {
762 6 : return 0;
763 : }
764 9 : IDE_RUN_LOGI("OpInfoRecord Stop Entry!");
765 9 : dumpSwitch &= ~OP_INFO_RECORD_DUMP;
766 9 : dumpSetting_.InitDumpSwitch(dumpSwitch);
767 15 : }
768 21 : for (auto& item : disableCallbackFunc_) {
769 12 : item.second(dumpSwitch, dumpConfigInfo_.data(), dumpConfigInfo_.size());
770 : }
771 9 : IDE_RUN_LOGI("OpInfoRecord success!");
772 9 : return 0;
773 : }
774 :
775 6 : const char* DumpManager::GetExceptionDumpPath()
776 : {
777 6 : std::lock_guard<std::mutex> lk(resourceMtx_);
778 6 : exceptionDumper_.CreateExtraDumpPath();
779 12 : return exceptionDumper_.GetExtraDumpCPath();
780 6 : }
781 :
782 36 : const char* DumpManager::GetDataDumpPath()
783 : {
784 36 : std::lock_guard<std::mutex> lk(resourceMtx_);
785 72 : return dumpSetting_.GetDumpCPath();
786 36 : }
787 :
788 27 : int32_t DumpManager::SaveFile(const char* data, size_t dataLen, const char* fileName, SaveType type)
789 : {
790 27 : Adx::Path filePath(opInfoRecordPath_);
791 27 : filePath.Concat(fileName);
792 27 : Adx::Path dirPath = filePath.ParentPath();
793 27 : if (!dirPath.Exist()) {
794 12 : if (!dirPath.CreateDirectory(true)) {
795 0 : IDE_LOGE("create directory[%s] failed", dirPath.GetCString());
796 0 : return -1;
797 : }
798 : }
799 27 : if (!dirPath.RealPath()) {
800 0 : IDE_LOGE("get real path failed, path:%s", dirPath.GetCString());
801 : }
802 :
803 27 : std::string canonicalPath = dirPath.GetString() + "/" + filePath.GetFileName();
804 27 : int32_t openFlag = 0;
805 27 : if (type == SaveType::OVERWRITE) {
806 15 : openFlag = O_CREAT | O_WRONLY | O_TRUNC;
807 : } else {
808 12 : openFlag = O_CREAT | O_WRONLY | O_APPEND;
809 : }
810 27 : File file(canonicalPath, openFlag);
811 :
812 27 : if (file.IsFileOpen() != 0) {
813 6 : IDE_LOGE("open file[%s] failed!", fileName);
814 6 : return -1;
815 : }
816 21 : int64_t ret = file.Write(data, dataLen);
817 21 : IDE_CTRL_VALUE_FAILED(ret >= 0, return -1, "Save file %s failed!", fileName);
818 :
819 21 : IDE_LOGI("DumpJsonToFile %s success!", fileName);
820 21 : return 0;
821 27 : }
822 :
823 48 : int32_t DumpManager::CallbackEnvExceptionDumpEvent(AdumpCallback callbackFunc)
824 : {
825 48 : if (isEnvExceptionDump_) {
826 0 : IDE_LOGI("Callback module when exception dump enabled with env.");
827 0 : if (exceptionDumper_.GetArgsExceptionStatus() || exceptionDumper_.GetCoredumpStatus()) {
828 0 : return callbackFunc(DUMP_SWITCH_L0_MACK, nullptr, 0);
829 0 : } else if (exceptionDumper_.GetExceptionStatus()) {
830 0 : return callbackFunc(DUMP_SWITCH_L1_MACK, nullptr, 0);
831 : }
832 : }
833 48 : return ADUMP_SUCCESS;
834 : }
835 :
836 : // DUMP 配置变化时，触发dump事件，同步回调用户接口
837 96 : int32_t DumpManager::HandleDumpEvent(uint32_t moduleId, DumpEnableAction action)
838 : {
839 96 : const uint64_t dumpSwitch = dumpSetting_.GetDumpSwitch();
840 96 : auto callbackFunc = disableCallbackFunc_[moduleId];
841 96 : if (action == DumpEnableAction::ENABLE) {
842 27 : callbackFunc = enableCallbackFunc_[moduleId];
843 : // 回调环境变量开启exception dump
844 27 : (void)CallbackEnvExceptionDumpEvent(callbackFunc);
845 : }
846 96 : if (action == DumpEnableAction::AUTO) {
847 : // 回调环境变量开启exception dump
848 18 : (void)CallbackEnvExceptionDumpEvent(enableCallbackFunc_[moduleId]);
849 18 : if (dumpConfigInfo_.data() == nullptr || dumpConfigInfo_.size() == 0U) {
850 3 : IDE_LOGW("Config data is null or empty. Not trigger HandleDumpEvent.");
851 3 : return ADUMP_SUCCESS;
852 : }
853 15 : if (dumpSwitch > 0U) {
854 15 : callbackFunc = enableCallbackFunc_[moduleId];
855 : }
856 : }
857 :
858 93 : if (!callbackFunc) {
859 15 : IDE_LOGE("No registered callback for module %u", moduleId);
860 15 : return ADUMP_FAILED;
861 : }
862 :
863 78 : IDE_LOGI("HandleDumpEvent callbackFunc start for module [%zu]", moduleId);
864 78 : IDE_LOGI("HandleDumpEvent callbackFunc switch [%" PRIu64 "]", dumpSwitch);
865 78 : IDE_LOGI(
866 : "HandleDumpEvent callbackFunc Dump config info: addr=%p, size=%zu", dumpConfigInfo_.data(),
867 : dumpConfigInfo_.size());
868 78 : int32_t result = callbackFunc(dumpSwitch, dumpConfigInfo_.data(), dumpConfigInfo_.size());
869 78 : IDE_LOGI("callbackFunc returned: %d", result);
870 78 : return result;
871 : }
872 :
873 : #ifdef __ADUMP_LLT
874 571 : void DumpManager::Reset()
875 : {
876 571 : registered_ = false;
877 571 : exceptionDumper_.Reset();
878 571 : }
879 :
880 9 : bool DumpManager::GetKFCInitStatus()
881 : {
882 9 : return isKFCInit_;
883 : }
884 :
885 6 : void DumpManager::SetKFCInitStatus(bool status)
886 : {
887 6 : isKFCInit_ = status;
888 6 : }
889 : #endif
890 :
891 18 : int32_t DumpManager::RegisterExceptionDumpCallback(ExceptionDumpCallback callback)
892 : {
893 18 : return exceptionDumper_.RegisterExceptionDumpCallback(callback);
894 : }
895 :
896 18 : int32_t DumpManager::UnregisterExceptionDumpCallback(ExceptionDumpCallback callback)
897 : {
898 18 : return exceptionDumper_.UnregisterExceptionDumpCallback(callback);
899 : }
900 :
901 : } // namespace Adx
|
