Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include <cctype>
12 : #include <cinttypes>
13 : #include <thread>
14 : #include <map>
15 : #include <cerrno>
16 : #include "str_utils.h"
17 : #include "sys_utils.h"
18 : #include "lib_path.h"
19 : #include "file_utils.h"
20 : #include "log/adx_log.h"
21 : #include "dump_config_converter.h"
22 : #include "kernel_dfx_dumper.h"
23 :
24 : namespace Adx {
25 : static const int32_t WAIT_TASK_INTERVAL_TIME = 500;
26 : static const int32_t TOTAL_RETRIES = 20U;
27 : static const std::string KERNEL_DFX_TYPE_ALL = "all";
28 : static const std::string KERNEL_DFX_TYPE_PRINTF = "printf";
29 : static const std::string KERNEL_DFX_TYPE_TENSOR = "tensor";
30 : static const std::string KERNEL_DFX_TYPE_ASSERT = "assert";
31 : static const std::string KERNEL_DFX_TYPE_TIMESTAMP = "timestamp";
32 : static const std::string KERNEL_DFX_TYPE_BLOCKINFO = "BlockInfo";
33 : static const std::map<uint32_t, std::string> DFX_CORE_TYPE_MAP = {
34 : {0U, "aic"}, {1U, "aiv"}, {2U, "simt"}, {3U, "aicpu"}};
35 :
36 : static const std::map<rtKernelDfxInfoType, std::string> DFX_TYPE_STR_MAP = {
37 : {rtKernelDfxInfoType::RT_KERNEL_DFX_INFO_DEFAULT, KERNEL_DFX_TYPE_ALL},
38 : {rtKernelDfxInfoType::RT_KERNEL_DFX_INFO_PRINTF, KERNEL_DFX_TYPE_PRINTF},
39 : {rtKernelDfxInfoType::RT_KERNEL_DFX_INFO_TENSOR, KERNEL_DFX_TYPE_TENSOR},
40 : {rtKernelDfxInfoType::RT_KERNEL_DFX_INFO_ASSERT, KERNEL_DFX_TYPE_ASSERT},
41 : {rtKernelDfxInfoType::RT_KERNEL_DFX_INFO_TIME_STAMP, KERNEL_DFX_TYPE_TIMESTAMP},
42 : {rtKernelDfxInfoType::RT_KERNEL_DFX_INFO_BLOCK_INFO, KERNEL_DFX_TYPE_BLOCKINFO}};
43 :
44 1 : void DumpKernelDfxInfoCallback(
45 : rtKernelDfxInfoType dfxType, uint32_t coreType, uint32_t coreId, const uint8_t* buffer, size_t length)
46 : {
47 1 : (void)KernelDfxDumper::Instance().DumpKernelDfxInfo(dfxType, coreType, coreId, buffer, length);
48 1 : }
49 :
50 6 : int32_t KernelDfxDumper::PushDfxInfoToQueue(DumpDfxInfo& dfxInfo)
51 : {
52 6 : IDE_CTRL_VALUE_WARN(taskInit_, return ADUMP_FAILED, "The dfx dump task is not started or has been stopped.");
53 6 : if (dumpDfxInfoQueue_->IsFull()) {
54 1 : IDE_LOGE("Cannot push the dfx info into the queue. Memory usage exceeds 85% or queue size exceeds 60!");
55 1 : return ADUMP_FAILED;
56 : } else {
57 5 : if (dumpDfxInfoQueue_->Push(dfxInfo)) {
58 4 : IDE_LOGI("Push the dfx info into the queue success.");
59 4 : return ADUMP_SUCCESS;
60 : } else {
61 1 : IDE_LOGE("Failed to push the dfx info into the queue.");
62 1 : return ADUMP_FAILED;
63 : }
64 : }
65 : }
66 :
67 26 : void KernelDfxDumper::RecordDfxInfo()
68 : {
69 26 : IDE_LOGI("The dfx dump task is started.");
70 24 : taskRunning_ = true;
71 : try {
72 42 : while (taskInit_ || (dumpDfxInfoQueue_ && !dumpDfxInfoQueue_->IsEmpty())) {
73 36 : DumpDfxInfo dfxInfo{"", nullptr, 0UL};
74 18 : if (dumpDfxInfoQueue_ && dumpDfxInfoQueue_->Pop(dfxInfo)) {
75 4 : RecordDfxInfoToDisk(dfxInfo);
76 : }
77 18 : }
78 0 : } catch (...) {
79 0 : IDE_LOGW("The dfx dump task is exit with exception!");
80 0 : }
81 24 : taskRunning_ = false;
82 24 : IDE_LOGI("The dfx dump task is exit.");
83 24 : }
84 :
85 6 : void KernelDfxDumper::RecordDfxInfoToDisk(DumpDfxInfo& dfxInfo)
86 : {
87 6 : if (dfxInfo.path.empty() || dfxInfo.data == nullptr || dfxInfo.length == 0UL) {
88 2 : IDE_LOGW(
89 : "The dfx info item is invalid! path=%s, data=%p, length=%u", dfxInfo.path.c_str(), dfxInfo.data.get(),
90 : dfxInfo.length);
91 4 : return;
92 : }
93 :
94 4 : IDE_LOGI("Pop the dfx info item success. path=%s, length=%u", dfxInfo.path.c_str(), dfxInfo.length);
95 4 : Path path = Path(dfxInfo.path);
96 4 : std::string fileName = path.GetFileName();
97 4 : path = path.ParentPath();
98 4 : IDE_CTRL_VALUE_FAILED(
99 : path.CreateDirectory(true), return, "Cannot create the dfx info directory for path[%s]", path.GetCString());
100 4 : IDE_CTRL_VALUE_FAILED(path.RealPath(), return, "Cannot get the real path for path[%s]", path.GetCString());
101 4 : IDE_CTRL_VALUE_FAILED(
102 : !FileUtils::IsDiskFull(path.GetString(), static_cast<uint64_t>(dfxInfo.length)), return,
103 : "Don't have enough free disk for %u bytes", dfxInfo.length);
104 3 : path.Concat(fileName);
105 3 : IdeErrorT err = FileUtils::WriteFile(path.GetString(), dfxInfo.data.get(), dfxInfo.length, -1);
106 3 : if (err != IDE_DAEMON_NONE_ERROR) {
107 1 : IDE_LOGE("Cannot write the dfx info into path[%s]! err: %d", path.GetCString(), err);
108 1 : return;
109 : }
110 2 : IDE_LOGI("Record the dfx info into path[%s] success.", path.GetCString());
111 6 : }
112 :
113 94 : int32_t KernelDfxDumper::InitTask()
114 : {
115 94 : if (taskInit_) {
116 67 : IDE_LOGI("The dfx dump task has been started, no need to start again.");
117 67 : return IDE_DAEMON_OK;
118 : }
119 : // 没有注册使能，不开启任务。在下一次注册使能时再开启任务
120 27 : if (!IsEnabled()) {
121 1 : IDE_LOGI("Dfx type has not been registered, no need to start the dfx dump task.");
122 1 : return IDE_DAEMON_OK;
123 : }
124 :
125 26 : IDE_LOGI("Begin to start the dfx dump task.");
126 26 : if (dumpDfxInfoQueue_ == nullptr) {
127 26 : dumpDfxInfoQueue_.reset(new (std::nothrow) BoundQueueMemory<DumpDfxInfo>());
128 26 : if (dumpDfxInfoQueue_ == nullptr) {
129 0 : IDE_LOGE("Failed to new dumpDfxInfoQueue");
130 0 : return IDE_DAEMON_ERROR;
131 : }
132 : }
133 26 : dumpDfxInfoQueue_->Init();
134 26 : taskInit_ = true;
135 :
136 : try {
137 26 : taskThread_ = std::thread(&KernelDfxDumper::RecordDfxInfo, this);
138 0 : } catch (std::exception& ex) {
139 0 : IDE_LOGE("Create the dfx dump task failed, message: %s", ex.what());
140 0 : taskInit_ = false;
141 0 : return IDE_DAEMON_ERROR;
142 0 : }
143 26 : return IDE_DAEMON_OK;
144 : }
145 :
146 44 : int32_t KernelDfxDumper::UnInitTask()
147 : {
148 44 : IDE_LOGI("Begin to stop the dfx dump task.");
149 44 : taskInit_ = false;
150 44 : if (dumpDfxInfoQueue_) {
151 28 : dumpDfxInfoQueue_->Quit();
152 : }
153 : // 超时等待任务退出，避免任务卡死
154 61 : for (int32_t i = 0; i < TOTAL_RETRIES && taskRunning_; ++i) {
155 17 : mmSleep(WAIT_TASK_INTERVAL_TIME);
156 : }
157 44 : if (taskThread_.joinable()) {
158 24 : if (taskRunning_) {
159 0 : IDE_LOGW("The dfx dump task maybe stuck, detach it.");
160 0 : taskThread_.detach();
161 : } else {
162 24 : taskThread_.join();
163 : }
164 24 : taskThread_ = std::thread();
165 : }
166 44 : return IDE_DAEMON_OK;
167 : }
168 :
169 39 : void KernelDfxDumper::UnInit()
170 : {
171 39 : std::lock_guard<std::mutex> lock(mutex_);
172 39 : destructed_ = true;
173 39 : UnInitTask();
174 39 : dumpPath_.clear();
175 39 : enabledDfxTypes_.clear();
176 39 : if (dumpDfxInfoQueue_) {
177 24 : dumpDfxInfoQueue_.reset();
178 : }
179 39 : }
180 :
181 5 : void KernelDfxDumper::PrepareFork() { Instance().mutex_.lock(); }
182 :
183 3 : void KernelDfxDumper::PostForkParent() { Instance().mutex_.unlock(); }
184 :
185 2 : void KernelDfxDumper::PostForkChild()
186 : {
187 2 : auto& instance = Instance();
188 2 : instance.taskInit_ = false;
189 2 : instance.taskRunning_ = false;
190 2 : instance.dumpPath_.clear();
191 2 : instance.enabledDfxTypes_.clear();
192 : // 释放线程控制权
193 2 : if (instance.taskThread_.joinable()) {
194 2 : instance.taskThread_.detach();
195 : }
196 : // 释放dumpDfxInfoQueue_控制权
197 2 : if (instance.dumpDfxInfoQueue_) {
198 2 : instance.dumpDfxInfoQueue_.release();
199 2 : instance.dumpDfxInfoQueue_ = nullptr;
200 : }
201 2 : instance.mutex_.unlock();
202 2 : }
203 :
204 3 : KernelDfxDumper::KernelDfxDumper()
205 : {
206 : int32_t ret =
207 3 : pthread_atfork(KernelDfxDumper::PrepareFork, KernelDfxDumper::PostForkParent, KernelDfxDumper::PostForkChild);
208 3 : if (ret != 0) {
209 0 : IDE_LOGW("call pthread_atfork failed, ret: %d", ret);
210 : }
211 3 : EnableDfxDumper();
212 3 : }
213 :
214 3 : KernelDfxDumper::~KernelDfxDumper() { UnInit(); }
215 :
216 26 : void KernelDfxDumper::EnableDfxDumper()
217 : {
218 26 : DumpDfxConfig dumpDfxConfig;
219 26 : if (DumpConfigConverter::EnableKernelDfxDumpWithEnv(dumpDfxConfig)) {
220 20 : if (EnableDfxDumper(dumpDfxConfig) != ADUMP_SUCCESS) {
221 3 : IDE_LOGW("Enable kernel dfx dump with env failed!");
222 : }
223 : }
224 26 : }
225 :
226 95 : bool KernelDfxDumper::InitDumpPath(const std::string& dumpPath)
227 : {
228 95 : IDE_CTRL_VALUE_FAILED(!dumpPath.empty(), return false, "The dfx info dump path is empty!");
229 95 : IDE_CTRL_VALUE_WARN(
230 : dumpPath_.empty(), return true, "The dfx info dump path has been set with [%s]", dumpPath_.c_str());
231 26 : Path path = Path(dumpPath).Append(SysUtils::GetCurrentTime());
232 26 : dumpPath_ = path.GetString();
233 26 : IDE_LOGI("Set the dfx info dump path with [%s]", dumpPath_.c_str());
234 26 : return true;
235 26 : }
236 :
237 12 : std::string KernelDfxDumper::GetDfxTypeStr(const rtKernelDfxInfoType rtDfxType)
238 : {
239 12 : auto it = DFX_TYPE_STR_MAP.find(rtDfxType);
240 14 : return it != DFX_TYPE_STR_MAP.end() ? it->second : "";
241 : }
242 :
243 12 : std::string KernelDfxDumper::GetCoreTypeStr(const uint32_t coreType)
244 : {
245 12 : auto it = DFX_CORE_TYPE_MAP.find(coreType);
246 14 : return it != DFX_CORE_TYPE_MAP.end() ? it->second : "";
247 : }
248 :
249 107 : int32_t KernelDfxDumper::EnableDfxDumper(const DumpDfxConfig config)
250 : {
251 107 : if (config.dfxTypes.empty() || config.dumpPath.empty()) {
252 12 : return ADUMP_SUCCESS;
253 : }
254 95 : std::lock_guard<std::mutex> lock(mutex_);
255 95 : destructed_ = false;
256 95 : std::set<rtKernelDfxInfoType> rtDfxTypes;
257 95 : GetRegisterDfxTypes(config.dfxTypes, rtDfxTypes);
258 206 : for (auto& rtDfxType : rtDfxTypes) {
259 111 : if (!IsEnabled(rtDfxType)) {
260 42 : rtError_t ret = rtSetKernelDfxInfoCallback(rtDfxType, DumpKernelDfxInfoCallback);
261 42 : IDE_CTRL_VALUE_FAILED(
262 : ret == RT_ERROR_NONE, return ADUMP_FAILED,
263 : "Register the dfx info dump callback to RTS failed! dfxType=%d, ret=%d", rtDfxType, ret);
264 42 : enabledDfxTypes_.insert(rtDfxType);
265 42 : IDE_LOGI("Register the dfx info dump callback to RTS success. dfxType=%d", rtDfxType);
266 : } else {
267 69 : IDE_LOGI("The dfx info dump callback has been registered to RTS. dfxType=%d", rtDfxType);
268 : }
269 : }
270 95 : IDE_CTRL_VALUE_FAILED(
271 : InitDumpPath(config.dumpPath), return ADUMP_FAILED, "Set the dfx info dump path[%s] failed!",
272 : config.dumpPath.c_str());
273 95 : IDE_CTRL_VALUE_FAILED(InitTask() == IDE_DAEMON_OK, return ADUMP_FAILED, "Init the task of dump dfx info failed!");
274 92 : return ADUMP_SUCCESS;
275 95 : }
276 :
277 95 : void KernelDfxDumper::GetRegisterDfxTypes(
278 : const std::vector<std::string>& cfgDfxTypes, std::set<rtKernelDfxInfoType>& rtDfxTypes)
279 : {
280 297 : for (auto& dfxType : cfgDfxTypes) {
281 : // 非default场景，都要再注册RT_KERNEL_DFX_INFO_BLOCK_INFO类型。
282 : // all/printf：不使用静态常量，解决so构造函数时通过环境变量使能而静态常量未初始化的问题。
283 107 : if (dfxType == "all") {
284 94 : rtDfxTypes.insert(rtKernelDfxInfoType::RT_KERNEL_DFX_INFO_DEFAULT);
285 13 : } else if (dfxType == "printf") {
286 3 : rtDfxTypes.insert(
287 : {rtKernelDfxInfoType::RT_KERNEL_DFX_INFO_PRINTF, rtKernelDfxInfoType::RT_KERNEL_DFX_INFO_BLOCK_INFO});
288 10 : } else if (dfxType == "tensor") {
289 4 : rtDfxTypes.insert(
290 : {rtKernelDfxInfoType::RT_KERNEL_DFX_INFO_TENSOR, rtKernelDfxInfoType::RT_KERNEL_DFX_INFO_BLOCK_INFO});
291 6 : } else if (dfxType == "assert") {
292 3 : rtDfxTypes.insert(
293 : {rtKernelDfxInfoType::RT_KERNEL_DFX_INFO_ASSERT, rtKernelDfxInfoType::RT_KERNEL_DFX_INFO_BLOCK_INFO});
294 3 : } else if (dfxType == "timestamp") {
295 3 : rtDfxTypes.insert(
296 : {rtKernelDfxInfoType::RT_KERNEL_DFX_INFO_TIME_STAMP,
297 : rtKernelDfxInfoType::RT_KERNEL_DFX_INFO_BLOCK_INFO});
298 : }
299 : }
300 95 : }
301 :
302 160 : bool KernelDfxDumper::IsEnabled(const rtKernelDfxInfoType dfxType)
303 : {
304 160 : return enabledDfxTypes_.find(dfxType) != enabledDfxTypes_.end();
305 : }
306 :
307 44 : bool KernelDfxDumper::IsEnabled() { return !enabledDfxTypes_.empty(); }
308 :
309 6 : std::string KernelDfxDumper::GetDfxInfoFilePath(uint32_t coreId, std::string& coreType)
310 : {
311 6 : std::string fileName = "asc_kernel_data_" + coreType + "_" + std::to_string(coreId) + ".bin";
312 12 : return Path(dumpPath_).Concat(fileName).GetString();
313 6 : }
314 :
315 12 : int32_t KernelDfxDumper::DumpKernelDfxInfo(
316 : rtKernelDfxInfoType dfxType, uint32_t coreType, uint32_t coreId, const uint8_t* buffer, size_t length)
317 : {
318 12 : std::lock_guard<std::mutex> lock(mutex_);
319 12 : IDE_CTRL_VALUE_WARN(!destructed_, return ADUMP_FAILED, "KernelDfxDumper has been destructed.");
320 :
321 12 : std::string dfxTypeStr = GetDfxTypeStr(dfxType);
322 12 : std::string coreTypeStr = GetCoreTypeStr(coreType);
323 12 : IDE_CTRL_VALUE_WARN(
324 : IsEnabled(dfxType), return ADUMP_FAILED, "dfxType=%d[%s] is not enabled, do not record the dfx info.", dfxType,
325 : dfxTypeStr.c_str());
326 18 : if (buffer == nullptr || length == 0UL || length > std::numeric_limits<uint32_t>::max() || dfxTypeStr.empty() ||
327 8 : coreTypeStr.empty()) {
328 3 : IDE_LOGW(
329 : "The dfx info is invalid! dfxType=%d[%s], coreType=%u[%s], coreId=%u, buffer=%p, length=%zu", dfxType,
330 : dfxTypeStr.c_str(), coreType, coreTypeStr.c_str(), coreId, buffer, length);
331 3 : return ADUMP_FAILED;
332 : }
333 7 : IDE_LOGI(
334 : "Receive kernel dfx info. dfxType=%d[%s], coreType=%u[%s], coreId=%u, buffer=%p, length=%zu", dfxType,
335 : dfxTypeStr.c_str(), coreType, coreTypeStr.c_str(), coreId, buffer, length);
336 :
337 14 : std::shared_ptr<uint8_t> data(new (std::nothrow) uint8_t[length], [](uint8_t* ptr) { delete[] ptr; });
338 7 : IDE_CTRL_VALUE_FAILED(data.get() != nullptr, return ADUMP_FAILED, "Cannot create the new dfx info buffer!");
339 7 : auto err = memcpy_s(data.get(), length, buffer, length);
340 7 : if (err != EOK) {
341 1 : IDE_LOGE("Cannot copy the dfx info to the new buffer! err: %d", err);
342 1 : return ADUMP_FAILED;
343 : }
344 6 : DumpDfxInfo dfxInfo{GetDfxInfoFilePath(coreId, coreTypeStr), data, static_cast<uint32_t>(length)};
345 6 : return PushDfxInfoToQueue(dfxInfo);
346 12 : }
347 : } // namespace Adx
|
