Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include <cctype>
12 : #include <cinttypes>
13 : #include <thread>
14 : #include <map>
15 : #include <cerrno>
16 : #include "str_utils.h"
17 : #include "sys_utils.h"
18 : #include "lib_path.h"
19 : #include "file_utils.h"
20 : #include "log/adx_log.h"
21 : #include "dump_config_converter.h"
22 : #include "kernel_dfx_dumper.h"
23 :
24 : namespace Adx {
25 : static const int32_t WAIT_TASK_INTERVAL_TIME = 500;
26 : static const int32_t TOTAL_RETRIES = 20U;
27 : static const std::string KERNEL_DFX_TYPE_ALL = "all";
28 : static const std::string KERNEL_DFX_TYPE_PRINTF = "printf";
29 : static const std::string KERNEL_DFX_TYPE_TENSOR = "tensor";
30 : static const std::string KERNEL_DFX_TYPE_ASSERT = "assert";
31 : static const std::string KERNEL_DFX_TYPE_TIMESTAMP = "timestamp";
32 : static const std::string KERNEL_DFX_TYPE_BLOCKINFO = "BlockInfo";
33 : static const std::map<uint32_t, std::string> DFX_CORE_TYPE_MAP = {
34 : {0U, "aic"},
35 : {1U, "aiv"},
36 : {2U, "simt"}
37 : };
38 :
39 : static const std::map<rtKernelDfxInfoType, std::string> DFX_TYPE_STR_MAP = {
40 : {rtKernelDfxInfoType::RT_KERNEL_DFX_INFO_DEFAULT, KERNEL_DFX_TYPE_ALL},
41 : {rtKernelDfxInfoType::RT_KERNEL_DFX_INFO_PRINTF, KERNEL_DFX_TYPE_PRINTF},
42 : {rtKernelDfxInfoType::RT_KERNEL_DFX_INFO_TENSOR, KERNEL_DFX_TYPE_TENSOR},
43 : {rtKernelDfxInfoType::RT_KERNEL_DFX_INFO_ASSERT, KERNEL_DFX_TYPE_ASSERT},
44 : {rtKernelDfxInfoType::RT_KERNEL_DFX_INFO_TIME_STAMP, KERNEL_DFX_TYPE_TIMESTAMP},
45 : {rtKernelDfxInfoType::RT_KERNEL_DFX_INFO_BLOCK_INFO, KERNEL_DFX_TYPE_BLOCKINFO}
46 : };
47 :
48 1 : void DumpKernelDfxInfoCallback(rtKernelDfxInfoType dfxType, uint32_t coreType, uint32_t coreId,
49 : const uint8_t *buffer, size_t length)
50 : {
51 1 : (void)KernelDfxDumper::Instance().DumpKernelDfxInfo(dfxType, coreType, coreId, buffer, length);
52 1 : }
53 :
54 5 : int32_t KernelDfxDumper::PushDfxInfoToQueue(DumpDfxInfo &dfxInfo)
55 : {
56 5 : IDE_CTRL_VALUE_WARN(taskInit_, return ADUMP_FAILED, "The dfx dump task is not started or has been stopped.");
57 5 : if (dumpDfxInfoQueue_->IsFull()) {
58 1 : IDE_LOGE("Cannot push the dfx info into the queue. Memory usage exceeds 85% or queue size exceeds 60!");
59 1 : return ADUMP_FAILED;
60 : } else {
61 4 : if (dumpDfxInfoQueue_->Push(dfxInfo)) {
62 3 : IDE_LOGI("Push the dfx info into the queue success.");
63 3 : return ADUMP_SUCCESS;
64 : } else {
65 1 : IDE_LOGE("Failed to push the dfx info into the queue.");
66 1 : return ADUMP_FAILED;
67 : }
68 : }
69 : }
70 :
71 25 : void KernelDfxDumper::RecordDfxInfo()
72 : {
73 25 : IDE_LOGI("The dfx dump task is started.");
74 23 : taskRunning_ = true;
75 : try {
76 40 : while (taskInit_ || (dumpDfxInfoQueue_ && !dumpDfxInfoQueue_->IsEmpty())) {
77 34 : DumpDfxInfo dfxInfo{"", nullptr, 0UL};
78 17 : if (dumpDfxInfoQueue_ && dumpDfxInfoQueue_->Pop(dfxInfo)) {
79 3 : RecordDfxInfoToDisk(dfxInfo);
80 : }
81 17 : }
82 0 : } catch (...) {
83 0 : IDE_LOGW("The dfx dump task is exit with exception!");
84 0 : }
85 23 : taskRunning_ = false;
86 23 : IDE_LOGI("The dfx dump task is exit.");
87 23 : }
88 :
89 5 : void KernelDfxDumper::RecordDfxInfoToDisk(DumpDfxInfo &dfxInfo)
90 : {
91 5 : if (dfxInfo.path.empty() || dfxInfo.data == nullptr || dfxInfo.length == 0UL) {
92 2 : IDE_LOGW("The dfx info item is invalid! path=%s, data=%p, length=%u",
93 : dfxInfo.path.c_str(), dfxInfo.data.get(), dfxInfo.length);
94 4 : return;
95 : }
96 :
97 3 : IDE_LOGI("Pop the dfx info item success. path=%s, length=%u", dfxInfo.path.c_str(), dfxInfo.length);
98 3 : Path path = Path(dfxInfo.path);
99 3 : std::string fileName = path.GetFileName();
100 3 : path = path.ParentPath();
101 3 : IDE_CTRL_VALUE_FAILED(path.CreateDirectory(true), return,
102 : "Cannot create the dfx info directory for path[%s]", path.GetCString());
103 3 : IDE_CTRL_VALUE_FAILED(path.RealPath(), return,
104 : "Cannot get the real path for path[%s]", path.GetCString());
105 3 : IDE_CTRL_VALUE_FAILED(!FileUtils::IsDiskFull(path.GetString(), static_cast<uint64_t>(dfxInfo.length)),
106 : return, "Don't have enough free disk for %u bytes", dfxInfo.length);
107 2 : path.Concat(fileName);
108 2 : IdeErrorT err = FileUtils::WriteFile(path.GetString(), dfxInfo.data.get(), dfxInfo.length, -1);
109 2 : if (err != IDE_DAEMON_NONE_ERROR) {
110 1 : IDE_LOGE("Cannot write the dfx info into path[%s]! err: %d", path.GetCString(), err);
111 1 : return;
112 : }
113 1 : IDE_LOGI("Record the dfx info into path[%s] success.", path.GetCString());
114 5 : }
115 :
116 93 : int32_t KernelDfxDumper::InitTask()
117 : {
118 93 : if (taskInit_) {
119 67 : IDE_LOGI("The dfx dump task has been started, no need to start again.");
120 67 : return IDE_DAEMON_OK;
121 : }
122 : // 没有注册使能，不开启任务。在下一次注册使能时再开启任务
123 26 : if (!IsEnabled()) {
124 1 : IDE_LOGI("Dfx type has not been registered, no need to start the dfx dump task.");
125 1 : return IDE_DAEMON_OK;
126 : }
127 :
128 25 : IDE_LOGI("Begin to start the dfx dump task.");
129 25 : if (dumpDfxInfoQueue_ == nullptr) {
130 25 : dumpDfxInfoQueue_.reset(new(std::nothrow) BoundQueueMemory<DumpDfxInfo>());
131 25 : if (dumpDfxInfoQueue_ == nullptr) {
132 0 : IDE_LOGE("Failed to new dumpDfxInfoQueue");
133 0 : return IDE_DAEMON_ERROR;
134 : }
135 : }
136 25 : dumpDfxInfoQueue_->Init();
137 25 : taskInit_ = true;
138 :
139 : try {
140 25 : taskThread_ = std::thread(&KernelDfxDumper::RecordDfxInfo, this);
141 0 : } catch (std::exception &ex) {
142 0 : IDE_LOGE("Create the dfx dump task failed, message: %s", ex.what());
143 0 : taskInit_ = false;
144 0 : return IDE_DAEMON_ERROR;
145 0 : }
146 25 : return IDE_DAEMON_OK;
147 : }
148 :
149 41 : int32_t KernelDfxDumper::UnInitTask()
150 : {
151 41 : IDE_LOGI("Begin to stop the dfx dump task.");
152 41 : taskInit_ = false;
153 41 : if (dumpDfxInfoQueue_) {
154 26 : dumpDfxInfoQueue_->Quit();
155 : }
156 : // 超时等待任务退出，避免任务卡死
157 57 : for (int32_t i = 0; i < TOTAL_RETRIES && taskRunning_; ++i) {
158 16 : mmSleep(WAIT_TASK_INTERVAL_TIME);
159 : }
160 41 : if (taskThread_.joinable()) {
161 23 : if (taskRunning_) {
162 0 : IDE_LOGW("The dfx dump task maybe stuck, detach it.");
163 0 : taskThread_.detach();
164 : } else {
165 23 : taskThread_.join();
166 : }
167 23 : taskThread_ = std::thread();
168 : }
169 41 : return IDE_DAEMON_OK;
170 : }
171 :
172 37 : void KernelDfxDumper::UnInit()
173 : {
174 37 : std::lock_guard<std::mutex> lock(mutex_);
175 37 : destructed_ = true;
176 37 : UnInitTask();
177 37 : dumpPath_.clear();
178 37 : enabledDfxTypes_.clear();
179 37 : if (dumpDfxInfoQueue_) {
180 23 : dumpDfxInfoQueue_.reset();
181 : }
182 37 : }
183 :
184 5 : void KernelDfxDumper::PrepareFork()
185 : {
186 5 : Instance().mutex_.lock();
187 5 : }
188 :
189 3 : void KernelDfxDumper::PostForkParent()
190 : {
191 3 : Instance().mutex_.unlock();
192 3 : }
193 :
194 2 : void KernelDfxDumper::PostForkChild()
195 : {
196 2 : auto &instance = Instance();
197 2 : instance.taskInit_ = false;
198 2 : instance.taskRunning_ = false;
199 2 : instance.dumpPath_.clear();
200 2 : instance.enabledDfxTypes_.clear();
201 : // 释放线程控制权
202 2 : if (instance.taskThread_.joinable()) {
203 2 : instance.taskThread_.detach();
204 : }
205 : // 释放dumpDfxInfoQueue_控制权
206 2 : if (instance.dumpDfxInfoQueue_) {
207 2 : instance.dumpDfxInfoQueue_.release();
208 2 : instance.dumpDfxInfoQueue_ = nullptr;
209 : }
210 2 : instance.mutex_.unlock();
211 2 : }
212 :
213 3 : KernelDfxDumper::KernelDfxDumper()
214 : {
215 3 : int32_t ret = pthread_atfork(
216 : KernelDfxDumper::PrepareFork, KernelDfxDumper::PostForkParent, KernelDfxDumper::PostForkChild);
217 3 : if (ret != 0) {
218 0 : IDE_LOGW("call pthread_atfork failed, ret: %d", ret);
219 : }
220 3 : EnableDfxDumper();
221 3 : }
222 :
223 3 : KernelDfxDumper::~KernelDfxDumper()
224 : {
225 3 : UnInit();
226 3 : }
227 :
228 26 : void KernelDfxDumper::EnableDfxDumper() {
229 26 : DumpDfxConfig dumpDfxConfig;
230 26 : if (DumpConfigConverter::EnableKernelDfxDumpWithEnv(dumpDfxConfig)) {
231 20 : if (EnableDfxDumper(dumpDfxConfig) != ADUMP_SUCCESS) {
232 3 : IDE_LOGW("Enable kernel dfx dump with env failed!");
233 : }
234 : }
235 26 : }
236 :
237 94 : bool KernelDfxDumper::InitDumpPath(const std::string &dumpPath)
238 : {
239 94 : IDE_CTRL_VALUE_FAILED(!dumpPath.empty(), return false, "The dfx info dump path is empty!");
240 94 : IDE_CTRL_VALUE_WARN(dumpPath_.empty(), return true,
241 : "The dfx info dump path has been set with [%s]", dumpPath_.c_str());
242 25 : Path path = Path(dumpPath).Append(SysUtils::GetCurrentTime());
243 25 : dumpPath_ = path.GetString();
244 25 : IDE_LOGI("Set the dfx info dump path with [%s]", dumpPath_.c_str());
245 25 : return true;
246 25 : }
247 :
248 11 : std::string KernelDfxDumper::GetDfxTypeStr(const rtKernelDfxInfoType rtDfxType)
249 : {
250 11 : auto it = DFX_TYPE_STR_MAP.find(rtDfxType);
251 13 : return it != DFX_TYPE_STR_MAP.end() ? it->second : "";
252 : }
253 :
254 11 : std::string KernelDfxDumper::GetCoreTypeStr(const uint32_t coreType)
255 : {
256 11 : auto it = DFX_CORE_TYPE_MAP.find(coreType);
257 13 : return it != DFX_CORE_TYPE_MAP.end() ? it->second : "";
258 : }
259 :
260 106 : int32_t KernelDfxDumper::EnableDfxDumper(const DumpDfxConfig config)
261 : {
262 106 : if (config.dfxTypes.empty() || config.dumpPath.empty()) {
263 12 : return ADUMP_SUCCESS;
264 : }
265 94 : std::lock_guard<std::mutex> lock(mutex_);
266 94 : destructed_ = false;
267 94 : std::set<rtKernelDfxInfoType> rtDfxTypes;
268 94 : GetRegisterDfxTypes(config.dfxTypes, rtDfxTypes);
269 204 : for (auto& rtDfxType : rtDfxTypes) {
270 110 : if (!IsEnabled(rtDfxType)) {
271 41 : rtError_t ret = rtSetKernelDfxInfoCallback(rtDfxType, DumpKernelDfxInfoCallback);
272 41 : IDE_CTRL_VALUE_FAILED(ret == RT_ERROR_NONE, return ADUMP_FAILED,
273 : "Register the dfx info dump callback to RTS failed! dfxType=%d, ret=%d", rtDfxType, ret);
274 41 : enabledDfxTypes_.insert(rtDfxType);
275 41 : IDE_LOGI("Register the dfx info dump callback to RTS success. dfxType=%d", rtDfxType);
276 : } else {
277 69 : IDE_LOGI("The dfx info dump callback has been registered to RTS. dfxType=%d", rtDfxType);
278 : }
279 : }
280 94 : IDE_CTRL_VALUE_FAILED(InitDumpPath(config.dumpPath), return ADUMP_FAILED,
281 : "Set the dfx info dump path[%s] failed!", config.dumpPath.c_str());
282 94 : IDE_CTRL_VALUE_FAILED(InitTask() == IDE_DAEMON_OK, return ADUMP_FAILED,
283 : "Init the task of dump dfx info failed!");
284 91 : return ADUMP_SUCCESS;
285 94 : }
286 :
287 94 : void KernelDfxDumper::GetRegisterDfxTypes(const std::vector<std::string> &cfgDfxTypes,
288 : std::set<rtKernelDfxInfoType> &rtDfxTypes)
289 : {
290 200 : for (auto& dfxType : cfgDfxTypes) {
291 : // 非default场景，都要再注册RT_KERNEL_DFX_INFO_BLOCK_INFO类型。
292 : // all/printf：不使用静态常量，解决so构造函数时通过环境变量使能而静态常量未初始化的问题。
293 106 : if (dfxType == "all") {
294 93 : rtDfxTypes.insert(rtKernelDfxInfoType::RT_KERNEL_DFX_INFO_DEFAULT);
295 13 : } else if (dfxType == "printf") {
296 3 : rtDfxTypes.insert({rtKernelDfxInfoType::RT_KERNEL_DFX_INFO_PRINTF,
297 : rtKernelDfxInfoType::RT_KERNEL_DFX_INFO_BLOCK_INFO});
298 10 : } else if (dfxType == "tensor") {
299 4 : rtDfxTypes.insert({rtKernelDfxInfoType::RT_KERNEL_DFX_INFO_TENSOR,
300 : rtKernelDfxInfoType::RT_KERNEL_DFX_INFO_BLOCK_INFO});
301 6 : } else if (dfxType == "assert") {
302 3 : rtDfxTypes.insert({rtKernelDfxInfoType::RT_KERNEL_DFX_INFO_ASSERT,
303 : rtKernelDfxInfoType::RT_KERNEL_DFX_INFO_BLOCK_INFO});
304 3 : } else if (dfxType == "timestamp") {
305 3 : rtDfxTypes.insert({rtKernelDfxInfoType::RT_KERNEL_DFX_INFO_TIME_STAMP,
306 : rtKernelDfxInfoType::RT_KERNEL_DFX_INFO_BLOCK_INFO});
307 : }
308 : }
309 94 : }
310 :
311 158 : bool KernelDfxDumper::IsEnabled(const rtKernelDfxInfoType dfxType)
312 : {
313 158 : return enabledDfxTypes_.find(dfxType) != enabledDfxTypes_.end();
314 : }
315 :
316 43 : bool KernelDfxDumper::IsEnabled()
317 : {
318 43 : return !enabledDfxTypes_.empty();
319 : }
320 :
321 5 : std::string KernelDfxDumper::GetDfxInfoFilePath(uint32_t coreId, std::string &coreType)
322 : {
323 5 : std::string fileName = "asc_kernel_data_" + coreType + "_" + std::to_string(coreId) + ".bin";
324 10 : return Path(dumpPath_).Concat(fileName).GetString();
325 5 : }
326 :
327 11 : int32_t KernelDfxDumper::DumpKernelDfxInfo(rtKernelDfxInfoType dfxType, uint32_t coreType, uint32_t coreId,
328 : const uint8_t *buffer, size_t length)
329 : {
330 11 : std::lock_guard<std::mutex> lock(mutex_);
331 11 : IDE_CTRL_VALUE_WARN(!destructed_, return ADUMP_FAILED, "KernelDfxDumper has been destructed.");
332 :
333 11 : std::string dfxTypeStr = GetDfxTypeStr(dfxType);
334 11 : std::string coreTypeStr = GetCoreTypeStr(coreType);
335 11 : IDE_CTRL_VALUE_WARN(IsEnabled(dfxType), return ADUMP_FAILED,
336 : "dfxType=%d[%s] is not enabled, do not record the dfx info.", dfxType, dfxTypeStr.c_str());
337 8 : if (buffer == nullptr || length == 0UL || length > std::numeric_limits<uint32_t>::max()
338 17 : || dfxTypeStr.empty() || coreTypeStr.empty()) {
339 3 : IDE_LOGW("The dfx info is invalid! dfxType=%d[%s], coreType=%u[%s], coreId=%u, buffer=%p, length=%zu",
340 : dfxType, dfxTypeStr.c_str(), coreType, coreTypeStr.c_str(), coreId, buffer, length);
341 3 : return ADUMP_FAILED;
342 : }
343 6 : IDE_LOGI("Receive kernel dfx info. dfxType=%d[%s], coreType=%u[%s], coreId=%u, buffer=%p, length=%zu",
344 : dfxType, dfxTypeStr.c_str(), coreType, coreTypeStr.c_str(), coreId, buffer, length);
345 :
346 12 : std::shared_ptr<uint8_t> data(new(std::nothrow) uint8_t[length], [](uint8_t* ptr) { delete[] ptr; });
347 6 : IDE_CTRL_VALUE_FAILED(data.get() != nullptr, return ADUMP_FAILED, "Cannot create the new dfx info buffer!");
348 6 : auto err = memcpy_s(data.get(), length, buffer, length);
349 6 : if (err != EOK) {
350 1 : IDE_LOGE("Cannot copy the dfx info to the new buffer! err: %d", err);
351 1 : return ADUMP_FAILED;
352 : }
353 5 : DumpDfxInfo dfxInfo{GetDfxInfoFilePath(coreId, coreTypeStr), data, static_cast<uint32_t>(length)};
354 5 : return PushDfxInfoToQueue(dfxInfo);
355 11 : }
356 : }
|
