Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "dump_args_callback.h"
12 : #include "dfx_args_parser.h"
13 : #include "dump_memory.h"
14 : #include "exception_info_common.h"
15 : #include "kernel_symbol_locator.h"
16 : #include "str_utils.h"
17 : #include "log/adx_log.h"
18 : #include "log/hdc_log.h"
19 :
20 : namespace Adx {
21 :
22 25 : DumpArgsCallback::DumpArgsCallback(const rtExceptionInfo &exception, const ExceptionDumpInfo &info,
23 25 : const std::string &dumpPath)
24 25 : : exception_(exception),
25 25 : info_(info),
26 25 : dumpPath_(dumpPath),
27 75 : dumpFilePath_(dumpPath + "/" +
28 100 : std::string(info.kernelDisplayName[0] != '\0' ? info.kernelDisplayName : "exception_info") + "." +
29 100 : std::to_string(exception.streamid) + "." + std::to_string(exception.taskid) + "." +
30 100 : std::to_string(info.coreType) + "." + std::to_string(info.coreId) + "." +
31 50 : SysUtils::GetCurrentTimeWithMillisecond()),
32 50 : dumpFile_(exception.deviceid, dumpFilePath_)
33 : {
34 25 : }
35 :
36 4 : int32_t DumpArgsCallback::DumpKernelBin()
37 : {
38 4 : std::string kernelName(info_.kernelName);
39 4 : if (info_.bin == nullptr || kernelName.empty()) {
40 3 : return ADUMP_SUCCESS;
41 : }
42 1 : IDE_LOGI("Dump kernel bin file. bin=%p, kernelName=%s", info_.bin, kernelName.c_str());
43 1 : KernelInfoCollector collector;
44 1 : int32_t ret = collector.InitFromBinHandle(info_.bin, kernelName);
45 1 : IDE_CTRL_VALUE_FAILED(ret == ADUMP_SUCCESS, return ADUMP_FAILED,
46 : "Init for dump kernel bin failed. ret=%d, bin=%p, kernelName=%s.", ret, info_.bin, kernelName.c_str());
47 :
48 : // 先同步落 _host.o，再做慢的 kernel_meta 搜索拷贝。落盘失败需传播，保持与拆分前一致的错误可观测性。
49 1 : ret = collector.DumpHostKernelBin(dumpPath_);
50 1 : IDE_CTRL_VALUE_WARN(ret == ADUMP_SUCCESS, return ADUMP_FAILED,
51 : "DumpHostKernelBin failed, kernelName=%s.", kernelName.c_str());
52 :
53 1 : ret = collector.StartCollectKernel(dumpPath_);
54 1 : IDE_CTRL_VALUE_WARN(ret == ADUMP_SUCCESS, return ADUMP_FAILED,
55 : "StartCollectKernel failed, kernelName=%s.", kernelName.c_str());
56 :
57 1 : IDE_LOGI("DumpKernelBin success, kernelName=%s.", kernelName.c_str());
58 1 : return ADUMP_SUCCESS;
59 4 : }
60 :
61 6 : int32_t DumpArgsCallback::DumpKernelErrorSymbols()
62 : {
63 6 : if (info_.bin == nullptr) {
64 2 : return ADUMP_SUCCESS;
65 : }
66 4 : KernelSymbolLocator locator;
67 4 : int32_t ret = locator.InitFromBinHandle(info_.bin);
68 4 : IDE_CTRL_VALUE_FAILED(ret == ADUMP_SUCCESS, return ADUMP_FAILED,
69 : "KernelSymbolLocator InitFromBinHandle failed for callback exception. ret=%d", ret);
70 3 : locator.UpdateStartPCFromDeviceAddr(info_.bin);
71 :
72 3 : ExceptionRegInfo exceptionRegInfo{0, nullptr};
73 3 : ret = ExceptionInfoCommon::GetExceptionRegInfo(exception_, exceptionRegInfo);
74 3 : IDE_CTRL_VALUE_WARN(ret == ADUMP_SUCCESS, return ADUMP_FAILED,
75 : "Get exception register information failed for callback exception. ret=%d", ret);
76 :
77 2 : ret = locator.LocateAndPrintErrorSymbolsForCore(info_.coreId, info_.coreType, exceptionRegInfo);
78 2 : IDE_CTRL_VALUE_WARN(ret == ADUMP_SUCCESS, return ADUMP_FAILED,
79 : "LocateAndPrintErrorSymbolsForCore failed for callback exception. ret=%d, coreId=%u, coreType=%u.",
80 : ret, info_.coreId, info_.coreType);
81 :
82 1 : return ADUMP_SUCCESS;
83 4 : }
84 :
85 3 : int32_t DumpArgsCallback::QueryDfxIsTikInfo(rtFuncHandle funcHandle)
86 : {
87 3 : size_t isTikSize = 0;
88 3 : rtError_t rtRet = rtFunctionGetMetaInfoSize(funcHandle, RT_FUNCTION_TYPE_L0_EXCEPTION_DFX_IS_TIK, &isTikSize);
89 3 : if (rtRet != RT_ERROR_NONE || isTikSize == 0 || isTikSize < sizeof(uint32_t)) {
90 1 : IDE_LOGW("rtFunctionGetMetaInfoSize for dfxIsTik failed, will set dfxIsTik with false. "
91 : "rtRet=%d, kernelName=%s", rtRet, info_.kernelName);
92 1 : return ADUMP_FAILED;
93 : }
94 :
95 2 : std::vector<uint8_t> isTikBuffer(isTikSize);
96 2 : rtRet = rtFunctionGetMetaInfo(funcHandle, RT_FUNCTION_TYPE_L0_EXCEPTION_DFX_IS_TIK,
97 2 : isTikBuffer.data(), isTikSize);
98 2 : if (rtRet != RT_ERROR_NONE) {
99 1 : IDE_LOGW("rtFunctionGetMetaInfo for isTik failed, will set dfxIsTik with false. "
100 : "ret=%d, kernelName=%s, isTikSize=%zu.", rtRet, info_.kernelName, isTikSize);
101 1 : return ADUMP_FAILED;
102 : }
103 :
104 1 : uint32_t isTik = 0;
105 1 : isTik = *reinterpret_cast<uint32_t*>(isTikBuffer.data());
106 1 : isTik_ = (isTik == 1U) ? true : false;
107 1 : IDE_LOGI("Query dfxIsTik info success. kernelName=%s, dfxIsTik=%u", info_.kernelName, isTik);
108 1 : return ADUMP_SUCCESS;
109 2 : }
110 :
111 7 : int32_t DumpArgsCallback::QueryDfxInfo(std::vector<uint8_t> &dfxBuffer)
112 : {
113 7 : rtFuncHandle funcHandle = nullptr;
114 7 : rtError_t rtRet = rtBinaryGetFunctionByName(info_.bin, info_.kernelName, &funcHandle);
115 7 : IDE_CTRL_VALUE_FAILED(rtRet == RT_ERROR_NONE && funcHandle != nullptr, return ADUMP_FAILED,
116 : "rtBinaryGetFunctionByName failed, ret=%d, bin=%p, kernelName=%s.", rtRet, info_.bin, info_.kernelName);
117 :
118 6 : size_t dfxSize = 0;
119 6 : rtRet = rtFunctionGetMetaInfoSize(funcHandle, RT_FUNCTION_TYPE_DFX_TYPE, &dfxSize);
120 6 : IDE_CTRL_VALUE_FAILED(rtRet == RT_ERROR_NONE && dfxSize > 0, return ADUMP_FAILED,
121 : "rtFunctionGetMetaInfoSize failed, ret=%d, kernelName=%s, dfxSize=%zu.", rtRet, info_.kernelName, dfxSize);
122 5 : IDE_CTRL_VALUE_FAILED(dfxSize <= UINT16_MAX, return ADUMP_FAILED,
123 : "Dfx size exceeds uint16_t max, dfxSize=%zu", dfxSize);
124 :
125 4 : dfxBuffer.resize(dfxSize);
126 4 : rtRet = rtFunctionGetMetaInfo(funcHandle, RT_FUNCTION_TYPE_DFX_TYPE, dfxBuffer.data(), dfxSize);
127 4 : IDE_CTRL_VALUE_FAILED(rtRet == RT_ERROR_NONE, return ADUMP_FAILED,
128 : "rtFunctionGetMetaInfo failed, ret=%d, kernelName=%s, dfxSize=%zu.", rtRet, info_.kernelName, dfxSize);
129 :
130 3 : (void)QueryDfxIsTikInfo(funcHandle);
131 :
132 3 : IDE_LOGI("Query kernel dfx args info success. kernelName=%s, dfxSize=%zu, isTik=%d",
133 : info_.kernelName, dfxSize, isTik_);
134 3 : return ADUMP_SUCCESS;
135 : }
136 :
137 9 : int32_t DumpArgsCallback::DumpDfxArgs()
138 : {
139 9 : if (info_.argAddr == nullptr || info_.argSize == 0 || info_.bin == nullptr || info_.kernelName[0] == '\0') {
140 2 : return ADUMP_SUCCESS;
141 : }
142 :
143 7 : IDE_LOGI("Begin to dump dfx args tensors. argAddr=%p, argSize=%u, bin=%p, kernelName=%s",
144 : info_.argAddr, info_.argSize, info_.bin, info_.kernelName);
145 :
146 7 : int32_t ret = QueryDfxInfo(dfxBuffer_);
147 7 : IDE_CTRL_VALUE_FAILED(ret == ADUMP_SUCCESS, return ADUMP_FAILED, "Query kernel dfx info failed.");
148 :
149 3 : DfxArgsParser parser;
150 3 : ret = parser.Init(info_.argAddr, info_.argSize, dfxBuffer_.data(), static_cast<uint16_t>(dfxBuffer_.size()));
151 3 : IDE_CTRL_VALUE_FAILED(ret == ADUMP_SUCCESS, return ADUMP_FAILED, "Dfx args parser init failed.");
152 :
153 3 : parser.SetIsTik(isTik_);
154 3 : ret = parser.InitTensorModeInfo();
155 3 : IDE_CHECK_RET(ret, return ADUMP_FAILED);
156 :
157 0 : ret = parser.ParseAll();
158 0 : IDE_CHECK_RET(ret, return ADUMP_FAILED);
159 :
160 0 : tensorBuffer_ = parser.GetTensors();
161 0 : workspaces_ = parser.GetWorkspaces();
162 0 : logRecord_ = parser.GetLogRecords();
163 :
164 0 : IDE_LOGI("Dfx args tensors are parsed finished. tensor size=%zu, workspace size=%zu.",
165 : tensorBuffer_.size(), workspaces_.size());
166 :
167 0 : dumpFile_.SetTensorBuffer(tensorBuffer_);
168 0 : dumpFile_.SetWorkspaces(workspaces_);
169 :
170 0 : IDE_LOGI("End to dump dfx args tensors.");
171 0 : return ADUMP_SUCCESS;
172 3 : }
173 :
174 6 : int32_t DumpArgsCallback::DumpExtraTensors()
175 : {
176 6 : if (info_.extraTensorNum == 0) {
177 2 : return ADUMP_SUCCESS;
178 : }
179 4 : if (info_.extraTensorNum > EXCEPTION_DUMP_MAX_TENSOR_NUM) {
180 1 : IDE_LOGE("Extra tensor size exceeds the maximum limit. realSize=%u, maxSize=%u",
181 : info_.extraTensorNum, EXCEPTION_DUMP_MAX_TENSOR_NUM);
182 1 : return ADUMP_FAILED;
183 : }
184 :
185 3 : IDE_LOGI("Begin to dump extra tensors. extra tensor size=%u", info_.extraTensorNum);
186 :
187 3 : const uint32_t baseOffset = static_cast<uint32_t>(tensorBuffer_.size() + workspaces_.size());
188 3 : std::vector<TensorInfo> tensors(info_.extraTensor, info_.extraTensor + info_.extraTensorNum);
189 9 : for (uint32_t i = 0; i < info_.extraTensorNum; ++i) {
190 6 : tensors[i].argsOffSet = baseOffset + i;
191 : }
192 3 : dumpFile_.SetTensors(tensors, logRecord_);
193 :
194 3 : IDE_LOGI("End to dump extra tensors.");
195 3 : return ADUMP_SUCCESS;
196 3 : }
197 :
198 6 : int32_t DumpArgsCallback::Dump()
199 : {
200 6 : int32_t ret = dumpFile_.Dump(logRecord_);
201 6 : IDE_CTRL_VALUE_FAILED(ret == ADUMP_SUCCESS, return ADUMP_FAILED,
202 : "[Dump][Exception] Write callback exception to file failed, file: %s", dumpFilePath_.c_str());
203 :
204 5 : (void)mmChmod(dumpFilePath_.c_str(), M_IRUSR); // readonly, 400
205 5 : IDE_LOGE("[Dump][Exception] dump exception to file, file: %s", dumpFilePath_.c_str());
206 5 : return ADUMP_SUCCESS;
207 : }
208 :
209 0 : void DumpArgsCallback::RecordDumpLog(const std::string &log)
210 : {
211 0 : IDE_LOGE("%s", log.c_str());
212 0 : logRecord_.emplace_back(log + "\n");
213 0 : }
214 :
215 : } // namespace Adx
|
