Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "dump_operator.h"
12 : #include "runtime/rt.h"
13 : #include "str_utils.h"
14 : #include "sys_utils.h"
15 : #include "file.h"
16 : #include "path.h"
17 : #include "dump_memory.h"
18 : #include "dump_datatype.h"
19 : #include "dump_file.h"
20 : #include "log/hdc_log.h"
21 : #include "dump_manager.h"
22 :
23 : namespace Adx {
24 : namespace {
25 : const std::set<char> INVALID_FILE_NAME_CHAR = {' ', '.', '/', '\\'};
26 : constexpr char REPLACE_FILE_NAME_CHAR = '_';
27 : } // namespace
28 :
29 10 : bool OpIdentity::operator==(const OpIdentity& rhs) const
30 : {
31 10 : return rhs.deviceId == deviceId && rhs.taskId == taskId && rhs.streamId == streamId && rhs.contextId == contextId;
32 : }
33 :
34 10 : std::string OpIdentity::GetString() const
35 : {
36 10 : std::stringstream ss;
37 10 : ss << "stream_id:" << streamId << ", task_id:" << taskId << ", context_id:" << contextId << ", thread_id:";
38 20 : return ss.str();
39 10 : }
40 :
41 147 : DumpOperator::DumpOperator(const OperatorInfoV2& opInfo) { Init(opInfo); }
42 :
43 148 : void DumpOperator::Init(const OperatorInfoV2& opInfo)
44 : {
45 : // init base info
46 148 : opName_ = opInfo.opName;
47 148 : opType_ = opInfo.opType;
48 :
49 148 : identity_.taskId = opInfo.taskId;
50 148 : identity_.streamId = opInfo.streamId;
51 148 : identity_.deviceId = opInfo.deviceId;
52 148 : identity_.contextId = opInfo.contextId;
53 :
54 : // init device info
55 148 : deviceInfos_ = opInfo.deviceInfos;
56 :
57 : // init additional info
58 148 : additions_ = opInfo.additionalInfo;
59 296 : auto item = additions_.find(DUMP_ADDITIONAL_IS_HOST_ARGS);
60 148 : if (item != additions_.end()) {
61 4 : isHostArgs_ = item->second;
62 : }
63 148 : InitDeviceArgs();
64 :
65 : // init tensor or workspace
66 315 : for (const auto& tensorInfo : opInfo.tensorInfos) {
67 19 : if (tensorInfo.tensorAddr == nullptr || tensorInfo.tensorSize == 0 ||
68 17 : tensorInfo.placement != TensorPlacement::kOnDeviceHbm) {
69 9 : continue;
70 : }
71 :
72 10 : if (tensorInfo.type == TensorType::INPUT) {
73 4 : inputTensors_.emplace_back(tensorInfo);
74 6 : } else if (tensorInfo.type == TensorType::OUTPUT) {
75 4 : outputTensors_.emplace_back(tensorInfo);
76 2 : } else if (tensorInfo.type == TensorType::WORKSPACE) {
77 2 : workspaces_.emplace_back(
78 2 : tensorInfo.tensorAddr, static_cast<uint64_t>(tensorInfo.tensorSize), tensorInfo.argsOffSet);
79 : }
80 : }
81 296 : return;
82 : }
83 :
84 148 : void DumpOperator::InitDeviceArgs()
85 : {
86 297 : for (const auto& deviceInfo : deviceInfos_) {
87 4 : if (deviceInfo.name == DEVICE_INFO_NAME_ARGS) {
88 3 : if (deviceInfo.addr == nullptr) {
89 1 : IDE_LOGW("The args before execute is null.");
90 3 : break;
91 : }
92 2 : void* argsAddr = nullptr;
93 2 : void* argsMem = nullptr;
94 2 : if (isHostArgs_ == "false") {
95 0 : argsMem = DumpMemory::CopyDeviceToHost(deviceInfo.addr, deviceInfo.length);
96 0 : if (argsMem == nullptr) {
97 0 : IDE_LOGE("Copy device args to host failed.");
98 0 : return;
99 : }
100 0 : argsAddr = argsMem;
101 : } else {
102 2 : argsAddr = deviceInfo.addr;
103 : }
104 4 : HOST_RT_MEMORY_GUARD(argsMem);
105 :
106 2 : uint64_t maxArgNum = deviceInfo.length / sizeof(uint64_t);
107 30 : for (uint64_t i = 0; i < maxArgNum; ++i) {
108 28 : hostArgs_.emplace_back(static_cast<void**>(argsAddr)[i]);
109 : }
110 2 : break;
111 2 : }
112 : }
113 : }
114 :
115 6 : bool DumpOperator::IsBelongTo(const OpIdentity& identity) const { return identity == identity_; }
116 :
117 80 : void DumpOperator::PrintAdditionInfo(const char* flag) const
118 : {
119 80 : std::string value;
120 160 : auto it = additions_.find(flag);
121 80 : if (it != additions_.end()) {
122 8 : value = it->second;
123 : }
124 80 : IDE_RUN_LOGI("[AIC_INFO] %s:%s", flag, value.c_str());
125 80 : }
126 :
127 8 : int32_t DumpOperator::LogExceptionInfo(const rtExceptionArgsInfo& argsInfo)
128 : {
129 8 : kernelCollector_.LoadKernelInfo(argsInfo);
130 :
131 : // log op basic info
132 8 : std::ostringstream oss;
133 : oss << "[AIC_INFO] node_name:" << opName_.c_str() << ", node_type:" << opType_.c_str() << ", "
134 8 : << identity_.GetString().c_str();
135 8 : IDE_RUN_LOGI("%s", oss.str().c_str());
136 8 : logRecord_.emplace_back(oss.str() + "\n");
137 8 : oss.str("");
138 : // log tensor info
139 9 : for (size_t i = 0U; i < inputTensors_.size(); ++i) {
140 1 : oss << "[Dump][Exception] begin to load input tensor, index:" << inputTensors_[i].GetArgsOffSet();
141 1 : RecordCurrentLog(oss);
142 1 : IDE_RUN_LOGI("[AIC_INFO] input:%zu;%s", i, GetTensorString(inputTensors_[i]).c_str());
143 1 : oss << "[Dump][Exception] end to load input tensor, index:" << inputTensors_[i].GetArgsOffSet();
144 1 : RecordCurrentLog(oss);
145 : }
146 9 : for (size_t i = 0U; i < outputTensors_.size(); ++i) {
147 1 : oss << "[Dump][Exception] begin to load output tensor, index:" << outputTensors_[i].GetArgsOffSet();
148 1 : RecordCurrentLog(oss);
149 1 : IDE_RUN_LOGI("[AIC_INFO] output:%zu;%s", i, GetTensorString(outputTensors_[i]).c_str());
150 1 : oss << "[Dump][Exception] end to load output tensor, index:" << outputTensors_[i].GetArgsOffSet();
151 1 : RecordCurrentLog(oss);
152 : }
153 9 : for (size_t i = 0U; i < workspaces_.size(); ++i) {
154 1 : oss << "[Dump][Exception] begin to load workspace, index:" << workspaces_[i].argsOffset;
155 1 : RecordCurrentLog(oss);
156 1 : oss << "[Dump][Exception] exception info dump args data, addr:" << workspaces_[i].addr
157 1 : << "; size:" << workspaces_[i].bytes << " bytes";
158 1 : RecordCurrentLog(oss);
159 1 : IDE_RUN_LOGI("[AIC_INFO] workspace:%zu;addr:%p;size:%llu bytes", i, workspaces_[i].addr, workspaces_[i].bytes);
160 1 : oss << "[Dump][Exception] end to load workspace, index:" << workspaces_[i].argsOffset;
161 1 : RecordCurrentLog(oss);
162 : }
163 :
164 : // log additional info
165 8 : PrintAdditionInfo(DUMP_ADDITIONAL_BLOCK_DIM);
166 8 : PrintAdditionInfo(DUMP_ADDITIONAL_WORKSPACE_BYTES);
167 8 : PrintAdditionInfo(DUMP_ADDITIONAL_WORKSPACE_ADDRS);
168 8 : PrintAdditionInfo(DUMP_ADDITIONAL_ALL_ATTRS);
169 8 : PrintAdditionInfo(DUMP_ADDITIONAL_DEV_FUNC);
170 8 : PrintAdditionInfo(DUMP_ADDITIONAL_TVM_MAGIC);
171 8 : PrintAdditionInfo(DUMP_ADDITIONAL_KERNEL_INFO);
172 8 : PrintAdditionInfo(DUMP_ADDITIONAL_TILING_KEY);
173 8 : PrintAdditionInfo(DUMP_ADDITIONAL_TILING_DATA);
174 8 : PrintAdditionInfo(DUMP_ADDITIONAL_OP_FILE_PATH);
175 :
176 : // log args
177 16 : return LogExceptionArgs(argsInfo);
178 8 : }
179 :
180 3 : int32_t DumpOperator::CopyOpKernelFile() const
181 : {
182 : // src file path
183 6 : auto it = additions_.find(DUMP_ADDITIONAL_OP_FILE_PATH);
184 5 : std::string opFilePath = it != additions_.cend() ? it->second : "./kernel_meta";
185 :
186 6 : it = additions_.find(DUMP_ADDITIONAL_DEV_FUNC);
187 3 : if (it == additions_.cend()) {
188 2 : IDE_LOGW("can't find [%s] in additional info.", DUMP_ADDITIONAL_DEV_FUNC);
189 2 : return ADUMP_FAILED;
190 : }
191 1 : std::string devFunc = it->second;
192 1 : std::string opFileName = devFunc.substr(0, devFunc.rfind("__"));
193 3 : Path srcFilePath = Path(opFilePath).Concat(opFileName).AddExtension(".o");
194 1 : if (!srcFilePath.RealPath()) {
195 1 : IDE_LOGW("Can not get realpath for src op file %s.", srcFilePath.GetCString());
196 1 : return ADUMP_FAILED;
197 : }
198 :
199 : // dst file path
200 0 : std::string currentDir = SysUtils::GetCurrentWorkDir();
201 0 : if (currentDir.empty()) {
202 0 : IDE_LOGE("Get current program work dir failed.");
203 0 : return ADUMP_FAILED;
204 : }
205 0 : Path dstFilePath = Path(currentDir).Concat(opFileName).AddExtension(".o");
206 :
207 : // copy file
208 0 : const auto ret = File::Copy(srcFilePath.GetString(), dstFilePath.GetString());
209 0 : if (ret != ADUMP_SUCCESS) {
210 0 : IDE_LOGE("Copy op kernel file from %s to %s failed.", srcFilePath.GetCString(), dstFilePath.GetCString());
211 0 : return ADUMP_FAILED;
212 : }
213 0 : return ADUMP_SUCCESS;
214 3 : }
215 :
216 4 : int32_t DumpOperator::RefreshAddrs(const rtExceptionArgsInfo& argsInfo)
217 : {
218 4 : void* hostMem = DumpMemory::CopyDeviceToHost(argsInfo.argAddr, argsInfo.argsize);
219 4 : if (hostMem == nullptr) {
220 1 : IDE_LOGE("Copy device args to host failed, ptr: %p, size: %lu bytes.", argsInfo.argAddr, argsInfo.argsize);
221 1 : return ADUMP_FAILED;
222 : }
223 6 : HOST_RT_MEMORY_GUARD(hostMem);
224 :
225 3 : IDE_LOGI("Op %s type %s refresh addr.", opName_.c_str(), opType_.c_str());
226 3 : void** argsOnHost = static_cast<void**>(hostMem);
227 3 : size_t maxArgNum = argsInfo.argsize / sizeof(uint64_t);
228 5 : for (size_t i = 0U; i < inputTensors_.size(); i++) {
229 2 : const void* oriAddr = inputTensors_[i].GetAddress();
230 2 : if (maxArgNum <= inputTensors_[i].GetArgsOffSet()) {
231 1 : IDE_LOGW(
232 : "Tensor args offset[%u] is larger than max arg num[%llu].", inputTensors_[i].GetArgsOffSet(),
233 : maxArgNum);
234 1 : continue;
235 : }
236 1 : const void* refreshAddr = *(argsOnHost + inputTensors_[i].GetArgsOffSet());
237 1 : IDE_LOGI("Input.%zu addr refresh from %p to %p.", i, oriAddr, refreshAddr);
238 1 : inputTensors_[i].SetAddress(refreshAddr);
239 : }
240 :
241 5 : for (size_t i = 0U; i < outputTensors_.size(); i++) {
242 2 : const void* oriAddr = outputTensors_[i].GetAddress();
243 2 : if (maxArgNum <= outputTensors_[i].GetArgsOffSet()) {
244 0 : IDE_LOGW(
245 : "Tensor args offset[%u] is larger than max arg num[%llu].", outputTensors_[i].GetArgsOffSet(),
246 : maxArgNum);
247 0 : continue;
248 : }
249 2 : const void* refreshAddr = *(argsOnHost + outputTensors_[i].GetArgsOffSet());
250 2 : IDE_LOGI("Output.%zu addr refresh from %p to %p.", i, oriAddr, refreshAddr);
251 2 : outputTensors_[i].SetAddress(refreshAddr);
252 : }
253 3 : return ADUMP_SUCCESS;
254 3 : }
255 :
256 2 : int32_t DumpOperator::DumpExceptionFile(const uint32_t deviceId, const std::string& dumpPath)
257 : {
258 2 : std::string dumpFilePath = GetDumpFilePath(dumpPath);
259 2 : IDE_LOGI("[Dump][Exception] The exception dump file path is %s", dumpFilePath.c_str());
260 :
261 2 : DumpFile dumpFile(deviceId, dumpFilePath);
262 2 : dumpFile.SetHeader(opName_);
263 2 : dumpFile.SetInputTensors(inputTensors_);
264 2 : dumpFile.SetOutputTensors(outputTensors_);
265 2 : dumpFile.SetWorkspaces(workspaces_);
266 :
267 2 : AdxLogFlush();
268 2 : const int32_t ret = dumpFile.Dump(logRecord_);
269 2 : if (ret != ADUMP_SUCCESS) {
270 0 : IDE_LOGE("[Dump][Exception] write exception to file failed, file: %s", dumpFilePath.c_str());
271 0 : return ADUMP_FAILED;
272 : }
273 2 : IDE_LOGE("[Dump][Exception] dump exception to file, file: %s", dumpFilePath.c_str());
274 2 : IDE_LOGI("[Dump][Exception] Dump exception info success.");
275 2 : return ADUMP_SUCCESS;
276 2 : }
277 :
278 2 : int32_t DumpOperator::DumpException(const uint32_t deviceId, const std::string& dumpPath)
279 : {
280 2 : int32_t ret = ADUMP_SUCCESS;
281 2 : if (DumpExceptionFile(deviceId, dumpPath) != ADUMP_SUCCESS) {
282 0 : ret = ADUMP_FAILED;
283 : }
284 2 : if (kernelCollector_.LoadKernelBinBuffer() != ADUMP_SUCCESS) {
285 0 : ret = ADUMP_FAILED;
286 : }
287 : // 先同步落 _host.o，再做慢的 kernel_meta 搜索拷贝。落盘失败需传播，保持与拆分前一致的错误可观测性。
288 2 : std::string hostOPath;
289 2 : if (kernelCollector_.DumpHostKernelBin(dumpPath, hostOPath) != ADUMP_SUCCESS) {
290 0 : ret = ADUMP_FAILED;
291 : }
292 2 : if (kernelCollector_.StartCollectKernel(dumpPath) != ADUMP_SUCCESS) {
293 0 : ret = ADUMP_FAILED;
294 : }
295 2 : return ret;
296 2 : }
297 :
298 0 : bool DumpOperator::IsTvmOperator() const
299 : {
300 0 : const auto it = additions_.find(DUMP_ADDITIONAL_IMPLY_TYPE);
301 0 : if (it == additions_.cend()) {
302 0 : return false;
303 : }
304 :
305 0 : int32_t implType = 0;
306 0 : if (!StrUtils::ToInteger(it->second, implType)) {
307 0 : IDE_LOGW("Convert imply_type string %s to int failed.", it->second.c_str());
308 0 : return false;
309 : }
310 0 : return static_cast<ImplyType>(implType) == ImplyType::TVM;
311 : }
312 :
313 2 : std::string DumpOperator::GetTensorString(const DumpTensor& tensor)
314 : {
315 2 : std::stringstream content;
316 4 : content << "shape:" << StrUtils::ToString(tensor.GetShape()) << ";"
317 4 : << "format:" << DumpDataType::FormatToSerialString(static_cast<int32_t>(tensor.GetFormat())) << ";"
318 4 : << "dtype:" << DumpDataType::DataTypeToSerialString(static_cast<int32_t>(tensor.GetDataType())) << ";"
319 8 : << "addr:" << tensor.GetAddress() << ";"
320 2 : << "size:" << tensor.GetSize() << " bytes";
321 2 : std::ostringstream oss;
322 2 : oss << "[Dump][Exception] exception info dump args data, addr:" << tensor.GetAddress()
323 2 : << "; size:" << tensor.GetSize() << " bytes";
324 2 : RecordCurrentLog(oss);
325 4 : return content.str();
326 2 : }
327 :
328 8 : int32_t DumpOperator::LogExceptionArgs(const rtExceptionArgsInfo& argsInfo) const
329 : {
330 8 : if (hostArgs_.size() == 0) {
331 7 : IDE_LOGI("Op %s args is empty, skip log args.", opName_.c_str());
332 : } else {
333 1 : void* const* argsMem = hostArgs_.data();
334 2 : PrintLog(argsMem, hostArgs_.size(), std::string(DEVICE_INFO_NAME_ARGS));
335 : }
336 :
337 8 : if (argsInfo.argAddr == nullptr) {
338 7 : IDE_LOGE("exception callback argAddr is null");
339 7 : return ADUMP_SUCCESS;
340 : }
341 1 : void* hostMem = DumpMemory::CopyDeviceToHost(argsInfo.argAddr, argsInfo.argsize);
342 1 : if (hostMem == nullptr) {
343 0 : IDE_LOGE("Copy device args to host failed, addr: %p, size: %u.", argsInfo.argAddr, argsInfo.argsize);
344 0 : return ADUMP_FAILED;
345 : }
346 2 : HOST_RT_MEMORY_GUARD(hostMem);
347 1 : void* const* argsOnHost = static_cast<void* const*>(hostMem);
348 1 : size_t argNum = argsInfo.argsize / sizeof(uint64_t);
349 1 : PrintLog(argsOnHost, argNum, "args after execute");
350 :
351 1 : return ADUMP_SUCCESS;
352 1 : }
353 :
354 2 : void DumpOperator::PrintLog(void* const* argsOnHost, size_t argNum, const std::string& tag) const
355 : {
356 2 : std::stringstream strStream;
357 2 : const uint32_t printNumEachTime = 20;
358 2 : uint32_t count = argNum / printNumEachTime;
359 2 : uint32_t left = argNum % printNumEachTime;
360 :
361 3 : for (uint32_t i = 0; i < count; ++i) {
362 1 : uint32_t begin = printNumEachTime * i;
363 1 : uint32_t end = printNumEachTime * (i + 1);
364 21 : for (uint32_t j = begin; j < end; ++j) {
365 20 : strStream << *(argsOnHost + j) << " ";
366 : }
367 1 : IDE_RUN_LOGI("[AIC_INFO] %s(%u to %u): %s", tag.c_str(), begin, end - 1, strStream.str().c_str());
368 2 : strStream.str("");
369 : }
370 :
371 2 : if (left != 0 && argNum > 0) {
372 2 : uint32_t begin = printNumEachTime * count;
373 2 : uint32_t end = argNum;
374 11 : for (uint32_t j = begin; j < end; ++j) {
375 9 : strStream << *(argsOnHost + j) << " ";
376 : }
377 2 : IDE_RUN_LOGI("[AIC_INFO] %s(%u to %u): %s", tag.c_str(), begin, end - 1, strStream.str().c_str());
378 : }
379 2 : }
380 :
381 2 : std::string DumpOperator::GetDumpFilePath(const std::string& dumpPath) const
382 : {
383 2 : std::string opType = StrUtils::Replace(opType_, INVALID_FILE_NAME_CHAR, REPLACE_FILE_NAME_CHAR);
384 2 : std::string opName = StrUtils::Replace(opName_, INVALID_FILE_NAME_CHAR, REPLACE_FILE_NAME_CHAR);
385 2 : uint32_t taskId = identity_.taskId;
386 : std::string dumpFileName =
387 2 : opType + "." + opName + "." + std::to_string(taskId) + "." + SysUtils::GetCurrentTimeWithMillisecond();
388 2 : Path dumpFilePath(dumpPath);
389 2 : dumpFilePath.Concat(dumpFileName);
390 4 : return dumpFilePath.GetString();
391 2 : }
392 :
393 9 : void DumpOperator::RecordCurrentLog(std::ostringstream& oss)
394 : {
395 9 : IDE_LOGE("%s", oss.str().c_str());
396 9 : logRecord_.emplace_back(oss.str() + "\n");
397 9 : oss.str("");
398 9 : }
399 : } // namespace Adx
|
