Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "dump_operator.h"
12 : #include "runtime/rt.h"
13 : #include "str_utils.h"
14 : #include "sys_utils.h"
15 : #include "file.h"
16 : #include "path.h"
17 : #include "dump_memory.h"
18 : #include "dump_datatype.h"
19 : #include "dump_file.h"
20 : #include "log/hdc_log.h"
21 : #include "dump_manager.h"
22 :
23 : namespace Adx {
24 : namespace {
25 : const std::set<char> INVALID_FILE_NAME_CHAR = {' ', '.', '/', '\\'};
26 : constexpr char REPLACE_FILE_NAME_CHAR = '_';
27 : } // namespace
28 :
29 10 : bool OpIdentity::operator==(const OpIdentity &rhs) const
30 : {
31 10 : return rhs.deviceId == deviceId && rhs.taskId == taskId && rhs.streamId == streamId && rhs.contextId == contextId;
32 : }
33 :
34 10 : std::string OpIdentity::GetString() const
35 : {
36 10 : std::stringstream ss;
37 10 : ss << "stream_id:" << streamId << ", task_id:" << taskId << ", context_id:" << contextId << ", thread_id:";
38 20 : return ss.str();
39 10 : }
40 :
41 147 : DumpOperator::DumpOperator(const OperatorInfoV2 &opInfo)
42 : {
43 147 : Init(opInfo);
44 147 : }
45 :
46 148 : void DumpOperator::Init(const OperatorInfoV2 &opInfo)
47 : {
48 : // init base info
49 148 : opName_ = opInfo.opName;
50 148 : opType_ = opInfo.opType;
51 :
52 148 : identity_.taskId = opInfo.taskId;
53 148 : identity_.streamId = opInfo.streamId;
54 148 : identity_.deviceId = opInfo.deviceId;
55 148 : identity_.contextId = opInfo.contextId;
56 :
57 : // init device info
58 148 : deviceInfos_ = opInfo.deviceInfos;
59 :
60 : // init additional info
61 148 : additions_ = opInfo.additionalInfo;
62 296 : auto item = additions_.find(DUMP_ADDITIONAL_IS_HOST_ARGS);
63 148 : if (item != additions_.end()) {
64 4 : isHostArgs_ = item->second;
65 : }
66 148 : InitDeviceArgs();
67 :
68 : // init tensor or workspace
69 167 : for (const auto &tensorInfo : opInfo.tensorInfos)
70 : {
71 19 : if (tensorInfo.tensorAddr == nullptr || tensorInfo.tensorSize == 0 || tensorInfo.placement != TensorPlacement::kOnDeviceHbm) {
72 9 : continue;
73 : }
74 :
75 10 : if (tensorInfo.type == TensorType::INPUT) {
76 4 : inputTensors_.emplace_back(tensorInfo);
77 : }
78 6 : else if (tensorInfo.type == TensorType::OUTPUT) {
79 4 : outputTensors_.emplace_back(tensorInfo);
80 : }
81 2 : else if (tensorInfo.type == TensorType::WORKSPACE) {
82 2 : workspaces_.emplace_back(tensorInfo.tensorAddr, static_cast<uint64_t>(tensorInfo.tensorSize),
83 2 : tensorInfo.argsOffSet);
84 : }
85 : }
86 296 : return;
87 : }
88 :
89 148 : void DumpOperator::InitDeviceArgs()
90 : {
91 149 : for (const auto &deviceInfo : deviceInfos_) {
92 4 : if (deviceInfo.name == DEVICE_INFO_NAME_ARGS) {
93 3 : if (deviceInfo.addr == nullptr) {
94 1 : IDE_LOGW("The args before execute is null.");
95 3 : break;
96 : }
97 2 : void *argsAddr = nullptr;
98 2 : void *argsMem = nullptr;
99 2 : if (isHostArgs_ == "false") {
100 0 : argsMem = DumpMemory::CopyDeviceToHost(deviceInfo.addr, deviceInfo.length);
101 0 : if (argsMem == nullptr) {
102 0 : IDE_LOGE("Copy device args to host failed.");
103 0 : return;
104 : }
105 0 : argsAddr = argsMem;
106 : } else {
107 2 : argsAddr = deviceInfo.addr;
108 : }
109 4 : HOST_RT_MEMORY_GUARD(argsMem);
110 :
111 2 : uint64_t maxArgNum = deviceInfo.length / sizeof(uint64_t);
112 30 : for (uint64_t i = 0; i < maxArgNum; ++i) {
113 28 : hostArgs_.emplace_back(static_cast<void **>(argsAddr)[i]);
114 : }
115 2 : break;
116 2 : }
117 : }
118 : }
119 :
120 6 : bool DumpOperator::IsBelongTo(const OpIdentity &identity) const
121 : {
122 6 : return identity == identity_;
123 : }
124 :
125 80 : void DumpOperator::PrintAdditionInfo(const char *flag) const
126 : {
127 80 : std::string value;
128 160 : auto it = additions_.find(flag);
129 80 : if (it != additions_.end()) {
130 8 : value = it->second;
131 : }
132 80 : IDE_RUN_LOGI("[AIC_INFO] %s:%s", flag, value.c_str());
133 80 : }
134 :
135 8 : int32_t DumpOperator::LogExceptionInfo(const rtExceptionArgsInfo &argsInfo)
136 : {
137 8 : kernelCollector_.LoadKernelInfo(argsInfo);
138 :
139 : // log op basic info
140 8 : std::ostringstream oss;
141 : oss << "[AIC_INFO] node_name:" << opName_.c_str() << ", node_type:" << opType_.c_str() << ", "
142 8 : << identity_.GetString().c_str();
143 8 : IDE_RUN_LOGI("%s", oss.str().c_str());
144 8 : logRecord_.emplace_back(oss.str() + "\n");
145 8 : oss.str("");
146 : // log tensor info
147 9 : for (size_t i = 0U; i < inputTensors_.size(); ++i) {
148 1 : oss << "[Dump][Exception] begin to load input tensor, index:" << inputTensors_[i].GetArgsOffSet();
149 1 : RecordCurrentLog(oss);
150 1 : IDE_RUN_LOGI("[AIC_INFO] input:%zu;%s", i, GetTensorString(inputTensors_[i]).c_str());
151 1 : oss << "[Dump][Exception] end to load input tensor, index:" << inputTensors_[i].GetArgsOffSet();
152 1 : RecordCurrentLog(oss);
153 : }
154 9 : for (size_t i = 0U; i < outputTensors_.size(); ++i) {
155 1 : oss << "[Dump][Exception] begin to load output tensor, index:" << outputTensors_[i].GetArgsOffSet();
156 1 : RecordCurrentLog(oss);
157 1 : IDE_RUN_LOGI("[AIC_INFO] output:%zu;%s", i, GetTensorString(outputTensors_[i]).c_str());
158 1 : oss << "[Dump][Exception] end to load output tensor, index:" << outputTensors_[i].GetArgsOffSet();
159 1 : RecordCurrentLog(oss);
160 : }
161 9 : for (size_t i = 0U; i < workspaces_.size(); ++i) {
162 1 : oss << "[Dump][Exception] begin to load workspace, index:" << workspaces_[i].argsOffset;
163 1 : RecordCurrentLog(oss);
164 1 : oss << "[Dump][Exception] exception info dump args data, addr:" << workspaces_[i].addr
165 1 : << "; size:" << workspaces_[i].bytes << " bytes";
166 1 : RecordCurrentLog(oss);
167 1 : IDE_RUN_LOGI("[AIC_INFO] workspace:%zu;addr:%p;size:%llu bytes", i, workspaces_[i].addr, workspaces_[i].bytes);
168 1 : oss << "[Dump][Exception] end to load workspace, index:" << workspaces_[i].argsOffset;
169 1 : RecordCurrentLog(oss);
170 : }
171 :
172 : // log additional info
173 8 : PrintAdditionInfo(DUMP_ADDITIONAL_BLOCK_DIM);
174 8 : PrintAdditionInfo(DUMP_ADDITIONAL_WORKSPACE_BYTES);
175 8 : PrintAdditionInfo(DUMP_ADDITIONAL_WORKSPACE_ADDRS);
176 8 : PrintAdditionInfo(DUMP_ADDITIONAL_ALL_ATTRS);
177 8 : PrintAdditionInfo(DUMP_ADDITIONAL_DEV_FUNC);
178 8 : PrintAdditionInfo(DUMP_ADDITIONAL_TVM_MAGIC);
179 8 : PrintAdditionInfo(DUMP_ADDITIONAL_KERNEL_INFO);
180 8 : PrintAdditionInfo(DUMP_ADDITIONAL_TILING_KEY);
181 8 : PrintAdditionInfo(DUMP_ADDITIONAL_TILING_DATA);
182 8 : PrintAdditionInfo(DUMP_ADDITIONAL_OP_FILE_PATH);
183 :
184 : // log args
185 16 : return LogExceptionArgs(argsInfo);
186 8 : }
187 :
188 3 : int32_t DumpOperator::CopyOpKernelFile() const
189 : {
190 : // src file path
191 6 : auto it = additions_.find(DUMP_ADDITIONAL_OP_FILE_PATH);
192 5 : std::string opFilePath = it != additions_.cend() ? it->second : "./kernel_meta";
193 :
194 6 : it = additions_.find(DUMP_ADDITIONAL_DEV_FUNC);
195 3 : if (it == additions_.cend()) {
196 2 : IDE_LOGW("can't find [%s] in additional info.", DUMP_ADDITIONAL_DEV_FUNC);
197 2 : return ADUMP_FAILED;
198 : }
199 1 : std::string devFunc = it->second;
200 1 : std::string opFileName = devFunc.substr(0, devFunc.rfind("__"));
201 3 : Path srcFilePath = Path(opFilePath).Concat(opFileName).AddExtension(".o");
202 1 : if (!srcFilePath.RealPath()) {
203 1 : IDE_LOGW("Can not get realpath for src op file %s.", srcFilePath.GetCString());
204 1 : return ADUMP_FAILED;
205 : }
206 :
207 : // dst file path
208 0 : std::string currentDir = SysUtils::GetCurrentWorkDir();
209 0 : if (currentDir.empty()) {
210 0 : IDE_LOGE("Get current program work dir failed.");
211 0 : return ADUMP_FAILED;
212 : }
213 0 : Path dstFilePath = Path(currentDir).Concat(opFileName).AddExtension(".o");
214 :
215 : // copy file
216 0 : const auto ret = File::Copy(srcFilePath.GetString(), dstFilePath.GetString());
217 0 : if (ret != ADUMP_SUCCESS) {
218 0 : IDE_LOGE("Copy op kernel file from %s to %s failed.", srcFilePath.GetCString(), dstFilePath.GetCString());
219 0 : return ADUMP_FAILED;
220 : }
221 0 : return ADUMP_SUCCESS;
222 3 : }
223 :
224 4 : int32_t DumpOperator::RefreshAddrs(const rtExceptionArgsInfo &argsInfo)
225 : {
226 4 : void *hostMem = DumpMemory::CopyDeviceToHost(argsInfo.argAddr, argsInfo.argsize);
227 4 : if (hostMem == nullptr) {
228 1 : IDE_LOGE("Copy device args to host failed, ptr: %p, size: %lu bytes.", argsInfo.argAddr, argsInfo.argsize);
229 1 : return ADUMP_FAILED;
230 : }
231 6 : HOST_RT_MEMORY_GUARD(hostMem);
232 :
233 3 : IDE_LOGI("Op %s type %s refresh addr.", opName_.c_str(), opType_.c_str());
234 3 : void **argsOnHost = static_cast<void **>(hostMem);
235 3 : size_t maxArgNum = argsInfo.argsize / sizeof(uint64_t);
236 5 : for (size_t i = 0U; i < inputTensors_.size(); i++) {
237 2 : const void *oriAddr = inputTensors_[i].GetAddress();
238 2 : if (maxArgNum <= inputTensors_[i].GetArgsOffSet()) {
239 1 : IDE_LOGW("Tensor args offset[%u] is larger than max arg num[%llu].", inputTensors_[i].GetArgsOffSet(),
240 : maxArgNum);
241 1 : continue;
242 : }
243 1 : const void *refreshAddr = *(argsOnHost + inputTensors_[i].GetArgsOffSet());
244 1 : IDE_LOGI("Input.%zu addr refresh from %p to %p.", i, oriAddr, refreshAddr);
245 1 : inputTensors_[i].SetAddress(refreshAddr);
246 : }
247 :
248 5 : for (size_t i = 0U; i < outputTensors_.size(); i++) {
249 2 : const void *oriAddr = outputTensors_[i].GetAddress();
250 2 : if (maxArgNum <= outputTensors_[i].GetArgsOffSet()) {
251 0 : IDE_LOGW("Tensor args offset[%u] is larger than max arg num[%llu].", outputTensors_[i].GetArgsOffSet(),
252 : maxArgNum);
253 0 : continue;
254 : }
255 2 : const void *refreshAddr = *(argsOnHost + outputTensors_[i].GetArgsOffSet());
256 2 : IDE_LOGI("Output.%zu addr refresh from %p to %p.", i, oriAddr, refreshAddr);
257 2 : outputTensors_[i].SetAddress(refreshAddr);
258 : }
259 3 : return ADUMP_SUCCESS;
260 3 : }
261 :
262 2 : int32_t DumpOperator::DumpExceptionFile(const uint32_t deviceId, const std::string &dumpPath)
263 : {
264 2 : std::string dumpFilePath = GetDumpFilePath(dumpPath);
265 2 : IDE_LOGI("[Dump][Exception] The exception dump file path is %s", dumpFilePath.c_str());
266 :
267 2 : DumpFile dumpFile(deviceId, dumpFilePath);
268 2 : dumpFile.SetHeader(opName_);
269 2 : dumpFile.SetInputTensors(inputTensors_);
270 2 : dumpFile.SetOutputTensors(outputTensors_);
271 2 : dumpFile.SetWorkspaces(workspaces_);
272 :
273 2 : AdxLogFlush();
274 2 : const int32_t ret = dumpFile.Dump(logRecord_);
275 2 : if (ret != ADUMP_SUCCESS) {
276 0 : IDE_LOGE("[Dump][Exception] write exception to file failed, file: %s", dumpFilePath.c_str());
277 0 : return ADUMP_FAILED;
278 : }
279 2 : IDE_LOGE("[Dump][Exception] dump exception to file, file: %s", dumpFilePath.c_str());
280 2 : IDE_LOGI("[Dump][Exception] Dump exception info success.");
281 2 : return ADUMP_SUCCESS;
282 2 : }
283 :
284 2 : int32_t DumpOperator::DumpException(const uint32_t deviceId, const std::string &dumpPath)
285 : {
286 2 : int32_t ret = ADUMP_SUCCESS;
287 2 : if (DumpExceptionFile(deviceId, dumpPath) != ADUMP_SUCCESS) {
288 0 : ret = ADUMP_FAILED;
289 : }
290 2 : if (kernelCollector_.LoadKernelBinBuffer() != ADUMP_SUCCESS){
291 0 : ret = ADUMP_FAILED;
292 : }
293 : // 先同步落 _host.o，再做慢的 kernel_meta 搜索拷贝。落盘失败需传播，保持与拆分前一致的错误可观测性。
294 2 : if (kernelCollector_.DumpHostKernelBin(dumpPath) != ADUMP_SUCCESS) {
295 0 : ret = ADUMP_FAILED;
296 : }
297 2 : if (kernelCollector_.StartCollectKernel(dumpPath) != ADUMP_SUCCESS) {
298 0 : ret = ADUMP_FAILED;
299 : }
300 2 : return ret;
301 : }
302 :
303 0 : bool DumpOperator::IsTvmOperator() const
304 : {
305 0 : const auto it = additions_.find(DUMP_ADDITIONAL_IMPLY_TYPE);
306 0 : if (it == additions_.cend()) {
307 0 : return false;
308 : }
309 :
310 0 : int32_t implType = 0;
311 0 : if (!StrUtils::ToInteger(it->second, implType)) {
312 0 : IDE_LOGW("Convert imply_type string %s to int failed.", it->second.c_str());
313 0 : return false;
314 : }
315 0 : return static_cast<ImplyType>(implType) == ImplyType::TVM;
316 : }
317 :
318 2 : std::string DumpOperator::GetTensorString(const DumpTensor &tensor)
319 : {
320 2 : std::stringstream content;
321 4 : content << "shape:" << StrUtils::ToString(tensor.GetShape()) << ";"
322 4 : << "format:" << DumpDataType::FormatToSerialString(static_cast<int32_t>(tensor.GetFormat())) << ";"
323 4 : << "dtype:" << DumpDataType::DataTypeToSerialString(static_cast<int32_t>(tensor.GetDataType())) << ";"
324 8 : << "addr:" << tensor.GetAddress() << ";"
325 2 : << "size:" << tensor.GetSize() << " bytes";
326 2 : std::ostringstream oss;
327 2 : oss << "[Dump][Exception] exception info dump args data, addr:" << tensor.GetAddress()
328 2 : << "; size:" << tensor.GetSize() << " bytes";
329 2 : RecordCurrentLog(oss);
330 4 : return content.str();
331 2 : }
332 :
333 8 : int32_t DumpOperator::LogExceptionArgs(const rtExceptionArgsInfo &argsInfo) const
334 : {
335 8 : if (hostArgs_.size() == 0) {
336 7 : IDE_LOGI("Op %s args is empty, skip log args.", opName_.c_str());
337 : } else {
338 1 : void *const *argsMem = hostArgs_.data();
339 2 : PrintLog(argsMem, hostArgs_.size(), std::string(DEVICE_INFO_NAME_ARGS));
340 : }
341 :
342 8 : if (argsInfo.argAddr == nullptr) {
343 7 : IDE_LOGE("exception callback argAddr is null");
344 7 : return ADUMP_SUCCESS;
345 : }
346 1 : void *hostMem = DumpMemory::CopyDeviceToHost(argsInfo.argAddr, argsInfo.argsize);
347 1 : if (hostMem == nullptr) {
348 0 : IDE_LOGE("Copy device args to host failed, addr: %p, size: %u.", argsInfo.argAddr, argsInfo.argsize);
349 0 : return ADUMP_FAILED;
350 : }
351 2 : HOST_RT_MEMORY_GUARD(hostMem);
352 1 : void *const *argsOnHost = static_cast<void *const *>(hostMem);
353 1 : size_t argNum = argsInfo.argsize / sizeof(uint64_t);
354 1 : PrintLog(argsOnHost, argNum, "args after execute");
355 :
356 1 : return ADUMP_SUCCESS;
357 1 : }
358 :
359 2 : void DumpOperator::PrintLog(void *const *argsOnHost, size_t argNum, const std::string &tag) const
360 : {
361 2 : std::stringstream strStream;
362 2 : const uint32_t printNumEachTime = 20;
363 2 : uint32_t count = argNum / printNumEachTime;
364 2 : uint32_t left = argNum % printNumEachTime;
365 :
366 3 : for (uint32_t i = 0; i < count; ++i) {
367 1 : uint32_t begin = printNumEachTime * i;
368 1 : uint32_t end = printNumEachTime * (i + 1);
369 21 : for (uint32_t j = begin; j < end; ++j) {
370 20 : strStream << *(argsOnHost + j) << " ";
371 : }
372 1 : IDE_RUN_LOGI("[AIC_INFO] %s(%u to %u): %s", tag.c_str(), begin, end - 1, strStream.str().c_str());
373 2 : strStream.str("");
374 : }
375 :
376 2 : if (left != 0 && argNum > 0) {
377 2 : uint32_t begin = printNumEachTime * count;
378 2 : uint32_t end = argNum;
379 11 : for (uint32_t j = begin; j < end; ++j) {
380 9 : strStream << *(argsOnHost + j) << " ";
381 : }
382 2 : IDE_RUN_LOGI("[AIC_INFO] %s(%u to %u): %s", tag.c_str(), begin, end - 1, strStream.str().c_str());
383 : }
384 2 : }
385 :
386 2 : std::string DumpOperator::GetDumpFilePath(const std::string &dumpPath) const
387 : {
388 2 : std::string opType = StrUtils::Replace(opType_, INVALID_FILE_NAME_CHAR, REPLACE_FILE_NAME_CHAR);
389 2 : std::string opName = StrUtils::Replace(opName_, INVALID_FILE_NAME_CHAR, REPLACE_FILE_NAME_CHAR);
390 2 : uint32_t taskId = identity_.taskId;
391 : std::string dumpFileName =
392 2 : opType + "." + opName + "." + std::to_string(taskId) + "." + SysUtils::GetCurrentTimeWithMillisecond();
393 2 : Path dumpFilePath(dumpPath);
394 2 : dumpFilePath.Concat(dumpFileName);
395 4 : return dumpFilePath.GetString();
396 2 : }
397 :
398 9 : void DumpOperator::RecordCurrentLog(std::ostringstream &oss)
399 : {
400 9 : IDE_LOGE("%s", oss.str().c_str());
401 9 : logRecord_.emplace_back(oss.str() + "\n");
402 9 : oss.str("");
403 9 : }
404 : } // namespace Adx
|
