Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 : #include <set>
11 : #include <cmath>
12 : #include "dump_args.h"
13 : #include "dfx_args_parser.h"
14 : #include <cinttypes>
15 : #include "path.h"
16 : #include "dump_file.h"
17 : #include "sys_utils.h"
18 : #include "dump_memory.h"
19 : #include "log/hdc_log.h"
20 : #include "exception_info_common.h"
21 : #include "dump_tensor_plugin.h"
22 : #include "adump_dsmi.h"
23 : #include "log/adx_log.h"
24 :
25 : namespace Adx {
26 : namespace {
27 : constexpr uint32_t ATOMIC_AND_INPUT_NUM_OFFSET = 2; // 2 - atomic index | input num(or total context size num)
28 : constexpr uint32_t CONTEXTID_AND_INPUT_NUM_OFFSET = 2; // 2 - context id | input num
29 : constexpr uint32_t SKIP_NUM_SHIFT_BITS = 32;
30 : constexpr uint64_t SKIP_NUM_MASK = 0x0FFFFFFFF00000000; // high 32 bits
31 : constexpr uint64_t INPUT_NUM_MASK = 0x0FFFFFFFF; // low 32 bits
32 : constexpr uint64_t SIZE_TYPE_MASK = 0x0FF00000000000000; // high 8 bits
33 : constexpr uint64_t SIZE_MASK = 0x0FFFFFFFFFFFFFF; // low 56 bits
34 : constexpr uint32_t SIZE_TYPE_SHIFT_BITS = 56;
35 : constexpr uint8_t NORMAL_TENSOR = 0;
36 : constexpr uint8_t NORMAL_PTR_TENSOR = 1;
37 : constexpr uint8_t SHAPE_PTR_TENSOR = 2;
38 : constexpr uint8_t TILING_DATA_PTR = 3;
39 : constexpr uint8_t WORKSPACE_TENSOR = 4;
40 : constexpr uint32_t ARGS_PER_STRING_MAX_LEN = 20;
41 : } // namespace
42 :
43 : template <typename T>
44 6 : int32_t DumpArgs::GetPointerValueByBigEndian(
45 : const uint8_t** ptr, T& value, uint64_t& currentDfxSize, uint16_t totalDfxSize) const
46 : {
47 6 : uint16_t bitOfByte = 8;
48 18 : for (size_t i = 0; i < sizeof(T); ++i) {
49 12 : if (currentDfxSize >= totalDfxSize) {
50 0 : IDE_LOGE(
51 : "The dfx data size[%" PRIu64 "] is larger than the total dfx size[%" PRIu16 "].", currentDfxSize,
52 : totalDfxSize);
53 0 : return ADUMP_FAILED;
54 : }
55 12 : value = (value) | (static_cast<uint64_t>(**ptr) << ((sizeof(T) - i - 1) * bitOfByte));
56 12 : currentDfxSize++;
57 12 : (*ptr)++;
58 : }
59 :
60 6 : return ADUMP_SUCCESS;
61 : }
62 :
63 : template <typename T>
64 81 : int32_t DumpArgs::GetPointerValueByLittleEndian(
65 : const uint8_t** ptr, T& value, uint64_t& currentDfxSize, uint16_t totalDfxSize) const
66 : {
67 81 : uint16_t bitOfByte = 8;
68 261 : for (size_t i = 0; i < sizeof(T); ++i) {
69 180 : if (currentDfxSize >= totalDfxSize) {
70 0 : IDE_LOGE(
71 : "The dfx data size[%" PRIu64 "] is larger than the total dfx size[%" PRIu16 "].", currentDfxSize,
72 : totalDfxSize);
73 0 : return ADUMP_FAILED;
74 : }
75 180 : value = (value) | (static_cast<uint64_t>(**ptr) << (i * bitOfByte));
76 180 : currentDfxSize++;
77 180 : (*ptr)++;
78 : }
79 :
80 81 : return ADUMP_SUCCESS;
81 : }
82 :
83 26 : int32_t DumpArgs::FindExceptionDfx(const rtExceptionArgsInfo_t& exceptionArgsInfo)
84 : {
85 26 : const uint8_t* metaDfxPtr = static_cast<const uint8_t*>(exceptionArgsInfo.exceptionKernelInfo.dfxAddr);
86 26 : uint16_t metaDfxSize = exceptionArgsInfo.exceptionKernelInfo.dfxSize;
87 26 : std::stringstream metaDfxStr;
88 4934 : for (uint16_t i = 0; i < metaDfxSize; ++i) {
89 4908 : metaDfxStr << int32_t(*(metaDfxPtr + i)) << ", ";
90 : }
91 26 : IDE_LOGI("The meta dfx raw data:%s ", metaDfxStr.str().c_str());
92 :
93 26 : uint64_t currDfxSize = 0;
94 65 : while (currDfxSize < metaDfxSize) {
95 39 : uint16_t dfxType = 0;
96 39 : uint16_t dfxLength = 0;
97 39 : int32_t ret = ADUMP_SUCCESS;
98 39 : if (exceptionArgsInfo.exceptionKernelInfo.elfDataFlag == ELF_DATA2MSB) {
99 3 : ret = GetPointerValueByBigEndian(&metaDfxPtr, dfxType, currDfxSize, metaDfxSize);
100 3 : IDE_CHECK_RET(ret, return ADUMP_FAILED);
101 3 : ret = GetPointerValueByBigEndian(&metaDfxPtr, dfxLength, currDfxSize, metaDfxSize);
102 3 : IDE_CHECK_RET(ret, return ADUMP_FAILED);
103 : } else {
104 36 : ret = GetPointerValueByLittleEndian(&metaDfxPtr, dfxType, currDfxSize, metaDfxSize);
105 36 : IDE_CHECK_RET(ret, return ADUMP_FAILED);
106 36 : ret = GetPointerValueByLittleEndian(&metaDfxPtr, dfxLength, currDfxSize, metaDfxSize);
107 36 : IDE_CHECK_RET(ret, return ADUMP_FAILED);
108 : }
109 39 : IDE_LOGI("The dfx type[%" PRIu16 "], dfx length[%" PRIu16 "]", dfxType, dfxLength);
110 39 : if (dfxType == TYPE_L0_EXCEPTION_DFX) {
111 25 : exceptionDfxPtr_ = metaDfxPtr;
112 25 : exceptionDfxSize_ = dfxLength;
113 : }
114 :
115 39 : if (dfxType == TYPE_L0_EXCEPTION_DFX_IS_TIK) {
116 9 : uint32_t isTik = 0;
117 9 : if (exceptionArgsInfo.exceptionKernelInfo.elfDataFlag == ELF_DATA2MSB) {
118 0 : ret = GetPointerValueByBigEndian(&metaDfxPtr, isTik, currDfxSize, metaDfxSize);
119 : } else {
120 9 : ret = GetPointerValueByLittleEndian(&metaDfxPtr, isTik, currDfxSize, metaDfxSize);
121 : }
122 9 : IDE_CHECK_RET(ret, return ADUMP_FAILED);
123 9 : isTik_ = (isTik == 1U) ? true : false;
124 : } else {
125 30 : metaDfxPtr += dfxLength;
126 30 : currDfxSize += dfxLength;
127 : }
128 : }
129 :
130 26 : if (exceptionDfxPtr_ == nullptr || exceptionDfxSize_ == 0) {
131 1 : IDE_LOGE("No exception dfx info, dfxAddr[%p]|dfxSize[%" PRIu16 "].", exceptionDfxPtr_, exceptionDfxSize_);
132 1 : return ADUMP_FAILED;
133 : }
134 :
135 25 : return ADUMP_SUCCESS;
136 26 : }
137 :
138 68 : int32_t DumpArgs::LoadArgsInfoWithDfx(const rtExceptionArgsInfo_t& exceptionArgsInfo)
139 : {
140 68 : if (exceptionArgsInfo.argAddr == nullptr || exceptionArgsInfo.argsize == 0 ||
141 49 : exceptionArgsInfo.exceptionKernelInfo.dfxAddr == nullptr ||
142 26 : exceptionArgsInfo.exceptionKernelInfo.dfxSize == 0) {
143 42 : IDE_LOGE(
144 : "In argAddr[%p]|argSize[%u]|dfxAddr[%p]|dfxSize[%u] has invalid attribute.", exceptionArgsInfo.argAddr,
145 : exceptionArgsInfo.argsize, exceptionArgsInfo.exceptionKernelInfo.dfxAddr,
146 : exceptionArgsInfo.exceptionKernelInfo.dfxSize);
147 42 : return ADUMP_FAILED;
148 : }
149 :
150 26 : argAddr_ = exceptionArgsInfo.argAddr;
151 26 : argSize_ = exceptionArgsInfo.argsize;
152 :
153 : // FindExceptionDfx 保留在 DumpArgs（从 meta dfx 定位 exception dfx）
154 26 : int32_t ret = FindExceptionDfx(exceptionArgsInfo);
155 26 : IDE_CHECK_RET(ret, return ADUMP_FAILED);
156 25 : IDE_LOGI("The dfx address[%p] and dfx size[%u].", exceptionDfxPtr_, exceptionDfxSize_);
157 :
158 : // 使用 DfxArgsParser
159 25 : DfxArgsParser parser;
160 25 : ret = parser.Init(argAddr_, argSize_, exceptionDfxPtr_, exceptionDfxSize_);
161 25 : IDE_CHECK_RET(ret, return ADUMP_FAILED);
162 :
163 19 : parser.SetIsTik(isTik_);
164 19 : ret = parser.InitTensorModeInfo();
165 19 : IDE_CHECK_RET(ret, return ADUMP_FAILED);
166 :
167 8 : ret = parser.ParseAll();
168 8 : IDE_CHECK_RET(ret, return ADUMP_FAILED);
169 :
170 : // 获取解析结果
171 6 : tensorBuffer_ = parser.GetTensors();
172 6 : workspace_ = parser.GetWorkspaces();
173 6 : mc2Space_ = parser.GetMc2Space();
174 6 : logRecord_ = parser.GetLogRecords();
175 :
176 6 : return ADUMP_SUCCESS;
177 25 : }
178 :
179 62 : int32_t DumpArgs::LoadArgsInfoWithSizeInfo(
180 : const rtExceptionArgsInfo_t& exceptionArgsInfo, const rtExceptionExpandInfo_t& exceptionExpandInfo)
181 : {
182 62 : if (CheckParam(exceptionArgsInfo, exceptionExpandInfo) != ADUMP_SUCCESS) {
183 42 : return ADUMP_FAILED;
184 : }
185 :
186 20 : if (InitAttributes(exceptionArgsInfo, exceptionExpandInfo) != ADUMP_SUCCESS) {
187 2 : return ADUMP_FAILED;
188 : }
189 :
190 18 : uint32_t maxArgNum = argSize_ / sizeof(uint64_t);
191 18 : if (static_cast<uint64_t>(skipNum_) + inputNum_ > maxArgNum) {
192 2 : IDE_LOGE(
193 : "[Dump][Exception] skip num[%" PRIu32 "] plus input num[%" PRIu32 "] is over the max arg num[%" PRIu32
194 : "], load failed.",
195 : skipNum_, inputNum_, maxArgNum);
196 2 : return ADUMP_FAILED;
197 : }
198 :
199 16 : auto data = DumpMemory::CopyDeviceToHostEx(argAddr_, argSize_);
200 16 : if (data == nullptr) {
201 0 : IDE_LOGE("Copy device args to host failed.");
202 0 : return ADUMP_FAILED;
203 : }
204 16 : const void** argOnHost = static_cast<const void**>(data);
205 16 : if (exceptionExpandInfo.type == RT_EXCEPTION_FFTS_PLUS) {
206 1 : LogArgsInfo(argOnHost, maxArgNum);
207 : }
208 :
209 16 : uint32_t argIndex = skipNum_;
210 16 : IDE_LOGE(
211 : "[Dump][Exception] the begin tensor's index of args is:%" PRIu32 ", args dump count[%" PRIu32 "]", argIndex,
212 : inputNum_);
213 113 : for (uint64_t sizeInfoIdx = sizeBeginIndex_; sizeInfoIdx < inputNum_ + sizeBeginIndex_; ++sizeInfoIdx) {
214 100 : int64_t size = *(reinterpret_cast<int64_t*>(&sizeInfo_[sizeInfoIdx]));
215 : // the size of the level-2 pointer is a negative number in earlier versions.
216 100 : if (size < 0) {
217 16 : size = std::abs(size);
218 16 : if (size > maxArgNum) {
219 3 : IDE_LOGE(
220 : "[Dump][Exception] size[%" PRId64 "] over arg max[%" PRIu32 "] is invalid, load failed.", size,
221 : maxArgNum);
222 6 : HOST_RT_MEMORY_GUARD(data);
223 3 : return ADUMP_FAILED;
224 3 : }
225 13 : sizeInfoIdx += size;
226 : } else {
227 84 : int32_t ret = LoadInputBuffer(argOnHost, argIndex, sizeInfoIdx);
228 84 : if (ret != ADUMP_SUCCESS) {
229 0 : HOST_RT_MEMORY_GUARD(data);
230 0 : return ADUMP_FAILED;
231 0 : }
232 : }
233 97 : argIndex++;
234 : }
235 13 : LoadTilingData();
236 26 : HOST_RT_MEMORY_GUARD(data);
237 :
238 13 : return ADUMP_SUCCESS;
239 13 : }
240 :
241 68 : int32_t DumpArgs::LoadArgsExceptionInfo(const rtExceptionInfo& exception)
242 : {
243 68 : rtExceptionArgsInfo_t exceptionArgsInfo{};
244 68 : rtExceptionExpandInfo_t exceptionExpandInfo = exception.expandInfo;
245 68 : if (ExceptionInfoCommon::GetExceptionInfo(exception, exceptionArgsInfo) != ADUMP_SUCCESS) {
246 0 : IDE_LOGW("Get exception args info failed.");
247 0 : return ADUMP_FAILED;
248 : }
249 68 : kernelCollector_->LoadKernelInfo(exceptionArgsInfo);
250 :
251 68 : if (LoadArgsInfoWithDfx(exceptionArgsInfo) == ADUMP_SUCCESS) {
252 6 : dumpWithDfxFlag_ = true;
253 : } else {
254 62 : IDE_LOGI("Can not load args info with dfx, use load args info with size info instead of it.");
255 62 : if (LoadArgsInfoWithSizeInfo(exceptionArgsInfo, exceptionExpandInfo) != ADUMP_SUCCESS) {
256 49 : return ADUMP_FAILED;
257 : }
258 : }
259 :
260 19 : streamId_ = std::to_string(exception.streamid);
261 19 : taskId_ = std::to_string(exception.taskid);
262 :
263 19 : return ADUMP_SUCCESS;
264 : }
265 :
266 62 : int32_t DumpArgs::CheckParam(
267 : const rtExceptionArgsInfo_t& exceptionArgsInfo, const rtExceptionExpandInfo_t& exceptionExpandInfo) const
268 : {
269 : // Except the aic/aiv error, other exceptions are also called back. Therefore, warning logs are generated here.
270 62 : if (exceptionArgsInfo.sizeInfo.infoAddr == nullptr || exceptionArgsInfo.sizeInfo.atomicIndex == 0 ||
271 25 : exceptionArgsInfo.argAddr == nullptr) {
272 37 : IDE_LOGW(
273 : "[Dump][Exception] in infoAddr[%p]|atomicIndex[%" PRIu32 "]|argAddr[%p] has invalid attribute.",
274 : exceptionArgsInfo.sizeInfo.infoAddr, exceptionArgsInfo.sizeInfo.atomicIndex, exceptionArgsInfo.argAddr);
275 37 : return ADUMP_FAILED;
276 : }
277 : // RT_EXCEPTION_AICORE indicates the normal mode
278 25 : if (exceptionExpandInfo.type == RT_EXCEPTION_AICORE && exceptionArgsInfo.argsize == 0) {
279 2 : IDE_LOGW("[Dump][Exception] the arg size[%" PRIu32 "] is invalid.", exceptionArgsInfo.argsize);
280 2 : return ADUMP_FAILED;
281 : }
282 :
283 23 : uint64_t* sizeInfo = static_cast<uint64_t*>(exceptionArgsInfo.sizeInfo.infoAddr);
284 23 : if (sizeInfo < g_chunk || sizeInfo > (g_chunk + RING_CHUNK_SIZE - 1)) {
285 1 : IDE_LOGE(
286 : "[Dump][Exception] the size info[%p] address may out of the chunk[%p] address range.", sizeInfo, g_chunk);
287 1 : return ADUMP_FAILED;
288 : }
289 22 : if (sizeInfo[0] != exceptionArgsInfo.sizeInfo.atomicIndex) {
290 2 : IDE_LOGE(
291 : "[Dump][Exception] args exception atomic index between %" PRIu64 " and %" PRIu32 " is different.",
292 : sizeInfo[0], exceptionArgsInfo.sizeInfo.atomicIndex);
293 2 : return ADUMP_FAILED;
294 : }
295 :
296 20 : return ADUMP_SUCCESS;
297 : }
298 :
299 20 : int32_t DumpArgs::InitAttributes(
300 : const rtExceptionArgsInfo_t& exceptionArgsInfo, const rtExceptionExpandInfo_t& exceptionExpandInfo)
301 : {
302 20 : sizeInfo_ = static_cast<uint64_t*>(exceptionArgsInfo.sizeInfo.infoAddr);
303 20 : argAddr_ = exceptionArgsInfo.argAddr;
304 :
305 : // RT_EXCEPTION_AICORE indicates the normal mode
306 20 : if (exceptionExpandInfo.type == RT_EXCEPTION_AICORE) {
307 18 : sizeBeginIndex_ = ATOMIC_AND_INPUT_NUM_OFFSET;
308 18 : skipNum_ = (sizeInfo_[1] & SKIP_NUM_MASK) >> SKIP_NUM_SHIFT_BITS;
309 18 : inputNum_ = sizeInfo_[1] & INPUT_NUM_MASK;
310 18 : if (sizeInfo_ + inputNum_ + ATOMIC_AND_INPUT_NUM_OFFSET > g_chunk + RING_CHUNK_SIZE + MAX_TENSOR_NUM) {
311 1 : IDE_LOGE(
312 : "[Dump][Exception] the value of size info[%p] plus input num[%" PRIu32
313 : "] may exceed the chunk[%p] address range.",
314 : sizeInfo_, inputNum_, g_chunk);
315 1 : return ADUMP_FAILED;
316 : }
317 17 : argSize_ = exceptionArgsInfo.argsize;
318 17 : return ADUMP_SUCCESS;
319 : }
320 :
321 : // RT_EXCEPTION_FFTS_PLUS indicates the ffts+ mode
322 2 : if (exceptionExpandInfo.type == RT_EXCEPTION_FFTS_PLUS) {
323 2 : uint64_t totalContextSizeNum = sizeInfo_[1];
324 2 : if (sizeInfo_ + totalContextSizeNum + ATOMIC_AND_INPUT_NUM_OFFSET >
325 2 : g_chunk + RING_CHUNK_SIZE + MAX_TENSOR_NUM) {
326 1 : IDE_LOGE(
327 : "[Dump][Exception] the value of size info[%p] plus total num[%" PRIu64
328 : "] may exceed the chunk[%p] address range.",
329 : sizeInfo_, totalContextSizeNum, g_chunk);
330 1 : return ADUMP_FAILED;
331 : }
332 1 : uint32_t contextBeginIndex = ATOMIC_AND_INPUT_NUM_OFFSET;
333 2 : for (uint64_t sizeInfoIdx = contextBeginIndex; sizeInfoIdx < totalContextSizeNum + contextBeginIndex;
334 : ++sizeInfoIdx) {
335 2 : uint32_t inputNumBeginIndex = sizeInfoIdx + CONTEXTID_AND_INPUT_NUM_OFFSET;
336 2 : skipNum_ = (sizeInfo_[inputNumBeginIndex] & SKIP_NUM_MASK) >> SKIP_NUM_SHIFT_BITS;
337 2 : inputNum_ = sizeInfo_[inputNumBeginIndex] & INPUT_NUM_MASK;
338 2 : IDE_LOGI(
339 : "[Dump][Exception] the context id[%" PRIu64 "] of size info and context id[%" PRIu16
340 : "] of exception info",
341 : sizeInfo_[sizeInfoIdx], exceptionExpandInfo.u.fftsPlusInfo.contextId);
342 2 : if (sizeInfo_[sizeInfoIdx] == exceptionExpandInfo.u.fftsPlusInfo.contextId) {
343 1 : argSize_ = sizeInfo_[sizeInfoIdx + 1];
344 1 : sizeBeginIndex_ = sizeInfoIdx + 3; // 3 - context id | args size | input num
345 1 : return ADUMP_SUCCESS;
346 : }
347 1 : sizeInfoIdx += inputNum_ + CONTEXTID_AND_INPUT_NUM_OFFSET;
348 : }
349 : }
350 :
351 : // log warning - the arg data in ffts+ mode may be in others contextid
352 0 : IDE_LOGW("Data mismatch, some attribute values cannot be obtained.");
353 0 : return ADUMP_FAILED;
354 : }
355 :
356 1 : void DumpArgs::LogArgsInfo(const void** argOnHost, uint32_t maxArgNum)
357 : {
358 1 : const uint32_t argsLogTimes = (maxArgNum % ARGS_PER_STRING_MAX_LEN > 0) ?
359 1 : ((maxArgNum / ARGS_PER_STRING_MAX_LEN) + 1) :
360 : (maxArgNum / ARGS_PER_STRING_MAX_LEN);
361 2 : for (uint32_t i = 1; i <= argsLogTimes; ++i) {
362 1 : std::stringstream ss;
363 1 : uint32_t endIndex = maxArgNum > (i * ARGS_PER_STRING_MAX_LEN) ? (i * ARGS_PER_STRING_MAX_LEN) : maxArgNum;
364 15 : for (uint32_t j = (i - 1) * ARGS_PER_STRING_MAX_LEN; j < endIndex; ++j) {
365 14 : ss << *(argOnHost + j) << ", ";
366 : }
367 1 : oss_ << "[AIC_INFO] args(" << (i - 1) * ARGS_PER_STRING_MAX_LEN << " to " << endIndex
368 1 : << ") after execute:" << ss.str().c_str();
369 1 : RecordCurrentLog();
370 1 : }
371 1 : IDE_LOGE("[AIC_INFO] after execute:args print end");
372 1 : }
373 :
374 84 : int32_t DumpArgs::LoadInputBuffer(const void** argOnHost, const uint32_t argIndex, uint64_t& sizeInfoIdx)
375 : {
376 84 : uint8_t sizeType = (sizeInfo_[sizeInfoIdx] & SIZE_TYPE_MASK) >> SIZE_TYPE_SHIFT_BITS;
377 84 : if (sizeType == NORMAL_TENSOR || sizeType == WORKSPACE_TENSOR) {
378 45 : oss_ << "[Dump][Exception] begin to load normal tensor, index:" << argIndex;
379 45 : RecordCurrentLog();
380 45 : uint64_t tensorSize = sizeInfo_[sizeInfoIdx] & SIZE_MASK;
381 45 : oss_ << "[Dump][Exception] exception info dump args data, addr:" << argOnHost[argIndex]
382 45 : << "; size:" << tensorSize << " bytes";
383 45 : RecordCurrentLog();
384 45 : inputBuffer_.emplace_back(argOnHost[argIndex], tensorSize, argIndex);
385 45 : oss_ << "[Dump][Exception] end to load normal tensor, index:" << argIndex;
386 45 : RecordCurrentLog();
387 84 : } else if (sizeType == NORMAL_PTR_TENSOR || sizeType == SHAPE_PTR_TENSOR) {
388 26 : if (LoadPointerTensor(argOnHost, argIndex, sizeInfoIdx) != ADUMP_SUCCESS) {
389 0 : return ADUMP_FAILED;
390 : }
391 13 : } else if (sizeType == TILING_DATA_PTR) {
392 13 : IDE_LOGE("[Dump][Exception] save tiling data, index:%" PRIu32, argIndex);
393 13 : tilingData_.emplace_back(argOnHost[argIndex], (sizeInfo_[sizeInfoIdx] & SIZE_MASK), argIndex);
394 : } else {
395 0 : IDE_LOGE("[Dump][Exception] args exception size type[%" PRIu8 "] is error.", sizeType);
396 0 : return ADUMP_FAILED;
397 : }
398 :
399 84 : return ADUMP_SUCCESS;
400 : }
401 :
402 26 : int32_t DumpArgs::LoadPointerTensor(const void** argOnHost, const uint32_t argIndex, uint64_t& sizeInfoIdx)
403 : {
404 26 : uint64_t addrBias = reinterpret_cast<uint64_t>(argOnHost[argIndex]) - reinterpret_cast<uint64_t>(argAddr_);
405 26 : if (addrBias >= argSize_) {
406 0 : IDE_LOGE(
407 : "[Dump][Exception]address bias[%p - %p] is over args size[%" PRIu64 "], invalid.", argOnHost[argIndex],
408 : argAddr_, argSize_);
409 0 : return ADUMP_FAILED;
410 : }
411 :
412 26 : uint64_t inputPtrAddr = addrBias + reinterpret_cast<uint64_t>(argOnHost);
413 26 : void* inputPtr = nullptr;
414 26 : std::string pointerType = "";
415 26 : uint8_t sizeType = (sizeInfo_[sizeInfoIdx] & SIZE_TYPE_MASK) >> SIZE_TYPE_SHIFT_BITS;
416 26 : if (sizeType == NORMAL_PTR_TENSOR) {
417 13 : inputPtr = reinterpret_cast<void*>(inputPtrAddr);
418 13 : pointerType = "normal pointer tensor";
419 13 : } else if (sizeType == SHAPE_PTR_TENSOR) {
420 13 : uint64_t offset = *(reinterpret_cast<uint64_t*>(inputPtrAddr));
421 13 : inputPtr = reinterpret_cast<void*>(inputPtrAddr + offset);
422 13 : pointerType = "shape pointer tensor";
423 : } else {
424 0 : IDE_LOGE("[Dump][Exception] args exception size type[%" PRIu8 "] is error.", sizeType);
425 0 : return ADUMP_FAILED;
426 : }
427 :
428 26 : IDE_LOGE(
429 : "[Dump][Exception] begin to load %s, index:%" PRIu32 ", addr:%p", pointerType.c_str(), argIndex,
430 : argOnHost[argIndex]);
431 :
432 26 : const void** endArgAddr = argOnHost + argSize_ / sizeof(uint64_t);
433 26 : uint64_t inputPtrNum = sizeInfo_[sizeInfoIdx] & SIZE_MASK;
434 78 : for (uint64_t i = 0; i < inputPtrNum; ++i) {
435 52 : uint64_t inputPtrSize = sizeInfo_[sizeInfoIdx + 1 + i];
436 52 : void** tensorPtr = static_cast<void**>(inputPtr) + i;
437 52 : oss_ << "[Dump][Exception] exception info dump args data, addr:" << *(tensorPtr) << "; size:" << inputPtrSize
438 52 : << " bytes";
439 52 : RecordCurrentLog();
440 52 : if (tensorPtr > endArgAddr) {
441 0 : IDE_LOGE("[Dump][Exception] args address[%p] is over args end address[%p].", tensorPtr, endArgAddr);
442 0 : return ADUMP_FAILED;
443 : }
444 52 : inputBuffer_.emplace_back(*(tensorPtr), inputPtrSize, argIndex);
445 : }
446 26 : sizeInfoIdx += inputPtrNum;
447 :
448 26 : IDE_LOGE(
449 : "[Dump][Exception] end to load %s, index:%" PRIu32 ", addr:%p", pointerType.c_str(), argIndex,
450 : argOnHost[argIndex]);
451 :
452 26 : return ADUMP_SUCCESS;
453 26 : }
454 :
455 13 : void DumpArgs::LoadTilingData()
456 : {
457 13 : if (tilingData_.size() == 0) {
458 0 : return;
459 : }
460 13 : IDE_LOGE("[Dump][Exception] begin to load tiling data.");
461 39 : for (const auto& it : tilingData_) {
462 13 : oss_ << "[Dump][Exception] exception info dump args data, addr:" << it.addr << "; size:" << it.length
463 13 : << " bytes";
464 13 : RecordCurrentLog();
465 13 : inputBuffer_.emplace_back(it.addr, it.length, it.argIndex);
466 : }
467 13 : IDE_LOGE("[Dump][Exception] end to load tiling data.");
468 : }
469 :
470 32 : int32_t DumpArgs::DumpArgsExceptionFile(const uint32_t deviceId, const std::string& dumpPath)
471 : {
472 32 : std::string dumpFilePath = GetDumpFilePath(dumpPath);
473 32 : IDE_LOGI("[Dump][Exception] The exception dump file path is %s", dumpFilePath.c_str());
474 :
475 32 : DumpFile dumpFile(deviceId, dumpFilePath);
476 32 : dumpFile.SetHeader("");
477 32 : if (dumpWithDfxFlag_) {
478 6 : dumpFile.SetTensorBuffer(tensorBuffer_);
479 6 : dumpFile.SetWorkspaces(workspace_);
480 : #if !defined(ADUMP_SOC_HOST) || ADUMP_SOC_HOST == 1
481 6 : dumpFile.SetMc2spaces(mc2Space_);
482 : #endif
483 : } else {
484 26 : dumpFile.SetInputBuffer(inputBuffer_);
485 : }
486 32 : AdxLogFlush();
487 32 : int32_t ret = dumpFile.Dump(logRecord_);
488 32 : if (ret != ADUMP_SUCCESS) {
489 1 : IDE_LOGE("[Dump][Exception] write exception to file failed, file: %s", dumpFilePath.c_str());
490 1 : return ADUMP_FAILED;
491 : }
492 :
493 31 : (void)mmChmod(dumpFilePath.c_str(), M_IRUSR); // readonly, 400
494 31 : IDE_LOGE("[Dump][Exception] dump exception to file, file: %s", dumpFilePath.c_str());
495 :
496 31 : IDE_LOGI("[Dump][Exception] Dump exception info success.");
497 31 : return ADUMP_SUCCESS;
498 32 : }
499 :
500 32 : int32_t DumpArgs::DumpArgsExceptionInfo(const uint32_t deviceId, const std::string& dumpPath)
501 : {
502 32 : int32_t ret = ADUMP_SUCCESS;
503 32 : if (DumpArgsExceptionFile(deviceId, dumpPath) != ADUMP_SUCCESS) {
504 1 : ret = ADUMP_FAILED;
505 : }
506 32 : if (StartCollectKernelAsync(kernelCollector_, dumpPath) != ADUMP_SUCCESS) {
507 2 : ret = ADUMP_FAILED;
508 : }
509 32 : return ret;
510 : }
511 :
512 32 : std::string DumpArgs::GetDumpFilePath(const std::string& dumpPath) const
513 : {
514 32 : std::string opType = "exception_info";
515 : std::string dumpFileName =
516 32 : opType + "." + streamId_ + "." + taskId_ + "." + SysUtils::GetCurrentTimeWithMillisecond();
517 32 : Path dumpFilePath(dumpPath);
518 32 : dumpFilePath.Concat(dumpFileName);
519 32 : dumpFilePath.RealPath();
520 64 : return dumpFilePath.GetString();
521 32 : }
522 :
523 201 : void DumpArgs::RecordCurrentLog()
524 : {
525 201 : IDE_LOGE("%s", oss_.str().c_str());
526 201 : logRecord_.emplace_back(oss_.str() + "\n");
527 402 : oss_.str("");
528 201 : }
529 :
530 1 : bool DumpArgs::DumpArgsDumpWithDfxFlag() const { return dumpWithDfxFlag_; }
531 :
532 1 : const std::vector<InputBuffer>& DumpArgs::DumpArgsGetInputBuffer() const { return inputBuffer_; }
533 :
534 0 : const std::vector<TensorBuffer>& DumpArgs::DumpArgsGetTensorBuffer() const { return tensorBuffer_; }
535 :
536 0 : const std::vector<DumpWorkspace>& DumpArgs::DumpArgsGetWorkSpace() const { return workspace_; }
537 :
538 : } // namespace Adx
|
