Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 : #include <set>
11 : #include <cmath>
12 : #include "dump_args.h"
13 : #include "dfx_args_parser.h"
14 : #include <cinttypes>
15 : #include "path.h"
16 : #include "dump_file.h"
17 : #include "sys_utils.h"
18 : #include "dump_memory.h"
19 : #include "log/hdc_log.h"
20 : #include "exception_info_common.h"
21 : #include "dump_tensor_plugin.h"
22 : #include "adump_dsmi.h"
23 : #include "log/adx_log.h"
24 :
25 : namespace Adx {
26 : namespace {
27 : constexpr uint32_t ATOMIC_AND_INPUT_NUM_OFFSET = 2; // 2 - atomic index | input num(or total context size num)
28 : constexpr uint32_t CONTEXTID_AND_INPUT_NUM_OFFSET = 2; // 2 - context id | input num
29 : constexpr uint32_t SKIP_NUM_SHIFT_BITS = 32;
30 : constexpr uint64_t SKIP_NUM_MASK = 0x0FFFFFFFF00000000; // high 32 bits
31 : constexpr uint64_t INPUT_NUM_MASK = 0x0FFFFFFFF; // low 32 bits
32 : constexpr uint64_t SIZE_TYPE_MASK = 0x0FF00000000000000; // high 8 bits
33 : constexpr uint64_t SIZE_MASK = 0x0FFFFFFFFFFFFFF; // low 56 bits
34 : constexpr uint32_t SIZE_TYPE_SHIFT_BITS = 56;
35 : constexpr uint8_t NORMAL_TENSOR = 0;
36 : constexpr uint8_t NORMAL_PTR_TENSOR = 1;
37 : constexpr uint8_t SHAPE_PTR_TENSOR = 2;
38 : constexpr uint8_t TILING_DATA_PTR = 3;
39 : constexpr uint8_t WORKSPACE_TENSOR = 4;
40 : constexpr uint32_t ARGS_PER_STRING_MAX_LEN = 20;
41 : } // namespace
42 :
43 : template <typename T>
44 6 : int32_t DumpArgs::GetPointerValueByBigEndian(const uint8_t **ptr, T &value, uint64_t ¤tDfxSize,
45 : uint16_t totalDfxSize) const
46 : {
47 6 : uint16_t bitOfByte = 8;
48 18 : for (size_t i = 0; i < sizeof(T); ++i) {
49 12 : if (currentDfxSize >= totalDfxSize) {
50 0 : IDE_LOGE("The dfx data size[%" PRIu64 "] is larger than the total dfx size[%" PRIu16 "].", currentDfxSize,
51 : totalDfxSize);
52 0 : return ADUMP_FAILED;
53 : }
54 12 : value = (value) | (static_cast<uint64_t>(**ptr) << ((sizeof(T) - i - 1) * bitOfByte));
55 12 : currentDfxSize++;
56 12 : (*ptr)++;
57 : }
58 :
59 6 : return ADUMP_SUCCESS;
60 : }
61 :
62 : template <typename T>
63 81 : int32_t DumpArgs::GetPointerValueByLittleEndian(const uint8_t **ptr, T &value, uint64_t ¤tDfxSize,
64 : uint16_t totalDfxSize) const
65 : {
66 81 : uint16_t bitOfByte = 8;
67 261 : for (size_t i = 0; i < sizeof(T); ++i) {
68 180 : if (currentDfxSize >= totalDfxSize) {
69 0 : IDE_LOGE("The dfx data size[%" PRIu64 "] is larger than the total dfx size[%" PRIu16 "].", currentDfxSize,
70 : totalDfxSize);
71 0 : return ADUMP_FAILED;
72 : }
73 180 : value = (value) | (static_cast<uint64_t>(**ptr) << (i * bitOfByte));
74 180 : currentDfxSize++;
75 180 : (*ptr)++;
76 : }
77 :
78 81 : return ADUMP_SUCCESS;
79 : }
80 :
81 26 : int32_t DumpArgs::FindExceptionDfx(const rtExceptionArgsInfo_t &exceptionArgsInfo)
82 : {
83 26 : const uint8_t *metaDfxPtr = static_cast<const uint8_t *>(exceptionArgsInfo.exceptionKernelInfo.dfxAddr);
84 26 : uint16_t metaDfxSize = exceptionArgsInfo.exceptionKernelInfo.dfxSize;
85 26 : std::stringstream metaDfxStr;
86 4934 : for (uint16_t i = 0; i < metaDfxSize; ++i) {
87 4908 : metaDfxStr << int32_t(*(metaDfxPtr + i)) << ", ";
88 : }
89 26 : IDE_LOGI("The meta dfx raw data:%s ", metaDfxStr.str().c_str());
90 :
91 26 : uint64_t currDfxSize = 0;
92 65 : while (currDfxSize < metaDfxSize) {
93 39 : uint16_t dfxType = 0;
94 39 : uint16_t dfxLength = 0;
95 39 : int32_t ret = ADUMP_SUCCESS;
96 39 : if (exceptionArgsInfo.exceptionKernelInfo.elfDataFlag == ELF_DATA2MSB) {
97 3 : ret = GetPointerValueByBigEndian(&metaDfxPtr, dfxType, currDfxSize, metaDfxSize);
98 3 : IDE_CHECK_RET(ret, return ADUMP_FAILED);
99 3 : ret = GetPointerValueByBigEndian(&metaDfxPtr, dfxLength, currDfxSize, metaDfxSize);
100 3 : IDE_CHECK_RET(ret, return ADUMP_FAILED);
101 : } else {
102 36 : ret = GetPointerValueByLittleEndian(&metaDfxPtr, dfxType, currDfxSize, metaDfxSize);
103 36 : IDE_CHECK_RET(ret, return ADUMP_FAILED);
104 36 : ret = GetPointerValueByLittleEndian(&metaDfxPtr, dfxLength, currDfxSize, metaDfxSize);
105 36 : IDE_CHECK_RET(ret, return ADUMP_FAILED);
106 : }
107 39 : IDE_LOGI("The dfx type[%" PRIu16 "], dfx length[%" PRIu16 "]", dfxType, dfxLength);
108 39 : if (dfxType == TYPE_L0_EXCEPTION_DFX) {
109 25 : exceptionDfxPtr_ = metaDfxPtr;
110 25 : exceptionDfxSize_ = dfxLength;
111 : }
112 :
113 39 : if (dfxType == TYPE_L0_EXCEPTION_DFX_IS_TIK) {
114 9 : uint32_t isTik = 0;
115 9 : if (exceptionArgsInfo.exceptionKernelInfo.elfDataFlag == ELF_DATA2MSB) {
116 0 : ret = GetPointerValueByBigEndian(&metaDfxPtr, isTik, currDfxSize, metaDfxSize);
117 : } else {
118 9 : ret = GetPointerValueByLittleEndian(&metaDfxPtr, isTik, currDfxSize, metaDfxSize);
119 : }
120 9 : IDE_CHECK_RET(ret, return ADUMP_FAILED);
121 9 : isTik_ = (isTik == 1U) ? true : false;
122 : } else {
123 30 : metaDfxPtr += dfxLength;
124 30 : currDfxSize += dfxLength;
125 : }
126 : }
127 :
128 26 : if (exceptionDfxPtr_ == nullptr || exceptionDfxSize_ == 0) {
129 1 : IDE_LOGE("No exception dfx info, dfxAddr[%p]|dfxSize[%" PRIu16 "].", exceptionDfxPtr_, exceptionDfxSize_);
130 1 : return ADUMP_FAILED;
131 : }
132 :
133 25 : return ADUMP_SUCCESS;
134 26 : }
135 :
136 67 : int32_t DumpArgs::LoadArgsInfoWithDfx(const rtExceptionArgsInfo_t &exceptionArgsInfo)
137 : {
138 67 : if (exceptionArgsInfo.argAddr == nullptr || exceptionArgsInfo.argsize == 0 ||
139 49 : exceptionArgsInfo.exceptionKernelInfo.dfxAddr == nullptr ||
140 26 : exceptionArgsInfo.exceptionKernelInfo.dfxSize == 0) {
141 41 : IDE_LOGE("In argAddr[%p]|argSize[%u]|dfxAddr[%p]|dfxSize[%u] has invalid attribute.",
142 : exceptionArgsInfo.argAddr, exceptionArgsInfo.argsize, exceptionArgsInfo.exceptionKernelInfo.dfxAddr,
143 : exceptionArgsInfo.exceptionKernelInfo.dfxSize);
144 41 : return ADUMP_FAILED;
145 : }
146 :
147 26 : argAddr_ = exceptionArgsInfo.argAddr;
148 26 : argSize_ = exceptionArgsInfo.argsize;
149 :
150 : // FindExceptionDfx 保留在 DumpArgs（从 meta dfx 定位 exception dfx）
151 26 : int32_t ret = FindExceptionDfx(exceptionArgsInfo);
152 26 : IDE_CHECK_RET(ret, return ADUMP_FAILED);
153 25 : IDE_LOGI("The dfx address[%p] and dfx size[%u].", exceptionDfxPtr_, exceptionDfxSize_);
154 :
155 : // 使用 DfxArgsParser
156 25 : DfxArgsParser parser;
157 25 : ret = parser.Init(argAddr_, argSize_, exceptionDfxPtr_, exceptionDfxSize_);
158 25 : IDE_CHECK_RET(ret, return ADUMP_FAILED);
159 :
160 19 : parser.SetIsTik(isTik_);
161 19 : ret = parser.InitTensorModeInfo();
162 19 : IDE_CHECK_RET(ret, return ADUMP_FAILED);
163 :
164 8 : ret = parser.ParseAll();
165 8 : IDE_CHECK_RET(ret, return ADUMP_FAILED);
166 :
167 : // 获取解析结果
168 6 : tensorBuffer_ = parser.GetTensors();
169 6 : workspace_ = parser.GetWorkspaces();
170 6 : mc2Space_ = parser.GetMc2Space();
171 6 : logRecord_ = parser.GetLogRecords();
172 :
173 6 : return ADUMP_SUCCESS;
174 25 : }
175 :
176 61 : int32_t DumpArgs::LoadArgsInfoWithSizeInfo(const rtExceptionArgsInfo_t &exceptionArgsInfo,
177 : const rtExceptionExpandInfo_t &exceptionExpandInfo)
178 : {
179 61 : if (CheckParam(exceptionArgsInfo, exceptionExpandInfo) != ADUMP_SUCCESS) {
180 41 : return ADUMP_FAILED;
181 : }
182 :
183 20 : if (InitAttributes(exceptionArgsInfo, exceptionExpandInfo) != ADUMP_SUCCESS) {
184 2 : return ADUMP_FAILED;
185 : }
186 :
187 18 : uint32_t maxArgNum = argSize_ / sizeof(uint64_t);
188 18 : if (static_cast<uint64_t>(skipNum_) + inputNum_ > maxArgNum) {
189 2 : IDE_LOGE("[Dump][Exception] skip num[%" PRIu32 "] plus input num[%" PRIu32 "] is over the max arg num[%" PRIu32
190 : "], load failed.",
191 : skipNum_, inputNum_, maxArgNum);
192 2 : return ADUMP_FAILED;
193 : }
194 :
195 16 : auto data = DumpMemory::CopyDeviceToHostEx(argAddr_, argSize_);
196 16 : if (data == nullptr) {
197 0 : IDE_LOGE("Copy device args to host failed.");
198 0 : return ADUMP_FAILED;
199 : }
200 16 : const void **argOnHost = static_cast<const void **>(data);
201 16 : if (exceptionExpandInfo.type == RT_EXCEPTION_FFTS_PLUS) {
202 1 : LogArgsInfo(argOnHost, maxArgNum);
203 : }
204 :
205 16 : uint32_t argIndex = skipNum_;
206 16 : IDE_LOGE("[Dump][Exception] the begin tensor's index of args is:%" PRIu32 ", args dump count[%" PRIu32 "]",
207 : argIndex, inputNum_);
208 113 : for (uint64_t sizeInfoIdx = sizeBeginIndex_; sizeInfoIdx < inputNum_ + sizeBeginIndex_; ++sizeInfoIdx) {
209 100 : int64_t size = *(reinterpret_cast<int64_t *>(&sizeInfo_[sizeInfoIdx]));
210 : // the size of the level-2 pointer is a negative number in earlier versions.
211 100 : if (size < 0) {
212 16 : size = std::abs(size);
213 16 : if (size > maxArgNum) {
214 3 : IDE_LOGE("[Dump][Exception] size[%" PRId64 "] over arg max[%" PRIu32 "] is invalid, load failed.", size,
215 : maxArgNum);
216 6 : HOST_RT_MEMORY_GUARD(data);
217 3 : return ADUMP_FAILED;
218 3 : }
219 13 : sizeInfoIdx += size;
220 : } else {
221 84 : int32_t ret = LoadInputBuffer(argOnHost, argIndex, sizeInfoIdx);
222 84 : if (ret != ADUMP_SUCCESS) {
223 0 : HOST_RT_MEMORY_GUARD(data);
224 0 : return ADUMP_FAILED;
225 0 : }
226 : }
227 97 : argIndex++;
228 : }
229 13 : LoadTilingData();
230 26 : HOST_RT_MEMORY_GUARD(data);
231 :
232 13 : return ADUMP_SUCCESS;
233 13 : }
234 :
235 67 : int32_t DumpArgs::LoadArgsExceptionInfo(const rtExceptionInfo &exception)
236 : {
237 67 : rtExceptionArgsInfo_t exceptionArgsInfo{};
238 67 : rtExceptionExpandInfo_t exceptionExpandInfo = exception.expandInfo;
239 67 : if (ExceptionInfoCommon::GetExceptionInfo(exception, exceptionArgsInfo) != ADUMP_SUCCESS) {
240 0 : IDE_LOGW("Get exception args info failed.");
241 0 : return ADUMP_FAILED;
242 : }
243 67 : kernelCollector_->LoadKernelInfo(exceptionArgsInfo);
244 :
245 67 : if (LoadArgsInfoWithDfx(exceptionArgsInfo) == ADUMP_SUCCESS) {
246 6 : dumpWithDfxFlag_ = true;
247 : } else {
248 61 : IDE_LOGI("Can not load args info with dfx, use load args info with size info instead of it.");
249 61 : if (LoadArgsInfoWithSizeInfo(exceptionArgsInfo, exceptionExpandInfo) != ADUMP_SUCCESS) {
250 48 : return ADUMP_FAILED;
251 : }
252 : }
253 :
254 19 : streamId_ = std::to_string(exception.streamid);
255 19 : taskId_ = std::to_string(exception.taskid);
256 :
257 19 : return ADUMP_SUCCESS;
258 : }
259 :
260 61 : int32_t DumpArgs::CheckParam(const rtExceptionArgsInfo_t &exceptionArgsInfo,
261 : const rtExceptionExpandInfo_t &exceptionExpandInfo) const
262 : {
263 : // Except the aic/aiv error, other exceptions are also called back. Therefore, warning logs are generated here.
264 61 : if (exceptionArgsInfo.sizeInfo.infoAddr == nullptr || exceptionArgsInfo.sizeInfo.atomicIndex == 0 ||
265 25 : exceptionArgsInfo.argAddr == nullptr) {
266 36 : IDE_LOGW("[Dump][Exception] in infoAddr[%p]|atomicIndex[%" PRIu32 "]|argAddr[%p] has invalid attribute.",
267 : exceptionArgsInfo.sizeInfo.infoAddr, exceptionArgsInfo.sizeInfo.atomicIndex,
268 : exceptionArgsInfo.argAddr);
269 36 : return ADUMP_FAILED;
270 : }
271 : // RT_EXCEPTION_AICORE indicates the normal mode
272 25 : if (exceptionExpandInfo.type == RT_EXCEPTION_AICORE && exceptionArgsInfo.argsize == 0) {
273 2 : IDE_LOGW("[Dump][Exception] the arg size[%" PRIu32 "] is invalid.", exceptionArgsInfo.argsize);
274 2 : return ADUMP_FAILED;
275 : }
276 :
277 23 : uint64_t *sizeInfo = static_cast<uint64_t *>(exceptionArgsInfo.sizeInfo.infoAddr);
278 23 : if (sizeInfo < g_chunk || sizeInfo > (g_chunk + RING_CHUNK_SIZE - 1)) {
279 1 : IDE_LOGE("[Dump][Exception] the size info[%p] address may out of the chunk[%p] address range.", sizeInfo,
280 : g_chunk);
281 1 : return ADUMP_FAILED;
282 : }
283 22 : if (sizeInfo[0] != exceptionArgsInfo.sizeInfo.atomicIndex) {
284 2 : IDE_LOGE("[Dump][Exception] args exception atomic index between %" PRIu64 " and %" PRIu32 " is different.",
285 : sizeInfo[0], exceptionArgsInfo.sizeInfo.atomicIndex);
286 2 : return ADUMP_FAILED;
287 : }
288 :
289 20 : return ADUMP_SUCCESS;
290 : }
291 :
292 20 : int32_t DumpArgs::InitAttributes(const rtExceptionArgsInfo_t &exceptionArgsInfo,
293 : const rtExceptionExpandInfo_t &exceptionExpandInfo)
294 : {
295 20 : sizeInfo_ = static_cast<uint64_t *>(exceptionArgsInfo.sizeInfo.infoAddr);
296 20 : argAddr_ = exceptionArgsInfo.argAddr;
297 :
298 : // RT_EXCEPTION_AICORE indicates the normal mode
299 20 : if (exceptionExpandInfo.type == RT_EXCEPTION_AICORE) {
300 18 : sizeBeginIndex_ = ATOMIC_AND_INPUT_NUM_OFFSET;
301 18 : skipNum_ = (sizeInfo_[1] & SKIP_NUM_MASK) >> SKIP_NUM_SHIFT_BITS;
302 18 : inputNum_ = sizeInfo_[1] & INPUT_NUM_MASK;
303 18 : if (sizeInfo_ + inputNum_ + ATOMIC_AND_INPUT_NUM_OFFSET > g_chunk + RING_CHUNK_SIZE + MAX_TENSOR_NUM) {
304 1 : IDE_LOGE("[Dump][Exception] the value of size info[%p] plus input num[%" PRIu32
305 : "] may exceed the chunk[%p] address range.",
306 : sizeInfo_, inputNum_, g_chunk);
307 1 : return ADUMP_FAILED;
308 : }
309 17 : argSize_ = exceptionArgsInfo.argsize;
310 17 : return ADUMP_SUCCESS;
311 : }
312 :
313 : // RT_EXCEPTION_FFTS_PLUS indicates the ffts+ mode
314 2 : if (exceptionExpandInfo.type == RT_EXCEPTION_FFTS_PLUS) {
315 2 : uint64_t totalContextSizeNum = sizeInfo_[1];
316 2 : if (sizeInfo_ + totalContextSizeNum + ATOMIC_AND_INPUT_NUM_OFFSET >
317 2 : g_chunk + RING_CHUNK_SIZE + MAX_TENSOR_NUM) {
318 1 : IDE_LOGE("[Dump][Exception] the value of size info[%p] plus total num[%" PRIu64
319 : "] may exceed the chunk[%p] address range.",
320 : sizeInfo_, totalContextSizeNum, g_chunk);
321 1 : return ADUMP_FAILED;
322 : }
323 1 : uint32_t contextBeginIndex = ATOMIC_AND_INPUT_NUM_OFFSET;
324 2 : for (uint64_t sizeInfoIdx = contextBeginIndex; sizeInfoIdx < totalContextSizeNum + contextBeginIndex;
325 : ++sizeInfoIdx) {
326 2 : uint32_t inputNumBeginIndex = sizeInfoIdx + CONTEXTID_AND_INPUT_NUM_OFFSET;
327 2 : skipNum_ = (sizeInfo_[inputNumBeginIndex] & SKIP_NUM_MASK) >> SKIP_NUM_SHIFT_BITS;
328 2 : inputNum_ = sizeInfo_[inputNumBeginIndex] & INPUT_NUM_MASK;
329 2 : IDE_LOGI("[Dump][Exception] the context id[%" PRIu64 "] of size info and context id[%" PRIu16
330 : "] of exception info",
331 : sizeInfo_[sizeInfoIdx], exceptionExpandInfo.u.fftsPlusInfo.contextId);
332 2 : if (sizeInfo_[sizeInfoIdx] == exceptionExpandInfo.u.fftsPlusInfo.contextId) {
333 1 : argSize_ = sizeInfo_[sizeInfoIdx + 1];
334 1 : sizeBeginIndex_ = sizeInfoIdx + 3; // 3 - context id | args size | input num
335 1 : return ADUMP_SUCCESS;
336 : }
337 1 : sizeInfoIdx += inputNum_ + CONTEXTID_AND_INPUT_NUM_OFFSET;
338 : }
339 : }
340 :
341 : // log warning - the arg data in ffts+ mode may be in others contextid
342 0 : IDE_LOGW("Data mismatch, some attribute values cannot be obtained.");
343 0 : return ADUMP_FAILED;
344 : }
345 :
346 1 : void DumpArgs::LogArgsInfo(const void **argOnHost, uint32_t maxArgNum)
347 : {
348 1 : const uint32_t argsLogTimes = (maxArgNum % ARGS_PER_STRING_MAX_LEN > 0) ?
349 1 : ((maxArgNum / ARGS_PER_STRING_MAX_LEN) + 1) :
350 : (maxArgNum / ARGS_PER_STRING_MAX_LEN);
351 2 : for (uint32_t i = 1; i <= argsLogTimes; ++i) {
352 1 : std::stringstream ss;
353 1 : uint32_t endIndex = maxArgNum > (i * ARGS_PER_STRING_MAX_LEN) ? (i * ARGS_PER_STRING_MAX_LEN) : maxArgNum;
354 15 : for (uint32_t j = (i - 1) * ARGS_PER_STRING_MAX_LEN; j < endIndex; ++j) {
355 14 : ss << *(argOnHost + j) << ", ";
356 : }
357 1 : oss_ << "[AIC_INFO] args(" << (i - 1) * ARGS_PER_STRING_MAX_LEN << " to " << endIndex
358 1 : << ") after execute:" << ss.str().c_str();
359 1 : RecordCurrentLog();
360 1 : }
361 1 : IDE_LOGE("[AIC_INFO] after execute:args print end");
362 1 : }
363 :
364 84 : int32_t DumpArgs::LoadInputBuffer(const void **argOnHost, const uint32_t argIndex, uint64_t &sizeInfoIdx)
365 : {
366 84 : uint8_t sizeType = (sizeInfo_[sizeInfoIdx] & SIZE_TYPE_MASK) >> SIZE_TYPE_SHIFT_BITS;
367 84 : if (sizeType == NORMAL_TENSOR || sizeType == WORKSPACE_TENSOR) {
368 45 : oss_ << "[Dump][Exception] begin to load normal tensor, index:" << argIndex;
369 45 : RecordCurrentLog();
370 45 : uint64_t tensorSize = sizeInfo_[sizeInfoIdx] & SIZE_MASK;
371 45 : oss_ << "[Dump][Exception] exception info dump args data, addr:" << argOnHost[argIndex]
372 45 : << "; size:" << tensorSize << " bytes";
373 45 : RecordCurrentLog();
374 45 : inputBuffer_.emplace_back(argOnHost[argIndex], tensorSize, argIndex);
375 45 : oss_ << "[Dump][Exception] end to load normal tensor, index:" << argIndex;
376 45 : RecordCurrentLog();
377 84 : } else if (sizeType == NORMAL_PTR_TENSOR || sizeType == SHAPE_PTR_TENSOR) {
378 26 : if (LoadPointerTensor(argOnHost, argIndex, sizeInfoIdx) != ADUMP_SUCCESS) {
379 0 : return ADUMP_FAILED;
380 : }
381 13 : } else if (sizeType == TILING_DATA_PTR) {
382 13 : IDE_LOGE("[Dump][Exception] save tiling data, index:%" PRIu32, argIndex);
383 13 : tilingData_.emplace_back(argOnHost[argIndex], (sizeInfo_[sizeInfoIdx] & SIZE_MASK), argIndex);
384 : } else {
385 0 : IDE_LOGE("[Dump][Exception] args exception size type[%" PRIu8 "] is error.", sizeType);
386 0 : return ADUMP_FAILED;
387 : }
388 :
389 84 : return ADUMP_SUCCESS;
390 : }
391 :
392 26 : int32_t DumpArgs::LoadPointerTensor(const void **argOnHost, const uint32_t argIndex, uint64_t &sizeInfoIdx)
393 : {
394 26 : uint64_t addrBias = reinterpret_cast<uint64_t>(argOnHost[argIndex]) - reinterpret_cast<uint64_t>(argAddr_);
395 26 : if (addrBias >= argSize_) {
396 0 : IDE_LOGE("[Dump][Exception]address bias[%p - %p] is over args size[%" PRIu64 "], invalid.", argOnHost[argIndex],
397 : argAddr_, argSize_);
398 0 : return ADUMP_FAILED;
399 : }
400 :
401 26 : uint64_t inputPtrAddr = addrBias + reinterpret_cast<uint64_t>(argOnHost);
402 26 : void *inputPtr = nullptr;
403 26 : std::string pointerType = "";
404 26 : uint8_t sizeType = (sizeInfo_[sizeInfoIdx] & SIZE_TYPE_MASK) >> SIZE_TYPE_SHIFT_BITS;
405 26 : if (sizeType == NORMAL_PTR_TENSOR) {
406 13 : inputPtr = reinterpret_cast<void *>(inputPtrAddr);
407 13 : pointerType = "normal pointer tensor";
408 13 : } else if (sizeType == SHAPE_PTR_TENSOR) {
409 13 : uint64_t offset = *(reinterpret_cast<uint64_t *>(inputPtrAddr));
410 13 : inputPtr = reinterpret_cast<void *>(inputPtrAddr + offset);
411 13 : pointerType = "shape pointer tensor";
412 : } else {
413 0 : IDE_LOGE("[Dump][Exception] args exception size type[%" PRIu8 "] is error.", sizeType);
414 0 : return ADUMP_FAILED;
415 : }
416 :
417 26 : IDE_LOGE("[Dump][Exception] begin to load %s, index:%" PRIu32 ", addr:%p", pointerType.c_str(), argIndex,
418 : argOnHost[argIndex]);
419 :
420 26 : const void **endArgAddr = argOnHost + argSize_ / sizeof(uint64_t);
421 26 : uint64_t inputPtrNum = sizeInfo_[sizeInfoIdx] & SIZE_MASK;
422 78 : for (uint64_t i = 0; i < inputPtrNum; ++i) {
423 52 : uint64_t inputPtrSize = sizeInfo_[sizeInfoIdx + 1 + i];
424 52 : void **tensorPtr = static_cast<void **>(inputPtr) + i;
425 52 : oss_ << "[Dump][Exception] exception info dump args data, addr:" << *(tensorPtr) << "; size:" << inputPtrSize
426 52 : << " bytes";
427 52 : RecordCurrentLog();
428 52 : if (tensorPtr > endArgAddr) {
429 0 : IDE_LOGE("[Dump][Exception] args address[%p] is over args end address[%p].", tensorPtr, endArgAddr);
430 0 : return ADUMP_FAILED;
431 : }
432 52 : inputBuffer_.emplace_back(*(tensorPtr), inputPtrSize, argIndex);
433 : }
434 26 : sizeInfoIdx += inputPtrNum;
435 :
436 26 : IDE_LOGE("[Dump][Exception] end to load %s, index:%" PRIu32 ", addr:%p", pointerType.c_str(), argIndex,
437 : argOnHost[argIndex]);
438 :
439 26 : return ADUMP_SUCCESS;
440 26 : }
441 :
442 13 : void DumpArgs::LoadTilingData()
443 : {
444 13 : if (tilingData_.size() == 0) {
445 0 : return;
446 : }
447 13 : IDE_LOGE("[Dump][Exception] begin to load tiling data.");
448 26 : for (const auto &it : tilingData_) {
449 13 : oss_ << "[Dump][Exception] exception info dump args data, addr:" << it.addr << "; size:" << it.length
450 13 : << " bytes";
451 13 : RecordCurrentLog();
452 13 : inputBuffer_.emplace_back(it.addr, it.length, it.argIndex);
453 : }
454 13 : IDE_LOGE("[Dump][Exception] end to load tiling data.");
455 : }
456 :
457 32 : int32_t DumpArgs::DumpArgsExceptionFile(const uint32_t deviceId, const std::string &dumpPath)
458 : {
459 32 : std::string dumpFilePath = GetDumpFilePath(dumpPath);
460 32 : IDE_LOGI("[Dump][Exception] The exception dump file path is %s", dumpFilePath.c_str());
461 :
462 32 : DumpFile dumpFile(deviceId, dumpFilePath);
463 32 : dumpFile.SetHeader("");
464 32 : if (dumpWithDfxFlag_) {
465 6 : dumpFile.SetTensorBuffer(tensorBuffer_);
466 6 : dumpFile.SetWorkspaces(workspace_);
467 : #if !defined(ADUMP_SOC_HOST) || ADUMP_SOC_HOST == 1
468 6 : dumpFile.SetMc2spaces(mc2Space_);
469 : #endif
470 : } else {
471 26 : dumpFile.SetInputBuffer(inputBuffer_);
472 : }
473 32 : AdxLogFlush();
474 32 : int32_t ret = dumpFile.Dump(logRecord_);
475 32 : if (ret != ADUMP_SUCCESS) {
476 1 : IDE_LOGE("[Dump][Exception] write exception to file failed, file: %s", dumpFilePath.c_str());
477 1 : return ADUMP_FAILED;
478 : }
479 :
480 31 : (void)mmChmod(dumpFilePath.c_str(), M_IRUSR); // readonly, 400
481 31 : IDE_LOGE("[Dump][Exception] dump exception to file, file: %s", dumpFilePath.c_str());
482 :
483 31 : IDE_LOGI("[Dump][Exception] Dump exception info success.");
484 31 : return ADUMP_SUCCESS;
485 32 : }
486 :
487 32 : int32_t DumpArgs::DumpArgsExceptionInfo(const uint32_t deviceId, const std::string &dumpPath)
488 : {
489 32 : int32_t ret = ADUMP_SUCCESS;
490 32 : if (DumpArgsExceptionFile(deviceId, dumpPath) != ADUMP_SUCCESS) {
491 1 : ret = ADUMP_FAILED;
492 : }
493 32 : if (StartCollectKernelAsync(kernelCollector_, dumpPath) != ADUMP_SUCCESS) {
494 2 : ret = ADUMP_FAILED;
495 : }
496 32 : return ret;
497 : }
498 :
499 32 : std::string DumpArgs::GetDumpFilePath(const std::string &dumpPath) const
500 : {
501 32 : std::string opType = "exception_info";
502 : std::string dumpFileName =
503 32 : opType + "." + streamId_ + "." + taskId_ + "." + SysUtils::GetCurrentTimeWithMillisecond();
504 32 : Path dumpFilePath(dumpPath);
505 32 : dumpFilePath.Concat(dumpFileName);
506 32 : dumpFilePath.RealPath();
507 64 : return dumpFilePath.GetString();
508 32 : }
509 :
510 201 : void DumpArgs::RecordCurrentLog()
511 : {
512 201 : IDE_LOGE("%s", oss_.str().c_str());
513 201 : logRecord_.emplace_back(oss_.str() + "\n");
514 402 : oss_.str("");
515 201 : }
516 :
517 1 : bool DumpArgs::DumpArgsDumpWithDfxFlag() const
518 : {
519 1 : return dumpWithDfxFlag_;
520 : }
521 :
522 1 : const std::vector<InputBuffer> &DumpArgs::DumpArgsGetInputBuffer() const
523 : {
524 1 : return inputBuffer_;
525 : }
526 :
527 0 : const std::vector<TensorBuffer> &DumpArgs::DumpArgsGetTensorBuffer() const
528 : {
529 0 : return tensorBuffer_;
530 : }
531 :
532 0 : const std::vector<DumpWorkspace> &DumpArgs::DumpArgsGetWorkSpace() const
533 : {
534 0 : return workspace_;
535 : }
536 :
537 : } // namespace Adx
|
