Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 : #include <map>
11 : #include "adump_pub.h"
12 : #include "log/adx_log.h"
13 : #include "runtime/mem.h"
14 : #include "adump_dsmi.h"
15 : #include "sys_utils.h"
16 : #include "path.h"
17 : #include "exception_info_common.h"
18 : #include "kernel_symbol_locator.h"
19 : #include "dump_core.h"
20 :
21 : namespace Adx {
22 8 : int32_t DumpCore::DumpCoreFile(const rtExceptionInfo &exception)
23 : {
24 8 : if (ExceptionInfoCommon::GetExceptionRegInfo(exception, exceptionRegInfo_) == ADUMP_SUCCESS) {
25 8 : KernelSymbolLocator::DumpErrorSymbols(exception, exceptionRegInfo_);
26 : }
27 8 : DumpCoreInfo(exception.deviceid);
28 8 : DumpGlobalMemory(exception);
29 8 : DumpResourceInit();
30 :
31 44 : for (uint16_t aicId : aiCoreIds_) {
32 36 : DumpLocalMemory(CORE_TYPE_AIC, aicId);
33 36 : DumpRegister(CORE_TYPE_AIC, aicId);
34 : }
35 62 : for (uint16_t aivId : aiVectorCoreIds_) {
36 54 : DumpLocalMemory(CORE_TYPE_AIV, aivId);
37 54 : DumpRegister(CORE_TYPE_AIV, aivId);
38 : }
39 :
40 8 : SaveCoreFile(exception);
41 :
42 8 : return ADUMP_SUCCESS;
43 : }
44 :
45 8 : void DumpCore::DumpGlobalMemory(const rtExceptionInfo &exception)
46 : {
47 8 : std::vector<GlobalMemInfo> memInfoList;
48 :
49 8 : rtExceptionArgsInfo_t exceptionArgsInfo{};
50 8 : if (ExceptionInfoCommon::GetExceptionInfo(exception, exceptionArgsInfo) != ADUMP_SUCCESS) {
51 0 : IDE_LOGE("Get exception args info failed.");
52 0 : return;
53 : }
54 8 : DumpArgsInfo(exceptionArgsInfo, memInfoList);
55 :
56 8 : DumpArgs args;
57 8 : if (args.LoadArgsExceptionInfo(exception) == ADUMP_SUCCESS) {
58 1 : if (args.DumpArgsDumpWithDfxFlag()) {
59 0 : DumpTensorBuffer(args, memInfoList);
60 0 : DumpWorkSpace(args, memInfoList);
61 : } else {
62 1 : DumpInput(args, memInfoList);
63 : }
64 : }
65 :
66 8 : DumpStack(exceptionArgsInfo.exceptionKernelInfo.bin, memInfoList);
67 8 : DumpDeviceKernelBin(exceptionArgsInfo.exceptionKernelInfo, memInfoList);
68 8 : DumpGlobalAuxInfo(memInfoList);
69 :
70 8 : DumpHostKernelBin(exceptionArgsInfo.exceptionKernelInfo);
71 8 : DumpHostFile(exceptionArgsInfo);
72 8 : }
73 :
74 8 : void DumpCore::DumpResourceInit()
75 : {
76 8 : char version[SOC_VERSION_SIZE] = {0};
77 8 : rtError_t ret = rtGetSocVersion(version, SOC_VERSION_SIZE);
78 8 : IDE_CTRL_VALUE_FAILED(ret == RT_ERROR_NONE, return, "Failed to get soc version");
79 8 : const std::string socVersion(version);
80 :
81 8 : IDE_CTRL_VALUE_FAILED(AdumpPlatformApi::GetAicoreSizeInfo(socVersion, bufferSize_),
82 : return, "Failed to read platform info from fe api.");
83 8 : IDE_LOGD("Get local buffer size: L0A: %llu, L0B: %llu, L0C: %llu, L1: %llu, UB: %llu",
84 : bufferSize_.l0aSize, bufferSize_.l0bSize, bufferSize_.l0cSize, bufferSize_.l1Size, bufferSize_.ubSize);
85 8 : }
86 :
87 8 : void DumpCore::DumpCoreInfo(uint32_t devId)
88 : {
89 8 : std::string data(sizeof(DevInfo), 0);
90 8 : DevInfo *devInfo = reinterpret_cast<DevInfo *>(const_cast<char *>(data.data()));
91 8 : devInfo->devId = devId;
92 8 : IDE_CTRL_VALUE_FAILED(AdumpDsmi::DrvGetPlatformType(devInfo->devType), return, "Get platform type failed.");
93 :
94 8 : rtError_t ret = rtDebugGetStalledCore(&devInfo->coreInfo);
95 8 : IDE_CTRL_VALUE_FAILED(ret == RT_ERROR_NONE, return, "Get core id failed, ret: %d", ret);
96 8 : IDE_LOGD("get core info: aicBitmap: 0x%llx 0x%llx, aivBitmap: 0x%llx 0x%llx", devInfo->coreInfo.aicBitmap0,
97 : devInfo->coreInfo.aicBitmap1, devInfo->coreInfo.aivBitmap0, devInfo->coreInfo.aivBitmap1);
98 :
99 520 : for (uint16_t i = 0; i < CORE_ID_BIT_MAP_SIZE; ++i) {
100 512 : if ((devInfo->coreInfo.aicBitmap0 & (1ULL << i)) != 0) {
101 8 : aiCoreIds_.emplace_back(i);
102 : }
103 512 : if ((devInfo->coreInfo.aicBitmap1 & (1ULL << i)) != 0) {
104 8 : aiCoreIds_.emplace_back(i + CORE_ID_BIT_MAP_SIZE);
105 : }
106 512 : if ((devInfo->coreInfo.aivBitmap0 & (1ULL << i)) != 0) {
107 8 : aiVectorCoreIds_.emplace_back(i);
108 : }
109 512 : if ((devInfo->coreInfo.aivBitmap1 & (1ULL << i)) != 0) {
110 16 : aiVectorCoreIds_.emplace_back(i + CORE_ID_BIT_MAP_SIZE);
111 : }
112 : }
113 :
114 8 : ELF::SectionPtr devtbl = coreFile_.AddSection(ASCEND_SHTYPE_DEVTBL, ASCEND_SHNAME_DEVTBL);
115 8 : IDE_CTRL_VALUE_FAILED(devtbl != nullptr, return, "Create %s section failed.", ASCEND_SHNAME_DEVTBL.c_str());
116 8 : devtbl->SetData(data);
117 8 : }
118 :
119 90 : void DumpCore::DumpLocalMemory(uint8_t coreType, uint16_t coreId)
120 : {
121 90 : std::string coreIdStr = std::to_string(ConvertCoreId(coreType, coreId));
122 90 : std::string sectionName = ASCEND_SHNAME_LOCAL + "." + coreIdStr;
123 90 : std::vector<LocalMemInfo> localMemInfoList;
124 :
125 90 : DumpBuffer(coreType, coreId, sectionName, localMemInfoList);
126 90 : DumpCache(coreType, coreId, sectionName, localMemInfoList);
127 90 : DumpLocalAuxInfo(coreIdStr, localMemInfoList);
128 90 : }
129 :
130 90 : void DumpCore::DumpCache(uint8_t coreType, uint16_t coreId, const std::string §ionName,
131 : std::vector<LocalMemInfo> &localMemInfoList)
132 : {
133 : // icache
134 90 : DumpCache(coreType, coreId, binParam_, sectionName, localMemInfoList);
135 :
136 : // dcache
137 90 : DumpCache(coreType, coreId, argsParam_, sectionName, localMemInfoList);
138 90 : DumpCache(coreType, coreId, tilingDataParam_, sectionName, localMemInfoList);
139 :
140 2390 : for (const auto &stackParam : stackParamList_) {
141 2300 : if (stackParam.coreType == coreType && stackParam.coreId == coreId) {
142 460 : DumpCache(coreType, coreId, stackParam, sectionName, localMemInfoList);
143 : }
144 : }
145 90 : }
146 :
147 730 : void DumpCore::DumpCache(uint8_t coreType, uint16_t coreId, const CacheParam &cacheParam, const std::string §ionName,
148 : std::vector<LocalMemInfo> &localMemInfoList)
149 : {
150 730 : if (cacheParam.memSize == 0) {
151 650 : return;
152 : }
153 640 : std::string cacheData(cacheParam.memSize, 0);
154 640 : rtDebugMemoryParam_t param =
155 640 : {coreType, 0, coreId, cacheParam.cacheType, 0, 0, cacheParam.memAddr, 0, cacheParam.memSize};
156 640 : param.dstAddr = reinterpret_cast<uint64_t>(cacheData.data());
157 640 : rtError_t ret = rtDebugReadAICore(¶m);
158 640 : IDE_CTRL_VALUE_FAILED(ret == RT_ERROR_NONE, return, "Failed to copy icache data from device, ret: %d", ret);
159 :
160 80 : ELF::SectionPtr localSec = coreFile_.AddSection(ASCEND_SHTYPE_LOCAL, sectionName);
161 80 : IDE_CTRL_VALUE_FAILED(localSec != nullptr, return, "Create %s section failed.", sectionName.c_str());
162 :
163 80 : LocalMemInfo localMemInfo = {param.memLen, localSec->GetIndex(), cacheParam.sectionIndex, param.debugMemType, 0};
164 80 : localMemInfoList.emplace_back(localMemInfo);
165 80 : localSec->SetData(cacheData);
166 80 : localSec->SetInfo(localMemInfoList.size() - 1);
167 :
168 80 : IDE_LOGD("Dump cache data success, core type: %hhu, core id: %hu, cache type: %d, addr: %llx, size: %llu",
169 : coreType, coreId, cacheParam.cacheType, cacheParam.memAddr, cacheParam.memSize);
170 640 : }
171 :
172 90 : void DumpCore::DumpBuffer(uint8_t coreType, uint16_t coreId, const std::string §ionName,
173 : std::vector<LocalMemInfo> &localMemInfoList)
174 : {
175 : std::vector<rtDebugMemoryParam_t> memParamList = {
176 90 : { CORE_TYPE_AIC, 0, coreId, RT_MEM_TYPE_L0A, 0, 0, 0, 0, bufferSize_.l0aSize },
177 90 : { CORE_TYPE_AIC, 0, coreId, RT_MEM_TYPE_L0B, 0, 0, 0, 0, bufferSize_.l0bSize },
178 90 : { CORE_TYPE_AIC, 0, coreId, RT_MEM_TYPE_L0C, 0, 0, 0, 0, bufferSize_.l0cSize },
179 90 : { CORE_TYPE_AIC, 0, coreId, RT_MEM_TYPE_L1, 0, 0, 0, 0, bufferSize_.l1Size },
180 90 : { CORE_TYPE_AIV, 0, coreId, RT_MEM_TYPE_UB, 0, 0, 0, 0, bufferSize_.ubSize },
181 180 : };
182 540 : for (auto &memParam : memParamList) {
183 450 : if (memParam.coreType != coreType) {
184 406 : continue;
185 : }
186 : // memLen zero check
187 198 : std::string localData(memParam.memLen, 0);
188 198 : memParam.dstAddr = reinterpret_cast<uint64_t>(localData.data());
189 :
190 198 : rtError_t ret = rtDebugReadAICore(&memParam);
191 198 : IDE_CTRL_VALUE_FAILED_NODO(ret == RT_ERROR_NONE, continue, "Failed to copy data from device, ret: %d", ret);
192 :
193 44 : ELF::SectionPtr localSec = coreFile_.AddSection(ASCEND_SHTYPE_LOCAL, sectionName);
194 44 : IDE_CTRL_VALUE_FAILED_NODO(localSec != nullptr, continue, "Create %s section failed.", sectionName.c_str());
195 :
196 44 : LocalMemInfo localMemInfo = {memParam.memLen, localSec->GetIndex(), 0, memParam.debugMemType, 0};
197 44 : localMemInfoList.emplace_back(localMemInfo);
198 44 : localSec->SetData(localData);
199 44 : localSec->SetInfo(localMemInfoList.size() - 1);
200 :
201 44 : IDE_LOGD("Dump local memory success, core type: %hhu, core id: %hu, type: %d, size: %llu",
202 : coreType, coreId, memParam.debugMemType, memParam.memLen);
203 198 : }
204 90 : }
205 :
206 90 : void DumpCore::DumpLocalAuxInfo(const std::string &coreIdStr, std::vector<LocalMemInfo> &localMemInfoList)
207 : {
208 90 : if (localMemInfoList.empty()) {
209 70 : return;
210 : }
211 :
212 20 : size_t totalSize = localMemInfoList.size() * sizeof(LocalMemInfo);
213 20 : std::string localData(reinterpret_cast<const char *>(localMemInfoList.data()), totalSize);
214 :
215 20 : std::string sectionName = ASCEND_SHNAME_AUXINFO_LOCAL + "." + coreIdStr;
216 20 : ELF::SectionPtr localSec = coreFile_.AddSection(ASCEND_SHTYPE_AUXINFO_LOCAL, sectionName);
217 20 : IDE_CTRL_VALUE_FAILED(localSec != nullptr, return, "Create %s section failed.", sectionName.c_str());
218 :
219 20 : localSec->SetData(localData);
220 20 : localSec->SetEntSize(sizeof(LocalMemInfo));
221 144 : for (const LocalMemInfo &localMemInfo : localMemInfoList) {
222 124 : ELF::SectionPtr sec = coreFile_.GetSectionByIndex(localMemInfo.sectionIndex);
223 124 : IDE_CTRL_VALUE_FAILED_NODO(sec != nullptr, continue, "Get section by index failed, index: %u",
224 : localMemInfo.sectionIndex);
225 124 : sec->SetLink(localSec->GetIndex());
226 124 : }
227 20 : }
228 :
229 8 : void DumpCore::SaveCoreFile(const rtExceptionInfo &exception)
230 : {
231 8 : std::string kernelName;
232 8 : rtExceptionArgsInfo_t exceptionArgsInfo{};
233 8 : if (ExceptionInfoCommon::GetExceptionInfo(exception, exceptionArgsInfo) != ADUMP_SUCCESS) {
234 0 : IDE_LOGE("Get exception args info failed.");
235 0 : kernelName = DEFAULT_KERNEL_NAME;
236 : } else {
237 : std::string rtKernelName(exceptionArgsInfo.exceptionKernelInfo.kernelName,
238 8 : exceptionArgsInfo.exceptionKernelInfo.kernelNameSize);
239 8 : kernelName = rtKernelName.empty() ? DEFAULT_KERNEL_NAME : rtKernelName;
240 8 : }
241 16 : std::string dumpFileName = kernelName + "." + std::to_string(exception.streamid) + "." +
242 24 : std::to_string(exception.taskid) + "." + SysUtils::GetCurrentTimeWithMillisecond() + ".core";
243 : // File names should not exceed the filesystem limits.
244 8 : dumpFileName = dumpFileName.length() > 255U ? dumpFileName.substr(dumpFileName.length() - 255U) : dumpFileName;
245 8 : Path dumpFilePath(path_);
246 8 : IDE_CTRL_VALUE_FAILED(dumpFilePath.RealPath(), return, "Get path %s real path failed, strerr=%s.",
247 : dumpFilePath.GetCString(), strerror(errno));
248 8 : std::string dir = dumpFilePath.GetString();
249 8 : dumpFilePath.Concat(dumpFileName);
250 8 : IDE_CTRL_VALUE_FAILED(dumpFilePath.ParentPath().GetString() == dir, return,
251 : "Check dump file path %s failed.", dumpFilePath.GetCString());
252 8 : coreFile_.Save(dumpFilePath.GetString());
253 8 : }
254 :
255 30 : int32_t DumpCore::D2HMemcpyWithNoCheck(const GlobalMemInfo &memInfo, std::string &data) const
256 : {
257 30 : std::vector<char> buffer(memInfo.size);
258 30 : rtError_t rtRet = rtMemcpyEx(static_cast<void *>(buffer.data()), memInfo.size,
259 30 : reinterpret_cast<void *>(memInfo.devAddr), memInfo.size, RT_MEMCPY_DEVICE_TO_HOST);
260 30 : if (rtRet != RT_ERROR_NONE) {
261 15 : IDE_LOGE("Call rtMemcpyEx failed, data type: %d, addr: 0x%llx, size: %llu, ret: %d",
262 : static_cast<int32_t>(memInfo.type), memInfo.devAddr, memInfo.size, rtRet);
263 15 : return ADUMP_FAILED;
264 : } else {
265 15 : data.assign(buffer.data(), memInfo.size);
266 : }
267 15 : return ADUMP_SUCCESS;
268 30 : }
269 :
270 53 : int32_t DumpCore::D2HMemcpyWithCheck(const GlobalMemInfo &memInfo, std::string &data) const
271 : {
272 53 : rtMemInfo_t info{};
273 53 : uint64_t *deviceAddr[1] = {reinterpret_cast<uint64_t *>(memInfo.devAddr)};
274 53 : info.addrInfo.addr = static_cast<uint64_t **>(deviceAddr);
275 53 : info.addrInfo.cnt = 1;
276 53 : info.addrInfo.memType = RT_MEM_MASK_DEV_TYPE | RT_MEM_MASK_RSVD_TYPE;
277 53 : info.addrInfo.flag = true;
278 53 : rtError_t rtRet = rtMemGetInfoByType(static_cast<int32_t>(devId_), RT_MEM_INFO_TYPE_ADDR_CHECK, &info);
279 53 : if ((rtRet != RT_ERROR_NONE) || (info.addrInfo.flag == false)) {
280 2 : IDE_LOGE("Check global memory addr 0x%llx invalid", memInfo.devAddr);
281 2 : return ADUMP_FAILED;
282 : }
283 :
284 51 : std::vector<char> buffer(memInfo.size);
285 51 : rtRet = rtMemcpy(static_cast<void *>(buffer.data()), memInfo.size,
286 51 : reinterpret_cast<void *>(memInfo.devAddr), memInfo.size, RT_MEMCPY_DEVICE_TO_HOST);
287 51 : if (rtRet != RT_ERROR_NONE) {
288 2 : IDE_LOGE("Call rtMemcpy failed, data type: %d, addr: 0x%llx, size: %llu, ret: %d",
289 : static_cast<int32_t>(memInfo.type), memInfo.devAddr, memInfo.size, rtRet);
290 2 : return ADUMP_FAILED;
291 : } else {
292 49 : data.assign(buffer.data(), memInfo.size);
293 : }
294 49 : return ADUMP_SUCCESS;
295 51 : }
296 :
297 84 : int32_t DumpCore::ProcessGlobalMemory(GlobalMemInfo &memInfo, std::vector<GlobalMemInfo> &memInfoList, bool checkAddr)
298 : {
299 84 : IDE_LOGI("Dump device data. data type: %hu, addr: 0x%llx, size: %llu", memInfo.type, memInfo.devAddr, memInfo.size);
300 84 : std::string curData(memInfo.size, 0);
301 84 : if (memInfo.size != 0) {
302 83 : if (checkAddr) {
303 53 : if (D2HMemcpyWithCheck(memInfo, curData) != ADUMP_SUCCESS) {
304 4 : memInfo.size |= INVALID_DATA_FLAG;
305 : }
306 : } else {
307 30 : if (D2HMemcpyWithNoCheck(memInfo, curData) != ADUMP_SUCCESS) {
308 15 : memInfo.size |= INVALID_DATA_FLAG;
309 : }
310 : }
311 : }
312 :
313 84 : ELF::SectionPtr curSection = coreFile_.AddSection(ASCEND_SHTYPE_GLOBAL, ASCEND_SHNAME_GLOBAL);
314 84 : IDE_CTRL_VALUE_FAILED(curSection != nullptr, return ADUMP_FAILED, "Create %s section failed.",
315 : ASCEND_SHNAME_GLOBAL.c_str());
316 :
317 84 : memInfo.sectionIndex = curSection->GetIndex();
318 84 : memInfoList.emplace_back(memInfo);
319 84 : curSection->SetInfo(memInfoList.size() - 1);
320 84 : curSection->SetData(curData);
321 84 : curSection->SetAddr(memInfo.devAddr);
322 84 : return ADUMP_SUCCESS;
323 84 : }
324 :
325 8 : void DumpCore::DumpArgsInfo(const rtExceptionArgsInfo_t &exceptionArgsInfo, std::vector<GlobalMemInfo> &memInfoList)
326 : {
327 8 : GlobalMemInfo memInfo = {
328 8 : .devAddr = reinterpret_cast<uint64_t>(exceptionArgsInfo.argAddr),
329 8 : .size = exceptionArgsInfo.argsize,
330 : .sectionIndex = 0,
331 : .type = DfxTensorType::ARGS,
332 : .reserve = 0,
333 : .extraInfo = {.coreInfo = {.coreId = 0}}
334 8 : };
335 :
336 8 : int32_t ret = ProcessGlobalMemory(memInfo, memInfoList);
337 8 : IDE_CTRL_VALUE_FAILED(ret == ADUMP_SUCCESS, return, "Save args to section failed.");
338 8 : argsParam_ = {0, 0, exceptionArgsInfo.argsize, memInfo.devAddr, memInfo.sectionIndex, RT_MEM_TYPE_DCACHE};
339 : }
340 :
341 1 : void DumpCore::DumpInput(const DumpArgs &args, std::vector<GlobalMemInfo> &memInfoList)
342 : {
343 1 : const std::vector<InputBuffer> inputBuffer = args.DumpArgsGetInputBuffer();
344 1 : int32_t ret = 0;
345 9 : for (const auto &input : inputBuffer) {
346 8 : if (input.addr == nullptr) { // skip placeholder tensor
347 0 : continue;
348 : }
349 8 : GlobalMemInfo memInfo = {
350 8 : .devAddr = reinterpret_cast<uint64_t>(input.addr),
351 8 : .size = input.length,
352 : .sectionIndex = 0,
353 : .type = DfxTensorType::INPUT_TENSOR,
354 : .reserve = 0,
355 : .extraInfo = {.shape = {.dim = 0, .dimSize = {0}}}
356 8 : };
357 :
358 8 : ret = ProcessGlobalMemory(memInfo, memInfoList);
359 8 : IDE_CTRL_VALUE_FAILED_NODO(ret == ADUMP_SUCCESS, continue,
360 : "Save input to section failed, addr: 0x%llx, size: %llu", memInfo.devAddr, memInfo.size);
361 : }
362 1 : }
363 :
364 0 : void DumpCore::DumpTensorBuffer(const DumpArgs &args, std::vector<GlobalMemInfo> &memInfoList)
365 : {
366 0 : const std::vector<TensorBuffer> tensorBuffer = args.DumpArgsGetTensorBuffer();
367 0 : int32_t ret = 0;
368 0 : for (const auto &tensor : tensorBuffer) {
369 0 : if (tensor.addr == nullptr) { // skip placeholder tensor
370 0 : continue;
371 : }
372 0 : IDE_CTRL_VALUE_FAILED_NODO(tensor.dimension < MAX_DIM_SIZE, continue,
373 : "Invalid dimension %llu, addr: %p, arg index: %u, tensor type %hu, pointer type: %hu",
374 : tensor.dimension, tensor.addr, tensor.argIndex, tensor.tensorType, tensor.pointerType);
375 :
376 0 : GlobalMemInfo memInfo = {
377 0 : .devAddr = reinterpret_cast<uint64_t>(tensor.addr),
378 0 : .size = tensor.GetTotalByteSize(),
379 : .sectionIndex = 0,
380 0 : .type = static_cast<DfxTensorType>(tensor.tensorType),
381 : .reserve = 0,
382 0 : .extraInfo = {.shape = {.dim = static_cast<uint32_t>(tensor.dimension), .dimSize = {0}}}
383 0 : };
384 0 : for (uint32_t i = 0; i < memInfo.extraInfo.shape.dim; ++i) {
385 0 : memInfo.extraInfo.shape.dimSize[i] = tensor.shape[i];
386 : }
387 0 : ret = ProcessGlobalMemory(memInfo, memInfoList);
388 0 : IDE_CTRL_VALUE_FAILED_NODO(ret == ADUMP_SUCCESS, continue,
389 : "Save tensor to section failed, addr: 0x%llx, size: %llu, type: %hu",
390 : memInfo.devAddr, memInfo.size, memInfo.type);
391 :
392 0 : if (tensor.tensorType == DfxTensorType::TILING_DATA) {
393 0 : tilingDataParam_ = {0, 0, memInfo.size, memInfo.devAddr, memInfo.sectionIndex, RT_MEM_TYPE_DCACHE};
394 : }
395 : }
396 0 : }
397 :
398 0 : void DumpCore::DumpWorkSpace(const DumpArgs &args, std::vector<GlobalMemInfo> &memInfoList)
399 : {
400 0 : const std::vector<DumpWorkspace> workSpace = args.DumpArgsGetWorkSpace();
401 0 : int32_t ret = 0;
402 0 : for (const auto &ws : workSpace) {
403 0 : GlobalMemInfo memInfo = {
404 0 : .devAddr = reinterpret_cast<uint64_t>(ws.addr),
405 0 : .size = ws.bytes,
406 : .sectionIndex = 0,
407 : .type = DfxTensorType::WORKSPACE_TENSOR,
408 : .reserve = 0,
409 : .extraInfo = {.shape = {.dim = 0, .dimSize = {0}}}
410 0 : };
411 :
412 0 : ret = ProcessGlobalMemory(memInfo, memInfoList);
413 0 : IDE_CTRL_VALUE_FAILED_NODO(ret == ADUMP_SUCCESS, continue,
414 : "Save input to section failed, addr: 0x%llx, size: %llu", memInfo.devAddr, memInfo.size);
415 : }
416 0 : }
417 :
418 8 : void DumpCore::DumpStack(const rtBinHandle &binHandle, std::vector<GlobalMemInfo> &memInfoList)
419 : {
420 44 : for (uint16_t aicId : aiCoreIds_) {
421 36 : IDE_LOGI("Dump core stack data. coreType: aic, coreId: %hu", aicId);
422 36 : DumpCoreStack(
423 : binHandle, CORE_TYPE_AIC, aicId, false, RT_STACK_TYPE_SCALAR, DfxTensorType::STACK, memInfoList);
424 36 : DumpCoreStack(
425 : binHandle, CORE_TYPE_AIC, aicId, true, RT_STACK_TYPE_SIMT, DfxTensorType::SIMT_STACK, memInfoList);
426 : }
427 62 : for (uint16_t aivId : aiVectorCoreIds_) {
428 54 : IDE_LOGI("Dump core stack data. coreType: aiv, coreId: %hu", aivId);
429 54 : DumpCoreStack(
430 : binHandle, CORE_TYPE_AIV, aivId, false, RT_STACK_TYPE_SCALAR, DfxTensorType::STACK, memInfoList);
431 54 : DumpCoreStack(
432 : binHandle, CORE_TYPE_AIV, aivId, true, RT_STACK_TYPE_SIMT, DfxTensorType::SIMT_STACK, memInfoList);
433 : }
434 8 : }
435 :
436 180 : void DumpCore::DumpCoreStack(const rtBinHandle& binHandle, const uint8_t coreType, const uint16_t coreId,
437 : bool checkAddr, const rtStackType_t rtStackType, DfxTensorType dumpStackType,
438 : std::vector<GlobalMemInfo> &memInfoList)
439 : {
440 180 : const void *stackAddr = nullptr;
441 180 : uint32_t stackSize = 0;
442 180 : rtError_t rtRet = rtGetStackBuffer(binHandle, 0U, rtStackType, coreType, coreId, &stackAddr, &stackSize);
443 180 : if (rtRet == ACL_ERROR_RT_FEATURE_NOT_SUPPORT) {
444 0 : IDE_LOGW("RTS does not support rtGetStackBuffer feature. stackType: %d, ret: %d", rtStackType, rtRet);
445 120 : return;
446 : }
447 180 : if ((rtRet != RT_ERROR_NONE) || (stackAddr == nullptr)) {
448 120 : IDE_LOGE("Call rtGetStackBuffer to get stack data failed, coreType: %hhu, coreId: %hu, ret: %d",
449 : coreType, coreId, static_cast<int32_t>(rtRet));
450 120 : return;
451 : }
452 60 : GlobalMemInfo memInfo = {
453 : .devAddr = reinterpret_cast<uint64_t>(stackAddr),
454 60 : .size = stackSize,
455 : .sectionIndex = 0,
456 : .type = dumpStackType,
457 : .reserve = 0,
458 120 : .extraInfo = {.coreInfo = {ConvertCoreId(coreType, coreId)}}
459 60 : };
460 :
461 60 : int32_t ret = ProcessGlobalMemory(memInfo, memInfoList, checkAddr);
462 60 : IDE_CTRL_VALUE_FAILED(ret == ADUMP_SUCCESS, return,
463 : "Save stack section failed, core type: %hhu, core id: %hu", coreType, coreId);
464 :
465 60 : stackParamList_.emplace_back(
466 60 : coreType, coreId, stackSize, memInfo.devAddr, memInfo.sectionIndex, RT_MEM_TYPE_DCACHE);
467 : }
468 :
469 8 : void DumpCore::DumpHostKernelBin(const rtExceptionKernelInfo_t &kernelInfo)
470 : {
471 8 : void *binAddr = nullptr;
472 8 : uint32_t binSize = 0;
473 8 : rtError_t rtRet = rtGetBinBuffer(kernelInfo.bin, RT_BIN_HOST_ADDR, &binAddr, &binSize);
474 8 : IDE_CTRL_VALUE_FAILED((rtRet == RT_ERROR_NONE) && (binAddr != nullptr), return,
475 : "Call rtGetBinBuffer get host kernel object failed, ret: %d", static_cast<int32_t>(rtRet));
476 :
477 8 : std::string hostBinData(static_cast<char*>(binAddr), binSize);
478 : ELF::SectionPtr curSection = coreFile_.AddSection(ASCEND_SHTYPE_HOST_KERNEL_OBJECT,
479 8 : ASCEND_SHNAME_HOST_KERNEL_OBJECT);
480 8 : IDE_CTRL_VALUE_FAILED(curSection != nullptr, return, "Create %s section failed.",
481 : ASCEND_SHNAME_HOST_KERNEL_OBJECT.c_str());
482 8 : curSection->SetData(hostBinData);
483 8 : }
484 :
485 8 : void DumpCore::DumpDeviceKernelBin(const rtExceptionKernelInfo_t &kernelInfo, std::vector<GlobalMemInfo> &memInfoList)
486 : {
487 8 : void *deviceBinAddr = nullptr;
488 8 : uint32_t binSize = 0;
489 8 : rtError_t rtRet = rtGetBinBuffer(kernelInfo.bin, RT_BIN_DEVICE_ADDR, &deviceBinAddr, &binSize);
490 8 : IDE_CTRL_VALUE_FAILED(rtRet == RT_ERROR_NONE, return,
491 : "Call rtGetBinBuffer get device kernel object failed, ret: %d", static_cast<int32_t>(rtRet));
492 :
493 8 : GlobalMemInfo memInfo = {
494 : .devAddr = reinterpret_cast<uint64_t>(deviceBinAddr),
495 8 : .size = binSize,
496 : .sectionIndex = 0,
497 : .type = DfxTensorType::DEVICE_KERNEL_OBJECT,
498 : .reserve = 0,
499 : .extraInfo = {.shape = {.dim = 0, .dimSize = {0}}}
500 8 : };
501 :
502 8 : int32_t ret = ProcessGlobalMemory(memInfo, memInfoList);
503 8 : IDE_CTRL_VALUE_FAILED(ret == ADUMP_SUCCESS, return,
504 : "Save device kernel object section failed, addr: %p, size: %u", deviceBinAddr, binSize);
505 :
506 8 : binParam_ = {0, 0, binSize, memInfo.devAddr, memInfo.sectionIndex, RT_MEM_TYPE_ICACHE};
507 : }
508 :
509 8 : void DumpCore::DumpHostFile(const rtExceptionArgsInfo_t &argsInfo)
510 : {
511 8 : KernelInfoCollector collector;
512 8 : collector.LoadKernelInfo(argsInfo);
513 8 : std::vector<std::string> searchPath = collector.GetSearchPath();
514 8 : std::string kernelName = collector.GetProcessedKernelName();
515 8 : IDE_CTRL_VALUE_FAILED(!kernelName.empty(), return, "Get kernel name failed.");
516 3 : IDE_LOGI("kernel name: %s", kernelName.c_str());
517 3 : DumpKernelInfo(kernelName);
518 :
519 3 : for (const auto &path : searchPath) {
520 3 : std::string jsonFilePath = collector.SearchJsonFiles(path, kernelName);
521 3 : if (jsonFilePath.empty()) {
522 0 : continue;
523 : }
524 3 : DumpHostFile(jsonFilePath, ASCEND_SHTYPE_FILE_KERNEL_JSON, ASCEND_SHNAME_FILE_KERNEL_JSON);
525 3 : std::string kernelFilePath = jsonFilePath.substr(0, jsonFilePath.size() - JSON_SUFFIX_LEN) + ".o";
526 3 : DumpHostFile(kernelFilePath, ASCEND_SHTYPE_FILE_KERNEL_OBJECT, ASCEND_SHNAME_FILE_KERNEL_OBJECT);
527 3 : break;
528 6 : }
529 18 : }
530 :
531 6 : void DumpCore::DumpHostFile(const std::string &filePath, uint32_t sectionType, const std::string §ionName)
532 : {
533 6 : char canonicalPath[PATH_MAX] = {0};
534 6 : IDE_CTRL_VALUE_FAILED(realpath(filePath.c_str(), canonicalPath) != nullptr, return,
535 : "Get file path %s realpath failed, strerr=%s", filePath.c_str(), strerror(errno));
536 6 : std::ifstream file(canonicalPath);
537 6 : IDE_CTRL_VALUE_FAILED(file.is_open(), return, "Open file failed, path: %s", canonicalPath);
538 :
539 : do {
540 6 : std::string content((std::istreambuf_iterator<char>(file)), std::istreambuf_iterator<char>());
541 6 : IDE_CTRL_VALUE_FAILED_NODO(!content.empty(), break, "Read file content failed, path: %s", canonicalPath);
542 :
543 6 : ELF::SectionPtr curSection = coreFile_.AddSection(sectionType, sectionName);
544 6 : IDE_CTRL_VALUE_FAILED_NODO(curSection != nullptr, break, "Create %s section failed.", sectionName.c_str());
545 :
546 6 : curSection->SetData(content);
547 6 : } while (0);
548 :
549 6 : file.close();
550 6 : }
551 :
552 3 : void DumpCore::DumpKernelInfo(const std::string &kernelName)
553 : {
554 3 : std::string content;
555 3 : uint32_t nameSize = static_cast<uint32_t>(kernelName.length());
556 3 : content.append(reinterpret_cast<const char *>(&nameSize), sizeof(nameSize));
557 3 : content.append(kernelName);
558 3 : ELF::SectionPtr curSection = coreFile_.AddSection(ASCEND_SHTYPE_KERNEL_INFO, ASCEND_SHNAME_KERNEL_INFO);
559 3 : IDE_CTRL_VALUE_FAILED_NODO(curSection != nullptr, return,
560 : "Create %s section failed.",ASCEND_SHNAME_KERNEL_INFO.c_str());
561 3 : curSection->SetData(content);
562 3 : }
563 :
564 8 : void DumpCore::DumpGlobalAuxInfo(const std::vector<GlobalMemInfo> &memInfoList)
565 : {
566 8 : ELF::SectionPtr infoSection = coreFile_.AddSection(ASCEND_SHTYPE_AUXINFO_GLOABL, ASCEND_SHNAME_AUXINFO_GLOABL);
567 8 : IDE_CTRL_VALUE_FAILED(infoSection != nullptr, return,
568 : "Create %s section failed.", ASCEND_SHNAME_AUXINFO_GLOABL.c_str());
569 :
570 8 : uint32_t index = infoSection->GetIndex();
571 92 : for (const GlobalMemInfo &memInfo : memInfoList) {
572 84 : ELF::SectionPtr section = coreFile_.GetSectionByIndex(memInfo.sectionIndex);
573 84 : IDE_CTRL_VALUE_FAILED_NODO(section != nullptr, continue, "Get section by index %u failed",
574 : memInfo.sectionIndex);
575 84 : section->SetLink(index);
576 84 : }
577 :
578 8 : size_t perLen = sizeof(GlobalMemInfo);
579 8 : size_t totalSize = perLen * memInfoList.size();
580 8 : infoSection->SetEntSize(perLen);
581 8 : std::string memInfoData(reinterpret_cast<const char *>(memInfoList.data()), totalSize);
582 8 : infoSection->SetData(memInfoData);
583 8 : }
584 : } // namespace Adx
|
