Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 : #include <map>
11 : #include "adump_pub.h"
12 : #include "log/adx_log.h"
13 : #include "runtime/mem.h"
14 : #include "adump_dsmi.h"
15 : #include "sys_utils.h"
16 : #include "path.h"
17 : #include "exception_info_common.h"
18 : #include "kernel_symbol_locator.h"
19 : #include "dump_core.h"
20 :
21 : namespace Adx {
22 8 : int32_t DumpCore::DumpCoreFile(const rtExceptionInfo& exception)
23 : {
24 8 : if (ExceptionInfoCommon::GetExceptionRegInfo(exception, exceptionRegInfo_) == ADUMP_SUCCESS) {
25 8 : KernelSymbolLocator::DumpErrorSymbols(exception, exceptionRegInfo_, path_);
26 : }
27 8 : DumpCoreInfo(exception.deviceid);
28 8 : DumpGlobalMemory(exception);
29 8 : DumpResourceInit();
30 :
31 52 : for (uint16_t aicId : aiCoreIds_) {
32 36 : DumpLocalMemory(CORE_TYPE_AIC, aicId);
33 36 : DumpRegister(CORE_TYPE_AIC, aicId);
34 : }
35 70 : for (uint16_t aivId : aiVectorCoreIds_) {
36 54 : DumpLocalMemory(CORE_TYPE_AIV, aivId);
37 54 : DumpRegister(CORE_TYPE_AIV, aivId);
38 : }
39 :
40 8 : SaveCoreFile(exception);
41 :
42 8 : return ADUMP_SUCCESS;
43 : }
44 :
45 8 : void DumpCore::DumpGlobalMemory(const rtExceptionInfo& exception)
46 : {
47 8 : std::vector<GlobalMemInfo> memInfoList;
48 :
49 8 : rtExceptionArgsInfo_t exceptionArgsInfo{};
50 8 : if (ExceptionInfoCommon::GetExceptionInfo(exception, exceptionArgsInfo) != ADUMP_SUCCESS) {
51 0 : IDE_LOGE("Get exception args info failed.");
52 0 : return;
53 : }
54 8 : DumpArgsInfo(exceptionArgsInfo, memInfoList);
55 :
56 8 : DumpArgs args;
57 8 : if (args.LoadArgsExceptionInfo(exception) == ADUMP_SUCCESS) {
58 1 : if (args.DumpArgsDumpWithDfxFlag()) {
59 0 : DumpTensorBuffer(args, memInfoList);
60 0 : DumpWorkSpace(args, memInfoList);
61 : } else {
62 1 : DumpInput(args, memInfoList);
63 : }
64 : }
65 :
66 8 : DumpStack(exceptionArgsInfo.exceptionKernelInfo.bin, memInfoList);
67 8 : DumpDeviceKernelBin(exceptionArgsInfo.exceptionKernelInfo, memInfoList);
68 8 : DumpGlobalAuxInfo(memInfoList);
69 :
70 8 : DumpHostKernelBin(exceptionArgsInfo.exceptionKernelInfo);
71 8 : DumpHostFile(exceptionArgsInfo);
72 8 : }
73 :
74 8 : void DumpCore::DumpResourceInit()
75 : {
76 8 : char version[SOC_VERSION_SIZE] = {0};
77 8 : rtError_t ret = rtGetSocVersion(version, SOC_VERSION_SIZE);
78 8 : IDE_CTRL_VALUE_FAILED(ret == RT_ERROR_NONE, return, "Failed to get soc version");
79 8 : const std::string socVersion(version);
80 :
81 8 : IDE_CTRL_VALUE_FAILED(
82 : AdumpPlatformApi::GetAicoreSizeInfo(socVersion, bufferSize_), return,
83 : "Failed to read platform info from fe api.");
84 8 : IDE_LOGD(
85 : "Get local buffer size: L0A: %llu, L0B: %llu, L0C: %llu, L1: %llu, UB: %llu", bufferSize_.l0aSize,
86 : bufferSize_.l0bSize, bufferSize_.l0cSize, bufferSize_.l1Size, bufferSize_.ubSize);
87 8 : }
88 :
89 8 : void DumpCore::DumpCoreInfo(uint32_t devId)
90 : {
91 8 : std::string data(sizeof(DevInfo), 0);
92 8 : DevInfo* devInfo = reinterpret_cast<DevInfo*>(const_cast<char*>(data.data()));
93 8 : devInfo->devId = devId;
94 8 : IDE_CTRL_VALUE_FAILED(AdumpDsmi::DrvGetPlatformType(devInfo->devType), return, "Get platform type failed.");
95 :
96 8 : rtError_t ret = rtDebugGetStalledCore(&devInfo->coreInfo);
97 8 : IDE_CTRL_VALUE_FAILED(ret == RT_ERROR_NONE, return, "Get core id failed, ret: %d", ret);
98 8 : IDE_LOGD(
99 : "get core info: aicBitmap: 0x%llx 0x%llx, aivBitmap: 0x%llx 0x%llx", devInfo->coreInfo.aicBitmap0,
100 : devInfo->coreInfo.aicBitmap1, devInfo->coreInfo.aivBitmap0, devInfo->coreInfo.aivBitmap1);
101 :
102 520 : for (uint16_t i = 0; i < CORE_ID_BIT_MAP_SIZE; ++i) {
103 512 : if ((devInfo->coreInfo.aicBitmap0 & (1ULL << i)) != 0) {
104 8 : aiCoreIds_.emplace_back(i);
105 : }
106 512 : if ((devInfo->coreInfo.aicBitmap1 & (1ULL << i)) != 0) {
107 8 : aiCoreIds_.emplace_back(i + CORE_ID_BIT_MAP_SIZE);
108 : }
109 512 : if ((devInfo->coreInfo.aivBitmap0 & (1ULL << i)) != 0) {
110 8 : aiVectorCoreIds_.emplace_back(i);
111 : }
112 512 : if ((devInfo->coreInfo.aivBitmap1 & (1ULL << i)) != 0) {
113 16 : aiVectorCoreIds_.emplace_back(i + CORE_ID_BIT_MAP_SIZE);
114 : }
115 : }
116 :
117 8 : ELF::SectionPtr devtbl = coreFile_.AddSection(ASCEND_SHTYPE_DEVTBL, ASCEND_SHNAME_DEVTBL);
118 8 : IDE_CTRL_VALUE_FAILED(devtbl != nullptr, return, "Create %s section failed.", ASCEND_SHNAME_DEVTBL.c_str());
119 8 : devtbl->SetData(data);
120 8 : }
121 :
122 90 : void DumpCore::DumpLocalMemory(uint8_t coreType, uint16_t coreId)
123 : {
124 90 : std::string coreIdStr = std::to_string(ConvertCoreId(coreType, coreId));
125 90 : std::string sectionName = ASCEND_SHNAME_LOCAL + "." + coreIdStr;
126 90 : std::vector<LocalMemInfo> localMemInfoList;
127 :
128 90 : DumpBuffer(coreType, coreId, sectionName, localMemInfoList);
129 90 : DumpCache(coreType, coreId, sectionName, localMemInfoList);
130 90 : DumpLocalAuxInfo(coreIdStr, localMemInfoList);
131 90 : }
132 :
133 90 : void DumpCore::DumpCache(
134 : uint8_t coreType, uint16_t coreId, const std::string& sectionName, std::vector<LocalMemInfo>& localMemInfoList)
135 : {
136 : // icache
137 90 : DumpCache(coreType, coreId, binParam_, sectionName, localMemInfoList);
138 :
139 : // dcache
140 90 : DumpCache(coreType, coreId, argsParam_, sectionName, localMemInfoList);
141 90 : DumpCache(coreType, coreId, tilingDataParam_, sectionName, localMemInfoList);
142 :
143 2480 : for (const auto& stackParam : stackParamList_) {
144 2300 : if (stackParam.coreType == coreType && stackParam.coreId == coreId) {
145 460 : DumpCache(coreType, coreId, stackParam, sectionName, localMemInfoList);
146 : }
147 : }
148 90 : }
149 :
150 730 : void DumpCore::DumpCache(
151 : uint8_t coreType, uint16_t coreId, const CacheParam& cacheParam, const std::string& sectionName,
152 : std::vector<LocalMemInfo>& localMemInfoList)
153 : {
154 730 : if (cacheParam.memSize == 0) {
155 650 : return;
156 : }
157 640 : std::string cacheData(cacheParam.memSize, 0);
158 640 : rtDebugMemoryParam_t param = {coreType, 0, coreId, cacheParam.cacheType, 0, 0, cacheParam.memAddr, 0,
159 640 : cacheParam.memSize};
160 640 : param.dstAddr = reinterpret_cast<uint64_t>(cacheData.data());
161 640 : rtError_t ret = rtDebugReadAICore(¶m);
162 640 : IDE_CTRL_VALUE_FAILED(ret == RT_ERROR_NONE, return, "Failed to copy icache data from device, ret: %d", ret);
163 :
164 80 : ELF::SectionPtr localSec = coreFile_.AddSection(ASCEND_SHTYPE_LOCAL, sectionName);
165 80 : IDE_CTRL_VALUE_FAILED(localSec != nullptr, return, "Create %s section failed.", sectionName.c_str());
166 :
167 80 : LocalMemInfo localMemInfo = {param.memLen, localSec->GetIndex(), cacheParam.sectionIndex, param.debugMemType, 0};
168 80 : localMemInfoList.emplace_back(localMemInfo);
169 80 : localSec->SetData(cacheData);
170 80 : localSec->SetInfo(localMemInfoList.size() - 1);
171 :
172 80 : IDE_LOGD(
173 : "Dump cache data success, core type: %hhu, core id: %hu, cache type: %d, addr: %llx, size: %llu", coreType,
174 : coreId, cacheParam.cacheType, cacheParam.memAddr, cacheParam.memSize);
175 640 : }
176 :
177 90 : void DumpCore::DumpBuffer(
178 : uint8_t coreType, uint16_t coreId, const std::string& sectionName, std::vector<LocalMemInfo>& localMemInfoList)
179 : {
180 : std::vector<rtDebugMemoryParam_t> memParamList = {
181 90 : {CORE_TYPE_AIC, 0, coreId, RT_MEM_TYPE_L0A, 0, 0, 0, 0, bufferSize_.l0aSize},
182 90 : {CORE_TYPE_AIC, 0, coreId, RT_MEM_TYPE_L0B, 0, 0, 0, 0, bufferSize_.l0bSize},
183 90 : {CORE_TYPE_AIC, 0, coreId, RT_MEM_TYPE_L0C, 0, 0, 0, 0, bufferSize_.l0cSize},
184 90 : {CORE_TYPE_AIC, 0, coreId, RT_MEM_TYPE_L1, 0, 0, 0, 0, bufferSize_.l1Size},
185 90 : {CORE_TYPE_AIV, 0, coreId, RT_MEM_TYPE_UB, 0, 0, 0, 0, bufferSize_.ubSize},
186 180 : };
187 630 : for (auto& memParam : memParamList) {
188 450 : if (memParam.coreType != coreType) {
189 406 : continue;
190 : }
191 : // memLen zero check
192 198 : std::string localData(memParam.memLen, 0);
193 198 : memParam.dstAddr = reinterpret_cast<uint64_t>(localData.data());
194 :
195 198 : rtError_t ret = rtDebugReadAICore(&memParam);
196 198 : IDE_CTRL_VALUE_FAILED_NODO(ret == RT_ERROR_NONE, continue, "Failed to copy data from device, ret: %d", ret);
197 :
198 44 : ELF::SectionPtr localSec = coreFile_.AddSection(ASCEND_SHTYPE_LOCAL, sectionName);
199 44 : IDE_CTRL_VALUE_FAILED_NODO(localSec != nullptr, continue, "Create %s section failed.", sectionName.c_str());
200 :
201 44 : LocalMemInfo localMemInfo = {memParam.memLen, localSec->GetIndex(), 0, memParam.debugMemType, 0};
202 44 : localMemInfoList.emplace_back(localMemInfo);
203 44 : localSec->SetData(localData);
204 44 : localSec->SetInfo(localMemInfoList.size() - 1);
205 :
206 44 : IDE_LOGD(
207 : "Dump local memory success, core type: %hhu, core id: %hu, type: %d, size: %llu", coreType, coreId,
208 : memParam.debugMemType, memParam.memLen);
209 198 : }
210 90 : }
211 :
212 90 : void DumpCore::DumpLocalAuxInfo(const std::string& coreIdStr, std::vector<LocalMemInfo>& localMemInfoList)
213 : {
214 90 : if (localMemInfoList.empty()) {
215 70 : return;
216 : }
217 :
218 20 : size_t totalSize = localMemInfoList.size() * sizeof(LocalMemInfo);
219 20 : std::string localData(reinterpret_cast<const char*>(localMemInfoList.data()), totalSize);
220 :
221 20 : std::string sectionName = ASCEND_SHNAME_AUXINFO_LOCAL + "." + coreIdStr;
222 20 : ELF::SectionPtr localSec = coreFile_.AddSection(ASCEND_SHTYPE_AUXINFO_LOCAL, sectionName);
223 20 : IDE_CTRL_VALUE_FAILED(localSec != nullptr, return, "Create %s section failed.", sectionName.c_str());
224 :
225 20 : localSec->SetData(localData);
226 20 : localSec->SetEntSize(sizeof(LocalMemInfo));
227 164 : for (const LocalMemInfo& localMemInfo : localMemInfoList) {
228 124 : ELF::SectionPtr sec = coreFile_.GetSectionByIndex(localMemInfo.sectionIndex);
229 124 : IDE_CTRL_VALUE_FAILED_NODO(
230 : sec != nullptr, continue, "Get section by index failed, index: %u", localMemInfo.sectionIndex);
231 124 : sec->SetLink(localSec->GetIndex());
232 124 : }
233 20 : }
234 :
235 8 : void DumpCore::SaveCoreFile(const rtExceptionInfo& exception)
236 : {
237 8 : std::string kernelName;
238 8 : rtExceptionArgsInfo_t exceptionArgsInfo{};
239 8 : if (ExceptionInfoCommon::GetExceptionInfo(exception, exceptionArgsInfo) != ADUMP_SUCCESS) {
240 0 : IDE_LOGE("Get exception args info failed.");
241 0 : kernelName = DEFAULT_KERNEL_NAME;
242 : } else {
243 : std::string rtKernelName(
244 8 : exceptionArgsInfo.exceptionKernelInfo.kernelName, exceptionArgsInfo.exceptionKernelInfo.kernelNameSize);
245 8 : kernelName = rtKernelName.empty() ? DEFAULT_KERNEL_NAME : rtKernelName;
246 8 : }
247 16 : std::string dumpFileName = kernelName + "." + std::to_string(exception.streamid) + "." +
248 32 : std::to_string(exception.taskid) + "." + SysUtils::GetCurrentTimeWithMillisecond() +
249 8 : ".core";
250 : // File names should not exceed the filesystem limits.
251 8 : dumpFileName = dumpFileName.length() > 255U ? dumpFileName.substr(dumpFileName.length() - 255U) : dumpFileName;
252 8 : Path dumpFilePath(path_);
253 8 : IDE_CTRL_VALUE_FAILED(
254 : dumpFilePath.RealPath(), return, "Get path %s real path failed, strerror=%s.", dumpFilePath.GetCString(),
255 : strerror(errno));
256 8 : std::string dir = dumpFilePath.GetString();
257 8 : dumpFilePath.Concat(dumpFileName);
258 8 : IDE_CTRL_VALUE_FAILED(
259 : dumpFilePath.ParentPath().GetString() == dir, return, "Check dump file path %s failed.",
260 : dumpFilePath.GetCString());
261 8 : coreFile_.Save(dumpFilePath.GetString());
262 8 : }
263 :
264 30 : int32_t DumpCore::D2HMemcpyWithNoCheck(const GlobalMemInfo& memInfo, std::string& data) const
265 : {
266 30 : std::vector<char> buffer(memInfo.size);
267 30 : rtError_t rtRet = rtMemcpyEx(
268 30 : static_cast<void*>(buffer.data()), memInfo.size, reinterpret_cast<void*>(memInfo.devAddr), memInfo.size,
269 : RT_MEMCPY_DEVICE_TO_HOST);
270 30 : if (rtRet != RT_ERROR_NONE) {
271 15 : IDE_LOGE(
272 : "Call rtMemcpyEx failed, data type: %d, addr: 0x%llx, size: %llu, ret: %d",
273 : static_cast<int32_t>(memInfo.type), memInfo.devAddr, memInfo.size, rtRet);
274 15 : return ADUMP_FAILED;
275 : } else {
276 15 : data.assign(buffer.data(), memInfo.size);
277 : }
278 15 : return ADUMP_SUCCESS;
279 30 : }
280 :
281 53 : int32_t DumpCore::D2HMemcpyWithCheck(const GlobalMemInfo& memInfo, std::string& data) const
282 : {
283 53 : rtMemInfo_t info{};
284 53 : uint64_t* deviceAddr[1] = {reinterpret_cast<uint64_t*>(memInfo.devAddr)};
285 53 : info.addrInfo.addr = static_cast<uint64_t**>(deviceAddr);
286 53 : info.addrInfo.cnt = 1;
287 53 : info.addrInfo.memType = RT_MEM_MASK_DEV_TYPE | RT_MEM_MASK_RSVD_TYPE;
288 53 : info.addrInfo.flag = true;
289 53 : rtError_t rtRet = rtMemGetInfoByType(static_cast<int32_t>(devId_), RT_MEM_INFO_TYPE_ADDR_CHECK, &info);
290 53 : if ((rtRet != RT_ERROR_NONE) || (info.addrInfo.flag == false)) {
291 2 : IDE_LOGE("Check global memory addr 0x%llx invalid", memInfo.devAddr);
292 2 : return ADUMP_FAILED;
293 : }
294 :
295 51 : std::vector<char> buffer(memInfo.size);
296 51 : rtRet = rtMemcpy(
297 51 : static_cast<void*>(buffer.data()), memInfo.size, reinterpret_cast<void*>(memInfo.devAddr), memInfo.size,
298 : RT_MEMCPY_DEVICE_TO_HOST);
299 51 : if (rtRet != RT_ERROR_NONE) {
300 2 : IDE_LOGE(
301 : "Call rtMemcpy failed, data type: %d, addr: 0x%llx, size: %llu, ret: %d",
302 : static_cast<int32_t>(memInfo.type), memInfo.devAddr, memInfo.size, rtRet);
303 2 : return ADUMP_FAILED;
304 : } else {
305 49 : data.assign(buffer.data(), memInfo.size);
306 : }
307 49 : return ADUMP_SUCCESS;
308 51 : }
309 :
310 84 : int32_t DumpCore::ProcessGlobalMemory(GlobalMemInfo& memInfo, std::vector<GlobalMemInfo>& memInfoList, bool checkAddr)
311 : {
312 84 : IDE_LOGI("Dump device data. data type: %hu, addr: 0x%llx, size: %llu", memInfo.type, memInfo.devAddr, memInfo.size);
313 84 : std::string curData(memInfo.size, 0);
314 84 : if (memInfo.size != 0) {
315 83 : if (checkAddr) {
316 53 : if (D2HMemcpyWithCheck(memInfo, curData) != ADUMP_SUCCESS) {
317 4 : memInfo.size |= INVALID_DATA_FLAG;
318 : }
319 : } else {
320 30 : if (D2HMemcpyWithNoCheck(memInfo, curData) != ADUMP_SUCCESS) {
321 15 : memInfo.size |= INVALID_DATA_FLAG;
322 : }
323 : }
324 : }
325 :
326 84 : ELF::SectionPtr curSection = coreFile_.AddSection(ASCEND_SHTYPE_GLOBAL, ASCEND_SHNAME_GLOBAL);
327 84 : IDE_CTRL_VALUE_FAILED(
328 : curSection != nullptr, return ADUMP_FAILED, "Create %s section failed.", ASCEND_SHNAME_GLOBAL.c_str());
329 :
330 84 : memInfo.sectionIndex = curSection->GetIndex();
331 84 : memInfoList.emplace_back(memInfo);
332 84 : curSection->SetInfo(memInfoList.size() - 1);
333 84 : curSection->SetData(curData);
334 84 : curSection->SetAddr(memInfo.devAddr);
335 84 : return ADUMP_SUCCESS;
336 84 : }
337 :
338 8 : void DumpCore::DumpArgsInfo(const rtExceptionArgsInfo_t& exceptionArgsInfo, std::vector<GlobalMemInfo>& memInfoList)
339 : {
340 8 : GlobalMemInfo memInfo = {
341 8 : .devAddr = reinterpret_cast<uint64_t>(exceptionArgsInfo.argAddr),
342 8 : .size = exceptionArgsInfo.argsize,
343 : .sectionIndex = 0,
344 : .type = DfxTensorType::ARGS,
345 : .reserve = 0,
346 8 : .extraInfo = {.coreInfo = {.coreId = 0}}};
347 :
348 8 : int32_t ret = ProcessGlobalMemory(memInfo, memInfoList);
349 8 : IDE_CTRL_VALUE_FAILED(ret == ADUMP_SUCCESS, return, "Save args to section failed.");
350 8 : argsParam_ = {0, 0, exceptionArgsInfo.argsize, memInfo.devAddr, memInfo.sectionIndex, RT_MEM_TYPE_DCACHE};
351 : }
352 :
353 1 : void DumpCore::DumpInput(const DumpArgs& args, std::vector<GlobalMemInfo>& memInfoList)
354 : {
355 1 : const std::vector<InputBuffer> inputBuffer = args.DumpArgsGetInputBuffer();
356 1 : int32_t ret = 0;
357 10 : for (const auto& input : inputBuffer) {
358 8 : if (input.addr == nullptr) { // skip placeholder tensor
359 0 : continue;
360 : }
361 8 : GlobalMemInfo memInfo = {
362 8 : .devAddr = reinterpret_cast<uint64_t>(input.addr),
363 8 : .size = input.length,
364 : .sectionIndex = 0,
365 : .type = DfxTensorType::INPUT_TENSOR,
366 : .reserve = 0,
367 8 : .extraInfo = {.shape = {.dim = 0, .dimSize = {0}}}};
368 :
369 8 : ret = ProcessGlobalMemory(memInfo, memInfoList);
370 8 : IDE_CTRL_VALUE_FAILED_NODO(
371 : ret == ADUMP_SUCCESS, continue, "Save input to section failed, addr: 0x%llx, size: %llu", memInfo.devAddr,
372 : memInfo.size);
373 : }
374 1 : }
375 :
376 0 : void DumpCore::DumpTensorBuffer(const DumpArgs& args, std::vector<GlobalMemInfo>& memInfoList)
377 : {
378 0 : const std::vector<TensorBuffer> tensorBuffer = args.DumpArgsGetTensorBuffer();
379 0 : int32_t ret = 0;
380 0 : for (const auto& tensor : tensorBuffer) {
381 0 : if (tensor.addr == nullptr) { // skip placeholder tensor
382 0 : continue;
383 : }
384 0 : IDE_CTRL_VALUE_FAILED_NODO(
385 : tensor.dimension < MAX_DIM_SIZE, continue,
386 : "Invalid dimension %llu, addr: %p, arg index: %u, tensor type %hu, pointer type: %hu", tensor.dimension,
387 : tensor.addr, tensor.argIndex, tensor.tensorType, tensor.pointerType);
388 :
389 0 : GlobalMemInfo memInfo = {
390 0 : .devAddr = reinterpret_cast<uint64_t>(tensor.addr),
391 0 : .size = tensor.GetTotalByteSize(),
392 : .sectionIndex = 0,
393 0 : .type = static_cast<DfxTensorType>(tensor.tensorType),
394 : .reserve = 0,
395 0 : .extraInfo = {.shape = {.dim = static_cast<uint32_t>(tensor.dimension), .dimSize = {0}}}};
396 0 : for (uint32_t i = 0; i < memInfo.extraInfo.shape.dim; ++i) {
397 0 : memInfo.extraInfo.shape.dimSize[i] = tensor.shape[i];
398 : }
399 0 : ret = ProcessGlobalMemory(memInfo, memInfoList);
400 0 : IDE_CTRL_VALUE_FAILED_NODO(
401 : ret == ADUMP_SUCCESS, continue, "Save tensor to section failed, addr: 0x%llx, size: %llu, type: %hu",
402 : memInfo.devAddr, memInfo.size, memInfo.type);
403 :
404 0 : if (tensor.tensorType == DfxTensorType::TILING_DATA) {
405 0 : tilingDataParam_ = {0, 0, memInfo.size, memInfo.devAddr, memInfo.sectionIndex, RT_MEM_TYPE_DCACHE};
406 : }
407 : }
408 0 : }
409 :
410 0 : void DumpCore::DumpWorkSpace(const DumpArgs& args, std::vector<GlobalMemInfo>& memInfoList)
411 : {
412 0 : const std::vector<DumpWorkspace> workSpace = args.DumpArgsGetWorkSpace();
413 0 : int32_t ret = 0;
414 0 : for (const auto& ws : workSpace) {
415 0 : GlobalMemInfo memInfo = {
416 0 : .devAddr = reinterpret_cast<uint64_t>(ws.addr),
417 0 : .size = ws.bytes,
418 : .sectionIndex = 0,
419 : .type = DfxTensorType::WORKSPACE_TENSOR,
420 : .reserve = 0,
421 0 : .extraInfo = {.shape = {.dim = 0, .dimSize = {0}}}};
422 :
423 0 : ret = ProcessGlobalMemory(memInfo, memInfoList);
424 0 : IDE_CTRL_VALUE_FAILED_NODO(
425 : ret == ADUMP_SUCCESS, continue, "Save input to section failed, addr: 0x%llx, size: %llu", memInfo.devAddr,
426 : memInfo.size);
427 : }
428 0 : }
429 :
430 8 : void DumpCore::DumpStack(const rtBinHandle& binHandle, std::vector<GlobalMemInfo>& memInfoList)
431 : {
432 52 : for (uint16_t aicId : aiCoreIds_) {
433 36 : IDE_LOGI("Dump core stack data. coreType: aic, coreId: %hu", aicId);
434 36 : DumpCoreStack(binHandle, CORE_TYPE_AIC, aicId, false, RT_STACK_TYPE_SCALAR, DfxTensorType::STACK, memInfoList);
435 36 : DumpCoreStack(
436 : binHandle, CORE_TYPE_AIC, aicId, true, RT_STACK_TYPE_SIMT, DfxTensorType::SIMT_STACK, memInfoList);
437 : }
438 70 : for (uint16_t aivId : aiVectorCoreIds_) {
439 54 : IDE_LOGI("Dump core stack data. coreType: aiv, coreId: %hu", aivId);
440 54 : DumpCoreStack(binHandle, CORE_TYPE_AIV, aivId, false, RT_STACK_TYPE_SCALAR, DfxTensorType::STACK, memInfoList);
441 54 : DumpCoreStack(
442 : binHandle, CORE_TYPE_AIV, aivId, true, RT_STACK_TYPE_SIMT, DfxTensorType::SIMT_STACK, memInfoList);
443 : }
444 8 : }
445 :
446 180 : void DumpCore::DumpCoreStack(
447 : const rtBinHandle& binHandle, const uint8_t coreType, const uint16_t coreId, bool checkAddr,
448 : const rtStackType_t rtStackType, DfxTensorType dumpStackType, std::vector<GlobalMemInfo>& memInfoList)
449 : {
450 180 : const void* stackAddr = nullptr;
451 180 : uint32_t stackSize = 0;
452 180 : rtError_t rtRet = rtGetStackBuffer(binHandle, 0U, rtStackType, coreType, coreId, &stackAddr, &stackSize);
453 180 : if (rtRet == ACL_ERROR_RT_FEATURE_NOT_SUPPORT) {
454 0 : IDE_LOGW("RTS does not support rtGetStackBuffer feature. stackType: %d, ret: %d", rtStackType, rtRet);
455 120 : return;
456 : }
457 180 : if ((rtRet != RT_ERROR_NONE) || (stackAddr == nullptr)) {
458 120 : IDE_LOGE(
459 : "Call rtGetStackBuffer to get stack data failed, coreType: %hhu, coreId: %hu, ret: %d", coreType, coreId,
460 : static_cast<int32_t>(rtRet));
461 120 : return;
462 : }
463 60 : GlobalMemInfo memInfo = {
464 : .devAddr = reinterpret_cast<uint64_t>(stackAddr),
465 60 : .size = stackSize,
466 : .sectionIndex = 0,
467 : .type = dumpStackType,
468 : .reserve = 0,
469 60 : .extraInfo = {.coreInfo = {ConvertCoreId(coreType, coreId)}}};
470 :
471 60 : int32_t ret = ProcessGlobalMemory(memInfo, memInfoList, checkAddr);
472 60 : IDE_CTRL_VALUE_FAILED(
473 : ret == ADUMP_SUCCESS, return, "Save stack section failed, core type: %hhu, core id: %hu", coreType, coreId);
474 :
475 60 : stackParamList_.emplace_back(
476 60 : coreType, coreId, stackSize, memInfo.devAddr, memInfo.sectionIndex, RT_MEM_TYPE_DCACHE);
477 : }
478 :
479 8 : void DumpCore::DumpHostKernelBin(const rtExceptionKernelInfo_t& kernelInfo)
480 : {
481 8 : void* binAddr = nullptr;
482 8 : uint32_t binSize = 0;
483 8 : rtError_t rtRet = rtGetBinBuffer(kernelInfo.bin, RT_BIN_HOST_ADDR, &binAddr, &binSize);
484 8 : IDE_CTRL_VALUE_FAILED(
485 : (rtRet == RT_ERROR_NONE) && (binAddr != nullptr), return,
486 : "Call rtGetBinBuffer get host kernel object failed, ret: %d", static_cast<int32_t>(rtRet));
487 :
488 8 : std::string hostBinData(static_cast<char*>(binAddr), binSize);
489 : ELF::SectionPtr curSection =
490 8 : coreFile_.AddSection(ASCEND_SHTYPE_HOST_KERNEL_OBJECT, ASCEND_SHNAME_HOST_KERNEL_OBJECT);
491 8 : IDE_CTRL_VALUE_FAILED(
492 : curSection != nullptr, return, "Create %s section failed.", ASCEND_SHNAME_HOST_KERNEL_OBJECT.c_str());
493 8 : curSection->SetData(hostBinData);
494 8 : }
495 :
496 8 : void DumpCore::DumpDeviceKernelBin(const rtExceptionKernelInfo_t& kernelInfo, std::vector<GlobalMemInfo>& memInfoList)
497 : {
498 8 : void* deviceBinAddr = nullptr;
499 8 : uint32_t binSize = 0;
500 8 : rtError_t rtRet = rtGetBinBuffer(kernelInfo.bin, RT_BIN_DEVICE_ADDR, &deviceBinAddr, &binSize);
501 8 : IDE_CTRL_VALUE_FAILED(
502 : rtRet == RT_ERROR_NONE, return, "Call rtGetBinBuffer get device kernel object failed, ret: %d",
503 : static_cast<int32_t>(rtRet));
504 :
505 8 : GlobalMemInfo memInfo = {
506 : .devAddr = reinterpret_cast<uint64_t>(deviceBinAddr),
507 8 : .size = binSize,
508 : .sectionIndex = 0,
509 : .type = DfxTensorType::DEVICE_KERNEL_OBJECT,
510 : .reserve = 0,
511 8 : .extraInfo = {.shape = {.dim = 0, .dimSize = {0}}}};
512 :
513 8 : int32_t ret = ProcessGlobalMemory(memInfo, memInfoList);
514 8 : IDE_CTRL_VALUE_FAILED(
515 : ret == ADUMP_SUCCESS, return, "Save device kernel object section failed, addr: %p, size: %u", deviceBinAddr,
516 : binSize);
517 :
518 8 : binParam_ = {0, 0, binSize, memInfo.devAddr, memInfo.sectionIndex, RT_MEM_TYPE_ICACHE};
519 : }
520 :
521 8 : void DumpCore::DumpHostFile(const rtExceptionArgsInfo_t& argsInfo)
522 : {
523 8 : KernelInfoCollector collector;
524 8 : collector.LoadKernelInfo(argsInfo);
525 8 : std::vector<std::string> searchPath = collector.GetSearchPath();
526 8 : std::string kernelName = collector.GetProcessedKernelName();
527 8 : IDE_CTRL_VALUE_FAILED(!kernelName.empty(), return, "Get kernel name failed.");
528 3 : IDE_LOGI("kernel name: %s", kernelName.c_str());
529 3 : DumpKernelInfo(kernelName);
530 :
531 6 : for (const auto& path : searchPath) {
532 3 : std::string jsonFilePath = collector.SearchJsonFiles(path, kernelName);
533 3 : if (jsonFilePath.empty()) {
534 0 : continue;
535 : }
536 3 : DumpHostFile(jsonFilePath, ASCEND_SHTYPE_FILE_KERNEL_JSON, ASCEND_SHNAME_FILE_KERNEL_JSON);
537 3 : std::string kernelFilePath = jsonFilePath.substr(0, jsonFilePath.size() - JSON_SUFFIX_LEN) + ".o";
538 3 : DumpHostFile(kernelFilePath, ASCEND_SHTYPE_FILE_KERNEL_OBJECT, ASCEND_SHNAME_FILE_KERNEL_OBJECT);
539 3 : break;
540 6 : }
541 18 : }
542 :
543 6 : void DumpCore::DumpHostFile(const std::string& filePath, uint32_t sectionType, const std::string& sectionName)
544 : {
545 6 : char canonicalPath[PATH_MAX] = {0};
546 6 : IDE_CTRL_VALUE_FAILED(
547 : realpath(filePath.c_str(), canonicalPath) != nullptr, return, "Get file path %s realpath failed, strerror=%s",
548 : filePath.c_str(), strerror(errno));
549 6 : std::ifstream file(canonicalPath);
550 6 : IDE_CTRL_VALUE_FAILED(file.is_open(), return, "Open file failed, path: %s", canonicalPath);
551 :
552 : do {
553 6 : std::string content((std::istreambuf_iterator<char>(file)), std::istreambuf_iterator<char>());
554 6 : IDE_CTRL_VALUE_FAILED_NODO(!content.empty(), break, "Read file content failed, path: %s", canonicalPath);
555 :
556 6 : ELF::SectionPtr curSection = coreFile_.AddSection(sectionType, sectionName);
557 6 : IDE_CTRL_VALUE_FAILED_NODO(curSection != nullptr, break, "Create %s section failed.", sectionName.c_str());
558 :
559 6 : curSection->SetData(content);
560 6 : } while (0);
561 :
562 6 : file.close();
563 6 : }
564 :
565 3 : void DumpCore::DumpKernelInfo(const std::string& kernelName)
566 : {
567 3 : std::string content;
568 3 : uint32_t nameSize = static_cast<uint32_t>(kernelName.length());
569 3 : content.append(reinterpret_cast<const char*>(&nameSize), sizeof(nameSize));
570 3 : content.append(kernelName);
571 3 : ELF::SectionPtr curSection = coreFile_.AddSection(ASCEND_SHTYPE_KERNEL_INFO, ASCEND_SHNAME_KERNEL_INFO);
572 3 : IDE_CTRL_VALUE_FAILED_NODO(
573 : curSection != nullptr, return, "Create %s section failed.", ASCEND_SHNAME_KERNEL_INFO.c_str());
574 3 : curSection->SetData(content);
575 3 : }
576 :
577 8 : void DumpCore::DumpGlobalAuxInfo(const std::vector<GlobalMemInfo>& memInfoList)
578 : {
579 8 : ELF::SectionPtr infoSection = coreFile_.AddSection(ASCEND_SHTYPE_AUXINFO_GLOABL, ASCEND_SHNAME_AUXINFO_GLOABL);
580 8 : IDE_CTRL_VALUE_FAILED(
581 : infoSection != nullptr, return, "Create %s section failed.", ASCEND_SHNAME_AUXINFO_GLOABL.c_str());
582 :
583 8 : uint32_t index = infoSection->GetIndex();
584 100 : for (const GlobalMemInfo& memInfo : memInfoList) {
585 84 : ELF::SectionPtr section = coreFile_.GetSectionByIndex(memInfo.sectionIndex);
586 84 : IDE_CTRL_VALUE_FAILED_NODO(
587 : section != nullptr, continue, "Get section by index %u failed", memInfo.sectionIndex);
588 84 : section->SetLink(index);
589 84 : }
590 :
591 8 : size_t perLen = sizeof(GlobalMemInfo);
592 8 : size_t totalSize = perLen * memInfoList.size();
593 8 : infoSection->SetEntSize(perLen);
594 8 : std::string memInfoData(reinterpret_cast<const char*>(memInfoList.data()), totalSize);
595 8 : infoSection->SetData(memInfoData);
596 8 : }
597 : } // namespace Adx
|
