Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "dump_file.h"
12 : #include <sstream>
13 : #include "runtime/mem.h"
14 : #include "sys_utils.h"
15 : #include "dump_datatype.h"
16 : #include "dump_memory.h"
17 : #include "log/hdc_log.h"
18 : #include "log/adx_log.h"
19 : #include "adump_dsmi.h"
20 : #include "adump_platform_manager.h"
21 : #include "dump_tensor_plugin.h"
22 : #include "hccl_mc2_define.h"
23 : #include "proto/dump_task.pb.h"
24 :
25 : namespace Adx {
26 : namespace {
27 : constexpr char DUMP_FILE_VERSION[] = "2.0";
28 : constexpr uint32_t MAX_DEV_NUM = 64;
29 : } // namespace
30 :
31 37 : void DumpFile::SetHeader(const std::string &opName)
32 : {
33 37 : header_.set_version(DUMP_FILE_VERSION);
34 37 : header_.set_dump_time(SysUtils::GetTimestamp());
35 37 : header_.set_op_name(opName);
36 37 : }
37 :
38 8 : void DumpFile::SetInputTensors(const std::vector<DumpTensor> &inputTensors)
39 : {
40 14 : for (size_t i = 0U; i < inputTensors.size(); ++i) {
41 6 : const DumpTensor &tensor = inputTensors[i];
42 6 : if (tensor.GetAddress() == nullptr) {
43 2 : continue;
44 : }
45 4 : toolkit::dump::OpInput input;
46 4 : auto ir_data_type = DumpDataType::GetIrDataType(static_cast<GeDataType>(tensor.GetDataType()));
47 4 : input.set_data_type(static_cast<toolkit::dump::OutputDataType>(ir_data_type));
48 4 : input.set_format(static_cast<toolkit::dump::OutputFormat>(tensor.GetFormat()));
49 4 : std::vector<int64_t> shape = tensor.GetShape();
50 13 : for (auto dim : shape)
51 : {
52 9 : input.mutable_shape()->add_dim(static_cast<uint64_t>(dim));
53 : }
54 4 : uint64_t size = static_cast<uint64_t>(tensor.GetSize());
55 4 : input.set_size(size);
56 4 : input.set_arg_index(tensor.GetArgsOffSet());
57 4 : input.set_input_type(static_cast<uint32_t>(DfxTensorType::INPUT_TENSOR));
58 4 : IDE_LOGI("[Set][DumpData] The input size exception is %llu bytes.", size);
59 4 : header_.mutable_input()->Add(std::move(input));
60 :
61 4 : inputs_.emplace_back(tensor.GetAddress(), size);
62 4 : }
63 8 : }
64 :
65 7 : void DumpFile::SetOutputTensors(const std::vector<DumpTensor> &outputTensors)
66 : {
67 10 : for (size_t i = 0U; i < outputTensors.size(); ++i) {
68 3 : const DumpTensor &tensor = outputTensors[i];
69 3 : if (tensor.GetAddress() == nullptr) {
70 1 : continue;
71 : }
72 2 : toolkit::dump::OpOutput output;
73 2 : auto ir_data_type = DumpDataType::GetIrDataType(static_cast<GeDataType>(tensor.GetDataType()));
74 2 : output.set_data_type(static_cast<toolkit::dump::OutputDataType>(ir_data_type));
75 2 : output.set_format(static_cast<toolkit::dump::OutputFormat>(tensor.GetFormat()));
76 2 : std::vector<int64_t> shape = tensor.GetShape();
77 6 : for (auto dim : shape) {
78 4 : output.mutable_shape()->add_dim(static_cast<uint64_t>(dim));
79 : }
80 2 : uint64_t size = static_cast<uint64_t>(tensor.GetSize());
81 2 : output.set_size(size);
82 2 : output.set_arg_index(tensor.GetArgsOffSet());
83 2 : IDE_LOGI("[Set][DumpData] The output size exception is %llu bytes.", size);
84 2 : header_.mutable_output()->Add(std::move(output));
85 :
86 2 : outputs_.emplace_back(tensor.GetAddress(), size);
87 2 : }
88 7 : }
89 :
90 14 : void DumpFile::SetWorkspaces(const std::vector<DumpWorkspace> &workspaces)
91 : {
92 14 : IDE_LOGI("[Set][DumpData] Workspace size is %zu bytes", workspaces.size());
93 22 : for (size_t i = 0U; i < workspaces.size(); ++i) {
94 8 : if (workspaces[i].addr == nullptr) {
95 1 : continue;
96 : }
97 7 : toolkit::dump::Workspace workspace;
98 7 : workspace.set_size(workspaces[i].bytes);
99 7 : workspace.set_arg_index(workspaces[i].argsOffset);
100 7 : IDE_LOGI("[Set][DumpData] The workspace size exception is %llu bytes.", workspaces[i].bytes);
101 7 : workspace.set_type(toolkit::dump::Workspace::LOG);
102 7 : header_.mutable_space()->Add(std::move(workspace));
103 :
104 7 : workspaces_.emplace_back(workspaces[i].addr, workspaces[i].bytes);
105 7 : }
106 14 : }
107 :
108 : #if !defined(ADUMP_SOC_HOST) || ADUMP_SOC_HOST == 1
109 6 : void DumpFile::SetMc2spaces(const std::vector<DumpWorkspace> &mc2Spaces)
110 : {
111 6 : IDE_CTRL_VALUE_WARN(SupportDumpMc2spaces(), return, "Not support to dump mc2 spaces data");
112 3 : IDE_LOGI("[Set][DumpData] mc2 space size is %zu", mc2Spaces.size());
113 5 : for (size_t i = 0U; i < mc2Spaces.size(); ++i) {
114 2 : if (mc2Spaces[i].addr == nullptr) {
115 0 : continue;
116 : }
117 2 : uint64_t opParamSize = 0;
118 2 : uint64_t newSize = GetMc2DataSize(mc2Spaces[i].addr, i, opParamSize);
119 2 : if (opParamSize == 0 || newSize == 0) {
120 0 : continue;
121 : }
122 2 : toolkit::dump::Workspace workspace;
123 2 : workspace.set_size(newSize);
124 2 : workspace.set_arg_index(mc2Spaces[i].argsOffset);
125 2 : workspace.set_type(toolkit::dump::Workspace::LOG);
126 2 : header_.mutable_space()->Add(std::move(workspace));
127 2 : IDE_LOGI("[Set][DumpData] The mc2 space size exception is %llu bytes.", newSize);
128 :
129 2 : mc2Spaces_.emplace_back(mc2Spaces[i].addr, opParamSize);
130 2 : }
131 : }
132 : #endif
133 :
134 28 : void DumpFile::SetInputBuffer(const std::vector<InputBuffer> input)
135 : {
136 28 : IDE_LOGI("[Set][DumpData] input buffer size is %zu", input.size());
137 126 : for (size_t i = 0U; i < input.size(); ++i) {
138 98 : if (input[i].addr == nullptr) {
139 2 : if (input[i].length != 0) {
140 2 : std::ostringstream oss;
141 2 : oss << "[Dump][Exception] the address maybe invalid, addr info: " << input[i].addr << ";"
142 2 : << input[i].length << ".";
143 2 : IDE_LOGE("%s", oss.str().c_str());
144 2 : logRecord_.emplace_back(oss.str() + "\n");
145 2 : }
146 2 : continue;
147 2 : }
148 96 : toolkit::dump::OpInput opInput;
149 96 : opInput.set_data_type(toolkit::dump::OutputDataType::DT_UNDEFINED);
150 96 : opInput.set_size(input[i].length);
151 96 : opInput.set_arg_index(input[i].argIndex);
152 96 : opInput.set_input_type(static_cast<uint32_t>(DfxTensorType::GENERAL_TENSOR));
153 96 : IDE_LOGI("[Set][DumpData] The input size exception is %llu", input[i].length);
154 96 : header_.mutable_input()->Add(std::move(opInput));
155 :
156 96 : inputBuffer_.emplace_back(input[i].addr, input[i].length);
157 96 : }
158 28 : }
159 :
160 10 : void DumpFile::SetTensorBuffer(const std::vector<TensorBuffer> &tensorBuffer)
161 : {
162 43 : for (size_t i = 0U; i < tensorBuffer.size(); ++i) {
163 33 : if (tensorBuffer[i].addr == nullptr) {
164 2 : if (tensorBuffer[i].size != 0) {
165 2 : std::ostringstream oss;
166 2 : oss << "[Dump][Exception] the address maybe invalid, addr info: " << tensorBuffer[i].addr << ";"
167 2 : << tensorBuffer[i].size << ".";
168 2 : IDE_LOGE("%s", oss.str().c_str());
169 2 : logRecord_.emplace_back(oss.str() + "\n");
170 2 : }
171 2 : continue;
172 2 : }
173 31 : if (tensorBuffer[i].tensorType <= DfxTensorType::INPUT_TENSOR) {
174 10 : toolkit::dump::OpInput input;
175 10 : input.set_data_type(toolkit::dump::OutputDataType::DT_UNDEFINED);
176 10 : uint64_t dimension = tensorBuffer[i].dimension;
177 10 : IDE_LOGI("[Set][DumpData] The input dim is %llu", dimension);
178 28 : for (uint64_t j = 0; j < dimension; ++j) {
179 18 : input.mutable_shape()->add_dim(tensorBuffer[i].shape[j]);
180 18 : IDE_LOGI("[Set][DumpData] The input shape is %llu", tensorBuffer[i].shape[j]);
181 : }
182 10 : uint64_t size = tensorBuffer[i].GetTotalByteSize();
183 10 : input.set_size(size);
184 10 : input.set_arg_index(tensorBuffer[i].argIndex);
185 10 : input.set_input_type(static_cast<uint32_t>(tensorBuffer[i].tensorType));
186 10 : IDE_LOGI("[Set][DumpData] The input size exception is %llu", size);
187 10 : header_.mutable_input()->Add(std::move(input));
188 :
189 10 : inputs_.emplace_back(tensorBuffer[i].addr, size);
190 36 : } else if (tensorBuffer[i].tensorType == DfxTensorType::OUTPUT_TENSOR ||
191 5 : tensorBuffer[i].tensorType == DfxTensorType::SHAPE_TENSOR) {
192 16 : toolkit::dump::OpOutput output;
193 16 : output.set_data_type(toolkit::dump::OutputDataType::DT_UNDEFINED);
194 16 : uint64_t dimension = tensorBuffer[i].dimension;
195 16 : IDE_LOGI("[Set][DumpData] The output dim is %llu", dimension);
196 40 : for (uint64_t j = 0; j < dimension; ++j) {
197 24 : output.mutable_shape()->add_dim(tensorBuffer[i].shape[j]);
198 24 : IDE_LOGI("[Set][DumpData] The output shape is %llu", tensorBuffer[i].shape[j]);
199 : }
200 16 : uint64_t size = tensorBuffer[i].GetTotalByteSize();
201 16 : output.set_size(size);
202 16 : output.set_arg_index(tensorBuffer[i].argIndex);
203 16 : IDE_LOGI("[Set][DumpData] The output size exception is %llu", size);
204 16 : header_.mutable_output()->Add(std::move(output));
205 :
206 16 : outputs_.emplace_back(tensorBuffer[i].addr, size);
207 21 : } else if (tensorBuffer[i].tensorType == DfxTensorType::TILING_DATA) {
208 5 : toolkit::dump::OpInput input;
209 5 : input.set_data_type(toolkit::dump::OutputDataType::DT_UNDEFINED);
210 5 : uint64_t size = tensorBuffer[i].GetTotalByteSize();
211 5 : input.set_size(size);
212 5 : input.set_arg_index(tensorBuffer[i].argIndex);
213 5 : input.set_input_type(static_cast<uint32_t>(tensorBuffer[i].tensorType));
214 5 : IDE_LOGI("[Set][DumpData] The tiling size exception is %llu", size);
215 5 : header_.mutable_input()->Add(std::move(input));
216 :
217 5 : inputs_.emplace_back(tensorBuffer[i].addr, size);
218 5 : }
219 : }
220 10 : }
221 :
222 27 : bool DumpFile::HasDumpData() const
223 : {
224 27 : if (!inputs_.empty() || !outputs_.empty() || !workspaces_.empty() || !inputBuffer_.empty()) {
225 21 : return true;
226 : }
227 : #if !defined(ADUMP_SOC_HOST) || ADUMP_SOC_HOST == 1
228 6 : if (!mc2Spaces_.empty()) {
229 0 : return true;
230 : }
231 : #endif
232 6 : return false;
233 : }
234 :
235 27 : int32_t DumpFile::Dump(std::vector<std::string> &record)
236 : {
237 27 : if (!HasDumpData()) {
238 6 : IDE_LOGI("No dump data, skip dump file.");
239 6 : return ADUMP_SUCCESS;
240 : }
241 :
242 21 : int32_t ret = file_.EnsureOpen();
243 21 : if (ret != ADUMP_SUCCESS) {
244 0 : return ret;
245 : }
246 :
247 21 : SetAicInfo(record);
248 :
249 21 : ret = WriteHeader();
250 21 : if (ret != ADUMP_SUCCESS) {
251 0 : return ret;
252 : }
253 :
254 21 : ret = WriteInputTensors();
255 21 : if (ret != ADUMP_SUCCESS) {
256 0 : return ret;
257 : }
258 :
259 21 : ret = WriteOutputTensors();
260 21 : if (ret != ADUMP_SUCCESS) {
261 0 : return ret;
262 : }
263 :
264 21 : ret = WriteWorkspace();
265 21 : if (ret != ADUMP_SUCCESS) {
266 0 : return ret;
267 : }
268 :
269 21 : ret = WriteMc2Space();
270 21 : if (ret != ADUMP_SUCCESS) {
271 0 : return ret;
272 : }
273 :
274 21 : ret = WriteInputBuffer();
275 21 : if (ret != ADUMP_SUCCESS) {
276 0 : return ret;
277 : }
278 21 : return ADUMP_SUCCESS;
279 : }
280 :
281 21 : void DumpFile::SetAicInfo(std::vector<std::string> &record)
282 : {
283 21 : IDE_LOGI("[Set][DumpData] Start to set aic info");
284 21 : std::ostringstream oss;
285 294 : for (const auto &str : record) {
286 273 : oss << str;
287 : }
288 21 : for (const auto &str : logRecord_) {
289 0 : oss << str;
290 : }
291 42 : header_.set_dfx_message(oss.str());
292 21 : record.clear();
293 21 : logRecord_.clear();
294 21 : }
295 :
296 21 : int32_t DumpFile::WriteHeader() const
297 : {
298 21 : std::string protoHeader;
299 21 : uint64_t protoHeaderSize = header_.ByteSizeLong();
300 21 : const bool pbRet = header_.SerializeToString(&protoHeader);
301 21 : if ((!pbRet) || (protoHeaderSize == 0U)) {
302 0 : IDE_LOGE("Serialize dump header proto msg failed, size is %lu", protoHeaderSize);
303 0 : return ADUMP_FAILED;
304 : }
305 :
306 21 : int64_t ret = file_.Write(reinterpret_cast<char *>(&protoHeaderSize), sizeof(protoHeaderSize));
307 21 : if (ret == EN_ERROR || ret == EN_INVALID_PARAM) {
308 0 : IDE_LOGE("Write header proto msg size to dump file failed, ret: %ld", ret);
309 0 : return ADUMP_FAILED;
310 : }
311 :
312 21 : ret = file_.Write(protoHeader.c_str(), protoHeaderSize);
313 21 : if (ret == EN_ERROR || ret == EN_INVALID_PARAM) {
314 0 : IDE_LOGE("Write header proto msg to dump file failed, ret: %ld", ret);
315 0 : return ADUMP_FAILED;
316 : }
317 21 : return ADUMP_SUCCESS;
318 21 : }
319 :
320 21 : int32_t DumpFile::WriteInputTensors()
321 : {
322 21 : IDE_LOGI("[Dump][ExceptionInput] Start to dump exception input");
323 37 : for (size_t i = 0U; i < inputs_.size(); ++i) {
324 16 : const auto ret = WriteDeviceDataToFile(inputs_[i]);
325 16 : if (ret != ADUMP_SUCCESS) {
326 0 : IDE_LOGE("[Dump][ExceptionInput] Dump %zu input device data failed.", i);
327 0 : return ADUMP_FAILED;
328 : }
329 : }
330 21 : return ADUMP_SUCCESS;
331 : }
332 :
333 21 : int32_t DumpFile::WriteOutputTensors()
334 : {
335 21 : IDE_LOGI("[Dump][ExceptionOutput] Start to dump exception output");
336 37 : for (size_t i = 0U; i < outputs_.size(); ++i) {
337 16 : const auto ret = WriteDeviceDataToFile(outputs_[i]);
338 16 : if (ret != ADUMP_SUCCESS) {
339 0 : IDE_LOGE("[Dump][ExceptionOutput] Dump %zu output device data failed.", i);
340 0 : return ADUMP_FAILED;
341 : }
342 : }
343 21 : return ADUMP_SUCCESS;
344 : }
345 :
346 21 : int32_t DumpFile::WriteWorkspace()
347 : {
348 21 : IDE_LOGI("[Dump][ExceptionWorkspace] Start to dump exception workspace");
349 27 : for (size_t i = 0U; i < workspaces_.size(); ++i) {
350 6 : const auto ret = WriteDeviceDataToFile(workspaces_[i]);
351 6 : if (ret != ADUMP_SUCCESS) {
352 0 : IDE_LOGE("[Dump][ExceptionWorkspace] Dump %zu workspace failed.", i);
353 0 : return ADUMP_FAILED;
354 : }
355 : }
356 21 : return ADUMP_SUCCESS;
357 : }
358 :
359 21 : int32_t DumpFile::WriteMc2Space() const
360 : {
361 : #if !defined(ADUMP_SOC_HOST) || ADUMP_SOC_HOST == 1
362 21 : IDE_LOGI("[Dump][ExceptionMc2Space] Start to dump exception mc2 space on device %u.", deviceId_);
363 23 : for (size_t idx = 0U; idx < mc2Spaces_.size(); ++idx) {
364 2 : auto ret = WriteDeviceDataToFile(mc2Spaces_[idx]);
365 2 : IDE_CTRL_VALUE_FAILED(ret == ADUMP_SUCCESS, return ADUMP_FAILED,
366 : "[Dump][ExceptionWorkspace] Dump %zu workspace failed.", idx);
367 :
368 2 : auto it = mc2Ctx_.find(idx);
369 2 : if (it != mc2Ctx_.end()) {
370 10 : for (const DeviceData &devData : it->second) {
371 8 : ret = WriteDeviceDataToFile(devData);
372 : }
373 : }
374 : }
375 : #endif
376 21 : return ADUMP_SUCCESS;
377 : }
378 :
379 21 : int32_t DumpFile::WriteInputBuffer()
380 : {
381 21 : IDE_LOGI("[Dump][ExceptionWorkspace] Start to dump exception args");
382 117 : for (size_t i = 0U; i < inputBuffer_.size(); ++i) {
383 96 : const auto ret = WriteDeviceDataToFile(inputBuffer_[i]);
384 96 : if (ret != ADUMP_SUCCESS) {
385 0 : IDE_LOGE("[Dump][ExceptionOutput] Dump %zu input args failed.", i);
386 0 : return ADUMP_FAILED;
387 : }
388 : }
389 21 : return ADUMP_SUCCESS;
390 : }
391 :
392 : #if !defined(ADUMP_SOC_HOST) || ADUMP_SOC_HOST == 1
393 6 : bool DumpFile::SupportDumpMc2spaces() const
394 : {
395 6 : auto *plat = ExceptionDumpManager::Get();
396 6 : if (plat == nullptr) {
397 2 : IDE_LOGW("[Dump][Exception] Platform unavailable, mc2 spaces dump disabled.");
398 2 : return false;
399 : }
400 4 : return plat->SupportMc2SpacesDump();
401 : }
402 :
403 2 : uint64_t DumpFile::ComputeStructSize() const
404 : {
405 2 : auto *plat = ExceptionDumpManager::Get();
406 2 : if (plat == nullptr) {
407 0 : IDE_LOGE("[Dump][Exception] Platform unavailable, cannot determine mc2 struct size.");
408 0 : return 0;
409 : }
410 2 : return plat->GetMc2StructSize();
411 : }
412 :
413 1 : void DumpFile::HandleMc2CtxV1(const void *hostData, uint64_t &totalSize, std::vector<DeviceData> &dataList) const
414 : {
415 1 : const HcclCombinOpParam *param = static_cast<const HcclCombinOpParam *>(hostData);
416 0 : dataList.emplace_back(reinterpret_cast<const void*>(param->mc2WorkSpace.workSpace),
417 1 : param->mc2WorkSpace.workSpaceSize);
418 1 : totalSize += param->mc2WorkSpace.workSpaceSize;
419 1 : IDE_LOGI("[Dump][Exception] MC2 workspace addr 0x%llx and size %llu bytes, rankId %u",
420 : param->mc2WorkSpace.workSpace, param->mc2WorkSpace.workSpaceSize, param->rankId);
421 :
422 1 : if (param->rankId < RANK_NUM) {
423 1 : dataList.emplace_back(reinterpret_cast<const void*>(param->windowsIn[param->rankId]), param->winSize);
424 1 : dataList.emplace_back(reinterpret_cast<const void*>(param->windowsOut[param->rankId]), param->winSize);
425 1 : IDE_LOGI("[Dump][Exception] MC2 winSize %llu bytes, windowsIn addr 0x%llx, windowsOut addr 0x%llx",
426 : param->winSize, param->windowsIn[param->rankId], param->windowsOut[param->rankId]);
427 : } else {
428 0 : dataList.emplace_back(nullptr, param->winSize); // windowsIn 占位
429 0 : dataList.emplace_back(nullptr, param->winSize); // windowsOut 占位
430 0 : IDE_LOGW("[Dump][Exception] MC2 winSize %llu bytes, rankId is invalid, max rankId %u", RANK_NUM);
431 : }
432 1 : totalSize += param->winSize + param->winSize;
433 :
434 1 : const void *deviceDataPtr = reinterpret_cast<const void*>(param->ibverbsData);
435 1 : dataList.emplace_back(deviceDataPtr, param->ibverbsDataSize);
436 1 : totalSize += param->ibverbsDataSize;
437 :
438 1 : if (param->rankId >= param->ibverbsDataSize / sizeof(IbVerbsData)) {
439 0 : IDE_LOGW("[Dump][Exception] MC2 ibverbsData size %llu bytes, rankId out of range", param->ibverbsDataSize);
440 0 : return;
441 : }
442 :
443 1 : int32_t ret = DumpMemory::CheckDeviceMemory(deviceId_, deviceDataPtr);
444 1 : if (ret == ADUMP_SUCCESS) {
445 1 : void *hostDataPtr = DumpMemory::CopyDeviceToHost(deviceDataPtr, param->ibverbsDataSize);
446 1 : if (hostDataPtr != nullptr) {
447 1 : IbVerbsData *ibverbsData = static_cast<IbVerbsData *>(hostDataPtr);
448 0 : dataList.emplace_back(reinterpret_cast<const void*>(ibverbsData[param->rankId].localInput.addr),
449 1 : ibverbsData[param->rankId].localInput.size);
450 0 : dataList.emplace_back(reinterpret_cast<const void*>(ibverbsData[param->rankId].localOutput.addr),
451 1 : ibverbsData[param->rankId].localOutput.size);
452 1 : totalSize += ibverbsData[param->rankId].localInput.size + ibverbsData[param->rankId].localOutput.size;
453 1 : IDE_LOGI("[Dump][Exception] MC2 ibverbsData localInput addr 0x%llx and size %llu bytes, "
454 : "localOutput addr 0x%llx and size %llu bytes",
455 : ibverbsData[param->rankId].localInput.addr, ibverbsData[param->rankId].localInput.size,
456 : ibverbsData[param->rankId].localOutput.addr, ibverbsData[param->rankId].localOutput.size);
457 1 : DumpMemory::FreeHost(hostDataPtr);
458 : }
459 : }
460 : }
461 :
462 2 : uint64_t DumpFile::HandleMc2Ctx(const void *hostData, uint64_t opParamSize, size_t index)
463 : {
464 2 : std::vector<DeviceData> dataList;
465 2 : uint64_t totalSize = opParamSize;
466 2 : if (opParamSize == sizeof(HcclOpResParam)) {
467 1 : const HcclOpResParam *param = static_cast<const HcclOpResParam *>(hostData);
468 0 : dataList.emplace_back(reinterpret_cast<const void*>(param->mc2WorkSpace.workSpace),
469 1 : param->mc2WorkSpace.workSpaceSize);
470 1 : dataList.emplace_back(reinterpret_cast<const void*>(param->localWindowsExp), param->winExpSize);
471 1 : totalSize += param->mc2WorkSpace.workSpaceSize + param->winExpSize;
472 1 : IDE_LOGI("[Dump][Exception] MC2 workspace addr 0x%llx and size %llu bytes, windowsExp addr 0x%llx and "
473 : "size %llu bytes",
474 : param->mc2WorkSpace.workSpace, param->mc2WorkSpace.workSpaceSize,
475 : param->localWindowsExp, param->winExpSize);
476 : } else {
477 1 : HandleMc2CtxV1(hostData, totalSize, dataList);
478 : }
479 2 : mc2Ctx_.emplace(index, dataList);
480 2 : return totalSize;
481 2 : }
482 :
483 2 : uint64_t DumpFile::GetMc2DataSize(const void *addr, size_t index, uint64_t &opParamSize)
484 : {
485 2 : if (addr == nullptr) {
486 0 : IDE_LOGW("The addr from index %zu is nullptr, nothing need to do.", index);
487 0 : return 0;
488 : }
489 2 : void *hostData = nullptr;
490 2 : opParamSize = ComputeStructSize();
491 2 : if (opParamSize == 0) {
492 0 : IDE_LOGE("[Dump][Exception] Get struct size failed, size is 0.");
493 0 : return 0;
494 : }
495 :
496 2 : int32_t ret = DumpMemory::CheckDeviceMemory(deviceId_, addr);
497 2 : if (ret != ADUMP_SUCCESS) {
498 0 : return 0;
499 : } else {
500 2 : hostData = DumpMemory::CopyDeviceToHost(addr, opParamSize);
501 2 : if (hostData == nullptr) {
502 0 : IDE_LOGE("[Dump][Exception] the address maybe invalid, D2H failed, addr info: %p;%llu.", addr, opParamSize);
503 0 : return 0;
504 : }
505 : }
506 4 : HOST_RT_MEMORY_GUARD(hostData);
507 :
508 2 : uint64_t totalSize = HandleMc2Ctx(hostData, opParamSize, index);
509 :
510 2 : return totalSize;
511 2 : }
512 : #endif
513 :
514 1 : void DumpFile::LogIsOtherDeviceAddress(const DeviceData &devData, rtMemInfo_t &memInfo) const
515 : {
516 1 : uint32_t devNum = AdumpDsmi::DrvGetDevNum();
517 1 : if (devNum == 0) {
518 0 : IDE_LOGE("[Dump][Exception] DrvGetDevNum failed");
519 0 : return;
520 : }
521 :
522 1 : uint32_t length = MAX_DEV_NUM;
523 1 : std::vector<uint32_t> deviceIds(length, 0);
524 1 : if (!AdumpDsmi::DrvGetDevIds(length, deviceIds)) {
525 0 : IDE_LOGE("[Dump][Exception] DrvGetDevIds failed, length: %u", length);
526 0 : return;
527 : }
528 :
529 9 : for (size_t i = 0; i < devNum; ++i) {
530 8 : if (deviceIds[i] == deviceId_) {
531 1 : continue;
532 : }
533 7 : rtError_t ret = rtMemGetInfoByType(deviceIds[i], RT_MEM_INFO_TYPE_ADDR_CHECK, &memInfo);
534 7 : if ((ret == RT_ERROR_NONE) && (memInfo.addrInfo.flag == true)) {
535 0 : IDE_LOGE("[Dump][Exception] the address[%p] is the valid address of device[%d]", devData.addr,
536 : deviceIds[i]);
537 0 : return;
538 : }
539 : }
540 1 : }
541 :
542 1 : int32_t DumpFile::AllocDefaultMemory(void **hostPtr, uint64_t size) const
543 : {
544 1 : rtError_t rtRet = rtMallocHost(hostPtr, size, static_cast<uint16_t>(IDEDD));
545 1 : if (rtRet != RT_ERROR_NONE) {
546 0 : IDE_LOGE("Call rtMallocHost failed, size: %lu, ret: 0x%X", size, rtRet);
547 0 : return ADUMP_FAILED;
548 : }
549 1 : errno_t ret = memset_s(*hostPtr, size, 0, size);
550 1 : if (ret != EOK) {
551 0 : IDE_LOGE("Call memset_s failed, ret: %d", ret);
552 0 : DumpMemory::FreeHost(*hostPtr);
553 0 : return ADUMP_FAILED;
554 : }
555 :
556 1 : return ADUMP_SUCCESS;
557 : }
558 :
559 144 : int32_t DumpFile::WriteDeviceDataToFile(const DeviceData &devData) const
560 : {
561 144 : if (devData.bytes == 0) {
562 20 : IDE_LOGW("Skip dump addr %p because size is 0", devData.addr);
563 20 : return ADUMP_SUCCESS;
564 : }
565 :
566 124 : void *hostData = nullptr;
567 :
568 124 : rtMemInfo_t memInfo{};
569 124 : uint64_t *deviceAddr[1] = {static_cast<uint64_t *>(const_cast<void *>(devData.addr))};
570 124 : memInfo.addrInfo.addr = static_cast<uint64_t **>(deviceAddr);
571 124 : memInfo.addrInfo.cnt = 1;
572 124 : memInfo.addrInfo.memType = RT_MEM_MASK_DEV_TYPE | RT_MEM_MASK_RSVD_TYPE;
573 124 : memInfo.addrInfo.flag = true;
574 124 : rtError_t err = rtMemGetInfoByType(deviceId_, RT_MEM_INFO_TYPE_ADDR_CHECK, &memInfo);
575 124 : if ((err != RT_ERROR_NONE) || (memInfo.addrInfo.flag == false)) {
576 1 : IDE_LOGE("[Dump][Exception] the address maybe invalid, check addr info failed, error code: %d, check flag: %u, "
577 : "addr info: %p;%llu.",
578 : static_cast<int32_t>(err), memInfo.addrInfo.flag, devData.addr, devData.bytes);
579 1 : LogIsOtherDeviceAddress(devData, memInfo);
580 1 : if (AllocDefaultMemory(&hostData, devData.bytes) != ADUMP_SUCCESS) {
581 0 : return ADUMP_FAILED;
582 : }
583 : } else {
584 123 : hostData = DumpMemory::CopyDeviceToHost(devData.addr, devData.bytes);
585 123 : if (hostData == nullptr) {
586 0 : IDE_LOGE("[Dump][Exception] the address maybe invalid, D2H failed, addr info: %p;%llu.", devData.addr,
587 : devData.bytes);
588 0 : if (AllocDefaultMemory(&hostData, devData.bytes) != ADUMP_SUCCESS) {
589 0 : return ADUMP_FAILED;
590 : }
591 : }
592 : }
593 :
594 248 : HOST_RT_MEMORY_GUARD(hostData);
595 :
596 124 : const int64_t ret = file_.Write(reinterpret_cast<char *>(hostData), devData.bytes);
597 124 : if (ret == EN_ERROR || ret == EN_INVALID_PARAM) {
598 0 : IDE_LOGE("Write data to dump file failed, ret: %ld", ret);
599 0 : return ADUMP_FAILED;
600 : }
601 124 : return ADUMP_SUCCESS;
602 124 : }
603 : } // namespace Adx
|
