Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "dump_file.h"
12 : #include <sstream>
13 : #include "runtime/mem.h"
14 : #include "sys_utils.h"
15 : #include "str_utils.h"
16 : #include "dump_datatype.h"
17 : #include "dump_memory.h"
18 : #include "log/hdc_log.h"
19 : #include "log/adx_log.h"
20 : #include "adump_dsmi.h"
21 : #include "adump_platform_manager.h"
22 : #include "dump_tensor_plugin.h"
23 : #include "hccl_mc2_define.h"
24 : #include "proto/dump_task.pb.h"
25 :
26 : namespace Adx {
27 : namespace {
28 : constexpr char DUMP_FILE_VERSION[] = "2.0";
29 : constexpr uint32_t MAX_DEV_NUM = 64;
30 : } // namespace
31 :
32 50 : void DumpFile::SetHeader(const std::string &opName)
33 : {
34 50 : header_.set_version(DUMP_FILE_VERSION);
35 50 : header_.set_dump_time(SysUtils::GetTimestamp());
36 50 : header_.set_op_name(opName);
37 50 : }
38 :
39 10 : void DumpFile::SetOpAttr(const std::string &name, const std::string &value)
40 : {
41 10 : toolkit::dump::OpAttr attr;
42 : attr.set_name(name);
43 : attr.set_value(value);
44 10 : header_.mutable_attr()->Add(std::move(attr));
45 10 : IDE_LOGI("[Set][DumpData] Set op attr, name: %s", name.c_str());
46 10 : }
47 :
48 21 : void DumpFile::SetInputTensors(const std::vector<DumpTensor> &inputTensors)
49 : {
50 40 : for (size_t i = 0U; i < inputTensors.size(); ++i) {
51 19 : const DumpTensor &tensor = inputTensors[i];
52 19 : if (tensor.GetAddress() == nullptr) {
53 2 : continue;
54 : }
55 17 : toolkit::dump::OpInput input;
56 17 : auto ir_data_type = DumpDataType::GetIrDataType(static_cast<GeDataType>(tensor.GetDataType()));
57 17 : input.set_data_type(static_cast<toolkit::dump::OutputDataType>(ir_data_type));
58 17 : input.set_format(static_cast<toolkit::dump::OutputFormat>(tensor.GetFormat()));
59 17 : std::vector<int64_t> shape = tensor.GetShape();
60 51 : for (auto dim : shape)
61 : {
62 34 : input.mutable_shape()->add_dim(static_cast<uint64_t>(dim));
63 : }
64 17 : uint64_t size = static_cast<uint64_t>(tensor.GetSize());
65 17 : input.set_size(size);
66 17 : input.set_arg_index(tensor.GetArgsOffSet());
67 17 : input.set_input_type(static_cast<uint32_t>(DfxTensorType::INPUT_TENSOR));
68 17 : IDE_LOGI("[Set][DumpData] The input size exception is %llu bytes.", size);
69 17 : header_.mutable_input()->Add(std::move(input));
70 :
71 17 : inputs_.emplace_back(tensor.GetAddress(), size);
72 17 : }
73 21 : }
74 :
75 19 : void DumpFile::SetOutputTensors(const std::vector<DumpTensor> &outputTensors)
76 : {
77 25 : for (size_t i = 0U; i < outputTensors.size(); ++i) {
78 6 : const DumpTensor &tensor = outputTensors[i];
79 6 : if (tensor.GetAddress() == nullptr) {
80 1 : continue;
81 : }
82 5 : toolkit::dump::OpOutput output;
83 5 : auto ir_data_type = DumpDataType::GetIrDataType(static_cast<GeDataType>(tensor.GetDataType()));
84 5 : output.set_data_type(static_cast<toolkit::dump::OutputDataType>(ir_data_type));
85 5 : output.set_format(static_cast<toolkit::dump::OutputFormat>(tensor.GetFormat()));
86 5 : std::vector<int64_t> shape = tensor.GetShape();
87 15 : for (auto dim : shape) {
88 10 : output.mutable_shape()->add_dim(static_cast<uint64_t>(dim));
89 : }
90 5 : uint64_t size = static_cast<uint64_t>(tensor.GetSize());
91 5 : output.set_size(size);
92 5 : output.set_arg_index(tensor.GetArgsOffSet());
93 5 : IDE_LOGI("[Set][DumpData] The output size exception is %llu bytes.", size);
94 5 : header_.mutable_output()->Add(std::move(output));
95 :
96 5 : outputs_.emplace_back(tensor.GetAddress(), size);
97 5 : }
98 19 : }
99 :
100 26 : void DumpFile::SetWorkspaces(const std::vector<DumpWorkspace> &workspaces)
101 : {
102 26 : IDE_LOGI("[Set][DumpData] Workspace size is %zu bytes", workspaces.size());
103 37 : for (size_t i = 0U; i < workspaces.size(); ++i) {
104 11 : if (workspaces[i].addr == nullptr) {
105 1 : continue;
106 : }
107 10 : toolkit::dump::Workspace workspace;
108 10 : workspace.set_size(workspaces[i].bytes);
109 10 : workspace.set_arg_index(workspaces[i].argsOffset);
110 10 : IDE_LOGI("[Set][DumpData] The workspace size exception is %llu bytes.", workspaces[i].bytes);
111 10 : workspace.set_type(toolkit::dump::Workspace::LOG);
112 10 : header_.mutable_space()->Add(std::move(workspace));
113 :
114 10 : workspaces_.emplace_back(workspaces[i].addr, workspaces[i].bytes);
115 10 : }
116 26 : }
117 :
118 15 : void DumpFile::SetTensors(const std::vector<TensorInfo> &tensors, std::vector<std::string> &record)
119 : {
120 15 : std::vector<DumpTensor> inputTensors;
121 15 : std::vector<DumpTensor> outputTensors;
122 15 : std::vector<DumpWorkspace> workspaces;
123 39 : for (size_t index = 0U; index < tensors.size(); ++index) {
124 24 : const TensorInfo &tensor = tensors[index];
125 24 : if (tensor.tensorAddr == nullptr || tensor.tensorSize == 0) {
126 1 : IDE_LOGW("[Set][DumpData] skip null/empty tensor[%zu], addr=%p, size=%zu, type=%d.",
127 : index, tensor.tensorAddr, tensor.tensorSize, static_cast<int32_t>(tensor.type));
128 1 : continue;
129 : }
130 :
131 : std::string info = StrUtils::Format(
132 : "[Dump][Exception] exception info dump tensor data, type=%s; index=%zu; shape=%s; format=%s; "
133 : "dtype=%s; address=%p; size=%zu bytes",
134 46 : DumpDataType::TensorTypeToSerialString(tensor.type).c_str(), index,
135 46 : StrUtils::ToString(tensor.shape).c_str(),
136 46 : DumpDataType::FormatToSerialString(tensor.format).c_str(),
137 23 : DumpDataType::DataTypeToSerialString(tensor.dataType).c_str(),
138 92 : tensor.tensorAddr, tensor.tensorSize);
139 23 : IDE_LOGI("%s", info.c_str());
140 23 : record.emplace_back(info + "\n");
141 :
142 23 : if (tensor.type == TensorType::INPUT) {
143 15 : inputTensors.emplace_back(tensor);
144 8 : } else if (tensor.type == TensorType::OUTPUT) {
145 4 : outputTensors.emplace_back(tensor);
146 4 : } else if (tensor.type == TensorType::WORKSPACE) {
147 4 : workspaces.emplace_back(tensor.tensorAddr, static_cast<uint64_t>(tensor.tensorSize), tensor.argsOffSet);
148 : }
149 23 : }
150 15 : SetInputTensors(inputTensors);
151 15 : SetOutputTensors(outputTensors);
152 15 : SetWorkspaces(workspaces);
153 15 : }
154 :
155 : #if !defined(ADUMP_SOC_HOST) || ADUMP_SOC_HOST == 1
156 6 : void DumpFile::SetMc2spaces(const std::vector<DumpWorkspace> &mc2Spaces)
157 : {
158 6 : IDE_CTRL_VALUE_WARN(SupportDumpMc2spaces(), return, "Not support to dump mc2 spaces data");
159 3 : IDE_LOGI("[Set][DumpData] mc2 space size is %zu", mc2Spaces.size());
160 5 : for (size_t i = 0U; i < mc2Spaces.size(); ++i) {
161 2 : if (mc2Spaces[i].addr == nullptr) {
162 0 : continue;
163 : }
164 2 : uint64_t opParamSize = 0;
165 2 : uint64_t newSize = GetMc2DataSize(mc2Spaces[i].addr, i, opParamSize);
166 2 : if (opParamSize == 0 || newSize == 0) {
167 0 : continue;
168 : }
169 2 : toolkit::dump::Workspace workspace;
170 2 : workspace.set_size(newSize);
171 2 : workspace.set_arg_index(mc2Spaces[i].argsOffset);
172 2 : workspace.set_type(toolkit::dump::Workspace::LOG);
173 2 : header_.mutable_space()->Add(std::move(workspace));
174 2 : IDE_LOGI("[Set][DumpData] The mc2 space size exception is %llu bytes.", newSize);
175 :
176 2 : mc2Spaces_.emplace_back(mc2Spaces[i].addr, opParamSize);
177 2 : }
178 : }
179 : #endif
180 :
181 28 : void DumpFile::SetInputBuffer(const std::vector<InputBuffer> input)
182 : {
183 28 : IDE_LOGI("[Set][DumpData] input buffer size is %zu", input.size());
184 126 : for (size_t i = 0U; i < input.size(); ++i) {
185 98 : if (input[i].addr == nullptr) {
186 2 : if (input[i].length != 0) {
187 2 : std::ostringstream oss;
188 2 : oss << "[Dump][Exception] the address maybe invalid, addr info: " << input[i].addr << ";"
189 2 : << input[i].length << ".";
190 2 : IDE_LOGE("%s", oss.str().c_str());
191 2 : logRecord_.emplace_back(oss.str() + "\n");
192 2 : }
193 2 : continue;
194 2 : }
195 96 : toolkit::dump::OpInput opInput;
196 96 : opInput.set_data_type(toolkit::dump::OutputDataType::DT_UNDEFINED);
197 96 : opInput.set_size(input[i].length);
198 96 : opInput.set_arg_index(input[i].argIndex);
199 96 : opInput.set_input_type(static_cast<uint32_t>(DfxTensorType::GENERAL_TENSOR));
200 96 : IDE_LOGI("[Set][DumpData] The input size exception is %llu", input[i].length);
201 96 : header_.mutable_input()->Add(std::move(opInput));
202 :
203 96 : inputBuffer_.emplace_back(input[i].addr, input[i].length);
204 96 : }
205 28 : }
206 :
207 10 : void DumpFile::SetTensorBuffer(const std::vector<TensorBuffer> &tensorBuffer)
208 : {
209 43 : for (size_t i = 0U; i < tensorBuffer.size(); ++i) {
210 33 : if (tensorBuffer[i].addr == nullptr) {
211 2 : if (tensorBuffer[i].size != 0) {
212 2 : std::ostringstream oss;
213 2 : oss << "[Dump][Exception] the address maybe invalid, addr info: " << tensorBuffer[i].addr << ";"
214 2 : << tensorBuffer[i].size << ".";
215 2 : IDE_LOGE("%s", oss.str().c_str());
216 2 : logRecord_.emplace_back(oss.str() + "\n");
217 2 : }
218 2 : continue;
219 2 : }
220 31 : if (tensorBuffer[i].tensorType <= DfxTensorType::INPUT_TENSOR) {
221 10 : toolkit::dump::OpInput input;
222 10 : input.set_data_type(toolkit::dump::OutputDataType::DT_UNDEFINED);
223 10 : uint64_t dimension = tensorBuffer[i].dimension;
224 10 : IDE_LOGI("[Set][DumpData] The input dim is %llu", dimension);
225 28 : for (uint64_t j = 0; j < dimension; ++j) {
226 18 : input.mutable_shape()->add_dim(tensorBuffer[i].shape[j]);
227 18 : IDE_LOGI("[Set][DumpData] The input shape is %llu", tensorBuffer[i].shape[j]);
228 : }
229 10 : uint64_t size = tensorBuffer[i].GetTotalByteSize();
230 10 : input.set_size(size);
231 10 : input.set_arg_index(tensorBuffer[i].argIndex);
232 10 : input.set_input_type(static_cast<uint32_t>(tensorBuffer[i].tensorType));
233 10 : IDE_LOGI("[Set][DumpData] The input size exception is %llu", size);
234 10 : header_.mutable_input()->Add(std::move(input));
235 :
236 10 : inputs_.emplace_back(tensorBuffer[i].addr, size);
237 36 : } else if (tensorBuffer[i].tensorType == DfxTensorType::OUTPUT_TENSOR ||
238 5 : tensorBuffer[i].tensorType == DfxTensorType::SHAPE_TENSOR) {
239 16 : toolkit::dump::OpOutput output;
240 16 : output.set_data_type(toolkit::dump::OutputDataType::DT_UNDEFINED);
241 16 : uint64_t dimension = tensorBuffer[i].dimension;
242 16 : IDE_LOGI("[Set][DumpData] The output dim is %llu", dimension);
243 40 : for (uint64_t j = 0; j < dimension; ++j) {
244 24 : output.mutable_shape()->add_dim(tensorBuffer[i].shape[j]);
245 24 : IDE_LOGI("[Set][DumpData] The output shape is %llu", tensorBuffer[i].shape[j]);
246 : }
247 16 : uint64_t size = tensorBuffer[i].GetTotalByteSize();
248 16 : output.set_size(size);
249 16 : output.set_arg_index(tensorBuffer[i].argIndex);
250 16 : IDE_LOGI("[Set][DumpData] The output size exception is %llu", size);
251 16 : header_.mutable_output()->Add(std::move(output));
252 :
253 16 : outputs_.emplace_back(tensorBuffer[i].addr, size);
254 21 : } else if (tensorBuffer[i].tensorType == DfxTensorType::TILING_DATA) {
255 5 : toolkit::dump::OpInput input;
256 5 : input.set_data_type(toolkit::dump::OutputDataType::DT_UNDEFINED);
257 5 : uint64_t size = tensorBuffer[i].GetTotalByteSize();
258 5 : input.set_size(size);
259 5 : input.set_arg_index(tensorBuffer[i].argIndex);
260 5 : input.set_input_type(static_cast<uint32_t>(tensorBuffer[i].tensorType));
261 5 : IDE_LOGI("[Set][DumpData] The tiling size exception is %llu", size);
262 5 : header_.mutable_input()->Add(std::move(input));
263 :
264 5 : inputs_.emplace_back(tensorBuffer[i].addr, size);
265 5 : }
266 : }
267 10 : }
268 :
269 28 : bool DumpFile::HasDumpData() const
270 : {
271 28 : if (!inputs_.empty() || !outputs_.empty() || !workspaces_.empty() || !inputBuffer_.empty()) {
272 22 : return true;
273 : }
274 : #if !defined(ADUMP_SOC_HOST) || ADUMP_SOC_HOST == 1
275 6 : if (!mc2Spaces_.empty()) {
276 0 : return true;
277 : }
278 : #endif
279 6 : return false;
280 : }
281 :
282 28 : int32_t DumpFile::Dump(std::vector<std::string> &record)
283 : {
284 28 : if (!HasDumpData()) {
285 6 : IDE_LOGI("No dump data, skip dump file.");
286 6 : return ADUMP_SUCCESS;
287 : }
288 :
289 22 : int32_t ret = file_.EnsureOpen();
290 22 : if (ret != ADUMP_SUCCESS) {
291 0 : return ret;
292 : }
293 :
294 22 : SetAicInfo(record);
295 :
296 22 : ret = WriteHeader();
297 22 : if (ret != ADUMP_SUCCESS) {
298 0 : return ret;
299 : }
300 :
301 22 : ret = WriteInputTensors();
302 22 : if (ret != ADUMP_SUCCESS) {
303 0 : return ret;
304 : }
305 :
306 22 : ret = WriteOutputTensors();
307 22 : if (ret != ADUMP_SUCCESS) {
308 0 : return ret;
309 : }
310 :
311 22 : ret = WriteWorkspace();
312 22 : if (ret != ADUMP_SUCCESS) {
313 0 : return ret;
314 : }
315 :
316 22 : ret = WriteMc2Space();
317 22 : if (ret != ADUMP_SUCCESS) {
318 0 : return ret;
319 : }
320 :
321 22 : ret = WriteInputBuffer();
322 22 : if (ret != ADUMP_SUCCESS) {
323 0 : return ret;
324 : }
325 22 : return ADUMP_SUCCESS;
326 : }
327 :
328 22 : void DumpFile::SetAicInfo(std::vector<std::string> &record)
329 : {
330 22 : IDE_LOGI("[Set][DumpData] Start to set aic info");
331 22 : std::ostringstream oss;
332 295 : for (const auto &str : record) {
333 273 : oss << str;
334 : }
335 22 : for (const auto &str : logRecord_) {
336 0 : oss << str;
337 : }
338 44 : header_.set_dfx_message(oss.str());
339 22 : record.clear();
340 22 : logRecord_.clear();
341 22 : }
342 :
343 22 : int32_t DumpFile::WriteHeader() const
344 : {
345 22 : std::string protoHeader;
346 22 : uint64_t protoHeaderSize = header_.ByteSizeLong();
347 22 : const bool pbRet = header_.SerializeToString(&protoHeader);
348 22 : if ((!pbRet) || (protoHeaderSize == 0U)) {
349 0 : IDE_LOGE("Serialize dump header proto msg failed, size is %lu", protoHeaderSize);
350 0 : return ADUMP_FAILED;
351 : }
352 :
353 22 : int64_t ret = file_.Write(reinterpret_cast<char *>(&protoHeaderSize), sizeof(protoHeaderSize));
354 22 : if (ret == EN_ERROR || ret == EN_INVALID_PARAM) {
355 0 : IDE_LOGE("Write header proto msg size to dump file failed, ret: %ld", ret);
356 0 : return ADUMP_FAILED;
357 : }
358 :
359 22 : ret = file_.Write(protoHeader.c_str(), protoHeaderSize);
360 22 : if (ret == EN_ERROR || ret == EN_INVALID_PARAM) {
361 0 : IDE_LOGE("Write header proto msg to dump file failed, ret: %ld", ret);
362 0 : return ADUMP_FAILED;
363 : }
364 22 : return ADUMP_SUCCESS;
365 22 : }
366 :
367 22 : int32_t DumpFile::WriteInputTensors()
368 : {
369 22 : IDE_LOGI("[Dump][ExceptionInput] Start to dump exception input");
370 39 : for (size_t i = 0U; i < inputs_.size(); ++i) {
371 17 : const auto ret = WriteDeviceDataToFile(inputs_[i]);
372 17 : if (ret != ADUMP_SUCCESS) {
373 0 : IDE_LOGE("[Dump][ExceptionInput] Dump %zu input device data failed.", i);
374 0 : return ADUMP_FAILED;
375 : }
376 : }
377 22 : return ADUMP_SUCCESS;
378 : }
379 :
380 22 : int32_t DumpFile::WriteOutputTensors()
381 : {
382 22 : IDE_LOGI("[Dump][ExceptionOutput] Start to dump exception output");
383 38 : for (size_t i = 0U; i < outputs_.size(); ++i) {
384 16 : const auto ret = WriteDeviceDataToFile(outputs_[i]);
385 16 : if (ret != ADUMP_SUCCESS) {
386 0 : IDE_LOGE("[Dump][ExceptionOutput] Dump %zu output device data failed.", i);
387 0 : return ADUMP_FAILED;
388 : }
389 : }
390 22 : return ADUMP_SUCCESS;
391 : }
392 :
393 22 : int32_t DumpFile::WriteWorkspace()
394 : {
395 22 : IDE_LOGI("[Dump][ExceptionWorkspace] Start to dump exception workspace");
396 28 : for (size_t i = 0U; i < workspaces_.size(); ++i) {
397 6 : const auto ret = WriteDeviceDataToFile(workspaces_[i]);
398 6 : if (ret != ADUMP_SUCCESS) {
399 0 : IDE_LOGE("[Dump][ExceptionWorkspace] Dump %zu workspace failed.", i);
400 0 : return ADUMP_FAILED;
401 : }
402 : }
403 22 : return ADUMP_SUCCESS;
404 : }
405 :
406 22 : int32_t DumpFile::WriteMc2Space() const
407 : {
408 : #if !defined(ADUMP_SOC_HOST) || ADUMP_SOC_HOST == 1
409 22 : IDE_LOGI("[Dump][ExceptionMc2Space] Start to dump exception mc2 space on device %u.", deviceId_);
410 24 : for (size_t idx = 0U; idx < mc2Spaces_.size(); ++idx) {
411 2 : auto ret = WriteDeviceDataToFile(mc2Spaces_[idx]);
412 2 : IDE_CTRL_VALUE_FAILED(ret == ADUMP_SUCCESS, return ADUMP_FAILED,
413 : "[Dump][ExceptionWorkspace] Dump %zu workspace failed.", idx);
414 :
415 2 : auto it = mc2Ctx_.find(idx);
416 2 : if (it != mc2Ctx_.end()) {
417 10 : for (const DeviceData &devData : it->second) {
418 8 : ret = WriteDeviceDataToFile(devData);
419 : }
420 : }
421 : }
422 : #endif
423 22 : return ADUMP_SUCCESS;
424 : }
425 :
426 22 : int32_t DumpFile::WriteInputBuffer()
427 : {
428 22 : IDE_LOGI("[Dump][ExceptionWorkspace] Start to dump exception args");
429 118 : for (size_t i = 0U; i < inputBuffer_.size(); ++i) {
430 96 : const auto ret = WriteDeviceDataToFile(inputBuffer_[i]);
431 96 : if (ret != ADUMP_SUCCESS) {
432 0 : IDE_LOGE("[Dump][ExceptionOutput] Dump %zu input args failed.", i);
433 0 : return ADUMP_FAILED;
434 : }
435 : }
436 22 : return ADUMP_SUCCESS;
437 : }
438 :
439 : #if !defined(ADUMP_SOC_HOST) || ADUMP_SOC_HOST == 1
440 6 : bool DumpFile::SupportDumpMc2spaces() const
441 : {
442 6 : auto *plat = ExceptionDumpManager::Get();
443 6 : if (plat == nullptr) {
444 2 : IDE_LOGW("[Dump][Exception] Platform unavailable, mc2 spaces dump disabled.");
445 2 : return false;
446 : }
447 4 : return plat->SupportMc2SpacesDump();
448 : }
449 :
450 2 : uint64_t DumpFile::ComputeStructSize() const
451 : {
452 2 : auto *plat = ExceptionDumpManager::Get();
453 2 : if (plat == nullptr) {
454 0 : IDE_LOGE("[Dump][Exception] Platform unavailable, cannot determine mc2 struct size.");
455 0 : return 0;
456 : }
457 2 : return plat->GetMc2StructSize();
458 : }
459 :
460 1 : void DumpFile::HandleMc2CtxV1(const void *hostData, uint64_t &totalSize, std::vector<DeviceData> &dataList) const
461 : {
462 1 : const HcclCombinOpParam *param = static_cast<const HcclCombinOpParam *>(hostData);
463 0 : dataList.emplace_back(reinterpret_cast<const void*>(param->mc2WorkSpace.workSpace),
464 1 : param->mc2WorkSpace.workSpaceSize);
465 1 : totalSize += param->mc2WorkSpace.workSpaceSize;
466 1 : IDE_LOGI("[Dump][Exception] MC2 workspace addr 0x%llx and size %llu bytes, rankId %u",
467 : param->mc2WorkSpace.workSpace, param->mc2WorkSpace.workSpaceSize, param->rankId);
468 :
469 1 : if (param->rankId < RANK_NUM) {
470 1 : dataList.emplace_back(reinterpret_cast<const void*>(param->windowsIn[param->rankId]), param->winSize);
471 1 : dataList.emplace_back(reinterpret_cast<const void*>(param->windowsOut[param->rankId]), param->winSize);
472 1 : IDE_LOGI("[Dump][Exception] MC2 winSize %llu bytes, windowsIn addr 0x%llx, windowsOut addr 0x%llx",
473 : param->winSize, param->windowsIn[param->rankId], param->windowsOut[param->rankId]);
474 : } else {
475 0 : dataList.emplace_back(nullptr, param->winSize); // windowsIn 占位
476 0 : dataList.emplace_back(nullptr, param->winSize); // windowsOut 占位
477 0 : IDE_LOGW("[Dump][Exception] MC2 winSize %llu bytes, rankId is invalid, max rankId %u", RANK_NUM);
478 : }
479 1 : totalSize += param->winSize + param->winSize;
480 :
481 1 : const void *deviceDataPtr = reinterpret_cast<const void*>(param->ibverbsData);
482 1 : dataList.emplace_back(deviceDataPtr, param->ibverbsDataSize);
483 1 : totalSize += param->ibverbsDataSize;
484 :
485 1 : if (param->rankId >= param->ibverbsDataSize / sizeof(IbVerbsData)) {
486 0 : IDE_LOGW("[Dump][Exception] MC2 ibverbsData size %llu bytes, rankId out of range", param->ibverbsDataSize);
487 0 : return;
488 : }
489 :
490 1 : int32_t ret = DumpMemory::CheckDeviceMemory(deviceId_, deviceDataPtr);
491 1 : if (ret == ADUMP_SUCCESS) {
492 1 : void *hostDataPtr = DumpMemory::CopyDeviceToHost(deviceDataPtr, param->ibverbsDataSize);
493 1 : if (hostDataPtr != nullptr) {
494 1 : IbVerbsData *ibverbsData = static_cast<IbVerbsData *>(hostDataPtr);
495 0 : dataList.emplace_back(reinterpret_cast<const void*>(ibverbsData[param->rankId].localInput.addr),
496 1 : ibverbsData[param->rankId].localInput.size);
497 0 : dataList.emplace_back(reinterpret_cast<const void*>(ibverbsData[param->rankId].localOutput.addr),
498 1 : ibverbsData[param->rankId].localOutput.size);
499 1 : totalSize += ibverbsData[param->rankId].localInput.size + ibverbsData[param->rankId].localOutput.size;
500 1 : IDE_LOGI("[Dump][Exception] MC2 ibverbsData localInput addr 0x%llx and size %llu bytes, "
501 : "localOutput addr 0x%llx and size %llu bytes",
502 : ibverbsData[param->rankId].localInput.addr, ibverbsData[param->rankId].localInput.size,
503 : ibverbsData[param->rankId].localOutput.addr, ibverbsData[param->rankId].localOutput.size);
504 1 : DumpMemory::FreeHost(hostDataPtr);
505 : }
506 : }
507 : }
508 :
509 2 : uint64_t DumpFile::HandleMc2Ctx(const void *hostData, uint64_t opParamSize, size_t index)
510 : {
511 2 : std::vector<DeviceData> dataList;
512 2 : uint64_t totalSize = opParamSize;
513 2 : if (opParamSize == sizeof(HcclOpResParam)) {
514 1 : const HcclOpResParam *param = static_cast<const HcclOpResParam *>(hostData);
515 0 : dataList.emplace_back(reinterpret_cast<const void*>(param->mc2WorkSpace.workSpace),
516 1 : param->mc2WorkSpace.workSpaceSize);
517 1 : dataList.emplace_back(reinterpret_cast<const void*>(param->localWindowsExp), param->winExpSize);
518 1 : totalSize += param->mc2WorkSpace.workSpaceSize + param->winExpSize;
519 1 : IDE_LOGI("[Dump][Exception] MC2 workspace addr 0x%llx and size %llu bytes, windowsExp addr 0x%llx and "
520 : "size %llu bytes",
521 : param->mc2WorkSpace.workSpace, param->mc2WorkSpace.workSpaceSize,
522 : param->localWindowsExp, param->winExpSize);
523 : } else {
524 1 : HandleMc2CtxV1(hostData, totalSize, dataList);
525 : }
526 2 : mc2Ctx_.emplace(index, dataList);
527 2 : return totalSize;
528 2 : }
529 :
530 2 : uint64_t DumpFile::GetMc2DataSize(const void *addr, size_t index, uint64_t &opParamSize)
531 : {
532 2 : if (addr == nullptr) {
533 0 : IDE_LOGW("The addr from index %zu is nullptr, nothing need to do.", index);
534 0 : return 0;
535 : }
536 2 : void *hostData = nullptr;
537 2 : opParamSize = ComputeStructSize();
538 2 : if (opParamSize == 0) {
539 0 : IDE_LOGE("[Dump][Exception] Get struct size failed, size is 0.");
540 0 : return 0;
541 : }
542 :
543 2 : int32_t ret = DumpMemory::CheckDeviceMemory(deviceId_, addr);
544 2 : if (ret != ADUMP_SUCCESS) {
545 0 : return 0;
546 : } else {
547 2 : hostData = DumpMemory::CopyDeviceToHost(addr, opParamSize);
548 2 : if (hostData == nullptr) {
549 0 : IDE_LOGE("[Dump][Exception] the address maybe invalid, D2H failed, addr info: %p;%llu.", addr, opParamSize);
550 0 : return 0;
551 : }
552 : }
553 4 : HOST_RT_MEMORY_GUARD(hostData);
554 :
555 2 : uint64_t totalSize = HandleMc2Ctx(hostData, opParamSize, index);
556 :
557 2 : return totalSize;
558 2 : }
559 : #endif
560 :
561 1 : void DumpFile::LogIsOtherDeviceAddress(const DeviceData &devData, rtMemInfo_t &memInfo) const
562 : {
563 1 : uint32_t devNum = AdumpDsmi::DrvGetDevNum();
564 1 : if (devNum == 0) {
565 0 : IDE_LOGE("[Dump][Exception] DrvGetDevNum failed");
566 0 : return;
567 : }
568 :
569 1 : uint32_t length = MAX_DEV_NUM;
570 1 : std::vector<uint32_t> deviceIds(length, 0);
571 1 : if (!AdumpDsmi::DrvGetDevIds(length, deviceIds)) {
572 0 : IDE_LOGE("[Dump][Exception] DrvGetDevIds failed, length: %u", length);
573 0 : return;
574 : }
575 :
576 9 : for (size_t i = 0; i < devNum; ++i) {
577 8 : if (deviceIds[i] == deviceId_) {
578 1 : continue;
579 : }
580 7 : rtError_t ret = rtMemGetInfoByType(deviceIds[i], RT_MEM_INFO_TYPE_ADDR_CHECK, &memInfo);
581 7 : if ((ret == RT_ERROR_NONE) && (memInfo.addrInfo.flag == true)) {
582 0 : IDE_LOGE("[Dump][Exception] the address[%p] is the valid address of device[%d]", devData.addr,
583 : deviceIds[i]);
584 0 : return;
585 : }
586 : }
587 1 : }
588 :
589 1 : int32_t DumpFile::AllocDefaultMemory(void **hostPtr, uint64_t size) const
590 : {
591 1 : rtError_t rtRet = rtMallocHost(hostPtr, size, static_cast<uint16_t>(IDEDD));
592 1 : if (rtRet != RT_ERROR_NONE) {
593 0 : IDE_LOGE("Call rtMallocHost failed, size: %lu, ret: 0x%X", size, rtRet);
594 0 : return ADUMP_FAILED;
595 : }
596 1 : errno_t ret = memset_s(*hostPtr, size, 0, size);
597 1 : if (ret != EOK) {
598 0 : IDE_LOGE("Call memset_s failed, ret: %d", ret);
599 0 : DumpMemory::FreeHost(*hostPtr);
600 0 : return ADUMP_FAILED;
601 : }
602 :
603 1 : return ADUMP_SUCCESS;
604 : }
605 :
606 145 : int32_t DumpFile::WriteDeviceDataToFile(const DeviceData &devData) const
607 : {
608 145 : if (devData.bytes == 0) {
609 20 : IDE_LOGW("Skip dump addr %p because size is 0", devData.addr);
610 20 : return ADUMP_SUCCESS;
611 : }
612 :
613 125 : void *hostData = nullptr;
614 :
615 125 : rtMemInfo_t memInfo{};
616 125 : uint64_t *deviceAddr[1] = {static_cast<uint64_t *>(const_cast<void *>(devData.addr))};
617 125 : memInfo.addrInfo.addr = static_cast<uint64_t **>(deviceAddr);
618 125 : memInfo.addrInfo.cnt = 1;
619 125 : memInfo.addrInfo.memType = RT_MEM_MASK_DEV_TYPE | RT_MEM_MASK_RSVD_TYPE;
620 125 : memInfo.addrInfo.flag = true;
621 125 : rtError_t err = rtMemGetInfoByType(deviceId_, RT_MEM_INFO_TYPE_ADDR_CHECK, &memInfo);
622 125 : if ((err != RT_ERROR_NONE) || (memInfo.addrInfo.flag == false)) {
623 1 : IDE_LOGE("[Dump][Exception] the address maybe invalid, check addr info failed, error code: %d, check flag: %u, "
624 : "addr info: %p;%llu.",
625 : static_cast<int32_t>(err), memInfo.addrInfo.flag, devData.addr, devData.bytes);
626 1 : LogIsOtherDeviceAddress(devData, memInfo);
627 1 : if (AllocDefaultMemory(&hostData, devData.bytes) != ADUMP_SUCCESS) {
628 0 : return ADUMP_FAILED;
629 : }
630 : } else {
631 124 : hostData = DumpMemory::CopyDeviceToHost(devData.addr, devData.bytes);
632 124 : if (hostData == nullptr) {
633 0 : IDE_LOGE("[Dump][Exception] the address maybe invalid, D2H failed, addr info: %p;%llu.", devData.addr,
634 : devData.bytes);
635 0 : if (AllocDefaultMemory(&hostData, devData.bytes) != ADUMP_SUCCESS) {
636 0 : return ADUMP_FAILED;
637 : }
638 : }
639 : }
640 :
641 250 : HOST_RT_MEMORY_GUARD(hostData);
642 :
643 125 : const int64_t ret = file_.Write(reinterpret_cast<char *>(hostData), devData.bytes);
644 125 : if (ret == EN_ERROR || ret == EN_INVALID_PARAM) {
645 0 : IDE_LOGE("Write data to dump file failed, ret: %ld", ret);
646 0 : return ADUMP_FAILED;
647 : }
648 125 : return ADUMP_SUCCESS;
649 125 : }
650 : } // namespace Adx
|
