Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "dump_file.h"
12 : #include <sstream>
13 : #include "runtime/mem.h"
14 : #include "sys_utils.h"
15 : #include "str_utils.h"
16 : #include "dump_datatype.h"
17 : #include "dump_memory.h"
18 : #include "log/hdc_log.h"
19 : #include "log/adx_log.h"
20 : #include "adump_dsmi.h"
21 : #include "adump_platform_manager.h"
22 : #include "dump_tensor_plugin.h"
23 : #include "hccl_mc2_define.h"
24 : #include "proto/dump_task.pb.h"
25 :
26 : namespace Adx {
27 : namespace {
28 : constexpr char DUMP_FILE_VERSION[] = "2.0";
29 : constexpr uint32_t MAX_DEV_NUM = 64;
30 : } // namespace
31 :
32 50 : void DumpFile::SetHeader(const std::string& opName)
33 : {
34 50 : header_.set_version(DUMP_FILE_VERSION);
35 50 : header_.set_dump_time(SysUtils::GetTimestamp());
36 50 : header_.set_op_name(opName);
37 50 : }
38 :
39 10 : void DumpFile::SetOpAttr(const std::string& name, const std::string& value)
40 : {
41 10 : toolkit::dump::OpAttr attr;
42 : attr.set_name(name);
43 : attr.set_value(value);
44 20 : header_.mutable_attr()->Add(std::move(attr));
45 10 : IDE_LOGI("[Set][DumpData] Set op attr, name: %s", name.c_str());
46 10 : }
47 :
48 21 : void DumpFile::SetInputTensors(const std::vector<DumpTensor>& inputTensors)
49 : {
50 40 : for (size_t i = 0U; i < inputTensors.size(); ++i) {
51 19 : const DumpTensor& tensor = inputTensors[i];
52 19 : if (tensor.GetAddress() == nullptr) {
53 2 : continue;
54 : }
55 17 : toolkit::dump::OpInput input;
56 17 : auto ir_data_type = DumpDataType::GetIrDataType(static_cast<GeDataType>(tensor.GetDataType()));
57 17 : input.set_data_type(static_cast<toolkit::dump::OutputDataType>(ir_data_type));
58 17 : input.set_format(static_cast<toolkit::dump::OutputFormat>(tensor.GetFormat()));
59 17 : std::vector<int64_t> shape = tensor.GetShape();
60 68 : for (auto dim : shape) {
61 34 : input.mutable_shape()->add_dim(static_cast<uint64_t>(dim));
62 : }
63 17 : uint64_t size = static_cast<uint64_t>(tensor.GetSize());
64 17 : input.set_size(size);
65 17 : input.set_arg_index(tensor.GetArgsOffSet());
66 17 : input.set_input_type(static_cast<uint32_t>(DfxTensorType::INPUT_TENSOR));
67 17 : IDE_LOGI("[Set][DumpData] The input size exception is %llu bytes.", size);
68 34 : header_.mutable_input()->Add(std::move(input));
69 :
70 17 : inputs_.emplace_back(tensor.GetAddress(), size);
71 17 : }
72 21 : }
73 :
74 19 : void DumpFile::SetOutputTensors(const std::vector<DumpTensor>& outputTensors)
75 : {
76 25 : for (size_t i = 0U; i < outputTensors.size(); ++i) {
77 6 : const DumpTensor& tensor = outputTensors[i];
78 6 : if (tensor.GetAddress() == nullptr) {
79 1 : continue;
80 : }
81 5 : toolkit::dump::OpOutput output;
82 5 : auto ir_data_type = DumpDataType::GetIrDataType(static_cast<GeDataType>(tensor.GetDataType()));
83 5 : output.set_data_type(static_cast<toolkit::dump::OutputDataType>(ir_data_type));
84 5 : output.set_format(static_cast<toolkit::dump::OutputFormat>(tensor.GetFormat()));
85 5 : std::vector<int64_t> shape = tensor.GetShape();
86 20 : for (auto dim : shape) {
87 10 : output.mutable_shape()->add_dim(static_cast<uint64_t>(dim));
88 : }
89 5 : uint64_t size = static_cast<uint64_t>(tensor.GetSize());
90 5 : output.set_size(size);
91 5 : output.set_arg_index(tensor.GetArgsOffSet());
92 5 : IDE_LOGI("[Set][DumpData] The output size exception is %llu bytes.", size);
93 10 : header_.mutable_output()->Add(std::move(output));
94 :
95 5 : outputs_.emplace_back(tensor.GetAddress(), size);
96 5 : }
97 19 : }
98 :
99 26 : void DumpFile::SetWorkspaces(const std::vector<DumpWorkspace>& workspaces)
100 : {
101 26 : IDE_LOGI("[Set][DumpData] Workspace size is %zu bytes", workspaces.size());
102 37 : for (size_t i = 0U; i < workspaces.size(); ++i) {
103 11 : if (workspaces[i].addr == nullptr) {
104 1 : continue;
105 : }
106 10 : toolkit::dump::Workspace workspace;
107 10 : workspace.set_size(workspaces[i].bytes);
108 10 : workspace.set_arg_index(workspaces[i].argsOffset);
109 10 : IDE_LOGI("[Set][DumpData] The workspace size exception is %llu bytes.", workspaces[i].bytes);
110 10 : workspace.set_type(toolkit::dump::Workspace::LOG);
111 20 : header_.mutable_space()->Add(std::move(workspace));
112 :
113 10 : workspaces_.emplace_back(workspaces[i].addr, workspaces[i].bytes);
114 10 : }
115 26 : }
116 :
117 15 : void DumpFile::SetTensors(const std::vector<TensorInfo>& tensors, std::vector<std::string>& record)
118 : {
119 15 : std::vector<DumpTensor> inputTensors;
120 15 : std::vector<DumpTensor> outputTensors;
121 15 : std::vector<DumpWorkspace> workspaces;
122 39 : for (size_t index = 0U; index < tensors.size(); ++index) {
123 24 : const TensorInfo& tensor = tensors[index];
124 24 : if (tensor.tensorAddr == nullptr || tensor.tensorSize == 0) {
125 1 : IDE_LOGW(
126 : "[Set][DumpData] skip null/empty tensor[%zu], addr=%p, size=%zu, type=%d.", index, tensor.tensorAddr,
127 : tensor.tensorSize, static_cast<int32_t>(tensor.type));
128 1 : continue;
129 : }
130 :
131 : std::string info = StrUtils::Format(
132 : "[Dump][Exception] exception info dump tensor data, type=%s; index=%zu; shape=%s; format=%s; "
133 : "dtype=%s; address=%p; size=%zu bytes",
134 46 : DumpDataType::TensorTypeToSerialString(tensor.type).c_str(), index,
135 69 : StrUtils::ToString(tensor.shape).c_str(), DumpDataType::FormatToSerialString(tensor.format).c_str(),
136 115 : DumpDataType::DataTypeToSerialString(tensor.dataType).c_str(), tensor.tensorAddr, tensor.tensorSize);
137 23 : IDE_LOGI("%s", info.c_str());
138 23 : record.emplace_back(info + "\n");
139 :
140 23 : if (tensor.type == TensorType::INPUT) {
141 15 : inputTensors.emplace_back(tensor);
142 8 : } else if (tensor.type == TensorType::OUTPUT) {
143 4 : outputTensors.emplace_back(tensor);
144 4 : } else if (tensor.type == TensorType::WORKSPACE) {
145 4 : workspaces.emplace_back(tensor.tensorAddr, static_cast<uint64_t>(tensor.tensorSize), tensor.argsOffSet);
146 : }
147 23 : }
148 15 : SetInputTensors(inputTensors);
149 15 : SetOutputTensors(outputTensors);
150 15 : SetWorkspaces(workspaces);
151 15 : }
152 :
153 : #if !defined(ADUMP_SOC_HOST) || ADUMP_SOC_HOST == 1
154 6 : void DumpFile::SetMc2spaces(const std::vector<DumpWorkspace>& mc2Spaces)
155 : {
156 6 : IDE_CTRL_VALUE_WARN(SupportDumpMc2spaces(), return, "Not support to dump mc2 spaces data");
157 3 : IDE_LOGI("[Set][DumpData] mc2 space size is %zu", mc2Spaces.size());
158 5 : for (size_t i = 0U; i < mc2Spaces.size(); ++i) {
159 2 : if (mc2Spaces[i].addr == nullptr) {
160 0 : continue;
161 : }
162 2 : uint64_t opParamSize = 0;
163 2 : uint64_t newSize = GetMc2DataSize(mc2Spaces[i].addr, i, opParamSize);
164 2 : if (opParamSize == 0 || newSize == 0) {
165 0 : continue;
166 : }
167 2 : toolkit::dump::Workspace workspace;
168 2 : workspace.set_size(newSize);
169 2 : workspace.set_arg_index(mc2Spaces[i].argsOffset);
170 2 : workspace.set_type(toolkit::dump::Workspace::LOG);
171 4 : header_.mutable_space()->Add(std::move(workspace));
172 2 : IDE_LOGI("[Set][DumpData] The mc2 space size exception is %llu bytes.", newSize);
173 :
174 2 : mc2Spaces_.emplace_back(mc2Spaces[i].addr, opParamSize);
175 2 : }
176 : }
177 : #endif
178 :
179 28 : void DumpFile::SetInputBuffer(const std::vector<InputBuffer> input)
180 : {
181 28 : IDE_LOGI("[Set][DumpData] input buffer size is %zu", input.size());
182 126 : for (size_t i = 0U; i < input.size(); ++i) {
183 98 : if (input[i].addr == nullptr) {
184 2 : if (input[i].length != 0) {
185 2 : std::ostringstream oss;
186 2 : oss << "[Dump][Exception] the address maybe invalid, addr info: " << input[i].addr << ";"
187 2 : << input[i].length << ".";
188 2 : IDE_LOGE("%s", oss.str().c_str());
189 2 : logRecord_.emplace_back(oss.str() + "\n");
190 2 : }
191 2 : continue;
192 2 : }
193 96 : toolkit::dump::OpInput opInput;
194 96 : opInput.set_data_type(toolkit::dump::OutputDataType::DT_UNDEFINED);
195 96 : opInput.set_size(input[i].length);
196 96 : opInput.set_arg_index(input[i].argIndex);
197 96 : opInput.set_input_type(static_cast<uint32_t>(DfxTensorType::GENERAL_TENSOR));
198 96 : IDE_LOGI("[Set][DumpData] The input size exception is %llu", input[i].length);
199 192 : header_.mutable_input()->Add(std::move(opInput));
200 :
201 96 : inputBuffer_.emplace_back(input[i].addr, input[i].length);
202 96 : }
203 28 : }
204 :
205 10 : void DumpFile::SetTensorBuffer(const std::vector<TensorBuffer>& tensorBuffer)
206 : {
207 43 : for (size_t i = 0U; i < tensorBuffer.size(); ++i) {
208 33 : if (tensorBuffer[i].addr == nullptr) {
209 2 : if (tensorBuffer[i].size != 0) {
210 2 : std::ostringstream oss;
211 2 : oss << "[Dump][Exception] the address maybe invalid, addr info: " << tensorBuffer[i].addr << ";"
212 2 : << tensorBuffer[i].size << ".";
213 2 : IDE_LOGE("%s", oss.str().c_str());
214 2 : logRecord_.emplace_back(oss.str() + "\n");
215 2 : }
216 2 : continue;
217 2 : }
218 31 : if (tensorBuffer[i].tensorType <= DfxTensorType::INPUT_TENSOR) {
219 10 : toolkit::dump::OpInput input;
220 10 : input.set_data_type(toolkit::dump::OutputDataType::DT_UNDEFINED);
221 10 : uint64_t dimension = tensorBuffer[i].dimension;
222 10 : IDE_LOGI("[Set][DumpData] The input dim is %llu", dimension);
223 28 : for (uint64_t j = 0; j < dimension; ++j) {
224 18 : input.mutable_shape()->add_dim(tensorBuffer[i].shape[j]);
225 18 : IDE_LOGI("[Set][DumpData] The input shape is %llu", tensorBuffer[i].shape[j]);
226 : }
227 10 : uint64_t size = tensorBuffer[i].GetTotalByteSize();
228 10 : input.set_size(size);
229 10 : input.set_arg_index(tensorBuffer[i].argIndex);
230 10 : input.set_input_type(static_cast<uint32_t>(tensorBuffer[i].tensorType));
231 10 : IDE_LOGI("[Set][DumpData] The input size exception is %llu", size);
232 20 : header_.mutable_input()->Add(std::move(input));
233 :
234 10 : inputs_.emplace_back(tensorBuffer[i].addr, size);
235 10 : } else if (
236 26 : tensorBuffer[i].tensorType == DfxTensorType::OUTPUT_TENSOR ||
237 5 : tensorBuffer[i].tensorType == DfxTensorType::SHAPE_TENSOR) {
238 16 : toolkit::dump::OpOutput output;
239 16 : output.set_data_type(toolkit::dump::OutputDataType::DT_UNDEFINED);
240 16 : uint64_t dimension = tensorBuffer[i].dimension;
241 16 : IDE_LOGI("[Set][DumpData] The output dim is %llu", dimension);
242 40 : for (uint64_t j = 0; j < dimension; ++j) {
243 24 : output.mutable_shape()->add_dim(tensorBuffer[i].shape[j]);
244 24 : IDE_LOGI("[Set][DumpData] The output shape is %llu", tensorBuffer[i].shape[j]);
245 : }
246 16 : uint64_t size = tensorBuffer[i].GetTotalByteSize();
247 16 : output.set_size(size);
248 16 : output.set_arg_index(tensorBuffer[i].argIndex);
249 16 : IDE_LOGI("[Set][DumpData] The output size exception is %llu", size);
250 32 : header_.mutable_output()->Add(std::move(output));
251 :
252 16 : outputs_.emplace_back(tensorBuffer[i].addr, size);
253 21 : } else if (tensorBuffer[i].tensorType == DfxTensorType::TILING_DATA) {
254 5 : toolkit::dump::OpInput input;
255 5 : input.set_data_type(toolkit::dump::OutputDataType::DT_UNDEFINED);
256 5 : uint64_t size = tensorBuffer[i].GetTotalByteSize();
257 5 : input.set_size(size);
258 5 : input.set_arg_index(tensorBuffer[i].argIndex);
259 5 : input.set_input_type(static_cast<uint32_t>(tensorBuffer[i].tensorType));
260 5 : IDE_LOGI("[Set][DumpData] The tiling size exception is %llu", size);
261 10 : header_.mutable_input()->Add(std::move(input));
262 :
263 5 : inputs_.emplace_back(tensorBuffer[i].addr, size);
264 5 : }
265 : }
266 10 : }
267 :
268 28 : bool DumpFile::HasDumpData() const
269 : {
270 28 : if (!inputs_.empty() || !outputs_.empty() || !workspaces_.empty() || !inputBuffer_.empty()) {
271 22 : return true;
272 : }
273 : #if !defined(ADUMP_SOC_HOST) || ADUMP_SOC_HOST == 1
274 6 : if (!mc2Spaces_.empty()) {
275 0 : return true;
276 : }
277 : #endif
278 6 : return false;
279 : }
280 :
281 28 : int32_t DumpFile::Dump(std::vector<std::string>& record)
282 : {
283 28 : if (!HasDumpData()) {
284 6 : IDE_LOGI("No dump data, skip dump file.");
285 6 : return ADUMP_SUCCESS;
286 : }
287 :
288 22 : int32_t ret = file_.EnsureOpen();
289 22 : if (ret != ADUMP_SUCCESS) {
290 0 : return ret;
291 : }
292 :
293 22 : SetAicInfo(record);
294 :
295 22 : ret = WriteHeader();
296 22 : if (ret != ADUMP_SUCCESS) {
297 0 : return ret;
298 : }
299 :
300 22 : ret = WriteInputTensors();
301 22 : if (ret != ADUMP_SUCCESS) {
302 0 : return ret;
303 : }
304 :
305 22 : ret = WriteOutputTensors();
306 22 : if (ret != ADUMP_SUCCESS) {
307 0 : return ret;
308 : }
309 :
310 22 : ret = WriteWorkspace();
311 22 : if (ret != ADUMP_SUCCESS) {
312 0 : return ret;
313 : }
314 :
315 22 : ret = WriteMc2Space();
316 22 : if (ret != ADUMP_SUCCESS) {
317 0 : return ret;
318 : }
319 :
320 22 : ret = WriteInputBuffer();
321 22 : if (ret != ADUMP_SUCCESS) {
322 0 : return ret;
323 : }
324 22 : return ADUMP_SUCCESS;
325 : }
326 :
327 22 : void DumpFile::SetAicInfo(std::vector<std::string>& record)
328 : {
329 22 : IDE_LOGI("[Set][DumpData] Start to set aic info");
330 22 : std::ostringstream oss;
331 317 : for (const auto& str : record) {
332 273 : oss << str;
333 : }
334 44 : for (const auto& str : logRecord_) {
335 0 : oss << str;
336 : }
337 44 : header_.set_dfx_message(oss.str());
338 22 : record.clear();
339 22 : logRecord_.clear();
340 22 : }
341 :
342 22 : int32_t DumpFile::WriteHeader() const
343 : {
344 22 : std::string protoHeader;
345 22 : uint64_t protoHeaderSize = header_.ByteSizeLong();
346 22 : const bool pbRet = header_.SerializeToString(&protoHeader);
347 22 : if ((!pbRet) || (protoHeaderSize == 0U)) {
348 0 : IDE_LOGE("Serialize dump header proto msg failed, size is %lu", protoHeaderSize);
349 0 : return ADUMP_FAILED;
350 : }
351 :
352 22 : int64_t ret = file_.Write(reinterpret_cast<char*>(&protoHeaderSize), sizeof(protoHeaderSize));
353 22 : if (ret == EN_ERROR || ret == EN_INVALID_PARAM) {
354 0 : IDE_LOGE("Write header proto msg size to dump file failed, ret: %ld", ret);
355 0 : return ADUMP_FAILED;
356 : }
357 :
358 22 : ret = file_.Write(protoHeader.c_str(), protoHeaderSize);
359 22 : if (ret == EN_ERROR || ret == EN_INVALID_PARAM) {
360 0 : IDE_LOGE("Write header proto msg to dump file failed, ret: %ld", ret);
361 0 : return ADUMP_FAILED;
362 : }
363 22 : return ADUMP_SUCCESS;
364 22 : }
365 :
366 22 : int32_t DumpFile::WriteInputTensors()
367 : {
368 22 : IDE_LOGI("[Dump][ExceptionInput] Start to dump exception input");
369 39 : for (size_t i = 0U; i < inputs_.size(); ++i) {
370 17 : const auto ret = WriteDeviceDataToFile(inputs_[i]);
371 17 : if (ret != ADUMP_SUCCESS) {
372 0 : IDE_LOGE("[Dump][ExceptionInput] Dump %zu input device data failed.", i);
373 0 : return ADUMP_FAILED;
374 : }
375 : }
376 22 : return ADUMP_SUCCESS;
377 : }
378 :
379 22 : int32_t DumpFile::WriteOutputTensors()
380 : {
381 22 : IDE_LOGI("[Dump][ExceptionOutput] Start to dump exception output");
382 38 : for (size_t i = 0U; i < outputs_.size(); ++i) {
383 16 : const auto ret = WriteDeviceDataToFile(outputs_[i]);
384 16 : if (ret != ADUMP_SUCCESS) {
385 0 : IDE_LOGE("[Dump][ExceptionOutput] Dump %zu output device data failed.", i);
386 0 : return ADUMP_FAILED;
387 : }
388 : }
389 22 : return ADUMP_SUCCESS;
390 : }
391 :
392 22 : int32_t DumpFile::WriteWorkspace()
393 : {
394 22 : IDE_LOGI("[Dump][ExceptionWorkspace] Start to dump exception workspace");
395 28 : for (size_t i = 0U; i < workspaces_.size(); ++i) {
396 6 : const auto ret = WriteDeviceDataToFile(workspaces_[i]);
397 6 : if (ret != ADUMP_SUCCESS) {
398 0 : IDE_LOGE("[Dump][ExceptionWorkspace] Dump %zu workspace failed.", i);
399 0 : return ADUMP_FAILED;
400 : }
401 : }
402 22 : return ADUMP_SUCCESS;
403 : }
404 :
405 22 : int32_t DumpFile::WriteMc2Space() const
406 : {
407 : #if !defined(ADUMP_SOC_HOST) || ADUMP_SOC_HOST == 1
408 22 : IDE_LOGI("[Dump][ExceptionMc2Space] Start to dump exception mc2 space on device %u.", deviceId_);
409 24 : for (size_t idx = 0U; idx < mc2Spaces_.size(); ++idx) {
410 2 : auto ret = WriteDeviceDataToFile(mc2Spaces_[idx]);
411 2 : IDE_CTRL_VALUE_FAILED(
412 : ret == ADUMP_SUCCESS, return ADUMP_FAILED, "[Dump][ExceptionWorkspace] Dump %zu workspace failed.", idx);
413 :
414 2 : auto it = mc2Ctx_.find(idx);
415 2 : if (it != mc2Ctx_.end()) {
416 12 : for (const DeviceData& devData : it->second) {
417 8 : ret = WriteDeviceDataToFile(devData);
418 : }
419 : }
420 : }
421 : #endif
422 22 : return ADUMP_SUCCESS;
423 : }
424 :
425 22 : int32_t DumpFile::WriteInputBuffer()
426 : {
427 22 : IDE_LOGI("[Dump][ExceptionWorkspace] Start to dump exception args");
428 118 : for (size_t i = 0U; i < inputBuffer_.size(); ++i) {
429 96 : const auto ret = WriteDeviceDataToFile(inputBuffer_[i]);
430 96 : if (ret != ADUMP_SUCCESS) {
431 0 : IDE_LOGE("[Dump][ExceptionOutput] Dump %zu input args failed.", i);
432 0 : return ADUMP_FAILED;
433 : }
434 : }
435 22 : return ADUMP_SUCCESS;
436 : }
437 :
438 : #if !defined(ADUMP_SOC_HOST) || ADUMP_SOC_HOST == 1
439 6 : bool DumpFile::SupportDumpMc2spaces() const
440 : {
441 6 : auto* plat = ExceptionDumpManager::Get();
442 6 : if (plat == nullptr) {
443 2 : IDE_LOGW("[Dump][Exception] Platform unavailable, mc2 spaces dump disabled.");
444 2 : return false;
445 : }
446 4 : return plat->SupportMc2SpacesDump();
447 : }
448 :
449 2 : uint64_t DumpFile::ComputeStructSize() const
450 : {
451 2 : auto* plat = ExceptionDumpManager::Get();
452 2 : if (plat == nullptr) {
453 0 : IDE_LOGE("[Dump][Exception] Platform unavailable, cannot determine mc2 struct size.");
454 0 : return 0;
455 : }
456 2 : return plat->GetMc2StructSize();
457 : }
458 :
459 1 : void DumpFile::HandleMc2CtxV1(const void* hostData, uint64_t& totalSize, std::vector<DeviceData>& dataList) const
460 : {
461 1 : const HcclCombinOpParam* param = static_cast<const HcclCombinOpParam*>(hostData);
462 1 : dataList.emplace_back(
463 1 : reinterpret_cast<const void*>(param->mc2WorkSpace.workSpace), param->mc2WorkSpace.workSpaceSize);
464 1 : totalSize += param->mc2WorkSpace.workSpaceSize;
465 1 : IDE_LOGI(
466 : "[Dump][Exception] MC2 workspace addr 0x%llx and size %llu bytes, rankId %u", param->mc2WorkSpace.workSpace,
467 : param->mc2WorkSpace.workSpaceSize, param->rankId);
468 :
469 1 : if (param->rankId < RANK_NUM) {
470 1 : dataList.emplace_back(reinterpret_cast<const void*>(param->windowsIn[param->rankId]), param->winSize);
471 1 : dataList.emplace_back(reinterpret_cast<const void*>(param->windowsOut[param->rankId]), param->winSize);
472 1 : IDE_LOGI(
473 : "[Dump][Exception] MC2 winSize %llu bytes, windowsIn addr 0x%llx, windowsOut addr 0x%llx", param->winSize,
474 : param->windowsIn[param->rankId], param->windowsOut[param->rankId]);
475 : } else {
476 0 : dataList.emplace_back(nullptr, param->winSize); // windowsIn 占位
477 0 : dataList.emplace_back(nullptr, param->winSize); // windowsOut 占位
478 0 : IDE_LOGW("[Dump][Exception] MC2 winSize %llu bytes, rankId is invalid, max rankId %u", RANK_NUM);
479 : }
480 1 : totalSize += param->winSize + param->winSize;
481 :
482 1 : const void* deviceDataPtr = reinterpret_cast<const void*>(param->ibverbsData);
483 1 : dataList.emplace_back(deviceDataPtr, param->ibverbsDataSize);
484 1 : totalSize += param->ibverbsDataSize;
485 :
486 1 : if (param->rankId >= param->ibverbsDataSize / sizeof(IbVerbsData)) {
487 0 : IDE_LOGW("[Dump][Exception] MC2 ibverbsData size %llu bytes, rankId out of range", param->ibverbsDataSize);
488 0 : return;
489 : }
490 :
491 1 : int32_t ret = DumpMemory::CheckDeviceMemory(deviceId_, deviceDataPtr);
492 1 : if (ret == ADUMP_SUCCESS) {
493 1 : void* hostDataPtr = DumpMemory::CopyDeviceToHost(deviceDataPtr, param->ibverbsDataSize);
494 1 : if (hostDataPtr != nullptr) {
495 1 : IbVerbsData* ibverbsData = static_cast<IbVerbsData*>(hostDataPtr);
496 1 : dataList.emplace_back(
497 0 : reinterpret_cast<const void*>(ibverbsData[param->rankId].localInput.addr),
498 1 : ibverbsData[param->rankId].localInput.size);
499 1 : dataList.emplace_back(
500 0 : reinterpret_cast<const void*>(ibverbsData[param->rankId].localOutput.addr),
501 1 : ibverbsData[param->rankId].localOutput.size);
502 1 : totalSize += ibverbsData[param->rankId].localInput.size + ibverbsData[param->rankId].localOutput.size;
503 1 : IDE_LOGI(
504 : "[Dump][Exception] MC2 ibverbsData localInput addr 0x%llx and size %llu bytes, "
505 : "localOutput addr 0x%llx and size %llu bytes",
506 : ibverbsData[param->rankId].localInput.addr, ibverbsData[param->rankId].localInput.size,
507 : ibverbsData[param->rankId].localOutput.addr, ibverbsData[param->rankId].localOutput.size);
508 1 : DumpMemory::FreeHost(hostDataPtr);
509 : }
510 : }
511 : }
512 :
513 2 : uint64_t DumpFile::HandleMc2Ctx(const void* hostData, uint64_t opParamSize, size_t index)
514 : {
515 2 : std::vector<DeviceData> dataList;
516 2 : uint64_t totalSize = opParamSize;
517 2 : if (opParamSize == sizeof(HcclOpResParam)) {
518 1 : const HcclOpResParam* param = static_cast<const HcclOpResParam*>(hostData);
519 1 : dataList.emplace_back(
520 1 : reinterpret_cast<const void*>(param->mc2WorkSpace.workSpace), param->mc2WorkSpace.workSpaceSize);
521 1 : dataList.emplace_back(reinterpret_cast<const void*>(param->localWindowsExp), param->winExpSize);
522 1 : totalSize += param->mc2WorkSpace.workSpaceSize + param->winExpSize;
523 1 : IDE_LOGI(
524 : "[Dump][Exception] MC2 workspace addr 0x%llx and size %llu bytes, windowsExp addr 0x%llx and "
525 : "size %llu bytes",
526 : param->mc2WorkSpace.workSpace, param->mc2WorkSpace.workSpaceSize, param->localWindowsExp,
527 : param->winExpSize);
528 : } else {
529 1 : HandleMc2CtxV1(hostData, totalSize, dataList);
530 : }
531 2 : mc2Ctx_.emplace(index, dataList);
532 2 : return totalSize;
533 2 : }
534 :
535 2 : uint64_t DumpFile::GetMc2DataSize(const void* addr, size_t index, uint64_t& opParamSize)
536 : {
537 2 : if (addr == nullptr) {
538 0 : IDE_LOGW("The addr from index %zu is nullptr, nothing need to do.", index);
539 0 : return 0;
540 : }
541 2 : void* hostData = nullptr;
542 2 : opParamSize = ComputeStructSize();
543 2 : if (opParamSize == 0) {
544 0 : IDE_LOGE("[Dump][Exception] Get struct size failed, size is 0.");
545 0 : return 0;
546 : }
547 :
548 2 : int32_t ret = DumpMemory::CheckDeviceMemory(deviceId_, addr);
549 2 : if (ret != ADUMP_SUCCESS) {
550 0 : return 0;
551 : } else {
552 2 : hostData = DumpMemory::CopyDeviceToHost(addr, opParamSize);
553 2 : if (hostData == nullptr) {
554 0 : IDE_LOGE("[Dump][Exception] the address maybe invalid, D2H failed, addr info: %p;%llu.", addr, opParamSize);
555 0 : return 0;
556 : }
557 : }
558 4 : HOST_RT_MEMORY_GUARD(hostData);
559 :
560 2 : uint64_t totalSize = HandleMc2Ctx(hostData, opParamSize, index);
561 :
562 2 : return totalSize;
563 2 : }
564 : #endif
565 :
566 1 : void DumpFile::LogIsOtherDeviceAddress(const DeviceData& devData, rtMemInfo_t& memInfo) const
567 : {
568 1 : uint32_t devNum = AdumpDsmi::DrvGetDevNum();
569 1 : if (devNum == 0) {
570 0 : IDE_LOGE("[Dump][Exception] DrvGetDevNum failed");
571 0 : return;
572 : }
573 :
574 1 : uint32_t length = MAX_DEV_NUM;
575 1 : std::vector<uint32_t> deviceIds(length, 0);
576 1 : if (!AdumpDsmi::DrvGetDevIds(length, deviceIds)) {
577 0 : IDE_LOGE("[Dump][Exception] DrvGetDevIds failed, length: %u", length);
578 0 : return;
579 : }
580 :
581 9 : for (size_t i = 0; i < devNum; ++i) {
582 8 : if (deviceIds[i] == deviceId_) {
583 1 : continue;
584 : }
585 7 : rtError_t ret = rtMemGetInfoByType(deviceIds[i], RT_MEM_INFO_TYPE_ADDR_CHECK, &memInfo);
586 7 : if ((ret == RT_ERROR_NONE) && (memInfo.addrInfo.flag == true)) {
587 0 : IDE_LOGE(
588 : "[Dump][Exception] the address[%p] is the valid address of device[%d]", devData.addr, deviceIds[i]);
589 0 : return;
590 : }
591 : }
592 1 : }
593 :
594 1 : int32_t DumpFile::AllocDefaultMemory(void** hostPtr, uint64_t size) const
595 : {
596 1 : rtError_t rtRet = rtMallocHost(hostPtr, size, static_cast<uint16_t>(IDEDD));
597 1 : if (rtRet != RT_ERROR_NONE) {
598 0 : IDE_LOGE("Call rtMallocHost failed, size: %lu, ret: 0x%X", size, rtRet);
599 0 : return ADUMP_FAILED;
600 : }
601 1 : errno_t ret = memset_s(*hostPtr, size, 0, size);
602 1 : if (ret != EOK) {
603 0 : IDE_LOGE("Call memset_s failed, ret: %d", ret);
604 0 : DumpMemory::FreeHost(*hostPtr);
605 0 : return ADUMP_FAILED;
606 : }
607 :
608 1 : return ADUMP_SUCCESS;
609 : }
610 :
611 145 : int32_t DumpFile::WriteDeviceDataToFile(const DeviceData& devData) const
612 : {
613 145 : if (devData.bytes == 0) {
614 20 : IDE_LOGW("Skip dump addr %p because size is 0", devData.addr);
615 20 : return ADUMP_SUCCESS;
616 : }
617 :
618 125 : void* hostData = nullptr;
619 :
620 125 : rtMemInfo_t memInfo{};
621 125 : uint64_t* deviceAddr[1] = {static_cast<uint64_t*>(const_cast<void*>(devData.addr))};
622 125 : memInfo.addrInfo.addr = static_cast<uint64_t**>(deviceAddr);
623 125 : memInfo.addrInfo.cnt = 1;
624 125 : memInfo.addrInfo.memType = RT_MEM_MASK_DEV_TYPE | RT_MEM_MASK_RSVD_TYPE;
625 125 : memInfo.addrInfo.flag = true;
626 125 : rtError_t err = rtMemGetInfoByType(deviceId_, RT_MEM_INFO_TYPE_ADDR_CHECK, &memInfo);
627 125 : if ((err != RT_ERROR_NONE) || (memInfo.addrInfo.flag == false)) {
628 1 : IDE_LOGE(
629 : "[Dump][Exception] the address maybe invalid, check addr info failed, error code: %d, check flag: %u, "
630 : "addr info: %p;%llu.",
631 : static_cast<int32_t>(err), memInfo.addrInfo.flag, devData.addr, devData.bytes);
632 1 : LogIsOtherDeviceAddress(devData, memInfo);
633 1 : if (AllocDefaultMemory(&hostData, devData.bytes) != ADUMP_SUCCESS) {
634 0 : return ADUMP_FAILED;
635 : }
636 : } else {
637 124 : hostData = DumpMemory::CopyDeviceToHost(devData.addr, devData.bytes);
638 124 : if (hostData == nullptr) {
639 0 : IDE_LOGE(
640 : "[Dump][Exception] the address maybe invalid, D2H failed, addr info: %p;%llu.", devData.addr,
641 : devData.bytes);
642 0 : if (AllocDefaultMemory(&hostData, devData.bytes) != ADUMP_SUCCESS) {
643 0 : return ADUMP_FAILED;
644 : }
645 : }
646 : }
647 :
648 250 : HOST_RT_MEMORY_GUARD(hostData);
649 :
650 125 : const int64_t ret = file_.Write(reinterpret_cast<char*>(hostData), devData.bytes);
651 125 : if (ret == EN_ERROR || ret == EN_INVALID_PARAM) {
652 0 : IDE_LOGE("Write data to dump file failed, ret: %ld", ret);
653 0 : return ADUMP_FAILED;
654 : }
655 125 : return ADUMP_SUCCESS;
656 125 : }
657 : } // namespace Adx
|
