Line data Source code
1 : /**
2 : * Copyright (c) 2026 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "dump_stream_info.h"
12 : #include "dump_manager.h"
13 : #include "adx_dump_record.h"
14 : #include "common_utils.h"
15 : #include "adump/adx_datadump_callback.h"
16 : #include "adx_msg_proto.h"
17 : #include "memory_utils.h"
18 : #include "sys_utils.h"
19 : #include "dump_datatype.h"
20 :
21 : namespace Adx {
22 :
23 : static std::atomic<uint64_t> g_dumpNumber(0);
24 :
25 24 : uint64_t GetNextDumpNumber()
26 : {
27 24 : return g_dumpNumber.fetch_add(1);
28 : }
29 :
30 3 : void DumpResourceSafeMap::WaitInterval(uint32_t intervalSec)
31 : {
32 3 : std::this_thread::sleep_for(std::chrono::seconds(intervalSec));
33 3 : }
34 :
35 36 : void DumpResourceSafeMap::CleanupThreadLoop()
36 : {
37 36 : IDE_LOGI("Cleanup thread started");
38 81 : while (cleanupThreadActive_.load()) {
39 72 : std::string key;
40 : {
41 72 : std::unique_lock<std::mutex> lock(cleanupMtx_);
42 174 : cleanupCv_.wait(lock, [this]() { return !cleanupQueue_.empty() || !cleanupThreadActive_.load(); });
43 :
44 72 : if (cleanupQueue_.empty() && !cleanupThreadActive_.load()) {
45 27 : IDE_LOGI("Cleanup thread exiting");
46 27 : return;
47 : }
48 :
49 45 : if (!cleanupQueue_.empty()) {
50 45 : key = cleanupQueue_.front();
51 45 : cleanupQueue_.pop();
52 : }
53 72 : }
54 :
55 45 : if (!key.empty()) {
56 45 : IDE_LOGI("Cleanup thread removing key: %s", key.c_str());
57 45 : remove(key);
58 : }
59 72 : }
60 : }
61 :
62 45 : void DumpResourceSafeMap::StartCleanupThread()
63 : {
64 45 : if (cleanupThreadActive_.load()) {
65 9 : return;
66 : }
67 :
68 36 : std::lock_guard<std::mutex> lock(cleanupMtx_);
69 36 : if (cleanupThreadActive_.load()) {
70 0 : return;
71 : }
72 36 : cleanupThreadActive_.store(true);
73 36 : cleanupThread_ = std::thread(&DumpResourceSafeMap::CleanupThreadLoop, this);
74 36 : IDE_LOGI("Cleanup thread started");
75 36 : }
76 :
77 219 : void DumpResourceSafeMap::StopCleanupThread()
78 : {
79 : {
80 219 : std::lock_guard<std::mutex> lock(cleanupMtx_);
81 219 : if (!cleanupThreadActive_.load()) {
82 183 : return;
83 : }
84 36 : cleanupThreadActive_.store(false);
85 219 : }
86 36 : cleanupCv_.notify_one();
87 :
88 36 : if (cleanupThread_.joinable()) {
89 36 : cleanupThread_.join();
90 : }
91 36 : IDE_LOGI("Cleanup thread stopped");
92 : }
93 :
94 45 : void DumpResourceSafeMap::EnqueueCleanup(const std::string key)
95 : {
96 45 : StartCleanupThread();
97 : {
98 45 : std::lock_guard<std::mutex> lock(cleanupMtx_);
99 45 : cleanupQueue_.push(key);
100 45 : }
101 45 : cleanupCv_.notify_one();
102 45 : IDE_LOGI("Enqueued key for cleanup: %s", key.c_str());
103 45 : }
104 :
105 21 : bool DumpResourceSafeMap::IsCleanupThreadActive()
106 : {
107 21 : return cleanupThreadActive_.load();
108 : }
109 :
110 132 : int32_t DumpStreamCreate(DumpStreamInfo** ptr)
111 : {
112 132 : if (ptr == nullptr) {
113 3 : IDE_LOGE("Dump stream create failed, ptr is null");
114 3 : return ADUMP_FAILED;
115 : }
116 :
117 129 : DumpStreamInfo* dumpPtr = new (std::nothrow) DumpStreamInfo();
118 129 : if (dumpPtr == nullptr) {
119 0 : IDE_LOGE("Dump stream malloc failed");
120 0 : *ptr = nullptr;
121 0 : return ADUMP_FAILED;
122 : }
123 :
124 129 : rtError_t ret = rtEventCreateExWithFlag(&(dumpPtr->mainStmEvt), RT_EVENT_DDSYNC_NS);
125 129 : if (ret != RT_ERROR_NONE) {
126 6 : IDE_LOGE("create main stream event failed");
127 6 : delete dumpPtr;
128 6 : return ret;
129 : }
130 :
131 123 : ret = rtEventCreateExWithFlag(&(dumpPtr->dumpStmEvt), RT_EVENT_DDSYNC_NS);
132 123 : if (ret != RT_ERROR_NONE) {
133 3 : IDE_LOGE("create dump stream event failed");
134 3 : rtEventDestroy(dumpPtr->mainStmEvt);
135 3 : delete dumpPtr;
136 3 : return ret;
137 : }
138 :
139 120 : ret = rtCtxGetCurrent(&(dumpPtr->ctx));
140 120 : if (ret != RT_ERROR_NONE) {
141 0 : IDE_LOGE("get current context failed, ret: %d", ret);
142 0 : rtEventDestroy(dumpPtr->mainStmEvt);
143 0 : rtEventDestroy(dumpPtr->dumpStmEvt);
144 0 : delete dumpPtr; // stm 尚未创建无需释放
145 0 : return ret;
146 : }
147 :
148 120 : ret = rtStreamCreate(&(dumpPtr->stm), 0);
149 120 : if (ret != RT_ERROR_NONE) {
150 3 : IDE_LOGE("create dump stream failed");
151 3 : rtEventDestroy(dumpPtr->mainStmEvt);
152 3 : rtEventDestroy(dumpPtr->dumpStmEvt);
153 3 : delete dumpPtr;
154 3 : return ret;
155 : }
156 :
157 117 : dumpPtr->dumpStmId = 0;
158 117 : ret = rtGetStreamId(dumpPtr->stm, reinterpret_cast<int32_t*>(&(dumpPtr->dumpStmId)));
159 117 : if (ret != RT_ERROR_NONE) {
160 3 : IDE_LOGE("get dump stream id failed, ret: %d", ret);
161 3 : rtEventDestroy(dumpPtr->mainStmEvt);
162 3 : rtEventDestroy(dumpPtr->dumpStmEvt);
163 3 : rtStreamDestroy(dumpPtr->stm);
164 3 : delete dumpPtr;
165 3 : return ret;
166 : }
167 :
168 114 : *ptr = dumpPtr;
169 114 : return ADUMP_SUCCESS;
170 : }
171 :
172 120 : void DumpStreamFree(DumpStreamInfo* ptr)
173 : {
174 120 : if (!ptr) {
175 6 : return;
176 : }
177 114 : if (ptr->mainStmEvt) {
178 114 : rtEventDestroy(ptr->mainStmEvt);
179 : }
180 114 : if (ptr->dumpStmEvt) {
181 114 : rtEventDestroy(ptr->dumpStmEvt);
182 : }
183 114 : if (ptr->stm) {
184 114 : if (ptr->ctx != nullptr) {
185 114 : rtError_t ret = rtCtxSetCurrent(ptr->ctx);
186 114 : if (ret != RT_ERROR_NONE) {
187 0 : IDE_LOGE("set context failed before stream destroy, ret: %d", ret);
188 0 : return;
189 : }
190 : }
191 114 : rtStreamDestroy(ptr->stm);
192 114 : ptr->stm = nullptr;
193 114 : ptr->ctx = nullptr; // 防止悬空
194 : }
195 114 : ptr->inputTensors.clear();
196 114 : ptr->outputTensors.clear();
197 114 : delete ptr;
198 : }
199 :
200 27 : std::string GenerateDumpFileName(const DumpStreamInfo* dumpInfoPtr)
201 : {
202 27 : const std::string& dumpPath = dumpInfoPtr->dumpPath;
203 27 : const std::string& opType = dumpInfoPtr->opType;
204 27 : const std::string& opName = dumpInfoPtr->opName;
205 27 : uint32_t taskId = dumpInfoPtr->taskId;
206 27 : uint32_t streamId = dumpInfoPtr->streamId;
207 27 : uint32_t contextId = dumpInfoPtr->contextId;
208 27 : uint32_t threadId = dumpInfoPtr->threadId;
209 27 : uint32_t deviceId = dumpInfoPtr->deviceId;
210 27 : uint64_t timestamp = dumpInfoPtr->timestamp;
211 27 : uint64_t dumpNumber = dumpInfoPtr->dumpNumber;
212 :
213 27 : std::ostringstream fileNameoss;
214 27 : if (!dumpPath.empty() && dumpPath.back() == '/') {
215 6 : fileNameoss << dumpPath << deviceId << "/" << opType << "." << opName << "." << dumpNumber << "." << taskId
216 6 : << "." << streamId << "." << timestamp;
217 : } else {
218 21 : fileNameoss << dumpPath << "/" << deviceId << "/" << opType << "." << opName << "." << dumpNumber << "."
219 21 : << taskId << "." << streamId << "." << timestamp;
220 : }
221 27 : if (contextId != 0 && threadId != 0) {
222 3 : fileNameoss << ".FFTSPLUS." << contextId << "." << threadId << "." << deviceId;
223 : }
224 54 : return fileNameoss.str();
225 27 : }
226 :
227 57 : void FillTensorProtoInfo(const std::vector<DumpTensor>& tensors, toolkit::dump::DumpData& data, bool isInput)
228 : {
229 84 : for (size_t i = 0; i < tensors.size(); i++) {
230 27 : const DumpTensor& item = tensors[i];
231 27 : int32_t format = item.GetFormat();
232 :
233 27 : if (isInput) {
234 15 : auto* opInput = data.add_input();
235 : // Convert data type using DumpDataType helper
236 30 : opInput->set_data_type(
237 : static_cast<toolkit::dump::OutputDataType>(
238 15 : DumpDataType::GetIrDataType(static_cast<GeDataType>(item.GetDataType()))));
239 15 : opInput->set_format(static_cast<toolkit::dump::OutputFormat>(GetPrimaryFormat(format)));
240 15 : opInput->set_sub_format(GetSubFormat(format));
241 : // Address for input
242 15 : opInput->set_address(reinterpret_cast<uint64_t>(item.GetAddress()));
243 15 : opInput->set_offset(item.GetArgsOffSet());
244 15 : opInput->set_size(item.GetSize());
245 :
246 15 : auto* shape = opInput->mutable_shape();
247 30 : for (auto dim : item.GetShape()) {
248 15 : shape->add_dim(dim);
249 15 : }
250 15 : auto* originShape = opInput->mutable_original_shape();
251 30 : for (auto dim : item.GetOriginShape()) {
252 15 : originShape->add_dim(dim);
253 15 : }
254 : } else {
255 12 : auto* opOutput = data.add_output();
256 : // Convert data type using DumpDataType helper
257 24 : opOutput->set_data_type(
258 : static_cast<toolkit::dump::OutputDataType>(
259 12 : DumpDataType::GetIrDataType(static_cast<GeDataType>(item.GetDataType()))));
260 12 : opOutput->set_format(static_cast<toolkit::dump::OutputFormat>(GetPrimaryFormat(format)));
261 12 : opOutput->set_sub_format(GetSubFormat(format));
262 : // Offset and address for output
263 12 : opOutput->set_offset(item.GetArgsOffSet());
264 12 : opOutput->set_address(reinterpret_cast<uint64_t>(item.GetAddress()));
265 12 : opOutput->set_size(item.GetSize());
266 :
267 12 : auto* shape = opOutput->mutable_shape();
268 24 : for (auto dim : item.GetShape()) {
269 12 : shape->add_dim(dim);
270 12 : }
271 12 : auto* originShape = opOutput->mutable_original_shape();
272 24 : for (auto dim : item.GetOriginShape()) {
273 12 : originShape->add_dim(dim);
274 12 : }
275 : }
276 : }
277 57 : }
278 :
279 24 : toolkit::dump::DumpData BuildDumpDataProto(const DumpStreamInfo* dumpInfoPtr)
280 : {
281 24 : toolkit::dump::DumpData dumpData;
282 : dumpData.set_version("2.0");
283 24 : dumpData.set_dump_time(dumpInfoPtr->timestamp);
284 24 : dumpData.set_op_name(dumpInfoPtr->opName);
285 :
286 24 : FillTensorProtoInfo(dumpInfoPtr->inputTensors, dumpData, true);
287 24 : FillTensorProtoInfo(dumpInfoPtr->outputTensors, dumpData, false);
288 :
289 24 : return dumpData;
290 0 : }
291 :
292 6 : size_t CalculateTensorDataSize(const std::vector<DumpTensor>& tensors)
293 : {
294 6 : size_t totalSize = 0;
295 15 : for (const auto& tensor : tensors) {
296 9 : totalSize += tensor.GetSize();
297 : }
298 6 : return totalSize;
299 : }
300 :
301 33 : int32_t DumpTensorPushToDumpQueue(
302 : void* dataBuf, uint32_t bufLen, const char* fileName, uint64_t offset, uint32_t isLastChunk)
303 : {
304 : int err;
305 33 : uint32_t dataLen = 0;
306 33 : IDE_RETURN_IF_CHECK_ASSIGN_32U_ADD(sizeof(DumpChunk), bufLen, dataLen, return IDE_DAEMON_INTERGER_REVERSED_ERROR);
307 33 : MsgProto* msg = AdxMsgProto::CreateMsgPacket(IDE_DUMP_REQ, 0, nullptr, dataLen);
308 33 : IDE_CTRL_VALUE_FAILED(msg != nullptr, return IDE_DAEMON_MALLOC_ERROR, "create message failed");
309 33 : SharedPtr<MsgProto> sendDataMsgPtr(msg, IdeXfree);
310 33 : msg = nullptr;
311 33 : DumpChunk* data = reinterpret_cast<DumpChunk*>(sendDataMsgPtr->data);
312 33 : err = strcpy_s(data->fileName, IDE_MAX_FILE_PATH, fileName);
313 33 : IDE_CTRL_VALUE_FAILED(err == EOK, return IDE_DAEMON_INVALID_PATH_ERROR, "copy file name failed");
314 33 : data->bufLen = bufLen;
315 33 : data->flag = 0;
316 33 : data->isLastChunk = isLastChunk;
317 33 : data->offset = static_cast<int64_t>(offset);
318 33 : IDE_LOGI(
319 : "dataLen: %u, bufLen: %u, flag: %d, isLastChunk: %u, offset: %ld, fileName: %s", dataLen, data->bufLen,
320 : data->flag, data->isLastChunk, data->offset, data->fileName);
321 33 : err = memcpy_s(data->dataBuf, data->bufLen, dataBuf, bufLen);
322 33 : IDE_CTRL_VALUE_FAILED(err == EOK, return IDE_DAEMON_UNKNOW_ERROR, "memcpy_s data buffer failed");
323 33 : HostDumpDataInfo dataInfo = {sendDataMsgPtr, dataLen};
324 33 : if (!AdxDumpRecord::Instance().RecordDumpDataToQueue(dataInfo)) {
325 6 : IDE_LOGW("dump data queue full");
326 6 : return ADUMP_FAILED;
327 : }
328 27 : IDE_LOGD("dump data process normal");
329 27 : return ADUMP_SUCCESS;
330 33 : }
331 :
332 30 : int32_t FlushCurrentChunk(ChunkContext& ctx, uint32_t isLastChunk)
333 : {
334 30 : if (ctx.offset == 0) {
335 3 : return ADUMP_SUCCESS;
336 : }
337 27 : int32_t ret = DumpTensorPushToDumpQueue(
338 27 : ctx.buffer.data(), static_cast<uint32_t>(ctx.offset), ctx.fileName.c_str(), -1, isLastChunk);
339 27 : if (ret != ADUMP_SUCCESS) {
340 3 : IDE_LOGW(
341 : "DumpTensorPushToDumpQueue failed, ret: %d, fileName: %s, offset: %zu", ret, ctx.fileName.c_str(),
342 : ctx.offset);
343 : }
344 :
345 27 : ctx.offset = 0;
346 27 : errno_t memRet = memset_s(ctx.buffer.data(), DUMP_SLICE_SIZE, 0, DUMP_SLICE_SIZE);
347 27 : if (memRet != EOK) {
348 0 : IDE_LOGW("memset_s failed, ret: %d", memRet);
349 : }
350 27 : return ret;
351 : }
352 :
353 30 : int32_t CopyTensorDataWithFlush(const DumpTensor& tensor, ChunkContext& ctx, bool isLastTensorForChunk)
354 : {
355 30 : if (tensor.GetAddress() == nullptr) {
356 6 : IDE_LOGE("tensor address is null");
357 6 : return ADUMP_FAILED;
358 : }
359 :
360 24 : size_t remainSize = tensor.GetSize();
361 24 : size_t srcOffset = 0;
362 24 : int32_t flushRet = ADUMP_SUCCESS;
363 42 : while (remainSize > 0) {
364 24 : size_t space = DUMP_SLICE_SIZE - ctx.offset;
365 24 : if (space == 0) {
366 3 : flushRet = FlushCurrentChunk(ctx, 0);
367 3 : if (flushRet != ADUMP_SUCCESS) {
368 0 : IDE_LOGE("FlushCurrentChunk failed, ret: %d", flushRet);
369 6 : return flushRet;
370 : }
371 3 : space = DUMP_SLICE_SIZE;
372 : }
373 :
374 24 : size_t copySize = std::min(space, remainSize);
375 : void* hostData =
376 24 : DumpMemory::CopyDeviceToHost(static_cast<const char*>(tensor.GetAddress()) + srcOffset, copySize);
377 24 : if (hostData == nullptr) {
378 6 : IDE_LOGE("CopyDeviceToHost failed, size: %zu", copySize);
379 6 : return ADUMP_FAILED;
380 : }
381 :
382 18 : errno_t ret = memcpy_s(ctx.buffer.data() + ctx.offset, space, hostData, copySize);
383 36 : HOST_RT_MEMORY_GUARD(hostData);
384 18 : if (ret != EOK) {
385 0 : IDE_LOGE("memcpy_s failed, ret: %d", ret);
386 0 : return ADUMP_FAILED;
387 : }
388 :
389 18 : ctx.offset += copySize;
390 18 : srcOffset += copySize;
391 18 : remainSize -= copySize;
392 :
393 18 : bool isLastChunk = (remainSize == 0) && isLastTensorForChunk;
394 18 : if (ctx.offset >= DUMP_SLICE_SIZE) {
395 0 : flushRet = FlushCurrentChunk(ctx, isLastChunk ? 1 : 0);
396 0 : if (flushRet != ADUMP_SUCCESS) {
397 0 : IDE_LOGE("FlushCurrentChunk failed, ret: %d", flushRet);
398 0 : return flushRet;
399 : }
400 : }
401 18 : }
402 18 : return ADUMP_SUCCESS;
403 : }
404 :
405 45 : int32_t CopyTensorsWithChunking(const std::vector<DumpTensor>& tensors, ChunkContext& ctx, bool isLastTensorList)
406 : {
407 45 : size_t tensorCount = tensors.size();
408 60 : for (size_t i = 0; i < tensorCount; ++i) {
409 21 : bool isLastTensorForChunk = isLastTensorList && (i == tensorCount - 1);
410 21 : int ret = CopyTensorDataWithFlush(tensors[i], ctx, isLastTensorForChunk);
411 21 : if (ret != ADUMP_SUCCESS) {
412 6 : IDE_LOGE("CopyTensorDataWithFlush failed, ret: %d", ret);
413 6 : return ret;
414 : }
415 : }
416 39 : return ADUMP_SUCCESS;
417 : }
418 :
419 21 : void DumpTensorToQueue(DumpStreamInfo* dumpInfoPtr)
420 : {
421 21 : if (dumpInfoPtr == nullptr) {
422 3 : IDE_LOGE("dumpInfoPtr is nullptr");
423 3 : return;
424 : }
425 :
426 18 : std::string fileName = GenerateDumpFileName(dumpInfoPtr);
427 18 : toolkit::dump::DumpData dumpData = BuildDumpDataProto(dumpInfoPtr);
428 :
429 18 : uint64_t protoSize = dumpData.ByteSizeLong();
430 18 : if (protoSize == 0 || protoSize > DUMP_SLICE_SIZE) {
431 0 : IDE_LOGW("%s protobuf size invalid: %lu", fileName.c_str(), protoSize);
432 0 : return;
433 : }
434 :
435 18 : std::vector<char> chunkBuffer(DUMP_SLICE_SIZE);
436 18 : size_t currentOffset = 0;
437 :
438 18 : *(reinterpret_cast<uint64_t*>(chunkBuffer.data() + currentOffset)) = protoSize;
439 18 : currentOffset += sizeof(uint64_t);
440 :
441 18 : if (!dumpData.SerializeToArray(chunkBuffer.data() + currentOffset, static_cast<int32_t>(protoSize))) {
442 0 : IDE_LOGE("SerializeToArray failed");
443 0 : return;
444 : }
445 18 : currentOffset += protoSize;
446 :
447 18 : ChunkContext ctx{chunkBuffer, currentOffset, fileName};
448 :
449 18 : size_t inputTensorSize = 0;
450 27 : for (size_t i = 0; i < dumpInfoPtr->inputTensors.size(); i++) {
451 9 : inputTensorSize += dumpInfoPtr->inputTensors[i].GetSize();
452 : }
453 :
454 18 : size_t outputTensorSize = 0;
455 24 : for (size_t i = 0; i < dumpInfoPtr->outputTensors.size(); i++) {
456 6 : outputTensorSize += dumpInfoPtr->outputTensors[i].GetSize();
457 : }
458 :
459 18 : if (CopyTensorsWithChunking(dumpInfoPtr->inputTensors, ctx, (outputTensorSize == 0)) != ADUMP_SUCCESS) {
460 0 : IDE_LOGE("%s copy input tensors failed", fileName.c_str());
461 0 : return;
462 : }
463 :
464 18 : if (CopyTensorsWithChunking(dumpInfoPtr->outputTensors, ctx, true) != ADUMP_SUCCESS) {
465 0 : IDE_LOGE("%s copy output tensors failed", fileName.c_str());
466 0 : return;
467 : }
468 :
469 18 : if (ctx.offset > 0) {
470 18 : (void)FlushCurrentChunk(ctx, 1);
471 : }
472 :
473 18 : IDE_LOGI("%s dump success, total size: %zu", fileName.c_str(),
474 : (sizeof(uint64_t) + protoSize + inputTensorSize + outputTensorSize));
475 18 : }
476 :
477 15 : int32_t CollectStreamContextInfo(
478 : aclrtStream mainStream, const std::string& opName, const std::string& opType, uint32_t& streamId, uint32_t& taskId,
479 : uint32_t& deviceId, std::string& dumpPath)
480 : {
481 15 : rtError_t ret = rtsStreamGetId(mainStream, reinterpret_cast<int32_t*>(&streamId));
482 15 : IDE_CTRL_VALUE_FAILED(
483 : (ret == RT_ERROR_NONE), return ADUMP_FAILED, "%s(%s) dump data : get main stream id failed, ret: %d",
484 : opName.c_str(), opType.c_str(), ret);
485 :
486 12 : ret = rtsGetThreadLastTaskId(&taskId);
487 12 : IDE_CTRL_VALUE_FAILED(
488 : (ret == RT_ERROR_NONE), return ADUMP_FAILED, "%s(%s) dump data : get task id failed, ret: %d", opName.c_str(),
489 : opType.c_str(), ret);
490 :
491 9 : int32_t deviceIdTmp = 0;
492 9 : ret = rtGetDevice(&deviceIdTmp);
493 9 : IDE_CTRL_VALUE_FAILED(
494 : (ret == RT_ERROR_NONE), return ADUMP_FAILED, "%s(%s) dump data : get device id failed, ret: %d", opName.c_str(),
495 : opType.c_str(), ret);
496 6 : deviceId = static_cast<uint32_t>(deviceIdTmp);
497 :
498 6 : dumpPath = DumpManager::Instance().GetDumpSetting().GetDumpPath();
499 6 : if (dumpPath.empty()) {
500 3 : IDE_LOGE("%s(%s) dump data : get dump path failed", opName.c_str(), opType.c_str());
501 3 : return ADUMP_FAILED;
502 : }
503 3 : return ADUMP_SUCCESS;
504 : }
505 :
506 12 : void DumpDataRecordInCaptureStream(void* fnArgs)
507 : {
508 12 : if (fnArgs == nullptr) {
509 3 : IDE_LOGE("create dump stream failed");
510 3 : return;
511 : }
512 :
513 9 : std::unique_ptr<std::shared_ptr<DumpStreamInfo>> callbackArg(static_cast<std::shared_ptr<DumpStreamInfo>*>(fnArgs));
514 9 : if (callbackArg == nullptr) {
515 0 : IDE_LOGE("callbackArg is nullptr");
516 0 : return;
517 : }
518 :
519 9 : std::shared_ptr<DumpStreamInfo> args = *callbackArg;
520 9 : if (args == nullptr) {
521 0 : IDE_LOGE("args is nullptr");
522 0 : return;
523 : }
524 :
525 9 : IDE_LOGI("%s input tensor size : %d, output tensor size : %d",
526 : args->opName.c_str(), args->inputTensors.size(), args->outputTensors.size());
527 9 : DumpTensorToQueue(args.get());
528 :
529 9 : DumpResourceSafeMap::Instance().EnqueueCleanup(args->mainStreamKey);
530 9 : }
531 :
532 12 : int32_t SetupAsyncDump(
533 : std::shared_ptr<DumpStreamInfo> dumpInfoPtr, const std::string& opName, const std::string& opType,
534 : aclrtStream mainStream)
535 : {
536 12 : rtError_t ret = rtEventRecord(dumpInfoPtr->mainStmEvt, mainStream);
537 12 : IDE_CTRL_VALUE_FAILED(
538 : ret == RT_ERROR_NONE, return ADUMP_FAILED, "%s(%s) main stream (%u) record event failed, ret: %d", opName.c_str(),
539 : opType.c_str(), dumpInfoPtr->streamId, ret);
540 :
541 9 : ret = rtStreamWaitEvent(dumpInfoPtr->stm, dumpInfoPtr->mainStmEvt);
542 9 : IDE_CTRL_VALUE_FAILED(
543 : ret == RT_ERROR_NONE, return ADUMP_FAILED, "%s(%s) dump stream (%u) wait event failed, ret: %d", opName.c_str(),
544 : opType.c_str(), dumpInfoPtr->dumpStmId, ret);
545 :
546 : // 创建指向 shared_ptr 的指针，确保DumpStreamInfo的引用计数不为0, 并通过unique_ptr来保证指针释放
547 6 : auto callbackArg = std::make_unique<std::shared_ptr<DumpStreamInfo>>(dumpInfoPtr);
548 : // 提前 release 避免与回调争抢所有权
549 6 : auto* rawContext = callbackArg.release();
550 6 : ret = rtsLaunchHostFunc(
551 6 : dumpInfoPtr->stm, reinterpret_cast<rtCallback_t>(DumpDataRecordInCaptureStream), (void*)rawContext);
552 6 : if (ret != RT_ERROR_NONE) {
553 3 : IDE_LOGE("%s(%s) launch host function register failed in dump stream (%u), ret: %d", opName.c_str(),
554 : opType.c_str(), dumpInfoPtr->dumpStmId, ret);
555 3 : delete rawContext;
556 3 : return ADUMP_FAILED;
557 : }
558 :
559 3 : ret = rtEventRecord(dumpInfoPtr->dumpStmEvt, dumpInfoPtr->stm);
560 3 : IDE_CTRL_VALUE_FAILED(
561 : ret == RT_ERROR_NONE, return ADUMP_FAILED, "%s(%s) dump stream (%u) record event failed, ret: %d", opName.c_str(),
562 : opType.c_str(), dumpInfoPtr->dumpStmId, ret);
563 :
564 3 : ret = rtStreamWaitEvent(mainStream, dumpInfoPtr->dumpStmEvt);
565 3 : IDE_CTRL_VALUE_FAILED(
566 : ret == RT_ERROR_NONE, return ADUMP_FAILED, "%s(%s) main stream (%u) wait event failed, ret: %d", opName.c_str(),
567 : opType.c_str(), dumpInfoPtr->streamId, ret);
568 :
569 3 : return ADUMP_SUCCESS;
570 6 : }
571 :
572 24 : int32_t GetDumpInfoFromMap(DumpInfoParams& params)
573 : {
574 24 : auto it = DumpResourceSafeMap::Instance().get(params.mainStreamKey);
575 24 : if (it != nullptr) {
576 6 : return ADUMP_SUCCESS;
577 : }
578 :
579 18 : DumpStreamInfo* dumpPtr = nullptr;
580 18 : int32_t ret = DumpStreamCreate(&dumpPtr);
581 18 : std::shared_ptr<DumpStreamInfo> dumpInfo(dumpPtr, DumpStreamFree);
582 18 : if (ret != ADUMP_SUCCESS) {
583 3 : IDE_LOGE("ceate dump info error, ret : %d", ret);
584 3 : return ADUMP_FAILED;
585 : }
586 15 : dumpPtr->mainStreamKey = params.mainStreamKey;
587 15 : dumpPtr->opType = params.opType;
588 15 : dumpPtr->opName = params.opName;
589 15 : dumpPtr->streamId = params.streamId;
590 15 : dumpPtr->taskId = params.taskId;
591 15 : dumpPtr->deviceId = params.deviceId;
592 15 : dumpPtr->contextId = params.contextId;
593 15 : dumpPtr->threadId = params.threadId;
594 15 : dumpPtr->timestamp = SysUtils::GetTimestamp();
595 15 : dumpPtr->dumpNumber = GetNextDumpNumber();
596 15 : dumpPtr->dumpPath = params.dumpPath;
597 15 : uint32_t dumpMode = DumpManager::Instance().GetDumpSetting().GetDumpMode();
598 15 : if ((dumpMode & DUMP_MODE_INPUT) != 0) {
599 18 : for (const auto& tensorInfo : params.inputTensors) {
600 3 : dumpPtr->inputTensors.emplace_back(tensorInfo);
601 : }
602 : }
603 :
604 15 : if ((dumpMode & DUMP_MODE_OUTPUT) != 0) {
605 18 : for (const auto& tensorInfo : params.outputTensors) {
606 3 : dumpPtr->outputTensors.emplace_back(tensorInfo);
607 : }
608 : }
609 15 : DumpResourceSafeMap::Instance().insert(params.mainStreamKey, dumpInfo);
610 15 : return ADUMP_SUCCESS;
611 24 : }
612 :
613 : } // namespace Adx
|
