Line data Source code
1 : /**
2 : * Copyright (c) 2026 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "dump_stream_info.h"
12 : #include "dump_manager.h"
13 : #include "adx_dump_record.h"
14 : #include "common_utils.h"
15 : #include "adump/adx_datadump_callback.h"
16 : #include "adx_msg_proto.h"
17 : #include "memory_utils.h"
18 : #include "sys_utils.h"
19 : #include "dump_datatype.h"
20 :
21 : namespace Adx {
22 :
23 : static std::atomic<uint64_t> g_dumpNumber(0);
24 :
25 48 : uint64_t GetNextDumpNumber() { return g_dumpNumber.fetch_add(1); }
26 :
27 3 : void DumpResourceSafeMap::WaitInterval(uint32_t intervalSec)
28 : {
29 3 : std::this_thread::sleep_for(std::chrono::seconds(intervalSec));
30 3 : }
31 :
32 36 : void DumpResourceSafeMap::CleanupThreadLoop()
33 : {
34 36 : IDE_LOGI("Cleanup thread started");
35 81 : while (cleanupThreadActive_.load()) {
36 75 : std::string key;
37 : {
38 75 : std::unique_lock<std::mutex> lock(cleanupMtx_);
39 183 : cleanupCv_.wait(lock, [this]() { return !cleanupQueue_.empty() || !cleanupThreadActive_.load(); });
40 :
41 75 : if (cleanupQueue_.empty() && !cleanupThreadActive_.load()) {
42 30 : IDE_LOGI("Cleanup thread exiting");
43 30 : return;
44 : }
45 :
46 45 : if (!cleanupQueue_.empty()) {
47 45 : key = cleanupQueue_.front();
48 45 : cleanupQueue_.pop();
49 : }
50 75 : }
51 :
52 45 : if (!key.empty()) {
53 45 : IDE_LOGI("Cleanup thread removing key: %s", key.c_str());
54 45 : remove(key);
55 : }
56 75 : }
57 : }
58 :
59 45 : void DumpResourceSafeMap::StartCleanupThread()
60 : {
61 45 : if (cleanupThreadActive_.load()) {
62 9 : return;
63 : }
64 :
65 36 : std::lock_guard<std::mutex> lock(cleanupMtx_);
66 36 : if (cleanupThreadActive_.load()) {
67 0 : return;
68 : }
69 36 : cleanupThreadActive_.store(true);
70 36 : cleanupThread_ = std::thread(&DumpResourceSafeMap::CleanupThreadLoop, this);
71 36 : IDE_LOGI("Cleanup thread started");
72 36 : }
73 :
74 219 : void DumpResourceSafeMap::StopCleanupThread()
75 : {
76 : {
77 219 : std::lock_guard<std::mutex> lock(cleanupMtx_);
78 219 : if (!cleanupThreadActive_.load()) {
79 183 : return;
80 : }
81 36 : cleanupThreadActive_.store(false);
82 219 : }
83 36 : cleanupCv_.notify_one();
84 :
85 36 : if (cleanupThread_.joinable()) {
86 36 : cleanupThread_.join();
87 : }
88 36 : IDE_LOGI("Cleanup thread stopped");
89 : }
90 :
91 45 : void DumpResourceSafeMap::EnqueueCleanup(const std::string key)
92 : {
93 45 : StartCleanupThread();
94 : {
95 45 : std::lock_guard<std::mutex> lock(cleanupMtx_);
96 45 : cleanupQueue_.push(key);
97 45 : }
98 45 : cleanupCv_.notify_one();
99 45 : IDE_LOGI("Enqueued key for cleanup: %s", key.c_str());
100 45 : }
101 :
102 21 : bool DumpResourceSafeMap::IsCleanupThreadActive() { return cleanupThreadActive_.load(); }
103 :
104 132 : int32_t DumpStreamCreate(DumpStreamInfo** ptr)
105 : {
106 132 : if (ptr == nullptr) {
107 3 : IDE_LOGE("Dump stream create failed, ptr is null");
108 3 : return ADUMP_FAILED;
109 : }
110 :
111 129 : DumpStreamInfo* dumpPtr = new (std::nothrow) DumpStreamInfo();
112 129 : if (dumpPtr == nullptr) {
113 0 : IDE_LOGE("Dump stream malloc failed");
114 0 : *ptr = nullptr;
115 0 : return ADUMP_FAILED;
116 : }
117 :
118 129 : rtError_t ret = rtEventCreateExWithFlag(&(dumpPtr->mainStmEvt), RT_EVENT_DDSYNC_NS);
119 129 : if (ret != RT_ERROR_NONE) {
120 6 : IDE_LOGE("create main stream event failed");
121 6 : delete dumpPtr;
122 6 : return ret;
123 : }
124 :
125 123 : ret = rtEventCreateExWithFlag(&(dumpPtr->dumpStmEvt), RT_EVENT_DDSYNC_NS);
126 123 : if (ret != RT_ERROR_NONE) {
127 3 : IDE_LOGE("create dump stream event failed");
128 3 : rtEventDestroy(dumpPtr->mainStmEvt);
129 3 : delete dumpPtr;
130 3 : return ret;
131 : }
132 :
133 120 : ret = rtCtxGetCurrent(&(dumpPtr->ctx));
134 120 : if (ret != RT_ERROR_NONE) {
135 0 : IDE_LOGE("get current context failed, ret: %d", ret);
136 0 : rtEventDestroy(dumpPtr->mainStmEvt);
137 0 : rtEventDestroy(dumpPtr->dumpStmEvt);
138 0 : delete dumpPtr; // stm 尚未创建无需释放
139 0 : return ret;
140 : }
141 :
142 120 : ret = rtStreamCreate(&(dumpPtr->stm), 0);
143 120 : if (ret != RT_ERROR_NONE) {
144 3 : IDE_LOGE("create dump stream failed");
145 3 : rtEventDestroy(dumpPtr->mainStmEvt);
146 3 : rtEventDestroy(dumpPtr->dumpStmEvt);
147 3 : delete dumpPtr;
148 3 : return ret;
149 : }
150 :
151 117 : dumpPtr->dumpStmId = 0;
152 117 : ret = rtGetStreamId(dumpPtr->stm, reinterpret_cast<int32_t*>(&(dumpPtr->dumpStmId)));
153 117 : if (ret != RT_ERROR_NONE) {
154 3 : IDE_LOGE("get dump stream id failed, ret: %d", ret);
155 3 : rtEventDestroy(dumpPtr->mainStmEvt);
156 3 : rtEventDestroy(dumpPtr->dumpStmEvt);
157 3 : rtStreamDestroy(dumpPtr->stm);
158 3 : delete dumpPtr;
159 3 : return ret;
160 : }
161 :
162 114 : *ptr = dumpPtr;
163 114 : return ADUMP_SUCCESS;
164 : }
165 :
166 120 : void DumpStreamFree(DumpStreamInfo* ptr)
167 : {
168 120 : if (!ptr) {
169 6 : return;
170 : }
171 114 : if (ptr->mainStmEvt) {
172 114 : rtEventDestroy(ptr->mainStmEvt);
173 : }
174 114 : if (ptr->dumpStmEvt) {
175 114 : rtEventDestroy(ptr->dumpStmEvt);
176 : }
177 114 : if (ptr->stm) {
178 114 : if (ptr->ctx != nullptr) {
179 114 : rtError_t ret = rtCtxSetCurrent(ptr->ctx);
180 114 : if (ret != RT_ERROR_NONE) {
181 0 : IDE_LOGE("set context failed before stream destroy, ret: %d", ret);
182 0 : return;
183 : }
184 : }
185 114 : rtStreamDestroy(ptr->stm);
186 114 : ptr->stm = nullptr;
187 114 : ptr->ctx = nullptr; // 防止悬空
188 : }
189 114 : ptr->inputTensors.clear();
190 114 : ptr->outputTensors.clear();
191 114 : delete ptr;
192 : }
193 :
194 27 : std::string GenerateDumpFileName(const DumpStreamInfo* dumpInfoPtr)
195 : {
196 27 : const std::string& dumpPath = dumpInfoPtr->dumpPath;
197 27 : const std::string& opType = dumpInfoPtr->opType;
198 27 : const std::string& opName = dumpInfoPtr->opName;
199 27 : uint32_t taskId = dumpInfoPtr->taskId;
200 27 : uint32_t streamId = dumpInfoPtr->streamId;
201 27 : uint32_t contextId = dumpInfoPtr->contextId;
202 27 : uint32_t threadId = dumpInfoPtr->threadId;
203 27 : uint32_t deviceId = dumpInfoPtr->deviceId;
204 27 : uint64_t timestamp = dumpInfoPtr->timestamp;
205 27 : uint64_t dumpNumber = dumpInfoPtr->dumpNumber;
206 :
207 27 : std::ostringstream fileNameoss;
208 27 : if (!dumpPath.empty() && dumpPath.back() == '/') {
209 6 : fileNameoss << dumpPath << deviceId << "/" << opType << "." << opName << "." << dumpNumber << "." << taskId
210 6 : << "." << streamId << "." << timestamp;
211 : } else {
212 21 : fileNameoss << dumpPath << "/" << deviceId << "/" << opType << "." << opName << "." << dumpNumber << "."
213 21 : << taskId << "." << streamId << "." << timestamp;
214 : }
215 27 : if (contextId != 0 && threadId != 0) {
216 3 : fileNameoss << ".FFTSPLUS." << contextId << "." << threadId << "." << deviceId;
217 : }
218 54 : return fileNameoss.str();
219 27 : }
220 :
221 57 : void FillTensorProtoInfo(const std::vector<DumpTensor>& tensors, toolkit::dump::DumpData& data, bool isInput)
222 : {
223 84 : for (size_t i = 0; i < tensors.size(); i++) {
224 27 : const DumpTensor& item = tensors[i];
225 27 : int32_t format = item.GetFormat();
226 :
227 27 : if (isInput) {
228 15 : auto* opInput = data.add_input();
229 : // Convert data type using DumpDataType helper
230 30 : opInput->set_data_type(static_cast<toolkit::dump::OutputDataType>(
231 15 : DumpDataType::GetIrDataType(static_cast<GeDataType>(item.GetDataType()))));
232 15 : opInput->set_format(static_cast<toolkit::dump::OutputFormat>(GetPrimaryFormat(format)));
233 15 : opInput->set_sub_format(GetSubFormat(format));
234 : // Address for input
235 15 : opInput->set_address(reinterpret_cast<uint64_t>(item.GetAddress()));
236 15 : opInput->set_offset(item.GetArgsOffSet());
237 15 : opInput->set_size(item.GetSize());
238 :
239 15 : auto* shape = opInput->mutable_shape();
240 45 : for (auto dim : item.GetShape()) {
241 15 : shape->add_dim(dim);
242 : }
243 15 : auto* originShape = opInput->mutable_original_shape();
244 45 : for (auto dim : item.GetOriginShape()) {
245 15 : originShape->add_dim(dim);
246 : }
247 : } else {
248 12 : auto* opOutput = data.add_output();
249 : // Convert data type using DumpDataType helper
250 24 : opOutput->set_data_type(static_cast<toolkit::dump::OutputDataType>(
251 12 : DumpDataType::GetIrDataType(static_cast<GeDataType>(item.GetDataType()))));
252 12 : opOutput->set_format(static_cast<toolkit::dump::OutputFormat>(GetPrimaryFormat(format)));
253 12 : opOutput->set_sub_format(GetSubFormat(format));
254 : // Offset and address for output
255 12 : opOutput->set_offset(item.GetArgsOffSet());
256 12 : opOutput->set_address(reinterpret_cast<uint64_t>(item.GetAddress()));
257 12 : opOutput->set_size(item.GetSize());
258 :
259 12 : auto* shape = opOutput->mutable_shape();
260 36 : for (auto dim : item.GetShape()) {
261 12 : shape->add_dim(dim);
262 : }
263 12 : auto* originShape = opOutput->mutable_original_shape();
264 36 : for (auto dim : item.GetOriginShape()) {
265 12 : originShape->add_dim(dim);
266 : }
267 : }
268 : }
269 57 : }
270 :
271 24 : toolkit::dump::DumpData BuildDumpDataProto(const DumpStreamInfo* dumpInfoPtr)
272 : {
273 24 : toolkit::dump::DumpData dumpData;
274 : dumpData.set_version("2.0");
275 24 : dumpData.set_dump_time(dumpInfoPtr->timestamp);
276 24 : dumpData.set_op_name(dumpInfoPtr->opName);
277 :
278 24 : FillTensorProtoInfo(dumpInfoPtr->inputTensors, dumpData, true);
279 24 : FillTensorProtoInfo(dumpInfoPtr->outputTensors, dumpData, false);
280 :
281 24 : return dumpData;
282 0 : }
283 :
284 6 : size_t CalculateTensorDataSize(const std::vector<DumpTensor>& tensors)
285 : {
286 6 : size_t totalSize = 0;
287 21 : for (const auto& tensor : tensors) {
288 9 : totalSize += tensor.GetSize();
289 : }
290 6 : return totalSize;
291 : }
292 :
293 33 : int32_t DumpTensorPushToDumpQueue(
294 : void* dataBuf, uint32_t bufLen, const char* fileName, uint64_t offset, uint32_t isLastChunk)
295 : {
296 : int err;
297 33 : uint32_t dataLen = 0;
298 33 : IDE_RETURN_IF_CHECK_ASSIGN_32U_ADD(sizeof(DumpChunk), bufLen, dataLen, return IDE_DAEMON_INTERGER_REVERSED_ERROR);
299 33 : MsgProto* msg = AdxMsgProto::CreateMsgPacket(IDE_DUMP_REQ, 0, nullptr, dataLen);
300 33 : IDE_CTRL_VALUE_FAILED(msg != nullptr, return IDE_DAEMON_MALLOC_ERROR, "create message failed");
301 33 : SharedPtr<MsgProto> sendDataMsgPtr(msg, IdeXfree);
302 33 : msg = nullptr;
303 33 : DumpChunk* data = reinterpret_cast<DumpChunk*>(sendDataMsgPtr->data);
304 33 : err = strcpy_s(data->fileName, IDE_MAX_FILE_PATH, fileName);
305 33 : IDE_CTRL_VALUE_FAILED(err == EOK, return IDE_DAEMON_INVALID_PATH_ERROR, "copy file name failed");
306 33 : data->bufLen = bufLen;
307 33 : data->flag = 0;
308 33 : data->isLastChunk = isLastChunk;
309 33 : data->offset = static_cast<int64_t>(offset);
310 33 : IDE_LOGI(
311 : "dataLen: %u, bufLen: %u, flag: %d, isLastChunk: %u, offset: %ld, fileName: %s", dataLen, data->bufLen,
312 : data->flag, data->isLastChunk, data->offset, data->fileName);
313 33 : err = memcpy_s(data->dataBuf, data->bufLen, dataBuf, bufLen);
314 33 : IDE_CTRL_VALUE_FAILED(err == EOK, return IDE_DAEMON_UNKNOW_ERROR, "memcpy_s data buffer failed");
315 33 : HostDumpDataInfo dataInfo = {sendDataMsgPtr, dataLen};
316 33 : if (!AdxDumpRecord::Instance().RecordDumpDataToQueue(dataInfo)) {
317 6 : IDE_LOGW("dump data queue full");
318 6 : return ADUMP_FAILED;
319 : }
320 27 : IDE_LOGD("dump data process normal");
321 27 : return ADUMP_SUCCESS;
322 33 : }
323 :
324 30 : int32_t FlushCurrentChunk(ChunkContext& ctx, uint32_t isLastChunk)
325 : {
326 30 : if (ctx.offset == 0) {
327 3 : return ADUMP_SUCCESS;
328 : }
329 27 : int32_t ret = DumpTensorPushToDumpQueue(
330 27 : ctx.buffer.data(), static_cast<uint32_t>(ctx.offset), ctx.fileName.c_str(), -1, isLastChunk);
331 27 : if (ret != ADUMP_SUCCESS) {
332 3 : IDE_LOGW(
333 : "DumpTensorPushToDumpQueue failed, ret: %d, fileName: %s, offset: %zu", ret, ctx.fileName.c_str(),
334 : ctx.offset);
335 : }
336 :
337 27 : ctx.offset = 0;
338 27 : errno_t memRet = memset_s(ctx.buffer.data(), DUMP_SLICE_SIZE, 0, DUMP_SLICE_SIZE);
339 27 : if (memRet != EOK) {
340 0 : IDE_LOGW("memset_s failed, ret: %d", memRet);
341 : }
342 27 : return ret;
343 : }
344 :
345 30 : int32_t CopyTensorDataWithFlush(const DumpTensor& tensor, ChunkContext& ctx, bool isLastTensorForChunk)
346 : {
347 30 : if (tensor.GetAddress() == nullptr) {
348 6 : IDE_LOGE("tensor address is null");
349 6 : return ADUMP_FAILED;
350 : }
351 :
352 24 : size_t remainSize = tensor.GetSize();
353 24 : size_t srcOffset = 0;
354 24 : int32_t flushRet = ADUMP_SUCCESS;
355 42 : while (remainSize > 0) {
356 24 : size_t space = DUMP_SLICE_SIZE - ctx.offset;
357 24 : if (space == 0) {
358 3 : flushRet = FlushCurrentChunk(ctx, 0);
359 3 : if (flushRet != ADUMP_SUCCESS) {
360 0 : IDE_LOGE("FlushCurrentChunk failed, ret: %d", flushRet);
361 6 : return flushRet;
362 : }
363 3 : space = DUMP_SLICE_SIZE;
364 : }
365 :
366 24 : size_t copySize = std::min(space, remainSize);
367 : void* hostData =
368 24 : DumpMemory::CopyDeviceToHost(static_cast<const char*>(tensor.GetAddress()) + srcOffset, copySize);
369 24 : if (hostData == nullptr) {
370 6 : IDE_LOGE("CopyDeviceToHost failed, size: %zu", copySize);
371 6 : return ADUMP_FAILED;
372 : }
373 :
374 18 : errno_t ret = memcpy_s(ctx.buffer.data() + ctx.offset, space, hostData, copySize);
375 36 : HOST_RT_MEMORY_GUARD(hostData);
376 18 : if (ret != EOK) {
377 0 : IDE_LOGE("memcpy_s failed, ret: %d", ret);
378 0 : return ADUMP_FAILED;
379 : }
380 :
381 18 : ctx.offset += copySize;
382 18 : srcOffset += copySize;
383 18 : remainSize -= copySize;
384 :
385 18 : bool isLastChunk = (remainSize == 0) && isLastTensorForChunk;
386 18 : if (ctx.offset >= DUMP_SLICE_SIZE) {
387 0 : flushRet = FlushCurrentChunk(ctx, isLastChunk ? 1 : 0);
388 0 : if (flushRet != ADUMP_SUCCESS) {
389 0 : IDE_LOGE("FlushCurrentChunk failed, ret: %d", flushRet);
390 0 : return flushRet;
391 : }
392 : }
393 18 : }
394 18 : return ADUMP_SUCCESS;
395 : }
396 :
397 45 : int32_t CopyTensorsWithChunking(const std::vector<DumpTensor>& tensors, ChunkContext& ctx, bool isLastTensorList)
398 : {
399 45 : size_t tensorCount = tensors.size();
400 60 : for (size_t i = 0; i < tensorCount; ++i) {
401 21 : bool isLastTensorForChunk = isLastTensorList && (i == tensorCount - 1);
402 21 : int ret = CopyTensorDataWithFlush(tensors[i], ctx, isLastTensorForChunk);
403 21 : if (ret != ADUMP_SUCCESS) {
404 6 : IDE_LOGE("CopyTensorDataWithFlush failed, ret: %d", ret);
405 6 : return ret;
406 : }
407 : }
408 39 : return ADUMP_SUCCESS;
409 : }
410 :
411 21 : void DumpTensorToQueue(DumpStreamInfo* dumpInfoPtr)
412 : {
413 21 : if (dumpInfoPtr == nullptr) {
414 3 : IDE_LOGE("dumpInfoPtr is nullptr");
415 3 : return;
416 : }
417 :
418 18 : std::string fileName = GenerateDumpFileName(dumpInfoPtr);
419 18 : toolkit::dump::DumpData dumpData = BuildDumpDataProto(dumpInfoPtr);
420 :
421 18 : uint64_t protoSize = dumpData.ByteSizeLong();
422 18 : if (protoSize == 0 || protoSize > DUMP_SLICE_SIZE) {
423 0 : IDE_LOGW("%s protobuf size invalid: %lu", fileName.c_str(), protoSize);
424 0 : return;
425 : }
426 :
427 18 : std::vector<char> chunkBuffer(DUMP_SLICE_SIZE);
428 18 : size_t currentOffset = 0;
429 :
430 18 : *(reinterpret_cast<uint64_t*>(chunkBuffer.data() + currentOffset)) = protoSize;
431 18 : currentOffset += sizeof(uint64_t);
432 :
433 18 : if (!dumpData.SerializeToArray(chunkBuffer.data() + currentOffset, static_cast<int32_t>(protoSize))) {
434 0 : IDE_LOGE("SerializeToArray failed");
435 0 : return;
436 : }
437 18 : currentOffset += protoSize;
438 :
439 18 : ChunkContext ctx{chunkBuffer, currentOffset, fileName};
440 :
441 18 : size_t inputTensorSize = 0;
442 27 : for (size_t i = 0; i < dumpInfoPtr->inputTensors.size(); i++) {
443 9 : inputTensorSize += dumpInfoPtr->inputTensors[i].GetSize();
444 : }
445 :
446 18 : size_t outputTensorSize = 0;
447 24 : for (size_t i = 0; i < dumpInfoPtr->outputTensors.size(); i++) {
448 6 : outputTensorSize += dumpInfoPtr->outputTensors[i].GetSize();
449 : }
450 :
451 18 : if (CopyTensorsWithChunking(dumpInfoPtr->inputTensors, ctx, (outputTensorSize == 0)) != ADUMP_SUCCESS) {
452 0 : IDE_LOGE("%s copy input tensors failed", fileName.c_str());
453 0 : return;
454 : }
455 :
456 18 : if (CopyTensorsWithChunking(dumpInfoPtr->outputTensors, ctx, true) != ADUMP_SUCCESS) {
457 0 : IDE_LOGE("%s copy output tensors failed", fileName.c_str());
458 0 : return;
459 : }
460 :
461 18 : if (ctx.offset > 0) {
462 18 : (void)FlushCurrentChunk(ctx, 1);
463 : }
464 :
465 18 : IDE_LOGI(
466 : "%s dump success, total size: %zu", fileName.c_str(),
467 : (sizeof(uint64_t) + protoSize + inputTensorSize + outputTensorSize));
468 18 : }
469 :
470 15 : int32_t CollectStreamContextInfo(
471 : aclrtStream mainStream, const std::string& opName, const std::string& opType, uint32_t& streamId, uint32_t& taskId,
472 : uint32_t& deviceId, std::string& dumpPath)
473 : {
474 15 : rtError_t ret = rtsStreamGetId(mainStream, reinterpret_cast<int32_t*>(&streamId));
475 15 : IDE_CTRL_VALUE_FAILED(
476 : (ret == RT_ERROR_NONE), return ADUMP_FAILED, "%s(%s) dump data : get main stream id failed, ret: %d",
477 : opName.c_str(), opType.c_str(), ret);
478 :
479 12 : ret = rtsGetThreadLastTaskId(&taskId);
480 12 : IDE_CTRL_VALUE_FAILED(
481 : (ret == RT_ERROR_NONE), return ADUMP_FAILED, "%s(%s) dump data : get task id failed, ret: %d", opName.c_str(),
482 : opType.c_str(), ret);
483 :
484 9 : int32_t deviceIdTmp = 0;
485 9 : ret = rtGetDevice(&deviceIdTmp);
486 9 : IDE_CTRL_VALUE_FAILED(
487 : (ret == RT_ERROR_NONE), return ADUMP_FAILED, "%s(%s) dump data : get device id failed, ret: %d", opName.c_str(),
488 : opType.c_str(), ret);
489 6 : deviceId = static_cast<uint32_t>(deviceIdTmp);
490 :
491 6 : dumpPath = DumpManager::Instance().GetDumpSetting().GetDumpPath();
492 6 : if (dumpPath.empty()) {
493 3 : IDE_LOGE("%s(%s) dump data : get dump path failed", opName.c_str(), opType.c_str());
494 3 : return ADUMP_FAILED;
495 : }
496 3 : return ADUMP_SUCCESS;
497 : }
498 :
499 12 : void DumpDataRecordInCaptureStream(void* fnArgs)
500 : {
501 12 : if (fnArgs == nullptr) {
502 3 : IDE_LOGE("create dump stream failed");
503 3 : return;
504 : }
505 :
506 9 : std::unique_ptr<std::shared_ptr<DumpStreamInfo>> callbackArg(static_cast<std::shared_ptr<DumpStreamInfo>*>(fnArgs));
507 9 : if (callbackArg == nullptr) {
508 0 : IDE_LOGE("callbackArg is nullptr");
509 0 : return;
510 : }
511 :
512 9 : std::shared_ptr<DumpStreamInfo> args = *callbackArg;
513 9 : if (args == nullptr) {
514 0 : IDE_LOGE("args is nullptr");
515 0 : return;
516 : }
517 :
518 9 : IDE_LOGI(
519 : "%s input tensor size : %d, output tensor size : %d", args->opName.c_str(), args->inputTensors.size(),
520 : args->outputTensors.size());
521 9 : DumpTensorToQueue(args.get());
522 :
523 9 : DumpResourceSafeMap::Instance().EnqueueCleanup(args->mainStreamKey);
524 9 : }
525 :
526 12 : int32_t SetupAsyncDump(
527 : std::shared_ptr<DumpStreamInfo> dumpInfoPtr, const std::string& opName, const std::string& opType,
528 : aclrtStream mainStream)
529 : {
530 12 : rtError_t ret = rtEventRecord(dumpInfoPtr->mainStmEvt, mainStream);
531 12 : IDE_CTRL_VALUE_FAILED(
532 : ret == RT_ERROR_NONE, return ADUMP_FAILED, "%s(%s) main stream (%u) record event failed, ret: %d",
533 : opName.c_str(), opType.c_str(), dumpInfoPtr->streamId, ret);
534 :
535 9 : ret = rtStreamWaitEvent(dumpInfoPtr->stm, dumpInfoPtr->mainStmEvt);
536 9 : IDE_CTRL_VALUE_FAILED(
537 : ret == RT_ERROR_NONE, return ADUMP_FAILED, "%s(%s) dump stream (%u) wait event failed, ret: %d",
538 : opName.c_str(), opType.c_str(), dumpInfoPtr->dumpStmId, ret);
539 :
540 : // 创建指向 shared_ptr 的指针，确保DumpStreamInfo的引用计数不为0, 并通过unique_ptr来保证指针释放
541 6 : auto callbackArg = std::make_unique<std::shared_ptr<DumpStreamInfo>>(dumpInfoPtr);
542 : // 提前 release 避免与回调争抢所有权
543 6 : auto* rawContext = callbackArg.release();
544 6 : ret = rtsLaunchHostFunc(
545 6 : dumpInfoPtr->stm, reinterpret_cast<rtCallback_t>(DumpDataRecordInCaptureStream), (void*)rawContext);
546 6 : if (ret != RT_ERROR_NONE) {
547 3 : IDE_LOGE(
548 : "%s(%s) launch host function register failed in dump stream (%u), ret: %d", opName.c_str(), opType.c_str(),
549 : dumpInfoPtr->dumpStmId, ret);
550 3 : delete rawContext;
551 3 : return ADUMP_FAILED;
552 : }
553 :
554 3 : ret = rtEventRecord(dumpInfoPtr->dumpStmEvt, dumpInfoPtr->stm);
555 3 : IDE_CTRL_VALUE_FAILED(
556 : ret == RT_ERROR_NONE, return ADUMP_FAILED, "%s(%s) dump stream (%u) record event failed, ret: %d",
557 : opName.c_str(), opType.c_str(), dumpInfoPtr->dumpStmId, ret);
558 :
559 3 : ret = rtStreamWaitEvent(mainStream, dumpInfoPtr->dumpStmEvt);
560 3 : IDE_CTRL_VALUE_FAILED(
561 : ret == RT_ERROR_NONE, return ADUMP_FAILED, "%s(%s) main stream (%u) wait event failed, ret: %d", opName.c_str(),
562 : opType.c_str(), dumpInfoPtr->streamId, ret);
563 :
564 3 : return ADUMP_SUCCESS;
565 6 : }
566 :
567 24 : int32_t GetDumpInfoFromMap(DumpInfoParams& params)
568 : {
569 24 : auto it = DumpResourceSafeMap::Instance().get(params.mainStreamKey);
570 24 : if (it != nullptr) {
571 6 : return ADUMP_SUCCESS;
572 : }
573 :
574 18 : DumpStreamInfo* dumpPtr = nullptr;
575 18 : int32_t ret = DumpStreamCreate(&dumpPtr);
576 18 : std::shared_ptr<DumpStreamInfo> dumpInfo(dumpPtr, DumpStreamFree);
577 18 : if (ret != ADUMP_SUCCESS) {
578 3 : IDE_LOGE("create dump info error, ret : %d", ret);
579 3 : return ADUMP_FAILED;
580 : }
581 15 : dumpPtr->mainStreamKey = params.mainStreamKey;
582 15 : dumpPtr->opType = params.opType;
583 15 : dumpPtr->opName = params.opName;
584 15 : dumpPtr->streamId = params.streamId;
585 15 : dumpPtr->taskId = params.taskId;
586 15 : dumpPtr->deviceId = params.deviceId;
587 15 : dumpPtr->contextId = params.contextId;
588 15 : dumpPtr->threadId = params.threadId;
589 15 : dumpPtr->timestamp = SysUtils::GetTimestamp();
590 15 : dumpPtr->dumpNumber = GetNextDumpNumber();
591 15 : dumpPtr->dumpPath = params.dumpPath;
592 15 : uint32_t dumpMode = DumpManager::Instance().GetDumpSetting().GetDumpMode();
593 15 : if ((dumpMode & DUMP_MODE_INPUT) != 0) {
594 33 : for (const auto& tensorInfo : params.inputTensors) {
595 3 : dumpPtr->inputTensors.emplace_back(tensorInfo);
596 : }
597 : }
598 :
599 15 : if ((dumpMode & DUMP_MODE_OUTPUT) != 0) {
600 33 : for (const auto& tensorInfo : params.outputTensors) {
601 3 : dumpPtr->outputTensors.emplace_back(tensorInfo);
602 : }
603 : }
604 15 : DumpResourceSafeMap::Instance().insert(params.mainStreamKey, dumpInfo);
605 15 : return ADUMP_SUCCESS;
606 24 : }
607 :
608 : } // namespace Adx
|
