Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 : #include "dump_task.h"
11 :
12 : #include <algorithm>
13 : #include <regex>
14 : #include <sstream>
15 : #include <vector>
16 : #include <map>
17 : #include <dlfcn.h>
18 : #include "aicpusd_drv_manager.h"
19 : #include "aicpusd_status.h"
20 : #include "aicpusd_monitor.h"
21 : #include "aicpusd_model_execute.h"
22 : #include "common/aicpusd_util.h"
23 : #include "securec.h"
24 : #include "metadef_types.h"
25 : #include "aicpusd_hal_interface_ref.h"
26 : #include "aicpusd_common.h"
27 :
28 : namespace AicpuSchedule {
29 : const size_t INTERNAL_STEP_SIZE = 2U;
30 : const uint32_t VERSION_0 = 0U;
31 : constexpr uint32_t OVERFLOW_DUMP_BIT = 0x04;
32 : constexpr uint32_t STATS_DUMP_BIT = 0x02;
33 : constexpr uint32_t TENSOR_DUMP_BIT = 0x01;
34 : constexpr int32_t MAX_TASK_SIZE = 2;
35 :
36 116 : OpDumpTaskManager &OpDumpTaskManager::GetInstance()
37 : {
38 116 : static OpDumpTaskManager instance;
39 116 : return instance;
40 : }
41 :
42 68 : void OpDumpTaskManager::GetOptionalParam(const aicpu::dump::OpMappingInfo &opMappingInfo,
43 : MappingInfoOptionalParam &optionalParam) const
44 : {
45 68 : if (opMappingInfo.model_name_param_case() == aicpu::dump::OpMappingInfo::kModelName) {
46 53 : optionalParam.modelName = opMappingInfo.model_name();
47 53 : ReplaceStringElem(optionalParam.modelName);
48 53 : optionalParam.hasModelName = true;
49 : }
50 68 : if (opMappingInfo.model_id_param_case() == aicpu::dump::OpMappingInfo::kModelId) {
51 53 : optionalParam.modelId = opMappingInfo.model_id();
52 53 : optionalParam.hasModelId = true;
53 : }
54 68 : if (opMappingInfo.step_id_case() == aicpu::dump::OpMappingInfo::kStepIdAddr) {
55 49 : optionalParam.stepIdAddr =
56 49 : PtrToPtr<void, uint64_t>(ValueToPtr(opMappingInfo.step_id_addr()));
57 49 : optionalParam.hasStepId = true;
58 : }
59 68 : if (opMappingInfo.iterations_per_loop_case() == aicpu::dump::OpMappingInfo::kIterationsPerLoopAddr) {
60 41 : optionalParam.iterationsPerLoopAddr =
61 41 : PtrToPtr<void, uint64_t>(ValueToPtr(opMappingInfo.iterations_per_loop_addr()));
62 41 : optionalParam.hasIterationsPerLoop = true;
63 : }
64 68 : if (opMappingInfo.loop_cond_case() == aicpu::dump::OpMappingInfo::kLoopCondAddr) {
65 41 : optionalParam.loopCondAddr =
66 41 : PtrToPtr<void, uint64_t>(ValueToPtr(opMappingInfo.loop_cond_addr()));
67 41 : optionalParam.hasLoopCond = true;
68 : }
69 68 : if (opMappingInfo.dump_switch_case() == aicpu::dump::OpMappingInfo::kDumpSwitchAddr) {
70 4 : optionalParam.dumpSwitchAddr =
71 4 : PtrToPtr<void, uint64_t>(ValueToPtr(opMappingInfo.dump_switch_addr()));
72 4 : optionalParam.hasDumpSwitch = true;
73 : }
74 68 : }
75 45 : int32_t OpDumpTaskManager::CreateOpDumpTask(std::shared_ptr<OpDumpTask> &opDumpTaskPtr,
76 : const int32_t hostPid, const uint32_t deviceId) const
77 : {
78 45 : DATADUMP_MAKE_SHARED(opDumpTaskPtr = std::make_shared<OpDumpTask>(hostPid, deviceId),
79 : return AICPU_SCHEDULE_ERROR_DUMP_FAILED);
80 45 : return AICPU_SCHEDULE_OK;
81 : }
82 :
83 42 : int32_t OpDumpTaskManager::Load(const aicpu::dump::OpMappingInfo &opMappingInfo, AicpuSqeAdapter &aicpuSqeAdapter)
84 : {
85 42 : const std::string dumpPath = opMappingInfo.dump_path();
86 42 : MappingInfoOptionalParam optionalParam;
87 42 : GetOptionalParam(opMappingInfo, optionalParam);
88 42 : const std::string dumpStepStr = opMappingInfo.dump_step();
89 :
90 42 : uint32_t streamId = INVALID_VAL;
91 42 : uint32_t taskId = INVALID_VAL;
92 42 : if (!aicpuSqeAdapter.IsAdapterInvalidParameter()) {
93 6 : AicpuSqeAdapter::AicpuDataDumpInfoLoad info {};
94 6 : aicpuSqeAdapter.GetAicpuDataDumpInfoLoad(info);
95 6 : streamId = info.stream_id;
96 6 : taskId = info.task_id;
97 : }
98 :
99 42 : DumpStep dumpStep;
100 42 : if (!GetDumpStepFromString(dumpStepStr, dumpStep)) {
101 3 : aicpusd_err("Step error[%s].", dumpStepStr.c_str());
102 3 : return AICPU_SCHEDULE_ERROR_DUMP_FAILED;
103 : }
104 39 : const int32_t hostPid = static_cast<int32_t>(AicpuSchedule::AicpuDrvManager::GetInstance().GetHostPid());
105 39 : const uint32_t deviceId = AicpuSchedule::AicpuDrvManager::GetInstance().GetDeviceId();
106 : {
107 39 : std::set<TaskInfo> tasksInfo;
108 39 : const std::lock_guard<std::mutex> mapLock(dumpTaskMapMtx_);
109 39 : int32_t ret = AICPU_SCHEDULE_OK;
110 70 : for (int32_t i = 0; i < opMappingInfo.task_size(); ++i) {
111 39 : aicpu::dump::Task task = opMappingInfo.task(i);
112 39 : aicpusd_info("Get load dump data from proto task id[%u], stream id[%u]", task.task_id(), task.stream_id());
113 39 : AicpuSqeAdapter::AicpuOpMappingDumpTaskInfo opmappingInfo(task.task_id(), task.stream_id(), taskId, streamId);
114 : const bool isOpMappingDumpTaskInfoVaild =
115 39 : aicpuSqeAdapter.IsOpMappingDumpTaskInfoVaild(opmappingInfo);
116 39 : if (!isOpMappingDumpTaskInfoVaild) {
117 2 : aicpusd_err("streamId or taskId is INVALID_VAL, task.task_id()[%u], task.stream_id()[%u], "
118 : "streamId[%u], taskId[%u]", opmappingInfo.proto_info_task_id,
119 : opmappingInfo.proto_info_stream_id, opmappingInfo.stream_id, opmappingInfo.task_id);
120 : }
121 39 : AicpuSqeAdapter::AicpuDumpTaskInfo dumpTaskInfo{};
122 39 : aicpuSqeAdapter.GetAicpuDumpTaskInfo(opmappingInfo, dumpTaskInfo);
123 39 : uint32_t contextIdTmp = INVALID_VAL;
124 39 : uint32_t threadIdTmp = INVALID_VAL;
125 39 : if (task.tasktype() == aicpu::dump::Task::FFTSPLUS) {
126 11 : const auto &contextFromMapInfo = task.context();
127 11 : uint32_t contextFromMapInfoSize = static_cast<uint32_t>(contextFromMapInfo.size());
128 16 : for (uint32_t j = 0U; j < contextFromMapInfoSize; ++j) {
129 9 : std::shared_ptr<OpDumpTask> opDumpTaskPtr = nullptr;
130 9 : ret = CreateOpDumpTask(opDumpTaskPtr, hostPid, deviceId);
131 9 : if (ret != AICPU_SCHEDULE_OK) {
132 1 : aicpusd_err("Create opDumpTask failed, hostPid[%d], deviceId[%u]", hostPid, deviceId);
133 1 : return ret;
134 : }
135 8 : aicpusd_memory_log("MallocMemory, func=new, size=%zu, purpose=data dumper", sizeof(OpDumpTask));
136 8 : if (opDumpTaskPtr == nullptr) {
137 1 : aicpusd_err("malloc memory for OpDumpTask object failed");
138 1 : return AICPU_SCHEDULE_ERROR_DUMP_FAILED;
139 : }
140 7 : DumpMode dumpMode = opMappingInfo.dump_data();
141 7 : aicpusd_info("Load Dump data is %d", static_cast<int32_t>(dumpMode));
142 7 : ret = opDumpTaskPtr->PreProcessOpMappingInfo(task,
143 : dumpPath,
144 : optionalParam,
145 : dumpStep,
146 : dumpMode);
147 7 : if (ret != AICPU_SCHEDULE_OK) {
148 : // when error occur, ge call unload op mapping info
149 2 : aicpusd_err("pre process op mapping info failed");
150 2 : return ret;
151 : }
152 5 : auto &item = contextFromMapInfo.at(j);
153 5 : opDumpTaskPtr->UpdatePreProcessFftsPlusInputAndOutput(item);
154 5 : contextIdTmp = item.context_id();
155 5 : threadIdTmp = item.thread_id();
156 5 : TaskInfo taskInfo(dumpTaskInfo.stream_id, dumpTaskInfo.task_id, contextIdTmp, threadIdTmp);
157 5 : aicpusd_info("[stream id:%u, task id:%u, context id:%u, thread id:%u]",
158 : dumpTaskInfo.stream_id, dumpTaskInfo.task_id, contextIdTmp, threadIdTmp);
159 5 : (void)dumpTaskMap_.insert(std::make_pair(taskInfo, std::move(opDumpTaskPtr)));
160 5 : (void)tasksInfo.insert(taskInfo);
161 9 : }
162 : } else {
163 28 : contextIdTmp = INVALID_VAL;
164 28 : threadIdTmp = INVALID_VAL;
165 28 : std::shared_ptr<OpDumpTask> opDumpTaskPtr = nullptr;
166 28 : ret = CreateOpDumpTask(opDumpTaskPtr, hostPid, deviceId);
167 28 : if (ret != AICPU_SCHEDULE_OK) {
168 1 : aicpusd_err("Create opDumpTask failed");
169 1 : return ret;
170 : }
171 27 : aicpusd_memory_log("MallocMemory, func=new, size=%zu, purpose=data dumper", sizeof(OpDumpTask));
172 27 : if (opDumpTaskPtr == nullptr) {
173 1 : aicpusd_err("malloc memory for OpDumpTask object failed");
174 1 : return AICPU_SCHEDULE_ERROR_DUMP_FAILED;
175 : }
176 26 : DumpMode dumpMode = opMappingInfo.dump_data();
177 26 : aicpusd_info("Load Dump data is %d", static_cast<int32_t>(dumpMode));
178 26 : ret = opDumpTaskPtr->PreProcessOpMappingInfo(task,
179 : dumpPath,
180 : optionalParam,
181 : dumpStep,
182 : dumpMode);
183 26 : if (ret != AICPU_SCHEDULE_OK) {
184 : // when error occur, ge call unload op mapping info
185 2 : aicpusd_err("pre process op mapping info failed");
186 2 : return ret;
187 : }
188 24 : TaskInfo taskInfo(dumpTaskInfo.stream_id, dumpTaskInfo.task_id, contextIdTmp, threadIdTmp);
189 24 : aicpusd_info("[stream id:%u, task id:%u, context id:%u, thread id:%u]",
190 : dumpTaskInfo.stream_id, dumpTaskInfo.task_id, contextIdTmp, threadIdTmp);
191 24 : (void)dumpTaskMap_.insert(std::make_pair(taskInfo, std::move(opDumpTaskPtr)));
192 24 : (void)tasksInfo.insert(taskInfo);
193 28 : }
194 39 : }
195 31 : if (optionalParam.hasModelId) {
196 26 : const auto iter = modelIdToTask_.find(optionalParam.modelId);
197 26 : if (iter == modelIdToTask_.end()) {
198 15 : (void)modelIdToTask_.insert(std::make_pair(optionalParam.modelId, std::move(tasksInfo)));
199 : } else {
200 20 : for (const auto item : tasksInfo) {
201 9 : (void)iter->second.insert(item);
202 : }
203 : }
204 : }
205 47 : }
206 31 : return AICPU_SCHEDULE_OK;
207 42 : }
208 :
209 27 : void OpDumpTaskManager::UnloadClearTaskInfo(const TaskInfo &dumpTaskInfo)
210 : {
211 27 : const auto range = dumpTaskMap_.equal_range(dumpTaskInfo);
212 27 : if (range.first != range.second) {
213 30 : for (auto item = range.first; item != range.second; ++item) {
214 16 : item->second->ClearBaseDumpData();
215 : }
216 : }
217 27 : (void)dumpTaskMap_.erase(dumpTaskInfo);
218 54 : return;
219 : }
220 :
221 12 : int32_t OpDumpTaskManager::Unload(const aicpu::dump::OpMappingInfo &opMappingInfo,
222 : AicpuSqeAdapter &aicpuSqeAdapter)
223 : {
224 12 : const std::lock_guard<std::mutex> mapLock(dumpTaskMapMtx_);
225 12 : uint32_t streamId = INVALID_VAL;
226 12 : uint32_t taskId = INVALID_VAL;
227 12 : if (!aicpuSqeAdapter.IsAdapterInvalidParameter()) {
228 0 : AicpuSqeAdapter::AicpuDataDumpInfoLoad info {};
229 0 : aicpuSqeAdapter.GetAicpuDataDumpInfoLoad(info);
230 0 : streamId = info.stream_id;
231 0 : taskId = info.task_id;
232 : }
233 24 : for (int32_t i = 0; i < opMappingInfo.task_size(); ++i) {
234 12 : const aicpu::dump::Task task = opMappingInfo.task(i);
235 12 : AicpuSqeAdapter::AicpuOpMappingDumpTaskInfo opmappingInfo(task.task_id(), task.stream_id(), taskId, streamId);
236 12 : AicpuSqeAdapter::AicpuDumpTaskInfo taskInfo{};
237 12 : aicpuSqeAdapter.GetAicpuDumpTaskInfo(opmappingInfo, taskInfo);
238 12 : uint32_t contextIdTmp = INVALID_VAL;
239 12 : uint32_t threadIdTmp = INVALID_VAL;
240 12 : const auto &contextFromMapInfo = task.context();
241 12 : uint32_t contextFromMapInfoSize = static_cast<uint32_t>(contextFromMapInfo.size());
242 12 : bool isFftsPlusUnload = false;
243 15 : for (uint32_t j = 0U; j < contextFromMapInfoSize; ++j) {
244 3 : auto &item = contextFromMapInfo.at(j);
245 3 : contextIdTmp = item.context_id();
246 3 : threadIdTmp = item.thread_id();
247 3 : const TaskInfo dumpTaskInfo(taskInfo.stream_id, taskInfo.task_id, contextIdTmp, threadIdTmp);
248 3 : UnloadClearTaskInfo(dumpTaskInfo);
249 3 : aicpusd_info("Unload stream id[%u], task id[%u], streamIdTmp[%u], taskIdTmp[%u],"
250 : " context id[%u], threadIdTmp[%u].", streamId, taskId, taskInfo.stream_id, taskInfo.task_id,
251 : contextIdTmp, threadIdTmp);
252 3 : isFftsPlusUnload = true;
253 : }
254 12 : if (isFftsPlusUnload == false) {
255 10 : const TaskInfo dumpTaskInfo(taskInfo.stream_id, taskInfo.task_id, contextIdTmp, threadIdTmp);
256 10 : UnloadClearTaskInfo(dumpTaskInfo);
257 10 : aicpusd_info("Unload op mapping info, stream id[%u], task id[%u], streamIdTmp[%u], taskIdTmp[%u],"
258 : " context id[%u], threadIdTmp[%u].", streamId, taskId, taskInfo.stream_id, taskInfo.task_id,
259 : contextIdTmp, threadIdTmp);
260 : }
261 12 : }
262 12 : MappingInfoOptionalParam optionalParam;
263 12 : GetOptionalParam(opMappingInfo, optionalParam);
264 12 : if (optionalParam.hasModelId) {
265 12 : const auto iter = modelIdToTask_.find(optionalParam.modelId);
266 12 : if (iter == modelIdToTask_.end()) {
267 1 : aicpusd_warn("no model id[%u], unload dump model failed", optionalParam.modelId);
268 1 : return AICPU_SCHEDULE_OK;
269 : }
270 25 : for (const auto taskInfo : iter->second) {
271 14 : UnloadClearTaskInfo(taskInfo);
272 14 : aicpusd_info("Check and clean data dump task resource, stream id[%u], task id[%u]"
273 : " context id[%u], thread id [%u].",
274 : taskInfo.streamId_, taskInfo.taskId_, taskInfo.contextId_, taskInfo.threadId_);
275 : }
276 11 : (void)modelIdToTask_.erase(iter);
277 11 : aicpusd_info("Unloaded model id[%u] successfully", optionalParam.modelId);
278 : }
279 11 : return AICPU_SCHEDULE_OK;
280 12 : }
281 :
282 33 : int32_t OpDumpTaskManager::LoadOpMappingInfo(const char_t * const infoAddr, const uint32_t len)
283 : {
284 33 : AicpuSqeAdapter aicpuSqeAdapter(0U);
285 66 : return LoadOpMappingInfo(infoAddr, len, aicpuSqeAdapter);
286 33 : }
287 :
288 60 : int32_t OpDumpTaskManager::LoadOpMappingInfo(const char_t * const infoAddr, const uint32_t len, AicpuSqeAdapter &aicpuSqeAdapter)
289 : {
290 60 : uint32_t streamId = INVALID_VAL;
291 60 : uint64_t taskId = INVALID_VAL;
292 60 : if (!aicpuSqeAdapter.IsAdapterInvalidParameter()) {
293 8 : AicpuSqeAdapter::AicpuDataDumpInfoLoad info {};
294 8 : aicpuSqeAdapter.GetAicpuDataDumpInfoLoad(info);
295 8 : streamId = info.stream_id;
296 8 : taskId = info.task_id;
297 : }
298 60 : aicpusd_info("Load op mapping info from ts, size[%u], streamId[%u], taskId[%u]", len, streamId, taskId);
299 60 : if (infoAddr == nullptr) {
300 4 : aicpusd_err("op mapping info addr is null");
301 4 : return AICPU_SCHEDULE_ERROR_DUMP_FAILED;
302 : }
303 56 : aicpu::dump::OpMappingInfo opMappingInfo;
304 56 : const std::string protoInfo(infoAddr, static_cast<uint64_t>(len));
305 56 : const bool parseRet = opMappingInfo.ParseFromString(protoInfo);
306 56 : if (!parseRet) {
307 3 : aicpusd_err("parse op mapping info failed");
308 3 : return AICPU_SCHEDULE_ERROR_DUMP_FAILED;
309 : }
310 53 : if (opMappingInfo.flag() == 0x01U) {
311 40 : return Load(opMappingInfo, aicpuSqeAdapter);
312 13 : } else if (opMappingInfo.flag() == 0x00U) {
313 12 : return Unload(opMappingInfo, aicpuSqeAdapter);
314 : } else {
315 1 : aicpusd_err("Flag [%d] invalid, allow [0x00, 0x01]", opMappingInfo.flag());
316 1 : return AICPU_SCHEDULE_ERROR_DUMP_FAILED;
317 : }
318 : return AICPU_SCHEDULE_OK;
319 56 : }
320 :
321 11 : int32_t OpDumpTaskManager::DumpOpInfo(const uint32_t streamId, const uint32_t taskId,
322 : const uint32_t streamId1, const uint32_t taskId1)
323 : {
324 11 : TaskInfoExt dumpTaskInfo;
325 11 : dumpTaskInfo.streamId_ = streamId;
326 11 : dumpTaskInfo.taskId_ = taskId;
327 11 : dumpTaskInfo.contextId_ = INVALID_VAL;
328 11 : dumpTaskInfo.threadId_ = INVALID_VAL;
329 11 : bool isDebug = ((streamId1 != INVALID_VAL) && (taskId1 != INVALID_VAL));
330 11 : uint32_t fileNameStreamId = isDebug ? streamId1 : streamId;
331 11 : uint32_t fileNameTaskId = isDebug ? taskId1 : taskId;
332 11 : DumpFileName dumpFileName(fileNameStreamId, fileNameTaskId);
333 22 : return DumpOpInfo(dumpTaskInfo, dumpFileName);
334 : }
335 :
336 5 : int32_t OpDumpTaskManager::DumpOpInfo(TaskInfoExt &dumpTaskInfo,
337 : const uint32_t streamId, const uint32_t taskId,
338 : const uint32_t contextId, const uint32_t threadId)
339 : {
340 5 : DumpFileName dumpFileName(streamId, taskId, contextId, threadId);
341 10 : return DumpOpInfo(dumpTaskInfo, dumpFileName);
342 : }
343 :
344 55 : int32_t OpDumpTaskManager::DumpOpInfo(TaskInfoExt &dumpTaskInfo,
345 : const DumpFileName &dumpFileName)
346 : {
347 : TaskInfo curTaskInfo(dumpTaskInfo.streamId_, dumpTaskInfo.taskId_,
348 55 : dumpTaskInfo.contextId_, dumpTaskInfo.threadId_);
349 55 : dumpTaskMapMtx_.lock();
350 55 : const auto range = dumpTaskMap_.equal_range(curTaskInfo);
351 55 : if (range.first == range.second) {
352 30 : dumpTaskMapMtx_.unlock();
353 30 : aicpusd_warn("task required to dump does not exist, stream id[%u], task id[%u], " \
354 : "context id[%u], thread id[%u].", dumpTaskInfo.streamId_,
355 : dumpTaskInfo.taskId_, dumpTaskInfo.contextId_, dumpTaskInfo.threadId_);
356 30 : return AICPU_SCHEDULE_OK;
357 : } else {
358 25 : std::vector<std::shared_ptr<OpDumpTask>> opDumptasks;
359 61 : for (auto item = range.first; item != range.second; ++item) {
360 36 : std::shared_ptr<OpDumpTask> opDumpTask = item->second;
361 36 : opDumptasks.push_back(std::move(opDumpTask));
362 36 : }
363 25 : int32_t ret = AICPU_SCHEDULE_OK;
364 25 : dumpTaskMapMtx_.unlock();
365 25 : aicpusd_run_info("require to dump op info, stream id=%u, task id=%u, context id=%u, " \
366 : "thread id=%u, task number=%zu", dumpTaskInfo.streamId_, dumpTaskInfo.taskId_,
367 : dumpTaskInfo.contextId_, dumpTaskInfo.threadId_, opDumptasks.size());
368 25 : uint32_t indexId = 0;
369 60 : for (auto item : opDumptasks) {
370 36 : auto startTime = std::chrono::steady_clock::now();
371 36 : const KfcDumpTask kfcDumpTaskinfo(dumpTaskInfo.streamId_, dumpTaskInfo.taskId_, indexId);
372 36 : MakeDumpOpInfoforKfc(kfcDumpTaskinfo, item);
373 36 : dumpTaskInfo.indexId_ = indexId;
374 36 : aicpusd_run_info("start to dump op info, stream id=%u, task id=%u, context id=%u, " \
375 : "thread id=%u, op name=%s", dumpTaskInfo.streamId_, dumpTaskInfo.taskId_,
376 : dumpTaskInfo.contextId_, dumpTaskInfo.threadId_, item->GetOpName().c_str());
377 36 : ret = item->DumpOpInfo(dumpTaskInfo, dumpFileName);
378 36 : auto endTime = std::chrono::steady_clock::now();
379 36 : double drUs = std::chrono::duration<double, std::micro>(endTime - startTime).count();
380 36 : aicpusd_run_info("end of dump op info, result=%d, stream id=%u, task id=%u, " \
381 : "context id=%u, thread id=%u, op name=%s, cost time is [%.2lf]us", ret, dumpTaskInfo.streamId_,
382 : dumpTaskInfo.taskId_, dumpTaskInfo.contextId_, dumpTaskInfo.threadId_,
383 : item->GetOpName().c_str(), drUs);
384 : UNUSED(drUs);
385 36 : if (ret != AICPU_SCHEDULE_OK) {
386 1 : ClearKfcDumpTaskInfo(kfcDumpTaskinfo);
387 1 : return ret;
388 : }
389 35 : indexId++;
390 35 : ClearKfcDumpTaskInfo(kfcDumpTaskinfo);
391 36 : }
392 : // process if task is end graph
393 24 : ProcessEndGraph(opDumptasks);
394 25 : }
395 24 : return AICPU_SCHEDULE_OK;
396 : }
397 :
398 12 : static bool CheckAndGetDumpBitmap(MappingInfoOptionalParam &optionalParam, uint64_t &switchBitMap)
399 : {
400 : // 检查是否有有效的dump开关
401 12 : if (!optionalParam.hasDumpSwitch) {
402 10 : aicpusd_info("Get op mapping info: no dump switch field.");
403 10 : return false;
404 : }
405 : // 检查dumpSwitchAddr是否有效
406 2 : if (optionalParam.dumpSwitchAddr == nullptr) {
407 1 : aicpusd_run_info("Get op mapping info: dump switch field exists but address is null.");
408 1 : return false;
409 : }
410 1 : switchBitMap = *(optionalParam.dumpSwitchAddr);
411 1 : aicpusd_info("Get op mapping info dump bitmap is [%llx].", switchBitMap);
412 1 : return true;
413 : }
414 :
415 14 : int32_t OpDumpTaskManager::DumpOpInfoForUnknowShape(const uint64_t opMappingInfoAddr,
416 : const uint64_t opMappingInfoLen) const
417 : {
418 14 : aicpusd_info("load op mapping info from dump task,value[%llu], size[%llu]",opMappingInfoAddr, opMappingInfoLen);
419 14 : if (opMappingInfoAddr == 0U) {
420 1 : aicpusd_err("op mapping info addr is null");
421 1 : return AICPU_SCHEDULE_ERROR_DUMP_FAILED;
422 : }
423 13 : aicpu::dump::OpMappingInfo opMappingInfo;
424 13 : const std::string protoInfo(PtrToPtr<const void, const char_t>(ValueToPtr(opMappingInfoAddr)),
425 13 : opMappingInfoLen);
426 13 : const bool parseRet = opMappingInfo.ParseFromString(protoInfo);
427 13 : if (!parseRet) {
428 1 : aicpusd_err("parse op mapping info failed, size[%u]", opMappingInfoLen);
429 1 : return AICPU_SCHEDULE_ERROR_DUMP_FAILED;
430 : }
431 12 : MappingInfoOptionalParam optionalParam;
432 12 : GetOptionalParam(opMappingInfo, optionalParam);
433 12 : uint64_t switchBitMap = 0UL;
434 12 : if (!CheckAndGetDumpBitmap(optionalParam, switchBitMap)) {
435 11 : return DoDump(opMappingInfo, optionalParam);
436 : } else {
437 1 : return DoDumpBySwitchBitmap(opMappingInfo, optionalParam, switchBitMap);
438 : }
439 13 : }
440 :
441 12 : int32_t OpDumpTaskManager::DoDump(const aicpu::dump::OpMappingInfo &opMappingInfo, const MappingInfoOptionalParam &optionalParam) const
442 : {
443 12 : const int32_t taskSize = opMappingInfo.task_size();
444 12 : if (taskSize != 1) {
445 2 : aicpusd_err("task number[%d] should be only one, op mapping info: %s",
446 : opMappingInfo.task_size(), opMappingInfo.DebugString().c_str());
447 2 : return AICPU_SCHEDULE_ERROR_DUMP_FAILED;
448 : }
449 10 : const std::string dumpPath = opMappingInfo.dump_path();
450 10 : const std::string dumpStepStr = opMappingInfo.dump_step();
451 :
452 10 : DumpStep dumpStep;
453 10 : if (!GetDumpStepFromString(dumpStepStr, dumpStep)) {
454 0 : return AICPU_SCHEDULE_ERROR_DUMP_FAILED;
455 : }
456 10 : const int32_t hostPid = static_cast<int32_t>(AicpuSchedule::AicpuDrvManager::GetInstance().GetHostPid());
457 10 : const uint32_t deviceId = AicpuSchedule::AicpuDrvManager::GetInstance().GetDeviceId();
458 10 : std::shared_ptr<OpDumpTask> opDumpTaskPtr = nullptr;
459 10 : int32_t ret = CreateOpDumpTask(opDumpTaskPtr, hostPid, deviceId);
460 10 : if (ret != AICPU_SCHEDULE_OK) {
461 2 : aicpusd_err("Create opDumpTask failed, hostPid[%d], deviceId[%u]", hostPid, deviceId);
462 2 : return ret;
463 : }
464 8 : aicpusd_memory_log("MallocMemory, func=new, size=%zu, purpose=data dumper", sizeof(OpDumpTask));
465 8 : if (opDumpTaskPtr == nullptr) {
466 2 : aicpusd_err("malloc memory for OpDumpTask object failed");
467 2 : return AICPU_SCHEDULE_ERROR_DUMP_FAILED;
468 : }
469 6 : const aicpu::dump::Task task = opMappingInfo.task(0);
470 6 : DumpMode dumpMode = opMappingInfo.dump_data();
471 6 : aicpusd_info("skipAddressConversion[true], hasStepId[%u], task_id[%u], stream_id[%u] dumpMode[%d]",
472 : static_cast<uint32_t>(optionalParam.hasStepId), task.task_id(), task.stream_id(),
473 : static_cast<int32_t>(dumpMode));
474 6 : ret = opDumpTaskPtr->PreProcessOpMappingInfo(task,
475 : dumpPath,
476 : optionalParam,
477 : dumpStep,
478 : dumpMode,
479 : true);
480 6 : if (ret != AICPU_SCHEDULE_OK) {
481 2 : aicpusd_err("pre process op mapping info failed, op mapping info: %s",
482 : opMappingInfo.DebugString().c_str());
483 2 : return ret;
484 : }
485 4 : AicpuSqeAdapter adapter(FeatureCtrl::GetTsMsgVersion());
486 4 : AicpuSqeAdapter::AicpuOpMappingDumpTaskInfo opmappingInfo(task.task_id(), task.stream_id(), INVALID_VAL, INVALID_VAL);
487 4 : AicpuSqeAdapter::AicpuDumpTaskInfo taskInfo{};
488 4 : adapter.GetAicpuDumpTaskInfo(opmappingInfo, taskInfo);
489 4 : auto startTime = std::chrono::steady_clock::now();
490 4 : aicpusd_run_info("start to dump op info, op name=%s", opDumpTaskPtr->GetOpName().c_str());
491 4 : const KfcDumpTask kfcDumpTaskinfo(taskInfo.stream_id, taskInfo.task_id, 0);
492 4 : const_cast<OpDumpTaskManager *>(this)->MakeDumpOpInfoforKfc(kfcDumpTaskinfo, opDumpTaskPtr);
493 4 : const TaskInfoExt dumpTaskInfo(taskInfo.stream_id, taskInfo.task_id, INVALID_VAL, INVALID_VAL, 0);
494 4 : DumpFileName dumpFileName(taskInfo.stream_id, taskInfo.task_id);
495 4 : ret = opDumpTaskPtr->DumpOpInfo(dumpTaskInfo, dumpFileName);
496 4 : auto endTime = std::chrono::steady_clock::now();
497 4 : double drUs = std::chrono::duration<double, std::micro>(endTime - startTime).count();
498 4 : aicpusd_run_info("end of dump op info, result=%d, op name=%s, cost time is [%.2lf]us",
499 : ret, opDumpTaskPtr->GetOpName().c_str(), drUs);
500 : UNUSED(drUs);
501 4 : opDumpTaskPtr->ClearBaseDumpData();
502 4 : const_cast<OpDumpTaskManager *>(this)->ClearKfcDumpTaskInfo(kfcDumpTaskinfo);
503 4 : return ret;
504 10 : }
505 :
506 2 : int32_t OpDumpTaskManager::GetAndClearOverflowStatus(const uint32_t deviceId, const uint32_t streamId, const uint32_t opType, uint32_t *status) const
507 : {
508 2 : TsCtrlMsgBody queryIn = {};
509 2 : queryIn.type = opType;
510 2 : queryIn.u.query_stream_overflow_status.stream_id = streamId;
511 2 : TsCtrlMsgBody queryResult = {};
512 2 : size_t queryResultCount = sizeof(TsCtrlMsgBody);
513 : TsDrvCtrlMsg para;
514 2 : para.tsid = 0U;
515 2 : para.msg_len = sizeof(TsCtrlMsgBody);
516 2 : para.msg = static_cast<void*>(&queryIn);
517 2 : if (!EnsureDeviceOpened(deviceId)) {
518 0 : aicpusd_err("The open device interface return failed, device id is %u, stream id is %u, op type is %u.", deviceId, streamId, opType);
519 0 : return AICPU_SCHEDULE_ERROR_FROM_DRV;
520 : }
521 2 : const drvError_t drvRet = halTsdrvCtl(deviceId, TSDRV_CTL_CMD_CTRL_MSG, static_cast<void*>(¶), sizeof(TsDrvCtrlMsg), static_cast<void*>(&queryResult), &queryResultCount);
522 2 : if (drvRet != DRV_ERROR_NONE) {
523 0 : aicpusd_err("The device id is %u, stream id is %u, op type is %u, return result is %d.", deviceId, streamId, opType, static_cast<int32_t>(drvRet));
524 0 : return static_cast<int32_t>(drvRet);
525 : }
526 2 : *status = queryResult.u.query_stream_overflow_status.status;
527 2 : aicpusd_run_info("The device id is %u, stream id is %u, op type is %u, overflow status result is %u.",
528 : deviceId, streamId, opType, queryResult.u.query_stream_overflow_status.status);
529 2 : return AICPU_SCHEDULE_OK;
530 : }
531 :
532 2 : bool OpDumpTaskManager::EnsureDeviceOpened(const uint32_t deviceId) const {
533 : static std::once_flag drvOpenflag;
534 : static bool isDeviceOpen = false;
535 2 : std::call_once(drvOpenflag, [&]() {
536 1 : struct drvDevInfo devInfo = {0};
537 1 : auto ret = drvDeviceOpen(PtrToPtr<void, void*>(&devInfo), deviceId);
538 1 : if ((ret != DRV_ERROR_NONE) && (ret != DRV_ERROR_REPEATED_INIT)) {
539 0 : aicpusd_err("Device %u open failed, ret=%d", deviceId, static_cast<int32_t>(ret));
540 0 : isDeviceOpen = false;
541 : } else {
542 1 : aicpusd_info("Device %u opened successfully", deviceId);
543 1 : isDeviceOpen = true;
544 : }
545 1 : });
546 2 : return isDeviceOpen;
547 : }
548 :
549 5 : int32_t OpDumpTaskManager::DoDumpBySwitchBitmap(const aicpu::dump::OpMappingInfo &opMappingInfo, const MappingInfoOptionalParam &optionalParam, const uint64_t switchBitMap) const
550 : {
551 : // 全0表示关闭dump开关，或者溢出检测开关开启但是halTsdrvCtl接口或者drvDeviceOpen不存在，不做dump处理
552 5 : if ((switchBitMap == 0UL) || ((switchBitMap & OVERFLOW_DUMP_BIT) != 0UL && ((&halTsdrvCtl == nullptr) || (&drvDeviceOpen == nullptr)))) {
553 0 : aicpusd_run_info("Get op mapping info dump bitmap is [%llx], halTsdrvCtl available is [%d], drvDeviceOpen available is [%d].", switchBitMap, &halTsdrvCtl == nullptr, &drvDeviceOpen == nullptr);
554 0 : return AICPU_SCHEDULE_OK;
555 : }
556 5 : DumpMode dumpMode = DumpMode::TENSOR_DUMP_DATA;
557 5 : if ((switchBitMap & STATS_DUMP_BIT) != 0UL) {
558 1 : dumpMode = DumpMode::STATS_DUMP_DATA;
559 : }
560 5 : if ((switchBitMap & TENSOR_DUMP_BIT) != 0UL) {
561 2 : dumpMode = DumpMode::TENSOR_DUMP_DATA;
562 : }
563 5 : const std::string dumpPath = opMappingInfo.dump_path();
564 5 : const std::string dumpStepStr = opMappingInfo.dump_step();
565 5 : DumpStep dumpStep;
566 5 : if (!GetDumpStepFromString(dumpStepStr, dumpStep)) {
567 0 : return AICPU_SCHEDULE_ERROR_DUMP_FAILED;
568 : }
569 5 : const int32_t hostPid = static_cast<int32_t>(AicpuSchedule::AicpuDrvManager::GetInstance().GetHostPid());
570 5 : const uint32_t deviceId = AicpuSchedule::AicpuDrvManager::GetInstance().GetDeviceId();
571 5 : const int32_t taskSize = opMappingInfo.task_size();
572 5 : if (taskSize > MAX_TASK_SIZE) {
573 0 : aicpusd_run_info("Task number[%d] exceeds 2, op mapping info: %s", taskSize, opMappingInfo.DebugString().c_str());
574 : }
575 5 : int32_t ret = AICPU_SCHEDULE_OK;
576 9 : for (int32_t i = 0; i < taskSize; i++) {
577 5 : const aicpu::dump::Task task = opMappingInfo.task(i);
578 5 : std::shared_ptr<OpDumpTask> opDumpTaskPtr = nullptr;
579 5 : ret = CreateOpDumpTask(opDumpTaskPtr, hostPid, deviceId);
580 5 : if (ret != AICPU_SCHEDULE_OK) {
581 0 : aicpusd_err("Create opDumpTask failed, hostPid[%d], deviceId[%u].", hostPid, deviceId);
582 0 : return ret;
583 : }
584 5 : aicpusd_memory_log("MallocMemory, func=new, size=%zu, purpose=data dumper.", sizeof(OpDumpTask));
585 5 : if (opDumpTaskPtr == nullptr) {
586 0 : aicpusd_err("Malloc memory for OpDumpTask object failed.");
587 0 : return AICPU_SCHEDULE_ERROR_DUMP_FAILED;
588 : }
589 5 : if ((switchBitMap & OVERFLOW_DUMP_BIT) != 0UL) {
590 2 : uint32_t overflowStatus = 0U;
591 2 : const int32_t getRet = GetAndClearOverflowStatus(deviceId, task.stream_id(), OP_QUERY_OVERFLOW, &overflowStatus);
592 2 : if (getRet != AICPU_SCHEDULE_OK) {
593 0 : aicpusd_err("Query overflow status interface returned: ret is %d, device id is %u, stream id is %u.", getRet, deviceId, task.stream_id());
594 1 : return AICPU_SCHEDULE_ERROR_FROM_DRV;
595 : }
596 2 : if (overflowStatus == 0U) { // result 为0表示不溢出，不做dump处理直接返回
597 1 : aicpusd_run_info("The query overflow status is %u, the device id is %u, and the stream id is %u.", overflowStatus, deviceId, task.stream_id());
598 1 : return AICPU_SCHEDULE_OK;
599 : }
600 : }
601 4 : aicpusd_info("Dump the parameter information : single or unknown shape flag[false], has step id[%u], task id[%u], stream id[%u], dump mode[%d].",
602 : static_cast<uint32_t>(optionalParam.hasStepId), task.task_id(), task.stream_id(),
603 : static_cast<int32_t>(dumpMode));
604 4 : ret = opDumpTaskPtr->PreProcessOpMappingInfo(task, dumpPath, optionalParam, dumpStep, dumpMode, false);
605 4 : if (ret != AICPU_SCHEDULE_OK) {
606 0 : aicpusd_err("Preprocess op mapping info failed. Op mapping info is %s", opMappingInfo.DebugString().c_str());
607 0 : return ret;
608 : }
609 4 : AicpuSqeAdapter adapter(static_cast<int16_t>(FeatureCtrl::GetTsMsgVersion()));
610 4 : AicpuSqeAdapter::AicpuOpMappingDumpTaskInfo opmappingInfo(task.task_id(), task.stream_id(), INVALID_VAL, INVALID_VAL);
611 4 : AicpuSqeAdapter::AicpuDumpTaskInfo taskInfo{};
612 4 : adapter.GetAicpuDumpTaskInfo(opmappingInfo, taskInfo);
613 4 : auto startTime = std::chrono::steady_clock::now();
614 4 : aicpusd_run_info("Start dumping operation information: op name is %s.", opDumpTaskPtr->GetOpName().c_str());
615 4 : const KfcDumpTask kfcDumpTaskinfo(taskInfo.stream_id, taskInfo.task_id, 0);
616 4 : const_cast<OpDumpTaskManager *>(this)->MakeDumpOpInfoforKfc(kfcDumpTaskinfo, opDumpTaskPtr);
617 4 : const TaskInfoExt dumpTaskInfo(taskInfo.stream_id, taskInfo.task_id, INVALID_VAL, INVALID_VAL, 0);
618 4 : DumpFileName dumpFileName(taskInfo.stream_id, taskInfo.task_id);
619 4 : ret = opDumpTaskPtr->DumpOpInfo(dumpTaskInfo, dumpFileName);
620 4 : auto endTime = std::chrono::steady_clock::now();
621 4 : const double drUs = std::chrono::duration<double, std::micro>(endTime - startTime).count();
622 4 : aicpusd_run_info("End of dump operation information: result is %d, op name is %s, cost time is [%.2lf]us.",
623 : ret, opDumpTaskPtr->GetOpName().c_str(), drUs);
624 : UNUSED(drUs);
625 4 : opDumpTaskPtr->ClearBaseDumpData();
626 4 : const_cast<OpDumpTaskManager *>(this)->ClearKfcDumpTaskInfo(kfcDumpTaskinfo);
627 6 : }
628 4 : return ret;
629 5 : }
630 :
631 24 : void OpDumpTaskManager::ProcessEndGraph(const std::vector<std::shared_ptr<OpDumpTask>> &opDumptasks)
632 : {
633 58 : for (const auto &item : opDumptasks) {
634 34 : if (item->IsEndGraph()) {
635 : uint32_t modelId;
636 4 : if (item->GetModelId(modelId)) {
637 3 : UpdateDumpNumByModelId(modelId);
638 : } else {
639 1 : item->UpdateDumpNum();
640 : }
641 : }
642 : }
643 24 : }
644 :
645 3 : void OpDumpTaskManager::UpdateDumpNumByModelId(const uint32_t modelId)
646 : {
647 3 : std::vector<std::shared_ptr<OpDumpTask>> opDumptasks;
648 : {
649 3 : const std::lock_guard<std::mutex> mapLock(dumpTaskMapMtx_);
650 3 : const auto iter = modelIdToTask_.find(modelId);
651 3 : if (iter != modelIdToTask_.end()) {
652 6 : for (auto &taskInfo : iter->second) {
653 3 : const auto range = dumpTaskMap_.equal_range(taskInfo);
654 12 : for (auto item = range.first; item != range.second; ++item) {
655 9 : std::shared_ptr<OpDumpTask> opDumpTask = item->second;
656 9 : opDumptasks.push_back(std::move(opDumpTask));
657 9 : }
658 : }
659 : }
660 3 : }
661 :
662 12 : for (auto &dumpTask : opDumptasks) {
663 9 : dumpTask->UpdateDumpNum();
664 : }
665 3 : }
666 :
667 :
668 7 : static void DoSplit(const std::string &externStr, std::vector<std::string> &ret, const std::string &delim)
669 : {
670 7 : size_t i = 0UL;
671 20 : while (i < externStr.size()) {
672 13 : const size_t pos = externStr.find(delim, i);
673 13 : if (pos != std::string::npos) {
674 13 : std::string s = externStr.substr(i, pos - i);
675 13 : if (!s.empty()) {
676 13 : ret.push_back(std::move(s));
677 : }
678 13 : i = pos + delim.size() - 1UL;
679 13 : }
680 13 : ++i;
681 : }
682 7 : }
683 :
684 51 : static std::vector<std::string> Split(const std::string &str, const std::string &delim)
685 : {
686 51 : std::vector<std::string> ret;
687 51 : if (!str.empty()) {
688 7 : const std::string externStr = str + delim;
689 7 : DoSplit(externStr, ret, delim);
690 7 : }
691 51 : return ret;
692 0 : }
693 :
694 8 : bool OpDumpTaskManager::MatchAndInsert(const std::string &step, DumpStep &tmpDumpStep) const
695 : {
696 : // smatch result
697 8 : const std::regex singleStepPatten("(\\s*)(\\d+)(\\s*)");
698 8 : const std::regex intervalStepPatten("((\\s*)(\\d+)(\\s*)){1}(-(\\s*)(\\d+)(\\s*))");
699 8 : if (std::regex_match(step, singleStepPatten)) {
700 4 : (void)tmpDumpStep.singleStep.insert(static_cast<uint64_t>(std::stoull(step)));
701 4 : } else if (std::regex_match(step, intervalStepPatten)) {
702 3 : const std::vector<std::string> intervalStepStr = Split(step, "-");
703 3 : if (intervalStepStr.size() == INTERNAL_STEP_SIZE) {
704 : IntervalStep intervalStep;
705 3 : intervalStep.start = static_cast<uint64_t>(std::stoull(intervalStepStr[0U]));
706 3 : intervalStep.end = static_cast<uint64_t>(std::stoull(intervalStepStr[1U]));
707 3 : if (intervalStep.start > intervalStep.end) {
708 3 : std::swap(intervalStep.start, intervalStep.end);
709 : }
710 3 : tmpDumpStep.intervalStep.push_back(std::move(intervalStep));
711 : }
712 3 : } else {
713 1 : aicpusd_err("invalid step[%s], please check.", step.c_str());
714 1 : return false;
715 : }
716 6 : return true;
717 9 : }
718 :
719 48 : bool OpDumpTaskManager::GetDumpStepFromString(const std::string &str, DumpStep &dumpStep) const
720 : {
721 : // split by |, such as "0|5-10"
722 48 : aicpusd_info("Step need to dump[%s].", str.c_str());
723 48 : const std::vector<std::string> steps = Split(str, "|");
724 48 : DumpStep tmpDumpStep;
725 54 : for (const auto &step : steps) {
726 : try {
727 7 : if (!MatchAndInsert(step, tmpDumpStep)) {
728 1 : return false;
729 : }
730 1 : } catch (std::out_of_range &e) {
731 1 : aicpusd_err("out of range of uint64_max[%llu], invalid step[%s], msg[%s], please check.",
732 : UINT64_MAX, step.c_str(), e.what());
733 1 : return false;
734 1 : } catch (...) {
735 0 : aicpusd_err("invalid step[%s], please check.", step.c_str());
736 0 : return false;
737 0 : }
738 : }
739 47 : dumpStep = std::move(tmpDumpStep);
740 47 : return true;
741 48 : }
742 :
743 3 : void OpDumpTaskManager::ClearResource()
744 : {
745 3 : const std::lock_guard<std::mutex> mapLock(dumpTaskMapMtx_);
746 3 : aicpusd_run_info("clear all resource of data dump");
747 3 : dumpTaskMap_.clear();
748 3 : modelIdToTask_.clear();
749 3 : kfcDumpTaskMap_.clear();
750 3 : kfcDumpInfoMap_.clear();
751 3 : custDumpTaskMap_.clear();
752 3 : }
753 :
754 : /*
755 : * 清除资源
756 : */
757 44 : void OpDumpTaskManager::ClearKfcDumpTaskInfo(const KfcDumpTask &kfcTaskinfo)
758 : {
759 44 : const std::lock_guard<std::mutex> mapLock(kfcDumpTaskMapMtx_);
760 44 : if (kfcDumpTaskMap_.find(kfcTaskinfo) != kfcDumpTaskMap_.end()) {
761 2 : (void)kfcDumpTaskMap_.erase(kfcTaskinfo);
762 2 : aicpusd_info("erase key of dump task map : streamId[%u], taskId[%u], index[%u]", kfcTaskinfo.streamId_, kfcTaskinfo.taskId_, kfcTaskinfo.index_);
763 : }
764 44 : if (kfcDumpInfoMap_.find(kfcTaskinfo) != kfcDumpInfoMap_.end()) {
765 2 : (void)kfcDumpInfoMap_.erase(kfcTaskinfo);
766 2 : aicpusd_info("erase key of dump info map : streamId[%u], taskId[%u], index[%u]", kfcTaskinfo.streamId_, kfcTaskinfo.taskId_, kfcTaskinfo.index_);
767 : }
768 44 : TaskInfo taskInfo(kfcTaskinfo.streamId_ , kfcTaskinfo.taskId_);
769 44 : if (custDumpTaskMap_.find(taskInfo) != custDumpTaskMap_.end()) {
770 0 : (void)custDumpTaskMap_.erase(taskInfo);
771 0 : aicpusd_info("erase key of cust dump task map : streamId[%u], taskId[%u], index[%u]", kfcTaskinfo.streamId_, kfcTaskinfo.taskId_, kfcTaskinfo.index_);
772 : }
773 88 : return;
774 44 : }
775 :
776 :
777 : /*
778 : * 查询是否是aicpu cust dump 任务
779 : */
780 1 : bool OpDumpTaskManager::IsCustDumpTask(const uint32_t streamId, const uint32_t taskId)
781 : {
782 1 : TaskInfo taskInfo(streamId, taskId);
783 1 : const auto iter = custDumpTaskMap_.find(taskInfo);
784 1 : if (iter == custDumpTaskMap_.end()) {
785 1 : aicpusd_info("Cannot find dump task [streamId:%u, taskId:%u] in cust dump task map.", streamId, taskId);
786 1 : return false;
787 : }
788 0 : aicpusd_info("Get dump task [streamId:%u, taskId:%u] flag is [%d].", streamId, taskId, iter->second);
789 0 : return iter->second;
790 : }
791 :
792 : /*
793 : * 获取是否是aicpu cust dump 任务
794 : */
795 11 : int32_t OpDumpTaskManager::SetCustDumpTaskFlag(const uint32_t streamId, const uint32_t taskId, const bool flag)
796 : {
797 11 : TaskInfo taskInfo(streamId, taskId);
798 11 : custDumpTaskMap_.insert(std::make_pair(taskInfo, flag));
799 11 : aicpusd_info("Set dump task [streamId:%u, taskId:%u] flag [%d] in cust dump task map.", streamId, taskId, flag);
800 11 : return AICPU_SCHEDULE_OK;
801 : }
802 :
803 : /*
804 : * 构造kfc需要的dump信息映射关系{steamid taskid index, task}
805 : */
806 52 : void OpDumpTaskManager::MakeDumpOpInfoforKfc(const KfcDumpTask &taskinfo, std::shared_ptr<OpDumpTask> dumpTask)
807 : {
808 52 : const std::lock_guard<std::mutex> mapLock(kfcDumpTaskMapMtx_);
809 52 : if (!(dumpTask->IsSupportKfcDump())) {
810 47 : aicpusd_info("task does not support kfc statistical dump, streamId[%u], taskId[%u].", taskinfo.streamId_, taskinfo.taskId_);
811 47 : return;
812 : }
813 5 : kfcDumpTaskMap_.insert(std::make_pair(taskinfo, dumpTask));
814 5 : aicpusd_info("Make kfc dump task map, streamId[%u], taskId[%u], index[%u]", taskinfo.streamId_, taskinfo.taskId_, taskinfo.index_);
815 5 : return;
816 52 : }
817 6 : int32_t OpDumpTaskManager::CreateKfcDumpInfo(std::shared_ptr<KfcDumpInfo> &kfcDumpInfoPtr) const
818 : {
819 6 : DATADUMP_MAKE_SHARED(kfcDumpInfoPtr = std::make_shared<KfcDumpInfo>(),
820 : return AICPU_SCHEDULE_ERROR_DUMP_FAILED);
821 6 : return AICPU_SCHEDULE_OK;
822 : }
823 : /*
824 : * 向kfc提供dump查询信息接口
825 : */
826 11 : int32_t OpDumpTaskManager::GetDumpOpTaskDataforKfc(const KfcDumpTask &taskKey, KfcDumpInfo **dumpInfo)
827 : {
828 11 : const std::lock_guard<std::mutex> mapLock(kfcDumpTaskMapMtx_);
829 11 : aicpusd_info("Get dump info for kfc, streamId[%u], taskId[%u], index[%u].",
830 : taskKey.streamId_, taskKey.taskId_, taskKey.index_);
831 : // dumpInfo 是否为空在上层调用处保证
832 11 : const auto iter = kfcDumpTaskMap_.find(taskKey);
833 11 : if (iter == kfcDumpTaskMap_.end()) {
834 2 : aicpusd_err("Cannot find dump task info key for kfc. streamId[%u], taskId[%u], index[%u].",
835 : taskKey.streamId_, taskKey.taskId_, taskKey.index_);
836 2 : return AICPU_SCHEDULE_ERROR_DUMP_FAILED;
837 : }
838 9 : if (iter->second == nullptr) {
839 1 : aicpusd_err("Get dump task info failed. streamId[%u], taskId[%u], index[%u]",
840 : taskKey.streamId_, taskKey.taskId_, taskKey.index_);
841 1 : return AICPU_SCHEDULE_ERROR_DUMP_FAILED;
842 : }
843 :
844 8 : std::shared_ptr<KfcDumpInfo> kfcDumpInfoPtr = nullptr;
845 8 : int32_t ret = CreateKfcDumpInfo(kfcDumpInfoPtr);
846 8 : if (ret != AICPU_SCHEDULE_OK) {
847 1 : aicpusd_err("Create opDumpTask failed streamId[%u], taskId[%u], index[%u]",
848 : taskKey.streamId_, taskKey.taskId_, taskKey.index_);
849 1 : return ret;
850 : }
851 7 : if (kfcDumpInfoPtr == nullptr) {
852 1 : aicpusd_err("malloc memory for KfcDumpInfo object failed");
853 1 : return AICPU_SCHEDULE_ERROR_DUMP_FAILED;
854 : }
855 6 : iter->second->GetKfcDumpInfo(kfcDumpInfoPtr);
856 6 : kfcDumpInfoMap_.insert(std::make_pair(taskKey, kfcDumpInfoPtr));
857 6 : *dumpInfo = kfcDumpInfoPtr.get();
858 6 : return AICPU_SCHEDULE_OK;
859 11 : }
860 :
861 : /*
862 : * 向kfc提供dump数据接口
863 : */
864 8 : int32_t OpDumpTaskManager::DumpOpTaskDataforKfc(const KfcDumpTask &taskKey, void *dumpData, uint32_t length) const
865 : {
866 8 : aicpusd_info("dump op task data for kfc, length [%u], streamId[%u], taskId[%u], index[%u]",
867 : length, taskKey.streamId_, taskKey.taskId_, taskKey.index_);
868 : // dumpData 是否为空在上层调用处保证
869 8 : const auto iter = kfcDumpTaskMap_.find(taskKey);
870 8 : if (iter == kfcDumpTaskMap_.end()) {
871 2 : aicpusd_err("Not get dump info for kfc. length [%u], streamId[%u], taskId[%u], index[%u].",
872 : length, taskKey.streamId_, taskKey.taskId_, taskKey.index_);
873 2 : return AICPU_SCHEDULE_ERROR_DUMP_FAILED;
874 : }
875 6 : if (iter->second == nullptr) {
876 1 : aicpusd_err("Get dump task failed. length [%u], streamId[%u], taskId[%u], index[%u]",
877 : length, taskKey.streamId_, taskKey.taskId_, taskKey.index_);
878 1 : return AICPU_SCHEDULE_ERROR_DUMP_FAILED;
879 : }
880 5 : const std::string filePath = iter->second->GetDumpPath();
881 5 : const std::string dumpFilePath = filePath + ".bin";
882 5 : const int32_t hostPid = iter->second->GetHostPid();
883 5 : const uint32_t deviceId = iter->second->GetDeviceId();
884 5 : const std::string opName = iter->second->GetOpName();
885 :
886 5 : IDE_SESSION ideSession = DumpSessionManager::GetInstance().GetSession(hostPid, deviceId);
887 5 : if (ideSession == nullptr) {
888 0 : aicpusd_err("op name[%s], call IdeDumpStart failed, path[%s]. length [%u], streamId[%u], taskId[%u], index[%u]",
889 : opName.c_str(), dumpFilePath.c_str(), length, taskKey.streamId_, taskKey.taskId_, taskKey.index_);
890 0 : return AICPU_SCHEDULE_ERROR_DUMP_FAILED;
891 : }
892 :
893 5 : return iter->second->Dump(dumpFilePath, PtrToPtr<void, char>(dumpData), static_cast<uint64_t>(length), ideSession, true);
894 5 : }
895 : } // namespace aicpu
|
