Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 : #include "dump_task.h"
11 :
12 : #include <algorithm>
13 : #include <regex>
14 : #include <sstream>
15 : #include <vector>
16 : #include <map>
17 : #include <dlfcn.h>
18 : #include "aicpusd_drv_manager.h"
19 : #include "aicpusd_status.h"
20 : #include "aicpusd_monitor.h"
21 : #include "aicpusd_model_execute.h"
22 : #include "common/aicpusd_util.h"
23 : #include "securec.h"
24 : #include "metadef_types.h"
25 : #include "aicpusd_hal_interface_ref.h"
26 : #include "aicpusd_common.h"
27 :
28 : namespace AicpuSchedule {
29 : const size_t INTERNAL_STEP_SIZE = 2U;
30 : const uint32_t VERSION_0 = 0U;
31 : constexpr uint32_t OVERFLOW_DUMP_BIT = 0x04;
32 : constexpr uint32_t STATS_DUMP_BIT = 0x02;
33 : constexpr uint32_t TENSOR_DUMP_BIT = 0x01;
34 : constexpr int32_t MAX_TASK_SIZE = 2;
35 :
36 116 : OpDumpTaskManager& OpDumpTaskManager::GetInstance()
37 : {
38 116 : static OpDumpTaskManager instance;
39 116 : return instance;
40 : }
41 :
42 68 : void OpDumpTaskManager::GetOptionalParam(
43 : const aicpu::dump::OpMappingInfo& opMappingInfo, MappingInfoOptionalParam& optionalParam) const
44 : {
45 68 : if (opMappingInfo.model_name_param_case() == aicpu::dump::OpMappingInfo::kModelName) {
46 53 : optionalParam.modelName = opMappingInfo.model_name();
47 53 : ReplaceStringElem(optionalParam.modelName);
48 53 : optionalParam.hasModelName = true;
49 : }
50 68 : if (opMappingInfo.model_id_param_case() == aicpu::dump::OpMappingInfo::kModelId) {
51 53 : optionalParam.modelId = opMappingInfo.model_id();
52 53 : optionalParam.hasModelId = true;
53 : }
54 68 : if (opMappingInfo.step_id_case() == aicpu::dump::OpMappingInfo::kStepIdAddr) {
55 49 : optionalParam.stepIdAddr = PtrToPtr<void, uint64_t>(ValueToPtr(opMappingInfo.step_id_addr()));
56 49 : optionalParam.hasStepId = true;
57 : }
58 68 : if (opMappingInfo.iterations_per_loop_case() == aicpu::dump::OpMappingInfo::kIterationsPerLoopAddr) {
59 41 : optionalParam.iterationsPerLoopAddr =
60 41 : PtrToPtr<void, uint64_t>(ValueToPtr(opMappingInfo.iterations_per_loop_addr()));
61 41 : optionalParam.hasIterationsPerLoop = true;
62 : }
63 68 : if (opMappingInfo.loop_cond_case() == aicpu::dump::OpMappingInfo::kLoopCondAddr) {
64 41 : optionalParam.loopCondAddr = PtrToPtr<void, uint64_t>(ValueToPtr(opMappingInfo.loop_cond_addr()));
65 41 : optionalParam.hasLoopCond = true;
66 : }
67 68 : if (opMappingInfo.dump_switch_case() == aicpu::dump::OpMappingInfo::kDumpSwitchAddr) {
68 4 : optionalParam.dumpSwitchAddr = PtrToPtr<void, uint64_t>(ValueToPtr(opMappingInfo.dump_switch_addr()));
69 4 : optionalParam.hasDumpSwitch = true;
70 : }
71 68 : }
72 45 : int32_t OpDumpTaskManager::CreateOpDumpTask(
73 : std::shared_ptr<OpDumpTask>& opDumpTaskPtr, const int32_t hostPid, const uint32_t deviceId) const
74 : {
75 45 : DATADUMP_MAKE_SHARED(
76 : opDumpTaskPtr = std::make_shared<OpDumpTask>(hostPid, deviceId), return AICPU_SCHEDULE_ERROR_DUMP_FAILED);
77 45 : return AICPU_SCHEDULE_OK;
78 : }
79 :
80 42 : int32_t OpDumpTaskManager::Load(const aicpu::dump::OpMappingInfo& opMappingInfo, AicpuSqeAdapter& aicpuSqeAdapter)
81 : {
82 42 : const std::string dumpPath = opMappingInfo.dump_path();
83 42 : MappingInfoOptionalParam optionalParam;
84 42 : GetOptionalParam(opMappingInfo, optionalParam);
85 42 : const std::string dumpStepStr = opMappingInfo.dump_step();
86 :
87 42 : uint32_t streamId = INVALID_VAL;
88 42 : uint32_t taskId = INVALID_VAL;
89 42 : if (!aicpuSqeAdapter.IsAdapterInvalidParameter()) {
90 6 : AicpuSqeAdapter::AicpuDataDumpInfoLoad info{};
91 6 : aicpuSqeAdapter.GetAicpuDataDumpInfoLoad(info);
92 6 : streamId = info.stream_id;
93 6 : taskId = info.task_id;
94 : }
95 :
96 42 : DumpStep dumpStep;
97 42 : if (!GetDumpStepFromString(dumpStepStr, dumpStep)) {
98 3 : aicpusd_err("Step error[%s].", dumpStepStr.c_str());
99 3 : return AICPU_SCHEDULE_ERROR_DUMP_FAILED;
100 : }
101 39 : const int32_t hostPid = static_cast<int32_t>(AicpuSchedule::AicpuDrvManager::GetInstance().GetHostPid());
102 39 : const uint32_t deviceId = AicpuSchedule::AicpuDrvManager::GetInstance().GetDeviceId();
103 : {
104 39 : std::set<TaskInfo> tasksInfo;
105 39 : const std::lock_guard<std::mutex> mapLock(dumpTaskMapMtx_);
106 39 : int32_t ret = AICPU_SCHEDULE_OK;
107 70 : for (int32_t i = 0; i < opMappingInfo.task_size(); ++i) {
108 39 : aicpu::dump::Task task = opMappingInfo.task(i);
109 39 : aicpusd_info("Get load dump data from proto task id[%u], stream id[%u]", task.task_id(), task.stream_id());
110 : AicpuSqeAdapter::AicpuOpMappingDumpTaskInfo opmappingInfo(
111 39 : task.task_id(), task.stream_id(), taskId, streamId);
112 39 : const bool isOpMappingDumpTaskInfoVaild = aicpuSqeAdapter.IsOpMappingDumpTaskInfoVaild(opmappingInfo);
113 39 : if (!isOpMappingDumpTaskInfoVaild) {
114 2 : aicpusd_err(
115 : "streamId or taskId is INVALID_VAL, task.task_id()[%u], task.stream_id()[%u], "
116 : "streamId[%u], taskId[%u]",
117 : opmappingInfo.proto_info_task_id, opmappingInfo.proto_info_stream_id, opmappingInfo.stream_id,
118 : opmappingInfo.task_id);
119 : }
120 39 : AicpuSqeAdapter::AicpuDumpTaskInfo dumpTaskInfo{};
121 39 : aicpuSqeAdapter.GetAicpuDumpTaskInfo(opmappingInfo, dumpTaskInfo);
122 39 : uint32_t contextIdTmp = INVALID_VAL;
123 39 : uint32_t threadIdTmp = INVALID_VAL;
124 39 : if (task.tasktype() == aicpu::dump::Task::FFTSPLUS) {
125 11 : const auto& contextFromMapInfo = task.context();
126 11 : uint32_t contextFromMapInfoSize = static_cast<uint32_t>(contextFromMapInfo.size());
127 16 : for (uint32_t j = 0U; j < contextFromMapInfoSize; ++j) {
128 9 : std::shared_ptr<OpDumpTask> opDumpTaskPtr = nullptr;
129 9 : ret = CreateOpDumpTask(opDumpTaskPtr, hostPid, deviceId);
130 9 : if (ret != AICPU_SCHEDULE_OK) {
131 1 : aicpusd_err("Create opDumpTask failed, hostPid[%d], deviceId[%u]", hostPid, deviceId);
132 1 : return ret;
133 : }
134 8 : aicpusd_memory_log("MallocMemory, func=new, size=%zu, purpose=data dumper", sizeof(OpDumpTask));
135 8 : if (opDumpTaskPtr == nullptr) {
136 1 : aicpusd_err("malloc memory for OpDumpTask object failed");
137 1 : return AICPU_SCHEDULE_ERROR_DUMP_FAILED;
138 : }
139 7 : DumpMode dumpMode = opMappingInfo.dump_data();
140 7 : aicpusd_info("Load Dump data is %d", static_cast<int32_t>(dumpMode));
141 7 : ret = opDumpTaskPtr->PreProcessOpMappingInfo(task, dumpPath, optionalParam, dumpStep, dumpMode);
142 7 : if (ret != AICPU_SCHEDULE_OK) {
143 : // when error occur, ge call unload op mapping info
144 2 : aicpusd_err("pre process op mapping info failed");
145 2 : return ret;
146 : }
147 5 : auto& item = contextFromMapInfo.at(j);
148 5 : opDumpTaskPtr->UpdatePreProcessFftsPlusInputAndOutput(item);
149 5 : contextIdTmp = item.context_id();
150 5 : threadIdTmp = item.thread_id();
151 5 : TaskInfo taskInfo(dumpTaskInfo.stream_id, dumpTaskInfo.task_id, contextIdTmp, threadIdTmp);
152 5 : aicpusd_info(
153 : "[stream id:%u, task id:%u, context id:%u, thread id:%u]", dumpTaskInfo.stream_id,
154 : dumpTaskInfo.task_id, contextIdTmp, threadIdTmp);
155 5 : (void)dumpTaskMap_.insert(std::make_pair(taskInfo, std::move(opDumpTaskPtr)));
156 5 : (void)tasksInfo.insert(taskInfo);
157 9 : }
158 : } else {
159 28 : contextIdTmp = INVALID_VAL;
160 28 : threadIdTmp = INVALID_VAL;
161 28 : std::shared_ptr<OpDumpTask> opDumpTaskPtr = nullptr;
162 28 : ret = CreateOpDumpTask(opDumpTaskPtr, hostPid, deviceId);
163 28 : if (ret != AICPU_SCHEDULE_OK) {
164 1 : aicpusd_err("Create opDumpTask failed");
165 1 : return ret;
166 : }
167 27 : aicpusd_memory_log("MallocMemory, func=new, size=%zu, purpose=data dumper", sizeof(OpDumpTask));
168 27 : if (opDumpTaskPtr == nullptr) {
169 1 : aicpusd_err("malloc memory for OpDumpTask object failed");
170 1 : return AICPU_SCHEDULE_ERROR_DUMP_FAILED;
171 : }
172 26 : DumpMode dumpMode = opMappingInfo.dump_data();
173 26 : aicpusd_info("Load Dump data is %d", static_cast<int32_t>(dumpMode));
174 26 : ret = opDumpTaskPtr->PreProcessOpMappingInfo(task, dumpPath, optionalParam, dumpStep, dumpMode);
175 26 : if (ret != AICPU_SCHEDULE_OK) {
176 : // when error occur, ge call unload op mapping info
177 2 : aicpusd_err("pre process op mapping info failed");
178 2 : return ret;
179 : }
180 24 : TaskInfo taskInfo(dumpTaskInfo.stream_id, dumpTaskInfo.task_id, contextIdTmp, threadIdTmp);
181 24 : aicpusd_info(
182 : "[stream id:%u, task id:%u, context id:%u, thread id:%u]", dumpTaskInfo.stream_id,
183 : dumpTaskInfo.task_id, contextIdTmp, threadIdTmp);
184 24 : (void)dumpTaskMap_.insert(std::make_pair(taskInfo, std::move(opDumpTaskPtr)));
185 24 : (void)tasksInfo.insert(taskInfo);
186 28 : }
187 39 : }
188 31 : if (optionalParam.hasModelId) {
189 26 : const auto iter = modelIdToTask_.find(optionalParam.modelId);
190 26 : if (iter == modelIdToTask_.end()) {
191 15 : (void)modelIdToTask_.insert(std::make_pair(optionalParam.modelId, std::move(tasksInfo)));
192 : } else {
193 20 : for (const auto item : tasksInfo) {
194 9 : (void)iter->second.insert(item);
195 : }
196 : }
197 : }
198 47 : }
199 31 : return AICPU_SCHEDULE_OK;
200 42 : }
201 :
202 27 : void OpDumpTaskManager::UnloadClearTaskInfo(const TaskInfo& dumpTaskInfo)
203 : {
204 27 : const auto range = dumpTaskMap_.equal_range(dumpTaskInfo);
205 27 : if (range.first != range.second) {
206 30 : for (auto item = range.first; item != range.second; ++item) {
207 16 : item->second->ClearBaseDumpData();
208 : }
209 : }
210 27 : (void)dumpTaskMap_.erase(dumpTaskInfo);
211 54 : return;
212 : }
213 :
214 12 : int32_t OpDumpTaskManager::Unload(const aicpu::dump::OpMappingInfo& opMappingInfo, AicpuSqeAdapter& aicpuSqeAdapter)
215 : {
216 12 : const std::lock_guard<std::mutex> mapLock(dumpTaskMapMtx_);
217 12 : uint32_t streamId = INVALID_VAL;
218 12 : uint32_t taskId = INVALID_VAL;
219 12 : if (!aicpuSqeAdapter.IsAdapterInvalidParameter()) {
220 0 : AicpuSqeAdapter::AicpuDataDumpInfoLoad info{};
221 0 : aicpuSqeAdapter.GetAicpuDataDumpInfoLoad(info);
222 0 : streamId = info.stream_id;
223 0 : taskId = info.task_id;
224 : }
225 24 : for (int32_t i = 0; i < opMappingInfo.task_size(); ++i) {
226 12 : const aicpu::dump::Task task = opMappingInfo.task(i);
227 12 : AicpuSqeAdapter::AicpuOpMappingDumpTaskInfo opmappingInfo(task.task_id(), task.stream_id(), taskId, streamId);
228 12 : AicpuSqeAdapter::AicpuDumpTaskInfo taskInfo{};
229 12 : aicpuSqeAdapter.GetAicpuDumpTaskInfo(opmappingInfo, taskInfo);
230 12 : uint32_t contextIdTmp = INVALID_VAL;
231 12 : uint32_t threadIdTmp = INVALID_VAL;
232 12 : const auto& contextFromMapInfo = task.context();
233 12 : uint32_t contextFromMapInfoSize = static_cast<uint32_t>(contextFromMapInfo.size());
234 12 : bool isFftsPlusUnload = false;
235 15 : for (uint32_t j = 0U; j < contextFromMapInfoSize; ++j) {
236 3 : auto& item = contextFromMapInfo.at(j);
237 3 : contextIdTmp = item.context_id();
238 3 : threadIdTmp = item.thread_id();
239 3 : const TaskInfo dumpTaskInfo(taskInfo.stream_id, taskInfo.task_id, contextIdTmp, threadIdTmp);
240 3 : UnloadClearTaskInfo(dumpTaskInfo);
241 3 : aicpusd_info(
242 : "Unload stream id[%u], task id[%u], streamIdTmp[%u], taskIdTmp[%u],"
243 : " context id[%u], threadIdTmp[%u].",
244 : streamId, taskId, taskInfo.stream_id, taskInfo.task_id, contextIdTmp, threadIdTmp);
245 3 : isFftsPlusUnload = true;
246 : }
247 12 : if (isFftsPlusUnload == false) {
248 10 : const TaskInfo dumpTaskInfo(taskInfo.stream_id, taskInfo.task_id, contextIdTmp, threadIdTmp);
249 10 : UnloadClearTaskInfo(dumpTaskInfo);
250 10 : aicpusd_info(
251 : "Unload op mapping info, stream id[%u], task id[%u], streamIdTmp[%u], taskIdTmp[%u],"
252 : " context id[%u], threadIdTmp[%u].",
253 : streamId, taskId, taskInfo.stream_id, taskInfo.task_id, contextIdTmp, threadIdTmp);
254 : }
255 12 : }
256 12 : MappingInfoOptionalParam optionalParam;
257 12 : GetOptionalParam(opMappingInfo, optionalParam);
258 12 : if (optionalParam.hasModelId) {
259 12 : const auto iter = modelIdToTask_.find(optionalParam.modelId);
260 12 : if (iter == modelIdToTask_.end()) {
261 1 : aicpusd_warn("no model id[%u], unload dump model failed", optionalParam.modelId);
262 1 : return AICPU_SCHEDULE_OK;
263 : }
264 25 : for (const auto taskInfo : iter->second) {
265 14 : UnloadClearTaskInfo(taskInfo);
266 14 : aicpusd_info(
267 : "Check and clean data dump task resource, stream id[%u], task id[%u]"
268 : " context id[%u], thread id [%u].",
269 : taskInfo.streamId_, taskInfo.taskId_, taskInfo.contextId_, taskInfo.threadId_);
270 : }
271 11 : (void)modelIdToTask_.erase(iter);
272 11 : aicpusd_info("Unloaded model id[%u] successfully", optionalParam.modelId);
273 : }
274 11 : return AICPU_SCHEDULE_OK;
275 12 : }
276 :
277 33 : int32_t OpDumpTaskManager::LoadOpMappingInfo(const char_t* const infoAddr, const uint32_t len)
278 : {
279 33 : AicpuSqeAdapter aicpuSqeAdapter(0U);
280 66 : return LoadOpMappingInfo(infoAddr, len, aicpuSqeAdapter);
281 33 : }
282 :
283 60 : int32_t OpDumpTaskManager::LoadOpMappingInfo(
284 : const char_t* const infoAddr, const uint32_t len, AicpuSqeAdapter& aicpuSqeAdapter)
285 : {
286 60 : uint32_t streamId = INVALID_VAL;
287 60 : uint64_t taskId = INVALID_VAL;
288 60 : if (!aicpuSqeAdapter.IsAdapterInvalidParameter()) {
289 8 : AicpuSqeAdapter::AicpuDataDumpInfoLoad info{};
290 8 : aicpuSqeAdapter.GetAicpuDataDumpInfoLoad(info);
291 8 : streamId = info.stream_id;
292 8 : taskId = info.task_id;
293 : }
294 60 : aicpusd_info("Load op mapping info from ts, size[%u], streamId[%u], taskId[%u]", len, streamId, taskId);
295 60 : if (infoAddr == nullptr) {
296 4 : aicpusd_err("op mapping info addr is null");
297 4 : return AICPU_SCHEDULE_ERROR_DUMP_FAILED;
298 : }
299 56 : aicpu::dump::OpMappingInfo opMappingInfo;
300 56 : const std::string protoInfo(infoAddr, static_cast<uint64_t>(len));
301 56 : const bool parseRet = opMappingInfo.ParseFromString(protoInfo);
302 56 : if (!parseRet) {
303 3 : aicpusd_err("parse op mapping info failed");
304 3 : return AICPU_SCHEDULE_ERROR_DUMP_FAILED;
305 : }
306 53 : if (opMappingInfo.flag() == 0x01U) {
307 40 : return Load(opMappingInfo, aicpuSqeAdapter);
308 13 : } else if (opMappingInfo.flag() == 0x00U) {
309 12 : return Unload(opMappingInfo, aicpuSqeAdapter);
310 : } else {
311 1 : aicpusd_err("Flag [%d] invalid, allow [0x00, 0x01]", opMappingInfo.flag());
312 1 : return AICPU_SCHEDULE_ERROR_DUMP_FAILED;
313 : }
314 : return AICPU_SCHEDULE_OK;
315 56 : }
316 :
317 11 : int32_t OpDumpTaskManager::DumpOpInfo(
318 : const uint32_t streamId, const uint32_t taskId, const uint32_t streamId1, const uint32_t taskId1)
319 : {
320 11 : TaskInfoExt dumpTaskInfo;
321 11 : dumpTaskInfo.streamId_ = streamId;
322 11 : dumpTaskInfo.taskId_ = taskId;
323 11 : dumpTaskInfo.contextId_ = INVALID_VAL;
324 11 : dumpTaskInfo.threadId_ = INVALID_VAL;
325 11 : bool isDebug = ((streamId1 != INVALID_VAL) && (taskId1 != INVALID_VAL));
326 11 : uint32_t fileNameStreamId = isDebug ? streamId1 : streamId;
327 11 : uint32_t fileNameTaskId = isDebug ? taskId1 : taskId;
328 11 : DumpFileName dumpFileName(fileNameStreamId, fileNameTaskId);
329 22 : return DumpOpInfo(dumpTaskInfo, dumpFileName);
330 : }
331 :
332 5 : int32_t OpDumpTaskManager::DumpOpInfo(
333 : TaskInfoExt& dumpTaskInfo, const uint32_t streamId, const uint32_t taskId, const uint32_t contextId,
334 : const uint32_t threadId)
335 : {
336 5 : DumpFileName dumpFileName(streamId, taskId, contextId, threadId);
337 10 : return DumpOpInfo(dumpTaskInfo, dumpFileName);
338 : }
339 :
340 55 : int32_t OpDumpTaskManager::DumpOpInfo(TaskInfoExt& dumpTaskInfo, const DumpFileName& dumpFileName)
341 : {
342 55 : TaskInfo curTaskInfo(dumpTaskInfo.streamId_, dumpTaskInfo.taskId_, dumpTaskInfo.contextId_, dumpTaskInfo.threadId_);
343 55 : dumpTaskMapMtx_.lock();
344 55 : const auto range = dumpTaskMap_.equal_range(curTaskInfo);
345 55 : if (range.first == range.second) {
346 30 : dumpTaskMapMtx_.unlock();
347 30 : aicpusd_warn(
348 : "task required to dump does not exist, stream id[%u], task id[%u], "
349 : "context id[%u], thread id[%u].",
350 : dumpTaskInfo.streamId_, dumpTaskInfo.taskId_, dumpTaskInfo.contextId_, dumpTaskInfo.threadId_);
351 30 : return AICPU_SCHEDULE_OK;
352 : } else {
353 25 : std::vector<std::shared_ptr<OpDumpTask>> opDumptasks;
354 61 : for (auto item = range.first; item != range.second; ++item) {
355 36 : std::shared_ptr<OpDumpTask> opDumpTask = item->second;
356 36 : opDumptasks.push_back(std::move(opDumpTask));
357 36 : }
358 25 : int32_t ret = AICPU_SCHEDULE_OK;
359 25 : dumpTaskMapMtx_.unlock();
360 25 : aicpusd_run_info(
361 : "require to dump op info, stream id=%u, task id=%u, context id=%u, "
362 : "thread id=%u, task number=%zu",
363 : dumpTaskInfo.streamId_, dumpTaskInfo.taskId_, dumpTaskInfo.contextId_, dumpTaskInfo.threadId_,
364 : opDumptasks.size());
365 25 : uint32_t indexId = 0;
366 60 : for (auto item : opDumptasks) {
367 36 : auto startTime = std::chrono::steady_clock::now();
368 36 : const KfcDumpTask kfcDumpTaskinfo(dumpTaskInfo.streamId_, dumpTaskInfo.taskId_, indexId);
369 36 : MakeDumpOpInfoforKfc(kfcDumpTaskinfo, item);
370 36 : dumpTaskInfo.indexId_ = indexId;
371 36 : aicpusd_run_info(
372 : "start to dump op info, stream id=%u, task id=%u, context id=%u, "
373 : "thread id=%u, op name=%s",
374 : dumpTaskInfo.streamId_, dumpTaskInfo.taskId_, dumpTaskInfo.contextId_, dumpTaskInfo.threadId_,
375 : item->GetOpName().c_str());
376 36 : ret = item->DumpOpInfo(dumpTaskInfo, dumpFileName);
377 36 : auto endTime = std::chrono::steady_clock::now();
378 36 : double drUs = std::chrono::duration<double, std::micro>(endTime - startTime).count();
379 36 : aicpusd_run_info(
380 : "end of dump op info, result=%d, stream id=%u, task id=%u, "
381 : "context id=%u, thread id=%u, op name=%s, cost time is [%.2lf]us",
382 : ret, dumpTaskInfo.streamId_, dumpTaskInfo.taskId_, dumpTaskInfo.contextId_, dumpTaskInfo.threadId_,
383 : item->GetOpName().c_str(), drUs);
384 : UNUSED(drUs);
385 36 : if (ret != AICPU_SCHEDULE_OK) {
386 1 : ClearKfcDumpTaskInfo(kfcDumpTaskinfo);
387 1 : return ret;
388 : }
389 35 : indexId++;
390 35 : ClearKfcDumpTaskInfo(kfcDumpTaskinfo);
391 36 : }
392 : // process if task is end graph
393 24 : ProcessEndGraph(opDumptasks);
394 25 : }
395 24 : return AICPU_SCHEDULE_OK;
396 : }
397 :
398 12 : static bool CheckAndGetDumpBitmap(MappingInfoOptionalParam& optionalParam, uint64_t& switchBitMap)
399 : {
400 : // 检查是否有有效的dump开关
401 12 : if (!optionalParam.hasDumpSwitch) {
402 10 : aicpusd_info("Get op mapping info: no dump switch field.");
403 10 : return false;
404 : }
405 : // 检查dumpSwitchAddr是否有效
406 2 : if (optionalParam.dumpSwitchAddr == nullptr) {
407 1 : aicpusd_run_info("Get op mapping info: dump switch field exists but address is null.");
408 1 : return false;
409 : }
410 1 : switchBitMap = *(optionalParam.dumpSwitchAddr);
411 1 : aicpusd_info("Get op mapping info dump bitmap is [%llx].", switchBitMap);
412 1 : return true;
413 : }
414 :
415 14 : int32_t OpDumpTaskManager::DumpOpInfoForUnknowShape(
416 : const uint64_t opMappingInfoAddr, const uint64_t opMappingInfoLen) const
417 : {
418 14 : aicpusd_info("load op mapping info from dump task,value[%llu], size[%llu]", opMappingInfoAddr, opMappingInfoLen);
419 14 : if (opMappingInfoAddr == 0U) {
420 1 : aicpusd_err("op mapping info addr is null");
421 1 : return AICPU_SCHEDULE_ERROR_DUMP_FAILED;
422 : }
423 13 : aicpu::dump::OpMappingInfo opMappingInfo;
424 13 : const std::string protoInfo(PtrToPtr<const void, const char_t>(ValueToPtr(opMappingInfoAddr)), opMappingInfoLen);
425 13 : const bool parseRet = opMappingInfo.ParseFromString(protoInfo);
426 13 : if (!parseRet) {
427 1 : aicpusd_err("parse op mapping info failed, size[%u]", opMappingInfoLen);
428 1 : return AICPU_SCHEDULE_ERROR_DUMP_FAILED;
429 : }
430 12 : MappingInfoOptionalParam optionalParam;
431 12 : GetOptionalParam(opMappingInfo, optionalParam);
432 12 : uint64_t switchBitMap = 0UL;
433 12 : if (!CheckAndGetDumpBitmap(optionalParam, switchBitMap)) {
434 11 : return DoDump(opMappingInfo, optionalParam);
435 : } else {
436 1 : return DoDumpBySwitchBitmap(opMappingInfo, optionalParam, switchBitMap);
437 : }
438 13 : }
439 :
440 12 : int32_t OpDumpTaskManager::DoDump(
441 : const aicpu::dump::OpMappingInfo& opMappingInfo, const MappingInfoOptionalParam& optionalParam) const
442 : {
443 12 : const int32_t taskSize = opMappingInfo.task_size();
444 12 : if (taskSize != 1) {
445 2 : aicpusd_err(
446 : "task number[%d] should be only one, op mapping info: %s", opMappingInfo.task_size(),
447 : opMappingInfo.DebugString().c_str());
448 2 : return AICPU_SCHEDULE_ERROR_DUMP_FAILED;
449 : }
450 10 : const std::string dumpPath = opMappingInfo.dump_path();
451 10 : const std::string dumpStepStr = opMappingInfo.dump_step();
452 :
453 10 : DumpStep dumpStep;
454 10 : if (!GetDumpStepFromString(dumpStepStr, dumpStep)) {
455 0 : return AICPU_SCHEDULE_ERROR_DUMP_FAILED;
456 : }
457 10 : const int32_t hostPid = static_cast<int32_t>(AicpuSchedule::AicpuDrvManager::GetInstance().GetHostPid());
458 10 : const uint32_t deviceId = AicpuSchedule::AicpuDrvManager::GetInstance().GetDeviceId();
459 10 : std::shared_ptr<OpDumpTask> opDumpTaskPtr = nullptr;
460 10 : int32_t ret = CreateOpDumpTask(opDumpTaskPtr, hostPid, deviceId);
461 10 : if (ret != AICPU_SCHEDULE_OK) {
462 2 : aicpusd_err("Create opDumpTask failed, hostPid[%d], deviceId[%u]", hostPid, deviceId);
463 2 : return ret;
464 : }
465 8 : aicpusd_memory_log("MallocMemory, func=new, size=%zu, purpose=data dumper", sizeof(OpDumpTask));
466 8 : if (opDumpTaskPtr == nullptr) {
467 2 : aicpusd_err("malloc memory for OpDumpTask object failed");
468 2 : return AICPU_SCHEDULE_ERROR_DUMP_FAILED;
469 : }
470 6 : const aicpu::dump::Task task = opMappingInfo.task(0);
471 6 : DumpMode dumpMode = opMappingInfo.dump_data();
472 6 : aicpusd_info(
473 : "skipAddressConversion[true], hasStepId[%u], task_id[%u], stream_id[%u] dumpMode[%d]",
474 : static_cast<uint32_t>(optionalParam.hasStepId), task.task_id(), task.stream_id(),
475 : static_cast<int32_t>(dumpMode));
476 6 : ret = opDumpTaskPtr->PreProcessOpMappingInfo(task, dumpPath, optionalParam, dumpStep, dumpMode, true);
477 6 : if (ret != AICPU_SCHEDULE_OK) {
478 2 : aicpusd_err("pre process op mapping info failed, op mapping info: %s", opMappingInfo.DebugString().c_str());
479 2 : return ret;
480 : }
481 4 : AicpuSqeAdapter adapter(FeatureCtrl::GetTsMsgVersion());
482 : AicpuSqeAdapter::AicpuOpMappingDumpTaskInfo opmappingInfo(
483 4 : task.task_id(), task.stream_id(), INVALID_VAL, INVALID_VAL);
484 4 : AicpuSqeAdapter::AicpuDumpTaskInfo taskInfo{};
485 4 : adapter.GetAicpuDumpTaskInfo(opmappingInfo, taskInfo);
486 4 : auto startTime = std::chrono::steady_clock::now();
487 4 : aicpusd_run_info("start to dump op info, op name=%s", opDumpTaskPtr->GetOpName().c_str());
488 4 : const KfcDumpTask kfcDumpTaskinfo(taskInfo.stream_id, taskInfo.task_id, 0);
489 4 : const_cast<OpDumpTaskManager*>(this)->MakeDumpOpInfoforKfc(kfcDumpTaskinfo, opDumpTaskPtr);
490 4 : const TaskInfoExt dumpTaskInfo(taskInfo.stream_id, taskInfo.task_id, INVALID_VAL, INVALID_VAL, 0);
491 4 : DumpFileName dumpFileName(taskInfo.stream_id, taskInfo.task_id);
492 4 : ret = opDumpTaskPtr->DumpOpInfo(dumpTaskInfo, dumpFileName);
493 4 : auto endTime = std::chrono::steady_clock::now();
494 4 : double drUs = std::chrono::duration<double, std::micro>(endTime - startTime).count();
495 4 : aicpusd_run_info(
496 : "end of dump op info, result=%d, op name=%s, cost time is [%.2lf]us", ret, opDumpTaskPtr->GetOpName().c_str(),
497 : drUs);
498 : UNUSED(drUs);
499 4 : opDumpTaskPtr->ClearBaseDumpData();
500 4 : const_cast<OpDumpTaskManager*>(this)->ClearKfcDumpTaskInfo(kfcDumpTaskinfo);
501 4 : return ret;
502 10 : }
503 :
504 2 : int32_t OpDumpTaskManager::GetAndClearOverflowStatus(
505 : const uint32_t deviceId, const uint32_t streamId, const uint32_t opType, uint32_t* status) const
506 : {
507 2 : TsCtrlMsgBody queryIn = {};
508 2 : queryIn.type = opType;
509 2 : queryIn.u.query_stream_overflow_status.stream_id = streamId;
510 2 : TsCtrlMsgBody queryResult = {};
511 2 : size_t queryResultCount = sizeof(TsCtrlMsgBody);
512 : TsDrvCtrlMsg para;
513 2 : para.tsid = 0U;
514 2 : para.msg_len = sizeof(TsCtrlMsgBody);
515 2 : para.msg = static_cast<void*>(&queryIn);
516 2 : if (!EnsureDeviceOpened(deviceId)) {
517 0 : aicpusd_err(
518 : "The open device interface return failed, device id is %u, stream id is %u, op type is %u.", deviceId,
519 : streamId, opType);
520 0 : return AICPU_SCHEDULE_ERROR_FROM_DRV;
521 : }
522 2 : const drvError_t drvRet = halTsdrvCtl(
523 : deviceId, TSDRV_CTL_CMD_CTRL_MSG, static_cast<void*>(¶), sizeof(TsDrvCtrlMsg),
524 : static_cast<void*>(&queryResult), &queryResultCount);
525 2 : if (drvRet != DRV_ERROR_NONE) {
526 0 : aicpusd_err(
527 : "The device id is %u, stream id is %u, op type is %u, return result is %d.", deviceId, streamId, opType,
528 : static_cast<int32_t>(drvRet));
529 0 : return static_cast<int32_t>(drvRet);
530 : }
531 2 : *status = queryResult.u.query_stream_overflow_status.status;
532 2 : aicpusd_run_info(
533 : "The device id is %u, stream id is %u, op type is %u, overflow status result is %u.", deviceId, streamId,
534 : opType, queryResult.u.query_stream_overflow_status.status);
535 2 : return AICPU_SCHEDULE_OK;
536 : }
537 :
538 2 : bool OpDumpTaskManager::EnsureDeviceOpened(const uint32_t deviceId) const
539 : {
540 : static std::once_flag drvOpenflag;
541 : static bool isDeviceOpen = false;
542 2 : std::call_once(drvOpenflag, [&]() {
543 1 : struct drvDevInfo devInfo = {0};
544 1 : auto ret = drvDeviceOpen(PtrToPtr<void, void*>(&devInfo), deviceId);
545 1 : if ((ret != DRV_ERROR_NONE) && (ret != DRV_ERROR_REPEATED_INIT)) {
546 0 : aicpusd_err("Device %u open failed, ret=%d", deviceId, static_cast<int32_t>(ret));
547 0 : isDeviceOpen = false;
548 : } else {
549 1 : aicpusd_info("Device %u opened successfully", deviceId);
550 1 : isDeviceOpen = true;
551 : }
552 1 : });
553 2 : return isDeviceOpen;
554 : }
555 :
556 5 : int32_t OpDumpTaskManager::DoDumpBySwitchBitmap(
557 : const aicpu::dump::OpMappingInfo& opMappingInfo, const MappingInfoOptionalParam& optionalParam,
558 : const uint64_t switchBitMap) const
559 : {
560 : // 全0表示关闭dump开关，或者溢出检测开关开启但是halTsdrvCtl接口或者drvDeviceOpen不存在，不做dump处理
561 5 : if ((switchBitMap == 0UL) ||
562 5 : ((switchBitMap & OVERFLOW_DUMP_BIT) != 0UL && ((&halTsdrvCtl == nullptr) || (&drvDeviceOpen == nullptr)))) {
563 0 : aicpusd_run_info(
564 : "Get op mapping info dump bitmap is [%llx], halTsdrvCtl available is [%d], drvDeviceOpen available is "
565 : "[%d].",
566 : switchBitMap, &halTsdrvCtl == nullptr, &drvDeviceOpen == nullptr);
567 0 : return AICPU_SCHEDULE_OK;
568 : }
569 5 : DumpMode dumpMode = DumpMode::TENSOR_DUMP_DATA;
570 5 : if ((switchBitMap & STATS_DUMP_BIT) != 0UL) {
571 1 : dumpMode = DumpMode::STATS_DUMP_DATA;
572 : }
573 5 : if ((switchBitMap & TENSOR_DUMP_BIT) != 0UL) {
574 2 : dumpMode = DumpMode::TENSOR_DUMP_DATA;
575 : }
576 5 : const std::string dumpPath = opMappingInfo.dump_path();
577 5 : const std::string dumpStepStr = opMappingInfo.dump_step();
578 5 : DumpStep dumpStep;
579 5 : if (!GetDumpStepFromString(dumpStepStr, dumpStep)) {
580 0 : return AICPU_SCHEDULE_ERROR_DUMP_FAILED;
581 : }
582 5 : const int32_t hostPid = static_cast<int32_t>(AicpuSchedule::AicpuDrvManager::GetInstance().GetHostPid());
583 5 : const uint32_t deviceId = AicpuSchedule::AicpuDrvManager::GetInstance().GetDeviceId();
584 5 : const int32_t taskSize = opMappingInfo.task_size();
585 5 : if (taskSize > MAX_TASK_SIZE) {
586 0 : aicpusd_run_info(
587 : "Task number[%d] exceeds 2, op mapping info: %s", taskSize, opMappingInfo.DebugString().c_str());
588 : }
589 5 : int32_t ret = AICPU_SCHEDULE_OK;
590 9 : for (int32_t i = 0; i < taskSize; i++) {
591 5 : const aicpu::dump::Task task = opMappingInfo.task(i);
592 5 : std::shared_ptr<OpDumpTask> opDumpTaskPtr = nullptr;
593 5 : ret = CreateOpDumpTask(opDumpTaskPtr, hostPid, deviceId);
594 5 : if (ret != AICPU_SCHEDULE_OK) {
595 0 : aicpusd_err("Create opDumpTask failed, hostPid[%d], deviceId[%u].", hostPid, deviceId);
596 0 : return ret;
597 : }
598 5 : aicpusd_memory_log("MallocMemory, func=new, size=%zu, purpose=data dumper.", sizeof(OpDumpTask));
599 5 : if (opDumpTaskPtr == nullptr) {
600 0 : aicpusd_err("Malloc memory for OpDumpTask object failed.");
601 0 : return AICPU_SCHEDULE_ERROR_DUMP_FAILED;
602 : }
603 5 : if ((switchBitMap & OVERFLOW_DUMP_BIT) != 0UL) {
604 2 : uint32_t overflowStatus = 0U;
605 : const int32_t getRet =
606 2 : GetAndClearOverflowStatus(deviceId, task.stream_id(), OP_QUERY_OVERFLOW, &overflowStatus);
607 2 : if (getRet != AICPU_SCHEDULE_OK) {
608 0 : aicpusd_err(
609 : "Query overflow status interface returned: ret is %d, device id is %u, stream id is %u.", getRet,
610 : deviceId, task.stream_id());
611 1 : return AICPU_SCHEDULE_ERROR_FROM_DRV;
612 : }
613 2 : if (overflowStatus == 0U) { // result 为0表示不溢出，不做dump处理直接返回
614 1 : aicpusd_run_info(
615 : "The query overflow status is %u, the device id is %u, and the stream id is %u.", overflowStatus,
616 : deviceId, task.stream_id());
617 1 : return AICPU_SCHEDULE_OK;
618 : }
619 : }
620 4 : aicpusd_info(
621 : "Dump the parameter information : single or unknown shape flag[false], has step id[%u], task id[%u], "
622 : "stream id[%u], dump mode[%d].",
623 : static_cast<uint32_t>(optionalParam.hasStepId), task.task_id(), task.stream_id(),
624 : static_cast<int32_t>(dumpMode));
625 4 : ret = opDumpTaskPtr->PreProcessOpMappingInfo(task, dumpPath, optionalParam, dumpStep, dumpMode, false);
626 4 : if (ret != AICPU_SCHEDULE_OK) {
627 0 : aicpusd_err(
628 : "Preprocess op mapping info failed. Op mapping info is %s", opMappingInfo.DebugString().c_str());
629 0 : return ret;
630 : }
631 4 : AicpuSqeAdapter adapter(static_cast<int16_t>(FeatureCtrl::GetTsMsgVersion()));
632 : AicpuSqeAdapter::AicpuOpMappingDumpTaskInfo opmappingInfo(
633 4 : task.task_id(), task.stream_id(), INVALID_VAL, INVALID_VAL);
634 4 : AicpuSqeAdapter::AicpuDumpTaskInfo taskInfo{};
635 4 : adapter.GetAicpuDumpTaskInfo(opmappingInfo, taskInfo);
636 4 : auto startTime = std::chrono::steady_clock::now();
637 4 : aicpusd_run_info("Start dumping operation information: op name is %s.", opDumpTaskPtr->GetOpName().c_str());
638 4 : const KfcDumpTask kfcDumpTaskinfo(taskInfo.stream_id, taskInfo.task_id, 0);
639 4 : const_cast<OpDumpTaskManager*>(this)->MakeDumpOpInfoforKfc(kfcDumpTaskinfo, opDumpTaskPtr);
640 4 : const TaskInfoExt dumpTaskInfo(taskInfo.stream_id, taskInfo.task_id, INVALID_VAL, INVALID_VAL, 0);
641 4 : DumpFileName dumpFileName(taskInfo.stream_id, taskInfo.task_id);
642 4 : ret = opDumpTaskPtr->DumpOpInfo(dumpTaskInfo, dumpFileName);
643 4 : auto endTime = std::chrono::steady_clock::now();
644 4 : const double drUs = std::chrono::duration<double, std::micro>(endTime - startTime).count();
645 4 : aicpusd_run_info(
646 : "End of dump operation information: result is %d, op name is %s, cost time is [%.2lf]us.", ret,
647 : opDumpTaskPtr->GetOpName().c_str(), drUs);
648 : UNUSED(drUs);
649 4 : opDumpTaskPtr->ClearBaseDumpData();
650 4 : const_cast<OpDumpTaskManager*>(this)->ClearKfcDumpTaskInfo(kfcDumpTaskinfo);
651 6 : }
652 4 : return ret;
653 5 : }
654 :
655 24 : void OpDumpTaskManager::ProcessEndGraph(const std::vector<std::shared_ptr<OpDumpTask>>& opDumptasks)
656 : {
657 58 : for (const auto& item : opDumptasks) {
658 34 : if (item->IsEndGraph()) {
659 : uint32_t modelId;
660 4 : if (item->GetModelId(modelId)) {
661 3 : UpdateDumpNumByModelId(modelId);
662 : } else {
663 1 : item->UpdateDumpNum();
664 : }
665 : }
666 : }
667 24 : }
668 :
669 3 : void OpDumpTaskManager::UpdateDumpNumByModelId(const uint32_t modelId)
670 : {
671 3 : std::vector<std::shared_ptr<OpDumpTask>> opDumptasks;
672 : {
673 3 : const std::lock_guard<std::mutex> mapLock(dumpTaskMapMtx_);
674 3 : const auto iter = modelIdToTask_.find(modelId);
675 3 : if (iter != modelIdToTask_.end()) {
676 6 : for (auto& taskInfo : iter->second) {
677 3 : const auto range = dumpTaskMap_.equal_range(taskInfo);
678 12 : for (auto item = range.first; item != range.second; ++item) {
679 9 : std::shared_ptr<OpDumpTask> opDumpTask = item->second;
680 9 : opDumptasks.push_back(std::move(opDumpTask));
681 9 : }
682 : }
683 : }
684 3 : }
685 :
686 12 : for (auto& dumpTask : opDumptasks) {
687 9 : dumpTask->UpdateDumpNum();
688 : }
689 3 : }
690 :
691 7 : static void DoSplit(const std::string& externStr, std::vector<std::string>& ret, const std::string& delim)
692 : {
693 7 : size_t i = 0UL;
694 20 : while (i < externStr.size()) {
695 13 : const size_t pos = externStr.find(delim, i);
696 13 : if (pos != std::string::npos) {
697 13 : std::string s = externStr.substr(i, pos - i);
698 13 : if (!s.empty()) {
699 13 : ret.push_back(std::move(s));
700 : }
701 13 : i = pos + delim.size() - 1UL;
702 13 : }
703 13 : ++i;
704 : }
705 7 : }
706 :
707 51 : static std::vector<std::string> Split(const std::string& str, const std::string& delim)
708 : {
709 51 : std::vector<std::string> ret;
710 51 : if (!str.empty()) {
711 7 : const std::string externStr = str + delim;
712 7 : DoSplit(externStr, ret, delim);
713 7 : }
714 51 : return ret;
715 0 : }
716 :
717 8 : bool OpDumpTaskManager::MatchAndInsert(const std::string& step, DumpStep& tmpDumpStep) const
718 : {
719 : // smatch result
720 8 : const std::regex singleStepPatten("(\\s*)(\\d+)(\\s*)");
721 8 : const std::regex intervalStepPatten("((\\s*)(\\d+)(\\s*)){1}(-(\\s*)(\\d+)(\\s*))");
722 8 : if (std::regex_match(step, singleStepPatten)) {
723 4 : (void)tmpDumpStep.singleStep.insert(static_cast<uint64_t>(std::stoull(step)));
724 4 : } else if (std::regex_match(step, intervalStepPatten)) {
725 3 : const std::vector<std::string> intervalStepStr = Split(step, "-");
726 3 : if (intervalStepStr.size() == INTERNAL_STEP_SIZE) {
727 : IntervalStep intervalStep;
728 3 : intervalStep.start = static_cast<uint64_t>(std::stoull(intervalStepStr[0U]));
729 3 : intervalStep.end = static_cast<uint64_t>(std::stoull(intervalStepStr[1U]));
730 3 : if (intervalStep.start > intervalStep.end) {
731 3 : std::swap(intervalStep.start, intervalStep.end);
732 : }
733 3 : tmpDumpStep.intervalStep.push_back(std::move(intervalStep));
734 : }
735 3 : } else {
736 1 : aicpusd_err("invalid step[%s], please check.", step.c_str());
737 1 : return false;
738 : }
739 6 : return true;
740 9 : }
741 :
742 48 : bool OpDumpTaskManager::GetDumpStepFromString(const std::string& str, DumpStep& dumpStep) const
743 : {
744 : // split by |, such as "0|5-10"
745 48 : aicpusd_info("Step need to dump[%s].", str.c_str());
746 48 : const std::vector<std::string> steps = Split(str, "|");
747 48 : DumpStep tmpDumpStep;
748 54 : for (const auto& step : steps) {
749 : try {
750 7 : if (!MatchAndInsert(step, tmpDumpStep)) {
751 1 : return false;
752 : }
753 1 : } catch (std::out_of_range& e) {
754 1 : aicpusd_err(
755 : "out of range of uint64_max[%llu], invalid step[%s], msg[%s], please check.", UINT64_MAX, step.c_str(),
756 : e.what());
757 1 : return false;
758 1 : } catch (...) {
759 0 : aicpusd_err("invalid step[%s], please check.", step.c_str());
760 0 : return false;
761 0 : }
762 : }
763 47 : dumpStep = std::move(tmpDumpStep);
764 47 : return true;
765 48 : }
766 :
767 3 : void OpDumpTaskManager::ClearResource()
768 : {
769 3 : const std::lock_guard<std::mutex> mapLock(dumpTaskMapMtx_);
770 3 : aicpusd_run_info("clear all resource of data dump");
771 3 : dumpTaskMap_.clear();
772 3 : modelIdToTask_.clear();
773 3 : kfcDumpTaskMap_.clear();
774 3 : kfcDumpInfoMap_.clear();
775 3 : custDumpTaskMap_.clear();
776 3 : }
777 :
778 : /*
779 : * 清除资源
780 : */
781 44 : void OpDumpTaskManager::ClearKfcDumpTaskInfo(const KfcDumpTask& kfcTaskinfo)
782 : {
783 44 : const std::lock_guard<std::mutex> mapLock(kfcDumpTaskMapMtx_);
784 44 : if (kfcDumpTaskMap_.find(kfcTaskinfo) != kfcDumpTaskMap_.end()) {
785 2 : (void)kfcDumpTaskMap_.erase(kfcTaskinfo);
786 2 : aicpusd_info(
787 : "erase key of dump task map : streamId[%u], taskId[%u], index[%u]", kfcTaskinfo.streamId_,
788 : kfcTaskinfo.taskId_, kfcTaskinfo.index_);
789 : }
790 44 : if (kfcDumpInfoMap_.find(kfcTaskinfo) != kfcDumpInfoMap_.end()) {
791 2 : (void)kfcDumpInfoMap_.erase(kfcTaskinfo);
792 2 : aicpusd_info(
793 : "erase key of dump info map : streamId[%u], taskId[%u], index[%u]", kfcTaskinfo.streamId_,
794 : kfcTaskinfo.taskId_, kfcTaskinfo.index_);
795 : }
796 44 : TaskInfo taskInfo(kfcTaskinfo.streamId_, kfcTaskinfo.taskId_);
797 44 : if (custDumpTaskMap_.find(taskInfo) != custDumpTaskMap_.end()) {
798 0 : (void)custDumpTaskMap_.erase(taskInfo);
799 0 : aicpusd_info(
800 : "erase key of cust dump task map : streamId[%u], taskId[%u], index[%u]", kfcTaskinfo.streamId_,
801 : kfcTaskinfo.taskId_, kfcTaskinfo.index_);
802 : }
803 88 : return;
804 44 : }
805 :
806 : /*
807 : * 查询是否是aicpu cust dump 任务
808 : */
809 1 : bool OpDumpTaskManager::IsCustDumpTask(const uint32_t streamId, const uint32_t taskId)
810 : {
811 1 : TaskInfo taskInfo(streamId, taskId);
812 1 : const auto iter = custDumpTaskMap_.find(taskInfo);
813 1 : if (iter == custDumpTaskMap_.end()) {
814 1 : aicpusd_info("Cannot find dump task [streamId:%u, taskId:%u] in cust dump task map.", streamId, taskId);
815 1 : return false;
816 : }
817 0 : aicpusd_info("Get dump task [streamId:%u, taskId:%u] flag is [%d].", streamId, taskId, iter->second);
818 0 : return iter->second;
819 : }
820 :
821 : /*
822 : * 获取是否是aicpu cust dump 任务
823 : */
824 11 : int32_t OpDumpTaskManager::SetCustDumpTaskFlag(const uint32_t streamId, const uint32_t taskId, const bool flag)
825 : {
826 11 : TaskInfo taskInfo(streamId, taskId);
827 11 : custDumpTaskMap_.insert(std::make_pair(taskInfo, flag));
828 11 : aicpusd_info("Set dump task [streamId:%u, taskId:%u] flag [%d] in cust dump task map.", streamId, taskId, flag);
829 11 : return AICPU_SCHEDULE_OK;
830 : }
831 :
832 : /*
833 : * 构造kfc需要的dump信息映射关系{steamid taskid index, task}
834 : */
835 52 : void OpDumpTaskManager::MakeDumpOpInfoforKfc(const KfcDumpTask& taskinfo, std::shared_ptr<OpDumpTask> dumpTask)
836 : {
837 52 : const std::lock_guard<std::mutex> mapLock(kfcDumpTaskMapMtx_);
838 52 : if (!(dumpTask->IsSupportKfcDump())) {
839 47 : aicpusd_info(
840 : "task does not support kfc statistical dump, streamId[%u], taskId[%u].", taskinfo.streamId_,
841 : taskinfo.taskId_);
842 47 : return;
843 : }
844 5 : kfcDumpTaskMap_.insert(std::make_pair(taskinfo, dumpTask));
845 5 : aicpusd_info(
846 : "Make kfc dump task map, streamId[%u], taskId[%u], index[%u]", taskinfo.streamId_, taskinfo.taskId_,
847 : taskinfo.index_);
848 5 : return;
849 52 : }
850 6 : int32_t OpDumpTaskManager::CreateKfcDumpInfo(std::shared_ptr<KfcDumpInfo>& kfcDumpInfoPtr) const
851 : {
852 6 : DATADUMP_MAKE_SHARED(kfcDumpInfoPtr = std::make_shared<KfcDumpInfo>(), return AICPU_SCHEDULE_ERROR_DUMP_FAILED);
853 6 : return AICPU_SCHEDULE_OK;
854 : }
855 : /*
856 : * 向kfc提供dump查询信息接口
857 : */
858 11 : int32_t OpDumpTaskManager::GetDumpOpTaskDataforKfc(const KfcDumpTask& taskKey, KfcDumpInfo** dumpInfo)
859 : {
860 11 : const std::lock_guard<std::mutex> mapLock(kfcDumpTaskMapMtx_);
861 11 : aicpusd_info(
862 : "Get dump info for kfc, streamId[%u], taskId[%u], index[%u].", taskKey.streamId_, taskKey.taskId_,
863 : taskKey.index_);
864 : // dumpInfo 是否为空在上层调用处保证
865 11 : const auto iter = kfcDumpTaskMap_.find(taskKey);
866 11 : if (iter == kfcDumpTaskMap_.end()) {
867 2 : aicpusd_err(
868 : "Cannot find dump task info key for kfc. streamId[%u], taskId[%u], index[%u].", taskKey.streamId_,
869 : taskKey.taskId_, taskKey.index_);
870 2 : return AICPU_SCHEDULE_ERROR_DUMP_FAILED;
871 : }
872 9 : if (iter->second == nullptr) {
873 1 : aicpusd_err(
874 : "Get dump task info failed. streamId[%u], taskId[%u], index[%u]", taskKey.streamId_, taskKey.taskId_,
875 : taskKey.index_);
876 1 : return AICPU_SCHEDULE_ERROR_DUMP_FAILED;
877 : }
878 :
879 8 : std::shared_ptr<KfcDumpInfo> kfcDumpInfoPtr = nullptr;
880 8 : int32_t ret = CreateKfcDumpInfo(kfcDumpInfoPtr);
881 8 : if (ret != AICPU_SCHEDULE_OK) {
882 1 : aicpusd_err(
883 : "Create opDumpTask failed streamId[%u], taskId[%u], index[%u]", taskKey.streamId_, taskKey.taskId_,
884 : taskKey.index_);
885 1 : return ret;
886 : }
887 7 : if (kfcDumpInfoPtr == nullptr) {
888 1 : aicpusd_err("malloc memory for KfcDumpInfo object failed");
889 1 : return AICPU_SCHEDULE_ERROR_DUMP_FAILED;
890 : }
891 6 : iter->second->GetKfcDumpInfo(kfcDumpInfoPtr);
892 6 : kfcDumpInfoMap_.insert(std::make_pair(taskKey, kfcDumpInfoPtr));
893 6 : *dumpInfo = kfcDumpInfoPtr.get();
894 6 : return AICPU_SCHEDULE_OK;
895 11 : }
896 :
897 : /*
898 : * 向kfc提供dump数据接口
899 : */
900 8 : int32_t OpDumpTaskManager::DumpOpTaskDataforKfc(const KfcDumpTask& taskKey, void* dumpData, uint32_t length) const
901 : {
902 8 : aicpusd_info(
903 : "dump op task data for kfc, length [%u], streamId[%u], taskId[%u], index[%u]", length, taskKey.streamId_,
904 : taskKey.taskId_, taskKey.index_);
905 : // dumpData 是否为空在上层调用处保证
906 8 : const auto iter = kfcDumpTaskMap_.find(taskKey);
907 8 : if (iter == kfcDumpTaskMap_.end()) {
908 2 : aicpusd_err(
909 : "Not get dump info for kfc. length [%u], streamId[%u], taskId[%u], index[%u].", length, taskKey.streamId_,
910 : taskKey.taskId_, taskKey.index_);
911 2 : return AICPU_SCHEDULE_ERROR_DUMP_FAILED;
912 : }
913 6 : if (iter->second == nullptr) {
914 1 : aicpusd_err(
915 : "Get dump task failed. length [%u], streamId[%u], taskId[%u], index[%u]", length, taskKey.streamId_,
916 : taskKey.taskId_, taskKey.index_);
917 1 : return AICPU_SCHEDULE_ERROR_DUMP_FAILED;
918 : }
919 5 : const std::string filePath = iter->second->GetDumpPath();
920 5 : const std::string dumpFilePath = filePath + ".bin";
921 5 : const int32_t hostPid = iter->second->GetHostPid();
922 5 : const uint32_t deviceId = iter->second->GetDeviceId();
923 5 : const std::string opName = iter->second->GetOpName();
924 :
925 5 : IDE_SESSION ideSession = DumpSessionManager::GetInstance().GetSession(hostPid, deviceId);
926 5 : if (ideSession == nullptr) {
927 0 : aicpusd_err(
928 : "op name[%s], call IdeDumpStart failed, path[%s]. length [%u], streamId[%u], taskId[%u], index[%u]",
929 : opName.c_str(), dumpFilePath.c_str(), length, taskKey.streamId_, taskKey.taskId_, taskKey.index_);
930 0 : return AICPU_SCHEDULE_ERROR_DUMP_FAILED;
931 : }
932 :
933 5 : return iter->second->Dump(
934 5 : dumpFilePath, PtrToPtr<void, char>(dumpData), static_cast<uint64_t>(length), ideSession, true);
935 5 : }
936 : } // namespace AicpuSchedule
|
