Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 : #include "aicpu_cust_sd_dump_process.h"
11 :
12 : #include <string.h>
13 : #include <sched.h>
14 : #include <semaphore.h>
15 : #include "aicpusd_drv_manager.h"
16 : #include "aicpusd_info.h"
17 : #include "aicpusd_util.h"
18 : #include "aicpu_cust_sd_proc_mgr_sys_operator_agent.h"
19 : #include "aicpusd_interface_process.h"
20 : #include "aicpu_cust_platform_info_process.h"
21 :
22 : namespace AicpuSchedule {
23 : namespace {
24 : constexpr const uint32_t DATA_DUMP_GRUOP_ID = 31U;
25 : constexpr const uint32_t DATA_DUMP_THREAD_INDEX = 0U;
26 : constexpr const uint64_t DATA_DUMP_EVENT_MASK = (1ULL << static_cast<uint32_t>(EVENT_CCPU_CTRL_MSG));
27 : constexpr const int32_t DATA_DUMP_TIMEOUT_INTERVAL = 3000;
28 : constexpr const uint32_t SLEEP_USECS = 50000U;
29 : };
30 :
31 85 : AicpuCustDumpProcess &AicpuCustDumpProcess::GetInstance()
32 : {
33 85 : static AicpuCustDumpProcess instance;
34 85 : return instance;
35 : }
36 :
37 1 : AicpuCustDumpProcess::~AicpuCustDumpProcess()
38 : {
39 1 : UnitDataDumpProcess();
40 1 : }
41 :
42 1 : AicpuCustDumpProcess::AicpuCustDumpProcess()
43 1 : : deviceId_(0U), runningFlag_(false), initFlag_(false)
44 : {
45 1 : }
46 :
47 10 : int32_t AicpuCustDumpProcess::InitDumpProcess(const uint32_t deviceId, const uint32_t workCnt)
48 : {
49 10 : const std::lock_guard<std::mutex> lk(initMutex_);
50 : uint32_t runMode;
51 10 : const aicpu::status_t status = aicpu::GetAicpuRunMode(runMode);
52 10 : if (status != aicpu::AICPU_ERROR_NONE) {
53 1 : aicpusd_err("Get current aicpu ctx failed.");
54 1 : return AICPU_SCHEDULE_ERROR_INNER_ERROR;
55 : }
56 9 : if (runMode != aicpu::AicpuRunMode::PROCESS_PCIE_MODE) {
57 1 : aicpusd_info("not pcie mode no need this thread");
58 1 : return AICPU_SCHEDULE_OK;
59 : }
60 8 : if ((initFlag_) || (workCnt == 0U)) {
61 5 : aicpusd_info("workcnt:%u", workCnt);
62 5 : return AICPU_SCHEDULE_OK;
63 : }
64 3 : initFlag_ = true;
65 3 : deviceId_ = deviceId;
66 3 : runningFlag_ = true;
67 3 : if (InitWaitConVec(workCnt) != AICPU_SCHEDULE_OK) {
68 1 : aicpusd_err("InitWaitConVec init failed");
69 1 : return AICPU_SCHEDULE_ERROR_INNER_ERROR;
70 : }
71 :
72 2 : const int32_t semRet = sem_init(&workerSme_, 0, 0U);
73 2 : if (semRet == -1) {
74 1 : aicpusd_err("workerSme_ init failed, %s", strerror(errno));
75 1 : return AICPU_SCHEDULE_ERROR_INIT_FAILED;
76 : }
77 :
78 : try {
79 1 : msgThread_ = std::thread(&AicpuCustDumpProcess::StartProcessEvent, this);
80 0 : } catch (std::exception &e) {
81 0 : aicpusd_err("create thread file:%s", e.what());
82 0 : return AICPU_SCHEDULE_ERROR_INIT_FAILED;
83 0 : }
84 1 : const int32_t semWaitRet = sem_wait(&workerSme_);
85 1 : if (semWaitRet == -1) {
86 0 : aicpusd_err("workerSme wait failed, %s", strerror(errno));
87 0 : return AICPU_SCHEDULE_ERROR_INIT_FAILED;
88 : }
89 1 : return AICPU_SCHEDULE_OK;
90 10 : }
91 :
92 8 : int32_t AicpuCustDumpProcess::SetDataDumpThreadAffinity() const
93 : {
94 8 : std::vector<uint32_t> ccpuIds = AicpuDrvManager::GetInstance().GetCcpuList();
95 8 : if (ccpuIds.empty()) {
96 0 : aicpusd_run_info("ccpu list is empty no need bind core");
97 0 : return AICPU_SCHEDULE_OK;
98 : }
99 16 : if (AicpuUtil::IsEnvValEqual(ENV_NAME_PROCMGR_AICPU_CPUSET, "1")) {
100 2 : const pid_t tid = static_cast<pid_t>(GetTid());
101 2 : auto ret = ProcMgrBindThread(tid, ccpuIds);
102 2 : uint32_t tryTimes = 0;
103 4 : while ((ret != 0U) && (tryTimes <= 1)) {
104 2 : aicpusd_warn("set affinity failed ret[%d], will try again", ret);
105 2 : (void)usleep(SLEEP_USECS);
106 2 : ret = ProcMgrBindThread(tid, ccpuIds);
107 2 : tryTimes++;
108 : }
109 2 : if (ret != 0) {
110 1 : aicpusd_err("aicpu bind tid by pm res[%d].", ret);
111 1 : return AICPU_SCHEDULE_ERROR_INNER_ERROR;
112 : } else {
113 1 : aicpusd_run_info("aicpu bind by pm success");
114 1 : return AICPU_SCHEDULE_OK;
115 : }
116 : } else {
117 6 : pthread_t threadId = pthread_self();
118 : cpu_set_t cpuset;
119 6 : CPU_ZERO(&cpuset);
120 48 : for (const auto cpuId: ccpuIds) {
121 42 : CPU_SET(cpuId, &cpuset);
122 42 : aicpusd_info("prepare bind threadId=%lu to cpuId=%u", threadId, static_cast<uint32_t>(cpuId));
123 : }
124 6 : const int32_t ret = pthread_setaffinity_np(threadId, sizeof(cpu_set_t), &cpuset);
125 6 : if (ret != 0) {
126 1 : aicpusd_run_warn(
127 : "Binding datadump thread to control CPU was not successful, will run in aicpu. ret=%d, reason=%s",
128 : ret, strerror(errno));
129 : } else {
130 5 : aicpusd_run_info("aicpu bind by self success");
131 : }
132 : }
133 :
134 6 : return AICPU_SCHEDULE_OK;
135 8 : }
136 :
137 5 : void AicpuCustDumpProcess::StartProcessEvent()
138 : {
139 5 : int32_t ret = halEschedCreateGrp(deviceId_, DATA_DUMP_GRUOP_ID, GRP_TYPE_BIND_CP_CPU);
140 5 : if (ret != DRV_ERROR_NONE) {
141 1 : aicpusd_err("halEschedCreateGrp dev[%u] group[%u] type[%d] failed, result[%d].",
142 : deviceId_, DATA_DUMP_GRUOP_ID, GRP_TYPE_BIND_CP_CPU, ret);
143 1 : sem_post(&workerSme_);
144 1 : return;
145 : }
146 :
147 4 : if (SetDataDumpThreadAffinity() != AICPU_SCHEDULE_OK) {
148 1 : aicpusd_err("datadump thread bind core failed");
149 1 : sem_post(&workerSme_);
150 1 : return;
151 : }
152 3 : ret = halEschedSubscribeEvent(deviceId_, DATA_DUMP_GRUOP_ID, DATA_DUMP_THREAD_INDEX, DATA_DUMP_EVENT_MASK);
153 3 : if (ret != DRV_ERROR_NONE) {
154 1 : aicpusd_err("halEschedSubscribeEvent failed, deviceId[%u], groupId[%u], threadIndex[%u]",
155 : deviceId_, DATA_DUMP_GRUOP_ID, DATA_DUMP_THREAD_INDEX);
156 1 : sem_post(&workerSme_);
157 1 : return;
158 : }
159 2 : ProcessMessage(0U);
160 2 : sem_post(&workerSme_);
161 2 : aicpusd_run_info("post work sem finish, dump process init finish. deviceId[%u]", deviceId_);
162 2 : LoopProcessEvent();
163 2 : aicpusd_run_info("dump thread stopped");
164 : }
165 2 : void AicpuCustDumpProcess::LoopProcessEvent()
166 : {
167 29 : while (runningFlag_) {
168 27 : ProcessMessage(DATA_DUMP_TIMEOUT_INTERVAL);
169 : }
170 2 : }
171 :
172 6 : int32_t AicpuCustDumpProcess::InitWaitConVec(const uint32_t workCnt)
173 : {
174 6 : waitCondVec_.resize(workCnt);
175 17 : for (size_t index = 0; index < waitCondVec_.size(); index++) {
176 14 : waitCondVec_[index].streamId = 0U;
177 14 : waitCondVec_[index].taskId = 0U;
178 14 : if (sem_init(&(waitCondVec_[index].waitFlag), 0, 0U) != 0) {
179 3 : waitCondVec_.clear();
180 3 : aicpusd_err("sem init failed %s", strerror(errno));
181 3 : return AICPU_SCHEDULE_ERROR_INNER_ERROR;
182 : }
183 : }
184 3 : return AICPU_SCHEDULE_OK;
185 : }
186 :
187 9 : int32_t AicpuCustDumpProcess::ProcessDumpMessage(const event_info &drvEventInfo)
188 : {
189 9 : if (static_cast<size_t>(drvEventInfo.priv.msg_len) != sizeof(AICPUDumpCustInfo)) {
190 3 : aicpusd_err("The len[%u] is not correct[%zu].",
191 : drvEventInfo.priv.msg_len, sizeof(AICPUDumpCustInfo));
192 3 : return AICPU_SCHEDULE_ERROR_PARAMETER_NOT_VALID;
193 : }
194 6 : const AICPUDumpCustInfo * const info = PtrToPtr<const char_t, const AICPUDumpCustInfo>(drvEventInfo.priv.msg);
195 : // find process function of the subevent id.
196 6 : const AICPUCustSubEvent drvEventId = static_cast<AICPUCustSubEvent>(drvEventInfo.comm.subevent_id);
197 6 : if (drvEventId != AICPU_SUB_EVENT_REPORT_CUST_DUMPDATA) {
198 1 : aicpusd_err("invalid subevent %u", static_cast<uint32_t>(drvEventId));
199 1 : return AICPU_SCHEDULE_ERROR_PARAMETER_NOT_VALID;
200 : }
201 5 : uint32_t threadIndex = info->threadIndex;
202 5 : uint32_t streamId = info->streamId;
203 5 : uint32_t taskId = info->taskId;
204 5 : aicpusd_info("Begin dump info, threadindex:%u, streamid:%u, taskid:%u.", threadIndex, streamId, taskId);
205 5 : if (waitCondVec_.size() > threadIndex) {
206 3 : if ((waitCondVec_[threadIndex].streamId == streamId) && (waitCondVec_[threadIndex].taskId == taskId)) {
207 2 : waitCondVec_[threadIndex].dumpRet = info->retCode;
208 2 : if (sem_post(&(waitCondVec_[threadIndex].waitFlag)) != 0) {
209 1 : aicpusd_err("sem post threadindex:%u failed, error:%s", threadIndex, strerror(errno));
210 1 : return AICPU_SCHEDULE_ERROR_INNER_ERROR;
211 : } else {
212 1 : aicpusd_info("active threadIndex:%u, streamId:%u, taskId:%u", threadIndex, streamId, taskId);
213 1 : return AICPU_SCHEDULE_OK;
214 : }
215 : } else {
216 1 : aicpusd_err("threadinex:%u, basestreamid:%u, basetaskid:%u, infostreamId:%u, infotaskId:%u",
217 : threadIndex, waitCondVec_[threadIndex].streamId, waitCondVec_[threadIndex].taskId,
218 : streamId, taskId);
219 1 : return AICPU_SCHEDULE_ERROR_PARAMETER_NOT_VALID;
220 : }
221 : } else {
222 2 : aicpusd_err("invalid thread index:%u", threadIndex);
223 2 : return AICPU_SCHEDULE_ERROR_PARAMETER_NOT_VALID;
224 : }
225 : return AICPU_SCHEDULE_ERROR_INNER_ERROR;
226 : }
227 3 : int32_t AicpuCustDumpProcess::ProcessPlatformInfoMessage(const event_info &drvEventInfo) const
228 : {
229 3 : return CustProcessLoadPlatform(&drvEventInfo);
230 : }
231 33 : int32_t AicpuCustDumpProcess::ActiveTheBlockThread(const event_info &drvEventInfo)
232 : {
233 33 : const AICPUCustSubEvent drvEventId = static_cast<AICPUCustSubEvent>(drvEventInfo.comm.subevent_id);
234 33 : if (drvEventId == AICPU_SUB_EVENT_REPORT_CUST_DUMPDATA) {
235 2 : return ProcessDumpMessage(drvEventInfo);
236 : }
237 31 : if (drvEventId == AICPU_SUB_EVENT_CUST_LOAD_PLATFORM) {
238 3 : return ProcessPlatformInfoMessage(drvEventInfo);
239 : }
240 28 : aicpusd_err("invalid Subevent Id [%u]", static_cast<uint32_t>(drvEventId));
241 28 : return AICPU_SCHEDULE_OK;
242 : }
243 33 : int32_t AicpuCustDumpProcess::ProcessMessage(const int32_t timeout)
244 : {
245 : event_info drvEventInfo;
246 33 : const int32_t retVal = halEschedWaitEvent(deviceId_, DATA_DUMP_GRUOP_ID, DATA_DUMP_THREAD_INDEX,
247 33 : timeout, &drvEventInfo);
248 33 : if (retVal == DRV_ERROR_NONE) {
249 28 : (void) ActiveTheBlockThread(drvEventInfo);
250 5 : } else if ((retVal == DRV_ERROR_SCHED_WAIT_TIMEOUT) || (retVal == DRV_ERROR_NO_EVENT)) {
251 : // if timeout, will continue wait event.
252 5 : } else if ((retVal == DRV_ERROR_SCHED_PROCESS_EXIT) || (retVal == DRV_ERROR_SCHED_PARA_ERR)) {
253 3 : if (runningFlag_) {
254 1 : runningFlag_ = false;
255 : }
256 3 : aicpusd_warn("Failed to get event, error code=%d, deviceId[%u], groupId[%u], threadIndex[%u]",
257 : retVal, deviceId_, DATA_DUMP_GRUOP_ID, DATA_DUMP_THREAD_INDEX);
258 2 : } else if (retVal == DRV_ERROR_SCHED_RUN_IN_ILLEGAL_CPU) {
259 1 : runningFlag_ = false;
260 1 : aicpusd_err("Cpu Illegal get event, error code[%d], deviceId[%u], groupId[%u], threadIndex[%u]",
261 : retVal, deviceId_, DATA_DUMP_GRUOP_ID, DATA_DUMP_THREAD_INDEX);
262 : } else {
263 : // record a error code
264 1 : aicpusd_err("Failed to get event, error code[%d], deviceId[%u], groupId[%u], threadIndex[%u]",
265 : retVal, deviceId_, DATA_DUMP_GRUOP_ID, DATA_DUMP_THREAD_INDEX);
266 : }
267 33 : return retVal;
268 : }
269 :
270 1 : int32_t AicpuCustDumpProcess::GetDataDumpRetCode(const uint32_t threadIndex)
271 : {
272 1 : int32_t retCode = waitCondVec_[threadIndex].dumpRet;
273 1 : waitCondVec_[threadIndex].dumpRet = -1;
274 1 : aicpusd_info("dump task finish retcode:%d, streamid:%u, taskId:%u, threadIndex:%u",
275 : retCode, waitCondVec_[threadIndex].streamId, waitCondVec_[threadIndex].taskId, threadIndex);
276 1 : return retCode;
277 : }
278 :
279 19 : void AicpuCustDumpProcess::UnitDataDumpProcess()
280 : {
281 19 : aicpusd_info("UnitDataDumpProcess start");
282 19 : const std::lock_guard<std::mutex> lk(initMutex_);
283 19 : if (!initFlag_) {
284 17 : aicpusd_info("already uninit no need uninit");
285 17 : return;
286 : }
287 2 : initFlag_ = false;
288 6 : for (size_t index = 0; index < waitCondVec_.size(); index++) {
289 4 : sem_post(&(waitCondVec_[index].waitFlag));
290 : }
291 2 : runningFlag_=false;
292 2 : aicpusd_info("sem post finish");
293 2 : if (msgThread_.joinable()) {
294 1 : msgThread_.join();
295 : }
296 2 : aicpusd_info("UnitDataDumpProcess finish");
297 19 : }
298 :
299 4 : int32_t AicpuCustDumpProcess::SendDumpMessageToAicpuSd(char_t *msg, const uint32_t msgLen) const
300 : {
301 4 : pid_t aicpuPid = AicpuScheduleInterface::GetInstance().GetAicpuSdProcId();
302 4 : event_summary drvEvent = {};
303 4 : drvEvent.dst_engine = CCPU_DEVICE;
304 4 : drvEvent.policy = ONLY;
305 4 : drvEvent.pid = aicpuPid;
306 4 : drvEvent.grp_id = DATA_DUMP_GRUOP_ID;
307 4 : drvEvent.event_id = EVENT_CCPU_CTRL_MSG;
308 4 : drvEvent.subevent_id = static_cast<uint32_t>(AICPU_SUB_EVENT_REPORT_CUST_DUMPDATA);
309 4 : drvEvent.msg_len = msgLen;
310 4 : drvEvent.msg = msg;
311 4 : const int32_t ret = halEschedSubmitEvent(deviceId_, &drvEvent);
312 4 : if (ret != DRV_ERROR_NONE) {
313 1 : aicpusd_err("halEschedSubmitEvent failed, ret:%d, devid:%u", ret, deviceId_);
314 1 : return AICPU_SCHEDULE_ERROR_DRV_ERR;
315 : }
316 3 : aicpusd_info("send dump msg success");
317 3 : return AICPU_SCHEDULE_OK;
318 : }
319 :
320 5 : int32_t AicpuCustDumpProcess::BeginDatadumpTask(const uint32_t threadIndex, const uint32_t streamId,
321 : const uint32_t taskId)
322 : {
323 5 : if (threadIndex >= waitCondVec_.size()) {
324 3 : aicpusd_err("invalid threadindex:%u", threadIndex);
325 3 : return AICPU_SCHEDULE_ERROR_PARAMETER_NOT_VALID;
326 : }
327 2 : waitCondVec_[threadIndex].streamId = streamId;
328 2 : waitCondVec_[threadIndex].taskId = taskId;
329 2 : AICPUDumpCustInfo subEventInfo = { };
330 2 : subEventInfo.retCode = -1;
331 2 : subEventInfo.streamId = streamId;
332 2 : subEventInfo.taskId = taskId;
333 2 : subEventInfo.threadIndex = threadIndex;
334 2 : if (SendDumpMessageToAicpuSd(PtrToPtr<AICPUDumpCustInfo, char_t>(&subEventInfo),
335 2 : static_cast<uint32_t>(sizeof(AICPUDumpCustInfo))) != AICPU_SCHEDULE_OK) {
336 0 : aicpusd_err("send msg failed streamid:%u, taskId:%u, threadIndex:%u", streamId, taskId, threadIndex);
337 0 : return AICPU_SCHEDULE_ERROR_DRV_ERR;
338 : }
339 2 : aicpusd_info("dump task begin success streamid:%u, taskId:%u, threadIndex:%u", streamId, taskId, threadIndex);
340 2 : if (sem_wait(&(waitCondVec_[threadIndex].waitFlag)) == -1) {
341 1 : aicpusd_err("try to wait by loop threadindex:%u", threadIndex);
342 1 : return AICPU_SCHEDULE_ERROR_DUMP_TASK_WAIT_ERROR;
343 : }
344 1 : const int32_t dumpRet = GetDataDumpRetCode(threadIndex);
345 1 : aicpusd_info("dump task finish streamid:%u, taskId:%u, threadIndex:%u, dumpRet:%d",
346 : streamId, taskId, threadIndex, dumpRet);
347 1 : return dumpRet;
348 : }
349 : }
|
