Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 : #include "aicpu_cust_sd_dump_process.h"
11 :
12 : #include <string.h>
13 : #include <sched.h>
14 : #include <semaphore.h>
15 : #include "aicpusd_drv_manager.h"
16 : #include "aicpusd_info.h"
17 : #include "aicpusd_util.h"
18 : #include "aicpu_cust_sd_proc_mgr_sys_operator_agent.h"
19 : #include "aicpusd_interface_process.h"
20 : #include "aicpu_cust_platform_info_process.h"
21 :
22 : namespace AicpuSchedule {
23 : namespace {
24 : constexpr const uint32_t DATA_DUMP_GRUOP_ID = 31U;
25 : constexpr const uint32_t DATA_DUMP_THREAD_INDEX = 0U;
26 : constexpr const uint64_t DATA_DUMP_EVENT_MASK = (1ULL << static_cast<uint32_t>(EVENT_CCPU_CTRL_MSG));
27 : constexpr const int32_t DATA_DUMP_TIMEOUT_INTERVAL = 3000;
28 : constexpr const uint32_t SLEEP_USECS = 50000U;
29 : }; // namespace
30 :
31 81 : AicpuCustDumpProcess& AicpuCustDumpProcess::GetInstance()
32 : {
33 81 : static AicpuCustDumpProcess instance;
34 81 : return instance;
35 : }
36 :
37 1 : AicpuCustDumpProcess::~AicpuCustDumpProcess() { UnitDataDumpProcess(); }
38 :
39 1 : AicpuCustDumpProcess::AicpuCustDumpProcess() : deviceId_(0U), runningFlag_(false), initFlag_(false) {}
40 :
41 10 : int32_t AicpuCustDumpProcess::InitDumpProcess(const uint32_t deviceId, const uint32_t workCnt)
42 : {
43 10 : const std::lock_guard<std::mutex> lk(initMutex_);
44 : uint32_t runMode;
45 10 : const aicpu::status_t status = aicpu::GetAicpuRunMode(runMode);
46 10 : if (status != aicpu::AICPU_ERROR_NONE) {
47 1 : aicpusd_err("Get current aicpu ctx failed.");
48 1 : return AICPU_SCHEDULE_ERROR_INNER_ERROR;
49 : }
50 9 : if (runMode != aicpu::AicpuRunMode::PROCESS_PCIE_MODE) {
51 1 : aicpusd_info("not pcie mode no need this thread");
52 1 : return AICPU_SCHEDULE_OK;
53 : }
54 8 : if ((initFlag_) || (workCnt == 0U)) {
55 5 : aicpusd_info("workcnt:%u", workCnt);
56 5 : return AICPU_SCHEDULE_OK;
57 : }
58 3 : initFlag_ = true;
59 3 : deviceId_ = deviceId;
60 3 : runningFlag_ = true;
61 3 : if (InitWaitConVec(workCnt) != AICPU_SCHEDULE_OK) {
62 1 : aicpusd_err("InitWaitConVec init failed");
63 1 : return AICPU_SCHEDULE_ERROR_INNER_ERROR;
64 : }
65 :
66 2 : const int32_t semRet = sem_init(&workerSme_, 0, 0U);
67 2 : if (semRet == -1) {
68 1 : aicpusd_err("workerSme_ init failed, %s", strerror(errno));
69 1 : return AICPU_SCHEDULE_ERROR_INIT_FAILED;
70 : }
71 :
72 : try {
73 1 : msgThread_ = std::thread(&AicpuCustDumpProcess::StartProcessEvent, this);
74 0 : } catch (std::exception& e) {
75 0 : aicpusd_err("create thread file:%s", e.what());
76 0 : return AICPU_SCHEDULE_ERROR_INIT_FAILED;
77 0 : }
78 1 : const int32_t semWaitRet = sem_wait(&workerSme_);
79 1 : if (semWaitRet == -1) {
80 0 : aicpusd_err("workerSme wait failed, %s", strerror(errno));
81 0 : return AICPU_SCHEDULE_ERROR_INIT_FAILED;
82 : }
83 1 : return AICPU_SCHEDULE_OK;
84 10 : }
85 :
86 8 : int32_t AicpuCustDumpProcess::SetDataDumpThreadAffinity() const
87 : {
88 8 : std::vector<uint32_t> ccpuIds = AicpuDrvManager::GetInstance().GetCcpuList();
89 8 : if (ccpuIds.empty()) {
90 0 : aicpusd_run_info("ccpu list is empty no need bind core");
91 0 : return AICPU_SCHEDULE_OK;
92 : }
93 16 : if (AicpuUtil::IsEnvValEqual(ENV_NAME_PROCMGR_AICPU_CPUSET, "1")) {
94 2 : const pid_t tid = static_cast<pid_t>(GetTid());
95 2 : auto ret = ProcMgrBindThread(tid, ccpuIds);
96 2 : uint32_t tryTimes = 0;
97 4 : while ((ret != 0U) && (tryTimes <= 1)) {
98 2 : aicpusd_warn("set affinity failed ret[%d], will try again", ret);
99 2 : (void)usleep(SLEEP_USECS);
100 2 : ret = ProcMgrBindThread(tid, ccpuIds);
101 2 : tryTimes++;
102 : }
103 2 : if (ret != 0) {
104 1 : aicpusd_err("aicpu bind tid by pm res[%d].", ret);
105 1 : return AICPU_SCHEDULE_ERROR_INNER_ERROR;
106 : } else {
107 1 : aicpusd_run_info("aicpu bind by pm success");
108 1 : return AICPU_SCHEDULE_OK;
109 : }
110 : } else {
111 6 : pthread_t threadId = pthread_self();
112 : cpu_set_t cpuset;
113 6 : CPU_ZERO(&cpuset);
114 48 : for (const auto cpuId : ccpuIds) {
115 42 : CPU_SET(cpuId, &cpuset);
116 42 : aicpusd_info("prepare bind threadId=%lu to cpuId=%u", threadId, static_cast<uint32_t>(cpuId));
117 : }
118 6 : const int32_t ret = pthread_setaffinity_np(threadId, sizeof(cpu_set_t), &cpuset);
119 6 : if (ret != 0) {
120 1 : aicpusd_run_warn(
121 : "Binding datadump thread to control CPU was not successful, will run in aicpu. ret=%d, reason=%s", ret,
122 : strerror(errno));
123 : } else {
124 5 : aicpusd_run_info("aicpu bind by self success");
125 : }
126 : }
127 :
128 6 : return AICPU_SCHEDULE_OK;
129 8 : }
130 :
131 5 : void AicpuCustDumpProcess::StartProcessEvent()
132 : {
133 5 : int32_t ret = halEschedCreateGrp(deviceId_, DATA_DUMP_GRUOP_ID, GRP_TYPE_BIND_CP_CPU);
134 5 : if (ret != DRV_ERROR_NONE) {
135 1 : aicpusd_err(
136 : "halEschedCreateGrp dev[%u] group[%u] type[%d] failed, result[%d].", deviceId_, DATA_DUMP_GRUOP_ID,
137 : GRP_TYPE_BIND_CP_CPU, ret);
138 1 : sem_post(&workerSme_);
139 1 : return;
140 : }
141 :
142 4 : if (SetDataDumpThreadAffinity() != AICPU_SCHEDULE_OK) {
143 1 : aicpusd_err("datadump thread bind core failed");
144 1 : sem_post(&workerSme_);
145 1 : return;
146 : }
147 3 : ret = halEschedSubscribeEvent(deviceId_, DATA_DUMP_GRUOP_ID, DATA_DUMP_THREAD_INDEX, DATA_DUMP_EVENT_MASK);
148 3 : if (ret != DRV_ERROR_NONE) {
149 1 : aicpusd_err(
150 : "halEschedSubscribeEvent failed, deviceId[%u], groupId[%u], threadIndex[%u]", deviceId_, DATA_DUMP_GRUOP_ID,
151 : DATA_DUMP_THREAD_INDEX);
152 1 : sem_post(&workerSme_);
153 1 : return;
154 : }
155 2 : ProcessMessage(0U);
156 2 : sem_post(&workerSme_);
157 2 : aicpusd_run_info("post work sem finish, dump process init finish. deviceId[%u]", deviceId_);
158 2 : LoopProcessEvent();
159 2 : aicpusd_run_info("dump thread stopped");
160 : }
161 2 : void AicpuCustDumpProcess::LoopProcessEvent()
162 : {
163 26 : while (runningFlag_) {
164 24 : ProcessMessage(DATA_DUMP_TIMEOUT_INTERVAL);
165 : }
166 2 : }
167 :
168 6 : int32_t AicpuCustDumpProcess::InitWaitConVec(const uint32_t workCnt)
169 : {
170 6 : waitCondVec_.resize(workCnt);
171 17 : for (size_t index = 0; index < waitCondVec_.size(); index++) {
172 14 : waitCondVec_[index].streamId = 0U;
173 14 : waitCondVec_[index].taskId = 0U;
174 14 : if (sem_init(&(waitCondVec_[index].waitFlag), 0, 0U) != 0) {
175 3 : waitCondVec_.clear();
176 3 : aicpusd_err("sem init failed %s", strerror(errno));
177 3 : return AICPU_SCHEDULE_ERROR_INNER_ERROR;
178 : }
179 : }
180 3 : return AICPU_SCHEDULE_OK;
181 : }
182 :
183 9 : int32_t AicpuCustDumpProcess::ProcessDumpMessage(const event_info& drvEventInfo)
184 : {
185 9 : if (static_cast<size_t>(drvEventInfo.priv.msg_len) != sizeof(AICPUDumpCustInfo)) {
186 3 : aicpusd_err("The len[%u] is not correct[%zu].", drvEventInfo.priv.msg_len, sizeof(AICPUDumpCustInfo));
187 3 : return AICPU_SCHEDULE_ERROR_PARAMETER_NOT_VALID;
188 : }
189 6 : const AICPUDumpCustInfo* const info = PtrToPtr<const char_t, const AICPUDumpCustInfo>(drvEventInfo.priv.msg);
190 : // find process function of the subevent id.
191 6 : const AICPUCustSubEvent drvEventId = static_cast<AICPUCustSubEvent>(drvEventInfo.comm.subevent_id);
192 6 : if (drvEventId != AICPU_SUB_EVENT_REPORT_CUST_DUMPDATA) {
193 1 : aicpusd_err("invalid subevent %u", static_cast<uint32_t>(drvEventId));
194 1 : return AICPU_SCHEDULE_ERROR_PARAMETER_NOT_VALID;
195 : }
196 5 : uint32_t threadIndex = info->threadIndex;
197 5 : uint32_t streamId = info->streamId;
198 5 : uint32_t taskId = info->taskId;
199 5 : aicpusd_info("Begin dump info, threadindex:%u, streamid:%u, taskid:%u.", threadIndex, streamId, taskId);
200 5 : if (waitCondVec_.size() > threadIndex) {
201 3 : if ((waitCondVec_[threadIndex].streamId == streamId) && (waitCondVec_[threadIndex].taskId == taskId)) {
202 2 : waitCondVec_[threadIndex].dumpRet = info->retCode;
203 2 : if (sem_post(&(waitCondVec_[threadIndex].waitFlag)) != 0) {
204 1 : aicpusd_err("sem post threadindex:%u failed, error:%s", threadIndex, strerror(errno));
205 1 : return AICPU_SCHEDULE_ERROR_INNER_ERROR;
206 : } else {
207 1 : aicpusd_info("active threadIndex:%u, streamId:%u, taskId:%u", threadIndex, streamId, taskId);
208 1 : return AICPU_SCHEDULE_OK;
209 : }
210 : } else {
211 1 : aicpusd_err(
212 : "threadinex:%u, basestreamid:%u, basetaskid:%u, infostreamId:%u, infotaskId:%u", threadIndex,
213 : waitCondVec_[threadIndex].streamId, waitCondVec_[threadIndex].taskId, streamId, taskId);
214 1 : return AICPU_SCHEDULE_ERROR_PARAMETER_NOT_VALID;
215 : }
216 : } else {
217 2 : aicpusd_err("invalid thread index:%u", threadIndex);
218 2 : return AICPU_SCHEDULE_ERROR_PARAMETER_NOT_VALID;
219 : }
220 : return AICPU_SCHEDULE_ERROR_INNER_ERROR;
221 : }
222 3 : int32_t AicpuCustDumpProcess::ProcessPlatformInfoMessage(const event_info& drvEventInfo) const
223 : {
224 3 : return CustProcessLoadPlatform(&drvEventInfo);
225 : }
226 :
227 3 : int32_t AicpuCustDumpProcess::AICPUEventSetDfxInfo(const event_info& drvEventInfo) const
228 : {
229 3 : aicpusd_info("Begin to process set dfx info event.");
230 3 : const auto& privEventInfo = drvEventInfo.priv;
231 3 : if (privEventInfo.msg_len != sizeof(AicpuDfxInfoReq)) {
232 1 : aicpusd_err("The event msg len[%u] is not correct[%zu].", privEventInfo.msg_len, sizeof(AicpuDfxInfoReq));
233 1 : return AICPU_SCHEDULE_ERROR_PARAMETER_NOT_VALID;
234 : }
235 :
236 2 : const AicpuDfxInfoReq* dfxInfoReq = PtrToPtr<const char_t, const AicpuDfxInfoReq>(privEventInfo.msg);
237 2 : aicpusd_info("Set dfx info for aicpusd, addr is %lu.", dfxInfoReq->infoAddr);
238 2 : aicpu::AicpuSetDfxInfo(dfxInfoReq->infoAddr);
239 :
240 2 : const event_sync_msg* syncHead = PtrToPtr<char_t, event_sync_msg>(privEventInfo.msg);
241 2 : event_summary replyEvent = {};
242 2 : int32_t ret = AICPU_SCHEDULE_OK;
243 2 : replyEvent.msg = PtrToPtr<int32_t, char_t>(&ret);
244 2 : replyEvent.msg_len = static_cast<uint32_t>(sizeof(int32_t));
245 2 : replyEvent.dst_engine = syncHead->dst_engine;
246 2 : replyEvent.policy = ONLY;
247 2 : replyEvent.pid = syncHead->pid;
248 2 : replyEvent.grp_id = syncHead->gid;
249 2 : const int32_t eventId = syncHead->event_id;
250 2 : replyEvent.event_id = static_cast<EVENT_ID>(eventId);
251 2 : replyEvent.subevent_id = syncHead->subevent_id;
252 2 : const auto drvRet = halEschedSubmitEvent(AicpuDrvManager::GetInstance().GetDeviceId(), &replyEvent);
253 2 : if (drvRet != DRV_ERROR_NONE) {
254 1 : aicpusd_err("Failed to submit aicpu event. ret is %d.", static_cast<int32_t>(drvRet));
255 1 : return AICPU_SCHEDULE_ERROR_DRV_ERR;
256 : }
257 1 : return ret;
258 : }
259 :
260 33 : int32_t AicpuCustDumpProcess::ActiveTheBlockThread(const event_info& drvEventInfo)
261 : {
262 33 : const AICPUCustSubEvent drvEventId = static_cast<AICPUCustSubEvent>(drvEventInfo.comm.subevent_id);
263 33 : if (drvEventId == AICPU_SUB_EVENT_REPORT_CUST_DUMPDATA) {
264 2 : return ProcessDumpMessage(drvEventInfo);
265 : }
266 31 : if (drvEventId == AICPU_SUB_EVENT_CUST_LOAD_PLATFORM) {
267 3 : return ProcessPlatformInfoMessage(drvEventInfo);
268 : }
269 28 : if (drvEventId == AICPU_SUB_EVENT_SET_DFX_INFO) {
270 3 : return AICPUEventSetDfxInfo(drvEventInfo);
271 : }
272 25 : aicpusd_err("invalid Subevent Id [%u]", static_cast<uint32_t>(drvEventId));
273 25 : return AICPU_SCHEDULE_OK;
274 : }
275 30 : int32_t AicpuCustDumpProcess::ProcessMessage(const int32_t timeout)
276 : {
277 : event_info drvEventInfo;
278 : const int32_t retVal =
279 30 : halEschedWaitEvent(deviceId_, DATA_DUMP_GRUOP_ID, DATA_DUMP_THREAD_INDEX, timeout, &drvEventInfo);
280 30 : if (retVal == DRV_ERROR_NONE) {
281 25 : (void)ActiveTheBlockThread(drvEventInfo);
282 5 : } else if ((retVal == DRV_ERROR_SCHED_WAIT_TIMEOUT) || (retVal == DRV_ERROR_NO_EVENT)) {
283 : // if timeout, will continue wait event.
284 5 : } else if ((retVal == DRV_ERROR_SCHED_PROCESS_EXIT) || (retVal == DRV_ERROR_SCHED_PARA_ERR)) {
285 3 : if (runningFlag_) {
286 1 : runningFlag_ = false;
287 : }
288 3 : aicpusd_warn(
289 : "Failed to get event, error code=%d, deviceId[%u], groupId[%u], threadIndex[%u]", retVal, deviceId_,
290 : DATA_DUMP_GRUOP_ID, DATA_DUMP_THREAD_INDEX);
291 2 : } else if (retVal == DRV_ERROR_SCHED_RUN_IN_ILLEGAL_CPU) {
292 1 : runningFlag_ = false;
293 1 : aicpusd_err(
294 : "Cpu Illegal get event, error code[%d], deviceId[%u], groupId[%u], threadIndex[%u]", retVal, deviceId_,
295 : DATA_DUMP_GRUOP_ID, DATA_DUMP_THREAD_INDEX);
296 : } else {
297 : // record a error code
298 1 : aicpusd_err(
299 : "Failed to get event, error code[%d], deviceId[%u], groupId[%u], threadIndex[%u]", retVal, deviceId_,
300 : DATA_DUMP_GRUOP_ID, DATA_DUMP_THREAD_INDEX);
301 : }
302 30 : return retVal;
303 : }
304 :
305 1 : int32_t AicpuCustDumpProcess::GetDataDumpRetCode(const uint32_t threadIndex)
306 : {
307 1 : int32_t retCode = waitCondVec_[threadIndex].dumpRet;
308 1 : waitCondVec_[threadIndex].dumpRet = -1;
309 1 : aicpusd_info(
310 : "dump task finish retcode:%d, streamid:%u, taskId:%u, threadIndex:%u", retCode,
311 : waitCondVec_[threadIndex].streamId, waitCondVec_[threadIndex].taskId, threadIndex);
312 1 : return retCode;
313 : }
314 :
315 19 : void AicpuCustDumpProcess::UnitDataDumpProcess()
316 : {
317 19 : aicpusd_info("UnitDataDumpProcess start");
318 19 : const std::lock_guard<std::mutex> lk(initMutex_);
319 19 : if (!initFlag_) {
320 17 : aicpusd_info("already uninit no need uninit");
321 17 : return;
322 : }
323 2 : initFlag_ = false;
324 6 : for (size_t index = 0; index < waitCondVec_.size(); index++) {
325 4 : sem_post(&(waitCondVec_[index].waitFlag));
326 : }
327 2 : runningFlag_ = false;
328 2 : aicpusd_info("sem post finish");
329 2 : if (msgThread_.joinable()) {
330 1 : msgThread_.join();
331 : }
332 2 : aicpusd_info("UnitDataDumpProcess finish");
333 19 : }
334 :
335 4 : int32_t AicpuCustDumpProcess::SendDumpMessageToAicpuSd(char_t* msg, const uint32_t msgLen) const
336 : {
337 4 : pid_t aicpuPid = AicpuScheduleInterface::GetInstance().GetAicpuSdProcId();
338 4 : event_summary drvEvent = {};
339 4 : drvEvent.dst_engine = CCPU_DEVICE;
340 4 : drvEvent.policy = ONLY;
341 4 : drvEvent.pid = aicpuPid;
342 4 : drvEvent.grp_id = DATA_DUMP_GRUOP_ID;
343 4 : drvEvent.event_id = EVENT_CCPU_CTRL_MSG;
344 4 : drvEvent.subevent_id = static_cast<uint32_t>(AICPU_SUB_EVENT_REPORT_CUST_DUMPDATA);
345 4 : drvEvent.msg_len = msgLen;
346 4 : drvEvent.msg = msg;
347 4 : const int32_t ret = halEschedSubmitEvent(deviceId_, &drvEvent);
348 4 : if (ret != DRV_ERROR_NONE) {
349 1 : aicpusd_err("halEschedSubmitEvent failed, ret:%d, devid:%u", ret, deviceId_);
350 1 : return AICPU_SCHEDULE_ERROR_DRV_ERR;
351 : }
352 3 : aicpusd_info("send dump msg success");
353 3 : return AICPU_SCHEDULE_OK;
354 : }
355 :
356 3 : int32_t AicpuCustDumpProcess::BeginDatadumpTask(
357 : const uint32_t threadIndex, const uint32_t streamId, const uint32_t taskId)
358 : {
359 3 : if (threadIndex >= waitCondVec_.size()) {
360 1 : aicpusd_err("invalid threadindex:%u", threadIndex);
361 1 : return AICPU_SCHEDULE_ERROR_PARAMETER_NOT_VALID;
362 : }
363 2 : waitCondVec_[threadIndex].streamId = streamId;
364 2 : waitCondVec_[threadIndex].taskId = taskId;
365 2 : AICPUDumpCustInfo subEventInfo = {};
366 2 : subEventInfo.retCode = -1;
367 2 : subEventInfo.streamId = streamId;
368 2 : subEventInfo.taskId = taskId;
369 2 : subEventInfo.threadIndex = threadIndex;
370 2 : if (SendDumpMessageToAicpuSd(
371 2 : PtrToPtr<AICPUDumpCustInfo, char_t>(&subEventInfo), static_cast<uint32_t>(sizeof(AICPUDumpCustInfo))) !=
372 : AICPU_SCHEDULE_OK) {
373 0 : aicpusd_err("send msg failed streamid:%u, taskId:%u, threadIndex:%u", streamId, taskId, threadIndex);
374 0 : return AICPU_SCHEDULE_ERROR_DRV_ERR;
375 : }
376 2 : aicpusd_info("dump task begin success streamid:%u, taskId:%u, threadIndex:%u", streamId, taskId, threadIndex);
377 2 : if (sem_wait(&(waitCondVec_[threadIndex].waitFlag)) == -1) {
378 1 : aicpusd_err("try to wait by loop threadindex:%u", threadIndex);
379 1 : return AICPU_SCHEDULE_ERROR_DUMP_TASK_WAIT_ERROR;
380 : }
381 1 : const int32_t dumpRet = GetDataDumpRetCode(threadIndex);
382 1 : aicpusd_info(
383 : "dump task finish streamid:%u, taskId:%u, threadIndex:%u, dumpRet:%d", streamId, taskId, threadIndex, dumpRet);
384 1 : return dumpRet;
385 : }
386 : } // namespace AicpuSchedule
|
