Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 : #include <string>
11 : #include <vector>
12 : #include <sstream>
13 : #include <unistd.h>
14 : #include <dlfcn.h>
15 : #include <sys/file.h>
16 : #include <sys/ioctl.h>
17 : #include <exception>
18 : #include "core/aicpusd_interface_process.h"
19 : #include "core/aicpusd_event_process.h"
20 : #include "core/aicpusd_event_manager.h"
21 : #include "core/aicpusd_resource_limit.h"
22 : #include "core/aicpusd_worker.h"
23 : #include "aicpusd_profiler.h"
24 : #include "aicpusd_lastword.h"
25 : #include "common/aicpusd_args_parser.h"
26 : #include "aicpusd_util.h"
27 : #include "aicpusd_status.h"
28 : #include "aicpusd_common.h"
29 : #include "securec.h"
30 : #include "tsd.h"
31 : #include "aicpusd_weak_log.h"
32 : #include "driver/ascend_hal.h"
33 : #include "common/aicpusd_context.h"
34 : #include "aicpusd_mc2_maintenance_thread_api.h"
35 : #include "aicpusd_feature_ctrl.h"
36 : #include "aicpusd_cust_dump_process.h"
37 : #include "aicpusd_send_platform_Info_to_custom.h"
38 :
39 : namespace AicpuSchedule {
40 : namespace {
41 : constexpr int32_t SUCCESS_VALUE = 0;
42 : const std::string ERROR_MSG_INVALID_PARAM = "E39001";
43 : const std::string ERROR_MSG_DRV_ERROR = "E39002";
44 : const std::string ERROR_MSG_CGROUP_FAILED = "E30007";
45 : const std::string ERROR_MSG_AICPU_INIT_FAILED = "E39004";
46 : const std::string QUEUE_SCHEDULE_SO_NAME = "libqueue_schedule.so";
47 : void *g_qslibHandle = nullptr;
48 :
49 26 : void SetLogLevel(AicpuSchedule::ArgsParser &startParams)
50 : {
51 26 : if (&dlog_setlevel != nullptr) {
52 26 : if (dlog_setlevel(-1, startParams.GetLogLevel(), startParams.GetEventLevel()) != SUCCESS_VALUE) {
53 4 : aicpusd_warn("Set log level failed");
54 : }
55 26 : if ((startParams.GetCcecpuLogLevel() >= DEBUG_LOG) && (startParams.GetCcecpuLogLevel() <= ERROR_LOG)) {
56 0 : if (dlog_setlevel(CCECPU, startParams.GetCcecpuLogLevel(), startParams.GetEventLevel()) != SUCCESS_VALUE) {
57 0 : aicpusd_warn("Set ccecpu log level failed");
58 : }
59 : }
60 26 : if ((startParams.GetAicpuLogLevel() >= DEBUG_LOG) && (startParams.GetAicpuLogLevel() <= ERROR_LOG)) {
61 0 : if (dlog_setlevel(AICPU, startParams.GetAicpuLogLevel(), startParams.GetEventLevel()) != SUCCESS_VALUE) {
62 0 : aicpusd_warn("Set aicpu log level failed");
63 : }
64 : }
65 : }
66 26 : LogAttr logAttr = {};
67 26 : logAttr.type = APPLICATION;
68 26 : logAttr.pid = startParams.GetHostPid();
69 26 : logAttr.deviceId = startParams.GetDeviceId();
70 26 : logAttr.mode = 0;
71 26 : if (DlogSetAttrAicpu(logAttr) != SUCCESS_VALUE) {
72 1 : aicpusd_warn("Set log attr failed");
73 : }
74 26 : }
75 : /**
76 : * attach host group based on parameters send by tsd.
77 : * @param grpNameListStr group name list
78 : * @param grpNameNum group name number
79 : * @return true:success, false:failed
80 : */
81 20 : bool AttachHostGroup(const std::vector<std::string> &groupNameVec, const uint32_t grpNameNum)
82 : {
83 20 : if (grpNameNum == 0U) {
84 4 : aicpusd_info("There is not group need to be attached");
85 4 : return true;
86 : }
87 16 : if (groupNameVec.empty()) {
88 1 : aicpusd_err("Aicpu start failed. Wrong parameters of groupNum[%d] with empty group name", grpNameNum);
89 1 : return false;
90 : }
91 15 : if (static_cast<uint32_t>(groupNameVec.size()) != grpNameNum) {
92 1 : aicpusd_err("Aicpu start failed. parse group name num[%d] is consistence with num[%u] in parameter",
93 : groupNameVec.size(), grpNameNum);
94 1 : return false;
95 : }
96 :
97 37 : for (size_t i = 0UL; i < groupNameVec.size(); ++i) {
98 24 : const auto drvRet = halGrpAttach(groupNameVec[i].c_str(), -1);
99 24 : if (drvRet != DRV_ERROR_NONE) {
100 1 : aicpusd_err("halGrpAttach group[%s] failed. ret[%d]", groupNameVec[i].c_str(), drvRet);
101 1 : return false;
102 : }
103 23 : aicpusd_run_info("halGrpAttach group[%s] successfully.", groupNameVec[i].c_str());
104 : }
105 13 : return true;
106 : }
107 :
108 9 : void ReportErrorMsg(const int32_t errCode, const uint32_t deviceId, const uint32_t hostPid, const uint32_t vfId)
109 : {
110 9 : std::string errStr;
111 9 : switch (errCode) {
112 1 : case AICPU_SCHEDULE_ERROR_PARAMETER_NOT_VALID: {
113 1 : errStr = ERROR_MSG_INVALID_PARAM;
114 1 : break;
115 : }
116 3 : case AICPU_SCHEDULE_ERROR_DRV_ERR: {
117 3 : errStr = ERROR_MSG_DRV_ERROR;
118 3 : break;
119 : }
120 1 : case AICPU_SCHEDULE_ERROR_CGROUP_FAILED: {
121 1 : errStr = ERROR_MSG_CGROUP_FAILED;
122 1 : break;
123 : }
124 1 : case AICPU_SCHEDULE_ERROR_INIT_FAILED: {
125 1 : errStr = ERROR_MSG_AICPU_INIT_FAILED;
126 1 : break;
127 : }
128 3 : default: {
129 3 : errStr = ERROR_MSG_AICPU_INIT_FAILED;
130 3 : break;
131 : }
132 : }
133 9 : DlogFlushAicpu();
134 9 : sleep(1); // sleep 1s to avoid log loss
135 9 : const int32_t ret = TsdReportStartOrStopErrCode(deviceId, TSD_COMPUTE, hostPid, vfId,
136 9 : errStr.c_str(), static_cast<uint32_t>(errStr.size()));
137 9 : if (ret != 0) {
138 1 : aicpusd_err("TsdReportStartOrStopErrCode failed. ret[%d]", ret);
139 1 : sleep(1); // sleep 1s to avoid log loss
140 : }
141 :
142 9 : DlogFlushAicpu();
143 9 : }
144 :
145 17 : void SendPidQosMsgToTsd(const uint32_t pidQos, const uint32_t deviceId, const uint32_t hostPid, const uint32_t vfId)
146 : {
147 : TsdCapabilityMsgInfo qosPidMsg;
148 17 : qosPidMsg.subCapabityType = TSD_EVENT_GET_CAPABILITY;
149 17 : qosPidMsg.resultInfo = pidQos;
150 17 : ReportMsgToTsd(deviceId, TSD_COMPUTE, hostPid, vfId, reinterpret_cast<const char_t * const>(&qosPidMsg));
151 17 : }
152 :
153 17 : void RegQueueScheduleModuleCallBack()
154 : {
155 17 : g_qslibHandle = dlopen(QUEUE_SCHEDULE_SO_NAME.c_str(), RTLD_LAZY);
156 17 : if (g_qslibHandle == nullptr) {
157 9 : aicpusd_warn("cannot open %s", QUEUE_SCHEDULE_SO_NAME.c_str());
158 9 : return;
159 : }
160 :
161 : const SubProcEventCallBackFuncInfo startQsFunc =
162 8 : reinterpret_cast<SubProcEventCallBackFuncInfo>(dlsym(g_qslibHandle, "StartQueueScheduleModule"));
163 8 : if (startQsFunc == nullptr) {
164 1 : aicpusd_err("cannot find StartQueueScheduleModule");
165 1 : (void)dlclose(g_qslibHandle);
166 1 : g_qslibHandle = nullptr;
167 1 : return;
168 : }
169 :
170 : SubProcEventCallBackInfo startCallBackInfo;
171 7 : startCallBackInfo.callBackFunc = startQsFunc;
172 7 : startCallBackInfo.eventType = TSD_EVENT_START_QS_MODULE;
173 7 : int32_t ret = RegEventMsgCallBackFunc(&startCallBackInfo);
174 7 : if (ret != 0) {
175 2 : aicpusd_err("reg StartQueueScheduleModule failed");
176 2 : (void)dlclose(g_qslibHandle);
177 2 : g_qslibHandle = nullptr;
178 2 : return;
179 : }
180 :
181 : const SubProcEventCallBackFuncInfo stopQsFunc =
182 5 : reinterpret_cast<SubProcEventCallBackFuncInfo>(dlsym(g_qslibHandle, "StopQueueScheduleModule"));
183 5 : if (stopQsFunc == nullptr) {
184 1 : UnRegEventMsgCallBackFunc(TSD_EVENT_START_QS_MODULE);
185 1 : aicpusd_err("cannot find StopQueueScheduleModule");
186 1 : (void)dlclose(g_qslibHandle);
187 1 : g_qslibHandle = nullptr;
188 1 : return;
189 : }
190 :
191 : SubProcEventCallBackInfo stopCallBackInfo;
192 4 : stopCallBackInfo.callBackFunc = stopQsFunc;
193 4 : stopCallBackInfo.eventType = TSD_EVENT_STOP_QS_MODULE;
194 :
195 4 : ret = RegEventMsgCallBackFunc(&stopCallBackInfo);
196 4 : if (ret != 0) {
197 1 : UnRegEventMsgCallBackFunc(TSD_EVENT_START_QS_MODULE);
198 1 : aicpusd_err("reg StopQueueScheduleModule failed");
199 1 : (void)dlclose(g_qslibHandle);
200 1 : g_qslibHandle = nullptr;
201 1 : return;
202 : }
203 :
204 3 : aicpusd_run_info("Register queue schedule module callback successfully.");
205 3 : return;
206 : }
207 :
208 17 : void RegCreateMc2MantenanceThreadCallBack()
209 : {
210 17 : SubProcEventCallBackInfo createMc2MantenanceThreadInfo = {};
211 17 : createMc2MantenanceThreadInfo.callBackFunc = CreateMc2MantenanceThread;
212 17 : createMc2MantenanceThreadInfo.eventType = TSD_EVENT_START_MC2_THREAD;
213 17 : int32_t ret = RegEventMsgCallBackFunc(&createMc2MantenanceThreadInfo);
214 17 : if (ret != 0) {
215 3 : aicpusd_err("reg Create Mc2 Maintenance Thread failed");
216 3 : return;
217 : }
218 : }
219 :
220 17 : void RegCreateDatadumpThreadCallBack()
221 : {
222 17 : SubProcEventCallBackInfo createDatadumpThreadInfo = {};
223 17 : createDatadumpThreadInfo.callBackFunc = CreateDatadumpThread;
224 17 : createDatadumpThreadInfo.eventType = TSD_EVENT_START_UDF_DATADUMP_THREAD;
225 17 : int32_t ret = RegEventMsgCallBackFunc(&createDatadumpThreadInfo);
226 17 : if (ret != 0) {
227 3 : aicpusd_err("reg Create datadump Thread failed");
228 3 : return;
229 : }
230 : }
231 :
232 17 : void RegCustProcessLoadPlatformCallBack()
233 : {
234 17 : SubProcEventCallBackInfo sendLoadPlatformInfoToCustInfo = {};
235 17 : sendLoadPlatformInfoToCustInfo.callBackFunc = SendLoadPlatformInfoToCust;
236 17 : sendLoadPlatformInfoToCustInfo.eventType = TSD_EVENT_LOAD_PLATFORM_TO_CUST;
237 17 : int32_t ret = RegEventMsgCallBackFunc(&sendLoadPlatformInfoToCustInfo);
238 17 : if (ret != 0) {
239 3 : aicpusd_err("reg send load platform to cust failed");
240 3 : return;
241 : }
242 : }
243 :
244 16 : void StopQsSubModuleInAicpu(const uint32_t deviceId, const uint32_t hostPid, const uint32_t vfId)
245 : {
246 16 : if (g_qslibHandle == nullptr) {
247 13 : aicpusd_warn("cannot open so %s", QUEUE_SCHEDULE_SO_NAME.c_str());
248 14 : return;
249 : }
250 3 : TsdSubEventInfo eventInfo = {};
251 3 : eventInfo.deviceId = deviceId;
252 3 : eventInfo.hostPid = hostPid;
253 3 : eventInfo.vfId = vfId;
254 : const SubProcEventCallBackFuncInfo stopQsFunc =
255 3 : reinterpret_cast<SubProcEventCallBackFuncInfo>(dlsym(g_qslibHandle, "StopQueueScheduleModule"));
256 3 : if (stopQsFunc == nullptr) {
257 1 : aicpusd_err("cannot find StopQueueScheduleModule");
258 1 : (void)dlclose(g_qslibHandle);
259 1 : g_qslibHandle = nullptr;
260 1 : return;
261 : }
262 2 : const int32_t ret = stopQsFunc(&eventInfo);
263 2 : if (ret != 0) {
264 0 : aicpusd_err("StopQueueScheduleModule fail");
265 : }
266 2 : (void)dlclose(g_qslibHandle);
267 2 : g_qslibHandle = nullptr;
268 : }
269 :
270 32 : bool SendVfMsgToDrv(const uint32_t cmd, const uint32_t deviceId, const uint32_t vfId)
271 : {
272 32 : if (FeatureCtrl::IsVfMode(deviceId, vfId)) {
273 6 : const int32_t fd = open("/dev/qos", O_RDWR);
274 6 : if (fd < 0) {
275 4 : aicpusd_warn("no such file. error[%s],errorno[%d]", strerror(errno), errno);
276 4 : return false;
277 : }
278 : VfMsgInfo vfMsg;
279 2 : vfMsg.deviceId = deviceId;
280 2 : vfMsg.vfId = vfId;
281 2 : const auto ret = ioctl(fd, cmd, PtrToPtr<VfMsgInfo, void>(&vfMsg));
282 2 : if (ret != 0) {
283 0 : aicpusd_warn("ioctl failed, error[%s],errorno[%d]", strerror(errno), errno);
284 0 : (void)close(fd);
285 0 : return false;
286 : }
287 2 : (void)close(fd);
288 : }
289 28 : return true;
290 : }
291 :
292 1 : void CloseQslibHandle()
293 : {
294 1 : if (g_qslibHandle != nullptr) {
295 0 : (void)dlclose(g_qslibHandle);
296 0 : g_qslibHandle = nullptr;
297 0 : return;
298 : }
299 : }
300 : } // namespace
301 : } // namespace AicpuSchedule
302 :
303 : /**
304 : * main of compute process.
305 : * @param argc argv length
306 : * @param argv arg values
307 : * @return 0:success, other:failed
308 : */
309 : #ifndef aicpusd_UT
310 : int32_t main(int32_t argc, char *argv[])
311 : #else
312 39 : int32_t ComputeProcessMain(int32_t argc, char* argv[])
313 : #endif
314 : {
315 : try {
316 39 : AicpuSchedule::ArgsParser startParams;
317 39 : if (!startParams.ParseArgs(argc, argv)) {
318 13 : return -1;
319 : }
320 26 : AicpuSchedule::SetLogLevel(startParams);
321 26 : aicpusd_run_info("Start parameter. %s", startParams.GetParaParsedStr().c_str());
322 :
323 26 : const uint32_t vfId = startParams.GetVfId();
324 26 : const pid_t pid = startParams.GetHostPid();
325 26 : const uint32_t deviceId = startParams.GetDeviceId();
326 26 : const uint32_t aicpuProcNum = startParams.GetAicpuProcNum();
327 26 : AicpuSchedule::AicpuScheduleInterface::GetInstance().SetBatchLoadMode(aicpuProcNum);
328 26 : if (!AicpuSchedule::AddToCgroup(deviceId, vfId)) {
329 1 : AicpuSchedule::ReportErrorMsg(AicpuSchedule::AICPU_SCHEDULE_ERROR_CGROUP_FAILED,
330 : deviceId, static_cast<uint32_t>(pid), vfId);
331 1 : return -1;
332 : }
333 : // Make sure AicpusdLastword are created first，to ensure the last exit
334 25 : AicpuSchedule::AicpusdLastword::GetInstance();
335 25 : AicpuSchedule::AicpuProfiler::ProfilerAgentInit();
336 25 : AicpuSchedule::AicpuEventManager::GetInstance().InitEventMgr(false, true, 0U);
337 25 : std::vector<uint32_t> deviceVec(1, deviceId);
338 :
339 75 : int32_t ret = AicpuSchedule::AicpuScheduleInterface::GetInstance().InitAICPUScheduler(
340 50 : deviceVec, pid, startParams.GetPidSign(), startParams.GetProfilingMode(),
341 50 : vfId, true, startParams.GetHostProcName());
342 25 : if (ret != AicpuSchedule::AICPU_SCHEDULE_OK) {
343 5 : aicpusd_err("Aicpu scheduler start failed, ret=%d", ret);
344 5 : AicpuSchedule::ReportErrorMsg(ret, deviceId, static_cast<uint32_t>(pid), vfId);
345 5 : (void)AicpuSchedule::AicpuScheduleInterface::GetInstance().StopAICPUScheduler(deviceVec, pid);
346 5 : return -1;
347 : }
348 :
349 20 : std::string libPath = "";
350 20 : (void)AicpuSchedule::AicpuUtil::GetEnvVal(AicpuSchedule::ENV_NAME_LD_LIBRARY_PATH, libPath);
351 20 : aicpusd_run_info("Aicpu schedule start successfully, LD_LIBRARY_PATH=%s", libPath.c_str());
352 :
353 : // start attach and init process.
354 : // Attach group before send success to make sure aicpusd own the authority to access mbuff send by RTS
355 20 : if (!AicpuSchedule::AttachHostGroup(startParams.GetGrpNameList(), startParams.GetGrpNameNum())) {
356 3 : aicpusd_err("AttachHostGroup execute failed.");
357 3 : AicpuSchedule::ReportErrorMsg(AicpuSchedule::AICPU_SCHEDULE_ERROR_DRV_ERR,
358 : deviceId, static_cast<uint32_t>(pid), vfId);
359 3 : return -1;
360 : }
361 :
362 17 : aicpusd_run_info("Aicpu schedule attach and init successfully.");
363 : // send pid qos to tsd current send default value 0 to tsd
364 17 : AicpuSchedule::SendPidQosMsgToTsd(static_cast<uint32_t>(PRIORITY_LEVEL1), deviceId,
365 : static_cast<uint32_t>(pid), vfId);
366 :
367 : // reg queue schedule start and stop callback func
368 17 : AicpuSchedule::RegQueueScheduleModuleCallBack();
369 17 : AicpuSchedule::RegCreateMc2MantenanceThreadCallBack();
370 17 : AicpuSchedule::RegCreateDatadumpThreadCallBack();
371 17 : AicpuSchedule::RegCustProcessLoadPlatformCallBack();
372 :
373 17 : const int32_t rspRet = StartupResponse(deviceVec[0U], TSD_COMPUTE, static_cast<uint32_t>(pid), vfId);
374 17 : if (rspRet != AicpuSchedule::SUCCESS_VALUE) {
375 1 : aicpusd_err("Aicpu startup response return not ok, error code[%d].", rspRet);
376 1 : AicpuSchedule::CloseQslibHandle();
377 1 : return -1;
378 : }
379 16 : aicpusd_run_info("Aicpu startup response return successfully.");
380 :
381 16 : (void)AicpuSchedule::SendVfMsgToDrv(VM_QOS_PROCESS_STARTUP, deviceId, vfId);
382 :
383 : // wait for shutdown
384 16 : const int32_t waitRet = WaitForShutDown(deviceVec[0U]);
385 16 : if (waitRet != AicpuSchedule::SUCCESS_VALUE) {
386 1 : aicpusd_err("wait for shut down return not ok return not ok, error code[%d].", waitRet);
387 1 : ret = -1;
388 : } else {
389 15 : aicpusd_info("wait for shut down return successfully.");
390 15 : ret = 0;
391 : }
392 :
393 16 : AicpuSchedule::StopQsSubModuleInAicpu(deviceId, static_cast<uint32_t>(pid), vfId);
394 16 : (void)AicpuSchedule::AicpuScheduleInterface::GetInstance().StopAICPUScheduler(deviceVec, pid);
395 16 : aicpusd_run_info("Aicpu schedule stopped");
396 :
397 16 : (void)AicpuSchedule::SendVfMsgToDrv(VM_QOS_PROCESS_SUSPEND, deviceId, vfId);
398 : // flush cache log to slogd
399 16 : DlogFlushAicpu();
400 16 : return ret;
401 39 : } catch (const std::exception &e) {
402 0 : aicpusd_err("Execute main failed, reason=%s", e.what());
403 0 : return -1;
404 0 : }
405 : }
|
