Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include <cstdint>
12 : #include <string>
13 : #include <sstream>
14 : #include <unistd.h>
15 : #include <csignal>
16 : #include <dlfcn.h>
17 : #include <fstream>
18 : #include "driver/ascend_hal.h"
19 : #include "common/bqs_status.h"
20 : #include "common/bqs_log.h"
21 : #include "common/bqs_util.h"
22 : #include "qs_interface_process.h"
23 : #include "server/bind_cpu_utils.h"
24 : #include "tsd.h"
25 : #include "securec.h"
26 : #include "queue_schedule/qs_client.h"
27 : #include "bqs_weak_log.h"
28 : #include "server/qs_args_parser.h"
29 : #include "queue_schedule_sub_module_interface.h"
30 : #include "bqs_feature_ctrl.h"
31 :
32 : namespace bqs {
33 : namespace {
34 : void *g_aicpuSdlibHandle = nullptr;
35 : const std::string AICPU_SCHEDULER_SO_NAME = "libaicpu_scheduler.so";
36 : // mutex for condition variable
37 : std::mutex g_kMtx;
38 : // condition variable
39 : std::condition_variable g_kCv;
40 : // exit flag
41 : #ifndef aicpusd_UT
42 : std::atomic<bool> g_kExitFlag{false};
43 : #else
44 : std::atomic<bool> g_kExitFlag{true};
45 : #endif
46 :
47 : /**
48 : * wait for shutdown.
49 : * @return void
50 : */
51 6 : static void WaitShutdown()
52 : {
53 6 : std::unique_lock<std::mutex> lk(bqs::g_kMtx);
54 6 : while (!bqs::g_kExitFlag.load()) {
55 0 : bqs::g_kCv.wait(lk);
56 : }
57 6 : BQS_LOG_INFO("Exit WaitShutdown");
58 6 : }
59 :
60 : /**
61 : * handle SIGTERM sig.
62 : * @return void
63 : */
64 0 : static void HandleSignal(const int32_t sig)
65 : {
66 : (void) sig;
67 0 : bqs::g_kExitFlag = true;
68 0 : bqs::g_kCv.notify_one();
69 0 : }
70 : }
71 :
72 8 : void ReportErrorMsg(const int32_t errCode, const uint32_t deviceId, const pid_t hostPid, const uint32_t vfId)
73 : {
74 8 : BQS_LOG_RUN_WARN("ReportErrorMsg errcode is %d.", errCode);
75 8 : std::string errStr;
76 8 : switch (errCode) {
77 0 : case BQS_STATUS_ATTACH_GROUP_FALED: {
78 0 : errStr = ERROR_MSG_ATTACH_GROUP_FAILED;
79 0 : break;
80 : }
81 8 : default: {
82 8 : errStr = ERROR_MSG_QS_INIT_FAILED;
83 8 : break;
84 : }
85 : }
86 8 : BQS_LOG_RUN_INFO("ReportErrorMsg msg is %s.", errStr.c_str());
87 8 : (void) TsdReportStartOrStopErrCode(deviceId, TSD_QS, static_cast<uint32_t>(hostPid), vfId, errStr.c_str(),
88 8 : static_cast<uint32_t>(errStr.size()));
89 8 : }
90 :
91 20 : void RegAicpuSchedulerModuleCallBack()
92 : {
93 20 : g_aicpuSdlibHandle = dlopen(AICPU_SCHEDULER_SO_NAME.c_str(), RTLD_LAZY);
94 20 : if (g_aicpuSdlibHandle == nullptr) {
95 14 : BQS_LOG_WARN("cannot open so %s", AICPU_SCHEDULER_SO_NAME.c_str());
96 14 : return;
97 : }
98 :
99 : const SubProcEventCallBackFuncInfo startAicpuSdFunc =
100 6 : reinterpret_cast<SubProcEventCallBackFuncInfo>(dlsym(g_aicpuSdlibHandle, "StartAicpuSchedulerModule"));
101 6 : if (startAicpuSdFunc == nullptr) {
102 1 : BQS_LOG_ERROR("cannot find StartAicpuSchedulerModule");
103 1 : (void)dlclose(g_aicpuSdlibHandle);
104 1 : g_aicpuSdlibHandle = nullptr;
105 1 : return;
106 : }
107 :
108 : SubProcEventCallBackInfo startCallBackInfo;
109 5 : startCallBackInfo.callBackFunc = startAicpuSdFunc;
110 5 : startCallBackInfo.eventType = TSD_EVENT_START_AICPU_SD_MODULE;
111 5 : int32_t ret = RegEventMsgCallBackFunc(&startCallBackInfo);
112 5 : if (ret != 0) {
113 1 : BQS_LOG_ERROR("reg StartAicpuSchedulerModule failed");
114 1 : (void)dlclose(g_aicpuSdlibHandle);
115 1 : g_aicpuSdlibHandle = nullptr;
116 1 : return;
117 : }
118 :
119 : const SubProcEventCallBackFuncInfo stopAicpuSdFunc =
120 4 : reinterpret_cast<SubProcEventCallBackFuncInfo>(dlsym(g_aicpuSdlibHandle, "StopAicpuSchedulerModule"));
121 4 : if (stopAicpuSdFunc == nullptr) {
122 1 : UnRegEventMsgCallBackFunc(TSD_EVENT_START_AICPU_SD_MODULE);
123 1 : BQS_LOG_ERROR("cannot find StopAicpuSchedulerModule");
124 1 : (void)dlclose(g_aicpuSdlibHandle);
125 1 : g_aicpuSdlibHandle = nullptr;
126 1 : return;
127 : }
128 :
129 : SubProcEventCallBackInfo stopCallBackInfo;
130 3 : stopCallBackInfo.callBackFunc = stopAicpuSdFunc;
131 3 : stopCallBackInfo.eventType = TSD_EVENT_STOP_AICPU_SD_MODULE;
132 :
133 3 : ret = RegEventMsgCallBackFunc(&stopCallBackInfo);
134 3 : if (ret != 0) {
135 1 : UnRegEventMsgCallBackFunc(TSD_EVENT_START_AICPU_SD_MODULE);
136 1 : BQS_LOG_ERROR("reg StopAicpuSchedulerModule failed");
137 1 : (void)dlclose(g_aicpuSdlibHandle);
138 1 : g_aicpuSdlibHandle = nullptr;
139 1 : return;
140 : }
141 2 : BQS_LOG_RUN_INFO("RegAicpuSchedulerModuleCallBack success");
142 2 : return;
143 : }
144 12 : void StopAiCpuSubModuleInQs(const uint32_t deviceId, const uint32_t hostPid, const uint32_t vfId)
145 : {
146 12 : if (g_aicpuSdlibHandle == nullptr) {
147 11 : BQS_LOG_WARN("cannot open so %s .", AICPU_SCHEDULER_SO_NAME.c_str());
148 11 : return;
149 : }
150 1 : TsdSubEventInfo eventInfo = {};
151 1 : eventInfo.deviceId = deviceId;
152 1 : eventInfo.hostPid = hostPid;
153 1 : eventInfo.vfId = vfId;
154 : const SubProcEventCallBackFuncInfo stopAicpuSdFunc =
155 1 : reinterpret_cast<SubProcEventCallBackFuncInfo>(dlsym(g_aicpuSdlibHandle, "StopAicpuSchedulerModule"));
156 1 : if (stopAicpuSdFunc == nullptr) {
157 0 : BQS_LOG_ERROR("cannot find StopAicpuSchedulerModule");
158 0 : (void)dlclose(g_aicpuSdlibHandle);
159 0 : g_aicpuSdlibHandle = nullptr;
160 0 : return;
161 : }
162 1 : const int32_t ret = stopAicpuSdFunc(&eventInfo);
163 1 : if (ret != 0) {
164 0 : BQS_LOG_ERROR("StopAicpuSchedulerModule fail");
165 : }
166 1 : (void)dlclose(g_aicpuSdlibHandle);
167 1 : g_aicpuSdlibHandle = nullptr;
168 : }
169 :
170 8 : void CloseAicpuSdlibHandle()
171 : {
172 8 : if (g_aicpuSdlibHandle != nullptr) {
173 1 : (void)dlclose(g_aicpuSdlibHandle);
174 1 : g_aicpuSdlibHandle = nullptr;
175 1 : return;
176 : }
177 : }
178 :
179 38 : void GetEnv(const char_t * const envName, std::string &envValue)
180 : {
181 38 : const size_t envValueMaxLen = 1024UL * 1024UL;
182 38 : const char_t * const envTemp = std::getenv(envName);
183 38 : if ((envTemp == nullptr) || (strnlen(envTemp, envValueMaxLen) >= envValueMaxLen)) {
184 20 : BQS_LOG_WARN("Get env[%s] failed.", envName);
185 20 : return;
186 : }
187 18 : envValue = envTemp;
188 : }
189 :
190 19 : void SetLogLevelWithEnv(bqs::ArgsParser &startParams)
191 : {
192 19 : std::string envLogLevel;
193 19 : std::string envEventLevel;
194 19 : GetEnv("ASCEND_GLOBAL_LOG_LEVEL", envLogLevel);
195 19 : GetEnv("ASCEND_GLOBAL_EVENT_ENABLE", envEventLevel);
196 19 : BQS_LOG_RUN_INFO("Set log with env, envLogLevel[%s], envEventLevel[%s]",
197 : envLogLevel.c_str(), envEventLevel.c_str());
198 : int32_t logLevel;
199 : int32_t eventLevel;
200 19 : if (!TransStrToInt(envLogLevel, logLevel)) {
201 1 : logLevel = ERROR_LOG;
202 : }
203 19 : if (!TransStrToInt(envEventLevel, eventLevel)) {
204 19 : eventLevel = EVENT_LOG;
205 : }
206 19 : startParams.SetLogLevel(logLevel, eventLevel);
207 19 : }
208 :
209 20 : void InitQsInitParams(bqs::InitQsParams &initQsParams, const uint32_t deviceId,
210 : const uint32_t hostPid, const uint32_t vfId,
211 : const std::vector<uint32_t> resVec, const bqs::ArgsParser &startParams)
212 : {
213 20 : initQsParams.deviceId = deviceId;
214 20 : initQsParams.enqueGroupId = static_cast<uint32_t>(bqs::EventGroupId::ENQUEUE_GROUP_ID);
215 20 : initQsParams.f2nfGroupId = static_cast<uint32_t>(bqs::EventGroupId::F2NF_GROUP_ID);
216 20 : initQsParams.reschedInterval = static_cast<uint32_t>(startParams.GetReschedInterval());
217 20 : initQsParams.runMode = startParams.GetDeployMode();
218 20 : initQsParams.pid = hostPid;
219 20 : initQsParams.vfId = vfId;
220 20 : initQsParams.pidSign = startParams.GetPidSign();
221 20 : initQsParams.qsInitGrpName = startParams.GetGroupName();
222 20 : initQsParams.schedPolicy = startParams.GetSchedPolicy();
223 20 : initQsParams.starter = startParams.GetStarter();
224 20 : initQsParams.profCfgData = startParams.GetProfCfgData();
225 20 : initQsParams.abnormalInterVal = static_cast<uint32_t>(startParams.GetAbnormalInterval());
226 20 : initQsParams.profFlag = startParams.GetProfFlag();
227 20 : initQsParams.enqueGroupIdExtra = bqs::EventGroupId::ENQUEUE_GROUP_ID_EXTRA;
228 20 : initQsParams.f2nfGroupIdExtra = bqs::EventGroupId::F2NF_GROUP_ID_EXTRA;
229 20 : uint32_t extraDevceId = 0U;
230 20 : if (resVec.size() > 1U) {
231 5 : extraDevceId = (resVec[0U] == deviceId) ? resVec[1U] : resVec[0U];
232 : }
233 20 : initQsParams.deviceIdExtra = extraDevceId;
234 20 : initQsParams.numaFlag = (resVec.size() > 1U);
235 20 : initQsParams.devIdVec = startParams.GetDevIdVec();
236 20 : initQsParams.needAttachGroup = true;
237 20 : }
238 : }
239 :
240 : /**
241 : * main of queue schedule
242 : * @param argc argv length
243 : * @param argv arg values
244 : * @return 0:success, other:failed
245 : */
246 : #ifndef aicpusd_UT
247 : int32_t main(int32_t argc, char_t *argv[])
248 : #else
249 32 : int32_t QueueScheduleMain(int32_t argc, char_t *argv[])
250 : #endif
251 : {
252 : try {
253 32 : BQS_LOG_RUN_INFO("QueueSchedule begin");
254 32 : bqs::ArgsParser startParams;
255 32 : if (!startParams.ParseArgs(argc, argv)) {
256 12 : BQS_LOG_ERROR("Parse args failed");
257 12 : return -1;
258 : }
259 :
260 20 : BQS_LOG_RUN_INFO("Start parameter. %s", startParams.GetParaParsedStr().c_str());
261 :
262 20 : if (!startParams.GetWithLogLevel()) {
263 19 : bqs::SetLogLevelWithEnv(startParams);
264 : }
265 :
266 20 : const uint32_t vfId = startParams.GetVfId();
267 20 : const uint32_t hostPid = startParams.GetHostPid();
268 20 : uint32_t deviceId = startParams.GetDeviceId();
269 20 : const std::vector<uint32_t> resVec = startParams.GetResvec();
270 20 : BQS_LOG_RUN_INFO("resVec size is %zu", resVec.size());
271 20 : if (resVec.size() >= 1) {
272 5 : deviceId = resVec[0];
273 : }
274 20 : if ((bqs::GetRunContext() == bqs::RunContext::DEVICE) && (!bqs::BindCpuUtils::AddToCgroup(deviceId, vfId))) {
275 0 : return -1;
276 : }
277 :
278 20 : bqs::RegAicpuSchedulerModuleCallBack();
279 :
280 20 : auto& qsInterface = bqs::QueueScheduleInterface::GetInstance();
281 :
282 20 : bqs::InitQsParams initQsParams = {};
283 20 : bqs::InitQsInitParams(initQsParams, deviceId, hostPid, vfId, resVec, startParams);
284 20 : const auto bsqStatus = qsInterface.InitQueueScheduler(initQsParams);
285 20 : if (bsqStatus != bqs::BQS_STATUS_OK) {
286 8 : BQS_LOG_ERROR("QueueSchedule start failed, ret=%d .", bsqStatus);
287 8 : bqs::ReportErrorMsg(bsqStatus, deviceId, hostPid, vfId);
288 8 : bqs::CloseAicpuSdlibHandle();
289 8 : return -1;
290 : }
291 12 : BQS_LOG_RUN_INFO("QueueSchedule start success");
292 : // wait for shutdown
293 12 : int32_t waitRet = 0;
294 21 : if ((startParams.GetStarter() == bqs::QsStartType::START_BY_TSD) &&
295 9 : (bqs::GetRunContext() != bqs::RunContext::HOST)) {
296 6 : waitRet = TsdWaitForShutdown(deviceId, TSD_QS, hostPid, vfId);
297 : } else {
298 : // register a signal handler
299 6 : (void) std::signal(SIGTERM, static_cast<sighandler_t>(&bqs::HandleSignal));
300 6 : bqs::WaitShutdown();
301 : }
302 12 : int32_t ret = 0;
303 12 : if (waitRet != 0) {
304 1 : BQS_LOG_ERROR("Tsd wait for shut down return not ok, error code[%d].", waitRet);
305 1 : ret = -1;
306 : } else {
307 11 : BQS_LOG_INFO("Tsd wait for shut down success");
308 11 : ret = 0;
309 : }
310 :
311 12 : bqs::StopAiCpuSubModuleInQs(deviceId, hostPid, vfId);
312 12 : qsInterface.WaitForStop();
313 12 : (void)qsInterface.Destroy();
314 12 : BQS_LOG_RUN_INFO("QueueSchedule exit");
315 12 : return ret;
316 32 : } catch(...) {
317 0 : BQS_LOG_ERROR("QueueSchedule run exception.");
318 0 : return -1;
319 0 : }
320 : }
|
