Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 : #include "aicpusd_threads_process.h"
11 :
12 : #include "ascend_hal.h"
13 : #include "profiling_adp.h"
14 : #include "aicpusd_status.h"
15 : #include "aicpusd_event_manager.h"
16 : #include "aicpu_context.h"
17 : #include "aicpusd_common.h"
18 : #include "aicpusd_drv_manager.h"
19 : #include "aicpusd_worker.h"
20 : #include "aicpusd_monitor.h"
21 : #include "aicpusd_hal_interface_ref.h"
22 : #include "aicpu_prof.h"
23 : #include "aicpu_cust_sd_dump_process.h"
24 : #include "aicpusd_util.h"
25 :
26 : namespace AicpuSchedule {
27 18 : ComputeProcess::ComputeProcess()
28 18 : : deviceId_(0U),
29 18 : hostPid_(-1),
30 18 : aicpuNum_(0U),
31 18 : profilingMode_(PROFILING_CLOSE),
32 18 : aicpuPid_(-1),
33 18 : vfId_(0U),
34 18 : runMode_(aicpu::AicpuRunMode::THREAD_MODE) {}
35 :
36 25 : ComputeProcess& ComputeProcess::GetInstance()
37 : {
38 25 : static ComputeProcess instance;
39 25 : return instance;
40 : }
41 :
42 8 : void ComputeProcess::UpdateProfilingSetting(uint32_t flag)
43 : {
44 8 : ProfilingMode profilingMode = PROFILING_CLOSE;
45 8 : bool kernelFlag = false;
46 8 : AicpuUtil::GetProfilingInfo(flag, profilingMode, kernelFlag);
47 8 : profilingMode_ = profilingMode;
48 8 : if (kernelFlag) {
49 1 : aicpu::UpdateMode(profilingMode == PROFILING_OPEN);
50 : }
51 8 : aicpusd_info("Update aicpu profiling mode success, flag[%u], profilingMode[%u], kernelFlag[%d],",
52 : flag, profilingMode, kernelFlag);
53 8 : }
54 :
55 12 : int32_t ComputeProcess::Start(const uint32_t deviceId,
56 : const pid_t hostPid,
57 : const uint32_t profilingMode,
58 : const pid_t aicpuPid,
59 : const uint32_t vfId,
60 : const aicpu::AicpuRunMode runMode)
61 : {
62 12 : aicpusd_info("AicpuCustSd start, deviceId[%u] hostpid[%d] profilingMode[%u] aicpuPid[%d] runMode[%d] vfId[%u].",
63 : deviceId, hostPid, profilingMode, aicpuPid, static_cast<int32_t>(runMode), vfId);
64 :
65 12 : const AicpuSchedule::AicpuDrvManager &drvMgr = AicpuSchedule::AicpuDrvManager::GetInstance();
66 12 : aicpuNum_ = drvMgr.GetAicpuNum();
67 :
68 12 : deviceId_ = deviceId;
69 12 : hostPid_ = hostPid;
70 12 : runMode_ = runMode;
71 12 : aicpuPid_ = aicpuPid;
72 12 : vfId_ = vfId;
73 :
74 12 : if ((aicpuNum_ > 0U) && (aicpu::InitTaskMonitorContext(aicpuNum_) != aicpu::AICPU_ERROR_NONE)) {
75 1 : aicpusd_err("Init task monitor context failed");
76 1 : return static_cast<int32_t>(ComputProcessRetCode::CP_RET_COMMON_ERROR);
77 : }
78 11 : if (runMode_ == aicpu::AicpuRunMode::PROCESS_PCIE_MODE) {
79 4 : if (&halMemBindSibling == nullptr) {
80 0 : aicpusd_err("Interface halMemBindSibling is not supported in current device");
81 0 : return AICPU_SCHEDULE_ERROR_DRV_ERR;
82 : }
83 4 : drvError_t drvRet = DRV_ERROR_NONE;
84 4 : if (AicpuSchedule::AicpuDrvManager::GetInstance().GetSafeVerifyFlag()) {
85 3 : drvRet = halMemBindSibling(hostPid, aicpuPid, vfId_, deviceId, SVM_MEM_BIND_SVM_GRP);
86 : } else {
87 : //new
88 1 : aicpusd_info("open prof so and bind sp group without alloc permission.");
89 1 : aicpu::LoadProfilingLib();
90 1 : drvRet = halMemBindSibling(hostPid, aicpuPid, vfId_, deviceId, SVM_MEM_BIND_SVM_GRP | SVM_MEM_BIND_SP_GRP_NO_ALLOC);
91 : }
92 :
93 4 : if (drvRet != DRV_ERROR_NONE) {
94 3 : aicpusd_err("Failed to halMemBindSibling, hostpid[%d], aicpusd pid[%d], deviceId[%u], vfId[%u], ret[%d].",
95 : hostPid, aicpuPid, deviceId, vfId, drvRet);
96 3 : return static_cast<int32_t>(ComputProcessRetCode::CP_RET_COMMON_ERROR);
97 : }
98 1 : aicpusd_info("Bind Sibling pid success, hostpid[%d] aicpusd pid[%d] deviceId[%u] vfId[%u].",
99 : hostPid, aicpuPid, deviceId, vfId);
100 : }
101 8 : aicpu::InitProfilingDataInfo(deviceId_, hostPid, CHANNEL_CUS_AICPU);
102 8 : UpdateProfilingSetting(profilingMode);
103 8 : if (profilingMode != 0U) {
104 8 : aicpu::LoadProfilingLib();
105 8 : aicpu::SetProfilingFlagForKFC(profilingMode);
106 8 : aicpu::UpdateMode((profilingMode & 1) == PROFILING_OPEN);
107 : }
108 8 : (void)aicpu::SetAicpuRunMode(runMode_);
109 8 : aicpu::SetCustAicpuSdFlag(true);
110 8 : const uint32_t ret = RegisterScheduleTask();
111 8 : if (ret != AICPU_SCHEDULE_OK) {
112 1 : aicpusd_err("Register aicpu split and random kernel scheduler failed.");
113 1 : return static_cast<int32_t>(ret);
114 : }
115 :
116 7 : const int32_t aicpuStartRet = AicpuSchedule::ThreadPool::Instance().CreateWorker();
117 7 : if (aicpuStartRet != AICPU_SCHEDULE_OK) {
118 1 : aicpusd_err("Drv create aicpu work tasks failed, ret[%d].", aicpuStartRet);
119 1 : return static_cast<int32_t>(ComputProcessRetCode::CP_RET_COMMON_ERROR);
120 : }
121 12 : AicpuCustDumpProcess::GetInstance().InitDumpProcess(deviceId,
122 6 : AicpuDrvManager::GetInstance().GetAicpuNum());
123 6 : aicpusd_info("Aicpu custom scheduler start succeed, deviceId[%u], hostpid[%d], profilingMode[%u], runMode[%d].",
124 : deviceId, hostPid, profilingMode, runMode_);
125 6 : return static_cast<int32_t>(ComputProcessRetCode::CP_RET_SUCCESS);
126 : }
127 :
128 24 : uint32_t ComputeProcess::RegisterScheduleTask()
129 : {
130 3 : const auto randomKernelScheduler = [this] (const aicpu::Closure &task) {
131 3 : return SubmitRandomKernelTask(task);
132 24 : };
133 10 : const auto splitKernelScheduler = [this] (const uint32_t parallelId, const int64_t shardNum,
134 : const std::queue<aicpu::Closure> &taskQueue) {
135 10 : const AICPUSharderTaskInfo taskInfo = {.parallelId=parallelId, .shardNum=shardNum};
136 20 : return SubmitSplitKernelTask(taskInfo, taskQueue);
137 24 : };
138 3 : const auto splitKernelGetProcesser = [this] () {
139 3 : return GetAndDoSplitKernelTask();
140 24 : };
141 :
142 24 : aicpu::SharderNonBlock::GetInstance().Register(aicpuNum_, randomKernelScheduler, splitKernelScheduler,
143 : splitKernelGetProcesser);
144 :
145 24 : return AICPU_SCHEDULE_OK;
146 : }
147 :
148 3 : uint32_t ComputeProcess::SubmitRandomKernelTask(const aicpu::Closure &task)
149 : {
150 3 : if (!randomKernelTask_.Enqueue(task)) {
151 1 : aicpusd_err("Add random kernel task failed.");
152 1 : return AICPU_SCHEDULE_ERROR_INNER_ERROR;
153 : }
154 :
155 2 : AICPUSubEventInfo aicpuEventInfo = {};
156 2 : event_summary eventInfoSummary = {};
157 2 : eventInfoSummary.pid = getpid();
158 2 : eventInfoSummary.event_id = EVENT_RANDOM_KERNEL;
159 2 : eventInfoSummary.msg = PtrToPtr<AICPUSubEventInfo, char_t>(&aicpuEventInfo);
160 2 : eventInfoSummary.msg_len = static_cast<uint32_t>(sizeof(AICPUSubEventInfo));
161 :
162 2 : const int32_t drvRet = halEschedSubmitEvent(deviceId_, &eventInfoSummary);
163 2 : if (drvRet != DRV_ERROR_NONE) {
164 1 : aicpusd_err("Submit random kernel event failed. ret=%d", drvRet);
165 1 : return AICPU_SCHEDULE_ERROR_DRV_ERR;
166 : }
167 :
168 1 : return AICPU_SCHEDULE_OK;
169 : }
170 :
171 10 : uint32_t ComputeProcess::SubmitSplitKernelTask(const AICPUSharderTaskInfo &taskInfo,
172 : const std::queue<aicpu::Closure> &taskQueue)
173 : {
174 10 : if (!splitKernelTask_.BatchAddTask(taskInfo, taskQueue)) {
175 1 : aicpusd_err("Add split kernel task to map failed, parallelId=%u", taskInfo.parallelId);
176 1 : return AICPU_SCHEDULE_ERROR_INNER_ERROR;
177 : }
178 :
179 9 : uint32_t ret = AICPU_SCHEDULE_OK;
180 9 : if (FeatureCtrl::ShouldSubmitTaskOneByOne()) {
181 2 : ret = SubmitBatchSplitKernelEventOneByOne(taskInfo);
182 : } else {
183 7 : ret = SubmitBatchSplitKernelEventDc(taskInfo);
184 : }
185 9 : if (ret != AICPU_SCHEDULE_OK) {
186 2 : aicpusd_err("Submit batch split kernel event failed. parallelId=%u, submitNum=%ld",
187 : taskInfo.parallelId, taskInfo.shardNum);
188 2 : return ret;
189 : }
190 :
191 7 : aicpusd_info("Submit split kernel event success. parallelId=%u, submitNum=%ld",
192 : taskInfo.parallelId, taskInfo.shardNum);
193 :
194 7 : return AICPU_SCHEDULE_OK;
195 : }
196 :
197 2 : uint32_t ComputeProcess::SubmitBatchSplitKernelEventOneByOne(const AICPUSharderTaskInfo &taskInfo) const
198 : {
199 2 : const uint32_t submitNum = static_cast<uint32_t>(taskInfo.shardNum);
200 3 : for (uint32_t i = 0U; i < submitNum; ++i) {
201 2 : const uint32_t ret = SubmitOneSplitKernelEvent(taskInfo);
202 2 : if (ret != AICPU_SCHEDULE_OK) {
203 1 : aicpusd_err("Submit single split kernel event failed. parallelId=%u, i=%u",
204 : taskInfo.parallelId, i);
205 1 : return ret;
206 : }
207 : }
208 :
209 1 : return AICPU_SCHEDULE_OK;
210 : }
211 :
212 7 : uint32_t ComputeProcess::SubmitBatchSplitKernelEventDc(const AICPUSharderTaskInfo &taskInfo)
213 : {
214 7 : AICPUSubEventInfo aicpuEventInfo = {};
215 7 : aicpuEventInfo.para.sharderTaskInfo = taskInfo;
216 7 : event_summary eventInfoSummary = {};
217 7 : eventInfoSummary.pid = getpid();
218 7 : eventInfoSummary.event_id = EVENT_SPLIT_KERNEL;
219 7 : eventInfoSummary.msg = PtrToPtr<AICPUSubEventInfo, char_t>(&aicpuEventInfo);
220 7 : eventInfoSummary.msg_len = static_cast<uint32_t>(sizeof(AICPUSubEventInfo));
221 7 : uint32_t submitSuccessNum = 0U;
222 7 : const uint32_t submitNum = static_cast<uint32_t>(taskInfo.shardNum);
223 7 : const int32_t drvRet = halEschedSubmitEventBatch(deviceId_, SHARED_EVENT_ENTRY,
224 7 : &eventInfoSummary, submitNum, &submitSuccessNum);
225 7 : if ((drvRet == DRV_ERROR_NONE) && (submitSuccessNum == submitNum)) {
226 4 : aicpusd_info("Batch submit split kernel event success, parallelId=%u, submitNum=%u",
227 : taskInfo.parallelId, submitNum);
228 4 : return AICPU_SCHEDULE_OK;
229 : }
230 :
231 : /*
232 : * The queue depth of event schedule is only dozens. If too many split kernel event are submited,
233 : * the queue will be full. Therefore, the main thread needs to process the task sending failure.
234 : */
235 3 : aicpusd_warn("Batch submit some of split kernel event success, ret=%d, parallelId=%u, submitNum=%u, "
236 : "submitSuccessNum=%u", drvRet, taskInfo.parallelId, submitNum, submitSuccessNum);
237 :
238 3 : const uint32_t remainNum = (drvRet == DRV_ERROR_NONE) ? submitNum - submitSuccessNum : submitNum;
239 5 : for (uint32_t i = 0U; i < remainNum; ++i) {
240 3 : if (!DoSplitKernelTask(taskInfo)) {
241 1 : aicpusd_err("Run single task failed after batch submit fail, parallelId=%u, submitNum=%u, "
242 : "remainNum=%u, i=%u", taskInfo.parallelId, submitNum, remainNum, i);
243 1 : return AICPU_SCHEDULE_ERROR_INNER_ERROR;
244 : }
245 : }
246 :
247 2 : return AICPU_SCHEDULE_OK;
248 : }
249 :
250 2 : uint32_t ComputeProcess::SubmitOneSplitKernelEvent(const AICPUSharderTaskInfo &taskInfo) const
251 : {
252 2 : AICPUSubEventInfo aicpuEventInfo = {};
253 2 : aicpuEventInfo.para.sharderTaskInfo = taskInfo;
254 2 : event_summary eventInfoSummary = {};
255 2 : eventInfoSummary.pid = getpid();
256 2 : eventInfoSummary.event_id = EVENT_SPLIT_KERNEL;
257 2 : eventInfoSummary.msg = PtrToPtr<AICPUSubEventInfo, char_t>(&aicpuEventInfo);
258 2 : eventInfoSummary.msg_len = static_cast<uint32_t>(sizeof(AICPUSubEventInfo));
259 :
260 2 : const int32_t drvRet = halEschedSubmitEvent(deviceId_, &eventInfoSummary);
261 2 : if (drvRet != DRV_ERROR_NONE) {
262 1 : aicpusd_err("Submit split kernel event failed. ret=%d, parallelId=%u",
263 : drvRet, taskInfo.parallelId);
264 1 : return AICPU_SCHEDULE_ERROR_DRV_ERR;
265 : }
266 :
267 1 : return AICPU_SCHEDULE_OK;
268 : }
269 :
270 3 : bool ComputeProcess::GetAndDoSplitKernelTask()
271 : {
272 3 : event_info eventInfo = {};
273 3 : const uint32_t threadIndex = aicpu::GetAicpuThreadIndex();
274 3 : const int32_t retVal = halEschedGetEvent(deviceId_, AicpuSchedule::DEFAULT_GROUP_ID, threadIndex,
275 3 : EVENT_SPLIT_KERNEL, &eventInfo);
276 3 : if (retVal == DRV_ERROR_NO_EVENT) {
277 1 : return true;
278 : }
279 2 : if (retVal != DRV_ERROR_NONE) {
280 1 : aicpusd_err("Cannot get event, threadIndex=%u, ret=%d", threadIndex, retVal);
281 1 : return false;
282 : }
283 :
284 1 : const AICPUSubEventInfo * const subEventInfo = PtrToPtr<const char_t, const AICPUSubEventInfo>(eventInfo.priv.msg);
285 1 : aicpusd_info("Begin to process split kernel event. parallelId=%u, threadIdx=%u, type=get",
286 : subEventInfo->para.sharderTaskInfo.parallelId, threadIndex);
287 :
288 1 : return DoSplitKernelTask(subEventInfo->para.sharderTaskInfo);
289 : }
290 :
291 5 : bool ComputeProcess::DoSplitKernelTask(const AICPUSharderTaskInfo &taskInfo)
292 : {
293 5 : aicpu::Closure task;
294 5 : if (!splitKernelTask_.PopTask(taskInfo, task)) {
295 1 : aicpusd_run_warn("Get split kernel task from map failed, parallelId=%u, %s",
296 : taskInfo.parallelId, splitKernelTask_.DebugString().c_str());
297 1 : return true;
298 : }
299 :
300 : try {
301 4 : task();
302 1 : } catch (std::exception &e) {
303 1 : aicpusd_err("Run split kernel task failed. parallelId=%u, exception=%s",
304 : taskInfo.parallelId, e.what());
305 1 : return false;
306 1 : }
307 :
308 3 : return true;
309 5 : }
310 :
311 3 : bool ComputeProcess::DoRandomKernelTask()
312 : {
313 3 : aicpu::Closure task;
314 3 : if (!randomKernelTask_.Dequeue(task)) {
315 1 : aicpusd_err("Get random kernel task from map failed, %s",
316 : randomKernelTask_.DebugString().c_str());
317 1 : return false;
318 : }
319 :
320 : try {
321 2 : task();
322 1 : } catch (std::exception &e) {
323 1 : aicpusd_err("Run random kernel task failed. exception=%s", e.what());
324 1 : return false;
325 1 : }
326 :
327 1 : return true;
328 3 : }
329 :
330 5 : void ComputeProcess::Stop()
331 : {
332 5 : splitKernelTask_.Clear();
333 5 : randomKernelTask_.Clear();
334 5 : if (profilingMode_ == PROFILING_OPEN) {
335 5 : aicpu::ReleaseProfiling();
336 : }
337 5 : }
338 : }
|
