Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 : #include "aicpusd_threads_process.h"
11 :
12 : #include "ascend_hal.h"
13 : #include "profiling_adp.h"
14 : #include "aicpusd_status.h"
15 : #include "aicpusd_event_manager.h"
16 : #include "aicpu_context.h"
17 : #include "aicpusd_common.h"
18 : #include "aicpusd_drv_manager.h"
19 : #include "aicpusd_worker.h"
20 : #include "aicpusd_monitor.h"
21 : #include "aicpusd_hal_interface_ref.h"
22 : #include "aicpu_prof.h"
23 : #include "aicpu_cust_sd_dump_process.h"
24 : #include "aicpusd_util.h"
25 :
26 : namespace AicpuSchedule {
27 18 : ComputeProcess::ComputeProcess()
28 18 : : deviceId_(0U),
29 18 : hostPid_(-1),
30 18 : aicpuNum_(0U),
31 18 : profilingMode_(PROFILING_CLOSE),
32 18 : aicpuPid_(-1),
33 18 : vfId_(0U),
34 18 : runMode_(aicpu::AicpuRunMode::THREAD_MODE)
35 18 : {}
36 :
37 25 : ComputeProcess& ComputeProcess::GetInstance()
38 : {
39 25 : static ComputeProcess instance;
40 25 : return instance;
41 : }
42 :
43 8 : void ComputeProcess::UpdateProfilingSetting(uint32_t flag)
44 : {
45 8 : ProfilingMode profilingMode = PROFILING_CLOSE;
46 8 : bool kernelFlag = false;
47 8 : AicpuUtil::GetProfilingInfo(flag, profilingMode, kernelFlag);
48 8 : profilingMode_ = profilingMode;
49 8 : if (kernelFlag) {
50 1 : aicpu::UpdateMode(profilingMode == PROFILING_OPEN);
51 : }
52 8 : aicpusd_info(
53 : "Update aicpu profiling mode success, flag[%u], profilingMode[%u], kernelFlag[%d],", flag, profilingMode,
54 : kernelFlag);
55 8 : }
56 :
57 12 : int32_t ComputeProcess::Start(
58 : const uint32_t deviceId, const pid_t hostPid, const uint32_t profilingMode, const pid_t aicpuPid,
59 : const uint32_t vfId, const aicpu::AicpuRunMode runMode)
60 : {
61 12 : aicpusd_info(
62 : "AicpuCustSd start, deviceId[%u] hostpid[%d] profilingMode[%u] aicpuPid[%d] runMode[%d] vfId[%u].", deviceId,
63 : hostPid, profilingMode, aicpuPid, static_cast<int32_t>(runMode), vfId);
64 :
65 12 : const AicpuSchedule::AicpuDrvManager& drvMgr = AicpuSchedule::AicpuDrvManager::GetInstance();
66 12 : aicpuNum_ = drvMgr.GetAicpuNum();
67 :
68 12 : deviceId_ = deviceId;
69 12 : hostPid_ = hostPid;
70 12 : runMode_ = runMode;
71 12 : aicpuPid_ = aicpuPid;
72 12 : vfId_ = vfId;
73 :
74 12 : if ((aicpuNum_ > 0U) && (aicpu::InitTaskMonitorContext(aicpuNum_) != aicpu::AICPU_ERROR_NONE)) {
75 1 : aicpusd_err("Init task monitor context failed");
76 1 : return static_cast<int32_t>(ComputProcessRetCode::CP_RET_COMMON_ERROR);
77 : }
78 11 : if (runMode_ == aicpu::AicpuRunMode::PROCESS_PCIE_MODE) {
79 4 : if (&halMemBindSibling == nullptr) {
80 0 : aicpusd_err("Interface halMemBindSibling is not supported in current device");
81 0 : return AICPU_SCHEDULE_ERROR_DRV_ERR;
82 : }
83 4 : drvError_t drvRet = DRV_ERROR_NONE;
84 4 : if (AicpuSchedule::AicpuDrvManager::GetInstance().GetSafeVerifyFlag()) {
85 3 : drvRet = halMemBindSibling(hostPid, aicpuPid, vfId_, deviceId, SVM_MEM_BIND_SVM_GRP);
86 : } else {
87 : // new
88 1 : aicpusd_info("open prof so and bind sp group without alloc permission.");
89 1 : aicpu::LoadProfilingLib();
90 1 : drvRet = halMemBindSibling(
91 : hostPid, aicpuPid, vfId_, deviceId, SVM_MEM_BIND_SVM_GRP | SVM_MEM_BIND_SP_GRP_NO_ALLOC);
92 : }
93 :
94 4 : if (drvRet != DRV_ERROR_NONE) {
95 3 : aicpusd_err(
96 : "Failed to halMemBindSibling, hostpid[%d], aicpusd pid[%d], deviceId[%u], vfId[%u], ret[%d].", hostPid,
97 : aicpuPid, deviceId, vfId, drvRet);
98 3 : return static_cast<int32_t>(ComputProcessRetCode::CP_RET_COMMON_ERROR);
99 : }
100 1 : aicpusd_info(
101 : "Bind Sibling pid success, hostpid[%d] aicpusd pid[%d] deviceId[%u] vfId[%u].", hostPid, aicpuPid, deviceId,
102 : vfId);
103 : }
104 8 : aicpu::InitProfilingDataInfo(deviceId_, hostPid, CHANNEL_CUS_AICPU);
105 8 : UpdateProfilingSetting(profilingMode);
106 8 : if (profilingMode != 0U) {
107 8 : aicpu::LoadProfilingLib();
108 8 : aicpu::SetProfilingFlagForKFC(profilingMode);
109 8 : aicpu::UpdateMode((profilingMode & 1) == PROFILING_OPEN);
110 : }
111 8 : (void)aicpu::SetAicpuRunMode(runMode_);
112 8 : aicpu::SetCustAicpuSdFlag(true);
113 8 : const uint32_t ret = RegisterScheduleTask();
114 8 : if (ret != AICPU_SCHEDULE_OK) {
115 1 : aicpusd_err("Register aicpu split and random kernel scheduler failed.");
116 1 : return static_cast<int32_t>(ret);
117 : }
118 :
119 7 : const int32_t aicpuStartRet = AicpuSchedule::ThreadPool::Instance().CreateWorker();
120 7 : if (aicpuStartRet != AICPU_SCHEDULE_OK) {
121 1 : aicpusd_err("Drv create aicpu work tasks failed, ret[%d].", aicpuStartRet);
122 1 : return static_cast<int32_t>(ComputProcessRetCode::CP_RET_COMMON_ERROR);
123 : }
124 6 : AicpuCustDumpProcess::GetInstance().InitDumpProcess(deviceId, AicpuDrvManager::GetInstance().GetAicpuNum());
125 6 : aicpusd_info(
126 : "Aicpu custom scheduler start succeed, deviceId[%u], hostpid[%d], profilingMode[%u], runMode[%d].", deviceId,
127 : hostPid, profilingMode, runMode_);
128 6 : return static_cast<int32_t>(ComputProcessRetCode::CP_RET_SUCCESS);
129 : }
130 :
131 24 : uint32_t ComputeProcess::RegisterScheduleTask()
132 : {
133 3 : const auto randomKernelScheduler = [this](const aicpu::Closure& task) { return SubmitRandomKernelTask(task); };
134 : const auto splitKernelScheduler =
135 10 : [this](const uint32_t parallelId, const int64_t shardNum, const std::queue<aicpu::Closure>& taskQueue) {
136 10 : const AICPUSharderTaskInfo taskInfo = {.parallelId = parallelId, .shardNum = shardNum};
137 20 : return SubmitSplitKernelTask(taskInfo, taskQueue);
138 24 : };
139 3 : const auto splitKernelGetProcesser = [this]() { return GetAndDoSplitKernelTask(); };
140 :
141 24 : aicpu::SharderNonBlock::GetInstance().Register(
142 : aicpuNum_, randomKernelScheduler, splitKernelScheduler, splitKernelGetProcesser);
143 :
144 24 : return AICPU_SCHEDULE_OK;
145 : }
146 :
147 3 : uint32_t ComputeProcess::SubmitRandomKernelTask(const aicpu::Closure& task)
148 : {
149 3 : if (!randomKernelTask_.Enqueue(task)) {
150 1 : aicpusd_err("Add random kernel task failed.");
151 1 : return AICPU_SCHEDULE_ERROR_INNER_ERROR;
152 : }
153 :
154 2 : AICPUSubEventInfo aicpuEventInfo = {};
155 2 : event_summary eventInfoSummary = {};
156 2 : eventInfoSummary.pid = getpid();
157 2 : eventInfoSummary.event_id = EVENT_RANDOM_KERNEL;
158 2 : eventInfoSummary.msg = PtrToPtr<AICPUSubEventInfo, char_t>(&aicpuEventInfo);
159 2 : eventInfoSummary.msg_len = static_cast<uint32_t>(sizeof(AICPUSubEventInfo));
160 :
161 2 : const int32_t drvRet = halEschedSubmitEvent(deviceId_, &eventInfoSummary);
162 2 : if (drvRet != DRV_ERROR_NONE) {
163 1 : aicpusd_err("Submit random kernel event failed. ret=%d", drvRet);
164 1 : return AICPU_SCHEDULE_ERROR_DRV_ERR;
165 : }
166 :
167 1 : return AICPU_SCHEDULE_OK;
168 : }
169 :
170 10 : uint32_t ComputeProcess::SubmitSplitKernelTask(
171 : const AICPUSharderTaskInfo& taskInfo, const std::queue<aicpu::Closure>& taskQueue)
172 : {
173 10 : if (!splitKernelTask_.BatchAddTask(taskInfo, taskQueue)) {
174 1 : aicpusd_err("Add split kernel task to map failed, parallelId=%u", taskInfo.parallelId);
175 1 : return AICPU_SCHEDULE_ERROR_INNER_ERROR;
176 : }
177 :
178 9 : uint32_t ret = AICPU_SCHEDULE_OK;
179 9 : if (FeatureCtrl::ShouldSubmitTaskOneByOne()) {
180 2 : ret = SubmitBatchSplitKernelEventOneByOne(taskInfo);
181 : } else {
182 7 : ret = SubmitBatchSplitKernelEventDc(taskInfo);
183 : }
184 9 : if (ret != AICPU_SCHEDULE_OK) {
185 2 : aicpusd_err(
186 : "Submit batch split kernel event failed. parallelId=%u, submitNum=%ld", taskInfo.parallelId,
187 : taskInfo.shardNum);
188 2 : return ret;
189 : }
190 :
191 7 : aicpusd_info(
192 : "Submit split kernel event success. parallelId=%u, submitNum=%ld", taskInfo.parallelId, taskInfo.shardNum);
193 :
194 7 : return AICPU_SCHEDULE_OK;
195 : }
196 :
197 2 : uint32_t ComputeProcess::SubmitBatchSplitKernelEventOneByOne(const AICPUSharderTaskInfo& taskInfo) const
198 : {
199 2 : const uint32_t submitNum = static_cast<uint32_t>(taskInfo.shardNum);
200 3 : for (uint32_t i = 0U; i < submitNum; ++i) {
201 2 : const uint32_t ret = SubmitOneSplitKernelEvent(taskInfo);
202 2 : if (ret != AICPU_SCHEDULE_OK) {
203 1 : aicpusd_err("Submit single split kernel event failed. parallelId=%u, i=%u", taskInfo.parallelId, i);
204 1 : return ret;
205 : }
206 : }
207 :
208 1 : return AICPU_SCHEDULE_OK;
209 : }
210 :
211 7 : uint32_t ComputeProcess::SubmitBatchSplitKernelEventDc(const AICPUSharderTaskInfo& taskInfo)
212 : {
213 7 : AICPUSubEventInfo aicpuEventInfo = {};
214 7 : aicpuEventInfo.para.sharderTaskInfo = taskInfo;
215 7 : event_summary eventInfoSummary = {};
216 7 : eventInfoSummary.pid = getpid();
217 7 : eventInfoSummary.event_id = EVENT_SPLIT_KERNEL;
218 7 : eventInfoSummary.msg = PtrToPtr<AICPUSubEventInfo, char_t>(&aicpuEventInfo);
219 7 : eventInfoSummary.msg_len = static_cast<uint32_t>(sizeof(AICPUSubEventInfo));
220 7 : uint32_t submitSuccessNum = 0U;
221 7 : const uint32_t submitNum = static_cast<uint32_t>(taskInfo.shardNum);
222 : const int32_t drvRet =
223 7 : halEschedSubmitEventBatch(deviceId_, SHARED_EVENT_ENTRY, &eventInfoSummary, submitNum, &submitSuccessNum);
224 7 : if ((drvRet == DRV_ERROR_NONE) && (submitSuccessNum == submitNum)) {
225 4 : aicpusd_info(
226 : "Batch submit split kernel event success, parallelId=%u, submitNum=%u", taskInfo.parallelId, submitNum);
227 4 : return AICPU_SCHEDULE_OK;
228 : }
229 :
230 : /*
231 : * The queue depth of event schedule is only dozens. If too many split kernel event are submited,
232 : * the queue will be full. Therefore, the main thread needs to process the task sending failure.
233 : */
234 3 : aicpusd_warn(
235 : "Batch submit some of split kernel event success, ret=%d, parallelId=%u, submitNum=%u, "
236 : "submitSuccessNum=%u",
237 : drvRet, taskInfo.parallelId, submitNum, submitSuccessNum);
238 :
239 3 : const uint32_t remainNum = (drvRet == DRV_ERROR_NONE) ? submitNum - submitSuccessNum : submitNum;
240 5 : for (uint32_t i = 0U; i < remainNum; ++i) {
241 3 : if (!DoSplitKernelTask(taskInfo)) {
242 1 : aicpusd_err(
243 : "Run single task failed after batch submit fail, parallelId=%u, submitNum=%u, "
244 : "remainNum=%u, i=%u",
245 : taskInfo.parallelId, submitNum, remainNum, i);
246 1 : return AICPU_SCHEDULE_ERROR_INNER_ERROR;
247 : }
248 : }
249 :
250 2 : return AICPU_SCHEDULE_OK;
251 : }
252 :
253 2 : uint32_t ComputeProcess::SubmitOneSplitKernelEvent(const AICPUSharderTaskInfo& taskInfo) const
254 : {
255 2 : AICPUSubEventInfo aicpuEventInfo = {};
256 2 : aicpuEventInfo.para.sharderTaskInfo = taskInfo;
257 2 : event_summary eventInfoSummary = {};
258 2 : eventInfoSummary.pid = getpid();
259 2 : eventInfoSummary.event_id = EVENT_SPLIT_KERNEL;
260 2 : eventInfoSummary.msg = PtrToPtr<AICPUSubEventInfo, char_t>(&aicpuEventInfo);
261 2 : eventInfoSummary.msg_len = static_cast<uint32_t>(sizeof(AICPUSubEventInfo));
262 :
263 2 : const int32_t drvRet = halEschedSubmitEvent(deviceId_, &eventInfoSummary);
264 2 : if (drvRet != DRV_ERROR_NONE) {
265 1 : aicpusd_err("Submit split kernel event failed. ret=%d, parallelId=%u", drvRet, taskInfo.parallelId);
266 1 : return AICPU_SCHEDULE_ERROR_DRV_ERR;
267 : }
268 :
269 1 : return AICPU_SCHEDULE_OK;
270 : }
271 :
272 3 : bool ComputeProcess::GetAndDoSplitKernelTask()
273 : {
274 3 : event_info eventInfo = {};
275 3 : const uint32_t threadIndex = aicpu::GetAicpuThreadIndex();
276 : const int32_t retVal =
277 3 : halEschedGetEvent(deviceId_, AicpuSchedule::DEFAULT_GROUP_ID, threadIndex, EVENT_SPLIT_KERNEL, &eventInfo);
278 3 : if (retVal == DRV_ERROR_NO_EVENT) {
279 1 : return true;
280 : }
281 2 : if (retVal != DRV_ERROR_NONE) {
282 1 : aicpusd_err("Cannot get event, threadIndex=%u, ret=%d", threadIndex, retVal);
283 1 : return false;
284 : }
285 :
286 1 : const AICPUSubEventInfo* const subEventInfo = PtrToPtr<const char_t, const AICPUSubEventInfo>(eventInfo.priv.msg);
287 1 : aicpusd_info(
288 : "Begin to process split kernel event. parallelId=%u, threadIdx=%u, type=get",
289 : subEventInfo->para.sharderTaskInfo.parallelId, threadIndex);
290 :
291 1 : return DoSplitKernelTask(subEventInfo->para.sharderTaskInfo);
292 : }
293 :
294 5 : bool ComputeProcess::DoSplitKernelTask(const AICPUSharderTaskInfo& taskInfo)
295 : {
296 5 : aicpu::Closure task;
297 5 : if (!splitKernelTask_.PopTask(taskInfo, task)) {
298 1 : aicpusd_run_warn(
299 : "Get split kernel task from map failed, parallelId=%u, %s", taskInfo.parallelId,
300 : splitKernelTask_.DebugString().c_str());
301 1 : return true;
302 : }
303 :
304 : try {
305 4 : task();
306 1 : } catch (std::exception& e) {
307 1 : aicpusd_err("Run split kernel task failed. parallelId=%u, exception=%s", taskInfo.parallelId, e.what());
308 1 : return false;
309 1 : }
310 :
311 3 : return true;
312 5 : }
313 :
314 3 : bool ComputeProcess::DoRandomKernelTask()
315 : {
316 3 : aicpu::Closure task;
317 3 : if (!randomKernelTask_.Dequeue(task)) {
318 1 : aicpusd_err("Get random kernel task from map failed, %s", randomKernelTask_.DebugString().c_str());
319 1 : return false;
320 : }
321 :
322 : try {
323 2 : task();
324 1 : } catch (std::exception& e) {
325 1 : aicpusd_err("Run random kernel task failed. exception=%s", e.what());
326 1 : return false;
327 1 : }
328 :
329 1 : return true;
330 3 : }
331 :
332 5 : void ComputeProcess::Stop()
333 : {
334 5 : splitKernelTask_.Clear();
335 5 : randomKernelTask_.Clear();
336 5 : if (profilingMode_ == PROFILING_OPEN) {
337 5 : aicpu::ReleaseProfiling();
338 : }
339 5 : }
340 : } // namespace AicpuSchedule
|
