Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "api_impl_david.hpp"
12 : #include "memory_task.h"
13 : #include "event_task.h"
14 : #include "ccu_stream.hpp"
15 : #include "runtime_handle_guard.h"
16 : #include "context.hpp"
17 : #include "context_manage.hpp"
18 : #include "stream_c.hpp"
19 : #include "aix_c.hpp"
20 : #include "aicpu_c.hpp"
21 : #include "fusion_c.hpp"
22 : #include "dvpp_c.hpp"
23 : #include "event_c.hpp"
24 : #include "ipc_event.hpp"
25 : #include "memcpy_c.hpp"
26 : #include "memory_c.hpp"
27 : #include "notify_c.hpp"
28 : #include "count_notify.hpp"
29 : #include "event_david.hpp"
30 : #include "model_c.hpp"
31 : #include "cond_c.hpp"
32 : #include "cond_enum_desc.hpp"
33 : #include "label_c.hpp"
34 : #include "label.hpp"
35 : #include "cmo_barrier_c.hpp"
36 : #include "profiler_c.hpp"
37 : #include "coredump_c.hpp"
38 : #include "thread_local_container.hpp"
39 : #include "inner_thread_local.hpp"
40 : #include "device_msg_handler.hpp"
41 : #include "task_david.hpp"
42 : #include "task_recycle.hpp"
43 : #include "fast_recover.hpp"
44 : #include "device/device_error_info.hpp"
45 : #include "capture_model_utils.hpp"
46 : #include "capture_adapt.hpp"
47 : #include "base_david.hpp"
48 : #include "common_task.h"
49 : #include "args_handle_allocator.hpp"
50 : #include "para_convertor.hpp"
51 : #include "runtime/kernel.h"
52 : #include "starsv2_base.hpp"
53 : #include "utils.h"
54 : #include "api_handle_guard.h"
55 : #include "error_message_manage.hpp"
56 : #include "capability.hpp"
57 : #include "notify_enum_desc.hpp"
58 : #include "task.hpp"
59 : #include "context_data_manage.h"
60 : #include "enum_desc.hpp"
61 :
62 : namespace cce {
63 : namespace runtime {
64 :
65 : rtError_t ApiImplDavid::KernelLaunch(
66 : const void* const stubFunc, const uint32_t coreDim, const rtArgsEx_t* const argsInfo, Stream* const stm,
67 : const rtTaskCfgInfo_t* const cfgInfo, const bool isLaunchVec)
68 : {
69 : COND_RETURN_WARN(
70 : isLaunchVec, RT_ERROR_FEATURE_NOT_SUPPORT, "chip type(%d) does not support.",
71 : static_cast<int32_t>(Runtime::Instance()->GetChipType()));
72 : RT_LOG(RT_LOG_DEBUG, "Launch kernel, stubFunc=%p, blockDim=%u.", stubFunc, coreDim);
73 :
74 : Context* const curCtx = CurrentContext();
75 : CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
76 :
77 : Stream* curStm = stm;
78 : if (curStm == nullptr) {
79 : curStm = curCtx->DefaultStream_();
80 : NULL_PTR_RETURN_MSG(curStm, RT_ERROR_STREAM_NULL);
81 : }
82 : COND_RETURN_AND_MSG_INVALID_CONTEXT_STREAM(curStm, curCtx, RT_ERROR_STREAM_CONTEXT);
83 :
84 : if ((cfgInfo != nullptr) && ((cfgInfo->dumpflag & RT_KERNEL_DUMPFLAG) != 0U)) {
85 : ERROR_RETURN_MSG_INNER(
86 : Runtime::Instance()->StartAicpuSd(curCtx->Device_()),
87 : "kernel launch with kernel dump flag failed, check and start tsd open aicpu sd error.");
88 : }
89 :
90 : TaskCfg taskCfg = {};
91 : (void)ConvertTaskCfgInfoToTaskCfg(taskCfg, cfgInfo);
92 : return StreamLaunchKernelV1(stubFunc, coreDim, argsInfo, curStm, &taskCfg, isLaunchVec);
93 : }
94 :
95 : rtError_t ApiImplDavid::KernelLaunchWithHandle(
96 : void* const hdl, const uint64_t tilingKey, const uint32_t coreDim, const rtArgsEx_t* const argsInfo,
97 : Stream* const stm, const rtTaskCfgInfo_t* const cfgInfo, const bool isLaunchVec)
98 : {
99 : COND_RETURN_WARN(
100 : isLaunchVec, RT_ERROR_FEATURE_NOT_SUPPORT, "chip type(%d) does not support.",
101 : static_cast<int32_t>(Runtime::Instance()->GetChipType()));
102 : RT_LOG(RT_LOG_DEBUG, "Launch kernel with hdl, blockDim=%u, tilingKey=%" PRIu64, coreDim, tilingKey);
103 :
104 : Context* const curCtx = CurrentContext();
105 : CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
106 :
107 : Stream* curStm = stm;
108 : if (curStm == nullptr) {
109 : curStm = curCtx->DefaultStream_();
110 : NULL_PTR_RETURN_MSG(curStm, RT_ERROR_STREAM_NULL);
111 : }
112 : COND_RETURN_AND_MSG_INVALID_CONTEXT_STREAM(curStm, curCtx, RT_ERROR_STREAM_CONTEXT);
113 :
114 : TaskCfg taskCfg = {};
115 : (void)ConvertTaskCfgInfoToTaskCfg(taskCfg, cfgInfo);
116 : return StreamLaunchKernelWithHandle(hdl, tilingKey, coreDim, argsInfo, curStm, &taskCfg, isLaunchVec);
117 : }
118 :
119 : rtError_t ApiImplDavid::LaunchKernel(
120 : Kernel* const kernel, uint32_t blockDim, const rtArgsEx_t* const argsInfo, Stream* const stm,
121 : const rtTaskCfgInfo_t* const cfgInfo)
122 : {
123 : RT_LOG(RT_LOG_DEBUG, "Launch kernel, blockDim=%u", blockDim);
124 : Context* const curCtx = CurrentContext();
125 : CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
126 :
127 : Stream* curStm = stm;
128 : if (curStm == nullptr) {
129 : curStm = curCtx->DefaultStream_();
130 : NULL_PTR_RETURN_MSG(curStm, RT_ERROR_STREAM_NULL);
131 : }
132 : COND_RETURN_AND_MSG_INVALID_CONTEXT_STREAM(curStm, curCtx, RT_ERROR_STREAM_CONTEXT);
133 : if (!kernel->Program_()->IsDeviceSoAndNameValid(curCtx->Device_()->Id_())) {
134 : RT_LOG(RT_LOG_WARNING, "kernel is invalid, device_id=%d", curCtx->Device_()->Id_());
135 : return RT_ERROR_KERNEL_INVALID;
136 : }
137 : TaskCfg taskCfg = {};
138 : (void)ConvertTaskCfgInfoToTaskCfg(taskCfg, cfgInfo);
139 :
140 : rtStreamLaunchKernelV2ExtendArgs_t launchKernelExtendArgs = {};
141 : launchKernelExtendArgs.argsInfo = argsInfo;
142 : launchKernelExtendArgs.taskCfg = &taskCfg;
143 : return StreamLaunchKernelV2(kernel, blockDim, curStm, &launchKernelExtendArgs);
144 : }
145 :
146 : rtError_t ApiImplDavid::CpuKernelLaunchExAll(
147 : const Kernel* const kernel, const uint32_t coreDim, rtCpuKernelArgs_t* argsInfo, Stream* const stm,
148 : const TaskCfg* const taskCfg)
149 : {
150 : // 由于tv是可选配置，可以通过有没有timeout可选配置来弱化RT_KERNEL_USE_SPECIAL_TIMEOUT的kernel flag
151 : // 对外不提供kernel flag, 如果taskCfg.extend.timeout为0时，代表永不超时,
152 : // 在等于0时，flag直接退化默认为RT_KERNEL_DEFAULT
153 : uint32_t flag = RT_KERNEL_DEFAULT;
154 : if (taskCfg->isBaseValid == 1U) {
155 : flag |= taskCfg->base.dumpflag;
156 : }
157 :
158 : const uint32_t kernelType = kernel->GetAicpuKernelType_();
159 : if ((kernelType != KERNEL_TYPE_FWK) && (kernelType != KERNEL_TYPE_AICPU) &&
160 : (kernelType != KERNEL_TYPE_AICPU_CUSTOM) && (kernelType != KERNEL_TYPE_AICPU_KFC)) {
161 : RT_LOG(RT_LOG_ERROR, "kernel type mismatch kernelType=%u.", kernelType);
162 : return RT_ERROR_KERNEL_TYPE;
163 : }
164 :
165 : const rtError_t error = StreamLaunchCpuKernelExWithArgs(
166 : coreDim, static_cast<const rtAicpuArgsEx_t*>(&argsInfo->baseArgs), taskCfg, stm, flag, kernelType, kernel,
167 : argsInfo->cpuParamHeadOffset);
168 :
169 : ERROR_RETURN_MSG_INNER(
170 : error, "Cpu kernel launch ex with args failed, check and start tsd open aicpu sd error=%#x.", error);
171 : return RT_ERROR_NONE;
172 : }
173 :
174 : rtError_t ApiImplDavid::LaunchKernelByHandle(
175 : Kernel* const kernel, uint32_t blockDim, const RtArgsHandle* const argHandle, Stream* const curStm,
176 : const TaskCfg& taskCfg)
177 : {
178 : COND_RETURN_ERROR(argHandle == nullptr, RT_ERROR_INVALID_VALUE, "args handle is nullptr");
179 :
180 : rtError_t error = RT_ERROR_NONE;
181 : const KernelRegisterType regType = kernel->GetKernelRegisterType();
182 : const uint8_t phNum = argHandle->placeHolderNum;
183 : // Cpu kernel
184 : if (regType == RT_KERNEL_REG_TYPE_CPU) {
185 : rtCpuKernelArgs_t cpuKernelArgs = {};
186 :
187 : if (phNum <= SPECIAL_ARGS_MAX_CNT) {
188 : rtHostInputInfo_t hostArgsInfos[SPECIAL_ARGS_MAX_CNT] = {};
189 : error = ConvertCpuArgsByArgsHandle(cpuKernelArgs, argHandle, hostArgsInfos, SPECIAL_ARGS_MAX_CNT);
190 : ERROR_RETURN_MSG_INNER(error, "convert args failed, error=%#x", static_cast<uint32_t>(error));
191 : return CpuKernelLaunchExAll(kernel, blockDim, &cpuKernelArgs, curStm, &taskCfg);
192 : }
193 : rtHostInputInfo_t* hostArgsInfos = new (std::nothrow) rtHostInputInfo_t[phNum];
194 : COND_RETURN_AND_MSG_OUTER(
195 : hostArgsInfos == nullptr, RT_ERROR_MEMORY_ALLOCATION, ErrorCode::EE1013,
196 : std::to_string(sizeof(rtHostInputInfo_t) * phNum), "new");
197 : error = ConvertCpuArgsByArgsHandle(cpuKernelArgs, argHandle, hostArgsInfos, phNum);
198 : COND_PROC_RETURN_ERROR(error != RT_ERROR_NONE, error, DELETE_A(hostArgsInfos), "convert args failed.");
199 : error = CpuKernelLaunchExAll(kernel, blockDim, &cpuKernelArgs, curStm, &taskCfg);
200 : DELETE_A(hostArgsInfos);
201 : ERROR_RETURN_MSG_INNER(error, "launch kernel failed, error=%#x", static_cast<uint32_t>(error));
202 :
203 : return RT_ERROR_NONE;
204 : }
205 :
206 : // Non Cpu Kernel
207 : rtArgsEx_t argsInfo = {};
208 : if (phNum <= SPECIAL_ARGS_MAX_CNT) {
209 : rtHostInputInfo_t specialArgsInfos[SPECIAL_ARGS_MAX_CNT];
210 : error = ConvertArgsByArgsHandle(argsInfo, argHandle, specialArgsInfos, SPECIAL_ARGS_MAX_CNT);
211 : ERROR_RETURN_MSG_INNER(error, "convert args failed, error=%#x", error);
212 : rtStreamLaunchKernelV2ExtendArgs_t launchKernelExtendArgs = {};
213 : launchKernelExtendArgs.argsInfo = &argsInfo;
214 : launchKernelExtendArgs.taskCfg = &taskCfg;
215 : return StreamLaunchKernelV2(kernel, blockDim, curStm, &launchKernelExtendArgs);
216 : }
217 :
218 : rtHostInputInfo_t* hostArgsInfos = new (std::nothrow) rtHostInputInfo_t[phNum];
219 : COND_RETURN_AND_MSG_OUTER(
220 : hostArgsInfos == nullptr, RT_ERROR_MEMORY_ALLOCATION, ErrorCode::EE1013,
221 : std::to_string(sizeof(rtHostInputInfo_t) * phNum), "new");
222 : error = ConvertArgsByArgsHandle(argsInfo, argHandle, hostArgsInfos, phNum);
223 : COND_PROC_RETURN_ERROR(error != RT_ERROR_NONE, error, DELETE_A(hostArgsInfos), "convert args failed.");
224 : rtStreamLaunchKernelV2ExtendArgs_t launchKernelExtendArgs = {};
225 : launchKernelExtendArgs.argsInfo = &argsInfo;
226 : launchKernelExtendArgs.taskCfg = &taskCfg;
227 : error = StreamLaunchKernelV2(kernel, blockDim, curStm, &launchKernelExtendArgs);
228 : DELETE_A(hostArgsInfos);
229 : ERROR_RETURN_MSG_INNER(error, "launch kernel failed, error=%#x", error);
230 :
231 : return RT_ERROR_NONE;
232 : }
233 :
234 : rtError_t ApiImplDavid::KernelLaunchEx(
235 : const char_t* const opName, const void* const args, const uint32_t argsSize, const uint32_t flags,
236 : Stream* const stm)
237 : {
238 : RT_LOG(RT_LOG_DEBUG, "Launch kernel ex, opName=%s, argsSize=%u, flags=%u.", opName, argsSize, flags);
239 :
240 : Context* const curCtx = CurrentContext();
241 : CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
242 :
243 : Stream* curStm = stm;
244 : if (curStm == nullptr) {
245 : curStm = curCtx->DefaultStream_();
246 : NULL_PTR_RETURN_MSG(curStm, RT_ERROR_STREAM_NULL);
247 : }
248 : COND_RETURN_AND_MSG_INVALID_CONTEXT_STREAM(curStm, curCtx, RT_ERROR_STREAM_CONTEXT);
249 :
250 : Runtime* const rtInstance = Runtime::Instance();
251 : COND_RETURN_ERROR(rtInstance == nullptr, RT_ERROR_INSTANCE_NULL, "Runtime instance is null.");
252 : ERROR_RETURN_MSG_INNER(
253 : rtInstance->StartAicpuSd(curCtx->Device_()),
254 : "Cpu kernel launch ex with args failed, check and start tsd open aicpu sd error.");
255 :
256 : return StreamLaunchKernelEx(args, argsSize, flags, curStm);
257 : }
258 :
259 : rtError_t ApiImplDavid::CpuKernelLaunch(
260 : const rtKernelLaunchNames_t* const launchNames, const uint32_t coreDim, const rtArgsEx_t* const argsInfo,
261 : Stream* const stm, const uint32_t flag)
262 : {
263 : RT_LOG(
264 : RT_LOG_DEBUG,
265 : "Launch cpu kernel, soName=%s, kernelName=%s, opName=%s, blockDim=%u, argsSize=%u, "
266 : "hostInputInfoNum=%hu, flag=%u.",
267 : launchNames->soName, launchNames->kernelName, launchNames->opName, coreDim, argsInfo->argsSize,
268 : argsInfo->hostInputInfoNum, flag);
269 :
270 : Context* const curCtx = CurrentContext();
271 : CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
272 :
273 : Stream* curStm = stm;
274 : if (curStm == nullptr) {
275 : curStm = curCtx->DefaultStream_();
276 : NULL_PTR_RETURN_MSG(curStm, RT_ERROR_STREAM_NULL);
277 : }
278 : COND_RETURN_AND_MSG_INVALID_CONTEXT_STREAM(curStm, curCtx, RT_ERROR_STREAM_CONTEXT);
279 :
280 : Runtime* const rtInstance = Runtime::Instance();
281 : COND_RETURN_ERROR_MSG_INNER(rtInstance == nullptr, RT_ERROR_INSTANCE_NULL, "Runtime instance is null.");
282 : ERROR_RETURN_MSG_INNER(
283 : rtInstance->StartAicpuSd(curCtx->Device_()),
284 : "Cpu kernel launch failed, check and start tsd open aicpu sd error.");
285 : return StreamLaunchCpuKernel(launchNames, coreDim, argsInfo, curStm, flag);
286 : }
287 :
288 : rtError_t ApiImplDavid::CpuKernelLaunchExWithArgs(
289 : const char_t* const opName, const uint32_t coreDim, const rtAicpuArgsEx_t* const argsInfo, Stream* const stm,
290 : const uint32_t flag, const uint32_t kernelType)
291 : {
292 : RT_LOG(
293 : RT_LOG_DEBUG,
294 : "Launch cpu kernel ex, opName=%s, blockDim=%u, argsSize=%u, hostInputInfoNum=%hu, "
295 : "flag=%u, kernelType=%u, isNoNeedH2DCopy=%u.",
296 : opName, coreDim, argsInfo->argsSize, argsInfo->hostInputInfoNum, flag, kernelType, argsInfo->isNoNeedH2DCopy);
297 :
298 : Context* const curCtx = CurrentContext();
299 : CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
300 :
301 : Stream* curStm = stm;
302 : if (curStm == nullptr) {
303 : curStm = curCtx->DefaultStream_();
304 : NULL_PTR_RETURN_MSG(curStm, RT_ERROR_STREAM_NULL);
305 : }
306 : COND_RETURN_AND_MSG_INVALID_CONTEXT_STREAM(curStm, curCtx, RT_ERROR_STREAM_CONTEXT);
307 :
308 : Runtime* const rtInstance = Runtime::Instance();
309 : COND_RETURN_ERROR_MSG_INNER(rtInstance == nullptr, RT_ERROR_INSTANCE_NULL, "Runtime instance is null.");
310 : ERROR_RETURN_MSG_INNER(
311 : rtInstance->StartAicpuSd(curCtx->Device_()),
312 : "Cpu kernel launch failed, check and start tsd open aicpu sd error.");
313 : return StreamLaunchCpuKernelExWithArgs(coreDim, argsInfo, nullptr, curStm, flag, kernelType, nullptr);
314 : }
315 :
316 : rtError_t ApiImplDavid::FusionLaunch(void* const fusionInfo, Stream* const stm, rtFusionArgsEx_t* argsInfo)
317 : {
318 : Context* const curCtx = CurrentContext();
319 : CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
320 :
321 : Stream* curStm = stm;
322 : if (curStm == nullptr) {
323 : curStm = curCtx->DefaultStream_();
324 : NULL_PTR_RETURN_MSG(curStm, RT_ERROR_STREAM_NULL);
325 : }
326 : COND_RETURN_AND_MSG_INVALID_CONTEXT_STREAM(curStm, curCtx, RT_ERROR_STREAM_CONTEXT);
327 : return LaunchFusionKernel(curStm, fusionInfo, argsInfo);
328 : }
329 :
330 : rtError_t ApiImplDavid::CCULaunch(rtCcuTaskInfo_t* taskInfo, Stream* const stm)
331 : {
332 : Context* const curCtx = CurrentContext();
333 : CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
334 :
335 : Stream* curStm = stm;
336 : if (curStm == nullptr) {
337 : curStm = curCtx->DefaultStream_();
338 : NULL_PTR_RETURN_MSG(curStm, RT_ERROR_STREAM_NULL);
339 : }
340 : COND_RETURN_AND_MSG_INVALID_CONTEXT_STREAM(curStm, curCtx, RT_ERROR_STREAM_CONTEXT);
341 : return StreamCCULaunch(curStm, taskInfo);
342 : }
343 :
344 : rtError_t ApiImplDavid::UbDevQueryInfo(rtUbDevQueryCmd cmd, void* devInfo)
345 : {
346 : Context* const curCtx = CurrentContext();
347 : CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
348 : Device* const dev = curCtx->Device_();
349 : NULL_PTR_RETURN_MSG(dev, RT_ERROR_DEVICE_NULL);
350 :
351 : return NpuDriver::QueryUbInfo(dev->Id_(), cmd, devInfo);
352 : }
353 :
354 : rtError_t ApiImplDavid::GetDevResAddress(const rtDevResInfo* const resInfo, rtDevResAddrInfo* const addrInfo)
355 : {
356 : Context* const curCtx = CurrentContext();
357 : CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
358 : Device* const dev = curCtx->Device_();
359 : NULL_PTR_RETURN_MSG(dev, RT_ERROR_DEVICE_NULL);
360 : uint64_t resAddr = 0U;
361 : uint32_t len = 0U;
362 :
363 : const rtError_t error = NpuDriver::GetDevResAddress(dev->Id_(), resInfo, &resAddr, &len);
364 : if (error == RT_ERROR_NONE) {
365 : *(addrInfo->resAddress) = resAddr;
366 : *(addrInfo->len) = len;
367 : }
368 :
369 : return error;
370 : }
371 :
372 : rtError_t ApiImplDavid::ReleaseDevResAddress(rtDevResInfo* const resInfo)
373 : {
374 : Context* const curCtx = CurrentContext();
375 : CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
376 : Device* const dev = curCtx->Device_();
377 : NULL_PTR_RETURN_MSG(dev, RT_ERROR_DEVICE_NULL);
378 :
379 : return NpuDriver::ReleaseDevResAddress(dev->Id_(), resInfo);
380 : }
381 :
382 : rtError_t ApiImplDavid::CmoTaskLaunch(const rtCmoTaskInfo_t* const taskInfo, Stream* const stm, const uint32_t flag)
383 : {
384 : RT_LOG(RT_LOG_DEBUG, "Cmo task launch, opCode=%hu, lengthInner=%u", taskInfo->opCode, taskInfo->lengthInner);
385 : Context* const curCtx = CurrentContext();
386 : CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
387 :
388 : Stream* curStm = stm;
389 : if (curStm == nullptr) {
390 : curStm = curCtx->DefaultStream_();
391 : NULL_PTR_RETURN_MSG(curStm, RT_ERROR_STREAM_NULL);
392 : }
393 : COND_RETURN_AND_MSG_INVALID_CONTEXT_STREAM(curStm, curCtx, RT_ERROR_STREAM_CONTEXT);
394 : return cce::runtime::CmoTaskLaunch(taskInfo, curStm, flag);
395 : }
396 :
397 : rtError_t ApiImplDavid::CmoAddrTaskLaunch(
398 : void* cmoAddrInfo, const uint64_t destMax, const rtCmoOpCode_t cmoOpCode, Stream* const stm, const uint32_t flag)
399 : {
400 : UNUSED(destMax);
401 : UNUSED(flag);
402 : RT_LOG(RT_LOG_DEBUG, "Cmo addr task launch, opCode=%s.", CmoOpCodeToString(cmoOpCode).c_str());
403 :
404 : Context* const curCtx = CurrentContext();
405 : CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
406 :
407 : Stream* curStm = stm;
408 : if (curStm == nullptr) {
409 : curStm = curCtx->DefaultStream_();
410 : NULL_PTR_RETURN_MSG(curStm, RT_ERROR_STREAM_NULL);
411 : }
412 : COND_RETURN_AND_MSG_INVALID_CONTEXT_STREAM(curStm, curCtx, RT_ERROR_STREAM_CONTEXT);
413 :
414 : return CmoAddrTaskLaunchForDavid(static_cast<rtDavidCmoAddrInfo*>(cmoAddrInfo), cmoOpCode, curStm);
415 : }
416 :
417 : rtError_t ApiImplDavid::EventCreate(Event** const evt, const uint64_t flag)
418 : {
419 : Context* const curCtx = CurrentContext();
420 : CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
421 : Device* const dev = curCtx->Device_();
422 : COND_RETURN_ERROR(dev == nullptr, RT_ERROR_INVALID_VALUE, "device is NULL.");
423 :
424 : *evt = new (std::nothrow) DavidEvent(dev, flag, curCtx);
425 : COND_RETURN_ERROR_MSG_CALL(ERR_MODULE_SYSTEM, *evt == nullptr, RT_ERROR_EVENT_NEW, "new event failed.");
426 : if (flag != RT_EVENT_DEFAULT) {
427 : const rtError_t error = (*evt)->GenEventId();
428 : COND_PROC_RETURN_ERROR(error != RT_ERROR_NONE, error, DELETE_O(*evt);
429 : , "Gen event id failed, device_id=%u, tsId=%u, retCode=%#x", dev->Id_(),
430 : dev->DevGetTsId(), static_cast<uint32_t>(error));
431 : }
432 : InitEmbeddedInnerHandle<Event>(*evt);
433 : dev->PushEvent(*evt);
434 : return RT_ERROR_NONE;
435 : }
436 :
437 : rtError_t ApiImplDavid::EventCreateEx(Event** const evt, const uint64_t flag)
438 : {
439 : Context* const curCtx = CurrentContext();
440 : CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
441 : Device* const dev = curCtx->Device_();
442 : COND_RETURN_ERROR(dev == nullptr, RT_ERROR_INVALID_VALUE, "device is NULL.");
443 :
444 : if (flag == RT_EVENT_IPC) {
445 : *evt = new (std::nothrow) IpcEvent(dev, flag, curCtx);
446 : COND_RETURN_AND_MSG_OUTER(
447 : *evt == nullptr, RT_ERROR_EVENT_NEW, ErrorCode::EE1013, std::to_string(sizeof(IpcEvent)), "new");
448 : const rtError_t error = (*evt)->Setup();
449 : COND_PROC_RETURN_ERROR(error != RT_ERROR_NONE, error, DELETE_O(*evt);, "setup failed, retCode=%#x", error);
450 : } else {
451 : *evt = new (std::nothrow) DavidEvent(dev, flag, curCtx, true);
452 : COND_RETURN_AND_MSG_OUTER(
453 : *evt == nullptr, RT_ERROR_EVENT_NEW, ErrorCode::EE1013, std::to_string(sizeof(DavidEvent)), "new");
454 : }
455 :
456 : InitEmbeddedInnerHandle<Event>(*evt);
457 : dev->PushEvent(*evt);
458 : return RT_ERROR_NONE;
459 : }
460 :
461 : rtError_t ApiImplDavid::EventDestroy(Event* evt)
462 : {
463 : ResetEmbeddedInnerHandle<Event>(evt);
464 : if (evt->GetEventFlag() == RT_EVENT_IPC) {
465 : IpcEvent* eventIpc = dynamic_cast<IpcEvent*>(evt);
466 : IpcEventDestroy(&eventIpc, MAX_INT32_NUM, true);
467 : } else {
468 : RT_LOG(RT_LOG_INFO, "event destroy event_id=%d.", evt->EventId_());
469 : TryToFreeEventIdAndDestroyEvent(&evt, evt->EventId_(), true);
470 : }
471 :
472 : return RT_ERROR_NONE;
473 : }
474 :
475 : rtError_t ApiImplDavid::EventRecord(Event* const evt, Stream* const stm, const uint32_t flag)
476 : {
477 : Context* const curCtx = CurrentContext();
478 : CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
479 : Stream* const curStm = (stm == nullptr) ? curCtx->DefaultStream_() : stm;
480 : NULL_STREAM_PTR_RETURN_MSG(curStm);
481 : const bool supportFlag = (evt->IsNewMode() || (evt->GetEventFlag() == RT_EVENT_DEFAULT)) && curStm->IsModelStream();
482 : COND_RETURN_WARN(
483 : supportFlag, RT_ERROR_FEATURE_NOT_SUPPORT,
484 : "Event record is not supported on a model-bound stream when the event was created in Ex mode "
485 : "(for example, via aclrtCreateEventExWithFlag or rtEventCreateExWithFlag) or "
486 : "eventFlag is RT_EVENT_DEFAULT, isNewMode=%d, eventFlag=%#" PRIx64 ", isModelStream=%d.",
487 : evt->IsNewMode(), evt->GetEventFlag(), curStm->IsModelStream());
488 : COND_RETURN_AND_MSG_INVALID_CONTEXT_STREAM(curStm, curCtx, RT_ERROR_STREAM_CONTEXT);
489 : if (flag == RT_EVENT_RECORD_EXTERNAL) {
490 : COND_RETURN_AND_MSG_OUTER(
491 : (!curStm->IsCapturing()), RT_ERROR_STREAM_NOT_CAPTURED, ErrorCode::EE1016, "Event recording",
492 : RtFmtMsg("Stream %d is not in the capture stage", curStm->Id_()));
493 : const rtError_t supportRet = CheckCaptureModelSupportExternalEvent(curStm->Device_(), true);
494 : if (supportRet != RT_ERROR_NONE) {
495 : return supportRet;
496 : }
497 : return Starsv2CaptureExternalEventRecord(evt, curStm);
498 : }
499 : if (evt->ToBeCaptured(curStm)) {
500 : COND_RETURN_WARN(
501 : !evt->IsNewMode(), RT_ERROR_FEATURE_NOT_SUPPORT,
502 : "Not support call rtEventCreate or rtEventCreateWithFlag without external flag, mode=%d", evt->IsNewMode());
503 : COND_RETURN_ERROR_MSG_INNER(
504 : !StreamFlagIsSupportCapture(curStm->Flags()), RT_ERROR_STREAM_INVALID,
505 : "stream flag does not support capture to model, flag=%u.", curStm->Flags());
506 : COND_RETURN_ERROR_MSG_INNER(
507 : curStm == curCtx->DefaultStream_(), RT_ERROR_STREAM_CAPTURE_IMPLICIT,
508 : "A disallowed implicit dependency from default stream.");
509 : COND_RETURN_WARN(
510 : evt->IsEventWithoutWaitTask(), RT_ERROR_NONE,
511 : "The event flag %" PRIu64 " is not supported in capture mode.", evt->GetEventFlag());
512 : const std::lock_guard<std::mutex> lk(curCtx->GetCaptureLock());
513 : if (evt->ToBeCaptured(curStm)) {
514 : const rtError_t retCode = CaptureRecordEvent(curCtx, evt, curStm);
515 : ERROR_PROC_RETURN_MSG_INNER(retCode, TerminateCapture(evt, curStm), "Capture event record failed.");
516 : return RT_ERROR_NONE;
517 : }
518 : }
519 : if (evt->GetEventFlag() == RT_EVENT_IPC) {
520 : return (dynamic_cast<IpcEvent*>(evt))->IpcEventRecordStarsV2(curStm);
521 : } else if (!evt->IsHardwareMode()) {
522 : return EvtRecordSoftwareMode(evt, curStm);
523 : } else {
524 : return EvtRecord(evt, curStm);
525 : }
526 : }
527 :
528 : rtError_t ApiImplDavid::EventReset(Event* const evt, Stream* const stm)
529 : {
530 : Context* const curCtx = CurrentContext();
531 : CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
532 : Stream* const curStm = (stm == nullptr) ? curCtx->DefaultStream_() : stm;
533 : NULL_STREAM_PTR_RETURN_MSG(curStm);
534 : const bool supportFlag =
535 : (evt->IsNewMode()) || ((evt->GetEventFlag() == RT_EVENT_DEFAULT) && curStm->IsModelStream());
536 : COND_RETURN_WARN(
537 : supportFlag, RT_ERROR_FEATURE_NOT_SUPPORT,
538 : "Not support current mode bind stm, mode=%d, flag=%" PRIu64 ", isModel=%d.", evt->IsNewMode(),
539 : evt->GetEventFlag(), curStm->IsModelStream());
540 : // David硬件默认event reset沿用历史no-op；software event reset需要下发写0任务，不能提前返回。
541 : if ((evt->GetEventFlag() == RT_EVENT_DEFAULT) && evt->IsHardwareMode() && !curStm->IsModelStream()) {
542 : return RT_ERROR_NONE;
543 : }
544 : COND_RETURN_AND_MSG_INVALID_CONTEXT_STREAM(curStm, curCtx, RT_ERROR_STREAM_CONTEXT);
545 : if (evt->IsCapturing()) {
546 : COND_RETURN_ERROR_MSG_INNER(
547 : !StreamFlagIsSupportCapture(curStm->Flags()), RT_ERROR_STREAM_INVALID,
548 : "stream flag does not support capture to model, flag=%u.", curStm->Flags());
549 : COND_RETURN_ERROR_MSG_INNER(
550 : curStm == curCtx->DefaultStream_(), RT_ERROR_STREAM_CAPTURE_IMPLICIT,
551 : "A disallowed implicit dependency from default stream.");
552 : COND_RETURN_ERROR_MSG_INNER(
553 : evt->IsEventWithoutWaitTask(), RT_ERROR_INVALID_VALUE,
554 : "The event flag %" PRIu64 " is not supported in capture mode.", evt->GetEventFlag());
555 : const std::lock_guard<std::mutex> lk(curCtx->GetCaptureLock());
556 : if (evt->IsCapturing()) {
557 : const rtError_t retCode = CaptureResetEvent(evt, curStm);
558 : ERROR_PROC_RETURN_MSG_INNER(retCode, TerminateCapture(evt, curStm), "Capture event record failed.");
559 : return RT_ERROR_NONE;
560 : }
561 : } else {
562 : // hardware event reset没有software写0任务可捕获，仍沿用capture限制；software event reset可转成mem-write任务。
563 : if ((curStm != curCtx->DefaultStream_()) && evt->IsHardwareMode() && (evt->ToBeCaptured(curStm))) {
564 : RT_LOG(RT_LOG_WARNING, "Not support call rtEventCreate or rtEventCreateWithFlag without external flag");
565 : return RT_ERROR_FEATURE_NOT_SUPPORT;
566 : }
567 : }
568 : return evt->IsHardwareMode() ? EvtReset(evt, curStm) : EvtResetSoftwareMode(evt, curStm);
569 : }
570 :
571 : rtError_t ApiImplDavid::LaunchKernelByArgsWithType(
572 : Kernel* const kernel, const uint32_t coreDim, Stream* stm, const RtArgsWithType* const argsWithType,
573 : TaskCfg& taskCfg)
574 : {
575 : rtError_t error = RT_ERROR_NONE;
576 : RT_LOG(
577 : RT_LOG_DEBUG, "LaunchKernelByArgsWithType, device_id=%u, add stream_id=%d, blockDim=%u, argsType=%u.",
578 : stm->Device_()->Id_(), stm->Id_(), coreDim, static_cast<uint32_t>(argsWithType->type));
579 : switch (argsWithType->type) {
580 : case RT_ARGS_NON_CPU_EX: {
581 : rtStreamLaunchKernelV2ExtendArgs_t launchKernelExtendArgs = {};
582 : launchKernelExtendArgs.argsInfo = argsWithType->args.nonCpuArgsInfo;
583 : launchKernelExtendArgs.taskCfg = &taskCfg;
584 : error = StreamLaunchKernelV2(kernel, coreDim, stm, &launchKernelExtendArgs);
585 : break;
586 : }
587 : case RT_ARGS_CPU_EX: {
588 : error = CpuKernelLaunchExAll(kernel, coreDim, argsWithType->args.cpuArgsInfo, stm, &taskCfg);
589 : break;
590 : }
591 : case RT_ARGS_HANDLE: {
592 : error = LaunchKernelByHandle(kernel, coreDim, argsWithType->args.argHandle, stm, taskCfg);
593 : break;
594 : }
595 : case RT_ARGS_ARRAY: {
596 : error = StreamLaunchArgsArray(kernel, coreDim, stm, argsWithType->args.argsArrayInfo, taskCfg);
597 : break;
598 : }
599 : case RT_SIMT_ARGS_ARRAY: {
600 : error = StreamLaunchSimtArgsArray(kernel, coreDim, stm, argsWithType->args.simtArgsArray, taskCfg);
601 : break;
602 : }
603 : case RT_SIMT_ARGS_HOST: {
604 : error = StreamLaunchSimtArgsHost(kernel, coreDim, stm, argsWithType->args.simtArgsHost, taskCfg);
605 : break;
606 : }
607 : default:
608 : error = RT_ERROR_INVALID_VALUE;
609 : RT_LOG_OUTER_MSG_INVALID_PARAM(
610 : argsWithType->type,
611 : "[" + std::to_string(RT_ARGS_NON_CPU_EX) + ", " + std::to_string(RT_ARGS_MAX) + ")");
612 : break;
613 : }
614 :
615 : return error;
616 : }
617 :
618 : rtError_t ApiImplDavid::StreamWaitEvent(
619 : Stream* const stm, Event* const evt, const uint32_t timeout, const uint32_t flag)
620 : {
621 : RT_LOG(RT_LOG_DEBUG, "Stream wait event, timeout=%us.", timeout);
622 : Context* const curCtx = CurrentContext();
623 : CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
624 : Stream* const curStm = (stm == nullptr) ? curCtx->DefaultStream_() : stm;
625 : NULL_STREAM_PTR_RETURN_MSG(curStm);
626 : const bool supFlag = ((evt->IsNewMode()) || (evt->GetEventFlag() == RT_EVENT_DEFAULT)) && curStm->IsModelStream();
627 : COND_RETURN_WARN(
628 : supFlag, RT_ERROR_FEATURE_NOT_SUPPORT,
629 : "Not support current mode bind stream, mode=%d, flag=%" PRIu64 ", isModel=%d.", evt->IsNewMode(),
630 : evt->GetEventFlag(), curStm->IsModelStream());
631 : COND_RETURN_AND_MSG_INVALID_CONTEXT_STREAM(curStm, curCtx, RT_ERROR_STREAM_CONTEXT);
632 : if (flag == RT_EVENT_WAIT_EXTERNAL) {
633 : COND_RETURN_AND_MSG_OUTER(
634 : (!curStm->IsCapturing()), RT_ERROR_STREAM_NOT_CAPTURED, ErrorCode::EE1016, "Triggering event waiting",
635 : RtFmtMsg("Stream %d is not in the capture stage", curStm->Id_()));
636 : const rtError_t supportRet = CheckCaptureModelSupportExternalEvent(curStm->Device_(), false);
637 : if (supportRet != RT_ERROR_NONE) {
638 : return supportRet;
639 : }
640 : return Starsv2CaptureExternalEventWait(evt, curStm);
641 : }
642 : if (evt->IsCapturing()) {
643 : COND_RETURN_ERROR_MSG_INNER(
644 : !StreamFlagIsSupportCapture(curStm->Flags()), RT_ERROR_STREAM_INVALID,
645 : "stream flag does not support capture to model, flag=%u, stream_id=%d.", curStm->Flags(), curStm->Id_());
646 : COND_RETURN_ERROR_MSG_INNER(
647 : curStm == curCtx->DefaultStream_(), RT_ERROR_STREAM_CAPTURE_IMPLICIT,
648 : "A disallowed implicit dependency from default stream.");
649 : COND_RETURN_ERROR_MSG_INNER(
650 : evt->IsEventWithoutWaitTask(), RT_ERROR_INVALID_VALUE,
651 : "The event flag %" PRIu64 " is not supported in capture mode.", evt->GetEventFlag());
652 : const std::lock_guard<std::mutex> lk(curCtx->GetCaptureLock());
653 : if (evt->IsCapturing()) {
654 : const rtError_t retCode = CaptureWaitEvent(curCtx, curStm, evt, timeout);
655 : ERROR_PROC_RETURN_MSG_INNER(
656 : retCode, TerminateCapture(evt, curStm), "Capture wait event failed, stream_id=%d.", curStm->Id_());
657 : return RT_ERROR_NONE;
658 : }
659 : } else {
660 : if (curStm->IsCapturing()) {
661 : if ((!(evt->IsNewMode())) && (evt->GetEventFlag() != RT_EVENT_EXTERNAL)) {
662 : RT_LOG(
663 : RT_LOG_WARNING,
664 : "Event created via the API rtEventCreate and rtEventCreateWithFlag are not"
665 : " supported, except for the RT_EVENT_EXTERNAL type, mode=%d, flag=%" PRIu64 "",
666 : evt->IsNewMode(), evt->GetEventFlag());
667 : return RT_ERROR_FEATURE_NOT_SUPPORT;
668 : }
669 : if ((evt->IsNewMode()) && (evt->HasRecord())) {
670 : // 1.Not capture event
671 : // 2.Be a capture stream
672 : // 3.Event was created using the rtCreateEventExWithFlag interface
673 : // 4. A record was added the single-operator stream
674 : RT_LOG_OUTER_MSG_IMPL(
675 : ErrorCode::EE1016, "Triggering event waiting",
676 : "The event wait task is not supported during model capture because the corresponding event record "
677 : "task is not "
678 : "in the current model. To wait for an event outside the model, see the usage of "
679 : "ACL_EVENT_WAIT_EXTERNAL of "
680 : "the aclrtStreamWaitEventWithFlag API in the API reference");
681 : return RT_ERROR_STREAM_CAPTURE_ISOLATION;
682 : }
683 : }
684 : }
685 :
686 : rtError_t error = RT_ERROR_NONE;
687 : if (evt->GetEventFlag() == RT_EVENT_IPC) {
688 : error = (dynamic_cast<IpcEvent*>(evt))->IpcEventWaitStarsV2(curStm);
689 : } else if ((!evt->IsHardwareMode()) && evt->HasRecord() && (evt->GetEventAddr() != nullptr)) {
690 : error = EvtWaitSoftwareMode(evt, curStm);
691 : } else {
692 : error = EvtWait(evt, curStm, timeout);
693 : }
694 : ERROR_RETURN(error, "Stream wait event failed.");
695 : return error;
696 : }
697 :
698 : rtError_t ApiImplDavid::SetMemcpyDesc(
699 : rtMemcpyDesc_t desc, const void* const srcAddr, const void* const dstAddr, const size_t count,
700 : const rtMemcpyKind kind, rtMemcpyConfig_t* const config)
701 : {
702 : RT_LOG(
703 : RT_LOG_INFO, "SetMemcpyDesc called, desc=%p, srcAddr=%p, dstAddr=%p, count=%zu, kind=%s, config=%p", desc,
704 : srcAddr, dstAddr, count, MemcpyNewKindToString(kind).c_str(), config);
705 : UNUSED(kind);
706 : UNUSED(config);
707 :
708 : Context* const curCtx = CurrentContext();
709 : CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
710 : const Device* const dev = curCtx->Device_();
711 :
712 : rtDavidMemcpyAddrInfo memcpyData;
713 : (void)memset_s(&memcpyData, sizeof(rtDavidMemcpyAddrInfo), 0, sizeof(rtDavidMemcpyAddrInfo));
714 :
715 : memcpyData.len = static_cast<uint32_t>(count);
716 : memcpyData.src = RtPtrToValue<const void*>(srcAddr);
717 : memcpyData.dst = RtPtrToValue<const void*>(dstAddr);
718 :
719 : constexpr uint64_t dstMax = MEMCPY_DESC_SIZE_V2; // 64U
720 : rtError_t error = RT_ERROR_NONE;
721 :
722 : error = dev->Driver_()->MemCopySync(
723 : static_cast<rtDavidMemcpyAddrInfo*>(desc), dstMax, &memcpyData, dstMax, RT_MEMCPY_HOST_TO_DEVICE);
724 : ERROR_RETURN(
725 : error, "Failed to memory copy stream info, device_id=%u, dstMax=%u, retCode=%#x.", dev->Id_(), dstMax,
726 : static_cast<uint32_t>(error));
727 :
728 : if (dev->Driver_()->GetRunMode() == RT_RUN_MODE_ONLINE) {
729 : error = dev->Driver_()->DevMemFlushCache(RtPtrToPtr<uintptr_t>(desc), static_cast<size_t>(dstMax));
730 : ERROR_RETURN(
731 : error, "Failed to flush stream info, device_id=%u, retCode=%#x", dev->Id_(), static_cast<uint32_t>(error));
732 : }
733 :
734 : RT_LOG(RT_LOG_INFO, "Set memcpyDesc info success, srcAddr=%p, dstAddr=%p, count=%llu", srcAddr, dstAddr, count);
735 : return RT_ERROR_NONE;
736 : }
737 :
738 : rtError_t ApiImplDavid::MemCopy2DAsync(
739 : void* const dst, const uint64_t dstPitch, const void* const src, const uint64_t srcPitch, const uint64_t width,
740 : const uint64_t height, Stream* const stm, const rtMemcpyKind_t kind, const rtMemcpyKind newKind)
741 : {
742 : UNUSED(newKind);
743 : RT_LOG(
744 : RT_LOG_DEBUG,
745 : "Sync memcpy2d, dstPitch=%" PRIu64 ", srcPitch=%" PRIu64 ", width=%" PRIu64 ", height=%" PRIu64 ", kind=%s.",
746 : dstPitch, srcPitch, width, height, MemcpyKindToStr(kind));
747 :
748 : rtError_t error = RT_ERROR_NONE;
749 : uint64_t remainSize = width * height;
750 : const uint64_t totalSize = remainSize;
751 : uint64_t realSize = 0UL;
752 : uint64_t fixedSize = 0UL;
753 : uint64_t srcoffset = 0UL;
754 : uint64_t dstoffset = 0UL;
755 : Context* const curCtx = CurrentContext();
756 : CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
757 :
758 : Stream* curStm = stm;
759 : if (curStm == nullptr) {
760 : curStm = curCtx->DefaultStream_();
761 : NULL_PTR_RETURN_MSG(curStm, RT_ERROR_STREAM_NULL);
762 : }
763 : COND_RETURN_AND_MSG_INVALID_CONTEXT_STREAM(curStm, curCtx, RT_ERROR_STREAM_CONTEXT);
764 :
765 : while (remainSize > 0UL) {
766 : if (kind == RT_MEMCPY_DEVICE_TO_DEVICE) {
767 : error = Memcpy2DAsync(
768 : (static_cast<char_t*>(dst)) + dstoffset, dstPitch, (static_cast<const char_t*>(src)) + srcoffset,
769 : srcPitch, width, height, kind, &realSize, curStm, fixedSize);
770 : dstoffset += dstPitch;
771 : srcoffset += srcPitch;
772 : } else {
773 : error = Memcpy2DAsync(dst, dstPitch, src, srcPitch, width, height, kind, &realSize, curStm, fixedSize);
774 : }
775 : COND_RETURN_WITH_NOLOG((error != RT_ERROR_NONE), error);
776 : if (Runtime::Instance()->GetConnectUbFlag() && (kind != RT_MEMCPY_DEVICE_TO_DEVICE)) {
777 : fixedSize = realSize;
778 : remainSize = totalSize - fixedSize;
779 : if (remainSize > 0UL && !(curStm->IsCapturing())) {
780 : error = curStm->Synchronize();
781 : ERROR_RETURN_MSG_INNER(
782 : error, "Failed to synchronize stream, retCode=%#x.", static_cast<uint32_t>(error));
783 : }
784 : } else {
785 : fixedSize += realSize;
786 : remainSize -= realSize;
787 : }
788 : }
789 : return error;
790 : }
791 :
792 : rtError_t ApiImplDavid::BatchMemcpyAsync(
793 : void** const dsts, const size_t* const destMaxs, void** const srcs, const size_t* const sizes, const size_t count,
794 : const rtMemcpyBatchAttr* const attrs, const size_t* const attrsIdxs, const size_t numAttrs, size_t* const failIdx,
795 : Stream* const stm)
796 : {
797 : Context* const curCtx = CurrentContext();
798 : CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
799 : Stream* curStm = stm;
800 : if (curStm == nullptr) {
801 : curStm = curCtx->DefaultStream_();
802 : NULL_PTR_RETURN_MSG(curStm, RT_ERROR_STREAM_NULL);
803 : }
804 : COND_RETURN_ERROR_MSG_INNER(
805 : curStm->Context_() != curCtx, RT_ERROR_STREAM_CONTEXT,
806 : "MemcopyBatch async failed, stream is not in current ctx, stream_id=%d.", curStm->Id_());
807 :
808 : rtError_t error = RT_ERROR_NONE;
809 : rtMemcpyBatchAttr memAttr = attrs[0];
810 : size_t attrIdx = 0U;
811 : rtPtrAttributes_t dstAttr = {};
812 : rtPtrAttributes_t srcAttr = {};
813 : uint64_t realCnt = 0UL;
814 : uint64_t realSize = 0UL;
815 : uint64_t remainCnt = count;
816 : uint64_t fixedCnt = 0UL;
817 : uint64_t fixedSize = 0UL;
818 : bool isD2HorH2DInvolvePageableMemory = false;
819 :
820 : for (size_t i = 0U; i < count; i++) {
821 : if (((attrIdx + 1U) < numAttrs) && (i >= attrsIdxs[attrIdx + 1U])) {
822 : attrIdx = attrIdx + 1U;
823 : memAttr = attrs[attrIdx];
824 : }
825 : error = ValidateMemCpyParamsAndAttributes(dsts[i], destMaxs[i], srcs[i], sizes[i], memAttr, dstAttr, srcAttr);
826 : COND_PROC_RETURN_ERROR(
827 : error != RT_ERROR_NONE, error, SetFailIndex(failIdx, i), "ValidateMemCpyParamsAndAttributes %u failed.", i);
828 :
829 : if (dstAttr.location.type == RT_MEMORY_LOC_UNREGISTERED ||
830 : srcAttr.location.type == RT_MEMORY_LOC_UNREGISTERED) {
831 : isD2HorH2DInvolvePageableMemory = true;
832 : }
833 : }
834 :
835 : if (isD2HorH2DInvolvePageableMemory) {
836 : COND_RETURN_AND_MSG_OUTER(
837 : curStm->IsCapturing(), RT_ERROR_INVALID_VALUE, ErrorCode::EE1016, "Asynchronous batch copy task",
838 : "The pageable memory copy task does not support graph capture");
839 : error = StreamSynchronize(curStm, -1);
840 : ERROR_RETURN(error, "StreamSynchronize failed, stream_id=%d.", curStm->Id_());
841 : RT_LOG(RT_LOG_DEBUG, "Stream Synchronize success, stream_id=%d.", curStm->Id_());
842 : return MemcpyBatch(
843 : dsts, srcs, const_cast<size_t*>(sizes), count, const_cast<rtMemcpyBatchAttr*>(attrs),
844 : const_cast<size_t*>(attrsIdxs), numAttrs, failIdx);
845 : }
846 : std::vector<void*> localDsts(dsts, dsts + count);
847 : std::vector<void*> localSrcs(srcs, srcs + count);
848 : std::vector<uint64_t> localSizes(sizes, sizes + count);
849 : while (remainCnt > 0UL) {
850 : AsyncDmaBatchInfo batchInfo = {localDsts.data(), localSrcs.data(), localSizes.data(),
851 : remainCnt, fixedCnt, fixedSize};
852 : error = MemcopyBatchAsync(batchInfo, &realCnt, &realSize, curStm);
853 : COND_RETURN_WITH_NOLOG((error != RT_ERROR_NONE), error);
854 : // realCnt 本次处理的
855 : fixedCnt = realCnt;
856 : fixedSize = realSize;
857 : remainCnt -= fixedCnt;
858 : if (remainCnt > 0UL && !(curStm->IsCapturing())) {
859 : error = curStm->Synchronize();
860 : ERROR_RETURN_MSG_INNER(error, "Failed to synchronize stream, retCode=%#x.", static_cast<uint32_t>(error));
861 : }
862 : }
863 :
864 : return error;
865 : }
866 :
867 : rtError_t ApiImplDavid::MemcpyBatchAsync(
868 : void** const dsts, const size_t* const destMaxs, void** const srcs, const size_t* const sizes, const size_t count,
869 : const rtMemcpyBatchAttr* const attrs, const size_t* const attrsIdxs, const size_t numAttrs, size_t* const failIdx,
870 : Stream* const stm)
871 : {
872 : Context* curCtx = Runtime::Instance()->CurrentContext();
873 : CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
874 : NULL_PTR_RETURN_MSG(curCtx->Device_(), RT_ERROR_DEVICE_NULL);
875 :
876 : Stream* curStm = stm;
877 : if (curStm == nullptr) {
878 : curStm = curCtx->DefaultStream_();
879 : NULL_PTR_RETURN_MSG(curStm, RT_ERROR_STREAM_NULL);
880 : }
881 :
882 : if (!NpuDriver::CheckIsSupportFeature(curCtx->Device_()->Id_(), FEATURE_MEMCPY_BATCH_ASYNC)) {
883 : // ub 单算子
884 : if (Runtime::Instance()->GetConnectUbFlag()) {
885 : return BatchMemcpyAsync(dsts, destMaxs, srcs, sizes, count, attrs, attrsIdxs, numAttrs, failIdx, curStm);
886 : } else {
887 : return LoopMemcpyAsync(dsts, destMaxs, srcs, sizes, count, attrs, attrsIdxs, numAttrs, failIdx, stm);
888 : }
889 : }
890 : return RT_ERROR_DRV_NOT_SUPPORT;
891 : }
892 :
893 : rtError_t ApiImplDavid::MemcpyAsync(
894 : void* const dst, const uint64_t destMax, const void* const src, const uint64_t cnt, const rtMemcpyKind_t kind,
895 : Stream* const stm, const rtTaskCfgInfo_t* const cfgInfo, const rtD2DAddrCfgInfo_t* const addrCfg, bool checkKind,
896 : const rtMemcpyConfig_t* const memcpyConfig)
897 : {
898 : UNUSED(checkKind);
899 : UNUSED(memcpyConfig);
900 : RT_LOG(RT_LOG_DEBUG, "Async memcpy, count=%" PRIu64 ", kind=%s", cnt, MemcpyKindToStr(kind));
901 :
902 : Context* const curCtx = CurrentContext();
903 : CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
904 :
905 : Stream* curStm = stm;
906 : if (curStm == nullptr) {
907 : curStm = curCtx->DefaultStream_();
908 : NULL_PTR_RETURN_MSG(curStm, RT_ERROR_STREAM_NULL);
909 : }
910 : COND_RETURN_AND_MSG_INVALID_CONTEXT_STREAM(curStm, curCtx, RT_ERROR_STREAM_CONTEXT);
911 :
912 : uint32_t transType = UINT32_MAX;
913 : rtError_t error = RT_ERROR_NONE;
914 : if (kind == RT_MEMCPY_DEVICE_TO_DEVICE) {
915 : error = ConvertD2DCpyType(curStm, transType, src, dst);
916 : if (error != RT_ERROR_NONE) {
917 : RT_LOG(RT_LOG_ERROR, "ConvertD2DCpyType failed, retCode=%#x.", static_cast<uint32_t>(error));
918 : return error;
919 : }
920 : }
921 : const uint64_t sqSize = CalculateMemcpyAsyncSingleMaxSize(kind, transType);
922 :
923 : uint64_t realSize = cnt;
924 : uint64_t remainSize = cnt;
925 : uint64_t doneSize = 0U;
926 : while (remainSize > 0U) {
927 : const uint64_t doingSize = (remainSize >= sqSize) ? sqSize : remainSize;
928 : realSize = doingSize;
929 : error = MemcopyAsync(
930 : (static_cast<char_t*>(dst)) + doneSize, destMax - doneSize, (static_cast<const char_t*>(src)) + doneSize,
931 : doingSize, kind, curStm, &realSize, nullptr, cfgInfo, addrCfg);
932 : if (error != RT_ERROR_NONE) {
933 : RT_LOG(RT_LOG_ERROR, "cnt=%lld, doingSize=%lld, realSize=%lld.", cnt, doingSize, realSize);
934 : return error;
935 : }
936 : doneSize += realSize;
937 : remainSize -= realSize;
938 : }
939 : return error;
940 : }
941 :
942 : rtError_t ApiImplDavid::ReduceAsync(
943 : void* const dst, const void* const src, const uint64_t cnt, const rtRecudeKind_t kind, const rtDataType_t type,
944 : Stream* const stm, const rtTaskCfgInfo_t* const cfgInfo)
945 : {
946 : RT_LOG(RT_LOG_INFO, "ReduceAsync, count=%" PRIu64 ", kind=%s.", cnt, ReduceKindToString(kind).c_str());
947 : Context* const curCtx = CurrentContext();
948 : CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
949 :
950 : Stream* curStm = stm;
951 : if (curStm == nullptr) {
952 : curStm = curCtx->DefaultStream_();
953 : NULL_PTR_RETURN_MSG(curStm, RT_ERROR_STREAM_NULL);
954 : }
955 : COND_RETURN_AND_MSG_INVALID_CONTEXT_STREAM(curStm, curCtx, RT_ERROR_STREAM_CONTEXT);
956 : return cce::runtime::ReduceAsync(dst, src, cnt, kind, type, curStm, cfgInfo);
957 : }
958 :
959 : rtError_t ApiImplDavid::ModelExit(Model* const mdl, Stream* const stm)
960 : {
961 : Context* const curCtx = CurrentContext();
962 : CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
963 :
964 : COND_RETURN_AND_MSG_INVALID_CONTEXT_STREAM(stm, curCtx, RT_ERROR_STREAM_CONTEXT);
965 : COND_RETURN_AND_MSG_INVALID_CONTEXT_MODEL(mdl, curCtx, RT_ERROR_MODEL_CONTEXT);
966 : const uint32_t modelExitNum = mdl->ModelExitNum_();
967 : COND_RETURN_AND_MSG_OUTER(
968 : modelExitNum >= 1U, RT_ERROR_MODEL_EXIT, ErrorCode::EE1011, "Model exiting", modelExitNum, "modelExitNum",
969 : RtFmtMsg("Model (model_id=%u) must exit only once", mdl->Id_()));
970 : COND_RETURN_AND_MSG_OUTER(
971 : stm->Model_() == nullptr, RT_ERROR_MODEL_EXIT_STREAM_UNBIND, ErrorCode::EE1017, "Model exiting", "stm",
972 : RtFmtMsg("Stream (stream_id=%d) is not bound to any model", stm->Id_()));
973 : COND_RETURN_AND_MSG_OUTER(
974 : stm->Model_()->Id_() != mdl->Id_(), RT_ERROR_MODEL_EXIT_ID, ErrorCode::EE1017, "Model exiting", "stm",
975 : RtFmtMsg(
976 : "The current stream (stream_id=%d) has been bound to another model (model_id=%u) which is different from "
977 : "the input model (model_id=%u). "
978 : "The input model must be the same as the model bound to the input stream",
979 : stm->Id_(), stm->Model_()->Id_(), mdl->Id_()));
980 : mdl->IncModelExitNum();
981 : return RT_ERROR_NONE;
982 : }
983 :
984 : rtError_t ApiImplDavid::MemsetAsync(
985 : void* const ptr, const uint64_t destMax, const uint32_t val, const uint64_t cnt, Stream* const stm)
986 : {
987 : Context* const curCtx = CurrentContext();
988 : CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
989 :
990 : Stream* curStm = stm;
991 : if (curStm == nullptr) {
992 : curStm = curCtx->DefaultStream_();
993 : NULL_PTR_RETURN_MSG(curStm, RT_ERROR_STREAM_NULL);
994 : }
995 : COND_RETURN_AND_MSG_INVALID_CONTEXT_STREAM(curStm, curCtx, RT_ERROR_STREAM_CONTEXT);
996 :
997 : RT_LOG(
998 : RT_LOG_DEBUG, "fillVal=%u, fillCount=%" PRIu64 ", destMax=%" PRIu64 ", stream_id=%d.", val, cnt, destMax,
999 : curStm->Id_());
1000 : return MemSetAsync(curStm, ptr, destMax, val, cnt);
1001 : }
1002 :
1003 : rtError_t ApiImplDavid::CntNotifyCreate(const int32_t deviceId, CountNotify** const retCntNotify, const uint32_t flag)
1004 : {
1005 : Context* const curCtx = CurrentContext();
1006 : CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
1007 : Device* const dev = curCtx->Device_();
1008 : COND_RETURN_ERROR(dev == nullptr, RT_ERROR_INVALID_VALUE, "device is NULL.");
1009 :
1010 : const uint32_t mc2FeatureFlag = dev->GetDevProperties().mc2FeatureFlag;
1011 : COND_RETURN_AND_MSG_OUTER_WITH_PARAM_NAME(
1012 : ((mc2FeatureFlag == 0U) && (flag == RT_NOTIFY_FLAG_DOWNLOAD_TO_DEV)), RT_ERROR_INVALID_VALUE,
1013 : NotifyFlagToString(flag), "flag", "RT_NOTIFY_FLAG_DEFAULT(0)");
1014 :
1015 : *retCntNotify = new (std::nothrow) CountNotify(static_cast<uint32_t>(deviceId), dev->DevGetTsId());
1016 : COND_RETURN_AND_MSG_OUTER(
1017 : *retCntNotify == nullptr, RT_ERROR_NOTIFY_NEW, ErrorCode::EE1013, std::to_string(sizeof(CountNotify)), "new");
1018 :
1019 : (*retCntNotify)->SetNotifyFlag(flag);
1020 : rtError_t error = (*retCntNotify)->Setup();
1021 : ERROR_PROC_RETURN_MSG_INNER(error, DELETE_O(*retCntNotify);
1022 : , "Count Notify create failed, setup failed, user device_id=%d, retCode=%#x", deviceId,
1023 : static_cast<uint32_t>(error));
1024 : return RT_ERROR_NONE;
1025 : }
1026 :
1027 : rtError_t ApiImplDavid::CntNotifyDestroy(CountNotify* const inCntNotify)
1028 : {
1029 : delete inCntNotify;
1030 : return RT_ERROR_NONE;
1031 : }
1032 :
1033 : rtError_t ApiImplDavid::CntNotifyRecord(
1034 : CountNotify* const inCntNotify, Stream* const stm, const rtCntNtyRecordInfo_t* const info)
1035 : {
1036 : Context* const curCtx = CurrentContext();
1037 : CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
1038 : Device* const dev = curCtx->Device_();
1039 : COND_RETURN_ERROR(dev == nullptr, RT_ERROR_INVALID_VALUE, "device is NULL.");
1040 :
1041 : Stream* targetStm = (stm == nullptr) ? curCtx->DefaultStream_() : stm;
1042 : NULL_STREAM_PTR_RETURN_MSG(targetStm);
1043 : COND_RETURN_AND_MSG_INVALID_CONTEXT_STREAM(targetStm, curCtx, RT_ERROR_STREAM_CONTEXT);
1044 :
1045 : const uint32_t countNotifyId = inCntNotify->GetCntNotifyId();
1046 : const rtError_t error = inCntNotify->Record(targetStm, info);
1047 : ERROR_RETURN_MSG_INNER(
1048 : error,
1049 : "Count Notify record failed, device_id=%u, stream_id=%d, count notify_id=%u,"
1050 : " retCode=%#x",
1051 : dev->Id_(), targetStm->Id_(), countNotifyId, static_cast<uint32_t>(error));
1052 :
1053 : return RT_ERROR_NONE;
1054 : }
1055 :
1056 : rtError_t ApiImplDavid::CntNotifyReset(CountNotify* const inCntNotify, Stream* const stm)
1057 : {
1058 : Context* const curCtx = CurrentContext();
1059 : CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
1060 : Device* const dev = curCtx->Device_();
1061 : COND_RETURN_ERROR(dev == nullptr, RT_ERROR_INVALID_VALUE, "device is NULL.");
1062 :
1063 : Stream* targetStm = (stm == nullptr) ? curCtx->DefaultStream_() : stm;
1064 : NULL_STREAM_PTR_RETURN_MSG(targetStm);
1065 : COND_RETURN_AND_MSG_INVALID_CONTEXT_STREAM(targetStm, curCtx, RT_ERROR_STREAM_CONTEXT);
1066 :
1067 : rtCntNtyRecordInfo_t info = {RECORD_STORE_MODE, 0U};
1068 : const uint32_t countNotifyId = inCntNotify->GetCntNotifyId();
1069 : const rtError_t error = inCntNotify->Record(targetStm, &info);
1070 : ERROR_RETURN_MSG_INNER(
1071 : error,
1072 : "Count Notify record failed, device_id=%u, stream_id=%d, count notify_id=%u,"
1073 : " retCode=%#x",
1074 : dev->Id_(), targetStm->Id_(), countNotifyId, static_cast<uint32_t>(error));
1075 : return RT_ERROR_NONE;
1076 : }
1077 :
1078 : rtError_t ApiImplDavid::CntNotifyWaitWithTimeout(
1079 : CountNotify* const inCntNotify, Stream* const stm, const rtCntNtyWaitInfo_t* const info)
1080 : {
1081 : Context* const curCtx = CurrentContext();
1082 : CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
1083 : Device* const dev = curCtx->Device_();
1084 : COND_RETURN_ERROR(dev == nullptr, RT_ERROR_INVALID_VALUE, "device is NULL.");
1085 :
1086 : Stream* targetStm = (stm == nullptr) ? curCtx->DefaultStream_() : stm;
1087 : NULL_STREAM_PTR_RETURN_MSG(targetStm);
1088 : COND_RETURN_AND_MSG_INVALID_CONTEXT_STREAM(targetStm, curCtx, RT_ERROR_STREAM_CONTEXT);
1089 :
1090 : const rtError_t error = inCntNotify->Wait(targetStm, info);
1091 : const uint32_t notifyId = inCntNotify->GetCntNotifyId();
1092 : ERROR_RETURN_MSG_INNER(
1093 : error,
1094 : "count notify wait failed, device_id=%u, stream_id=%d, count notify_id=%u,"
1095 : " time_out = %u, retCode=%#x",
1096 : dev->Id_(), targetStm->Id_(), notifyId, info->timeout, static_cast<uint32_t>(error));
1097 : return RT_ERROR_NONE;
1098 : }
1099 :
1100 : rtError_t ApiImplDavid::GetCntNotifyId(CountNotify* const inCntNotify, uint32_t* const notifyId)
1101 : {
1102 : *notifyId = inCntNotify->GetCntNotifyId();
1103 : return RT_ERROR_NONE;
1104 : }
1105 :
1106 : rtError_t ApiImplDavid::GetCntNotifyAddress(
1107 : CountNotify* const inCntNotify, uint64_t* const cntNotifyAddress, rtNotifyType_t const regType)
1108 : {
1109 : uint64_t addr;
1110 : Context* const curCtx = CurrentContext();
1111 : CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
1112 : Device* const dev = curCtx->Device_();
1113 : COND_RETURN_ERROR(dev == nullptr, RT_ERROR_INVALID_VALUE, "device is NULL.");
1114 : if (!dev->IsSupportFeature(RtOptionalFeatureType::RT_FEATURE_DEVICE_AICPUSD_LATER_PROCEDURE)) {
1115 : // Driver now only support NOTIFY_CNT_ST_SLICE for count notify
1116 : COND_RETURN_AND_MSG_OUTER_WITH_PARAM_DESC(
1117 : (regType != NOTIFY_CNT_ST_SLICE), RT_ERROR_INVALID_VALUE,
1118 : "Obtaining the on-device address of a CntNotify object", regType, std::to_string(NOTIFY_CNT_ST_SLICE));
1119 : } else {
1120 : COND_RETURN_ERROR(
1121 : regType == NOTIFY_TABLE_SLICE, RT_ERROR_INVALID_VALUE,
1122 : "CntNotify does not support getting notify table address.");
1123 : }
1124 : const rtError_t error = inCntNotify->GetCntNotifyAddress(addr, regType);
1125 : ERROR_RETURN_MSG_INNER(
1126 : error, "GetCntNotifyAddress failed, device_id=%d, retCode=%#x", dev->Id_(), static_cast<uint32_t>(error));
1127 : RT_LOG(RT_LOG_INFO, "GetCntNotifyAddress ok, device_id=%d, addr=%#" PRIx64, dev->Id_(), addr);
1128 : *cntNotifyAddress = addr;
1129 : return RT_ERROR_NONE;
1130 : }
1131 :
1132 : rtError_t ApiImplDavid::NotifyWait(Notify* const inNotify, Stream* const stm, const uint32_t timeOut)
1133 : {
1134 : Context* const curCtx = CurrentContext();
1135 : CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
1136 :
1137 : Stream* curStm = stm;
1138 : if (curStm == nullptr) {
1139 : curStm = curCtx->DefaultStream_();
1140 : NULL_PTR_RETURN_MSG(curStm, RT_ERROR_STREAM_NULL);
1141 : }
1142 :
1143 : COND_RETURN_AND_MSG_INVALID_CONTEXT_STREAM(curStm, curCtx, RT_ERROR_STREAM_CONTEXT);
1144 : COND_RETURN_AND_MSG_OUTER(
1145 : inNotify->CheckIpcNotifyDevId() != RT_ERROR_NONE, RT_ERROR_INVALID_VALUE, ErrorCode::EE1012,
1146 : "Waiting for a Notify", curCtx->Device_()->Id_(), "current deviceId",
1147 : RtFmtMsg(
1148 : "The device (device_id=%u) cannot deliver the notify wait task."
1149 : " The notify wait task must be delivered on the device (device_id=%u) where the IPC Notify is created",
1150 : curCtx->Device_()->Id_(), inNotify->GetDeviceId()));
1151 : const uint32_t timeOutTmp = timeOut;
1152 : const rtError_t error = NtyWait(inNotify, curStm, timeOutTmp);
1153 : const uint32_t notifyId = inNotify->GetNotifyId();
1154 : ERROR_RETURN_MSG_INNER(
1155 : error, "notify wait failed, notify_id=%u, time_out = %u, retCode=%#x", notifyId, timeOutTmp,
1156 : static_cast<uint32_t>(error));
1157 : return RT_ERROR_NONE;
1158 : }
1159 :
1160 : rtError_t ApiImplDavid::NotifyRecord(Notify* const inNotify, Stream* const stm)
1161 : {
1162 : Context* const curCtx = CurrentContext();
1163 : CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
1164 :
1165 : Stream* curStm = stm;
1166 : if (curStm == nullptr) {
1167 : curStm = curCtx->DefaultStream_();
1168 : NULL_PTR_RETURN_MSG(curStm, RT_ERROR_STREAM_NULL);
1169 : }
1170 : COND_RETURN_AND_MSG_INVALID_CONTEXT_STREAM(curStm, curCtx, RT_ERROR_STREAM_CONTEXT);
1171 :
1172 : const uint32_t notifyId = inNotify->GetNotifyId();
1173 : const rtError_t error = NtyRecord(inNotify, curStm);
1174 : ERROR_RETURN_MSG_INNER(
1175 : error, "Notify record failed, notify_id=%u, retCode=%#x", notifyId, static_cast<uint32_t>(error));
1176 : return RT_ERROR_NONE;
1177 : }
1178 :
1179 : rtError_t ApiImplDavid::NotifyReset(Notify* const inNotify)
1180 : {
1181 : Context* const curCtx = CurrentContext();
1182 : CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
1183 : Stream* curStm = curCtx->GetCtrlSQStream();
1184 : NULL_STREAM_PTR_RETURN_MSG(curStm);
1185 :
1186 : const uint32_t notifyId = inNotify->GetNotifyId();
1187 : const rtError_t error = NtyReset(inNotify, curStm);
1188 : ERROR_RETURN_MSG_INNER(
1189 : error, "Notify reset failed, device_id=%u, notify_id=%u, is_ipc_notify=%d, retCode=%#x",
1190 : curStm->Device_()->Id_(), notifyId, inNotify->IsIpcNotify(), static_cast<uint32_t>(error));
1191 : return RT_ERROR_NONE;
1192 : }
1193 :
1194 : rtError_t ApiImplDavid::DatadumpInfoLoad(const void* const dumpInfo, const uint32_t length, const uint32_t flag)
1195 : {
1196 : UNUSED(flag);
1197 : RT_LOG(RT_LOG_DEBUG, "length=%u, flag=%u.", length, flag);
1198 : Context* const curCtx = CurrentContext();
1199 : CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
1200 :
1201 : Runtime* const rtInstance = Runtime::Instance();
1202 : COND_RETURN_ERROR_MSG_INNER(rtInstance == nullptr, RT_ERROR_INSTANCE_NULL, "Runtime instance is null.");
1203 : ERROR_RETURN_MSG_INNER(
1204 : rtInstance->StartAicpuSd(curCtx->Device_()),
1205 : "Data dump info load failed, check and start tsd open aicpu sd error.");
1206 : return StreamDatadumpInfoLoad(dumpInfo, length, curCtx->DefaultStream_());
1207 : }
1208 :
1209 : rtError_t ApiImplDavid::DebugRegister(
1210 : Model* const mdl, const uint32_t flag, const void* const addr, uint32_t* const streamId, uint32_t* const taskId)
1211 : {
1212 : Context* const curCtx = CurrentContext();
1213 : CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
1214 :
1215 : COND_RETURN_AND_MSG_INVALID_CONTEXT_MODEL(mdl, curCtx, RT_ERROR_MODEL_CONTEXT);
1216 : return ModelDebugRegister(mdl, flag, addr, streamId, taskId, curCtx->DefaultStream_());
1217 : }
1218 :
1219 : rtError_t ApiImplDavid::DebugUnRegister(Model* const mdl)
1220 : {
1221 : Context* const curCtx = CurrentContext();
1222 : CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
1223 :
1224 : COND_RETURN_AND_MSG_INVALID_CONTEXT_MODEL(mdl, curCtx, RT_ERROR_MODEL_CONTEXT);
1225 : return ModelDebugUnRegister(mdl, curCtx->DefaultStream_());
1226 : }
1227 :
1228 : rtError_t ApiImplDavid::DebugRegisterForStream(
1229 : Stream* const stm, const uint32_t flag, const void* const addr, uint32_t* const streamId, uint32_t* const taskId)
1230 : {
1231 : Context* const curCtx = CurrentContext();
1232 : CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
1233 :
1234 : COND_RETURN_AND_MSG_INVALID_CONTEXT_STREAM(stm, curCtx, RT_ERROR_STREAM_CONTEXT);
1235 : return StreamDebugRegister(stm, flag, addr, streamId, taskId);
1236 : }
1237 :
1238 : rtError_t ApiImplDavid::DebugUnRegisterForStream(Stream* const stm)
1239 : {
1240 : Context* const curCtx = CurrentContext();
1241 : CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
1242 :
1243 : COND_RETURN_AND_MSG_INVALID_CONTEXT_STREAM(stm, curCtx, RT_ERROR_STREAM_CONTEXT);
1244 : return StreamDebugUnRegister(stm);
1245 : }
1246 :
1247 : rtError_t ApiImplDavid::GetDevRunningStreamSnapshotMsg(const rtGetMsgCallback callback)
1248 : {
1249 : const std::function<rtError_t(Device* const dev)> getDevHangMsgForDev = [callback](Device* const dev) -> rtError_t {
1250 : DeviceStreamSnapshotHandler devStreamSnapshotHandler(dev, callback);
1251 : rtError_t error = devStreamSnapshotHandler.Init();
1252 : ERROR_RETURN(
1253 : error, "Init device stream snapshot msg handler failed, retCode=%#x.", static_cast<uint32_t>(error));
1254 :
1255 : error = SyncGetDeviceMsg(
1256 : dev, devStreamSnapshotHandler.GetDevMemAddr(), devStreamSnapshotHandler.GetDevMemSize(),
1257 : RT_GET_DEV_RUNNING_STREAM_SNAPSHOT_MSG);
1258 :
1259 : ERROR_RETURN(error, "Sync get device msg failed, retCode=%#x.", static_cast<uint32_t>(error));
1260 :
1261 : error = devStreamSnapshotHandler.HandleMsg();
1262 : ERROR_RETURN_MSG_INNER(
1263 : error, "Failed to handle get stream snapshot msg, retCode=%#x.", static_cast<uint32_t>(error));
1264 : return RT_ERROR_NONE;
1265 : };
1266 : return Runtime::Instance()->ProcessForAllOpenDevice(getDevHangMsgForDev, false);
1267 : }
1268 :
1269 : rtError_t ApiImplDavid::NpuClearFloatStatus(const uint32_t checkMode, Stream* const stm)
1270 : {
1271 : NULL_STREAM_PTR_RETURN_MSG(stm);
1272 : Context* const curCtx = CurrentContext();
1273 : CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
1274 : COND_RETURN_AND_MSG_INVALID_CONTEXT_STREAM(stm, curCtx, RT_ERROR_STREAM_CONTEXT);
1275 : return StreamNpuClearFloatStatus(checkMode, stm, false);
1276 : }
1277 :
1278 : rtError_t ApiImplDavid::NpuGetFloatStatus(
1279 : void* const outputAddrPtr, const uint64_t outputSize, const uint32_t checkMode, Stream* const stm)
1280 : {
1281 : Context* const curCtx = CurrentContext();
1282 : CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
1283 : Stream* const targetStm = (stm == nullptr) ? curCtx->DefaultStream_() : stm;
1284 : NULL_STREAM_PTR_RETURN_MSG(targetStm);
1285 : COND_RETURN_AND_MSG_INVALID_CONTEXT_STREAM(targetStm, curCtx, RT_ERROR_STREAM_CONTEXT);
1286 : return StreamNpuGetFloatStatus(outputAddrPtr, outputSize, checkMode, targetStm, false);
1287 : }
1288 :
1289 : rtError_t ApiImplDavid::NpuClearFloatDebugStatus(const uint32_t checkMode, Stream* const stm)
1290 : {
1291 : NULL_STREAM_PTR_RETURN_MSG(stm);
1292 : Context* const curCtx = CurrentContext();
1293 : CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
1294 : COND_RETURN_AND_MSG_INVALID_CONTEXT_STREAM(stm, curCtx, RT_ERROR_STREAM_CONTEXT);
1295 : return StreamNpuClearFloatStatus(checkMode, stm, true);
1296 : }
1297 :
1298 : rtError_t ApiImplDavid::NpuGetFloatDebugStatus(
1299 : void* const outputAddrPtr, const uint64_t outputSize, const uint32_t checkMode, Stream* const stm)
1300 : {
1301 : NULL_STREAM_PTR_RETURN_MSG(stm);
1302 : Context* const curCtx = CurrentContext();
1303 : CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
1304 : COND_RETURN_AND_MSG_INVALID_CONTEXT_STREAM(stm, curCtx, RT_ERROR_STREAM_CONTEXT);
1305 : return StreamNpuGetFloatStatus(outputAddrPtr, outputSize, checkMode, stm, true);
1306 : }
1307 :
1308 : rtError_t ApiImplDavid::GetDeviceSatStatus(void* const outputAddrPtr, const uint64_t outputSize, Stream* const stm)
1309 : {
1310 : RT_LOG(RT_LOG_DEBUG, "Start to get sat status.");
1311 : uint64_t realSize = 0U;
1312 : rtError_t error = RT_ERROR_NONE;
1313 : Context* const curCtx = CurrentContext();
1314 : CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
1315 :
1316 : Stream* curStm = stm;
1317 : if (curStm == nullptr) {
1318 : curStm = curCtx->DefaultStream_();
1319 : NULL_PTR_RETURN_MSG(curStm, RT_ERROR_STREAM_NULL);
1320 : }
1321 : COND_RETURN_AND_MSG_INVALID_CONTEXT_STREAM(curStm, curCtx, RT_ERROR_STREAM_CONTEXT);
1322 :
1323 : error = StreamGetSatStatus(outputSize, curStm);
1324 : COND_RETURN_WITH_NOLOG((error != RT_ERROR_NONE), error);
1325 :
1326 : error = MemcopyAsync(
1327 : outputAddrPtr, outputSize, curCtx->CtxGetOverflowAddr(), outputSize, RT_MEMCPY_DEVICE_TO_DEVICE, curStm,
1328 : &realSize, nullptr, nullptr);
1329 : if (error != RT_ERROR_NONE) {
1330 : RT_LOG(RT_LOG_ERROR, "MemcpyAsync failed destMax=%llu.", outputSize);
1331 : }
1332 :
1333 : return error;
1334 : }
1335 :
1336 : rtError_t ApiImplDavid::SetStreamOverflowSwitch(Stream* const stm, const uint32_t flags)
1337 : {
1338 : Context* const curCtx = CurrentContext();
1339 : CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
1340 :
1341 : Stream* const targetStm = (stm == nullptr) ? curCtx->DefaultStream_() : stm;
1342 : NULL_STREAM_PTR_RETURN_MSG(targetStm);
1343 : COND_RETURN_AND_MSG_INVALID_CONTEXT_STREAM(targetStm, curCtx, RT_ERROR_STREAM_CONTEXT);
1344 : return SetOverflowSwitchOnStream(targetStm, flags);
1345 : }
1346 :
1347 : rtError_t ApiImplDavid::SetStreamTag(Stream* const stm, const uint32_t geOpTag)
1348 : {
1349 : RT_LOG(RT_LOG_DEBUG, "geOpTag=%#x.", geOpTag);
1350 : Context* const curCtx = CurrentContext();
1351 : CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
1352 : Stream* const targetStm = (stm == nullptr) ? curCtx->DefaultStream_() : stm;
1353 : NULL_STREAM_PTR_RETURN_MSG(targetStm);
1354 : COND_RETURN_AND_MSG_INVALID_CONTEXT_STREAM(targetStm, curCtx, RT_ERROR_STREAM_CONTEXT);
1355 : return SetTagOnStream(targetStm, geOpTag);
1356 : }
1357 :
1358 : rtError_t ApiImplDavid::UbDbSend(rtUbDbInfo_t* const dbInfo, Stream* const stm)
1359 : {
1360 : Context* const curCtx = CurrentContext();
1361 : CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
1362 : Stream* curStm = stm;
1363 : if (curStm == nullptr) {
1364 : curStm = curCtx->DefaultStream_();
1365 : NULL_STREAM_PTR_RETURN_MSG(curStm);
1366 : }
1367 :
1368 : COND_RETURN_AND_MSG_INVALID_CONTEXT_STREAM(curStm, curCtx, RT_ERROR_STREAM_CONTEXT);
1369 : return StreamUbDbSend(dbInfo, curStm, static_cast<uint16_t>(UbDmaSqeSource::RT_UBDMA_SOURCE_API));
1370 : }
1371 :
1372 : rtError_t ApiImplDavid::UbDirectSend(rtUbWqeInfo_t* const wqeInfo, Stream* const stm)
1373 : {
1374 : Context* const curCtx = CurrentContext();
1375 : CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
1376 : Stream* curStm = stm;
1377 : if (curStm == nullptr) {
1378 : curStm = curCtx->DefaultStream_();
1379 : NULL_STREAM_PTR_RETURN_MSG(curStm);
1380 : }
1381 :
1382 : COND_RETURN_AND_MSG_INVALID_CONTEXT_STREAM(curStm, curCtx, RT_ERROR_STREAM_CONTEXT);
1383 : COND_RETURN_ERROR_MSG_INNER(
1384 : curStm->GetBindFlag(), RT_ERROR_STREAM_INVALID, "UbDirectSend not support model stream, stream_id=%d.",
1385 : curStm->Id_());
1386 : COND_RETURN_WARN(
1387 : curStm->IsCapturing(), RT_ERROR_FEATURE_NOT_SUPPORT, "Ub direct tasks cannot be delivered in capture mode.");
1388 :
1389 : return StreamUbDirectSend(wqeInfo, curStm);
1390 : }
1391 :
1392 : rtError_t ApiImplDavid::StreamClear(Stream* const stm, rtClearStep_t step)
1393 : {
1394 : UNUSED(stm);
1395 : UNUSED(step);
1396 : RT_LOG(RT_LOG_WARNING, "Chip type(%d) does not support.", Runtime::Instance()->GetChipType());
1397 : return RT_ERROR_FEATURE_NOT_SUPPORT;
1398 : }
1399 :
1400 : rtError_t ApiImplDavid::NopTask(Stream* const stm)
1401 : {
1402 : Context* const curCtx = CurrentContext();
1403 : CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
1404 : NULL_STREAM_PTR_RETURN_MSG(stm);
1405 : COND_RETURN_AND_MSG_INVALID_CONTEXT_STREAM(stm, curCtx, RT_ERROR_STREAM_CONTEXT);
1406 :
1407 : return StreamNopTask(stm);
1408 : }
1409 :
1410 : rtError_t ApiImplDavid::AicpuInfoLoad(const void* const aicpuInfo, const uint32_t length)
1411 : {
1412 : RT_LOG(RT_LOG_DEBUG, "length=%u.", length);
1413 : Context* const curCtx = CurrentContext();
1414 : CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
1415 :
1416 : Runtime* const rtInstance = Runtime::Instance();
1417 : COND_RETURN_ERROR_MSG_INNER(rtInstance == nullptr, RT_ERROR_INSTANCE_NULL, "Runtime instance is null.");
1418 : ERROR_RETURN_MSG_INNER(
1419 : rtInstance->StartAicpuSd(curCtx->Device_()),
1420 : "aicpu info load failed, check and start tsd open aicpu sd error.");
1421 : return StreamAicpuInfoLoad(curCtx->DefaultStream_(), aicpuInfo, length);
1422 : }
1423 :
1424 : rtError_t ApiImplDavid::SubscribeReport(const uint64_t threadId, Stream* const stm)
1425 : {
1426 : rtError_t ret = RT_ERROR_NONE;
1427 : Stream* curStm = stm;
1428 : if (curStm == nullptr) {
1429 : Context* const curCtx = CurrentContext();
1430 : CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
1431 : curStm = curCtx->DefaultStream_();
1432 : NULL_STREAM_PTR_RETURN_MSG(curStm);
1433 : }
1434 : Notify* curNotify = nullptr;
1435 : ret = NotifyCreate(static_cast<int32_t>(curStm->Device_()->Id_()), &curNotify, RT_NOTIFY_DEFAULT);
1436 : ERROR_RETURN(ret, "Call NotifyCreate failed, ret=%#x.", ret);
1437 : ret = Runtime::Instance()->SubscribeReport(threadId, curStm, static_cast<void*>(curNotify));
1438 : if (ret != RT_ERROR_NONE) {
1439 : (void)NotifyDestroy(curNotify);
1440 : }
1441 : return ret;
1442 : }
1443 :
1444 : static rtError_t ProcessReportForBlockCqe(
1445 : Device* const dev, rtHostFuncCqReport_t* const report, const uint32_t deviceId, uint32_t tsId)
1446 : {
1447 : if (report->isBlock != 0) {
1448 : Runtime* const rt = Runtime::Instance();
1449 : Driver* const curDrv = dev->Driver_();
1450 : Notify* curNotify = nullptr;
1451 : rtError_t ret = rt->GetNotifyByStreamId(deviceId, static_cast<int32_t>(report->streamId), &curNotify);
1452 : ERROR_RETURN_MSG_INNER(ret, "Call GetNotifyByStreamId failed for block callback, ret=%#x.", ret);
1453 : ret = curDrv->WriteNotifyRecord(deviceId, tsId, curNotify->GetNotifyId());
1454 : ERROR_RETURN_MSG_INNER(ret, "Call WriteNotifyRecord failed for block callback, ret=%#x.", ret);
1455 : }
1456 : return RT_ERROR_NONE;
1457 : }
1458 :
1459 : rtError_t ApiImplDavid::ProcessReport(const int32_t timeout, const bool noLog)
1460 : {
1461 : uint64_t cqidBit[HOST_CALLBACK_SQCQ_BIT / 64U] = {0ULL};
1462 : uint32_t deviceId = 0U;
1463 : uint32_t tsId = 0U;
1464 : uint32_t groupId = 0U;
1465 : uint64_t bit = 0U;
1466 :
1467 : Runtime* const rt = Runtime::Instance();
1468 :
1469 : const uint64_t threadId = PidTidFetcher::GetCurrentUserTid();
1470 : rtError_t ret = rt->GetGroupIdByThreadId(threadId, &deviceId, &tsId, &groupId, noLog);
1471 : COND_RETURN_WARN_WITH_NOLOG_SWITCH(
1472 : ret != RT_ERROR_NONE, noLog, ret, "get groupId fail, threadIdentifier=%" PRIu64 ", retCode=%#x", threadId, ret);
1473 : rt->LockGroupId(groupId);
1474 : std::function<void()> const func = [rt, groupId]() { rt->UnlockGroupId(groupId); };
1475 : const ScopeGuard groupIdGuarder(func);
1476 :
1477 : Context* priCtx = rt->GetPriCtxByDeviceId(deviceId, tsId);
1478 : if (priCtx == nullptr) {
1479 : priCtx = CurrentContext();
1480 : }
1481 :
1482 : if (noLog) {
1483 : CHECK_CONTEXT_VALID_WITH_PROC_RETURN(priCtx, RT_ERROR_CONTEXT_NULL, );
1484 : } else {
1485 : CHECK_CONTEXT_VALID_WITH_RETURN(priCtx, RT_ERROR_CONTEXT_NULL);
1486 : }
1487 :
1488 : Device* const dev = priCtx->Device_();
1489 : Driver* const curDrv = dev->Driver_();
1490 :
1491 : ret = curDrv->CqReportIrqWait(deviceId, tsId, groupId, timeout, &cqidBit[0], HOST_CALLBACK_SQCQ_BIT / 64U);
1492 : COND_RETURN_WARN_WITH_NOLOG_SWITCH(ret != RT_ERROR_NONE, noLog, ret, "CqReportIrqWait, retCode=%#x", ret);
1493 : RT_LOG(RT_LOG_DEBUG, "IrqWait groupId=%u, threadIdentifier=%" PRIu64, groupId, threadId);
1494 :
1495 : // per uint64_t num has 64 bit
1496 : for (uint32_t index = 0U; index < (HOST_CALLBACK_SQCQ_BIT / 64U); index++) {
1497 : for (;; RT_BITMAP_CLR(cqidBit[index], bit)) {
1498 : bit = BitScan(cqidBit[index]);
1499 : if (bit >= 64U) { // 64 bit for uint64_t
1500 : break;
1501 : }
1502 : // left move 6 is multiply 64
1503 : const uint32_t cqidValue = (index << 6U) + static_cast<uint32_t>(bit);
1504 : uint32_t cnt = 0U;
1505 : rtHostFuncCqReport_t* report = nullptr;
1506 : ret = curDrv->CqReportGet(deviceId, tsId, cqidValue, &report, &cnt);
1507 : if (unlikely((report == nullptr) || (cnt == 0U))) {
1508 : continue;
1509 : }
1510 : RT_LOG(RT_LOG_DEBUG, "get report info num=%u from cqid = %u.", cnt, cqidValue);
1511 :
1512 : COND_RETURN_WARN_WITH_NOLOG_SWITCH(
1513 : ret != RT_ERROR_NONE, noLog, ret, "CqReportGet failed, retCode=%#x", ret);
1514 : for (uint32_t idx = 0U; idx < cnt; idx++) {
1515 : const rtCallback_t hostFunc = RtValueToPtr<rtCallback_t>(report[idx].hostFuncCbPtr);
1516 : NULL_PTR_RETURN_MSG(hostFunc, RT_ERROR_DRV_REPORT);
1517 :
1518 : RT_LOG(
1519 : RT_LOG_INFO, "report[%u], streamId=%hu, taskId=%hu, eventNotifyId=%hu, isBlock=%hhu", idx,
1520 : report[idx].streamId, report[idx].taskId, report[idx].eventId, report[idx].isBlock);
1521 :
1522 : ProcessHostFunc(report[idx].hostFuncCbPtr, report[idx].fnDataPtr, dev, report[idx].streamId);
1523 : ret = ProcessReportForBlockCqe(dev, &report[idx], deviceId, tsId);
1524 : ERROR_RETURN(ret, "process block cqe fail, ret=%#x.", ret);
1525 : ret = curDrv->CqReportRelease(&report[idx], deviceId, cqidValue, tsId, noLog);
1526 : }
1527 : }
1528 : }
1529 :
1530 : return ret;
1531 : }
1532 :
1533 : rtError_t ApiImplDavid::ModelTaskUpdate(
1534 : Stream* desStm, uint32_t desTaskId, Stream* sinkStm, rtMdlTaskUpdateInfo_t* para)
1535 : {
1536 : RT_LOG(
1537 : RT_LOG_INFO, "ModelTaskUpdate, desStm=%d, desTaskId=%u, sinkStm=%d", desStm->Id_(), desTaskId, sinkStm->Id_());
1538 : Context* const curCtx = CurrentContext();
1539 : CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
1540 :
1541 : return MdlTaskUpdate(desStm, desTaskId, sinkStm, para);
1542 : }
1543 :
1544 : rtError_t ApiImplDavid::CallbackLaunch(
1545 : const rtCallback_t callBackFunc, void* const fnData, Stream* const stm, const bool isBlock)
1546 : {
1547 : Context* const curCtx = CurrentContext();
1548 : CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
1549 :
1550 : Stream* curStm = stm;
1551 : if (curStm == nullptr) {
1552 : curStm = curCtx->DefaultStream_();
1553 : NULL_STREAM_PTR_RETURN_MSG(curStm);
1554 : }
1555 : COND_RETURN_AND_MSG_INVALID_CONTEXT_STREAM(curStm, curCtx, RT_ERROR_STREAM_CONTEXT);
1556 : COND_RETURN_ERROR_MSG_INNER(
1557 : !curStm->IsHostFuncCbReg(), RT_ERROR_STREAM_NO_CB_REG,
1558 : "The stream used by this user's callback function is not registered to any thread, retCode=%#x",
1559 : static_cast<uint32_t>(RT_ERROR_STREAM_NO_CB_REG));
1560 : if (isBlock) {
1561 : const rtError_t ret = CallbackLaunchForDavidWithBlock(callBackFunc, fnData, curStm, MAX_UINT64_NUM);
1562 : ERROR_RETURN(ret, "Call CallbackLaunch failed for block callback, ret=%#x.", ret);
1563 : return ret;
1564 : }
1565 : return CallbackLaunchForDavidNoBlock(callBackFunc, fnData, curStm, MAX_UINT64_NUM);
1566 : }
1567 :
1568 : rtError_t ApiImplDavid::ModelAbort(Model* const mdl)
1569 : {
1570 : NULL_PTR_RETURN_MSG_OUTER_WITH_FUNC_DESC(mdl, RT_ERROR_MODEL_NULL, "Aborting the model running instance");
1571 : Context* const curCtx = CurrentContext();
1572 : CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
1573 : COND_RETURN_AND_MSG_INVALID_CONTEXT_MODEL(mdl, curCtx, RT_ERROR_MODEL_CONTEXT);
1574 : return MdlAbort(mdl);
1575 : }
1576 :
1577 : rtError_t ApiImplDavid::ModelEndGraph(Model* const mdl, Stream* const stm, const uint32_t flags)
1578 : {
1579 : RT_LOG(RT_LOG_DEBUG, "model add end graph task model_id=%u, stream_id=%d, flags=%u", mdl->Id_(), stm->Id_(), flags);
1580 : Context* const curCtx = CurrentContext();
1581 : CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
1582 :
1583 : COND_RETURN_AND_MSG_INVALID_CONTEXT_STREAM(stm, curCtx, RT_ERROR_STREAM_CONTEXT);
1584 : COND_RETURN_AND_MSG_INVALID_CONTEXT_MODEL(mdl, curCtx, RT_ERROR_MODEL_CONTEXT);
1585 :
1586 : if ((flags & RT_KERNEL_DUMPFLAG) != 0U) {
1587 : ERROR_RETURN_MSG_INNER(
1588 : Runtime::Instance()->StartAicpuSd(curCtx->Device_()),
1589 : "Model end graph with kernel dump flag failed, check and start tsd open aicpu sd error.");
1590 : }
1591 : return MdlAddEndGraph(mdl, stm, flags);
1592 : }
1593 :
1594 : rtError_t ApiImplDavid::StreamSwitchEx(
1595 : void* const ptr, const rtCondition_t condition, void* const valuePtr, Stream* const trueStream, Stream* const stm,
1596 : const rtSwitchDataType_t dataType)
1597 : {
1598 : RT_LOG(
1599 : RT_LOG_DEBUG, "Stream switch, condition=%s, dataType=%s.", ConditionToString(condition).c_str(),
1600 : SwitchDataTypeToString(dataType).c_str());
1601 : Context* const curCtx = CurrentContext();
1602 : CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
1603 : COND_RETURN_AND_MSG_INVALID_CONTEXT_STREAM(stm, curCtx, RT_ERROR_STREAM_CONTEXT);
1604 : COND_RETURN_AND_MSG_OUTER(
1605 : !stm->IsModelStream(), RT_ERROR_STREAM_MODEL, ErrorCode::EE1011,
1606 : "Switching between streams based on conditions", 0, "stm->modelNum",
1607 : RtFmtMsg("The stream (stream_id=%d) is not bound to a model", stm->Id_()));
1608 : COND_RETURN_AND_MSG_OUTER(
1609 : !trueStream->IsModelStream(), RT_ERROR_STREAM_MODEL, ErrorCode::EE1011,
1610 : "Switching between streams based on conditions", 0, "trueStream->modelNum",
1611 : RtFmtMsg("The stream (stream_id=%d) is not bound to a model", trueStream->Id_()));
1612 : return CondStreamSwitchEx(ptr, condition, valuePtr, trueStream, stm, dataType, curCtx);
1613 : }
1614 :
1615 : rtError_t ApiImplDavid::LabelSet(Label* const lbl, Stream* const stm)
1616 : {
1617 : Context* const curCtx = CurrentContext();
1618 : CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
1619 : COND_RETURN_AND_MSG_INVALID_CONTEXT_LABEL(lbl, curCtx, RT_ERROR_LABEL_CONTEXT);
1620 : COND_RETURN_AND_MSG_INVALID_CONTEXT_STREAM(stm, curCtx, RT_ERROR_STREAM_CONTEXT);
1621 : return CondLabelSet(lbl, stm);
1622 : }
1623 :
1624 : rtError_t ApiImplDavid::ProfilerTrace(const uint64_t id, const bool notifyFlag, const uint32_t flags, Stream* const stm)
1625 : {
1626 : UNUSED(id);
1627 : UNUSED(notifyFlag);
1628 : UNUSED(flags);
1629 : Context* const curCtx = CurrentContext();
1630 : CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
1631 :
1632 : Stream* curStm = stm;
1633 : if (curStm == nullptr) {
1634 : curStm = curCtx->DefaultStream_();
1635 : NULL_STREAM_PTR_RETURN_MSG(curStm);
1636 : }
1637 : COND_RETURN_AND_MSG_INVALID_CONTEXT_STREAM(curStm, curCtx, RT_ERROR_STREAM_CONTEXT);
1638 : return RT_ERROR_NONE;
1639 : }
1640 :
1641 : rtError_t ApiImplDavid::ProfilerTraceEx(const uint64_t id, const uint64_t modelId, const uint16_t tagId, Stream* stm)
1642 : {
1643 : Context* const curCtx = CurrentContext();
1644 : CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
1645 :
1646 : if (stm == nullptr) {
1647 : stm = curCtx->DefaultStream_();
1648 : NULL_PTR_RETURN_MSG(stm, RT_ERROR_STREAM_NULL);
1649 : }
1650 : COND_RETURN_AND_MSG_INVALID_CONTEXT_STREAM(stm, curCtx, RT_ERROR_STREAM_CONTEXT);
1651 : return ProfTraceEx(id, modelId, tagId, stm, curCtx);
1652 : }
1653 :
1654 : rtError_t ApiImplDavid::WriteValue(rtWriteValueInfo_t* const info, Stream* const stm)
1655 : {
1656 : Context* const curCtx = CurrentContext();
1657 : CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
1658 :
1659 : Stream* curStm = stm;
1660 : if (curStm == nullptr) {
1661 : curStm = curCtx->DefaultStream_();
1662 : NULL_PTR_RETURN_MSG(curStm, RT_ERROR_STREAM_NULL);
1663 : }
1664 : COND_RETURN_AND_MSG_INVALID_CONTEXT_STREAM(curStm, curCtx, RT_ERROR_STREAM_CONTEXT);
1665 :
1666 : return StreamWriteValue(info, curStm);
1667 : }
1668 :
1669 : rtError_t ApiImplDavid::WriteValuePtr(void* const writeValueInfo, Stream* const stm, void* const pointedAddr)
1670 : {
1671 : Context* const curCtx = CurrentContext();
1672 : CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
1673 :
1674 : Stream* curStm = stm;
1675 : if (curStm == nullptr) {
1676 : curStm = curCtx->DefaultStream_();
1677 : NULL_PTR_RETURN_MSG(curStm, RT_ERROR_STREAM_NULL);
1678 : }
1679 : COND_RETURN_AND_MSG_INVALID_CONTEXT_STREAM(curStm, curCtx, RT_ERROR_STREAM_CONTEXT);
1680 : return StreamWriteValuePtr(static_cast<rtWriteValueInfo_t*>(writeValueInfo), curStm, pointedAddr);
1681 : }
1682 :
1683 : rtError_t ApiImplDavid::StarsTaskLaunch(
1684 : const void* const sqe, const uint32_t sqeLen, Stream* const stm, const uint32_t flag)
1685 : {
1686 : RT_LOG(RT_LOG_DEBUG, "Stars launch, sqeLen=%u, flag=%u.", sqeLen, flag);
1687 : Context* const curCtx = CurrentContext();
1688 : CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
1689 :
1690 : Stream* curStm = stm;
1691 : if (curStm == nullptr) {
1692 : curStm = curCtx->DefaultStream_();
1693 : NULL_PTR_RETURN_MSG(curStm, RT_ERROR_STREAM_NULL);
1694 : }
1695 : COND_RETURN_AND_MSG_INVALID_CONTEXT_STREAM(curStm, curCtx, RT_ERROR_STREAM_CONTEXT);
1696 : COND_RETURN_WARN(
1697 : curStm->IsCapturing(), RT_ERROR_FEATURE_NOT_SUPPORT, "stars task cannot be delivered in capture mode.");
1698 : return StarsLaunch(sqe, sqeLen, curStm, flag);
1699 : }
1700 :
1701 : rtError_t ApiImplDavid::LaunchDvppTask(const void* sqe, uint32_t sqeLen, Stream* stm, rtDvppCfg_t* cfg)
1702 : {
1703 : RT_LOG(RT_LOG_INFO, "Start to launch dvpp task.");
1704 :
1705 : // Retrieve the current context and validate its validity
1706 : Context* const curCtx = CurrentContext();
1707 : CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
1708 :
1709 : // Determine the stream to use; if the provided stream is null, use the default stream
1710 : Stream* curStm = (stm == nullptr) ? curCtx->DefaultStream_() : stm;
1711 : NULL_STREAM_PTR_RETURN_MSG(curStm);
1712 :
1713 : // Verify that the stream belongs to the current context
1714 : COND_RETURN_AND_MSG_INVALID_CONTEXT_STREAM(curStm, curCtx, RT_ERROR_STREAM_CONTEXT);
1715 :
1716 : bool isCmdListNotFree = false;
1717 : rtError_t error = GetIsCmdListNotFreeValByDvppCfg(cfg, isCmdListNotFree);
1718 : ERROR_RETURN_MSG_INNER(
1719 : error, "Failed to get dvpp cmdlist not free flag, streamId=%u, retCode=%#x", curStm->Id_(), error);
1720 :
1721 : const uint32_t flag = isCmdListNotFree ? RT_KERNEL_CMDLIST_NOT_FREE : RT_KERNEL_DEFAULT;
1722 : COND_RETURN_WARN(
1723 : curStm->IsCapturing(), RT_ERROR_FEATURE_NOT_SUPPORT, "DVPP tasks cannot be delivered in capture mode.");
1724 : error = StarsTaskLaunch(sqe, sqeLen, curStm, flag);
1725 : ERROR_RETURN(error, "Failed to launch Dvpp task");
1726 : return error;
1727 : }
1728 :
1729 : rtError_t ApiImplDavid::MultipleTaskInfoLaunch(
1730 : const rtMultipleTaskInfo_t* const taskInfo, Stream* const stm, const uint32_t flag)
1731 : {
1732 : for (size_t idx = 0U; idx < taskInfo->taskNum; idx++) {
1733 : if (taskInfo->taskDesc[idx].type == RT_MULTIPLE_TASK_TYPE_DVPP) {
1734 : RT_LOG(
1735 : RT_LOG_DEBUG, "Launch dvpp task, dvppSqeType=%hhu, pos=%hu",
1736 : taskInfo->taskDesc[idx].u.dvppTaskDesc.sqe.sqeHeader.type,
1737 : taskInfo->taskDesc[idx].u.dvppTaskDesc.aicpuTaskPos);
1738 : } else if (taskInfo->taskDesc[idx].type == RT_MULTIPLE_TASK_TYPE_AICPU) {
1739 : RT_LOG(
1740 : RT_LOG_DEBUG,
1741 : "Launch aicpu task, soName=%s, kernelName=%s, opName=%s, blockDim=%hu, isUnderstudyOp=%hu,"
1742 : " argsSize=%u, hostInputInfoNum=%hu",
1743 : taskInfo->taskDesc[idx].u.aicpuTaskDesc.kernelLaunchNames.soName,
1744 : taskInfo->taskDesc[idx].u.aicpuTaskDesc.kernelLaunchNames.kernelName,
1745 : taskInfo->taskDesc[idx].u.aicpuTaskDesc.kernelLaunchNames.opName,
1746 : taskInfo->taskDesc[idx].u.aicpuTaskDesc.blockDim,
1747 : taskInfo->taskDesc[idx].u.aicpuTaskDesc.isUnderstudyOp,
1748 : taskInfo->taskDesc[idx].u.aicpuTaskDesc.argsInfo.argsSize,
1749 : taskInfo->taskDesc[idx].u.aicpuTaskDesc.argsInfo.hostInputInfoNum);
1750 : } else {
1751 : Kernel* hdl = RtPtrToPtr<Kernel*>(taskInfo->taskDesc[idx].u.aicpuTaskDescByHandle.funcHdl);
1752 : RT_LOG(
1753 : RT_LOG_DEBUG,
1754 : "launch aicpu task by handle, soName=%s, funcName=%s, opName=%s, blockDim=%hu, isUnderstudyOp=%hu,"
1755 : " argsSize=%u, hostInputInfoNum=%hu,",
1756 : hdl->GetCpuKernelSo().c_str(), hdl->GetCpuFuncName().c_str(), hdl->GetCpuOpType().c_str(),
1757 : taskInfo->taskDesc[idx].u.aicpuTaskDescByHandle.blockDim,
1758 : taskInfo->taskDesc[idx].u.aicpuTaskDescByHandle.isUnderstudyOp,
1759 : taskInfo->taskDesc[idx].u.aicpuTaskDescByHandle.argsInfo.argsSize,
1760 : taskInfo->taskDesc[idx].u.aicpuTaskDescByHandle.argsInfo.hostInputInfoNum);
1761 : }
1762 : }
1763 :
1764 : Context* const curCtx = CurrentContext();
1765 : CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
1766 :
1767 : Stream* curStm = stm;
1768 : if (curStm == nullptr) {
1769 : curStm = curCtx->DefaultStream_();
1770 : NULL_PTR_RETURN_MSG(curStm, RT_ERROR_STREAM_NULL);
1771 : }
1772 : COND_RETURN_AND_MSG_INVALID_CONTEXT_STREAM(curStm, curCtx, RT_ERROR_STREAM_CONTEXT);
1773 : COND_RETURN_WARN(
1774 : curStm->IsCapturing(), RT_ERROR_FEATURE_NOT_SUPPORT, "DVPP tasks cannot be delivered in capture mode.");
1775 : return LaunchMultipleTaskInfo(taskInfo, curStm, flag);
1776 : }
1777 :
1778 : rtError_t ApiImplDavid::DvppWaitGroupReport(
1779 : DvppGrp* const grp, rtDvppGrpCallback const callBackFunc, const int32_t timeout)
1780 : {
1781 : Context* const curCtx = grp->getContext();
1782 : CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
1783 : return DvppWaitGroup(curCtx->Device_(), grp, callBackFunc, timeout);
1784 : }
1785 :
1786 : static rtError_t SetLimitSizeByType(const rtLimitType_t type, const uint32_t val)
1787 : {
1788 : Runtime* rt = Runtime::Instance();
1789 : uint32_t alignVal = (val + STACK_PHY_BASE_ALIGN_LEN - 1U) / STACK_PHY_BASE_ALIGN_LEN * STACK_PHY_BASE_ALIGN_LEN;
1790 : switch (type) {
1791 : case RT_LIMIT_TYPE_SIMT_STACK_SIZE:
1792 : rt->SetSimtWarpStkSize(alignVal * RT_MAX_THREAD_NUM_PER_WARP);
1793 : break;
1794 : case RT_LIMIT_TYPE_SIMT_DVG_WARP_STACK_SIZE:
1795 : rt->SetSimtDvgWarpStkSize(alignVal);
1796 : break;
1797 : default:
1798 : RT_LOG_OUTER_MSG_WITH_FUNC(
1799 : ErrorCode::EE1003, LimitTypeToString(type), "type",
1800 : LimitTypeToString(RT_LIMIT_TYPE_SIMT_STACK_SIZE) + " or " +
1801 : LimitTypeToString(RT_LIMIT_TYPE_SIMT_DVG_WARP_STACK_SIZE));
1802 : return RT_ERROR_DEVICE_LIMIT;
1803 : }
1804 : COND_RETURN_AND_MSG_OUTER_WITH_PARAM(
1805 : (rt->GetSimtWarpStkSize() == 0) && (rt->GetSimtDvgWarpStkSize() == 0), RT_ERROR_INVALID_VALUE, 0,
1806 : "non-zero stack size");
1807 : return RT_ERROR_NONE;
1808 : }
1809 :
1810 : rtError_t ApiImplDavid::DeviceSetLimit(const int32_t devId, const rtLimitType_t type, const uint32_t val)
1811 : {
1812 : RT_LOG(
1813 : RT_LOG_DEBUG, "drv devId=%u, type=%s, value=%u.", static_cast<uint32_t>(devId), LimitTypeToString(type).c_str(),
1814 : val);
1815 : rtError_t error = RT_ERROR_NONE;
1816 : Runtime* rt = Runtime::Instance();
1817 : COND_RETURN_ERROR_MSG_INNER(rt == nullptr, RT_ERROR_INSTANCE_NULL, "Runtime instance is null.");
1818 : if (type == RT_LIMIT_TYPE_LOW_POWER_TIMEOUT) {
1819 : Context* const curCtx = CurrentContext();
1820 : CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
1821 : RT_LOG(
1822 : RT_LOG_WARNING, "DeviceSetLimit, drv devId=%u, type=%s, value=%u.", static_cast<uint32_t>(devId),
1823 : LimitTypeToString(type).c_str(), val);
1824 : return error;
1825 : } else if (type == RT_LIMIT_TYPE_STACK_SIZE) {
1826 : std::unique_lock<std::mutex> lock(rt->GetSimtStackMutex());
1827 : rt->SetDeviceCustomerStackSize(val);
1828 : return RT_ERROR_NONE;
1829 : } else if (type == RT_LIMIT_TYPE_SIMD_PRINTF_FIFO_SIZE_PER_CORE) {
1830 : std::unique_lock<std::mutex> lock(rt->GetSimdFifoMutex());
1831 : return rt->SetSimdPrintFifoSize(val);
1832 : } else if (type == RT_LIMIT_TYPE_SIMT_PRINTF_FIFO_SIZE) {
1833 : std::unique_lock<std::mutex> lock(rt->GetSimtFifoMutex());
1834 : return rt->SetSimtPrintFifoSize(val);
1835 : } else {
1836 : // no op
1837 : }
1838 :
1839 : error = SetLimitSizeByType(type, val);
1840 : COND_RETURN_ERROR_MSG_INNER(
1841 : error != RT_ERROR_NONE, error, "Set simt stack size failed, drv devId=%u, retCode=%#x.",
1842 : static_cast<uint32_t>(devId), static_cast<uint32_t>(error));
1843 : return RT_ERROR_NONE;
1844 : }
1845 :
1846 : rtError_t ApiImplDavid::DeviceGetLimit(const rtLimitType_t type, uint32_t* val)
1847 : {
1848 : RT_LOG(RT_LOG_DEBUG, "type=%s.", LimitTypeToString(type).c_str());
1849 : Runtime* rt = Runtime::Instance();
1850 : COND_RETURN_ERROR_MSG_INNER(rt == nullptr, RT_ERROR_INSTANCE_NULL, "Runtime instance is null.");
1851 : if (type == RT_LIMIT_TYPE_LOW_POWER_TIMEOUT) {
1852 : Context* const curCtx = CurrentContext();
1853 : CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
1854 : RT_LOG(RT_LOG_WARNING, "DeviceGetLimit, type=%s.", LimitTypeToString(type).c_str());
1855 : *val = 0;
1856 : } else if (type == RT_LIMIT_TYPE_STACK_SIZE) {
1857 : *val = rt->GetDeviceCustomerStackSize();
1858 : } else if (type == RT_LIMIT_TYPE_SIMD_PRINTF_FIFO_SIZE_PER_CORE) {
1859 : std::unique_lock<std::mutex> lock(rt->GetSimdFifoMutex());
1860 : *val = rt->GetSimdPrintFifoSize();
1861 : } else if (type == RT_LIMIT_TYPE_SIMT_PRINTF_FIFO_SIZE) {
1862 : std::unique_lock<std::mutex> lock(rt->GetSimtFifoMutex());
1863 : *val = rt->GetSimtPrintFifoSize();
1864 : } else if (type == RT_LIMIT_TYPE_SIMT_STACK_SIZE) {
1865 : *val = static_cast<uint32_t>(rt->GetSimtWarpStkSize());
1866 : } else if (type == RT_LIMIT_TYPE_SIMT_DVG_WARP_STACK_SIZE) {
1867 : *val = rt->GetSimtDvgWarpStkSize();
1868 : } else {
1869 : RT_LOG(RT_LOG_WARNING, "Limit type is not supported, type=%s", LimitTypeToString(type).c_str());
1870 : return RT_ERROR_FEATURE_NOT_SUPPORT;
1871 : }
1872 : RT_LOG(RT_LOG_INFO, "DeviceGetLimit success, type=%s, val=%u.", LimitTypeToString(type).c_str(), *val);
1873 : return RT_ERROR_NONE;
1874 : }
1875 :
1876 : rtError_t ApiImplDavid::StreamTaskAbort(Stream* const stm)
1877 : {
1878 : Stream* curStm = stm;
1879 : if (curStm == nullptr) {
1880 : Context* const curCtx = CurrentContext();
1881 : CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
1882 : curStm = curCtx->DefaultStream_();
1883 : NULL_STREAM_PTR_RETURN_MSG(curStm);
1884 : } else {
1885 : const bool isValid = ContextManage::CheckStreamPtrIsValid(curStm);
1886 : COND_RETURN_AND_MSG_OUTER(
1887 : !isValid, RT_ERROR_INVALID_VALUE, ErrorCode::EE1017, "Aborting a stream task", "stm",
1888 : RtFmtMsg("Stream (stream_id=%d) does not belong to any context", curStm->Id_()));
1889 : }
1890 :
1891 : return curStm->StreamAbort();
1892 : }
1893 :
1894 : rtError_t ApiImplDavid::StreamAbort(Stream* const stm)
1895 : {
1896 : Context* const curCtx = CurrentContext();
1897 : CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
1898 : Stream* curStm = (stm == nullptr) ? curCtx->DefaultStream_() : stm;
1899 : NULL_STREAM_PTR_RETURN_MSG(curStm);
1900 :
1901 : COND_RETURN_AND_MSG_INVALID_CONTEXT_STREAM(curStm, curCtx, RT_ERROR_STREAM_CONTEXT);
1902 : COND_RETURN_AND_MSG_OUTER(
1903 : ((curStm->Flags() & RT_STREAM_PERSISTENT) != 0U), RT_ERROR_STREAM_INVALID, ErrorCode::EE1006, "Stream aborting",
1904 : "Aborting persistent stream",
1905 : "The stream flag contains ACL_STREAM_PERSISTENT(0x4) and the stream cannot be aborted");
1906 : return curStm->StreamAbort();
1907 : }
1908 :
1909 : rtError_t ApiImplDavid::StreamStop(Stream* const stm)
1910 : {
1911 : Context* const curCtx = CurrentContext();
1912 : CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
1913 : COND_RETURN_AND_MSG_INVALID_CONTEXT_STREAM(stm, curCtx, RT_ERROR_STREAM_CONTEXT);
1914 : COND_RETURN_ERROR_MSG_INNER(
1915 : stm->GetBindFlag(), RT_ERROR_STREAM_INVALID, "StreamStop not support model stream, stream_id=%d.", stm->Id_());
1916 : return stm->StreamStop();
1917 : }
1918 : rtError_t ApiImplDavid::StreamRecover(Stream* const stm)
1919 : {
1920 : const bool isValid = ContextManage::CheckStreamPtrIsValid(stm);
1921 : COND_RETURN_AND_MSG_OUTER(
1922 : !isValid, RT_ERROR_INVALID_VALUE, ErrorCode::EE1017, "Resuming tasks in a stream", "stm",
1923 : RtFmtMsg("Stream (stream_id=%d) does not belong to any context", stm->Id_()));
1924 : return stm->StreamRecoverAbort();
1925 : }
1926 :
1927 : rtError_t ApiImplDavid::StreamTaskClean(Stream* const stm)
1928 : {
1929 : Context* const curCtx = CurrentContext();
1930 : CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
1931 : COND_RETURN_AND_MSG_INVALID_CONTEXT_STREAM(stm, curCtx, RT_ERROR_STREAM_CONTEXT);
1932 : return stm->StreamTaskClean();
1933 : }
1934 :
1935 : rtError_t ApiImplDavid::DeviceResourceClean(int32_t devId) { return ContextManage::DeviceResourceClean(devId); }
1936 :
1937 : rtError_t ApiImplDavid::LabelGotoEx(Label* const lbl, Stream* const stm)
1938 : {
1939 : Context* const curCtx = CurrentContext();
1940 : CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
1941 : COND_RETURN_AND_MSG_INVALID_CONTEXT_STREAM(stm, curCtx, RT_ERROR_STREAM_CONTEXT);
1942 : COND_RETURN_AND_MSG_INVALID_CONTEXT_LABEL(lbl, curCtx, RT_ERROR_LABEL_CONTEXT);
1943 : RT_LOG_OUTER_MSG_WITH_FUNC_DESC(ErrorCode::EE1005, "Label redirection");
1944 : return RT_ERROR_FEATURE_NOT_SUPPORT;
1945 : }
1946 :
1947 : rtError_t ApiImplDavid::GetMemUceInfo(const uint32_t deviceId, rtMemUceInfo* memUceInfo)
1948 : {
1949 : rtError_t error;
1950 : rtErrorInfo errorInfo = {};
1951 : error = GetMemUceInfoProc(deviceId, &errorInfo);
1952 : COND_RETURN_WARN(
1953 : error == RT_ERROR_FEATURE_NOT_SUPPORT, RT_ERROR_FEATURE_NOT_SUPPORT, "Not support get mem uce info.");
1954 : if (error != RT_ERROR_NONE) {
1955 : RT_LOG(RT_LOG_ERROR, "Get mem uce info failed, drv devId=%u, error=%d.", deviceId, error);
1956 : return error;
1957 : }
1958 :
1959 : rtMemUceArray* memUceArray = &(errorInfo.detail.uceInfo);
1960 : memUceInfo->devid = deviceId;
1961 : memUceInfo->count = memUceArray->arraySize;
1962 : errno_t ret = memcpy_s(
1963 : memUceInfo->repairAddr, sizeof(memUceInfo->repairAddr), memUceArray->repairAddrArray,
1964 : sizeof(memUceArray->repairAddrArray));
1965 : if (ret != EOK) {
1966 : const std::string retStr = std::to_string(ret);
1967 : std::stringstream ss;
1968 : ss << std::hex << "dest=0x" << RtPtrToValue(memUceInfo->repairAddr) << ", src=0x"
1969 : << RtPtrToValue(memUceArray->repairAddrArray) << std::dec << ", destMax=" << sizeof(memUceInfo->repairAddr)
1970 : << ", count=" << sizeof(memUceArray->repairAddrArray) << ".";
1971 : RT_LOG_OUTER_MSG_IMPL(ErrorCode::EE1020, __func__, "memcpy_s", retStr.c_str(), strerror(ret), ss.str().c_str());
1972 : return RT_ERROR_INVALID_VALUE;
1973 : }
1974 :
1975 : return RT_ERROR_NONE;
1976 : }
1977 :
1978 : rtError_t ApiImplDavid::DeviceTaskAbort(const int32_t devId, const uint32_t timeout)
1979 : {
1980 : return DavidDeviceTaskAbort(devId, timeout);
1981 : }
1982 :
1983 : static rtError_t L2BufferErrProc(const uint32_t deviceId, rtErrorInfo* const errorInfo)
1984 : {
1985 : errorInfo->errorType = RT_ERROR_L2;
1986 :
1987 : uint32_t resume_cnt = MAX_UINT32_NUM;
1988 : int32_t buf_size = sizeof(resume_cnt);
1989 : const rtError_t error = NpuDriver::GetDeviceInfoByBuff(
1990 : deviceId, MODULE_TYPE_L2BUFF, INFO_TYPE_L2BUFF_RESUME_CNT, static_cast<void*>(&resume_cnt), &buf_size);
1991 : COND_RETURN_WARN(
1992 : error == RT_ERROR_FEATURE_NOT_SUPPORT, RT_ERROR_FEATURE_NOT_SUPPORT, "Not support get fault event info.");
1993 : if ((error != RT_ERROR_NONE) || (resume_cnt == MAX_UINT32_NUM) || (buf_size != sizeof(resume_cnt))) {
1994 : RT_LOG(
1995 : RT_LOG_ERROR, "Calling drv api halGetDeviceInfoByBuff failed, resume_cnt=%u, buf_size=%d, error=%#x.",
1996 : resume_cnt, buf_size, static_cast<uint32_t>(error));
1997 : return RT_ERROR_DRV_ERR;
1998 : }
1999 :
2000 : if (resume_cnt == 0U) {
2001 : errorInfo->tryRepair = 1U;
2002 : } else {
2003 : errorInfo->tryRepair = 0U;
2004 : RT_LOG(RT_LOG_INFO, "Resume cnt is not zero, recovery may have been triggered.");
2005 : }
2006 : return error;
2007 : }
2008 :
2009 : static void AicoreErrorProc(const Device* const dev, rtErrorInfo* const errorInfo)
2010 : {
2011 : errorInfo->hasDetail = 1U;
2012 : errorInfo->errorType = RT_ERROR_AICORE;
2013 :
2014 : const uint32_t recoverCnt = dev->GetAixErrRecoverCnt();
2015 : if (recoverCnt == 0U) {
2016 : errorInfo->tryRepair = 1U;
2017 : } else {
2018 : errorInfo->tryRepair = 0U;
2019 : RT_LOG(RT_LOG_INFO, "Aicore recover cnt is not zero or other fault event exists, recoverCnt=%u.", recoverCnt);
2020 : }
2021 :
2022 : return;
2023 : }
2024 :
2025 : static void AicoreUnknownErrorProc(rtErrorInfo* const errorInfo)
2026 : {
2027 : errorInfo->errorType = RT_ERROR_AICORE;
2028 : errorInfo->hasDetail = 1U;
2029 : errorInfo->detail.aicoreErrType = RT_AICORE_ERROR_UNKNOWN;
2030 : }
2031 :
2032 : static void UnknowErrorProc(const Context* const curCtx, rtErrorInfo* const errorInfo)
2033 : {
2034 : if (curCtx->GetFailureError() == RT_ERROR_NONE) {
2035 : errorInfo->errorType = RT_NO_ERROR;
2036 : } else {
2037 : errorInfo->errorType = RT_ERROR_OTHERS;
2038 : }
2039 : }
2040 :
2041 : rtError_t ApiImplDavid::GetErrorVerbose(const uint32_t deviceId, rtErrorInfo* const errorInfo)
2042 : {
2043 : errorInfo->hasDetail = 0U;
2044 : errorInfo->tryRepair = 0U;
2045 : errorInfo->errorType = RT_NO_ERROR;
2046 : const uint32_t tsId = InnerThreadLocalContainer::GetTsId();
2047 : Context* const ctx = Runtime::Instance()->GetPriCtxByDeviceId(deviceId, tsId);
2048 : COND_RETURN_WARN(ctx == nullptr, RT_ERROR_NONE, "Device[%u] has no fault.", deviceId);
2049 : Device* const dev = ctx->Device_();
2050 : COND_RETURN_WARN(dev == nullptr, RT_ERROR_NONE, "Device[%u] has no fault.", deviceId);
2051 :
2052 : rtError_t error = RT_ERROR_NONE;
2053 : const DeviceFaultType faultType = dev->GetDeviceFaultType();
2054 : RT_LOG(RT_LOG_DEBUG, "start GetErrorVerbose, device_id=%u, type=%u", deviceId, faultType);
2055 : switch (faultType) {
2056 : case DeviceFaultType::L2_BUFFER_ERROR:
2057 : error = L2BufferErrProc(deviceId, errorInfo);
2058 : break;
2059 : case DeviceFaultType::HBM_UCE_ERROR:
2060 : error = GetMemUceInfoProc(deviceId, errorInfo);
2061 : errorInfo->errorType = RT_ERROR_MEMORY;
2062 : break;
2063 : case DeviceFaultType::AICORE_SW_ERROR:
2064 : errorInfo->detail.aicoreErrType = RT_AICORE_ERROR_SW;
2065 : AicoreErrorProc(dev, errorInfo);
2066 : break;
2067 : case DeviceFaultType::AICORE_HW_L_ERROR:
2068 : errorInfo->detail.aicoreErrType = RT_AICORE_ERROR_HW_LOCAL;
2069 : AicoreErrorProc(dev, errorInfo);
2070 : break;
2071 : case DeviceFaultType::AICORE_UNKNOWN_ERROR:
2072 : AicoreUnknownErrorProc(errorInfo);
2073 : break;
2074 : case DeviceFaultType::LINK_ERROR:
2075 : errorInfo->errorType = RT_ERROR_LINK;
2076 : errorInfo->tryRepair = 1U;
2077 : break;
2078 : case DeviceFaultType::L3_PORT_ERROR:
2079 : errorInfo->errorType = RT_ERROR_L3_PORT;
2080 : errorInfo->tryRepair = 1U;
2081 : break;
2082 : default:
2083 : UnknowErrorProc(ctx, errorInfo);
2084 : break;
2085 : }
2086 : return error;
2087 : }
2088 :
2089 : static rtError_t L2BufferErrorResume(Device* const dev, const uint32_t deviceId)
2090 : {
2091 : uint32_t buf_context = DRV_L2BUFF_CLEAN;
2092 : const rtError_t error = NpuDriver::SetDeviceInfoByBuff(
2093 : deviceId, MODULE_TYPE_L2BUFF, INFO_TYPE_L2BUFF_RESUME, static_cast<void*>(&buf_context), sizeof(buf_context));
2094 : COND_RETURN_WARN(
2095 : error == RT_ERROR_FEATURE_NOT_SUPPORT, RT_ERROR_FEATURE_NOT_SUPPORT, "Not support l2 buffer resume.");
2096 : COND_PROC(
2097 : (error != RT_ERROR_NONE), RT_LOG(
2098 : RT_LOG_ERROR, "L2 buffer err repair failed, deviceId=%u, retCode=%#x.", deviceId,
2099 : static_cast<uint32_t>(error)));
2100 : dev->SetDeviceFaultType(DeviceFaultType::NO_ERROR);
2101 : return error;
2102 : }
2103 :
2104 : static rtError_t L3PortRepairResume(Device* const dev)
2105 : {
2106 : halRepairFaultInfo repairInfo = {};
2107 : repairInfo.fault_type = HAL_REPAIR_FAULT_TYPE_UBMEM;
2108 : const rtError_t error = NpuDriver::L3PortRepair(dev->Id_(), &repairInfo);
2109 : COND_RETURN_WARN(
2110 : error == RT_ERROR_FEATURE_NOT_SUPPORT, RT_ERROR_FEATURE_NOT_SUPPORT, "Not support l3 prot resume.");
2111 : COND_PROC(
2112 : (error != RT_ERROR_NONE), RT_LOG(
2113 : RT_LOG_ERROR, "l3 port err repair failed, deviceId=%u, retCode=%#x.", dev->Id_(),
2114 : static_cast<uint32_t>(error)));
2115 : dev->SetDeviceFaultType(DeviceFaultType::NO_ERROR);
2116 : return error;
2117 : }
2118 :
2119 : static void L3PortErrorStatusReset(Device* const dev)
2120 : {
2121 : dev->SetDeviceStatus(RT_ERROR_NONE);
2122 : const ReadProtect rp(&ContextDataManage::Instance().GetSetRwLock());
2123 : for (Context* const ctx : ContextDataManage::Instance().GetSetObj()) {
2124 : if (!ContextManage::IsContextOnDevice(ctx, static_cast<int32_t>(dev->Id_()))) {
2125 : continue;
2126 : }
2127 : ctx->SetStreamsStatus(RT_ERROR_NONE);
2128 : ctx->SetFailureError(RT_ERROR_NONE);
2129 : }
2130 : }
2131 :
2132 : rtError_t ApiImplDavid::RepairError(const uint32_t deviceId, const rtErrorInfo* const errorInfo)
2133 : {
2134 : rtError_t error = RT_ERROR_NONE;
2135 : const uint32_t tsId = InnerThreadLocalContainer::GetTsId();
2136 : Device* const dev = Runtime::Instance()->GetDevice(deviceId, tsId);
2137 : NULL_PTR_RETURN_MSG(dev, RT_ERROR_DEVICE_NULL);
2138 : switch (errorInfo->errorType) {
2139 : case RT_ERROR_L2:
2140 : error = L2BufferErrorResume(dev, deviceId);
2141 : break;
2142 : case RT_ERROR_AICORE:
2143 : dev->SetAixErrRecoverCnt();
2144 : dev->SetDeviceFaultType(DeviceFaultType::NO_ERROR);
2145 : break;
2146 : case RT_ERROR_MEMORY:
2147 : error = MemUceErrorResume(dev, deviceId, errorInfo);
2148 : break;
2149 : case RT_ERROR_LINK:
2150 : dev->SetDeviceFaultType(DeviceFaultType::NO_ERROR);
2151 : break;
2152 : case RT_ERROR_L3_PORT:
2153 : error = L3PortRepairResume(dev);
2154 : L3PortErrorStatusReset(dev);
2155 : break;
2156 : default:
2157 : error = RT_ERROR_INVALID_VALUE;
2158 1 : RT_LOG(
2159 : RT_LOG_ERROR, "Does not support current error type [%s]",
2160 : ErrorTypeToString(errorInfo->errorType).c_str());
2161 : break;
2162 : }
2163 : dev->SetBaseTime();
2164 : return error;
2165 : }
2166 :
2167 : rtError_t ApiImplDavid::GetStackBuffer(
2168 : const rtBinHandle binHandle, uint32_t deviceId, const uint32_t stackType, const uint32_t coreType,
2169 : const uint32_t coreId, const void** stack, uint32_t* stackSize)
2170 : {
2171 : RT_LOG(
2172 : RT_LOG_DEBUG, "Get stack buffer, bin handle %p, stackType %u, coreType %u, coreId %u", binHandle, stackType,
2173 : coreType, coreId);
2174 : return GetStackBufferInfo(binHandle, deviceId, stackType, coreType, coreId, stack, stackSize);
2175 : }
2176 :
2177 : rtError_t ApiImplDavid::DebugReadAICore(rtDebugMemoryParam_t* const param) { return ReadAICoreDebugInfo(param); }
2178 :
2179 : rtError_t ApiImplDavid::StarsLaunchSubscribeProc(
2180 : Stream* const stm, const rtCallback_t callBackFunc, void* const fnData, const bool needSubscribe,
2181 : const uint64_t threadId)
2182 : {
2183 : rtError_t ret = RT_ERROR_NONE;
2184 : Runtime* const rtInstance = Runtime::Instance();
2185 : if (needSubscribe && !(stm->IsCapturing())) {
2186 : Notify* curNotify = nullptr;
2187 : ret = NotifyCreate(static_cast<int32_t>(stm->Device_()->Id_()), &curNotify, RT_NOTIFY_DEFAULT);
2188 : ERROR_RETURN(ret, "Call NotifyCreate failed for callback, ret=%#x.", ret);
2189 : ret = rtInstance->SubscribeCallback(threadId, stm, static_cast<void*>(curNotify));
2190 : if (ret != RT_ERROR_NONE) {
2191 : (void)NotifyDestroy(curNotify);
2192 : }
2193 : }
2194 :
2195 : return CallbackLaunchForDavidWithBlock(callBackFunc, fnData, stm, threadId);
2196 : }
2197 :
2198 : rtError_t ApiImplDavid::LaunchHostFunc(Stream* const stm, const rtCallback_t callBackFunc, void* const fnData)
2199 : {
2200 : Context* const curCtx = CurrentContext();
2201 : CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
2202 : Stream* curStm = stm;
2203 : if (curStm == nullptr) {
2204 : curStm = curCtx->DefaultStream_();
2205 : NULL_STREAM_PTR_RETURN_MSG(curStm);
2206 : }
2207 : COND_RETURN_AND_MSG_INVALID_CONTEXT_STREAM(curStm, curCtx, RT_ERROR_STREAM_CONTEXT);
2208 : Runtime* const rtInstance = Runtime::Instance();
2209 : Device* const dev = curCtx->Device_();
2210 : NULL_PTR_RETURN_MSG(dev, RT_ERROR_INVALID_VALUE);
2211 : // lock first Check whether the thread exists. If the thread does not exist, create a thread in context level.
2212 : curCtx->callbackTheadMutex_.lock();
2213 : if (!curCtx->GetCallBackThreadExistFlag()) {
2214 : if (curCtx->CreateContextCallBackThread() != RT_ERROR_NONE) {
2215 : curCtx->callbackTheadMutex_.unlock();
2216 : RT_LOG_INNER_MSG(RT_LOG_ERROR, "Failed to create callback thread.");
2217 : return RT_ERROR_MEMORY_ALLOCATION;
2218 : }
2219 : curCtx->SetCallBackThreadExistFlag();
2220 : }
2221 : curCtx->callbackTheadMutex_.unlock();
2222 : // if new stream should subscribe in map first; else launchcallback Directly
2223 : const bool isNeedSubscribe = rtInstance->JudgeNeedSubscribe(curCtx->GetCallBackThreadId(), curStm, dev->Id_());
2224 :
2225 : return StarsLaunchSubscribeProc(curStm, callBackFunc, fnData, isNeedSubscribe, curCtx->GetCallBackThreadId());
2226 : }
2227 :
2228 : rtError_t ApiImplDavid::MemWriteValue(
2229 : const void* const devAddr, const uint64_t value, const uint32_t flag, Stream* const stm)
2230 : {
2231 : Context* const curCtx = CurrentContext();
2232 : CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
2233 :
2234 : Stream* curStm = stm;
2235 : if (curStm == nullptr) {
2236 : curStm = curCtx->DefaultStream_();
2237 : NULL_STREAM_PTR_RETURN_MSG(curStm);
2238 : }
2239 :
2240 : COND_RETURN_AND_MSG_INVALID_CONTEXT_STREAM(curStm, curCtx, RT_ERROR_STREAM_CONTEXT);
2241 :
2242 : return cce::runtime::MemWriteValue(devAddr, value, flag, curStm);
2243 : }
2244 :
2245 : rtError_t ApiImplDavid::MemWaitValue(
2246 : const void* const devAddr, const uint64_t value, const uint32_t flag, Stream* const stm)
2247 : {
2248 : Context* const curCtx = CurrentContext();
2249 : CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
2250 :
2251 : Stream* curStm = stm;
2252 : if (curStm == nullptr) {
2253 : curStm = curCtx->DefaultStream_();
2254 : NULL_STREAM_PTR_RETURN_MSG(curStm);
2255 : }
2256 :
2257 : COND_RETURN_AND_MSG_INVALID_CONTEXT_STREAM(curStm, curCtx, RT_ERROR_STREAM_CONTEXT);
2258 :
2259 : return cce::runtime::MemWaitValue(devAddr, value, flag, curStm);
2260 : }
2261 :
2262 : rtError_t ApiImplDavid::StreamAddCondTask(rtCondTaskParams params, Stream* const stm, uint32_t flags)
2263 : {
2264 : CondHandle* realHandle = nullptr;
2265 : rtError_t error = StreamAddCondTaskParasCheck(params, stm, &realHandle);
2266 : COND_RETURN_ERROR(error != RT_ERROR_NONE, error, "condition task parameters check failed.");
2267 :
2268 : Context* const curCtx = CurrentContext();
2269 : CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
2270 : error = curCtx->CreateSubCaptureModels(realHandle, params, stm);
2271 : ERROR_RETURN_MSG_INNER(
2272 : error, "Create sub capture model failed, condition type=%s, condition size=%u, retCode=%#x.",
2273 : CondTaskTypeToString(params.type).c_str(), params.size, static_cast<uint32_t>(error));
2274 :
2275 : return cce::runtime::StreamAddCondTask(realHandle, params, stm, flags);
2276 : }
2277 :
2278 : rtError_t ApiImplDavid::IpcSetMemoryAttr(const char* name, uint32_t type, uint64_t attr)
2279 : {
2280 : RT_LOG(RT_LOG_DEBUG, "Set ipc memory attribute. name=%s, type=%u, attr=%" PRIu64 ".", name, type, attr);
2281 :
2282 : Context* const curCtx = CurrentContext();
2283 : CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
2284 : const rtError_t error = curCtx->Device_()->Driver_()->CheckIpcMapRoute(name, attr, curCtx->Device_()->Id_());
2285 : COND_RETURN_WITH_NOLOG(error != RT_ERROR_NONE, error);
2286 : const std::unique_lock<std::mutex> lock(Runtime::Instance()->GetIpcMemNameLock());
2287 : std::unordered_map<std::string, ipcMemInfo_t>& ipcMemNameMap = Runtime::Instance()->GetIpcMemNameMap();
2288 : std::string ipcName(name);
2289 : auto it = ipcMemNameMap.find(ipcName);
2290 : if (it == ipcMemNameMap.end()) {
2291 : ipcMemInfo_t& info = ipcMemNameMap[ipcName];
2292 : info.latestAttr = attr;
2293 : } else {
2294 : it->second.latestAttr = attr;
2295 : }
2296 : return RT_ERROR_NONE;
2297 : }
2298 : } // namespace runtime
2299 : } // namespace cce
|
