Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "operator_kernel_post_process_dynamic_output.h"
12 :
13 : #include "aicpusd_status.h"
14 : #include "aicpusd_model_execute.h"
15 : #include "aicpusd_resource_manager.h"
16 : #include "operator_kernel_common.h"
17 :
18 : namespace AicpuSchedule {
19 : namespace {
20 : const std::string KERNEL_POST_PROCESS_DYNAMIC_OUTPUT = "postprocessDynamicOutput";
21 : const std::string KERNEL_POST_PROCESS_DYNAMIC_OUTPUT_V2 = "postprocessDynamicOutputV2";
22 :
23 : enum class OutputType {
24 : STATIC_OUTPUT = 0,
25 : DYNAMIC_OUTPUT_WITH_MAXSIZE = 1,
26 : DYNAMIC_OUTPUT_WITHOUT_MAXSIZE = 2
27 : };
28 : } // namespace
29 :
30 1 : int32_t OperatorKernelPostProcessDynamicOutput::Compute(const AicpuTaskInfo &kernelTaskInfo,
31 : const RunContext &taskContext)
32 : {
33 1 : aicpusd_info("Start ModelPostprocessDynamicOutput. modelId=%u, streamId=%u, taskId=%u.",
34 : taskContext.modelId, kernelTaskInfo.streamID, kernelTaskInfo.taskID);
35 1 : if (kernelTaskInfo.paraBase == 0UL) {
36 0 : aicpusd_err("kernelTaskInfo.paraBase is null");
37 0 : return AICPU_SCHEDULE_ERROR_PARAMETER_NOT_VALID;
38 : }
39 :
40 : const PostprocessDynamicOutputKernelArgs * const param =
41 1 : PtrToPtr<void, PostprocessDynamicOutputKernelArgs>(ValueToPtr(kernelTaskInfo.paraBase));
42 1 : if ((param->outputsNum > 0U) && ((param->outputDynamicFlagsAddr == 0U) || (param->outputMbufAddrsAddr == 0U))) {
43 0 : aicpusd_err("Parameter invalid: outputsNum[%u], outputDynamicFlagsAddr[%u], outputMbufAddrsAddr[%u]",
44 : param->outputsNum, param->outputDynamicFlagsAddr, param->outputMbufAddrsAddr);
45 0 : return AICPU_SCHEDULE_ERROR_PARAMETER_NOT_VALID;
46 : }
47 :
48 1 : const auto copyRet = CopyTensorDesc(param, taskContext);
49 1 : if (copyRet != AICPU_SCHEDULE_OK) {
50 0 : return copyRet;
51 : }
52 :
53 1 : return FreeMbuf(param, taskContext);
54 : }
55 :
56 8 : int32_t OperatorKernelPostProcessDynamicOutput::CopyTensorDesc(const PostprocessDynamicOutputKernelArgs * const param,
57 : const RunContext &taskContext) const
58 : {
59 8 : const uint32_t * const dynamicFlags = PtrToPtr<void, uint32_t>(ValueToPtr(param->outputDynamicFlagsAddr));
60 8 : uint64_t *const outputPptrs = PtrToPtr<void, uint64_t>(ValueToPtr(param->outputMbufAddrsAddr));
61 8 : RuntimeTensorDesc *dynamicSrcDesc = nullptr;
62 8 : RuntimeTensorDesc *staticSrcDesc = PtrToPtr<void, RuntimeTensorDesc>(ValueToPtr(param->outputStaticTensorDescAddr));
63 :
64 8 : void *customBuf = nullptr;
65 8 : uint32_t customBufSize = 0U;
66 12 : for (size_t index = 0U; index < param->outputsNum; ++index) {
67 10 : aicpusd_info("dynamicFlags[%zu] is %u.", index, dynamicFlags[index]);
68 10 : if (outputPptrs[index] == 0U) {
69 1 : aicpusd_err("the [%zu]th outputMbufPtr is null", index);
70 1 : return AICPU_SCHEDULE_ERROR_PARAMETER_NOT_VALID;
71 : }
72 :
73 9 : if ((dynamicFlags[index] == static_cast<uint32_t>(OutputType::STATIC_OUTPUT)) ||
74 7 : (dynamicFlags[index] == static_cast<uint32_t>(OutputType::DYNAMIC_OUTPUT_WITH_MAXSIZE))) {
75 5 : Mbuf *const outputMbuf = *(reinterpret_cast<Mbuf **>(outputPptrs[index]));
76 5 : const auto ret = AllocatedOutput(param, outputMbuf, dynamicFlags[index], index,
77 : &dynamicSrcDesc, &staticSrcDesc);
78 5 : if (ret != AICPU_SCHEDULE_OK) {
79 2 : return ret;
80 : }
81 3 : continue;
82 3 : }
83 :
84 4 : if (dynamicFlags[index] == static_cast<uint32_t>(OutputType::DYNAMIC_OUTPUT_WITHOUT_MAXSIZE)) {
85 : const auto getHeadRet =
86 4 : GetMbufHeadFromResp(reinterpret_cast<Mbuf **>(param->respMsgMbufAddr), taskContext, &customBuf,
87 : &customBufSize);
88 4 : if (getHeadRet != AICPU_SCHEDULE_OK) {
89 2 : return getHeadRet;
90 : }
91 2 : const auto ret = PostProcessForOutputToAllocate(param, reinterpret_cast<Mbuf **>(outputPptrs[index]),
92 : index, &dynamicSrcDesc, customBuf, customBufSize, taskContext);
93 2 : if (ret != AICPU_SCHEDULE_OK) {
94 1 : return ret;
95 : }
96 : }
97 : }
98 2 : return AICPU_SCHEDULE_OK;
99 : }
100 :
101 8 : int32_t OperatorKernelPostProcessDynamicOutput::AllocatedOutput(const PostprocessDynamicOutputKernelArgs * const param,
102 : Mbuf *const outputMbuf, const uint32_t dynamicFlag,
103 : const size_t index,
104 : RuntimeTensorDesc **dynamicSrcDescPptr,
105 : RuntimeTensorDesc **staticSrcDescPptr) const
106 : {
107 8 : if (outputMbuf == nullptr) {
108 2 : aicpusd_err("the [%zu]th outputMbuf is null, dynamicFlag is %u.", index, dynamicFlag);
109 2 : return AICPU_SCHEDULE_ERROR_PARAMETER_NOT_VALID;
110 : }
111 6 : void *dataPtr = nullptr;
112 6 : const auto dataRet = halMbufGetBuffAddr(outputMbuf, &dataPtr);
113 6 : if (dataRet != static_cast<int32_t>(DRV_ERROR_NONE)) {
114 1 : aicpusd_err("Failed to get data ptr, ret[%d].", dataRet);
115 1 : return AICPU_SCHEDULE_ERROR_FROM_DRV;
116 : }
117 5 : RuntimeTensorDesc *runtimeTensorDesc = nullptr;
118 5 : if (dynamicFlag > 0U) {
119 2 : if (*dynamicSrcDescPptr == nullptr) {
120 2 : const auto tensorRet = GetRuntimeTensor(param, dynamicSrcDescPptr);
121 2 : if (tensorRet != AICPU_SCHEDULE_OK) {
122 0 : return tensorRet;
123 : }
124 : }
125 2 : runtimeTensorDesc = (*dynamicSrcDescPptr)++;
126 2 : const auto setLenRet = PostprocessSetDataLen(runtimeTensorDesc, outputMbuf, index);
127 2 : if (setLenRet != AICPU_SCHEDULE_OK) {
128 1 : return setLenRet;
129 : }
130 : } else {
131 3 : runtimeTensorDesc = (*staticSrcDescPptr)++;
132 : }
133 :
134 4 : const errno_t eRet = memcpy_s(dataPtr, sizeof(RuntimeTensorDesc), runtimeTensorDesc, sizeof(RuntimeTensorDesc));
135 4 : if (eRet != EOK) {
136 1 : aicpusd_err("Data copy failed, ret[%d].", eRet);
137 1 : return AICPU_SCHEDULE_ERROR_SAFE_FUNCTION_ERR;
138 : }
139 3 : return AICPU_SCHEDULE_OK;
140 : }
141 :
142 3 : int32_t OperatorKernelPostProcessDynamicOutput::GetRuntimeTensor(const PostprocessDynamicOutputKernelArgs * const param,
143 : RuntimeTensorDesc **dynamicSrcDescPtr) const
144 : {
145 3 : Mbuf **const respPptrs = reinterpret_cast<Mbuf **>(param->respMsgMbufAddr);
146 3 : if (respPptrs == nullptr) {
147 0 : aicpusd_err("respMsgMbufAddr is null");
148 0 : return AICPU_SCHEDULE_ERROR_PARAMETER_NOT_VALID;
149 : }
150 3 : Mbuf *const respMbuf = *respPptrs;
151 3 : if (respMbuf == nullptr) {
152 0 : aicpusd_err("respMsgMbuf is null");
153 0 : return AICPU_SCHEDULE_ERROR_PARAMETER_NOT_VALID;
154 : }
155 3 : void *dataPtr = nullptr;
156 3 : const auto dataRet = halMbufGetBuffAddr(respMbuf, &dataPtr);
157 3 : if (dataRet != static_cast<int32_t>(DRV_ERROR_NONE)) {
158 0 : aicpusd_err("Failed to get data ptr, ret[%d].", dataRet);
159 0 : return AICPU_SCHEDULE_ERROR_FROM_DRV;
160 : }
161 :
162 3 : *dynamicSrcDescPtr = PtrToPtr<void, RuntimeTensorDesc>(dataPtr);
163 3 : return AICPU_SCHEDULE_OK;
164 : }
165 :
166 2 : int32_t OperatorKernelPostProcessDynamicOutput::PostprocessSetDataLen(const RuntimeTensorDesc * const runtimeTensorDesc,
167 : Mbuf *const outputMbuf, const size_t index) const
168 : {
169 2 : const uint64_t mbufLen = runtimeTensorDesc->dataSize + static_cast<uint64_t>(sizeof(RuntimeTensorDesc));
170 2 : const auto setLenRet = halMbufSetDataLen(outputMbuf, mbufLen);
171 2 : if (setLenRet != static_cast<int32_t>(DRV_ERROR_NONE)) {
172 1 : uint64_t outputMbufLen = 0UL;
173 1 : const auto getSizeRet = halMbufGetBuffSize(outputMbuf, &outputMbufLen);
174 1 : if ((getSizeRet != static_cast<int32_t>(DRV_ERROR_NONE)) ||
175 1 : (outputMbufLen < sizeof(RuntimeTensorDesc))) {
176 0 : aicpusd_err("Fail to get buff size for [%zu]th mbuf, ret=[%d], outputMbufLen[%lu]",
177 : index, getSizeRet, outputMbufLen);
178 : }
179 1 : if (outputMbufLen >= sizeof(RuntimeTensorDesc)) {
180 1 : outputMbufLen -= sizeof(RuntimeTensorDesc);
181 : }
182 1 : aicpusd_err("set [%zu]th mbuf's datalen to %lu fail, mbuf's len is %lu, dataSize is %lu, ret is %d",
183 : index, mbufLen, outputMbufLen, runtimeTensorDesc->dataSize, setLenRet);
184 1 : return AICPU_SCHEDULE_ERROR_FROM_DRV;
185 : }
186 1 : aicpusd_info("set [%zu]th mbuf's datalen to %lu success.", index, mbufLen);
187 1 : return AICPU_SCHEDULE_OK;
188 : }
189 :
190 4 : int32_t OperatorKernelPostProcessDynamicOutput::GetMbufHeadFromResp(Mbuf **const respMbufPtr,
191 : const RunContext &taskContext,
192 : void **const customBufPtr,
193 : uint32_t *const customBufSizePtr) const
194 : {
195 4 : if (*customBufPtr == nullptr) {
196 4 : if ((respMbufPtr == nullptr) || (*respMbufPtr == nullptr)) {
197 1 : aicpusd_err("Invalid response Mbuf for model[%u].", taskContext.modelId);
198 1 : return AICPU_SCHEDULE_ERROR_PARAMETER_NOT_VALID;
199 : }
200 3 : Mbuf *const respMbuf = *respMbufPtr;
201 3 : const auto ret = halMbufGetPrivInfo(respMbuf, customBufPtr, customBufSizePtr);
202 3 : if ((ret != static_cast<int32_t>(DRV_ERROR_NONE)) || (*customBufPtr == nullptr)) {
203 1 : aicpusd_err("Failed to get customBuf from reponse Mbuf for model[%u], ret[%d].",
204 : taskContext.modelId, ret);
205 1 : return AICPU_SCHEDULE_ERROR_FROM_DRV;
206 : }
207 : }
208 2 : return AICPU_SCHEDULE_OK;
209 : }
210 :
211 7 : int32_t OperatorKernelPostProcessDynamicOutput::PostProcessForOutputToAllocate(
212 : const PostprocessDynamicOutputKernelArgs * const param, Mbuf **const outputMbufPtr, const size_t index,
213 : RuntimeTensorDesc **dynamicSrcDescPptr, void *const customBuf, const uint32_t customBufSize,
214 : const RunContext &taskContext) const
215 : {
216 7 : if (*dynamicSrcDescPptr == nullptr) {
217 2 : const auto tensorRet = GetRuntimeTensor(param, dynamicSrcDescPptr);
218 2 : if (tensorRet != AICPU_SCHEDULE_OK) {
219 0 : return tensorRet;
220 : }
221 : }
222 :
223 7 : RuntimeTensorDesc *runtimeTensorDesc = (*dynamicSrcDescPptr)++;
224 7 : if (runtimeTensorDesc == nullptr) {
225 1 : aicpusd_err("the runtimeTensor of [%zu]th output is invalid.", index);
226 1 : return AICPU_SCHEDULE_ERROR_PARAMETER_NOT_VALID;
227 : }
228 :
229 6 : return MakeupDynamicOutputMbuf(outputMbufPtr, index, runtimeTensorDesc, customBuf, customBufSize, taskContext);
230 : }
231 :
232 6 : int32_t OperatorKernelPostProcessDynamicOutput::MakeupDynamicOutputMbuf(Mbuf **const outputMbufPtr, const size_t index,
233 : const RuntimeTensorDesc *const runtimeTensorDesc,
234 : void *const customBuf,
235 : const uint32_t customBufSize,
236 : const RunContext &taskContext) const
237 : {
238 6 : const uint64_t allocSize = runtimeTensorDesc->dataSize + static_cast<uint64_t>(sizeof(RuntimeTensorDesc));
239 6 : Mbuf *mbuf = BufManager::GetInstance().MallocAndGuardBufU64(static_cast<uint64_t>(allocSize),
240 6 : taskContext.modelId);
241 6 : if (mbuf == nullptr) {
242 1 : aicpusd_err("model[%u] alloc mbuf fail, size: %zu.", taskContext.modelId, allocSize);
243 1 : return AICPU_SCHEDULE_ERROR_FROM_DRV;
244 : }
245 :
246 0 : const ScopeGuard mbufGuard([&mbuf, &taskContext]() {
247 5 : if (mbuf != nullptr) {
248 4 : aicpusd_info("Free mbuf in ScopeGuard");
249 4 : const auto modelPtr = AicpuModelManager::GetInstance().GetModel(taskContext.modelId);
250 4 : if (modelPtr == nullptr) {
251 0 : aicpusd_err("cannot get aicpuModel by modelId:[%u]!", taskContext.modelId);
252 : }
253 4 : if (modelPtr != nullptr) {
254 4 : (void) modelPtr->UnGardModelBuf(mbuf);
255 : }
256 4 : (void) halMbufFree(mbuf);
257 : }
258 5 : });
259 :
260 5 : const auto copyRet = OperatorKernelCommon::CopyMbufHeadInfo(customBuf, customBufSize, mbuf);
261 5 : if (copyRet != AICPU_SCHEDULE_OK) {
262 1 : aicpusd_err("model[%u] copy head fail for [%zu]th output.", taskContext.modelId, index);
263 1 : return copyRet;
264 : }
265 :
266 4 : void *dataPtr = nullptr;
267 4 : const auto dataRet = halMbufGetBuffAddr(mbuf, &dataPtr);
268 4 : if (dataRet != static_cast<int32_t>(DRV_ERROR_NONE)) {
269 1 : aicpusd_err("Failed to get data ptr, ret[%d].", dataRet);
270 1 : return AICPU_SCHEDULE_ERROR_FROM_DRV;
271 : }
272 : // copy (runtimeTensorDesc, sizeof(RuntimeTensorDesc)) to mbuf's data
273 3 : bool cpyRet = OptimizedMemCopy(dataPtr, sizeof(RuntimeTensorDesc), runtimeTensorDesc, sizeof(RuntimeTensorDesc));
274 3 : if (!cpyRet) {
275 2 : aicpusd_err("model[%u] copy tensordesc for [%zu]th output failed.", taskContext.modelId, index);
276 2 : return AICPU_SCHEDULE_ERROR_SAFE_FUNCTION_ERR;
277 : }
278 1 : dataPtr = ValueToPtr(PtrToValue(dataPtr) + sizeof(RuntimeTensorDesc));
279 : // copy (dataAddr, dataSize) to mbuf's offset of sizeof(RuntimeTensorDesc)
280 1 : cpyRet = OptimizedMemCopy(dataPtr, static_cast<size_t>(runtimeTensorDesc->dataSize),
281 1 : ValueToPtr(runtimeTensorDesc->dataAddr), static_cast<size_t>(runtimeTensorDesc->dataSize));
282 1 : if (!cpyRet) {
283 0 : aicpusd_err("model[%u] copy data for [%zu]th output failed.", taskContext.modelId, index);
284 0 : return AICPU_SCHEDULE_ERROR_SAFE_FUNCTION_ERR;
285 : }
286 :
287 1 : *outputMbufPtr = mbuf;
288 1 : mbuf = nullptr;
289 1 : return AICPU_SCHEDULE_OK;
290 5 : }
291 :
292 1 : bool OperatorKernelPostProcessDynamicOutput::IsSupportSdmaCopy() const {
293 1 : return (&halSdmaCopy != nullptr);
294 : }
295 :
296 5 : bool OperatorKernelPostProcessDynamicOutput::OptimizedMemCopy(void *const dst_data, const size_t dst_size,
297 : const void *const src_data, const size_t src_size) const {
298 5 : if (IsSupportSdmaCopy()) {
299 1 : const auto ret = halSdmaCopy(reinterpret_cast<DVdeviceptr>(dst_data), dst_size,
300 : reinterpret_cast<DVdeviceptr>(src_data), src_size);
301 1 : if (ret != DRV_ERROR_NONE) {
302 1 : aicpusd_err("Failed to call halSdmaCopy, driver api ret:%d, dst_size:%zu, src_size:%zu.",
303 : static_cast<int32_t>(ret), dst_size, src_size);
304 : }
305 1 : return (ret == DRV_ERROR_NONE);
306 : }
307 4 : return AicpuUtil::BiggerMemCpy(dst_data, dst_size, src_data, src_size);
308 : }
309 :
310 2 : int32_t OperatorKernelPostProcessDynamicOutput::FreeMbuf(const PostprocessDynamicOutputKernelArgs * const param,
311 : const RunContext &taskContext) const
312 : {
313 : // free (respMsgMbufAddr, inputMbufAddrsAddr(uniq)) and unguard them
314 2 : AicpuModel * const modelPtr = AicpuModelManager::GetInstance().GetModel(taskContext.modelId);
315 2 : if (modelPtr == nullptr) {
316 0 : aicpusd_err("cannot get aicpuModel by modelId:[%u]!", taskContext.modelId);
317 0 : return AICPU_SCHEDULE_ERROR_PARAMETER_NOT_VALID;
318 : }
319 2 : Mbuf **const respPptrs = reinterpret_cast<Mbuf **>(param->respMsgMbufAddr);
320 2 : if ((respPptrs != nullptr) && (*respPptrs != nullptr)) {
321 2 : (void)halMbufFree(*respPptrs);
322 2 : (void)modelPtr->UnGardModelBuf(*respPptrs);
323 : }
324 2 : if ((param->inputsNum > 0U) && (param->inputMbufAddrsAddr == 0U)) {
325 0 : aicpusd_err("Invalid inputsNum[%u], inputMbufAddrsAddr[%u]", param->inputsNum,
326 : param->inputMbufAddrsAddr);
327 0 : return AICPU_SCHEDULE_ERROR_PARAMETER_NOT_VALID;
328 : }
329 2 : uint64_t *const inputPptrs = PtrToPtr<void, uint64_t>(ValueToPtr(param->inputMbufAddrsAddr));
330 2 : std::unordered_set<Mbuf*> freedInput;
331 4 : for (size_t index = 0U; index < param->inputsNum; ++index) {
332 2 : Mbuf *const inputMbuf = *(reinterpret_cast<Mbuf **>(inputPptrs[index]));
333 2 : if (freedInput.count(inputMbuf) != 0U) {
334 0 : continue;
335 : }
336 2 : (void)halMbufFree(inputMbuf);
337 2 : (void)modelPtr->UnGardModelBuf(inputMbuf);
338 2 : freedInput.insert(inputMbuf);
339 : }
340 2 : return AICPU_SCHEDULE_OK;
341 2 : }
342 :
343 :
344 6 : REGISTER_OPERATOR_KERNEL(KERNEL_POST_PROCESS_DYNAMIC_OUTPUT, OperatorKernelPostProcessDynamicOutput);
345 6 : REGISTER_OPERATOR_KERNEL(KERNEL_POST_PROCESS_DYNAMIC_OUTPUT_V2, OperatorKernelPostProcessDynamicOutput);
346 : } // namespace AicpuSchedule
|
