Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "operator_kernel_post_process_dynamic_output.h"
12 :
13 : #include "aicpusd_status.h"
14 : #include "aicpusd_model_execute.h"
15 : #include "aicpusd_resource_manager.h"
16 : #include "operator_kernel_common.h"
17 :
18 : namespace AicpuSchedule {
19 : namespace {
20 : const std::string KERNEL_POST_PROCESS_DYNAMIC_OUTPUT = "postprocessDynamicOutput";
21 : const std::string KERNEL_POST_PROCESS_DYNAMIC_OUTPUT_V2 = "postprocessDynamicOutputV2";
22 :
23 : enum class OutputType { STATIC_OUTPUT = 0, DYNAMIC_OUTPUT_WITH_MAXSIZE = 1, DYNAMIC_OUTPUT_WITHOUT_MAXSIZE = 2 };
24 : } // namespace
25 :
26 1 : int32_t OperatorKernelPostProcessDynamicOutput::Compute(
27 : const AicpuTaskInfo& kernelTaskInfo, const RunContext& taskContext)
28 : {
29 1 : aicpusd_info(
30 : "Start ModelPostprocessDynamicOutput. modelId=%u, streamId=%u, taskId=%u.", taskContext.modelId,
31 : kernelTaskInfo.streamID, kernelTaskInfo.taskID);
32 1 : if (kernelTaskInfo.paraBase == 0UL) {
33 0 : aicpusd_err("kernelTaskInfo.paraBase is null");
34 0 : return AICPU_SCHEDULE_ERROR_PARAMETER_NOT_VALID;
35 : }
36 :
37 : const PostprocessDynamicOutputKernelArgs* const param =
38 1 : PtrToPtr<void, PostprocessDynamicOutputKernelArgs>(ValueToPtr(kernelTaskInfo.paraBase));
39 1 : if ((param->outputsNum > 0U) && ((param->outputDynamicFlagsAddr == 0U) || (param->outputMbufAddrsAddr == 0U))) {
40 0 : aicpusd_err(
41 : "Parameter invalid: outputsNum[%u], outputDynamicFlagsAddr[%u], outputMbufAddrsAddr[%u]", param->outputsNum,
42 : param->outputDynamicFlagsAddr, param->outputMbufAddrsAddr);
43 0 : return AICPU_SCHEDULE_ERROR_PARAMETER_NOT_VALID;
44 : }
45 :
46 1 : const auto copyRet = CopyTensorDesc(param, taskContext);
47 1 : if (copyRet != AICPU_SCHEDULE_OK) {
48 0 : return copyRet;
49 : }
50 :
51 1 : return FreeMbuf(param, taskContext);
52 : }
53 :
54 8 : int32_t OperatorKernelPostProcessDynamicOutput::CopyTensorDesc(
55 : const PostprocessDynamicOutputKernelArgs* const param, const RunContext& taskContext) const
56 : {
57 8 : const uint32_t* const dynamicFlags = PtrToPtr<void, uint32_t>(ValueToPtr(param->outputDynamicFlagsAddr));
58 8 : uint64_t* const outputPptrs = PtrToPtr<void, uint64_t>(ValueToPtr(param->outputMbufAddrsAddr));
59 8 : RuntimeTensorDesc* dynamicSrcDesc = nullptr;
60 8 : RuntimeTensorDesc* staticSrcDesc = PtrToPtr<void, RuntimeTensorDesc>(ValueToPtr(param->outputStaticTensorDescAddr));
61 :
62 8 : void* customBuf = nullptr;
63 8 : uint32_t customBufSize = 0U;
64 12 : for (size_t index = 0U; index < param->outputsNum; ++index) {
65 10 : aicpusd_info("dynamicFlags[%zu] is %u.", index, dynamicFlags[index]);
66 10 : if (outputPptrs[index] == 0U) {
67 1 : aicpusd_err("the [%zu]th outputMbufPtr is null", index);
68 1 : return AICPU_SCHEDULE_ERROR_PARAMETER_NOT_VALID;
69 : }
70 :
71 9 : if ((dynamicFlags[index] == static_cast<uint32_t>(OutputType::STATIC_OUTPUT)) ||
72 7 : (dynamicFlags[index] == static_cast<uint32_t>(OutputType::DYNAMIC_OUTPUT_WITH_MAXSIZE))) {
73 5 : Mbuf* const outputMbuf = *(reinterpret_cast<Mbuf**>(outputPptrs[index]));
74 : const auto ret =
75 5 : AllocatedOutput(param, outputMbuf, dynamicFlags[index], index, &dynamicSrcDesc, &staticSrcDesc);
76 5 : if (ret != AICPU_SCHEDULE_OK) {
77 2 : return ret;
78 : }
79 3 : continue;
80 3 : }
81 :
82 4 : if (dynamicFlags[index] == static_cast<uint32_t>(OutputType::DYNAMIC_OUTPUT_WITHOUT_MAXSIZE)) {
83 8 : const auto getHeadRet = GetMbufHeadFromResp(
84 4 : reinterpret_cast<Mbuf**>(param->respMsgMbufAddr), taskContext, &customBuf, &customBufSize);
85 4 : if (getHeadRet != AICPU_SCHEDULE_OK) {
86 2 : return getHeadRet;
87 : }
88 4 : const auto ret = PostProcessForOutputToAllocate(
89 2 : param, reinterpret_cast<Mbuf**>(outputPptrs[index]), index, &dynamicSrcDesc, customBuf, customBufSize,
90 : taskContext);
91 2 : if (ret != AICPU_SCHEDULE_OK) {
92 1 : return ret;
93 : }
94 : }
95 : }
96 2 : return AICPU_SCHEDULE_OK;
97 : }
98 :
99 8 : int32_t OperatorKernelPostProcessDynamicOutput::AllocatedOutput(
100 : const PostprocessDynamicOutputKernelArgs* const param, Mbuf* const outputMbuf, const uint32_t dynamicFlag,
101 : const size_t index, RuntimeTensorDesc** dynamicSrcDescPptr, RuntimeTensorDesc** staticSrcDescPptr) const
102 : {
103 8 : if (outputMbuf == nullptr) {
104 2 : aicpusd_err("the [%zu]th outputMbuf is null, dynamicFlag is %u.", index, dynamicFlag);
105 2 : return AICPU_SCHEDULE_ERROR_PARAMETER_NOT_VALID;
106 : }
107 6 : void* dataPtr = nullptr;
108 6 : const auto dataRet = halMbufGetBuffAddr(outputMbuf, &dataPtr);
109 6 : if (dataRet != static_cast<int32_t>(DRV_ERROR_NONE)) {
110 1 : aicpusd_err("Failed to get data ptr, ret[%d].", dataRet);
111 1 : return AICPU_SCHEDULE_ERROR_FROM_DRV;
112 : }
113 5 : RuntimeTensorDesc* runtimeTensorDesc = nullptr;
114 5 : if (dynamicFlag > 0U) {
115 2 : if (*dynamicSrcDescPptr == nullptr) {
116 2 : const auto tensorRet = GetRuntimeTensor(param, dynamicSrcDescPptr);
117 2 : if (tensorRet != AICPU_SCHEDULE_OK) {
118 0 : return tensorRet;
119 : }
120 : }
121 2 : runtimeTensorDesc = (*dynamicSrcDescPptr)++;
122 2 : const auto setLenRet = PostprocessSetDataLen(runtimeTensorDesc, outputMbuf, index);
123 2 : if (setLenRet != AICPU_SCHEDULE_OK) {
124 1 : return setLenRet;
125 : }
126 : } else {
127 3 : runtimeTensorDesc = (*staticSrcDescPptr)++;
128 : }
129 :
130 4 : const errno_t eRet = memcpy_s(dataPtr, sizeof(RuntimeTensorDesc), runtimeTensorDesc, sizeof(RuntimeTensorDesc));
131 4 : if (eRet != EOK) {
132 1 : aicpusd_err("Data copy failed, ret[%d].", eRet);
133 1 : return AICPU_SCHEDULE_ERROR_SAFE_FUNCTION_ERR;
134 : }
135 3 : return AICPU_SCHEDULE_OK;
136 : }
137 :
138 3 : int32_t OperatorKernelPostProcessDynamicOutput::GetRuntimeTensor(
139 : const PostprocessDynamicOutputKernelArgs* const param, RuntimeTensorDesc** dynamicSrcDescPtr) const
140 : {
141 3 : Mbuf** const respPptrs = reinterpret_cast<Mbuf**>(param->respMsgMbufAddr);
142 3 : if (respPptrs == nullptr) {
143 0 : aicpusd_err("respMsgMbufAddr is null");
144 0 : return AICPU_SCHEDULE_ERROR_PARAMETER_NOT_VALID;
145 : }
146 3 : Mbuf* const respMbuf = *respPptrs;
147 3 : if (respMbuf == nullptr) {
148 0 : aicpusd_err("respMsgMbuf is null");
149 0 : return AICPU_SCHEDULE_ERROR_PARAMETER_NOT_VALID;
150 : }
151 3 : void* dataPtr = nullptr;
152 3 : const auto dataRet = halMbufGetBuffAddr(respMbuf, &dataPtr);
153 3 : if (dataRet != static_cast<int32_t>(DRV_ERROR_NONE)) {
154 0 : aicpusd_err("Failed to get data ptr, ret[%d].", dataRet);
155 0 : return AICPU_SCHEDULE_ERROR_FROM_DRV;
156 : }
157 :
158 3 : *dynamicSrcDescPtr = PtrToPtr<void, RuntimeTensorDesc>(dataPtr);
159 3 : return AICPU_SCHEDULE_OK;
160 : }
161 :
162 2 : int32_t OperatorKernelPostProcessDynamicOutput::PostprocessSetDataLen(
163 : const RuntimeTensorDesc* const runtimeTensorDesc, Mbuf* const outputMbuf, const size_t index) const
164 : {
165 2 : const uint64_t mbufLen = runtimeTensorDesc->dataSize + static_cast<uint64_t>(sizeof(RuntimeTensorDesc));
166 2 : const auto setLenRet = halMbufSetDataLen(outputMbuf, mbufLen);
167 2 : if (setLenRet != static_cast<int32_t>(DRV_ERROR_NONE)) {
168 1 : uint64_t outputMbufLen = 0UL;
169 1 : const auto getSizeRet = halMbufGetBuffSize(outputMbuf, &outputMbufLen);
170 1 : if ((getSizeRet != static_cast<int32_t>(DRV_ERROR_NONE)) || (outputMbufLen < sizeof(RuntimeTensorDesc))) {
171 0 : aicpusd_err(
172 : "Fail to get buff size for [%zu]th mbuf, ret=[%d], outputMbufLen[%lu]", index, getSizeRet,
173 : outputMbufLen);
174 : }
175 1 : if (outputMbufLen >= sizeof(RuntimeTensorDesc)) {
176 1 : outputMbufLen -= sizeof(RuntimeTensorDesc);
177 : }
178 1 : aicpusd_err(
179 : "set [%zu]th mbuf's datalen to %lu fail, mbuf's len is %lu, dataSize is %lu, ret is %d", index, mbufLen,
180 : outputMbufLen, runtimeTensorDesc->dataSize, setLenRet);
181 1 : return AICPU_SCHEDULE_ERROR_FROM_DRV;
182 : }
183 1 : aicpusd_info("set [%zu]th mbuf's datalen to %lu success.", index, mbufLen);
184 1 : return AICPU_SCHEDULE_OK;
185 : }
186 :
187 4 : int32_t OperatorKernelPostProcessDynamicOutput::GetMbufHeadFromResp(
188 : Mbuf** const respMbufPtr, const RunContext& taskContext, void** const customBufPtr,
189 : uint32_t* const customBufSizePtr) const
190 : {
191 4 : if (*customBufPtr == nullptr) {
192 4 : if ((respMbufPtr == nullptr) || (*respMbufPtr == nullptr)) {
193 1 : aicpusd_err("Invalid response Mbuf for model[%u].", taskContext.modelId);
194 1 : return AICPU_SCHEDULE_ERROR_PARAMETER_NOT_VALID;
195 : }
196 3 : Mbuf* const respMbuf = *respMbufPtr;
197 3 : const auto ret = halMbufGetPrivInfo(respMbuf, customBufPtr, customBufSizePtr);
198 3 : if ((ret != static_cast<int32_t>(DRV_ERROR_NONE)) || (*customBufPtr == nullptr)) {
199 1 : aicpusd_err("Failed to get customBuf from reponse Mbuf for model[%u], ret[%d].", taskContext.modelId, ret);
200 1 : return AICPU_SCHEDULE_ERROR_FROM_DRV;
201 : }
202 : }
203 2 : return AICPU_SCHEDULE_OK;
204 : }
205 :
206 7 : int32_t OperatorKernelPostProcessDynamicOutput::PostProcessForOutputToAllocate(
207 : const PostprocessDynamicOutputKernelArgs* const param, Mbuf** const outputMbufPtr, const size_t index,
208 : RuntimeTensorDesc** dynamicSrcDescPptr, void* const customBuf, const uint32_t customBufSize,
209 : const RunContext& taskContext) const
210 : {
211 7 : if (*dynamicSrcDescPptr == nullptr) {
212 2 : const auto tensorRet = GetRuntimeTensor(param, dynamicSrcDescPptr);
213 2 : if (tensorRet != AICPU_SCHEDULE_OK) {
214 0 : return tensorRet;
215 : }
216 : }
217 :
218 7 : RuntimeTensorDesc* runtimeTensorDesc = (*dynamicSrcDescPptr)++;
219 7 : if (runtimeTensorDesc == nullptr) {
220 1 : aicpusd_err("the runtimeTensor of [%zu]th output is invalid.", index);
221 1 : return AICPU_SCHEDULE_ERROR_PARAMETER_NOT_VALID;
222 : }
223 :
224 6 : return MakeupDynamicOutputMbuf(outputMbufPtr, index, runtimeTensorDesc, customBuf, customBufSize, taskContext);
225 : }
226 :
227 6 : int32_t OperatorKernelPostProcessDynamicOutput::MakeupDynamicOutputMbuf(
228 : Mbuf** const outputMbufPtr, const size_t index, const RuntimeTensorDesc* const runtimeTensorDesc,
229 : void* const customBuf, const uint32_t customBufSize, const RunContext& taskContext) const
230 : {
231 6 : const uint64_t allocSize = runtimeTensorDesc->dataSize + static_cast<uint64_t>(sizeof(RuntimeTensorDesc));
232 6 : Mbuf* mbuf = BufManager::GetInstance().MallocAndGuardBufU64(static_cast<uint64_t>(allocSize), taskContext.modelId);
233 6 : if (mbuf == nullptr) {
234 1 : aicpusd_err("model[%u] alloc mbuf fail, size: %zu.", taskContext.modelId, allocSize);
235 1 : return AICPU_SCHEDULE_ERROR_FROM_DRV;
236 : }
237 :
238 0 : const ScopeGuard mbufGuard([&mbuf, &taskContext]() {
239 5 : if (mbuf != nullptr) {
240 4 : aicpusd_info("Free mbuf in ScopeGuard");
241 4 : const auto modelPtr = AicpuModelManager::GetInstance().GetModel(taskContext.modelId);
242 4 : if (modelPtr == nullptr) {
243 0 : aicpusd_err("cannot get aicpuModel by modelId:[%u]!", taskContext.modelId);
244 : }
245 4 : if (modelPtr != nullptr) {
246 4 : (void)modelPtr->UnGardModelBuf(mbuf);
247 : }
248 4 : (void)halMbufFree(mbuf);
249 : }
250 5 : });
251 :
252 5 : const auto copyRet = OperatorKernelCommon::CopyMbufHeadInfo(customBuf, customBufSize, mbuf);
253 5 : if (copyRet != AICPU_SCHEDULE_OK) {
254 1 : aicpusd_err("model[%u] copy head fail for [%zu]th output.", taskContext.modelId, index);
255 1 : return copyRet;
256 : }
257 :
258 4 : void* dataPtr = nullptr;
259 4 : const auto dataRet = halMbufGetBuffAddr(mbuf, &dataPtr);
260 4 : if (dataRet != static_cast<int32_t>(DRV_ERROR_NONE)) {
261 1 : aicpusd_err("Failed to get data ptr, ret[%d].", dataRet);
262 1 : return AICPU_SCHEDULE_ERROR_FROM_DRV;
263 : }
264 : // copy (runtimeTensorDesc, sizeof(RuntimeTensorDesc)) to mbuf's data
265 3 : bool cpyRet = OptimizedMemCopy(dataPtr, sizeof(RuntimeTensorDesc), runtimeTensorDesc, sizeof(RuntimeTensorDesc));
266 3 : if (!cpyRet) {
267 2 : aicpusd_err("model[%u] copy tensordesc for [%zu]th output failed.", taskContext.modelId, index);
268 2 : return AICPU_SCHEDULE_ERROR_SAFE_FUNCTION_ERR;
269 : }
270 1 : dataPtr = ValueToPtr(PtrToValue(dataPtr) + sizeof(RuntimeTensorDesc));
271 : // copy (dataAddr, dataSize) to mbuf's offset of sizeof(RuntimeTensorDesc)
272 2 : cpyRet = OptimizedMemCopy(
273 1 : dataPtr, static_cast<size_t>(runtimeTensorDesc->dataSize), ValueToPtr(runtimeTensorDesc->dataAddr),
274 1 : static_cast<size_t>(runtimeTensorDesc->dataSize));
275 1 : if (!cpyRet) {
276 0 : aicpusd_err("model[%u] copy data for [%zu]th output failed.", taskContext.modelId, index);
277 0 : return AICPU_SCHEDULE_ERROR_SAFE_FUNCTION_ERR;
278 : }
279 :
280 1 : *outputMbufPtr = mbuf;
281 1 : mbuf = nullptr;
282 1 : return AICPU_SCHEDULE_OK;
283 5 : }
284 :
285 1 : bool OperatorKernelPostProcessDynamicOutput::IsSupportSdmaCopy() const { return (&halSdmaCopy != nullptr); }
286 :
287 5 : bool OperatorKernelPostProcessDynamicOutput::OptimizedMemCopy(
288 : void* const dst_data, const size_t dst_size, const void* const src_data, const size_t src_size) const
289 : {
290 5 : if (IsSupportSdmaCopy()) {
291 1 : const auto ret = halSdmaCopy(
292 : reinterpret_cast<DVdeviceptr>(dst_data), dst_size, reinterpret_cast<DVdeviceptr>(src_data), src_size);
293 1 : if (ret != DRV_ERROR_NONE) {
294 1 : aicpusd_err(
295 : "Failed to call halSdmaCopy, driver api ret:%d, dst_size:%zu, src_size:%zu.", static_cast<int32_t>(ret),
296 : dst_size, src_size);
297 : }
298 1 : return (ret == DRV_ERROR_NONE);
299 : }
300 4 : return AicpuUtil::BiggerMemCpy(dst_data, dst_size, src_data, src_size);
301 : }
302 :
303 2 : int32_t OperatorKernelPostProcessDynamicOutput::FreeMbuf(
304 : const PostprocessDynamicOutputKernelArgs* const param, const RunContext& taskContext) const
305 : {
306 : // free (respMsgMbufAddr, inputMbufAddrsAddr(uniq)) and unguard them
307 2 : AicpuModel* const modelPtr = AicpuModelManager::GetInstance().GetModel(taskContext.modelId);
308 2 : if (modelPtr == nullptr) {
309 0 : aicpusd_err("cannot get aicpuModel by modelId:[%u]!", taskContext.modelId);
310 0 : return AICPU_SCHEDULE_ERROR_PARAMETER_NOT_VALID;
311 : }
312 2 : Mbuf** const respPptrs = reinterpret_cast<Mbuf**>(param->respMsgMbufAddr);
313 2 : if ((respPptrs != nullptr) && (*respPptrs != nullptr)) {
314 2 : (void)halMbufFree(*respPptrs);
315 2 : (void)modelPtr->UnGardModelBuf(*respPptrs);
316 : }
317 2 : if ((param->inputsNum > 0U) && (param->inputMbufAddrsAddr == 0U)) {
318 0 : aicpusd_err("Invalid inputsNum[%u], inputMbufAddrsAddr[%u]", param->inputsNum, param->inputMbufAddrsAddr);
319 0 : return AICPU_SCHEDULE_ERROR_PARAMETER_NOT_VALID;
320 : }
321 2 : uint64_t* const inputPptrs = PtrToPtr<void, uint64_t>(ValueToPtr(param->inputMbufAddrsAddr));
322 2 : std::unordered_set<Mbuf*> freedInput;
323 4 : for (size_t index = 0U; index < param->inputsNum; ++index) {
324 2 : Mbuf* const inputMbuf = *(reinterpret_cast<Mbuf**>(inputPptrs[index]));
325 2 : if (freedInput.count(inputMbuf) != 0U) {
326 0 : continue;
327 : }
328 2 : (void)halMbufFree(inputMbuf);
329 2 : (void)modelPtr->UnGardModelBuf(inputMbuf);
330 2 : freedInput.insert(inputMbuf);
331 : }
332 2 : return AICPU_SCHEDULE_OK;
333 2 : }
334 :
335 6 : REGISTER_OPERATOR_KERNEL(KERNEL_POST_PROCESS_DYNAMIC_OUTPUT, OperatorKernelPostProcessDynamicOutput);
336 6 : REGISTER_OPERATOR_KERNEL(KERNEL_POST_PROCESS_DYNAMIC_OUTPUT_V2, OperatorKernelPostProcessDynamicOutput);
337 : } // namespace AicpuSchedule
|
