Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "operator_kernel_model_prepare_output.h"
12 :
13 : #include "aicpusd_status.h"
14 : #include "aicpusd_monitor.h"
15 : #include "aicpusd_profiler.h"
16 : #include "aicpusd_resource_manager.h"
17 : #include "aicpusd_model_execute.h"
18 : #include "aicpusd_model_statistic.h"
19 : #include "operator_kernel_common.h"
20 :
21 :
22 : namespace AicpuSchedule {
23 : namespace {
24 : const std::string KERNEL_MODEL_PREPARE_OUTPUT = "modelPrepareOutput";
25 : const std::string KERNEL_MODEL_PREPARE_OUTPUT_WITH_TENSOR_DESC = "modelPrepareOutputWithTensorDesc";
26 : const std::string KERNEL_BUFFER_PREPARE_OUTPUT = "bufferPrepareOutput";
27 : const std::string KERNEL_BUFFER_PREPARE_OUTPUT_WITH_TENSOR_DESC = "bufferPrepareOutputWithTensorDesc";
28 : } // namespace
29 :
30 8 : int32_t OperatorKernelPrepareOutputBase::PrepareOutput(ProcessOutputInfo &outputInfo, const RunContext &taskContext,
31 : const bool zeroCpy, RuntimeTensorDesc *const tensorDesc) const
32 : {
33 : // point to Mbuf
34 8 : auto inMBuf = reinterpret_cast<Mbuf **>(static_cast<uintptr_t>(outputInfo.inMBuf));
35 8 : auto outMBuf = reinterpret_cast<Mbuf **>(static_cast<uintptr_t>(outputInfo.outMBuf));
36 8 : const bool inOrOutMbufIsNull = ((inMBuf == nullptr) || (outMBuf == nullptr));
37 8 : if (inOrOutMbufIsNull) {
38 1 : aicpusd_err("PrepareOutput param inMBuf or outMBuf is null, inMBuf[%llx], outMbuf[%llx].", outputInfo.inMBuf,
39 : outputInfo.outMBuf);
40 1 : return AICPU_SCHEDULE_ERROR_PARAMETER_NOT_VALID;
41 : }
42 7 : g_aicpuProfiler.SetPrepareOutStart();
43 7 : if (tensorDesc != nullptr) {
44 1 : if ((UINT32_MAX - static_cast<uint32_t>(sizeof(RuntimeTensorDesc))) < outputInfo.dataSize) {
45 0 : aicpusd_err("PrepareOutput param is invalid, output data size[%u] + sizeof(RuntimeTensorDesc)[%zu] "
46 : "will overflow.", outputInfo.dataSize, sizeof(RuntimeTensorDesc));
47 0 : return AICPU_SCHEDULE_ERROR_PARAMETER_NOT_VALID;
48 : }
49 1 : *outMBuf = BufManager::GetInstance().MallocAndGuardBuf(
50 1 : outputInfo.dataSize + static_cast<uint32_t>(sizeof(RuntimeTensorDesc)), taskContext.modelId);
51 2 : AicpuSdModelStatistic::GetInstance().StatNNModelOutput(taskContext.modelId, tensorDesc,
52 1 : GetStaticNNOutPutIndex(taskContext.modelId));
53 : } else {
54 6 : *outMBuf = BufManager::GetInstance().MallocAndGuardBuf(outputInfo.dataSize, taskContext.modelId);
55 : }
56 :
57 7 : if (*outMBuf == nullptr) {
58 3 : aicpusd_err("Failed to alloc mbuf, dataSize[%u], modelId[%u].", outputInfo.dataSize, taskContext.modelId);
59 3 : AicpuMonitor::GetInstance().SendKillMsgToTsd();
60 3 : return AICPU_SCHEDULE_ERROR_FROM_DRV;
61 : }
62 4 : if (tensorDesc != nullptr) {
63 1 : void *basePtr = nullptr;
64 1 : const auto ret = halMbufGetBuffAddr(*outMBuf, &basePtr);
65 1 : if ((ret != DRV_ERROR_NONE) || (basePtr == nullptr)) {
66 0 : aicpusd_err("Failed to call halMbufGetBuffAddr, ret[%d].", ret);
67 0 : return AICPU_SCHEDULE_ERROR_DRV_ERR;
68 : }
69 :
70 1 : tensorDesc->dataAddr = PtrToValue(basePtr) + static_cast<uint64_t>(sizeof(RuntimeTensorDesc));
71 1 : const errno_t eRet = memcpy_s(basePtr, sizeof(RuntimeTensorDesc), tensorDesc, sizeof(RuntimeTensorDesc));
72 1 : if (eRet != EOK) {
73 0 : return AICPU_SCHEDULE_ERROR_SAFE_FUNCTION_ERR;
74 : }
75 : }
76 4 : g_aicpuProfiler.SetPrepareOutEnd();
77 4 : if (!zeroCpy) {
78 2 : const auto ret = PrepareOutputNonZeroCpy(outputInfo, *outMBuf, tensorDesc);
79 2 : if (ret != AICPU_SCHEDULE_OK) {
80 0 : return ret;
81 : }
82 : }
83 :
84 4 : void *inputHeaderBuf = nullptr;
85 4 : uint32_t inputHeadSize = 0U;
86 4 : const auto drvRet = halMbufGetPrivInfo(*inMBuf, &inputHeaderBuf, &inputHeadSize);
87 4 : if (drvRet != DRV_ERROR_NONE) {
88 1 : aicpusd_err("Failed to get head info in input information, ret[%d].", drvRet);
89 1 : return AICPU_SCHEDULE_ERROR_FROM_DRV;
90 : }
91 3 : const auto ret = OperatorKernelCommon::CopyMbufHeadInfo(inputHeaderBuf, inputHeadSize, *outMBuf);
92 3 : if (ret != AICPU_SCHEDULE_OK) {
93 0 : return ret;
94 : }
95 :
96 3 : return AICPU_SCHEDULE_OK;
97 : }
98 :
99 3 : int32_t OperatorKernelPrepareOutputBase::PrepareOutputNonZeroCpy(const ProcessOutputInfo &outputInfo,
100 : Mbuf * const outMBuf,
101 : RuntimeTensorDesc *const tensorDesc) const
102 : {
103 3 : void *dataPtr = nullptr;
104 3 : const auto ret = halMbufGetBuffAddr(outMBuf, &dataPtr);
105 3 : if ((ret != DRV_ERROR_NONE) || (dataPtr == nullptr)) {
106 1 : aicpusd_err("Failed to get data or data is nullptr, ret[%d].", ret);
107 1 : return AICPU_SCHEDULE_ERROR_DRV_ERR;
108 : }
109 :
110 2 : if (tensorDesc != nullptr) {
111 1 : dataPtr = ValueToPtr(PtrToValue(dataPtr) + sizeof(RuntimeTensorDesc));
112 : }
113 :
114 2 : const errno_t eRet = memcpy_s(dataPtr, static_cast<size_t>(outputInfo.dataSize),
115 2 : ValueToPtr(outputInfo.srcPtr),
116 2 : static_cast<size_t>(outputInfo.dataSize));
117 2 : if (eRet != EOK) {
118 0 : aicpusd_err("Failed to memcpy, ret[%d].", eRet);
119 0 : return AICPU_SCHEDULE_ERROR_SAFE_FUNCTION_ERR;
120 : }
121 2 : return AICPU_SCHEDULE_OK;
122 : }
123 :
124 1 : int32_t OperatorKernelPrepareOutputBase::GetStaticNNOutPutIndex(const uint32_t modelId) const
125 : {
126 1 : const auto model = AicpuModelManager::GetInstance().GetModel(modelId);
127 1 : if (model == nullptr) {
128 1 : aicpusd_err("model:%u is null", modelId);
129 1 : return -1;
130 : }
131 0 : return (static_cast<int32_t>(model->GetCurStaticNNModelOutputIndex()));
132 : }
133 :
134 3 : void OperatorKernelPrepareOutputBase::MarkStaticNNOutPutIndex(const uint32_t modelId) const
135 : {
136 3 : const auto model = AicpuModelManager::GetInstance().GetModel(modelId);
137 3 : if (model == nullptr) {
138 3 : aicpusd_err("model:%u is null", modelId);
139 3 : return;
140 : }
141 0 : model->IncreaseStaticNNModelOutputIndex();
142 0 : return;
143 : }
144 :
145 3 : int32_t OperatorKernelPrepareOutputBase::PrepareOutWithTensorDesc(const AicpuTaskInfo &kernelTaskInfo,
146 : const bool zeroCpy,
147 : const RunContext &taskContext) const
148 : {
149 3 : MarkStaticNNOutPutIndex(taskContext.modelId);
150 3 : ProcessOutputInfo * const info = PtrToPtr<void, ProcessOutputInfo>(ValueToPtr(kernelTaskInfo.paraBase));
151 3 : if (info == nullptr) {
152 1 : aicpusd_err("AicpuTaskInfo.paramBase is null, modelId[%u], streamId[%u], taskId[%u]",
153 : taskContext.modelId, taskContext.streamId, kernelTaskInfo.taskID);
154 1 : return AICPU_SCHEDULE_ERROR_PARAMETER_NOT_VALID;
155 : }
156 2 : if ((UINT64_MAX - static_cast<uint64_t>(sizeof(ProcessOutputInfo))) < kernelTaskInfo.paraBase) {
157 1 : aicpusd_err("AicpuTaskInfo.paramBase[%lu] + sizeof(ProcessOutputInfo)[%zu] will overflow, modelId[%u], "
158 : "streamId[%u], taskId[%u]", kernelTaskInfo.paraBase, sizeof(ProcessOutputInfo), taskContext.modelId,
159 : taskContext.streamId, kernelTaskInfo.taskID);
160 1 : return AICPU_SCHEDULE_ERROR_PARAMETER_NOT_VALID;
161 : }
162 1 : RuntimeTensorDesc *const tensorDesc = PtrToPtr<void, RuntimeTensorDesc>(
163 1 : ValueToPtr(kernelTaskInfo.paraBase + static_cast<uint64_t>(sizeof(ProcessOutputInfo))));
164 1 : return PrepareOutput(*info, taskContext, zeroCpy, tensorDesc);
165 : }
166 :
167 3 : int32_t OperatorKernelModelPrepareOutput::Compute(const AicpuTaskInfo &kernelTaskInfo, const RunContext &taskContext)
168 : {
169 3 : ProcessOutputInfo * const info = reinterpret_cast<ProcessOutputInfo *>(
170 3 : static_cast<uintptr_t>(kernelTaskInfo.paraBase));
171 3 : if (info == nullptr) {
172 1 : aicpusd_err("ModelPrepareOut kernelTaskInfo paramBase is null, modelId[%u], streamId[%u], taskId[%u]",
173 : taskContext.modelId, taskContext.streamId, kernelTaskInfo.taskID);
174 1 : return AICPU_SCHEDULE_ERROR_PARAMETER_NOT_VALID;
175 : }
176 2 : return PrepareOutput(*info, taskContext, false, nullptr);
177 : }
178 :
179 2 : int32_t OperatorKernelModelPrepareOutputWithTensorDesc::Compute(const AicpuTaskInfo &kernelTaskInfo,
180 : const RunContext &taskContext)
181 : {
182 2 : return PrepareOutWithTensorDesc(kernelTaskInfo, false, taskContext);
183 : }
184 :
185 1 : int32_t OperatorKernelBufferPrepareOutput::Compute(const AicpuTaskInfo &kernelTaskInfo, const RunContext &taskContext)
186 : {
187 1 : ProcessOutputInfo * const info = PtrToPtr<void, ProcessOutputInfo>(
188 1 : ValueToPtr(kernelTaskInfo.paraBase));
189 1 : if (info == nullptr) {
190 0 : aicpusd_err("ModelPrepareOut kernelTaskInfo paramBase is null, modelId[%u], streamId[%u], taskId[%u]",
191 : taskContext.modelId, taskContext.streamId, kernelTaskInfo.taskID);
192 0 : return AICPU_SCHEDULE_ERROR_PARAMETER_NOT_VALID;
193 : }
194 1 : return PrepareOutput(*info, taskContext, true, nullptr);
195 : }
196 :
197 1 : int32_t OperatorKernelBufferPrepareOutputWithTensorDesc::Compute(const AicpuTaskInfo &kernelTaskInfo,
198 : const RunContext &taskContext)
199 : {
200 1 : return PrepareOutWithTensorDesc(kernelTaskInfo, true, taskContext);
201 : }
202 :
203 6 : REGISTER_OPERATOR_KERNEL(KERNEL_MODEL_PREPARE_OUTPUT, OperatorKernelModelPrepareOutput);
204 6 : REGISTER_OPERATOR_KERNEL(KERNEL_MODEL_PREPARE_OUTPUT_WITH_TENSOR_DESC, OperatorKernelModelPrepareOutputWithTensorDesc);
205 6 : REGISTER_OPERATOR_KERNEL(KERNEL_BUFFER_PREPARE_OUTPUT, OperatorKernelBufferPrepareOutput);
206 6 : REGISTER_OPERATOR_KERNEL(KERNEL_BUFFER_PREPARE_OUTPUT_WITH_TENSOR_DESC, OperatorKernelBufferPrepareOutputWithTensorDesc);
207 : } // namespace AicpuSchedule
|
