Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "operator_kernel_model_prepare_output.h"
12 :
13 : #include "aicpusd_status.h"
14 : #include "aicpusd_monitor.h"
15 : #include "aicpusd_profiler.h"
16 : #include "aicpusd_resource_manager.h"
17 : #include "aicpusd_model_execute.h"
18 : #include "aicpusd_model_statistic.h"
19 : #include "operator_kernel_common.h"
20 :
21 : namespace AicpuSchedule {
22 : namespace {
23 : const std::string KERNEL_MODEL_PREPARE_OUTPUT = "modelPrepareOutput";
24 : const std::string KERNEL_MODEL_PREPARE_OUTPUT_WITH_TENSOR_DESC = "modelPrepareOutputWithTensorDesc";
25 : const std::string KERNEL_BUFFER_PREPARE_OUTPUT = "bufferPrepareOutput";
26 : const std::string KERNEL_BUFFER_PREPARE_OUTPUT_WITH_TENSOR_DESC = "bufferPrepareOutputWithTensorDesc";
27 : } // namespace
28 :
29 8 : int32_t OperatorKernelPrepareOutputBase::PrepareOutput(
30 : ProcessOutputInfo& outputInfo, const RunContext& taskContext, const bool zeroCpy,
31 : RuntimeTensorDesc* const tensorDesc) const
32 : {
33 : // point to Mbuf
34 8 : auto inMBuf = reinterpret_cast<Mbuf**>(static_cast<uintptr_t>(outputInfo.inMBuf));
35 8 : auto outMBuf = reinterpret_cast<Mbuf**>(static_cast<uintptr_t>(outputInfo.outMBuf));
36 8 : const bool inOrOutMbufIsNull = ((inMBuf == nullptr) || (outMBuf == nullptr));
37 8 : if (inOrOutMbufIsNull) {
38 1 : aicpusd_err(
39 : "PrepareOutput param inMBuf or outMBuf is null, inMBuf[%llx], outMbuf[%llx].", outputInfo.inMBuf,
40 : outputInfo.outMBuf);
41 1 : return AICPU_SCHEDULE_ERROR_PARAMETER_NOT_VALID;
42 : }
43 7 : g_aicpuProfiler.SetPrepareOutStart();
44 7 : if (tensorDesc != nullptr) {
45 1 : if ((UINT32_MAX - static_cast<uint32_t>(sizeof(RuntimeTensorDesc))) < outputInfo.dataSize) {
46 0 : aicpusd_err(
47 : "PrepareOutput param is invalid, output data size[%u] + sizeof(RuntimeTensorDesc)[%zu] "
48 : "will overflow.",
49 : outputInfo.dataSize, sizeof(RuntimeTensorDesc));
50 0 : return AICPU_SCHEDULE_ERROR_PARAMETER_NOT_VALID;
51 : }
52 1 : *outMBuf = BufManager::GetInstance().MallocAndGuardBuf(
53 1 : outputInfo.dataSize + static_cast<uint32_t>(sizeof(RuntimeTensorDesc)), taskContext.modelId);
54 1 : AicpuSdModelStatistic::GetInstance().StatNNModelOutput(
55 1 : taskContext.modelId, tensorDesc, GetStaticNNOutPutIndex(taskContext.modelId));
56 : } else {
57 6 : *outMBuf = BufManager::GetInstance().MallocAndGuardBuf(outputInfo.dataSize, taskContext.modelId);
58 : }
59 :
60 7 : if (*outMBuf == nullptr) {
61 3 : aicpusd_err("Failed to alloc mbuf, dataSize[%u], modelId[%u].", outputInfo.dataSize, taskContext.modelId);
62 3 : AicpuMonitor::GetInstance().SendKillMsgToTsd();
63 3 : return AICPU_SCHEDULE_ERROR_FROM_DRV;
64 : }
65 4 : if (tensorDesc != nullptr) {
66 1 : void* basePtr = nullptr;
67 1 : const auto ret = halMbufGetBuffAddr(*outMBuf, &basePtr);
68 1 : if ((ret != DRV_ERROR_NONE) || (basePtr == nullptr)) {
69 0 : aicpusd_err("Failed to call halMbufGetBuffAddr, ret[%d].", ret);
70 0 : return AICPU_SCHEDULE_ERROR_DRV_ERR;
71 : }
72 :
73 1 : tensorDesc->dataAddr = PtrToValue(basePtr) + static_cast<uint64_t>(sizeof(RuntimeTensorDesc));
74 1 : const errno_t eRet = memcpy_s(basePtr, sizeof(RuntimeTensorDesc), tensorDesc, sizeof(RuntimeTensorDesc));
75 1 : if (eRet != EOK) {
76 0 : return AICPU_SCHEDULE_ERROR_SAFE_FUNCTION_ERR;
77 : }
78 : }
79 4 : g_aicpuProfiler.SetPrepareOutEnd();
80 4 : if (!zeroCpy) {
81 2 : const auto ret = PrepareOutputNonZeroCpy(outputInfo, *outMBuf, tensorDesc);
82 2 : if (ret != AICPU_SCHEDULE_OK) {
83 0 : return ret;
84 : }
85 : }
86 :
87 4 : void* inputHeaderBuf = nullptr;
88 4 : uint32_t inputHeadSize = 0U;
89 4 : const auto drvRet = halMbufGetPrivInfo(*inMBuf, &inputHeaderBuf, &inputHeadSize);
90 4 : if (drvRet != DRV_ERROR_NONE) {
91 1 : aicpusd_err("Failed to get head info in input information, ret[%d].", drvRet);
92 1 : return AICPU_SCHEDULE_ERROR_FROM_DRV;
93 : }
94 3 : const auto ret = OperatorKernelCommon::CopyMbufHeadInfo(inputHeaderBuf, inputHeadSize, *outMBuf);
95 3 : if (ret != AICPU_SCHEDULE_OK) {
96 0 : return ret;
97 : }
98 :
99 3 : return AICPU_SCHEDULE_OK;
100 : }
101 :
102 3 : int32_t OperatorKernelPrepareOutputBase::PrepareOutputNonZeroCpy(
103 : const ProcessOutputInfo& outputInfo, Mbuf* const outMBuf, RuntimeTensorDesc* const tensorDesc) const
104 : {
105 3 : void* dataPtr = nullptr;
106 3 : const auto ret = halMbufGetBuffAddr(outMBuf, &dataPtr);
107 3 : if ((ret != DRV_ERROR_NONE) || (dataPtr == nullptr)) {
108 1 : aicpusd_err("Failed to get data or data is nullptr, ret[%d].", ret);
109 1 : return AICPU_SCHEDULE_ERROR_DRV_ERR;
110 : }
111 :
112 2 : if (tensorDesc != nullptr) {
113 1 : dataPtr = ValueToPtr(PtrToValue(dataPtr) + sizeof(RuntimeTensorDesc));
114 : }
115 :
116 6 : const errno_t eRet = memcpy_s(
117 2 : dataPtr, static_cast<size_t>(outputInfo.dataSize), ValueToPtr(outputInfo.srcPtr),
118 2 : static_cast<size_t>(outputInfo.dataSize));
119 2 : if (eRet != EOK) {
120 0 : aicpusd_err("Failed to memcpy, ret[%d].", eRet);
121 0 : return AICPU_SCHEDULE_ERROR_SAFE_FUNCTION_ERR;
122 : }
123 2 : return AICPU_SCHEDULE_OK;
124 : }
125 :
126 1 : int32_t OperatorKernelPrepareOutputBase::GetStaticNNOutPutIndex(const uint32_t modelId) const
127 : {
128 1 : const auto model = AicpuModelManager::GetInstance().GetModel(modelId);
129 1 : if (model == nullptr) {
130 1 : aicpusd_err("model:%u is null", modelId);
131 1 : return -1;
132 : }
133 0 : return (static_cast<int32_t>(model->GetCurStaticNNModelOutputIndex()));
134 : }
135 :
136 3 : void OperatorKernelPrepareOutputBase::MarkStaticNNOutPutIndex(const uint32_t modelId) const
137 : {
138 3 : const auto model = AicpuModelManager::GetInstance().GetModel(modelId);
139 3 : if (model == nullptr) {
140 3 : aicpusd_err("model:%u is null", modelId);
141 3 : return;
142 : }
143 0 : model->IncreaseStaticNNModelOutputIndex();
144 0 : return;
145 : }
146 :
147 3 : int32_t OperatorKernelPrepareOutputBase::PrepareOutWithTensorDesc(
148 : const AicpuTaskInfo& kernelTaskInfo, const bool zeroCpy, const RunContext& taskContext) const
149 : {
150 3 : MarkStaticNNOutPutIndex(taskContext.modelId);
151 3 : ProcessOutputInfo* const info = PtrToPtr<void, ProcessOutputInfo>(ValueToPtr(kernelTaskInfo.paraBase));
152 3 : if (info == nullptr) {
153 1 : aicpusd_err(
154 : "AicpuTaskInfo.paramBase is null, modelId[%u], streamId[%u], taskId[%u]", taskContext.modelId,
155 : taskContext.streamId, kernelTaskInfo.taskID);
156 1 : return AICPU_SCHEDULE_ERROR_PARAMETER_NOT_VALID;
157 : }
158 2 : if ((UINT64_MAX - static_cast<uint64_t>(sizeof(ProcessOutputInfo))) < kernelTaskInfo.paraBase) {
159 1 : aicpusd_err(
160 : "AicpuTaskInfo.paramBase[%lu] + sizeof(ProcessOutputInfo)[%zu] will overflow, modelId[%u], "
161 : "streamId[%u], taskId[%u]",
162 : kernelTaskInfo.paraBase, sizeof(ProcessOutputInfo), taskContext.modelId, taskContext.streamId,
163 : kernelTaskInfo.taskID);
164 1 : return AICPU_SCHEDULE_ERROR_PARAMETER_NOT_VALID;
165 : }
166 1 : RuntimeTensorDesc* const tensorDesc = PtrToPtr<void, RuntimeTensorDesc>(
167 1 : ValueToPtr(kernelTaskInfo.paraBase + static_cast<uint64_t>(sizeof(ProcessOutputInfo))));
168 1 : return PrepareOutput(*info, taskContext, zeroCpy, tensorDesc);
169 : }
170 :
171 3 : int32_t OperatorKernelModelPrepareOutput::Compute(const AicpuTaskInfo& kernelTaskInfo, const RunContext& taskContext)
172 : {
173 3 : ProcessOutputInfo* const info =
174 3 : reinterpret_cast<ProcessOutputInfo*>(static_cast<uintptr_t>(kernelTaskInfo.paraBase));
175 3 : if (info == nullptr) {
176 1 : aicpusd_err(
177 : "ModelPrepareOut kernelTaskInfo paramBase is null, modelId[%u], streamId[%u], taskId[%u]",
178 : taskContext.modelId, taskContext.streamId, kernelTaskInfo.taskID);
179 1 : return AICPU_SCHEDULE_ERROR_PARAMETER_NOT_VALID;
180 : }
181 2 : return PrepareOutput(*info, taskContext, false, nullptr);
182 : }
183 :
184 2 : int32_t OperatorKernelModelPrepareOutputWithTensorDesc::Compute(
185 : const AicpuTaskInfo& kernelTaskInfo, const RunContext& taskContext)
186 : {
187 2 : return PrepareOutWithTensorDesc(kernelTaskInfo, false, taskContext);
188 : }
189 :
190 1 : int32_t OperatorKernelBufferPrepareOutput::Compute(const AicpuTaskInfo& kernelTaskInfo, const RunContext& taskContext)
191 : {
192 1 : ProcessOutputInfo* const info = PtrToPtr<void, ProcessOutputInfo>(ValueToPtr(kernelTaskInfo.paraBase));
193 1 : if (info == nullptr) {
194 0 : aicpusd_err(
195 : "ModelPrepareOut kernelTaskInfo paramBase is null, modelId[%u], streamId[%u], taskId[%u]",
196 : taskContext.modelId, taskContext.streamId, kernelTaskInfo.taskID);
197 0 : return AICPU_SCHEDULE_ERROR_PARAMETER_NOT_VALID;
198 : }
199 1 : return PrepareOutput(*info, taskContext, true, nullptr);
200 : }
201 :
202 1 : int32_t OperatorKernelBufferPrepareOutputWithTensorDesc::Compute(
203 : const AicpuTaskInfo& kernelTaskInfo, const RunContext& taskContext)
204 : {
205 1 : return PrepareOutWithTensorDesc(kernelTaskInfo, true, taskContext);
206 : }
207 :
208 6 : REGISTER_OPERATOR_KERNEL(KERNEL_MODEL_PREPARE_OUTPUT, OperatorKernelModelPrepareOutput);
209 6 : REGISTER_OPERATOR_KERNEL(KERNEL_MODEL_PREPARE_OUTPUT_WITH_TENSOR_DESC, OperatorKernelModelPrepareOutputWithTensorDesc);
210 6 : REGISTER_OPERATOR_KERNEL(KERNEL_BUFFER_PREPARE_OUTPUT, OperatorKernelBufferPrepareOutput);
211 6 : REGISTER_OPERATOR_KERNEL(
212 : KERNEL_BUFFER_PREPARE_OUTPUT_WITH_TENSOR_DESC, OperatorKernelBufferPrepareOutputWithTensorDesc);
213 : } // namespace AicpuSchedule
|
