Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "operator_kernel_common.h"
12 :
13 : #include <cstring>
14 : #include "aicpusd_msg_send.h"
15 : #include "aicpusd_drv_manager.h"
16 : #include "aicpusd_model_execute.h"
17 :
18 : namespace AicpuSchedule {
19 9 : int32_t OperatorKernelCommon::SendAICPUSubEvent(char_t* const msg, const uint32_t msgLen, const uint32_t subEventId)
20 : {
21 9 : if (msg == nullptr) {
22 1 : aicpusd_err("The message is nullptr");
23 1 : return AICPU_SCHEDULE_ERROR_INVALID_EVENT_SUBMIT;
24 : }
25 :
26 8 : if (msgLen == 0U) {
27 1 : aicpusd_err("The size of message is zero");
28 1 : return AICPU_SCHEDULE_ERROR_INVALID_EVENT_SUBMIT;
29 : }
30 7 : event_summary eventInfoSummary = {};
31 7 : eventInfoSummary.pid = getpid();
32 7 : eventInfoSummary.event_id = EVENT_AICPU_MSG;
33 7 : eventInfoSummary.subevent_id = subEventId;
34 7 : eventInfoSummary.msg = msg;
35 7 : eventInfoSummary.msg_len = msgLen;
36 :
37 7 : const uint32_t deviceId = AicpuDrvManager::GetInstance().GetDeviceId();
38 7 : AicpuMsgSend::SetSchedSubmitEvent(deviceId, eventInfoSummary);
39 7 : return AICPU_SCHEDULE_OK;
40 : }
41 :
42 69 : void OperatorKernelCommon::TraceQueueData(
43 : const RunContext& taskContext, void* const headBuf, const uint32_t headSize, const char_t* const marker)
44 : {
45 69 : MbufHeadMsg* const msg = GetHeadMsgForTrace(headBuf, static_cast<size_t>(headSize), marker);
46 69 : DoTraceQueueData(taskContext, msg, marker);
47 69 : }
48 :
49 85 : MbufHeadMsg* OperatorKernelCommon::GetHeadMsgForTrace(
50 : void* const headBuf, const size_t headSize, const char_t* const marker)
51 : {
52 85 : if (&CheckLogLevel != nullptr) {
53 85 : if (CheckLogLevel(static_cast<int32_t>(CCECPU), DLOG_INFO) != 1) {
54 1 : return nullptr;
55 : }
56 : }
57 :
58 84 : if ((headBuf == nullptr) || (headSize < sizeof(MbufHeadMsg)) || (marker == nullptr)) {
59 67 : return nullptr;
60 : }
61 :
62 17 : MbufHeadMsg* const msg = PtrToPtr<uint8_t, MbufHeadMsg>(
63 : PtrAdd<uint8_t>(PtrToPtr<void, uint8_t>(headBuf), MBUF_HEAD_MAX_SIZE, headSize - sizeof(MbufHeadMsg)));
64 17 : return msg;
65 : }
66 :
67 81 : void OperatorKernelCommon::DoTraceQueueData(
68 : const RunContext& taskContext, const MbufHeadMsg* const msg, const char_t* const marker)
69 : {
70 81 : if (msg == nullptr) {
71 65 : return;
72 : }
73 16 : if (std::strncmp("Dequeued", marker, DEQUEUED_SIZE) == 0) {
74 12 : const auto model = AicpuModelManager::GetInstance().GetModel(taskContext.modelId);
75 12 : if (model == nullptr) {
76 1 : aicpusd_err(
77 : "Cannot get model by modelId:[%u], streamId[%u], transId[%lu].", taskContext.modelId,
78 : taskContext.streamId, msg->transId);
79 1 : return;
80 : }
81 11 : (void)model->SetModelTransId(msg->transId);
82 : }
83 15 : aicpusd_info(
84 : "%s: transId[%lu], routeLabel[%u], retCode[%d], modelId[%u], streamId[%u]", marker, msg->transId,
85 : msg->routeLabel, msg->retCode, taskContext.modelId, taskContext.streamId);
86 : }
87 :
88 16 : std::shared_ptr<MbufHeadMsg> OperatorKernelCommon::BackupHeadMsg(
89 : void* const headBuf, const uint32_t headSize, const char_t* const marker)
90 : {
91 16 : MbufHeadMsg* const msg = GetHeadMsgForTrace(headBuf, static_cast<size_t>(headSize), marker);
92 16 : if (msg == nullptr) {
93 11 : return nullptr;
94 : }
95 5 : std::shared_ptr<MbufHeadMsg> backupMsg(new (std::nothrow) MbufHeadMsg());
96 5 : AICPUSD_CHECK((backupMsg != nullptr), nullptr, "alloc backup headmsg failed.");
97 :
98 5 : const auto cpyRet = memcpy_s(backupMsg.get(), sizeof(MbufHeadMsg), msg, sizeof(MbufHeadMsg));
99 5 : if (cpyRet != EOK) {
100 1 : aicpusd_warn("Memcpy headmsg failed, cpyLen[%zu], ret=[%d].", sizeof(MbufHeadMsg), cpyRet);
101 1 : return nullptr;
102 : }
103 4 : return backupMsg;
104 5 : }
105 :
106 34 : int32_t OperatorKernelCommon::CopyMbufHeadInfo(
107 : const void* const srcHeaderBuf, const uint32_t srcHeadSize, Mbuf* destMbuf)
108 : {
109 34 : if (srcHeaderBuf == nullptr) {
110 1 : aicpusd_err("malloc srcHeaderBuf is nullptr.");
111 1 : return AICPU_SCHEDULE_ERROR_INNER_ERROR;
112 : }
113 33 : if (destMbuf == nullptr) {
114 1 : aicpusd_err("malloc destMbuf is nullptr.");
115 1 : return AICPU_SCHEDULE_ERROR_INNER_ERROR;
116 : }
117 :
118 32 : void* destHeaderBuf = nullptr;
119 32 : uint32_t destHeadSize = 0U;
120 32 : const auto ret = halMbufGetPrivInfo(destMbuf, &destHeaderBuf, &destHeadSize);
121 32 : if (ret != DRV_ERROR_NONE) {
122 1 : aicpusd_err("Failed to get head info in dest information, ret[%d].", ret);
123 1 : return AICPU_SCHEDULE_ERROR_FROM_DRV;
124 : }
125 31 : if (destHeaderBuf == nullptr) {
126 1 : aicpusd_err("Failed to get head info from dest buffer.");
127 1 : return AICPU_SCHEDULE_ERROR_MALLOC_MEM_FAIL_THROUGH_DRV;
128 : }
129 30 : if (srcHeadSize != destHeadSize) {
130 0 : aicpusd_err("the src head size[%u] is not equal to the dest size[%u].", srcHeadSize, destHeadSize);
131 0 : return AICPU_SCHEDULE_ERROR_MALLOC_MEM_FAIL_THROUGH_DRV;
132 : }
133 30 : if (srcHeadSize == 0U) {
134 0 : aicpusd_err("Failed to get size.");
135 0 : return AICPU_SCHEDULE_ERROR_MALLOC_MEM_FAIL_THROUGH_DRV;
136 : }
137 :
138 : const errno_t eRet =
139 30 : memcpy_s(destHeaderBuf, static_cast<uint64_t>(destHeadSize), srcHeaderBuf, static_cast<uint64_t>(srcHeadSize));
140 30 : if (eRet != EOK) {
141 0 : aicpusd_err("Failed to memcpy, ret[%d].", eRet);
142 0 : return AICPU_SCHEDULE_ERROR_SAFE_FUNCTION_ERR;
143 : }
144 30 : return AICPU_SCHEDULE_OK;
145 : }
146 :
147 89 : int32_t OperatorKernelCommon::GetMbufDataPtr(const uint64_t srcAddr, void** dataAddrPtr)
148 : {
149 89 : if (dataAddrPtr == nullptr) {
150 0 : aicpusd_err("Mbuf data ptr is null.");
151 0 : return AICPU_SCHEDULE_ERROR_PARAMETER_NOT_VALID;
152 : }
153 89 : const auto mbufPptr = reinterpret_cast<Mbuf**>(static_cast<uintptr_t>(srcAddr));
154 89 : if (mbufPptr == nullptr) {
155 2 : aicpusd_err("mbufPptr is null.");
156 2 : return AICPU_SCHEDULE_ERROR_PARAMETER_NOT_VALID;
157 : }
158 87 : if (*mbufPptr == nullptr) {
159 1 : aicpusd_err("*mbufPptr is null.");
160 1 : return AICPU_SCHEDULE_ERROR_PARAMETER_NOT_VALID;
161 : }
162 :
163 86 : const auto ret = halMbufGetBuffAddr(*mbufPptr, dataAddrPtr);
164 86 : if (ret != DRV_ERROR_NONE) {
165 2 : aicpusd_err("Failed to get data ptr, ret[%d].", ret);
166 2 : return AICPU_SCHEDULE_ERROR_FROM_DRV;
167 : }
168 84 : return AICPU_SCHEDULE_OK;
169 : }
170 :
171 5 : int32_t OperatorKernelCommon::UpdateDataPtr(
172 : const uint64_t mbufAddr, const int32_t fusionOffset, void*& dataPtr, uint64_t& totalOffset)
173 : {
174 5 : uint64_t dataSize = 0UL;
175 5 : int32_t ret = OperatorKernelCommon::GetMbufDataSize(mbufAddr, dataSize);
176 5 : if (ret != AICPU_SCHEDULE_OK) {
177 1 : aicpusd_err("Failed to get mbuf data size, ret = %d.", ret);
178 1 : return ret;
179 : }
180 4 : if (dataSize < sizeof(RuntimeTensorDesc)) {
181 2 : aicpusd_err("Mbuf data size[%lu] is invalid, must >= %zu.", dataSize, sizeof(RuntimeTensorDesc));
182 2 : return AICPU_SCHEDULE_ERROR_PARAMETER_NOT_VALID;
183 : }
184 2 : FusionInfo info = {};
185 2 : info.dataSize = dataSize;
186 2 : ret = OperatorKernelCommon::DoUpdateDataPtr(info, fusionOffset, dataPtr);
187 2 : if (ret == AICPU_SCHEDULE_OK) {
188 2 : totalOffset = info.lastDataOffset;
189 : }
190 2 : return ret;
191 : }
192 :
193 4 : int32_t OperatorKernelCommon::DoUpdateDataPtr(FusionInfo& info, const int32_t fusionOffset, void*& dataPtr)
194 : {
195 4 : if (fusionOffset < info.lastFusionOffset) {
196 1 : aicpusd_err("Invalid fusionOffset[%d] vs lastFusionOffset[%d].", fusionOffset, info.lastFusionOffset);
197 1 : return AICPU_SCHEDULE_ERROR_PARAMETER_NOT_VALID;
198 : }
199 3 : auto baseAddr = PtrToPtr<void, uint8_t>(dataPtr);
200 3 : uint64_t totalOffset = info.lastDataOffset;
201 6 : for (int32_t i = info.lastFusionOffset; i < fusionOffset; ++i) {
202 3 : if (totalOffset > info.dataSize - sizeof(RuntimeTensorDesc)) {
203 0 : aicpusd_err("Fusion size is invalid, must <= data size[%lu].", info.dataSize);
204 0 : return AICPU_SCHEDULE_ERROR_PARAMETER_NOT_VALID;
205 : }
206 3 : auto tensorDesc = PtrToPtr<uint8_t, RuntimeTensorDesc>(baseAddr + totalOffset);
207 3 : totalOffset += sizeof(RuntimeTensorDesc);
208 :
209 3 : if (tensorDesc->dataSize > info.dataSize - totalOffset) {
210 0 : aicpusd_err(
211 : "Tensor dataSize[%lu] is invalid, must <= mbuf dataSize[%lu] - offset[%lu].", tensorDesc->dataSize,
212 : info.dataSize, totalOffset);
213 0 : return AICPU_SCHEDULE_ERROR_PARAMETER_NOT_VALID;
214 : }
215 3 : totalOffset += tensorDesc->dataSize;
216 : }
217 3 : aicpusd_info("Fusion offset index = %d, total offset byte size = %lu.", fusionOffset, totalOffset);
218 3 : if (totalOffset > info.dataSize - sizeof(RuntimeTensorDesc)) {
219 0 : aicpusd_err(
220 : "Fusion offset[%lu] is invalid, must <= data size[%u] - %zu.", totalOffset, info.dataSize,
221 : sizeof(RuntimeTensorDesc));
222 0 : return AICPU_SCHEDULE_ERROR_PARAMETER_NOT_VALID;
223 : }
224 3 : dataPtr = PtrToPtr<uint8_t, void>(baseAddr + totalOffset);
225 3 : info.lastFusionOffset = fusionOffset;
226 3 : info.lastDataOffset = totalOffset;
227 3 : return AICPU_SCHEDULE_OK;
228 : }
229 :
230 10 : int32_t OperatorKernelCommon::GetMbufDataSize(const uint64_t srcAddr, uint64_t& dataSize)
231 : {
232 10 : const auto mbufPptr = reinterpret_cast<Mbuf**>(static_cast<uintptr_t>(srcAddr));
233 10 : if (mbufPptr == nullptr) {
234 0 : aicpusd_err("mbufPptr is null.");
235 0 : return AICPU_SCHEDULE_ERROR_PARAMETER_NOT_VALID;
236 : }
237 10 : if (*mbufPptr == nullptr) {
238 0 : aicpusd_err("*mbufPptr is null.");
239 0 : return AICPU_SCHEDULE_ERROR_PARAMETER_NOT_VALID;
240 : }
241 :
242 10 : const auto ret = halMbufGetBuffSize(*mbufPptr, &dataSize);
243 10 : if (ret != DRV_ERROR_NONE) {
244 1 : aicpusd_err("Failed to get data size, ret[%d].", ret);
245 1 : return AICPU_SCHEDULE_ERROR_FROM_DRV;
246 : }
247 9 : return AICPU_SCHEDULE_OK;
248 : }
249 :
250 5 : int32_t OperatorKernelCommon::ParseTensorDescAndCalcDataSize(
251 : const RuntimeTensorDesc* const srcTensorDesc, uint32_t& dataSize)
252 : {
253 5 : if (srcTensorDesc->shape[0] > MAX_DIM_SIZE) {
254 0 : aicpusd_err("Max shape size[%lld], but got shape size[%lld]", MAX_DIM_SIZE, srcTensorDesc->shape[0]);
255 0 : return AICPU_SCHEDULE_ERROR_PARAMETER_NOT_VALID;
256 : }
257 :
258 5 : int64_t size = 0;
259 : const int32_t ret =
260 5 : AicpuUtil::CalcDataSizeByShape(&(srcTensorDesc->shape[1]), srcTensorDesc->shape[0], srcTensorDesc->dtype, size);
261 5 : if ((ret != AICPU_SCHEDULE_OK) || (size < 0) || (size > static_cast<int64_t>(UINT32_MAX))) {
262 0 : aicpusd_err("Get data size by shape failed, ret[%d], size[%lld]", ret, size);
263 0 : return AICPU_SCHEDULE_ERROR_PARAMETER_NOT_VALID;
264 : }
265 5 : dataSize = static_cast<uint32_t>(size);
266 5 : return AICPU_SCHEDULE_OK;
267 : }
268 :
269 4 : int32_t OperatorKernelCommon::GetMbufAddrAndSize(
270 : Mbuf* mbuf, void** dataPptr, uint64_t* dataLenPtr, uint32_t modelId, bool allowOnlyDesc)
271 : {
272 4 : if (mbuf == nullptr) {
273 0 : aicpusd_err("null mbuf for model[%u]", modelId);
274 0 : return AICPU_SCHEDULE_ERROR_PARAMETER_NOT_VALID;
275 : }
276 4 : const uint64_t dataLenThreshold = allowOnlyDesc ? sizeof(RuntimeTensorDesc) : sizeof(RuntimeTensorDesc) + 1U;
277 4 : auto ret = halMbufGetBuffSize(mbuf, dataLenPtr);
278 4 : if ((ret != static_cast<int32_t>(DRV_ERROR_NONE)) || (*dataLenPtr < dataLenThreshold)) {
279 1 : aicpusd_err(
280 : "Fail to get buff size for mbuf, model:[%u], ret=[%d], dataLen[%u] vs dataLenThreshold[%u]", modelId, ret,
281 : *dataLenPtr, dataLenThreshold);
282 1 : return AICPU_SCHEDULE_ERROR_DRV_ERR;
283 : }
284 :
285 3 : ret = halMbufGetBuffAddr(mbuf, dataPptr);
286 3 : if ((ret != static_cast<int32_t>(DRV_ERROR_NONE)) || (*dataPptr == nullptr)) {
287 0 : aicpusd_err("Failed to get data or data is nullptr, ret[%d].", ret);
288 0 : return AICPU_SCHEDULE_ERROR_DRV_ERR;
289 : }
290 :
291 3 : return AICPU_SCHEDULE_OK;
292 : }
293 :
294 : } // namespace AicpuSchedule
|
