Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "operator_kernel_common.h"
12 :
13 : #include <cstring>
14 : #include "aicpusd_msg_send.h"
15 : #include "aicpusd_drv_manager.h"
16 : #include "aicpusd_model_execute.h"
17 :
18 :
19 : namespace AicpuSchedule {
20 9 : int32_t OperatorKernelCommon::SendAICPUSubEvent(char_t * const msg, const uint32_t msgLen, const uint32_t subEventId)
21 : {
22 9 : if (msg == nullptr) {
23 1 : aicpusd_err("The message is nullptr");
24 1 : return AICPU_SCHEDULE_ERROR_INVAILD_EVENT_SUBMIT;
25 : }
26 :
27 8 : if (msgLen == 0U) {
28 1 : aicpusd_err("The size of message is zero");
29 1 : return AICPU_SCHEDULE_ERROR_INVAILD_EVENT_SUBMIT;
30 : }
31 7 : event_summary eventInfoSummary = {};
32 7 : eventInfoSummary.pid = getpid();
33 7 : eventInfoSummary.event_id = EVENT_AICPU_MSG;
34 7 : eventInfoSummary.subevent_id = subEventId;
35 7 : eventInfoSummary.msg = msg;
36 7 : eventInfoSummary.msg_len = msgLen;
37 :
38 7 : const uint32_t deviceId = AicpuDrvManager::GetInstance().GetDeviceId();
39 7 : AicpuMsgSend::SetSchedSubmitEvent(deviceId, eventInfoSummary);
40 7 : return AICPU_SCHEDULE_OK;
41 : }
42 :
43 69 : void OperatorKernelCommon::TraceQueueData(const RunContext &taskContext, void * const headBuf, const uint32_t headSize,
44 : const char_t* const marker)
45 : {
46 69 : MbufHeadMsg * const msg = GetHeadMsgForTrace(headBuf, static_cast<size_t>(headSize), marker);
47 69 : DoTraceQueueData(taskContext, msg, marker);
48 69 : }
49 :
50 85 : MbufHeadMsg *OperatorKernelCommon::GetHeadMsgForTrace(void * const headBuf, const size_t headSize,
51 : const char_t *const marker)
52 : {
53 85 : if (&CheckLogLevel != nullptr) {
54 85 : if (CheckLogLevel(static_cast<int32_t>(CCECPU), DLOG_INFO) != 1) {
55 1 : return nullptr;
56 : }
57 : }
58 :
59 84 : if ((headBuf == nullptr) || (headSize < sizeof(MbufHeadMsg)) || (marker == nullptr)) {
60 67 : return nullptr;
61 : }
62 :
63 17 : MbufHeadMsg * const msg = PtrToPtr<uint8_t, MbufHeadMsg>(PtrAdd<uint8_t>(PtrToPtr<void, uint8_t>(headBuf),
64 : MBUF_HEAD_MAX_SIZE, headSize - sizeof(MbufHeadMsg)));
65 17 : return msg;
66 : }
67 :
68 81 : void OperatorKernelCommon::DoTraceQueueData(const RunContext &taskContext, const MbufHeadMsg *const msg,
69 : const char_t *const marker)
70 : {
71 81 : if (msg == nullptr) {
72 65 : return;
73 : }
74 16 : if (std::strncmp("Dequeued", marker, DEQUEUED_SIZE) == 0) {
75 12 : const auto model = AicpuModelManager::GetInstance().GetModel(taskContext.modelId);
76 12 : if (model == nullptr) {
77 1 : aicpusd_err("Cannot get model by modelId:[%u], streamId[%u], transId[%lu].", taskContext.modelId,
78 : taskContext.streamId, msg->transId);
79 1 : return;
80 : }
81 11 : (void)model->SetModelTransId(msg->transId);
82 : }
83 15 : aicpusd_info("%s: transId[%lu], routeLabel[%u], retCode[%d], modelId[%u], streamId[%u]", marker, msg->transId,
84 : msg->routeLabel, msg->retCode, taskContext.modelId, taskContext.streamId);
85 : }
86 :
87 16 : std::shared_ptr<MbufHeadMsg> OperatorKernelCommon::BackupHeadMsg(void * const headBuf, const uint32_t headSize,
88 : const char_t* const marker)
89 : {
90 16 : MbufHeadMsg * const msg = GetHeadMsgForTrace(headBuf, static_cast<size_t>(headSize), marker);
91 16 : if (msg == nullptr) {
92 11 : return nullptr;
93 : }
94 5 : std::shared_ptr<MbufHeadMsg> backupMsg(new (std::nothrow) MbufHeadMsg());
95 5 : AICPUSD_CHECK((backupMsg != nullptr), nullptr, "alloc backup headmsg failed.");
96 :
97 5 : const auto cpyRet = memcpy_s(backupMsg.get(), sizeof(MbufHeadMsg), msg, sizeof(MbufHeadMsg));
98 5 : if (cpyRet != EOK) {
99 1 : aicpusd_warn("Memcpy headmsg failed, cpyLen[%zu], ret=[%d].", sizeof(MbufHeadMsg), cpyRet);
100 1 : return nullptr;
101 : }
102 4 : return backupMsg;
103 5 : }
104 :
105 34 : int32_t OperatorKernelCommon::CopyMbufHeadInfo(const void * const srcHeaderBuf, const uint32_t srcHeadSize,
106 : Mbuf *destMbuf)
107 : {
108 34 : if (srcHeaderBuf == nullptr) {
109 1 : aicpusd_err("malloc srcHeaderBuf is nullptr.");
110 1 : return AICPU_SCHEDULE_ERROR_INNER_ERROR;
111 : }
112 33 : if (destMbuf == nullptr) {
113 1 : aicpusd_err("malloc destMbuf is nullptr.");
114 1 : return AICPU_SCHEDULE_ERROR_INNER_ERROR;
115 : }
116 :
117 32 : void *destHeaderBuf = nullptr;
118 32 : uint32_t destHeadSize = 0U;
119 32 : const auto ret = halMbufGetPrivInfo(destMbuf, &destHeaderBuf, &destHeadSize);
120 32 : if (ret != DRV_ERROR_NONE) {
121 1 : aicpusd_err("Failed to get head info in dest information, ret[%d].", ret);
122 1 : return AICPU_SCHEDULE_ERROR_FROM_DRV;
123 : }
124 31 : if (destHeaderBuf == nullptr) {
125 1 : aicpusd_err("Failed to get head info from dest buffer.");
126 1 : return AICPU_SCHEDULE_ERROR_MALLOC_MEM_FAIL_THROUGH_DRV;
127 : }
128 30 : if (srcHeadSize != destHeadSize) {
129 0 : aicpusd_err("the src head size[%u] is not equal to the dest size[%u].", srcHeadSize, destHeadSize);
130 0 : return AICPU_SCHEDULE_ERROR_MALLOC_MEM_FAIL_THROUGH_DRV;
131 : }
132 30 : if (srcHeadSize == 0U) {
133 0 : aicpusd_err("Failed to get size.");
134 0 : return AICPU_SCHEDULE_ERROR_MALLOC_MEM_FAIL_THROUGH_DRV;
135 : }
136 :
137 : const errno_t eRet =
138 30 : memcpy_s(destHeaderBuf, static_cast<uint64_t>(destHeadSize), srcHeaderBuf, static_cast<uint64_t>(srcHeadSize));
139 30 : if (eRet != EOK) {
140 0 : aicpusd_err("Failed to memcpy, ret[%d].", eRet);
141 0 : return AICPU_SCHEDULE_ERROR_SAFE_FUNCTION_ERR;
142 : }
143 30 : return AICPU_SCHEDULE_OK;
144 : }
145 :
146 89 : int32_t OperatorKernelCommon::GetMbufDataPtr(const uint64_t srcAddr, void **dataAddrPtr)
147 : {
148 89 : if (dataAddrPtr == nullptr) {
149 0 : aicpusd_err("Mbuf data ptr is null.");
150 0 : return AICPU_SCHEDULE_ERROR_PARAMETER_NOT_VALID;
151 : }
152 89 : const auto mbufPptr = reinterpret_cast<Mbuf **>(static_cast<uintptr_t>(srcAddr));
153 89 : if (mbufPptr == nullptr) {
154 2 : aicpusd_err("mbufPptr is null.");
155 2 : return AICPU_SCHEDULE_ERROR_PARAMETER_NOT_VALID;
156 : }
157 87 : if (*mbufPptr == nullptr) {
158 1 : aicpusd_err("*mbufPptr is null.");
159 1 : return AICPU_SCHEDULE_ERROR_PARAMETER_NOT_VALID;
160 : }
161 :
162 86 : const auto ret = halMbufGetBuffAddr(*mbufPptr, dataAddrPtr);
163 86 : if (ret != DRV_ERROR_NONE) {
164 2 : aicpusd_err("Failed to get data ptr, ret[%d].", ret);
165 2 : return AICPU_SCHEDULE_ERROR_FROM_DRV;
166 : }
167 84 : return AICPU_SCHEDULE_OK;
168 : }
169 :
170 5 : int32_t OperatorKernelCommon::UpdateDataPtr(const uint64_t mbufAddr, const int32_t fusionOffset,
171 : void *&dataPtr, uint64_t &totalOffset)
172 : {
173 5 : uint64_t dataSize = 0UL;
174 5 : int32_t ret = OperatorKernelCommon::GetMbufDataSize(mbufAddr, dataSize);
175 5 : if (ret != AICPU_SCHEDULE_OK) {
176 1 : aicpusd_err("Failed to get mbuf data size, ret = %d.", ret);
177 1 : return ret;
178 : }
179 4 : if (dataSize < sizeof(RuntimeTensorDesc)) {
180 2 : aicpusd_err("Mbuf data size[%lu] is invalid, must >= %zu.", dataSize, sizeof(RuntimeTensorDesc));
181 2 : return AICPU_SCHEDULE_ERROR_PARAMETER_NOT_VALID;
182 : }
183 2 : FusionInfo info = {};
184 2 : info.dataSize = dataSize;
185 2 : ret = OperatorKernelCommon::DoUpdateDataPtr(info, fusionOffset, dataPtr);
186 2 : if (ret == AICPU_SCHEDULE_OK) {
187 2 : totalOffset = info.lastDataOffset;
188 : }
189 2 : return ret;
190 : }
191 :
192 4 : int32_t OperatorKernelCommon::DoUpdateDataPtr(FusionInfo &info, const int32_t fusionOffset, void *&dataPtr)
193 : {
194 4 : if (fusionOffset < info.lastFusionOffset) {
195 1 : aicpusd_err("Invalid fusionOffset[%d] vs lastFusionOffset[%d].", fusionOffset, info.lastFusionOffset);
196 1 : return AICPU_SCHEDULE_ERROR_PARAMETER_NOT_VALID;
197 : }
198 3 : auto baseAddr = PtrToPtr<void, uint8_t>(dataPtr);
199 3 : uint64_t totalOffset = info.lastDataOffset;
200 6 : for (int32_t i = info.lastFusionOffset; i < fusionOffset; ++i) {
201 3 : if (totalOffset > info.dataSize - sizeof(RuntimeTensorDesc)) {
202 0 : aicpusd_err("Fusion size is invalid, must <= data size[%lu].", info.dataSize);
203 0 : return AICPU_SCHEDULE_ERROR_PARAMETER_NOT_VALID;
204 : }
205 3 : auto tensorDesc = PtrToPtr<uint8_t, RuntimeTensorDesc>(baseAddr + totalOffset);
206 3 : totalOffset += sizeof(RuntimeTensorDesc);
207 :
208 3 : if (tensorDesc->dataSize > info.dataSize - totalOffset) {
209 0 : aicpusd_err("Tensor dataSize[%lu] is invalid, must <= mbuf dataSize[%lu] - offset[%lu].",
210 : tensorDesc->dataSize, info.dataSize, totalOffset);
211 0 : return AICPU_SCHEDULE_ERROR_PARAMETER_NOT_VALID;
212 : }
213 3 : totalOffset += tensorDesc->dataSize;
214 : }
215 3 : aicpusd_info("Fusion offset index = %d, total offset byte size = %lu.", fusionOffset, totalOffset);
216 3 : if (totalOffset > info.dataSize - sizeof(RuntimeTensorDesc)) {
217 0 : aicpusd_err("Fusion offset[%lu] is invalid, must <= data size[%u] - %zu.",
218 : totalOffset, info.dataSize, sizeof(RuntimeTensorDesc));
219 0 : return AICPU_SCHEDULE_ERROR_PARAMETER_NOT_VALID;
220 : }
221 3 : dataPtr = PtrToPtr<uint8_t, void>(baseAddr + totalOffset);
222 3 : info.lastFusionOffset = fusionOffset;
223 3 : info.lastDataOffset = totalOffset;
224 3 : return AICPU_SCHEDULE_OK;
225 : }
226 :
227 10 : int32_t OperatorKernelCommon::GetMbufDataSize(const uint64_t srcAddr, uint64_t &dataSize)
228 : {
229 10 : const auto mbufPptr = reinterpret_cast<Mbuf **>(static_cast<uintptr_t>(srcAddr));
230 10 : if (mbufPptr == nullptr) {
231 0 : aicpusd_err("mbufPptr is null.");
232 0 : return AICPU_SCHEDULE_ERROR_PARAMETER_NOT_VALID;
233 : }
234 10 : if (*mbufPptr == nullptr) {
235 0 : aicpusd_err("*mbufPptr is null.");
236 0 : return AICPU_SCHEDULE_ERROR_PARAMETER_NOT_VALID;
237 : }
238 :
239 10 : const auto ret = halMbufGetBuffSize(*mbufPptr, &dataSize);
240 10 : if (ret != DRV_ERROR_NONE) {
241 1 : aicpusd_err("Failed to get data size, ret[%d].", ret);
242 1 : return AICPU_SCHEDULE_ERROR_FROM_DRV;
243 : }
244 9 : return AICPU_SCHEDULE_OK;
245 : }
246 :
247 5 : int32_t OperatorKernelCommon::ParseTensorDescAndCalcDataSize(const RuntimeTensorDesc * const srcTensorDesc,
248 : uint32_t &dataSize)
249 : {
250 5 : if (srcTensorDesc->shape[0] > MAX_DIM_SIZE) {
251 0 : aicpusd_err("Max shape size[%lld], but got shape size[%lld]", MAX_DIM_SIZE, srcTensorDesc->shape[0]);
252 0 : return AICPU_SCHEDULE_ERROR_PARAMETER_NOT_VALID;
253 : }
254 :
255 5 : int64_t size = 0;
256 10 : const int32_t ret = AicpuUtil::CalcDataSizeByShape(&(srcTensorDesc->shape[1]), srcTensorDesc->shape[0],
257 5 : srcTensorDesc->dtype, size);
258 5 : if ((ret != AICPU_SCHEDULE_OK) || (size < 0) || (size > static_cast<int64_t>(UINT32_MAX))) {
259 0 : aicpusd_err("Get data size by shape failed, ret[%d], size[%lld]", ret, size);
260 0 : return AICPU_SCHEDULE_ERROR_PARAMETER_NOT_VALID;
261 : }
262 5 : dataSize = static_cast<uint32_t>(size);
263 5 : return AICPU_SCHEDULE_OK;
264 : }
265 :
266 4 : int32_t OperatorKernelCommon::GetMbufAddrAndSize(Mbuf *mbuf, void **dataPptr, uint64_t *dataLenPtr, uint32_t modelId,
267 : bool allowOnlyDesc)
268 : {
269 4 : if (mbuf == nullptr) {
270 0 : aicpusd_err("null mbuf for model[%u]", modelId);
271 0 : return AICPU_SCHEDULE_ERROR_PARAMETER_NOT_VALID;
272 : }
273 4 : const uint64_t dataLenThreshold = allowOnlyDesc ? sizeof(RuntimeTensorDesc) : sizeof(RuntimeTensorDesc) + 1U;
274 4 : auto ret = halMbufGetBuffSize(mbuf, dataLenPtr);
275 4 : if ((ret != static_cast<int32_t>(DRV_ERROR_NONE)) || (*dataLenPtr < dataLenThreshold)) {
276 1 : aicpusd_err("Fail to get buff size for mbuf, model:[%u], ret=[%d], dataLen[%u] vs dataLenThreshold[%u]",
277 : modelId, ret, *dataLenPtr, dataLenThreshold);
278 1 : return AICPU_SCHEDULE_ERROR_DRV_ERR;
279 : }
280 :
281 3 : ret = halMbufGetBuffAddr(mbuf, dataPptr);
282 3 : if ((ret != static_cast<int32_t>(DRV_ERROR_NONE)) || (*dataPptr == nullptr)) {
283 0 : aicpusd_err("Failed to get data or data is nullptr, ret[%d].", ret);
284 0 : return AICPU_SCHEDULE_ERROR_DRV_ERR;
285 : }
286 :
287 3 : return AICPU_SCHEDULE_OK;
288 : }
289 :
290 : } // namespace AicpuSchedule
|
