Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "operator_kernel_zero_cpy.h"
12 :
13 : #include "aicpusd_status.h"
14 : #include "operator_kernel_common.h"
15 :
16 :
17 : namespace AicpuSchedule {
18 : namespace {
19 : const std::string KERNEL_ZERO_CPY = "zeroCpy";
20 : const std::string KERNEL_ZERO_CPY_V2 = "zeroCpyV2";
21 : const std::string KERNEL_CPU_ZERO_CPY = "cpuZeroCpy";
22 : } // namespace
23 :
24 5 : int32_t OperatorKernelZeroCpy::Compute(const AicpuTaskInfo &kernelTaskInfo, const RunContext &taskContext)
25 : {
26 5 : AddrMapInfo *mapInfo = PtrToPtr<void, AddrMapInfo>(ValueToPtr(kernelTaskInfo.paraBase));
27 5 : if (mapInfo == nullptr) {
28 1 : aicpusd_err("ModelZeroCpy kernelTaskInfo paramBase is null, modelId[%u], streamId[%u], taskId[%u]",
29 : taskContext.modelId, taskContext.streamId, kernelTaskInfo.taskID);
30 1 : return AICPU_SCHEDULE_ERROR_PARAMETER_NOT_VALID;
31 : }
32 :
33 4 : const uint64_t *const srcAddrList = PtrToPtr<void, uint64_t>(ValueToPtr(mapInfo->srcAddrList));
34 4 : const uint64_t *const dstAddrList = PtrToPtr<void, uint64_t>(ValueToPtr(mapInfo->dstAddrList));
35 4 : if ((srcAddrList == nullptr) || (dstAddrList == nullptr)) {
36 1 : aicpusd_err("Failed to zero copy, srcAddrList or dstAddrList is null, modelId[%u], streamId[%u]",
37 : taskContext.modelId, taskContext.streamId);
38 1 : return AICPU_SCHEDULE_ERROR_INNER_ERROR;
39 : }
40 4 : for (uint32_t i = 0U; i < mapInfo->addrNum; i++) {
41 3 : void *dataPtr = nullptr;
42 3 : const int32_t ret = OperatorKernelCommon::GetMbufDataPtr(srcAddrList[i], &dataPtr);
43 3 : if (ret != AICPU_SCHEDULE_OK) {
44 2 : aicpusd_err("Failed to get mbuf data addr. srcAddrList[%u] is [%lu].", i, srcAddrList[i]);
45 2 : return ret;
46 : }
47 1 : const auto dstPtr = reinterpret_cast<void **>(static_cast<uintptr_t>(dstAddrList[i]));
48 1 : *dstPtr = dataPtr;
49 : }
50 1 : return AICPU_SCHEDULE_OK;
51 : }
52 :
53 11 : int32_t OperatorKernelZeroCpyV2::Compute(const AicpuTaskInfo &kernelTaskInfo, const RunContext &taskContext)
54 : {
55 11 : AddrMapInfoV2 * const mapInfo = reinterpret_cast<AddrMapInfoV2 *>(static_cast<uintptr_t>(kernelTaskInfo.paraBase));
56 11 : if (mapInfo == nullptr) {
57 1 : aicpusd_err("ModelDynZeroCpy kernelTaskInfo paramBase is null, modelId[%u], streamId[%u], taskId[%u]",
58 : taskContext.modelId, taskContext.streamId, kernelTaskInfo.taskID);
59 1 : return AICPU_SCHEDULE_ERROR_PARAMETER_NOT_VALID;
60 : }
61 :
62 10 : return DoCompute(*mapInfo, taskContext);
63 : }
64 :
65 10 : int32_t OperatorKernelZeroCpyV2::DoCompute(AddrMapInfoV2 &mapInfo, const RunContext &taskContext) const
66 : {
67 10 : const uint64_t * const srcAddrList = PtrToPtr<void, uint64_t>(ValueToPtr(static_cast<uintptr_t>(mapInfo.srcAddrList)));
68 10 : const uint64_t * const dstAddrList = PtrToPtr<void, uint64_t>(ValueToPtr(static_cast<uintptr_t>(mapInfo.dstAddrList)));
69 10 : const int32_t * const isNoTilingList = PtrToPtr<void, int32_t>(ValueToPtr(static_cast<uintptr_t>(mapInfo.isNoTilingList)));
70 10 : if ((srcAddrList == nullptr) || (dstAddrList == nullptr) || (isNoTilingList == nullptr)) {
71 0 : aicpusd_err("Failed to zero copy, srcAddrList, dstAddrList or isNoTilingList is null, "
72 : "modelId[%u], streamId[%u]", taskContext.modelId, taskContext.streamId);
73 0 : return AICPU_SCHEDULE_ERROR_INNER_ERROR;
74 : }
75 :
76 10 : std::vector<int32_t> fusionOffsets(mapInfo.addrNum);
77 10 : if (mapInfo.len >= sizeof(uint32_t) + sizeof(uint64_t) + sizeof(uint64_t)) {
78 : const uint64_t *const fusionOffsetListAddr =
79 7 : PtrToPtr<char, uint64_t>(mapInfo.extendInfo + sizeof(uint32_t) + sizeof(uint64_t));
80 7 : const int32_t ret = ResolveFusionOffsets(fusionOffsetListAddr, mapInfo.addrNum, fusionOffsets);
81 7 : if (ret != AICPU_SCHEDULE_OK) {
82 2 : aicpusd_err("Failed to resolve fusion offsets, addr num = %u, ret = %d.", mapInfo.addrNum, ret);
83 2 : return ret;
84 : }
85 : }
86 :
87 8 : std::unordered_map<uint64_t, FusionInfo> fusionMap;
88 12 : for (uint32_t i = 0U; i < mapInfo.addrNum; i++) {
89 8 : void *srcDataPtr = nullptr;
90 8 : auto result = OperatorKernelCommon::GetMbufDataPtr(srcAddrList[i], &srcDataPtr);
91 8 : if (result != AICPU_SCHEDULE_OK) {
92 0 : aicpusd_err("Failed to get mbuf data addr. srcAddrList[%u] is [%lu].", i, srcAddrList[i]);
93 4 : return result;
94 : }
95 :
96 8 : if (fusionOffsets[i] > 0) {
97 5 : result = UpdateDataPtrExtend(srcAddrList[i], fusionOffsets[i], srcDataPtr, fusionMap);
98 5 : if (result != AICPU_SCHEDULE_OK) {
99 4 : aicpusd_err("Failed to update data addr. fusion offset[%d]", fusionOffsets[i]);
100 4 : return result;
101 : }
102 : }
103 4 : const auto dstPtr = reinterpret_cast<void **>(static_cast<uintptr_t>(dstAddrList[i]));
104 4 : *dstPtr = srcDataPtr;
105 : // if notiling will not skip
106 4 : if (isNoTilingList[i] != 0) {
107 3 : RuntimeTensorDesc * const tensorDesc = PtrToPtr<void, RuntimeTensorDesc>(srcDataPtr);
108 3 : tensorDesc->dataAddr =
109 3 : static_cast<uint64_t>(reinterpret_cast<uintptr_t>(srcDataPtr) + sizeof(RuntimeTensorDesc));
110 3 : const int32_t *destIsTilingList = nullptr;
111 3 : if (mapInfo.len >= sizeof(uint32_t) + sizeof(uint64_t)) {
112 : const uint64_t *const destIsTilingListPtr =
113 2 : PtrToPtr<char, uint64_t>(mapInfo.extendInfo + sizeof(uint32_t));
114 2 : destIsTilingList = PtrToPtr<void, int32_t>(ValueToPtr(*destIsTilingListPtr));
115 : }
116 3 : if ((destIsTilingList != nullptr) && (destIsTilingList[i] == static_cast<int32_t>(true))) {
117 2 : *dstPtr = ValueToPtr(PtrToValue(srcDataPtr) + sizeof(RuntimeTensorDesc));
118 : }
119 1 : } else if (mapInfo.len >= sizeof(uint32_t)) {
120 1 : const uint32_t skipSize = *(PtrToPtr<char, uint32_t>(mapInfo.extendInfo));
121 1 : const uint64_t baseAddr = PtrToValue(srcDataPtr);
122 1 : if ((UINT64_MAX - baseAddr) < static_cast<uint64_t>(skipSize)) {
123 0 : aicpusd_err("AddrMapInfoV2 extendInfo skip size[%u] + baseAddr will overflow, modelId[%u], "
124 : "streamId[%u].", skipSize, taskContext.modelId, taskContext.streamId);
125 0 : return AICPU_SCHEDULE_ERROR_PARAMETER_NOT_VALID;
126 : }
127 1 : *dstPtr = ValueToPtr(baseAddr + static_cast<uint64_t>(skipSize));
128 : } else {}
129 :
130 4 : aicpusd_info("Zero cpy task success, addr index[%u], is notiling[%d].", i, isNoTilingList[i]);
131 : }
132 4 : return AICPU_SCHEDULE_OK;
133 10 : }
134 :
135 7 : int32_t OperatorKernelZeroCpyV2::ResolveFusionOffsets(const uint64_t *const fusionOffsetListAddr,
136 : const uint32_t addrNum,
137 : std::vector<int32_t> &fusionOffsets) const
138 : {
139 7 : if (fusionOffsetListAddr == nullptr) {
140 0 : aicpusd_err("The fusion offset list addr is null.");
141 0 : return AICPU_SCHEDULE_ERROR_PARAMETER_NOT_VALID;
142 : }
143 7 : auto fusionOffsetList = PtrToPtr<void, int32_t>(ValueToPtr(*fusionOffsetListAddr));
144 7 : if (fusionOffsetList == nullptr) {
145 2 : aicpusd_err("The fusion offset list is null");
146 2 : return AICPU_SCHEDULE_ERROR_PARAMETER_NOT_VALID;
147 : }
148 10 : for (uint32_t i = 0; i < addrNum; ++i) {
149 5 : fusionOffsets[i] = fusionOffsetList[i];
150 5 : aicpusd_info("Get fusion offset success, index = %u, fusion offset = %d.", i, fusionOffsetList[i]);
151 : }
152 5 : return AICPU_SCHEDULE_OK;
153 : }
154 :
155 6 : int32_t OperatorKernelZeroCpyV2::UpdateDataPtrExtend(const uint64_t mbufAddr, const int32_t fusionOffset, void *&dataPtr,
156 : std::unordered_map<uint64_t, FusionInfo> &fusionMap) const
157 : {
158 6 : const auto iter = fusionMap.find(mbufAddr);
159 6 : if (iter == fusionMap.end()) {
160 5 : uint64_t dataSize = 0UL;
161 5 : const int32_t ret = OperatorKernelCommon::GetMbufDataSize(mbufAddr, dataSize);
162 5 : if ((ret != AICPU_SCHEDULE_OK) || (dataSize < sizeof(RuntimeTensorDesc))) {
163 4 : aicpusd_err("Failed to get mbuf data size, ret = %d, dataSize[%lu] vs threshold[%zu]",
164 : ret, dataSize, sizeof(RuntimeTensorDesc));
165 4 : return (ret != AICPU_SCHEDULE_OK) ? ret : AICPU_SCHEDULE_ERROR_PARAMETER_NOT_VALID;
166 : }
167 1 : FusionInfo info = {};
168 1 : info.dataSize = dataSize;
169 1 : fusionMap[mbufAddr] = info;
170 : }
171 :
172 2 : auto &fusionInfo = fusionMap[mbufAddr];
173 2 : if (fusionInfo.lastFusionOffset > fusionOffset) {
174 1 : fusionInfo.lastFusionOffset = 0;
175 1 : fusionInfo.lastDataOffset = 0U;
176 : }
177 :
178 2 : return OperatorKernelCommon::DoUpdateDataPtr(fusionInfo, fusionOffset, dataPtr);
179 : }
180 :
181 3 : int32_t OperatorKernelCpuZeroCpy::Compute(const AicpuTaskInfo &kernelTaskInfo, const RunContext &taskContext)
182 : {
183 3 : AddrMapInfo *mapInfo = reinterpret_cast<AddrMapInfo *>(static_cast<uintptr_t>(kernelTaskInfo.paraBase));
184 3 : if (mapInfo == nullptr) {
185 1 : aicpusd_err("ModelZeroCpy kernelTaskInfo paramBase is null, modelId[%u], streamId[%u], taskId[%u]",
186 : taskContext.modelId, taskContext.streamId, kernelTaskInfo.taskID);
187 1 : return AICPU_SCHEDULE_ERROR_PARAMETER_NOT_VALID;
188 : }
189 2 : const uint64_t *const srcAddrList = PtrToPtr<void, uint64_t>(ValueToPtr(static_cast<uintptr_t>(mapInfo->srcAddrList)));
190 2 : const uint64_t *const dstAddrList = PtrToPtr<void, uint64_t>(ValueToPtr(static_cast<uintptr_t>(mapInfo->dstAddrList)));
191 2 : if ((srcAddrList == nullptr) || (dstAddrList == nullptr)) {
192 1 : aicpusd_err("Failed to zero copy, srcAddrList or dstAddrList is null, modelId[%u], streamId[%u]",
193 : taskContext.modelId, taskContext.streamId);
194 1 : return AICPU_SCHEDULE_ERROR_INNER_ERROR;
195 : }
196 1 : aicpusd_info("copy from %p to %p", srcAddrList, dstAddrList);
197 3 : for (uint32_t i = 0U; i < mapInfo->addrNum; i++) {
198 2 : const auto dstPtr = reinterpret_cast<void **>(ValueToPtr(dstAddrList[static_cast<size_t>(i)]));
199 2 : const auto srcPtr = reinterpret_cast<void **>(ValueToPtr(srcAddrList[static_cast<size_t>(i)]));
200 2 : *dstPtr = *srcPtr;
201 2 : aicpusd_info("copy element %p, srcPptr %p, dstPptr %p", *dstPtr, srcPtr, dstPtr);
202 : }
203 1 : return AICPU_SCHEDULE_OK;
204 : }
205 :
206 :
207 6 : REGISTER_OPERATOR_KERNEL(KERNEL_ZERO_CPY, OperatorKernelZeroCpy);
208 6 : REGISTER_OPERATOR_KERNEL(KERNEL_ZERO_CPY_V2, OperatorKernelZeroCpyV2);
209 6 : REGISTER_OPERATOR_KERNEL(KERNEL_CPU_ZERO_CPY, OperatorKernelCpuZeroCpy);
210 : } // namespace AicpuSchedule
|
