Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "operator_kernel_zero_cpy.h"
12 :
13 : #include "aicpusd_status.h"
14 : #include "operator_kernel_common.h"
15 :
16 : namespace AicpuSchedule {
17 : namespace {
18 : const std::string KERNEL_ZERO_CPY = "zeroCpy";
19 : const std::string KERNEL_ZERO_CPY_V2 = "zeroCpyV2";
20 : const std::string KERNEL_CPU_ZERO_CPY = "cpuZeroCpy";
21 : } // namespace
22 :
23 5 : int32_t OperatorKernelZeroCpy::Compute(const AicpuTaskInfo& kernelTaskInfo, const RunContext& taskContext)
24 : {
25 5 : AddrMapInfo* mapInfo = PtrToPtr<void, AddrMapInfo>(ValueToPtr(kernelTaskInfo.paraBase));
26 5 : if (mapInfo == nullptr) {
27 1 : aicpusd_err(
28 : "ModelZeroCpy kernelTaskInfo paramBase is null, modelId[%u], streamId[%u], taskId[%u]", taskContext.modelId,
29 : taskContext.streamId, kernelTaskInfo.taskID);
30 1 : return AICPU_SCHEDULE_ERROR_PARAMETER_NOT_VALID;
31 : }
32 :
33 4 : const uint64_t* const srcAddrList = PtrToPtr<void, uint64_t>(ValueToPtr(mapInfo->srcAddrList));
34 4 : const uint64_t* const dstAddrList = PtrToPtr<void, uint64_t>(ValueToPtr(mapInfo->dstAddrList));
35 4 : if ((srcAddrList == nullptr) || (dstAddrList == nullptr)) {
36 1 : aicpusd_err(
37 : "Failed to zero copy, srcAddrList or dstAddrList is null, modelId[%u], streamId[%u]", taskContext.modelId,
38 : taskContext.streamId);
39 1 : return AICPU_SCHEDULE_ERROR_INNER_ERROR;
40 : }
41 4 : for (uint32_t i = 0U; i < mapInfo->addrNum; i++) {
42 3 : void* dataPtr = nullptr;
43 3 : const int32_t ret = OperatorKernelCommon::GetMbufDataPtr(srcAddrList[i], &dataPtr);
44 3 : if (ret != AICPU_SCHEDULE_OK) {
45 2 : aicpusd_err("Failed to get mbuf data addr. srcAddrList[%u] is [%lu].", i, srcAddrList[i]);
46 2 : return ret;
47 : }
48 1 : const auto dstPtr = reinterpret_cast<void**>(static_cast<uintptr_t>(dstAddrList[i]));
49 1 : *dstPtr = dataPtr;
50 : }
51 1 : return AICPU_SCHEDULE_OK;
52 : }
53 :
54 11 : int32_t OperatorKernelZeroCpyV2::Compute(const AicpuTaskInfo& kernelTaskInfo, const RunContext& taskContext)
55 : {
56 11 : AddrMapInfoV2* const mapInfo = reinterpret_cast<AddrMapInfoV2*>(static_cast<uintptr_t>(kernelTaskInfo.paraBase));
57 11 : if (mapInfo == nullptr) {
58 1 : aicpusd_err(
59 : "ModelDynZeroCpy kernelTaskInfo paramBase is null, modelId[%u], streamId[%u], taskId[%u]",
60 : taskContext.modelId, taskContext.streamId, kernelTaskInfo.taskID);
61 1 : return AICPU_SCHEDULE_ERROR_PARAMETER_NOT_VALID;
62 : }
63 :
64 10 : return DoCompute(*mapInfo, taskContext);
65 : }
66 :
67 10 : int32_t OperatorKernelZeroCpyV2::DoCompute(AddrMapInfoV2& mapInfo, const RunContext& taskContext) const
68 : {
69 : const uint64_t* const srcAddrList =
70 10 : PtrToPtr<void, uint64_t>(ValueToPtr(static_cast<uintptr_t>(mapInfo.srcAddrList)));
71 : const uint64_t* const dstAddrList =
72 10 : PtrToPtr<void, uint64_t>(ValueToPtr(static_cast<uintptr_t>(mapInfo.dstAddrList)));
73 : const int32_t* const isNoTilingList =
74 10 : PtrToPtr<void, int32_t>(ValueToPtr(static_cast<uintptr_t>(mapInfo.isNoTilingList)));
75 10 : if ((srcAddrList == nullptr) || (dstAddrList == nullptr) || (isNoTilingList == nullptr)) {
76 0 : aicpusd_err(
77 : "Failed to zero copy, srcAddrList, dstAddrList or isNoTilingList is null, "
78 : "modelId[%u], streamId[%u]",
79 : taskContext.modelId, taskContext.streamId);
80 0 : return AICPU_SCHEDULE_ERROR_INNER_ERROR;
81 : }
82 :
83 10 : std::vector<int32_t> fusionOffsets(mapInfo.addrNum);
84 10 : if (mapInfo.len >= sizeof(uint32_t) + sizeof(uint64_t) + sizeof(uint64_t)) {
85 : const uint64_t* const fusionOffsetListAddr =
86 7 : PtrToPtr<char, uint64_t>(mapInfo.extendInfo + sizeof(uint32_t) + sizeof(uint64_t));
87 7 : const int32_t ret = ResolveFusionOffsets(fusionOffsetListAddr, mapInfo.addrNum, fusionOffsets);
88 7 : if (ret != AICPU_SCHEDULE_OK) {
89 2 : aicpusd_err("Failed to resolve fusion offsets, addr num = %u, ret = %d.", mapInfo.addrNum, ret);
90 2 : return ret;
91 : }
92 : }
93 :
94 8 : std::unordered_map<uint64_t, FusionInfo> fusionMap;
95 12 : for (uint32_t i = 0U; i < mapInfo.addrNum; i++) {
96 8 : void* srcDataPtr = nullptr;
97 8 : auto result = OperatorKernelCommon::GetMbufDataPtr(srcAddrList[i], &srcDataPtr);
98 8 : if (result != AICPU_SCHEDULE_OK) {
99 0 : aicpusd_err("Failed to get mbuf data addr. srcAddrList[%u] is [%lu].", i, srcAddrList[i]);
100 4 : return result;
101 : }
102 :
103 8 : if (fusionOffsets[i] > 0) {
104 5 : result = UpdateDataPtrExtend(srcAddrList[i], fusionOffsets[i], srcDataPtr, fusionMap);
105 5 : if (result != AICPU_SCHEDULE_OK) {
106 4 : aicpusd_err("Failed to update data addr. fusion offset[%d]", fusionOffsets[i]);
107 4 : return result;
108 : }
109 : }
110 4 : const auto dstPtr = reinterpret_cast<void**>(static_cast<uintptr_t>(dstAddrList[i]));
111 4 : *dstPtr = srcDataPtr;
112 : // if notiling will not skip
113 4 : if (isNoTilingList[i] != 0) {
114 3 : RuntimeTensorDesc* const tensorDesc = PtrToPtr<void, RuntimeTensorDesc>(srcDataPtr);
115 3 : tensorDesc->dataAddr =
116 3 : static_cast<uint64_t>(reinterpret_cast<uintptr_t>(srcDataPtr) + sizeof(RuntimeTensorDesc));
117 3 : const int32_t* destIsTilingList = nullptr;
118 3 : if (mapInfo.len >= sizeof(uint32_t) + sizeof(uint64_t)) {
119 : const uint64_t* const destIsTilingListPtr =
120 2 : PtrToPtr<char, uint64_t>(mapInfo.extendInfo + sizeof(uint32_t));
121 2 : destIsTilingList = PtrToPtr<void, int32_t>(ValueToPtr(*destIsTilingListPtr));
122 : }
123 3 : if ((destIsTilingList != nullptr) && (destIsTilingList[i] == static_cast<int32_t>(true))) {
124 2 : *dstPtr = ValueToPtr(PtrToValue(srcDataPtr) + sizeof(RuntimeTensorDesc));
125 : }
126 1 : } else if (mapInfo.len >= sizeof(uint32_t)) {
127 1 : const uint32_t skipSize = *(PtrToPtr<char, uint32_t>(mapInfo.extendInfo));
128 1 : const uint64_t baseAddr = PtrToValue(srcDataPtr);
129 1 : if ((UINT64_MAX - baseAddr) < static_cast<uint64_t>(skipSize)) {
130 0 : aicpusd_err(
131 : "AddrMapInfoV2 extendInfo skip size[%u] + baseAddr will overflow, modelId[%u], "
132 : "streamId[%u].",
133 : skipSize, taskContext.modelId, taskContext.streamId);
134 0 : return AICPU_SCHEDULE_ERROR_PARAMETER_NOT_VALID;
135 : }
136 1 : *dstPtr = ValueToPtr(baseAddr + static_cast<uint64_t>(skipSize));
137 : } else {
138 : }
139 :
140 4 : aicpusd_info("Zero cpy task success, addr index[%u], is notiling[%d].", i, isNoTilingList[i]);
141 : }
142 4 : return AICPU_SCHEDULE_OK;
143 10 : }
144 :
145 7 : int32_t OperatorKernelZeroCpyV2::ResolveFusionOffsets(
146 : const uint64_t* const fusionOffsetListAddr, const uint32_t addrNum, std::vector<int32_t>& fusionOffsets) const
147 : {
148 7 : if (fusionOffsetListAddr == nullptr) {
149 0 : aicpusd_err("The fusion offset list addr is null.");
150 0 : return AICPU_SCHEDULE_ERROR_PARAMETER_NOT_VALID;
151 : }
152 7 : auto fusionOffsetList = PtrToPtr<void, int32_t>(ValueToPtr(*fusionOffsetListAddr));
153 7 : if (fusionOffsetList == nullptr) {
154 2 : aicpusd_err("The fusion offset list is null");
155 2 : return AICPU_SCHEDULE_ERROR_PARAMETER_NOT_VALID;
156 : }
157 10 : for (uint32_t i = 0; i < addrNum; ++i) {
158 5 : fusionOffsets[i] = fusionOffsetList[i];
159 5 : aicpusd_info("Get fusion offset success, index = %u, fusion offset = %d.", i, fusionOffsetList[i]);
160 : }
161 5 : return AICPU_SCHEDULE_OK;
162 : }
163 :
164 6 : int32_t OperatorKernelZeroCpyV2::UpdateDataPtrExtend(
165 : const uint64_t mbufAddr, const int32_t fusionOffset, void*& dataPtr,
166 : std::unordered_map<uint64_t, FusionInfo>& fusionMap) const
167 : {
168 6 : const auto iter = fusionMap.find(mbufAddr);
169 6 : if (iter == fusionMap.end()) {
170 5 : uint64_t dataSize = 0UL;
171 5 : const int32_t ret = OperatorKernelCommon::GetMbufDataSize(mbufAddr, dataSize);
172 5 : if ((ret != AICPU_SCHEDULE_OK) || (dataSize < sizeof(RuntimeTensorDesc))) {
173 4 : aicpusd_err(
174 : "Failed to get mbuf data size, ret = %d, dataSize[%lu] vs threshold[%zu]", ret, dataSize,
175 : sizeof(RuntimeTensorDesc));
176 4 : return (ret != AICPU_SCHEDULE_OK) ? ret : AICPU_SCHEDULE_ERROR_PARAMETER_NOT_VALID;
177 : }
178 1 : FusionInfo info = {};
179 1 : info.dataSize = dataSize;
180 1 : fusionMap[mbufAddr] = info;
181 : }
182 :
183 2 : auto& fusionInfo = fusionMap[mbufAddr];
184 2 : if (fusionInfo.lastFusionOffset > fusionOffset) {
185 1 : fusionInfo.lastFusionOffset = 0;
186 1 : fusionInfo.lastDataOffset = 0U;
187 : }
188 :
189 2 : return OperatorKernelCommon::DoUpdateDataPtr(fusionInfo, fusionOffset, dataPtr);
190 : }
191 :
192 3 : int32_t OperatorKernelCpuZeroCpy::Compute(const AicpuTaskInfo& kernelTaskInfo, const RunContext& taskContext)
193 : {
194 3 : AddrMapInfo* mapInfo = reinterpret_cast<AddrMapInfo*>(static_cast<uintptr_t>(kernelTaskInfo.paraBase));
195 3 : if (mapInfo == nullptr) {
196 1 : aicpusd_err(
197 : "ModelZeroCpy kernelTaskInfo paramBase is null, modelId[%u], streamId[%u], taskId[%u]", taskContext.modelId,
198 : taskContext.streamId, kernelTaskInfo.taskID);
199 1 : return AICPU_SCHEDULE_ERROR_PARAMETER_NOT_VALID;
200 : }
201 : const uint64_t* const srcAddrList =
202 2 : PtrToPtr<void, uint64_t>(ValueToPtr(static_cast<uintptr_t>(mapInfo->srcAddrList)));
203 : const uint64_t* const dstAddrList =
204 2 : PtrToPtr<void, uint64_t>(ValueToPtr(static_cast<uintptr_t>(mapInfo->dstAddrList)));
205 2 : if ((srcAddrList == nullptr) || (dstAddrList == nullptr)) {
206 1 : aicpusd_err(
207 : "Failed to zero copy, srcAddrList or dstAddrList is null, modelId[%u], streamId[%u]", taskContext.modelId,
208 : taskContext.streamId);
209 1 : return AICPU_SCHEDULE_ERROR_INNER_ERROR;
210 : }
211 1 : aicpusd_info("copy from %p to %p", srcAddrList, dstAddrList);
212 3 : for (uint32_t i = 0U; i < mapInfo->addrNum; i++) {
213 2 : const auto dstPtr = reinterpret_cast<void**>(ValueToPtr(dstAddrList[static_cast<size_t>(i)]));
214 2 : const auto srcPtr = reinterpret_cast<void**>(ValueToPtr(srcAddrList[static_cast<size_t>(i)]));
215 2 : *dstPtr = *srcPtr;
216 2 : aicpusd_info("copy element %p, srcPptr %p, dstPptr %p", *dstPtr, srcPtr, dstPtr);
217 : }
218 1 : return AICPU_SCHEDULE_OK;
219 : }
220 :
221 6 : REGISTER_OPERATOR_KERNEL(KERNEL_ZERO_CPY, OperatorKernelZeroCpy);
222 6 : REGISTER_OPERATOR_KERNEL(KERNEL_ZERO_CPY_V2, OperatorKernelZeroCpyV2);
223 6 : REGISTER_OPERATOR_KERNEL(KERNEL_CPU_ZERO_CPY, OperatorKernelCpuZeroCpy);
224 : } // namespace AicpuSchedule
|
