Line data Source code
1 : /**
2 : * Copyright (c) 2026 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include <stdlib.h>
12 : #include <stdint.h>
13 : #include <errno.h>
14 : #include "securec.h"
15 : #include "dl_hal_function.h"
16 : #include "dl_ibverbs_function.h"
17 : #include "dl_ibv_extend_function.h"
18 : #include "hccp_nda.h"
19 : #include "ra_rs_err.h"
20 : #include "rs.h"
21 : #include "rs_inner.h"
22 : #include "rs_drv_rdma.h"
23 : #include "rs_rdma.h"
24 : #include "rs_nda.h"
25 :
26 2 : RS_ATTRI_VISI_DEF int RsNdaGetDirectFlag(unsigned int phyId, unsigned int rdevIndex, int *directFlag)
27 : {
28 2 : struct ibv_device_attr_extend extDevAttr = {0};
29 2 : struct RsRdevCb *rdevCb = NULL;
30 2 : int ret = 0;
31 :
32 2 : ret = RsQueryRdevCb(phyId, rdevIndex, &rdevCb);
33 2 : CHK_PRT_RETURN(ret != 0, hccp_err("RsQueryRdevCb phyId:%u rdevIndex:%u ret:%d", phyId, rdevIndex, ret), ret);
34 :
35 1 : if (rdevCb->ibCtxEx == NULL) {
36 1 : hccp_warn("ibCtxEx is null, phyId:%u rdevIndex:%u", phyId, rdevIndex);
37 1 : *directFlag = DIRECT_FLAG_NOTSUPP;
38 1 : return ret;
39 : }
40 :
41 0 : ret = RsIbvQueryDeviceExtend(rdevCb->ibCtxEx, &extDevAttr);
42 0 : CHK_PRT_RETURN(ret != 0,
43 : hccp_err("RsIbvQueryDeviceExtend failed, phyId:%u rdevIndex:%u ret:%d", phyId, rdevIndex, ret), ret);
44 :
45 0 : if ((extDevAttr.ext_cap & IBV_EXTEND_DEV_NDA) == 0) {
46 0 : hccp_warn("dev not support NDA, phyId:%u rdevIndex:%u ext_cap:0x%x", phyId, rdevIndex, extDevAttr.ext_cap);
47 0 : *directFlag = DIRECT_FLAG_NOTSUPP;
48 0 : return ret;
49 : }
50 :
51 0 : *directFlag = rdevCb->directFlag;
52 0 : return ret;
53 : }
54 :
55 32 : STATIC int RsNdaGetDirectFlagByDevAttr(struct ibv_device_attr *deviceAttr)
56 : {
57 : struct {
58 : uint32_t vendorId;
59 : uint32_t vendorPartID;
60 32 : } vendorInfoList[] = {
61 : {0x19E5U, 0x0230U},
62 : {0x19E5U, 0x0231U},
63 : {0x19E5U, 0x0232U},
64 : {0xCC08U, 0x8200U},
65 : {0xCC08U, 0x8201U},
66 : };
67 32 : unsigned int listSize = sizeof(vendorInfoList) / sizeof(vendorInfoList[0]);
68 : unsigned int i;
69 :
70 192 : for (i = 0; i < listSize; i++) {
71 160 : if ((deviceAttr->vendor_id == vendorInfoList[i].vendorId) &&
72 0 : (deviceAttr->vendor_part_id == vendorInfoList[i].vendorPartID)) {
73 0 : return DIRECT_FLAG_UB;
74 : }
75 : }
76 :
77 32 : return DIRECT_FLAG_PCIE;
78 : }
79 :
80 6 : STATIC int RsGetNdaPcieDbCb(struct RsNdaCb *ndaCb, uint64_t hva, struct NdaPcieDbCb **ndaDbCb)
81 : {
82 6 : struct NdaPcieDbCb *dbCbCurr = NULL;
83 6 : struct NdaPcieDbCb *dbCbNext = NULL;
84 :
85 6 : RS_LIST_GET_HEAD_ENTRY(dbCbCurr, dbCbNext, &ndaCb->ndaPcieCb.ndaDbList, list, struct NdaPcieDbCb);
86 6 : for (; (&dbCbCurr->list) != &ndaCb->ndaPcieCb.ndaDbList;
87 0 : dbCbCurr = dbCbNext, dbCbNext = list_entry(dbCbNext->list.next, struct NdaPcieDbCb, list)) {
88 3 : if (dbCbCurr->hva == hva) {
89 3 : *ndaDbCb = dbCbCurr;
90 3 : return 0;
91 : }
92 : }
93 :
94 3 : *ndaDbCb = NULL;
95 3 : hccp_info("ndaDbCb for hva:0x%llx does not exist", hva);
96 3 : return -ENODEV;
97 : }
98 :
99 7 : STATIC int RsGetNdaUbDbCb(struct RsNdaCb *ndaCb, uint64_t guidL, uint64_t guidH, struct NdaUbDbCb **ndaDbCb)
100 : {
101 7 : struct NdaUbDbCb *dbCbCurr = NULL;
102 7 : struct NdaUbDbCb *dbCbNext = NULL;
103 :
104 7 : RS_LIST_GET_HEAD_ENTRY(dbCbCurr, dbCbNext, &ndaCb->ndaUbCb.ndaDbList, list, struct NdaUbDbCb);
105 7 : for (; (&dbCbCurr->list) != &ndaCb->ndaUbCb.ndaDbList;
106 0 : dbCbCurr = dbCbNext, dbCbNext = list_entry(dbCbNext->list.next, struct NdaUbDbCb, list)) {
107 3 : if (dbCbCurr->guidL == guidL && dbCbCurr->guidH == guidH) {
108 3 : *ndaDbCb = dbCbCurr;
109 3 : return 0;
110 : }
111 : }
112 :
113 4 : *ndaDbCb = NULL;
114 4 : hccp_info("ndaDbCb for guidL:0x%llx guidH:0x%llx does not exist", guidL, guidH);
115 4 : return -ENODEV;
116 : }
117 :
118 0 : STATIC void *RsNdaPcieAlloc(size_t size)
119 : {
120 0 : struct RsNdaCb *ndaCb = (struct RsNdaCb *)gRsCb->ndaCb;
121 0 : void *ptr = NULL;
122 :
123 0 : CHK_PRT_RETURN(ndaCb == NULL || ndaCb->ndaOps.alloc == NULL,
124 : hccp_err("ndaCb or ndaOps.alloc is NULL, chipId:%u", gRsCb->chipId), NULL);
125 :
126 0 : ptr = ndaCb->ndaOps.alloc(size);
127 0 : CHK_PRT_RETURN(ptr == NULL, hccp_err("ptr alloc failed"), NULL);
128 :
129 0 : return ptr;
130 : }
131 :
132 0 : STATIC void RsNdaPcieFree(void *ptr)
133 : {
134 0 : struct RsNdaCb *ndaCb = (struct RsNdaCb *)gRsCb->ndaCb;
135 :
136 0 : if (ndaCb == NULL || ndaCb->ndaOps.free == NULL) {
137 0 : hccp_err("gRsCb->ndaCb or ndaOps.free is NULL, chipId:%u", gRsCb->chipId);
138 0 : return;
139 : }
140 :
141 0 : ndaCb->ndaOps.free(ptr);
142 0 : ptr = NULL;
143 : }
144 :
145 4 : STATIC void *RsNdaUbAlloc(size_t size)
146 : {
147 4 : struct RsNdaCb *ndaCb = (struct RsNdaCb *)gRsCb->ndaCb;
148 4 : struct DVattribute attr = {0};
149 4 : void *ptr = NULL;
150 4 : int ret = 0;
151 :
152 4 : CHK_PRT_RETURN(ndaCb == NULL || ndaCb->ndaOps.alloc == NULL || ndaCb->ndaOps.free == NULL,
153 : hccp_err("ndaCb or alloc or free is NULL, chipId:%u", gRsCb->chipId), NULL);
154 :
155 2 : ptr = ndaCb->ndaOps.alloc(size);
156 2 : CHK_PRT_RETURN(ptr == NULL, hccp_err("ptr alloc failed"), NULL);
157 :
158 2 : ret = DlDrvMemGetAttribute((uint64_t)(uintptr_t)ptr, &attr);
159 2 : if (ret != 0) {
160 1 : hccp_err("DlDrvMemGetAttribute failed, ret:%d", ret);
161 1 : goto free_ptr;
162 : }
163 :
164 1 : if (attr.memType == DV_MEM_LOCK_DEV) {
165 1 : ret = DlHalMemRegUbSegment(attr.devId, (uint64_t)(uintptr_t)ptr, size);
166 1 : if (ret != 0) {
167 1 : hccp_err("DlHalMemRegUbSegment failed, ret:%d devId:%u size:%zu", ret, attr.devId, size);
168 1 : goto free_ptr;
169 : }
170 : }
171 :
172 0 : return ptr;
173 :
174 2 : free_ptr:
175 2 : ndaCb->ndaOps.free(ptr);
176 2 : ptr = NULL;
177 2 : return NULL;
178 : }
179 :
180 0 : STATIC void RsNdaUbFree(void *ptr)
181 : {
182 0 : struct RsNdaCb *ndaCb = (struct RsNdaCb *)gRsCb->ndaCb;
183 0 : struct DVattribute attr = {0};
184 0 : int ret = 0;
185 :
186 0 : if (ndaCb == NULL || ndaCb->ndaOps.free == NULL) {
187 0 : hccp_err("gRsCb->ndaCb or ndaOps.free is NULL, chipId:%u", gRsCb->chipId);
188 0 : return;
189 : }
190 :
191 0 : ret = DlDrvMemGetAttribute((uint64_t)(uintptr_t)ptr, &attr);
192 0 : if (ret != 0) {
193 0 : hccp_err("DlDrvMemGetAttribute failed, ret:%d", ret);
194 0 : goto free_ptr;
195 : }
196 :
197 0 : if (attr.memType == DV_MEM_LOCK_DEV) {
198 0 : (void)DlHalMemUnRegUbSegment(attr.devId, (uint64_t)(uintptr_t)ptr);
199 : }
200 :
201 0 : free_ptr:
202 0 : ndaCb->ndaOps.free(ptr);
203 0 : ptr = NULL;
204 : }
205 :
206 0 : STATIC void RsNdaMemset(void *dst, int value, size_t count)
207 : {
208 0 : struct RsNdaCb *ndaCb = (struct RsNdaCb *)gRsCb->ndaCb;
209 :
210 0 : if (ndaCb == NULL || ndaCb->ndaOps.memset_s == NULL) {
211 0 : hccp_err("gRsCb->ndaCb or ndaOps.memset_s is NULL, chipId:%u", gRsCb->chipId);
212 0 : return;
213 : }
214 :
215 0 : ndaCb->ndaOps.memset_s(dst, value, count);
216 : }
217 :
218 0 : STATIC int RsNdaMemcpy(void *dst, size_t dstSize, void *src, size_t srcSize, uint32_t direct)
219 : {
220 0 : struct RsNdaCb *ndaCb = (struct RsNdaCb *)gRsCb->ndaCb;
221 :
222 0 : if (ndaCb == NULL || ndaCb->ndaOps.memcpy_s == NULL) {
223 0 : hccp_err("gRsCb->ndaCb or ndaOps.memcpy_s is NULL, chipId:%u", gRsCb->chipId);
224 0 : return -ENODEV;
225 : }
226 :
227 0 : return ndaCb->ndaOps.memcpy_s(dst, dstSize, src, srcSize, direct);
228 : }
229 :
230 5 : STATIC void RsNdaFreeUbDbCb(struct NdaUbDbCb *ndaDbCb)
231 : {
232 5 : if (ndaDbCb->refCnt != 0) {
233 1 : return;
234 : }
235 :
236 4 : RsListDel(&ndaDbCb->list);
237 4 : free(ndaDbCb);
238 4 : ndaDbCb = NULL;
239 : }
240 :
241 4 : STATIC void *RsNdaDbMmapHostVa(struct RsNdaCb *ndaCb, struct doorbell_map_desc *desc)
242 : {
243 4 : uint64_t alignHva = AlignDown(desc->hva, (uint64_t)RA_RS_4K_PAGE_SIZE);
244 4 : uint64_t alignSize = AlignUp(desc->size, (uint64_t)RA_RS_4K_PAGE_SIZE);
245 4 : uint64_t offset = desc->hva - alignHva;
246 4 : unsigned int logicId = gRsCb->logicId;
247 4 : struct NdaPcieDbCb *ndaDbCb = NULL;
248 4 : void *dbDva = NULL;
249 4 : int ret = 0;
250 :
251 4 : ret = RsGetNdaPcieDbCb(ndaCb, alignHva, &ndaDbCb);
252 4 : if (ret == 0) {
253 1 : ndaDbCb->refCnt++;
254 1 : return (void *)(uintptr_t)(ndaDbCb->dva + offset);
255 : }
256 :
257 3 : ndaDbCb = (struct NdaPcieDbCb *)calloc(1, sizeof(struct NdaPcieDbCb));
258 3 : CHK_PRT_RETURN(ndaDbCb == NULL, hccp_err("ndaDbCb calloc failed"), NULL);
259 :
260 3 : ret = DlHalHostRegister((void *)(uintptr_t)alignHva, alignSize, HOST_IO_MAP_DEV, logicId, &dbDva);
261 3 : if (ret != 0) {
262 2 : hccp_err("register host failed, chipId:%u logicId:%u ret:%d alignHva:0x%llx", gRsCb->chipId, logicId, ret,
263 : alignHva);
264 2 : free(ndaDbCb);
265 2 : ndaDbCb = NULL;
266 2 : return NULL;
267 : }
268 :
269 1 : ndaDbCb->hva = alignHva;
270 1 : ndaDbCb->dva = (uint64_t)(uintptr_t)dbDva;
271 1 : ndaDbCb->refCnt++;
272 1 : RsListAddTail(&ndaDbCb->list, &ndaCb->ndaPcieCb.ndaDbList);
273 1 : return (void *)(uintptr_t)(ndaDbCb->dva + offset);
274 : }
275 :
276 7 : STATIC void RsNdaMapPrivPrepare(struct doorbell_map_desc *desc, struct NdaUbResMapPrivInfo *resMapIn)
277 : {
278 7 : resMapIn->guid_l = desc->ub_res.guid_l;
279 7 : resMapIn->guid_h = desc->ub_res.guid_h;
280 7 : resMapIn->db_idx = desc->ub_res.bits.offset / (uint64_t)RA_RS_4K_PAGE_SIZE;
281 7 : resMapIn->db_num = 1;
282 7 : return;
283 : }
284 :
285 5 : STATIC void *RsNdaDbMmapUbRes(struct RsNdaCb *ndaCb, struct doorbell_map_desc *desc)
286 : {
287 5 : struct NdaUbResMapPrivInfo resMapIn = {0};
288 5 : struct res_map_info_out resInfoOut = {0};
289 5 : struct res_map_info_in resInfoIn = {0};
290 5 : unsigned int logicId = gRsCb->logicId;
291 5 : struct NdaUbDbCb *ndaDbCb = NULL;
292 5 : uint64_t dva = 0;
293 5 : int ret = 0;
294 :
295 5 : ret = RsGetNdaUbDbCb(ndaCb, desc->ub_res.guid_l, desc->ub_res.guid_h, &ndaDbCb);
296 5 : if (ret == 0) {
297 1 : goto map_db;
298 : }
299 :
300 4 : ndaDbCb = (struct NdaUbDbCb *)calloc(1, sizeof(struct NdaUbDbCb));
301 4 : CHK_PRT_RETURN(ndaDbCb == NULL, hccp_err("ndaDbCb calloc failed"), NULL);
302 4 : ndaDbCb->guidL = desc->ub_res.guid_l;
303 4 : ndaDbCb->guidH = desc->ub_res.guid_h;
304 4 : ndaDbCb->guidIdx = ndaCb->ndaUbCb.ndaDbGuidCnt;
305 4 : RsListAddTail(&ndaDbCb->list, &ndaCb->ndaUbCb.ndaDbList);
306 4 : ndaCb->ndaUbCb.ndaDbGuidCnt++;
307 :
308 5 : map_db:
309 5 : RsNdaMapPrivPrepare(desc, &resMapIn);
310 5 : resInfoIn.target_proc_type = PROCESS_CP1;
311 5 : resInfoIn.res_type = RES_ADDR_TYPE_NDA_URMA_DB;
312 5 : resInfoIn.res_id = RsNdaGenerateResId(resMapIn.db_idx, ndaDbCb->guidIdx);
313 5 : resInfoIn.priv_len = sizeof(struct NdaUbResMapPrivInfo);
314 5 : resInfoIn.priv = (void *)&resMapIn;
315 5 : ret = DlHalResAddrMapV2(logicId, &resInfoIn, &resInfoOut);
316 5 : if (ret != 0) {
317 3 : hccp_err("DlHalResAddrMapV2 failed, chipId:%u logicId:%u resId:0x%x ret:%d", gRsCb->chipId, logicId,
318 : resInfoIn.res_id, ret);
319 3 : RsNdaFreeUbDbCb(ndaDbCb);
320 3 : return NULL;
321 : }
322 :
323 2 : ndaDbCb->refCnt++;
324 2 : dva = resInfoOut.va + (desc->ub_res.bits.offset % (uint64_t)RA_RS_4K_PAGE_SIZE);
325 2 : return (void *)(uintptr_t)dva;
326 : }
327 :
328 11 : STATIC void *RsNdaDbMmap(struct doorbell_map_desc *desc)
329 : {
330 11 : struct RsNdaCb *ndaCb = (struct RsNdaCb *)gRsCb->ndaCb;
331 :
332 11 : CHK_PRT_RETURN(desc == NULL, hccp_err("desc is null"), NULL);
333 11 : CHK_PRT_RETURN(ndaCb == NULL, hccp_err("ndaCb is null, chipId:%u", gRsCb->chipId), NULL);
334 :
335 11 : if (desc->type == DB_MAP_MODE_HOST_VA) {
336 4 : return RsNdaDbMmapHostVa(ndaCb, desc);
337 7 : } else if (desc->type == DB_MAP_MODE_UB_RES) {
338 5 : return RsNdaDbMmapUbRes(ndaCb, desc);
339 : } else {
340 2 : hccp_err("invalid desc->type:%u, chipId:%u", desc->type, gRsCb->chipId);
341 2 : return NULL;
342 : }
343 : }
344 :
345 3 : STATIC int RsNdaDbUnmapHostVa(struct RsNdaCb *ndaCb, void *ptr, struct doorbell_map_desc *desc)
346 : {
347 : (void)ptr;
348 3 : uint64_t alignHva = AlignDown(desc->hva, (uint64_t)RA_RS_4K_PAGE_SIZE);
349 3 : unsigned int logicId = gRsCb->logicId;
350 3 : struct NdaPcieDbCb *ndaDbCb = NULL;
351 3 : int ret = 0;
352 :
353 3 : ret = RsGetNdaPcieDbCb(ndaCb, alignHva, &ndaDbCb);
354 3 : CHK_PRT_RETURN(ret != 0,
355 : hccp_err("RsGetNdaPcieDbCb failed, hva:0x%llx chipId:%u ret:%d", alignHva, gRsCb->chipId, ret), ret);
356 :
357 2 : ndaDbCb->refCnt--;
358 2 : if (ndaDbCb->refCnt != 0) {
359 1 : return ret;
360 : }
361 :
362 1 : ret = DlHalHostUnRegisterEx((void *)(uintptr_t)alignHva, logicId, HOST_IO_MAP_DEV);
363 1 : if (ret != 0) {
364 0 : hccp_err("DlHalHostUnRegisterEx failed, chipId:%u logicId:%u ret:%d", gRsCb->chipId, logicId, ret);
365 : }
366 :
367 1 : RsListDel(&ndaDbCb->list);
368 1 : free(ndaDbCb);
369 1 : ndaDbCb = NULL;
370 1 : return ret;
371 : }
372 :
373 3 : STATIC int RsNdaDbUnmapUbRes(struct RsNdaCb *ndaCb, void *ptr, struct doorbell_map_desc *desc)
374 : {
375 : (void)ptr;
376 3 : struct NdaUbResMapPrivInfo resMapIn = {0};
377 3 : struct res_map_info_in resInfoIn = {0};
378 3 : unsigned int logicId = gRsCb->logicId;
379 3 : struct NdaUbDbCb *ndaDbCb = NULL;
380 3 : int ret = 0;
381 :
382 3 : ret = RsGetNdaUbDbCb(ndaCb, desc->ub_res.guid_l, desc->ub_res.guid_h, &ndaDbCb);
383 3 : CHK_PRT_RETURN(ret != 0,
384 : hccp_err("RsGetNdaUbDbCb failed, chipId:%u guidL:0x%llx guidH:0x%llx", gRsCb->chipId, desc->ub_res.guid_l,
385 : desc->ub_res.guid_h),
386 : ret);
387 :
388 2 : RsNdaMapPrivPrepare(desc, &resMapIn);
389 2 : resInfoIn.target_proc_type = PROCESS_CP1;
390 2 : resInfoIn.res_type = RES_ADDR_TYPE_NDA_URMA_DB;
391 2 : resInfoIn.res_id = RsNdaGenerateResId(resMapIn.db_idx, ndaDbCb->guidIdx);
392 2 : resInfoIn.priv_len = sizeof(struct NdaUbResMapPrivInfo);
393 2 : resInfoIn.priv = (void *)&resMapIn;
394 2 : ret = DlHalResAddrUnmapV2(logicId, &resInfoIn);
395 2 : if (ret != 0) {
396 0 : hccp_err("DlHalResAddrUnmapV2 failed, chipId:%u logicId:%u resId:0x%x ret:%d", gRsCb->chipId, logicId,
397 : resInfoIn.res_id, ret);
398 : }
399 :
400 2 : ndaDbCb->refCnt--;
401 2 : RsNdaFreeUbDbCb(ndaDbCb);
402 2 : return ret;
403 : }
404 :
405 6 : STATIC int RsNdaDbUnmap(void *ptr, struct doorbell_map_desc *desc)
406 : {
407 6 : struct RsNdaCb *ndaCb = (struct RsNdaCb *)gRsCb->ndaCb;
408 :
409 6 : CHK_PRT_RETURN(ptr == NULL || desc == NULL, hccp_err("ptr or desc is null"), -EINVAL);
410 6 : CHK_PRT_RETURN(ndaCb == NULL, hccp_err("ndaCb is null, chipId:%u", gRsCb->chipId), -ENODEV);
411 :
412 6 : if (desc->type == DB_MAP_MODE_HOST_VA) {
413 3 : return RsNdaDbUnmapHostVa(ndaCb, ptr, desc);
414 3 : } else if (desc->type == DB_MAP_MODE_UB_RES) {
415 3 : return RsNdaDbUnmapUbRes(ndaCb, ptr, desc);
416 : } else {
417 0 : hccp_err("invalid desc->type:%u, chipId:%u", desc->type, gRsCb->chipId);
418 0 : return -EINVAL;
419 : }
420 : }
421 :
422 32 : STATIC void RsNdaCbInitCb(struct RsNdaCb *ndaCb)
423 : {
424 32 : ndaCb->ndaPcieCb.ibvExOps.alloc = RsNdaPcieAlloc;
425 32 : ndaCb->ndaPcieCb.ibvExOps.free = RsNdaPcieFree;
426 32 : ndaCb->ndaPcieCb.ibvExOps.db_mmap = RsNdaDbMmap;
427 32 : ndaCb->ndaPcieCb.ibvExOps.db_unmap = RsNdaDbUnmap;
428 32 : ndaCb->ndaPcieCb.ibvExOps.memset_s = RsNdaMemset;
429 32 : ndaCb->ndaPcieCb.ibvExOps.memcpy_s = RsNdaMemcpy;
430 :
431 32 : ndaCb->ndaUbCb.ibvExOps.alloc = RsNdaUbAlloc;
432 32 : ndaCb->ndaUbCb.ibvExOps.free = RsNdaUbFree;
433 32 : ndaCb->ndaUbCb.ibvExOps.db_mmap = RsNdaDbMmap;
434 32 : ndaCb->ndaUbCb.ibvExOps.db_unmap = RsNdaDbUnmap;
435 32 : ndaCb->ndaUbCb.ibvExOps.memset_s = RsNdaMemset;
436 32 : ndaCb->ndaUbCb.ibvExOps.memcpy_s = RsNdaMemcpy;
437 :
438 32 : RS_INIT_LIST_HEAD(&ndaCb->ndaPcieCb.ndaDbList);
439 32 : RS_INIT_LIST_HEAD(&ndaCb->ndaUbCb.ndaDbList);
440 32 : }
441 :
442 32 : int RsInitNdaCb(struct RsRdevCb *rdevCb)
443 : {
444 32 : struct RsNdaCb *ndaCb = NULL;
445 32 : int count = 0;
446 32 : int ret = 0;
447 :
448 32 : rdevCb->directFlag = RsNdaGetDirectFlagByDevAttr(&rdevCb->deviceAttr);
449 :
450 32 : rdevCb->ibCtxEx = RsIbvOpenExtend(rdevCb->ibCtx);
451 32 : if (rdevCb->ibCtxEx == NULL) {
452 0 : hccp_warn("ibCtxEx is null");
453 0 : return 0;
454 : }
455 :
456 32 : count = __sync_fetch_and_add(&rdevCb->rsCb->ndaCbRefCnt, 1);
457 32 : if (count > 0) {
458 0 : hccp_info("RsNdaCb exist, ndaCbRefCnt:%d", rdevCb->rsCb->ndaCbRefCnt);
459 0 : return 0;
460 : }
461 :
462 32 : ndaCb = (struct RsNdaCb *)calloc(1, sizeof(struct RsNdaCb));
463 32 : if (ndaCb == NULL) {
464 0 : hccp_err("calloc for ndaCb failed");
465 0 : ret = -ENOMEM;
466 0 : goto calloc_err;
467 : }
468 :
469 32 : RsNdaCbInitCb(ndaCb);
470 :
471 32 : rdevCb->rsCb->ndaCb = (void *)ndaCb;
472 32 : return ret;
473 :
474 0 : calloc_err:
475 0 : (void)__sync_fetch_and_sub(&rdevCb->rsCb->ndaCbRefCnt, 1);
476 0 : (void)RsIbvCloseExtend(rdevCb->ibCtxEx);
477 0 : rdevCb->ibCtxEx = NULL;
478 0 : return ret;
479 : }
480 :
481 32 : void RsDeinitNdaCb(struct RsRdevCb *rdevCb)
482 : {
483 32 : if (rdevCb->ibCtxEx == NULL) {
484 1 : return;
485 : }
486 :
487 31 : (void)RsIbvCloseExtend(rdevCb->ibCtxEx);
488 31 : rdevCb->ibCtxEx = NULL;
489 :
490 31 : if (__sync_fetch_and_sub(&rdevCb->rsCb->ndaCbRefCnt, 1) > 1) {
491 0 : return;
492 : }
493 :
494 31 : free(rdevCb->rsCb->ndaCb);
495 31 : rdevCb->rsCb->ndaCb = NULL;
496 : }
497 :
498 4 : STATIC void RsNdaInitExOps(struct RsNdaCb *ndaCb, uint32_t dmaMode, struct NdaOps *ops, struct ibv_extend_ops **extOps)
499 : {
500 4 : ndaCb->ndaOps.alloc = ops->alloc;
501 4 : ndaCb->ndaOps.free = ops->free;
502 4 : ndaCb->ndaOps.memset_s = ops->memset_s;
503 4 : ndaCb->ndaOps.memcpy_s = ops->memcpy_s;
504 4 : if (dmaMode == QBUF_DMA_MODE_DEFAULT) {
505 2 : *extOps = &ndaCb->ndaPcieCb.ibvExOps;
506 2 : } else if (dmaMode == QBUF_DMA_MODE_INDEP_UB) {
507 2 : *extOps = &ndaCb->ndaUbCb.ibvExOps;
508 : } else {
509 0 : *extOps = NULL;
510 : }
511 4 : }
512 :
513 2 : STATIC void RsNdaCqInitExPrepare(struct NdaCqInitAttr *attr, struct RsNdaCb *ndaCb,
514 : struct ibv_cq_init_attr_extend *cqInitAttrEx)
515 : {
516 2 : (void)memcpy_s(&cqInitAttrEx->attr, sizeof(struct ibv_cq_init_attr_ex), &attr->attr,
517 : sizeof(struct ibv_cq_init_attr_ex));
518 2 : cqInitAttrEx->cq_cap_flag = attr->cqCapFlag;
519 2 : cqInitAttrEx->type = attr->dmaMode;
520 2 : RsNdaInitExOps(ndaCb, attr->dmaMode, attr->ops, &cqInitAttrEx->ops);
521 2 : }
522 :
523 1 : STATIC int RsNdaCqCreateEx(struct RsRdevCb *rdevCb, struct ibv_cq_init_attr_extend *cqInitAttrEx,
524 : struct NdaCqInfo *info, void **ibvCqExt)
525 : {
526 1 : struct ibv_cq_extend *cqExt = NULL;
527 1 : int ret = 0;
528 :
529 1 : cqExt = RsIbvCreateCqExtend(rdevCb->ibCtxEx, cqInitAttrEx);
530 1 : CHK_PRT_RETURN(cqExt == NULL, hccp_err("RsNdaCreateCqExtend failed, errno:%d", errno), -ENOMEM);
531 :
532 1 : ret = memcpy_s(&info->cqInfo, sizeof(struct queueInfo), &cqExt->cq_info, sizeof(struct queue_info));
533 1 : CHK_PRT_RETURN(ret != 0, hccp_err("memcpy_s queue_info failed, ret:%d", ret), -ESAFEFUNC);
534 1 : ret = memcpy_s(info->resv, sizeof(info->resv), cqExt->resv, sizeof(cqExt->resv));
535 1 : CHK_PRT_RETURN(ret != 0, hccp_err("memcpy_s resv failed, ret:%d", ret), -ESAFEFUNC);
536 1 : info->cq = cqExt->cq;
537 1 : *ibvCqExt = cqExt;
538 1 : return 0;
539 : }
540 :
541 5 : RS_ATTRI_VISI_DEF int RsNdaCqCreate(unsigned int phyId, unsigned int rdevIndex, struct NdaCqInitAttr *attr,
542 : struct NdaCqInfo *info, void **ibvCqExt)
543 : {
544 5 : struct ibv_cq_init_attr_extend cqInitAttrEx = {0};
545 5 : struct RsRdevCb *rdevCb = NULL;
546 5 : struct RsNdaCb *ndaCb = NULL;
547 5 : int ret = 0;
548 :
549 5 : CHK_PRT_RETURN(attr == NULL || info == NULL || ibvCqExt == NULL,
550 : hccp_err("attr or info or ibvCqExt is NULL, phyId:%u", phyId), -EINVAL);
551 4 : CHK_PRT_RETURN(attr->dmaMode >= QBUF_DMA_MODE_MAX,
552 : hccp_err("param err, dmaMode:%u >= %u, phyId:%u", attr->dmaMode, QBUF_DMA_MODE_MAX, phyId), -EINVAL);
553 :
554 3 : ret = RsQueryRdevCb(phyId, rdevIndex, &rdevCb);
555 3 : CHK_PRT_RETURN(ret != 0, hccp_err("RsQueryRdevCb failed, phyId:%u rdevIndex:%u ret:%d", phyId, rdevIndex, ret),
556 : ret);
557 :
558 2 : ndaCb = (struct RsNdaCb *)rdevCb->rsCb->ndaCb;
559 2 : CHK_PRT_RETURN(ndaCb == NULL, hccp_err("ndaCb is NULL, does not support nda, phyId:%u", phyId), -ENODEV);
560 :
561 2 : RsNdaCqInitExPrepare(attr, ndaCb, &cqInitAttrEx);
562 :
563 2 : ret = RsNdaCqCreateEx(rdevCb, &cqInitAttrEx, info, ibvCqExt);
564 2 : CHK_PRT_RETURN(ret != 0, hccp_err("RsNdaCqCreateEx failed, phyId:%u rdevIndex:%u ret:%d", phyId, rdevIndex, ret),
565 : ret);
566 :
567 1 : return ret;
568 : }
569 :
570 1 : RS_ATTRI_VISI_DEF int RsNdaCqDestroy(unsigned int phyId, unsigned int rdevIndex, void *ibvCqExt)
571 : {
572 1 : struct RsRdevCb *rdevCb = NULL;
573 1 : int ret = 0;
574 :
575 1 : ret = RsQueryRdevCb(phyId, rdevIndex, &rdevCb);
576 1 : CHK_PRT_RETURN(ret != 0, hccp_err("RsQueryRdevCb failed, phyId:%u rdevIndex:%u ret:%d", phyId, rdevIndex, ret),
577 : ret);
578 :
579 1 : ret = RsIbvDestroyCqExtend(rdevCb->ibCtxEx, ibvCqExt);
580 1 : CHK_PRT_RETURN(ret != 0,
581 : hccp_err("RsIbvDestroyCqExtend failed, phyId:%u rdevIndex:%u ret:%d", phyId, rdevIndex, ret), ret);
582 :
583 1 : return ret;
584 : }
585 :
586 2 : STATIC int RsBuildUpNdaQpcb(struct RsRdevCb *rdevCb, struct ibv_qp_init_attr *qpInitAttr, struct RsQpCb **qpCb)
587 : {
588 2 : int ret = 0;
589 :
590 2 : *qpCb = (struct RsQpCb *)calloc(1, sizeof(struct RsQpCb));
591 2 : CHK_PRT_RETURN(*qpCb == NULL, hccp_err("RsQpCb calloc failed"), -ENOMEM);
592 :
593 2 : ret = pthread_mutex_init(&(*qpCb)->qpMutex, NULL);
594 2 : if (ret != 0) {
595 0 : hccp_err("pthread_mutex_init failed, ret:%d", ret);
596 0 : goto qp_mutex_init_err;
597 : }
598 :
599 2 : (*qpCb)->rdevCb = rdevCb;
600 2 : RS_INIT_LIST_HEAD(&(*qpCb)->mrList);
601 2 : RS_INIT_LIST_HEAD(&(*qpCb)->remMrList);
602 2 : (*qpCb)->state = RS_QP_STATUS_DISCONNECT;
603 2 : (*qpCb)->ibPd = rdevCb->ibPd;
604 2 : (*qpCb)->txDepth = qpInitAttr->cap.max_send_wr;
605 2 : (*qpCb)->rxDepth = qpInitAttr->cap.max_recv_wr;
606 :
607 2 : (*qpCb)->numRecvCqEvents = 0;
608 2 : (*qpCb)->numSendCqEvents = 0;
609 2 : (*qpCb)->qosAttr.tc = (RS_ROCE_DSCP_33 & RS_DSCP_MASK) << RS_DSCP_OFF;
610 2 : (*qpCb)->qosAttr.sl = RS_ROCE_4_SL;
611 2 : (*qpCb)->timeout = RS_QP_ATTR_TIMEOUT;
612 2 : (*qpCb)->retryCnt = RS_QP_ATTR_RETRY_CNT;
613 :
614 2 : return ret;
615 :
616 0 : qp_mutex_init_err:
617 0 : free(*qpCb);
618 0 : *qpCb = NULL;
619 0 : return ret;
620 : }
621 :
622 2 : STATIC void RsNdaQpInitExPrepare(struct RsRdevCb *rdevCb, struct NdaQpInitAttr *attr,
623 : struct ibv_qp_init_attr_extend *qpInitAttrEx)
624 : {
625 2 : struct RsNdaCb *ndaCb = (struct RsNdaCb *)rdevCb->rsCb->ndaCb;
626 :
627 2 : qpInitAttrEx->pd = rdevCb->ibPd;
628 2 : (void)memcpy_s(&qpInitAttrEx->attr, sizeof(struct ibv_qp_init_attr), &attr->attr, sizeof(struct ibv_qp_init_attr));
629 2 : qpInitAttrEx->qp_cap_flag = attr->qpCapFlag;
630 2 : qpInitAttrEx->type = attr->dmaMode;
631 2 : RsNdaInitExOps(ndaCb, attr->dmaMode, attr->ops, &qpInitAttrEx->ops);
632 2 : }
633 :
634 1 : STATIC int RsNdaQpGetQpInfo(struct RsQpCb *qpCb, struct NdaQpInfo *info)
635 : {
636 1 : int ret = 0;
637 :
638 1 : info->qp = qpCb->ibQp;
639 1 : ret = memcpy_s(&info->sqInfo, sizeof(struct queueInfo), &qpCb->ibQpEx->sq_info, sizeof(struct queue_info));
640 1 : CHK_PRT_RETURN(ret != 0, hccp_err("memcpy_s sq_info failed, ret:%d", ret), -ESAFEFUNC);
641 1 : ret = memcpy_s(&info->rqInfo, sizeof(struct queueInfo), &qpCb->ibQpEx->rq_info, sizeof(struct queue_info));
642 1 : CHK_PRT_RETURN(ret != 0, hccp_err("memcpy_s rq_info failed, ret:%d", ret), -ESAFEFUNC);
643 1 : ret = memcpy_s(info->resv, sizeof(info->resv), qpCb->ibQpEx->resv, sizeof(qpCb->ibQpEx->resv));
644 1 : CHK_PRT_RETURN(ret != 0, hccp_err("memcpy_s resv failed, ret:%d", ret), -ESAFEFUNC);
645 :
646 1 : return ret;
647 : }
648 :
649 3 : STATIC int RsNdaQpCreateEx(struct RsQpCb *qpCb, struct ibv_qp_init_attr_extend *qpInitAttrEx, struct NdaQpInfo *info)
650 : {
651 3 : struct ibv_qp_attr qpAttr = {0};
652 3 : struct ibv_port_attr attr = {0};
653 3 : struct RsRdevCb *rdevCb = NULL;
654 3 : int ret = 0;
655 :
656 3 : rdevCb = qpCb->rdevCb;
657 3 : qpCb->ibQpEx = RsIbvCreateQpExtend(rdevCb->ibCtxEx, qpInitAttrEx);
658 3 : CHK_PRT_RETURN(qpCb->ibQpEx == NULL, hccp_err("RsNdaCreateQpExtend failed, errno:%d", errno), -ENOMEM);
659 :
660 3 : qpCb->ibQp = qpCb->ibQpEx->qp;
661 3 : ret = RsIbvQueryQp(qpCb->ibQp, &qpAttr, IBV_QP_CAP, &qpInitAttrEx->attr);
662 3 : if (ret != 0) {
663 1 : hccp_err("query qp attr failed ret:%d", ret);
664 1 : ret = -EOPENSRC;
665 1 : goto nda_init_qp_err;
666 : }
667 :
668 2 : ret = RsDrvQpInfoRelated(qpCb, rdevCb, &attr, &qpAttr);
669 2 : if (ret != 0) {
670 1 : hccp_err("qp info related failed ret:%d", ret);
671 1 : goto nda_init_qp_err;
672 : }
673 :
674 1 : ret = RsNdaQpGetQpInfo(qpCb, info);
675 1 : if (ret != 0) {
676 0 : hccp_err("RsNdaQpGetQpInfo failed ret:%d", ret);
677 0 : goto nda_init_qp_err;
678 : }
679 :
680 1 : hccp_info("chip_id:%u, rdevIndex:%u, qp:%d create succ", rdevCb->rsCb->chipId, rdevCb->rdevIndex,
681 : qpCb->qpInfoLo.qpn);
682 1 : return ret;
683 :
684 2 : nda_init_qp_err:
685 2 : (void)RsIbvDestroyQpExtend(rdevCb->ibCtxEx, qpCb->ibQpEx);
686 2 : qpCb->ibQpEx = NULL;
687 2 : return ret;
688 : }
689 :
690 5 : RS_ATTRI_VISI_DEF int RsNdaQpCreate(unsigned int phyId, unsigned int rdevIndex, struct NdaQpInitAttr *attr,
691 : struct NdaQpInfo *info, struct RsQpResp *qpResp)
692 : {
693 5 : struct ibv_qp_init_attr_extend qpInitAttrEx = {0};
694 5 : struct RsRdevCb *rdevCb = NULL;
695 5 : struct RsNdaCb *ndaCb = NULL;
696 5 : struct RsQpCb *qpCb = NULL;
697 5 : int ret = 0;
698 :
699 5 : CHK_PRT_RETURN(attr == NULL || info == NULL || qpResp == NULL,
700 : hccp_err("attr or info or qpResp is NULL, phyId:%u", phyId), -EINVAL);
701 4 : CHK_PRT_RETURN(attr->dmaMode >= QBUF_DMA_MODE_MAX,
702 : hccp_err("param err, dmaMode:%u >= %u, phyId:%u", attr->dmaMode, QBUF_DMA_MODE_MAX, phyId), -EINVAL);
703 :
704 3 : ret = RsQueryRdevCb(phyId, rdevIndex, &rdevCb);
705 3 : CHK_PRT_RETURN(ret != 0, hccp_err("RsQueryRdevCb phyId:%u rdevIndex:%u ret:%d", phyId, rdevIndex, ret), ret);
706 :
707 2 : ndaCb = (struct RsNdaCb *)rdevCb->rsCb->ndaCb;
708 2 : CHK_PRT_RETURN(ndaCb == NULL, hccp_err("ndaCb is NULL, phyId:%u rdevIndex:%u", phyId, rdevIndex), -ENODEV);
709 :
710 2 : ret = RsBuildUpNdaQpcb(rdevCb, &attr->attr, &qpCb);
711 2 : CHK_PRT_RETURN(ret != 0, hccp_err("RsBuildUpNdaQpcb failed, ret:%d", ret), ret);
712 :
713 2 : RsNdaQpInitExPrepare(rdevCb, attr, &qpInitAttrEx);
714 :
715 2 : ret = RsNdaQpCreateEx(qpCb, &qpInitAttrEx, info);
716 2 : if (ret != 0) {
717 1 : hccp_err("create nda qp create extend failed, ret:%d", ret);
718 1 : goto create_qp_err;
719 : }
720 :
721 1 : RS_PTHREAD_MUTEX_LOCK(&rdevCb->rdevMutex);
722 1 : RsListAddTail(&qpCb->list, &rdevCb->qpList);
723 1 : rdevCb->qpCnt++;
724 1 : RS_PTHREAD_MUTEX_ULOCK(&rdevCb->rdevMutex);
725 1 : qpResp->qpn = (unsigned int)qpCb->qpInfoLo.qpn;
726 1 : qpResp->gidIdx = (unsigned int)qpCb->qpInfoLo.gidIdx;
727 1 : qpResp->psn = (unsigned int)qpCb->qpInfoLo.psn;
728 1 : qpResp->gid = qpCb->qpInfoLo.gid;
729 1 : qpResp->directFlag = rdevCb->directFlag;
730 1 : return 0;
731 :
732 1 : create_qp_err:
733 1 : pthread_mutex_destroy(&qpCb->qpMutex);
734 1 : free(qpCb);
735 1 : qpCb = NULL;
736 1 : return ret;
737 : }
738 :
739 1 : RS_ATTRI_VISI_DEF int RsNdaQpDestroy(unsigned int phyId, unsigned int rdevIndex, unsigned int qpn)
740 : {
741 1 : struct RsQpCb *qpCb = NULL;
742 1 : int ret = 0;
743 :
744 1 : ret = RsQpn2qpcb(phyId, rdevIndex, qpn, &qpCb);
745 1 : CHK_PRT_RETURN(ret != 0 || qpCb == NULL, hccp_err("get qp cb failed, qpn:%u, ret:%d", qpn, ret), ret);
746 :
747 1 : RS_PTHREAD_MUTEX_LOCK(&qpCb->rdevCb->rdevMutex);
748 1 : RsListDel(&qpCb->list);
749 1 : qpCb->rdevCb->qpCnt--;
750 1 : RS_PTHREAD_MUTEX_ULOCK(&qpCb->rdevCb->rdevMutex);
751 :
752 1 : RsMrRelease(qpCb);
753 :
754 1 : ret = RsIbvDestroyQpExtend(qpCb->rdevCb->ibCtxEx, qpCb->ibQpEx);
755 1 : if (ret != 0) {
756 0 : hccp_err("qp:%u destroy extend failed, ret:%d", qpn, ret);
757 : }
758 1 : qpCb->ibQpEx = NULL;
759 :
760 1 : pthread_mutex_destroy(&qpCb->qpMutex);
761 :
762 1 : free(qpCb);
763 1 : qpCb = NULL;
764 1 : return ret;
765 : }
766 :
767 6 : RS_ATTRI_VISI_DEF int RsGetQpHyperFeature(unsigned int phyId, unsigned int rdevIndex, unsigned int qpn,
768 : struct HyperFeature *hyperFeature)
769 : {
770 6 : struct ibv_hyroce_feature feature = {0};
771 6 : struct RsQpCb *qpCb = NULL;
772 6 : int ret = 0;
773 :
774 6 : CHK_PRT_RETURN(hyperFeature == NULL, hccp_warn("hyperFeature is NULL, phyId:%u", phyId), ret);
775 :
776 6 : ret = RsQpn2qpcb(phyId, rdevIndex, qpn, &qpCb);
777 6 : CHK_PRT_RETURN(ret != 0 || qpCb == NULL, hccp_warn("get qp cb unsuccessful qpn %u, ret %d", qpn, ret), ret);
778 :
779 4 : if (qpCb->rdevCb->ibCtxEx == NULL) {
780 1 : return 0;
781 : }
782 :
783 3 : ret = RsIbvQueryQpSupportedHyroceFeature(qpCb->rdevCb->ibCtxEx, qpCb->ibQp, qpCb->qosAttr.sl, qpCb->qosAttr.tc,
784 : &feature);
785 3 : CHK_PRT_RETURN(ret != 0,
786 : hccp_warn("RsIbvQueryQpSupportedHyroceFeature unsuccessful, qpn %u ret %d errno:%d", qpn, ret, errno), ret);
787 :
788 2 : ret = memcpy_s(hyperFeature, sizeof(struct HyperFeature), &feature, sizeof(struct ibv_hyroce_feature));
789 2 : CHK_PRT_RETURN(ret != 0,
790 : hccp_warn("memcpy_s feature unsuccessful, qpn:%u ret:%d HyperFeature len:%zu "
791 : "ibv_hyroce_feature len:%zu",
792 : qpn, ret, sizeof(struct HyperFeature), sizeof(struct ibv_hyroce_feature)),
793 : ret);
794 :
795 1 : hccp_dbg("get QpHyperFeature successful qpn %u", qpn);
796 1 : return ret;
797 : }
|
