Line data Source code
1 : /**
2 : * Copyright (c) 2026 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include <stdlib.h>
12 : #include <stdint.h>
13 : #include <errno.h>
14 : #include "securec.h"
15 : #include "dl_hal_function.h"
16 : #include "dl_ibverbs_function.h"
17 : #include "dl_ibv_extend_function.h"
18 : #include "hccp_nda.h"
19 : #include "ra_rs_err.h"
20 : #include "rs.h"
21 : #include "rs_inner.h"
22 : #include "rs_drv_rdma.h"
23 : #include "rs_rdma.h"
24 : #include "rs_nda.h"
25 :
26 2 : RS_ATTRI_VISI_DEF int RsNdaGetDirectFlag(unsigned int phyId, unsigned int rdevIndex, int *directFlag)
27 : {
28 2 : struct ibv_device_attr_extend extDevAttr = {0};
29 2 : struct RsRdevCb *rdevCb = NULL;
30 2 : int ret = 0;
31 :
32 2 : ret = RsQueryRdevCb(phyId, rdevIndex, &rdevCb);
33 2 : CHK_PRT_RETURN(ret != 0, hccp_err("RsQueryRdevCb phyId:%u rdevIndex:%u ret:%d", phyId, rdevIndex, ret), ret);
34 :
35 1 : if (rdevCb->ibCtxEx == NULL) {
36 1 : *directFlag = DIRECT_FLAG_NOTSUPP;
37 1 : return ret;
38 : }
39 :
40 0 : ret = RsIbvQueryDeviceExtend(rdevCb->ibCtxEx, &extDevAttr);
41 0 : CHK_PRT_RETURN(ret != 0,
42 : hccp_err("RsIbvQueryDeviceExtend failed, phyId:%u rdevIndex:%u ret:%d", phyId, rdevIndex, ret), ret);
43 :
44 0 : if ((extDevAttr.ext_cap & IBV_EXTEND_DEV_NDA) == 0) {
45 0 : *directFlag = DIRECT_FLAG_NOTSUPP;
46 0 : return ret;
47 : }
48 :
49 0 : *directFlag = rdevCb->directFlag;
50 0 : return ret;
51 : }
52 :
53 32 : STATIC int RsNdaGetDirectFlagByDevAttr(struct ibv_device_attr *deviceAttr)
54 : {
55 : struct {
56 : uint32_t vendorId;
57 : uint32_t vendorPartID;
58 32 : } vendorInfoList[] = {
59 : {0x19E5U, 0x0230U},
60 : {0x19E5U, 0x0231U},
61 : {0x19E5U, 0x0232U},
62 : {0xCC08U, 0x8200U},
63 : {0xCC08U, 0x8201U},
64 : };
65 32 : unsigned int listSize = sizeof(vendorInfoList) / sizeof(vendorInfoList[0]);
66 : unsigned int i;
67 :
68 192 : for (i = 0; i < listSize; i++) {
69 160 : if ((deviceAttr->vendor_id == vendorInfoList[i].vendorId) &&
70 0 : (deviceAttr->vendor_part_id == vendorInfoList[i].vendorPartID)) {
71 0 : return DIRECT_FLAG_UB;
72 : }
73 : }
74 :
75 32 : return DIRECT_FLAG_PCIE;
76 : }
77 :
78 6 : STATIC int RsGetNdaPcieDbCb(struct RsNdaCb *ndaCb, uint64_t hva, struct NdaPcieDbCb **ndaDbCb)
79 : {
80 6 : struct NdaPcieDbCb *dbCbCurr = NULL;
81 6 : struct NdaPcieDbCb *dbCbNext = NULL;
82 :
83 6 : RS_LIST_GET_HEAD_ENTRY(dbCbCurr, dbCbNext, &ndaCb->ndaPcieCb.ndaDbList, list, struct NdaPcieDbCb);
84 6 : for (; (&dbCbCurr->list) != &ndaCb->ndaPcieCb.ndaDbList;
85 0 : dbCbCurr = dbCbNext, dbCbNext = list_entry(dbCbNext->list.next, struct NdaPcieDbCb, list)) {
86 3 : if (dbCbCurr->hva == hva) {
87 3 : *ndaDbCb = dbCbCurr;
88 3 : return 0;
89 : }
90 : }
91 :
92 3 : *ndaDbCb = NULL;
93 3 : hccp_info("ndaDbCb for hva:0x%llx does not exist", hva);
94 3 : return -ENODEV;
95 : }
96 :
97 7 : STATIC int RsGetNdaUbDbCb(struct RsNdaCb *ndaCb, uint64_t guidL, uint64_t guidH, struct NdaUbDbCb **ndaDbCb)
98 : {
99 7 : struct NdaUbDbCb *dbCbCurr = NULL;
100 7 : struct NdaUbDbCb *dbCbNext = NULL;
101 :
102 7 : RS_LIST_GET_HEAD_ENTRY(dbCbCurr, dbCbNext, &ndaCb->ndaUbCb.ndaDbList, list, struct NdaUbDbCb);
103 7 : for (; (&dbCbCurr->list) != &ndaCb->ndaUbCb.ndaDbList;
104 0 : dbCbCurr = dbCbNext, dbCbNext = list_entry(dbCbNext->list.next, struct NdaUbDbCb, list)) {
105 3 : if (dbCbCurr->guidL == guidL && dbCbCurr->guidH == guidH) {
106 3 : *ndaDbCb = dbCbCurr;
107 3 : return 0;
108 : }
109 : }
110 :
111 4 : *ndaDbCb = NULL;
112 4 : hccp_info("ndaDbCb for guidL:0x%llx guidH:0x%llx does not exist", guidL, guidH);
113 4 : return -ENODEV;
114 : }
115 :
116 0 : STATIC void *RsNdaPcieAlloc(size_t size)
117 : {
118 0 : struct RsNdaCb *ndaCb = (struct RsNdaCb *)gRsCb->ndaCb;
119 0 : void *ptr = NULL;
120 :
121 0 : CHK_PRT_RETURN(ndaCb == NULL || ndaCb->ndaOps.alloc == NULL,
122 : hccp_err("ndaCb or ndaOps.alloc is NULL, chipId:%u", gRsCb->chipId), NULL);
123 :
124 0 : ptr = ndaCb->ndaOps.alloc(size);
125 0 : CHK_PRT_RETURN(ptr == NULL, hccp_err("ptr alloc failed"), NULL);
126 :
127 0 : return ptr;
128 : }
129 :
130 0 : STATIC void RsNdaPcieFree(void *ptr)
131 : {
132 0 : struct RsNdaCb *ndaCb = (struct RsNdaCb *)gRsCb->ndaCb;
133 :
134 0 : if (ndaCb == NULL || ndaCb->ndaOps.free == NULL) {
135 0 : hccp_err("gRsCb->ndaCb or ndaOps.free is NULL, chipId:%u", gRsCb->chipId);
136 0 : return;
137 : }
138 :
139 0 : ndaCb->ndaOps.free(ptr);
140 0 : ptr = NULL;
141 : }
142 :
143 4 : STATIC void *RsNdaUbAlloc(size_t size)
144 : {
145 4 : struct RsNdaCb *ndaCb = (struct RsNdaCb *)gRsCb->ndaCb;
146 4 : struct DVattribute attr = {0};
147 4 : void *ptr = NULL;
148 4 : int ret = 0;
149 :
150 4 : CHK_PRT_RETURN(ndaCb == NULL || ndaCb->ndaOps.alloc == NULL || ndaCb->ndaOps.free == NULL,
151 : hccp_err("ndaCb or alloc or free is NULL, chipId:%u", gRsCb->chipId), NULL);
152 :
153 2 : ptr = ndaCb->ndaOps.alloc(size);
154 2 : CHK_PRT_RETURN(ptr == NULL, hccp_err("ptr alloc failed"), NULL);
155 :
156 2 : ret = DlDrvMemGetAttribute((uint64_t)(uintptr_t)ptr, &attr);
157 2 : if (ret != 0) {
158 1 : hccp_err("DlDrvMemGetAttribute failed, ret:%d", ret);
159 1 : goto free_ptr;
160 : }
161 :
162 1 : if (attr.memType == DV_MEM_LOCK_DEV) {
163 1 : ret = DlHalMemRegUbSegment(attr.devId, (uint64_t)(uintptr_t)ptr, size);
164 1 : if (ret != 0) {
165 1 : hccp_err("DlHalMemRegUbSegment failed, ret:%d devId:%u size:%zu", ret, attr.devId, size);
166 1 : goto free_ptr;
167 : }
168 : }
169 :
170 0 : return ptr;
171 :
172 2 : free_ptr:
173 2 : ndaCb->ndaOps.free(ptr);
174 2 : ptr = NULL;
175 2 : return NULL;
176 : }
177 :
178 0 : STATIC void RsNdaUbFree(void *ptr)
179 : {
180 0 : struct RsNdaCb *ndaCb = (struct RsNdaCb *)gRsCb->ndaCb;
181 0 : struct DVattribute attr = {0};
182 0 : int ret = 0;
183 :
184 0 : if (ndaCb == NULL || ndaCb->ndaOps.free == NULL) {
185 0 : hccp_err("gRsCb->ndaCb or ndaOps.free is NULL, chipId:%u", gRsCb->chipId);
186 0 : return;
187 : }
188 :
189 0 : ret = DlDrvMemGetAttribute((uint64_t)(uintptr_t)ptr, &attr);
190 0 : if (ret != 0) {
191 0 : hccp_err("DlDrvMemGetAttribute failed, ret:%d", ret);
192 0 : goto free_ptr;
193 : }
194 :
195 0 : if (attr.memType == DV_MEM_LOCK_DEV) {
196 0 : (void)DlHalMemUnRegUbSegment(attr.devId, (uint64_t)(uintptr_t)ptr);
197 : }
198 :
199 0 : free_ptr:
200 0 : ndaCb->ndaOps.free(ptr);
201 0 : ptr = NULL;
202 : }
203 :
204 0 : STATIC void RsNdaMemset(void *dst, int value, size_t count)
205 : {
206 0 : struct RsNdaCb *ndaCb = (struct RsNdaCb *)gRsCb->ndaCb;
207 :
208 0 : if (ndaCb == NULL || ndaCb->ndaOps.memset_s == NULL) {
209 0 : hccp_err("gRsCb->ndaCb or ndaOps.memset_s is NULL, chipId:%u", gRsCb->chipId);
210 0 : return;
211 : }
212 :
213 0 : ndaCb->ndaOps.memset_s(dst, value, count);
214 : }
215 :
216 0 : STATIC int RsNdaMemcpy(void *dst, size_t dstSize, void *src, size_t srcSize, uint32_t direct)
217 : {
218 0 : struct RsNdaCb *ndaCb = (struct RsNdaCb *)gRsCb->ndaCb;
219 :
220 0 : if (ndaCb == NULL || ndaCb->ndaOps.memcpy_s == NULL) {
221 0 : hccp_err("gRsCb->ndaCb or ndaOps.memcpy_s is NULL, chipId:%u", gRsCb->chipId);
222 0 : return -ENODEV;
223 : }
224 :
225 0 : return ndaCb->ndaOps.memcpy_s(dst, dstSize, src, srcSize, direct);
226 : }
227 :
228 5 : STATIC void RsNdaFreeUbDbCb(struct NdaUbDbCb *ndaDbCb)
229 : {
230 5 : if (ndaDbCb->refCnt != 0) {
231 1 : return;
232 : }
233 :
234 4 : RsListDel(&ndaDbCb->list);
235 4 : free(ndaDbCb);
236 4 : ndaDbCb = NULL;
237 : }
238 :
239 4 : STATIC void *RsNdaDbMmapHostVa(struct RsNdaCb *ndaCb, struct doorbell_map_desc *desc)
240 : {
241 4 : uint64_t alignHva = AlignDown(desc->hva, (uint64_t)RA_RS_4K_PAGE_SIZE);
242 4 : uint64_t alignSize = AlignUp(desc->size, (uint64_t)RA_RS_4K_PAGE_SIZE);
243 4 : uint64_t offset = desc->hva - alignHva;
244 4 : unsigned int logicId = gRsCb->logicId;
245 4 : struct NdaPcieDbCb *ndaDbCb = NULL;
246 4 : void *dbDva = NULL;
247 4 : int ret = 0;
248 :
249 4 : ret = RsGetNdaPcieDbCb(ndaCb, alignHva, &ndaDbCb);
250 4 : if (ret == 0) {
251 1 : ndaDbCb->refCnt++;
252 1 : return (void *)(uintptr_t)(ndaDbCb->dva + offset);
253 : }
254 :
255 3 : ndaDbCb = (struct NdaPcieDbCb *)calloc(1, sizeof(struct NdaPcieDbCb));
256 3 : CHK_PRT_RETURN(ndaDbCb == NULL, hccp_err("ndaDbCb calloc failed"), NULL);
257 :
258 3 : ret = DlHalHostRegister((void *)(uintptr_t)alignHva, alignSize, HOST_IO_MAP_DEV, logicId, &dbDva);
259 3 : if (ret != 0) {
260 2 : hccp_err("register host failed, chipId:%u logicId:%u ret:%d alignHva:0x%llx", gRsCb->chipId, logicId, ret,
261 : alignHva);
262 2 : free(ndaDbCb);
263 2 : ndaDbCb = NULL;
264 2 : return NULL;
265 : }
266 :
267 1 : ndaDbCb->hva = alignHva;
268 1 : ndaDbCb->dva = (uint64_t)(uintptr_t)dbDva;
269 1 : ndaDbCb->refCnt++;
270 1 : RsListAddTail(&ndaDbCb->list, &ndaCb->ndaPcieCb.ndaDbList);
271 1 : return (void *)(uintptr_t)(ndaDbCb->dva + offset);
272 : }
273 :
274 7 : STATIC void RsNdaMapPrivPrepare(struct doorbell_map_desc *desc, struct NdaUbResMapPrivInfo *resMapIn)
275 : {
276 7 : resMapIn->guid_l = desc->ub_res.guid_l;
277 7 : resMapIn->guid_h = desc->ub_res.guid_h;
278 7 : resMapIn->db_idx = desc->ub_res.bits.offset / (uint64_t)RA_RS_4K_PAGE_SIZE;
279 7 : resMapIn->db_num = 1;
280 7 : return;
281 : }
282 :
283 5 : STATIC void *RsNdaDbMmapUbRes(struct RsNdaCb *ndaCb, struct doorbell_map_desc *desc)
284 : {
285 5 : struct NdaUbResMapPrivInfo resMapIn = {0};
286 5 : struct res_map_info_out resInfoOut = {0};
287 5 : struct res_map_info_in resInfoIn = {0};
288 5 : unsigned int logicId = gRsCb->logicId;
289 5 : struct NdaUbDbCb *ndaDbCb = NULL;
290 5 : uint64_t dva = 0;
291 5 : int ret = 0;
292 :
293 5 : ret = RsGetNdaUbDbCb(ndaCb, desc->ub_res.guid_l, desc->ub_res.guid_h, &ndaDbCb);
294 5 : if (ret == 0) {
295 1 : goto map_db;
296 : }
297 :
298 4 : ndaDbCb = (struct NdaUbDbCb *)calloc(1, sizeof(struct NdaUbDbCb));
299 4 : CHK_PRT_RETURN(ndaDbCb == NULL, hccp_err("ndaDbCb calloc failed"), NULL);
300 4 : ndaDbCb->guidL = desc->ub_res.guid_l;
301 4 : ndaDbCb->guidH = desc->ub_res.guid_h;
302 4 : ndaDbCb->guidIdx = ndaCb->ndaUbCb.ndaDbGuidCnt;
303 4 : RsListAddTail(&ndaDbCb->list, &ndaCb->ndaUbCb.ndaDbList);
304 4 : ndaCb->ndaUbCb.ndaDbGuidCnt++;
305 :
306 5 : map_db:
307 5 : RsNdaMapPrivPrepare(desc, &resMapIn);
308 5 : resInfoIn.target_proc_type = PROCESS_CP1;
309 5 : resInfoIn.res_type = RES_ADDR_TYPE_NDA_URMA_DB;
310 5 : resInfoIn.res_id = RsNdaGenerateResId(resMapIn.db_idx, ndaDbCb->guidIdx);
311 5 : resInfoIn.priv_len = sizeof(struct NdaUbResMapPrivInfo);
312 5 : resInfoIn.priv = (void *)&resMapIn;
313 5 : ret = DlHalResAddrMapV2(logicId, &resInfoIn, &resInfoOut);
314 5 : if (ret != 0) {
315 3 : hccp_err("DlHalResAddrMapV2 failed, chipId:%u logicId:%u resId:0x%x ret:%d", gRsCb->chipId, logicId,
316 : resInfoIn.res_id, ret);
317 3 : RsNdaFreeUbDbCb(ndaDbCb);
318 3 : return NULL;
319 : }
320 :
321 2 : ndaDbCb->refCnt++;
322 2 : dva = resInfoOut.va + (desc->ub_res.bits.offset % (uint64_t)RA_RS_4K_PAGE_SIZE);
323 2 : return (void *)(uintptr_t)dva;
324 : }
325 :
326 11 : STATIC void *RsNdaDbMmap(struct doorbell_map_desc *desc)
327 : {
328 11 : struct RsNdaCb *ndaCb = (struct RsNdaCb *)gRsCb->ndaCb;
329 :
330 11 : CHK_PRT_RETURN(desc == NULL, hccp_err("desc is null"), NULL);
331 11 : CHK_PRT_RETURN(ndaCb == NULL, hccp_err("ndaCb is null, chipId:%u", gRsCb->chipId), NULL);
332 :
333 11 : if (desc->type == DB_MAP_MODE_HOST_VA) {
334 4 : return RsNdaDbMmapHostVa(ndaCb, desc);
335 7 : } else if (desc->type == DB_MAP_MODE_UB_RES) {
336 5 : return RsNdaDbMmapUbRes(ndaCb, desc);
337 : } else {
338 2 : hccp_err("invalid desc->type:%u, chipId:%u", desc->type, gRsCb->chipId);
339 2 : return NULL;
340 : }
341 : }
342 :
343 3 : STATIC int RsNdaDbUnmapHostVa(struct RsNdaCb *ndaCb, void *ptr, struct doorbell_map_desc *desc)
344 : {
345 3 : uint64_t alignHva = AlignDown(desc->hva, (uint64_t)RA_RS_4K_PAGE_SIZE);
346 3 : unsigned int logicId = gRsCb->logicId;
347 3 : struct NdaPcieDbCb *ndaDbCb = NULL;
348 3 : int ret = 0;
349 :
350 3 : ret = RsGetNdaPcieDbCb(ndaCb, alignHva, &ndaDbCb);
351 3 : CHK_PRT_RETURN(ret != 0,
352 : hccp_err("RsGetNdaPcieDbCb failed, hva:0x%llx chipId:%u ret:%d", alignHva, gRsCb->chipId, ret), ret);
353 :
354 2 : ndaDbCb->refCnt--;
355 2 : if (ndaDbCb->refCnt != 0) {
356 1 : return ret;
357 : }
358 :
359 1 : ret = DlHalHostUnRegisterEx((void *)(uintptr_t)alignHva, logicId, HOST_IO_MAP_DEV);
360 1 : if (ret != 0) {
361 0 : hccp_err("DlHalHostUnRegisterEx failed, chipId:%u logicId:%u ret:%d", gRsCb->chipId, logicId, ret);
362 : }
363 :
364 1 : RsListDel(&ndaDbCb->list);
365 1 : free(ndaDbCb);
366 1 : ndaDbCb = NULL;
367 1 : return ret;
368 : }
369 :
370 3 : STATIC int RsNdaDbUnmapUbRes(struct RsNdaCb *ndaCb, void *ptr, struct doorbell_map_desc *desc)
371 : {
372 3 : struct NdaUbResMapPrivInfo resMapIn = {0};
373 3 : struct res_map_info_in resInfoIn = {0};
374 3 : unsigned int logicId = gRsCb->logicId;
375 3 : struct NdaUbDbCb *ndaDbCb = NULL;
376 3 : int ret = 0;
377 :
378 3 : ret = RsGetNdaUbDbCb(ndaCb, desc->ub_res.guid_l, desc->ub_res.guid_h, &ndaDbCb);
379 3 : CHK_PRT_RETURN(ret != 0,
380 : hccp_err("RsGetNdaUbDbCb failed, chipId:%u guidL:0x%llx guidH:0x%llx", gRsCb->chipId, desc->ub_res.guid_l,
381 : desc->ub_res.guid_h),
382 : ret);
383 :
384 2 : RsNdaMapPrivPrepare(desc, &resMapIn);
385 2 : resInfoIn.target_proc_type = PROCESS_CP1;
386 2 : resInfoIn.res_type = RES_ADDR_TYPE_NDA_URMA_DB;
387 2 : resInfoIn.res_id = RsNdaGenerateResId(resMapIn.db_idx, ndaDbCb->guidIdx);
388 2 : resInfoIn.priv_len = sizeof(struct NdaUbResMapPrivInfo);
389 2 : resInfoIn.priv = (void *)&resMapIn;
390 2 : ret = DlHalResAddrUnmapV2(logicId, &resInfoIn);
391 2 : if (ret != 0) {
392 0 : hccp_err("DlHalResAddrUnmapV2 failed, chipId:%u logicId:%u resId:0x%x ret:%d", gRsCb->chipId, logicId,
393 : resInfoIn.res_id, ret);
394 : }
395 :
396 2 : ndaDbCb->refCnt--;
397 2 : RsNdaFreeUbDbCb(ndaDbCb);
398 2 : return ret;
399 : }
400 :
401 6 : STATIC int RsNdaDbUnmap(void *ptr, struct doorbell_map_desc *desc)
402 : {
403 6 : struct RsNdaCb *ndaCb = (struct RsNdaCb *)gRsCb->ndaCb;
404 :
405 6 : CHK_PRT_RETURN(ptr == NULL || desc == NULL, hccp_err("ptr or desc is null"), -EINVAL);
406 6 : CHK_PRT_RETURN(ndaCb == NULL, hccp_err("ndaCb is null, chipId:%u", gRsCb->chipId), -ENODEV);
407 :
408 6 : if (desc->type == DB_MAP_MODE_HOST_VA) {
409 3 : return RsNdaDbUnmapHostVa(ndaCb, ptr, desc);
410 3 : } else if (desc->type == DB_MAP_MODE_UB_RES) {
411 3 : return RsNdaDbUnmapUbRes(ndaCb, ptr, desc);
412 : } else {
413 0 : hccp_err("invalid desc->type:%u, chipId:%u", desc->type, gRsCb->chipId);
414 0 : return -EINVAL;
415 : }
416 : }
417 :
418 32 : STATIC void RsNdaCbInitCb(struct RsNdaCb *ndaCb)
419 : {
420 32 : ndaCb->ndaPcieCb.ibvExOps.alloc = RsNdaPcieAlloc;
421 32 : ndaCb->ndaPcieCb.ibvExOps.free = RsNdaPcieFree;
422 32 : ndaCb->ndaPcieCb.ibvExOps.db_mmap = RsNdaDbMmap;
423 32 : ndaCb->ndaPcieCb.ibvExOps.db_unmap = RsNdaDbUnmap;
424 32 : ndaCb->ndaPcieCb.ibvExOps.memset_s = RsNdaMemset;
425 32 : ndaCb->ndaPcieCb.ibvExOps.memcpy_s = RsNdaMemcpy;
426 :
427 32 : ndaCb->ndaUbCb.ibvExOps.alloc = RsNdaUbAlloc;
428 32 : ndaCb->ndaUbCb.ibvExOps.free = RsNdaUbFree;
429 32 : ndaCb->ndaUbCb.ibvExOps.db_mmap = RsNdaDbMmap;
430 32 : ndaCb->ndaUbCb.ibvExOps.db_unmap = RsNdaDbUnmap;
431 32 : ndaCb->ndaUbCb.ibvExOps.memset_s = RsNdaMemset;
432 32 : ndaCb->ndaUbCb.ibvExOps.memcpy_s = RsNdaMemcpy;
433 :
434 32 : RS_INIT_LIST_HEAD(&ndaCb->ndaPcieCb.ndaDbList);
435 32 : RS_INIT_LIST_HEAD(&ndaCb->ndaUbCb.ndaDbList);
436 32 : }
437 :
438 32 : int RsInitNdaCb(struct RsRdevCb *rdevCb)
439 : {
440 32 : struct RsNdaCb *ndaCb = NULL;
441 32 : int count = 0;
442 32 : int ret = 0;
443 :
444 32 : rdevCb->directFlag = RsNdaGetDirectFlagByDevAttr(&rdevCb->deviceAttr);
445 :
446 32 : rdevCb->ibCtxEx = RsIbvOpenExtend(rdevCb->ibCtx);
447 32 : if (rdevCb->ibCtxEx == NULL) {
448 0 : return 0;
449 : }
450 :
451 32 : count = __sync_fetch_and_add(&rdevCb->rsCb->ndaCbRefCnt, 1);
452 32 : if (count > 0) {
453 0 : return 0;
454 : }
455 :
456 32 : ndaCb = (struct RsNdaCb *)calloc(1, sizeof(struct RsNdaCb));
457 32 : if (ndaCb == NULL) {
458 0 : hccp_err("calloc for ndaCb failed");
459 0 : ret = -ENOMEM;
460 0 : goto calloc_err;
461 : }
462 :
463 32 : RsNdaCbInitCb(ndaCb);
464 :
465 32 : rdevCb->rsCb->ndaCb = (void *)ndaCb;
466 32 : return ret;
467 :
468 0 : calloc_err:
469 0 : (void)__sync_fetch_and_sub(&rdevCb->rsCb->ndaCbRefCnt, 1);
470 0 : (void)RsIbvCloseExtend(rdevCb->ibCtxEx);
471 0 : rdevCb->ibCtxEx = NULL;
472 0 : return ret;
473 : }
474 :
475 32 : void RsDeinitNdaCb(struct RsRdevCb *rdevCb)
476 : {
477 32 : if (rdevCb->ibCtxEx == NULL) {
478 1 : return;
479 : }
480 :
481 31 : (void)RsIbvCloseExtend(rdevCb->ibCtxEx);
482 31 : rdevCb->ibCtxEx = NULL;
483 :
484 31 : if (__sync_fetch_and_sub(&rdevCb->rsCb->ndaCbRefCnt, 1) > 1) {
485 0 : return;
486 : }
487 :
488 31 : free(rdevCb->rsCb->ndaCb);
489 31 : rdevCb->rsCb->ndaCb = NULL;
490 : }
491 :
492 4 : STATIC void RsNdaInitExOps(struct RsNdaCb *ndaCb, uint32_t dmaMode, struct NdaOps *ops, struct ibv_extend_ops **extOps)
493 : {
494 4 : ndaCb->ndaOps.alloc = ops->alloc;
495 4 : ndaCb->ndaOps.free = ops->free;
496 4 : ndaCb->ndaOps.memset_s = ops->memset_s;
497 4 : ndaCb->ndaOps.memcpy_s = ops->memcpy_s;
498 4 : if (dmaMode == QBUF_DMA_MODE_DEFAULT) {
499 2 : *extOps = &ndaCb->ndaPcieCb.ibvExOps;
500 2 : } else if (dmaMode == QBUF_DMA_MODE_INDEP_UB) {
501 2 : *extOps = &ndaCb->ndaUbCb.ibvExOps;
502 : } else {
503 0 : *extOps = NULL;
504 : }
505 4 : }
506 :
507 2 : STATIC void RsNdaCqInitExPrepare(struct NdaCqInitAttr *attr, struct RsNdaCb *ndaCb,
508 : struct ibv_cq_init_attr_extend *cqInitAttrEx)
509 : {
510 2 : (void)memcpy_s(&cqInitAttrEx->attr, sizeof(struct ibv_cq_init_attr_ex), &attr->attr,
511 : sizeof(struct ibv_cq_init_attr_ex));
512 2 : cqInitAttrEx->cq_cap_flag = attr->cqCapFlag;
513 2 : cqInitAttrEx->type = attr->dmaMode;
514 2 : RsNdaInitExOps(ndaCb, attr->dmaMode, attr->ops, &cqInitAttrEx->ops);
515 2 : }
516 :
517 1 : STATIC int RsNdaCqCreateEx(struct RsRdevCb *rdevCb, struct ibv_cq_init_attr_extend *cqInitAttrEx,
518 : struct NdaCqInfo *info, void **ibvCqExt)
519 : {
520 1 : struct ibv_cq_extend *cqExt = NULL;
521 1 : int ret = 0;
522 :
523 1 : cqExt = RsIbvCreateCqExtend(rdevCb->ibCtxEx, cqInitAttrEx);
524 1 : CHK_PRT_RETURN(cqExt == NULL, hccp_err("RsNdaCreateCqExtend failed, errno:%d", errno), -ENOMEM);
525 :
526 1 : ret = memcpy_s(&info->cqInfo, sizeof(struct queueInfo), &cqExt->cq_info, sizeof(struct queue_info));
527 1 : CHK_PRT_RETURN(ret != 0, hccp_err("memcpy_s queue_info failed, ret:%d", ret), -ESAFEFUNC);
528 1 : ret = memcpy_s(info->resv, sizeof(info->resv), cqExt->resv, sizeof(cqExt->resv));
529 1 : CHK_PRT_RETURN(ret != 0, hccp_err("memcpy_s resv failed, ret:%d", ret), -ESAFEFUNC);
530 1 : info->cq = cqExt->cq;
531 1 : *ibvCqExt = cqExt;
532 1 : return 0;
533 : }
534 :
535 5 : RS_ATTRI_VISI_DEF int RsNdaCqCreate(unsigned int phyId, unsigned int rdevIndex, struct NdaCqInitAttr *attr,
536 : struct NdaCqInfo *info, void **ibvCqExt)
537 : {
538 5 : struct ibv_cq_init_attr_extend cqInitAttrEx = {0};
539 5 : struct RsRdevCb *rdevCb = NULL;
540 5 : struct RsNdaCb *ndaCb = NULL;
541 5 : int ret = 0;
542 :
543 5 : CHK_PRT_RETURN(attr == NULL || info == NULL, hccp_err("attr or info is NULL, phyId:%u", phyId), -EINVAL);
544 4 : CHK_PRT_RETURN(attr->dmaMode >= QBUF_DMA_MODE_MAX,
545 : hccp_err("param err, dmaMode:%u >= %u, phyId:%u", attr->dmaMode, QBUF_DMA_MODE_MAX, phyId), -EINVAL);
546 :
547 3 : ret = RsQueryRdevCb(phyId, rdevIndex, &rdevCb);
548 3 : CHK_PRT_RETURN(ret != 0, hccp_err("RsQueryRdevCb failed, phyId:%u rdevIndex:%u ret:%d", phyId, rdevIndex, ret),
549 : ret);
550 :
551 2 : ndaCb = (struct RsNdaCb *)rdevCb->rsCb->ndaCb;
552 2 : CHK_PRT_RETURN(ndaCb == NULL, hccp_err("ndaCb is NULL, does not support nda, phyId:%u", phyId), -ENODEV);
553 :
554 2 : RsNdaCqInitExPrepare(attr, ndaCb, &cqInitAttrEx);
555 :
556 2 : ret = RsNdaCqCreateEx(rdevCb, &cqInitAttrEx, info, ibvCqExt);
557 2 : CHK_PRT_RETURN(ret != 0, hccp_err("RsNdaCqCreateEx failed, phyId:%u rdevIndex:%u ret:%d", phyId, rdevIndex, ret),
558 : ret);
559 :
560 1 : return ret;
561 : }
562 :
563 1 : RS_ATTRI_VISI_DEF int RsNdaCqDestroy(unsigned int phyId, unsigned int rdevIndex, void *ibvCqExt)
564 : {
565 1 : struct RsRdevCb *rdevCb = NULL;
566 1 : int ret = 0;
567 :
568 1 : ret = RsQueryRdevCb(phyId, rdevIndex, &rdevCb);
569 1 : CHK_PRT_RETURN(ret != 0, hccp_err("RsQueryRdevCb failed, phyId:%u rdevIndex:%u ret:%d", phyId, rdevIndex, ret),
570 : ret);
571 :
572 1 : ret = RsIbvDestroyCqExtend(rdevCb->ibCtxEx, ibvCqExt);
573 1 : CHK_PRT_RETURN(ret != 0,
574 : hccp_err("RsIbvDestroyCqExtend failed, phyId:%u rdevIndex:%u ret:%d", phyId, rdevIndex, ret), ret);
575 :
576 1 : return ret;
577 : }
578 :
579 2 : STATIC int RsBuildUpNdaQpcb(struct RsRdevCb *rdevCb, struct ibv_qp_init_attr *qpInitAttr, struct RsQpCb **qpCb)
580 : {
581 2 : int ret = 0;
582 :
583 2 : *qpCb = (struct RsQpCb *)calloc(1, sizeof(struct RsQpCb));
584 2 : CHK_PRT_RETURN(*qpCb == NULL, hccp_err("RsQpCb calloc failed"), -ENOMEM);
585 :
586 2 : ret = pthread_mutex_init(&(*qpCb)->qpMutex, NULL);
587 2 : if (ret != 0) {
588 0 : hccp_err("pthread_mutex_init failed, ret:%d", ret);
589 0 : goto qp_mutex_init_err;
590 : }
591 :
592 2 : (*qpCb)->rdevCb = rdevCb;
593 2 : RS_INIT_LIST_HEAD(&(*qpCb)->mrList);
594 2 : RS_INIT_LIST_HEAD(&(*qpCb)->remMrList);
595 2 : (*qpCb)->state = RS_QP_STATUS_DISCONNECT;
596 2 : (*qpCb)->ibPd = rdevCb->ibPd;
597 2 : (*qpCb)->txDepth = qpInitAttr->cap.max_send_wr;
598 2 : (*qpCb)->rxDepth = qpInitAttr->cap.max_recv_wr;
599 :
600 2 : (*qpCb)->numRecvCqEvents = 0;
601 2 : (*qpCb)->numSendCqEvents = 0;
602 2 : (*qpCb)->qosAttr.tc = (RS_ROCE_DSCP_33 & RS_DSCP_MASK) << RS_DSCP_OFF;
603 2 : (*qpCb)->qosAttr.sl = RS_ROCE_4_SL;
604 2 : (*qpCb)->timeout = RS_QP_ATTR_TIMEOUT;
605 2 : (*qpCb)->retryCnt = RS_QP_ATTR_RETRY_CNT;
606 :
607 2 : return ret;
608 :
609 0 : qp_mutex_init_err:
610 0 : free(*qpCb);
611 0 : *qpCb = NULL;
612 0 : return ret;
613 : }
614 :
615 2 : STATIC void RsNdaQpInitExPrepare(struct RsRdevCb *rdevCb, struct NdaQpInitAttr *attr,
616 : struct ibv_qp_init_attr_extend *qpInitAttrEx)
617 : {
618 2 : struct RsNdaCb *ndaCb = (struct RsNdaCb *)rdevCb->rsCb->ndaCb;
619 :
620 2 : qpInitAttrEx->pd = rdevCb->ibPd;
621 2 : (void)memcpy_s(&qpInitAttrEx->attr, sizeof(struct ibv_qp_init_attr), &attr->attr, sizeof(struct ibv_qp_init_attr));
622 2 : qpInitAttrEx->qp_cap_flag = attr->qpCapFlag;
623 2 : qpInitAttrEx->type = attr->dmaMode;
624 2 : RsNdaInitExOps(ndaCb, attr->dmaMode, attr->ops, &qpInitAttrEx->ops);
625 2 : }
626 :
627 1 : STATIC int RsNdaQpGetQpInfo(struct RsQpCb *qpCb, struct NdaQpInfo *info)
628 : {
629 1 : int ret = 0;
630 :
631 1 : info->qp = qpCb->ibQp;
632 1 : ret = memcpy_s(&info->sqInfo, sizeof(struct queueInfo), &qpCb->ibQpEx->sq_info, sizeof(struct queue_info));
633 1 : CHK_PRT_RETURN(ret != 0, hccp_err("memcpy_s sq_info failed, ret:%d", ret), -ESAFEFUNC);
634 1 : ret = memcpy_s(&info->rqInfo, sizeof(struct queueInfo), &qpCb->ibQpEx->rq_info, sizeof(struct queue_info));
635 1 : CHK_PRT_RETURN(ret != 0, hccp_err("memcpy_s rq_info failed, ret:%d", ret), -ESAFEFUNC);
636 1 : ret = memcpy_s(info->resv, sizeof(info->resv), qpCb->ibQpEx->resv, sizeof(qpCb->ibQpEx->resv));
637 1 : CHK_PRT_RETURN(ret != 0, hccp_err("memcpy_s resv failed, ret:%d", ret), -ESAFEFUNC);
638 :
639 1 : return ret;
640 : }
641 :
642 3 : STATIC int RsNdaQpCreateEx(struct RsQpCb *qpCb, struct ibv_qp_init_attr_extend *qpInitAttrEx, struct NdaQpInfo *info)
643 : {
644 3 : struct ibv_qp_attr qpAttr = {0};
645 3 : struct ibv_port_attr attr = {0};
646 3 : struct RsRdevCb *rdevCb = NULL;
647 3 : int ret = 0;
648 :
649 3 : rdevCb = qpCb->rdevCb;
650 3 : qpCb->ibQpEx = RsIbvCreateQpExtend(rdevCb->ibCtxEx, qpInitAttrEx);
651 3 : CHK_PRT_RETURN(qpCb->ibQpEx == NULL, hccp_err("RsNdaCreateQpExtend failed, errno:%d", errno), -ENOMEM);
652 :
653 3 : qpCb->ibQp = qpCb->ibQpEx->qp;
654 3 : ret = RsIbvQueryQp(qpCb->ibQp, &qpAttr, IBV_QP_CAP, &qpInitAttrEx->attr);
655 3 : if (ret != 0) {
656 1 : hccp_err("query qp attr failed ret:%d", ret);
657 1 : ret = -EOPENSRC;
658 1 : goto nda_init_qp_err;
659 : }
660 :
661 2 : ret = RsDrvQpInfoRelated(qpCb, rdevCb, &attr, &qpAttr);
662 2 : if (ret != 0) {
663 1 : hccp_err("qp info related failed ret:%d", ret);
664 1 : goto nda_init_qp_err;
665 : }
666 :
667 1 : ret = RsNdaQpGetQpInfo(qpCb, info);
668 1 : if (ret != 0) {
669 0 : hccp_err("RsNdaQpGetQpInfo failed ret:%d", ret);
670 0 : goto nda_init_qp_err;
671 : }
672 :
673 1 : hccp_info("chip_id:%u, rdevIndex:%u, qp:%d create succ", rdevCb->rsCb->chipId, rdevCb->rdevIndex,
674 : qpCb->qpInfoLo.qpn);
675 1 : return ret;
676 :
677 2 : nda_init_qp_err:
678 2 : (void)RsIbvDestroyQpExtend(rdevCb->ibCtxEx, qpCb->ibQpEx);
679 2 : qpCb->ibQpEx = NULL;
680 2 : return ret;
681 : }
682 :
683 5 : RS_ATTRI_VISI_DEF int RsNdaQpCreate(unsigned int phyId, unsigned int rdevIndex, struct NdaQpInitAttr *attr,
684 : struct NdaQpInfo *info, struct RsQpResp *qpResp)
685 : {
686 5 : struct ibv_qp_init_attr_extend qpInitAttrEx = {0};
687 5 : struct RsRdevCb *rdevCb = NULL;
688 5 : struct RsNdaCb *ndaCb = NULL;
689 5 : struct RsQpCb *qpCb = NULL;
690 5 : int ret = 0;
691 :
692 5 : CHK_PRT_RETURN(attr == NULL || info == NULL || qpResp == NULL,
693 : hccp_err("attr or info or qpResp is NULL, phyId:%u", phyId), -EINVAL);
694 4 : CHK_PRT_RETURN(attr->dmaMode >= QBUF_DMA_MODE_MAX,
695 : hccp_err("param err, dmaMode:%u >= %u, phyId:%u", attr->dmaMode, QBUF_DMA_MODE_MAX, phyId), -EINVAL);
696 :
697 3 : ret = RsQueryRdevCb(phyId, rdevIndex, &rdevCb);
698 3 : CHK_PRT_RETURN(ret != 0, hccp_err("RsQueryRdevCb phyId:%u rdevIndex:%u ret:%d", phyId, rdevIndex, ret), ret);
699 :
700 2 : ndaCb = (struct RsNdaCb *)rdevCb->rsCb->ndaCb;
701 2 : CHK_PRT_RETURN(ndaCb == NULL, hccp_err("ndaCb is NULL, phyId:%u rdevIndex:%u", phyId, rdevIndex), -ENODEV);
702 :
703 2 : ret = RsBuildUpNdaQpcb(rdevCb, &attr->attr, &qpCb);
704 2 : CHK_PRT_RETURN(ret != 0, hccp_err("RsBuildUpNdaQpcb failed, ret:%d", ret), ret);
705 :
706 2 : RsNdaQpInitExPrepare(rdevCb, attr, &qpInitAttrEx);
707 :
708 2 : ret = RsNdaQpCreateEx(qpCb, &qpInitAttrEx, info);
709 2 : if (ret != 0) {
710 1 : hccp_err("create nda qp create extend failed, ret:%d", ret);
711 1 : goto create_qp_err;
712 : }
713 :
714 1 : RS_PTHREAD_MUTEX_LOCK(&rdevCb->rdevMutex);
715 1 : RsListAddTail(&qpCb->list, &rdevCb->qpList);
716 1 : rdevCb->qpCnt++;
717 1 : RS_PTHREAD_MUTEX_ULOCK(&rdevCb->rdevMutex);
718 1 : qpResp->qpn = (unsigned int)qpCb->qpInfoLo.qpn;
719 1 : qpResp->gidIdx = (unsigned int)qpCb->qpInfoLo.gidIdx;
720 1 : qpResp->psn = (unsigned int)qpCb->qpInfoLo.psn;
721 1 : qpResp->gid = qpCb->qpInfoLo.gid;
722 1 : qpResp->directFlag = rdevCb->directFlag;
723 1 : return 0;
724 :
725 1 : create_qp_err:
726 1 : pthread_mutex_destroy(&qpCb->qpMutex);
727 1 : free(qpCb);
728 1 : qpCb = NULL;
729 1 : return ret;
730 : }
731 :
732 1 : RS_ATTRI_VISI_DEF int RsNdaQpDestroy(unsigned int phyId, unsigned int rdevIndex, unsigned int qpn)
733 : {
734 1 : struct RsQpCb *qpCb = NULL;
735 1 : int ret = 0;
736 :
737 1 : ret = RsQpn2qpcb(phyId, rdevIndex, qpn, &qpCb);
738 1 : CHK_PRT_RETURN(ret != 0 || qpCb == NULL, hccp_err("get qp cb failed, qpn:%u, ret:%d", qpn, ret), ret);
739 :
740 1 : RS_PTHREAD_MUTEX_LOCK(&qpCb->rdevCb->rdevMutex);
741 1 : RsListDel(&qpCb->list);
742 1 : qpCb->rdevCb->qpCnt--;
743 1 : RS_PTHREAD_MUTEX_ULOCK(&qpCb->rdevCb->rdevMutex);
744 :
745 1 : RsMrRelease(qpCb);
746 :
747 1 : ret = RsIbvDestroyQpExtend(qpCb->rdevCb->ibCtxEx, qpCb->ibQpEx);
748 1 : if (ret != 0) {
749 0 : hccp_err("qp:%u destroy extend failed, ret:%d", qpn, ret);
750 : }
751 1 : qpCb->ibQpEx = NULL;
752 :
753 1 : pthread_mutex_destroy(&qpCb->qpMutex);
754 :
755 1 : free(qpCb);
756 1 : qpCb = NULL;
757 1 : return ret;
758 : }
759 :
760 6 : RS_ATTRI_VISI_DEF int RsGetQpHyperFeature(unsigned int phyId, unsigned int rdevIndex, unsigned int qpn,
761 : struct HyperFeature *hyperFeature)
762 : {
763 6 : struct ibv_hyroce_feature feature = {0};
764 6 : struct RsQpCb *qpCb = NULL;
765 6 : int ret = 0;
766 :
767 6 : CHK_PRT_RETURN(hyperFeature == NULL, hccp_warn("hyperFeature is NULL, phyId:%u", phyId), ret);
768 :
769 6 : ret = RsQpn2qpcb(phyId, rdevIndex, qpn, &qpCb);
770 6 : CHK_PRT_RETURN(ret != 0 || qpCb == NULL, hccp_warn("get qp cb unsuccessful qpn %u, ret %d", qpn, ret), ret);
771 :
772 4 : if (qpCb->rdevCb->ibCtxEx == NULL) {
773 1 : return 0;
774 : }
775 :
776 3 : ret = RsIbvQueryQpSupportedHyroceFeature(qpCb->rdevCb->ibCtxEx, qpCb->ibQp, qpCb->qosAttr.sl, qpCb->qosAttr.tc,
777 : &feature);
778 3 : CHK_PRT_RETURN(ret != 0,
779 : hccp_warn("RsIbvQueryQpSupportedHyroceFeature unsuccessful, qpn %u ret %d errno:%d", qpn, ret, errno), ret);
780 :
781 2 : ret = memcpy_s(hyperFeature, sizeof(struct HyperFeature), &feature, sizeof(struct ibv_hyroce_feature));
782 2 : CHK_PRT_RETURN(ret != 0,
783 : hccp_warn("memcpy_s feature unsuccessful, qpn:%u ret:%d HyperFeature len:%zu "
784 : "ibv_hyroce_feature len:%zu",
785 : qpn, ret, sizeof(struct HyperFeature), sizeof(struct ibv_hyroce_feature)),
786 : ret);
787 :
788 1 : hccp_dbg("get QpHyperFeature successful qpn %u", qpn);
789 1 : return ret;
790 : }
|
