Line data Source code
1 : /**
2 : * Copyright (c) 2026 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include <stdlib.h>
12 : #include <stdint.h>
13 : #include <errno.h>
14 : #include "securec.h"
15 : #include "dl_hal_function.h"
16 : #include "dl_ibverbs_function.h"
17 : #include "dl_ibv_extend_function.h"
18 : #include "hccp_nda.h"
19 : #include "ra_rs_err.h"
20 : #include "rs.h"
21 : #include "rs_inner.h"
22 : #include "rs_drv_rdma.h"
23 : #include "rs_rdma.h"
24 : #include "rs_nda.h"
25 :
26 2 : RS_ATTRI_VISI_DEF int RsNdaGetDirectFlag(unsigned int phyId, unsigned int rdevIndex, int *directFlag)
27 : {
28 2 : struct ibv_device_attr_extend extDevAttr = {0};
29 2 : struct RsRdevCb *rdevCb = NULL;
30 2 : int ret = 0;
31 :
32 2 : ret = RsQueryRdevCb(phyId, rdevIndex, &rdevCb);
33 2 : CHK_PRT_RETURN(ret != 0, hccp_err("RsQueryRdevCb phyId:%u rdevIndex:%u ret:%d", phyId, rdevIndex, ret), ret);
34 :
35 1 : if (rdevCb->ibCtxEx == NULL) {
36 1 : *directFlag = DIRECT_FLAG_NOTSUPP;
37 1 : return ret;
38 : }
39 :
40 0 : ret = RsIbvQueryDeviceExtend(rdevCb->ibCtxEx, &extDevAttr);
41 0 : CHK_PRT_RETURN(ret != 0,
42 : hccp_err("RsIbvQueryDeviceExtend failed, phyId:%u rdevIndex:%u ret:%d", phyId, rdevIndex, ret), ret);
43 :
44 0 : if ((extDevAttr.ext_cap & IBV_EXTEND_DEV_NDA) == 0) {
45 0 : *directFlag = DIRECT_FLAG_NOTSUPP;
46 0 : return ret;
47 : }
48 :
49 0 : *directFlag = rdevCb->directFlag;
50 0 : return ret;
51 : }
52 :
53 32 : STATIC int RsNdaGetDirectFlagByDevAttr(struct ibv_device_attr *deviceAttr)
54 : {
55 : struct {
56 : uint32_t vendorId;
57 : uint32_t vendorPartID;
58 32 : } vendorInfoList[] = {
59 : {0x19E5U, 0x0230U},
60 : {0x19E5U, 0x0231U},
61 : {0x19E5U, 0x0232U},
62 : {0xCC08U, 0x8200U},
63 : {0xCC08U, 0x8201U},
64 : };
65 32 : unsigned int listSize = sizeof(vendorInfoList) / sizeof(vendorInfoList[0]);
66 : unsigned int i;
67 :
68 192 : for (i = 0; i < listSize; i++) {
69 160 : if ((deviceAttr->vendor_id == vendorInfoList[i].vendorId) &&
70 0 : (deviceAttr->vendor_part_id == vendorInfoList[i].vendorPartID)) {
71 0 : return DIRECT_FLAG_UB;
72 : }
73 : }
74 :
75 32 : return DIRECT_FLAG_PCIE;
76 : }
77 :
78 6 : STATIC int RsGetNdaPcieDbCb(struct RsNdaCb *ndaCb, uint64_t hva, struct NdaPcieDbCb **ndaDbCb)
79 : {
80 6 : struct NdaPcieDbCb *dbCbCurr = NULL;
81 6 : struct NdaPcieDbCb *dbCbNext = NULL;
82 :
83 6 : RS_LIST_GET_HEAD_ENTRY(dbCbCurr, dbCbNext, &ndaCb->ndaPcieCb.ndaDbList, list, struct NdaPcieDbCb);
84 6 : for (; (&dbCbCurr->list) != &ndaCb->ndaPcieCb.ndaDbList;
85 0 : dbCbCurr = dbCbNext, dbCbNext = list_entry(dbCbNext->list.next, struct NdaPcieDbCb, list)) {
86 3 : if (dbCbCurr->hva == hva) {
87 3 : *ndaDbCb = dbCbCurr;
88 3 : return 0;
89 : }
90 : }
91 :
92 3 : *ndaDbCb = NULL;
93 3 : hccp_info("ndaDbCb for hva:0x%llx does not exist", hva);
94 3 : return -ENODEV;
95 : }
96 :
97 7 : STATIC int RsGetNdaUbDbCb(struct RsNdaCb *ndaCb, uint64_t guidL, uint64_t guidH, struct NdaUbDbCb **ndaDbCb)
98 : {
99 7 : struct NdaUbDbCb *dbCbCurr = NULL;
100 7 : struct NdaUbDbCb *dbCbNext = NULL;
101 :
102 7 : RS_LIST_GET_HEAD_ENTRY(dbCbCurr, dbCbNext, &ndaCb->ndaUbCb.ndaDbList, list, struct NdaUbDbCb);
103 7 : for (; (&dbCbCurr->list) != &ndaCb->ndaUbCb.ndaDbList;
104 0 : dbCbCurr = dbCbNext, dbCbNext = list_entry(dbCbNext->list.next, struct NdaUbDbCb, list)) {
105 3 : if (dbCbCurr->guidL == guidL && dbCbCurr->guidH == guidH) {
106 3 : *ndaDbCb = dbCbCurr;
107 3 : return 0;
108 : }
109 : }
110 :
111 4 : *ndaDbCb = NULL;
112 4 : hccp_info("ndaDbCb for guidL:0x%llx guidH:0x%llx does not exist", guidL, guidH);
113 4 : return -ENODEV;
114 : }
115 :
116 0 : STATIC void *RsNdaPcieAlloc(size_t size)
117 : {
118 0 : struct RsNdaCb *ndaCb = (struct RsNdaCb *)gRsCb->ndaCb;
119 0 : void *ptr = NULL;
120 :
121 0 : CHK_PRT_RETURN(ndaCb == NULL || ndaCb->ndaOps.alloc == NULL,
122 : hccp_err("ndaCb or ndaOps.alloc is NULL, chipId:%u", gRsCb->chipId), NULL);
123 :
124 0 : ptr = ndaCb->ndaOps.alloc(size);
125 0 : CHK_PRT_RETURN(ptr == NULL, hccp_err("ptr alloc failed"), NULL);
126 :
127 0 : return ptr;
128 : }
129 :
130 0 : STATIC void RsNdaPcieFree(void *ptr)
131 : {
132 0 : struct RsNdaCb *ndaCb = (struct RsNdaCb *)gRsCb->ndaCb;
133 :
134 0 : if (ndaCb == NULL || ndaCb->ndaOps.free == NULL) {
135 0 : hccp_err("gRsCb->ndaCb or ndaOps.free is NULL, chipId:%u", gRsCb->chipId);
136 0 : return;
137 : }
138 :
139 0 : ndaCb->ndaOps.free(ptr);
140 0 : ptr = NULL;
141 : }
142 :
143 4 : STATIC void *RsNdaUbAlloc(size_t size)
144 : {
145 4 : struct RsNdaCb *ndaCb = (struct RsNdaCb *)gRsCb->ndaCb;
146 4 : struct DVattribute attr = {0};
147 4 : void *ptr = NULL;
148 4 : int ret = 0;
149 :
150 4 : CHK_PRT_RETURN(ndaCb == NULL || ndaCb->ndaOps.alloc == NULL || ndaCb->ndaOps.free == NULL,
151 : hccp_err("ndaCb or alloc or free is NULL, chipId:%u", gRsCb->chipId), NULL);
152 :
153 2 : ptr = ndaCb->ndaOps.alloc(size);
154 2 : CHK_PRT_RETURN(ptr == NULL, hccp_err("ptr alloc failed"), NULL);
155 :
156 2 : ret = DlDrvMemGetAttribute((uint64_t)(uintptr_t)ptr, &attr);
157 2 : if (ret != 0) {
158 1 : hccp_err("DlDrvMemGetAttribute failed, ret:%d", ret);
159 1 : goto free_ptr;
160 : }
161 :
162 1 : if (attr.memType == DV_MEM_LOCK_DEV) {
163 1 : ret = DlHalMemRegUbSegment(attr.devId, (uint64_t)(uintptr_t)ptr, size);
164 1 : if (ret != 0) {
165 1 : hccp_err("DlHalMemRegUbSegment failed, ret:%d devId:%u size:%zu", ret, attr.devId, size);
166 1 : goto free_ptr;
167 : }
168 : }
169 :
170 0 : return ptr;
171 :
172 2 : free_ptr:
173 2 : ndaCb->ndaOps.free(ptr);
174 2 : ptr = NULL;
175 2 : return NULL;
176 : }
177 :
178 0 : STATIC void RsNdaUbFree(void *ptr)
179 : {
180 0 : struct RsNdaCb *ndaCb = (struct RsNdaCb *)gRsCb->ndaCb;
181 0 : struct DVattribute attr = {0};
182 0 : int ret = 0;
183 :
184 0 : if (ndaCb == NULL || ndaCb->ndaOps.free == NULL) {
185 0 : hccp_err("gRsCb->ndaCb or ndaOps.free is NULL, chipId:%u", gRsCb->chipId);
186 0 : return;
187 : }
188 :
189 0 : ret = DlDrvMemGetAttribute((uint64_t)(uintptr_t)ptr, &attr);
190 0 : if (ret != 0) {
191 0 : hccp_err("DlDrvMemGetAttribute failed, ret:%d", ret);
192 0 : goto free_ptr;
193 : }
194 :
195 0 : if (attr.memType == DV_MEM_LOCK_DEV) {
196 0 : (void)DlHalMemUnRegUbSegment(attr.devId, (uint64_t)(uintptr_t)ptr);
197 : }
198 :
199 0 : free_ptr:
200 0 : ndaCb->ndaOps.free(ptr);
201 0 : ptr = NULL;
202 : }
203 :
204 0 : STATIC void RsNdaMemset(void *dst, int value, size_t count)
205 : {
206 0 : struct RsNdaCb *ndaCb = (struct RsNdaCb *)gRsCb->ndaCb;
207 :
208 0 : if (ndaCb == NULL || ndaCb->ndaOps.memset_s == NULL) {
209 0 : hccp_err("gRsCb->ndaCb or ndaOps.memset_s is NULL, chipId:%u", gRsCb->chipId);
210 0 : return;
211 : }
212 :
213 0 : ndaCb->ndaOps.memset_s(dst, value, count);
214 : }
215 :
216 0 : STATIC int RsNdaMemcpy(void *dst, size_t dstSize, void *src, size_t srcSize, uint32_t direct)
217 : {
218 0 : struct RsNdaCb *ndaCb = (struct RsNdaCb *)gRsCb->ndaCb;
219 :
220 0 : if (ndaCb == NULL || ndaCb->ndaOps.memcpy_s == NULL) {
221 0 : hccp_err("gRsCb->ndaCb or ndaOps.memcpy_s is NULL, chipId:%u", gRsCb->chipId);
222 0 : return -ENODEV;
223 : }
224 :
225 0 : return ndaCb->ndaOps.memcpy_s(dst, dstSize, src, srcSize, direct);
226 : }
227 :
228 5 : STATIC void RsNdaFreeUbDbCb(struct NdaUbDbCb *ndaDbCb)
229 : {
230 5 : if (ndaDbCb->refCnt != 0) {
231 1 : return;
232 : }
233 :
234 4 : RsListDel(&ndaDbCb->list);
235 4 : free(ndaDbCb);
236 4 : ndaDbCb = NULL;
237 : }
238 :
239 4 : STATIC void *RsNdaDbMmapHostVa(struct RsNdaCb *ndaCb, struct doorbell_map_desc *desc)
240 : {
241 4 : uint64_t alignHva = AlignDown(desc->hva, (uint64_t)RA_RS_4K_PAGE_SIZE);
242 4 : uint64_t alignSize = AlignUp(desc->size, (uint64_t)RA_RS_4K_PAGE_SIZE);
243 4 : uint64_t offset = desc->hva - alignHva;
244 4 : unsigned int logicId = gRsCb->logicId;
245 4 : struct NdaPcieDbCb *ndaDbCb = NULL;
246 4 : void *dbDva = NULL;
247 4 : int ret = 0;
248 :
249 4 : ret = RsGetNdaPcieDbCb(ndaCb, alignHva, &ndaDbCb);
250 4 : if (ret == 0) {
251 1 : ndaDbCb->refCnt++;
252 1 : return (void *)(uintptr_t)(ndaDbCb->dva + offset);
253 : }
254 :
255 3 : ndaDbCb = (struct NdaPcieDbCb *)calloc(1, sizeof(struct NdaPcieDbCb));
256 3 : CHK_PRT_RETURN(ndaDbCb == NULL, hccp_err("ndaDbCb calloc failed"), NULL);
257 :
258 3 : ret = DlHalHostRegister((void *)(uintptr_t)alignHva, alignSize, HOST_IO_MAP_DEV, logicId, &dbDva);
259 3 : if (ret != 0) {
260 2 : hccp_err("register host failed, chipId:%u logicId:%u ret:%d alignHva:0x%llx", gRsCb->chipId, logicId, ret,
261 : alignHva);
262 2 : free(ndaDbCb);
263 2 : ndaDbCb = NULL;
264 2 : return NULL;
265 : }
266 :
267 1 : ndaDbCb->hva = alignHva;
268 1 : ndaDbCb->dva = (uint64_t)(uintptr_t)dbDva;
269 1 : ndaDbCb->refCnt++;
270 1 : RsListAddTail(&ndaDbCb->list, &ndaCb->ndaPcieCb.ndaDbList);
271 1 : return (void *)(uintptr_t)(ndaDbCb->dva + offset);
272 : }
273 :
274 7 : STATIC void RsNdaMapPrivPrepare(struct doorbell_map_desc *desc, struct NdaUbResMapPrivInfo *resMapIn)
275 : {
276 7 : resMapIn->guid_l = desc->ub_res.guid_l;
277 7 : resMapIn->guid_h = desc->ub_res.guid_h;
278 7 : resMapIn->db_idx = desc->ub_res.bits.offset / (uint64_t)RA_RS_4K_PAGE_SIZE;
279 7 : resMapIn->db_num = 1;
280 7 : return;
281 : }
282 :
283 5 : STATIC void *RsNdaDbMmapUbRes(struct RsNdaCb *ndaCb, struct doorbell_map_desc *desc)
284 : {
285 5 : struct NdaUbResMapPrivInfo resMapIn = {0};
286 5 : struct res_map_info_out resInfoOut = {0};
287 5 : struct res_map_info_in resInfoIn = {0};
288 5 : unsigned int logicId = gRsCb->logicId;
289 5 : struct NdaUbDbCb *ndaDbCb = NULL;
290 5 : uint64_t dva = 0;
291 5 : int ret = 0;
292 :
293 5 : ret = RsGetNdaUbDbCb(ndaCb, desc->ub_res.guid_l, desc->ub_res.guid_h, &ndaDbCb);
294 5 : if (ret == 0) {
295 1 : goto map_db;
296 : }
297 :
298 4 : ndaDbCb = (struct NdaUbDbCb *)calloc(1, sizeof(struct NdaUbDbCb));
299 4 : CHK_PRT_RETURN(ndaDbCb == NULL, hccp_err("ndaDbCb calloc failed"), NULL);
300 4 : ndaDbCb->guidL = desc->ub_res.guid_l;
301 4 : ndaDbCb->guidH = desc->ub_res.guid_h;
302 4 : ndaDbCb->guidIdx = ndaCb->ndaUbCb.ndaDbGuidCnt;
303 4 : RsListAddTail(&ndaDbCb->list, &ndaCb->ndaUbCb.ndaDbList);
304 4 : ndaCb->ndaUbCb.ndaDbGuidCnt++;
305 :
306 5 : map_db:
307 5 : RsNdaMapPrivPrepare(desc, &resMapIn);
308 5 : resInfoIn.target_proc_type = PROCESS_CP1;
309 5 : resInfoIn.res_type = RES_ADDR_TYPE_NDA_URMA_DB;
310 5 : resInfoIn.res_id = RsNdaGenerateResId(resMapIn.db_idx, ndaDbCb->guidIdx);
311 5 : resInfoIn.priv_len = sizeof(struct NdaUbResMapPrivInfo);
312 5 : resInfoIn.priv = (void *)&resMapIn;
313 5 : ret = DlHalResAddrMapV2(logicId, &resInfoIn, &resInfoOut);
314 5 : if (ret != 0) {
315 3 : hccp_err("DlHalResAddrMapV2 failed, chipId:%u logicId:%u resId:0x%x ret:%d", gRsCb->chipId, logicId,
316 : resInfoIn.res_id, ret);
317 3 : RsNdaFreeUbDbCb(ndaDbCb);
318 3 : return NULL;
319 : }
320 :
321 2 : ndaDbCb->refCnt++;
322 2 : dva = resInfoOut.va + (desc->ub_res.bits.offset % (uint64_t)RA_RS_4K_PAGE_SIZE);
323 2 : return (void *)(uintptr_t)dva;
324 : }
325 :
326 11 : STATIC void *RsNdaDbMmap(struct doorbell_map_desc *desc)
327 : {
328 11 : struct RsNdaCb *ndaCb = (struct RsNdaCb *)gRsCb->ndaCb;
329 :
330 11 : CHK_PRT_RETURN(desc == NULL, hccp_err("desc is null"), NULL);
331 11 : CHK_PRT_RETURN(ndaCb == NULL, hccp_err("ndaCb is null, chipId:%u", gRsCb->chipId), NULL);
332 :
333 11 : if (desc->type == DB_MAP_MODE_HOST_VA) {
334 4 : return RsNdaDbMmapHostVa(ndaCb, desc);
335 7 : } else if (desc->type == DB_MAP_MODE_UB_RES) {
336 5 : return RsNdaDbMmapUbRes(ndaCb, desc);
337 : } else {
338 2 : hccp_err("invalid desc->type:%u, chipId:%u", desc->type, gRsCb->chipId);
339 2 : return NULL;
340 : }
341 : }
342 :
343 3 : STATIC int RsNdaDbUnmapHostVa(struct RsNdaCb *ndaCb, void *ptr, struct doorbell_map_desc *desc)
344 : {
345 : (void)ptr;
346 3 : uint64_t alignHva = AlignDown(desc->hva, (uint64_t)RA_RS_4K_PAGE_SIZE);
347 3 : unsigned int logicId = gRsCb->logicId;
348 3 : struct NdaPcieDbCb *ndaDbCb = NULL;
349 3 : int ret = 0;
350 :
351 3 : ret = RsGetNdaPcieDbCb(ndaCb, alignHva, &ndaDbCb);
352 3 : CHK_PRT_RETURN(ret != 0,
353 : hccp_err("RsGetNdaPcieDbCb failed, hva:0x%llx chipId:%u ret:%d", alignHva, gRsCb->chipId, ret), ret);
354 :
355 2 : ndaDbCb->refCnt--;
356 2 : if (ndaDbCb->refCnt != 0) {
357 1 : return ret;
358 : }
359 :
360 1 : ret = DlHalHostUnRegisterEx((void *)(uintptr_t)alignHva, logicId, HOST_IO_MAP_DEV);
361 1 : if (ret != 0) {
362 0 : hccp_err("DlHalHostUnRegisterEx failed, chipId:%u logicId:%u ret:%d", gRsCb->chipId, logicId, ret);
363 : }
364 :
365 1 : RsListDel(&ndaDbCb->list);
366 1 : free(ndaDbCb);
367 1 : ndaDbCb = NULL;
368 1 : return ret;
369 : }
370 :
371 3 : STATIC int RsNdaDbUnmapUbRes(struct RsNdaCb *ndaCb, void *ptr, struct doorbell_map_desc *desc)
372 : {
373 : (void)ptr;
374 3 : struct NdaUbResMapPrivInfo resMapIn = {0};
375 3 : struct res_map_info_in resInfoIn = {0};
376 3 : unsigned int logicId = gRsCb->logicId;
377 3 : struct NdaUbDbCb *ndaDbCb = NULL;
378 3 : int ret = 0;
379 :
380 3 : ret = RsGetNdaUbDbCb(ndaCb, desc->ub_res.guid_l, desc->ub_res.guid_h, &ndaDbCb);
381 3 : CHK_PRT_RETURN(ret != 0,
382 : hccp_err("RsGetNdaUbDbCb failed, chipId:%u guidL:0x%llx guidH:0x%llx", gRsCb->chipId, desc->ub_res.guid_l,
383 : desc->ub_res.guid_h),
384 : ret);
385 :
386 2 : RsNdaMapPrivPrepare(desc, &resMapIn);
387 2 : resInfoIn.target_proc_type = PROCESS_CP1;
388 2 : resInfoIn.res_type = RES_ADDR_TYPE_NDA_URMA_DB;
389 2 : resInfoIn.res_id = RsNdaGenerateResId(resMapIn.db_idx, ndaDbCb->guidIdx);
390 2 : resInfoIn.priv_len = sizeof(struct NdaUbResMapPrivInfo);
391 2 : resInfoIn.priv = (void *)&resMapIn;
392 2 : ret = DlHalResAddrUnmapV2(logicId, &resInfoIn);
393 2 : if (ret != 0) {
394 0 : hccp_err("DlHalResAddrUnmapV2 failed, chipId:%u logicId:%u resId:0x%x ret:%d", gRsCb->chipId, logicId,
395 : resInfoIn.res_id, ret);
396 : }
397 :
398 2 : ndaDbCb->refCnt--;
399 2 : RsNdaFreeUbDbCb(ndaDbCb);
400 2 : return ret;
401 : }
402 :
403 6 : STATIC int RsNdaDbUnmap(void *ptr, struct doorbell_map_desc *desc)
404 : {
405 6 : struct RsNdaCb *ndaCb = (struct RsNdaCb *)gRsCb->ndaCb;
406 :
407 6 : CHK_PRT_RETURN(ptr == NULL || desc == NULL, hccp_err("ptr or desc is null"), -EINVAL);
408 6 : CHK_PRT_RETURN(ndaCb == NULL, hccp_err("ndaCb is null, chipId:%u", gRsCb->chipId), -ENODEV);
409 :
410 6 : if (desc->type == DB_MAP_MODE_HOST_VA) {
411 3 : return RsNdaDbUnmapHostVa(ndaCb, ptr, desc);
412 3 : } else if (desc->type == DB_MAP_MODE_UB_RES) {
413 3 : return RsNdaDbUnmapUbRes(ndaCb, ptr, desc);
414 : } else {
415 0 : hccp_err("invalid desc->type:%u, chipId:%u", desc->type, gRsCb->chipId);
416 0 : return -EINVAL;
417 : }
418 : }
419 :
420 32 : STATIC void RsNdaCbInitCb(struct RsNdaCb *ndaCb)
421 : {
422 32 : ndaCb->ndaPcieCb.ibvExOps.alloc = RsNdaPcieAlloc;
423 32 : ndaCb->ndaPcieCb.ibvExOps.free = RsNdaPcieFree;
424 32 : ndaCb->ndaPcieCb.ibvExOps.db_mmap = RsNdaDbMmap;
425 32 : ndaCb->ndaPcieCb.ibvExOps.db_unmap = RsNdaDbUnmap;
426 32 : ndaCb->ndaPcieCb.ibvExOps.memset_s = RsNdaMemset;
427 32 : ndaCb->ndaPcieCb.ibvExOps.memcpy_s = RsNdaMemcpy;
428 :
429 32 : ndaCb->ndaUbCb.ibvExOps.alloc = RsNdaUbAlloc;
430 32 : ndaCb->ndaUbCb.ibvExOps.free = RsNdaUbFree;
431 32 : ndaCb->ndaUbCb.ibvExOps.db_mmap = RsNdaDbMmap;
432 32 : ndaCb->ndaUbCb.ibvExOps.db_unmap = RsNdaDbUnmap;
433 32 : ndaCb->ndaUbCb.ibvExOps.memset_s = RsNdaMemset;
434 32 : ndaCb->ndaUbCb.ibvExOps.memcpy_s = RsNdaMemcpy;
435 :
436 32 : RS_INIT_LIST_HEAD(&ndaCb->ndaPcieCb.ndaDbList);
437 32 : RS_INIT_LIST_HEAD(&ndaCb->ndaUbCb.ndaDbList);
438 32 : }
439 :
440 32 : int RsInitNdaCb(struct RsRdevCb *rdevCb)
441 : {
442 32 : struct RsNdaCb *ndaCb = NULL;
443 32 : int count = 0;
444 32 : int ret = 0;
445 :
446 32 : rdevCb->directFlag = RsNdaGetDirectFlagByDevAttr(&rdevCb->deviceAttr);
447 :
448 32 : rdevCb->ibCtxEx = RsIbvOpenExtend(rdevCb->ibCtx);
449 32 : if (rdevCb->ibCtxEx == NULL) {
450 0 : return 0;
451 : }
452 :
453 32 : count = __sync_fetch_and_add(&rdevCb->rsCb->ndaCbRefCnt, 1);
454 32 : if (count > 0) {
455 0 : return 0;
456 : }
457 :
458 32 : ndaCb = (struct RsNdaCb *)calloc(1, sizeof(struct RsNdaCb));
459 32 : if (ndaCb == NULL) {
460 0 : hccp_err("calloc for ndaCb failed");
461 0 : ret = -ENOMEM;
462 0 : goto calloc_err;
463 : }
464 :
465 32 : RsNdaCbInitCb(ndaCb);
466 :
467 32 : rdevCb->rsCb->ndaCb = (void *)ndaCb;
468 32 : return ret;
469 :
470 0 : calloc_err:
471 0 : (void)__sync_fetch_and_sub(&rdevCb->rsCb->ndaCbRefCnt, 1);
472 0 : (void)RsIbvCloseExtend(rdevCb->ibCtxEx);
473 0 : rdevCb->ibCtxEx = NULL;
474 0 : return ret;
475 : }
476 :
477 32 : void RsDeinitNdaCb(struct RsRdevCb *rdevCb)
478 : {
479 32 : if (rdevCb->ibCtxEx == NULL) {
480 1 : return;
481 : }
482 :
483 31 : (void)RsIbvCloseExtend(rdevCb->ibCtxEx);
484 31 : rdevCb->ibCtxEx = NULL;
485 :
486 31 : if (__sync_fetch_and_sub(&rdevCb->rsCb->ndaCbRefCnt, 1) > 1) {
487 0 : return;
488 : }
489 :
490 31 : free(rdevCb->rsCb->ndaCb);
491 31 : rdevCb->rsCb->ndaCb = NULL;
492 : }
493 :
494 4 : STATIC void RsNdaInitExOps(struct RsNdaCb *ndaCb, uint32_t dmaMode, struct NdaOps *ops, struct ibv_extend_ops **extOps)
495 : {
496 4 : ndaCb->ndaOps.alloc = ops->alloc;
497 4 : ndaCb->ndaOps.free = ops->free;
498 4 : ndaCb->ndaOps.memset_s = ops->memset_s;
499 4 : ndaCb->ndaOps.memcpy_s = ops->memcpy_s;
500 4 : if (dmaMode == QBUF_DMA_MODE_DEFAULT) {
501 2 : *extOps = &ndaCb->ndaPcieCb.ibvExOps;
502 2 : } else if (dmaMode == QBUF_DMA_MODE_INDEP_UB) {
503 2 : *extOps = &ndaCb->ndaUbCb.ibvExOps;
504 : } else {
505 0 : *extOps = NULL;
506 : }
507 4 : }
508 :
509 2 : STATIC void RsNdaCqInitExPrepare(struct NdaCqInitAttr *attr, struct RsNdaCb *ndaCb,
510 : struct ibv_cq_init_attr_extend *cqInitAttrEx)
511 : {
512 2 : (void)memcpy_s(&cqInitAttrEx->attr, sizeof(struct ibv_cq_init_attr_ex), &attr->attr,
513 : sizeof(struct ibv_cq_init_attr_ex));
514 2 : cqInitAttrEx->cq_cap_flag = attr->cqCapFlag;
515 2 : cqInitAttrEx->type = attr->dmaMode;
516 2 : RsNdaInitExOps(ndaCb, attr->dmaMode, attr->ops, &cqInitAttrEx->ops);
517 2 : }
518 :
519 1 : STATIC int RsNdaCqCreateEx(struct RsRdevCb *rdevCb, struct ibv_cq_init_attr_extend *cqInitAttrEx,
520 : struct NdaCqInfo *info, void **ibvCqExt)
521 : {
522 1 : struct ibv_cq_extend *cqExt = NULL;
523 1 : int ret = 0;
524 :
525 1 : cqExt = RsIbvCreateCqExtend(rdevCb->ibCtxEx, cqInitAttrEx);
526 1 : CHK_PRT_RETURN(cqExt == NULL, hccp_err("RsNdaCreateCqExtend failed, errno:%d", errno), -ENOMEM);
527 :
528 1 : ret = memcpy_s(&info->cqInfo, sizeof(struct queueInfo), &cqExt->cq_info, sizeof(struct queue_info));
529 1 : CHK_PRT_RETURN(ret != 0, hccp_err("memcpy_s queue_info failed, ret:%d", ret), -ESAFEFUNC);
530 1 : ret = memcpy_s(info->resv, sizeof(info->resv), cqExt->resv, sizeof(cqExt->resv));
531 1 : CHK_PRT_RETURN(ret != 0, hccp_err("memcpy_s resv failed, ret:%d", ret), -ESAFEFUNC);
532 1 : info->cq = cqExt->cq;
533 1 : *ibvCqExt = cqExt;
534 1 : return 0;
535 : }
536 :
537 5 : RS_ATTRI_VISI_DEF int RsNdaCqCreate(unsigned int phyId, unsigned int rdevIndex, struct NdaCqInitAttr *attr,
538 : struct NdaCqInfo *info, void **ibvCqExt)
539 : {
540 5 : struct ibv_cq_init_attr_extend cqInitAttrEx = {0};
541 5 : struct RsRdevCb *rdevCb = NULL;
542 5 : struct RsNdaCb *ndaCb = NULL;
543 5 : int ret = 0;
544 :
545 5 : CHK_PRT_RETURN(attr == NULL || info == NULL, hccp_err("attr or info is NULL, phyId:%u", phyId), -EINVAL);
546 4 : CHK_PRT_RETURN(attr->dmaMode >= QBUF_DMA_MODE_MAX,
547 : hccp_err("param err, dmaMode:%u >= %u, phyId:%u", attr->dmaMode, QBUF_DMA_MODE_MAX, phyId), -EINVAL);
548 :
549 3 : ret = RsQueryRdevCb(phyId, rdevIndex, &rdevCb);
550 3 : CHK_PRT_RETURN(ret != 0, hccp_err("RsQueryRdevCb failed, phyId:%u rdevIndex:%u ret:%d", phyId, rdevIndex, ret),
551 : ret);
552 :
553 2 : ndaCb = (struct RsNdaCb *)rdevCb->rsCb->ndaCb;
554 2 : CHK_PRT_RETURN(ndaCb == NULL, hccp_err("ndaCb is NULL, does not support nda, phyId:%u", phyId), -ENODEV);
555 :
556 2 : RsNdaCqInitExPrepare(attr, ndaCb, &cqInitAttrEx);
557 :
558 2 : ret = RsNdaCqCreateEx(rdevCb, &cqInitAttrEx, info, ibvCqExt);
559 2 : CHK_PRT_RETURN(ret != 0, hccp_err("RsNdaCqCreateEx failed, phyId:%u rdevIndex:%u ret:%d", phyId, rdevIndex, ret),
560 : ret);
561 :
562 1 : return ret;
563 : }
564 :
565 1 : RS_ATTRI_VISI_DEF int RsNdaCqDestroy(unsigned int phyId, unsigned int rdevIndex, void *ibvCqExt)
566 : {
567 1 : struct RsRdevCb *rdevCb = NULL;
568 1 : int ret = 0;
569 :
570 1 : ret = RsQueryRdevCb(phyId, rdevIndex, &rdevCb);
571 1 : CHK_PRT_RETURN(ret != 0, hccp_err("RsQueryRdevCb failed, phyId:%u rdevIndex:%u ret:%d", phyId, rdevIndex, ret),
572 : ret);
573 :
574 1 : ret = RsIbvDestroyCqExtend(rdevCb->ibCtxEx, ibvCqExt);
575 1 : CHK_PRT_RETURN(ret != 0,
576 : hccp_err("RsIbvDestroyCqExtend failed, phyId:%u rdevIndex:%u ret:%d", phyId, rdevIndex, ret), ret);
577 :
578 1 : return ret;
579 : }
580 :
581 2 : STATIC int RsBuildUpNdaQpcb(struct RsRdevCb *rdevCb, struct ibv_qp_init_attr *qpInitAttr, struct RsQpCb **qpCb)
582 : {
583 2 : int ret = 0;
584 :
585 2 : *qpCb = (struct RsQpCb *)calloc(1, sizeof(struct RsQpCb));
586 2 : CHK_PRT_RETURN(*qpCb == NULL, hccp_err("RsQpCb calloc failed"), -ENOMEM);
587 :
588 2 : ret = pthread_mutex_init(&(*qpCb)->qpMutex, NULL);
589 2 : if (ret != 0) {
590 0 : hccp_err("pthread_mutex_init failed, ret:%d", ret);
591 0 : goto qp_mutex_init_err;
592 : }
593 :
594 2 : (*qpCb)->rdevCb = rdevCb;
595 2 : RS_INIT_LIST_HEAD(&(*qpCb)->mrList);
596 2 : RS_INIT_LIST_HEAD(&(*qpCb)->remMrList);
597 2 : (*qpCb)->state = RS_QP_STATUS_DISCONNECT;
598 2 : (*qpCb)->ibPd = rdevCb->ibPd;
599 2 : (*qpCb)->txDepth = qpInitAttr->cap.max_send_wr;
600 2 : (*qpCb)->rxDepth = qpInitAttr->cap.max_recv_wr;
601 :
602 2 : (*qpCb)->numRecvCqEvents = 0;
603 2 : (*qpCb)->numSendCqEvents = 0;
604 2 : (*qpCb)->qosAttr.tc = (RS_ROCE_DSCP_33 & RS_DSCP_MASK) << RS_DSCP_OFF;
605 2 : (*qpCb)->qosAttr.sl = RS_ROCE_4_SL;
606 2 : (*qpCb)->timeout = RS_QP_ATTR_TIMEOUT;
607 2 : (*qpCb)->retryCnt = RS_QP_ATTR_RETRY_CNT;
608 :
609 2 : return ret;
610 :
611 0 : qp_mutex_init_err:
612 0 : free(*qpCb);
613 0 : *qpCb = NULL;
614 0 : return ret;
615 : }
616 :
617 2 : STATIC void RsNdaQpInitExPrepare(struct RsRdevCb *rdevCb, struct NdaQpInitAttr *attr,
618 : struct ibv_qp_init_attr_extend *qpInitAttrEx)
619 : {
620 2 : struct RsNdaCb *ndaCb = (struct RsNdaCb *)rdevCb->rsCb->ndaCb;
621 :
622 2 : qpInitAttrEx->pd = rdevCb->ibPd;
623 2 : (void)memcpy_s(&qpInitAttrEx->attr, sizeof(struct ibv_qp_init_attr), &attr->attr, sizeof(struct ibv_qp_init_attr));
624 2 : qpInitAttrEx->qp_cap_flag = attr->qpCapFlag;
625 2 : qpInitAttrEx->type = attr->dmaMode;
626 2 : RsNdaInitExOps(ndaCb, attr->dmaMode, attr->ops, &qpInitAttrEx->ops);
627 2 : }
628 :
629 1 : STATIC int RsNdaQpGetQpInfo(struct RsQpCb *qpCb, struct NdaQpInfo *info)
630 : {
631 1 : int ret = 0;
632 :
633 1 : info->qp = qpCb->ibQp;
634 1 : ret = memcpy_s(&info->sqInfo, sizeof(struct queueInfo), &qpCb->ibQpEx->sq_info, sizeof(struct queue_info));
635 1 : CHK_PRT_RETURN(ret != 0, hccp_err("memcpy_s sq_info failed, ret:%d", ret), -ESAFEFUNC);
636 1 : ret = memcpy_s(&info->rqInfo, sizeof(struct queueInfo), &qpCb->ibQpEx->rq_info, sizeof(struct queue_info));
637 1 : CHK_PRT_RETURN(ret != 0, hccp_err("memcpy_s rq_info failed, ret:%d", ret), -ESAFEFUNC);
638 1 : ret = memcpy_s(info->resv, sizeof(info->resv), qpCb->ibQpEx->resv, sizeof(qpCb->ibQpEx->resv));
639 1 : CHK_PRT_RETURN(ret != 0, hccp_err("memcpy_s resv failed, ret:%d", ret), -ESAFEFUNC);
640 :
641 1 : return ret;
642 : }
643 :
644 3 : STATIC int RsNdaQpCreateEx(struct RsQpCb *qpCb, struct ibv_qp_init_attr_extend *qpInitAttrEx, struct NdaQpInfo *info)
645 : {
646 3 : struct ibv_qp_attr qpAttr = {0};
647 3 : struct ibv_port_attr attr = {0};
648 3 : struct RsRdevCb *rdevCb = NULL;
649 3 : int ret = 0;
650 :
651 3 : rdevCb = qpCb->rdevCb;
652 3 : qpCb->ibQpEx = RsIbvCreateQpExtend(rdevCb->ibCtxEx, qpInitAttrEx);
653 3 : CHK_PRT_RETURN(qpCb->ibQpEx == NULL, hccp_err("RsNdaCreateQpExtend failed, errno:%d", errno), -ENOMEM);
654 :
655 3 : qpCb->ibQp = qpCb->ibQpEx->qp;
656 3 : ret = RsIbvQueryQp(qpCb->ibQp, &qpAttr, IBV_QP_CAP, &qpInitAttrEx->attr);
657 3 : if (ret != 0) {
658 1 : hccp_err("query qp attr failed ret:%d", ret);
659 1 : ret = -EOPENSRC;
660 1 : goto nda_init_qp_err;
661 : }
662 :
663 2 : ret = RsDrvQpInfoRelated(qpCb, rdevCb, &attr, &qpAttr);
664 2 : if (ret != 0) {
665 1 : hccp_err("qp info related failed ret:%d", ret);
666 1 : goto nda_init_qp_err;
667 : }
668 :
669 1 : ret = RsNdaQpGetQpInfo(qpCb, info);
670 1 : if (ret != 0) {
671 0 : hccp_err("RsNdaQpGetQpInfo failed ret:%d", ret);
672 0 : goto nda_init_qp_err;
673 : }
674 :
675 1 : hccp_info("chip_id:%u, rdevIndex:%u, qp:%d create succ", rdevCb->rsCb->chipId, rdevCb->rdevIndex,
676 : qpCb->qpInfoLo.qpn);
677 1 : return ret;
678 :
679 2 : nda_init_qp_err:
680 2 : (void)RsIbvDestroyQpExtend(rdevCb->ibCtxEx, qpCb->ibQpEx);
681 2 : qpCb->ibQpEx = NULL;
682 2 : return ret;
683 : }
684 :
685 5 : RS_ATTRI_VISI_DEF int RsNdaQpCreate(unsigned int phyId, unsigned int rdevIndex, struct NdaQpInitAttr *attr,
686 : struct NdaQpInfo *info, struct RsQpResp *qpResp)
687 : {
688 5 : struct ibv_qp_init_attr_extend qpInitAttrEx = {0};
689 5 : struct RsRdevCb *rdevCb = NULL;
690 5 : struct RsNdaCb *ndaCb = NULL;
691 5 : struct RsQpCb *qpCb = NULL;
692 5 : int ret = 0;
693 :
694 5 : CHK_PRT_RETURN(attr == NULL || info == NULL || qpResp == NULL,
695 : hccp_err("attr or info or qpResp is NULL, phyId:%u", phyId), -EINVAL);
696 4 : CHK_PRT_RETURN(attr->dmaMode >= QBUF_DMA_MODE_MAX,
697 : hccp_err("param err, dmaMode:%u >= %u, phyId:%u", attr->dmaMode, QBUF_DMA_MODE_MAX, phyId), -EINVAL);
698 :
699 3 : ret = RsQueryRdevCb(phyId, rdevIndex, &rdevCb);
700 3 : CHK_PRT_RETURN(ret != 0, hccp_err("RsQueryRdevCb phyId:%u rdevIndex:%u ret:%d", phyId, rdevIndex, ret), ret);
701 :
702 2 : ndaCb = (struct RsNdaCb *)rdevCb->rsCb->ndaCb;
703 2 : CHK_PRT_RETURN(ndaCb == NULL, hccp_err("ndaCb is NULL, phyId:%u rdevIndex:%u", phyId, rdevIndex), -ENODEV);
704 :
705 2 : ret = RsBuildUpNdaQpcb(rdevCb, &attr->attr, &qpCb);
706 2 : CHK_PRT_RETURN(ret != 0, hccp_err("RsBuildUpNdaQpcb failed, ret:%d", ret), ret);
707 :
708 2 : RsNdaQpInitExPrepare(rdevCb, attr, &qpInitAttrEx);
709 :
710 2 : ret = RsNdaQpCreateEx(qpCb, &qpInitAttrEx, info);
711 2 : if (ret != 0) {
712 1 : hccp_err("create nda qp create extend failed, ret:%d", ret);
713 1 : goto create_qp_err;
714 : }
715 :
716 1 : RS_PTHREAD_MUTEX_LOCK(&rdevCb->rdevMutex);
717 1 : RsListAddTail(&qpCb->list, &rdevCb->qpList);
718 1 : rdevCb->qpCnt++;
719 1 : RS_PTHREAD_MUTEX_ULOCK(&rdevCb->rdevMutex);
720 1 : qpResp->qpn = (unsigned int)qpCb->qpInfoLo.qpn;
721 1 : qpResp->gidIdx = (unsigned int)qpCb->qpInfoLo.gidIdx;
722 1 : qpResp->psn = (unsigned int)qpCb->qpInfoLo.psn;
723 1 : qpResp->gid = qpCb->qpInfoLo.gid;
724 1 : qpResp->directFlag = rdevCb->directFlag;
725 1 : return 0;
726 :
727 1 : create_qp_err:
728 1 : pthread_mutex_destroy(&qpCb->qpMutex);
729 1 : free(qpCb);
730 1 : qpCb = NULL;
731 1 : return ret;
732 : }
733 :
734 1 : RS_ATTRI_VISI_DEF int RsNdaQpDestroy(unsigned int phyId, unsigned int rdevIndex, unsigned int qpn)
735 : {
736 1 : struct RsQpCb *qpCb = NULL;
737 1 : int ret = 0;
738 :
739 1 : ret = RsQpn2qpcb(phyId, rdevIndex, qpn, &qpCb);
740 1 : CHK_PRT_RETURN(ret != 0 || qpCb == NULL, hccp_err("get qp cb failed, qpn:%u, ret:%d", qpn, ret), ret);
741 :
742 1 : RS_PTHREAD_MUTEX_LOCK(&qpCb->rdevCb->rdevMutex);
743 1 : RsListDel(&qpCb->list);
744 1 : qpCb->rdevCb->qpCnt--;
745 1 : RS_PTHREAD_MUTEX_ULOCK(&qpCb->rdevCb->rdevMutex);
746 :
747 1 : RsMrRelease(qpCb);
748 :
749 1 : ret = RsIbvDestroyQpExtend(qpCb->rdevCb->ibCtxEx, qpCb->ibQpEx);
750 1 : if (ret != 0) {
751 0 : hccp_err("qp:%u destroy extend failed, ret:%d", qpn, ret);
752 : }
753 1 : qpCb->ibQpEx = NULL;
754 :
755 1 : pthread_mutex_destroy(&qpCb->qpMutex);
756 :
757 1 : free(qpCb);
758 1 : qpCb = NULL;
759 1 : return ret;
760 : }
761 :
762 6 : RS_ATTRI_VISI_DEF int RsGetQpHyperFeature(unsigned int phyId, unsigned int rdevIndex, unsigned int qpn,
763 : struct HyperFeature *hyperFeature)
764 : {
765 6 : struct ibv_hyroce_feature feature = {0};
766 6 : struct RsQpCb *qpCb = NULL;
767 6 : int ret = 0;
768 :
769 6 : CHK_PRT_RETURN(hyperFeature == NULL, hccp_warn("hyperFeature is NULL, phyId:%u", phyId), ret);
770 :
771 6 : ret = RsQpn2qpcb(phyId, rdevIndex, qpn, &qpCb);
772 6 : CHK_PRT_RETURN(ret != 0 || qpCb == NULL, hccp_warn("get qp cb unsuccessful qpn %u, ret %d", qpn, ret), ret);
773 :
774 4 : if (qpCb->rdevCb->ibCtxEx == NULL) {
775 1 : return 0;
776 : }
777 :
778 3 : ret = RsIbvQueryQpSupportedHyroceFeature(qpCb->rdevCb->ibCtxEx, qpCb->ibQp, qpCb->qosAttr.sl, qpCb->qosAttr.tc,
779 : &feature);
780 3 : CHK_PRT_RETURN(ret != 0,
781 : hccp_warn("RsIbvQueryQpSupportedHyroceFeature unsuccessful, qpn %u ret %d errno:%d", qpn, ret, errno), ret);
782 :
783 2 : ret = memcpy_s(hyperFeature, sizeof(struct HyperFeature), &feature, sizeof(struct ibv_hyroce_feature));
784 2 : CHK_PRT_RETURN(ret != 0,
785 : hccp_warn("memcpy_s feature unsuccessful, qpn:%u ret:%d HyperFeature len:%zu "
786 : "ibv_hyroce_feature len:%zu",
787 : qpn, ret, sizeof(struct HyperFeature), sizeof(struct ibv_hyroce_feature)),
788 : ret);
789 :
790 1 : hccp_dbg("get QpHyperFeature successful qpn %u", qpn);
791 1 : return ret;
792 : }
|
