Line data Source code
1 : /**
2 : * Copyright (c) 2026 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include <stdlib.h>
12 : #include <stdint.h>
13 : #include <errno.h>
14 : #include "securec.h"
15 : #include "dl_hal_function.h"
16 : #include "dl_ibverbs_function.h"
17 : #include "dl_ibv_extend_function.h"
18 : #include "hccp_nda.h"
19 : #include "ra_rs_err.h"
20 : #include "rs.h"
21 : #include "rs_inner.h"
22 : #include "rs_drv_rdma.h"
23 : #include "rs_rdma.h"
24 : #include "rs_nda.h"
25 :
26 2 : RS_ATTRI_VISI_DEF int RsNdaGetDirectFlag(unsigned int phyId, unsigned int rdevIndex, int *directFlag)
27 : {
28 2 : struct ibv_device_attr_extend extDevAttr = {0};
29 2 : struct RsRdevCb *rdevCb = NULL;
30 2 : int ret = 0;
31 :
32 2 : ret = RsQueryRdevCb(phyId, rdevIndex, &rdevCb);
33 2 : CHK_PRT_RETURN(ret != 0, hccp_err("RsQueryRdevCb phyId:%u rdevIndex:%u ret:%d", phyId, rdevIndex, ret), ret);
34 :
35 1 : if (rdevCb->ibCtxEx == NULL) {
36 1 : *directFlag = DIRECT_FLAG_NOTSUPP;
37 1 : return ret;
38 : }
39 :
40 0 : ret = RsIbvQueryDeviceExtend(rdevCb->ibCtxEx, &extDevAttr);
41 0 : CHK_PRT_RETURN(ret != 0, hccp_err("RsIbvQueryDeviceExtend failed, phyId:%u rdevIndex:%u ret:%d",
42 : phyId, rdevIndex, ret), ret);
43 :
44 0 : if ((extDevAttr.ext_cap & IBV_EXTEND_DEV_NDA) == 0) {
45 0 : *directFlag = DIRECT_FLAG_NOTSUPP;
46 0 : return ret;
47 : }
48 :
49 0 : *directFlag = rdevCb->directFlag;
50 0 : return ret;
51 : }
52 :
53 31 : STATIC int RsNdaGetDirectFlagByDevAttr(struct ibv_device_attr *deviceAttr)
54 : {
55 : struct {
56 : uint32_t vendorId;
57 : uint32_t vendorPartID;
58 31 : } vendorInfoList[] = {
59 : {0x19E5U, 0x0230U}, {0x19E5U, 0x0231U}, {0x19E5U, 0x0232U}, {0xCC08U, 0x8200U}, {0xCC08U, 0x8201U},
60 : };
61 31 : unsigned int listSize = sizeof(vendorInfoList) / sizeof(vendorInfoList[0]);
62 : unsigned int i;
63 :
64 186 : for (i = 0; i < listSize; i++) {
65 155 : if ((deviceAttr->vendor_id == vendorInfoList[i].vendorId) &&
66 0 : (deviceAttr->vendor_part_id == vendorInfoList[i].vendorPartID)) {
67 0 : return DIRECT_FLAG_UB;
68 : }
69 : }
70 :
71 31 : return DIRECT_FLAG_PCIE;
72 : }
73 :
74 6 : STATIC int RsGetNdaPcieDbCb(struct RsNdaCb *ndaCb, uint64_t hva, struct NdaPcieDbCb **ndaDbCb)
75 : {
76 6 : struct NdaPcieDbCb *dbCbCurr = NULL;
77 6 : struct NdaPcieDbCb *dbCbNext = NULL;
78 :
79 6 : RS_LIST_GET_HEAD_ENTRY(dbCbCurr, dbCbNext, &ndaCb->ndaPcieCb.ndaDbList, list, struct NdaPcieDbCb);
80 6 : for (; (&dbCbCurr->list) != &ndaCb->ndaPcieCb.ndaDbList;
81 0 : dbCbCurr = dbCbNext, dbCbNext = list_entry(dbCbNext->list.next, struct NdaPcieDbCb, list)) {
82 3 : if (dbCbCurr->hva == hva) {
83 3 : *ndaDbCb = dbCbCurr;
84 3 : return 0;
85 : }
86 : }
87 :
88 3 : *ndaDbCb = NULL;
89 3 : hccp_info("ndaDbCb for hva:0x%llx does not exist", hva);
90 3 : return -ENODEV;
91 : }
92 :
93 7 : STATIC int RsGetNdaUbDbCb(struct RsNdaCb *ndaCb, uint64_t guidL, uint64_t guidH, struct NdaUbDbCb **ndaDbCb)
94 : {
95 7 : struct NdaUbDbCb *dbCbCurr = NULL;
96 7 : struct NdaUbDbCb *dbCbNext = NULL;
97 :
98 7 : RS_LIST_GET_HEAD_ENTRY(dbCbCurr, dbCbNext, &ndaCb->ndaUbCb.ndaDbList, list, struct NdaUbDbCb);
99 7 : for (; (&dbCbCurr->list) != &ndaCb->ndaUbCb.ndaDbList;
100 0 : dbCbCurr = dbCbNext, dbCbNext = list_entry(dbCbNext->list.next, struct NdaUbDbCb, list)) {
101 3 : if (dbCbCurr->guidL == guidL && dbCbCurr->guidH == guidH) {
102 3 : *ndaDbCb = dbCbCurr;
103 3 : return 0;
104 : }
105 : }
106 :
107 4 : *ndaDbCb = NULL;
108 4 : hccp_info("ndaDbCb for guidL:0x%llx guidH:0x%llx does not exist", guidL, guidH);
109 4 : return -ENODEV;
110 : }
111 :
112 0 : STATIC void *RsNdaPcieAlloc(size_t size)
113 : {
114 0 : struct RsNdaCb *ndaCb = (struct RsNdaCb *)gRsCb->ndaCb;
115 0 : void *ptr = NULL;
116 :
117 0 : CHK_PRT_RETURN(ndaCb == NULL || ndaCb->ndaOps.alloc == NULL, hccp_err("ndaCb or ndaOps.alloc is NULL, chipId:%u",
118 : gRsCb->chipId), NULL);
119 :
120 0 : ptr = ndaCb->ndaOps.alloc(size);
121 0 : CHK_PRT_RETURN(ptr == NULL, hccp_err("ptr alloc failed"), NULL);
122 :
123 0 : return ptr;
124 : }
125 :
126 0 : STATIC void RsNdaPcieFree(void *ptr)
127 : {
128 0 : struct RsNdaCb *ndaCb = (struct RsNdaCb *)gRsCb->ndaCb;
129 :
130 0 : if (ndaCb == NULL || ndaCb->ndaOps.free == NULL) {
131 0 : hccp_err("gRsCb->ndaCb or ndaOps.free is NULL, chipId:%u", gRsCb->chipId);
132 0 : return;
133 : }
134 :
135 0 : ndaCb->ndaOps.free(ptr);
136 0 : ptr = NULL;
137 : }
138 :
139 4 : STATIC void *RsNdaUbAlloc(size_t size)
140 : {
141 4 : struct RsNdaCb *ndaCb = (struct RsNdaCb *)gRsCb->ndaCb;
142 4 : struct DVattribute attr = {0};
143 4 : void *ptr = NULL;
144 4 : int ret = 0;
145 :
146 4 : CHK_PRT_RETURN(ndaCb == NULL || ndaCb->ndaOps.alloc == NULL || ndaCb->ndaOps.free == NULL,
147 : hccp_err("ndaCb or alloc or free is NULL, chipId:%u", gRsCb->chipId), NULL);
148 :
149 2 : ptr = ndaCb->ndaOps.alloc(size);
150 2 : CHK_PRT_RETURN(ptr == NULL, hccp_err("ptr alloc failed"), NULL);
151 :
152 2 : ret = DlDrvMemGetAttribute((uint64_t)(uintptr_t)ptr, &attr);
153 2 : if (ret != 0) {
154 1 : hccp_err("DlDrvMemGetAttribute failed, ret:%d", ret);
155 1 : goto free_ptr;
156 : }
157 :
158 1 : if (attr.memType == DV_MEM_LOCK_DEV) {
159 1 : ret = DlHalMemRegUbSegment(attr.devId, (uint64_t)(uintptr_t)ptr, size);
160 1 : if (ret != 0) {
161 1 : hccp_err("DlHalMemRegUbSegment failed, ret:%d devId:%u size:%zu", ret, attr.devId, size);
162 1 : goto free_ptr;
163 : }
164 : }
165 :
166 0 : return ptr;
167 :
168 2 : free_ptr:
169 2 : ndaCb->ndaOps.free(ptr);
170 2 : ptr = NULL;
171 2 : return NULL;
172 : }
173 :
174 0 : STATIC void RsNdaUbFree(void *ptr)
175 : {
176 0 : struct RsNdaCb *ndaCb = (struct RsNdaCb *)gRsCb->ndaCb;
177 0 : struct DVattribute attr = {0};
178 0 : int ret = 0;
179 :
180 0 : if (ndaCb == NULL || ndaCb->ndaOps.free == NULL) {
181 0 : hccp_err("gRsCb->ndaCb or ndaOps.free is NULL, chipId:%u", gRsCb->chipId);
182 0 : return;
183 : }
184 :
185 0 : ret = DlDrvMemGetAttribute((uint64_t)(uintptr_t)ptr, &attr);
186 0 : if (ret != 0) {
187 0 : hccp_err("DlDrvMemGetAttribute failed, ret:%d", ret);
188 0 : goto free_ptr;
189 : }
190 :
191 0 : if (attr.memType == DV_MEM_LOCK_DEV) {
192 0 : (void)DlHalMemUnRegUbSegment(attr.devId, (uint64_t)(uintptr_t)ptr);
193 : }
194 :
195 0 : free_ptr:
196 0 : ndaCb->ndaOps.free(ptr);
197 0 : ptr = NULL;
198 : }
199 :
200 0 : STATIC void RsNdaMemset(void *dst, int value, size_t count)
201 : {
202 0 : struct RsNdaCb *ndaCb = (struct RsNdaCb *)gRsCb->ndaCb;
203 :
204 0 : if (ndaCb == NULL || ndaCb->ndaOps.memset_s == NULL) {
205 0 : hccp_err("gRsCb->ndaCb or ndaOps.memset_s is NULL, chipId:%u", gRsCb->chipId);
206 0 : return;
207 : }
208 :
209 0 : ndaCb->ndaOps.memset_s(dst, value, count);
210 : }
211 :
212 0 : STATIC int RsNdaMemcpy(void *dst, size_t dstSize, void *src, size_t srcSize, uint32_t direct)
213 : {
214 0 : struct RsNdaCb *ndaCb = (struct RsNdaCb *)gRsCb->ndaCb;
215 :
216 0 : if (ndaCb == NULL || ndaCb->ndaOps.memcpy_s == NULL) {
217 0 : hccp_err("gRsCb->ndaCb or ndaOps.memcpy_s is NULL, chipId:%u", gRsCb->chipId);
218 0 : return -ENODEV;
219 : }
220 :
221 0 : return ndaCb->ndaOps.memcpy_s(dst, dstSize, src, srcSize, direct);
222 : }
223 :
224 4 : STATIC void *RsNdaDbMmapHostVa(struct RsNdaCb *ndaCb, struct doorbell_map_desc *desc)
225 : {
226 4 : uint64_t alignHva = AlignDown(desc->hva, (uint64_t)RA_RS_4K_PAGE_SIZE);
227 4 : uint64_t alignSize = AlignUp(desc->size, (uint64_t)RA_RS_4K_PAGE_SIZE);
228 4 : uint64_t offset = desc->hva - alignHva;
229 4 : unsigned int logicId = gRsCb->logicId;
230 4 : struct NdaPcieDbCb *ndaDbCb = NULL;
231 4 : void *dbDva = NULL;
232 4 : int ret = 0;
233 :
234 4 : ret = RsGetNdaPcieDbCb(ndaCb, alignHva, &ndaDbCb);
235 4 : if (ret == 0) {
236 1 : ndaDbCb->refCnt++;
237 1 : return (void *)(uintptr_t)(ndaDbCb->dva + offset);
238 : }
239 :
240 3 : ndaDbCb = (struct NdaPcieDbCb *)calloc(1, sizeof(struct NdaPcieDbCb));
241 3 : CHK_PRT_RETURN(ndaDbCb == NULL, hccp_err("ndaDbCb calloc failed"), NULL);
242 :
243 3 : ret = DlHalHostRegister((void *)(uintptr_t)alignHva, alignSize, HOST_IO_MAP_DEV, logicId, &dbDva);
244 3 : if (ret != 0) {
245 2 : hccp_err("register host failed, chipId:%u logicId:%u ret:%d alignHva:0x%llx",
246 : gRsCb->chipId, logicId, ret, alignHva);
247 2 : free(ndaDbCb);
248 2 : ndaDbCb = NULL;
249 2 : return NULL;
250 : }
251 :
252 1 : ndaDbCb->hva = alignHva;
253 1 : ndaDbCb->dva = (uint64_t )(uintptr_t)dbDva;
254 1 : ndaDbCb->refCnt++;
255 1 : RsListAddTail(&ndaDbCb->list, &ndaCb->ndaPcieCb.ndaDbList);
256 1 : return (void *)(uintptr_t)(ndaDbCb->dva + offset);
257 : }
258 :
259 5 : STATIC void RsNdaMapPrivPrepare(struct doorbell_map_desc *desc, struct NdaUbResMapPrivInfo *resMapIn)
260 : {
261 5 : resMapIn->guid_l = desc->ub_res.guid_l;
262 5 : resMapIn->guid_h = desc->ub_res.guid_h;
263 5 : resMapIn->db_idx = desc->ub_res.bits.offset / (uint64_t )RA_RS_4K_PAGE_SIZE;
264 5 : resMapIn->db_num = 1;
265 5 : return;
266 : }
267 :
268 5 : STATIC void *RsNdaDbMmapUbRes(struct RsNdaCb *ndaCb, struct doorbell_map_desc *desc)
269 : {
270 5 : struct NdaUbResMapPrivInfo resMapIn = {0};
271 5 : struct res_map_info_out resInfoOut = {0};
272 5 : struct res_map_info_in resInfoIn = {0};
273 5 : unsigned int logicId = gRsCb->logicId;
274 5 : struct NdaUbDbCb *ndaGuidCb = NULL;
275 5 : int ret = 0;
276 :
277 5 : ret = RsGetNdaUbDbCb(ndaCb, desc->ub_res.guid_l, desc->ub_res.guid_h, &ndaGuidCb);
278 5 : if (ret == 0) {
279 1 : ndaGuidCb->refCnt++;
280 1 : return (void *)(uintptr_t)ndaGuidCb->dva;
281 : }
282 :
283 4 : ndaGuidCb = (struct NdaUbDbCb *)calloc(1, sizeof(struct NdaUbDbCb));
284 4 : CHK_PRT_RETURN(ndaGuidCb == NULL, hccp_err("ndaGuidCb calloc failed"), NULL);
285 :
286 4 : RsNdaMapPrivPrepare(desc, &resMapIn);
287 4 : resInfoIn.target_proc_type = PROCESS_CP1;
288 4 : resInfoIn.res_type = RES_ADDR_TYPE_NDA_URMA_DB;
289 4 : resInfoIn.res_id = RsNdaGenerateResId(resMapIn.db_idx, ndaCb->ndaUbCb.ndaDbGuidCnt);
290 4 : resInfoIn.priv_len = sizeof(struct NdaUbResMapPrivInfo);
291 4 : resInfoIn.priv = (void *)&resMapIn;
292 4 : ret = DlHalResAddrMapV2(logicId, &resInfoIn, &resInfoOut);
293 4 : if (ret != 0) {
294 3 : hccp_err("map resAddr failed, chipId:%u logicId:%u ret:%d", gRsCb->chipId, logicId, ret);
295 3 : free(ndaGuidCb);
296 3 : ndaGuidCb = NULL;
297 3 : return NULL;
298 : }
299 :
300 1 : ndaGuidCb->dva = resInfoOut.va + (desc->ub_res.bits.offset % (uint64_t)RA_RS_4K_PAGE_SIZE);
301 1 : ndaGuidCb->guidL = desc->ub_res.guid_l;
302 1 : ndaGuidCb->guidH = desc->ub_res.guid_h;
303 1 : ndaGuidCb->guidIdx = ndaCb->ndaUbCb.ndaDbGuidCnt;
304 :
305 1 : ndaGuidCb->refCnt++;
306 1 : RsListAddTail(&ndaGuidCb->list, &ndaCb->ndaUbCb.ndaDbList);
307 1 : ndaCb->ndaUbCb.ndaDbGuidCnt++;
308 1 : return (void *)(uintptr_t)ndaGuidCb->dva;
309 : }
310 :
311 11 : STATIC void *RsNdaDbMmap(struct doorbell_map_desc *desc)
312 : {
313 11 : struct RsNdaCb *ndaCb = (struct RsNdaCb *)gRsCb->ndaCb;
314 :
315 11 : CHK_PRT_RETURN(desc == NULL, hccp_err("desc is null"), NULL);
316 11 : CHK_PRT_RETURN(ndaCb == NULL, hccp_err("ndaCb is null, chipId:%u", gRsCb->chipId), NULL);
317 :
318 11 : if (desc->type == DB_MAP_MODE_HOST_VA) {
319 4 : return RsNdaDbMmapHostVa(ndaCb, desc);
320 7 : } else if (desc->type == DB_MAP_MODE_UB_RES) {
321 5 : return RsNdaDbMmapUbRes(ndaCb, desc);
322 : } else {
323 2 : hccp_err("invalid desc->type:%u, chipId:%u", desc->type, gRsCb->chipId);
324 2 : return NULL;
325 : }
326 : }
327 :
328 3 : STATIC int RsNdaDbUnmapHostVa(struct RsNdaCb *ndaCb, void *ptr, struct doorbell_map_desc *desc)
329 : {
330 3 : uint64_t alignHva = AlignDown(desc->hva, (uint64_t)RA_RS_4K_PAGE_SIZE);
331 3 : unsigned int logicId = gRsCb->logicId;
332 3 : struct NdaPcieDbCb *ndaDbCb = NULL;
333 3 : int ret = 0;
334 :
335 3 : ret = RsGetNdaPcieDbCb(ndaCb, alignHva, &ndaDbCb);
336 3 : CHK_PRT_RETURN(ret != 0, hccp_err("RsGetNdaPcieDbCb failed, hva:0x%llx chipId:%u ret:%d",
337 : alignHva, gRsCb->chipId, ret), ret);
338 :
339 2 : ndaDbCb->refCnt--;
340 2 : if (ndaDbCb->refCnt != 0) {
341 1 : return ret;
342 : }
343 :
344 1 : ret = DlHalHostUnRegisterEx((void *)(uintptr_t)alignHva, logicId, HOST_IO_MAP_DEV);
345 1 : if (ret != 0) {
346 0 : hccp_err("DlHalHostUnRegisterEx failed, chipId:%u logicId:%u ret:%d", gRsCb->chipId, logicId, ret);
347 : }
348 :
349 1 : RsListDel(&ndaDbCb->list);
350 1 : free(ndaDbCb);
351 1 : ndaDbCb = NULL;
352 1 : return ret;
353 : }
354 :
355 3 : STATIC int RsNdaDbUnmapUbRes(struct RsNdaCb *ndaCb, void *ptr, struct doorbell_map_desc *desc)
356 : {
357 3 : struct NdaUbResMapPrivInfo resMapIn = {0};
358 3 : struct res_map_info_in resInfoIn = {0};
359 3 : unsigned int logicId = gRsCb->logicId;
360 3 : struct NdaUbDbCb *ndaGuidCb = NULL;
361 3 : int ret = 0;
362 :
363 3 : ret = RsGetNdaUbDbCb(ndaCb, desc->ub_res.guid_l, desc->ub_res.guid_h, &ndaGuidCb);
364 3 : CHK_PRT_RETURN(ret != 0, hccp_err("RsGetNdaUbDbCb failed, chipId:%u guidL:0x%llx guidH:0x%llx",
365 : gRsCb->chipId, desc->ub_res.guid_l, desc->ub_res.guid_h), ret);
366 :
367 2 : ndaGuidCb->refCnt--;
368 2 : if (ndaGuidCb->refCnt != 0) {
369 1 : return ret;
370 : }
371 :
372 1 : RsNdaMapPrivPrepare(desc, &resMapIn);
373 1 : resInfoIn.target_proc_type = PROCESS_CP1;
374 1 : resInfoIn.res_type = RES_ADDR_TYPE_NDA_URMA_DB;
375 1 : resInfoIn.res_id = RsNdaGenerateResId(resMapIn.db_idx, ndaGuidCb->guidIdx);
376 1 : resInfoIn.priv_len = sizeof(struct NdaUbResMapPrivInfo);
377 1 : resInfoIn.priv = (void *)&resMapIn;
378 1 : ret = DlHalResAddrUnmapV2(logicId, &resInfoIn);
379 1 : if (ret != 0) {
380 0 : hccp_err("DlHalResAddrUnmapV2 failed, chipId:%u logicId:%u ret:%d", gRsCb->chipId, logicId, ret);
381 : }
382 :
383 1 : RsListDel(&ndaGuidCb->list);
384 1 : free(ndaGuidCb);
385 1 : ndaGuidCb = NULL;
386 1 : return ret;
387 : }
388 :
389 6 : STATIC int RsNdaDbUnmap(void *ptr, struct doorbell_map_desc *desc)
390 : {
391 6 : struct RsNdaCb *ndaCb = (struct RsNdaCb *)gRsCb->ndaCb;
392 :
393 6 : CHK_PRT_RETURN(ptr == NULL || desc == NULL, hccp_err("ptr or desc is null"), -EINVAL);
394 6 : CHK_PRT_RETURN(ndaCb == NULL, hccp_err("ndaCb is null, chipId:%u", gRsCb->chipId), -ENODEV);
395 :
396 6 : if (desc->type == DB_MAP_MODE_HOST_VA) {
397 3 : return RsNdaDbUnmapHostVa(ndaCb, ptr, desc);
398 3 : } else if (desc->type == DB_MAP_MODE_UB_RES) {
399 3 : return RsNdaDbUnmapUbRes(ndaCb, ptr, desc);
400 : } else {
401 0 : hccp_err("invalid desc->type:%u, chipId:%u", desc->type, gRsCb->chipId);
402 0 : return -EINVAL;
403 : }
404 : }
405 :
406 31 : STATIC void RsNdaCbInitCb(struct RsNdaCb *ndaCb)
407 : {
408 31 : ndaCb->ndaPcieCb.ibvExOps.alloc = RsNdaPcieAlloc;
409 31 : ndaCb->ndaPcieCb.ibvExOps.free = RsNdaPcieFree;
410 31 : ndaCb->ndaPcieCb.ibvExOps.db_mmap = RsNdaDbMmap;
411 31 : ndaCb->ndaPcieCb.ibvExOps.db_unmap = RsNdaDbUnmap;
412 31 : ndaCb->ndaPcieCb.ibvExOps.memset_s = RsNdaMemset;
413 31 : ndaCb->ndaPcieCb.ibvExOps.memcpy_s = RsNdaMemcpy;
414 :
415 31 : ndaCb->ndaUbCb.ibvExOps.alloc = RsNdaUbAlloc;
416 31 : ndaCb->ndaUbCb.ibvExOps.free = RsNdaUbFree;
417 31 : ndaCb->ndaUbCb.ibvExOps.db_mmap = RsNdaDbMmap;
418 31 : ndaCb->ndaUbCb.ibvExOps.db_unmap = RsNdaDbUnmap;
419 31 : ndaCb->ndaUbCb.ibvExOps.memset_s = RsNdaMemset;
420 31 : ndaCb->ndaUbCb.ibvExOps.memcpy_s = RsNdaMemcpy;
421 :
422 31 : RS_INIT_LIST_HEAD(&ndaCb->ndaPcieCb.ndaDbList);
423 31 : RS_INIT_LIST_HEAD(&ndaCb->ndaUbCb.ndaDbList);
424 31 : }
425 :
426 31 : int RsInitNdaCb(struct RsRdevCb *rdevCb)
427 : {
428 31 : struct RsNdaCb *ndaCb = NULL;
429 31 : int count = 0;
430 31 : int ret = 0;
431 :
432 31 : rdevCb->directFlag = RsNdaGetDirectFlagByDevAttr(&rdevCb->deviceAttr);
433 :
434 31 : rdevCb->ibCtxEx = RsIbvOpenExtend(rdevCb->ibCtx);
435 31 : if (rdevCb->ibCtxEx == NULL) {
436 0 : return 0;
437 : }
438 :
439 31 : count = __sync_fetch_and_add(&rdevCb->rsCb->ndaCbRefCnt, 1);
440 31 : if (count > 0) {
441 0 : return 0;
442 : }
443 :
444 31 : ndaCb = (struct RsNdaCb *)calloc(1, sizeof(struct RsNdaCb));
445 31 : if (ndaCb == NULL) {
446 0 : hccp_err("calloc for ndaCb failed");
447 0 : ret = -ENOMEM;
448 0 : goto calloc_err;
449 : }
450 :
451 31 : RsNdaCbInitCb(ndaCb);
452 :
453 31 : rdevCb->rsCb->ndaCb = (void *)ndaCb;
454 31 : return ret;
455 :
456 0 : calloc_err:
457 0 : (void)__sync_fetch_and_sub(&rdevCb->rsCb->ndaCbRefCnt, 1);
458 0 : (void)RsIbvCloseExtend(rdevCb->ibCtxEx);
459 0 : rdevCb->ibCtxEx = NULL;
460 0 : return ret;
461 : }
462 :
463 31 : void RsDeinitNdaCb(struct RsRdevCb *rdevCb)
464 : {
465 31 : if (rdevCb->ibCtxEx == NULL) {
466 0 : return;
467 : }
468 :
469 31 : (void)RsIbvCloseExtend(rdevCb->ibCtxEx);
470 31 : rdevCb->ibCtxEx = NULL;
471 :
472 31 : if (__sync_fetch_and_sub(&rdevCb->rsCb->ndaCbRefCnt, 1) > 1) {
473 0 : return;
474 : }
475 :
476 31 : free(rdevCb->rsCb->ndaCb);
477 31 : rdevCb->rsCb->ndaCb = NULL;
478 : }
479 :
480 4 : STATIC void RsNdaInitExOps(struct RsNdaCb *ndaCb, uint32_t dmaMode, struct NdaOps *ops, struct ibv_extend_ops **extOps)
481 : {
482 4 : ndaCb->ndaOps.alloc = ops->alloc;
483 4 : ndaCb->ndaOps.free = ops->free;
484 4 : ndaCb->ndaOps.memset_s = ops->memset_s;
485 4 : ndaCb->ndaOps.memcpy_s = ops->memcpy_s;
486 4 : if (dmaMode == QBUF_DMA_MODE_DEFAULT) {
487 2 : *extOps = &ndaCb->ndaPcieCb.ibvExOps;
488 2 : } else if (dmaMode == QBUF_DMA_MODE_INDEP_UB) {
489 2 : *extOps = &ndaCb->ndaUbCb.ibvExOps;
490 : } else {
491 0 : *extOps = NULL;
492 : }
493 4 : }
494 :
495 2 : STATIC void RsNdaCqInitExPrepare(struct NdaCqInitAttr *attr, struct RsNdaCb *ndaCb,
496 : struct ibv_cq_init_attr_extend *cqInitAttrEx)
497 : {
498 2 : (void)memcpy_s(&cqInitAttrEx->attr, sizeof(struct ibv_cq_init_attr_ex), &attr->attr,
499 : sizeof(struct ibv_cq_init_attr_ex));
500 2 : cqInitAttrEx->cq_cap_flag = attr->cqCapFlag;
501 2 : cqInitAttrEx->type = attr->dmaMode;
502 2 : RsNdaInitExOps(ndaCb, attr->dmaMode, attr->ops, &cqInitAttrEx->ops);
503 2 : }
504 :
505 1 : STATIC int RsNdaCqCreateEx(struct RsRdevCb *rdevCb, struct ibv_cq_init_attr_extend *cqInitAttrEx,
506 : struct NdaCqInfo *info, void **ibvCqExt)
507 : {
508 1 : struct ibv_cq_extend *cqExt = NULL;
509 1 : int ret = 0;
510 :
511 1 : cqExt = RsIbvCreateCqExtend(rdevCb->ibCtxEx, cqInitAttrEx);
512 1 : CHK_PRT_RETURN(cqExt == NULL, hccp_err("RsNdaCreateCqExtend failed, errno:%d", errno), -ENOMEM);
513 :
514 1 : ret = memcpy_s(&info->cqInfo, sizeof(struct queueInfo), &cqExt->cq_info, sizeof(struct queue_info));
515 1 : CHK_PRT_RETURN(ret != 0, hccp_err("memcpy_s queue_info failed, ret:%d", ret), -ESAFEFUNC);
516 1 : ret = memcpy_s(info->resv, sizeof(info->resv), cqExt->resv, sizeof(cqExt->resv));
517 1 : CHK_PRT_RETURN(ret != 0, hccp_err("memcpy_s resv failed, ret:%d", ret), -ESAFEFUNC);
518 1 : info->cq = cqExt->cq;
519 1 : *ibvCqExt = cqExt;
520 1 : return 0;
521 : }
522 :
523 5 : RS_ATTRI_VISI_DEF int RsNdaCqCreate(unsigned int phyId, unsigned int rdevIndex, struct NdaCqInitAttr *attr,
524 : struct NdaCqInfo *info, void **ibvCqExt)
525 : {
526 5 : struct ibv_cq_init_attr_extend cqInitAttrEx = {0};
527 5 : struct RsRdevCb *rdevCb = NULL;
528 5 : struct RsNdaCb *ndaCb = NULL;
529 5 : int ret = 0;
530 :
531 5 : CHK_PRT_RETURN(attr == NULL || info == NULL, hccp_err("attr or info is NULL, phyId:%u", phyId), -EINVAL);
532 4 : CHK_PRT_RETURN(attr->dmaMode >= QBUF_DMA_MODE_MAX, hccp_err("param err, dmaMode:%u >= %u, phyId:%u",
533 : attr->dmaMode, QBUF_DMA_MODE_MAX, phyId), -EINVAL);
534 :
535 3 : ret = RsQueryRdevCb(phyId, rdevIndex, &rdevCb);
536 3 : CHK_PRT_RETURN(ret != 0, hccp_err("RsQueryRdevCb failed, phyId:%u rdevIndex:%u ret:%d", phyId, rdevIndex, ret),
537 : ret);
538 :
539 2 : ndaCb = (struct RsNdaCb *)rdevCb->rsCb->ndaCb;
540 2 : CHK_PRT_RETURN(ndaCb == NULL, hccp_err("ndaCb is NULL, does not support nda, phyId:%u", phyId), -ENODEV);
541 :
542 2 : RsNdaCqInitExPrepare(attr, ndaCb, &cqInitAttrEx);
543 :
544 2 : ret = RsNdaCqCreateEx(rdevCb, &cqInitAttrEx, info, ibvCqExt);
545 2 : CHK_PRT_RETURN(ret != 0, hccp_err("RsNdaCqCreateEx failed, phyId:%u rdevIndex:%u ret:%d", phyId, rdevIndex, ret),
546 : ret);
547 :
548 1 : return ret;
549 : }
550 :
551 1 : RS_ATTRI_VISI_DEF int RsNdaCqDestroy(unsigned int phyId, unsigned int rdevIndex, void *ibvCqExt)
552 : {
553 1 : struct RsRdevCb *rdevCb = NULL;
554 1 : int ret = 0;
555 :
556 1 : ret = RsQueryRdevCb(phyId, rdevIndex, &rdevCb);
557 1 : CHK_PRT_RETURN(ret != 0, hccp_err("RsQueryRdevCb failed, phyId:%u rdevIndex:%u ret:%d", phyId, rdevIndex, ret),
558 : ret);
559 :
560 1 : ret = RsIbvDestroyCqExtend(rdevCb->ibCtxEx, ibvCqExt);
561 1 : CHK_PRT_RETURN(ret != 0, hccp_err("RsIbvDestroyCqExtend failed, phyId:%u rdevIndex:%u ret:%d",
562 : phyId, rdevIndex, ret), ret);
563 :
564 1 : return ret;
565 : }
566 :
567 2 : STATIC int RsBuildUpNdaQpcb(struct RsRdevCb *rdevCb, struct ibv_qp_init_attr *qpInitAttr, struct RsQpCb **qpCb)
568 : {
569 2 : int ret = 0;
570 :
571 2 : *qpCb = (struct RsQpCb *)calloc(1, sizeof(struct RsQpCb));
572 2 : CHK_PRT_RETURN(*qpCb == NULL, hccp_err("RsQpCb calloc failed"), -ENOMEM);
573 :
574 2 : ret = pthread_mutex_init(&(*qpCb)->qpMutex, NULL);
575 2 : if (ret != 0) {
576 0 : hccp_err("pthread_mutex_init failed, ret:%d", ret);
577 0 : goto qp_mutex_init_err;
578 : }
579 :
580 2 : (*qpCb)->rdevCb = rdevCb;
581 2 : RS_INIT_LIST_HEAD(&(*qpCb)->mrList);
582 2 : RS_INIT_LIST_HEAD(&(*qpCb)->remMrList);
583 2 : (*qpCb)->state = RS_QP_STATUS_DISCONNECT;
584 2 : (*qpCb)->ibPd = rdevCb->ibPd;
585 2 : (*qpCb)->txDepth = qpInitAttr->cap.max_send_wr;
586 2 : (*qpCb)->rxDepth = qpInitAttr->cap.max_recv_wr;
587 :
588 2 : (*qpCb)->numRecvCqEvents = 0;
589 2 : (*qpCb)->numSendCqEvents = 0;
590 2 : (*qpCb)->qosAttr.tc = (RS_ROCE_DSCP_33 & RS_DSCP_MASK) << RS_DSCP_OFF;
591 2 : (*qpCb)->qosAttr.sl = RS_ROCE_4_SL;
592 2 : (*qpCb)->timeout = RS_QP_ATTR_TIMEOUT;
593 2 : (*qpCb)->retryCnt = RS_QP_ATTR_RETRY_CNT;
594 :
595 2 : return ret;
596 :
597 0 : qp_mutex_init_err:
598 0 : free(*qpCb);
599 0 : *qpCb = NULL;
600 0 : return ret;
601 : }
602 :
603 2 : STATIC void RsNdaQpInitExPrepare(struct RsRdevCb *rdevCb, struct NdaQpInitAttr *attr,
604 : struct ibv_qp_init_attr_extend *qpInitAttrEx)
605 : {
606 2 : struct RsNdaCb *ndaCb = (struct RsNdaCb *)rdevCb->rsCb->ndaCb;
607 :
608 2 : qpInitAttrEx->pd = rdevCb->ibPd;
609 2 : (void)memcpy_s(&qpInitAttrEx->attr, sizeof(struct ibv_qp_init_attr), &attr->attr, sizeof(struct ibv_qp_init_attr));
610 2 : qpInitAttrEx->qp_cap_flag = attr->qpCapFlag;
611 2 : qpInitAttrEx->type = attr->dmaMode;
612 2 : RsNdaInitExOps(ndaCb, attr->dmaMode, attr->ops, &qpInitAttrEx->ops);
613 2 : }
614 :
615 1 : STATIC int RsNdaQpGetQpInfo(struct RsQpCb *qpCb, struct NdaQpInfo *info)
616 : {
617 1 : int ret = 0;
618 :
619 1 : info->qp = qpCb->ibQp;
620 1 : ret = memcpy_s(&info->sqInfo, sizeof(struct queueInfo), &qpCb->ibQpEx->sq_info, sizeof(struct queue_info));
621 1 : CHK_PRT_RETURN(ret != 0, hccp_err("memcpy_s sq_info failed, ret:%d", ret), -ESAFEFUNC);
622 1 : ret = memcpy_s(&info->rqInfo, sizeof(struct queueInfo), &qpCb->ibQpEx->rq_info, sizeof(struct queue_info));
623 1 : CHK_PRT_RETURN(ret != 0, hccp_err("memcpy_s rq_info failed, ret:%d", ret), -ESAFEFUNC);
624 1 : ret = memcpy_s(info->resv, sizeof(info->resv), qpCb->ibQpEx->resv, sizeof(qpCb->ibQpEx->resv));
625 1 : CHK_PRT_RETURN(ret != 0, hccp_err("memcpy_s resv failed, ret:%d", ret), -ESAFEFUNC);
626 :
627 1 : return ret;
628 : }
629 :
630 3 : STATIC int RsNdaQpCreateEx(struct RsQpCb *qpCb, struct ibv_qp_init_attr_extend *qpInitAttrEx, struct NdaQpInfo *info)
631 : {
632 3 : struct ibv_qp_attr qpAttr = {0};
633 3 : struct ibv_port_attr attr = {0};
634 3 : struct RsRdevCb *rdevCb = NULL;
635 3 : int ret = 0;
636 :
637 3 : rdevCb = qpCb->rdevCb;
638 3 : qpCb->ibQpEx = RsIbvCreateQpExtend(rdevCb->ibCtxEx, qpInitAttrEx);
639 3 : CHK_PRT_RETURN(qpCb->ibQpEx == NULL, hccp_err("RsNdaCreateQpExtend failed, errno:%d", errno), -ENOMEM);
640 :
641 3 : qpCb->ibQp = qpCb->ibQpEx->qp;
642 3 : ret = RsIbvQueryQp(qpCb->ibQp, &qpAttr, IBV_QP_CAP, &qpInitAttrEx->attr);
643 3 : if (ret != 0) {
644 1 : hccp_err("query qp attr failed ret:%d", ret);
645 1 : ret = -EOPENSRC;
646 1 : goto nda_init_qp_err;
647 : }
648 :
649 2 : ret = RsDrvQpInfoRelated(qpCb, rdevCb, &attr, &qpAttr);
650 2 : if (ret != 0) {
651 1 : hccp_err("qp info related failed ret:%d", ret);
652 1 : goto nda_init_qp_err;
653 : }
654 :
655 1 : ret = RsNdaQpGetQpInfo(qpCb, info);
656 1 : if (ret != 0) {
657 0 : hccp_err("RsNdaQpGetQpInfo failed ret:%d", ret);
658 0 : goto nda_init_qp_err;
659 : }
660 :
661 1 : hccp_info("chip_id:%u, rdevIndex:%u, qp:%d create succ", rdevCb->rsCb->chipId, rdevCb->rdevIndex,
662 : qpCb->qpInfoLo.qpn);
663 1 : return ret;
664 :
665 2 : nda_init_qp_err:
666 2 : (void)RsIbvDestroyQpExtend(rdevCb->ibCtxEx, qpCb->ibQpEx);
667 2 : qpCb->ibQpEx = NULL;
668 2 : return ret;
669 : }
670 :
671 5 : RS_ATTRI_VISI_DEF int RsNdaQpCreate(unsigned int phyId, unsigned int rdevIndex, struct NdaQpInitAttr *attr,
672 : struct NdaQpInfo *info, struct RsQpResp *qpResp)
673 : {
674 5 : struct ibv_qp_init_attr_extend qpInitAttrEx = {0};
675 5 : struct RsRdevCb *rdevCb = NULL;
676 5 : struct RsNdaCb *ndaCb = NULL;
677 5 : struct RsQpCb *qpCb = NULL;
678 5 : int ret = 0;
679 :
680 5 : CHK_PRT_RETURN(attr == NULL || info == NULL || qpResp == NULL, hccp_err("attr or info or qpResp is NULL, phyId:%u",
681 : phyId), -EINVAL);
682 4 : CHK_PRT_RETURN(attr->dmaMode >= QBUF_DMA_MODE_MAX, hccp_err("param err, dmaMode:%u >= %u, phyId:%u",
683 : attr->dmaMode, QBUF_DMA_MODE_MAX, phyId), -EINVAL);
684 :
685 3 : ret = RsQueryRdevCb(phyId, rdevIndex, &rdevCb);
686 3 : CHK_PRT_RETURN(ret != 0, hccp_err("RsQueryRdevCb phyId:%u rdevIndex:%u ret:%d", phyId, rdevIndex, ret), ret);
687 :
688 2 : ndaCb = (struct RsNdaCb *)rdevCb->rsCb->ndaCb;
689 2 : CHK_PRT_RETURN(ndaCb == NULL, hccp_err("ndaCb is NULL, phyId:%u rdevIndex:%u", phyId, rdevIndex), -ENODEV);
690 :
691 2 : ret = RsBuildUpNdaQpcb(rdevCb, &attr->attr, &qpCb);
692 2 : CHK_PRT_RETURN(ret != 0, hccp_err("RsBuildUpNdaQpcb failed, ret:%d", ret), ret);
693 :
694 2 : RsNdaQpInitExPrepare(rdevCb, attr, &qpInitAttrEx);
695 :
696 2 : ret = RsNdaQpCreateEx(qpCb, &qpInitAttrEx, info);
697 2 : if (ret != 0) {
698 1 : hccp_err("create nda qp create extend failed, ret:%d", ret);
699 1 : goto create_qp_err;
700 : }
701 :
702 1 : RS_PTHREAD_MUTEX_LOCK(&rdevCb->rdevMutex);
703 1 : RsListAddTail(&qpCb->list, &rdevCb->qpList);
704 1 : rdevCb->qpCnt++;
705 1 : RS_PTHREAD_MUTEX_ULOCK(&rdevCb->rdevMutex);
706 1 : qpResp->qpn = (unsigned int)qpCb->qpInfoLo.qpn;
707 1 : qpResp->gidIdx = (unsigned int)qpCb->qpInfoLo.gidIdx;
708 1 : qpResp->psn = (unsigned int)qpCb->qpInfoLo.psn;
709 1 : qpResp->gid = qpCb->qpInfoLo.gid;
710 1 : qpResp->directFlag = rdevCb->directFlag;
711 1 : return 0;
712 :
713 1 : create_qp_err:
714 1 : pthread_mutex_destroy(&qpCb->qpMutex);
715 1 : free(qpCb);
716 1 : qpCb = NULL;
717 1 : return ret;
718 : }
719 :
720 1 : RS_ATTRI_VISI_DEF int RsNdaQpDestroy(unsigned int phyId, unsigned int rdevIndex, unsigned int qpn)
721 : {
722 1 : struct RsQpCb *qpCb = NULL;
723 1 : int ret = 0;
724 :
725 1 : ret = RsQpn2qpcb(phyId, rdevIndex, qpn, &qpCb);
726 1 : CHK_PRT_RETURN(ret != 0 || qpCb == NULL, hccp_err("get qp cb failed, qpn:%u, ret:%d", qpn, ret), ret);
727 :
728 1 : RS_PTHREAD_MUTEX_LOCK(&qpCb->rdevCb->rdevMutex);
729 1 : RsListDel(&qpCb->list);
730 1 : qpCb->rdevCb->qpCnt--;
731 1 : RS_PTHREAD_MUTEX_ULOCK(&qpCb->rdevCb->rdevMutex);
732 :
733 1 : RsMrRelease(qpCb);
734 :
735 1 : ret = RsIbvDestroyQpExtend(qpCb->rdevCb->ibCtxEx, qpCb->ibQpEx);
736 1 : if (ret != 0) {
737 0 : hccp_err("qp:%u destroy extend failed, ret:%d", qpn, ret);
738 : }
739 1 : qpCb->ibQpEx = NULL;
740 :
741 1 : pthread_mutex_destroy(&qpCb->qpMutex);
742 :
743 1 : free(qpCb);
744 1 : qpCb = NULL;
745 1 : return ret;
746 : }
|
