Line data Source code
1 : /**
2 : * Copyright (c) 2026 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include <stdlib.h>
12 : #include <stdint.h>
13 : #include <errno.h>
14 : #include "securec.h"
15 : #include "dl_hal_function.h"
16 : #include "dl_ibverbs_function.h"
17 : #include "dl_ibv_extend_function.h"
18 : #include "hccp_nda.h"
19 : #include "ra_rs_err.h"
20 : #include "rs.h"
21 : #include "rs_inner.h"
22 : #include "rs_drv_rdma.h"
23 : #include "rs_rdma.h"
24 : #include "rs_nda.h"
25 :
26 2 : RS_ATTRI_VISI_DEF int RsNdaGetDirectFlag(unsigned int phyId, unsigned int rdevIndex, int *directFlag)
27 : {
28 2 : struct ibv_device_attr_extend extDevAttr = {0};
29 2 : struct RsRdevCb *rdevCb = NULL;
30 2 : int ret = 0;
31 :
32 2 : ret = RsQueryRdevCb(phyId, rdevIndex, &rdevCb);
33 2 : CHK_PRT_RETURN(ret != 0, hccp_err("RsQueryRdevCb phyId:%u rdevIndex:%u ret:%d", phyId, rdevIndex, ret), ret);
34 :
35 1 : if (rdevCb->ibCtxEx == NULL) {
36 1 : *directFlag = DIRECT_FLAG_NOTSUPP;
37 1 : return ret;
38 : }
39 :
40 0 : ret = RsIbvQueryDeviceExtend(rdevCb->ibCtxEx, &extDevAttr);
41 0 : CHK_PRT_RETURN(ret != 0, hccp_err("RsIbvQueryDeviceExtend failed, phyId:%u rdevIndex:%u ret:%d",
42 : phyId, rdevIndex, ret), ret);
43 :
44 0 : if ((extDevAttr.ext_cap & IBV_EXTEND_DEV_NDA) == 0) {
45 0 : *directFlag = DIRECT_FLAG_NOTSUPP;
46 0 : return ret;
47 : }
48 :
49 0 : *directFlag = rdevCb->directFlag;
50 0 : return ret;
51 : }
52 :
53 31 : STATIC int RsNdaGetDirectFlagByDevAttr(struct ibv_device_attr *deviceAttr)
54 : {
55 : struct {
56 : uint32_t vendorId;
57 : uint32_t vendorPartID;
58 31 : } vendorInfoList[] = {
59 : {0x19E5U, 0x0230U}, {0x19E5U, 0x0231U}, {0x19E5U, 0x0232U}, {0xCC08U, 0x8200U}, {0xCC08U, 0x8201U},
60 : };
61 31 : unsigned int listSize = sizeof(vendorInfoList) / sizeof(vendorInfoList[0]);
62 : unsigned int i;
63 :
64 186 : for (i = 0; i < listSize; i++) {
65 155 : if ((deviceAttr->vendor_id == vendorInfoList[i].vendorId) &&
66 0 : (deviceAttr->vendor_part_id == vendorInfoList[i].vendorPartID)) {
67 0 : return DIRECT_FLAG_UB;
68 : }
69 : }
70 :
71 31 : return DIRECT_FLAG_PCIE;
72 : }
73 :
74 6 : STATIC int RsGetNdaPcieDbCb(struct RsNdaCb *ndaCb, uint64_t hva, struct NdaPcieDbCb **ndaDbCb)
75 : {
76 6 : struct NdaPcieDbCb *dbCbCurr = NULL;
77 6 : struct NdaPcieDbCb *dbCbNext = NULL;
78 :
79 6 : RS_LIST_GET_HEAD_ENTRY(dbCbCurr, dbCbNext, &ndaCb->ndaPcieCb.ndaDbList, list, struct NdaPcieDbCb);
80 6 : for (; (&dbCbCurr->list) != &ndaCb->ndaPcieCb.ndaDbList;
81 0 : dbCbCurr = dbCbNext, dbCbNext = list_entry(dbCbNext->list.next, struct NdaPcieDbCb, list)) {
82 3 : if (dbCbCurr->hva == hva) {
83 3 : *ndaDbCb = dbCbCurr;
84 3 : return 0;
85 : }
86 : }
87 :
88 3 : *ndaDbCb = NULL;
89 3 : hccp_info("ndaDbCb for hva:0x%llx does not exist", hva);
90 3 : return -ENODEV;
91 : }
92 :
93 7 : STATIC int RsGetNdaUbDbCb(struct RsNdaCb *ndaCb, uint64_t guidL, uint64_t guidH, struct NdaUbDbCb **ndaDbCb)
94 : {
95 7 : struct NdaUbDbCb *dbCbCurr = NULL;
96 7 : struct NdaUbDbCb *dbCbNext = NULL;
97 :
98 7 : RS_LIST_GET_HEAD_ENTRY(dbCbCurr, dbCbNext, &ndaCb->ndaUbCb.ndaDbList, list, struct NdaUbDbCb);
99 7 : for (; (&dbCbCurr->list) != &ndaCb->ndaUbCb.ndaDbList;
100 0 : dbCbCurr = dbCbNext, dbCbNext = list_entry(dbCbNext->list.next, struct NdaUbDbCb, list)) {
101 3 : if (dbCbCurr->guidL == guidL && dbCbCurr->guidH == guidH) {
102 3 : *ndaDbCb = dbCbCurr;
103 3 : return 0;
104 : }
105 : }
106 :
107 4 : *ndaDbCb = NULL;
108 4 : hccp_info("ndaDbCb for guidL:0x%llx guidH:0x%llx does not exist", guidL, guidH);
109 4 : return -ENODEV;
110 : }
111 :
112 0 : STATIC void *RsNdaPcieAlloc(size_t size)
113 : {
114 0 : struct RsNdaCb *ndaCb = (struct RsNdaCb *)gRsCb->ndaCb;
115 0 : void *ptr = NULL;
116 :
117 0 : CHK_PRT_RETURN(ndaCb == NULL || ndaCb->ndaOps.alloc == NULL, hccp_err("ndaCb or ndaOps.alloc is NULL, chipId:%u",
118 : gRsCb->chipId), NULL);
119 :
120 0 : ptr = ndaCb->ndaOps.alloc(size);
121 0 : CHK_PRT_RETURN(ptr == NULL, hccp_err("ptr alloc failed"), NULL);
122 :
123 0 : return ptr;
124 : }
125 :
126 0 : STATIC void RsNdaPcieFree(void *ptr)
127 : {
128 0 : struct RsNdaCb *ndaCb = (struct RsNdaCb *)gRsCb->ndaCb;
129 :
130 0 : if (ndaCb == NULL || ndaCb->ndaOps.free == NULL) {
131 0 : hccp_err("gRsCb->ndaCb or ndaOps.free is NULL, chipId:%u", gRsCb->chipId);
132 0 : return;
133 : }
134 :
135 0 : ndaCb->ndaOps.free(ptr);
136 0 : ptr = NULL;
137 : }
138 :
139 4 : STATIC void *RsNdaUbAlloc(size_t size)
140 : {
141 4 : struct RsNdaCb *ndaCb = (struct RsNdaCb *)gRsCb->ndaCb;
142 4 : struct DVattribute attr = {0};
143 4 : void *ptr = NULL;
144 4 : int ret = 0;
145 :
146 4 : CHK_PRT_RETURN(ndaCb == NULL || ndaCb->ndaOps.alloc == NULL || ndaCb->ndaOps.free == NULL,
147 : hccp_err("ndaCb or alloc or free is NULL, chipId:%u", gRsCb->chipId), NULL);
148 :
149 2 : ptr = ndaCb->ndaOps.alloc(size);
150 2 : CHK_PRT_RETURN(ptr == NULL, hccp_err("ptr alloc failed"), NULL);
151 :
152 2 : ret = DlDrvMemGetAttribute((uint64_t)(uintptr_t)ptr, &attr);
153 2 : if (ret != 0) {
154 1 : hccp_err("DlDrvMemGetAttribute failed, ret:%d", ret);
155 1 : goto free_ptr;
156 : }
157 :
158 1 : if (attr.memType == DV_MEM_LOCK_DEV) {
159 1 : ret = DlHalMemRegUbSegment(attr.devId, (uint64_t)(uintptr_t)ptr, size);
160 1 : if (ret != 0) {
161 1 : hccp_err("DlHalMemRegUbSegment failed, ret:%d devId:%u size:%zu", ret, attr.devId, size);
162 1 : goto free_ptr;
163 : }
164 : }
165 :
166 0 : return ptr;
167 :
168 2 : free_ptr:
169 2 : ndaCb->ndaOps.free(ptr);
170 2 : ptr = NULL;
171 2 : return NULL;
172 : }
173 :
174 0 : STATIC void RsNdaUbFree(void *ptr)
175 : {
176 0 : struct RsNdaCb *ndaCb = (struct RsNdaCb *)gRsCb->ndaCb;
177 0 : struct DVattribute attr = {0};
178 0 : int ret = 0;
179 :
180 0 : if (ndaCb == NULL || ndaCb->ndaOps.free == NULL) {
181 0 : hccp_err("gRsCb->ndaCb or ndaOps.free is NULL, chipId:%u", gRsCb->chipId);
182 0 : return;
183 : }
184 :
185 0 : ret = DlDrvMemGetAttribute((uint64_t)(uintptr_t)ptr, &attr);
186 0 : if (ret != 0) {
187 0 : hccp_err("DlDrvMemGetAttribute failed, ret:%d", ret);
188 0 : goto free_ptr;
189 : }
190 :
191 0 : if (attr.memType == DV_MEM_LOCK_DEV) {
192 0 : (void)DlHalMemUnRegUbSegment(attr.devId, (uint64_t)(uintptr_t)ptr);
193 : }
194 :
195 0 : free_ptr:
196 0 : ndaCb->ndaOps.free(ptr);
197 0 : ptr = NULL;
198 : }
199 :
200 0 : STATIC void RsNdaMemset(void *dst, int value, size_t count)
201 : {
202 0 : struct RsNdaCb *ndaCb = (struct RsNdaCb *)gRsCb->ndaCb;
203 :
204 0 : if (ndaCb == NULL || ndaCb->ndaOps.memset_s == NULL) {
205 0 : hccp_err("gRsCb->ndaCb or ndaOps.memset_s is NULL, chipId:%u", gRsCb->chipId);
206 0 : return;
207 : }
208 :
209 0 : ndaCb->ndaOps.memset_s(dst, value, count);
210 : }
211 :
212 0 : STATIC int RsNdaMemcpy(void *dst, size_t dstSize, void *src, size_t srcSize, uint32_t direct)
213 : {
214 0 : struct RsNdaCb *ndaCb = (struct RsNdaCb *)gRsCb->ndaCb;
215 :
216 0 : if (ndaCb == NULL || ndaCb->ndaOps.memcpy_s == NULL) {
217 0 : hccp_err("gRsCb->ndaCb or ndaOps.memcpy_s is NULL, chipId:%u", gRsCb->chipId);
218 0 : return -ENODEV;
219 : }
220 :
221 0 : return ndaCb->ndaOps.memcpy_s(dst, dstSize, src, srcSize, direct);
222 : }
223 :
224 5 : STATIC void RsNdaFreeUbDbCb(struct NdaUbDbCb *ndaDbCb)
225 : {
226 5 : if (ndaDbCb->refCnt != 0) {
227 1 : return;
228 : }
229 :
230 4 : RsListDel(&ndaDbCb->list);
231 4 : free(ndaDbCb);
232 4 : ndaDbCb = NULL;
233 : }
234 :
235 4 : STATIC void *RsNdaDbMmapHostVa(struct RsNdaCb *ndaCb, struct doorbell_map_desc *desc)
236 : {
237 4 : uint64_t alignHva = AlignDown(desc->hva, (uint64_t)RA_RS_4K_PAGE_SIZE);
238 4 : uint64_t alignSize = AlignUp(desc->size, (uint64_t)RA_RS_4K_PAGE_SIZE);
239 4 : uint64_t offset = desc->hva - alignHva;
240 4 : unsigned int logicId = gRsCb->logicId;
241 4 : struct NdaPcieDbCb *ndaDbCb = NULL;
242 4 : void *dbDva = NULL;
243 4 : int ret = 0;
244 :
245 4 : ret = RsGetNdaPcieDbCb(ndaCb, alignHva, &ndaDbCb);
246 4 : if (ret == 0) {
247 1 : ndaDbCb->refCnt++;
248 1 : return (void *)(uintptr_t)(ndaDbCb->dva + offset);
249 : }
250 :
251 3 : ndaDbCb = (struct NdaPcieDbCb *)calloc(1, sizeof(struct NdaPcieDbCb));
252 3 : CHK_PRT_RETURN(ndaDbCb == NULL, hccp_err("ndaDbCb calloc failed"), NULL);
253 :
254 3 : ret = DlHalHostRegister((void *)(uintptr_t)alignHva, alignSize, HOST_IO_MAP_DEV, logicId, &dbDva);
255 3 : if (ret != 0) {
256 2 : hccp_err("register host failed, chipId:%u logicId:%u ret:%d alignHva:0x%llx",
257 : gRsCb->chipId, logicId, ret, alignHva);
258 2 : free(ndaDbCb);
259 2 : ndaDbCb = NULL;
260 2 : return NULL;
261 : }
262 :
263 1 : ndaDbCb->hva = alignHva;
264 1 : ndaDbCb->dva = (uint64_t )(uintptr_t)dbDva;
265 1 : ndaDbCb->refCnt++;
266 1 : RsListAddTail(&ndaDbCb->list, &ndaCb->ndaPcieCb.ndaDbList);
267 1 : return (void *)(uintptr_t)(ndaDbCb->dva + offset);
268 : }
269 :
270 7 : STATIC void RsNdaMapPrivPrepare(struct doorbell_map_desc *desc, struct NdaUbResMapPrivInfo *resMapIn)
271 : {
272 7 : resMapIn->guid_l = desc->ub_res.guid_l;
273 7 : resMapIn->guid_h = desc->ub_res.guid_h;
274 7 : resMapIn->db_idx = desc->ub_res.bits.offset / (uint64_t )RA_RS_4K_PAGE_SIZE;
275 7 : resMapIn->db_num = 1;
276 7 : return;
277 : }
278 :
279 5 : STATIC void *RsNdaDbMmapUbRes(struct RsNdaCb *ndaCb, struct doorbell_map_desc *desc)
280 : {
281 5 : struct NdaUbResMapPrivInfo resMapIn = {0};
282 5 : struct res_map_info_out resInfoOut = {0};
283 5 : struct res_map_info_in resInfoIn = {0};
284 5 : unsigned int logicId = gRsCb->logicId;
285 5 : struct NdaUbDbCb *ndaDbCb = NULL;
286 5 : uint64_t dva = 0;
287 5 : int ret = 0;
288 :
289 5 : ret = RsGetNdaUbDbCb(ndaCb, desc->ub_res.guid_l, desc->ub_res.guid_h, &ndaDbCb);
290 5 : if (ret == 0) {
291 1 : goto map_db;
292 : }
293 :
294 4 : ndaDbCb = (struct NdaUbDbCb *)calloc(1, sizeof(struct NdaUbDbCb));
295 4 : CHK_PRT_RETURN(ndaDbCb == NULL, hccp_err("ndaDbCb calloc failed"), NULL);
296 4 : ndaDbCb->guidL = desc->ub_res.guid_l;
297 4 : ndaDbCb->guidH = desc->ub_res.guid_h;
298 4 : ndaDbCb->guidIdx = ndaCb->ndaUbCb.ndaDbGuidCnt;
299 4 : RsListAddTail(&ndaDbCb->list, &ndaCb->ndaUbCb.ndaDbList);
300 4 : ndaCb->ndaUbCb.ndaDbGuidCnt++;
301 :
302 5 : map_db:
303 5 : RsNdaMapPrivPrepare(desc, &resMapIn);
304 5 : resInfoIn.target_proc_type = PROCESS_CP1;
305 5 : resInfoIn.res_type = RES_ADDR_TYPE_NDA_URMA_DB;
306 5 : resInfoIn.res_id = RsNdaGenerateResId(resMapIn.db_idx, ndaDbCb->guidIdx);
307 5 : resInfoIn.priv_len = sizeof(struct NdaUbResMapPrivInfo);
308 5 : resInfoIn.priv = (void *)&resMapIn;
309 5 : ret = DlHalResAddrMapV2(logicId, &resInfoIn, &resInfoOut);
310 5 : if (ret != 0) {
311 3 : hccp_err("DlHalResAddrMapV2 failed, chipId:%u logicId:%u resId:0x%x ret:%d",
312 : gRsCb->chipId, logicId, resInfoIn.res_id, ret);
313 3 : RsNdaFreeUbDbCb(ndaDbCb);
314 3 : return NULL;
315 : }
316 :
317 2 : ndaDbCb->refCnt++;
318 2 : dva = resInfoOut.va + (desc->ub_res.bits.offset % (uint64_t)RA_RS_4K_PAGE_SIZE);
319 2 : return (void *)(uintptr_t)dva;
320 : }
321 :
322 11 : STATIC void *RsNdaDbMmap(struct doorbell_map_desc *desc)
323 : {
324 11 : struct RsNdaCb *ndaCb = (struct RsNdaCb *)gRsCb->ndaCb;
325 :
326 11 : CHK_PRT_RETURN(desc == NULL, hccp_err("desc is null"), NULL);
327 11 : CHK_PRT_RETURN(ndaCb == NULL, hccp_err("ndaCb is null, chipId:%u", gRsCb->chipId), NULL);
328 :
329 11 : if (desc->type == DB_MAP_MODE_HOST_VA) {
330 4 : return RsNdaDbMmapHostVa(ndaCb, desc);
331 7 : } else if (desc->type == DB_MAP_MODE_UB_RES) {
332 5 : return RsNdaDbMmapUbRes(ndaCb, desc);
333 : } else {
334 2 : hccp_err("invalid desc->type:%u, chipId:%u", desc->type, gRsCb->chipId);
335 2 : return NULL;
336 : }
337 : }
338 :
339 3 : STATIC int RsNdaDbUnmapHostVa(struct RsNdaCb *ndaCb, void *ptr, struct doorbell_map_desc *desc)
340 : {
341 3 : uint64_t alignHva = AlignDown(desc->hva, (uint64_t)RA_RS_4K_PAGE_SIZE);
342 3 : unsigned int logicId = gRsCb->logicId;
343 3 : struct NdaPcieDbCb *ndaDbCb = NULL;
344 3 : int ret = 0;
345 :
346 3 : ret = RsGetNdaPcieDbCb(ndaCb, alignHva, &ndaDbCb);
347 3 : CHK_PRT_RETURN(ret != 0, hccp_err("RsGetNdaPcieDbCb failed, hva:0x%llx chipId:%u ret:%d",
348 : alignHva, gRsCb->chipId, ret), ret);
349 :
350 2 : ndaDbCb->refCnt--;
351 2 : if (ndaDbCb->refCnt != 0) {
352 1 : return ret;
353 : }
354 :
355 1 : ret = DlHalHostUnRegisterEx((void *)(uintptr_t)alignHva, logicId, HOST_IO_MAP_DEV);
356 1 : if (ret != 0) {
357 0 : hccp_err("DlHalHostUnRegisterEx failed, chipId:%u logicId:%u ret:%d", gRsCb->chipId, logicId, ret);
358 : }
359 :
360 1 : RsListDel(&ndaDbCb->list);
361 1 : free(ndaDbCb);
362 1 : ndaDbCb = NULL;
363 1 : return ret;
364 : }
365 :
366 3 : STATIC int RsNdaDbUnmapUbRes(struct RsNdaCb *ndaCb, void *ptr, struct doorbell_map_desc *desc)
367 : {
368 3 : struct NdaUbResMapPrivInfo resMapIn = {0};
369 3 : struct res_map_info_in resInfoIn = {0};
370 3 : unsigned int logicId = gRsCb->logicId;
371 3 : struct NdaUbDbCb *ndaDbCb = NULL;
372 3 : int ret = 0;
373 :
374 3 : ret = RsGetNdaUbDbCb(ndaCb, desc->ub_res.guid_l, desc->ub_res.guid_h, &ndaDbCb);
375 3 : CHK_PRT_RETURN(ret != 0, hccp_err("RsGetNdaUbDbCb failed, chipId:%u guidL:0x%llx guidH:0x%llx",
376 : gRsCb->chipId, desc->ub_res.guid_l, desc->ub_res.guid_h), ret);
377 :
378 2 : RsNdaMapPrivPrepare(desc, &resMapIn);
379 2 : resInfoIn.target_proc_type = PROCESS_CP1;
380 2 : resInfoIn.res_type = RES_ADDR_TYPE_NDA_URMA_DB;
381 2 : resInfoIn.res_id = RsNdaGenerateResId(resMapIn.db_idx, ndaDbCb->guidIdx);
382 2 : resInfoIn.priv_len = sizeof(struct NdaUbResMapPrivInfo);
383 2 : resInfoIn.priv = (void *)&resMapIn;
384 2 : ret = DlHalResAddrUnmapV2(logicId, &resInfoIn);
385 2 : if (ret != 0) {
386 0 : hccp_err("DlHalResAddrUnmapV2 failed, chipId:%u logicId:%u resId:0x%x ret:%d",
387 : gRsCb->chipId, logicId, resInfoIn.res_id, ret);
388 : }
389 :
390 2 : ndaDbCb->refCnt--;
391 2 : RsNdaFreeUbDbCb(ndaDbCb);
392 2 : return ret;
393 : }
394 :
395 6 : STATIC int RsNdaDbUnmap(void *ptr, struct doorbell_map_desc *desc)
396 : {
397 6 : struct RsNdaCb *ndaCb = (struct RsNdaCb *)gRsCb->ndaCb;
398 :
399 6 : CHK_PRT_RETURN(ptr == NULL || desc == NULL, hccp_err("ptr or desc is null"), -EINVAL);
400 6 : CHK_PRT_RETURN(ndaCb == NULL, hccp_err("ndaCb is null, chipId:%u", gRsCb->chipId), -ENODEV);
401 :
402 6 : if (desc->type == DB_MAP_MODE_HOST_VA) {
403 3 : return RsNdaDbUnmapHostVa(ndaCb, ptr, desc);
404 3 : } else if (desc->type == DB_MAP_MODE_UB_RES) {
405 3 : return RsNdaDbUnmapUbRes(ndaCb, ptr, desc);
406 : } else {
407 0 : hccp_err("invalid desc->type:%u, chipId:%u", desc->type, gRsCb->chipId);
408 0 : return -EINVAL;
409 : }
410 : }
411 :
412 31 : STATIC void RsNdaCbInitCb(struct RsNdaCb *ndaCb)
413 : {
414 31 : ndaCb->ndaPcieCb.ibvExOps.alloc = RsNdaPcieAlloc;
415 31 : ndaCb->ndaPcieCb.ibvExOps.free = RsNdaPcieFree;
416 31 : ndaCb->ndaPcieCb.ibvExOps.db_mmap = RsNdaDbMmap;
417 31 : ndaCb->ndaPcieCb.ibvExOps.db_unmap = RsNdaDbUnmap;
418 31 : ndaCb->ndaPcieCb.ibvExOps.memset_s = RsNdaMemset;
419 31 : ndaCb->ndaPcieCb.ibvExOps.memcpy_s = RsNdaMemcpy;
420 :
421 31 : ndaCb->ndaUbCb.ibvExOps.alloc = RsNdaUbAlloc;
422 31 : ndaCb->ndaUbCb.ibvExOps.free = RsNdaUbFree;
423 31 : ndaCb->ndaUbCb.ibvExOps.db_mmap = RsNdaDbMmap;
424 31 : ndaCb->ndaUbCb.ibvExOps.db_unmap = RsNdaDbUnmap;
425 31 : ndaCb->ndaUbCb.ibvExOps.memset_s = RsNdaMemset;
426 31 : ndaCb->ndaUbCb.ibvExOps.memcpy_s = RsNdaMemcpy;
427 :
428 31 : RS_INIT_LIST_HEAD(&ndaCb->ndaPcieCb.ndaDbList);
429 31 : RS_INIT_LIST_HEAD(&ndaCb->ndaUbCb.ndaDbList);
430 31 : }
431 :
432 31 : int RsInitNdaCb(struct RsRdevCb *rdevCb)
433 : {
434 31 : struct RsNdaCb *ndaCb = NULL;
435 31 : int count = 0;
436 31 : int ret = 0;
437 :
438 31 : rdevCb->directFlag = RsNdaGetDirectFlagByDevAttr(&rdevCb->deviceAttr);
439 :
440 31 : rdevCb->ibCtxEx = RsIbvOpenExtend(rdevCb->ibCtx);
441 31 : if (rdevCb->ibCtxEx == NULL) {
442 0 : return 0;
443 : }
444 :
445 31 : count = __sync_fetch_and_add(&rdevCb->rsCb->ndaCbRefCnt, 1);
446 31 : if (count > 0) {
447 0 : return 0;
448 : }
449 :
450 31 : ndaCb = (struct RsNdaCb *)calloc(1, sizeof(struct RsNdaCb));
451 31 : if (ndaCb == NULL) {
452 0 : hccp_err("calloc for ndaCb failed");
453 0 : ret = -ENOMEM;
454 0 : goto calloc_err;
455 : }
456 :
457 31 : RsNdaCbInitCb(ndaCb);
458 :
459 31 : rdevCb->rsCb->ndaCb = (void *)ndaCb;
460 31 : return ret;
461 :
462 0 : calloc_err:
463 0 : (void)__sync_fetch_and_sub(&rdevCb->rsCb->ndaCbRefCnt, 1);
464 0 : (void)RsIbvCloseExtend(rdevCb->ibCtxEx);
465 0 : rdevCb->ibCtxEx = NULL;
466 0 : return ret;
467 : }
468 :
469 31 : void RsDeinitNdaCb(struct RsRdevCb *rdevCb)
470 : {
471 31 : if (rdevCb->ibCtxEx == NULL) {
472 0 : return;
473 : }
474 :
475 31 : (void)RsIbvCloseExtend(rdevCb->ibCtxEx);
476 31 : rdevCb->ibCtxEx = NULL;
477 :
478 31 : if (__sync_fetch_and_sub(&rdevCb->rsCb->ndaCbRefCnt, 1) > 1) {
479 0 : return;
480 : }
481 :
482 31 : free(rdevCb->rsCb->ndaCb);
483 31 : rdevCb->rsCb->ndaCb = NULL;
484 : }
485 :
486 4 : STATIC void RsNdaInitExOps(struct RsNdaCb *ndaCb, uint32_t dmaMode, struct NdaOps *ops, struct ibv_extend_ops **extOps)
487 : {
488 4 : ndaCb->ndaOps.alloc = ops->alloc;
489 4 : ndaCb->ndaOps.free = ops->free;
490 4 : ndaCb->ndaOps.memset_s = ops->memset_s;
491 4 : ndaCb->ndaOps.memcpy_s = ops->memcpy_s;
492 4 : if (dmaMode == QBUF_DMA_MODE_DEFAULT) {
493 2 : *extOps = &ndaCb->ndaPcieCb.ibvExOps;
494 2 : } else if (dmaMode == QBUF_DMA_MODE_INDEP_UB) {
495 2 : *extOps = &ndaCb->ndaUbCb.ibvExOps;
496 : } else {
497 0 : *extOps = NULL;
498 : }
499 4 : }
500 :
501 2 : STATIC void RsNdaCqInitExPrepare(struct NdaCqInitAttr *attr, struct RsNdaCb *ndaCb,
502 : struct ibv_cq_init_attr_extend *cqInitAttrEx)
503 : {
504 2 : (void)memcpy_s(&cqInitAttrEx->attr, sizeof(struct ibv_cq_init_attr_ex), &attr->attr,
505 : sizeof(struct ibv_cq_init_attr_ex));
506 2 : cqInitAttrEx->cq_cap_flag = attr->cqCapFlag;
507 2 : cqInitAttrEx->type = attr->dmaMode;
508 2 : RsNdaInitExOps(ndaCb, attr->dmaMode, attr->ops, &cqInitAttrEx->ops);
509 2 : }
510 :
511 1 : STATIC int RsNdaCqCreateEx(struct RsRdevCb *rdevCb, struct ibv_cq_init_attr_extend *cqInitAttrEx,
512 : struct NdaCqInfo *info, void **ibvCqExt)
513 : {
514 1 : struct ibv_cq_extend *cqExt = NULL;
515 1 : int ret = 0;
516 :
517 1 : cqExt = RsIbvCreateCqExtend(rdevCb->ibCtxEx, cqInitAttrEx);
518 1 : CHK_PRT_RETURN(cqExt == NULL, hccp_err("RsNdaCreateCqExtend failed, errno:%d", errno), -ENOMEM);
519 :
520 1 : ret = memcpy_s(&info->cqInfo, sizeof(struct queueInfo), &cqExt->cq_info, sizeof(struct queue_info));
521 1 : CHK_PRT_RETURN(ret != 0, hccp_err("memcpy_s queue_info failed, ret:%d", ret), -ESAFEFUNC);
522 1 : ret = memcpy_s(info->resv, sizeof(info->resv), cqExt->resv, sizeof(cqExt->resv));
523 1 : CHK_PRT_RETURN(ret != 0, hccp_err("memcpy_s resv failed, ret:%d", ret), -ESAFEFUNC);
524 1 : info->cq = cqExt->cq;
525 1 : *ibvCqExt = cqExt;
526 1 : return 0;
527 : }
528 :
529 5 : RS_ATTRI_VISI_DEF int RsNdaCqCreate(unsigned int phyId, unsigned int rdevIndex, struct NdaCqInitAttr *attr,
530 : struct NdaCqInfo *info, void **ibvCqExt)
531 : {
532 5 : struct ibv_cq_init_attr_extend cqInitAttrEx = {0};
533 5 : struct RsRdevCb *rdevCb = NULL;
534 5 : struct RsNdaCb *ndaCb = NULL;
535 5 : int ret = 0;
536 :
537 5 : CHK_PRT_RETURN(attr == NULL || info == NULL, hccp_err("attr or info is NULL, phyId:%u", phyId), -EINVAL);
538 4 : CHK_PRT_RETURN(attr->dmaMode >= QBUF_DMA_MODE_MAX, hccp_err("param err, dmaMode:%u >= %u, phyId:%u",
539 : attr->dmaMode, QBUF_DMA_MODE_MAX, phyId), -EINVAL);
540 :
541 3 : ret = RsQueryRdevCb(phyId, rdevIndex, &rdevCb);
542 3 : CHK_PRT_RETURN(ret != 0, hccp_err("RsQueryRdevCb failed, phyId:%u rdevIndex:%u ret:%d", phyId, rdevIndex, ret),
543 : ret);
544 :
545 2 : ndaCb = (struct RsNdaCb *)rdevCb->rsCb->ndaCb;
546 2 : CHK_PRT_RETURN(ndaCb == NULL, hccp_err("ndaCb is NULL, does not support nda, phyId:%u", phyId), -ENODEV);
547 :
548 2 : RsNdaCqInitExPrepare(attr, ndaCb, &cqInitAttrEx);
549 :
550 2 : ret = RsNdaCqCreateEx(rdevCb, &cqInitAttrEx, info, ibvCqExt);
551 2 : CHK_PRT_RETURN(ret != 0, hccp_err("RsNdaCqCreateEx failed, phyId:%u rdevIndex:%u ret:%d", phyId, rdevIndex, ret),
552 : ret);
553 :
554 1 : return ret;
555 : }
556 :
557 1 : RS_ATTRI_VISI_DEF int RsNdaCqDestroy(unsigned int phyId, unsigned int rdevIndex, void *ibvCqExt)
558 : {
559 1 : struct RsRdevCb *rdevCb = NULL;
560 1 : int ret = 0;
561 :
562 1 : ret = RsQueryRdevCb(phyId, rdevIndex, &rdevCb);
563 1 : CHK_PRT_RETURN(ret != 0, hccp_err("RsQueryRdevCb failed, phyId:%u rdevIndex:%u ret:%d", phyId, rdevIndex, ret),
564 : ret);
565 :
566 1 : ret = RsIbvDestroyCqExtend(rdevCb->ibCtxEx, ibvCqExt);
567 1 : CHK_PRT_RETURN(ret != 0, hccp_err("RsIbvDestroyCqExtend failed, phyId:%u rdevIndex:%u ret:%d",
568 : phyId, rdevIndex, ret), ret);
569 :
570 1 : return ret;
571 : }
572 :
573 2 : STATIC int RsBuildUpNdaQpcb(struct RsRdevCb *rdevCb, struct ibv_qp_init_attr *qpInitAttr, struct RsQpCb **qpCb)
574 : {
575 2 : int ret = 0;
576 :
577 2 : *qpCb = (struct RsQpCb *)calloc(1, sizeof(struct RsQpCb));
578 2 : CHK_PRT_RETURN(*qpCb == NULL, hccp_err("RsQpCb calloc failed"), -ENOMEM);
579 :
580 2 : ret = pthread_mutex_init(&(*qpCb)->qpMutex, NULL);
581 2 : if (ret != 0) {
582 0 : hccp_err("pthread_mutex_init failed, ret:%d", ret);
583 0 : goto qp_mutex_init_err;
584 : }
585 :
586 2 : (*qpCb)->rdevCb = rdevCb;
587 2 : RS_INIT_LIST_HEAD(&(*qpCb)->mrList);
588 2 : RS_INIT_LIST_HEAD(&(*qpCb)->remMrList);
589 2 : (*qpCb)->state = RS_QP_STATUS_DISCONNECT;
590 2 : (*qpCb)->ibPd = rdevCb->ibPd;
591 2 : (*qpCb)->txDepth = qpInitAttr->cap.max_send_wr;
592 2 : (*qpCb)->rxDepth = qpInitAttr->cap.max_recv_wr;
593 :
594 2 : (*qpCb)->numRecvCqEvents = 0;
595 2 : (*qpCb)->numSendCqEvents = 0;
596 2 : (*qpCb)->qosAttr.tc = (RS_ROCE_DSCP_33 & RS_DSCP_MASK) << RS_DSCP_OFF;
597 2 : (*qpCb)->qosAttr.sl = RS_ROCE_4_SL;
598 2 : (*qpCb)->timeout = RS_QP_ATTR_TIMEOUT;
599 2 : (*qpCb)->retryCnt = RS_QP_ATTR_RETRY_CNT;
600 :
601 2 : return ret;
602 :
603 0 : qp_mutex_init_err:
604 0 : free(*qpCb);
605 0 : *qpCb = NULL;
606 0 : return ret;
607 : }
608 :
609 2 : STATIC void RsNdaQpInitExPrepare(struct RsRdevCb *rdevCb, struct NdaQpInitAttr *attr,
610 : struct ibv_qp_init_attr_extend *qpInitAttrEx)
611 : {
612 2 : struct RsNdaCb *ndaCb = (struct RsNdaCb *)rdevCb->rsCb->ndaCb;
613 :
614 2 : qpInitAttrEx->pd = rdevCb->ibPd;
615 2 : (void)memcpy_s(&qpInitAttrEx->attr, sizeof(struct ibv_qp_init_attr), &attr->attr, sizeof(struct ibv_qp_init_attr));
616 2 : qpInitAttrEx->qp_cap_flag = attr->qpCapFlag;
617 2 : qpInitAttrEx->type = attr->dmaMode;
618 2 : RsNdaInitExOps(ndaCb, attr->dmaMode, attr->ops, &qpInitAttrEx->ops);
619 2 : }
620 :
621 1 : STATIC int RsNdaQpGetQpInfo(struct RsQpCb *qpCb, struct NdaQpInfo *info)
622 : {
623 1 : int ret = 0;
624 :
625 1 : info->qp = qpCb->ibQp;
626 1 : ret = memcpy_s(&info->sqInfo, sizeof(struct queueInfo), &qpCb->ibQpEx->sq_info, sizeof(struct queue_info));
627 1 : CHK_PRT_RETURN(ret != 0, hccp_err("memcpy_s sq_info failed, ret:%d", ret), -ESAFEFUNC);
628 1 : ret = memcpy_s(&info->rqInfo, sizeof(struct queueInfo), &qpCb->ibQpEx->rq_info, sizeof(struct queue_info));
629 1 : CHK_PRT_RETURN(ret != 0, hccp_err("memcpy_s rq_info failed, ret:%d", ret), -ESAFEFUNC);
630 1 : ret = memcpy_s(info->resv, sizeof(info->resv), qpCb->ibQpEx->resv, sizeof(qpCb->ibQpEx->resv));
631 1 : CHK_PRT_RETURN(ret != 0, hccp_err("memcpy_s resv failed, ret:%d", ret), -ESAFEFUNC);
632 :
633 1 : return ret;
634 : }
635 :
636 3 : STATIC int RsNdaQpCreateEx(struct RsQpCb *qpCb, struct ibv_qp_init_attr_extend *qpInitAttrEx, struct NdaQpInfo *info)
637 : {
638 3 : struct ibv_qp_attr qpAttr = {0};
639 3 : struct ibv_port_attr attr = {0};
640 3 : struct RsRdevCb *rdevCb = NULL;
641 3 : int ret = 0;
642 :
643 3 : rdevCb = qpCb->rdevCb;
644 3 : qpCb->ibQpEx = RsIbvCreateQpExtend(rdevCb->ibCtxEx, qpInitAttrEx);
645 3 : CHK_PRT_RETURN(qpCb->ibQpEx == NULL, hccp_err("RsNdaCreateQpExtend failed, errno:%d", errno), -ENOMEM);
646 :
647 3 : qpCb->ibQp = qpCb->ibQpEx->qp;
648 3 : ret = RsIbvQueryQp(qpCb->ibQp, &qpAttr, IBV_QP_CAP, &qpInitAttrEx->attr);
649 3 : if (ret != 0) {
650 1 : hccp_err("query qp attr failed ret:%d", ret);
651 1 : ret = -EOPENSRC;
652 1 : goto nda_init_qp_err;
653 : }
654 :
655 2 : ret = RsDrvQpInfoRelated(qpCb, rdevCb, &attr, &qpAttr);
656 2 : if (ret != 0) {
657 1 : hccp_err("qp info related failed ret:%d", ret);
658 1 : goto nda_init_qp_err;
659 : }
660 :
661 1 : ret = RsNdaQpGetQpInfo(qpCb, info);
662 1 : if (ret != 0) {
663 0 : hccp_err("RsNdaQpGetQpInfo failed ret:%d", ret);
664 0 : goto nda_init_qp_err;
665 : }
666 :
667 1 : hccp_info("chip_id:%u, rdevIndex:%u, qp:%d create succ", rdevCb->rsCb->chipId, rdevCb->rdevIndex,
668 : qpCb->qpInfoLo.qpn);
669 1 : return ret;
670 :
671 2 : nda_init_qp_err:
672 2 : (void)RsIbvDestroyQpExtend(rdevCb->ibCtxEx, qpCb->ibQpEx);
673 2 : qpCb->ibQpEx = NULL;
674 2 : return ret;
675 : }
676 :
677 5 : RS_ATTRI_VISI_DEF int RsNdaQpCreate(unsigned int phyId, unsigned int rdevIndex, struct NdaQpInitAttr *attr,
678 : struct NdaQpInfo *info, struct RsQpResp *qpResp)
679 : {
680 5 : struct ibv_qp_init_attr_extend qpInitAttrEx = {0};
681 5 : struct RsRdevCb *rdevCb = NULL;
682 5 : struct RsNdaCb *ndaCb = NULL;
683 5 : struct RsQpCb *qpCb = NULL;
684 5 : int ret = 0;
685 :
686 5 : CHK_PRT_RETURN(attr == NULL || info == NULL || qpResp == NULL, hccp_err("attr or info or qpResp is NULL, phyId:%u",
687 : phyId), -EINVAL);
688 4 : CHK_PRT_RETURN(attr->dmaMode >= QBUF_DMA_MODE_MAX, hccp_err("param err, dmaMode:%u >= %u, phyId:%u",
689 : attr->dmaMode, QBUF_DMA_MODE_MAX, phyId), -EINVAL);
690 :
691 3 : ret = RsQueryRdevCb(phyId, rdevIndex, &rdevCb);
692 3 : CHK_PRT_RETURN(ret != 0, hccp_err("RsQueryRdevCb phyId:%u rdevIndex:%u ret:%d", phyId, rdevIndex, ret), ret);
693 :
694 2 : ndaCb = (struct RsNdaCb *)rdevCb->rsCb->ndaCb;
695 2 : CHK_PRT_RETURN(ndaCb == NULL, hccp_err("ndaCb is NULL, phyId:%u rdevIndex:%u", phyId, rdevIndex), -ENODEV);
696 :
697 2 : ret = RsBuildUpNdaQpcb(rdevCb, &attr->attr, &qpCb);
698 2 : CHK_PRT_RETURN(ret != 0, hccp_err("RsBuildUpNdaQpcb failed, ret:%d", ret), ret);
699 :
700 2 : RsNdaQpInitExPrepare(rdevCb, attr, &qpInitAttrEx);
701 :
702 2 : ret = RsNdaQpCreateEx(qpCb, &qpInitAttrEx, info);
703 2 : if (ret != 0) {
704 1 : hccp_err("create nda qp create extend failed, ret:%d", ret);
705 1 : goto create_qp_err;
706 : }
707 :
708 1 : RS_PTHREAD_MUTEX_LOCK(&rdevCb->rdevMutex);
709 1 : RsListAddTail(&qpCb->list, &rdevCb->qpList);
710 1 : rdevCb->qpCnt++;
711 1 : RS_PTHREAD_MUTEX_ULOCK(&rdevCb->rdevMutex);
712 1 : qpResp->qpn = (unsigned int)qpCb->qpInfoLo.qpn;
713 1 : qpResp->gidIdx = (unsigned int)qpCb->qpInfoLo.gidIdx;
714 1 : qpResp->psn = (unsigned int)qpCb->qpInfoLo.psn;
715 1 : qpResp->gid = qpCb->qpInfoLo.gid;
716 1 : qpResp->directFlag = rdevCb->directFlag;
717 1 : return 0;
718 :
719 1 : create_qp_err:
720 1 : pthread_mutex_destroy(&qpCb->qpMutex);
721 1 : free(qpCb);
722 1 : qpCb = NULL;
723 1 : return ret;
724 : }
725 :
726 1 : RS_ATTRI_VISI_DEF int RsNdaQpDestroy(unsigned int phyId, unsigned int rdevIndex, unsigned int qpn)
727 : {
728 1 : struct RsQpCb *qpCb = NULL;
729 1 : int ret = 0;
730 :
731 1 : ret = RsQpn2qpcb(phyId, rdevIndex, qpn, &qpCb);
732 1 : CHK_PRT_RETURN(ret != 0 || qpCb == NULL, hccp_err("get qp cb failed, qpn:%u, ret:%d", qpn, ret), ret);
733 :
734 1 : RS_PTHREAD_MUTEX_LOCK(&qpCb->rdevCb->rdevMutex);
735 1 : RsListDel(&qpCb->list);
736 1 : qpCb->rdevCb->qpCnt--;
737 1 : RS_PTHREAD_MUTEX_ULOCK(&qpCb->rdevCb->rdevMutex);
738 :
739 1 : RsMrRelease(qpCb);
740 :
741 1 : ret = RsIbvDestroyQpExtend(qpCb->rdevCb->ibCtxEx, qpCb->ibQpEx);
742 1 : if (ret != 0) {
743 0 : hccp_err("qp:%u destroy extend failed, ret:%d", qpn, ret);
744 : }
745 1 : qpCb->ibQpEx = NULL;
746 :
747 1 : pthread_mutex_destroy(&qpCb->qpMutex);
748 :
749 1 : free(qpCb);
750 1 : qpCb = NULL;
751 1 : return ret;
752 : }
|
