Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #define _GNU_SOURCE
12 : #include <unistd.h>
13 : #include <stdlib.h>
14 : #include <netinet/in.h>
15 : #include <arpa/inet.h>
16 : #include <dlfcn.h>
17 : #include <sys/types.h>
18 : #include <sys/stat.h>
19 : #include <sys/epoll.h>
20 : #include <sys/eventfd.h>
21 : #include <sys/socket.h>
22 : #include <errno.h>
23 : #include "securec.h"
24 : #include "rs.h"
25 : #include "ra_rs_err.h"
26 : #include "rs_common_inner.h"
27 : #include "rs_inner.h"
28 : #include "rs_rdma_inner.h"
29 : #include "rs_epoll.h"
30 : #include "dl_hal_function.h"
31 : #include "dl_ibverbs_function.h"
32 : #include "dl_ibv_extend_function.h"
33 : #include "rs_drv_socket.h"
34 : #include "rs_drv_rdma.h"
35 : #include "rs_rdma.h"
36 :
37 : unsigned int gRsSendWrNum = 0;
38 :
39 : STATIC struct RsListHead gRsTypicalCqList;
40 : STATIC pthread_mutex_t gRsTypicalCqMutex = PTHREAD_MUTEX_INITIALIZER;
41 :
42 2 : STATIC void RsBufPrint(char *addr, int len)
43 : {
44 : int i;
45 :
46 130 : for (i = 0; i < len; i++) {
47 128 : hccp_info("0x%02x ", *(addr + i));
48 : }
49 2 : }
50 :
51 137 : STATIC int RsGetQpcb(struct RsRdevCb *rdevCb, uint32_t qpn, struct RsQpCb **qpCb)
52 : {
53 137 : struct RsQpCb *qpCbTmp = NULL;
54 137 : struct RsQpCb *qpCbTmp2 = NULL;
55 :
56 137 : RS_LIST_GET_HEAD_ENTRY(qpCbTmp, qpCbTmp2, &rdevCb->qpList, list, struct RsQpCb);
57 180 : for (; (&qpCbTmp->list) != &rdevCb->qpList;
58 43 : qpCbTmp = qpCbTmp2, qpCbTmp2 = list_entry(qpCbTmp2->list.next, struct RsQpCb, list)) {
59 171 : if (qpCbTmp->ibQp->qp_num == qpn) {
60 128 : *qpCb = qpCbTmp;
61 128 : return 0;
62 : }
63 : }
64 :
65 9 : *qpCb = NULL;
66 9 : hccp_err("qp_cb for qp %u is not available", qpn);
67 :
68 9 : return -ENODEV;
69 : }
70 :
71 140 : int RsQpn2qpcb(unsigned int phyId, unsigned int rdevIndex, uint32_t qpn, struct RsQpCb **qpCb)
72 : {
73 : int ret;
74 : unsigned int chipId;
75 140 : struct rs_cb *rsCb = NULL;
76 140 : struct RsRdevCb *rdevCb = NULL;
77 :
78 140 : CHK_PRT_RETURN(phyId >= RS_MAX_DEV_NUM, hccp_err("rs set param error! phyId:%u", phyId), -EINVAL);
79 :
80 137 : ret = rsGetLocalDevIDByHostDevID(phyId, &chipId);
81 137 : CHK_PRT_RETURN(ret, hccp_err("rs_qpn2qpcb rsGetLocalDevIDByHostDevID phyId[%u] invalid, ret:%d", phyId, ret), ret);
82 :
83 137 : ret = RsDev2rscb(chipId, &rsCb, false);
84 137 : CHK_PRT_RETURN(ret, hccp_err("rs_qpn2qpcb get rs_cb failed, ret:%d", ret), -ENODEV);
85 :
86 137 : ret = RsGetRdevCb(rsCb, rdevIndex, &rdevCb);
87 137 : CHK_PRT_RETURN(ret, hccp_err("rs_get_rdev_cb failed! ret:%d, rdevIndex:%u", ret, rdevIndex), ret);
88 :
89 137 : ret = RsGetQpcb(rdevCb, qpn, qpCb);
90 137 : CHK_PRT_RETURN(ret, hccp_err("rs_get_qpcb failed! ret:%d, qpn:%u", ret, qpn), ret);
91 :
92 128 : return 0;
93 : }
94 :
95 44 : STATIC int RsGetMrcb(struct RsQpCb *qpCb, uint64_t addr, struct RsMrCb **mrCb, struct RsListHead *mrList)
96 : {
97 44 : struct RsMrCb *mrTmp = NULL;
98 44 : struct RsMrCb *mrTmp2 = NULL;
99 :
100 44 : RS_PTHREAD_MUTEX_LOCK(&qpCb->qpMutex);
101 44 : RS_LIST_GET_HEAD_ENTRY(mrTmp, mrTmp2, mrList, list, struct RsMrCb);
102 86 : for (; (&mrTmp->list) != mrList; mrTmp = mrTmp2, mrTmp2 = list_entry(mrTmp2->list.next, struct RsMrCb, list)) {
103 66 : if ((mrTmp->mrInfo.addr <= addr) && (addr < mrTmp->mrInfo.addr + mrTmp->mrInfo.len)) {
104 24 : *mrCb = mrTmp;
105 24 : RS_PTHREAD_MUTEX_ULOCK(&qpCb->qpMutex);
106 24 : return 0;
107 : }
108 : }
109 :
110 20 : *mrCb = NULL;
111 20 : RS_PTHREAD_MUTEX_ULOCK(&qpCb->qpMutex);
112 :
113 20 : hccp_info("cannot find mrcb for addr@0x%lx !", addr);
114 :
115 20 : return -ENODEV;
116 : }
117 :
118 18 : STATIC void *RsNotifyMrListAdd(struct RsQpCb *qpCb, const char *buf)
119 : {
120 : int ret;
121 : struct RsMrCb *notifyMrCb;
122 :
123 18 : notifyMrCb = calloc(1, sizeof(struct RsMrCb));
124 18 : CHK_PRT_RETURN(notifyMrCb == NULL, hccp_err("notify_mr_cb calloc failed"), NULL);
125 16 : ret = memcpy_s(¬ifyMrCb->mrInfo, sizeof(struct RsMrInfo), &((const struct RsQpInfo *)buf)->notifyMr,
126 : sizeof(struct RsMrInfo));
127 16 : if (ret) {
128 1 : hccp_err("memcpy_s failed, ret:%d, src_len:%u, dst_len:%u", ret, sizeof(struct RsMrInfo),
129 : sizeof(struct RsMrInfo));
130 1 : free(notifyMrCb);
131 1 : notifyMrCb = NULL;
132 1 : return NULL;
133 : }
134 :
135 15 : hccp_info("qpn is %d, rdevIndex:%u, chipId %u, recv notify va is 0x%llx, notify size is %llu", qpCb->qpInfoLo.qpn,
136 : qpCb->rdevCb->rdevIndex, qpCb->rdevCb->rsCb->chipId, notifyMrCb->mrInfo.addr, notifyMrCb->mrInfo.len);
137 :
138 15 : RsListAddTail(¬ifyMrCb->list, &qpCb->remMrList);
139 :
140 15 : return notifyMrCb;
141 : }
142 :
143 21 : STATIC int RsQpStateModify(struct RsQpCb *qpCb)
144 : {
145 21 : struct ibv_qp_init_attr initAttr = {0};
146 21 : struct ibv_qp_attr attr = {0};
147 : enum ibv_qp_state state;
148 : int ret;
149 :
150 : // see ib_modify_qp_is_ok for status modify, only support modify qp from INIT to RTR
151 21 : ret = RsIbvQueryQp(qpCb->ibQp, &attr, IBV_QP_STATE, &initAttr);
152 21 : if (ret != 0) {
153 1 : hccp_warn("rs_ibv_query_qp qpn:%d unsuccessful, ret:%d", qpCb->qpInfoLo.qpn, ret);
154 1 : state = IBV_QPS_UNKNOWN;
155 : } else {
156 20 : state = attr.qp_state;
157 : }
158 :
159 : // disallow modify qp from IBV_QPS_RTS to IBV_QPS_RTS
160 21 : if (state == IBV_QPS_RTS) {
161 1 : hccp_err("qpn:%d disallow modify from %d", qpCb->qpInfoLo.qpn, state);
162 1 : return -EINVAL;
163 : }
164 :
165 20 : hccp_info("qpn:%d state:%d start modify", qpCb->qpInfoLo.qpn, state);
166 :
167 : // modify qp from others to RESET
168 20 : if (state != IBV_QPS_RESET && state != IBV_QPS_INIT && state != IBV_QPS_RTR) {
169 1 : ret = RsDrvQpStateModifytoReset(qpCb);
170 1 : CHK_PRT_RETURN(ret, hccp_err("qpn:%d modify %d to reset failed, ret:%d", qpCb->qpInfoLo.qpn, state, ret), ret);
171 0 : state = IBV_QPS_RESET;
172 : }
173 :
174 : // modify qp from RESET to INIT
175 19 : if (state == IBV_QPS_RESET) {
176 19 : ret = RsDrvQpStateModifytoInit(qpCb, &attr);
177 19 : CHK_PRT_RETURN(ret, hccp_err("qpn:%d modify %d to init failed, ret %d", qpCb->qpInfoLo.qpn, state, ret), ret);
178 18 : state = IBV_QPS_INIT;
179 : }
180 :
181 : // modify qp from INIT to RTR
182 18 : if (state == IBV_QPS_INIT) {
183 18 : ret = RsDrvQpStateModifytoRtr(qpCb, &attr);
184 18 : CHK_PRT_RETURN(ret, hccp_err("qpn:%d modify %d to rtr failed, ret %d", qpCb->qpInfoLo.qpn, state, ret), ret);
185 16 : state = IBV_QPS_RTR;
186 : }
187 :
188 : // modify qp from RTR to RTS
189 16 : if (state == IBV_QPS_RTR) {
190 16 : ret = RsDrvQpStateModifytoRts(qpCb, &attr);
191 16 : CHK_PRT_RETURN(ret, hccp_err("qpn:%d modify %d to rts failed, ret %d", qpCb->qpInfoLo.qpn, state, ret), ret);
192 : }
193 :
194 16 : hccp_info("local qpn[%d] remote qpn[%d] modify succ", qpCb->qpInfoLo.qpn, qpCb->qpInfoRem.qpn);
195 :
196 16 : return 0;
197 : }
198 :
199 17 : STATIC int RsEpollRecvQpHandle(struct RsQpCb *qpCb, const char *bufTmp)
200 : {
201 : int ret;
202 17 : float timeCost = 0.0;
203 :
204 17 : ret = memcpy_s(&qpCb->qpInfoRem, sizeof(struct RsQpInfo), bufTmp, sizeof(struct RsQpInfo));
205 17 : CHK_PRT_RETURN(ret,
206 : hccp_err("memcpy_s failed[%d], dest size:%d, src size:%d", ret, sizeof(struct RsQpInfo),
207 : sizeof(struct RsQpInfo)),
208 : -ENOMEM);
209 :
210 : /* modify qp state to RTR/RTS */
211 17 : ret = RsQpStateModify(qpCb);
212 17 : CHK_PRT_RETURN(ret,
213 : hccp_err("rs_qp_state_modify local qpn[%d] remote qpn[%d] failed ret[%d]", qpCb->qpInfoLo.qpn,
214 : qpCb->qpInfoRem.qpn, ret),
215 : ret);
216 :
217 16 : RsGetCurTime(&qpCb->endTime);
218 16 : HccpTimeInterval(&qpCb->endTime, &qpCb->startTime, &timeCost);
219 16 : if (timeCost > RS_EXPECT_TIME_MAX) {
220 2 : hccp_warn("local qpn[%d] remote qpn [%d] connect success cost[%f] more than[%f]ms!", qpCb->qpInfoLo.qpn,
221 : qpCb->qpInfoRem.qpn, timeCost, RS_EXPECT_TIME_MAX);
222 : } else {
223 14 : hccp_info("local qpn[%d] remote qpn [%d] connect success! cost [%f] ms", qpCb->qpInfoLo.qpn,
224 : qpCb->qpInfoRem.qpn, timeCost);
225 : }
226 :
227 16 : hccp_info("qp [%d] state has been migrate to RTS!, qpCb state is %d", qpCb->qpInfoLo.qpn, qpCb->state);
228 :
229 16 : return 0;
230 : }
231 :
232 14 : STATIC void *RsEpollRecvMrHandle(struct RsQpCb *qpCb, const char *bufTmp)
233 : {
234 : int ret;
235 : struct RsMrCb *mrCb;
236 :
237 14 : mrCb = calloc(1, sizeof(struct RsMrCb));
238 14 : CHK_PRT_RETURN(mrCb == NULL, hccp_err("mr_cb calloc failed"), NULL);
239 13 : ret = memcpy_s(&mrCb->mrInfo, sizeof(struct RsMrInfo), bufTmp, sizeof(struct RsMrInfo));
240 13 : if (ret) {
241 0 : hccp_err("memcpy_s failed[%d], dest size:%u, src size:%u", ret, sizeof(struct RsMrInfo),
242 : sizeof(struct RsMrInfo));
243 0 : free(mrCb);
244 0 : mrCb = NULL;
245 0 : return NULL;
246 : }
247 :
248 13 : RsListAddTail(&mrCb->list, &qpCb->remMrList);
249 :
250 13 : hccp_info("recv mr addr is 0x%llx", mrCb->mrInfo.addr);
251 13 : hccp_info("recv mr len is %llu", mrCb->mrInfo.len);
252 :
253 13 : return mrCb;
254 : }
255 :
256 17 : STATIC int RsCmdQpInfoHandle(struct RsQpCb *qpCb, unsigned int totalSize, const char *bufTmp, unsigned int curSize,
257 : bool *flag)
258 : {
259 : int ret;
260 17 : CHK_PRT_RETURN((totalSize - curSize) < sizeof(struct RsQpInfo),
261 : hccp_info("qp_info remain size"
262 : "[%u] < size [%u], wait for next recv",
263 : totalSize - curSize, sizeof(struct RsQpInfo)),
264 : -EINVAL);
265 :
266 17 : ret = RsEpollRecvQpHandle(qpCb, bufTmp);
267 17 : CHK_PRT_RETURN(ret, hccp_err("rs_epoll_recv_qp_handle failed! ret[%d]", ret), ret);
268 :
269 16 : RsNotifyMrListAdd(qpCb, bufTmp);
270 16 : hccp_info("rs_notify_mr_list_add");
271 :
272 16 : *flag = true;
273 16 : hccp_info("qp_info cur_size(%u) len(%u) !", curSize, sizeof(struct RsQpInfo));
274 :
275 16 : return 0;
276 : }
277 :
278 14 : STATIC int RsCmdMrInfoHandle(struct RsQpCb *qpCb, unsigned int totalSize, const char *bufTmp, unsigned int curSize,
279 : bool *flag)
280 : {
281 14 : CHK_PRT_RETURN((totalSize - curSize) < sizeof(struct RsMrInfo),
282 : hccp_info("mr_info remain size"
283 : "[%u] < size [%u], wait for next recv",
284 : totalSize - curSize, sizeof(struct RsMrInfo)),
285 : -EINVAL);
286 :
287 14 : (void)RsEpollRecvMrHandle(qpCb, bufTmp);
288 :
289 14 : *flag = true;
290 :
291 14 : hccp_info("mr_info cur_size(%u) len(%u) !", curSize, sizeof(struct RsMrInfo));
292 :
293 14 : return 0;
294 : }
295 :
296 14 : STATIC int RsCmdLenInfoHandle(struct RsQpCb *qpCb, unsigned int totalSize, const char *bufTmp, unsigned int curSize,
297 : bool *flag)
298 : {
299 14 : CHK_PRT_RETURN((totalSize - curSize) < sizeof(struct RsQpLenInfo),
300 : hccp_info("len_info remain size"
301 : "[%u] < size [%u], wait for next recv",
302 : totalSize - curSize, sizeof(struct RsQpLenInfo)),
303 : -EINVAL);
304 :
305 14 : qpCb->expectLen = *((const uint32_t *)(bufTmp + sizeof(uint32_t)));
306 :
307 14 : *flag = true;
308 :
309 14 : return 0;
310 : }
311 :
312 15 : STATIC void RsEpollRecvHandleRemain(struct RsQpCb *qpCb, unsigned int totalSize, unsigned int curSize, bool flag,
313 : const char *bufTmp)
314 : {
315 15 : int ret = 0;
316 :
317 15 : qpCb->remainSize = totalSize - curSize;
318 15 : if ((qpCb->remainSize > 0) && (flag == true)) {
319 1 : ret = memcpy_s(qpCb->qpMrBuf, RS_BUF_SIZE, bufTmp, qpCb->remainSize);
320 1 : if (ret) {
321 1 : hccp_err("memcpy_s failed, ret:%d, remainSize:%u", ret, qpCb->remainSize);
322 1 : return;
323 : }
324 : }
325 :
326 14 : return;
327 : }
328 :
329 18 : STATIC void RsEpollRecvHandle(struct RsQpCb *qpCb, char *buf, int size)
330 : {
331 18 : unsigned int totalSize = qpCb->remainSize + (unsigned int)size;
332 18 : char *bufTmp = (char *)qpCb->qpMrBuf;
333 18 : unsigned int curSize = 0;
334 18 : bool flag = false;
335 : uint32_t cmd;
336 : int ret;
337 :
338 18 : hccp_info("Message for qp:%d, qpCb->remainSize:%u, size:%d", qpCb->qpInfoLo.qpn, qpCb->remainSize, size);
339 18 : ret = memcpy_s(qpCb->qpMrBuf + qpCb->remainSize, RS_BUF_SIZE - qpCb->remainSize, buf, size);
340 18 : if (ret) {
341 2 : hccp_err("memcpy_s failed, ret:%d, remainSize:%u, size:%d", ret, qpCb->remainSize, size);
342 4 : return;
343 : }
344 :
345 : do {
346 47 : cmd = *((uint32_t *)bufTmp);
347 47 : switch (cmd) {
348 17 : case RS_CMD_QP_INFO:
349 17 : ret = RsCmdQpInfoHandle(qpCb, totalSize, bufTmp, curSize, &flag);
350 17 : if (ret) {
351 1 : goto out;
352 : }
353 :
354 16 : curSize += sizeof(struct RsQpInfo);
355 16 : bufTmp = qpCb->qpMrBuf + curSize;
356 16 : break;
357 14 : case RS_CMD_MR_INFO:
358 14 : ret = RsCmdMrInfoHandle(qpCb, totalSize, bufTmp, curSize, &flag);
359 14 : if (ret) {
360 0 : goto out;
361 : }
362 :
363 14 : curSize += sizeof(struct RsMrInfo);
364 14 : bufTmp = qpCb->qpMrBuf + curSize;
365 14 : break;
366 14 : case RS_CMD_LEN_INFO:
367 14 : ret = RsCmdLenInfoHandle(qpCb, totalSize, bufTmp, curSize, &flag);
368 14 : if (ret) {
369 0 : goto out;
370 : }
371 14 : curSize += sizeof(struct RsQpLenInfo);
372 14 : bufTmp = qpCb->qpMrBuf + curSize;
373 14 : break;
374 2 : default:
375 2 : hccp_warn("qp %d, unknown cmd(0x%x)!", qpCb->qpInfoLo.qpn, cmd);
376 2 : RsBufPrint(buf, size);
377 2 : return;
378 : }
379 44 : } while (curSize < totalSize);
380 :
381 13 : out:
382 14 : RsEpollRecvHandleRemain(qpCb, totalSize, curSize, flag, bufTmp);
383 : }
384 :
385 27 : STATIC void RsQpMrRecvHandle(int fd, struct RsQpCb *qpCb)
386 : {
387 : char buf[RS_BUF_SIZE];
388 : int size;
389 : int ret;
390 :
391 27 : RS_PTHREAD_MUTEX_LOCK(&qpCb->qpMutex);
392 :
393 27 : size = RsSocketRecv(fd, buf, RS_BUF_SIZE - qpCb->remainSize);
394 27 : hccp_dbg("fd %d qpn %d read size = %d, qpCb->remainSize:%u", fd, qpCb->qpInfoLo.qpn, size, qpCb->remainSize);
395 :
396 27 : if (size > 0) {
397 13 : qpCb->recvLen += (uint32_t)size;
398 13 : RsEpollRecvHandle(qpCb, buf, size);
399 14 : } else if (size == 0) {
400 0 : hccp_dbg("fd %d read size = %d, remote fd has been closed, fd cannot use !", fd, size);
401 : #ifdef CA_CONFIG_LLT
402 0 : qpCb->state = RS_QP_STATUS_REM_FD_CLOSE;
403 : #endif
404 : } else {
405 14 : ret = errno;
406 14 : hccp_dbg("no data available, errno:%d", ret);
407 : }
408 27 : RS_PTHREAD_MUTEX_ULOCK(&qpCb->qpMutex);
409 :
410 27 : return;
411 : }
412 :
413 0 : STATIC int RsHandleQpMrEpollEvent(struct RsRdevCb *rdevCb, int fd)
414 : {
415 : struct RsQpCb *qpCb;
416 0 : struct RsQpCb *qpCb2 = NULL;
417 :
418 : /* QP event, QP info exchange */
419 0 : RS_LIST_GET_HEAD_ENTRY(qpCb, qpCb2, &rdevCb->qpList, list, struct RsQpCb);
420 0 : for (; (&qpCb->list) != &rdevCb->qpList; qpCb = qpCb2, qpCb2 = list_entry(qpCb2->list.next, struct RsQpCb, list)) {
421 0 : if (qpCb->channel == NULL) {
422 0 : continue;
423 : }
424 0 : if (qpCb->srqContext != NULL && qpCb->srqContext->channel->fd == fd) {
425 0 : hccp_dbg("fd %d poll cq!", fd);
426 0 : RsDrvPollSrqCqHandle(qpCb);
427 0 : return 0;
428 : }
429 0 : if (fd == qpCb->channel->fd) {
430 0 : hccp_dbg("fd %d poll cq!", fd);
431 0 : RsDrvPollCqHandle(qpCb);
432 0 : return 0;
433 : }
434 : }
435 0 : return -ENODEV;
436 : }
437 :
438 0 : int RsEpollEventQpMrInHandle(struct rs_cb *rsCb, int fd)
439 : {
440 : int ret;
441 0 : struct RsRdevCb *rdevCbTmp = NULL;
442 0 : struct RsRdevCb *rdevCbTmp2 = NULL;
443 :
444 0 : if (rsCb->protocol != PROTOCOL_RDMA) {
445 0 : return -ENODEV;
446 : }
447 :
448 0 : RS_LIST_GET_HEAD_ENTRY(rdevCbTmp, rdevCbTmp2, &rsCb->rdevList, list, struct RsRdevCb);
449 0 : for (; (&rdevCbTmp->list) != &rsCb->rdevList;
450 0 : rdevCbTmp = rdevCbTmp2, rdevCbTmp2 = list_entry(rdevCbTmp2->list.next, struct RsRdevCb, list)) {
451 0 : RS_PTHREAD_MUTEX_LOCK(&rdevCbTmp->rdevMutex);
452 0 : ret = RsHandleQpMrEpollEvent(rdevCbTmp, fd);
453 0 : RS_PTHREAD_MUTEX_ULOCK(&rdevCbTmp->rdevMutex);
454 0 : if (ret == 0) {
455 0 : return 0;
456 : }
457 : }
458 0 : return -ENODEV;
459 : }
460 :
461 28 : STATIC int RsMrInfoSync(struct RsMrCb *mrCb)
462 : {
463 : int ret;
464 :
465 28 : hccp_info("mr state:%d, addr:0x%lx", mrCb->state, mrCb->mrInfo.addr);
466 :
467 28 : CHK_PRT_RETURN(mrCb->state & RS_MR_STATE_SYNCED,
468 : hccp_warn("mr synced ! mr_cb->flag[%d] & [%d] != 0", mrCb->state, RS_MR_STATE_SYNCED), 0);
469 :
470 : /*
471 : * no socket available for MR_INFO exchange if allowed
472 : * need exchange when socket available
473 : */
474 27 : CHK_PRT_RETURN(mrCb->qpCb->connInfo == NULL, hccp_warn("no conn available !"), 0);
475 :
476 27 : CHK_PRT_RETURN(mrCb->qpCb->state == RS_QP_STATUS_REM_FD_CLOSE,
477 : hccp_warn("remote qp fd closed, "
478 : "can not use it anymore! status[%d](RS_QP_STATUS_REM_FD_CLOSE)",
479 : mrCb->qpCb->state),
480 : -EFAULT);
481 :
482 27 : CHK_PRT_RETURN(mrCb->qpCb->connInfo->connfd == RS_FD_INVALID,
483 : hccp_warn("mr info sync failed! fd not ready! connfd[%d](RS_FD_INVALID)", mrCb->qpCb->connInfo->connfd),
484 : -ENETUNREACH);
485 :
486 26 : mrCb->mrInfo.cmd = (unsigned int)RS_CMD_MR_INFO;
487 26 : ret = RsSocketSend(mrCb->qpCb->connInfo->connfd, &mrCb->mrInfo, sizeof(struct RsMrInfo));
488 26 : CHK_PRT_RETURN(ret != sizeof(struct RsMrInfo),
489 : hccp_err("mr_info send %d/%ld incomplete", ret, sizeof(struct RsMrInfo)), -EAGAIN);
490 :
491 25 : mrCb->qpCb->sendLen += (uint32_t)ret;
492 25 : mrCb->state |= RS_MR_STATE_SYNCED;
493 25 : hccp_info("after send mr state:%d, addr:0x%lx", mrCb->state, mrCb->mrInfo.addr);
494 :
495 25 : return 0;
496 : }
497 :
498 5 : STATIC int RsMrPrepareRoceSign(unsigned int phyId, struct RsRdevCb *devCb, struct roce_process_sign *roceSign)
499 : {
500 5 : unsigned int tmpPhyId = phyId;
501 : unsigned int chipId;
502 : int ret;
503 :
504 : // reg mr with backup phyId
505 5 : if (devCb->backupInfo.backupFlag) {
506 0 : tmpPhyId = devCb->backupInfo.rdevInfo.phyId;
507 0 : ret = DlDrvGetLocalDevIdByHostDevId(tmpPhyId, &chipId);
508 : } else {
509 5 : ret = rsGetLocalDevIDByHostDevID(tmpPhyId, &chipId);
510 : }
511 5 : CHK_PRT_RETURN(ret != 0,
512 : hccp_err("get chipId failed, ret %d, phyid[%u] backupFlag[%d]", ret, tmpPhyId, devCb->backupInfo.backupFlag),
513 : -EACCES);
514 5 : roceSign->tgid = devCb->rsCb->pRsSign.tgid;
515 5 : roceSign->devid = chipId;
516 5 : roceSign->vfid = 0;
517 5 : ret = strcpy_s(roceSign->sign, PROCESS_RS_SIGN_LENGTH, devCb->rsCb->pRsSign.sign);
518 5 : CHK_PRT_RETURN(ret != 0, hccp_err("Invalid pid sign, ret(%d)", ret), -ESAFEFUNC);
519 5 : return 0;
520 : }
521 :
522 8 : STATIC int RsMrPreReg(unsigned int phyId, struct RsQpCb *qpCb, struct RsMrCb *mrCb, struct RdmaMrRegInfo *mrRegInfo)
523 : {
524 8 : unsigned long long len = mrRegInfo->len;
525 8 : struct roce_process_sign roceSign = {0};
526 8 : int access = mrRegInfo->access;
527 8 : char *addr = mrRegInfo->addr;
528 : int ret;
529 :
530 8 : if (qpCb->rdevCb->rsCb->hccpMode == NETWORK_PEER_ONLINE || qpCb->rdevCb->rsCb->hccpMode == NETWORK_ONLINE ||
531 7 : qpCb->isExp == RS_NOT_EXP) {
532 3 : mrCb->ibMr = RsDrvMrReg(qpCb->ibPd, addr, len, access);
533 3 : CHK_PRT_RETURN(mrCb->ibMr == NULL, hccp_err("rs_drv_mr_reg addr is NULL len[%lld] failed ", len), -EACCES);
534 : } else {
535 5 : ret = RsMrPrepareRoceSign(phyId, qpCb->rdevCb, &roceSign);
536 5 : CHK_PRT_RETURN(ret != 0, hccp_err("RsMrPrepareRoceSign failed, ret(%d)", ret), ret);
537 5 : mrCb->ibMr = RsDrvExpMrReg(qpCb->ibPd, addr, len, access, roceSign);
538 5 : CHK_PRT_RETURN(mrCb->ibMr == NULL, hccp_err("rs_drv_exp_mr_reg addr is NULL len[%lld] failed ", len), -EACCES);
539 : }
540 :
541 8 : mrCb->mrInfo.cmd = (unsigned int)RS_CMD_MR_INFO;
542 8 : mrCb->mrInfo.addr = (uintptr_t)addr;
543 8 : mrCb->mrInfo.len = len;
544 8 : mrCb->mrInfo.rkey = mrCb->ibMr->rkey;
545 :
546 8 : RS_PTHREAD_MUTEX_LOCK(&qpCb->qpMutex);
547 8 : RsListAddTail(&mrCb->list, &qpCb->mrList);
548 8 : RS_PTHREAD_MUTEX_ULOCK(&qpCb->qpMutex);
549 :
550 8 : qpCb->mrNum++;
551 8 : return 0;
552 : }
553 :
554 41 : STATIC int RsCallocMr(int num, struct RsMrCb **mrCb)
555 : {
556 41 : CHK_PRT_RETURN(num <= 0, hccp_err("invalid num for mr calloc"), -EINVAL);
557 :
558 40 : *mrCb = calloc(num, sizeof(struct RsMrCb));
559 40 : CHK_PRT_RETURN((*mrCb) == NULL, hccp_err("calloc mr_cb failed"), -ENOMEM);
560 40 : return 0;
561 : }
562 :
563 54 : STATIC int RsCallocQpcb(int num, struct RsQpCb **qpCb)
564 : {
565 54 : if (num <= 0) {
566 1 : return -EINVAL;
567 : }
568 :
569 53 : *qpCb = calloc(num, sizeof(struct RsQpCb));
570 53 : if ((*qpCb) == NULL) {
571 1 : return -ENOMEM;
572 : }
573 :
574 52 : return 0;
575 : }
576 :
577 10 : RS_ATTRI_VISI_DEF int RsMrReg(unsigned int phyId, unsigned int rdevIndex, unsigned int qpn,
578 : struct RdmaMrRegInfo *mrRegInfo)
579 : {
580 : int ret;
581 10 : struct RsQpCb *qpCb = NULL;
582 10 : struct RsMrCb *mrCb = NULL;
583 :
584 10 : CHK_PRT_RETURN(mrRegInfo == NULL || mrRegInfo->addr == NULL || mrRegInfo->len == 0 || phyId >= RS_MAX_DEV_NUM,
585 : hccp_err("param err, NULL pointer or phyId:%u >= [%d]", phyId, RS_MAX_DEV_NUM), -EINVAL);
586 :
587 9 : hccp_info("qpn[%u], len[0x%llx], access[%d]", qpn, mrRegInfo->len, mrRegInfo->access);
588 :
589 9 : ret = RsQpn2qpcb(phyId, rdevIndex, qpn, &qpCb);
590 9 : CHK_PRT_RETURN(ret, hccp_err("rs_qpn2qpcb qpn[%d] ret[%d] failed ", qpn, ret), ret);
591 :
592 8 : CHK_PRT_RETURN(qpCb->mrNum >= RS_MR_NUM_MAX, hccp_err("Exceeded the maximum MR limit %d", qpCb->mrNum), -EINVAL);
593 :
594 8 : ret = RsGetMrcb(qpCb, (uintptr_t)mrRegInfo->addr, &mrCb, &qpCb->mrList);
595 8 : if (ret == 0) {
596 0 : hccp_warn("mr already registered");
597 0 : goto found;
598 : }
599 :
600 8 : ret = RsCallocMr(1, &mrCb);
601 8 : CHK_PRT_RETURN(ret, hccp_err("calloc mr failed"), ret);
602 :
603 8 : mrCb->qpCb = qpCb;
604 :
605 8 : ret = RsMrPreReg(phyId, qpCb, mrCb, mrRegInfo);
606 8 : if (ret) {
607 0 : hccp_err("pre reg mr failed, qpn %u, ret %d", qpn, ret);
608 0 : goto reg_err;
609 : }
610 :
611 8 : found:
612 8 : mrRegInfo->lkey = mrCb->ibMr->lkey;
613 8 : mrRegInfo->rkey = mrCb->ibMr->rkey;
614 :
615 8 : hccp_info("rs_mr_reg succ, state:%u", mrCb->state);
616 8 : return 0;
617 :
618 0 : reg_err:
619 0 : free(mrCb);
620 0 : mrCb = NULL;
621 :
622 0 : return ret;
623 : }
624 :
625 10 : RS_ATTRI_VISI_DEF int RsMrDereg(unsigned int phyId, unsigned int rdevIndex, unsigned int qpn, char *addr)
626 : {
627 : int ret;
628 10 : struct RsQpCb *qpCb = NULL;
629 10 : struct RsMrCb *mrCb = NULL;
630 :
631 10 : hccp_dbg("start rs_mr_dereg");
632 10 : RS_CHECK_POINTER_NULL_RETURN_INT(addr);
633 9 : CHK_PRT_RETURN(phyId >= RS_MAX_DEV_NUM, hccp_err("phyId:%u >= [%d], is invalid", phyId, RS_MAX_DEV_NUM), -EINVAL);
634 :
635 9 : ret = RsQpn2qpcb(phyId, rdevIndex, qpn, &qpCb);
636 9 : CHK_PRT_RETURN(ret, hccp_err("rs_qpn2qpcb failed ret[%d]", ret), ret);
637 :
638 7 : CHK_PRT_RETURN(RsGetMrcb(qpCb, (uintptr_t)addr, &mrCb, &qpCb->mrList),
639 : hccp_err("rs_get_mrcb failed, qpn[%u] addr[0x%llx]", qpn, (unsigned long long)(uintptr_t)addr), -EFAULT);
640 :
641 6 : ret = RsDrvMrDereg(mrCb->ibMr);
642 6 : CHK_PRT_RETURN(ret, hccp_err("rs_drv_mr_dereg failed ret[%d] ", ret), -EACCES);
643 :
644 6 : RS_PTHREAD_MUTEX_LOCK(&qpCb->qpMutex);
645 6 : RsListDel(&mrCb->list);
646 6 : free(mrCb);
647 6 : mrCb = NULL;
648 6 : RS_PTHREAD_MUTEX_ULOCK(&qpCb->qpMutex);
649 6 : qpCb->mrNum--;
650 :
651 6 : hccp_dbg("qpn[%u] succ", qpn);
652 :
653 6 : return 0;
654 : }
655 :
656 2 : STATIC int RsRegisterUbSegment(int directFlag, uint64_t addr, uint64_t len)
657 : {
658 2 : struct DVattribute attr = {0};
659 2 : int ret = 0;
660 :
661 2 : if (directFlag != DIRECT_FLAG_UB) {
662 2 : return 0;
663 : }
664 :
665 0 : ret = DlDrvMemGetAttribute(addr, &attr);
666 0 : CHK_PRT_RETURN(ret != 0, hccp_err("DlDrvMemGetAttribute failed, ret:%d", ret), ret);
667 :
668 0 : if (attr.memType != DV_MEM_LOCK_DEV) { // not support host memory
669 0 : return ret;
670 : }
671 :
672 0 : ret = DlHalMemRegUbSegment(attr.devId, addr, len);
673 0 : CHK_PRT_RETURN(ret != 0, hccp_err("DlHalMemRegUbSegment failed, ret:%d devId:%u len:%u", ret, attr.devId, len),
674 : ret);
675 :
676 0 : return ret;
677 : }
678 :
679 2 : STATIC int RsUnRegisterUbSegment(int directFlag, uint64_t addr)
680 : {
681 2 : struct DVattribute attr = {0};
682 2 : int ret = 0;
683 :
684 2 : if (directFlag != DIRECT_FLAG_UB) {
685 2 : return 0;
686 : }
687 :
688 0 : ret = DlDrvMemGetAttribute(addr, &attr);
689 0 : CHK_PRT_RETURN(ret != 0, hccp_err("DrvMemGetAttribute failed, ret:%d", ret), ret);
690 :
691 0 : if (attr.memType != DV_MEM_LOCK_DEV) { // not support host memory
692 0 : return ret;
693 : }
694 :
695 0 : ret = DlHalMemUnRegUbSegment(attr.devId, addr);
696 0 : if (ret != 0) {
697 0 : hccp_err("DlHalMemUnRegUbSegment failed, ret:%d devId:%u", ret, attr.devId);
698 : }
699 :
700 0 : return ret;
701 : }
702 :
703 3 : RS_ATTRI_VISI_DEF int RsRegisterMr(unsigned int phyId, unsigned int rdevIndex, struct RdmaMrRegInfo *mrRegInfo,
704 : void **mrHandle)
705 : {
706 3 : RS_CHECK_POINTER_NULL_RETURN_INT(mrHandle);
707 :
708 : int ret;
709 : unsigned int chipId;
710 3 : struct RsRdevCb *rdevCb = NULL;
711 3 : struct ibv_mr *rsMrHandle = NULL;
712 :
713 3 : CHK_PRT_RETURN(mrRegInfo == NULL || mrRegInfo->addr == NULL || mrRegInfo->len == 0 || phyId >= RS_MAX_DEV_NUM,
714 : hccp_err("param err, NULL pointer or phyId:%u >= [%d]", phyId, RS_MAX_DEV_NUM), -EINVAL);
715 :
716 2 : hccp_info("[rs_register_mr] len[0x%llx], access[%d]", mrRegInfo->len, mrRegInfo->access);
717 :
718 2 : ret = rsGetLocalDevIDByHostDevID(phyId, &chipId);
719 2 : CHK_PRT_RETURN(ret, hccp_err("rs_register_mr rsGetLocalDevIDByHostDevID phyId[%u] invalid, ret %d", phyId, ret),
720 : ret);
721 :
722 2 : ret = RsRdev2rdevCb(chipId, rdevIndex, &rdevCb);
723 2 : CHK_PRT_RETURN(ret != 0 || rdevCb == NULL, hccp_err("rs_rdev2rdev_cb for chip_id[%u] failed, ret %d", chipId, ret),
724 : ret);
725 :
726 2 : ret = RsRegisterUbSegment(rdevCb->directFlag, (uint64_t)(uintptr_t)mrRegInfo->addr, mrRegInfo->len);
727 2 : if (ret != 0) {
728 0 : hccp_err("RsRegisterUbSegment failed, ret[%d] vendor_id[0x%x] part_id[0x%x] directFlag[%u] "
729 : "addr[0x%llx] len[0x%llx]",
730 : ret, rdevCb->deviceAttr.vendor_id, rdevCb->deviceAttr.vendor_part_id, rdevCb->directFlag,
731 : (uint64_t)(uintptr_t)mrRegInfo->addr, mrRegInfo->len);
732 0 : goto mem_reg_err;
733 : }
734 :
735 2 : *mrHandle = (void *)RsDrvMrReg(rdevCb->ibPd, mrRegInfo->addr, mrRegInfo->len, mrRegInfo->access);
736 2 : if (*mrHandle == NULL) {
737 1 : hccp_warn("rs_drv_mr_reg addr is NULL len[0x%llx] access[%d] unsuccessful errno[%d]", mrRegInfo->len,
738 : mrRegInfo->access, errno);
739 1 : goto mr_reg_err;
740 : }
741 :
742 1 : rsMrHandle = (struct ibv_mr *)*mrHandle;
743 1 : mrRegInfo->lkey = rsMrHandle->lkey;
744 1 : mrRegInfo->rkey = rsMrHandle->rkey;
745 :
746 1 : hccp_info("rs_register_mr succ");
747 1 : return ret;
748 1 : mr_reg_err:
749 1 : (void)RsUnRegisterUbSegment(rdevCb->directFlag, (uint64_t)(uintptr_t)mrRegInfo->addr);
750 1 : mem_reg_err:
751 1 : mrRegInfo->lkey = 0;
752 :
753 1 : return ret;
754 : }
755 :
756 2 : STATIC int RsInitTypicalMrCb(unsigned int phyId, struct RdmaMrRegInfo *mrRegInfo, struct RsRdevCb *devCb,
757 : struct RsMrCb *mrCb)
758 : {
759 2 : unsigned long long len = mrRegInfo->len;
760 2 : struct roce_process_sign roceSign = {0};
761 2 : char *addr = (char *)mrRegInfo->addr;
762 2 : int access = mrRegInfo->access;
763 : int ret;
764 :
765 2 : if (devCb->rsCb->hccpMode == NETWORK_PEER_ONLINE || devCb->rsCb->hccpMode == NETWORK_ONLINE) {
766 2 : mrCb->ibMr = RsDrvMrReg(devCb->ibPd, addr, len, access);
767 2 : CHK_PRT_RETURN(mrCb->ibMr == NULL,
768 : hccp_err("rs_drv_mr_reg ibMr is NULL len[%lld] failed, errno:%d", len, errno), -EACCES);
769 : } else {
770 0 : ret = RsMrPrepareRoceSign(phyId, devCb, &roceSign);
771 0 : CHK_PRT_RETURN(ret != 0, hccp_err("RsMrPrepareRoceSign failed, ret(%d)", ret), ret);
772 0 : mrCb->ibMr = RsDrvExpMrReg(devCb->ibPd, addr, len, access, roceSign);
773 0 : CHK_PRT_RETURN(mrCb->ibMr == NULL,
774 : hccp_err("rs_drv_exp_mr_reg ibMr is NULL len[%lld] failed, errno:%d", len, errno), -EACCES);
775 : }
776 :
777 2 : mrCb->mrInfo.addr = (uintptr_t)addr;
778 2 : mrCb->mrInfo.len = len;
779 2 : mrCb->mrInfo.rkey = mrCb->ibMr->rkey;
780 :
781 2 : RS_PTHREAD_MUTEX_LOCK(&devCb->rdevMutex);
782 2 : RsListAddTail(&mrCb->list, &devCb->typicalMrList);
783 2 : RS_PTHREAD_MUTEX_ULOCK(&devCb->rdevMutex);
784 :
785 2 : return 0;
786 : }
787 :
788 1 : RS_ATTRI_VISI_DEF int RsTypicalRegisterMrV1(unsigned int phyId, unsigned int rdevIndex, struct RdmaMrRegInfo *mrRegInfo,
789 : void **mrHandle)
790 : {
791 1 : RS_CHECK_POINTER_NULL_RETURN_INT(mrHandle);
792 :
793 1 : struct RsMrCb *typicalMrCb = NULL;
794 1 : struct RsRdevCb *rdevCb = NULL;
795 : unsigned int chipId;
796 : int ret;
797 :
798 1 : CHK_PRT_RETURN(mrRegInfo == NULL || mrRegInfo->addr == NULL || mrRegInfo->len == 0 || phyId >= RS_MAX_DEV_NUM,
799 : hccp_err("param err, NULL pointer or phyId:%u >= [%d]", phyId, RS_MAX_DEV_NUM), -EINVAL);
800 :
801 1 : hccp_info("[rs_typical_register_mr] len[0x%llx], access[%d]", mrRegInfo->len, mrRegInfo->access);
802 :
803 1 : ret = rsGetLocalDevIDByHostDevID(phyId, &chipId);
804 1 : CHK_PRT_RETURN(ret != 0,
805 : hccp_err("rs_typical_register_mr rsGetLocalDevIDByHostDevID phyId[%u] invalid, ret %d", phyId, ret), ret);
806 :
807 1 : ret = RsRdev2rdevCb(chipId, rdevIndex, &rdevCb);
808 1 : CHK_PRT_RETURN(ret != 0 || rdevCb == NULL, hccp_err("rs_rdev2rdev_cb for chip_id[%u] failed, ret %d", chipId, ret),
809 : ret);
810 :
811 1 : ret = RsQueryMrCb(rdevCb, (uint64_t)(uintptr_t)mrRegInfo->addr, &typicalMrCb, &rdevCb->typicalMrList);
812 1 : if (ret == 0) {
813 0 : hccp_warn("typical mr already registered");
814 0 : goto found;
815 : }
816 :
817 1 : typicalMrCb = calloc(1, sizeof(struct RsMrCb));
818 1 : CHK_PRT_RETURN(typicalMrCb == NULL, hccp_err("calloc typical_mr_cb failed"), -ENOMEM);
819 1 : typicalMrCb->devCb = rdevCb;
820 :
821 1 : ret = RsInitTypicalMrCb(phyId, mrRegInfo, rdevCb, typicalMrCb);
822 1 : if (ret != 0) {
823 0 : hccp_err("rs_init_typical_mr_cb failed, devIndex[%u], ret[%d]", rdevIndex, ret);
824 0 : goto reg_err;
825 : }
826 :
827 1 : found:
828 1 : *mrHandle = typicalMrCb->ibMr;
829 1 : mrRegInfo->lkey = typicalMrCb->ibMr->lkey;
830 1 : mrRegInfo->rkey = typicalMrCb->ibMr->rkey;
831 1 : hccp_info("rs_typical_register_mr succ, state:%d", typicalMrCb->state);
832 1 : return 0;
833 :
834 0 : reg_err:
835 0 : free(typicalMrCb);
836 0 : typicalMrCb = NULL;
837 0 : return ret;
838 : }
839 :
840 1 : RS_ATTRI_VISI_DEF int RsTypicalRegisterMr(unsigned int phyId, unsigned int rdevIndex, struct RdmaMrRegInfo *mrRegInfo,
841 : void **mrHandle)
842 : {
843 1 : RS_CHECK_POINTER_NULL_RETURN_INT(mrHandle);
844 :
845 1 : struct RsMrCb *typicalMrCb = NULL;
846 1 : struct RsRdevCb *rdevCb = NULL;
847 : unsigned int chipId;
848 : int ret;
849 :
850 1 : CHK_PRT_RETURN(mrRegInfo == NULL || mrRegInfo->addr == NULL || mrRegInfo->len == 0 || phyId >= RS_MAX_DEV_NUM,
851 : hccp_err("param err, NULL pointer or phyId:%u >= [%d]", phyId, RS_MAX_DEV_NUM), -EINVAL);
852 :
853 1 : hccp_info("start register len[0x%llx], access[%d]", mrRegInfo->len, mrRegInfo->access);
854 :
855 1 : ret = rsGetLocalDevIDByHostDevID(phyId, &chipId);
856 1 : CHK_PRT_RETURN(ret != 0, hccp_err("rsGetLocalDevIDByHostDevID phyId[%u] invalid, ret %d", phyId, ret), ret);
857 :
858 1 : ret = RsRdev2rdevCb(chipId, rdevIndex, &rdevCb);
859 1 : CHK_PRT_RETURN(ret != 0 || rdevCb == NULL, hccp_err("rs_rdev2rdev_cb for chip_id[%u] failed, ret %d", chipId, ret),
860 : ret);
861 :
862 1 : typicalMrCb = calloc(1, sizeof(struct RsMrCb));
863 1 : CHK_PRT_RETURN(typicalMrCb == NULL, hccp_err("calloc typical_mr_cb failed"), -ENOMEM);
864 1 : typicalMrCb->devCb = rdevCb;
865 :
866 1 : ret = RsInitTypicalMrCb(phyId, mrRegInfo, rdevCb, typicalMrCb);
867 1 : if (ret != 0) {
868 0 : hccp_err("rs_init_typical_mr_cb failed, devIndex[%u], ret[%d]", rdevIndex, ret);
869 0 : goto reg_err;
870 : }
871 :
872 : // resv len as 1 to save addr for later unreg to query
873 1 : typicalMrCb->mrInfo.addr = (uint64_t)(uintptr_t)typicalMrCb->ibMr;
874 1 : typicalMrCb->mrInfo.len = 1U;
875 1 : *mrHandle = typicalMrCb->ibMr;
876 1 : mrRegInfo->lkey = typicalMrCb->ibMr->lkey;
877 1 : mrRegInfo->rkey = typicalMrCb->ibMr->rkey;
878 1 : hccp_info("register succ, state:%d", typicalMrCb->state);
879 1 : return 0;
880 :
881 0 : reg_err:
882 0 : free(typicalMrCb);
883 0 : typicalMrCb = NULL;
884 0 : return ret;
885 : }
886 :
887 4 : RS_ATTRI_VISI_DEF int RsRemapMr(unsigned int phyId, unsigned int rdevIndex, struct MemRemapInfo memList[],
888 : unsigned int memNum)
889 : {
890 4 : struct RsRdevCb *devCb = NULL;
891 4 : struct RsMrCb *mrCurr = NULL;
892 4 : struct RsMrCb *mrNext = NULL;
893 4 : unsigned long long addr = 0;
894 4 : bool isMemMatched = false;
895 : unsigned int chipId;
896 : unsigned int i;
897 : int ret;
898 :
899 4 : CHK_PRT_RETURN(phyId >= RS_MAX_DEV_NUM, hccp_err("phyId:%u >= %d, is invalid", phyId, RS_MAX_DEV_NUM), -EINVAL);
900 :
901 4 : ret = rsGetLocalDevIDByHostDevID(phyId, &chipId);
902 4 : CHK_PRT_RETURN(ret, hccp_err("rsGetLocalDevIDByHostDevID failed, phyId:%u invalid, ret:%d", phyId, ret), ret);
903 :
904 4 : ret = RsRdev2rdevCb(chipId, rdevIndex, &devCb);
905 4 : CHK_PRT_RETURN(devCb == NULL, hccp_err("rs_rdev2rdev_cb failed, chipId:%u, ret:%d", chipId, ret), -ENODEV);
906 :
907 6 : for (i = 0; i < memNum; i++) {
908 4 : isMemMatched = false;
909 4 : addr = (uint64_t)(uintptr_t)memList[i].addr;
910 4 : RS_PTHREAD_MUTEX_LOCK(&devCb->rdevMutex);
911 4 : RS_LIST_GET_HEAD_ENTRY(mrCurr, mrNext, &devCb->typicalMrList, list, struct RsMrCb);
912 7 : for (; (&mrCurr->list) != &devCb->typicalMrList;
913 3 : mrCurr = mrNext, mrNext = list_entry(mrNext->list.next, struct RsMrCb, list)) {
914 : // mem is out range of mr, continue to find next matching mr
915 3 : if ((addr < (uint64_t)(uintptr_t)mrCurr->ibMr->addr) || (memList[i].size > mrCurr->ibMr->length) ||
916 2 : (addr + memList[i].size < addr) ||
917 2 : (addr + memList[i].size > (uint64_t)(uintptr_t)mrCurr->ibMr->addr + mrCurr->ibMr->length)) {
918 1 : continue;
919 : }
920 :
921 : // each mr remap each corresponding mem
922 2 : ret = RsRoceRemapMr(mrCurr->ibMr, (struct hns_roce_mr_remap_info *)(void *)&memList[i], 1);
923 2 : if (ret != 0) {
924 0 : hccp_err("remap %u-th mem failed, ret:%d addr:0x%llx size:0x%llx", i, ret, addr, memList[i].size);
925 0 : RS_PTHREAD_MUTEX_ULOCK(&devCb->rdevMutex);
926 0 : return ret;
927 : }
928 2 : isMemMatched = true;
929 : }
930 4 : RS_PTHREAD_MUTEX_ULOCK(&devCb->rdevMutex);
931 :
932 4 : if (!isMemMatched) {
933 2 : hccp_err("find %u-th mem failed, addr:0x%llx size:0x%llx", i, addr, memList[i].size);
934 2 : return -ENODEV;
935 : }
936 2 : hccp_dbg("remap %u-th mem success, addr:0x%llx size:0x%llx", i, addr, memList[i].size);
937 : }
938 :
939 2 : return 0;
940 : }
941 :
942 2 : RS_ATTRI_VISI_DEF int RsTypicalDeregisterMr(unsigned int phyId, unsigned int devIndex, unsigned long long addr)
943 : {
944 2 : struct RsMrCb *typicalMrCb = NULL;
945 2 : struct RsRdevCb *devCb = NULL;
946 : unsigned int chipId;
947 : int ret;
948 :
949 2 : hccp_info("typical mr unreg start, addr[%llu]", addr);
950 2 : CHK_PRT_RETURN(phyId >= RS_MAX_DEV_NUM, hccp_err("phyId:%u >= %d, is invalid", phyId, RS_MAX_DEV_NUM), -EINVAL);
951 :
952 2 : ret = rsGetLocalDevIDByHostDevID(phyId, &chipId);
953 2 : CHK_PRT_RETURN(ret, hccp_err("phyId[%u] invalid, ret %d", phyId, ret), ret);
954 :
955 2 : ret = RsRdev2rdevCb(chipId, devIndex, &devCb);
956 2 : CHK_PRT_RETURN(ret != 0 || devCb == NULL,
957 : hccp_err("rs_rdev2rdev_cb get dev_cb failed for chip_id[%u], ret[%d]", chipId, ret), -ENODEV);
958 :
959 2 : ret = RsQueryMrCb(devCb, addr, &typicalMrCb, &devCb->typicalMrList);
960 2 : CHK_PRT_RETURN(ret, hccp_err("rs_query_mr_cb failed ret[%d]", ret), ret);
961 :
962 2 : ret = RsDrvMrDereg(typicalMrCb->ibMr);
963 2 : CHK_PRT_RETURN(ret, hccp_err("rs_drv_mr_dereg failed ret[%d]", ret), -EACCES);
964 :
965 2 : RS_PTHREAD_MUTEX_LOCK(&devCb->rdevMutex);
966 2 : RsListDel(&typicalMrCb->list);
967 2 : free(typicalMrCb);
968 2 : typicalMrCb = NULL;
969 2 : RS_PTHREAD_MUTEX_ULOCK(&devCb->rdevMutex);
970 :
971 2 : hccp_info("devIndex[%u] succ", devIndex);
972 :
973 2 : return 0;
974 : }
975 :
976 2 : RS_ATTRI_VISI_DEF int RsDeregisterMr(unsigned int phyId, unsigned int rdevIndex, void *mrHandle)
977 : {
978 2 : RS_CHECK_POINTER_NULL_RETURN_INT(mrHandle);
979 :
980 1 : struct ibv_mr *rsMrHandle = (struct ibv_mr *)mrHandle;
981 1 : uint64_t addr = (uint64_t)(uintptr_t)rsMrHandle->addr;
982 1 : struct RsRdevCb *devCb = NULL;
983 : unsigned int chipId;
984 1 : int ret = 0;
985 :
986 1 : ret = rsGetLocalDevIDByHostDevID(phyId, &chipId);
987 1 : CHK_PRT_RETURN(ret != 0, hccp_err("phyId[%u] invalid, ret %d", phyId, ret), ret);
988 :
989 1 : ret = RsRdev2rdevCb(chipId, rdevIndex, &devCb);
990 1 : CHK_PRT_RETURN(ret != 0 || devCb == NULL,
991 : hccp_err("rs_rdev2rdev_cb get dev_cb failed for chip_id[%u], ret[%d]", chipId, ret), -ENODEV);
992 :
993 1 : ret = RsDrvMrDereg(rsMrHandle);
994 1 : CHK_PRT_RETURN(ret != 0, hccp_err("rs_drv_mr_dereg failed ret[%d]", ret), -EACCES);
995 :
996 1 : ret = RsUnRegisterUbSegment(devCb->directFlag, addr);
997 1 : CHK_PRT_RETURN(ret != 0,
998 : hccp_err("RsUnRegisterUbSegment failed ret[%d], vendor_id[0x%x] part_id[0x%x]"
999 : " directFlag[%u]",
1000 : ret, devCb->deviceAttr.vendor_id, devCb->deviceAttr.vendor_part_id, devCb->directFlag),
1001 : ret);
1002 :
1003 1 : hccp_info("rs_deregister_mr succ");
1004 1 : return 0;
1005 : }
1006 :
1007 7 : RS_ATTRI_VISI_DEF int RsSendWr(unsigned int phyId, unsigned int rdevIndex, uint32_t qpn, struct SendWr *wr,
1008 : struct SendWrRsp *wrRsp)
1009 : {
1010 : int ret;
1011 7 : struct RsQpCb *qpCb = NULL;
1012 7 : struct RsMrCb *mrCb = NULL;
1013 7 : struct RsMrCb *remMrCb = NULL;
1014 :
1015 7 : RS_CHECK_POINTER_NULL_RETURN_INT(wr);
1016 7 : RS_CHECK_POINTER_NULL_RETURN_INT(wr->bufList);
1017 7 : RS_CHECK_POINTER_NULL_RETURN_INT(wrRsp);
1018 :
1019 7 : CHK_PRT_RETURN(phyId >= RS_MAX_DEV_NUM, hccp_err("phyId:%u >= [%d], is invalid", phyId, RS_MAX_DEV_NUM), -EINVAL);
1020 :
1021 7 : CHK_PRT_RETURN(wr->bufNum > MAX_SGE_NUM || wr->bufNum == 0, hccp_err("invalid buf_num[%u]!", wr->bufNum), -EINVAL);
1022 :
1023 6 : CHK_PRT_RETURN(wr->bufList->len > RS_SGLIST_LEN_MAX || wr->bufList->len == 0,
1024 : hccp_err("sg list "
1025 : "len is more than 2G, len[%u]",
1026 : wr->bufList->len),
1027 : -EINVAL);
1028 :
1029 5 : if (RsQpn2qpcb(phyId, rdevIndex, qpn, &qpCb)) {
1030 1 : return -EACCES;
1031 : }
1032 :
1033 4 : qpCb->sendWrNum++;
1034 :
1035 4 : hccp_info("qpn %d, bufList[0].addr is 0x%llx", qpn, wr->bufList[0].addr);
1036 4 : if (RsGetMrcb(qpCb, wr->bufList[0].addr, &mrCb, &qpCb->mrList)) {
1037 1 : hccp_err("qpn %d, bufList[0].addr[0x%llx] len[0x%x] is invalid.", qpn, wr->bufList[0].addr, wr->bufList[0].len);
1038 1 : return -EFAULT;
1039 : }
1040 :
1041 : // send op no need to check & get remote mr
1042 3 : if (wr->op != RA_WR_SEND && wr->op != RA_WR_SEND_WITH_IMM) {
1043 3 : hccp_info("remote wr dst addr is 0x%llx", wr->dstAddr);
1044 3 : if (RsGetMrcb(qpCb, wr->dstAddr, &remMrCb, &qpCb->remMrList)) {
1045 1 : hccp_err("qpn %d, remote wr dst addr[0x%llx] len[0x%x] is invalid.", qpn, wr->dstAddr, wr->bufList[0].len);
1046 1 : return -ENOENT;
1047 : }
1048 : }
1049 :
1050 2 : ret = RsDrvSendExp(qpCb, mrCb, remMrCb, wr, wrRsp);
1051 2 : if (ret) {
1052 0 : hccp_err("send exp failed qpn %u, ret %d", qpn, ret);
1053 : }
1054 2 : gRsSendWrNum++;
1055 2 : return ret;
1056 : }
1057 :
1058 5 : STATIC void BuildUpWrWithKey(struct WrInfo *wr, struct ibv_sge *list, struct ibv_send_wr *ibWr)
1059 : {
1060 5 : list->addr = (uintptr_t)wr->memList.addr;
1061 5 : list->length = wr->memList.len;
1062 5 : list->lkey = wr->memList.lkey;
1063 :
1064 5 : ibWr->sg_list = list;
1065 5 : ibWr->opcode = wr->op;
1066 5 : ibWr->send_flags = (unsigned int)wr->sendFlags;
1067 5 : ibWr->imm_data = htobe32(wr->immData);
1068 :
1069 5 : ibWr->num_sge = 1; /* only support one sge */
1070 5 : ibWr->wr_id = wr->wrId;
1071 5 : if (wr->op != IBV_WR_SEND && wr->op != IBV_WR_SEND_WITH_IMM) {
1072 5 : ibWr->wr.rdma.rkey = wr->rkey;
1073 5 : ibWr->wr.rdma.remote_addr = wr->dstAddr;
1074 : }
1075 5 : }
1076 :
1077 0 : STATIC void RsSendBuildUpWr(struct RsMrCb *mrCb, struct WrInfo *wr, struct ibv_sge *list, struct ibv_send_wr *ibWr)
1078 : {
1079 0 : list->addr = (uintptr_t)wr->memList.addr;
1080 0 : list->lkey = mrCb->ibMr->lkey;
1081 0 : list->length = wr->memList.len;
1082 :
1083 0 : ibWr->sg_list = list;
1084 0 : ibWr->opcode = wr->op;
1085 0 : ibWr->imm_data = htobe32(wr->immData);
1086 0 : ibWr->send_flags = (unsigned int)wr->sendFlags;
1087 :
1088 0 : ibWr->num_sge = 1; /* only support one sge */
1089 0 : ibWr->wr_id = wr->wrId;
1090 0 : }
1091 :
1092 2 : STATIC void RsWirteAndReadBuildUpWr(struct RsMrCb *mrCb, struct RsMrCb *remMrCb, struct WrInfo *wr,
1093 : struct ibv_sge *list, struct ibv_send_wr *ibWr)
1094 : {
1095 2 : list->addr = (uintptr_t)wr->memList.addr;
1096 2 : list->length = wr->memList.len;
1097 2 : list->lkey = mrCb->ibMr->lkey;
1098 :
1099 2 : ibWr->sg_list = list;
1100 2 : ibWr->opcode = wr->op;
1101 2 : ibWr->send_flags = (unsigned int)wr->sendFlags;
1102 2 : ibWr->imm_data = htobe32(wr->immData);
1103 :
1104 2 : ibWr->num_sge = 1; /* only support one sge */
1105 2 : ibWr->wr_id = wr->wrId;
1106 2 : ibWr->wr.rdma.rkey = remMrCb->mrInfo.rkey;
1107 2 : ibWr->wr.rdma.remote_addr = wr->dstAddr;
1108 2 : }
1109 :
1110 13 : STATIC int RsBuildUpWrList(struct WrInfo *wrList, struct RsQpCb *qpCb, struct ibv_sge *list, struct ibv_send_wr *ibWr,
1111 : unsigned int i)
1112 : {
1113 13 : struct RsMrCb *mrCb = NULL;
1114 13 : struct RsMrCb *remMrCb = NULL;
1115 13 : CHK_PRT_RETURN(wrList[i].memList.len > RS_SGLIST_LEN_MAX,
1116 : hccp_err("sg list len is more than 2G, len[%u]", wrList[i].memList.len), -EINVAL);
1117 :
1118 12 : hccp_dbg("qpn %d, bufList[0].addr is 0x%llx", qpCb->ibQp->qp_num, wrList[i].memList.addr);
1119 12 : if (RsGetMrcb(qpCb, wrList[i].memList.addr, &mrCb, &qpCb->mrList)) {
1120 2 : hccp_err("qpn %d, bufList[0].addr[0x%llx] len[0x%x] is invalid.", qpCb->ibQp->qp_num, wrList[i].memList.addr,
1121 : wrList[i].memList.len);
1122 2 : return -EFAULT;
1123 : }
1124 :
1125 : // send op no need to check & get remote mr
1126 10 : if (wrList[i].op != IBV_WR_SEND && wrList[i].op != IBV_WR_SEND_WITH_IMM) {
1127 10 : hccp_dbg("remote wr dst addr is 0x%llx", wrList[i].dstAddr);
1128 10 : if (RsGetMrcb(qpCb, wrList[i].dstAddr, &remMrCb, &qpCb->remMrList)) {
1129 8 : hccp_err("qpn %d, remote wr dst addr[0x%llx] len[0x%x] is invalid.", qpCb->ibQp->qp_num, wrList[i].dstAddr,
1130 : wrList[i].memList.len);
1131 8 : return -ENOENT;
1132 : }
1133 2 : RsWirteAndReadBuildUpWr(mrCb, remMrCb, &wrList[i], &list[i], &ibWr[i]);
1134 : } else {
1135 0 : RsSendBuildUpWr(mrCb, &wrList[i], &list[i], &ibWr[i]);
1136 : }
1137 :
1138 2 : return 0;
1139 : }
1140 :
1141 6 : STATIC int RsBuildUpWrListWithKey(struct WrInfo *wrList, struct ibv_sge *list, struct ibv_send_wr *ibWr, unsigned int i)
1142 : {
1143 6 : CHK_PRT_RETURN(wrList[i].memList.len > RS_SGLIST_LEN_MAX,
1144 : hccp_err("sg list len is more than 2G, len[%u]", wrList[i].memList.len), -EINVAL);
1145 :
1146 5 : BuildUpWrWithKey(&wrList[i], &list[i], &ibWr[i]);
1147 5 : return 0;
1148 : }
1149 :
1150 8 : STATIC int RsSendNormalWrlist(struct RsQpCb *qpCb, struct WrInfo *wrList, unsigned int sendNum,
1151 : unsigned int *completeNum, unsigned int keyFlag)
1152 : {
1153 : int ret;
1154 : unsigned int i, j;
1155 :
1156 8 : struct ibv_send_wr *badWr = NULL;
1157 8 : CHK_PRT_RETURN(sendNum > MAX_WR_NUM || sendNum == 0, hccp_err("send num[%u] is invalid!", sendNum), -EINVAL);
1158 8 : struct ibv_send_wr *ibWr = (struct ibv_send_wr *)calloc(sendNum, sizeof(struct ibv_send_wr));
1159 8 : CHK_PRT_RETURN(ibWr == NULL, hccp_err("calloc ib_wr failed!"), -ENOSPC);
1160 :
1161 8 : struct ibv_sge *list = (struct ibv_sge *)calloc(sendNum, sizeof(struct ibv_sge));
1162 8 : if (list == NULL) {
1163 0 : hccp_err("calloc list failed!");
1164 0 : ret = -ENOSPC;
1165 0 : goto alloc_fail;
1166 : }
1167 :
1168 10 : for (i = 0; i < sendNum; i++) {
1169 6 : ret = (keyFlag == 0) ? RsBuildUpWrList(wrList, qpCb, list, ibWr, i)
1170 8 : : RsBuildUpWrListWithKey(wrList, list, ibWr, i);
1171 8 : if (ret) {
1172 6 : goto input_err;
1173 : }
1174 2 : j = i + 1;
1175 2 : ibWr[i].next = (i < sendNum - 1) ? &ibWr[j] : NULL;
1176 : }
1177 :
1178 2 : ret = RsIbvPostSend(qpCb->ibQp, &ibWr[0], &badWr);
1179 2 : if (ret == 0) {
1180 2 : *completeNum = sendNum;
1181 0 : } else if (ret == -ENOMEM) {
1182 0 : *completeNum = (unsigned int)((void *)badWr - (void *)ibWr) / sizeof(struct ibv_send_wr);
1183 0 : hccp_dbg("post send wqe overflow, completeNum[%d]", *completeNum);
1184 : } else {
1185 0 : hccp_err("ibv_post_send failed, ret[%d]", ret);
1186 0 : *completeNum = 0;
1187 : }
1188 2 : qpCb->sendWrNum = qpCb->sendWrNum + (*completeNum);
1189 :
1190 8 : input_err:
1191 8 : free(list);
1192 8 : list = NULL;
1193 8 : alloc_fail:
1194 8 : free(ibWr);
1195 8 : ibWr = NULL;
1196 8 : return (ret == -ENOMEM) ? 0 : ret;
1197 : }
1198 :
1199 11 : STATIC int RsSendExpWrlist(struct RsQpCb *qpCb, struct WrInfo *wrList, unsigned int sendNum, struct SendWrRsp *wrRsp,
1200 : unsigned int *completeNum, unsigned int keyFlag)
1201 : {
1202 11 : struct ibv_post_send_ext_attr extAttr = {0};
1203 11 : struct ibv_post_send_ext_resp extRsp = {0};
1204 11 : struct ibv_send_wr *badWr = NULL;
1205 11 : struct wr_exp_rsp expRsp = {0};
1206 11 : struct ibv_send_wr ibWr = {0};
1207 11 : struct ibv_sge list = {0};
1208 : unsigned int i;
1209 11 : int ret = 0;
1210 :
1211 14 : for (i = 0; i < sendNum; i++) {
1212 : // reuse code: only need to build up one wr once a time
1213 7 : ret = (keyFlag == 0) ? RsBuildUpWrList(&wrList[i], qpCb, &list, &ibWr, 0)
1214 11 : : RsBuildUpWrListWithKey(&wrList[i], &list, &ibWr, 0);
1215 11 : if (ret != 0) {
1216 6 : hccp_err("qpn:%u key_flag:%u build_up_wr i:%u failed, ret:%d", qpCb->ibQp->qp_num, keyFlag, i, ret);
1217 6 : break;
1218 : }
1219 :
1220 5 : if (wrList[i].op == RA_WR_RDMA_WRITE_WITH_NOTIFY || wrList[i].op == RA_WR_RDMA_REDUCE_WRITE ||
1221 5 : wrList[i].op == RA_WR_RDMA_REDUCE_WRITE_WITH_NOTIFY) {
1222 1 : ibWr.imm_data = htobe32((wrList[i].aux.notifyOffset & WRITE_NOTIFY_OFFSET_MASK) |
1223 : WRITE_NOTIFY_VALUE_RECORD);
1224 1 : extAttr.reduce_op = wrList[i].aux.reduceType;
1225 1 : extAttr.reduce_type = wrList[i].aux.dataType;
1226 1 : ret = RsIbvExtPostSend(qpCb->ibQp, &ibWr, &badWr, &extAttr, &extRsp);
1227 1 : expRsp.wqe_index = extRsp.wqe_index;
1228 1 : expRsp.db_info = extRsp.db_info;
1229 1 : hccp_dbg("rs_ibv_ext_post_send, op = [%x], immData = [0x%lx], reduce_op = [%d],reduceType = [%d]",
1230 : ibWr.opcode, ibWr.imm_data, extAttr.reduce_op, extAttr.reduce_type);
1231 : } else {
1232 4 : ret = RsIbvExpPostSend(qpCb->ibQp, &ibWr, &badWr, &expRsp);
1233 4 : hccp_dbg("rs_ibv_exp_post_send, op = [%x], remoteAddr = [0x%llx], size = [%d]", ibWr.opcode,
1234 : ibWr.wr.rdma.remote_addr, ibWr.sg_list->length);
1235 : }
1236 :
1237 5 : if (ret != 0) {
1238 2 : if (ret == -ENOMEM) {
1239 1 : hccp_warn("qpn:%u rs_ibv_exp_post_send i:%u unsuccessful, ret %d", qpCb->ibQp->qp_num, i, ret);
1240 : } else {
1241 1 : hccp_err("qpn:%u rs_ibv_exp_post_send i:%u failed, ret %d", qpCb->ibQp->qp_num, i, ret);
1242 : }
1243 2 : break;
1244 : }
1245 :
1246 3 : qpCb->sendWrNum++;
1247 :
1248 3 : if (qpCb->qpMode == RA_RS_GDR_TMPL_QP_MODE) {
1249 1 : wrRsp[i].wqeTmp.sqIndex = (unsigned int)qpCb->sqIndex;
1250 1 : wrRsp[i].wqeTmp.wqeIndex = expRsp.wqe_index;
1251 2 : } else if (qpCb->qpMode == RA_RS_OP_QP_MODE || qpCb->qpMode == RA_RS_GDR_ASYN_QP_MODE) {
1252 2 : wrRsp[i].db.dbIndex = (unsigned int)qpCb->dbIndex;
1253 2 : wrRsp[i].db.dbInfo = expRsp.db_info;
1254 : }
1255 : }
1256 :
1257 11 : hccp_dbg("complete_num[%d], ret[%d]", i, ret);
1258 11 : *completeNum = i;
1259 11 : return (ret == -ENOMEM) ? 0 : ret;
1260 : }
1261 :
1262 15 : RS_ATTRI_VISI_DEF int RsSendWrlist(struct RsWrlistBaseInfo baseInfo, struct WrInfo *wrList, unsigned int sendNum,
1263 : struct SendWrRsp *wrRsp, unsigned int *completeNum)
1264 : {
1265 : int ret;
1266 : unsigned int phyId, rdevIndex, qpn;
1267 15 : struct RsQpCb *qpCb = NULL;
1268 :
1269 15 : RS_CHECK_POINTER_NULL_RETURN_INT(wrList);
1270 15 : RS_CHECK_POINTER_NULL_RETURN_INT(wrRsp);
1271 15 : CHK_PRT_RETURN(sendNum > MAX_WR_NUM || sendNum == 0 || baseInfo.phyId >= RS_MAX_DEV_NUM,
1272 : hccp_err("send_num[%u] or phyId:%u >= [%d], is invalid", sendNum, baseInfo.phyId, RS_MAX_DEV_NUM), -EINVAL);
1273 :
1274 14 : phyId = baseInfo.phyId;
1275 14 : rdevIndex = baseInfo.rdevIndex;
1276 14 : qpn = baseInfo.qpn;
1277 :
1278 14 : CHK_PRT_RETURN(RsQpn2qpcb(phyId, rdevIndex, qpn, &qpCb), hccp_err("rs_qpn2qpcb failed, physical id[%u]", phyId),
1279 : -EACCES);
1280 :
1281 : // only allow normal qp to call this func when ai_op_support not set
1282 13 : if (qpCb->qpMode == RA_RS_NOR_QP_MODE && qpCb->aiOpSupport == 0) {
1283 6 : ret = RsSendNormalWrlist(qpCb, wrList, sendNum, completeNum, baseInfo.keyFlag);
1284 : } else {
1285 7 : ret = RsSendExpWrlist(qpCb, wrList, sendNum, wrRsp, completeNum, baseInfo.keyFlag);
1286 : }
1287 13 : return ret;
1288 : }
1289 :
1290 0 : RS_ATTRI_VISI_DEF int RsRecvWrlist(struct RsWrlistBaseInfo baseInfo, struct RecvWrlistData *wr, unsigned int recvNum,
1291 : unsigned int *completeNum)
1292 : {
1293 0 : struct RsQpCb *qpCb = NULL;
1294 :
1295 0 : RS_CHECK_POINTER_NULL_RETURN_INT(wr);
1296 0 : CHK_PRT_RETURN(recvNum > MAX_WR_NUM || recvNum == 0 || baseInfo.phyId >= RS_MAX_DEV_NUM,
1297 : hccp_err("recv_num[%u] or phyId:%u >= [%d], is invalid", recvNum, baseInfo.phyId, RS_MAX_DEV_NUM), -EINVAL);
1298 :
1299 0 : CHK_PRT_RETURN(RsQpn2qpcb(baseInfo.phyId, baseInfo.rdevIndex, baseInfo.qpn, &qpCb),
1300 : hccp_err("rs_qpn2qpcb failed, physical id[%u]", baseInfo.phyId), -EACCES);
1301 :
1302 0 : return RsDrvPostRecv(qpCb, wr, recvNum, completeNum);
1303 : }
1304 :
1305 0 : RS_ATTRI_VISI_DEF int RsSetHostPid(uint32_t phyId, pid_t hostPid, const char *pidSign)
1306 : {
1307 : int ret;
1308 : unsigned int chipId;
1309 0 : struct rs_cb *rsCb = NULL;
1310 :
1311 0 : RS_CHECK_POINTER_NULL_RETURN_INT(pidSign);
1312 0 : CHK_PRT_RETURN(phyId >= RS_MAX_DEV_NUM, hccp_err("rs_set_host_pid rs set param error ! phyId:%u", phyId), -EINVAL);
1313 :
1314 0 : ret = rsGetLocalDevIDByHostDevID(phyId, &chipId);
1315 0 : CHK_PRT_RETURN(ret, hccp_err("rs_set_host_pid rsGetLocalDevIDByHostDevID phyId invalid, ret %d", ret), ret);
1316 :
1317 0 : hccp_info("phyId[%u] host_pid[%d]", chipId, hostPid);
1318 :
1319 0 : ret = RsDev2rscb(chipId, &rsCb, false);
1320 0 : CHK_PRT_RETURN(ret, hccp_err("get rs cb failed, chipId:%u", chipId), ret);
1321 :
1322 0 : rsCb->pRsSign.tgid = hostPid;
1323 0 : ret = strcpy_s(rsCb->pRsSign.sign, PROCESS_RS_SIGN_LENGTH, pidSign);
1324 0 : CHK_PRT_RETURN(ret, hccp_err("copy sign failed, ret %d", ret), -ESAFEFUNC);
1325 :
1326 0 : return 0;
1327 : }
1328 :
1329 6 : RS_ATTRI_VISI_DEF int RsRdevGetPortStatus(unsigned int phyId, unsigned int rdevIndex, enum PortStatus *status)
1330 : {
1331 6 : struct ibv_port_attr portAttr = {0};
1332 6 : struct RsRdevCb *rdevCb = NULL;
1333 : unsigned int chipId;
1334 : int ret;
1335 :
1336 6 : CHK_PRT_RETURN(phyId >= RS_MAX_DEV_NUM, hccp_err("phyId:%u >= [%d], is invalid", phyId, RS_MAX_DEV_NUM), -EINVAL);
1337 4 : CHK_PRT_RETURN(status == NULL, hccp_err("param err! status is NULL"), -EINVAL);
1338 :
1339 3 : ret = rsGetLocalDevIDByHostDevID(phyId, &chipId);
1340 3 : CHK_PRT_RETURN(ret, hccp_err("rsGetLocalDevIDByHostDevID failed, phyId[%u] invalid, ret %d", phyId, ret), ret);
1341 :
1342 3 : ret = RsRdev2rdevCb(chipId, rdevIndex, &rdevCb);
1343 3 : CHK_PRT_RETURN(ret != 0 || rdevCb == NULL, hccp_err("rs_rdev2rdev_cb for chip_id[%u] failed, ret %d", chipId, ret),
1344 : ret);
1345 :
1346 2 : ret = RsIbvQueryPort(rdevCb->ibCtx, rdevCb->ibPort, &portAttr);
1347 2 : CHK_PRT_RETURN(ret, hccp_err("ibv_query_port failed ret[%d]", ret), -EOPENSRC);
1348 :
1349 1 : *status = portAttr.state == IBV_PORT_ACTIVE ? PORT_STATUS_ACTIVE : PORT_STATUS_DOWN;
1350 :
1351 1 : hccp_dbg("phyId:%u port_attr.state:%u status:%u", phyId, portAttr.state, *status);
1352 1 : return 0;
1353 : }
1354 :
1355 3 : RS_ATTRI_VISI_DEF int RsGetNotifyMrInfo(unsigned int phyId, unsigned int rdevIndex, struct MrInfoT *info)
1356 : {
1357 3 : struct RsRdevCb *rdevCb = NULL;
1358 3 : struct rs_cb *rsCb = NULL;
1359 : unsigned int chipId;
1360 : int ret;
1361 :
1362 3 : CHK_PRT_RETURN(phyId >= RS_MAX_DEV_NUM, hccp_err("phyId:%u >= [%d], is invalid", phyId, RS_MAX_DEV_NUM), -EINVAL);
1363 :
1364 2 : CHK_PRT_RETURN(info == NULL, hccp_err("param err! info is NULL"), -EINVAL);
1365 :
1366 1 : ret = rsGetLocalDevIDByHostDevID(phyId, &chipId);
1367 1 : CHK_PRT_RETURN(ret, hccp_err("phyId[%u] invalid, ret:%d", phyId, ret), ret);
1368 :
1369 1 : ret = RsDev2rscb(chipId, &rsCb, false);
1370 1 : CHK_PRT_RETURN(ret, hccp_err("get rs_cb failed, ret:%d", ret), -ENODEV);
1371 :
1372 1 : ret = RsGetRdevCb(rsCb, rdevIndex, &rdevCb);
1373 1 : CHK_PRT_RETURN(ret, hccp_err("rs_get_rdev_cb failed!, ret:%d, rdevIndex:%u", ret, rdevIndex), ret);
1374 :
1375 1 : info->addr = (void *)(uintptr_t)rdevCb->notifyVaBase;
1376 1 : info->size = rdevCb->notifySize;
1377 1 : info->access = rdevCb->notifyAccess;
1378 1 : info->lkey = rdevCb->notifyMr->lkey;
1379 :
1380 1 : return 0;
1381 : }
1382 :
1383 4 : RS_ATTRI_VISI_DEF int RsNotifyCfgSet(unsigned int phyId, unsigned long long va, unsigned long long size)
1384 : {
1385 : int ret;
1386 : unsigned int chipId;
1387 4 : struct rs_cb *rsCb = NULL;
1388 :
1389 4 : RS_CHECK_POINTER_NULL_RETURN_INT((void *)(uintptr_t)va);
1390 :
1391 4 : CHK_PRT_RETURN(phyId >= RS_MAX_DEV_NUM ||
1392 : (size != MAX_NOTIFY_SIZE_CLOUD && size != NOTIFY_NUM_MAX_V2 && size != NOTIFY_NUM_MAX_V3),
1393 : hccp_err("rs_notify_cfg_set rs set param error ! phyId[%u] size[%llu]", phyId, size), -EINVAL);
1394 :
1395 3 : ret = rsGetLocalDevIDByHostDevID(phyId, &chipId);
1396 3 : CHK_PRT_RETURN(ret, hccp_err("rs_notify_cfg_set phyId invalid, ret %d, phyId:%u", ret, phyId), ret);
1397 :
1398 1 : ret = RsDev2rscb(chipId, &rsCb, false);
1399 1 : CHK_PRT_RETURN(ret, hccp_err("get rs cb failed, chipId:%u", chipId), ret);
1400 :
1401 1 : rsCb->notifyVaBase = va;
1402 1 : rsCb->notifySize = size;
1403 :
1404 1 : return 0;
1405 : }
1406 :
1407 0 : RS_ATTRI_VISI_DEF int RsNotifyCfgGet(unsigned int phyId, unsigned long long *va, unsigned long long *size)
1408 : {
1409 : int ret;
1410 : unsigned int chipId;
1411 0 : struct rs_cb *rsCb = NULL;
1412 :
1413 0 : RS_CHECK_POINTER_NULL_RETURN_INT(va);
1414 0 : RS_CHECK_POINTER_NULL_RETURN_INT(size);
1415 :
1416 0 : CHK_PRT_RETURN(phyId >= RS_MAX_DEV_NUM, hccp_err("rs_notify_cfg_get rs set param error ! phyId:%u", phyId),
1417 : -EINVAL);
1418 :
1419 0 : ret = rsGetLocalDevIDByHostDevID(phyId, &chipId);
1420 0 : CHK_PRT_RETURN(ret, hccp_err("rs_notify_cfg_get phyId invalid, ret %d, phyId:%u", ret, phyId), ret);
1421 :
1422 0 : ret = RsDev2rscb(chipId, &rsCb, false);
1423 0 : CHK_PRT_RETURN(ret, hccp_err("get rs cb failed, chipId:%u", chipId), ret);
1424 :
1425 0 : *va = rsCb->notifyVaBase;
1426 0 : *size = rsCb->notifySize;
1427 :
1428 0 : return 0;
1429 : }
1430 :
1431 6 : RS_ATTRI_VISI_DEF int RsSetTsqpDepth(unsigned int phyId, unsigned int rdevIndex, unsigned int tempDepth,
1432 : unsigned int *qpNum)
1433 : {
1434 : (void)phyId;
1435 : (void)rdevIndex;
1436 : (void)tempDepth;
1437 : (void)qpNum;
1438 : #ifdef CUSTOM_INTERFACE
1439 6 : struct RsRdevCb *rdevCb = NULL;
1440 6 : unsigned int sqDepth = 0;
1441 6 : unsigned int chipId = 0;
1442 : int ret;
1443 :
1444 6 : if (!RsIsCustomInterfaceSupported()) {
1445 0 : return 0;
1446 : }
1447 6 : CHK_PRT_RETURN(phyId >= RS_MAX_DEV_NUM, hccp_err("rs_set_tsqp_depth param error ! phyId:%d", phyId), -EINVAL);
1448 :
1449 5 : CHK_PRT_RETURN(qpNum == NULL, hccp_err("rs_set_tsqp_depth qp_num is NULL, param error!"), -EINVAL);
1450 :
1451 4 : CHK_PRT_RETURN(tempDepth < RS_MIN_TEMPTH_DEPTH || tempDepth > RS_MAX_TEMPTH_DEPTH,
1452 : hccp_err("param error!"
1453 : "temp_depth[%u] can not be smaller than [%d] or bigger than [%d]",
1454 : tempDepth, RS_MIN_TEMPTH_DEPTH, RS_MAX_TEMPTH_DEPTH),
1455 : -EINVAL);
1456 :
1457 3 : ret = rsGetLocalDevIDByHostDevID(phyId, &chipId);
1458 3 : CHK_PRT_RETURN(ret, hccp_err("phyId[%u] invalid, ret %d", phyId, ret), ret);
1459 :
1460 2 : ret = RsRdev2rdevCb(chipId, rdevIndex, &rdevCb);
1461 2 : CHK_PRT_RETURN(ret || rdevCb == NULL,
1462 : hccp_err("rs_set_tsqp_depth rs_rdev2rdev_cb for chip_id[%u] "
1463 : "failed, ret %d",
1464 : chipId, ret),
1465 : ret);
1466 :
1467 1 : ret = RsRoceSetTsqpDepth(rdevCb->devName, rdevIndex, tempDepth, qpNum, &sqDepth);
1468 1 : CHK_PRT_RETURN(ret, hccp_err("rs_roce_set_tsqp_depth failed, ret %d, devName[%s]", ret, rdevCb->devName), ret);
1469 :
1470 0 : rdevCb->txDepth = sqDepth;
1471 0 : rdevCb->rxDepth = sqDepth;
1472 0 : rdevCb->qpMaxNum = *qpNum;
1473 : #endif
1474 0 : return 0;
1475 : }
1476 :
1477 5 : RS_ATTRI_VISI_DEF int RsGetTsqpDepth(unsigned int phyId, unsigned int rdevIndex, unsigned int *tempDepth,
1478 : unsigned int *qpNum)
1479 : {
1480 : (void)phyId;
1481 : (void)rdevIndex;
1482 : (void)tempDepth;
1483 : (void)qpNum;
1484 : #ifdef CUSTOM_INTERFACE
1485 5 : struct RsRdevCb *rdevCb = NULL;
1486 5 : unsigned int sqDepth = 0;
1487 5 : unsigned int chipId = 0;
1488 : int ret;
1489 :
1490 5 : if (!RsIsCustomInterfaceSupported()) {
1491 0 : return 0;
1492 : }
1493 5 : CHK_PRT_RETURN(phyId >= RS_MAX_DEV_NUM, hccp_err("param error ! phyId:%d", phyId), -EINVAL);
1494 :
1495 4 : CHK_PRT_RETURN(tempDepth == NULL || qpNum == NULL,
1496 : hccp_err("temp_depth or qp_num is NULL,"
1497 : "param error!"),
1498 : -EINVAL);
1499 :
1500 3 : ret = rsGetLocalDevIDByHostDevID(phyId, &chipId);
1501 3 : CHK_PRT_RETURN(ret, hccp_err("phyId[%u] invalid, ret %d", phyId, ret), ret);
1502 :
1503 2 : ret = RsRdev2rdevCb(chipId, rdevIndex, &rdevCb);
1504 2 : CHK_PRT_RETURN(ret != 0 || rdevCb == NULL,
1505 : hccp_err("rs_get_tsqp_depth rs_rdev2rdev_cb for chip_id[%u] "
1506 : "failed, ret %d",
1507 : chipId, ret),
1508 : ret);
1509 :
1510 1 : ret = RsRoceGetTsqpDepth(rdevCb->devName, rdevIndex, tempDepth, qpNum, &sqDepth);
1511 1 : CHK_PRT_RETURN(ret, hccp_err("rs_roce_get_tsqp_depth failed, ret %d, devName[%s]", ret, rdevCb->devName), ret);
1512 : #endif
1513 0 : return 0;
1514 : }
1515 :
1516 45 : STATIC void RsSetQpDepthAttr(struct RsRdevCb *rdevCb, struct RsQpCb *qpCb, struct RsQpNorm *qpNorm)
1517 : {
1518 45 : if (qpCb->qpMode == RA_RS_GDR_TMPL_QP_MODE) {
1519 41 : qpCb->txDepth = rdevCb->txDepth;
1520 41 : qpCb->rxDepth = rdevCb->rxDepth;
1521 : } else {
1522 4 : if (rdevCb->rsCb->hccpMode == NETWORK_OFFLINE) {
1523 2 : qpCb->txDepth = RS_QP_TX_DEPTH_OFFLINE;
1524 2 : qpCb->rxDepth = RS_QP_RX_DEPTH_OFFLINE;
1525 : } else {
1526 2 : qpCb->txDepth = RS_QP_TX_DEPTH_ONLINE;
1527 2 : qpCb->rxDepth = RS_QP_RX_DEPTH_ONLINE;
1528 : }
1529 : }
1530 :
1531 45 : if (qpNorm->isExp != 0 && qpNorm->qpMode != RA_RS_NOR_QP_MODE) {
1532 43 : if (rdevCb->rsCb->hccpMode == NETWORK_PEER_ONLINE) {
1533 2 : qpCb->txDepth = (qpCb->qpMode != RA_RS_GDR_TMPL_QP_MODE) ? RS_QP_TX_DEPTH_PEER_ONLINE : qpCb->txDepth;
1534 2 : qpCb->rxDepth = (qpCb->qpMode != RA_RS_GDR_TMPL_QP_MODE) ? RS_QP_TX_DEPTH_PEER_ONLINE : qpCb->rxDepth;
1535 : } else {
1536 41 : qpCb->txDepth = (qpCb->qpMode != RA_RS_GDR_TMPL_QP_MODE && qpCb->qpMode != RA_RS_GDR_ASYN_QP_MODE)
1537 : ? RS_QP_32K_DEPTH
1538 41 : : qpCb->txDepth;
1539 : }
1540 43 : qpCb->sendSgeNum = 1;
1541 43 : qpCb->recvSgeNum = 1;
1542 : } else {
1543 2 : if (rdevCb->rsCb->hccpMode == NETWORK_PEER_ONLINE) {
1544 0 : qpCb->txDepth = (qpCb->qpMode != RA_RS_GDR_TMPL_QP_MODE) ? RS_QP_TX_DEPTH_PEER_ONLINE : qpCb->txDepth;
1545 0 : qpCb->rxDepth = (qpCb->qpMode != RA_RS_GDR_TMPL_QP_MODE) ? RS_QP_TX_DEPTH_PEER_ONLINE : qpCb->rxDepth;
1546 : } else {
1547 2 : qpCb->txDepth = (qpCb->qpMode != RA_RS_GDR_TMPL_QP_MODE) ? RS_QP_TX_DEPTH : qpCb->txDepth;
1548 2 : qpCb->rxDepth = (qpCb->qpMode != RA_RS_GDR_TMPL_QP_MODE) ? RS_QP_TX_DEPTH : qpCb->rxDepth;
1549 : }
1550 2 : qpCb->sendSgeNum = RS_QP_ATTR_MAX_SEND_SGE;
1551 2 : qpCb->recvSgeNum = 1;
1552 : }
1553 45 : }
1554 :
1555 45 : STATIC int RsQpcbInit(struct RsRdevCb *rdevCb, struct RsQpCb *qpCb, struct RsQpNorm *qpNorm)
1556 : {
1557 : #define RS_DRV_CQ_DEPTH 16384
1558 : #define RS_DRV_CQ_128_DEPTH 128
1559 : #define RS_DRV_CQ_8K_DEPTH 8192
1560 : #define RS_DRV_CQ_32K_DEPTH 32768
1561 45 : int qpMode = qpNorm->qpMode;
1562 : int ret;
1563 :
1564 45 : qpCb->rdevCb = rdevCb;
1565 45 : RS_INIT_LIST_HEAD(&qpCb->mrList);
1566 45 : RS_INIT_LIST_HEAD(&qpCb->remMrList);
1567 :
1568 45 : qpCb->qpMode = qpMode;
1569 45 : qpCb->eqNum = 0;
1570 45 : qpCb->numRecvCqEvents = 0;
1571 45 : qpCb->numSendCqEvents = 0;
1572 45 : qpCb->state = RS_QP_STATUS_DISCONNECT;
1573 45 : qpCb->ibPd = rdevCb->ibPd;
1574 :
1575 : // cq attr
1576 45 : if (qpNorm->isExt == 1) {
1577 : // update TEMP & ASYN mode cq depth from 32K to 8K due to memory issue
1578 2 : qpCb->sendCqDepth = (qpMode != RA_RS_GDR_TMPL_QP_MODE && qpMode != RA_RS_GDR_ASYN_QP_MODE) ? RS_DRV_CQ_32K_DEPTH
1579 4 : : RS_DRV_CQ_8K_DEPTH;
1580 2 : qpCb->recvCqDepth = RS_DRV_CQ_128_DEPTH;
1581 : } else {
1582 43 : qpCb->sendCqDepth = RS_DRV_CQ_DEPTH;
1583 43 : qpCb->recvCqDepth = RS_DRV_CQ_DEPTH;
1584 : }
1585 :
1586 : // qp attr
1587 45 : RsSetQpDepthAttr(rdevCb, qpCb, qpNorm);
1588 :
1589 45 : qpCb->memAlign = qpNorm->memAlign;
1590 :
1591 45 : qpCb->channel = RsIbvCreateCompChannel(rdevCb->ibCtx);
1592 45 : CHK_PRT_RETURN(qpCb->channel == NULL, hccp_err("ibv_create_comp_channel failed! errno(%d)", errno), -EINVAL);
1593 44 : qpCb->qosAttr.tc = (RS_ROCE_DSCP_33 & RS_DSCP_MASK) << RS_DSCP_OFF;
1594 44 : qpCb->qosAttr.sl = RS_ROCE_4_SL;
1595 44 : qpCb->timeout = RS_QP_ATTR_TIMEOUT;
1596 44 : qpCb->retryCnt = RS_QP_ATTR_RETRY_CNT;
1597 :
1598 44 : ret = RsEpollCtl(rdevCb->rsCb->connCb.epollfd, EPOLL_CTL_ADD, qpCb->channel->fd, EPOLLIN | EPOLLRDHUP);
1599 : #ifndef CA_CONFIG_LLT
1600 : if (ret) {
1601 : RsIbvDestroyCompChannel(qpCb->channel);
1602 : hccp_err("add channel fd failed ret %d", ret);
1603 : return ret;
1604 : }
1605 : #endif
1606 44 : return 0;
1607 : }
1608 :
1609 48 : STATIC int RsQpcbDeinit(struct RsRdevCb *rdevCb, struct RsQpCb *qpCb)
1610 : {
1611 : int ret;
1612 :
1613 48 : if (qpCb == NULL || qpCb->channel == NULL) {
1614 0 : hccp_err("qp_cb or qp_cb->channel is NULL!");
1615 0 : return -EINVAL;
1616 : }
1617 :
1618 48 : ret = RsEpollCtl(rdevCb->rsCb->connCb.epollfd, EPOLL_CTL_DEL, qpCb->channel->fd, EPOLLIN | EPOLLRDHUP);
1619 : #ifndef CA_CONFIG_LLT
1620 : if (ret) {
1621 : hccp_err("del channel fd failed ret %d", ret);
1622 : }
1623 : #endif
1624 :
1625 48 : if (qpCb->channel != NULL) {
1626 48 : RsIbvDestroyCompChannel(qpCb->channel);
1627 48 : qpCb->channel = NULL;
1628 : }
1629 : #ifndef CA_CONFIG_LLT
1630 : return ret;
1631 : #else
1632 48 : return 0;
1633 : #endif
1634 : }
1635 :
1636 32 : STATIC int RsQpNotifyMr(struct RsRdevCb *rdevCb, struct RsQpCb *qpCb, uint32_t *qpn)
1637 : {
1638 : int ret;
1639 32 : struct RsMrCb *notifyMrNode = NULL;
1640 :
1641 32 : ret = RsCallocMr(1, ¬ifyMrNode);
1642 32 : CHK_PRT_RETURN(ret, hccp_err("notify_mr_cb malloc failed"), ret);
1643 :
1644 32 : RS_PTHREAD_MUTEX_LOCK(&rdevCb->rdevMutex);
1645 32 : RsListAddTail(&qpCb->list, &rdevCb->qpList);
1646 32 : RS_PTHREAD_MUTEX_ULOCK(&rdevCb->rdevMutex);
1647 :
1648 32 : if (rdevCb->notifyType != NO_USE) {
1649 32 : notifyMrNode->qpCb = qpCb;
1650 32 : notifyMrNode->ibMr = rdevCb->notifyMr;
1651 32 : notifyMrNode->mrInfo.addr = rdevCb->notifyVaBase;
1652 32 : notifyMrNode->mrInfo.len = rdevCb->notifySize;
1653 32 : notifyMrNode->mrInfo.rkey = notifyMrNode->ibMr->rkey;
1654 : } else {
1655 0 : notifyMrNode->qpCb = qpCb;
1656 0 : notifyMrNode->ibMr = NULL;
1657 : }
1658 :
1659 32 : RS_PTHREAD_MUTEX_LOCK(&qpCb->qpMutex);
1660 32 : RsListAddTail(¬ifyMrNode->list, &qpCb->mrList);
1661 32 : RS_PTHREAD_MUTEX_ULOCK(&qpCb->qpMutex);
1662 32 : rdevCb->qpCnt++;
1663 32 : *qpn = qpCb->ibQp->qp_num;
1664 :
1665 32 : hccp_info("rs qp %d create OK!", *qpn);
1666 :
1667 32 : return 0;
1668 : }
1669 :
1670 53 : STATIC int RsQpQueryInfo(unsigned int phyId, unsigned int rdevIndex, struct RsRdevCb **rdevCb, int qpMode)
1671 : {
1672 : int ret;
1673 : unsigned int chipId;
1674 53 : struct rs_cb *rsCb = NULL;
1675 :
1676 53 : CHK_PRT_RETURN(phyId >= RS_MAX_DEV_NUM, hccp_err("rs_qp_query_info rs set param error! phyId:%u", phyId), -EINVAL);
1677 :
1678 53 : ret = rsGetLocalDevIDByHostDevID(phyId, &chipId);
1679 53 : CHK_PRT_RETURN(ret, hccp_err("rs_qp_query_info phyId[%u] invalid, ret:%d", phyId, ret), ret);
1680 :
1681 52 : ret = RsDev2rscb(chipId, &rsCb, false);
1682 52 : CHK_PRT_RETURN(ret, hccp_err("rs_qp_query_info get rs_cb failed, ret:%d", ret), -ENODEV);
1683 :
1684 52 : ret = RsGetRdevCb(rsCb, rdevIndex, rdevCb);
1685 52 : CHK_PRT_RETURN(ret, hccp_err("rs_get_rdev_cb failed! ret:%d, rdevIndex:%u", ret, rdevIndex), ret);
1686 :
1687 52 : if (qpMode == RA_RS_GDR_TMPL_QP_MODE) {
1688 46 : CHK_PRT_RETURN((*rdevCb)->qpCnt >= (*rdevCb)->qpMaxNum,
1689 : hccp_err("Exceeded the maximum QP limit(%u)", (*rdevCb)->qpMaxNum), -EINVAL);
1690 : } else {
1691 6 : CHK_PRT_RETURN((*rdevCb)->qpCnt >= RS_QP_NUM_MAX,
1692 : hccp_err("Exceeded the maximum QP limit(%u)", (*rdevCb)->qpCnt), -EINVAL);
1693 : }
1694 :
1695 50 : return 0;
1696 : }
1697 :
1698 50 : STATIC int RsInitMemPool(struct RsQpCb *qpCb)
1699 : {
1700 50 : struct roce_mem_cq_qp_attr memAttr = {0};
1701 : int ret;
1702 :
1703 50 : if ((qpCb->qpMode != RA_RS_OP_QP_MODE && qpCb->qpMode != RA_RS_OP_QP_MODE_EXT) ||
1704 4 : qpCb->memAlign != LITE_ALIGN_2MB) {
1705 49 : return 0;
1706 : }
1707 :
1708 : // init mem_pool and store mem_data in mem_resp
1709 1 : memAttr.mem_align = qpCb->memAlign;
1710 1 : memAttr.send_qp_depth = qpCb->txDepth;
1711 1 : memAttr.send_cq_depth = (unsigned int)qpCb->sendCqDepth;
1712 1 : memAttr.send_sge_num = qpCb->sendSgeNum;
1713 1 : memAttr.recv_qp_depth = qpCb->rxDepth;
1714 1 : memAttr.recv_cq_depth = (unsigned int)qpCb->recvCqDepth;
1715 1 : memAttr.recv_sge_num = qpCb->recvSgeNum;
1716 1 : memAttr.use_resv_mem = qpCb->useResvMem;
1717 1 : memAttr.resv_mem_pool_id = qpCb->resvMemPoolId;
1718 1 : memAttr.ctx = qpCb->rdevCb->ibCtx;
1719 :
1720 1 : ret = RsRoceInitMemPool(&memAttr, &qpCb->memResp.memData, qpCb->rdevCb->rsCb->chipId);
1721 1 : if (ret != 0) {
1722 1 : hccp_err("rs_roce_init_mem_pool failed, ret=%d, chipId=%u", ret, qpCb->rdevCb->rsCb->chipId);
1723 : }
1724 1 : return ret;
1725 : }
1726 :
1727 50 : STATIC void RsDeinitMemPool(struct RsQpCb *qpCb)
1728 : {
1729 50 : if ((qpCb->qpMode != RA_RS_OP_QP_MODE && qpCb->qpMode != RA_RS_OP_QP_MODE_EXT) ||
1730 4 : qpCb->memAlign != LITE_ALIGN_2MB) {
1731 49 : return;
1732 : }
1733 :
1734 1 : (void)RsRoceDeinitMemPool(qpCb->memResp.memData.mem_idx);
1735 : }
1736 :
1737 46 : STATIC int RsAllocQpcb(struct RsRdevCb *rdevCb, struct RsQpCb **qpCb, struct RsQpNorm *qpNorm)
1738 : {
1739 : int ret;
1740 :
1741 46 : ret = RsCallocQpcb(1, qpCb);
1742 46 : CHK_PRT_RETURN(ret, hccp_err("alloc mem for qp_cb failed, ret:%d errno:%d", ret, errno), -ENOMEM);
1743 :
1744 45 : ret = pthread_mutex_init(&(*qpCb)->qpMutex, NULL);
1745 45 : if (ret) {
1746 0 : hccp_err("pthread_mutex_init failed, ret %d", ret);
1747 0 : goto qp_mutex_init_err;
1748 : }
1749 :
1750 45 : ret = pthread_mutex_init(&(*qpCb)->cqeErrInfo.mutex, NULL);
1751 45 : if (ret) {
1752 0 : hccp_err("pthread_mutex_init failed, ret %d", ret);
1753 0 : goto cqe_mutex_init_err;
1754 : }
1755 :
1756 45 : ret = RsQpcbInit(rdevCb, *qpCb, qpNorm);
1757 45 : if (ret) {
1758 1 : hccp_err("create qp tx rx failed ret %d", ret);
1759 1 : goto rs_qpcb_init_err;
1760 : }
1761 :
1762 44 : ret = RsInitMemPool(*qpCb);
1763 44 : if (ret) {
1764 0 : hccp_err("init mem pool failed ret %d", ret);
1765 0 : goto rs_init_mem_err;
1766 : }
1767 :
1768 44 : ret = RsDrvCreateCq(*qpCb, qpNorm->isExt);
1769 44 : if (ret) {
1770 1 : hccp_err("create cq failed ret %d", ret);
1771 1 : goto create_cq_err;
1772 : }
1773 :
1774 43 : return 0;
1775 :
1776 1 : create_cq_err:
1777 1 : RsDeinitMemPool(*qpCb);
1778 :
1779 1 : rs_init_mem_err:
1780 1 : RsQpcbDeinit(rdevCb, *qpCb);
1781 :
1782 2 : rs_qpcb_init_err:
1783 2 : pthread_mutex_destroy(&(*qpCb)->cqeErrInfo.mutex);
1784 :
1785 2 : cqe_mutex_init_err:
1786 2 : pthread_mutex_destroy(&(*qpCb)->qpMutex);
1787 :
1788 2 : qp_mutex_init_err:
1789 2 : free(*qpCb);
1790 2 : *qpCb = NULL;
1791 :
1792 2 : return ret;
1793 : }
1794 :
1795 15 : STATIC void RsFreeQpcb(struct RsRdevCb *rdevCb, struct RsQpCb *qpCb)
1796 : {
1797 15 : RsDrvDestroyCq(qpCb);
1798 15 : RsDeinitMemPool(qpCb);
1799 15 : (void)RsQpcbDeinit(rdevCb, qpCb);
1800 15 : pthread_mutex_destroy(&qpCb->cqeErrInfo.mutex);
1801 15 : pthread_mutex_destroy(&qpCb->qpMutex);
1802 15 : free(qpCb);
1803 15 : qpCb = NULL;
1804 15 : }
1805 :
1806 48 : RS_ATTRI_VISI_DEF int RsQpCreate(unsigned int phyId, unsigned int rdevIndex, struct RsQpNorm qpNorm,
1807 : struct RsQpResp *qpResp)
1808 : {
1809 48 : struct RsRdevCb *rdevCb = NULL;
1810 48 : struct RsQpCb *qpCb = NULL;
1811 : int ret;
1812 :
1813 48 : RS_QP_PARA_CHECK(phyId);
1814 47 : CHK_PRT_RETURN(qpResp == NULL, hccp_err("qp_resp is NULL!"), -EINVAL);
1815 :
1816 47 : ret = RsQpQueryInfo(phyId, rdevIndex, &rdevCb, qpNorm.qpMode);
1817 47 : CHK_PRT_RETURN(ret, hccp_err("query qp info failed:%d", ret), ret);
1818 :
1819 46 : ret = RsAllocQpcb(rdevCb, &qpCb, &qpNorm);
1820 46 : CHK_PRT_RETURN(ret, hccp_err("alloc mem for qp_cb failed, ret:%d", ret), ret);
1821 :
1822 43 : ret = RsDrvQpCreate(qpCb, &qpNorm);
1823 43 : if (ret) {
1824 14 : hccp_err("create drv qp create failed:%d", ret);
1825 14 : goto create_qp_err;
1826 : }
1827 :
1828 29 : ret = ibv_req_notify_cq(qpCb->ibSendCq, 0);
1829 29 : if (ret) {
1830 1 : hccp_err("Couldn't request send CQ notification, ret:%d", ret);
1831 1 : ret = -EOPENSRC;
1832 1 : goto ret_noritfy_cq;
1833 : }
1834 :
1835 28 : ret = ibv_req_notify_cq(qpCb->ibRecvCq, 0);
1836 28 : if (ret) {
1837 0 : hccp_err("Couldn't request recv CQ notification, ret:%d", ret);
1838 0 : ret = -EOPENSRC;
1839 0 : goto ret_noritfy_cq;
1840 : }
1841 :
1842 28 : ret = RsQpNotifyMr(rdevCb, qpCb, &qpResp->qpn); // alloc mr
1843 28 : if (ret) {
1844 0 : hccp_err("store qp notify mr failed:%d", ret);
1845 0 : goto ret_noritfy_cq;
1846 : }
1847 :
1848 28 : if (qpNorm.isExp) {
1849 26 : qpCb->isExp = RS_IS_EXP;
1850 : } else {
1851 2 : qpCb->isExp = RS_NOT_EXP;
1852 : }
1853 :
1854 28 : qpResp->qpn = (unsigned int)qpCb->qpInfoLo.qpn;
1855 28 : qpResp->gidIdx = (unsigned int)qpCb->qpInfoLo.gidIdx;
1856 28 : qpResp->psn = (unsigned int)qpCb->qpInfoLo.psn;
1857 28 : qpResp->gid = qpCb->qpInfoLo.gid;
1858 :
1859 28 : return 0;
1860 :
1861 1 : ret_noritfy_cq:
1862 1 : RsDrvQpDestroy(qpCb);
1863 :
1864 15 : create_qp_err:
1865 15 : RsFreeQpcb(rdevCb, qpCb);
1866 15 : return ret;
1867 : }
1868 :
1869 4 : STATIC int RsQpcbInitWithAttrs(struct RsRdevCb *rdevCb, struct RsQpCb *qpCb, struct RsQpNormWithAttrs *qpNorm)
1870 : {
1871 : int ret;
1872 :
1873 4 : qpCb->rdevCb = rdevCb;
1874 4 : RS_INIT_LIST_HEAD(&qpCb->mrList);
1875 4 : RS_INIT_LIST_HEAD(&qpCb->remMrList);
1876 :
1877 4 : qpCb->qpMode = qpNorm->extAttrs.qpMode;
1878 4 : qpCb->numRecvCqEvents = 0;
1879 4 : qpCb->numSendCqEvents = 0;
1880 4 : qpCb->state = RS_QP_STATUS_DISCONNECT;
1881 4 : qpCb->ibPd = rdevCb->ibPd;
1882 :
1883 4 : qpCb->txDepth = qpNorm->extAttrs.qpAttr.cap.max_send_wr;
1884 4 : qpCb->rxDepth = qpNorm->extAttrs.qpAttr.cap.max_send_wr;
1885 4 : qpCb->sendSgeNum = qpNorm->extAttrs.qpAttr.cap.max_send_sge;
1886 4 : qpCb->recvSgeNum = qpNorm->extAttrs.qpAttr.cap.max_recv_sge;
1887 4 : qpCb->sendCqDepth = qpNorm->extAttrs.cqAttr.sendCqDepth;
1888 4 : qpCb->recvCqDepth = qpNorm->extAttrs.cqAttr.recvCqDepth;
1889 4 : qpCb->memAlign = qpNorm->extAttrs.memAlign;
1890 :
1891 4 : qpCb->channel = RsIbvCreateCompChannel(rdevCb->ibCtx);
1892 4 : CHK_PRT_RETURN(qpCb->channel == NULL, hccp_err("ibv_create_comp_channel failed! errno(%d)", errno), -EINVAL);
1893 4 : qpCb->qosAttr.tc = (RS_ROCE_DSCP_33 & RS_DSCP_MASK) << RS_DSCP_OFF;
1894 4 : qpCb->qosAttr.sl = RS_ROCE_4_SL;
1895 4 : qpCb->timeout = RS_QP_ATTR_TIMEOUT;
1896 4 : qpCb->retryCnt = RS_QP_ATTR_RETRY_CNT;
1897 :
1898 4 : qpCb->udpSport = qpNorm->extAttrs.udpSport;
1899 :
1900 4 : qpCb->aiOpSupport = qpNorm->aiOpSupport;
1901 4 : qpCb->grpId = rdevCb->rsCb->grpId;
1902 4 : qpCb->cqCstmFlag = qpNorm->extAttrs.dataPlaneFlag.bs.cqCstm;
1903 4 : qpCb->useResvMem = qpNorm->extAttrs.cstmFlag.bs.useResvMem;
1904 4 : qpCb->resvMemPoolId = qpNorm->extAttrs.resvMemPoolId;
1905 :
1906 4 : ret = RsEpollCtl(rdevCb->rsCb->connCb.epollfd, EPOLL_CTL_ADD, qpCb->channel->fd, EPOLLIN | EPOLLRDHUP);
1907 : #ifndef CA_CONFIG_LLT
1908 : if (ret) {
1909 : RsIbvDestroyCompChannel(qpCb->channel);
1910 : hccp_err("add channel fd failed ret %d", ret);
1911 : return ret;
1912 : }
1913 : #endif
1914 4 : return 0;
1915 : }
1916 :
1917 4 : STATIC int RsAllocQpcbWithAttrs(struct RsRdevCb *rdevCb, struct RsQpCb **qpCb, struct RsQpNormWithAttrs *qpNorm)
1918 : {
1919 : int ret;
1920 :
1921 4 : ret = RsCallocQpcb(1, qpCb);
1922 4 : CHK_PRT_RETURN(ret, hccp_err("alloc mem for qp_cb failed, ret:%d errno:%d", ret, errno), -ENOMEM);
1923 :
1924 4 : ret = pthread_mutex_init(&(*qpCb)->qpMutex, NULL);
1925 4 : if (ret) {
1926 0 : hccp_err("pthread_mutex_init failed, ret %d", ret);
1927 0 : goto qp_mutex_init_err;
1928 : }
1929 :
1930 4 : ret = pthread_mutex_init(&(*qpCb)->cqeErrInfo.mutex, NULL);
1931 4 : if (ret) {
1932 0 : hccp_err("pthread_mutex_init failed, ret %d", ret);
1933 0 : goto cqe_mutex_init_err;
1934 : }
1935 :
1936 4 : ret = RsQpcbInitWithAttrs(rdevCb, *qpCb, qpNorm);
1937 4 : if (ret) {
1938 0 : hccp_err("create qp tx rx failed ret %d", ret);
1939 0 : goto rs_qpcb_init_err;
1940 : }
1941 :
1942 4 : ret = RsInitMemPool(*qpCb);
1943 4 : if (ret) {
1944 0 : hccp_err("init mem pool failed ret %d", ret);
1945 0 : goto rs_init_mem_err;
1946 : }
1947 :
1948 4 : ret = RsDrvCreateCqWithAttrs(*qpCb, qpNorm->isExt, &qpNorm->extAttrs.cqAttr);
1949 4 : if (ret) {
1950 0 : hccp_err("create cq failed ret %d", ret);
1951 0 : goto create_cq_err;
1952 : }
1953 :
1954 4 : return 0;
1955 :
1956 0 : create_cq_err:
1957 0 : RsDeinitMemPool(*qpCb);
1958 :
1959 0 : rs_init_mem_err:
1960 0 : RsQpcbDeinit(rdevCb, *qpCb);
1961 :
1962 0 : rs_qpcb_init_err:
1963 0 : pthread_mutex_destroy(&(*qpCb)->cqeErrInfo.mutex);
1964 :
1965 0 : cqe_mutex_init_err:
1966 0 : pthread_mutex_destroy(&(*qpCb)->qpMutex);
1967 :
1968 0 : qp_mutex_init_err:
1969 0 : free(*qpCb);
1970 0 : *qpCb = NULL;
1971 :
1972 0 : return ret;
1973 : }
1974 :
1975 9 : STATIC int RsQpCheckQpNorm(struct RsQpNormWithAttrs *qpNorm, int *qpMode)
1976 : {
1977 9 : CHK_PRT_RETURN(qpNorm == NULL, hccp_err("qp_norm is NULL!"), -EINVAL);
1978 8 : CHK_PRT_RETURN(qpNorm->extAttrs.version != QP_CREATE_WITH_ATTR_VERSION,
1979 : hccp_err("attr version[%d] mismatch, expect [%d]", qpNorm->extAttrs.version, QP_CREATE_WITH_ATTR_VERSION),
1980 : -EINVAL);
1981 :
1982 7 : *qpMode = qpNorm->extAttrs.qpMode;
1983 7 : if (*qpMode < 0 || *qpMode >= RA_RS_ERR_QP_MODE) {
1984 1 : hccp_err("qp_mode[%d] must be greater than or equal to 0 and less than %d", *qpMode, RA_RS_ERR_QP_MODE);
1985 1 : return -EINVAL;
1986 : }
1987 :
1988 6 : if (*qpMode == RA_RS_OP_QP_MODE_EXT) {
1989 1 : *qpMode = RA_RS_OP_QP_MODE;
1990 : }
1991 :
1992 6 : qpNorm->extAttrs.qpMode = *qpMode;
1993 6 : return 0;
1994 : }
1995 :
1996 : #ifdef CUSTOM_INTERFACE
1997 8 : STATIC void RsQpPrepareCqDataPlaneInfo(struct ibv_cq *ibCq, struct AiDataPlaneCq *dataPlaneCq)
1998 : {
1999 8 : struct hns_roce_cq_data_plane_info cqInfo = {0};
2000 :
2001 8 : (void)RsRoceGetCqDataPlaneInfo(ibCq, &cqInfo);
2002 8 : dataPlaneCq->cqn = cqInfo.cqn;
2003 8 : dataPlaneCq->bufAddr = cqInfo.buf_addr;
2004 8 : dataPlaneCq->cqeSize = cqInfo.cqe_size;
2005 8 : dataPlaneCq->depth = cqInfo.depth;
2006 8 : dataPlaneCq->headAddr = cqInfo.head_addr;
2007 8 : dataPlaneCq->tailAddr = cqInfo.tail_addr;
2008 8 : dataPlaneCq->swdbAddr = cqInfo.swdb_addr;
2009 8 : dataPlaneCq->dbReg = cqInfo.db_reg;
2010 8 : hccp_info("cqn:%u buf_addr:0x%llx cqe_size:%u depth:%u head_addr:0x%llx tail_addr:0x%llx swdb_addr:0x%llx",
2011 : dataPlaneCq->cqn, dataPlaneCq->bufAddr, dataPlaneCq->cqeSize, dataPlaneCq->depth, dataPlaneCq->headAddr,
2012 : dataPlaneCq->tailAddr, dataPlaneCq->swdbAddr);
2013 8 : }
2014 :
2015 8 : STATIC void RsQpPrepareWqDataPlaneInfo(struct hns_roce_wq_data_plane_info *wqInfo, struct AiDataPlaneWq *dataPlaneWq)
2016 : {
2017 8 : dataPlaneWq->wqn = wqInfo->wqn;
2018 8 : dataPlaneWq->bufAddr = wqInfo->buf_addr;
2019 8 : dataPlaneWq->wqebbSize = wqInfo->wqebb_size;
2020 8 : dataPlaneWq->depth = wqInfo->depth;
2021 8 : dataPlaneWq->headAddr = wqInfo->head_addr;
2022 8 : dataPlaneWq->tailAddr = wqInfo->tail_addr;
2023 8 : dataPlaneWq->swdbAddr = wqInfo->swdb_addr;
2024 8 : dataPlaneWq->dbReg = wqInfo->db_reg;
2025 8 : hccp_info("wqn:%u buf_addr:0x%llx wqebb_size:%u depth:%u head_addr:%u tail_addr:%u swdb_addr:0x%llx",
2026 : dataPlaneWq->wqn, dataPlaneWq->bufAddr, dataPlaneWq->wqebbSize, dataPlaneWq->depth, dataPlaneWq->headAddr,
2027 : dataPlaneWq->tailAddr, dataPlaneWq->swdbAddr);
2028 8 : }
2029 :
2030 4 : STATIC void RsQpPrepareQpDataPlaneInfo(struct ibv_qp *ibQp, struct AiDataPlaneWq *dataPlaneSq,
2031 : struct AiDataPlaneWq *dataPlaneRq)
2032 : {
2033 4 : struct hns_roce_qp_data_plane_info qpInfo = {0};
2034 :
2035 4 : (void)RsRoceGetQpDataPlaneInfo(ibQp, &qpInfo);
2036 4 : RsQpPrepareWqDataPlaneInfo(&qpInfo.sq, dataPlaneSq);
2037 4 : RsQpPrepareWqDataPlaneInfo(&qpInfo.rq, dataPlaneRq);
2038 4 : }
2039 :
2040 4 : STATIC void RsQpPrepareDataPlaneInfo(struct RsQpNormWithAttrs *qpNorm, struct RsQpCb *qpCb,
2041 : struct RsQpRespWithAttrs *qpResp)
2042 : {
2043 : // skip to prepare cq data plane info
2044 4 : if (qpNorm->extAttrs.dataPlaneFlag.bs.cqCstm != 0) {
2045 4 : qpResp->aiScqAddr = (unsigned long long)(uintptr_t)qpCb->ibSendCq;
2046 4 : qpResp->aiRcqAddr = (unsigned long long)(uintptr_t)qpCb->ibRecvCq;
2047 4 : RsQpPrepareCqDataPlaneInfo(qpCb->ibSendCq, &qpResp->dataPlaneInfo.scq);
2048 4 : RsQpPrepareCqDataPlaneInfo(qpCb->ibRecvCq, &qpResp->dataPlaneInfo.rcq);
2049 : }
2050 :
2051 : // skip to prepare qp data plane info
2052 4 : if (qpNorm->aiOpSupport != 0) {
2053 4 : RsQpPrepareQpDataPlaneInfo(qpCb->ibQp, &qpResp->dataPlaneInfo.sq, &qpResp->dataPlaneInfo.rq);
2054 : }
2055 4 : }
2056 : #endif
2057 :
2058 4 : STATIC void RsQpPrepareQpResp(struct RsQpNormWithAttrs *qpNorm, struct RsQpCb *qpCb, struct RsQpRespWithAttrs *qpResp)
2059 : {
2060 4 : if (qpNorm->isExp != 0) {
2061 4 : qpCb->isExp = RS_IS_EXP;
2062 : } else {
2063 0 : qpCb->isExp = RS_NOT_EXP;
2064 : }
2065 :
2066 4 : qpResp->aiQpAddr = (unsigned long long)(uintptr_t)qpCb->ibQp;
2067 4 : qpResp->sqIndex = (unsigned int)qpCb->sqIndex;
2068 4 : qpResp->dbIndex = (unsigned int)qpCb->dbIndex;
2069 4 : qpResp->gidIdx = (unsigned int)qpCb->qpInfoLo.gidIdx;
2070 4 : qpResp->psn = (unsigned int)qpCb->qpInfoLo.psn;
2071 :
2072 : #ifdef CUSTOM_INTERFACE
2073 4 : if (RsIsCustomInterfaceSupported()) {
2074 4 : RsQpPrepareDataPlaneInfo(qpNorm, qpCb, qpResp);
2075 : }
2076 : #endif
2077 :
2078 4 : return;
2079 : }
2080 :
2081 12 : RS_ATTRI_VISI_DEF int RsQpCreateWithAttrs(unsigned int phyId, unsigned int rdevIndex, struct RsQpNormWithAttrs *qpNorm,
2082 : struct RsQpRespWithAttrs *qpResp)
2083 : {
2084 12 : struct RsRdevCb *rdevCb = NULL;
2085 12 : struct RsQpCb *qpCb = NULL;
2086 : int qpMode;
2087 : int ret;
2088 :
2089 12 : RS_QP_PARA_CHECK(phyId);
2090 10 : CHK_PRT_RETURN(qpResp == NULL, hccp_err("qp_resp is NULL!"), -EINVAL);
2091 :
2092 9 : ret = RsQpCheckQpNorm(qpNorm, &qpMode);
2093 9 : CHK_PRT_RETURN(ret != 0, hccp_err("check qp mode failed, ret:%d", ret), ret);
2094 :
2095 6 : ret = RsQpQueryInfo(phyId, rdevIndex, &rdevCb, qpMode);
2096 6 : CHK_PRT_RETURN(ret, hccp_err("query qp info failed:%d", ret), ret);
2097 :
2098 4 : ret = RsAllocQpcbWithAttrs(rdevCb, &qpCb, qpNorm);
2099 4 : CHK_PRT_RETURN(ret, hccp_err("alloc mem for qp_cb failed, ret:%d", ret), ret);
2100 :
2101 4 : ret = RsDrvQpCreateWithAttrs(qpCb, qpNorm);
2102 4 : if (ret) {
2103 0 : hccp_err("create drv qp create failed:%d", ret);
2104 0 : goto create_qp_err;
2105 : }
2106 :
2107 4 : ret = ibv_req_notify_cq(qpCb->ibSendCq, 0);
2108 4 : if (ret) {
2109 0 : hccp_err("Couldn't request send CQ notification, ret:%d", ret);
2110 0 : ret = -EOPENSRC;
2111 0 : goto ret_noritfy_cq;
2112 : }
2113 :
2114 4 : ret = ibv_req_notify_cq(qpCb->ibRecvCq, 0);
2115 4 : if (ret) {
2116 0 : hccp_err("Couldn't request recv CQ notification, ret:%d", ret);
2117 0 : ret = -EOPENSRC;
2118 0 : goto ret_noritfy_cq;
2119 : }
2120 :
2121 4 : ret = RsQpNotifyMr(rdevCb, qpCb, &qpResp->qpn); // alloc mr
2122 4 : if (ret) {
2123 0 : hccp_err("store qp notify mr failed:%d", ret);
2124 0 : goto ret_noritfy_cq;
2125 : }
2126 :
2127 4 : RsQpPrepareQpResp(qpNorm, qpCb, qpResp);
2128 :
2129 4 : return 0;
2130 :
2131 0 : ret_noritfy_cq:
2132 0 : RsDrvQpDestroy(qpCb);
2133 :
2134 0 : create_qp_err:
2135 0 : RsFreeQpcb(rdevCb, qpCb);
2136 0 : return ret;
2137 : }
2138 :
2139 35 : void RsMrRelease(struct RsQpCb *qpCb)
2140 : {
2141 35 : struct RsMrCb *mrTmp2 = NULL;
2142 35 : struct RsMrCb *mrTmp = NULL;
2143 :
2144 35 : RS_PTHREAD_MUTEX_LOCK(&qpCb->qpMutex);
2145 35 : RS_LIST_GET_HEAD_ENTRY(mrTmp, mrTmp2, &qpCb->mrList, list, struct RsMrCb);
2146 69 : for (; (&mrTmp->list) != &qpCb->mrList;
2147 34 : mrTmp = mrTmp2, mrTmp2 = list_entry(mrTmp2->list.next, struct RsMrCb, list)) {
2148 34 : if (mrTmp->ibMr != qpCb->rdevCb->notifyMr) {
2149 2 : (void)RsDrvMrDereg(mrTmp->ibMr);
2150 : }
2151 34 : RsListDel(&mrTmp->list);
2152 34 : free(mrTmp);
2153 34 : mrTmp = NULL;
2154 : }
2155 :
2156 35 : RS_LIST_GET_HEAD_ENTRY(mrTmp, mrTmp2, &qpCb->remMrList, list, struct RsMrCb);
2157 66 : for (; (&mrTmp->list) != &qpCb->remMrList;
2158 31 : mrTmp = mrTmp2, mrTmp2 = list_entry(mrTmp2->list.next, struct RsMrCb, list)) {
2159 31 : RsListDel(&mrTmp->list);
2160 31 : free(mrTmp);
2161 31 : mrTmp = NULL;
2162 : }
2163 35 : RS_PTHREAD_MUTEX_ULOCK(&qpCb->qpMutex);
2164 35 : }
2165 :
2166 34 : STATIC void RsQpRelease(struct RsQpCb *qpCb)
2167 : {
2168 34 : RS_PTHREAD_MUTEX_LOCK(&qpCb->rdevCb->rdevMutex);
2169 34 : RsListDel(&qpCb->list);
2170 34 : RS_PTHREAD_MUTEX_ULOCK(&qpCb->rdevCb->rdevMutex);
2171 34 : RsIbvAckCqEvents(qpCb->ibSendCq, qpCb->numSendCqEvents);
2172 34 : RsIbvAckCqEvents(qpCb->ibRecvCq, qpCb->numRecvCqEvents);
2173 :
2174 : // dereg mr
2175 34 : RsMrRelease(qpCb);
2176 34 : }
2177 :
2178 34 : RS_ATTRI_VISI_DEF int RsQpDestroy(unsigned int phyId, unsigned int rdevIndex, unsigned int qpn)
2179 : {
2180 34 : struct RsQpCb *qpCb = NULL;
2181 : int ret;
2182 :
2183 34 : RS_QP_PARA_CHECK(phyId);
2184 34 : ret = RsQpn2qpcb(phyId, rdevIndex, qpn, &qpCb);
2185 34 : CHK_PRT_RETURN(ret != 0 || qpCb == NULL, hccp_err("get qp cb failed! qpn %u, ret %d", qpn, ret), ret);
2186 :
2187 32 : RsQpRelease(qpCb);
2188 :
2189 : // destroy qp
2190 32 : RsDrvQpDestroy(qpCb);
2191 32 : RsDrvDestroyCq(qpCb);
2192 32 : RsDeinitMemPool(qpCb);
2193 :
2194 32 : qpCb->rdevCb->qpCnt--;
2195 32 : ret = RsQpcbDeinit(qpCb->rdevCb, qpCb);
2196 32 : if (ret) {
2197 0 : hccp_err("rs_qpcb_deinit failed! ret[%d]", ret);
2198 : }
2199 :
2200 32 : pthread_mutex_destroy(&qpCb->cqeErrInfo.mutex);
2201 32 : pthread_mutex_destroy(&qpCb->qpMutex);
2202 32 : hccp_info("qp %d destroy qp, send wr[%u].", qpn, qpCb->sendWrNum);
2203 :
2204 32 : free(qpCb);
2205 32 : qpCb = NULL;
2206 32 : return ret;
2207 : }
2208 :
2209 0 : RS_ATTRI_VISI_DEF int RsQpDestroyWithoutCQ(unsigned int phyId, unsigned int rdevIndex, unsigned int qpn)
2210 : {
2211 0 : struct RsQpCb *qpCb = NULL;
2212 : int ret;
2213 :
2214 0 : RS_QP_PARA_CHECK(phyId);
2215 0 : ret = RsQpn2qpcb(phyId, rdevIndex, qpn, &qpCb);
2216 0 : CHK_PRT_RETURN(ret != 0 || qpCb == NULL, hccp_err("get qp cb failed! qpn %u, ret %d", qpn, ret), ret);
2217 :
2218 0 : RsQpRelease(qpCb);
2219 :
2220 : // destroy qp
2221 0 : RsDrvQpDestroy(qpCb);
2222 0 : RsDeinitMemPool(qpCb);
2223 :
2224 0 : qpCb->rdevCb->qpCnt--;
2225 0 : ret = RsQpcbDeinit(qpCb->rdevCb, qpCb);
2226 0 : if (ret) {
2227 0 : hccp_err("rs_qpcb_deinit failed! ret[%d]", ret);
2228 : }
2229 :
2230 0 : pthread_mutex_destroy(&qpCb->cqeErrInfo.mutex);
2231 0 : pthread_mutex_destroy(&qpCb->qpMutex);
2232 0 : hccp_info("qp %d destroy qp without cq, send wr[%u].", qpn, qpCb->sendWrNum);
2233 :
2234 0 : free(qpCb);
2235 0 : qpCb = NULL;
2236 0 : return ret;
2237 : }
2238 :
2239 27 : static void RsQpConnectAsyncMr(const struct RsQpCb *qpCb)
2240 : {
2241 : int ret;
2242 27 : struct RsMrCb *mrCb = NULL;
2243 27 : struct RsMrCb *mrCb2 = NULL;
2244 :
2245 27 : RS_LIST_GET_HEAD_ENTRY(mrCb, mrCb2, &qpCb->mrList, list, struct RsMrCb);
2246 53 : for (; (&mrCb->list) != &qpCb->mrList; mrCb = mrCb2, mrCb2 = list_entry(mrCb2->list.next, struct RsMrCb, list)) {
2247 26 : ret = RsMrInfoSync(mrCb);
2248 26 : if (ret) {
2249 0 : hccp_warn("rs_mr_info_sync unsuccessful, ret:%d", ret);
2250 : }
2251 : }
2252 27 : }
2253 :
2254 27 : STATIC void RsQpConnectAsyncQpcbSet(int fd, struct RsQpCb *qpCb)
2255 : {
2256 : int ret;
2257 27 : ret = RsSocketSend(fd, &qpCb->qpInfoLo, sizeof(struct RsQpInfo));
2258 27 : if (ret == sizeof(struct RsQpInfo)) {
2259 27 : qpCb->sendLen += (uint32_t)ret;
2260 27 : qpCb->state = RS_QP_STATUS_CONNECTING;
2261 : } else {
2262 0 : qpCb->state = RS_QP_STATUS_TIMEOUT;
2263 : }
2264 27 : }
2265 :
2266 27 : STATIC void RsQpConnectAsyncLength(int fd, struct RsQpCb *qpCb)
2267 : {
2268 : int ret;
2269 : struct RsQpLenInfo msg;
2270 :
2271 27 : msg.cmd = RS_CMD_LEN_INFO;
2272 27 : msg.len = qpCb->sendLen;
2273 :
2274 27 : ret = RsSocketSend(fd, &msg, sizeof(struct RsQpLenInfo));
2275 27 : if (ret != sizeof(struct RsQpLenInfo)) {
2276 0 : qpCb->state = RS_QP_STATUS_TIMEOUT;
2277 : }
2278 27 : }
2279 :
2280 32 : static int RsQpConnectAsyncInitPara(struct RsQpConnPara qpConnPara, int fd, struct RsQpCb **qpCb,
2281 : struct RsConnInfo **conn)
2282 : {
2283 : int ret;
2284 :
2285 32 : CHK_PRT_RETURN(qpConnPara.phyId >= RS_MAX_DEV_NUM, hccp_err("param error ! phyId:%u", qpConnPara.phyId), -EINVAL);
2286 :
2287 32 : CHK_PRT_RETURN(fd < 0, hccp_err("param error ! fd:%d must be non-negative", fd), -EINVAL);
2288 :
2289 30 : ret = RsQpn2qpcb(qpConnPara.phyId, qpConnPara.rdevIndex, qpConnPara.qpn, qpCb);
2290 30 : CHK_PRT_RETURN(ret, hccp_err("get qpcb failed, qpn %u, ret %d", qpConnPara.qpn, ret), ret);
2291 :
2292 28 : ret = RsFd2conn(fd, conn);
2293 28 : CHK_PRT_RETURN(ret, hccp_err("get conn failed, fd %d, ret %d", fd, ret), ret);
2294 :
2295 28 : RsGetCurTime(&((*qpCb)->startTime));
2296 28 : (*qpCb)->sendLen = 0;
2297 28 : (*qpCb)->recvLen = 0;
2298 28 : (*qpCb)->expectLen = 0;
2299 28 : (*qpCb)->connInfo = *conn;
2300 :
2301 28 : return 0;
2302 : }
2303 :
2304 2 : STATIC int RsTypicalQpStateModifytoRtr(struct RsQpCb *qpCb, struct TypicalQp *localQpInfo,
2305 : struct TypicalQp *remoteQpInfo)
2306 : {
2307 2 : struct ibv_port_attr portAttr = {0};
2308 2 : union ibv_gid remoteInfoGid = {0};
2309 2 : struct ibv_qp_attr attr = {0};
2310 : int ret;
2311 :
2312 2 : attr.qp_state = IBV_QPS_RTR;
2313 2 : attr.dest_qp_num = remoteQpInfo->qpn;
2314 2 : attr.rq_psn = remoteQpInfo->psn;
2315 2 : attr.min_rnr_timer = RS_QP_ATTR_MIN_RNR_TIMER;
2316 2 : (attr.ah_attr).is_global = 0;
2317 2 : (attr.ah_attr).sl = localQpInfo->sl;
2318 2 : (attr.ah_attr).src_path_bits = 0;
2319 2 : (attr.ah_attr).port_num = qpCb->rdevCb->ibPort;
2320 :
2321 2 : attr.path_mtu = RsDrvSetMtu(qpCb);
2322 2 : CHK_PRT_RETURN(attr.path_mtu < IBV_MTU_1024,
2323 : hccp_err("qpn[%u] failed to set mtu, mtu[%d] < [%d]", localQpInfo->qpn, attr.path_mtu, IBV_MTU_1024), -EPERM);
2324 2 : if (qpCb->rdevCb->rsCb->hccpMode == NETWORK_PEER_ONLINE) {
2325 2 : attr.max_dest_rd_atomic = RS_MAX_RD_ATOMIC_NUM_PEER_ONLINE;
2326 : } else {
2327 0 : attr.max_dest_rd_atomic = RS_MAX_RD_ATOMIC_NUM;
2328 : }
2329 2 : (attr.ah_attr).grh.traffic_class = localQpInfo->tc;
2330 : // get gid_idx dynamically to avoid gid_idx changed issue: refresh gid_idx when it changed
2331 2 : ret = RsDrvGetGidIndex(qpCb->rdevCb, &portAttr, &qpCb->qpInfoLo.gidIdx);
2332 2 : if (ret == 0 && localQpInfo->gidIdx != (uint32_t)qpCb->qpInfoLo.gidIdx) {
2333 0 : hccp_warn("qpn[%u] qp_mode[%d] refresh gid_idx[%u] to [%d]", localQpInfo->qpn, qpCb->qpMode,
2334 : localQpInfo->gidIdx, qpCb->qpInfoLo.gidIdx);
2335 0 : localQpInfo->gidIdx = (uint32_t)qpCb->qpInfoLo.gidIdx;
2336 : }
2337 :
2338 2 : (void)memcpy_s(remoteInfoGid.raw, HCCP_GID_RAW_LEN, remoteQpInfo->gid, HCCP_GID_RAW_LEN);
2339 2 : if (remoteInfoGid.global.interface_id) {
2340 2 : attr.ah_attr.is_global = 1;
2341 2 : attr.ah_attr.grh.hop_limit = 1;
2342 2 : attr.ah_attr.grh.dgid = remoteInfoGid;
2343 2 : attr.ah_attr.grh.sgid_index = localQpInfo->gidIdx;
2344 : }
2345 :
2346 2 : ret = RsIbvModifyQp(qpCb->ibQp, &attr,
2347 : IBV_QP_STATE | IBV_QP_AV | IBV_QP_PATH_MTU | IBV_QP_DEST_QPN | IBV_QP_RQ_PSN | IBV_QP_MAX_DEST_RD_ATOMIC |
2348 : IBV_QP_MIN_RNR_TIMER);
2349 2 : CHK_PRT_RETURN(ret,
2350 : hccp_err("[modifyto_rtr]local_qpn[%u] remote_qpn[%u] ibv_modify_qp failed ret[%d], errno[%d]", localQpInfo->qpn,
2351 : remoteQpInfo->qpn, ret, errno),
2352 : -EOPENSRC);
2353 2 : hccp_info("qp qos attr: qpn[%u] tc[%u] sl[%u]", localQpInfo->qpn, localQpInfo->tc, localQpInfo->sl);
2354 2 : return 0;
2355 : }
2356 :
2357 2 : STATIC int RsTypicalQpStateModifytoRts(struct RsQpCb *qpCb, struct TypicalQp *localQpInfo)
2358 : {
2359 2 : struct ibv_qp_attr attr = {0};
2360 : int ret;
2361 :
2362 2 : attr.qp_state = IBV_QPS_RTS;
2363 2 : attr.timeout = (uint8_t)localQpInfo->retryTime;
2364 2 : attr.retry_cnt = (uint8_t)localQpInfo->retryCnt;
2365 2 : attr.rnr_retry = RS_QP_ATTR_RNR_RETRY;
2366 2 : attr.sq_psn = localQpInfo->psn;
2367 2 : if (qpCb->rdevCb->rsCb->hccpMode == NETWORK_PEER_ONLINE) {
2368 2 : attr.max_rd_atomic = RS_MAX_RD_ATOMIC_NUM_PEER_ONLINE;
2369 : } else {
2370 0 : attr.max_rd_atomic = RS_MAX_RD_ATOMIC_NUM;
2371 : }
2372 :
2373 2 : ret = RsIbvModifyQp(qpCb->ibQp, &attr,
2374 : IBV_QP_STATE | IBV_QP_TIMEOUT | IBV_QP_RETRY_CNT | IBV_QP_RNR_RETRY | IBV_QP_SQ_PSN | IBV_QP_MAX_QP_RD_ATOMIC);
2375 2 : CHK_PRT_RETURN(ret != 0,
2376 : hccp_err("[modifyto_rts]local_qpn[%u] ibv_modify_qp failed ret[%d], errno[%d]", localQpInfo->qpn, ret, errno),
2377 : -EOPENSRC);
2378 :
2379 2 : hccp_info("qp rdma attr: qpn[%u] timeout[%u] retrycnt[%u]", localQpInfo->qpn, localQpInfo->retryTime,
2380 : localQpInfo->retryCnt);
2381 2 : return 0;
2382 : }
2383 :
2384 2 : STATIC void RsTypicalQpModifyInfoRelated(struct RsQpCb *qpCb, struct TypicalQp *localQpInfo,
2385 : struct TypicalQp *remoteQpInfo)
2386 : {
2387 2 : qpCb->state = RS_QP_STATUS_CONNECTED;
2388 : // local qp info related: no need to relate qpn, psn, gid_idx, gid
2389 2 : qpCb->qosAttr.tc = (unsigned char)localQpInfo->tc;
2390 2 : qpCb->qosAttr.sl = (unsigned char)localQpInfo->sl;
2391 2 : qpCb->retryCnt = localQpInfo->retryCnt;
2392 2 : qpCb->timeout = localQpInfo->retryTime;
2393 : // remote qp info related
2394 2 : qpCb->qpInfoRem.qpn = (int)remoteQpInfo->qpn;
2395 2 : qpCb->qpInfoRem.psn = (int)remoteQpInfo->psn;
2396 2 : qpCb->qpInfoRem.gidIdx = (int)remoteQpInfo->gidIdx;
2397 2 : (void)memcpy_s(qpCb->qpInfoRem.gid.raw, HCCP_GID_RAW_LEN, remoteQpInfo->gid, HCCP_GID_RAW_LEN);
2398 2 : }
2399 :
2400 2 : STATIC void RsTypicalQpModifyExtend(struct RsQpCb *qpCb, struct TypicalQp *localQpInfo, struct TypicalQp *remoteQpInfo)
2401 : {
2402 2 : struct ibv_hyroce_feature output = {0};
2403 2 : struct ibv_hyroce_feature input = {0};
2404 2 : struct ibv_qp_attr_extend attr = {0};
2405 2 : uint32_t needMoreNego = 0;
2406 2 : int ret = 0;
2407 :
2408 2 : if (qpCb->rdevCb->ibCtxEx == NULL) {
2409 0 : return;
2410 : }
2411 :
2412 2 : if (localQpInfo->tc != qpCb->qosAttr.tc || localQpInfo->sl != qpCb->qosAttr.sl) {
2413 2 : hccp_warn("localQpInfo tc:%u sl:%u is not equal to qpCb tc:%u sl:%u", localQpInfo->tc, localQpInfo->sl,
2414 : qpCb->qosAttr.tc, qpCb->qosAttr.sl);
2415 2 : return;
2416 : }
2417 :
2418 0 : ret = memcpy_s(&input, sizeof(struct ibv_hyroce_feature), &remoteQpInfo->feature, sizeof(struct HyperFeature));
2419 0 : if (ret != 0) {
2420 0 : hccp_warn("memcpy_s feature unsuccessful, ret:%d qpn:%u ibv_hyroce_feature len:%zu HyperFeature len:%zu", ret,
2421 : qpCb->ibQp->qp_num, sizeof(struct ibv_hyroce_feature), sizeof(struct HyperFeature));
2422 0 : return;
2423 : }
2424 :
2425 0 : ret = RsIbvNegoQpHyroceFeature(qpCb->rdevCb->ibCtxEx, qpCb->ibQp, &input, &output, &needMoreNego);
2426 0 : if (ret != 0) {
2427 0 : hccp_warn("RsIbvNegoQpHyroceFeature unsuccessful, ret:%d qpn:%u errno:%d", ret, qpCb->ibQp->qp_num, errno);
2428 0 : return;
2429 : }
2430 :
2431 0 : attr.qp = qpCb->ibQp;
2432 0 : (void)memcpy_s(&attr.feature, sizeof(struct ibv_hyroce_feature), &output, sizeof(struct ibv_hyroce_feature));
2433 0 : attr.udp_src_port = localQpInfo->udpSport;
2434 0 : ret = RsIbvModifyQpExtend(qpCb->rdevCb->ibCtxEx, &attr,
2435 : IBV_QP_ATTR_EXTEND_UDP_SRC_PORT | IBV_QP_ATTR_EXTEND_HYROCE_FEATURE);
2436 0 : if (ret != 0) {
2437 0 : hccp_warn("RsIbvModifyQpExtend unsuccessful, ret:%d qpn:%u errno:%d", ret, qpCb->ibQp->qp_num, errno);
2438 0 : return;
2439 : }
2440 :
2441 0 : hccp_dbg("RsIbvModifyQpExtend successful, qpn:%u", qpCb->ibQp->qp_num);
2442 0 : return;
2443 : }
2444 :
2445 2 : STATIC int RsTypicalQueryQpAttr(struct RsQpCb *qpCb, struct TypicalQpAttr *qpAttr)
2446 : {
2447 2 : unsigned int qpAttrMask = HNS_ROCE_AI_QPC_UDPSPN;
2448 2 : struct hns_roce_qpc_attr_val qpAttrVal = {0};
2449 2 : struct ibv_qp_init_attr initAttr = {0};
2450 2 : struct ibv_qp_attr ibvQpAttr = {0};
2451 2 : int ret = 0;
2452 :
2453 : (void)qpAttrMask;
2454 : (void)qpAttrVal;
2455 : #ifdef CUSTOM_INTERFACE
2456 2 : if (RsIsCustomInterfaceSupported()) {
2457 2 : ret = RsRoceQueryQpc(qpCb->ibQp, &qpAttrVal, qpAttrMask);
2458 2 : if (ret != 0) {
2459 2 : hccp_warn("qpn:%d query qpc unsuccessful, ret %d", qpCb->qpInfoLo.qpn, ret);
2460 : } else {
2461 0 : qpCb->udpSport = qpAttrVal.udp_sport;
2462 : }
2463 : }
2464 : #endif
2465 :
2466 2 : ret = RsIbvQueryQp(qpCb->ibQp, &ibvQpAttr, IBV_QP_PATH_MTU, &initAttr);
2467 2 : CHK_PRT_RETURN(ret, hccp_err("RsIbvQueryQp failed, ret:%d errno:%d", ret, errno), -EOPENSRC);
2468 :
2469 2 : if (qpCb->ibQpEx != NULL) {
2470 0 : qpAttr->vendorPrivInfo = qpCb->ibQpEx->vendor_priv_info.value;
2471 : }
2472 :
2473 2 : qpAttr->udpSport = qpCb->udpSport;
2474 2 : qpAttr->pathMtu = (int)ibvQpAttr.path_mtu;
2475 2 : return 0;
2476 : }
2477 :
2478 2 : RS_ATTRI_VISI_DEF int RsTypicalQpModify(unsigned int phyId, unsigned int rdevIndex, struct TypicalQp localQpInfo,
2479 : struct TypicalQp remoteQpInfo, struct TypicalQpAttr *qpAttr)
2480 : {
2481 2 : struct ibv_qp_init_attr initAttr = {0};
2482 2 : struct ibv_qp_attr attr = {0};
2483 2 : struct RsQpCb *qpCb = NULL;
2484 : int ret;
2485 :
2486 2 : CHK_PRT_RETURN(phyId >= RS_MAX_DEV_NUM, hccp_err("[modify]phyId:%u >= [%d], is invalid", phyId, RS_MAX_DEV_NUM),
2487 : -EINVAL);
2488 :
2489 2 : CHK_PRT_RETURN(RsQpn2qpcb(phyId, rdevIndex, localQpInfo.qpn, &qpCb),
2490 : hccp_err("[modify]rs_qpn2qpcb qpn:%u failed, phyId[%u]", localQpInfo.qpn, phyId), -EACCES);
2491 :
2492 2 : CHK_PRT_RETURN(qpCb->state == RS_QP_STATUS_CONNECTED,
2493 : hccp_info("local_qpn:%u remote_qpn:%u already been connected, no need to modify again", localQpInfo.qpn,
2494 : remoteQpInfo.qpn),
2495 : 0);
2496 :
2497 : // see ib_modify_qp_is_ok for status modify, only support modify qp from INIT to RTR
2498 2 : ret = RsIbvQueryQp(qpCb->ibQp, &attr, IBV_QP_STATE, &initAttr);
2499 2 : CHK_PRT_RETURN(ret != 0 || attr.qp_state != IBV_QPS_INIT,
2500 : hccp_err("query qpn:%u failed, ret:%d or state:%d != %d", localQpInfo.qpn, ret, attr.qp_state, IBV_QPS_INIT),
2501 : -EOPENSRC);
2502 :
2503 2 : RsTypicalQpModifyExtend(qpCb, &localQpInfo, &remoteQpInfo);
2504 :
2505 2 : ret = RsTypicalQpStateModifytoRtr(qpCb, &localQpInfo, &remoteQpInfo);
2506 2 : CHK_PRT_RETURN(ret != 0,
2507 : hccp_err("[modify]local_qpn:%u remote_qpn:%u modify to rtr failed, ret %d", localQpInfo.qpn, remoteQpInfo.qpn,
2508 : ret),
2509 : ret);
2510 :
2511 2 : ret = RsTypicalQpStateModifytoRts(qpCb, &localQpInfo);
2512 2 : CHK_PRT_RETURN(ret != 0,
2513 : hccp_err("[modify]local_qpn:%u remote_qpn:%u modify to rts failed, ret %d", localQpInfo.qpn, remoteQpInfo.qpn,
2514 : ret),
2515 : ret);
2516 :
2517 2 : ret = RsTypicalQueryQpAttr(qpCb, qpAttr);
2518 2 : CHK_PRT_RETURN(ret != 0, hccp_err("RsTypicalQueryQpAttr failed, ret %d local_qpn:%u", ret, localQpInfo.qpn), ret);
2519 :
2520 2 : RsTypicalQpModifyInfoRelated(qpCb, &localQpInfo, &remoteQpInfo);
2521 :
2522 2 : hccp_info("local_qpn:%u remote_qpn:%u modify succ, udpSport:%u", localQpInfo.qpn, remoteQpInfo.qpn, qpCb->udpSport);
2523 :
2524 2 : return 0;
2525 : }
2526 :
2527 2 : STATIC int RsQpStateBatchModifytoPause(struct RsQpCb *qpCb)
2528 : {
2529 : int ret;
2530 :
2531 2 : ret = RsDrvQpStateModifytoReset(qpCb);
2532 2 : CHK_PRT_RETURN(ret, hccp_err("qp modify to reset failed, ret %d", ret), ret);
2533 :
2534 2 : hccp_info("local qpn[%d] remote qpn[%d] modify to pause succ", qpCb->qpInfoLo.qpn, qpCb->qpInfoRem.qpn);
2535 2 : return 0;
2536 : }
2537 :
2538 2 : STATIC int RsQpStateBatchModifytoConnected(struct RsQpCb *qpCb)
2539 : {
2540 : struct ibv_qp_attr attr;
2541 : int ret;
2542 :
2543 2 : ret = memset_s(&attr, sizeof(struct ibv_qp_attr), 0, sizeof(struct ibv_qp_attr));
2544 2 : CHK_PRT_RETURN(ret, hccp_err("memset_s attr failed ret %d", ret), -ESAFEFUNC);
2545 :
2546 2 : ret = RsDrvQpStateModifytoInit(qpCb, &attr);
2547 2 : CHK_PRT_RETURN(ret, hccp_err("qp modify to init failed, ret %d", ret), ret);
2548 2 : ret = RsDrvQpStateModifytoRtr(qpCb, &attr);
2549 2 : CHK_PRT_RETURN(ret, hccp_err("qp modify to rtr failed, ret %d", ret), ret);
2550 2 : ret = RsDrvQpStateModifytoRts(qpCb, &attr);
2551 2 : CHK_PRT_RETURN(ret, hccp_err("qp modify to rts failed, ret %d", ret), ret);
2552 :
2553 2 : hccp_info("local qpn[%d] remote qpn[%d] modify to rts succ", qpCb->qpInfoLo.qpn, qpCb->qpInfoRem.qpn);
2554 2 : return 0;
2555 : }
2556 :
2557 3 : RS_ATTRI_VISI_DEF int RsQpBatchModify(unsigned int phyId, unsigned int rdevIndex, int status, int qpn[], int qpnNum)
2558 : {
2559 3 : struct RsQpCb *qpCb = NULL;
2560 : int ret;
2561 : int i;
2562 :
2563 3 : CHK_PRT_RETURN(phyId >= RS_MAX_DEV_NUM, hccp_err("[modify]phyId:%u >= [%d], is invalid", phyId, RS_MAX_DEV_NUM),
2564 : -EINVAL);
2565 :
2566 7 : for (i = 0; i < qpnNum; i++) {
2567 5 : CHK_PRT_RETURN(RsQpn2qpcb(phyId, rdevIndex, (uint32_t)qpn[i], &qpCb),
2568 : hccp_err("[modify]rs_qpn2qpcb failed, phyId[%u]", phyId), -EACCES);
2569 :
2570 : /*
2571 : * see ib_modify_qp_is_ok for status modify
2572 : * only support modify qp from STATUS_PAUSE(RESET) to STATUS_CONNECTED(INIT)
2573 : */
2574 5 : if (status == RS_QP_STATUS_CONNECTED && qpCb->state == RS_QP_STATUS_PAUSE) {
2575 2 : ret = RsQpStateBatchModifytoConnected(qpCb);
2576 2 : CHK_PRT_RETURN(ret,
2577 : hccp_err("modify_qp qpn[%d]:%d to connected failed, ret[%d] phyId[%u]", i, qpn[i], ret, phyId), ret);
2578 3 : } else if (status == RS_QP_STATUS_PAUSE) {
2579 2 : ret = RsQpStateBatchModifytoPause(qpCb);
2580 2 : CHK_PRT_RETURN(ret,
2581 : hccp_err("modify_qp qpn[%d]:%d to pause failed, ret[%d] phyId[%u]", i, qpn[i], ret, phyId), ret);
2582 : } else {
2583 1 : hccp_err("modify_qp qpn[%d]:%d failed, not support to modify status[%d] to status[%d], phyId[%u]", i,
2584 : qpn[i], qpCb->state, status, phyId);
2585 1 : return -EINVAL;
2586 : }
2587 :
2588 4 : qpCb->state = status;
2589 : }
2590 :
2591 2 : return 0;
2592 : }
2593 :
2594 2 : RS_ATTRI_VISI_DEF int RsSetQpLbValue(unsigned int phyId, unsigned int rdevIndex, unsigned int qpn, int lbValue)
2595 : {
2596 2 : struct RsQpCb *qpCb = NULL;
2597 2 : int ret = 0;
2598 :
2599 2 : ret = RsQpn2qpcb(phyId, rdevIndex, qpn, &qpCb);
2600 2 : CHK_PRT_RETURN(ret != 0, hccp_err("RsQpn2qpcb failed ret:%d", ret), ret);
2601 :
2602 1 : return RsRoceSetQpLbValue(qpCb->ibQp, lbValue);
2603 : }
2604 :
2605 3 : RS_ATTRI_VISI_DEF int RsGetQpLbValue(unsigned int phyId, unsigned int rdevIndex, unsigned int qpn, int *lbValue)
2606 : {
2607 3 : struct RsQpCb *qpCb = NULL;
2608 3 : int ret = 0;
2609 :
2610 3 : ret = RsQpn2qpcb(phyId, rdevIndex, qpn, &qpCb);
2611 3 : CHK_PRT_RETURN(ret != 0, hccp_err("RsQpn2qpcb failed ret:%d", ret), ret);
2612 :
2613 2 : return RsRoceGetQpLbValue(qpCb->ibQp, lbValue);
2614 : }
2615 :
2616 32 : RS_ATTRI_VISI_DEF int RsQpConnectAsync(unsigned int phyId, unsigned int rdevIndex, unsigned int qpn, int fd)
2617 : {
2618 : int ret;
2619 32 : struct RsQpCb *qpCb = NULL;
2620 32 : struct RsConnInfo *conn = NULL;
2621 : struct RsQpConnPara qpConnPara;
2622 32 : hccp_info("qp:%d, fd:%d", qpn, fd);
2623 :
2624 32 : qpConnPara.phyId = phyId;
2625 32 : qpConnPara.rdevIndex = rdevIndex;
2626 32 : qpConnPara.qpn = qpn;
2627 32 : ret = RsQpConnectAsyncInitPara(qpConnPara, fd, &qpCb, &conn);
2628 32 : CHK_PRT_RETURN(ret, hccp_err("rs_qp_connect_async_init_para failed, qpn %u, ret %d", qpn, ret), ret);
2629 :
2630 28 : RS_PTHREAD_MUTEX_LOCK(&qpCb->qpMutex);
2631 :
2632 28 : if (qpCb->state == RS_QP_STATUS_REM_FD_CLOSE) {
2633 0 : hccp_warn("remote qp fd close, can not use it anymore!");
2634 0 : RS_PTHREAD_MUTEX_ULOCK(&qpCb->qpMutex);
2635 0 : return -EFAULT;
2636 : }
2637 :
2638 28 : if ((qpCb->state == RS_QP_STATUS_CONNECTED) || (qpCb->state == RS_QP_STATUS_CONNECTING)) {
2639 1 : hccp_warn("qp %d has already sync! state[%d]", qpCb->qpInfoLo.qpn, qpCb->state);
2640 1 : RS_PTHREAD_MUTEX_ULOCK(&qpCb->qpMutex);
2641 1 : return -EEXIST;
2642 : }
2643 :
2644 27 : RsQpConnectAsyncQpcbSet(fd, qpCb);
2645 :
2646 27 : hccp_info("after socket fd %d send QP %u, chipId %u, state:%d!", fd, qpn, qpCb->rdevCb->rsCb->chipId, qpCb->state);
2647 :
2648 27 : RS_PTHREAD_MUTEX_ULOCK(&qpCb->qpMutex);
2649 :
2650 27 : RsQpMrRecvHandle(fd, qpCb);
2651 :
2652 27 : RsQpConnectAsyncMr(qpCb);
2653 :
2654 27 : RsQpConnectAsyncLength(fd, qpCb);
2655 :
2656 27 : hccp_info("QP %d async done, state:%d!", qpn, qpCb->state);
2657 :
2658 27 : return 0;
2659 : }
2660 :
2661 1 : RS_ATTRI_VISI_DEF int RsGetQpStatus(unsigned int phyId, unsigned int rdevIndex, unsigned int qpn,
2662 : struct RsQpStatusInfo *qpInfo)
2663 : {
2664 1 : unsigned int qpAttrMask = HNS_ROCE_AI_QPC_UDPSPN;
2665 1 : struct hns_roce_qpc_attr_val qpAttrVal = {0};
2666 1 : struct RsQpCb *qpCb = NULL;
2667 : int ret;
2668 :
2669 : (void)qpAttrMask;
2670 : (void)qpAttrVal;
2671 1 : CHK_PRT_RETURN(qpInfo == NULL, hccp_err("param error, qpInfo is NULL"), -EINVAL);
2672 :
2673 1 : CHK_PRT_RETURN(phyId >= RS_MAX_DEV_NUM, hccp_err("phyId:%u >= [%d], is invalid", phyId, RS_MAX_DEV_NUM), -EINVAL);
2674 :
2675 1 : ret = RsQpn2qpcb(phyId, rdevIndex, qpn, &qpCb);
2676 1 : CHK_PRT_RETURN(ret, hccp_err("get qp cb failed, qpn:%u, ret %d", qpn, ret), ret);
2677 :
2678 : // qp state is CONNECTED, no need to handle
2679 1 : if (qpCb->state == RS_QP_STATUS_CONNECTED) {
2680 1 : goto update_qp_cb;
2681 : }
2682 :
2683 : // modify state to CONNECTED
2684 0 : if (qpCb->expectLen == qpCb->recvLen - sizeof(struct RsQpLenInfo)) {
2685 0 : qpCb->state = RS_QP_STATUS_CONNECTED;
2686 : } else {
2687 0 : RsQpMrRecvHandle(qpCb->connInfo->connfd, qpCb);
2688 0 : goto out;
2689 : }
2690 :
2691 1 : update_qp_cb:
2692 : #ifdef CUSTOM_INTERFACE
2693 1 : if (RsIsCustomInterfaceSupported()) {
2694 1 : ret = RsRoceQueryQpc(qpCb->ibQp, &qpAttrVal, qpAttrMask);
2695 1 : if (ret != 0) {
2696 1 : hccp_warn("qpn:%d query qpc unsuccessful, ret %d", qpCb->qpInfoLo.qpn, ret);
2697 : } else {
2698 0 : qpCb->udpSport = qpAttrVal.udp_sport;
2699 : }
2700 : }
2701 : #endif
2702 0 : out:
2703 1 : hccp_dbg("qp:%u, state:%d, udpSport:%u", qpn, qpCb->state, qpCb->udpSport);
2704 1 : qpInfo->status = qpCb->state;
2705 1 : qpInfo->udpSport = qpCb->udpSport;
2706 :
2707 1 : return 0;
2708 : }
2709 :
2710 3 : RS_ATTRI_VISI_DEF int RsGetQpContext(unsigned int phyId, unsigned int rdevIndex, unsigned int qpn, void **qp,
2711 : void **sendCq, void **recvCq)
2712 : {
2713 : int ret;
2714 3 : struct RsQpCb *qpCb = NULL;
2715 :
2716 3 : CHK_PRT_RETURN(phyId >= RS_MAX_DEV_NUM, hccp_err("phyId:%u >= [%d], is invalid", phyId, RS_MAX_DEV_NUM), -EINVAL);
2717 :
2718 2 : ret = RsQpn2qpcb(phyId, rdevIndex, qpn, &qpCb);
2719 2 : CHK_PRT_RETURN(ret, hccp_err("rs_qpn2qpcb failed ret[%d]", ret), ret);
2720 :
2721 1 : *qp = qpCb->ibQp;
2722 1 : *sendCq = qpCb->ibSendCq;
2723 1 : *recvCq = qpCb->ibRecvCq;
2724 :
2725 1 : hccp_dbg("qpn[%u] succ", qpn);
2726 :
2727 1 : return 0;
2728 : }
2729 :
2730 26 : int RsQueryRdevCb(unsigned int phyId, unsigned int rdevIndex, struct RsRdevCb **rdevCb)
2731 : {
2732 : int ret;
2733 : unsigned int chipId;
2734 26 : struct rs_cb *rsCb = NULL;
2735 :
2736 26 : RS_QP_PARA_CHECK(phyId);
2737 :
2738 24 : ret = rsGetLocalDevIDByHostDevID(phyId, &chipId);
2739 24 : CHK_PRT_RETURN(ret, hccp_err("rs_query_rdev_cb phyId[%u] invalid, ret:%d", phyId, ret), ret);
2740 :
2741 23 : ret = RsDev2rscb(chipId, &rsCb, false);
2742 23 : CHK_PRT_RETURN(ret, hccp_err("rs_query_rdev_cb get rs_cb failed, ret:%d", ret), -ENODEV);
2743 :
2744 23 : ret = RsGetRdevCb(rsCb, rdevIndex, rdevCb);
2745 23 : CHK_PRT_RETURN(ret, hccp_err("rs_get_rdev_cb failed! ret:%d, rdevIndex:%u", ret, rdevIndex), ret);
2746 :
2747 23 : return 0;
2748 : }
2749 :
2750 3 : RS_ATTRI_VISI_DEF int RsGetLbMax(unsigned int phyId, unsigned int rdevIndex, int *lbMax)
2751 : {
2752 3 : struct RsRdevCb *rdevCb = NULL;
2753 3 : int ret = 0;
2754 :
2755 3 : ret = RsQueryRdevCb(phyId, rdevIndex, &rdevCb);
2756 3 : CHK_PRT_RETURN(ret != 0, hccp_err("RsQueryRdevCb phyId:%u rdev_index:%u ret:%d", phyId, rdevIndex, ret), ret);
2757 :
2758 2 : return RsRoceGetQpNum(rdevCb->ibCtx, lbMax);
2759 : }
2760 :
2761 3 : STATIC int RsBuildUpQpcb(struct RsCqContext *cqContext, struct ibv_qp_init_attr *qpInitAttr, struct RsQpCb **qpCb)
2762 : {
2763 : int ret;
2764 :
2765 3 : ret = RsCallocQpcb(1, qpCb);
2766 3 : CHK_PRT_RETURN(ret, hccp_err("alloc mem for qp_cb failed, ret:%d errno:%d", ret, errno), -ENOMEM);
2767 :
2768 3 : ret = pthread_mutex_init(&(*qpCb)->qpMutex, NULL);
2769 3 : if (ret) {
2770 0 : hccp_err("pthread_mutex_init failed, ret %d", ret);
2771 0 : goto pthread_mutex_init_err;
2772 : }
2773 :
2774 3 : (*qpCb)->rdevCb = cqContext->rdevCb;
2775 3 : RS_INIT_LIST_HEAD(&(*qpCb)->mrList);
2776 3 : RS_INIT_LIST_HEAD(&(*qpCb)->remMrList);
2777 :
2778 3 : (*qpCb)->eqNum = cqContext->eqNum;
2779 3 : (*qpCb)->channel = cqContext->channel;
2780 3 : (*qpCb)->ibSendCq = cqContext->ibSendCq;
2781 3 : (*qpCb)->ibRecvCq = cqContext->ibRecvCq;
2782 3 : (*qpCb)->sendEvent = cqContext->sendEvent;
2783 3 : (*qpCb)->recvEvent = cqContext->recvEvent;
2784 3 : (*qpCb)->numRecvCqEvents = 0;
2785 3 : (*qpCb)->numSendCqEvents = 0;
2786 3 : (*qpCb)->srqContext = cqContext->srqContext;
2787 3 : (*qpCb)->state = RS_QP_STATUS_DISCONNECT;
2788 3 : (*qpCb)->ibPd = cqContext->rdevCb->ibPd;
2789 3 : (*qpCb)->txDepth = qpInitAttr->cap.max_send_wr;
2790 3 : (*qpCb)->rxDepth = qpInitAttr->cap.max_recv_wr;
2791 3 : (*qpCb)->qosAttr.tc = (RS_ROCE_DSCP_33 & RS_DSCP_MASK) << RS_DSCP_OFF;
2792 3 : (*qpCb)->qosAttr.sl = RS_ROCE_4_SL;
2793 3 : (*qpCb)->timeout = RS_QP_ATTR_TIMEOUT;
2794 3 : (*qpCb)->retryCnt = RS_QP_ATTR_RETRY_CNT;
2795 :
2796 3 : return 0;
2797 :
2798 0 : pthread_mutex_init_err:
2799 0 : free(*qpCb);
2800 0 : (*qpCb) = NULL;
2801 0 : return ret;
2802 : }
2803 :
2804 6 : RS_ATTRI_VISI_DEF int RsCreateCqEvent(struct RsCqContext *cqContext, struct CqAttr *attr)
2805 : {
2806 : int ret;
2807 6 : cqContext->channel = RsIbvCreateCompChannel(cqContext->rdevCb->ibCtx);
2808 :
2809 6 : if (cqContext->channel == NULL) {
2810 1 : hccp_err("ibv_create_comp_channel failed, ret %d, errno(%d)", -EINVAL, errno);
2811 1 : return -EINVAL;
2812 : }
2813 :
2814 5 : hccp_info("comp channel fd[%d].", cqContext->channel->fd);
2815 5 : ret = RsEpollCtl(cqContext->rdevCb->rsCb->connCb.epollfd, EPOLL_CTL_ADD, cqContext->channel->fd,
2816 : EPOLLIN | EPOLLRDHUP);
2817 : #ifndef CA_CONFIG_LLT
2818 : if (ret) {
2819 : hccp_err("add channel fd failed ret %d", ret);
2820 : goto rs_cq_epoll_ctl_err;
2821 : }
2822 : #endif
2823 :
2824 5 : ret = RsDrvCreateCqEvent(cqContext, attr);
2825 5 : if (ret) {
2826 0 : hccp_err("create drv cq event failed:%d", ret);
2827 0 : goto rs_cq_create_err;
2828 : }
2829 :
2830 5 : return ret;
2831 0 : rs_cq_create_err:
2832 0 : ret = RsEpollCtl(cqContext->rdevCb->rsCb->connCb.epollfd, EPOLL_CTL_DEL, cqContext->channel->fd,
2833 : EPOLLIN | EPOLLRDHUP);
2834 : #ifndef CA_CONFIG_LLT
2835 : if (ret) {
2836 : hccp_err("del channel fd failed ret %d", ret);
2837 : }
2838 : #endif
2839 0 : rs_cq_epoll_ctl_err:
2840 0 : if (cqContext->channel != NULL) {
2841 0 : RsIbvDestroyCompChannel(cqContext->channel);
2842 0 : cqContext->channel = NULL;
2843 : }
2844 0 : return ret;
2845 : }
2846 :
2847 5 : RS_ATTRI_VISI_DEF int RsCqCreate(unsigned int phyId, unsigned int rdevIndex, struct CqAttr *attr)
2848 : {
2849 : int ret;
2850 5 : struct RsRdevCb *rdevCb = NULL;
2851 5 : struct RsCqContext *cqContext = NULL;
2852 :
2853 5 : ret = RsQueryRdevCb(phyId, rdevIndex, &rdevCb);
2854 5 : if (ret) {
2855 0 : hccp_err("rs_query_rdev_cb phyId[%u] rdev_index[%u], ret %d", phyId, rdevIndex, ret);
2856 0 : return ret;
2857 : }
2858 :
2859 5 : cqContext = calloc(1, sizeof(struct RsCqContext));
2860 5 : if (cqContext == NULL) {
2861 0 : return -ENOMEM;
2862 : }
2863 5 : cqContext->rdevCb = rdevCb;
2864 5 : cqContext->eqNum = 0;
2865 5 : if (attr->sendChannel == NULL && attr->recvChannel == NULL) {
2866 3 : if (*attr->ibSendCq == NULL && *attr->ibRecvCq != NULL) {
2867 : // 只创建sq cq
2868 1 : cqContext->cqCreateMode = RS_SQ_CQ_CREATE;
2869 1 : cqContext->ibRecvCq = *attr->ibRecvCq;
2870 1 : cqContext->srqContext = attr->srqContext;
2871 : } else {
2872 : // 创建sq&rq cq
2873 2 : cqContext->cqCreateMode = RS_NORMAL_CQ_CREATE;
2874 : }
2875 3 : ret = RsCreateCqEvent(cqContext, attr);
2876 3 : if (ret) {
2877 0 : hccp_err("create cq event failed:%d", ret);
2878 0 : goto rs_cq_create_err;
2879 : }
2880 2 : } else if (attr->sendChannel != NULL && attr->recvChannel != NULL) {
2881 : // 使用输入comp channel创建sq&rq
2882 1 : ret = RsDrvCreateCqWithChannel(cqContext, attr);
2883 1 : if (ret) {
2884 0 : hccp_err("create drv cq with channel failed:%d", ret);
2885 0 : goto rs_cq_create_err;
2886 : }
2887 : } else {
2888 1 : hccp_err("rs create cq failed, sendChannel or recvChannel is NULL.");
2889 1 : ret = -EPERM;
2890 1 : goto rs_cq_create_err;
2891 : }
2892 :
2893 4 : *attr->qpContext = cqContext;
2894 4 : return 0;
2895 :
2896 1 : rs_cq_create_err:
2897 1 : free(cqContext);
2898 1 : cqContext = NULL;
2899 :
2900 1 : return ret;
2901 : }
2902 :
2903 0 : RS_ATTRI_VISI_DEF int RsTypicalCqCreate(unsigned int phyId, unsigned int rdevIndex, unsigned int cqDepth,
2904 : unsigned int *cqn)
2905 : {
2906 : struct RsTypicalCqEntry *entry;
2907 : struct RsTypicalCqEntry *tmp;
2908 : int ret;
2909 0 : struct RsRdevCb *rdevCb = NULL;
2910 0 : struct ibv_cq *ibCq = NULL;
2911 : struct rdma_lite_device_cq_attr deviceCqAttr;
2912 :
2913 0 : ret = RsQueryRdevCb(phyId, rdevIndex, &rdevCb);
2914 0 : if (ret) {
2915 0 : hccp_err("rs_query_rdev_cb phyId[%u] rdevIndex[%u], ret %d", phyId, rdevIndex, ret);
2916 0 : return ret;
2917 : }
2918 :
2919 0 : ret = RsDrvTypicalCqCreate(rdevCb, cqDepth, cqn, &ibCq, &deviceCqAttr);
2920 0 : if (ret) {
2921 0 : hccp_err("rs_drv_typical_cq_create failed, cqDepth[%u] ret[%d]", cqDepth, ret);
2922 0 : return ret;
2923 : }
2924 :
2925 0 : pthread_mutex_lock(&gRsTypicalCqMutex);
2926 0 : if (gRsTypicalCqList.next == NULL) {
2927 0 : RS_INIT_LIST_HEAD(&gRsTypicalCqList);
2928 : }
2929 :
2930 0 : RS_LIST_GET_HEAD_ENTRY(tmp, entry, &gRsTypicalCqList, list, struct RsTypicalCqEntry);
2931 0 : for (; &tmp->list != &gRsTypicalCqList;
2932 0 : tmp = entry, entry = list_entry(entry->list.next, struct RsTypicalCqEntry, list)) {
2933 0 : if (tmp->phyId == phyId && tmp->rdevIndex == rdevIndex && tmp->cqn == *cqn) {
2934 0 : tmp->ibCq = ibCq;
2935 0 : tmp->deviceCqAttr = deviceCqAttr;
2936 0 : pthread_mutex_unlock(&gRsTypicalCqMutex);
2937 0 : hccp_info("RsTypicalCqCreate updated: phyId[%u] rdevIndex[%u] cqn[%u] cqDepth[%u]", phyId, rdevIndex, *cqn,
2938 : cqDepth);
2939 0 : return 0;
2940 : }
2941 : }
2942 :
2943 0 : entry = calloc(1, sizeof(struct RsTypicalCqEntry));
2944 0 : if (entry == NULL) {
2945 0 : pthread_mutex_unlock(&gRsTypicalCqMutex);
2946 0 : hccp_err("RsTypicalCqCreate calloc failed, cqn[%u]", *cqn);
2947 0 : return -ENOMEM;
2948 : }
2949 0 : entry->phyId = phyId;
2950 0 : entry->rdevIndex = rdevIndex;
2951 0 : entry->cqn = *cqn;
2952 0 : entry->ibCq = ibCq;
2953 0 : entry->deviceCqAttr = deviceCqAttr;
2954 0 : RsListAddTail(&entry->list, &gRsTypicalCqList);
2955 0 : pthread_mutex_unlock(&gRsTypicalCqMutex);
2956 :
2957 0 : hccp_info("RsTypicalCqCreate success: phyId[%u] rdevIndex[%u] cqn[%u] cqDepth[%u]", phyId, rdevIndex, *cqn,
2958 : cqDepth);
2959 :
2960 0 : return 0;
2961 : }
2962 :
2963 0 : RS_ATTRI_VISI_DEF int RsTypicalCqDestroy(unsigned int phyId, unsigned int rdevIndex, unsigned int cqn)
2964 : {
2965 : struct RsTypicalCqEntry *entry;
2966 : struct RsTypicalCqEntry *tmp;
2967 : int ret;
2968 :
2969 0 : pthread_mutex_lock(&gRsTypicalCqMutex);
2970 0 : if (gRsTypicalCqList.next == NULL) {
2971 0 : pthread_mutex_unlock(&gRsTypicalCqMutex);
2972 0 : hccp_err("rs_typical_cq_destroy: cqn[%u] not found phyId[%u] rdevIndex[%u]", cqn, phyId, rdevIndex);
2973 0 : return -EINVAL;
2974 : }
2975 :
2976 0 : RS_LIST_GET_HEAD_ENTRY(tmp, entry, &gRsTypicalCqList, list, struct RsTypicalCqEntry);
2977 0 : for (; &tmp->list != &gRsTypicalCqList;
2978 0 : tmp = entry, entry = list_entry(entry->list.next, struct RsTypicalCqEntry, list)) {
2979 0 : if (tmp->phyId == phyId && tmp->rdevIndex == rdevIndex && tmp->cqn == cqn) {
2980 0 : RsListDel(&tmp->list);
2981 0 : pthread_mutex_unlock(&gRsTypicalCqMutex);
2982 :
2983 0 : if (tmp->ibCq != NULL) {
2984 0 : ret = RsIbvDestroyCq(tmp->ibCq);
2985 0 : if (ret) {
2986 0 : hccp_err("rs_ibv_destroy_cq failed cqn[%u] ret[%d]", cqn, ret);
2987 0 : free(tmp);
2988 0 : return ret;
2989 : }
2990 : }
2991 0 : free(tmp);
2992 0 : hccp_info("RsTypicalCqDestroy success: phyId[%u] rdevIndex[%u] cqn[%u]", phyId, rdevIndex, cqn);
2993 0 : return 0;
2994 : }
2995 : }
2996 0 : pthread_mutex_unlock(&gRsTypicalCqMutex);
2997 :
2998 0 : hccp_err("rs_typical_cq_destroy: cqn[%u] not found phyId[%u] rdevIndex[%u]", cqn, phyId, rdevIndex);
2999 0 : return -EINVAL;
3000 : }
3001 :
3002 0 : RS_ATTRI_VISI_DEF int RsGetLiteCqAttr(unsigned int phyId, unsigned int rdevIndex, unsigned int cqn,
3003 : struct rdma_lite_device_cq_attr *deviceCqAttr)
3004 : {
3005 : struct RsTypicalCqEntry *entry;
3006 : struct RsTypicalCqEntry *tmp;
3007 : int ret;
3008 :
3009 0 : RS_CHECK_POINTER_NULL_RETURN_INT(deviceCqAttr);
3010 :
3011 0 : pthread_mutex_lock(&gRsTypicalCqMutex);
3012 0 : if (gRsTypicalCqList.next == NULL) {
3013 0 : pthread_mutex_unlock(&gRsTypicalCqMutex);
3014 0 : hccp_err("rs_get_lite_cq_attr: cqn[%u] not found phyId[%u] rdevIndex[%u]", cqn, phyId, rdevIndex);
3015 0 : return -EINVAL;
3016 : }
3017 :
3018 0 : RS_LIST_GET_HEAD_ENTRY(tmp, entry, &gRsTypicalCqList, list, struct RsTypicalCqEntry);
3019 0 : for (; &tmp->list != &gRsTypicalCqList;
3020 0 : tmp = entry, entry = list_entry(entry->list.next, struct RsTypicalCqEntry, list)) {
3021 0 : if (tmp->phyId == phyId && tmp->rdevIndex == rdevIndex && tmp->cqn == cqn) {
3022 0 : ret = memcpy_s(deviceCqAttr, sizeof(*deviceCqAttr), &tmp->deviceCqAttr, sizeof(tmp->deviceCqAttr));
3023 0 : pthread_mutex_unlock(&gRsTypicalCqMutex);
3024 0 : if (ret) {
3025 0 : hccp_err("memcpy_s failed, ret:%d", ret);
3026 0 : return ret;
3027 : }
3028 0 : hccp_info("RsGetLiteCqAttr success: cqn[%u] depth[%u]", cqn, deviceCqAttr->depth);
3029 0 : return 0;
3030 : }
3031 : }
3032 0 : pthread_mutex_unlock(&gRsTypicalCqMutex);
3033 :
3034 0 : hccp_err("rs_get_lite_cq_attr: cqn[%u] not found phyId[%u] rdevIndex[%u]", cqn, phyId, rdevIndex);
3035 0 : return -EINVAL;
3036 : }
3037 :
3038 0 : RS_ATTRI_VISI_DEF int RsQpCreateWithCQWithAttrs(unsigned int phyId, unsigned int rdevIndex, unsigned int sendCqn,
3039 : unsigned int recvCqn, struct RsQpNormWithAttrs *qpNorm, struct RsQpRespWithAttrs *qpResp)
3040 : {
3041 : struct RsTypicalCqEntry *entry;
3042 : struct RsTypicalCqEntry *tmp;
3043 0 : struct RsRdevCb *rdevCb = NULL;
3044 0 : struct RsQpCb *qpCb = NULL;
3045 0 : struct ibv_cq *sendIbCq = NULL;
3046 0 : struct ibv_cq *recvIbCq = NULL;
3047 : struct rdma_lite_device_cq_attr sendDeviceCqAttr;
3048 : struct rdma_lite_device_cq_attr recvDeviceCqAttr;
3049 : int qpMode;
3050 : int ret;
3051 0 : bool sendFound = false;
3052 0 : bool recvFound = false;
3053 :
3054 0 : RS_QP_PARA_CHECK(phyId);
3055 0 : CHK_PRT_RETURN(qpResp == NULL, hccp_err("qp_resp is NULL!"), -EINVAL);
3056 :
3057 0 : ret = RsQpCheckQpNorm(qpNorm, &qpMode);
3058 0 : CHK_PRT_RETURN(ret != 0, hccp_err("check qp mode failed, ret:%d", ret), ret);
3059 :
3060 0 : ret = RsQpQueryInfo(phyId, rdevIndex, &rdevCb, qpMode);
3061 0 : CHK_PRT_RETURN(ret, hccp_err("query qp info failed:%d", ret), ret);
3062 :
3063 0 : pthread_mutex_lock(&gRsTypicalCqMutex);
3064 0 : if (gRsTypicalCqList.next != NULL) {
3065 0 : RS_LIST_GET_HEAD_ENTRY(tmp, entry, &gRsTypicalCqList, list, struct RsTypicalCqEntry);
3066 0 : for (; &tmp->list != &gRsTypicalCqList;
3067 0 : tmp = entry, entry = list_entry(entry->list.next, struct RsTypicalCqEntry, list)) {
3068 0 : if (tmp->phyId == phyId && tmp->rdevIndex == rdevIndex && tmp->cqn == sendCqn) {
3069 0 : sendIbCq = tmp->ibCq;
3070 0 : sendDeviceCqAttr = tmp->deviceCqAttr;
3071 0 : sendFound = true;
3072 0 : break;
3073 : }
3074 : }
3075 : }
3076 0 : pthread_mutex_unlock(&gRsTypicalCqMutex);
3077 0 : CHK_PRT_RETURN(!sendFound,
3078 : hccp_err("send cq not found: sendCqn[%u] phyId[%u] rdevIndex[%u]", sendCqn, phyId, rdevIndex), -EINVAL);
3079 :
3080 0 : pthread_mutex_lock(&gRsTypicalCqMutex);
3081 0 : if (gRsTypicalCqList.next != NULL) {
3082 0 : RS_LIST_GET_HEAD_ENTRY(tmp, entry, &gRsTypicalCqList, list, struct RsTypicalCqEntry);
3083 0 : for (; &tmp->list != &gRsTypicalCqList;
3084 0 : tmp = entry, entry = list_entry(entry->list.next, struct RsTypicalCqEntry, list)) {
3085 0 : if (tmp->phyId == phyId && tmp->rdevIndex == rdevIndex && tmp->cqn == recvCqn) {
3086 0 : recvIbCq = tmp->ibCq;
3087 0 : recvDeviceCqAttr = tmp->deviceCqAttr;
3088 0 : recvFound = true;
3089 0 : break;
3090 : }
3091 : }
3092 : }
3093 0 : pthread_mutex_unlock(&gRsTypicalCqMutex);
3094 0 : CHK_PRT_RETURN(!recvFound,
3095 : hccp_err("recv cq not found: recvCqn[%u] phyId[%u] rdevIndex[%u]", recvCqn, phyId, rdevIndex), -EINVAL);
3096 :
3097 0 : ret = RsCallocQpcb(1, &qpCb);
3098 0 : CHK_PRT_RETURN(ret, hccp_err("alloc mem for qp_cb failed, ret:%d errno:%d", ret, errno), -ENOMEM);
3099 :
3100 0 : ret = pthread_mutex_init(&qpCb->qpMutex, NULL);
3101 0 : if (ret) {
3102 0 : hccp_err("pthread_mutex_init failed, ret %d", ret);
3103 0 : goto qp_mutex_init_err;
3104 : }
3105 :
3106 0 : ret = pthread_mutex_init(&qpCb->cqeErrInfo.mutex, NULL);
3107 0 : if (ret) {
3108 0 : hccp_err("pthread_mutex_init failed, ret %d", ret);
3109 0 : goto cqe_mutex_init_err;
3110 : }
3111 :
3112 0 : ret = RsQpcbInitWithAttrs(rdevCb, qpCb, qpNorm);
3113 0 : if (ret) {
3114 0 : hccp_err("create qp tx rx failed ret %d", ret);
3115 0 : goto rs_qpcb_init_err;
3116 : }
3117 :
3118 0 : ret = RsInitMemPool(qpCb);
3119 0 : if (ret) {
3120 0 : hccp_err("init mem pool failed ret %d", ret);
3121 0 : goto rs_init_mem_err;
3122 : }
3123 :
3124 : // Assign pre-existing CQs (instead of RsDrvCreateCqWithAttrs)
3125 0 : qpCb->ibSendCq = sendIbCq;
3126 0 : qpCb->ibRecvCq = recvIbCq;
3127 0 : qpCb->qpResp.sendCqData = sendDeviceCqAttr;
3128 0 : qpCb->qpResp.recvCqData = recvDeviceCqAttr;
3129 0 : qpCb->sendCqDepth = sendDeviceCqAttr.depth;
3130 0 : qpCb->recvCqDepth = recvDeviceCqAttr.depth;
3131 :
3132 0 : ret = RsDrvQpCreateWithAttrs(qpCb, qpNorm);
3133 0 : if (ret) {
3134 0 : hccp_err("Create drv qp create failed:%d", ret);
3135 0 : goto create_qp_err;
3136 : }
3137 :
3138 0 : ret = ibv_req_notify_cq(qpCb->ibSendCq, 0);
3139 0 : if (ret) {
3140 0 : hccp_err("Can't request send CQ notification, ret:%d", ret);
3141 0 : ret = -EOPENSRC;
3142 0 : goto ret_noritfy_cq;
3143 : }
3144 :
3145 0 : ret = ibv_req_notify_cq(qpCb->ibRecvCq, 0);
3146 0 : if (ret) {
3147 0 : hccp_err("Can't request recv CQ notification, ret:%d", ret);
3148 0 : ret = -EOPENSRC;
3149 0 : goto ret_noritfy_cq;
3150 : }
3151 :
3152 0 : ret = RsQpNotifyMr(rdevCb, qpCb, &qpResp->qpn);
3153 0 : if (ret) {
3154 0 : hccp_err("Store qp notify mr failed:%d", ret);
3155 0 : goto ret_noritfy_cq;
3156 : }
3157 :
3158 0 : RsQpPrepareQpResp(qpNorm, qpCb, qpResp);
3159 :
3160 0 : return 0;
3161 :
3162 0 : ret_noritfy_cq:
3163 0 : RsDrvQpDestroy(qpCb);
3164 :
3165 0 : create_qp_err:
3166 : // Do NOT call RsDrvDestroyCq — CQs are not owned by this QP
3167 0 : RsDeinitMemPool(qpCb);
3168 0 : (void)RsQpcbDeinit(rdevCb, qpCb);
3169 :
3170 0 : rs_init_mem_err:
3171 0 : rs_qpcb_init_err:
3172 0 : pthread_mutex_destroy(&qpCb->cqeErrInfo.mutex);
3173 :
3174 0 : cqe_mutex_init_err:
3175 0 : pthread_mutex_destroy(&qpCb->qpMutex);
3176 :
3177 0 : qp_mutex_init_err:
3178 0 : free(qpCb);
3179 0 : qpCb = NULL;
3180 :
3181 0 : return ret;
3182 : }
3183 :
3184 7 : RS_ATTRI_VISI_DEF int RsCqDestroy(unsigned int phyId, unsigned int rdevIndex, struct CqAttr *attr)
3185 : {
3186 : int ret;
3187 7 : struct RsRdevCb *rdevCb = NULL;
3188 7 : struct RsCqContext *cqContext = NULL;
3189 :
3190 7 : ret = RsQueryRdevCb(phyId, rdevIndex, &rdevCb);
3191 7 : CHK_PRT_RETURN(ret, hccp_err("rs_query_rdev_cb phyId[%u] rdev_index[%u], ret %d", phyId, rdevIndex, ret), ret);
3192 :
3193 7 : cqContext = *attr->qpContext;
3194 :
3195 7 : ret = RsDrvDestroyCqEvent(cqContext);
3196 7 : if (ret) {
3197 0 : hccp_err("rs_drv_destroy_cq_event failed ret %d", ret);
3198 : }
3199 :
3200 7 : if (cqContext->channel != NULL) {
3201 5 : ret = RsEpollCtl(rdevCb->rsCb->connCb.epollfd, EPOLL_CTL_DEL, cqContext->channel->fd, EPOLLIN | EPOLLRDHUP);
3202 : #ifndef CA_CONFIG_LLT
3203 : if (ret) {
3204 : hccp_err("del channel fd failed ret %d", ret);
3205 : }
3206 : #endif
3207 5 : RsIbvDestroyCompChannel(cqContext->channel);
3208 5 : cqContext->channel = NULL;
3209 : }
3210 :
3211 7 : free(cqContext);
3212 7 : cqContext = NULL;
3213 :
3214 7 : return ret;
3215 : }
3216 :
3217 4 : RS_ATTRI_VISI_DEF int RsNormalQpCreate(unsigned int phyId, unsigned int rdevIndex, struct ibv_qp_init_attr *qpInitAttr,
3218 : struct RsQpResp *qpResp, void **qp)
3219 : {
3220 4 : struct RsCqContext *cqContext = NULL;
3221 4 : struct RsRdevCb *rdevCb = NULL;
3222 4 : struct RsQpCb *qpCb = NULL;
3223 : int ret;
3224 :
3225 4 : CHK_PRT_RETURN(qpResp == NULL, hccp_err("qp_resp is NULL!"), -EINVAL);
3226 4 : ret = RsQueryRdevCb(phyId, rdevIndex, &rdevCb);
3227 4 : CHK_PRT_RETURN(ret != 0, hccp_err("rs_query_rdev_cb phyId[%u] rdev_index[%u], ret %d", phyId, rdevIndex, ret), ret);
3228 :
3229 4 : CHK_PRT_RETURN(qpInitAttr == NULL, hccp_err("qp_init_attr is NULL!"), -EINVAL);
3230 :
3231 4 : cqContext = qpInitAttr->qp_context;
3232 4 : CHK_PRT_RETURN(cqContext == NULL, hccp_err("cq_context is NULL!"), -EINVAL);
3233 3 : CHK_PRT_RETURN(rdevCb != cqContext->rdevCb,
3234 : hccp_err("rs_query_rdev_cb phyId[%u] rdev_index[%u],"
3235 : "rdevCb is invalid.",
3236 : phyId, rdevIndex),
3237 : -EINVAL);
3238 :
3239 3 : ret = RsBuildUpQpcb(cqContext, qpInitAttr, &qpCb);
3240 3 : CHK_PRT_RETURN(ret, hccp_err("rs_build_up_qpcb failed, ret:%d", ret), ret);
3241 :
3242 3 : ret = RsDrvNormalQpCreate(qpCb, qpInitAttr);
3243 3 : if (ret) {
3244 1 : hccp_err("create drv qp create failed:%d", ret);
3245 1 : goto create_qp_err;
3246 : }
3247 :
3248 2 : RS_PTHREAD_MUTEX_LOCK(&rdevCb->rdevMutex);
3249 2 : RsListAddTail(&qpCb->list, &rdevCb->qpList);
3250 2 : RS_PTHREAD_MUTEX_ULOCK(&rdevCb->rdevMutex);
3251 2 : rdevCb->qpCnt++;
3252 2 : *qp = qpCb->ibQp;
3253 2 : qpResp->qpn = (unsigned int)qpCb->qpInfoLo.qpn;
3254 2 : qpResp->gidIdx = (unsigned int)qpCb->qpInfoLo.gidIdx;
3255 2 : qpResp->psn = (unsigned int)qpCb->qpInfoLo.psn;
3256 2 : qpResp->gid = qpCb->qpInfoLo.gid;
3257 :
3258 2 : hccp_info("qp %d create qp.", qpResp->qpn);
3259 :
3260 2 : return 0;
3261 :
3262 1 : create_qp_err:
3263 1 : pthread_mutex_destroy(&qpCb->qpMutex);
3264 1 : free(qpCb);
3265 1 : qpCb = NULL;
3266 1 : return ret;
3267 : }
3268 :
3269 2 : RS_ATTRI_VISI_DEF int RsNormalQpDestroy(unsigned int phyId, unsigned int rdevIndex, unsigned int qpn)
3270 : {
3271 2 : struct RsQpCb *qpCb = NULL;
3272 : int ret;
3273 :
3274 2 : RS_QP_PARA_CHECK(phyId);
3275 2 : ret = RsQpn2qpcb(phyId, rdevIndex, qpn, &qpCb);
3276 2 : CHK_PRT_RETURN(ret != 0 || qpCb == NULL, hccp_err("get qp cb failed qpn %u, ret %d", qpn, ret), ret);
3277 :
3278 2 : RsQpRelease(qpCb);
3279 :
3280 : // destroy qp
3281 2 : RsDrvQpDestroy(qpCb);
3282 :
3283 2 : qpCb->rdevCb->qpCnt--;
3284 :
3285 2 : pthread_mutex_destroy(&qpCb->qpMutex);
3286 2 : hccp_info("qp %d destroy qp, send wr[%u].", qpn, qpCb->sendWrNum);
3287 :
3288 2 : free(qpCb);
3289 2 : qpCb = NULL;
3290 2 : return ret;
3291 : }
3292 :
3293 4 : RS_ATTRI_VISI_DEF int RsCreateCompChannel(unsigned int phyId, unsigned int rdevIndex, void **compChannel)
3294 : {
3295 : int ret;
3296 : unsigned int chipId;
3297 :
3298 4 : struct RsRdevCb *rdevCb = NULL;
3299 :
3300 4 : CHK_PRT_RETURN(compChannel == NULL || phyId >= RS_MAX_DEV_NUM,
3301 : hccp_err("param err, NULL pointer or phyId:%u >= [%d]", phyId, RS_MAX_DEV_NUM), -EINVAL);
3302 :
3303 4 : ret = rsGetLocalDevIDByHostDevID(phyId, &chipId);
3304 4 : CHK_PRT_RETURN(ret,
3305 : hccp_err("rs_create_comp_channel rsGetLocalDevIDByHostDevID phyId[%u] invalid, ret %d", phyId, ret), ret);
3306 :
3307 3 : ret = RsRdev2rdevCb(chipId, rdevIndex, &rdevCb);
3308 3 : CHK_PRT_RETURN(ret != 0 || rdevCb == NULL, hccp_err("rs_rdev2rdev_cb for chip_id[%u] failed, ret %d", chipId, ret),
3309 : ret);
3310 :
3311 2 : *compChannel = (void *)RsIbvCreateCompChannel(rdevCb->ibCtx);
3312 2 : if (*compChannel == NULL) {
3313 1 : hccp_err("rs_ibv_create_comp_channel failed, errno(%d)", errno);
3314 1 : return -EOPENSRC;
3315 : }
3316 1 : hccp_info("create comp channel success!");
3317 1 : return 0;
3318 : }
3319 :
3320 2 : RS_ATTRI_VISI_DEF int RsDestroyCompChannel(void *compChannel)
3321 : {
3322 : int ret;
3323 2 : struct ibv_comp_channel *rsCompChannel = (struct ibv_comp_channel *)compChannel;
3324 :
3325 2 : ret = RsIbvDestroyCompChannel(rsCompChannel);
3326 2 : CHK_PRT_RETURN(ret, hccp_err("rs_destroy_comp_channel failed."), ret);
3327 1 : hccp_info("destroy comp channel success!");
3328 :
3329 1 : return 0;
3330 : }
3331 :
3332 5 : RS_ATTRI_VISI_DEF int RsCreateSrq(unsigned int phyId, unsigned int rdevIndex, struct SrqAttr *attr)
3333 : {
3334 : int ret;
3335 5 : struct RsRdevCb *rdevCb = NULL;
3336 5 : struct RsCqContext *cqContext = NULL;
3337 :
3338 5 : CHK_PRT_RETURN(attr == NULL || attr->context == NULL || attr->ibRecvCq == NULL || attr->ibSrq == NULL ||
3339 : phyId >= RS_MAX_DEV_NUM,
3340 : hccp_err("param err, NULL pointer or phyId:%u >= [%d]", phyId, RS_MAX_DEV_NUM), -EINVAL);
3341 :
3342 5 : ret = RsQueryRdevCb(phyId, rdevIndex, &rdevCb);
3343 5 : CHK_PRT_RETURN(ret, hccp_err("rs_query_rdev_cb phyId[%u] rdev_index[%u], ret %d", phyId, rdevIndex, ret), ret);
3344 :
3345 4 : cqContext = calloc(1, sizeof(struct RsCqContext));
3346 4 : if (cqContext == NULL) {
3347 1 : return -ENOMEM;
3348 : }
3349 :
3350 3 : cqContext->rdevCb = rdevCb;
3351 3 : cqContext->eqNum = 0;
3352 3 : cqContext->cqCreateMode = RS_SRQ_CQ_CREATE;
3353 3 : *attr->context = cqContext;
3354 :
3355 3 : struct CqAttr cqAttr = {0};
3356 3 : cqAttr.recvCqDepth = attr->cqDepth;
3357 3 : cqAttr.recvCqEventId = attr->srqEventId;
3358 3 : cqAttr.ibRecvCq = attr->ibRecvCq;
3359 : // 创建srq cq
3360 3 : ret = RsCreateCqEvent(cqContext, &cqAttr);
3361 3 : if (ret) {
3362 1 : hccp_err("rs_create_cq_event create cq failed! ret:%d", ret);
3363 1 : goto create_cq_event_err;
3364 : }
3365 2 : cqContext->ibSrqCq = *attr->ibRecvCq;
3366 :
3367 2 : struct ibv_srq_init_attr srqInitAttr = {.attr = {.max_wr = attr->srqDepth, .max_sge = attr->maxSge}};
3368 2 : hccp_info("max_wr [%u], max_sge[%u]", srqInitAttr.attr.max_wr, srqInitAttr.attr.max_sge);
3369 :
3370 : // 创建srq
3371 2 : *attr->ibSrq = RsIbvCreateSrq(rdevCb->ibPd, &srqInitAttr);
3372 2 : if (*attr->ibSrq == NULL) {
3373 1 : hccp_err("rs_ibv_create_srq failed.");
3374 1 : ret = -EOPENSRC;
3375 1 : goto create_srq_err;
3376 : }
3377 1 : hccp_info("create srq success!");
3378 :
3379 1 : return 0;
3380 1 : create_cq_event_err:
3381 2 : create_srq_err:
3382 2 : cqAttr.qpContext = attr->context;
3383 2 : RsCqDestroy(phyId, rdevIndex, &cqAttr);
3384 :
3385 2 : return ret;
3386 : }
3387 :
3388 1 : RS_ATTRI_VISI_DEF int RsDestroySrq(unsigned int phyId, unsigned int rdevIndex, struct SrqAttr *attr)
3389 : {
3390 : int ret;
3391 :
3392 1 : CHK_PRT_RETURN(*attr->context == NULL || *attr->ibSrq == NULL || phyId >= RS_MAX_DEV_NUM,
3393 : hccp_err("param err, NULL pointer or phyId:%u >= [%d]", phyId, RS_MAX_DEV_NUM), -EINVAL);
3394 :
3395 1 : struct CqAttr cqAttr = {0};
3396 1 : struct RsCqContext *cqContext = *attr->context;
3397 1 : cqAttr.qpContext = attr->context;
3398 1 : RsIbvAckCqEvents(cqContext->ibSrqCq, cqContext->numRecvCqEvents);
3399 :
3400 : // 销毁srq cq
3401 1 : ret = RsCqDestroy(phyId, rdevIndex, &cqAttr);
3402 1 : CHK_PRT_RETURN(ret, hccp_err("rs_cq_destroy destroy cq failed! ret:%d", ret), ret);
3403 :
3404 1 : ret = RsIbvDestroySrq(*attr->ibSrq);
3405 1 : CHK_PRT_RETURN(ret, hccp_err("rs_ibv_destroy_srq failed."), ret);
3406 :
3407 1 : return 0;
3408 : }
3409 :
3410 2 : RS_ATTRI_VISI_DEF int RsGetLiteSupport(unsigned int phyId, unsigned int rdevIndex, int *supportLite)
3411 : {
3412 : int ret;
3413 : unsigned int chipId;
3414 2 : struct RsRdevCb *rdevCb = NULL;
3415 :
3416 2 : RS_CHECK_POINTER_NULL_RETURN_INT(supportLite);
3417 :
3418 2 : CHK_PRT_RETURN(phyId >= RS_MAX_DEV_NUM, hccp_err("rs set param error ! phyId:%u", phyId), -EINVAL);
3419 2 : ret = rsGetLocalDevIDByHostDevID(phyId, &chipId);
3420 2 : CHK_PRT_RETURN(ret, hccp_err("phyId[%u] invalid, ret %d", phyId, ret), ret);
3421 :
3422 2 : ret = RsRdev2rdevCb(chipId, rdevIndex, &rdevCb);
3423 2 : CHK_PRT_RETURN(ret != 0 || rdevCb == NULL, hccp_err("rs_rdev2rdev_cb for chip_id[%u] failed, ret %d", chipId, ret),
3424 : ret);
3425 :
3426 2 : rdevCb->supportLite = 1;
3427 2 : *supportLite = rdevCb->supportLite;
3428 :
3429 2 : return 0;
3430 : }
3431 :
3432 2 : RS_ATTRI_VISI_DEF int RsGetLiteRdevCap(unsigned int phyId, unsigned int rdevIndex, struct LiteRdevCapResp *resp)
3433 : {
3434 : int ret;
3435 : unsigned int chipId;
3436 2 : struct RsRdevCb *rdevCb = NULL;
3437 :
3438 2 : RS_CHECK_POINTER_NULL_RETURN_INT(resp);
3439 :
3440 2 : CHK_PRT_RETURN(phyId >= RS_MAX_DEV_NUM, hccp_err("rs set param error ! phyId:%u", phyId), -EINVAL);
3441 2 : ret = rsGetLocalDevIDByHostDevID(phyId, &chipId);
3442 2 : CHK_PRT_RETURN(ret, hccp_err("phyId[%u] invalid, ret %d", phyId, ret), ret);
3443 :
3444 2 : ret = RsRdev2rdevCb(chipId, rdevIndex, &rdevCb);
3445 2 : CHK_PRT_RETURN(ret != 0 || rdevCb == NULL, hccp_err("rs_rdev2rdev_cb for chip_id[%u] failed, ret %d", chipId, ret),
3446 : ret);
3447 :
3448 2 : ret = RsIbvExpQueryDevice(rdevCb->ibCtx, &resp->cap);
3449 2 : CHK_PRT_RETURN(ret != 0, hccp_err("rs_ibv_exp_query_device for phyId[%u] failed, ret %d", phyId, ret), ret);
3450 :
3451 2 : ret = memcpy_s(resp, sizeof(struct dev_cap_info), (void *)&resp->cap, sizeof(resp->cap));
3452 2 : if (ret) {
3453 0 : hccp_err("memcpy_s failed, ret:%d, src_len:%u, dst_len:%u", ret, (unsigned int)sizeof(resp->cap),
3454 : (unsigned int)sizeof(struct dev_cap_info));
3455 0 : return ret;
3456 : }
3457 :
3458 2 : return 0;
3459 : }
3460 :
3461 2 : RS_ATTRI_VISI_DEF int RsGetLiteQpCqAttr(unsigned int phyId, unsigned int rdevIndex, unsigned int qpn,
3462 : struct LiteQpCqAttrResp *resp)
3463 : {
3464 : int ret;
3465 2 : struct RsQpCb *qpCb = NULL;
3466 :
3467 2 : RS_CHECK_POINTER_NULL_RETURN_INT(resp);
3468 :
3469 2 : RS_QP_PARA_CHECK(phyId);
3470 2 : ret = RsQpn2qpcb(phyId, rdevIndex, qpn, &qpCb);
3471 2 : CHK_PRT_RETURN(ret != 0 || qpCb == NULL, hccp_err("get qp cb failed qpn %u, ret %d", qpn, ret), ret);
3472 :
3473 2 : ret = memcpy_s(resp, sizeof(struct LiteQpCqAttrResp), (void *)&qpCb->qpResp, sizeof(qpCb->qpResp));
3474 2 : if (ret) {
3475 0 : hccp_err("memcpy_s failed, ret:%d, src_len:%u, dst_len:%u", ret, (unsigned int)sizeof(qpCb->qpResp),
3476 : (unsigned int)sizeof(struct LiteQpCqAttrResp));
3477 0 : return ret;
3478 : }
3479 :
3480 2 : return 0;
3481 : }
3482 :
3483 0 : RS_ATTRI_VISI_DEF int RsGetLiteQpAttr(unsigned int phyId, unsigned int rdevIndex, unsigned int qpn,
3484 : struct LiteQpAttrResp *resp)
3485 : {
3486 : int ret;
3487 0 : struct RsQpCb *qpCb = NULL;
3488 :
3489 0 : RS_CHECK_POINTER_NULL_RETURN_INT(resp);
3490 :
3491 0 : RS_QP_PARA_CHECK(phyId);
3492 0 : ret = RsQpn2qpcb(phyId, rdevIndex, qpn, &qpCb);
3493 0 : CHK_PRT_RETURN(ret != 0 || qpCb == NULL, hccp_err("get qp cb failed qpn %u, ret %d", qpn, ret), ret);
3494 :
3495 0 : ret = memcpy_s(resp, sizeof(struct LiteQpAttrResp), (void *)&qpCb->qpResp.qpData, sizeof(qpCb->qpResp.qpData));
3496 0 : if (ret) {
3497 0 : hccp_err("memcpy_s failed, ret:%d, src_len:%u, dst_len:%u", ret, (unsigned int)sizeof(qpCb->qpResp.qpData),
3498 : (unsigned int)sizeof(struct LiteQpAttrResp));
3499 0 : return ret;
3500 : }
3501 :
3502 0 : return 0;
3503 : }
3504 :
3505 2 : RS_ATTRI_VISI_DEF int RsGetLiteMemAttr(unsigned int phyId, unsigned int rdevIndex, unsigned int qpn,
3506 : struct LiteMemAttrResp *resp)
3507 : {
3508 : int ret;
3509 2 : struct RsQpCb *qpCb = NULL;
3510 :
3511 2 : RS_CHECK_POINTER_NULL_RETURN_INT(resp);
3512 :
3513 2 : RS_QP_PARA_CHECK(phyId);
3514 2 : ret = RsQpn2qpcb(phyId, rdevIndex, qpn, &qpCb);
3515 2 : CHK_PRT_RETURN(ret != 0 || qpCb == NULL, hccp_err("get qp cb failed qpn %u, ret %d", qpn, ret), ret);
3516 :
3517 2 : ret = memcpy_s(resp, sizeof(struct LiteMemAttrResp), (void *)&qpCb->memResp, sizeof(qpCb->memResp));
3518 2 : if (ret) {
3519 0 : hccp_err("memcpy_s failed, ret:%d, src_len:%u, dst_len:%u", ret, (unsigned int)sizeof(qpCb->memResp),
3520 : (unsigned int)sizeof(struct LiteMemAttrResp));
3521 0 : return ret;
3522 : }
3523 :
3524 2 : return 0;
3525 : }
3526 :
3527 4 : STATIC void RsGetMrInfo(struct RsQpCb *qpCb, struct LiteMrInfo *mr, uint32_t maxMrNum, struct RsListHead *mrList)
3528 : {
3529 4 : struct RsMrCb *mrTmp = NULL;
3530 4 : struct RsMrCb *mrTmp2 = NULL;
3531 4 : uint32_t i = 0;
3532 :
3533 4 : RS_PTHREAD_MUTEX_LOCK(&qpCb->qpMutex);
3534 4 : RS_LIST_GET_HEAD_ENTRY(mrTmp, mrTmp2, mrList, list, struct RsMrCb);
3535 6 : for (; (&mrTmp->list) != mrList; mrTmp = mrTmp2, mrTmp2 = list_entry(mrTmp2->list.next, struct RsMrCb, list)) {
3536 2 : if (i < maxMrNum) {
3537 2 : mr[i].key = mrTmp->mrInfo.rkey;
3538 2 : mr[i].addr = mrTmp->mrInfo.addr;
3539 2 : mr[i].len = mrTmp->mrInfo.len;
3540 2 : i++;
3541 : } else {
3542 0 : break;
3543 : }
3544 : }
3545 :
3546 4 : RS_PTHREAD_MUTEX_ULOCK(&qpCb->qpMutex);
3547 4 : }
3548 :
3549 2 : RS_ATTRI_VISI_DEF int RsGetLiteConnectedInfo(unsigned int phyId, unsigned int rdevIndex, unsigned int qpn,
3550 : struct LiteConnectedInfoResp *resp)
3551 : {
3552 : int ret;
3553 2 : struct RsQpCb *qpCb = NULL;
3554 :
3555 2 : RS_CHECK_POINTER_NULL_RETURN_INT(resp);
3556 2 : RS_QP_PARA_CHECK(phyId);
3557 2 : ret = RsQpn2qpcb(phyId, rdevIndex, qpn, &qpCb);
3558 2 : CHK_PRT_RETURN(ret != 0 || qpCb == NULL, hccp_err("get qp cb failed qpn %u, ret %d", qpn, ret), ret);
3559 :
3560 2 : resp->state = (unsigned int)qpCb->state;
3561 2 : if (resp->state == RS_QP_STATUS_CONNECTED) {
3562 2 : RsGetMrInfo(qpCb, &resp->localMr[0], RA_MR_MAX_NUM, &qpCb->mrList);
3563 2 : RsGetMrInfo(qpCb, &resp->remMr[0], RA_MR_MAX_NUM, &qpCb->remMrList);
3564 2 : resp->qosAttr.sl = qpCb->qosAttr.sl;
3565 2 : resp->qosAttr.tc = qpCb->qosAttr.tc;
3566 : }
3567 :
3568 2 : return 0;
3569 : }
|
