Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #define _GNU_SOURCE
12 : #include <unistd.h>
13 : #include <stdlib.h>
14 : #include <netinet/in.h>
15 : #include <arpa/inet.h>
16 : #include <dlfcn.h>
17 : #include <sys/types.h>
18 : #include <sys/stat.h>
19 : #include <sys/epoll.h>
20 : #include <sys/eventfd.h>
21 : #include <sys/socket.h>
22 : #include <errno.h>
23 : #include "securec.h"
24 : #include "rs.h"
25 : #include "ra_rs_err.h"
26 : #include "rs_common_inner.h"
27 : #include "rs_inner.h"
28 : #include "rs_rdma_inner.h"
29 : #include "rs_epoll.h"
30 : #include "dl_hal_function.h"
31 : #include "dl_ibverbs_function.h"
32 : #include "dl_ibv_extend_function.h"
33 : #include "rs_drv_socket.h"
34 : #include "rs_drv_rdma.h"
35 : #include "rs_rdma.h"
36 :
37 : unsigned int gRsSendWrNum = 0;
38 :
39 : STATIC struct RsListHead gRsTypicalCqList;
40 : STATIC pthread_mutex_t gRsTypicalCqMutex = PTHREAD_MUTEX_INITIALIZER;
41 :
42 2 : STATIC void RsBufPrint(char *addr, int len)
43 : {
44 : int i;
45 :
46 130 : for (i = 0; i < len; i++) {
47 128 : hccp_info("0x%02x ", *(addr + i));
48 : }
49 2 : }
50 :
51 137 : STATIC int RsGetQpcb(struct RsRdevCb *rdevCb, uint32_t qpn, struct RsQpCb **qpCb)
52 : {
53 137 : struct RsQpCb *qpCbTmp = NULL;
54 137 : struct RsQpCb *qpCbTmp2 = NULL;
55 :
56 137 : RS_LIST_GET_HEAD_ENTRY(qpCbTmp, qpCbTmp2, &rdevCb->qpList, list, struct RsQpCb);
57 180 : for (; (&qpCbTmp->list) != &rdevCb->qpList;
58 43 : qpCbTmp = qpCbTmp2, qpCbTmp2 = list_entry(qpCbTmp2->list.next, struct RsQpCb, list)) {
59 171 : if (qpCbTmp->ibQp->qp_num == qpn) {
60 128 : *qpCb = qpCbTmp;
61 128 : return 0;
62 : }
63 : }
64 :
65 9 : *qpCb = NULL;
66 9 : hccp_err("qp_cb for qp %u do not available!", qpn);
67 :
68 9 : return -ENODEV;
69 : }
70 :
71 140 : int RsQpn2qpcb(unsigned int phyId, unsigned int rdevIndex, uint32_t qpn, struct RsQpCb **qpCb)
72 : {
73 : int ret;
74 : unsigned int chipId;
75 140 : struct rs_cb *rsCb = NULL;
76 140 : struct RsRdevCb *rdevCb = NULL;
77 :
78 140 : CHK_PRT_RETURN(phyId >= RS_MAX_DEV_NUM, hccp_err("rs set param error! phyId:%u", phyId), -EINVAL);
79 :
80 137 : ret = rsGetLocalDevIDByHostDevID(phyId, &chipId);
81 137 : CHK_PRT_RETURN(ret, hccp_err("rs_qpn2qpcb rsGetLocalDevIDByHostDevID phyId[%u] invalid, ret:%d", phyId, ret), ret);
82 :
83 137 : ret = RsDev2rscb(chipId, &rsCb, false);
84 137 : CHK_PRT_RETURN(ret, hccp_err("rs_qpn2qpcb get rs_cb failed, ret:%d", ret), -ENODEV);
85 :
86 137 : ret = RsGetRdevCb(rsCb, rdevIndex, &rdevCb);
87 137 : CHK_PRT_RETURN(ret, hccp_err("rs_get_rdev_cb failed! ret:%d, rdevIndex:%u", ret, rdevIndex), ret);
88 :
89 137 : ret = RsGetQpcb(rdevCb, qpn, qpCb);
90 137 : CHK_PRT_RETURN(ret, hccp_err("rs_get_qpcb failed! ret:%d, qpn:%u", ret, qpn), ret);
91 :
92 128 : return 0;
93 : }
94 :
95 44 : STATIC int RsGetMrcb(struct RsQpCb *qpCb, uint64_t addr, struct RsMrCb **mrCb, struct RsListHead *mrList)
96 : {
97 44 : struct RsMrCb *mrTmp = NULL;
98 44 : struct RsMrCb *mrTmp2 = NULL;
99 :
100 44 : RS_PTHREAD_MUTEX_LOCK(&qpCb->qpMutex);
101 44 : RS_LIST_GET_HEAD_ENTRY(mrTmp, mrTmp2, mrList, list, struct RsMrCb);
102 86 : for (; (&mrTmp->list) != mrList; mrTmp = mrTmp2, mrTmp2 = list_entry(mrTmp2->list.next, struct RsMrCb, list)) {
103 66 : if ((mrTmp->mrInfo.addr <= addr) && (addr < mrTmp->mrInfo.addr + mrTmp->mrInfo.len)) {
104 24 : *mrCb = mrTmp;
105 24 : RS_PTHREAD_MUTEX_ULOCK(&qpCb->qpMutex);
106 24 : return 0;
107 : }
108 : }
109 :
110 20 : *mrCb = NULL;
111 20 : RS_PTHREAD_MUTEX_ULOCK(&qpCb->qpMutex);
112 :
113 20 : hccp_info("cannot find mrcb for addr@0x%lx !", addr);
114 :
115 20 : return -ENODEV;
116 : }
117 :
118 18 : STATIC void *RsNotifyMrListAdd(struct RsQpCb *qpCb, const char *buf)
119 : {
120 : int ret;
121 : struct RsMrCb *notifyMrCb;
122 :
123 18 : notifyMrCb = calloc(1, sizeof(struct RsMrCb));
124 18 : CHK_PRT_RETURN(notifyMrCb == NULL, hccp_err("notify_mr_cb calloc failed"), NULL);
125 16 : ret = memcpy_s(¬ifyMrCb->mrInfo, sizeof(struct RsMrInfo), &((const struct RsQpInfo *)buf)->notifyMr,
126 : sizeof(struct RsMrInfo));
127 16 : if (ret) {
128 1 : hccp_err("memcpy_s failed, ret:%d, src_len:%u, dst_len:%u", ret, sizeof(struct RsMrInfo),
129 : sizeof(struct RsMrInfo));
130 1 : free(notifyMrCb);
131 1 : notifyMrCb = NULL;
132 1 : return NULL;
133 : }
134 :
135 15 : hccp_info("qpn is %d, rdevIndex:%u, chipId %u, recv notify va is 0x%llx, notify size is %llu", qpCb->qpInfoLo.qpn,
136 : qpCb->rdevCb->rdevIndex, qpCb->rdevCb->rsCb->chipId, notifyMrCb->mrInfo.addr, notifyMrCb->mrInfo.len);
137 :
138 15 : RsListAddTail(¬ifyMrCb->list, &qpCb->remMrList);
139 :
140 15 : return notifyMrCb;
141 : }
142 :
143 21 : STATIC int RsQpStateModify(struct RsQpCb *qpCb)
144 : {
145 21 : struct ibv_qp_init_attr initAttr = {0};
146 21 : struct ibv_qp_attr attr = {0};
147 : enum ibv_qp_state state;
148 : int ret;
149 :
150 : // see ib_modify_qp_is_ok for status modify, only support modify qp from INIT to RTR
151 21 : ret = RsIbvQueryQp(qpCb->ibQp, &attr, IBV_QP_STATE, &initAttr);
152 21 : if (ret != 0) {
153 1 : hccp_warn("rs_ibv_query_qp qpn:%d unsuccessful, ret:%d", qpCb->qpInfoLo.qpn, ret);
154 1 : state = IBV_QPS_UNKNOWN;
155 : } else {
156 20 : state = attr.qp_state;
157 : }
158 :
159 : // disallow modify qp from IBV_QPS_RTS to IBV_QPS_RTS
160 21 : if (state == IBV_QPS_RTS) {
161 1 : hccp_err("qpn:%d disallow modify from %d", qpCb->qpInfoLo.qpn, state);
162 1 : return -EINVAL;
163 : }
164 :
165 20 : hccp_info("qpn:%d state:%d start modify", qpCb->qpInfoLo.qpn, state);
166 :
167 : // modify qp from others to RESET
168 20 : if (state != IBV_QPS_RESET && state != IBV_QPS_INIT && state != IBV_QPS_RTR) {
169 1 : ret = RsDrvQpStateModifytoReset(qpCb);
170 1 : CHK_PRT_RETURN(ret, hccp_err("qpn:%d modify %d to reset failed, ret:%d", qpCb->qpInfoLo.qpn, state, ret), ret);
171 0 : state = IBV_QPS_RESET;
172 : }
173 :
174 : // modify qp from RESET to INIT
175 19 : if (state == IBV_QPS_RESET) {
176 19 : ret = RsDrvQpStateModifytoInit(qpCb, &attr);
177 19 : CHK_PRT_RETURN(ret, hccp_err("qpn:%d modify %d to init failed, ret %d", qpCb->qpInfoLo.qpn, state, ret), ret);
178 18 : state = IBV_QPS_INIT;
179 : }
180 :
181 : // modify qp from INIT to RTR
182 18 : if (state == IBV_QPS_INIT) {
183 18 : ret = RsDrvQpStateModifytoRtr(qpCb, &attr);
184 18 : CHK_PRT_RETURN(ret, hccp_err("qpn:%d modify %d to rtr failed, ret %d", qpCb->qpInfoLo.qpn, state, ret), ret);
185 16 : state = IBV_QPS_RTR;
186 : }
187 :
188 : // modify qp from RTR to RTS
189 16 : if (state == IBV_QPS_RTR) {
190 16 : ret = RsDrvQpStateModifytoRts(qpCb, &attr);
191 16 : CHK_PRT_RETURN(ret, hccp_err("qpn:%d modify %d to rts failed, ret %d", qpCb->qpInfoLo.qpn, state, ret), ret);
192 : }
193 :
194 16 : hccp_info("local qpn[%d] remote qpn[%d] modify succ", qpCb->qpInfoLo.qpn, qpCb->qpInfoRem.qpn);
195 :
196 16 : return 0;
197 : }
198 :
199 17 : STATIC int RsEpollRecvQpHandle(struct RsQpCb *qpCb, const char *bufTmp)
200 : {
201 : int ret;
202 17 : float timeCost = 0.0;
203 :
204 17 : ret = memcpy_s(&qpCb->qpInfoRem, sizeof(struct RsQpInfo), bufTmp, sizeof(struct RsQpInfo));
205 17 : CHK_PRT_RETURN(ret,
206 : hccp_err("memcpy_s failed[%d], dest size:%d, src size:%d", ret, sizeof(struct RsQpInfo),
207 : sizeof(struct RsQpInfo)),
208 : -ENOMEM);
209 :
210 : /* modify qp state to RTR/RTS */
211 17 : ret = RsQpStateModify(qpCb);
212 17 : CHK_PRT_RETURN(ret,
213 : hccp_err("rs_qp_state_modify local qpn[%d] remote qpn[%d] failed ret[%d]", qpCb->qpInfoLo.qpn,
214 : qpCb->qpInfoRem.qpn, ret),
215 : ret);
216 :
217 16 : RsGetCurTime(&qpCb->endTime);
218 16 : HccpTimeInterval(&qpCb->endTime, &qpCb->startTime, &timeCost);
219 16 : if (timeCost > RS_EXPECT_TIME_MAX) {
220 2 : hccp_warn("local qpn[%d] remote qpn [%d] connect success cost[%f] more than[%f]ms!", qpCb->qpInfoLo.qpn,
221 : qpCb->qpInfoRem.qpn, timeCost, RS_EXPECT_TIME_MAX);
222 : } else {
223 14 : hccp_info("local qpn[%d] remote qpn [%d] connect success! cost [%f] ms", qpCb->qpInfoLo.qpn,
224 : qpCb->qpInfoRem.qpn, timeCost);
225 : }
226 :
227 16 : hccp_info("qp [%d] state has been migrate to RTS!, qpCb state is %d", qpCb->qpInfoLo.qpn, qpCb->state);
228 :
229 16 : return 0;
230 : }
231 :
232 14 : STATIC void *RsEpollRecvMrHandle(struct RsQpCb *qpCb, const char *bufTmp)
233 : {
234 : int ret;
235 : struct RsMrCb *mrCb;
236 :
237 14 : mrCb = calloc(1, sizeof(struct RsMrCb));
238 14 : CHK_PRT_RETURN(mrCb == NULL, hccp_err("mr_cb calloc failed"), NULL);
239 13 : ret = memcpy_s(&mrCb->mrInfo, sizeof(struct RsMrInfo), bufTmp, sizeof(struct RsMrInfo));
240 13 : if (ret) {
241 0 : hccp_err("memcpy_s failed[%d], dest size:%u, src size:%u", ret, sizeof(struct RsMrInfo),
242 : sizeof(struct RsMrInfo));
243 0 : free(mrCb);
244 0 : mrCb = NULL;
245 0 : return NULL;
246 : }
247 :
248 13 : RsListAddTail(&mrCb->list, &qpCb->remMrList);
249 :
250 13 : hccp_info("recv mr addr is 0x%llx", mrCb->mrInfo.addr);
251 13 : hccp_info("recv mr len is %llu", mrCb->mrInfo.len);
252 :
253 13 : return mrCb;
254 : }
255 :
256 17 : STATIC int RsCmdQpInfoHandle(struct RsQpCb *qpCb, unsigned int totalSize, const char *bufTmp, unsigned int curSize,
257 : bool *flag)
258 : {
259 : int ret;
260 17 : CHK_PRT_RETURN((totalSize - curSize) < sizeof(struct RsQpInfo),
261 : hccp_info("qp_info remain size"
262 : "[%u] < size [%u], wait for next recv",
263 : totalSize - curSize, sizeof(struct RsQpInfo)),
264 : -EINVAL);
265 :
266 17 : ret = RsEpollRecvQpHandle(qpCb, bufTmp);
267 17 : CHK_PRT_RETURN(ret, hccp_err("rs_epoll_recv_qp_handle failed! ret[%d]", ret), ret);
268 :
269 16 : RsNotifyMrListAdd(qpCb, bufTmp);
270 16 : hccp_info("rs_notify_mr_list_add");
271 :
272 16 : *flag = true;
273 16 : hccp_info("qp_info cur_size(%u) len(%u) !", curSize, sizeof(struct RsQpInfo));
274 :
275 16 : return 0;
276 : }
277 :
278 14 : STATIC int RsCmdMrInfoHandle(struct RsQpCb *qpCb, unsigned int totalSize, const char *bufTmp, unsigned int curSize,
279 : bool *flag)
280 : {
281 14 : CHK_PRT_RETURN((totalSize - curSize) < sizeof(struct RsMrInfo),
282 : hccp_info("mr_info remain size"
283 : "[%u] < size [%u], wait for next recv",
284 : totalSize - curSize, sizeof(struct RsMrInfo)),
285 : -EINVAL);
286 :
287 14 : (void)RsEpollRecvMrHandle(qpCb, bufTmp);
288 :
289 14 : *flag = true;
290 :
291 14 : hccp_info("mr_info cur_size(%u) len(%u) !", curSize, sizeof(struct RsMrInfo));
292 :
293 14 : return 0;
294 : }
295 :
296 14 : STATIC int RsCmdLenInfoHandle(struct RsQpCb *qpCb, unsigned int totalSize, const char *bufTmp, unsigned int curSize,
297 : bool *flag)
298 : {
299 14 : CHK_PRT_RETURN((totalSize - curSize) < sizeof(struct RsQpLenInfo),
300 : hccp_info("len_info remain size"
301 : "[%u] < size [%u], wait for next recv",
302 : totalSize - curSize, sizeof(struct RsQpLenInfo)),
303 : -EINVAL);
304 :
305 14 : qpCb->expectLen = *((const uint32_t *)(bufTmp + sizeof(uint32_t)));
306 :
307 14 : *flag = true;
308 :
309 14 : return 0;
310 : }
311 :
312 15 : STATIC void RsEpollRecvHandleRemain(struct RsQpCb *qpCb, unsigned int totalSize, unsigned int curSize, bool flag,
313 : const char *bufTmp)
314 : {
315 15 : int ret = 0;
316 :
317 15 : qpCb->remainSize = totalSize - curSize;
318 15 : if ((qpCb->remainSize > 0) && (flag == true)) {
319 1 : ret = memcpy_s(qpCb->qpMrBuf, RS_BUF_SIZE, bufTmp, qpCb->remainSize);
320 1 : if (ret) {
321 1 : hccp_err("memcpy_s failed, ret:%d, remainSize:%u", ret, qpCb->remainSize);
322 1 : return;
323 : }
324 : }
325 :
326 14 : return;
327 : }
328 :
329 18 : STATIC void RsEpollRecvHandle(struct RsQpCb *qpCb, char *buf, int size)
330 : {
331 18 : unsigned int totalSize = qpCb->remainSize + (unsigned int)size;
332 18 : char *bufTmp = (char *)qpCb->qpMrBuf;
333 18 : unsigned int curSize = 0;
334 18 : bool flag = false;
335 : uint32_t cmd;
336 : int ret;
337 :
338 18 : hccp_info("Message for qp:%d, qpCb->remainSize:%u, size:%d", qpCb->qpInfoLo.qpn, qpCb->remainSize, size);
339 18 : ret = memcpy_s(qpCb->qpMrBuf + qpCb->remainSize, RS_BUF_SIZE - qpCb->remainSize, buf, size);
340 18 : if (ret) {
341 2 : hccp_err("memcpy_s failed, ret:%d, remainSize:%u, size:%d", ret, qpCb->remainSize, size);
342 4 : return;
343 : }
344 :
345 : do {
346 47 : cmd = *((uint32_t *)bufTmp);
347 47 : switch (cmd) {
348 17 : case RS_CMD_QP_INFO:
349 17 : ret = RsCmdQpInfoHandle(qpCb, totalSize, bufTmp, curSize, &flag);
350 17 : if (ret) {
351 1 : goto out;
352 : }
353 :
354 16 : curSize += sizeof(struct RsQpInfo);
355 16 : bufTmp = qpCb->qpMrBuf + curSize;
356 16 : break;
357 14 : case RS_CMD_MR_INFO:
358 14 : ret = RsCmdMrInfoHandle(qpCb, totalSize, bufTmp, curSize, &flag);
359 14 : if (ret) {
360 0 : goto out;
361 : }
362 :
363 14 : curSize += sizeof(struct RsMrInfo);
364 14 : bufTmp = qpCb->qpMrBuf + curSize;
365 14 : break;
366 14 : case RS_CMD_LEN_INFO:
367 14 : ret = RsCmdLenInfoHandle(qpCb, totalSize, bufTmp, curSize, &flag);
368 14 : if (ret) {
369 0 : goto out;
370 : }
371 14 : curSize += sizeof(struct RsQpLenInfo);
372 14 : bufTmp = qpCb->qpMrBuf + curSize;
373 14 : break;
374 2 : default:
375 2 : hccp_warn("qp %d, unknown cmd(0x%x)!", qpCb->qpInfoLo.qpn, cmd);
376 2 : RsBufPrint(buf, size);
377 2 : return;
378 : }
379 44 : } while (curSize < totalSize);
380 :
381 13 : out:
382 14 : RsEpollRecvHandleRemain(qpCb, totalSize, curSize, flag, bufTmp);
383 : }
384 :
385 27 : STATIC void RsQpMrRecvHandle(int fd, struct RsQpCb *qpCb)
386 : {
387 : char buf[RS_BUF_SIZE];
388 : int size;
389 : int ret;
390 :
391 27 : RS_PTHREAD_MUTEX_LOCK(&qpCb->qpMutex);
392 :
393 27 : size = RsSocketRecv(fd, buf, RS_BUF_SIZE - qpCb->remainSize);
394 27 : hccp_dbg("fd %d qpn %d read size = %d, qpCb->remainSize:%u", fd, qpCb->qpInfoLo.qpn, size, qpCb->remainSize);
395 :
396 27 : if (size > 0) {
397 13 : qpCb->recvLen += (uint32_t)size;
398 13 : RsEpollRecvHandle(qpCb, buf, size);
399 14 : } else if (size == 0) {
400 0 : hccp_dbg("fd %d read size = %d, remote fd has been closed, fd cannot use !", fd, size);
401 : #ifdef CA_CONFIG_LLT
402 0 : qpCb->state = RS_QP_STATUS_REM_FD_CLOSE;
403 : #endif
404 : } else {
405 14 : ret = errno;
406 14 : hccp_dbg("no data available, errno:%d", ret);
407 : }
408 27 : RS_PTHREAD_MUTEX_ULOCK(&qpCb->qpMutex);
409 :
410 27 : return;
411 : }
412 :
413 0 : STATIC int RsHandleQpMrEpollEvent(struct RsRdevCb *rdevCb, int fd)
414 : {
415 : struct RsQpCb *qpCb;
416 0 : struct RsQpCb *qpCb2 = NULL;
417 :
418 : /* QP event, QP info exchange */
419 0 : RS_LIST_GET_HEAD_ENTRY(qpCb, qpCb2, &rdevCb->qpList, list, struct RsQpCb);
420 0 : for (; (&qpCb->list) != &rdevCb->qpList; qpCb = qpCb2, qpCb2 = list_entry(qpCb2->list.next, struct RsQpCb, list)) {
421 0 : if (qpCb->channel == NULL) {
422 0 : continue;
423 : }
424 0 : if (qpCb->srqContext != NULL && qpCb->srqContext->channel->fd == fd) {
425 0 : hccp_dbg("fd %d poll cq!", fd);
426 0 : RsDrvPollSrqCqHandle(qpCb);
427 0 : return 0;
428 : }
429 0 : if (fd == qpCb->channel->fd) {
430 0 : hccp_dbg("fd %d poll cq!", fd);
431 0 : RsDrvPollCqHandle(qpCb);
432 0 : return 0;
433 : }
434 : }
435 0 : return -ENODEV;
436 : }
437 :
438 0 : int RsEpollEventQpMrInHandle(struct rs_cb *rsCb, int fd)
439 : {
440 : int ret;
441 0 : struct RsRdevCb *rdevCbTmp = NULL;
442 0 : struct RsRdevCb *rdevCbTmp2 = NULL;
443 :
444 0 : if (rsCb->protocol != PROTOCOL_RDMA) {
445 0 : return -ENODEV;
446 : }
447 :
448 0 : RS_LIST_GET_HEAD_ENTRY(rdevCbTmp, rdevCbTmp2, &rsCb->rdevList, list, struct RsRdevCb);
449 0 : for (; (&rdevCbTmp->list) != &rsCb->rdevList;
450 0 : rdevCbTmp = rdevCbTmp2, rdevCbTmp2 = list_entry(rdevCbTmp2->list.next, struct RsRdevCb, list)) {
451 0 : RS_PTHREAD_MUTEX_LOCK(&rdevCbTmp->rdevMutex);
452 0 : ret = RsHandleQpMrEpollEvent(rdevCbTmp, fd);
453 0 : RS_PTHREAD_MUTEX_ULOCK(&rdevCbTmp->rdevMutex);
454 0 : if (ret == 0) {
455 0 : return 0;
456 : }
457 : }
458 0 : return -ENODEV;
459 : }
460 :
461 28 : STATIC int RsMrInfoSync(struct RsMrCb *mrCb)
462 : {
463 : int ret;
464 :
465 28 : hccp_info("mr state:%d, addr:0x%lx", mrCb->state, mrCb->mrInfo.addr);
466 :
467 28 : CHK_PRT_RETURN(mrCb->state & RS_MR_STATE_SYNCED,
468 : hccp_warn("mr synced ! mr_cb->flag[%d] & [%d] != 0", mrCb->state, RS_MR_STATE_SYNCED), 0);
469 :
470 : /*
471 : * no socket available for MR_INFO exchange if allowed
472 : * need exchange when socket available
473 : */
474 27 : CHK_PRT_RETURN(mrCb->qpCb->connInfo == NULL, hccp_warn("no conn available !"), 0);
475 :
476 27 : CHK_PRT_RETURN(mrCb->qpCb->state == RS_QP_STATUS_REM_FD_CLOSE,
477 : hccp_warn("remote qp fd closed,"
478 : "cann not use it anymore! status[%d](RS_QP_STATUS_REM_FD_CLOSE)",
479 : mrCb->qpCb->state),
480 : -EFAULT);
481 :
482 27 : CHK_PRT_RETURN(mrCb->qpCb->connInfo->connfd == RS_FD_INVALID,
483 : hccp_warn("rm info sync failed! fd not ready!"
484 : "connfd[%d](RS_FD_INVALID)",
485 : mrCb->qpCb->connInfo->connfd),
486 : -ENETUNREACH);
487 :
488 26 : mrCb->mrInfo.cmd = (unsigned int)RS_CMD_MR_INFO;
489 26 : ret = RsSocketSend(mrCb->qpCb->connInfo->connfd, &mrCb->mrInfo, sizeof(struct RsMrInfo));
490 26 : CHK_PRT_RETURN(ret != sizeof(struct RsMrInfo),
491 : hccp_err("mr_info send %d/%ld incomplete", ret, sizeof(struct RsMrInfo)), -EAGAIN);
492 :
493 25 : mrCb->qpCb->sendLen += (uint32_t)ret;
494 25 : mrCb->state |= RS_MR_STATE_SYNCED;
495 25 : hccp_info("after send mr state:%d, addr:0x%lx", mrCb->state, mrCb->mrInfo.addr);
496 :
497 25 : return 0;
498 : }
499 :
500 5 : STATIC int RsMrPrepareRoceSign(unsigned int phyId, struct RsRdevCb *devCb, struct roce_process_sign *roceSign)
501 : {
502 5 : unsigned int tmpPhyId = phyId;
503 : unsigned int chipId;
504 : int ret;
505 :
506 : // reg mr with backup phyId
507 5 : if (devCb->backupInfo.backupFlag) {
508 0 : tmpPhyId = devCb->backupInfo.rdevInfo.phyId;
509 0 : ret = DlDrvGetLocalDevIdByHostDevId(tmpPhyId, &chipId);
510 : } else {
511 5 : ret = rsGetLocalDevIDByHostDevID(tmpPhyId, &chipId);
512 : }
513 5 : CHK_PRT_RETURN(ret != 0,
514 : hccp_err("get chipId failed, ret %d, phyid[%u] backupFlag[%d]", ret, tmpPhyId, devCb->backupInfo.backupFlag),
515 : -EACCES);
516 5 : roceSign->tgid = devCb->rsCb->pRsSign.tgid;
517 5 : roceSign->devid = chipId;
518 5 : roceSign->vfid = 0;
519 5 : ret = strcpy_s(roceSign->sign, PROCESS_RS_SIGN_LENGTH, devCb->rsCb->pRsSign.sign);
520 5 : CHK_PRT_RETURN(ret != 0, hccp_err("Invalid pid sign, ret(%d)", ret), -ESAFEFUNC);
521 5 : return 0;
522 : }
523 :
524 8 : STATIC int RsMrPreReg(unsigned int phyId, struct RsQpCb *qpCb, struct RsMrCb *mrCb, struct RdmaMrRegInfo *mrRegInfo)
525 : {
526 8 : unsigned long long len = mrRegInfo->len;
527 8 : struct roce_process_sign roceSign = {0};
528 8 : int access = mrRegInfo->access;
529 8 : char *addr = mrRegInfo->addr;
530 : int ret;
531 :
532 8 : if (qpCb->rdevCb->rsCb->hccpMode == NETWORK_PEER_ONLINE || qpCb->rdevCb->rsCb->hccpMode == NETWORK_ONLINE ||
533 7 : qpCb->isExp == RS_NOT_EXP) {
534 3 : mrCb->ibMr = RsDrvMrReg(qpCb->ibPd, addr, len, access);
535 3 : CHK_PRT_RETURN(mrCb->ibMr == NULL, hccp_err("rs_drv_mr_reg addr is NULL len[%lld] failed ", len), -EACCES);
536 : } else {
537 5 : ret = RsMrPrepareRoceSign(phyId, qpCb->rdevCb, &roceSign);
538 5 : CHK_PRT_RETURN(ret != 0, hccp_err("RsMrPrepareRoceSign failed, ret(%d)", ret), ret);
539 5 : mrCb->ibMr = RsDrvExpMrReg(qpCb->ibPd, addr, len, access, roceSign);
540 5 : CHK_PRT_RETURN(mrCb->ibMr == NULL, hccp_err("rs_drv_exp_mr_reg addr is NULL len[%lld] failed ", len), -EACCES);
541 : }
542 :
543 8 : mrCb->mrInfo.cmd = (unsigned int)RS_CMD_MR_INFO;
544 8 : mrCb->mrInfo.addr = (uintptr_t)addr;
545 8 : mrCb->mrInfo.len = len;
546 8 : mrCb->mrInfo.rkey = mrCb->ibMr->rkey;
547 :
548 8 : RS_PTHREAD_MUTEX_LOCK(&qpCb->qpMutex);
549 8 : RsListAddTail(&mrCb->list, &qpCb->mrList);
550 8 : RS_PTHREAD_MUTEX_ULOCK(&qpCb->qpMutex);
551 :
552 8 : qpCb->mrNum++;
553 8 : return 0;
554 : }
555 :
556 41 : STATIC int RsCallocMr(int num, struct RsMrCb **mrCb)
557 : {
558 41 : CHK_PRT_RETURN(num <= 0, hccp_err("invalid num for mr calloc"), -EINVAL);
559 :
560 40 : *mrCb = calloc(num, sizeof(struct RsMrCb));
561 40 : CHK_PRT_RETURN((*mrCb) == NULL, hccp_err("calloc mr_cb failed"), -ENOMEM);
562 40 : return 0;
563 : }
564 :
565 54 : STATIC int RsCallocQpcb(int num, struct RsQpCb **qpCb)
566 : {
567 54 : if (num <= 0) {
568 1 : return -EINVAL;
569 : }
570 :
571 53 : *qpCb = calloc(num, sizeof(struct RsQpCb));
572 53 : if ((*qpCb) == NULL) {
573 1 : return -ENOMEM;
574 : }
575 :
576 52 : return 0;
577 : }
578 :
579 10 : RS_ATTRI_VISI_DEF int RsMrReg(unsigned int phyId, unsigned int rdevIndex, unsigned int qpn,
580 : struct RdmaMrRegInfo *mrRegInfo)
581 : {
582 : int ret;
583 10 : struct RsQpCb *qpCb = NULL;
584 10 : struct RsMrCb *mrCb = NULL;
585 :
586 10 : CHK_PRT_RETURN(mrRegInfo == NULL || mrRegInfo->addr == NULL || mrRegInfo->len == 0 || phyId >= RS_MAX_DEV_NUM,
587 : hccp_err("param err, NULL pointer or phyId:%u >= [%d]", phyId, RS_MAX_DEV_NUM), -EINVAL);
588 :
589 9 : hccp_info("qpn[%u], len[0x%llx], access[%d]", qpn, mrRegInfo->len, mrRegInfo->access);
590 :
591 9 : ret = RsQpn2qpcb(phyId, rdevIndex, qpn, &qpCb);
592 9 : CHK_PRT_RETURN(ret, hccp_err("rs_qpn2qpcb qpn[%d] ret[%d] failed ", qpn, ret), ret);
593 :
594 8 : CHK_PRT_RETURN(qpCb->mrNum >= RS_MR_NUM_MAX, hccp_err("Exceeded the maximum MR limit %d", qpCb->mrNum), -EINVAL);
595 :
596 8 : ret = RsGetMrcb(qpCb, (uintptr_t)mrRegInfo->addr, &mrCb, &qpCb->mrList);
597 8 : if (ret == 0) {
598 0 : hccp_warn("mr already registered");
599 0 : goto found;
600 : }
601 :
602 8 : ret = RsCallocMr(1, &mrCb);
603 8 : CHK_PRT_RETURN(ret, hccp_err("calloc mr failed"), ret);
604 :
605 8 : mrCb->qpCb = qpCb;
606 :
607 8 : ret = RsMrPreReg(phyId, qpCb, mrCb, mrRegInfo);
608 8 : if (ret) {
609 0 : hccp_err("pre reg mr failed, qpn %u, ret %d", qpn, ret);
610 0 : goto reg_err;
611 : }
612 :
613 8 : found:
614 8 : mrRegInfo->lkey = mrCb->ibMr->lkey;
615 8 : mrRegInfo->rkey = mrCb->ibMr->rkey;
616 :
617 8 : hccp_info("rs_mr_reg succ, state:%u", mrCb->state);
618 8 : return 0;
619 :
620 0 : reg_err:
621 0 : free(mrCb);
622 0 : mrCb = NULL;
623 :
624 0 : return ret;
625 : }
626 :
627 10 : RS_ATTRI_VISI_DEF int RsMrDereg(unsigned int phyId, unsigned int rdevIndex, unsigned int qpn, char *addr)
628 : {
629 : int ret;
630 10 : struct RsQpCb *qpCb = NULL;
631 10 : struct RsMrCb *mrCb = NULL;
632 :
633 10 : hccp_dbg("start rs_mr_dereg");
634 10 : RS_CHECK_POINTER_NULL_RETURN_INT(addr);
635 9 : CHK_PRT_RETURN(phyId >= RS_MAX_DEV_NUM, hccp_err("phyId:%u >= [%d], is invalid", phyId, RS_MAX_DEV_NUM), -EINVAL);
636 :
637 9 : ret = RsQpn2qpcb(phyId, rdevIndex, qpn, &qpCb);
638 9 : CHK_PRT_RETURN(ret, hccp_err("rs_qpn2qpcb failed ret[%d]", ret), ret);
639 :
640 7 : CHK_PRT_RETURN(RsGetMrcb(qpCb, (uintptr_t)addr, &mrCb, &qpCb->mrList),
641 : hccp_err("rs_get_mrcb failed "
642 : "g_rs_send_wr_num[%u]",
643 : gRsSendWrNum),
644 : -EFAULT);
645 :
646 6 : ret = RsDrvMrDereg(mrCb->ibMr);
647 6 : CHK_PRT_RETURN(ret, hccp_err("rs_drv_mr_dereg failed ret[%d] ", ret), -EACCES);
648 :
649 6 : RS_PTHREAD_MUTEX_LOCK(&qpCb->qpMutex);
650 6 : RsListDel(&mrCb->list);
651 6 : free(mrCb);
652 6 : mrCb = NULL;
653 6 : RS_PTHREAD_MUTEX_ULOCK(&qpCb->qpMutex);
654 6 : qpCb->mrNum--;
655 :
656 6 : hccp_dbg("qpn[%u] succ", qpn);
657 :
658 6 : return 0;
659 : }
660 :
661 2 : STATIC int RsRegisterUbSegment(int directFlag, uint64_t addr, uint64_t len)
662 : {
663 2 : struct DVattribute attr = {0};
664 2 : int ret = 0;
665 :
666 2 : if (directFlag != DIRECT_FLAG_UB) {
667 2 : return 0;
668 : }
669 :
670 0 : ret = DlDrvMemGetAttribute(addr, &attr);
671 0 : CHK_PRT_RETURN(ret != 0, hccp_err("DlDrvMemGetAttribute failed, ret:%d", ret), ret);
672 :
673 0 : if (attr.memType != DV_MEM_LOCK_DEV) { // not support host memory
674 0 : return ret;
675 : }
676 :
677 0 : ret = DlHalMemRegUbSegment(attr.devId, addr, len);
678 0 : CHK_PRT_RETURN(ret != 0, hccp_err("DlHalMemRegUbSegment failed, ret:%d devId:%u len:%u", ret, attr.devId, len),
679 : ret);
680 :
681 0 : return ret;
682 : }
683 :
684 2 : STATIC int RsUnRegisterUbSegment(int directFlag, uint64_t addr)
685 : {
686 2 : struct DVattribute attr = {0};
687 2 : int ret = 0;
688 :
689 2 : if (directFlag != DIRECT_FLAG_UB) {
690 2 : return 0;
691 : }
692 :
693 0 : ret = DlDrvMemGetAttribute(addr, &attr);
694 0 : CHK_PRT_RETURN(ret != 0, hccp_err("DrvMemGetAttribute failed, ret:%d", ret), ret);
695 :
696 0 : if (attr.memType != DV_MEM_LOCK_DEV) { // not support host memory
697 0 : return ret;
698 : }
699 :
700 0 : ret = DlHalMemUnRegUbSegment(attr.devId, addr);
701 0 : if (ret != 0) {
702 0 : hccp_err("DlHalMemUnRegUbSegment failed, ret:%d devId:%u", ret, attr.devId);
703 : }
704 :
705 0 : return ret;
706 : }
707 :
708 3 : RS_ATTRI_VISI_DEF int RsRegisterMr(unsigned int phyId, unsigned int rdevIndex, struct RdmaMrRegInfo *mrRegInfo,
709 : void **mrHandle)
710 : {
711 3 : RS_CHECK_POINTER_NULL_RETURN_INT(mrHandle);
712 :
713 : int ret;
714 : unsigned int chipId;
715 3 : struct RsRdevCb *rdevCb = NULL;
716 3 : struct ibv_mr *rsMrHandle = NULL;
717 :
718 3 : CHK_PRT_RETURN(mrRegInfo == NULL || mrRegInfo->addr == NULL || mrRegInfo->len == 0 || phyId >= RS_MAX_DEV_NUM,
719 : hccp_err("param err, NULL pointer or phyId:%u >= [%d]", phyId, RS_MAX_DEV_NUM), -EINVAL);
720 :
721 2 : hccp_info("[rs_register_mr] len[0x%llx], access[%d]", mrRegInfo->len, mrRegInfo->access);
722 :
723 2 : ret = rsGetLocalDevIDByHostDevID(phyId, &chipId);
724 2 : CHK_PRT_RETURN(ret, hccp_err("rs_register_mr rsGetLocalDevIDByHostDevID phyId[%u] invalid, ret %d", phyId, ret),
725 : ret);
726 :
727 2 : ret = RsRdev2rdevCb(chipId, rdevIndex, &rdevCb);
728 2 : CHK_PRT_RETURN(ret != 0 || rdevCb == NULL, hccp_err("rs_rdev2rdev_cb for chip_id[%u] failed, ret %d", chipId, ret),
729 : ret);
730 :
731 2 : ret = RsRegisterUbSegment(rdevCb->directFlag, (uint64_t)(uintptr_t)mrRegInfo->addr, mrRegInfo->len);
732 2 : if (ret != 0) {
733 0 : hccp_err("RsRegisterUbSegment failed, ret[%d] vendor_id[0x%x] part_id[0x%x] directFlag[%u] "
734 : "addr[0x%llx] len[0x%llx]",
735 : ret, rdevCb->deviceAttr.vendor_id, rdevCb->deviceAttr.vendor_part_id, rdevCb->directFlag,
736 : (uint64_t)(uintptr_t)mrRegInfo->addr, mrRegInfo->len);
737 0 : goto mem_reg_err;
738 : }
739 :
740 2 : *mrHandle = (void *)RsDrvMrReg(rdevCb->ibPd, mrRegInfo->addr, mrRegInfo->len, mrRegInfo->access);
741 2 : if (*mrHandle == NULL) {
742 1 : hccp_warn("rs_drv_mr_reg addr is NULL len[0x%llx] access[%d] unsuccessful errno[%d]", mrRegInfo->len,
743 : mrRegInfo->access, errno);
744 1 : goto mr_reg_err;
745 : }
746 :
747 1 : rsMrHandle = (struct ibv_mr *)*mrHandle;
748 1 : mrRegInfo->lkey = rsMrHandle->lkey;
749 1 : mrRegInfo->rkey = rsMrHandle->rkey;
750 :
751 1 : hccp_info("rs_register_mr succ");
752 1 : return ret;
753 1 : mr_reg_err:
754 1 : (void)RsUnRegisterUbSegment(rdevCb->directFlag, (uint64_t)(uintptr_t)mrRegInfo->addr);
755 1 : mem_reg_err:
756 1 : mrRegInfo->lkey = 0;
757 :
758 1 : return ret;
759 : }
760 :
761 2 : STATIC int RsInitTypicalMrCb(unsigned int phyId, struct RdmaMrRegInfo *mrRegInfo, struct RsRdevCb *devCb,
762 : struct RsMrCb *mrCb)
763 : {
764 2 : unsigned long long len = mrRegInfo->len;
765 2 : struct roce_process_sign roceSign = {0};
766 2 : char *addr = (char *)mrRegInfo->addr;
767 2 : int access = mrRegInfo->access;
768 : int ret;
769 :
770 2 : if (devCb->rsCb->hccpMode == NETWORK_PEER_ONLINE || devCb->rsCb->hccpMode == NETWORK_ONLINE) {
771 2 : mrCb->ibMr = RsDrvMrReg(devCb->ibPd, addr, len, access);
772 2 : CHK_PRT_RETURN(mrCb->ibMr == NULL, hccp_err("rs_drv_mr_reg addr is NULL len[%lld] failed", len), -EACCES);
773 : } else {
774 0 : ret = RsMrPrepareRoceSign(phyId, devCb, &roceSign);
775 0 : CHK_PRT_RETURN(ret != 0, hccp_err("RsMrPrepareRoceSign failed, ret(%d)", ret), ret);
776 0 : mrCb->ibMr = RsDrvExpMrReg(devCb->ibPd, addr, len, access, roceSign);
777 0 : CHK_PRT_RETURN(mrCb->ibMr == NULL, hccp_err("rs_drv_exp_mr_reg addr is NULL len[%lld] failed", len), -EACCES);
778 : }
779 :
780 2 : mrCb->mrInfo.addr = (uintptr_t)addr;
781 2 : mrCb->mrInfo.len = len;
782 2 : mrCb->mrInfo.rkey = mrCb->ibMr->rkey;
783 :
784 2 : RS_PTHREAD_MUTEX_LOCK(&devCb->rdevMutex);
785 2 : RsListAddTail(&mrCb->list, &devCb->typicalMrList);
786 2 : RS_PTHREAD_MUTEX_ULOCK(&devCb->rdevMutex);
787 :
788 2 : return 0;
789 : }
790 :
791 1 : RS_ATTRI_VISI_DEF int RsTypicalRegisterMrV1(unsigned int phyId, unsigned int rdevIndex, struct RdmaMrRegInfo *mrRegInfo,
792 : void **mrHandle)
793 : {
794 1 : RS_CHECK_POINTER_NULL_RETURN_INT(mrHandle);
795 :
796 1 : struct RsMrCb *typicalMrCb = NULL;
797 1 : struct RsRdevCb *rdevCb = NULL;
798 : unsigned int chipId;
799 : int ret;
800 :
801 1 : CHK_PRT_RETURN(mrRegInfo == NULL || mrRegInfo->addr == NULL || mrRegInfo->len == 0 || phyId >= RS_MAX_DEV_NUM,
802 : hccp_err("param err, NULL pointer or phyId:%u >= [%d]", phyId, RS_MAX_DEV_NUM), -EINVAL);
803 :
804 1 : hccp_info("[rs_typical_register_mr] len[0x%llx], access[%d]", mrRegInfo->len, mrRegInfo->access);
805 :
806 1 : ret = rsGetLocalDevIDByHostDevID(phyId, &chipId);
807 1 : CHK_PRT_RETURN(ret != 0,
808 : hccp_err("rs_typical_register_mr rsGetLocalDevIDByHostDevID phyId[%u] invalid, ret %d", phyId, ret), ret);
809 :
810 1 : ret = RsRdev2rdevCb(chipId, rdevIndex, &rdevCb);
811 1 : CHK_PRT_RETURN(ret != 0 || rdevCb == NULL, hccp_err("rs_rdev2rdev_cb for chip_id[%u] failed, ret %d", chipId, ret),
812 : ret);
813 :
814 1 : ret = RsQueryMrCb(rdevCb, (uint64_t)(uintptr_t)mrRegInfo->addr, &typicalMrCb, &rdevCb->typicalMrList);
815 1 : if (ret == 0) {
816 0 : hccp_warn("typical mr already registered");
817 0 : goto found;
818 : }
819 :
820 1 : typicalMrCb = calloc(1, sizeof(struct RsMrCb));
821 1 : CHK_PRT_RETURN(typicalMrCb == NULL, hccp_err("calloc typical_mr_cb failed"), -ENOMEM);
822 1 : typicalMrCb->devCb = rdevCb;
823 :
824 1 : ret = RsInitTypicalMrCb(phyId, mrRegInfo, rdevCb, typicalMrCb);
825 1 : if (ret != 0) {
826 0 : hccp_err("rs_init_typical_mr_cb failed, devIndex[%u], ret[%d]", rdevIndex, ret);
827 0 : goto reg_err;
828 : }
829 :
830 1 : found:
831 1 : *mrHandle = typicalMrCb->ibMr;
832 1 : mrRegInfo->lkey = typicalMrCb->ibMr->lkey;
833 1 : mrRegInfo->rkey = typicalMrCb->ibMr->rkey;
834 1 : hccp_info("rs_typical_register_mr succ, state:%d", typicalMrCb->state);
835 1 : return 0;
836 :
837 0 : reg_err:
838 0 : free(typicalMrCb);
839 0 : typicalMrCb = NULL;
840 0 : return ret;
841 : }
842 :
843 1 : RS_ATTRI_VISI_DEF int RsTypicalRegisterMr(unsigned int phyId, unsigned int rdevIndex, struct RdmaMrRegInfo *mrRegInfo,
844 : void **mrHandle)
845 : {
846 1 : RS_CHECK_POINTER_NULL_RETURN_INT(mrHandle);
847 :
848 1 : struct RsMrCb *typicalMrCb = NULL;
849 1 : struct RsRdevCb *rdevCb = NULL;
850 : unsigned int chipId;
851 : int ret;
852 :
853 1 : CHK_PRT_RETURN(mrRegInfo == NULL || mrRegInfo->addr == NULL || mrRegInfo->len == 0 || phyId >= RS_MAX_DEV_NUM,
854 : hccp_err("param err, NULL pointer or phyId:%u >= [%d]", phyId, RS_MAX_DEV_NUM), -EINVAL);
855 :
856 1 : hccp_info("start register len[0x%llx], access[%d]", mrRegInfo->len, mrRegInfo->access);
857 :
858 1 : ret = rsGetLocalDevIDByHostDevID(phyId, &chipId);
859 1 : CHK_PRT_RETURN(ret != 0, hccp_err("rsGetLocalDevIDByHostDevID phyId[%u] invalid, ret %d", phyId, ret), ret);
860 :
861 1 : ret = RsRdev2rdevCb(chipId, rdevIndex, &rdevCb);
862 1 : CHK_PRT_RETURN(ret != 0 || rdevCb == NULL, hccp_err("rs_rdev2rdev_cb for chip_id[%u] failed, ret %d", chipId, ret),
863 : ret);
864 :
865 1 : typicalMrCb = calloc(1, sizeof(struct RsMrCb));
866 1 : CHK_PRT_RETURN(typicalMrCb == NULL, hccp_err("calloc typical_mr_cb failed"), -ENOMEM);
867 1 : typicalMrCb->devCb = rdevCb;
868 :
869 1 : ret = RsInitTypicalMrCb(phyId, mrRegInfo, rdevCb, typicalMrCb);
870 1 : if (ret != 0) {
871 0 : hccp_err("rs_init_typical_mr_cb failed, devIndex[%u], ret[%d]", rdevIndex, ret);
872 0 : goto reg_err;
873 : }
874 :
875 : // resv len as 1 to save addr for later unreg to query
876 1 : typicalMrCb->mrInfo.addr = (uint64_t)(uintptr_t)typicalMrCb->ibMr;
877 1 : typicalMrCb->mrInfo.len = 1U;
878 1 : *mrHandle = typicalMrCb->ibMr;
879 1 : mrRegInfo->lkey = typicalMrCb->ibMr->lkey;
880 1 : mrRegInfo->rkey = typicalMrCb->ibMr->rkey;
881 1 : hccp_info("register succ, state:%d", typicalMrCb->state);
882 1 : return 0;
883 :
884 0 : reg_err:
885 0 : free(typicalMrCb);
886 0 : typicalMrCb = NULL;
887 0 : return ret;
888 : }
889 :
890 4 : RS_ATTRI_VISI_DEF int RsRemapMr(unsigned int phyId, unsigned int rdevIndex, struct MemRemapInfo memList[],
891 : unsigned int memNum)
892 : {
893 4 : struct RsRdevCb *devCb = NULL;
894 4 : struct RsMrCb *mrCurr = NULL;
895 4 : struct RsMrCb *mrNext = NULL;
896 4 : unsigned long long addr = 0;
897 4 : bool isMemMatched = false;
898 : unsigned int chipId;
899 : unsigned int i;
900 : int ret;
901 :
902 4 : CHK_PRT_RETURN(phyId >= RS_MAX_DEV_NUM, hccp_err("phyId:%u >= %d, is invalid", phyId, RS_MAX_DEV_NUM), -EINVAL);
903 :
904 4 : ret = rsGetLocalDevIDByHostDevID(phyId, &chipId);
905 4 : CHK_PRT_RETURN(ret, hccp_err("rsGetLocalDevIDByHostDevID failed, phyId:%u invalid, ret:%d", phyId, ret), ret);
906 :
907 4 : ret = RsRdev2rdevCb(chipId, rdevIndex, &devCb);
908 4 : CHK_PRT_RETURN(devCb == NULL, hccp_err("rs_rdev2rdev_cb failed, chipId:%u, ret:%d", chipId, ret), -ENODEV);
909 :
910 6 : for (i = 0; i < memNum; i++) {
911 4 : isMemMatched = false;
912 4 : addr = (uint64_t)(uintptr_t)memList[i].addr;
913 4 : RS_PTHREAD_MUTEX_LOCK(&devCb->rdevMutex);
914 4 : RS_LIST_GET_HEAD_ENTRY(mrCurr, mrNext, &devCb->typicalMrList, list, struct RsMrCb);
915 7 : for (; (&mrCurr->list) != &devCb->typicalMrList;
916 3 : mrCurr = mrNext, mrNext = list_entry(mrNext->list.next, struct RsMrCb, list)) {
917 : // mem is out range of mr, continue to find next matching mr
918 3 : if ((addr < (uint64_t)(uintptr_t)mrCurr->ibMr->addr) || (memList[i].size > mrCurr->ibMr->length) ||
919 2 : (addr + memList[i].size < addr) ||
920 2 : (addr + memList[i].size > (uint64_t)(uintptr_t)mrCurr->ibMr->addr + mrCurr->ibMr->length)) {
921 1 : continue;
922 : }
923 :
924 : // each mr remap each corresponding mem
925 2 : ret = RsRoceRemapMr(mrCurr->ibMr, (struct hns_roce_mr_remap_info *)(void *)&memList[i], 1);
926 2 : if (ret != 0) {
927 0 : hccp_err("remap %u-th mem failed, ret:%d addr:0x%llx size:0x%llx", i, ret, addr, memList[i].size);
928 0 : RS_PTHREAD_MUTEX_ULOCK(&devCb->rdevMutex);
929 0 : return ret;
930 : }
931 2 : isMemMatched = true;
932 : }
933 4 : RS_PTHREAD_MUTEX_ULOCK(&devCb->rdevMutex);
934 :
935 4 : if (!isMemMatched) {
936 2 : hccp_err("find %u-th mem failed, addr:0x%llx size:0x%llx", i, addr, memList[i].size);
937 2 : return -ENODEV;
938 : }
939 2 : hccp_dbg("remap %u-th mem success, addr:0x%llx size:0x%llx", i, addr, memList[i].size);
940 : }
941 :
942 2 : return 0;
943 : }
944 :
945 2 : RS_ATTRI_VISI_DEF int RsTypicalDeregisterMr(unsigned int phyId, unsigned int devIndex, unsigned long long addr)
946 : {
947 2 : struct RsMrCb *typicalMrCb = NULL;
948 2 : struct RsRdevCb *devCb = NULL;
949 : unsigned int chipId;
950 : int ret;
951 :
952 2 : hccp_info("typical mr unreg start, addr[%llu]", addr);
953 2 : CHK_PRT_RETURN(phyId >= RS_MAX_DEV_NUM, hccp_err("phyId:%u >= %d, is invalid", phyId, RS_MAX_DEV_NUM), -EINVAL);
954 :
955 2 : ret = rsGetLocalDevIDByHostDevID(phyId, &chipId);
956 2 : CHK_PRT_RETURN(ret, hccp_err("phyId[%u] invalid, ret %d", phyId, ret), ret);
957 :
958 2 : ret = RsRdev2rdevCb(chipId, devIndex, &devCb);
959 2 : CHK_PRT_RETURN(ret != 0 || devCb == NULL,
960 : hccp_err("rs_rdev2rdev_cb get dev_cb failed for chip_id[%u], ret[%d]", chipId, ret), -ENODEV);
961 :
962 2 : ret = RsQueryMrCb(devCb, addr, &typicalMrCb, &devCb->typicalMrList);
963 2 : CHK_PRT_RETURN(ret, hccp_err("rs_query_mr_cb failed ret[%d]", ret), ret);
964 :
965 2 : ret = RsDrvMrDereg(typicalMrCb->ibMr);
966 2 : CHK_PRT_RETURN(ret, hccp_err("rs_drv_mr_dereg failed ret[%d]", ret), -EACCES);
967 :
968 2 : RS_PTHREAD_MUTEX_LOCK(&devCb->rdevMutex);
969 2 : RsListDel(&typicalMrCb->list);
970 2 : free(typicalMrCb);
971 2 : typicalMrCb = NULL;
972 2 : RS_PTHREAD_MUTEX_ULOCK(&devCb->rdevMutex);
973 :
974 2 : hccp_info("devIndex[%u] succ", devIndex);
975 :
976 2 : return 0;
977 : }
978 :
979 2 : RS_ATTRI_VISI_DEF int RsDeregisterMr(unsigned int phyId, unsigned int rdevIndex, void *mrHandle)
980 : {
981 2 : RS_CHECK_POINTER_NULL_RETURN_INT(mrHandle);
982 :
983 1 : struct ibv_mr *rsMrHandle = (struct ibv_mr *)mrHandle;
984 1 : uint64_t addr = (uint64_t)(uintptr_t)rsMrHandle->addr;
985 1 : struct RsRdevCb *devCb = NULL;
986 : unsigned int chipId;
987 1 : int ret = 0;
988 :
989 1 : ret = rsGetLocalDevIDByHostDevID(phyId, &chipId);
990 1 : CHK_PRT_RETURN(ret != 0, hccp_err("phyId[%u] invalid, ret %d", phyId, ret), ret);
991 :
992 1 : ret = RsRdev2rdevCb(chipId, rdevIndex, &devCb);
993 1 : CHK_PRT_RETURN(ret != 0 || devCb == NULL,
994 : hccp_err("rs_rdev2rdev_cb get dev_cb failed for chip_id[%u], ret[%d]", chipId, ret), -ENODEV);
995 :
996 1 : ret = RsDrvMrDereg(rsMrHandle);
997 1 : CHK_PRT_RETURN(ret != 0, hccp_err("rs_drv_mr_dereg failed ret[%d]", ret), -EACCES);
998 :
999 1 : ret = RsUnRegisterUbSegment(devCb->directFlag, addr);
1000 1 : CHK_PRT_RETURN(ret != 0,
1001 : hccp_err("RsUnRegisterUbSegment failed ret[%d], vendor_id[0x%x] part_id[0x%x]"
1002 : " directFlag[%u]",
1003 : ret, devCb->deviceAttr.vendor_id, devCb->deviceAttr.vendor_part_id, devCb->directFlag),
1004 : ret);
1005 :
1006 1 : hccp_info("rs_deregister_mr succ");
1007 1 : return 0;
1008 : }
1009 :
1010 7 : RS_ATTRI_VISI_DEF int RsSendWr(unsigned int phyId, unsigned int rdevIndex, uint32_t qpn, struct SendWr *wr,
1011 : struct SendWrRsp *wrRsp)
1012 : {
1013 : int ret;
1014 7 : struct RsQpCb *qpCb = NULL;
1015 7 : struct RsMrCb *mrCb = NULL;
1016 7 : struct RsMrCb *remMrCb = NULL;
1017 :
1018 7 : RS_CHECK_POINTER_NULL_RETURN_INT(wr);
1019 7 : RS_CHECK_POINTER_NULL_RETURN_INT(wr->bufList);
1020 7 : RS_CHECK_POINTER_NULL_RETURN_INT(wrRsp);
1021 :
1022 7 : CHK_PRT_RETURN(phyId >= RS_MAX_DEV_NUM, hccp_err("phyId:%u >= [%d], is invalid", phyId, RS_MAX_DEV_NUM), -EINVAL);
1023 :
1024 7 : CHK_PRT_RETURN(wr->bufNum > MAX_SGE_NUM || wr->bufNum == 0, hccp_err("invalid buf_num[%u]!", wr->bufNum), -EINVAL);
1025 :
1026 6 : CHK_PRT_RETURN(wr->bufList->len > RS_SGLIST_LEN_MAX || wr->bufList->len == 0,
1027 : hccp_err("sg list"
1028 : "len is more than 2G, len[%u]",
1029 : wr->bufList->len),
1030 : -EINVAL);
1031 :
1032 5 : if (RsQpn2qpcb(phyId, rdevIndex, qpn, &qpCb)) {
1033 1 : return -EACCES;
1034 : }
1035 :
1036 4 : qpCb->sendWrNum++;
1037 :
1038 4 : hccp_info("qpn %d, bufList[0].addr is 0x%llx", qpn, wr->bufList[0].addr);
1039 4 : if (RsGetMrcb(qpCb, wr->bufList[0].addr, &mrCb, &qpCb->mrList)) {
1040 1 : hccp_err("qpn %d, bufList[0].addr[0x%llx] len[0x%x] is invalid.", qpn, wr->bufList[0].addr, wr->bufList[0].len);
1041 1 : return -EFAULT;
1042 : }
1043 :
1044 : // send op no need to check & get remote mr
1045 3 : if (wr->op != RA_WR_SEND && wr->op != RA_WR_SEND_WITH_IMM) {
1046 3 : hccp_info("remote wr dst addr is 0x%llx", wr->dstAddr);
1047 3 : if (RsGetMrcb(qpCb, wr->dstAddr, &remMrCb, &qpCb->remMrList)) {
1048 1 : hccp_err("qpn %d, remote wr dst addr[0x%llx] len[0x%x] is invalid.", qpn, wr->dstAddr, wr->bufList[0].len);
1049 1 : return -ENOENT;
1050 : }
1051 : }
1052 :
1053 2 : ret = RsDrvSendExp(qpCb, mrCb, remMrCb, wr, wrRsp);
1054 2 : if (ret) {
1055 0 : hccp_err("send exp failed qpn %u, ret %d", qpn, ret);
1056 : }
1057 2 : gRsSendWrNum++;
1058 2 : return ret;
1059 : }
1060 :
1061 5 : STATIC void BuildUpWrWithKey(struct WrInfo *wr, struct ibv_sge *list, struct ibv_send_wr *ibWr)
1062 : {
1063 5 : list->addr = (uintptr_t)wr->memList.addr;
1064 5 : list->length = wr->memList.len;
1065 5 : list->lkey = wr->memList.lkey;
1066 :
1067 5 : ibWr->sg_list = list;
1068 5 : ibWr->opcode = wr->op;
1069 5 : ibWr->send_flags = (unsigned int)wr->sendFlags;
1070 5 : ibWr->imm_data = htobe32(wr->immData);
1071 :
1072 5 : ibWr->num_sge = 1; /* only support one sge */
1073 5 : ibWr->wr_id = wr->wrId;
1074 5 : if (wr->op != IBV_WR_SEND && wr->op != IBV_WR_SEND_WITH_IMM) {
1075 5 : ibWr->wr.rdma.rkey = wr->rkey;
1076 5 : ibWr->wr.rdma.remote_addr = wr->dstAddr;
1077 : }
1078 5 : }
1079 :
1080 0 : STATIC void RsSendBuildUpWr(struct RsMrCb *mrCb, struct WrInfo *wr, struct ibv_sge *list, struct ibv_send_wr *ibWr)
1081 : {
1082 0 : list->addr = (uintptr_t)wr->memList.addr;
1083 0 : list->lkey = mrCb->ibMr->lkey;
1084 0 : list->length = wr->memList.len;
1085 :
1086 0 : ibWr->sg_list = list;
1087 0 : ibWr->opcode = wr->op;
1088 0 : ibWr->imm_data = htobe32(wr->immData);
1089 0 : ibWr->send_flags = (unsigned int)wr->sendFlags;
1090 :
1091 0 : ibWr->num_sge = 1; /* only support one sge */
1092 0 : ibWr->wr_id = wr->wrId;
1093 0 : }
1094 :
1095 2 : STATIC void RsWirteAndReadBuildUpWr(struct RsMrCb *mrCb, struct RsMrCb *remMrCb, struct WrInfo *wr,
1096 : struct ibv_sge *list, struct ibv_send_wr *ibWr)
1097 : {
1098 2 : list->addr = (uintptr_t)wr->memList.addr;
1099 2 : list->length = wr->memList.len;
1100 2 : list->lkey = mrCb->ibMr->lkey;
1101 :
1102 2 : ibWr->sg_list = list;
1103 2 : ibWr->opcode = wr->op;
1104 2 : ibWr->send_flags = (unsigned int)wr->sendFlags;
1105 2 : ibWr->imm_data = htobe32(wr->immData);
1106 :
1107 2 : ibWr->num_sge = 1; /* only support one sge */
1108 2 : ibWr->wr_id = wr->wrId;
1109 2 : ibWr->wr.rdma.rkey = remMrCb->mrInfo.rkey;
1110 2 : ibWr->wr.rdma.remote_addr = wr->dstAddr;
1111 2 : }
1112 :
1113 13 : STATIC int RsBuildUpWrList(struct WrInfo *wrList, struct RsQpCb *qpCb, struct ibv_sge *list, struct ibv_send_wr *ibWr,
1114 : unsigned int i)
1115 : {
1116 13 : struct RsMrCb *mrCb = NULL;
1117 13 : struct RsMrCb *remMrCb = NULL;
1118 13 : CHK_PRT_RETURN(wrList[i].memList.len > RS_SGLIST_LEN_MAX,
1119 : hccp_err("sg list len is more than 2G, len[%u]", wrList[i].memList.len), -EINVAL);
1120 :
1121 12 : hccp_dbg("qpn %d, bufList[0].addr is 0x%llx", qpCb->ibQp->qp_num, wrList[i].memList.addr);
1122 12 : if (RsGetMrcb(qpCb, wrList[i].memList.addr, &mrCb, &qpCb->mrList)) {
1123 2 : hccp_err("qpn %d, bufList[0].addr[0x%llx] len[0x%x] is invalid.", qpCb->ibQp->qp_num, wrList[i].memList.addr,
1124 : wrList[i].memList.len);
1125 2 : return -EFAULT;
1126 : }
1127 :
1128 : // send op no need to check & get remote mr
1129 10 : if (wrList[i].op != IBV_WR_SEND && wrList[i].op != IBV_WR_SEND_WITH_IMM) {
1130 10 : hccp_dbg("remote wr dst addr is 0x%llx", wrList[i].dstAddr);
1131 10 : if (RsGetMrcb(qpCb, wrList[i].dstAddr, &remMrCb, &qpCb->remMrList)) {
1132 8 : hccp_err("qpn %d, remote wr dst addr[0x%llx] len[0x%x] is invalid.", qpCb->ibQp->qp_num, wrList[i].dstAddr,
1133 : wrList[i].memList.len);
1134 8 : return -ENOENT;
1135 : }
1136 2 : RsWirteAndReadBuildUpWr(mrCb, remMrCb, &wrList[i], &list[i], &ibWr[i]);
1137 : } else {
1138 0 : RsSendBuildUpWr(mrCb, &wrList[i], &list[i], &ibWr[i]);
1139 : }
1140 :
1141 2 : return 0;
1142 : }
1143 :
1144 6 : STATIC int RsBuildUpWrListWithKey(struct WrInfo *wrList, struct ibv_sge *list, struct ibv_send_wr *ibWr, unsigned int i)
1145 : {
1146 6 : CHK_PRT_RETURN(wrList[i].memList.len > RS_SGLIST_LEN_MAX,
1147 : hccp_err("sg list len is more than 2G, len[%u]", wrList[i].memList.len), -EINVAL);
1148 :
1149 5 : BuildUpWrWithKey(&wrList[i], &list[i], &ibWr[i]);
1150 5 : return 0;
1151 : }
1152 :
1153 8 : STATIC int RsSendNormalWrlist(struct RsQpCb *qpCb, struct WrInfo *wrList, unsigned int sendNum,
1154 : unsigned int *completeNum, unsigned int keyFlag)
1155 : {
1156 : int ret;
1157 : unsigned int i, j;
1158 :
1159 8 : struct ibv_send_wr *badWr = NULL;
1160 8 : CHK_PRT_RETURN(sendNum > MAX_WR_NUM || sendNum == 0, hccp_err("send num[%u] is invalid!", sendNum), -EINVAL);
1161 8 : struct ibv_send_wr *ibWr = (struct ibv_send_wr *)calloc(sendNum, sizeof(struct ibv_send_wr));
1162 8 : CHK_PRT_RETURN(ibWr == NULL, hccp_err("calloc ib_wr failed!"), -ENOSPC);
1163 :
1164 8 : struct ibv_sge *list = (struct ibv_sge *)calloc(sendNum, sizeof(struct ibv_sge));
1165 8 : if (list == NULL) {
1166 0 : hccp_err("calloc list failed!");
1167 0 : ret = -ENOSPC;
1168 0 : goto alloc_fail;
1169 : }
1170 :
1171 10 : for (i = 0; i < sendNum; i++) {
1172 6 : ret = (keyFlag == 0) ? RsBuildUpWrList(wrList, qpCb, list, ibWr, i)
1173 8 : : RsBuildUpWrListWithKey(wrList, list, ibWr, i);
1174 8 : if (ret) {
1175 6 : goto input_err;
1176 : }
1177 2 : j = i + 1;
1178 2 : ibWr[i].next = (i < sendNum - 1) ? &ibWr[j] : NULL;
1179 : }
1180 :
1181 2 : ret = RsIbvPostSend(qpCb->ibQp, &ibWr[0], &badWr);
1182 2 : if (ret == 0) {
1183 2 : *completeNum = sendNum;
1184 0 : } else if (ret == -ENOMEM) {
1185 0 : *completeNum = (unsigned int)((void *)badWr - (void *)ibWr) / sizeof(struct ibv_send_wr);
1186 0 : hccp_dbg("post send wqe overflow, completeNum[%d]", *completeNum);
1187 : } else {
1188 0 : hccp_err("ibv_post_send failed, ret[%d]", ret);
1189 0 : *completeNum = 0;
1190 : }
1191 2 : qpCb->sendWrNum = qpCb->sendWrNum + (*completeNum);
1192 :
1193 8 : input_err:
1194 8 : free(list);
1195 8 : list = NULL;
1196 8 : alloc_fail:
1197 8 : free(ibWr);
1198 8 : ibWr = NULL;
1199 8 : return (ret == -ENOMEM) ? 0 : ret;
1200 : }
1201 :
1202 11 : STATIC int RsSendExpWrlist(struct RsQpCb *qpCb, struct WrInfo *wrList, unsigned int sendNum, struct SendWrRsp *wrRsp,
1203 : unsigned int *completeNum, unsigned int keyFlag)
1204 : {
1205 11 : struct ibv_post_send_ext_attr extAttr = {0};
1206 11 : struct ibv_post_send_ext_resp extRsp = {0};
1207 11 : struct ibv_send_wr *badWr = NULL;
1208 11 : struct wr_exp_rsp expRsp = {0};
1209 11 : struct ibv_send_wr ibWr = {0};
1210 11 : struct ibv_sge list = {0};
1211 : unsigned int i;
1212 11 : int ret = 0;
1213 :
1214 14 : for (i = 0; i < sendNum; i++) {
1215 : // reuse code: only need to build up one wr once a time
1216 7 : ret = (keyFlag == 0) ? RsBuildUpWrList(&wrList[i], qpCb, &list, &ibWr, 0)
1217 11 : : RsBuildUpWrListWithKey(&wrList[i], &list, &ibWr, 0);
1218 11 : if (ret != 0) {
1219 6 : hccp_err("qpn:%u key_flag:%u build_up_wr i:%u failed, ret:%d", qpCb->ibQp->qp_num, keyFlag, i, ret);
1220 6 : break;
1221 : }
1222 :
1223 5 : if (wrList[i].op == RA_WR_RDMA_WRITE_WITH_NOTIFY || wrList[i].op == RA_WR_RDMA_REDUCE_WRITE ||
1224 5 : wrList[i].op == RA_WR_RDMA_REDUCE_WRITE_WITH_NOTIFY) {
1225 1 : ibWr.imm_data = htobe32((wrList[i].aux.notifyOffset & WRITE_NOTIFY_OFFSET_MASK) |
1226 : WRITE_NOTIFY_VALUE_RECORD);
1227 1 : extAttr.reduce_op = wrList[i].aux.reduceType;
1228 1 : extAttr.reduce_type = wrList[i].aux.dataType;
1229 1 : ret = RsIbvExtPostSend(qpCb->ibQp, &ibWr, &badWr, &extAttr, &extRsp);
1230 1 : expRsp.wqe_index = extRsp.wqe_index;
1231 1 : expRsp.db_info = extRsp.db_info;
1232 1 : hccp_dbg("rs_ibv_ext_post_send, op = [%x], immData = [0x%lx], reduce_op = [%d],reduceType = [%d]",
1233 : ibWr.opcode, ibWr.imm_data, extAttr.reduce_op, extAttr.reduce_type);
1234 : } else {
1235 4 : ret = RsIbvExpPostSend(qpCb->ibQp, &ibWr, &badWr, &expRsp);
1236 4 : hccp_dbg("rs_ibv_exp_post_send, op = [%x], remoteAddr = [0x%llx], size = [%d]", ibWr.opcode,
1237 : ibWr.wr.rdma.remote_addr, ibWr.sg_list->length);
1238 : }
1239 :
1240 5 : if (ret != 0) {
1241 2 : if (ret == -ENOMEM) {
1242 1 : hccp_warn("qpn:%u rs_ibv_exp_post_send i:%u unsuccessful, ret %d", qpCb->ibQp->qp_num, i, ret);
1243 : } else {
1244 1 : hccp_err("qpn:%u rs_ibv_exp_post_send i:%u failed, ret %d", qpCb->ibQp->qp_num, i, ret);
1245 : }
1246 2 : break;
1247 : }
1248 :
1249 3 : qpCb->sendWrNum++;
1250 :
1251 3 : if (qpCb->qpMode == RA_RS_GDR_TMPL_QP_MODE) {
1252 1 : wrRsp[i].wqeTmp.sqIndex = (unsigned int)qpCb->sqIndex;
1253 1 : wrRsp[i].wqeTmp.wqeIndex = expRsp.wqe_index;
1254 2 : } else if (qpCb->qpMode == RA_RS_OP_QP_MODE || qpCb->qpMode == RA_RS_GDR_ASYN_QP_MODE) {
1255 2 : wrRsp[i].db.dbIndex = (unsigned int)qpCb->dbIndex;
1256 2 : wrRsp[i].db.dbInfo = expRsp.db_info;
1257 : }
1258 : }
1259 :
1260 11 : hccp_dbg("complete_num[%d], ret[%d]", i, ret);
1261 11 : *completeNum = i;
1262 11 : return (ret == -ENOMEM) ? 0 : ret;
1263 : }
1264 :
1265 15 : RS_ATTRI_VISI_DEF int RsSendWrlist(struct RsWrlistBaseInfo baseInfo, struct WrInfo *wrList, unsigned int sendNum,
1266 : struct SendWrRsp *wrRsp, unsigned int *completeNum)
1267 : {
1268 : int ret;
1269 : unsigned int phyId, rdevIndex, qpn;
1270 15 : struct RsQpCb *qpCb = NULL;
1271 :
1272 15 : RS_CHECK_POINTER_NULL_RETURN_INT(wrList);
1273 15 : RS_CHECK_POINTER_NULL_RETURN_INT(wrRsp);
1274 15 : CHK_PRT_RETURN(sendNum > MAX_WR_NUM || sendNum == 0 || baseInfo.phyId >= RS_MAX_DEV_NUM,
1275 : hccp_err("send_num[%u] or phyId:%u >= [%d], is invalid", sendNum, baseInfo.phyId, RS_MAX_DEV_NUM), -EINVAL);
1276 :
1277 14 : phyId = baseInfo.phyId;
1278 14 : rdevIndex = baseInfo.rdevIndex;
1279 14 : qpn = baseInfo.qpn;
1280 :
1281 14 : CHK_PRT_RETURN(RsQpn2qpcb(phyId, rdevIndex, qpn, &qpCb), hccp_err("rs_qpn2qpcb failed, physical id[%u]", phyId),
1282 : -EACCES);
1283 :
1284 : // only allow normal qp to call this func when ai_op_support not set
1285 13 : if (qpCb->qpMode == RA_RS_NOR_QP_MODE && qpCb->aiOpSupport == 0) {
1286 6 : ret = RsSendNormalWrlist(qpCb, wrList, sendNum, completeNum, baseInfo.keyFlag);
1287 : } else {
1288 7 : ret = RsSendExpWrlist(qpCb, wrList, sendNum, wrRsp, completeNum, baseInfo.keyFlag);
1289 : }
1290 13 : return ret;
1291 : }
1292 :
1293 0 : RS_ATTRI_VISI_DEF int RsRecvWrlist(struct RsWrlistBaseInfo baseInfo, struct RecvWrlistData *wr, unsigned int recvNum,
1294 : unsigned int *completeNum)
1295 : {
1296 0 : struct RsQpCb *qpCb = NULL;
1297 :
1298 0 : RS_CHECK_POINTER_NULL_RETURN_INT(wr);
1299 0 : CHK_PRT_RETURN(recvNum > MAX_WR_NUM || recvNum == 0 || baseInfo.phyId >= RS_MAX_DEV_NUM,
1300 : hccp_err("recv_num[%u] or phyId:%u >= [%d], is invalid", recvNum, baseInfo.phyId, RS_MAX_DEV_NUM), -EINVAL);
1301 :
1302 0 : CHK_PRT_RETURN(RsQpn2qpcb(baseInfo.phyId, baseInfo.rdevIndex, baseInfo.qpn, &qpCb),
1303 : hccp_err("rs_qpn2qpcb failed, physical id[%u]", baseInfo.phyId), -EACCES);
1304 :
1305 0 : return RsDrvPostRecv(qpCb, wr, recvNum, completeNum);
1306 : }
1307 :
1308 0 : RS_ATTRI_VISI_DEF int RsSetHostPid(uint32_t phyId, pid_t hostPid, const char *pidSign)
1309 : {
1310 : int ret;
1311 : unsigned int chipId;
1312 0 : struct rs_cb *rsCb = NULL;
1313 :
1314 0 : RS_CHECK_POINTER_NULL_RETURN_INT(pidSign);
1315 0 : CHK_PRT_RETURN(phyId >= RS_MAX_DEV_NUM, hccp_err("rs_set_host_pid rs set param error ! phyId:%u", phyId), -EINVAL);
1316 :
1317 0 : ret = rsGetLocalDevIDByHostDevID(phyId, &chipId);
1318 0 : CHK_PRT_RETURN(ret, hccp_err("rs_set_host_pid rsGetLocalDevIDByHostDevID phyId invalid, ret %d", ret), ret);
1319 :
1320 0 : hccp_info("phyId[%u] host_pid[%d]", chipId, hostPid);
1321 :
1322 0 : ret = RsDev2rscb(chipId, &rsCb, false);
1323 0 : CHK_PRT_RETURN(ret, hccp_err("get rs cb failed, chipId:%u", chipId), ret);
1324 :
1325 0 : rsCb->pRsSign.tgid = hostPid;
1326 0 : ret = strcpy_s(rsCb->pRsSign.sign, PROCESS_RS_SIGN_LENGTH, pidSign);
1327 0 : CHK_PRT_RETURN(ret, hccp_err("copy sign failed, ret %d", ret), -ESAFEFUNC);
1328 :
1329 0 : return 0;
1330 : }
1331 :
1332 6 : RS_ATTRI_VISI_DEF int RsRdevGetPortStatus(unsigned int phyId, unsigned int rdevIndex, enum PortStatus *status)
1333 : {
1334 6 : struct ibv_port_attr portAttr = {0};
1335 6 : struct RsRdevCb *rdevCb = NULL;
1336 : unsigned int chipId;
1337 : int ret;
1338 :
1339 6 : CHK_PRT_RETURN(phyId >= RS_MAX_DEV_NUM, hccp_err("phyId:%u >= [%d], is invalid", phyId, RS_MAX_DEV_NUM), -EINVAL);
1340 4 : CHK_PRT_RETURN(status == NULL, hccp_err("param err! status is NULL"), -EINVAL);
1341 :
1342 3 : ret = rsGetLocalDevIDByHostDevID(phyId, &chipId);
1343 3 : CHK_PRT_RETURN(ret, hccp_err("rsGetLocalDevIDByHostDevID failed, phyId[%u] invalid, ret %d", phyId, ret), ret);
1344 :
1345 3 : ret = RsRdev2rdevCb(chipId, rdevIndex, &rdevCb);
1346 3 : CHK_PRT_RETURN(ret != 0 || rdevCb == NULL, hccp_err("rs_rdev2rdev_cb for chip_id[%u] failed, ret %d", chipId, ret),
1347 : ret);
1348 :
1349 2 : ret = RsIbvQueryPort(rdevCb->ibCtx, rdevCb->ibPort, &portAttr);
1350 2 : CHK_PRT_RETURN(ret, hccp_err("ibv_query_port failed ret[%d]", ret), -EOPENSRC);
1351 :
1352 1 : *status = portAttr.state == IBV_PORT_ACTIVE ? PORT_STATUS_ACTIVE : PORT_STATUS_DOWN;
1353 :
1354 1 : hccp_dbg("phyId:%u port_attr.state:%u status:%u", phyId, portAttr.state, *status);
1355 1 : return 0;
1356 : }
1357 :
1358 3 : RS_ATTRI_VISI_DEF int RsGetNotifyMrInfo(unsigned int phyId, unsigned int rdevIndex, struct MrInfoT *info)
1359 : {
1360 3 : struct RsRdevCb *rdevCb = NULL;
1361 3 : struct rs_cb *rsCb = NULL;
1362 : unsigned int chipId;
1363 : int ret;
1364 :
1365 3 : CHK_PRT_RETURN(phyId >= RS_MAX_DEV_NUM, hccp_err("phyId:%u >= [%d], is invalid", phyId, RS_MAX_DEV_NUM), -EINVAL);
1366 :
1367 2 : CHK_PRT_RETURN(info == NULL, hccp_err("param err! info is NULL"), -EINVAL);
1368 :
1369 1 : ret = rsGetLocalDevIDByHostDevID(phyId, &chipId);
1370 1 : CHK_PRT_RETURN(ret, hccp_err("phyId[%u] invalid, ret:%d", phyId, ret), ret);
1371 :
1372 1 : ret = RsDev2rscb(chipId, &rsCb, false);
1373 1 : CHK_PRT_RETURN(ret, hccp_err("get rs_cb failed, ret:%d", ret), -ENODEV);
1374 :
1375 1 : ret = RsGetRdevCb(rsCb, rdevIndex, &rdevCb);
1376 1 : CHK_PRT_RETURN(ret, hccp_err("rs_get_rdev_cb failed!, ret:%d, rdevIndex:%u", ret, rdevIndex), ret);
1377 :
1378 1 : info->addr = (void *)(uintptr_t)rdevCb->notifyVaBase;
1379 1 : info->size = rdevCb->notifySize;
1380 1 : info->access = rdevCb->notifyAccess;
1381 1 : info->lkey = rdevCb->notifyMr->lkey;
1382 :
1383 1 : return 0;
1384 : }
1385 :
1386 4 : RS_ATTRI_VISI_DEF int RsNotifyCfgSet(unsigned int phyId, unsigned long long va, unsigned long long size)
1387 : {
1388 : int ret;
1389 : unsigned int chipId;
1390 4 : struct rs_cb *rsCb = NULL;
1391 :
1392 4 : RS_CHECK_POINTER_NULL_RETURN_INT((void *)(uintptr_t)va);
1393 :
1394 4 : CHK_PRT_RETURN(phyId >= RS_MAX_DEV_NUM ||
1395 : (size != MAX_NOTIFY_SIZE_CLOUD && size != NOTIFY_NUM_MAX_V2 && size != NOTIFY_NUM_MAX_V3),
1396 : hccp_err("rs_notify_cfg_set rs set param error ! phyId[%u] size[%llu]", phyId, size), -EINVAL);
1397 :
1398 3 : ret = rsGetLocalDevIDByHostDevID(phyId, &chipId);
1399 3 : CHK_PRT_RETURN(ret, hccp_err("rs_notify_cfg_set phyId invalid, ret %d, phyId:%u", ret, phyId), ret);
1400 :
1401 1 : ret = RsDev2rscb(chipId, &rsCb, false);
1402 1 : CHK_PRT_RETURN(ret, hccp_err("get rs cb failed, chipId:%u", chipId), ret);
1403 :
1404 1 : rsCb->notifyVaBase = va;
1405 1 : rsCb->notifySize = size;
1406 :
1407 1 : return 0;
1408 : }
1409 :
1410 0 : RS_ATTRI_VISI_DEF int RsNotifyCfgGet(unsigned int phyId, unsigned long long *va, unsigned long long *size)
1411 : {
1412 : int ret;
1413 : unsigned int chipId;
1414 0 : struct rs_cb *rsCb = NULL;
1415 :
1416 0 : RS_CHECK_POINTER_NULL_RETURN_INT(va);
1417 0 : RS_CHECK_POINTER_NULL_RETURN_INT(size);
1418 :
1419 0 : CHK_PRT_RETURN(phyId >= RS_MAX_DEV_NUM, hccp_err("rs_notify_cfg_get rs set param error ! phyId:%u", phyId),
1420 : -EINVAL);
1421 :
1422 0 : ret = rsGetLocalDevIDByHostDevID(phyId, &chipId);
1423 0 : CHK_PRT_RETURN(ret, hccp_err("rs_notify_cfg_get phyId invalid, ret %d, phyId:%u", ret, phyId), ret);
1424 :
1425 0 : ret = RsDev2rscb(chipId, &rsCb, false);
1426 0 : CHK_PRT_RETURN(ret, hccp_err("get rs cb failed, chipId:%u", chipId), ret);
1427 :
1428 0 : *va = rsCb->notifyVaBase;
1429 0 : *size = rsCb->notifySize;
1430 :
1431 0 : return 0;
1432 : }
1433 :
1434 6 : RS_ATTRI_VISI_DEF int RsSetTsqpDepth(unsigned int phyId, unsigned int rdevIndex, unsigned int tempDepth,
1435 : unsigned int *qpNum)
1436 : {
1437 : #ifdef CUSTOM_INTERFACE
1438 6 : struct RsRdevCb *rdevCb = NULL;
1439 6 : unsigned int sqDepth = 0;
1440 6 : unsigned int chipId = 0;
1441 : int ret;
1442 :
1443 6 : if (!RsIsCustomInterfaceSupported()) {
1444 0 : return 0;
1445 : }
1446 6 : CHK_PRT_RETURN(phyId >= RS_MAX_DEV_NUM, hccp_err("rs_set_tsqp_depth param error ! phyId:%d", phyId), -EINVAL);
1447 :
1448 5 : CHK_PRT_RETURN(qpNum == NULL, hccp_err("rs_set_tsqp_depth qp_num is NULL, param error!"), -EINVAL);
1449 :
1450 4 : CHK_PRT_RETURN(tempDepth < RS_MIN_TEMPTH_DEPTH || tempDepth > RS_MAX_TEMPTH_DEPTH,
1451 : hccp_err("param error!"
1452 : "temp_depth[%u] can not smaller than [%d] or bigerr than [%d]",
1453 : tempDepth, RS_MIN_TEMPTH_DEPTH, RS_MAX_TEMPTH_DEPTH),
1454 : -EINVAL);
1455 :
1456 3 : ret = rsGetLocalDevIDByHostDevID(phyId, &chipId);
1457 3 : CHK_PRT_RETURN(ret, hccp_err("phyId[%u] invalid, ret %d", phyId, ret), ret);
1458 :
1459 2 : ret = RsRdev2rdevCb(chipId, rdevIndex, &rdevCb);
1460 2 : CHK_PRT_RETURN(ret || rdevCb == NULL,
1461 : hccp_err("rs_set_tsqp_depth rs_rdev2rdev_cb for chip_id[%u]"
1462 : "failed, ret %d",
1463 : chipId, ret),
1464 : ret);
1465 :
1466 1 : ret = RsRoceSetTsqpDepth(rdevCb->devName, rdevIndex, tempDepth, qpNum, &sqDepth);
1467 1 : CHK_PRT_RETURN(ret, hccp_err("rs_roce_set_tsqp_depth failed, ret %d, devName[%s]", ret, rdevCb->devName), ret);
1468 :
1469 0 : rdevCb->txDepth = sqDepth;
1470 0 : rdevCb->rxDepth = sqDepth;
1471 0 : rdevCb->qpMaxNum = *qpNum;
1472 : #endif
1473 0 : return 0;
1474 : }
1475 :
1476 5 : RS_ATTRI_VISI_DEF int RsGetTsqpDepth(unsigned int phyId, unsigned int rdevIndex, unsigned int *tempDepth,
1477 : unsigned int *qpNum)
1478 : {
1479 : #ifdef CUSTOM_INTERFACE
1480 5 : struct RsRdevCb *rdevCb = NULL;
1481 5 : unsigned int sqDepth = 0;
1482 5 : unsigned int chipId = 0;
1483 : int ret;
1484 :
1485 5 : if (!RsIsCustomInterfaceSupported()) {
1486 0 : return 0;
1487 : }
1488 5 : CHK_PRT_RETURN(phyId >= RS_MAX_DEV_NUM, hccp_err("param error ! phyId:%d", phyId), -EINVAL);
1489 :
1490 4 : CHK_PRT_RETURN(tempDepth == NULL || qpNum == NULL,
1491 : hccp_err("temp_depth or qp_num is NULL,"
1492 : "param error!"),
1493 : -EINVAL);
1494 :
1495 3 : ret = rsGetLocalDevIDByHostDevID(phyId, &chipId);
1496 3 : CHK_PRT_RETURN(ret, hccp_err("phyId[%u] invalid, ret %d", phyId, ret), ret);
1497 :
1498 2 : ret = RsRdev2rdevCb(chipId, rdevIndex, &rdevCb);
1499 2 : CHK_PRT_RETURN(ret != 0 || rdevCb == NULL,
1500 : hccp_err("rs_get_tsqp_depth rs_rdev2rdev_cb for chip_id[%u]"
1501 : "failed, ret %d",
1502 : chipId, ret),
1503 : ret);
1504 :
1505 1 : ret = RsRoceGetTsqpDepth(rdevCb->devName, rdevIndex, tempDepth, qpNum, &sqDepth);
1506 1 : CHK_PRT_RETURN(ret, hccp_err("rs_roce_get_tsqp_depth failed, ret %d, devName[%s]", ret, rdevCb->devName), ret);
1507 : #endif
1508 0 : return 0;
1509 : }
1510 :
1511 45 : STATIC void RsSetQpDepthAttr(struct RsRdevCb *rdevCb, struct RsQpCb *qpCb, struct RsQpNorm *qpNorm)
1512 : {
1513 45 : if (qpCb->qpMode == RA_RS_GDR_TMPL_QP_MODE) {
1514 41 : qpCb->txDepth = rdevCb->txDepth;
1515 41 : qpCb->rxDepth = rdevCb->rxDepth;
1516 : } else {
1517 4 : if (rdevCb->rsCb->hccpMode == NETWORK_OFFLINE) {
1518 2 : qpCb->txDepth = RS_QP_TX_DEPTH_OFFLINE;
1519 2 : qpCb->rxDepth = RS_QP_RX_DEPTH_OFFLINE;
1520 : } else {
1521 2 : qpCb->txDepth = RS_QP_TX_DEPTH_ONLINE;
1522 2 : qpCb->rxDepth = RS_QP_RX_DEPTH_ONLINE;
1523 : }
1524 : }
1525 :
1526 45 : if (qpNorm->isExp != 0 && qpNorm->qpMode != RA_RS_NOR_QP_MODE) {
1527 43 : if (rdevCb->rsCb->hccpMode == NETWORK_PEER_ONLINE) {
1528 2 : qpCb->txDepth = (qpCb->qpMode != RA_RS_GDR_TMPL_QP_MODE) ? RS_QP_TX_DEPTH_PEER_ONLINE : qpCb->txDepth;
1529 2 : qpCb->rxDepth = (qpCb->qpMode != RA_RS_GDR_TMPL_QP_MODE) ? RS_QP_TX_DEPTH_PEER_ONLINE : qpCb->rxDepth;
1530 : } else {
1531 41 : qpCb->txDepth = (qpCb->qpMode != RA_RS_GDR_TMPL_QP_MODE && qpCb->qpMode != RA_RS_GDR_ASYN_QP_MODE)
1532 : ? RS_QP_32K_DEPTH
1533 41 : : qpCb->txDepth;
1534 : }
1535 43 : qpCb->sendSgeNum = 1;
1536 43 : qpCb->recvSgeNum = 1;
1537 : } else {
1538 2 : if (rdevCb->rsCb->hccpMode == NETWORK_PEER_ONLINE) {
1539 0 : qpCb->txDepth = (qpCb->qpMode != RA_RS_GDR_TMPL_QP_MODE) ? RS_QP_TX_DEPTH_PEER_ONLINE : qpCb->txDepth;
1540 0 : qpCb->rxDepth = (qpCb->qpMode != RA_RS_GDR_TMPL_QP_MODE) ? RS_QP_TX_DEPTH_PEER_ONLINE : qpCb->rxDepth;
1541 : } else {
1542 2 : qpCb->txDepth = (qpCb->qpMode != RA_RS_GDR_TMPL_QP_MODE) ? RS_QP_TX_DEPTH : qpCb->txDepth;
1543 2 : qpCb->rxDepth = (qpCb->qpMode != RA_RS_GDR_TMPL_QP_MODE) ? RS_QP_TX_DEPTH : qpCb->rxDepth;
1544 : }
1545 2 : qpCb->sendSgeNum = RS_QP_ATTR_MAX_SEND_SGE;
1546 2 : qpCb->recvSgeNum = 1;
1547 : }
1548 45 : }
1549 :
1550 45 : STATIC int RsQpcbInit(struct RsRdevCb *rdevCb, struct RsQpCb *qpCb, struct RsQpNorm *qpNorm)
1551 : {
1552 : #define RS_DRV_CQ_DEPTH 16384
1553 : #define RS_DRV_CQ_128_DEPTH 128
1554 : #define RS_DRV_CQ_8K_DEPTH 8192
1555 : #define RS_DRV_CQ_32K_DEPTH 32768
1556 45 : int qpMode = qpNorm->qpMode;
1557 : int ret;
1558 :
1559 45 : qpCb->rdevCb = rdevCb;
1560 45 : RS_INIT_LIST_HEAD(&qpCb->mrList);
1561 45 : RS_INIT_LIST_HEAD(&qpCb->remMrList);
1562 :
1563 45 : qpCb->qpMode = qpMode;
1564 45 : qpCb->eqNum = 0;
1565 45 : qpCb->numRecvCqEvents = 0;
1566 45 : qpCb->numSendCqEvents = 0;
1567 45 : qpCb->state = RS_QP_STATUS_DISCONNECT;
1568 45 : qpCb->ibPd = rdevCb->ibPd;
1569 :
1570 : // cq attr
1571 45 : if (qpNorm->isExt == 1) {
1572 : // update TEMP & ASYN mode cq depth from 32K to 8K due to memory issue
1573 2 : qpCb->sendCqDepth = (qpMode != RA_RS_GDR_TMPL_QP_MODE && qpMode != RA_RS_GDR_ASYN_QP_MODE) ? RS_DRV_CQ_32K_DEPTH
1574 4 : : RS_DRV_CQ_8K_DEPTH;
1575 2 : qpCb->recvCqDepth = RS_DRV_CQ_128_DEPTH;
1576 : } else {
1577 43 : qpCb->sendCqDepth = RS_DRV_CQ_DEPTH;
1578 43 : qpCb->recvCqDepth = RS_DRV_CQ_DEPTH;
1579 : }
1580 :
1581 : // qp attr
1582 45 : RsSetQpDepthAttr(rdevCb, qpCb, qpNorm);
1583 :
1584 45 : qpCb->memAlign = qpNorm->memAlign;
1585 :
1586 45 : qpCb->channel = RsIbvCreateCompChannel(rdevCb->ibCtx);
1587 45 : CHK_PRT_RETURN(qpCb->channel == NULL, hccp_err("ibv_create_comp_channel failed! errno(%d)", errno), -EINVAL);
1588 44 : qpCb->qosAttr.tc = (RS_ROCE_DSCP_33 & RS_DSCP_MASK) << RS_DSCP_OFF;
1589 44 : qpCb->qosAttr.sl = RS_ROCE_4_SL;
1590 44 : qpCb->timeout = RS_QP_ATTR_TIMEOUT;
1591 44 : qpCb->retryCnt = RS_QP_ATTR_RETRY_CNT;
1592 :
1593 44 : ret = RsEpollCtl(rdevCb->rsCb->connCb.epollfd, EPOLL_CTL_ADD, qpCb->channel->fd, EPOLLIN | EPOLLRDHUP);
1594 : #ifndef CA_CONFIG_LLT
1595 : if (ret) {
1596 : RsIbvDestroyCompChannel(qpCb->channel);
1597 : hccp_err("add channel fd failed ret %d", ret);
1598 : return ret;
1599 : }
1600 : #endif
1601 44 : return 0;
1602 : }
1603 :
1604 48 : STATIC int RsQpcbDeinit(struct RsRdevCb *rdevCb, struct RsQpCb *qpCb)
1605 : {
1606 : int ret;
1607 :
1608 48 : if (qpCb == NULL || qpCb->channel == NULL) {
1609 0 : hccp_err("qp_cb or qp_cb->channel is NULL!");
1610 0 : return -EINVAL;
1611 : }
1612 :
1613 48 : ret = RsEpollCtl(rdevCb->rsCb->connCb.epollfd, EPOLL_CTL_DEL, qpCb->channel->fd, EPOLLIN | EPOLLRDHUP);
1614 : #ifndef CA_CONFIG_LLT
1615 : if (ret) {
1616 : hccp_err("del channel fd failed ret %d", ret);
1617 : }
1618 : #endif
1619 :
1620 48 : if (qpCb->channel != NULL) {
1621 48 : RsIbvDestroyCompChannel(qpCb->channel);
1622 48 : qpCb->channel = NULL;
1623 : }
1624 : #ifndef CA_CONFIG_LLT
1625 : return ret;
1626 : #else
1627 48 : return 0;
1628 : #endif
1629 : }
1630 :
1631 32 : STATIC int RsQpNotifyMr(struct RsRdevCb *rdevCb, struct RsQpCb *qpCb, uint32_t *qpn)
1632 : {
1633 : int ret;
1634 32 : struct RsMrCb *notifyMrNode = NULL;
1635 :
1636 32 : ret = RsCallocMr(1, ¬ifyMrNode);
1637 32 : CHK_PRT_RETURN(ret, hccp_err("notify_mr_cb malloc failed"), ret);
1638 :
1639 32 : RS_PTHREAD_MUTEX_LOCK(&rdevCb->rdevMutex);
1640 32 : RsListAddTail(&qpCb->list, &rdevCb->qpList);
1641 32 : RS_PTHREAD_MUTEX_ULOCK(&rdevCb->rdevMutex);
1642 :
1643 32 : if (rdevCb->notifyType != NO_USE) {
1644 32 : notifyMrNode->qpCb = qpCb;
1645 32 : notifyMrNode->ibMr = rdevCb->notifyMr;
1646 32 : notifyMrNode->mrInfo.addr = rdevCb->notifyVaBase;
1647 32 : notifyMrNode->mrInfo.len = rdevCb->notifySize;
1648 32 : notifyMrNode->mrInfo.rkey = notifyMrNode->ibMr->rkey;
1649 : } else {
1650 0 : notifyMrNode->qpCb = qpCb;
1651 0 : notifyMrNode->ibMr = NULL;
1652 : }
1653 :
1654 32 : RS_PTHREAD_MUTEX_LOCK(&qpCb->qpMutex);
1655 32 : RsListAddTail(¬ifyMrNode->list, &qpCb->mrList);
1656 32 : RS_PTHREAD_MUTEX_ULOCK(&qpCb->qpMutex);
1657 32 : rdevCb->qpCnt++;
1658 32 : *qpn = qpCb->ibQp->qp_num;
1659 :
1660 32 : hccp_info("rs qp %d create OK!", *qpn);
1661 :
1662 32 : return 0;
1663 : }
1664 :
1665 53 : STATIC int RsQpQueryInfo(unsigned int phyId, unsigned int rdevIndex, struct RsRdevCb **rdevCb, int qpMode)
1666 : {
1667 : int ret;
1668 : unsigned int chipId;
1669 53 : struct rs_cb *rsCb = NULL;
1670 :
1671 53 : CHK_PRT_RETURN(phyId >= RS_MAX_DEV_NUM, hccp_err("rs_qp_query_info rs set param error! phyId:%u", phyId), -EINVAL);
1672 :
1673 53 : ret = rsGetLocalDevIDByHostDevID(phyId, &chipId);
1674 53 : CHK_PRT_RETURN(ret, hccp_err("rs_qp_query_info phyId[%u] invalid, ret:%d", phyId, ret), ret);
1675 :
1676 52 : ret = RsDev2rscb(chipId, &rsCb, false);
1677 52 : CHK_PRT_RETURN(ret, hccp_err("rs_qp_query_info get rs_cb failed, ret:%d", ret), -ENODEV);
1678 :
1679 52 : ret = RsGetRdevCb(rsCb, rdevIndex, rdevCb);
1680 52 : CHK_PRT_RETURN(ret, hccp_err("rs_get_rdev_cb failed! ret:%d, rdevIndex:%u", ret, rdevIndex), ret);
1681 :
1682 52 : if (qpMode == RA_RS_GDR_TMPL_QP_MODE) {
1683 46 : CHK_PRT_RETURN((*rdevCb)->qpCnt >= (*rdevCb)->qpMaxNum,
1684 : hccp_err("Exceeded the maximum QP limit(%u)", (*rdevCb)->qpMaxNum), -EINVAL);
1685 : } else {
1686 6 : CHK_PRT_RETURN((*rdevCb)->qpCnt >= RS_QP_NUM_MAX,
1687 : hccp_err("Exceeded the maximum QP limit(%u)", (*rdevCb)->qpCnt), -EINVAL);
1688 : }
1689 :
1690 50 : return 0;
1691 : }
1692 :
1693 50 : STATIC int RsInitMemPool(struct RsQpCb *qpCb)
1694 : {
1695 50 : struct roce_mem_cq_qp_attr memAttr = {0};
1696 : int ret;
1697 :
1698 50 : if ((qpCb->qpMode != RA_RS_OP_QP_MODE && qpCb->qpMode != RA_RS_OP_QP_MODE_EXT) ||
1699 4 : qpCb->memAlign != LITE_ALIGN_2MB) {
1700 49 : return 0;
1701 : }
1702 :
1703 : // init mem_pool and store mem_data in mem_resp
1704 1 : memAttr.mem_align = qpCb->memAlign;
1705 1 : memAttr.send_qp_depth = qpCb->txDepth;
1706 1 : memAttr.send_cq_depth = (unsigned int)qpCb->sendCqDepth;
1707 1 : memAttr.send_sge_num = qpCb->sendSgeNum;
1708 1 : memAttr.recv_qp_depth = qpCb->rxDepth;
1709 1 : memAttr.recv_cq_depth = (unsigned int)qpCb->recvCqDepth;
1710 1 : memAttr.recv_sge_num = qpCb->recvSgeNum;
1711 1 : memAttr.use_resv_mem = qpCb->useResvMem;
1712 1 : memAttr.resv_mem_pool_id = qpCb->resvMemPoolId;
1713 1 : memAttr.ctx = qpCb->rdevCb->ibCtx;
1714 :
1715 1 : ret = RsRoceInitMemPool(&memAttr, &qpCb->memResp.memData, qpCb->rdevCb->rsCb->chipId);
1716 1 : if (ret != 0) {
1717 1 : hccp_err("rs_roce_init_mem_pool failed, ret=%d, chipId=%u", ret, qpCb->rdevCb->rsCb->chipId);
1718 : }
1719 1 : return ret;
1720 : }
1721 :
1722 50 : STATIC void RsDeinitMemPool(struct RsQpCb *qpCb)
1723 : {
1724 50 : if ((qpCb->qpMode != RA_RS_OP_QP_MODE && qpCb->qpMode != RA_RS_OP_QP_MODE_EXT) ||
1725 4 : qpCb->memAlign != LITE_ALIGN_2MB) {
1726 49 : return;
1727 : }
1728 :
1729 1 : (void)RsRoceDeinitMemPool(qpCb->memResp.memData.mem_idx);
1730 : }
1731 :
1732 46 : STATIC int RsAllocQpcb(struct RsRdevCb *rdevCb, struct RsQpCb **qpCb, struct RsQpNorm *qpNorm)
1733 : {
1734 : int ret;
1735 :
1736 46 : ret = RsCallocQpcb(1, qpCb);
1737 46 : CHK_PRT_RETURN(ret, hccp_err("alloc mem for qp_cb failed, ret:%d errno:%d", ret, errno), -ENOMEM);
1738 :
1739 45 : ret = pthread_mutex_init(&(*qpCb)->qpMutex, NULL);
1740 45 : if (ret) {
1741 0 : hccp_err("pthread_mutex_init failed, ret %d", ret);
1742 0 : goto qp_mutex_init_err;
1743 : }
1744 :
1745 45 : ret = pthread_mutex_init(&(*qpCb)->cqeErrInfo.mutex, NULL);
1746 45 : if (ret) {
1747 0 : hccp_err("pthread_mutex_init failed, ret %d", ret);
1748 0 : goto cqe_mutex_init_err;
1749 : }
1750 :
1751 45 : ret = RsQpcbInit(rdevCb, *qpCb, qpNorm);
1752 45 : if (ret) {
1753 1 : hccp_err("create qp tx rx failed ret %d", ret);
1754 1 : goto rs_qpcb_init_err;
1755 : }
1756 :
1757 44 : ret = RsInitMemPool(*qpCb);
1758 44 : if (ret) {
1759 0 : hccp_err("init mem pool failed ret %d", ret);
1760 0 : goto rs_init_mem_err;
1761 : }
1762 :
1763 44 : ret = RsDrvCreateCq(*qpCb, qpNorm->isExt);
1764 44 : if (ret) {
1765 1 : hccp_err("create cq failed ret %d", ret);
1766 1 : goto create_cq_err;
1767 : }
1768 :
1769 43 : return 0;
1770 :
1771 1 : create_cq_err:
1772 1 : RsDeinitMemPool(*qpCb);
1773 :
1774 1 : rs_init_mem_err:
1775 1 : RsQpcbDeinit(rdevCb, *qpCb);
1776 :
1777 2 : rs_qpcb_init_err:
1778 2 : pthread_mutex_destroy(&(*qpCb)->cqeErrInfo.mutex);
1779 :
1780 2 : cqe_mutex_init_err:
1781 2 : pthread_mutex_destroy(&(*qpCb)->qpMutex);
1782 :
1783 2 : qp_mutex_init_err:
1784 2 : free(*qpCb);
1785 2 : *qpCb = NULL;
1786 :
1787 2 : return ret;
1788 : }
1789 :
1790 15 : STATIC void RsFreeQpcb(struct RsRdevCb *rdevCb, struct RsQpCb *qpCb)
1791 : {
1792 15 : RsDrvDestroyCq(qpCb);
1793 15 : RsDeinitMemPool(qpCb);
1794 15 : (void)RsQpcbDeinit(rdevCb, qpCb);
1795 15 : pthread_mutex_destroy(&qpCb->cqeErrInfo.mutex);
1796 15 : pthread_mutex_destroy(&qpCb->qpMutex);
1797 15 : free(qpCb);
1798 15 : qpCb = NULL;
1799 15 : }
1800 :
1801 48 : RS_ATTRI_VISI_DEF int RsQpCreate(unsigned int phyId, unsigned int rdevIndex, struct RsQpNorm qpNorm,
1802 : struct RsQpResp *qpResp)
1803 : {
1804 48 : struct RsRdevCb *rdevCb = NULL;
1805 48 : struct RsQpCb *qpCb = NULL;
1806 : int ret;
1807 :
1808 48 : RS_QP_PARA_CHECK(phyId);
1809 47 : CHK_PRT_RETURN(qpResp == NULL, hccp_err("qp_resp is NULL!"), -EINVAL);
1810 :
1811 47 : ret = RsQpQueryInfo(phyId, rdevIndex, &rdevCb, qpNorm.qpMode);
1812 47 : CHK_PRT_RETURN(ret, hccp_err("query qp info failed:%d", ret), ret);
1813 :
1814 46 : ret = RsAllocQpcb(rdevCb, &qpCb, &qpNorm);
1815 46 : CHK_PRT_RETURN(ret, hccp_err("alloc mem for qp_cb failed, ret:%d", ret), ret);
1816 :
1817 43 : ret = RsDrvQpCreate(qpCb, &qpNorm);
1818 43 : if (ret) {
1819 14 : hccp_err("create drv qp create failed:%d", ret);
1820 14 : goto create_qp_err;
1821 : }
1822 :
1823 29 : ret = ibv_req_notify_cq(qpCb->ibSendCq, 0);
1824 29 : if (ret) {
1825 1 : hccp_err("Couldn't request send CQ notification, ret:%d", ret);
1826 1 : ret = -EOPENSRC;
1827 1 : goto ret_noritfy_cq;
1828 : }
1829 :
1830 28 : ret = ibv_req_notify_cq(qpCb->ibRecvCq, 0);
1831 28 : if (ret) {
1832 0 : hccp_err("Couldn't request recv CQ notification, ret:%d", ret);
1833 0 : ret = -EOPENSRC;
1834 0 : goto ret_noritfy_cq;
1835 : }
1836 :
1837 28 : ret = RsQpNotifyMr(rdevCb, qpCb, &qpResp->qpn); // alloc mr
1838 28 : if (ret) {
1839 0 : hccp_err("store qp notify mr failed:%d", ret);
1840 0 : goto ret_noritfy_cq;
1841 : }
1842 :
1843 28 : if (qpNorm.isExp) {
1844 26 : qpCb->isExp = RS_IS_EXP;
1845 : } else {
1846 2 : qpCb->isExp = RS_NOT_EXP;
1847 : }
1848 :
1849 28 : qpResp->qpn = (unsigned int)qpCb->qpInfoLo.qpn;
1850 28 : qpResp->gidIdx = (unsigned int)qpCb->qpInfoLo.gidIdx;
1851 28 : qpResp->psn = (unsigned int)qpCb->qpInfoLo.psn;
1852 28 : qpResp->gid = qpCb->qpInfoLo.gid;
1853 :
1854 28 : return 0;
1855 :
1856 1 : ret_noritfy_cq:
1857 1 : RsDrvQpDestroy(qpCb);
1858 :
1859 15 : create_qp_err:
1860 15 : RsFreeQpcb(rdevCb, qpCb);
1861 15 : return ret;
1862 : }
1863 :
1864 4 : STATIC int RsQpcbInitWithAttrs(struct RsRdevCb *rdevCb, struct RsQpCb *qpCb, struct RsQpNormWithAttrs *qpNorm)
1865 : {
1866 : int ret;
1867 :
1868 4 : qpCb->rdevCb = rdevCb;
1869 4 : RS_INIT_LIST_HEAD(&qpCb->mrList);
1870 4 : RS_INIT_LIST_HEAD(&qpCb->remMrList);
1871 :
1872 4 : qpCb->qpMode = qpNorm->extAttrs.qpMode;
1873 4 : qpCb->numRecvCqEvents = 0;
1874 4 : qpCb->numSendCqEvents = 0;
1875 4 : qpCb->state = RS_QP_STATUS_DISCONNECT;
1876 4 : qpCb->ibPd = rdevCb->ibPd;
1877 :
1878 4 : qpCb->txDepth = qpNorm->extAttrs.qpAttr.cap.max_send_wr;
1879 4 : qpCb->rxDepth = qpNorm->extAttrs.qpAttr.cap.max_send_wr;
1880 4 : qpCb->sendSgeNum = qpNorm->extAttrs.qpAttr.cap.max_send_sge;
1881 4 : qpCb->recvSgeNum = qpNorm->extAttrs.qpAttr.cap.max_recv_sge;
1882 4 : qpCb->sendCqDepth = qpNorm->extAttrs.cqAttr.sendCqDepth;
1883 4 : qpCb->recvCqDepth = qpNorm->extAttrs.cqAttr.recvCqDepth;
1884 4 : qpCb->memAlign = qpNorm->extAttrs.memAlign;
1885 :
1886 4 : qpCb->channel = RsIbvCreateCompChannel(rdevCb->ibCtx);
1887 4 : CHK_PRT_RETURN(qpCb->channel == NULL, hccp_err("ibv_create_comp_channel failed! errno(%d)", errno), -EINVAL);
1888 4 : qpCb->qosAttr.tc = (RS_ROCE_DSCP_33 & RS_DSCP_MASK) << RS_DSCP_OFF;
1889 4 : qpCb->qosAttr.sl = RS_ROCE_4_SL;
1890 4 : qpCb->timeout = RS_QP_ATTR_TIMEOUT;
1891 4 : qpCb->retryCnt = RS_QP_ATTR_RETRY_CNT;
1892 :
1893 4 : qpCb->udpSport = qpNorm->extAttrs.udpSport;
1894 :
1895 4 : qpCb->aiOpSupport = qpNorm->aiOpSupport;
1896 4 : qpCb->grpId = rdevCb->rsCb->grpId;
1897 4 : qpCb->cqCstmFlag = qpNorm->extAttrs.dataPlaneFlag.bs.cqCstm;
1898 4 : qpCb->useResvMem = qpNorm->extAttrs.cstmFlag.bs.useResvMem;
1899 4 : qpCb->resvMemPoolId = qpNorm->extAttrs.resvMemPoolId;
1900 :
1901 4 : ret = RsEpollCtl(rdevCb->rsCb->connCb.epollfd, EPOLL_CTL_ADD, qpCb->channel->fd, EPOLLIN | EPOLLRDHUP);
1902 : #ifndef CA_CONFIG_LLT
1903 : if (ret) {
1904 : RsIbvDestroyCompChannel(qpCb->channel);
1905 : hccp_err("add channel fd failed ret %d", ret);
1906 : return ret;
1907 : }
1908 : #endif
1909 4 : return 0;
1910 : }
1911 :
1912 4 : STATIC int RsAllocQpcbWithAttrs(struct RsRdevCb *rdevCb, struct RsQpCb **qpCb, struct RsQpNormWithAttrs *qpNorm)
1913 : {
1914 : int ret;
1915 :
1916 4 : ret = RsCallocQpcb(1, qpCb);
1917 4 : CHK_PRT_RETURN(ret, hccp_err("alloc mem for qp_cb failed, ret:%d errno:%d", ret, errno), -ENOMEM);
1918 :
1919 4 : ret = pthread_mutex_init(&(*qpCb)->qpMutex, NULL);
1920 4 : if (ret) {
1921 0 : hccp_err("pthread_mutex_init failed, ret %d", ret);
1922 0 : goto qp_mutex_init_err;
1923 : }
1924 :
1925 4 : ret = pthread_mutex_init(&(*qpCb)->cqeErrInfo.mutex, NULL);
1926 4 : if (ret) {
1927 0 : hccp_err("pthread_mutex_init failed, ret %d", ret);
1928 0 : goto cqe_mutex_init_err;
1929 : }
1930 :
1931 4 : ret = RsQpcbInitWithAttrs(rdevCb, *qpCb, qpNorm);
1932 4 : if (ret) {
1933 0 : hccp_err("create qp tx rx failed ret %d", ret);
1934 0 : goto rs_qpcb_init_err;
1935 : }
1936 :
1937 4 : ret = RsInitMemPool(*qpCb);
1938 4 : if (ret) {
1939 0 : hccp_err("init mem pool failed ret %d", ret);
1940 0 : goto rs_init_mem_err;
1941 : }
1942 :
1943 4 : ret = RsDrvCreateCqWithAttrs(*qpCb, qpNorm->isExt, &qpNorm->extAttrs.cqAttr);
1944 4 : if (ret) {
1945 0 : hccp_err("create cq failed ret %d", ret);
1946 0 : goto create_cq_err;
1947 : }
1948 :
1949 4 : return 0;
1950 :
1951 0 : create_cq_err:
1952 0 : RsDeinitMemPool(*qpCb);
1953 :
1954 0 : rs_init_mem_err:
1955 0 : RsQpcbDeinit(rdevCb, *qpCb);
1956 :
1957 0 : rs_qpcb_init_err:
1958 0 : pthread_mutex_destroy(&(*qpCb)->cqeErrInfo.mutex);
1959 :
1960 0 : cqe_mutex_init_err:
1961 0 : pthread_mutex_destroy(&(*qpCb)->qpMutex);
1962 :
1963 0 : qp_mutex_init_err:
1964 0 : free(*qpCb);
1965 0 : *qpCb = NULL;
1966 :
1967 0 : return ret;
1968 : }
1969 :
1970 9 : STATIC int RsQpCheckQpNorm(struct RsQpNormWithAttrs *qpNorm, int *qpMode)
1971 : {
1972 9 : CHK_PRT_RETURN(qpNorm == NULL, hccp_err("qp_norm is NULL!"), -EINVAL);
1973 8 : CHK_PRT_RETURN(qpNorm->extAttrs.version != QP_CREATE_WITH_ATTR_VERSION,
1974 : hccp_err("attr version[%d] mismatch, expect [%d]", qpNorm->extAttrs.version, QP_CREATE_WITH_ATTR_VERSION),
1975 : -EINVAL);
1976 :
1977 7 : *qpMode = qpNorm->extAttrs.qpMode;
1978 7 : if (*qpMode < 0 || *qpMode >= RA_RS_ERR_QP_MODE) {
1979 1 : hccp_err("qp_mode[%d] must greater or equal to 0 and less than %d", *qpMode, RA_RS_ERR_QP_MODE);
1980 1 : return -EINVAL;
1981 : }
1982 :
1983 6 : if (*qpMode == RA_RS_OP_QP_MODE_EXT) {
1984 1 : *qpMode = RA_RS_OP_QP_MODE;
1985 : }
1986 :
1987 6 : qpNorm->extAttrs.qpMode = *qpMode;
1988 6 : return 0;
1989 : }
1990 :
1991 : #ifdef CUSTOM_INTERFACE
1992 8 : STATIC void RsQpPrepareCqDataPlaneInfo(struct ibv_cq *ibCq, struct AiDataPlaneCq *dataPlaneCq)
1993 : {
1994 8 : struct hns_roce_cq_data_plane_info cqInfo = {0};
1995 :
1996 8 : (void)RsRoceGetCqDataPlaneInfo(ibCq, &cqInfo);
1997 8 : dataPlaneCq->cqn = cqInfo.cqn;
1998 8 : dataPlaneCq->bufAddr = cqInfo.buf_addr;
1999 8 : dataPlaneCq->cqeSize = cqInfo.cqe_size;
2000 8 : dataPlaneCq->depth = cqInfo.depth;
2001 8 : dataPlaneCq->headAddr = cqInfo.head_addr;
2002 8 : dataPlaneCq->tailAddr = cqInfo.tail_addr;
2003 8 : dataPlaneCq->swdbAddr = cqInfo.swdb_addr;
2004 8 : dataPlaneCq->dbReg = cqInfo.db_reg;
2005 8 : hccp_info("cqn:%u buf_addr:0x%llx cqe_size:%u depth:%u head_addr:0x%llx tail_addr:0x%llx swdb_addr:0x%llx",
2006 : dataPlaneCq->cqn, dataPlaneCq->bufAddr, dataPlaneCq->cqeSize, dataPlaneCq->depth, dataPlaneCq->headAddr,
2007 : dataPlaneCq->tailAddr, dataPlaneCq->swdbAddr);
2008 8 : }
2009 :
2010 8 : STATIC void RsQpPrepareWqDataPlaneInfo(struct hns_roce_wq_data_plane_info *wqInfo, struct AiDataPlaneWq *dataPlaneWq)
2011 : {
2012 8 : dataPlaneWq->wqn = wqInfo->wqn;
2013 8 : dataPlaneWq->bufAddr = wqInfo->buf_addr;
2014 8 : dataPlaneWq->wqebbSize = wqInfo->wqebb_size;
2015 8 : dataPlaneWq->depth = wqInfo->depth;
2016 8 : dataPlaneWq->headAddr = wqInfo->head_addr;
2017 8 : dataPlaneWq->tailAddr = wqInfo->tail_addr;
2018 8 : dataPlaneWq->swdbAddr = wqInfo->swdb_addr;
2019 8 : dataPlaneWq->dbReg = wqInfo->db_reg;
2020 8 : hccp_info("wqn:%u buf_addr:0x%llx wqebb_size:%u depth:%u head_addr:%u tail_addr:%u swdb_addr:0x%llx",
2021 : dataPlaneWq->wqn, dataPlaneWq->bufAddr, dataPlaneWq->wqebbSize, dataPlaneWq->depth, dataPlaneWq->headAddr,
2022 : dataPlaneWq->tailAddr, dataPlaneWq->swdbAddr);
2023 8 : }
2024 :
2025 4 : STATIC void RsQpPrepareQpDataPlaneInfo(struct ibv_qp *ibQp, struct AiDataPlaneWq *dataPlaneSq,
2026 : struct AiDataPlaneWq *dataPlaneRq)
2027 : {
2028 4 : struct hns_roce_qp_data_plane_info qpInfo = {0};
2029 :
2030 4 : (void)RsRoceGetQpDataPlaneInfo(ibQp, &qpInfo);
2031 4 : RsQpPrepareWqDataPlaneInfo(&qpInfo.sq, dataPlaneSq);
2032 4 : RsQpPrepareWqDataPlaneInfo(&qpInfo.rq, dataPlaneRq);
2033 4 : }
2034 :
2035 4 : STATIC void RsQpPrepareDataPlaneInfo(struct RsQpNormWithAttrs *qpNorm, struct RsQpCb *qpCb,
2036 : struct RsQpRespWithAttrs *qpResp)
2037 : {
2038 : // skip to prepare cq data plane info
2039 4 : if (qpNorm->extAttrs.dataPlaneFlag.bs.cqCstm != 0) {
2040 4 : qpResp->aiScqAddr = (unsigned long long)(uintptr_t)qpCb->ibSendCq;
2041 4 : qpResp->aiRcqAddr = (unsigned long long)(uintptr_t)qpCb->ibRecvCq;
2042 4 : RsQpPrepareCqDataPlaneInfo(qpCb->ibSendCq, &qpResp->dataPlaneInfo.scq);
2043 4 : RsQpPrepareCqDataPlaneInfo(qpCb->ibRecvCq, &qpResp->dataPlaneInfo.rcq);
2044 : }
2045 :
2046 : // skip to prepare qp data plane info
2047 4 : if (qpNorm->aiOpSupport != 0) {
2048 4 : RsQpPrepareQpDataPlaneInfo(qpCb->ibQp, &qpResp->dataPlaneInfo.sq, &qpResp->dataPlaneInfo.rq);
2049 : }
2050 4 : }
2051 : #endif
2052 :
2053 4 : STATIC void RsQpPrepareQpResp(struct RsQpNormWithAttrs *qpNorm, struct RsQpCb *qpCb, struct RsQpRespWithAttrs *qpResp)
2054 : {
2055 4 : if (qpNorm->isExp != 0) {
2056 4 : qpCb->isExp = RS_IS_EXP;
2057 : } else {
2058 0 : qpCb->isExp = RS_NOT_EXP;
2059 : }
2060 :
2061 4 : qpResp->aiQpAddr = (unsigned long long)(uintptr_t)qpCb->ibQp;
2062 4 : qpResp->sqIndex = (unsigned int)qpCb->sqIndex;
2063 4 : qpResp->dbIndex = (unsigned int)qpCb->dbIndex;
2064 4 : qpResp->gidIdx = (unsigned int)qpCb->qpInfoLo.gidIdx;
2065 4 : qpResp->psn = (unsigned int)qpCb->qpInfoLo.psn;
2066 :
2067 : #ifdef CUSTOM_INTERFACE
2068 4 : if (RsIsCustomInterfaceSupported()) {
2069 4 : RsQpPrepareDataPlaneInfo(qpNorm, qpCb, qpResp);
2070 : }
2071 : #endif
2072 :
2073 4 : return;
2074 : }
2075 :
2076 12 : RS_ATTRI_VISI_DEF int RsQpCreateWithAttrs(unsigned int phyId, unsigned int rdevIndex, struct RsQpNormWithAttrs *qpNorm,
2077 : struct RsQpRespWithAttrs *qpResp)
2078 : {
2079 12 : struct RsRdevCb *rdevCb = NULL;
2080 12 : struct RsQpCb *qpCb = NULL;
2081 : int qpMode;
2082 : int ret;
2083 :
2084 12 : RS_QP_PARA_CHECK(phyId);
2085 10 : CHK_PRT_RETURN(qpResp == NULL, hccp_err("qp_resp is NULL!"), -EINVAL);
2086 :
2087 9 : ret = RsQpCheckQpNorm(qpNorm, &qpMode);
2088 9 : CHK_PRT_RETURN(ret != 0, hccp_err("check qp mode failed, ret:%d", ret), ret);
2089 :
2090 6 : ret = RsQpQueryInfo(phyId, rdevIndex, &rdevCb, qpMode);
2091 6 : CHK_PRT_RETURN(ret, hccp_err("query qp info failed:%d", ret), ret);
2092 :
2093 4 : ret = RsAllocQpcbWithAttrs(rdevCb, &qpCb, qpNorm);
2094 4 : CHK_PRT_RETURN(ret, hccp_err("alloc mem for qp_cb failed, ret:%d", ret), ret);
2095 :
2096 4 : ret = RsDrvQpCreateWithAttrs(qpCb, qpNorm);
2097 4 : if (ret) {
2098 0 : hccp_err("create drv qp create failed:%d", ret);
2099 0 : goto create_qp_err;
2100 : }
2101 :
2102 4 : ret = ibv_req_notify_cq(qpCb->ibSendCq, 0);
2103 4 : if (ret) {
2104 0 : hccp_err("Couldn't request send CQ notification, ret:%d", ret);
2105 0 : ret = -EOPENSRC;
2106 0 : goto ret_noritfy_cq;
2107 : }
2108 :
2109 4 : ret = ibv_req_notify_cq(qpCb->ibRecvCq, 0);
2110 4 : if (ret) {
2111 0 : hccp_err("Couldn't request recv CQ notification, ret:%d", ret);
2112 0 : ret = -EOPENSRC;
2113 0 : goto ret_noritfy_cq;
2114 : }
2115 :
2116 4 : ret = RsQpNotifyMr(rdevCb, qpCb, &qpResp->qpn); // alloc mr
2117 4 : if (ret) {
2118 0 : hccp_err("store qp notify mr failed:%d", ret);
2119 0 : goto ret_noritfy_cq;
2120 : }
2121 :
2122 4 : RsQpPrepareQpResp(qpNorm, qpCb, qpResp);
2123 :
2124 4 : return 0;
2125 :
2126 0 : ret_noritfy_cq:
2127 0 : RsDrvQpDestroy(qpCb);
2128 :
2129 0 : create_qp_err:
2130 0 : RsFreeQpcb(rdevCb, qpCb);
2131 0 : return ret;
2132 : }
2133 :
2134 35 : void RsMrRelease(struct RsQpCb *qpCb)
2135 : {
2136 35 : struct RsMrCb *mrTmp2 = NULL;
2137 35 : struct RsMrCb *mrTmp = NULL;
2138 :
2139 35 : RS_PTHREAD_MUTEX_LOCK(&qpCb->qpMutex);
2140 35 : RS_LIST_GET_HEAD_ENTRY(mrTmp, mrTmp2, &qpCb->mrList, list, struct RsMrCb);
2141 69 : for (; (&mrTmp->list) != &qpCb->mrList;
2142 34 : mrTmp = mrTmp2, mrTmp2 = list_entry(mrTmp2->list.next, struct RsMrCb, list)) {
2143 34 : if (mrTmp->ibMr != qpCb->rdevCb->notifyMr) {
2144 2 : (void)RsDrvMrDereg(mrTmp->ibMr);
2145 : }
2146 34 : RsListDel(&mrTmp->list);
2147 34 : free(mrTmp);
2148 34 : mrTmp = NULL;
2149 : }
2150 :
2151 35 : RS_LIST_GET_HEAD_ENTRY(mrTmp, mrTmp2, &qpCb->remMrList, list, struct RsMrCb);
2152 66 : for (; (&mrTmp->list) != &qpCb->remMrList;
2153 31 : mrTmp = mrTmp2, mrTmp2 = list_entry(mrTmp2->list.next, struct RsMrCb, list)) {
2154 31 : RsListDel(&mrTmp->list);
2155 31 : free(mrTmp);
2156 31 : mrTmp = NULL;
2157 : }
2158 35 : RS_PTHREAD_MUTEX_ULOCK(&qpCb->qpMutex);
2159 35 : }
2160 :
2161 34 : STATIC void RsQpRelease(struct RsQpCb *qpCb)
2162 : {
2163 34 : RS_PTHREAD_MUTEX_LOCK(&qpCb->rdevCb->rdevMutex);
2164 34 : RsListDel(&qpCb->list);
2165 34 : RS_PTHREAD_MUTEX_ULOCK(&qpCb->rdevCb->rdevMutex);
2166 34 : RsIbvAckCqEvents(qpCb->ibSendCq, qpCb->numSendCqEvents);
2167 34 : RsIbvAckCqEvents(qpCb->ibRecvCq, qpCb->numRecvCqEvents);
2168 :
2169 : // dereg mr
2170 34 : RsMrRelease(qpCb);
2171 34 : }
2172 :
2173 34 : RS_ATTRI_VISI_DEF int RsQpDestroy(unsigned int phyId, unsigned int rdevIndex, unsigned int qpn)
2174 : {
2175 34 : struct RsQpCb *qpCb = NULL;
2176 : int ret;
2177 :
2178 34 : RS_QP_PARA_CHECK(phyId);
2179 34 : ret = RsQpn2qpcb(phyId, rdevIndex, qpn, &qpCb);
2180 34 : CHK_PRT_RETURN(ret != 0 || qpCb == NULL, hccp_err("get qp cb failed! qpn %u, ret %d", qpn, ret), ret);
2181 :
2182 32 : RsQpRelease(qpCb);
2183 :
2184 : // destroy qp
2185 32 : RsDrvQpDestroy(qpCb);
2186 32 : RsDrvDestroyCq(qpCb);
2187 32 : RsDeinitMemPool(qpCb);
2188 :
2189 32 : qpCb->rdevCb->qpCnt--;
2190 32 : ret = RsQpcbDeinit(qpCb->rdevCb, qpCb);
2191 32 : if (ret) {
2192 0 : hccp_err("rs_qpcb_deinit failed! ret[%d]", ret);
2193 : }
2194 :
2195 32 : pthread_mutex_destroy(&qpCb->cqeErrInfo.mutex);
2196 32 : pthread_mutex_destroy(&qpCb->qpMutex);
2197 32 : hccp_info("qp %d destroy qp, send wr[%u].", qpn, qpCb->sendWrNum);
2198 :
2199 32 : free(qpCb);
2200 32 : qpCb = NULL;
2201 32 : return ret;
2202 : }
2203 :
2204 0 : RS_ATTRI_VISI_DEF int RsQpDestroyWithoutCQ(unsigned int phyId, unsigned int rdevIndex, unsigned int qpn)
2205 : {
2206 0 : struct RsQpCb *qpCb = NULL;
2207 : int ret;
2208 :
2209 0 : RS_QP_PARA_CHECK(phyId);
2210 0 : ret = RsQpn2qpcb(phyId, rdevIndex, qpn, &qpCb);
2211 0 : CHK_PRT_RETURN(ret != 0 || qpCb == NULL, hccp_err("get qp cb failed! qpn %u, ret %d", qpn, ret), ret);
2212 :
2213 0 : RsQpRelease(qpCb);
2214 :
2215 : // destroy qp
2216 0 : RsDrvQpDestroy(qpCb);
2217 0 : RsDeinitMemPool(qpCb);
2218 :
2219 0 : qpCb->rdevCb->qpCnt--;
2220 0 : ret = RsQpcbDeinit(qpCb->rdevCb, qpCb);
2221 0 : if (ret) {
2222 0 : hccp_err("rs_qpcb_deinit failed! ret[%d]", ret);
2223 : }
2224 :
2225 0 : pthread_mutex_destroy(&qpCb->cqeErrInfo.mutex);
2226 0 : pthread_mutex_destroy(&qpCb->qpMutex);
2227 0 : hccp_info("qp %d destroy qp without cq, send wr[%u].", qpn, qpCb->sendWrNum);
2228 :
2229 0 : free(qpCb);
2230 0 : qpCb = NULL;
2231 0 : return ret;
2232 : }
2233 :
2234 27 : static void RsQpConnectAsyncMr(const struct RsQpCb *qpCb)
2235 : {
2236 : int ret;
2237 27 : struct RsMrCb *mrCb = NULL;
2238 27 : struct RsMrCb *mrCb2 = NULL;
2239 :
2240 27 : RS_LIST_GET_HEAD_ENTRY(mrCb, mrCb2, &qpCb->mrList, list, struct RsMrCb);
2241 53 : for (; (&mrCb->list) != &qpCb->mrList; mrCb = mrCb2, mrCb2 = list_entry(mrCb2->list.next, struct RsMrCb, list)) {
2242 26 : ret = RsMrInfoSync(mrCb);
2243 26 : if (ret) {
2244 0 : hccp_warn("rs_mr_info_sync unsuccessful, ret:%d", ret);
2245 : }
2246 : }
2247 27 : }
2248 :
2249 27 : STATIC void RsQpConnectAsyncQpcbSet(int fd, struct RsQpCb *qpCb)
2250 : {
2251 : int ret;
2252 27 : ret = RsSocketSend(fd, &qpCb->qpInfoLo, sizeof(struct RsQpInfo));
2253 27 : if (ret == sizeof(struct RsQpInfo)) {
2254 27 : qpCb->sendLen += (uint32_t)ret;
2255 27 : qpCb->state = RS_QP_STATUS_CONNECTING;
2256 : } else {
2257 0 : qpCb->state = RS_QP_STATUS_TIMEOUT;
2258 : }
2259 27 : }
2260 :
2261 27 : STATIC void RsQpConnectAsyncLength(int fd, struct RsQpCb *qpCb)
2262 : {
2263 : int ret;
2264 : struct RsQpLenInfo msg;
2265 :
2266 27 : msg.cmd = RS_CMD_LEN_INFO;
2267 27 : msg.len = qpCb->sendLen;
2268 :
2269 27 : ret = RsSocketSend(fd, &msg, sizeof(struct RsQpLenInfo));
2270 27 : if (ret != sizeof(struct RsQpLenInfo)) {
2271 0 : qpCb->state = RS_QP_STATUS_TIMEOUT;
2272 : }
2273 27 : }
2274 :
2275 32 : static int RsQpConnectAsyncInitPara(struct RsQpConnPara qpConnPara, int fd, struct RsQpCb **qpCb,
2276 : struct RsConnInfo **conn)
2277 : {
2278 : int ret;
2279 :
2280 32 : CHK_PRT_RETURN(qpConnPara.phyId >= RS_MAX_DEV_NUM, hccp_err("param error ! phyId:%u", qpConnPara.phyId), -EINVAL);
2281 :
2282 32 : CHK_PRT_RETURN(fd < 0, hccp_err("param error ! fd:%d must bigger than 0", fd), -EINVAL);
2283 :
2284 30 : ret = RsQpn2qpcb(qpConnPara.phyId, qpConnPara.rdevIndex, qpConnPara.qpn, qpCb);
2285 30 : CHK_PRT_RETURN(ret, hccp_err("get qpcb failed, qpn %u, ret %d", qpConnPara.qpn, ret), ret);
2286 :
2287 28 : ret = RsFd2conn(fd, conn);
2288 28 : CHK_PRT_RETURN(ret, hccp_err("get conn failed, fd %d, ret %d", fd, ret), ret);
2289 :
2290 28 : RsGetCurTime(&((*qpCb)->startTime));
2291 28 : (*qpCb)->sendLen = 0;
2292 28 : (*qpCb)->recvLen = 0;
2293 28 : (*qpCb)->expectLen = 0;
2294 28 : (*qpCb)->connInfo = *conn;
2295 :
2296 28 : return 0;
2297 : }
2298 :
2299 2 : STATIC int RsTypicalQpStateModifytoRtr(struct RsQpCb *qpCb, struct TypicalQp *localQpInfo,
2300 : struct TypicalQp *remoteQpInfo)
2301 : {
2302 2 : struct ibv_port_attr portAttr = {0};
2303 2 : union ibv_gid remoteInfoGid = {0};
2304 2 : struct ibv_qp_attr attr = {0};
2305 : int ret;
2306 :
2307 2 : attr.qp_state = IBV_QPS_RTR;
2308 2 : attr.dest_qp_num = remoteQpInfo->qpn;
2309 2 : attr.rq_psn = remoteQpInfo->psn;
2310 2 : attr.min_rnr_timer = RS_QP_ATTR_MIN_RNR_TIMER;
2311 2 : (attr.ah_attr).is_global = 0;
2312 2 : (attr.ah_attr).sl = localQpInfo->sl;
2313 2 : (attr.ah_attr).src_path_bits = 0;
2314 2 : (attr.ah_attr).port_num = qpCb->rdevCb->ibPort;
2315 :
2316 2 : attr.path_mtu = RsDrvSetMtu(qpCb);
2317 2 : CHK_PRT_RETURN(attr.path_mtu < IBV_MTU_1024,
2318 : hccp_err("qpn[%u] failed to set mtu, mtu[%d] < [%d]", localQpInfo->qpn, attr.path_mtu, IBV_MTU_1024), -EPERM);
2319 2 : if (qpCb->rdevCb->rsCb->hccpMode == NETWORK_PEER_ONLINE) {
2320 2 : attr.max_dest_rd_atomic = RS_MAX_RD_ATOMIC_NUM_PEER_ONLINE;
2321 : } else {
2322 0 : attr.max_dest_rd_atomic = RS_MAX_RD_ATOMIC_NUM;
2323 : }
2324 2 : (attr.ah_attr).grh.traffic_class = localQpInfo->tc;
2325 : // get gid_idx dynamically to avoid gid_idx changed issue: refresh gid_idx when it changed
2326 2 : ret = RsDrvGetGidIndex(qpCb->rdevCb, &portAttr, &qpCb->qpInfoLo.gidIdx);
2327 2 : if (ret == 0 && localQpInfo->gidIdx != (uint32_t)qpCb->qpInfoLo.gidIdx) {
2328 0 : hccp_warn("qpn[%u] qp_mode[%d] refresh gid_idx[%u] to [%d]", localQpInfo->qpn, qpCb->qpMode,
2329 : localQpInfo->gidIdx, qpCb->qpInfoLo.gidIdx);
2330 0 : localQpInfo->gidIdx = (uint32_t)qpCb->qpInfoLo.gidIdx;
2331 : }
2332 :
2333 2 : (void)memcpy_s(remoteInfoGid.raw, HCCP_GID_RAW_LEN, remoteQpInfo->gid, HCCP_GID_RAW_LEN);
2334 2 : if (remoteInfoGid.global.interface_id) {
2335 2 : attr.ah_attr.is_global = 1;
2336 2 : attr.ah_attr.grh.hop_limit = 1;
2337 2 : attr.ah_attr.grh.dgid = remoteInfoGid;
2338 2 : attr.ah_attr.grh.sgid_index = localQpInfo->gidIdx;
2339 : }
2340 :
2341 2 : ret = RsIbvModifyQp(qpCb->ibQp, &attr,
2342 : IBV_QP_STATE | IBV_QP_AV | IBV_QP_PATH_MTU | IBV_QP_DEST_QPN | IBV_QP_RQ_PSN | IBV_QP_MAX_DEST_RD_ATOMIC |
2343 : IBV_QP_MIN_RNR_TIMER);
2344 2 : CHK_PRT_RETURN(ret,
2345 : hccp_err("[modifyto_rtr]local_qpn[%u] remote_qpn[%u] ibv_modify_qp failed ret[%d], errno[%d]", localQpInfo->qpn,
2346 : remoteQpInfo->qpn, ret, errno),
2347 : -EOPENSRC);
2348 2 : hccp_info("qp qos attr: qpn[%u] tc[%u] sl[%u]", localQpInfo->qpn, localQpInfo->tc, localQpInfo->sl);
2349 2 : return 0;
2350 : }
2351 :
2352 2 : STATIC int RsTypicalQpStateModifytoRts(struct RsQpCb *qpCb, struct TypicalQp *localQpInfo)
2353 : {
2354 2 : struct ibv_qp_attr attr = {0};
2355 : int ret;
2356 :
2357 2 : attr.qp_state = IBV_QPS_RTS;
2358 2 : attr.timeout = (uint8_t)localQpInfo->retryTime;
2359 2 : attr.retry_cnt = (uint8_t)localQpInfo->retryCnt;
2360 2 : attr.rnr_retry = RS_QP_ATTR_RNR_RETRY;
2361 2 : attr.sq_psn = localQpInfo->psn;
2362 2 : if (qpCb->rdevCb->rsCb->hccpMode == NETWORK_PEER_ONLINE) {
2363 2 : attr.max_rd_atomic = RS_MAX_RD_ATOMIC_NUM_PEER_ONLINE;
2364 : } else {
2365 0 : attr.max_rd_atomic = RS_MAX_RD_ATOMIC_NUM;
2366 : }
2367 :
2368 2 : ret = RsIbvModifyQp(qpCb->ibQp, &attr,
2369 : IBV_QP_STATE | IBV_QP_TIMEOUT | IBV_QP_RETRY_CNT | IBV_QP_RNR_RETRY | IBV_QP_SQ_PSN | IBV_QP_MAX_QP_RD_ATOMIC);
2370 2 : CHK_PRT_RETURN(ret != 0,
2371 : hccp_err("[modifyto_rts]local_qpn[%u] ibv_modify_qp failed ret[%d], errno[%d]", localQpInfo->qpn, ret, errno),
2372 : -EOPENSRC);
2373 :
2374 2 : hccp_info("qp rdma attr: qpn[%u] timeout[%u] retrycnt[%u]", localQpInfo->qpn, localQpInfo->retryTime,
2375 : localQpInfo->retryCnt);
2376 2 : return 0;
2377 : }
2378 :
2379 2 : STATIC void RsTypicalQpModifyInfoRelated(struct RsQpCb *qpCb, struct TypicalQp *localQpInfo,
2380 : struct TypicalQp *remoteQpInfo)
2381 : {
2382 2 : qpCb->state = RS_QP_STATUS_CONNECTED;
2383 : // local qp info related: no need to relate qpn, psn, gid_idx, gid
2384 2 : qpCb->qosAttr.tc = (unsigned char)localQpInfo->tc;
2385 2 : qpCb->qosAttr.sl = (unsigned char)localQpInfo->sl;
2386 2 : qpCb->retryCnt = localQpInfo->retryCnt;
2387 2 : qpCb->timeout = localQpInfo->retryTime;
2388 : // remote qp info related
2389 2 : qpCb->qpInfoRem.qpn = (int)remoteQpInfo->qpn;
2390 2 : qpCb->qpInfoRem.psn = (int)remoteQpInfo->psn;
2391 2 : qpCb->qpInfoRem.gidIdx = (int)remoteQpInfo->gidIdx;
2392 2 : (void)memcpy_s(qpCb->qpInfoRem.gid.raw, HCCP_GID_RAW_LEN, remoteQpInfo->gid, HCCP_GID_RAW_LEN);
2393 2 : }
2394 :
2395 2 : STATIC void RsTypicalQpModifyExtend(struct RsQpCb *qpCb, struct TypicalQp *localQpInfo, struct TypicalQp *remoteQpInfo)
2396 : {
2397 2 : struct ibv_hyroce_feature output = {0};
2398 2 : struct ibv_hyroce_feature input = {0};
2399 2 : struct ibv_qp_attr_extend attr = {0};
2400 2 : uint32_t needMoreNego = 0;
2401 2 : int ret = 0;
2402 :
2403 2 : if (qpCb->rdevCb->ibCtxEx == NULL) {
2404 0 : return;
2405 : }
2406 :
2407 2 : if (localQpInfo->tc != qpCb->qosAttr.tc || localQpInfo->sl != qpCb->qosAttr.sl) {
2408 2 : hccp_warn("localQpInfo tc:%u sl:%u is not equal to qpCb tc:%u sl:%u", localQpInfo->tc, localQpInfo->sl,
2409 : qpCb->qosAttr.tc, qpCb->qosAttr.sl);
2410 2 : return;
2411 : }
2412 :
2413 0 : ret = memcpy_s(&input, sizeof(struct ibv_hyroce_feature), &remoteQpInfo->feature, sizeof(struct HyperFeature));
2414 0 : if (ret != 0) {
2415 0 : hccp_warn("memcpy_s feature unsuccessful, ret:%d qpn:%u ibv_hyroce_feature len:%zu HyperFeature len:%zu", ret,
2416 : qpCb->ibQp->qp_num, sizeof(struct ibv_hyroce_feature), sizeof(struct HyperFeature));
2417 0 : return;
2418 : }
2419 :
2420 0 : ret = RsIbvNegoQpHyroceFeature(qpCb->rdevCb->ibCtxEx, qpCb->ibQp, &input, &output, &needMoreNego);
2421 0 : if (ret != 0) {
2422 0 : hccp_warn("RsIbvNegoQpHyroceFeature unsuccessful, ret:%d qpn:%u errno:%d", ret, qpCb->ibQp->qp_num, errno);
2423 0 : return;
2424 : }
2425 :
2426 0 : attr.qp = qpCb->ibQp;
2427 0 : (void)memcpy_s(&attr.feature, sizeof(struct ibv_hyroce_feature), &output, sizeof(struct ibv_hyroce_feature));
2428 0 : attr.udp_src_port = localQpInfo->udpSport;
2429 0 : ret = RsIbvModifyQpExtend(qpCb->rdevCb->ibCtxEx, &attr,
2430 : IBV_QP_ATTR_EXTEND_UDP_SRC_PORT | IBV_QP_ATTR_EXTEND_HYROCE_FEATURE);
2431 0 : if (ret != 0) {
2432 0 : hccp_warn("RsIbvModifyQpExtend unsuccessful, ret:%d qpn:%u errno:%d", ret, qpCb->ibQp->qp_num, errno);
2433 0 : return;
2434 : }
2435 :
2436 0 : hccp_dbg("RsIbvModifyQpExtend successful, qpn:%u", qpCb->ibQp->qp_num);
2437 0 : return;
2438 : }
2439 :
2440 2 : STATIC int RsTypicalQueryQpAttr(struct RsQpCb *qpCb, struct TypicalQpAttr *qpAttr)
2441 : {
2442 2 : unsigned int qpAttrMask = HNS_ROCE_AI_QPC_UDPSPN;
2443 2 : struct hns_roce_qpc_attr_val qpAttrVal = {0};
2444 2 : struct ibv_qp_init_attr initAttr = {0};
2445 2 : struct ibv_qp_attr ibvQpAttr = {0};
2446 2 : int ret = 0;
2447 :
2448 : (void)qpAttrMask;
2449 : (void)qpAttrVal;
2450 : #ifdef CUSTOM_INTERFACE
2451 2 : if (RsIsCustomInterfaceSupported()) {
2452 2 : ret = RsRoceQueryQpc(qpCb->ibQp, &qpAttrVal, qpAttrMask);
2453 2 : if (ret != 0) {
2454 2 : hccp_warn("qpn:%d query qpc unsuccessful, ret %d", qpCb->qpInfoLo.qpn, ret);
2455 : } else {
2456 0 : qpCb->udpSport = qpAttrVal.udp_sport;
2457 : }
2458 : }
2459 : #endif
2460 :
2461 2 : ret = RsIbvQueryQp(qpCb->ibQp, &ibvQpAttr, IBV_QP_PATH_MTU, &initAttr);
2462 2 : CHK_PRT_RETURN(ret, hccp_err("RsIbvQueryQp failed, ret:%d errno:%d", ret, errno), -EOPENSRC);
2463 :
2464 2 : if (qpCb->ibQpEx != NULL) {
2465 0 : qpAttr->vendorPrivInfo = qpCb->ibQpEx->vendor_priv_info.value;
2466 : }
2467 :
2468 2 : qpAttr->udpSport = qpCb->udpSport;
2469 2 : qpAttr->pathMtu = (int)ibvQpAttr.path_mtu;
2470 2 : return 0;
2471 : }
2472 :
2473 2 : RS_ATTRI_VISI_DEF int RsTypicalQpModify(unsigned int phyId, unsigned int rdevIndex, struct TypicalQp localQpInfo,
2474 : struct TypicalQp remoteQpInfo, struct TypicalQpAttr *qpAttr)
2475 : {
2476 2 : struct ibv_qp_init_attr initAttr = {0};
2477 2 : struct ibv_qp_attr attr = {0};
2478 2 : struct RsQpCb *qpCb = NULL;
2479 : int ret;
2480 :
2481 2 : CHK_PRT_RETURN(phyId >= RS_MAX_DEV_NUM, hccp_err("[modify]phyId:%u >= [%d], is invalid", phyId, RS_MAX_DEV_NUM),
2482 : -EINVAL);
2483 :
2484 2 : CHK_PRT_RETURN(RsQpn2qpcb(phyId, rdevIndex, localQpInfo.qpn, &qpCb),
2485 : hccp_err("[modify]rs_qpn2qpcb qpn:%u failed, phyId[%u]", localQpInfo.qpn, phyId), -EACCES);
2486 :
2487 2 : CHK_PRT_RETURN(qpCb->state == RS_QP_STATUS_CONNECTED,
2488 : hccp_info("local_qpn:%u remote_qpn:%u already been connected, no need to modify again", localQpInfo.qpn,
2489 : remoteQpInfo.qpn),
2490 : 0);
2491 :
2492 : // see ib_modify_qp_is_ok for status modify, only support modify qp from INIT to RTR
2493 2 : ret = RsIbvQueryQp(qpCb->ibQp, &attr, IBV_QP_STATE, &initAttr);
2494 2 : CHK_PRT_RETURN(ret != 0 || attr.qp_state != IBV_QPS_INIT,
2495 : hccp_err("query qpn:%u failed, ret:%d or state:%d != %d", localQpInfo.qpn, ret, attr.qp_state, IBV_QPS_INIT),
2496 : -EOPENSRC);
2497 :
2498 2 : RsTypicalQpModifyExtend(qpCb, &localQpInfo, &remoteQpInfo);
2499 :
2500 2 : ret = RsTypicalQpStateModifytoRtr(qpCb, &localQpInfo, &remoteQpInfo);
2501 2 : CHK_PRT_RETURN(ret != 0,
2502 : hccp_err("[modify]local_qpn:%u remote_qpn:%u modify to rtr failed, ret %d", localQpInfo.qpn, remoteQpInfo.qpn,
2503 : ret),
2504 : ret);
2505 :
2506 2 : ret = RsTypicalQpStateModifytoRts(qpCb, &localQpInfo);
2507 2 : CHK_PRT_RETURN(ret != 0,
2508 : hccp_err("[modify]local_qpn:%u remote_qpn:%u modify to rts failed, ret %d", localQpInfo.qpn, remoteQpInfo.qpn,
2509 : ret),
2510 : ret);
2511 :
2512 2 : ret = RsTypicalQueryQpAttr(qpCb, qpAttr);
2513 2 : CHK_PRT_RETURN(ret != 0, hccp_err("RsTypicalQueryQpAttr failed, ret %d local_qpn:%u", ret, localQpInfo.qpn), ret);
2514 :
2515 2 : RsTypicalQpModifyInfoRelated(qpCb, &localQpInfo, &remoteQpInfo);
2516 :
2517 2 : hccp_info("local_qpn:%u remote_qpn:%u modify succ, udpSport:%u", localQpInfo.qpn, remoteQpInfo.qpn, qpCb->udpSport);
2518 :
2519 2 : return 0;
2520 : }
2521 :
2522 2 : STATIC int RsQpStateBatchModifytoPause(struct RsQpCb *qpCb)
2523 : {
2524 : int ret;
2525 :
2526 2 : ret = RsDrvQpStateModifytoReset(qpCb);
2527 2 : CHK_PRT_RETURN(ret, hccp_err("qp modify to reset failed, ret %d", ret), ret);
2528 :
2529 2 : hccp_info("local qpn[%d] remote qpn[%d] modify to pause succ", qpCb->qpInfoLo.qpn, qpCb->qpInfoRem.qpn);
2530 2 : return 0;
2531 : }
2532 :
2533 2 : STATIC int RsQpStateBatchModifytoConnected(struct RsQpCb *qpCb)
2534 : {
2535 : struct ibv_qp_attr attr;
2536 : int ret;
2537 :
2538 2 : ret = memset_s(&attr, sizeof(struct ibv_qp_attr), 0, sizeof(struct ibv_qp_attr));
2539 2 : CHK_PRT_RETURN(ret, hccp_err("memset_s attr failed ret %d", ret), -ESAFEFUNC);
2540 :
2541 2 : ret = RsDrvQpStateModifytoInit(qpCb, &attr);
2542 2 : CHK_PRT_RETURN(ret, hccp_err("qp modify to init failed, ret %d", ret), ret);
2543 2 : ret = RsDrvQpStateModifytoRtr(qpCb, &attr);
2544 2 : CHK_PRT_RETURN(ret, hccp_err("qp modify to rtr failed, ret %d", ret), ret);
2545 2 : ret = RsDrvQpStateModifytoRts(qpCb, &attr);
2546 2 : CHK_PRT_RETURN(ret, hccp_err("qp modify to rts failed, ret %d", ret), ret);
2547 :
2548 2 : hccp_info("local qpn[%d] remote qpn[%d] modify to rts succ", qpCb->qpInfoLo.qpn, qpCb->qpInfoRem.qpn);
2549 2 : return 0;
2550 : }
2551 :
2552 3 : RS_ATTRI_VISI_DEF int RsQpBatchModify(unsigned int phyId, unsigned int rdevIndex, int status, int qpn[], int qpnNum)
2553 : {
2554 3 : struct RsQpCb *qpCb = NULL;
2555 : int ret;
2556 : int i;
2557 :
2558 3 : CHK_PRT_RETURN(phyId >= RS_MAX_DEV_NUM, hccp_err("[modify]phyId:%u >= [%d], is invalid", phyId, RS_MAX_DEV_NUM),
2559 : -EINVAL);
2560 :
2561 7 : for (i = 0; i < qpnNum; i++) {
2562 5 : CHK_PRT_RETURN(RsQpn2qpcb(phyId, rdevIndex, (uint32_t)qpn[i], &qpCb),
2563 : hccp_err("[modify]rs_qpn2qpcb failed, phyId[%u]", phyId), -EACCES);
2564 :
2565 : /*
2566 : * see ib_modify_qp_is_ok for status modify
2567 : * only support modify qp from STATUS_PAUSE(RESET) to STATUS_CONNECTED(INIT)
2568 : */
2569 5 : if (status == RS_QP_STATUS_CONNECTED && qpCb->state == RS_QP_STATUS_PAUSE) {
2570 2 : ret = RsQpStateBatchModifytoConnected(qpCb);
2571 2 : CHK_PRT_RETURN(ret,
2572 : hccp_err("modify_qp qpn[%d]:%d to connected failed, ret[%d] phyId[%u]", i, qpn[i], ret, phyId), ret);
2573 3 : } else if (status == RS_QP_STATUS_PAUSE) {
2574 2 : ret = RsQpStateBatchModifytoPause(qpCb);
2575 2 : CHK_PRT_RETURN(ret,
2576 : hccp_err("modify_qp qpn[%d]:%d to pause failed, ret[%d] phyId[%u]", i, qpn[i], ret, phyId), ret);
2577 : } else {
2578 1 : hccp_err("modify_qp qpn[%d]:%d failed, not support to modify status[%d] to status[%d], phyId[%u]", i,
2579 : qpn[i], qpCb->state, status, phyId);
2580 1 : return -EINVAL;
2581 : }
2582 :
2583 4 : qpCb->state = status;
2584 : }
2585 :
2586 2 : return 0;
2587 : }
2588 :
2589 2 : RS_ATTRI_VISI_DEF int RsSetQpLbValue(unsigned int phyId, unsigned int rdevIndex, unsigned int qpn, int lbValue)
2590 : {
2591 2 : struct RsQpCb *qpCb = NULL;
2592 2 : int ret = 0;
2593 :
2594 2 : ret = RsQpn2qpcb(phyId, rdevIndex, qpn, &qpCb);
2595 2 : CHK_PRT_RETURN(ret != 0, hccp_err("RsQpn2qpcb failed ret:%d", ret), ret);
2596 :
2597 1 : return RsRoceSetQpLbValue(qpCb->ibQp, lbValue);
2598 : }
2599 :
2600 3 : RS_ATTRI_VISI_DEF int RsGetQpLbValue(unsigned int phyId, unsigned int rdevIndex, unsigned int qpn, int *lbValue)
2601 : {
2602 3 : struct RsQpCb *qpCb = NULL;
2603 3 : int ret = 0;
2604 :
2605 3 : ret = RsQpn2qpcb(phyId, rdevIndex, qpn, &qpCb);
2606 3 : CHK_PRT_RETURN(ret != 0, hccp_err("RsQpn2qpcb failed ret:%d", ret), ret);
2607 :
2608 2 : return RsRoceGetQpLbValue(qpCb->ibQp, lbValue);
2609 : }
2610 :
2611 32 : RS_ATTRI_VISI_DEF int RsQpConnectAsync(unsigned int phyId, unsigned int rdevIndex, unsigned int qpn, int fd)
2612 : {
2613 : int ret;
2614 32 : struct RsQpCb *qpCb = NULL;
2615 32 : struct RsConnInfo *conn = NULL;
2616 : struct RsQpConnPara qpConnPara;
2617 32 : hccp_info("qp:%d, fd:%d", qpn, fd);
2618 :
2619 32 : qpConnPara.phyId = phyId;
2620 32 : qpConnPara.rdevIndex = rdevIndex;
2621 32 : qpConnPara.qpn = qpn;
2622 32 : ret = RsQpConnectAsyncInitPara(qpConnPara, fd, &qpCb, &conn);
2623 32 : CHK_PRT_RETURN(ret, hccp_err("rs_qp_connect_async_init_para failed, qpn %u, ret %d", qpn, ret), ret);
2624 :
2625 28 : RS_PTHREAD_MUTEX_LOCK(&qpCb->qpMutex);
2626 :
2627 28 : if (qpCb->state == RS_QP_STATUS_REM_FD_CLOSE) {
2628 0 : hccp_warn("remote qp fd close, can not use it anymore!");
2629 0 : RS_PTHREAD_MUTEX_ULOCK(&qpCb->qpMutex);
2630 0 : return -EFAULT;
2631 : }
2632 :
2633 28 : if ((qpCb->state == RS_QP_STATUS_CONNECTED) || (qpCb->state == RS_QP_STATUS_CONNECTING)) {
2634 1 : hccp_warn("qp %d has already sync! state[%d]", qpCb->qpInfoLo.qpn, qpCb->state);
2635 1 : RS_PTHREAD_MUTEX_ULOCK(&qpCb->qpMutex);
2636 1 : return -EEXIST;
2637 : }
2638 :
2639 27 : RsQpConnectAsyncQpcbSet(fd, qpCb);
2640 :
2641 27 : hccp_info("after socket fd %d send QP %u, chipId %u, state:%d!", fd, qpn, qpCb->rdevCb->rsCb->chipId, qpCb->state);
2642 :
2643 27 : RS_PTHREAD_MUTEX_ULOCK(&qpCb->qpMutex);
2644 :
2645 27 : RsQpMrRecvHandle(fd, qpCb);
2646 :
2647 27 : RsQpConnectAsyncMr(qpCb);
2648 :
2649 27 : RsQpConnectAsyncLength(fd, qpCb);
2650 :
2651 27 : hccp_info("QP %d async done, state:%d!", qpn, qpCb->state);
2652 :
2653 27 : return 0;
2654 : }
2655 :
2656 1 : RS_ATTRI_VISI_DEF int RsGetQpStatus(unsigned int phyId, unsigned int rdevIndex, unsigned int qpn,
2657 : struct RsQpStatusInfo *qpInfo)
2658 : {
2659 1 : unsigned int qpAttrMask = HNS_ROCE_AI_QPC_UDPSPN;
2660 1 : struct hns_roce_qpc_attr_val qpAttrVal = {0};
2661 1 : struct RsQpCb *qpCb = NULL;
2662 : int ret;
2663 :
2664 : (void)qpAttrMask;
2665 : (void)qpAttrVal;
2666 1 : CHK_PRT_RETURN(qpInfo == NULL, hccp_err("param error, qpInfo is NULL"), -EINVAL);
2667 :
2668 1 : CHK_PRT_RETURN(phyId >= RS_MAX_DEV_NUM, hccp_err("phyId:%u >= [%d], is invalid", phyId, RS_MAX_DEV_NUM), -EINVAL);
2669 :
2670 1 : ret = RsQpn2qpcb(phyId, rdevIndex, qpn, &qpCb);
2671 1 : CHK_PRT_RETURN(ret, hccp_err("get qp cb failed, qpn:%u, ret %d", qpn, ret), ret);
2672 :
2673 : // qp state is CONNECTED, no need to handle
2674 1 : if (qpCb->state == RS_QP_STATUS_CONNECTED) {
2675 1 : goto update_qp_cb;
2676 : }
2677 :
2678 : // modify state to CONNECTED
2679 0 : if (qpCb->expectLen == qpCb->recvLen - sizeof(struct RsQpLenInfo)) {
2680 0 : qpCb->state = RS_QP_STATUS_CONNECTED;
2681 : } else {
2682 0 : RsQpMrRecvHandle(qpCb->connInfo->connfd, qpCb);
2683 0 : goto out;
2684 : }
2685 :
2686 1 : update_qp_cb:
2687 : #ifdef CUSTOM_INTERFACE
2688 1 : if (RsIsCustomInterfaceSupported()) {
2689 1 : ret = RsRoceQueryQpc(qpCb->ibQp, &qpAttrVal, qpAttrMask);
2690 1 : if (ret != 0) {
2691 1 : hccp_warn("qpn:%d query qpc unsuccessful, ret %d", qpCb->qpInfoLo.qpn, ret);
2692 : } else {
2693 0 : qpCb->udpSport = qpAttrVal.udp_sport;
2694 : }
2695 : }
2696 : #endif
2697 0 : out:
2698 1 : hccp_dbg("qp:%u, state:%d, udpSport:%u", qpn, qpCb->state, qpCb->udpSport);
2699 1 : qpInfo->status = qpCb->state;
2700 1 : qpInfo->udpSport = qpCb->udpSport;
2701 :
2702 1 : return 0;
2703 : }
2704 :
2705 3 : RS_ATTRI_VISI_DEF int RsGetQpContext(unsigned int phyId, unsigned int rdevIndex, unsigned int qpn, void **qp,
2706 : void **sendCq, void **recvCq)
2707 : {
2708 : int ret;
2709 3 : struct RsQpCb *qpCb = NULL;
2710 :
2711 3 : CHK_PRT_RETURN(phyId >= RS_MAX_DEV_NUM, hccp_err("phyId:%u >= [%d], is invalid", phyId, RS_MAX_DEV_NUM), -EINVAL);
2712 :
2713 2 : ret = RsQpn2qpcb(phyId, rdevIndex, qpn, &qpCb);
2714 2 : CHK_PRT_RETURN(ret, hccp_err("rs_qpn2qpcb failed ret[%d]", ret), ret);
2715 :
2716 1 : *qp = qpCb->ibQp;
2717 1 : *sendCq = qpCb->ibSendCq;
2718 1 : *recvCq = qpCb->ibRecvCq;
2719 :
2720 1 : hccp_dbg("qpn[%u] succ", qpn);
2721 :
2722 1 : return 0;
2723 : }
2724 :
2725 26 : int RsQueryRdevCb(unsigned int phyId, unsigned int rdevIndex, struct RsRdevCb **rdevCb)
2726 : {
2727 : int ret;
2728 : unsigned int chipId;
2729 26 : struct rs_cb *rsCb = NULL;
2730 :
2731 26 : RS_QP_PARA_CHECK(phyId);
2732 :
2733 24 : ret = rsGetLocalDevIDByHostDevID(phyId, &chipId);
2734 24 : CHK_PRT_RETURN(ret, hccp_err("rs_query_rdev_cb phyId[%u] invalid, ret:%d", phyId, ret), ret);
2735 :
2736 23 : ret = RsDev2rscb(chipId, &rsCb, false);
2737 23 : CHK_PRT_RETURN(ret, hccp_err("rs_query_rdev_cb get rs_cb failed, ret:%d", ret), -ENODEV);
2738 :
2739 23 : ret = RsGetRdevCb(rsCb, rdevIndex, rdevCb);
2740 23 : CHK_PRT_RETURN(ret, hccp_err("rs_get_rdev_cb failed! ret:%d, rdevIndex:%u", ret, rdevIndex), ret);
2741 :
2742 23 : return 0;
2743 : }
2744 :
2745 3 : RS_ATTRI_VISI_DEF int RsGetLbMax(unsigned int phyId, unsigned int rdevIndex, int *lbMax)
2746 : {
2747 3 : struct RsRdevCb *rdevCb = NULL;
2748 3 : int ret = 0;
2749 :
2750 3 : ret = RsQueryRdevCb(phyId, rdevIndex, &rdevCb);
2751 3 : CHK_PRT_RETURN(ret != 0, hccp_err("RsQueryRdevCb phyId:%u rdev_index:%u ret:%d", phyId, rdevIndex, ret), ret);
2752 :
2753 2 : return RsRoceGetQpNum(rdevCb->ibCtx, lbMax);
2754 : }
2755 :
2756 3 : STATIC int RsBuildUpQpcb(struct RsCqContext *cqContext, struct ibv_qp_init_attr *qpInitAttr, struct RsQpCb **qpCb)
2757 : {
2758 : int ret;
2759 :
2760 3 : ret = RsCallocQpcb(1, qpCb);
2761 3 : CHK_PRT_RETURN(ret, hccp_err("alloc mem for qp_cb failed, ret:%d errno:%d", ret, errno), -ENOMEM);
2762 :
2763 3 : ret = pthread_mutex_init(&(*qpCb)->qpMutex, NULL);
2764 3 : if (ret) {
2765 0 : hccp_err("pthread_mutex_init failed, ret %d", ret);
2766 0 : goto pthread_mutex_init_err;
2767 : }
2768 :
2769 3 : (*qpCb)->rdevCb = cqContext->rdevCb;
2770 3 : RS_INIT_LIST_HEAD(&(*qpCb)->mrList);
2771 3 : RS_INIT_LIST_HEAD(&(*qpCb)->remMrList);
2772 :
2773 3 : (*qpCb)->eqNum = cqContext->eqNum;
2774 3 : (*qpCb)->channel = cqContext->channel;
2775 3 : (*qpCb)->ibSendCq = cqContext->ibSendCq;
2776 3 : (*qpCb)->ibRecvCq = cqContext->ibRecvCq;
2777 3 : (*qpCb)->sendEvent = cqContext->sendEvent;
2778 3 : (*qpCb)->recvEvent = cqContext->recvEvent;
2779 3 : (*qpCb)->numRecvCqEvents = 0;
2780 3 : (*qpCb)->numSendCqEvents = 0;
2781 3 : (*qpCb)->srqContext = cqContext->srqContext;
2782 3 : (*qpCb)->state = RS_QP_STATUS_DISCONNECT;
2783 3 : (*qpCb)->ibPd = cqContext->rdevCb->ibPd;
2784 3 : (*qpCb)->txDepth = qpInitAttr->cap.max_send_wr;
2785 3 : (*qpCb)->rxDepth = qpInitAttr->cap.max_recv_wr;
2786 3 : (*qpCb)->qosAttr.tc = (RS_ROCE_DSCP_33 & RS_DSCP_MASK) << RS_DSCP_OFF;
2787 3 : (*qpCb)->qosAttr.sl = RS_ROCE_4_SL;
2788 3 : (*qpCb)->timeout = RS_QP_ATTR_TIMEOUT;
2789 3 : (*qpCb)->retryCnt = RS_QP_ATTR_RETRY_CNT;
2790 :
2791 3 : return 0;
2792 :
2793 0 : pthread_mutex_init_err:
2794 0 : free(*qpCb);
2795 0 : (*qpCb) = NULL;
2796 0 : return ret;
2797 : }
2798 :
2799 6 : RS_ATTRI_VISI_DEF int RsCreateCqEvent(struct RsCqContext *cqContext, struct CqAttr *attr)
2800 : {
2801 : int ret;
2802 6 : cqContext->channel = RsIbvCreateCompChannel(cqContext->rdevCb->ibCtx);
2803 :
2804 6 : if (cqContext->channel == NULL) {
2805 1 : hccp_err("ibv_create_comp_channel failed, ret %d, errno(%d)", -EINVAL, errno);
2806 1 : return -EINVAL;
2807 : }
2808 :
2809 5 : hccp_info("comp channel fd[%d].", cqContext->channel->fd);
2810 5 : ret = RsEpollCtl(cqContext->rdevCb->rsCb->connCb.epollfd, EPOLL_CTL_ADD, cqContext->channel->fd,
2811 : EPOLLIN | EPOLLRDHUP);
2812 : #ifndef CA_CONFIG_LLT
2813 : if (ret) {
2814 : hccp_err("add channel fd failed ret %d", ret);
2815 : goto rs_cq_epoll_ctl_err;
2816 : }
2817 : #endif
2818 :
2819 5 : ret = RsDrvCreateCqEvent(cqContext, attr);
2820 5 : if (ret) {
2821 0 : hccp_err("create drv cq event failed:%d", ret);
2822 0 : goto rs_cq_create_err;
2823 : }
2824 :
2825 5 : return ret;
2826 0 : rs_cq_create_err:
2827 0 : ret = RsEpollCtl(cqContext->rdevCb->rsCb->connCb.epollfd, EPOLL_CTL_DEL, cqContext->channel->fd,
2828 : EPOLLIN | EPOLLRDHUP);
2829 : #ifndef CA_CONFIG_LLT
2830 : if (ret) {
2831 : hccp_err("del channel fd failed ret %d", ret);
2832 : }
2833 : #endif
2834 0 : rs_cq_epoll_ctl_err:
2835 0 : if (cqContext->channel != NULL) {
2836 0 : RsIbvDestroyCompChannel(cqContext->channel);
2837 0 : cqContext->channel = NULL;
2838 : }
2839 0 : return ret;
2840 : }
2841 :
2842 5 : RS_ATTRI_VISI_DEF int RsCqCreate(unsigned int phyId, unsigned int rdevIndex, struct CqAttr *attr)
2843 : {
2844 : int ret;
2845 5 : struct RsRdevCb *rdevCb = NULL;
2846 5 : struct RsCqContext *cqContext = NULL;
2847 :
2848 5 : ret = RsQueryRdevCb(phyId, rdevIndex, &rdevCb);
2849 5 : if (ret) {
2850 0 : hccp_err("rs_query_rdev_cb phyId[%u] rdev_index[%u], ret %d", phyId, rdevIndex, ret);
2851 0 : return ret;
2852 : }
2853 :
2854 5 : cqContext = calloc(1, sizeof(struct RsCqContext));
2855 5 : if (cqContext == NULL) {
2856 0 : return -ENOMEM;
2857 : }
2858 5 : cqContext->rdevCb = rdevCb;
2859 5 : cqContext->eqNum = 0;
2860 5 : if (attr->sendChannel == NULL && attr->recvChannel == NULL) {
2861 3 : if (*attr->ibSendCq == NULL && *attr->ibRecvCq != NULL) {
2862 : // 只创建sq cq
2863 1 : cqContext->cqCreateMode = RS_SQ_CQ_CREATE;
2864 1 : cqContext->ibRecvCq = *attr->ibRecvCq;
2865 1 : cqContext->srqContext = attr->srqContext;
2866 : } else {
2867 : // 创建sq&rq cq
2868 2 : cqContext->cqCreateMode = RS_NORMAL_CQ_CREATE;
2869 : }
2870 3 : ret = RsCreateCqEvent(cqContext, attr);
2871 3 : if (ret) {
2872 0 : hccp_err("create cq event failed:%d", ret);
2873 0 : goto rs_cq_create_err;
2874 : }
2875 2 : } else if (attr->sendChannel != NULL && attr->recvChannel != NULL) {
2876 : // 使用输入comp channel创建sq&rq
2877 1 : ret = RsDrvCreateCqWithChannel(cqContext, attr);
2878 1 : if (ret) {
2879 0 : hccp_err("create drv cq with channel failed:%d", ret);
2880 0 : goto rs_cq_create_err;
2881 : }
2882 : } else {
2883 1 : hccp_err("rs create cq failed, sendChannel or recvChannel is NULL.");
2884 1 : ret = -EPERM;
2885 1 : goto rs_cq_create_err;
2886 : }
2887 :
2888 4 : *attr->qpContext = cqContext;
2889 4 : return 0;
2890 :
2891 1 : rs_cq_create_err:
2892 1 : free(cqContext);
2893 1 : cqContext = NULL;
2894 :
2895 1 : return ret;
2896 : }
2897 :
2898 0 : RS_ATTRI_VISI_DEF int RsTypicalCqCreate(unsigned int phyId, unsigned int rdevIndex, unsigned int cqDepth,
2899 : unsigned int *cqn)
2900 : {
2901 : struct RsTypicalCqEntry *entry;
2902 : struct RsTypicalCqEntry *tmp;
2903 : int ret;
2904 0 : struct RsRdevCb *rdevCb = NULL;
2905 0 : struct ibv_cq *ibCq = NULL;
2906 : struct rdma_lite_device_cq_attr deviceCqAttr;
2907 :
2908 0 : ret = RsQueryRdevCb(phyId, rdevIndex, &rdevCb);
2909 0 : if (ret) {
2910 0 : hccp_err("rs_query_rdev_cb phyId[%u] rdevIndex[%u], ret %d", phyId, rdevIndex, ret);
2911 0 : return ret;
2912 : }
2913 :
2914 0 : ret = RsDrvTypicalCqCreate(rdevCb, cqDepth, cqn, &ibCq, &deviceCqAttr);
2915 0 : if (ret) {
2916 0 : hccp_err("rs_drv_typical_cq_create failed, cqDepth[%u] ret[%d]", cqDepth, ret);
2917 0 : return ret;
2918 : }
2919 :
2920 0 : pthread_mutex_lock(&gRsTypicalCqMutex);
2921 0 : if (gRsTypicalCqList.next == NULL) {
2922 0 : RS_INIT_LIST_HEAD(&gRsTypicalCqList);
2923 : }
2924 :
2925 0 : RS_LIST_GET_HEAD_ENTRY(tmp, entry, &gRsTypicalCqList, list, struct RsTypicalCqEntry);
2926 0 : for (; &tmp->list != &gRsTypicalCqList;
2927 0 : tmp = entry, entry = list_entry(entry->list.next, struct RsTypicalCqEntry, list)) {
2928 0 : if (tmp->phyId == phyId && tmp->rdevIndex == rdevIndex && tmp->cqn == *cqn) {
2929 0 : tmp->ibCq = ibCq;
2930 0 : tmp->deviceCqAttr = deviceCqAttr;
2931 0 : pthread_mutex_unlock(&gRsTypicalCqMutex);
2932 0 : hccp_info("RsTypicalCqCreate updated: phyId[%u] rdevIndex[%u] cqn[%u] cqDepth[%u]", phyId, rdevIndex, *cqn,
2933 : cqDepth);
2934 0 : return 0;
2935 : }
2936 : }
2937 :
2938 0 : entry = calloc(1, sizeof(struct RsTypicalCqEntry));
2939 0 : if (entry == NULL) {
2940 0 : pthread_mutex_unlock(&gRsTypicalCqMutex);
2941 0 : hccp_err("RsTypicalCqCreate calloc failed, cqn[%u]", *cqn);
2942 0 : return -ENOMEM;
2943 : }
2944 0 : entry->phyId = phyId;
2945 0 : entry->rdevIndex = rdevIndex;
2946 0 : entry->cqn = *cqn;
2947 0 : entry->ibCq = ibCq;
2948 0 : entry->deviceCqAttr = deviceCqAttr;
2949 0 : RsListAddTail(&entry->list, &gRsTypicalCqList);
2950 0 : pthread_mutex_unlock(&gRsTypicalCqMutex);
2951 :
2952 0 : hccp_info("RsTypicalCqCreate success: phyId[%u] rdevIndex[%u] cqn[%u] cqDepth[%u]", phyId, rdevIndex, *cqn,
2953 : cqDepth);
2954 :
2955 0 : return 0;
2956 : }
2957 :
2958 0 : RS_ATTRI_VISI_DEF int RsTypicalCqDestroy(unsigned int phyId, unsigned int rdevIndex, unsigned int cqn)
2959 : {
2960 : struct RsTypicalCqEntry *entry;
2961 : struct RsTypicalCqEntry *tmp;
2962 : int ret;
2963 :
2964 0 : pthread_mutex_lock(&gRsTypicalCqMutex);
2965 0 : if (gRsTypicalCqList.next == NULL) {
2966 0 : pthread_mutex_unlock(&gRsTypicalCqMutex);
2967 0 : hccp_err("rs_typical_cq_destroy: cqn[%u] not found phyId[%u] rdevIndex[%u]", cqn, phyId, rdevIndex);
2968 0 : return -EINVAL;
2969 : }
2970 :
2971 0 : RS_LIST_GET_HEAD_ENTRY(tmp, entry, &gRsTypicalCqList, list, struct RsTypicalCqEntry);
2972 0 : for (; &tmp->list != &gRsTypicalCqList;
2973 0 : tmp = entry, entry = list_entry(entry->list.next, struct RsTypicalCqEntry, list)) {
2974 0 : if (tmp->phyId == phyId && tmp->rdevIndex == rdevIndex && tmp->cqn == cqn) {
2975 0 : RsListDel(&tmp->list);
2976 0 : pthread_mutex_unlock(&gRsTypicalCqMutex);
2977 :
2978 0 : if (tmp->ibCq != NULL) {
2979 0 : ret = RsIbvDestroyCq(tmp->ibCq);
2980 0 : if (ret) {
2981 0 : hccp_err("rs_ibv_destroy_cq failed cqn[%u] ret[%d]", cqn, ret);
2982 0 : free(tmp);
2983 0 : return ret;
2984 : }
2985 : }
2986 0 : free(tmp);
2987 0 : hccp_info("RsTypicalCqDestroy success: phyId[%u] rdevIndex[%u] cqn[%u]", phyId, rdevIndex, cqn);
2988 0 : return 0;
2989 : }
2990 : }
2991 0 : pthread_mutex_unlock(&gRsTypicalCqMutex);
2992 :
2993 0 : hccp_err("rs_typical_cq_destroy: cqn[%u] not found phyId[%u] rdevIndex[%u]", cqn, phyId, rdevIndex);
2994 0 : return -EINVAL;
2995 : }
2996 :
2997 0 : RS_ATTRI_VISI_DEF int RsGetLiteCqAttr(unsigned int phyId, unsigned int rdevIndex, unsigned int cqn,
2998 : struct rdma_lite_device_cq_attr *deviceCqAttr)
2999 : {
3000 : struct RsTypicalCqEntry *entry;
3001 : struct RsTypicalCqEntry *tmp;
3002 : int ret;
3003 :
3004 0 : RS_CHECK_POINTER_NULL_RETURN_INT(deviceCqAttr);
3005 :
3006 0 : pthread_mutex_lock(&gRsTypicalCqMutex);
3007 0 : if (gRsTypicalCqList.next == NULL) {
3008 0 : pthread_mutex_unlock(&gRsTypicalCqMutex);
3009 0 : hccp_err("rs_get_lite_cq_attr: cqn[%u] not found phyId[%u] rdevIndex[%u]", cqn, phyId, rdevIndex);
3010 0 : return -EINVAL;
3011 : }
3012 :
3013 0 : RS_LIST_GET_HEAD_ENTRY(tmp, entry, &gRsTypicalCqList, list, struct RsTypicalCqEntry);
3014 0 : for (; &tmp->list != &gRsTypicalCqList;
3015 0 : tmp = entry, entry = list_entry(entry->list.next, struct RsTypicalCqEntry, list)) {
3016 0 : if (tmp->phyId == phyId && tmp->rdevIndex == rdevIndex && tmp->cqn == cqn) {
3017 0 : ret = memcpy_s(deviceCqAttr, sizeof(*deviceCqAttr), &tmp->deviceCqAttr, sizeof(tmp->deviceCqAttr));
3018 0 : pthread_mutex_unlock(&gRsTypicalCqMutex);
3019 0 : if (ret) {
3020 0 : hccp_err("memcpy_s failed, ret:%d", ret);
3021 0 : return ret;
3022 : }
3023 0 : hccp_info("RsGetLiteCqAttr success: cqn[%u] depth[%u]", cqn, deviceCqAttr->depth);
3024 0 : return 0;
3025 : }
3026 : }
3027 0 : pthread_mutex_unlock(&gRsTypicalCqMutex);
3028 :
3029 0 : hccp_err("rs_get_lite_cq_attr: cqn[%u] not found phyId[%u] rdevIndex[%u]", cqn, phyId, rdevIndex);
3030 0 : return -EINVAL;
3031 : }
3032 :
3033 0 : RS_ATTRI_VISI_DEF int RsQpCreateWithCQWithAttrs(unsigned int phyId, unsigned int rdevIndex, unsigned int sendCqn,
3034 : unsigned int recvCqn, struct RsQpNormWithAttrs *qpNorm, struct RsQpRespWithAttrs *qpResp)
3035 : {
3036 : struct RsTypicalCqEntry *entry;
3037 : struct RsTypicalCqEntry *tmp;
3038 0 : struct RsRdevCb *rdevCb = NULL;
3039 0 : struct RsQpCb *qpCb = NULL;
3040 0 : struct ibv_cq *sendIbCq = NULL;
3041 0 : struct ibv_cq *recvIbCq = NULL;
3042 : struct rdma_lite_device_cq_attr sendDeviceCqAttr;
3043 : struct rdma_lite_device_cq_attr recvDeviceCqAttr;
3044 : int qpMode;
3045 : int ret;
3046 0 : bool sendFound = false;
3047 0 : bool recvFound = false;
3048 :
3049 0 : RS_QP_PARA_CHECK(phyId);
3050 0 : CHK_PRT_RETURN(qpResp == NULL, hccp_err("qp_resp is NULL!"), -EINVAL);
3051 :
3052 0 : ret = RsQpCheckQpNorm(qpNorm, &qpMode);
3053 0 : CHK_PRT_RETURN(ret != 0, hccp_err("check qp mode failed, ret:%d", ret), ret);
3054 :
3055 0 : ret = RsQpQueryInfo(phyId, rdevIndex, &rdevCb, qpMode);
3056 0 : CHK_PRT_RETURN(ret, hccp_err("query qp info failed:%d", ret), ret);
3057 :
3058 0 : pthread_mutex_lock(&gRsTypicalCqMutex);
3059 0 : if (gRsTypicalCqList.next != NULL) {
3060 0 : RS_LIST_GET_HEAD_ENTRY(tmp, entry, &gRsTypicalCqList, list, struct RsTypicalCqEntry);
3061 0 : for (; &tmp->list != &gRsTypicalCqList;
3062 0 : tmp = entry, entry = list_entry(entry->list.next, struct RsTypicalCqEntry, list)) {
3063 0 : if (tmp->phyId == phyId && tmp->rdevIndex == rdevIndex && tmp->cqn == sendCqn) {
3064 0 : sendIbCq = tmp->ibCq;
3065 0 : sendDeviceCqAttr = tmp->deviceCqAttr;
3066 0 : sendFound = true;
3067 0 : break;
3068 : }
3069 : }
3070 : }
3071 0 : pthread_mutex_unlock(&gRsTypicalCqMutex);
3072 0 : CHK_PRT_RETURN(!sendFound,
3073 : hccp_err("send cq not found: sendCqn[%u] phyId[%u] rdevIndex[%u]", sendCqn, phyId, rdevIndex), -EINVAL);
3074 :
3075 0 : pthread_mutex_lock(&gRsTypicalCqMutex);
3076 0 : if (gRsTypicalCqList.next != NULL) {
3077 0 : RS_LIST_GET_HEAD_ENTRY(tmp, entry, &gRsTypicalCqList, list, struct RsTypicalCqEntry);
3078 0 : for (; &tmp->list != &gRsTypicalCqList;
3079 0 : tmp = entry, entry = list_entry(entry->list.next, struct RsTypicalCqEntry, list)) {
3080 0 : if (tmp->phyId == phyId && tmp->rdevIndex == rdevIndex && tmp->cqn == recvCqn) {
3081 0 : recvIbCq = tmp->ibCq;
3082 0 : recvDeviceCqAttr = tmp->deviceCqAttr;
3083 0 : recvFound = true;
3084 0 : break;
3085 : }
3086 : }
3087 : }
3088 0 : pthread_mutex_unlock(&gRsTypicalCqMutex);
3089 0 : CHK_PRT_RETURN(!recvFound,
3090 : hccp_err("recv cq not found: recvCqn[%u] phyId[%u] rdevIndex[%u]", recvCqn, phyId, rdevIndex), -EINVAL);
3091 :
3092 0 : ret = RsCallocQpcb(1, &qpCb);
3093 0 : CHK_PRT_RETURN(ret, hccp_err("alloc mem for qp_cb failed, ret:%d errno:%d", ret, errno), -ENOMEM);
3094 :
3095 0 : ret = pthread_mutex_init(&qpCb->qpMutex, NULL);
3096 0 : if (ret) {
3097 0 : hccp_err("pthread_mutex_init failed, ret %d", ret);
3098 0 : goto qp_mutex_init_err;
3099 : }
3100 :
3101 0 : ret = pthread_mutex_init(&qpCb->cqeErrInfo.mutex, NULL);
3102 0 : if (ret) {
3103 0 : hccp_err("pthread_mutex_init failed, ret %d", ret);
3104 0 : goto cqe_mutex_init_err;
3105 : }
3106 :
3107 0 : ret = RsQpcbInitWithAttrs(rdevCb, qpCb, qpNorm);
3108 0 : if (ret) {
3109 0 : hccp_err("create qp tx rx failed ret %d", ret);
3110 0 : goto rs_qpcb_init_err;
3111 : }
3112 :
3113 0 : ret = RsInitMemPool(qpCb);
3114 0 : if (ret) {
3115 0 : hccp_err("init mem pool failed ret %d", ret);
3116 0 : goto rs_init_mem_err;
3117 : }
3118 :
3119 : // Assign pre-existing CQs (instead of RsDrvCreateCqWithAttrs)
3120 0 : qpCb->ibSendCq = sendIbCq;
3121 0 : qpCb->ibRecvCq = recvIbCq;
3122 0 : qpCb->qpResp.sendCqData = sendDeviceCqAttr;
3123 0 : qpCb->qpResp.recvCqData = recvDeviceCqAttr;
3124 0 : qpCb->sendCqDepth = sendDeviceCqAttr.depth;
3125 0 : qpCb->recvCqDepth = recvDeviceCqAttr.depth;
3126 :
3127 0 : ret = RsDrvQpCreateWithAttrs(qpCb, qpNorm);
3128 0 : if (ret) {
3129 0 : hccp_err("Create drv qp create failed:%d", ret);
3130 0 : goto create_qp_err;
3131 : }
3132 :
3133 0 : ret = ibv_req_notify_cq(qpCb->ibSendCq, 0);
3134 0 : if (ret) {
3135 0 : hccp_err("Can't request send CQ notification, ret:%d", ret);
3136 0 : ret = -EOPENSRC;
3137 0 : goto ret_noritfy_cq;
3138 : }
3139 :
3140 0 : ret = ibv_req_notify_cq(qpCb->ibRecvCq, 0);
3141 0 : if (ret) {
3142 0 : hccp_err("Can't request recv CQ notification, ret:%d", ret);
3143 0 : ret = -EOPENSRC;
3144 0 : goto ret_noritfy_cq;
3145 : }
3146 :
3147 0 : ret = RsQpNotifyMr(rdevCb, qpCb, &qpResp->qpn);
3148 0 : if (ret) {
3149 0 : hccp_err("Store qp notify mr failed:%d", ret);
3150 0 : goto ret_noritfy_cq;
3151 : }
3152 :
3153 0 : RsQpPrepareQpResp(qpNorm, qpCb, qpResp);
3154 :
3155 0 : return 0;
3156 :
3157 0 : ret_noritfy_cq:
3158 0 : RsDrvQpDestroy(qpCb);
3159 :
3160 0 : create_qp_err:
3161 : // Do NOT call RsDrvDestroyCq — CQs are not owned by this QP
3162 0 : RsDeinitMemPool(qpCb);
3163 0 : (void)RsQpcbDeinit(rdevCb, qpCb);
3164 :
3165 0 : rs_init_mem_err:
3166 0 : rs_qpcb_init_err:
3167 0 : pthread_mutex_destroy(&qpCb->cqeErrInfo.mutex);
3168 :
3169 0 : cqe_mutex_init_err:
3170 0 : pthread_mutex_destroy(&qpCb->qpMutex);
3171 :
3172 0 : qp_mutex_init_err:
3173 0 : free(qpCb);
3174 0 : qpCb = NULL;
3175 :
3176 0 : return ret;
3177 : }
3178 :
3179 7 : RS_ATTRI_VISI_DEF int RsCqDestroy(unsigned int phyId, unsigned int rdevIndex, struct CqAttr *attr)
3180 : {
3181 : int ret;
3182 7 : struct RsRdevCb *rdevCb = NULL;
3183 7 : struct RsCqContext *cqContext = NULL;
3184 :
3185 7 : ret = RsQueryRdevCb(phyId, rdevIndex, &rdevCb);
3186 7 : CHK_PRT_RETURN(ret, hccp_err("rs_query_rdev_cb phyId[%u] rdev_index[%u], ret %d", phyId, rdevIndex, ret), ret);
3187 :
3188 7 : cqContext = *attr->qpContext;
3189 :
3190 7 : ret = RsDrvDestroyCqEvent(cqContext);
3191 7 : if (ret) {
3192 0 : hccp_err("rs_drv_destroy_cq_event failed ret %d", ret);
3193 : }
3194 :
3195 7 : if (cqContext->channel != NULL) {
3196 5 : ret = RsEpollCtl(rdevCb->rsCb->connCb.epollfd, EPOLL_CTL_DEL, cqContext->channel->fd, EPOLLIN | EPOLLRDHUP);
3197 : #ifndef CA_CONFIG_LLT
3198 : if (ret) {
3199 : hccp_err("del channel fd failed ret %d", ret);
3200 : }
3201 : #endif
3202 5 : RsIbvDestroyCompChannel(cqContext->channel);
3203 5 : cqContext->channel = NULL;
3204 : }
3205 :
3206 7 : free(cqContext);
3207 7 : cqContext = NULL;
3208 :
3209 7 : return ret;
3210 : }
3211 :
3212 4 : RS_ATTRI_VISI_DEF int RsNormalQpCreate(unsigned int phyId, unsigned int rdevIndex, struct ibv_qp_init_attr *qpInitAttr,
3213 : struct RsQpResp *qpResp, void **qp)
3214 : {
3215 4 : struct RsCqContext *cqContext = NULL;
3216 4 : struct RsRdevCb *rdevCb = NULL;
3217 4 : struct RsQpCb *qpCb = NULL;
3218 : int ret;
3219 :
3220 4 : CHK_PRT_RETURN(qpResp == NULL, hccp_err("qp_resp is NULL!"), -EINVAL);
3221 4 : ret = RsQueryRdevCb(phyId, rdevIndex, &rdevCb);
3222 4 : CHK_PRT_RETURN(ret != 0, hccp_err("rs_query_rdev_cb phyId[%u] rdev_index[%u], ret %d", phyId, rdevIndex, ret), ret);
3223 :
3224 4 : CHK_PRT_RETURN(qpInitAttr == NULL, hccp_err("qp_init_attr is NULL!"), -EINVAL);
3225 :
3226 4 : cqContext = qpInitAttr->qp_context;
3227 4 : CHK_PRT_RETURN(cqContext == NULL, hccp_err("cq_context is NULL!"), -EINVAL);
3228 3 : CHK_PRT_RETURN(rdevCb != cqContext->rdevCb,
3229 : hccp_err("rs_query_rdev_cb phyId[%u] rdev_index[%u],"
3230 : "rdevCb is invalid.",
3231 : phyId, rdevIndex),
3232 : -EINVAL);
3233 :
3234 3 : ret = RsBuildUpQpcb(cqContext, qpInitAttr, &qpCb);
3235 3 : CHK_PRT_RETURN(ret, hccp_err("rs_build_up_qpcb failed, ret:%d", ret), ret);
3236 :
3237 3 : ret = RsDrvNormalQpCreate(qpCb, qpInitAttr);
3238 3 : if (ret) {
3239 1 : hccp_err("create drv qp create failed:%d", ret);
3240 1 : goto create_qp_err;
3241 : }
3242 :
3243 2 : RS_PTHREAD_MUTEX_LOCK(&rdevCb->rdevMutex);
3244 2 : RsListAddTail(&qpCb->list, &rdevCb->qpList);
3245 2 : RS_PTHREAD_MUTEX_ULOCK(&rdevCb->rdevMutex);
3246 2 : rdevCb->qpCnt++;
3247 2 : *qp = qpCb->ibQp;
3248 2 : qpResp->qpn = (unsigned int)qpCb->qpInfoLo.qpn;
3249 2 : qpResp->gidIdx = (unsigned int)qpCb->qpInfoLo.gidIdx;
3250 2 : qpResp->psn = (unsigned int)qpCb->qpInfoLo.psn;
3251 2 : qpResp->gid = qpCb->qpInfoLo.gid;
3252 :
3253 2 : hccp_info("qp %d create qp.", qpResp->qpn);
3254 :
3255 2 : return 0;
3256 :
3257 1 : create_qp_err:
3258 1 : pthread_mutex_destroy(&qpCb->qpMutex);
3259 1 : free(qpCb);
3260 1 : qpCb = NULL;
3261 1 : return ret;
3262 : }
3263 :
3264 2 : RS_ATTRI_VISI_DEF int RsNormalQpDestroy(unsigned int phyId, unsigned int rdevIndex, unsigned int qpn)
3265 : {
3266 2 : struct RsQpCb *qpCb = NULL;
3267 : int ret;
3268 :
3269 2 : RS_QP_PARA_CHECK(phyId);
3270 2 : ret = RsQpn2qpcb(phyId, rdevIndex, qpn, &qpCb);
3271 2 : CHK_PRT_RETURN(ret != 0 || qpCb == NULL, hccp_err("get qp cb failed qpn %u, ret %d", qpn, ret), ret);
3272 :
3273 2 : RsQpRelease(qpCb);
3274 :
3275 : // destroy qp
3276 2 : RsDrvQpDestroy(qpCb);
3277 :
3278 2 : qpCb->rdevCb->qpCnt--;
3279 :
3280 2 : pthread_mutex_destroy(&qpCb->qpMutex);
3281 2 : hccp_info("qp %d destroy qp, send wr[%u].", qpn, qpCb->sendWrNum);
3282 :
3283 2 : free(qpCb);
3284 2 : qpCb = NULL;
3285 2 : return ret;
3286 : }
3287 :
3288 4 : RS_ATTRI_VISI_DEF int RsCreateCompChannel(unsigned int phyId, unsigned int rdevIndex, void **compChannel)
3289 : {
3290 : int ret;
3291 : unsigned int chipId;
3292 :
3293 4 : struct RsRdevCb *rdevCb = NULL;
3294 :
3295 4 : CHK_PRT_RETURN(compChannel == NULL || phyId >= RS_MAX_DEV_NUM,
3296 : hccp_err("param err, NULL pointer or phyId:%u >= [%d]", phyId, RS_MAX_DEV_NUM), -EINVAL);
3297 :
3298 4 : ret = rsGetLocalDevIDByHostDevID(phyId, &chipId);
3299 4 : CHK_PRT_RETURN(ret,
3300 : hccp_err("rs_create_comp_channel rsGetLocalDevIDByHostDevID phyId[%u] invalid, ret %d", phyId, ret), ret);
3301 :
3302 3 : ret = RsRdev2rdevCb(chipId, rdevIndex, &rdevCb);
3303 3 : CHK_PRT_RETURN(ret != 0 || rdevCb == NULL, hccp_err("rs_rdev2rdev_cb for chip_id[%u] failed, ret %d", chipId, ret),
3304 : ret);
3305 :
3306 2 : *compChannel = (void *)RsIbvCreateCompChannel(rdevCb->ibCtx);
3307 2 : if (*compChannel == NULL) {
3308 1 : hccp_err("rs_ibv_create_comp_channel failed, errno(%d)", errno);
3309 1 : return -EOPENSRC;
3310 : }
3311 1 : hccp_info("create comp channel success!");
3312 1 : return 0;
3313 : }
3314 :
3315 2 : RS_ATTRI_VISI_DEF int RsDestroyCompChannel(void *compChannel)
3316 : {
3317 : int ret;
3318 2 : struct ibv_comp_channel *rsCompChannel = (struct ibv_comp_channel *)compChannel;
3319 :
3320 2 : ret = RsIbvDestroyCompChannel(rsCompChannel);
3321 2 : CHK_PRT_RETURN(ret, hccp_err("rs_destroy_comp_channel failed."), ret);
3322 1 : hccp_info("destroy comp channel success!");
3323 :
3324 1 : return 0;
3325 : }
3326 :
3327 5 : RS_ATTRI_VISI_DEF int RsCreateSrq(unsigned int phyId, unsigned int rdevIndex, struct SrqAttr *attr)
3328 : {
3329 : int ret;
3330 5 : struct RsRdevCb *rdevCb = NULL;
3331 5 : struct RsCqContext *cqContext = NULL;
3332 :
3333 5 : CHK_PRT_RETURN(attr == NULL || attr->context == NULL || attr->ibRecvCq == NULL || attr->ibSrq == NULL ||
3334 : phyId >= RS_MAX_DEV_NUM,
3335 : hccp_err("param err, NULL pointer or phyId:%u >= [%d]", phyId, RS_MAX_DEV_NUM), -EINVAL);
3336 :
3337 5 : ret = RsQueryRdevCb(phyId, rdevIndex, &rdevCb);
3338 5 : CHK_PRT_RETURN(ret, hccp_err("rs_query_rdev_cb phyId[%u] rdev_index[%u], ret %d", phyId, rdevIndex, ret), ret);
3339 :
3340 4 : cqContext = calloc(1, sizeof(struct RsCqContext));
3341 4 : if (cqContext == NULL) {
3342 1 : return -ENOMEM;
3343 : }
3344 :
3345 3 : cqContext->rdevCb = rdevCb;
3346 3 : cqContext->eqNum = 0;
3347 3 : cqContext->cqCreateMode = RS_SRQ_CQ_CREATE;
3348 3 : *attr->context = cqContext;
3349 :
3350 3 : struct CqAttr cqAttr = {0};
3351 3 : cqAttr.recvCqDepth = attr->cqDepth;
3352 3 : cqAttr.recvCqEventId = attr->srqEventId;
3353 3 : cqAttr.ibRecvCq = attr->ibRecvCq;
3354 : // 创建srq cq
3355 3 : ret = RsCreateCqEvent(cqContext, &cqAttr);
3356 3 : if (ret) {
3357 1 : hccp_err("rs_create_cq_event create cq failed! ret:%d", ret);
3358 1 : goto create_cq_event_err;
3359 : }
3360 2 : cqContext->ibSrqCq = *attr->ibRecvCq;
3361 :
3362 2 : struct ibv_srq_init_attr srqInitAttr = {.attr = {.max_wr = attr->srqDepth, .max_sge = attr->maxSge}};
3363 2 : hccp_info("max_wr [%u], max_sge[%u]", srqInitAttr.attr.max_wr, srqInitAttr.attr.max_sge);
3364 :
3365 : // 创建srq
3366 2 : *attr->ibSrq = RsIbvCreateSrq(rdevCb->ibPd, &srqInitAttr);
3367 2 : if (*attr->ibSrq == NULL) {
3368 1 : hccp_err("rs_ibv_create_srq failed.");
3369 1 : ret = -EOPENSRC;
3370 1 : goto create_srq_err;
3371 : }
3372 1 : hccp_info("create srq success!");
3373 :
3374 1 : return 0;
3375 1 : create_cq_event_err:
3376 2 : create_srq_err:
3377 2 : cqAttr.qpContext = attr->context;
3378 2 : RsCqDestroy(phyId, rdevIndex, &cqAttr);
3379 :
3380 2 : return ret;
3381 : }
3382 :
3383 1 : RS_ATTRI_VISI_DEF int RsDestroySrq(unsigned int phyId, unsigned int rdevIndex, struct SrqAttr *attr)
3384 : {
3385 : int ret;
3386 :
3387 1 : CHK_PRT_RETURN(*attr->context == NULL || *attr->ibSrq == NULL || phyId >= RS_MAX_DEV_NUM,
3388 : hccp_err("param err, NULL pointer or phyId:%u >= [%d]", phyId, RS_MAX_DEV_NUM), -EINVAL);
3389 :
3390 1 : struct CqAttr cqAttr = {0};
3391 1 : struct RsCqContext *cqContext = *attr->context;
3392 1 : cqAttr.qpContext = attr->context;
3393 1 : RsIbvAckCqEvents(cqContext->ibSrqCq, cqContext->numRecvCqEvents);
3394 :
3395 : // 销毁srq cq
3396 1 : ret = RsCqDestroy(phyId, rdevIndex, &cqAttr);
3397 1 : CHK_PRT_RETURN(ret, hccp_err("rs_cq_destroy destroy cq failed! ret:%d", ret), ret);
3398 :
3399 1 : ret = RsIbvDestroySrq(*attr->ibSrq);
3400 1 : CHK_PRT_RETURN(ret, hccp_err("rs_ibv_destroy_srq failed."), ret);
3401 :
3402 1 : return 0;
3403 : }
3404 :
3405 2 : RS_ATTRI_VISI_DEF int RsGetLiteSupport(unsigned int phyId, unsigned int rdevIndex, int *supportLite)
3406 : {
3407 : int ret;
3408 : unsigned int chipId;
3409 2 : struct RsRdevCb *rdevCb = NULL;
3410 :
3411 2 : RS_CHECK_POINTER_NULL_RETURN_INT(supportLite);
3412 :
3413 2 : CHK_PRT_RETURN(phyId >= RS_MAX_DEV_NUM, hccp_err("rs set param error ! phyId:%u", phyId), -EINVAL);
3414 2 : ret = rsGetLocalDevIDByHostDevID(phyId, &chipId);
3415 2 : CHK_PRT_RETURN(ret, hccp_err("phyId[%u] invalid, ret %d", phyId, ret), ret);
3416 :
3417 2 : ret = RsRdev2rdevCb(chipId, rdevIndex, &rdevCb);
3418 2 : CHK_PRT_RETURN(ret != 0 || rdevCb == NULL, hccp_err("rs_rdev2rdev_cb for chip_id[%u] failed, ret %d", chipId, ret),
3419 : ret);
3420 :
3421 2 : rdevCb->supportLite = 1;
3422 2 : *supportLite = rdevCb->supportLite;
3423 :
3424 2 : return 0;
3425 : }
3426 :
3427 2 : RS_ATTRI_VISI_DEF int RsGetLiteRdevCap(unsigned int phyId, unsigned int rdevIndex, struct LiteRdevCapResp *resp)
3428 : {
3429 : int ret;
3430 : unsigned int chipId;
3431 2 : struct RsRdevCb *rdevCb = NULL;
3432 :
3433 2 : RS_CHECK_POINTER_NULL_RETURN_INT(resp);
3434 :
3435 2 : CHK_PRT_RETURN(phyId >= RS_MAX_DEV_NUM, hccp_err("rs set param error ! phyId:%u", phyId), -EINVAL);
3436 2 : ret = rsGetLocalDevIDByHostDevID(phyId, &chipId);
3437 2 : CHK_PRT_RETURN(ret, hccp_err("phyId[%u] invalid, ret %d", phyId, ret), ret);
3438 :
3439 2 : ret = RsRdev2rdevCb(chipId, rdevIndex, &rdevCb);
3440 2 : CHK_PRT_RETURN(ret != 0 || rdevCb == NULL, hccp_err("rs_rdev2rdev_cb for chip_id[%u] failed, ret %d", chipId, ret),
3441 : ret);
3442 :
3443 2 : ret = RsIbvExpQueryDevice(rdevCb->ibCtx, &resp->cap);
3444 2 : CHK_PRT_RETURN(ret != 0, hccp_err("rs_ibv_exp_query_device for phyId[%u] failed, ret %d", phyId, ret), ret);
3445 :
3446 2 : ret = memcpy_s(resp, sizeof(struct dev_cap_info), (void *)&resp->cap, sizeof(resp->cap));
3447 2 : if (ret) {
3448 0 : hccp_err("memcpy_s failed, ret:%d, src_len:%u, dst_len:%u", ret, (unsigned int)sizeof(resp->cap),
3449 : (unsigned int)sizeof(struct dev_cap_info));
3450 0 : return ret;
3451 : }
3452 :
3453 2 : return 0;
3454 : }
3455 :
3456 2 : RS_ATTRI_VISI_DEF int RsGetLiteQpCqAttr(unsigned int phyId, unsigned int rdevIndex, unsigned int qpn,
3457 : struct LiteQpCqAttrResp *resp)
3458 : {
3459 : int ret;
3460 2 : struct RsQpCb *qpCb = NULL;
3461 :
3462 2 : RS_CHECK_POINTER_NULL_RETURN_INT(resp);
3463 :
3464 2 : RS_QP_PARA_CHECK(phyId);
3465 2 : ret = RsQpn2qpcb(phyId, rdevIndex, qpn, &qpCb);
3466 2 : CHK_PRT_RETURN(ret != 0 || qpCb == NULL, hccp_err("get qp cb failed qpn %u, ret %d", qpn, ret), ret);
3467 :
3468 2 : ret = memcpy_s(resp, sizeof(struct LiteQpCqAttrResp), (void *)&qpCb->qpResp, sizeof(qpCb->qpResp));
3469 2 : if (ret) {
3470 0 : hccp_err("memcpy_s failed, ret:%d, src_len:%u, dst_len:%u", ret, (unsigned int)sizeof(qpCb->qpResp),
3471 : (unsigned int)sizeof(struct LiteQpCqAttrResp));
3472 0 : return ret;
3473 : }
3474 :
3475 2 : return 0;
3476 : }
3477 :
3478 0 : RS_ATTRI_VISI_DEF int RsGetLiteQpAttr(unsigned int phyId, unsigned int rdevIndex, unsigned int qpn,
3479 : struct LiteQpAttrResp *resp)
3480 : {
3481 : int ret;
3482 0 : struct RsQpCb *qpCb = NULL;
3483 :
3484 0 : RS_CHECK_POINTER_NULL_RETURN_INT(resp);
3485 :
3486 0 : RS_QP_PARA_CHECK(phyId);
3487 0 : ret = RsQpn2qpcb(phyId, rdevIndex, qpn, &qpCb);
3488 0 : CHK_PRT_RETURN(ret != 0 || qpCb == NULL, hccp_err("get qp cb failed qpn %u, ret %d", qpn, ret), ret);
3489 :
3490 0 : ret = memcpy_s(resp, sizeof(struct LiteQpAttrResp), (void *)&qpCb->qpResp.qpData, sizeof(qpCb->qpResp.qpData));
3491 0 : if (ret) {
3492 0 : hccp_err("memcpy_s failed, ret:%d, src_len:%u, dst_len:%u", ret, (unsigned int)sizeof(qpCb->qpResp.qpData),
3493 : (unsigned int)sizeof(struct LiteQpAttrResp));
3494 0 : return ret;
3495 : }
3496 :
3497 0 : return 0;
3498 : }
3499 :
3500 2 : RS_ATTRI_VISI_DEF int RsGetLiteMemAttr(unsigned int phyId, unsigned int rdevIndex, unsigned int qpn,
3501 : struct LiteMemAttrResp *resp)
3502 : {
3503 : int ret;
3504 2 : struct RsQpCb *qpCb = NULL;
3505 :
3506 2 : RS_CHECK_POINTER_NULL_RETURN_INT(resp);
3507 :
3508 2 : RS_QP_PARA_CHECK(phyId);
3509 2 : ret = RsQpn2qpcb(phyId, rdevIndex, qpn, &qpCb);
3510 2 : CHK_PRT_RETURN(ret != 0 || qpCb == NULL, hccp_err("get qp cb failed qpn %u, ret %d", qpn, ret), ret);
3511 :
3512 2 : ret = memcpy_s(resp, sizeof(struct LiteMemAttrResp), (void *)&qpCb->memResp, sizeof(qpCb->memResp));
3513 2 : if (ret) {
3514 0 : hccp_err("memcpy_s failed, ret:%d, src_len:%u, dst_len:%u", ret, (unsigned int)sizeof(qpCb->memResp),
3515 : (unsigned int)sizeof(struct LiteMemAttrResp));
3516 0 : return ret;
3517 : }
3518 :
3519 2 : return 0;
3520 : }
3521 :
3522 4 : STATIC void RsGetMrInfo(struct RsQpCb *qpCb, struct LiteMrInfo *mr, uint32_t maxMrNum, struct RsListHead *mrList)
3523 : {
3524 4 : struct RsMrCb *mrTmp = NULL;
3525 4 : struct RsMrCb *mrTmp2 = NULL;
3526 4 : uint32_t i = 0;
3527 :
3528 4 : RS_PTHREAD_MUTEX_LOCK(&qpCb->qpMutex);
3529 4 : RS_LIST_GET_HEAD_ENTRY(mrTmp, mrTmp2, mrList, list, struct RsMrCb);
3530 6 : for (; (&mrTmp->list) != mrList; mrTmp = mrTmp2, mrTmp2 = list_entry(mrTmp2->list.next, struct RsMrCb, list)) {
3531 2 : if (i < maxMrNum) {
3532 2 : mr[i].key = mrTmp->mrInfo.rkey;
3533 2 : mr[i].addr = mrTmp->mrInfo.addr;
3534 2 : mr[i].len = mrTmp->mrInfo.len;
3535 2 : i++;
3536 : } else {
3537 0 : break;
3538 : }
3539 : }
3540 :
3541 4 : RS_PTHREAD_MUTEX_ULOCK(&qpCb->qpMutex);
3542 4 : }
3543 :
3544 2 : RS_ATTRI_VISI_DEF int RsGetLiteConnectedInfo(unsigned int phyId, unsigned int rdevIndex, unsigned int qpn,
3545 : struct LiteConnectedInfoResp *resp)
3546 : {
3547 : int ret;
3548 2 : struct RsQpCb *qpCb = NULL;
3549 :
3550 2 : RS_CHECK_POINTER_NULL_RETURN_INT(resp);
3551 2 : RS_QP_PARA_CHECK(phyId);
3552 2 : ret = RsQpn2qpcb(phyId, rdevIndex, qpn, &qpCb);
3553 2 : CHK_PRT_RETURN(ret != 0 || qpCb == NULL, hccp_err("get qp cb failed qpn %u, ret %d", qpn, ret), ret);
3554 :
3555 2 : resp->state = (unsigned int)qpCb->state;
3556 2 : if (resp->state == RS_QP_STATUS_CONNECTED) {
3557 2 : RsGetMrInfo(qpCb, &resp->localMr[0], RA_MR_MAX_NUM, &qpCb->mrList);
3558 2 : RsGetMrInfo(qpCb, &resp->remMr[0], RA_MR_MAX_NUM, &qpCb->remMrList);
3559 2 : resp->qosAttr.sl = qpCb->qosAttr.sl;
3560 2 : resp->qosAttr.tc = qpCb->qosAttr.tc;
3561 : }
3562 :
3563 2 : return 0;
3564 : }
|
