Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #define _GNU_SOURCE
12 : #include <unistd.h>
13 : #include <stdlib.h>
14 : #include <netinet/in.h>
15 : #include <arpa/inet.h>
16 : #include <dlfcn.h>
17 : #include <sys/types.h>
18 : #include <sys/stat.h>
19 : #include <sys/epoll.h>
20 : #include <sys/eventfd.h>
21 : #include <sys/socket.h>
22 : #include <errno.h>
23 : #include "securec.h"
24 : #include "rs.h"
25 : #include "ra_rs_err.h"
26 : #include "rs_common_inner.h"
27 : #include "rs_inner.h"
28 : #include "rs_rdma_inner.h"
29 : #include "rs_epoll.h"
30 : #include "dl_hal_function.h"
31 : #include "dl_ibverbs_function.h"
32 : #include "dl_ibv_extend_function.h"
33 : #include "rs_drv_socket.h"
34 : #include "rs_drv_rdma.h"
35 : #include "rs_rdma.h"
36 :
37 : unsigned int gRsSendWrNum = 0;
38 :
39 : STATIC struct RsListHead gRsTypicalCqList;
40 : STATIC pthread_mutex_t gRsTypicalCqMutex = PTHREAD_MUTEX_INITIALIZER;
41 :
42 2 : STATIC void RsBufPrint(char *addr, int len)
43 : {
44 : int i;
45 :
46 130 : for (i = 0; i < len; i++) {
47 128 : hccp_info("0x%02x ", *(addr + i));
48 : }
49 2 : }
50 :
51 137 : STATIC int RsGetQpcb(struct RsRdevCb *rdevCb, uint32_t qpn, struct RsQpCb **qpCb)
52 : {
53 137 : struct RsQpCb *qpCbTmp = NULL;
54 137 : struct RsQpCb *qpCbTmp2 = NULL;
55 :
56 137 : RS_LIST_GET_HEAD_ENTRY(qpCbTmp, qpCbTmp2, &rdevCb->qpList, list, struct RsQpCb);
57 180 : for (; (&qpCbTmp->list) != &rdevCb->qpList;
58 43 : qpCbTmp = qpCbTmp2, qpCbTmp2 = list_entry(qpCbTmp2->list.next, struct RsQpCb, list)) {
59 171 : if (qpCbTmp->ibQp->qp_num == qpn) {
60 128 : *qpCb = qpCbTmp;
61 128 : return 0;
62 : }
63 : }
64 :
65 9 : *qpCb = NULL;
66 9 : hccp_err("qp_cb for qp %u do not available!", qpn);
67 :
68 9 : return -ENODEV;
69 : }
70 :
71 140 : int RsQpn2qpcb(unsigned int phyId, unsigned int rdevIndex, uint32_t qpn, struct RsQpCb **qpCb)
72 : {
73 : int ret;
74 : unsigned int chipId;
75 140 : struct rs_cb *rsCb = NULL;
76 140 : struct RsRdevCb *rdevCb = NULL;
77 :
78 140 : CHK_PRT_RETURN(phyId >= RS_MAX_DEV_NUM, hccp_err("rs set param error! phyId:%u", phyId), -EINVAL);
79 :
80 137 : ret = rsGetLocalDevIDByHostDevID(phyId, &chipId);
81 137 : CHK_PRT_RETURN(ret, hccp_err("rs_qpn2qpcb rsGetLocalDevIDByHostDevID phyId[%u] invalid, ret:%d", phyId, ret), ret);
82 :
83 137 : ret = RsDev2rscb(chipId, &rsCb, false);
84 137 : CHK_PRT_RETURN(ret, hccp_err("rs_qpn2qpcb get rs_cb failed, ret:%d", ret), -ENODEV);
85 :
86 137 : ret = RsGetRdevCb(rsCb, rdevIndex, &rdevCb);
87 137 : CHK_PRT_RETURN(ret, hccp_err("rs_get_rdev_cb failed! ret:%d, rdevIndex:%u", ret, rdevIndex), ret);
88 :
89 137 : ret = RsGetQpcb(rdevCb, qpn, qpCb);
90 137 : CHK_PRT_RETURN(ret, hccp_err("rs_get_qpcb failed! ret:%d, qpn:%u", ret, qpn), ret);
91 :
92 128 : return 0;
93 : }
94 :
95 44 : STATIC int RsGetMrcb(struct RsQpCb *qpCb, uint64_t addr, struct RsMrCb **mrCb, struct RsListHead *mrList)
96 : {
97 44 : struct RsMrCb *mrTmp = NULL;
98 44 : struct RsMrCb *mrTmp2 = NULL;
99 :
100 44 : RS_PTHREAD_MUTEX_LOCK(&qpCb->qpMutex);
101 44 : RS_LIST_GET_HEAD_ENTRY(mrTmp, mrTmp2, mrList, list, struct RsMrCb);
102 86 : for (; (&mrTmp->list) != mrList; mrTmp = mrTmp2, mrTmp2 = list_entry(mrTmp2->list.next, struct RsMrCb, list)) {
103 66 : if ((mrTmp->mrInfo.addr <= addr) && (addr < mrTmp->mrInfo.addr + mrTmp->mrInfo.len)) {
104 24 : *mrCb = mrTmp;
105 24 : RS_PTHREAD_MUTEX_ULOCK(&qpCb->qpMutex);
106 24 : return 0;
107 : }
108 : }
109 :
110 20 : *mrCb = NULL;
111 20 : RS_PTHREAD_MUTEX_ULOCK(&qpCb->qpMutex);
112 :
113 20 : hccp_info("cannot find mrcb for addr@0x%lx !", addr);
114 :
115 20 : return -ENODEV;
116 : }
117 :
118 18 : STATIC void *RsNotifyMrListAdd(struct RsQpCb *qpCb, const char *buf)
119 : {
120 : int ret;
121 : struct RsMrCb *notifyMrCb;
122 :
123 18 : notifyMrCb = calloc(1, sizeof(struct RsMrCb));
124 18 : CHK_PRT_RETURN(notifyMrCb == NULL, hccp_err("notify_mr_cb calloc failed"), NULL);
125 16 : ret = memcpy_s(¬ifyMrCb->mrInfo, sizeof(struct RsMrInfo), &((const struct RsQpInfo *)buf)->notifyMr,
126 : sizeof(struct RsMrInfo));
127 16 : if (ret) {
128 1 : hccp_err("memcpy_s failed, ret:%d, src_len:%u, dst_len:%u", ret, sizeof(struct RsMrInfo),
129 : sizeof(struct RsMrInfo));
130 1 : free(notifyMrCb);
131 1 : notifyMrCb = NULL;
132 1 : return NULL;
133 : }
134 :
135 15 : hccp_info("qpn is %d, rdevIndex:%u, chipId %u, recv notify va is 0x%llx, notify size is %llu", qpCb->qpInfoLo.qpn,
136 : qpCb->rdevCb->rdevIndex, qpCb->rdevCb->rsCb->chipId, notifyMrCb->mrInfo.addr, notifyMrCb->mrInfo.len);
137 :
138 15 : RsListAddTail(¬ifyMrCb->list, &qpCb->remMrList);
139 :
140 15 : return notifyMrCb;
141 : }
142 :
143 21 : STATIC int RsQpStateModify(struct RsQpCb *qpCb)
144 : {
145 21 : struct ibv_qp_init_attr initAttr = {0};
146 21 : struct ibv_qp_attr attr = {0};
147 : enum ibv_qp_state state;
148 : int ret;
149 :
150 : // see ib_modify_qp_is_ok for status modify, only support modify qp from INIT to RTR
151 21 : ret = RsIbvQueryQp(qpCb->ibQp, &attr, IBV_QP_STATE, &initAttr);
152 21 : if (ret != 0) {
153 1 : hccp_warn("rs_ibv_query_qp qpn:%d unsuccessful, ret:%d", qpCb->qpInfoLo.qpn, ret);
154 1 : state = IBV_QPS_UNKNOWN;
155 : } else {
156 20 : state = attr.qp_state;
157 : }
158 :
159 : // disallow modify qp from IBV_QPS_RTS to IBV_QPS_RTS
160 21 : if (state == IBV_QPS_RTS) {
161 1 : hccp_err("qpn:%d disallow modify from %d", qpCb->qpInfoLo.qpn, state);
162 1 : return -EINVAL;
163 : }
164 :
165 20 : hccp_info("qpn:%d state:%d start modify", qpCb->qpInfoLo.qpn, state);
166 :
167 : // modify qp from others to RESET
168 20 : if (state != IBV_QPS_RESET && state != IBV_QPS_INIT && state != IBV_QPS_RTR) {
169 1 : ret = RsDrvQpStateModifytoReset(qpCb);
170 1 : CHK_PRT_RETURN(ret, hccp_err("qpn:%d modify %d to reset failed, ret:%d", qpCb->qpInfoLo.qpn, state, ret), ret);
171 0 : state = IBV_QPS_RESET;
172 : }
173 :
174 : // modify qp from RESET to INIT
175 19 : if (state == IBV_QPS_RESET) {
176 19 : ret = RsDrvQpStateModifytoInit(qpCb, &attr);
177 19 : CHK_PRT_RETURN(ret, hccp_err("qpn:%d modify %d to init failed, ret %d", qpCb->qpInfoLo.qpn, state, ret), ret);
178 18 : state = IBV_QPS_INIT;
179 : }
180 :
181 : // modify qp from INIT to RTR
182 18 : if (state == IBV_QPS_INIT) {
183 18 : ret = RsDrvQpStateModifytoRtr(qpCb, &attr);
184 18 : CHK_PRT_RETURN(ret, hccp_err("qpn:%d modify %d to rtr failed, ret %d", qpCb->qpInfoLo.qpn, state, ret), ret);
185 16 : state = IBV_QPS_RTR;
186 : }
187 :
188 : // modify qp from RTR to RTS
189 16 : if (state == IBV_QPS_RTR) {
190 16 : ret = RsDrvQpStateModifytoRts(qpCb, &attr);
191 16 : CHK_PRT_RETURN(ret, hccp_err("qpn:%d modify %d to rts failed, ret %d", qpCb->qpInfoLo.qpn, state, ret), ret);
192 : }
193 :
194 16 : hccp_info("local qpn[%d] remote qpn[%d] modify succ", qpCb->qpInfoLo.qpn, qpCb->qpInfoRem.qpn);
195 :
196 16 : return 0;
197 : }
198 :
199 17 : STATIC int RsEpollRecvQpHandle(struct RsQpCb *qpCb, const char *bufTmp)
200 : {
201 : int ret;
202 17 : float timeCost = 0.0;
203 :
204 17 : ret = memcpy_s(&qpCb->qpInfoRem, sizeof(struct RsQpInfo), bufTmp, sizeof(struct RsQpInfo));
205 17 : CHK_PRT_RETURN(ret,
206 : hccp_err("memcpy_s failed[%d], dest size:%d, src size:%d", ret, sizeof(struct RsQpInfo),
207 : sizeof(struct RsQpInfo)),
208 : -ENOMEM);
209 :
210 : /* modify qp state to RTR/RTS */
211 17 : ret = RsQpStateModify(qpCb);
212 17 : CHK_PRT_RETURN(ret,
213 : hccp_err("rs_qp_state_modify local qpn[%d] remote qpn[%d] failed ret[%d]", qpCb->qpInfoLo.qpn,
214 : qpCb->qpInfoRem.qpn, ret),
215 : ret);
216 :
217 16 : RsGetCurTime(&qpCb->endTime);
218 16 : HccpTimeInterval(&qpCb->endTime, &qpCb->startTime, &timeCost);
219 16 : if (timeCost > RS_EXPECT_TIME_MAX) {
220 2 : hccp_warn("local qpn[%d] remote qpn [%d] connect success cost[%f] more than[%f]ms!", qpCb->qpInfoLo.qpn,
221 : qpCb->qpInfoRem.qpn, timeCost, RS_EXPECT_TIME_MAX);
222 : } else {
223 14 : hccp_info("local qpn[%d] remote qpn [%d] connect success! cost [%f] ms", qpCb->qpInfoLo.qpn,
224 : qpCb->qpInfoRem.qpn, timeCost);
225 : }
226 :
227 16 : hccp_info("qp [%d] state has been migrate to RTS!, qpCb state is %d", qpCb->qpInfoLo.qpn, qpCb->state);
228 :
229 16 : return 0;
230 : }
231 :
232 14 : STATIC void *RsEpollRecvMrHandle(struct RsQpCb *qpCb, const char *bufTmp)
233 : {
234 : int ret;
235 : struct RsMrCb *mrCb;
236 :
237 14 : mrCb = calloc(1, sizeof(struct RsMrCb));
238 14 : CHK_PRT_RETURN(mrCb == NULL, hccp_err("mr_cb calloc failed"), NULL);
239 13 : ret = memcpy_s(&mrCb->mrInfo, sizeof(struct RsMrInfo), bufTmp, sizeof(struct RsMrInfo));
240 13 : if (ret) {
241 0 : hccp_err("memcpy_s failed[%d], dest size:%u, src size:%u", ret, sizeof(struct RsMrInfo),
242 : sizeof(struct RsMrInfo));
243 0 : free(mrCb);
244 0 : mrCb = NULL;
245 0 : return NULL;
246 : }
247 :
248 13 : RsListAddTail(&mrCb->list, &qpCb->remMrList);
249 :
250 13 : hccp_info("recv mr addr is 0x%llx", mrCb->mrInfo.addr);
251 13 : hccp_info("recv mr len is %llu", mrCb->mrInfo.len);
252 :
253 13 : return mrCb;
254 : }
255 :
256 17 : STATIC int RsCmdQpInfoHandle(struct RsQpCb *qpCb, unsigned int totalSize, const char *bufTmp, unsigned int curSize,
257 : bool *flag)
258 : {
259 : int ret;
260 17 : CHK_PRT_RETURN((totalSize - curSize) < sizeof(struct RsQpInfo),
261 : hccp_info("qp_info remain size"
262 : "[%u] < size [%u], wait for next recv",
263 : totalSize - curSize, sizeof(struct RsQpInfo)),
264 : -EINVAL);
265 :
266 17 : ret = RsEpollRecvQpHandle(qpCb, bufTmp);
267 17 : CHK_PRT_RETURN(ret, hccp_err("rs_epoll_recv_qp_handle failed! ret[%d]", ret), ret);
268 :
269 16 : RsNotifyMrListAdd(qpCb, bufTmp);
270 16 : hccp_info("rs_notify_mr_list_add");
271 :
272 16 : *flag = true;
273 16 : hccp_info("qp_info cur_size(%u) len(%u) !", curSize, sizeof(struct RsQpInfo));
274 :
275 16 : return 0;
276 : }
277 :
278 14 : STATIC int RsCmdMrInfoHandle(struct RsQpCb *qpCb, unsigned int totalSize, const char *bufTmp, unsigned int curSize,
279 : bool *flag)
280 : {
281 14 : CHK_PRT_RETURN((totalSize - curSize) < sizeof(struct RsMrInfo),
282 : hccp_info("mr_info remain size"
283 : "[%u] < size [%u], wait for next recv",
284 : totalSize - curSize, sizeof(struct RsMrInfo)),
285 : -EINVAL);
286 :
287 14 : (void)RsEpollRecvMrHandle(qpCb, bufTmp);
288 :
289 14 : *flag = true;
290 :
291 14 : hccp_info("mr_info cur_size(%u) len(%u) !", curSize, sizeof(struct RsMrInfo));
292 :
293 14 : return 0;
294 : }
295 :
296 14 : STATIC int RsCmdLenInfoHandle(struct RsQpCb *qpCb, unsigned int totalSize, const char *bufTmp, unsigned int curSize,
297 : bool *flag)
298 : {
299 14 : CHK_PRT_RETURN((totalSize - curSize) < sizeof(struct RsQpLenInfo),
300 : hccp_info("len_info remain size"
301 : "[%u] < size [%u], wait for next recv",
302 : totalSize - curSize, sizeof(struct RsQpLenInfo)),
303 : -EINVAL);
304 :
305 14 : qpCb->expectLen = *((const uint32_t *)(bufTmp + sizeof(uint32_t)));
306 :
307 14 : *flag = true;
308 :
309 14 : return 0;
310 : }
311 :
312 15 : STATIC void RsEpollRecvHandleRemain(struct RsQpCb *qpCb, unsigned int totalSize, unsigned int curSize, bool flag,
313 : const char *bufTmp)
314 : {
315 15 : int ret = 0;
316 :
317 15 : qpCb->remainSize = totalSize - curSize;
318 15 : if ((qpCb->remainSize > 0) && (flag == true)) {
319 1 : ret = memcpy_s(qpCb->qpMrBuf, RS_BUF_SIZE, bufTmp, qpCb->remainSize);
320 1 : if (ret) {
321 1 : hccp_err("memcpy_s failed, ret:%d, remainSize:%u", ret, qpCb->remainSize);
322 1 : return;
323 : }
324 : }
325 :
326 14 : return;
327 : }
328 :
329 18 : STATIC void RsEpollRecvHandle(struct RsQpCb *qpCb, char *buf, int size)
330 : {
331 18 : unsigned int totalSize = qpCb->remainSize + (unsigned int)size;
332 18 : char *bufTmp = (char *)qpCb->qpMrBuf;
333 18 : unsigned int curSize = 0;
334 18 : bool flag = false;
335 : uint32_t cmd;
336 : int ret;
337 :
338 18 : hccp_info("Message for qp:%d, qpCb->remainSize:%u, size:%d", qpCb->qpInfoLo.qpn, qpCb->remainSize, size);
339 18 : ret = memcpy_s(qpCb->qpMrBuf + qpCb->remainSize, RS_BUF_SIZE - qpCb->remainSize, buf, size);
340 18 : if (ret) {
341 2 : hccp_err("memcpy_s failed, ret:%d, remainSize:%u, size:%d", ret, qpCb->remainSize, size);
342 4 : return;
343 : }
344 :
345 : do {
346 47 : cmd = *((uint32_t *)bufTmp);
347 47 : switch (cmd) {
348 17 : case RS_CMD_QP_INFO:
349 17 : ret = RsCmdQpInfoHandle(qpCb, totalSize, bufTmp, curSize, &flag);
350 17 : if (ret) {
351 1 : goto out;
352 : }
353 :
354 16 : curSize += sizeof(struct RsQpInfo);
355 16 : bufTmp = qpCb->qpMrBuf + curSize;
356 16 : break;
357 14 : case RS_CMD_MR_INFO:
358 14 : ret = RsCmdMrInfoHandle(qpCb, totalSize, bufTmp, curSize, &flag);
359 14 : if (ret) {
360 0 : goto out;
361 : }
362 :
363 14 : curSize += sizeof(struct RsMrInfo);
364 14 : bufTmp = qpCb->qpMrBuf + curSize;
365 14 : break;
366 14 : case RS_CMD_LEN_INFO:
367 14 : ret = RsCmdLenInfoHandle(qpCb, totalSize, bufTmp, curSize, &flag);
368 14 : if (ret) {
369 0 : goto out;
370 : }
371 14 : curSize += sizeof(struct RsQpLenInfo);
372 14 : bufTmp = qpCb->qpMrBuf + curSize;
373 14 : break;
374 2 : default:
375 2 : hccp_warn("qp %d, unknown cmd(0x%x)!", qpCb->qpInfoLo.qpn, cmd);
376 2 : RsBufPrint(buf, size);
377 2 : return;
378 : }
379 44 : } while (curSize < totalSize);
380 :
381 13 : out:
382 14 : RsEpollRecvHandleRemain(qpCb, totalSize, curSize, flag, bufTmp);
383 : }
384 :
385 27 : STATIC void RsQpMrRecvHandle(int fd, struct RsQpCb *qpCb)
386 : {
387 : char buf[RS_BUF_SIZE];
388 : int size;
389 : int ret;
390 :
391 27 : RS_PTHREAD_MUTEX_LOCK(&qpCb->qpMutex);
392 :
393 27 : size = RsSocketRecv(fd, buf, RS_BUF_SIZE - qpCb->remainSize);
394 27 : hccp_dbg("fd %d qpn %d read size = %d, qpCb->remainSize:%u", fd, qpCb->qpInfoLo.qpn, size, qpCb->remainSize);
395 :
396 27 : if (size > 0) {
397 13 : qpCb->recvLen += (uint32_t)size;
398 13 : RsEpollRecvHandle(qpCb, buf, size);
399 14 : } else if (size == 0) {
400 0 : hccp_dbg("fd %d read size = %d, remote fd has been closed, fd cannot use !", fd, size);
401 : #ifdef CA_CONFIG_LLT
402 0 : qpCb->state = RS_QP_STATUS_REM_FD_CLOSE;
403 : #endif
404 : } else {
405 14 : ret = errno;
406 14 : hccp_dbg("no data available, errno:%d", ret);
407 : }
408 27 : RS_PTHREAD_MUTEX_ULOCK(&qpCb->qpMutex);
409 :
410 27 : return;
411 : }
412 :
413 0 : STATIC int RsHandleQpMrEpollEvent(struct RsRdevCb *rdevCb, int fd)
414 : {
415 : struct RsQpCb *qpCb;
416 0 : struct RsQpCb *qpCb2 = NULL;
417 :
418 : /* QP event, QP info exchange */
419 0 : RS_LIST_GET_HEAD_ENTRY(qpCb, qpCb2, &rdevCb->qpList, list, struct RsQpCb);
420 0 : for (; (&qpCb->list) != &rdevCb->qpList; qpCb = qpCb2, qpCb2 = list_entry(qpCb2->list.next, struct RsQpCb, list)) {
421 0 : if (qpCb->channel == NULL) {
422 0 : continue;
423 : }
424 0 : if (qpCb->srqContext != NULL && qpCb->srqContext->channel->fd == fd) {
425 0 : hccp_dbg("fd %d poll cq!", fd);
426 0 : RsDrvPollSrqCqHandle(qpCb);
427 0 : return 0;
428 : }
429 0 : if (fd == qpCb->channel->fd) {
430 0 : hccp_dbg("fd %d poll cq!", fd);
431 0 : RsDrvPollCqHandle(qpCb);
432 0 : return 0;
433 : }
434 : }
435 0 : return -ENODEV;
436 : }
437 :
438 0 : int RsEpollEventQpMrInHandle(struct rs_cb *rsCb, int fd)
439 : {
440 : int ret;
441 0 : struct RsRdevCb *rdevCbTmp = NULL;
442 0 : struct RsRdevCb *rdevCbTmp2 = NULL;
443 :
444 0 : if (rsCb->protocol != PROTOCOL_RDMA) {
445 0 : return -ENODEV;
446 : }
447 :
448 0 : RS_LIST_GET_HEAD_ENTRY(rdevCbTmp, rdevCbTmp2, &rsCb->rdevList, list, struct RsRdevCb);
449 0 : for (; (&rdevCbTmp->list) != &rsCb->rdevList;
450 0 : rdevCbTmp = rdevCbTmp2, rdevCbTmp2 = list_entry(rdevCbTmp2->list.next, struct RsRdevCb, list)) {
451 0 : RS_PTHREAD_MUTEX_LOCK(&rdevCbTmp->rdevMutex);
452 0 : ret = RsHandleQpMrEpollEvent(rdevCbTmp, fd);
453 0 : RS_PTHREAD_MUTEX_ULOCK(&rdevCbTmp->rdevMutex);
454 0 : if (ret == 0) {
455 0 : return 0;
456 : }
457 : }
458 0 : return -ENODEV;
459 : }
460 :
461 28 : STATIC int RsMrInfoSync(struct RsMrCb *mrCb)
462 : {
463 : int ret;
464 :
465 28 : hccp_info("mr state:%d, addr:0x%lx", mrCb->state, mrCb->mrInfo.addr);
466 :
467 28 : CHK_PRT_RETURN(mrCb->state & RS_MR_STATE_SYNCED,
468 : hccp_warn("mr synced ! mr_cb->flag[%d] & [%d] != 0", mrCb->state, RS_MR_STATE_SYNCED), 0);
469 :
470 : /*
471 : * no socket available for MR_INFO exchange if allowed
472 : * need exchange when socket available
473 : */
474 27 : CHK_PRT_RETURN(mrCb->qpCb->connInfo == NULL, hccp_warn("no conn available !"), 0);
475 :
476 27 : CHK_PRT_RETURN(mrCb->qpCb->state == RS_QP_STATUS_REM_FD_CLOSE,
477 : hccp_warn("remote qp fd closed,"
478 : "cann not use it anymore! status[%d](RS_QP_STATUS_REM_FD_CLOSE)",
479 : mrCb->qpCb->state),
480 : -EFAULT);
481 :
482 27 : CHK_PRT_RETURN(mrCb->qpCb->connInfo->connfd == RS_FD_INVALID,
483 : hccp_warn("rm info sync failed! fd not ready!"
484 : "connfd[%d](RS_FD_INVALID)",
485 : mrCb->qpCb->connInfo->connfd),
486 : -ENETUNREACH);
487 :
488 26 : mrCb->mrInfo.cmd = (unsigned int)RS_CMD_MR_INFO;
489 26 : ret = RsSocketSend(mrCb->qpCb->connInfo->connfd, &mrCb->mrInfo, sizeof(struct RsMrInfo));
490 26 : CHK_PRT_RETURN(ret != sizeof(struct RsMrInfo),
491 : hccp_err("mr_info send %d/%ld incomplete", ret, sizeof(struct RsMrInfo)), -EAGAIN);
492 :
493 25 : mrCb->qpCb->sendLen += (uint32_t)ret;
494 25 : mrCb->state |= RS_MR_STATE_SYNCED;
495 25 : hccp_info("after send mr state:%d, addr:0x%lx", mrCb->state, mrCb->mrInfo.addr);
496 :
497 25 : return 0;
498 : }
499 :
500 5 : STATIC int RsMrPrepareRoceSign(unsigned int phyId, struct RsRdevCb *devCb, struct roce_process_sign *roceSign)
501 : {
502 5 : unsigned int tmpPhyId = phyId;
503 : unsigned int chipId;
504 : int ret;
505 :
506 : // reg mr with backup phyId
507 5 : if (devCb->backupInfo.backupFlag) {
508 0 : tmpPhyId = devCb->backupInfo.rdevInfo.phyId;
509 0 : ret = DlDrvGetLocalDevIdByHostDevId(tmpPhyId, &chipId);
510 : } else {
511 5 : ret = rsGetLocalDevIDByHostDevID(tmpPhyId, &chipId);
512 : }
513 5 : CHK_PRT_RETURN(ret != 0,
514 : hccp_err("get chipId failed, ret %d, phyid[%u] backupFlag[%d]", ret, tmpPhyId, devCb->backupInfo.backupFlag),
515 : -EACCES);
516 5 : roceSign->tgid = devCb->rsCb->pRsSign.tgid;
517 5 : roceSign->devid = chipId;
518 5 : roceSign->vfid = 0;
519 5 : ret = strcpy_s(roceSign->sign, PROCESS_RS_SIGN_LENGTH, devCb->rsCb->pRsSign.sign);
520 5 : CHK_PRT_RETURN(ret != 0, hccp_err("Invalid pid sign, ret(%d)", ret), -ESAFEFUNC);
521 5 : return 0;
522 : }
523 :
524 8 : STATIC int RsMrPreReg(unsigned int phyId, struct RsQpCb *qpCb, struct RsMrCb *mrCb, struct RdmaMrRegInfo *mrRegInfo)
525 : {
526 8 : unsigned long long len = mrRegInfo->len;
527 8 : struct roce_process_sign roceSign = {0};
528 8 : int access = mrRegInfo->access;
529 8 : char *addr = mrRegInfo->addr;
530 : int ret;
531 :
532 8 : if (qpCb->rdevCb->rsCb->hccpMode == NETWORK_PEER_ONLINE || qpCb->rdevCb->rsCb->hccpMode == NETWORK_ONLINE ||
533 7 : qpCb->isExp == RS_NOT_EXP) {
534 3 : mrCb->ibMr = RsDrvMrReg(qpCb->ibPd, addr, len, access);
535 3 : CHK_PRT_RETURN(mrCb->ibMr == NULL, hccp_err("rs_drv_mr_reg addr is NULL len[%lld] failed ", len), -EACCES);
536 : } else {
537 5 : ret = RsMrPrepareRoceSign(phyId, qpCb->rdevCb, &roceSign);
538 5 : CHK_PRT_RETURN(ret != 0, hccp_err("RsMrPrepareRoceSign failed, ret(%d)", ret), ret);
539 5 : mrCb->ibMr = RsDrvExpMrReg(qpCb->ibPd, addr, len, access, roceSign);
540 5 : CHK_PRT_RETURN(mrCb->ibMr == NULL, hccp_err("rs_drv_exp_mr_reg addr is NULL len[%lld] failed ", len), -EACCES);
541 : }
542 :
543 8 : mrCb->mrInfo.cmd = (unsigned int)RS_CMD_MR_INFO;
544 8 : mrCb->mrInfo.addr = (uintptr_t)addr;
545 8 : mrCb->mrInfo.len = len;
546 8 : mrCb->mrInfo.rkey = mrCb->ibMr->rkey;
547 :
548 8 : RS_PTHREAD_MUTEX_LOCK(&qpCb->qpMutex);
549 8 : RsListAddTail(&mrCb->list, &qpCb->mrList);
550 8 : RS_PTHREAD_MUTEX_ULOCK(&qpCb->qpMutex);
551 :
552 8 : qpCb->mrNum++;
553 8 : return 0;
554 : }
555 :
556 41 : STATIC int RsCallocMr(int num, struct RsMrCb **mrCb)
557 : {
558 41 : CHK_PRT_RETURN(num <= 0, hccp_err("invalid num for mr calloc"), -EINVAL);
559 :
560 40 : *mrCb = calloc(num, sizeof(struct RsMrCb));
561 40 : CHK_PRT_RETURN((*mrCb) == NULL, hccp_err("calloc mr_cb failed"), -ENOMEM);
562 40 : return 0;
563 : }
564 :
565 54 : STATIC int RsCallocQpcb(int num, struct RsQpCb **qpCb)
566 : {
567 54 : if (num <= 0) {
568 1 : return -EINVAL;
569 : }
570 :
571 53 : *qpCb = calloc(num, sizeof(struct RsQpCb));
572 53 : if ((*qpCb) == NULL) {
573 1 : return -ENOMEM;
574 : }
575 :
576 52 : return 0;
577 : }
578 :
579 10 : RS_ATTRI_VISI_DEF int RsMrReg(unsigned int phyId, unsigned int rdevIndex, unsigned int qpn,
580 : struct RdmaMrRegInfo *mrRegInfo)
581 : {
582 : int ret;
583 10 : struct RsQpCb *qpCb = NULL;
584 10 : struct RsMrCb *mrCb = NULL;
585 :
586 10 : CHK_PRT_RETURN(mrRegInfo == NULL || mrRegInfo->addr == NULL || mrRegInfo->len == 0 || phyId >= RS_MAX_DEV_NUM,
587 : hccp_err("param err, NULL pointer or phyId:%u >= [%d]", phyId, RS_MAX_DEV_NUM), -EINVAL);
588 :
589 9 : hccp_info("qpn[%u], len[0x%llx], access[%d]", qpn, mrRegInfo->len, mrRegInfo->access);
590 :
591 9 : ret = RsQpn2qpcb(phyId, rdevIndex, qpn, &qpCb);
592 9 : CHK_PRT_RETURN(ret, hccp_err("rs_qpn2qpcb qpn[%d] ret[%d] failed ", qpn, ret), ret);
593 :
594 8 : CHK_PRT_RETURN(qpCb->mrNum >= RS_MR_NUM_MAX, hccp_err("Exceeded the maximum MR limit %d", qpCb->mrNum), -EINVAL);
595 :
596 8 : ret = RsGetMrcb(qpCb, (uintptr_t)mrRegInfo->addr, &mrCb, &qpCb->mrList);
597 8 : if (ret == 0) {
598 0 : hccp_warn("mr already registered");
599 0 : goto found;
600 : }
601 :
602 8 : ret = RsCallocMr(1, &mrCb);
603 8 : CHK_PRT_RETURN(ret, hccp_err("calloc mr failed"), ret);
604 :
605 8 : mrCb->qpCb = qpCb;
606 :
607 8 : ret = RsMrPreReg(phyId, qpCb, mrCb, mrRegInfo);
608 8 : if (ret) {
609 0 : hccp_err("pre reg mr failed, qpn %u, ret %d", qpn, ret);
610 0 : goto reg_err;
611 : }
612 :
613 8 : found:
614 8 : mrRegInfo->lkey = mrCb->ibMr->lkey;
615 8 : mrRegInfo->rkey = mrCb->ibMr->rkey;
616 :
617 8 : hccp_info("rs_mr_reg succ, state:%u", mrCb->state);
618 8 : return 0;
619 :
620 0 : reg_err:
621 0 : free(mrCb);
622 0 : mrCb = NULL;
623 :
624 0 : return ret;
625 : }
626 :
627 10 : RS_ATTRI_VISI_DEF int RsMrDereg(unsigned int phyId, unsigned int rdevIndex, unsigned int qpn, char *addr)
628 : {
629 : int ret;
630 10 : struct RsQpCb *qpCb = NULL;
631 10 : struct RsMrCb *mrCb = NULL;
632 :
633 10 : hccp_dbg("start rs_mr_dereg");
634 10 : RS_CHECK_POINTER_NULL_RETURN_INT(addr);
635 9 : CHK_PRT_RETURN(phyId >= RS_MAX_DEV_NUM, hccp_err("phyId:%u >= [%d], is invalid", phyId, RS_MAX_DEV_NUM), -EINVAL);
636 :
637 9 : ret = RsQpn2qpcb(phyId, rdevIndex, qpn, &qpCb);
638 9 : CHK_PRT_RETURN(ret, hccp_err("rs_qpn2qpcb failed ret[%d]", ret), ret);
639 :
640 7 : CHK_PRT_RETURN(RsGetMrcb(qpCb, (uintptr_t)addr, &mrCb, &qpCb->mrList),
641 : hccp_err("rs_get_mrcb failed "
642 : "g_rs_send_wr_num[%u]",
643 : gRsSendWrNum),
644 : -EFAULT);
645 :
646 6 : ret = RsDrvMrDereg(mrCb->ibMr);
647 6 : CHK_PRT_RETURN(ret, hccp_err("rs_drv_mr_dereg failed ret[%d] ", ret), -EACCES);
648 :
649 6 : RS_PTHREAD_MUTEX_LOCK(&qpCb->qpMutex);
650 6 : RsListDel(&mrCb->list);
651 6 : free(mrCb);
652 6 : mrCb = NULL;
653 6 : RS_PTHREAD_MUTEX_ULOCK(&qpCb->qpMutex);
654 6 : qpCb->mrNum--;
655 :
656 6 : hccp_dbg("qpn[%u] succ", qpn);
657 :
658 6 : return 0;
659 : }
660 :
661 2 : STATIC int RsRegisterUbSegment(int directFlag, uint64_t addr, uint64_t len)
662 : {
663 2 : struct DVattribute attr = {0};
664 2 : int ret = 0;
665 :
666 2 : if (directFlag != DIRECT_FLAG_UB) {
667 2 : return 0;
668 : }
669 :
670 0 : ret = DlDrvMemGetAttribute(addr, &attr);
671 0 : CHK_PRT_RETURN(ret != 0, hccp_err("DlDrvMemGetAttribute failed, ret:%d", ret), ret);
672 :
673 0 : if (attr.memType != DV_MEM_LOCK_DEV) { // not support host memory
674 0 : return ret;
675 : }
676 :
677 0 : ret = DlHalMemRegUbSegment(attr.devId, addr, len);
678 0 : CHK_PRT_RETURN(ret != 0, hccp_err("DlHalMemRegUbSegment failed, ret:%d devId:%u len:%u", ret, attr.devId, len),
679 : ret);
680 :
681 0 : return ret;
682 : }
683 :
684 2 : STATIC int RsUnRegisterUbSegment(int directFlag, uint64_t addr)
685 : {
686 2 : struct DVattribute attr = {0};
687 2 : int ret = 0;
688 :
689 2 : if (directFlag != DIRECT_FLAG_UB) {
690 2 : return 0;
691 : }
692 :
693 0 : ret = DlDrvMemGetAttribute(addr, &attr);
694 0 : CHK_PRT_RETURN(ret != 0, hccp_err("DrvMemGetAttribute failed, ret:%d", ret), ret);
695 :
696 0 : if (attr.memType != DV_MEM_LOCK_DEV) { // not support host memory
697 0 : return ret;
698 : }
699 :
700 0 : ret = DlHalMemUnRegUbSegment(attr.devId, addr);
701 0 : if (ret != 0) {
702 0 : hccp_err("DlHalMemUnRegUbSegment failed, ret:%d devId:%u", ret, attr.devId);
703 : }
704 :
705 0 : return ret;
706 : }
707 :
708 3 : RS_ATTRI_VISI_DEF int RsRegisterMr(unsigned int phyId, unsigned int rdevIndex, struct RdmaMrRegInfo *mrRegInfo,
709 : void **mrHandle)
710 : {
711 3 : RS_CHECK_POINTER_NULL_RETURN_INT(mrHandle);
712 :
713 : int ret;
714 : unsigned int chipId;
715 3 : struct RsRdevCb *rdevCb = NULL;
716 3 : struct ibv_mr *rsMrHandle = NULL;
717 :
718 3 : CHK_PRT_RETURN(mrRegInfo == NULL || mrRegInfo->addr == NULL || mrRegInfo->len == 0 || phyId >= RS_MAX_DEV_NUM,
719 : hccp_err("param err, NULL pointer or phyId:%u >= [%d]", phyId, RS_MAX_DEV_NUM), -EINVAL);
720 :
721 2 : hccp_info("[rs_register_mr] len[0x%llx], access[%d]", mrRegInfo->len, mrRegInfo->access);
722 :
723 2 : ret = rsGetLocalDevIDByHostDevID(phyId, &chipId);
724 2 : CHK_PRT_RETURN(ret, hccp_err("rs_register_mr rsGetLocalDevIDByHostDevID phyId[%u] invalid, ret %d", phyId, ret),
725 : ret);
726 :
727 2 : ret = RsRdev2rdevCb(chipId, rdevIndex, &rdevCb);
728 2 : CHK_PRT_RETURN(ret != 0 || rdevCb == NULL, hccp_err("rs_rdev2rdev_cb for chip_id[%u] failed, ret %d", chipId, ret),
729 : ret);
730 :
731 2 : ret = RsRegisterUbSegment(rdevCb->directFlag, (uint64_t)(uintptr_t)mrRegInfo->addr, mrRegInfo->len);
732 2 : if (ret != 0) {
733 0 : hccp_err("RsRegisterUbSegment failed, ret[%d] vendor_id[0x%x] part_id[0x%x] directFlag[%u] "
734 : "addr[0x%llx] len[0x%llx]",
735 : ret, rdevCb->deviceAttr.vendor_id, rdevCb->deviceAttr.vendor_part_id, rdevCb->directFlag,
736 : (uint64_t)(uintptr_t)mrRegInfo->addr, mrRegInfo->len);
737 0 : goto mem_reg_err;
738 : }
739 :
740 2 : *mrHandle = (void *)RsDrvMrReg(rdevCb->ibPd, mrRegInfo->addr, mrRegInfo->len, mrRegInfo->access);
741 2 : if (*mrHandle == NULL) {
742 1 : hccp_warn("rs_drv_mr_reg addr is NULL len[0x%llx] access[%d] unsuccessful errno[%d]", mrRegInfo->len,
743 : mrRegInfo->access, errno);
744 1 : goto mr_reg_err;
745 : }
746 :
747 1 : rsMrHandle = (struct ibv_mr *)*mrHandle;
748 1 : mrRegInfo->lkey = rsMrHandle->lkey;
749 1 : mrRegInfo->rkey = rsMrHandle->rkey;
750 :
751 1 : hccp_info("rs_register_mr succ");
752 1 : return ret;
753 1 : mr_reg_err:
754 1 : (void)RsUnRegisterUbSegment(rdevCb->directFlag, (uint64_t)(uintptr_t)mrRegInfo->addr);
755 1 : mem_reg_err:
756 1 : mrRegInfo->lkey = 0;
757 :
758 1 : return ret;
759 : }
760 :
761 2 : STATIC int RsInitTypicalMrCb(unsigned int phyId, struct RdmaMrRegInfo *mrRegInfo, struct RsRdevCb *devCb,
762 : struct RsMrCb *mrCb)
763 : {
764 2 : unsigned long long len = mrRegInfo->len;
765 2 : struct roce_process_sign roceSign = {0};
766 2 : char *addr = (char *)mrRegInfo->addr;
767 2 : int access = mrRegInfo->access;
768 : int ret;
769 :
770 2 : if (devCb->rsCb->hccpMode == NETWORK_PEER_ONLINE || devCb->rsCb->hccpMode == NETWORK_ONLINE) {
771 2 : mrCb->ibMr = RsDrvMrReg(devCb->ibPd, addr, len, access);
772 2 : CHK_PRT_RETURN(mrCb->ibMr == NULL, hccp_err("rs_drv_mr_reg addr is NULL len[%lld] failed", len), -EACCES);
773 : } else {
774 0 : ret = RsMrPrepareRoceSign(phyId, devCb, &roceSign);
775 0 : CHK_PRT_RETURN(ret != 0, hccp_err("RsMrPrepareRoceSign failed, ret(%d)", ret), ret);
776 0 : mrCb->ibMr = RsDrvExpMrReg(devCb->ibPd, addr, len, access, roceSign);
777 0 : CHK_PRT_RETURN(mrCb->ibMr == NULL, hccp_err("rs_drv_exp_mr_reg addr is NULL len[%lld] failed", len), -EACCES);
778 : }
779 :
780 2 : mrCb->mrInfo.addr = (uintptr_t)addr;
781 2 : mrCb->mrInfo.len = len;
782 2 : mrCb->mrInfo.rkey = mrCb->ibMr->rkey;
783 :
784 2 : RS_PTHREAD_MUTEX_LOCK(&devCb->rdevMutex);
785 2 : RsListAddTail(&mrCb->list, &devCb->typicalMrList);
786 2 : RS_PTHREAD_MUTEX_ULOCK(&devCb->rdevMutex);
787 :
788 2 : return 0;
789 : }
790 :
791 1 : RS_ATTRI_VISI_DEF int RsTypicalRegisterMrV1(unsigned int phyId, unsigned int rdevIndex, struct RdmaMrRegInfo *mrRegInfo,
792 : void **mrHandle)
793 : {
794 1 : RS_CHECK_POINTER_NULL_RETURN_INT(mrHandle);
795 :
796 1 : struct RsMrCb *typicalMrCb = NULL;
797 1 : struct RsRdevCb *rdevCb = NULL;
798 : unsigned int chipId;
799 : int ret;
800 :
801 1 : CHK_PRT_RETURN(mrRegInfo == NULL || mrRegInfo->addr == NULL || mrRegInfo->len == 0 || phyId >= RS_MAX_DEV_NUM,
802 : hccp_err("param err, NULL pointer or phyId:%u >= [%d]", phyId, RS_MAX_DEV_NUM), -EINVAL);
803 :
804 1 : hccp_info("[rs_typical_register_mr] len[0x%llx], access[%d]", mrRegInfo->len, mrRegInfo->access);
805 :
806 1 : ret = rsGetLocalDevIDByHostDevID(phyId, &chipId);
807 1 : CHK_PRT_RETURN(ret != 0,
808 : hccp_err("rs_typical_register_mr rsGetLocalDevIDByHostDevID phyId[%u] invalid, ret %d", phyId, ret), ret);
809 :
810 1 : ret = RsRdev2rdevCb(chipId, rdevIndex, &rdevCb);
811 1 : CHK_PRT_RETURN(ret != 0 || rdevCb == NULL, hccp_err("rs_rdev2rdev_cb for chip_id[%u] failed, ret %d", chipId, ret),
812 : ret);
813 :
814 1 : ret = RsQueryMrCb(rdevCb, (uint64_t)(uintptr_t)mrRegInfo->addr, &typicalMrCb, &rdevCb->typicalMrList);
815 1 : if (ret == 0) {
816 0 : hccp_warn("typical mr already registered");
817 0 : goto found;
818 : }
819 :
820 1 : typicalMrCb = calloc(1, sizeof(struct RsMrCb));
821 1 : CHK_PRT_RETURN(typicalMrCb == NULL, hccp_err("calloc typical_mr_cb failed"), -ENOMEM);
822 1 : typicalMrCb->devCb = rdevCb;
823 :
824 1 : ret = RsInitTypicalMrCb(phyId, mrRegInfo, rdevCb, typicalMrCb);
825 1 : if (ret != 0) {
826 0 : hccp_err("rs_init_typical_mr_cb failed, devIndex[%u], ret[%d]", rdevIndex, ret);
827 0 : goto reg_err;
828 : }
829 :
830 1 : found:
831 1 : *mrHandle = typicalMrCb->ibMr;
832 1 : mrRegInfo->lkey = typicalMrCb->ibMr->lkey;
833 1 : mrRegInfo->rkey = typicalMrCb->ibMr->rkey;
834 1 : hccp_info("rs_typical_register_mr succ, state:%d", typicalMrCb->state);
835 1 : return 0;
836 :
837 0 : reg_err:
838 0 : free(typicalMrCb);
839 0 : typicalMrCb = NULL;
840 0 : return ret;
841 : }
842 :
843 1 : RS_ATTRI_VISI_DEF int RsTypicalRegisterMr(unsigned int phyId, unsigned int rdevIndex, struct RdmaMrRegInfo *mrRegInfo,
844 : void **mrHandle)
845 : {
846 1 : RS_CHECK_POINTER_NULL_RETURN_INT(mrHandle);
847 :
848 1 : struct RsMrCb *typicalMrCb = NULL;
849 1 : struct RsRdevCb *rdevCb = NULL;
850 : unsigned int chipId;
851 : int ret;
852 :
853 1 : CHK_PRT_RETURN(mrRegInfo == NULL || mrRegInfo->addr == NULL || mrRegInfo->len == 0 || phyId >= RS_MAX_DEV_NUM,
854 : hccp_err("param err, NULL pointer or phyId:%u >= [%d]", phyId, RS_MAX_DEV_NUM), -EINVAL);
855 :
856 1 : hccp_info("start register len[0x%llx], access[%d]", mrRegInfo->len, mrRegInfo->access);
857 :
858 1 : ret = rsGetLocalDevIDByHostDevID(phyId, &chipId);
859 1 : CHK_PRT_RETURN(ret != 0, hccp_err("rsGetLocalDevIDByHostDevID phyId[%u] invalid, ret %d", phyId, ret), ret);
860 :
861 1 : ret = RsRdev2rdevCb(chipId, rdevIndex, &rdevCb);
862 1 : CHK_PRT_RETURN(ret != 0 || rdevCb == NULL, hccp_err("rs_rdev2rdev_cb for chip_id[%u] failed, ret %d", chipId, ret),
863 : ret);
864 :
865 1 : typicalMrCb = calloc(1, sizeof(struct RsMrCb));
866 1 : CHK_PRT_RETURN(typicalMrCb == NULL, hccp_err("calloc typical_mr_cb failed"), -ENOMEM);
867 1 : typicalMrCb->devCb = rdevCb;
868 :
869 1 : ret = RsInitTypicalMrCb(phyId, mrRegInfo, rdevCb, typicalMrCb);
870 1 : if (ret != 0) {
871 0 : hccp_err("rs_init_typical_mr_cb failed, devIndex[%u], ret[%d]", rdevIndex, ret);
872 0 : goto reg_err;
873 : }
874 :
875 : // resv len as 1 to save addr for later unreg to query
876 1 : typicalMrCb->mrInfo.addr = (uint64_t)(uintptr_t)typicalMrCb->ibMr;
877 1 : typicalMrCb->mrInfo.len = 1U;
878 1 : *mrHandle = typicalMrCb->ibMr;
879 1 : mrRegInfo->lkey = typicalMrCb->ibMr->lkey;
880 1 : mrRegInfo->rkey = typicalMrCb->ibMr->rkey;
881 1 : hccp_info("register succ, state:%d", typicalMrCb->state);
882 1 : return 0;
883 :
884 0 : reg_err:
885 0 : free(typicalMrCb);
886 0 : typicalMrCb = NULL;
887 0 : return ret;
888 : }
889 :
890 4 : RS_ATTRI_VISI_DEF int RsRemapMr(unsigned int phyId, unsigned int rdevIndex, struct MemRemapInfo memList[],
891 : unsigned int memNum)
892 : {
893 4 : struct RsRdevCb *devCb = NULL;
894 4 : struct RsMrCb *mrCurr = NULL;
895 4 : struct RsMrCb *mrNext = NULL;
896 4 : unsigned long long addr = 0;
897 4 : bool isMemMatched = false;
898 : unsigned int chipId;
899 : unsigned int i;
900 : int ret;
901 :
902 4 : CHK_PRT_RETURN(phyId >= RS_MAX_DEV_NUM, hccp_err("phyId:%u >= %d, is invalid", phyId, RS_MAX_DEV_NUM), -EINVAL);
903 :
904 4 : ret = rsGetLocalDevIDByHostDevID(phyId, &chipId);
905 4 : CHK_PRT_RETURN(ret, hccp_err("rsGetLocalDevIDByHostDevID failed, phyId:%u invalid, ret:%d", phyId, ret), ret);
906 :
907 4 : ret = RsRdev2rdevCb(chipId, rdevIndex, &devCb);
908 4 : CHK_PRT_RETURN(devCb == NULL, hccp_err("rs_rdev2rdev_cb failed, chipId:%u, ret:%d", chipId, ret), -ENODEV);
909 :
910 6 : for (i = 0; i < memNum; i++) {
911 4 : isMemMatched = false;
912 4 : addr = (uint64_t)(uintptr_t)memList[i].addr;
913 4 : RS_PTHREAD_MUTEX_LOCK(&devCb->rdevMutex);
914 4 : RS_LIST_GET_HEAD_ENTRY(mrCurr, mrNext, &devCb->typicalMrList, list, struct RsMrCb);
915 7 : for (; (&mrCurr->list) != &devCb->typicalMrList;
916 3 : mrCurr = mrNext, mrNext = list_entry(mrNext->list.next, struct RsMrCb, list)) {
917 : // mem is out range of mr, continue to find next matching mr
918 3 : if ((addr < (uint64_t)(uintptr_t)mrCurr->ibMr->addr) || (memList[i].size > mrCurr->ibMr->length) ||
919 2 : (addr + memList[i].size < addr) ||
920 2 : (addr + memList[i].size > (uint64_t)(uintptr_t)mrCurr->ibMr->addr + mrCurr->ibMr->length)) {
921 1 : continue;
922 : }
923 :
924 : // each mr remap each corresponding mem
925 2 : ret = RsRoceRemapMr(mrCurr->ibMr, (struct hns_roce_mr_remap_info *)(void *)&memList[i], 1);
926 2 : if (ret != 0) {
927 0 : hccp_err("remap %u-th mem failed, ret:%d addr:0x%llx size:0x%llx", i, ret, addr, memList[i].size);
928 0 : RS_PTHREAD_MUTEX_ULOCK(&devCb->rdevMutex);
929 0 : return ret;
930 : }
931 2 : isMemMatched = true;
932 : }
933 4 : RS_PTHREAD_MUTEX_ULOCK(&devCb->rdevMutex);
934 :
935 4 : if (!isMemMatched) {
936 2 : hccp_err("find %u-th mem failed, addr:0x%llx size:0x%llx", i, addr, memList[i].size);
937 2 : return -ENODEV;
938 : }
939 2 : hccp_dbg("remap %u-th mem success, addr:0x%llx size:0x%llx", i, addr, memList[i].size);
940 : }
941 :
942 2 : return 0;
943 : }
944 :
945 2 : RS_ATTRI_VISI_DEF int RsTypicalDeregisterMr(unsigned int phyId, unsigned int devIndex, unsigned long long addr)
946 : {
947 2 : struct RsMrCb *typicalMrCb = NULL;
948 2 : struct RsRdevCb *devCb = NULL;
949 : unsigned int chipId;
950 : int ret;
951 :
952 2 : hccp_info("typical mr unreg start, addr[%llu]", addr);
953 2 : CHK_PRT_RETURN(phyId >= RS_MAX_DEV_NUM, hccp_err("phyId:%u >= %d, is invalid", phyId, RS_MAX_DEV_NUM), -EINVAL);
954 :
955 2 : ret = rsGetLocalDevIDByHostDevID(phyId, &chipId);
956 2 : CHK_PRT_RETURN(ret, hccp_err("phyId[%u] invalid, ret %d", phyId, ret), ret);
957 :
958 2 : ret = RsRdev2rdevCb(chipId, devIndex, &devCb);
959 2 : CHK_PRT_RETURN(ret != 0 || devCb == NULL,
960 : hccp_err("rs_rdev2rdev_cb get dev_cb failed for chip_id[%u], ret[%d]", chipId, ret), -ENODEV);
961 :
962 2 : ret = RsQueryMrCb(devCb, addr, &typicalMrCb, &devCb->typicalMrList);
963 2 : CHK_PRT_RETURN(ret, hccp_err("rs_query_mr_cb failed ret[%d]", ret), ret);
964 :
965 2 : ret = RsDrvMrDereg(typicalMrCb->ibMr);
966 2 : CHK_PRT_RETURN(ret, hccp_err("rs_drv_mr_dereg failed ret[%d]", ret), -EACCES);
967 :
968 2 : RS_PTHREAD_MUTEX_LOCK(&devCb->rdevMutex);
969 2 : RsListDel(&typicalMrCb->list);
970 2 : free(typicalMrCb);
971 2 : typicalMrCb = NULL;
972 2 : RS_PTHREAD_MUTEX_ULOCK(&devCb->rdevMutex);
973 :
974 2 : hccp_info("devIndex[%u] succ", devIndex);
975 :
976 2 : return 0;
977 : }
978 :
979 2 : RS_ATTRI_VISI_DEF int RsDeregisterMr(unsigned int phyId, unsigned int rdevIndex, void *mrHandle)
980 : {
981 2 : RS_CHECK_POINTER_NULL_RETURN_INT(mrHandle);
982 :
983 1 : struct ibv_mr *rsMrHandle = (struct ibv_mr *)mrHandle;
984 1 : uint64_t addr = (uint64_t)(uintptr_t)rsMrHandle->addr;
985 1 : struct RsRdevCb *devCb = NULL;
986 : unsigned int chipId;
987 1 : int ret = 0;
988 :
989 1 : ret = rsGetLocalDevIDByHostDevID(phyId, &chipId);
990 1 : CHK_PRT_RETURN(ret != 0, hccp_err("phyId[%u] invalid, ret %d", phyId, ret), ret);
991 :
992 1 : ret = RsRdev2rdevCb(chipId, rdevIndex, &devCb);
993 1 : CHK_PRT_RETURN(ret != 0 || devCb == NULL,
994 : hccp_err("rs_rdev2rdev_cb get dev_cb failed for chip_id[%u], ret[%d]", chipId, ret), -ENODEV);
995 :
996 1 : ret = RsDrvMrDereg(rsMrHandle);
997 1 : CHK_PRT_RETURN(ret != 0, hccp_err("rs_drv_mr_dereg failed ret[%d]", ret), -EACCES);
998 :
999 1 : ret = RsUnRegisterUbSegment(devCb->directFlag, addr);
1000 1 : CHK_PRT_RETURN(ret != 0,
1001 : hccp_err("RsUnRegisterUbSegment failed ret[%d], vendor_id[0x%x] part_id[0x%x]"
1002 : " directFlag[%u]",
1003 : ret, devCb->deviceAttr.vendor_id, devCb->deviceAttr.vendor_part_id, devCb->directFlag),
1004 : ret);
1005 :
1006 1 : hccp_info("rs_deregister_mr succ");
1007 1 : return 0;
1008 : }
1009 :
1010 7 : RS_ATTRI_VISI_DEF int RsSendWr(unsigned int phyId, unsigned int rdevIndex, uint32_t qpn, struct SendWr *wr,
1011 : struct SendWrRsp *wrRsp)
1012 : {
1013 : int ret;
1014 7 : struct RsQpCb *qpCb = NULL;
1015 7 : struct RsMrCb *mrCb = NULL;
1016 7 : struct RsMrCb *remMrCb = NULL;
1017 :
1018 7 : RS_CHECK_POINTER_NULL_RETURN_INT(wr);
1019 7 : RS_CHECK_POINTER_NULL_RETURN_INT(wr->bufList);
1020 7 : RS_CHECK_POINTER_NULL_RETURN_INT(wrRsp);
1021 :
1022 7 : CHK_PRT_RETURN(phyId >= RS_MAX_DEV_NUM, hccp_err("phyId:%u >= [%d], is invalid", phyId, RS_MAX_DEV_NUM), -EINVAL);
1023 :
1024 7 : CHK_PRT_RETURN(wr->bufNum > MAX_SGE_NUM || wr->bufNum == 0, hccp_err("invalid buf_num[%u]!", wr->bufNum), -EINVAL);
1025 :
1026 6 : CHK_PRT_RETURN(wr->bufList->len > RS_SGLIST_LEN_MAX || wr->bufList->len == 0,
1027 : hccp_err("sg list"
1028 : "len is more than 2G, len[%u]",
1029 : wr->bufList->len),
1030 : -EINVAL);
1031 :
1032 5 : if (RsQpn2qpcb(phyId, rdevIndex, qpn, &qpCb)) {
1033 1 : return -EACCES;
1034 : }
1035 :
1036 4 : qpCb->sendWrNum++;
1037 :
1038 4 : hccp_info("qpn %d, bufList[0].addr is 0x%llx", qpn, wr->bufList[0].addr);
1039 4 : if (RsGetMrcb(qpCb, wr->bufList[0].addr, &mrCb, &qpCb->mrList)) {
1040 1 : hccp_err("qpn %d, bufList[0].addr[0x%llx] len[0x%x] is invalid.", qpn, wr->bufList[0].addr, wr->bufList[0].len);
1041 1 : return -EFAULT;
1042 : }
1043 :
1044 : // send op no need to check & get remote mr
1045 3 : if (wr->op != RA_WR_SEND && wr->op != RA_WR_SEND_WITH_IMM) {
1046 3 : hccp_info("remote wr dst addr is 0x%llx", wr->dstAddr);
1047 3 : if (RsGetMrcb(qpCb, wr->dstAddr, &remMrCb, &qpCb->remMrList)) {
1048 1 : hccp_err("qpn %d, remote wr dst addr[0x%llx] len[0x%x] is invalid.", qpn, wr->dstAddr, wr->bufList[0].len);
1049 1 : return -ENOENT;
1050 : }
1051 : }
1052 :
1053 2 : ret = RsDrvSendExp(qpCb, mrCb, remMrCb, wr, wrRsp);
1054 2 : if (ret) {
1055 0 : hccp_err("send exp failed qpn %u, ret %d", qpn, ret);
1056 : }
1057 2 : gRsSendWrNum++;
1058 2 : return ret;
1059 : }
1060 :
1061 5 : STATIC void BuildUpWrWithKey(struct WrInfo *wr, struct ibv_sge *list, struct ibv_send_wr *ibWr)
1062 : {
1063 5 : list->addr = (uintptr_t)wr->memList.addr;
1064 5 : list->length = wr->memList.len;
1065 5 : list->lkey = wr->memList.lkey;
1066 :
1067 5 : ibWr->sg_list = list;
1068 5 : ibWr->opcode = wr->op;
1069 5 : ibWr->send_flags = (unsigned int)wr->sendFlags;
1070 5 : ibWr->imm_data = htobe32(wr->immData);
1071 :
1072 5 : ibWr->num_sge = 1; /* only support one sge */
1073 5 : ibWr->wr_id = wr->wrId;
1074 5 : if (wr->op != IBV_WR_SEND && wr->op != IBV_WR_SEND_WITH_IMM) {
1075 5 : ibWr->wr.rdma.rkey = wr->rkey;
1076 5 : ibWr->wr.rdma.remote_addr = wr->dstAddr;
1077 : }
1078 5 : }
1079 :
1080 0 : STATIC void RsSendBuildUpWr(struct RsMrCb *mrCb, struct WrInfo *wr, struct ibv_sge *list, struct ibv_send_wr *ibWr)
1081 : {
1082 0 : list->addr = (uintptr_t)wr->memList.addr;
1083 0 : list->lkey = mrCb->ibMr->lkey;
1084 0 : list->length = wr->memList.len;
1085 :
1086 0 : ibWr->sg_list = list;
1087 0 : ibWr->opcode = wr->op;
1088 0 : ibWr->imm_data = htobe32(wr->immData);
1089 0 : ibWr->send_flags = (unsigned int)wr->sendFlags;
1090 :
1091 0 : ibWr->num_sge = 1; /* only support one sge */
1092 0 : ibWr->wr_id = wr->wrId;
1093 0 : }
1094 :
1095 2 : STATIC void RsWirteAndReadBuildUpWr(struct RsMrCb *mrCb, struct RsMrCb *remMrCb, struct WrInfo *wr,
1096 : struct ibv_sge *list, struct ibv_send_wr *ibWr)
1097 : {
1098 2 : list->addr = (uintptr_t)wr->memList.addr;
1099 2 : list->length = wr->memList.len;
1100 2 : list->lkey = mrCb->ibMr->lkey;
1101 :
1102 2 : ibWr->sg_list = list;
1103 2 : ibWr->opcode = wr->op;
1104 2 : ibWr->send_flags = (unsigned int)wr->sendFlags;
1105 2 : ibWr->imm_data = htobe32(wr->immData);
1106 :
1107 2 : ibWr->num_sge = 1; /* only support one sge */
1108 2 : ibWr->wr_id = wr->wrId;
1109 2 : ibWr->wr.rdma.rkey = remMrCb->mrInfo.rkey;
1110 2 : ibWr->wr.rdma.remote_addr = wr->dstAddr;
1111 2 : }
1112 :
1113 13 : STATIC int RsBuildUpWrList(struct WrInfo *wrList, struct RsQpCb *qpCb, struct ibv_sge *list, struct ibv_send_wr *ibWr,
1114 : unsigned int i)
1115 : {
1116 13 : struct RsMrCb *mrCb = NULL;
1117 13 : struct RsMrCb *remMrCb = NULL;
1118 13 : CHK_PRT_RETURN(wrList[i].memList.len > RS_SGLIST_LEN_MAX,
1119 : hccp_err("sg list len is more than 2G, len[%u]", wrList[i].memList.len), -EINVAL);
1120 :
1121 12 : hccp_dbg("qpn %d, bufList[0].addr is 0x%llx", qpCb->ibQp->qp_num, wrList[i].memList.addr);
1122 12 : if (RsGetMrcb(qpCb, wrList[i].memList.addr, &mrCb, &qpCb->mrList)) {
1123 2 : hccp_err("qpn %d, bufList[0].addr[0x%llx] len[0x%x] is invalid.", qpCb->ibQp->qp_num, wrList[i].memList.addr,
1124 : wrList[i].memList.len);
1125 2 : return -EFAULT;
1126 : }
1127 :
1128 : // send op no need to check & get remote mr
1129 10 : if (wrList[i].op != IBV_WR_SEND && wrList[i].op != IBV_WR_SEND_WITH_IMM) {
1130 10 : hccp_dbg("remote wr dst addr is 0x%llx", wrList[i].dstAddr);
1131 10 : if (RsGetMrcb(qpCb, wrList[i].dstAddr, &remMrCb, &qpCb->remMrList)) {
1132 8 : hccp_err("qpn %d, remote wr dst addr[0x%llx] len[0x%x] is invalid.", qpCb->ibQp->qp_num, wrList[i].dstAddr,
1133 : wrList[i].memList.len);
1134 8 : return -ENOENT;
1135 : }
1136 2 : RsWirteAndReadBuildUpWr(mrCb, remMrCb, &wrList[i], &list[i], &ibWr[i]);
1137 : } else {
1138 0 : RsSendBuildUpWr(mrCb, &wrList[i], &list[i], &ibWr[i]);
1139 : }
1140 :
1141 2 : return 0;
1142 : }
1143 :
1144 6 : STATIC int RsBuildUpWrListWithKey(struct WrInfo *wrList, struct ibv_sge *list, struct ibv_send_wr *ibWr, unsigned int i)
1145 : {
1146 6 : CHK_PRT_RETURN(wrList[i].memList.len > RS_SGLIST_LEN_MAX,
1147 : hccp_err("sg list len is more than 2G, len[%u]", wrList[i].memList.len), -EINVAL);
1148 :
1149 5 : BuildUpWrWithKey(&wrList[i], &list[i], &ibWr[i]);
1150 5 : return 0;
1151 : }
1152 :
1153 8 : STATIC int RsSendNormalWrlist(struct RsQpCb *qpCb, struct WrInfo *wrList, unsigned int sendNum,
1154 : unsigned int *completeNum, unsigned int keyFlag)
1155 : {
1156 : int ret;
1157 : unsigned int i, j;
1158 :
1159 8 : struct ibv_send_wr *badWr = NULL;
1160 8 : CHK_PRT_RETURN(sendNum > MAX_WR_NUM || sendNum == 0, hccp_err("send num[%u] is invalid!", sendNum), -EINVAL);
1161 8 : struct ibv_send_wr *ibWr = (struct ibv_send_wr *)calloc(sendNum, sizeof(struct ibv_send_wr));
1162 8 : CHK_PRT_RETURN(ibWr == NULL, hccp_err("calloc ib_wr failed!"), -ENOSPC);
1163 :
1164 8 : struct ibv_sge *list = (struct ibv_sge *)calloc(sendNum, sizeof(struct ibv_sge));
1165 8 : if (list == NULL) {
1166 0 : hccp_err("calloc list failed!");
1167 0 : ret = -ENOSPC;
1168 0 : goto alloc_fail;
1169 : }
1170 :
1171 10 : for (i = 0; i < sendNum; i++) {
1172 6 : ret = (keyFlag == 0) ? RsBuildUpWrList(wrList, qpCb, list, ibWr, i)
1173 8 : : RsBuildUpWrListWithKey(wrList, list, ibWr, i);
1174 8 : if (ret) {
1175 6 : goto input_err;
1176 : }
1177 2 : j = i + 1;
1178 2 : ibWr[i].next = (i < sendNum - 1) ? &ibWr[j] : NULL;
1179 : }
1180 :
1181 2 : ret = RsIbvPostSend(qpCb->ibQp, &ibWr[0], &badWr);
1182 2 : if (ret == 0) {
1183 2 : *completeNum = sendNum;
1184 0 : } else if (ret == -ENOMEM) {
1185 0 : *completeNum = (unsigned int)((void *)badWr - (void *)ibWr) / sizeof(struct ibv_send_wr);
1186 0 : hccp_dbg("post send wqe overflow, completeNum[%d]", *completeNum);
1187 : } else {
1188 0 : hccp_err("ibv_post_send failed, ret[%d]", ret);
1189 0 : *completeNum = 0;
1190 : }
1191 2 : qpCb->sendWrNum = qpCb->sendWrNum + (*completeNum);
1192 :
1193 8 : input_err:
1194 8 : free(list);
1195 8 : list = NULL;
1196 8 : alloc_fail:
1197 8 : free(ibWr);
1198 8 : ibWr = NULL;
1199 8 : return (ret == -ENOMEM) ? 0 : ret;
1200 : }
1201 :
1202 11 : STATIC int RsSendExpWrlist(struct RsQpCb *qpCb, struct WrInfo *wrList, unsigned int sendNum, struct SendWrRsp *wrRsp,
1203 : unsigned int *completeNum, unsigned int keyFlag)
1204 : {
1205 11 : struct ibv_post_send_ext_attr extAttr = {0};
1206 11 : struct ibv_post_send_ext_resp extRsp = {0};
1207 11 : struct ibv_send_wr *badWr = NULL;
1208 11 : struct wr_exp_rsp expRsp = {0};
1209 11 : struct ibv_send_wr ibWr = {0};
1210 11 : struct ibv_sge list = {0};
1211 : unsigned int i;
1212 11 : int ret = 0;
1213 :
1214 14 : for (i = 0; i < sendNum; i++) {
1215 : // reuse code: only need to build up one wr once a time
1216 7 : ret = (keyFlag == 0) ? RsBuildUpWrList(&wrList[i], qpCb, &list, &ibWr, 0)
1217 11 : : RsBuildUpWrListWithKey(&wrList[i], &list, &ibWr, 0);
1218 11 : if (ret != 0) {
1219 6 : hccp_err("qpn:%u key_flag:%u build_up_wr i:%u failed, ret:%d", qpCb->ibQp->qp_num, keyFlag, i, ret);
1220 6 : break;
1221 : }
1222 :
1223 5 : if (wrList[i].op == RA_WR_RDMA_WRITE_WITH_NOTIFY || wrList[i].op == RA_WR_RDMA_REDUCE_WRITE ||
1224 5 : wrList[i].op == RA_WR_RDMA_REDUCE_WRITE_WITH_NOTIFY) {
1225 1 : ibWr.imm_data = htobe32((wrList[i].aux.notifyOffset & WRITE_NOTIFY_OFFSET_MASK) |
1226 : WRITE_NOTIFY_VALUE_RECORD);
1227 1 : extAttr.reduce_op = wrList[i].aux.reduceType;
1228 1 : extAttr.reduce_type = wrList[i].aux.dataType;
1229 1 : ret = RsIbvExtPostSend(qpCb->ibQp, &ibWr, &badWr, &extAttr, &extRsp);
1230 1 : expRsp.wqe_index = extRsp.wqe_index;
1231 1 : expRsp.db_info = extRsp.db_info;
1232 1 : hccp_dbg("rs_ibv_ext_post_send, op = [%x], immData = [0x%lx], reduce_op = [%d],reduceType = [%d]",
1233 : ibWr.opcode, ibWr.imm_data, extAttr.reduce_op, extAttr.reduce_type);
1234 : } else {
1235 4 : ret = RsIbvExpPostSend(qpCb->ibQp, &ibWr, &badWr, &expRsp);
1236 4 : hccp_dbg("rs_ibv_exp_post_send, op = [%x], remoteAddr = [0x%llx], size = [%d]", ibWr.opcode,
1237 : ibWr.wr.rdma.remote_addr, ibWr.sg_list->length);
1238 : }
1239 :
1240 5 : if (ret != 0) {
1241 2 : if (ret == -ENOMEM) {
1242 1 : hccp_warn("qpn:%u rs_ibv_exp_post_send i:%u unsuccessful, ret %d", qpCb->ibQp->qp_num, i, ret);
1243 : } else {
1244 1 : hccp_err("qpn:%u rs_ibv_exp_post_send i:%u failed, ret %d", qpCb->ibQp->qp_num, i, ret);
1245 : }
1246 2 : break;
1247 : }
1248 :
1249 3 : qpCb->sendWrNum++;
1250 :
1251 3 : if (qpCb->qpMode == RA_RS_GDR_TMPL_QP_MODE) {
1252 1 : wrRsp[i].wqeTmp.sqIndex = (unsigned int)qpCb->sqIndex;
1253 1 : wrRsp[i].wqeTmp.wqeIndex = expRsp.wqe_index;
1254 2 : } else if (qpCb->qpMode == RA_RS_OP_QP_MODE || qpCb->qpMode == RA_RS_GDR_ASYN_QP_MODE) {
1255 2 : wrRsp[i].db.dbIndex = (unsigned int)qpCb->dbIndex;
1256 2 : wrRsp[i].db.dbInfo = expRsp.db_info;
1257 : }
1258 : }
1259 :
1260 11 : hccp_dbg("complete_num[%d], ret[%d]", i, ret);
1261 11 : *completeNum = i;
1262 11 : return (ret == -ENOMEM) ? 0 : ret;
1263 : }
1264 :
1265 15 : RS_ATTRI_VISI_DEF int RsSendWrlist(struct RsWrlistBaseInfo baseInfo, struct WrInfo *wrList, unsigned int sendNum,
1266 : struct SendWrRsp *wrRsp, unsigned int *completeNum)
1267 : {
1268 : int ret;
1269 : unsigned int phyId, rdevIndex, qpn;
1270 15 : struct RsQpCb *qpCb = NULL;
1271 :
1272 15 : RS_CHECK_POINTER_NULL_RETURN_INT(wrList);
1273 15 : RS_CHECK_POINTER_NULL_RETURN_INT(wrRsp);
1274 15 : CHK_PRT_RETURN(sendNum > MAX_WR_NUM || sendNum == 0 || baseInfo.phyId >= RS_MAX_DEV_NUM,
1275 : hccp_err("send_num[%u] or phyId:%u >= [%d], is invalid", sendNum, baseInfo.phyId, RS_MAX_DEV_NUM), -EINVAL);
1276 :
1277 14 : phyId = baseInfo.phyId;
1278 14 : rdevIndex = baseInfo.rdevIndex;
1279 14 : qpn = baseInfo.qpn;
1280 :
1281 14 : CHK_PRT_RETURN(RsQpn2qpcb(phyId, rdevIndex, qpn, &qpCb), hccp_err("rs_qpn2qpcb failed, physical id[%u]", phyId),
1282 : -EACCES);
1283 :
1284 : // only allow normal qp to call this func when ai_op_support not set
1285 13 : if (qpCb->qpMode == RA_RS_NOR_QP_MODE && qpCb->aiOpSupport == 0) {
1286 6 : ret = RsSendNormalWrlist(qpCb, wrList, sendNum, completeNum, baseInfo.keyFlag);
1287 : } else {
1288 7 : ret = RsSendExpWrlist(qpCb, wrList, sendNum, wrRsp, completeNum, baseInfo.keyFlag);
1289 : }
1290 13 : return ret;
1291 : }
1292 :
1293 0 : RS_ATTRI_VISI_DEF int RsRecvWrlist(struct RsWrlistBaseInfo baseInfo, struct RecvWrlistData *wr, unsigned int recvNum,
1294 : unsigned int *completeNum)
1295 : {
1296 0 : struct RsQpCb *qpCb = NULL;
1297 :
1298 0 : RS_CHECK_POINTER_NULL_RETURN_INT(wr);
1299 0 : CHK_PRT_RETURN(recvNum > MAX_WR_NUM || recvNum == 0 || baseInfo.phyId >= RS_MAX_DEV_NUM,
1300 : hccp_err("recv_num[%u] or phyId:%u >= [%d], is invalid", recvNum, baseInfo.phyId, RS_MAX_DEV_NUM), -EINVAL);
1301 :
1302 0 : CHK_PRT_RETURN(RsQpn2qpcb(baseInfo.phyId, baseInfo.rdevIndex, baseInfo.qpn, &qpCb),
1303 : hccp_err("rs_qpn2qpcb failed, physical id[%u]", baseInfo.phyId), -EACCES);
1304 :
1305 0 : return RsDrvPostRecv(qpCb, wr, recvNum, completeNum);
1306 : }
1307 :
1308 0 : RS_ATTRI_VISI_DEF int RsSetHostPid(uint32_t phyId, pid_t hostPid, const char *pidSign)
1309 : {
1310 : int ret;
1311 : unsigned int chipId;
1312 0 : struct rs_cb *rsCb = NULL;
1313 :
1314 0 : RS_CHECK_POINTER_NULL_RETURN_INT(pidSign);
1315 0 : CHK_PRT_RETURN(phyId >= RS_MAX_DEV_NUM, hccp_err("rs_set_host_pid rs set param error ! phyId:%u", phyId), -EINVAL);
1316 :
1317 0 : ret = rsGetLocalDevIDByHostDevID(phyId, &chipId);
1318 0 : CHK_PRT_RETURN(ret, hccp_err("rs_set_host_pid rsGetLocalDevIDByHostDevID phyId invalid, ret %d", ret), ret);
1319 :
1320 0 : hccp_info("phyId[%u] host_pid[%d]", chipId, hostPid);
1321 :
1322 0 : ret = RsDev2rscb(chipId, &rsCb, false);
1323 0 : CHK_PRT_RETURN(ret, hccp_err("get rs cb failed, chipId:%u", chipId), ret);
1324 :
1325 0 : rsCb->pRsSign.tgid = hostPid;
1326 0 : ret = strcpy_s(rsCb->pRsSign.sign, PROCESS_RS_SIGN_LENGTH, pidSign);
1327 0 : CHK_PRT_RETURN(ret, hccp_err("copy sign failed, ret %d", ret), -ESAFEFUNC);
1328 :
1329 0 : return 0;
1330 : }
1331 :
1332 6 : RS_ATTRI_VISI_DEF int RsRdevGetPortStatus(unsigned int phyId, unsigned int rdevIndex, enum PortStatus *status)
1333 : {
1334 6 : struct ibv_port_attr portAttr = {0};
1335 6 : struct RsRdevCb *rdevCb = NULL;
1336 : unsigned int chipId;
1337 : int ret;
1338 :
1339 6 : CHK_PRT_RETURN(phyId >= RS_MAX_DEV_NUM, hccp_err("phyId:%u >= [%d], is invalid", phyId, RS_MAX_DEV_NUM), -EINVAL);
1340 4 : CHK_PRT_RETURN(status == NULL, hccp_err("param err! status is NULL"), -EINVAL);
1341 :
1342 3 : ret = rsGetLocalDevIDByHostDevID(phyId, &chipId);
1343 3 : CHK_PRT_RETURN(ret, hccp_err("rsGetLocalDevIDByHostDevID failed, phyId[%u] invalid, ret %d", phyId, ret), ret);
1344 :
1345 3 : ret = RsRdev2rdevCb(chipId, rdevIndex, &rdevCb);
1346 3 : CHK_PRT_RETURN(ret != 0 || rdevCb == NULL, hccp_err("rs_rdev2rdev_cb for chip_id[%u] failed, ret %d", chipId, ret),
1347 : ret);
1348 :
1349 2 : ret = RsIbvQueryPort(rdevCb->ibCtx, rdevCb->ibPort, &portAttr);
1350 2 : CHK_PRT_RETURN(ret, hccp_err("ibv_query_port failed ret[%d]", ret), -EOPENSRC);
1351 :
1352 1 : *status = portAttr.state == IBV_PORT_ACTIVE ? PORT_STATUS_ACTIVE : PORT_STATUS_DOWN;
1353 :
1354 1 : hccp_dbg("phyId:%u port_attr.state:%u status:%u", phyId, portAttr.state, *status);
1355 1 : return 0;
1356 : }
1357 :
1358 3 : RS_ATTRI_VISI_DEF int RsGetNotifyMrInfo(unsigned int phyId, unsigned int rdevIndex, struct MrInfoT *info)
1359 : {
1360 3 : struct RsRdevCb *rdevCb = NULL;
1361 3 : struct rs_cb *rsCb = NULL;
1362 : unsigned int chipId;
1363 : int ret;
1364 :
1365 3 : CHK_PRT_RETURN(phyId >= RS_MAX_DEV_NUM, hccp_err("phyId:%u >= [%d], is invalid", phyId, RS_MAX_DEV_NUM), -EINVAL);
1366 :
1367 2 : CHK_PRT_RETURN(info == NULL, hccp_err("param err! info is NULL"), -EINVAL);
1368 :
1369 1 : ret = rsGetLocalDevIDByHostDevID(phyId, &chipId);
1370 1 : CHK_PRT_RETURN(ret, hccp_err("phyId[%u] invalid, ret:%d", phyId, ret), ret);
1371 :
1372 1 : ret = RsDev2rscb(chipId, &rsCb, false);
1373 1 : CHK_PRT_RETURN(ret, hccp_err("get rs_cb failed, ret:%d", ret), -ENODEV);
1374 :
1375 1 : ret = RsGetRdevCb(rsCb, rdevIndex, &rdevCb);
1376 1 : CHK_PRT_RETURN(ret, hccp_err("rs_get_rdev_cb failed!, ret:%d, rdevIndex:%u", ret, rdevIndex), ret);
1377 :
1378 1 : info->addr = (void *)(uintptr_t)rdevCb->notifyVaBase;
1379 1 : info->size = rdevCb->notifySize;
1380 1 : info->access = rdevCb->notifyAccess;
1381 1 : info->lkey = rdevCb->notifyMr->lkey;
1382 :
1383 1 : return 0;
1384 : }
1385 :
1386 4 : RS_ATTRI_VISI_DEF int RsNotifyCfgSet(unsigned int phyId, unsigned long long va, unsigned long long size)
1387 : {
1388 : int ret;
1389 : unsigned int chipId;
1390 4 : struct rs_cb *rsCb = NULL;
1391 :
1392 4 : RS_CHECK_POINTER_NULL_RETURN_INT((void *)(uintptr_t)va);
1393 :
1394 4 : CHK_PRT_RETURN(phyId >= RS_MAX_DEV_NUM ||
1395 : (size != MAX_NOTIFY_SIZE_CLOUD && size != NOTIFY_NUM_MAX_V2 && size != NOTIFY_NUM_MAX_V3),
1396 : hccp_err("rs_notify_cfg_set rs set param error ! phyId[%u] size[%llu]", phyId, size), -EINVAL);
1397 :
1398 3 : ret = rsGetLocalDevIDByHostDevID(phyId, &chipId);
1399 3 : CHK_PRT_RETURN(ret, hccp_err("rs_notify_cfg_set phyId invalid, ret %d, phyId:%u", ret, phyId), ret);
1400 :
1401 1 : ret = RsDev2rscb(chipId, &rsCb, false);
1402 1 : CHK_PRT_RETURN(ret, hccp_err("get rs cb failed, chipId:%u", chipId), ret);
1403 :
1404 1 : rsCb->notifyVaBase = va;
1405 1 : rsCb->notifySize = size;
1406 :
1407 1 : return 0;
1408 : }
1409 :
1410 0 : RS_ATTRI_VISI_DEF int RsNotifyCfgGet(unsigned int phyId, unsigned long long *va, unsigned long long *size)
1411 : {
1412 : int ret;
1413 : unsigned int chipId;
1414 0 : struct rs_cb *rsCb = NULL;
1415 :
1416 0 : RS_CHECK_POINTER_NULL_RETURN_INT(va);
1417 0 : RS_CHECK_POINTER_NULL_RETURN_INT(size);
1418 :
1419 0 : CHK_PRT_RETURN(phyId >= RS_MAX_DEV_NUM, hccp_err("rs_notify_cfg_get rs set param error ! phyId:%u", phyId),
1420 : -EINVAL);
1421 :
1422 0 : ret = rsGetLocalDevIDByHostDevID(phyId, &chipId);
1423 0 : CHK_PRT_RETURN(ret, hccp_err("rs_notify_cfg_get phyId invalid, ret %d, phyId:%u", ret, phyId), ret);
1424 :
1425 0 : ret = RsDev2rscb(chipId, &rsCb, false);
1426 0 : CHK_PRT_RETURN(ret, hccp_err("get rs cb failed, chipId:%u", chipId), ret);
1427 :
1428 0 : *va = rsCb->notifyVaBase;
1429 0 : *size = rsCb->notifySize;
1430 :
1431 0 : return 0;
1432 : }
1433 :
1434 6 : RS_ATTRI_VISI_DEF int RsSetTsqpDepth(unsigned int phyId, unsigned int rdevIndex, unsigned int tempDepth,
1435 : unsigned int *qpNum)
1436 : {
1437 : (void)phyId;
1438 : (void)rdevIndex;
1439 : (void)tempDepth;
1440 : (void)qpNum;
1441 : #ifdef CUSTOM_INTERFACE
1442 6 : struct RsRdevCb *rdevCb = NULL;
1443 6 : unsigned int sqDepth = 0;
1444 6 : unsigned int chipId = 0;
1445 : int ret;
1446 :
1447 6 : if (!RsIsCustomInterfaceSupported()) {
1448 0 : return 0;
1449 : }
1450 6 : CHK_PRT_RETURN(phyId >= RS_MAX_DEV_NUM, hccp_err("rs_set_tsqp_depth param error ! phyId:%d", phyId), -EINVAL);
1451 :
1452 5 : CHK_PRT_RETURN(qpNum == NULL, hccp_err("rs_set_tsqp_depth qp_num is NULL, param error!"), -EINVAL);
1453 :
1454 4 : CHK_PRT_RETURN(tempDepth < RS_MIN_TEMPTH_DEPTH || tempDepth > RS_MAX_TEMPTH_DEPTH,
1455 : hccp_err("param error!"
1456 : "temp_depth[%u] can not smaller than [%d] or bigerr than [%d]",
1457 : tempDepth, RS_MIN_TEMPTH_DEPTH, RS_MAX_TEMPTH_DEPTH),
1458 : -EINVAL);
1459 :
1460 3 : ret = rsGetLocalDevIDByHostDevID(phyId, &chipId);
1461 3 : CHK_PRT_RETURN(ret, hccp_err("phyId[%u] invalid, ret %d", phyId, ret), ret);
1462 :
1463 2 : ret = RsRdev2rdevCb(chipId, rdevIndex, &rdevCb);
1464 2 : CHK_PRT_RETURN(ret || rdevCb == NULL,
1465 : hccp_err("rs_set_tsqp_depth rs_rdev2rdev_cb for chip_id[%u]"
1466 : "failed, ret %d",
1467 : chipId, ret),
1468 : ret);
1469 :
1470 1 : ret = RsRoceSetTsqpDepth(rdevCb->devName, rdevIndex, tempDepth, qpNum, &sqDepth);
1471 1 : CHK_PRT_RETURN(ret, hccp_err("rs_roce_set_tsqp_depth failed, ret %d, devName[%s]", ret, rdevCb->devName), ret);
1472 :
1473 0 : rdevCb->txDepth = sqDepth;
1474 0 : rdevCb->rxDepth = sqDepth;
1475 0 : rdevCb->qpMaxNum = *qpNum;
1476 : #endif
1477 0 : return 0;
1478 : }
1479 :
1480 5 : RS_ATTRI_VISI_DEF int RsGetTsqpDepth(unsigned int phyId, unsigned int rdevIndex, unsigned int *tempDepth,
1481 : unsigned int *qpNum)
1482 : {
1483 : (void)phyId;
1484 : (void)rdevIndex;
1485 : (void)tempDepth;
1486 : (void)qpNum;
1487 : #ifdef CUSTOM_INTERFACE
1488 5 : struct RsRdevCb *rdevCb = NULL;
1489 5 : unsigned int sqDepth = 0;
1490 5 : unsigned int chipId = 0;
1491 : int ret;
1492 :
1493 5 : if (!RsIsCustomInterfaceSupported()) {
1494 0 : return 0;
1495 : }
1496 5 : CHK_PRT_RETURN(phyId >= RS_MAX_DEV_NUM, hccp_err("param error ! phyId:%d", phyId), -EINVAL);
1497 :
1498 4 : CHK_PRT_RETURN(tempDepth == NULL || qpNum == NULL,
1499 : hccp_err("temp_depth or qp_num is NULL,"
1500 : "param error!"),
1501 : -EINVAL);
1502 :
1503 3 : ret = rsGetLocalDevIDByHostDevID(phyId, &chipId);
1504 3 : CHK_PRT_RETURN(ret, hccp_err("phyId[%u] invalid, ret %d", phyId, ret), ret);
1505 :
1506 2 : ret = RsRdev2rdevCb(chipId, rdevIndex, &rdevCb);
1507 2 : CHK_PRT_RETURN(ret != 0 || rdevCb == NULL,
1508 : hccp_err("rs_get_tsqp_depth rs_rdev2rdev_cb for chip_id[%u]"
1509 : "failed, ret %d",
1510 : chipId, ret),
1511 : ret);
1512 :
1513 1 : ret = RsRoceGetTsqpDepth(rdevCb->devName, rdevIndex, tempDepth, qpNum, &sqDepth);
1514 1 : CHK_PRT_RETURN(ret, hccp_err("rs_roce_get_tsqp_depth failed, ret %d, devName[%s]", ret, rdevCb->devName), ret);
1515 : #endif
1516 0 : return 0;
1517 : }
1518 :
1519 45 : STATIC void RsSetQpDepthAttr(struct RsRdevCb *rdevCb, struct RsQpCb *qpCb, struct RsQpNorm *qpNorm)
1520 : {
1521 45 : if (qpCb->qpMode == RA_RS_GDR_TMPL_QP_MODE) {
1522 41 : qpCb->txDepth = rdevCb->txDepth;
1523 41 : qpCb->rxDepth = rdevCb->rxDepth;
1524 : } else {
1525 4 : if (rdevCb->rsCb->hccpMode == NETWORK_OFFLINE) {
1526 2 : qpCb->txDepth = RS_QP_TX_DEPTH_OFFLINE;
1527 2 : qpCb->rxDepth = RS_QP_RX_DEPTH_OFFLINE;
1528 : } else {
1529 2 : qpCb->txDepth = RS_QP_TX_DEPTH_ONLINE;
1530 2 : qpCb->rxDepth = RS_QP_RX_DEPTH_ONLINE;
1531 : }
1532 : }
1533 :
1534 45 : if (qpNorm->isExp != 0 && qpNorm->qpMode != RA_RS_NOR_QP_MODE) {
1535 43 : if (rdevCb->rsCb->hccpMode == NETWORK_PEER_ONLINE) {
1536 2 : qpCb->txDepth = (qpCb->qpMode != RA_RS_GDR_TMPL_QP_MODE) ? RS_QP_TX_DEPTH_PEER_ONLINE : qpCb->txDepth;
1537 2 : qpCb->rxDepth = (qpCb->qpMode != RA_RS_GDR_TMPL_QP_MODE) ? RS_QP_TX_DEPTH_PEER_ONLINE : qpCb->rxDepth;
1538 : } else {
1539 41 : qpCb->txDepth = (qpCb->qpMode != RA_RS_GDR_TMPL_QP_MODE && qpCb->qpMode != RA_RS_GDR_ASYN_QP_MODE)
1540 : ? RS_QP_32K_DEPTH
1541 41 : : qpCb->txDepth;
1542 : }
1543 43 : qpCb->sendSgeNum = 1;
1544 43 : qpCb->recvSgeNum = 1;
1545 : } else {
1546 2 : if (rdevCb->rsCb->hccpMode == NETWORK_PEER_ONLINE) {
1547 0 : qpCb->txDepth = (qpCb->qpMode != RA_RS_GDR_TMPL_QP_MODE) ? RS_QP_TX_DEPTH_PEER_ONLINE : qpCb->txDepth;
1548 0 : qpCb->rxDepth = (qpCb->qpMode != RA_RS_GDR_TMPL_QP_MODE) ? RS_QP_TX_DEPTH_PEER_ONLINE : qpCb->rxDepth;
1549 : } else {
1550 2 : qpCb->txDepth = (qpCb->qpMode != RA_RS_GDR_TMPL_QP_MODE) ? RS_QP_TX_DEPTH : qpCb->txDepth;
1551 2 : qpCb->rxDepth = (qpCb->qpMode != RA_RS_GDR_TMPL_QP_MODE) ? RS_QP_TX_DEPTH : qpCb->rxDepth;
1552 : }
1553 2 : qpCb->sendSgeNum = RS_QP_ATTR_MAX_SEND_SGE;
1554 2 : qpCb->recvSgeNum = 1;
1555 : }
1556 45 : }
1557 :
1558 45 : STATIC int RsQpcbInit(struct RsRdevCb *rdevCb, struct RsQpCb *qpCb, struct RsQpNorm *qpNorm)
1559 : {
1560 : #define RS_DRV_CQ_DEPTH 16384
1561 : #define RS_DRV_CQ_128_DEPTH 128
1562 : #define RS_DRV_CQ_8K_DEPTH 8192
1563 : #define RS_DRV_CQ_32K_DEPTH 32768
1564 45 : int qpMode = qpNorm->qpMode;
1565 : int ret;
1566 :
1567 45 : qpCb->rdevCb = rdevCb;
1568 45 : RS_INIT_LIST_HEAD(&qpCb->mrList);
1569 45 : RS_INIT_LIST_HEAD(&qpCb->remMrList);
1570 :
1571 45 : qpCb->qpMode = qpMode;
1572 45 : qpCb->eqNum = 0;
1573 45 : qpCb->numRecvCqEvents = 0;
1574 45 : qpCb->numSendCqEvents = 0;
1575 45 : qpCb->state = RS_QP_STATUS_DISCONNECT;
1576 45 : qpCb->ibPd = rdevCb->ibPd;
1577 :
1578 : // cq attr
1579 45 : if (qpNorm->isExt == 1) {
1580 : // update TEMP & ASYN mode cq depth from 32K to 8K due to memory issue
1581 2 : qpCb->sendCqDepth = (qpMode != RA_RS_GDR_TMPL_QP_MODE && qpMode != RA_RS_GDR_ASYN_QP_MODE) ? RS_DRV_CQ_32K_DEPTH
1582 4 : : RS_DRV_CQ_8K_DEPTH;
1583 2 : qpCb->recvCqDepth = RS_DRV_CQ_128_DEPTH;
1584 : } else {
1585 43 : qpCb->sendCqDepth = RS_DRV_CQ_DEPTH;
1586 43 : qpCb->recvCqDepth = RS_DRV_CQ_DEPTH;
1587 : }
1588 :
1589 : // qp attr
1590 45 : RsSetQpDepthAttr(rdevCb, qpCb, qpNorm);
1591 :
1592 45 : qpCb->memAlign = qpNorm->memAlign;
1593 :
1594 45 : qpCb->channel = RsIbvCreateCompChannel(rdevCb->ibCtx);
1595 45 : CHK_PRT_RETURN(qpCb->channel == NULL, hccp_err("ibv_create_comp_channel failed! errno(%d)", errno), -EINVAL);
1596 44 : qpCb->qosAttr.tc = (RS_ROCE_DSCP_33 & RS_DSCP_MASK) << RS_DSCP_OFF;
1597 44 : qpCb->qosAttr.sl = RS_ROCE_4_SL;
1598 44 : qpCb->timeout = RS_QP_ATTR_TIMEOUT;
1599 44 : qpCb->retryCnt = RS_QP_ATTR_RETRY_CNT;
1600 :
1601 44 : ret = RsEpollCtl(rdevCb->rsCb->connCb.epollfd, EPOLL_CTL_ADD, qpCb->channel->fd, EPOLLIN | EPOLLRDHUP);
1602 : #ifndef CA_CONFIG_LLT
1603 : if (ret) {
1604 : RsIbvDestroyCompChannel(qpCb->channel);
1605 : hccp_err("add channel fd failed ret %d", ret);
1606 : return ret;
1607 : }
1608 : #endif
1609 44 : return 0;
1610 : }
1611 :
1612 48 : STATIC int RsQpcbDeinit(struct RsRdevCb *rdevCb, struct RsQpCb *qpCb)
1613 : {
1614 : int ret;
1615 :
1616 48 : if (qpCb == NULL || qpCb->channel == NULL) {
1617 0 : hccp_err("qp_cb or qp_cb->channel is NULL!");
1618 0 : return -EINVAL;
1619 : }
1620 :
1621 48 : ret = RsEpollCtl(rdevCb->rsCb->connCb.epollfd, EPOLL_CTL_DEL, qpCb->channel->fd, EPOLLIN | EPOLLRDHUP);
1622 : #ifndef CA_CONFIG_LLT
1623 : if (ret) {
1624 : hccp_err("del channel fd failed ret %d", ret);
1625 : }
1626 : #endif
1627 :
1628 48 : if (qpCb->channel != NULL) {
1629 48 : RsIbvDestroyCompChannel(qpCb->channel);
1630 48 : qpCb->channel = NULL;
1631 : }
1632 : #ifndef CA_CONFIG_LLT
1633 : return ret;
1634 : #else
1635 48 : return 0;
1636 : #endif
1637 : }
1638 :
1639 32 : STATIC int RsQpNotifyMr(struct RsRdevCb *rdevCb, struct RsQpCb *qpCb, uint32_t *qpn)
1640 : {
1641 : int ret;
1642 32 : struct RsMrCb *notifyMrNode = NULL;
1643 :
1644 32 : ret = RsCallocMr(1, ¬ifyMrNode);
1645 32 : CHK_PRT_RETURN(ret, hccp_err("notify_mr_cb malloc failed"), ret);
1646 :
1647 32 : RS_PTHREAD_MUTEX_LOCK(&rdevCb->rdevMutex);
1648 32 : RsListAddTail(&qpCb->list, &rdevCb->qpList);
1649 32 : RS_PTHREAD_MUTEX_ULOCK(&rdevCb->rdevMutex);
1650 :
1651 32 : if (rdevCb->notifyType != NO_USE) {
1652 32 : notifyMrNode->qpCb = qpCb;
1653 32 : notifyMrNode->ibMr = rdevCb->notifyMr;
1654 32 : notifyMrNode->mrInfo.addr = rdevCb->notifyVaBase;
1655 32 : notifyMrNode->mrInfo.len = rdevCb->notifySize;
1656 32 : notifyMrNode->mrInfo.rkey = notifyMrNode->ibMr->rkey;
1657 : } else {
1658 0 : notifyMrNode->qpCb = qpCb;
1659 0 : notifyMrNode->ibMr = NULL;
1660 : }
1661 :
1662 32 : RS_PTHREAD_MUTEX_LOCK(&qpCb->qpMutex);
1663 32 : RsListAddTail(¬ifyMrNode->list, &qpCb->mrList);
1664 32 : RS_PTHREAD_MUTEX_ULOCK(&qpCb->qpMutex);
1665 32 : rdevCb->qpCnt++;
1666 32 : *qpn = qpCb->ibQp->qp_num;
1667 :
1668 32 : hccp_info("rs qp %d create OK!", *qpn);
1669 :
1670 32 : return 0;
1671 : }
1672 :
1673 53 : STATIC int RsQpQueryInfo(unsigned int phyId, unsigned int rdevIndex, struct RsRdevCb **rdevCb, int qpMode)
1674 : {
1675 : int ret;
1676 : unsigned int chipId;
1677 53 : struct rs_cb *rsCb = NULL;
1678 :
1679 53 : CHK_PRT_RETURN(phyId >= RS_MAX_DEV_NUM, hccp_err("rs_qp_query_info rs set param error! phyId:%u", phyId), -EINVAL);
1680 :
1681 53 : ret = rsGetLocalDevIDByHostDevID(phyId, &chipId);
1682 53 : CHK_PRT_RETURN(ret, hccp_err("rs_qp_query_info phyId[%u] invalid, ret:%d", phyId, ret), ret);
1683 :
1684 52 : ret = RsDev2rscb(chipId, &rsCb, false);
1685 52 : CHK_PRT_RETURN(ret, hccp_err("rs_qp_query_info get rs_cb failed, ret:%d", ret), -ENODEV);
1686 :
1687 52 : ret = RsGetRdevCb(rsCb, rdevIndex, rdevCb);
1688 52 : CHK_PRT_RETURN(ret, hccp_err("rs_get_rdev_cb failed! ret:%d, rdevIndex:%u", ret, rdevIndex), ret);
1689 :
1690 52 : if (qpMode == RA_RS_GDR_TMPL_QP_MODE) {
1691 46 : CHK_PRT_RETURN((*rdevCb)->qpCnt >= (*rdevCb)->qpMaxNum,
1692 : hccp_err("Exceeded the maximum QP limit(%u)", (*rdevCb)->qpMaxNum), -EINVAL);
1693 : } else {
1694 6 : CHK_PRT_RETURN((*rdevCb)->qpCnt >= RS_QP_NUM_MAX,
1695 : hccp_err("Exceeded the maximum QP limit(%u)", (*rdevCb)->qpCnt), -EINVAL);
1696 : }
1697 :
1698 50 : return 0;
1699 : }
1700 :
1701 50 : STATIC int RsInitMemPool(struct RsQpCb *qpCb)
1702 : {
1703 50 : struct roce_mem_cq_qp_attr memAttr = {0};
1704 : int ret;
1705 :
1706 50 : if ((qpCb->qpMode != RA_RS_OP_QP_MODE && qpCb->qpMode != RA_RS_OP_QP_MODE_EXT) ||
1707 4 : qpCb->memAlign != LITE_ALIGN_2MB) {
1708 49 : return 0;
1709 : }
1710 :
1711 : // init mem_pool and store mem_data in mem_resp
1712 1 : memAttr.mem_align = qpCb->memAlign;
1713 1 : memAttr.send_qp_depth = qpCb->txDepth;
1714 1 : memAttr.send_cq_depth = (unsigned int)qpCb->sendCqDepth;
1715 1 : memAttr.send_sge_num = qpCb->sendSgeNum;
1716 1 : memAttr.recv_qp_depth = qpCb->rxDepth;
1717 1 : memAttr.recv_cq_depth = (unsigned int)qpCb->recvCqDepth;
1718 1 : memAttr.recv_sge_num = qpCb->recvSgeNum;
1719 1 : memAttr.use_resv_mem = qpCb->useResvMem;
1720 1 : memAttr.resv_mem_pool_id = qpCb->resvMemPoolId;
1721 1 : memAttr.ctx = qpCb->rdevCb->ibCtx;
1722 :
1723 1 : ret = RsRoceInitMemPool(&memAttr, &qpCb->memResp.memData, qpCb->rdevCb->rsCb->chipId);
1724 1 : if (ret != 0) {
1725 1 : hccp_err("rs_roce_init_mem_pool failed, ret=%d, chipId=%u", ret, qpCb->rdevCb->rsCb->chipId);
1726 : }
1727 1 : return ret;
1728 : }
1729 :
1730 50 : STATIC void RsDeinitMemPool(struct RsQpCb *qpCb)
1731 : {
1732 50 : if ((qpCb->qpMode != RA_RS_OP_QP_MODE && qpCb->qpMode != RA_RS_OP_QP_MODE_EXT) ||
1733 4 : qpCb->memAlign != LITE_ALIGN_2MB) {
1734 49 : return;
1735 : }
1736 :
1737 1 : (void)RsRoceDeinitMemPool(qpCb->memResp.memData.mem_idx);
1738 : }
1739 :
1740 46 : STATIC int RsAllocQpcb(struct RsRdevCb *rdevCb, struct RsQpCb **qpCb, struct RsQpNorm *qpNorm)
1741 : {
1742 : int ret;
1743 :
1744 46 : ret = RsCallocQpcb(1, qpCb);
1745 46 : CHK_PRT_RETURN(ret, hccp_err("alloc mem for qp_cb failed, ret:%d errno:%d", ret, errno), -ENOMEM);
1746 :
1747 45 : ret = pthread_mutex_init(&(*qpCb)->qpMutex, NULL);
1748 45 : if (ret) {
1749 0 : hccp_err("pthread_mutex_init failed, ret %d", ret);
1750 0 : goto qp_mutex_init_err;
1751 : }
1752 :
1753 45 : ret = pthread_mutex_init(&(*qpCb)->cqeErrInfo.mutex, NULL);
1754 45 : if (ret) {
1755 0 : hccp_err("pthread_mutex_init failed, ret %d", ret);
1756 0 : goto cqe_mutex_init_err;
1757 : }
1758 :
1759 45 : ret = RsQpcbInit(rdevCb, *qpCb, qpNorm);
1760 45 : if (ret) {
1761 1 : hccp_err("create qp tx rx failed ret %d", ret);
1762 1 : goto rs_qpcb_init_err;
1763 : }
1764 :
1765 44 : ret = RsInitMemPool(*qpCb);
1766 44 : if (ret) {
1767 0 : hccp_err("init mem pool failed ret %d", ret);
1768 0 : goto rs_init_mem_err;
1769 : }
1770 :
1771 44 : ret = RsDrvCreateCq(*qpCb, qpNorm->isExt);
1772 44 : if (ret) {
1773 1 : hccp_err("create cq failed ret %d", ret);
1774 1 : goto create_cq_err;
1775 : }
1776 :
1777 43 : return 0;
1778 :
1779 1 : create_cq_err:
1780 1 : RsDeinitMemPool(*qpCb);
1781 :
1782 1 : rs_init_mem_err:
1783 1 : RsQpcbDeinit(rdevCb, *qpCb);
1784 :
1785 2 : rs_qpcb_init_err:
1786 2 : pthread_mutex_destroy(&(*qpCb)->cqeErrInfo.mutex);
1787 :
1788 2 : cqe_mutex_init_err:
1789 2 : pthread_mutex_destroy(&(*qpCb)->qpMutex);
1790 :
1791 2 : qp_mutex_init_err:
1792 2 : free(*qpCb);
1793 2 : *qpCb = NULL;
1794 :
1795 2 : return ret;
1796 : }
1797 :
1798 15 : STATIC void RsFreeQpcb(struct RsRdevCb *rdevCb, struct RsQpCb *qpCb)
1799 : {
1800 15 : RsDrvDestroyCq(qpCb);
1801 15 : RsDeinitMemPool(qpCb);
1802 15 : (void)RsQpcbDeinit(rdevCb, qpCb);
1803 15 : pthread_mutex_destroy(&qpCb->cqeErrInfo.mutex);
1804 15 : pthread_mutex_destroy(&qpCb->qpMutex);
1805 15 : free(qpCb);
1806 15 : qpCb = NULL;
1807 15 : }
1808 :
1809 48 : RS_ATTRI_VISI_DEF int RsQpCreate(unsigned int phyId, unsigned int rdevIndex, struct RsQpNorm qpNorm,
1810 : struct RsQpResp *qpResp)
1811 : {
1812 48 : struct RsRdevCb *rdevCb = NULL;
1813 48 : struct RsQpCb *qpCb = NULL;
1814 : int ret;
1815 :
1816 48 : RS_QP_PARA_CHECK(phyId);
1817 47 : CHK_PRT_RETURN(qpResp == NULL, hccp_err("qp_resp is NULL!"), -EINVAL);
1818 :
1819 47 : ret = RsQpQueryInfo(phyId, rdevIndex, &rdevCb, qpNorm.qpMode);
1820 47 : CHK_PRT_RETURN(ret, hccp_err("query qp info failed:%d", ret), ret);
1821 :
1822 46 : ret = RsAllocQpcb(rdevCb, &qpCb, &qpNorm);
1823 46 : CHK_PRT_RETURN(ret, hccp_err("alloc mem for qp_cb failed, ret:%d", ret), ret);
1824 :
1825 43 : ret = RsDrvQpCreate(qpCb, &qpNorm);
1826 43 : if (ret) {
1827 14 : hccp_err("create drv qp create failed:%d", ret);
1828 14 : goto create_qp_err;
1829 : }
1830 :
1831 29 : ret = ibv_req_notify_cq(qpCb->ibSendCq, 0);
1832 29 : if (ret) {
1833 1 : hccp_err("Couldn't request send CQ notification, ret:%d", ret);
1834 1 : ret = -EOPENSRC;
1835 1 : goto ret_noritfy_cq;
1836 : }
1837 :
1838 28 : ret = ibv_req_notify_cq(qpCb->ibRecvCq, 0);
1839 28 : if (ret) {
1840 0 : hccp_err("Couldn't request recv CQ notification, ret:%d", ret);
1841 0 : ret = -EOPENSRC;
1842 0 : goto ret_noritfy_cq;
1843 : }
1844 :
1845 28 : ret = RsQpNotifyMr(rdevCb, qpCb, &qpResp->qpn); // alloc mr
1846 28 : if (ret) {
1847 0 : hccp_err("store qp notify mr failed:%d", ret);
1848 0 : goto ret_noritfy_cq;
1849 : }
1850 :
1851 28 : if (qpNorm.isExp) {
1852 26 : qpCb->isExp = RS_IS_EXP;
1853 : } else {
1854 2 : qpCb->isExp = RS_NOT_EXP;
1855 : }
1856 :
1857 28 : qpResp->qpn = (unsigned int)qpCb->qpInfoLo.qpn;
1858 28 : qpResp->gidIdx = (unsigned int)qpCb->qpInfoLo.gidIdx;
1859 28 : qpResp->psn = (unsigned int)qpCb->qpInfoLo.psn;
1860 28 : qpResp->gid = qpCb->qpInfoLo.gid;
1861 :
1862 28 : return 0;
1863 :
1864 1 : ret_noritfy_cq:
1865 1 : RsDrvQpDestroy(qpCb);
1866 :
1867 15 : create_qp_err:
1868 15 : RsFreeQpcb(rdevCb, qpCb);
1869 15 : return ret;
1870 : }
1871 :
1872 4 : STATIC int RsQpcbInitWithAttrs(struct RsRdevCb *rdevCb, struct RsQpCb *qpCb, struct RsQpNormWithAttrs *qpNorm)
1873 : {
1874 : int ret;
1875 :
1876 4 : qpCb->rdevCb = rdevCb;
1877 4 : RS_INIT_LIST_HEAD(&qpCb->mrList);
1878 4 : RS_INIT_LIST_HEAD(&qpCb->remMrList);
1879 :
1880 4 : qpCb->qpMode = qpNorm->extAttrs.qpMode;
1881 4 : qpCb->numRecvCqEvents = 0;
1882 4 : qpCb->numSendCqEvents = 0;
1883 4 : qpCb->state = RS_QP_STATUS_DISCONNECT;
1884 4 : qpCb->ibPd = rdevCb->ibPd;
1885 :
1886 4 : qpCb->txDepth = qpNorm->extAttrs.qpAttr.cap.max_send_wr;
1887 4 : qpCb->rxDepth = qpNorm->extAttrs.qpAttr.cap.max_send_wr;
1888 4 : qpCb->sendSgeNum = qpNorm->extAttrs.qpAttr.cap.max_send_sge;
1889 4 : qpCb->recvSgeNum = qpNorm->extAttrs.qpAttr.cap.max_recv_sge;
1890 4 : qpCb->sendCqDepth = qpNorm->extAttrs.cqAttr.sendCqDepth;
1891 4 : qpCb->recvCqDepth = qpNorm->extAttrs.cqAttr.recvCqDepth;
1892 4 : qpCb->memAlign = qpNorm->extAttrs.memAlign;
1893 :
1894 4 : qpCb->channel = RsIbvCreateCompChannel(rdevCb->ibCtx);
1895 4 : CHK_PRT_RETURN(qpCb->channel == NULL, hccp_err("ibv_create_comp_channel failed! errno(%d)", errno), -EINVAL);
1896 4 : qpCb->qosAttr.tc = (RS_ROCE_DSCP_33 & RS_DSCP_MASK) << RS_DSCP_OFF;
1897 4 : qpCb->qosAttr.sl = RS_ROCE_4_SL;
1898 4 : qpCb->timeout = RS_QP_ATTR_TIMEOUT;
1899 4 : qpCb->retryCnt = RS_QP_ATTR_RETRY_CNT;
1900 :
1901 4 : qpCb->udpSport = qpNorm->extAttrs.udpSport;
1902 :
1903 4 : qpCb->aiOpSupport = qpNorm->aiOpSupport;
1904 4 : qpCb->grpId = rdevCb->rsCb->grpId;
1905 4 : qpCb->cqCstmFlag = qpNorm->extAttrs.dataPlaneFlag.bs.cqCstm;
1906 4 : qpCb->useResvMem = qpNorm->extAttrs.cstmFlag.bs.useResvMem;
1907 4 : qpCb->resvMemPoolId = qpNorm->extAttrs.resvMemPoolId;
1908 :
1909 4 : ret = RsEpollCtl(rdevCb->rsCb->connCb.epollfd, EPOLL_CTL_ADD, qpCb->channel->fd, EPOLLIN | EPOLLRDHUP);
1910 : #ifndef CA_CONFIG_LLT
1911 : if (ret) {
1912 : RsIbvDestroyCompChannel(qpCb->channel);
1913 : hccp_err("add channel fd failed ret %d", ret);
1914 : return ret;
1915 : }
1916 : #endif
1917 4 : return 0;
1918 : }
1919 :
1920 4 : STATIC int RsAllocQpcbWithAttrs(struct RsRdevCb *rdevCb, struct RsQpCb **qpCb, struct RsQpNormWithAttrs *qpNorm)
1921 : {
1922 : int ret;
1923 :
1924 4 : ret = RsCallocQpcb(1, qpCb);
1925 4 : CHK_PRT_RETURN(ret, hccp_err("alloc mem for qp_cb failed, ret:%d errno:%d", ret, errno), -ENOMEM);
1926 :
1927 4 : ret = pthread_mutex_init(&(*qpCb)->qpMutex, NULL);
1928 4 : if (ret) {
1929 0 : hccp_err("pthread_mutex_init failed, ret %d", ret);
1930 0 : goto qp_mutex_init_err;
1931 : }
1932 :
1933 4 : ret = pthread_mutex_init(&(*qpCb)->cqeErrInfo.mutex, NULL);
1934 4 : if (ret) {
1935 0 : hccp_err("pthread_mutex_init failed, ret %d", ret);
1936 0 : goto cqe_mutex_init_err;
1937 : }
1938 :
1939 4 : ret = RsQpcbInitWithAttrs(rdevCb, *qpCb, qpNorm);
1940 4 : if (ret) {
1941 0 : hccp_err("create qp tx rx failed ret %d", ret);
1942 0 : goto rs_qpcb_init_err;
1943 : }
1944 :
1945 4 : ret = RsInitMemPool(*qpCb);
1946 4 : if (ret) {
1947 0 : hccp_err("init mem pool failed ret %d", ret);
1948 0 : goto rs_init_mem_err;
1949 : }
1950 :
1951 4 : ret = RsDrvCreateCqWithAttrs(*qpCb, qpNorm->isExt, &qpNorm->extAttrs.cqAttr);
1952 4 : if (ret) {
1953 0 : hccp_err("create cq failed ret %d", ret);
1954 0 : goto create_cq_err;
1955 : }
1956 :
1957 4 : return 0;
1958 :
1959 0 : create_cq_err:
1960 0 : RsDeinitMemPool(*qpCb);
1961 :
1962 0 : rs_init_mem_err:
1963 0 : RsQpcbDeinit(rdevCb, *qpCb);
1964 :
1965 0 : rs_qpcb_init_err:
1966 0 : pthread_mutex_destroy(&(*qpCb)->cqeErrInfo.mutex);
1967 :
1968 0 : cqe_mutex_init_err:
1969 0 : pthread_mutex_destroy(&(*qpCb)->qpMutex);
1970 :
1971 0 : qp_mutex_init_err:
1972 0 : free(*qpCb);
1973 0 : *qpCb = NULL;
1974 :
1975 0 : return ret;
1976 : }
1977 :
1978 9 : STATIC int RsQpCheckQpNorm(struct RsQpNormWithAttrs *qpNorm, int *qpMode)
1979 : {
1980 9 : CHK_PRT_RETURN(qpNorm == NULL, hccp_err("qp_norm is NULL!"), -EINVAL);
1981 8 : CHK_PRT_RETURN(qpNorm->extAttrs.version != QP_CREATE_WITH_ATTR_VERSION,
1982 : hccp_err("attr version[%d] mismatch, expect [%d]", qpNorm->extAttrs.version, QP_CREATE_WITH_ATTR_VERSION),
1983 : -EINVAL);
1984 :
1985 7 : *qpMode = qpNorm->extAttrs.qpMode;
1986 7 : if (*qpMode < 0 || *qpMode >= RA_RS_ERR_QP_MODE) {
1987 1 : hccp_err("qp_mode[%d] must greater or equal to 0 and less than %d", *qpMode, RA_RS_ERR_QP_MODE);
1988 1 : return -EINVAL;
1989 : }
1990 :
1991 6 : if (*qpMode == RA_RS_OP_QP_MODE_EXT) {
1992 1 : *qpMode = RA_RS_OP_QP_MODE;
1993 : }
1994 :
1995 6 : qpNorm->extAttrs.qpMode = *qpMode;
1996 6 : return 0;
1997 : }
1998 :
1999 : #ifdef CUSTOM_INTERFACE
2000 8 : STATIC void RsQpPrepareCqDataPlaneInfo(struct ibv_cq *ibCq, struct AiDataPlaneCq *dataPlaneCq)
2001 : {
2002 8 : struct hns_roce_cq_data_plane_info cqInfo = {0};
2003 :
2004 8 : (void)RsRoceGetCqDataPlaneInfo(ibCq, &cqInfo);
2005 8 : dataPlaneCq->cqn = cqInfo.cqn;
2006 8 : dataPlaneCq->bufAddr = cqInfo.buf_addr;
2007 8 : dataPlaneCq->cqeSize = cqInfo.cqe_size;
2008 8 : dataPlaneCq->depth = cqInfo.depth;
2009 8 : dataPlaneCq->headAddr = cqInfo.head_addr;
2010 8 : dataPlaneCq->tailAddr = cqInfo.tail_addr;
2011 8 : dataPlaneCq->swdbAddr = cqInfo.swdb_addr;
2012 8 : dataPlaneCq->dbReg = cqInfo.db_reg;
2013 8 : hccp_info("cqn:%u buf_addr:0x%llx cqe_size:%u depth:%u head_addr:0x%llx tail_addr:0x%llx swdb_addr:0x%llx",
2014 : dataPlaneCq->cqn, dataPlaneCq->bufAddr, dataPlaneCq->cqeSize, dataPlaneCq->depth, dataPlaneCq->headAddr,
2015 : dataPlaneCq->tailAddr, dataPlaneCq->swdbAddr);
2016 8 : }
2017 :
2018 8 : STATIC void RsQpPrepareWqDataPlaneInfo(struct hns_roce_wq_data_plane_info *wqInfo, struct AiDataPlaneWq *dataPlaneWq)
2019 : {
2020 8 : dataPlaneWq->wqn = wqInfo->wqn;
2021 8 : dataPlaneWq->bufAddr = wqInfo->buf_addr;
2022 8 : dataPlaneWq->wqebbSize = wqInfo->wqebb_size;
2023 8 : dataPlaneWq->depth = wqInfo->depth;
2024 8 : dataPlaneWq->headAddr = wqInfo->head_addr;
2025 8 : dataPlaneWq->tailAddr = wqInfo->tail_addr;
2026 8 : dataPlaneWq->swdbAddr = wqInfo->swdb_addr;
2027 8 : dataPlaneWq->dbReg = wqInfo->db_reg;
2028 8 : hccp_info("wqn:%u buf_addr:0x%llx wqebb_size:%u depth:%u head_addr:%u tail_addr:%u swdb_addr:0x%llx",
2029 : dataPlaneWq->wqn, dataPlaneWq->bufAddr, dataPlaneWq->wqebbSize, dataPlaneWq->depth, dataPlaneWq->headAddr,
2030 : dataPlaneWq->tailAddr, dataPlaneWq->swdbAddr);
2031 8 : }
2032 :
2033 4 : STATIC void RsQpPrepareQpDataPlaneInfo(struct ibv_qp *ibQp, struct AiDataPlaneWq *dataPlaneSq,
2034 : struct AiDataPlaneWq *dataPlaneRq)
2035 : {
2036 4 : struct hns_roce_qp_data_plane_info qpInfo = {0};
2037 :
2038 4 : (void)RsRoceGetQpDataPlaneInfo(ibQp, &qpInfo);
2039 4 : RsQpPrepareWqDataPlaneInfo(&qpInfo.sq, dataPlaneSq);
2040 4 : RsQpPrepareWqDataPlaneInfo(&qpInfo.rq, dataPlaneRq);
2041 4 : }
2042 :
2043 4 : STATIC void RsQpPrepareDataPlaneInfo(struct RsQpNormWithAttrs *qpNorm, struct RsQpCb *qpCb,
2044 : struct RsQpRespWithAttrs *qpResp)
2045 : {
2046 : // skip to prepare cq data plane info
2047 4 : if (qpNorm->extAttrs.dataPlaneFlag.bs.cqCstm != 0) {
2048 4 : qpResp->aiScqAddr = (unsigned long long)(uintptr_t)qpCb->ibSendCq;
2049 4 : qpResp->aiRcqAddr = (unsigned long long)(uintptr_t)qpCb->ibRecvCq;
2050 4 : RsQpPrepareCqDataPlaneInfo(qpCb->ibSendCq, &qpResp->dataPlaneInfo.scq);
2051 4 : RsQpPrepareCqDataPlaneInfo(qpCb->ibRecvCq, &qpResp->dataPlaneInfo.rcq);
2052 : }
2053 :
2054 : // skip to prepare qp data plane info
2055 4 : if (qpNorm->aiOpSupport != 0) {
2056 4 : RsQpPrepareQpDataPlaneInfo(qpCb->ibQp, &qpResp->dataPlaneInfo.sq, &qpResp->dataPlaneInfo.rq);
2057 : }
2058 4 : }
2059 : #endif
2060 :
2061 4 : STATIC void RsQpPrepareQpResp(struct RsQpNormWithAttrs *qpNorm, struct RsQpCb *qpCb, struct RsQpRespWithAttrs *qpResp)
2062 : {
2063 4 : if (qpNorm->isExp != 0) {
2064 4 : qpCb->isExp = RS_IS_EXP;
2065 : } else {
2066 0 : qpCb->isExp = RS_NOT_EXP;
2067 : }
2068 :
2069 4 : qpResp->aiQpAddr = (unsigned long long)(uintptr_t)qpCb->ibQp;
2070 4 : qpResp->sqIndex = (unsigned int)qpCb->sqIndex;
2071 4 : qpResp->dbIndex = (unsigned int)qpCb->dbIndex;
2072 4 : qpResp->gidIdx = (unsigned int)qpCb->qpInfoLo.gidIdx;
2073 4 : qpResp->psn = (unsigned int)qpCb->qpInfoLo.psn;
2074 :
2075 : #ifdef CUSTOM_INTERFACE
2076 4 : if (RsIsCustomInterfaceSupported()) {
2077 4 : RsQpPrepareDataPlaneInfo(qpNorm, qpCb, qpResp);
2078 : }
2079 : #endif
2080 :
2081 4 : return;
2082 : }
2083 :
2084 12 : RS_ATTRI_VISI_DEF int RsQpCreateWithAttrs(unsigned int phyId, unsigned int rdevIndex, struct RsQpNormWithAttrs *qpNorm,
2085 : struct RsQpRespWithAttrs *qpResp)
2086 : {
2087 12 : struct RsRdevCb *rdevCb = NULL;
2088 12 : struct RsQpCb *qpCb = NULL;
2089 : int qpMode;
2090 : int ret;
2091 :
2092 12 : RS_QP_PARA_CHECK(phyId);
2093 10 : CHK_PRT_RETURN(qpResp == NULL, hccp_err("qp_resp is NULL!"), -EINVAL);
2094 :
2095 9 : ret = RsQpCheckQpNorm(qpNorm, &qpMode);
2096 9 : CHK_PRT_RETURN(ret != 0, hccp_err("check qp mode failed, ret:%d", ret), ret);
2097 :
2098 6 : ret = RsQpQueryInfo(phyId, rdevIndex, &rdevCb, qpMode);
2099 6 : CHK_PRT_RETURN(ret, hccp_err("query qp info failed:%d", ret), ret);
2100 :
2101 4 : ret = RsAllocQpcbWithAttrs(rdevCb, &qpCb, qpNorm);
2102 4 : CHK_PRT_RETURN(ret, hccp_err("alloc mem for qp_cb failed, ret:%d", ret), ret);
2103 :
2104 4 : ret = RsDrvQpCreateWithAttrs(qpCb, qpNorm);
2105 4 : if (ret) {
2106 0 : hccp_err("create drv qp create failed:%d", ret);
2107 0 : goto create_qp_err;
2108 : }
2109 :
2110 4 : ret = ibv_req_notify_cq(qpCb->ibSendCq, 0);
2111 4 : if (ret) {
2112 0 : hccp_err("Couldn't request send CQ notification, ret:%d", ret);
2113 0 : ret = -EOPENSRC;
2114 0 : goto ret_noritfy_cq;
2115 : }
2116 :
2117 4 : ret = ibv_req_notify_cq(qpCb->ibRecvCq, 0);
2118 4 : if (ret) {
2119 0 : hccp_err("Couldn't request recv CQ notification, ret:%d", ret);
2120 0 : ret = -EOPENSRC;
2121 0 : goto ret_noritfy_cq;
2122 : }
2123 :
2124 4 : ret = RsQpNotifyMr(rdevCb, qpCb, &qpResp->qpn); // alloc mr
2125 4 : if (ret) {
2126 0 : hccp_err("store qp notify mr failed:%d", ret);
2127 0 : goto ret_noritfy_cq;
2128 : }
2129 :
2130 4 : RsQpPrepareQpResp(qpNorm, qpCb, qpResp);
2131 :
2132 4 : return 0;
2133 :
2134 0 : ret_noritfy_cq:
2135 0 : RsDrvQpDestroy(qpCb);
2136 :
2137 0 : create_qp_err:
2138 0 : RsFreeQpcb(rdevCb, qpCb);
2139 0 : return ret;
2140 : }
2141 :
2142 35 : void RsMrRelease(struct RsQpCb *qpCb)
2143 : {
2144 35 : struct RsMrCb *mrTmp2 = NULL;
2145 35 : struct RsMrCb *mrTmp = NULL;
2146 :
2147 35 : RS_PTHREAD_MUTEX_LOCK(&qpCb->qpMutex);
2148 35 : RS_LIST_GET_HEAD_ENTRY(mrTmp, mrTmp2, &qpCb->mrList, list, struct RsMrCb);
2149 69 : for (; (&mrTmp->list) != &qpCb->mrList;
2150 34 : mrTmp = mrTmp2, mrTmp2 = list_entry(mrTmp2->list.next, struct RsMrCb, list)) {
2151 34 : if (mrTmp->ibMr != qpCb->rdevCb->notifyMr) {
2152 2 : (void)RsDrvMrDereg(mrTmp->ibMr);
2153 : }
2154 34 : RsListDel(&mrTmp->list);
2155 34 : free(mrTmp);
2156 34 : mrTmp = NULL;
2157 : }
2158 :
2159 35 : RS_LIST_GET_HEAD_ENTRY(mrTmp, mrTmp2, &qpCb->remMrList, list, struct RsMrCb);
2160 66 : for (; (&mrTmp->list) != &qpCb->remMrList;
2161 31 : mrTmp = mrTmp2, mrTmp2 = list_entry(mrTmp2->list.next, struct RsMrCb, list)) {
2162 31 : RsListDel(&mrTmp->list);
2163 31 : free(mrTmp);
2164 31 : mrTmp = NULL;
2165 : }
2166 35 : RS_PTHREAD_MUTEX_ULOCK(&qpCb->qpMutex);
2167 35 : }
2168 :
2169 34 : STATIC void RsQpRelease(struct RsQpCb *qpCb)
2170 : {
2171 34 : RS_PTHREAD_MUTEX_LOCK(&qpCb->rdevCb->rdevMutex);
2172 34 : RsListDel(&qpCb->list);
2173 34 : RS_PTHREAD_MUTEX_ULOCK(&qpCb->rdevCb->rdevMutex);
2174 34 : RsIbvAckCqEvents(qpCb->ibSendCq, qpCb->numSendCqEvents);
2175 34 : RsIbvAckCqEvents(qpCb->ibRecvCq, qpCb->numRecvCqEvents);
2176 :
2177 : // dereg mr
2178 34 : RsMrRelease(qpCb);
2179 34 : }
2180 :
2181 34 : RS_ATTRI_VISI_DEF int RsQpDestroy(unsigned int phyId, unsigned int rdevIndex, unsigned int qpn)
2182 : {
2183 34 : struct RsQpCb *qpCb = NULL;
2184 : int ret;
2185 :
2186 34 : RS_QP_PARA_CHECK(phyId);
2187 34 : ret = RsQpn2qpcb(phyId, rdevIndex, qpn, &qpCb);
2188 34 : CHK_PRT_RETURN(ret != 0 || qpCb == NULL, hccp_err("get qp cb failed! qpn %u, ret %d", qpn, ret), ret);
2189 :
2190 32 : RsQpRelease(qpCb);
2191 :
2192 : // destroy qp
2193 32 : RsDrvQpDestroy(qpCb);
2194 32 : RsDrvDestroyCq(qpCb);
2195 32 : RsDeinitMemPool(qpCb);
2196 :
2197 32 : qpCb->rdevCb->qpCnt--;
2198 32 : ret = RsQpcbDeinit(qpCb->rdevCb, qpCb);
2199 32 : if (ret) {
2200 0 : hccp_err("rs_qpcb_deinit failed! ret[%d]", ret);
2201 : }
2202 :
2203 32 : pthread_mutex_destroy(&qpCb->cqeErrInfo.mutex);
2204 32 : pthread_mutex_destroy(&qpCb->qpMutex);
2205 32 : hccp_info("qp %d destroy qp, send wr[%u].", qpn, qpCb->sendWrNum);
2206 :
2207 32 : free(qpCb);
2208 32 : qpCb = NULL;
2209 32 : return ret;
2210 : }
2211 :
2212 0 : RS_ATTRI_VISI_DEF int RsQpDestroyWithoutCQ(unsigned int phyId, unsigned int rdevIndex, unsigned int qpn)
2213 : {
2214 0 : struct RsQpCb *qpCb = NULL;
2215 : int ret;
2216 :
2217 0 : RS_QP_PARA_CHECK(phyId);
2218 0 : ret = RsQpn2qpcb(phyId, rdevIndex, qpn, &qpCb);
2219 0 : CHK_PRT_RETURN(ret != 0 || qpCb == NULL, hccp_err("get qp cb failed! qpn %u, ret %d", qpn, ret), ret);
2220 :
2221 0 : RsQpRelease(qpCb);
2222 :
2223 : // destroy qp
2224 0 : RsDrvQpDestroy(qpCb);
2225 0 : RsDeinitMemPool(qpCb);
2226 :
2227 0 : qpCb->rdevCb->qpCnt--;
2228 0 : ret = RsQpcbDeinit(qpCb->rdevCb, qpCb);
2229 0 : if (ret) {
2230 0 : hccp_err("rs_qpcb_deinit failed! ret[%d]", ret);
2231 : }
2232 :
2233 0 : pthread_mutex_destroy(&qpCb->cqeErrInfo.mutex);
2234 0 : pthread_mutex_destroy(&qpCb->qpMutex);
2235 0 : hccp_info("qp %d destroy qp without cq, send wr[%u].", qpn, qpCb->sendWrNum);
2236 :
2237 0 : free(qpCb);
2238 0 : qpCb = NULL;
2239 0 : return ret;
2240 : }
2241 :
2242 27 : static void RsQpConnectAsyncMr(const struct RsQpCb *qpCb)
2243 : {
2244 : int ret;
2245 27 : struct RsMrCb *mrCb = NULL;
2246 27 : struct RsMrCb *mrCb2 = NULL;
2247 :
2248 27 : RS_LIST_GET_HEAD_ENTRY(mrCb, mrCb2, &qpCb->mrList, list, struct RsMrCb);
2249 53 : for (; (&mrCb->list) != &qpCb->mrList; mrCb = mrCb2, mrCb2 = list_entry(mrCb2->list.next, struct RsMrCb, list)) {
2250 26 : ret = RsMrInfoSync(mrCb);
2251 26 : if (ret) {
2252 0 : hccp_warn("rs_mr_info_sync unsuccessful, ret:%d", ret);
2253 : }
2254 : }
2255 27 : }
2256 :
2257 27 : STATIC void RsQpConnectAsyncQpcbSet(int fd, struct RsQpCb *qpCb)
2258 : {
2259 : int ret;
2260 27 : ret = RsSocketSend(fd, &qpCb->qpInfoLo, sizeof(struct RsQpInfo));
2261 27 : if (ret == sizeof(struct RsQpInfo)) {
2262 27 : qpCb->sendLen += (uint32_t)ret;
2263 27 : qpCb->state = RS_QP_STATUS_CONNECTING;
2264 : } else {
2265 0 : qpCb->state = RS_QP_STATUS_TIMEOUT;
2266 : }
2267 27 : }
2268 :
2269 27 : STATIC void RsQpConnectAsyncLength(int fd, struct RsQpCb *qpCb)
2270 : {
2271 : int ret;
2272 : struct RsQpLenInfo msg;
2273 :
2274 27 : msg.cmd = RS_CMD_LEN_INFO;
2275 27 : msg.len = qpCb->sendLen;
2276 :
2277 27 : ret = RsSocketSend(fd, &msg, sizeof(struct RsQpLenInfo));
2278 27 : if (ret != sizeof(struct RsQpLenInfo)) {
2279 0 : qpCb->state = RS_QP_STATUS_TIMEOUT;
2280 : }
2281 27 : }
2282 :
2283 32 : static int RsQpConnectAsyncInitPara(struct RsQpConnPara qpConnPara, int fd, struct RsQpCb **qpCb,
2284 : struct RsConnInfo **conn)
2285 : {
2286 : int ret;
2287 :
2288 32 : CHK_PRT_RETURN(qpConnPara.phyId >= RS_MAX_DEV_NUM, hccp_err("param error ! phyId:%u", qpConnPara.phyId), -EINVAL);
2289 :
2290 32 : CHK_PRT_RETURN(fd < 0, hccp_err("param error ! fd:%d must bigger than 0", fd), -EINVAL);
2291 :
2292 30 : ret = RsQpn2qpcb(qpConnPara.phyId, qpConnPara.rdevIndex, qpConnPara.qpn, qpCb);
2293 30 : CHK_PRT_RETURN(ret, hccp_err("get qpcb failed, qpn %u, ret %d", qpConnPara.qpn, ret), ret);
2294 :
2295 28 : ret = RsFd2conn(fd, conn);
2296 28 : CHK_PRT_RETURN(ret, hccp_err("get conn failed, fd %d, ret %d", fd, ret), ret);
2297 :
2298 28 : RsGetCurTime(&((*qpCb)->startTime));
2299 28 : (*qpCb)->sendLen = 0;
2300 28 : (*qpCb)->recvLen = 0;
2301 28 : (*qpCb)->expectLen = 0;
2302 28 : (*qpCb)->connInfo = *conn;
2303 :
2304 28 : return 0;
2305 : }
2306 :
2307 2 : STATIC int RsTypicalQpStateModifytoRtr(struct RsQpCb *qpCb, struct TypicalQp *localQpInfo,
2308 : struct TypicalQp *remoteQpInfo)
2309 : {
2310 2 : struct ibv_port_attr portAttr = {0};
2311 2 : union ibv_gid remoteInfoGid = {0};
2312 2 : struct ibv_qp_attr attr = {0};
2313 : int ret;
2314 :
2315 2 : attr.qp_state = IBV_QPS_RTR;
2316 2 : attr.dest_qp_num = remoteQpInfo->qpn;
2317 2 : attr.rq_psn = remoteQpInfo->psn;
2318 2 : attr.min_rnr_timer = RS_QP_ATTR_MIN_RNR_TIMER;
2319 2 : (attr.ah_attr).is_global = 0;
2320 2 : (attr.ah_attr).sl = localQpInfo->sl;
2321 2 : (attr.ah_attr).src_path_bits = 0;
2322 2 : (attr.ah_attr).port_num = qpCb->rdevCb->ibPort;
2323 :
2324 2 : attr.path_mtu = RsDrvSetMtu(qpCb);
2325 2 : CHK_PRT_RETURN(attr.path_mtu < IBV_MTU_1024,
2326 : hccp_err("qpn[%u] failed to set mtu, mtu[%d] < [%d]", localQpInfo->qpn, attr.path_mtu, IBV_MTU_1024), -EPERM);
2327 2 : if (qpCb->rdevCb->rsCb->hccpMode == NETWORK_PEER_ONLINE) {
2328 2 : attr.max_dest_rd_atomic = RS_MAX_RD_ATOMIC_NUM_PEER_ONLINE;
2329 : } else {
2330 0 : attr.max_dest_rd_atomic = RS_MAX_RD_ATOMIC_NUM;
2331 : }
2332 2 : (attr.ah_attr).grh.traffic_class = localQpInfo->tc;
2333 : // get gid_idx dynamically to avoid gid_idx changed issue: refresh gid_idx when it changed
2334 2 : ret = RsDrvGetGidIndex(qpCb->rdevCb, &portAttr, &qpCb->qpInfoLo.gidIdx);
2335 2 : if (ret == 0 && localQpInfo->gidIdx != (uint32_t)qpCb->qpInfoLo.gidIdx) {
2336 0 : hccp_warn("qpn[%u] qp_mode[%d] refresh gid_idx[%u] to [%d]", localQpInfo->qpn, qpCb->qpMode,
2337 : localQpInfo->gidIdx, qpCb->qpInfoLo.gidIdx);
2338 0 : localQpInfo->gidIdx = (uint32_t)qpCb->qpInfoLo.gidIdx;
2339 : }
2340 :
2341 2 : (void)memcpy_s(remoteInfoGid.raw, HCCP_GID_RAW_LEN, remoteQpInfo->gid, HCCP_GID_RAW_LEN);
2342 2 : if (remoteInfoGid.global.interface_id) {
2343 2 : attr.ah_attr.is_global = 1;
2344 2 : attr.ah_attr.grh.hop_limit = 1;
2345 2 : attr.ah_attr.grh.dgid = remoteInfoGid;
2346 2 : attr.ah_attr.grh.sgid_index = localQpInfo->gidIdx;
2347 : }
2348 :
2349 2 : ret = RsIbvModifyQp(qpCb->ibQp, &attr,
2350 : IBV_QP_STATE | IBV_QP_AV | IBV_QP_PATH_MTU | IBV_QP_DEST_QPN | IBV_QP_RQ_PSN | IBV_QP_MAX_DEST_RD_ATOMIC |
2351 : IBV_QP_MIN_RNR_TIMER);
2352 2 : CHK_PRT_RETURN(ret,
2353 : hccp_err("[modifyto_rtr]local_qpn[%u] remote_qpn[%u] ibv_modify_qp failed ret[%d], errno[%d]", localQpInfo->qpn,
2354 : remoteQpInfo->qpn, ret, errno),
2355 : -EOPENSRC);
2356 2 : hccp_info("qp qos attr: qpn[%u] tc[%u] sl[%u]", localQpInfo->qpn, localQpInfo->tc, localQpInfo->sl);
2357 2 : return 0;
2358 : }
2359 :
2360 2 : STATIC int RsTypicalQpStateModifytoRts(struct RsQpCb *qpCb, struct TypicalQp *localQpInfo)
2361 : {
2362 2 : struct ibv_qp_attr attr = {0};
2363 : int ret;
2364 :
2365 2 : attr.qp_state = IBV_QPS_RTS;
2366 2 : attr.timeout = (uint8_t)localQpInfo->retryTime;
2367 2 : attr.retry_cnt = (uint8_t)localQpInfo->retryCnt;
2368 2 : attr.rnr_retry = RS_QP_ATTR_RNR_RETRY;
2369 2 : attr.sq_psn = localQpInfo->psn;
2370 2 : if (qpCb->rdevCb->rsCb->hccpMode == NETWORK_PEER_ONLINE) {
2371 2 : attr.max_rd_atomic = RS_MAX_RD_ATOMIC_NUM_PEER_ONLINE;
2372 : } else {
2373 0 : attr.max_rd_atomic = RS_MAX_RD_ATOMIC_NUM;
2374 : }
2375 :
2376 2 : ret = RsIbvModifyQp(qpCb->ibQp, &attr,
2377 : IBV_QP_STATE | IBV_QP_TIMEOUT | IBV_QP_RETRY_CNT | IBV_QP_RNR_RETRY | IBV_QP_SQ_PSN | IBV_QP_MAX_QP_RD_ATOMIC);
2378 2 : CHK_PRT_RETURN(ret != 0,
2379 : hccp_err("[modifyto_rts]local_qpn[%u] ibv_modify_qp failed ret[%d], errno[%d]", localQpInfo->qpn, ret, errno),
2380 : -EOPENSRC);
2381 :
2382 2 : hccp_info("qp rdma attr: qpn[%u] timeout[%u] retrycnt[%u]", localQpInfo->qpn, localQpInfo->retryTime,
2383 : localQpInfo->retryCnt);
2384 2 : return 0;
2385 : }
2386 :
2387 2 : STATIC void RsTypicalQpModifyInfoRelated(struct RsQpCb *qpCb, struct TypicalQp *localQpInfo,
2388 : struct TypicalQp *remoteQpInfo)
2389 : {
2390 2 : qpCb->state = RS_QP_STATUS_CONNECTED;
2391 : // local qp info related: no need to relate qpn, psn, gid_idx, gid
2392 2 : qpCb->qosAttr.tc = (unsigned char)localQpInfo->tc;
2393 2 : qpCb->qosAttr.sl = (unsigned char)localQpInfo->sl;
2394 2 : qpCb->retryCnt = localQpInfo->retryCnt;
2395 2 : qpCb->timeout = localQpInfo->retryTime;
2396 : // remote qp info related
2397 2 : qpCb->qpInfoRem.qpn = (int)remoteQpInfo->qpn;
2398 2 : qpCb->qpInfoRem.psn = (int)remoteQpInfo->psn;
2399 2 : qpCb->qpInfoRem.gidIdx = (int)remoteQpInfo->gidIdx;
2400 2 : (void)memcpy_s(qpCb->qpInfoRem.gid.raw, HCCP_GID_RAW_LEN, remoteQpInfo->gid, HCCP_GID_RAW_LEN);
2401 2 : }
2402 :
2403 2 : STATIC void RsTypicalQpModifyExtend(struct RsQpCb *qpCb, struct TypicalQp *localQpInfo, struct TypicalQp *remoteQpInfo)
2404 : {
2405 2 : struct ibv_hyroce_feature output = {0};
2406 2 : struct ibv_hyroce_feature input = {0};
2407 2 : struct ibv_qp_attr_extend attr = {0};
2408 2 : uint32_t needMoreNego = 0;
2409 2 : int ret = 0;
2410 :
2411 2 : if (qpCb->rdevCb->ibCtxEx == NULL) {
2412 0 : return;
2413 : }
2414 :
2415 2 : if (localQpInfo->tc != qpCb->qosAttr.tc || localQpInfo->sl != qpCb->qosAttr.sl) {
2416 2 : hccp_warn("localQpInfo tc:%u sl:%u is not equal to qpCb tc:%u sl:%u", localQpInfo->tc, localQpInfo->sl,
2417 : qpCb->qosAttr.tc, qpCb->qosAttr.sl);
2418 2 : return;
2419 : }
2420 :
2421 0 : ret = memcpy_s(&input, sizeof(struct ibv_hyroce_feature), &remoteQpInfo->feature, sizeof(struct HyperFeature));
2422 0 : if (ret != 0) {
2423 0 : hccp_warn("memcpy_s feature unsuccessful, ret:%d qpn:%u ibv_hyroce_feature len:%zu HyperFeature len:%zu", ret,
2424 : qpCb->ibQp->qp_num, sizeof(struct ibv_hyroce_feature), sizeof(struct HyperFeature));
2425 0 : return;
2426 : }
2427 :
2428 0 : ret = RsIbvNegoQpHyroceFeature(qpCb->rdevCb->ibCtxEx, qpCb->ibQp, &input, &output, &needMoreNego);
2429 0 : if (ret != 0) {
2430 0 : hccp_warn("RsIbvNegoQpHyroceFeature unsuccessful, ret:%d qpn:%u errno:%d", ret, qpCb->ibQp->qp_num, errno);
2431 0 : return;
2432 : }
2433 :
2434 0 : attr.qp = qpCb->ibQp;
2435 0 : (void)memcpy_s(&attr.feature, sizeof(struct ibv_hyroce_feature), &output, sizeof(struct ibv_hyroce_feature));
2436 0 : attr.udp_src_port = localQpInfo->udpSport;
2437 0 : ret = RsIbvModifyQpExtend(qpCb->rdevCb->ibCtxEx, &attr,
2438 : IBV_QP_ATTR_EXTEND_UDP_SRC_PORT | IBV_QP_ATTR_EXTEND_HYROCE_FEATURE);
2439 0 : if (ret != 0) {
2440 0 : hccp_warn("RsIbvModifyQpExtend unsuccessful, ret:%d qpn:%u errno:%d", ret, qpCb->ibQp->qp_num, errno);
2441 0 : return;
2442 : }
2443 :
2444 0 : hccp_dbg("RsIbvModifyQpExtend successful, qpn:%u", qpCb->ibQp->qp_num);
2445 0 : return;
2446 : }
2447 :
2448 2 : STATIC int RsTypicalQueryQpAttr(struct RsQpCb *qpCb, struct TypicalQpAttr *qpAttr)
2449 : {
2450 2 : unsigned int qpAttrMask = HNS_ROCE_AI_QPC_UDPSPN;
2451 2 : struct hns_roce_qpc_attr_val qpAttrVal = {0};
2452 2 : struct ibv_qp_init_attr initAttr = {0};
2453 2 : struct ibv_qp_attr ibvQpAttr = {0};
2454 2 : int ret = 0;
2455 :
2456 : (void)qpAttrMask;
2457 : (void)qpAttrVal;
2458 : #ifdef CUSTOM_INTERFACE
2459 2 : if (RsIsCustomInterfaceSupported()) {
2460 2 : ret = RsRoceQueryQpc(qpCb->ibQp, &qpAttrVal, qpAttrMask);
2461 2 : if (ret != 0) {
2462 2 : hccp_warn("qpn:%d query qpc unsuccessful, ret %d", qpCb->qpInfoLo.qpn, ret);
2463 : } else {
2464 0 : qpCb->udpSport = qpAttrVal.udp_sport;
2465 : }
2466 : }
2467 : #endif
2468 :
2469 2 : ret = RsIbvQueryQp(qpCb->ibQp, &ibvQpAttr, IBV_QP_PATH_MTU, &initAttr);
2470 2 : CHK_PRT_RETURN(ret, hccp_err("RsIbvQueryQp failed, ret:%d errno:%d", ret, errno), -EOPENSRC);
2471 :
2472 2 : if (qpCb->ibQpEx != NULL) {
2473 0 : qpAttr->vendorPrivInfo = qpCb->ibQpEx->vendor_priv_info.value;
2474 : }
2475 :
2476 2 : qpAttr->udpSport = qpCb->udpSport;
2477 2 : qpAttr->pathMtu = (int)ibvQpAttr.path_mtu;
2478 2 : return 0;
2479 : }
2480 :
2481 2 : RS_ATTRI_VISI_DEF int RsTypicalQpModify(unsigned int phyId, unsigned int rdevIndex, struct TypicalQp localQpInfo,
2482 : struct TypicalQp remoteQpInfo, struct TypicalQpAttr *qpAttr)
2483 : {
2484 2 : struct ibv_qp_init_attr initAttr = {0};
2485 2 : struct ibv_qp_attr attr = {0};
2486 2 : struct RsQpCb *qpCb = NULL;
2487 : int ret;
2488 :
2489 2 : CHK_PRT_RETURN(phyId >= RS_MAX_DEV_NUM, hccp_err("[modify]phyId:%u >= [%d], is invalid", phyId, RS_MAX_DEV_NUM),
2490 : -EINVAL);
2491 :
2492 2 : CHK_PRT_RETURN(RsQpn2qpcb(phyId, rdevIndex, localQpInfo.qpn, &qpCb),
2493 : hccp_err("[modify]rs_qpn2qpcb qpn:%u failed, phyId[%u]", localQpInfo.qpn, phyId), -EACCES);
2494 :
2495 2 : CHK_PRT_RETURN(qpCb->state == RS_QP_STATUS_CONNECTED,
2496 : hccp_info("local_qpn:%u remote_qpn:%u already been connected, no need to modify again", localQpInfo.qpn,
2497 : remoteQpInfo.qpn),
2498 : 0);
2499 :
2500 : // see ib_modify_qp_is_ok for status modify, only support modify qp from INIT to RTR
2501 2 : ret = RsIbvQueryQp(qpCb->ibQp, &attr, IBV_QP_STATE, &initAttr);
2502 2 : CHK_PRT_RETURN(ret != 0 || attr.qp_state != IBV_QPS_INIT,
2503 : hccp_err("query qpn:%u failed, ret:%d or state:%d != %d", localQpInfo.qpn, ret, attr.qp_state, IBV_QPS_INIT),
2504 : -EOPENSRC);
2505 :
2506 2 : RsTypicalQpModifyExtend(qpCb, &localQpInfo, &remoteQpInfo);
2507 :
2508 2 : ret = RsTypicalQpStateModifytoRtr(qpCb, &localQpInfo, &remoteQpInfo);
2509 2 : CHK_PRT_RETURN(ret != 0,
2510 : hccp_err("[modify]local_qpn:%u remote_qpn:%u modify to rtr failed, ret %d", localQpInfo.qpn, remoteQpInfo.qpn,
2511 : ret),
2512 : ret);
2513 :
2514 2 : ret = RsTypicalQpStateModifytoRts(qpCb, &localQpInfo);
2515 2 : CHK_PRT_RETURN(ret != 0,
2516 : hccp_err("[modify]local_qpn:%u remote_qpn:%u modify to rts failed, ret %d", localQpInfo.qpn, remoteQpInfo.qpn,
2517 : ret),
2518 : ret);
2519 :
2520 2 : ret = RsTypicalQueryQpAttr(qpCb, qpAttr);
2521 2 : CHK_PRT_RETURN(ret != 0, hccp_err("RsTypicalQueryQpAttr failed, ret %d local_qpn:%u", ret, localQpInfo.qpn), ret);
2522 :
2523 2 : RsTypicalQpModifyInfoRelated(qpCb, &localQpInfo, &remoteQpInfo);
2524 :
2525 2 : hccp_info("local_qpn:%u remote_qpn:%u modify succ, udpSport:%u", localQpInfo.qpn, remoteQpInfo.qpn, qpCb->udpSport);
2526 :
2527 2 : return 0;
2528 : }
2529 :
2530 2 : STATIC int RsQpStateBatchModifytoPause(struct RsQpCb *qpCb)
2531 : {
2532 : int ret;
2533 :
2534 2 : ret = RsDrvQpStateModifytoReset(qpCb);
2535 2 : CHK_PRT_RETURN(ret, hccp_err("qp modify to reset failed, ret %d", ret), ret);
2536 :
2537 2 : hccp_info("local qpn[%d] remote qpn[%d] modify to pause succ", qpCb->qpInfoLo.qpn, qpCb->qpInfoRem.qpn);
2538 2 : return 0;
2539 : }
2540 :
2541 2 : STATIC int RsQpStateBatchModifytoConnected(struct RsQpCb *qpCb)
2542 : {
2543 : struct ibv_qp_attr attr;
2544 : int ret;
2545 :
2546 2 : ret = memset_s(&attr, sizeof(struct ibv_qp_attr), 0, sizeof(struct ibv_qp_attr));
2547 2 : CHK_PRT_RETURN(ret, hccp_err("memset_s attr failed ret %d", ret), -ESAFEFUNC);
2548 :
2549 2 : ret = RsDrvQpStateModifytoInit(qpCb, &attr);
2550 2 : CHK_PRT_RETURN(ret, hccp_err("qp modify to init failed, ret %d", ret), ret);
2551 2 : ret = RsDrvQpStateModifytoRtr(qpCb, &attr);
2552 2 : CHK_PRT_RETURN(ret, hccp_err("qp modify to rtr failed, ret %d", ret), ret);
2553 2 : ret = RsDrvQpStateModifytoRts(qpCb, &attr);
2554 2 : CHK_PRT_RETURN(ret, hccp_err("qp modify to rts failed, ret %d", ret), ret);
2555 :
2556 2 : hccp_info("local qpn[%d] remote qpn[%d] modify to rts succ", qpCb->qpInfoLo.qpn, qpCb->qpInfoRem.qpn);
2557 2 : return 0;
2558 : }
2559 :
2560 3 : RS_ATTRI_VISI_DEF int RsQpBatchModify(unsigned int phyId, unsigned int rdevIndex, int status, int qpn[], int qpnNum)
2561 : {
2562 3 : struct RsQpCb *qpCb = NULL;
2563 : int ret;
2564 : int i;
2565 :
2566 3 : CHK_PRT_RETURN(phyId >= RS_MAX_DEV_NUM, hccp_err("[modify]phyId:%u >= [%d], is invalid", phyId, RS_MAX_DEV_NUM),
2567 : -EINVAL);
2568 :
2569 7 : for (i = 0; i < qpnNum; i++) {
2570 5 : CHK_PRT_RETURN(RsQpn2qpcb(phyId, rdevIndex, (uint32_t)qpn[i], &qpCb),
2571 : hccp_err("[modify]rs_qpn2qpcb failed, phyId[%u]", phyId), -EACCES);
2572 :
2573 : /*
2574 : * see ib_modify_qp_is_ok for status modify
2575 : * only support modify qp from STATUS_PAUSE(RESET) to STATUS_CONNECTED(INIT)
2576 : */
2577 5 : if (status == RS_QP_STATUS_CONNECTED && qpCb->state == RS_QP_STATUS_PAUSE) {
2578 2 : ret = RsQpStateBatchModifytoConnected(qpCb);
2579 2 : CHK_PRT_RETURN(ret,
2580 : hccp_err("modify_qp qpn[%d]:%d to connected failed, ret[%d] phyId[%u]", i, qpn[i], ret, phyId), ret);
2581 3 : } else if (status == RS_QP_STATUS_PAUSE) {
2582 2 : ret = RsQpStateBatchModifytoPause(qpCb);
2583 2 : CHK_PRT_RETURN(ret,
2584 : hccp_err("modify_qp qpn[%d]:%d to pause failed, ret[%d] phyId[%u]", i, qpn[i], ret, phyId), ret);
2585 : } else {
2586 1 : hccp_err("modify_qp qpn[%d]:%d failed, not support to modify status[%d] to status[%d], phyId[%u]", i,
2587 : qpn[i], qpCb->state, status, phyId);
2588 1 : return -EINVAL;
2589 : }
2590 :
2591 4 : qpCb->state = status;
2592 : }
2593 :
2594 2 : return 0;
2595 : }
2596 :
2597 2 : RS_ATTRI_VISI_DEF int RsSetQpLbValue(unsigned int phyId, unsigned int rdevIndex, unsigned int qpn, int lbValue)
2598 : {
2599 2 : struct RsQpCb *qpCb = NULL;
2600 2 : int ret = 0;
2601 :
2602 2 : ret = RsQpn2qpcb(phyId, rdevIndex, qpn, &qpCb);
2603 2 : CHK_PRT_RETURN(ret != 0, hccp_err("RsQpn2qpcb failed ret:%d", ret), ret);
2604 :
2605 1 : return RsRoceSetQpLbValue(qpCb->ibQp, lbValue);
2606 : }
2607 :
2608 3 : RS_ATTRI_VISI_DEF int RsGetQpLbValue(unsigned int phyId, unsigned int rdevIndex, unsigned int qpn, int *lbValue)
2609 : {
2610 3 : struct RsQpCb *qpCb = NULL;
2611 3 : int ret = 0;
2612 :
2613 3 : ret = RsQpn2qpcb(phyId, rdevIndex, qpn, &qpCb);
2614 3 : CHK_PRT_RETURN(ret != 0, hccp_err("RsQpn2qpcb failed ret:%d", ret), ret);
2615 :
2616 2 : return RsRoceGetQpLbValue(qpCb->ibQp, lbValue);
2617 : }
2618 :
2619 32 : RS_ATTRI_VISI_DEF int RsQpConnectAsync(unsigned int phyId, unsigned int rdevIndex, unsigned int qpn, int fd)
2620 : {
2621 : int ret;
2622 32 : struct RsQpCb *qpCb = NULL;
2623 32 : struct RsConnInfo *conn = NULL;
2624 : struct RsQpConnPara qpConnPara;
2625 32 : hccp_info("qp:%d, fd:%d", qpn, fd);
2626 :
2627 32 : qpConnPara.phyId = phyId;
2628 32 : qpConnPara.rdevIndex = rdevIndex;
2629 32 : qpConnPara.qpn = qpn;
2630 32 : ret = RsQpConnectAsyncInitPara(qpConnPara, fd, &qpCb, &conn);
2631 32 : CHK_PRT_RETURN(ret, hccp_err("rs_qp_connect_async_init_para failed, qpn %u, ret %d", qpn, ret), ret);
2632 :
2633 28 : RS_PTHREAD_MUTEX_LOCK(&qpCb->qpMutex);
2634 :
2635 28 : if (qpCb->state == RS_QP_STATUS_REM_FD_CLOSE) {
2636 0 : hccp_warn("remote qp fd close, can not use it anymore!");
2637 0 : RS_PTHREAD_MUTEX_ULOCK(&qpCb->qpMutex);
2638 0 : return -EFAULT;
2639 : }
2640 :
2641 28 : if ((qpCb->state == RS_QP_STATUS_CONNECTED) || (qpCb->state == RS_QP_STATUS_CONNECTING)) {
2642 1 : hccp_warn("qp %d has already sync! state[%d]", qpCb->qpInfoLo.qpn, qpCb->state);
2643 1 : RS_PTHREAD_MUTEX_ULOCK(&qpCb->qpMutex);
2644 1 : return -EEXIST;
2645 : }
2646 :
2647 27 : RsQpConnectAsyncQpcbSet(fd, qpCb);
2648 :
2649 27 : hccp_info("after socket fd %d send QP %u, chipId %u, state:%d!", fd, qpn, qpCb->rdevCb->rsCb->chipId, qpCb->state);
2650 :
2651 27 : RS_PTHREAD_MUTEX_ULOCK(&qpCb->qpMutex);
2652 :
2653 27 : RsQpMrRecvHandle(fd, qpCb);
2654 :
2655 27 : RsQpConnectAsyncMr(qpCb);
2656 :
2657 27 : RsQpConnectAsyncLength(fd, qpCb);
2658 :
2659 27 : hccp_info("QP %d async done, state:%d!", qpn, qpCb->state);
2660 :
2661 27 : return 0;
2662 : }
2663 :
2664 1 : RS_ATTRI_VISI_DEF int RsGetQpStatus(unsigned int phyId, unsigned int rdevIndex, unsigned int qpn,
2665 : struct RsQpStatusInfo *qpInfo)
2666 : {
2667 1 : unsigned int qpAttrMask = HNS_ROCE_AI_QPC_UDPSPN;
2668 1 : struct hns_roce_qpc_attr_val qpAttrVal = {0};
2669 1 : struct RsQpCb *qpCb = NULL;
2670 : int ret;
2671 :
2672 : (void)qpAttrMask;
2673 : (void)qpAttrVal;
2674 1 : CHK_PRT_RETURN(qpInfo == NULL, hccp_err("param error, qpInfo is NULL"), -EINVAL);
2675 :
2676 1 : CHK_PRT_RETURN(phyId >= RS_MAX_DEV_NUM, hccp_err("phyId:%u >= [%d], is invalid", phyId, RS_MAX_DEV_NUM), -EINVAL);
2677 :
2678 1 : ret = RsQpn2qpcb(phyId, rdevIndex, qpn, &qpCb);
2679 1 : CHK_PRT_RETURN(ret, hccp_err("get qp cb failed, qpn:%u, ret %d", qpn, ret), ret);
2680 :
2681 : // qp state is CONNECTED, no need to handle
2682 1 : if (qpCb->state == RS_QP_STATUS_CONNECTED) {
2683 1 : goto update_qp_cb;
2684 : }
2685 :
2686 : // modify state to CONNECTED
2687 0 : if (qpCb->expectLen == qpCb->recvLen - sizeof(struct RsQpLenInfo)) {
2688 0 : qpCb->state = RS_QP_STATUS_CONNECTED;
2689 : } else {
2690 0 : RsQpMrRecvHandle(qpCb->connInfo->connfd, qpCb);
2691 0 : goto out;
2692 : }
2693 :
2694 1 : update_qp_cb:
2695 : #ifdef CUSTOM_INTERFACE
2696 1 : if (RsIsCustomInterfaceSupported()) {
2697 1 : ret = RsRoceQueryQpc(qpCb->ibQp, &qpAttrVal, qpAttrMask);
2698 1 : if (ret != 0) {
2699 1 : hccp_warn("qpn:%d query qpc unsuccessful, ret %d", qpCb->qpInfoLo.qpn, ret);
2700 : } else {
2701 0 : qpCb->udpSport = qpAttrVal.udp_sport;
2702 : }
2703 : }
2704 : #endif
2705 0 : out:
2706 1 : hccp_dbg("qp:%u, state:%d, udpSport:%u", qpn, qpCb->state, qpCb->udpSport);
2707 1 : qpInfo->status = qpCb->state;
2708 1 : qpInfo->udpSport = qpCb->udpSport;
2709 :
2710 1 : return 0;
2711 : }
2712 :
2713 3 : RS_ATTRI_VISI_DEF int RsGetQpContext(unsigned int phyId, unsigned int rdevIndex, unsigned int qpn, void **qp,
2714 : void **sendCq, void **recvCq)
2715 : {
2716 : int ret;
2717 3 : struct RsQpCb *qpCb = NULL;
2718 :
2719 3 : CHK_PRT_RETURN(phyId >= RS_MAX_DEV_NUM, hccp_err("phyId:%u >= [%d], is invalid", phyId, RS_MAX_DEV_NUM), -EINVAL);
2720 :
2721 2 : ret = RsQpn2qpcb(phyId, rdevIndex, qpn, &qpCb);
2722 2 : CHK_PRT_RETURN(ret, hccp_err("rs_qpn2qpcb failed ret[%d]", ret), ret);
2723 :
2724 1 : *qp = qpCb->ibQp;
2725 1 : *sendCq = qpCb->ibSendCq;
2726 1 : *recvCq = qpCb->ibRecvCq;
2727 :
2728 1 : hccp_dbg("qpn[%u] succ", qpn);
2729 :
2730 1 : return 0;
2731 : }
2732 :
2733 26 : int RsQueryRdevCb(unsigned int phyId, unsigned int rdevIndex, struct RsRdevCb **rdevCb)
2734 : {
2735 : int ret;
2736 : unsigned int chipId;
2737 26 : struct rs_cb *rsCb = NULL;
2738 :
2739 26 : RS_QP_PARA_CHECK(phyId);
2740 :
2741 24 : ret = rsGetLocalDevIDByHostDevID(phyId, &chipId);
2742 24 : CHK_PRT_RETURN(ret, hccp_err("rs_query_rdev_cb phyId[%u] invalid, ret:%d", phyId, ret), ret);
2743 :
2744 23 : ret = RsDev2rscb(chipId, &rsCb, false);
2745 23 : CHK_PRT_RETURN(ret, hccp_err("rs_query_rdev_cb get rs_cb failed, ret:%d", ret), -ENODEV);
2746 :
2747 23 : ret = RsGetRdevCb(rsCb, rdevIndex, rdevCb);
2748 23 : CHK_PRT_RETURN(ret, hccp_err("rs_get_rdev_cb failed! ret:%d, rdevIndex:%u", ret, rdevIndex), ret);
2749 :
2750 23 : return 0;
2751 : }
2752 :
2753 3 : RS_ATTRI_VISI_DEF int RsGetLbMax(unsigned int phyId, unsigned int rdevIndex, int *lbMax)
2754 : {
2755 3 : struct RsRdevCb *rdevCb = NULL;
2756 3 : int ret = 0;
2757 :
2758 3 : ret = RsQueryRdevCb(phyId, rdevIndex, &rdevCb);
2759 3 : CHK_PRT_RETURN(ret != 0, hccp_err("RsQueryRdevCb phyId:%u rdev_index:%u ret:%d", phyId, rdevIndex, ret), ret);
2760 :
2761 2 : return RsRoceGetQpNum(rdevCb->ibCtx, lbMax);
2762 : }
2763 :
2764 3 : STATIC int RsBuildUpQpcb(struct RsCqContext *cqContext, struct ibv_qp_init_attr *qpInitAttr, struct RsQpCb **qpCb)
2765 : {
2766 : int ret;
2767 :
2768 3 : ret = RsCallocQpcb(1, qpCb);
2769 3 : CHK_PRT_RETURN(ret, hccp_err("alloc mem for qp_cb failed, ret:%d errno:%d", ret, errno), -ENOMEM);
2770 :
2771 3 : ret = pthread_mutex_init(&(*qpCb)->qpMutex, NULL);
2772 3 : if (ret) {
2773 0 : hccp_err("pthread_mutex_init failed, ret %d", ret);
2774 0 : goto pthread_mutex_init_err;
2775 : }
2776 :
2777 3 : (*qpCb)->rdevCb = cqContext->rdevCb;
2778 3 : RS_INIT_LIST_HEAD(&(*qpCb)->mrList);
2779 3 : RS_INIT_LIST_HEAD(&(*qpCb)->remMrList);
2780 :
2781 3 : (*qpCb)->eqNum = cqContext->eqNum;
2782 3 : (*qpCb)->channel = cqContext->channel;
2783 3 : (*qpCb)->ibSendCq = cqContext->ibSendCq;
2784 3 : (*qpCb)->ibRecvCq = cqContext->ibRecvCq;
2785 3 : (*qpCb)->sendEvent = cqContext->sendEvent;
2786 3 : (*qpCb)->recvEvent = cqContext->recvEvent;
2787 3 : (*qpCb)->numRecvCqEvents = 0;
2788 3 : (*qpCb)->numSendCqEvents = 0;
2789 3 : (*qpCb)->srqContext = cqContext->srqContext;
2790 3 : (*qpCb)->state = RS_QP_STATUS_DISCONNECT;
2791 3 : (*qpCb)->ibPd = cqContext->rdevCb->ibPd;
2792 3 : (*qpCb)->txDepth = qpInitAttr->cap.max_send_wr;
2793 3 : (*qpCb)->rxDepth = qpInitAttr->cap.max_recv_wr;
2794 3 : (*qpCb)->qosAttr.tc = (RS_ROCE_DSCP_33 & RS_DSCP_MASK) << RS_DSCP_OFF;
2795 3 : (*qpCb)->qosAttr.sl = RS_ROCE_4_SL;
2796 3 : (*qpCb)->timeout = RS_QP_ATTR_TIMEOUT;
2797 3 : (*qpCb)->retryCnt = RS_QP_ATTR_RETRY_CNT;
2798 :
2799 3 : return 0;
2800 :
2801 0 : pthread_mutex_init_err:
2802 0 : free(*qpCb);
2803 0 : (*qpCb) = NULL;
2804 0 : return ret;
2805 : }
2806 :
2807 6 : RS_ATTRI_VISI_DEF int RsCreateCqEvent(struct RsCqContext *cqContext, struct CqAttr *attr)
2808 : {
2809 : int ret;
2810 6 : cqContext->channel = RsIbvCreateCompChannel(cqContext->rdevCb->ibCtx);
2811 :
2812 6 : if (cqContext->channel == NULL) {
2813 1 : hccp_err("ibv_create_comp_channel failed, ret %d, errno(%d)", -EINVAL, errno);
2814 1 : return -EINVAL;
2815 : }
2816 :
2817 5 : hccp_info("comp channel fd[%d].", cqContext->channel->fd);
2818 5 : ret = RsEpollCtl(cqContext->rdevCb->rsCb->connCb.epollfd, EPOLL_CTL_ADD, cqContext->channel->fd,
2819 : EPOLLIN | EPOLLRDHUP);
2820 : #ifndef CA_CONFIG_LLT
2821 : if (ret) {
2822 : hccp_err("add channel fd failed ret %d", ret);
2823 : goto rs_cq_epoll_ctl_err;
2824 : }
2825 : #endif
2826 :
2827 5 : ret = RsDrvCreateCqEvent(cqContext, attr);
2828 5 : if (ret) {
2829 0 : hccp_err("create drv cq event failed:%d", ret);
2830 0 : goto rs_cq_create_err;
2831 : }
2832 :
2833 5 : return ret;
2834 0 : rs_cq_create_err:
2835 0 : ret = RsEpollCtl(cqContext->rdevCb->rsCb->connCb.epollfd, EPOLL_CTL_DEL, cqContext->channel->fd,
2836 : EPOLLIN | EPOLLRDHUP);
2837 : #ifndef CA_CONFIG_LLT
2838 : if (ret) {
2839 : hccp_err("del channel fd failed ret %d", ret);
2840 : }
2841 : #endif
2842 0 : rs_cq_epoll_ctl_err:
2843 0 : if (cqContext->channel != NULL) {
2844 0 : RsIbvDestroyCompChannel(cqContext->channel);
2845 0 : cqContext->channel = NULL;
2846 : }
2847 0 : return ret;
2848 : }
2849 :
2850 5 : RS_ATTRI_VISI_DEF int RsCqCreate(unsigned int phyId, unsigned int rdevIndex, struct CqAttr *attr)
2851 : {
2852 : int ret;
2853 5 : struct RsRdevCb *rdevCb = NULL;
2854 5 : struct RsCqContext *cqContext = NULL;
2855 :
2856 5 : ret = RsQueryRdevCb(phyId, rdevIndex, &rdevCb);
2857 5 : if (ret) {
2858 0 : hccp_err("rs_query_rdev_cb phyId[%u] rdev_index[%u], ret %d", phyId, rdevIndex, ret);
2859 0 : return ret;
2860 : }
2861 :
2862 5 : cqContext = calloc(1, sizeof(struct RsCqContext));
2863 5 : if (cqContext == NULL) {
2864 0 : return -ENOMEM;
2865 : }
2866 5 : cqContext->rdevCb = rdevCb;
2867 5 : cqContext->eqNum = 0;
2868 5 : if (attr->sendChannel == NULL && attr->recvChannel == NULL) {
2869 3 : if (*attr->ibSendCq == NULL && *attr->ibRecvCq != NULL) {
2870 : // 只创建sq cq
2871 1 : cqContext->cqCreateMode = RS_SQ_CQ_CREATE;
2872 1 : cqContext->ibRecvCq = *attr->ibRecvCq;
2873 1 : cqContext->srqContext = attr->srqContext;
2874 : } else {
2875 : // 创建sq&rq cq
2876 2 : cqContext->cqCreateMode = RS_NORMAL_CQ_CREATE;
2877 : }
2878 3 : ret = RsCreateCqEvent(cqContext, attr);
2879 3 : if (ret) {
2880 0 : hccp_err("create cq event failed:%d", ret);
2881 0 : goto rs_cq_create_err;
2882 : }
2883 2 : } else if (attr->sendChannel != NULL && attr->recvChannel != NULL) {
2884 : // 使用输入comp channel创建sq&rq
2885 1 : ret = RsDrvCreateCqWithChannel(cqContext, attr);
2886 1 : if (ret) {
2887 0 : hccp_err("create drv cq with channel failed:%d", ret);
2888 0 : goto rs_cq_create_err;
2889 : }
2890 : } else {
2891 1 : hccp_err("rs create cq failed, sendChannel or recvChannel is NULL.");
2892 1 : ret = -EPERM;
2893 1 : goto rs_cq_create_err;
2894 : }
2895 :
2896 4 : *attr->qpContext = cqContext;
2897 4 : return 0;
2898 :
2899 1 : rs_cq_create_err:
2900 1 : free(cqContext);
2901 1 : cqContext = NULL;
2902 :
2903 1 : return ret;
2904 : }
2905 :
2906 0 : RS_ATTRI_VISI_DEF int RsTypicalCqCreate(unsigned int phyId, unsigned int rdevIndex, unsigned int cqDepth,
2907 : unsigned int *cqn)
2908 : {
2909 : struct RsTypicalCqEntry *entry;
2910 : struct RsTypicalCqEntry *tmp;
2911 : int ret;
2912 0 : struct RsRdevCb *rdevCb = NULL;
2913 0 : struct ibv_cq *ibCq = NULL;
2914 : struct rdma_lite_device_cq_attr deviceCqAttr;
2915 :
2916 0 : ret = RsQueryRdevCb(phyId, rdevIndex, &rdevCb);
2917 0 : if (ret) {
2918 0 : hccp_err("rs_query_rdev_cb phyId[%u] rdevIndex[%u], ret %d", phyId, rdevIndex, ret);
2919 0 : return ret;
2920 : }
2921 :
2922 0 : ret = RsDrvTypicalCqCreate(rdevCb, cqDepth, cqn, &ibCq, &deviceCqAttr);
2923 0 : if (ret) {
2924 0 : hccp_err("rs_drv_typical_cq_create failed, cqDepth[%u] ret[%d]", cqDepth, ret);
2925 0 : return ret;
2926 : }
2927 :
2928 0 : pthread_mutex_lock(&gRsTypicalCqMutex);
2929 0 : if (gRsTypicalCqList.next == NULL) {
2930 0 : RS_INIT_LIST_HEAD(&gRsTypicalCqList);
2931 : }
2932 :
2933 0 : RS_LIST_GET_HEAD_ENTRY(tmp, entry, &gRsTypicalCqList, list, struct RsTypicalCqEntry);
2934 0 : for (; &tmp->list != &gRsTypicalCqList;
2935 0 : tmp = entry, entry = list_entry(entry->list.next, struct RsTypicalCqEntry, list)) {
2936 0 : if (tmp->phyId == phyId && tmp->rdevIndex == rdevIndex && tmp->cqn == *cqn) {
2937 0 : tmp->ibCq = ibCq;
2938 0 : tmp->deviceCqAttr = deviceCqAttr;
2939 0 : pthread_mutex_unlock(&gRsTypicalCqMutex);
2940 0 : hccp_info("RsTypicalCqCreate updated: phyId[%u] rdevIndex[%u] cqn[%u] cqDepth[%u]", phyId, rdevIndex, *cqn,
2941 : cqDepth);
2942 0 : return 0;
2943 : }
2944 : }
2945 :
2946 0 : entry = calloc(1, sizeof(struct RsTypicalCqEntry));
2947 0 : if (entry == NULL) {
2948 0 : pthread_mutex_unlock(&gRsTypicalCqMutex);
2949 0 : hccp_err("RsTypicalCqCreate calloc failed, cqn[%u]", *cqn);
2950 0 : return -ENOMEM;
2951 : }
2952 0 : entry->phyId = phyId;
2953 0 : entry->rdevIndex = rdevIndex;
2954 0 : entry->cqn = *cqn;
2955 0 : entry->ibCq = ibCq;
2956 0 : entry->deviceCqAttr = deviceCqAttr;
2957 0 : RsListAddTail(&entry->list, &gRsTypicalCqList);
2958 0 : pthread_mutex_unlock(&gRsTypicalCqMutex);
2959 :
2960 0 : hccp_info("RsTypicalCqCreate success: phyId[%u] rdevIndex[%u] cqn[%u] cqDepth[%u]", phyId, rdevIndex, *cqn,
2961 : cqDepth);
2962 :
2963 0 : return 0;
2964 : }
2965 :
2966 0 : RS_ATTRI_VISI_DEF int RsTypicalCqDestroy(unsigned int phyId, unsigned int rdevIndex, unsigned int cqn)
2967 : {
2968 : struct RsTypicalCqEntry *entry;
2969 : struct RsTypicalCqEntry *tmp;
2970 : int ret;
2971 :
2972 0 : pthread_mutex_lock(&gRsTypicalCqMutex);
2973 0 : if (gRsTypicalCqList.next == NULL) {
2974 0 : pthread_mutex_unlock(&gRsTypicalCqMutex);
2975 0 : hccp_err("rs_typical_cq_destroy: cqn[%u] not found phyId[%u] rdevIndex[%u]", cqn, phyId, rdevIndex);
2976 0 : return -EINVAL;
2977 : }
2978 :
2979 0 : RS_LIST_GET_HEAD_ENTRY(tmp, entry, &gRsTypicalCqList, list, struct RsTypicalCqEntry);
2980 0 : for (; &tmp->list != &gRsTypicalCqList;
2981 0 : tmp = entry, entry = list_entry(entry->list.next, struct RsTypicalCqEntry, list)) {
2982 0 : if (tmp->phyId == phyId && tmp->rdevIndex == rdevIndex && tmp->cqn == cqn) {
2983 0 : RsListDel(&tmp->list);
2984 0 : pthread_mutex_unlock(&gRsTypicalCqMutex);
2985 :
2986 0 : if (tmp->ibCq != NULL) {
2987 0 : ret = RsIbvDestroyCq(tmp->ibCq);
2988 0 : if (ret) {
2989 0 : hccp_err("rs_ibv_destroy_cq failed cqn[%u] ret[%d]", cqn, ret);
2990 0 : free(tmp);
2991 0 : return ret;
2992 : }
2993 : }
2994 0 : free(tmp);
2995 0 : hccp_info("RsTypicalCqDestroy success: phyId[%u] rdevIndex[%u] cqn[%u]", phyId, rdevIndex, cqn);
2996 0 : return 0;
2997 : }
2998 : }
2999 0 : pthread_mutex_unlock(&gRsTypicalCqMutex);
3000 :
3001 0 : hccp_err("rs_typical_cq_destroy: cqn[%u] not found phyId[%u] rdevIndex[%u]", cqn, phyId, rdevIndex);
3002 0 : return -EINVAL;
3003 : }
3004 :
3005 0 : RS_ATTRI_VISI_DEF int RsGetLiteCqAttr(unsigned int phyId, unsigned int rdevIndex, unsigned int cqn,
3006 : struct rdma_lite_device_cq_attr *deviceCqAttr)
3007 : {
3008 : struct RsTypicalCqEntry *entry;
3009 : struct RsTypicalCqEntry *tmp;
3010 : int ret;
3011 :
3012 0 : RS_CHECK_POINTER_NULL_RETURN_INT(deviceCqAttr);
3013 :
3014 0 : pthread_mutex_lock(&gRsTypicalCqMutex);
3015 0 : if (gRsTypicalCqList.next == NULL) {
3016 0 : pthread_mutex_unlock(&gRsTypicalCqMutex);
3017 0 : hccp_err("rs_get_lite_cq_attr: cqn[%u] not found phyId[%u] rdevIndex[%u]", cqn, phyId, rdevIndex);
3018 0 : return -EINVAL;
3019 : }
3020 :
3021 0 : RS_LIST_GET_HEAD_ENTRY(tmp, entry, &gRsTypicalCqList, list, struct RsTypicalCqEntry);
3022 0 : for (; &tmp->list != &gRsTypicalCqList;
3023 0 : tmp = entry, entry = list_entry(entry->list.next, struct RsTypicalCqEntry, list)) {
3024 0 : if (tmp->phyId == phyId && tmp->rdevIndex == rdevIndex && tmp->cqn == cqn) {
3025 0 : ret = memcpy_s(deviceCqAttr, sizeof(*deviceCqAttr), &tmp->deviceCqAttr, sizeof(tmp->deviceCqAttr));
3026 0 : pthread_mutex_unlock(&gRsTypicalCqMutex);
3027 0 : if (ret) {
3028 0 : hccp_err("memcpy_s failed, ret:%d", ret);
3029 0 : return ret;
3030 : }
3031 0 : hccp_info("RsGetLiteCqAttr success: cqn[%u] depth[%u]", cqn, deviceCqAttr->depth);
3032 0 : return 0;
3033 : }
3034 : }
3035 0 : pthread_mutex_unlock(&gRsTypicalCqMutex);
3036 :
3037 0 : hccp_err("rs_get_lite_cq_attr: cqn[%u] not found phyId[%u] rdevIndex[%u]", cqn, phyId, rdevIndex);
3038 0 : return -EINVAL;
3039 : }
3040 :
3041 0 : RS_ATTRI_VISI_DEF int RsQpCreateWithCQWithAttrs(unsigned int phyId, unsigned int rdevIndex, unsigned int sendCqn,
3042 : unsigned int recvCqn, struct RsQpNormWithAttrs *qpNorm, struct RsQpRespWithAttrs *qpResp)
3043 : {
3044 : struct RsTypicalCqEntry *entry;
3045 : struct RsTypicalCqEntry *tmp;
3046 0 : struct RsRdevCb *rdevCb = NULL;
3047 0 : struct RsQpCb *qpCb = NULL;
3048 0 : struct ibv_cq *sendIbCq = NULL;
3049 0 : struct ibv_cq *recvIbCq = NULL;
3050 : struct rdma_lite_device_cq_attr sendDeviceCqAttr;
3051 : struct rdma_lite_device_cq_attr recvDeviceCqAttr;
3052 : int qpMode;
3053 : int ret;
3054 0 : bool sendFound = false;
3055 0 : bool recvFound = false;
3056 :
3057 0 : RS_QP_PARA_CHECK(phyId);
3058 0 : CHK_PRT_RETURN(qpResp == NULL, hccp_err("qp_resp is NULL!"), -EINVAL);
3059 :
3060 0 : ret = RsQpCheckQpNorm(qpNorm, &qpMode);
3061 0 : CHK_PRT_RETURN(ret != 0, hccp_err("check qp mode failed, ret:%d", ret), ret);
3062 :
3063 0 : ret = RsQpQueryInfo(phyId, rdevIndex, &rdevCb, qpMode);
3064 0 : CHK_PRT_RETURN(ret, hccp_err("query qp info failed:%d", ret), ret);
3065 :
3066 0 : pthread_mutex_lock(&gRsTypicalCqMutex);
3067 0 : if (gRsTypicalCqList.next != NULL) {
3068 0 : RS_LIST_GET_HEAD_ENTRY(tmp, entry, &gRsTypicalCqList, list, struct RsTypicalCqEntry);
3069 0 : for (; &tmp->list != &gRsTypicalCqList;
3070 0 : tmp = entry, entry = list_entry(entry->list.next, struct RsTypicalCqEntry, list)) {
3071 0 : if (tmp->phyId == phyId && tmp->rdevIndex == rdevIndex && tmp->cqn == sendCqn) {
3072 0 : sendIbCq = tmp->ibCq;
3073 0 : sendDeviceCqAttr = tmp->deviceCqAttr;
3074 0 : sendFound = true;
3075 0 : break;
3076 : }
3077 : }
3078 : }
3079 0 : pthread_mutex_unlock(&gRsTypicalCqMutex);
3080 0 : CHK_PRT_RETURN(!sendFound,
3081 : hccp_err("send cq not found: sendCqn[%u] phyId[%u] rdevIndex[%u]", sendCqn, phyId, rdevIndex), -EINVAL);
3082 :
3083 0 : pthread_mutex_lock(&gRsTypicalCqMutex);
3084 0 : if (gRsTypicalCqList.next != NULL) {
3085 0 : RS_LIST_GET_HEAD_ENTRY(tmp, entry, &gRsTypicalCqList, list, struct RsTypicalCqEntry);
3086 0 : for (; &tmp->list != &gRsTypicalCqList;
3087 0 : tmp = entry, entry = list_entry(entry->list.next, struct RsTypicalCqEntry, list)) {
3088 0 : if (tmp->phyId == phyId && tmp->rdevIndex == rdevIndex && tmp->cqn == recvCqn) {
3089 0 : recvIbCq = tmp->ibCq;
3090 0 : recvDeviceCqAttr = tmp->deviceCqAttr;
3091 0 : recvFound = true;
3092 0 : break;
3093 : }
3094 : }
3095 : }
3096 0 : pthread_mutex_unlock(&gRsTypicalCqMutex);
3097 0 : CHK_PRT_RETURN(!recvFound,
3098 : hccp_err("recv cq not found: recvCqn[%u] phyId[%u] rdevIndex[%u]", recvCqn, phyId, rdevIndex), -EINVAL);
3099 :
3100 0 : ret = RsCallocQpcb(1, &qpCb);
3101 0 : CHK_PRT_RETURN(ret, hccp_err("alloc mem for qp_cb failed, ret:%d errno:%d", ret, errno), -ENOMEM);
3102 :
3103 0 : ret = pthread_mutex_init(&qpCb->qpMutex, NULL);
3104 0 : if (ret) {
3105 0 : hccp_err("pthread_mutex_init failed, ret %d", ret);
3106 0 : goto qp_mutex_init_err;
3107 : }
3108 :
3109 0 : ret = pthread_mutex_init(&qpCb->cqeErrInfo.mutex, NULL);
3110 0 : if (ret) {
3111 0 : hccp_err("pthread_mutex_init failed, ret %d", ret);
3112 0 : goto cqe_mutex_init_err;
3113 : }
3114 :
3115 0 : ret = RsQpcbInitWithAttrs(rdevCb, qpCb, qpNorm);
3116 0 : if (ret) {
3117 0 : hccp_err("create qp tx rx failed ret %d", ret);
3118 0 : goto rs_qpcb_init_err;
3119 : }
3120 :
3121 0 : ret = RsInitMemPool(qpCb);
3122 0 : if (ret) {
3123 0 : hccp_err("init mem pool failed ret %d", ret);
3124 0 : goto rs_init_mem_err;
3125 : }
3126 :
3127 : // Assign pre-existing CQs (instead of RsDrvCreateCqWithAttrs)
3128 0 : qpCb->ibSendCq = sendIbCq;
3129 0 : qpCb->ibRecvCq = recvIbCq;
3130 0 : qpCb->qpResp.sendCqData = sendDeviceCqAttr;
3131 0 : qpCb->qpResp.recvCqData = recvDeviceCqAttr;
3132 0 : qpCb->sendCqDepth = sendDeviceCqAttr.depth;
3133 0 : qpCb->recvCqDepth = recvDeviceCqAttr.depth;
3134 :
3135 0 : ret = RsDrvQpCreateWithAttrs(qpCb, qpNorm);
3136 0 : if (ret) {
3137 0 : hccp_err("Create drv qp create failed:%d", ret);
3138 0 : goto create_qp_err;
3139 : }
3140 :
3141 0 : ret = ibv_req_notify_cq(qpCb->ibSendCq, 0);
3142 0 : if (ret) {
3143 0 : hccp_err("Can't request send CQ notification, ret:%d", ret);
3144 0 : ret = -EOPENSRC;
3145 0 : goto ret_noritfy_cq;
3146 : }
3147 :
3148 0 : ret = ibv_req_notify_cq(qpCb->ibRecvCq, 0);
3149 0 : if (ret) {
3150 0 : hccp_err("Can't request recv CQ notification, ret:%d", ret);
3151 0 : ret = -EOPENSRC;
3152 0 : goto ret_noritfy_cq;
3153 : }
3154 :
3155 0 : ret = RsQpNotifyMr(rdevCb, qpCb, &qpResp->qpn);
3156 0 : if (ret) {
3157 0 : hccp_err("Store qp notify mr failed:%d", ret);
3158 0 : goto ret_noritfy_cq;
3159 : }
3160 :
3161 0 : RsQpPrepareQpResp(qpNorm, qpCb, qpResp);
3162 :
3163 0 : return 0;
3164 :
3165 0 : ret_noritfy_cq:
3166 0 : RsDrvQpDestroy(qpCb);
3167 :
3168 0 : create_qp_err:
3169 : // Do NOT call RsDrvDestroyCq — CQs are not owned by this QP
3170 0 : RsDeinitMemPool(qpCb);
3171 0 : (void)RsQpcbDeinit(rdevCb, qpCb);
3172 :
3173 0 : rs_init_mem_err:
3174 0 : rs_qpcb_init_err:
3175 0 : pthread_mutex_destroy(&qpCb->cqeErrInfo.mutex);
3176 :
3177 0 : cqe_mutex_init_err:
3178 0 : pthread_mutex_destroy(&qpCb->qpMutex);
3179 :
3180 0 : qp_mutex_init_err:
3181 0 : free(qpCb);
3182 0 : qpCb = NULL;
3183 :
3184 0 : return ret;
3185 : }
3186 :
3187 7 : RS_ATTRI_VISI_DEF int RsCqDestroy(unsigned int phyId, unsigned int rdevIndex, struct CqAttr *attr)
3188 : {
3189 : int ret;
3190 7 : struct RsRdevCb *rdevCb = NULL;
3191 7 : struct RsCqContext *cqContext = NULL;
3192 :
3193 7 : ret = RsQueryRdevCb(phyId, rdevIndex, &rdevCb);
3194 7 : CHK_PRT_RETURN(ret, hccp_err("rs_query_rdev_cb phyId[%u] rdev_index[%u], ret %d", phyId, rdevIndex, ret), ret);
3195 :
3196 7 : cqContext = *attr->qpContext;
3197 :
3198 7 : ret = RsDrvDestroyCqEvent(cqContext);
3199 7 : if (ret) {
3200 0 : hccp_err("rs_drv_destroy_cq_event failed ret %d", ret);
3201 : }
3202 :
3203 7 : if (cqContext->channel != NULL) {
3204 5 : ret = RsEpollCtl(rdevCb->rsCb->connCb.epollfd, EPOLL_CTL_DEL, cqContext->channel->fd, EPOLLIN | EPOLLRDHUP);
3205 : #ifndef CA_CONFIG_LLT
3206 : if (ret) {
3207 : hccp_err("del channel fd failed ret %d", ret);
3208 : }
3209 : #endif
3210 5 : RsIbvDestroyCompChannel(cqContext->channel);
3211 5 : cqContext->channel = NULL;
3212 : }
3213 :
3214 7 : free(cqContext);
3215 7 : cqContext = NULL;
3216 :
3217 7 : return ret;
3218 : }
3219 :
3220 4 : RS_ATTRI_VISI_DEF int RsNormalQpCreate(unsigned int phyId, unsigned int rdevIndex, struct ibv_qp_init_attr *qpInitAttr,
3221 : struct RsQpResp *qpResp, void **qp)
3222 : {
3223 4 : struct RsCqContext *cqContext = NULL;
3224 4 : struct RsRdevCb *rdevCb = NULL;
3225 4 : struct RsQpCb *qpCb = NULL;
3226 : int ret;
3227 :
3228 4 : CHK_PRT_RETURN(qpResp == NULL, hccp_err("qp_resp is NULL!"), -EINVAL);
3229 4 : ret = RsQueryRdevCb(phyId, rdevIndex, &rdevCb);
3230 4 : CHK_PRT_RETURN(ret != 0, hccp_err("rs_query_rdev_cb phyId[%u] rdev_index[%u], ret %d", phyId, rdevIndex, ret), ret);
3231 :
3232 4 : CHK_PRT_RETURN(qpInitAttr == NULL, hccp_err("qp_init_attr is NULL!"), -EINVAL);
3233 :
3234 4 : cqContext = qpInitAttr->qp_context;
3235 4 : CHK_PRT_RETURN(cqContext == NULL, hccp_err("cq_context is NULL!"), -EINVAL);
3236 3 : CHK_PRT_RETURN(rdevCb != cqContext->rdevCb,
3237 : hccp_err("rs_query_rdev_cb phyId[%u] rdev_index[%u],"
3238 : "rdevCb is invalid.",
3239 : phyId, rdevIndex),
3240 : -EINVAL);
3241 :
3242 3 : ret = RsBuildUpQpcb(cqContext, qpInitAttr, &qpCb);
3243 3 : CHK_PRT_RETURN(ret, hccp_err("rs_build_up_qpcb failed, ret:%d", ret), ret);
3244 :
3245 3 : ret = RsDrvNormalQpCreate(qpCb, qpInitAttr);
3246 3 : if (ret) {
3247 1 : hccp_err("create drv qp create failed:%d", ret);
3248 1 : goto create_qp_err;
3249 : }
3250 :
3251 2 : RS_PTHREAD_MUTEX_LOCK(&rdevCb->rdevMutex);
3252 2 : RsListAddTail(&qpCb->list, &rdevCb->qpList);
3253 2 : RS_PTHREAD_MUTEX_ULOCK(&rdevCb->rdevMutex);
3254 2 : rdevCb->qpCnt++;
3255 2 : *qp = qpCb->ibQp;
3256 2 : qpResp->qpn = (unsigned int)qpCb->qpInfoLo.qpn;
3257 2 : qpResp->gidIdx = (unsigned int)qpCb->qpInfoLo.gidIdx;
3258 2 : qpResp->psn = (unsigned int)qpCb->qpInfoLo.psn;
3259 2 : qpResp->gid = qpCb->qpInfoLo.gid;
3260 :
3261 2 : hccp_info("qp %d create qp.", qpResp->qpn);
3262 :
3263 2 : return 0;
3264 :
3265 1 : create_qp_err:
3266 1 : pthread_mutex_destroy(&qpCb->qpMutex);
3267 1 : free(qpCb);
3268 1 : qpCb = NULL;
3269 1 : return ret;
3270 : }
3271 :
3272 2 : RS_ATTRI_VISI_DEF int RsNormalQpDestroy(unsigned int phyId, unsigned int rdevIndex, unsigned int qpn)
3273 : {
3274 2 : struct RsQpCb *qpCb = NULL;
3275 : int ret;
3276 :
3277 2 : RS_QP_PARA_CHECK(phyId);
3278 2 : ret = RsQpn2qpcb(phyId, rdevIndex, qpn, &qpCb);
3279 2 : CHK_PRT_RETURN(ret != 0 || qpCb == NULL, hccp_err("get qp cb failed qpn %u, ret %d", qpn, ret), ret);
3280 :
3281 2 : RsQpRelease(qpCb);
3282 :
3283 : // destroy qp
3284 2 : RsDrvQpDestroy(qpCb);
3285 :
3286 2 : qpCb->rdevCb->qpCnt--;
3287 :
3288 2 : pthread_mutex_destroy(&qpCb->qpMutex);
3289 2 : hccp_info("qp %d destroy qp, send wr[%u].", qpn, qpCb->sendWrNum);
3290 :
3291 2 : free(qpCb);
3292 2 : qpCb = NULL;
3293 2 : return ret;
3294 : }
3295 :
3296 4 : RS_ATTRI_VISI_DEF int RsCreateCompChannel(unsigned int phyId, unsigned int rdevIndex, void **compChannel)
3297 : {
3298 : int ret;
3299 : unsigned int chipId;
3300 :
3301 4 : struct RsRdevCb *rdevCb = NULL;
3302 :
3303 4 : CHK_PRT_RETURN(compChannel == NULL || phyId >= RS_MAX_DEV_NUM,
3304 : hccp_err("param err, NULL pointer or phyId:%u >= [%d]", phyId, RS_MAX_DEV_NUM), -EINVAL);
3305 :
3306 4 : ret = rsGetLocalDevIDByHostDevID(phyId, &chipId);
3307 4 : CHK_PRT_RETURN(ret,
3308 : hccp_err("rs_create_comp_channel rsGetLocalDevIDByHostDevID phyId[%u] invalid, ret %d", phyId, ret), ret);
3309 :
3310 3 : ret = RsRdev2rdevCb(chipId, rdevIndex, &rdevCb);
3311 3 : CHK_PRT_RETURN(ret != 0 || rdevCb == NULL, hccp_err("rs_rdev2rdev_cb for chip_id[%u] failed, ret %d", chipId, ret),
3312 : ret);
3313 :
3314 2 : *compChannel = (void *)RsIbvCreateCompChannel(rdevCb->ibCtx);
3315 2 : if (*compChannel == NULL) {
3316 1 : hccp_err("rs_ibv_create_comp_channel failed, errno(%d)", errno);
3317 1 : return -EOPENSRC;
3318 : }
3319 1 : hccp_info("create comp channel success!");
3320 1 : return 0;
3321 : }
3322 :
3323 2 : RS_ATTRI_VISI_DEF int RsDestroyCompChannel(void *compChannel)
3324 : {
3325 : int ret;
3326 2 : struct ibv_comp_channel *rsCompChannel = (struct ibv_comp_channel *)compChannel;
3327 :
3328 2 : ret = RsIbvDestroyCompChannel(rsCompChannel);
3329 2 : CHK_PRT_RETURN(ret, hccp_err("rs_destroy_comp_channel failed."), ret);
3330 1 : hccp_info("destroy comp channel success!");
3331 :
3332 1 : return 0;
3333 : }
3334 :
3335 5 : RS_ATTRI_VISI_DEF int RsCreateSrq(unsigned int phyId, unsigned int rdevIndex, struct SrqAttr *attr)
3336 : {
3337 : int ret;
3338 5 : struct RsRdevCb *rdevCb = NULL;
3339 5 : struct RsCqContext *cqContext = NULL;
3340 :
3341 5 : CHK_PRT_RETURN(attr == NULL || attr->context == NULL || attr->ibRecvCq == NULL || attr->ibSrq == NULL ||
3342 : phyId >= RS_MAX_DEV_NUM,
3343 : hccp_err("param err, NULL pointer or phyId:%u >= [%d]", phyId, RS_MAX_DEV_NUM), -EINVAL);
3344 :
3345 5 : ret = RsQueryRdevCb(phyId, rdevIndex, &rdevCb);
3346 5 : CHK_PRT_RETURN(ret, hccp_err("rs_query_rdev_cb phyId[%u] rdev_index[%u], ret %d", phyId, rdevIndex, ret), ret);
3347 :
3348 4 : cqContext = calloc(1, sizeof(struct RsCqContext));
3349 4 : if (cqContext == NULL) {
3350 1 : return -ENOMEM;
3351 : }
3352 :
3353 3 : cqContext->rdevCb = rdevCb;
3354 3 : cqContext->eqNum = 0;
3355 3 : cqContext->cqCreateMode = RS_SRQ_CQ_CREATE;
3356 3 : *attr->context = cqContext;
3357 :
3358 3 : struct CqAttr cqAttr = {0};
3359 3 : cqAttr.recvCqDepth = attr->cqDepth;
3360 3 : cqAttr.recvCqEventId = attr->srqEventId;
3361 3 : cqAttr.ibRecvCq = attr->ibRecvCq;
3362 : // 创建srq cq
3363 3 : ret = RsCreateCqEvent(cqContext, &cqAttr);
3364 3 : if (ret) {
3365 1 : hccp_err("rs_create_cq_event create cq failed! ret:%d", ret);
3366 1 : goto create_cq_event_err;
3367 : }
3368 2 : cqContext->ibSrqCq = *attr->ibRecvCq;
3369 :
3370 2 : struct ibv_srq_init_attr srqInitAttr = {.attr = {.max_wr = attr->srqDepth, .max_sge = attr->maxSge}};
3371 2 : hccp_info("max_wr [%u], max_sge[%u]", srqInitAttr.attr.max_wr, srqInitAttr.attr.max_sge);
3372 :
3373 : // 创建srq
3374 2 : *attr->ibSrq = RsIbvCreateSrq(rdevCb->ibPd, &srqInitAttr);
3375 2 : if (*attr->ibSrq == NULL) {
3376 1 : hccp_err("rs_ibv_create_srq failed.");
3377 1 : ret = -EOPENSRC;
3378 1 : goto create_srq_err;
3379 : }
3380 1 : hccp_info("create srq success!");
3381 :
3382 1 : return 0;
3383 1 : create_cq_event_err:
3384 2 : create_srq_err:
3385 2 : cqAttr.qpContext = attr->context;
3386 2 : RsCqDestroy(phyId, rdevIndex, &cqAttr);
3387 :
3388 2 : return ret;
3389 : }
3390 :
3391 1 : RS_ATTRI_VISI_DEF int RsDestroySrq(unsigned int phyId, unsigned int rdevIndex, struct SrqAttr *attr)
3392 : {
3393 : int ret;
3394 :
3395 1 : CHK_PRT_RETURN(*attr->context == NULL || *attr->ibSrq == NULL || phyId >= RS_MAX_DEV_NUM,
3396 : hccp_err("param err, NULL pointer or phyId:%u >= [%d]", phyId, RS_MAX_DEV_NUM), -EINVAL);
3397 :
3398 1 : struct CqAttr cqAttr = {0};
3399 1 : struct RsCqContext *cqContext = *attr->context;
3400 1 : cqAttr.qpContext = attr->context;
3401 1 : RsIbvAckCqEvents(cqContext->ibSrqCq, cqContext->numRecvCqEvents);
3402 :
3403 : // 销毁srq cq
3404 1 : ret = RsCqDestroy(phyId, rdevIndex, &cqAttr);
3405 1 : CHK_PRT_RETURN(ret, hccp_err("rs_cq_destroy destroy cq failed! ret:%d", ret), ret);
3406 :
3407 1 : ret = RsIbvDestroySrq(*attr->ibSrq);
3408 1 : CHK_PRT_RETURN(ret, hccp_err("rs_ibv_destroy_srq failed."), ret);
3409 :
3410 1 : return 0;
3411 : }
3412 :
3413 2 : RS_ATTRI_VISI_DEF int RsGetLiteSupport(unsigned int phyId, unsigned int rdevIndex, int *supportLite)
3414 : {
3415 : int ret;
3416 : unsigned int chipId;
3417 2 : struct RsRdevCb *rdevCb = NULL;
3418 :
3419 2 : RS_CHECK_POINTER_NULL_RETURN_INT(supportLite);
3420 :
3421 2 : CHK_PRT_RETURN(phyId >= RS_MAX_DEV_NUM, hccp_err("rs set param error ! phyId:%u", phyId), -EINVAL);
3422 2 : ret = rsGetLocalDevIDByHostDevID(phyId, &chipId);
3423 2 : CHK_PRT_RETURN(ret, hccp_err("phyId[%u] invalid, ret %d", phyId, ret), ret);
3424 :
3425 2 : ret = RsRdev2rdevCb(chipId, rdevIndex, &rdevCb);
3426 2 : CHK_PRT_RETURN(ret != 0 || rdevCb == NULL, hccp_err("rs_rdev2rdev_cb for chip_id[%u] failed, ret %d", chipId, ret),
3427 : ret);
3428 :
3429 2 : rdevCb->supportLite = 1;
3430 2 : *supportLite = rdevCb->supportLite;
3431 :
3432 2 : return 0;
3433 : }
3434 :
3435 2 : RS_ATTRI_VISI_DEF int RsGetLiteRdevCap(unsigned int phyId, unsigned int rdevIndex, struct LiteRdevCapResp *resp)
3436 : {
3437 : int ret;
3438 : unsigned int chipId;
3439 2 : struct RsRdevCb *rdevCb = NULL;
3440 :
3441 2 : RS_CHECK_POINTER_NULL_RETURN_INT(resp);
3442 :
3443 2 : CHK_PRT_RETURN(phyId >= RS_MAX_DEV_NUM, hccp_err("rs set param error ! phyId:%u", phyId), -EINVAL);
3444 2 : ret = rsGetLocalDevIDByHostDevID(phyId, &chipId);
3445 2 : CHK_PRT_RETURN(ret, hccp_err("phyId[%u] invalid, ret %d", phyId, ret), ret);
3446 :
3447 2 : ret = RsRdev2rdevCb(chipId, rdevIndex, &rdevCb);
3448 2 : CHK_PRT_RETURN(ret != 0 || rdevCb == NULL, hccp_err("rs_rdev2rdev_cb for chip_id[%u] failed, ret %d", chipId, ret),
3449 : ret);
3450 :
3451 2 : ret = RsIbvExpQueryDevice(rdevCb->ibCtx, &resp->cap);
3452 2 : CHK_PRT_RETURN(ret != 0, hccp_err("rs_ibv_exp_query_device for phyId[%u] failed, ret %d", phyId, ret), ret);
3453 :
3454 2 : ret = memcpy_s(resp, sizeof(struct dev_cap_info), (void *)&resp->cap, sizeof(resp->cap));
3455 2 : if (ret) {
3456 0 : hccp_err("memcpy_s failed, ret:%d, src_len:%u, dst_len:%u", ret, (unsigned int)sizeof(resp->cap),
3457 : (unsigned int)sizeof(struct dev_cap_info));
3458 0 : return ret;
3459 : }
3460 :
3461 2 : return 0;
3462 : }
3463 :
3464 2 : RS_ATTRI_VISI_DEF int RsGetLiteQpCqAttr(unsigned int phyId, unsigned int rdevIndex, unsigned int qpn,
3465 : struct LiteQpCqAttrResp *resp)
3466 : {
3467 : int ret;
3468 2 : struct RsQpCb *qpCb = NULL;
3469 :
3470 2 : RS_CHECK_POINTER_NULL_RETURN_INT(resp);
3471 :
3472 2 : RS_QP_PARA_CHECK(phyId);
3473 2 : ret = RsQpn2qpcb(phyId, rdevIndex, qpn, &qpCb);
3474 2 : CHK_PRT_RETURN(ret != 0 || qpCb == NULL, hccp_err("get qp cb failed qpn %u, ret %d", qpn, ret), ret);
3475 :
3476 2 : ret = memcpy_s(resp, sizeof(struct LiteQpCqAttrResp), (void *)&qpCb->qpResp, sizeof(qpCb->qpResp));
3477 2 : if (ret) {
3478 0 : hccp_err("memcpy_s failed, ret:%d, src_len:%u, dst_len:%u", ret, (unsigned int)sizeof(qpCb->qpResp),
3479 : (unsigned int)sizeof(struct LiteQpCqAttrResp));
3480 0 : return ret;
3481 : }
3482 :
3483 2 : return 0;
3484 : }
3485 :
3486 0 : RS_ATTRI_VISI_DEF int RsGetLiteQpAttr(unsigned int phyId, unsigned int rdevIndex, unsigned int qpn,
3487 : struct LiteQpAttrResp *resp)
3488 : {
3489 : int ret;
3490 0 : struct RsQpCb *qpCb = NULL;
3491 :
3492 0 : RS_CHECK_POINTER_NULL_RETURN_INT(resp);
3493 :
3494 0 : RS_QP_PARA_CHECK(phyId);
3495 0 : ret = RsQpn2qpcb(phyId, rdevIndex, qpn, &qpCb);
3496 0 : CHK_PRT_RETURN(ret != 0 || qpCb == NULL, hccp_err("get qp cb failed qpn %u, ret %d", qpn, ret), ret);
3497 :
3498 0 : ret = memcpy_s(resp, sizeof(struct LiteQpAttrResp), (void *)&qpCb->qpResp.qpData, sizeof(qpCb->qpResp.qpData));
3499 0 : if (ret) {
3500 0 : hccp_err("memcpy_s failed, ret:%d, src_len:%u, dst_len:%u", ret, (unsigned int)sizeof(qpCb->qpResp.qpData),
3501 : (unsigned int)sizeof(struct LiteQpAttrResp));
3502 0 : return ret;
3503 : }
3504 :
3505 0 : return 0;
3506 : }
3507 :
3508 2 : RS_ATTRI_VISI_DEF int RsGetLiteMemAttr(unsigned int phyId, unsigned int rdevIndex, unsigned int qpn,
3509 : struct LiteMemAttrResp *resp)
3510 : {
3511 : int ret;
3512 2 : struct RsQpCb *qpCb = NULL;
3513 :
3514 2 : RS_CHECK_POINTER_NULL_RETURN_INT(resp);
3515 :
3516 2 : RS_QP_PARA_CHECK(phyId);
3517 2 : ret = RsQpn2qpcb(phyId, rdevIndex, qpn, &qpCb);
3518 2 : CHK_PRT_RETURN(ret != 0 || qpCb == NULL, hccp_err("get qp cb failed qpn %u, ret %d", qpn, ret), ret);
3519 :
3520 2 : ret = memcpy_s(resp, sizeof(struct LiteMemAttrResp), (void *)&qpCb->memResp, sizeof(qpCb->memResp));
3521 2 : if (ret) {
3522 0 : hccp_err("memcpy_s failed, ret:%d, src_len:%u, dst_len:%u", ret, (unsigned int)sizeof(qpCb->memResp),
3523 : (unsigned int)sizeof(struct LiteMemAttrResp));
3524 0 : return ret;
3525 : }
3526 :
3527 2 : return 0;
3528 : }
3529 :
3530 4 : STATIC void RsGetMrInfo(struct RsQpCb *qpCb, struct LiteMrInfo *mr, uint32_t maxMrNum, struct RsListHead *mrList)
3531 : {
3532 4 : struct RsMrCb *mrTmp = NULL;
3533 4 : struct RsMrCb *mrTmp2 = NULL;
3534 4 : uint32_t i = 0;
3535 :
3536 4 : RS_PTHREAD_MUTEX_LOCK(&qpCb->qpMutex);
3537 4 : RS_LIST_GET_HEAD_ENTRY(mrTmp, mrTmp2, mrList, list, struct RsMrCb);
3538 6 : for (; (&mrTmp->list) != mrList; mrTmp = mrTmp2, mrTmp2 = list_entry(mrTmp2->list.next, struct RsMrCb, list)) {
3539 2 : if (i < maxMrNum) {
3540 2 : mr[i].key = mrTmp->mrInfo.rkey;
3541 2 : mr[i].addr = mrTmp->mrInfo.addr;
3542 2 : mr[i].len = mrTmp->mrInfo.len;
3543 2 : i++;
3544 : } else {
3545 0 : break;
3546 : }
3547 : }
3548 :
3549 4 : RS_PTHREAD_MUTEX_ULOCK(&qpCb->qpMutex);
3550 4 : }
3551 :
3552 2 : RS_ATTRI_VISI_DEF int RsGetLiteConnectedInfo(unsigned int phyId, unsigned int rdevIndex, unsigned int qpn,
3553 : struct LiteConnectedInfoResp *resp)
3554 : {
3555 : int ret;
3556 2 : struct RsQpCb *qpCb = NULL;
3557 :
3558 2 : RS_CHECK_POINTER_NULL_RETURN_INT(resp);
3559 2 : RS_QP_PARA_CHECK(phyId);
3560 2 : ret = RsQpn2qpcb(phyId, rdevIndex, qpn, &qpCb);
3561 2 : CHK_PRT_RETURN(ret != 0 || qpCb == NULL, hccp_err("get qp cb failed qpn %u, ret %d", qpn, ret), ret);
3562 :
3563 2 : resp->state = (unsigned int)qpCb->state;
3564 2 : if (resp->state == RS_QP_STATUS_CONNECTED) {
3565 2 : RsGetMrInfo(qpCb, &resp->localMr[0], RA_MR_MAX_NUM, &qpCb->mrList);
3566 2 : RsGetMrInfo(qpCb, &resp->remMr[0], RA_MR_MAX_NUM, &qpCb->remMrList);
3567 2 : resp->qosAttr.sl = qpCb->qosAttr.sl;
3568 2 : resp->qosAttr.tc = qpCb->qosAttr.tc;
3569 : }
3570 :
3571 2 : return 0;
3572 : }
|
