Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include <errno.h>
12 : #include <infiniband/verbs.h>
13 : #include "securec.h"
14 : #include "dl_hal_function.h"
15 : #include "dl_ibverbs_function.h"
16 : #include "hccp_common.h"
17 : #include "rs.h"
18 : #include "ra_rs_err.h"
19 : #include "rs_inner.h"
20 : #include "rs_epoll.h"
21 : #include "rs_drv_socket.h"
22 : #include "rs_drv_rdma.h"
23 : #include "rs_ping_inner.h"
24 : #ifndef HNS_ROCE_LLT
25 : #include <dlog_pub.h>
26 : #endif
27 : #include "rs_ping_roce.h"
28 :
29 : #define RS_PING_ROCE_RECV_WC_NUM 16
30 :
31 : struct ibv_wc gPingQpRecvWc[RS_PING_ROCE_RECV_WC_NUM] = {0};
32 : struct ibv_wc gPongQpRecvWc[RS_PING_ROCE_RECV_WC_NUM] = {0};
33 :
34 3 : STATIC bool RsPingRoceCheckFd(struct RsPingCtxCb *pingCb, int fd)
35 : {
36 3 : if (pingCb->pingQp.channel != NULL && pingCb->pingQp.channel->fd == fd) {
37 1 : hccp_dbg("ping_qp rq, channel->fd:%d poll cq", fd);
38 1 : return true;
39 : }
40 2 : return false;
41 : }
42 :
43 2 : STATIC bool RsPongRoceCheckFd(struct RsPingCtxCb *pingCb, int fd)
44 : {
45 2 : if (pingCb->pongQp.channel != NULL && pingCb->pongQp.channel->fd == fd) {
46 2 : hccp_dbg("pong_qp rq, channel->fd:%d poll cq", fd);
47 2 : return true;
48 : }
49 0 : return false;
50 : }
51 :
52 0 : STATIC int RsPingCbGetDevRdevIndex(struct RsPingCtxCb *pingCb, int index)
53 : {
54 : (void)pingCb;
55 : (void)index;
56 : #ifdef CUSTOM_INTERFACE
57 0 : struct roce_dev_data rdevData = {0};
58 : int ret;
59 :
60 0 : if (RsIsCustomInterfaceSupported()) {
61 0 : RS_PTHREAD_MUTEX_LOCK(&pingCb->pingMutex);
62 0 : pingCb->rdevCb.devName = RsIbvGetDeviceName(pingCb->rdevCb.devList[index]);
63 0 : ret = RsRoceGetRoceDevData(pingCb->rdevCb.devName, &rdevData);
64 0 : if (ret != 0) {
65 0 : hccp_err("rs_roce_get_roce_dev_data failed, ret:%d, devName:%s", ret, pingCb->rdevCb.devName);
66 0 : RS_PTHREAD_MUTEX_ULOCK(&pingCb->pingMutex);
67 0 : return ret;
68 : }
69 0 : pingCb->devIndex = rdevData.rdev_index; // rdev_index is same to port_id
70 0 : RS_PTHREAD_MUTEX_ULOCK(&pingCb->pingMutex);
71 : }
72 : #endif
73 0 : return 0;
74 : }
75 :
76 2 : STATIC int RsPingCbGetIbCtxAndIndex(struct rdev *rdevInfo, struct RsPingCtxCb *pingCb)
77 : {
78 2 : struct ibv_context *ibCtx = NULL;
79 : int ret;
80 : int i;
81 :
82 4 : for (i = 0; (i < pingCb->rdevCb.devNum) && (pingCb->rdevCb.devList[i] != NULL); ++i) {
83 3 : ibCtx = RsIbvOpenDevice(pingCb->rdevCb.devList[i]);
84 3 : CHK_PRT_RETURN(ibCtx == NULL, hccp_err("ibv_open_device failed!"), -ENODEV);
85 3 : ret = RsQueryGid(*rdevInfo, ibCtx, pingCb->rdevCb.ibPort, &pingCb->rdevCb.gidIdx);
86 3 : if (ret == 0) {
87 1 : ret = RsPingCbGetDevRdevIndex(pingCb, i);
88 1 : if (ret != 0) {
89 0 : hccp_err("rs_ping_cb_get_dev_rdev_index failed, ret:%d", ret);
90 0 : RsIbvCloseDevice(ibCtx);
91 0 : return ret;
92 : }
93 1 : pingCb->rdevCb.ibCtx = ibCtx;
94 1 : ret = RsIbvQueryGid(ibCtx, pingCb->rdevCb.ibPort, pingCb->rdevCb.gidIdx, &pingCb->rdevCb.gid);
95 1 : if (ret != 0) {
96 1 : RsIbvCloseDevice(ibCtx);
97 1 : hccp_err("query gid failed gid_idx %d, ret %d", pingCb->rdevCb.gidIdx, ret);
98 1 : return -EOPENSRC;
99 : }
100 0 : return 0;
101 2 : } else if (ret == -EEXIST) {
102 2 : RsIbvCloseDevice(ibCtx);
103 : } else {
104 0 : hccp_err("rs_query_gid failed, ret:%d", ret);
105 0 : RsIbvCloseDevice(ibCtx);
106 0 : return ret;
107 : }
108 : }
109 :
110 1 : CHK_PRT_RETURN(i == pingCb->rdevCb.devNum,
111 : hccp_err("can not find ib_ctx for phyId[%u] local_ip[0x%x] "
112 : "in dev_list!",
113 : rdevInfo->phyId, rdevInfo->localIp.addr.s_addr),
114 : -ENODEV);
115 0 : return 0;
116 : }
117 :
118 3 : STATIC int RsPingCommonModifyLocalQp(struct RsPingCtxCb *pingCb, struct RsPingLocalQpCb *qpCb)
119 : {
120 : struct ibv_qp_init_attr initAttr;
121 3 : struct ibv_qp_attr attr = {0};
122 : int ret;
123 :
124 3 : ret = RsIbvQueryQp(qpCb->ibQp, &attr, IBV_QP_STATE, &initAttr);
125 3 : CHK_PRT_RETURN(ret != 0 || attr.qp_state != IBV_QPS_RESET,
126 : hccp_err("rs_ibv_query_qp qpn:%u fail, ret:%d attr.qp_state:%d != %d", qpCb->ibQp->qp_num, ret, attr.qp_state,
127 : IBV_QPS_RESET),
128 : -EOPENSRC);
129 :
130 2 : attr.qp_state = IBV_QPS_INIT;
131 2 : attr.pkey_index = 0;
132 2 : attr.port_num = pingCb->rdevCb.ibPort;
133 2 : attr.qkey = qpCb->qkey;
134 2 : ret = RsIbvModifyQp(qpCb->ibQp, &attr, IBV_QP_STATE | IBV_QP_PKEY_INDEX | IBV_QP_PORT | IBV_QP_QKEY);
135 2 : CHK_PRT_RETURN(ret != 0,
136 : hccp_err("rs_ibv_modify_qp qpn:%u to init fail, ret:%d, errno:%d", qpCb->ibQp->qp_num, ret, errno), -EOPENSRC);
137 :
138 1 : attr.qp_state = IBV_QPS_RTR;
139 1 : ret = RsIbvModifyQp(qpCb->ibQp, &attr, IBV_QP_STATE);
140 1 : CHK_PRT_RETURN(ret != 0,
141 : hccp_err("rs_ibv_modify_qp qpn:%u to rtr fail, ret:%d, errno:%d", qpCb->ibQp->qp_num, ret, errno), -EOPENSRC);
142 :
143 1 : attr.qp_state = IBV_QPS_RTS;
144 1 : attr.sq_psn = 0;
145 1 : ret = RsIbvModifyQp(qpCb->ibQp, &attr, IBV_QP_STATE | IBV_QP_SQ_PSN);
146 1 : CHK_PRT_RETURN(ret != 0,
147 : hccp_err("rs_ibv_modify_qp qpn:%u to rts fail, ret:%d, errno:%d", qpCb->ibQp->qp_num, ret, errno), -EOPENSRC);
148 :
149 1 : return 0;
150 : }
151 :
152 2 : STATIC int RsPingCommonInitLocalQp(struct rs_cb *rscb, struct RsPingCtxCb *pingCb, union PingQpAttr *attr,
153 : struct RsPingLocalQpCb *qpCb)
154 : {
155 2 : struct ibv_exp_qp_init_attr qpInitAttr = {0};
156 2 : struct rdma_lite_device_qp_attr qpResp = {0};
157 : int randNum;
158 : int ret;
159 :
160 2 : hccp_info("cq_attr{%d %d, %d %d}", attr->rdma.cqAttr.sendCqDepth, attr->rdma.cqAttr.sendCqCompVector,
161 : attr->rdma.cqAttr.recvCqDepth, attr->rdma.cqAttr.recvCqCompVector);
162 :
163 : // create send cq with attr
164 2 : qpCb->sendCq.depth = attr->rdma.cqAttr.sendCqDepth;
165 2 : qpCb->sendCq.compVector = attr->rdma.cqAttr.sendCqCompVector;
166 2 : qpCb->sendCq.ibCq = RsIbvCreateCq(pingCb->rdevCb.ibCtx, qpCb->sendCq.depth, NULL, NULL, qpCb->sendCq.compVector);
167 2 : qpCb->sendCq.maxRecvWcNum = RS_PING_ROCE_RECV_WC_NUM;
168 2 : ret = -errno;
169 2 : CHK_PRT_RETURN(qpCb->sendCq.ibCq == NULL, hccp_err("rs_ibv_create_cq send cq fail, ret:%d", ret), ret);
170 :
171 : // create channel & create recv cq with attr
172 1 : qpCb->channel = RsIbvCreateCompChannel(pingCb->rdevCb.ibCtx);
173 1 : if (qpCb->channel == NULL) {
174 0 : ret = -errno;
175 0 : hccp_err("rs_ibv_create_comp_channel failed! ret:%d", ret);
176 0 : goto create_channel_fail;
177 : }
178 1 : ret = RsEpollCtl(rscb->connCb.epollfd, EPOLL_CTL_ADD, qpCb->channel->fd, EPOLLIN | EPOLLRDHUP);
179 1 : if (ret != 0) {
180 0 : hccp_err("RsEpollCtl failed! epollfd:%d fd:%d ret:%d", rscb->connCb.epollfd, qpCb->channel->fd, ret);
181 0 : goto epoll_ctl_fail;
182 : }
183 1 : qpCb->recvCq.depth = attr->rdma.cqAttr.recvCqDepth;
184 1 : qpCb->recvCq.compVector = attr->rdma.cqAttr.recvCqCompVector;
185 1 : qpCb->recvCq.ibCq = RsIbvCreateCq(pingCb->rdevCb.ibCtx, qpCb->recvCq.depth, NULL, qpCb->channel,
186 : qpCb->recvCq.compVector);
187 1 : qpCb->recvCq.maxRecvWcNum = RS_PING_ROCE_RECV_WC_NUM;
188 1 : if (qpCb->recvCq.ibCq == NULL) {
189 0 : ret = -errno;
190 0 : hccp_err("rs_ibv_create_cq recv cq fail, ret:%d", ret);
191 0 : goto create_rcq_fail;
192 : }
193 :
194 : // create qp with attr
195 1 : (void)RsDrvGetRandomNum(&randNum);
196 : // clear bit IB_QP_SET_QKEY to avoid modify_qp to INIT failed
197 1 : qpCb->qkey = (uint32_t)(((uint32_t)randNum) & (~(1U << 31U)));
198 1 : (void)memcpy_s(&qpCb->qpCap, sizeof(struct ibv_qp_cap), &attr->rdma.qpAttr.cap, sizeof(struct ibv_qp_cap));
199 1 : qpCb->udpSport = attr->rdma.qpAttr.udpSport;
200 1 : qpInitAttr.attr.send_cq = qpCb->sendCq.ibCq;
201 1 : qpInitAttr.attr.recv_cq = qpCb->recvCq.ibCq;
202 1 : (void)memcpy_s(&qpInitAttr.attr.cap, sizeof(struct ibv_qp_cap), &qpCb->qpCap, sizeof(struct ibv_qp_cap));
203 1 : qpInitAttr.attr.qp_type = IBV_QPT_UD;
204 1 : qpInitAttr.udp_sport = attr->rdma.qpAttr.udpSport;
205 :
206 1 : hccp_info("qkey:%u udp_sport:%u qp_cap{%u %u %u %u %u}", qpCb->qkey, qpCb->udpSport,
207 : attr->rdma.qpAttr.cap.maxSendWr, attr->rdma.qpAttr.cap.maxRecvWr, attr->rdma.qpAttr.cap.maxSendSge,
208 : attr->rdma.qpAttr.cap.maxRecvSge, attr->rdma.qpAttr.cap.maxInlineData);
209 1 : qpCb->ibQp = RsIbvExpCreateQp(pingCb->rdevCb.ibPd, &qpInitAttr, &qpResp);
210 1 : if (qpCb->ibQp == NULL) {
211 1 : ret = -errno;
212 1 : hccp_err("rs_ibv_exp_create_qp qp fail, ret:%d", ret);
213 1 : goto create_qp_fail;
214 : }
215 :
216 0 : ret = RsPingCommonModifyLocalQp(pingCb, qpCb);
217 0 : if (ret != 0) {
218 0 : hccp_err("rs_ping_common_modify_local_qp failed, ret:%d", ret);
219 0 : goto modify_qp_fail;
220 : }
221 :
222 0 : ret = RsIbvReqNotifyCq(qpCb->recvCq.ibCq, 0);
223 0 : if (ret != 0) {
224 0 : hccp_err("rs_ibv_req_notify_cq failed, ret:%d", ret);
225 0 : goto modify_qp_fail;
226 : }
227 :
228 0 : hccp_run_info("qpn:%u create success, cqAttr{%d %d, %d %d} qkey:%u udpSport:%u qpCap{%u %u %u %u %u}",
229 : qpCb->ibQp->qp_num, attr->rdma.cqAttr.sendCqDepth, attr->rdma.cqAttr.sendCqCompVector,
230 : attr->rdma.cqAttr.recvCqDepth, attr->rdma.cqAttr.recvCqCompVector, qpCb->qkey, qpCb->udpSport,
231 : attr->rdma.qpAttr.cap.maxSendWr, attr->rdma.qpAttr.cap.maxRecvWr, attr->rdma.qpAttr.cap.maxSendSge,
232 : attr->rdma.qpAttr.cap.maxRecvSge, attr->rdma.qpAttr.cap.maxInlineData);
233 :
234 0 : return 0;
235 :
236 0 : modify_qp_fail:
237 0 : (void)RsIbvDestroyQp(qpCb->ibQp);
238 1 : create_qp_fail:
239 1 : (void)RsIbvDestroyCq(qpCb->recvCq.ibCq);
240 1 : create_rcq_fail:
241 1 : (void)RsEpollCtl(rscb->connCb.epollfd, EPOLL_CTL_DEL, qpCb->channel->fd, EPOLLIN | EPOLLRDHUP);
242 1 : epoll_ctl_fail:
243 1 : (void)RsIbvDestroyCompChannel(qpCb->channel);
244 1 : create_channel_fail:
245 1 : (void)RsIbvDestroyCq(qpCb->sendCq.ibCq);
246 1 : return ret;
247 : }
248 :
249 0 : STATIC int RsPingCommonInitMrCb(struct rs_cb *rscb, struct RsPingCtxCb *pingCb, struct RsPingMrCb *mrCb)
250 : {
251 0 : unsigned long flag = 0;
252 0 : uint32_t idx = 0;
253 : int ret;
254 :
255 0 : hccp_info("payload_offset:%u len:0x%llx sge_num:%u grp_id:%u", mrCb->payloadOffset, mrCb->len, mrCb->sgeNum,
256 : rscb->grpId);
257 :
258 0 : ret = pthread_mutex_init(&mrCb->mutex, NULL);
259 0 : CHK_PRT_RETURN(ret != 0, hccp_err("pthread_mutex_init mr_cb mutex failed, ret:%d", ret), ret);
260 :
261 0 : flag = ((unsigned long)pingCb->logicDevid << BUFF_FLAGS_DEVID_OFFSET) | BUFF_SP_SVM;
262 0 : ret = DlHalBuffAllocAlignEx(mrCb->len, (unsigned int)RA_RS_4K_PAGE_SIZE, flag, (int)rscb->grpId,
263 0 : (void **)&mrCb->addr);
264 0 : if (ret != 0) {
265 0 : hccp_err("DlHalBuffAllocAlignEx failed, length:0x%llx, dev_id:0x%x, flag:0x%lx, grpId:%u, ret:%d", mrCb->len,
266 : pingCb->logicDevid, flag, rscb->grpId, ret);
267 0 : goto alloc_fail;
268 : }
269 :
270 0 : mrCb->ibMr = RsDrvMrReg(pingCb->rdevCb.ibPd, (char *)(uintptr_t)mrCb->addr, mrCb->len,
271 : IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE | IBV_ACCESS_REMOTE_READ);
272 0 : if (mrCb->ibMr == NULL) {
273 0 : ret = -errno;
274 0 : hccp_err("rs_ibv_reg_mr fail, ret:%d addr:0x%llx len:0x%llx", ret, mrCb->addr, mrCb->len);
275 0 : goto mr_reg_fail;
276 : }
277 :
278 : // init sge list
279 0 : mrCb->sgeList = calloc(mrCb->sgeNum, sizeof(struct ibv_sge));
280 0 : if (mrCb->sgeList == NULL) {
281 0 : ret = -errno;
282 0 : hccp_err("calloc fail, ret:%d sgeNum:%u", ret, mrCb->sgeNum);
283 0 : goto calloc_fail;
284 : }
285 0 : for (idx = 0; idx < mrCb->sgeNum; idx++) {
286 0 : mrCb->sgeList[idx].lkey = mrCb->ibMr->lkey;
287 0 : mrCb->sgeList[idx].length = mrCb->payloadOffset;
288 0 : if (idx == 0) {
289 0 : mrCb->sgeList[idx].addr = mrCb->addr;
290 : } else {
291 0 : mrCb->sgeList[idx].addr = mrCb->sgeList[idx - 1].addr + mrCb->payloadOffset;
292 : }
293 : }
294 0 : mrCb->sgeIdx = 0;
295 :
296 0 : hccp_info("addr:0x%llx lkey:%u ", mrCb->addr, mrCb->ibMr->lkey);
297 :
298 0 : return 0;
299 :
300 0 : calloc_fail:
301 0 : (void)RsDrvMrDereg(mrCb->ibMr);
302 0 : mr_reg_fail:
303 0 : (void)DlHalBuffFree((void *)(uintptr_t)mrCb->addr);
304 0 : alloc_fail:
305 0 : (void)pthread_mutex_destroy(&mrCb->mutex);
306 0 : return ret;
307 : }
308 :
309 4 : STATIC void RsPingCommonDeinitMrCb(struct RsPingMrCb *mrCb)
310 : {
311 4 : hccp_dbg("addr:0x%llx len:%llu", mrCb->addr, mrCb->len);
312 4 : free(mrCb->sgeList);
313 4 : mrCb->sgeList = NULL;
314 4 : (void)RsDrvMrDereg(mrCb->ibMr);
315 4 : (void)DlHalBuffFree((void *)(uintptr_t)mrCb->addr);
316 4 : (void)pthread_mutex_destroy(&mrCb->mutex);
317 4 : }
318 :
319 2 : STATIC int RsPingPongInitLocalBuffer(struct rs_cb *rscb, struct PingInitAttr *attr, struct PingInitInfo *info,
320 : struct RsPingCtxCb *pingCb)
321 : {
322 : int ret;
323 :
324 : // prepare ping_qp send mr
325 2 : pingCb->pingQp.sendMrCb.payloadOffset = PING_TOTAL_PAYLOAD_MAX_SIZE;
326 2 : pingCb->pingQp.sendMrCb.len = pingCb->pingQp.qpCap.max_send_wr * pingCb->pingQp.sendMrCb.payloadOffset;
327 2 : pingCb->pingQp.sendMrCb.sgeNum = pingCb->pingQp.qpCap.max_send_wr;
328 2 : ret = RsPingCommonInitMrCb(rscb, pingCb, &pingCb->pingQp.sendMrCb);
329 2 : CHK_PRT_RETURN(ret != 0, hccp_err("rs_ping_common_init_mr_cb ping_qp send_mr_cb failed, ret %d", ret), ret);
330 : // prepare ping_qp recv mr
331 1 : pingCb->pingQp.recvMrCb.payloadOffset = PING_TOTAL_PAYLOAD_MAX_SIZE;
332 1 : pingCb->pingQp.recvMrCb.len = pingCb->pingQp.qpCap.max_recv_wr * pingCb->pingQp.recvMrCb.payloadOffset;
333 1 : pingCb->pingQp.recvMrCb.sgeNum = pingCb->pingQp.qpCap.max_recv_wr;
334 1 : ret = RsPingCommonInitMrCb(rscb, pingCb, &pingCb->pingQp.recvMrCb);
335 1 : if (ret != 0) {
336 0 : hccp_err("rs_ping_common_init_mr_cb ping_qp recv_mr_cb failed, ret %d", ret);
337 0 : goto init_ping_qp_recv_mr_fail;
338 : }
339 :
340 : // prepare pong_qp send mr
341 1 : pingCb->pongQp.sendMrCb.payloadOffset = PING_TOTAL_PAYLOAD_MAX_SIZE;
342 1 : pingCb->pongQp.sendMrCb.len = pingCb->pongQp.qpCap.max_send_wr * pingCb->pongQp.sendMrCb.payloadOffset;
343 1 : pingCb->pongQp.sendMrCb.sgeNum = pingCb->pongQp.qpCap.max_send_wr;
344 1 : ret = RsPingCommonInitMrCb(rscb, pingCb, &pingCb->pongQp.sendMrCb);
345 1 : if (ret != 0) {
346 0 : hccp_err("rs_ping_common_init_mr_cb pong_qp send_mr_cb failed, ret %d", ret);
347 0 : goto init_pong_qp_send_mr_fail;
348 : }
349 : // prepare pong_qp recv mr
350 1 : pingCb->pongQp.recvMrCb.payloadOffset = PING_TOTAL_PAYLOAD_MAX_SIZE;
351 1 : pingCb->pongQp.recvMrCb.len = attr->bufferSize;
352 1 : pingCb->pongQp.recvMrCb.sgeNum = attr->bufferSize / pingCb->pongQp.recvMrCb.payloadOffset;
353 1 : ret = RsPingCommonInitMrCb(rscb, pingCb, &pingCb->pongQp.recvMrCb);
354 1 : if (ret != 0) {
355 0 : hccp_err("rs_ping_common_init_mr_cb pong_qp recv_mr_cb failed, ret %d", ret);
356 0 : goto init_pong_qp_recv_mr_fail;
357 : }
358 1 : info->result.bufferVa = pingCb->pongQp.recvMrCb.addr;
359 1 : info->result.bufferSize = attr->bufferSize;
360 1 : info->result.payloadOffset = pingCb->pongQp.recvMrCb.payloadOffset;
361 1 : info->result.headerSize = RS_PING_PAYLOAD_HEADER_RESV_GRH + RS_PING_PAYLOAD_HEADER_RESV_CUSTOM;
362 :
363 1 : return 0;
364 :
365 0 : init_pong_qp_recv_mr_fail:
366 0 : RsPingCommonDeinitMrCb(&pingCb->pongQp.sendMrCb);
367 0 : init_pong_qp_send_mr_fail:
368 0 : RsPingCommonDeinitMrCb(&pingCb->pingQp.recvMrCb);
369 0 : init_ping_qp_recv_mr_fail:
370 0 : RsPingCommonDeinitMrCb(&pingCb->pingQp.sendMrCb);
371 0 : return ret;
372 : }
373 :
374 2 : STATIC int RsPingCommonPostRecv(struct RsPingLocalQpCb *qpCb)
375 : {
376 2 : struct ibv_recv_wr *badWr = NULL;
377 2 : struct ibv_recv_wr wr = {0};
378 2 : struct ibv_sge list = {0};
379 : uint32_t sgeIdx;
380 : int ret;
381 :
382 2 : RS_PTHREAD_MUTEX_LOCK(&qpCb->recvMrCb.mutex);
383 2 : sgeIdx = qpCb->recvMrCb.sgeIdx;
384 2 : (void)memcpy_s(&list, sizeof(struct ibv_sge), &qpCb->recvMrCb.sgeList[sgeIdx], sizeof(struct ibv_sge));
385 2 : qpCb->recvMrCb.sgeIdx = (sgeIdx + 1) % qpCb->recvMrCb.sgeNum;
386 2 : RS_PTHREAD_MUTEX_ULOCK(&qpCb->recvMrCb.mutex);
387 :
388 2 : wr.wr_id = (uint64_t)sgeIdx;
389 2 : wr.next = NULL;
390 2 : wr.sg_list = &list;
391 2 : wr.num_sge = 1;
392 :
393 2 : ret = RsIbvPostRecv(qpCb->ibQp, &wr, &badWr);
394 2 : if (ret != 0) {
395 1 : hccp_err("rs_ibv_post_recv failed, ret:%d", ret);
396 1 : return ret;
397 : }
398 :
399 1 : return 0;
400 : }
401 :
402 2 : STATIC int RsPingCommonInitPostRecvAll(struct RsPingLocalQpCb *qpCb)
403 : {
404 2 : int ret = 0;
405 : uint32_t i;
406 :
407 : // prepare RQ wqe
408 3 : for (i = qpCb->recvMrCb.sgeIdx; i < qpCb->recvMrCb.sgeNum && i < qpCb->qpCap.max_recv_wr; i++) {
409 2 : ret = RsPingCommonPostRecv(qpCb);
410 2 : if (ret != 0) {
411 1 : hccp_err("rs_ping_common_post_recv %u-th rqe failed, ret:%d", i, ret);
412 1 : break;
413 : }
414 : }
415 :
416 2 : return ret;
417 : }
418 :
419 1 : STATIC void RsPingCommonDeinitLocalBuffer(struct RsPingCtxCb *pingCb)
420 : {
421 1 : RsPingCommonDeinitMrCb(&pingCb->pongQp.recvMrCb);
422 1 : RsPingCommonDeinitMrCb(&pingCb->pongQp.sendMrCb);
423 1 : RsPingCommonDeinitMrCb(&pingCb->pingQp.recvMrCb);
424 1 : RsPingCommonDeinitMrCb(&pingCb->pingQp.sendMrCb);
425 1 : }
426 :
427 2 : STATIC void RsPingCommonDeinitLocalQp(struct rs_cb *rscb, struct RsPingCtxCb *pingCb, struct RsPingLocalQpCb *qpCb)
428 : {
429 : (void)pingCb;
430 2 : if (qpCb == NULL || qpCb->channel == NULL) {
431 1 : hccp_err("qp_cb is NULL or qp_cb->channel is NULL");
432 1 : return;
433 : }
434 :
435 1 : (void)RsIbvDestroyQp(qpCb->ibQp);
436 1 : RsIbvAckCqEvents(qpCb->recvCq.ibCq, qpCb->recvCq.numEvents);
437 1 : qpCb->recvCq.numEvents = 0;
438 1 : (void)RsIbvDestroyCq(qpCb->recvCq.ibCq);
439 1 : (void)RsEpollCtl(rscb->connCb.epollfd, EPOLL_CTL_DEL, qpCb->channel->fd, EPOLLIN | EPOLLRDHUP);
440 1 : (void)RsIbvDestroyCompChannel(qpCb->channel);
441 1 : qpCb->channel = NULL;
442 1 : (void)RsIbvDestroyCq(qpCb->sendCq.ibCq);
443 : }
444 :
445 4 : STATIC int RsPingPongInitLocalInfo(struct rs_cb *rscb, struct PingInitAttr *attr, struct PingInitInfo *info,
446 : struct RsPingCtxCb *pingCb)
447 : {
448 : int ret;
449 :
450 4 : ret = RsPingCommonInitLocalQp(rscb, pingCb, &attr->client, &pingCb->pingQp);
451 4 : CHK_PRT_RETURN(ret != 0, hccp_err("init ping_qp failed, ret:%d", ret), ret);
452 3 : info->client.version = 0;
453 3 : (void)memcpy_s(&info->client.rdma.gid, sizeof(union HccpGid), &pingCb->rdevCb.gid, sizeof(union ibv_gid));
454 3 : info->client.rdma.qpn = pingCb->pingQp.ibQp->qp_num;
455 3 : info->client.rdma.qkey = pingCb->pingQp.qkey;
456 :
457 3 : ret = RsPingCommonInitLocalQp(rscb, pingCb, &attr->server, &pingCb->pongQp);
458 3 : if (ret != 0) {
459 0 : hccp_err("init pong_qp failed, ret:%d", ret);
460 0 : goto init_pong_qp_fail;
461 : }
462 3 : info->server.version = 0;
463 3 : (void)memcpy_s(&info->server.rdma.gid, sizeof(union HccpGid), &pingCb->rdevCb.gid, sizeof(union ibv_gid));
464 3 : info->server.rdma.qpn = pingCb->pongQp.ibQp->qp_num;
465 3 : info->server.rdma.qkey = pingCb->pongQp.qkey;
466 :
467 3 : ret = RsPingPongInitLocalBuffer(rscb, attr, info, pingCb);
468 3 : if (ret != 0) {
469 1 : hccp_err("init buffer failed, ret:%d", ret);
470 1 : goto init_buffer_fail;
471 : }
472 :
473 2 : ret = RsPingCommonInitPostRecvAll(&pingCb->pingQp);
474 2 : if (ret != 0) {
475 1 : hccp_err("ping_qp post recv failed, ret:%d", ret);
476 1 : goto post_recv_fail;
477 : }
478 1 : ret = RsPingCommonInitPostRecvAll(&pingCb->pongQp);
479 1 : if (ret != 0) {
480 0 : hccp_err("pong_qp post recv failed, ret:%d", ret);
481 0 : goto post_recv_fail;
482 : }
483 :
484 1 : return 0;
485 :
486 1 : post_recv_fail:
487 1 : RsPingCommonDeinitLocalBuffer(pingCb);
488 2 : init_buffer_fail:
489 2 : RsPingCommonDeinitLocalQp(rscb, pingCb, &pingCb->pongQp);
490 2 : init_pong_qp_fail:
491 2 : RsPingCommonDeinitLocalQp(rscb, pingCb, &pingCb->pingQp);
492 2 : return ret;
493 : }
494 :
495 1 : STATIC int RsPingRocePingCbInit(unsigned int phyId, struct PingInitAttr *attr, struct PingInitInfo *info,
496 : unsigned int *devIndex, struct RsPingCtxCb *pingCb)
497 : {
498 1 : struct rdev *rdevInfo = &attr->dev.rdma;
499 1 : struct rs_cb *rscb = NULL;
500 : int ret;
501 :
502 1 : ret = RsGetRsCb(phyId, &rscb);
503 1 : CHK_PRT_RETURN(ret != 0, hccp_err("RsGetRsCb failed, phyId[%u] invalid, ret %d", phyId, ret), ret);
504 :
505 : // prepare input attr
506 1 : pingCb->rdevCb.ip.family = (uint32_t)rdevInfo->family;
507 1 : pingCb->rdevCb.ip.binAddr = rdevInfo->localIp;
508 1 : ret = RsInetNtop(rdevInfo->family, &rdevInfo->localIp, pingCb->rdevCb.ip.readAddr, RS_MAX_IP_LEN);
509 1 : CHK_PRT_RETURN(ret != 0, hccp_err("rs_inet_ntop failed, ret %d", ret), -EINVAL);
510 1 : (void)memcpy_s(&pingCb->commInfo, sizeof(struct PingLocalCommInfo), &attr->commInfo,
511 : sizeof(struct PingLocalCommInfo));
512 :
513 : // open device & alloc global pd
514 1 : pingCb->rdevCb.devList = RsIbvGetDeviceList(&pingCb->rdevCb.devNum);
515 1 : if (pingCb->rdevCb.devList == NULL || pingCb->rdevCb.devNum == 0) {
516 0 : hccp_err("dev_list is NULL or dev_num[%d] is 0", pingCb->rdevCb.devNum);
517 0 : ret = -ENODEV;
518 0 : goto get_device_list_fail;
519 : }
520 :
521 1 : pingCb->rdevCb.ibPort = RS_PORT_DEF;
522 1 : ret = RsPingCbGetIbCtxAndIndex(rdevInfo, pingCb);
523 1 : if (ret != 0) {
524 1 : hccp_err("rs_ping_cb_get_ib_ctx_and_index failed, ret:%d", ret);
525 1 : goto get_ib_ctx_and_index_fail;
526 : }
527 :
528 0 : pingCb->rdevCb.ibPd = RsIbvAllocPd(pingCb->rdevCb.ibCtx);
529 0 : if (pingCb->rdevCb.ibPd == NULL) {
530 0 : hccp_err("rs_ibv_alloc_pd failed, errno:%d", errno);
531 0 : ret = -ENOMEM;
532 0 : goto alloc_pd_fail;
533 : }
534 :
535 : // init cq & qp & mr info, prepare output info
536 0 : info->version = 0;
537 0 : ret = RsPingPongInitLocalInfo(rscb, attr, info, pingCb);
538 0 : if (ret != 0) {
539 0 : hccp_err("rs_ping_pong_init_local_info failed, ret=%d phyId:%u", ret, rdevInfo->phyId);
540 0 : goto init_local_info_fail;
541 : }
542 :
543 0 : *devIndex = pingCb->devIndex;
544 0 : return 0;
545 :
546 0 : init_local_info_fail:
547 0 : (void)RsIbvDeallocPd(pingCb->rdevCb.ibPd);
548 0 : alloc_pd_fail:
549 0 : (void)RsIbvCloseDevice(pingCb->rdevCb.ibCtx);
550 1 : get_ib_ctx_and_index_fail:
551 1 : RsIbvFreeDeviceList(pingCb->rdevCb.devList);
552 1 : get_device_list_fail:
553 1 : (void)pthread_mutex_destroy(&pingCb->pingMutex);
554 1 : (void)pthread_mutex_destroy(&pingCb->pongMutex);
555 1 : return ret;
556 : }
557 :
558 6 : STATIC bool RsPingCommonCompareRdmaInfo(struct PingQpInfo *a, struct PingQpInfo *b)
559 : {
560 6 : if (a->rdma.qpn != b->rdma.qpn) {
561 1 : return false;
562 : }
563 5 : if (a->rdma.qkey != b->rdma.qkey) {
564 1 : return false;
565 : }
566 4 : if (memcmp(&a->rdma.gid, &b->rdma.gid, sizeof(union HccpGid)) != 0) {
567 1 : return false;
568 : }
569 3 : return true;
570 : }
571 :
572 3 : STATIC int RsPingRoceFindTargetNode(struct RsPingCtxCb *pingCb, struct PingQpInfo *target,
573 : struct RsPingTargetInfo **node)
574 : {
575 3 : struct RsPingTargetInfo *targetNext = NULL;
576 3 : struct RsPingTargetInfo *targetCurr = NULL;
577 :
578 3 : RS_PTHREAD_MUTEX_LOCK(&pingCb->pingMutex);
579 3 : RS_LIST_GET_HEAD_ENTRY(targetCurr, targetNext, &pingCb->pingList, list, struct RsPingTargetInfo);
580 3 : for (; (&targetCurr->list) != &pingCb->pingList;
581 0 : targetCurr = targetNext, targetNext = list_entry(targetNext->list.next, struct RsPingTargetInfo, list)) {
582 1 : if (RsPingCommonCompareRdmaInfo(&targetCurr->qpInfo, target)) {
583 1 : *node = targetCurr;
584 1 : RS_PTHREAD_MUTEX_ULOCK(&pingCb->pingMutex);
585 1 : return 0;
586 : }
587 : }
588 2 : RS_PTHREAD_MUTEX_ULOCK(&pingCb->pingMutex);
589 :
590 2 : hccp_info("ping target node for qpn:%u gid:%016llx:%016llx not found", target->rdma.qpn,
591 : target->rdma.gid.global.subnetPrefix, target->rdma.gid.global.interfaceId);
592 2 : return -ENODEV;
593 : }
594 :
595 0 : STATIC int RsPingCommonCreateAh(struct RsPingCtxCb *pingCb, struct PingLocalCommInfo *localInfo,
596 : struct PingQpInfo *remoteInfo, struct ibv_ah **ah)
597 : {
598 0 : struct ibv_exp_ah_attr attrx = {0};
599 0 : struct ibv_global_route grh = {0};
600 0 : struct ibv_ah_attr attr = {0};
601 0 : struct ibv_ah *ahTmp = NULL;
602 0 : int ret = 0;
603 :
604 0 : (void)memcpy_s(&grh.dgid, sizeof(union ibv_gid), &remoteInfo->rdma.gid, sizeof(union HccpGid));
605 0 : grh.flow_label = localInfo->rdma.flowLabel;
606 0 : grh.sgid_index = (uint8_t)pingCb->rdevCb.gidIdx;
607 0 : grh.hop_limit = localInfo->rdma.hopLimit;
608 0 : grh.traffic_class = localInfo->rdma.qosAttr.tc;
609 :
610 0 : attr.grh = grh;
611 0 : attr.sl = localInfo->rdma.qosAttr.sl;
612 0 : attr.is_global = 1;
613 0 : attr.port_num = pingCb->rdevCb.ibPort;
614 0 : attrx.attr = attr;
615 0 : attrx.udp_sport = localInfo->rdma.udpSport;
616 :
617 0 : hccp_dbg("remote_qpn:%u flow_label:%u sgid_index:%u hop_limit:%u traffic_class:%u sl:%u is_global:%u "
618 : "port_num:%u udp_sport:%u",
619 : remoteInfo->rdma.qpn, grh.flow_label, grh.sgid_index, grh.hop_limit, grh.traffic_class, attr.sl, attr.is_global,
620 : attr.port_num, attrx.udp_sport);
621 :
622 0 : ahTmp = RsIbvExpCreateAh(pingCb->rdevCb.ibPd, &attrx);
623 0 : if (ahTmp == NULL) {
624 0 : ret = -EFAULT;
625 0 : hccp_err("rs_ibv_exp_create_ah failed, errno:%d", errno);
626 0 : return ret;
627 : }
628 :
629 0 : *ah = ahTmp;
630 0 : return ret;
631 : }
632 :
633 1 : STATIC int RsPingRoceAllocTargetNode(struct RsPingCtxCb *pingCb, struct PingTargetInfo *target,
634 : struct RsPingTargetInfo **node)
635 : {
636 1 : struct RsPingTargetInfo *targetInfo = NULL;
637 : int ret;
638 :
639 1 : targetInfo = (struct RsPingTargetInfo *)calloc(1, sizeof(struct RsPingTargetInfo));
640 1 : CHK_PRT_RETURN(targetInfo == NULL, hccp_err("calloc target_info fail! errno:%d", errno), -ENOMEM);
641 :
642 1 : ret = pthread_mutex_init(&targetInfo->tripMutex, NULL);
643 1 : if (ret != 0) {
644 0 : hccp_err("pthread_mutex_init trip_mutex failed, ret:%d", ret);
645 0 : goto free_target_info;
646 : }
647 :
648 1 : targetInfo->payloadSize = target->payload.size;
649 1 : if (target->payload.size > 0) {
650 1 : targetInfo->payloadBuffer = (char *)calloc(1, target->payload.size);
651 1 : if (targetInfo->payloadBuffer == NULL) {
652 0 : hccp_err("calloc payload_buffer fail! size:%u errno:%d", target->payload.size, errno);
653 0 : ret = -ENOMEM;
654 0 : goto free_trip_mutex;
655 : }
656 1 : (void)memcpy_s(targetInfo->payloadBuffer, target->payload.size, target->payload.buffer, target->payload.size);
657 : }
658 :
659 1 : (void)memcpy_s(&targetInfo->qpInfo, sizeof(struct PingQpInfo), &target->remoteInfo.qpInfo,
660 : sizeof(struct PingQpInfo));
661 1 : ret = RsPingCommonCreateAh(pingCb, &target->localInfo, &target->remoteInfo.qpInfo, &targetInfo->ah);
662 1 : if (ret != 0) {
663 0 : hccp_err("rs_ping_common_create_ah fail! ret:%d", ret);
664 0 : goto free_payload_buffer;
665 : }
666 :
667 1 : targetInfo->resultSummary.rttMin = ~0;
668 1 : targetInfo->state = RS_PING_PONG_TARGET_READY;
669 1 : *node = targetInfo;
670 :
671 1 : return 0;
672 0 : free_payload_buffer:
673 0 : if (target->payload.size > 0 && targetInfo->payloadBuffer != NULL) {
674 0 : free(targetInfo->payloadBuffer);
675 0 : targetInfo->payloadBuffer = NULL;
676 : }
677 0 : free_trip_mutex:
678 0 : (void)pthread_mutex_destroy(&targetInfo->tripMutex);
679 0 : free_target_info:
680 0 : free(targetInfo);
681 0 : targetInfo = NULL;
682 0 : return ret;
683 : }
684 :
685 1 : STATIC void RsPingRoceResetRecvBuffer(struct RsPingCtxCb *pingCb)
686 : {
687 1 : RS_PTHREAD_MUTEX_LOCK(&pingCb->pongQp.recvMrCb.mutex);
688 1 : (void)memset_s((void *)(uintptr_t)pingCb->pongQp.recvMrCb.addr, pingCb->pongQp.recvMrCb.len, 0,
689 : pingCb->pongQp.recvMrCb.len);
690 1 : RS_PTHREAD_MUTEX_ULOCK(&pingCb->pongQp.recvMrCb.mutex);
691 1 : }
692 :
693 2 : STATIC void RsPingQpBuildUpWr(struct RsPingTargetInfo *target, struct ibv_sge *list, struct ibv_send_wr *wr)
694 : {
695 2 : wr->wr_id = target->uuid;
696 2 : wr->next = NULL;
697 2 : wr->sg_list = list;
698 2 : wr->num_sge = 1;
699 2 : wr->opcode = IBV_WR_SEND;
700 2 : wr->send_flags = IBV_SEND_SIGNALED;
701 2 : wr->wr.ud.ah = target->ah;
702 2 : wr->wr.ud.remote_qpn = target->qpInfo.rdma.qpn;
703 2 : wr->wr.ud.remote_qkey = target->qpInfo.rdma.qkey;
704 2 : }
705 :
706 2 : STATIC int RsPingRocePostSend(struct RsPingCtxCb *pingCb, struct RsPingTargetInfo *target)
707 : {
708 2 : struct RsPingPayloadHeader *header = NULL;
709 2 : struct ibv_send_wr *badWr = NULL;
710 2 : struct timeval timestamp = {0};
711 2 : struct ibv_send_wr wr = {0};
712 2 : struct ibv_sge list = {0};
713 : uint32_t sgeIdx;
714 2 : int ret = 0;
715 :
716 2 : hccp_dbg("target uuid:0x%llx state:%d payload_size:%u qpn:%u gid:%016llx:%016llx", target->uuid, target->state,
717 : target->payloadSize, target->qpInfo.rdma.qpn, target->qpInfo.rdma.gid.global.subnetPrefix,
718 : target->qpInfo.rdma.gid.global.interfaceId);
719 :
720 2 : RS_PTHREAD_MUTEX_LOCK(&pingCb->pingQp.sendMrCb.mutex);
721 2 : sgeIdx = pingCb->pingQp.sendMrCb.sgeIdx;
722 2 : (void)memcpy_s(&list, sizeof(struct ibv_sge), &pingCb->pingQp.sendMrCb.sgeList[sgeIdx], sizeof(struct ibv_sge));
723 2 : pingCb->pingQp.sendMrCb.sgeIdx = (sgeIdx + 1) % pingCb->pingQp.sendMrCb.sgeNum;
724 2 : RS_PTHREAD_MUTEX_ULOCK(&pingCb->pingQp.sendMrCb.mutex);
725 :
726 : // prepare ping_qp send buffer
727 2 : (void)memset_s((void *)(uintptr_t)list.addr, list.length, 0, list.length);
728 2 : header = (struct RsPingPayloadHeader *)(uintptr_t)list.addr;
729 2 : header->type = RS_PING_TYPE_ROCE_DETECT;
730 2 : (void)memcpy_s(&header->server.rdma.gid, sizeof(union HccpGid), &pingCb->rdevCb.gid, sizeof(union ibv_gid));
731 2 : header->server.rdma.qpn = pingCb->pongQp.ibQp->qp_num;
732 2 : header->server.rdma.qkey = pingCb->pongQp.qkey;
733 2 : (void)memcpy_s(&header->target, sizeof(struct PingQpInfo), &target->qpInfo, sizeof(struct PingQpInfo));
734 :
735 2 : if (target->payloadSize > 0) {
736 2 : ret = memcpy_s((void *)(uintptr_t)(list.addr + RS_PING_PAYLOAD_HEADER_RESV_CUSTOM),
737 2 : (list.length - RS_PING_PAYLOAD_HEADER_RESV_CUSTOM), (void *)target->payloadBuffer, target->payloadSize);
738 2 : CHK_PRT_RETURN(ret != 0,
739 : hccp_err("memcpy_s buffer payload_size:%u list.length:%u failed, ret:%d", target->payloadSize,
740 : (list.length - RS_PING_PAYLOAD_HEADER_RESV_CUSTOM), ret),
741 : -ESAFEFUNC);
742 : }
743 2 : list.length = RS_PING_PAYLOAD_HEADER_RESV_CUSTOM + target->payloadSize;
744 :
745 2 : RsPingQpBuildUpWr(target, &list, &wr);
746 :
747 : // record timestamp t1
748 2 : (void)gettimeofday(×tamp, NULL);
749 2 : header->timestamp.tvSec1 = (uint64_t)timestamp.tv_sec;
750 2 : header->timestamp.tvUsec1 = (uint64_t)timestamp.tv_usec;
751 2 : header->taskId = pingCb->taskId;
752 2 : header->magic = 0x55AA;
753 :
754 2 : ret = RsIbvPostSend(pingCb->pingQp.ibQp, &wr, &badWr);
755 2 : if (ret != 0) {
756 1 : hccp_err("rs_ibv_post_send qpn:%u failed, ret:%d", pingCb->pingQp.ibQp->qp_num, ret);
757 1 : RS_PTHREAD_MUTEX_LOCK(&target->tripMutex);
758 1 : target->state = RS_PING_PONG_TARGET_ERROR;
759 1 : RS_PTHREAD_MUTEX_ULOCK(&target->tripMutex);
760 : }
761 2 : return ret;
762 : }
763 :
764 3 : STATIC int RsPingRocePollScq(struct RsPingCtxCb *pingCb, struct RsPingTargetInfo *target)
765 : {
766 3 : struct ibv_wc wc = {0};
767 : int polledCnt;
768 :
769 3 : polledCnt = RsIbvPollCq(pingCb->pingQp.sendCq.ibCq, 1, &wc);
770 3 : if (polledCnt != 1) {
771 1 : hccp_err("uuid:0x%llx rs_ibv_poll_cq polled_cnt:%d", target->uuid, polledCnt);
772 1 : target->state = RS_PING_PONG_TARGET_ERROR;
773 1 : return -ENODATA;
774 : }
775 2 : if (wc.status != IBV_WC_SUCCESS) {
776 1 : target->state = RS_PING_PONG_TARGET_ERROR;
777 1 : hccp_err("wr_id:0x%llx error cqe %s(%d)", wc.wr_id, RsIbvWcStatusStr(wc.status), wc.status);
778 1 : return -EOPENSRC;
779 : }
780 1 : return 0;
781 : }
782 :
783 6 : STATIC int RsPingRocePollRcq(struct RsPingCtxCb *pingCb, int *polledCnt, struct timeval *timestamp2)
784 : {
785 6 : struct ibv_cq *evCq = NULL;
786 6 : void *evCtx = NULL;
787 : int ret;
788 :
789 : // record timestamp t2
790 6 : (void)gettimeofday(timestamp2, NULL);
791 :
792 6 : ret = RsIbvGetCqEvent(pingCb->pingQp.channel, &evCq, &evCtx);
793 6 : if (ret != 0) {
794 1 : hccp_err("rs_ibv_get_cq_event ping_qp.channel failed, ret:%d", ret);
795 1 : return -EOPENSRC;
796 : }
797 :
798 5 : if (evCq != pingCb->pingQp.recvCq.ibCq) {
799 0 : hccp_err("CQ event for unknown CQ");
800 0 : return -EOPENSRC;
801 : }
802 5 : pingCb->pingQp.recvCq.numEvents++;
803 :
804 5 : *polledCnt = RsIbvPollCq(evCq, pingCb->pingQp.recvCq.maxRecvWcNum, gPingQpRecvWc);
805 5 : CHK_PRT_RETURN(*polledCnt > pingCb->pingQp.recvCq.maxRecvWcNum || *polledCnt < 0,
806 : hccp_err("ping_poll_rcq failed, ret:%d", *polledCnt), -EOPENSRC);
807 :
808 4 : return 0;
809 : }
810 :
811 2 : STATIC int RsPingCommonPollScq(struct RsPingLocalQpCb *qpCb)
812 : {
813 2 : struct ibv_wc wc = {0};
814 : int polledCnt;
815 :
816 2 : polledCnt = RsIbvPollCq(qpCb->sendCq.ibCq, 1, &wc);
817 2 : if (polledCnt < 0) {
818 1 : hccp_warn("rs_ibv_poll_cq unsuccessful, polledCnt:%d", polledCnt);
819 1 : } else if (polledCnt > 0) {
820 1 : if (wc.status != IBV_WC_SUCCESS) {
821 0 : hccp_err("wr_id:0x%llx error cqe %s(%d)", wc.wr_id, RsIbvWcStatusStr(wc.status), wc.status);
822 0 : return -EOPENSRC;
823 : }
824 : }
825 :
826 2 : return 0;
827 : }
828 :
829 2 : STATIC int RsPongFindTargetNode(struct RsPingCtxCb *pingCb, struct PingQpInfo *target, struct RsPongTargetInfo **node)
830 : {
831 2 : struct RsPongTargetInfo *targetNext = NULL;
832 2 : struct RsPongTargetInfo *targetCurr = NULL;
833 :
834 2 : RS_CHECK_POINTER_NULL_WITH_RET(pingCb);
835 2 : RS_PTHREAD_MUTEX_LOCK(&pingCb->pongMutex);
836 2 : RS_LIST_GET_HEAD_ENTRY(targetCurr, targetNext, &pingCb->pongList, list, struct RsPongTargetInfo);
837 2 : for (; (&targetCurr->list) != &pingCb->pongList;
838 0 : targetCurr = targetNext, targetNext = list_entry(targetNext->list.next, struct RsPongTargetInfo, list)) {
839 1 : if (RsPingCommonCompareRdmaInfo(&targetCurr->qpInfo, target)) {
840 1 : *node = targetCurr;
841 1 : RS_PTHREAD_MUTEX_ULOCK(&pingCb->pongMutex);
842 1 : return 0;
843 : }
844 : }
845 1 : RS_PTHREAD_MUTEX_ULOCK(&pingCb->pongMutex);
846 :
847 1 : hccp_info("pong target node for qpn:%u gid:%016llx:%016llx not found", target->rdma.qpn,
848 : target->rdma.gid.global.subnetPrefix, target->rdma.gid.global.interfaceId);
849 1 : return -ENODEV;
850 : }
851 :
852 2 : STATIC int RsPongFindAllocTargetNode(struct RsPingCtxCb *pingCb, struct PingQpInfo *target,
853 : struct RsPongTargetInfo **node)
854 : {
855 2 : struct RsPongTargetInfo *targetInfo = NULL;
856 : int ret;
857 :
858 2 : ret = RsPongFindTargetNode(pingCb, target, node);
859 2 : if (ret == 0 && (*node)->state == RS_PING_PONG_TARGET_READY) {
860 0 : return 0;
861 2 : } else if (ret == 0) {
862 1 : targetInfo = *node;
863 1 : hccp_info("delete pong target uuid:0x%llx state:%d, realloc again", targetInfo->uuid, targetInfo->state);
864 1 : RsListDel(&targetInfo->list);
865 1 : if (targetInfo->ah) {
866 0 : (void)RsIbvDestroyAh(targetInfo->ah);
867 : }
868 1 : free(targetInfo);
869 1 : targetInfo = NULL;
870 : }
871 :
872 2 : targetInfo = (struct RsPongTargetInfo *)calloc(1, sizeof(struct RsPongTargetInfo));
873 2 : CHK_PRT_RETURN(targetInfo == NULL, hccp_err("calloc target_info fail! errno:%d", errno), -ENOMEM);
874 :
875 2 : (void)memcpy_s(&targetInfo->qpInfo, sizeof(struct PingQpInfo), target, sizeof(struct PingQpInfo));
876 2 : ret = RsPingCommonCreateAh(pingCb, &pingCb->commInfo, target, &targetInfo->ah);
877 2 : if (ret != 0) {
878 1 : hccp_err("rs_ping_common_create_ah fail! ret:%d", ret);
879 1 : goto free_target_info;
880 : }
881 :
882 1 : targetInfo->state = RS_PING_PONG_TARGET_READY;
883 1 : *node = targetInfo;
884 :
885 1 : RS_PTHREAD_MUTEX_LOCK(&pingCb->pongMutex);
886 1 : targetInfo->uuid = (uint64_t)pingCb->pongNum << 32U;
887 1 : RsListAddTail(&targetInfo->list, &pingCb->pongList);
888 1 : pingCb->pongNum++;
889 1 : RS_PTHREAD_MUTEX_ULOCK(&pingCb->pongMutex);
890 :
891 1 : return 0;
892 :
893 1 : free_target_info:
894 1 : free(targetInfo);
895 1 : return ret;
896 : }
897 :
898 4 : STATIC int RsPongPostSend(struct RsPingCtxCb *pingCb, struct ibv_wc *wc, struct timeval *timestamp2)
899 : {
900 4 : struct RsPongTargetInfo *targetInfo = NULL;
901 4 : struct RsPingPayloadHeader *header = NULL;
902 4 : struct ibv_send_wr *badWr = NULL;
903 4 : struct timeval timestamp3 = {0};
904 4 : struct ibv_sge recvList = {0};
905 4 : struct ibv_sge sendList = {0};
906 4 : struct ibv_send_wr wr = {0};
907 : uint32_t recvSgeIdx;
908 : uint32_t sendSgeIdx;
909 4 : int ret = 0;
910 :
911 : // poll send cq
912 4 : (void)RsPingCommonPollScq(&pingCb->pongQp);
913 :
914 : // handle detect packet & send response packet
915 4 : recvSgeIdx = (uint32_t)wc->wr_id;
916 4 : if (recvSgeIdx >= pingCb->pingQp.recvMrCb.sgeNum) {
917 1 : hccp_err("param err recv_sge_idx:%u >= sge_num:%u", recvSgeIdx, pingCb->pingQp.recvMrCb.sgeNum);
918 1 : return -EIO;
919 : }
920 3 : (void)memcpy_s(&recvList, sizeof(struct ibv_sge), &pingCb->pingQp.recvMrCb.sgeList[recvSgeIdx],
921 : sizeof(struct ibv_sge));
922 :
923 3 : RS_PTHREAD_MUTEX_LOCK(&pingCb->pongQp.sendMrCb.mutex);
924 3 : sendSgeIdx = pingCb->pongQp.sendMrCb.sgeIdx;
925 3 : (void)memcpy_s(&sendList, sizeof(struct ibv_sge), &pingCb->pongQp.sendMrCb.sgeList[sendSgeIdx],
926 : sizeof(struct ibv_sge));
927 3 : pingCb->pongQp.sendMrCb.sgeIdx = (sendSgeIdx + 1) % pingCb->pongQp.sendMrCb.sgeNum;
928 3 : RS_PTHREAD_MUTEX_ULOCK(&pingCb->pongQp.sendMrCb.mutex);
929 :
930 : // UD consume 40 Bytes for GRH
931 3 : if (wc->byte_len < RS_PING_PAYLOAD_HEADER_RESV_GRH || wc->byte_len > PING_TOTAL_PAYLOAD_MAX_SIZE) {
932 0 : hccp_err("param err wc->byte_len:%u < %u or wc->byte_len:%u > %u", wc->byte_len,
933 : RS_PING_PAYLOAD_HEADER_RESV_GRH, wc->byte_len, PING_TOTAL_PAYLOAD_MAX_SIZE);
934 0 : return -EIO;
935 : }
936 3 : ret = memcpy_s((void *)(uintptr_t)sendList.addr, sendList.length,
937 3 : (void *)(uintptr_t)(recvList.addr + RS_PING_PAYLOAD_HEADER_RESV_GRH),
938 3 : wc->byte_len - RS_PING_PAYLOAD_HEADER_RESV_GRH);
939 3 : CHK_PRT_RETURN(ret != 0,
940 : hccp_err("memcpy_s buffer wc->byte_len:%u send_list.length:%u failed, ret:%d", wc->byte_len, sendList.length,
941 : ret),
942 : -ESAFEFUNC);
943 3 : sendList.length = wc->byte_len - RS_PING_PAYLOAD_HEADER_RESV_GRH;
944 3 : header = (struct RsPingPayloadHeader *)(uintptr_t)sendList.addr;
945 3 : header->type = RS_PING_TYPE_ROCE_RESPONSE;
946 :
947 3 : ret = RsPongFindAllocTargetNode(pingCb, &header->server, &targetInfo);
948 3 : if (ret != 0) {
949 1 : hccp_err("rs_pong_find_alloc_target_node failed, ret:%d", ret);
950 1 : return ret;
951 : }
952 :
953 2 : wr.wr_id = targetInfo->uuid;
954 2 : wr.next = NULL;
955 2 : wr.sg_list = &sendList;
956 2 : wr.num_sge = 1;
957 2 : wr.opcode = IBV_WR_SEND;
958 2 : wr.send_flags = IBV_SEND_SIGNALED;
959 2 : wr.wr.ud.ah = targetInfo->ah;
960 2 : wr.wr.ud.remote_qpn = targetInfo->qpInfo.rdma.qpn;
961 2 : wr.wr.ud.remote_qkey = targetInfo->qpInfo.rdma.qkey;
962 :
963 : // record timestamp t3
964 2 : (void)gettimeofday(×tamp3, NULL);
965 2 : header->timestamp.tvSec2 = (uint64_t)timestamp2->tv_sec;
966 2 : header->timestamp.tvUsec2 = (uint64_t)timestamp2->tv_usec;
967 2 : header->timestamp.tvSec3 = (uint64_t)timestamp3.tv_sec;
968 2 : header->timestamp.tvUsec3 = (uint64_t)timestamp3.tv_usec;
969 2 : header->magic = 0xAA55;
970 :
971 2 : ret = RsIbvPostSend(pingCb->pongQp.ibQp, &wr, &badWr);
972 2 : if (ret != 0) {
973 1 : targetInfo->state = RS_PING_PONG_TARGET_ERROR;
974 1 : hccp_err("rs_ibv_post_send failed, ret:%d", ret);
975 1 : return ret;
976 : }
977 :
978 1 : return ret;
979 : }
980 :
981 3 : STATIC void RsPongRoceHandleSend(struct RsPingCtxCb *pingCb, int polledCnt, struct timeval *timestamp2)
982 : {
983 3 : struct ibv_wc *wc = NULL;
984 : int ret, i;
985 :
986 3 : wc = gPingQpRecvWc;
987 6 : for (i = 0; i < polledCnt; i++) {
988 3 : if (wc[i].status != IBV_WC_SUCCESS) {
989 0 : hccp_err("wr_id:0x%llx error cqe %s(%d)", wc[i].wr_id, RsIbvWcStatusStr(wc[i].status), wc[i].status);
990 0 : continue;
991 : }
992 :
993 3 : ret = RsPongPostSend(pingCb, &wc[i], timestamp2);
994 3 : if (ret != 0) {
995 1 : hccp_err("rs_pong_post_send failed, wrId:0x%llx", wc[i].wr_id);
996 1 : continue;
997 : }
998 :
999 2 : ret = RsPingCommonPostRecv(&pingCb->pingQp);
1000 2 : if (ret != 0) {
1001 1 : hccp_err("rs_ping_common_post_recv failed, ret:%d", ret);
1002 1 : continue;
1003 : }
1004 : }
1005 :
1006 3 : ret = RsIbvReqNotifyCq(pingCb->pingQp.recvCq.ibCq, 0);
1007 3 : if (ret != 0) {
1008 1 : hccp_err("rs_ibv_req_notify_cq failed, ret:%d", ret);
1009 : }
1010 :
1011 3 : return;
1012 : }
1013 :
1014 4 : STATIC int RsPongResolveResponsePacket(struct RsPingCtxCb *pingCb, uint32_t sgeIdx, struct timeval *timestamp4)
1015 : {
1016 4 : struct RsPingTargetInfo *targetInfo = NULL;
1017 4 : struct RsPingPayloadHeader *header = NULL;
1018 4 : struct ibv_sge *recvList = NULL;
1019 : uint32_t rtt;
1020 : int ret;
1021 :
1022 4 : recvList = &pingCb->pongQp.recvMrCb.sgeList[sgeIdx];
1023 : // UD consume 40 Bytes for GRH
1024 4 : header = (struct RsPingPayloadHeader *)(uintptr_t)(recvList->addr + RS_PING_PAYLOAD_HEADER_RESV_GRH);
1025 4 : if (header->taskId != pingCb->taskId) {
1026 1 : hccp_warn("drop received packet, recv_task_id:%u, curr_task_id:%u", header->taskId, pingCb->taskId);
1027 1 : return 0;
1028 : }
1029 :
1030 3 : header->timestamp.tvSec4 = (uint64_t)timestamp4->tv_sec;
1031 3 : header->timestamp.tvUsec4 = (uint64_t)timestamp4->tv_usec;
1032 3 : rtt = RsPingGetTripTime(&header->timestamp);
1033 3 : ret = RsPingRoceFindTargetNode(pingCb, &header->target, &targetInfo);
1034 3 : if (ret != 0) {
1035 1 : hccp_err("rs_ping_roce_find_target_node failed, ret:%d qpn:%u gid:%016llx:%016llx rtt:%u", ret,
1036 : header->target.rdma.qpn, header->target.rdma.gid.global.subnetPrefix,
1037 : header->target.rdma.gid.global.interfaceId, rtt);
1038 1 : return ret;
1039 : }
1040 :
1041 2 : (void)memset_s((void *)header, RS_PING_PAYLOAD_HEADER_MASK_SIZE, 0, RS_PING_PAYLOAD_HEADER_MASK_SIZE);
1042 2 : RS_PTHREAD_MUTEX_LOCK(&targetInfo->tripMutex);
1043 2 : targetInfo->resultSummary.recvCnt++;
1044 2 : targetInfo->resultSummary.taskId = header->taskId;
1045 : // rtt timeout, increase timeout_cnt
1046 2 : if ((targetInfo->resultSummary.taskAttr.timeoutInterval * RS_PING_MSEC_TO_USEC) < rtt) {
1047 1 : targetInfo->resultSummary.timeoutCnt++;
1048 1 : hccp_dbg("recv_cnt:%u timeout_interval:%u rtt:%u timeout_cnt:%u", targetInfo->resultSummary.recvCnt,
1049 : targetInfo->resultSummary.taskAttr.timeoutInterval, rtt, targetInfo->resultSummary.timeoutCnt);
1050 1 : RS_PTHREAD_MUTEX_ULOCK(&targetInfo->tripMutex);
1051 1 : return 0;
1052 : }
1053 :
1054 : // handle rtt_min, rtt_max, rtt_avg
1055 1 : if (targetInfo->resultSummary.rttMin > rtt) {
1056 0 : targetInfo->resultSummary.rttMin = rtt;
1057 : }
1058 1 : if (targetInfo->resultSummary.rttMax < rtt) {
1059 1 : targetInfo->resultSummary.rttMax = rtt;
1060 : }
1061 1 : if (targetInfo->resultSummary.rttAvg == 0) {
1062 1 : targetInfo->resultSummary.rttAvg = rtt;
1063 : }
1064 1 : targetInfo->resultSummary.rttAvg = (targetInfo->resultSummary.rttAvg + rtt) / 2U;
1065 1 : RS_PTHREAD_MUTEX_ULOCK(&targetInfo->tripMutex);
1066 1 : return 0;
1067 : }
1068 :
1069 6 : STATIC void RsPongRocePollRcq(struct RsPingCtxCb *pingCb)
1070 : {
1071 6 : struct timeval timestamp = {0};
1072 6 : struct ibv_cq *evCq = NULL;
1073 6 : struct ibv_wc *wc = NULL;
1074 : uint32_t recvSgeIdx;
1075 6 : void *evCtx = NULL;
1076 : int polledCnt, i;
1077 : int ret;
1078 :
1079 : // record timestamp t4
1080 6 : (void)gettimeofday(×tamp, NULL);
1081 :
1082 6 : ret = RsIbvGetCqEvent(pingCb->pongQp.channel, &evCq, &evCtx);
1083 6 : if (ret != 0) {
1084 1 : hccp_err("rs_ibv_get_cq_event pong_qp.channel failed, ret:%d", ret);
1085 1 : return;
1086 : }
1087 :
1088 5 : if (evCq != pingCb->pongQp.recvCq.ibCq) {
1089 0 : hccp_err("CQ event for unknown CQ");
1090 0 : return;
1091 : }
1092 5 : pingCb->pongQp.recvCq.numEvents++;
1093 :
1094 5 : polledCnt = RsIbvPollCq(evCq, pingCb->pongQp.recvCq.maxRecvWcNum, gPongQpRecvWc);
1095 5 : if (polledCnt > pingCb->pongQp.recvCq.maxRecvWcNum || polledCnt < 0) {
1096 2 : hccp_err("rs_ibv_poll_cq failed, ret:%d", polledCnt);
1097 2 : return;
1098 : }
1099 :
1100 3 : wc = gPongQpRecvWc;
1101 6 : for (i = 0; i < polledCnt; i++) {
1102 3 : if (wc[i].status != IBV_WC_SUCCESS) {
1103 0 : hccp_err("wr_id:0x%llx error cqe %s(%d)", wc[i].wr_id, RsIbvWcStatusStr(wc[i].status), wc[i].status);
1104 0 : continue;
1105 : }
1106 3 : recvSgeIdx = (uint32_t)wc[i].wr_id;
1107 3 : if (recvSgeIdx >= pingCb->pongQp.recvMrCb.sgeNum) {
1108 3 : hccp_err("param err recv_sge_idx:%u > sge_num:%u", recvSgeIdx, pingCb->pongQp.recvMrCb.sgeNum);
1109 3 : continue;
1110 : }
1111 :
1112 : // handle response packet result
1113 0 : ret = RsPongResolveResponsePacket(pingCb, recvSgeIdx, ×tamp);
1114 0 : if (ret != 0) {
1115 0 : continue;
1116 : }
1117 :
1118 0 : ret = RsPingCommonPostRecv(&pingCb->pongQp);
1119 0 : if (ret != 0) {
1120 0 : continue;
1121 : }
1122 : }
1123 :
1124 3 : ret = RsIbvReqNotifyCq(evCq, 0);
1125 3 : if (ret != 0) {
1126 1 : hccp_err("rs_ibv_req_notify_cq failed, ret:%d", ret);
1127 : }
1128 :
1129 3 : return;
1130 : }
1131 :
1132 2 : STATIC int RsPingRoceGetTargetResult(struct RsPingCtxCb *pingCb, struct PingTargetCommInfo *target,
1133 : struct PingResultInfo *result)
1134 : {
1135 2 : struct RsPingTargetInfo *targetInfo = NULL;
1136 : int ret;
1137 :
1138 2 : ret = RsPingRoceFindTargetNode(pingCb, &target->qpInfo, &targetInfo);
1139 2 : if (ret != 0) {
1140 1 : hccp_err("rs_ping_roce_find_target_node failed, ret:%d qpn:%u gid:%016llx:%016llx", ret,
1141 : target->qpInfo.rdma.qpn, target->qpInfo.rdma.gid.global.subnetPrefix,
1142 : target->qpInfo.rdma.gid.global.interfaceId);
1143 1 : return ret;
1144 : }
1145 :
1146 1 : (void)memcpy_s(&result->summary, sizeof(struct PingResultSummary), &targetInfo->resultSummary,
1147 : sizeof(struct PingResultSummary));
1148 1 : if (targetInfo->state == RS_PING_PONG_TARGET_FINISH) {
1149 0 : result->state = PING_RESULT_STATE_VALID;
1150 : } else {
1151 1 : result->state = PING_RESULT_STATE_INVALID;
1152 : }
1153 :
1154 1 : hccp_dbg("ip:0x%llx qpn:%u, state:%d sendCnt:%u recvCnt:%u timeoutCnt:%u rttMin:%u rttMax:%u rttAvg:%u",
1155 : target->ip.addr.s_addr, target->qpInfo.rdma.qpn, result->state, result->summary.sendCnt,
1156 : result->summary.recvCnt, result->summary.timeoutCnt, result->summary.rttMin, result->summary.rttMax,
1157 : result->summary.rttAvg);
1158 :
1159 1 : return 0;
1160 : }
1161 :
1162 1 : STATIC void RsPingRoceFreeTargetNode(struct RsPingCtxCb *pingCb, struct RsPingTargetInfo *targetInfo)
1163 : {
1164 1 : RS_PTHREAD_MUTEX_LOCK(&pingCb->pingMutex);
1165 1 : RsListDel(&targetInfo->list);
1166 1 : RS_PTHREAD_MUTEX_ULOCK(&pingCb->pingMutex);
1167 :
1168 1 : if (targetInfo->payloadSize > 0 && targetInfo->payloadBuffer != NULL) {
1169 1 : free(targetInfo->payloadBuffer);
1170 1 : targetInfo->payloadBuffer = NULL;
1171 : }
1172 :
1173 1 : if (targetInfo->ah) {
1174 0 : (void)RsIbvDestroyAh(targetInfo->ah);
1175 : }
1176 1 : return;
1177 : }
1178 :
1179 1 : STATIC void RsPingPongDelTargetList(struct RsPingCtxCb *pingCb)
1180 : {
1181 1 : struct RsPongTargetInfo *pongNext = NULL;
1182 1 : struct RsPingTargetInfo *pingNext = NULL;
1183 1 : struct RsPongTargetInfo *pongCurr = NULL;
1184 1 : struct RsPingTargetInfo *pingCurr = NULL;
1185 :
1186 : // del ping_list
1187 1 : RS_PTHREAD_MUTEX_LOCK(&pingCb->pingMutex);
1188 1 : RS_LIST_GET_HEAD_ENTRY(pingCurr, pingNext, &pingCb->pingList, list, struct RsPingTargetInfo);
1189 2 : for (; (&pingCurr->list) != &pingCb->pingList;
1190 1 : pingCurr = pingNext, pingNext = list_entry(pingNext->list.next, struct RsPingTargetInfo, list)) {
1191 1 : RsListDel(&pingCurr->list);
1192 1 : if (pingCurr->payloadSize > 0 && pingCurr->payloadBuffer != NULL) {
1193 0 : free(pingCurr->payloadBuffer);
1194 0 : pingCurr->payloadBuffer = NULL;
1195 : }
1196 1 : if (pingCurr->ah) {
1197 0 : (void)RsIbvDestroyAh(pingCurr->ah);
1198 : }
1199 1 : (void)pthread_mutex_destroy(&pingCurr->tripMutex);
1200 1 : free(pingCurr);
1201 1 : pingCurr = NULL;
1202 : }
1203 1 : RS_PTHREAD_MUTEX_ULOCK(&pingCb->pingMutex);
1204 :
1205 : // del pong_list
1206 1 : RS_PTHREAD_MUTEX_LOCK(&pingCb->pongMutex);
1207 1 : RS_LIST_GET_HEAD_ENTRY(pongCurr, pongNext, &pingCb->pongList, list, struct RsPongTargetInfo);
1208 2 : for (; (&pongCurr->list) != &pingCb->pongList;
1209 1 : pongCurr = pongNext, pongNext = list_entry(pongNext->list.next, struct RsPongTargetInfo, list)) {
1210 1 : RsListDel(&pongCurr->list);
1211 1 : if (pongCurr->ah) {
1212 0 : (void)RsIbvDestroyAh(pongCurr->ah);
1213 : }
1214 1 : free(pongCurr);
1215 1 : pongCurr = NULL;
1216 : }
1217 1 : RS_PTHREAD_MUTEX_ULOCK(&pingCb->pongMutex);
1218 1 : }
1219 :
1220 2 : STATIC void RsPingRocePingCbDeinit(unsigned int phyId, struct RsPingCtxCb *pingCb)
1221 : {
1222 2 : struct rs_cb *rscb = NULL;
1223 : int ret;
1224 :
1225 2 : ret = RsGetRsCb(phyId, &rscb);
1226 2 : if (ret != 0) {
1227 1 : hccp_err("RsGetRsCb failed, phyId[%u] invalid, ret %d", phyId, ret);
1228 1 : return;
1229 : }
1230 :
1231 1 : RS_PTHREAD_MUTEX_LOCK(&pingCb->pingMutex);
1232 1 : pingCb->taskStatus = RS_PING_TASK_RESET;
1233 1 : RS_PTHREAD_MUTEX_ULOCK(&pingCb->pingMutex);
1234 :
1235 1 : RsPingPongDelTargetList(pingCb);
1236 :
1237 1 : RsPingCommonDeinitLocalQp(rscb, pingCb, &pingCb->pongQp);
1238 1 : RsPingCommonDeinitLocalQp(rscb, pingCb, &pingCb->pingQp);
1239 1 : RsPingCommonDeinitLocalBuffer(pingCb);
1240 1 : (void)RsIbvDeallocPd(pingCb->rdevCb.ibPd);
1241 1 : (void)RsIbvCloseDevice(pingCb->rdevCb.ibCtx);
1242 1 : RsIbvFreeDeviceList(pingCb->rdevCb.devList);
1243 : }
1244 :
1245 2 : STATIC void RsPingRoceAddTargetSuccess(struct PingTargetInfo *target, struct RsPingTargetInfo *targetInfo)
1246 : {
1247 2 : hccp_info("target ip:0x%llx payload_size:%u add success, qpn:%u uuid:0x%llx", target->remoteInfo.ip.addr.s_addr,
1248 : target->payload.size, targetInfo->qpInfo.rdma.qpn, targetInfo->uuid);
1249 2 : }
1250 :
1251 1 : STATIC void RsPingRocePingCbInitSuccess(unsigned int phyId, struct PingInitAttr *attr, unsigned int devIndex)
1252 : {
1253 1 : hccp_run_info("ping_cb init success, phyId:%u, localIp:0x%x, devIndex:%u", phyId,
1254 : attr->dev.rdma.localIp.addr.s_addr, devIndex);
1255 1 : }
1256 :
1257 1 : STATIC void RsPingRoceCannotFindTargetNode(unsigned int i, int ret, struct PingTargetCommInfo target,
1258 : unsigned int phyId)
1259 : {
1260 1 : hccp_err("rs_ping_roce_find_target_node i:%u failed, ret:%d ip:0x%llx qpn:%u phyId:%u", i, ret,
1261 : target.ip.addr.s_addr, target.qpInfo.rdma.qpn, phyId);
1262 1 : }
1263 :
1264 : struct RsPingPongOps gRsPingRoceOps = {
1265 : .checkPingFd = RsPingRoceCheckFd,
1266 : .checkPongFd = RsPongRoceCheckFd,
1267 : .initPingCb = RsPingRocePingCbInit,
1268 : .pingFindTargetNode = RsPingRoceFindTargetNode,
1269 : .pingAllocTargetNode = RsPingRoceAllocTargetNode,
1270 : .resetRecvBuffer = RsPingRoceResetRecvBuffer,
1271 : .pingPostSend = RsPingRocePostSend,
1272 : .pingPollScq = RsPingRocePollScq,
1273 : .pingPollRcq = RsPingRocePollRcq,
1274 : .pongHandleSend = RsPongRoceHandleSend,
1275 : .pongPollRcq = RsPongRocePollRcq,
1276 : .getTargetResult = RsPingRoceGetTargetResult,
1277 : .pingFreeTargetNode = RsPingRoceFreeTargetNode,
1278 : .deinitPingCb = RsPingRocePingCbDeinit,
1279 : };
1280 :
1281 : struct RsPingPongDfx gRsPingRoceDfx = {
1282 : .addTargetSuccess = RsPingRoceAddTargetSuccess,
1283 : .initPingCbSuccess = RsPingRocePingCbInitSuccess,
1284 : .pingCannotFindTargetNode = RsPingRoceCannotFindTargetNode,
1285 : };
1286 :
1287 27 : struct RsPingPongOps *RsPingRoceGetOps(void)
1288 : {
1289 27 : return &gRsPingRoceOps;
1290 : }
1291 :
1292 27 : struct RsPingPongDfx *RsPingRoceGetDfx(void)
1293 : {
1294 27 : return &gRsPingRoceDfx;
1295 : }
|
