Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include <errno.h>
12 : #include <infiniband/verbs.h>
13 : #include "securec.h"
14 : #include "dl_hal_function.h"
15 : #include "dl_ibverbs_function.h"
16 : #include "hccp_common.h"
17 : #include "rs.h"
18 : #include "ra_rs_err.h"
19 : #include "rs_inner.h"
20 : #include "rs_epoll.h"
21 : #include "rs_drv_socket.h"
22 : #include "rs_drv_rdma.h"
23 : #include "rs_ping_inner.h"
24 : #ifndef HNS_ROCE_LLT
25 : #include <dlog_pub.h>
26 : #endif
27 : #include "rs_ping_roce.h"
28 :
29 : #define RS_PING_ROCE_RECV_WC_NUM 16
30 :
31 : struct ibv_wc gPingQpRecvWc[RS_PING_ROCE_RECV_WC_NUM] = { 0 };
32 : struct ibv_wc gPongQpRecvWc[RS_PING_ROCE_RECV_WC_NUM] = { 0 };
33 :
34 3 : STATIC bool RsPingRoceCheckFd(struct RsPingCtxCb *pingCb, int fd)
35 : {
36 3 : if (pingCb->pingQp.channel != NULL && pingCb->pingQp.channel->fd == fd) {
37 1 : hccp_dbg("ping_qp rq, channel->fd:%d poll cq", fd);
38 1 : return true;
39 : }
40 2 : return false;
41 : }
42 :
43 2 : STATIC bool RsPongRoceCheckFd(struct RsPingCtxCb *pingCb, int fd)
44 : {
45 2 : if (pingCb->pongQp.channel != NULL && pingCb->pongQp.channel->fd == fd) {
46 2 : hccp_dbg("pong_qp rq, channel->fd:%d poll cq", fd);
47 2 : return true;
48 : }
49 0 : return false;
50 : }
51 :
52 0 : STATIC int RsPingCbGetDevRdevIndex(struct RsPingCtxCb *pingCb, int index)
53 : {
54 : #ifdef CUSTOM_INTERFACE
55 0 : struct roce_dev_data rdevData = { 0 };
56 : int ret;
57 :
58 0 : if (RsIsCustomInterfaceSupported()) {
59 0 : RS_PTHREAD_MUTEX_LOCK(&pingCb->pingMutex);
60 0 : pingCb->rdevCb.devName = RsIbvGetDeviceName(pingCb->rdevCb.devList[index]);
61 0 : ret = RsRoceGetRoceDevData(pingCb->rdevCb.devName, &rdevData);
62 0 : if (ret != 0) {
63 0 : hccp_err("rs_roce_get_roce_dev_data failed, ret:%d, devName:%s", ret, pingCb->rdevCb.devName);
64 0 : RS_PTHREAD_MUTEX_ULOCK(&pingCb->pingMutex);
65 0 : return ret;
66 : }
67 0 : pingCb->devIndex = rdevData.rdev_index; // rdev_index is same to port_id
68 0 : RS_PTHREAD_MUTEX_ULOCK(&pingCb->pingMutex);
69 : }
70 : #endif
71 0 : return 0;
72 : }
73 :
74 2 : STATIC int RsPingCbGetIbCtxAndIndex(struct rdev *rdevInfo, struct RsPingCtxCb *pingCb)
75 : {
76 2 : struct ibv_context *ibCtx = NULL;
77 : int ret;
78 : int i;
79 :
80 4 : for (i = 0; (i < pingCb->rdevCb.devNum) && (pingCb->rdevCb.devList[i] != NULL); ++i) {
81 3 : ibCtx = RsIbvOpenDevice(pingCb->rdevCb.devList[i]);
82 3 : CHK_PRT_RETURN(ibCtx == NULL, hccp_err("ibv_open_device failed!"), -ENODEV);
83 3 : ret = RsQueryGid(*rdevInfo, ibCtx, pingCb->rdevCb.ibPort, &pingCb->rdevCb.gidIdx);
84 3 : if (ret == 0) {
85 1 : ret = RsPingCbGetDevRdevIndex(pingCb, i);
86 1 : if (ret != 0) {
87 0 : hccp_err("rs_ping_cb_get_dev_rdev_index failed, ret:%d", ret);
88 0 : RsIbvCloseDevice(ibCtx);
89 0 : return ret;
90 : }
91 1 : pingCb->rdevCb.ibCtx = ibCtx;
92 1 : ret = RsIbvQueryGid(ibCtx, pingCb->rdevCb.ibPort, pingCb->rdevCb.gidIdx, &pingCb->rdevCb.gid);
93 1 : if (ret != 0) {
94 1 : RsIbvCloseDevice(ibCtx);
95 1 : hccp_err("query gid failed gid_idx %d, ret %d", pingCb->rdevCb.gidIdx, ret);
96 1 : return -EOPENSRC;
97 : }
98 0 : return 0;
99 2 : } else if (ret == -EEXIST) {
100 2 : RsIbvCloseDevice(ibCtx);
101 : } else {
102 0 : hccp_err("rs_query_gid failed, ret:%d", ret);
103 0 : RsIbvCloseDevice(ibCtx);
104 0 : return ret;
105 : }
106 : }
107 :
108 1 : CHK_PRT_RETURN(i == pingCb->rdevCb.devNum, hccp_err("can not find ib_ctx for phyId[%u] local_ip[0x%x] "
109 : "in dev_list!", rdevInfo->phyId, rdevInfo->localIp.addr.s_addr), -ENODEV);
110 0 : return 0;
111 : }
112 :
113 3 : STATIC int RsPingCommonModifyLocalQp(struct RsPingCtxCb *pingCb, struct RsPingLocalQpCb *qpCb)
114 : {
115 : struct ibv_qp_init_attr initAttr;
116 3 : struct ibv_qp_attr attr = { 0 };
117 : int ret;
118 :
119 3 : ret = RsIbvQueryQp(qpCb->ibQp, &attr, IBV_QP_STATE, &initAttr);
120 3 : CHK_PRT_RETURN(ret != 0 || attr.qp_state != IBV_QPS_RESET,
121 : hccp_err("rs_ibv_query_qp qpn:%u fail, ret:%d attr.qp_state:%d != %d",
122 : qpCb->ibQp->qp_num, ret, attr.qp_state, IBV_QPS_RESET), -EOPENSRC);
123 :
124 2 : attr.qp_state = IBV_QPS_INIT;
125 2 : attr.pkey_index = 0;
126 2 : attr.port_num = pingCb->rdevCb.ibPort;
127 2 : attr.qkey = qpCb->qkey;
128 2 : ret = RsIbvModifyQp(qpCb->ibQp, &attr, IBV_QP_STATE | IBV_QP_PKEY_INDEX | IBV_QP_PORT | IBV_QP_QKEY);
129 2 : CHK_PRT_RETURN(ret != 0, hccp_err("rs_ibv_modify_qp qpn:%u to init fail, ret:%d, errno:%d",
130 : qpCb->ibQp->qp_num, ret, errno), -EOPENSRC);
131 :
132 1 : attr.qp_state = IBV_QPS_RTR;
133 1 : ret = RsIbvModifyQp(qpCb->ibQp, &attr, IBV_QP_STATE);
134 1 : CHK_PRT_RETURN(ret != 0, hccp_err("rs_ibv_modify_qp qpn:%u to rtr fail, ret:%d, errno:%d",
135 : qpCb->ibQp->qp_num, ret, errno), -EOPENSRC);
136 :
137 1 : attr.qp_state = IBV_QPS_RTS;
138 1 : attr.sq_psn = 0;
139 1 : ret = RsIbvModifyQp(qpCb->ibQp, &attr, IBV_QP_STATE | IBV_QP_SQ_PSN);
140 1 : CHK_PRT_RETURN(ret != 0, hccp_err("rs_ibv_modify_qp qpn:%u to rts fail, ret:%d, errno:%d",
141 : qpCb->ibQp->qp_num, ret, errno), -EOPENSRC);
142 :
143 1 : return 0;
144 : }
145 :
146 2 : STATIC int RsPingCommonInitLocalQp(struct rs_cb *rscb, struct RsPingCtxCb *pingCb, union PingQpAttr *attr,
147 : struct RsPingLocalQpCb *qpCb)
148 : {
149 2 : struct ibv_exp_qp_init_attr qpInitAttr = { 0 };
150 2 : struct rdma_lite_device_qp_attr qpResp = { 0 };
151 : int randNum;
152 : int ret;
153 :
154 2 : hccp_info("cq_attr{%d %d, %d %d}", attr->rdma.cqAttr.sendCqDepth, attr->rdma.cqAttr.sendCqCompVector,
155 : attr->rdma.cqAttr.recvCqDepth, attr->rdma.cqAttr.recvCqCompVector);
156 :
157 : // create send cq with attr
158 2 : qpCb->sendCq.depth = attr->rdma.cqAttr.sendCqDepth;
159 2 : qpCb->sendCq.compVector = attr->rdma.cqAttr.sendCqCompVector;
160 2 : qpCb->sendCq.ibCq = RsIbvCreateCq(pingCb->rdevCb.ibCtx, qpCb->sendCq.depth, NULL, NULL,
161 : qpCb->sendCq.compVector);
162 2 : qpCb->sendCq.maxRecvWcNum = RS_PING_ROCE_RECV_WC_NUM;
163 2 : ret = -errno;
164 2 : CHK_PRT_RETURN(qpCb->sendCq.ibCq == NULL, hccp_err("rs_ibv_create_cq send cq fail, ret:%d", ret), ret);
165 :
166 : // create channel & create recv cq with attr
167 1 : qpCb->channel = RsIbvCreateCompChannel(pingCb->rdevCb.ibCtx);
168 1 : if (qpCb->channel == NULL) {
169 0 : ret = -errno;
170 0 : hccp_err("rs_ibv_create_comp_channel failed! ret:%d", ret);
171 0 : goto create_channel_fail;
172 : }
173 1 : ret = RsEpollCtl(rscb->connCb.epollfd, EPOLL_CTL_ADD, qpCb->channel->fd, EPOLLIN | EPOLLRDHUP);
174 1 : if (ret != 0) {
175 0 : hccp_err("RsEpollCtl failed! epollfd:%d fd:%d ret:%d", rscb->connCb.epollfd, qpCb->channel->fd, ret);
176 0 : goto epoll_ctl_fail;
177 : }
178 1 : qpCb->recvCq.depth = attr->rdma.cqAttr.recvCqDepth;
179 1 : qpCb->recvCq.compVector = attr->rdma.cqAttr.recvCqCompVector;
180 1 : qpCb->recvCq.ibCq = RsIbvCreateCq(pingCb->rdevCb.ibCtx, qpCb->recvCq.depth, NULL, qpCb->channel,
181 : qpCb->recvCq.compVector);
182 1 : qpCb->recvCq.maxRecvWcNum = RS_PING_ROCE_RECV_WC_NUM;
183 1 : if (qpCb->recvCq.ibCq == NULL) {
184 0 : ret = -errno;
185 0 : hccp_err("rs_ibv_create_cq recv cq fail, ret:%d", ret);
186 0 : goto create_rcq_fail;
187 : }
188 :
189 : // create qp with attr
190 1 : (void)RsDrvGetRandomNum(&randNum);
191 : // clear bit IB_QP_SET_QKEY to avoid modify_qp to INIT failed
192 1 : qpCb->qkey = (uint32_t)(((uint32_t)randNum) & (~(1U << 31U)));
193 1 : (void)memcpy_s(&qpCb->qpCap, sizeof(struct ibv_qp_cap), &attr->rdma.qpAttr.cap, sizeof(struct ibv_qp_cap));
194 1 : qpCb->udpSport = attr->rdma.qpAttr.udpSport;
195 1 : qpInitAttr.attr.send_cq = qpCb->sendCq.ibCq;
196 1 : qpInitAttr.attr.recv_cq = qpCb->recvCq.ibCq;
197 1 : (void)memcpy_s(&qpInitAttr.attr.cap, sizeof(struct ibv_qp_cap), &qpCb->qpCap, sizeof(struct ibv_qp_cap));
198 1 : qpInitAttr.attr.qp_type = IBV_QPT_UD;
199 1 : qpInitAttr.udp_sport = attr->rdma.qpAttr.udpSport;
200 :
201 1 : hccp_info("qkey:%u udp_sport:%u qp_cap{%u %u %u %u %u}", qpCb->qkey, qpCb->udpSport,
202 : attr->rdma.qpAttr.cap.maxSendWr, attr->rdma.qpAttr.cap.maxRecvWr, attr->rdma.qpAttr.cap.maxSendSge,
203 : attr->rdma.qpAttr.cap.maxRecvSge, attr->rdma.qpAttr.cap.maxInlineData);
204 1 : qpCb->ibQp = RsIbvExpCreateQp(pingCb->rdevCb.ibPd, &qpInitAttr, &qpResp);
205 1 : if (qpCb->ibQp == NULL) {
206 1 : ret = -errno;
207 1 : hccp_err("rs_ibv_exp_create_qp qp fail, ret:%d", ret);
208 1 : goto create_qp_fail;
209 : }
210 :
211 0 : ret = RsPingCommonModifyLocalQp(pingCb, qpCb);
212 0 : if (ret != 0) {
213 0 : hccp_err("rs_ping_common_modify_local_qp failed, ret:%d", ret);
214 0 : goto modify_qp_fail;
215 : }
216 :
217 0 : ret = RsIbvReqNotifyCq(qpCb->recvCq.ibCq, 0);
218 0 : if (ret != 0) {
219 0 : hccp_err("rs_ibv_req_notify_cq failed, ret:%d", ret);
220 0 : goto modify_qp_fail;
221 : }
222 :
223 0 : hccp_run_info("qpn:%u create success, cqAttr{%d %d, %d %d} qkey:%u udpSport:%u qpCap{%u %u %u %u %u}",
224 : qpCb->ibQp->qp_num, attr->rdma.cqAttr.sendCqDepth, attr->rdma.cqAttr.sendCqCompVector,
225 : attr->rdma.cqAttr.recvCqDepth, attr->rdma.cqAttr.recvCqCompVector, qpCb->qkey, qpCb->udpSport,
226 : attr->rdma.qpAttr.cap.maxSendWr, attr->rdma.qpAttr.cap.maxRecvWr, attr->rdma.qpAttr.cap.maxSendSge,
227 : attr->rdma.qpAttr.cap.maxRecvSge, attr->rdma.qpAttr.cap.maxInlineData);
228 :
229 0 : return 0;
230 :
231 0 : modify_qp_fail:
232 0 : (void)RsIbvDestroyQp(qpCb->ibQp);
233 1 : create_qp_fail:
234 1 : (void)RsIbvDestroyCq(qpCb->recvCq.ibCq);
235 1 : create_rcq_fail:
236 1 : (void)RsEpollCtl(rscb->connCb.epollfd, EPOLL_CTL_DEL, qpCb->channel->fd, EPOLLIN | EPOLLRDHUP);
237 1 : epoll_ctl_fail:
238 1 : (void)RsIbvDestroyCompChannel(qpCb->channel);
239 1 : create_channel_fail:
240 1 : (void)RsIbvDestroyCq(qpCb->sendCq.ibCq);
241 1 : return ret;
242 : }
243 :
244 0 : STATIC int RsPingCommonInitMrCb(struct rs_cb *rscb, struct RsPingCtxCb *pingCb, struct RsPingMrCb *mrCb)
245 : {
246 0 : unsigned long flag = 0;
247 0 : uint32_t idx = 0;
248 : int ret;
249 :
250 0 : hccp_info("payload_offset:%u len:0x%llx sge_num:%u grp_id:%u",
251 : mrCb->payloadOffset, mrCb->len, mrCb->sgeNum, rscb->grpId);
252 :
253 0 : ret = pthread_mutex_init(&mrCb->mutex, NULL);
254 0 : CHK_PRT_RETURN(ret != 0, hccp_err("pthread_mutex_init mr_cb mutex failed, ret:%d", ret), ret);
255 :
256 0 : flag = ((unsigned long)pingCb->logicDevid << BUFF_FLAGS_DEVID_OFFSET) | BUFF_SP_SVM;
257 0 : ret = DlHalBuffAllocAlignEx(mrCb->len, (unsigned int)RA_RS_4K_PAGE_SIZE, flag,
258 0 : (int)rscb->grpId, (void **)&mrCb->addr);
259 0 : if (ret != 0) {
260 0 : hccp_err("DlHalBuffAllocAlignEx failed, length:0x%llx, dev_id:0x%x, flag:0x%lx, grpId:%u, ret:%d",
261 : mrCb->len, pingCb->logicDevid, flag, rscb->grpId, ret);
262 0 : goto alloc_fail;
263 : }
264 :
265 0 : mrCb->ibMr = RsDrvMrReg(pingCb->rdevCb.ibPd, (char *)(uintptr_t)mrCb->addr, mrCb->len,
266 : IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE | IBV_ACCESS_REMOTE_READ);
267 0 : if (mrCb->ibMr == NULL) {
268 0 : ret = -errno;
269 0 : hccp_err("rs_ibv_reg_mr fail, ret:%d addr:0x%llx len:0x%llx", ret, mrCb->addr, mrCb->len);
270 0 : goto mr_reg_fail;
271 : }
272 :
273 : // init sge list
274 0 : mrCb->sgeList = calloc(mrCb->sgeNum, sizeof(struct ibv_sge));
275 0 : if (mrCb->sgeList == NULL) {
276 0 : ret = -errno;
277 0 : hccp_err("calloc fail, ret:%d sgeNum:%u", ret, mrCb->sgeNum);
278 0 : goto calloc_fail;
279 : }
280 0 : for (idx = 0; idx < mrCb->sgeNum; idx++) {
281 0 : mrCb->sgeList[idx].lkey = mrCb->ibMr->lkey;
282 0 : mrCb->sgeList[idx].length = mrCb->payloadOffset;
283 0 : if (idx == 0) {
284 0 : mrCb->sgeList[idx].addr = mrCb->addr;
285 : } else {
286 0 : mrCb->sgeList[idx].addr = mrCb->sgeList[idx - 1].addr + mrCb->payloadOffset;
287 : }
288 : }
289 0 : mrCb->sgeIdx = 0;
290 :
291 0 : hccp_info("addr:0x%llx lkey:%u ", mrCb->addr, mrCb->ibMr->lkey);
292 :
293 0 : return 0;
294 :
295 0 : calloc_fail:
296 0 : (void)RsDrvMrDereg(mrCb->ibMr);
297 0 : mr_reg_fail:
298 0 : (void)DlHalBuffFree((void *)(uintptr_t)mrCb->addr);
299 0 : alloc_fail:
300 0 : (void)pthread_mutex_destroy(&mrCb->mutex);
301 0 : return ret;
302 : }
303 :
304 4 : STATIC void RsPingCommonDeinitMrCb(struct RsPingMrCb *mrCb)
305 : {
306 4 : hccp_dbg("addr:0x%llx len:%llu", mrCb->addr, mrCb->len);
307 4 : free(mrCb->sgeList);
308 4 : mrCb->sgeList = NULL;
309 4 : (void)RsDrvMrDereg(mrCb->ibMr);
310 4 : (void)DlHalBuffFree((void *)(uintptr_t)mrCb->addr);
311 4 : (void)pthread_mutex_destroy(&mrCb->mutex);
312 4 : }
313 :
314 2 : STATIC int RsPingPongInitLocalBuffer(struct rs_cb *rscb, struct PingInitAttr *attr, struct PingInitInfo *info,
315 : struct RsPingCtxCb *pingCb)
316 : {
317 : int ret;
318 :
319 : // prepare ping_qp send mr
320 2 : pingCb->pingQp.sendMrCb.payloadOffset = PING_TOTAL_PAYLOAD_MAX_SIZE;
321 2 : pingCb->pingQp.sendMrCb.len = pingCb->pingQp.qpCap.max_send_wr * pingCb->pingQp.sendMrCb.payloadOffset;
322 2 : pingCb->pingQp.sendMrCb.sgeNum = pingCb->pingQp.qpCap.max_send_wr;
323 2 : ret = RsPingCommonInitMrCb(rscb, pingCb, &pingCb->pingQp.sendMrCb);
324 2 : CHK_PRT_RETURN(ret != 0, hccp_err("rs_ping_common_init_mr_cb ping_qp send_mr_cb failed, ret %d", ret), ret);
325 : // prepare ping_qp recv mr
326 1 : pingCb->pingQp.recvMrCb.payloadOffset = PING_TOTAL_PAYLOAD_MAX_SIZE;
327 1 : pingCb->pingQp.recvMrCb.len = pingCb->pingQp.qpCap.max_recv_wr * pingCb->pingQp.recvMrCb.payloadOffset;
328 1 : pingCb->pingQp.recvMrCb.sgeNum = pingCb->pingQp.qpCap.max_recv_wr;
329 1 : ret = RsPingCommonInitMrCb(rscb, pingCb, &pingCb->pingQp.recvMrCb);
330 1 : if (ret != 0) {
331 0 : hccp_err("rs_ping_common_init_mr_cb ping_qp recv_mr_cb failed, ret %d", ret);
332 0 : goto init_ping_qp_recv_mr_fail;
333 : }
334 :
335 : // prepare pong_qp send mr
336 1 : pingCb->pongQp.sendMrCb.payloadOffset = PING_TOTAL_PAYLOAD_MAX_SIZE;
337 1 : pingCb->pongQp.sendMrCb.len = pingCb->pongQp.qpCap.max_send_wr * pingCb->pongQp.sendMrCb.payloadOffset;
338 1 : pingCb->pongQp.sendMrCb.sgeNum = pingCb->pongQp.qpCap.max_send_wr;
339 1 : ret = RsPingCommonInitMrCb(rscb, pingCb, &pingCb->pongQp.sendMrCb);
340 1 : if (ret != 0) {
341 0 : hccp_err("rs_ping_common_init_mr_cb pong_qp send_mr_cb failed, ret %d", ret);
342 0 : goto init_pong_qp_send_mr_fail;
343 : }
344 : // prepare pong_qp recv mr
345 1 : pingCb->pongQp.recvMrCb.payloadOffset = PING_TOTAL_PAYLOAD_MAX_SIZE;
346 1 : pingCb->pongQp.recvMrCb.len = attr->bufferSize;
347 1 : pingCb->pongQp.recvMrCb.sgeNum = attr->bufferSize / pingCb->pongQp.recvMrCb.payloadOffset;
348 1 : ret = RsPingCommonInitMrCb(rscb, pingCb, &pingCb->pongQp.recvMrCb);
349 1 : if (ret != 0) {
350 0 : hccp_err("rs_ping_common_init_mr_cb pong_qp recv_mr_cb failed, ret %d", ret);
351 0 : goto init_pong_qp_recv_mr_fail;
352 : }
353 1 : info->result.bufferVa = pingCb->pongQp.recvMrCb.addr;
354 1 : info->result.bufferSize = attr->bufferSize;
355 1 : info->result.payloadOffset = pingCb->pongQp.recvMrCb.payloadOffset;
356 1 : info->result.headerSize = RS_PING_PAYLOAD_HEADER_RESV_GRH + RS_PING_PAYLOAD_HEADER_RESV_CUSTOM;
357 :
358 1 : return 0;
359 :
360 0 : init_pong_qp_recv_mr_fail:
361 0 : RsPingCommonDeinitMrCb(&pingCb->pongQp.sendMrCb);
362 0 : init_pong_qp_send_mr_fail:
363 0 : RsPingCommonDeinitMrCb(&pingCb->pingQp.recvMrCb);
364 0 : init_ping_qp_recv_mr_fail:
365 0 : RsPingCommonDeinitMrCb(&pingCb->pingQp.sendMrCb);
366 0 : return ret;
367 : }
368 :
369 2 : STATIC int RsPingCommonPostRecv(struct RsPingLocalQpCb *qpCb)
370 : {
371 2 : struct ibv_recv_wr *badWr = NULL;
372 2 : struct ibv_recv_wr wr = { 0 };
373 2 : struct ibv_sge list = { 0 };
374 : uint32_t sgeIdx;
375 : int ret;
376 :
377 2 : RS_PTHREAD_MUTEX_LOCK(&qpCb->recvMrCb.mutex);
378 2 : sgeIdx = qpCb->recvMrCb.sgeIdx;
379 2 : (void)memcpy_s(&list, sizeof(struct ibv_sge), &qpCb->recvMrCb.sgeList[sgeIdx], sizeof(struct ibv_sge));
380 2 : qpCb->recvMrCb.sgeIdx = (sgeIdx + 1) % qpCb->recvMrCb.sgeNum;
381 2 : RS_PTHREAD_MUTEX_ULOCK(&qpCb->recvMrCb.mutex);
382 :
383 2 : wr.wr_id = (uint64_t)sgeIdx;
384 2 : wr.next = NULL;
385 2 : wr.sg_list = &list;
386 2 : wr.num_sge = 1;
387 :
388 2 : ret = RsIbvPostRecv(qpCb->ibQp, &wr, &badWr);
389 2 : if (ret != 0) {
390 1 : hccp_err("rs_ibv_post_recv failed, ret:%d", ret);
391 1 : return ret;
392 : }
393 :
394 1 : return 0;
395 : }
396 :
397 2 : STATIC int RsPingCommonInitPostRecvAll(struct RsPingLocalQpCb *qpCb)
398 : {
399 2 : int ret = 0;
400 : uint32_t i;
401 :
402 : // prepare RQ wqe
403 3 : for (i = qpCb->recvMrCb.sgeIdx; i < qpCb->recvMrCb.sgeNum && i < qpCb->qpCap.max_recv_wr; i++) {
404 2 : ret = RsPingCommonPostRecv(qpCb);
405 2 : if (ret != 0) {
406 1 : hccp_err("rs_ping_common_post_recv %u-th rqe failed, ret:%d", i, ret);
407 1 : break;
408 : }
409 : }
410 :
411 2 : return ret;
412 : }
413 :
414 1 : STATIC void RsPingCommonDeinitLocalBuffer(struct RsPingCtxCb *pingCb)
415 : {
416 1 : RsPingCommonDeinitMrCb(&pingCb->pongQp.recvMrCb);
417 1 : RsPingCommonDeinitMrCb(&pingCb->pongQp.sendMrCb);
418 1 : RsPingCommonDeinitMrCb(&pingCb->pingQp.recvMrCb);
419 1 : RsPingCommonDeinitMrCb(&pingCb->pingQp.sendMrCb);
420 1 : }
421 :
422 2 : STATIC void RsPingCommonDeinitLocalQp(struct rs_cb *rscb, struct RsPingCtxCb *pingCb,
423 : struct RsPingLocalQpCb *qpCb)
424 : {
425 2 : if (qpCb == NULL || qpCb->channel == NULL) {
426 1 : hccp_err("qp_cb is NULL or qp_cb->channel is NULL");
427 1 : return;
428 : }
429 :
430 1 : (void)RsIbvDestroyQp(qpCb->ibQp);
431 1 : RsIbvAckCqEvents(qpCb->recvCq.ibCq, qpCb->recvCq.numEvents);
432 1 : qpCb->recvCq.numEvents = 0;
433 1 : (void)RsIbvDestroyCq(qpCb->recvCq.ibCq);
434 1 : (void)RsEpollCtl(rscb->connCb.epollfd, EPOLL_CTL_DEL, qpCb->channel->fd, EPOLLIN | EPOLLRDHUP);
435 1 : (void)RsIbvDestroyCompChannel(qpCb->channel);
436 1 : qpCb->channel = NULL;
437 1 : (void)RsIbvDestroyCq(qpCb->sendCq.ibCq);
438 : }
439 :
440 4 : STATIC int RsPingPongInitLocalInfo(struct rs_cb *rscb, struct PingInitAttr *attr, struct PingInitInfo *info,
441 : struct RsPingCtxCb *pingCb)
442 : {
443 : int ret;
444 :
445 4 : ret = RsPingCommonInitLocalQp(rscb, pingCb, &attr->client, &pingCb->pingQp);
446 4 : CHK_PRT_RETURN(ret != 0, hccp_err("init ping_qp failed, ret:%d", ret), ret);
447 3 : info->client.version = 0;
448 3 : (void)memcpy_s(&info->client.rdma.gid, sizeof(union HccpGid), &pingCb->rdevCb.gid, sizeof(union ibv_gid));
449 3 : info->client.rdma.qpn = pingCb->pingQp.ibQp->qp_num;
450 3 : info->client.rdma.qkey = pingCb->pingQp.qkey;
451 :
452 3 : ret = RsPingCommonInitLocalQp(rscb, pingCb, &attr->server, &pingCb->pongQp);
453 3 : if (ret != 0) {
454 0 : hccp_err("init pong_qp failed, ret:%d", ret);
455 0 : goto init_pong_qp_fail;
456 : }
457 3 : info->server.version = 0;
458 3 : (void)memcpy_s(&info->server.rdma.gid, sizeof(union HccpGid), &pingCb->rdevCb.gid, sizeof(union ibv_gid));
459 3 : info->server.rdma.qpn = pingCb->pongQp.ibQp->qp_num;
460 3 : info->server.rdma.qkey = pingCb->pongQp.qkey;
461 :
462 3 : ret = RsPingPongInitLocalBuffer(rscb, attr, info, pingCb);
463 3 : if (ret != 0) {
464 1 : hccp_err("init buffer failed, ret:%d", ret);
465 1 : goto init_buffer_fail;
466 : }
467 :
468 2 : ret = RsPingCommonInitPostRecvAll(&pingCb->pingQp);
469 2 : if (ret != 0) {
470 1 : hccp_err("ping_qp post recv failed, ret:%d", ret);
471 1 : goto post_recv_fail;
472 : }
473 1 : ret = RsPingCommonInitPostRecvAll(&pingCb->pongQp);
474 1 : if (ret != 0) {
475 0 : hccp_err("pong_qp post recv failed, ret:%d", ret);
476 0 : goto post_recv_fail;
477 : }
478 :
479 1 : return 0;
480 :
481 1 : post_recv_fail:
482 1 : RsPingCommonDeinitLocalBuffer(pingCb);
483 2 : init_buffer_fail:
484 2 : RsPingCommonDeinitLocalQp(rscb, pingCb, &pingCb->pongQp);
485 2 : init_pong_qp_fail:
486 2 : RsPingCommonDeinitLocalQp(rscb, pingCb, &pingCb->pingQp);
487 2 : return ret;
488 : }
489 :
490 1 : STATIC int RsPingRocePingCbInit(unsigned int phyId, struct PingInitAttr *attr, struct PingInitInfo *info,
491 : unsigned int *devIndex, struct RsPingCtxCb *pingCb)
492 : {
493 1 : struct rdev *rdevInfo = &attr->dev.rdma;
494 1 : struct rs_cb *rscb = NULL;
495 : int ret;
496 :
497 1 : ret = RsGetRsCb(phyId, &rscb);
498 1 : CHK_PRT_RETURN(ret != 0, hccp_err("RsGetRsCb failed, phyId[%u] invalid, ret %d", phyId, ret), ret);
499 :
500 : // prepare input attr
501 1 : pingCb->rdevCb.ip.family = (uint32_t)rdevInfo->family;
502 1 : pingCb->rdevCb.ip.binAddr = rdevInfo->localIp;
503 1 : ret = RsInetNtop(rdevInfo->family, &rdevInfo->localIp, pingCb->rdevCb.ip.readAddr, RS_MAX_IP_LEN);
504 1 : CHK_PRT_RETURN(ret != 0, hccp_err("rs_inet_ntop failed, ret %d", ret), -EINVAL);
505 1 : (void)memcpy_s(&pingCb->commInfo, sizeof(struct PingLocalCommInfo), &attr->commInfo,
506 : sizeof(struct PingLocalCommInfo));
507 :
508 : // open device & alloc global pd
509 1 : pingCb->rdevCb.devList = RsIbvGetDeviceList(&pingCb->rdevCb.devNum);
510 1 : if (pingCb->rdevCb.devList == NULL || pingCb->rdevCb.devNum == 0) {
511 0 : hccp_err("dev_list is NULL or dev_num[%d] is 0", pingCb->rdevCb.devNum);
512 0 : ret = -ENODEV;
513 0 : goto get_device_list_fail;
514 : }
515 :
516 1 : pingCb->rdevCb.ibPort = RS_PORT_DEF;
517 1 : ret = RsPingCbGetIbCtxAndIndex(rdevInfo, pingCb);
518 1 : if (ret != 0) {
519 1 : hccp_err("rs_ping_cb_get_ib_ctx_and_index failed, ret:%d", ret);
520 1 : goto get_ib_ctx_and_index_fail;
521 : }
522 :
523 0 : pingCb->rdevCb.ibPd = RsIbvAllocPd(pingCb->rdevCb.ibCtx);
524 0 : if (pingCb->rdevCb.ibPd == NULL) {
525 0 : hccp_err("rs_ibv_alloc_pd failed, errno:%d", errno);
526 0 : ret = -ENOMEM;
527 0 : goto alloc_pd_fail;
528 : }
529 :
530 : // init cq & qp & mr info, prepare output info
531 0 : info->version = 0;
532 0 : ret = RsPingPongInitLocalInfo(rscb, attr, info, pingCb);
533 0 : if (ret != 0) {
534 0 : hccp_err("rs_ping_pong_init_local_info failed, ret=%d phyId:%u", ret, rdevInfo->phyId);
535 0 : goto init_local_info_fail;
536 : }
537 :
538 0 : *devIndex = pingCb->devIndex;
539 0 : return 0;
540 :
541 0 : init_local_info_fail:
542 0 : (void)RsIbvDeallocPd(pingCb->rdevCb.ibPd);
543 0 : alloc_pd_fail:
544 0 : (void)RsIbvCloseDevice(pingCb->rdevCb.ibCtx);
545 1 : get_ib_ctx_and_index_fail:
546 1 : RsIbvFreeDeviceList(pingCb->rdevCb.devList);
547 1 : get_device_list_fail:
548 1 : (void)pthread_mutex_destroy(&pingCb->pingMutex);
549 1 : (void)pthread_mutex_destroy(&pingCb->pongMutex);
550 1 : return ret;
551 : }
552 :
553 6 : STATIC bool RsPingCommonCompareRdmaInfo(struct PingQpInfo *a, struct PingQpInfo *b)
554 : {
555 6 : if (a->rdma.qpn != b->rdma.qpn) {
556 1 : return false;
557 : }
558 5 : if (a->rdma.qkey != b->rdma.qkey) {
559 1 : return false;
560 : }
561 4 : if (memcmp(&a->rdma.gid, &b->rdma.gid, sizeof(union HccpGid)) != 0) {
562 1 : return false;
563 : }
564 3 : return true;
565 : }
566 :
567 3 : STATIC int RsPingRoceFindTargetNode(struct RsPingCtxCb *pingCb, struct PingQpInfo *target,
568 : struct RsPingTargetInfo **node)
569 : {
570 3 : struct RsPingTargetInfo *targetNext = NULL;
571 3 : struct RsPingTargetInfo *targetCurr = NULL;
572 :
573 3 : RS_PTHREAD_MUTEX_LOCK(&pingCb->pingMutex);
574 3 : RS_LIST_GET_HEAD_ENTRY(targetCurr, targetNext, &pingCb->pingList, list, struct RsPingTargetInfo);
575 3 : for (; (&targetCurr->list) != &pingCb->pingList;
576 0 : targetCurr = targetNext, targetNext = list_entry(targetNext->list.next, struct RsPingTargetInfo, list)) {
577 1 : if (RsPingCommonCompareRdmaInfo(&targetCurr->qpInfo, target)) {
578 1 : *node = targetCurr;
579 1 : RS_PTHREAD_MUTEX_ULOCK(&pingCb->pingMutex);
580 1 : return 0;
581 : }
582 : }
583 2 : RS_PTHREAD_MUTEX_ULOCK(&pingCb->pingMutex);
584 :
585 2 : hccp_info("ping target node for qpn:%u gid:%016llx:%016llx not found", target->rdma.qpn,
586 : target->rdma.gid.global.subnetPrefix, target->rdma.gid.global.interfaceId);
587 2 : return -ENODEV;
588 : }
589 :
590 0 : STATIC int RsPingCommonCreateAh(struct RsPingCtxCb *pingCb, struct PingLocalCommInfo *localInfo,
591 : struct PingQpInfo *remoteInfo, struct ibv_ah **ah)
592 : {
593 0 : struct ibv_exp_ah_attr attrx = { 0 };
594 0 : struct ibv_global_route grh = { 0 };
595 0 : struct ibv_ah_attr attr = { 0 };
596 0 : struct ibv_ah *ahTmp = NULL;
597 0 : int ret = 0;
598 :
599 0 : (void)memcpy_s(&grh.dgid, sizeof(union ibv_gid), &remoteInfo->rdma.gid, sizeof(union HccpGid));
600 0 : grh.flow_label = localInfo->rdma.flowLabel;
601 0 : grh.sgid_index = (uint8_t)pingCb->rdevCb.gidIdx;
602 0 : grh.hop_limit = localInfo->rdma.hopLimit;
603 0 : grh.traffic_class = localInfo->rdma.qosAttr.tc;
604 :
605 0 : attr.grh = grh;
606 0 : attr.sl = localInfo->rdma.qosAttr.sl;
607 0 : attr.is_global = 1;
608 0 : attr.port_num = pingCb->rdevCb.ibPort;
609 0 : attrx.attr = attr;
610 0 : attrx.udp_sport = localInfo->rdma.udpSport;
611 :
612 0 : hccp_dbg("remote_qpn:%u flow_label:%u sgid_index:%u hop_limit:%u traffic_class:%u sl:%u is_global:%u "
613 : "port_num:%u udp_sport:%u", remoteInfo->rdma.qpn, grh.flow_label, grh.sgid_index, grh.hop_limit,
614 : grh.traffic_class, attr.sl, attr.is_global, attr.port_num, attrx.udp_sport);
615 :
616 0 : ahTmp = RsIbvExpCreateAh(pingCb->rdevCb.ibPd, &attrx);
617 0 : if (ahTmp == NULL) {
618 0 : ret = -EFAULT;
619 0 : hccp_err("rs_ibv_exp_create_ah failed, errno:%d", errno);
620 0 : return ret;
621 : }
622 :
623 0 : *ah = ahTmp;
624 0 : return ret;
625 : }
626 :
627 1 : STATIC int RsPingRoceAllocTargetNode(struct RsPingCtxCb *pingCb, struct PingTargetInfo *target,
628 : struct RsPingTargetInfo **node)
629 : {
630 1 : struct RsPingTargetInfo *targetInfo = NULL;
631 : int ret;
632 :
633 1 : targetInfo = (struct RsPingTargetInfo *)calloc(1, sizeof(struct RsPingTargetInfo));
634 1 : CHK_PRT_RETURN(targetInfo == NULL, hccp_err("calloc target_info fail! errno:%d", errno), -ENOMEM);
635 :
636 1 : ret = pthread_mutex_init(&targetInfo->tripMutex, NULL);
637 1 : if (ret != 0) {
638 0 : hccp_err("pthread_mutex_init trip_mutex failed, ret:%d", ret);
639 0 : goto free_target_info;
640 : }
641 :
642 1 : targetInfo->payloadSize = target->payload.size;
643 1 : if (target->payload.size > 0) {
644 1 : targetInfo->payloadBuffer = (char *)calloc(1, target->payload.size);
645 1 : if (targetInfo->payloadBuffer == NULL) {
646 0 : hccp_err("calloc payload_buffer fail! size:%u errno:%d", target->payload.size, errno);
647 0 : ret = -ENOMEM;
648 0 : goto free_trip_mutex;
649 : }
650 1 : (void)memcpy_s(targetInfo->payloadBuffer, target->payload.size, target->payload.buffer, target->payload.size);
651 : }
652 :
653 1 : (void)memcpy_s(&targetInfo->qpInfo, sizeof(struct PingQpInfo),
654 1 : &target->remoteInfo.qpInfo, sizeof(struct PingQpInfo));
655 1 : ret = RsPingCommonCreateAh(pingCb, &target->localInfo, &target->remoteInfo.qpInfo, &targetInfo->ah);
656 1 : if (ret != 0) {
657 0 : hccp_err("rs_ping_common_create_ah fail! ret:%d", ret);
658 0 : goto free_payload_buffer;
659 : }
660 :
661 1 : targetInfo->resultSummary.rttMin = ~0;
662 1 : targetInfo->state = RS_PING_PONG_TARGET_READY;
663 1 : *node = targetInfo;
664 :
665 1 : return 0;
666 0 : free_payload_buffer:
667 0 : if (target->payload.size > 0 && targetInfo->payloadBuffer != NULL) {
668 0 : free(targetInfo->payloadBuffer);
669 0 : targetInfo->payloadBuffer = NULL;
670 : }
671 0 : free_trip_mutex:
672 0 : (void)pthread_mutex_destroy(&targetInfo->tripMutex);
673 0 : free_target_info:
674 0 : free(targetInfo);
675 0 : targetInfo = NULL;
676 0 : return ret;
677 : }
678 :
679 1 : STATIC void RsPingRoceResetRecvBuffer(struct RsPingCtxCb *pingCb)
680 : {
681 1 : RS_PTHREAD_MUTEX_LOCK(&pingCb->pongQp.recvMrCb.mutex);
682 1 : (void)memset_s((void *)(uintptr_t)pingCb->pongQp.recvMrCb.addr, pingCb->pongQp.recvMrCb.len,
683 : 0, pingCb->pongQp.recvMrCb.len);
684 1 : RS_PTHREAD_MUTEX_ULOCK(&pingCb->pongQp.recvMrCb.mutex);
685 1 : }
686 :
687 2 : STATIC void RsPingQpBuildUpWr(struct RsPingTargetInfo *target, struct ibv_sge *list, struct ibv_send_wr *wr)
688 : {
689 2 : wr->wr_id = target->uuid;
690 2 : wr->next = NULL;
691 2 : wr->sg_list = list;
692 2 : wr->num_sge = 1;
693 2 : wr->opcode = IBV_WR_SEND;
694 2 : wr->send_flags = IBV_SEND_SIGNALED;
695 2 : wr->wr.ud.ah = target->ah;
696 2 : wr->wr.ud.remote_qpn = target->qpInfo.rdma.qpn;
697 2 : wr->wr.ud.remote_qkey = target->qpInfo.rdma.qkey;
698 2 : }
699 :
700 2 : STATIC int RsPingRocePostSend(struct RsPingCtxCb *pingCb, struct RsPingTargetInfo *target)
701 : {
702 2 : struct RsPingPayloadHeader *header = NULL;
703 2 : struct ibv_send_wr *badWr = NULL;
704 2 : struct timeval timestamp = { 0 };
705 2 : struct ibv_send_wr wr = { 0 };
706 2 : struct ibv_sge list = { 0 };
707 : uint32_t sgeIdx;
708 2 : int ret = 0;
709 :
710 2 : hccp_dbg("target uuid:0x%llx state:%d payload_size:%u qpn:%u gid:%016llx:%016llx",
711 : target->uuid, target->state, target->payloadSize, target->qpInfo.rdma.qpn,
712 : target->qpInfo.rdma.gid.global.subnetPrefix, target->qpInfo.rdma.gid.global.interfaceId);
713 :
714 2 : RS_PTHREAD_MUTEX_LOCK(&pingCb->pingQp.sendMrCb.mutex);
715 2 : sgeIdx = pingCb->pingQp.sendMrCb.sgeIdx;
716 2 : (void)memcpy_s(&list, sizeof(struct ibv_sge),
717 2 : &pingCb->pingQp.sendMrCb.sgeList[sgeIdx], sizeof(struct ibv_sge));
718 2 : pingCb->pingQp.sendMrCb.sgeIdx = (sgeIdx + 1) % pingCb->pingQp.sendMrCb.sgeNum;
719 2 : RS_PTHREAD_MUTEX_ULOCK(&pingCb->pingQp.sendMrCb.mutex);
720 :
721 : // prepare ping_qp send buffer
722 2 : (void)memset_s((void *)(uintptr_t)list.addr, list.length, 0, list.length);
723 2 : header = (struct RsPingPayloadHeader *)(uintptr_t)list.addr;
724 2 : header->type = RS_PING_TYPE_ROCE_DETECT;
725 2 : (void)memcpy_s(&header->server.rdma.gid, sizeof(union HccpGid), &pingCb->rdevCb.gid, sizeof(union ibv_gid));
726 2 : header->server.rdma.qpn = pingCb->pongQp.ibQp->qp_num;
727 2 : header->server.rdma.qkey = pingCb->pongQp.qkey;
728 2 : (void)memcpy_s(&header->target, sizeof(struct PingQpInfo), &target->qpInfo, sizeof(struct PingQpInfo));
729 :
730 2 : if (target->payloadSize > 0) {
731 2 : ret = memcpy_s((void *)(uintptr_t)(list.addr + RS_PING_PAYLOAD_HEADER_RESV_CUSTOM),
732 2 : (list.length - RS_PING_PAYLOAD_HEADER_RESV_CUSTOM),
733 2 : (void *)target->payloadBuffer, target->payloadSize);
734 2 : CHK_PRT_RETURN(ret != 0, hccp_err("memcpy_s buffer payload_size:%u list.length:%u failed, ret:%d",
735 : target->payloadSize, (list.length - RS_PING_PAYLOAD_HEADER_RESV_CUSTOM), ret), -ESAFEFUNC);
736 : }
737 2 : list.length = RS_PING_PAYLOAD_HEADER_RESV_CUSTOM + target->payloadSize;
738 :
739 2 : RsPingQpBuildUpWr(target, &list, &wr);
740 :
741 : // record timestamp t1
742 2 : (void)gettimeofday(×tamp, NULL);
743 2 : header->timestamp.tvSec1 = (uint64_t)timestamp.tv_sec;
744 2 : header->timestamp.tvUsec1 = (uint64_t)timestamp.tv_usec;
745 2 : header->taskId = pingCb->taskId;
746 2 : header->magic = 0x55AA;
747 :
748 2 : ret = RsIbvPostSend(pingCb->pingQp.ibQp, &wr, &badWr);
749 2 : if (ret != 0) {
750 1 : hccp_err("rs_ibv_post_send qpn:%u failed, ret:%d", pingCb->pingQp.ibQp->qp_num, ret);
751 1 : RS_PTHREAD_MUTEX_LOCK(&target->tripMutex);
752 1 : target->state = RS_PING_PONG_TARGET_ERROR;
753 1 : RS_PTHREAD_MUTEX_ULOCK(&target->tripMutex);
754 : }
755 2 : return ret;
756 : }
757 :
758 3 : STATIC int RsPingRocePollScq(struct RsPingCtxCb *pingCb, struct RsPingTargetInfo *target)
759 : {
760 3 : struct ibv_wc wc = { 0 };
761 : int polledCnt;
762 :
763 3 : polledCnt = RsIbvPollCq(pingCb->pingQp.sendCq.ibCq, 1, &wc);
764 3 : if (polledCnt != 1) {
765 1 : hccp_err("uuid:0x%llx rs_ibv_poll_cq polled_cnt:%d", target->uuid, polledCnt);
766 1 : target->state = RS_PING_PONG_TARGET_ERROR;
767 1 : return -ENODATA;
768 : }
769 2 : if (wc.status != IBV_WC_SUCCESS) {
770 1 : target->state = RS_PING_PONG_TARGET_ERROR;
771 1 : hccp_err("wr_id:0x%llx error cqe %s(%d)", wc.wr_id, RsIbvWcStatusStr(wc.status), wc.status);
772 1 : return -EOPENSRC;
773 : }
774 1 : return 0;
775 : }
776 :
777 6 : STATIC int RsPingRocePollRcq(struct RsPingCtxCb *pingCb, int *polledCnt, struct timeval *timestamp2)
778 : {
779 6 : struct ibv_cq *evCq = NULL;
780 6 : void *evCtx = NULL;
781 : int ret;
782 :
783 : // record timestamp t2
784 6 : (void)gettimeofday(timestamp2, NULL);
785 :
786 6 : ret = RsIbvGetCqEvent(pingCb->pingQp.channel, &evCq, &evCtx);
787 6 : if (ret != 0) {
788 1 : hccp_err("rs_ibv_get_cq_event ping_qp.channel failed, ret:%d", ret);
789 1 : return -EOPENSRC;
790 : }
791 :
792 5 : if (evCq != pingCb->pingQp.recvCq.ibCq) {
793 0 : hccp_err("CQ event for unknown CQ");
794 0 : return -EOPENSRC;
795 : }
796 5 : pingCb->pingQp.recvCq.numEvents++;
797 :
798 5 : *polledCnt = RsIbvPollCq(evCq, pingCb->pingQp.recvCq.maxRecvWcNum, gPingQpRecvWc);
799 5 : CHK_PRT_RETURN(*polledCnt > pingCb->pingQp.recvCq.maxRecvWcNum || *polledCnt < 0,
800 : hccp_err("ping_poll_rcq failed, ret:%d", *polledCnt), -EOPENSRC);
801 :
802 4 : return 0;
803 : }
804 :
805 2 : STATIC int RsPingCommonPollScq(struct RsPingLocalQpCb *qpCb)
806 : {
807 2 : struct ibv_wc wc = { 0 };
808 : int polledCnt;
809 :
810 2 : polledCnt = RsIbvPollCq(qpCb->sendCq.ibCq, 1, &wc);
811 2 : if (polledCnt < 0) {
812 1 : hccp_warn("rs_ibv_poll_cq unsuccessful, polledCnt:%d", polledCnt);
813 1 : } else if (polledCnt > 0) {
814 1 : if (wc.status != IBV_WC_SUCCESS) {
815 0 : hccp_err("wr_id:0x%llx error cqe %s(%d)", wc.wr_id, RsIbvWcStatusStr(wc.status), wc.status);
816 0 : return -EOPENSRC;
817 : }
818 : }
819 :
820 2 : return 0;
821 : }
822 :
823 2 : STATIC int RsPongFindTargetNode(struct RsPingCtxCb *pingCb, struct PingQpInfo *target,
824 : struct RsPongTargetInfo **node)
825 : {
826 2 : struct RsPongTargetInfo *targetNext = NULL;
827 2 : struct RsPongTargetInfo *targetCurr = NULL;
828 :
829 2 : RS_CHECK_POINTER_NULL_WITH_RET(pingCb);
830 2 : RS_PTHREAD_MUTEX_LOCK(&pingCb->pongMutex);
831 2 : RS_LIST_GET_HEAD_ENTRY(targetCurr, targetNext, &pingCb->pongList, list, struct RsPongTargetInfo);
832 2 : for (; (&targetCurr->list) != &pingCb->pongList;
833 0 : targetCurr = targetNext, targetNext = list_entry(targetNext->list.next, struct RsPongTargetInfo, list)) {
834 1 : if (RsPingCommonCompareRdmaInfo(&targetCurr->qpInfo, target)) {
835 1 : *node = targetCurr;
836 1 : RS_PTHREAD_MUTEX_ULOCK(&pingCb->pongMutex);
837 1 : return 0;
838 : }
839 : }
840 1 : RS_PTHREAD_MUTEX_ULOCK(&pingCb->pongMutex);
841 :
842 1 : hccp_info("pong target node for qpn:%u gid:%016llx:%016llx not found", target->rdma.qpn,
843 : target->rdma.gid.global.subnetPrefix, target->rdma.gid.global.interfaceId);
844 1 : return -ENODEV;
845 : }
846 :
847 2 : STATIC int RsPongFindAllocTargetNode(struct RsPingCtxCb *pingCb, struct PingQpInfo *target,
848 : struct RsPongTargetInfo **node)
849 : {
850 2 : struct RsPongTargetInfo *targetInfo = NULL;
851 : int ret;
852 :
853 2 : ret = RsPongFindTargetNode(pingCb, target, node);
854 2 : if (ret == 0 && (*node)->state == RS_PING_PONG_TARGET_READY) {
855 0 : return 0;
856 2 : } else if (ret == 0) {
857 1 : targetInfo = *node;
858 1 : hccp_info("delete pong target uuid:0x%llx state:%d, realloc again", targetInfo->uuid, targetInfo->state);
859 1 : RsListDel(&targetInfo->list);
860 1 : if (targetInfo->ah) {
861 0 : (void)RsIbvDestroyAh(targetInfo->ah);
862 : }
863 1 : free(targetInfo);
864 1 : targetInfo = NULL;
865 : }
866 :
867 2 : targetInfo = (struct RsPongTargetInfo *)calloc(1, sizeof(struct RsPongTargetInfo));
868 2 : CHK_PRT_RETURN(targetInfo == NULL, hccp_err("calloc target_info fail! errno:%d", errno), -ENOMEM);
869 :
870 2 : (void)memcpy_s(&targetInfo->qpInfo, sizeof(struct PingQpInfo), target, sizeof(struct PingQpInfo));
871 2 : ret = RsPingCommonCreateAh(pingCb, &pingCb->commInfo, target, &targetInfo->ah);
872 2 : if (ret != 0) {
873 1 : hccp_err("rs_ping_common_create_ah fail! ret:%d", ret);
874 1 : goto free_target_info;
875 : }
876 :
877 1 : targetInfo->state = RS_PING_PONG_TARGET_READY;
878 1 : *node = targetInfo;
879 :
880 1 : RS_PTHREAD_MUTEX_LOCK(&pingCb->pongMutex);
881 1 : targetInfo->uuid = (uint64_t)pingCb->pongNum << 32U;
882 1 : RsListAddTail(&targetInfo->list, &pingCb->pongList);
883 1 : pingCb->pongNum++;
884 1 : RS_PTHREAD_MUTEX_ULOCK(&pingCb->pongMutex);
885 :
886 1 : return 0;
887 :
888 1 : free_target_info:
889 1 : free(targetInfo);
890 1 : return ret;
891 : }
892 :
893 4 : STATIC int RsPongPostSend(struct RsPingCtxCb *pingCb, struct ibv_wc *wc, struct timeval *timestamp2)
894 : {
895 4 : struct RsPongTargetInfo *targetInfo = NULL;
896 4 : struct RsPingPayloadHeader *header = NULL;
897 4 : struct ibv_send_wr *badWr = NULL;
898 4 : struct timeval timestamp3 = { 0 };
899 4 : struct ibv_sge recvList = { 0 };
900 4 : struct ibv_sge sendList = { 0 };
901 4 : struct ibv_send_wr wr = { 0 };
902 : uint32_t recvSgeIdx;
903 : uint32_t sendSgeIdx;
904 4 : int ret = 0;
905 :
906 : // poll send cq
907 4 : (void)RsPingCommonPollScq(&pingCb->pongQp);
908 :
909 : // handle detect packet & send response packet
910 4 : recvSgeIdx = (uint32_t)wc->wr_id;
911 4 : if (recvSgeIdx >= pingCb->pingQp.recvMrCb.sgeNum) {
912 1 : hccp_err("param err recv_sge_idx:%u >= sge_num:%u", recvSgeIdx, pingCb->pingQp.recvMrCb.sgeNum);
913 1 : return -EIO;
914 : }
915 3 : (void)memcpy_s(&recvList, sizeof(struct ibv_sge),
916 3 : &pingCb->pingQp.recvMrCb.sgeList[recvSgeIdx], sizeof(struct ibv_sge));
917 :
918 3 : RS_PTHREAD_MUTEX_LOCK(&pingCb->pongQp.sendMrCb.mutex);
919 3 : sendSgeIdx = pingCb->pongQp.sendMrCb.sgeIdx;
920 3 : (void)memcpy_s(&sendList, sizeof(struct ibv_sge),
921 3 : &pingCb->pongQp.sendMrCb.sgeList[sendSgeIdx], sizeof(struct ibv_sge));
922 3 : pingCb->pongQp.sendMrCb.sgeIdx = (sendSgeIdx + 1) % pingCb->pongQp.sendMrCb.sgeNum;
923 3 : RS_PTHREAD_MUTEX_ULOCK(&pingCb->pongQp.sendMrCb.mutex);
924 :
925 : // UD consume 40 Bytes for GRH
926 3 : if (wc->byte_len < RS_PING_PAYLOAD_HEADER_RESV_GRH || wc->byte_len > PING_TOTAL_PAYLOAD_MAX_SIZE) {
927 0 : hccp_err("param err wc->byte_len:%u < %u or wc->byte_len:%u > %u", wc->byte_len,
928 : RS_PING_PAYLOAD_HEADER_RESV_GRH, wc->byte_len, PING_TOTAL_PAYLOAD_MAX_SIZE);
929 0 : return -EIO;
930 : }
931 3 : ret = memcpy_s((void *)(uintptr_t)sendList.addr, sendList.length,
932 3 : (void *)(uintptr_t)(recvList.addr + RS_PING_PAYLOAD_HEADER_RESV_GRH),
933 3 : wc->byte_len - RS_PING_PAYLOAD_HEADER_RESV_GRH);
934 3 : CHK_PRT_RETURN(ret != 0, hccp_err("memcpy_s buffer wc->byte_len:%u send_list.length:%u failed, ret:%d",
935 : wc->byte_len, sendList.length, ret), -ESAFEFUNC);
936 3 : sendList.length = wc->byte_len - RS_PING_PAYLOAD_HEADER_RESV_GRH;
937 3 : header = (struct RsPingPayloadHeader *)(uintptr_t)sendList.addr;
938 3 : header->type = RS_PING_TYPE_ROCE_RESPONSE;
939 :
940 3 : ret = RsPongFindAllocTargetNode(pingCb, &header->server, &targetInfo);
941 3 : if (ret != 0) {
942 1 : hccp_err("rs_pong_find_alloc_target_node failed, ret:%d", ret);
943 1 : return ret;
944 : }
945 :
946 2 : wr.wr_id = targetInfo->uuid;
947 2 : wr.next = NULL;
948 2 : wr.sg_list = &sendList;
949 2 : wr.num_sge = 1;
950 2 : wr.opcode = IBV_WR_SEND;
951 2 : wr.send_flags = IBV_SEND_SIGNALED;
952 2 : wr.wr.ud.ah = targetInfo->ah;
953 2 : wr.wr.ud.remote_qpn = targetInfo->qpInfo.rdma.qpn;
954 2 : wr.wr.ud.remote_qkey = targetInfo->qpInfo.rdma.qkey;
955 :
956 : // record timestamp t3
957 2 : (void)gettimeofday(×tamp3, NULL);
958 2 : header->timestamp.tvSec2 = (uint64_t)timestamp2->tv_sec;
959 2 : header->timestamp.tvUsec2 = (uint64_t)timestamp2->tv_usec;
960 2 : header->timestamp.tvSec3 = (uint64_t)timestamp3.tv_sec;
961 2 : header->timestamp.tvUsec3 = (uint64_t)timestamp3.tv_usec;
962 2 : header->magic = 0xAA55;
963 :
964 2 : ret = RsIbvPostSend(pingCb->pongQp.ibQp, &wr, &badWr);
965 2 : if (ret != 0) {
966 1 : targetInfo->state = RS_PING_PONG_TARGET_ERROR;
967 1 : hccp_err("rs_ibv_post_send failed, ret:%d", ret);
968 1 : return ret;
969 : }
970 :
971 1 : return ret;
972 : }
973 :
974 3 : STATIC void RsPongRoceHandleSend(struct RsPingCtxCb *pingCb, int polledCnt, struct timeval *timestamp2)
975 : {
976 3 : struct ibv_wc *wc = NULL;
977 : int ret, i;
978 :
979 3 : wc = gPingQpRecvWc;
980 6 : for (i = 0; i < polledCnt; i++) {
981 3 : if (wc[i].status != IBV_WC_SUCCESS) {
982 0 : hccp_err("wr_id:0x%llx error cqe %s(%d)", wc[i].wr_id, RsIbvWcStatusStr(wc[i].status), wc[i].status);
983 0 : continue;
984 : }
985 :
986 3 : ret = RsPongPostSend(pingCb, &wc[i], timestamp2);
987 3 : if (ret != 0) {
988 1 : hccp_err("rs_pong_post_send failed, wrId:0x%llx", wc[i].wr_id);
989 1 : continue;
990 : }
991 :
992 2 : ret = RsPingCommonPostRecv(&pingCb->pingQp);
993 2 : if (ret != 0) {
994 1 : hccp_err("rs_ping_common_post_recv failed, ret:%d", ret);
995 1 : continue;
996 : }
997 : }
998 :
999 3 : ret = RsIbvReqNotifyCq(pingCb->pingQp.recvCq.ibCq, 0);
1000 3 : if (ret != 0) {
1001 1 : hccp_err("rs_ibv_req_notify_cq failed, ret:%d", ret);
1002 : }
1003 :
1004 3 : return;
1005 : }
1006 :
1007 4 : STATIC int RsPongResolveResponsePacket(struct RsPingCtxCb *pingCb, uint32_t sgeIdx, struct timeval *timestamp4)
1008 : {
1009 4 : struct RsPingTargetInfo *targetInfo = NULL;
1010 4 : struct RsPingPayloadHeader *header = NULL;
1011 4 : struct ibv_sge *recvList = NULL;
1012 : uint32_t rtt;
1013 : int ret;
1014 :
1015 4 : recvList = &pingCb->pongQp.recvMrCb.sgeList[sgeIdx];
1016 : // UD consume 40 Bytes for GRH
1017 4 : header = (struct RsPingPayloadHeader *)(uintptr_t)(recvList->addr + RS_PING_PAYLOAD_HEADER_RESV_GRH);
1018 4 : if (header->taskId != pingCb->taskId) {
1019 1 : hccp_warn("drop received packet, recv_task_id:%u, curr_task_id:%u", header->taskId, pingCb->taskId);
1020 1 : return 0;
1021 : }
1022 :
1023 3 : header->timestamp.tvSec4 = (uint64_t)timestamp4->tv_sec;
1024 3 : header->timestamp.tvUsec4 = (uint64_t)timestamp4->tv_usec;
1025 3 : rtt = RsPingGetTripTime(&header->timestamp);
1026 3 : ret = RsPingRoceFindTargetNode(pingCb, &header->target, &targetInfo);
1027 3 : if (ret != 0) {
1028 1 : hccp_err("rs_ping_roce_find_target_node failed, ret:%d qpn:%u gid:%016llx:%016llx rtt:%u", ret,
1029 : header->target.rdma.qpn, header->target.rdma.gid.global.subnetPrefix,
1030 : header->target.rdma.gid.global.interfaceId, rtt);
1031 1 : return ret;
1032 : }
1033 :
1034 2 : (void)memset_s((void *)header, RS_PING_PAYLOAD_HEADER_MASK_SIZE, 0, RS_PING_PAYLOAD_HEADER_MASK_SIZE);
1035 2 : RS_PTHREAD_MUTEX_LOCK(&targetInfo->tripMutex);
1036 2 : targetInfo->resultSummary.recvCnt++;
1037 2 : targetInfo->resultSummary.taskId = header->taskId;
1038 : // rtt timeout, increase timeout_cnt
1039 2 : if ((targetInfo->resultSummary.taskAttr.timeoutInterval * RS_PING_MSEC_TO_USEC) < rtt) {
1040 1 : targetInfo->resultSummary.timeoutCnt++;
1041 1 : hccp_dbg("recv_cnt:%u timeout_interval:%u rtt:%u timeout_cnt:%u", targetInfo->resultSummary.recvCnt,
1042 : targetInfo->resultSummary.taskAttr.timeoutInterval, rtt, targetInfo->resultSummary.timeoutCnt);
1043 1 : RS_PTHREAD_MUTEX_ULOCK(&targetInfo->tripMutex);
1044 1 : return 0;
1045 : }
1046 :
1047 : // handle rtt_min, rtt_max, rtt_avg
1048 1 : if (targetInfo->resultSummary.rttMin > rtt) {
1049 0 : targetInfo->resultSummary.rttMin = rtt;
1050 : }
1051 1 : if (targetInfo->resultSummary.rttMax < rtt) {
1052 1 : targetInfo->resultSummary.rttMax = rtt;
1053 : }
1054 1 : if (targetInfo->resultSummary.rttAvg == 0) {
1055 1 : targetInfo->resultSummary.rttAvg = rtt;
1056 : }
1057 1 : targetInfo->resultSummary.rttAvg = (targetInfo->resultSummary.rttAvg + rtt) / 2U;
1058 1 : RS_PTHREAD_MUTEX_ULOCK(&targetInfo->tripMutex);
1059 1 : return 0;
1060 : }
1061 :
1062 6 : STATIC void RsPongRocePollRcq(struct RsPingCtxCb *pingCb)
1063 : {
1064 6 : struct timeval timestamp = { 0 };
1065 6 : struct ibv_cq *evCq = NULL;
1066 6 : struct ibv_wc *wc = NULL;
1067 : uint32_t recvSgeIdx;
1068 6 : void *evCtx = NULL;
1069 : int polledCnt, i;
1070 : int ret;
1071 :
1072 : // record timestamp t4
1073 6 : (void)gettimeofday(×tamp, NULL);
1074 :
1075 6 : ret = RsIbvGetCqEvent(pingCb->pongQp.channel, &evCq, &evCtx);
1076 6 : if (ret != 0) {
1077 1 : hccp_err("rs_ibv_get_cq_event pong_qp.channel failed, ret:%d", ret);
1078 1 : return;
1079 : }
1080 :
1081 5 : if (evCq != pingCb->pongQp.recvCq.ibCq) {
1082 0 : hccp_err("CQ event for unknown CQ");
1083 0 : return;
1084 : }
1085 5 : pingCb->pongQp.recvCq.numEvents++;
1086 :
1087 5 : polledCnt = RsIbvPollCq(evCq, pingCb->pongQp.recvCq.maxRecvWcNum, gPongQpRecvWc);
1088 5 : if (polledCnt > pingCb->pongQp.recvCq.maxRecvWcNum || polledCnt < 0) {
1089 2 : hccp_err("rs_ibv_poll_cq failed, ret:%d", polledCnt);
1090 2 : return;
1091 : }
1092 :
1093 3 : wc = gPongQpRecvWc;
1094 6 : for (i = 0; i < polledCnt; i++) {
1095 3 : if (wc[i].status != IBV_WC_SUCCESS) {
1096 0 : hccp_err("wr_id:0x%llx error cqe %s(%d)", wc[i].wr_id, RsIbvWcStatusStr(wc[i].status), wc[i].status);
1097 0 : continue;
1098 : }
1099 3 : recvSgeIdx = (uint32_t)wc[i].wr_id;
1100 3 : if (recvSgeIdx >= pingCb->pongQp.recvMrCb.sgeNum) {
1101 3 : hccp_err("param err recv_sge_idx:%u > sge_num:%u", recvSgeIdx, pingCb->pongQp.recvMrCb.sgeNum);
1102 3 : continue;
1103 : }
1104 :
1105 : // handle response packet result
1106 0 : ret = RsPongResolveResponsePacket(pingCb, recvSgeIdx, ×tamp);
1107 0 : if (ret != 0) {
1108 0 : continue;
1109 : }
1110 :
1111 0 : ret = RsPingCommonPostRecv(&pingCb->pongQp);
1112 0 : if (ret != 0) {
1113 0 : continue;
1114 : }
1115 : }
1116 :
1117 3 : ret = RsIbvReqNotifyCq(evCq, 0);
1118 3 : if (ret != 0) {
1119 1 : hccp_err("rs_ibv_req_notify_cq failed, ret:%d", ret);
1120 : }
1121 :
1122 3 : return;
1123 : }
1124 :
1125 2 : STATIC int RsPingRoceGetTargetResult(struct RsPingCtxCb *pingCb, struct PingTargetCommInfo *target,
1126 : struct PingResultInfo *result)
1127 : {
1128 2 : struct RsPingTargetInfo *targetInfo = NULL;
1129 : int ret;
1130 :
1131 2 : ret = RsPingRoceFindTargetNode(pingCb, &target->qpInfo, &targetInfo);
1132 2 : if (ret != 0) {
1133 1 : hccp_err("rs_ping_roce_find_target_node failed, ret:%d qpn:%u gid:%016llx:%016llx", ret,
1134 : target->qpInfo.rdma.qpn, target->qpInfo.rdma.gid.global.subnetPrefix,
1135 : target->qpInfo.rdma.gid.global.interfaceId);
1136 1 : return ret;
1137 : }
1138 :
1139 1 : (void)memcpy_s(&result->summary, sizeof(struct PingResultSummary), &targetInfo->resultSummary,
1140 : sizeof(struct PingResultSummary));
1141 1 : if (targetInfo->state == RS_PING_PONG_TARGET_FINISH) {
1142 0 : result->state = PING_RESULT_STATE_VALID;
1143 : } else {
1144 1 : result->state = PING_RESULT_STATE_INVALID;
1145 : }
1146 :
1147 1 : hccp_dbg("ip:0x%llx qpn:%u, state:%d sendCnt:%u recvCnt:%u timeoutCnt:%u rttMin:%u rttMax:%u rttAvg:%u",
1148 : target->ip.addr.s_addr, target->qpInfo.rdma.qpn, result->state, result->summary.sendCnt,
1149 : result->summary.recvCnt, result->summary.timeoutCnt, result->summary.rttMin, result->summary.rttMax,
1150 : result->summary.rttAvg);
1151 :
1152 1 : return 0;
1153 : }
1154 :
1155 1 : STATIC void RsPingRoceFreeTargetNode(struct RsPingCtxCb *pingCb, struct RsPingTargetInfo *targetInfo)
1156 : {
1157 1 : RS_PTHREAD_MUTEX_LOCK(&pingCb->pingMutex);
1158 1 : RsListDel(&targetInfo->list);
1159 1 : RS_PTHREAD_MUTEX_ULOCK(&pingCb->pingMutex);
1160 :
1161 1 : if (targetInfo->payloadSize > 0 && targetInfo->payloadBuffer != NULL) {
1162 1 : free(targetInfo->payloadBuffer);
1163 1 : targetInfo->payloadBuffer = NULL;
1164 : }
1165 :
1166 1 : if (targetInfo->ah) {
1167 0 : (void)RsIbvDestroyAh(targetInfo->ah);
1168 : }
1169 1 : return;
1170 : }
1171 :
1172 1 : STATIC void RsPingPongDelTargetList(struct RsPingCtxCb *pingCb)
1173 : {
1174 1 : struct RsPongTargetInfo *pongNext = NULL;
1175 1 : struct RsPingTargetInfo *pingNext = NULL;
1176 1 : struct RsPongTargetInfo *pongCurr = NULL;
1177 1 : struct RsPingTargetInfo *pingCurr = NULL;
1178 :
1179 : // del ping_list
1180 1 : RS_PTHREAD_MUTEX_LOCK(&pingCb->pingMutex);
1181 1 : RS_LIST_GET_HEAD_ENTRY(pingCurr, pingNext, &pingCb->pingList, list, struct RsPingTargetInfo);
1182 2 : for (; (&pingCurr->list) != &pingCb->pingList;
1183 1 : pingCurr = pingNext, pingNext = list_entry(pingNext->list.next, struct RsPingTargetInfo, list)) {
1184 1 : RsListDel(&pingCurr->list);
1185 1 : if (pingCurr->payloadSize > 0 && pingCurr->payloadBuffer != NULL) {
1186 0 : free(pingCurr->payloadBuffer);
1187 0 : pingCurr->payloadBuffer = NULL;
1188 : }
1189 1 : if (pingCurr->ah) {
1190 0 : (void)RsIbvDestroyAh(pingCurr->ah);
1191 : }
1192 1 : (void)pthread_mutex_destroy(&pingCurr->tripMutex);
1193 1 : free(pingCurr);
1194 1 : pingCurr = NULL;
1195 : }
1196 1 : RS_PTHREAD_MUTEX_ULOCK(&pingCb->pingMutex);
1197 :
1198 : // del pong_list
1199 1 : RS_PTHREAD_MUTEX_LOCK(&pingCb->pongMutex);
1200 1 : RS_LIST_GET_HEAD_ENTRY(pongCurr, pongNext, &pingCb->pongList, list, struct RsPongTargetInfo);
1201 2 : for (; (&pongCurr->list) != &pingCb->pongList;
1202 1 : pongCurr = pongNext, pongNext = list_entry(pongNext->list.next, struct RsPongTargetInfo, list)) {
1203 1 : RsListDel(&pongCurr->list);
1204 1 : if (pongCurr->ah) {
1205 0 : (void)RsIbvDestroyAh(pongCurr->ah);
1206 : }
1207 1 : free(pongCurr);
1208 1 : pongCurr = NULL;
1209 : }
1210 1 : RS_PTHREAD_MUTEX_ULOCK(&pingCb->pongMutex);
1211 1 : }
1212 :
1213 2 : STATIC void RsPingRocePingCbDeinit(unsigned int phyId, struct RsPingCtxCb *pingCb)
1214 : {
1215 2 : struct rs_cb *rscb = NULL;
1216 : int ret;
1217 :
1218 2 : ret = RsGetRsCb(phyId, &rscb);
1219 2 : if (ret != 0) {
1220 1 : hccp_err("RsGetRsCb failed, phyId[%u] invalid, ret %d", phyId, ret);
1221 1 : return;
1222 : }
1223 :
1224 1 : RS_PTHREAD_MUTEX_LOCK(&pingCb->pingMutex);
1225 1 : pingCb->taskStatus = RS_PING_TASK_RESET;
1226 1 : RS_PTHREAD_MUTEX_ULOCK(&pingCb->pingMutex);
1227 :
1228 1 : RsPingPongDelTargetList(pingCb);
1229 :
1230 1 : RsPingCommonDeinitLocalQp(rscb, pingCb, &pingCb->pongQp);
1231 1 : RsPingCommonDeinitLocalQp(rscb, pingCb, &pingCb->pingQp);
1232 1 : RsPingCommonDeinitLocalBuffer(pingCb);
1233 1 : (void)RsIbvDeallocPd(pingCb->rdevCb.ibPd);
1234 1 : (void)RsIbvCloseDevice(pingCb->rdevCb.ibCtx);
1235 1 : RsIbvFreeDeviceList(pingCb->rdevCb.devList);
1236 : }
1237 :
1238 2 : STATIC void RsPingRoceAddTargetSuccess(struct PingTargetInfo *target, struct RsPingTargetInfo *targetInfo)
1239 : {
1240 2 : hccp_info("target ip:0x%llx payload_size:%u add success, qpn:%u uuid:0x%llx",
1241 : target->remoteInfo.ip.addr.s_addr, target->payload.size, targetInfo->qpInfo.rdma.qpn, targetInfo->uuid);
1242 2 : }
1243 :
1244 1 : STATIC void RsPingRocePingCbInitSuccess(unsigned int phyId, struct PingInitAttr *attr, unsigned int devIndex)
1245 : {
1246 1 : hccp_run_info("ping_cb init success, phyId:%u, localIp:0x%x, devIndex:%u",
1247 : phyId, attr->dev.rdma.localIp.addr.s_addr, devIndex);
1248 1 : }
1249 :
1250 1 : STATIC void RsPingRoceCannotFindTargetNode(unsigned int i, int ret, struct PingTargetCommInfo target,
1251 : unsigned int phyId)
1252 : {
1253 1 : hccp_err("rs_ping_roce_find_target_node i:%u failed, ret:%d ip:0x%llx qpn:%u phyId:%u",i, ret,
1254 : target.ip.addr.s_addr, target.qpInfo.rdma.qpn, phyId);
1255 1 : }
1256 :
1257 : struct RsPingPongOps gRsPingRoceOps = {
1258 : .checkPingFd = RsPingRoceCheckFd,
1259 : .checkPongFd = RsPongRoceCheckFd,
1260 : .initPingCb = RsPingRocePingCbInit,
1261 : .pingFindTargetNode = RsPingRoceFindTargetNode,
1262 : .pingAllocTargetNode = RsPingRoceAllocTargetNode,
1263 : .resetRecvBuffer = RsPingRoceResetRecvBuffer,
1264 : .pingPostSend = RsPingRocePostSend,
1265 : .pingPollScq = RsPingRocePollScq,
1266 : .pingPollRcq = RsPingRocePollRcq,
1267 : .pongHandleSend = RsPongRoceHandleSend,
1268 : .pongPollRcq = RsPongRocePollRcq,
1269 : .getTargetResult = RsPingRoceGetTargetResult,
1270 : .pingFreeTargetNode = RsPingRoceFreeTargetNode,
1271 : .deinitPingCb = RsPingRocePingCbDeinit,
1272 : };
1273 :
1274 : struct RsPingPongDfx gRsPingRoceDfx = {
1275 : .addTargetSuccess = RsPingRoceAddTargetSuccess,
1276 : .initPingCbSuccess = RsPingRocePingCbInitSuccess,
1277 : .pingCannotFindTargetNode = RsPingRoceCannotFindTargetNode,
1278 : };
1279 :
1280 27 : struct RsPingPongOps *RsPingRoceGetOps(void) {
1281 27 : return &gRsPingRoceOps;
1282 : }
1283 :
1284 27 : struct RsPingPongDfx *RsPingRoceGetDfx(void) {
1285 27 : return &gRsPingRoceDfx;
1286 : }
|
