Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include <errno.h>
12 : #include <infiniband/verbs.h>
13 : #include "securec.h"
14 : #include "dl_hal_function.h"
15 : #include "dl_ibverbs_function.h"
16 : #include "hccp_common.h"
17 : #include "rs.h"
18 : #include "ra_rs_err.h"
19 : #include "rs_inner.h"
20 : #include "rs_epoll.h"
21 : #include "rs_drv_socket.h"
22 : #include "rs_drv_rdma.h"
23 : #include "rs_ping_inner.h"
24 : #ifndef HNS_ROCE_LLT
25 : #include <dlog_pub.h>
26 : #endif
27 : #include "rs_ping_roce.h"
28 :
29 : #define RS_PING_ROCE_RECV_WC_NUM 16
30 :
31 : struct ibv_wc gPingQpRecvWc[RS_PING_ROCE_RECV_WC_NUM] = {0};
32 : struct ibv_wc gPongQpRecvWc[RS_PING_ROCE_RECV_WC_NUM] = {0};
33 :
34 3 : STATIC bool RsPingRoceCheckFd(struct RsPingCtxCb *pingCb, int fd)
35 : {
36 3 : if (pingCb->pingQp.channel != NULL && pingCb->pingQp.channel->fd == fd) {
37 1 : hccp_dbg("ping_qp rq, channel->fd:%d poll cq", fd);
38 1 : return true;
39 : }
40 2 : return false;
41 : }
42 :
43 2 : STATIC bool RsPongRoceCheckFd(struct RsPingCtxCb *pingCb, int fd)
44 : {
45 2 : if (pingCb->pongQp.channel != NULL && pingCb->pongQp.channel->fd == fd) {
46 2 : hccp_dbg("pong_qp rq, channel->fd:%d poll cq", fd);
47 2 : return true;
48 : }
49 0 : return false;
50 : }
51 :
52 0 : STATIC int RsPingCbGetDevRdevIndex(struct RsPingCtxCb *pingCb, int index)
53 : {
54 : #ifdef CUSTOM_INTERFACE
55 0 : struct roce_dev_data rdevData = {0};
56 : int ret;
57 :
58 0 : if (RsIsCustomInterfaceSupported()) {
59 0 : RS_PTHREAD_MUTEX_LOCK(&pingCb->pingMutex);
60 0 : pingCb->rdevCb.devName = RsIbvGetDeviceName(pingCb->rdevCb.devList[index]);
61 0 : ret = RsRoceGetRoceDevData(pingCb->rdevCb.devName, &rdevData);
62 0 : if (ret != 0) {
63 0 : hccp_err("rs_roce_get_roce_dev_data failed, ret:%d, devName:%s", ret, pingCb->rdevCb.devName);
64 0 : RS_PTHREAD_MUTEX_ULOCK(&pingCb->pingMutex);
65 0 : return ret;
66 : }
67 0 : pingCb->devIndex = rdevData.rdev_index; // rdev_index is same to port_id
68 0 : RS_PTHREAD_MUTEX_ULOCK(&pingCb->pingMutex);
69 : }
70 : #endif
71 0 : return 0;
72 : }
73 :
74 2 : STATIC int RsPingCbGetIbCtxAndIndex(struct rdev *rdevInfo, struct RsPingCtxCb *pingCb)
75 : {
76 2 : struct ibv_context *ibCtx = NULL;
77 : int ret;
78 : int i;
79 :
80 4 : for (i = 0; (i < pingCb->rdevCb.devNum) && (pingCb->rdevCb.devList[i] != NULL); ++i) {
81 3 : ibCtx = RsIbvOpenDevice(pingCb->rdevCb.devList[i]);
82 3 : CHK_PRT_RETURN(ibCtx == NULL, hccp_err("ibv_open_device failed!"), -ENODEV);
83 3 : ret = RsQueryGid(*rdevInfo, ibCtx, pingCb->rdevCb.ibPort, &pingCb->rdevCb.gidIdx);
84 3 : if (ret == 0) {
85 1 : ret = RsPingCbGetDevRdevIndex(pingCb, i);
86 1 : if (ret != 0) {
87 0 : hccp_err("rs_ping_cb_get_dev_rdev_index failed, ret:%d", ret);
88 0 : RsIbvCloseDevice(ibCtx);
89 0 : return ret;
90 : }
91 1 : pingCb->rdevCb.ibCtx = ibCtx;
92 1 : ret = RsIbvQueryGid(ibCtx, pingCb->rdevCb.ibPort, pingCb->rdevCb.gidIdx, &pingCb->rdevCb.gid);
93 1 : if (ret != 0) {
94 1 : RsIbvCloseDevice(ibCtx);
95 1 : hccp_err("query gid failed gid_idx %d, ret %d", pingCb->rdevCb.gidIdx, ret);
96 1 : return -EOPENSRC;
97 : }
98 0 : return 0;
99 2 : } else if (ret == -EEXIST) {
100 2 : RsIbvCloseDevice(ibCtx);
101 : } else {
102 0 : hccp_err("rs_query_gid failed, ret:%d", ret);
103 0 : RsIbvCloseDevice(ibCtx);
104 0 : return ret;
105 : }
106 : }
107 :
108 1 : CHK_PRT_RETURN(i == pingCb->rdevCb.devNum,
109 : hccp_err("can not find ib_ctx for phyId[%u] local_ip[0x%x] "
110 : "in dev_list!",
111 : rdevInfo->phyId, rdevInfo->localIp.addr.s_addr),
112 : -ENODEV);
113 0 : return 0;
114 : }
115 :
116 3 : STATIC int RsPingCommonModifyLocalQp(struct RsPingCtxCb *pingCb, struct RsPingLocalQpCb *qpCb)
117 : {
118 : struct ibv_qp_init_attr initAttr;
119 3 : struct ibv_qp_attr attr = {0};
120 : int ret;
121 :
122 3 : ret = RsIbvQueryQp(qpCb->ibQp, &attr, IBV_QP_STATE, &initAttr);
123 3 : CHK_PRT_RETURN(ret != 0 || attr.qp_state != IBV_QPS_RESET,
124 : hccp_err("rs_ibv_query_qp qpn:%u fail, ret:%d attr.qp_state:%d != %d", qpCb->ibQp->qp_num, ret, attr.qp_state,
125 : IBV_QPS_RESET),
126 : -EOPENSRC);
127 :
128 2 : attr.qp_state = IBV_QPS_INIT;
129 2 : attr.pkey_index = 0;
130 2 : attr.port_num = pingCb->rdevCb.ibPort;
131 2 : attr.qkey = qpCb->qkey;
132 2 : ret = RsIbvModifyQp(qpCb->ibQp, &attr, IBV_QP_STATE | IBV_QP_PKEY_INDEX | IBV_QP_PORT | IBV_QP_QKEY);
133 2 : CHK_PRT_RETURN(ret != 0,
134 : hccp_err("rs_ibv_modify_qp qpn:%u to init fail, ret:%d, errno:%d", qpCb->ibQp->qp_num, ret, errno), -EOPENSRC);
135 :
136 1 : attr.qp_state = IBV_QPS_RTR;
137 1 : ret = RsIbvModifyQp(qpCb->ibQp, &attr, IBV_QP_STATE);
138 1 : CHK_PRT_RETURN(ret != 0,
139 : hccp_err("rs_ibv_modify_qp qpn:%u to rtr fail, ret:%d, errno:%d", qpCb->ibQp->qp_num, ret, errno), -EOPENSRC);
140 :
141 1 : attr.qp_state = IBV_QPS_RTS;
142 1 : attr.sq_psn = 0;
143 1 : ret = RsIbvModifyQp(qpCb->ibQp, &attr, IBV_QP_STATE | IBV_QP_SQ_PSN);
144 1 : CHK_PRT_RETURN(ret != 0,
145 : hccp_err("rs_ibv_modify_qp qpn:%u to rts fail, ret:%d, errno:%d", qpCb->ibQp->qp_num, ret, errno), -EOPENSRC);
146 :
147 1 : return 0;
148 : }
149 :
150 2 : STATIC int RsPingCommonInitLocalQp(struct rs_cb *rscb, struct RsPingCtxCb *pingCb, union PingQpAttr *attr,
151 : struct RsPingLocalQpCb *qpCb)
152 : {
153 2 : struct ibv_exp_qp_init_attr qpInitAttr = {0};
154 2 : struct rdma_lite_device_qp_attr qpResp = {0};
155 : int randNum;
156 : int ret;
157 :
158 2 : hccp_info("cq_attr{%d %d, %d %d}", attr->rdma.cqAttr.sendCqDepth, attr->rdma.cqAttr.sendCqCompVector,
159 : attr->rdma.cqAttr.recvCqDepth, attr->rdma.cqAttr.recvCqCompVector);
160 :
161 : // create send cq with attr
162 2 : qpCb->sendCq.depth = attr->rdma.cqAttr.sendCqDepth;
163 2 : qpCb->sendCq.compVector = attr->rdma.cqAttr.sendCqCompVector;
164 2 : qpCb->sendCq.ibCq = RsIbvCreateCq(pingCb->rdevCb.ibCtx, qpCb->sendCq.depth, NULL, NULL, qpCb->sendCq.compVector);
165 2 : qpCb->sendCq.maxRecvWcNum = RS_PING_ROCE_RECV_WC_NUM;
166 2 : ret = -errno;
167 2 : CHK_PRT_RETURN(qpCb->sendCq.ibCq == NULL, hccp_err("rs_ibv_create_cq send cq fail, ret:%d", ret), ret);
168 :
169 : // create channel & create recv cq with attr
170 1 : qpCb->channel = RsIbvCreateCompChannel(pingCb->rdevCb.ibCtx);
171 1 : if (qpCb->channel == NULL) {
172 0 : ret = -errno;
173 0 : hccp_err("rs_ibv_create_comp_channel failed! ret:%d", ret);
174 0 : goto create_channel_fail;
175 : }
176 1 : ret = RsEpollCtl(rscb->connCb.epollfd, EPOLL_CTL_ADD, qpCb->channel->fd, EPOLLIN | EPOLLRDHUP);
177 1 : if (ret != 0) {
178 0 : hccp_err("RsEpollCtl failed! epollfd:%d fd:%d ret:%d", rscb->connCb.epollfd, qpCb->channel->fd, ret);
179 0 : goto epoll_ctl_fail;
180 : }
181 1 : qpCb->recvCq.depth = attr->rdma.cqAttr.recvCqDepth;
182 1 : qpCb->recvCq.compVector = attr->rdma.cqAttr.recvCqCompVector;
183 1 : qpCb->recvCq.ibCq = RsIbvCreateCq(pingCb->rdevCb.ibCtx, qpCb->recvCq.depth, NULL, qpCb->channel,
184 : qpCb->recvCq.compVector);
185 1 : qpCb->recvCq.maxRecvWcNum = RS_PING_ROCE_RECV_WC_NUM;
186 1 : if (qpCb->recvCq.ibCq == NULL) {
187 0 : ret = -errno;
188 0 : hccp_err("rs_ibv_create_cq recv cq fail, ret:%d", ret);
189 0 : goto create_rcq_fail;
190 : }
191 :
192 : // create qp with attr
193 1 : (void)RsDrvGetRandomNum(&randNum);
194 : // clear bit IB_QP_SET_QKEY to avoid modify_qp to INIT failed
195 1 : qpCb->qkey = (uint32_t)(((uint32_t)randNum) & (~(1U << 31U)));
196 1 : (void)memcpy_s(&qpCb->qpCap, sizeof(struct ibv_qp_cap), &attr->rdma.qpAttr.cap, sizeof(struct ibv_qp_cap));
197 1 : qpCb->udpSport = attr->rdma.qpAttr.udpSport;
198 1 : qpInitAttr.attr.send_cq = qpCb->sendCq.ibCq;
199 1 : qpInitAttr.attr.recv_cq = qpCb->recvCq.ibCq;
200 1 : (void)memcpy_s(&qpInitAttr.attr.cap, sizeof(struct ibv_qp_cap), &qpCb->qpCap, sizeof(struct ibv_qp_cap));
201 1 : qpInitAttr.attr.qp_type = IBV_QPT_UD;
202 1 : qpInitAttr.udp_sport = attr->rdma.qpAttr.udpSport;
203 :
204 1 : hccp_info("qkey:%u udp_sport:%u qp_cap{%u %u %u %u %u}", qpCb->qkey, qpCb->udpSport,
205 : attr->rdma.qpAttr.cap.maxSendWr, attr->rdma.qpAttr.cap.maxRecvWr, attr->rdma.qpAttr.cap.maxSendSge,
206 : attr->rdma.qpAttr.cap.maxRecvSge, attr->rdma.qpAttr.cap.maxInlineData);
207 1 : qpCb->ibQp = RsIbvExpCreateQp(pingCb->rdevCb.ibPd, &qpInitAttr, &qpResp);
208 1 : if (qpCb->ibQp == NULL) {
209 1 : ret = -errno;
210 1 : hccp_err("rs_ibv_exp_create_qp qp fail, ret:%d", ret);
211 1 : goto create_qp_fail;
212 : }
213 :
214 0 : ret = RsPingCommonModifyLocalQp(pingCb, qpCb);
215 0 : if (ret != 0) {
216 0 : hccp_err("rs_ping_common_modify_local_qp failed, ret:%d", ret);
217 0 : goto modify_qp_fail;
218 : }
219 :
220 0 : ret = RsIbvReqNotifyCq(qpCb->recvCq.ibCq, 0);
221 0 : if (ret != 0) {
222 0 : hccp_err("rs_ibv_req_notify_cq failed, ret:%d", ret);
223 0 : goto modify_qp_fail;
224 : }
225 :
226 0 : hccp_run_info("qpn:%u create success, cqAttr{%d %d, %d %d} qkey:%u udpSport:%u qpCap{%u %u %u %u %u}",
227 : qpCb->ibQp->qp_num, attr->rdma.cqAttr.sendCqDepth, attr->rdma.cqAttr.sendCqCompVector,
228 : attr->rdma.cqAttr.recvCqDepth, attr->rdma.cqAttr.recvCqCompVector, qpCb->qkey, qpCb->udpSport,
229 : attr->rdma.qpAttr.cap.maxSendWr, attr->rdma.qpAttr.cap.maxRecvWr, attr->rdma.qpAttr.cap.maxSendSge,
230 : attr->rdma.qpAttr.cap.maxRecvSge, attr->rdma.qpAttr.cap.maxInlineData);
231 :
232 0 : return 0;
233 :
234 0 : modify_qp_fail:
235 0 : (void)RsIbvDestroyQp(qpCb->ibQp);
236 1 : create_qp_fail:
237 1 : (void)RsIbvDestroyCq(qpCb->recvCq.ibCq);
238 1 : create_rcq_fail:
239 1 : (void)RsEpollCtl(rscb->connCb.epollfd, EPOLL_CTL_DEL, qpCb->channel->fd, EPOLLIN | EPOLLRDHUP);
240 1 : epoll_ctl_fail:
241 1 : (void)RsIbvDestroyCompChannel(qpCb->channel);
242 1 : create_channel_fail:
243 1 : (void)RsIbvDestroyCq(qpCb->sendCq.ibCq);
244 1 : return ret;
245 : }
246 :
247 0 : STATIC int RsPingCommonInitMrCb(struct rs_cb *rscb, struct RsPingCtxCb *pingCb, struct RsPingMrCb *mrCb)
248 : {
249 0 : unsigned long flag = 0;
250 0 : uint32_t idx = 0;
251 : int ret;
252 :
253 0 : hccp_info("payload_offset:%u len:0x%llx sge_num:%u grp_id:%u", mrCb->payloadOffset, mrCb->len, mrCb->sgeNum,
254 : rscb->grpId);
255 :
256 0 : ret = pthread_mutex_init(&mrCb->mutex, NULL);
257 0 : CHK_PRT_RETURN(ret != 0, hccp_err("pthread_mutex_init mr_cb mutex failed, ret:%d", ret), ret);
258 :
259 0 : flag = ((unsigned long)pingCb->logicDevid << BUFF_FLAGS_DEVID_OFFSET) | BUFF_SP_SVM;
260 0 : ret = DlHalBuffAllocAlignEx(mrCb->len, (unsigned int)RA_RS_4K_PAGE_SIZE, flag, (int)rscb->grpId,
261 0 : (void **)&mrCb->addr);
262 0 : if (ret != 0) {
263 0 : hccp_err("DlHalBuffAllocAlignEx failed, length:0x%llx, dev_id:0x%x, flag:0x%lx, grpId:%u, ret:%d", mrCb->len,
264 : pingCb->logicDevid, flag, rscb->grpId, ret);
265 0 : goto alloc_fail;
266 : }
267 :
268 0 : mrCb->ibMr = RsDrvMrReg(pingCb->rdevCb.ibPd, (char *)(uintptr_t)mrCb->addr, mrCb->len,
269 : IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE | IBV_ACCESS_REMOTE_READ);
270 0 : if (mrCb->ibMr == NULL) {
271 0 : ret = -errno;
272 0 : hccp_err("rs_ibv_reg_mr fail, ret:%d addr:0x%llx len:0x%llx", ret, mrCb->addr, mrCb->len);
273 0 : goto mr_reg_fail;
274 : }
275 :
276 : // init sge list
277 0 : mrCb->sgeList = calloc(mrCb->sgeNum, sizeof(struct ibv_sge));
278 0 : if (mrCb->sgeList == NULL) {
279 0 : ret = -errno;
280 0 : hccp_err("calloc fail, ret:%d sgeNum:%u", ret, mrCb->sgeNum);
281 0 : goto calloc_fail;
282 : }
283 0 : for (idx = 0; idx < mrCb->sgeNum; idx++) {
284 0 : mrCb->sgeList[idx].lkey = mrCb->ibMr->lkey;
285 0 : mrCb->sgeList[idx].length = mrCb->payloadOffset;
286 0 : if (idx == 0) {
287 0 : mrCb->sgeList[idx].addr = mrCb->addr;
288 : } else {
289 0 : mrCb->sgeList[idx].addr = mrCb->sgeList[idx - 1].addr + mrCb->payloadOffset;
290 : }
291 : }
292 0 : mrCb->sgeIdx = 0;
293 :
294 0 : hccp_info("addr:0x%llx lkey:%u ", mrCb->addr, mrCb->ibMr->lkey);
295 :
296 0 : return 0;
297 :
298 0 : calloc_fail:
299 0 : (void)RsDrvMrDereg(mrCb->ibMr);
300 0 : mr_reg_fail:
301 0 : (void)DlHalBuffFree((void *)(uintptr_t)mrCb->addr);
302 0 : alloc_fail:
303 0 : (void)pthread_mutex_destroy(&mrCb->mutex);
304 0 : return ret;
305 : }
306 :
307 4 : STATIC void RsPingCommonDeinitMrCb(struct RsPingMrCb *mrCb)
308 : {
309 4 : hccp_dbg("addr:0x%llx len:%llu", mrCb->addr, mrCb->len);
310 4 : free(mrCb->sgeList);
311 4 : mrCb->sgeList = NULL;
312 4 : (void)RsDrvMrDereg(mrCb->ibMr);
313 4 : (void)DlHalBuffFree((void *)(uintptr_t)mrCb->addr);
314 4 : (void)pthread_mutex_destroy(&mrCb->mutex);
315 4 : }
316 :
317 2 : STATIC int RsPingPongInitLocalBuffer(struct rs_cb *rscb, struct PingInitAttr *attr, struct PingInitInfo *info,
318 : struct RsPingCtxCb *pingCb)
319 : {
320 : int ret;
321 :
322 : // prepare ping_qp send mr
323 2 : pingCb->pingQp.sendMrCb.payloadOffset = PING_TOTAL_PAYLOAD_MAX_SIZE;
324 2 : pingCb->pingQp.sendMrCb.len = pingCb->pingQp.qpCap.max_send_wr * pingCb->pingQp.sendMrCb.payloadOffset;
325 2 : pingCb->pingQp.sendMrCb.sgeNum = pingCb->pingQp.qpCap.max_send_wr;
326 2 : ret = RsPingCommonInitMrCb(rscb, pingCb, &pingCb->pingQp.sendMrCb);
327 2 : CHK_PRT_RETURN(ret != 0, hccp_err("rs_ping_common_init_mr_cb ping_qp send_mr_cb failed, ret %d", ret), ret);
328 : // prepare ping_qp recv mr
329 1 : pingCb->pingQp.recvMrCb.payloadOffset = PING_TOTAL_PAYLOAD_MAX_SIZE;
330 1 : pingCb->pingQp.recvMrCb.len = pingCb->pingQp.qpCap.max_recv_wr * pingCb->pingQp.recvMrCb.payloadOffset;
331 1 : pingCb->pingQp.recvMrCb.sgeNum = pingCb->pingQp.qpCap.max_recv_wr;
332 1 : ret = RsPingCommonInitMrCb(rscb, pingCb, &pingCb->pingQp.recvMrCb);
333 1 : if (ret != 0) {
334 0 : hccp_err("rs_ping_common_init_mr_cb ping_qp recv_mr_cb failed, ret %d", ret);
335 0 : goto init_ping_qp_recv_mr_fail;
336 : }
337 :
338 : // prepare pong_qp send mr
339 1 : pingCb->pongQp.sendMrCb.payloadOffset = PING_TOTAL_PAYLOAD_MAX_SIZE;
340 1 : pingCb->pongQp.sendMrCb.len = pingCb->pongQp.qpCap.max_send_wr * pingCb->pongQp.sendMrCb.payloadOffset;
341 1 : pingCb->pongQp.sendMrCb.sgeNum = pingCb->pongQp.qpCap.max_send_wr;
342 1 : ret = RsPingCommonInitMrCb(rscb, pingCb, &pingCb->pongQp.sendMrCb);
343 1 : if (ret != 0) {
344 0 : hccp_err("rs_ping_common_init_mr_cb pong_qp send_mr_cb failed, ret %d", ret);
345 0 : goto init_pong_qp_send_mr_fail;
346 : }
347 : // prepare pong_qp recv mr
348 1 : pingCb->pongQp.recvMrCb.payloadOffset = PING_TOTAL_PAYLOAD_MAX_SIZE;
349 1 : pingCb->pongQp.recvMrCb.len = attr->bufferSize;
350 1 : pingCb->pongQp.recvMrCb.sgeNum = attr->bufferSize / pingCb->pongQp.recvMrCb.payloadOffset;
351 1 : ret = RsPingCommonInitMrCb(rscb, pingCb, &pingCb->pongQp.recvMrCb);
352 1 : if (ret != 0) {
353 0 : hccp_err("rs_ping_common_init_mr_cb pong_qp recv_mr_cb failed, ret %d", ret);
354 0 : goto init_pong_qp_recv_mr_fail;
355 : }
356 1 : info->result.bufferVa = pingCb->pongQp.recvMrCb.addr;
357 1 : info->result.bufferSize = attr->bufferSize;
358 1 : info->result.payloadOffset = pingCb->pongQp.recvMrCb.payloadOffset;
359 1 : info->result.headerSize = RS_PING_PAYLOAD_HEADER_RESV_GRH + RS_PING_PAYLOAD_HEADER_RESV_CUSTOM;
360 :
361 1 : return 0;
362 :
363 0 : init_pong_qp_recv_mr_fail:
364 0 : RsPingCommonDeinitMrCb(&pingCb->pongQp.sendMrCb);
365 0 : init_pong_qp_send_mr_fail:
366 0 : RsPingCommonDeinitMrCb(&pingCb->pingQp.recvMrCb);
367 0 : init_ping_qp_recv_mr_fail:
368 0 : RsPingCommonDeinitMrCb(&pingCb->pingQp.sendMrCb);
369 0 : return ret;
370 : }
371 :
372 2 : STATIC int RsPingCommonPostRecv(struct RsPingLocalQpCb *qpCb)
373 : {
374 2 : struct ibv_recv_wr *badWr = NULL;
375 2 : struct ibv_recv_wr wr = {0};
376 2 : struct ibv_sge list = {0};
377 : uint32_t sgeIdx;
378 : int ret;
379 :
380 2 : RS_PTHREAD_MUTEX_LOCK(&qpCb->recvMrCb.mutex);
381 2 : sgeIdx = qpCb->recvMrCb.sgeIdx;
382 2 : (void)memcpy_s(&list, sizeof(struct ibv_sge), &qpCb->recvMrCb.sgeList[sgeIdx], sizeof(struct ibv_sge));
383 2 : qpCb->recvMrCb.sgeIdx = (sgeIdx + 1) % qpCb->recvMrCb.sgeNum;
384 2 : RS_PTHREAD_MUTEX_ULOCK(&qpCb->recvMrCb.mutex);
385 :
386 2 : wr.wr_id = (uint64_t)sgeIdx;
387 2 : wr.next = NULL;
388 2 : wr.sg_list = &list;
389 2 : wr.num_sge = 1;
390 :
391 2 : ret = RsIbvPostRecv(qpCb->ibQp, &wr, &badWr);
392 2 : if (ret != 0) {
393 1 : hccp_err("rs_ibv_post_recv failed, ret:%d", ret);
394 1 : return ret;
395 : }
396 :
397 1 : return 0;
398 : }
399 :
400 2 : STATIC int RsPingCommonInitPostRecvAll(struct RsPingLocalQpCb *qpCb)
401 : {
402 2 : int ret = 0;
403 : uint32_t i;
404 :
405 : // prepare RQ wqe
406 3 : for (i = qpCb->recvMrCb.sgeIdx; i < qpCb->recvMrCb.sgeNum && i < qpCb->qpCap.max_recv_wr; i++) {
407 2 : ret = RsPingCommonPostRecv(qpCb);
408 2 : if (ret != 0) {
409 1 : hccp_err("rs_ping_common_post_recv %u-th rqe failed, ret:%d", i, ret);
410 1 : break;
411 : }
412 : }
413 :
414 2 : return ret;
415 : }
416 :
417 1 : STATIC void RsPingCommonDeinitLocalBuffer(struct RsPingCtxCb *pingCb)
418 : {
419 1 : RsPingCommonDeinitMrCb(&pingCb->pongQp.recvMrCb);
420 1 : RsPingCommonDeinitMrCb(&pingCb->pongQp.sendMrCb);
421 1 : RsPingCommonDeinitMrCb(&pingCb->pingQp.recvMrCb);
422 1 : RsPingCommonDeinitMrCb(&pingCb->pingQp.sendMrCb);
423 1 : }
424 :
425 2 : STATIC void RsPingCommonDeinitLocalQp(struct rs_cb *rscb, struct RsPingCtxCb *pingCb, struct RsPingLocalQpCb *qpCb)
426 : {
427 2 : if (qpCb == NULL || qpCb->channel == NULL) {
428 1 : hccp_err("qp_cb is NULL or qp_cb->channel is NULL");
429 1 : return;
430 : }
431 :
432 1 : (void)RsIbvDestroyQp(qpCb->ibQp);
433 1 : RsIbvAckCqEvents(qpCb->recvCq.ibCq, qpCb->recvCq.numEvents);
434 1 : qpCb->recvCq.numEvents = 0;
435 1 : (void)RsIbvDestroyCq(qpCb->recvCq.ibCq);
436 1 : (void)RsEpollCtl(rscb->connCb.epollfd, EPOLL_CTL_DEL, qpCb->channel->fd, EPOLLIN | EPOLLRDHUP);
437 1 : (void)RsIbvDestroyCompChannel(qpCb->channel);
438 1 : qpCb->channel = NULL;
439 1 : (void)RsIbvDestroyCq(qpCb->sendCq.ibCq);
440 : }
441 :
442 4 : STATIC int RsPingPongInitLocalInfo(struct rs_cb *rscb, struct PingInitAttr *attr, struct PingInitInfo *info,
443 : struct RsPingCtxCb *pingCb)
444 : {
445 : int ret;
446 :
447 4 : ret = RsPingCommonInitLocalQp(rscb, pingCb, &attr->client, &pingCb->pingQp);
448 4 : CHK_PRT_RETURN(ret != 0, hccp_err("init ping_qp failed, ret:%d", ret), ret);
449 3 : info->client.version = 0;
450 3 : (void)memcpy_s(&info->client.rdma.gid, sizeof(union HccpGid), &pingCb->rdevCb.gid, sizeof(union ibv_gid));
451 3 : info->client.rdma.qpn = pingCb->pingQp.ibQp->qp_num;
452 3 : info->client.rdma.qkey = pingCb->pingQp.qkey;
453 :
454 3 : ret = RsPingCommonInitLocalQp(rscb, pingCb, &attr->server, &pingCb->pongQp);
455 3 : if (ret != 0) {
456 0 : hccp_err("init pong_qp failed, ret:%d", ret);
457 0 : goto init_pong_qp_fail;
458 : }
459 3 : info->server.version = 0;
460 3 : (void)memcpy_s(&info->server.rdma.gid, sizeof(union HccpGid), &pingCb->rdevCb.gid, sizeof(union ibv_gid));
461 3 : info->server.rdma.qpn = pingCb->pongQp.ibQp->qp_num;
462 3 : info->server.rdma.qkey = pingCb->pongQp.qkey;
463 :
464 3 : ret = RsPingPongInitLocalBuffer(rscb, attr, info, pingCb);
465 3 : if (ret != 0) {
466 1 : hccp_err("init buffer failed, ret:%d", ret);
467 1 : goto init_buffer_fail;
468 : }
469 :
470 2 : ret = RsPingCommonInitPostRecvAll(&pingCb->pingQp);
471 2 : if (ret != 0) {
472 1 : hccp_err("ping_qp post recv failed, ret:%d", ret);
473 1 : goto post_recv_fail;
474 : }
475 1 : ret = RsPingCommonInitPostRecvAll(&pingCb->pongQp);
476 1 : if (ret != 0) {
477 0 : hccp_err("pong_qp post recv failed, ret:%d", ret);
478 0 : goto post_recv_fail;
479 : }
480 :
481 1 : return 0;
482 :
483 1 : post_recv_fail:
484 1 : RsPingCommonDeinitLocalBuffer(pingCb);
485 2 : init_buffer_fail:
486 2 : RsPingCommonDeinitLocalQp(rscb, pingCb, &pingCb->pongQp);
487 2 : init_pong_qp_fail:
488 2 : RsPingCommonDeinitLocalQp(rscb, pingCb, &pingCb->pingQp);
489 2 : return ret;
490 : }
491 :
492 1 : STATIC int RsPingRocePingCbInit(unsigned int phyId, struct PingInitAttr *attr, struct PingInitInfo *info,
493 : unsigned int *devIndex, struct RsPingCtxCb *pingCb)
494 : {
495 1 : struct rdev *rdevInfo = &attr->dev.rdma;
496 1 : struct rs_cb *rscb = NULL;
497 : int ret;
498 :
499 1 : ret = RsGetRsCb(phyId, &rscb);
500 1 : CHK_PRT_RETURN(ret != 0, hccp_err("RsGetRsCb failed, phyId[%u] invalid, ret %d", phyId, ret), ret);
501 :
502 : // prepare input attr
503 1 : pingCb->rdevCb.ip.family = (uint32_t)rdevInfo->family;
504 1 : pingCb->rdevCb.ip.binAddr = rdevInfo->localIp;
505 1 : ret = RsInetNtop(rdevInfo->family, &rdevInfo->localIp, pingCb->rdevCb.ip.readAddr, RS_MAX_IP_LEN);
506 1 : CHK_PRT_RETURN(ret != 0, hccp_err("rs_inet_ntop failed, ret %d", ret), -EINVAL);
507 1 : (void)memcpy_s(&pingCb->commInfo, sizeof(struct PingLocalCommInfo), &attr->commInfo,
508 : sizeof(struct PingLocalCommInfo));
509 :
510 : // open device & alloc global pd
511 1 : pingCb->rdevCb.devList = RsIbvGetDeviceList(&pingCb->rdevCb.devNum);
512 1 : if (pingCb->rdevCb.devList == NULL || pingCb->rdevCb.devNum == 0) {
513 0 : hccp_err("dev_list is NULL or dev_num[%d] is 0", pingCb->rdevCb.devNum);
514 0 : ret = -ENODEV;
515 0 : goto get_device_list_fail;
516 : }
517 :
518 1 : pingCb->rdevCb.ibPort = RS_PORT_DEF;
519 1 : ret = RsPingCbGetIbCtxAndIndex(rdevInfo, pingCb);
520 1 : if (ret != 0) {
521 1 : hccp_err("rs_ping_cb_get_ib_ctx_and_index failed, ret:%d", ret);
522 1 : goto get_ib_ctx_and_index_fail;
523 : }
524 :
525 0 : pingCb->rdevCb.ibPd = RsIbvAllocPd(pingCb->rdevCb.ibCtx);
526 0 : if (pingCb->rdevCb.ibPd == NULL) {
527 0 : hccp_err("rs_ibv_alloc_pd failed, errno:%d", errno);
528 0 : ret = -ENOMEM;
529 0 : goto alloc_pd_fail;
530 : }
531 :
532 : // init cq & qp & mr info, prepare output info
533 0 : info->version = 0;
534 0 : ret = RsPingPongInitLocalInfo(rscb, attr, info, pingCb);
535 0 : if (ret != 0) {
536 0 : hccp_err("rs_ping_pong_init_local_info failed, ret=%d phyId:%u", ret, rdevInfo->phyId);
537 0 : goto init_local_info_fail;
538 : }
539 :
540 0 : *devIndex = pingCb->devIndex;
541 0 : return 0;
542 :
543 0 : init_local_info_fail:
544 0 : (void)RsIbvDeallocPd(pingCb->rdevCb.ibPd);
545 0 : alloc_pd_fail:
546 0 : (void)RsIbvCloseDevice(pingCb->rdevCb.ibCtx);
547 1 : get_ib_ctx_and_index_fail:
548 1 : RsIbvFreeDeviceList(pingCb->rdevCb.devList);
549 1 : get_device_list_fail:
550 1 : (void)pthread_mutex_destroy(&pingCb->pingMutex);
551 1 : (void)pthread_mutex_destroy(&pingCb->pongMutex);
552 1 : return ret;
553 : }
554 :
555 6 : STATIC bool RsPingCommonCompareRdmaInfo(struct PingQpInfo *a, struct PingQpInfo *b)
556 : {
557 6 : if (a->rdma.qpn != b->rdma.qpn) {
558 1 : return false;
559 : }
560 5 : if (a->rdma.qkey != b->rdma.qkey) {
561 1 : return false;
562 : }
563 4 : if (memcmp(&a->rdma.gid, &b->rdma.gid, sizeof(union HccpGid)) != 0) {
564 1 : return false;
565 : }
566 3 : return true;
567 : }
568 :
569 3 : STATIC int RsPingRoceFindTargetNode(struct RsPingCtxCb *pingCb, struct PingQpInfo *target,
570 : struct RsPingTargetInfo **node)
571 : {
572 3 : struct RsPingTargetInfo *targetNext = NULL;
573 3 : struct RsPingTargetInfo *targetCurr = NULL;
574 :
575 3 : RS_PTHREAD_MUTEX_LOCK(&pingCb->pingMutex);
576 3 : RS_LIST_GET_HEAD_ENTRY(targetCurr, targetNext, &pingCb->pingList, list, struct RsPingTargetInfo);
577 3 : for (; (&targetCurr->list) != &pingCb->pingList;
578 0 : targetCurr = targetNext, targetNext = list_entry(targetNext->list.next, struct RsPingTargetInfo, list)) {
579 1 : if (RsPingCommonCompareRdmaInfo(&targetCurr->qpInfo, target)) {
580 1 : *node = targetCurr;
581 1 : RS_PTHREAD_MUTEX_ULOCK(&pingCb->pingMutex);
582 1 : return 0;
583 : }
584 : }
585 2 : RS_PTHREAD_MUTEX_ULOCK(&pingCb->pingMutex);
586 :
587 2 : hccp_info("ping target node for qpn:%u gid:%016llx:%016llx not found", target->rdma.qpn,
588 : target->rdma.gid.global.subnetPrefix, target->rdma.gid.global.interfaceId);
589 2 : return -ENODEV;
590 : }
591 :
592 0 : STATIC int RsPingCommonCreateAh(struct RsPingCtxCb *pingCb, struct PingLocalCommInfo *localInfo,
593 : struct PingQpInfo *remoteInfo, struct ibv_ah **ah)
594 : {
595 0 : struct ibv_exp_ah_attr attrx = {0};
596 0 : struct ibv_global_route grh = {0};
597 0 : struct ibv_ah_attr attr = {0};
598 0 : struct ibv_ah *ahTmp = NULL;
599 0 : int ret = 0;
600 :
601 0 : (void)memcpy_s(&grh.dgid, sizeof(union ibv_gid), &remoteInfo->rdma.gid, sizeof(union HccpGid));
602 0 : grh.flow_label = localInfo->rdma.flowLabel;
603 0 : grh.sgid_index = (uint8_t)pingCb->rdevCb.gidIdx;
604 0 : grh.hop_limit = localInfo->rdma.hopLimit;
605 0 : grh.traffic_class = localInfo->rdma.qosAttr.tc;
606 :
607 0 : attr.grh = grh;
608 0 : attr.sl = localInfo->rdma.qosAttr.sl;
609 0 : attr.is_global = 1;
610 0 : attr.port_num = pingCb->rdevCb.ibPort;
611 0 : attrx.attr = attr;
612 0 : attrx.udp_sport = localInfo->rdma.udpSport;
613 :
614 0 : hccp_dbg("remote_qpn:%u flow_label:%u sgid_index:%u hop_limit:%u traffic_class:%u sl:%u is_global:%u "
615 : "port_num:%u udp_sport:%u",
616 : remoteInfo->rdma.qpn, grh.flow_label, grh.sgid_index, grh.hop_limit, grh.traffic_class, attr.sl, attr.is_global,
617 : attr.port_num, attrx.udp_sport);
618 :
619 0 : ahTmp = RsIbvExpCreateAh(pingCb->rdevCb.ibPd, &attrx);
620 0 : if (ahTmp == NULL) {
621 0 : ret = -EFAULT;
622 0 : hccp_err("rs_ibv_exp_create_ah failed, errno:%d", errno);
623 0 : return ret;
624 : }
625 :
626 0 : *ah = ahTmp;
627 0 : return ret;
628 : }
629 :
630 1 : STATIC int RsPingRoceAllocTargetNode(struct RsPingCtxCb *pingCb, struct PingTargetInfo *target,
631 : struct RsPingTargetInfo **node)
632 : {
633 1 : struct RsPingTargetInfo *targetInfo = NULL;
634 : int ret;
635 :
636 1 : targetInfo = (struct RsPingTargetInfo *)calloc(1, sizeof(struct RsPingTargetInfo));
637 1 : CHK_PRT_RETURN(targetInfo == NULL, hccp_err("calloc target_info fail! errno:%d", errno), -ENOMEM);
638 :
639 1 : ret = pthread_mutex_init(&targetInfo->tripMutex, NULL);
640 1 : if (ret != 0) {
641 0 : hccp_err("pthread_mutex_init trip_mutex failed, ret:%d", ret);
642 0 : goto free_target_info;
643 : }
644 :
645 1 : targetInfo->payloadSize = target->payload.size;
646 1 : if (target->payload.size > 0) {
647 1 : targetInfo->payloadBuffer = (char *)calloc(1, target->payload.size);
648 1 : if (targetInfo->payloadBuffer == NULL) {
649 0 : hccp_err("calloc payload_buffer fail! size:%u errno:%d", target->payload.size, errno);
650 0 : ret = -ENOMEM;
651 0 : goto free_trip_mutex;
652 : }
653 1 : (void)memcpy_s(targetInfo->payloadBuffer, target->payload.size, target->payload.buffer, target->payload.size);
654 : }
655 :
656 1 : (void)memcpy_s(&targetInfo->qpInfo, sizeof(struct PingQpInfo), &target->remoteInfo.qpInfo,
657 : sizeof(struct PingQpInfo));
658 1 : ret = RsPingCommonCreateAh(pingCb, &target->localInfo, &target->remoteInfo.qpInfo, &targetInfo->ah);
659 1 : if (ret != 0) {
660 0 : hccp_err("rs_ping_common_create_ah fail! ret:%d", ret);
661 0 : goto free_payload_buffer;
662 : }
663 :
664 1 : targetInfo->resultSummary.rttMin = ~0;
665 1 : targetInfo->state = RS_PING_PONG_TARGET_READY;
666 1 : *node = targetInfo;
667 :
668 1 : return 0;
669 0 : free_payload_buffer:
670 0 : if (target->payload.size > 0 && targetInfo->payloadBuffer != NULL) {
671 0 : free(targetInfo->payloadBuffer);
672 0 : targetInfo->payloadBuffer = NULL;
673 : }
674 0 : free_trip_mutex:
675 0 : (void)pthread_mutex_destroy(&targetInfo->tripMutex);
676 0 : free_target_info:
677 0 : free(targetInfo);
678 0 : targetInfo = NULL;
679 0 : return ret;
680 : }
681 :
682 1 : STATIC void RsPingRoceResetRecvBuffer(struct RsPingCtxCb *pingCb)
683 : {
684 1 : RS_PTHREAD_MUTEX_LOCK(&pingCb->pongQp.recvMrCb.mutex);
685 1 : (void)memset_s((void *)(uintptr_t)pingCb->pongQp.recvMrCb.addr, pingCb->pongQp.recvMrCb.len, 0,
686 : pingCb->pongQp.recvMrCb.len);
687 1 : RS_PTHREAD_MUTEX_ULOCK(&pingCb->pongQp.recvMrCb.mutex);
688 1 : }
689 :
690 2 : STATIC void RsPingQpBuildUpWr(struct RsPingTargetInfo *target, struct ibv_sge *list, struct ibv_send_wr *wr)
691 : {
692 2 : wr->wr_id = target->uuid;
693 2 : wr->next = NULL;
694 2 : wr->sg_list = list;
695 2 : wr->num_sge = 1;
696 2 : wr->opcode = IBV_WR_SEND;
697 2 : wr->send_flags = IBV_SEND_SIGNALED;
698 2 : wr->wr.ud.ah = target->ah;
699 2 : wr->wr.ud.remote_qpn = target->qpInfo.rdma.qpn;
700 2 : wr->wr.ud.remote_qkey = target->qpInfo.rdma.qkey;
701 2 : }
702 :
703 2 : STATIC int RsPingRocePostSend(struct RsPingCtxCb *pingCb, struct RsPingTargetInfo *target)
704 : {
705 2 : struct RsPingPayloadHeader *header = NULL;
706 2 : struct ibv_send_wr *badWr = NULL;
707 2 : struct timeval timestamp = {0};
708 2 : struct ibv_send_wr wr = {0};
709 2 : struct ibv_sge list = {0};
710 : uint32_t sgeIdx;
711 2 : int ret = 0;
712 :
713 2 : hccp_dbg("target uuid:0x%llx state:%d payload_size:%u qpn:%u gid:%016llx:%016llx", target->uuid, target->state,
714 : target->payloadSize, target->qpInfo.rdma.qpn, target->qpInfo.rdma.gid.global.subnetPrefix,
715 : target->qpInfo.rdma.gid.global.interfaceId);
716 :
717 2 : RS_PTHREAD_MUTEX_LOCK(&pingCb->pingQp.sendMrCb.mutex);
718 2 : sgeIdx = pingCb->pingQp.sendMrCb.sgeIdx;
719 2 : (void)memcpy_s(&list, sizeof(struct ibv_sge), &pingCb->pingQp.sendMrCb.sgeList[sgeIdx], sizeof(struct ibv_sge));
720 2 : pingCb->pingQp.sendMrCb.sgeIdx = (sgeIdx + 1) % pingCb->pingQp.sendMrCb.sgeNum;
721 2 : RS_PTHREAD_MUTEX_ULOCK(&pingCb->pingQp.sendMrCb.mutex);
722 :
723 : // prepare ping_qp send buffer
724 2 : (void)memset_s((void *)(uintptr_t)list.addr, list.length, 0, list.length);
725 2 : header = (struct RsPingPayloadHeader *)(uintptr_t)list.addr;
726 2 : header->type = RS_PING_TYPE_ROCE_DETECT;
727 2 : (void)memcpy_s(&header->server.rdma.gid, sizeof(union HccpGid), &pingCb->rdevCb.gid, sizeof(union ibv_gid));
728 2 : header->server.rdma.qpn = pingCb->pongQp.ibQp->qp_num;
729 2 : header->server.rdma.qkey = pingCb->pongQp.qkey;
730 2 : (void)memcpy_s(&header->target, sizeof(struct PingQpInfo), &target->qpInfo, sizeof(struct PingQpInfo));
731 :
732 2 : if (target->payloadSize > 0) {
733 2 : ret = memcpy_s((void *)(uintptr_t)(list.addr + RS_PING_PAYLOAD_HEADER_RESV_CUSTOM),
734 2 : (list.length - RS_PING_PAYLOAD_HEADER_RESV_CUSTOM), (void *)target->payloadBuffer, target->payloadSize);
735 2 : CHK_PRT_RETURN(ret != 0,
736 : hccp_err("memcpy_s buffer payload_size:%u list.length:%u failed, ret:%d", target->payloadSize,
737 : (list.length - RS_PING_PAYLOAD_HEADER_RESV_CUSTOM), ret),
738 : -ESAFEFUNC);
739 : }
740 2 : list.length = RS_PING_PAYLOAD_HEADER_RESV_CUSTOM + target->payloadSize;
741 :
742 2 : RsPingQpBuildUpWr(target, &list, &wr);
743 :
744 : // record timestamp t1
745 2 : (void)gettimeofday(×tamp, NULL);
746 2 : header->timestamp.tvSec1 = (uint64_t)timestamp.tv_sec;
747 2 : header->timestamp.tvUsec1 = (uint64_t)timestamp.tv_usec;
748 2 : header->taskId = pingCb->taskId;
749 2 : header->magic = 0x55AA;
750 :
751 2 : ret = RsIbvPostSend(pingCb->pingQp.ibQp, &wr, &badWr);
752 2 : if (ret != 0) {
753 1 : hccp_err("rs_ibv_post_send qpn:%u failed, ret:%d", pingCb->pingQp.ibQp->qp_num, ret);
754 1 : RS_PTHREAD_MUTEX_LOCK(&target->tripMutex);
755 1 : target->state = RS_PING_PONG_TARGET_ERROR;
756 1 : RS_PTHREAD_MUTEX_ULOCK(&target->tripMutex);
757 : }
758 2 : return ret;
759 : }
760 :
761 3 : STATIC int RsPingRocePollScq(struct RsPingCtxCb *pingCb, struct RsPingTargetInfo *target)
762 : {
763 3 : struct ibv_wc wc = {0};
764 : int polledCnt;
765 :
766 3 : polledCnt = RsIbvPollCq(pingCb->pingQp.sendCq.ibCq, 1, &wc);
767 3 : if (polledCnt != 1) {
768 1 : hccp_err("uuid:0x%llx rs_ibv_poll_cq polled_cnt:%d", target->uuid, polledCnt);
769 1 : target->state = RS_PING_PONG_TARGET_ERROR;
770 1 : return -ENODATA;
771 : }
772 2 : if (wc.status != IBV_WC_SUCCESS) {
773 1 : target->state = RS_PING_PONG_TARGET_ERROR;
774 1 : hccp_err("wr_id:0x%llx error cqe %s(%d)", wc.wr_id, RsIbvWcStatusStr(wc.status), wc.status);
775 1 : return -EOPENSRC;
776 : }
777 1 : return 0;
778 : }
779 :
780 6 : STATIC int RsPingRocePollRcq(struct RsPingCtxCb *pingCb, int *polledCnt, struct timeval *timestamp2)
781 : {
782 6 : struct ibv_cq *evCq = NULL;
783 6 : void *evCtx = NULL;
784 : int ret;
785 :
786 : // record timestamp t2
787 6 : (void)gettimeofday(timestamp2, NULL);
788 :
789 6 : ret = RsIbvGetCqEvent(pingCb->pingQp.channel, &evCq, &evCtx);
790 6 : if (ret != 0) {
791 1 : hccp_err("rs_ibv_get_cq_event ping_qp.channel failed, ret:%d", ret);
792 1 : return -EOPENSRC;
793 : }
794 :
795 5 : if (evCq != pingCb->pingQp.recvCq.ibCq) {
796 0 : hccp_err("CQ event for unknown CQ");
797 0 : return -EOPENSRC;
798 : }
799 5 : pingCb->pingQp.recvCq.numEvents++;
800 :
801 5 : *polledCnt = RsIbvPollCq(evCq, pingCb->pingQp.recvCq.maxRecvWcNum, gPingQpRecvWc);
802 5 : CHK_PRT_RETURN(*polledCnt > pingCb->pingQp.recvCq.maxRecvWcNum || *polledCnt < 0,
803 : hccp_err("ping_poll_rcq failed, ret:%d", *polledCnt), -EOPENSRC);
804 :
805 4 : return 0;
806 : }
807 :
808 2 : STATIC int RsPingCommonPollScq(struct RsPingLocalQpCb *qpCb)
809 : {
810 2 : struct ibv_wc wc = {0};
811 : int polledCnt;
812 :
813 2 : polledCnt = RsIbvPollCq(qpCb->sendCq.ibCq, 1, &wc);
814 2 : if (polledCnt < 0) {
815 1 : hccp_warn("rs_ibv_poll_cq unsuccessful, polledCnt:%d", polledCnt);
816 1 : } else if (polledCnt > 0) {
817 1 : if (wc.status != IBV_WC_SUCCESS) {
818 0 : hccp_err("wr_id:0x%llx error cqe %s(%d)", wc.wr_id, RsIbvWcStatusStr(wc.status), wc.status);
819 0 : return -EOPENSRC;
820 : }
821 : }
822 :
823 2 : return 0;
824 : }
825 :
826 2 : STATIC int RsPongFindTargetNode(struct RsPingCtxCb *pingCb, struct PingQpInfo *target, struct RsPongTargetInfo **node)
827 : {
828 2 : struct RsPongTargetInfo *targetNext = NULL;
829 2 : struct RsPongTargetInfo *targetCurr = NULL;
830 :
831 2 : RS_CHECK_POINTER_NULL_WITH_RET(pingCb);
832 2 : RS_PTHREAD_MUTEX_LOCK(&pingCb->pongMutex);
833 2 : RS_LIST_GET_HEAD_ENTRY(targetCurr, targetNext, &pingCb->pongList, list, struct RsPongTargetInfo);
834 2 : for (; (&targetCurr->list) != &pingCb->pongList;
835 0 : targetCurr = targetNext, targetNext = list_entry(targetNext->list.next, struct RsPongTargetInfo, list)) {
836 1 : if (RsPingCommonCompareRdmaInfo(&targetCurr->qpInfo, target)) {
837 1 : *node = targetCurr;
838 1 : RS_PTHREAD_MUTEX_ULOCK(&pingCb->pongMutex);
839 1 : return 0;
840 : }
841 : }
842 1 : RS_PTHREAD_MUTEX_ULOCK(&pingCb->pongMutex);
843 :
844 1 : hccp_info("pong target node for qpn:%u gid:%016llx:%016llx not found", target->rdma.qpn,
845 : target->rdma.gid.global.subnetPrefix, target->rdma.gid.global.interfaceId);
846 1 : return -ENODEV;
847 : }
848 :
849 2 : STATIC int RsPongFindAllocTargetNode(struct RsPingCtxCb *pingCb, struct PingQpInfo *target,
850 : struct RsPongTargetInfo **node)
851 : {
852 2 : struct RsPongTargetInfo *targetInfo = NULL;
853 : int ret;
854 :
855 2 : ret = RsPongFindTargetNode(pingCb, target, node);
856 2 : if (ret == 0 && (*node)->state == RS_PING_PONG_TARGET_READY) {
857 0 : return 0;
858 2 : } else if (ret == 0) {
859 1 : targetInfo = *node;
860 1 : hccp_info("delete pong target uuid:0x%llx state:%d, realloc again", targetInfo->uuid, targetInfo->state);
861 1 : RsListDel(&targetInfo->list);
862 1 : if (targetInfo->ah) {
863 0 : (void)RsIbvDestroyAh(targetInfo->ah);
864 : }
865 1 : free(targetInfo);
866 1 : targetInfo = NULL;
867 : }
868 :
869 2 : targetInfo = (struct RsPongTargetInfo *)calloc(1, sizeof(struct RsPongTargetInfo));
870 2 : CHK_PRT_RETURN(targetInfo == NULL, hccp_err("calloc target_info fail! errno:%d", errno), -ENOMEM);
871 :
872 2 : (void)memcpy_s(&targetInfo->qpInfo, sizeof(struct PingQpInfo), target, sizeof(struct PingQpInfo));
873 2 : ret = RsPingCommonCreateAh(pingCb, &pingCb->commInfo, target, &targetInfo->ah);
874 2 : if (ret != 0) {
875 1 : hccp_err("rs_ping_common_create_ah fail! ret:%d", ret);
876 1 : goto free_target_info;
877 : }
878 :
879 1 : targetInfo->state = RS_PING_PONG_TARGET_READY;
880 1 : *node = targetInfo;
881 :
882 1 : RS_PTHREAD_MUTEX_LOCK(&pingCb->pongMutex);
883 1 : targetInfo->uuid = (uint64_t)pingCb->pongNum << 32U;
884 1 : RsListAddTail(&targetInfo->list, &pingCb->pongList);
885 1 : pingCb->pongNum++;
886 1 : RS_PTHREAD_MUTEX_ULOCK(&pingCb->pongMutex);
887 :
888 1 : return 0;
889 :
890 1 : free_target_info:
891 1 : free(targetInfo);
892 1 : return ret;
893 : }
894 :
895 4 : STATIC int RsPongPostSend(struct RsPingCtxCb *pingCb, struct ibv_wc *wc, struct timeval *timestamp2)
896 : {
897 4 : struct RsPongTargetInfo *targetInfo = NULL;
898 4 : struct RsPingPayloadHeader *header = NULL;
899 4 : struct ibv_send_wr *badWr = NULL;
900 4 : struct timeval timestamp3 = {0};
901 4 : struct ibv_sge recvList = {0};
902 4 : struct ibv_sge sendList = {0};
903 4 : struct ibv_send_wr wr = {0};
904 : uint32_t recvSgeIdx;
905 : uint32_t sendSgeIdx;
906 4 : int ret = 0;
907 :
908 : // poll send cq
909 4 : (void)RsPingCommonPollScq(&pingCb->pongQp);
910 :
911 : // handle detect packet & send response packet
912 4 : recvSgeIdx = (uint32_t)wc->wr_id;
913 4 : if (recvSgeIdx >= pingCb->pingQp.recvMrCb.sgeNum) {
914 1 : hccp_err("param err recv_sge_idx:%u >= sge_num:%u", recvSgeIdx, pingCb->pingQp.recvMrCb.sgeNum);
915 1 : return -EIO;
916 : }
917 3 : (void)memcpy_s(&recvList, sizeof(struct ibv_sge), &pingCb->pingQp.recvMrCb.sgeList[recvSgeIdx],
918 : sizeof(struct ibv_sge));
919 :
920 3 : RS_PTHREAD_MUTEX_LOCK(&pingCb->pongQp.sendMrCb.mutex);
921 3 : sendSgeIdx = pingCb->pongQp.sendMrCb.sgeIdx;
922 3 : (void)memcpy_s(&sendList, sizeof(struct ibv_sge), &pingCb->pongQp.sendMrCb.sgeList[sendSgeIdx],
923 : sizeof(struct ibv_sge));
924 3 : pingCb->pongQp.sendMrCb.sgeIdx = (sendSgeIdx + 1) % pingCb->pongQp.sendMrCb.sgeNum;
925 3 : RS_PTHREAD_MUTEX_ULOCK(&pingCb->pongQp.sendMrCb.mutex);
926 :
927 : // UD consume 40 Bytes for GRH
928 3 : if (wc->byte_len < RS_PING_PAYLOAD_HEADER_RESV_GRH || wc->byte_len > PING_TOTAL_PAYLOAD_MAX_SIZE) {
929 0 : hccp_err("param err wc->byte_len:%u < %u or wc->byte_len:%u > %u", wc->byte_len,
930 : RS_PING_PAYLOAD_HEADER_RESV_GRH, wc->byte_len, PING_TOTAL_PAYLOAD_MAX_SIZE);
931 0 : return -EIO;
932 : }
933 3 : ret = memcpy_s((void *)(uintptr_t)sendList.addr, sendList.length,
934 3 : (void *)(uintptr_t)(recvList.addr + RS_PING_PAYLOAD_HEADER_RESV_GRH),
935 3 : wc->byte_len - RS_PING_PAYLOAD_HEADER_RESV_GRH);
936 3 : CHK_PRT_RETURN(ret != 0,
937 : hccp_err("memcpy_s buffer wc->byte_len:%u send_list.length:%u failed, ret:%d", wc->byte_len, sendList.length,
938 : ret),
939 : -ESAFEFUNC);
940 3 : sendList.length = wc->byte_len - RS_PING_PAYLOAD_HEADER_RESV_GRH;
941 3 : header = (struct RsPingPayloadHeader *)(uintptr_t)sendList.addr;
942 3 : header->type = RS_PING_TYPE_ROCE_RESPONSE;
943 :
944 3 : ret = RsPongFindAllocTargetNode(pingCb, &header->server, &targetInfo);
945 3 : if (ret != 0) {
946 1 : hccp_err("rs_pong_find_alloc_target_node failed, ret:%d", ret);
947 1 : return ret;
948 : }
949 :
950 2 : wr.wr_id = targetInfo->uuid;
951 2 : wr.next = NULL;
952 2 : wr.sg_list = &sendList;
953 2 : wr.num_sge = 1;
954 2 : wr.opcode = IBV_WR_SEND;
955 2 : wr.send_flags = IBV_SEND_SIGNALED;
956 2 : wr.wr.ud.ah = targetInfo->ah;
957 2 : wr.wr.ud.remote_qpn = targetInfo->qpInfo.rdma.qpn;
958 2 : wr.wr.ud.remote_qkey = targetInfo->qpInfo.rdma.qkey;
959 :
960 : // record timestamp t3
961 2 : (void)gettimeofday(×tamp3, NULL);
962 2 : header->timestamp.tvSec2 = (uint64_t)timestamp2->tv_sec;
963 2 : header->timestamp.tvUsec2 = (uint64_t)timestamp2->tv_usec;
964 2 : header->timestamp.tvSec3 = (uint64_t)timestamp3.tv_sec;
965 2 : header->timestamp.tvUsec3 = (uint64_t)timestamp3.tv_usec;
966 2 : header->magic = 0xAA55;
967 :
968 2 : ret = RsIbvPostSend(pingCb->pongQp.ibQp, &wr, &badWr);
969 2 : if (ret != 0) {
970 1 : targetInfo->state = RS_PING_PONG_TARGET_ERROR;
971 1 : hccp_err("rs_ibv_post_send failed, ret:%d", ret);
972 1 : return ret;
973 : }
974 :
975 1 : return ret;
976 : }
977 :
978 3 : STATIC void RsPongRoceHandleSend(struct RsPingCtxCb *pingCb, int polledCnt, struct timeval *timestamp2)
979 : {
980 3 : struct ibv_wc *wc = NULL;
981 : int ret, i;
982 :
983 3 : wc = gPingQpRecvWc;
984 6 : for (i = 0; i < polledCnt; i++) {
985 3 : if (wc[i].status != IBV_WC_SUCCESS) {
986 0 : hccp_err("wr_id:0x%llx error cqe %s(%d)", wc[i].wr_id, RsIbvWcStatusStr(wc[i].status), wc[i].status);
987 0 : continue;
988 : }
989 :
990 3 : ret = RsPongPostSend(pingCb, &wc[i], timestamp2);
991 3 : if (ret != 0) {
992 1 : hccp_err("rs_pong_post_send failed, wrId:0x%llx", wc[i].wr_id);
993 1 : continue;
994 : }
995 :
996 2 : ret = RsPingCommonPostRecv(&pingCb->pingQp);
997 2 : if (ret != 0) {
998 1 : hccp_err("rs_ping_common_post_recv failed, ret:%d", ret);
999 1 : continue;
1000 : }
1001 : }
1002 :
1003 3 : ret = RsIbvReqNotifyCq(pingCb->pingQp.recvCq.ibCq, 0);
1004 3 : if (ret != 0) {
1005 1 : hccp_err("rs_ibv_req_notify_cq failed, ret:%d", ret);
1006 : }
1007 :
1008 3 : return;
1009 : }
1010 :
1011 4 : STATIC int RsPongResolveResponsePacket(struct RsPingCtxCb *pingCb, uint32_t sgeIdx, struct timeval *timestamp4)
1012 : {
1013 4 : struct RsPingTargetInfo *targetInfo = NULL;
1014 4 : struct RsPingPayloadHeader *header = NULL;
1015 4 : struct ibv_sge *recvList = NULL;
1016 : uint32_t rtt;
1017 : int ret;
1018 :
1019 4 : recvList = &pingCb->pongQp.recvMrCb.sgeList[sgeIdx];
1020 : // UD consume 40 Bytes for GRH
1021 4 : header = (struct RsPingPayloadHeader *)(uintptr_t)(recvList->addr + RS_PING_PAYLOAD_HEADER_RESV_GRH);
1022 4 : if (header->taskId != pingCb->taskId) {
1023 1 : hccp_warn("drop received packet, recv_task_id:%u, curr_task_id:%u", header->taskId, pingCb->taskId);
1024 1 : return 0;
1025 : }
1026 :
1027 3 : header->timestamp.tvSec4 = (uint64_t)timestamp4->tv_sec;
1028 3 : header->timestamp.tvUsec4 = (uint64_t)timestamp4->tv_usec;
1029 3 : rtt = RsPingGetTripTime(&header->timestamp);
1030 3 : ret = RsPingRoceFindTargetNode(pingCb, &header->target, &targetInfo);
1031 3 : if (ret != 0) {
1032 1 : hccp_err("rs_ping_roce_find_target_node failed, ret:%d qpn:%u gid:%016llx:%016llx rtt:%u", ret,
1033 : header->target.rdma.qpn, header->target.rdma.gid.global.subnetPrefix,
1034 : header->target.rdma.gid.global.interfaceId, rtt);
1035 1 : return ret;
1036 : }
1037 :
1038 2 : (void)memset_s((void *)header, RS_PING_PAYLOAD_HEADER_MASK_SIZE, 0, RS_PING_PAYLOAD_HEADER_MASK_SIZE);
1039 2 : RS_PTHREAD_MUTEX_LOCK(&targetInfo->tripMutex);
1040 2 : targetInfo->resultSummary.recvCnt++;
1041 2 : targetInfo->resultSummary.taskId = header->taskId;
1042 : // rtt timeout, increase timeout_cnt
1043 2 : if ((targetInfo->resultSummary.taskAttr.timeoutInterval * RS_PING_MSEC_TO_USEC) < rtt) {
1044 1 : targetInfo->resultSummary.timeoutCnt++;
1045 1 : hccp_dbg("recv_cnt:%u timeout_interval:%u rtt:%u timeout_cnt:%u", targetInfo->resultSummary.recvCnt,
1046 : targetInfo->resultSummary.taskAttr.timeoutInterval, rtt, targetInfo->resultSummary.timeoutCnt);
1047 1 : RS_PTHREAD_MUTEX_ULOCK(&targetInfo->tripMutex);
1048 1 : return 0;
1049 : }
1050 :
1051 : // handle rtt_min, rtt_max, rtt_avg
1052 1 : if (targetInfo->resultSummary.rttMin > rtt) {
1053 0 : targetInfo->resultSummary.rttMin = rtt;
1054 : }
1055 1 : if (targetInfo->resultSummary.rttMax < rtt) {
1056 1 : targetInfo->resultSummary.rttMax = rtt;
1057 : }
1058 1 : if (targetInfo->resultSummary.rttAvg == 0) {
1059 1 : targetInfo->resultSummary.rttAvg = rtt;
1060 : }
1061 1 : targetInfo->resultSummary.rttAvg = (targetInfo->resultSummary.rttAvg + rtt) / 2U;
1062 1 : RS_PTHREAD_MUTEX_ULOCK(&targetInfo->tripMutex);
1063 1 : return 0;
1064 : }
1065 :
1066 6 : STATIC void RsPongRocePollRcq(struct RsPingCtxCb *pingCb)
1067 : {
1068 6 : struct timeval timestamp = {0};
1069 6 : struct ibv_cq *evCq = NULL;
1070 6 : struct ibv_wc *wc = NULL;
1071 : uint32_t recvSgeIdx;
1072 6 : void *evCtx = NULL;
1073 : int polledCnt, i;
1074 : int ret;
1075 :
1076 : // record timestamp t4
1077 6 : (void)gettimeofday(×tamp, NULL);
1078 :
1079 6 : ret = RsIbvGetCqEvent(pingCb->pongQp.channel, &evCq, &evCtx);
1080 6 : if (ret != 0) {
1081 1 : hccp_err("rs_ibv_get_cq_event pong_qp.channel failed, ret:%d", ret);
1082 1 : return;
1083 : }
1084 :
1085 5 : if (evCq != pingCb->pongQp.recvCq.ibCq) {
1086 0 : hccp_err("CQ event for unknown CQ");
1087 0 : return;
1088 : }
1089 5 : pingCb->pongQp.recvCq.numEvents++;
1090 :
1091 5 : polledCnt = RsIbvPollCq(evCq, pingCb->pongQp.recvCq.maxRecvWcNum, gPongQpRecvWc);
1092 5 : if (polledCnt > pingCb->pongQp.recvCq.maxRecvWcNum || polledCnt < 0) {
1093 2 : hccp_err("rs_ibv_poll_cq failed, ret:%d", polledCnt);
1094 2 : return;
1095 : }
1096 :
1097 3 : wc = gPongQpRecvWc;
1098 6 : for (i = 0; i < polledCnt; i++) {
1099 3 : if (wc[i].status != IBV_WC_SUCCESS) {
1100 0 : hccp_err("wr_id:0x%llx error cqe %s(%d)", wc[i].wr_id, RsIbvWcStatusStr(wc[i].status), wc[i].status);
1101 0 : continue;
1102 : }
1103 3 : recvSgeIdx = (uint32_t)wc[i].wr_id;
1104 3 : if (recvSgeIdx >= pingCb->pongQp.recvMrCb.sgeNum) {
1105 3 : hccp_err("param err recv_sge_idx:%u > sge_num:%u", recvSgeIdx, pingCb->pongQp.recvMrCb.sgeNum);
1106 3 : continue;
1107 : }
1108 :
1109 : // handle response packet result
1110 0 : ret = RsPongResolveResponsePacket(pingCb, recvSgeIdx, ×tamp);
1111 0 : if (ret != 0) {
1112 0 : continue;
1113 : }
1114 :
1115 0 : ret = RsPingCommonPostRecv(&pingCb->pongQp);
1116 0 : if (ret != 0) {
1117 0 : continue;
1118 : }
1119 : }
1120 :
1121 3 : ret = RsIbvReqNotifyCq(evCq, 0);
1122 3 : if (ret != 0) {
1123 1 : hccp_err("rs_ibv_req_notify_cq failed, ret:%d", ret);
1124 : }
1125 :
1126 3 : return;
1127 : }
1128 :
1129 2 : STATIC int RsPingRoceGetTargetResult(struct RsPingCtxCb *pingCb, struct PingTargetCommInfo *target,
1130 : struct PingResultInfo *result)
1131 : {
1132 2 : struct RsPingTargetInfo *targetInfo = NULL;
1133 : int ret;
1134 :
1135 2 : ret = RsPingRoceFindTargetNode(pingCb, &target->qpInfo, &targetInfo);
1136 2 : if (ret != 0) {
1137 1 : hccp_err("rs_ping_roce_find_target_node failed, ret:%d qpn:%u gid:%016llx:%016llx", ret,
1138 : target->qpInfo.rdma.qpn, target->qpInfo.rdma.gid.global.subnetPrefix,
1139 : target->qpInfo.rdma.gid.global.interfaceId);
1140 1 : return ret;
1141 : }
1142 :
1143 1 : (void)memcpy_s(&result->summary, sizeof(struct PingResultSummary), &targetInfo->resultSummary,
1144 : sizeof(struct PingResultSummary));
1145 1 : if (targetInfo->state == RS_PING_PONG_TARGET_FINISH) {
1146 0 : result->state = PING_RESULT_STATE_VALID;
1147 : } else {
1148 1 : result->state = PING_RESULT_STATE_INVALID;
1149 : }
1150 :
1151 1 : hccp_dbg("ip:0x%llx qpn:%u, state:%d sendCnt:%u recvCnt:%u timeoutCnt:%u rttMin:%u rttMax:%u rttAvg:%u",
1152 : target->ip.addr.s_addr, target->qpInfo.rdma.qpn, result->state, result->summary.sendCnt,
1153 : result->summary.recvCnt, result->summary.timeoutCnt, result->summary.rttMin, result->summary.rttMax,
1154 : result->summary.rttAvg);
1155 :
1156 1 : return 0;
1157 : }
1158 :
1159 1 : STATIC void RsPingRoceFreeTargetNode(struct RsPingCtxCb *pingCb, struct RsPingTargetInfo *targetInfo)
1160 : {
1161 1 : RS_PTHREAD_MUTEX_LOCK(&pingCb->pingMutex);
1162 1 : RsListDel(&targetInfo->list);
1163 1 : RS_PTHREAD_MUTEX_ULOCK(&pingCb->pingMutex);
1164 :
1165 1 : if (targetInfo->payloadSize > 0 && targetInfo->payloadBuffer != NULL) {
1166 1 : free(targetInfo->payloadBuffer);
1167 1 : targetInfo->payloadBuffer = NULL;
1168 : }
1169 :
1170 1 : if (targetInfo->ah) {
1171 0 : (void)RsIbvDestroyAh(targetInfo->ah);
1172 : }
1173 1 : return;
1174 : }
1175 :
1176 1 : STATIC void RsPingPongDelTargetList(struct RsPingCtxCb *pingCb)
1177 : {
1178 1 : struct RsPongTargetInfo *pongNext = NULL;
1179 1 : struct RsPingTargetInfo *pingNext = NULL;
1180 1 : struct RsPongTargetInfo *pongCurr = NULL;
1181 1 : struct RsPingTargetInfo *pingCurr = NULL;
1182 :
1183 : // del ping_list
1184 1 : RS_PTHREAD_MUTEX_LOCK(&pingCb->pingMutex);
1185 1 : RS_LIST_GET_HEAD_ENTRY(pingCurr, pingNext, &pingCb->pingList, list, struct RsPingTargetInfo);
1186 2 : for (; (&pingCurr->list) != &pingCb->pingList;
1187 1 : pingCurr = pingNext, pingNext = list_entry(pingNext->list.next, struct RsPingTargetInfo, list)) {
1188 1 : RsListDel(&pingCurr->list);
1189 1 : if (pingCurr->payloadSize > 0 && pingCurr->payloadBuffer != NULL) {
1190 0 : free(pingCurr->payloadBuffer);
1191 0 : pingCurr->payloadBuffer = NULL;
1192 : }
1193 1 : if (pingCurr->ah) {
1194 0 : (void)RsIbvDestroyAh(pingCurr->ah);
1195 : }
1196 1 : (void)pthread_mutex_destroy(&pingCurr->tripMutex);
1197 1 : free(pingCurr);
1198 1 : pingCurr = NULL;
1199 : }
1200 1 : RS_PTHREAD_MUTEX_ULOCK(&pingCb->pingMutex);
1201 :
1202 : // del pong_list
1203 1 : RS_PTHREAD_MUTEX_LOCK(&pingCb->pongMutex);
1204 1 : RS_LIST_GET_HEAD_ENTRY(pongCurr, pongNext, &pingCb->pongList, list, struct RsPongTargetInfo);
1205 2 : for (; (&pongCurr->list) != &pingCb->pongList;
1206 1 : pongCurr = pongNext, pongNext = list_entry(pongNext->list.next, struct RsPongTargetInfo, list)) {
1207 1 : RsListDel(&pongCurr->list);
1208 1 : if (pongCurr->ah) {
1209 0 : (void)RsIbvDestroyAh(pongCurr->ah);
1210 : }
1211 1 : free(pongCurr);
1212 1 : pongCurr = NULL;
1213 : }
1214 1 : RS_PTHREAD_MUTEX_ULOCK(&pingCb->pongMutex);
1215 1 : }
1216 :
1217 2 : STATIC void RsPingRocePingCbDeinit(unsigned int phyId, struct RsPingCtxCb *pingCb)
1218 : {
1219 2 : struct rs_cb *rscb = NULL;
1220 : int ret;
1221 :
1222 2 : ret = RsGetRsCb(phyId, &rscb);
1223 2 : if (ret != 0) {
1224 1 : hccp_err("RsGetRsCb failed, phyId[%u] invalid, ret %d", phyId, ret);
1225 1 : return;
1226 : }
1227 :
1228 1 : RS_PTHREAD_MUTEX_LOCK(&pingCb->pingMutex);
1229 1 : pingCb->taskStatus = RS_PING_TASK_RESET;
1230 1 : RS_PTHREAD_MUTEX_ULOCK(&pingCb->pingMutex);
1231 :
1232 1 : RsPingPongDelTargetList(pingCb);
1233 :
1234 1 : RsPingCommonDeinitLocalQp(rscb, pingCb, &pingCb->pongQp);
1235 1 : RsPingCommonDeinitLocalQp(rscb, pingCb, &pingCb->pingQp);
1236 1 : RsPingCommonDeinitLocalBuffer(pingCb);
1237 1 : (void)RsIbvDeallocPd(pingCb->rdevCb.ibPd);
1238 1 : (void)RsIbvCloseDevice(pingCb->rdevCb.ibCtx);
1239 1 : RsIbvFreeDeviceList(pingCb->rdevCb.devList);
1240 : }
1241 :
1242 2 : STATIC void RsPingRoceAddTargetSuccess(struct PingTargetInfo *target, struct RsPingTargetInfo *targetInfo)
1243 : {
1244 2 : hccp_info("target ip:0x%llx payload_size:%u add success, qpn:%u uuid:0x%llx", target->remoteInfo.ip.addr.s_addr,
1245 : target->payload.size, targetInfo->qpInfo.rdma.qpn, targetInfo->uuid);
1246 2 : }
1247 :
1248 1 : STATIC void RsPingRocePingCbInitSuccess(unsigned int phyId, struct PingInitAttr *attr, unsigned int devIndex)
1249 : {
1250 1 : hccp_run_info("ping_cb init success, phyId:%u, localIp:0x%x, devIndex:%u", phyId,
1251 : attr->dev.rdma.localIp.addr.s_addr, devIndex);
1252 1 : }
1253 :
1254 1 : STATIC void RsPingRoceCannotFindTargetNode(unsigned int i, int ret, struct PingTargetCommInfo target,
1255 : unsigned int phyId)
1256 : {
1257 1 : hccp_err("rs_ping_roce_find_target_node i:%u failed, ret:%d ip:0x%llx qpn:%u phyId:%u", i, ret,
1258 : target.ip.addr.s_addr, target.qpInfo.rdma.qpn, phyId);
1259 1 : }
1260 :
1261 : struct RsPingPongOps gRsPingRoceOps = {
1262 : .checkPingFd = RsPingRoceCheckFd,
1263 : .checkPongFd = RsPongRoceCheckFd,
1264 : .initPingCb = RsPingRocePingCbInit,
1265 : .pingFindTargetNode = RsPingRoceFindTargetNode,
1266 : .pingAllocTargetNode = RsPingRoceAllocTargetNode,
1267 : .resetRecvBuffer = RsPingRoceResetRecvBuffer,
1268 : .pingPostSend = RsPingRocePostSend,
1269 : .pingPollScq = RsPingRocePollScq,
1270 : .pingPollRcq = RsPingRocePollRcq,
1271 : .pongHandleSend = RsPongRoceHandleSend,
1272 : .pongPollRcq = RsPongRocePollRcq,
1273 : .getTargetResult = RsPingRoceGetTargetResult,
1274 : .pingFreeTargetNode = RsPingRoceFreeTargetNode,
1275 : .deinitPingCb = RsPingRocePingCbDeinit,
1276 : };
1277 :
1278 : struct RsPingPongDfx gRsPingRoceDfx = {
1279 : .addTargetSuccess = RsPingRoceAddTargetSuccess,
1280 : .initPingCbSuccess = RsPingRocePingCbInitSuccess,
1281 : .pingCannotFindTargetNode = RsPingRoceCannotFindTargetNode,
1282 : };
1283 :
1284 27 : struct RsPingPongOps *RsPingRoceGetOps(void)
1285 : {
1286 27 : return &gRsPingRoceOps;
1287 : }
1288 :
1289 27 : struct RsPingPongDfx *RsPingRoceGetDfx(void)
1290 : {
1291 27 : return &gRsPingRoceDfx;
1292 : }
|
