Line data Source code
1 : /**
2 : * Copyright (c) 2026 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #ifndef RDMA_VENDOR_1825_OPS_H
12 : #define RDMA_VENDOR_1825_OPS_H
13 :
14 : #include "rma_conn_lite.h"
15 : #include "rdma_vendor_base_ops.h"
16 :
17 : namespace Hccl {
18 :
19 : // 先默认所有WQE都只占一个WQEBB
20 : #define WQEBB_SHIFT 6
21 :
22 : #define ROCE_WQE_OWNERBIT_SHIFT 7
23 : #define ROCE_WQE_CTRL_VALUE 0x40
24 : #define ROCE_SQ_VA_VALUE 0x20
25 : #define ROCE_SQ_SIGNAL_SHIFT 7
26 : #define ROCE_WQE_CQE_SIGNAL_SHIFT 7
27 : #define ROCE_WQE_CMP_TASK_LEN1 1u /* wqe cl is set to 1 */
28 : #define ROCE_WQE_CMP_TASK_LEN_SHIFT 28
29 : #define ROCE_TASK_SEG_ALIGN 8
30 : #define ROCE_WQE_FAST_DMA_SHIFT 10
31 : #define ROCE_WQE_SSN_MASK 0x3
32 : #define ROCE_WQE_SSN_SHIFT 12
33 : #define ROCE_WQE_DATA_SEG_SHIFT 4
34 : #define ROCE_WQE_TASK_SEG_LAST_EXT_LEN 4
35 : #define ROCE_WQE_TASK_REDUCE_OP_OFFSET 4
36 : #define WQE_SECTION_ALIGN_SHIFT 3
37 :
38 : #define ROCE_WQE_NEXT_SGE_INVALID (1u << 31)
39 :
40 : using Roce3CtrlSeg = struct {
41 : // Control Segment
42 : uint8_t owner_sl; /* ownerbit:ctrl_section_length:csl:dif_sl = 1:2:2:3; */
43 : uint8_t df_tsl; /* cr:df:va:tal = 1:1:1:5; */
44 : uint16_t wf_bdsl; /* cf:wf:wqe_msn:fde:fast:drvsl:bdsl = 1:1:2:1:1:2:8; */
45 : uint32_t cl_pi; /* cl:signature:mask_pi = 4:8:20; */
46 : uint64_t db; /* used by direct wqe*/
47 : };
48 :
49 : union Roce3TaskSeg {
50 : struct {
51 : uint32_t xrcSrqn : 18;
52 : uint32_t ext : 1;
53 : uint32_t dif : 1;
54 : uint32_t rsvd0 : 3;
55 : uint32_t so : 1;
56 : uint32_t opType : 5;
57 : uint32_t signal : 1;
58 : uint32_t fence : 1;
59 : uint32_t se : 1; /* solited event flag; */
60 : } dw0;
61 : uint32_t value;
62 : };
63 :
64 : using Roce3TaskWqeSeg = struct {
65 : // Task Wqe Segment
66 : union Roce3TaskSeg tskSeg;
67 : uint32_t dataLen;
68 : uint32_t immData;
69 : union {
70 : struct {
71 : uint32_t lastExtLen : 8;
72 : uint32_t cmdLen : 8;
73 : uint32_t pi : 16;
74 : } bs;
75 :
76 : uint32_t feth; /* cflush feth header */
77 : uint32_t value;
78 : } dw3;
79 : uint64_t va; /* to indicate remote buf address; */
80 : uint32_t rkey; /* to indicate remote mr buf; */
81 : uint32_t ulp; /* ulp预留字段 */
82 : };
83 :
84 : static constexpr uint32_t DTYPE_INVALID = 0xFFu;
85 :
86 : static const uint32_t Roce3ReduceDataTypeMap[] = {
87 : /* [DINT8] = */ 0x0, // INT8
88 : /* [DINT16] = */ 0x1, // INT16
89 : /* [DINT32] = */ 0x2, // INT32
90 : /* [DFP16] = */ 0x6, // FP16_NORMAL
91 : /* [DFP32] = */ 0x7, // FP32
92 : /* [DINT64] = */ DTYPE_INVALID, // 硬件不支持
93 : /* [DUINT32] = */ DTYPE_INVALID, // 硬件不支持
94 : };
95 :
96 : static const uint32_t Roce3ReduceOpTypeMap[] = {
97 : /* [SUM] = */ 0xA, // ADD
98 : /* [PROD] = */ DTYPE_INVALID, // 求积, 硬件不支持
99 : /* [MAX] = */ 0x8, // MAX
100 : /* [MIN] = */ 0x9, // MIN
101 : /* [EQUAL] = */ 0xB, // EQUAL
102 : };
103 :
104 : using Roce3WqeDataSeg = struct {
105 : uint64_t bufAddr; /* buffer address that wqe sge indicate; */
106 : uint32_t rLen; /* buffer length that wqe sge indicate; */
107 : uint32_t leKey; /* buffer lkey that wqe sge indicate; */
108 : };
109 :
110 : using Roce3WqeEntry = struct {
111 : Roce3CtrlSeg ctrl;
112 : Roce3TaskWqeSeg task;
113 : Roce3WqeDataSeg data;
114 : };
115 :
116 : // Cqe related
117 : constexpr uint32_t ROCE_CQE_OPCODE_SHIFT = 27;
118 : constexpr uint32_t ROCE_CQE_OPCODE_MASK = 0x1f;
119 : constexpr uint32_t ROCE_CQ_UPDATE_CI_MASK = 0xffffff; /* cq arm db ci mask; */
120 : constexpr uint32_t ROCE_CQE_MAX_GEN_NUM = 1024;
121 : constexpr uint32_t ROCE_CQE_OWNERBIT_SHIFT = 31;
122 : constexpr uint32_t ROCE_CQE_QPN_MASK = 0xfffff;
123 :
124 : enum class RoceCqeType : uint32_t {
125 : ERROR = 0x1e, /* indicate the cqe is an error cqe */
126 : RESIZE = 0x16, /* indicate the cqe is an resize type cqe */
127 : INVALID = 0x1f
128 : };
129 :
130 : struct Roce3CqeEntry{
131 : uint32_t owner_id_qpn; /* ownerbit: cqe_size:dif_en:wq_id:error_code:qpn = 1:2:1:4:4:20; */
132 : uint32_t op_sr_wqebb; /* opcode:s_r:inline_r:merge:fake:rsvd:linkwqe_used:wqebb_index = 5:1:1:1:1:2:1:20; */
133 : uint32_t byte_cnt; /* reflect the data size we reveive; */
134 : uint32_t imm_data; /* the immediate data we receive */
135 :
136 : union {
137 : struct {
138 : uint8_t smac_h[2]; /* smac value; */
139 : uint16_t vlan_id_pri; /* vlan_id:vlan_priority:reserved:smac = 12:1:3:16; */
140 : } mac_vlanid;
141 : } dw4;
142 : union {
143 : uint8_t smac_l[4]; /* smac value */
144 : uint32_t wqe_num; /* merged wqe num */
145 : };
146 : uint32_t vlan_queue_index; /* vlan:force_loopback:reserved:RQPN = 2:1:5:24; */
147 : uint8_t syndrome; /* record syndrome code; */
148 : uint8_t reserved2;
149 : uint16_t wqe_counter; /* the wqe index, valid for sq; */
150 : };
151 :
152 : // Doorbell related
153 : constexpr uint32_t ROCE_SQ_DOORBELL_TYPE = 21;
154 : constexpr uint32_t ROCE_INIT_SQ_DB_SGIT_IDX = 1;
155 :
156 : using Roce3DbEntry = struct {
157 : union {
158 : struct {
159 : uint32_t qpn : 20; /* indicate the sq qpn; */
160 : uint32_t cntxSize : 2; /* indicate the qpc size; */
161 : uint32_t r : 1; /* reserved bit; */
162 : uint32_t c : 1;
163 : uint32_t cos : 3;
164 : uint32_t type : 5;
165 :
166 : uint32_t pi : 8;
167 : uint32_t resv : 8;
168 : uint32_t xrcvld : 1;
169 : uint32_t vxlan : 1;
170 : uint32_t mtuShift : 3;
171 : uint32_t sgidIndex : 7;
172 : uint32_t subType : 4;
173 : } bs;
174 : uint64_t db_value;
175 : } dw0;
176 : };
177 :
178 : enum class ROCE3_OPCODE : uint32_t {
179 : ROCE_OPCODE_RDMA_WRITE = 4U,
180 : ROCE_OPCODE_RDMA_READ = 8U
181 : };
182 :
183 :
184 : // post_send helper
185 : struct Roce3PostSendParams {
186 : const SqeConfigLite &cfg;
187 : const RmaBufSliceLite &loc;
188 : const RmtRmaBufSliceLite &rmt;
189 : const uint32_t opCode;
190 :
191 : Roce3PostSendParams(const Roce3PostSendParams &) = delete;
192 : Roce3PostSendParams &operator=(const Roce3PostSendParams &) = delete;
193 : };
194 :
195 : class Rdma1825Ops : public RdmaBaseOps {
196 : public:
197 39 : Rdma1825Ops(RdmaSqContextLite *sqContext, RdmaCqContextLite *cqContext)
198 39 : : RdmaBaseOps(sqContext, cqContext) {}
199 :
200 78 : ~Rdma1825Ops() override {}
201 :
202 7 : HcclResult BuildDoorbell(u64 &dbAddr, u64 &dbValue) override
203 : {
204 : Roce3DbEntry dbEntry;
205 7 : dbEntry.dw0.db_value = 0;
206 7 : dbEntry.dw0.bs.r = 0;
207 7 : dbEntry.dw0.bs.c = 0;
208 7 : dbEntry.dw0.bs.cntxSize = 1;
209 7 : dbEntry.dw0.bs.qpn = sqContext_->qpn;
210 7 : dbEntry.dw0.bs.subType = 0;
211 7 : dbEntry.dw0.bs.resv = 0;
212 7 : dbEntry.dw0.bs.pi = ((sqHead_ >> 8) & 0xff);
213 7 : dbEntry.dw0.bs.sgidIndex = ROCE_INIT_SQ_DB_SGIT_IDX;
214 7 : dbEntry.dw0.bs.type = ROCE_SQ_DOORBELL_TYPE;
215 7 : dbEntry.dw0.bs.mtuShift = sqContext_->mtuShift;
216 7 : dbEntry.dw0.bs.cos = 0x7;
217 :
218 7 : dbAddr = sqContext_->dbHwVa;
219 7 : dbValue = dbEntry.dw0.db_value;
220 :
221 21 : HCCL_INFO("[Rdma1825Ops::%s] SQ DB ready, qpn[%u], sqHead[%u], dbAddr[0x%llx], dbValue[0x%llx]",
222 : __func__, sqContext_->qpn, sqHead_, dbAddr, dbValue);
223 7 : return HCCL_SUCCESS;
224 : }
225 :
226 3 : HcclResult BuildCqDoorbell(u64 &dbAddr, u64 &dbValue) override
227 : {
228 3 : if (cqDbFlush_) {
229 2 : dbAddr = cqContext_->dbSwVa;
230 :
231 2 : const uint32_t dbValue32 = Htonl32(cqTail_ & ROCE_CQ_UPDATE_CI_MASK);
232 2 : dbValue = dbValue32;
233 :
234 : // Ring Cq Soft DB
235 2 : auto ret = memcpy_sp(reinterpret_cast<void *>(dbAddr), sizeof(uint32_t), &dbValue32, sizeof(uint32_t));
236 2 : if (UNLIKELY(ret != 0)) {
237 0 : THROW<InternalException>(
238 0 : StringFormat("[Rdma1825Ops::%s] write soft Cq DB failed, ret = %d", __func__, ret));
239 : }
240 6 : HCCL_INFO("[Rdma1825Ops::%s] CQ DB updated, cqTail[%u], dbAddr[0x%llx], dbValue[0x%llx]",
241 : __func__, cqTail_, dbAddr, dbValue);
242 : } else {
243 1 : dbAddr = 0;
244 1 : dbValue = 0;
245 : }
246 :
247 3 : cqDbFlush_ = false;
248 :
249 3 : return HCCL_SUCCESS;
250 : }
251 :
252 : protected:
253 3 : HcclResult BuildReadWqe(const RmaBufSliceLite &loc, const RmtRmaBufSliceLite &rmt, const SqeConfigLite &cfg) override
254 : {
255 3 : Roce3WqeEntry wqe{};
256 3 : Roce3PostSendParams params{cfg, loc, rmt, static_cast<uint32_t>(ROCE3_OPCODE::ROCE_OPCODE_RDMA_READ)};
257 :
258 3 : CHK_RET(FillCtrlSeg(&wqe, params));
259 3 : CHK_RET(FillTaskSeg(&wqe, params));
260 3 : CHK_RET(FillDataSeg(&wqe, params));
261 :
262 3 : CHK_RET(CommitWqe(&wqe, sizeof(Roce3WqeEntry)));
263 9 : HCCL_INFO("[Rdma1825Ops::%s] WQE committed, qpn[%u], sqHead[%u], loc[0x%llx], rmt[0x%llx], size[%u]",
264 : __func__, sqContext_->qpn, sqHead_, loc.GetAddr(), rmt.GetAddr(), loc.GetSize());
265 3 : return HCCL_SUCCESS;
266 : }
267 :
268 6 : HcclResult BuildWriteWqe(const RmaBufSliceLite &loc, const RmtRmaBufSliceLite &rmt, const SqeConfigLite &cfg) override
269 : {
270 6 : Roce3WqeEntry wqe{};
271 6 : Roce3PostSendParams params{cfg, loc, rmt, static_cast<uint32_t>(ROCE3_OPCODE::ROCE_OPCODE_RDMA_WRITE)};
272 :
273 6 : CHK_RET(FillCtrlSeg(&wqe, params));
274 6 : CHK_RET(FillTaskSeg(&wqe, params));
275 6 : CHK_RET(FillDataSeg(&wqe, params));
276 :
277 6 : CHK_RET(CommitWqe(&wqe, sizeof(Roce3WqeEntry)));
278 18 : HCCL_INFO("[Rdma1825Ops::%s] WQE committed, qpn[%u], sqHead[%u], loc[0x%llx], rmt[0x%llx], size[%u]",
279 : __func__, sqContext_->qpn, sqHead_, loc.GetAddr(), rmt.GetAddr(), loc.GetSize());
280 6 : return HCCL_SUCCESS;
281 : }
282 :
283 2 : HcclResult BuildWriteReduceWqe(const RmaBufSliceLite &loc, const RmtRmaBufSliceLite &rmt,
284 : const SqeConfigLite &cfg, DataType dataType, ReduceOp reduceOp) override
285 : {
286 2 : Roce3WqeEntry wqe{};
287 2 : Roce3PostSendParams params{cfg, loc, rmt, static_cast<uint32_t>(ROCE3_OPCODE::ROCE_OPCODE_RDMA_WRITE)};
288 :
289 2 : CHK_RET(FillCtrlSeg(&wqe, params));
290 2 : CHK_RET(FillTaskSeg(&wqe, params));
291 2 : CHK_RET(FillReduceTaskSeg(&wqe, params, dataType, reduceOp));
292 2 : CHK_RET(FillDataSeg(&wqe, params));
293 :
294 2 : CHK_RET(CommitWqe(&wqe, sizeof(Roce3WqeEntry)));
295 6 : HCCL_INFO("[Rdma1825Ops::%s] WQE committed, qpn[%u], sqHead[%u], loc[0x%llx], rmt[0x%llx], size[%u], "
296 : "dataType[%u], reduceOp[%u]",
297 : __func__, sqContext_->qpn, sqHead_, loc.GetAddr(), rmt.GetAddr(), loc.GetSize(),
298 : static_cast<uint32_t>(dataType), static_cast<uint32_t>(reduceOp));
299 2 : return HCCL_SUCCESS;
300 : }
301 :
302 0 : HcclResult WriteInvalidWqebb(uint32_t nextIdx) override
303 : {
304 0 : const uint32_t sqDepth = sqContext_->depth;
305 0 : const uint32_t sqMask = sqDepth - 1U;
306 0 : uint8_t ownerSl = ((nextIdx & sqDepth) == 0) ? 0xff : 0x7f;
307 :
308 0 : auto *dst = reinterpret_cast<void *>(
309 0 : sqContext_->sqVa + static_cast<uint64_t>(nextIdx & sqMask) * sqContext_->wqeSize);
310 :
311 0 : auto ret = memcpy_sp(dst, sizeof(ownerSl), &ownerSl, sizeof(ownerSl));
312 0 : if (UNLIKELY(ret != 0)) {
313 0 : THROW<InternalException>(
314 0 : StringFormat("[Rdma1825Ops::%s] write invalid wqebb failed, ret = %d", __func__, ret));
315 : }
316 0 : return HCCL_SUCCESS;
317 : }
318 :
319 3 : int32_t PollCqImpl(int32_t numEntries, std::vector<int32_t> &errList) override
320 : {
321 3 : int32_t pollNum = 0;
322 3 : CqPollStatus ret = CqPollStatus::ERROR;
323 :
324 4 : while (pollNum < numEntries) {
325 3 : ret = PollOne(errList);
326 3 : if (ret != CqPollStatus::SUCCESS) {
327 2 : if (ret != CqPollStatus::EMPTY && ret != CqPollStatus::ERROR) {
328 0 : HCCL_ERROR("[Rdma1825Ops::%s][Poll cq] Poll CQ error, ret: %d", __func__,
329 : static_cast<int32_t>(ret));
330 : }
331 2 : break;
332 : }
333 :
334 : // Update Cq CI
335 1 : cqTail_ += 1;
336 :
337 1 : ++pollNum;
338 : }
339 :
340 3 : if ((pollNum != 0) || (ret == CqPollStatus::ERROR)) {
341 : // Need to flush Cq DB
342 2 : cqDbFlush_ = true;
343 : }
344 :
345 3 : return (ret == CqPollStatus::ERROR) ? static_cast<int32_t>(ret) : pollNum;
346 : }
347 :
348 : private:
349 11 : HcclResult FillCtrlSeg(Roce3WqeEntry *wqe, const Roce3PostSendParams ¶ms) const
350 : {
351 : // 赋值本wqe的owner
352 11 : uint8_t owner = (sqHead_ & (sqContext_->depth)) == 0 ? 0 : 1;
353 11 : wqe->ctrl.owner_sl = (owner << ROCE_WQE_OWNERBIT_SHIFT) | ROCE_WQE_CTRL_VALUE;
354 :
355 : // 不产生cqe + task字段的长度
356 11 : wqe->ctrl.df_tsl = ((params.cfg.cqeEn == true) ? (1U << ROCE_SQ_SIGNAL_SHIFT) : 0) | ROCE_SQ_VA_VALUE;
357 11 : wqe->ctrl.df_tsl |= sizeof(Roce3TaskWqeSeg) / ROCE_TASK_SEG_ALIGN;
358 :
359 : // fast_dma + SSN + sge长度
360 11 : wqe->ctrl.wf_bdsl = Htons16(static_cast<uint16_t>(0 << ROCE_WQE_FAST_DMA_SHIFT));
361 11 : wqe->ctrl.wf_bdsl |= Htons16((sqHead_ & ROCE_WQE_SSN_MASK) << ROCE_WQE_SSN_SHIFT);
362 11 : wqe->ctrl.wf_bdsl |= Htons16(static_cast<uint16_t>(static_cast<uint32_t>(1) << (ROCE_WQE_DATA_SEG_SHIFT - WQE_SECTION_ALIGN_SHIFT)));
363 :
364 : // cl
365 11 : wqe->ctrl.cl_pi = Htonl32(ROCE_WQE_CMP_TASK_LEN1 << ROCE_WQE_CMP_TASK_LEN_SHIFT);
366 :
367 11 : return HCCL_SUCCESS;
368 : }
369 :
370 11 : HcclResult FillTaskSeg(Roce3WqeEntry *wqe, const Roce3PostSendParams ¶ms) const
371 : {
372 : // ----- Task Seg -----
373 11 : wqe->task.tskSeg.value = 0;
374 11 : wqe->task.tskSeg.dw0.signal = !!((wqe->ctrl.df_tsl & (1U << ROCE_WQE_CQE_SIGNAL_SHIFT)) > 0);
375 11 : wqe->task.tskSeg.dw0.fence = params.cfg.fence;
376 11 : wqe->task.tskSeg.dw0.opType = params.opCode;
377 11 : wqe->task.tskSeg.dw0.se = 0;
378 11 : wqe->task.tskSeg.value = Htonl32(wqe->task.tskSeg.value);
379 :
380 11 : wqe->task.dataLen = Htonl32(params.loc.GetSize());
381 11 : wqe->task.immData = 0;
382 11 : wqe->task.dw3.value = (params.opCode == static_cast<uint32_t>(ROCE3_OPCODE::ROCE_OPCODE_RDMA_READ) ? Htonl32(ROCE_WQE_TASK_SEG_LAST_EXT_LEN) : 0);
383 11 : wqe->task.va = Htonll64(params.rmt.GetAddr());
384 11 : wqe->task.rkey = Htonl32(params.rmt.GetRkey());
385 11 : wqe->task.ulp = Htonl32(params.loc.GetLkey() & 0xffff);
386 :
387 11 : return HCCL_SUCCESS;
388 : }
389 :
390 2 : HcclResult FillReduceTaskSeg(Roce3WqeEntry *wqe, const Roce3PostSendParams ¶ms, DataType dataType, ReduceOp reduceOp) const
391 : {
392 : // Hcomm Type -> Roce3 Type
393 2 : uint32_t dataTypeIdx = static_cast<uint32_t>(dataType);
394 2 : if (dataTypeIdx >= sizeof(Roce3ReduceDataTypeMap) / sizeof(uint32_t)) {
395 0 : HCCL_ERROR("[Rdma1825Ops::%s] invalid DataType %u", __func__, dataTypeIdx);
396 0 : return HCCL_E_PARA;
397 : }
398 2 : uint32_t reduceDataType = Roce3ReduceDataTypeMap[dataTypeIdx];
399 2 : if (reduceDataType == DTYPE_INVALID) {
400 0 : HCCL_ERROR("[Rdma1825Ops::%s] DataType %u not support by inline reduce", __func__, dataTypeIdx);
401 0 : return HCCL_E_NOT_SUPPORT;
402 : }
403 :
404 2 : uint32_t reduceOpIdx = static_cast<uint32_t>(reduceOp);
405 2 : if (reduceOpIdx >= sizeof(Roce3ReduceOpTypeMap) / sizeof(uint32_t)) {
406 0 : HCCL_ERROR("[Rdma1825Ops::%s] invalid ReduceOpType %u", __func__, reduceOpIdx);
407 0 : return HCCL_E_PARA;
408 : }
409 2 : uint32_t reduceOpType = Roce3ReduceOpTypeMap[reduceOpIdx];
410 2 : if (reduceOpType == DTYPE_INVALID) {
411 0 : HCCL_ERROR("[Rdma1825Ops::%s] ReduceOpType %u not support by inline reduce", __func__, reduceOpIdx);
412 0 : return HCCL_E_NOT_SUPPORT;
413 : }
414 :
415 : // Reduce data
416 2 : uint32_t mptIndex = params.loc.GetLkey() & 0xffff;
417 2 : uint32_t reduceOpInfo = ((reduceOpType << ROCE_WQE_TASK_REDUCE_OP_OFFSET) | reduceDataType);
418 :
419 : // ----- Task Reduce Seg -----
420 2 : wqe->task.ulp = Htonl32(mptIndex | reduceOpInfo);
421 :
422 2 : return HCCL_SUCCESS;
423 : }
424 :
425 11 : HcclResult FillDataSeg(Roce3WqeEntry *wqe, const Roce3PostSendParams ¶ms) const
426 : {
427 : // ----- Data Seg -----
428 11 : wqe->data.bufAddr = Htonll64(static_cast<uint64_t>(params.loc.GetAddr()));
429 11 : wqe->data.leKey = Htonl32((params.loc.GetLkey() & (~ROCE_WQE_NEXT_SGE_INVALID)) | ROCE_WQE_NEXT_SGE_INVALID); // 当前sge就是最后一个
430 11 : wqe->data.rLen = Htonl32(params.loc.GetSize());
431 :
432 11 : return HCCL_SUCCESS;
433 : }
434 :
435 0 : CqPollStatus PollOne(std::vector<int32_t> &errList)
436 : {
437 0 : bool need_poll = true;
438 0 : while (need_poll) {
439 0 : Roce3CqeEntry *rcqe = nullptr;
440 0 : rcqe = Roce3GetOneCqe(cqHead_);
441 0 : if (rcqe == nullptr) {
442 0 : return CqPollStatus::EMPTY;
443 : }
444 :
445 0 : ++cqHead_;
446 :
447 0 : uint32_t syndrome = static_cast<uint32_t>(rcqe->syndrome);
448 0 : uint32_t ownerIdQpn = static_cast<uint32_t>(rcqe->owner_id_qpn);
449 0 : auto cqeType = static_cast<RoceCqeType>(
450 0 : (rcqe->op_sr_wqebb >> ROCE_CQE_OPCODE_SHIFT) & ROCE_CQE_OPCODE_MASK);
451 : // 存在错误
452 0 : if (cqeType == RoceCqeType::ERROR) {
453 0 : const uint32_t cqeErrCode = (ownerIdQpn >> 20U) & 0xfU;
454 :
455 0 : HCCL_ERROR("[Rdma1825Ops::%s][Poll cq] CQE error, qpn[%u], syndrome[%u], cqeErrCode[%u], "
456 : "wqeCounter[%u]",
457 : __func__, ownerIdQpn & ROCE_CQE_QPN_MASK, syndrome, cqeErrCode,
458 : static_cast<uint32_t>(rcqe->wqe_counter));
459 :
460 : // 后续优化为具体ErrorCode
461 0 : int32_t ERROR_CODE = 1;
462 0 : errList.push_back(ERROR_CODE);
463 0 : return CqPollStatus::ERROR;
464 : }
465 :
466 0 : HCCL_INFO("[Rdma1825Ops::%s][Poll cq] CQE completed, cqHead[%u], qpn[%u], syndrome[%u], "
467 : "wqeCounter[%u]",
468 : __func__, cqHead_, ownerIdQpn & ROCE_CQE_QPN_MASK, syndrome,
469 : static_cast<uint32_t>(rcqe->wqe_counter));
470 0 : need_poll = false;
471 : }
472 :
473 0 : return CqPollStatus::SUCCESS;
474 : }
475 :
476 0 : Roce3CqeEntry* Roce3GetOneCqe(uint32_t consIndex)
477 : {
478 0 : uint32_t cqeSize = 64;
479 0 : uint32_t cqDepthReal = cqContext_->cqDepth + ROCE_CQE_MAX_GEN_NUM; // Must be power of 2
480 0 : uint32_t cqeMask = cqDepthReal - 1;
481 0 : uint32_t cqeSlot = consIndex & cqeMask;
482 :
483 : // Calculate Cqe Addr
484 0 : auto cqeAddr = reinterpret_cast<Roce3CqeEntry *>(
485 0 : reinterpret_cast<uint8_t *>(cqContext_->cqVa) + static_cast<size_t>(cqeSlot * cqeSize));
486 :
487 : // Memcpy Cqe
488 0 : auto ret = memcpy_sp(&cqeReadback_, sizeof(cqeReadback_), cqeAddr, sizeof(cqeReadback_));
489 0 : if (UNLIKELY(ret != 0)) {
490 0 : THROW<InternalException>(StringFormat("[Rdma1825Ops::%s] memcpy_sp cqe failed, ret = %d", __func__, ret));
491 : }
492 :
493 : // Cqe Details Parse
494 0 : uint32_t opSrWqebb = cqeReadback_.op_sr_wqebb;
495 0 : uint32_t ownerIdQpn = cqeReadback_.owner_id_qpn;
496 0 : auto cqeType = static_cast<RoceCqeType>((opSrWqebb >> ROCE_CQE_OPCODE_SHIFT) & ROCE_CQE_OPCODE_MASK);
497 0 : if (cqeType == RoceCqeType::INVALID) {
498 0 : HCCL_INFO("[Rdma1825Ops::%s][Poll cq] CQE invalid, consIndex[%u], slot[%u]", __func__, consIndex,
499 : cqeSlot);
500 0 : return nullptr;
501 : }
502 :
503 : // Judge Owner bit
504 0 : uint32_t calHwOwner = static_cast<uint32_t>((consIndex & cqDepthReal) == 0);
505 0 : uint32_t curCqeOwner = !((ownerIdQpn & (1U << ROCE_CQE_OWNERBIT_SHIFT)) == 0);
506 0 : if ((calHwOwner ^ curCqeOwner) == 0) {
507 0 : HCCL_INFO("[Rdma1825Ops::%s][Poll cq] CQE owner not ready, consIndex[%u], slot[%u], "
508 : "calHwOwner[%u], curCqeOwner[%u]",
509 : __func__, consIndex, cqeSlot, calHwOwner, curCqeOwner);
510 0 : return nullptr;
511 : }
512 :
513 0 : return &cqeReadback_;
514 : }
515 :
516 : Roce3CqeEntry cqeReadback_{};
517 : };
518 :
519 : }
520 : #endif // RDMA_VENDOR_1825_OPS_H
|
