Line data Source code
1 : /**
2 : * Copyright (c) 2026 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #ifndef RDMA_VENDOR_1825_OPS_H
12 : #define RDMA_VENDOR_1825_OPS_H
13 :
14 : #include "rma_conn_lite.h"
15 : #include "rdma_vendor_base_ops.h"
16 :
17 : namespace Hccl {
18 :
19 : // 先默认所有WQE都只占一个WQEBB
20 : #define WQEBB_SHIFT 6
21 :
22 : #define ROCE_WQE_OWNERBIT_SHIFT 7
23 : #define ROCE_WQE_CTRL_VALUE 0x40
24 : #define ROCE_SQ_VA_VALUE 0x20
25 : #define ROCE_SQ_SIGNAL_SHIFT 7
26 : #define ROCE_WQE_CQE_SIGNAL_SHIFT 7
27 : #define ROCE_WQE_CMP_TASK_LEN1 1u /* wqe cl is set to 1 */
28 : #define ROCE_WQE_CMP_TASK_LEN_SHIFT 28
29 : #define ROCE_TASK_SEG_ALIGN 8
30 : #define ROCE_WQE_FAST_DMA_SHIFT 10
31 : #define ROCE_WQE_SSN_MASK 0x3
32 : #define ROCE_WQE_SSN_SHIFT 12
33 : #define ROCE_WQE_DATA_SEG_SHIFT 4
34 : #define ROCE_WQE_TASK_SEG_LAST_EXT_LEN 4
35 : #define ROCE_WQE_TASK_REDUCE_OP_OFFSET 4
36 : #define WQE_SECTION_ALIGN_SHIFT 3
37 :
38 : #define ROCE_WQE_NEXT_SGE_INVALID (1u << 31)
39 :
40 : using Roce3CtrlSeg = struct {
41 : // Control Segment
42 : uint8_t owner_sl; /* ownerbit:ctrl_section_length:csl:dif_sl = 1:2:2:3; */
43 : uint8_t df_tsl; /* cr:df:va:tal = 1:1:1:5; */
44 : uint16_t wf_bdsl; /* cf:wf:wqe_msn:fde:fast:drvsl:bdsl = 1:1:2:1:1:2:8; */
45 : uint32_t cl_pi; /* cl:signature:mask_pi = 4:8:20; */
46 : uint64_t db; /* used by direct wqe*/
47 : };
48 :
49 : union Roce3TaskSeg {
50 : struct {
51 : uint32_t xrcSrqn : 18;
52 : uint32_t ext : 1;
53 : uint32_t dif : 1;
54 : uint32_t rsvd0 : 3;
55 : uint32_t so : 1;
56 : uint32_t opType : 5;
57 : uint32_t signal : 1;
58 : uint32_t fence : 1;
59 : uint32_t se : 1; /* solited event flag; */
60 : } dw0;
61 : uint32_t value;
62 : };
63 :
64 : using Roce3TaskWqeSeg = struct {
65 : // Task Wqe Segment
66 : union Roce3TaskSeg tskSeg;
67 : uint32_t dataLen;
68 : uint32_t immData;
69 : union {
70 : struct {
71 : uint32_t lastExtLen : 8;
72 : uint32_t cmdLen : 8;
73 : uint32_t pi : 16;
74 : } bs;
75 :
76 : uint32_t feth; /* cflush feth header */
77 : uint32_t value;
78 : } dw3;
79 : uint64_t va; /* to indicate remote buf address; */
80 : uint32_t rkey; /* to indicate remote mr buf; */
81 : uint32_t ulp; /* ulp预留字段 */
82 : };
83 :
84 : static constexpr uint32_t DTYPE_INVALID = 0xFFu;
85 :
86 : static const uint32_t Roce3ReduceDataTypeMap[] = {
87 : /* [DINT8] = */ 0x0, // INT8
88 : /* [DINT16] = */ 0x1, // INT16
89 : /* [DINT32] = */ 0x2, // INT32
90 : /* [DFP16] = */ 0x6, // FP16_NORMAL
91 : /* [DFP32] = */ 0x7, // FP32
92 : /* [DINT64] = */ DTYPE_INVALID, // 硬件不支持
93 : /* [DUINT32] = */ DTYPE_INVALID, // 硬件不支持
94 : };
95 :
96 : static const uint32_t Roce3ReduceOpTypeMap[] = {
97 : /* [SUM] = */ 0xA, // ADD
98 : /* [PROD] = */ DTYPE_INVALID, // 求积, 硬件不支持
99 : /* [MAX] = */ 0x8, // MAX
100 : /* [MIN] = */ 0x9, // MIN
101 : /* [EQUAL] = */ 0xB, // EQUAL
102 : };
103 :
104 : using Roce3WqeDataSeg = struct {
105 : uint64_t bufAddr; /* buffer address that wqe sge indicate; */
106 : uint32_t rLen; /* buffer length that wqe sge indicate; */
107 : uint32_t leKey; /* buffer lkey that wqe sge indicate; */
108 : };
109 :
110 : using Roce3WqeEntry = struct {
111 : Roce3CtrlSeg ctrl;
112 : Roce3TaskWqeSeg task;
113 : Roce3WqeDataSeg data;
114 : };
115 :
116 : // Cqe related
117 : constexpr uint32_t ROCE_CQE_OPCODE_SHIFT = 27;
118 : constexpr uint32_t ROCE_CQE_OPCODE_MASK = 0x1f;
119 : constexpr uint32_t ROCE_CQ_UPDATE_CI_MASK = 0xffffff; /* cq arm db ci mask; */
120 : constexpr uint32_t ROCE_CQE_MAX_GEN_NUM = 1024;
121 : constexpr uint32_t ROCE_CQE_OWNERBIT_SHIFT = 31;
122 : constexpr uint32_t ROCE_CQE_QPN_MASK = 0xfffff;
123 :
124 : enum class RoceCqeType : uint32_t {
125 : ERROR = 0x1e, /* indicate the cqe is an error cqe */
126 : RESIZE = 0x16, /* indicate the cqe is an resize type cqe */
127 : INVALID = 0x1f
128 : };
129 :
130 : struct Roce3CqeEntry {
131 : uint32_t owner_id_qpn; /* ownerbit: cqe_size:dif_en:wq_id:error_code:qpn = 1:2:1:4:4:20; */
132 : uint32_t op_sr_wqebb; /* opcode:s_r:inline_r:merge:fake:rsvd:linkwqe_used:wqebb_index = 5:1:1:1:1:2:1:20; */
133 : uint32_t byte_cnt; /* reflect the data size we reveive; */
134 : uint32_t imm_data; /* the immediate data we receive */
135 :
136 : union {
137 : struct {
138 : uint8_t smac_h[2]; /* smac value; */
139 : uint16_t vlan_id_pri; /* vlan_id:vlan_priority:reserved:smac = 12:1:3:16; */
140 : } mac_vlanid;
141 : } dw4;
142 : union {
143 : uint8_t smac_l[4]; /* smac value */
144 : uint32_t wqe_num; /* merged wqe num */
145 : };
146 : uint32_t vlan_queue_index; /* vlan:force_loopback:reserved:RQPN = 2:1:5:24; */
147 : uint8_t syndrome; /* record syndrome code; */
148 : uint8_t reserved2;
149 : uint16_t wqe_counter; /* the wqe index, valid for sq; */
150 : };
151 :
152 : // Doorbell related
153 : constexpr uint32_t ROCE_SQ_DOORBELL_TYPE = 21;
154 : constexpr uint32_t ROCE_INIT_SQ_DB_SGIT_IDX = 1;
155 :
156 : using Roce3DbEntry = struct {
157 : union {
158 : struct {
159 : uint32_t qpn : 20; /* indicate the sq qpn; */
160 : uint32_t cntxSize : 2; /* indicate the qpc size; */
161 : uint32_t r : 1; /* reserved bit; */
162 : uint32_t c : 1;
163 : uint32_t cos : 3;
164 : uint32_t type : 5;
165 :
166 : uint32_t pi : 8;
167 : uint32_t resv : 8;
168 : uint32_t xrcvld : 1;
169 : uint32_t vxlan : 1;
170 : uint32_t mtuShift : 3;
171 : uint32_t sgidIndex : 7;
172 : uint32_t subType : 4;
173 : } bs;
174 : uint64_t db_value;
175 : } dw0;
176 : };
177 :
178 : enum class ROCE3_OPCODE : uint32_t { ROCE_OPCODE_RDMA_WRITE = 4U, ROCE_OPCODE_RDMA_READ = 8U };
179 :
180 : // post_send helper
181 : struct Roce3PostSendParams {
182 : const SqeConfigLite& cfg;
183 : const RmaBufSliceLite& loc;
184 : const RmtRmaBufSliceLite& rmt;
185 : const uint32_t opCode;
186 :
187 : Roce3PostSendParams(const Roce3PostSendParams&) = delete;
188 : Roce3PostSendParams& operator=(const Roce3PostSendParams&) = delete;
189 : };
190 :
191 : class Rdma1825Ops : public RdmaBaseOps {
192 : public:
193 39 : Rdma1825Ops(RdmaSqContextLite* sqContext, RdmaCqContextLite* cqContext) : RdmaBaseOps(sqContext, cqContext) {}
194 :
195 78 : ~Rdma1825Ops() override {}
196 :
197 7 : HcclResult BuildDoorbell(u64& dbAddr, u64& dbValue) override
198 : {
199 : Roce3DbEntry dbEntry;
200 7 : dbEntry.dw0.db_value = 0;
201 7 : dbEntry.dw0.bs.r = 0;
202 7 : dbEntry.dw0.bs.c = 0;
203 7 : dbEntry.dw0.bs.cntxSize = 1;
204 7 : dbEntry.dw0.bs.qpn = sqContext_->qpn;
205 7 : dbEntry.dw0.bs.subType = 0;
206 7 : dbEntry.dw0.bs.resv = 0;
207 7 : dbEntry.dw0.bs.pi = ((sqHead_ >> 8) & 0xff);
208 7 : dbEntry.dw0.bs.sgidIndex = ROCE_INIT_SQ_DB_SGIT_IDX;
209 7 : dbEntry.dw0.bs.type = ROCE_SQ_DOORBELL_TYPE;
210 :
211 7 : dbEntry.dw0.bs.mtuShift = static_cast<uint32_t>(
212 7 : (sqContext_->dbVendorSpecified >> UB_DB_VENDOR_MTUSHIFT_SHIFT) & UB_DB_VENDOR_FIELD_MASK);
213 7 : dbEntry.dw0.bs.cos = static_cast<uint32_t>(
214 7 : (sqContext_->dbVendorSpecified >> UB_DB_VENDOR_COS_SHIFT) & UB_DB_VENDOR_FIELD_MASK);
215 :
216 7 : dbAddr = sqContext_->dbHwVa;
217 7 : dbValue = dbEntry.dw0.db_value;
218 :
219 21 : HCCL_INFO(
220 : "[Rdma1825Ops::%s] SQ DB ready, qpn[%u], sqHead[%u], mtuShift[%u], cos[%u], "
221 : "dbAddr[0x%llx], dbValue[0x%llx]",
222 : __func__, sqContext_->qpn, sqHead_, dbEntry.dw0.bs.mtuShift, dbEntry.dw0.bs.cos, dbAddr, dbValue);
223 7 : return HCCL_SUCCESS;
224 : }
225 :
226 3 : HcclResult BuildCqDoorbell(u64& dbAddr, u64& dbValue) override
227 : {
228 3 : if (cqDbFlush_) {
229 2 : dbAddr = cqContext_->dbSwVa;
230 :
231 2 : const uint32_t dbValue32 = Htonl32(cqTail_ & ROCE_CQ_UPDATE_CI_MASK);
232 2 : dbValue = dbValue32;
233 :
234 : // Ring Cq Soft DB
235 2 : auto ret = memcpy_sp(reinterpret_cast<void*>(dbAddr), sizeof(uint32_t), &dbValue32, sizeof(uint32_t));
236 2 : if (UNLIKELY(ret != 0)) {
237 0 : THROW<InternalException>(
238 0 : StringFormat("[Rdma1825Ops::%s] write soft Cq DB failed, ret = %d", __func__, ret));
239 : }
240 6 : HCCL_INFO(
241 : "[Rdma1825Ops::%s] CQ DB updated, cqTail[%u], dbAddr[0x%llx], dbValue[0x%llx]", __func__, cqTail_,
242 : dbAddr, dbValue);
243 : } else {
244 1 : dbAddr = 0;
245 1 : dbValue = 0;
246 : }
247 :
248 3 : cqDbFlush_ = false;
249 :
250 3 : return HCCL_SUCCESS;
251 : }
252 :
253 : protected:
254 : HcclResult
255 3 : BuildReadWqe(const RmaBufSliceLite& loc, const RmtRmaBufSliceLite& rmt, const SqeConfigLite& cfg) override
256 : {
257 3 : Roce3WqeEntry wqe{};
258 3 : Roce3PostSendParams params{cfg, loc, rmt, static_cast<uint32_t>(ROCE3_OPCODE::ROCE_OPCODE_RDMA_READ)};
259 :
260 3 : CHK_RET(FillCtrlSeg(&wqe, params));
261 3 : CHK_RET(FillTaskSeg(&wqe, params));
262 3 : CHK_RET(FillDataSeg(&wqe, params));
263 :
264 3 : CHK_RET(CommitWqe(&wqe, sizeof(Roce3WqeEntry)));
265 9 : HCCL_INFO(
266 : "[Rdma1825Ops::%s] WQE committed, qpn[%u], sqHead[%u], loc[0x%llx], rmt[0x%llx], size[%u]", __func__,
267 : sqContext_->qpn, sqHead_, loc.GetAddr(), rmt.GetAddr(), loc.GetSize());
268 3 : return HCCL_SUCCESS;
269 : }
270 :
271 : HcclResult
272 6 : BuildWriteWqe(const RmaBufSliceLite& loc, const RmtRmaBufSliceLite& rmt, const SqeConfigLite& cfg) override
273 : {
274 6 : Roce3WqeEntry wqe{};
275 6 : Roce3PostSendParams params{cfg, loc, rmt, static_cast<uint32_t>(ROCE3_OPCODE::ROCE_OPCODE_RDMA_WRITE)};
276 :
277 6 : CHK_RET(FillCtrlSeg(&wqe, params));
278 6 : CHK_RET(FillTaskSeg(&wqe, params));
279 6 : CHK_RET(FillDataSeg(&wqe, params));
280 :
281 6 : CHK_RET(CommitWqe(&wqe, sizeof(Roce3WqeEntry)));
282 18 : HCCL_INFO(
283 : "[Rdma1825Ops::%s] WQE committed, qpn[%u], sqHead[%u], loc[0x%llx], rmt[0x%llx], size[%u]", __func__,
284 : sqContext_->qpn, sqHead_, loc.GetAddr(), rmt.GetAddr(), loc.GetSize());
285 6 : return HCCL_SUCCESS;
286 : }
287 :
288 2 : HcclResult BuildWriteReduceWqe(
289 : const RmaBufSliceLite& loc, const RmtRmaBufSliceLite& rmt, const SqeConfigLite& cfg, DataType dataType,
290 : ReduceOp reduceOp) override
291 : {
292 2 : Roce3WqeEntry wqe{};
293 2 : Roce3PostSendParams params{cfg, loc, rmt, static_cast<uint32_t>(ROCE3_OPCODE::ROCE_OPCODE_RDMA_WRITE)};
294 :
295 2 : CHK_RET(FillCtrlSeg(&wqe, params));
296 2 : CHK_RET(FillTaskSeg(&wqe, params));
297 2 : CHK_RET(FillReduceTaskSeg(&wqe, params, dataType, reduceOp));
298 2 : CHK_RET(FillDataSeg(&wqe, params));
299 :
300 2 : CHK_RET(CommitWqe(&wqe, sizeof(Roce3WqeEntry)));
301 6 : HCCL_INFO(
302 : "[Rdma1825Ops::%s] WQE committed, qpn[%u], sqHead[%u], loc[0x%llx], rmt[0x%llx], size[%u], "
303 : "dataType[%u], reduceOp[%u]",
304 : __func__, sqContext_->qpn, sqHead_, loc.GetAddr(), rmt.GetAddr(), loc.GetSize(),
305 : static_cast<uint32_t>(dataType), static_cast<uint32_t>(reduceOp));
306 2 : return HCCL_SUCCESS;
307 : }
308 :
309 0 : HcclResult WriteInvalidWqebb(uint32_t nextIdx) override
310 : {
311 0 : const uint32_t sqDepth = sqContext_->depth;
312 0 : const uint32_t sqMask = sqDepth - 1U;
313 0 : uint8_t ownerSl = ((nextIdx & sqDepth) == 0) ? 0xff : 0x7f;
314 :
315 0 : auto* dst
316 0 : = reinterpret_cast<void*>(sqContext_->sqVa + static_cast<uint64_t>(nextIdx & sqMask) * sqContext_->wqeSize);
317 :
318 0 : auto ret = memcpy_sp(dst, sizeof(ownerSl), &ownerSl, sizeof(ownerSl));
319 0 : if (UNLIKELY(ret != 0)) {
320 0 : THROW<InternalException>(
321 0 : StringFormat("[Rdma1825Ops::%s] write invalid wqebb failed, ret = %d", __func__, ret));
322 : }
323 0 : return HCCL_SUCCESS;
324 : }
325 :
326 3 : int32_t PollCqImpl(int32_t numEntries, std::vector<int32_t>& errList) override
327 : {
328 3 : int32_t pollNum = 0;
329 3 : CqPollStatus ret = CqPollStatus::ERROR;
330 :
331 4 : while (pollNum < numEntries) {
332 3 : ret = PollOne(errList);
333 3 : if (ret != CqPollStatus::SUCCESS) {
334 2 : if (ret != CqPollStatus::EMPTY && ret != CqPollStatus::ERROR) {
335 0 : HCCL_ERROR(
336 : "[Rdma1825Ops::%s][Poll cq] Poll CQ error, ret: %d", __func__, static_cast<int32_t>(ret));
337 : }
338 2 : break;
339 : }
340 :
341 : // Update Cq CI
342 1 : cqTail_ += 1;
343 :
344 1 : ++pollNum;
345 : }
346 :
347 3 : if ((pollNum != 0) || (ret == CqPollStatus::ERROR)) {
348 : // Need to flush Cq DB
349 2 : cqDbFlush_ = true;
350 : }
351 :
352 3 : return (ret == CqPollStatus::ERROR) ? static_cast<int32_t>(ret) : pollNum;
353 : }
354 :
355 : private:
356 11 : HcclResult FillCtrlSeg(Roce3WqeEntry* wqe, const Roce3PostSendParams& params) const
357 : {
358 : // 赋值本wqe的owner
359 11 : uint8_t owner = (sqHead_ & (sqContext_->depth)) == 0 ? 0 : 1;
360 11 : wqe->ctrl.owner_sl = (owner << ROCE_WQE_OWNERBIT_SHIFT) | ROCE_WQE_CTRL_VALUE;
361 :
362 : // 不产生cqe + task字段的长度
363 11 : wqe->ctrl.df_tsl = ((params.cfg.cqeEn == true) ? (1U << ROCE_SQ_SIGNAL_SHIFT) : 0) | ROCE_SQ_VA_VALUE;
364 11 : wqe->ctrl.df_tsl |= sizeof(Roce3TaskWqeSeg) / ROCE_TASK_SEG_ALIGN;
365 :
366 : // fast_dma + SSN + sge长度
367 11 : wqe->ctrl.wf_bdsl = Htons16(static_cast<uint16_t>(0 << ROCE_WQE_FAST_DMA_SHIFT));
368 11 : wqe->ctrl.wf_bdsl |= Htons16((sqHead_ & ROCE_WQE_SSN_MASK) << ROCE_WQE_SSN_SHIFT);
369 11 : wqe->ctrl.wf_bdsl |= Htons16(
370 : static_cast<uint16_t>(static_cast<uint32_t>(1) << (ROCE_WQE_DATA_SEG_SHIFT - WQE_SECTION_ALIGN_SHIFT)));
371 :
372 : // cl
373 11 : wqe->ctrl.cl_pi = Htonl32(ROCE_WQE_CMP_TASK_LEN1 << ROCE_WQE_CMP_TASK_LEN_SHIFT);
374 :
375 11 : return HCCL_SUCCESS;
376 : }
377 :
378 11 : HcclResult FillTaskSeg(Roce3WqeEntry* wqe, const Roce3PostSendParams& params) const
379 : {
380 : // ----- Task Seg -----
381 11 : wqe->task.tskSeg.value = 0;
382 11 : wqe->task.tskSeg.dw0.signal = !!((wqe->ctrl.df_tsl & (1U << ROCE_WQE_CQE_SIGNAL_SHIFT)) > 0);
383 11 : wqe->task.tskSeg.dw0.fence = params.cfg.fence;
384 11 : wqe->task.tskSeg.dw0.opType = params.opCode;
385 11 : wqe->task.tskSeg.dw0.se = 0;
386 11 : wqe->task.tskSeg.value = Htonl32(wqe->task.tskSeg.value);
387 :
388 11 : wqe->task.dataLen = Htonl32(params.loc.GetSize());
389 11 : wqe->task.immData = 0;
390 : wqe->task.dw3.value
391 11 : = (params.opCode == static_cast<uint32_t>(ROCE3_OPCODE::ROCE_OPCODE_RDMA_READ) ?
392 3 : Htonl32(ROCE_WQE_TASK_SEG_LAST_EXT_LEN) :
393 : 0);
394 11 : wqe->task.va = Htonll64(params.rmt.GetAddr());
395 11 : wqe->task.rkey = Htonl32(params.rmt.GetRkey());
396 11 : wqe->task.ulp = Htonl32(params.loc.GetLkey() & 0xffff);
397 :
398 11 : return HCCL_SUCCESS;
399 : }
400 :
401 : HcclResult
402 2 : FillReduceTaskSeg(Roce3WqeEntry* wqe, const Roce3PostSendParams& params, DataType dataType, ReduceOp reduceOp) const
403 : {
404 : // Hcomm Type -> Roce3 Type
405 2 : uint32_t dataTypeIdx = static_cast<uint32_t>(dataType);
406 2 : if (dataTypeIdx >= sizeof(Roce3ReduceDataTypeMap) / sizeof(uint32_t)) {
407 0 : HCCL_ERROR("[Rdma1825Ops::%s] invalid DataType %u", __func__, dataTypeIdx);
408 0 : return HCCL_E_PARA;
409 : }
410 2 : uint32_t reduceDataType = Roce3ReduceDataTypeMap[dataTypeIdx];
411 2 : if (reduceDataType == DTYPE_INVALID) {
412 0 : HCCL_ERROR("[Rdma1825Ops::%s] DataType %u not support by inline reduce", __func__, dataTypeIdx);
413 0 : return HCCL_E_NOT_SUPPORT;
414 : }
415 :
416 2 : uint32_t reduceOpIdx = static_cast<uint32_t>(reduceOp);
417 2 : if (reduceOpIdx >= sizeof(Roce3ReduceOpTypeMap) / sizeof(uint32_t)) {
418 0 : HCCL_ERROR("[Rdma1825Ops::%s] invalid ReduceOpType %u", __func__, reduceOpIdx);
419 0 : return HCCL_E_PARA;
420 : }
421 2 : uint32_t reduceOpType = Roce3ReduceOpTypeMap[reduceOpIdx];
422 2 : if (reduceOpType == DTYPE_INVALID) {
423 0 : HCCL_ERROR("[Rdma1825Ops::%s] ReduceOpType %u not support by inline reduce", __func__, reduceOpIdx);
424 0 : return HCCL_E_NOT_SUPPORT;
425 : }
426 :
427 : // Reduce data
428 2 : uint32_t mptIndex = params.loc.GetLkey() & 0xffff;
429 2 : uint32_t reduceOpInfo = ((reduceOpType << ROCE_WQE_TASK_REDUCE_OP_OFFSET) | reduceDataType);
430 :
431 : // ----- Task Reduce Seg -----
432 2 : wqe->task.ulp = Htonl32(mptIndex | reduceOpInfo);
433 :
434 2 : return HCCL_SUCCESS;
435 : }
436 :
437 11 : HcclResult FillDataSeg(Roce3WqeEntry* wqe, const Roce3PostSendParams& params) const
438 : {
439 : // ----- Data Seg -----
440 11 : wqe->data.bufAddr = Htonll64(static_cast<uint64_t>(params.loc.GetAddr()));
441 11 : wqe->data.leKey = Htonl32(
442 11 : (params.loc.GetLkey() & (~ROCE_WQE_NEXT_SGE_INVALID)) | ROCE_WQE_NEXT_SGE_INVALID); // 当前sge就是最后一个
443 11 : wqe->data.rLen = Htonl32(params.loc.GetSize());
444 :
445 11 : return HCCL_SUCCESS;
446 : }
447 :
448 0 : CqPollStatus PollOne(std::vector<int32_t>& errList)
449 : {
450 0 : bool need_poll = true;
451 0 : while (need_poll) {
452 0 : Roce3CqeEntry* rcqe = nullptr;
453 0 : rcqe = Roce3GetOneCqe(cqHead_);
454 0 : if (rcqe == nullptr) {
455 0 : return CqPollStatus::EMPTY;
456 : }
457 :
458 0 : ++cqHead_;
459 :
460 0 : uint32_t syndrome = static_cast<uint32_t>(rcqe->syndrome);
461 0 : uint32_t ownerIdQpn = static_cast<uint32_t>(rcqe->owner_id_qpn);
462 0 : auto cqeType
463 0 : = static_cast<RoceCqeType>((rcqe->op_sr_wqebb >> ROCE_CQE_OPCODE_SHIFT) & ROCE_CQE_OPCODE_MASK);
464 : // 存在错误
465 0 : if (cqeType == RoceCqeType::ERROR) {
466 0 : const uint32_t cqeErrCode = (ownerIdQpn >> 20U) & 0xfU;
467 :
468 0 : HCCL_ERROR(
469 : "[Rdma1825Ops::%s][Poll cq] CQE error, qpn[%u], syndrome[%u], cqeErrCode[%u], "
470 : "wqeCounter[%u]",
471 : __func__, ownerIdQpn & ROCE_CQE_QPN_MASK, syndrome, cqeErrCode,
472 : static_cast<uint32_t>(rcqe->wqe_counter));
473 :
474 : // 后续优化为具体ErrorCode
475 0 : int32_t ERROR_CODE = 1;
476 0 : errList.push_back(ERROR_CODE);
477 0 : return CqPollStatus::ERROR;
478 : }
479 :
480 0 : HCCL_INFO(
481 : "[Rdma1825Ops::%s][Poll cq] CQE completed, cqHead[%u], qpn[%u], syndrome[%u], "
482 : "wqeCounter[%u]",
483 : __func__, cqHead_, ownerIdQpn & ROCE_CQE_QPN_MASK, syndrome, static_cast<uint32_t>(rcqe->wqe_counter));
484 0 : need_poll = false;
485 : }
486 :
487 0 : return CqPollStatus::SUCCESS;
488 : }
489 :
490 0 : Roce3CqeEntry* Roce3GetOneCqe(uint32_t consIndex)
491 : {
492 0 : uint32_t cqeSize = 64;
493 0 : uint32_t cqDepthReal = cqContext_->cqDepth + ROCE_CQE_MAX_GEN_NUM; // Must be power of 2
494 0 : uint32_t cqeMask = cqDepthReal - 1;
495 0 : uint32_t cqeSlot = consIndex & cqeMask;
496 :
497 : // Calculate Cqe Addr
498 0 : auto cqeAddr = reinterpret_cast<Roce3CqeEntry*>(
499 0 : reinterpret_cast<uint8_t*>(cqContext_->cqVa) + static_cast<size_t>(cqeSlot * cqeSize));
500 :
501 : // Memcpy Cqe
502 0 : auto ret = memcpy_sp(&cqeReadback_, sizeof(cqeReadback_), cqeAddr, sizeof(cqeReadback_));
503 0 : if (UNLIKELY(ret != 0)) {
504 0 : THROW<InternalException>(StringFormat("[Rdma1825Ops::%s] memcpy_sp cqe failed, ret = %d", __func__, ret));
505 : }
506 :
507 : // Cqe Details Parse
508 0 : uint32_t opSrWqebb = cqeReadback_.op_sr_wqebb;
509 0 : uint32_t ownerIdQpn = cqeReadback_.owner_id_qpn;
510 0 : auto cqeType = static_cast<RoceCqeType>((opSrWqebb >> ROCE_CQE_OPCODE_SHIFT) & ROCE_CQE_OPCODE_MASK);
511 0 : if (cqeType == RoceCqeType::INVALID) {
512 0 : HCCL_INFO("[Rdma1825Ops::%s][Poll cq] CQE invalid, consIndex[%u], slot[%u]", __func__, consIndex, cqeSlot);
513 0 : return nullptr;
514 : }
515 :
516 : // Judge Owner bit
517 0 : uint32_t calHwOwner = static_cast<uint32_t>((consIndex & cqDepthReal) == 0);
518 0 : uint32_t curCqeOwner = !((ownerIdQpn & (1U << ROCE_CQE_OWNERBIT_SHIFT)) == 0);
519 0 : if ((calHwOwner ^ curCqeOwner) == 0) {
520 0 : HCCL_INFO(
521 : "[Rdma1825Ops::%s][Poll cq] CQE owner not ready, consIndex[%u], slot[%u], "
522 : "calHwOwner[%u], curCqeOwner[%u]",
523 : __func__, consIndex, cqeSlot, calHwOwner, curCqeOwner);
524 0 : return nullptr;
525 : }
526 :
527 0 : return &cqeReadback_;
528 : }
529 :
530 : Roce3CqeEntry cqeReadback_{};
531 : };
532 :
533 : } // namespace Hccl
534 : #endif // RDMA_VENDOR_1825_OPS_H
|
