Line data Source code
1 : /**
2 : * Copyright (c) 2026 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #ifndef RDMA_BASE_VENDOR_OPS_H
12 : #define RDMA_BASE_VENDOR_OPS_H
13 :
14 : #include <chrono>
15 : #include <cstddef>
16 : #include <cstdint>
17 : #include <vector>
18 : #include <securec.h>
19 :
20 : #include "exception_util.h"
21 : #include "internal_exception.h"
22 : #include "log.h"
23 : #include "rma_buf_slice_lite.h" // RmaBufSliceLite
24 : #include "rmt_rma_buf_slice_lite.h" // RmtRmaBufSliceLite
25 : #include "data_type.h" // DataType
26 : #include "reduce_op.h" // ReduceOp
27 :
28 : namespace Hccl {
29 :
30 : struct SqeConfigLite;
31 :
32 : struct RdmaSqContextLite {
33 : uint32_t qpn;
34 : uint64_t sqVa;
35 : uint32_t wqeSize;
36 : uint32_t depth;
37 : uint64_t headAddr;
38 : uint64_t tailAddr;
39 : uint64_t dbHwVa;
40 : uint64_t dbSwVa;
41 : uint8_t sl;
42 : uint64_t dbVendorSpecified;
43 : };
44 :
45 : struct RdmaCqContextLite {
46 : uint32_t cqn;
47 : uint64_t cqVa;
48 : uint32_t cqeSize;
49 : uint32_t cqDepth;
50 : uint64_t headAddr;
51 : uint64_t tailAddr;
52 : uint64_t dbHwVa;
53 : uint64_t dbSwVa;
54 : uint64_t dbVendorSpecified;
55 : };
56 :
57 : // DbVendorSpecified bitfield layout, matching Roce3DbEntry.dw0.bs:
58 : // cos at bits 24-26 (3 bits)
59 : // mtuShift at bits 50-52 (3 bits)
60 : static constexpr uint32_t UB_DB_VENDOR_COS_SHIFT = 24;
61 : static constexpr uint32_t UB_DB_VENDOR_MTUSHIFT_SHIFT = 50;
62 : static constexpr uint32_t UB_DB_VENDOR_FIELD_MASK = 0x7;
63 :
64 : // Necessary helper funcs
65 : constexpr uint32_t BITS_1BYTE = 8;
66 : constexpr uint32_t BITS_3BYTE = 24;
67 : constexpr uint32_t BITS_5BYTE = 40;
68 : constexpr uint32_t BITS_7BYTE = 56;
69 :
70 33 : inline uint16_t Htons16(uint16_t x) { return (((x & 0xffULL) << BITS_1BYTE) | ((x & 0xff00ULL) >> BITS_1BYTE)); }
71 :
72 86 : inline uint32_t Htonl32(uint32_t x)
73 : {
74 86 : return ((x & 0x000000ffU) << BITS_3BYTE) | ((x & 0x0000ff00U) << BITS_1BYTE) | ((x & 0x00ff0000U) >> BITS_1BYTE)
75 86 : | ((x & 0xff000000U) >> BITS_3BYTE);
76 : }
77 :
78 22 : inline uint64_t Htonll64(uint64_t x)
79 : {
80 22 : return ((x & 0x00000000000000ffULL) << BITS_7BYTE) | ((x & 0x000000000000ff00ULL) << BITS_5BYTE)
81 22 : | ((x & 0x0000000000ff0000ULL) << BITS_3BYTE) | ((x & 0x00000000ff000000ULL) << BITS_1BYTE)
82 22 : | ((x & 0x000000ff00000000ULL) >> BITS_1BYTE) | ((x & 0x0000ff0000000000ULL) >> BITS_3BYTE)
83 22 : | ((x & 0x00ff000000000000ULL) >> BITS_5BYTE) | ((x & 0xff00000000000000ULL) >> BITS_7BYTE);
84 : }
85 :
86 : enum class CqPollStatus : int32_t {
87 : SUCCESS = 0, /* indicate poll once cqe successfully; */
88 : EMPTY = -1, /* indicate the cq is empty when poll cq; */
89 : ERROR = -2, /* indicate the error when poll cq; */
90 : REROLL = -3, /* 返回到repoll标识 */
91 : CONTINUE = 1, /* indicate continue the process */
92 : };
93 :
94 : class RdmaBaseOps {
95 : public:
96 39 : RdmaBaseOps(RdmaSqContextLite* sqContext, RdmaCqContextLite* cqContext)
97 39 : : sqContext_(sqContext),
98 39 : cqContext_(cqContext)
99 39 : {}
100 39 : virtual ~RdmaBaseOps() = default;
101 :
102 : // 上层接口，不关心具体vendor类型
103 3 : HcclResult Read(const RmaBufSliceLite& loc, const RmtRmaBufSliceLite& rmt, const SqeConfigLite& cfg)
104 : {
105 : // Read需要占用1个wr位置, 确定Sq存在空位
106 3 : constexpr int ReadWqeCount = 1;
107 3 : CHK_RET(WaitSqFree(ReadWqeCount));
108 :
109 : // 进入vendor特有wqe组装接口, 组装完wqe直接下发, opCode不支持直接返回HCCL_E_NOT_SUPPORT
110 3 : CHK_RET(BuildReadWqe(loc, rmt, cfg));
111 :
112 : // 更新Sq队列PI值
113 3 : CHK_RET(UpdateSqPI());
114 :
115 3 : return HCCL_SUCCESS;
116 : }
117 :
118 6 : HcclResult Write(const RmaBufSliceLite& loc, const RmtRmaBufSliceLite& rmt, const SqeConfigLite& cfg)
119 : {
120 : // Write需要占用1个wr位置, 确定Sq存在空位
121 6 : constexpr int WriteWqeCount = 1;
122 6 : CHK_RET(WaitSqFree(WriteWqeCount));
123 :
124 : // 进入vendor特有wqe组装接口, 组装完wqe直接下发
125 6 : CHK_RET(BuildWriteWqe(loc, rmt, cfg));
126 :
127 : // 更新Sq队列PI值
128 6 : CHK_RET(UpdateSqPI());
129 :
130 6 : return HCCL_SUCCESS;
131 : }
132 :
133 2 : HcclResult WriteReduce(
134 : const RmaBufSliceLite& loc, const RmtRmaBufSliceLite& rmt, const SqeConfigLite& cfg, DataType dataType,
135 : ReduceOp reduceOp)
136 : {
137 : // Inline Reduce Write需要占用1个wr位置, 确定Sq存在空位
138 2 : constexpr int WriteReduceWqeCount = 1;
139 2 : CHK_RET(WaitSqFree(WriteReduceWqeCount));
140 :
141 : // 进入vendor特有wqe组装接口, 组装完wqe直接下发
142 2 : CHK_RET(BuildWriteReduceWqe(loc, rmt, cfg, dataType, reduceOp));
143 :
144 : // 更新Sq队列PI值
145 2 : CHK_RET(UpdateSqPI());
146 :
147 2 : return HCCL_SUCCESS;
148 : }
149 :
150 3 : HcclResult PollCq(int32_t numEntries, int32_t timeOut, std::vector<int32_t>& errList)
151 : {
152 3 : auto timeLimit = std::chrono::milliseconds(timeOut);
153 3 : auto startTime = std::chrono::steady_clock::now();
154 :
155 3 : int32_t totalPollNum = 0;
156 3 : int32_t ret = 0;
157 :
158 4 : while (totalPollNum < numEntries) {
159 : // 逐一Poll Cq，并处理cqe
160 3 : ret = PollCqImpl(numEntries - totalPollNum, errList);
161 3 : if (ret == static_cast<int32_t>(CqPollStatus::ERROR)) {
162 3 : HCCL_ERROR("[RdmaBaseOps::%s][Poll cq] Poll Cq Error.", __func__);
163 1 : return HCCL_E_REMOTE;
164 : }
165 :
166 2 : if (ret > 0) {
167 : // Update cqe in this loop
168 1 : totalPollNum += ret;
169 :
170 : // Update Sq Tail
171 1 : sqTail_ += ret;
172 :
173 : // continue poll cqe
174 1 : continue;
175 : }
176 :
177 1 : if ((std::chrono::steady_clock::now() - startTime) > timeLimit) {
178 3 : HCCL_ERROR(
179 : "[RdmaBaseOps::%s][Poll cq] Poll Cq timeout, expected[%d], actual[%d], lastRet[%d]", __func__,
180 : numEntries, totalPollNum, ret);
181 1 : return HCCL_E_TIMEOUT;
182 : }
183 : }
184 :
185 3 : HCCL_INFO(
186 : "[RdmaBaseOps::%s][Poll cq] Poll Cq success, expected[%d], actual[%d]", __func__, numEntries, totalPollNum);
187 1 : return HCCL_SUCCESS;
188 : }
189 :
190 : // 准备Doorbell(厂商实现)
191 : virtual HcclResult BuildDoorbell(u64& dbAddr, u64& dbValue) = 0;
192 :
193 : // 准备CqDoorbell(厂商实现)
194 : virtual HcclResult BuildCqDoorbell(u64& dbAddr, u64& dbValue) = 0;
195 :
196 : protected:
197 : // 软件侧只维护Sq PI，Sq CI由硬件维护
198 : u32 sqHead_{0};
199 : u32 sqTail_{0};
200 :
201 : u32 cqHead_{0};
202 : u32 cqTail_{0};
203 : bool cqDbFlush_ = false;
204 :
205 : RdmaSqContextLite* sqContext_;
206 : RdmaCqContextLite* cqContext_;
207 :
208 : // 默认超时时间 30 ms
209 : const std::chrono::milliseconds timeout_ = std::chrono::milliseconds(30U);
210 :
211 : // vendor扩展点: 每个原子op一个虚函数
212 : // 默认 NOT_SUPPORT, 各个vendor 只重写自己支持的
213 0 : virtual HcclResult BuildReadWqe(const RmaBufSliceLite& loc, const RmtRmaBufSliceLite& rmt, const SqeConfigLite& cfg)
214 : {
215 : (void)loc;
216 : (void)rmt;
217 : (void)cfg;
218 0 : HCCL_ERROR("[RdmaBaseOps::%s] This Backend Not support Read Now.", __func__);
219 0 : return HCCL_E_NOT_SUPPORT;
220 : }
221 :
222 : virtual HcclResult
223 0 : BuildWriteWqe(const RmaBufSliceLite& loc, const RmtRmaBufSliceLite& rmt, const SqeConfigLite& cfg)
224 : {
225 : (void)loc;
226 : (void)rmt;
227 : (void)cfg;
228 0 : HCCL_ERROR("[RdmaBaseOps::%s] This Backend Not support Write Now.", __func__);
229 0 : return HCCL_E_NOT_SUPPORT;
230 : }
231 :
232 0 : virtual HcclResult BuildWriteReduceWqe(
233 : const RmaBufSliceLite& locNotify, const RmtRmaBufSliceLite& notify, const SqeConfigLite& cfg, DataType dataType,
234 : ReduceOp reduceOp)
235 : {
236 : (void)locNotify;
237 : (void)notify;
238 : (void)cfg;
239 : (void)dataType;
240 : (void)reduceOp;
241 0 : HCCL_ERROR("[RdmaBaseOps::%s] This Backend Not support WriteReduce Now.", __func__);
242 0 : return HCCL_E_NOT_SUPPORT;
243 : }
244 :
245 0 : virtual HcclResult WriteInvalidWqebb(uint32_t nextIdx)
246 : {
247 : (void)nextIdx;
248 0 : return HCCL_SUCCESS;
249 : }
250 :
251 0 : virtual int32_t PollCqImpl(int32_t numEntries, [[maybe_unused]] std::vector<int32_t>& errList)
252 : {
253 : (void)numEntries;
254 0 : HCCL_ERROR("[RdmaBaseOps::%s] This Backend Not support PollCq Now.", __func__);
255 0 : return HCCL_E_NOT_SUPPORT;
256 : }
257 :
258 : // 搬运wqe(通用实现), 把 Wqe 写到 SQ
259 0 : HcclResult CommitWqe(const void* wqe, uint32_t wqeSize)
260 : {
261 0 : HCCL_INFO("[RdmaBaseOps::%s] Memcpy wqe start, Now SQ PI: [%u]", __func__, sqHead_);
262 :
263 : // 写wqe到va
264 0 : auto sqDepth = sqContext_->depth;
265 0 : uint32_t sqPIMask = sqDepth - 1;
266 0 : u8* va = reinterpret_cast<u8*>(sqContext_->sqVa + (sqHead_ & sqPIMask) * wqeSize);
267 :
268 0 : HCCL_INFO(
269 : "[RdmaBaseOps][Wqe Write] before copy, sqHead[%u], slot[%u], sqVa[0x%llx], dst[0x%llx], size[%u]", sqHead_,
270 : sqHead_ & sqPIMask, static_cast<unsigned long long>(sqContext_->sqVa),
271 : reinterpret_cast<unsigned long long>(va), wqeSize);
272 :
273 0 : auto ret = memcpy_sp(va, wqeSize, wqe, wqeSize);
274 0 : if (UNLIKELY(ret != 0)) {
275 0 : THROW<InternalException>(StringFormat("[RdmaBaseOps::%s] memcpy_s failed, ret = %d", __func__, ret));
276 : }
277 :
278 : // pi维护用于传入DB Send用于Rtsq 敲door bell
279 0 : sqHead_ = sqHead_ + 1;
280 :
281 : // Write InValid Wqebb
282 0 : CHK_RET(WriteInvalidWqebb(sqHead_));
283 :
284 0 : HCCL_INFO("[RdmaBaseOps::%s] Memcpy wqe end, Now SQ PI: [%u]", __func__, sqHead_);
285 0 : return HCCL_SUCCESS;
286 : }
287 :
288 0 : HcclResult WaitSqFree(uint32_t wqeNum)
289 : {
290 : // wq_overflow
291 0 : bool timeOutFlag = false;
292 0 : auto startTime = std::chrono::steady_clock::now();
293 :
294 0 : HCCL_INFO("[RdmaBaseOps::%s] Operate: sqTail = %u", __func__, sqTail_);
295 0 : while (!timeOutFlag) {
296 : // sq 队列能放下，直接成功返回
297 0 : if (static_cast<uint32_t>(sqHead_ - sqTail_ + wqeNum) <= sqContext_->depth) {
298 0 : return HCCL_SUCCESS;
299 : }
300 :
301 0 : timeOutFlag = (std::chrono::steady_clock::now() - startTime) > timeout_;
302 : }
303 :
304 : // 超时处理
305 0 : HCCL_ERROR("[RdmaBaseOps::%s] Sq is Full !! Operate: sqTail = %u Failed. ", __func__, sqTail_);
306 0 : return HCCL_E_TIMEOUT;
307 : }
308 :
309 : // 将PI更新到硬件可见地址
310 0 : HcclResult UpdateSqPI()
311 : {
312 : // 更新Sq PI指针
313 0 : uint32_t sqHeadNum = Htonl32(sqHead_);
314 :
315 0 : HCCL_INFO(
316 : "[RdmaBaseOps][Wqe Write] write soft PI, sqHead host[%u], dbSwVa[0x%llx]", sqHead_,
317 : static_cast<unsigned long long>(sqContext_->dbSwVa));
318 :
319 : auto status
320 0 : = memcpy_sp(reinterpret_cast<void*>(sqContext_->dbSwVa), sizeof(uint32_t), &sqHeadNum, sizeof(uint32_t));
321 0 : if (UNLIKELY(status != 0)) {
322 0 : THROW<InternalException>(StringFormat("[RdmaBaseOps::%s] Ring Sw DB failed, ret = %d", __func__, status));
323 : }
324 0 : return HCCL_SUCCESS;
325 : }
326 : };
327 :
328 : } // namespace Hccl
329 : #endif // RDMA_BASE_VENDOR_OPS_H
|
