Line data Source code
1 : /**
2 : * Copyright (c) 2026 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "./dev_rdma_connection_v2.h"
12 : #include "log.h"
13 : #include "acl/acl_rt.h"
14 : #include "hccp.h"
15 :
16 : namespace hcomm {
17 : static constexpr uint32_t DEFAULT_CQN = 0;
18 : // DbVendorSpecified bitfield layout, matching Roce3DbEntry.dw0.bs:
19 : // cos at bits 24-26 (3 bits)
20 : // mtuShift at bits 50-52 (3 bits)
21 : static constexpr uint32_t UB_DB_VENDOR_COS_SHIFT = 24;
22 : static constexpr uint32_t UB_DB_VENDOR_MTUSHIFT_SHIFT = 50;
23 : static constexpr uint32_t UB_DB_VENDOR_FIELD_MASK = 0x7;
24 :
25 44 : DevRdmaConnectionV2::DevRdmaConnectionV2(Hccl::Socket* socket, RdmaHandle rdmaHandle, uint32_t cqAttrFlags)
26 44 : : socket_(socket),
27 44 : rdmaHandle_(rdmaHandle),
28 44 : cqAttrFlags_(cqAttrFlags)
29 44 : {}
30 :
31 44 : HcclResult DevRdmaConnectionV2::Init()
32 : {
33 44 : if (rdmaConnStatus_ != RdmaConnStatus::CLOSED) {
34 0 : HCCL_INFO(
35 : "[DevRdmaConnectionV2][%s] status[%s] is not need init.", __func__, rdmaConnStatus_.Describe().c_str());
36 0 : return HCCL_SUCCESS;
37 : }
38 :
39 44 : GetNdaOps();
40 44 : CHK_RET(GetDirectFlag());
41 44 : CHK_RET(GetDmaMode());
42 :
43 44 : rdmaConnStatus_ = RdmaConnStatus::INIT;
44 44 : return HCCL_SUCCESS;
45 : }
46 :
47 48 : DevRdmaConnectionV2::~DevRdmaConnectionV2()
48 : {
49 44 : if (rdmaConnStatus_ == RdmaConnStatus::CLOSED || rdmaConnStatus_ == RdmaConnStatus::INIT) {
50 40 : return;
51 : }
52 4 : HcclResult ret = DestroyQp();
53 4 : if (ret != HCCL_SUCCESS) {
54 0 : HCCL_ERROR("[%s]call DestroyQp failed: hcclRet -> %d", __func__, ret);
55 : }
56 164 : }
57 :
58 5 : std::string DevRdmaConnectionV2::Describe() const
59 : {
60 5 : return Hccl::StringFormat("DevRdmaConnectionV2[status=%s]", rdmaConnStatus_.Describe().c_str());
61 : }
62 :
63 2 : static void* NdaAlloc(size_t size)
64 : {
65 2 : void* ptr = nullptr;
66 2 : aclError ret = aclrtMalloc(&ptr, size, static_cast<aclrtMemMallocPolicy>(ACL_MEM_TYPE_HIGH_BAND_WIDTH));
67 2 : if (ret != ACL_SUCCESS) {
68 1 : HCCL_ERROR("[NdaAlloc] aclrtMalloc failed, ret[%d], size[%zu]", ret, size);
69 1 : return nullptr;
70 : }
71 1 : return ptr;
72 : }
73 :
74 3 : static void NdaFree(void* ptr)
75 : {
76 3 : if (ptr == nullptr) {
77 1 : return;
78 : }
79 2 : aclError ret = aclrtFree(ptr);
80 2 : if (ret != ACL_SUCCESS) {
81 1 : HCCL_ERROR("[NdaFree] aclrtFree failed, ret[%d], ptr[%p]", ret, ptr);
82 : }
83 : }
84 :
85 2 : static void NdaMemset(void* dst, int value, size_t count)
86 : {
87 2 : aclError ret = aclrtMemset(dst, count, value, count);
88 2 : if (ret != ACL_SUCCESS) {
89 1 : HCCL_ERROR("[NdaMemset] aclrtMemset failed, ret[%d], dst[%p], value[%d], count[%zu]", ret, dst, value, count);
90 : }
91 2 : }
92 :
93 6 : static int NdaMemcpy(void* dst, size_t dstSize, void* src, size_t srcSize, uint32_t direct)
94 : {
95 6 : aclrtMemcpyKind kind = ACL_MEMCPY_DEFAULT;
96 6 : switch (direct) {
97 1 : case MEMCPY_DIRECT_HOST_TO_HOST: {
98 1 : kind = ACL_MEMCPY_HOST_TO_HOST;
99 1 : break;
100 : }
101 2 : case MEMCPY_DIRECT_HOST_TO_DEVICE: {
102 2 : kind = ACL_MEMCPY_HOST_TO_DEVICE;
103 2 : break;
104 : }
105 1 : case MEMCPY_DIRECT_DEVICE_TO_HOST: {
106 1 : kind = ACL_MEMCPY_DEVICE_TO_HOST;
107 1 : break;
108 : }
109 1 : case MEMCPY_DIRECT_DEVICE_TO_DEVICE: {
110 1 : kind = ACL_MEMCPY_DEVICE_TO_DEVICE;
111 1 : break;
112 : }
113 1 : default: {
114 1 : HCCL_ERROR("[MemcpyKindTranslate]Not support the memory copy type[%d].", direct);
115 1 : return -1;
116 : }
117 : }
118 5 : aclError ret = aclrtMemcpy(dst, dstSize, src, srcSize, kind);
119 5 : if (ret != ACL_SUCCESS) {
120 1 : HCCL_ERROR(
121 : "[NdaMemcpy] aclrtMemcpy failed, ret[%d], dst[%p], src[%p], dstSize[%zu], srcSize[%zu]", ret, dst, src,
122 : dstSize, srcSize);
123 1 : return -1;
124 : }
125 4 : return 0;
126 : }
127 :
128 44 : void DevRdmaConnectionV2::GetNdaOps()
129 : {
130 44 : ndaOps_ = {.alloc = NdaAlloc, .free = NdaFree, .memset_s = NdaMemset, .memcpy_s = NdaMemcpy};
131 44 : }
132 :
133 44 : HcclResult DevRdmaConnectionV2::GetDirectFlag()
134 : {
135 44 : s32 ret = RaNdaGetDirectFlag(rdmaHandle_, &directFlag_);
136 44 : if (ret != 0) {
137 0 : HCCL_ERROR(
138 : "[DevRdmaConnectionV2][GetDirectFlag]errNo[0x%016llx] get directFlag fail. "
139 : "return[%d], params: rdmaHandle[%p], directFlag[%d]",
140 : HCCL_ERROR_CODE(HCCL_E_INTERNAL), ret, rdmaHandle_, directFlag_);
141 0 : return HCCL_E_INTERNAL;
142 : }
143 44 : return HCCL_SUCCESS;
144 : }
145 :
146 44 : HcclResult DevRdmaConnectionV2::GetDmaMode()
147 : {
148 44 : switch (directFlag_) {
149 44 : case DIRECT_FLAG_PCIE: {
150 44 : dmaMode_ = QBUF_DMA_MODE_DEFAULT;
151 44 : break;
152 : }
153 0 : case DIRECT_FLAG_UB: {
154 0 : dmaMode_ = QBUF_DMA_MODE_INDEP_UB;
155 0 : break;
156 : }
157 0 : default: {
158 0 : HCCL_ERROR("[GetDmaMode]Not support the directFlag [%d].", directFlag_);
159 0 : dmaMode_ = QBUF_DMA_MODE_MAX;
160 0 : return HCCL_E_INTERNAL;
161 : }
162 : }
163 44 : HCCL_INFO("[GetDmaMode] directFlag[%d], dmaMode[%d]", directFlag_, dmaMode_);
164 44 : return HCCL_SUCCESS;
165 : }
166 :
167 4 : HcclResult DevRdmaConnectionV2::CreateQp()
168 : {
169 4 : if (socket_->GetStatus() != Hccl::SocketStatus::OK) {
170 0 : HCCL_WARNING("[DevRdmaConnectionV2::CreateQp] socket status is not ok, please");
171 0 : return HCCL_E_AGAIN;
172 : }
173 :
174 : // cqAttrFlags不为默认值0时，仅QBUF_DMA_MODE_DEFAULT（云脉网卡）支持配置，NPU网卡（INDEP_UB）不支持
175 4 : uint32_t cqAttrFlags = 0;
176 4 : if (cqAttrFlags_ != 0) {
177 0 : if (dmaMode_ == QBUF_DMA_MODE_INDEP_UB) {
178 0 : HCCL_ERROR(
179 : "[DevRdmaConnectionV2::CreateQp] NPU NIC does not support configuring cqAttrFlags[%u].", cqAttrFlags_);
180 0 : return HCCL_E_NOT_SUPPORT;
181 : }
182 0 : cqAttrFlags = cqAttrFlags_;
183 0 : HCCL_WARNING("[DevRdmaConnectionV2::CreateQp] CqAttrFlags is configured to [%u].", cqAttrFlags);
184 : }
185 :
186 4 : CHK_RET(Hccl::HrtRaNdaCqCreate(rdmaHandle_, &ndaOps_, dmaMode_, cqAttrFlags, &ndaCqInfo_, &cqHandle_));
187 :
188 4 : CHK_RET(Hccl::HrtRaNdaQpCreate(rdmaHandle_, &ndaOps_, dmaMode_, &ndaCqInfo_, &ndaQpInfo_, &qpHandle_));
189 :
190 4 : SqPiMem_ = hccl::DeviceMem::alloc(sizeof(void*));
191 4 : SqCiMem_ = hccl::DeviceMem::alloc(sizeof(void*));
192 4 : CqPiMem_ = hccl::DeviceMem::alloc(sizeof(void*));
193 4 : CqCiMem_ = hccl::DeviceMem::alloc(sizeof(void*));
194 4 : CHK_PRT_RET(
195 : !SqPiMem_ || !SqCiMem_ || !CqPiMem_ || !CqCiMem_,
196 : HCCL_ERROR(
197 : "%s DeviceMem::alloc for SqPi_ or SqCi_ or CqPi_ or CqCi_ failed, size=%zu", __func__, sizeof(void*)),
198 : HCCL_E_MEMORY);
199 :
200 4 : rdmaConnStatus_ = RdmaConnStatus::QP_CREATED;
201 4 : return HCCL_SUCCESS;
202 : }
203 :
204 4 : HcclResult DevRdmaConnectionV2::DestroyQp()
205 : {
206 4 : if (rdmaConnStatus_ == RdmaConnStatus::CLOSED || rdmaConnStatus_ == RdmaConnStatus::INIT) {
207 0 : return HCCL_SUCCESS;
208 : }
209 :
210 4 : HcclResult ret = HCCL_SUCCESS;
211 :
212 4 : if (qpHandle_ != nullptr) {
213 4 : Hccl::HrtRaQpDestroy(qpHandle_);
214 4 : qpHandle_ = nullptr;
215 : }
216 :
217 4 : if (cqHandle_ != nullptr) {
218 4 : ret = Hccl::HrtRaNdaCqDestroy(rdmaHandle_, cqHandle_);
219 4 : if (ret != HCCL_SUCCESS) {
220 0 : HCCL_ERROR("[DevRdmaConnectionV2::%s] HrtRaNdaCqDestroy failed, ret[%d]", __func__, ret);
221 : }
222 4 : cqHandle_ = nullptr;
223 : }
224 :
225 4 : SqPiMem_ = hccl::DeviceMem();
226 4 : SqCiMem_ = hccl::DeviceMem();
227 4 : CqPiMem_ = hccl::DeviceMem();
228 4 : CqCiMem_ = hccl::DeviceMem();
229 :
230 4 : rdmaConnStatus_ = RdmaConnStatus::CLOSED;
231 4 : return ret;
232 : }
233 :
234 4 : HcclResult DevRdmaConnectionV2::GetExchangeDto(std::unique_ptr<Hccl::Serializable>& locQpAttrserial)
235 : {
236 4 : if (rdmaConnStatus_ != RdmaConnStatus::QP_CREATED && rdmaConnStatus_ != RdmaConnStatus::QP_MODIFIED) {
237 0 : HCCL_ERROR(
238 : "[DevRdmaConnectionV2][%s] status[%s] is not expected.", __func__, rdmaConnStatus_.Describe().c_str());
239 0 : return HCCL_E_AGAIN;
240 : }
241 :
242 : struct QpAttr localQpAttr;
243 4 : s32 ret = RaGetQpAttr(qpHandle_, &localQpAttr);
244 4 : if (ret != 0) {
245 0 : HCCL_ERROR("[DevRdmaConnectionV2::%s]RaGetQpAttr failed, ret[%d]", __func__, ret);
246 0 : return HCCL_E_ROCE_CONNECT;
247 : }
248 4 : std::unique_ptr<ExchangeRdmaConnDto> dto = nullptr;
249 4 : EXCEPTION_CATCH(
250 : dto = std::make_unique<ExchangeRdmaConnDto>(localQpAttr.qpn, localQpAttr.psn, localQpAttr.gidIdx),
251 : return HCCL_E_PTR);
252 4 : CHK_SAFETY_FUNC_RET(memcpy_s(dto->gid_, HCCP_GID_RAW_LEN, localQpAttr.gid, HCCP_GID_RAW_LEN));
253 4 : locQpAttrserial = std::unique_ptr<Hccl::Serializable>(std::move(dto));
254 4 : return HCCL_SUCCESS;
255 4 : }
256 :
257 5 : HcclResult DevRdmaConnectionV2::ParseRmtExchangeDto(const Hccl::Serializable& rmtDto)
258 : {
259 5 : auto dto = dynamic_cast<const ExchangeRdmaConnDto&>(rmtDto);
260 5 : HCCL_INFO("[DevRdmaConnectionV2][%s] remoteConnDto[%s]", __func__, dto.Describe().c_str());
261 5 : rmtQpAttr_.psn = dto.psn_;
262 5 : rmtQpAttr_.qpn = dto.qpn_;
263 5 : rmtQpAttr_.gid_idx = dto.gid_idx_;
264 5 : CHK_SAFETY_FUNC_RET(memcpy_s(rmtQpAttr_.gid, HCCP_GID_RAW_LEN, dto.gid_, HCCP_GID_RAW_LEN));
265 5 : return HCCL_SUCCESS;
266 5 : }
267 :
268 1 : HcclResult DevRdmaConnectionV2::ModifyQp()
269 : {
270 1 : if (rdmaConnStatus_ == RdmaConnStatus::QP_MODIFIED) {
271 0 : HCCL_WARNING(
272 : "[DevRdmaConnectionV2][%s] modify qp already, status[%s].", __func__, rdmaConnStatus_.Describe().c_str());
273 0 : return HCCL_SUCCESS;
274 : }
275 1 : if (rdmaConnStatus_ != RdmaConnStatus::QP_CREATED) {
276 0 : HCCL_ERROR(
277 : "[DevRdmaConnectionV2][%s] status[%s] is not expected.", __func__, rdmaConnStatus_.Describe().c_str());
278 0 : return HCCL_E_AGAIN;
279 : }
280 :
281 1 : if (!rmtQpAttr_.IsValid()) {
282 0 : HCCL_ERROR("[DevRdmaConnectionV2][%s] romate Qp Attr is empty, exchange qp attr first", __func__);
283 0 : return HCCL_E_INTERNAL;
284 : }
285 :
286 : struct QpAttr localQpAttr;
287 1 : s32 ret = RaGetQpAttr(qpHandle_, &localQpAttr);
288 1 : if (ret != 0) {
289 0 : HCCL_ERROR("[DevRdmaConnectionV2::%s]RaGetQpAttr failed, ret[%d]", __func__, ret);
290 0 : return HCCL_E_ROCE_CONNECT;
291 : }
292 :
293 : struct TypicalQp localQp;
294 : struct TypicalQp rmtQp;
295 1 : localQp.sl = qpInfo_.serviceLevel;
296 1 : localQp.tc = qpInfo_.trafficClass;
297 1 : localQp.retryCnt = qpInfo_.retryCnt;
298 1 : localQp.retryTime = qpInfo_.retryInterval;
299 1 : localQp.qpn = localQpAttr.qpn;
300 1 : localQp.psn = localQpAttr.psn;
301 1 : localQp.gidIdx = localQpAttr.gidIdx;
302 1 : (void)memcpy_s(localQp.gid, HCCP_GID_RAW_LEN, localQpAttr.gid, HCCP_GID_RAW_LEN);
303 1 : rmtQp.sl = qpInfo_.serviceLevel;
304 1 : rmtQp.tc = qpInfo_.trafficClass;
305 1 : rmtQp.retryCnt = qpInfo_.retryCnt;
306 1 : rmtQp.retryTime = qpInfo_.retryInterval;
307 1 : rmtQp.qpn = rmtQpAttr_.qpn;
308 1 : rmtQp.psn = rmtQpAttr_.psn;
309 1 : rmtQp.gidIdx = rmtQpAttr_.gid_idx;
310 1 : (void)memcpy_s(rmtQp.gid, HCCP_GID_RAW_LEN, rmtQpAttr_.gid, HCCP_GID_RAW_LEN);
311 1 : ret = RaTypicalQpModify(qpHandle_, &localQp, &rmtQp);
312 1 : if (ret != 0) {
313 0 : HCCL_ERROR("[modify][ra_qp]modify qp failed, ret(%d)", ret);
314 0 : return HCCL_E_ROCE_CONNECT;
315 : }
316 1 : rdmaConnStatus_ = RdmaConnStatus::QP_MODIFIED;
317 1 : return HCCL_SUCCESS;
318 : }
319 :
320 9 : HcclResult DevRdmaConnectionV2::BuildSqContext(SqContext* context)
321 : {
322 9 : if (context == nullptr) {
323 0 : HCCL_ERROR("[GetSqContext] Invalid null pointer for context.");
324 0 : return HCCL_E_PTR;
325 : }
326 :
327 : // 零初始化roceSq, 确保不同dmaMode下未使用字段有确定值
328 9 : (void)memset_s(
329 9 : &context->contextInfo.roceSq, sizeof(context->contextInfo.roceSq), 0, sizeof(context->contextInfo.roceSq));
330 :
331 : struct QpAttr localQpAttr;
332 9 : s32 ret = RaGetQpAttr(qpHandle_, &localQpAttr);
333 9 : if (ret != 0) {
334 0 : HCCL_ERROR("[DevRdmaConnectionV2::%s]RaGetQpAttr failed, ret[%d]", __func__, ret);
335 0 : return HCCL_E_ROCE_CONNECT;
336 : }
337 :
338 9 : context->type = SQ_CONTEXT_TYPE_ROCE;
339 9 : context->contextInfo.roceSq.qpn = localQpAttr.qpn;
340 9 : context->contextInfo.roceSq.sqVa = ndaQpInfo_.sqInfo.qBuf.base;
341 9 : context->contextInfo.roceSq.wqeSize = ndaQpInfo_.sqInfo.qBuf.entrySize;
342 9 : context->contextInfo.roceSq.depth = ndaQpInfo_.sqInfo.qBuf.entryCnt;
343 9 : context->contextInfo.roceSq.headAddr = reinterpret_cast<uint64_t>(SqPiMem_.ptr());
344 9 : context->contextInfo.roceSq.tailAddr = reinterpret_cast<uint64_t>(SqCiMem_.ptr());
345 9 : context->contextInfo.roceSq.sl = qpInfo_.serviceLevel;
346 9 : context->contextInfo.roceSq.dbHwVa = reinterpret_cast<uint64_t>(ndaQpInfo_.sqInfo.dbHwVa.iovBase);
347 :
348 9 : if (dmaMode_ == QBUF_DMA_MODE_INDEP_UB) {
349 0 : context->contextInfo.roceSq.dbSwVa = reinterpret_cast<uint64_t>(ndaQpInfo_.sqInfo.dbrPiVa.iovBase);
350 0 : uint8_t mtuShift = static_cast<uint8_t>(localQpAttr.pathMtu - 1);
351 0 : uint8_t dbCos = static_cast<uint8_t>(localQpAttr.vendorPrivInfo & 0xFF);
352 0 : context->contextInfo.roceSq.dbVendorSpecified
353 0 : |= (static_cast<uint64_t>(mtuShift & UB_DB_VENDOR_FIELD_MASK) << UB_DB_VENDOR_MTUSHIFT_SHIFT);
354 0 : context->contextInfo.roceSq.dbVendorSpecified
355 0 : |= (static_cast<uint64_t>(dbCos & UB_DB_VENDOR_FIELD_MASK) << UB_DB_VENDOR_COS_SHIFT);
356 :
357 0 : HCCL_INFO("[DevRdmaConnectionV2][%s] mtuShift=%u, dbCos=%u", __func__, mtuShift, dbCos);
358 : }
359 :
360 9 : HCCL_INFO(
361 : "[DevRdmaConnectionV2][%s] type=%u, QPN=%u, SQ_VA=0x%llx, WQE_SIZE=%u, "
362 : "SQ_DEPTH=%u, SQ_HEAD_ADDR=0x%llx, SQ_TAIL_ADDR=0x%llx, "
363 : "SL=%u, DB_HW_VA=0x%llx, DB_SW_VA=0x%llx,"
364 : "DbVendorSpecified=0x%llx",
365 : __func__, context->type, context->contextInfo.roceSq.qpn, context->contextInfo.roceSq.sqVa,
366 : context->contextInfo.roceSq.wqeSize, context->contextInfo.roceSq.depth, context->contextInfo.roceSq.headAddr,
367 : context->contextInfo.roceSq.tailAddr, context->contextInfo.roceSq.sl, context->contextInfo.roceSq.dbHwVa,
368 : context->contextInfo.roceSq.dbSwVa, context->contextInfo.roceSq.dbVendorSpecified);
369 9 : return HCCL_SUCCESS;
370 : }
371 :
372 9 : HcclResult DevRdmaConnectionV2::BuildCqContext(CqContext* context)
373 : {
374 9 : if (context == nullptr) {
375 0 : HCCL_ERROR("[GetCqContext] Invalid null pointer for context.");
376 0 : return HCCL_E_PTR;
377 : }
378 :
379 : // 零初始化roceCq, 确保不同dmaMode下未使用字段有确定值
380 9 : (void)memset_s(
381 9 : &context->contextInfo.roceCq, sizeof(context->contextInfo.roceCq), 0, sizeof(context->contextInfo.roceCq));
382 :
383 9 : context->type = CQ_CONTEXT_TYPE_ROCE;
384 9 : context->contextInfo.roceCq.cqVa = ndaCqInfo_.cqInfo.qBuf.base;
385 9 : context->contextInfo.roceCq.cqeSize = ndaCqInfo_.cqInfo.qBuf.entrySize;
386 9 : context->contextInfo.roceCq.cqDepth = ndaCqInfo_.cqInfo.qBuf.entryCnt;
387 9 : context->contextInfo.roceCq.headAddr = reinterpret_cast<uint64_t>(CqPiMem_.ptr());
388 9 : context->contextInfo.roceCq.tailAddr = reinterpret_cast<uint64_t>(CqCiMem_.ptr());
389 : // PCIe模式(DEFAULT): 仅使用硬DB; UBNIC模式(INDEP_UB): 仅使用软DB
390 9 : if (dmaMode_ == QBUF_DMA_MODE_DEFAULT) {
391 9 : context->contextInfo.roceCq.cqn = ndaCqInfo_.resv[0]; // 云脉网卡NDA直驱，协商ndaCqInfo_.resv[0]字段为CQN
392 9 : context->contextInfo.roceCq.dbHwVa = reinterpret_cast<uint64_t>(ndaCqInfo_.cqInfo.dbHwVa.iovBase);
393 : } else {
394 0 : context->contextInfo.roceCq.cqn = DEFAULT_CQN;
395 0 : context->contextInfo.roceCq.dbSwVa = reinterpret_cast<uint64_t>(ndaCqInfo_.cqInfo.dbrCiVa.iovBase);
396 : }
397 :
398 9 : HCCL_INFO(
399 : "[DevRdmaConnectionV2][%s] type=%u, CQN=%u, CQ_VA=0x%llx, CQE_SIZE=%u, CQ_DEPTH=%u, "
400 : "CQ_HEAD_ADDR=0x%llx, CQ_TAIL_ADDR=0x%llx, DB_HW_VA=0x%llx, DB_SW_VA=0x%llx]",
401 : __func__, context->type, context->contextInfo.roceCq.cqn, context->contextInfo.roceCq.cqVa,
402 : context->contextInfo.roceCq.cqeSize, context->contextInfo.roceCq.cqDepth, context->contextInfo.roceCq.headAddr,
403 : context->contextInfo.roceCq.tailAddr, context->contextInfo.roceCq.dbHwVa, context->contextInfo.roceCq.dbSwVa);
404 :
405 9 : return HCCL_SUCCESS;
406 : }
407 :
408 4 : std::vector<char> DevRdmaConnectionV2::GetSqUniqueId() const
409 : {
410 4 : HCCL_DEBUG("start packing sq uniqueId");
411 : struct QpAttr localQpAttr;
412 4 : s32 ret = RaGetQpAttr(qpHandle_, &localQpAttr);
413 4 : if (ret != 0) {
414 0 : HCCL_ERROR("[DevRdmaConnectionV2::%s]RaGetQpAttr failed, ret[%d]", __func__, ret);
415 0 : return {};
416 : }
417 4 : uint8_t mtuShift = static_cast<uint8_t>(localQpAttr.pathMtu - 1);
418 4 : uint8_t dbCos = static_cast<uint8_t>(localQpAttr.vendorPrivInfo & 0xFF);
419 4 : uint64_t dbVendorSpecified = 0;
420 4 : dbVendorSpecified |= (static_cast<uint64_t>(mtuShift & UB_DB_VENDOR_FIELD_MASK) << UB_DB_VENDOR_MTUSHIFT_SHIFT);
421 4 : dbVendorSpecified |= (static_cast<uint64_t>(dbCos & UB_DB_VENDOR_FIELD_MASK) << UB_DB_VENDOR_COS_SHIFT);
422 :
423 4 : Hccl::BinaryStream binaryStream;
424 : // 打包1825网卡NDA直驱资源
425 4 : binaryStream << localQpAttr.qpn;
426 4 : binaryStream << ndaQpInfo_.sqInfo.qBuf.base;
427 4 : binaryStream << ndaQpInfo_.sqInfo.qBuf.entrySize;
428 4 : binaryStream << ndaQpInfo_.sqInfo.qBuf.entryCnt;
429 4 : binaryStream << reinterpret_cast<uint64_t>(SqPiMem_.ptr());
430 4 : binaryStream << reinterpret_cast<uint64_t>(SqCiMem_.ptr());
431 4 : binaryStream << reinterpret_cast<uint64_t>(ndaQpInfo_.sqInfo.dbHwVa.iovBase);
432 4 : binaryStream << reinterpret_cast<uint64_t>(ndaQpInfo_.sqInfo.dbrPiVa.iovBase);
433 4 : binaryStream << static_cast<uint8_t>(qpInfo_.serviceLevel);
434 4 : binaryStream << dbVendorSpecified;
435 :
436 4 : HCCL_DEBUG(
437 : "[DevRdmaConnectionV2][%s] mtuShift[%d], dbCos[%u], dbVendorSpecified[0x%llx]", __func__, mtuShift, dbCos,
438 : dbVendorSpecified);
439 :
440 4 : std::vector<char> result;
441 4 : binaryStream.Dump(result);
442 4 : return result;
443 4 : }
444 :
445 4 : std::vector<char> DevRdmaConnectionV2::GetCqUniqueId() const
446 : {
447 4 : HCCL_DEBUG("start packing cq uniqueId");
448 4 : Hccl::BinaryStream binaryStream;
449 : // 打包1825网卡NDA直驱资源
450 4 : binaryStream << DEFAULT_CQN;
451 4 : binaryStream << ndaCqInfo_.cqInfo.qBuf.base;
452 4 : binaryStream << ndaCqInfo_.cqInfo.qBuf.entrySize;
453 4 : binaryStream << ndaCqInfo_.cqInfo.qBuf.entryCnt;
454 4 : binaryStream << reinterpret_cast<uint64_t>(CqPiMem_.ptr());
455 4 : binaryStream << reinterpret_cast<uint64_t>(CqCiMem_.ptr());
456 4 : binaryStream << reinterpret_cast<uint64_t>(ndaCqInfo_.cqInfo.dbrCiVa.iovBase);
457 4 : binaryStream << static_cast<uint64_t>(0); // CQ DbVendorSpecified placeholder
458 :
459 4 : std::vector<char> result;
460 4 : binaryStream.Dump(result);
461 4 : return result;
462 4 : }
463 :
464 4 : std::vector<char> DevRdmaConnectionV2::GetUniqueId() const
465 : {
466 4 : HCCL_DEBUG("start packing conn uniqueId");
467 4 : Hccl::BinaryStream binaryStream;
468 4 : binaryStream << dmaMode_;
469 4 : binaryStream << GetSqUniqueId();
470 4 : binaryStream << GetCqUniqueId();
471 :
472 4 : std::vector<char> result;
473 4 : binaryStream.Dump(result);
474 4 : return result;
475 4 : }
476 :
477 : } // namespace hcomm
|
