Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 : #include <adapter_hccp.h>
11 : #include <securec.h>
12 : #include <unordered_map>
13 : #include <sys/socket.h>
14 : #include <netdb.h>
15 : #include <arpa/inet.h>
16 : #include <sys/types.h>
17 : #include <ifaddrs.h>
18 : #include <adapter_rts.h>
19 : #include <mutex>
20 : #include <memory>
21 : #include <unordered_set>
22 :
23 : #include "network/hccp_common.h"
24 : #include "externalinput.h"
25 : #include "dlra_function.h"
26 : #include "log.h"
27 : #include "../host/transport_ibverbs_pub.h"
28 : #include "config_plf_log.h"
29 :
30 : using namespace hccl;
31 : using namespace std;
32 :
33 : /* 检查函数返回值是否为ROCE_ENOMEM_RET, 记录指定日志, 并返回HCCL_E_OOM, 内存大小取决于qp深度配置 */
34 : #define CHK_OOM_RET(ret, qpInfo) \
35 : do { \
36 : if ((ret) == ROCE_ENOMEM_RET) { \
37 : RPT_ENV_ERR(true, "EI0011", \
38 : std::vector<std::string>({"memory_size"}), \
39 : std::vector<std::string>({"262144~3145728"})); \
40 : HCCL_ERROR("[%s] ra qp create fail, reason: out of memory. qpInfo:[%s], return: ret[%d]", \
41 : __func__, (qpInfo), (ret)); \
42 : return HCCL_E_OOM; \
43 : } \
44 : } while (0)
45 :
46 : constexpr u32 MAX_NUM_OF_BATCH_CONN = 16;
47 : constexpr u32 MAX_CQ_DEPTH = 65535;
48 : constexpr u32 MAX_INLINE_DATA = 128;
49 : constexpr u32 MAX_WR_NUM = 1024;
50 : constexpr u32 MAX_RECV_SGE_NUM = 1;
51 : constexpr u32 REPEAT_RAINIT_ERROR_CODE = 328002;
52 : constexpr u32 REPEAT_LISTEN_ERROR_CODE = 128205;
53 :
54 : // network 获取版本信息参数
55 : constexpr u32 SOCKET_BATCH_CLOSE_INTERFACE = 1;
56 : constexpr u32 SOCKET_BATCH_CLOSE_SUP_VER = 2;
57 : constexpr u32 QP_ATTR_QOS_INTERFACE = 29; // RA_RS_SET_QP_ATTR_QOS 的 opcode为29
58 : constexpr u32 QP_ATTR_TIMEOUT_INTERFACE = 30; // RA_RS_SET_QP_ATTR_TIMEOUT 的 opcode为30
59 : constexpr u32 QP_ATTR_RETRY_CNT_INTERFACE = 31; // RA_RS_SET_QP_ATTR_RETRY_CNT 的 opcode为31
60 : constexpr u32 QP_ATTR_QOS_SUP_VER = 1; // 当前支持的版本号为1
61 :
62 : constexpr u32 IFNUM_INTERFACE = 33; // RA_RS_GET_IFNUM的opcode为33
63 : constexpr u32 IFNUM_INTERFACE_VERSION = 1; // 支持的RA_RS_GET_IFNUM_VERSION为1
64 :
65 : constexpr u32 IFADDRS_V2_INTERFACE = 38; // RA_RS_GET_IFADDRS_V2的opcode为38
66 : constexpr u32 IFADDRS_V2_INTERFACE_VERSTOIN = 3; // 支持获取chip上所有ip addr的IFADDRS_V2_INTERFACE_VERSTOIN为3
67 :
68 : constexpr u32 RDEV_INIT_WITH_BACKUP = 81; // RA_RS_RDEV_INIT_WITH_BACKUP的opcode为81
69 : constexpr u32 RDEV_INIT_WITH_BACKUP_SUP_VER = 1; // 当前支持的版本号为1
70 :
71 : constexpr u32 ALL_NIC_NUM_910_93 = 2; // 910_93 上最大网卡数量
72 : constexpr u32 ALL_NIC_NUM_910_A2 = 1; // 910 A2 上最大网卡数量
73 : constexpr u32 MAX_ALL_NIC_NUM = ALL_NIC_NUM_910_93; // 最大可能的网卡数量
74 :
75 : constexpr u32 QP_ATTR_TIMEOUT_SUPPORT_VER = 1; // 当前支持配置RDMA TimeOut的版本号为1
76 : constexpr u32 QP_ATTR_RETRY_CNT_SUPPORT_VER = 1; // 当前支持配置RDMA RetryCnt的版本号为1
77 :
78 : constexpr u32 CQE_ERR_INFO_INTERFACE = 32; // RA_RS_GET_CQE_ERR_INFO 的 opcode为32
79 : constexpr u32 CQE_ERR_INFO_LIST_INTERFACE = 80; // RA_RS_GET_CQE_ERR_INFO_LIST 的 opcode为80
80 : constexpr u32 CQE_ERR_INFO_SUP_VER = 1; // 当前支持的版本号为1
81 :
82 : constexpr u32 QP_CREATE_WITH_ATTRS_INTERFACE = 39; // RA_RS_QP_CREATE_WITH_ATTRS 的 opcode为39
83 : constexpr u32 QP_CREATE_WITH_ATTRS_SUP_VER = 1; // 当前支持的版本号为1
84 :
85 : constexpr u32 SOCKET_VNIC_IP_INFOS_INTERFACE = 55; // RA_RS_GET_VNIC_IP_INFOS 的 opcode为55
86 : constexpr u32 SOCKET_VNIC_IP_INFOS_SUP_VER = 1; // 当前支持的版本号为1
87 :
88 : constexpr u32 GET_NOTIFY_BA = 14; // RA_RS_GET_NOTIFY_BA 的 opcode为14
89 : constexpr u32 GET_NOTIFY_BA_VERSION = 2; // 当前支持的版本号为2
90 :
91 : constexpr u32 SEND_NORMAL_WRLIST = 83 ;
92 : constexpr u32 SEND_NORMAL_WRLIST_VERSION = 1 ;
93 :
94 : constexpr u32 TLV_INIT = 87;
95 : constexpr u32 TLV_DEINIT = 88;
96 : constexpr u32 TLV_REQUEST = 89;
97 : constexpr u32 TLV_VERSION = 1 ;
98 :
99 : constexpr u32 GET_TLS_ENABLE = 95;
100 : constexpr u32 TLS_ENABLE_VERSION = 1;
101 : // handle ref
102 : constexpr u32 FIRST_HANDLE_REF = 1;
103 :
104 : constexpr s32 HCCL_SEND_CQ_DEPTH_DEFAULT = (8 * 1024); // HCCL 默认的scq深度
105 :
106 : constexpr u32 TYPICAL_QP_MODIFY = 46; // opcode: RA_RS_TYPICAL_QP_MODIFY
107 : constexpr u32 TYPICAL_QP_MODIFY_VERSION = 2; // 支持QP解耦socket建链版本号
108 :
109 : constexpr u32 SOCKET_ABORT = 97; // opcode: RA_RS_SOCKET_ABORT
110 : constexpr u32 SOCKET_ABORT_VERSION = 1; // 支持socket abort的版本号
111 :
112 : constexpr u32 RS_INIT = 15; // opcode: RA_RS_INIT
113 : constexpr u32 RS_INIT_SUPPORT_ASYNC_VERSION = 2; // 支持socket async的版本号
114 :
115 : constexpr u32 ROCE_ENOMEM_RET = 328100; // 创建qp时由于内存不足的错误返回值
116 :
117 : template <typename T>
118 : struct HandleInfo {
119 : std::mutex handleMutex;
120 : std::unordered_map<u32, T> handleMap;
121 : std::unordered_map<T, u32> handleRef;
122 : };
123 :
124 : HandleInfo<SocketHandle> g_socketHandleInfo;
125 : HandleInfo<RdmaHandle> g_rdmaHandleInfo;
126 :
127 : #if T_DESC("RDMA异步", true)
128 0 : HcclResult HrtRaQpCreate(RdmaHandle rdmaHandle, int flag, int qpMode, QpHandle &qpHandle)
129 : {
130 0 : string qpInfo = string("rdmaHandle:") + to_string(reinterpret_cast<intptr_t>(rdmaHandle)) + string("qpHandle:") +
131 0 : to_string(reinterpret_cast<intptr_t>(&qpHandle)) + string("flag:") + to_string(flag) + string("qpMode:") +
132 0 : to_string(qpMode);
133 :
134 0 : s32 ret = DlRaFunction::GetInstance().dlRaQpCreate(rdmaHandle, flag, qpMode, &qpHandle);
135 :
136 0 : CHK_OOM_RET(ret, qpInfo.c_str());
137 :
138 0 : CHK_PRT_RET(ret != 0 || (qpHandle == nullptr),
139 : HCCL_ERROR("[Create][RaQp]errNo[0x%016llx] ra qp create fail. qpInfo:[%s], return: ret[%d]",
140 : HCCL_ERROR_CODE(HCCL_E_NETWORK), qpInfo.c_str(), ret),
141 : HCCL_E_NETWORK);
142 :
143 0 : struct QpAttr attr{};
144 0 : CHK_RET(hrtRaGetQpAttr(qpHandle, &attr));
145 0 : s32 deviceId = 0;
146 0 : if (hrtGetDevice(&deviceId) != HCCL_SUCCESS) {
147 0 : deviceId = -1;
148 : }
149 0 : PLF_CONFIG_DEBUG(PLF_RES, "Create Qp para: deviceId[%d] qpn[%u] qpInfo[%s]", deviceId, attr.qpn, qpInfo.c_str());
150 0 : return HCCL_SUCCESS;
151 0 : }
152 :
153 0 : HcclResult hrtRaTypicalQpCreate(RdmaHandle rdmaHandle, int flag,
154 : int qpMode, struct TypicalQp* qpInfo, QpHandle &qpHandle)
155 : {
156 0 : std::string qpInfoStr = std::string("rdmaHandle:") + std::to_string(reinterpret_cast<intptr_t>(rdmaHandle)) + \
157 0 : std::string("flag:") + std::to_string(flag) + std::string("qpMode:") + std::to_string(qpMode) + \
158 0 : std::to_string(reinterpret_cast<intptr_t>(&qpHandle));
159 :
160 0 : s32 ret = DlRaFunction::GetInstance().dlRaTypicalQpCreate(rdmaHandle, flag, qpMode, qpInfo, &qpHandle);
161 :
162 0 : CHK_OOM_RET(ret, qpInfoStr.c_str());
163 :
164 0 : RPT_ENV_ERR(ret != 0 || (qpHandle == nullptr), "EI0007",
165 : std::vector<std::string>({"resource_type", "resource_info"}), std::vector<std::string>({"qp", "CreateQp"}));
166 :
167 0 : CHK_PRT_RET(ret != 0 || (qpHandle == nullptr), HCCL_ERROR("[%s][%s]errNo[0x%016llx] ra qp create fail. "\
168 : "params: flag[%d], qpMode[%d]. return: ret[%d]", LOG_KEYWORDS_INIT_GROUP.c_str(), LOG_KEYWORDS_RESOURCE.c_str(),
169 : HCCL_ERROR_CODE(HCCL_E_NETWORK), flag, qpMode, ret), HCCL_E_NETWORK);
170 :
171 0 : s32 deviceId = 0;
172 0 : if (hrtGetDevice(&deviceId) != HCCL_SUCCESS) {
173 0 : deviceId = -1;
174 : }
175 0 : PLF_CONFIG_DEBUG(PLF_RES, "Create Qp para: deviceId[%d] qpn[%u] qpInfo[%s]", deviceId, qpInfo->qpn, qpInfoStr.c_str());
176 0 : return HCCL_SUCCESS;
177 0 : }
178 :
179 0 : HcclResult CreateTypicalCq(RdmaHandle rdmaHandle, u32 cqDepth, u32 &cqn, void **cqHandle)
180 : {
181 0 : HCCL_DEBUG("CreateTypicalCq cqDepth[%u]", cqDepth);
182 :
183 0 : s32 ret = DlRaFunction::GetInstance().dlRaTypicalCqCreate(rdmaHandle, cqDepth, &cqn, cqHandle);
184 0 : CHK_PRT_RET(ret != 0 || (*cqHandle == NULL),
185 : HCCL_ERROR("[CreateTypicalCq]create typical cq failed. ret[%d]", ret), HCCL_E_NETWORK);
186 0 : return HCCL_SUCCESS;
187 : }
188 :
189 0 : HcclResult DestroyTypicalCq(RdmaHandle rdmaHandle, u32 cqn, void *cqHandle)
190 : {
191 0 : HCCL_DEBUG("DestroyTypicalCq cqn[%u]", cqn);
192 :
193 0 : s32 ret = DlRaFunction::GetInstance().dlRaTypicalCqDestroy(rdmaHandle, cqn, cqHandle);
194 0 : CHK_PRT_RET(ret != 0,
195 : HCCL_ERROR("[DestroyTypicalCq]destroy typical cq failed. ret[%d]", ret), HCCL_E_NETWORK);
196 0 : return HCCL_SUCCESS;
197 : }
198 :
199 0 : HcclResult HrtRaQpDestroyWithoutCQ(QpHandle handle)
200 : {
201 0 : s32 ret = DlRaFunction::GetInstance().dlRaQpDestroyWithoutCQ(handle);
202 0 : CHK_PRT_RET(ret != 0,
203 : HCCL_ERROR("[HrtRaQpDestroyWithoutCQ]destroy qp without cq failed. ret[%d]", ret), HCCL_E_NETWORK);
204 0 : return HCCL_SUCCESS;
205 : }
206 :
207 30 : HcclResult hrtRaTypicalQpModify(QpHandle qpHandle, struct TypicalQp* localQpInfo, struct TypicalQp* remoteQpInfo)
208 : {
209 120 : std::string qpInfo = std::string("qpHandle:") + std::to_string(reinterpret_cast<intptr_t>(qpHandle)) + \
210 180 : std::string("localQpInfo:") + std::to_string(reinterpret_cast<intptr_t>(&localQpInfo)) + \
211 150 : std::string("remoteQpInfo:") + std::to_string(reinterpret_cast<intptr_t>(&remoteQpInfo));
212 :
213 30 : s32 ret = DlRaFunction::GetInstance().dlRaTypicalQpModify(qpHandle, localQpInfo, remoteQpInfo);
214 30 : RPT_ENV_ERR(ret != 0, "EI0007",
215 : std::vector<std::string>({"resource_type", "resource_info"}), std::vector<std::string>({"qp", "ModifyQp"}));
216 :
217 30 : CHK_PRT_RET(ret == ROCE_EOPENSRC , HCCL_RUN_WARNING("[%s][%s]ra qp modify need retry.",
218 : LOG_KEYWORDS_INIT_GROUP.c_str(), LOG_KEYWORDS_RESOURCE.c_str()), HCCL_E_AGAIN);
219 30 : CHK_PRT_RET(ret != 0 , HCCL_ERROR("[%s][%s]errNo[0x%016llx] ra qp modify fail. return: ret[%d]", \
220 : LOG_KEYWORDS_INIT_GROUP.c_str(), LOG_KEYWORDS_RESOURCE.c_str(), HCCL_ERROR_CODE(HCCL_E_NETWORK), ret), HCCL_E_NETWORK);
221 30 : return HCCL_SUCCESS;
222 30 : }
223 :
224 0 : HcclResult hrtRaTypicalSendWr(QpHandle handle, struct SendWr *wr, struct SendWrRsp *opRsp)
225 : {
226 0 : s32 ret = 0;
227 0 : auto startTime = std::chrono::steady_clock::now();
228 0 : auto timeout = std::chrono::seconds(GetExternalInputHcclLinkTimeOut());
229 :
230 0 : HCCL_DEBUG("ra send wr");
231 : while (true) {
232 0 : ret = DlRaFunction::GetInstance().dlRaTypicalSendWr(handle, wr, opRsp);
233 0 : if (!ret) {
234 0 : break; // 成功跳出
235 0 : } else if ((ret == SOCK_ENOENT) || (ret == SOCK_EAGAIN) ||
236 0 : (GetWorkflowMode() == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE && ret == ROCE_ENOMEM)) {
237 0 : bool bTimeout = ((std::chrono::steady_clock::now() - startTime) >= timeout);
238 0 : CHK_PRT_RET(bTimeout, HCCL_ERROR("[Send][RaWr]errNo[0x%016llx] ra get send async timeout[%d s]. "\
239 : "return[%d], params: send_wrAddr[%p], opRspAddr[%p]",
240 : HCCL_ERROR_CODE(HCCL_E_ROCE_TRANSFER), timeout, ret, wr, opRsp), HCCL_E_ROCE_TRANSFER);
241 0 : SaluSleep(ONE_MILLISECOND_OF_USLEEP);
242 : } else {
243 0 : HCCL_ERROR("[Send][RaWr]ra send async fail. return[%d], para: send_wrAddr[%p], "\
244 : "opRspAddr[%p].", ret, wr, opRsp);
245 0 : return HCCL_E_ROCE_TRANSFER; // 非-2/-11场景错误，不轮询，直接退出
246 : }
247 0 : }
248 0 : return HCCL_SUCCESS;
249 : }
250 :
251 6 : HcclResult HrtRaQpDestroy(QpHandle handle)
252 : {
253 6 : struct QpAttr attr{};
254 6 : CHK_RET(hrtRaGetQpAttr(handle, &attr));
255 6 : s32 deviceId = 0;
256 6 : if (hrtGetDevice(&deviceId) != HCCL_SUCCESS) {
257 0 : deviceId = -1;
258 : }
259 6 : PLF_CONFIG_DEBUG(PLF_RES, "Destroy Qp para: deviceId[%d] qpn[%u]", deviceId, attr.qpn);
260 :
261 6 : s32 ret = 0;
262 6 : auto startTime = chrono::steady_clock::now();
263 6 : auto timeout = chrono::seconds(GetExternalInputHcclLinkTimeOut());
264 : while (true) {
265 6 : ret = DlRaFunction::GetInstance().dlRaQpDestroy(handle);
266 6 : if (!ret) {
267 0 : break; // 成功跳出
268 6 : } else if (ret == ROCE_EAGAIN) {
269 0 : bool bTimeout = ((chrono::steady_clock::now() - startTime) >= timeout);
270 0 : CHK_PRT_RET(bTimeout, HCCL_ERROR("[Destroy][RaQp]errNo[0x%016llx] ra qp destroy timeout[%d s]. "\
271 : "return[%d].", HCCL_ERROR_CODE(HCCL_E_NETWORK), timeout, ret), HCCL_E_NETWORK);
272 0 : SaluSleep(ONE_MILLISECOND_OF_USLEEP);
273 : } else {
274 6 : HCCL_ERROR("[Destroy][RaQp]errNo[0x%016llx] ra qp destroy fail. return[%d].", \
275 : HCCL_ERROR_CODE(HCCL_E_NETWORK), ret);
276 6 : return HCCL_E_NETWORK; // 非ra限速场景错误，不轮询，直接退出
277 : }
278 0 : }
279 0 : return HCCL_SUCCESS;
280 : }
281 :
282 0 : HcclResult HrtRaGetQpDepth(RdmaHandle rdmaHandle, unsigned int *tempDepth, unsigned int *qpNum)
283 : {
284 0 : CHK_PTR_NULL(rdmaHandle);
285 :
286 0 : s32 ret = DlRaFunction::GetInstance().dlRaGetQpDepth(rdmaHandle, tempDepth, qpNum);
287 0 : CHK_PRT_RET(ret != 0, HCCL_ERROR("[HrtRaGetQpDepth]errNo[0x%016llx] ra get qp depth fail. return[%d]",
288 : HCCL_ERROR_CODE(HCCL_E_NETWORK), ret), HCCL_E_NETWORK);
289 0 : return HCCL_SUCCESS;
290 : }
291 :
292 0 : HcclResult HrtRaSetQpDepth(RdmaHandle rdmaHandle, unsigned int tempDepth, unsigned int *qpNum)
293 : {
294 0 : CHK_PTR_NULL(rdmaHandle);
295 :
296 0 : s32 ret = DlRaFunction::GetInstance().dlRaSetQpDepth(rdmaHandle, tempDepth, qpNum);
297 0 : CHK_PRT_RET(ret != 0, HCCL_ERROR("[dlRaSetQpDepth]errNo[0x%016llx] ra set qp depth fail. return[%d]",
298 : HCCL_ERROR_CODE(HCCL_E_NETWORK), ret), HCCL_E_NETWORK);
299 0 : return HCCL_SUCCESS;
300 : }
301 :
302 0 : HcclResult HrtRaQpNonBlockConnectAsync(QpHandle handle, const SocketHandle sockHandle)
303 : {
304 0 : s32 ret = DlRaFunction::GetInstance().dlRaQpConnectAsync(handle, sockHandle);
305 0 : if (ret == 0) {
306 0 : return HCCL_SUCCESS;
307 0 : } else if (ret == ROCE_EAGAIN) {
308 0 : return HCCL_E_AGAIN;
309 : } else {
310 0 : HCCL_ERROR("[HrtRaQpNonBlockConnectAsync]errNo[0x%016llx] ra qp connect async fail. return[%d].",\
311 : HCCL_ERROR_CODE(HCCL_E_NETWORK), ret);
312 0 : return HCCL_E_NETWORK;
313 : }
314 :
315 : return HCCL_SUCCESS;
316 : }
317 :
318 0 : HcclResult HrtRaQpConnectAsync(QpHandle handle, const SocketHandle sockHandle, std::function<bool()> needStop, u32 timeout)
319 : {
320 0 : s32 ret = 0;
321 0 : auto startTime = chrono::steady_clock::now();
322 : const chrono::seconds timeoutSec = chrono::seconds(
323 0 : timeout > 0 ? timeout : GetExternalInputHcclLinkTimeOut());
324 : while (true) {
325 0 : CHK_PRT_RET(needStop(), HCCL_ERROR("Terminating operation due to external request"), HCCL_E_INTERNAL);
326 :
327 0 : ret = DlRaFunction::GetInstance().dlRaQpConnectAsync(handle, sockHandle);
328 0 : if (!ret) {
329 0 : break; // 成功跳出
330 0 : } else if (ret == SOCK_EAGAIN) {
331 0 : bool bTimeout = ((chrono::steady_clock::now() - startTime) >= timeoutSec);
332 0 : CHK_PRT_RET(bTimeout, HCCL_ERROR("[ConnectAsync][RaQp]errNo[0x%016llx] ra qp connect async "\
333 : "timeout[%lld s]. return[%d].", HCCL_ERROR_CODE(HCCL_E_NETWORK), timeoutSec, ret), HCCL_E_NETWORK);
334 0 : SaluSleep(ONE_MILLISECOND_OF_USLEEP);
335 : } else {
336 0 : HCCL_ERROR("[ConnectAsync][RaQp]errNo[0x%016llx] ra qp connect async fail. return[%d]",\
337 : HCCL_ERROR_CODE(HCCL_E_NETWORK), ret);
338 0 : return HCCL_E_NETWORK; // 非ra限速场景错误，不轮询，直接退出
339 : }
340 0 : }
341 0 : return HCCL_SUCCESS;
342 : }
343 :
344 0 : s32 hrtGetRaQpStatus(QpHandle handle, int *status)
345 : {
346 0 : return DlRaFunction::GetInstance().dlRaGetQpStatus(handle, status);
347 : }
348 :
349 0 : HcclResult HrtRaMrReg(QpHandle handle, struct MrInfoT *mrInfo)
350 : {
351 0 : CHK_PTR_NULL(mrInfo);
352 0 : HCCL_DEBUG("ra mr reg: addr[%p], size[%llu], access[%d].", mrInfo->addr, mrInfo->size, mrInfo->access);
353 0 : s32 ret = DlRaFunction::GetInstance().dlRaMrReg(handle, mrInfo);
354 0 : CHK_PRT_RET(ret != 0, HCCL_ERROR("[Reg][RaMr]errNo[0x%016llx] ra mr reg fail. return[%d], params: "\
355 : "addr[%p], size[%llu], access[%d]", HCCL_ERROR_CODE(HCCL_E_NETWORK), ret, mrInfo->addr, mrInfo->size,
356 : mrInfo->access), HCCL_E_NETWORK);
357 0 : return HCCL_SUCCESS;
358 : }
359 :
360 0 : HcclResult HrtRaMrDereg(QpHandle handle, struct MrInfoT *mrInfo)
361 : {
362 0 : CHK_PTR_NULL(mrInfo);
363 0 : HCCL_INFO("ra mr dereg: qphandle[%p], addr[%p], size[%llu Byte], access[%d].",
364 : handle, mrInfo->addr, mrInfo->size, mrInfo->access);
365 0 : s32 ret = DlRaFunction::GetInstance().dlRaMrDereg(handle, mrInfo);
366 0 : CHK_PRT_RET(ret != 0, HCCL_ERROR("[Dereg][RaMr]errNo[0x%016llx] ra mr dereg fail. return[%d], params: "\
367 : "addr[%p], size[%llu Byte], access[%d]", HCCL_ERROR_CODE(HCCL_E_NETWORK), ret, mrInfo->addr,
368 : mrInfo->size, mrInfo->access), HCCL_E_NETWORK);
369 0 : return HCCL_SUCCESS;
370 : }
371 :
372 92 : HcclResult hrtRaRegGlobalMr(const RdmaHandle rdmaHandle, struct MrInfoT &mrInfo, MrHandle &mrHandle)
373 : {
374 92 : CHK_PTR_NULL(rdmaHandle);
375 92 : CHK_PTR_NULL(mrInfo.addr);
376 92 : CHK_PRT_RET((mrInfo.size <= 0), HCCL_ERROR("[hrtRaRegGlobalMr]memory size[%llu Byte] should be greater than 0.",
377 : mrInfo.size), HCCL_E_PARA);
378 :
379 92 : s32 ret = DlRaFunction::GetInstance().dlRaRegGlobalMr(rdmaHandle, &mrInfo, &mrHandle);
380 92 : CHK_PRT_RET(ret != 0, HCCL_ERROR("[hrtRaRegGlobalMr]errNo[0x%016llx] ra reg global mr fail. return[%d], params: "
381 : "addr[%p], size[%llu Byte], access[%d]", HCCL_ERROR_CODE(HCCL_E_NETWORK),
382 : ret, mrInfo.addr, mrInfo.size, mrInfo.access), HCCL_E_NETWORK);
383 92 : HCCL_DEBUG("[hrtRaRegGlobalMr]ra reg global mr: addr[%p], size[%llu Byte], access[%d]",\
384 : mrInfo.addr, mrInfo.size, mrInfo.access);
385 92 : return HCCL_SUCCESS;
386 : }
387 :
388 92 : HcclResult hrtRaDeRegGlobalMr(const RdmaHandle rdmaHandle, MrHandle mrHandle)
389 : {
390 92 : CHK_PTR_NULL(rdmaHandle);
391 92 : CHK_PTR_NULL(mrHandle);
392 :
393 92 : HCCL_DEBUG("[hrtRaDeRegGlobalMr]ra dereg global.");
394 92 : s32 ret = DlRaFunction::GetInstance().dlRaDeRegGlobalMr(rdmaHandle, mrHandle);
395 92 : CHK_PRT_RET(ret != 0, HCCL_ERROR("[hrtRaDeRegGlobalMr]errNo[0x%016llx] ra dereg global mr fail. return[%d]",\
396 : HCCL_ERROR_CODE(HCCL_E_NETWORK), ret), HCCL_E_NETWORK);
397 :
398 92 : return HCCL_SUCCESS;
399 : }
400 :
401 0 : HcclResult HrtRaSendWr(QpHandle handle, struct SendWr *wr, struct SendWrRsp *opRsp)
402 : {
403 0 : s32 ret = 0;
404 0 : auto startTime = chrono::steady_clock::now();
405 0 : auto timeout = chrono::seconds(GetExternalInputHcclLinkTimeOut());
406 :
407 0 : HCCL_DEBUG("ra send wr.");
408 : while (true) {
409 0 : ret = DlRaFunction::GetInstance().dlRaSendWr(handle, wr, opRsp);
410 0 : if (!ret) {
411 0 : break; // 成功跳出
412 0 : } else if ((ret == SOCK_ENOENT) || (ret == ROCE_EAGAIN) ||
413 0 : (GetWorkflowMode() == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE && ret == ROCE_ENOMEM)) {
414 0 : bool bTimeout = ((chrono::steady_clock::now() - startTime) >= timeout);
415 0 : CHK_PRT_RET(bTimeout, HCCL_ERROR("[Send][RaWr]errNo[0x%016llx] ra get send async timeout[%d s]. "\
416 : "return[%d], params: send_wrAddr[%p], opRspAddr[%p]",
417 : HCCL_ERROR_CODE(HCCL_E_ROCE_TRANSFER), timeout, ret, wr, opRsp), HCCL_E_ROCE_TRANSFER);
418 0 : SaluSleep(ONE_MILLISECOND_OF_USLEEP);
419 : } else {
420 0 : HCCL_ERROR("[Send][RaWr]ra send async fail. return[%d], para: send_wrAddr[%p], "\
421 : "opRspAddr[%p].", ret, wr, opRsp);
422 0 : return HCCL_E_ROCE_TRANSFER; // 非-2/-11场景错误，不轮询，直接退出
423 : }
424 0 : }
425 :
426 0 : return HCCL_SUCCESS;
427 : }
428 :
429 77 : HcclResult HrtRaSendWrV2(QpHandle handle, struct SendWrV2 *wr, struct SendWrRsp *opRsp, HcclWorkflowMode workflowMode)
430 : {
431 77 : s32 ret = 0;
432 77 : auto startTime = std::chrono::steady_clock::now();
433 77 : auto timeout = std::chrono::seconds(GetExternalInputHcclLinkTimeOut());
434 :
435 77 : HCCL_DEBUG("ra send wr.");
436 : while (true) {
437 77 : ret = DlRaFunction::GetInstance().dlRaSendWrV2(handle, wr, opRsp);
438 77 : if (!ret) {
439 77 : break; // 成功跳出
440 0 : } else if ((ret == SOCK_ENOENT) || (ret == ROCE_EAGAIN) ||
441 0 : (workflowMode == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE && ret == ROCE_ENOMEM)) {
442 0 : HCCL_WARNING("after 1ms sendwr, ret=%d", ret);
443 0 : bool bTimeout = ((std::chrono::steady_clock::now() - startTime) >= timeout);
444 0 : CHK_PRT_RET(bTimeout, HCCL_ERROR("[Send][RaWr]errNo[0x%016llx] ra get send async timeout[%d s]. "\
445 : "return[%d], params: send_wrAddr[%p], opRspAddr[%p]",
446 : HCCL_ERROR_CODE(HCCL_E_ROCE_TRANSFER), timeout, ret, wr, opRsp), HCCL_E_ROCE_TRANSFER);
447 0 : SaluSleep(ONE_MILLISECOND_OF_USLEEP);
448 0 : } else {
449 0 : HCCL_ERROR("[Send][RaWr]ra send async fail. return[%d], para: send_wrAddr[%p], "\
450 : "opRspAddr[%p].", ret, wr, opRsp);
451 0 : return HCCL_E_ROCE_TRANSFER; // 非-2/-11场景错误，不轮询，直接退出
452 : }
453 0 : }
454 :
455 77 : return HCCL_SUCCESS;
456 : }
457 :
458 0 : HcclResult HrtRaSendWrVerbs(QpHandle handle, struct SendWrVerbs *wr, struct SendWrRsp *opRsp)
459 : {
460 0 : s32 ret = 0;
461 0 : auto startTime = std::chrono::steady_clock::now();
462 0 : auto timeout = std::chrono::seconds(GetExternalInputHcclLinkTimeOut());
463 :
464 0 : HCCL_DEBUG("ra send wr verbs.");
465 : while (true) {
466 0 : ret = DlRaFunction::GetInstance().dlRaSendWrVerbs(handle, wr, opRsp);
467 0 : if (!ret) {
468 0 : break;
469 0 : } else if ((ret == SOCK_ENOENT) || (ret == ROCE_EAGAIN) ||
470 0 : (GetWorkflowMode() == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE && ret == ROCE_ENOMEM)) {
471 0 : bool bTimeOut = ((std::chrono::steady_clock::now() - startTime) >= timeout);
472 0 : CHK_PRT_RET(bTimeOut, HCCL_ERROR("[HrtRaSendWrVerbs][RaWr]errNo[0x%016llx] ra get send async timeout[%d s]. "\
473 : "return[%d], params: send_wrAddr[%p], opRspAddr[%p]",
474 : HCCL_ERROR_CODE(HCCL_E_ROCE_TRANSFER), timeout, ret, wr, opRsp), HCCL_E_ROCE_TRANSFER);
475 0 : SaluSleep(ONE_MILLISECOND_OF_USLEEP);
476 : } else {
477 0 : HCCL_ERROR("[HrtRaSendWrVerbs][RaWr]ra send async fail. return[%d], para: send_wrAddr[%p], "\
478 : "opRspAddr[%p].", ret, wr, opRsp);
479 0 : return HCCL_E_ROCE_TRANSFER;
480 : }
481 0 : }
482 :
483 0 : return HCCL_SUCCESS;
484 : }
485 :
486 0 : HcclResult HrtRaRecvWrVerbs(QpHandle handle, struct RecvWrVerbs *wr)
487 : {
488 0 : s32 ret = 0;
489 0 : auto startTime = std::chrono::steady_clock::now();
490 0 : auto timeout = std::chrono::seconds(GetExternalInputHcclLinkTimeOut());
491 :
492 0 : HCCL_DEBUG("ra recv wr verbs.");
493 : while (true) {
494 0 : ret = DlRaFunction::GetInstance().dlRaRecvWrVerbs(handle, wr);
495 0 : if (!ret) {
496 0 : break;
497 0 : } else if ((ret == SOCK_ENOENT) || (ret == ROCE_EAGAIN) ||
498 0 : (GetWorkflowMode() == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE && ret == ROCE_ENOMEM)) {
499 0 : bool bTimeout = ((std::chrono::steady_clock::now() - startTime) >= timeout);
500 0 : CHK_PRT_RET(bTimeout, HCCL_ERROR("[Recv][RaWr]errNo[0x%016llx] ra get recv async timeout[%d s]. "\
501 : "return[%d], params: recv_wrAddr[%p]",
502 : HCCL_ERROR_CODE(HCCL_E_ROCE_TRANSFER), timeout, ret, wr), HCCL_E_ROCE_TRANSFER);
503 0 : SaluSleep(ONE_MILLISECOND_OF_USLEEP);
504 : } else {
505 0 : HCCL_ERROR("[Recv][RaWr]ra recv async fail. return[%d], para: recv_wrAddr[%p].",
506 : ret, wr);
507 0 : return HCCL_E_ROCE_TRANSFER;
508 : }
509 0 : }
510 :
511 0 : return HCCL_SUCCESS;
512 : }
513 :
514 0 : s32 hrtRaPollCq(QpHandle handle, bool is_send_cq, unsigned int num, void *wc)
515 : {
516 0 : CHK_PTR_NULL(handle);
517 0 : CHK_PTR_NULL(wc);
518 :
519 0 : u32 ret = DlRaFunction::GetInstance().dlRaPollCq(handle, is_send_cq, num, wc);
520 0 : CHK_PRT_RET(static_cast<u32>(ret) > num, HCCL_ERROR("[hrtRaPollCq] PollCq fail. return[%d]", ret), ret);
521 0 : return ret;
522 : }
523 :
524 0 : s32 HrtRaPollTypicalCq(void* cqHandle, u32 num, void *wc)
525 : {
526 0 : CHK_PTR_NULL(cqHandle);
527 0 : CHK_PTR_NULL(wc);
528 0 : u32 ret = DlRaFunction::GetInstance().dlRaPollTypicalCq(cqHandle, num, wc);
529 0 : CHK_PRT_RET(static_cast<u32>(ret) > num, HCCL_ERROR("[HrtRaPollTypicalCq] PollCq fail. return[%d]", ret), ret);
530 0 : return ret;
531 : }
532 :
533 0 : HcclResult hrtRaQpBatchModify(RdmaHandle rdmaHandle, QpHandle qpHandle[], unsigned int num, int expectStatus)
534 : {
535 0 : if (DlRaFunction::GetInstance().dlRaQpBatchModify == nullptr) {
536 0 : HCCL_ERROR("[Send][RaQpBatchModify]driver package does not support ra_qp_batch_modify interface, "\
537 : "please change new one");
538 0 : return HCCL_E_NOT_SUPPORT;
539 : }
540 0 : s32 ret = DlRaFunction::GetInstance().dlRaQpBatchModify(rdmaHandle, &qpHandle[0], num, expectStatus);
541 0 : CHK_PRT_RET(ret != 0 || (qpHandle[0] == nullptr),
542 : HCCL_ERROR("[BatchModify][RaQp]errNo[0x%016llx] ra qp batch modify fail. "\
543 : "params: num[%u], expectStatus[%d]. return: ret[%d]", \
544 : HCCL_ERROR_CODE(HCCL_E_NETWORK), num, expectStatus), HCCL_E_NETWORK);
545 0 : return HCCL_SUCCESS;
546 : }
547 :
548 0 : HcclResult HrtRaSendWrlist(QpHandle handle, struct SendWrlistData wr[], struct SendWrRsp opRsp[],
549 : unsigned int sendNum, unsigned int *completeNum)
550 : {
551 0 : if (DlRaFunction::GetInstance().dlRaSendWrlist == nullptr) {
552 0 : HCCL_ERROR("[Send][RaWrlist]driver package does not support hrtRaSendWrlist interface, "\
553 : "please change new one");
554 0 : return HCCL_E_NOT_SUPPORT;
555 : }
556 0 : s32 ret = 0;
557 0 : auto startTime = chrono::steady_clock::now();
558 0 : auto timeout = chrono::seconds(GetExternalInputHcclLinkTimeOut());
559 0 : u32 remainNum = sendNum;
560 0 : unsigned int completeNumLocal = 0;
561 0 : *completeNum = 0;
562 : while (true) {
563 0 : if (remainNum > sendNum) {
564 0 : HCCL_ERROR("[Send][RaWr]ra wr list send async fail. return[%d], remainNum[%u], "\
565 : "sendNum[%u].", HCCL_E_ROCE_TRANSFER, remainNum, sendNum);
566 0 : return HCCL_E_ROCE_TRANSFER; // 非-2/-11场景错误，不轮询，直接退出
567 : }
568 0 : if (remainNum == 0) {
569 0 : break;
570 : }
571 0 : ret = DlRaFunction::GetInstance().dlRaSendWrlist(
572 0 : handle, wr + (sendNum - remainNum), opRsp + (sendNum - remainNum), remainNum, &completeNumLocal);
573 0 : *completeNum += completeNumLocal;
574 0 : if (!ret) {
575 0 : break; // 成功跳出
576 0 : } else if ((ret == SOCK_ENOENT) || (ret == ROCE_EAGAIN) ||
577 0 : (GetWorkflowMode() == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE && ret == ROCE_ENOMEM)) {
578 0 : remainNum -= completeNumLocal;
579 0 : bool bTimeout = ((chrono::steady_clock::now() - startTime) >= timeout);
580 0 : CHK_PRT_RET(bTimeout, HCCL_ERROR("[Send][RaWrList]errNo[0x%016llx] ra send wrlsit async timeout[%d s]. "\
581 : "return[%d], params: send_wrAddr[%p], opRspAddr[%p]",
582 : HCCL_ERROR_CODE(HCCL_E_ROCE_TRANSFER), timeout, ret, wr, opRsp), HCCL_E_ROCE_TRANSFER);
583 0 : SaluSleep(ONE_MILLISECOND_OF_USLEEP);
584 : } else {
585 0 : HCCL_ERROR("[Send][RaWr]ra wr list send async fail. return[%d], para: send_wrAddr[%p], dst_addr[%p],"\
586 : " bufAddr[%p], bufLen[%u], opRspAddr[%p].", ret, wr, wr->dstAddr, wr->memList.addr, wr->memList.len, opRsp);
587 0 : return HCCL_E_ROCE_TRANSFER; // 非-2/-11场景错误，不轮询，直接退出
588 : }
589 0 : }
590 :
591 0 : return HCCL_SUCCESS;
592 : }
593 :
594 0 : HcclResult HrtRaSendWrlistExt(QpHandle handle, struct SendWrlistDataExt wr[], struct SendWrRsp opRsp[],
595 : unsigned int sendNum, unsigned int *completeNum)
596 : {
597 : DevType deviceType;
598 0 : CHK_RET(hrtGetDeviceType(deviceType));
599 0 : if (deviceType != DevType::DEV_TYPE_910B && deviceType != DevType::DEV_TYPE_910_93) {
600 0 : vector<SendWrlistData> wqeList(sendNum);
601 0 : struct SendWrlistData* data = wqeList.data();
602 0 : for (unsigned int i = 0; i < sendNum; i++) {
603 0 : s32 sret = memcpy_s(&data[i], sizeof(SendWrlistData), &wr[i], sizeof(SendWrlistData));
604 0 : CHK_PRT_RET(sret != EOK, HCCL_ERROR("[WqeList][Add]add wqe list, memcpy wqe failed. errorno[%d]", sret),
605 : HCCL_E_MEMORY);
606 : }
607 0 : CHK_RET(HrtRaSendWrlist(handle, data, opRsp, sendNum, completeNum));
608 0 : } else {
609 : static bool flag = false;
610 0 : if (UNLIKELY(flag == false)) {
611 0 : if (UNLIKELY(DlRaFunction::GetInstance().dlRaSendWrlistExt == nullptr)) {
612 0 : HCCL_ERROR("[Send][RaWrlistExt]driver package does not support hrtRaSendWrlist interface, "\
613 : "please change new one");
614 0 : return HCCL_E_NOT_SUPPORT;
615 : }
616 0 : flag = true;
617 : }
618 :
619 0 : s32 ret = 0;
620 0 : auto startTime = chrono::steady_clock::now();
621 0 : auto timeout = chrono::seconds(GetExternalInputHcclLinkTimeOut());
622 0 : u32 remainNum = sendNum;
623 0 : unsigned int completeNumLocal = 0;
624 0 : *completeNum = 0;
625 : while (true) {
626 0 : if (remainNum > sendNum) {
627 0 : HCCL_ERROR("[Send][RaWr]ra wr list send async fail. return[%d], remainNum[%u], "\
628 : "sendNum[%u].", HCCL_E_ROCE_TRANSFER, remainNum, sendNum);
629 0 : return HCCL_E_ROCE_TRANSFER;
630 : }
631 0 : if (remainNum == 0) {
632 0 : break;
633 : }
634 0 : ret = DlRaFunction::GetInstance().dlRaSendWrlistExt(
635 0 : handle, wr + (sendNum - remainNum), opRsp + (sendNum - remainNum), remainNum, &completeNumLocal);
636 0 : *completeNum += completeNumLocal;
637 0 : if (!ret) {
638 0 : break; // 成功跳出
639 0 : } else if ((ret == SOCK_ENOENT) || (ret == ROCE_EAGAIN) ||
640 0 : (GetWorkflowMode() == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE && ret == ROCE_ENOMEM)) {
641 0 : remainNum -= completeNumLocal;
642 0 : bool bTimeout = ((chrono::steady_clock::now() - startTime) >= timeout);
643 0 : CHK_PRT_RET(bTimeout, HCCL_ERROR("[Send][RaWr]errNo[0x%016llx] ra wrlist send async timeout[%d s]. "\
644 : "return[%d], params: send_wrAddr[%p], opRspAddr[%p]",
645 : HCCL_ERROR_CODE(HCCL_E_ROCE_TRANSFER), timeout, ret, wr, opRsp), HCCL_E_ROCE_TRANSFER);
646 0 : SaluSleep(ONE_MILLISECOND_OF_USLEEP);
647 : } else {
648 0 : HCCL_ERROR("[Send][RaWr]ra wrlist send async fail. return[%d], para: send_wrAddr[%p], "\
649 : "opRspAddr[%p].", ret, wr, opRsp);
650 0 : return HCCL_E_ROCE_TRANSFER; // 非-2/-11场景错误，不轮询，直接退出
651 : }
652 0 : }
653 : }
654 :
655 0 : return HCCL_SUCCESS;
656 : }
657 :
658 0 : HcclResult HrtRaSendNormalWrlist(QpHandle handle, struct WrInfo wr[], struct SendWrRsp opRsp[],
659 : unsigned int sendNum, unsigned int *completeNum)
660 : {
661 0 : if (UNLIKELY(DlRaFunction::GetInstance().dlRaSendWrlist == nullptr)) {
662 0 : HCCL_ERROR("[Send][RaWrlist]driver package does not support hrtRaSendWrlist interface, "\
663 : "please change new one");
664 0 : return HCCL_E_NOT_SUPPORT;
665 : }
666 0 : s32 ret = 0;
667 0 : auto startTime = chrono::steady_clock::now();
668 0 : auto timeout = chrono::seconds(GetExternalInputHcclLinkTimeOut());
669 0 : u32 remainNum = sendNum;
670 0 : unsigned int completeNumLocal = 0;
671 0 : *completeNum = 0;
672 : while (true) {
673 0 : if (UNLIKELY(remainNum > sendNum)) {
674 0 : HCCL_ERROR("[Send][RaWr]ra wr list send async fail. return[%d], remainNum[%u], "\
675 : "sendNum[%u].", HCCL_E_ROCE_TRANSFER, remainNum, sendNum);
676 0 : return HCCL_E_ROCE_TRANSFER; // 非-2/-11场景错误，不轮询，直接退出
677 : }
678 0 : if (remainNum == 0) {
679 0 : break;
680 : }
681 0 : ret = DlRaFunction::GetInstance().dlRaSendNormalWrlist(
682 0 : handle, wr + (sendNum - remainNum), opRsp + (sendNum - remainNum), remainNum, &completeNumLocal);
683 0 : *completeNum += completeNumLocal;
684 0 : if (!ret) {
685 0 : break; // 成功跳出
686 : }
687 0 : if ((ret == ROCE_ENOENT) || (ret == ROCE_EAGAIN) || ret == ROCE_ENOMEM) {
688 0 : remainNum -= completeNumLocal;
689 0 : bool bTimeout = ((chrono::steady_clock::now() - startTime) >= timeout);
690 0 : CHK_PRT_RET(bTimeout, HCCL_ERROR("[Send][HrtRaSendNormalWrlist]errNo[0x%016llx] ra send wrlsit async timeout[%d s]. "\
691 : "return[%d], params: send_wrAddr[%p], opRspAddr[%p]",
692 : HCCL_ERROR_CODE(HCCL_E_ROCE_TRANSFER), timeout, ret, wr, opRsp), HCCL_E_ROCE_TRANSFER);
693 0 : SaluSleep(ONE_MILLISECOND_OF_USLEEP);
694 0 : } else {
695 0 : HCCL_ERROR("[Send][RaWr]ra wr list send async fail. return[%d], para: send_wrAddr[%p], dst_addr[%p],"\
696 : " bufAddr[%p], bufLen[%u], opRspAddr[%p].", ret, wr, wr->dstAddr, wr->memList.addr, wr->memList.len, opRsp);
697 0 : return HCCL_E_ROCE_TRANSFER; // 非-2/-11场景错误，不轮询，直接退出
698 : }
699 0 : }
700 :
701 0 : return HCCL_SUCCESS;
702 : }
703 :
704 :
705 0 : HcclResult HrtRaGetNotifyBaseAddr(RdmaHandle handle, u64 *va, u64 *size, std::function<bool()> needStop)
706 : {
707 0 : s32 ret = 0;
708 0 : auto startTime = chrono::steady_clock::now();
709 0 : auto timeout = chrono::seconds(GetExternalInputHcclLinkTimeOut());
710 : while (true) {
711 0 : CHK_PRT_RET(needStop(), HCCL_ERROR("Terminating operation due to external request"), HCCL_E_INTERNAL);
712 :
713 0 : ret = DlRaFunction::GetInstance().dlRaGetNotifyBaseAddr(handle, va, size);
714 0 : if (!ret) {
715 0 : break; // 成功跳出
716 0 : } else if (ret == ROCE_EAGAIN) {
717 0 : bool bTimeout = ((chrono::steady_clock::now() - startTime) >= timeout);
718 0 : CHK_PRT_RET(bTimeout, HCCL_ERROR("[Get][RaNotifyBaseAddr]errNo[0x%016llx] ra get notify base addr "\
719 : "timeout[%d s]. return[%d], params: va[0x%llx], size[%llu Byte]",
720 : HCCL_ERROR_CODE(HCCL_E_NETWORK), timeout, ret, *va, *size), HCCL_E_NETWORK);
721 0 : SaluSleep(ONE_MILLISECOND_OF_USLEEP);
722 : } else {
723 0 : HCCL_ERROR("[Get][RaNotifyBaseAddr]errNo[0x%016llx] ra get notify base addr fail. return[%d], params: "\
724 : "va[0x%llx], size[%llu]", HCCL_ERROR_CODE(HCCL_E_NETWORK), ret, *va, *size);
725 0 : return HCCL_E_NETWORK; // 非ra限速场景错误，不轮询，直接退出
726 : }
727 0 : }
728 0 : return HCCL_SUCCESS;
729 : }
730 :
731 0 : HcclResult HrtRaGetNotifyMrInfo(u32 phyId, RdmaHandle handle, struct MrInfoT *mrInfo)
732 : {
733 0 : s32 ret = 0;
734 0 : u32 getNotifyBaVersion = 0;
735 0 : HcclResult vRet = hrtRaGetInterfaceVersion(phyId, GET_NOTIFY_BA, &getNotifyBaVersion);
736 0 : if (vRet != HCCL_SUCCESS || getNotifyBaVersion < GET_NOTIFY_BA_VERSION) {
737 0 : HCCL_ERROR("this package does not support HrtRaGetNotifyMrInfo for device, please change new package");
738 0 : return HCCL_E_NOT_SUPPORT;
739 : }
740 0 : auto startTime = chrono::steady_clock::now();
741 0 : auto timeout = chrono::seconds(GetExternalInputHcclLinkTimeOut());
742 : while (true) {
743 0 : ret = DlRaFunction::GetInstance().dlRaGetNotifyMrInfo(handle, mrInfo);
744 0 : if (!ret) {
745 0 : break; // 成功跳出
746 0 : } else if (ret == ROCE_EAGAIN) {
747 0 : bool bTimeout = ((chrono::steady_clock::now() - startTime) >= timeout);
748 0 : CHK_PRT_RET(bTimeout, HCCL_ERROR("[Get][RaGetNotifyMrInfo]errNo[0x%016llx] ra get notify mr info "\
749 : "timeout[%d s]. return[%d]",
750 : HCCL_ERROR_CODE(HCCL_E_NETWORK), timeout, ret),
751 : HCCL_E_NETWORK);
752 0 : SaluSleep(ONE_MILLISECOND_OF_USLEEP);
753 : } else {
754 0 : HCCL_ERROR("[Get][RaGetNotifyMrInfo]errNo[0x%016llx] ra get notify mr info fail. return[%d]",
755 : HCCL_ERROR_CODE(HCCL_E_NETWORK), ret);
756 0 : return HCCL_E_NETWORK;
757 : }
758 0 : }
759 0 : return HCCL_SUCCESS;
760 : }
761 :
762 228 : HcclResult HrtRaInit(struct RaInitConfig *config)
763 : {
764 228 : CHK_RET(DlRaFunction::GetInstance().DlRaFunctionInit());
765 228 : s32 ret = 0;
766 228 : auto startTime = chrono::steady_clock::now();
767 228 : auto timeout = chrono::seconds(GetExternalInputHcclLinkTimeOut());
768 : while (true) {
769 228 : ret = DlRaFunction::GetInstance().dlRaInit(config);
770 228 : if (!ret) {
771 228 : break; // 成功跳出
772 0 : } else if (ret == HCCP_EAGAIN) {
773 0 : bool bTimeout = ((chrono::steady_clock::now() - startTime) >= timeout);
774 0 : CHK_PRT_RET(bTimeout, HCCL_ERROR("[Init][Ra]errNo[0x%016llx] ra init timeout[%lld s]. return[%d], "\
775 : "phyId[%u], nicPosition[%u], hdcType[%d]", HCCL_ERROR_CODE(HCCL_E_TIMEOUT), timeout, ret,\
776 : config->phyId, config->nicPosition, config->hdcType), HCCL_E_TIMEOUT);
777 0 : SaluSleep(ONE_MILLISECOND_OF_USLEEP);
778 : } else {
779 0 : if (ret == REPEAT_RAINIT_ERROR_CODE) {
780 0 : HCCL_RUN_WARNING("ra init repeatedly, return. phyId[%u] nicPosition[%u] hdcType[%d]",
781 : config->phyId, config->nicPosition, config->hdcType);
782 0 : return HCCL_E_PARA;
783 : }
784 0 : HCCL_ERROR("[Init][Ra]errNo[0x%016llx] ra init fail ret[%d] phyId[%u] nicPosition[%u] hdcType[%d]", \
785 : HCCL_ERROR_CODE(HCCL_E_NETWORK), ret, config->phyId, config->nicPosition, config->hdcType);
786 0 : return HCCL_E_NETWORK; // 非ra限速场景错误，不轮询。直接退出
787 : }
788 0 : }
789 228 : HCCL_INFO("init ra success.");
790 228 : return HCCL_SUCCESS;
791 : }
792 :
793 0 : HcclResult HrtRaRdmaInit(int mode, u32 notifyType, struct rdev rdevInfo, RdmaHandle &rdmaHandle)
794 : {
795 0 : s32 ret = DlRaFunction::GetInstance().dlRaRdmaInit(mode, notifyType, rdevInfo, &rdmaHandle);
796 0 : RPT_INPUT_ERR(ret == HCCP_ELINKDOWN,
797 : "EI0009",
798 : vector<string>({"device_id", "reason"}),
799 : vector<string>({std::to_string(rdevInfo.phyId), "The network port is down"})
800 : );
801 : #ifndef HCCD
802 0 : vector<HcclIpAddress> deviceIp;
803 0 : CHK_RET(hrtRaGetDeviceIP(rdevInfo.phyId, deviceIp));
804 0 : CHK_PRT_RET(deviceIp.size() < 1,
805 : HCCL_ERROR("Get ip address failed, phyId[%u]", rdevInfo.phyId), HCCL_E_INTERNAL);
806 0 : RPT_INPUT_ERR(ret == HCCP_EINVALIDIPS,
807 : "EI0014",
808 : vector<string>({ "value", "variable" ,"expect" }),
809 : vector<string>({ string(HcclIpAddress(rdevInfo.localIp.addr.s_addr).GetReadableIP()),
810 : "IP", string(deviceIp[0].GetReadableIP()) })
811 : );
812 : #endif
813 0 : CHK_PRT_CONT(ret == HCCP_EINVALIDIPS,
814 : HCCL_ERROR("[%s][%s]the IP address in the ranktable is inconsistent with the IP address of the network adapter.",
815 : LOG_KEYWORDS_INIT_GROUP.c_str(), LOG_KEYWORDS_RANKTABLE_CHECK.c_str()));
816 :
817 0 : CHK_PRT_RET(ret == HCCP_ELINKDOWN , HCCL_RUN_WARNING("ra rdma init need retry."), HCCL_E_AGAIN);
818 0 : CHK_PRT_RET(ret != 0 || (rdmaHandle == nullptr), HCCL_ERROR("[Init][RaRdma]errNo[0x%016llx] rdma init fail. "\
819 : "params: mode[%d]. notifyType[%u] phyId[%u] family[%d] s_addr[%u] ret[%d]", HCCL_ERROR_CODE(HCCL_E_INTERNAL),\
820 : mode, notifyType, rdevInfo.phyId, rdevInfo.family, rdevInfo.localIp.addr.s_addr, ret), HCCL_E_INTERNAL);
821 0 : return HCCL_SUCCESS;
822 0 : }
823 :
824 34 : HcclResult HrtRaRdmaInitWithAttr(struct RdevInitInfo &init_info, const struct rdev &rdevInfo, RdmaHandle &rdmaHandle)
825 : {
826 34 : HCCL_INFO("mode:[%d], NotifyTypeT:[%u], enabled910aLite:[%d], disabledLiteThread:[%d], enabled2mbLite:[%d]",
827 : init_info.mode, init_info.notifyType, init_info.enabled910aLite, init_info.disabledLiteThread,
828 : init_info.enabled2mbLite);
829 :
830 34 : s32 ret = DlRaFunction::GetInstance().dlRaRdmaInitWithAttr(init_info, rdevInfo, &rdmaHandle);
831 34 : RPT_INPUT_ERR(ret == HCCP_ELINKDOWN,
832 : "EI0009",
833 : vector<string>({"device_id", "reason"}),
834 : vector<string>({std::to_string(rdevInfo.phyId), "The network port is down"})
835 : );
836 34 : CHK_PRT_CONT(ret == HCCP_ELINKDOWN,
837 : HCCL_ERROR("[%s][%s]rdma init failed because RoCE link status is down, please check the network adapter configuration.",
838 : LOG_KEYWORDS_INIT_GROUP.c_str(), LOG_KEYWORDS_RESOURCE.c_str()));
839 : #ifndef HCCD
840 34 : if (init_info.mode != NETWORK_PEER_ONLINE) {
841 34 : vector<HcclIpAddress> deviceIp;
842 34 : CHK_RET(hrtRaGetDeviceIP(rdevInfo.phyId, deviceIp));
843 34 : CHK_PRT_RET(deviceIp.size() < 1,
844 : HCCL_ERROR("Get ip address failed, phyId[%u]", rdevInfo.phyId), HCCL_E_INTERNAL);
845 34 : RPT_INPUT_ERR(ret == HCCP_EINVALIDIPS,
846 : "EI0014",
847 : vector<string>({ "value", "variable" ,"expect" }),
848 : vector<string>({ string(HcclIpAddress(rdevInfo.localIp.addr.s_addr).GetReadableIP()), "IP", string(deviceIp[0].GetReadableIP()) })
849 : );
850 34 : }
851 : #endif
852 34 : CHK_PRT_CONT(ret == HCCP_EINVALIDIPS,
853 : HCCL_ERROR("[%s][%s]the IP address in the ranktable is inconsistent with the IP address of the network adapter.",
854 : LOG_KEYWORDS_INIT_GROUP.c_str(), LOG_KEYWORDS_RANKTABLE_CHECK.c_str()));
855 :
856 34 : CHK_PRT_RET(ret != 0 || (rdmaHandle == nullptr), HCCL_ERROR("[Init][RaRdma]errNo[0x%016llx] rdma init fail. "\
857 : "return: ret[%d]", HCCL_ERROR_CODE(HCCL_E_NETWORK), ret), HCCL_E_NETWORK);
858 34 : return HCCL_SUCCESS;
859 0 : }
860 :
861 1 : HcclResult HrtRdmaInitWithBackupAttr(struct RdevInitInfo &init_info, struct rdev &rdevInfo,
862 : struct rdev &backupRdevInfo, RdmaHandle &rdmaHandle)
863 : {
864 1 : HCCL_INFO("[%s]mode:[%d], NotifyTypeT:[%u], enabled910aLite:[%d], disabledLiteThread:[%d], "
865 : "enabled2mbLite:[%d]", __func__, init_info.mode, init_info.notifyType, init_info.enabled910aLite,
866 : init_info.disabledLiteThread, init_info.enabled2mbLite);
867 :
868 : // 获取版本号查看是否兼容
869 1 : u32 rdmainitBackupVersion = 0;
870 1 : HcclResult vRet = hrtRaGetInterfaceVersion(rdevInfo.phyId, RDEV_INIT_WITH_BACKUP, &rdmainitBackupVersion);
871 1 : if (vRet != HCCL_SUCCESS || rdmainitBackupVersion < RDEV_INIT_WITH_BACKUP_SUP_VER) {
872 1 : HCCL_WARNING("this package does not support HrtRdmaInitWithBackupAttr, please change new package.");
873 1 : return HCCL_E_NOT_SUPPORT;
874 : }
875 :
876 0 : s32 ret = DlRaFunction::GetInstance().dlRaRdmaInitWithBackupAttr(&init_info, &rdevInfo, &backupRdevInfo, &rdmaHandle);
877 0 : RPT_INPUT_ERR(ret == HCCP_ELINKDOWN,
878 : "EI0009",
879 : vector<string>({"device_id", "reason"}),
880 : vector<string>({std::to_string(rdevInfo.phyId), "The network port is down"})
881 : );
882 0 : CHK_PRT_CONT(ret == HCCP_ELINKDOWN,
883 : HCCL_ERROR("[%s][%s]rdma init failed because RoCE link status is down, please check the network adapter configuration.",
884 : LOG_KEYWORDS_INIT_GROUP.c_str(), LOG_KEYWORDS_RESOURCE.c_str()));
885 : #ifndef HCCD
886 0 : vector<HcclIpAddress> deviceIp;
887 0 : CHK_RET(hrtRaGetDeviceIP(rdevInfo.phyId, deviceIp));
888 0 : CHK_PRT_RET(deviceIp.size() < 1,
889 : HCCL_ERROR("Get ip address failed, phyId[%u]", rdevInfo.phyId), HCCL_E_INTERNAL);
890 0 : RPT_INPUT_ERR(ret == HCCP_EINVALIDIPS,
891 : "EI0014",
892 : vector<string>({ "value", "variable" ,"expect" }),
893 : vector<string>({ string(HcclIpAddress(rdevInfo.localIp.addr.s_addr).GetReadableIP()), "IP", string(deviceIp[0].GetReadableIP()) })
894 : );
895 : #endif
896 0 : CHK_PRT_CONT(ret == HCCP_EINVALIDIPS,
897 : HCCL_ERROR("[%s][%s]the IP address in the ranktable is inconsistent with the IP address of the network adapter.",
898 : LOG_KEYWORDS_INIT_GROUP.c_str(), LOG_KEYWORDS_RANKTABLE_CHECK.c_str()));
899 :
900 0 : CHK_PRT_RET(ret != 0 || (rdmaHandle == nullptr), HCCL_ERROR("[Init][RaRdma]errNo[0x%016llx] rdma init fail. "\
901 : "return: ret[%d]", HCCL_ERROR_CODE(HCCL_E_NETWORK), ret), HCCL_E_NETWORK);
902 0 : return HCCL_SUCCESS;
903 0 : }
904 :
905 0 : HcclResult HrtRaRdmaInitRef(int mode, u32 notifyType, const struct rdev &rdevInfo, RdmaHandle &rdmaHandle)
906 : {
907 0 : lock_guard<mutex> lock(g_rdmaHandleInfo.handleMutex);
908 0 : if (g_rdmaHandleInfo.handleMap.find(rdevInfo.localIp.addr.s_addr) !=
909 0 : g_rdmaHandleInfo.handleMap.end()) {
910 0 : HCCL_DEBUG("The rdmaHandle[%p] corresponding to the ipAddr[%u] has been initialized.",
911 : rdmaHandle, rdevInfo.localIp.addr.s_addr);
912 :
913 0 : rdmaHandle = g_rdmaHandleInfo.handleMap[rdevInfo.localIp.addr.s_addr];
914 0 : g_rdmaHandleInfo.handleRef[rdmaHandle]++;
915 0 : return HCCL_SUCCESS;
916 : }
917 :
918 0 : CHK_RET(HrtRaRdmaInit(mode, notifyType, rdevInfo, rdmaHandle));
919 0 : g_rdmaHandleInfo.handleMap[rdevInfo.localIp.addr.s_addr] = rdmaHandle;
920 0 : g_rdmaHandleInfo.handleRef[rdmaHandle] = FIRST_HANDLE_REF;
921 0 : return HCCL_SUCCESS;
922 0 : }
923 :
924 0 : HcclResult HrtRaRdmaGetHandle(unsigned int phyId, RdmaHandle &rdmaHandle)
925 : {
926 0 : CHK_SMART_PTR_NULL(DlRaFunction::GetInstance().dlRaRdmaGetHandle);
927 0 : s32 ret = DlRaFunction::GetInstance().dlRaRdmaGetHandle(phyId, &rdmaHandle);
928 :
929 0 : CHK_PRT_RET(ret != 0 || (rdmaHandle == nullptr), HCCL_ERROR("[Get][RdmaHandle]errNo[0x%016llx] "\
930 : "get rdma handle fail. return: ret[%d]", HCCL_ERROR_CODE(HCCL_E_NETWORK), ret), HCCL_E_NETWORK);
931 :
932 0 : HCCL_DEBUG("get rdma handle success.");
933 0 : return HCCL_SUCCESS;
934 : }
935 :
936 35 : HcclResult HrtGetRdmaLiteStatus(RdmaHandle rdmaHandle, int *supportLite)
937 : {
938 35 : if (rdmaHandle == nullptr) {
939 0 : HCCL_ERROR("[Get][RdmaLiteStatus]rdmaHandle is nullptr, please input the correct rdmaHandle");
940 0 : return HCCL_E_PTR;
941 : }
942 35 : s32 ret = DlRaFunction::GetInstance().dlRaGetRdmaLiteStatus(rdmaHandle, supportLite);
943 35 : CHK_PRT_RET(ret != 0, HCCL_ERROR("[Get][RdmaLiteStatus]errNo[0x%016llx] get rdma lite status fail. "\
944 : "return: ret[%d]", HCCL_ERROR_CODE(HCCL_E_NETWORK), ret), HCCL_E_NETWORK);
945 :
946 35 : return HCCL_SUCCESS;
947 : }
948 :
949 231 : HcclResult HrtRaDeInit(struct RaInitConfig *config)
950 : {
951 231 : s32 ret = 0;
952 231 : auto startTime = chrono::steady_clock::now();
953 231 : auto timeout = chrono::seconds(GetExternalInputHcclLinkTimeOut());
954 : while (true) {
955 231 : ret = DlRaFunction::GetInstance().dlRaDeInit(config);
956 231 : if (!ret) {
957 231 : break; // 成功跳出
958 0 : } else if (ret == HCCP_EAGAIN) {
959 0 : bool bTimeout = ((chrono::steady_clock::now() - startTime) >= timeout);
960 0 : CHK_PRT_RET(bTimeout, HCCL_ERROR("[DeInit][Ra]errNo[0x%016llx] ra deinit timeout[%lld s]. return[%d], "\
961 : "phyId[%u] nicPosition[%u] hdcType[%d]", HCCL_ERROR_CODE(HCCL_E_TIMEOUT), timeout, ret,\
962 : config->phyId, config->nicPosition, config->hdcType), HCCL_E_TIMEOUT);
963 0 : SaluSleep(ONE_MILLISECOND_OF_USLEEP);
964 : } else {
965 0 : HCCL_ERROR("[DeInit][Ra]errNo[0x%016llx] ra deinit fail. ret[%d] phyId[%u] nicPosition[%u] hdcType[%d]", \
966 : HCCL_ERROR_CODE(HCCL_E_NETWORK), ret, config->phyId, config->nicPosition, config->hdcType);
967 0 : return HCCL_E_NETWORK; // 非ra限速场景错误，不轮询。直接退出
968 : }
969 0 : }
970 231 : return HCCL_SUCCESS;
971 : }
972 :
973 36 : HcclResult HrtRaRdmaDeInit(RdmaHandle &rdmaHandle, u32 notifyType)
974 : {
975 36 : CHK_PTR_NULL(rdmaHandle);
976 36 : s32 ret = DlRaFunction::GetInstance().dlRaRdmaDeInit(rdmaHandle, notifyType);
977 36 : if (ret != HCCL_SUCCESS) {
978 2 : HCCL_ERROR("[DeInit][RaRdma] rdmaHandle[%p]", rdmaHandle);
979 2 : rdmaHandle = nullptr;
980 : }
981 36 : CHK_PRT_RET(ret != 0, HCCL_ERROR("[DeInit][RaRdma]errNo[0x%016llx] rt rdev deinit fail. return[%d]."\
982 : "notifyType[%u]", HCCL_ERROR_CODE(HCCL_E_NETWORK), ret, notifyType), HCCL_E_NETWORK);
983 34 : return HCCL_SUCCESS;
984 : }
985 :
986 0 : HcclResult HrtRaRdmaDeInitRef(RdmaHandle &rdmaHandle, u32 notifyType)
987 : {
988 0 : lock_guard<mutex> lock(g_rdmaHandleInfo.handleMutex);
989 0 : g_rdmaHandleInfo.handleRef[rdmaHandle]--;
990 0 : if (g_rdmaHandleInfo.handleRef[rdmaHandle] == 0) {
991 0 : HCCL_DEBUG("This rdmaHandle[%p] is about to be deinitialized.", rdmaHandle);
992 0 : CHK_RET(HrtRaRdmaDeInit(rdmaHandle, notifyType));
993 0 : auto it = g_rdmaHandleInfo.handleMap.begin();
994 0 : while (it != g_rdmaHandleInfo.handleMap.end()) {
995 0 : if (it->second == rdmaHandle) {
996 0 : it = g_rdmaHandleInfo.handleMap.erase(it);
997 : } else {
998 0 : ++it;
999 : }
1000 : }
1001 :
1002 0 : g_rdmaHandleInfo.handleRef.erase(rdmaHandle);
1003 : }
1004 :
1005 0 : return HCCL_SUCCESS;
1006 0 : }
1007 :
1008 51 : HcclResult hrtRaSocketInit(int mode, struct rdev rdevInfo, SocketHandle &socketHandle)
1009 : {
1010 51 : s32 ret = DlRaFunction::GetInstance().dlRaSocketInit(mode, rdevInfo, &socketHandle);
1011 :
1012 51 : CHK_PRT_RET(ret != 0 || (socketHandle == nullptr), HCCL_ERROR("[Init][RaSock]errNo[0x%016llx] "\
1013 : "ra socket init fail. params: mode[%d]. return: ret[%d] phyId[%u] family[%d] s_addr[%u]",
1014 : HCCL_ERROR_CODE(HCCL_E_INTERNAL), mode, ret, rdevInfo.phyId, rdevInfo.family, rdevInfo.localIp.addr.s_addr),
1015 : HCCL_E_INTERNAL);
1016 :
1017 51 : HCCL_INFO("socket init success, ip[%u] device id[%u], socketHandle[%p]",
1018 : rdevInfo.localIp.addr.s_addr, rdevInfo.phyId, socketHandle);
1019 51 : return HCCL_SUCCESS;
1020 : }
1021 :
1022 22 : HcclResult hrtRaSocketInitV1(int mode, struct SocketInitInfoT socket_init, SocketHandle &socketHandle)
1023 : {
1024 22 : s32 ret = DlRaFunction::GetInstance().dlRaSocketInitV1(mode, socket_init, &socketHandle);
1025 :
1026 22 : CHK_PRT_RET(ret != 0 || (socketHandle == nullptr),
1027 : HCCL_ERROR("[Init][RaSockV1]errNo[0x%016llx] ra socket v1 init fail. params: mode[%d]. return: ret[%d]",
1028 : HCCL_ERROR_CODE(HCCL_E_NETWORK), mode, ret),
1029 : HCCL_E_NETWORK);
1030 22 : HCCL_INFO("socket init v1 success, socketHandle[%p]", socketHandle);
1031 22 : return HCCL_SUCCESS;
1032 : }
1033 :
1034 0 : HcclResult hrtRaSocketInitRef(int mode, const struct rdev &rdevInfo, SocketHandle &socketHandle)
1035 : {
1036 0 : lock_guard<mutex> lock(g_socketHandleInfo.handleMutex);
1037 0 : if (g_socketHandleInfo.handleMap.find(rdevInfo.localIp.addr.s_addr) !=
1038 0 : g_socketHandleInfo.handleMap.end()) {
1039 0 : HCCL_DEBUG("The socketHandle[%p] corresponding to the ipAddr[%u] has been initialized.",
1040 : socketHandle, rdevInfo.localIp.addr.s_addr);
1041 :
1042 0 : socketHandle = g_socketHandleInfo.handleMap[rdevInfo.localIp.addr.s_addr];
1043 0 : g_socketHandleInfo.handleRef[socketHandle]++;
1044 0 : return HCCL_SUCCESS;
1045 : }
1046 :
1047 0 : CHK_RET(hrtRaSocketInit(mode, rdevInfo, socketHandle));
1048 0 : g_socketHandleInfo.handleMap[rdevInfo.localIp.addr.s_addr] = socketHandle;
1049 0 : g_socketHandleInfo.handleRef[socketHandle] = FIRST_HANDLE_REF;
1050 0 : return HCCL_SUCCESS;
1051 0 : }
1052 :
1053 71 : HcclResult hrtRaSocketDeInit(SocketHandle &socketHandle)
1054 : {
1055 71 : CHK_PTR_NULL(socketHandle);
1056 71 : s32 ret = DlRaFunction::GetInstance().dlRaSocketDeInit(socketHandle);
1057 71 : if (ret != HCCL_SUCCESS) {
1058 0 : HCCL_ERROR("[DeInit][RaSocket] socketHandle[%p]", socketHandle);
1059 0 : socketHandle = nullptr;
1060 : }
1061 71 : CHK_PRT_RET(ret != 0, HCCL_ERROR("[DeInit][RaSocket]errNo[0x%016llx] rt socket deinit fail. return[%d]",\
1062 : HCCL_ERROR_CODE(HCCL_E_NETWORK), ret), HCCL_E_NETWORK);
1063 71 : return HCCL_SUCCESS;
1064 : }
1065 :
1066 0 : HcclResult hrtRaSocketDeInitRef(SocketHandle &socketHandle)
1067 : {
1068 0 : lock_guard<mutex> lock(g_socketHandleInfo.handleMutex);
1069 0 : g_socketHandleInfo.handleRef[socketHandle]--;
1070 0 : if (g_socketHandleInfo.handleRef[socketHandle] == 0) {
1071 0 : HCCL_DEBUG("This socketHandle[%p] is about to be deinitialized.", socketHandle);
1072 0 : CHK_RET(hrtRaSocketDeInit(socketHandle));
1073 0 : auto it = g_socketHandleInfo.handleMap.begin();
1074 0 : while (it != g_socketHandleInfo.handleMap.end()) {
1075 0 : if (it->second == socketHandle) {
1076 0 : it = g_socketHandleInfo.handleMap.erase(it);
1077 : } else {
1078 0 : ++it;
1079 : }
1080 : }
1081 :
1082 0 : g_socketHandleInfo.handleRef.erase(socketHandle);
1083 : }
1084 :
1085 0 : return HCCL_SUCCESS;
1086 0 : }
1087 :
1088 41 : HcclResult hrtRaSocketNonBlockListenStart(struct SocketListenInfoT conn[], u32 num)
1089 : {
1090 41 : CheckConnPort(conn, num);
1091 41 : s32 ret = DlRaFunction::GetInstance().dlRaSocketListenStart(conn, num);
1092 41 : if (ret == SOCK_EAGAIN) {
1093 0 : return HCCL_E_AGAIN;
1094 41 : } else if (ret == SOCK_EADDRINUSE) {
1095 0 : HCCL_INFO("ra socket listen could not start, due to the port[%u] has already been bound. "
1096 : "please try another port or check the port status", (num > 0 ? conn[0].port : HCCL_INVALID_PORT));
1097 0 : return HCCL_E_UNAVAIL;
1098 41 : } else if (ret != HCCL_SUCCESS) {
1099 0 : HCCL_ERROR("errNo[0x%016llx] ra socket listen start fail. return[%d], num[%u]",
1100 : HCCL_ERROR_CODE(HCCL_E_TCP_CONNECT), ret, num);
1101 0 : for (u32 idx = 0; idx < num; idx++) {
1102 0 : HCCL_ERROR("cur idx[%u] port[%u] phase[%u] err[%u]",
1103 : idx, conn[idx].port, conn[idx].phase, conn[idx].err);
1104 : }
1105 0 : return HCCL_E_TCP_CONNECT;
1106 : }
1107 :
1108 41 : return HCCL_SUCCESS;
1109 : }
1110 :
1111 0 : HcclResult hrtRaSocketAcceptCreditAdd(struct SocketListenInfoT conn[], u32 num, u32 creditLimit)
1112 : {
1113 0 : s32 ret = 0;
1114 0 : ret = DlRaFunction::GetInstance().dlRaSocketAcceptCreditAdd(conn, num, creditLimit);
1115 0 : CHK_PRT_RET(ret != 0, HCCL_ERROR("socket accept credit add failed, ret[%d], port[%u], creditLimit[%d]",
1116 : ret, conn[0].port, creditLimit), HCCL_E_TCP_CONNECT);
1117 0 : return HCCL_SUCCESS;
1118 : }
1119 :
1120 41 : HcclResult hrtRaSocketListenStart(struct SocketListenInfoT conn[], u32 num)
1121 : {
1122 41 : s32 ret = 0;
1123 41 : auto startTime = chrono::steady_clock::now();
1124 41 : auto timeout = chrono::seconds(GetExternalInputHcclLinkTimeOut());
1125 41 : CHK_PRT_RET(num == 0, HCCL_ERROR("[ListenStart][RaSocket] num is zero"), HCCL_E_PARA);
1126 : while (true) {
1127 41 : ret = hrtRaSocketNonBlockListenStart(conn, num);
1128 41 : if (ret == 0) {
1129 41 : break; // 成功跳出
1130 0 : } else if (ret == HCCL_E_AGAIN) {
1131 0 : bool bTimeout = ((chrono::steady_clock::now() - startTime) >= timeout);
1132 0 : RPT_CALL_ERR(bTimeout, "ra socket listen failed. timeout[%d s], return[%d], num[%u]",
1133 : GetExternalInputHcclLinkTimeOut(), ret, num);
1134 :
1135 0 : CHK_PRT_RET(bTimeout, HCCL_ERROR("[ListenStart][RaSocket]errNo[0x%016llx] ra socket listen start "
1136 : "timeout[%d s]. return[%d]", HCCL_ERROR_CODE(HCCL_E_TIMEOUT),
1137 : GetExternalInputHcclLinkTimeOut(), ret), HCCL_E_TIMEOUT);
1138 0 : SaluSleep(ONE_MILLISECOND_OF_USLEEP);
1139 0 : } else if (ret == HCCL_E_UNAVAIL) {
1140 0 : return HCCL_E_UNAVAIL;
1141 : } else {
1142 0 : HCCL_ERROR("[hrtRaSocketListenStart]ra socket listen start fail, ret[%d]", ret);
1143 0 : return HCCL_E_TCP_CONNECT;
1144 : }
1145 0 : }
1146 41 : return HCCL_SUCCESS;
1147 : }
1148 :
1149 39 : HcclResult hrtRaSocketListenStop(struct SocketListenInfoT conn[], u32 num)
1150 : {
1151 39 : s32 ret = 0;
1152 39 : auto startTime = chrono::steady_clock::now();
1153 39 : auto timeout = chrono::seconds(GetExternalInputHcclLinkTimeOut());
1154 39 : CheckConnPort(conn, num);
1155 : while (true) {
1156 39 : ret = DlRaFunction::GetInstance().dlRaSocketListenStop(conn, num);
1157 39 : if (!ret || ret == SOCK_ENODEV) {
1158 : break; // 成功跳出
1159 0 : } else if (ret == SOCK_EAGAIN) {
1160 0 : bool bTimeout = ((chrono::steady_clock::now() - startTime) >= timeout);
1161 0 : if (!bTimeout) {
1162 0 : SaluSleep(ONE_MILLISECOND_OF_USLEEP);
1163 0 : continue;
1164 : }
1165 0 : HCCL_ERROR("[ListenStop][RaSocket]errNo[0x%016llx] ra socket listen stop fail timeout[%d]s, ret[%d], num[%u]",
1166 : HCCL_ERROR_CODE(HCCL_E_TIMEOUT), timeout, ret, num);
1167 0 : for (u32 idx = 0; idx < num; idx++) {
1168 0 : HCCL_ERROR("cur idx[%u] port[%u] phase[%u] err[%u]", idx, conn[idx].port, conn[idx].phase, conn[idx].err);
1169 : }
1170 0 : return HCCL_E_TIMEOUT;
1171 : } else {
1172 0 : HCCL_ERROR("[ListenStop][RaSocket]errNo[0x%016llx] ra socket listen stop fail. return[%d], num[%u]",\
1173 : HCCL_ERROR_CODE(HCCL_E_TCP_CONNECT), ret, num);
1174 0 : for (u32 idx = 0; idx < num; idx++) {
1175 0 : HCCL_ERROR("cur idx[%u] port[%u] phase[%u] err[%u]",
1176 : idx, conn[idx].port, conn[idx].phase, conn[idx].err);
1177 : }
1178 0 : return HCCL_E_TCP_CONNECT; // 非ra限速场景错误，不轮询，直接退出
1179 : }
1180 0 : }
1181 39 : return HCCL_SUCCESS;
1182 : }
1183 :
1184 1 : HcclResult hrtRaSocketNonBlockBatchAbort(SocketConnectInfoT conn[], u32 num)
1185 : {
1186 1 : CheckConnPort(conn, num);
1187 1 : s32 ret = DlRaFunction::GetInstance().dlRaSocketBatchAbort(conn, num);
1188 1 : if (ret == 0) {
1189 1 : return HCCL_SUCCESS;
1190 0 : } else if (ret == SOCK_EAGAIN) {
1191 0 : return HCCL_E_AGAIN;
1192 : } else {
1193 0 : HCCL_ERROR("[hrtRaSocketNonBlockBatchAbort]errNo[0x%016llx] ra socket batch abort fail. "\
1194 : "return[%d], num[%u]", HCCL_ERROR_CODE(HCCL_E_TCP_CONNECT), ret, num);
1195 0 : for (u32 idx = 0; idx < num; idx++) {
1196 0 : HCCL_ERROR("cur idx[%u] remoteIp[%u] port[%u] tag[%s]",
1197 : idx, conn[idx].remoteIp.addr.s_addr, conn[idx].port, conn[idx].tag);
1198 : }
1199 0 : return HCCL_E_TCP_CONNECT;
1200 : }
1201 :
1202 : return HCCL_SUCCESS;
1203 : }
1204 :
1205 1 : HcclResult IsSupportRaSocketAbort(bool& isSupportRaSocketAbort)
1206 : {
1207 1 : isSupportRaSocketAbort = false;
1208 1 : s32 deviceLogicID = -1;
1209 1 : u32 devicePhyId = 0;
1210 1 : CHK_RET(hrtGetDevice(&deviceLogicID));
1211 1 : CHK_RET(hrtGetDevicePhyIdByIndex(static_cast<u32>(deviceLogicID), devicePhyId));
1212 1 : u32 configVersion = 0;
1213 :
1214 : // 获取版本号查看是否兼容
1215 1 : HcclResult ret = hrtRaGetInterfaceVersion(devicePhyId, SOCKET_ABORT, &configVersion);
1216 1 : CHK_PRT_RET(ret == HCCL_E_NETWORK, HCCL_ERROR("[IsSupportRaSendNormalWrlist]hrtRaGetInterfaceVersion "\
1217 : "failed, interface[%u]", SOCKET_ABORT), ret);
1218 1 : if (ret == HCCL_E_NOT_SUPPORT) {
1219 0 : HCCL_WARNING("this package does not support hrtRaGetInterfaceVersion, please change new package");
1220 0 : return HCCL_SUCCESS;
1221 : }
1222 :
1223 1 : if (configVersion >= SOCKET_ABORT_VERSION) {
1224 1 : isSupportRaSocketAbort = true;
1225 : }
1226 1 : HCCL_INFO("isSupportRaSocketAbort support:%d, configVersion:%d", isSupportRaSocketAbort, configVersion);
1227 1 : return HCCL_SUCCESS;
1228 : }
1229 :
1230 0 : HcclResult hrtRaSocketNonBlockBatchConnect(SocketConnectInfoT conn[], u32 num)
1231 : {
1232 0 : CheckConnPort(conn, num);
1233 0 : s32 ret = DlRaFunction::GetInstance().dlRaSocketBatchConnect(conn, num);
1234 0 : if (ret == 0) {
1235 0 : return HCCL_SUCCESS;
1236 0 : } else if (ret == SOCK_EAGAIN) {
1237 0 : return HCCL_E_AGAIN;
1238 : } else {
1239 0 : HCCL_ERROR("[HrtRaQpNonBlockConnectAsync]errNo[0x%016llx] ra socket batch connect fail. "\
1240 : "return[%d], num[%u]", HCCL_ERROR_CODE(HCCL_E_TCP_CONNECT), ret, num);
1241 0 : for (u32 idx = 0; idx < num; idx++) {
1242 0 : HCCL_ERROR("cur idx[%u] remoteIp[%u] port[%u] tag[%s]",
1243 : idx, conn[idx].remoteIp.addr.s_addr, conn[idx].port, conn[idx].tag);
1244 : }
1245 0 : return HCCL_E_TCP_CONNECT;
1246 : }
1247 :
1248 : return HCCL_SUCCESS;
1249 : }
1250 :
1251 7 : HcclResult SocketBatchConnect(SocketConnectInfoT conn[], u32 num, std::function<bool()> needStop)
1252 : {
1253 7 : s32 ret = 0;
1254 7 : auto startTime = chrono::steady_clock::now();
1255 7 : auto timeout = chrono::seconds(GetExternalInputHcclLinkTimeOut());
1256 7 : CheckConnPort(conn, num);
1257 : while (true) {
1258 7 : CHK_PRT_RET(needStop(), HCCL_ERROR("Terminating operation due to external request"), HCCL_E_INTERNAL);
1259 :
1260 7 : ret = DlRaFunction::GetInstance().dlRaSocketBatchConnect(conn, num);
1261 7 : if (!ret) {
1262 7 : break; // 成功跳出
1263 0 : } else if (ret == SOCK_EAGAIN) {
1264 0 : bool bTimeout = ((chrono::steady_clock::now() - startTime) >= timeout);
1265 0 : RPT_CALL_ERR(bTimeout, "ra socket batch connect failed. timeout[%d s], return[%d]",
1266 : GetExternalInputHcclLinkTimeOut(), ret);
1267 0 : CHK_PRT_RET(bTimeout, HCCL_ERROR("[BatchConnect][RaSocket]errNo[0x%016llx] ra socket batch connect "\
1268 : "timeout[%lld s]. return[%d]", HCCL_ERROR_CODE(HCCL_E_TIMEOUT),\
1269 : GetExternalInputHcclLinkTimeOut(), ret), HCCL_E_TIMEOUT);
1270 0 : SaluSleep(ONE_MILLISECOND_OF_USLEEP);
1271 : } else {
1272 0 : RPT_CALL_ERR_PRT("ra socket batch connect failed. return[%d]", ret);
1273 0 : HCCL_ERROR("[BatchConnect][RaSocket]errNo[0x%016llx] ra socket batch connect fail. return[%d], params: ",\
1274 : HCCL_ERROR_CODE(HCCL_E_TCP_CONNECT), ret);
1275 0 : return HCCL_E_TCP_CONNECT; // 非ra限速场景错误，不轮询，直接退出
1276 : }
1277 0 : }
1278 7 : return HCCL_SUCCESS;
1279 : }
1280 :
1281 7 : HcclResult hrtRaSocketBatchConnect(struct SocketConnectInfoT conn[], u32 num, u32 maxLen, std::function<bool()> needStop)
1282 : {
1283 7 : CHK_PTR_NULL(conn);
1284 7 : CHK_PRT_RET((num > maxLen) || (num == 0), HCCL_ERROR("[hrtRaSocketBatchConnect][RaSocket]ra socket batch connect "\
1285 : "para error, num[%u], maxLen[%u]", num, maxLen), HCCL_E_PARA);
1286 :
1287 7 : HCCL_INFO("batch connect, port[%u], remoteip[%x]", conn[0].port, conn[0].remoteIp);
1288 : // batchConnect函数指针。底层接口一次最多建链16条，超过16条调用多次batch connect
1289 7 : u32 exeNum = 0;
1290 7 : SocketConnectInfoT *connBase = conn;
1291 14 : while (num > 0) {
1292 7 : exeNum = num > MAX_NUM_OF_BATCH_CONN ? MAX_NUM_OF_BATCH_CONN : num;
1293 7 : CHK_RET(SocketBatchConnect(connBase, exeNum, needStop));
1294 7 : connBase += exeNum;
1295 7 : num -= exeNum;
1296 : }
1297 :
1298 7 : return HCCL_SUCCESS;
1299 : }
1300 :
1301 11 : HcclResult hrtRaSocketBatchClose(struct SocketCloseInfoT conn[], u32 num, u32 maxLen)
1302 : {
1303 11 : CHK_PTR_NULL(conn);
1304 11 : HCCL_INFO("ra socket batch close fdhandle[%p]", conn->fdHandle);
1305 11 : CHK_PRT_RET((num > maxLen) || (num == 0), HCCL_ERROR("[BatchClose][RaSocket]ra socket batch connect para error "\
1306 : "num[%u], maxLen[%u]", num, maxLen), HCCL_E_PARA);
1307 11 : s32 ret = 0;
1308 11 : auto startTime = chrono::steady_clock::now();
1309 11 : auto timeout = chrono::seconds(GetExternalInputHcclLinkTimeOut());
1310 : while (true) {
1311 11 : ret = DlRaFunction::GetInstance().dlRaSocketBatchClose(conn, num);
1312 11 : if (!ret) {
1313 11 : break; // 成功跳出
1314 0 : } else if (ret == SOCK_EAGAIN) {
1315 0 : bool bTimeout = ((chrono::steady_clock::now() - startTime) >= timeout);
1316 0 : if (!bTimeout) {
1317 0 : SaluSleep(ONE_MILLISECOND_OF_USLEEP);
1318 0 : continue;
1319 : }
1320 0 : HCCL_ERROR("[BatchClose][RaSocket]errNo[0x%016llx] ra socket batch close timeout[%d s], ret[%d], num[%u]",
1321 : HCCL_ERROR_CODE(HCCL_E_TIMEOUT), timeout, ret, num);
1322 0 : for (u32 idx = 0; idx < num; idx++) {
1323 0 : HCCL_ERROR("cur idx[%u] disuseLinger[%d]", idx, conn[idx].disuseLinger);
1324 : }
1325 0 : return HCCL_E_TIMEOUT;
1326 : } else {
1327 0 : HCCL_ERROR("[BatchClose][RaSocket]errNo[0x%016llx] ra socket batch close fail. return[%d], num[%u]",\
1328 : HCCL_ERROR_CODE(HCCL_E_TCP_CONNECT), ret, num);
1329 0 : for (u32 idx = 0; idx < num; idx++) {
1330 0 : HCCL_ERROR("cur idx[%u] disuseLinger[%d]", idx, conn[idx].disuseLinger);
1331 : }
1332 0 : return HCCL_E_TCP_CONNECT; // 非ra限速场景错误，不轮询，直接退出
1333 : }
1334 0 : }
1335 11 : HCCL_INFO("ra socket batch close success,take time [%lld]us",
1336 : std::chrono::duration_cast<std::chrono::microseconds>(chrono::steady_clock::now() - startTime));
1337 11 : return HCCL_SUCCESS;
1338 : }
1339 :
1340 41 : s32 hrtRaGetSockets(u32 role, struct SocketInfoT conn[], u32 num, u32 *connectedNum)
1341 : {
1342 41 : return DlRaFunction::GetInstance().dlRaGetSockets(role, conn, num, connectedNum);
1343 : }
1344 :
1345 0 : HcclResult hrtRaNonBlockGetSockets(u32 role, struct SocketInfoT conn[], u32 num, u32 *connectedNum)
1346 : {
1347 0 : CHK_PTR_NULL(conn);
1348 0 : CHK_PRT_RET(num == 0, HCCL_ERROR("[hrtRaBlockGetSockets]ra get rasocket para error, num[%d]", num), HCCL_E_PARA);
1349 0 : s32 ret = DlRaFunction::GetInstance().dlRaGetSockets(role, conn, num, connectedNum);
1350 0 : if (ret == 0) {
1351 0 : return HCCL_SUCCESS;
1352 0 : } else if (ret == SOCK_EAGAIN) {
1353 0 : return HCCL_E_AGAIN;
1354 : } else {
1355 0 : HCCL_ERROR("[hrtRaNonBlockGetSockets]get ra socket error. role[%u], num[%u], ret[%d], connected num[%u]", \
1356 : role, num, ret, *connectedNum);
1357 0 : for (u32 idx = 0; idx < num; idx++) {
1358 0 : HCCL_ERROR("cur idx[%u] socketHandle[%u] s_addr[%u] tag[%s]", idx, conn[idx].socketHandle,
1359 : conn[idx].remoteIp.addr.s_addr, conn[idx].tag);
1360 : }
1361 0 : return HCCL_E_TCP_CONNECT;
1362 : }
1363 :
1364 : return HCCL_SUCCESS;
1365 : }
1366 :
1367 0 : HcclResult hrtRaBlockGetSockets(u32 role, struct SocketInfoT conn[], u32 num)
1368 : {
1369 0 : CHK_PTR_NULL(conn);
1370 0 : CHK_PRT_RET(num == 0, HCCL_ERROR("[hrtRaBlockGetSockets]ra get rasocket para error"), HCCL_E_PARA);
1371 : s32 sockRet;
1372 0 : u32 gotSocketsCnt = 0;
1373 0 : auto startTime = chrono::steady_clock::now();
1374 0 : auto timeout = chrono::seconds(GetExternalInputHcclLinkTimeOut());
1375 : while (true) {
1376 0 : if ((chrono::steady_clock::now() - startTime) >= timeout) {
1377 0 : HCCL_ERROR("[hrtRaBlockGetSockets] get rasocket timeout role[%u], num[%u], goten[%u], "\
1378 : "timeout[%lld s], the HCCL_CONNECT_TIMEOUT may be insufficient.", role, num, gotSocketsCnt, timeout);
1379 0 : return HCCL_E_TIMEOUT;
1380 : }
1381 0 : u32 connectedNum = 0;
1382 0 : sockRet = hrtRaGetSockets(role, conn, num, &connectedNum);
1383 0 : if ((connectedNum == 0 && sockRet == 0) || (sockRet == SOCK_EAGAIN)) {
1384 0 : SaluSleep(ONE_MILLISECOND_OF_USLEEP);
1385 0 : } else if (sockRet != 0) {
1386 0 : HCCL_ERROR("[Get][RaSocket]get rasocket error. role[%u], num[%u], sockRet[%d], connectednum[%u]", \
1387 : role, num, sockRet, connectedNum);
1388 0 : return HCCL_E_TCP_CONNECT;
1389 : } else {
1390 0 : gotSocketsCnt += connectedNum;
1391 0 : if (gotSocketsCnt == num) {
1392 0 : HCCL_INFO("block get sockets success, socket num[%u]", gotSocketsCnt);
1393 0 : break;
1394 0 : } else if (gotSocketsCnt > num) {
1395 0 : HCCL_ERROR("[Get][RaSocket]total Sockets[%u], more than needed num[%u]!", gotSocketsCnt, num);
1396 0 : return HCCL_E_TCP_CONNECT;
1397 : } else {
1398 0 : SaluSleep(ONE_MILLISECOND_OF_USLEEP);
1399 : }
1400 : }
1401 0 : }
1402 0 : return HCCL_SUCCESS;
1403 : }
1404 :
1405 :
1406 0 : HcclResult hrtRaSocketNonBlockSendHeterog(const FdHandle fdHandle, const void *data, u64 size, u64 *sentSize)
1407 : {
1408 0 : if (size > SOCKET_SEND_MAX_SIZE) {
1409 0 : HCCL_ERROR("[hrtRaSocketNonBlockSend]errNo[0x%016llx] ra socket send size is too large, " \
1410 : "data[%p], size[%llu Byte]", HCCL_ERROR_CODE(HCCL_E_NETWORK), data, size);
1411 0 : return HCCL_E_PARA;
1412 : }
1413 0 : s32 ret = DlRaFunction::GetInstance().dlRaSocketSend(fdHandle, data, size, sentSize);
1414 0 : if (ret == 0) {
1415 0 : return HCCL_SUCCESS;
1416 0 : } else if (ret == SOCK_EAGAIN) {
1417 0 : return HCCL_E_AGAIN;
1418 : } else {
1419 0 : HCCL_RUN_INFO("[hrtRaSocketNonBlockSend]ra socket send failed, data[%p], size[%llu Byte], "\
1420 : "sent[%llu Byte], ret[%d]", data, size, *sentSize, ret);
1421 0 : return HCCL_E_NETWORK;
1422 : }
1423 :
1424 : return HCCL_SUCCESS;
1425 : }
1426 :
1427 0 : s32 hrtRaSocketNonBlockSend(const FdHandle fdHandle, const void *data, u64 size, u64 *sentSize)
1428 : {
1429 0 : return DlRaFunction::GetInstance().dlRaSocketSend(fdHandle, data, size, sentSize);
1430 : }
1431 :
1432 0 : HcclResult hrtRaSocketNonBlockSendHeart(const FdHandle fdHandle, const void *data, u64 size, u64 *sentSize)
1433 : {
1434 0 : if (size > SOCKET_SEND_MAX_SIZE) {
1435 0 : HCCL_ERROR("[hrtRaSocketNonBlockSend]errNo[0x%016llx] ra socket send size is too large, " \
1436 : "data[%p], size[%llu]", HCCL_ERROR_CODE(HCCL_E_NETWORK), data, size);
1437 0 : return HCCL_E_PARA;
1438 : }
1439 0 : s32 ret = DlRaFunction::GetInstance().dlRaSocketSend(fdHandle, data, size, sentSize);
1440 0 : if (ret == 0) {
1441 0 : return HCCL_SUCCESS;
1442 0 : } else if (ret == SOCK_EAGAIN) {
1443 0 : return HCCL_E_AGAIN;
1444 0 : } else if (ret == SOCK_CLOSE) {
1445 0 : return HCCL_E_INTERNAL; // 暂时用这个错误表示hccp进程异常退出
1446 : } else {
1447 0 : HCCL_WARNING("[hrtRaSocketNonBlockSend]ra socket send failed, fdHandle[%p], data[%p], size[%llu], "\
1448 : "sent[%llu], ret[%d], errno[%d][%s]", fdHandle, data, size, *sentSize, ret, errno, strerror(errno));
1449 0 : return HCCL_E_NETWORK;
1450 : }
1451 :
1452 : return HCCL_SUCCESS;
1453 : }
1454 :
1455 10 : HcclResult hrtRaSocketBlockSend(const FdHandle fdHandle, const void *data, u64 sendSize, std::function<bool()> needStop)
1456 : {
1457 10 : CHK_PTR_NULL(data);
1458 10 : if (sendSize > SOCKET_SEND_MAX_SIZE) {
1459 0 : HCCL_ERROR("[Send][RaSocket]errNo[0x%016llx] ra socket send size is too large, " \
1460 : "data[%p], size[%llu Byte]", HCCL_ERROR_CODE(HCCL_E_NETWORK), data, sendSize);
1461 0 : return HCCL_E_PARA;
1462 : }
1463 10 : s64 ret = 0;
1464 10 : void *sendData = const_cast<void *>(data);
1465 : const chrono::seconds timeout = chrono::seconds(
1466 10 : GetExternalInputHcclLinkTimeOut());
1467 10 : const auto start = chrono::steady_clock::now();
1468 10 : u64 totalSentSize = 0;
1469 10 : u64 sentSize = 0;
1470 :
1471 10 : HCCL_DEBUG("before ra socket send, para: data[%p], size[%llu Byte]", sendData, sendSize);
1472 :
1473 : while (true) {
1474 10 : CHK_PRT_RET(needStop(), HCCL_ERROR("Terminating operation due to external request"), HCCL_E_INTERNAL);
1475 :
1476 : // 底层ra_socket_send host网卡无限制，device网卡由于HDC通道限制的限制有大小限制(目前大小为64KB)
1477 10 : ret = DlRaFunction::GetInstance().dlRaSocketSend(fdHandle,
1478 10 : reinterpret_cast<void *>(reinterpret_cast<uintptr_t>(sendData) + totalSentSize),
1479 : sendSize - totalSentSize, &sentSize);
1480 10 : HCCL_DEBUG("ra socket send, data[%p], size[%llu Byte] send size[%llu Byte]", sendData, sendSize, totalSentSize);
1481 10 : if (ret == 0) {
1482 10 : totalSentSize += sentSize;
1483 10 : if (totalSentSize == sendSize) { // 只有完全发送完才返回成功
1484 10 : break;
1485 : }
1486 :
1487 0 : CHK_PRT_RET((totalSentSize > sendSize),
1488 : HCCL_ERROR("[Send][RaSocket]errNo[0x%016llx] ra socket send failed, " \
1489 : "data[%p], size[%llu Byte], retSize[%llu Byte]", HCCL_ERROR_CODE(HCCL_E_NETWORK),
1490 : data, sendSize, sentSize), HCCL_E_NETWORK);
1491 0 : SaluSleep(ONE_HUNDRED_MICROSECOND_OF_USLEEP);
1492 0 : } else if (ret == SOCK_EAGAIN) {
1493 : /* ra速率限制 retry */
1494 0 : SaluSleep(ONE_MILLISECOND_OF_USLEEP);
1495 : } else {
1496 0 : HCCL_ERROR("[Send][RaSocket]errNo[0x%016llx] ra socket send failed, data[%p], size[%llu], "\
1497 : "sent[%llu Byte], ret[%d]", HCCL_ERROR_CODE(HCCL_E_NETWORK), data, sendSize, sentSize, ret);
1498 0 : return HCCL_E_NETWORK;
1499 : }
1500 :
1501 : /* 获取当前时间，如果耗时超过timeout，则返回错误 */
1502 : const auto elapsed =
1503 0 : chrono::duration_cast<chrono::seconds>(chrono::steady_clock::now() - start);
1504 0 : if (elapsed > timeout) {
1505 0 : HCCL_ERROR("[Send][RaSocket]errNo[0x%016llx] Wait timeout for sockets send, data[%p], "\
1506 : "size[%llu Byte], sentsize[%llu Byte]", HCCL_ERROR_CODE(HCCL_E_NETWORK), data, sendSize, sentSize);
1507 0 : return HCCL_E_TIMEOUT;
1508 : }
1509 0 : }
1510 10 : HCCL_DEBUG("ra socket send finished.");
1511 10 : return HCCL_SUCCESS;
1512 : }
1513 :
1514 0 : s32 hrtRaSocketRecv(const FdHandle fdHandle, void *data, u64 size, u64 *recvSize)
1515 : {
1516 0 : return DlRaFunction::GetInstance().dlRaSocketRecv(fdHandle, data, size, recvSize);
1517 : }
1518 :
1519 0 : HcclResult hrtRaSocketNonBlockRecvHeterog(const FdHandle fdHandle, void *data, u64 size, u64 *recvSize)
1520 : {
1521 0 : s32 ret = DlRaFunction::GetInstance().dlRaSocketRecv(fdHandle, data, size, recvSize);
1522 0 : if (ret == 0) {
1523 0 : return HCCL_SUCCESS;
1524 0 : } else if (ret == SOCK_EAGAIN) {
1525 0 : return HCCL_E_AGAIN;
1526 : } else {
1527 0 : HCCL_RUN_INFO("[hrtRaSocketNonBlockRecv]ra socket recv failed, data[%p], size[%llu Byte], "\
1528 : "recv[%llu Byte], ret[%d], errno[%d][%s]", data, size, recvSize, ret, errno, strerror(errno));
1529 0 : return HCCL_E_TCP_TRANSFER;
1530 : }
1531 :
1532 : return HCCL_SUCCESS;
1533 : }
1534 :
1535 0 : s32 hrtRaSocketNonBlockRecv(const FdHandle fdHandle, void *data, u64 size, u64 *recvSize)
1536 : {
1537 0 : return DlRaFunction::GetInstance().dlRaSocketRecv(fdHandle, data, size, recvSize);;
1538 : }
1539 :
1540 0 : HcclResult hrtRaSocketNonBlockRecvHeart(const FdHandle fdHandle, void *data, u64 size, u64 *recvSize)
1541 : {
1542 0 : s32 ret = DlRaFunction::GetInstance().dlRaSocketRecv(fdHandle, data, size, recvSize);
1543 0 : if (ret == 0) {
1544 0 : return HCCL_SUCCESS;
1545 0 : } else if (ret == SOCK_EAGAIN) {
1546 0 : return HCCL_E_AGAIN;
1547 0 : } else if (ret == SOCK_CLOSE) {
1548 0 : return HCCL_E_INTERNAL; //暂时用这个错误码表示hccp进程异常退出
1549 : } else {
1550 0 : HCCL_WARNING("[hrtRaSocketNonBlockRecvHeart]ra socket recv failed, data[%p], size[%llu], "\
1551 : "recv[%llu], ret[%d], errno[%d][%s]", data, size, recvSize, ret, errno, strerror(errno));
1552 0 : return HCCL_E_TCP_TRANSFER;
1553 : }
1554 : return HCCL_SUCCESS;
1555 : }
1556 :
1557 9 : HcclResult hrtRaSocketBlockRecv(const FdHandle fdHandle, void *data, u64 size, std::function<bool()> needStop, u32 timeout)
1558 : {
1559 9 : auto startTime = chrono::steady_clock::now();
1560 9 : void *recvData = const_cast<void *>(data);
1561 9 : u64 recvSize = 0;
1562 9 : s32 rtRet = 0;
1563 9 : u64 getedLen = 0;
1564 : const chrono::seconds timeoutSec = chrono::seconds(
1565 9 : timeout > 0 ? timeout : GetExternalInputHcclLinkTimeOut());
1566 :
1567 9 : HCCL_DEBUG("before ra socket recv, para: data[%p], size[%llu]", recvData, size);
1568 : while (true) {
1569 9 : CHK_PRT_RET(needStop(), HCCL_ERROR("Terminating operation due to external request"), HCCL_E_INTERNAL);
1570 :
1571 9 : if ((chrono::steady_clock::now() - startTime) >= timeoutSec) {
1572 0 : HCCL_ERROR("[Recv][RaSocket]errNo[0x%016llx] Wait timeout for sockets recv, data[%p], "\
1573 : "size[%llu Byte], recvSize[%llu Byte] timeout[%lld s]. Peerrank did not send the data in time. " \
1574 : "Check whether the peerrank is abnormal.", \
1575 : HCCL_ERROR_CODE(HCCL_E_NETWORK), data, size, recvSize, timeoutSec);
1576 0 : return HCCL_E_TIMEOUT;
1577 : }
1578 9 : rtRet = DlRaFunction::GetInstance().dlRaSocketRecv(fdHandle,
1579 9 : reinterpret_cast<void *>(reinterpret_cast<uintptr_t>(recvData) + getedLen), size - getedLen, &recvSize);
1580 9 : if ((rtRet == 0) && (recvSize > 0)) { // 接收完成，也有可能要多次接收
1581 9 : getedLen += recvSize;
1582 9 : CHK_PRT_RET(getedLen > size, HCCL_ERROR("[Recv][RaSocket]errNo[0x%016llx] socket receive "\
1583 : "rtSize[%llu Byte] bigger size[%zu Byte]", HCCL_ERROR_CODE(HCCL_E_TCP_TRANSFER), getedLen, size),
1584 : HCCL_E_TCP_TRANSFER);
1585 9 : if (getedLen == size) {
1586 9 : break;
1587 : }
1588 0 : } else if ((rtRet == 0) && (recvSize == 0)) {
1589 0 : HCCL_ERROR("[Recv][RaSocket]recv fail, bufLen[%llu], recLen[%llu]", size, recvSize);
1590 0 : return HCCL_E_TCP_TRANSFER;
1591 0 : } else if (rtRet == SOCK_EAGAIN) {
1592 : /* 尚未接收到数据,延时1ms */
1593 0 : SaluSleep(ONE_MILLISECOND_OF_USLEEP);
1594 0 : continue;
1595 0 : } else if (rtRet != 0) { // 等于0为连接关闭，小于0的其他场景为出错
1596 0 : HCCL_ERROR("[Recv][RaSocket]errNo[0x%016llx] recv fail, data[%p], size[%llu], rtRet[%d]",
1597 : HCCL_ERROR_CODE(HCCL_E_TCP_TRANSFER), data, size, rtRet);
1598 0 : return HCCL_E_TCP_TRANSFER;
1599 : }
1600 : }
1601 9 : HCCL_DEBUG("ra socket receive finished");
1602 9 : return HCCL_SUCCESS;
1603 : }
1604 :
1605 0 : HcclResult IsSupportHdcAsync(bool &isSupportHdcAsync)
1606 : {
1607 0 : isSupportHdcAsync = false;
1608 0 : s32 deviceLogicID = -1;
1609 0 : u32 devicePhyId = 0;
1610 0 : CHK_RET(hrtGetDevice(&deviceLogicID));
1611 0 : CHK_RET(hrtGetDevicePhyIdByIndex(static_cast<u32>(deviceLogicID), devicePhyId));
1612 0 : u32 version = 0;
1613 :
1614 : // 获取版本号查看是否兼容
1615 0 : HcclResult ret = hrtRaGetInterfaceVersion(devicePhyId, RS_INIT, &version);
1616 0 : CHK_PRT_RET(ret == HCCL_E_NETWORK, HCCL_ERROR("[IsSupportHdcAsync]hrtRaGetInterfaceVersion "\
1617 : "failed, interface[%u]", RS_INIT), ret);
1618 0 : if (ret == HCCL_E_NOT_SUPPORT) {
1619 0 : HCCL_WARNING("this package does not support hrtRaGetInterfaceVersion, please change new package");
1620 0 : return HCCL_SUCCESS;
1621 : }
1622 :
1623 0 : if (version >= RS_INIT_SUPPORT_ASYNC_VERSION) {
1624 0 : isSupportHdcAsync = true;
1625 : }
1626 :
1627 0 : HCCL_INFO("[IsSupportHdcAsync] isSupportHdcAsync[%d], version[%d]", isSupportHdcAsync, version);
1628 0 : return HCCL_SUCCESS;
1629 : }
1630 :
1631 3 : s32 hrtRaSocketSendAsync(const FdHandle fdHandle, const void *data, u64 size, u64 *sentSize, void **reqHandle)
1632 : {
1633 3 : if (DlRaFunction::GetInstance().dlRaSocketSendAsync == nullptr) {
1634 1 : HCCL_WARNING("this package does not support hrtRaSocketSendAsync, please change new package");
1635 1 : return OTHERS_ENOTSUPP;
1636 : }
1637 2 : return DlRaFunction::GetInstance().dlRaSocketSendAsync(fdHandle, data, size, sentSize, reqHandle);
1638 : }
1639 :
1640 3 : s32 hrtRaSocketRecvAsync(const FdHandle fdHandle, void *data, u64 size, u64 *receivedSize, void **reqHandle)
1641 : {
1642 3 : if (DlRaFunction::GetInstance().dlRaSocketRecvAsync == nullptr) {
1643 1 : HCCL_WARNING("this package does not support hrtRaSocketRecvAsync, please change new package");
1644 1 : return OTHERS_ENOTSUPP;
1645 : }
1646 2 : return DlRaFunction::GetInstance().dlRaSocketRecvAsync(fdHandle, data, size, receivedSize, reqHandle);
1647 : }
1648 :
1649 5 : s32 hrtRaSocketGetAsyncReqResult(void *reqHandle, s32 *reqResult)
1650 : {
1651 5 : if (DlRaFunction::GetInstance().dlRaGetAsyncReqResult == nullptr) {
1652 1 : HCCL_WARNING("this package does not support hrtRaSocketGetAsyncReqResult, please change new package");
1653 1 : return OTHERS_ENOTSUPP;
1654 : }
1655 4 : return DlRaFunction::GetInstance().dlRaGetAsyncReqResult(reqHandle, reqResult);
1656 : }
1657 :
1658 14 : HcclResult hrtGetHostIf(vector<pair<string, HcclIpAddress>> &hostIfs, u32 devPhyId)
1659 : {
1660 14 : struct RaGetIfattr config = {0};
1661 14 : config.phyId = devPhyId;
1662 14 : config.nicPosition = static_cast<u32>(NICDeployment::NIC_DEPLOYMENT_HOST);
1663 14 : config.isAll = false;
1664 :
1665 14 : u32 ifAddrNum = 0;
1666 14 : CHK_RET(hrtGetIfNum(config, ifAddrNum));
1667 14 : HCCL_RUN_INFO("[Get][HostIf]hrtGetIfNum success. ifAddrNum[%u].", ifAddrNum);
1668 14 : if (ifAddrNum == 0) {
1669 0 : HCCL_WARNING("[Get][HostIf]there is no valid host interface, ifAddrNum[%u].", ifAddrNum);
1670 0 : return HCCL_SUCCESS;
1671 : }
1672 :
1673 : struct InterfaceInfo *ifAddrInfos;
1674 14 : NEW_NOTHROW(ifAddrInfos, struct InterfaceInfo[ifAddrNum], return HCCL_E_MEMORY);
1675 14 : shared_ptr<struct InterfaceInfo> ifAddrInfoPtrs(ifAddrInfos, default_delete<struct InterfaceInfo[]>());
1676 :
1677 14 : s32 sRet = memset_s(ifAddrInfos, ifAddrNum * sizeof(InterfaceInfo), 0, ifAddrNum * sizeof(InterfaceInfo));
1678 14 : if (sRet != EOK) {
1679 0 : HCCL_ERROR("[Get][HostIf]errNo[0x%016llx] memoryset ifAddrInfos to 0 failed. params: "\
1680 : "dest[%p], dest_size[%zu Byte], count[%zu]", HCCL_ERROR_CODE(HCCL_E_SYSCALL), ifAddrInfos,
1681 : ifAddrNum * sizeof(InterfaceInfo), ifAddrNum * sizeof(InterfaceInfo));
1682 0 : return HCCL_E_SYSCALL;
1683 : }
1684 14 : CHK_RET(hrtGetIfAddress(config, ifAddrInfos, ifAddrNum));
1685 :
1686 70 : for (u32 i = 0; i < ifAddrNum; i++) {
1687 : HcclInAddr temp;
1688 56 : temp.addr = ifAddrInfos[i].ifaddr.ip.addr;
1689 56 : temp.addr6 = ifAddrInfos[i].ifaddr.ip.addr6;
1690 56 : HcclIpAddress ipInfo(ifAddrInfos[i].family, temp);
1691 56 : CHK_PRT_RET(ipInfo.IsInvalid(), HCCL_ERROR("ip is invalid."), HCCL_E_PARA);
1692 112 : CHK_RET(ipInfo.SetIfName(ifAddrInfos[i].ifname));
1693 56 : CHK_RET(ipInfo.SetScopeID(ifAddrInfos[i].scopeId));
1694 56 : hostIfs.push_back({ifAddrInfos[i].ifname, ipInfo});
1695 56 : HCCL_INFO("[Get][HostIf]hrtGetIfAddress: idx[%u] ifname[%s] ip[%s]",
1696 : i, ifAddrInfos[i].ifname, ipInfo.GetReadableAddress());
1697 56 : }
1698 :
1699 14 : return HCCL_SUCCESS;
1700 14 : }
1701 :
1702 0 : HcclResult hrtEpollCtlAdd(const FdHandle fdHandle, RaEpollEvent event)
1703 : {
1704 0 : s32 ret = DlRaFunction::GetInstance().dlRaEpollCtlAdd(fdHandle, event);
1705 0 : CHK_PRT_RET(ret != 0, HCCL_ERROR("[Add][EpollCtl] failed"), HCCL_E_NETWORK);
1706 0 : return HCCL_SUCCESS;
1707 : }
1708 :
1709 0 : HcclResult hrtEpollCtlMod(const FdHandle fdHandle, RaEpollEvent event)
1710 : {
1711 0 : s32 ret = DlRaFunction::GetInstance().dlRaEpollCtlMod(fdHandle, event);
1712 0 : CHK_PRT_RET(ret != 0, HCCL_ERROR("[Mod][EpollCtl] failed"), HCCL_E_NETWORK);
1713 0 : return HCCL_SUCCESS;
1714 : }
1715 :
1716 0 : HcclResult hrtEpollCtlDel(const FdHandle fdHandle)
1717 : {
1718 0 : s32 ret = DlRaFunction::GetInstance().dlRaEpollCtlDel(fdHandle);
1719 0 : CHK_PRT_RET(ret != 0, HCCL_ERROR("[Del][EpollCtl] failed"), HCCL_E_NETWORK);
1720 0 : return HCCL_SUCCESS;
1721 : }
1722 :
1723 0 : HcclResult hrtSetRecvDataCallback(const SocketHandle socketHandle, const void *callback)
1724 : {
1725 0 : s32 ret = DlRaFunction::GetInstance().dlRaSetRecvDataCallback(socketHandle, callback);
1726 0 : CHK_PRT_RET(ret != 0, HCCL_ERROR("[Set][RecvDataCallback] failed"), HCCL_E_NETWORK);
1727 0 : return HCCL_SUCCESS;
1728 : }
1729 : #endif
1730 :
1731 : #if T_DESC("WhiteList", true)
1732 :
1733 18 : HcclResult hrtRaSocketSetWhiteListStatus(u32 enable)
1734 : {
1735 18 : s32 ret = DlRaFunction::GetInstance().dlRaSocketSetWhiteListStatus(enable);
1736 18 : CHK_PRT_RET(ret != 0,
1737 : HCCL_ERROR("[Set][WhiteListStatus]errNo[0x%016llx] ra socket set white list fail, return[%d]." \
1738 : " para: enable[%u]", HCCL_ERROR_CODE(HCCL_E_TCP_CONNECT), ret, enable), HCCL_E_TCP_CONNECT);
1739 18 : HCCL_INFO("set host socket whitelist status[%u] success.", enable);
1740 18 : return HCCL_SUCCESS;
1741 : }
1742 :
1743 0 : HcclResult hrtRaSocketGetWhiteListStatus(u32 &enable)
1744 : {
1745 0 : s32 ret = DlRaFunction::GetInstance().dlRaSocketGetWhiteListStatus(&enable);
1746 0 : CHK_PRT_RET(ret != 0,
1747 : HCCL_ERROR("[Get][WhiteListStatus]errNo[0x%016llx] ra socket get white list fail, return[%d].",
1748 : HCCL_ERROR_CODE(HCCL_E_TCP_CONNECT), ret), HCCL_E_TCP_CONNECT);
1749 0 : return HCCL_SUCCESS;
1750 : }
1751 :
1752 3 : HcclResult hrtRaSocketWhiteListAdd(SocketHandle socketHandle, struct SocketWlistInfoT whiteList[], u32 num)
1753 : {
1754 3 : HCCL_INFO("add white list: num[%u].", num);
1755 6 : for (u32 i = 0; i < num; i++) {
1756 3 : HCCL_DEBUG("add white list: idx[%u], remoteIp[%u], tag[%s].", i, whiteList[i].remoteIp.addr.s_addr,
1757 : whiteList[i].tag);
1758 3 : s32 ret = DlRaFunction::GetInstance().dlRaSocketWhiteListAdd(socketHandle, whiteList + i, 1);
1759 3 : CHK_PRT_RET(ret != 0,
1760 : HCCL_ERROR("[Add][RaSocketWhiteList]errNo[0x%016llx] ra white list add fail, return[%d].",\
1761 : HCCL_ERROR_CODE(HCCL_E_TCP_CONNECT), ret), HCCL_E_TCP_CONNECT);
1762 : }
1763 :
1764 3 : return HCCL_SUCCESS;
1765 : }
1766 :
1767 1 : HcclResult hrtRaSocketWhiteListDel(SocketHandle socketHandle, struct SocketWlistInfoT whiteList[], u32 num)
1768 : {
1769 1 : HCCL_DEBUG("delete white list: num[%u].", num);
1770 2 : for (u32 i = 0; i < num; i++) {
1771 1 : HCCL_DEBUG("del white list: idx[%u], remoteIp[%u], tag[%s].", i, whiteList[i].remoteIp.addr.s_addr,
1772 : whiteList[i].tag);
1773 1 : s32 ret = DlRaFunction::GetInstance().dlRaSocketWhiteListDel(socketHandle, whiteList + i, 1);
1774 1 : CHK_PRT_RET(ret != 0,
1775 : HCCL_ERROR("[Del][RaSocketWhiteList]errNo[0x%016llx] ra white list del fail, return[%d].",\
1776 : HCCL_ERROR_CODE(HCCL_E_TCP_CONNECT), ret), HCCL_E_TCP_CONNECT);
1777 : }
1778 :
1779 1 : return HCCL_SUCCESS;
1780 : }
1781 :
1782 : #endif
1783 :
1784 80 : HcclResult hrtGetIfNum(struct RaGetIfattr &config, u32 &num)
1785 : {
1786 : #ifndef HCCD
1787 80 : if (DlRaFunction::GetInstance().dlRaGetIfNum == nullptr) {
1788 0 : HCCL_WARNING("this package does not support hrtGetIfNum, please change new package");
1789 0 : return HCCL_SUCCESS;
1790 : }
1791 :
1792 80 : s32 ret = DlRaFunction::GetInstance().dlRaGetIfNum(&config, &num);
1793 80 : constexpr s32 MAX_SUPPORT_IFNUM = 65536;
1794 80 : CHK_PRT_RET((ret != 0 || num > MAX_SUPPORT_IFNUM), HCCL_ERROR("[Get][IfNum]errNo[0x%016llx] ra get if num fail."
1795 : " ret[%d], num[%u] should be less than [%u]", \
1796 : HCCL_ERROR_CODE(HCCL_E_TCP_CONNECT), ret, num, MAX_SUPPORT_IFNUM), HCCL_E_TCP_CONNECT);
1797 80 : return HCCL_SUCCESS;
1798 : #else
1799 : HCCL_ERROR("[hrtGetIfNum]Does not support this interface.");
1800 : return HCCL_E_NOT_SUPPORT;
1801 : #endif
1802 : }
1803 :
1804 80 : HcclResult hrtGetIfAddress(struct RaGetIfattr &config, struct InterfaceInfo ifaddrInfos[], u32 &num)
1805 : {
1806 : #ifndef HCCD
1807 80 : CHK_PRT_RET(num == 0, HCCL_ERROR("[Get][IfAddress]errNo[0x%016llx] ra get if address fail. input param num[%u] "\
1808 : "is invalid.", HCCL_ERROR_CODE(HCCL_E_INTERNAL), num), HCCL_E_INTERNAL);
1809 80 : s32 ret = DlRaFunction::GetInstance().dlRaGetIfAddress(&config, ifaddrInfos, &num);
1810 80 : CHK_PRT_RET(ret != 0, HCCL_ERROR("[Get][IfAddress]errNo[0x%016llx] ra get if address fail. ret[%d], num[%u]", \
1811 : HCCL_ERROR_CODE(HCCL_E_TCP_CONNECT), ret, num), HCCL_E_TCP_CONNECT);
1812 80 : return HCCL_SUCCESS;
1813 : #else
1814 : HCCL_ERROR("[hrtGetIfAddress]Does not support this interface.");
1815 : return HCCL_E_NOT_SUPPORT;
1816 : #endif
1817 : }
1818 :
1819 66 : HcclResult hrtRaGetDeviceIP(u32 devicePhyId, vector<HcclIpAddress> &ipAddr)
1820 : {
1821 66 : struct RaGetIfattr config = {0};
1822 66 : config.phyId = devicePhyId;
1823 66 : config.nicPosition = static_cast<u32>(NICDeployment::NIC_DEPLOYMENT_DEVICE);
1824 66 : config.isAll = false;
1825 :
1826 66 : u32 ifAddrNum = HCCL_DEVICE_NIC_NUM;
1827 66 : CHK_RET(hrtGetIfNum(config, ifAddrNum));
1828 66 : ifAddrNum = ifAddrNum > HCCL_DEVICE_NIC_NUM ? HCCL_DEVICE_NIC_NUM : ifAddrNum;
1829 66 : HCCL_RUN_INFO("[Get][DeviceIP]hrtGetIfNum success. ifAddrNum[%u].", ifAddrNum);
1830 :
1831 66 : if (ifAddrNum == 0) {
1832 0 : HCCL_WARNING("[Get][DeviceIP]device has no ip information, phyId[%u]", devicePhyId);
1833 0 : return HCCL_SUCCESS;
1834 : }
1835 :
1836 : struct InterfaceInfo ifAddrInfos[HCCL_DEVICE_NIC_NUM];
1837 66 : s32 sRet = memset_s(ifAddrInfos, sizeof(InterfaceInfo) * HCCL_DEVICE_NIC_NUM, 0, \
1838 : sizeof(InterfaceInfo) * HCCL_DEVICE_NIC_NUM);
1839 66 : CHK_PRT_RET(sRet != EOK, HCCL_ERROR("[Get][DeviceIP]errNo[0x%016llx] memoryset ifAddrInfos to 0 failed. params: "\
1840 : "dest[%p], dest_size[%zu Byte], count[%zu]", HCCL_ERROR_CODE(HCCL_E_SYSCALL), ifAddrInfos,
1841 : sizeof(InterfaceInfo) * HCCL_DEVICE_NIC_NUM, sizeof(InterfaceInfo) * HCCL_DEVICE_NIC_NUM), HCCL_E_SYSCALL);
1842 :
1843 66 : CHK_RET(hrtGetIfAddress(config, ifAddrInfos, ifAddrNum));
1844 :
1845 66 : CHK_PRT_RET(ifAddrNum > HCCL_DEVICE_NIC_NUM,
1846 : HCCL_ERROR("[Get][DeviceIP]hrtGetIfAddress fail. ifAddrNum[%u] should be below %u", ifAddrNum,
1847 : HCCL_DEVICE_NIC_NUM), HCCL_E_TCP_CONNECT);
1848 :
1849 198 : for (u32 i = 0; i < ifAddrNum; i++) {
1850 : HcclInAddr temp;
1851 132 : temp.addr = ifAddrInfos[i].ifaddr.ip.addr;
1852 132 : temp.addr6 = ifAddrInfos[i].ifaddr.ip.addr6;
1853 132 : HcclIpAddress ipInfo(ifAddrInfos[i].family, temp);
1854 132 : CHK_PRT_RET(ipInfo.IsInvalid(), HCCL_ERROR("ip is invalid."), HCCL_E_PARA);
1855 264 : CHK_RET(ipInfo.SetIfName(ifAddrInfos[i].ifname));
1856 132 : CHK_RET(ipInfo.SetScopeID(ifAddrInfos[i].scopeId));
1857 132 : ipAddr.push_back(ipInfo);
1858 132 : HCCL_RUN_INFO("[Get][DeviceIP]hrtGetIfAddress: idx[%u] ifname[%s] ip[%s]",
1859 : i, ifAddrInfos[i].ifname, ipInfo.GetReadableAddress());
1860 132 : }
1861 :
1862 66 : return HCCL_SUCCESS;
1863 : }
1864 :
1865 :
1866 1 : HcclResult hrtRaGetDeviceAllNicIP(vector<vector<HcclIpAddress>> &ipAddr)
1867 : {
1868 1 : s32 deviceLogicID = -1;
1869 1 : u32 devicePhyId = 0;
1870 1 : CHK_RET(hrtGetDevice(&deviceLogicID));
1871 1 : CHK_RET(hrtGetDevicePhyIdByIndex(static_cast<u32>(deviceLogicID), devicePhyId));
1872 : // 获取版本号查看是否兼容
1873 1 : u32 ifnumVersion = 0;
1874 1 : HcclResult vRet = hrtRaGetInterfaceVersion(devicePhyId, IFADDRS_V2_INTERFACE, &ifnumVersion);
1875 1 : if (vRet != HCCL_SUCCESS || ifnumVersion < IFADDRS_V2_INTERFACE_VERSTOIN) {
1876 0 : HCCL_WARNING("this package does not support hrtRaGetDeviceAllNicIP, please change new package.");
1877 0 : return HCCL_SUCCESS;
1878 : }
1879 1 : DevType deviceType = DevType::DEV_TYPE_COUNT;
1880 1 : CHK_RET(hrtGetDeviceType(deviceType));
1881 1 : CHK_PRT_RET(deviceType != DevType::DEV_TYPE_910_93 && deviceType != DevType::DEV_TYPE_910B,
1882 : HCCL_ERROR("[Get][DeviceAllNicIP] is not supported on device type[%d]. Please check device type.", deviceType),
1883 : HCCL_E_NOT_SUPPORT);
1884 :
1885 1 : struct RaGetIfattr config = {0};
1886 1 : config.phyId = devicePhyId;
1887 1 : config.nicPosition = static_cast<u32>(NICDeployment::NIC_DEPLOYMENT_DEVICE);
1888 1 : config.isAll = true;
1889 :
1890 1 : u32 nicNum = deviceType == DevType::DEV_TYPE_910_93 ? ALL_NIC_NUM_910_93 : ALL_NIC_NUM_910_A2;
1891 1 : u32 maxNicIpNum = HCCL_DEVICE_NIC_NUM * nicNum;
1892 :
1893 1 : u32 ifAddrNum = maxNicIpNum;
1894 1 : CHK_RET(hrtGetIfNum(config, ifAddrNum));
1895 1 : ifAddrNum = ifAddrNum > maxNicIpNum ? maxNicIpNum : ifAddrNum;
1896 1 : HCCL_RUN_INFO("[Get][DeviceAllNicIP]hrtGetIfNum success. ifAddrNum[%u].", ifAddrNum);
1897 :
1898 1 : if (ifAddrNum == 0) {
1899 0 : HCCL_WARNING("[Get][DeviceAllNicIP]device has no ip information, phyId[%u]", devicePhyId);
1900 0 : return HCCL_SUCCESS;
1901 : }
1902 :
1903 1 : struct InterfaceInfo ifAddrInfos[HCCL_DEVICE_NIC_NUM * MAX_ALL_NIC_NUM] = {0};
1904 1 : CHK_RET(hrtGetIfAddress(config, ifAddrInfos, ifAddrNum));
1905 1 : CHK_PRT_RET(ifAddrNum > maxNicIpNum,
1906 : HCCL_ERROR("[Get][DeviceAllNicIP]hrtGetIfAddress fail. ifAddrNum[%u] should be below %u", ifAddrNum,
1907 : maxNicIpNum), HCCL_E_TCP_CONNECT);
1908 :
1909 1 : unordered_map<string, size_t> ifname2Index;
1910 2 : for (u32 i = 0; i < ifAddrNum; i++) {
1911 : HcclInAddr temp;
1912 1 : temp.addr = ifAddrInfos[i].ifaddr.ip.addr;
1913 1 : temp.addr6 = ifAddrInfos[i].ifaddr.ip.addr6;
1914 1 : HcclIpAddress ipInfo(ifAddrInfos[i].family, temp);
1915 1 : CHK_PRT_RET(ipInfo.IsInvalid(), HCCL_ERROR("ip is invalid."), HCCL_E_PARA);
1916 2 : CHK_RET(ipInfo.SetIfName(ifAddrInfos[i].ifname));
1917 1 : CHK_RET(ipInfo.SetScopeID(ifAddrInfos[i].scopeId));
1918 3 : if (ifname2Index.find(ifAddrInfos[i].ifname) == ifname2Index.end()) {
1919 1 : ifname2Index.emplace(ifAddrInfos[i].ifname, ipAddr.size());
1920 1 : ipAddr.emplace_back(vector<HcclIpAddress>());
1921 : }
1922 1 : ipAddr[ifname2Index[ifAddrInfos[i].ifname]].push_back(ipInfo);
1923 1 : HCCL_RUN_INFO("[Get][DeviceAllNicIP]hrtGetIfAddress: idx[%u] ifname[%s] ip[%s]",
1924 : i, ifAddrInfos[i].ifname, ipInfo.GetReadableAddress());
1925 1 : }
1926 :
1927 1 : return HCCL_SUCCESS;
1928 1 : }
1929 :
1930 326 : HcclResult hrtRaGetInterfaceVersion(unsigned int phyId, unsigned int interfaceOpcode, unsigned int* interfaceVersion)
1931 : {
1932 326 : HCCL_DEBUG("hrtRaGetInterfaceVersion phyId[%u], opCode[%u]", phyId, interfaceOpcode);
1933 326 : if (DlRaFunction::GetInstance().dlRaGetInterfaceVersion == nullptr) {
1934 201 : HCCL_WARNING("driver package does not support hrtRaGetInterfaceVersion, please change new package");
1935 201 : return HCCL_E_NOT_SUPPORT;
1936 : }
1937 125 : s32 ret = DlRaFunction::GetInstance().dlRaGetInterfaceVersion(phyId, interfaceOpcode, interfaceVersion);
1938 125 : CHK_PRT_RET(ret != 0, HCCL_ERROR("[Get][InterfaceVersion]errNo[0x%016llx] ra get interface version fail. ret[%d]",
1939 : HCCL_ERROR_CODE(HCCL_E_NETWORK), ret), HCCL_E_NETWORK);
1940 125 : HCCL_INFO("hrtRaGetInterfaceVersion phyId[%u], opCode[%u], version[%u]",
1941 : phyId, interfaceOpcode, *interfaceVersion);
1942 125 : return HCCL_SUCCESS;
1943 : }
1944 :
1945 0 : HcclResult GetIsSupSockBatchCloseImmed(u32 phyId, bool& isSupportBatchClose)
1946 : {
1947 0 : u32 batchCloseVersion = 0;
1948 0 : isSupportBatchClose = false;
1949 : // 获取版本号看是否兼容
1950 0 : HcclResult ret = hrtRaGetInterfaceVersion(phyId, SOCKET_BATCH_CLOSE_INTERFACE, &batchCloseVersion);
1951 0 : CHK_PRT_RET(ret == HCCL_E_NETWORK, HCCL_ERROR("[Get][IsSupSockBatchCloseImmed]comm base hrtRaGetInterfaceVersion "\
1952 : "failed, interface[%u]", SOCKET_BATCH_CLOSE_INTERFACE), ret);
1953 0 : if (ret == HCCL_E_NOT_SUPPORT) {
1954 0 : HCCL_WARNING("this package does not support hrtRaGetInterfaceVersion, please change new package");
1955 0 : return HCCL_SUCCESS;
1956 : }
1957 0 : if (batchCloseVersion >= SOCKET_BATCH_CLOSE_SUP_VER) {
1958 0 : isSupportBatchClose = true;
1959 : }
1960 0 : return HCCL_SUCCESS;
1961 : }
1962 :
1963 2 : HcclResult hrtRaCreateCq(RdmaHandle handle, struct CqAttr* attr)
1964 : {
1965 2 : CHK_PTR_NULL(handle);
1966 2 : CHK_PTR_NULL(attr);
1967 2 : CHK_PTR_NULL(attr->ibSendCq);
1968 2 : CHK_PTR_NULL(attr->ibRecvCq);
1969 2 : CHK_PTR_NULL(attr->qpContext);
1970 2 : HCCL_DEBUG("ra create cq: sendCqDepth[%d], recvCqDepth[%d], sendCqEventId[%d], recvCqEventId[%d]",
1971 : attr->sendCqDepth, attr->recvCqDepth, attr->sendCqEventId, attr->recvCqEventId);
1972 2 : s32 ret = DlRaFunction::GetInstance().dlRaCreateCq(handle, attr);
1973 2 : CHK_PRT_RET(ret != 0, HCCL_ERROR("[Create][RaCq]errNo[0x%016llx] ra create cq fail. return[%d] "\
1974 : "sendCqDepth[%d], recvCqDepth[%d], sendCqEventId[%d], recvCqEventId[%d]",\
1975 : HCCL_ERROR_CODE(HCCL_E_INTERNAL), ret, attr->sendCqDepth, attr->recvCqDepth, attr->sendCqEventId,\
1976 : attr->recvCqEventId), HCCL_E_INTERNAL);
1977 2 : return HCCL_SUCCESS;
1978 : }
1979 :
1980 : map<string, vector<CqInfo>> g_qpRecords;
1981 : mutex g_qpRecordsMutex;
1982 2 : HcclResult CreateCq(RdmaHandle rdmaHandle, CqInfo& cq)
1983 : {
1984 2 : struct CqAttr attr = {};
1985 2 : attr.qpContext = &cq.context;
1986 2 : attr.ibSendCq = &cq.sq;
1987 2 : attr.ibRecvCq = &cq.rq;
1988 2 : attr.sendCqDepth = cq.depth;
1989 2 : attr.recvCqDepth = cq.depth;
1990 :
1991 2 : attr.sendCqEventId = cq.sqEvent;
1992 2 : attr.recvCqEventId = cq.rqEvent;
1993 2 : attr.sendChannel = cq.sendChannel;
1994 2 : attr.recvChannel = cq.recvChannel;
1995 2 : attr.srqContext = cq.srqContext;
1996 2 : CHK_RET(hrtRaCreateCq(rdmaHandle, &attr));
1997 2 : return HCCL_SUCCESS;
1998 : }
1999 :
2000 0 : HcclResult hrtRaDestroyCq(RdmaHandle handle, struct CqAttr* attr)
2001 : {
2002 0 : CHK_PTR_NULL(handle);
2003 0 : CHK_PTR_NULL(attr);
2004 0 : s32 ret = DlRaFunction::GetInstance().dlRaDestroyCq(handle, attr);
2005 0 : CHK_PRT_RET(ret != 0, HCCL_ERROR("[Destroy][RaCq]errNo[0x%016llx] ra destroy cq fail. ret[%d]",\
2006 : HCCL_ERROR_CODE(HCCL_E_NETWORK), ret), HCCL_E_NETWORK);
2007 0 : return HCCL_SUCCESS;
2008 : }
2009 :
2010 0 : HcclResult hrtRaNormalQpCreate(RdmaHandle handle, struct ibv_qp_init_attr* initAttr, QpHandle &qpHandle,
2011 : struct ibv_qp* &qp)
2012 : {
2013 0 : CHK_PTR_NULL(handle);
2014 0 : CHK_PTR_NULL(initAttr);
2015 0 : HCCL_DEBUG("ra normal qp create: initAttr[%p]", initAttr);
2016 0 : s32 ret = DlRaFunction::GetInstance().dlRaNormalQpCreate(handle, initAttr, &qpHandle,
2017 : reinterpret_cast<void **>(&qp));
2018 :
2019 0 : std::string qpInfo = std::string("qp_type[") + std::to_string(initAttr->qp_type) + std::string("] ") +
2020 0 : std::string("max_inline_data[") + std::to_string(initAttr->cap.max_inline_data) + std::string("] ") +
2021 0 : std::string("max_send_wr[") + std::to_string(initAttr->cap.max_send_wr) + std::string("] ") +
2022 0 : std::string("max_send_sge[") + std::to_string(initAttr->cap.max_send_sge) + std::string("] ") +
2023 0 : std::string("max_recv_wr[") + std::to_string(initAttr->cap.max_recv_wr) + std::string("] ") +
2024 0 : std::string("max_recv_sge[") + std::to_string(initAttr->cap.max_recv_sge) + std::string("]");
2025 :
2026 0 : CHK_OOM_RET(ret, qpInfo.c_str());
2027 :
2028 0 : CHK_PRT_RET(ret != 0, HCCL_ERROR("[Create][NormalQp]errNo[0x%016llx] ra create normal qp fail.ret[%d]"
2029 : "qp_type[%u] max_inline_data[%u] max_send_wr[%u] max_send_sge[%u] max_recv_wr[%u] max_recv_sge[%u]",\
2030 : HCCL_ERROR_CODE(HCCL_E_INTERNAL), ret, initAttr->qp_type, initAttr->cap.max_inline_data, initAttr->cap.max_send_wr,
2031 : initAttr->cap.max_send_sge, initAttr->cap.max_recv_wr, initAttr->cap.max_recv_sge), HCCL_E_INTERNAL);
2032 :
2033 0 : struct QpAttr attr{};
2034 0 : CHK_RET(hrtRaGetQpAttr(qpHandle, &attr));
2035 0 : s32 deviceId = 0;
2036 0 : if (hrtGetDevice(&deviceId) != HCCL_SUCCESS) {
2037 0 : deviceId = -1;
2038 : }
2039 0 : PLF_CONFIG_DEBUG(PLF_RES,
2040 : "Create Qp para: deviceId[%d] qpn[%u] qp_type[%u] max_inline_data[%u] max_send_wr[%u] max_send_sge[%u] "\
2041 : "max_recv_wr[%u] max_recv_sge[%u]", deviceId, attr.qpn, initAttr->qp_type, initAttr->cap.max_inline_data,
2042 : initAttr->cap.max_send_wr, initAttr->cap.max_send_sge, initAttr->cap.max_recv_wr, initAttr->cap.max_recv_sge);
2043 0 : return HCCL_SUCCESS;
2044 0 : }
2045 :
2046 0 : HcclResult hrtRaNormalQpDestroy(QpHandle qpHandle)
2047 : {
2048 0 : struct QpAttr attr{};
2049 0 : CHK_RET(hrtRaGetQpAttr(qpHandle, &attr));
2050 0 : s32 deviceId = 0;
2051 0 : if (hrtGetDevice(&deviceId) != HCCL_SUCCESS) {
2052 0 : deviceId = -1;
2053 : }
2054 0 : PLF_CONFIG_DEBUG(PLF_RES, "Destroy Qp para: deviceId[%d] qpn[%u]", deviceId, attr.qpn);
2055 :
2056 0 : CHK_PTR_NULL(qpHandle);
2057 0 : s32 ret = DlRaFunction::GetInstance().dlRaNormalQpDestroy(qpHandle);
2058 0 : CHK_PRT_RET(ret != 0, HCCL_ERROR("[Destroy][NormalQp]errNo[0x%016llx] ra destroy normal qp fail. ret[%d] qpHandle[%p]",\
2059 : HCCL_ERROR_CODE(HCCL_E_NETWORK), ret, qpHandle), HCCL_E_NETWORK);
2060 0 : return HCCL_SUCCESS;
2061 : }
2062 :
2063 2 : HcclResult DestroyCq(RdmaHandle rdmaHandle, CqInfo& cq)
2064 : {
2065 : struct CqAttr attr;
2066 2 : attr.qpContext = &cq.context;
2067 2 : attr.ibSendCq = &cq.sq;
2068 2 : attr.ibRecvCq = &cq.rq;
2069 2 : CHK_RET(hrtRaDestroyCq(rdmaHandle, &attr));
2070 2 : return HCCL_SUCCESS;
2071 : }
2072 :
2073 4 : HcclResult ConstructQpAttrs(s32 qpMode, struct QpExtAttrs &attrs, const QueueDepthAttr& qpDepth, bool isWorkFlowLib)
2074 : {
2075 4 : HCCL_INFO("[ConstructQpAttrs][qpDepth]sendCqDepth[%u], recvCqDepth[%u], sqDepth[%u], rqDepth[%u]", qpDepth.sendCqDepth, qpDepth.recvCqDepth,
2076 : qpDepth.sqDepth, qpDepth.rqDepth);
2077 4 : CHK_PRT_RET(CheckQpDepth(qpDepth.sendCqDepth) != HCCL_SUCCESS,
2078 : HCCL_ERROR("[CheckQpDepth]sendCqDepth[%u] is invalid, sendCqDepth should be power of 2 and in [%u, %u]",
2079 : qpDepth.sendCqDepth, QP_DEPTH_MIN, QP_DEPTH_MAX);, HCCL_E_PARA);
2080 4 : CHK_PRT_RET(CheckQpDepth(qpDepth.recvCqDepth) != HCCL_SUCCESS,
2081 : HCCL_ERROR("[CheckQpDepth]recvCqDepth[%u] is invalid, recvCqDepth should be power of 2 and in [%u, %u]",
2082 : qpDepth.recvCqDepth, QP_DEPTH_MIN, QP_DEPTH_MAX);, HCCL_E_PARA);
2083 4 : CHK_PRT_RET(CheckQpDepth(qpDepth.sqDepth) != HCCL_SUCCESS,
2084 : HCCL_ERROR("[CheckQpDepth]sqDepth[%u] is invalid, sqDepth should be power of 2 and in [%u, %u]",
2085 : qpDepth.sqDepth, QP_DEPTH_MIN, QP_DEPTH_MAX);, HCCL_E_PARA);
2086 4 : CHK_PRT_RET(CheckQpDepth(qpDepth.rqDepth) != HCCL_SUCCESS,
2087 : HCCL_ERROR("[CheckQpDepth]rqDepth[%u] is invalid, rqDepth should be power of 2 and in [%u, %u]",
2088 : qpDepth.rqDepth, QP_DEPTH_MIN, QP_DEPTH_MAX);, HCCL_E_PARA);
2089 :
2090 4 : attrs.qpMode = qpMode;
2091 4 : attrs.version = QP_CREATE_WITH_ATTR_VERSION;
2092 4 : attrs.cqAttr.recvCqDepth = (qpDepth.recvCqDepth == INVALID_UINT) ? DEFAULT_MAX_RECV_CQ_DEPTH : qpDepth.recvCqDepth;
2093 4 : attrs.qpAttr.cap.max_inline_data = DEFAULT_MAX_INLINE_DATA;
2094 4 : attrs.qpAttr.cap.max_send_sge = DEFAULT_MAX_SEND_SGE;
2095 4 : attrs.qpAttr.cap.max_recv_wr = (qpDepth.rqDepth == INVALID_UINT) ? DEFAULT_MAX_RECV_WR : qpDepth.rqDepth;
2096 4 : attrs.qpAttr.cap.max_recv_sge = DEFAULT_MAX_RECV_SGE;
2097 4 : attrs.qpAttr.qp_type = IBV_QPT_RC;
2098 :
2099 4 : if (qpDepth.sqDepth == INVALID_UINT) {
2100 4 : if (qpMode == OFFLINE_QP_MODE_EXT || isWorkFlowLib) {
2101 0 : attrs.qpAttr.cap.max_send_wr = DEFAULT_OFFLINE_MAX_SEND_WR;
2102 : } else {
2103 4 : attrs.qpAttr.cap.max_send_wr = DEFAULT_OPBASE_MAX_SEND_WR;
2104 : }
2105 : } else {
2106 0 : attrs.qpAttr.cap.max_send_wr = qpDepth.sqDepth;
2107 : }
2108 4 : if (qpDepth.sendCqDepth == INVALID_UINT) {
2109 4 : attrs.cqAttr.sendCqDepth = DEFAULT_MAX_SEND_CQ_DEPTH;
2110 4 : if (qpMode == OFFLINE_QP_MODE_EXT || qpMode == OFFLINE_QP_MODE || isWorkFlowLib) {
2111 0 : attrs.cqAttr.sendCqDepth = HCCL_SEND_CQ_DEPTH_DEFAULT;
2112 : }
2113 : } else {
2114 0 : attrs.cqAttr.sendCqDepth = qpDepth.sendCqDepth;
2115 : }
2116 4 : HCCL_INFO("[ConstructQpAttrs][attr]sendCqDepth[%d], recvCqDepth[%d], max_send_wr[%u], max_recv_wr[%u]", attrs.cqAttr.sendCqDepth,
2117 : attrs.cqAttr.recvCqDepth, attrs.qpAttr.cap.max_send_wr, attrs.qpAttr.cap.max_recv_wr);
2118 4 : return HCCL_SUCCESS;
2119 : }
2120 :
2121 3 : HcclResult CreateQp(RdmaHandle rdmaHandle, int& flag, s32& qpMode, QpInfo& qp, bool isESMode)
2122 : {
2123 3 : HCCL_INFO("CreateQp qpMode[%d], isESMode[%d].", qpMode, isESMode);
2124 3 : if (isESMode && (qpMode == OFFLINE_QP_MODE_EXT || qpMode == OPBASE_QP_MODE_EXT)) {
2125 0 : struct QpExtAttrs attrs{};
2126 0 : QueueDepthAttr qpDepth{};
2127 0 : CHK_RET(ConstructQpAttrs(qpMode, attrs, qpDepth));
2128 0 : attrs.udpSport = 0x0;
2129 0 : attrs.qpAttr.cap.max_send_wr = HETEROG_OFFLINE_EXT_MAX_SEND_WR;
2130 0 : attrs.cqAttr.sendCqDepth = DEFAULT_MAX_ONE_SIDED_SEND_CQ_DEPTH;
2131 0 : CHK_RET(hrtRaQpCreateWithAttrs(rdmaHandle, &attrs, qp.qpHandle));
2132 0 : } else {
2133 3 : CHK_RET(HrtRaQpCreate(rdmaHandle, flag, qpMode, qp.qpHandle));
2134 : }
2135 :
2136 : // Hdc模式下HCCP不支持hrtRaGetQpContext接口
2137 3 : HcclResult ret = SetQpAttrQos(qp.qpHandle, qp.trafficClass, qp.serviceLevel);
2138 3 : if (ret != HCCL_SUCCESS) {
2139 1 : HCCL_ERROR("[CreateQp] SetQpAttrQos fail, ret[%d], destroy QP", ret);
2140 1 : HrtRaQpDestroy(qp.qpHandle);
2141 1 : return ret;
2142 : }
2143 : // 配置RDMA Timeout时间
2144 2 : ret = SetQpAttrTimeOut(qp.qpHandle);
2145 2 : if (ret != HCCL_SUCCESS) {
2146 1 : HCCL_ERROR("[CreateQp] SetQpAttrTimeOut fail, ret[%d], destroy QP", ret);
2147 1 : HrtRaQpDestroy(qp.qpHandle);
2148 1 : return ret;
2149 : }
2150 : // 配置RDMA Retry Cnt重传次数
2151 1 : ret = SetQpAttrRetryCnt(qp.qpHandle);
2152 1 : if (ret != HCCL_SUCCESS) {
2153 1 : HCCL_ERROR("[CreateQp] SetQpAttrRetryCnt fail, ret[%d], destroy QP", ret);
2154 1 : HrtRaQpDestroy(qp.qpHandle);
2155 1 : return ret;
2156 : }
2157 :
2158 0 : return HCCL_SUCCESS;
2159 : }
2160 :
2161 4 : HcclResult CreateNormalQp(RdmaHandle rdmaHandle, QpInfo& qp)
2162 : {
2163 : struct ibv_qp_init_attr ibQpAttr;
2164 4 : CHK_SAFETY_FUNC_RET(memset_s(&ibQpAttr, sizeof(ibv_qp_init_attr), 0, sizeof(ibv_qp_init_attr)));
2165 4 : ibQpAttr.qp_context= qp.context;
2166 4 : ibQpAttr.send_cq = qp.sendCq;
2167 4 : ibQpAttr.recv_cq = qp.recvCq;
2168 4 : ibQpAttr.srq = qp.srq;
2169 4 : ibQpAttr.qp_type = IBV_QPT_RC;
2170 4 : ibQpAttr.cap.max_inline_data = MAX_INLINE_DATA;
2171 4 : ibQpAttr.cap.max_send_wr = qp.attr.maxWr;
2172 4 : ibQpAttr.cap.max_send_sge = qp.attr.maxSendSge;
2173 4 : ibQpAttr.cap.max_recv_wr = (qp.srq == nullptr ? qp.attr.maxWr : 0);
2174 4 : ibQpAttr.cap.max_recv_sge = (qp.srq == nullptr ? qp.attr.maxRecvSge : 0);
2175 4 : CHK_RET(hrtRaNormalQpCreate(rdmaHandle, &ibQpAttr, qp.qpHandle, qp.qp));
2176 2 : HcclResult ret = SetQpAttrQos(qp.qpHandle, qp.trafficClass, qp.serviceLevel);
2177 2 : if (ret != HCCL_SUCCESS) {
2178 0 : HCCL_ERROR("[CreateNormalQp] SetQpAttrQos fail, ret[%d], destroy QP", ret);
2179 0 : HrtRaQpDestroy(qp.qpHandle);
2180 0 : return ret;
2181 : }
2182 : // 配置RDMA Timeout时间
2183 2 : ret = SetQpAttrTimeOut(qp.qpHandle);
2184 2 : if (ret != HCCL_SUCCESS) {
2185 1 : HCCL_ERROR("[CreateNormalQp] SetQpAttrTimeOut fail, ret[%d], destroy QP", ret);
2186 1 : HrtRaQpDestroy(qp.qpHandle);
2187 1 : return ret;
2188 : }
2189 : // 配置RDMA Retry Cnt重传次数
2190 1 : ret = SetQpAttrRetryCnt(qp.qpHandle);
2191 1 : if (ret != HCCL_SUCCESS) {
2192 1 : HCCL_ERROR("[CreateNormalQp] SetQpAttrRetryCnt fail, ret[%d], destroy QP", ret);
2193 1 : HrtRaQpDestroy(qp.qpHandle);
2194 1 : return ret;
2195 : }
2196 :
2197 0 : return HCCL_SUCCESS;
2198 : }
2199 :
2200 1 : HcclResult CreateCqAndQp(RdmaHandle &rdmaHandle, string &label, QpConfig &config, QpInfo &info)
2201 : {
2202 1 : unique_lock<mutex> lock(g_qpRecordsMutex);
2203 1 : bool createCq = false;
2204 1 : if (g_qpRecords[label].empty()) {
2205 1 : HCCL_INFO("create cq: label[%s] is empty, need create cq.", label.c_str());
2206 1 : createCq = true;
2207 0 : } else if ((g_qpRecords[label].back().depth - g_qpRecords[label].back().used) < config.maxWr) {
2208 0 : HCCL_INFO("create cq: label[%s] has %u qp, last cq used %u, need create cq.",
2209 : label.c_str(), g_qpRecords[label].size(), g_qpRecords[label].back().used);
2210 0 : createCq = true;
2211 : } else {
2212 0 : HCCL_INFO("create cq: label[%s] has %u qp, last cq used %u, not need create cq.",
2213 : label.c_str(), g_qpRecords[label].size(), g_qpRecords[label].back().used);
2214 : }
2215 :
2216 1 : if (createCq) {
2217 1 : CqInfo cq(nullptr, info.srqCq, nullptr, MAX_CQ_DEPTH, config.sqEvent, config.rqEvent, info.srqContext);
2218 1 : CHK_RET(CreateCq(rdmaHandle, cq));
2219 : QpInfo qp(config, rdmaHandle, nullptr, nullptr, cq.context, cq.sq, cq.rq, info.srq,
2220 1 : info.srqCq, info.srqContext);
2221 1 : HcclResult ret = CreateNormalQp(rdmaHandle, qp);
2222 1 : if (ret != HCCL_SUCCESS) {
2223 1 : HCCL_ERROR("[CreateCqAndQp] CreateNormalQp fail, ret[%d], destroy CQ", ret);
2224 1 : DestroyCq(rdmaHandle, cq);
2225 1 : return ret;
2226 : }
2227 :
2228 0 : cq.used += qp.attr.maxWr;
2229 0 : cq.qps.push_back(qp);
2230 0 : g_qpRecords[label].push_back(cq);
2231 0 : info = qp;
2232 2 : } else {
2233 0 : QpInfo qp(config, rdmaHandle, nullptr, nullptr, g_qpRecords[label].back().context,
2234 0 : g_qpRecords[label].back().sq, g_qpRecords[label].back().rq, info.srq, info.srqCq, info.srqContext);
2235 0 : CHK_RET(CreateNormalQp(rdmaHandle, qp));
2236 :
2237 0 : g_qpRecords[label].back().used += config.maxWr;
2238 0 : g_qpRecords[label].back().qps.push_back(qp);
2239 0 : info = qp;
2240 0 : }
2241 0 : return HCCL_SUCCESS;
2242 1 : }
2243 :
2244 0 : HcclResult CreateQpWithSharedCq(RdmaHandle rdmaHandle, HcclIpAddress &selfIp, HcclIpAddress &peerIp, s32 sqEvent,
2245 : s32 rqEvent, QpInfo &info, s32 qpAppend, u32 maxSegNum)
2246 : {
2247 0 : QpConfig config(selfIp, peerIp, MAX_WR_NUM, maxSegNum, MAX_RECV_SGE_NUM, sqEvent, rqEvent);
2248 :
2249 0 : string label = string(selfIp.GetReadableIP()) + "_" + string(peerIp.GetReadableIP()) + "_" +
2250 0 : to_string(config.sqEvent) + "_" + to_string(config.rqEvent) + "_" + to_string(qpAppend);
2251 :
2252 0 : HCCL_RUN_INFO("CreateQpWithSharedCq selfIp[%s] peerIp[%s] maxWr[%u] maxSendSge[%u] maxRecvSge[%u]"
2253 : "sqEvent[%d] rqEvent[%d]", selfIp.GetReadableIP(), peerIp.GetReadableIP(),
2254 : config.maxWr, config.maxSendSge, config.maxRecvSge, config.sqEvent, config.rqEvent);
2255 0 : CHK_RET(CreateCqAndQp(rdmaHandle, label, config, info));
2256 0 : return HCCL_SUCCESS;
2257 0 : }
2258 :
2259 0 : HcclResult DestroyQpWithSharedCq(const QpInfo &info, s32 qpAppend)
2260 : {
2261 0 : if (info.qpHandle == nullptr) {
2262 0 : return HCCL_SUCCESS;
2263 : }
2264 :
2265 0 : string label = string(info.attr.selfIp.GetReadableIP()) + "_" + string(info.attr.peerIp.GetReadableIP()) + "_" +
2266 0 : to_string(info.attr.sqEvent) + "_" + to_string(info.attr.rqEvent) + "_" + to_string(qpAppend);
2267 :
2268 0 : unique_lock<mutex> lock(g_qpRecordsMutex);
2269 0 : if (g_qpRecords[label].empty()) {
2270 0 : HCCL_ERROR("qp label[%s] no exist.", label.c_str());
2271 0 : return HCCL_E_PARA;
2272 : } else {
2273 0 : for (auto itCq = g_qpRecords[label].begin(); itCq != g_qpRecords[label].end(); itCq++) {
2274 0 : if ((*itCq).context == info.context) {
2275 0 : for (auto itQp = (*itCq).qps.begin(); itQp != (*itCq).qps.end(); itQp++) {
2276 0 : if ((*itQp).qpHandle == info.qpHandle) {
2277 0 : HCCL_INFO("destroy qpHandle");
2278 0 : CHK_RET(hrtRaNormalQpDestroy(info.qpHandle));
2279 0 : (*itCq).qps.erase(itQp);
2280 0 : if ((*itCq).used > info.attr.maxWr) {
2281 0 : (*itCq).used -= info.attr.maxWr;
2282 0 : } else if ((*itCq).used == info.attr.maxWr) {
2283 0 : HCCL_INFO("destroy cq:%p", (*itCq).context);
2284 0 : CHK_RET(DestroyCq(info.rdmaHandle, *itCq));
2285 0 : g_qpRecords[label].erase(itCq);
2286 : } else {
2287 0 : HCCL_ERROR("DestroyQp: cq used[%u] should be greater than the qp maxwr[%u]", (*itCq).used,
2288 : info.attr.maxWr);
2289 0 : return HCCL_E_PARA;
2290 : }
2291 0 : return HCCL_SUCCESS;
2292 : }
2293 : }
2294 0 : HCCL_ERROR("DestroyQp: the qp is no exist");
2295 0 : return HCCL_E_PARA;
2296 : }
2297 : }
2298 0 : HCCL_ERROR("DestroyQp: the cq is no exist");
2299 0 : return HCCL_E_PARA;
2300 : }
2301 0 : }
2302 :
2303 1 : HcclResult CreateQpWithCq(RdmaHandle rdmaHandle, s32 sqEvent, s32 rqEvent,
2304 : void *sendChannel, void *recvChannel, QpInfo &info, bool isHdcMode, bool isESMode)
2305 : {
2306 1 : struct ibv_comp_channel *sChannel = reinterpret_cast<struct ibv_comp_channel *>(sendChannel);
2307 1 : struct ibv_comp_channel *rChannel = reinterpret_cast<struct ibv_comp_channel *>(recvChannel);
2308 :
2309 1 : QpConfig config(MAX_WR_NUM, MAX_SEND_SGE_NUM, MAX_RECV_SGE_NUM, sqEvent, rqEvent);
2310 : CqInfo cq(nullptr, nullptr, nullptr, MAX_CQ_DEPTH, config.sqEvent, config.rqEvent, info.srqContext,
2311 1 : sChannel, rChannel);
2312 1 : if (!isHdcMode) {
2313 : // hdc模式下hccp没有对外提供创建CQ的接口
2314 1 : CHK_RET(CreateCq(rdmaHandle, cq));
2315 : }
2316 : QpInfo qp(config, rdmaHandle, nullptr, nullptr, cq.context, cq.sq, cq.rq, info.srq, info.srqCq, info.srqContext,
2317 1 : sChannel, rChannel, info.trafficClass, info.serviceLevel);
2318 :
2319 1 : if (isHdcMode) {
2320 0 : CHK_RET(CreateQp(rdmaHandle, info.flag, info.qpMode, qp, isESMode));
2321 0 : info.qpHandle = qp.qpHandle;
2322 0 : info.qp = qp.qp;
2323 0 : info.sendCq = qp.sendCq;
2324 0 : info.recvCq = qp.recvCq;
2325 : } else {
2326 1 : HcclResult ret = CreateNormalQp(rdmaHandle, qp);
2327 1 : if (ret != HCCL_SUCCESS) {
2328 1 : HCCL_ERROR("[CreateQpWithCq] CreateNormalQp fail, ret[%d], destroy CQ", ret);
2329 1 : DestroyCq(rdmaHandle, cq);
2330 1 : return ret;
2331 : }
2332 0 : info = qp;
2333 : }
2334 0 : return HCCL_SUCCESS;
2335 1 : }
2336 :
2337 0 : HcclResult DestroyQpWithCq(const QpInfo& info, bool isHdcMode)
2338 : {
2339 0 : if (info.qpHandle == nullptr) {
2340 0 : return HCCL_SUCCESS;
2341 : }
2342 :
2343 0 : if (isHdcMode) {
2344 0 : CHK_RET(HrtRaQpDestroy(info.qpHandle));
2345 : } else {
2346 0 : CHK_RET(hrtRaNormalQpDestroy(info.qpHandle));
2347 : }
2348 :
2349 0 : CqInfo cq;
2350 0 : cq.context = info.context;
2351 0 : cq.rq = info.recvCq;
2352 0 : cq.sq = info.sendCq;
2353 0 : if (!isHdcMode) {
2354 0 : CHK_RET(DestroyCq(info.rdmaHandle, cq));
2355 : }
2356 :
2357 0 : return HCCL_SUCCESS;
2358 0 : }
2359 :
2360 4 : HcclResult CreateAiQp(RdmaHandle rdmaHandle, struct AiQpInfo &aiQpInfo, QpInfo &info, u32 devicePhyId)
2361 : {
2362 4 : struct QpExtAttrs attrs{};
2363 4 : QueueDepthAttr qpDepth{};
2364 4 : CHK_RET(ConstructQpAttrs(info.qpMode, attrs, qpDepth, false));
2365 4 : attrs.qpAttr.cap.max_send_wr = HETEROG_OFFLINE_EXT_MAX_SEND_WR;
2366 4 : attrs.cqAttr.sendCqDepth = DEFAULT_MAX_ONE_SIDED_SEND_CQ_DEPTH;
2367 4 : attrs.udpSport = 0;
2368 :
2369 4 : CHK_RET(hrtRaAiQpCreate(devicePhyId, rdmaHandle, &attrs, &aiQpInfo, info.qpHandle));
2370 :
2371 4 : HcclResult ret = SetQpAttrQos(info.qpHandle, info.trafficClass, info.serviceLevel);
2372 4 : if (ret != HCCL_SUCCESS) {
2373 1 : HCCL_ERROR("[CreateAiQp] SetQpAttrQos fail, ret[%d], destroy qpHandle", ret);
2374 1 : HrtRaQpDestroy(info.qpHandle);
2375 1 : return ret;
2376 : }
2377 3 : ret = SetQpAttrTimeOut(info.qpHandle);
2378 3 : if (ret != HCCL_SUCCESS) {
2379 1 : HCCL_ERROR("[CreateAiQp] SetQpAttrTimeOut fail, ret[%d], destroy qpHandle", ret);
2380 1 : HrtRaQpDestroy(info.qpHandle);
2381 1 : return ret;
2382 : }
2383 2 : ret = SetQpAttrRetryCnt(info.qpHandle);
2384 2 : if (ret != HCCL_SUCCESS) {
2385 1 : HCCL_ERROR("[CreateAiQp] SetQpAttrRetryCnt fail, ret[%d], destroy qpHandle", ret);
2386 1 : HrtRaQpDestroy(info.qpHandle);
2387 1 : return ret;
2388 : }
2389 :
2390 1 : info.qp = reinterpret_cast<struct ibv_qp *>(aiQpInfo.aiQpAddr);
2391 1 : if (info.qp == nullptr) {
2392 1 : HCCL_ERROR("info.qp is nullptr.");
2393 1 : HrtRaQpDestroy(info.qpHandle);
2394 1 : return HCCL_E_PARA;
2395 : }
2396 :
2397 0 : info.sendCq = reinterpret_cast<struct ibv_cq *>(aiQpInfo.aiScqAddr);
2398 0 : info.recvCq = reinterpret_cast<struct ibv_cq *>(aiQpInfo.aiRcqAddr);
2399 :
2400 0 : return HCCL_SUCCESS;
2401 : }
2402 :
2403 0 : HcclResult DestroyAiQp(const QpInfo &info)
2404 : {
2405 0 : if (info.qpHandle == nullptr) {
2406 0 : return HCCL_SUCCESS;
2407 : }
2408 :
2409 0 : CHK_RET(HrtRaQpDestroy(info.qpHandle));
2410 :
2411 0 : return HCCL_SUCCESS;
2412 : }
2413 :
2414 0 : HcclResult hrtRaSetQpAttrQos(QpHandle qpHandle, struct QosAttr &attr)
2415 : {
2416 0 : s32 ret = DlRaFunction::GetInstance().dlRaSetQpAttrQos(qpHandle, &attr);
2417 0 : CHK_PRT_RET(ret != 0, HCCL_ERROR("[Set][SqAttr]set qp attr qos failed tc[%u] sl[%u] ret[%d]",\
2418 : attr.tc, attr.sl, ret), HCCL_E_NETWORK);
2419 0 : return HCCL_SUCCESS;
2420 : }
2421 :
2422 0 : HcclResult hrtRaSetQpAttrTimeOut(QpHandle qpHandle, u32 &timeOut)
2423 : {
2424 0 : s32 ret = DlRaFunction::GetInstance().dlRaSetQpAttrTimeOut(qpHandle, &timeOut);
2425 0 : CHK_PRT_RET(ret != 0, HCCL_ERROR("[Set][SqAttr]set qp attr timeout[%u s] failed ret[%d]",\
2426 : timeOut, ret), HCCL_E_NETWORK);
2427 0 : return HCCL_SUCCESS;
2428 : }
2429 :
2430 0 : HcclResult hrtRaSetQpAttrRetryCnt(QpHandle qpHandle, u32 &retryCnt)
2431 : {
2432 0 : s32 ret = DlRaFunction::GetInstance().dlRaSetQpAttrRetryCnt(qpHandle, &retryCnt);
2433 0 : CHK_PRT_RET(ret != 0, HCCL_ERROR("[Set][SqAttr]set qp attr retrycnt[%u] failed ret[%d]",
2434 : retryCnt, ret), HCCL_E_NETWORK);
2435 0 : return HCCL_SUCCESS;
2436 : }
2437 :
2438 9 : HcclResult SetQpAttrQos(QpHandle qpHandle, u32 tc, u32 sl)
2439 : {
2440 9 : struct QosAttr qosAttr = {0};
2441 9 : if (tc == HCCL_COMM_TRAFFIC_CLASS_CONFIG_NOT_SET && sl == HCCL_COMM_SERVICE_LEVEL_CONFIG_NOT_SET) {
2442 0 : qosAttr.tc = GetExternalInputRdmaTrafficClass();
2443 0 : qosAttr.sl = GetExternalInputRdmaServerLevel();
2444 0 : HCCL_INFO("[%s]set qp qos success by environment variable or default value, TC[%u] SL[%u]",
2445 : __func__, qosAttr.tc, qosAttr.sl);
2446 : } else {
2447 9 : qosAttr.tc = tc;
2448 9 : qosAttr.sl = sl;
2449 9 : HCCL_INFO("[%s]set qp qos success by config, TC[%u] SL[%u]", __func__, qosAttr.tc, qosAttr.sl);
2450 : }
2451 :
2452 9 : CHK_RET(hrtRaSetQpAttrQos(qpHandle, qosAttr));
2453 7 : HCCL_INFO("[%s]rdmaTrafficClass[%u], rdmaServerLevel[%u].", __func__, qosAttr.tc, qosAttr.sl);
2454 :
2455 7 : return HCCL_SUCCESS;
2456 : }
2457 :
2458 7 : HcclResult SetQpAttrTimeOut(QpHandle qpHandle)
2459 : {
2460 7 : u32 rdmaTimeOut = GetExternalInputRdmaTimeOut();
2461 7 : CHK_RET(hrtRaSetQpAttrTimeOut(qpHandle, rdmaTimeOut));
2462 4 : HCCL_INFO("[SetQpAttrTimeOut]rdmaTimeOut[%u].", rdmaTimeOut);
2463 :
2464 4 : return HCCL_SUCCESS;
2465 : }
2466 :
2467 4 : HcclResult SetQpAttrRetryCnt(QpHandle qpHandle)
2468 : {
2469 4 : u32 rdmaRetryCnt = GetExternalInputRdmaRetryCnt();
2470 4 : CHK_RET(hrtRaSetQpAttrRetryCnt(qpHandle, rdmaRetryCnt));
2471 1 : HCCL_INFO("[SetQpAttrRetryCnt]rdmaRetryCnt[%u].", rdmaRetryCnt);
2472 :
2473 1 : return HCCL_SUCCESS;
2474 : }
2475 :
2476 0 : HcclResult hrtRaCreateCompChannel(RdmaHandle rdmaHandle, void **compChannel)
2477 : {
2478 0 : s32 ret = DlRaFunction::GetInstance().dlRaCreateCompChannel(rdmaHandle, compChannel);
2479 0 : CHK_PRT_RET(ret != 0, HCCL_ERROR("[Create][CompChannel]errNo[0x%016llx] ra create comp channel fail. "
2480 : "return[%d], params: rdmaHandle[%p], compChannel[%p]", HCCL_ERROR_CODE(HCCL_E_NETWORK),
2481 : ret, rdmaHandle, compChannel), HCCL_E_NETWORK);
2482 :
2483 0 : return HCCL_SUCCESS;
2484 : }
2485 :
2486 0 : HcclResult hrtRaDestroyCompChannel(RdmaHandle rdmaHandle, void *compChannel)
2487 : {
2488 0 : s32 ret = DlRaFunction::GetInstance().dlRaDestroyCompChannel(rdmaHandle, compChannel);
2489 0 : CHK_PRT_RET(ret != 0, HCCL_ERROR("[Destroy][CompChannel]errNo[0x%016llx] ra destroy normal qp fail. "
2490 : "return[%d], params: rdmaHandle[%p], compChannel[%p]", HCCL_ERROR_CODE(HCCL_E_NETWORK),
2491 : ret, rdmaHandle, compChannel), HCCL_E_NETWORK);
2492 :
2493 0 : return HCCL_SUCCESS;
2494 : }
2495 :
2496 0 : HcclResult hrtRaGetCqeErrInfo(unsigned int phyId, struct CqeErrInfo *info)
2497 : {
2498 0 : s32 ret = DlRaFunction::GetInstance().dlRaGetCqeErrInfo(phyId, info);
2499 0 : CHK_PRT_RET(ret != 0, HCCL_ERROR("[hrtRaGetCqeErrInfo]Get Cqe err info failed"), HCCL_E_NETWORK);
2500 0 : return HCCL_SUCCESS;
2501 : }
2502 0 : HcclResult hrtRaGetCqeErrInfoList(RdmaHandle rdmaHandle, struct CqeErrInfo *infolist, u32 *num)
2503 : {
2504 0 : CHK_PTR_NULL(rdmaHandle);
2505 0 : CHK_PTR_NULL(DlRaFunction::GetInstance().dlRaGetCqeErrInfoList);
2506 0 : s32 ret = DlRaFunction::GetInstance().dlRaGetCqeErrInfoList(rdmaHandle, infolist, num);
2507 0 : CHK_PRT_RET(ret != 0, HCCL_ERROR("[dlRaGetCqeErrInfoList]Get Cqe err info list failed"), HCCL_E_NETWORK);
2508 0 : return HCCL_SUCCESS;
2509 : }
2510 :
2511 0 : HcclResult IsSuppCqeErrInfoListConfig(bool& supCqeErrInfoListConfig)
2512 : {
2513 0 : u32 phyId = 0; // phyId无实际意义，这里直接传入0
2514 0 : u32 configVersion = 0;
2515 0 : supCqeErrInfoListConfig = false;
2516 :
2517 : // 获取版本号查看是否兼容
2518 0 : HcclResult ret = hrtRaGetInterfaceVersion(phyId, CQE_ERR_INFO_LIST_INTERFACE, &configVersion);
2519 0 : CHK_PRT_RET(ret == HCCL_E_NETWORK, HCCL_ERROR("[IsSuppportCqeErrInfoListConfig]hrtRaGetInterfaceVersion "\
2520 : "failed, interface[%u]", CQE_ERR_INFO_INTERFACE), ret);
2521 0 : if (ret == HCCL_E_NOT_SUPPORT) {
2522 0 : HCCL_WARNING("this package does not support hrtRaGetInterfaceVersion, please change new package");
2523 0 : return HCCL_SUCCESS;
2524 : }
2525 :
2526 0 : if (configVersion >= CQE_ERR_INFO_SUP_VER) {
2527 0 : supCqeErrInfoListConfig = true;
2528 : }
2529 0 : HCCL_INFO("IsSuppportCqeErrInfoListConfig support:%d", supCqeErrInfoListConfig);
2530 0 : return HCCL_SUCCESS;
2531 : }
2532 :
2533 0 : HcclResult IsSupportRaSendNormalWrlist(bool& isSupportRaSendNormalWrlist)
2534 : {
2535 0 : s32 deviceLogicID = -1;
2536 0 : u32 devicePhyId = 0;
2537 0 : CHK_RET(hrtGetDevice(&deviceLogicID));
2538 0 : CHK_RET(hrtGetDevicePhyIdByIndex(static_cast<u32>(deviceLogicID), devicePhyId));
2539 0 : u32 configVersion = 0;
2540 0 : isSupportRaSendNormalWrlist = false;
2541 :
2542 : // 获取版本号查看是否兼容
2543 0 : HcclResult ret = hrtRaGetInterfaceVersion(devicePhyId, SEND_NORMAL_WRLIST, &configVersion);
2544 0 : CHK_PRT_RET(ret == HCCL_E_NETWORK, HCCL_ERROR("[IsSupportRaSendNormalWrlist]hrtRaGetInterfaceVersion "\
2545 : "failed, interface[%u]", CQE_ERR_INFO_INTERFACE), ret);
2546 0 : if (ret == HCCL_E_NOT_SUPPORT) {
2547 0 : HCCL_WARNING("this package does not support hrtRaGetInterfaceVersion, please change new package");
2548 0 : return HCCL_SUCCESS;
2549 : }
2550 :
2551 0 : if (configVersion >= SEND_NORMAL_WRLIST_VERSION) {
2552 0 : isSupportRaSendNormalWrlist = true;
2553 : }
2554 0 : HCCL_INFO("IsSupportRaSendNormalWrlist support:%d", isSupportRaSendNormalWrlist);
2555 0 : return HCCL_SUCCESS;
2556 : }
2557 :
2558 :
2559 6 : HcclResult hrtRaGetQpAttr(QpHandle qpHandle, struct QpAttr *attr)
2560 : {
2561 6 : s32 ret = DlRaFunction::GetInstance().dlRaGetQpAttr(qpHandle, attr);
2562 6 : CHK_PRT_RET(ret != 0, HCCL_ERROR("Get qpn info failed"), HCCL_E_NETWORK);
2563 6 : return HCCL_SUCCESS;
2564 : }
2565 :
2566 0 : HcclResult hrtRaCreateSrq(RdmaHandle rdmaHandle, SrqInfo &srqInfo)
2567 : {
2568 0 : struct SrqAttr attr = {nullptr};
2569 0 : attr.ibSrq = &srqInfo.srq;
2570 0 : attr.ibRecvCq = &srqInfo.srqCq;
2571 0 : attr.maxSge = MAX_RECV_SGE_NUM;
2572 0 : attr.context = &srqInfo.context;
2573 0 : attr.srqEventId = srqInfo.srqEvent;
2574 0 : attr.srqDepth = srqInfo.srqDepth;
2575 0 : attr.cqDepth = MAX_CQ_DEPTH;
2576 0 : s32 ret = DlRaFunction::GetInstance().dlRaCreateSrq(rdmaHandle, &attr);
2577 0 : CHK_PRT_RET(ret != 0, HCCL_ERROR("[Create][Srq]errNo[0x%016llx] ra create srq fail. "
2578 : "return[%d], params: rdmaHandle[%p]", HCCL_ERROR_CODE(HCCL_E_NETWORK),
2579 : ret, rdmaHandle), HCCL_E_NETWORK);
2580 :
2581 0 : return HCCL_SUCCESS;
2582 : }
2583 :
2584 0 : HcclResult hrtRaDestroySrq(RdmaHandle rdmaHandle, SrqInfo &srqInfo)
2585 : {
2586 0 : struct SrqAttr attr = {nullptr};
2587 0 : attr.context = &srqInfo.context;
2588 0 : attr.ibSrq = &srqInfo.srq;
2589 0 : s32 ret = DlRaFunction::GetInstance().dlRaDestroyeSrq(rdmaHandle, &attr);
2590 0 : CHK_PRT_RET(ret != 0, HCCL_ERROR("[Destroy][Srq]errNo[0x%016llx] ra destroy normal qp fail. "
2591 : "return[%d], params: rdmaHandle[%p]", HCCL_ERROR_CODE(HCCL_E_NETWORK),
2592 : ret, rdmaHandle), HCCL_E_NETWORK);
2593 0 : return HCCL_SUCCESS;
2594 : }
2595 :
2596 13 : HcclResult hrtRaCreateEventHandle(s32 &eventHandle)
2597 : {
2598 13 : if (DlRaFunction::GetInstance().dlRaCreateEventHandle == nullptr) {
2599 0 : HCCL_ERROR("driver package does not support hrtRaCreateEventHandle, please change new package");
2600 0 : return HCCL_E_NOT_SUPPORT;
2601 : }
2602 13 : s32 ret = DlRaFunction::GetInstance().dlRaCreateEventHandle(&eventHandle);
2603 13 : CHK_PRT_RET(ret != 0, HCCL_ERROR("Create event handle failed, ret is [%d]", ret), HCCL_E_NETWORK);
2604 13 : return HCCL_SUCCESS;
2605 : }
2606 :
2607 0 : HcclResult hrtRaCtlEventHandle(s32 eventHandle, const FdHandle fdHandle, int opCode, HcclEpollEvent event)
2608 : {
2609 0 : if (DlRaFunction::GetInstance().dlRaCtlEventHandle == nullptr) {
2610 0 : HCCL_ERROR("driver package does not support hrtRaCtlEventHandle, please change new package");
2611 0 : return HCCL_E_NOT_SUPPORT;
2612 : }
2613 0 : RaEpollEvent epollEvent = static_cast<RaEpollEvent>(event);
2614 0 : CHK_PRT_RET((epollEvent < RA_EPOLLIN) && (epollEvent >= RA_EPOLLINVALD),
2615 : HCCL_ERROR("epoll event[%d] is invalid", epollEvent), HCCL_E_NETWORK);
2616 0 : s32 ret = DlRaFunction::GetInstance().dlRaCtlEventHandle(eventHandle, fdHandle, opCode, epollEvent);
2617 0 : CHK_PRT_RET(ret != 0, HCCL_ERROR("Control event handle failed, ret is [%d]", ret), HCCL_E_NETWORK);
2618 0 : return HCCL_SUCCESS;
2619 : }
2620 :
2621 0 : HcclResult hrtRaWaitEventHandle(s32 eventHandle, std::vector<SocketEventInfo> &eventInfos, s32 timeOut,
2622 : u32 maxEvents, u32 &eventsNum)
2623 : {
2624 0 : if (DlRaFunction::GetInstance().dlRaWaitEventHandle == nullptr) {
2625 0 : HCCL_ERROR("driver package does not support hrtRaWaitEventHandle, please change new package");
2626 0 : return HCCL_E_NOT_SUPPORT;
2627 : }
2628 0 : std::vector<struct SocketEventInfoT> raEventInfos(maxEvents);
2629 0 : s32 ret = DlRaFunction::GetInstance().dlRaWaitEventHandle(eventHandle, raEventInfos.data(), timeOut, maxEvents,
2630 : &eventsNum);
2631 0 : CHK_PRT_RET(ret != 0, HCCL_ERROR("Wait event handle failed, ret is [%d]", ret), HCCL_E_NETWORK);
2632 0 : for (u32 i = 0; i < eventsNum; i++) {
2633 0 : eventInfos[i].fdHandle = raEventInfos[i].fdHandle;
2634 : }
2635 0 : return HCCL_SUCCESS;
2636 0 : }
2637 :
2638 13 : HcclResult hrtRaDestroyEventHandle(s32 &eventHandle)
2639 : {
2640 13 : if (DlRaFunction::GetInstance().dlRaDestroyEventHandle == nullptr) {
2641 0 : HCCL_ERROR("driver package does not support hrtRaDestroyEventHandle, please change new package");
2642 0 : return HCCL_E_NOT_SUPPORT;
2643 : }
2644 13 : s32 ret = DlRaFunction::GetInstance().dlRaDestroyEventHandle(&eventHandle);
2645 13 : CHK_PRT_RET(ret != 0, HCCL_ERROR("Destroy event handle failed, ret is [%d]", ret), HCCL_E_NETWORK);
2646 13 : return HCCL_SUCCESS;
2647 : }
2648 :
2649 0 : HcclResult hrtRaQpCreateWithAttrs(RdmaHandle rdmaHandle, struct QpExtAttrs *attrs, QpHandle &qpHandle)
2650 : {
2651 0 : string qpInfo = string("rdmaHandle:[") + to_string(reinterpret_cast<intptr_t>(rdmaHandle)) + string("],qpHandle:[") +
2652 0 : to_string(reinterpret_cast<intptr_t>(&qpHandle)) + string("]; qp attr:[qpMode:") + to_string(attrs->qpMode) +
2653 0 : string(",udpSport:") + to_string(attrs->udpSport) + string(",version:") + to_string(attrs->version) +
2654 0 : string(",memAlign:") + to_string(attrs->memAlign) + string("]; cq attr: [sendCqDepth:") +
2655 0 : to_string(attrs->cqAttr.sendCqDepth) + string(",recvCqDepth:") + to_string(attrs->cqAttr.recvCqDepth) +
2656 0 : string(",sendCqCompVector:") + to_string(attrs->cqAttr.sendCqCompVector) +
2657 0 : string(",recvCqCompVector:") + to_string(attrs->cqAttr.recvCqCompVector) + string(",cap.max_send_wr:") +
2658 0 : to_string(attrs->qpAttr.cap.max_send_wr) + string(",cap.max_recv_wr:") +
2659 0 : to_string(attrs->qpAttr.cap.max_recv_wr) + "]";
2660 :
2661 0 : s32 ret = DlRaFunction::GetInstance().dlRaQpCreateWithAttrs(rdmaHandle, attrs, &qpHandle);
2662 0 : if (ret == ROCE_ENOMEM_RET && GetExternalInputRdmaFastPost()) {
2663 0 : HCCL_ERROR("[%s]create qp failed because of memory error, you can try to unset HCCL_RDMA_PCIE_DIRECT_POST_NOSTRICT and execute again",
2664 : __func__);
2665 : }
2666 :
2667 0 : CHK_OOM_RET(ret, qpInfo.c_str());
2668 :
2669 0 : CHK_PRT_RET(ret != 0 || (qpHandle == nullptr),
2670 : HCCL_ERROR("[Create][RaQp]errNo[0x%016llx] ra qp create with attrs fail. qpInfo:[%s], return: ret[%d]",
2671 : HCCL_ERROR_CODE(HCCL_E_NETWORK), qpInfo.c_str(), ret),
2672 : HCCL_E_NETWORK);
2673 :
2674 0 : struct QpAttr attr{};
2675 0 : CHK_RET(hrtRaGetQpAttr(qpHandle, &attr));
2676 0 : s32 deviceId = 0;
2677 0 : if (hrtGetDevice(&deviceId) != HCCL_SUCCESS) {
2678 0 : deviceId = -1;
2679 : }
2680 0 : PLF_CONFIG_DEBUG(PLF_RES, "Create Qp para: deviceId[%d] qpn[%u] qpInfo[%s]", deviceId, attr.qpn, qpInfo.c_str());
2681 0 : return HCCL_SUCCESS;
2682 0 : }
2683 :
2684 0 : HcclResult hrtRaQpCreateWithCQWithAttrs(RdmaHandle rdmaHandle, struct QpExtAttrs *attrs,
2685 : unsigned int sendCqn, unsigned int recvCqn, QpHandle &qpHandle)
2686 : {
2687 0 : s32 ret = DlRaFunction::GetInstance().dlRaQpCreateWithCQWithAttrs(rdmaHandle, attrs, sendCqn, recvCqn, &qpHandle);
2688 0 : if (ret != 0 || qpHandle == nullptr) {
2689 0 : HCCL_ERROR("[Create][RaQpWithCQ] ra qp create with cq with attrs fail. ret[%d]", ret);
2690 0 : return HCCL_E_NETWORK;
2691 : }
2692 :
2693 0 : struct QpAttr attr{};
2694 0 : CHK_RET(hrtRaGetQpAttr(qpHandle, &attr));
2695 0 : s32 deviceId = 0;
2696 0 : if (hrtGetDevice(&deviceId) != HCCL_SUCCESS) {
2697 0 : deviceId = -1;
2698 : }
2699 0 : PLF_CONFIG_DEBUG(PLF_RES, "Create QpWithCQ para: deviceId[%d] qpn[%u]", deviceId, attr.qpn);
2700 0 : return HCCL_SUCCESS;
2701 : }
2702 :
2703 0 : HcclResult hrtRaAiQpCreate(u32 phyId, RdmaHandle rdmaHandle, struct QpExtAttrs *attrs,
2704 : struct AiQpInfo *info, QpHandle &qpHandle)
2705 : {
2706 0 : u32 aiQpCreateVersion = 0;
2707 0 : HcclResult vRet = hrtRaGetInterfaceVersion(phyId, AI_QP_CREATE, &aiQpCreateVersion);
2708 0 : if (vRet != HCCL_SUCCESS || aiQpCreateVersion < AI_QP_CREATE_VERSION) {
2709 0 : HCCL_ERROR("this package does not support hrtRaAiQpCreate for device, please change new package");
2710 0 : return HCCL_E_NOT_SUPPORT;
2711 : }
2712 0 : s32 ret = DlRaFunction::GetInstance().dlRaAiQpCreate(rdmaHandle, attrs, info, &qpHandle);
2713 :
2714 0 : string qpInfo = string("qp attr:[qpMode:") + to_string(attrs->qpMode) +
2715 0 : string(",udpSport:") + to_string(attrs->udpSport) + string(",version:") + to_string(attrs->version) +
2716 0 : string(",memAlign:") + to_string(attrs->memAlign) + string("]; cq attr: [sendCqDepth:") +
2717 0 : to_string(attrs->cqAttr.sendCqDepth) + string(",recvCqDepth:") + to_string(attrs->cqAttr.recvCqDepth) +
2718 0 : string(",sendCqCompVector:") + to_string(attrs->cqAttr.sendCqCompVector) +
2719 0 : string(",recvCqCompVector:") + to_string(attrs->cqAttr.recvCqCompVector) + string(",cap.max_send_wr:") +
2720 0 : to_string(attrs->qpAttr.cap.max_send_wr) + string(",cap.max_recv_wr:") +
2721 0 : to_string(attrs->qpAttr.cap.max_recv_wr) + "]";
2722 :
2723 0 : CHK_OOM_RET(ret, qpInfo.c_str());
2724 :
2725 0 : CHK_PRT_RET(ret != 0 || (qpHandle == nullptr),
2726 : HCCL_ERROR("[Create][RaAiQp]errNo[0x%016llx] ra ai qp create fail. "
2727 : "return: ret[%d]",
2728 : HCCL_ERROR_CODE(HCCL_E_NETWORK),
2729 : ret),
2730 : HCCL_E_NETWORK);
2731 :
2732 0 : struct QpAttr attr{};
2733 0 : CHK_RET(hrtRaGetQpAttr(qpHandle, &attr));
2734 0 : s32 deviceId = 0;
2735 0 : if (hrtGetDevice(&deviceId) != HCCL_SUCCESS) {
2736 0 : deviceId = -1;
2737 : }
2738 0 : PLF_CONFIG_DEBUG(PLF_RES,
2739 : "Create Qp para: deviceId[%d] qpn[%u] sq_depth[%u] rq_depth[%u] scq_depth[%u] rcq_depth[%u]",
2740 : deviceId, attr.qpn, attrs->qpAttr.cap.max_send_wr, attrs->qpAttr.cap.max_recv_wr,
2741 : attrs->cqAttr.sendCqDepth, attrs->cqAttr.recvCqDepth);
2742 0 : return HCCL_SUCCESS;
2743 0 : }
2744 :
2745 0 : HcclResult hrtRaRecvWrlist(QpHandle handle, struct RecvWrlistData *wr, unsigned int recvNum,
2746 : unsigned int *completeNum)
2747 : {
2748 0 : if (DlRaFunction::GetInstance().dlRaRecvWrlist == nullptr) {
2749 0 : HCCL_ERROR("[Recv][RaWrlist]driver package does not support hrtRaRecvWrlist interface, "\
2750 : "please change new one");
2751 0 : return HCCL_E_NOT_SUPPORT;
2752 : }
2753 0 : s32 ret = 0;
2754 0 : auto startTime = std::chrono::steady_clock::now();
2755 0 : auto timeout = std::chrono::seconds(GetExternalInputHcclLinkTimeOut());
2756 0 : u32 remainNum = 0;
2757 0 : unsigned int completeNumLocal = 0;
2758 0 : *completeNum = 0;
2759 : while (true) {
2760 0 : if (remainNum == recvNum) {
2761 0 : break;
2762 : }
2763 :
2764 0 : ret = DlRaFunction::GetInstance().dlRaRecvWrlist(handle, wr + remainNum, recvNum, &completeNumLocal);
2765 0 : *completeNum += completeNumLocal;
2766 :
2767 0 : if (!ret) {
2768 0 : break; // 成功跳出
2769 0 : } else if ((ret == SOCK_ENOENT) || (ret == SOCK_EAGAIN) ||
2770 0 : (GetWorkflowMode() == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE && ret == ROCE_ENOMEM)) {
2771 0 : remainNum += completeNumLocal;
2772 0 : bool bTimeout = ((std::chrono::steady_clock::now() - startTime) >= timeout);
2773 0 : CHK_PRT_RET(bTimeout, HCCL_ERROR("[Recv][RaWrList]errNo[0x%016llx] ra Recv wrlsit async timeout[%d s]. "\
2774 : "return[%d], params: send_wrAddr[%p]",
2775 : HCCL_ERROR_CODE(HCCL_E_ROCE_TRANSFER), timeout, ret, wr), HCCL_E_ROCE_TRANSFER);
2776 0 : SaluSleep(ONE_MILLISECOND_OF_USLEEP);
2777 : } else {
2778 0 : HCCL_ERROR("[Recv][RaWr]ra wr list Recv async fail. return[%d], para: Recv_wrAddr[%p]", ret, wr);
2779 0 : return HCCL_E_ROCE_TRANSFER; // 非-2/-11场景错误，不轮询，直接退出
2780 : }
2781 0 : }
2782 0 : return HCCL_SUCCESS;
2783 : }
2784 :
2785 : std::mutex g_deviceVnicIpMutex;
2786 : map<u32, HcclIpAddress> g_deviceIdVnicInfoMap; // 记录deviceid和vnic ip的关系，用于非超节点模式server内查询，避免重复查询
2787 : map<u32, HcclIpAddress> g_sdidVnicInfoMap; // 记录sdid和vnic ip的关系，用于超节点模式，避免重复查询
2788 962 : HcclResult IsSuppportRaGetSocketVnicIps(bool& supportGetSocketVnicIp)
2789 : {
2790 962 : u32 phyId = 0; // phyId无实际意义，这里直接传入0
2791 962 : u32 supportGetSocketVnicIpVersion = 0;
2792 962 : supportGetSocketVnicIp = false;
2793 : // 获取版本号查看是否兼容
2794 962 : HcclResult ret = hrtRaGetInterfaceVersion(phyId, SOCKET_VNIC_IP_INFOS_INTERFACE, &supportGetSocketVnicIpVersion);
2795 960 : CHK_PRT_RET(ret == HCCL_E_NETWORK, HCCL_ERROR("[IsSuppportRaGetSocketVnicIps]hrtRaGetInterfaceVersion "\
2796 : "failed, interface[%u]", SOCKET_VNIC_IP_INFOS_INTERFACE), ret);
2797 960 : if (ret == HCCL_E_NOT_SUPPORT) {
2798 2 : HCCL_WARNING("this package does not support hrtRaGetInterfaceVersion, please change new package");
2799 2 : return HCCL_SUCCESS;
2800 : }
2801 :
2802 958 : if (supportGetSocketVnicIpVersion >= SOCKET_VNIC_IP_INFOS_SUP_VER) {
2803 812 : supportGetSocketVnicIp = true;
2804 : }
2805 :
2806 958 : return HCCL_SUCCESS;
2807 : }
2808 :
2809 807 : HcclResult hrtRaGetSocketVnicIpInfos(u32 phyId, enum IdType type, vector<u32> deviceIds,
2810 : vector<HcclIpAddress> &vnicIPs)
2811 : {
2812 807 : u32 vnicIpNum = deviceIds.size();
2813 809 : CHK_PRT_RET(vnicIpNum == 0, HCCL_ERROR("[hrtRaGetSocketVnicIpInfos]ra get VnicIp para error, num[%u]", vnicIpNum),
2814 : HCCL_E_PARA);
2815 809 : unique_lock<mutex> lock(g_deviceVnicIpMutex);
2816 814 : std::map<u32, HcclIpAddress> &vnicInfoMap = (type == PHY_ID_VNIC_IP) ? g_deviceIdVnicInfoMap : g_sdidVnicInfoMap;
2817 1628 : for (u32 i = 0; i < vnicIpNum; i++) {
2818 814 : HcclIpAddress vnicIP;
2819 814 : auto iter = vnicInfoMap.find(deviceIds[i]);
2820 : // 缓存查找到，直接从缓存获取
2821 814 : if (iter != vnicInfoMap.end()) {
2822 806 : vnicIP = iter->second;
2823 806 : HCCL_INFO("[hrtRaGetSocketVnicIpInfos] vnicInfoMap deviceIds[%u] found, Ip[%s]",
2824 : deviceIds[i], vnicIP.GetReadableAddress());
2825 : } else {
2826 8 : struct IpInfo vnicIpInfo = {};
2827 8 : s32 sRet = memset_s(&vnicIpInfo, sizeof(IpInfo), 0, sizeof(IpInfo));
2828 8 : CHK_PRT_RET(sRet != EOK,
2829 : HCCL_ERROR("[hrtRaGetSocketVnicIpInfos]errNo[0x%016llx] memset vnicIpInfo to 0 failed."
2830 : "params: dest[%p], dest_size[%zu], count[%zu]",
2831 : HCCL_ERROR_CODE(HCCL_E_SYSCALL), &vnicIpInfo, sizeof(IpInfo), sizeof(IpInfo)),
2832 : HCCL_E_SYSCALL);
2833 8 : s32 ret = DlRaFunction::GetInstance().dlRaGetSocketVnicIpInfos(phyId, type, &deviceIds[i], 1, &vnicIpInfo);
2834 8 : CHK_PRT_RET(ret != 0,
2835 : HCCL_ERROR("[hrtRaGetSocketVnicIpInfo]errNo[0x%016llx] ra get VnicIpfail. ret[%d]",
2836 : HCCL_ERROR_CODE(HCCL_E_TCP_CONNECT), ret),
2837 : HCCL_E_TCP_CONNECT);
2838 :
2839 : HcclInAddr temp;
2840 8 : temp.addr = vnicIpInfo.ip.addr;
2841 8 : temp.addr6 = vnicIpInfo.ip.addr6;
2842 8 : HcclIpAddress ipInfo(vnicIpInfo.family, temp);
2843 8 : CHK_PRT_RET(ipInfo.IsInvalid(), HCCL_ERROR("ip is invalid."), HCCL_E_PARA);
2844 8 : vnicInfoMap.insert({ deviceIds[i], ipInfo });
2845 8 : vnicIP = ipInfo;
2846 8 : HCCL_INFO("[hrtRaGetSocketVnicIpInfos] add vnicInfoMap, deviceIds[%u], Ip[%s]",
2847 : deviceIds[i], vnicIP.GetReadableAddress());
2848 8 : }
2849 814 : vnicIPs.push_back(vnicIP);
2850 814 : }
2851 814 : return HCCL_SUCCESS;
2852 814 : }
2853 :
2854 213 : HcclResult H2DTlvInit(struct TlvInitInfo *init_info, uint32_t *buffer_size, void **tlv_handle)
2855 : {
2856 213 : u32 tlvVersion = 0;
2857 213 : u32 phyId = 0; // phyId无实际意义，这里直接传入0
2858 213 : HcclResult vRet = hrtRaGetInterfaceVersion(phyId, TLV_INIT, &tlvVersion);
2859 213 : if (vRet != HCCL_SUCCESS || tlvVersion < TLV_VERSION) {
2860 212 : HCCL_WARNING("this package does not support H2DTlvInit for device, please change new package");
2861 212 : return HCCL_E_NOT_SUPPORT;
2862 : }
2863 :
2864 1 : s32 ret = DlRaFunction::GetInstance().dlH2DTlvInit(init_info, buffer_size, tlv_handle);
2865 1 : CHK_PRT_RET(ret != 0, HCCL_WARNING("[H2DTlvInit]errNo[0x%016llx] dlH2DTlvInit fail. "
2866 : "return: ret[%d]", HCCL_ERROR_CODE(HCCL_E_NETWORK), ret), HCCL_E_NETWORK);
2867 1 : return HCCL_SUCCESS;
2868 : }
2869 :
2870 0 : HcclResult H2DTlvRequest(void *tlv_handle, unsigned int module_type, struct TlvMsg *send_msg, struct TlvMsg *recv_msg)
2871 : {
2872 0 : u32 tlvVersion = 0;
2873 0 : u32 phyId = 0; // phyId无实际意义，这里直接传入0
2874 0 : HcclResult vRet = hrtRaGetInterfaceVersion(phyId, TLV_REQUEST, &tlvVersion);
2875 0 : if (vRet != HCCL_SUCCESS || tlvVersion < TLV_VERSION) {
2876 0 : HCCL_WARNING("this package does not support H2DTlvRequest for device, please change new package");
2877 0 : return HCCL_E_NOT_SUPPORT;
2878 : }
2879 :
2880 0 : if (DlRaFunction::GetInstance().dlH2DTlvRequest == nullptr) {
2881 0 : HCCL_WARNING("driver package does not support H2DTlvRequest, please change new package");
2882 0 : return HCCL_E_NOT_SUPPORT;
2883 : }
2884 :
2885 0 : s32 ret = DlRaFunction::GetInstance().dlH2DTlvRequest(tlv_handle, module_type, send_msg, recv_msg);
2886 0 : CHK_PRT_RET(ret != 0, HCCL_WARNING("[H2DTlvRequest]errNo[0x%016llx] dlH2DTlvRequest fail. module_type[%u]"
2887 : "return: ret[%d]", HCCL_ERROR_CODE(HCCL_E_NETWORK), module_type, ret), HCCL_E_NETWORK);
2888 0 : return HCCL_SUCCESS;
2889 : }
2890 :
2891 0 : HcclResult H2DTlvDeinit(void *tlv_handle)
2892 : {
2893 0 : u32 tlvVersion = 0;
2894 0 : u32 phyId = 0; // phyId无实际意义，这里直接传入0
2895 0 : HcclResult vRet = hrtRaGetInterfaceVersion(phyId, TLV_DEINIT, &tlvVersion);
2896 0 : if (vRet != HCCL_SUCCESS || tlvVersion < TLV_VERSION) {
2897 0 : HCCL_WARNING("this package does not support H2DTlvDeinit for device, please change new package");
2898 0 : return HCCL_E_NOT_SUPPORT;
2899 : }
2900 :
2901 0 : s32 ret = DlRaFunction::GetInstance().dlH2DTlvDeinit(tlv_handle);
2902 0 : CHK_PRT_RET(ret != 0, HCCL_WARNING("[H2DTlvDeinit]errNo[0x%016llx] ra tlv deinit fail. "
2903 : "return: ret[%d]", HCCL_ERROR_CODE(HCCL_E_NETWORK), ret), HCCL_E_NETWORK);
2904 0 : return HCCL_SUCCESS;
2905 : }
2906 :
2907 962 : HcclResult hrtRaGetSingleSocketVnicIpInfo(u32 phyId, DeviceIdType deviceIdType, u32 deviceId,
2908 : HcclIpAddress &vnicIP)
2909 : {
2910 962 : bool supportGetSocketVnicIp = false;
2911 962 : IsSuppportRaGetSocketVnicIps(supportGetSocketVnicIp);
2912 959 : if (!supportGetSocketVnicIp) {
2913 : // 非超节点场景，如果不支持查询vnicip，返回成功,继续使用phyid作为vnicip; 超节点如不支持，返错退出
2914 148 : return (deviceIdType == DeviceIdType::DEVICE_ID_TYPE_PHY_ID) ? (HCCL_SUCCESS) : (HCCL_E_NOT_SUPPORT);
2915 : }
2916 811 : std::vector<u32> deviceIds;
2917 812 : vector<HcclIpAddress> vnicIPs;
2918 812 : IdType idType = static_cast<IdType>(deviceIdType);
2919 812 : deviceIds.push_back(deviceId);
2920 808 : CHK_RET(hrtRaGetSocketVnicIpInfos(phyId, idType, deviceIds, vnicIPs));
2921 814 : vnicIP = vnicIPs[0];
2922 814 : HCCL_INFO("Get available Vnic info success, phyId[%u], deviceIdType[%d], deviceId[0x%x], Vnic ip[%s]", phyId, idType, deviceId,
2923 : vnicIP.GetReadableAddress());
2924 814 : return HCCL_SUCCESS;
2925 814 : }
2926 :
2927 2 : HcclResult hrtRaPingInit(struct PingInitAttr *initAttr, struct PingInitInfo *initInfo, void **pingHandle)
2928 : {
2929 2 : if (DlRaFunction::GetInstance().dlRaPingInit == nullptr) {
2930 1 : HCCL_ERROR("driver package does not support hrtRaPingInit, please change new package");
2931 1 : return HCCL_E_NOT_SUPPORT;
2932 : }
2933 1 : s32 ret = DlRaFunction::GetInstance().dlRaPingInit(initAttr, initInfo, pingHandle);
2934 1 : CHK_PRT_RET(ret != 0, HCCL_ERROR("Rping init failed, ret is [%d]", ret), HCCL_E_NOT_SUPPORT);
2935 1 : return HCCL_SUCCESS;
2936 : }
2937 :
2938 2 : HcclResult hrtRaPingDeinit(void *pingHandle)
2939 : {
2940 2 : if (DlRaFunction::GetInstance().dlRaPingDeinit == nullptr) {
2941 1 : HCCL_ERROR("driver package does not support hrtRaPingDeinit, please change new package");
2942 1 : return HCCL_E_NOT_SUPPORT;
2943 : }
2944 1 : s32 ret = DlRaFunction::GetInstance().dlRaPingDeinit(pingHandle);
2945 1 : CHK_PRT_RET(ret != 0, HCCL_ERROR("Rping deinit failed, ret is [%d]", ret), HCCL_E_NOT_SUPPORT);
2946 1 : return HCCL_SUCCESS;
2947 : }
2948 :
2949 2 : HcclResult hrtRaPingTargetAdd(void *pingHandle, struct PingTargetInfo target[], uint32_t num)
2950 : {
2951 2 : if (DlRaFunction::GetInstance().dlRaPingTargetAdd == nullptr) {
2952 1 : HCCL_ERROR("driver package does not support hrtRaPingTargetAdd, please change new package");
2953 1 : return HCCL_E_NOT_SUPPORT;
2954 : }
2955 1 : s32 ret = DlRaFunction::GetInstance().dlRaPingTargetAdd(pingHandle, target, num);
2956 1 : CHK_PRT_RET(ret != 0, HCCL_ERROR("Rping add target failed, ret is [%d], num[%u]", ret, num), HCCL_E_NOT_SUPPORT);
2957 1 : return HCCL_SUCCESS;
2958 : }
2959 :
2960 2 : HcclResult hrtRaPingTargetDel(void *pingHandle, struct PingTargetCommInfo target[], uint32_t num)
2961 : {
2962 2 : if (DlRaFunction::GetInstance().dlRaPingTargetDel == nullptr) {
2963 1 : HCCL_ERROR("driver package does not support hrtRaPingTargetDel, please change new package");
2964 1 : return HCCL_E_NOT_SUPPORT;
2965 : }
2966 1 : s32 ret = DlRaFunction::GetInstance().dlRaPingTargetDel(pingHandle, target, num);
2967 1 : CHK_PRT_RET(ret != 0, HCCL_ERROR("Rping delete target failed, ret is [%d], num[%u]", ret, num), HCCL_E_NOT_SUPPORT);
2968 1 : return HCCL_SUCCESS;
2969 : }
2970 :
2971 2 : HcclResult hrtRaPingTaskStart(void *pingHandle, struct PingTaskAttr *attr)
2972 : {
2973 2 : if (DlRaFunction::GetInstance().dlRaPingTaskStart == nullptr) {
2974 1 : HCCL_ERROR("driver package does not support hrtRaPingTaskStart, please change new package");
2975 1 : return HCCL_E_NOT_SUPPORT;
2976 : }
2977 1 : s32 ret = DlRaFunction::GetInstance().dlRaPingTaskStart(pingHandle, attr);
2978 1 : CHK_PRT_RET(ret != 0, HCCL_ERROR("Rping start task failed, ret is [%d]", ret), HCCL_E_NOT_SUPPORT);
2979 1 : return HCCL_SUCCESS;
2980 : }
2981 :
2982 2 : HcclResult hrtRaPingTaskStop(void *pingHandle)
2983 : {
2984 2 : if (DlRaFunction::GetInstance().dlRaPingTaskStop == nullptr) {
2985 1 : HCCL_ERROR("driver package does not support hrtRaPingTaskStop, please change new package");
2986 1 : return HCCL_E_NOT_SUPPORT;
2987 : }
2988 1 : s32 ret = DlRaFunction::GetInstance().dlRaPingTaskStop(pingHandle);
2989 1 : CHK_PRT_RET(ret != 0, HCCL_ERROR("Rping stop task failed, ret is [%d]", ret), HCCL_E_NOT_SUPPORT);
2990 1 : return HCCL_SUCCESS;
2991 : }
2992 :
2993 2 : HcclResult hrtRaPingGetResults(void *pingHandle, struct PingTargetResult target[], uint32_t *num)
2994 : {
2995 2 : if (DlRaFunction::GetInstance().dlRaPingGetResults == nullptr) {
2996 1 : HCCL_ERROR("driver package does not support hrtRaPingGetResults, please change new package");
2997 1 : return HCCL_E_NOT_SUPPORT;
2998 : }
2999 1 : s32 ret = DlRaFunction::GetInstance().dlRaPingGetResults(pingHandle, target, num);
3000 1 : CHK_PRT_RET(ret == ROCE_EAGAIN, HCCL_WARNING("Rping get results busy, try again", ret), HCCL_E_AGAIN);
3001 1 : CHK_PRT_RET(ret != 0, HCCL_ERROR("Rping get results failed, ret is [%d]", ret), HCCL_E_NOT_SUPPORT);
3002 1 : return HCCL_SUCCESS;
3003 : }
3004 :
3005 1 : HcclResult hrtRaIsFirstUsed(s32 insId, bool &used)
3006 : {
3007 1 : CHK_SMART_PTR_NULL(DlRaFunction::GetInstance().dlRaIsFirstUsed);
3008 1 : s32 ret = DlRaFunction::GetInstance().dlRaIsFirstUsed(insId);
3009 :
3010 1 : CHK_PRT_RET(ret != 0 && (ret != static_cast<s32>(true)), HCCL_ERROR("[hrtRaIsFirstUsed]errNo[0x%016llx] "
3011 : "failed ret[%d]", HCCL_ERROR_CODE(HCCL_E_NETWORK), ret), HCCL_E_NETWORK);
3012 :
3013 1 : used = ret == 0 ? false : true;
3014 :
3015 1 : HCCL_DEBUG("hrtRaIsFirstUsed insId[%d] success.", insId);
3016 1 : return HCCL_SUCCESS;
3017 : }
3018 :
3019 0 : HcclResult hrtRaIsLastUsed(s32 insId, bool &used)
3020 : {
3021 0 : CHK_SMART_PTR_NULL(DlRaFunction::GetInstance().dlRaIsLastUsed);
3022 0 : s32 ret = DlRaFunction::GetInstance().dlRaIsLastUsed(insId);
3023 :
3024 0 : CHK_PRT_RET(ret != 0 && (ret != static_cast<s32>(true)), HCCL_ERROR("[hrtRaIsLastUsed]errNo[0x%016llx] "
3025 : "failed ret[%d]", HCCL_ERROR_CODE(HCCL_E_NETWORK), ret), HCCL_E_NETWORK);
3026 :
3027 0 : used = ret == 0 ? false : true;
3028 :
3029 0 : HCCL_DEBUG("hrtRaIsLastUsed insId[%d] success.", insId);
3030 0 : return HCCL_SUCCESS;
3031 : }
3032 :
3033 0 : HcclResult hrtRaRdevGetPortStatus(RdmaHandle rdmaHandle, enum PortStatus *status)
3034 : {
3035 0 : CHK_PTR_NULL(rdmaHandle);
3036 0 : CHK_SMART_PTR_NULL(DlRaFunction::GetInstance().dlRaRdevGetPortStatus);
3037 0 : s32 ret = DlRaFunction::GetInstance().dlRaRdevGetPortStatus(rdmaHandle, status);
3038 :
3039 0 : CHK_PRT_RET(ret != 0, HCCL_ERROR("[hrtRaRdevGetPortStatus]errNo[0x%016llx] "
3040 : "failed ret[%d]", HCCL_ERROR_CODE(HCCL_E_NETWORK), ret), HCCL_E_NETWORK);
3041 0 : return HCCL_SUCCESS;
3042 : }
3043 :
3044 0 : HcclResult HrtRaRemapMr(RdmaHandle rdmaHandle, struct MemRemapInfo info[], unsigned int num)
3045 : {
3046 0 : CHK_PTR_NULL(rdmaHandle);
3047 0 : if (UNLIKELY(DlRaFunction::GetInstance().dlRaRemapMr == nullptr)) {
3048 0 : HCCL_ERROR("driver package does not support HrtRaRemapMr, please change new package");
3049 0 : return HCCL_E_NETWORK;
3050 : };
3051 0 : s32 ret = DlRaFunction::GetInstance().dlRaRemapMr(rdmaHandle, info, num);
3052 :
3053 0 : CHK_PRT_RET(ret != 0, HCCL_ERROR("[HrtRaRemapMr]errNo[0x%016llx] "
3054 : "failed ret[%d]", HCCL_ERROR_CODE(HCCL_E_NETWORK), ret), HCCL_E_NETWORK);
3055 0 : return HCCL_SUCCESS;
3056 : }
3057 :
3058 11 : HcclResult CreateQpWithDepthConfig(RdmaHandle rdmaHandle, s32 qpMode, const QpConfigInfo& qpConfig, QpHandle &qpHandle, struct TypicalQp& qpInfo)
3059 : {
3060 11 : HCCL_DEBUG("CreateQp qpMode[%d], sq_depth[%u], rq_depth[%u], scq_depth[%u], rcq_depth[%u], TC[%u], SL[%u], rdmaRetryCnt[%u], rdmaTimeOut[%u]",
3061 : qpMode, qpConfig.sq_depth, qpConfig.rq_depth, qpConfig.scq_depth, qpConfig.rcq_depth, qpInfo.tc, qpInfo.sl, qpInfo.retryCnt,
3062 : qpInfo.retryTime);
3063 :
3064 11 : struct QpExtAttrs ext_attrs{};
3065 11 : ext_attrs.qpMode = qpMode;
3066 11 : ext_attrs.cqAttr.sendCqDepth = qpConfig.scq_depth;
3067 11 : ext_attrs.cqAttr.recvCqDepth = qpConfig.rcq_depth;
3068 11 : ext_attrs.qpAttr.cap.max_send_wr = qpConfig.sq_depth;
3069 11 : ext_attrs.qpAttr.cap.max_recv_wr = qpConfig.rq_depth;
3070 11 : ext_attrs.version = QP_CREATE_WITH_ATTR_VERSION;
3071 11 : ext_attrs.qpAttr.cap.max_inline_data = DEFAULT_MAX_INLINE_DATA;
3072 11 : ext_attrs.qpAttr.cap.max_send_sge = DEFAULT_MAX_SEND_SGE;
3073 11 : ext_attrs.qpAttr.cap.max_recv_sge = DEFAULT_MAX_RECV_SGE;
3074 11 : ext_attrs.qpAttr.qp_type = IBV_QPT_RC;
3075 11 : ext_attrs.udpSport = 0x0;
3076 11 : ext_attrs.cstmFlag.bs.useResvMem = qpConfig.use_resv_mem;
3077 11 : ext_attrs.resvMemPoolId = qpConfig.resv_mem_pool_id;
3078 11 : s32 deviceLogicID = -1;
3079 11 : u32 devicePhyId = 0;
3080 11 : CHK_RET(hrtGetDevice(&deviceLogicID));
3081 11 : u32 typicalQpModifyVersion = 0;
3082 11 : CHK_RET(hrtGetDevicePhyIdByIndex(static_cast<u32>(deviceLogicID), devicePhyId));
3083 : // ra_qp_create_with_attrs创建的QP, 后续要使用ra_typical_qp_modify 需要判断ra_typical_qp_modify对应opcode:RA_RS_TYPICAL_QP_MODIFY是否支持支持QP解耦socket建链
3084 11 : HcclResult vRet = hrtRaGetInterfaceVersion(devicePhyId, TYPICAL_QP_MODIFY, &typicalQpModifyVersion);
3085 11 : if (vRet != HCCL_SUCCESS || typicalQpModifyVersion < TYPICAL_QP_MODIFY_VERSION) {
3086 2 : HCCL_ERROR("this package does not support CreateQpWithDepthConfig for device, please change new package");
3087 2 : return HCCL_E_NOT_SUPPORT;
3088 : }
3089 :
3090 9 : CHK_RET(hrtRaQpCreateWithAttrs(rdmaHandle, &ext_attrs, qpHandle));
3091 :
3092 9 : struct QpAttr attr{};
3093 9 : HcclResult ret = hrtRaGetQpAttr(qpHandle, &attr);
3094 9 : if (ret != HCCL_SUCCESS) {
3095 0 : HCCL_ERROR("[CreateQpWithDepthConfig] hrtRaGetQpAttr failed, ret[%d].", ret);
3096 0 : HrtRaQpDestroy(qpHandle);
3097 0 : return ret;
3098 : }
3099 9 : qpInfo.qpn = attr.qpn;
3100 9 : qpInfo.gidIdx = attr.gidIdx;
3101 153 : for (uint32_t i = 0; i < HCCP_GID_RAW_LEN; i++) {
3102 144 : qpInfo.gid[i] = attr.gid[i];
3103 : }
3104 9 : qpInfo.psn = attr.psn;
3105 9 : HCCL_DEBUG("CreateQpWithDepthConfig qpn[%u], gidIdx[%u], psn[%u]", qpInfo.qpn, qpInfo.gidIdx, qpInfo.psn);
3106 9 : return HCCL_SUCCESS;
3107 : }
3108 :
3109 0 : HcclResult CreateQpWithCQConfig(RdmaHandle rdmaHandle, s32 qpMode, const QpConfigWithCQInfo& qpConfig,
3110 : QpHandle &qpHandle, struct TypicalQp& qpInfo)
3111 : {
3112 0 : HCCL_INFO("CreateQpWithCQ qpMode[%d], sq_depth[%u], rq_depth[%u], scq_depth[%u], rcq_depth[%u], "
3113 : "sendCqn[%u], recvCqn[%u], use_resv_mem[%u], resv_mem_pool_id[%u], "
3114 : "sq_sig_all[%d], max_send_sge[%u], max_recv_sge[%u], max_inline_data[%u]",
3115 : qpMode, qpConfig.sq_depth, qpConfig.rq_depth, qpConfig.scq_depth, qpConfig.rcq_depth,
3116 : qpConfig.sendCqn, qpConfig.recvCqn, qpConfig.use_resv_mem, qpConfig.resv_mem_pool_id,
3117 : qpConfig.sq_sig_all, qpConfig.max_send_sge, qpConfig.max_recv_sge, qpConfig.max_inline_data);
3118 :
3119 0 : struct QpExtAttrs ext_attrs{};
3120 0 : ext_attrs.qpMode = qpMode;
3121 0 : ext_attrs.cqAttr.sendCqDepth = qpConfig.scq_depth;
3122 0 : ext_attrs.cqAttr.recvCqDepth = qpConfig.rcq_depth;
3123 0 : ext_attrs.qpAttr.cap.max_send_wr = qpConfig.sq_depth;
3124 0 : ext_attrs.qpAttr.cap.max_recv_wr = qpConfig.rq_depth;
3125 0 : ext_attrs.version = QP_CREATE_WITH_ATTR_VERSION;
3126 0 : ext_attrs.qpAttr.cap.max_inline_data = qpConfig.max_inline_data;
3127 0 : ext_attrs.qpAttr.cap.max_send_sge = qpConfig.max_send_sge;
3128 0 : ext_attrs.qpAttr.cap.max_recv_sge = qpConfig.max_recv_sge;
3129 0 : ext_attrs.qpAttr.qp_type = IBV_QPT_RC;
3130 0 : ext_attrs.qpAttr.sq_sig_all = qpConfig.sq_sig_all;
3131 0 : ext_attrs.udpSport = 0x0;
3132 0 : ext_attrs.cstmFlag.bs.useResvMem = qpConfig.use_resv_mem;
3133 0 : ext_attrs.resvMemPoolId = qpConfig.resv_mem_pool_id;
3134 :
3135 0 : CHK_RET(hrtRaQpCreateWithCQWithAttrs(rdmaHandle, &ext_attrs, qpConfig.sendCqn, qpConfig.recvCqn, qpHandle));
3136 :
3137 0 : struct QpAttr attr{};
3138 0 : HcclResult ret = hrtRaGetQpAttr(qpHandle, &attr);
3139 0 : if (ret != HCCL_SUCCESS) {
3140 0 : HCCL_ERROR("[CreateQpWithCQConfig] hrtRaGetQpAttr failed, ret[%d].", ret);
3141 0 : HrtRaQpDestroy(qpHandle);
3142 0 : return ret;
3143 : }
3144 0 : qpInfo.qpn = attr.qpn;
3145 0 : qpInfo.gidIdx = attr.gidIdx;
3146 0 : for (uint32_t i = 0; i < HCCP_GID_RAW_LEN; i++) {
3147 0 : qpInfo.gid[i] = attr.gid[i];
3148 : }
3149 0 : qpInfo.psn = attr.psn;
3150 0 : HCCL_DEBUG("CreateQpWithCQConfig qpn[%u], gidIdx[%u], psn[%u]", qpInfo.qpn, qpInfo.gidIdx, qpInfo.psn);
3151 0 : return HCCL_SUCCESS;
3152 : }
3153 :
3154 13 : HcclResult HrtRaGetTlsEnable(struct RaInfo *info, bool *tlsEnable)
3155 : {
3156 13 : u32 tlsVersion = 0;
3157 13 : u32 phyId = 0; // phyId无实际意义，这里直接传入0
3158 13 : HcclResult vRet = hrtRaGetInterfaceVersion(phyId, GET_TLS_ENABLE, &tlsVersion);
3159 13 : if (vRet != HCCL_SUCCESS || tlsVersion < TLS_ENABLE_VERSION) {
3160 0 : HCCL_WARNING("this package does not support HrtRaGetTlsEnable for device, please change new package");
3161 0 : return HCCL_E_NOT_SUPPORT;
3162 : }
3163 13 : HCCL_DEBUG("HrtRaGetTlsEnable tlsVersion[%u]", tlsVersion);
3164 13 : s32 ret = DlRaFunction::GetInstance().dlRaRaGetTlsEnable(info, tlsEnable);
3165 13 : CHK_PRT_RET(ret != 0, HCCL_ERROR("[HrtRaGetTlsEnable]errNo[0x%016llx] "
3166 : "failed ret[%d]", HCCL_ERROR_CODE(HCCL_E_NETWORK), ret), HCCL_E_NETWORK);
3167 13 : HCCL_INFO("HrtRaGetTlsEnable phyId[%u], tlsEnable[%d]", info->phyId, *tlsEnable);
3168 13 : return HCCL_SUCCESS;
3169 : }
3170 :
3171 0 : HcclResult SnapShotSaveAction(s32 networkMode, u32 devicePhyId, HcclSaveSnapShotAction action)
3172 : {
3173 0 : HCCL_INFO("%s networkMode[%d], devicePhyId[%u], action[%d]", __func__, networkMode, devicePhyId, action);
3174 0 : struct RaInfo raInfo = {};
3175 0 : raInfo.mode = networkMode;
3176 0 : raInfo.phyId = devicePhyId;
3177 0 : s32 ret = DlRaFunction::GetInstance().dlRaSaveSnapShot(&raInfo, static_cast<enum SaveSnapshotAction>(action));
3178 0 : CHK_PRT_RET(ret != 0, HCCL_ERROR("%s errNo[0x%016llx] failed ret[%d], networkMode[%d], phyId[%u], action[%d]",
3179 : __func__, HCCL_ERROR_CODE(HCCL_E_NETWORK), ret, networkMode, devicePhyId, action), HCCL_E_NETWORK);
3180 0 : return HCCL_SUCCESS;
3181 : }
3182 :
3183 0 : HcclResult SnapShotRestoreAction(s32 networkMode, u32 devicePhyId)
3184 : {
3185 0 : HCCL_INFO("%s networkMode[%d], devicePhyId[%u]", __func__, networkMode, devicePhyId);
3186 : struct RaInfo raInfo;
3187 0 : raInfo.mode = networkMode;
3188 0 : raInfo.phyId = devicePhyId;
3189 0 : s32 ret = DlRaFunction::GetInstance().dlRaRestoreSnapShot(&raInfo);
3190 0 : CHK_PRT_RET(ret != 0, HCCL_ERROR("%s errNo[0x%016llx] failed ret[%d], networkMode[%d], phyId[%u]",
3191 : __func__, HCCL_ERROR_CODE(HCCL_E_NETWORK), ret, networkMode, devicePhyId), HCCL_E_NETWORK);
3192 0 : return HCCL_SUCCESS;
3193 : }
3194 :
3195 31 : HcclResult HrtRaGetHccnCfg(s32 networkMode, u32 devicePhyId, enum HccnCfgKeyT key, std::string &value)
3196 : {
3197 31 : u32 raGetHccnCfg = 0;
3198 31 : HcclResult vRet = hrtRaGetInterfaceVersion(devicePhyId, GET_HCCH_CFG, &raGetHccnCfg);
3199 : static bool isPrintWarning = false;
3200 62 : if (vRet != HCCL_SUCCESS || raGetHccnCfg < GET_HCCH_CFG_VERSION ||
3201 31 : UNLIKELY(DlRaFunction::GetInstance().dlRaGetHccnCfg == nullptr)) {
3202 31 : if (!isPrintWarning) {
3203 1 : HCCL_WARNING("[HrtRaGetHccnCfg] this package does not support HrtRaGetHccnCfg for device, "
3204 : "please change new package ret[%d], version[%lu]",
3205 : static_cast<int>(vRet),
3206 : raGetHccnCfg);
3207 1 : isPrintWarning = true;
3208 : }
3209 31 : return HCCL_SUCCESS;
3210 : }
3211 :
3212 0 : if ((key == HccnCfgKeyT::HCCN_RESV_MEM_INFO) && (raGetHccnCfg <= GET_HCCH_CFG_VERSION)) {
3213 0 : HCCL_WARNING("[HrtRaGetHccnCfg] this package does not support resvMem for device, "
3214 : "please change new package ret[%d], version[%lu]",
3215 : static_cast<int>(vRet), raGetHccnCfg);
3216 0 : return HCCL_SUCCESS;
3217 : }
3218 :
3219 0 : struct RaInfo raInfo = {};
3220 0 : raInfo.mode = networkMode;
3221 0 : raInfo.phyId = devicePhyId;
3222 :
3223 0 : HccnCfgKey hccnKey{HccnCfgKey::HCCN_CFG_UDP_PORT_MODE};
3224 0 : switch (key) {
3225 0 : case HccnCfgKeyT::HCCN_UDP_PORT_MODE:
3226 0 : hccnKey = HccnCfgKey::HCCN_CFG_UDP_PORT_MODE;
3227 0 : break;
3228 0 : case HccnCfgKeyT::HCCN_MULTI_QP_COUNT:
3229 0 : hccnKey = HccnCfgKey::HCCN_CFG_MULTI_QP_COUNT;
3230 0 : break;
3231 0 : case HccnCfgKeyT::HCCN_MULTI_QP_UDP_PORTS:
3232 0 : hccnKey = HccnCfgKey::HCCN_CFG_MULTI_QP_UDP_PORTS;
3233 0 : break;
3234 0 : case HccnCfgKeyT::HCCN_RESV_MEM_INFO:
3235 0 : hccnKey = HccnCfgKey::HCCN_CFG_RESV_MEM_INFO;
3236 0 : break;
3237 0 : default:
3238 0 : HCCL_ERROR("[HrtRaGetHccnCfg]not support key[%d]", key);
3239 0 : return HCCL_E_PARA;
3240 : }
3241 :
3242 0 : constexpr std::uint32_t READ_MAX_LEN = 1024 * 2;
3243 0 : std::vector<char> buffer(READ_MAX_LEN);
3244 0 : int actualLen = static_cast<int>(buffer.size());
3245 0 : s32 ret = DlRaFunction::GetInstance().dlRaGetHccnCfg(&raInfo, hccnKey, buffer.data(), &actualLen);
3246 0 : if (ret == 0 && actualLen == 0) { // 文件不存在的话 HCCP长度返回0,且ret为0
3247 0 : HCCL_WARNING("[HrtRaGetHccnCfg] device networkMode[%d] with phyId[%u], "
3248 : "get hccn config key[%d] info is empty. Possible reasons: "
3249 : "1. Device not need to use multi_qp/nslb-dp settings. "
3250 : " 2. In this package, hccn_tool not support multi_qp/nslb-dp settings. "
3251 : " 3. The right key not exist in device's config file or key's value is empty.",
3252 : networkMode,
3253 : devicePhyId,
3254 : key);
3255 0 : value.assign(buffer.data(), actualLen);
3256 0 : return HCCL_SUCCESS;
3257 : }
3258 0 : CHK_PRT_RET(ret != 0, // 其他
3259 : HCCL_ERROR("[HrtRaGetHccnCfg]errNo[0x%016llx] error occurred."
3260 : " networkMode[%d], devicePhyId[%u], key[%d], return: ret[%d]",
3261 : HCCL_ERROR_CODE(HCCL_E_NETWORK),
3262 : networkMode,
3263 : devicePhyId,
3264 : key,
3265 : ret),
3266 : HCCL_E_NETWORK);
3267 0 : value.assign(buffer.data(), actualLen != 0 && buffer[actualLen - 1] == '\0' ? actualLen - 1 : actualLen);
3268 0 : HCCL_DEBUG("[HrtRaGetHccnCfg]devicePhyId[%u] key[%d], value[%s], value len[%d]",devicePhyId, key, value.c_str(),
3269 : actualLen);
3270 0 : return HCCL_SUCCESS;
3271 : }
3272 :
3273 0 : HcclResult hrtRaGetSecRandom(struct RaInfo *info, unsigned int* token)
3274 : {
3275 0 : if (DlRaFunction::GetInstance().dlRaGetSecRandom == nullptr) {
3276 0 : HCCL_ERROR("driver package does not support dlRaGetSecRandom, please change new package");
3277 0 : return HCCL_E_NOT_SUPPORT;
3278 : }
3279 0 : s32 ret = DlRaFunction::GetInstance().dlRaGetSecRandom(info, token);
3280 0 : if (ret != 0) {
3281 0 : HCCL_ERROR("[HrtRaGetSecRandom] RaGetSecRandom failed, call interface, ret[%d]", ret);
3282 0 : return HCCL_E_NETWORK;
3283 : }
3284 0 : return HCCL_SUCCESS;
3285 : }
3286 :
3287 0 : HcclResult hrtRaGetDevEidInfoNum(RaInfo info, unsigned int* num)
3288 : {
3289 0 : if (DlRaFunction::GetInstance().dlRaGetDevEidInfoNum == nullptr) {
3290 0 : HCCL_ERROR("driver package does not support dlRaGetDevEidInfoNum, please change new package");
3291 0 : return HCCL_E_NOT_SUPPORT;
3292 : }
3293 0 : s32 ret = DlRaFunction::GetInstance().dlRaGetDevEidInfoNum(info, num);
3294 0 : if (ret != 0) {
3295 0 : HCCL_ERROR("[HrtRaGetSecRandom] RaGetDevEidInfoNum failed, call interface, ret[%d]", ret);
3296 0 : return HCCL_E_NETWORK;
3297 : }
3298 0 : return HCCL_SUCCESS;
3299 : }
3300 :
3301 0 : HcclResult hrtRaGetDevEidInfoList(RaInfo info, struct HccpDevEidInfo *eid_info, unsigned int* num)
3302 : {
3303 0 : if (DlRaFunction::GetInstance().dlRaGetDevEidInfoList == nullptr) {
3304 0 : HCCL_ERROR("driver package does not support dlRaGetDevEidInfoNum, please change new package");
3305 0 : return HCCL_E_NOT_SUPPORT;
3306 : }
3307 0 : s32 ret = DlRaFunction::GetInstance().dlRaGetDevEidInfoList(info, eid_info, num);
3308 0 : if (ret != 0) {
3309 0 : HCCL_ERROR("[HrtRaGetSecRandom] RaGetDevEidInfoList failed, call interface, ret[%d]", ret);
3310 0 : return HCCL_E_NETWORK;
3311 : }
3312 0 : return HCCL_SUCCESS;
3313 : }
|
