Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 : #include "orion_adapter_hccp.h"
11 : #include <chrono>
12 : #include <unistd.h>
13 : #include <memory>
14 : #include <unordered_map>
15 : #include "sal.h"
16 : #include "network_api_exception.h"
17 : #include "internal_exception.h"
18 : #include "hccp.h"
19 : #include "hccp_tlv.h"
20 : #include "hccp_ctx.h"
21 : #include "hccp_async.h"
22 : #include "env_config.h"
23 : #include "hccp_common.h"
24 : #include "exception_util.h"
25 : #include "adapter_error_manager_pub.h"
26 :
27 : using namespace std;
28 :
29 : namespace Hccl {
30 : constexpr u32 ONE_HUNDRED_MICROSECOND_OF_USLEEP = 100;
31 : constexpr u32 ONE_MILLISECOND_OF_USLEEP = 1000;
32 : constexpr unsigned int SOCKET_NUM_ONE = 1;
33 : constexpr u32 MAX_NUM_OF_WHITE_LIST_NUM = 16;
34 : constexpr u32 AUTO_LISTEN_PORT = 0;
35 : constexpr u64 SOCKET_SEND_MAX_SIZE = 0x7FFFFFFFFFFFFFFF;
36 : constexpr u32 MAX_WR_NUM = 1024;
37 : constexpr u32 MAX_SEND_SGE_NUM = 1;
38 : constexpr u32 MAX_RECV_SGE_NUM = 1;
39 : constexpr u32 MAX_CQ_DEPTH = 65535;
40 : constexpr u32 NDA_CQ_DEPTH_FOR_UBNIC = 31 * 1024;
41 : constexpr u32 NDA_CQ_DEPTH_FOR_XSCDV = 32 * 1024;
42 : constexpr u32 MAX_INLINE_DATA = 64;
43 : constexpr u32 RA_TLV_REQUEST_UNAVAIL = 128308;
44 : constexpr u32 ROCE_ENOMEM_RET = 328100;
45 : constexpr u32 GET_TP_ATTR_OPCODE = 106;
46 : constexpr u32 GET_TLS_ENABLE_OPCODE = 95;
47 : constexpr u32 GET_TLS_ENABLE_VERSION = 1;
48 : constexpr u32 GET_TP_ATTR_VERSION = 2;
49 :
50 : const std::unordered_map<HrtNetworkMode, NetworkMode, EnumClassHash> HRT_NETWORK_MODE_MAP
51 : = {{HrtNetworkMode::PEER, NetworkMode::NETWORK_PEER_ONLINE}, {HrtNetworkMode::HDC, NetworkMode::NETWORK_OFFLINE}};
52 :
53 : s32 g_linkTimeout = 0;
54 60 : inline s32 EnvLinkTimeoutGet()
55 : {
56 60 : g_linkTimeout = g_linkTimeout != 0 ? g_linkTimeout : EnvConfig::GetInstance().GetSocketConfig().GetLinkTimeOut();
57 60 : return g_linkTimeout;
58 : }
59 :
60 11 : HcclResult HrtRaGetTlsStatus(struct RaInfo *info, TlsStatus &tlsStatus)
61 : {
62 11 : tlsStatus = TlsStatus::UNKNOWN;
63 14 : CHK_PTR_NULL(info);
64 :
65 10 : u32 tlsVersion = 0;
66 10 : s32 versionRet = RaGetInterfaceVersion(info->phyId, GET_TLS_ENABLE_OPCODE, &tlsVersion);
67 10 : if (versionRet != 0 || tlsVersion < GET_TLS_ENABLE_VERSION) {
68 21 : HCCL_WARNING("[HrtRaGetTlsStatus] this package does not support RaGetTlsEnable for device, "
69 : "please change new package. ret[%d], tlsVersion[%u].", versionRet, tlsVersion);
70 7 : return HCCL_E_NOT_SUPPORT;
71 : }
72 :
73 3 : bool tlsEnable = false;
74 3 : s32 ret = RaGetTlsEnable(info, &tlsEnable);
75 3 : if (ret != 0) {
76 1 : tlsStatus = TlsStatus::DISABLE;
77 3 : HCCL_ERROR("[HrtRaGetTlsStatus] errNo[0x%016llx] failed ret[%d], phyId[%u]",
78 : HCCL_ERROR_CODE(HCCL_E_NETWORK), ret, info->phyId);
79 1 : return HCCL_E_NETWORK;
80 : }
81 :
82 2 : tlsStatus = tlsEnable ? TlsStatus::ENABLE : TlsStatus::DISABLE;
83 6 : HCCL_INFO("[HrtRaGetTlsStatus] phyId[%u], tlsEnable[%d], tlsStatus[%d]",
84 : info->phyId, tlsEnable, static_cast<s32>(tlsStatus));
85 2 : return HCCL_SUCCESS;
86 : }
87 :
88 88 : inline union HccpIpAddr IpAddressToHccpIpAddr(IpAddress &addr)
89 : {
90 : union HccpIpAddr hccpIpAddr;
91 88 : if (addr.GetFamily() == AF_INET) {
92 88 : hccpIpAddr.addr = addr.GetBinaryAddress().addr;
93 : } else {
94 0 : hccpIpAddr.addr6 = addr.GetBinaryAddress().addr6;
95 : }
96 88 : return hccpIpAddr;
97 : }
98 :
99 2 : inline IpAddress IfAddrInfoToIpAddress(struct InterfaceInfo info)
100 : {
101 : BinaryAddr addr;
102 2 : if (info.family == AF_INET) {
103 2 : addr.addr = info.ifaddr.ip.addr;
104 : } else {
105 0 : addr.addr6 = info.ifaddr.ip.addr6;
106 : }
107 4 : return IpAddress(addr, info.family, info.scopeId);
108 : }
109 :
110 6 : void* HrtRaTlvInit(HRaTlvInitConfig &cfg)
111 : {
112 18 : HCCL_INFO("[Init][RaTlv] Input params: version=[%d], phyId=[%u], mode=[%u]",
113 : cfg.version, cfg.phyId, cfg.mode);
114 6 : struct TlvInitInfo init_info {};
115 6 : init_info.version = cfg.version;
116 6 : init_info.phyId = cfg.phyId;
117 6 : init_info.nicPosition = HRT_NETWORK_MODE_MAP.at(cfg.mode);
118 :
119 6 : s32 ret = 0;
120 : unsigned int buffer_size;
121 : void *tlv_handle;
122 :
123 6 : ret = RaTlvInit(&init_info, &buffer_size, &tlv_handle);
124 6 : if (ret != 0 || tlv_handle == nullptr) {
125 4 : MACRO_THROW(NetworkApiException, StringFormat("[Init][RaTlv]errNo[0x%016llx] ra tlv init fail. params: mode=%u, device id=%u, version=%d, tlv_handle=%p, return: ret[%d]",
126 : HCCL_ERROR_CODE(HcclResult::HCCL_E_NETWORK), cfg.mode, init_info.phyId, init_info.version, tlv_handle, ret));
127 : }
128 :
129 15 : HCCL_INFO("tlv init success, device id[%u]", init_info.phyId);
130 :
131 5 : return tlv_handle;
132 : }
133 :
134 4 : HcclResult HrtRaTlvRequest(void* tlv_handle, u32 tlv_module_type, u32 tlv_ccu_msg_type)
135 : {
136 4 : CHK_PTR_NULL(tlv_handle);
137 :
138 12 : HCCL_INFO("[Request][RaTlv] Input params: tlv_handle=[%p], tlv_module_type=[%u], tlv_ccu_msg_type=[%u]",
139 : tlv_handle, tlv_module_type, tlv_ccu_msg_type);
140 4 : s32 ret = 0;
141 :
142 4 : struct TlvMsg send_msg {};
143 4 : struct TlvMsg recv_msg {};
144 4 : send_msg.type = tlv_ccu_msg_type;
145 :
146 4 : ret = RaTlvRequest(tlv_handle, tlv_module_type, &send_msg, &recv_msg);
147 4 : if (ret != 0) {
148 1 : if (ret == RA_TLV_REQUEST_UNAVAIL || ret == OTHERS_ENOTSUPP) {
149 0 : HCCL_WARNING("[HrtRaTlvRequest]ra tlv request UNAVAIL. return: ret[%d]", ret);
150 0 : return HCCL_E_UNAVAIL;
151 : }
152 4 : MACRO_THROW(NetworkApiException, StringFormat("[Request][RaTlv]errNo[0x%016llx] ra tlv request fail. params: tlv_handle=%p, tlv_module_type=%u, tlv_ccu_msg_type=%u, return: ret=%d",
153 : HCCL_ERROR_CODE(HcclResult::HCCL_E_NETWORK), tlv_handle, tlv_module_type, tlv_ccu_msg_type, ret));
154 : }
155 :
156 9 : HCCL_INFO("tlv request success, tlv module type[%u], message type[%u]", tlv_module_type, tlv_ccu_msg_type);
157 3 : return HCCL_SUCCESS;
158 : }
159 :
160 4147 : void HrtRaTlvRequestForCustomChannel(void* tlvHandle, u32 msgType, void *customIn, void *customOut)
161 : {
162 8294 : CHECK_NULLPTR(tlvHandle, "[HrtRaTlvRequestForCustomChannel] tlvHandle is nullptr!");
163 8292 : CHECK_NULLPTR(customIn, "[HrtRaTlvRequestForCustomChannel] customIn is nullptr!");
164 4146 : CHECK_NULLPTR(customOut, "[HrtRaTlvRequestForCustomChannel] customOut is nullptr!");
165 :
166 4144 : struct TlvMsg sendMsg {};
167 4144 : sendMsg.type = msgType;
168 4144 : sendMsg.length = sizeof(CustomChanInfoIn);
169 4144 : sendMsg.data = static_cast<char *>(customIn);
170 :
171 4144 : struct TlvMsg recvMsg {};
172 4144 : recvMsg.type = msgType;
173 4144 : recvMsg.length = sizeof(CustomChanInfoOut);
174 4144 : recvMsg.data = static_cast<char *>(customOut);
175 :
176 4144 : s32 ret = RaTlvRequest(tlvHandle, TLV_MODULE_TYPE_CCU, &sendMsg, &recvMsg);
177 4144 : if (ret != 0) {
178 4 : MACRO_THROW(NetworkApiException, StringFormat("[%s] RaTlvRequest fail, ret[%d]",
179 : __func__, ret));
180 : }
181 4143 : }
182 :
183 6 : void HrtRaTlvDeInit(void* tlv_handle)
184 : {
185 6 : CHECK_NULLPTR(tlv_handle, "[HrtRaTlvDeInit] tlv_handle is nullptr!");
186 :
187 6 : s32 ret = 0;
188 :
189 6 : ret = RaTlvDeinit(tlv_handle);
190 6 : if (ret != 0) {
191 4 : MACRO_THROW(NetworkApiException, StringFormat("[DeInit][RaTlv]errNo[0x%016llx] ra tlv deinit fail. params: tlv_handle=%p, return: ret=%d",
192 : HCCL_ERROR_CODE(HcclResult::HCCL_E_NETWORK), tlv_handle, ret));
193 : }
194 5 : }
195 :
196 11 : void HrtRaInit(HRaInitConfig &cfg)
197 : {
198 33 : HCCL_INFO("[Init][Ra] Input params: phyId=[%u], mode=[%u]",
199 : cfg.phyId, cfg.mode);
200 :
201 11 : struct RaInitConfig config {};
202 11 : config.phyId = cfg.phyId;
203 11 : config.nicPosition = HRT_NETWORK_MODE_MAP.at(cfg.mode);
204 11 : config.hdcType = PID_HDC_TYPE;
205 11 : config.enableHdcAsync = true;
206 :
207 11 : s32 ret = 0;
208 11 : auto startTime = std::chrono::steady_clock::now();
209 11 : auto timeout = std::chrono::seconds(EnvLinkTimeoutGet());
210 :
211 : while (true) {
212 11 : ret = RaInit(&config);
213 11 : if (!ret) {
214 10 : break; // 成功跳出
215 1 : } else if (ret == SOCK_EAGAIN) {
216 0 : bool bTimeout = ((std::chrono::steady_clock::now() - startTime) >= timeout);
217 0 : if (bTimeout) {
218 0 : MACRO_THROW(NetworkApiException, StringFormat("[Init][Ra]errNo[0x%016llx], ra init timeout[%lld], phy_id=%u, nic_position=%u, ret=%d",
219 : HCCL_ERROR_CODE(HcclResult::HCCL_E_NETWORK), timeout, config.phyId, config.nicPosition, ret));
220 : }
221 0 : SaluSleep(ONE_MILLISECOND_OF_USLEEP);
222 : } else {
223 : // 非ra限速场景错误，不轮询。直接退出
224 4 : MACRO_THROW(NetworkApiException, StringFormat("[Init][Ra]errNo[0x%016llx] ra init fail, phy_id=%u, nic_position=%u, ret=%d",
225 : HCCL_ERROR_CODE(HcclResult::HCCL_E_NETWORK), config.phyId, config.nicPosition, ret));
226 : }
227 0 : }
228 30 : HCCL_INFO("init ra success,return: ret[%d]", ret);
229 10 : }
230 :
231 4 : void HrtRaDeInit(HRaInitConfig &cfg)
232 : {
233 12 : HCCL_INFO("[DeInit][Ra] Input params: phyId=[%u], mode=[%u]",
234 : cfg.phyId, cfg.mode);
235 4 : struct RaInitConfig config {};
236 4 : config.phyId = cfg.phyId;
237 4 : config.nicPosition = HRT_NETWORK_MODE_MAP.at(cfg.mode);
238 4 : config.hdcType = PID_HDC_TYPE;
239 :
240 4 : s32 ret = 0;
241 4 : auto startTime = std::chrono::steady_clock::now();
242 4 : auto timeout = std::chrono::seconds(EnvLinkTimeoutGet());
243 : while (true) {
244 4 : ret = RaDeinit(&config);
245 4 : if (!ret) {
246 9 : HCCL_INFO("deinit ra success,return: ret[%d]", ret);
247 3 : break; // 成功跳出
248 1 : } else if (ret == SOCK_EAGAIN) {
249 0 : bool bTimeout = ((std::chrono::steady_clock::now() - startTime) >= timeout);
250 0 : if (bTimeout) {
251 0 : MACRO_THROW(NetworkApiException, StringFormat("[DeInit][Ra]errNo[0x%016llx] ra deinit timeout[%lld], phy_id=%u, nic_position=%u, ret=%d",
252 : HCCL_ERROR_CODE(HcclResult::HCCL_E_NETWORK), timeout, config.phyId, config.nicPosition, ret));
253 : }
254 0 : SaluSleep(ONE_MILLISECOND_OF_USLEEP);
255 : } else {
256 : // 非ra限速场景错误，不轮询。直接退出
257 4 : MACRO_THROW(NetworkApiException, StringFormat("[DeInit][Ra]errNo[0x%016llx] ra deinit fail, phy_id=%u, nic_position=%u, ret=%d",
258 : HCCL_ERROR_CODE(HcclResult::HCCL_E_NETWORK), config.phyId, config.nicPosition, ret));
259 : }
260 0 : }
261 3 : }
262 :
263 0 : static void SocketBatchConnect(SocketConnectInfoT conn[], u32 num)
264 : {
265 0 : CHECK_NULLPTR(conn, "[SocketBatchConnect] conn is nullptr!");
266 0 : HCCL_INFO("[BatchConnect][RaSocket] Input params: num=%u", num);
267 0 : s32 ret = 0;
268 0 : auto startTime = std::chrono::steady_clock::now();
269 0 : auto timeout = std::chrono::seconds(EnvLinkTimeoutGet());
270 : while (true) {
271 0 : ret = RaSocketBatchConnect(conn, num);
272 0 : if (!ret) {
273 0 : HCCL_INFO("socket batch connect success, ret=%d", ret);
274 0 : break; // 成功跳出
275 0 : } else if (ret == SOCK_EAGAIN) {
276 0 : bool bTimeout = ((std::chrono::steady_clock::now() - startTime) >= timeout);
277 0 : if (bTimeout) {
278 0 : MACRO_THROW(NetworkApiException, StringFormat("[BatchConnect][RaSocket]errNo[0x%016llx] ra socket batch connect, timeout[%lld]. return[%d], params: num[%u]",
279 : HCCL_ERROR_CODE(HcclResult::HCCL_E_TCP_CONNECT), timeout, ret, num));
280 : }
281 0 : SaluSleep(ONE_MILLISECOND_OF_USLEEP);
282 : } else {
283 0 : MACRO_THROW(NetworkApiException, StringFormat("[BatchConnect][RaSocket]errNo[0x%016llx] ra socket batch connect fail, return[%d], params: num[%u]",
284 : HCCL_ERROR_CODE(HcclResult::HCCL_E_TCP_CONNECT), ret, num));
285 : }
286 0 : }
287 0 : }
288 :
289 0 : void HrtRaSocketConnectOne(RaSocketConnectParam &in)
290 : {
291 0 : HCCL_INFO("[ConnectOne][RaSocket] Input params: socketHandle=%p, remoteIp=%s, port=%u, tag=%s",
292 : in.socketHandle, in.remoteIp.Describe().c_str(), in.port, in.tag.c_str());
293 :
294 0 : struct SocketConnectInfoT connInfo {};
295 0 : connInfo.socketHandle = in.socketHandle;
296 0 : connInfo.remoteIp = IpAddressToHccpIpAddr(in.remoteIp);
297 0 : connInfo.port = in.port;
298 :
299 0 : int sret = strcpy_s(connInfo.tag, sizeof(connInfo.tag), in.tag.c_str());
300 0 : if (sret != 0) {
301 : string msg
302 0 : = StringFormat("[HrtRaSocketConnectOne] copy tag[%s] to hccp tag failed, in.tag size[%d], connInfo.tag size[%d], ret[%d]", in.tag.c_str(), sizeof(in.tag.c_str()), sizeof(connInfo.tag), sret);
303 0 : MACRO_THROW(NetworkApiException, msg);
304 0 : }
305 :
306 0 : HCCL_INFO("Socket Connect tag=[%s], remoteIp[%s]", connInfo.tag, in.remoteIp.Describe().c_str());
307 0 : SocketBatchConnect(&connInfo, 1);
308 0 : }
309 :
310 11 : static void HRaSocketBatchClose(struct SocketCloseInfoT conn[], u32 num)
311 : {
312 11 : CHECK_NULLPTR(conn, "[HRaSocketBatchClose] conn is nullptr!");
313 33 : HCCL_INFO("[BatchClose][RaSocket] Input params: num=%u", num);
314 33 : HCCL_INFO("ra socket batch close");
315 11 : s32 ret = 0;
316 11 : auto startTime = std::chrono::steady_clock::now();
317 11 : auto timeout = std::chrono::seconds(EnvLinkTimeoutGet());
318 : while (true) {
319 11 : ret = RaSocketBatchClose(conn, num);
320 11 : if (!ret) {
321 33 : HCCL_INFO("socket batch close success, ret=%d", ret);
322 11 : break; // 成功跳出
323 0 : } else if (ret == SOCK_EAGAIN) {
324 0 : bool bTimeout = ((std::chrono::steady_clock::now() - startTime) >= timeout);
325 0 : if (bTimeout) {
326 0 : MACRO_THROW(NetworkApiException, StringFormat("[BatchClose][RaSocket]errNo[0x%016llx] ra socket batch close, timeout[%d], return[%d], params: num[%u]",
327 : HCCL_ERROR_CODE(HcclResult::HCCL_E_TCP_CONNECT), timeout, ret, num));
328 : }
329 0 : SaluSleep(ONE_MILLISECOND_OF_USLEEP);
330 : } else {
331 : // 非ra限速场景错误，不轮询，直接退出
332 0 : MACRO_THROW(NetworkApiException, StringFormat("[BatchClose][RaSocket]errNo[0x%016llx] ra socket batch close fail, return[%d], params: num[%u]",
333 : HCCL_ERROR_CODE(HcclResult::HCCL_E_TCP_CONNECT), ret, num));
334 : }
335 0 : }
336 11 : }
337 :
338 11 : void HrtRaSocketCloseOne(RaSocketCloseParam &in)
339 : {
340 33 : HCCL_INFO("[CloseOne][RaSocket] Input params: socketHandle=%p, fdHandle=%p", in.socketHandle, in.fdHandle);
341 11 : struct SocketCloseInfoT closeInfo = {};
342 11 : closeInfo.fdHandle = in.fdHandle;
343 11 : closeInfo.socketHandle = in.socketHandle;
344 :
345 11 : HRaSocketBatchClose(&closeInfo, 1);
346 11 : }
347 :
348 0 : static void ReportAddrInUseError(const IpAddress &localIp, u32 port, HrtNetworkMode netMode)
349 : {
350 0 : std::string errMsg = "The IP address " + std::string(localIp.Describe().c_str()) +
351 0 : " and port " + std::to_string(port) + " have already been bound.";
352 0 : if (netMode == HrtNetworkMode::PEER) {
353 0 : RPT_INPUT_ERR(true, "EI0019", std::vector<std::string>({"reason"}),
354 : std::vector<std::string>({errMsg}));
355 : } else {
356 0 : RPT_INPUT_ERR(true, "EI0020", std::vector<std::string>({"reason"}),
357 : std::vector<std::string>({errMsg}));
358 : }
359 0 : }
360 :
361 11 : static void HRaSocketListenStart(struct SocketListenInfoT conn[], u32 num, const IpAddress &localIp, HrtNetworkMode netMode)
362 : {
363 11 : CHECK_NULLPTR(conn, "[HRaSocketListenStart] conn is nullptr!");
364 33 : HCCL_INFO("[ListenStart][RaSocket] Input params: num=%u", num);
365 11 : s32 ret = 0;
366 11 : auto startTime = std::chrono::steady_clock::now();
367 11 : auto timeout = std::chrono::seconds(EnvLinkTimeoutGet());
368 :
369 : while (true) {
370 959 : ret = RaSocketListenStart(conn, num);
371 959 : if (ret == 0) {
372 30 : HCCL_INFO("socket listen start success, ret=%d", ret);
373 10 : break;
374 949 : } else if (ret == SOCK_EAGAIN) {
375 949 : bool bTimeout = ((std::chrono::steady_clock::now() - startTime) >= timeout);
376 949 : if (bTimeout) {
377 4 : MACRO_THROW(NetworkApiException, StringFormat("[ListenStart][RaSocket]errNo[0x%016llx] ra socket listen start, timeout[%d], return[%d], params: num[%u]",
378 : HCCL_ERROR_CODE(HcclResult::HCCL_E_TCP_CONNECT), EnvLinkTimeoutGet(), ret, num));
379 : }
380 948 : SaluSleep(ONE_MILLISECOND_OF_USLEEP);
381 0 : } else if (ret == SOCK_EADDRINUSE) {
382 0 : u32 port = (num > 0) ? conn[0].port : HCCL_INVALID_PORT;
383 0 : ReportAddrInUseError(localIp, port, netMode);
384 0 : MACRO_THROW(NetworkApiException, StringFormat("[%s]ra socket listen could not start, due to the port[%u] has already been bound. please try"
385 : " another port or check the port status", __func__, port));
386 0 : } else if (ret == SOCK_EADDRNOTAVAIL){
387 0 : MACRO_THROW(NetworkApiException, StringFormat("[%s] Socket listen start fail: "
388 : "IP address is not available, please check the IP address configuration, return[%d]", __func__, ret));
389 : } else {
390 : // 非ra限速场景错误，不轮询，直接退出
391 0 : MACRO_THROW(NetworkApiException, StringFormat("[ListenStart][RaSocket]errNo[0x%016llx] ra socket listen start fail, return[%d], params: num[%u]",
392 : HCCL_ERROR_CODE(HcclResult::HCCL_E_TCP_CONNECT), EnvLinkTimeoutGet(), ret, num));
393 : }
394 948 : }
395 10 : }
396 :
397 3 : static bool RaSocketTryListenStart(struct SocketListenInfoT conn[], u32 num, const IpAddress &localIp, HrtNetworkMode netMode)
398 : {
399 3 : CHECK_NULLPTR(conn, "[RaSocketTryListenStart] conn is nullptr!");
400 9 : HCCL_INFO("[TryListenStart][RaSocket] Input params: num=%u", num);
401 3 : s32 ret = RaSocketListenStart(conn, num);
402 3 : if (ret == 0) {
403 2 : return true;
404 1 : } else if (ret == SOCK_EAGAIN) {
405 0 : HCCL_INFO("[%s] listen eagain", __func__);
406 0 : return true;
407 1 : } else if (ret == SOCK_EADDRINUSE) {
408 0 : u32 port = (num > 0) ? conn[0].port : HCCL_INVALID_PORT;
409 0 : ReportAddrInUseError(localIp, port, netMode);
410 0 : HCCL_INFO("[%s]ra socket listen could not start, due to the port[%u] has already been bound. please try"
411 : " another port or check the port status", __func__, port);
412 0 : return false;
413 1 : } else if (ret == SOCK_EADDRNOTAVAIL){
414 4 : MACRO_THROW(NetworkApiException, StringFormat("[%s] Socket listen start fail: "
415 : "IP address is not available, please check the IP address configuration, return[%d]", __func__, ret));
416 : } else {
417 : // 非ra限速场景错误，不轮询，直接退出
418 0 : MACRO_THROW(NetworkApiException, StringFormat("[TryListenStart][RaSocket]errNo[0x%016llx] ra socket listen start fail, return[%d], params: num[%u]",
419 : HCCL_ERROR_CODE(HcclResult::HCCL_E_TCP_CONNECT), ret, num));
420 : }
421 : }
422 :
423 12 : static void HRaSocketListenStop(struct SocketListenInfoT conn[], u32 num)
424 : {
425 12 : CHECK_NULLPTR(conn, "[HRaSocketListenStop] conn is nullptr!");
426 36 : HCCL_INFO("[ListenStop][RaSocket] Input params: num=%u", num);
427 12 : s32 ret = 0;
428 12 : auto startTime = std::chrono::steady_clock::now();
429 12 : auto timeout = std::chrono::seconds(EnvLinkTimeoutGet());
430 : while (true) {
431 12 : ret = RaSocketListenStop(conn, num);
432 12 : if (!ret || ret == 228202) { // 待修改: 同步版本后 228202 修改为 SOCK_ENODEV
433 36 : HCCL_INFO("socket listen stop success, ret=%d", ret);
434 12 : break; // 成功跳出
435 0 : } else if (ret == SOCK_EAGAIN) {
436 0 : bool bTimeout = ((std::chrono::steady_clock::now() - startTime) >= timeout);
437 0 : if (bTimeout) {
438 0 : MACRO_THROW(NetworkApiException, StringFormat("[ListenStop][RaSocket]errNo[0x%016llx] ra socket listen stop fail, timeout[%d], return[%d], params: num[%u]",
439 : HCCL_ERROR_CODE(HcclResult::HCCL_E_TCP_CONNECT), timeout, ret, num));
440 : }
441 0 : SaluSleep(ONE_MILLISECOND_OF_USLEEP);
442 : } else {
443 : // 非ra限速场景错误，不轮询，直接退出
444 0 : MACRO_THROW(NetworkApiException, StringFormat("[ListenStop][RaSocket]errNo[0x%016llx] ra socket listen stop fail, return[%d], params: num[%u]",
445 : HCCL_ERROR_CODE(HcclResult::HCCL_E_TCP_CONNECT), ret, num));
446 : }
447 0 : }
448 12 : }
449 :
450 11 : void HrtRaSocketListenOneStart(RaSocketListenParam &in, HrtNetworkMode netMode)
451 : {
452 33 : HCCL_INFO("[ListenStart][RaSocket] Input params: socketHandle: %p, port: %u", in.socketHandle, in.port);
453 11 : struct SocketListenInfoT listenInfo {};
454 11 : listenInfo.socketHandle = in.socketHandle;
455 11 : listenInfo.port = in.port;
456 11 : HRaSocketListenStart(&listenInfo, 1, in.localIp, netMode);
457 10 : }
458 :
459 3 : bool HrtRaSocketTryListenOneStart(RaSocketListenParam &in, HrtNetworkMode netMode)
460 : {
461 9 : HCCL_INFO("[TryListenOneStart][RaSocket] Input params: socketHandle: %p, port: %u", in.socketHandle, in.port);
462 3 : struct SocketListenInfoT listenInfo {};
463 3 : listenInfo.socketHandle = in.socketHandle;
464 3 : listenInfo.port = in.port;
465 3 : bool ret = RaSocketTryListenStart(&listenInfo, 1, in.localIp, netMode);
466 2 : if (ret && in.port == AUTO_LISTEN_PORT) {
467 0 : in.port = listenInfo.port;
468 : }
469 2 : return ret;
470 : }
471 :
472 12 : void HrtRaSocketListenOneStop(RaSocketListenParam &in)
473 : {
474 36 : HCCL_INFO("[ListenOneStop][RaSocket] Input params: socketHandle: %p, port: %u",in.socketHandle, in.port);
475 12 : struct SocketListenInfoT listenInfo {};
476 12 : listenInfo.socketHandle = in.socketHandle;
477 12 : listenInfo.port = in.port;
478 12 : HRaSocketListenStop(&listenInfo, 1);
479 12 : }
480 :
481 0 : void RaBlockGetSockets(u32 role, SocketInfoT conn[], u32 num, u32 timeoutSec) // 修改为内部函数，不对外
482 : {
483 0 : CHECK_NULLPTR(conn, "[RaBlockGetSockets] conn is nullptr!");
484 0 : HCCL_INFO("[GetSockets][RaBlock] Input params: role=%u, num=%u, timeoutSec=%u", role, num, timeoutSec);
485 : s32 sockRet;
486 0 : u32 gotSocketsCnt = 0;
487 0 : auto startTime = std::chrono::steady_clock::now();
488 0 : auto linkTimeout = std::chrono::seconds(EnvLinkTimeoutGet());
489 0 : auto timeout = (timeoutSec > 0 && timeoutSec < linkTimeout.count())
490 0 : ? std::chrono::seconds(timeoutSec) : linkTimeout;
491 : while (true) {
492 0 : if ((std::chrono::steady_clock::now() - startTime) >= timeout) {
493 0 : MACRO_THROW(NetworkApiException, StringFormat("[HrtRaBlockGetSockets] get rasocket timeout role[%u], num[%u], gotSocketsCnt[%u], timeout[%lld]s",
494 : role, num, gotSocketsCnt, timeout));
495 : }
496 0 : u32 connectedNum = 0;
497 0 : sockRet = RaGetSockets(role, conn, num, &connectedNum);
498 0 : if ((connectedNum == 0 && sockRet == 0) || (sockRet == SOCK_EAGAIN)) {
499 0 : SaluSleep(ONE_MILLISECOND_OF_USLEEP);
500 0 : } else if (sockRet != 0) {
501 0 : MACRO_THROW(NetworkApiException, StringFormat("[Get][RaSocket]get rasocket error. role[%u], num[%u], sockRet[%d], connectednum[%u]", role, num, sockRet, connectedNum));
502 : } else {
503 0 : gotSocketsCnt += connectedNum;
504 0 : if (gotSocketsCnt == num) {
505 0 : HCCL_INFO("block get sockets success, socket num[%u]", gotSocketsCnt);
506 0 : break;
507 0 : } else if (gotSocketsCnt > num) {
508 0 : MACRO_THROW(NetworkApiException, StringFormat("[Get][RaSocket]total Sockets[%u], more than needed num[%u]!", gotSocketsCnt, num));
509 : } else {
510 0 : SaluSleep(ONE_MILLISECOND_OF_USLEEP);
511 : }
512 : }
513 0 : }
514 0 : }
515 :
516 0 : RaSocketFdHandleParam HrtRaBlockGetOneSocket(u32 role, RaSocketGetParam ¶m, u32 timeout)
517 : {
518 0 : HCCL_INFO("[GetOneSocket][RaSocket] Input params: role=%u,socketHandle=%p, fdHandle=%p, remoteIp=%s, timeout=%u",
519 : role, param.socketHandle, param.fdHandle, param.remoteIp.Describe().c_str(), timeout);
520 0 : struct SocketInfoT socketInfo {};
521 :
522 0 : socketInfo.socketHandle = param.socketHandle;
523 0 : socketInfo.fdHandle = param.fdHandle;
524 0 : socketInfo.remoteIp = IpAddressToHccpIpAddr(param.remoteIp);
525 0 : socketInfo.status = SOCKET_NOT_CONNECTED;
526 :
527 0 : int sret = strcpy_s(socketInfo.tag, sizeof(socketInfo.tag), param.tag.c_str());
528 0 : if (sret != 0) {
529 0 : MACRO_THROW(NetworkApiException, StringFormat("[HrtRaBlockGetOneSocket] copy tag[%s] to hccp failed, ret=%d, role=%u,socketHandle=%p, fdHandle=%p, remoteIp=%s, socketInfo.tag size=%d, param.tag size=%d",
530 : param.tag.c_str(), sret, role, param.socketHandle, param.fdHandle, param.remoteIp.Describe().c_str(), sizeof(socketInfo.tag), sizeof(param.tag.c_str())));
531 : }
532 :
533 0 : HCCL_INFO("Socket Get tag=[%s], remoteIp[%s], ret[%d]", socketInfo.tag, param.remoteIp.Describe().c_str(), sret);
534 0 : RaBlockGetSockets(role, &socketInfo, 1, timeout);
535 :
536 0 : return RaSocketFdHandleParam(socketInfo.fdHandle, socketInfo.status);
537 : }
538 :
539 0 : void HrtRaSocketBlockSend(const FdHandle fdHandle, const void *data, u32 sendSize)
540 : {
541 0 : CHECK_NULLPTR(fdHandle, "[HrtRaSocketBlockSend] fdHandle is nullptr!");
542 0 : CHECK_NULLPTR(data, "[HrtRaSocketBlockSend] data is nullptr!");
543 0 : s32 ret = 0;
544 0 : void *sendData = const_cast<void *>(data);
545 0 : const std::chrono::seconds timeout = std::chrono::seconds(EnvLinkTimeoutGet());
546 0 : const auto start = std::chrono::steady_clock::now();
547 0 : u32 totalSentSize = 0;
548 0 : unsigned long long sentSize = 0;
549 :
550 0 : HCCL_INFO("before ra socket send, para: fdHandle[%p], data[%p], size[%u]", fdHandle, sendData, sendSize);
551 :
552 : while (true) {
553 : // 底层ra_socket_send host网卡无限制，device网卡由于HDC通道限制的限制有大小限制(目前大小为64KB)
554 0 : ret = RaSocketSend(fdHandle, reinterpret_cast<void *>(reinterpret_cast<uintptr_t>(sendData) + totalSentSize),
555 0 : sendSize - totalSentSize, &sentSize);
556 0 : HCCL_INFO("ra socket send, data[%p], size[%u] send size[%u]", sendData, sendSize, totalSentSize);
557 0 : if (ret == 0) {
558 0 : totalSentSize += sentSize;
559 0 : if (totalSentSize == sendSize) { // 只有完全发送完才返回成功
560 0 : break;
561 : }
562 :
563 0 : if (totalSentSize > sendSize) {
564 0 : MACRO_THROW(NetworkApiException, StringFormat("[Send][RaSocket]errNo[0x%016llx] ra socket send failed, fdHandle=%p, data=%p, size=%u, retSize=%u",
565 : HCCL_ERROR_CODE(HcclResult::HCCL_E_NETWORK), fdHandle, data, sendSize, sentSize));
566 : }
567 0 : SaluSleep(ONE_HUNDRED_MICROSECOND_OF_USLEEP);
568 0 : } else if (ret == SOCK_EAGAIN) {
569 : /* ra速率限制 retry */
570 0 : SaluSleep(ONE_MILLISECOND_OF_USLEEP);
571 : } else {
572 0 : MACRO_THROW(NetworkApiException, StringFormat("[Send][RaSocket]errNo[0x%016llx] ra socket send failed, fdHandle=%p, data=%p, size=%u, retSize=%u, ret=%d",
573 : HCCL_ERROR_CODE(HcclResult::HCCL_E_NETWORK), fdHandle, data, sendSize, sentSize, ret));
574 : }
575 : /* 获取当前时间，如果耗时超过timeout，则返回错误 */
576 0 : const auto elapsed = std::chrono::duration_cast<std::chrono::seconds>(std::chrono::steady_clock::now() - start);
577 0 : if (elapsed > timeout) {
578 0 : MACRO_THROW(NetworkApiException, StringFormat("[Send][RaSocket]errNo[0x%016llx] Wait timeout for sockets send, fdHandle[%p], data[%p], size[%u], retsize[%u], ret[%d]",
579 : HCCL_ERROR_CODE(HcclResult::HCCL_E_NETWORK), fdHandle, data, sendSize, sentSize, ret));
580 : }
581 0 : }
582 0 : HCCL_INFO("ra socket send finished,ret[%d]", ret);
583 0 : }
584 :
585 1 : s32 HrtRaSocketNonBlockSendNormal(const FdHandle fdHandle, void *data, u64 size, u64 *sentSize)
586 : {
587 2 : CHECK_NULLPTR(fdHandle, "[HrtRaSocketNonBlockSend] fdHandle is nullptr!");
588 2 : CHECK_NULLPTR(data, "[HrtRaSocketNonBlockSend] data is nullptr!");
589 1 : CHECK_NULLPTR(sentSize, "[HrtRaSocketNonBlockSend] sentSize is nullptr!");
590 3 : HCCL_INFO("[HrtRaSocketNonBlockSend] Input params: fdHandle=%p,data=%p, size=%llu, sentSize=%llu",
591 : fdHandle, data, size, *sentSize);
592 1 : if (size > SOCKET_SEND_MAX_SIZE) {
593 0 : MACRO_THROW(NetworkApiException, StringFormat("[hrtRaSocketNonBlockSend]errNo[0x%016llx] ra socket send size is too large, "
594 : "data[%p], size[%llu], fdHandle[%p], send size[%llu]", HCCL_ERROR_CODE(HcclResult::HCCL_E_NETWORK), data, size, fdHandle, *sentSize));
595 : }
596 :
597 1 : return RaSocketSend(fdHandle, data, size, sentSize);
598 : }
599 :
600 0 : bool HrtRaSocketNonBlockSend(const FdHandle fdHandle, void *data, u64 size, u64 *sentSize)
601 : {
602 0 : s32 ret = HrtRaSocketNonBlockSendNormal(fdHandle, data, size, sentSize);
603 0 : if (ret == 0 || ret == SOCK_EAGAIN) {
604 0 : HCCL_INFO("[HrtRaSocketNonBlockSend] ra socket send, data[%p], size[%llu], send size[%llu], ret[%d]", data, size, *sentSize, ret);
605 0 : return true;
606 : } else {
607 0 : HCCL_ERROR("call RaSocketSend failed, fdHandle=%p, data=%p, size=%llu, sentSize=%llu, ret[%d]",
608 : fdHandle, data, size, *sentSize, ret);
609 0 : return false;
610 : }
611 : }
612 :
613 1 : HcclResult HrtRaSocketNonBlockSendHeart(const FdHandle fdHandle, void *data, u64 size, u64 *sentSize)
614 : {
615 1 : s32 ret = HrtRaSocketNonBlockSendNormal(fdHandle, data, size, sentSize);
616 1 : if (ret == 0) {
617 1 : return HCCL_SUCCESS;
618 0 : } else if (ret == SOCK_EAGAIN) {
619 0 : return HCCL_E_AGAIN;
620 0 : } else if (ret == SOCK_CLOSE) {
621 0 : return HCCL_E_INTERNAL; //暂时用这个错误码表示hccp进程异常退出
622 : } else {
623 0 : HCCL_WARNING("[HrtRaSocketNonBlockSend]ra socket send failed, data[%p], size[%llu], send size[%llu], ret[%d]",
624 : data, size, *sentSize, ret);
625 0 : return HCCL_E_NETWORK;
626 : }
627 : }
628 :
629 1 : HcclResult HrtRaSocketNonBlockRecvHeart(const FdHandle fdHandle, void *data, u64 size, u64 *recvSize)
630 : {
631 2 : CHECK_NULLPTR(fdHandle, "[HrtRaSocketNonBlockRecv] fdHandle is nullptr!");
632 2 : CHECK_NULLPTR(data, "[HrtRaSocketNonBlockRecv] data is nullptr!");
633 1 : CHECK_NULLPTR(recvSize, "[HrtRaSocketNonBlockRecv] recvSize is nullptr!");
634 3 : HCCL_DEBUG("[HrtRaSocketNonBlockRecv] Input params: fdHandle=%p,data=%p, size=%llu, recvSize=%llu",
635 : fdHandle, data, size, *recvSize);
636 :
637 1 : s32 ret = RaSocketRecv(fdHandle, data, size, recvSize);
638 1 : if (ret == 0) {
639 1 : return HCCL_SUCCESS;
640 0 : } else if (ret == SOCK_EAGAIN) {
641 0 : return HCCL_E_AGAIN;
642 0 : } else if (ret == SOCK_CLOSE) {
643 0 : return HCCL_E_INTERNAL; //暂时用这个错误码表示hccp进程异常退出
644 : } else {
645 0 : HCCL_WARNING("[HrtRaSocketNonBlockRecv]ra socket recv failed, data[%p], size[%llu], "\
646 : "recv[%llu], ret[%d], errno[%d][%s]", data, size, recvSize, ret, errno, strerror(errno));
647 0 : return HCCL_E_TCP_TRANSFER;
648 : }
649 : return HCCL_SUCCESS;
650 : }
651 :
652 6 : void HrtRaSocketBlockRecv(const FdHandle fdHandle, void *data, u32 size)
653 : {
654 6 : auto startTime = std::chrono::steady_clock::now();
655 6 : unsigned long long recvSize = 0;
656 6 : s32 rtRet = 0;
657 6 : u32 getedLen = 0;
658 6 : const std::chrono::seconds timeout = std::chrono::seconds(EnvLinkTimeoutGet());
659 :
660 12 : CHECK_NULLPTR(fdHandle, "[HrtRaSocketBlockRecv] fdHandle is nullptr!");
661 6 : CHECK_NULLPTR(data, "[HrtRaSocketBlockRecv] data is nullptr!");
662 18 : HCCL_INFO("before ra socket recv, para: fdHandle[%p], data[%p], size[%u]", fdHandle, data, size);
663 : while (true) {
664 6 : if ((std::chrono::steady_clock::now() - startTime) >= timeout) {
665 : std::string errMsg = StringFormat(
666 : "[Recv][RaSocket]errNo[0x%016llx] Wait timeout for sockets recv, data[%p], "
667 : "size[%u], recvSize[%u], fdHandle[%p], ret[%d]",
668 : HCCL_ERROR_CODE(HcclResult::HCCL_E_NETWORK), data, size, recvSize, fdHandle, rtRet
669 1 : );
670 3 : HCCL_ERROR("%s", errMsg.c_str());
671 3 : HCCL_ERROR("Please check the following reasons:");
672 3 : HCCL_ERROR("1. check the firewall configuration or try to disable the firewall.");
673 3 : HCCL_ERROR("2. check error log on the other process or thread.");
674 4 : MACRO_THROW(NetworkApiException, errMsg);
675 1 : }
676 10 : rtRet = RaSocketRecv(fdHandle, reinterpret_cast<void *>(reinterpret_cast<uintptr_t>(data) + getedLen),
677 5 : size - getedLen, &recvSize);
678 5 : if ((rtRet == 0) && (recvSize > 0)) { // 接收完成，也有可能要多次接收
679 2 : getedLen += recvSize;
680 2 : if (getedLen > size) {
681 4 : MACRO_THROW(NetworkApiException, StringFormat("[Recv][RaSocket]errNo[0x%016llx] socket receive call RaSocketRecv failed,"
682 : "rtSize[%u], bigger size[%zu], fdHandle[%p], data[%p], retSize[%u], ret[%d]",
683 : HCCL_ERROR_CODE(HcclResult::HCCL_E_TCP_TRANSFER), getedLen, size, fdHandle, data, recvSize, rtRet));
684 : }
685 1 : if (getedLen == size) {
686 1 : break;
687 : }
688 3 : } else if ((rtRet == 0) && (recvSize == 0)) {
689 4 : MACRO_THROW(NetworkApiException, StringFormat("[Recv][RaSocket]recv fail, fdHandle=%p, data=%p, bufLen=%u, recLen=%lld, ret=%d", fdHandle, data, size, recvSize, rtRet));
690 2 : } else if (rtRet == SOCK_ESOCKCLOSED || rtRet == SOCK_CLOSE) { // 连接关闭，出错
691 8 : MACRO_THROW(NetworkApiException, StringFormat("[Recv][RaSocket]errNo[0x%016llx] recv fail, call RaSocketRecv failed, sock_esockclosed, fdhandle=%p, data=%p, bufLen=%u, recLen=%lld, ret=%d",
692 : HCCL_ERROR_CODE(HcclResult::HCCL_E_TCP_TRANSFER), fdHandle, data, size, recvSize, rtRet));
693 0 : } else if (rtRet != 0) {
694 0 : SaluSleep(ONE_MILLISECOND_OF_USLEEP); // 尚未接收到数据,延时1ms
695 0 : continue;
696 : }
697 0 : }
698 3 : HCCL_INFO("ra socket receive finished. ret[%d]", rtRet);
699 1 : }
700 :
701 2 : SocketHandle HrtRaSocketInit(HrtNetworkMode netMode, RaInterface &in)
702 : {
703 2 : int mode = HRT_NETWORK_MODE_MAP.at(netMode);
704 2 : struct rdev rdevInfo {};
705 2 : rdevInfo.phyId = in.phyId;
706 2 : rdevInfo.family = in.address.GetFamily();
707 2 : rdevInfo.localIp = IpAddressToHccpIpAddr(in.address);
708 :
709 6 : HCCL_INFO("[HrtRaSocketInit] Input params: mode=%u, ip=%u, device id=%u, family=%u",
710 : mode, rdevInfo.localIp.addr.s_addr, rdevInfo.phyId, rdevInfo.family);
711 :
712 2 : SocketHandle socketHandle = nullptr;
713 2 : s32 ret = RaSocketInit(mode, rdevInfo, &socketHandle);
714 2 : if (ret != 0 || (socketHandle == nullptr)) {
715 4 : MACRO_THROW(NetworkApiException, StringFormat("[Init][RaSock]errNo[0x%016llx] ra socket init fail, call RaSocketInit failed, params: mode=%u, ip=%u, device id=%u, family=%u. return: ret=%d",
716 : HCCL_ERROR_CODE(HcclResult::HCCL_E_NETWORK), mode, rdevInfo.localIp.addr.s_addr, rdevInfo.phyId, rdevInfo.family, ret));
717 : }
718 :
719 3 : HCCL_INFO("socket init success, ip[%u], device id[%u], ret[%d]", rdevInfo.localIp.addr.s_addr, rdevInfo.phyId, ret);
720 1 : return socketHandle;
721 : }
722 :
723 2 : void HrtRaSocketDeInit(SocketHandle socketHandle)
724 : {
725 2 : CHECK_NULLPTR(socketHandle, "[HrtRaSocketDeInit] socketHandle is nullptr!");
726 6 : HCCL_INFO("[HrtRaSocketDeInit] Input params: socketHandle=%p", socketHandle);
727 :
728 2 : s32 ret = RaSocketDeinit(socketHandle);
729 2 : if (ret != 0) {
730 0 : MACRO_THROW(NetworkApiException, StringFormat("[DeInit][RaSocket]errNo[0x%016llx] rt socket deinit fail. call RaSocketDeinit failed, params: socketHandle[%p], return: ret[%d]",
731 : HCCL_ERROR_CODE(HcclResult::HCCL_E_NETWORK), socketHandle, ret));
732 : }
733 2 : }
734 :
735 0 : void HrtRaSocketSetWhiteListStatus(u32 enable)
736 : {
737 0 : HCCL_INFO("[HrtRaSocketSetWhiteListStatus] Input params: enable=%u", enable);
738 0 : s32 ret = RaSocketSetWhiteListStatus(enable);
739 0 : if (ret != 0) {
740 0 : MACRO_THROW(NetworkApiException, StringFormat("[Set][WhiteListStatus]errNo[0x%016llx] ra socekt set white list fail, call RaSocketSetWhiteListStatus failed, params: enable[%u], return: ret[%d]",
741 : HCCL_ERROR_CODE(HcclResult::HCCL_E_TCP_CONNECT), enable, ret));
742 : }
743 :
744 0 : HCCL_INFO("set host socket whitelist status[%u] success.", enable);
745 0 : }
746 :
747 0 : u32 HrtRaSocketGetWhiteListStatus()
748 : {
749 : u32 enable;
750 0 : s32 ret = RaSocketGetWhiteListStatus(&enable);
751 0 : if (ret != 0) {
752 0 : MACRO_THROW(NetworkApiException, StringFormat("[Get][WhiteListStatus]errNo[0x%016llx] ra socekt get whilte list fail, call RaSocketGetWhiteListStatus failed, return: ret[%d]",
753 : HCCL_ERROR_CODE(HcclResult::HCCL_E_TCP_CONNECT), ret));
754 : }
755 :
756 0 : HCCL_INFO("get host socket whitelist status[%u] success.", enable);
757 0 : return enable;
758 : }
759 :
760 5 : void HrtRaSocketWhiteListAdd(SocketHandle socketHandle, vector<RaSocketWhitelist> &wlists)
761 : {
762 5 : CHECK_NULLPTR(socketHandle, "[HrtRaSocketWhiteListAdd] socketHandle is nullptr!");
763 15 : HCCL_INFO("[HrtRaSocketWhiteListAdd] Input params: socketHandle=%p", socketHandle);
764 :
765 5 : vector<struct SocketWlistInfoT> wlistInfoVec;
766 5 : wlistInfoVec.reserve(MAX_NUM_OF_WHITE_LIST_NUM);
767 5 : size_t wlistNum = wlists.size();
768 5 : size_t startIdx = 0;
769 8 : while (wlistNum > 0) {
770 5 : size_t addListNum = wlistNum > MAX_NUM_OF_WHITE_LIST_NUM ? MAX_NUM_OF_WHITE_LIST_NUM : wlistNum;
771 9 : for (size_t idx = startIdx; idx < addListNum + startIdx; idx++) {
772 5 : struct SocketWlistInfoT wlistInfo {};
773 5 : wlistInfo.connLimit = wlists[idx].connLimit;
774 5 : wlistInfo.remoteIp = IpAddressToHccpIpAddr(wlists[idx].remoteIp);
775 :
776 5 : int sret = strcpy_s(wlistInfo.tag, sizeof(wlistInfo.tag), wlists[idx].tag.c_str());
777 5 : if (sret != EOK) {
778 4 : MACRO_THROW(InternalException, StringFormat("[Add][RaSocketWhiteList]errNo[0x%016llx]errName[HCCL_E_MEMORY] memory copy failed. params: socketHandle[%p], return: ret[%d], wlistInfo.tag size=%zu, wlists[%zu].tag size=%zu",
779 : HCOM_ERROR_CODE(HcclResult::HCCL_E_MEMORY), socketHandle, sret, sizeof(wlistInfo.tag), idx, sizeof(wlists[idx].tag.c_str())));
780 : }
781 12 : HCCL_INFO("add whitelistInfo tag=[%s], remoteIp[%s]",
782 : wlistInfo.tag, wlists[idx].remoteIp.Describe().c_str());
783 4 : wlistInfoVec.push_back(wlistInfo);
784 : }
785 :
786 4 : s32 ret = RaSocketWhiteListAdd(socketHandle, wlistInfoVec.data(), wlistInfoVec.size());
787 4 : if (ret != 0) {
788 4 : MACRO_THROW(NetworkApiException, StringFormat("[Add][RaSocketWhiteList]errNo[0x%016llx]errName[HCCL_E_TCP_CONNECT] ra white list add fail, call RaSocketWhiteListAdd failed, socketHandle[%p], num=%zu, return[%d].",
789 : HCCL_ERROR_CODE(HcclResult::HCCL_E_TCP_CONNECT), socketHandle, wlistInfoVec.size() + startIdx, ret));
790 : }
791 9 : HCCL_INFO("add white list: num[%zu], remain [%zu].", addListNum, (wlistNum - addListNum));
792 :
793 3 : wlistInfoVec.clear();
794 3 : wlistNum -= addListNum;
795 3 : startIdx += addListNum;
796 : }
797 9 : HCCL_INFO("[HrtRaSocketWhiteListAdd] Success. Total add num [%zu]", wlists.size());
798 5 : }
799 :
800 4 : void HrtRaSocketWhiteListDel(SocketHandle socketHandle, vector<RaSocketWhitelist> &wlists)
801 : {
802 4 : CHECK_NULLPTR(socketHandle, "[HrtRaSocketWhiteListDel] socketHandle is nullptr!");
803 12 : HCCL_INFO("[HrtRaSocketWhiteListDel] Input params: socketHandle=%p", socketHandle);
804 :
805 4 : vector<struct SocketWlistInfoT> wlistInfoVec;
806 4 : wlistInfoVec.reserve(MAX_NUM_OF_WHITE_LIST_NUM);
807 4 : size_t wlistNum = wlists.size();
808 4 : size_t startIdx = 0;
809 10 : while (wlistNum > 0) {
810 7 : size_t delListNum = wlistNum > MAX_NUM_OF_WHITE_LIST_NUM ? MAX_NUM_OF_WHITE_LIST_NUM : wlistNum;
811 66 : for (size_t idx = startIdx; idx < delListNum + startIdx; idx++) {
812 59 : struct SocketWlistInfoT wlistInfo {};
813 59 : wlistInfo.connLimit = wlists[idx].connLimit;
814 59 : wlistInfo.remoteIp = IpAddressToHccpIpAddr(wlists[idx].remoteIp);
815 :
816 59 : int sret = strcpy_s(wlistInfo.tag, sizeof(wlistInfo.tag), wlists[idx].tag.c_str());
817 59 : if (sret != EOK) {
818 : auto msg = StringFormat("[Del][RaSocketWhiteList]errNo[0x%016llx] memory copy failed. ret[%d], wlistInfo.tag size[%zu], wlists[%zu].tag size[%zu]",
819 0 : HCOM_ERROR_CODE(HcclResult::HCCL_E_MEMORY), sret, sizeof(wlistInfo.tag), idx, sizeof(wlists[idx].tag.c_str()));
820 0 : MACRO_THROW(InternalException, msg);
821 0 : }
822 59 : wlistInfoVec.push_back(wlistInfo);
823 : }
824 :
825 7 : s32 ret = RaSocketWhiteListDel(socketHandle, wlistInfoVec.data(), wlistInfoVec.size());
826 7 : if (ret != 0) {
827 4 : MACRO_THROW(NetworkApiException, StringFormat("[Del][RaSocketWhiteList]errNo[0x%016llx] ra white list del fail, call RaSocketWhiteListDel failed, num=%zu, return[%d].",
828 : HCCL_ERROR_CODE(HcclResult::HCCL_E_TCP_CONNECT), wlists.size(), ret));
829 : }
830 18 : HCCL_INFO("del white list: num[%zu], remain [%zu].", delListNum, (wlistNum - delListNum));
831 :
832 6 : wlistInfoVec.clear();
833 6 : wlistNum -= delListNum;
834 6 : startIdx += delListNum;
835 : }
836 9 : HCCL_INFO("[HrtRaSocketWhiteListDel] Success. Total delete num[%zu]", wlists.size());
837 4 : }
838 :
839 : std::mutex g_deviceVnicIpMutex;
840 : std::map<u32, IpAddress> g_deviceIdVnicInfoMap; // 记录deviceid和vnic ip的关系，用于server内查询，避免重复查询
841 :
842 0 : void HrtRaSocketGetVnicIpInfos(u32 phyId, DeviceIdType deviceIdType, u32 deviceId, IpAddress &vnicIP)
843 : {
844 0 : std::lock_guard<std::mutex> lock(g_deviceVnicIpMutex);
845 0 : auto iter = g_deviceIdVnicInfoMap.find(deviceId);
846 0 : if (iter != g_deviceIdVnicInfoMap.end()) {
847 : // 缓存查找到，直接从缓存获取
848 0 : vnicIP = iter->second;
849 0 : HCCL_INFO("[HrtRaSocketGetVnicIpInfos] vnicInfoMap deviceId[%u] found, Ip[%s]",
850 : deviceId, vnicIP.Describe().c_str());
851 0 : return;
852 : }
853 0 : struct IpInfo vnicIpInfo = {};
854 0 : (void)memset_s(&vnicIpInfo, sizeof(IpInfo), 0, sizeof(IpInfo));
855 0 : IdType idType = static_cast<IdType>(deviceIdType);
856 0 : auto ret = RaSocketGetVnicIpInfos(phyId, idType, &deviceId, 1, &vnicIpInfo);
857 0 : if (ret != 0) {
858 0 : HCCL_ERROR("[hrtRaGetSocketVnicIpInfo]ra get VnicIpfail. ret[%d]", ret);
859 0 : throw NetworkApiException(StringFormat("call hrtRaGetSocketVnicIpInfo failed, ret=%llu", ret));
860 : }
861 : BinaryAddr temp;
862 0 : temp.addr = vnicIpInfo.ip.addr;
863 0 : temp.addr6 = vnicIpInfo.ip.addr6;
864 0 : IpAddress ipInfo(temp, vnicIpInfo.family);
865 0 : if (ipInfo.IsInvalid()) {
866 0 : HCCL_ERROR("vnicIp is invalid.");
867 0 : throw NetworkApiException("vnicIp is invalid.");
868 : }
869 0 : g_deviceIdVnicInfoMap.insert({ deviceId, ipInfo });
870 0 : vnicIP = ipInfo;
871 0 : HCCL_INFO("[hrtRaGetSocketVnicIpInfos] add vnicInfoMap, deviceIds[%u], Ip[%s]",
872 : deviceId, vnicIP.Describe().c_str());
873 0 : }
874 :
875 7 : static u32 HrtGetIfNum(struct RaGetIfattr &config)
876 : {
877 21 : HCCL_INFO("[HrtGetIfNum] Input params: phyId=%u, nicPosistion=%u", config.phyId, config.nicPosition);
878 :
879 7 : u32 num = 0;
880 7 : s32 ret = RaGetIfnum(&config, &num);
881 7 : if (ret != 0) {
882 4 : MACRO_THROW(NetworkApiException, StringFormat("[Get][IfNum]errNo[0x%016llx] ra get if num fail. call RaGetIfnum failed, Input params: phyId=%u, nicPosistion=%u, return: ret[%d], num[%u]",
883 : HCCL_ERROR_CODE(HcclResult::HCCL_E_TCP_CONNECT), config.phyId, config.nicPosition, ret, num));
884 : }
885 6 : return num;
886 : }
887 :
888 3 : static void HrtGetIfAddress(struct RaGetIfattr &config, InterfaceInfo ifaddrInfos[], u32 &num)
889 : {
890 3 : CHECK_NULLPTR(ifaddrInfos, "[HrtGetIfAddress] ifaddrInfos is nullptr!");
891 9 : HCCL_INFO("[HrtGetIfAddress] Input params: phyId=%u, nicPosition=%u, num=%u", config.phyId, config.nicPosition, num);
892 :
893 3 : s32 ret = RaGetIfaddrs(&config, ifaddrInfos, &num);
894 3 : if (ret != 0) {
895 4 : MACRO_THROW(NetworkApiException, StringFormat("[Get][IfAddress]errNo[0x%016llx] ra get if address fail. call RaGetIfaddrs failed, Input params: phyId=%u, nicPosistion=%u, return: ret[%d], num[%u]",
896 : HCCL_ERROR_CODE(HcclResult::HCCL_E_TCP_CONNECT), config.phyId, config.nicPosition, ret, num));
897 : }
898 2 : }
899 :
900 4 : std::vector<std::pair<std::string, IpAddress>> HrtGetHostIf(u32 devPhyId)
901 : {
902 12 : HCCL_INFO("[HrtGetHostIf] Input params: devPhyId=%u", devPhyId);
903 4 : std::vector<std::pair<std::string, IpAddress>> hostIfs;
904 4 : struct RaGetIfattr config = {};
905 4 : config.phyId = devPhyId;
906 4 : config.nicPosition = static_cast<u32>(NetworkMode::NETWORK_PEER_ONLINE);
907 :
908 4 : u32 ifAddrNum = HrtGetIfNum(config);
909 9 : HCCL_RUN_INFO("[Get][HostIf]hrtGetIfNum success. ifAddrNum[%u].", ifAddrNum);
910 3 : if (ifAddrNum == 0) {
911 3 : HCCL_WARNING("[Get][HostIf]there is no valid host interface, ifAddrNum[%u].", ifAddrNum);
912 1 : return hostIfs;
913 : }
914 :
915 6 : std::shared_ptr<struct InterfaceInfo> ifAddrInfoPtrs(new InterfaceInfo[ifAddrNum](),
916 4 : std::default_delete<InterfaceInfo[]>());
917 2 : struct InterfaceInfo *ifAddrInfos = ifAddrInfoPtrs.get();
918 :
919 2 : (void)memset_s(ifAddrInfos, ifAddrNum * sizeof(InterfaceInfo), 0, ifAddrNum * sizeof(InterfaceInfo));
920 :
921 2 : HrtGetIfAddress(config, ifAddrInfos, ifAddrNum);
922 :
923 2 : for (u32 i = 0; i < ifAddrNum; i++) {
924 1 : IpAddress ip = IfAddrInfoToIpAddress(ifAddrInfos[i]);
925 1 : hostIfs.emplace_back(ifAddrInfos[i].ifname, ip);
926 3 : HCCL_INFO("HrtGetIfAddress: idx[%u], ifName[%s], ip[%s]", i, ifAddrInfos[i].ifname, ip.GetIpStr().c_str());
927 : }
928 :
929 1 : return hostIfs;
930 4 : }
931 :
932 3 : vector<IpAddress> HrtGetDeviceIp(u32 devicePhyId, NetworkMode netWorkMode)
933 : {
934 9 : HCCL_INFO("[HrtGetDeviceIp] Input params: devicePhyId=%u", devicePhyId);
935 3 : vector<IpAddress> ipAddr;
936 3 : struct RaGetIfattr config = {};
937 3 : config.phyId = devicePhyId;
938 3 : config.nicPosition = static_cast<u32>(netWorkMode);
939 :
940 3 : u32 ifAddrNum = HrtGetIfNum(config);
941 9 : HCCL_RUN_INFO("[Get][DeviceIP]hrtGetIfNum success. ifAddrNum[%u].", ifAddrNum);
942 :
943 3 : if (ifAddrNum == 0) {
944 6 : HCCL_WARNING("[Get][DeviceIP]device has no ip information, phy_id[%u]", devicePhyId);
945 2 : return ipAddr;
946 : }
947 :
948 3 : std::shared_ptr<struct InterfaceInfo> ifAddrInfoPtrs(new InterfaceInfo[ifAddrNum](),
949 2 : std::default_delete<InterfaceInfo[]>());
950 1 : struct InterfaceInfo *ifAddrInfos = ifAddrInfoPtrs.get();
951 :
952 1 : (void)memset_s(ifAddrInfos, ifAddrNum * sizeof(InterfaceInfo), 0, ifAddrNum * sizeof(InterfaceInfo));
953 :
954 1 : HrtGetIfAddress(config, ifAddrInfos, ifAddrNum);
955 :
956 2 : for (u32 i = 0; i < ifAddrNum; i++) {
957 1 : IpAddress ip = IfAddrInfoToIpAddress(ifAddrInfos[i]);
958 1 : ipAddr.emplace_back(ip);
959 3 : HCCL_INFO("HrtGetIfAddress: idx[%u], ifName[%s], ip[%s]", i, ifAddrInfos[i].ifname, ip.GetIpStr().c_str());
960 : }
961 :
962 1 : return ipAddr;
963 1 : }
964 :
965 2 : RdmaHandle HrtRaRdmaInit(HrtNetworkMode netMode, RaInterface &in)
966 : {
967 2 : RdmaHandle rdmaHandle = nullptr;
968 2 : int mode = HRT_NETWORK_MODE_MAP.at(netMode);
969 2 : unsigned int notifyType = netMode == HrtNetworkMode::PEER ? NO_USE : NOTIFY;
970 6 : HCCL_INFO("[HrtRaRdmaInit] Input params: mode=%d, phyId=%u", mode, in.phyId);
971 2 : struct rdev rdevInfo {};
972 2 : rdevInfo.phyId = in.phyId;
973 2 : rdevInfo.family = in.address.GetFamily();
974 2 : rdevInfo.localIp = IpAddressToHccpIpAddr(in.address);
975 2 : s32 ret = RaRdevInit(mode, notifyType, rdevInfo, &rdmaHandle);
976 2 : RPT_INPUT_ERR(ret == HCCP_ELINKDOWN,
977 : "EI0009",
978 : vector<string>({"device_id", "reason"}),
979 : vector<string>({std::to_string(rdevInfo.phyId), "The network port is down"})
980 : );
981 2 : if (ret != 0 || (rdmaHandle == nullptr)) {
982 8 : MACRO_THROW(NetworkApiException, StringFormat("[Init][RaRdma]errNo[0x%016llx] rdma init fail. call RaRdevInit failed, Input params: phyId=%u, mode=%u, return: ret[%d]",
983 : HCCL_ERROR_CODE(HcclResult::HCCL_E_NETWORK), in.phyId, mode, ret));
984 : }
985 0 : return rdmaHandle;
986 0 : }
987 :
988 12 : void HrtRaRdmaDeInit(RdmaHandle rdmaHandle, HrtNetworkMode netMode)
989 : {
990 12 : CHECK_NULLPTR(rdmaHandle, "[HrtRaRdmaDeInit] rdmaHandle is nullptr!");
991 36 : HCCL_INFO("[HrtRaRdmaDeInit] Input params: rdmaHandle=%p, netMode=%d", rdmaHandle, netMode);
992 12 : unsigned int notifyType = netMode == HrtNetworkMode::PEER ? NO_USE : NOTIFY;
993 12 : s32 ret = RaRdevDeinit(rdmaHandle, notifyType);
994 12 : if (ret != 0) {
995 0 : MACRO_THROW(NetworkApiException, StringFormat("[DeInit][RaRdma]errNo[0x%016llx] rt rdev deinit fail. call RaRdevDeinit failed, rdmaHandle=%p, return[%d].",
996 : HCCL_ERROR_CODE(HcclResult::HCCL_E_NETWORK), rdmaHandle, ret));
997 : }
998 12 : }
999 :
1000 0 : void HrtRaGetNotifyBaseAddr(RdmaHandle rdmaHandle, u64 *va, u64 *size)
1001 : {
1002 0 : CHECK_NULLPTR(rdmaHandle, "[HrtRaGetNotifyBaseAddr] rdmaHandle is nullptr!");
1003 0 : CHECK_NULLPTR(va, "[HrtRaGetNotifyBaseAddr] va is nullptr!");
1004 0 : CHECK_NULLPTR(size, "[HrtRaGetNotifyBaseAddr] size is nullptr!");
1005 :
1006 0 : HCCL_INFO("[HrtRaGetNotifyBaseAddr] Input params: rdmaHandle=%p, va=%llu, size=%llu", rdmaHandle, *va, *size);
1007 0 : auto startTime = std::chrono::steady_clock::now();
1008 0 : auto timeout = std::chrono::seconds(EnvLinkTimeoutGet());
1009 : while (true) {
1010 : unsigned long long notifyVa;
1011 : unsigned long long notifySize;
1012 0 : s32 ret = RaGetNotifyBaseAddr(rdmaHandle, ¬ifyVa, ¬ifySize);
1013 0 : if (ret == 0) {
1014 0 : *va = notifyVa;
1015 0 : *size = notifySize;
1016 0 : break;
1017 0 : } else if (ret == SOCK_EAGAIN) {
1018 0 : bool bTimeout = ((std::chrono::steady_clock::now() - startTime) >= timeout);
1019 0 : if (bTimeout != 0) {
1020 0 : HCCL_ERROR("[Get][RaNotifyBaseAddr]errNo[0x%016llx] ra get notify base addr "
1021 : "timeout[%d]. return[%d], params: rdmaHandle[%p], va[0x%llx], size[%llu]",
1022 : HCCL_ERROR_CODE(HcclResult::HCCL_E_NETWORK), timeout, ret, rdmaHandle, notifyVa, notifySize);
1023 : }
1024 0 : SaluSleep(ONE_MILLISECOND_OF_USLEEP);
1025 : } else {
1026 0 : MACRO_THROW(NetworkApiException, StringFormat("[Get][RaNotifyBaseAddr]errNo[0x%016llx] ra get notify base addr fail, call RaGetNotifyBaseAddr failed,"
1027 : "return[%d], params: va[0x%llx], size[%llu], rdmaHandle=%p",
1028 : HCCL_ERROR_CODE(HcclResult::HCCL_E_NETWORK), ret, notifyVa, notifySize, rdmaHandle));
1029 : }
1030 0 : }
1031 0 : }
1032 :
1033 1 : QpHandle HrtRaQpCreate(RdmaHandle rdmaHandle, int flag, int qpMode)
1034 : {
1035 1 : CHECK_NULLPTR(rdmaHandle, "[HrtRaQpCreate] rdmaHandle is nullptr!");
1036 3 : HCCL_INFO("[HrtRaQpCreate] Input params: rdmaHandle=%p, flag=%d, qpMode=%d", rdmaHandle, flag, qpMode);
1037 1 : QpHandle connHandle = nullptr;
1038 :
1039 1 : s32 ret = RaQpCreate(rdmaHandle, flag, qpMode, &connHandle);
1040 1 : if (ret != 0 || connHandle == nullptr) {
1041 1 : RPT_INPUT_ERR(ret == ROCE_ENOMEM_RET, "EI0011", std::vector<std::string>({"memory_size"}), // A3是当ROCE_ENOMEM_RET才上报EI0011,内存大小取决于qp深度配置
1042 : std::vector<std::string>({"262144~3145728"}));
1043 4 : MACRO_THROW(NetworkApiException, StringFormat("[Create][RaQp]errNo[0x%016llx] ra qp create fail. call RaGetNotifyBaseAddr, params: rdmaHandle[%p], flag[%d], qpMode[%d], connHandle[%p]. return: ret[%d]",
1044 : HCCL_ERROR_CODE(HcclResult::HCCL_E_NETWORK), rdmaHandle, flag, qpMode, connHandle, ret));
1045 : }
1046 0 : return connHandle;
1047 : }
1048 :
1049 9 : void HrtRaQpDestroy(QpHandle qpHandle)
1050 : {
1051 9 : CHECK_NULLPTR(qpHandle, "[HrtRaQpDestroy] qpHandle is nullptr!");
1052 27 : HCCL_INFO("[HrtRaQpDestroy] Input params: qpHandle=%p", qpHandle);
1053 9 : auto startTime = std::chrono::steady_clock::now();
1054 9 : auto timeout = std::chrono::seconds(EnvLinkTimeoutGet());
1055 : while (true) {
1056 9 : s32 ret = RaQpDestroy(qpHandle);
1057 9 : if (ret == 0) {
1058 9 : break;
1059 0 : } else if (ret == SOCK_EAGAIN) {
1060 0 : bool bTimeout = ((std::chrono::steady_clock::now() - startTime) >= timeout);
1061 0 : if (bTimeout != 0) {
1062 0 : MACRO_THROW(NetworkApiException, StringFormat("[Destroy][RaQp]errNo[0x%016llx] ra qp destroy timeout[%d]. "
1063 : "qpHandle[%p], return[%d].",
1064 : HCCL_ERROR_CODE(HcclResult::HCCL_E_NETWORK), timeout, qpHandle, ret));
1065 : }
1066 0 : SaluSleep(ONE_MILLISECOND_OF_USLEEP);
1067 : } else {
1068 0 : MACRO_THROW(NetworkApiException, StringFormat("[Destroy][RaQp]errNo[0x%016llx] ra qp destroy fail. call RaQpDestroy failed, qpHandle[%p], return[%d].",
1069 : HCCL_ERROR_CODE(HcclResult::HCCL_E_NETWORK), qpHandle, ret));
1070 : }
1071 0 : }
1072 9 : }
1073 :
1074 0 : void HrtRaQpConnectAsync(QpHandle qpHandle, FdHandle fdHandle)
1075 : {
1076 0 : CHECK_NULLPTR(qpHandle, "[HrtRaQpConnectAsync] qpHandle is nullptr!");
1077 0 : CHECK_NULLPTR(fdHandle, "[HrtRaQpConnectAsync] fdHandle is nullptr!");
1078 :
1079 0 : HCCL_INFO("[HrtRaQpConnectAsync] Input params: qpHandle=%p, fdHandle=%p", qpHandle, fdHandle);
1080 0 : auto startTime = std::chrono::steady_clock::now();
1081 0 : auto timeout = std::chrono::seconds(EnvLinkTimeoutGet());
1082 : while (true) {
1083 0 : s32 ret = RaQpConnectAsync(qpHandle, fdHandle);
1084 0 : if (ret == 0) {
1085 0 : break;
1086 0 : } else if (ret == SOCK_EAGAIN) {
1087 0 : bool bTimeout = ((std::chrono::steady_clock::now() - startTime) >= timeout);
1088 0 : if (bTimeout != 0) {
1089 0 : HCCL_ERROR("[ConnectAsync][RaQp]errNo[0x%016llx] ra qp connect async "
1090 : "timeout[%lld]. qpHandle=%p, fdHandle=%p, return[%d].",
1091 : HCCL_ERROR_CODE(HcclResult::HCCL_E_NETWORK), timeout, qpHandle, fdHandle, ret);
1092 : }
1093 0 : SaluSleep(ONE_MILLISECOND_OF_USLEEP);
1094 : } else {
1095 0 : MACRO_THROW(NetworkApiException, StringFormat("[ConnectAsync][RaQp]errNo[0x%016llx] ra qp connect async fail. call RaQpConnectAsync failed, qpHandle=%p, fdHandle=%p, return[%d]",
1096 : HCCL_ERROR_CODE(HcclResult::HCCL_E_NETWORK), qpHandle, fdHandle, ret));
1097 : }
1098 0 : }
1099 0 : }
1100 :
1101 1 : int HrtGetRaQpStatus(QpHandle qpHandle)
1102 : {
1103 1 : CHECK_NULLPTR(qpHandle, "[HrtGetRaQpStatus] qpHandle is nullptr!");
1104 3 : HCCL_INFO("[HrtGetRaQpStatus] Input params: qpHandle=%p", qpHandle);
1105 1 : int status = 0;
1106 1 : s32 ret = RaGetQpStatus(qpHandle, &status);
1107 1 : if (ret != 0) {
1108 4 : MACRO_THROW(NetworkApiException, StringFormat("[GetStatus][RaQp]errNo[0x%016llx] ra qp get status failed. call ra_get_status failed, qpHandle[%p], return[%d]",
1109 : HCCL_ERROR_CODE(HcclResult::HCCL_E_NETWORK), qpHandle, ret));
1110 : }
1111 0 : return status;
1112 : }
1113 :
1114 1 : void HrtRaMrReg(QpHandle qpHandle, RaMrInfo &info)
1115 : {
1116 1 : CHECK_NULLPTR(qpHandle, "[HrtRaMrReg] qpHandle is nullptr!");
1117 1 : struct MrInfoT mrInfo = {};
1118 1 : mrInfo.addr = info.addr;
1119 1 : mrInfo.size = info.size;
1120 1 : mrInfo.access = info.access;
1121 1 : mrInfo.lkey = info.lkey;
1122 3 : HCCL_INFO("ra mr reg: qpHandle[%p], addr[%p], size[%llu], access[%d]", qpHandle, mrInfo.addr, mrInfo.size, mrInfo.access);
1123 1 : s32 ret = RaMrReg(qpHandle, &mrInfo);
1124 1 : if (ret != 0) {
1125 4 : MACRO_THROW(NetworkApiException, StringFormat("[Reg][RaMr]errNo[0x%016llx] ra mr reg fail. call RaMrReg failed, return[%d], params: qpHandle[%p], addr[%p], size[%llu], access[%d]",
1126 : HCCL_ERROR_CODE(HcclResult::HCCL_E_NETWORK), ret, qpHandle, mrInfo.addr, mrInfo.size, mrInfo.access));
1127 : }
1128 0 : }
1129 :
1130 1 : void HrtRaMrDereg(QpHandle qpHandle, RaMrInfo &info)
1131 : {
1132 1 : CHECK_NULLPTR(qpHandle, "[HrtRaMrDereg] qpHandle is nullptr!");
1133 1 : struct MrInfoT mrInfo = {};
1134 1 : mrInfo.addr = info.addr;
1135 1 : mrInfo.size = info.size;
1136 1 : mrInfo.access = info.access;
1137 1 : mrInfo.lkey = info.lkey;
1138 3 : HCCL_INFO("ra mr dereg: qpHandle[%p], addr[%p], size[%llu], access[%d]", qpHandle, mrInfo.addr, mrInfo.size, mrInfo.access);
1139 1 : s32 ret = RaMrDereg(qpHandle, &mrInfo);
1140 1 : if (ret != 0) {
1141 : string msg = StringFormat("call RaMrDereg failed, qpHandle=%p, addr=%p, size=%llu, access=%d", qpHandle,
1142 1 : mrInfo.addr, mrInfo.size, mrInfo.access);
1143 4 : MACRO_THROW(NetworkApiException, msg);
1144 1 : }
1145 0 : }
1146 :
1147 3 : static void HrtRaSendWr(QpHandle qpHandle, struct SendWr *wr, struct SendWrRsp *opRsp)
1148 : {
1149 6 : CHECK_NULLPTR(qpHandle, "[HrtRaSendWr] qpHandle is nullptr!");
1150 6 : CHECK_NULLPTR(wr, "[HrtRaSendWr] wr is nullptr!");
1151 3 : CHECK_NULLPTR(opRsp, "[HrtRaSendWr] opRsp is nullptr!");
1152 9 : HCCL_INFO("[HrtRaSendWr] Input params: qpHandle=%p, send_wrAddr=%p, opRspAddr=%p", qpHandle, wr, opRsp);
1153 3 : auto startTime = std::chrono::steady_clock::now();
1154 3 : auto timeout = std::chrono::seconds(EnvLinkTimeoutGet());
1155 : while (true) {
1156 3 : s32 ret = RaSendWr(qpHandle, wr, opRsp);
1157 3 : if (ret == 0) {
1158 3 : break;
1159 0 : } else if (ret == SOCK_ENOENT || ret == SOCK_EAGAIN) {
1160 0 : bool bTimeout = ((std::chrono::steady_clock::now() - startTime) >= timeout);
1161 0 : if (bTimeout) {
1162 0 : HCCL_ERROR("[Send][RaWr]errNo[0x%016llx] ra get send async timeout[%d]. "
1163 : "return[%d], params: qpHandle[%p], send_wrAddr[%p], opRspAddr[%p]",
1164 : HCCL_ERROR_CODE(HcclResult::HCCL_E_ROCE_TRANSFER), timeout, ret, qpHandle, wr, opRsp);
1165 0 : SaluSleep(ONE_MILLISECOND_OF_USLEEP);
1166 : }
1167 0 : } else {
1168 : string msg
1169 0 : = StringFormat("call RaSendWr failed, qpHandle=%p, send_wrAddr=%p opRspAddr=%p", qpHandle, wr, opRsp);
1170 0 : MACRO_THROW(NetworkApiException, msg);
1171 0 : }
1172 0 : }
1173 3 : }
1174 :
1175 3 : RaSendWrResp HrtRaSendOneWr(QpHandle qpHandle, HRaSendWr &in)
1176 : {
1177 3 : CHECK_NULLPTR(qpHandle, "[HrtRaSendOneWr] qpHandle is nullptr!");
1178 9 : HCCL_INFO("[HrtRaSendOneWr] Input params: qpHandle=%p, locAddr=0x%llx, len=%u, rmtAddr=0x%llx, op=%u, sendFlag=%d", qpHandle, in.locAddr, in.len, in.rmtAddr, in.op, in.sendFlag);
1179 3 : struct SgList bufList {};
1180 3 : bufList.addr = in.locAddr;
1181 3 : bufList.len = in.len;
1182 :
1183 3 : struct SendWr wr = {};
1184 3 : wr.op = in.op;
1185 3 : wr.dstAddr = in.rmtAddr;
1186 3 : wr.sendFlag = in.sendFlag;
1187 3 : wr.bufNum = 1; // 此处list只有一个，设置为1
1188 3 : wr.bufList = &bufList;
1189 3 : struct SendWrRsp opRsp = {};
1190 3 : HrtRaSendWr(qpHandle, &wr, &opRsp);
1191 :
1192 6 : return RaSendWrResp(opRsp.wqeTmp.sqIndex, opRsp.wqeTmp.wqeIndex, opRsp.db.dbIndex, opRsp.db.dbInfo);
1193 : }
1194 :
1195 0 : string HrtRaGetKeyDescribe(const u8 *key, u32 len)
1196 : {
1197 0 : CHECK_NULLPTR(key, "[HrtRaGetKeyDescribe] key is nullptr!");
1198 0 : HCCL_INFO("[HrtRaGetKeyDescribe] Input params: key=%d, len=%u", *key, len);
1199 0 : string desc = "0x";
1200 0 : for (u32 idx = 0; idx < len; idx++) {
1201 0 : desc += StringFormat("%02x", key[idx]);
1202 : }
1203 0 : return desc;
1204 0 : }
1205 :
1206 11 : RdmaHandle HrtRaUbCtxInit(const HrtRaUbCtxInitParam &in)
1207 : {
1208 33 : HCCL_INFO("[HrtRaUbCtxInit] Input params: mode=%d, phyId=%u, addr=%s", in.mode, in.phyId, in.addr.GetIpStr().c_str());
1209 11 : struct CtxInitCfg initCfg {};
1210 11 : initCfg.mode = HRT_NETWORK_MODE_MAP.at(in.mode);
1211 :
1212 11 : struct CtxInitAttr ctxInfo {};
1213 11 : ctxInfo.phyId = in.phyId;
1214 : // urma_create_context(eidIndex) 决定 ctx 的 local EID，须与 GetTpList/Import 使用的链路 EID 一致
1215 11 : ctxInfo.ub.eidIndex = 0U;
1216 11 : const Eid linkEid = in.addr.GetEid();
1217 : try {
1218 11 : const vector<HrtDevEidInfo> eidInfoList = HrtRaGetDevEidInfoList(HRaInfo(in.mode, in.phyId));
1219 11 : bool matched = false;
1220 11 : for (const auto &eidInfo : eidInfoList) {
1221 0 : if (eidInfo.ipAddress.GetEid() == linkEid) {
1222 0 : ctxInfo.ub.eidIndex = eidInfo.eidIndex;
1223 0 : matched = true;
1224 0 : HCCL_INFO("[HrtRaUbCtxInit] linkEid[%s] matched eidIndex[%u].",
1225 : in.addr.Describe().c_str(), eidInfo.eidIndex);
1226 0 : break;
1227 : }
1228 : }
1229 11 : if (!matched) {
1230 33 : HCCL_WARNING("[HrtRaUbCtxInit] linkEid[%s] not found in dev eid list(size[%zu]), "
1231 : "fallback eidIndex[0].", in.addr.Describe().c_str(), eidInfoList.size());
1232 : }
1233 11 : } catch (const NetworkApiException &) {
1234 0 : HCCL_WARNING("[HrtRaUbCtxInit] HrtRaGetDevEidInfoList failed, fallback eidIndex[0], addr[%s].",
1235 : in.addr.Describe().c_str());
1236 0 : }
1237 33 : HCCL_INFO("[HrtRaUbCtxInit] use eid[%s] eidIndex[%u]", in.addr.Describe().c_str(), ctxInfo.ub.eidIndex);
1238 11 : s32 sRet = memcpy_s(ctxInfo.ub.eid.raw, sizeof(ctxInfo.ub.eid.raw), in.addr.GetEid().raw, sizeof(in.addr.GetEid().raw));
1239 11 : if (sRet != EOK) {
1240 0 : MACRO_THROW(InternalException, StringFormat("[HrtRaUbCtxInit]memcpy_s failed. sRet[%d]", sRet));
1241 : }
1242 :
1243 : RdmaHandle handle;
1244 11 : s32 ret = RaCtxInit(&initCfg, &ctxInfo, &handle);
1245 11 : if (ret != 0) {
1246 : string msg = StringFormat(
1247 : "[Init][RaUbCtx]errNo[0x%016llx] ub ctx init fail, mode[%d], phyId[%u], addr[%s], ret[%d]",
1248 0 : HCCL_ERROR_CODE(HcclResult::HCCL_E_NETWORK), in.mode, in.phyId, in.addr.GetIpStr().c_str(), ret);
1249 0 : MACRO_THROW(NetworkApiException, msg);
1250 0 : }
1251 11 : return handle;
1252 : }
1253 :
1254 28 : void HrtRaUbCtxDestroy(RdmaHandle handle)
1255 : {
1256 28 : CHECK_NULLPTR(handle, "[HrtRaUbCtxDestroy] handle is nullptr!");
1257 84 : HCCL_INFO("[HrtRaUbCtxDestroy] rdmaHandle[%llu].", handle);
1258 28 : s32 ret = RaCtxDeinit(handle);
1259 28 : if (ret != 0) {
1260 : string msg = StringFormat("[DeInit][RaRdma]errNo[0x%016llx] rt ctx deinit fail. handle[%p], return[%d].",
1261 0 : HCCL_ERROR_CODE(HcclResult::HCCL_E_NETWORK), handle, ret);
1262 0 : MACRO_THROW(NetworkApiException, msg);
1263 0 : }
1264 28 : }
1265 :
1266 153 : std::pair<TokenIdHandle, uint32_t> RaUbAllocTokenIdHandle(RdmaHandle handle)
1267 : {
1268 153 : CHECK_NULLPTR(handle, "[RaUbAllocTokenIdHandle] handle is nullptr!");
1269 459 : HCCL_INFO("[RaUbAllocTokenIdHandle] rdmaHandle[%p].", handle);
1270 153 : struct HccpTokenId out {};
1271 153 : void *tokenIdHandle = nullptr;
1272 153 : s32 ret = RaCtxTokenIdAlloc(handle, &out, &tokenIdHandle);
1273 153 : if (ret != 0) {
1274 0 : string msg = StringFormat("%s failed, set=%d, rdmaHandle=%p", __func__, ret, handle);
1275 0 : MACRO_THROW(NetworkApiException, msg);
1276 0 : }
1277 459 : HCCL_INFO("[RaUbAllocTokenIdHandle] tokenIdHandle[%p], rdmaHandle[%p]", tokenIdHandle, handle);
1278 153 : return {reinterpret_cast<TokenIdHandle>(tokenIdHandle), out.tokenId >> URMA_TOKEN_ID_RIGHT_SHIFT};
1279 : }
1280 :
1281 52 : void RaUbFreeTokenIdHandle(RdmaHandle handle, TokenIdHandle tokenIdHandle)
1282 : {
1283 52 : CHECK_NULLPTR(handle, "[RaUbFreeTokenIdHandle] handle is nullptr!");
1284 156 : HCCL_INFO("[RaUbFreeTokenIdHandle] rdmaHandle[%p], tokenIdHandle[0x%llx].", handle, tokenIdHandle);
1285 52 : s32 ret = RaCtxTokenIdFree(handle, reinterpret_cast<void *>(tokenIdHandle));
1286 52 : if (ret != 0) {
1287 : string msg = StringFormat("%s failed, set=%d, rdmaHandle=%p, tokenIdHandle=0x%llx.",
1288 1 : __func__, ret, handle, tokenIdHandle);
1289 4 : MACRO_THROW(NetworkApiException, msg);
1290 1 : }
1291 51 : }
1292 :
1293 : constexpr u64 UB_MEM_PAGE_SIZE = 4096;
1294 :
1295 622 : std::pair<u64, u64> BufAlign(u64 addr, u64 size)
1296 : {
1297 1866 : HCCL_INFO("[BufAlign] Input params: addr=0x%llx, size=%llu", addr, size);
1298 : // 待解决: 正式方案待讨论
1299 622 : u64 pageSize = UB_MEM_PAGE_SIZE;
1300 622 : u64 newAddr = addr & (~(static_cast<u64>(pageSize - 1))); // UB内存注册要求起始地址4k对齐
1301 622 : u64 offset = addr - newAddr;
1302 622 : u64 newSize = size + offset;
1303 1866 : HCCL_INFO("UB mem info: newAddr[%llx], newSize[%llu]", newAddr, newSize);
1304 :
1305 1244 : return std::make_pair(newAddr, newSize);
1306 : }
1307 :
1308 604 : HrtRaUbLocalMemRegOutParam HrtRaUbLocalMemReg(RdmaHandle handle, const HrtRaUbLocMemRegParam &in)
1309 : {
1310 604 : CHECK_NULLPTR(handle, "[HrtRaUbLocalMemReg] handle is nullptr!");
1311 1812 : HCCL_INFO("[HrtRaUbLocalMemReg] Input params: handle=%p, addr=0x%llx, size=%llu", handle, in.addr, in.size);
1312 604 : struct MrRegInfoT info {};
1313 604 : info.in.mem.addr = in.addr;
1314 604 : info.in.mem.size = in.size;
1315 :
1316 604 : info.in.ub.flags.value = 0;
1317 604 : info.in.ub.flags.bs.tokenPolicy = TOKEN_POLICY_PLAIN_TEXT;
1318 604 : info.in.ub.flags.bs.tokenIdValid = 1;
1319 604 : info.in.ub.flags.bs.access = MEM_SEG_ACCESS_READ | MEM_SEG_ACCESS_WRITE
1320 : | MEM_SEG_ACCESS_ATOMIC;
1321 604 : info.in.ub.flags.bs.nonPin = in.nonPin;
1322 604 : info.in.ub.tokenValue = in.tokenValue;
1323 604 : info.in.ub.tokenIdHandle = reinterpret_cast<void *>(in.tokenIdHandle);
1324 :
1325 604 : void *lmemHandle = nullptr;
1326 604 : s32 ret = RaCtxLmemRegister(handle, &info, &lmemHandle);
1327 604 : if (ret != 0) {
1328 0 : string msg = StringFormat("localMemReg failed, addr=0x%llx, size=0x%llx", in.addr, in.size);
1329 0 : MACRO_THROW(NetworkApiException, msg);
1330 0 : }
1331 :
1332 604 : HrtRaUbLocalMemRegOutParam out;
1333 604 : s32 sRet = memcpy_s(out.key, sizeof(out.key), info.out.key.value, info.out.key.size);
1334 604 : if (sRet != EOK) {
1335 0 : MACRO_THROW(InternalException, StringFormat("[HrtRaUbLocalMemReg]memcpy_s failed. sRet[%d]", sRet));
1336 : }
1337 :
1338 1812 : HCCL_INFO("[HrtRaUbLocalMemReg]UbLocalMemReg key.size=%u", info.out.key.size);
1339 604 : out.keySize = info.out.key.size;
1340 604 : out.handle = reinterpret_cast<LocMemHandle>(lmemHandle);
1341 604 : out.targetSegVa = info.out.ub.targetSegHandle;
1342 604 : info.in.ub.tokenValue = 0;
1343 1812 : HCCL_INFO("[HrtRaUbLocalMemReg]UB mem reg info: in.addr[%llx], in.size[%llu], out.targetSegVa[%llu]",
1344 : in.addr, in.size, out.targetSegVa);
1345 1208 : return out;
1346 : }
1347 :
1348 18 : void HrtRaUbLocalMemUnreg(RdmaHandle rdmaHandle, LocMemHandle lmemHandle)
1349 : {
1350 18 : CHECK_NULLPTR(rdmaHandle, "[HrtRaUbLocalMemUnreg] rdmaHandle is nullptr!");
1351 54 : HCCL_INFO("[HrtRaUbLocalMemUnreg] Input params: rdmaHandle=%p, lmemHandle=0x%llx", rdmaHandle, lmemHandle);
1352 18 : s32 ret = RaCtxLmemUnregister(rdmaHandle, reinterpret_cast<void *>(lmemHandle));
1353 18 : if (ret != 0) {
1354 0 : string msg = StringFormat("localMemUnreg failed, rdmaHandle=%p, lmemHandle=0x%llx", rdmaHandle, lmemHandle);
1355 0 : MACRO_THROW(NetworkApiException, msg);
1356 0 : }
1357 18 : }
1358 :
1359 1 : HrtRaUbRemMemImportedOutParam HrtRaUbRemoteMemImport(RdmaHandle handle, u8 *key, u32 keyLen, u32 tokenValue)
1360 : {
1361 2 : CHECK_NULLPTR(handle, "[HrtRaUbRemoteMemImport] handle is nullptr!");
1362 1 : CHECK_NULLPTR(key, "[HrtRaUbRemoteMemImport] key is nullptr!");
1363 3 : HCCL_INFO("[HrtRaUbRemoteMemImport] Input params: handle=%p, key=%d, keyLen=%u", handle, *key, keyLen);
1364 1 : struct MrImportInfoT info {};
1365 1 : int res = memcpy_s(info.in.key.value, sizeof(info.in.key.value), key, keyLen);
1366 1 : if (res != 0) {
1367 0 : MACRO_THROW(InternalException, StringFormat("[%s] memcpy_s failed, ret = %d, params: handle=%p, key=%d, keyLen=%u", __func__, res, handle, *key, keyLen));
1368 : }
1369 1 : info.in.key.size = keyLen;
1370 :
1371 1 : info.in.ub.tokenValue = tokenValue;
1372 1 : info.in.ub.mappingAddr = 0;
1373 1 : info.in.ub.flags.value = 0;
1374 1 : info.in.ub.flags.bs.access = MEM_SEG_ACCESS_READ | MEM_SEG_ACCESS_WRITE
1375 : | MEM_SEG_ACCESS_ATOMIC;
1376 :
1377 1 : void *rmemHandle = nullptr;
1378 1 : s32 ret = RaCtxRmemImport(handle, &info, &rmemHandle);
1379 1 : if (ret != 0) {
1380 0 : string msg = StringFormat("ubRemoteMemImport failed!");
1381 0 : MACRO_THROW(NetworkApiException, msg);
1382 0 : }
1383 :
1384 1 : HrtRaUbRemMemImportedOutParam out;
1385 1 : out.handle = reinterpret_cast<LocMemHandle>(rmemHandle);
1386 1 : out.targetSegVa = info.out.ub.targetSegHandle;
1387 1 : info.in.ub.tokenValue = 0;
1388 1 : return out;
1389 : }
1390 5 : void HrtRaUbRemoteMemUnimport(RdmaHandle rdmaHandle, RemMemHandle rmemHandle)
1391 : {
1392 5 : CHECK_NULLPTR(rdmaHandle, "[HrtRaUbRemoteMemUnimport] rdmaHandle is nullptr!");
1393 15 : HCCL_INFO("[HrtRaUbRemoteMemUnimport] Input params: rdmaHandle=%p, rmemHandle=0x%llx", rdmaHandle, rmemHandle);
1394 5 : s32 ret = RaCtxRmemUnimport(rdmaHandle, reinterpret_cast<void *>(rmemHandle));
1395 5 : if (ret != 0) {
1396 : string msg
1397 0 : = StringFormat("ubRemoteMemUnimport failed, rdmaHandle=%p, rmemHandle=0x%llx", rdmaHandle, rmemHandle);
1398 0 : MACRO_THROW(NetworkApiException, msg);
1399 0 : }
1400 5 : }
1401 :
1402 : const std::map<HrtUbJfcMode, JfcMode> HRT_UB_JFC_MODE_MAP = {{HrtUbJfcMode::NORMAL, JfcMode::JFC_MODE_NORMAL},
1403 : {HrtUbJfcMode::STARS_POLL, JfcMode::JFC_MODE_STARS_POLL},
1404 : {HrtUbJfcMode::CCU_POLL, JfcMode::JFC_MODE_CCU_POLL},
1405 : {HrtUbJfcMode::USER_CTL, JfcMode::JFC_MODE_USER_CTL_NORMAL}};
1406 :
1407 : constexpr u32 CQ_DEPTH = 2 * 1024 * 1024 / 64;
1408 : constexpr u32 CCU_CQ_DEPTH = 64;
1409 :
1410 7 : JfcHandle HrtRaUbCreateJfc(RdmaHandle handle, CqCreateInfo& cqInfo, HrtUbJfcMode mode)
1411 : {
1412 7 : CHECK_NULLPTR(handle, "[HrtRaUbCreateJfc] handle is nullptr!");
1413 21 : HCCL_INFO("[HrtRaUbCreateJfc] Input params: handle=%p, mode=%d", handle, mode);
1414 7 : struct CqInfoT info {};
1415 :
1416 7 : info.in.chanHandle = nullptr;
1417 7 : if (mode == HrtUbJfcMode::CCU_POLL) {
1418 2 : info.in.depth = CCU_CQ_DEPTH;
1419 : } else {
1420 5 : info.in.depth = CQ_DEPTH;
1421 : }
1422 7 : info.in.ub.userCtx = 0;
1423 7 : info.in.ub.mode = HRT_UB_JFC_MODE_MAP.at(mode);
1424 7 : info.in.ub.ceqn = 0;
1425 7 : info.in.ub.flag.value = 0;
1426 :
1427 7 : void *jfcHandle = nullptr;
1428 :
1429 7 : s32 ret = RaCtxCqCreate(handle, &info, &jfcHandle);
1430 7 : if (ret != 0) {
1431 0 : string msg = StringFormat("ubCreateCq failed, rdmaHandle=%p,", handle);
1432 0 : MACRO_THROW(NetworkApiException, msg);
1433 0 : }
1434 :
1435 7 : cqInfo.va = info.out.va;
1436 21 : HCCL_INFO("HrtRaUbCreateJfc va[%llu] mode[%u] jfcHandle[%p]",
1437 : cqInfo.va, info.in.ub.mode, jfcHandle);
1438 7 : return reinterpret_cast<JfcHandle>(jfcHandle);
1439 : }
1440 :
1441 5 : void HrtRaUbDestroyJfc(RdmaHandle handle, JfcHandle jfcHandle)
1442 : {
1443 6 : CHECK_NULLPTR(handle, "[HrtRaUbDestroyJfc] handle is nullptr!");
1444 12 : HCCL_INFO("[HrtRaUbDestroyJfc] Input params: handle=%p, jfcHandle=0x%llx", handle, jfcHandle);
1445 4 : s32 ret = RaCtxCqDestroy(handle, reinterpret_cast<void *>(jfcHandle));
1446 4 : if (ret != 0) {
1447 0 : string msg = StringFormat("ubCqDestroy failed, rdmaHandle=%p, jfcHandle=0x%llx", handle, jfcHandle);
1448 0 : MACRO_THROW(NetworkApiException, msg);
1449 0 : }
1450 4 : }
1451 :
1452 :
1453 1 : JfcHandle HrtRaUbCreateJfcUserCtl(RdmaHandle handle, CqCreateInfo& cqInfo)
1454 : {
1455 1 : CHECK_NULLPTR(handle, "[HrtRaUbCreateJfcUserCtl] handle is nullptr!");
1456 3 : HCCL_INFO("[HrtRaUbCreateJfcUserCtl] Input params: handle=%p", handle);
1457 1 : struct CqInfoT info {};
1458 :
1459 1 : info.in.chanHandle = nullptr;
1460 1 : info.in.depth = CQ_DEPTH;
1461 1 : info.in.ub.userCtx = 0;
1462 1 : info.in.ub.mode = JfcMode::JFC_MODE_USER_CTL_NORMAL;
1463 1 : info.in.ub.ceqn = 0;
1464 1 : info.in.ub.flag.value = 0;
1465 :
1466 1 : void *jfcHandle = nullptr;
1467 :
1468 1 : s32 ret = RaCtxCqCreate(handle, &info, &jfcHandle);
1469 1 : if (ret != 0) {
1470 0 : string msg = StringFormat("ubCreateCq failed, rdmaHandle=%p,", handle);
1471 0 : THROW<NetworkApiException>(msg);
1472 0 : }
1473 :
1474 3 : HCCL_INFO("[HrtRaUbCreateJfcUserCtl] jfcId[%u], cqVA[%llx], cqeSize[%u], cqDepth[%u], dbAddr[%llx]",
1475 : info.out.id, info.out.bufAddr, info.out.cqeSize, CQ_DEPTH, info.out.swdbAddr);
1476 :
1477 1 : cqInfo.va = info.out.bufAddr;
1478 1 : cqInfo.id = info.out.id;
1479 1 : cqInfo.cqeSize = info.out.cqeSize;
1480 1 : cqInfo.cqDepth = CQ_DEPTH;
1481 1 : cqInfo.swdbAddr = info.out.swdbAddr;
1482 :
1483 1 : return reinterpret_cast<JfcHandle>(jfcHandle);
1484 : }
1485 :
1486 : const std::map<HrtTransportMode, TransportModeT> HRT_TRANSPORT_MODE_MAP
1487 : = {{HrtTransportMode::RM, TransportModeT::CONN_RM}};
1488 :
1489 : const std::map<HrtJettyMode, JettyMode> HRT_JETTY_MODE_MAP
1490 : = {{HrtJettyMode::STANDARD, JettyMode::JETTY_MODE_URMA_NORMAL},
1491 : {HrtJettyMode::HOST_OFFLOAD, JettyMode::JETTY_MODE_USER_CTL_NORMAL},
1492 : {HrtJettyMode::HOST_OPBASE, JettyMode::JETTY_MODE_USER_CTL_NORMAL},
1493 : {HrtJettyMode::DEV_USED, JettyMode::JETTY_MODE_USER_CTL_NORMAL},
1494 : {HrtJettyMode::CACHE_LOCK_DWQE, JettyMode::JETTY_MODE_CACHE_LOCK_DWQE},
1495 : {HrtJettyMode::CCU_CCUM_CACHE, JettyMode::JETTY_MODE_CCU}};
1496 :
1497 : constexpr u8 RNR_RETRY = 7;
1498 : constexpr u32 RQ_DEPTH = 256;
1499 :
1500 135 : static struct QpCreateAttr GetQpCreateAttr(const HrtRaUbCreateJettyParam &in)
1501 : {
1502 135 : struct QpCreateAttr attr {};
1503 135 : attr.scqHandle = reinterpret_cast<void *>(in.sjfcHandle);
1504 135 : attr.rcqHandle = reinterpret_cast<void *>(in.rjfcHandle);
1505 135 : attr.srqHandle = reinterpret_cast<void *>(in.sjfcHandle);
1506 135 : attr.rqDepth = RQ_DEPTH;
1507 135 : attr.sqDepth = in.sqDepth;
1508 135 : attr.transportMode = HRT_TRANSPORT_MODE_MAP.at(in.transMode);
1509 135 : attr.ub.mode = HRT_JETTY_MODE_MAP.at(in.jettyMode);
1510 :
1511 135 : attr.ub.tokenValue = in.tokenValue;
1512 135 : attr.ub.tokenIdHandle = reinterpret_cast<void *>(in.tokenIdHandle);
1513 135 : attr.ub.flag.value = 0;
1514 : /* errTime配置值：0-31
1515 : 0-7代表芯片配置值b00:512ms
1516 : 8-15代表芯片配置值b01:1s
1517 : 16-23代表芯片配置值b10:8s
1518 : 24-31代表芯片配置值b11:32s
1519 : */
1520 135 : attr.ub.errTimeout = in.errTimeout;
1521 135 : attr.ub.priority = static_cast<uint8_t>(in.qos & 0xFU);
1522 135 : attr.ub.rnrRetry = RNR_RETRY;
1523 135 : attr.ub.flag.bs.shareJfr = 1;
1524 135 : attr.ub.jettyId = in.jettyId;
1525 : // 在continue模式下+配置了wqe的fence标记，并且远端有一些权限校验错误/内存异常错误，硬件会直接挂死
1526 : // jfs_flag 的 error_suspend 设置为 1，
1527 135 : attr.ub.jfsFlag.bs.errorSuspend = 1;
1528 :
1529 135 : attr.ub.extMode.sqebbNum = in.sqDepth;
1530 135 : if (in.jettyMode == HrtJettyMode::HOST_OFFLOAD) {
1531 6 : attr.ub.extMode.piType = 1;
1532 6 : attr.ub.extMode.cstmFlag.bs.sqCstm = 0; // 表示不指定Va，由HCCP返回Va
1533 129 : } else if (in.jettyMode == HrtJettyMode::CCU_CCUM_CACHE) {
1534 26 : attr.ub.tokenValue = in.tokenValue;
1535 26 : attr.ub.extMode.cstmFlag.bs.sqCstm = 1;
1536 26 : attr.ub.extMode.sq.buffSize = in.sqBufSize;
1537 26 : attr.ub.extMode.sq.buffVa = in.sqBufVa;
1538 205 : } else if (in.jettyMode == HrtJettyMode::DEV_USED ||
1539 102 : in.jettyMode == HrtJettyMode::CACHE_LOCK_DWQE) {
1540 1 : attr.ub.extMode.cstmFlag.bs.sqCstm = 0; // 表示不指定Va，由HCCP返回Va
1541 1 : attr.ub.extMode.sq.buffSize = in.sqBufSize;
1542 1 : attr.ub.extMode.sq.buffVa = in.sqBufVa;
1543 : } // 预埋HrtJettyMode::CACHE_LOCK_DWQE类型，当前流程暂未使用
1544 :
1545 : // 其他Mode暂时不需要额外更新特定字段
1546 405 : HCCL_INFO("Create jetty, input params: attr.ub.jettyId[%u], attr.rqDepth[%u], "
1547 : "attr.sqDepth[%u], attr.transportMode[%d], attr.ub.mode[%d], "
1548 : "attr.ub.extMode.sqebbNum[%u], attr.ub.extMode.sq.buffVa[%llx], "
1549 : "attr.ub.extMode.sq.buffSize[%u], attr.ub.extMode.piType[%u], attr.ub.priority[%u], timeout[%u].",
1550 : attr.ub.jettyId, attr.rqDepth, attr.sqDepth, attr.transportMode, attr.ub.mode,
1551 : attr.ub.extMode.sqebbNum, attr.ub.extMode.sq.buffVa, attr.ub.extMode.sq.buffSize,
1552 : attr.ub.extMode.piType, attr.ub.priority, attr.ub.errTimeout);
1553 135 : return attr;
1554 : }
1555 :
1556 29 : HrtRaUbJettyCreatedOutParam HrtRaUbCreateJetty(RdmaHandle handle, const HrtRaUbCreateJettyParam &in)
1557 : {
1558 29 : CHECK_NULLPTR(handle, "[HrtRaUbCreateJetty] handle is nullptr!");
1559 87 : HCCL_INFO("[HrtRaUbCreateJetty] Input params: handle=%p", handle);
1560 29 : struct QpCreateAttr attr = GetQpCreateAttr(in);
1561 :
1562 29 : struct QpCreateInfo info {};
1563 29 : void *qpHandle = nullptr;
1564 29 : s32 ret = RaCtxQpCreate(handle, &attr, &info, &qpHandle);
1565 29 : if (ret != 0) {
1566 0 : string msg = StringFormat("ubCreateJetty failed, rdmaHandle=%p,", handle);
1567 0 : MACRO_THROW(NetworkApiException, msg);
1568 0 : }
1569 :
1570 29 : HrtRaUbJettyCreatedOutParam out;
1571 29 : out.handle = reinterpret_cast<JettyHandle>(qpHandle);
1572 29 : out.id = info.ub.id;
1573 29 : out.uasid = info.ub.uasid;
1574 29 : out.jettyVa = info.va;
1575 29 : out.dbVa = info.ub.dbAddr;
1576 29 : out.dbTokenId = info.ub.dbTokenId >> URMA_TOKEN_ID_RIGHT_SHIFT;
1577 29 : out.sqBuffVa = info.ub.sqBuffVa; // 适配HCCP修改，jettybufva由HCCP提供，不再由HCCL分配
1578 :
1579 29 : s32 sRet = memcpy_s(out.key, sizeof(out.key), info.key.value, info.key.size);
1580 29 : if (sRet != EOK) {
1581 0 : MACRO_THROW(InternalException, StringFormat("HrtRaUbCreateJetty memcpy_s failed. sRet[%d], params: handle=%p", sRet, handle));
1582 : }
1583 29 : out.keySize = info.key.size;
1584 29 : attr.ub.tokenValue = 0;
1585 87 : HCCL_INFO("Create jetty success, handle[%llu] jettyVa[%llu]", out.handle, out.jettyVa);
1586 58 : return out;
1587 : }
1588 :
1589 5 : void HrtRaUbDestroyJetty(JettyHandle jettyHandle)
1590 : {
1591 15 : HCCL_INFO("[HrtRaUbDestroyJetty] Input params: jettyHandle=0x%llx", jettyHandle);
1592 5 : s32 ret = RaCtxQpDestroy(reinterpret_cast<void *>(jettyHandle));
1593 5 : if (ret != 0) {
1594 0 : string msg = StringFormat("ubDestroyJetty failed, jettyHandle=0x%llx", jettyHandle);
1595 0 : MACRO_THROW(NetworkApiException, msg);
1596 0 : }
1597 5 : }
1598 :
1599 25 : static HrtRaUbJettyImportedOutParam ImportJetty(RdmaHandle handle, u8 *key, u32 keyLen,
1600 : u32 tokenValue, JettyImportExpCfg cfg, JettyImportMode mode, TpProtocol protocol = TpProtocol::INVALID)
1601 : {
1602 50 : CHECK_NULLPTR(handle, "[ImportJetty] handle is nullptr!");
1603 25 : CHECK_NULLPTR(key, "[ImportJetty] key is nullptr!");
1604 75 : HCCL_INFO("[ImportJetty] Input params: handle=%p, key=%d, keyLen=%u, mode=%d", handle, *key, keyLen, mode);
1605 25 : if (mode == JettyImportMode::JETTY_IMPORT_MODE_NORMAL) {
1606 0 : MACRO_THROW(NotSupportException, StringFormat("[%s] currently not support JETTY_IMPORT_MODE_NORMAL.",
1607 : __func__));
1608 : }
1609 :
1610 25 : struct QpImportInfoT info {};
1611 :
1612 25 : int res = memcpy_s(info.in.key.value, sizeof(info.in.key.value), key, keyLen);
1613 25 : if (res != 0) {
1614 0 : MACRO_THROW(InternalException, StringFormat("[%s] memcpy_s failed, ret = %d", __func__, res));
1615 : }
1616 25 : info.in.key.size = keyLen;
1617 :
1618 25 : info.in.ub.mode = mode;
1619 25 : info.in.ub.tokenValue = tokenValue;
1620 25 : info.in.ub.policy = JettyGrpPolicy::JETTY_GRP_POLICY_RR;
1621 25 : info.in.ub.type = TargetType::TARGET_TYPE_JETTY;
1622 :
1623 25 : info.in.ub.flag.value = 0;
1624 25 : info.in.ub.flag.bs.tokenPolicy = TOKEN_POLICY_PLAIN_TEXT;
1625 :
1626 25 : info.in.ub.expImportCfg = cfg;
1627 :
1628 26 : if (protocol != TpProtocol::TP && protocol != TpProtocol::CTP && protocol != TpProtocol::UBOE
1629 26 : && protocol != TpProtocol::UBG) {
1630 4 : MACRO_THROW(NetworkApiException, StringFormat("[%s] failed, tp protocol[%s] is not expected.",
1631 : __func__, protocol.Describe().c_str()));
1632 : }
1633 : // tpType: 0->RTP, 1->CTP
1634 24 : info.in.ub.tpType = protocol == TpProtocol::TP ? 0 : 1;
1635 :
1636 24 : void *remQpHandle = nullptr;
1637 24 : s32 ret = RaCtxQpImport(handle, &info, &remQpHandle);
1638 24 : if (ret != 0) {
1639 0 : string msg = StringFormat("UbImportJetty failed, rdmaHandle=%p,", handle);
1640 0 : MACRO_THROW(NetworkApiException, msg);
1641 0 : }
1642 :
1643 24 : HrtRaUbJettyImportedOutParam out;
1644 24 : out.handle = reinterpret_cast<TargetJettyHandle>(remQpHandle);
1645 24 : out.targetJettyVa = info.out.ub.tjettyHandle;
1646 24 : out.tpn = info.out.ub.tpn;
1647 :
1648 72 : HCCL_INFO("ImportJetty handle[%llu] targetJettyVa[%llu] tpn[%u]", out.handle, out.targetJettyVa, out.tpn);
1649 24 : info.in.ub.tokenValue = 0;
1650 48 : return out;
1651 : }
1652 :
1653 42 : static struct JettyImportExpCfg GetTpImportCfg(const JettyImportCfg &jettyImportCfg)
1654 : {
1655 42 : struct JettyImportExpCfg cfg = {};
1656 :
1657 42 : cfg.tpHandle = jettyImportCfg.localTpHandle;
1658 42 : cfg.peerTpHandle = jettyImportCfg.remoteTpHandle;
1659 42 : cfg.tag = jettyImportCfg.localTag;
1660 42 : cfg.txPsn = jettyImportCfg.localPsn;
1661 42 : cfg.rxPsn = jettyImportCfg.remotePsn;
1662 :
1663 126 : HCCL_INFO("GetTpImportCfg tpHandle[%llu] peerTpHandle[%llu] tag[%llu] txPsn[%llu] rxPsn[%llu]",
1664 : cfg.tpHandle, cfg.peerTpHandle, cfg.tag, cfg.txPsn, cfg.rxPsn);
1665 :
1666 42 : return cfg;
1667 : }
1668 :
1669 0 : HrtRaUbJettyImportedOutParam RaUbImportJetty(RdmaHandle handle, u8 *key, u32 keyLen, u32 tokenValue)
1670 : {
1671 0 : CHECK_NULLPTR(handle, "[RaUbImportJetty] handle is nullptr!");
1672 0 : CHECK_NULLPTR(key, "[RaUbImportJetty] key is nullptr!");
1673 0 : HCCL_INFO("[RaUbImportJetty] Input params: handle=%p, key=%d, keyLen=%u", handle, *key, keyLen);
1674 : // 该接口仅适配非管控面模式，当前不期望使用
1675 0 : struct JettyImportExpCfg cfg = {};
1676 0 : const auto mode = JettyImportMode::JETTY_IMPORT_MODE_NORMAL;
1677 0 : return ImportJetty(handle, key, keyLen, tokenValue, cfg, mode);
1678 : }
1679 :
1680 25 : HrtRaUbJettyImportedOutParam RaUbTpImportJetty(RdmaHandle handle, u8 *key, u32 keyLen,
1681 : u32 tokenValue, const JettyImportCfg &jettyImportCfg)
1682 : {
1683 50 : CHECK_NULLPTR(handle, "[RaUbTpImportJetty] handle is nullptr!");
1684 25 : CHECK_NULLPTR(key, "[RaUbTpImportJetty] key is nullptr!");
1685 75 : HCCL_INFO("[RaUbTpImportJetty] Input params: handle=%p", handle);
1686 25 : struct JettyImportExpCfg cfg = GetTpImportCfg(jettyImportCfg);
1687 25 : const auto mode = JettyImportMode::JETTY_IMPORT_MODE_EXP;
1688 49 : return ImportJetty(handle, key, keyLen, tokenValue, cfg, mode, jettyImportCfg.protocol);
1689 : }
1690 :
1691 1 : void HrtRaUbUnimportJetty(RdmaHandle handle, TargetJettyHandle targetJettyHandle)
1692 : {
1693 1 : CHECK_NULLPTR(handle, "[HrtRaUbUnimportJetty] handle is nullptr!");
1694 3 : HCCL_INFO("[HrtRaUbUnimportJetty] Input params: handle=%p, targetJettyHandle=0x%llx", handle, targetJettyHandle);
1695 1 : s32 ret = RaCtxQpUnimport(reinterpret_cast<void *>(handle), reinterpret_cast<void *>(targetJettyHandle));
1696 1 : if (ret != 0) {
1697 : string msg
1698 0 : = StringFormat("ubCqDestroy failed, rdmaHandle=%p, targetJettyHandle=0x%llx", handle, targetJettyHandle);
1699 0 : MACRO_THROW(NetworkApiException, msg);
1700 0 : }
1701 1 : }
1702 :
1703 1 : void HrtRaUbJettyBind(JettyHandle jettyHandle, TargetJettyHandle targetJettyHandle)
1704 : {
1705 3 : HCCL_INFO("[HrtRaUbJettyBind] Input params: jettyHandle=0x%llx, targetJettyHandle=0x%llx", jettyHandle, targetJettyHandle);
1706 1 : s32 ret = RaCtxQpBind(reinterpret_cast<void *>(jettyHandle), reinterpret_cast<void *>(targetJettyHandle));
1707 1 : if (ret != 0) {
1708 : string msg = StringFormat("ubJettyBind failed, jettyHandle=0x%llx, targetJettyHandle=0x%llx", jettyHandle,
1709 0 : targetJettyHandle);
1710 0 : MACRO_THROW(NetworkApiException, msg);
1711 0 : }
1712 1 : }
1713 :
1714 1 : void HrtRaUbJettyUnbind(JettyHandle jettyHandle)
1715 : {
1716 3 : HCCL_INFO("[HrtRaUbJettyUnbind] Input params: jettyHandle=0x%llx", jettyHandle);
1717 1 : s32 ret = RaCtxQpUnbind(reinterpret_cast<void *>(jettyHandle));
1718 1 : if (ret != 0) {
1719 0 : string msg = StringFormat("ubJettyUnbind failed, jettyHandle=0x%llx", jettyHandle);
1720 0 : MACRO_THROW(NetworkApiException, msg);
1721 0 : }
1722 1 : }
1723 :
1724 : const std::map<HrtUbSendWrOpCode, RaUbOpcode> HRT_UB_SEND_WR_OP_CODE_MAP
1725 : = {{HrtUbSendWrOpCode::WRITE, RaUbOpcode::RA_UB_OPC_WRITE},
1726 : {HrtUbSendWrOpCode::WRITE_WITH_NOTIFY, RaUbOpcode::RA_UB_OPC_WRITE_NOTIFY},
1727 : {HrtUbSendWrOpCode::READ, RaUbOpcode::RA_UB_OPC_READ},
1728 : {HrtUbSendWrOpCode::NOP, RaUbOpcode::RA_UB_OPC_NOP}};
1729 :
1730 : const std::map<ReduceOp, u8> HRT_UB_REDUCE_OP_CODE_MAP
1731 : = {{ReduceOp::SUM, 0xA}, {ReduceOp::MAX, 0x8}, {ReduceOp::MIN, 0x9}};
1732 :
1733 : const std::map<DataType, u8> HRT_UB_REDUCE_DATA_TYPE_MAP
1734 : = {{DataType::INT8, 0x0}, {DataType::INT16, 0x1}, {DataType::INT32, 0x2}, {DataType::UINT8, 0x3},
1735 : {DataType::UINT16, 0x4}, {DataType::UINT32, 0x5}, {DataType::FP16, 0x6}, {DataType::FP32, 0x7},
1736 : {DataType::BFP16, 0x8}, {DataType::BF16_SAT, 0x9}};
1737 :
1738 9 : static void ConstructWrSge(HrtRaUbSendWrReqParam &in, struct WrSgeList &sge)
1739 : {
1740 9 : sge.addr = in.localAddr;
1741 9 : sge.len = in.size;
1742 9 : sge.lmemHandle = reinterpret_cast<void *>(in.lmemHandle);
1743 9 : }
1744 :
1745 9 : static void ConstructSendWrReq(HrtRaUbSendWrReqParam &in, struct WrSgeList &sge, struct SendWrData &sendWr)
1746 : {
1747 : // 看一下hccp测试用例的入参
1748 9 : sendWr.numSge = 1;
1749 9 : sendWr.sges = &sge;
1750 9 : sendWr.remoteAddr = in.remoteAddr;
1751 9 : sendWr.rmemHandle = reinterpret_cast<void *>(in.rmemHandle);
1752 9 : sendWr.ub.userCtx = 0;
1753 9 : sendWr.ub.opcode = HRT_UB_SEND_WR_OP_CODE_MAP.at(in.opcode);
1754 9 : sendWr.ub.flags.value = 0;
1755 9 : sendWr.ub.flags.bs.compOrder = 1;
1756 9 : sendWr.ub.flags.bs.completeEnable = in.cqeEn;
1757 9 : sendWr.ub.flags.bs.fence = 1;
1758 9 : sendWr.ub.flags.bs.solicitedEnable = 1;
1759 9 : sendWr.ub.remQpHandle = reinterpret_cast<void *>(in.handle);
1760 9 : sendWr.ub.flags.bs.inlineFlag = in.inlineFlag;
1761 9 : if (sendWr.ub.flags.bs.inlineFlag) {
1762 3 : sendWr.inlineData = in.inlineData;
1763 3 : sendWr.inlineSize = in.size;
1764 : }
1765 9 : sendWr.ub.reduceInfo.reduceEn = in.inlineReduceFlag;
1766 9 : if (sendWr.ub.reduceInfo.reduceEn) {
1767 4 : sendWr.ub.reduceInfo.reduceOpcode = HRT_UB_REDUCE_OP_CODE_MAP.at(in.reduceOp);
1768 4 : sendWr.ub.reduceInfo.reduceDataType = HRT_UB_REDUCE_DATA_TYPE_MAP.at(in.dataType);
1769 : }
1770 9 : if (sendWr.ub.opcode == RaUbOpcode::RA_UB_OPC_WRITE_NOTIFY) {
1771 3 : sendWr.ub.notifyInfo.notifyData = in.notifyData;
1772 3 : sendWr.ub.notifyInfo.notifyAddr = in.notifyAddr;
1773 3 : sendWr.ub.notifyInfo.notifyHandle = reinterpret_cast<void *>(in.notifyHandle);
1774 : }
1775 9 : }
1776 :
1777 9 : HrtRaUbSendWrRespParam HrtRaUbPostSend(JettyHandle jettyHandle, HrtRaUbSendWrReqParam &in)
1778 : {
1779 9 : struct WrSgeList sge = {};
1780 9 : struct SendWrData sendWr {};
1781 :
1782 9 : ConstructWrSge(in, sge);
1783 9 : ConstructSendWrReq(in, sge, sendWr);
1784 :
1785 27 : HCCL_INFO("Sge addr = 0x%llx", in.localAddr);
1786 27 : HCCL_INFO("SendWR lmemHandle = 0x%llx", in.lmemHandle); // 和notifyFixedValue能否对齐
1787 27 : HCCL_INFO("SendWR rmemHandle = 0x%llx", in.rmemHandle); // remote
1788 27 : HCCL_INFO("SendWR remote addr = 0x%llx", in.remoteAddr);
1789 27 : HCCL_INFO("SendWR remote qp handle = 0x%llx", in.handle);
1790 27 : HCCL_INFO("SendWR jetty handle = 0x%llx", jettyHandle);
1791 :
1792 9 : SendWrResp sendWrResp{};
1793 :
1794 9 : u32 compNum = 0;
1795 9 : s32 ret = RaBatchSendWr(reinterpret_cast<void *>(jettyHandle), &sendWr, &sendWrResp, 1, &compNum);
1796 9 : if (ret != 0) {
1797 0 : string msg = StringFormat("UbJettySendWr failed, jettyHandle=0x%llx,", jettyHandle);
1798 0 : MACRO_THROW(NetworkApiException, msg);
1799 0 : }
1800 9 : HrtRaUbSendWrRespParam out;
1801 9 : out.dieId = sendWrResp.doorbellInfo.dieId;
1802 9 : out.funcId = sendWrResp.doorbellInfo.funcId;
1803 9 : out.jettyId = sendWrResp.doorbellInfo.jettyId;
1804 9 : out.piVal = sendWrResp.doorbellInfo.piVal;
1805 9 : out.dwqeSize = sendWrResp.doorbellInfo.dwqeSize;
1806 9 : ret = memcpy_s(out.dwqe, sizeof(out.dwqe), sendWrResp.doorbellInfo.dwqe, out.dwqeSize);
1807 9 : if (ret != 0) {
1808 0 : string msg = StringFormat("HrtRaUbPostSend copy dwqe failed, ret=%d", ret);
1809 0 : MACRO_THROW(InternalException, msg);
1810 0 : }
1811 :
1812 18 : return out;
1813 : }
1814 :
1815 15 : std::pair<uint32_t, uint32_t> HraGetDieAndFuncId(RdmaHandle handle)
1816 : {
1817 15 : CHECK_NULLPTR(handle, "[HraGetDieAndFuncId] handle is nullptr!");
1818 45 : HCCL_INFO("[HraGetDieAndFuncId] Input params: handle=%p", handle);
1819 15 : struct DevBaseAttr out {};
1820 15 : auto ret = RaGetDevBaseAttr(handle, &out);
1821 15 : if (ret != 0) {
1822 0 : MACRO_THROW(NetworkApiException, StringFormat("[%s] call ra_get_dev_base_attr failed, error code =%d.", __func__, ret));
1823 : }
1824 30 : return std::make_pair(out.ub.dieId, out.ub.funcId);
1825 : }
1826 :
1827 2 : bool HraGetRtpEnable(RdmaHandle handle)
1828 : {
1829 2 : struct DevBaseAttr out {};
1830 2 : auto ret = RaGetDevBaseAttr(handle, &out);
1831 2 : if (ret != 0) {
1832 0 : THROW<NetworkApiException>(StringFormat("[%s] call RaGetDevBaseAttr failed, error code =%d.", __func__, ret));
1833 : }
1834 :
1835 6 : HCCL_RUN_INFO("[%s] rmTpCap[%u] rcTpCap[%u] umTpCap[%u] tpFeat[%u]",
1836 : __func__, out.ub.rmTpCap.value, out.ub.rcTpCap.value, out.ub.umTpCap.value, out.ub.tpFeat.value);
1837 :
1838 18 : for (int i = 0; i < MAX_PRIORITY_CNT; i++) {
1839 17 : const CtxSlInfo &priorityInfo = out.ub.priorityInfo[i];
1840 51 : HCCL_RUN_INFO("[%s] priorityInfo[%d]: SL[%u] tpType[%u] rtp[%u]",
1841 : __func__, i, priorityInfo.SL, priorityInfo.tpType.value, priorityInfo.tpType.bs.rtp);
1842 17 : if (priorityInfo.tpType.bs.rtp == 1) {
1843 1 : return true;
1844 : }
1845 : }
1846 1 : return false;
1847 : }
1848 :
1849 2 : void HrtRaUbPostNops(JettyHandle jettyHandle, JettyHandle remoteJettyHandle, const u32 numNop)
1850 : {
1851 6 : HCCL_INFO("HrtRaUbPostNops: jettyHandle[0x%llx], remoteJettyHandle[0x%llx], numNop[%u]", jettyHandle, remoteJettyHandle, numNop);
1852 131 : struct SendWrData sendWrList[numNop] = {};
1853 131 : for (auto &sendWr : sendWrList) {
1854 129 : sendWr.ub.opcode = HRT_UB_SEND_WR_OP_CODE_MAP.at(HrtUbSendWrOpCode::NOP);
1855 387 : HCCL_INFO("SendWR opcode = %u", static_cast<u32>(sendWr.ub.opcode));
1856 : }
1857 2 : sendWrList[numNop - 1].ub.flags.bs.completeEnable = 1;
1858 :
1859 131 : SendWrResp sendWrRespList[numNop] = {};
1860 2 : u32 compNum = 0;
1861 2 : s32 ret = RaBatchSendWr(reinterpret_cast<void *>(jettyHandle), sendWrList, sendWrRespList, numNop, &compNum);
1862 2 : if (ret != 0) {
1863 1 : string msg = StringFormat("UbJettySendWr failed, jettyHandle=0x%llx,", jettyHandle);
1864 4 : MACRO_THROW(NetworkApiException, msg);
1865 1 : }
1866 3 : }
1867 :
1868 1 : void RaUbUpdateCi(JettyHandle jettyHandle, u32 ci)
1869 : {
1870 3 : HCCL_INFO("RaUbUpdateCi: jettyHandle=0x%llx, ci=%u", jettyHandle, ci);
1871 1 : s32 ret = RaCtxUpdateCi(reinterpret_cast<void *>(jettyHandle), ci);
1872 1 : if (ret != 0) {
1873 1 : string msg = StringFormat("UbUpdateCi failed, ret=%d, jettyHandle=0x%llx, ci=%u", ret, jettyHandle, ci);
1874 4 : MACRO_THROW(NetworkApiException, msg);
1875 1 : }
1876 0 : }
1877 :
1878 67 : inline string HccpEidDesc(union HccpEid& hccpEid)
1879 : {
1880 : return StringFormat("HccpEid[%016llx:%016llx]",
1881 67 : static_cast<unsigned long long>(be64toh(hccpEid.in6.subnetPrefix)),
1882 134 : static_cast<unsigned long long>(be64toh(hccpEid.in6.interfaceId)));
1883 : }
1884 :
1885 3 : inline IpAddress HccpEidToIpAddress(union HccpEid& hccpEid)
1886 : {
1887 3 : Eid eid{};
1888 9 : HCCL_INFO("[HccpEidToIpAddress] %s", HccpEidDesc(hccpEid).c_str());
1889 3 : s32 sRet = memcpy_s(eid.raw, sizeof(eid.raw), hccpEid.raw, sizeof(hccpEid.raw));
1890 3 : if (sRet != EOK) {
1891 0 : MACRO_THROW(InternalException, StringFormat("[HccpEidToIpAddress]memcpy_s failed. sRet[%d]", sRet));
1892 : }
1893 6 : return IpAddress(eid);
1894 : }
1895 :
1896 21 : std::vector<HrtDevEidInfo> HrtRaGetDevEidInfoList(const HRaInfo &raInfo)
1897 : {
1898 21 : std::vector<HrtDevEidInfo> hrtDevEidInfo;
1899 21 : struct RaInfo info {};
1900 21 : u32 num = 0;
1901 :
1902 21 : info.mode = HRT_NETWORK_MODE_MAP.at(raInfo.mode);
1903 21 : info.phyId = raInfo.phyId;
1904 :
1905 63 : HCCL_INFO("[HrtRaGetDevEidInfoList] Input params: mode=%d, phyId=%u", info.mode, info.phyId);
1906 21 : s32 ret = RaGetDevEidInfoNum(info, &num);
1907 21 : if (ret != 0) {
1908 1 : string msg = StringFormat("call RaGetDevEidInfoNum failed, error code =%d.", ret);
1909 4 : MACRO_THROW(NetworkApiException, msg);
1910 1 : }
1911 :
1912 24 : struct HccpDevEidInfo infoList[num] = {};
1913 20 : ret = RaGetDevEidInfoList(info, infoList, &num);
1914 20 : if (ret != 0) {
1915 1 : string msg = StringFormat("call RaGetDevEidInfoList failed, error code =%d.", ret);
1916 4 : MACRO_THROW(NetworkApiException, msg);
1917 1 : }
1918 :
1919 19 : hrtDevEidInfo.resize(num);
1920 21 : for (u32 i = 0; i < num; i++) {
1921 2 : hrtDevEidInfo[i].name = (infoList[i].name);
1922 2 : hrtDevEidInfo[i].ipAddress = HccpEidToIpAddress(infoList[i].eid);
1923 2 : hrtDevEidInfo[i].type = infoList[i].type;
1924 2 : hrtDevEidInfo[i].eidIndex = infoList[i].eidIndex;
1925 2 : hrtDevEidInfo[i].dieId = infoList[i].dieId;
1926 2 : hrtDevEidInfo[i].chipId = infoList[i].chipId;
1927 2 : hrtDevEidInfo[i].funcId = infoList[i].funcId;
1928 2 : hrtDevEidInfo[i].devFeature = infoList[i].devFeature;
1929 6 : HCCL_INFO("[%s] HrtDevEidInfo[%d]: name[%s], ipAddress[%s], type[%u], "
1930 : "eidIndex[%u], dieId[%u], chipId[%u], funcId[%u], devFeature[%u]",
1931 : __func__, i, hrtDevEidInfo[i].name.c_str(),
1932 : hrtDevEidInfo[i].ipAddress.Describe().c_str(),
1933 : hrtDevEidInfo[i].type, hrtDevEidInfo[i].eidIndex,
1934 : hrtDevEidInfo[i].dieId, hrtDevEidInfo[i].chipId,
1935 : hrtDevEidInfo[i].funcId, hrtDevEidInfo[i].devFeature);
1936 : }
1937 :
1938 19 : return hrtDevEidInfo;
1939 23 : }
1940 :
1941 79 : ReqHandleResult HrtRaGetAsyncReqResult(RequestHandle &reqHandle)
1942 : {
1943 79 : if (reqHandle == 0) {
1944 3 : HCCL_ERROR("[%s] failed, reqHandle is 0.params: reqHandle=0x%llx", __func__, reqHandle);
1945 1 : return ReqHandleResult::INVALID_PARA;
1946 : }
1947 :
1948 78 : int reqResult = 0;
1949 78 : s32 ret = RaGetAsyncReqResult(reinterpret_cast<void *>(reqHandle), &reqResult);
1950 : // 返回 OTHERS_EAGAIN 代表查询到异步任务未完成，需要重新查询，此时保留handle
1951 78 : if (ret == OTHERS_EAGAIN) {
1952 1 : return ReqHandleResult::NOT_COMPLETED;
1953 : }
1954 :
1955 : // 返回码非0代表调用查询接口失败，当前仅入参错误时触发
1956 77 : if (ret != 0) {
1957 4 : MACRO_THROW(NetworkApiException, StringFormat("[%s] failed, call interface error[%d], "
1958 : "reqhandle[%llu].", __func__, ret, reqHandle));
1959 : }
1960 :
1961 76 : RequestHandle tmpReqHandle = reqHandle;
1962 76 : reqHandle = 0;
1963 : // 返回码为 0 时，reqResult为异步任务完成结果，0代表成功，其他值代表失败
1964 : // SOCK_EAGAIN 为 socket 类执行结果，代表 socket 接口失败需要重试
1965 76 : if (reqResult == SOCK_EAGAIN) {
1966 1 : return ReqHandleResult::SOCK_E_AGAIN;
1967 : }
1968 :
1969 75 : if (reqResult != 0) {
1970 4 : MACRO_THROW(NetworkApiException, StringFormat("[%s] failed, the asynchronous request "
1971 : "error[%d], reqhandle[%llu].", __func__, reqResult, tmpReqHandle));
1972 : }
1973 :
1974 74 : return ReqHandleResult::COMPLETED;
1975 : }
1976 :
1977 6 : RequestHandle RaSocketConnectOneAsync(RaSocketConnectParam &in)
1978 : {
1979 18 : HCCL_INFO("[RaSocketConnectOneAsync] Input params: socketHandle=%p, remoteIp=%s, port=%u, tag=%s", in.socketHandle, in.remoteIp.Describe().c_str(), in.port, in.tag.c_str());
1980 6 : struct SocketConnectInfoT connInfo {};
1981 6 : connInfo.socketHandle = in.socketHandle;
1982 6 : connInfo.remoteIp = IpAddressToHccpIpAddr(in.remoteIp);
1983 6 : connInfo.port = in.port;
1984 :
1985 6 : int sret = strcpy_s(connInfo.tag, sizeof(connInfo.tag), in.tag.c_str());
1986 6 : if (sret != 0) {
1987 0 : MACRO_THROW(NetworkApiException, StringFormat(
1988 : "[%s] copy tag[%s] to hccp tag failed, ret=%d, connInfo.tag size=%zu, in.tag size=%zu",
1989 : __func__, in.tag.c_str(), sret, sizeof(connInfo.tag), sizeof(in.tag.c_str())));
1990 : }
1991 :
1992 18 : HCCL_INFO("Socket Connect tag=[%s], remoteIp[%s]", connInfo.tag, in.remoteIp.Describe().c_str());
1993 6 : void *raReqHandle = nullptr;
1994 6 : int ret = RaSocketBatchConnectAsync(&connInfo, SOCKET_NUM_ONE, &raReqHandle);
1995 6 : if (ret != 0) {
1996 0 : MACRO_THROW(NetworkApiException, StringFormat(
1997 : "[BatchConnect][RaSocket]errNo[0x%016llx] ra socket batch connect fail. return[%d]",
1998 : HCCL_ERROR_CODE(HcclResult::HCCL_E_TCP_CONNECT), ret));
1999 : }
2000 :
2001 6 : return reinterpret_cast<RequestHandle>(raReqHandle);
2002 : }
2003 :
2004 1 : RequestHandle RaSocketCloseOneAsync(RaSocketCloseParam &in)
2005 : {
2006 3 : HCCL_INFO("[RaSocketCloseOneAsync] Input params: socketHandle=%p, fdHandle=%p", in.socketHandle, in.fdHandle);
2007 1 : struct SocketCloseInfoT closeInfo = {};
2008 1 : closeInfo.fdHandle = in.fdHandle;
2009 1 : closeInfo.socketHandle = in.socketHandle;
2010 :
2011 1 : void *raReqHandle = nullptr;
2012 1 : int ret = RaSocketBatchCloseAsync(&closeInfo, SOCKET_NUM_ONE, &raReqHandle);
2013 1 : if (ret != 0) {
2014 0 : MACRO_THROW(NetworkApiException, StringFormat(
2015 : "[BatchClose][RaSocket]errNo[0x%016llx] ra socket batch close fail. return[%d]",
2016 : HCCL_ERROR_CODE(HcclResult::HCCL_E_TCP_CONNECT), ret));
2017 : }
2018 :
2019 1 : return reinterpret_cast<RequestHandle>(raReqHandle);
2020 : }
2021 :
2022 3 : RequestHandle RaSocketListenOneStartAsync(SocketListenInfoT* listenInfo)
2023 : {
2024 3 : if (listenInfo == nullptr) {
2025 0 : MACRO_THROW(NetworkApiException, StringFormat(
2026 : "errNo[0x%016llx] listenInfo is nullptr.",
2027 : HCCL_ERROR_CODE(HcclResult::HCCL_E_TCP_CONNECT)));
2028 : }
2029 9 : HCCL_INFO("[RaSocketListenOneStartAsync] Input params: listenInfo=%p, port=%u", listenInfo, listenInfo->port);
2030 :
2031 3 : void *raReqHandle = nullptr;
2032 3 : int ret = RaSocketListenStartAsync(listenInfo, SOCKET_NUM_ONE, &raReqHandle);
2033 3 : if (ret != 0) {
2034 0 : MACRO_THROW(NetworkApiException, StringFormat(
2035 : "errNo[0x%016llx] ra socket listen start fail. return[%d]",
2036 : HCCL_ERROR_CODE(HcclResult::HCCL_E_TCP_CONNECT), ret));
2037 : }
2038 :
2039 3 : return reinterpret_cast<RequestHandle>(raReqHandle);
2040 : }
2041 :
2042 1 : RequestHandle RaSocketListenOneStopAsync(RaSocketListenParam &in)
2043 : {
2044 3 : HCCL_INFO("[RaSocketListenOneStopAsync] Input params: socketHandle=%p, port=%u", in.socketHandle, in.port);
2045 1 : struct SocketListenInfoT listenInfo {};
2046 1 : listenInfo.socketHandle = in.socketHandle;
2047 1 : listenInfo.port = in.port;
2048 :
2049 1 : void *raReqHandle = nullptr;
2050 1 : int ret = RaSocketListenStopAsync(&listenInfo, SOCKET_NUM_ONE, &raReqHandle);
2051 1 : if (ret != 0) {
2052 0 : MACRO_THROW(NetworkApiException, StringFormat(
2053 : "[ListenStop][RaSocket]errNo[0x%016llx] ra socket listen stop fail. return[%d]",
2054 : HCCL_ERROR_CODE(HcclResult::HCCL_E_TCP_CONNECT), ret));
2055 : }
2056 :
2057 1 : return reinterpret_cast<RequestHandle>(raReqHandle);
2058 : }
2059 :
2060 11 : RaSocketFdHandleParam RaGetOneSocket(u32 role, RaSocketGetParam ¶m)
2061 : {
2062 11 : struct SocketInfoT socketInfo {};
2063 :
2064 11 : socketInfo.socketHandle = param.socketHandle;
2065 11 : socketInfo.fdHandle = param.fdHandle;
2066 11 : socketInfo.remoteIp = IpAddressToHccpIpAddr(param.remoteIp);
2067 11 : socketInfo.status = SOCKET_NOT_CONNECTED;
2068 :
2069 11 : int sret = strcpy_s(socketInfo.tag, sizeof(socketInfo.tag), param.tag.c_str());
2070 11 : if (sret != 0) {
2071 4 : MACRO_THROW(NetworkApiException, StringFormat("[%s] failed, copy tag[%s] to hccp failed, ret=%d, socketInfo.tag size=%zu, param.tag size=%zu",
2072 : __func__, param.tag.c_str(), sret, sizeof(socketInfo.tag), sizeof(param.tag.c_str())));
2073 : }
2074 :
2075 10 : u32 connectedNum = 0;
2076 10 : s32 sockRet = RaGetSockets(role, &socketInfo, SOCKET_NUM_ONE, &connectedNum);
2077 10 : if ((connectedNum == 0 && sockRet == 0) || sockRet == SOCK_EAGAIN) {
2078 : // 更新为 connecting 状态，表示连接未完成
2079 0 : socketInfo.status = SOCKET_CONNECTING;
2080 0 : return RaSocketFdHandleParam(socketInfo.fdHandle, socketInfo.status);
2081 : }
2082 :
2083 10 : if (sockRet != 0) {
2084 0 : MACRO_THROW(NetworkApiException, StringFormat("[%s] failed, call interface error[%d], "
2085 : "role[%u], num[%u], connectednum[%u]", __func__, sockRet, role, SOCKET_NUM_ONE, connectedNum));
2086 : }
2087 :
2088 10 : if (connectedNum > SOCKET_NUM_ONE) {
2089 4 : MACRO_THROW(NetworkApiException, StringFormat("[%s] failed, connetedNum[%u] is more "
2090 : "than expected[%u], role[%u], num[%u], connectednum[%u]", __func__, connectedNum,
2091 : SOCKET_NUM_ONE, sockRet, role, SOCKET_NUM_ONE, connectedNum));
2092 : }
2093 :
2094 9 : return RaSocketFdHandleParam(socketInfo.fdHandle, socketInfo.status);
2095 : }
2096 :
2097 5 : RequestHandle HrtRaSocketSendAsync(const FdHandle fdHandle, const void *data, u32 size,
2098 : unsigned long long &sentSize)
2099 : {
2100 10 : CHECK_NULLPTR(fdHandle, "[HrtRaSocketSendAsync] fdHandle is nullptr!");
2101 5 : CHECK_NULLPTR(data, "[HrtRaSocketSendAsync] data is nullptr!");
2102 15 : HCCL_INFO("[HrtRaSocketSendAsync] Input params: fdHandle=%p, data=%p, size=%u, sentSize=%llu", fdHandle, data, size, sentSize);
2103 5 : void *raReqHandle = nullptr;
2104 5 : s32 ret = RaSocketSendAsync(fdHandle, data, size, &sentSize, &raReqHandle);
2105 5 : if (ret != 0 || !raReqHandle) {
2106 0 : MACRO_THROW(NetworkApiException, StringFormat("[%s] failed, call interface error[%d] "
2107 : "raReqHandle[%p], fdHandle[%p], data[%p], size[%u], sentSize[%u].",
2108 : __func__, ret, raReqHandle, fdHandle, data, size, sentSize));
2109 : }
2110 :
2111 5 : return reinterpret_cast<RequestHandle>(raReqHandle);
2112 : }
2113 :
2114 5 : RequestHandle HrtRaSocketRecvAsync(const FdHandle fdHandle, void *data, u32 size,
2115 : unsigned long long &recvSize)
2116 : {
2117 10 : CHECK_NULLPTR(fdHandle, "[HrtRaSocketRecvAsync] fdHandle is nullptr!");
2118 5 : CHECK_NULLPTR(data, "[HrtRaSocketRecvAsync] data is nullptr!");
2119 15 : HCCL_INFO("[HrtRaSocketRecvAsync] Input params: fdHandle=%p, data=%p, size=%u, recvSize=%llu", fdHandle, data, size, recvSize);
2120 5 : void *raReqHandle = nullptr;
2121 5 : s32 ret = RaSocketRecvAsync(fdHandle, data, size, &recvSize, &raReqHandle);
2122 5 : if (ret != 0 || !raReqHandle) {
2123 0 : MACRO_THROW(NetworkApiException, StringFormat("[%s] failed, call interface error[%d], "
2124 : "raReqHandle[%p], fdHandle[%p], data[%p], size[%u], recvSize[%u].",
2125 : __func__, ret, raReqHandle, fdHandle, data, size, recvSize));
2126 : }
2127 :
2128 5 : return reinterpret_cast<RequestHandle>(raReqHandle);
2129 : }
2130 :
2131 1 : RequestHandle RaUbLocalMemRegAsync(RdmaHandle handle, const HrtRaUbLocMemRegParam &in,
2132 : vector<char_t> &out, void *&lmemHandle)
2133 : {
2134 2 : CHECK_NULLPTR(handle, "[RaUbLocalMemRegAsync] handle is nullptr!");
2135 1 : CHECK_NULLPTR(lmemHandle, "[RaUbLocalMemRegAsync] lmemHandle is nullptr!");
2136 3 : HCCL_INFO("[RaUbLocalMemRegAsync] Input params: handle=%p, addr=0x%llx, size=0x%llx, lmemHandle=%p", handle, in.addr, in.size, lmemHandle);
2137 1 : u64 pageSize = UB_MEM_PAGE_SIZE;
2138 1 : u64 newAddr = in.addr & (~(static_cast<u64>(pageSize - 1))); // UB内存注册要求起始地址4k对齐
2139 1 : u64 offset = in.addr - newAddr;
2140 1 : u64 newSize = in.size + offset + 4;
2141 :
2142 1 : out.resize(sizeof(struct MrRegInfoT));
2143 1 : struct MrRegInfoT *info = reinterpret_cast<struct MrRegInfoT *>(out.data());
2144 1 : info->in.mem.addr = newAddr;
2145 1 : info->in.mem.size = newSize;
2146 :
2147 1 : info->in.ub.flags.value = 0;
2148 1 : info->in.ub.flags.bs.tokenPolicy = TOKEN_POLICY_PLAIN_TEXT;
2149 1 : info->in.ub.flags.bs.tokenIdValid = 1;
2150 1 : info->in.ub.flags.bs.access = MEM_SEG_ACCESS_READ
2151 : | MEM_SEG_ACCESS_WRITE | MEM_SEG_ACCESS_ATOMIC;
2152 1 : info->in.ub.flags.bs.nonPin = in.nonPin;
2153 1 : info->in.ub.tokenValue = in.tokenValue;
2154 1 : info->in.ub.tokenIdHandle = reinterpret_cast<void *>(in.tokenIdHandle);
2155 :
2156 1 : void *raReqHandle = nullptr;
2157 1 : s32 ret = RaCtxLmemRegisterAsync(handle, info, &lmemHandle, &raReqHandle);
2158 1 : if (ret != 0 || !raReqHandle) {
2159 0 : MACRO_THROW(NetworkApiException, StringFormat("[%s] failed, call interface "
2160 : "error[%d], raReqHandle[%p], addr=0x%llx, size=0x%llx",
2161 : __func__, ret, raReqHandle, in.addr, in.size));
2162 : }
2163 1 : info->in.ub.tokenValue = 0;
2164 3 : HCCL_INFO("[%s] ok, get handle[%llu].", __func__, reinterpret_cast<RequestHandle>(raReqHandle));
2165 1 : return reinterpret_cast<RequestHandle>(raReqHandle);
2166 : }
2167 :
2168 1 : RequestHandle RaUbLocalMemUnregAsync(RdmaHandle rdmaHandle, LocMemHandle lmemHandle)
2169 : {
2170 1 : CHECK_NULLPTR(rdmaHandle, "[RaUbLocalMemUnregAsync] rdmaHandle is nullptr!");
2171 3 : HCCL_INFO("[RaUbLocalMemUnregAsync] Input params: rdmaHandle=%p, lmemHandle=0x%llx", rdmaHandle, lmemHandle);
2172 1 : void *raReqHandle = nullptr;
2173 1 : s32 ret = RaCtxLmemUnregisterAsync(rdmaHandle,
2174 : reinterpret_cast<void *>(lmemHandle), &raReqHandle);
2175 1 : if (ret != 0 || !raReqHandle) {
2176 0 : MACRO_THROW(NetworkApiException, StringFormat("[%s] failed, call interface error[%d] "
2177 : "raReqResult[%p], rdmaHandle=%p, lmemHandle=0x%llx.", __func__, ret, raReqHandle,
2178 : rdmaHandle, lmemHandle));
2179 : }
2180 :
2181 3 : HCCL_INFO("[%s] ok, get handle[%llu]", __func__, reinterpret_cast<RequestHandle>(raReqHandle));
2182 1 : return reinterpret_cast<RequestHandle>(raReqHandle);
2183 : }
2184 :
2185 106 : RequestHandle RaUbCreateJettyAsync(const RdmaHandle handle, const HrtRaUbCreateJettyParam &in,
2186 : vector<char_t> &out, void *&jettyHandle)
2187 : {
2188 106 : struct QpCreateAttr attr = GetQpCreateAttr(in);
2189 :
2190 106 : void *raReqHandle = nullptr;
2191 106 : out.resize(sizeof(QpCreateInfo));
2192 106 : s32 ret = RaCtxQpCreateAsync(handle, &attr, reinterpret_cast<QpCreateInfo *>(out.data()),
2193 : &jettyHandle, &raReqHandle);
2194 106 : if (ret != 0 || !raReqHandle) {
2195 0 : MACRO_THROW(NetworkApiException, StringFormat("[%s] failed, call interface error[%d], raReqHandle[%p], "
2196 : "rdmaHanlde[%p].", __func__, ret, raReqHandle, handle));
2197 : }
2198 106 : attr.ub.tokenValue = 0;
2199 318 : HCCL_INFO("[%s] ok, get handle[%llu].", __func__, reinterpret_cast<RequestHandle>(raReqHandle));
2200 106 : return reinterpret_cast<RequestHandle>(raReqHandle);
2201 : }
2202 :
2203 1 : RequestHandle RaUbDestroyJettyAsync(void *jettyHandle)
2204 : {
2205 1 : CHECK_NULLPTR(jettyHandle, "[RaUbDestroyJettyAsync] jettyHandle is nullptr!");
2206 3 : HCCL_INFO("[RaUbDestroyJettyAsync] Input params: jettyHandle=%p", jettyHandle);
2207 1 : void *raReqHandle = nullptr;
2208 1 : s32 ret = RaCtxQpDestroyAsync(jettyHandle, &raReqHandle);
2209 1 : if (ret != 0) {
2210 0 : MACRO_THROW(NetworkApiException, StringFormat("[%s] failed, call interface error[%d] raReqHandle[%p], "
2211 : "jettyHandle[%p].", __func__, ret, raReqHandle, jettyHandle));
2212 : }
2213 :
2214 3 : HCCL_INFO("[%s] ok, get handle[%llu].", __func__, reinterpret_cast<RequestHandle>(raReqHandle));
2215 1 : return reinterpret_cast<RequestHandle>(raReqHandle);
2216 : }
2217 :
2218 32 : inline HccpEid IpAddressToHccpEid(const IpAddress &ipAddr)
2219 : {
2220 32 : HccpEid eid = {};
2221 96 : HCCL_INFO("EID ipAddr[%s]", ipAddr.Describe().c_str());
2222 32 : s32 sRet = memcpy_s(eid.raw, sizeof(eid.raw), ipAddr.GetEid().raw, sizeof(ipAddr.GetEid().raw));
2223 32 : if (sRet != EOK) {
2224 0 : MACRO_THROW(InternalException, StringFormat("[IpAddressToHccpEid]memcpy_s failed. sRet[%d], dest[%p], destSize[%zu], src[%p], srcSize[%zu]",
2225 : sRet, eid.raw, sizeof(eid.raw), ipAddr.GetEid().raw, sizeof(ipAddr.GetEid().raw)));
2226 : }
2227 96 : HCCL_INFO("[IpAddressToHccpEid] %s", HccpEidDesc(eid).c_str());
2228 32 : return eid;
2229 : }
2230 :
2231 16 : RequestHandle RaUbGetTpInfoAsync(const RdmaHandle rdmaHandle, const RaUbGetTpInfoParam ¶m,
2232 : vector<char_t> &out, uint32_t &num)
2233 : {
2234 16 : CHECK_NULLPTR(rdmaHandle, "[RaUbGetTpInfoAsync] rdmaHandle is nullptr!");
2235 48 : HCCL_INFO("[RaUbGetTpInfoAsync] Input params: rdmaHandle=%p, num=%u", rdmaHandle, num);
2236 16 : const auto &locAddr = param.locAddr;
2237 16 : const auto &rmtAddr = param.rmtAddr;
2238 16 : const auto &tpProtocol = param.tpProtocol;
2239 :
2240 16 : struct GetTpCfg cfg{};
2241 : // UBG与TP同属RTP传输，需使能rtp位；UBOE走独立uboe位
2242 16 : cfg.flag.bs.rtp = (tpProtocol == TpProtocol::TP || tpProtocol == TpProtocol::UBG) ? 1 : 0;
2243 16 : cfg.flag.bs.ctp = tpProtocol == TpProtocol::CTP ? 1 : 0;
2244 16 : cfg.flag.bs.uboe = (tpProtocol == TpProtocol::UBOE) ? 1 : 0;
2245 16 : cfg.transMode = TransportModeT::CONN_RM; // 当前只使用RM Jetty
2246 16 : cfg.localEid = IpAddressToHccpEid(locAddr);
2247 48 : HCCL_INFO("RaUbGetTpInfoAsync cfg.localEid=%s", HccpEidDesc(cfg.localEid).c_str());
2248 16 : cfg.peerEid = IpAddressToHccpEid(rmtAddr);
2249 48 : HCCL_INFO("RaUbGetTpInfoAsync cfg.peerEid=%s", HccpEidDesc(cfg.peerEid).c_str());
2250 :
2251 : // 须至少容纳 TP_HANDLE_REQUEST_NUM 条 HccpTpInfo，避免 RS 按 num 写多条时越界破坏堆
2252 16 : out.resize(static_cast<size_t>(TP_HANDLE_REQUEST_NUM) * sizeof(struct HccpTpInfo));
2253 16 : struct HccpTpInfo *info = reinterpret_cast<struct HccpTpInfo *>(out.data());
2254 :
2255 16 : void *raReqHandle = nullptr;
2256 16 : num = TP_HANDLE_REQUEST_NUM; // 指定需要从管控面申请tp handle的数量, hccp 会返回实际个数
2257 16 : s32 ret = RaGetTpInfoListAsync(rdmaHandle, &cfg, info, &num, &raReqHandle);
2258 16 : if (ret != 0 || !raReqHandle) {
2259 4 : MACRO_THROW(NetworkApiException, StringFormat("[%s] failed, call interface error[%d] raReqHandle[%p], "
2260 : "rdmaHandle[%p], locAddr[%s], rmtAddr[%s].", __func__, ret, raReqHandle, rdmaHandle,
2261 : locAddr.Describe().c_str(), rmtAddr.Describe().c_str()));
2262 : }
2263 :
2264 45 : HCCL_INFO("[%s] ok, get handle[%llu].", __func__, reinterpret_cast<RequestHandle>(raReqHandle));
2265 15 : return reinterpret_cast<RequestHandle>(raReqHandle);
2266 : }
2267 :
2268 0 : void RaUbGetTpInfo(const RdmaHandle rdmaHandle, const RaUbGetTpInfoParam ¶m,
2269 : vector<char_t> &out, uint32_t &num)
2270 : {
2271 0 : CHECK_NULLPTR(rdmaHandle, "[RaUbGetTpInfo] rdmaHandle is nullptr!");
2272 0 : HCCL_INFO("[RaUbGetTpInfo] Input params: rdmaHandle=%p, num=%u", rdmaHandle, num);
2273 0 : const auto &locAddr = param.locAddr;
2274 0 : const auto &rmtAddr = param.rmtAddr;
2275 0 : const auto &tpProtocol = param.tpProtocol;
2276 :
2277 0 : struct GetTpCfg cfg{};
2278 : // UBG与TP同属RTP传输，需使能rtp位；UBOE走独立uboe位
2279 0 : cfg.flag.bs.rtp = (tpProtocol == TpProtocol::TP || tpProtocol == TpProtocol::UBG) ? 1 : 0;
2280 0 : cfg.flag.bs.ctp = tpProtocol == TpProtocol::CTP ? 1 : 0;
2281 0 : cfg.transMode = TransportModeT::CONN_RM; // 当前只使用RM Jetty
2282 0 : cfg.localEid = IpAddressToHccpEid(locAddr);
2283 0 : HCCL_INFO("RaUbGetTpInfo cfg.localEid=%s", HccpEidDesc(cfg.localEid).c_str());
2284 0 : cfg.peerEid = IpAddressToHccpEid(rmtAddr);
2285 0 : HCCL_INFO("RaUbGetTpInfo cfg.peerEid=%s", HccpEidDesc(cfg.peerEid).c_str());
2286 :
2287 0 : out.resize(static_cast<size_t>(TP_HANDLE_REQUEST_NUM) * sizeof(struct HccpTpInfo));
2288 0 : struct HccpTpInfo *info = reinterpret_cast<struct HccpTpInfo *>(out.data());
2289 :
2290 0 : num = TP_HANDLE_REQUEST_NUM; // 指定需要从管控面申请tp handle的数量, hccp 会返回实际个数
2291 0 : s32 ret = RaCtxGetTpInfoList(rdmaHandle, &cfg, info, &num);
2292 0 : if (ret != 0) {
2293 0 : MACRO_THROW(NetworkApiException, StringFormat("[%s] failed, call interface error[%d], "
2294 : "rdmaHandle[%p], locAddr[%s], rmtAddr[%s].", __func__, ret, rdmaHandle,
2295 : locAddr.Describe().c_str(), rmtAddr.Describe().c_str()));
2296 : }
2297 :
2298 0 : HCCL_INFO("[%s] ok, get handle[%llu].", __func__);
2299 0 : }
2300 :
2301 17 : static RequestHandle ImportJettyAsync(RdmaHandle rdmaHandle, const HrtRaUbJettyImportedInParam &in,
2302 : vector<char_t> &out, void *&remQpHandle, const JettyImportExpCfg &cfg, JettyImportMode mode,
2303 : TpProtocol protocol = TpProtocol::INVALID)
2304 : {
2305 17 : CHECK_NULLPTR(rdmaHandle, "[ImportJettyAsync] rdmaHandle is nullptr!");
2306 51 : HCCL_INFO("[ImportJettyAsync] Input params: rdmaHandle=%p, remQpHandle=%p", rdmaHandle, remQpHandle);
2307 17 : if (mode == JettyImportMode::JETTY_IMPORT_MODE_NORMAL) {
2308 0 : MACRO_THROW(NotSupportException, StringFormat("[%s] currently not support JETTY_IMPORT_MODE_NORMAL.",
2309 : __func__));
2310 : }
2311 :
2312 17 : out.resize(sizeof(QpImportInfoT));
2313 17 : struct QpImportInfoT *info = reinterpret_cast<QpImportInfoT *>(out.data());
2314 :
2315 17 : s32 ret = memcpy_s(info->in.key.value, sizeof(info->in.key.value), in.key, in.keyLen);
2316 17 : if (ret != 0) {
2317 0 : MACRO_THROW(InternalException, StringFormat("[%s] memcpy_s failed, ret=%d.", __func__, ret));
2318 : }
2319 :
2320 17 : info->in.key.size = in.keyLen;
2321 17 : info->in.ub.mode = mode;
2322 17 : info->in.ub.tokenValue = in.tokenValue;
2323 17 : info->in.ub.policy = JettyGrpPolicy::JETTY_GRP_POLICY_RR;
2324 17 : info->in.ub.type = TargetType::TARGET_TYPE_JETTY;
2325 :
2326 17 : info->in.ub.flag.value = 0;
2327 17 : info->in.ub.flag.bs.tokenPolicy = TOKEN_POLICY_PLAIN_TEXT;
2328 :
2329 17 : info->in.ub.expImportCfg = cfg;
2330 :
2331 31 : if (protocol != TpProtocol::TP && protocol != TpProtocol::CTP && protocol != TpProtocol::UBOE
2332 31 : && protocol != TpProtocol::UBG) {
2333 0 : MACRO_THROW(NetworkApiException, StringFormat("[%s] failed, tp protocol[%s] is not expected, %s.",
2334 : __func__, protocol.Describe().c_str()));
2335 : }
2336 : // tpType: 0->RTP, 1->CTP
2337 17 : info->in.ub.tpType = protocol == TpProtocol::TP ? 0 : 1;
2338 :
2339 17 : void *raReqHandle = nullptr;
2340 17 : ret = RaCtxQpImportAsync(rdmaHandle, info, &remQpHandle, &raReqHandle);
2341 17 : if (ret != 0 || !raReqHandle) {
2342 0 : MACRO_THROW(NetworkApiException, StringFormat("[%s] failed, call interface error[%d] raReqHandle[%p], "
2343 : "rdmaHandle[%p].", __func__, ret, raReqHandle, rdmaHandle));
2344 : }
2345 17 : info->in.ub.tokenValue = 0;
2346 51 : HCCL_INFO("[%s] ok, get handle[%llu]", __func__, reinterpret_cast<RequestHandle>(raReqHandle));
2347 17 : return reinterpret_cast<RequestHandle>(raReqHandle);
2348 : }
2349 :
2350 0 : RequestHandle RaUbImportJettyAsync(const RdmaHandle rdmaHandle, const HrtRaUbJettyImportedInParam &in,
2351 : vector<char_t> &out, void *&remQpHandle)
2352 : {
2353 0 : CHECK_NULLPTR(rdmaHandle, "[RaUbImportJettyAsync] rdmaHandle is nullptr!");
2354 0 : HCCL_INFO("[RaUbImportJettyAsync] Input params: rdmaHandle=%p, remQpHandle=%p", rdmaHandle, remQpHandle);
2355 : // 该接口仅适配非管控面模式，当前不期望使用
2356 0 : struct JettyImportExpCfg cfg = {};
2357 0 : const auto mode = JettyImportMode::JETTY_IMPORT_MODE_NORMAL;
2358 0 : return ImportJettyAsync(rdmaHandle, in, out, remQpHandle, cfg, mode);
2359 : }
2360 :
2361 17 : RequestHandle RaUbTpImportJettyAsync(const RdmaHandle rdmaHandle, const HrtRaUbJettyImportedInParam &in,
2362 : vector<char_t> &out, void *&remQpHandle)
2363 : {
2364 17 : CHECK_NULLPTR(rdmaHandle, "[RaUbTpImportJettyAsync] rdmaHandle is nullptr!");
2365 51 : HCCL_INFO("[RaUbTpImportJettyAsync] Input params: rdmaHandle=%p, remQpHandle=%p", rdmaHandle, remQpHandle);
2366 17 : struct JettyImportExpCfg cfg = GetTpImportCfg(in.jettyImportCfg);
2367 17 : const auto mode = JettyImportMode::JETTY_IMPORT_MODE_EXP;
2368 34 : return ImportJettyAsync(rdmaHandle, in, out, remQpHandle, cfg, mode, in.jettyImportCfg.protocol);
2369 : }
2370 :
2371 1 : RequestHandle RaUbUnimportJettyAsync(void *targetJettyHandle)
2372 : {
2373 1 : CHECK_NULLPTR(targetJettyHandle, "[RaUbUnimportJettyAsync] targetJettyHandle is nullptr!");
2374 3 : HCCL_INFO("[RaUbUnimportJettyAsync] Input params: targetJettyHandle=%p", targetJettyHandle);
2375 1 : void *raReqHandle = nullptr;
2376 1 : s32 ret = RaCtxQpUnimportAsync(targetJettyHandle, &raReqHandle);
2377 1 : if (ret != 0 || !raReqHandle) {
2378 0 : MACRO_THROW(NetworkApiException, StringFormat("[%s] failed, call interface error[%d] raReqHandle[%p], "
2379 : "targetJettyHandle[%p].", __func__, ret, raReqHandle, targetJettyHandle));
2380 : }
2381 :
2382 3 : HCCL_INFO("[%s] ok, get handle[%llu].", __func__, reinterpret_cast<RequestHandle>(raReqHandle));
2383 1 : return reinterpret_cast<RequestHandle>(raReqHandle);
2384 : }
2385 :
2386 0 : HcclResult HrtRaWaitEventHandle(int event_handle, std::vector<SocketEventInfo> &event_infos, int timeout,
2387 : unsigned int maxevents, u32 &events_num)
2388 : {
2389 0 : HCCL_INFO("[HrtRaWaitEventHandle] Input params: event_handle=%d, timeout=%d, maxevents=%u, events_num=%u", event_handle, timeout, maxevents, events_num);
2390 0 : std::vector<struct SocketEventInfoT> raEventInfos(maxevents);
2391 0 : s32 ret = RaWaitEventHandle(event_handle, raEventInfos.data(), timeout, maxevents, &events_num);
2392 0 : CHK_PRT_RET(ret != 0, HCCL_ERROR("[%s] failed, call RaWaitEventHandle error ret[%d].", __func__, ret), HCCL_E_NETWORK);
2393 0 : for (u32 i = 0; i < events_num; i++) {
2394 0 : event_infos[i].fdHandle = raEventInfos[i].fdHandle;
2395 : }
2396 0 : return HCCL_SUCCESS;
2397 0 : }
2398 :
2399 1 : void HrtRaGetSecRandom(u32 *value, u32 &devPhyId)
2400 : {
2401 1 : CHECK_NULLPTR(value, "[HrtRaGetSecRandom] value is nullptr!");
2402 3 : HCCL_INFO("[HrtRaGetSecRandom] Input params: value=%u, devPhyId=%u", *value, devPhyId);
2403 1 : struct RaInfo raInfo = {};
2404 1 : raInfo.mode = HrtNetworkMode::HDC;
2405 1 : raInfo.phyId = devPhyId;
2406 :
2407 1 : s32 ret = RaGetSecRandom(&raInfo, value);
2408 1 : if (ret != 0) {
2409 0 : MACRO_THROW(NetworkApiException, StringFormat("[%s] failed, call interface error[%d]. params: value=%u, devPhyId=%u", __func__, ret, *value, devPhyId));
2410 : }
2411 3 : HCCL_INFO("[HrtRaGetSecRandom] Input params: value=%u, devPhyId=%u", *value, devPhyId);
2412 1 : }
2413 0 : HcclResult HrtRaCreateQpWithCq(RdmaHandle rdmaHandle, s32 sqEvent, s32 rqEvent,
2414 : void *sendChannel, void *recvChannel, QpInfo &info, bool isHdcMode)
2415 : {
2416 0 : CHK_PTR_NULL(rdmaHandle);
2417 0 : CHK_PTR_NULL(sendChannel);
2418 0 : CHK_PTR_NULL(recvChannel);
2419 0 : HCCL_INFO("[HrtRaCreateQpWithCq] Input params: rdmaHandle=%p, sqEvent=%d, rqEvent=%d, sendChannel=%p, recvChannel=%p", rdmaHandle, sqEvent, rqEvent, sendChannel, recvChannel);
2420 0 : struct ibv_comp_channel *sChannel = reinterpret_cast<struct ibv_comp_channel *>(sendChannel);
2421 0 : struct ibv_comp_channel *rChannel = reinterpret_cast<struct ibv_comp_channel *>(recvChannel);
2422 :
2423 0 : QpConfig config(MAX_WR_NUM, MAX_SEND_SGE_NUM, MAX_RECV_SGE_NUM, sqEvent, rqEvent);
2424 : CqInfo cq(nullptr, nullptr, nullptr, MAX_CQ_DEPTH, config.sqEvent, config.rqEvent, info.srqContext,
2425 0 : sChannel, rChannel);
2426 : // hdc模式下hccp没有对外提供创建CQ的接口
2427 0 : if (!isHdcMode) {
2428 0 : CHK_RET(HrtRaCreateCq(rdmaHandle, cq));
2429 : }
2430 0 : info.attr = config;
2431 0 : info.rdmaHandle = rdmaHandle;
2432 0 : info.context = cq.context;
2433 0 : info.sendCq = cq.sq;
2434 0 : info.recvCq = cq.rq;
2435 0 : info.recvChannel = rChannel;
2436 0 : info.sendChannel = sChannel;
2437 :
2438 0 : if (isHdcMode) {
2439 0 : TRY_CATCH_RETURN(info.qpHandle = HrtRaQpCreate(rdmaHandle, info.flag, info.qpMode));
2440 : } else {
2441 0 : CHK_RET(HrtRaNormalQpCreate(rdmaHandle, info));
2442 : }
2443 :
2444 0 : return HCCL_SUCCESS;
2445 0 : }
2446 :
2447 0 : HcclResult HrtRaDestroyQpWithCq(const QpInfo& info, bool isHdcMode)
2448 : {
2449 0 : if (info.qpHandle == nullptr) {
2450 0 : return HCCL_SUCCESS;
2451 : }
2452 :
2453 0 : if (isHdcMode) {
2454 0 : TRY_CATCH_RETURN(HrtRaQpDestroy(info.qpHandle));
2455 : } else {
2456 0 : CHK_RET(HrtRaNormalQpDestroy(info.qpHandle));
2457 0 : CqInfo cq;
2458 0 : cq.context = info.context;
2459 0 : cq.rq = info.recvCq;
2460 0 : cq.sq = info.sendCq;
2461 0 : CHK_RET(HrtRaDestroyCq(info.rdmaHandle, cq));
2462 0 : }
2463 :
2464 0 : return HCCL_SUCCESS;
2465 : }
2466 :
2467 : // ra_cq_create
2468 0 : HcclResult HrtRaCreateCq(RdmaHandle rdmaHandle, CqInfo& cq)
2469 : {
2470 0 : CHK_PTR_NULL(rdmaHandle);
2471 0 : HCCL_INFO("[HrtRaCreateCq] Input params: rdmaHandle=%p, sq=%p, rq=%p, context=%p", rdmaHandle, cq.sq, cq.rq, cq.context);
2472 :
2473 0 : struct CqAttr attr{};
2474 0 : attr.qpContext = &(cq.context);
2475 0 : attr.ibSendCq = &(cq.sq);
2476 0 : attr.ibRecvCq = &(cq.rq);
2477 0 : attr.sendCqDepth = cq.depth;
2478 0 : attr.recvCqDepth = cq.depth;
2479 0 : attr.sendCqEventId = cq.sqEvent;
2480 0 : attr.recvCqEventId = cq.rqEvent;
2481 0 : attr.sendChannel = cq.sendChannel;
2482 0 : attr.recvChannel = cq.recvChannel;
2483 0 : attr.srqContext = cq.srqContext;
2484 :
2485 0 : HCCL_DEBUG("ra create cq: send_cq_depth[%d], recv_cq_depth[%d], send_cq_event_id[%d], recv_cq_event_id[%d]",
2486 : attr.sendCqDepth, attr.recvCqDepth, attr.sendCqEventId, attr.recvCqEventId);
2487 0 : s32 ret = RaCqCreate(rdmaHandle, &attr);
2488 0 : CHK_PRT_RET(ret != 0,
2489 : HCCL_ERROR("[HrtRaCreateCq] errNo[0x%016llx] RaCqCreate fail. "
2490 : "return[%d], params: rdmaHandle[%p], sq[%p], rq[%p], context[%p]",
2491 : HCCL_ERROR_CODE(HCCL_E_NETWORK), ret, rdmaHandle, cq.sq, cq.rq, cq.context),
2492 : HCCL_E_NETWORK);
2493 0 : if (cq.sq == nullptr || cq.rq == nullptr || cq.context == nullptr) {
2494 0 : HCCL_ERROR("[HrtRaCreateCq] cq member[sq:%p, rq:%p, context:%p] is nullptr, ret[%d]", cq.sq, cq.rq, cq.context, ret);
2495 0 : return HCCL_E_PARA;
2496 : }
2497 0 : return HCCL_SUCCESS;
2498 : }
2499 : // ra_cq_destroy
2500 0 : HcclResult HrtRaDestroyCq(RdmaHandle rdmaHandle, CqInfo& cq)
2501 : {
2502 0 : CHK_PTR_NULL(rdmaHandle);
2503 0 : HCCL_INFO("[HrtRaDestroyCq] Input params: rdmaHandle=%p, sq=%p, rq=%p, context=%p", rdmaHandle, cq.sq, cq.rq, cq.context);
2504 0 : struct CqAttr attr = {};
2505 0 : attr.qpContext = &cq.context;
2506 0 : attr.ibSendCq = &cq.sq;
2507 0 : attr.ibRecvCq = &cq.rq;
2508 0 : s32 ret = RaCqDestroy(rdmaHandle, &attr);
2509 0 : CHK_PRT_RET(ret != 0,
2510 : HCCL_ERROR("[HrtRaDestroyCq] errNo[0x%016llx] RaCqDestroy failed, call interface error. "
2511 : "return[%d], params: rdmaHandle[%p], sq[%p], rq[%p], context[%p]",
2512 : HCCL_ERROR_CODE(HCCL_E_NETWORK), ret, rdmaHandle, cq.sq, cq.rq, cq.context),
2513 : HCCL_E_NETWORK);
2514 0 : return HCCL_SUCCESS;
2515 : }
2516 :
2517 : // ra_normal_qp_create
2518 0 : HcclResult HrtRaNormalQpCreate(RdmaHandle rdmaHandle, QpInfo& qp)
2519 : {
2520 0 : CHK_PTR_NULL(rdmaHandle);
2521 0 : HCCL_INFO("[HrtRaNormalQpCreate] Input params: rdmaHandle=%p, context=%p", rdmaHandle, qp.context);
2522 0 : struct ibv_qp_init_attr ibQpAttr = {};
2523 0 : CHK_SAFETY_FUNC_RET(memset_s(&ibQpAttr, sizeof(ibv_qp_init_attr), 0, sizeof(ibv_qp_init_attr)));
2524 0 : ibQpAttr.qp_context= qp.context;
2525 0 : ibQpAttr.send_cq = qp.sendCq;
2526 0 : ibQpAttr.recv_cq = qp.recvCq;
2527 0 : ibQpAttr.srq = qp.srq;
2528 0 : ibQpAttr.qp_type = IBV_QPT_RC;
2529 0 : ibQpAttr.cap.max_inline_data = MAX_INLINE_DATA;
2530 0 : ibQpAttr.cap.max_send_wr = qp.attr.maxWr;
2531 0 : ibQpAttr.cap.max_send_sge = qp.attr.maxSendSge;
2532 0 : ibQpAttr.cap.max_recv_wr = (qp.srq == nullptr ? qp.attr.maxWr : 0);
2533 0 : ibQpAttr.cap.max_recv_sge = (qp.srq == nullptr ? qp.attr.maxRecvSge : 0);
2534 0 : s32 ret = RaNormalQpCreate(rdmaHandle, &ibQpAttr, &(qp.qpHandle), reinterpret_cast<void **>(&(qp.qp)));
2535 0 : RPT_INPUT_ERR(ret == ROCE_ENOMEM_RET, "EI0011", std::vector<std::string>({"memory_size"}), // A3是当ROCE_ENOMEM_RET才上报EI0011,内存大小取决于qp深度配置
2536 : std::vector<std::string>({"262144~3145728"}));
2537 0 : CHK_PRT_RET(ret != 0, HCCL_ERROR("[Create][NormalQp]errNo[0x%016llx] RaNormalQpCreate fail. return[%d], params: rdmaHandle[%p], context[%p]",\
2538 : HCCL_ERROR_CODE(HCCL_E_NETWORK), ret, rdmaHandle, qp.context), HCCL_E_NETWORK);
2539 0 : return HCCL_SUCCESS;
2540 : }
2541 :
2542 0 : HcclResult HrtRaNormalQpDestroy(QpHandle qpHandle)
2543 : {
2544 0 : CHK_PTR_NULL(qpHandle);
2545 0 : HCCL_INFO("[HrtRaNormalQpDestroy] Input params: qpHandle=%p", qpHandle);
2546 0 : s32 ret = RaNormalQpDestroy(qpHandle);
2547 0 : CHK_PRT_RET(ret != 0, HCCL_ERROR("[Destroy][NormalQp]errNo[0x%016llx] ra destroy normal qp fail. return[%d], params: rdmaHandle[%p]",\
2548 : HCCL_ERROR_CODE(HCCL_E_NETWORK), ret, qpHandle), HCCL_E_NETWORK);
2549 0 : return HCCL_SUCCESS;
2550 : }
2551 :
2552 0 : HcclResult HrtRaNdaQpCreate(RdmaHandle rdmaHandle, NdaOps *ndaOps, uint32_t dmaMode, NdaCqInfo *cqInfo, NdaQpInfo *qpInfo, QpHandle *qpHandle)
2553 : {
2554 0 : CHK_PTR_NULL(rdmaHandle);
2555 0 : CHK_PTR_NULL(ndaOps);
2556 0 : HCCL_INFO("[HrtRaNdaQpCreate] Input params: rdmaHandle=%p dmaMode=%u", rdmaHandle, dmaMode);
2557 :
2558 : struct ibv_qp_init_attr ibQpAttr;
2559 0 : CHK_SAFETY_FUNC_RET(memset_s(&ibQpAttr, sizeof(ibv_qp_init_attr), 0, sizeof(ibv_qp_init_attr)));
2560 0 : ibQpAttr.qp_context= nullptr;
2561 0 : ibQpAttr.send_cq = cqInfo->cq;
2562 0 : ibQpAttr.recv_cq = cqInfo->cq;
2563 0 : ibQpAttr.srq = nullptr;
2564 0 : ibQpAttr.qp_type = IBV_QPT_RC;
2565 0 : ibQpAttr.cap.max_inline_data = MAX_INLINE_DATA;
2566 0 : ibQpAttr.cap.max_send_wr = MAX_WR_NUM;
2567 0 : ibQpAttr.cap.max_send_sge = MAX_SEND_SGE_NUM;
2568 0 : ibQpAttr.cap.max_recv_wr = MAX_WR_NUM;
2569 0 : ibQpAttr.cap.max_recv_sge = MAX_RECV_SGE_NUM;
2570 :
2571 : struct NdaQpInitAttr qpAttr;
2572 0 : qpAttr.attr = ibQpAttr;
2573 0 : qpAttr.qpCapFlag = 0;
2574 0 : qpAttr.dmaMode = dmaMode;
2575 0 : qpAttr.ops = ndaOps;
2576 :
2577 0 : s32 ret = RaNdaQpCreate(rdmaHandle, &qpAttr, qpInfo, qpHandle);
2578 0 : CHK_PRT_RET(ret != 0 || qpInfo == nullptr || qpHandle == nullptr,
2579 : HCCL_ERROR("[Create][NdaQp]errNo[0x%016llx] RaNdaQpCreate fail. return[%d], "
2580 : "params: rdmaHandle[%p] dmaMode[%u]",
2581 : HCCL_ERROR_CODE(HCCL_E_NETWORK), ret, rdmaHandle, dmaMode), HCCL_E_NETWORK);
2582 0 : return HCCL_SUCCESS;
2583 : }
2584 :
2585 2 : HcclResult HrtRaNdaCqCreate(RdmaHandle rdmaHandle, NdaOps *ndaOps, uint32_t dmaMode, NdaCqInfo *cqInfo, CqHandle *cqHandle)
2586 : {
2587 2 : CHK_PTR_NULL(rdmaHandle);
2588 2 : CHK_PTR_NULL(ndaOps);
2589 6 : HCCL_INFO("[HrtRaNdaCqCreate] Input params: rdmaHandle=%p dmaMode=%u", rdmaHandle, dmaMode);
2590 :
2591 : struct ibv_cq_init_attr_ex ibCqAttr;
2592 2 : CHK_SAFETY_FUNC_RET(memset_s(&ibCqAttr, sizeof(ibv_cq_init_attr_ex), 0, sizeof(ibv_cq_init_attr_ex)));
2593 2 : if (dmaMode == QBUF_DMA_MODE_INDEP_UB) {
2594 1 : ibCqAttr.cqe = NDA_CQ_DEPTH_FOR_UBNIC;
2595 : } else {
2596 1 : ibCqAttr.cqe = NDA_CQ_DEPTH_FOR_XSCDV;
2597 : }
2598 2 : ibCqAttr.cq_context = nullptr;
2599 2 : ibCqAttr.channel = nullptr;
2600 2 : ibCqAttr.comp_vector = 0;
2601 2 : ibCqAttr.wc_flags = 0;
2602 2 : ibCqAttr.comp_mask = 0;
2603 2 : ibCqAttr.flags = 0;
2604 :
2605 : struct NdaCqInitAttr cqAttr;
2606 2 : cqAttr.attr = ibCqAttr;
2607 2 : cqAttr.cqCapFlag = 0;
2608 2 : cqAttr.dmaMode = dmaMode;
2609 2 : cqAttr.ops = ndaOps;
2610 :
2611 2 : s32 ret = RaNdaCqCreate(rdmaHandle, &cqAttr, cqInfo, cqHandle);
2612 2 : CHK_PRT_RET(ret != 0 || cqInfo == nullptr || cqHandle == nullptr,
2613 : HCCL_ERROR("[Create][NdaCq]errNo[0x%016llx] RaNdaCqCreate fail. return[%d], "
2614 : "params: rdmaHandle[%p] dmaMode[%u]",
2615 : HCCL_ERROR_CODE(HCCL_E_NETWORK), ret, rdmaHandle, dmaMode), HCCL_E_NETWORK);
2616 2 : return HCCL_SUCCESS;
2617 : }
2618 :
2619 0 : HcclResult HrtRaNdaCqDestroy(RdmaHandle rdmaHandle, CqHandle cqHandle)
2620 : {
2621 0 : CHK_PTR_NULL(rdmaHandle);
2622 0 : CHK_PTR_NULL(cqHandle);
2623 0 : HCCL_INFO("[HrtRaNdaCqDestroy] Input params: rdmaHandle=%p cqHandle=%p", rdmaHandle, cqHandle);
2624 :
2625 0 : s32 ret = RaNdaCqDestroy(rdmaHandle, cqHandle);
2626 0 : CHK_PRT_RET(ret != 0,
2627 : HCCL_ERROR("[RaNdaCqDestroy] errNo[0x%016llx] RaNdaCqDestroy failed, call interface error. "
2628 : "return[%d], params: rdmaHandle[%p] cqHandle[%p]",
2629 : HCCL_ERROR_CODE(HCCL_E_NETWORK), ret, rdmaHandle, cqHandle),
2630 : HCCL_E_NETWORK);
2631 0 : return HCCL_SUCCESS;
2632 : }
2633 :
2634 0 : HcclResult RaBatchQueryJettyStatus(const std::vector<JettyHandle> &jettyHandles, std::vector<JettyStatus> &jettyAttrs, u32 &num)
2635 : {
2636 0 : if (jettyHandles.size() != num) {
2637 0 : HCCL_ERROR("jettyHandles size[%zu] not equal to num[%u]", jettyHandles.size(), num);
2638 0 : return HCCL_E_PARA;
2639 : }
2640 0 : std::vector<struct JettyAttr> raJettyAttrs(MAX_JETTY_QUERY_NUM);
2641 0 : void* qp_handle[jettyHandles.size()];
2642 0 : for (size_t i = 0; i < jettyHandles.size(); ++i) {
2643 0 : qp_handle[i] = reinterpret_cast<void*>(jettyHandles[i]);
2644 : }
2645 0 : auto ret = RaCtxQpQueryBatch(qp_handle, raJettyAttrs.data(), &num);
2646 0 : if (ret != 0) {
2647 0 : HCCL_ERROR("RaBatchQueryJettyAttr failed.");
2648 0 : return HCCL_E_NETWORK;
2649 : }
2650 0 : if (num != jettyHandles.size()) {
2651 0 : HCCL_ERROR("jettyAttrs num[%zu] not equal to input jettyHandles size[%zu]", num, jettyHandles.size());
2652 0 : return HCCL_E_PARA;
2653 : }
2654 :
2655 0 : for (u32 i = 0; i < num; i++) {
2656 0 : JettyStatus jettyStatus = static_cast<JettyStatus::Value>(static_cast<int>(raJettyAttrs[i].state));
2657 0 : jettyAttrs.push_back(jettyStatus);
2658 : }
2659 0 : return HCCL_SUCCESS;
2660 0 : }
2661 :
2662 0 : HcclResult RaGetAuxInfo(const RdmaHandle rdmaHandle, AuxInfoIn auxInfoIn, AuxInfoOut &auxInfoOut)
2663 : {
2664 : HccpAuxInfoIn in;
2665 0 : in.type = static_cast<HccpAuxInfoInType>(static_cast<int>(auxInfoIn.auxInfoInType));
2666 0 : if (auxInfoIn.auxInfoInType == AuxInfoInType::AUX_INFO_IN_TYPE_CQE) {
2667 0 : in.cqe.status = auxInfoIn.cqe.status;
2668 0 : in.cqe.sR = auxInfoIn.cqe.sR;
2669 0 : } else if (auxInfoIn.auxInfoInType == AuxInfoInType::AUX_INFO_IN_TYPE_AE) {
2670 0 : in.ae.eventType = auxInfoIn.ae.eventType;
2671 : }
2672 :
2673 : HccpAuxInfoOut out;
2674 0 : auto ret = RaCtxGetAuxInfo(rdmaHandle, &in, &out);
2675 0 : if (ret != 0) {
2676 0 : HCCL_ERROR("RaGetAuxInfo failed.");
2677 0 : return HCCL_E_NETWORK;
2678 : }
2679 :
2680 0 : auxInfoOut.auxInfoNum = out.auxInfoNum;
2681 0 : for (uint32_t i = 0; i < out.auxInfoNum; i++) {
2682 0 : auxInfoOut.auxInfoTypes[i] = out.auxInfoType[i];
2683 0 : auxInfoOut.auxInfoValues[i] = out.auxInfoValue[i];
2684 : }
2685 0 : return HCCL_SUCCESS;
2686 : }
2687 :
2688 7 : HcclResult HrtRaCtxQpDestoryBatch(const RdmaHandle handle, const std::unordered_set<JettyHandle> &jettyHandles, std::vector<JettyHandle> &failJettyHandles)
2689 : {
2690 7 : std::vector<void*> qp_handle;
2691 7 : failJettyHandles.clear();
2692 21 : for (auto jettyHandle : jettyHandles) {
2693 14 : qp_handle.push_back(reinterpret_cast<void*>(jettyHandle));
2694 : }
2695 7 : unsigned int delNum = min(qp_handle.size(), static_cast<size_t>(MAX_DELETE_JETTY_NUMS));
2696 7 : std::vector<void*> del_qp_handle;
2697 : while (true) {
2698 8 : void *raReqHandle = nullptr;
2699 8 : delNum = min(qp_handle.size(), static_cast<size_t>(MAX_DELETE_JETTY_NUMS));
2700 8 : del_qp_handle.assign(qp_handle.begin(), qp_handle.begin() + delNum);
2701 8 : auto ret = RaCtxQpDestroyBatchAsync(handle, del_qp_handle.data(), &delNum, &raReqHandle);
2702 8 : if (ret != 0) {
2703 3 : HCCL_ERROR("[%s] failed, ret is [%d].", __func__, ret);
2704 3 : return HCCL_E_INTERNAL;
2705 : }
2706 :
2707 7 : RequestHandle reqHandle = reinterpret_cast<RequestHandle>(raReqHandle);
2708 7 : auto startTime = std::chrono::steady_clock::now();
2709 7 : constexpr uint32_t pollTimeoutMs = 10000; // 轮询超时时间10s
2710 7 : auto waitPollTimeOutMs = std::chrono::milliseconds(pollTimeoutMs);
2711 : while (true) {
2712 5504310 : if ((std::chrono::steady_clock::now() - startTime) >= waitPollTimeOutMs) {
2713 3 : HCCL_ERROR("[%s]poll timeout, originalJettyCount[%zu], undeleteJettyCount[%zu].", __func__, jettyHandles.size(), failJettyHandles.size());
2714 1 : return HCCL_E_TIMEOUT;
2715 : }
2716 5504309 : ReqHandleResult result = ReqHandleResult::INVALID_PARA;
2717 5504309 : TRY_CATCH_RETURN(result = HrtRaGetAsyncReqResult(reqHandle));
2718 5504309 : if (result == ReqHandleResult::NOT_COMPLETED) {
2719 5504303 : continue;
2720 6 : } else if (result == ReqHandleResult::COMPLETED) {
2721 6 : break;
2722 : } else {
2723 0 : HCCL_ERROR("[%s] failed, result[%s] is unexpected.", __func__, result.Describe().c_str());
2724 0 : return HCCL_E_INTERNAL;
2725 : }
2726 5504303 : }
2727 :
2728 : // 检查是否删除完成
2729 6 : if (delNum > del_qp_handle.size()) {
2730 3 : HCCL_ERROR("[%s] run RaCtxQpDestroyBatchAsync error, del jetty num[%u] greater than all jetty num[%zu].", __func__, delNum, del_qp_handle.size());
2731 1 : return HCCL_E_INTERNAL;
2732 5 : } else if (del_qp_handle.size() == delNum) {
2733 3 : qp_handle.erase(qp_handle.begin(), qp_handle.begin() + delNum);
2734 : } else {
2735 2 : failJettyHandles.push_back(reinterpret_cast<JettyHandle>(del_qp_handle[delNum]));
2736 2 : qp_handle.erase(qp_handle.begin(), qp_handle.begin() + delNum + 1);
2737 : }
2738 5 : if (qp_handle.size() == 0) {
2739 4 : break;
2740 : }
2741 1 : }
2742 12 : HCCL_INFO("[%s] run success, originalJettyCount[%zu], undeleteJettyCount[%zu].", __func__, jettyHandles.size(), failJettyHandles.size());
2743 4 : return HCCL_SUCCESS;
2744 7 : }
2745 :
2746 : struct ccu_mem_info {
2747 : unsigned int long long mem_va;
2748 : unsigned int mem_size;
2749 : unsigned int resv[1];
2750 : };
2751 :
2752 : struct ccu_mem_rsp {
2753 : unsigned int die_id;
2754 : unsigned int num;
2755 : struct ccu_mem_info list[64U];
2756 : };
2757 :
2758 6 : void HrtSetMemInfoList(struct CcuMemInfo *memInfoList, uint32_t count, struct ccu_mem_info *recvMemList) {
2759 114 : for (size_t i = 0; i < count; ++i) {
2760 108 : memInfoList[i].memVa = recvMemList[i].mem_va;
2761 108 : memInfoList[i].memSize = recvMemList[i].mem_size;
2762 : }
2763 6 : }
2764 :
2765 6 : HcclResult HrtGetCcuMemInfo(void* tlv_handle, uint32_t udieIdx, uint64_t memTypeBitmap, struct CcuMemInfo *memInfoList, uint32_t count)
2766 : {
2767 6 : s32 ret = 0;
2768 6 : u32 tlv_module_type = TLV_MODULE_TYPE_CCU;
2769 :
2770 6 : struct TlvMsg send_msg = {};
2771 6 : struct TlvMsg recv_msg = {};
2772 : // 使用unique_ptr管理动态分配的内存，实现RAII
2773 6 : auto send_data = std::make_unique<char[]>(sizeof(CcuMemReq));
2774 6 : auto recv_data = std::make_unique<char[]>(sizeof(ccu_mem_rsp));
2775 :
2776 : // 初始化请求消息
2777 6 : send_msg.type = MSG_TYPE_CCU_GET_MEM_INFO;
2778 6 : send_msg.length = sizeof(CcuMemReq);
2779 6 : send_msg.data = send_data.get();
2780 :
2781 6 : auto req = reinterpret_cast<CcuMemReq*>(send_msg.data);
2782 6 : req->udieIdx = udieIdx;
2783 6 : req->memTypeBitmap = memTypeBitmap;
2784 :
2785 : // 初始化响应消息
2786 6 : recv_msg.type = 0;
2787 6 : recv_msg.length = sizeof(ccu_mem_rsp);
2788 6 : recv_msg.data = recv_data.get();
2789 :
2790 6 : auto rsp = reinterpret_cast<ccu_mem_rsp*>(recv_msg.data);
2791 6 : rsp->die_id = 0;
2792 6 : rsp->num = 0;
2793 18 : std::fill(std::begin(rsp->list), std::end(rsp->list), ccu_mem_info{});
2794 :
2795 6 : ret = RaTlvRequest(tlv_handle, tlv_module_type, &send_msg, &recv_msg);
2796 6 : if (ret != 0) {
2797 0 : if (ret == RA_TLV_REQUEST_UNAVAIL) {
2798 0 : HCCL_WARNING("[HrtGetCcuMemInfo]ra tlv request UNAVAIL. return: ret[%d]", ret);
2799 0 : return HCCL_E_UNAVAIL;
2800 : }
2801 0 : HCCL_ERROR("[Request][RaTlv]errNo[0x%016llx] ra tlv request fail. return: ret[%d], module type[%u], message type[%u]",
2802 : HCCL_ERROR_CODE(HcclResult::HCCL_E_NETWORK), ret, tlv_module_type, send_msg.type);
2803 0 : throw NetworkApiException(StringFormat("call ra_tlv_request failed"));
2804 : }
2805 6 : HrtSetMemInfoList(memInfoList, count, rsp->list);
2806 18 : HCCL_INFO("tlv request success, tlv module type[%u], message type[%u]", tlv_module_type, send_msg.type);
2807 6 : return HCCL_SUCCESS;
2808 6 : }
2809 :
2810 4 : HcclResult HrtRaGetEidByIp(RdmaHandle handle, const vector<IpAddress>& ipV4AddrList, vector<IpAddress>& eidAddrList)
2811 : {
2812 12 : HCCL_INFO("[HrtRaGetEidByIp] begain, ipV4AddrList size=%zu", ipV4AddrList.size());
2813 4 : size_t ipV4AddrListSize = ipV4AddrList.size();
2814 4 : unsigned int num = ipV4AddrListSize;
2815 7 : IpInfo ipInfoList[num] = {};
2816 7 : for (size_t i = 0; i < num; i++) {
2817 3 : auto ipAddress = ipV4AddrList.at(i);
2818 9 : HCCL_INFO("[HrtRaGetEidByIp] ipV4AddrList[%d][%s]", i, ipAddress.Describe().c_str());
2819 3 : ipInfoList[i].family = ipAddress.GetFamily();
2820 3 : ipInfoList[i].ip = IpAddressToHccpIpAddr(ipAddress);
2821 : }
2822 :
2823 7 : union HccpEid eidList[num] = {};
2824 4 : s32 ret = RaGetEidByIp(handle, ipInfoList, eidList, &num);
2825 4 : if (ret != 0) {
2826 3 : HCCL_WARNING("call RaGetEidByIp failed, error code =%d.", ret);
2827 1 : return HCCL_E_INTERNAL;
2828 : }
2829 :
2830 3 : if (num != ipV4AddrList.size()) {
2831 3 : HCCL_ERROR("call RaGetEidByIp failed, The number of ipInfoList and eidList is inconsistent, "
2832 : "ipV4AddrList size =%zu, eidList size =%u",
2833 : ipV4AddrList.size(),
2834 : num);
2835 1 : return HCCL_E_INTERNAL;
2836 : }
2837 :
2838 3 : for (unsigned int i = 0; i < num; i++) {
2839 1 : IpAddress eidAddr = HccpEidToIpAddress(eidList[i]);
2840 1 : eidAddrList.push_back(eidAddr);
2841 : }
2842 6 : HCCL_INFO("[HrtRaGetEidByIp] success, eidAddrList size=%zu", eidAddrList.size());
2843 2 : return HCCL_SUCCESS;
2844 4 : }
2845 :
2846 3 : HcclResult WaitRequestResult(void* raReqHandle, RequestHandle& reqHandle)
2847 : {
2848 3 : reqHandle = reinterpret_cast<RequestHandle>(raReqHandle);
2849 3 : auto startTime = std::chrono::steady_clock::now();
2850 3 : constexpr uint32_t pollTimeoutMs = 10000; // 轮询超时时间
2851 3 : auto waitPollTimeOutMs = std::chrono::milliseconds(pollTimeoutMs);
2852 : while (true) {
2853 3 : if ((std::chrono::steady_clock::now() - startTime) >= waitPollTimeOutMs) {
2854 0 : HCCL_ERROR("[WaitRequestResult] poll timeout.");
2855 1 : return HCCL_E_TIMEOUT; // 超时报错
2856 : }
2857 :
2858 3 : ReqHandleResult result = ReqHandleResult::INVALID_PARA;
2859 3 : TRY_CATCH_RETURN(result = HrtRaGetAsyncReqResult(reqHandle));
2860 :
2861 : // 结果判断
2862 3 : if (result == ReqHandleResult::NOT_COMPLETED) {
2863 0 : continue;
2864 3 : } else if (result == ReqHandleResult::COMPLETED) {
2865 2 : break;
2866 : } else {
2867 3 : HCCL_ERROR("[WaitRequestResult] failed, result[%s] is unexpected.", result.Describe().c_str());
2868 1 : return HCCL_E_INTERNAL;
2869 : }
2870 0 : }
2871 :
2872 2 : return HCCL_SUCCESS;
2873 : }
2874 :
2875 3 : HcclResult HrtRaSetTpAttrAsync(RdmaHandle handle, uint64_t tpHandle, uint32_t attrBitmap, TpAttr& attr, RequestHandle& reqHandle)
2876 : {
2877 9 : HCCL_INFO("[HrtRaSetTpAttrAsync] begain, reqHandle[%llu]", reqHandle);
2878 3 : void *raReqHandle = nullptr;
2879 3 : s32 ret = RaSetTpAttrAsync(handle, tpHandle, attrBitmap, &attr, &raReqHandle);
2880 3 : if (ret != 0) {
2881 1 : string msg = StringFormat("call RaSetTpAttrAsync failed, error code =%d.", ret);
2882 1 : THROW<NetworkApiException>(msg);
2883 1 : }
2884 :
2885 5 : CHK_RET(WaitRequestResult(raReqHandle, reqHandle));
2886 3 : HCCL_INFO("[HrtRaSetTpAttrAsync] success, reqHandle[%llu]", reqHandle);
2887 1 : return HCCL_SUCCESS;
2888 : }
2889 :
2890 29 : HcclResult HrtRaGetTpAttrAsync(u32 phyId, RdmaHandle handle, uint64_t tpHandle, uint32_t& attrBitmap, TpAttr& attr, RequestHandle& reqHandle)
2891 : {
2892 87 : HCCL_INFO("[HrtRaGetTpAttrAsync] begain, reqHandle[%llu]", reqHandle);
2893 29 : u32 tpAttrVersion = 0;
2894 29 : s32 ret = RaGetInterfaceVersion(phyId, GET_TP_ATTR_OPCODE, &tpAttrVersion);
2895 29 : if (ret != 0 || tpAttrVersion < GET_TP_ATTR_VERSION) {
2896 81 : HCCL_ERROR("this package does not support RaGetTpAttrAsync for device, please change new package, ret=%d, tpAttrVersion=%u.", ret, tpAttrVersion);
2897 27 : return HCCL_E_NOT_SUPPORT;
2898 : }
2899 2 : void *raReqHandle = nullptr;
2900 2 : ret = RaGetTpAttrAsync(handle, tpHandle, &attrBitmap, &attr, &raReqHandle);
2901 2 : if (ret != 0) {
2902 1 : string msg = StringFormat("call RaGetTpAttrAsync failed, error code =%d.", ret);
2903 1 : THROW<NetworkApiException>(msg);
2904 1 : }
2905 :
2906 1 : CHK_RET(WaitRequestResult(raReqHandle, reqHandle));
2907 3 : HCCL_INFO("[HrtRaGetTpAttrAsync] success, reqHandle[%llu]", reqHandle);
2908 1 : return HCCL_SUCCESS;
2909 : }
2910 :
2911 3 : HcclResult HrtGetUboeFlagEnable(const u32 devPhyId)
2912 : {
2913 3 : u32 uboeVersion = 0;
2914 3 : s32 versionRet = RaGetInterfaceVersion(devPhyId, GET_UBOE_FLAG_ENABLE_OPCODE, &uboeVersion);
2915 6 : CHK_PRT_RET(versionRet != 0,
2916 : HCCL_ERROR("[%s] RaGetInterfaceVersion failed, devPhyId=%u, versionRet=%d", __func__, devPhyId, versionRet),
2917 : HCCL_E_INTERNAL);
2918 5 : CHK_PRT_RET(uboeVersion < GET_UBOE_FLAG_ENABLE_VERSION,
2919 : HCCL_ERROR("[%s] this package does not support to get uboe flag, "
2920 : "please change new package. uboeVersion[%u].", __func__, uboeVersion), HCCL_E_NOT_SUPPORT);
2921 1 : return HCCL_SUCCESS;
2922 : }
2923 :
2924 : } // namespace Hccl
|
