Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include <hccl/hccl_types.h>
12 : #include "dlhal_function.h"
13 : #include "dlra_function.h"
14 : #include "sal_pub.h"
15 : #include "adapter_hccp.h"
16 : #include "network_manager_pub.h"
17 : #include "externalinput_pub.h"
18 : #include "hccl_network.h"
19 :
20 : namespace hccl {
21 : HcclResult
22 256 : NetDevContext::Init(NicType nicType, s32 devicePhyId, s32 deviceLogicId, HcclIpAddress localIp, HcclIpAddress backupIp)
23 : {
24 256 : devicePhyId_ = devicePhyId;
25 256 : deviceLogicId_ = deviceLogicId;
26 256 : localIp_ = localIp;
27 256 : backupIp_ = backupIp;
28 256 : nicType_ = nicType;
29 :
30 256 : if (nicType == NicType::VNIC_TYPE || nicType == NicType::DEVICE_NIC_TYPE) {
31 231 : nicDeployment_ = NICDeployment::NIC_DEPLOYMENT_DEVICE;
32 : } else {
33 25 : nicDeployment_ = NICDeployment::NIC_DEPLOYMENT_HOST;
34 : }
35 :
36 256 : if (static_cast<s32>(devicePhyId) == HOST_DEVICE_ID) {
37 0 : deviceLogicId_ = 0;
38 : }
39 :
40 256 : if (nicDeployment_ == NICDeployment::NIC_DEPLOYMENT_HOST) {
41 25 : CHK_RET(NetworkManager::GetInstance(deviceLogicId_).StartHostNet(localIp, hostSocketHandle_));
42 : }
43 :
44 256 : return HCCL_SUCCESS;
45 : }
46 :
47 0 : HcclResult NetDevContext::GetinfoConfig(const HcclNetDevInfos* info)
48 : {
49 0 : CHK_PTR_NULL(info);
50 0 : devicePhyId_ = info->devicePhyId;
51 0 : isBackup_ = info->isBackup;
52 0 : u32 deviceLogicId = 0;
53 0 : CHK_RET(hrtGetDeviceIndexByPhyId(devicePhyId_, deviceLogicId));
54 0 : deviceLogicId_ = deviceLogicId;
55 0 : CHK_RET(ConvertIP(info->addr));
56 0 : netDevDeployment_ = info->netdevDeployment;
57 0 : protoType_ = info->addr.protoType;
58 :
59 0 : HCCL_INFO("[NetDevContext][InitV2] netDevDeployment_ = [%u], protoType_ = [%u]", netDevDeployment_, protoType_);
60 0 : return HCCL_SUCCESS;
61 : }
62 :
63 0 : HcclResult NetDevContext::ConvertIP(const HcclAddress address)
64 : {
65 0 : s32 family = AF_INET;
66 : HcclInAddr temp;
67 0 : if (address.type == HCCL_ADDR_TYPE_IP_V4) {
68 0 : family = AF_INET;
69 0 : temp.addr = address.addr;
70 0 : } else if (address.type == HCCL_ADDR_TYPE_IP_V6) {
71 0 : family = AF_INET6;
72 0 : temp.addr6 = address.addr6;
73 : } else {
74 0 : HCCL_ERROR(
75 : "[NetDevContext][InitV2]this addrType [%u] is not supported, please check the configuration.",
76 : address.type);
77 0 : return HCCL_E_PARA;
78 : }
79 0 : HcclIpAddress localIptemp(family, temp);
80 0 : localIp_ = localIptemp;
81 0 : return HCCL_SUCCESS;
82 0 : }
83 :
84 : // 初始化进程和设备
85 0 : HcclResult NetDevContext::InitV2(const HcclNetDevInfos* info)
86 : {
87 0 : CHK_PTR_NULL(info);
88 0 : CHK_RET(GetinfoConfig(info));
89 : // 需要保存新的协议类型
90 0 : if (netDevDeployment_ == HcclNetDevDeployment::HCCL_NETDEV_DEPLOYMENT_DEVICE) {
91 0 : switch (protoType_) {
92 0 : case HCCL_PROTO_TYPE_ROCE: {
93 0 : nicType_ = NicType::DEVICE_NIC_TYPE;
94 0 : nicDeployment_ = NICDeployment::NIC_DEPLOYMENT_DEVICE;
95 0 : bool rdmaFlag = !GetExternalInputHcclIsTcpMode();
96 0 : if (!rdmaFlag) {
97 0 : HCCL_ERROR(
98 : "[NetDevContext][InitV2]rdmaFlag and protoType are not equal, please check the configuration.");
99 0 : return HCCL_E_PARA;
100 : }
101 : NetworkMode netMode;
102 0 : NetworkManager::GetInstance(deviceLogicId_).GetNetworkMode(netMode);
103 : NotifyTypeT notifyType;
104 0 : NetworkManager::GetInstance(deviceLogicId_).GetNotifyType(notifyType);
105 0 : CHK_RET(NetworkManager::GetInstance(deviceLogicId_)
106 : .CreateRdmaHandle(localIp_, isBackup_, netMode, notifyType, netDevDeployment_));
107 0 : CHK_RET(NetworkManager::GetInstance(deviceLogicId_).CreateNicSocketHandle(localIp_));
108 0 : NetworkManager::GetInstance(deviceLogicId_).GetRdmaHandleByIpAddr(localIp_, handle_);
109 :
110 0 : CHK_PTR_NULL(handle_);
111 0 : HCCL_INFO("[NetDevContext][InitV2]Deployment is device and proto is roce");
112 0 : break;
113 : }
114 0 : case HCCL_PROTO_TYPE_BUS: {
115 0 : nicType_ = NicType::VNIC_TYPE;
116 0 : nicDeployment_ = NICDeployment::NIC_DEPLOYMENT_DEVICE;
117 0 : CHK_RET(NetworkManager::GetInstance(deviceLogicId_).CreateVnicSocketHandle(localIp_));
118 0 : RaResourceInfo raResourceInfo;
119 0 : NetworkManager::GetInstance(deviceLogicId_).GetRaResourceInfo(raResourceInfo);
120 0 : IpSocket& sock = raResourceInfo.vnicSocketMap[localIp_];
121 0 : handle_ = sock.nicSocketHandle;
122 0 : CHK_PTR_NULL(handle_);
123 0 : HCCL_INFO("[NetDevContext][InitV2]Deployment is device and proto is bus");
124 0 : break;
125 0 : }
126 :
127 0 : case HCCL_PROTO_TYPE_TCP: {
128 0 : nicType_ = NicType::DEVICE_NIC_TYPE;
129 0 : nicDeployment_ = NICDeployment::NIC_DEPLOYMENT_DEVICE;
130 0 : bool rdmaFlag = !GetExternalInputHcclIsTcpMode();
131 0 : if (rdmaFlag) {
132 0 : HCCL_ERROR("[NetDevContext][InitV2]rdmaFlag is ERROR, please check the configuration.");
133 0 : return HCCL_E_PARA;
134 : }
135 0 : CHK_RET(NetworkManager::GetInstance(deviceLogicId_).CreateNicSocketHandle(localIp_));
136 0 : NetworkManager::GetInstance(deviceLogicId_).GetNicHandleByIpAddr(localIp_, handle_);
137 0 : CHK_PTR_NULL(handle_);
138 0 : HCCL_INFO("[NetDevContext][InitV2]Deployment is device and proto is tcp");
139 0 : break;
140 : }
141 :
142 0 : default: // 保留
143 0 : HCCL_ERROR(
144 : "[NetDevContext][DeinitV2]this prototype [%u] is not supported in device mode, please check the "
145 : "configuration.",
146 : protoType_);
147 0 : return HCCL_E_NOT_SUPPORT;
148 : }
149 0 : } else if (netDevDeployment_ == HcclNetDevDeployment::HCCL_NETDEV_DEPLOYMENT_HOST) {
150 0 : switch (protoType_) {
151 0 : case HCCL_PROTO_TYPE_TCP: {
152 0 : nicType_ = NicType::HOST_NIC_TYPE;
153 0 : nicDeployment_ = NICDeployment::NIC_DEPLOYMENT_HOST;
154 0 : CHK_RET(NetworkManager::GetInstance(deviceLogicId_).CreateHostSocketHandle(localIp_, handle_));
155 0 : RaResourceInfo raResourceInfo;
156 0 : NetworkManager::GetInstance(deviceLogicId_).GetRaResourceInfo(raResourceInfo);
157 0 : IpSocket& sock = raResourceInfo.hostNetSocketMap[localIp_];
158 0 : handle_ = sock.nicSocketHandle;
159 0 : CHK_PTR_NULL(handle_);
160 0 : HCCL_INFO("[NetDevContext][InitV2]Deployment is host and proto is tcp");
161 0 : break;
162 0 : }
163 0 : case HCCL_PROTO_TYPE_ROCE: {
164 0 : nicType_ = NicType::HOST_NIC_TYPE;
165 0 : nicDeployment_ = NICDeployment::NIC_DEPLOYMENT_HOST;
166 0 : NetworkMode netMode = NETWORK_PEER_ONLINE;
167 0 : NotifyTypeT notifyType = NOTIFY;
168 0 : CHK_RET(NetworkManager::GetInstance(deviceLogicId_)
169 : .CreateRdmaHandle(localIp_, isBackup_, netMode, notifyType, netDevDeployment_));
170 0 : NetworkManager::GetInstance(deviceLogicId_).GetRdmaHandleByIpAddr(localIp_, handle_);
171 0 : CHK_PTR_NULL(handle_);
172 0 : HCCL_INFO("[NetDevContext][InitV2]Deployment is host and proto is roce");
173 0 : break;
174 : }
175 0 : default: // 保留
176 0 : HCCL_ERROR(
177 : "[NetDevContext][DeinitV2]this prototype [%u] is not supported in host mode, please check the "
178 : "configuration.",
179 : protoType_);
180 0 : return HCCL_E_NOT_SUPPORT;
181 : }
182 : } else {
183 0 : HCCL_ERROR(
184 : "[NetDevContext][DeinitV2]this Deployment [%u] is not supported, please check the configuration.",
185 : netDevDeployment_);
186 0 : return HCCL_E_NOT_SUPPORT;
187 : // 保留
188 : }
189 :
190 0 : return HCCL_SUCCESS;
191 : }
192 :
193 230 : HcclResult NetDevContext::Deinit()
194 : {
195 230 : if (nicDeployment_ == NICDeployment::NIC_DEPLOYMENT_HOST) {
196 25 : CHK_RET(NetworkManager::GetInstance(deviceLogicId_).StopHostNet(hostSocketHandle_, localIp_));
197 : }
198 230 : return HCCL_SUCCESS;
199 : }
200 :
201 0 : HcclResult NetDevContext::DeinitV2()
202 : {
203 0 : if (netDevDeployment_ == HcclNetDevDeployment::HCCL_NETDEV_DEPLOYMENT_DEVICE) {
204 0 : switch (protoType_) {
205 0 : case HCCL_PROTO_TYPE_ROCE:
206 0 : CHK_RET(NetworkManager::GetInstance(deviceLogicId_).StopRdmaHandle(localIp_, netDevDeployment_));
207 0 : break;
208 0 : case HCCL_PROTO_TYPE_BUS:
209 0 : CHK_RET(NetworkManager::GetInstance(deviceLogicId_).StopVnicSocketHandle(localIp_));
210 0 : break;
211 0 : case HCCL_PROTO_TYPE_TCP:
212 0 : CHK_RET(NetworkManager::GetInstance(deviceLogicId_).StopNicSocketHandle(localIp_));
213 0 : break;
214 0 : default: // 保留
215 0 : HCCL_ERROR(
216 : "[NetDevContext][DeinitV2]this prototype [%u] is not supported in host mode, please check the "
217 : "configuration.",
218 : protoType_);
219 0 : return HCCL_E_NOT_SUPPORT;
220 : }
221 0 : } else if (netDevDeployment_ == HcclNetDevDeployment::HCCL_NETDEV_DEPLOYMENT_HOST) {
222 0 : switch (protoType_) {
223 0 : case HCCL_PROTO_TYPE_TCP:
224 0 : CHK_RET(NetworkManager::GetInstance(deviceLogicId_).StopHostSocketHandle(localIp_));
225 0 : break;
226 0 : case HCCL_PROTO_TYPE_ROCE:
227 0 : CHK_RET(NetworkManager::GetInstance(deviceLogicId_).StopRdmaHandle(localIp_, netDevDeployment_));
228 0 : break;
229 0 : default: // 保留
230 0 : HCCL_ERROR(
231 : "[NetDevContext][DeinitV2]this prototype [%u] is not supported in device mode, please check the "
232 : "configuration.",
233 : protoType_);
234 0 : return HCCL_E_NOT_SUPPORT;
235 : }
236 : } else {
237 0 : HCCL_ERROR(
238 : "[NetDevContext][DeinitV2]this Deployment [%u] is not supported, please check the configuration.",
239 : netDevDeployment_);
240 0 : return HCCL_E_NOT_SUPPORT;
241 : }
242 0 : return HCCL_SUCCESS;
243 : }
244 :
245 13 : void NetDevContext::SetTlsStatus(TlsStatus tlsStatus)
246 : {
247 13 : tlsStatus_ = tlsStatus;
248 13 : HCCL_INFO("[NetDevContext][SetTlsStatus]devicePhyId[%d], set tlsStatus[%d]", devicePhyId_, tlsStatus);
249 13 : return;
250 : }
251 :
252 13 : void NetDevContext::SetIsNotNeedGetTlsStatus(bool isNotNeedGetTlsStatus)
253 : {
254 13 : isNotNeedGetTlsStatus_ = isNotNeedGetTlsStatus;
255 13 : return;
256 : }
257 : } // namespace hccl
258 :
259 : HcclResult
260 210 : HcclNetInit(NICDeployment nicDeploy, s32 devicePhyId, s32 deviceLogicId, bool enableWhitelistFlag, bool hasBackup)
261 : {
262 210 : CHK_RET(hccl::DlRaFunction::GetInstance().DlRaFunctionInit());
263 210 : if (nicDeploy == NICDeployment::NIC_DEPLOYMENT_DEVICE) {
264 190 : CHK_RET(hccl::DlHalFunction::GetInstance().DlHalFunctionInit());
265 : bool isHostUseDevNic;
266 189 : CHK_RET(IsHostUseDevNic(isHostUseDevNic));
267 189 : u32 tempDevicePhyId = hasBackup ? static_cast<u32>(devicePhyId) : hccl::DEFAULT_PHY_ID;
268 189 : HCCL_DEBUG(
269 : "[%s]start NetworkManager Init, deviceLogicId[%u], devicePhyId[%u], nicDeploy[%d], hasBackup[%d],"
270 : " tempDevicePhyId[%u]",
271 : __func__, deviceLogicId, devicePhyId, nicDeploy, hasBackup, tempDevicePhyId);
272 189 : CHK_RET(hccl::NetworkManager::GetInstance(deviceLogicId)
273 : .Init(
274 : NICDeployment::NIC_DEPLOYMENT_DEVICE, enableWhitelistFlag, tempDevicePhyId, isHostUseDevNic,
275 : hasBackup));
276 : } else {
277 21 : CHK_RET(hccl::NetworkManager::GetInstance(deviceLogicId)
278 : .Init(NICDeployment::NIC_DEPLOYMENT_HOST, enableWhitelistFlag, devicePhyId));
279 : }
280 :
281 209 : return HCCL_SUCCESS;
282 : }
283 :
284 203 : HcclResult HcclNetDeInit(NICDeployment nicDeploy, [[maybe_unused]] s32 devicePhyId, s32 deviceLogicId, bool hasBackup)
285 : {
286 203 : CHK_RET(hccl::NetworkManager::GetInstance(deviceLogicId).DeInit(nicDeploy, false, hasBackup));
287 202 : return HCCL_SUCCESS;
288 : }
289 :
290 239 : HcclResult HcclNetOpenDev(
291 : HcclNetDevCtx* netDevCtx, NicType nicType, s32 devicePhyId, s32 deviceLogicId, hccl::HcclIpAddress localIp,
292 : hccl::HcclIpAddress backupIp)
293 : {
294 239 : CHK_PTR_NULL(netDevCtx);
295 :
296 239 : hccl::NetDevContext* pNetDevCtx = new (std::nothrow) hccl::NetDevContext();
297 239 : CHK_PTR_NULL(pNetDevCtx);
298 :
299 239 : HcclResult ret = pNetDevCtx->Init(nicType, devicePhyId, deviceLogicId, localIp, backupIp);
300 239 : if (ret != HCCL_SUCCESS) {
301 1 : HCCL_ERROR("[Init][Port]Init fail. ret[%u]", ret);
302 1 : delete pNetDevCtx;
303 1 : pNetDevCtx = nullptr;
304 1 : return ret;
305 : }
306 :
307 238 : *netDevCtx = pNetDevCtx;
308 :
309 238 : return HCCL_SUCCESS;
310 : }
311 :
312 230 : void HcclNetCloseDev(HcclNetDevCtx netDevCtx)
313 : {
314 230 : if (netDevCtx == nullptr) {
315 0 : HCCL_ERROR("[HcclNetCloseDev] netDevCtx is nullptr");
316 0 : return;
317 : }
318 230 : hccl::NetDevContext* pNetDevCtx = static_cast<hccl::NetDevContext*>(netDevCtx);
319 :
320 230 : HcclResult ret = pNetDevCtx->Deinit();
321 230 : if (ret != HCCL_SUCCESS) {
322 0 : HCCL_ERROR("[DeInit][Port]DeInit fail. ret[%u]", ret);
323 : }
324 :
325 230 : delete pNetDevCtx;
326 : }
327 :
328 87 : HcclResult HcclNetDevGetNicType(HcclNetDevCtx netDevCtx, NicType* nicType)
329 : {
330 87 : CHK_PTR_NULL(netDevCtx);
331 87 : CHK_PTR_NULL(nicType);
332 87 : hccl::NetDevContext* pNetDevCtx = static_cast<hccl::NetDevContext*>(netDevCtx);
333 :
334 87 : *nicType = pNetDevCtx->GetNicType();
335 86 : return HCCL_SUCCESS;
336 : }
337 :
338 290 : HcclResult HcclNetDevGetLocalIp(HcclNetDevCtx netDevCtx, hccl::HcclIpAddress& localIp)
339 : {
340 290 : CHK_PTR_NULL(netDevCtx);
341 290 : hccl::NetDevContext* pNetDevCtx = static_cast<hccl::NetDevContext*>(netDevCtx);
342 :
343 290 : localIp = pNetDevCtx->GetLocalIp();
344 289 : return HCCL_SUCCESS;
345 : }
346 :
347 20 : HcclResult HcclNetDevGetProtoType(HcclNetDevCtx netDevCtx, u32& proto)
348 : {
349 20 : CHK_PTR_NULL(netDevCtx);
350 20 : hccl::NetDevContext* pNetDevCtx = static_cast<hccl::NetDevContext*>(netDevCtx);
351 :
352 20 : proto = (u32)pNetDevCtx->GetProtoType();
353 20 : return HCCL_SUCCESS;
354 : }
355 :
356 0 : HcclResult HcclNetDevSetProtoType(HcclNetDevCtx netDevCtx, u32 proto)
357 : {
358 0 : CHK_PTR_NULL(netDevCtx);
359 0 : hccl::NetDevContext* pNetDevCtx = static_cast<hccl::NetDevContext*>(netDevCtx);
360 :
361 0 : pNetDevCtx->SetProtoType(proto);
362 0 : return HCCL_SUCCESS;
363 : }
364 :
365 0 : HcclResult HcclNetDevGetPortStatus(HcclNetDevCtx netDevCtx, bool& portStatus)
366 : {
367 0 : CHK_PTR_NULL(netDevCtx);
368 0 : hccl::NetDevContext* pNetDevCtx = static_cast<hccl::NetDevContext*>(netDevCtx);
369 0 : u32 devicePhyId = static_cast<u32>(pNetDevCtx->GetPhyId());
370 0 : RdmaHandle rdmaHandle = nullptr;
371 : enum PortStatus status;
372 0 : CHK_RET(HrtRaRdmaGetHandle(devicePhyId, rdmaHandle));
373 0 : CHK_RET(hrtRaRdevGetPortStatus(rdmaHandle, &status));
374 0 : portStatus = (status == PORT_STATUS_ACTIVE);
375 0 : HCCL_RUN_INFO("[HcclNetDevGetPortStatus]devicePhysicID_[%u], portStatus_[%d]", devicePhyId, portStatus);
376 0 : return HCCL_SUCCESS;
377 : }
378 :
379 38 : HcclResult HcclNetDevGetTlsStatus(HcclNetDevCtx netDevCtx, TlsStatus* tlsStatus)
380 : {
381 38 : CHK_PTR_NULL(netDevCtx);
382 38 : CHK_PTR_NULL(tlsStatus);
383 :
384 38 : hccl::NetDevContext* pNetDevCtx = static_cast<hccl::NetDevContext*>(netDevCtx);
385 38 : std::lock_guard<std::mutex> lock(pNetDevCtx->mu_);
386 : // tls开关状态多个通信域只需要查询一次，后续一直使用第一次查询结果
387 38 : if (pNetDevCtx->IsNotNeedGetTlsStatus()) {
388 25 : *tlsStatus = pNetDevCtx->GettlsStatus();
389 25 : return HCCL_SUCCESS;
390 : }
391 :
392 13 : u32 devicePhyId = static_cast<u32>(pNetDevCtx->GetPhyId());
393 13 : struct RaInfo raInfo = {};
394 13 : raInfo.mode = static_cast<int>(pNetDevCtx->GetNicDeployment());
395 13 : raInfo.phyId = devicePhyId;
396 13 : bool tlsEnable = false;
397 13 : HcclResult ret = HrtRaGetTlsEnable(&raInfo, &tlsEnable);
398 13 : if (ret == HCCL_E_NOT_SUPPORT) {
399 0 : pNetDevCtx->SetTlsStatus(TlsStatus::UNKNOWN);
400 13 : } else if (tlsEnable) {
401 0 : pNetDevCtx->SetTlsStatus(TlsStatus::ENABLE);
402 : } else {
403 13 : pNetDevCtx->SetTlsStatus(TlsStatus::DISABLE);
404 : }
405 13 : *tlsStatus = pNetDevCtx->GettlsStatus();
406 13 : pNetDevCtx->SetIsNotNeedGetTlsStatus(true);
407 13 : return ret;
408 38 : }
|
