Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "network_manager.h"
12 : #include "externalinput_pub.h"
13 : #include "device_capacity.h"
14 : #include "adapter_tdt.h"
15 : #include "adapter_rts.h"
16 : #include "adapter_hccp.h"
17 : #include "adapter_hal.h"
18 : #include "adapter_error_manager.h"
19 : #include "dlhal_function.h"
20 : #include "adapter_hccp_common.h"
21 : namespace hccl {
22 :
23 : using namespace std;
24 :
25 : constexpr u32 SOCKET_LISTEN_AUTO_INTERFACE = 4;
26 : constexpr u32 SOCKET_LISTEN_AUTO_INTERFACE_VERSION = 3;
27 :
28 : NetworkManager* NetworkManager::nmInstance[MAX_DEV_NUM] = {nullptr};
29 : std::atomic<unsigned> NetworkManager::InitTool::initCount(0);
30 :
31 1736 : NetworkManager::InitTool::InitTool()
32 : {
33 1736 : if (initCount.load() == 0) {
34 1716 : for (u32 i = 0; i < MAX_DEV_NUM; i++) {
35 1664 : NetworkManager::nmInstance[i] = new (std::nothrow) NetworkManager;
36 1664 : CHK_PRT_CONT(
37 : NetworkManager::nmInstance[i] == nullptr,
38 : HCCL_ERROR("[NetworkManager][InitTool] nmInstance[%u] new failed", i));
39 : }
40 : }
41 1736 : ++initCount;
42 1736 : }
43 :
44 1736 : NetworkManager::InitTool::~InitTool()
45 : {
46 1736 : --initCount;
47 1736 : if (initCount.load() == 0) {
48 1716 : for (u32 i = 0; i < MAX_DEV_NUM; i++) {
49 1664 : if (NetworkManager::nmInstance[i] != nullptr) {
50 1664 : delete NetworkManager::nmInstance[i];
51 1664 : NetworkManager::nmInstance[i] = nullptr;
52 : }
53 : }
54 : }
55 1736 : }
56 :
57 1664 : NetworkManager::NetworkManager()
58 1664 : : deviceLogicId_(INVALID_INT),
59 1664 : devicePhyId_(INVALID_UINT),
60 1664 : isHostUseDevNic_(false),
61 1664 : notifyType_(NO_USE)
62 1664 : {}
63 :
64 1664 : NetworkManager::~NetworkManager()
65 : {
66 1664 : Destroy();
67 1664 : isRaDeInit_ = false;
68 1664 : }
69 :
70 1197 : NetworkManager& NetworkManager::GetInstance(s32 deviceLogicID)
71 : {
72 1197 : HCCL_INFO("NetworkManager::GetInstance deviceLogicID[%u].", deviceLogicID);
73 1202 : if (deviceLogicID == HOST_DEVICE_ID) {
74 4 : nmInstance[DEFAULT_DEVICE_LOGIC_ID]->deviceLogicId_ = DEFAULT_DEVICE_LOGIC_ID;
75 4 : return *(nmInstance[DEFAULT_DEVICE_LOGIC_ID]);
76 : }
77 :
78 1198 : if (static_cast<u32>(deviceLogicID) >= MAX_DEV_NUM || deviceLogicID <= HOST_DEVICE_ID) {
79 0 : HCCL_WARNING("[Get][Instance]deviceLogicID[%d] is invalid", deviceLogicID);
80 0 : nmInstance[DEFAULT_DEVICE_LOGIC_ID]->deviceLogicId_ = DEFAULT_DEVICE_LOGIC_ID;
81 0 : return *(nmInstance[DEFAULT_DEVICE_LOGIC_ID]);
82 : }
83 1198 : nmInstance[deviceLogicID]->deviceLogicId_ = deviceLogicID;
84 1198 : return *(nmInstance[deviceLogicID]);
85 : }
86 :
87 413 : HcclResult NetworkManager::TsdCapabilityGet(bool& supportMultiProcHCCP)
88 : {
89 413 : int32_t type = TSD_CAPABILITY_MUTIPLE_HCCP;
90 413 : bool* resultPtr = &supportMultiProcHCCP;
91 413 : CHK_RET(hrtTsdCapabilityGet(deviceLogicId_, type, static_cast<uint64_t>(reinterpret_cast<uintptr_t>(resultPtr))));
92 413 : return HCCL_SUCCESS;
93 : }
94 :
95 0 : HcclResult NetworkManager::GetNicIp(uint32_t devicePhyId, HcclAddress** addr, uint32_t* len)
96 : {
97 0 : vector<HcclIpAddress> tempIp;
98 0 : std::unique_lock<std::mutex> lock(memResMutex_);
99 0 : if (nicIpAddrs_.size() == 0) {
100 0 : CHK_RET(hrtRaGetDeviceIP(devicePhyId, tempIp));
101 0 : nicIpAddrs_.resize(tempIp.size());
102 0 : for (size_t i = 0; i < tempIp.size(); ++i) {
103 0 : CHK_RET(HcclIpAddressConvertHcclAddr(&nicIpAddrs_[i], &tempIp[i]));
104 : }
105 : }
106 0 : *addr = this->nicIpAddrs_.data();
107 0 : *len = this->nicIpAddrs_.size();
108 0 : lock.unlock();
109 0 : return HCCL_SUCCESS;
110 0 : }
111 :
112 0 : HcclResult NetworkManager::TsdProcessOpen(bool hasBackup)
113 : {
114 0 : s32 locaLogDevid = 0;
115 0 : hrtGetDevice(&locaLogDevid);
116 0 : if (locaLogDevid != deviceLogicId_) {
117 0 : hrtSetDevice(deviceLogicId_);
118 : }
119 : // 校验是否为新版本驱动，旧版本驱动不支持配置backupPhyId，报错返回
120 0 : s32 halAPIVersion = 0;
121 0 : CHK_RET(DlHalFunction::GetInstance().DlHalFunctionInit());
122 0 : CHK_RET(hrtHalGetAPIVersion(halAPIVersion));
123 0 : HCCL_INFO(
124 : "[%s]params: halAPIVersion[%d], BACKUP_DEVICE_LOG_DEV_VERSION[%d]", __func__, halAPIVersion,
125 : BACKUP_DEVICE_LOG_DEV_VERSION);
126 0 : if (halAPIVersion < BACKUP_DEVICE_LOG_DEV_VERSION) {
127 0 : HCCL_WARNING(
128 : "[%s]this package does not support obtaining backUp HCCP Log in PLOG. halAPIVersion[%d]", __func__,
129 : halAPIVersion);
130 : }
131 0 : isTsdProcessOpen_ = true;
132 0 : if (!hasBackup || halAPIVersion < BACKUP_DEVICE_LOG_DEV_VERSION) {
133 0 : std::string extPam("--hdcType=" + std::to_string(HDC_SERVICE_TYPE_RDMA_V2));
134 0 : rtNetServiceOpenArgs openArgs = {};
135 0 : rtProcExtParam extParam{};
136 0 : extParam.paramInfo = extPam.c_str();
137 0 : extParam.paramLen = extPam.size();
138 0 : openArgs.extParamList = &extParam;
139 0 : openArgs.extParamCnt = 1UL;
140 : // 根据pid粒度拉起hccp的rs进程
141 0 : CHK_RET(hrtOpenNetService(&openArgs));
142 0 : HCCL_INFO(
143 : "[%s]hrtOpenNetService success, subPid[%d], devicePhyId_[%u], deviceLogicId_[%d], hasBackup[%u]", __func__,
144 : subPid_, devicePhyId_, deviceLogicId_, hasBackup);
145 0 : } else {
146 : // 获取chip上另一个die的logicalID
147 0 : u32 deviceBackUpPhyId = 0;
148 0 : CHK_RET(hrtGetPairDevicePhyId(devicePhyId_, deviceBackUpPhyId));
149 0 : rtNetServiceOpenArgs openArgs = {};
150 : std::string extPams[TSD_OPEN_EXT_PARA_NUM]
151 0 : = {std::string("--hdcType=" + std::to_string(HDC_SERVICE_TYPE_RDMA_V2)),
152 0 : std::string("--backupPhyId=" + std::to_string(deviceBackUpPhyId))};
153 0 : rtProcExtParam extParams[TSD_OPEN_EXT_PARA_NUM]{};
154 0 : for (u32 i = 0; i < TSD_OPEN_EXT_PARA_NUM; i++) {
155 0 : extParams[i].paramInfo = extPams[i].c_str();
156 0 : extParams[i].paramLen = extPams[i].size();
157 : }
158 0 : openArgs.extParamList = extParams;
159 0 : openArgs.extParamCnt = TSD_OPEN_EXT_PARA_NUM;
160 : // 根据pid粒度拉起hccp的rs进程
161 0 : CHK_RET(hrtOpenNetService(&openArgs));
162 0 : HCCL_INFO(
163 : "[%s]hrtOpenNetService success, subPid[%u], "
164 : "devicePhyId_[%u], deviceLogicId_[%d], deviceBackUpPhyId[%u], hasBackup[%u]",
165 : __func__, subPid_, devicePhyId_, deviceLogicId_, deviceBackUpPhyId, hasBackup);
166 0 : }
167 :
168 0 : if (locaLogDevid != deviceLogicId_) {
169 0 : hrtSetDevice(locaLogDevid);
170 : }
171 0 : return HCCL_SUCCESS;
172 : }
173 :
174 2 : HcclResult NetworkManager::PrepareInit(NICDeployment nicDeploy, u32 devicePhyId, s32& ref)
175 : {
176 2 : HCCL_INFO("nicDeploy = [%u], devicePhyId = [%u] ", nicDeploy, devicePhyId);
177 2 : if (nicDeploy == NICDeployment::NIC_DEPLOYMENT_HOST) {
178 0 : ref = hostNicInitRef_.Ref();
179 2 : } else if (nicDeploy == NICDeployment::NIC_DEPLOYMENT_DEVICE) {
180 2 : ref = deviceNicInitRef_.Ref();
181 : } else {
182 0 : HCCL_ERROR(
183 : "[NetworkManager][PrepareInit]NetworkManager: init nic failed, nicPosition[%u] is not supported.",
184 : static_cast<u32>(nicDeploy));
185 0 : return HCCL_E_INTERNAL;
186 : }
187 2 : if (ref > 1) {
188 1 : HCCL_INFO("NetworkManager: init nic, nicPosition[%u] ref[%u], skip.", static_cast<u32>(nicDeploy), ref);
189 1 : return HCCL_SUCCESS;
190 : }
191 :
192 1 : if (devicePhyId != INVALID_UINT) {
193 0 : devicePhyId_ = devicePhyId;
194 : } else {
195 : // 初始化ra资源(dev信息带入逻辑ID)
196 1 : CHK_RET(hrtGetDevice(&deviceLogicId_));
197 1 : CHK_RET(hrtGetDevicePhyIdByIndex(static_cast<u32>(deviceLogicId_), devicePhyId_));
198 : }
199 1 : return HCCL_SUCCESS;
200 : }
201 1 : HcclResult NetworkManager::GetConfigAndRaInit(struct RaInitConfig& config, bool isHdcV2, NICDeployment nicDeploy)
202 : {
203 : // DC场景网卡与进程在同一侧，需要设置为类似host网卡模式
204 1 : config.nicPosition = Is310PDevice() ? 0 : static_cast<u32>(nicDeploy);
205 1 : config.phyId = devicePhyId_;
206 :
207 1 : if (isHdcV2) {
208 : // 使用HDC_SERVICE_TYPE_RDMA_V2指定进程粒度
209 0 : config.hdcType = HDC_SERVICE_TYPE_RDMA_V2;
210 0 : HCCL_DEBUG(
211 : "[%s]hdcType is set to HDC_SERVICE_TYPE_RDMA_V2"
212 : "devicePhyId[%u], deviceLogicId_[%d]",
213 : __func__, devicePhyId_, deviceLogicId_);
214 : }
215 1 : HcclResult ret = HrtRaInit(&config);
216 1 : RPT_CALL_ERR(
217 : ret != HCCL_SUCCESS, "ra init failed,return[%d] devicePhyId_[%u], nicPosition[%u]", ret, devicePhyId_,
218 : static_cast<u32>(nicDeploy));
219 1 : CHK_PRT_RET(
220 : ret != HCCL_SUCCESS,
221 : HCCL_ERROR(
222 : "[NetworkManager][Init]errNo[0x%016llx] ra init failed,return[%d] devicePhyId_[%u], "
223 : "nicPosition[%u]",
224 : HCCL_ERROR_CODE(ret), ret, devicePhyId_, static_cast<u32>(nicDeploy)),
225 : HCCL_E_NETWORK);
226 :
227 1 : return HCCL_SUCCESS;
228 : }
229 :
230 1 : HcclResult NetworkManager::GetTsdOpen(NICDeployment nicDeploy, bool hasBackup, bool& supportMultiProcHCCP)
231 : {
232 1 : if (nicDeploy == NICDeployment::NIC_DEPLOYMENT_DEVICE && !Is310PDevice()) { // DC场景不需要拉起tsd
233 1 : CHK_RET(DlTdtFunction::GetInstance().DlTdtFunctionInit());
234 1 : CHK_RET(TsdCapabilityGet(supportMultiProcHCCP));
235 1 : HCCL_INFO("[NetworkManager][Init]supportMultiProcHCCP[%u], hasBackup[%u]", supportMultiProcHCCP, hasBackup);
236 1 : if (supportMultiProcHCCP || hasBackup) {
237 : // 根据pid粒度拉起hccp的rs进程
238 0 : CHK_RET(TsdProcessOpen(hasBackup));
239 0 : HCCL_INFO(
240 : "[NetworkManager][Init]open tsd by process success, devicePhyId[%u], deviceLogicId_[%d].", devicePhyId_,
241 : deviceLogicId_);
242 0 : } else {
243 : // device 网卡初始化前需要拉起 hccp.
244 1 : CHK_RET(hrtOpenTsd());
245 1 : HCCL_INFO(
246 : "[%s]NetworkManager open tsd success, devicePhyId[%u], deviceLogicId_[%d]", __func__, devicePhyId_,
247 : deviceLogicId_);
248 : }
249 : }
250 1 : return HCCL_SUCCESS;
251 : }
252 : /* init network resource */
253 252 : HcclResult NetworkManager::Init(
254 : NICDeployment nicDeploy, bool enableWhitelistFlag, u32 devicePhyId, bool isHostUseDevNic, bool hasBackup)
255 : {
256 252 : s32 ref = 0;
257 252 : if (nicDeploy == NICDeployment::NIC_DEPLOYMENT_HOST) {
258 25 : ref = hostNicInitRef_.Ref();
259 227 : } else if (nicDeploy == NICDeployment::NIC_DEPLOYMENT_DEVICE) {
260 227 : ref = deviceNicInitRef_.Ref();
261 : } else {
262 0 : HCCL_ERROR(
263 : "[NetworkManager][Init]NetworkManager: init nic failed, nicPosition[%u] is not supported.",
264 : static_cast<u32>(nicDeploy));
265 0 : return HCCL_E_INTERNAL;
266 : }
267 252 : if (ref > 1) {
268 24 : HCCL_INFO("NetworkManager: init nic, nicPosition[%u] ref[%u], skip.", static_cast<u32>(nicDeploy), ref);
269 24 : return HCCL_SUCCESS;
270 : }
271 :
272 228 : if (devicePhyId != INVALID_UINT) {
273 38 : devicePhyId_ = devicePhyId;
274 : } else {
275 : // 初始化ra资源(dev信息带入逻辑ID)
276 190 : CHK_RET(hrtGetDevice(&deviceLogicId_));
277 190 : CHK_RET(hrtGetDevicePhyIdByIndex(static_cast<u32>(deviceLogicId_), devicePhyId_));
278 : }
279 228 : HCCL_INFO(
280 : "NetworkManager: devicePhyId[%u], devicePhyId_[%u] deviceLogicId_[%u], hasBackup[%d]", devicePhyId,
281 : devicePhyId_, deviceLogicId_, hasBackup);
282 :
283 228 : bool supportMultiProcHCCP = false;
284 228 : if (nicDeploy == NICDeployment::NIC_DEPLOYMENT_DEVICE && !Is310PDevice()) { // DC场景不需要拉起tsd
285 207 : CHK_RET(DlTdtFunction::GetInstance().DlTdtFunctionInit());
286 207 : CHK_RET(TsdCapabilityGet(supportMultiProcHCCP));
287 207 : HCCL_INFO("[NetworkManager][Init]supportMultiProcHCCP[%u], hasBackup[%u]", supportMultiProcHCCP, hasBackup);
288 207 : if (supportMultiProcHCCP || hasBackup) {
289 : // 根据pid粒度拉起hccp的rs进程
290 0 : CHK_RET(TsdProcessOpen(hasBackup));
291 0 : HCCL_INFO(
292 : "[NetworkManager][Init]open tsd by process success, devicePhyId[%u], deviceLogicId_[%d].", devicePhyId_,
293 : deviceLogicId_);
294 0 : } else {
295 : // device 网卡初始化前需要拉起 hccp.
296 207 : CHK_RET(hrtOpenTsd());
297 207 : HCCL_INFO(
298 : "[%s]NetworkManager open tsd success, devicePhyId[%u], deviceLogicId_[%d]", __func__, devicePhyId_,
299 : deviceLogicId_);
300 : }
301 : }
302 :
303 228 : struct RaInitConfig config = {DEFAULT_INIT_PHY_ID, DEFAULT_INIT_NIC_POS, DEFAULT_HDC_TYPE, false};
304 228 : u32 enableWhiteList = (GetExternalInputHcclEnableWhitelist() == HCCL_WHITELIST_ON) ? 1 : 0;
305 228 : if (GetRemoteIsHdc() && IsGeneralServer()) {
306 0 : HCCL_INFO("General server NetworkManager open Whitelist");
307 0 : enableWhitelistFlag = true;
308 0 : enableWhiteList = true;
309 : }
310 228 : if (nicDeploy == NICDeployment::NIC_DEPLOYMENT_HOST && enableWhitelistFlag) {
311 16 : CHK_RET(hrtRaSocketSetWhiteListStatus(enableWhiteList));
312 : }
313 : // DC场景网卡与进程在同一侧，需要设置为类似host网卡模式
314 228 : config.nicPosition = Is310PDevice() ? 0 : static_cast<u32>(nicDeploy);
315 228 : config.phyId = devicePhyId_;
316 228 : HCCL_DEBUG(
317 : "[%s]config.phyId = %u, nicPosition[%u], hasBackup[%d], devicePhyId_[%u], deviceLogicId_[%d], "
318 : "devicePhyId[%u]",
319 : __func__, config.phyId, config.nicPosition, hasBackup, devicePhyId_, deviceLogicId_, devicePhyId);
320 :
321 228 : if (nicDeploy == NICDeployment::NIC_DEPLOYMENT_DEVICE && !Is310PDevice() && (supportMultiProcHCCP || hasBackup)) {
322 : // 使用HDC_SERVICE_TYPE_RDMA_V2指定进程粒度
323 0 : config.hdcType = HDC_SERVICE_TYPE_RDMA_V2;
324 0 : HCCL_DEBUG(
325 : "[%s]hdcType is set to HDC_SERVICE_TYPE_RDMA_V2, hasBackup[%d], nicDeploy[%d], "
326 : "devicePhyId[%u], deviceLogicId_[%d]",
327 : __func__, hasBackup, nicDeploy, devicePhyId_, deviceLogicId_);
328 : }
329 228 : DevType devType = DevType::DEV_TYPE_COUNT;
330 228 : CHK_RET(hrtGetDeviceType(devType));
331 228 : if (devType == DevType::DEV_TYPE_910_93 && nicDeploy == NICDeployment::NIC_DEPLOYMENT_DEVICE) {
332 8 : isEnableHdcAsync_ = true;
333 8 : config.enableHdcAsync = true;
334 : }
335 228 : HCCL_INFO(
336 : "[%s]config.phyId[%u], config.nicPosition[%u], config.hdcType[%d], config.enableHdcAsync[%d]", __func__,
337 : config.phyId, config.nicPosition, config.hdcType, config.enableHdcAsync);
338 228 : HcclResult ret = HrtRaInit(&config);
339 228 : RPT_CALL_ERR(
340 : ret != HCCL_SUCCESS, "ra init failed,return[%d] devicePhyId_[%u], nicPosition[%u]", ret, devicePhyId_,
341 : static_cast<u32>(nicDeploy));
342 228 : CHK_PRT_RET(
343 : ret != HCCL_SUCCESS,
344 : HCCL_ERROR(
345 : "[NetworkManager][Init]errNo[0x%016llx] ra init failed,return[%d] devicePhyId_[%u], "
346 : "nicPosition[%u]",
347 : HCCL_ERROR_CODE(ret), ret, devicePhyId_, static_cast<u32>(nicDeploy)),
348 : HCCL_E_NETWORK);
349 :
350 227 : HCCL_INFO(
351 : "NetworkManager nicDeploy[%u] deviceLogicId_[%d] devicePhyId_[%u] init ra OK, nicSocketMap size[%u], "
352 : "isHostUseDevNic[%d]",
353 : static_cast<u32>(nicDeploy), deviceLogicId_, devicePhyId_, raResourceInfo_.nicSocketMap.size(),
354 : isHostUseDevNic);
355 227 : return HCCL_SUCCESS;
356 : }
357 :
358 2 : HcclResult NetworkManager::InitV2(NICDeployment nicDeploy, bool isBackup, u32 devicePhyId, bool isHostUseDevNic)
359 : {
360 2 : s32 ref = 0;
361 2 : CHK_RET(PrepareInit(nicDeploy, devicePhyId, ref));
362 2 : if (ref > 1) {
363 1 : HCCL_INFO("NetworkManager: initv2 nic, nicPosition[%u] ref[%u], skip.", static_cast<u32>(nicDeploy), ref);
364 1 : return HCCL_SUCCESS;
365 : }
366 1 : HCCL_INFO(
367 : "NetworkManager InitV2: devicePhyId[%u], devicePhyId_[%u] deviceLogicId_[%u], isBackup[%d]", devicePhyId,
368 : devicePhyId_, deviceLogicId_, isBackup);
369 :
370 1 : bool supportMultiProcHCCP = false;
371 1 : CHK_RET(GetTsdOpen(nicDeploy, isBackup, supportMultiProcHCCP));
372 :
373 1 : struct RaInitConfig config = {DEFAULT_INIT_PHY_ID, DEFAULT_INIT_NIC_POS, DEFAULT_HDC_TYPE, false};
374 : bool isHdcV2
375 1 : = (nicDeploy == NICDeployment::NIC_DEPLOYMENT_DEVICE && !Is310PDevice() && (supportMultiProcHCCP || isBackup));
376 1 : if (isHdcV2) {
377 0 : HCCL_DEBUG(
378 : "[%s]hdcType is set to HDC_SERVICE_TYPE_RDMA_V2, isBackup[%d], nicDeploy[%d], "
379 : "devicePhyId[%u], deviceLogicId_[%d]",
380 : __func__, isBackup, nicDeploy, devicePhyId_, deviceLogicId_);
381 : }
382 1 : CHK_RET(GetConfigAndRaInit(config, isHdcV2, nicDeploy));
383 :
384 1 : HCCL_INFO(
385 : "NetworkManager nicDeploy[%u] deviceLogicId_[%d] devicePhyId_[%u] init ra OK, nicSocketMap size[%u], "
386 : "isHostUseDevNic[%d]",
387 : static_cast<u32>(nicDeploy), deviceLogicId_, devicePhyId_, raResourceInfo_.nicSocketMap.size(),
388 : isHostUseDevNic);
389 1 : return HCCL_SUCCESS;
390 : }
391 :
392 0 : HcclResult NetworkManager::HeterogStartListen(const HcclIpAddress& ipAddr, u32 port)
393 : {
394 0 : HCCL_DEBUG("HeterogStartListen ipAddr[%s] port[%u]", ipAddr.GetReadableAddress(), port);
395 0 : SocketHandle nicSocketHandle = raResourceInfo_.nicSocketMap[ipAddr].nicSocketHandle;
396 :
397 0 : CHK_PRT_RET(
398 : port > MAX_PORT_ID || port < MIN_PORT_ID,
399 : HCCL_ERROR(
400 : "[NetworkManager][HeterogStartListen] port error[%u], valid range[%u, %u]", port, MIN_PORT_ID, MAX_PORT_ID),
401 : HCCL_E_NETWORK);
402 :
403 0 : if (nicSocketHandle != nullptr
404 0 : && raResourceInfo_.nicSocketMap[ipAddr].listenedPort.find(port)
405 0 : == raResourceInfo_.nicSocketMap[ipAddr].listenedPort.end()
406 0 : && IPPortListenRefMapHost_[ipAddr][port].Ref() == FIRST_LISTEN) {
407 0 : CHK_RET(StartListenSocket(nicSocketHandle, port));
408 0 : raResourceInfo_.nicSocketMap[ipAddr].listenedPort.insert(port);
409 0 : raResourceInfo_.hostNetSocketMap[ipAddr].listenedPort.insert(port);
410 : }
411 0 : return HCCL_SUCCESS;
412 : }
413 :
414 1 : HcclResult NetworkManager::HeterogInit(u32 devId, const HcclIpAddress& ipAddr, u32 port)
415 : {
416 1 : s32 ref = hostNicInitRef_.Ref();
417 1 : if (ref > 1) {
418 0 : HCCL_INFO("NetworkManager: heterog init nic, ref[%u], skip", ref);
419 0 : return HCCL_SUCCESS;
420 : }
421 :
422 1 : CHK_RET(hrtRaSocketSetWhiteListStatus(0));
423 :
424 : // 暂缺获取物理id的手段
425 1 : RaInitConfig config = {DEFAULT_INIT_PHY_ID, DEFAULT_INIT_NIC_POS, DEFAULT_HDC_TYPE, false};
426 1 : config.phyId = ((static_cast<s32>(devId) == HOST_DEVICE_ID) ? 0 : devId);
427 1 : config.nicPosition = static_cast<u32>(NICDeployment::NIC_DEPLOYMENT_HOST);
428 1 : HCCL_INFO("HeterogInit call HrtRaInit.");
429 1 : CHK_RET(HrtRaInit(&config));
430 :
431 1 : struct rdev nicRdevInfo = {};
432 1 : nicRdevInfo.phyId = devId;
433 1 : nicRdevInfo.family = ipAddr.GetFamily();
434 1 : nicRdevInfo.localIp.addr = ipAddr.GetBinaryAddress().addr;
435 1 : nicRdevInfo.localIp.addr6 = ipAddr.GetBinaryAddress().addr6;
436 1 : SocketHandle socketHandle = nullptr;
437 1 : HcclResult ret = hrtRaSocketInit(NETWORK_PEER_ONLINE, nicRdevInfo, socketHandle);
438 1 : if (ret != HCCL_SUCCESS) {
439 1 : HCCL_ERROR(
440 : "[HeterogInit]errNo[0x%016llx] ra socket init failed, ip[%s], return[%d]",
441 : HCCL_ERROR_CODE(HCCL_E_TCP_CONNECT), ipAddr.GetReadableAddress(), ret);
442 1 : HrtRaDeInit(&config);
443 1 : return HCCL_E_TCP_CONNECT;
444 : }
445 0 : HCCL_INFO("ip[%s] socket init OK", ipAddr.GetReadableAddress());
446 :
447 0 : IpSocket ipSocketInfo;
448 0 : ipSocketInfo.nicSocketHandle = socketHandle;
449 :
450 0 : raResourceInfo_.nicSocketMap.insert(std::make_pair(ipAddr, ipSocketInfo));
451 0 : raResourceInfo_.hostNetSocketMap.insert(std::make_pair(ipAddr, ipSocketInfo));
452 :
453 : /* device网卡初始化暂不考虑 */
454 0 : if (!GetExternalInputHcclIsTcpMode()) {
455 0 : ret = InitRdmaHandle(devId, ipAddr);
456 0 : if (ret != HCCL_SUCCESS) {
457 0 : HCCL_ERROR("[HeterogInit] InitRdmaHandle fail, ret[%d], destroy resource.", ret);
458 0 : hrtRaSocketDeInit(socketHandle);
459 0 : HrtRaDeInit(&config);
460 0 : return ret;
461 : }
462 : }
463 :
464 0 : ret = HeterogStartListen(ipAddr, port);
465 0 : if (ret != HCCL_SUCCESS) {
466 0 : HCCL_ERROR("[HeterogInit] HeterogStartListen fail, ret[%d], destroy resource.", ret);
467 0 : if (!GetExternalInputHcclIsTcpMode()) {
468 0 : auto& ipSock = raResourceInfo_.nicSocketMap[ipAddr];
469 0 : if (ipSock.nicRdmaHandle != nullptr) {
470 0 : (void)HrtRaRdmaDeInit(ipSock.nicRdmaHandle, NO_USE);
471 : }
472 : }
473 0 : hrtRaSocketDeInit(socketHandle);
474 0 : HrtRaDeInit(&config);
475 0 : raResourceInfo_.nicSocketMap.erase(ipAddr);
476 0 : return ret;
477 : }
478 0 : return HCCL_SUCCESS;
479 0 : }
480 :
481 0 : HcclResult NetworkManager::HeterogStopListen(const HcclIpAddress& ipAddr, u32 port, bool isNeedDeinit)
482 : {
483 0 : HCCL_DEBUG("HeterogStopListen ipAddr[%s] port[%u]", ipAddr.GetReadableAddress(), port);
484 0 : SocketHandle nicSocketHandle = raResourceInfo_.nicSocketMap[ipAddr].nicSocketHandle;
485 :
486 0 : if (nicSocketHandle != nullptr && raResourceInfo_.nicSocketMap[ipAddr].listenedPort.size() > 0) {
487 0 : if (raResourceInfo_.nicSocketMap[ipAddr].listenedPort.find(port)
488 0 : != raResourceInfo_.nicSocketMap[ipAddr].listenedPort.end()
489 0 : && IPPortListenRefMapHost_[ipAddr][port].Unref() == LAST_RELEASE) {
490 : // 重复raInit时不用去停止监听，但是要处理引用计数，否则析构会出问题
491 0 : if (isRaInitRepeated_) {
492 0 : return HCCL_SUCCESS;
493 : }
494 0 : HCCL_INFO("HeterogStopListen socket listen stop, socket port[%u]", port);
495 0 : CHK_RET(StopListenSocket(nicSocketHandle, port));
496 0 : raResourceInfo_.nicSocketMap[ipAddr].listenedPort.erase(port);
497 0 : raResourceInfo_.hostNetSocketMap[ipAddr].listenedPort.erase(port);
498 : }
499 :
500 0 : if (isNeedDeinit) {
501 0 : CHK_RET(hrtRaSocketDeInit(nicSocketHandle));
502 : }
503 : }
504 0 : return HCCL_SUCCESS;
505 : }
506 :
507 1 : HcclResult NetworkManager::HeterogDeinit(u32 devId, const HcclIpAddress& ipAddr, u32 port)
508 : {
509 1 : s32 ref = hostNicInitRef_.Unref();
510 1 : if (ref > 0) {
511 0 : HCCL_INFO("NetworkManager: heterog deinit nic success, ref[%u], skip.", ref);
512 0 : return HCCL_SUCCESS;
513 1 : } else if (ref < 0) {
514 1 : HCCL_ERROR("[NetworkManager][DeInit]NetworkManager: heterog deinit nic failed, nic has already deinit.");
515 1 : return HCCL_E_INTERNAL;
516 : }
517 :
518 0 : CHK_RET(HeterogStopListen(ipAddr, port, true));
519 :
520 0 : SocketHandle nicRdmaHandle = raResourceInfo_.nicSocketMap[ipAddr].nicRdmaHandle;
521 0 : if (!GetExternalInputHcclIsTcpMode() && nicRdmaHandle != nullptr) {
522 0 : CHK_RET(HrtRaRdmaDeInit(nicRdmaHandle, NO_USE));
523 : }
524 :
525 0 : RaInitConfig config = {DEFAULT_INIT_PHY_ID, DEFAULT_INIT_NIC_POS, DEFAULT_HDC_TYPE, false};
526 0 : config.phyId = devId;
527 0 : config.nicPosition = static_cast<u32>(NICDeployment::NIC_DEPLOYMENT_HOST);
528 :
529 0 : HCCL_INFO("HeterogDeinit call HrtRaDeInit.");
530 0 : CHK_RET(HrtRaDeInit(&config));
531 :
532 0 : raResourceInfo_.nicSocketMap.erase(raResourceInfo_.nicSocketMap.find(ipAddr));
533 0 : raResourceInfo_.hostNetSocketMap.erase(raResourceInfo_.hostNetSocketMap.find(ipAddr));
534 :
535 0 : return HCCL_SUCCESS;
536 : }
537 :
538 1 : HcclResult NetworkManager::CloseHccpProcess()
539 : {
540 1 : std::unique_lock<std::mutex> lock(hccpProcInfoMutex_);
541 1 : if (isTsdProcessOpen_ == true) {
542 0 : s32 locaLogDevid = 0;
543 0 : hrtGetDevice(&locaLogDevid);
544 0 : if (locaLogDevid != deviceLogicId_) {
545 0 : hrtSetDevice(deviceLogicId_);
546 : }
547 0 : CHK_RET(hrtCloseNetService());
548 0 : isTsdProcessOpen_ = false;
549 0 : if (locaLogDevid != deviceLogicId_) {
550 0 : hrtSetDevice(locaLogDevid);
551 : }
552 : }
553 1 : return HCCL_SUCCESS;
554 1 : }
555 :
556 1 : HcclResult NetworkManager::PrepareDeInit(s32& ref, NICDeployment nicDeploy)
557 : {
558 1 : if (nicDeploy == NICDeployment::NIC_DEPLOYMENT_HOST) {
559 1 : ref = hostNicInitRef_.Unref();
560 0 : } else if (nicDeploy == NICDeployment::NIC_DEPLOYMENT_DEVICE) {
561 0 : ref = deviceNicInitRef_.Unref();
562 : } else {
563 0 : HCCL_ERROR(
564 : "[NetworkManager][PrepareDeInit]NetworkManager: deinit nic failed, nicPosition[%u] is not supported.",
565 : static_cast<u32>(nicDeploy));
566 0 : return HCCL_E_INTERNAL;
567 : }
568 :
569 1 : if (ref > 0) {
570 0 : HCCL_INFO(
571 : "NetworkManager: PrepareDeInit nic success, nicPosition[%u] ref[%u], skip.", static_cast<u32>(nicDeploy),
572 : ref);
573 0 : return HCCL_SUCCESS;
574 1 : } else if (ref < 0) {
575 1 : HCCL_ERROR(
576 : "[NetworkManager][PrepareDeInit]NetworkManager: deinit nic failed, nicPosition[%u] has already deinit.",
577 : nicDeploy);
578 1 : return HCCL_E_INTERNAL;
579 : }
580 0 : return HCCL_SUCCESS;
581 : }
582 :
583 0 : HcclResult NetworkManager::GetConfigAndRaDeinit(
584 : struct RaInitConfig& config, NICDeployment nicDeploy, bool& isMultiProc, bool hasBackup)
585 : {
586 0 : if (nicDeploy == NICDeployment::NIC_DEPLOYMENT_DEVICE && !Is310PDevice()) {
587 0 : CHK_RET(DlTdtFunction::GetInstance().DlTdtFunctionInit());
588 0 : bool supportMultiProcHCCP = false;
589 0 : CHK_RET(TsdCapabilityGet(supportMultiProcHCCP));
590 0 : if (supportMultiProcHCCP || hasBackup) {
591 0 : isMultiProc = true;
592 0 : config.hdcType = HDC_SERVICE_TYPE_RDMA_V2;
593 : }
594 : }
595 :
596 0 : HcclResult ret = HrtRaDeInit(&config);
597 0 : CHK_PRT_RET(
598 : ret != HCCL_SUCCESS,
599 : HCCL_ERROR(
600 : "[NetworkManager][DeInit]ra deinit failed. para: nicdeploy[%u], phyId[%u], ret[%u]", config.nicPosition,
601 : config.phyId, ret),
602 : ret);
603 0 : if (IsGeneralServer()) {
604 0 : isRaDeInit_ = true;
605 : }
606 0 : return HCCL_SUCCESS;
607 : }
608 :
609 234 : HcclResult NetworkManager::DeInit(NICDeployment nicDeploy, [[maybe_unused]] bool resetDeviceFlag, bool hasBackup)
610 : {
611 234 : s32 ref = 0;
612 234 : if (nicDeploy == NICDeployment::NIC_DEPLOYMENT_HOST) {
613 20 : ref = hostNicInitRef_.Unref();
614 214 : } else if (nicDeploy == NICDeployment::NIC_DEPLOYMENT_DEVICE) {
615 217 : ref = deviceNicInitRef_.Unref();
616 : } else {
617 0 : HCCL_ERROR(
618 : "[NetworkManager][DeInit]NetworkManager: deinit nic failed, nicPosition[%u] is not supported.",
619 : static_cast<u32>(nicDeploy));
620 0 : return HCCL_E_INTERNAL;
621 : }
622 :
623 237 : if (ref > 0) {
624 15 : HCCL_INFO(
625 : "NetworkManager: deinit nic success, nicPosition[%u] ref[%u], skip.", static_cast<u32>(nicDeploy), ref);
626 15 : return HCCL_SUCCESS;
627 222 : } else if (ref < 0) {
628 1 : HCCL_ERROR(
629 : "[NetworkManager][DeInit]NetworkManager: deinit nic failed, nicPosition[%u] has already deinit.",
630 : nicDeploy);
631 1 : return HCCL_E_INTERNAL;
632 : }
633 :
634 221 : struct RaInitConfig config = {DEFAULT_INIT_PHY_ID, DEFAULT_INIT_NIC_POS, DEFAULT_HDC_TYPE, isEnableHdcAsync_};
635 221 : config.nicPosition = Is310PDevice() ? 0 : static_cast<u32>(nicDeploy);
636 221 : config.phyId = devicePhyId_;
637 :
638 221 : bool isMultiProc = false;
639 221 : if (nicDeploy == NICDeployment::NIC_DEPLOYMENT_DEVICE && !Is310PDevice()) {
640 206 : CHK_RET(DlTdtFunction::GetInstance().DlTdtFunctionInit());
641 206 : bool supportMultiProcHCCP = false;
642 206 : CHK_RET(TsdCapabilityGet(supportMultiProcHCCP));
643 206 : if (supportMultiProcHCCP || hasBackup) {
644 1 : isMultiProc = true;
645 1 : config.hdcType = HDC_SERVICE_TYPE_RDMA_V2;
646 : }
647 : }
648 :
649 221 : HcclResult ret = HrtRaDeInit(&config);
650 221 : CHK_PRT_RET(
651 : ret != HCCL_SUCCESS,
652 : HCCL_ERROR(
653 : "[NetworkManager][DeInit]ra deinit failed. para: nicdeploy[%u], phyId[%u], ret[%u]", config.nicPosition,
654 : config.phyId, ret),
655 : ret);
656 221 : if (IsGeneralServer()) {
657 0 : isRaDeInit_ = true;
658 : }
659 :
660 221 : if (isMultiProc) {
661 1 : CHK_RET(CloseHccpProcess());
662 1 : HCCL_INFO(
663 : "[%s]finish CloseHccpProcess, phyId[%u], hdcType[%d], nicDeployment[%d], hasBackup[%d]", __func__,
664 : config.phyId, config.hdcType, nicDeploy, hasBackup);
665 : }
666 :
667 221 : HCCL_INFO("NetworkManager: deinit nic success, nicPosition[%u] ref[%u].", static_cast<u32>(nicDeploy), ref);
668 221 : return HCCL_SUCCESS;
669 : }
670 :
671 0 : HcclResult NetworkManager::DeInitV2(NICDeployment nicDeploy, bool isBackup, [[maybe_unused]] bool resetDeviceFlag)
672 : {
673 0 : s32 ref = 0;
674 0 : CHK_RET(PrepareDeInit(ref, nicDeploy));
675 0 : if (ref > 0) {
676 0 : HCCL_INFO(
677 : "NetworkManager: DeInitV2 nic success, nicPosition[%u] ref[%u], skip.", static_cast<u32>(nicDeploy), ref);
678 0 : return HCCL_SUCCESS;
679 : }
680 :
681 0 : struct RaInitConfig config = {DEFAULT_INIT_PHY_ID, DEFAULT_INIT_NIC_POS, DEFAULT_HDC_TYPE, false};
682 0 : config.nicPosition = Is310PDevice() ? 0 : static_cast<u32>(nicDeploy);
683 0 : config.phyId = devicePhyId_;
684 :
685 0 : bool isMultiProc = false;
686 0 : CHK_RET(GetConfigAndRaDeinit(config, nicDeploy, isMultiProc, isBackup));
687 :
688 0 : if (isMultiProc) {
689 0 : CHK_RET(CloseHccpProcess());
690 0 : HCCL_INFO(
691 : "[%s]finish CloseHccpProcess, phyId[%u], hdcType[%d], nicDeployment[%d], isBackup[%d]", __func__,
692 : config.phyId, config.hdcType, nicDeploy, isBackup);
693 : }
694 :
695 0 : HCCL_INFO("NetworkManager: DeInitV2 success, nicPosition[%u] ref[%u].", static_cast<u32>(nicDeploy), ref);
696 0 : return HCCL_SUCCESS;
697 : }
698 :
699 24 : HcclResult NetworkManager::StartVnic(HcclIpAddress localIp, u32& port)
700 : {
701 24 : CHK_PRT_RET(
702 : deviceNicInitRef_.Count() <= 0, HCCL_ERROR("[Start][Vnic]can't start vnic socket before init device nic!"),
703 : HCCL_E_INTERNAL);
704 19 : CHK_PRT_RET(Is310PDevice(), HCCL_INFO("DC does not need vnic"), HCCL_SUCCESS);
705 19 : CHK_PRT_RET(
706 : port > MAX_PORT_ID, HCCL_ERROR("[Start][Vnic]invalid port id[%u], max port id[%u]", port, MAX_PORT_ID),
707 : HCCL_E_PARA);
708 :
709 19 : auto sockInfo = raResourceInfo_.vnicSocketMap.find(localIp);
710 19 : if (sockInfo == raResourceInfo_.vnicSocketMap.end()) {
711 14 : IpSocket tempSock;
712 14 : raResourceInfo_.vnicSocketMap.insert(std::make_pair(localIp, tempSock)); // 本IP占位
713 14 : HCCL_INFO("[Start][Vnic]device[%u] Start Vnic insert ip[%s]", devicePhyId_, localIp.GetReadableAddress());
714 14 : }
715 :
716 19 : IpSocket& sock = raResourceInfo_.vnicSocketMap[localIp];
717 19 : if (sock.listenedPort.size() == 0 && sock.nicSocketHandle == nullptr) {
718 14 : HcclResult ret = InitDeviceSocket(devicePhyId_, localIp, sock.nicSocketHandle);
719 14 : CHK_PRT_RET(
720 : ret != HCCL_SUCCESS,
721 : HCCL_ERROR(
722 : "[Start][Vnic]errNo[0x%016llx] ra vnic init socket failed, devid[%u], ipAddr[%s], return[%d]",
723 : HCCL_ERROR_CODE(HCCL_E_TCP_CONNECT), devicePhyId_, localIp.GetReadableAddress(), ret),
724 : ret);
725 : } else {
726 5 : HCCL_INFO(
727 : "[Start][Vnic]vnic socket has inited, ipAddr[%s] port[%u], skip init.", localIp.GetReadableAddress(), port);
728 : }
729 :
730 19 : if (sock.listenedPort.find(port) != sock.listenedPort.end()) {
731 5 : HCCL_WARNING("[Start][Vnic]ipAddr[%s] port[%u] is already listened.", localIp.GetReadableAddress(), port);
732 : } else {
733 : // 作为socket server端启动监听(虚拟网卡)
734 14 : bool isAutoPort = port == 0;
735 14 : HCCL_INFO("[Start][Vnic]trying to listen on ip[%s] port[%u].", localIp.GetReadableAddress(), port);
736 14 : CHK_RET(CheckAutoListenVersion(isAutoPort));
737 14 : HcclResult ret = StartListenSocket(sock.nicSocketHandle, port); /* 只拉起1个vnic */
738 14 : CHK_PRT_RET(
739 : ret == HCCL_E_UNAVAIL,
740 : HCCL_INFO(
741 : "[Start][StartVnic]Could not start listening socket for IP [%s] and port [%u].",
742 : localIp.GetReadableAddress(), port),
743 : ret);
744 14 : CHK_PRT_RET(
745 : ret != HCCL_SUCCESS,
746 : HCCL_ERROR(
747 : "[Start][Vnic]errNo[0x%016llx] ra inner listen start failed, "
748 : "devid[%u], ip[%s], port[%u], return[%d]",
749 : HCCL_ERROR_CODE(ret), devicePhyId_, localIp.GetReadableAddress(), port, ret),
750 : ret);
751 14 : sock.listenedPort.insert(port);
752 14 : HCCL_RUN_INFO(
753 : "[Start][Vnic]Listen on ip[%s], port[%u] success, devPhyId[%u], devLogicId[%u], isAutoPort[%d]",
754 : localIp.GetReadableAddress(), port, devicePhyId_, deviceLogicId_, isAutoPort);
755 : }
756 19 : int refCount = IPPortListenRefMapVnicDevice_[localIp][port].Ref();
757 19 : HCCL_INFO(
758 : "NetworkManager devicePhyId_[%u] ip[%s] port[%u] start vnic OK. refCount[%d]", devicePhyId_,
759 : localIp.GetReadableAddress(), port, refCount);
760 19 : return HCCL_SUCCESS;
761 : }
762 :
763 : // 此处只进行socket的创建 不listen
764 0 : HcclResult NetworkManager::CreateVnicSocketHandle(HcclIpAddress localIp)
765 : {
766 0 : CHK_PRT_RET(
767 : deviceNicInitRef_.Count() == 0,
768 : HCCL_ERROR("[NetworkManager][CreateVnicSocketHandle]"
769 : "can't start vnic socket before init device nic!"),
770 : HCCL_E_INTERNAL);
771 0 : OccupyIp(localIp, raResourceInfo_.vnicSocketMap);
772 0 : CHK_PRT_RET(Is310PDevice(), HCCL_INFO("DC does not need vnic"), HCCL_SUCCESS);
773 :
774 0 : IpSocket& sock = raResourceInfo_.vnicSocketMap[localIp];
775 0 : if (sock.listenedPort.size() == 0 && sock.nicSocketHandle == nullptr) {
776 0 : HcclResult ret = InitDeviceSocket(devicePhyId_, localIp, sock.nicSocketHandle);
777 0 : CHK_PRT_RET(
778 : ret != HCCL_SUCCESS,
779 : HCCL_ERROR(
780 : "[NetworkManager][CreateVnicSocketHandle]errNo[0x%016llx] ra vnic init socket failed, "
781 : "devid[%u], return[%d]",
782 : HCCL_ERROR_CODE(HCCL_E_TCP_CONNECT), devicePhyId_, ret),
783 : ret);
784 : } else {
785 0 : HCCL_INFO(
786 : "[NetworkManager][CreateVnicSocketHandle] socket has inited, ipAddr[%s], skip.",
787 : localIp.GetReadableAddress());
788 : }
789 0 : HCCL_INFO(
790 : "[NetworkManager][CreateVnicSocketHandle] CreateVnicSocketHandle OK, ipAddr[%s].",
791 : localIp.GetReadableAddress());
792 0 : return HCCL_SUCCESS;
793 : }
794 :
795 6 : HcclResult NetworkManager::StopVnic(const HcclIpAddress& localIp, u32 port)
796 : {
797 6 : auto it = raResourceInfo_.vnicSocketMap.find(localIp);
798 6 : CHK_PRT_RET(
799 : it == raResourceInfo_.vnicSocketMap.end(),
800 : HCCL_ERROR("[Stop][Vnic]ip[%s] is not found in vnicSocketMap, port[%u].", localIp.GetReadableAddress(), port),
801 : HCCL_E_INTERNAL);
802 6 : IpSocket& ipSock = it->second;
803 :
804 6 : int count = IPPortListenRefMapVnicDevice_[localIp][port].Unref();
805 6 : CHK_PRT_RET(
806 : count > 0,
807 : HCCL_INFO("[Stop][Vnic]ip[%s] port[%u] ref[%d] skip stop.", localIp.GetReadableAddress(), port, count),
808 : HCCL_SUCCESS);
809 6 : CHK_PRT_RET(
810 : count < 0,
811 : HCCL_WARNING(
812 : "[Stop][Vnic]ip[%s] port[%u] devicePhyId_[%u] vnic stopped ERROR, refcount[%d].",
813 : localIp.GetReadableAddress(), port, devicePhyId_, count),
814 : HCCL_SUCCESS);
815 :
816 : // Stop Listen
817 6 : CHK_PRT_RET(
818 : ipSock.nicSocketHandle != nullptr && StopListenSocket(ipSock.nicSocketHandle, port) != HCCL_SUCCESS,
819 : HCCL_ERROR(
820 : "[Stop][Vnic]errNo[0x%016llx] stop vnic socket failed,devid[%u], ip[%s], port[%u]",
821 : HCCL_ERROR_CODE(HCCL_E_TCP_CONNECT), devicePhyId_, localIp.GetReadableAddress(), port),
822 : HCCL_E_INTERNAL);
823 6 : ipSock.listenedPort.erase(port);
824 :
825 : // DeInit Socket
826 6 : if (ipSock.listenedPort.size() == 0 && ipSock.nicSocketHandle != nullptr) {
827 6 : HcclResult ret = hrtRaSocketDeInit(ipSock.nicSocketHandle);
828 6 : CHK_PRT_RET(
829 : ret != HCCL_SUCCESS,
830 : HCCL_ERROR(
831 : "[Stop][Vnic]errNo[0x%016llx] stop vnic socket failed,devid[%u], ip[%s], return[%d]",
832 : HCCL_ERROR_CODE(HCCL_E_TCP_CONNECT), devicePhyId_, localIp.GetReadableAddress(), ret),
833 : HCCL_E_INTERNAL);
834 6 : raResourceInfo_.vnicSocketMap.erase(localIp);
835 : }
836 6 : return HCCL_SUCCESS;
837 : }
838 :
839 1 : HcclResult NetworkManager::StopVnicSocketHandle(const HcclIpAddress& localIp)
840 : {
841 1 : auto it = raResourceInfo_.vnicSocketMap.find(localIp);
842 1 : CHK_PRT_RET(
843 : it == raResourceInfo_.vnicSocketMap.end(),
844 : HCCL_ERROR(
845 : "[NetworkManager][StopVnicSocketHandle]ip[%s] is not found in vnicSocketMap.",
846 : localIp.GetReadableAddress()),
847 : HCCL_E_INTERNAL);
848 :
849 1 : IpSocket& ipSock = it->second;
850 : // 关闭该ip下的全部端口的listen
851 2 : for (auto& port : ipSock.listenedPort) {
852 1 : if (ipSock.nicSocketHandle != nullptr && StopListenSocket(ipSock.nicSocketHandle, port) != HCCL_SUCCESS) {
853 0 : HCCL_ERROR(
854 : "[NetworkManager][StopVnicSocketHandle]errNo[0x%016llx] stop vnic listen failed, "
855 : "devid[%u], ip[%s], port[%u]",
856 : HCCL_ERROR_CODE(HCCL_E_TCP_CONNECT), devicePhyId_, localIp.GetReadableAddress(), port);
857 0 : break;
858 : }
859 1 : IPPortListenRefMapVnicDevice_[localIp][port].Clear();
860 1 : HCCL_INFO(
861 : "[NetworkManager][StopVnicSocketHandle] ip[%s] stop listen port[%u]", localIp.GetReadableAddress(), port);
862 : }
863 1 : ipSock.listenedPort.clear();
864 :
865 : // 销毁socket
866 1 : CHK_PRT_RET(
867 : ipSock.nicSocketHandle != nullptr && hrtRaSocketDeInit(ipSock.nicSocketHandle) != HCCL_SUCCESS,
868 : HCCL_ERROR("[Stop][NicsSocket]VNIC socket deInit not successfully"), HCCL_E_NETWORK);
869 1 : ipSock.nicSocketHandle = nullptr;
870 1 : raResourceInfo_.vnicSocketMap.erase(localIp);
871 :
872 1 : HCCL_INFO(
873 : "[NetworkManager][StopVnicSocketHandle] devid[%u] ip[%s] stop vnic socket success", devicePhyId_,
874 : localIp.GetReadableAddress());
875 1 : return HCCL_SUCCESS;
876 : }
877 :
878 39 : HcclResult NetworkManager::StartNic(const HcclIpAddress& ipAddr, u32& port, bool rdmaFlag, HcclIpAddress ipAddrBackup)
879 : {
880 39 : CHK_PRT_RET(
881 : deviceNicInitRef_.Count() == 0, HCCL_ERROR("[Start][Nic]can't start nic socket before init device nic!"),
882 : HCCL_E_INTERNAL);
883 38 : auto sockInfo = raResourceInfo_.nicSocketMap.find(ipAddr);
884 38 : if (sockInfo == raResourceInfo_.nicSocketMap.end()) {
885 37 : IpSocket tempSock;
886 37 : raResourceInfo_.nicSocketMap.insert(std::make_pair(ipAddr, tempSock)); // 本IP占位
887 37 : HCCL_INFO("device[%u] Start Nic insert ip[%s]", devicePhyId_, ipAddr.GetReadableAddress());
888 37 : }
889 :
890 38 : IpSocket& sock = raResourceInfo_.nicSocketMap[ipAddr];
891 38 : if (sock.listenedPort.size() == 0) {
892 37 : if (sock.nicSocketHandle == nullptr) {
893 37 : HcclResult ret = InitDeviceSocket(devicePhyId_, ipAddr, sock.nicSocketHandle);
894 37 : CHK_PRT_RET(
895 : ret != HCCL_SUCCESS,
896 : HCCL_ERROR(
897 : "[Start][Nic]errNo[0x%016llx] ra nic init socket failed, devid[%u], return[%d]",
898 : HCCL_ERROR_CODE(HCCL_E_TCP_CONNECT), devicePhyId_, ret),
899 : ret);
900 : }
901 : } else {
902 1 : HCCL_INFO("NetworkManager: socket has inited, ipAddr[%s] port[%u], skip.", ipAddr.GetReadableAddress(), port);
903 : }
904 38 : if (sock.nicRdmaHandle == nullptr && rdmaFlag) {
905 : NetworkMode netMode;
906 34 : GetNetworkMode(netMode);
907 34 : CHK_RET(GetNotifyType(notifyType_));
908 68 : HcclResult ret = InitRDMA(
909 34 : devicePhyId_, ipAddr, netMode, notifyType_, sock.nicRdmaHandle, isDisableLiteThread_, false, ipAddrBackup);
910 34 : CHK_PRT_RET(
911 : ret != HCCL_SUCCESS,
912 : HCCL_ERROR(
913 : "[Start][Nic]errNo[0x%016llx] ra nic init rdma failed, devid[%u], return[%d]",
914 : HCCL_ERROR_CODE(HCCL_E_NETWORK), devicePhyId_, ret),
915 : HCCL_E_NETWORK);
916 :
917 : int supportLite;
918 34 : ret = HrtGetRdmaLiteStatus(sock.nicRdmaHandle, &supportLite);
919 34 : CHK_PRT_RET(
920 : ret != HCCL_SUCCESS,
921 : HCCL_ERROR(
922 : "[Get][RdmaLiteStatus]errNo[0x%016llx] get rdma lite status failed, return[%d]",
923 : HCCL_ERROR_CODE(HCCL_E_TCP_CONNECT), ret),
924 : HCCL_E_TCP_CONNECT);
925 34 : isRdmaLiteEn_ = (supportLite == 1);
926 34 : } else {
927 4 : HCCL_INFO(
928 : "[Start][Nic] requesting not init rdma , or already init rdma, "
929 : "rdmaFlag[%u], ipAddr[%s], port[%u], skip.",
930 : rdmaFlag, ipAddr.GetReadableAddress(), port);
931 : }
932 : // 如果port id传入的值无效值0xFFFFFFFF, 不启动监听
933 38 : CHK_PRT_RET(port == MAX_VALUE_U32, HCCL_INFO("[Start][Nic] port id[%u], skip listen socket", port), HCCL_SUCCESS);
934 6 : CHK_PRT_RET(
935 : port > MAX_PORT_ID, HCCL_ERROR("[Start][Nic]invalid port id[%u], max port id[%u]", port, MAX_PORT_ID),
936 : HCCL_E_INTERNAL);
937 6 : if (sock.listenedPort.find(port) != sock.listenedPort.end()) {
938 0 : HCCL_WARNING("port[%u] is already listened.", port);
939 : } else {
940 6 : bool isAutoPort = port == 0;
941 6 : HCCL_INFO("[Start][Nic]trying to listen on ip[%s] port[%u].", ipAddr.GetReadableAddress(), port);
942 6 : CHK_RET(CheckAutoListenVersion(isAutoPort));
943 6 : HcclResult ret = StartListenSocket(sock.nicSocketHandle, port);
944 6 : CHK_PRT_RET(
945 : ret == HCCL_E_UNAVAIL,
946 : HCCL_INFO(
947 : "[Start][Nic]Could not start listening socket for IP [%s] and port [%u].", ipAddr.GetReadableAddress(),
948 : port),
949 : ret);
950 6 : CHK_PRT_RET(
951 : ret != HCCL_SUCCESS,
952 : HCCL_ERROR(
953 : "[Start][Nic]errNo[0x%016llx] ra inner listen start failed, "
954 : "devid[%u], ip[%s], port[%u], return[%d]",
955 : HCCL_ERROR_CODE(HCCL_E_TCP_CONNECT), devicePhyId_, ipAddr.GetReadableAddress(), port, ret),
956 : HCCL_E_TCP_CONNECT);
957 6 : HCCL_INFO("port[%u] listen start OK", port);
958 6 : sock.listenedPort.insert(port);
959 6 : HCCL_RUN_INFO(
960 : "[Start][Nic]Listen on ip[%s], port[%u] success, devPhyId[%u], devLogicId[%u], isAutoPort[%d]",
961 : ipAddr.GetReadableAddress(), port, devicePhyId_, deviceLogicId_, isAutoPort);
962 : }
963 6 : int refCount = IPPortListenRefMapDevice_[ipAddr][port].Ref();
964 6 : HCCL_INFO("Nic ip[%s] port[%u] refcount is [%d]", ipAddr.GetReadableAddress(), port, refCount);
965 6 : return HCCL_SUCCESS;
966 : }
967 :
968 2 : void NetworkManager::SetDisableLiteThread(bool disable) { isDisableLiteThread_ = disable; }
969 :
970 0 : HcclResult NetworkManager::CreateNicSocketHandle(const HcclIpAddress& ipAddr)
971 : {
972 0 : CHK_PRT_RET(
973 : deviceNicInitRef_.Count() == 0,
974 : HCCL_ERROR("[NetworkManager][CreateNicSocketHandle]can't start nic socket before init device nic!"),
975 : HCCL_E_INTERNAL);
976 0 : OccupyIp(ipAddr, raResourceInfo_.nicSocketMap);
977 :
978 0 : IpSocket& sock = raResourceInfo_.nicSocketMap[ipAddr];
979 0 : if (sock.listenedPort.size() == 0) {
980 0 : if (sock.nicSocketHandle == nullptr) {
981 0 : HcclResult ret = InitDeviceSocket(devicePhyId_, ipAddr, sock.nicSocketHandle);
982 0 : CHK_PRT_RET(
983 : ret != HCCL_SUCCESS,
984 : HCCL_ERROR(
985 : "[NetworkManager][CreateNicSocketHandle]errNo[0x%016llx] ra nic init socket failed, devid[%u], "
986 : "return[%d]",
987 : HCCL_ERROR_CODE(HCCL_E_TCP_CONNECT), devicePhyId_, ret),
988 : ret);
989 : }
990 : } else {
991 0 : HCCL_INFO(
992 : "[NetworkManager][CreateNicSocketHandle] socket has inited, ipAddr[%s], skip.",
993 : ipAddr.GetReadableAddress());
994 : }
995 0 : HCCL_INFO(
996 : "[NetworkManager][CreateNicSocketHandle] CreateNicSocketHandle OK, ipAddr[%s].", ipAddr.GetReadableAddress());
997 0 : return HCCL_SUCCESS;
998 : }
999 :
1000 0 : HcclResult NetworkManager::CreateRdmaHandle(
1001 : const HcclIpAddress& ipAddr, bool isBackup, NetworkMode netMode, NotifyTypeT notifyType,
1002 : HcclNetDevDeployment netDevDeployment)
1003 : {
1004 : // device侧如果传入的ip是备份ip,那么给InitRDMA传递的两个主备ip(两个相同)都是备份ip hostrdma没有备份 不受影响
1005 0 : HcclIpAddress ipAddrBackup(ipAddr.GetFamily(), ipAddr.GetBinaryAddress());
1006 0 : if (!isBackup) {
1007 0 : HCCL_INFO("[NetworkManager][CreateRdmaHandle] ipAddr[%s] is not backup", ipAddr.GetReadableAddress());
1008 0 : ipAddrBackup.clear();
1009 : }
1010 0 : CHK_PRT_RET(
1011 : (deviceNicInitRef_.Count() == 0 && hostNicInitRef_.Count() == 0),
1012 : HCCL_ERROR("[NetworkManager][CreateRdmaHandle]can't start nic socket before init device nic!"),
1013 : HCCL_E_INTERNAL);
1014 0 : switch (netDevDeployment) {
1015 0 : case HcclNetDevDeployment::HCCL_NETDEV_DEPLOYMENT_DEVICE: {
1016 : // ip占位
1017 0 : OccupyIp(ipAddr, raResourceInfo_.nicSocketMap);
1018 : // 初始化socket
1019 0 : IpSocket& sock = raResourceInfo_.nicSocketMap[ipAddr];
1020 0 : if (sock.nicRdmaHandle == nullptr) {
1021 0 : notifyType_ = notifyType;
1022 0 : HcclResult ret = InitRDMA(
1023 0 : devicePhyId_, ipAddr, netMode, notifyType_, sock.nicRdmaHandle, false, false, ipAddrBackup);
1024 0 : CHK_PRT_RET(
1025 : ret != HCCL_SUCCESS,
1026 : HCCL_ERROR(
1027 : "[NetworkManager][CreateRdmaHandle]errNo[0x%016llx] ra nic init rdma failed, devid[%u], "
1028 : "return[%d]",
1029 : HCCL_ERROR_CODE(HCCL_E_NETWORK), devicePhyId_, ret),
1030 : HCCL_E_NETWORK);
1031 : } else {
1032 0 : HCCL_INFO(
1033 : "[NetworkManager][CreateRdmaHandle] requesting not init rdma , or already init rdma, "
1034 : "ipAddr[%s], skip.",
1035 : ipAddr.GetReadableAddress());
1036 : }
1037 : // device-roce需要开启rdmalite
1038 0 : RdmaSupportLite(sock.nicRdmaHandle);
1039 0 : break;
1040 : }
1041 0 : case HcclNetDevDeployment::HCCL_NETDEV_DEPLOYMENT_HOST: {
1042 : // 本ip占位
1043 0 : OccupyIp(ipAddr, raResourceInfo_.hostNetSocketMap);
1044 : // 初始化socket
1045 0 : IpSocket& sock = raResourceInfo_.hostNetSocketMap[ipAddr];
1046 0 : if (sock.nicRdmaHandle == nullptr) {
1047 0 : notifyType_ = NOTIFY;
1048 0 : HcclResult ret = InitRDMA(devicePhyId_, ipAddr, netMode, notifyType_, sock.nicRdmaHandle);
1049 0 : CHK_PRT_RET(
1050 : ret != HCCL_SUCCESS,
1051 : HCCL_ERROR(
1052 : "[Start][Nic]errNo[0x%016llx] ra nic init rdma failed, devid[%u], return[%d]",
1053 : HCCL_ERROR_CODE(HCCL_E_NETWORK), devicePhyId_, ret),
1054 : HCCL_E_NETWORK);
1055 : } else {
1056 0 : HCCL_INFO(
1057 : "[NetworkManager][CreateRdmaHandle] requesting not init rdma , or already init rdma, "
1058 : "ipAddr[%s], skip.",
1059 : ipAddr.GetReadableAddress());
1060 : }
1061 : // host rdma的额外占位
1062 0 : auto temp = raResourceInfo_.nicSocketMap.find(ipAddr);
1063 0 : if (temp != raResourceInfo_.nicSocketMap.end()) {
1064 0 : HCCL_ERROR(
1065 : "[NetworkManager][CreateRdmaHandle] ipAddr[%s] has already occupied.", ipAddr.GetReadableAddress());
1066 0 : return HCCL_E_INTERNAL;
1067 : }
1068 0 : raResourceInfo_.nicSocketMap.insert(std::make_pair(ipAddr, sock));
1069 0 : break;
1070 : }
1071 0 : default: {
1072 0 : HCCL_ERROR(
1073 : "[NetworkManager][CreateRdmaHandle]this Deployment [%u] is not supported, please check the "
1074 : "configuration.",
1075 : netDevDeployment);
1076 0 : return HCCL_E_NOT_SUPPORT;
1077 : }
1078 : }
1079 0 : return HCCL_SUCCESS;
1080 0 : }
1081 :
1082 0 : HcclResult NetworkManager::RdmaSupportLite(RdmaHandle rdmaHandle)
1083 : {
1084 : int supportLite;
1085 0 : HcclResult ret = HrtGetRdmaLiteStatus(rdmaHandle, &supportLite);
1086 0 : CHK_PRT_RET(
1087 : ret != HCCL_SUCCESS,
1088 : HCCL_ERROR(
1089 : "[Get][RdmaLiteStatus]errNo[0x%016llx] get rdma lite status failed, return[%d]",
1090 : HCCL_ERROR_CODE(HCCL_E_TCP_CONNECT), ret),
1091 : HCCL_E_TCP_CONNECT);
1092 0 : isRdmaLiteEn_ = (supportLite == 1);
1093 0 : return HCCL_SUCCESS;
1094 : }
1095 :
1096 0 : HcclResult NetworkManager::StopRdmaHandle(const HcclIpAddress& ipAddr, HcclNetDevDeployment netDevDeployment)
1097 : {
1098 : // rdma没有socket 没有listen
1099 :
1100 : // 销毁socket
1101 0 : switch (netDevDeployment) {
1102 0 : case HcclNetDevDeployment::HCCL_NETDEV_DEPLOYMENT_DEVICE: {
1103 0 : auto it = raResourceInfo_.nicSocketMap.find(ipAddr);
1104 0 : if (it == raResourceInfo_.nicSocketMap.end()) {
1105 0 : HCCL_WARNING(
1106 : "[Stop][NicsSocket]ip[%s] not found in nicSocketMap, may already cleaned.",
1107 : ipAddr.GetReadableAddress());
1108 0 : return HCCL_SUCCESS;
1109 : }
1110 0 : IpSocket& ipSock = it->second;
1111 0 : if (ipSock.nicRdmaHandle != nullptr && HrtRaRdmaDeInit(ipSock.nicRdmaHandle, notifyType_) != HCCL_SUCCESS) {
1112 0 : HCCL_ERROR("[Stop][rdma]NIC rdev deInit not successfully, notifyType_[%d]", notifyType_);
1113 0 : return HCCL_E_NETWORK;
1114 : }
1115 0 : ipSock.nicRdmaHandle = nullptr;
1116 : // 如果该ip下既没有tcp也没有rdma那么移除该ip
1117 0 : if (ipSock.nicSocketHandle == nullptr && ipSock.nicRdmaHandle == nullptr) {
1118 0 : raResourceInfo_.nicSocketMap.erase(ipAddr);
1119 : }
1120 0 : break;
1121 : }
1122 0 : case HcclNetDevDeployment::HCCL_NETDEV_DEPLOYMENT_HOST: {
1123 0 : auto it = raResourceInfo_.hostNetSocketMap.find(ipAddr);
1124 0 : CHK_PRT_RET(
1125 : it == raResourceInfo_.hostNetSocketMap.end(),
1126 : HCCL_ERROR("[Stop][rdma]ip[%s] is not found in hostNetSocketMap.", ipAddr.GetReadableAddress()),
1127 : HCCL_E_INTERNAL);
1128 0 : IpSocket& ipSock = it->second;
1129 0 : if (ipSock.nicRdmaHandle != nullptr && HrtRaRdmaDeInit(ipSock.nicRdmaHandle, notifyType_) != HCCL_SUCCESS) {
1130 0 : HCCL_ERROR("[Stop][rdma]NIC rdev deInit not successfully, notifyType_[%d]", notifyType_);
1131 0 : return HCCL_E_NETWORK;
1132 : }
1133 0 : ipSock.nicRdmaHandle = nullptr;
1134 0 : raResourceInfo_.nicSocketMap.erase(ipAddr); // 移除额外的占位
1135 : // 如果该ip下没有tcp也没有rdma那么移除该ip
1136 0 : if (ipSock.nicSocketHandle == nullptr && ipSock.nicRdmaHandle == nullptr) {
1137 0 : raResourceInfo_.hostNetSocketMap.erase(ipAddr);
1138 : }
1139 0 : break;
1140 : }
1141 0 : default: {
1142 0 : HCCL_ERROR(
1143 : "[NetworkManager][StopRdmaHandle]this Deployment [%u] is not supported, please check the "
1144 : "configuration.",
1145 : netDevDeployment);
1146 0 : return HCCL_E_NOT_SUPPORT;
1147 : }
1148 : }
1149 :
1150 0 : HCCL_INFO(
1151 : "[NetworkManager] [StopRdmaHandle] devicePhyId_[%u] StopRdmaHandle success ip [%s]", devicePhyId_,
1152 : ipAddr.GetReadableAddress());
1153 0 : return HCCL_SUCCESS; // stop socket。port 数清零时自动关闭socket
1154 : }
1155 :
1156 2 : HcclResult NetworkManager::StopNicSocketHandle(const HcclIpAddress& ipAddr)
1157 : {
1158 2 : auto it = raResourceInfo_.nicSocketMap.find(ipAddr);
1159 2 : CHK_PRT_RET(
1160 : it == raResourceInfo_.nicSocketMap.end(),
1161 : HCCL_ERROR(
1162 : "[NetworkManager][StopNicSocketHandle]ip[%s] is not found in nicSocketMap.", ipAddr.GetReadableAddress()),
1163 : HCCL_E_INTERNAL);
1164 0 : IpSocket& ipSock = it->second;
1165 0 : HcclResult ret = HCCL_SUCCESS;
1166 : // 关闭该ip下的全部端口的listen
1167 0 : for (auto& port : ipSock.listenedPort) {
1168 0 : ret = StopNicsSocketListen(ipAddr, port);
1169 0 : CHK_PRT_RET(
1170 : ret != HCCL_SUCCESS,
1171 : HCCL_ERROR(
1172 : "[NetworkManager][StopNicSocketHandle]errNo[0x%016llx] stop nic listen failed,devid[%u], ip[%s], "
1173 : "port[%u], return[%d]",
1174 : HCCL_ERROR_CODE(HCCL_E_TCP_CONNECT), devicePhyId_, ipAddr.GetReadableAddress(), port, ret),
1175 : HCCL_E_INTERNAL);
1176 0 : ipSock.listenedPort.erase(port);
1177 0 : IPPortListenRefMapDevice_[ipAddr][port].Clear(); // port计数更新
1178 0 : HCCL_WARNING(
1179 : "[NetworkManager][StopNicSocketHandle] ip [%s] stop listen port [%u] refcount is [%d]",
1180 : ipAddr.GetReadableAddress(), port, IPPortListenRefMapDevice_[ipAddr][port].Count());
1181 : }
1182 :
1183 : // 销毁socket
1184 0 : if (ipSock.nicSocketHandle != nullptr && hrtRaSocketDeInit(ipSock.nicSocketHandle) != HCCL_SUCCESS) {
1185 0 : HCCL_ERROR("[Stop][NicsSocket]NIC socket deInit not successfully");
1186 0 : return HCCL_E_NETWORK;
1187 : }
1188 0 : ipSock.nicSocketHandle = nullptr;
1189 :
1190 : // 如果该ip下既没有tcp也没有rdma那么移除该ip
1191 0 : if (ipSock.nicSocketHandle == nullptr && ipSock.nicRdmaHandle == nullptr) {
1192 0 : raResourceInfo_.nicSocketMap.erase(ipAddr);
1193 : }
1194 :
1195 0 : return HCCL_SUCCESS;
1196 : }
1197 :
1198 36 : HcclResult NetworkManager::StopNic(const HcclIpAddress& ipAddr, u32 port)
1199 : {
1200 36 : auto it = raResourceInfo_.nicSocketMap.find(ipAddr);
1201 36 : CHK_PRT_RET(
1202 : it == raResourceInfo_.nicSocketMap.end(),
1203 : HCCL_ERROR("[Stop][Nic]ip[%s] is not found in nicSocketMap, port[%u].", ipAddr.GetReadableAddress(), port),
1204 : HCCL_E_INTERNAL);
1205 36 : IpSocket& ipSock = it->second;
1206 36 : HcclResult ret = HCCL_SUCCESS;
1207 : // 传入端口号为无效值0xFFFFFFFF，未启动监听，不需要stop listen
1208 36 : if (port != MAX_VALUE_U32) {
1209 4 : CHK_PRT_RET(
1210 : IPPortListenRefMapDevice_[ipAddr][port].Unref() > 0,
1211 : HCCL_INFO(
1212 : "[Stop][Nic]ip[%s] port[%u] ref[%d] skip stop.", ipAddr.GetReadableAddress(), port,
1213 : IPPortListenRefMapDevice_[ipAddr][port].Count()),
1214 : HCCL_SUCCESS);
1215 4 : ret = StopNicsSocketListen(ipAddr, port);
1216 4 : CHK_PRT_RET(
1217 : ret != HCCL_SUCCESS,
1218 : HCCL_ERROR(
1219 : "[Stop][Nic]errNo[0x%016llx] stop nic socket failed,devid[%u], ip[%s], port[%u], return[%d]",
1220 : HCCL_ERROR_CODE(HCCL_E_TCP_CONNECT), devicePhyId_, ipAddr.GetReadableAddress(), port, ret),
1221 : HCCL_E_INTERNAL);
1222 4 : ipSock.listenedPort.erase(port);
1223 : }
1224 :
1225 36 : if (ipSock.listenedPort.size() == 0) {
1226 35 : ret = StopNicsSocket(ipAddr);
1227 35 : CHK_PRT_RET(
1228 : ret != HCCL_SUCCESS,
1229 : HCCL_ERROR(
1230 : "[Stop][Nic]errNo[0x%016llx] stop nic socket failed,devid[%u], ip[%s], return[%d]",
1231 : HCCL_ERROR_CODE(HCCL_E_TCP_CONNECT), devicePhyId_, ipAddr.GetReadableAddress(), ret),
1232 : HCCL_E_INTERNAL);
1233 33 : raResourceInfo_.nicSocketMap.erase(ipAddr);
1234 : }
1235 34 : return HCCL_SUCCESS;
1236 : }
1237 5 : HcclResult NetworkManager::StopAllDeviceNicSockets()
1238 : {
1239 5 : HcclResult ret = HCCL_SUCCESS;
1240 11 : for (auto& itSocket : raResourceInfo_.nicSocketMap) {
1241 6 : std::set<u32> listenedPorts = itSocket.second.listenedPort;
1242 7 : for (auto itPort : listenedPorts) {
1243 3 : ret = StopNicsSocketListen(itSocket.first, itPort);
1244 3 : if (ret != HCCL_SUCCESS) {
1245 2 : HCCL_ERROR(
1246 : "[Stop][AllDeviceNicSockets]errNo[0x%016llx] stop nic socket failed,devid[%u],ip[%s], "
1247 : "port[%u],return[%d]",
1248 : HCCL_ERROR_CODE(HCCL_E_TCP_CONNECT), devicePhyId_, itSocket.first.GetReadableAddress(), itPort,
1249 : ret);
1250 2 : itSocket.second.listenedPort.erase(itPort);
1251 2 : break;
1252 : }
1253 : }
1254 6 : ret = StopNicsSocket(itSocket.first);
1255 6 : if (ret != HCCL_SUCCESS) {
1256 2 : HCCL_ERROR(
1257 : "[Stop][AllDeviceNicSockets]errNo[0x%016llx] stop nic socket failed,devid[%u],ip[%s],return[%d]",
1258 : HCCL_ERROR_CODE(HCCL_E_TCP_CONNECT), devicePhyId_, itSocket.first.GetReadableAddress(), ret);
1259 2 : if (itSocket.second.nicRdmaHandle != nullptr) {
1260 0 : HCCL_ERROR("[StopVnicSocketHandle] itSocket.second.nicRdmaHandle is not nullptr.");
1261 0 : (void)HrtRaRdmaDeInit(itSocket.second.nicRdmaHandle, notifyType_);
1262 0 : itSocket.second.nicRdmaHandle = nullptr;
1263 : }
1264 2 : if (itSocket.second.nicSocketHandle != nullptr) {
1265 2 : HCCL_ERROR("[StopVnicSocketHandle] itSocket.second.nicSocketHandle is not nullptr.");
1266 2 : (void)HrtRaRdmaDeInit(itSocket.second.nicSocketHandle, notifyType_);
1267 2 : itSocket.second.nicSocketHandle = nullptr;
1268 : }
1269 : }
1270 6 : }
1271 :
1272 5 : raResourceInfo_.nicSocketMap.clear();
1273 5 : return HCCL_SUCCESS;
1274 : }
1275 :
1276 7 : HcclResult NetworkManager::StopAllDeviceVnicSockets()
1277 : {
1278 7 : HcclResult ret = HCCL_SUCCESS;
1279 :
1280 14 : for (auto itSocket : raResourceInfo_.vnicSocketMap) {
1281 7 : HCCL_WARNING(
1282 : "vnicSocketMap ip[%s] is not released when NetworkManager Destroy, force releasing",
1283 : itSocket.first.GetReadableAddress());
1284 7 : if (itSocket.second.nicSocketHandle != nullptr) {
1285 14 : for (auto itPort : itSocket.second.listenedPort) {
1286 7 : ret = StopListenSocket(itSocket.second.nicSocketHandle, itPort);
1287 7 : if (ret != HCCL_SUCCESS) {
1288 0 : HCCL_ERROR(
1289 : "[Stop][AllDeviceVnicSockets]errNo[0x%016llx] stop vnic socket listen failed, "
1290 : "devid[%u], ip[%s], port[%u], return[%d]",
1291 : HCCL_ERROR_CODE(HCCL_E_TCP_CONNECT), devicePhyId_, itSocket.first.GetReadableAddress(), itPort,
1292 : ret);
1293 : }
1294 7 : IPPortListenRefMapVnicDevice_[itSocket.first][itPort].Clear();
1295 : }
1296 7 : ret = hrtRaSocketDeInit(itSocket.second.nicSocketHandle);
1297 7 : if (ret != HCCL_SUCCESS) {
1298 1 : HCCL_ERROR(
1299 : "[Stop][AllDeviceVnicSockets]errNo[0x%016llx] deinit vnic socket failed, "
1300 : "devid[%u], ip[%s], return[%d]",
1301 : HCCL_ERROR_CODE(HCCL_E_TCP_CONNECT), devicePhyId_, itSocket.first.GetReadableAddress(), ret);
1302 : }
1303 7 : itSocket.second.nicSocketHandle = nullptr;
1304 : }
1305 7 : }
1306 7 : raResourceInfo_.vnicSocketMap.clear();
1307 :
1308 7 : return HCCL_SUCCESS;
1309 : }
1310 :
1311 34 : HcclResult NetworkManager::InitRDMA(
1312 : u32 devicePhysicID, const HcclIpAddress& ipAddr, NetworkMode netMode, NotifyTypeT notifyType,
1313 : RdmaHandle& rdmaHandle, bool disabledLiteThread, bool enable910ALite, HcclIpAddress ipAddrBackup)
1314 : {
1315 34 : struct rdev nicRdevInfo = {};
1316 34 : nicRdevInfo.phyId = devicePhysicID;
1317 34 : nicRdevInfo.family = ipAddr.GetFamily();
1318 34 : nicRdevInfo.localIp.addr = ipAddr.GetBinaryAddress().addr;
1319 34 : nicRdevInfo.localIp.addr6 = ipAddr.GetBinaryAddress().addr6;
1320 :
1321 34 : struct RdevInitInfo init_info = {};
1322 34 : init_info.mode = netMode;
1323 34 : init_info.notifyType = notifyType;
1324 34 : init_info.disabledLiteThread = disabledLiteThread;
1325 34 : init_info.enabled910aLite = enable910ALite;
1326 34 : init_info.enabled2mbLite = GetExternalInputRdmaFastPost();
1327 :
1328 34 : HcclResult ret = HCCL_SUCCESS;
1329 34 : HCCL_DEBUG("isRaInitRepeated_[%d]", isRaInitRepeated_);
1330 34 : if (isRaInitRepeated_) {
1331 : // 重复RaInit时，调用此接口获取相同的rdmaHandle，防止重新生成
1332 0 : ret = HrtRaRdmaGetHandle(devicePhysicID, rdmaHandle);
1333 : } else {
1334 34 : if (!ipAddrBackup.IsInvalid()) {
1335 : struct rdev nicRdevInfoback;
1336 0 : CHK_RET(hrtGetPairDevicePhyId(devicePhysicID, nicRdevInfoback.phyId));
1337 0 : nicRdevInfoback.family = ipAddrBackup.GetFamily();
1338 0 : nicRdevInfoback.localIp.addr = ipAddrBackup.GetBinaryAddress().addr;
1339 0 : nicRdevInfoback.localIp.addr6 = ipAddrBackup.GetBinaryAddress().addr6;
1340 0 : HCCL_DEBUG(
1341 : "[%s]backup rdev info: ipAddr[%s], ipAddrBackup[%s]", __func__, ipAddr.GetReadableIP(),
1342 : ipAddrBackup.GetReadableIP());
1343 0 : ret = HrtRdmaInitWithBackupAttr(init_info, nicRdevInfo, nicRdevInfoback, rdmaHandle);
1344 : } else {
1345 34 : ret = HrtRaRdmaInitWithAttr(init_info, nicRdevInfo, rdmaHandle);
1346 : }
1347 : }
1348 :
1349 34 : CHK_PRT_RET(
1350 : ret != HCCL_SUCCESS,
1351 : HCCL_ERROR(
1352 : "[Init][RDMA]errNo[0x%016llx] ra rdma init failed, devid[%u] ip[%s], notifyType[%d], return[%d]",
1353 : HCCL_ERROR_CODE(HCCL_E_TCP_CONNECT), devicePhysicID, ipAddr.GetReadableAddress(), notifyType, ret),
1354 : HCCL_E_TCP_CONNECT);
1355 34 : HCCL_INFO(
1356 : "devicePhyId[%u], ip[%s] disabledLiteThread[%u] enabled910aLite[%u] rdmaHandle[%p] rdma init OK",
1357 : devicePhysicID, ipAddr.GetReadableAddress(), disabledLiteThread, enable910ALite, rdmaHandle);
1358 :
1359 34 : return HCCL_SUCCESS;
1360 : }
1361 :
1362 152 : bool NetworkManager::GetRdmaLiteStatus() { return isRdmaLiteEn_; }
1363 :
1364 34 : HcclResult NetworkManager::GetNotifyType(NotifyTypeT& notifyType) const
1365 : {
1366 34 : DevType deviceType = DevType::DEV_TYPE_COUNT;
1367 34 : CHK_RET(hrtGetDeviceType(deviceType));
1368 34 : if (deviceType == DevType::DEV_TYPE_910 || deviceType == DevType::DEV_TYPE_910B
1369 0 : || deviceType == DevType::DEV_TYPE_910_93) {
1370 34 : notifyType = NOTIFY;
1371 0 : } else if (deviceType == DevType::DEV_TYPE_310P3 || deviceType == DevType::DEV_TYPE_310P1) {
1372 0 : notifyType = EVENTID;
1373 : } else {
1374 0 : HCCL_ERROR("[Init][DeviceRDMA]devType[%d] is invalid", deviceType);
1375 0 : return HCCL_E_PARA;
1376 : }
1377 34 : return HCCL_SUCCESS;
1378 : }
1379 :
1380 85 : void NetworkManager::GetNetworkMode(NetworkMode& netMode) const
1381 : {
1382 85 : if (Is310PDevice()) {
1383 0 : netMode = NETWORK_PEER_ONLINE;
1384 : } else {
1385 85 : netMode = NETWORK_OFFLINE;
1386 : }
1387 85 : }
1388 :
1389 51 : HcclResult NetworkManager::InitDeviceSocket(u32 devicePhysicID, const HcclIpAddress& ipAddr, SocketHandle& socketHandle)
1390 : {
1391 51 : struct rdev nicRdevInfo = {};
1392 51 : nicRdevInfo.phyId = devicePhysicID;
1393 51 : nicRdevInfo.family = ipAddr.GetFamily();
1394 51 : nicRdevInfo.localIp.addr = ipAddr.GetBinaryAddress().addr;
1395 51 : nicRdevInfo.localIp.addr6 = ipAddr.GetBinaryAddress().addr6;
1396 :
1397 : NetworkMode netMode;
1398 51 : GetNetworkMode(netMode);
1399 :
1400 51 : HcclResult ret = hrtRaSocketInit(netMode, nicRdevInfo, socketHandle);
1401 51 : CHK_PRT_RET(
1402 : ret != HCCL_SUCCESS,
1403 : HCCL_ERROR(
1404 : "[Init][DeviceSocket]ra socket init failed, network mode[%d] devid[%u] ip[%s], return[%d]", netMode,
1405 : devicePhysicID, ipAddr.GetReadableAddress(), ret),
1406 : HCCL_E_TCP_CONNECT);
1407 51 : HCCL_INFO("devicePhyId[%u], ip[%s] socket init OK", devicePhysicID, ipAddr.GetReadableAddress());
1408 :
1409 51 : return HCCL_SUCCESS;
1410 : }
1411 :
1412 : HcclResult
1413 22 : NetworkManager::StartHostNetAndListen(const HcclIpAddress& ipAddr, SocketHandle& socketHandle, u32& port, bool rdmaFlag)
1414 : {
1415 22 : CHK_PRT_RET(
1416 : (hostNicInitRef_.Count() == 0),
1417 : HCCL_ERROR("[Start][HostNetAndListen]cannot start nic socket before host nic inited!"), HCCL_E_INTERNAL);
1418 22 : auto sockInfo = raResourceInfo_.hostNetSocketMap.find(ipAddr);
1419 22 : if (sockInfo == raResourceInfo_.hostNetSocketMap.end()) {
1420 2 : IpSocket tempSock;
1421 2 : raResourceInfo_.hostNetSocketMap.insert(std::make_pair(ipAddr, tempSock)); // 本IP占位
1422 2 : HCCL_INFO("device[%u] Start host nic insert Ip[%s]", devicePhyId_, ipAddr.GetReadableAddress());
1423 2 : }
1424 :
1425 22 : HcclResult ret = HCCL_SUCCESS;
1426 22 : IpSocket& sock = raResourceInfo_.hostNetSocketMap[ipAddr];
1427 22 : if (sock.nicSocketHandle == nullptr) {
1428 2 : ret = InitHostSocket(ipAddr, sock.nicSocketHandle);
1429 2 : CHK_PRT_RET(
1430 : ret != HCCL_SUCCESS,
1431 : HCCL_ERROR(
1432 : "[Start][HostNetAndListen]start host socket failed, devid[%u], ip[%s] return[%d]", devicePhyId_,
1433 : ipAddr.GetReadableAddress(), ret),
1434 : ret);
1435 : }
1436 22 : if (sock.nicRdmaHandle == nullptr && rdmaFlag) {
1437 0 : notifyType_ = EVENTID;
1438 0 : ret = InitRDMA(devicePhyId_, ipAddr, NETWORK_PEER_ONLINE, notifyType_, sock.nicRdmaHandle);
1439 0 : CHK_PRT_RET(
1440 : ret != HCCL_SUCCESS,
1441 : HCCL_ERROR(
1442 : "[Start][Nic]errNo[0x%016llx] ra nic init rdma failed, devid[%u], return[%d]",
1443 : HCCL_ERROR_CODE(HCCL_E_NETWORK), devicePhyId_, ret),
1444 : HCCL_E_NETWORK);
1445 : }
1446 22 : if (IPPortListenRefMapHost_[ipAddr][port].Count() == 0 && sock.listenedPort.find(port) == sock.listenedPort.end()) {
1447 21 : bool isAutoPort = port == 0;
1448 21 : HCCL_INFO("[Start][HostNetAndListen]trying to listen on ip[%s] port[%u].", ipAddr.GetReadableAddress(), port);
1449 21 : ret = StartListenSocket(sock.nicSocketHandle, port);
1450 21 : CHK_PRT_RET(
1451 : ret == HCCL_E_UNAVAIL,
1452 : HCCL_INFO(
1453 : "[Start][HostNetAndListen]Could not start listening socket for IP [%s] and port [%u].",
1454 : ipAddr.GetReadableAddress(), port),
1455 : ret);
1456 21 : CHK_PRT_RET(
1457 : ret != HCCL_SUCCESS,
1458 : HCCL_ERROR(
1459 : "[Start][HostNetAndListen]start host socket failed, devid[%u], ip[%s], port[%u], return[%d]",
1460 : devicePhyId_, ipAddr.GetReadableAddress(), port, ret),
1461 : ret);
1462 21 : sock.listenedPort.insert(port);
1463 21 : HCCL_RUN_INFO(
1464 : "[Start][HostNetAndListen]Listen on ip[%s], port[%u] success, "
1465 : "devPhyId[%u], devLogicId[%u], isAutoPort[%d]",
1466 : ipAddr.GetReadableAddress(), port, devicePhyId_, deviceLogicId_, isAutoPort);
1467 : }
1468 22 : int refCount = IPPortListenRefMapHost_[ipAddr][port].Ref();
1469 22 : HCCL_INFO("host ip[%s] port[%u] refcount is [%d]", ipAddr.GetReadableAddress(), port, refCount);
1470 :
1471 22 : socketHandle = sock.nicSocketHandle;
1472 22 : raResourceInfo_.nicSocketMap.insert(std::make_pair(ipAddr, sock));
1473 22 : hostNicSocketClientRef_[ipAddr].Ref();
1474 22 : HCCL_INFO("HostNet, ip[%s] port[%u] socket init OK", ipAddr.GetReadableAddress(), port);
1475 22 : return HCCL_SUCCESS;
1476 : }
1477 :
1478 45 : HcclResult NetworkManager::CheckSocketInfo(const SocketHandle socketHandle, const HcclIpAddress& ipAddr, u32 port) const
1479 : {
1480 45 : auto iterIP = raResourceInfo_.hostNetSocketMap.find(ipAddr);
1481 45 : CHK_PRT_RET(
1482 : (iterIP == raResourceInfo_.hostNetSocketMap.end()),
1483 : HCCL_ERROR(
1484 : "[Check][SocketInfo]ip[%s] port[%u] has not been started. ip is invalid.", ipAddr.GetReadableAddress(),
1485 : port),
1486 : HCCL_E_INTERNAL);
1487 :
1488 45 : CHK_PRT_RET(
1489 : (socketHandle != iterIP->second.nicSocketHandle),
1490 : HCCL_ERROR(
1491 : "[Check][SocketInfo]ip[%s] port[%u] has not been started. socketHandle is invalid",
1492 : ipAddr.GetReadableAddress(), port),
1493 : HCCL_E_INTERNAL);
1494 :
1495 45 : if (port != NO_LISTEN_PORT) {
1496 20 : CHK_PRT_RET(
1497 : (iterIP->second.listenedPort.count(port) == 0),
1498 : HCCL_ERROR(
1499 : "[Check][SocketInfo]ip[%s] port[%u] has not been started. port is invalid", ipAddr.GetReadableAddress(),
1500 : port),
1501 : HCCL_E_INTERNAL);
1502 : }
1503 45 : return HCCL_SUCCESS;
1504 : }
1505 :
1506 20 : HcclResult NetworkManager::StopHostNetAndListen(SocketHandle socketHandle, const HcclIpAddress& ipAddr, u32 port)
1507 : {
1508 20 : std::unique_lock<std::mutex> lock(raLock_);
1509 20 : CHK_PRT_RET(
1510 : (hostNicInitRef_.Count() == 0),
1511 : HCCL_ERROR("[Stop][HostNetAndListen]cannot start nic socket before host nic inited!"), HCCL_E_INTERNAL);
1512 :
1513 20 : CHK_RET(CheckSocketInfo(socketHandle, ipAddr, port));
1514 :
1515 20 : if (IPPortListenRefMapHost_[ipAddr][port].Unref() == 0) {
1516 20 : CHK_RET(StopListenSocket(socketHandle, port)); /* 当前只拉起一个server */
1517 :
1518 20 : HCCL_INFO("ip[%s] port[%u] stop success.", ipAddr.GetReadableAddress(), port);
1519 20 : raResourceInfo_.hostNetSocketMap[ipAddr].listenedPort.erase(port);
1520 20 : raResourceInfo_.nicSocketMap[ipAddr].listenedPort.erase(port);
1521 : } else {
1522 0 : HCCL_INFO(
1523 : "ip[%s] port[%u] skip stop. ref[%d].", ipAddr.GetReadableAddress(), port,
1524 : IPPortListenRefMapHost_[ipAddr][port].Count());
1525 : }
1526 :
1527 20 : if (hostNicSocketClientRef_[ipAddr].Unref() == 0
1528 20 : && raResourceInfo_.hostNetSocketMap[ipAddr].listenedPort.size() == 0) {
1529 0 : CHK_RET(hrtRaSocketDeInit(socketHandle));
1530 0 : raResourceInfo_.hostNetSocketMap.erase(ipAddr);
1531 0 : raResourceInfo_.nicSocketMap.erase(ipAddr);
1532 0 : HCCL_INFO("ip[%s] port[%u] deinit success.", ipAddr.GetReadableAddress(), port);
1533 : }
1534 :
1535 20 : return HCCL_SUCCESS;
1536 20 : }
1537 2 : HcclResult NetworkManager::StopAllHostNicSockets()
1538 : {
1539 2 : HcclResult ret = HCCL_SUCCESS;
1540 4 : for (auto itSocket : raResourceInfo_.hostNetSocketMap) {
1541 3 : for (auto itPort : itSocket.second.listenedPort) {
1542 1 : ret = (StopListenSocket(itSocket.second.nicSocketHandle, itPort));
1543 1 : if (ret != HCCL_SUCCESS) {
1544 1 : HCCL_ERROR(
1545 : "[Stop][StopAllHostNicSockets]errNo[0x%016llx] stop listen socket failed,"
1546 : "devid[%u], ip[%s], port[%u], return[%d].",
1547 : HCCL_ERROR_CODE(HCCL_E_TCP_CONNECT), devicePhyId_, itSocket.first.GetReadableAddress(), itPort,
1548 : ret);
1549 : } else {
1550 0 : HCCL_INFO("ip[%s] port[%u] stop success.", itSocket.first.GetReadableAddress(), itPort);
1551 : }
1552 : }
1553 2 : ret = hrtRaSocketDeInit(itSocket.second.nicSocketHandle);
1554 2 : if (ret != HCCL_SUCCESS) {
1555 1 : HCCL_ERROR(
1556 : "[Stop][StopAllHostNicSockets]errNo[0x%016llx] deinit socket failed, devid[%u], ip[%s], return[%d]",
1557 : HCCL_ERROR_CODE(HCCL_E_TCP_CONNECT), devicePhyId_, itSocket.first.GetReadableAddress(), ret);
1558 : } else {
1559 1 : HCCL_INFO("ip[%s] deinit success.", itSocket.first.GetReadableAddress());
1560 : }
1561 2 : hostNicSocketClientRef_[itSocket.first].Clear();
1562 2 : }
1563 :
1564 2 : raResourceInfo_.hostNetSocketMap.clear();
1565 2 : return HCCL_SUCCESS;
1566 : }
1567 : /* destroy network resource */
1568 1673 : HcclResult NetworkManager::Destroy()
1569 : {
1570 : /* 停止nic ra的监听 */
1571 1673 : if (raResourceInfo_.nicSocketMap.size() != 0) {
1572 7 : for (auto it : raResourceInfo_.nicSocketMap) {
1573 4 : HCCL_WARNING("nicSocketMap[%s] is not stopped when NetworkManager Destroy", it.first.GetReadableAddress());
1574 4 : }
1575 : // StartHostNetAndListen等函数中，同一地址在nicSocketMap和hostNetSocketMap内同时插入
1576 : // 此处StopAllDeviceNicSockets()销毁nicSocketMap后，StopAllHostNicSockets内会发生重复销毁导致core
1577 : // 为了避免此种情况，同时尽量减少对既有函数的修改扩散影响，此处同步对hostNetSocketMap进行清理
1578 : // 最终修改方案需要重构本类，解除两个MAP的耦合
1579 :
1580 7 : for (auto& it : raResourceInfo_.nicSocketMap) {
1581 4 : raResourceInfo_.hostNetSocketMap.erase(it.first); // key不存在则不会删除
1582 : }
1583 3 : StopAllDeviceNicSockets();
1584 : }
1585 :
1586 : /* 停止vnic ra的监听 */
1587 1673 : if (raResourceInfo_.vnicSocketMap.size() != 0) {
1588 5 : StopAllDeviceVnicSockets();
1589 : }
1590 :
1591 : /* 停止host nic ra的监听 */
1592 1673 : if (raResourceInfo_.hostNetSocketMap.size() != 0) {
1593 0 : for (auto it : raResourceInfo_.hostNetSocketMap) {
1594 0 : HCCL_WARNING(
1595 : "hostNicSocketMap[%s] is not stopped when NetworkManager Destroy", it.first.GetReadableAddress());
1596 0 : }
1597 0 : StopAllHostNicSockets();
1598 : }
1599 :
1600 : /* 释放ra资源 重复RaInit时，不再调用内部DeInit */
1601 1673 : HCCL_DEBUG("Destroy call HrtRaDeInit.");
1602 1673 : if (deviceNicInitRef_.Count() != 0 && !isRaDeInit_) {
1603 5 : HCCL_WARNING("device Nic is not deinit when NetworkManager Destroy. ref[%d]", deviceNicInitRef_.Count());
1604 5 : struct RaInitConfig config = {DEFAULT_INIT_PHY_ID, DEFAULT_INIT_NIC_POS, DEFAULT_HDC_TYPE, isEnableHdcAsync_};
1605 5 : GetDeviceRaInitConfig(config);
1606 5 : config.nicPosition = static_cast<u32>(NICDeployment::NIC_DEPLOYMENT_DEVICE);
1607 5 : if (HrtRaDeInit(&config) != HCCL_SUCCESS) {
1608 0 : HCCL_ERROR("ra deinit failed. para: nicdeploy[%u], phyId[%u]", config.nicPosition, config.phyId);
1609 : }
1610 :
1611 5 : isRaDeInit_ = true;
1612 5 : deviceNicInitRef_.Clear();
1613 : }
1614 1673 : if (hostNicInitRef_.Count() != 0) {
1615 6 : HCCL_WARNING("host Nic is not deinit when NetworkManager Destroy. ref[%d]", hostNicInitRef_.Count());
1616 6 : struct RaInitConfig config = {DEFAULT_INIT_PHY_ID, DEFAULT_INIT_NIC_POS, DEFAULT_HDC_TYPE, isEnableHdcAsync_};
1617 6 : config.nicPosition = static_cast<u32>(NICDeployment::NIC_DEPLOYMENT_HOST);
1618 6 : config.phyId = devicePhyId_;
1619 6 : if (HrtRaDeInit(&config) != HCCL_SUCCESS) {
1620 1 : HCCL_ERROR("ra deinit failed. para: nicdeploy[%u], phyId[%u]", config.nicPosition, config.phyId);
1621 : }
1622 6 : hostNicInitRef_.Clear();
1623 : }
1624 1673 : HCCL_INFO("destroy all nic/vnic.");
1625 1673 : return HCCL_SUCCESS;
1626 : }
1627 :
1628 6 : HcclResult NetworkManager::StopNicsSocketListen(const HcclIpAddress& ipAddr, u32 port)
1629 : {
1630 6 : auto it = raResourceInfo_.nicSocketMap.find(ipAddr);
1631 6 : CHK_PRT_RET(
1632 : it == raResourceInfo_.nicSocketMap.end(),
1633 : HCCL_ERROR(
1634 : "[Stop][NicsSocketPort]ip[%s] port[%u] is not found in nicSocketMap.", ipAddr.GetReadableAddress(), port),
1635 : HCCL_E_INTERNAL);
1636 6 : IpSocket& ipSock = it->second;
1637 6 : bool portFound = false;
1638 13 : for (auto itPort : ipSock.listenedPort) {
1639 7 : if (itPort == port) {
1640 6 : portFound = true;
1641 : }
1642 : }
1643 6 : CHK_PRT_RET(
1644 : !portFound,
1645 : HCCL_ERROR("[Stop][NicsSocketPort]PORT(ip[%s] port[%u]) is not found.", ipAddr.GetReadableAddress(), port),
1646 : HCCL_E_INTERNAL);
1647 6 : if (ipSock.nicSocketHandle != nullptr && StopListenSocket(ipSock.nicSocketHandle, port)) {
1648 1 : HCCL_ERROR("[Stop][NicsSocketPort]NIC socket listen is not stopped successfully");
1649 1 : return HCCL_E_NETWORK;
1650 : }
1651 5 : return HCCL_SUCCESS;
1652 : }
1653 :
1654 40 : HcclResult NetworkManager::StopNicsSocket(const HcclIpAddress& ipAddr)
1655 : {
1656 40 : auto it = raResourceInfo_.nicSocketMap.find(ipAddr);
1657 40 : CHK_PRT_RET(
1658 : it == raResourceInfo_.nicSocketMap.end(),
1659 : HCCL_ERROR("[Stop][NicsSocket]ip[%s] is not found in nicSocketMap.", ipAddr.GetReadableAddress()),
1660 : HCCL_E_INTERNAL);
1661 40 : IpSocket& ipSock = it->second;
1662 40 : if (ipSock.nicRdmaHandle != nullptr && HrtRaRdmaDeInit(ipSock.nicRdmaHandle, notifyType_) != HCCL_SUCCESS) {
1663 2 : HCCL_ERROR("[Stop][NicsSocket]NIC rdev deInit not successfully, notifyType_[%d]", notifyType_);
1664 2 : return HCCL_E_NETWORK;
1665 : }
1666 38 : ipSock.nicRdmaHandle = nullptr;
1667 38 : if (ipSock.nicSocketHandle != nullptr && hrtRaSocketDeInit(ipSock.nicSocketHandle) != HCCL_SUCCESS) {
1668 1 : HCCL_ERROR("[Stop][NicsSocket]NIC socket deInit not successfully");
1669 1 : return HCCL_E_NETWORK;
1670 : }
1671 37 : ipSock.nicSocketHandle = nullptr;
1672 37 : return HCCL_SUCCESS;
1673 : }
1674 :
1675 : HcclResult
1676 0 : NetworkManager::InitRdmaHandle(u32 devId, const HcclIpAddress& ipAddr, bool disabledLiteThread, bool enable910ALite)
1677 : {
1678 0 : if (raResourceInfo_.nicSocketMap[ipAddr].nicRdmaHandle != nullptr
1679 0 : && raResourceInfo_.hostNetSocketMap[ipAddr].nicRdmaHandle != nullptr) {
1680 0 : HCCL_INFO("NetworkManager: RdmaInit already nic");
1681 0 : return HCCL_SUCCESS;
1682 : }
1683 :
1684 0 : u32 devicePhyId = ((static_cast<s32>(devId) == HOST_DEVICE_ID) ? 0 : devId);
1685 0 : RdmaHandle rdmaHandle = nullptr;
1686 : // 模式和notify类型按照是否为hdc模式进行赋值
1687 0 : NetworkMode initRdmaMode = (isHostUseDevNic_) ? NETWORK_OFFLINE : NETWORK_PEER_ONLINE;
1688 0 : NotifyTypeT notifyType = (isHostUseDevNic_) ? NotifyTypeT::NOTIFY : NotifyTypeT::NO_USE;
1689 : HcclResult ret
1690 0 : = InitRDMA(devicePhyId, ipAddr, initRdmaMode, notifyType, rdmaHandle, disabledLiteThread, enable910ALite);
1691 0 : CHK_PRT_RET(
1692 : ret != HCCL_SUCCESS,
1693 : HCCL_ERROR(
1694 : "[Init][RDMA]errNo[0x%016llx] ra rdma init failed, ip[%s], return[%d]", HCCL_ERROR_CODE(HCCL_E_TCP_CONNECT),
1695 : ipAddr.GetReadableAddress(), ret),
1696 : HCCL_E_TCP_CONNECT);
1697 0 : HCCL_INFO("ip[%s] rdma init OK", ipAddr.GetReadableAddress());
1698 0 : CHK_PTR_NULL(rdmaHandle);
1699 0 : raResourceInfo_.nicSocketMap[ipAddr].nicRdmaHandle = rdmaHandle;
1700 0 : raResourceInfo_.hostNetSocketMap[ipAddr].nicRdmaHandle = rdmaHandle;
1701 0 : return HCCL_SUCCESS;
1702 : }
1703 :
1704 260 : HcclResult NetworkManager::GetRaResourceInfo(RaResourceInfo& raResourceInfo)
1705 : {
1706 260 : raResourceInfo = raResourceInfo_;
1707 256 : return HCCL_SUCCESS;
1708 : }
1709 :
1710 1 : HcclResult NetworkManager::PsWorkerRaInit(
1711 : u32 devId, const HcclIpAddress& ipAddr, u32 port, bool isHostUseDevNic, bool remoteIsHdc, bool isBoardVersion)
1712 : {
1713 1 : HCCL_INFO("PsWorkerRaInit, devicePhyId[%u], deviceLogicId_[%d]", devicePhyId_, deviceLogicId_);
1714 : // 引用计数
1715 1 : u32 devicePhyId = ((static_cast<s32>(devId) == HOST_DEVICE_ID) ? 0 : devId);
1716 1 : std::string ipAddrStr(ipAddr.GetReadableAddress());
1717 1 : if (ipAddrStr == "127.0.0.1") {
1718 0 : hostNicInitRef_.Ref();
1719 0 : HCCL_INFO("hostNicInitRef_[%d]", hostNicInitRef_.Count());
1720 : } else {
1721 1 : bool fistUsed{false};
1722 1 : deviceNicInitRef_.Ref();
1723 1 : CHK_RET(hrtRaIsFirstUsed(devicePhyId, fistUsed));
1724 1 : HCCL_INFO(
1725 : "deviceNicInitRef_[%d] firstUsed[%u] devicePhyId[%u]", deviceNicInitRef_.Count(), fistUsed, devicePhyId);
1726 1 : if (deviceNicInitRef_.Count() == 1 && !fistUsed) {
1727 0 : isRaInitRepeated_ = true;
1728 1 : } else if (!fistUsed) {
1729 0 : HCCL_INFO("[NetworkManager] PsWorkerRa is not firstUsed");
1730 0 : return HCCL_SUCCESS;
1731 : }
1732 : }
1733 :
1734 1 : CHK_PRT_RET(
1735 : (raResourceInfo_.nicSocketMap.count(ipAddr) != 0),
1736 : HCCL_INFO("NetworkManager: PsWorkerRa already Init, ipAddr[%s]", ipAddr.GetReadableAddress()), HCCL_SUCCESS);
1737 1 : isHostUseDevNic_ = isHostUseDevNic;
1738 :
1739 : // hdc模式下需要先拉起device上的hccp进程
1740 1 : if (isHostUseDevNic_) {
1741 : // 拉起device进程
1742 0 : if (devId != INVALID_UINT) {
1743 0 : devicePhyId_ = devId;
1744 : } else {
1745 : // 初始化ra资源(dev信息带入逻辑ID)
1746 0 : CHK_RET(hrtGetDevice(&deviceLogicId_));
1747 0 : CHK_RET(hrtGetDevicePhyIdByIndex(static_cast<u32>(deviceLogicId_), devicePhyId_));
1748 : }
1749 :
1750 : // device 网卡初始化前需要拉起 hccp .
1751 0 : rtNetServiceOpenArgs openArgs = {};
1752 0 : rtProcExtParam extParam{};
1753 0 : std::string extPam("--hdcType=" + std::to_string(PID_HDC_TYPE));
1754 :
1755 0 : extParam.paramInfo = extPam.c_str();
1756 0 : extParam.paramLen = extPam.size();
1757 0 : openArgs.extParamList = &extParam;
1758 0 : openArgs.extParamCnt = 1UL;
1759 0 : isTsdProcessOpen_ = true;
1760 0 : CHK_RET(hrtOpenNetService(&openArgs));
1761 0 : HCCL_INFO(
1762 : "NetworkManager open tsd success, devicePhyId[%u], deviceLogicId_[%d], subPid[%lld]", devicePhyId_,
1763 : deviceLogicId_, static_cast<s64>(subPid_));
1764 0 : }
1765 :
1766 1 : bool isOpenWhiteList = false;
1767 1 : if (!isBoardVersion && remoteIsHdc && IsGeneralServer()) {
1768 0 : HCCL_INFO("general server, ps open WhiteList");
1769 0 : isOpenWhiteList = true;
1770 : }
1771 :
1772 1 : CHK_RET(hrtRaSocketSetWhiteListStatus(static_cast<u32>(isOpenWhiteList)));
1773 :
1774 1 : RaInitConfig config = {DEFAULT_INIT_PHY_ID, DEFAULT_INIT_NIC_POS, DEFAULT_HDC_TYPE, false};
1775 1 : config.phyId = devicePhyId;
1776 1 : if (ipAddrStr == "127.0.0.1") {
1777 0 : config.nicPosition = static_cast<u32>(NICDeployment::NIC_DEPLOYMENT_HOST);
1778 : } else {
1779 1 : config.nicPosition = static_cast<u32>(isHostUseDevNic_);
1780 : }
1781 1 : if (isHostUseDevNic_) {
1782 0 : config.hdcType = PID_HDC_TYPE;
1783 : }
1784 :
1785 1 : if (!isRaInitRepeated_) {
1786 1 : HCCL_INFO(
1787 : "PsWorkerRaInit call HrtRaInit. devicePhyId[%u] isRaInitRepeated_[%u]", devicePhyId_, isRaInitRepeated_);
1788 1 : HcclResult hcclRet = HrtRaInit(&config);
1789 1 : if (hcclRet != HCCL_SUCCESS) {
1790 0 : HCCL_ERROR("[PsWorkerRaInit] ra init failed, ret[%d].", hcclRet);
1791 0 : if (isTsdProcessOpen_) {
1792 0 : (void)hrtCloseNetService();
1793 0 : isTsdProcessOpen_ = false;
1794 : }
1795 0 : return hcclRet;
1796 : }
1797 : }
1798 :
1799 1 : struct rdev nicRdevInfo = {};
1800 1 : nicRdevInfo.phyId = devicePhyId;
1801 1 : nicRdevInfo.family = ipAddr.GetFamily();
1802 1 : nicRdevInfo.localIp.addr = ipAddr.GetBinaryAddress().addr;
1803 1 : nicRdevInfo.localIp.addr6 = ipAddr.GetBinaryAddress().addr6;
1804 1 : SocketHandle socketHandle = nullptr;
1805 1 : NetworkMode raSocketInitMode = (isHostUseDevNic_) ? NETWORK_OFFLINE : NETWORK_PEER_ONLINE;
1806 1 : HcclResult ret = hrtRaSocketInit(raSocketInitMode, nicRdevInfo, socketHandle);
1807 1 : if (ret != HCCL_SUCCESS) {
1808 1 : HCCL_ERROR(
1809 : "[Init][HostSocket]errNo[0x%016llx] ra socket init failed, ip[%s], return[%d]",
1810 : HCCL_ERROR_CODE(HCCL_E_TCP_CONNECT), ipAddr.GetReadableAddress(), ret);
1811 1 : if (!isRaInitRepeated_) {
1812 1 : (void)HrtRaDeInit(&config);
1813 : }
1814 1 : if (isTsdProcessOpen_) {
1815 0 : (void)hrtCloseNetService();
1816 0 : isTsdProcessOpen_ = false;
1817 : }
1818 1 : return HCCL_E_TCP_CONNECT;
1819 : }
1820 0 : HCCL_INFO(
1821 : "ip[%s] socket init OK, devicePhyId_[%u], socketHandle[%llu]", ipAddr.GetReadableAddress(), devicePhyId_,
1822 : hash<void*>{}(socketHandle));
1823 :
1824 0 : IpSocket ipSocketInfo;
1825 0 : ipSocketInfo.nicSocketHandle = socketHandle;
1826 :
1827 0 : raResourceInfo_.nicSocketMap.insert(std::make_pair(ipAddr, ipSocketInfo));
1828 0 : raResourceInfo_.hostNetSocketMap.insert(std::make_pair(ipAddr, ipSocketInfo));
1829 :
1830 0 : ret = HeterogStartListen(ipAddr, port);
1831 0 : if (ret != HCCL_SUCCESS) {
1832 0 : HCCL_ERROR("[PsWorkerRaInit] HeterogStartListen failed, ret[%d]", ret);
1833 0 : (void)hrtRaSocketDeInit(socketHandle);
1834 0 : raResourceInfo_.nicSocketMap.erase(ipAddr);
1835 0 : raResourceInfo_.hostNetSocketMap.erase(ipAddr);
1836 0 : if (!isRaInitRepeated_) {
1837 0 : (void)HrtRaDeInit(&config);
1838 : }
1839 0 : if (isTsdProcessOpen_) {
1840 0 : (void)hrtCloseNetService();
1841 0 : isTsdProcessOpen_ = false;
1842 : }
1843 0 : return ret;
1844 : }
1845 :
1846 0 : return HCCL_SUCCESS;
1847 1 : }
1848 :
1849 : // 最后一次ra_deinit时才关闭device的hccp进程。ES场景主要使用
1850 0 : HcclResult NetworkManager::CloseHccpSubProc()
1851 : {
1852 0 : if (!isHostUseDevNic_ || subPid_ == 0) {
1853 0 : HCCL_INFO("No need to close hccp sub proc, devicePhyId[%u], subPid[%lld]", devicePhyId_, subPid_);
1854 0 : return HCCL_SUCCESS;
1855 : }
1856 0 : HCCL_INFO(
1857 : "NetworkManager ProcessCloseSubProcList HDC devicePhyId[%u], deviceLogicId_[%d], subPid[%lld]", devicePhyId_,
1858 : deviceLogicId_, static_cast<s64>(subPid_));
1859 0 : s32 locaLogDevid = 0;
1860 0 : hrtGetDevice(&locaLogDevid);
1861 0 : if (locaLogDevid != deviceLogicId_) {
1862 0 : hrtSetDevice(deviceLogicId_);
1863 : }
1864 0 : HcclResult ret = hrtCloseNetService();
1865 0 : if (locaLogDevid != deviceLogicId_) {
1866 0 : hrtSetDevice(locaLogDevid);
1867 : }
1868 0 : CHK_RET(ret);
1869 0 : subPid_ = 0;
1870 :
1871 0 : return HCCL_SUCCESS;
1872 : }
1873 :
1874 16 : HcclResult NetworkManager::PingMeshRaPingInit(u32 devLogicId, u32 devPhyId, RaInitConfig* config)
1875 : {
1876 : // 引用计数
1877 16 : deviceLogicId_ = static_cast<s32>(devLogicId);
1878 16 : devicePhyId_ = ((static_cast<s32>(devPhyId) == HOST_DEVICE_ID) ? 0 : devPhyId);
1879 16 : isHostUseDevNic_ = true;
1880 16 : isRaInitRepeated_ = false;
1881 :
1882 : // hccp侧初始化ping mesh资源
1883 16 : CHK_RET(HrtRaInit(config));
1884 16 : HCCL_INFO(
1885 : "[HCCN][PingMeshRaPingInit]Device[%u] config.hdcType[%d], config.nicPosition[%u], config.phyId[%u].",
1886 : deviceLogicId_, config->hdcType, config->nicPosition, config->phyId);
1887 16 : deviceNicInitRef_.Ref();
1888 :
1889 16 : return HCCL_SUCCESS;
1890 : }
1891 :
1892 15 : HcclResult NetworkManager::PingMeshRaPingDeinit()
1893 : {
1894 : // 引用计数
1895 15 : isRaInitRepeated_ = false;
1896 :
1897 15 : struct RaInitConfig config = {DEFAULT_INIT_PHY_ID, DEFAULT_INIT_NIC_POS, DEFAULT_HDC_TYPE, false};
1898 15 : GetDeviceRaInitConfig(config);
1899 15 : CHK_RET(HrtRaDeInit(&config));
1900 15 : deviceNicInitRef_.Unref();
1901 :
1902 15 : return HCCL_SUCCESS;
1903 : }
1904 :
1905 20 : void NetworkManager::GetDeviceRaInitConfig(RaInitConfig& config)
1906 : {
1907 20 : u32 devicePhyId = ((static_cast<s32>(devicePhyId_) == HOST_DEVICE_ID) ? 0 : devicePhyId_);
1908 20 : HCCL_INFO("RaDeinit devicePhyId_[%u] devicePhyId[%u]", devicePhyId_, devicePhyId);
1909 :
1910 20 : config.phyId = devicePhyId;
1911 20 : config.nicPosition = static_cast<u32>(isHostUseDevNic_);
1912 :
1913 20 : if (isHostUseDevNic_) {
1914 16 : config.hdcType = PID_HDC_TYPE;
1915 : }
1916 20 : }
1917 :
1918 0 : HcclResult NetworkManager::PsWorkerRaDeinit(u32 devId, const HcclIpAddress& ipAddr, u32 port)
1919 : {
1920 0 : string ipAddrStr(ipAddr.GetReadableAddress());
1921 0 : u32 devicePhyId = ((static_cast<s32>(devicePhyId_) == HOST_DEVICE_ID) ? 0 : devicePhyId_);
1922 0 : if (ipAddrStr == "127.0.0.1") {
1923 0 : hostNicInitRef_.Unref();
1924 0 : HCCL_INFO("hostNicInitRef_[%d]", hostNicInitRef_.Count());
1925 : } else {
1926 0 : bool lastUsed{false};
1927 0 : deviceNicInitRef_.Unref();
1928 0 : CHK_RET(hrtRaIsLastUsed(devicePhyId, lastUsed));
1929 0 : HCCL_INFO(
1930 : "deviceNicInitRef_[%d] lastUsed[%u] devicePhyId[%u]", deviceNicInitRef_.Count(), lastUsed, devicePhyId);
1931 0 : if (deviceNicInitRef_.Count() == 0 && !lastUsed) {
1932 0 : isRaInitRepeated_ = true;
1933 0 : } else if (lastUsed) {
1934 0 : isRaInitRepeated_ = false;
1935 0 : } else if (deviceNicInitRef_.Count() > 0) {
1936 0 : HCCL_INFO("[NetworkManager] PsWorkerRa is not lastUsed");
1937 0 : return HCCL_SUCCESS;
1938 : }
1939 : }
1940 0 : CHK_PRT_RET(
1941 : (raResourceInfo_.nicSocketMap.count(ipAddr) == 0),
1942 : HCCL_INFO("NetworkManager: PsWorkerRa already Deinit, ipAddr[%s]", ipAddr.GetReadableAddress()), HCCL_SUCCESS);
1943 :
1944 0 : HCCL_INFO("PsWorkerRaDeinit devId[%u], isRaInitRepeated[%d]", devId, isRaInitRepeated_);
1945 : // 重复RaInit时，不再调用内部DeInit
1946 0 : if (!isRaInitRepeated_) {
1947 0 : CHK_RET(HeterogStopListen(ipAddr, port, true));
1948 : }
1949 :
1950 0 : SocketHandle nicRdmaHandle = raResourceInfo_.nicSocketMap[ipAddr].nicRdmaHandle;
1951 0 : if (!GetExternalInputHcclIsTcpMode() && nicRdmaHandle != nullptr && !isRaInitRepeated_) {
1952 : // Helper的PS临时暂不调用 CHK_RET(HrtRaRdmaDeInit(nicRdmaHandle, (isHostUseDevNic_) ?
1953 : // NotifyTypeT::NOTIFY : NotifyTypeT::NO_USE));
1954 0 : HCCL_INFO("Not call RaRdmaDeInit devicePhyId[%u]", devicePhyId_);
1955 : }
1956 :
1957 0 : struct RaInitConfig config = {DEFAULT_INIT_PHY_ID, DEFAULT_INIT_NIC_POS, DEFAULT_HDC_TYPE, false};
1958 0 : GetDeviceRaInitConfig(config);
1959 :
1960 0 : if (ipAddrStr == "127.0.0.1") {
1961 0 : if (hostNicInitRef_.Count() == 0) {
1962 0 : HCCL_INFO("PsWorkerRaDeinit call hrtRaDeInit. devicePhyId[%u]", devicePhyId_);
1963 0 : config.nicPosition = static_cast<u32>(NICDeployment::NIC_DEPLOYMENT_HOST);
1964 0 : CHK_RET(HrtRaDeInit(&config));
1965 : }
1966 : } else {
1967 0 : if (deviceNicInitRef_.Count() == 0) {
1968 0 : HCCL_INFO("PsWorkerRaDeinit call HrtRaDeInit. devicePhyId[%u]", devicePhyId_);
1969 0 : if (!isRaInitRepeated_) {
1970 0 : CHK_RET(HrtRaDeInit(&config));
1971 0 : CHK_RET(CloseHccpSubProc());
1972 : }
1973 : }
1974 : }
1975 :
1976 0 : raResourceInfo_.nicSocketMap.erase(raResourceInfo_.nicSocketMap.find(ipAddr));
1977 0 : raResourceInfo_.hostNetSocketMap.erase(raResourceInfo_.hostNetSocketMap.find(ipAddr));
1978 :
1979 0 : return HCCL_SUCCESS;
1980 0 : }
1981 :
1982 22 : HcclResult NetworkManager::InitHostSocket(const HcclIpAddress& addr, SocketHandle& socketHandle) const
1983 : {
1984 22 : struct SocketInitInfoT socketInitInfo = {};
1985 22 : socketInitInfo.rdevInfo.family = addr.GetFamily();
1986 22 : socketInitInfo.rdevInfo.phyId = devicePhyId_;
1987 22 : socketInitInfo.rdevInfo.localIp.addr = addr.GetBinaryAddress().addr;
1988 22 : socketInitInfo.rdevInfo.localIp.addr6 = addr.GetBinaryAddress().addr6;
1989 22 : socketInitInfo.scopeId = addr.GetScopeID();
1990 22 : HcclResult ret = hrtRaSocketInitV1(NETWORK_PEER_ONLINE, socketInitInfo, socketHandle);
1991 22 : CHK_PRT_RET(
1992 : ret != HCCL_SUCCESS,
1993 : HCCL_ERROR(
1994 : "[Init][HostSocket]errNo[0x%016llx] ra socket init v1 failed, ip[%s], return[%d]",
1995 : HCCL_ERROR_CODE(HCCL_E_TCP_CONNECT), addr.GetReadableAddress(), ret),
1996 : HCCL_E_TCP_CONNECT);
1997 22 : HCCL_INFO("ip[%s] socket init OK", addr.GetReadableAddress());
1998 22 : return HCCL_SUCCESS;
1999 : }
2000 :
2001 25 : HcclResult NetworkManager::StopHostNet(SocketHandle socketHandle, const HcclIpAddress& ipAddr)
2002 : {
2003 25 : std::unique_lock<std::mutex> lock(raLock_);
2004 25 : CHK_PRT_RET(
2005 : (hostNicInitRef_.Count() == 0), HCCL_ERROR("[Stop][HostNet]cannot start nic socket before host nic inited!"),
2006 : HCCL_E_INTERNAL);
2007 :
2008 25 : CHK_RET(CheckSocketInfo(socketHandle, ipAddr));
2009 :
2010 25 : if (hostNicSocketClientRef_[ipAddr].Unref() == 0
2011 25 : && raResourceInfo_.hostNetSocketMap[ipAddr].listenedPort.size() == 0) {
2012 20 : CHK_RET(hrtRaSocketDeInit(socketHandle));
2013 20 : HCCL_INFO("ip[%s] deinit success.", ipAddr.GetReadableAddress());
2014 :
2015 20 : raResourceInfo_.hostNetSocketMap.erase(ipAddr);
2016 20 : raResourceInfo_.nicSocketMap.erase(ipAddr);
2017 : }
2018 25 : return HCCL_SUCCESS;
2019 25 : }
2020 :
2021 : // 从ip查handle 把ip下所有的listen全stop
2022 0 : HcclResult NetworkManager::StopHostSocketHandle(const HcclIpAddress& ipAddr)
2023 : {
2024 0 : std::unique_lock<std::mutex> lock(raLock_);
2025 0 : CHK_PRT_RET(
2026 : (hostNicInitRef_.Count() == 0),
2027 : HCCL_ERROR("[NetworkManager][StopHostSocketHandle]cannot start nic socket before host nic inited!"),
2028 : HCCL_E_INTERNAL);
2029 :
2030 0 : auto sockInfo = raResourceInfo_.hostNetSocketMap.find(ipAddr);
2031 0 : auto ipIt = IPPortListenRefMapHost_.find(ipAddr);
2032 0 : CHK_PRT_RET(
2033 : (sockInfo == raResourceInfo_.hostNetSocketMap.end()),
2034 : HCCL_ERROR("[NetworkManager][StopHostSocketHandle]ipAddr is invalid"), HCCL_E_INTERNAL);
2035 0 : IpSocket& sock = raResourceInfo_.hostNetSocketMap[ipAddr];
2036 0 : CHK_RET(CheckSocketInfo(sock.nicSocketHandle, ipAddr));
2037 :
2038 : // 停止所有的listen
2039 0 : if (ipIt != IPPortListenRefMapHost_.end()) {
2040 0 : for (auto& portIt : ipIt->second) {
2041 0 : u32 port = portIt.first;
2042 0 : if (IPPortListenRefMapHost_[ipAddr][port].Count() > 0) {
2043 0 : CHK_RET(StopListenSocket(sock.nicSocketHandle, port));
2044 0 : HCCL_WARNING(
2045 : "[NetworkManager][StopHostSocketHandle] ip [%s] stop listen port [%u]", ipAddr.GetReadableAddress(),
2046 : port);
2047 0 : raResourceInfo_.hostNetSocketMap[ipAddr].listenedPort.erase(port);
2048 0 : IPPortListenRefMapHost_[ipAddr][port].Clear(); // 引用计数归0
2049 : }
2050 : }
2051 : }
2052 0 : CHK_PRT_RET(
2053 : (raResourceInfo_.hostNetSocketMap[ipAddr].listenedPort.size() != 0),
2054 : HCCL_ERROR(
2055 : "[NetworkManager][StopHostSocketHandle]IPPortListenRefMapHost_[%s] is unequal to "
2056 : "hostNetSocketMap[%s].listenedPort",
2057 : ipAddr.GetReadableAddress(), ipAddr.GetReadableAddress()),
2058 : HCCL_E_INTERNAL);
2059 : // 删除socket 删除IP
2060 0 : CHK_RET(hrtRaSocketDeInit(sock.nicSocketHandle));
2061 0 : HCCL_INFO("[NetworkManager][StopHostSocketHandle] ip [%s] deinit success.", ipAddr.GetReadableAddress());
2062 0 : sock.nicSocketHandle = nullptr;
2063 : // 没有host和rdma时 删除ip
2064 0 : if (sock.nicSocketHandle == nullptr && sock.nicRdmaHandle == nullptr) {
2065 0 : raResourceInfo_.hostNetSocketMap.erase(ipAddr);
2066 : }
2067 0 : hostNicSocketClientRef_[ipAddr].Clear();
2068 0 : return HCCL_SUCCESS;
2069 0 : }
2070 :
2071 25 : HcclResult NetworkManager::StartHostNet(const HcclIpAddress& ipAddr, SocketHandle& socketHandle)
2072 : {
2073 25 : CHK_RET(CreateHostSocketHandle(ipAddr, socketHandle));
2074 25 : return HCCL_SUCCESS;
2075 : }
2076 :
2077 25 : HcclResult NetworkManager::CreateHostSocketHandle(const HcclIpAddress& ipAddr, SocketHandle& socketHandle)
2078 : {
2079 25 : CHK_PRT_RET(
2080 : (hostNicInitRef_.Count() == 0),
2081 : HCCL_ERROR("[CreateHostSocketHandle]cannot start nic socket before host nic inited!"), HCCL_E_INTERNAL);
2082 25 : OccupyIp(ipAddr, raResourceInfo_.hostNetSocketMap);
2083 25 : IpSocket& sock = raResourceInfo_.hostNetSocketMap[ipAddr];
2084 25 : if (sock.nicSocketHandle == nullptr) {
2085 20 : CHK_RET(InitHostSocket(ipAddr, sock.nicSocketHandle));
2086 : }
2087 :
2088 25 : socketHandle = sock.nicSocketHandle;
2089 25 : HCCL_INFO("ip[%s] socket start success socketHandle[%p]", ipAddr.GetReadableAddress(), socketHandle);
2090 25 : hostNicSocketClientRef_[ipAddr].Ref(); // 引用计数
2091 25 : return HCCL_SUCCESS;
2092 : }
2093 :
2094 41 : HcclResult NetworkManager::StartListenSocket(const SocketHandle socketHandle, u32& port) const
2095 : {
2096 41 : struct SocketListenInfoT serverInfo = {};
2097 41 : serverInfo.socketHandle = const_cast<SocketHandle>(socketHandle);
2098 41 : serverInfo.port = port;
2099 41 : if (isRaInitRepeated_) {
2100 0 : return HCCL_SUCCESS;
2101 : }
2102 41 : bool isAutoPort = port == AUTO_LISTEN_PORT;
2103 41 : HcclResult ret = hrtRaSocketListenStart(&serverInfo, 1);
2104 41 : CHK_PRT_RET(
2105 : ret == HCCL_E_UNAVAIL,
2106 : HCCL_INFO(
2107 : "socket port[%u] has already been bound. Could not start listening host nic. please use an idle port.",
2108 : port),
2109 : ret);
2110 41 : RPT_CALL_ERR(ret != HCCL_SUCCESS, "host nic listen start failed, port[%u], return[%d]", port, ret);
2111 41 : CHK_PRT_RET(ret != HCCL_SUCCESS, HCCL_ERROR("host nic listen start failed, port[%u], return[%d]", port, ret), ret);
2112 41 : if (isAutoPort) {
2113 0 : port = serverInfo.port;
2114 0 : CHK_PRT_RET(
2115 : port == AUTO_LISTEN_PORT, HCCL_ERROR("start listen on a port selected by os automatically failed"),
2116 : HCCL_E_NOT_SUPPORT);
2117 0 : HCCL_RUN_INFO("start listen on port[%u] by auto success.", port);
2118 : }
2119 41 : return HCCL_SUCCESS;
2120 : }
2121 :
2122 39 : HcclResult NetworkManager::StopListenSocket(const SocketHandle socketHandle, u32 port) const
2123 : {
2124 39 : struct SocketListenInfoT serverInfo = {};
2125 39 : serverInfo.socketHandle = const_cast<SocketHandle>(socketHandle);
2126 39 : serverInfo.port = port;
2127 39 : HcclResult ret = hrtRaSocketListenStop(&serverInfo, 1);
2128 39 : RPT_CALL_ERR(ret != HCCL_SUCCESS, "socket listen stop failed, port[%u], return[%d]", port, ret);
2129 39 : CHK_PRT_RET(ret != HCCL_SUCCESS, HCCL_ERROR("socket listen stop failed, port[%u], return[%d]", port, ret), ret);
2130 39 : return HCCL_SUCCESS;
2131 : }
2132 :
2133 32 : HcclResult NetworkManager::GetRdmaHandleByIpAddr(const HcclIpAddress& ipAddr, RdmaHandle& rdmaHandle)
2134 : {
2135 32 : auto it = raResourceInfo_.nicSocketMap.find(ipAddr);
2136 32 : CHK_PRT_RET(
2137 : it == raResourceInfo_.nicSocketMap.end(),
2138 : HCCL_ERROR("GetRdmaHandleByIpAddr ip[%s] is not found in nicSocketMap.", ipAddr.GetReadableAddress()),
2139 : HCCL_E_INTERNAL);
2140 32 : rdmaHandle = raResourceInfo_.nicSocketMap[ipAddr].nicRdmaHandle;
2141 32 : CHK_PTR_NULL(rdmaHandle);
2142 32 : return HCCL_SUCCESS;
2143 : }
2144 :
2145 0 : HcclResult NetworkManager::GetNicHandleByIpAddr(const HcclIpAddress& ipAddr, SocketHandle& nicHandle)
2146 : {
2147 0 : auto it = raResourceInfo_.nicSocketMap.find(ipAddr);
2148 0 : CHK_PRT_RET(
2149 : it == raResourceInfo_.nicSocketMap.end(),
2150 : HCCL_ERROR("GetNicHandleByIpAddr ip[%s] is not found in nicSocketMap.", ipAddr.GetReadableAddress()),
2151 : HCCL_E_INTERNAL);
2152 0 : nicHandle = raResourceInfo_.nicSocketMap[ipAddr].nicSocketHandle;
2153 0 : CHK_PTR_NULL(nicHandle);
2154 0 : return HCCL_SUCCESS;
2155 : }
2156 :
2157 21 : HcclResult NetworkManager::CheckAutoListenVersion(bool isAutoPort)
2158 : {
2159 21 : if (isAutoPort) {
2160 1 : u32 listenStartVersion = 0;
2161 1 : HcclResult vRet = hrtRaGetInterfaceVersion(devicePhyId_, SOCKET_LISTEN_AUTO_INTERFACE, &listenStartVersion);
2162 1 : HCCL_INFO("[CheckAutoListenVersion] listen start version[%u].", listenStartVersion);
2163 1 : CHK_PRT_RET(
2164 : vRet != HCCL_SUCCESS || listenStartVersion < SOCKET_LISTEN_AUTO_INTERFACE_VERSION,
2165 : HCCL_ERROR(
2166 : "this package does not support hrtRaSocketNonBlockListenStart to "
2167 : "listen automatically with port %u, please change new package.",
2168 : AUTO_LISTEN_PORT),
2169 : HCCL_E_NOT_SUPPORT);
2170 : }
2171 20 : return HCCL_SUCCESS;
2172 : }
2173 :
2174 : // 新旧ip类型转换
2175 0 : HcclResult NetworkManager::HcclIpAddressConvertHcclAddr(HcclAddress* hccladdr, HcclIpAddress* hcclIP)
2176 : {
2177 0 : CHK_PTR_NULL(hcclIP);
2178 0 : CHK_PTR_NULL(hccladdr);
2179 0 : if (hcclIP->GetFamily() == AF_INET) {
2180 0 : hccladdr->type = HCCL_ADDR_TYPE_IP_V4;
2181 0 : hccladdr->addr = hcclIP->GetBinaryAddress().addr;
2182 0 : } else if (hcclIP->GetFamily() == AF_INET6) {
2183 0 : hccladdr->type = HCCL_ADDR_TYPE_IP_V6;
2184 0 : hccladdr->addr6 = hcclIP->GetBinaryAddress().addr6;
2185 : } else {
2186 0 : HCCL_ERROR("[HcclIpAddressConvertingHcclAddr]ERROR IP type!");
2187 0 : return HCCL_E_PARA;
2188 : }
2189 0 : return HCCL_SUCCESS;
2190 : }
2191 :
2192 25 : HcclResult NetworkManager::OccupyIp(const HcclIpAddress& ipAddr, std::map<hccl::HcclIpAddress, IpSocket>& socketMap)
2193 : {
2194 25 : auto sockInfo = socketMap.find(ipAddr);
2195 25 : if (sockInfo == socketMap.end()) {
2196 20 : IpSocket tempSock;
2197 20 : socketMap.insert(std::make_pair(ipAddr, tempSock)); // 本IP占位
2198 20 : HCCL_INFO("[NetworkManager][OccupyIp] device[%u] insert ip[%s]", devicePhyId_, ipAddr.GetReadableAddress());
2199 20 : } else {
2200 5 : HCCL_INFO(
2201 : "[NetworkManager][OccupyIp] device[%u] ip[%s] has already occupied.", devicePhyId_,
2202 : ipAddr.GetReadableAddress());
2203 : }
2204 25 : return HCCL_SUCCESS;
2205 : }
2206 : } // namespace hccl
|
