Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "network_manager.h"
12 : #include "externalinput_pub.h"
13 : #include "device_capacity.h"
14 : #include "adapter_tdt.h"
15 : #include "adapter_rts.h"
16 : #include "adapter_hccp.h"
17 : #include "adapter_hal.h"
18 : #include "adapter_error_manager.h"
19 : #include "dlhal_function.h"
20 : #include "adapter_hccp_common.h"
21 : namespace hccl {
22 :
23 : using namespace std;
24 :
25 : constexpr u32 SOCKET_LISTEN_AUTO_INTERFACE = 4;
26 : constexpr u32 SOCKET_LISTEN_AUTO_INTERFACE_VERSION = 3;
27 :
28 : NetworkManager* NetworkManager::nmInstance[MAX_DEV_NUM] = {nullptr};
29 : std::atomic<unsigned> NetworkManager::InitTool::initCount(0);
30 :
31 1455 : NetworkManager::InitTool::InitTool()
32 : {
33 1455 : if (initCount.load() == 0) {
34 1419 : for (u32 i = 0; i < MAX_DEV_NUM; i++) {
35 1376 : NetworkManager::nmInstance[i] = new (std::nothrow) NetworkManager;
36 1376 : CHK_PRT_CONT(NetworkManager::nmInstance[i] == nullptr,
37 : HCCL_ERROR("[NetworkManager][InitTool] nmInstance[%u] new failed", i));
38 : }
39 : }
40 1455 : ++initCount;
41 1455 : }
42 :
43 1455 : NetworkManager::InitTool::~InitTool()
44 : {
45 1455 : --initCount;
46 1455 : if (initCount.load() == 0) {
47 1419 : for (u32 i = 0; i < MAX_DEV_NUM; i++) {
48 1376 : if (NetworkManager::nmInstance[i] != nullptr) {
49 1376 : delete NetworkManager::nmInstance[i];
50 1376 : NetworkManager::nmInstance[i] = nullptr;
51 : }
52 : }
53 : }
54 1455 : }
55 :
56 1376 : NetworkManager::NetworkManager()
57 1376 : : deviceLogicId_(INVALID_INT),
58 1376 : devicePhyId_(INVALID_UINT),
59 1376 : isHostUseDevNic_(false),
60 1376 : notifyType_(NO_USE)
61 : {
62 1376 : }
63 :
64 1376 : NetworkManager::~NetworkManager()
65 : {
66 1376 : Destroy();
67 1376 : isRaDeInit_ = false;
68 1376 : }
69 :
70 1157 : NetworkManager &NetworkManager::GetInstance(s32 deviceLogicID)
71 : {
72 1157 : HCCL_INFO("NetworkManager::GetInstance deviceLogicID[%u].", deviceLogicID);
73 1160 : if (deviceLogicID == HOST_DEVICE_ID) {
74 4 : nmInstance[DEFAULT_DEVICE_LOGIC_ID]->deviceLogicId_ = DEFAULT_DEVICE_LOGIC_ID;
75 4 : return *(nmInstance[DEFAULT_DEVICE_LOGIC_ID]);
76 : }
77 :
78 1156 : if (static_cast<u32>(deviceLogicID) >= MAX_DEV_NUM || deviceLogicID <= HOST_DEVICE_ID) {
79 0 : HCCL_WARNING("[Get][Instance]deviceLogicID[%d] is invalid", deviceLogicID);
80 0 : nmInstance[DEFAULT_DEVICE_LOGIC_ID]->deviceLogicId_ = DEFAULT_DEVICE_LOGIC_ID;
81 0 : return *(nmInstance[DEFAULT_DEVICE_LOGIC_ID]);
82 : }
83 1156 : nmInstance[deviceLogicID]->deviceLogicId_ = deviceLogicID;
84 1156 : return *(nmInstance[deviceLogicID]);
85 : }
86 :
87 410 : HcclResult NetworkManager::TsdCapabilityGet(bool &supportMultiProcHCCP)
88 : {
89 410 : int32_t type = TSD_CAPABILITY_MUTIPLE_HCCP;
90 410 : bool *resultPtr = &supportMultiProcHCCP;
91 410 : CHK_RET(hrtTsdCapabilityGet(deviceLogicId_, type, static_cast<uint64_t>(reinterpret_cast<uintptr_t>(resultPtr))));
92 410 : return HCCL_SUCCESS;
93 : }
94 :
95 0 : HcclResult NetworkManager::GetNicIp(uint32_t devicePhyId, HcclAddress** addr, uint32_t *len)
96 : {
97 0 : vector<HcclIpAddress> tempIp;
98 0 : std::unique_lock<std::mutex> lock(memResMutex_);
99 0 : if (nicIpAddrs_.size() == 0) {
100 0 : CHK_RET(hrtRaGetDeviceIP(devicePhyId, tempIp));
101 0 : nicIpAddrs_.resize(tempIp.size());
102 0 : for (size_t i = 0; i < tempIp.size(); ++i) {
103 0 : CHK_RET(HcclIpAddressConvertHcclAddr(&nicIpAddrs_[i], &tempIp[i]));
104 : }
105 : }
106 0 : *addr = this->nicIpAddrs_.data();
107 0 : *len = this->nicIpAddrs_.size();
108 0 : lock.unlock();
109 0 : return HCCL_SUCCESS;
110 0 : }
111 :
112 0 : HcclResult NetworkManager::TsdProcessOpen(bool hasBackup)
113 : {
114 0 : s32 locaLogDevid = 0;
115 0 : hrtGetDevice(&locaLogDevid);
116 0 : if (locaLogDevid != deviceLogicId_) {
117 0 : hrtSetDevice(deviceLogicId_);
118 : }
119 : // 校验是否为新版本驱动，旧版本驱动不支持配置backupPhyId，报错返回
120 0 : s32 halAPIVersion = 0;
121 0 : CHK_RET(DlHalFunction::GetInstance().DlHalFunctionInit());
122 0 : CHK_RET(hrtHalGetAPIVersion(halAPIVersion));
123 0 : HCCL_INFO("[%s]params: halAPIVersion[%d], BACKUP_DEVICE_LOG_DEV_VERSION[%d]", __func__, halAPIVersion,
124 : BACKUP_DEVICE_LOG_DEV_VERSION);
125 0 : if (halAPIVersion < BACKUP_DEVICE_LOG_DEV_VERSION) {
126 0 : HCCL_WARNING("[%s]this package does not support obtaining backUp HCCP Log in PLOG. halAPIVersion[%d]",
127 : __func__, halAPIVersion);
128 : }
129 0 : isTsdProcessOpen_ = true;
130 0 : if (!hasBackup || halAPIVersion < BACKUP_DEVICE_LOG_DEV_VERSION) {
131 0 : std::string extPam("--hdcType=" + std::to_string(HDC_SERVICE_TYPE_RDMA_V2));
132 : rtNetServiceOpenArgs openArgs;
133 0 : rtProcExtParam extParam{};
134 0 : extParam.paramInfo = extPam.c_str();
135 0 : extParam.paramLen = extPam.size();
136 0 : openArgs.extParamList = &extParam;
137 0 : openArgs.extParamCnt = 1UL;
138 : // 根据pid粒度拉起hccp的rs进程
139 0 : CHK_RET(hrtOpenNetService(&openArgs));
140 0 : HCCL_INFO("[%s]hrtOpenNetService success, subPid[%d], devicePhyId_[%u], deviceLogicId_[%d], hasBackup[%u]",
141 : __func__, subPid_, devicePhyId_, deviceLogicId_, hasBackup);
142 0 : } else {
143 : // 获取chip上另一个die的logicalID
144 0 : u32 deviceBackUpPhyId = 0;
145 0 : CHK_RET(hrtGetPairDevicePhyId(devicePhyId_, deviceBackUpPhyId));
146 : rtNetServiceOpenArgs openArgs;
147 : std::string extPams[TSD_OPEN_EXT_PARA_NUM] =
148 0 : {std::string("--hdcType=" + std::to_string(HDC_SERVICE_TYPE_RDMA_V2)),
149 0 : std::string("--backupPhyId=" + std::to_string(deviceBackUpPhyId))};
150 0 : rtProcExtParam extParams[TSD_OPEN_EXT_PARA_NUM] {};
151 0 : for (u32 i = 0; i < TSD_OPEN_EXT_PARA_NUM; i++) {
152 0 : extParams[i].paramInfo = extPams[i].c_str();
153 0 : extParams[i].paramLen = extPams[i].size();
154 : }
155 0 : openArgs.extParamList = extParams;
156 0 : openArgs.extParamCnt = TSD_OPEN_EXT_PARA_NUM;
157 : // 根据pid粒度拉起hccp的rs进程
158 0 : CHK_RET(hrtOpenNetService(&openArgs));
159 0 : HCCL_INFO("[%s]hrtOpenNetService success, subPid[%u], "
160 : "devicePhyId_[%u], deviceLogicId_[%d], deviceBackUpPhyId[%u], hasBackup[%u]",
161 : __func__, subPid_, devicePhyId_, deviceLogicId_, deviceBackUpPhyId, hasBackup);
162 0 : }
163 :
164 0 : if (locaLogDevid != deviceLogicId_) {
165 0 : hrtSetDevice(locaLogDevid);
166 : }
167 0 : return HCCL_SUCCESS;
168 : }
169 :
170 0 : HcclResult NetworkManager::PrepareInit(NICDeployment nicDeploy, u32 devicePhyId, s32 &ref)
171 : {
172 0 : HCCL_INFO("nicDeploy = [%u], devicePhyId = [%u] ",nicDeploy,devicePhyId);
173 0 : if (nicDeploy == NICDeployment::NIC_DEPLOYMENT_HOST) {
174 0 : ref = hostNicInitRef_.Ref();
175 0 : } else if (nicDeploy == NICDeployment::NIC_DEPLOYMENT_DEVICE) {
176 0 : ref = deviceNicInitRef_.Ref();
177 : } else {
178 0 : HCCL_ERROR("[NetworkManager][PrepareInit]NetworkManager: init nic failed, nicPosition[%u] is not supported.",
179 : static_cast<u32>(nicDeploy));
180 0 : return HCCL_E_INTERNAL;
181 : }
182 0 : if (ref > 1) {
183 0 : HCCL_INFO("NetworkManager: init nic, nicPosition[%u] ref[%u], skip.", static_cast<u32>(nicDeploy), ref);
184 0 : return HCCL_SUCCESS;
185 : }
186 :
187 0 : if (devicePhyId != INVALID_UINT) {
188 0 : devicePhyId_ = devicePhyId;
189 : } else {
190 : // 初始化ra资源(dev信息带入逻辑ID)
191 0 : CHK_RET(hrtGetDevice(&deviceLogicId_));
192 0 : CHK_RET(hrtGetDevicePhyIdByIndex(static_cast<u32>(deviceLogicId_), devicePhyId_));
193 : }
194 0 : return HCCL_SUCCESS;
195 : }
196 0 : HcclResult NetworkManager::GetConfigAndRaInit(struct RaInitConfig &config, bool isHdcV2, NICDeployment nicDeploy)
197 : {
198 : // DC场景网卡与进程在同一侧，需要设置为类似host网卡模式
199 0 : config.nicPosition = Is310PDevice() ? 0 : static_cast<u32>(nicDeploy);
200 0 : config.phyId = devicePhyId_;
201 :
202 0 : if (isHdcV2) {
203 : // 使用HDC_SERVICE_TYPE_RDMA_V2指定进程粒度
204 0 : config.hdcType = HDC_SERVICE_TYPE_RDMA_V2;
205 0 : HCCL_DEBUG("[%s]hdcType is set to HDC_SERVICE_TYPE_RDMA_V2"
206 : "devicePhyId[%u], deviceLogicId_[%d]", __func__, devicePhyId_, deviceLogicId_);
207 : }
208 0 : HcclResult ret = HrtRaInit(&config);
209 0 : RPT_CALL_ERR(ret != HCCL_SUCCESS,
210 : "ra init failed,return[%d] devicePhyId_[%u], nicPosition[%u]", ret, devicePhyId_,
211 : static_cast<u32>(nicDeploy));
212 0 : CHK_PRT_RET(ret != HCCL_SUCCESS,
213 : HCCL_ERROR("[NetworkManager][Init]errNo[0x%016llx] ra init failed,return[%d] devicePhyId_[%u], "
214 : "nicPosition[%u]", HCCL_ERROR_CODE(ret), ret, devicePhyId_, static_cast<u32>(nicDeploy)), HCCL_E_NETWORK);
215 :
216 0 : return HCCL_SUCCESS;
217 : }
218 :
219 0 : HcclResult NetworkManager:: GetTsdOpen(NICDeployment nicDeploy, bool hasBackup, bool &supportMultiProcHCCP)
220 : {
221 0 : if (nicDeploy == NICDeployment::NIC_DEPLOYMENT_DEVICE && !Is310PDevice()) { // DC场景不需要拉起tsd
222 0 : CHK_RET(DlTdtFunction::GetInstance().DlTdtFunctionInit());
223 0 : CHK_RET(TsdCapabilityGet(supportMultiProcHCCP));
224 0 : HCCL_INFO("[NetworkManager][Init]supportMultiProcHCCP[%u], hasBackup[%u]", supportMultiProcHCCP, hasBackup);
225 0 : if (supportMultiProcHCCP || hasBackup) {
226 : // 根据pid粒度拉起hccp的rs进程
227 0 : CHK_RET(TsdProcessOpen(hasBackup));
228 0 : HCCL_INFO("[NetworkManager][Init]open tsd by process success, devicePhyId[%u], deviceLogicId_[%d].",
229 : devicePhyId_, deviceLogicId_);
230 0 : } else {
231 : // device 网卡初始化前需要拉起 hccp.
232 0 : CHK_RET(hrtOpenTsd());
233 0 : HCCL_INFO("[%s]NetworkManager open tsd success, devicePhyId[%u], deviceLogicId_[%d]",
234 : __func__, devicePhyId_, deviceLogicId_);
235 : }
236 : }
237 0 : return HCCL_SUCCESS;
238 : }
239 : /* init network resource */
240 252 : HcclResult NetworkManager::Init(NICDeployment nicDeploy, bool enableWhitelistFlag, u32 devicePhyId,
241 : bool isHostUseDevNic, bool hasBackup)
242 : {
243 252 : s32 ref = 0;
244 252 : if (nicDeploy == NICDeployment::NIC_DEPLOYMENT_HOST) {
245 25 : ref = hostNicInitRef_.Ref();
246 227 : } else if (nicDeploy == NICDeployment::NIC_DEPLOYMENT_DEVICE) {
247 227 : ref = deviceNicInitRef_.Ref();
248 : } else {
249 0 : HCCL_ERROR("[NetworkManager][Init]NetworkManager: init nic failed, nicPosition[%u] is not supported.",
250 : static_cast<u32>(nicDeploy));
251 0 : return HCCL_E_INTERNAL;
252 : }
253 252 : if (ref > 1) {
254 25 : HCCL_INFO("NetworkManager: init nic, nicPosition[%u] ref[%u], skip.", static_cast<u32>(nicDeploy), ref);
255 25 : return HCCL_SUCCESS;
256 : }
257 :
258 227 : if (devicePhyId != INVALID_UINT) {
259 37 : devicePhyId_ = devicePhyId;
260 : } else {
261 : // 初始化ra资源(dev信息带入逻辑ID)
262 190 : CHK_RET(hrtGetDevice(&deviceLogicId_));
263 190 : CHK_RET(hrtGetDevicePhyIdByIndex(static_cast<u32>(deviceLogicId_), devicePhyId_));
264 : }
265 227 : HCCL_INFO("NetworkManager: devicePhyId[%u], devicePhyId_[%u] deviceLogicId_[%u], hasBackup[%d]", devicePhyId,
266 : devicePhyId_, deviceLogicId_, hasBackup);
267 :
268 227 : bool supportMultiProcHCCP = false;
269 227 : if (nicDeploy == NICDeployment::NIC_DEPLOYMENT_DEVICE && !Is310PDevice()) { // DC场景不需要拉起tsd
270 206 : CHK_RET(DlTdtFunction::GetInstance().DlTdtFunctionInit());
271 206 : CHK_RET(TsdCapabilityGet(supportMultiProcHCCP));
272 206 : HCCL_INFO("[NetworkManager][Init]supportMultiProcHCCP[%u], hasBackup[%u]", supportMultiProcHCCP, hasBackup);
273 206 : if (supportMultiProcHCCP || hasBackup) {
274 : // 根据pid粒度拉起hccp的rs进程
275 0 : CHK_RET(TsdProcessOpen(hasBackup));
276 0 : HCCL_INFO("[NetworkManager][Init]open tsd by process success, devicePhyId[%u], deviceLogicId_[%d].",
277 : devicePhyId_, deviceLogicId_);
278 0 : } else {
279 : // device 网卡初始化前需要拉起 hccp.
280 206 : CHK_RET(hrtOpenTsd());
281 206 : HCCL_INFO("[%s]NetworkManager open tsd success, devicePhyId[%u], deviceLogicId_[%d]",
282 : __func__, devicePhyId_, deviceLogicId_);
283 : }
284 : }
285 :
286 227 : struct RaInitConfig config = { DEFAULT_INIT_PHY_ID, DEFAULT_INIT_NIC_POS, DEFAULT_HDC_TYPE, false };
287 227 : u32 enableWhiteList = (GetExternalInputHcclEnableWhitelist() == HCCL_WHITELIST_ON) ? 1 : 0;
288 227 : if (GetRemoteIsHdc() && IsGeneralServer()) {
289 0 : HCCL_INFO("General server NetworkManager open Whitelist");
290 0 : enableWhitelistFlag = true;
291 0 : enableWhiteList = true;
292 : }
293 227 : if (nicDeploy == NICDeployment::NIC_DEPLOYMENT_HOST && enableWhitelistFlag) {
294 16 : CHK_RET(hrtRaSocketSetWhiteListStatus(enableWhiteList));
295 : }
296 : // DC场景网卡与进程在同一侧，需要设置为类似host网卡模式
297 227 : config.nicPosition = Is310PDevice() ? 0 : static_cast<u32>(nicDeploy);
298 227 : config.phyId = devicePhyId_;
299 227 : HCCL_DEBUG("[%s]config.phyId = %u, nicPosition[%u], hasBackup[%d], devicePhyId_[%u], deviceLogicId_[%d], "
300 : "devicePhyId[%u]", __func__, config.phyId, config.nicPosition, hasBackup, devicePhyId_, deviceLogicId_,
301 : devicePhyId);
302 :
303 227 : if (nicDeploy == NICDeployment::NIC_DEPLOYMENT_DEVICE && !Is310PDevice() && (supportMultiProcHCCP || hasBackup)) {
304 : // 使用HDC_SERVICE_TYPE_RDMA_V2指定进程粒度
305 0 : config.hdcType = HDC_SERVICE_TYPE_RDMA_V2;
306 0 : HCCL_DEBUG("[%s]hdcType is set to HDC_SERVICE_TYPE_RDMA_V2, hasBackup[%d], nicDeploy[%d], "
307 : "devicePhyId[%u], deviceLogicId_[%d]", __func__, hasBackup, nicDeploy, devicePhyId_, deviceLogicId_);
308 : }
309 : DevType devType;
310 227 : CHK_RET(hrtGetDeviceType(devType));
311 227 : if (devType == DevType::DEV_TYPE_910_93 && nicDeploy == NICDeployment::NIC_DEPLOYMENT_DEVICE) {
312 8 : isEnableHdcAsync_ = true;
313 8 : config.enableHdcAsync = true;
314 : }
315 227 : HCCL_INFO("[%s]config.phyId[%u], config.nicPosition[%u], config.hdcType[%d], config.enableHdcAsync[%d]",
316 : __func__, config.phyId, config.nicPosition, config.hdcType, config.enableHdcAsync);
317 227 : HcclResult ret = HrtRaInit(&config);
318 227 : RPT_CALL_ERR(ret != HCCL_SUCCESS,
319 : "ra init failed,return[%d] devicePhyId_[%u], nicPosition[%u]", ret, devicePhyId_,
320 : static_cast<u32>(nicDeploy));
321 227 : CHK_PRT_RET(ret != HCCL_SUCCESS,
322 : HCCL_ERROR("[NetworkManager][Init]errNo[0x%016llx] ra init failed,return[%d] devicePhyId_[%u], "
323 : "nicPosition[%u]", HCCL_ERROR_CODE(ret), ret, devicePhyId_, static_cast<u32>(nicDeploy)), HCCL_E_NETWORK);
324 :
325 226 : HCCL_INFO("NetworkManager nicDeploy[%u] deviceLogicId_[%d] devicePhyId_[%u] init ra OK, nicSocketMap size[%u], "
326 : "isHostUseDevNic[%d]", static_cast<u32>(nicDeploy), deviceLogicId_, devicePhyId_,
327 : raResourceInfo_.nicSocketMap.size(), isHostUseDevNic);
328 226 : return HCCL_SUCCESS;
329 : }
330 :
331 0 : HcclResult NetworkManager::InitV2(NICDeployment nicDeploy, bool isBackup, u32 devicePhyId, bool isHostUseDevNic)
332 : {
333 0 : s32 ref = 0;
334 0 : CHK_RET(PrepareInit(nicDeploy, devicePhyId, ref));
335 0 : if (ref > 1) {
336 0 : HCCL_INFO("NetworkManager: initv2 nic, nicPosition[%u] ref[%u], skip.", static_cast<u32>(nicDeploy), ref);
337 0 : return HCCL_SUCCESS;
338 : }
339 0 : HCCL_INFO("NetworkManager InitV2: devicePhyId[%u], devicePhyId_[%u] deviceLogicId_[%u], isBackup[%d]", devicePhyId,
340 : devicePhyId_, deviceLogicId_, isBackup);
341 :
342 0 : bool supportMultiProcHCCP = false;
343 0 : CHK_RET(GetTsdOpen(nicDeploy, isBackup, supportMultiProcHCCP));
344 :
345 0 : struct RaInitConfig config = { DEFAULT_INIT_PHY_ID, DEFAULT_INIT_NIC_POS, DEFAULT_HDC_TYPE, false};
346 0 : bool isHdcV2 = (nicDeploy == NICDeployment::NIC_DEPLOYMENT_DEVICE && !Is310PDevice() && (supportMultiProcHCCP || isBackup));
347 0 : if (isHdcV2) {
348 0 : HCCL_DEBUG("[%s]hdcType is set to HDC_SERVICE_TYPE_RDMA_V2, isBackup[%d], nicDeploy[%d], "
349 : "devicePhyId[%u], deviceLogicId_[%d]", __func__, isBackup, nicDeploy, devicePhyId_, deviceLogicId_);
350 : }
351 0 : CHK_RET(GetConfigAndRaInit(config, isHdcV2, nicDeploy));
352 :
353 0 : HCCL_INFO("NetworkManager nicDeploy[%u] deviceLogicId_[%d] devicePhyId_[%u] init ra OK, nicSocketMap size[%u], "
354 : "isHostUseDevNic[%d]", static_cast<u32>(nicDeploy), deviceLogicId_, devicePhyId_,
355 : raResourceInfo_.nicSocketMap.size(), isHostUseDevNic);
356 0 : return HCCL_SUCCESS;
357 : }
358 :
359 0 : HcclResult NetworkManager::HeterogStartListen(const HcclIpAddress &ipAddr, u32 port)
360 : {
361 0 : HCCL_DEBUG("HeterogStartListen ipAddr[%s] port[%u]", ipAddr.GetReadableAddress(), port);
362 0 : SocketHandle nicSocketHandle = raResourceInfo_.nicSocketMap[ipAddr].nicSocketHandle;
363 :
364 0 : CHK_PRT_RET(port > MAX_PORT_ID || port < MIN_PORT_ID,
365 : HCCL_ERROR("[NetworkManager][HeterogStartListen] port error[%u]", port), HCCL_E_NETWORK);
366 :
367 0 : if (nicSocketHandle != nullptr && raResourceInfo_.nicSocketMap[ipAddr].listenedPort.find(port) ==
368 0 : raResourceInfo_.nicSocketMap[ipAddr].listenedPort.end() &&
369 0 : IPPortListenRefMapHost_[ipAddr][port].Ref() == FIRST_LISTEN) {
370 0 : CHK_RET(StartListenSocket(nicSocketHandle, port));
371 0 : raResourceInfo_.nicSocketMap[ipAddr].listenedPort.insert(port);
372 0 : raResourceInfo_.hostNetSocketMap[ipAddr].listenedPort.insert(port);
373 : }
374 0 : return HCCL_SUCCESS;
375 : }
376 :
377 1 : HcclResult NetworkManager::HeterogInit(u32 devId, const HcclIpAddress &ipAddr, u32 port)
378 : {
379 1 : s32 ref = hostNicInitRef_.Ref();
380 1 : if (ref > 1) {
381 0 : HCCL_INFO("NetworkManager: heterog init nic, ref[%u], skip", ref);
382 0 : return HCCL_SUCCESS;
383 : }
384 :
385 1 : CHK_RET(hrtRaSocketSetWhiteListStatus(0));
386 :
387 : // 暂缺获取物理id的手段
388 1 : RaInitConfig config = { DEFAULT_INIT_PHY_ID, DEFAULT_INIT_NIC_POS, DEFAULT_HDC_TYPE, false };
389 1 : config.phyId = ((static_cast<s32>(devId) == HOST_DEVICE_ID) ? 0 : devId);
390 1 : config.nicPosition = static_cast<u32>(NICDeployment::NIC_DEPLOYMENT_HOST);
391 1 : HCCL_INFO("HeterogInit call HrtRaInit.");
392 1 : CHK_RET(HrtRaInit(&config));
393 :
394 1 : struct rdev nicRdevInfo = {};
395 1 : nicRdevInfo.phyId = devId;
396 1 : nicRdevInfo.family = ipAddr.GetFamily();
397 1 : nicRdevInfo.localIp.addr = ipAddr.GetBinaryAddress().addr;
398 1 : nicRdevInfo.localIp.addr6 = ipAddr.GetBinaryAddress().addr6;
399 1 : SocketHandle socketHandle = nullptr;
400 1 : HcclResult ret = hrtRaSocketInit(NETWORK_PEER_ONLINE, nicRdevInfo, socketHandle);
401 1 : if (ret != HCCL_SUCCESS) {
402 1 : HCCL_ERROR("[HeterogInit]errNo[0x%016llx] ra socket init failed, ip[%s], return[%d]",
403 : HCCL_ERROR_CODE(HCCL_E_TCP_CONNECT), ipAddr.GetReadableAddress(), ret);
404 1 : HrtRaDeInit(&config);
405 1 : return HCCL_E_TCP_CONNECT;
406 : }
407 0 : HCCL_INFO("ip[%s] socket init OK", ipAddr.GetReadableAddress());
408 :
409 0 : IpSocket ipSocketInfo;
410 0 : ipSocketInfo.nicSocketHandle = socketHandle;
411 :
412 0 : raResourceInfo_.nicSocketMap.insert(std::make_pair(ipAddr, ipSocketInfo));
413 0 : raResourceInfo_.hostNetSocketMap.insert(std::make_pair(ipAddr, ipSocketInfo));
414 :
415 : /* device网卡初始化暂不考虑 */
416 0 : if (!GetExternalInputHcclIsTcpMode()) {
417 0 : ret = InitRdmaHandle(devId, ipAddr);
418 0 : if (ret != HCCL_SUCCESS) {
419 0 : HCCL_ERROR("[HeterogInit] InitRdmaHandle fail, ret[%d], destroy resource.", ret);
420 0 : hrtRaSocketDeInit(socketHandle);
421 0 : HrtRaDeInit(&config);
422 0 : return ret;
423 : }
424 : }
425 :
426 0 : ret = HeterogStartListen(ipAddr, port);
427 0 : if (ret != HCCL_SUCCESS) {
428 0 : HCCL_ERROR("[HeterogInit] HeterogStartListen fail, ret[%d], destroy resource.", ret);
429 0 : if (!GetExternalInputHcclIsTcpMode()) {
430 0 : auto& ipSock = raResourceInfo_.nicSocketMap[ipAddr];
431 0 : if (ipSock.nicRdmaHandle != nullptr) {
432 0 : (void)HrtRaRdmaDeInit(ipSock.nicRdmaHandle, NO_USE);
433 : }
434 : }
435 0 : hrtRaSocketDeInit(socketHandle);
436 0 : HrtRaDeInit(&config);
437 0 : raResourceInfo_.nicSocketMap.erase(ipAddr);
438 0 : return ret;
439 : }
440 0 : return HCCL_SUCCESS;
441 0 : }
442 :
443 0 : HcclResult NetworkManager::HeterogStopListen(const HcclIpAddress &ipAddr, u32 port, bool isNeedDeinit)
444 : {
445 0 : HCCL_DEBUG("HeterogStopListen ipAddr[%s] port[%u]", ipAddr.GetReadableAddress(), port);
446 0 : SocketHandle nicSocketHandle = raResourceInfo_.nicSocketMap[ipAddr].nicSocketHandle;
447 :
448 0 : if (nicSocketHandle != nullptr && raResourceInfo_.nicSocketMap[ipAddr].listenedPort.size() > 0) {
449 0 : if (raResourceInfo_.nicSocketMap[ipAddr].listenedPort.find(port) !=
450 0 : raResourceInfo_.nicSocketMap[ipAddr].listenedPort.end() &&
451 0 : IPPortListenRefMapHost_[ipAddr][port].Unref() == LAST_RELEASE) {
452 : // 重复raInit时不用去停止监听，但是要处理引用计数，否则析构会出问题
453 0 : if (isRaInitRepeated_) {
454 0 : return HCCL_SUCCESS;
455 : }
456 0 : HCCL_INFO("HeterogStopListen socket listen stop, socket port[%u]", port);
457 0 : CHK_RET(StopListenSocket(nicSocketHandle, port));
458 0 : raResourceInfo_.nicSocketMap[ipAddr].listenedPort.erase(port);
459 0 : raResourceInfo_.hostNetSocketMap[ipAddr].listenedPort.erase(port);
460 : }
461 :
462 0 : if (isNeedDeinit) {
463 0 : CHK_RET(hrtRaSocketDeInit(nicSocketHandle));
464 : }
465 : }
466 0 : return HCCL_SUCCESS;
467 : }
468 :
469 1 : HcclResult NetworkManager::HeterogDeinit(u32 devId, const HcclIpAddress &ipAddr, u32 port)
470 : {
471 1 : s32 ref = hostNicInitRef_.Unref();
472 1 : if (ref > 0) {
473 0 : HCCL_INFO("NetworkManager: heterog deinit nic success, ref[%u], skip.", ref);
474 0 : return HCCL_SUCCESS;
475 1 : } else if (ref < 0) {
476 1 : HCCL_ERROR("[NetworkManager][DeInit]NetworkManager: heterog deinit nic failed, nic has already deinit.");
477 1 : return HCCL_E_INTERNAL;
478 : }
479 :
480 0 : CHK_RET(HeterogStopListen(ipAddr, port, true));
481 :
482 0 : SocketHandle nicRdmaHandle = raResourceInfo_.nicSocketMap[ipAddr].nicRdmaHandle;
483 0 : if (!GetExternalInputHcclIsTcpMode() && nicRdmaHandle != nullptr) {
484 0 : CHK_RET(HrtRaRdmaDeInit(nicRdmaHandle, NO_USE));
485 : }
486 :
487 0 : RaInitConfig config = { DEFAULT_INIT_PHY_ID, DEFAULT_INIT_NIC_POS, DEFAULT_HDC_TYPE, false };
488 0 : config.phyId = devId;
489 0 : config.nicPosition = static_cast<u32>(NICDeployment::NIC_DEPLOYMENT_HOST);
490 :
491 0 : HCCL_INFO("HeterogDeinit call HrtRaDeInit.");
492 0 : CHK_RET(HrtRaDeInit(&config));
493 :
494 0 : raResourceInfo_.nicSocketMap.erase(raResourceInfo_.nicSocketMap.find(ipAddr));
495 0 : raResourceInfo_.hostNetSocketMap.erase(raResourceInfo_.hostNetSocketMap.find(ipAddr));
496 :
497 0 : return HCCL_SUCCESS;
498 : }
499 :
500 1 : HcclResult NetworkManager::CloseHccpProcess()
501 : {
502 1 : std::unique_lock<std::mutex> lock(hccpProcInfoMutex_);
503 1 : if (isTsdProcessOpen_ == true) {
504 0 : s32 locaLogDevid = 0;
505 0 : hrtGetDevice(&locaLogDevid);
506 0 : if (locaLogDevid != deviceLogicId_) {
507 0 : hrtSetDevice(deviceLogicId_);
508 : }
509 0 : CHK_RET(hrtCloseNetService());
510 0 : isTsdProcessOpen_ = false;
511 0 : if (locaLogDevid != deviceLogicId_) {
512 0 : hrtSetDevice(locaLogDevid);
513 : }
514 : }
515 1 : return HCCL_SUCCESS;
516 1 : }
517 :
518 1 : HcclResult NetworkManager::PrepareDeInit(s32 &ref, NICDeployment nicDeploy)
519 : {
520 1 : if (nicDeploy == NICDeployment::NIC_DEPLOYMENT_HOST) {
521 1 : ref = hostNicInitRef_.Unref();
522 0 : } else if (nicDeploy == NICDeployment::NIC_DEPLOYMENT_DEVICE) {
523 0 : ref = deviceNicInitRef_.Unref();
524 : } else {
525 0 : HCCL_ERROR("[NetworkManager][PrepareDeInit]NetworkManager: deinit nic failed, nicPosition[%u] is not supported.",
526 : static_cast<u32>(nicDeploy));
527 0 : return HCCL_E_INTERNAL;
528 : }
529 :
530 1 : if (ref > 0) {
531 0 : HCCL_INFO("NetworkManager: PrepareDeInit nic success, nicPosition[%u] ref[%u], skip.", static_cast<u32>(nicDeploy),
532 : ref);
533 0 : return HCCL_SUCCESS;
534 1 : } else if (ref < 0) {
535 1 : HCCL_ERROR("[NetworkManager][PrepareDeInit]NetworkManager: deinit nic failed, nicPosition[%u] has already deinit.",
536 : nicDeploy);
537 1 : return HCCL_E_INTERNAL;
538 : }
539 0 : return HCCL_SUCCESS;
540 : }
541 :
542 0 : HcclResult NetworkManager::GetConfigAndRaDeinit(struct RaInitConfig &config, NICDeployment nicDeploy, bool &isMultiProc, bool hasBackup)
543 : {
544 0 : if (nicDeploy == NICDeployment::NIC_DEPLOYMENT_DEVICE && !Is310PDevice()) {
545 0 : CHK_RET(DlTdtFunction::GetInstance().DlTdtFunctionInit());
546 0 : bool supportMultiProcHCCP = false;
547 0 : CHK_RET(TsdCapabilityGet(supportMultiProcHCCP));
548 0 : if (supportMultiProcHCCP || hasBackup) {
549 0 : isMultiProc = true;
550 0 : config.hdcType = HDC_SERVICE_TYPE_RDMA_V2;
551 : }
552 : }
553 :
554 0 : HcclResult ret = HrtRaDeInit(&config);
555 0 : CHK_PRT_RET(ret != HCCL_SUCCESS,
556 : HCCL_ERROR("[NetworkManager][DeInit]ra deinit failed. para: nicdeploy[%u], phyId[%u], ret[%u]",
557 : config.nicPosition, config.phyId, ret), ret);
558 0 : if (IsGeneralServer()) {
559 0 : isRaDeInit_ = true;
560 : }
561 0 : return HCCL_SUCCESS;
562 : }
563 :
564 234 : HcclResult NetworkManager::DeInit(NICDeployment nicDeploy, bool resetDeviceFlag, bool hasBackup)
565 : {
566 234 : s32 ref = 0;
567 234 : if (nicDeploy == NICDeployment::NIC_DEPLOYMENT_HOST) {
568 20 : ref = hostNicInitRef_.Unref();
569 214 : } else if (nicDeploy == NICDeployment::NIC_DEPLOYMENT_DEVICE) {
570 217 : ref = deviceNicInitRef_.Unref();
571 : } else {
572 0 : HCCL_ERROR("[NetworkManager][DeInit]NetworkManager: deinit nic failed, nicPosition[%u] is not supported.",
573 : static_cast<u32>(nicDeploy));
574 0 : return HCCL_E_INTERNAL;
575 : }
576 :
577 237 : if (ref > 0) {
578 16 : HCCL_INFO("NetworkManager: deinit nic success, nicPosition[%u] ref[%u], skip.", static_cast<u32>(nicDeploy),
579 : ref);
580 16 : return HCCL_SUCCESS;
581 221 : } else if (ref < 0) {
582 1 : HCCL_ERROR("[NetworkManager][DeInit]NetworkManager: deinit nic failed, nicPosition[%u] has already deinit.",
583 : nicDeploy);
584 1 : return HCCL_E_INTERNAL;
585 : }
586 :
587 220 : struct RaInitConfig config = { DEFAULT_INIT_PHY_ID, DEFAULT_INIT_NIC_POS, DEFAULT_HDC_TYPE, isEnableHdcAsync_ };
588 220 : config.nicPosition = Is310PDevice() ? 0 : static_cast<u32>(nicDeploy);
589 220 : config.phyId = devicePhyId_;
590 :
591 220 : bool isMultiProc = false;
592 220 : if (nicDeploy == NICDeployment::NIC_DEPLOYMENT_DEVICE && !Is310PDevice()) {
593 205 : CHK_RET(DlTdtFunction::GetInstance().DlTdtFunctionInit());
594 205 : bool supportMultiProcHCCP = false;
595 205 : CHK_RET(TsdCapabilityGet(supportMultiProcHCCP));
596 205 : if (supportMultiProcHCCP || hasBackup) {
597 1 : isMultiProc = true;
598 1 : config.hdcType = HDC_SERVICE_TYPE_RDMA_V2;
599 : }
600 : }
601 :
602 220 : HcclResult ret = HrtRaDeInit(&config);
603 219 : CHK_PRT_RET(ret != HCCL_SUCCESS,
604 : HCCL_ERROR("[NetworkManager][DeInit]ra deinit failed. para: nicdeploy[%u], phyId[%u], ret[%u]",
605 : config.nicPosition, config.phyId, ret), ret);
606 219 : if (IsGeneralServer()) {
607 0 : isRaDeInit_ = true;
608 : }
609 :
610 220 : if (isMultiProc) {
611 1 : CHK_RET(CloseHccpProcess());
612 1 : HCCL_INFO("[%s]finish CloseHccpProcess, phyId[%u], hdcType[%d], nicDeployment[%d], hasBackup[%d]",
613 : __func__, config.phyId, config.hdcType, nicDeploy, hasBackup);
614 : }
615 :
616 220 : HCCL_INFO("NetworkManager: deinit nic success, nicPosition[%u] ref[%u].", static_cast<u32>(nicDeploy), ref);
617 220 : return HCCL_SUCCESS;
618 : }
619 :
620 0 : HcclResult NetworkManager::DeInitV2(NICDeployment nicDeploy, bool isBackup, bool resetDeviceFlag)
621 : {
622 0 : s32 ref = 0;
623 0 : CHK_RET(PrepareDeInit(ref, nicDeploy));
624 0 : if (ref > 0) {
625 0 : HCCL_INFO("NetworkManager: DeInitV2 nic success, nicPosition[%u] ref[%u], skip.", static_cast<u32>(nicDeploy),
626 : ref);
627 0 : return HCCL_SUCCESS;
628 : }
629 :
630 0 : struct RaInitConfig config = { DEFAULT_INIT_PHY_ID, DEFAULT_INIT_NIC_POS, DEFAULT_HDC_TYPE, false };
631 0 : config.nicPosition = Is310PDevice() ? 0 : static_cast<u32>(nicDeploy);
632 0 : config.phyId = devicePhyId_;
633 :
634 0 : bool isMultiProc = false;
635 0 : CHK_RET(GetConfigAndRaDeinit(config, nicDeploy, isMultiProc, isBackup));
636 :
637 0 : if (isMultiProc) {
638 0 : CHK_RET(CloseHccpProcess());
639 0 : HCCL_INFO("[%s]finish CloseHccpProcess, phyId[%u], hdcType[%d], nicDeployment[%d], isBackup[%d]",
640 : __func__, config.phyId, config.hdcType, nicDeploy, isBackup);
641 : }
642 :
643 0 : HCCL_INFO("NetworkManager: DeInitV2 success, nicPosition[%u] ref[%u].", static_cast<u32>(nicDeploy), ref);
644 0 : return HCCL_SUCCESS;
645 : }
646 :
647 24 : HcclResult NetworkManager::StartVnic(HcclIpAddress localIp, u32 &port)
648 : {
649 24 : CHK_PRT_RET(deviceNicInitRef_.Count() <= 0,
650 : HCCL_ERROR("[Start][Vnic]can't start vnic socket before init device nic!"), HCCL_E_INTERNAL);
651 19 : CHK_PRT_RET(Is310PDevice(), HCCL_INFO("DC does not need vnic"), HCCL_SUCCESS);
652 19 : CHK_PRT_RET(port > MAX_PORT_ID, HCCL_ERROR("[Start][Vnic]invalid port id[%u]", port), HCCL_E_PARA);
653 :
654 19 : auto sockInfo = raResourceInfo_.vnicSocketMap.find(localIp);
655 19 : if (sockInfo == raResourceInfo_.vnicSocketMap.end()) {
656 14 : IpSocket tempSock;
657 14 : raResourceInfo_.vnicSocketMap.insert(std::make_pair(localIp, tempSock)); // 本IP占位
658 14 : HCCL_INFO("[Start][Vnic]device[%u] Start Vnic insert ip[%s]", devicePhyId_, localIp.GetReadableAddress());
659 14 : }
660 :
661 19 : IpSocket &sock = raResourceInfo_.vnicSocketMap[localIp];
662 19 : if (sock.listenedPort.size() == 0 && sock.nicSocketHandle == nullptr) {
663 14 : HcclResult ret = InitDeviceSocket(devicePhyId_, localIp, sock.nicSocketHandle);
664 14 : CHK_PRT_RET(ret != HCCL_SUCCESS,
665 : HCCL_ERROR("[Start][Vnic]errNo[0x%016llx] ra vnic init socket failed, devid[%u], ipAddr[%s], return[%d]",
666 : HCCL_ERROR_CODE(HCCL_E_TCP_CONNECT), devicePhyId_, localIp.GetReadableAddress(), ret), ret);
667 : } else {
668 5 : HCCL_INFO("[Start][Vnic]vnic socket has inited, ipAddr[%s] port[%u], skip init.",
669 : localIp.GetReadableAddress(), port);
670 : }
671 :
672 19 : if (sock.listenedPort.find(port) != sock.listenedPort.end()) {
673 5 : HCCL_WARNING("[Start][Vnic]ipAddr[%s] port[%u] is already listened.", localIp.GetReadableAddress(), port);
674 : } else {
675 : // 作为socket server端启动监听(虚拟网卡)
676 14 : bool isAutoPort = port == 0;
677 14 : HCCL_INFO("[Start][Vnic]trying to listen on ip[%s] port[%u].", localIp.GetReadableAddress(), port);
678 14 : CHK_RET(CheckAutoListenVersion(isAutoPort));
679 14 : HcclResult ret = StartListenSocket(sock.nicSocketHandle, port); /* 只拉起1个vnic */
680 14 : CHK_PRT_RET(ret == HCCL_E_UNAVAIL,
681 : HCCL_INFO("[Start][StartVnic]Could not start listening socket for IP [%s] and port [%u].",
682 : localIp.GetReadableAddress(), port), ret);
683 14 : CHK_PRT_RET(ret != HCCL_SUCCESS,
684 : HCCL_ERROR("[Start][Vnic]errNo[0x%016llx] ra inner listen start failed, "
685 : "devid[%u], ip[%s], port[%u], return[%d]",
686 : HCCL_ERROR_CODE(ret), devicePhyId_, localIp.GetReadableAddress(), port, ret), ret);
687 14 : sock.listenedPort.insert(port);
688 14 : HCCL_RUN_INFO("[Start][Vnic]Listen on ip[%s], port[%u] success, devPhyId[%u], devLogicId[%u], isAutoPort[%d]",
689 : localIp.GetReadableAddress(), port, devicePhyId_, deviceLogicId_, isAutoPort);
690 : }
691 19 : int refCount = IPPortListenRefMapVnicDevice_[localIp][port].Ref();
692 19 : HCCL_INFO("NetworkManager devicePhyId_[%u] ip[%s] port[%u] start vnic OK. refCount[%d]",
693 : devicePhyId_, localIp.GetReadableAddress(), port, refCount);
694 19 : return HCCL_SUCCESS;
695 : }
696 :
697 : // 此处只进行socket的创建 不listen
698 0 : HcclResult NetworkManager::CreateVnicSocketHandle(HcclIpAddress localIp)
699 : {
700 0 : CHK_PRT_RET(!deviceNicInitRef_.Count(), HCCL_ERROR("[NetworkManager][CreateVnicSocketHandle]"
701 : "can't start vnic socket before init device nic!"), HCCL_E_INTERNAL);
702 0 : OccupyIp(localIp, raResourceInfo_.vnicSocketMap);
703 0 : CHK_PRT_RET(Is310PDevice(), HCCL_INFO("DC does not need vnic"), HCCL_SUCCESS);
704 :
705 0 : IpSocket &sock = raResourceInfo_.vnicSocketMap[localIp];
706 0 : if (sock.listenedPort.size() == 0 && sock.nicSocketHandle == nullptr) {
707 0 : HcclResult ret = InitDeviceSocket(devicePhyId_, localIp, sock.nicSocketHandle);
708 0 : CHK_PRT_RET(ret != HCCL_SUCCESS,
709 : HCCL_ERROR("[NetworkManager][CreateVnicSocketHandle]errNo[0x%016llx] ra vnic init socket failed, "
710 : "devid[%u], return[%d]", HCCL_ERROR_CODE(HCCL_E_TCP_CONNECT), devicePhyId_, ret), ret);
711 : } else {
712 0 : HCCL_INFO("[NetworkManager][CreateVnicSocketHandle] socket has inited, ipAddr[%s], skip.",
713 : localIp.GetReadableAddress());
714 : }
715 0 : HCCL_INFO("[NetworkManager][CreateVnicSocketHandle] CreateVnicSocketHandle OK, ipAddr[%s].",
716 : localIp.GetReadableAddress());
717 0 : return HCCL_SUCCESS;
718 : }
719 :
720 6 : HcclResult NetworkManager::StopVnic(const HcclIpAddress &localIp, u32 port)
721 : {
722 6 : auto it = raResourceInfo_.vnicSocketMap.find(localIp);
723 6 : CHK_PRT_RET(it == raResourceInfo_.vnicSocketMap.end(),
724 : HCCL_ERROR("[Stop][Vnic]ip[%s] is not found in vnicSocketMap, port[%u].", localIp.GetReadableAddress(), port),
725 : HCCL_E_INTERNAL);
726 6 : IpSocket &ipSock = it->second;
727 :
728 6 : int count = IPPortListenRefMapVnicDevice_[localIp][port].Unref();
729 6 : CHK_PRT_RET(count > 0,
730 : HCCL_INFO("[Stop][Vnic]ip[%s] port[%u] ref[%d] skip stop.", localIp.GetReadableAddress(), port, count),
731 : HCCL_SUCCESS);
732 6 : CHK_PRT_RET(count < 0,
733 : HCCL_INFO("[Stop][Vnic]ip[%s] port[%u] devicePhyId_[%u] vnic stopped ERROR, refcount[%d].",
734 : localIp.GetReadableAddress(), port, devicePhyId_, count), HCCL_SUCCESS);
735 :
736 : // Stop Listen
737 6 : CHK_PRT_RET(ipSock.nicSocketHandle != nullptr && StopListenSocket(ipSock.nicSocketHandle, port) != HCCL_SUCCESS,
738 : HCCL_ERROR("[Stop][Vnic]errNo[0x%016llx] stop vnic socket failed,devid[%u], ip[%s], port[%u]",
739 : HCCL_ERROR_CODE(HCCL_E_TCP_CONNECT), devicePhyId_, localIp.GetReadableAddress(), port), HCCL_E_INTERNAL);
740 6 : ipSock.listenedPort.erase(port);
741 :
742 : // DeInit Socket
743 6 : if (ipSock.listenedPort.size() == 0 && ipSock.nicSocketHandle != nullptr) {
744 6 : HcclResult ret = hrtRaSocketDeInit(ipSock.nicSocketHandle);
745 6 : CHK_PRT_RET(ret != HCCL_SUCCESS,
746 : HCCL_ERROR("[Stop][Vnic]errNo[0x%016llx] stop vnic socket failed,devid[%u], ip[%s], return[%d]",
747 : HCCL_ERROR_CODE(HCCL_E_TCP_CONNECT), devicePhyId_, localIp.GetReadableAddress(), ret),
748 : HCCL_E_INTERNAL);
749 6 : raResourceInfo_.vnicSocketMap.erase(localIp);
750 : }
751 6 : return HCCL_SUCCESS;
752 : }
753 :
754 1 : HcclResult NetworkManager::StopVnicSocketHandle(const HcclIpAddress &localIp)
755 : {
756 1 : auto it = raResourceInfo_.vnicSocketMap.find(localIp);
757 1 : CHK_PRT_RET(it == raResourceInfo_.vnicSocketMap.end(),
758 : HCCL_ERROR("[NetworkManager][StopVnicSocketHandle]ip[%s] is not found in vnicSocketMap.",
759 : localIp.GetReadableAddress()), HCCL_E_INTERNAL);
760 :
761 1 : IpSocket &ipSock = it->second;
762 : // 关闭该ip下的全部端口的listen
763 2 : for (auto &port : ipSock.listenedPort) {
764 1 : if (ipSock.nicSocketHandle != nullptr && StopListenSocket(ipSock.nicSocketHandle, port) != HCCL_SUCCESS) {
765 0 : HCCL_ERROR("[NetworkManager][StopVnicSocketHandle]errNo[0x%016llx] stop vnic listen failed, "
766 : "devid[%u], ip[%s], port[%u]", HCCL_ERROR_CODE(HCCL_E_TCP_CONNECT), devicePhyId_,
767 : localIp.GetReadableAddress(), port);
768 0 : break;
769 : }
770 1 : IPPortListenRefMapVnicDevice_[localIp][port].Clear();
771 1 : HCCL_INFO("[NetworkManager][StopVnicSocketHandle] ip[%s] stop listen port[%u]",
772 : localIp.GetReadableAddress(), port);
773 : }
774 1 : ipSock.listenedPort.clear();
775 :
776 : // 销毁socket
777 1 : CHK_PRT_RET(ipSock.nicSocketHandle != nullptr && hrtRaSocketDeInit(ipSock.nicSocketHandle),
778 : HCCL_ERROR("[Stop][NicsSocket]VNIC socket deInit not successfully"), HCCL_E_NETWORK);
779 1 : ipSock.nicSocketHandle = nullptr;
780 1 : raResourceInfo_.vnicSocketMap.erase(localIp);
781 :
782 1 : HCCL_INFO("[NetworkManager][StopVnicSocketHandle] devid[%u] ip[%s] stop vnic socket success",
783 : devicePhyId_, localIp.GetReadableAddress());
784 1 : return HCCL_SUCCESS;
785 : }
786 :
787 39 : HcclResult NetworkManager::StartNic(const HcclIpAddress &ipAddr, u32 &port, bool rdmaFlag,
788 : HcclIpAddress ipAddrBackup)
789 : {
790 39 : CHK_PRT_RET(!deviceNicInitRef_.Count(), HCCL_ERROR("[Start][Nic]can't start nic socket before init device nic!"),
791 : HCCL_E_INTERNAL);
792 38 : auto sockInfo = raResourceInfo_.nicSocketMap.find(ipAddr);
793 38 : if (sockInfo == raResourceInfo_.nicSocketMap.end()) {
794 37 : IpSocket tempSock;
795 37 : raResourceInfo_.nicSocketMap.insert(std::make_pair(ipAddr, tempSock)); // 本IP占位
796 37 : HCCL_INFO("device[%u] Start Nic insert ip[%s]", devicePhyId_, ipAddr.GetReadableAddress());
797 37 : }
798 :
799 38 : IpSocket &sock = raResourceInfo_.nicSocketMap[ipAddr];
800 38 : if (sock.listenedPort.size() == 0) {
801 37 : if (sock.nicSocketHandle == nullptr) {
802 37 : HcclResult ret = InitDeviceSocket(devicePhyId_, ipAddr, sock.nicSocketHandle);
803 37 : CHK_PRT_RET(ret != HCCL_SUCCESS,
804 : HCCL_ERROR("[Start][Nic]errNo[0x%016llx] ra nic init socket failed, devid[%u], return[%d]",
805 : HCCL_ERROR_CODE(HCCL_E_TCP_CONNECT), devicePhyId_, ret),
806 : ret);
807 : }
808 : } else {
809 1 : HCCL_INFO("NetworkManager: socket has inited, ipAddr[%s] port[%u], skip.", ipAddr.GetReadableAddress(), port);
810 : }
811 38 : if (sock.nicRdmaHandle == nullptr && rdmaFlag) {
812 : NetworkMode netMode;
813 34 : GetNetworkMode(netMode);
814 34 : CHK_RET(GetNotifyType(notifyType_));
815 34 : HcclResult ret = InitRDMA(devicePhyId_, ipAddr, netMode, notifyType_, sock.nicRdmaHandle, isDisableLiteThread_,
816 : false, ipAddrBackup);
817 34 : CHK_PRT_RET(ret != HCCL_SUCCESS,
818 : HCCL_ERROR("[Start][Nic]errNo[0x%016llx] ra nic init rdma failed, devid[%u], return[%d]",
819 : HCCL_ERROR_CODE(HCCL_E_NETWORK), devicePhyId_, ret),
820 : HCCL_E_NETWORK);
821 :
822 : int supportLite;
823 34 : ret = HrtGetRdmaLiteStatus(sock.nicRdmaHandle, &supportLite);
824 34 : CHK_PRT_RET(ret != HCCL_SUCCESS,
825 : HCCL_ERROR("[Get][RdmaLiteStatus]errNo[0x%016llx] get rdma lite status failed, return[%d]",
826 : HCCL_ERROR_CODE(HCCL_E_TCP_CONNECT), ret),
827 : HCCL_E_TCP_CONNECT);
828 34 : isRdmaLiteEn_ = (supportLite == 1);
829 34 : } else {
830 4 : HCCL_INFO("[Start][Nic] requesting not init rdma , or already init rdma, "
831 : "rdmaFlag[%u], ipAddr[%s], port[%u], skip.",
832 : rdmaFlag, ipAddr.GetReadableAddress(), port);
833 : }
834 : // 如果port id传入的值无效值0xFFFFFFFF, 不启动监听
835 38 : CHK_PRT_RET(port == MAX_VALUE_U32, HCCL_INFO("[Start][Nic] port id[%u], skip listen socket", port), HCCL_SUCCESS);
836 6 : CHK_PRT_RET(port > MAX_PORT_ID, HCCL_ERROR("[Start][Nic]invalid port id[%u]", port), HCCL_E_INTERNAL);
837 6 : if (sock.listenedPort.find(port) != sock.listenedPort.end()) {
838 0 : HCCL_WARNING("port[%u] is already listened.", port);
839 : } else {
840 6 : bool isAutoPort = port == 0;
841 6 : HCCL_INFO("[Start][Nic]trying to listen on ip[%s] port[%u].", ipAddr.GetReadableAddress(), port);
842 6 : CHK_RET(CheckAutoListenVersion(isAutoPort));
843 6 : HcclResult ret = StartListenSocket(sock.nicSocketHandle, port);
844 6 : CHK_PRT_RET(ret == HCCL_E_UNAVAIL,
845 : HCCL_INFO("[Start][Nic]Could not start listening socket for IP [%s] and port [%u].",
846 : ipAddr.GetReadableAddress(), port), ret);
847 6 : CHK_PRT_RET(ret != HCCL_SUCCESS,
848 : HCCL_ERROR("[Start][Nic]errNo[0x%016llx] ra inner listen start failed, "
849 : "devid[%u], ip[%s], port[%u], return[%d]",
850 : HCCL_ERROR_CODE(HCCL_E_TCP_CONNECT), devicePhyId_, ipAddr.GetReadableAddress(), port, ret),
851 : HCCL_E_TCP_CONNECT);
852 6 : HCCL_INFO("port[%u] listen start OK", port);
853 6 : sock.listenedPort.insert(port);
854 6 : HCCL_RUN_INFO("[Start][Nic]Listen on ip[%s], port[%u] success, devPhyId[%u], devLogicId[%u], isAutoPort[%d]",
855 : ipAddr.GetReadableAddress(), port, devicePhyId_, deviceLogicId_, isAutoPort);
856 : }
857 6 : int refCount = IPPortListenRefMapDevice_[ipAddr][port].Ref();
858 6 : HCCL_INFO("Nic ip[%s] port[%u] refcount is [%d]", ipAddr.GetReadableAddress(), port, refCount);
859 6 : return HCCL_SUCCESS;
860 : }
861 :
862 2 : void NetworkManager::SetDisableLiteThread(bool disable)
863 : {
864 2 : isDisableLiteThread_ = disable;
865 2 : }
866 :
867 0 : HcclResult NetworkManager::CreateNicSocketHandle(const HcclIpAddress &ipAddr)
868 : {
869 0 : CHK_PRT_RET(!deviceNicInitRef_.Count(), HCCL_ERROR("[NetworkManager][CreateNicSocketHandle]can't start nic socket before init device nic!"),
870 : HCCL_E_INTERNAL);
871 0 : OccupyIp(ipAddr, raResourceInfo_.nicSocketMap);
872 :
873 0 : IpSocket &sock = raResourceInfo_.nicSocketMap[ipAddr];
874 0 : if (sock.listenedPort.size() == 0) {
875 0 : if (sock.nicSocketHandle == nullptr) {
876 0 : HcclResult ret = InitDeviceSocket(devicePhyId_, ipAddr, sock.nicSocketHandle);
877 0 : CHK_PRT_RET(ret != HCCL_SUCCESS,
878 : HCCL_ERROR("[NetworkManager][CreateNicSocketHandle]errNo[0x%016llx] ra nic init socket failed, devid[%u], return[%d]",
879 : HCCL_ERROR_CODE(HCCL_E_TCP_CONNECT), devicePhyId_, ret),
880 : ret);
881 : }
882 : } else {
883 0 : HCCL_INFO("[NetworkManager][CreateNicSocketHandle] socket has inited, ipAddr[%s], skip.", ipAddr.GetReadableAddress());
884 : }
885 0 : HCCL_INFO("[NetworkManager][CreateNicSocketHandle] CreateNicSocketHandle OK, ipAddr[%s].", ipAddr.GetReadableAddress());
886 0 : return HCCL_SUCCESS;
887 : }
888 :
889 0 : HcclResult NetworkManager::CreateRdmaHandle(const HcclIpAddress &ipAddr, bool isBackup, NetworkMode netMode, NotifyTypeT notifyType, HcclNetDevDeployment netDevDeployment)
890 : {
891 : // device侧如果传入的ip是备份ip,那么给InitRDMA传递的两个主备ip(两个相同)都是备份ip hostrdma没有备份 不受影响
892 0 : HcclIpAddress ipAddrBackup(ipAddr.GetFamily(), ipAddr.GetBinaryAddress());
893 0 : if (!isBackup) {
894 0 : HCCL_INFO("[NetworkManager][CreateRdmaHandle] ipAddr[%s] is not backup", ipAddr.GetReadableAddress());
895 0 : ipAddrBackup.clear();
896 : }
897 0 : CHK_PRT_RET(!(deviceNicInitRef_.Count() || hostNicInitRef_.Count()), HCCL_ERROR("[NetworkManager][CreateRdmaHandle]can't start nic socket before init device nic!"),
898 : HCCL_E_INTERNAL);
899 0 : switch(netDevDeployment) {
900 0 : case HcclNetDevDeployment::HCCL_NETDEV_DEPLOYMENT_DEVICE:
901 : {
902 : // ip占位
903 0 : OccupyIp(ipAddr, raResourceInfo_.nicSocketMap);
904 : // 初始化socket
905 0 : IpSocket &sock = raResourceInfo_.nicSocketMap[ipAddr];
906 0 : if (sock.nicRdmaHandle == nullptr) {
907 0 : notifyType_ = notifyType;
908 0 : HcclResult ret = InitRDMA(devicePhyId_, ipAddr, netMode, notifyType_, sock.nicRdmaHandle, false,
909 : false, ipAddrBackup);
910 0 : CHK_PRT_RET(ret != HCCL_SUCCESS,
911 : HCCL_ERROR("[NetworkManager][CreateRdmaHandle]errNo[0x%016llx] ra nic init rdma failed, devid[%u], return[%d]",
912 : HCCL_ERROR_CODE(HCCL_E_NETWORK), devicePhyId_, ret),
913 : HCCL_E_NETWORK);
914 : } else {
915 0 : HCCL_INFO("[NetworkManager][CreateRdmaHandle] requesting not init rdma , or already init rdma, "
916 : "ipAddr[%s], skip.", ipAddr.GetReadableAddress());
917 : }
918 : // device-roce需要开启rdmalite
919 0 : RdmaSupportLite(sock.nicRdmaHandle);
920 0 : break;
921 : }
922 0 : case HcclNetDevDeployment::HCCL_NETDEV_DEPLOYMENT_HOST:
923 : {
924 : // 本ip占位
925 0 : OccupyIp(ipAddr, raResourceInfo_.hostNetSocketMap);
926 : // 初始化socket
927 0 : IpSocket &sock = raResourceInfo_.hostNetSocketMap[ipAddr];
928 0 : if (sock.nicRdmaHandle == nullptr) {
929 0 : notifyType_ = NOTIFY;
930 0 : HcclResult ret = InitRDMA(devicePhyId_, ipAddr, netMode, notifyType_, sock.nicRdmaHandle);
931 0 : CHK_PRT_RET(ret != HCCL_SUCCESS,
932 : HCCL_ERROR("[Start][Nic]errNo[0x%016llx] ra nic init rdma failed, devid[%u], return[%d]",
933 : HCCL_ERROR_CODE(HCCL_E_NETWORK), devicePhyId_, ret),
934 : HCCL_E_NETWORK);
935 : } else {
936 0 : HCCL_INFO("[NetworkManager][CreateRdmaHandle] requesting not init rdma , or already init rdma, "
937 : "ipAddr[%s], skip.", ipAddr.GetReadableAddress());
938 : }
939 : // host rdma的额外占位
940 0 : auto temp = raResourceInfo_.nicSocketMap.find(ipAddr);
941 0 : if (temp != raResourceInfo_.nicSocketMap.end()) {
942 0 : HCCL_ERROR("[NetworkManager][CreateRdmaHandle] ipAddr[%s] has already occupied.", ipAddr.GetReadableAddress());
943 0 : return HCCL_E_INTERNAL;
944 : }
945 0 : raResourceInfo_.nicSocketMap.insert(std::make_pair(ipAddr, sock));
946 0 : break;
947 : }
948 0 : default:
949 : {
950 0 : HCCL_ERROR("[NetworkManager][CreateRdmaHandle]this Deployment [%u] is not supported, please check the configuration.", netDevDeployment);
951 0 : return HCCL_E_NOT_SUPPORT;
952 : }
953 : }
954 0 : return HCCL_SUCCESS;
955 0 : }
956 :
957 0 : HcclResult NetworkManager::RdmaSupportLite(RdmaHandle rdmaHandle)
958 : {
959 : int supportLite;
960 0 : HcclResult ret = HrtGetRdmaLiteStatus(rdmaHandle, &supportLite);
961 0 : CHK_PRT_RET(ret != HCCL_SUCCESS,
962 : HCCL_ERROR("[Get][RdmaLiteStatus]errNo[0x%016llx] get rdma lite status failed, return[%d]",
963 : HCCL_ERROR_CODE(HCCL_E_TCP_CONNECT), ret),
964 : HCCL_E_TCP_CONNECT);
965 0 : isRdmaLiteEn_ = (supportLite == 1);
966 0 : return HCCL_SUCCESS;
967 : }
968 :
969 0 : HcclResult NetworkManager::StopRdmaHandle(const HcclIpAddress &ipAddr, HcclNetDevDeployment netDevDeployment)
970 : {
971 : // rdma没有socket 没有listen
972 :
973 : // 销毁socket
974 0 : switch (netDevDeployment) {
975 0 : case HcclNetDevDeployment::HCCL_NETDEV_DEPLOYMENT_DEVICE:{
976 0 : auto it = raResourceInfo_.nicSocketMap.find(ipAddr);
977 0 : if (it == raResourceInfo_.nicSocketMap.end()) {
978 0 : HCCL_WARNING("[Stop][NicsSocket]ip[%s] not found in nicSocketMap, may already cleaned.",
979 : ipAddr.GetReadableAddress());
980 0 : return HCCL_SUCCESS;
981 : }
982 0 : IpSocket &ipSock = it->second;
983 0 : if (ipSock.nicRdmaHandle != nullptr && HrtRaRdmaDeInit(ipSock.nicRdmaHandle, notifyType_)) {
984 0 : HCCL_ERROR("[Stop][rmda]NIC rdev deInit not successfully, notifyType_[%d]", notifyType_);
985 0 : return HCCL_E_NETWORK;
986 : }
987 0 : ipSock.nicRdmaHandle = nullptr;
988 : // 如果该ip下既没有tcp也没有rdma那么移除该ip
989 0 : if (ipSock.nicSocketHandle == nullptr && ipSock.nicRdmaHandle == nullptr) {
990 0 : raResourceInfo_.nicSocketMap.erase(ipAddr);
991 : }
992 0 : break;
993 : }
994 0 : case HcclNetDevDeployment::HCCL_NETDEV_DEPLOYMENT_HOST:{
995 0 : auto it = raResourceInfo_.hostNetSocketMap.find(ipAddr);
996 0 : CHK_PRT_RET(it == raResourceInfo_.hostNetSocketMap.end(),
997 : HCCL_ERROR("[Stop][rdma]ip[%s] is not found in hostNetSocketMap.", ipAddr.GetReadableAddress()),
998 : HCCL_E_INTERNAL);
999 0 : IpSocket &ipSock = it->second;
1000 0 : if (ipSock.nicRdmaHandle != nullptr && HrtRaRdmaDeInit(ipSock.nicRdmaHandle, notifyType_)) {
1001 0 : HCCL_ERROR("[Stop][rdma]NIC rdev deInit not successfully, notifyType_[%d]", notifyType_);
1002 0 : return HCCL_E_NETWORK;
1003 : }
1004 0 : ipSock.nicRdmaHandle = nullptr;
1005 0 : raResourceInfo_.nicSocketMap.erase(ipAddr); // 移除额外的占位
1006 : // 如果该ip下没有tcp也没有rdma那么移除该ip
1007 0 : if (ipSock.nicSocketHandle == nullptr && ipSock.nicRdmaHandle == nullptr) {
1008 0 : raResourceInfo_.hostNetSocketMap.erase(ipAddr);
1009 : }
1010 0 : break;
1011 : }
1012 0 : default: {
1013 0 : HCCL_ERROR("[NetworkManager][StopRdmaHandle]this Deployment [%u] is not supported, please check the configuration.", netDevDeployment);
1014 0 : return HCCL_E_NOT_SUPPORT;
1015 : }
1016 : }
1017 :
1018 0 : HCCL_INFO("[NetworkManager] [StopRdmaHandle] devicePhyId_[%u] StopRdmaHandle success ip [%s]", devicePhyId_, ipAddr.GetReadableAddress());
1019 0 : return HCCL_SUCCESS; // stop socket。port 数清零时自动关闭socket
1020 : }
1021 :
1022 2 : HcclResult NetworkManager::StopNicSocketHandle(const HcclIpAddress &ipAddr)
1023 : {
1024 2 : auto it = raResourceInfo_.nicSocketMap.find(ipAddr);
1025 2 : CHK_PRT_RET(it == raResourceInfo_.nicSocketMap.end(),
1026 : HCCL_ERROR("[NetworkManager][StopNicSocketHandle]ip[%s] is not found in nicSocketMap.", ipAddr.GetReadableAddress()),
1027 : HCCL_E_INTERNAL);
1028 0 : IpSocket &ipSock = it->second;
1029 : HcclResult ret;
1030 : // 关闭该ip下的全部端口的listen
1031 0 : for (auto &port : ipSock.listenedPort) {
1032 0 : ret = StopNicsSocketListen(ipAddr, port);
1033 0 : CHK_PRT_RET(ret != HCCL_SUCCESS,
1034 : HCCL_ERROR("[NetworkManager][StopNicSocketHandle]errNo[0x%016llx] stop nic listen failed,devid[%u], ip[%s], port[%u], return[%d]",
1035 : HCCL_ERROR_CODE(HCCL_E_TCP_CONNECT), devicePhyId_, ipAddr.GetReadableAddress(), port, ret),
1036 : HCCL_E_INTERNAL);
1037 0 : ipSock.listenedPort.erase(port);
1038 0 : IPPortListenRefMapDevice_[ipAddr][port].Clear(); // port计数更新
1039 0 : HCCL_WARNING("[NetworkManager][StopNicSocketHandle] ip [%s] stop listen port [%u] refcount is [%d]", ipAddr.GetReadableAddress(), port, IPPortListenRefMapDevice_[ipAddr][port].Count());
1040 : }
1041 :
1042 : // 销毁socket
1043 0 : if (ipSock.nicSocketHandle != nullptr && hrtRaSocketDeInit(ipSock.nicSocketHandle)) {
1044 0 : HCCL_ERROR("[Stop][NicsSocket]NIC socket deInit not successfully");
1045 0 : return HCCL_E_NETWORK;
1046 : }
1047 0 : ipSock.nicSocketHandle = nullptr;
1048 :
1049 : // 如果该ip下既没有tcp也没有rdma那么移除该ip
1050 0 : if (ipSock.nicSocketHandle == nullptr && ipSock.nicRdmaHandle == nullptr) {
1051 0 : raResourceInfo_.nicSocketMap.erase(ipAddr);
1052 : }
1053 :
1054 0 : return HCCL_SUCCESS;
1055 : }
1056 :
1057 36 : HcclResult NetworkManager::StopNic(const HcclIpAddress &ipAddr, u32 port)
1058 : {
1059 36 : auto it = raResourceInfo_.nicSocketMap.find(ipAddr);
1060 36 : CHK_PRT_RET(it == raResourceInfo_.nicSocketMap.end(),
1061 : HCCL_ERROR("[Stop][Nic]ip[%s] is not found in nicSocketMap, port[%u].", ipAddr.GetReadableAddress(), port),
1062 : HCCL_E_INTERNAL);
1063 36 : IpSocket &ipSock = it->second;
1064 : HcclResult ret;
1065 : // 传入端口号为无效值0xFFFFFFFF，未启动监听，不需要stop listen
1066 36 : if (port != MAX_VALUE_U32) {
1067 4 : CHK_PRT_RET(IPPortListenRefMapDevice_[ipAddr][port].Unref() > 0,
1068 : HCCL_INFO("[Stop][Nic]ip[%s] port[%u] ref[%d] skip stop.", ipAddr.GetReadableAddress(), port,
1069 : IPPortListenRefMapDevice_[ipAddr][port].Count()),
1070 : HCCL_SUCCESS);
1071 4 : ret = StopNicsSocketListen(ipAddr, port);
1072 4 : CHK_PRT_RET(ret != HCCL_SUCCESS,
1073 : HCCL_ERROR("[Stop][Nic]errNo[0x%016llx] stop nic socket failed,devid[%u], ip[%s], port[%u], return[%d]",
1074 : HCCL_ERROR_CODE(HCCL_E_TCP_CONNECT), devicePhyId_, ipAddr.GetReadableAddress(), port, ret),
1075 : HCCL_E_INTERNAL);
1076 4 : ipSock.listenedPort.erase(port);
1077 : }
1078 :
1079 36 : if (ipSock.listenedPort.size() == 0) {
1080 35 : ret = StopNicsSocket(ipAddr);
1081 35 : CHK_PRT_RET(ret != HCCL_SUCCESS,
1082 : HCCL_ERROR("[Stop][Nic]errNo[0x%016llx] stop nic socket failed,devid[%u], ip[%s], return[%d]",
1083 : HCCL_ERROR_CODE(HCCL_E_TCP_CONNECT), devicePhyId_, ipAddr.GetReadableAddress(), ret),
1084 : HCCL_E_INTERNAL);
1085 33 : raResourceInfo_.nicSocketMap.erase(ipAddr);
1086 : }
1087 34 : return HCCL_SUCCESS;
1088 : }
1089 5 : HcclResult NetworkManager::StopAllDeviceNicSockets()
1090 : {
1091 : HcclResult ret;
1092 11 : for (auto &itSocket : raResourceInfo_.nicSocketMap) {
1093 6 : std::set<u32> listenedPorts = itSocket.second.listenedPort;
1094 7 : for (auto itPort : listenedPorts) {
1095 3 : ret = StopNicsSocketListen(itSocket.first, itPort);
1096 3 : if (ret != HCCL_SUCCESS) {
1097 2 : HCCL_ERROR("[Stop][AllDeviceNicSockets]errNo[0x%016llx] stop nic socket failed,devid[%u],ip[%s], "
1098 : "port[%u],return[%d]",
1099 : HCCL_ERROR_CODE(HCCL_E_TCP_CONNECT), devicePhyId_, itSocket.first.GetReadableAddress(), itPort, ret);
1100 2 : itSocket.second.listenedPort.erase(itPort);
1101 2 : break;
1102 : }
1103 : }
1104 6 : ret = StopNicsSocket(itSocket.first);
1105 6 : if (ret != HCCL_SUCCESS) {
1106 2 : HCCL_ERROR("[Stop][AllDeviceNicSockets]errNo[0x%016llx] stop nic socket failed,devid[%u],ip[%s],return[%d]",
1107 : HCCL_ERROR_CODE(HCCL_E_TCP_CONNECT), devicePhyId_, itSocket.first.GetReadableAddress(), ret);
1108 2 : if (itSocket.second.nicRdmaHandle != nullptr) {
1109 0 : HCCL_ERROR("[StopVnicSocketHandle] itSocket.second.nicRdmaHandle is not nullptr.");
1110 0 : (void)HrtRaRdmaDeInit(itSocket.second.nicRdmaHandle, notifyType_);
1111 0 : itSocket.second.nicRdmaHandle = nullptr;
1112 : }
1113 2 : if (itSocket.second.nicSocketHandle != nullptr) {
1114 2 : HCCL_ERROR("[StopVnicSocketHandle] itSocket.second.nicSocketHandle is not nullptr.");
1115 2 : (void)HrtRaRdmaDeInit(itSocket.second.nicSocketHandle, notifyType_);
1116 2 : itSocket.second.nicSocketHandle = nullptr;
1117 : }
1118 : }
1119 6 : }
1120 :
1121 5 : raResourceInfo_.nicSocketMap.clear();
1122 5 : return HCCL_SUCCESS;
1123 : }
1124 :
1125 7 : HcclResult NetworkManager::StopAllDeviceVnicSockets()
1126 : {
1127 : HcclResult ret;
1128 :
1129 14 : for (auto itSocket : raResourceInfo_.vnicSocketMap) {
1130 7 : HCCL_WARNING("vnicSocketMap ip[%s] is not released when NetworkManager Destroy, force releasing",
1131 : itSocket.first.GetReadableAddress());
1132 7 : if (itSocket.second.nicSocketHandle != nullptr) {
1133 14 : for (auto itPort : itSocket.second.listenedPort) {
1134 7 : ret = StopListenSocket(itSocket.second.nicSocketHandle, itPort);
1135 7 : if (ret != HCCL_SUCCESS) {
1136 0 : HCCL_ERROR("[Stop][AllDeviceVnicSockets]errNo[0x%016llx] stop vnic socket listen failed, "
1137 : "devid[%u], ip[%s], port[%u], return[%d]", HCCL_ERROR_CODE(HCCL_E_TCP_CONNECT),
1138 : devicePhyId_, itSocket.first.GetReadableAddress(), itPort, ret);
1139 : }
1140 7 : IPPortListenRefMapVnicDevice_[itSocket.first][itPort].Clear();
1141 : }
1142 7 : ret = hrtRaSocketDeInit(itSocket.second.nicSocketHandle);
1143 7 : if (ret != HCCL_SUCCESS) {
1144 1 : HCCL_ERROR("[Stop][AllDeviceVnicSockets]errNo[0x%016llx] deinit vnic socket failed, "
1145 : "devid[%u], ip[%s], return[%d]", HCCL_ERROR_CODE(HCCL_E_TCP_CONNECT), devicePhyId_,
1146 : itSocket.first.GetReadableAddress(), ret);
1147 : }
1148 7 : itSocket.second.nicSocketHandle = nullptr;
1149 : }
1150 7 : }
1151 7 : raResourceInfo_.vnicSocketMap.clear();
1152 :
1153 7 : return HCCL_SUCCESS;
1154 : }
1155 :
1156 34 : HcclResult NetworkManager::InitRDMA(u32 devicePhysicID, const HcclIpAddress &ipAddr, NetworkMode netMode,
1157 : NotifyTypeT notifyType, RdmaHandle &rdmaHandle, bool disabledLiteThread, bool enable910ALite,
1158 : HcclIpAddress ipAddrBackup)
1159 : {
1160 : struct rdev nicRdevInfo;
1161 34 : nicRdevInfo.phyId = devicePhysicID;
1162 34 : nicRdevInfo.family = ipAddr.GetFamily();
1163 34 : nicRdevInfo.localIp.addr = ipAddr.GetBinaryAddress().addr;
1164 34 : nicRdevInfo.localIp.addr6 = ipAddr.GetBinaryAddress().addr6;
1165 :
1166 34 : struct RdevInitInfo init_info = { DEFAULT_INIT_RDMA_CONFIG };
1167 34 : init_info.mode = netMode;
1168 34 : init_info.notifyType = notifyType;
1169 34 : init_info.disabledLiteThread = disabledLiteThread;
1170 34 : init_info.enabled910aLite = enable910ALite;
1171 34 : init_info.enabled2mbLite = GetExternalInputRdmaFastPost();
1172 :
1173 : HcclResult ret;
1174 34 : HCCL_DEBUG("isRaInitRepeated_[%d]", isRaInitRepeated_);
1175 34 : if (isRaInitRepeated_) {
1176 : // 重复RaInit时，调用此接口获取相同的rdmaHandle，防止重新生成
1177 0 : ret = HrtRaRdmaGetHandle(devicePhysicID, rdmaHandle);
1178 : } else {
1179 34 : if (!ipAddrBackup.IsInvalid()) {
1180 : struct rdev nicRdevInfoback;
1181 0 : CHK_RET(hrtGetPairDevicePhyId(devicePhysicID, nicRdevInfoback.phyId));
1182 0 : nicRdevInfoback.family = ipAddrBackup.GetFamily();
1183 0 : nicRdevInfoback.localIp.addr = ipAddrBackup.GetBinaryAddress().addr;
1184 0 : nicRdevInfoback.localIp.addr6 = ipAddrBackup.GetBinaryAddress().addr6;
1185 0 : HCCL_DEBUG("[%s]backup rdev info: ipAddr[%s], ipAddrBackup[%s]", __func__,
1186 : ipAddr.GetReadableIP(), ipAddrBackup.GetReadableIP());
1187 0 : ret = HrtRdmaInitWithBackupAttr(init_info, nicRdevInfo, nicRdevInfoback, rdmaHandle);
1188 : } else {
1189 34 : ret = HrtRaRdmaInitWithAttr(init_info, nicRdevInfo, rdmaHandle);
1190 : }
1191 : }
1192 :
1193 34 : CHK_PRT_RET(ret != HCCL_SUCCESS,
1194 : HCCL_ERROR("[Init][RDMA]errNo[0x%016llx] ra rdma init failed, devid[%u] ip[%s], notifyType[%d], return[%d]",
1195 : HCCL_ERROR_CODE(HCCL_E_TCP_CONNECT), devicePhysicID, ipAddr.GetReadableAddress(), notifyType, ret),
1196 : HCCL_E_TCP_CONNECT);
1197 34 : HCCL_INFO("devicePhyId[%u], ip[%s] disabledLiteThread[%u] enabled910aLite[%u] rdmaHandle[%p] rdma init OK",
1198 : devicePhysicID, ipAddr.GetReadableAddress(), disabledLiteThread, enable910ALite, rdmaHandle);
1199 :
1200 34 : return HCCL_SUCCESS;
1201 : }
1202 :
1203 151 : bool NetworkManager::GetRdmaLiteStatus()
1204 : {
1205 151 : return isRdmaLiteEn_;
1206 : }
1207 :
1208 34 : HcclResult NetworkManager::GetNotifyType(NotifyTypeT ¬ifyType) const
1209 : {
1210 : DevType deviceType;
1211 34 : CHK_RET(hrtGetDeviceType(deviceType));
1212 34 : if (deviceType == DevType::DEV_TYPE_910 || deviceType == DevType::DEV_TYPE_910B ||
1213 0 : deviceType == DevType::DEV_TYPE_910_93) {
1214 34 : notifyType = NOTIFY;
1215 0 : } else if (deviceType == DevType::DEV_TYPE_310P3 || deviceType == DevType::DEV_TYPE_310P1) {
1216 0 : notifyType = EVENTID;
1217 : } else {
1218 0 : HCCL_ERROR("[Init][DeviceRDMA]devType[%d] is invalid", deviceType);
1219 0 : return HCCL_E_PARA;
1220 : }
1221 34 : return HCCL_SUCCESS;
1222 : }
1223 :
1224 85 : void NetworkManager::GetNetworkMode(NetworkMode &netMode) const
1225 : {
1226 85 : if (Is310PDevice()) {
1227 0 : netMode = NETWORK_PEER_ONLINE;
1228 : } else {
1229 85 : netMode = NETWORK_OFFLINE;
1230 : }
1231 85 : }
1232 :
1233 51 : HcclResult NetworkManager::InitDeviceSocket(u32 devicePhysicID, const HcclIpAddress &ipAddr, SocketHandle &socketHandle)
1234 : {
1235 : struct rdev nicRdevInfo;
1236 51 : nicRdevInfo.phyId = devicePhysicID;
1237 51 : nicRdevInfo.family = ipAddr.GetFamily();
1238 51 : nicRdevInfo.localIp.addr = ipAddr.GetBinaryAddress().addr;
1239 51 : nicRdevInfo.localIp.addr6 = ipAddr.GetBinaryAddress().addr6;
1240 :
1241 : NetworkMode netMode;
1242 51 : GetNetworkMode(netMode);
1243 :
1244 51 : HcclResult ret = hrtRaSocketInit(netMode, nicRdevInfo, socketHandle);
1245 51 : CHK_PRT_RET(ret != HCCL_SUCCESS,
1246 : HCCL_ERROR("[Init][DeviceSocket]ra socket init failed, network mode[%d] devid[%u] ip[%s], return[%d]", netMode,
1247 : devicePhysicID, ipAddr.GetReadableAddress(), ret),
1248 : HCCL_E_TCP_CONNECT);
1249 51 : HCCL_INFO("devicePhyId[%u], ip[%s] socket init OK", devicePhysicID, ipAddr.GetReadableAddress());
1250 :
1251 51 : return HCCL_SUCCESS;
1252 : }
1253 :
1254 22 : HcclResult NetworkManager::StartHostNetAndListen(const HcclIpAddress &ipAddr, SocketHandle &socketHandle, u32 &port,
1255 : bool rdmaFlag)
1256 : {
1257 22 : CHK_PRT_RET((hostNicInitRef_.Count() == 0),
1258 : HCCL_ERROR("[Start][HostNetAndListen]cannot start nic socket before host nic inited!"), HCCL_E_INTERNAL);
1259 22 : auto sockInfo = raResourceInfo_.hostNetSocketMap.find(ipAddr);
1260 22 : if (sockInfo == raResourceInfo_.hostNetSocketMap.end()) {
1261 2 : IpSocket tempSock;
1262 2 : raResourceInfo_.hostNetSocketMap.insert(std::make_pair(ipAddr, tempSock)); // 本IP占位
1263 2 : HCCL_INFO("device[%u] Start host nic insert Ip[%s]", devicePhyId_, ipAddr.GetReadableAddress());
1264 2 : }
1265 :
1266 : HcclResult ret;
1267 22 : IpSocket &sock = raResourceInfo_.hostNetSocketMap[ipAddr];
1268 22 : if (sock.nicSocketHandle == nullptr) {
1269 2 : ret = InitHostSocket(ipAddr, sock.nicSocketHandle);
1270 2 : CHK_PRT_RET(ret != HCCL_SUCCESS,
1271 : HCCL_ERROR("[Start][HostNetAndListen]start host socket failed, devid[%u], ip[%s] return[%d]", devicePhyId_,
1272 : ipAddr.GetReadableAddress(), ret),
1273 : ret);
1274 : }
1275 22 : if (sock.nicRdmaHandle == nullptr && rdmaFlag) {
1276 0 : notifyType_ = EVENTID;
1277 0 : ret = InitRDMA(devicePhyId_, ipAddr, NETWORK_PEER_ONLINE, notifyType_, sock.nicRdmaHandle);
1278 0 : CHK_PRT_RET(ret != HCCL_SUCCESS,
1279 : HCCL_ERROR("[Start][Nic]errNo[0x%016llx] ra nic init rdma failed, devid[%u], return[%d]",
1280 : HCCL_ERROR_CODE(HCCL_E_NETWORK), devicePhyId_, ret),
1281 : HCCL_E_NETWORK);
1282 : }
1283 22 : if (IPPortListenRefMapHost_[ipAddr][port].Count() == 0 && sock.listenedPort.find(port) == sock.listenedPort.end()) {
1284 21 : bool isAutoPort = port == 0;
1285 21 : HCCL_INFO("[Start][HostNetAndListen]trying to listen on ip[%s] port[%u].", ipAddr.GetReadableAddress(), port);
1286 21 : ret = StartListenSocket(sock.nicSocketHandle, port);
1287 21 : CHK_PRT_RET(ret == HCCL_E_UNAVAIL,
1288 : HCCL_INFO("[Start][HostNetAndListen]Could not start listening socket for IP [%s] and port [%u].",
1289 : ipAddr.GetReadableAddress(), port), ret);
1290 21 : CHK_PRT_RET(ret != HCCL_SUCCESS,
1291 : HCCL_ERROR("[Start][HostNetAndListen]start host socket failed, devid[%u], ip[%s], port[%u], return[%d]",
1292 : devicePhyId_, ipAddr.GetReadableAddress(), port, ret),
1293 : ret);
1294 21 : sock.listenedPort.insert(port);
1295 21 : HCCL_RUN_INFO("[Start][HostNetAndListen]Listen on ip[%s], port[%u] success, "
1296 : "devPhyId[%u], devLogicId[%u], isAutoPort[%d]",
1297 : ipAddr.GetReadableAddress(), port, devicePhyId_, deviceLogicId_, isAutoPort);
1298 : }
1299 22 : int refCount = IPPortListenRefMapHost_[ipAddr][port].Ref();
1300 22 : HCCL_INFO("host ip[%s] port[%u] refcount is [%d]", ipAddr.GetReadableAddress(), port, refCount);
1301 :
1302 22 : socketHandle = sock.nicSocketHandle;
1303 22 : raResourceInfo_.nicSocketMap.insert(std::make_pair(ipAddr, sock));
1304 22 : hostNicSocketClientRef_[ipAddr].Ref();
1305 22 : HCCL_INFO("HostNet, ip[%s] port[%u] socket init OK", ipAddr.GetReadableAddress(), port);
1306 22 : return HCCL_SUCCESS;
1307 : }
1308 :
1309 45 : HcclResult NetworkManager::CheckSocketInfo(const SocketHandle socketHandle, const HcclIpAddress &ipAddr, u32 port) const
1310 : {
1311 45 : auto iterIP = raResourceInfo_.hostNetSocketMap.find(ipAddr);
1312 45 : CHK_PRT_RET((iterIP == raResourceInfo_.hostNetSocketMap.end()),
1313 : HCCL_ERROR("[Check][SocketInfo]ip[%s] port[%u] has not been started. ip is invalid.",
1314 : ipAddr.GetReadableAddress(), port),
1315 : HCCL_E_INTERNAL);
1316 :
1317 45 : CHK_PRT_RET((socketHandle != iterIP->second.nicSocketHandle),
1318 : HCCL_ERROR("[Check][SocketInfo]ip[%s] port[%u] has not been started. socketHandle is invalid",
1319 : ipAddr.GetReadableAddress(), port),
1320 : HCCL_E_INTERNAL);
1321 :
1322 45 : if (port != NO_LISTEN_PORT) {
1323 20 : CHK_PRT_RET((iterIP->second.listenedPort.count(port) == 0),
1324 : HCCL_ERROR("[Check][SocketInfo]ip[%s] port[%u] has not been started. port is invalid",
1325 : ipAddr.GetReadableAddress(), port),
1326 : HCCL_E_INTERNAL);
1327 : }
1328 45 : return HCCL_SUCCESS;
1329 : }
1330 :
1331 20 : HcclResult NetworkManager::StopHostNetAndListen(SocketHandle socketHandle, const HcclIpAddress &ipAddr, u32 port)
1332 : {
1333 20 : std::unique_lock<std::mutex> lock(raLock_);
1334 20 : CHK_PRT_RET((hostNicInitRef_.Count() == 0),
1335 : HCCL_ERROR("[Stop][HostNetAndListen]cannot start nic socket before host nic inited!"), HCCL_E_INTERNAL);
1336 :
1337 20 : CHK_RET(CheckSocketInfo(socketHandle, ipAddr, port));
1338 :
1339 20 : if (IPPortListenRefMapHost_[ipAddr][port].Unref() == 0) {
1340 20 : CHK_RET(StopListenSocket(socketHandle, port)); /* 当前只拉起一个server */
1341 :
1342 20 : HCCL_INFO("ip[%s] port[%u] stop success.", ipAddr.GetReadableAddress(), port);
1343 20 : raResourceInfo_.hostNetSocketMap[ipAddr].listenedPort.erase(port);
1344 20 : raResourceInfo_.nicSocketMap[ipAddr].listenedPort.erase(port);
1345 : } else {
1346 0 : HCCL_INFO("ip[%s] port[%u] skip stop. ref[%d].", ipAddr.GetReadableAddress(), port,
1347 : IPPortListenRefMapHost_[ipAddr][port].Count());
1348 : }
1349 :
1350 20 : if (hostNicSocketClientRef_[ipAddr].Unref() == 0 && raResourceInfo_.hostNetSocketMap[ipAddr].listenedPort.size() == 0) {
1351 0 : CHK_RET(hrtRaSocketDeInit(socketHandle));
1352 0 : raResourceInfo_.hostNetSocketMap.erase(ipAddr);
1353 0 : raResourceInfo_.nicSocketMap.erase(ipAddr);
1354 0 : HCCL_INFO("ip[%s] port[%u] deinit success.", ipAddr.GetReadableAddress(), port);
1355 : }
1356 :
1357 20 : return HCCL_SUCCESS;
1358 20 : }
1359 2 : HcclResult NetworkManager::StopAllHostNicSockets()
1360 : {
1361 : HcclResult ret;
1362 4 : for (auto itSocket : raResourceInfo_.hostNetSocketMap) {
1363 3 : for (auto itPort : itSocket.second.listenedPort) {
1364 1 : ret = (StopListenSocket(itSocket.second.nicSocketHandle, itPort));
1365 1 : if (ret != HCCL_SUCCESS) {
1366 1 : HCCL_ERROR("[Stop][StopAllHostNicSockets]errNo[0x%016llx] stop listen socket failed,"
1367 : "devid[%u], ip[%s], port[%u], return[%d].", HCCL_ERROR_CODE(HCCL_E_TCP_CONNECT),
1368 : devicePhyId_, itSocket.first.GetReadableAddress(), itPort, ret);
1369 : } else {
1370 0 : HCCL_INFO("ip[%s] port[%u] stop success.", itSocket.first.GetReadableAddress(), itPort);
1371 : }
1372 : }
1373 2 : ret = hrtRaSocketDeInit(itSocket.second.nicSocketHandle);
1374 2 : if (ret != HCCL_SUCCESS) {
1375 1 : HCCL_ERROR("[Stop][StopAllHostNicSockets]errNo[0x%016llx] deinit socket failed, devid[%u], ip[%s], return[%d]",
1376 : HCCL_ERROR_CODE(HCCL_E_TCP_CONNECT), devicePhyId_, itSocket.first.GetReadableAddress(), ret);
1377 : } else {
1378 1 : HCCL_INFO("ip[%s] deinit success.", itSocket.first.GetReadableAddress());
1379 : }
1380 2 : hostNicSocketClientRef_[itSocket.first].Clear();
1381 2 : }
1382 :
1383 2 : raResourceInfo_.hostNetSocketMap.clear();
1384 2 : return HCCL_SUCCESS;
1385 : }
1386 : /* destroy network resource */
1387 1385 : HcclResult NetworkManager::Destroy()
1388 : {
1389 : /* 停止nic ra的监听 */
1390 1385 : if (raResourceInfo_.nicSocketMap.size() != 0) {
1391 7 : for (auto it : raResourceInfo_.nicSocketMap) {
1392 4 : HCCL_WARNING("nicSocketMap[%s] is not stopped when NetworkManager Destroy", it.first.GetReadableAddress());
1393 4 : }
1394 : //StartHostNetAndListen等函数中，同一地址在nicSocketMap和hostNetSocketMap内同时插入
1395 : //此处StopAllDeviceNicSockets()销毁nicSocketMap后，StopAllHostNicSockets内会发生重复销毁导致core
1396 : //为了避免此种情况，同时尽量减少对既有函数的修改扩散影响，此处同步对hostNetSocketMap进行清理
1397 : //最终修改方案需要重构本类，解除两个MAP的耦合
1398 :
1399 7 : for (auto &it : raResourceInfo_.nicSocketMap) {
1400 4 : raResourceInfo_.hostNetSocketMap.erase(it.first); // key不存在则不会删除
1401 : }
1402 3 : StopAllDeviceNicSockets();
1403 : }
1404 :
1405 : /* 停止vnic ra的监听 */
1406 1385 : if (raResourceInfo_.vnicSocketMap.size() != 0) {
1407 5 : StopAllDeviceVnicSockets();
1408 : }
1409 :
1410 : /* 停止host nic ra的监听 */
1411 1385 : if (raResourceInfo_.hostNetSocketMap.size() != 0) {
1412 0 : for (auto it : raResourceInfo_.hostNetSocketMap) {
1413 0 : HCCL_WARNING("hostNicSocketMap[%s] is not stop when NetworkManager Destroy", it.first.GetReadableAddress());
1414 0 : }
1415 0 : StopAllHostNicSockets();
1416 : }
1417 :
1418 : /* 释放ra资源 重复RaInit时，不再调用内部DeInit */
1419 1385 : HCCL_DEBUG("Destroy call HrtRaDeInit.");
1420 1385 : if (deviceNicInitRef_.Count() != 0 && !isRaDeInit_) {
1421 4 : HCCL_WARNING("device Nic is not deinit when NetworkManager Destroy. ref[%d]", deviceNicInitRef_.Count());
1422 4 : struct RaInitConfig config = { DEFAULT_INIT_PHY_ID, DEFAULT_INIT_NIC_POS, DEFAULT_HDC_TYPE, isEnableHdcAsync_ };
1423 4 : GetDeviceRaInitConfig(config);
1424 4 : config.nicPosition = static_cast<u32>(NICDeployment::NIC_DEPLOYMENT_DEVICE);
1425 4 : if (HrtRaDeInit(&config) != HCCL_SUCCESS) {
1426 0 : HCCL_ERROR("ra deinit failed. para: nicdeploy[%u], phyId[%u]", config.nicPosition, config.phyId);
1427 : }
1428 :
1429 4 : isRaDeInit_ = true;
1430 4 : deviceNicInitRef_.Clear();
1431 : }
1432 1385 : if (hostNicInitRef_.Count() != 0) {
1433 6 : HCCL_WARNING("host Nic is not deinit when NetworkManager Destroy. ref[%d]", hostNicInitRef_.Count());
1434 6 : struct RaInitConfig config = { DEFAULT_INIT_PHY_ID, DEFAULT_INIT_NIC_POS, DEFAULT_HDC_TYPE, isEnableHdcAsync_ };
1435 6 : config.nicPosition = static_cast<u32>(NICDeployment::NIC_DEPLOYMENT_HOST);
1436 6 : config.phyId = devicePhyId_;
1437 6 : if (HrtRaDeInit(&config) != HCCL_SUCCESS) {
1438 1 : HCCL_ERROR("ra deinit failed. para: nicdeploy[%u], phyId[%u]", config.nicPosition, config.phyId);
1439 : }
1440 6 : hostNicInitRef_.Clear();
1441 : }
1442 1385 : HCCL_INFO("destroy all nic/vnic.");
1443 1385 : return HCCL_SUCCESS;
1444 : }
1445 :
1446 6 : HcclResult NetworkManager::StopNicsSocketListen(const HcclIpAddress &ipAddr, u32 port)
1447 : {
1448 6 : auto it = raResourceInfo_.nicSocketMap.find(ipAddr);
1449 6 : CHK_PRT_RET(it == raResourceInfo_.nicSocketMap.end(),
1450 : HCCL_ERROR("[Stop][NicsSocketPort]ip[%s] port[%u] is not found in nicSocketMap.", ipAddr.GetReadableAddress(),
1451 : port),
1452 : HCCL_E_INTERNAL);
1453 6 : IpSocket &ipSock = it->second;
1454 6 : bool portFound = false;
1455 13 : for (auto itPort : ipSock.listenedPort) {
1456 7 : if (itPort == port) {
1457 6 : portFound = true;
1458 : }
1459 : }
1460 6 : CHK_PRT_RET(!portFound,
1461 : HCCL_ERROR("[Stop][NicsSocketPort]PORT(ip[%s] port[%u]) is not found.", ipAddr.GetReadableAddress(), port),
1462 : HCCL_E_INTERNAL);
1463 6 : if (ipSock.nicSocketHandle != nullptr && StopListenSocket(ipSock.nicSocketHandle, port)) {
1464 1 : HCCL_ERROR("[Stop][NicsSocketPort]NIC socket listen is not stopped successfully");
1465 1 : return HCCL_E_NETWORK;
1466 : }
1467 5 : return HCCL_SUCCESS;
1468 : }
1469 :
1470 40 : HcclResult NetworkManager::StopNicsSocket(const HcclIpAddress &ipAddr)
1471 : {
1472 40 : auto it = raResourceInfo_.nicSocketMap.find(ipAddr);
1473 40 : CHK_PRT_RET(it == raResourceInfo_.nicSocketMap.end(),
1474 : HCCL_ERROR("[Stop][NicsSocket]ip[%s] is not found in nicSocketMap.", ipAddr.GetReadableAddress()),
1475 : HCCL_E_INTERNAL);
1476 40 : IpSocket &ipSock = it->second;
1477 40 : if (ipSock.nicRdmaHandle != nullptr && HrtRaRdmaDeInit(ipSock.nicRdmaHandle, notifyType_)) {
1478 2 : HCCL_ERROR("[Stop][NicsSocket]NIC rdev deInit not successfully, notifyType_[%d]", notifyType_);
1479 2 : return HCCL_E_NETWORK;
1480 : }
1481 38 : ipSock.nicRdmaHandle = nullptr;
1482 38 : if (ipSock.nicSocketHandle != nullptr && hrtRaSocketDeInit(ipSock.nicSocketHandle)) {
1483 1 : HCCL_ERROR("[Stop][NicsSocket]NIC socket deInit not successfully");
1484 1 : return HCCL_E_NETWORK;
1485 : }
1486 37 : ipSock.nicSocketHandle = nullptr;
1487 37 : return HCCL_SUCCESS;
1488 : }
1489 :
1490 0 : HcclResult NetworkManager::InitRdmaHandle(u32 devId, const HcclIpAddress &ipAddr, bool disabledLiteThread,
1491 : bool enable910ALite)
1492 : {
1493 0 : if (raResourceInfo_.nicSocketMap[ipAddr].nicRdmaHandle != nullptr &&
1494 0 : raResourceInfo_.hostNetSocketMap[ipAddr].nicRdmaHandle != nullptr) {
1495 0 : HCCL_INFO("NetworkManager: RdmaInit already nic");
1496 0 : return HCCL_SUCCESS;
1497 : }
1498 :
1499 0 : u32 devicePhyId = ((static_cast<s32>(devId) == HOST_DEVICE_ID) ? 0 : devId);
1500 0 : RdmaHandle rdmaHandle = nullptr;
1501 : // 模式和notify类型按照是否为hdc模式进行赋值
1502 0 : NetworkMode initRdmaMode = (isHostUseDevNic_) ? NETWORK_OFFLINE : NETWORK_PEER_ONLINE;
1503 0 : NotifyTypeT notifyType = (isHostUseDevNic_) ? NotifyTypeT::NOTIFY : NotifyTypeT::NO_USE;
1504 0 : HcclResult ret = InitRDMA(devicePhyId, ipAddr, initRdmaMode, notifyType, rdmaHandle,
1505 : disabledLiteThread, enable910ALite);
1506 0 : CHK_PRT_RET(ret != HCCL_SUCCESS,
1507 : HCCL_ERROR("[Init][RDMA]errNo[0x%016llx] ra rdma init failed, ip[%s], return[%d]",
1508 : HCCL_ERROR_CODE(HCCL_E_TCP_CONNECT), ipAddr.GetReadableAddress(), ret),
1509 : HCCL_E_TCP_CONNECT);
1510 0 : HCCL_INFO("ip[%s] rdma init OK", ipAddr.GetReadableAddress());
1511 0 : CHK_PTR_NULL(rdmaHandle);
1512 0 : raResourceInfo_.nicSocketMap[ipAddr].nicRdmaHandle = rdmaHandle;
1513 0 : raResourceInfo_.hostNetSocketMap[ipAddr].nicRdmaHandle = rdmaHandle;
1514 0 : return HCCL_SUCCESS;
1515 : }
1516 :
1517 220 : HcclResult NetworkManager::GetRaResourceInfo(RaResourceInfo &raResourceInfo)
1518 : {
1519 220 : raResourceInfo = raResourceInfo_;
1520 218 : return HCCL_SUCCESS;
1521 : }
1522 :
1523 1 : HcclResult NetworkManager::PsWorkerRaInit(u32 devId, const HcclIpAddress &ipAddr, u32 port, bool isHostUseDevNic,
1524 : bool remoteIsHdc, bool isBoardVersion)
1525 : {
1526 1 : HCCL_INFO("PsWorkerRaInit, devicePhyId[%u], deviceLogicId_[%d]", devicePhyId_, deviceLogicId_);
1527 : // 引用计数
1528 1 : u32 devicePhyId = ((static_cast<s32>(devId) == HOST_DEVICE_ID) ? 0 : devId);
1529 1 : std::string ipAddrStr(ipAddr.GetReadableAddress());
1530 1 : if (ipAddrStr == "127.0.0.1") {
1531 0 : hostNicInitRef_.Ref();
1532 0 : HCCL_INFO("hostNicInitRef_[%d]", hostNicInitRef_.Count());
1533 : } else {
1534 1 : bool fistUsed{ false };
1535 1 : deviceNicInitRef_.Ref();
1536 1 : CHK_RET(hrtRaIsFirstUsed(devicePhyId, fistUsed));
1537 1 : HCCL_INFO("deviceNicInitRef_[%d] fistUsed[%u] devicePhyId[%u]", deviceNicInitRef_.Count(), fistUsed, devicePhyId);
1538 1 : if (deviceNicInitRef_.Count() == 1 && !fistUsed) {
1539 0 : isRaInitRepeated_ = true;
1540 1 : } else if (!fistUsed) {
1541 0 : HCCL_INFO("[NetworkManager] PsWorkerRa is not fistUsed");
1542 0 : return HCCL_SUCCESS;
1543 : }
1544 : }
1545 :
1546 1 : CHK_PRT_RET((raResourceInfo_.nicSocketMap.count(ipAddr) != 0),
1547 : HCCL_INFO("NetworkManager: PsWorkerRa already Init, ipAddr[%s]", ipAddr.GetReadableAddress()), HCCL_SUCCESS);
1548 1 : isHostUseDevNic_ = isHostUseDevNic;
1549 :
1550 : // hdc模式下需要先拉起device上的hccp进程
1551 1 : if (isHostUseDevNic_) {
1552 : // 拉起device进程
1553 0 : if (devId != INVALID_UINT) {
1554 0 : devicePhyId_ = devId;
1555 : } else {
1556 : // 初始化ra资源(dev信息带入逻辑ID)
1557 0 : CHK_RET(hrtGetDevice(&deviceLogicId_));
1558 0 : CHK_RET(hrtGetDevicePhyIdByIndex(static_cast<u32>(deviceLogicId_), devicePhyId_));
1559 : }
1560 :
1561 : // device 网卡初始化前需要拉起 hccp .
1562 : rtNetServiceOpenArgs openArgs;
1563 0 : rtProcExtParam extParam{};
1564 0 : std::string extPam("--hdcType=" + std::to_string(PID_HDC_TYPE));
1565 :
1566 0 : extParam.paramInfo = extPam.c_str();
1567 0 : extParam.paramLen = extPam.size();
1568 0 : openArgs.extParamList = &extParam;
1569 0 : openArgs.extParamCnt = 1UL;
1570 0 : isTsdProcessOpen_ = true;
1571 0 : CHK_RET(hrtOpenNetService(&openArgs));
1572 0 : HCCL_INFO("NetworkManager open tsd success, devicePhyId[%u], deviceLogicId_[%d], subPid[%lld]",
1573 : devicePhyId_, deviceLogicId_, static_cast<s64>(subPid_));
1574 0 : }
1575 :
1576 1 : bool isOpenWhiteList = false;
1577 1 : if (!isBoardVersion && remoteIsHdc && IsGeneralServer()) {
1578 0 : HCCL_INFO("general server, ps open WhiteList");
1579 0 : isOpenWhiteList = true;
1580 : }
1581 :
1582 1 : CHK_RET(hrtRaSocketSetWhiteListStatus(static_cast<u32>(isOpenWhiteList)));
1583 :
1584 1 : RaInitConfig config = { DEFAULT_INIT_PHY_ID, DEFAULT_INIT_NIC_POS, DEFAULT_HDC_TYPE, false };
1585 1 : config.phyId = devicePhyId;
1586 1 : if (ipAddrStr == "127.0.0.1") {
1587 0 : config.nicPosition = static_cast<u32>(NICDeployment::NIC_DEPLOYMENT_HOST);
1588 : } else {
1589 1 : config.nicPosition = static_cast<u32>(isHostUseDevNic_);
1590 : }
1591 1 : if (isHostUseDevNic_) {
1592 0 : config.hdcType = PID_HDC_TYPE;
1593 : }
1594 :
1595 1 : if (!isRaInitRepeated_) {
1596 1 : HCCL_INFO("PsWorkerRaInit call HrtRaInit. devicePhyId[%u] isRaInitRepeated_[%u]", devicePhyId_, isRaInitRepeated_);
1597 1 : HcclResult hcclRet = HrtRaInit(&config);
1598 1 : if (hcclRet != HCCL_SUCCESS) {
1599 0 : HCCL_ERROR("[PsWorkerRaInit] ra init failed, ret[%d].", hcclRet);
1600 0 : if (isTsdProcessOpen_) {
1601 0 : (void)hrtCloseNetService();
1602 0 : isTsdProcessOpen_ = false;
1603 : }
1604 0 : return hcclRet;
1605 : }
1606 : }
1607 :
1608 1 : struct rdev nicRdevInfo = {};
1609 1 : nicRdevInfo.phyId = devicePhyId;
1610 1 : nicRdevInfo.family = ipAddr.GetFamily();
1611 1 : nicRdevInfo.localIp.addr = ipAddr.GetBinaryAddress().addr;
1612 1 : nicRdevInfo.localIp.addr6 = ipAddr.GetBinaryAddress().addr6;
1613 1 : SocketHandle socketHandle = nullptr;
1614 1 : NetworkMode raSocketInitMode = (isHostUseDevNic_) ? NETWORK_OFFLINE : NETWORK_PEER_ONLINE;
1615 1 : HcclResult ret = hrtRaSocketInit(raSocketInitMode, nicRdevInfo, socketHandle);
1616 1 : if (ret != HCCL_SUCCESS) {
1617 1 : HCCL_ERROR("[Init][HostSocket]errNo[0x%016llx] ra socket init failed, ip[%s], return[%d]",
1618 : HCCL_ERROR_CODE(HCCL_E_TCP_CONNECT), ipAddr.GetReadableAddress(), ret);
1619 1 : if (!isRaInitRepeated_) {
1620 1 : (void)HrtRaDeInit(&config);
1621 : }
1622 1 : if (isTsdProcessOpen_) {
1623 0 : (void)hrtCloseNetService();
1624 0 : isTsdProcessOpen_ = false;
1625 : }
1626 1 : return HCCL_E_TCP_CONNECT;
1627 : }
1628 0 : HCCL_INFO("ip[%s] socket init OK, devicePhyId_[%u], socketHandle[%llu]", ipAddr.GetReadableAddress(), devicePhyId_,
1629 : hash<void *>{}(socketHandle));
1630 :
1631 0 : IpSocket ipSocketInfo;
1632 0 : ipSocketInfo.nicSocketHandle = socketHandle;
1633 :
1634 0 : raResourceInfo_.nicSocketMap.insert(std::make_pair(ipAddr, ipSocketInfo));
1635 0 : raResourceInfo_.hostNetSocketMap.insert(std::make_pair(ipAddr, ipSocketInfo));
1636 :
1637 0 : ret = HeterogStartListen(ipAddr, port);
1638 0 : if (ret != HCCL_SUCCESS) {
1639 0 : HCCL_ERROR("[PsWorkerRaInit] HeterogStartListen failed, ret[%d]", ret);
1640 0 : (void)hrtRaSocketDeInit(socketHandle);
1641 0 : raResourceInfo_.nicSocketMap.erase(ipAddr);
1642 0 : raResourceInfo_.hostNetSocketMap.erase(ipAddr);
1643 0 : if (!isRaInitRepeated_) {
1644 0 : (void)HrtRaDeInit(&config);
1645 : }
1646 0 : if (isTsdProcessOpen_) {
1647 0 : (void)hrtCloseNetService();
1648 0 : isTsdProcessOpen_ = false;
1649 : }
1650 0 : return ret;
1651 : }
1652 :
1653 0 : return HCCL_SUCCESS;
1654 1 : }
1655 :
1656 : // 最后一次ra_deinit时才关闭device的hccp进程。ES场景主要使用
1657 0 : HcclResult NetworkManager::CloseHccpSubProc()
1658 : {
1659 0 : if (!isHostUseDevNic_ || subPid_ == 0) {
1660 0 : HCCL_INFO("No need to close hccp sub proc, devicePhyId[%u], subPid[%lld]",
1661 : devicePhyId_, subPid_);
1662 0 : return HCCL_SUCCESS;
1663 : }
1664 0 : HCCL_INFO("NetworkManager ProcessCloseSubProcList HDC devicePhyId[%u], deviceLogicId_[%d], subPid[%lld]",
1665 : devicePhyId_, deviceLogicId_, static_cast<s64>(subPid_));
1666 0 : s32 locaLogDevid = 0;
1667 0 : hrtGetDevice(&locaLogDevid);
1668 0 : if (locaLogDevid != deviceLogicId_) {
1669 0 : hrtSetDevice(deviceLogicId_);
1670 : }
1671 0 : HcclResult ret = hrtCloseNetService();
1672 0 : if (locaLogDevid != deviceLogicId_) {
1673 0 : hrtSetDevice(locaLogDevid);
1674 : }
1675 0 : CHK_RET(ret);
1676 0 : subPid_ = 0;
1677 :
1678 0 : return HCCL_SUCCESS;
1679 : }
1680 :
1681 16 : HcclResult NetworkManager::PingMeshRaPingInit(u32 devLogicId, u32 devPhyId, RaInitConfig *config)
1682 : {
1683 : // 引用计数
1684 16 : deviceLogicId_ = static_cast<s32>(devLogicId);
1685 16 : devicePhyId_ = ((static_cast<s32>(devPhyId) == HOST_DEVICE_ID) ? 0 : devPhyId);
1686 16 : isHostUseDevNic_ = true;
1687 16 : isRaInitRepeated_ = false;
1688 :
1689 : // hccp侧初始化ping mesh资源
1690 16 : CHK_RET(HrtRaInit(config));
1691 16 : HCCL_INFO("[HCCN][PingMeshRaPingInit]Device[%u] config.hdcType[%d], config.nicPosition[%u], config.phyId[%u].",
1692 : deviceLogicId_, config->hdcType, config->nicPosition, config->phyId);
1693 16 : deviceNicInitRef_.Ref();
1694 :
1695 16 : return HCCL_SUCCESS;
1696 : }
1697 :
1698 15 : HcclResult NetworkManager::PingMeshRaPingDeinit()
1699 : {
1700 : // 引用计数
1701 15 : isRaInitRepeated_ = false;
1702 :
1703 15 : struct RaInitConfig config = { DEFAULT_INIT_PHY_ID, DEFAULT_INIT_NIC_POS, DEFAULT_HDC_TYPE, false };
1704 15 : GetDeviceRaInitConfig(config);
1705 15 : CHK_RET(HrtRaDeInit(&config));
1706 15 : deviceNicInitRef_.Unref();
1707 :
1708 15 : return HCCL_SUCCESS;
1709 : }
1710 :
1711 19 : void NetworkManager::GetDeviceRaInitConfig(RaInitConfig &config)
1712 : {
1713 19 : u32 devicePhyId = ((static_cast<s32>(devicePhyId_) == HOST_DEVICE_ID) ? 0 : devicePhyId_);
1714 19 : HCCL_INFO("RaDeinit devicePhyId_[%u] devicePhyId[%u]", devicePhyId_, devicePhyId);
1715 :
1716 19 : config.phyId = devicePhyId;
1717 19 : config.nicPosition = static_cast<u32>(isHostUseDevNic_);
1718 :
1719 19 : if (isHostUseDevNic_) {
1720 16 : config.hdcType = PID_HDC_TYPE;
1721 : }
1722 19 : }
1723 :
1724 0 : HcclResult NetworkManager::PsWorkerRaDeinit(u32 devId, const HcclIpAddress &ipAddr, u32 port)
1725 : {
1726 0 : string ipAddrStr(ipAddr.GetReadableAddress());
1727 0 : u32 devicePhyId = ((static_cast<s32>(devicePhyId_) == HOST_DEVICE_ID) ? 0 : devicePhyId_);
1728 0 : if (ipAddrStr == "127.0.0.1") {
1729 0 : hostNicInitRef_.Unref();
1730 0 : HCCL_INFO("hostNicInitRef_[%d]", hostNicInitRef_.Count());
1731 : } else {
1732 0 : bool lastUsed{ false };
1733 0 : deviceNicInitRef_.Unref();
1734 0 : CHK_RET(hrtRaIsLastUsed(devicePhyId, lastUsed));
1735 0 : HCCL_INFO("deviceNicInitRef_[%d] lastUsed[%u] devicePhyId[%u]", deviceNicInitRef_.Count(), lastUsed, devicePhyId);
1736 0 : if (deviceNicInitRef_.Count() == 0 && !lastUsed) {
1737 0 : isRaInitRepeated_ = true;
1738 0 : } else if (lastUsed) {
1739 0 : isRaInitRepeated_ = false;
1740 0 : } else if (deviceNicInitRef_.Count() > 0) {
1741 0 : HCCL_INFO("[NetworkManager] PsWorkerRa is not lastUsed");
1742 0 : return HCCL_SUCCESS;
1743 : }
1744 : }
1745 0 : CHK_PRT_RET((raResourceInfo_.nicSocketMap.count(ipAddr) == 0),
1746 : HCCL_INFO("NetworkManager: PsWorkerRa already Deinit, ipAddr[%s]", ipAddr.GetReadableAddress()), HCCL_SUCCESS);
1747 :
1748 0 : HCCL_INFO("PsWorkerRaDeinit devId[%u], isRaInitRepeated[%d]", devId, isRaInitRepeated_);
1749 : // 重复RaInit时，不再调用内部DeInit
1750 0 : if (!isRaInitRepeated_) {
1751 0 : CHK_RET(HeterogStopListen(ipAddr, port, true));
1752 : }
1753 :
1754 0 : SocketHandle nicRdmaHandle = raResourceInfo_.nicSocketMap[ipAddr].nicRdmaHandle;
1755 0 : if (!GetExternalInputHcclIsTcpMode() && nicRdmaHandle != nullptr && !isRaInitRepeated_) {
1756 : // Helper的PS临时暂不调用 CHK_RET(HrtRaRdmaDeInit(nicRdmaHandle, (isHostUseDevNic_) ?
1757 : // NotifyTypeT::NOTIFY : NotifyTypeT::NO_USE));
1758 0 : HCCL_INFO("Not call RaRdmaDeInit devicePhyId[%u]", devicePhyId_);
1759 : }
1760 :
1761 0 : struct RaInitConfig config = { DEFAULT_INIT_PHY_ID, DEFAULT_INIT_NIC_POS, DEFAULT_HDC_TYPE, false };
1762 0 : GetDeviceRaInitConfig(config);
1763 :
1764 0 : if (ipAddrStr == "127.0.0.1") {
1765 0 : if (hostNicInitRef_.Count() == 0) {
1766 0 : HCCL_INFO("PsWorkerRaDeinit call hrtRaDeInit. devicePhyId[%u]", devicePhyId_);
1767 0 : config.nicPosition = static_cast<u32>(NICDeployment::NIC_DEPLOYMENT_HOST);
1768 0 : CHK_RET(HrtRaDeInit(&config));
1769 : }
1770 : } else {
1771 0 : if (deviceNicInitRef_.Count() == 0) {
1772 0 : HCCL_INFO("PsWorkerRaDeinit call HrtRaDeInit. devicePhyId[%u]", devicePhyId_);
1773 0 : if (!isRaInitRepeated_) {
1774 0 : CHK_RET(HrtRaDeInit(&config));
1775 0 : CHK_RET(CloseHccpSubProc());
1776 : }
1777 : }
1778 : }
1779 :
1780 0 : raResourceInfo_.nicSocketMap.erase(raResourceInfo_.nicSocketMap.find(ipAddr));
1781 0 : raResourceInfo_.hostNetSocketMap.erase(raResourceInfo_.hostNetSocketMap.find(ipAddr));
1782 :
1783 0 : return HCCL_SUCCESS;
1784 0 : }
1785 :
1786 22 : HcclResult NetworkManager::InitHostSocket(const HcclIpAddress &addr, SocketHandle &socketHandle) const
1787 : {
1788 22 : struct SocketInitInfoT socketInitInfo = {};
1789 22 : socketInitInfo.rdevInfo.family = addr.GetFamily();
1790 22 : socketInitInfo.rdevInfo.phyId = devicePhyId_;
1791 22 : socketInitInfo.rdevInfo.localIp.addr = addr.GetBinaryAddress().addr;
1792 22 : socketInitInfo.rdevInfo.localIp.addr6 = addr.GetBinaryAddress().addr6;
1793 22 : socketInitInfo.scopeId = addr.GetScopeID();
1794 22 : HcclResult ret = hrtRaSocketInitV1(NETWORK_PEER_ONLINE, socketInitInfo, socketHandle);
1795 22 : CHK_PRT_RET(ret != HCCL_SUCCESS,
1796 : HCCL_ERROR("[Init][HostSocket]errNo[0x%016llx] ra socket init v1 failed, ip[%s], return[%d]",
1797 : HCCL_ERROR_CODE(HCCL_E_TCP_CONNECT), addr.GetReadableAddress(), ret), HCCL_E_TCP_CONNECT);
1798 22 : HCCL_INFO("ip[%s] socket init OK", addr.GetReadableAddress());
1799 22 : return HCCL_SUCCESS;
1800 : }
1801 :
1802 25 : HcclResult NetworkManager::StopHostNet(SocketHandle socketHandle, const HcclIpAddress &ipAddr)
1803 : {
1804 25 : std::unique_lock<std::mutex> lock(raLock_);
1805 25 : CHK_PRT_RET((hostNicInitRef_.Count() == 0),
1806 : HCCL_ERROR("[Stop][HostNet]cannot start nic socket before host nic inited!"), HCCL_E_INTERNAL);
1807 :
1808 25 : CHK_RET(CheckSocketInfo(socketHandle, ipAddr));
1809 :
1810 45 : if (hostNicSocketClientRef_[ipAddr].Unref() == 0 &&
1811 20 : raResourceInfo_.hostNetSocketMap[ipAddr].listenedPort.size() == 0) {
1812 20 : CHK_RET(hrtRaSocketDeInit(socketHandle));
1813 20 : HCCL_INFO("ip[%s] deinit success.", ipAddr.GetReadableAddress());
1814 :
1815 20 : raResourceInfo_.hostNetSocketMap.erase(ipAddr);
1816 20 : raResourceInfo_.nicSocketMap.erase(ipAddr);
1817 : }
1818 25 : return HCCL_SUCCESS;
1819 25 : }
1820 :
1821 : // 从ip查handle 把ip下所有的listen全stop
1822 0 : HcclResult NetworkManager::StopHostSocketHandle(const HcclIpAddress &ipAddr)
1823 : {
1824 0 : std::unique_lock<std::mutex> lock(raLock_);
1825 0 : CHK_PRT_RET((hostNicInitRef_.Count() == 0),
1826 : HCCL_ERROR("[NetworkManager][StopHostSocketHandle]cannot start nic socket before host nic inited!"), HCCL_E_INTERNAL);
1827 :
1828 0 : auto sockInfo = raResourceInfo_.hostNetSocketMap.find(ipAddr);
1829 0 : auto ipIt = IPPortListenRefMapHost_.find(ipAddr);
1830 0 : CHK_PRT_RET((sockInfo == raResourceInfo_.hostNetSocketMap.end()),
1831 : HCCL_ERROR("[NetworkManager][StopHostSocketHandle]ipAddr is invalid"), HCCL_E_INTERNAL);
1832 0 : IpSocket &sock = raResourceInfo_.hostNetSocketMap[ipAddr];
1833 0 : CHK_RET(CheckSocketInfo(sock.nicSocketHandle, ipAddr));
1834 :
1835 : // 停止所有的listen
1836 0 : if (ipIt != IPPortListenRefMapHost_.end()) {
1837 0 : for (auto &portIt : ipIt->second) {
1838 0 : u32 port = portIt.first;
1839 0 : if (IPPortListenRefMapHost_[ipAddr][port].Count() > 0) {
1840 0 : CHK_RET(StopListenSocket(sock.nicSocketHandle, port));
1841 0 : HCCL_WARNING("[NetworkManager][StopHostSocketHandle] ip [%s] stop listen port [%u]", ipAddr.GetReadableAddress(), port);
1842 0 : raResourceInfo_.hostNetSocketMap[ipAddr].listenedPort.erase(port);
1843 0 : IPPortListenRefMapHost_[ipAddr][port].Clear(); // 引用计数归0
1844 : }
1845 : }
1846 : }
1847 0 : CHK_PRT_RET((raResourceInfo_.hostNetSocketMap[ipAddr].listenedPort.size() != 0),
1848 : HCCL_ERROR("[NetworkManager][StopHostSocketHandle]IPPortListenRefMapHost_[%s] is unequal to hostNetSocketMap[%s].listenedPort",
1849 : ipAddr.GetReadableAddress(), ipAddr.GetReadableAddress()),
1850 : HCCL_E_INTERNAL);
1851 : // 删除socket 删除IP
1852 0 : CHK_RET(hrtRaSocketDeInit(sock.nicSocketHandle));
1853 0 : HCCL_INFO("[NetworkManager][StopHostSocketHandle] ip [%s] deinit success.", ipAddr.GetReadableAddress());
1854 0 : sock.nicSocketHandle = nullptr;
1855 : // 没有host和rdma时 删除ip
1856 0 : if (sock.nicSocketHandle == nullptr && sock.nicRdmaHandle == nullptr) {
1857 0 : raResourceInfo_.hostNetSocketMap.erase(ipAddr);
1858 : }
1859 0 : hostNicSocketClientRef_[ipAddr].Clear();
1860 0 : return HCCL_SUCCESS;
1861 0 : }
1862 :
1863 25 : HcclResult NetworkManager::StartHostNet(const HcclIpAddress &ipAddr, SocketHandle &socketHandle)
1864 : {
1865 25 : CHK_RET(CreateHostSocketHandle(ipAddr, socketHandle));
1866 25 : return HCCL_SUCCESS;
1867 : }
1868 :
1869 25 : HcclResult NetworkManager::CreateHostSocketHandle(const HcclIpAddress &ipAddr, SocketHandle &socketHandle)
1870 : {
1871 25 : CHK_PRT_RET((hostNicInitRef_.Count() == 0),
1872 : HCCL_ERROR("[CreateHostSocketHandle]cannot start nic socket before host nic inited!"), HCCL_E_INTERNAL);
1873 25 : OccupyIp(ipAddr, raResourceInfo_.hostNetSocketMap);
1874 25 : IpSocket &sock = raResourceInfo_.hostNetSocketMap[ipAddr];
1875 25 : if (sock.nicSocketHandle == nullptr) {
1876 20 : CHK_RET(InitHostSocket(ipAddr, sock.nicSocketHandle));
1877 : }
1878 :
1879 25 : socketHandle = sock.nicSocketHandle;
1880 25 : HCCL_INFO("ip[%s] socket start success socketHandle[%p]", ipAddr.GetReadableAddress(), socketHandle);
1881 25 : hostNicSocketClientRef_[ipAddr].Ref(); // 引用计数
1882 25 : return HCCL_SUCCESS;
1883 : }
1884 :
1885 41 : HcclResult NetworkManager::StartListenSocket(const SocketHandle socketHandle, u32 &port) const
1886 : {
1887 41 : struct SocketListenInfoT serverInfo = {};
1888 41 : serverInfo.socketHandle = const_cast<SocketHandle>(socketHandle);
1889 41 : serverInfo.port = port;
1890 41 : if (isRaInitRepeated_) {
1891 0 : return HCCL_SUCCESS;
1892 : }
1893 41 : bool isAutoPort = port == AUTO_LISTEN_PORT;
1894 41 : HcclResult ret = hrtRaSocketListenStart(&serverInfo, 1);
1895 41 : CHK_PRT_RET(ret == HCCL_E_UNAVAIL,
1896 : HCCL_INFO("socket port[%u] has already been bound. Could not start listening host nic. please use an idle port.",
1897 : port), ret);
1898 41 : RPT_CALL_ERR(ret != HCCL_SUCCESS,
1899 : "host nic listen start failed, port[%u], return[%d]", port, ret);
1900 41 : CHK_PRT_RET(ret != HCCL_SUCCESS,
1901 : HCCL_ERROR("host nic listen start failed, port[%u], return[%d]", port, ret), ret);
1902 41 : if (isAutoPort) {
1903 0 : port = serverInfo.port;
1904 0 : CHK_PRT_RET(port == AUTO_LISTEN_PORT,
1905 : HCCL_ERROR("start listen on a port selected by os automatically failed"),
1906 : HCCL_E_NOT_SUPPORT);
1907 0 : HCCL_RUN_INFO("start listen on port[%u] by auto success.", port);
1908 : }
1909 41 : return HCCL_SUCCESS;
1910 : }
1911 :
1912 39 : HcclResult NetworkManager::StopListenSocket(const SocketHandle socketHandle, u32 port) const
1913 : {
1914 : struct SocketListenInfoT serverInfo;
1915 39 : serverInfo.socketHandle = const_cast<SocketHandle>(socketHandle);
1916 39 : serverInfo.port = port;
1917 39 : HcclResult ret = hrtRaSocketListenStop(&serverInfo, 1);
1918 39 : RPT_CALL_ERR(ret != HCCL_SUCCESS, "socket listen stop failed, port[%u], return[%d]", port, ret);
1919 39 : CHK_PRT_RET(ret != HCCL_SUCCESS, HCCL_ERROR("socket listen stop failed, port[%u], return[%d]", port, ret), ret);
1920 39 : return HCCL_SUCCESS;
1921 : }
1922 :
1923 32 : HcclResult NetworkManager::GetRdmaHandleByIpAddr(const HcclIpAddress &ipAddr, RdmaHandle &rdmaHandle)
1924 : {
1925 32 : auto it = raResourceInfo_.nicSocketMap.find(ipAddr);
1926 32 : CHK_PRT_RET(it == raResourceInfo_.nicSocketMap.end(),
1927 : HCCL_ERROR("GetRdmaHandleByIpAddr ip[%s] is not found in nicSocketMap.", ipAddr.GetReadableAddress()),
1928 : HCCL_E_INTERNAL);
1929 32 : rdmaHandle = raResourceInfo_.nicSocketMap[ipAddr].nicRdmaHandle;
1930 32 : CHK_PTR_NULL(rdmaHandle);
1931 32 : return HCCL_SUCCESS;
1932 : }
1933 :
1934 0 : HcclResult NetworkManager::GetNicHandleByIpAddr(const HcclIpAddress &ipAddr, SocketHandle &nicHandle)
1935 : {
1936 0 : auto it = raResourceInfo_.nicSocketMap.find(ipAddr);
1937 0 : CHK_PRT_RET(it == raResourceInfo_.nicSocketMap.end(),
1938 : HCCL_ERROR("GetNicHandleByIpAddr ip[%s] is not found in nicSocketMap.", ipAddr.GetReadableAddress()),
1939 : HCCL_E_INTERNAL);
1940 0 : nicHandle = raResourceInfo_.nicSocketMap[ipAddr].nicSocketHandle;
1941 0 : CHK_PTR_NULL(nicHandle);
1942 0 : return HCCL_SUCCESS;
1943 : }
1944 :
1945 21 : HcclResult NetworkManager::CheckAutoListenVersion(bool isAutoPort)
1946 : {
1947 21 : if (isAutoPort) {
1948 1 : u32 listenStartVersion = 0;
1949 1 : HcclResult vRet = hrtRaGetInterfaceVersion(devicePhyId_, SOCKET_LISTEN_AUTO_INTERFACE, &listenStartVersion);
1950 1 : HCCL_INFO("[CheckAutoListenVersion] listen start version[%u].", listenStartVersion);
1951 1 : CHK_PRT_RET(vRet != HCCL_SUCCESS || listenStartVersion < SOCKET_LISTEN_AUTO_INTERFACE_VERSION,
1952 : HCCL_ERROR("this package does not support hrtRaSocketNonBlockListenStart to "
1953 : "listen automatically with port %u, please change new package.", AUTO_LISTEN_PORT),
1954 : HCCL_E_NOT_SUPPORT);
1955 : }
1956 20 : return HCCL_SUCCESS;
1957 : }
1958 :
1959 : // 新旧ip类型转换
1960 0 : HcclResult NetworkManager::HcclIpAddressConvertHcclAddr(HcclAddress *hccladdr, HcclIpAddress *hcclIP) {
1961 0 : CHK_PTR_NULL(hcclIP);
1962 0 : CHK_PTR_NULL(hccladdr);
1963 0 : if (hcclIP->GetFamily() == AF_INET) {
1964 0 : hccladdr->type = HCCL_ADDR_TYPE_IP_V4;
1965 0 : hccladdr->addr = hcclIP->GetBinaryAddress().addr;
1966 0 : } else if (hcclIP->GetFamily() == AF_INET6) {
1967 0 : hccladdr->type = HCCL_ADDR_TYPE_IP_V6;
1968 0 : hccladdr->addr6 = hcclIP->GetBinaryAddress().addr6;
1969 : } else {
1970 0 : HCCL_ERROR("[HcclIpAddressConvertingHcclAddr]ERROR IP type!");
1971 0 : return HCCL_E_PARA;
1972 : }
1973 0 : return HCCL_SUCCESS;
1974 : }
1975 :
1976 25 : HcclResult NetworkManager::OccupyIp(const HcclIpAddress &ipAddr, std::map<hccl::HcclIpAddress, IpSocket> &socketMap)
1977 : {
1978 25 : auto sockInfo = socketMap.find(ipAddr);
1979 25 : if (sockInfo == socketMap.end()) {
1980 20 : IpSocket tempSock;
1981 20 : socketMap.insert(std::make_pair(ipAddr, tempSock)); // 本IP占位
1982 20 : HCCL_INFO("[NetworkManager][OccupyIp] device[%u] insert ip[%s]", devicePhyId_, ipAddr.GetReadableAddress());
1983 20 : } else {
1984 5 : HCCL_INFO("[NetworkManager][OccupyIp] device[%u] ip[%s] has already occupied.", devicePhyId_, ipAddr.GetReadableAddress());
1985 : }
1986 25 : return HCCL_SUCCESS;
1987 : }
1988 : }
|
