Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "p2p_enable_manager.h"
12 : #include "log.h"
13 : #include "env_config_v2.h"
14 :
15 : namespace Hccl {
16 :
17 888 : P2PEnableManager& P2PEnableManager::GetInstance()
18 : {
19 888 : static P2PEnableManager p2pEnableManager;
20 888 : return p2pEnableManager;
21 : }
22 :
23 39 : HcclResult P2PEnableManager::EnableP2P(std::vector<uint32_t> remoteDevices)
24 : {
25 39 : auto localDeviceLogicID = HrtGetDevice();
26 :
27 42 : for (auto remoteDevicePhysicID : remoteDevices) {
28 3 : CHK_RET(EnableP2P(localDeviceLogicID, remoteDevicePhysicID));
29 : }
30 39 : return HCCL_SUCCESS;
31 : }
32 :
33 3 : HcclResult P2PEnableManager::EnableP2P(uint32_t localDeviceLogicID, uint32_t remoteDevicePhysicID)
34 : {
35 3 : std::unique_lock<std::mutex> lock(connectionsLock_[localDeviceLogicID]);
36 3 : auto& iterLocalDevice = connectionsInfo_[localDeviceLogicID];
37 3 : auto iterRemoteDevice = iterLocalDevice.find(remoteDevicePhysicID);
38 3 : if ((iterRemoteDevice == iterLocalDevice.end()) || (iterRemoteDevice->second.reference == 0)) {
39 3 : auto localDevicePhysicID = HrtGetDevicePhyIdByIndex(localDeviceLogicID);
40 3 : CHK_RET(HrtEnableP2P(localDeviceLogicID, remoteDevicePhysicID));
41 9 : HCCL_INFO(
42 : "[EnableP2P]enable p2p: local logic id:%u, local physic id:%u, remote physic id:%u.", localDeviceLogicID,
43 : localDevicePhysicID, remoteDevicePhysicID);
44 3 : iterLocalDevice[remoteDevicePhysicID].status = P2PStatus::P2P_STATUS_ENABLING;
45 3 : iterLocalDevice[remoteDevicePhysicID].reference++;
46 3 : return HCCL_SUCCESS;
47 : } else {
48 : // 使已执行过 enable，且未执行过 disable，不重复执行 enable p2p。
49 0 : iterLocalDevice[remoteDevicePhysicID].reference++;
50 : }
51 0 : return HCCL_SUCCESS;
52 3 : }
53 :
54 0 : HcclResult P2PEnableManager::WaitP2PEnabled(std::vector<uint32_t> remoteDevices)
55 : {
56 0 : auto localDeviceLogicID = HrtGetDevice();
57 :
58 0 : for (auto& remoteDevicePhysicID : remoteDevices) {
59 0 : CHK_RET(WaitP2PEnabled(localDeviceLogicID, remoteDevicePhysicID));
60 : }
61 0 : return HCCL_SUCCESS;
62 : }
63 :
64 0 : HcclResult P2PEnableManager::WaitP2PEnabled(uint32_t localDeviceLogicID, uint32_t remoteDevicePhysicID)
65 : {
66 0 : std::unique_lock<std::mutex> lock(connectionsLock_[localDeviceLogicID]);
67 :
68 0 : auto& iterLocalDevice = connectionsInfo_[localDeviceLogicID];
69 0 : auto iterRemoteDevice = iterLocalDevice.find(remoteDevicePhysicID);
70 0 : bool bErr = (iterRemoteDevice == iterLocalDevice.end()) || (iterRemoteDevice->second.reference == 0)
71 0 : || (iterRemoteDevice->second.status == P2PStatus::P2P_STATUS_DISABLED);
72 0 : CHK_PRT_RET(
73 : bErr,
74 : HCCL_ERROR(
75 : "[Wait][P2PEnabled]wait p2p enabled failed. enable operation has not been executed, "
76 : "ret[%u]. device info: local logic id:%u, remote physic id:%u.",
77 : HCCL_E_INTERNAL, localDeviceLogicID, remoteDevicePhysicID),
78 : HCCL_E_INTERNAL);
79 :
80 0 : if (iterRemoteDevice->second.status == P2PStatus::P2P_STATUS_ENABLED) {
81 0 : return HCCL_SUCCESS;
82 : } else {
83 0 : auto localDevicePhysicID = HrtGetDevicePhyIdByIndex(localDeviceLogicID);
84 :
85 0 : CHK_RET(WaitP2PConnected(localDeviceLogicID, remoteDevicePhysicID));
86 0 : HCCL_INFO(
87 : "[Wait]enable p2p: local logic id:%u, local physic id:%u, remote physic id:%u.", localDeviceLogicID,
88 : localDevicePhysicID, remoteDevicePhysicID);
89 0 : iterLocalDevice[remoteDevicePhysicID].status = P2PStatus::P2P_STATUS_ENABLED;
90 : }
91 0 : return HCCL_SUCCESS;
92 0 : }
93 :
94 0 : HcclResult P2PEnableManager::WaitP2PConnected(uint32_t localDeviceLogicID, uint32_t remoteDevicePhysicID)
95 : {
96 : // 读取P2P状态超时时间
97 0 : const std::chrono::seconds timeout(EnvConfig::GetInstance().GetSocketConfig().GetLinkTimeOut());
98 0 : const std::chrono::milliseconds checkP2PTimeInterval(1); // 轮询P2P状态时间 1ms
99 0 : const auto start = TIME_NOW();
100 :
101 : while (true) {
102 0 : uint32_t status = DRV_P2P_STATUS_DISABLE;
103 0 : CHK_RET(HrtGetP2PStatus(localDeviceLogicID, remoteDevicePhysicID, &status));
104 :
105 0 : if (status == DRV_P2P_STATUS_ENABLE) {
106 0 : HCCL_INFO(
107 : "connected p2p success, take time [%lld]us. device info: local logic id:%u, remote physic id:%u.",
108 : DURATION_US(TIME_NOW() - start), localDeviceLogicID, remoteDevicePhysicID);
109 0 : return HCCL_SUCCESS;
110 : }
111 0 : std::this_thread::sleep_for(checkP2PTimeInterval);
112 : /* 获取当前时间，如果耗时超过timeout，则返回错误 */
113 0 : const auto elapsed = std::chrono::duration_cast<std::chrono::seconds>(TIME_NOW() - start);
114 0 : if (elapsed > timeout) {
115 0 : HCCL_ERROR(
116 : "[Wait][P2PConnected]connected p2p timeout, timeout:%d s. local logicDevid:%u, "
117 : "remote physic id:%u.",
118 : EnvConfig::GetInstance().GetSocketConfig().GetLinkTimeOut(), localDeviceLogicID, remoteDevicePhysicID);
119 0 : return HCCL_E_DRV;
120 : }
121 0 : }
122 : return HCCL_SUCCESS;
123 : }
124 :
125 848 : HcclResult P2PEnableManager::DisableP2P(uint32_t localDeviceLogicID, std::vector<uint32_t> remoteDevices)
126 : {
127 : try {
128 854 : for (auto& remoteDevicePhysicID : remoteDevices) {
129 6 : CHK_RET(DisableP2P(localDeviceLogicID, remoteDevicePhysicID));
130 : }
131 0 : } catch (HcclException& e) {
132 0 : HCCL_ERROR("%s", e.what());
133 0 : return e.GetErrorCode();
134 0 : } catch (...) {
135 0 : HCCL_ERROR("Unknown error occurs!");
136 0 : return HcclResult::HCCL_E_INTERNAL;
137 0 : }
138 848 : return HCCL_SUCCESS;
139 : }
140 :
141 6 : HcclResult P2PEnableManager::DisableP2P(uint32_t localDeviceLogicID, uint32_t remoteDevicePhysicID)
142 : {
143 6 : std::unique_lock<std::mutex> lock(connectionsLock_[localDeviceLogicID]);
144 6 : auto& iterLocalDevice = connectionsInfo_[localDeviceLogicID];
145 6 : auto iterRemoteDevice = iterLocalDevice.find(remoteDevicePhysicID);
146 6 : if ((iterRemoteDevice == iterLocalDevice.end()) || (iterRemoteDevice->second.reference == 0)) {
147 18 : HCCL_WARNING(
148 : "there is no p2p connections, no need to disable p2p. "
149 : "device info: local logic id:%u, remote physic id:%u.",
150 : localDeviceLogicID, remoteDevicePhysicID);
151 6 : return HCCL_SUCCESS;
152 : }
153 :
154 0 : iterRemoteDevice->second.reference--;
155 0 : if (iterRemoteDevice->second.reference == 0) {
156 0 : auto localDevicePhysicID = HrtGetDevicePhyIdByIndex(localDeviceLogicID);
157 0 : HCCL_INFO(
158 : "disable p2p: local logic id:%u, local physic id:%u, remote physic id:%u.", localDeviceLogicID,
159 : localDevicePhysicID, remoteDevicePhysicID);
160 0 : CHK_RET(HrtDisableP2P(localDeviceLogicID, remoteDevicePhysicID));
161 0 : iterLocalDevice[remoteDevicePhysicID].status = P2PStatus::P2P_STATUS_DISABLED;
162 : }
163 0 : return HCCL_SUCCESS;
164 6 : }
165 :
166 1 : P2PEnableManager::~P2PEnableManager() {}
167 :
168 : } // namespace Hccl
|
