Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "hcomm_res_mgr.h"
12 :
13 : #include <mutex>
14 :
15 : #include "hccl_common.h"
16 : #include "comm_engine_utils.h"
17 : #include "launch_device.h"
18 : #include "launch_aicpu.h"
19 :
20 : // orion 通用平台层单例
21 : #include "hccp_hdc_manager.h"
22 : #include "hccp_peer_manager.h"
23 : #include "hccp_tlv_hdc_manager.h"
24 : #include "rdma_handle_manager.h"
25 : #include "inner_net_dev_manager.h"
26 : #include "socket_handle_manager.h"
27 : #include "host_socket_handle_manager.h"
28 : #include "tp_manager.h"
29 : #include "endpoint_monitor.h"
30 : // legacy ccu单例
31 : #include "ccu_component.h"
32 : #include "ccu_res_batch_allocator_legacy.h"
33 : #include "../../../legacy/ascend950/unified_platform/ccu/ccu_context/ccu_context_mgr_imp.h"
34 : // 开源开放 ccu单例
35 : #include "hccp_tlv_hdc_mgr.h"
36 : #include "tp_mgr.h"
37 : #include "ccu_comp.h"
38 : #include "resources/ccu/ccu_device/ccu_res_batch_allocator.h"
39 : #include "ccu_kernel_mgr.h"
40 : #include "ccu_instance_mgr.h"
41 : #include "../endpoint_pairs/sockets/socket_process.h"
42 : #include "dpu_notify/dpu_notify_manager.h"
43 : #include "server_socket_mgr.h"
44 : #include "server_socket_manager.h"
45 : #include "adapter_rts_common.h"
46 :
47 : namespace hcomm {
48 :
49 : static std::mutex g_deviceResetRegMutex;
50 : static std::mutex g_deviceRefreshRegMutex;
51 : static std::mutex g_deviceRefreshMutex;
52 : static bool g_deviceRefreshCallbackRegistered = false;
53 : static bool g_deviceResetCallbackRegistered = false;
54 :
55 : aclrtBinHandle HcommResMgr::binHandle_ = nullptr;
56 : std::mutex HcommResMgr::binHandleMtx_;
57 :
58 1 : HcclResult HcommResMgr::EnsureKernelBinLoaded(CommEngine engine)
59 : {
60 1 : if (engine != COMM_ENGINE_AICPU && engine != COMM_ENGINE_AICPU_TS) {
61 0 : HCCL_INFO(
62 : "[%s] engine[%s] kernel loading not required", __func__,
63 : GetEnumToString(GetCommEngineStatusStrMap(), engine).c_str());
64 0 : return HCCL_SUCCESS;
65 : }
66 1 : std::lock_guard<std::mutex> lock(binHandleMtx_);
67 1 : if (binHandle_ != nullptr) {
68 0 : return HCCL_SUCCESS;
69 : }
70 1 : std::string jsonPath;
71 1 : CHK_RET(hccl::GetKernelFilePath(jsonPath));
72 1 : jsonPath += "ccl_kernel.json";
73 :
74 1 : HcclResult ret = hccl::LoadBinaryFromFile(jsonPath.c_str(), ACL_RT_BINARY_LOAD_OPT_CPU_KERNEL_MODE, 0, binHandle_);
75 1 : CHK_PRT_RET(
76 : ret != HCCL_SUCCESS, HCCL_ERROR("[%s] load aicpu file fail, path[%s]", __func__, jsonPath.c_str()), ret);
77 1 : return HCCL_SUCCESS;
78 1 : }
79 :
80 1 : aclrtBinHandle HcommResMgr::GetBinHandle() { return binHandle_; }
81 :
82 : // HcommBaseResMgr
83 :
84 1 : void HcommBaseResMgr::Init()
85 : {
86 : // 临时方案：只声明单例对象做生命周期控制，不执行业务动作
87 : // 未来需要将各种单例转为该数据结构的成员变量
88 : // devicePhyId 目前不影响流程，只是触发静态对象声明
89 1 : DpuNotifyManager::GetInstance();
90 1 : Hccl::HccpHdcManager::GetInstance();
91 1 : Hccl::HccpPeerManager::GetInstance();
92 1 : Hccl::HccpTlvHdcManager::GetInstance();
93 1 : Hccl::RdmaHandleManager::GetInstance();
94 1 : Hccl::InnerNetDevManager::GetInstance();
95 1 : Hccl::SocketHandleManager::GetInstance();
96 1 : Hccl::HostSocketHandleManager::GetInstance();
97 1 : SocketMgr::GetInstance(devPhyId_);
98 1 : Hccl::TpManager::GetInstance(devPhyId_);
99 1 : (void)EndpointMonitor::GetHolder(devPhyId_);
100 :
101 1 : Hccl::CcuComponent::GetInstance(devPhyId_);
102 1 : Hccl::CcuResBatchAllocator::GetInstance(devPhyId_);
103 1 : Hccl::CtxMgrImp::GetInstance(devPhyId_);
104 :
105 : // 开源开放架构下CCU模式新增类型单例，当前混跑时不使用
106 1 : HccpTlvHdcMgr::GetInstance(devPhyId_);
107 1 : TpMgr::GetInstance(devPhyId_);
108 1 : CcuComponent::GetInstance(devPhyId_);
109 1 : CcuResBatchAllocator::GetInstance(devPhyId_);
110 1 : CcuKernelMgr::GetInstance(devPhyId_);
111 1 : CcuInstanceMgr::GetInstance(devPhyId_);
112 1 : SocketProcess::GetInstance(devPhyId_);
113 1 : }
114 :
115 : // HcommResMgr
116 :
117 2 : HcommResMgr::HcommResMgr() = default;
118 :
119 2 : HcommResMgr::~HcommResMgr()
120 : {
121 2 : g_deviceRefreshCallbackRegistered = false;
122 2 : g_deviceResetCallbackRegistered = false;
123 2 : UnregisterDeviceRefreshCallback();
124 2 : }
125 :
126 245 : HcommResMgr& HcommResMgr::GetInstance()
127 : {
128 245 : static HcommResMgr instance;
129 245 : return instance;
130 : }
131 :
132 196 : void HcommResMgr::InitDevice(uint32_t devicePhyId)
133 : {
134 196 : uint32_t devPhyId = devicePhyId;
135 196 : if (devPhyId >= MAX_MODULE_DEVICE_NUM) {
136 0 : HCCL_WARNING(
137 : "[HcommResMgr][%s] use the backup device, devPhyId[%u] should be "
138 : "less than %u.",
139 : __func__, devPhyId, MAX_MODULE_DEVICE_NUM);
140 0 : devPhyId = MAX_MODULE_DEVICE_NUM; // 使用备份设备
141 : }
142 196 : if (!isInitialized_[devPhyId]) {
143 1 : deviceResMgrs_[devPhyId].SetDevPhyId(devPhyId);
144 1 : deviceResMgrs_[devPhyId].Init();
145 1 : isInitialized_[devPhyId] = true;
146 : }
147 196 : }
148 :
149 196 : HcommBaseResMgr& HcommResMgr::GetDeviceResMgr(uint32_t devicePhyId)
150 : {
151 196 : InitDevice(devicePhyId);
152 196 : uint32_t devPhyId = devicePhyId;
153 196 : if (devPhyId >= MAX_MODULE_DEVICE_NUM) {
154 0 : devPhyId = MAX_MODULE_DEVICE_NUM;
155 : }
156 196 : return deviceResMgrs_[devPhyId];
157 : }
158 :
159 49 : ConfigMgr& HcommResMgr::GetConfigMgr() { return configMgr_; }
160 :
161 0 : static void OnDeviceResetPre(int32_t deviceId, aclrtDeviceState state, [[maybe_unused]] void* args)
162 : {
163 : try {
164 0 : if (state != ACL_RT_DEVICE_STATE_RESET_PRE) {
165 0 : return;
166 : }
167 0 : HCCL_INFO("[%s] deviceId[%d] state[%d] ", __func__, deviceId, static_cast<int>(state));
168 :
169 0 : u32 devPhyId = 0;
170 0 : HcclResult ret = hrtGetDevicePhyIdByIndex(static_cast<u32>(deviceId), devPhyId);
171 0 : if (ret != HCCL_SUCCESS) {
172 0 : HCCL_WARNING("[%s] hrtGetDevicePhyIdByIndex failed, deviceId[%d] ret[%d]", __func__, deviceId, ret);
173 0 : return;
174 : }
175 0 : SocketMgr::DeInit(devPhyId);
176 0 : ServerSocketMgr::DeInit(devPhyId);
177 0 : ServerSocketManager::GetInstance().DeInit(devPhyId);
178 0 : Hccl::RdmaHandleManager::GetInstance().DeInit(devPhyId);
179 0 : Hccl::SocketHandleManager::GetInstance().DeInit(devPhyId);
180 0 : Hccl::HccpHdcManager::GetInstance().DeInit(deviceId);
181 0 : } catch (const std::exception& e) {
182 0 : HCCL_WARNING("[%s] exception caught:%s", __func__, e.what());
183 0 : } catch (...) {
184 0 : HCCL_WARNING("[%s] unknown exception caught", __func__);
185 0 : }
186 : }
187 :
188 87 : void HcommResMgr::RegisterDeviceResetCallback()
189 : {
190 87 : std::lock_guard<std::mutex> lock(g_deviceResetRegMutex);
191 87 : if (g_deviceResetCallbackRegistered) {
192 83 : return;
193 : }
194 4 : aclError ret = aclrtRegDeviceStateCallback("hcomm_res_mgr", OnDeviceResetPre, nullptr);
195 4 : if (ret != ACL_SUCCESS) {
196 0 : HCCL_WARNING("[RegisterDeviceResetCallback] aclrtRegDeviceStateCallback failed, ret[%d]", ret);
197 0 : return;
198 : }
199 4 : g_deviceResetCallbackRegistered = true;
200 4 : HCCL_INFO("[%s] aclrtRegDeviceStateCallback success", __func__);
201 87 : }
202 :
203 11 : static void OnDeviceStateRefresh(int32_t deviceId, aclrtDeviceState state, [[maybe_unused]] void* args)
204 : {
205 11 : std::lock_guard<std::mutex> lock(g_deviceRefreshMutex);
206 : try {
207 11 : if (state != ACL_RT_DEVICE_STATE_SET_POST) {
208 6 : return;
209 : }
210 8 : HCCL_INFO("[%s] deviceId[%d] state[%d]", __func__, deviceId, static_cast<int>(state));
211 8 : s32 deviceLogicId = 0;
212 8 : HcclResult ret = hrtGetDeviceRefresh(&deviceLogicId);
213 6 : if (ret != HCCL_SUCCESS) {
214 1 : HCCL_WARNING("[%s] hrtGetDeviceRefresh failed, deviceId[%d] ret[%d]", __func__, deviceId, ret);
215 1 : return;
216 : }
217 :
218 5 : u32 devicePhyId = 0;
219 5 : ret = hrtGetDevicePhyIdByIndex(static_cast<u32>(deviceLogicId), devicePhyId, true);
220 5 : if (ret != HCCL_SUCCESS) {
221 1 : HCCL_WARNING("[%s] hrtGetDevicePhyIdByIndex failed, deviceId[%d] ret[%d]", __func__, deviceId, ret);
222 1 : return;
223 : }
224 :
225 4 : DevType deviceType = DevType::DEV_TYPE_COUNT;
226 4 : ret = hrtGetDeviceType(deviceType);
227 4 : if (ret != HCCL_SUCCESS) {
228 1 : HCCL_WARNING("[%s] hrtGetDeviceType failed, deviceId[%d] ret[%d]", __func__, deviceId, ret);
229 1 : return;
230 : }
231 3 : HCCL_INFO(
232 : "[%s] refresh success, deviceLogicId[%d] devicePhyId[%d], deviceType[%d]", __func__, deviceLogicId,
233 : devicePhyId, static_cast<int>(deviceType));
234 2 : } catch (const std::exception& e) {
235 1 : HCCL_WARNING("[%s] exception caught:%s", __func__, e.what());
236 2 : } catch (...) {
237 1 : HCCL_WARNING("[%s] unknown exception caught", __func__);
238 1 : }
239 11 : }
240 :
241 91 : void HcommResMgr::RegisterDeviceRefreshCallback()
242 : {
243 91 : std::lock_guard<std::mutex> lock(g_deviceRefreshRegMutex);
244 91 : if (g_deviceRefreshCallbackRegistered) {
245 86 : return;
246 : }
247 5 : aclError ret = aclrtRegDeviceStateCallback("hcomm_refresh_device", OnDeviceStateRefresh, nullptr);
248 5 : if (ret != ACL_SUCCESS) {
249 1 : HCCL_WARNING("[%s] aclrtRegDeviceStateCallback failed, ret[%d]", __func__, ret);
250 1 : return;
251 : }
252 4 : g_deviceRefreshCallbackRegistered = true;
253 4 : HCCL_INFO("[%s] aclrtRegDeviceStateCallback success, regName[%s]", __func__, "hcomm_refresh_device");
254 91 : }
255 :
256 6 : void HcommResMgr::UnregisterDeviceRefreshCallback()
257 : {
258 6 : aclError ret = aclrtRegDeviceStateCallback("hcomm_refresh_device", nullptr, nullptr);
259 6 : if (ret != ACL_SUCCESS) {
260 1 : HCCL_WARNING(
261 : "[%s] aclrtRegDeviceStateCallback unregister failed, "
262 : "regName[%s] ret[%d]",
263 : __func__, "hcomm_refresh_device", ret);
264 : }
265 6 : HCCL_INFO("[%s] unregister success", __func__);
266 6 : }
267 :
268 : } // namespace hcomm
|
