Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "coll_comm_mgr.h"
12 : #include "cluster_monitor.h"
13 : #include "hcomm_c_adpt.h"
14 : #include "hcom_common.h"
15 :
16 : namespace hccl {
17 :
18 2990 : CollCommMgr& CollCommMgr::GetInstance()
19 : {
20 2990 : static CollCommMgr instance;
21 2990 : return instance;
22 : }
23 :
24 9 : CollCommMgr::~CollCommMgr()
25 : {
26 9 : HCCL_INFO("[CollCommMgr][~CollCommMgr] destruct begin.");
27 594 : for (auto& monitor : clusterMonitor_) {
28 585 : (void)monitor.DeInit();
29 : }
30 9 : HCCL_INFO("[CollCommMgr][~CollCommMgr] destruct end.");
31 9 : }
32 :
33 179 : hcomm::ClusterMonitor& CollCommMgr::GetClusterMonitor(s32 deviceLogicId)
34 : {
35 179 : if (static_cast<u32>(deviceLogicId) >= MAX_MODULE_DEVICE_NUM) {
36 0 : HCCL_WARNING(
37 : "[ClusterMonitor][%s]deviceLogicId[%d] >= %u, invalid", __func__, deviceLogicId, MAX_MODULE_DEVICE_NUM);
38 0 : return clusterMonitor_[0];
39 : }
40 179 : return clusterMonitor_[deviceLogicId];
41 : }
42 :
43 31 : HcclResult CollCommMgr::TryReserveCcuMsComm(s32 deviceLogicId, const std::string& commId, bool& reserved)
44 : {
45 31 : reserved = false;
46 31 : if (deviceLogicId < 0 || static_cast<u32>(deviceLogicId) >= MAX_MODULE_DEVICE_NUM || commId.empty()) {
47 2 : HCCL_ERROR(
48 : "[%s] invalid parameter, deviceLogicId[%d], max device num[%u], commId empty[%d].", __func__, deviceLogicId,
49 : MAX_MODULE_DEVICE_NUM, commId.empty());
50 2 : return HCCL_E_PARA;
51 : }
52 :
53 29 : std::lock_guard<std::mutex> lock(ccuMsCommMutex_);
54 29 : auto& owner = ccuMsCommIds_[deviceLogicId];
55 29 : if (owner.empty()) {
56 21 : owner = commId;
57 21 : reserved = true;
58 : }
59 29 : return HCCL_SUCCESS;
60 29 : }
61 :
62 10 : void CollCommMgr::ReleaseCcuMsComm(s32 deviceLogicId, const std::string& commId)
63 : {
64 10 : if (deviceLogicId < 0 || static_cast<u32>(deviceLogicId) >= MAX_MODULE_DEVICE_NUM) {
65 0 : HCCL_WARNING(
66 : "[%s] deviceLogicId[%d] is invalid, max device num[%u].", __func__, deviceLogicId, MAX_MODULE_DEVICE_NUM);
67 0 : return;
68 : }
69 :
70 10 : std::lock_guard<std::mutex> lock(ccuMsCommMutex_);
71 10 : auto& owner = ccuMsCommIds_[deviceLogicId];
72 10 : if (owner == commId) {
73 9 : owner.clear();
74 : }
75 10 : }
76 :
77 319 : OrderLaunchThreadMgr& CollCommMgr::GetOrderLaunchThreadMgr(s32 deviceLogicId)
78 : {
79 319 : if (deviceLogicId < 0 || static_cast<u32>(deviceLogicId) >= MAX_MODULE_DEVICE_NUM) {
80 0 : HCCL_WARNING(
81 : "[CollCommMgr][%s]deviceLogicId[%d] >= %u, invalid", __func__, deviceLogicId, MAX_MODULE_DEVICE_NUM);
82 0 : return orderLaunchThreadMgrs_[0];
83 : }
84 319 : return orderLaunchThreadMgrs_[deviceLogicId];
85 : }
86 :
87 157 : void CollCommMgr::RegisteCollComm(CollComm* collComm)
88 : {
89 157 : std::lock_guard<std::mutex> lock(mutex_);
90 157 : allCollComms_[collComm->GetCommId()] = collComm;
91 : // 注册到需要的地方
92 157 : taskAbortHandler_.Register(collComm);
93 157 : (void)GetOrderLaunchThreadMgr(collComm->GetDeviceLogicId()).RegisterOrderLaunch(collComm->GetCommId());
94 157 : }
95 :
96 162 : void CollCommMgr::UnRegisteCollComm(CollComm* collComm)
97 : {
98 162 : std::lock_guard<std::mutex> lock(mutex_);
99 162 : allCollComms_.erase(collComm->GetCommId());
100 : // 从通信域里面注销
101 162 : taskAbortHandler_.UnRegister(collComm);
102 162 : (void)GetClusterMonitor(collComm->GetDeviceLogicId()).UnRegisterToClusterMonitor(collComm);
103 162 : (void)GetOrderLaunchThreadMgr(collComm->GetDeviceLogicId()).UnRegisterOrderLaunch(collComm->GetCommId());
104 162 : }
105 :
106 8 : const std::unordered_map<std::string, CollComm*>& CollCommMgr::GetAllCollComms() const { return allCollComms_; }
107 :
108 77 : void CollCommMgr::InitBaseCommRes(uint32_t devId) { (void)HcommResMgrInit(devId); }
109 :
110 2673 : HcclOpInfoCtx& CollCommMgr::LegacyGetOpHcomInfo(uint32_t devId)
111 : {
112 2673 : if (devId >= MAX_MODULE_DEVICE_NUM + 1) {
113 1 : devId = MAX_MODULE_DEVICE_NUM;
114 : }
115 : // baseCommInited_ 无需加锁：本函数在生产路径中始终由 LegacyGetHcclExistDeviceOpInfoCtx /
116 : // LegacyGetHcclOpInfoCtx 在 opHcomInfosMutex_ 锁内调用
117 2673 : if (!baseCommInited_[devId]) {
118 76 : InitBaseCommRes(devId);
119 76 : baseCommInited_[devId] = true;
120 : }
121 2673 : return opHcomInfos_[devId];
122 : }
123 :
124 1000 : HcclOpInfoCtx& CollCommMgr::LegacyGetHcclExistDeviceOpInfoCtx(s32& devId)
125 : {
126 1000 : std::lock_guard<std::mutex> lock(opHcomInfosMutex_);
127 1000 : auto& opHcomInfo = LegacyGetOpHcomInfo(devId);
128 1000 : if (!opHcomInfo.isUsed) {
129 10 : HCCL_INFO("[LegacyGetHcclOpInfoCtx] Set device, use devId[%d] ", devId);
130 10 : auto& backUpOpHcomInfo = LegacyGetOpHcomInfo(MAX_MODULE_DEVICE_NUM);
131 10 : if (backUpOpHcomInfo.isUsed) {
132 1 : devId = MAX_MODULE_DEVICE_NUM;
133 1 : HCCL_INFO("[LegacyGetHcclOpInfoCtx] Used cover bottom devId[%d]", devId);
134 1 : return backUpOpHcomInfo;
135 : }
136 : }
137 :
138 999 : HCCL_INFO("[LegacyGetHcclExistDeviceOpInfoCtx] use devId[%d] opHcomInfos", devId);
139 999 : opHcomInfo.isUsed = true;
140 999 : return opHcomInfo;
141 1000 : }
142 :
143 996 : HcclOpInfoCtx& CollCommMgr::LegacyGetHcclOpInfoCtx(s32& devId)
144 : {
145 996 : if (HcclGetDeviceId() == HCCL_SUCCESS) {
146 994 : return LegacyGetHcclExistDeviceOpInfoCtx(devId);
147 : }
148 :
149 2 : std::lock_guard<std::mutex> lock(opHcomInfosMutex_);
150 72 : for (u32 i = 0; i < MAX_MODULE_DEVICE_NUM; i++) {
151 71 : auto& opHcomInfo = LegacyGetOpHcomInfo(i);
152 71 : if (opHcomInfo.isUsed) {
153 1 : devId = i;
154 1 : HCCL_INFO("[LegacyGetHcclOpInfoCtx] Not set device, Used devId[%u] ", i);
155 1 : return opHcomInfo;
156 : }
157 : }
158 :
159 1 : devId = MAX_MODULE_DEVICE_NUM;
160 1 : auto& backUpOpHcomInfo = LegacyGetOpHcomInfo(devId);
161 1 : backUpOpHcomInfo.isUsed = true;
162 1 : HCCL_INFO("[LegacyGetHcclOpInfoCtx] Used cover bottom devId[%d]", devId);
163 1 : return backUpOpHcomInfo;
164 2 : }
165 :
166 : } // namespace hccl
|
