Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 : #include "coll_comm_mgr.h"
11 : #include "cluster_monitor.h"
12 : #include "hcomm_c_adpt.h"
13 : #include "hcom_common.h"
14 :
15 : namespace hccl {
16 :
17 2990 : CollCommMgr& CollCommMgr::GetInstance()
18 : {
19 2990 : static CollCommMgr instance;
20 2990 : return instance;
21 : }
22 :
23 9 : CollCommMgr::~CollCommMgr()
24 : {
25 9 : HCCL_INFO("[CollCommMgr][~CollCommMgr] destruct begin.");
26 594 : for (auto& monitor : clusterMonitor_) {
27 585 : (void)monitor.DeInit();
28 : }
29 9 : HCCL_INFO("[CollCommMgr][~CollCommMgr] destruct end.");
30 9 : }
31 :
32 179 : hcomm::ClusterMonitor& CollCommMgr::GetClusterMonitor(s32 deviceLogicId)
33 : {
34 179 : if (static_cast<u32>(deviceLogicId) >= MAX_MODULE_DEVICE_NUM) {
35 0 : HCCL_WARNING(
36 : "[ClusterMonitor][%s]deviceLogicId[%d] >= %u, invalid", __func__, deviceLogicId, MAX_MODULE_DEVICE_NUM);
37 0 : return clusterMonitor_[0];
38 : }
39 179 : return clusterMonitor_[deviceLogicId];
40 : }
41 :
42 31 : HcclResult CollCommMgr::TryReserveCcuMsComm(s32 deviceLogicId, const std::string& commId, bool& reserved)
43 : {
44 31 : reserved = false;
45 31 : if (deviceLogicId < 0 || static_cast<u32>(deviceLogicId) >= MAX_MODULE_DEVICE_NUM || commId.empty()) {
46 2 : HCCL_ERROR(
47 : "[%s] invalid parameter, deviceLogicId[%d], max device num[%u], commId empty[%d].", __func__, deviceLogicId,
48 : MAX_MODULE_DEVICE_NUM, commId.empty());
49 2 : return HCCL_E_PARA;
50 : }
51 :
52 30 : std::lock_guard<std::mutex> lock(ccuMsCommMutex_);
53 29 : auto& owner = ccuMsCommIds_[deviceLogicId];
54 29 : if (owner.empty()) {
55 21 : owner = commId;
56 21 : reserved = true;
57 : }
58 29 : return HCCL_SUCCESS;
59 29 : }
60 :
61 10 : void CollCommMgr::ReleaseCcuMsComm(s32 deviceLogicId, const std::string& commId)
62 : {
63 10 : if (deviceLogicId < 0 || static_cast<u32>(deviceLogicId) >= MAX_MODULE_DEVICE_NUM) {
64 0 : HCCL_WARNING(
65 : "[%s] deviceLogicId[%d] is invalid, max device num[%u].", __func__, deviceLogicId, MAX_MODULE_DEVICE_NUM);
66 0 : return;
67 : }
68 :
69 10 : std::lock_guard<std::mutex> lock(ccuMsCommMutex_);
70 10 : auto& owner = ccuMsCommIds_[deviceLogicId];
71 10 : if (owner == commId) {
72 9 : owner.clear();
73 : }
74 10 : }
75 :
76 319 : OrderLaunchThreadMgr& CollCommMgr::GetOrderLaunchThreadMgr(s32 deviceLogicId)
77 : {
78 319 : if (deviceLogicId < 0 || static_cast<u32>(deviceLogicId) >= MAX_MODULE_DEVICE_NUM) {
79 0 : HCCL_WARNING(
80 : "[CollCommMgr][%s]deviceLogicId[%d] >= %u, invalid", __func__, deviceLogicId, MAX_MODULE_DEVICE_NUM);
81 0 : return orderLaunchThreadMgrs_[0];
82 : }
83 319 : return orderLaunchThreadMgrs_[deviceLogicId];
84 : }
85 :
86 157 : void CollCommMgr::RegisteCollComm(CollComm* collComm)
87 : {
88 157 : std::lock_guard<std::mutex> lock(mutex_);
89 157 : allCollComms_[collComm->GetCommId()] = collComm;
90 : // 注册到需要的地方
91 157 : taskAbortHandler_.Register(collComm);
92 157 : (void)GetOrderLaunchThreadMgr(collComm->GetDeviceLogicId()).RegisterOrderLaunch(collComm->GetCommId());
93 157 : }
94 :
95 162 : void CollCommMgr::UnRegisteCollComm(CollComm* collComm)
96 : {
97 162 : std::lock_guard<std::mutex> lock(mutex_);
98 162 : allCollComms_.erase(collComm->GetCommId());
99 : // 从通信域里面注销
100 162 : taskAbortHandler_.UnRegister(collComm);
101 162 : (void)GetClusterMonitor(collComm->GetDeviceLogicId()).UnRegisterToClusterMonitor(collComm);
102 162 : (void)GetOrderLaunchThreadMgr(collComm->GetDeviceLogicId()).UnRegisterOrderLaunch(collComm->GetCommId());
103 162 : }
104 :
105 8 : const std::unordered_map<std::string, CollComm*>& CollCommMgr::GetAllCollComms() const { return allCollComms_; }
106 :
107 77 : void CollCommMgr::InitBaseCommRes(uint32_t devId) { (void)HcommResMgrInit(devId); }
108 :
109 2673 : HcclOpInfoCtx& CollCommMgr::LegacyGetOpHcomInfo(uint32_t devId)
110 : {
111 2673 : if (devId >= MAX_MODULE_DEVICE_NUM + 1) {
112 1 : devId = MAX_MODULE_DEVICE_NUM;
113 : }
114 : // baseCommInited_ 无需加锁：本函数在生产路径中始终由 LegacyGetHcclExistDeviceOpInfoCtx /
115 : // LegacyGetHcclOpInfoCtx 在 opHcomInfosMutex_ 锁内调用
116 2673 : if (!baseCommInited_[devId]) {
117 76 : InitBaseCommRes(devId);
118 76 : baseCommInited_[devId] = true;
119 : }
120 2673 : return opHcomInfos_[devId];
121 : }
122 :
123 1000 : HcclOpInfoCtx& CollCommMgr::LegacyGetHcclExistDeviceOpInfoCtx(s32& devId)
124 : {
125 1000 : std::lock_guard<std::mutex> lock(opHcomInfosMutex_);
126 1000 : auto& opHcomInfo = LegacyGetOpHcomInfo(devId);
127 1000 : if (!opHcomInfo.isUsed) {
128 10 : HCCL_INFO("[LegacyGetHcclOpInfoCtx] Set device, use devId[%d] ", devId);
129 10 : auto& backUpOpHcomInfo = LegacyGetOpHcomInfo(MAX_MODULE_DEVICE_NUM);
130 10 : if (backUpOpHcomInfo.isUsed) {
131 1 : devId = MAX_MODULE_DEVICE_NUM;
132 1 : HCCL_INFO("[LegacyGetHcclOpInfoCtx] Used cover bottom devId[%d]", devId);
133 1 : return backUpOpHcomInfo;
134 : }
135 : }
136 :
137 999 : HCCL_INFO("[LegacyGetHcclExistDeviceOpInfoCtx] use devId[%d] opHcomInfos", devId);
138 999 : opHcomInfo.isUsed = true;
139 999 : return opHcomInfo;
140 1000 : }
141 :
142 996 : HcclOpInfoCtx& CollCommMgr::LegacyGetHcclOpInfoCtx(s32& devId)
143 : {
144 996 : if (HcclGetDeviceId() == HCCL_SUCCESS) {
145 994 : return LegacyGetHcclExistDeviceOpInfoCtx(devId);
146 : }
147 :
148 2 : std::lock_guard<std::mutex> lock(opHcomInfosMutex_);
149 72 : for (u32 i = 0; i < MAX_MODULE_DEVICE_NUM; i++) {
150 71 : auto& opHcomInfo = LegacyGetOpHcomInfo(i);
151 71 : if (opHcomInfo.isUsed) {
152 1 : devId = i;
153 1 : HCCL_INFO("[LegacyGetHcclOpInfoCtx] Not set device, Used devId[%u] ", i);
154 1 : return opHcomInfo;
155 : }
156 : }
157 :
158 1 : devId = MAX_MODULE_DEVICE_NUM;
159 1 : auto& backUpOpHcomInfo = LegacyGetOpHcomInfo(devId);
160 1 : backUpOpHcomInfo.isUsed = true;
161 1 : HCCL_INFO("[LegacyGetHcclOpInfoCtx] Used cover bottom devId[%d]", devId);
162 1 : return backUpOpHcomInfo;
163 2 : }
164 :
165 : } // namespace hccl
|
