Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "hccl_ccu_res.h"
12 :
13 : #include <mutex>
14 :
15 : #include "hccl_comm_pub.h"
16 :
17 : #include "exception_handler.h"
18 : #include "ccu_instance_mgr.h"
19 : #include "ccu_device_res.h"
20 :
21 6 : static HcclResult CreateCcuInsByFixedResNum(
22 : const char* funcName, const std::string& commId, hccl::MyRank* myRank, CcuInsHandle& ccuInsHandle)
23 : {
24 6 : const auto opExpansionMode = myRank->GetOpExpansionMode();
25 6 : auto ccuInsType = hccl::OpExpansionModeToCcuInstanceType(opExpansionMode);
26 6 : if (ccuInsType == CcuInstanceType::CCU_UNUSED) {
27 1 : HCCL_WARNING(
28 : "[%s] failed to get ccu instance, commId[%s] op expansion mode[%u].", funcName, commId.c_str(),
29 : opExpansionMode);
30 1 : return HcclResult::HCCL_E_UNAVAIL;
31 : }
32 :
33 5 : CcuInsHandle newHandle = 0;
34 5 : auto ccuRet = CcuResult::CCU_SUCCESS;
35 : // CCU_MS 资源不足（CCU_E_UNAVAIL）时降级到 CCU_SCHED 重试一次；CCU_SCHED 不再继续降级
36 : while (true) {
37 7 : ccuRet = HcommCcuInsCreateLegacy(ccuInsType, &newHandle);
38 7 : if (ccuRet != CcuResult::CCU_E_UNAVAIL || ccuInsType != CcuInstanceType::CCU_MS) {
39 : break;
40 : }
41 2 : HCCL_WARNING(
42 : "[%s] ccu instance resource unavailable for CCU_MS, fallback to CCU_SCHED, commId[%s].", funcName,
43 : commId.c_str());
44 2 : ccuInsType = CcuInstanceType::CCU_SCHED;
45 : }
46 5 : if (ccuRet == CcuResult::CCU_E_UNAVAIL) {
47 1 : HCCL_WARNING(
48 : "[%s] failed to create ccu instance, resources are unavailable, "
49 : "commId[%s] insType[%d] ret[%d].",
50 : funcName, commId.c_str(), ccuInsType, ccuRet);
51 1 : return static_cast<HcclResult>(ccuRet);
52 : }
53 4 : if (ccuRet != CcuResult::CCU_SUCCESS) {
54 1 : HCCL_ERROR(
55 : "[%s] failed to create ccu instance, commId[%s] insType[%d] ret[%d].", funcName, commId.c_str(), ccuInsType,
56 : ccuRet);
57 1 : return static_cast<HcclResult>(ccuRet);
58 : }
59 3 : myRank->SetCcuInstance(newHandle);
60 3 : ccuInsHandle = newHandle;
61 3 : return HcclResult::HCCL_SUCCESS;
62 : }
63 :
64 : /**
65 : * @note 职责：集合通信的通信域CCU管理的C接口的C到C++适配
66 : */
67 12 : HcclResult HcclCommQueryCcuIns(HcclComm comm, CcuInsHandle* insHandles, uint32_t* insNum)
68 : {
69 : EXCEPTION_HANDLE_BEGIN
70 :
71 12 : HcclUs startut = TIME_NOW();
72 :
73 18 : CHK_PTR_NULL(comm);
74 11 : CHK_PTR_NULL(insHandles);
75 10 : CHK_PTR_NULL(insNum);
76 9 : auto* hcclComm = static_cast<hccl::hcclComm*>(comm);
77 9 : const auto& commId = hcclComm->GetIdentifier();
78 9 : HCCL_INFO("[%s] CommId[%s] query ccu instance.", __func__, commId.c_str());
79 :
80 : // CCU不支持A5之前代际
81 9 : if (!hcclComm->IsCommunicatorV2()) {
82 1 : HCCL_RUN_WARNING("[%s] is not supported.", __func__);
83 1 : return HcclResult::HCCL_E_NOT_SUPPORT;
84 : }
85 :
86 8 : auto* collComm = hcclComm->GetCollComm();
87 8 : CHK_PTR_NULL(collComm);
88 8 : auto* myRank = collComm->GetMyRank();
89 8 : CHK_PTR_NULL(myRank);
90 :
91 : // 查询通信域自有的 ccuInsHandle_；为 0 时按 opExpansionMode 创建
92 8 : auto ccuInsHandle = myRank->GetCcuInstance();
93 8 : if (ccuInsHandle == 0) {
94 6 : auto ret = CreateCcuInsByFixedResNum(__func__, commId, myRank, ccuInsHandle);
95 6 : if (ret != HcclResult::HCCL_SUCCESS) {
96 3 : HCCL_WARNING("[%s] failed to get ccu instance, commId[%s] ret[%d]", __func__, commId.c_str(), ret);
97 3 : return ret;
98 : }
99 : }
100 :
101 5 : insHandles[0] = ccuInsHandle;
102 5 : *insNum = 1;
103 5 : HCCL_INFO("[%s] success, take time [%lld]us.", __func__, DURATION_US(TIME_NOW() - startut).count());
104 :
105 9 : EXCEPTION_HANDLE_END
106 5 : return HcclResult::HCCL_SUCCESS;
107 : }
108 :
109 7 : HcclResult HcclCommQueryAssignedCcuIns(HcclComm comm, CcuInsHandle* insHandles, uint32_t* insNum)
110 : {
111 : EXCEPTION_HANDLE_BEGIN
112 :
113 7 : HcclUs startut = TIME_NOW();
114 :
115 12 : CHK_PTR_NULL(comm);
116 6 : CHK_PTR_NULL(insHandles);
117 5 : CHK_PTR_NULL(insNum);
118 4 : auto* hcclComm = static_cast<hccl::hcclComm*>(comm);
119 4 : const auto& commId = hcclComm->GetIdentifier();
120 4 : HCCL_INFO("[%s] CommId[%s] query assigned ccu instance.", __func__, commId.c_str());
121 :
122 : // CCU不支持A5之前代际
123 4 : if (!hcclComm->IsCommunicatorV2()) {
124 1 : HCCL_RUN_WARNING("[%s] is not supported.", __func__);
125 1 : return HcclResult::HCCL_E_NOT_SUPPORT;
126 : }
127 :
128 3 : auto* collComm = hcclComm->GetCollComm();
129 3 : CHK_PTR_NULL(collComm);
130 3 : auto* myRank = collComm->GetMyRank();
131 3 : CHK_PTR_NULL(myRank);
132 :
133 : // 查询绑定的 assignedCcuInsHandle_；未绑定时返回 UNAVAIL，不创建
134 3 : auto assignedCcuInsHandle = myRank->GetAssignedCcuInstance();
135 3 : if (assignedCcuInsHandle == 0) {
136 2 : HCCL_WARNING("[%s] assigned ccu instance not exist, commId[%s].", __func__, commId.c_str());
137 2 : return HcclResult::HCCL_E_UNAVAIL;
138 : }
139 :
140 1 : insHandles[0] = assignedCcuInsHandle;
141 1 : *insNum = 1;
142 1 : HCCL_INFO("[%s] success, take time [%lld]us.", __func__, DURATION_US(TIME_NOW() - startut).count());
143 :
144 4 : EXCEPTION_HANDLE_END
145 1 : return HcclResult::HCCL_SUCCESS;
146 : }
147 :
148 14 : HcclResult HcclCommAssignCcuIns(HcclComm comm, CcuInsHandle insHandle)
149 : {
150 : EXCEPTION_HANDLE_BEGIN
151 :
152 14 : HcclUs startut = TIME_NOW();
153 :
154 22 : CHK_PTR_NULL(comm);
155 13 : auto* hcclComm = static_cast<hccl::hcclComm*>(comm);
156 13 : const auto& commId = hcclComm->GetIdentifier();
157 13 : HCCL_INFO(
158 : "[%s] CommId[%s] assign ccu instance[%llu].", __func__, commId.c_str(),
159 : static_cast<unsigned long long>(insHandle));
160 :
161 13 : if (insHandle == 0) {
162 1 : HCCL_ERROR(
163 : "[%s] failed, commId[%s] insHandle[%llu] is invalid.", __func__, commId.c_str(),
164 : static_cast<unsigned long long>(insHandle));
165 1 : return HcclResult::HCCL_E_PARA;
166 : }
167 :
168 12 : if (!hcclComm->IsCommunicatorV2()) {
169 1 : HCCL_RUN_WARNING("[%s] is not supported.", __func__);
170 1 : return HcclResult::HCCL_E_NOT_SUPPORT;
171 : }
172 :
173 11 : auto* collComm = hcclComm->GetCollComm();
174 11 : CHK_PTR_NULL(collComm);
175 10 : auto* myRank = collComm->GetMyRank();
176 10 : CHK_PTR_NULL(myRank);
177 :
178 : {
179 : // 仅保证多个 Assign 调用之间并发安全，Query 和通信域销毁由调用方保证不与 Assign 并发。
180 : static std::mutex assignCcuInsMutex;
181 9 : std::lock_guard<std::mutex> lock(assignCcuInsMutex);
182 :
183 9 : auto oldInsHandle = myRank->GetAssignedCcuInstance();
184 9 : if (oldInsHandle != 0) {
185 3 : HCCL_ERROR(
186 : "[%s] failed, commId[%s] already has assigned ccu instance[%llu], "
187 : "new instance[%llu] will not be assigned.",
188 : __func__, commId.c_str(), static_cast<unsigned long long>(oldInsHandle),
189 : static_cast<unsigned long long>(insHandle));
190 3 : return HcclResult::HCCL_E_PARA;
191 : }
192 :
193 6 : const auto devLogicId = collComm->GetDeviceLogicId();
194 6 : auto* ccuIns = hcomm::CcuInstanceMgr::GetInstance(devLogicId).Get(insHandle);
195 5 : if (ccuIns == nullptr) {
196 1 : HCCL_ERROR(
197 : "[%s] failed, commId[%s] ccu instance[%llu] is not found.", __func__, commId.c_str(),
198 : static_cast<unsigned long long>(insHandle));
199 1 : return HcclResult::HCCL_E_NOT_FOUND;
200 : }
201 :
202 4 : myRank->SetAssignedCcuInstance(insHandle);
203 9 : }
204 :
205 4 : HCCL_INFO(
206 : "[%s] success, commId[%s] ccu instance[%llu], take time [%lld]us.", __func__, commId.c_str(),
207 : static_cast<unsigned long long>(insHandle), DURATION_US(TIME_NOW() - startut));
208 :
209 14 : EXCEPTION_HANDLE_END
210 4 : return HcclResult::HCCL_SUCCESS;
211 : }
|
