Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 : #ifndef HCCLV2_TP_MANAGER_H
11 : #define HCCLV2_TP_MANAGER_H
12 :
13 : #include <cstdint>
14 : #include <functional>
15 : #include <mutex>
16 : #include <vector>
17 : #include <unordered_map>
18 :
19 : #include "hccl_types.h"
20 : #include "ip_address.h"
21 : #include "orion_adapter_hccp.h"
22 :
23 : namespace Hccl {
24 :
25 : constexpr uint8_t TA_GEAR_INDEX_0 = 0;
26 : constexpr uint8_t TA_GEAR_INDEX_1 = 1;
27 : constexpr uint8_t TA_GEAR_INDEX_2 = 2;
28 : constexpr uint8_t TA_GEAR_INDEX_3 = 3;
29 :
30 : constexpr uint8_t TA_HW_GEAR0_BASE = 0;
31 : constexpr uint8_t TA_HW_GEAR1_BASE = 8;
32 : constexpr uint8_t TA_HW_GEAR2_BASE = 16;
33 : constexpr uint8_t TA_HW_GEAR3_BASE = 24;
34 :
35 : static constexpr uint32_t TA_TIMEOUT_MS_GEAR0 = 512;
36 : static constexpr uint32_t TA_TIMEOUT_MS_GEAR1 = 1000;
37 : static constexpr uint32_t TA_TIMEOUT_MS_GEAR2 = 8000;
38 : static constexpr uint32_t TA_TIMEOUT_MS_GEAR3 = 32000;
39 :
40 : constexpr uint8_t AT_GEAR_MIN = 0;
41 : constexpr uint8_t AT_GEAR_MAX = 3;
42 : constexpr uint8_t AT_GEAR_DEFAULT = 2;
43 : constexpr uint32_t AT_TIMEOUT_MAP[4] = {16, 128, 1000, 4000};
44 :
45 : /// 与 GetTpInfo / ReleaseTpInfo 中 info、req 两级 map 的 qos 键一致（param.qos 低 8 位）
46 : using QosKey = uint32_t;
47 :
48 : /*
49 : * TP信息，当前申请TpHandle，不感知具体TP信息，当前仅支持TP与CTP
50 : * tpHandle: 对应管控面的TPID与相关资源，URMA通过引用计数管理申请和销毁TP
51 : */
52 : using TpHandle = uint64_t;
53 : struct TpInfo {
54 : TpHandle tpHandle{0};
55 : uint32_t mappedJettyPriority{0};
56 : bool hasMappedJettyPriority{false};
57 :
58 503 : TpInfo() = default;
59 : TpInfo(const TpHandle handle)
60 : : tpHandle(handle) {}
61 : };
62 :
63 : struct TpAttrInfo {
64 : struct TpAttr tpAttr{};
65 :
66 20 : TpAttrInfo() = default;
67 4 : TpAttrInfo(const struct TpAttr &attr)
68 4 : : tpAttr(attr) {}
69 : };
70 :
71 : struct GetTpAttrParam {
72 : TpHandle tpHandle{0};
73 : uint32_t attrBitmap{0};
74 :
75 : GetTpAttrParam() = default;
76 14 : GetTpAttrParam(const TpHandle handle, const uint32_t bitmap)
77 14 : : tpHandle(handle), attrBitmap(bitmap) {}
78 :
79 4 : std::string Describe() const {
80 : return Hccl::StringFormat("GetTpAttrParam[tpHandle=0x%llx, attrBitmap=0x%x]",
81 4 : tpHandle, attrBitmap);
82 : }
83 : };
84 :
85 : class TpManager {
86 : public:
87 : static TpManager &GetInstance(const int32_t deviceLogicId);
88 : void Init();
89 : /// `isSync==true`：同步路径（HOST_NET ctx + RaCtx* 接口），一次返回 SUCCESS。
90 : /// `isSync==false`：异步三阶段，轮询返回 HCCL_E_AGAIN。
91 : HcclResult GetTpInfo(const RaUbGetTpInfoParam ¶m, TpInfo &tpInfo, bool isSync = false);
92 : // unimport jetty 会 URMA 销毁 tp 资源，hccl 配套删除记录
93 : HcclResult ReleaseTpInfo(const RaUbGetTpInfoParam ¶m, const TpInfo &tpInfo);
94 : HcclResult GetTpAttr(const GetTpAttrParam ¶m, TpAttrInfo &tpAttrInfo, RdmaHandle rdmaHandle);
95 : HcclResult ReleaseTpAttr(const TpHandle tpHandle, const TpAttrInfo &tpAttrInfo);
96 :
97 : static HcclResult GetTpTotalTimeout(const TpAttrInfo &tpAttrInfo, uint32_t &tpTimeOutMs);
98 : static uint8_t CalcTaTimeout(const TpAttrInfo &tpAttrInfo);
99 : static uint32_t TaHwValueToMs(uint8_t hwValue);
100 : static uint8_t FindMinTaHwValue(uint32_t tpTotalTimeoutMs);
101 :
102 : private:
103 : bool initFlag{false};
104 : uint32_t devLogicId{0};
105 : uint32_t devPhyId{0};
106 :
107 : struct TpInfoCtx {
108 : TpInfo tpInfo{};
109 : uint32_t useCnt{0};
110 :
111 17 : TpInfoCtx() = default;
112 17 : TpInfoCtx(const TpInfo &info, const uint32_t cnt)
113 17 : : tpInfo(info), useCnt(cnt) {}
114 : };
115 :
116 : struct TpAttrCtx {
117 : TpAttrInfo tpAttrInfo{};
118 : uint32_t useCnt{0};
119 :
120 4 : TpAttrCtx() = default;
121 4 : TpAttrCtx(const TpAttrInfo &info, const uint32_t cnt)
122 4 : : tpAttrInfo(info), useCnt(cnt) {}
123 : };
124 :
125 : /*
126 : * Request上下文，保存查询TP信息相关调用异步接口出参
127 : * handle: 异步接口调用handle，用于查询处理结果
128 : * tpInfoNum: 查询到的TP信息个数，当前为复用TP，只会申请1个
129 : * dataBuffer: 查询到的TP信息数据，原始数据保留缓冲区
130 : */
131 : struct RequestCtx {
132 : enum class ReqPhase : uint8_t { WAIT_LIST = 0, WAIT_TP_ATTR = 1 };
133 : ReqPhase phase{ReqPhase::WAIT_LIST};
134 : RequestHandle handle{0};
135 : uint32_t tpInfoNum{0};
136 : std::vector<char_t> dataBuffer;
137 : TpAttr tpAttr{};
138 : uint32_t tpAttrBitmap{0};
139 : };
140 :
141 : struct TpAttrRequestCtx {
142 : RequestHandle handle{0};
143 : struct TpAttr tpAttr{};
144 : };
145 :
146 : /// 三级索引：先按本端 IP，再按对端 IP，最后按 QoS 键（`QosKey`：`param.qos & 0xFF`，与 next `TpMgr` 一致）。
147 : using InfoQosMap = std::unordered_map<uint32_t, TpInfoCtx>;
148 : using InfoRmtMap = std::unordered_map<IpAddress, InfoQosMap>;
149 : using InfoCtxMap = std::unordered_map<IpAddress, InfoRmtMap>;
150 : using ReqQosMap = std::unordered_map<uint32_t, RequestCtx>;
151 : using ReqRmtMap = std::unordered_map<IpAddress, ReqQosMap>;
152 : using ReqCtxMap = std::unordered_map<IpAddress, ReqRmtMap>;
153 :
154 : using TpAttrCtxMap = std::unordered_map<TpHandle, TpAttrCtx>;
155 : using TpAttrReqCtxMap = std::unordered_map<TpHandle, TpAttrRequestCtx>;
156 :
157 : InfoCtxMap ctpInfoMap;
158 : ReqCtxMap ctpReqMap;
159 :
160 : InfoCtxMap tpInfoMap;
161 : ReqCtxMap tpReqMap;
162 :
163 : InfoCtxMap uboeInfoMap;
164 : ReqCtxMap uboeReqMap;
165 :
166 : InfoCtxMap ubgInfoMap;
167 : ReqCtxMap ubgReqMap;
168 :
169 : TpAttrCtxMap tpAttrCtxMap;
170 : TpAttrReqCtxMap tpAttrReqCtxMap;
171 :
172 : std::mutex ctpInfoMutex;
173 : std::mutex ctpReqMutex;
174 :
175 : std::mutex tpInfoMutex;
176 : std::mutex tpReqMutex;
177 :
178 : std::mutex uboeInfoMutex;
179 : std::mutex uboeReqMutex;
180 :
181 : std::mutex ubgInfoMutex;
182 : std::mutex ubgReqMutex;
183 :
184 : std::mutex tpAttrCtxMutex;
185 : std::mutex tpAttrReqMutex;
186 :
187 67 : TpManager() = default;
188 67 : ~TpManager() = default;
189 : TpManager(const TpManager &that) = delete;
190 : TpManager &operator=(const TpManager &that) = delete;
191 :
192 : HcclResult FindAndGetTpInfo(const RaUbGetTpInfoParam ¶m, TpInfo &tpInfo);
193 : HcclResult RunSyncGetTpInfo(const RaUbGetTpInfoParam ¶m, TpInfo &tpInfo);
194 : HcclResult RunAsyncGetTpInfo(const RaUbGetTpInfoParam ¶m, TpInfo &tpInfo);
195 : HcclResult StoreTpInfoResult(const RaUbGetTpInfoParam ¶m, TpInfo &tpInfo);
196 : HcclResult SyncGetFirstTpAttrForSlPolicy(const RaUbGetTpInfoParam ¶m, uint64_t firstTpHandle, TpAttr &tpAttr,
197 : uint32_t &attrBitmap) const;
198 : HcclResult AdvanceDeviceWaitListPhase(const RaUbGetTpInfoParam ¶m, RequestCtx &reqCtx, ReqQosMap &qosReqMap,
199 : ReqQosMap::iterator it, std::unique_lock<std::mutex> &reqCtxLock, TpInfo &tpInfo);
200 : void StartGetTpInfoListRequest(const RaUbGetTpInfoParam ¶m, RequestCtx &reqCtx, bool isSync) const;
201 : HcclResult FindAndGetTpAttr(const TpHandle tpHandle, TpAttrInfo &tpAttrInfo);
202 : HcclResult StartGetTpAttrRequest(const GetTpAttrParam ¶m, TpAttrRequestCtx &reqCtx, RdmaHandle rdmaHandle) const;
203 : HcclResult HandleCompletedTpAttrRequest(const TpAttrRequestCtx reqCtx, const TpHandle tpHandle,
204 : TpAttrInfo &tpAttrInfo);
205 :
206 : void StartGetTpAttrForFirstTpDevice(const RaUbGetTpInfoParam ¶m, RequestCtx &reqCtx) const;
207 : HcclResult HandleCompletedRequest(const RequestCtx reqCtx, const RaUbGetTpInfoParam ¶m, TpInfo &tpInfo,
208 : bool withSlPolicy);
209 : HcclResult MapTpInfoFromTpAttr(const RaUbGetTpInfoParam ¶m, const RequestCtx &reqCtx, TpInfo &outTpInfo,
210 : bool isSync);
211 :
212 : bool CheckRequestResult(RequestHandle &reqHandle) const;
213 : InfoCtxMap &GetInfoCtxMap(const TpProtocol tpProtocol);
214 : ReqCtxMap &GetReqCtxMap(const TpProtocol tpProtocol);
215 : std::mutex &GetInfoCtxMutex(const TpProtocol tpProtocol);
216 : std::mutex &GetReqCtxMutex(const TpProtocol tpProtocol);
217 : };
218 :
219 : /// UbConnection 释放 TpInfo：Release 键须与 GetTpInfo 时的业务 QoS 一致
220 : void ReleaseUbConnectionTp(int32_t devLogicId, const IpAddress &locAddr, const IpAddress &rmtAddr,
221 : TpProtocol tpProtocol, TpInfo &tpInfo, uint32_t requestQos);
222 :
223 : } // namespace Hccl
224 :
225 : #endif // HCCLV2_TP_MANAGER_H
|
