Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #ifndef HCCLV2_TP_MANAGER_H
12 : #define HCCLV2_TP_MANAGER_H
13 :
14 : #include <cstdint>
15 : #include <functional>
16 : #include <mutex>
17 : #include <vector>
18 : #include <unordered_map>
19 :
20 : #include "hccl_types.h"
21 : #include "ip_address.h"
22 : #include "orion_adapter_hccp.h"
23 :
24 : namespace Hccl {
25 :
26 : constexpr uint8_t TA_GEAR_INDEX_0 = 0;
27 : constexpr uint8_t TA_GEAR_INDEX_1 = 1;
28 : constexpr uint8_t TA_GEAR_INDEX_2 = 2;
29 : constexpr uint8_t TA_GEAR_INDEX_3 = 3;
30 :
31 : constexpr uint8_t TA_HW_GEAR0_BASE = 0;
32 : constexpr uint8_t TA_HW_GEAR1_BASE = 8;
33 : constexpr uint8_t TA_HW_GEAR2_BASE = 16;
34 : constexpr uint8_t TA_HW_GEAR3_BASE = 24;
35 :
36 : static constexpr uint32_t TA_TIMEOUT_MS_GEAR0 = 512;
37 : static constexpr uint32_t TA_TIMEOUT_MS_GEAR1 = 1000;
38 : static constexpr uint32_t TA_TIMEOUT_MS_GEAR2 = 8000;
39 : static constexpr uint32_t TA_TIMEOUT_MS_GEAR3 = 32000;
40 :
41 : constexpr uint8_t AT_GEAR_MIN = 0;
42 : constexpr uint8_t AT_GEAR_MAX = 3;
43 : constexpr uint8_t AT_GEAR_DEFAULT = 2;
44 : constexpr uint32_t AT_TIMEOUT_MAP[4] = {16, 128, 1000, 4000};
45 :
46 : /// 与 GetTpInfo / ReleaseTpInfo 中 info、req 两级 map 的 qos 键一致（param.qos 低 8 位）
47 : using QosKey = uint32_t;
48 :
49 : /*
50 : * TP信息，当前申请TpHandle，不感知具体TP信息，当前仅支持TP与CTP
51 : * tpHandle: 对应管控面的TPID与相关资源，URMA通过引用计数管理申请和销毁TP
52 : */
53 : using TpHandle = uint64_t;
54 : struct TpInfo {
55 : TpHandle tpHandle{0};
56 : uint32_t mappedJettyPriority{0};
57 : bool hasMappedJettyPriority{false};
58 :
59 530 : TpInfo() = default;
60 : TpInfo(const TpHandle handle) : tpHandle(handle) {}
61 : };
62 :
63 : struct TpAttrInfo {
64 : struct TpAttr tpAttr {};
65 :
66 20 : TpAttrInfo() = default;
67 4 : TpAttrInfo(const struct TpAttr& attr) : tpAttr(attr) {}
68 : };
69 :
70 : struct GetTpAttrParam {
71 : TpHandle tpHandle{0};
72 : uint32_t attrBitmap{0};
73 :
74 : GetTpAttrParam() = default;
75 14 : GetTpAttrParam(const TpHandle handle, const uint32_t bitmap) : tpHandle(handle), attrBitmap(bitmap) {}
76 :
77 4 : std::string Describe() const
78 : {
79 4 : return Hccl::StringFormat("GetTpAttrParam[tpHandle=0x%llx, attrBitmap=0x%x]", tpHandle, attrBitmap);
80 : }
81 : };
82 :
83 : class TpManager {
84 : public:
85 : static TpManager& GetInstance(const int32_t deviceLogicId);
86 : void Init();
87 : /// `isSync==true`：同步路径（HOST_NET ctx + HrtRa* 同步接口），一次返回 SUCCESS。
88 : /// `isSync==false`：异步三阶段，轮询返回 HCCL_E_AGAIN。
89 : HcclResult GetTpInfo(const RaUbGetTpInfoParam& param, TpInfo& tpInfo, bool isSync = false);
90 : // unimport jetty 会 URMA 销毁 tp 资源，hccl 配套删除记录
91 : HcclResult ReleaseTpInfo(const RaUbGetTpInfoParam& param, const TpInfo& tpInfo);
92 : HcclResult GetTpAttr(const GetTpAttrParam& param, TpAttrInfo& tpAttrInfo, RdmaHandle rdmaHandle);
93 : HcclResult ReleaseTpAttr(const TpHandle tpHandle, const TpAttrInfo& tpAttrInfo);
94 :
95 : static HcclResult GetTpTotalTimeout(const TpAttrInfo& tpAttrInfo, uint32_t& tpTimeOutMs);
96 : static uint8_t CalcTaTimeout(const TpAttrInfo& tpAttrInfo);
97 : static uint32_t TaHwValueToMs(uint8_t hwValue);
98 : static uint8_t FindMinTaHwValue(uint32_t tpTotalTimeoutMs);
99 :
100 : private:
101 : bool initFlag{false};
102 : uint32_t devLogicId{0};
103 : uint32_t devPhyId{0};
104 :
105 : struct TpInfoCtx {
106 : TpInfo tpInfo{};
107 : uint32_t useCnt{0};
108 :
109 17 : TpInfoCtx() = default;
110 17 : TpInfoCtx(const TpInfo& info, const uint32_t cnt) : tpInfo(info), useCnt(cnt) {}
111 : };
112 :
113 : struct TpAttrCtx {
114 : TpAttrInfo tpAttrInfo{};
115 : uint32_t useCnt{0};
116 :
117 4 : TpAttrCtx() = default;
118 4 : TpAttrCtx(const TpAttrInfo& info, const uint32_t cnt) : tpAttrInfo(info), useCnt(cnt) {}
119 : };
120 :
121 : /*
122 : * Request上下文，保存查询TP信息相关调用异步接口出参
123 : * handle: 异步接口调用handle，用于查询处理结果
124 : * tpInfoNum: 查询到的TP信息个数，当前为复用TP，只会申请1个
125 : * dataBuffer: 查询到的TP信息数据，原始数据保留缓冲区
126 : */
127 : struct RequestCtx {
128 : enum class ReqPhase : uint8_t { WAIT_LIST = 0, WAIT_TP_ATTR = 1 };
129 : ReqPhase phase{ReqPhase::WAIT_LIST};
130 : RequestHandle handle{0};
131 : uint32_t tpInfoNum{0};
132 : std::vector<char_t> dataBuffer;
133 : TpAttr tpAttr{};
134 : uint32_t tpAttrBitmap{0};
135 : };
136 :
137 : struct TpAttrRequestCtx {
138 : RequestHandle handle{0};
139 : struct TpAttr tpAttr {};
140 : uint32_t attrBitmap{0};
141 : };
142 :
143 : /// 三级索引：先按本端 IP，再按对端 IP，最后按 QoS 键（`QosKey`：`param.qos & 0xFF`，与 next `TpMgr` 一致）。
144 : using InfoQosMap = std::unordered_map<uint32_t, TpInfoCtx>;
145 : using InfoRmtMap = std::unordered_map<IpAddress, InfoQosMap>;
146 : using InfoCtxMap = std::unordered_map<IpAddress, InfoRmtMap>;
147 : using ReqQosMap = std::unordered_map<uint32_t, RequestCtx>;
148 : using ReqRmtMap = std::unordered_map<IpAddress, ReqQosMap>;
149 : using ReqCtxMap = std::unordered_map<IpAddress, ReqRmtMap>;
150 :
151 : using TpAttrCtxMap = std::unordered_map<TpHandle, TpAttrCtx>;
152 : using TpAttrReqCtxMap = std::unordered_map<TpHandle, TpAttrRequestCtx>;
153 :
154 : InfoCtxMap ctpInfoMap;
155 : ReqCtxMap ctpReqMap;
156 :
157 : InfoCtxMap tpInfoMap;
158 : ReqCtxMap tpReqMap;
159 :
160 : InfoCtxMap uboeInfoMap;
161 : ReqCtxMap uboeReqMap;
162 :
163 : InfoCtxMap ubRtpInfoMap;
164 : ReqCtxMap ubRtpReqMap;
165 :
166 : TpAttrCtxMap tpAttrCtxMap;
167 : TpAttrReqCtxMap tpAttrReqCtxMap;
168 :
169 : std::mutex ctpInfoMutex;
170 : std::mutex ctpReqMutex;
171 :
172 : std::mutex tpInfoMutex;
173 : std::mutex tpReqMutex;
174 :
175 : std::mutex uboeInfoMutex;
176 : std::mutex uboeReqMutex;
177 :
178 : std::mutex ubRtpInfoMutex;
179 : std::mutex ubRtpReqMutex;
180 :
181 : std::mutex tpAttrCtxMutex;
182 : std::mutex tpAttrReqMutex;
183 :
184 67 : TpManager() = default;
185 67 : ~TpManager() = default;
186 : TpManager(const TpManager& that) = delete;
187 : TpManager& operator=(const TpManager& that) = delete;
188 :
189 : HcclResult FindAndGetTpInfo(const RaUbGetTpInfoParam& param, TpInfo& tpInfo);
190 : HcclResult RunSyncGetTpInfo(const RaUbGetTpInfoParam& param, TpInfo& tpInfo);
191 : HcclResult RunAsyncGetTpInfo(const RaUbGetTpInfoParam& param, TpInfo& tpInfo);
192 : HcclResult StoreTpInfoResult(const RaUbGetTpInfoParam& param, TpInfo& tpInfo);
193 : HcclResult SyncGetFirstTpAttrForSlPolicy(
194 : const RaUbGetTpInfoParam& param, uint64_t firstTpHandle, TpAttr& tpAttr, uint32_t& attrBitmap) const;
195 : HcclResult AdvanceDeviceWaitListPhase(
196 : const RaUbGetTpInfoParam& param, RequestCtx& reqCtx, ReqQosMap& qosReqMap, ReqQosMap::iterator it,
197 : std::unique_lock<std::mutex>& reqCtxLock, TpInfo& tpInfo);
198 : void StartGetTpInfoListRequest(const RaUbGetTpInfoParam& param, RequestCtx& reqCtx, bool isSync) const;
199 : HcclResult FindAndGetTpAttr(const TpHandle tpHandle, TpAttrInfo& tpAttrInfo);
200 : HcclResult
201 : StartGetTpAttrRequest(const GetTpAttrParam& param, TpAttrRequestCtx& reqCtx, RdmaHandle rdmaHandle) const;
202 : HcclResult
203 : HandleCompletedTpAttrRequest(const TpAttrRequestCtx reqCtx, const TpHandle tpHandle, TpAttrInfo& tpAttrInfo);
204 :
205 : void StartGetTpAttrForFirstTpDevice(const RaUbGetTpInfoParam& param, RequestCtx& reqCtx) const;
206 : HcclResult
207 : HandleCompletedRequest(const RequestCtx reqCtx, const RaUbGetTpInfoParam& param, TpInfo& tpInfo, bool withSlPolicy);
208 : HcclResult
209 : MapTpInfoFromTpAttr(const RaUbGetTpInfoParam& param, const RequestCtx& reqCtx, TpInfo& outTpInfo, bool isSync);
210 :
211 : bool CheckRequestResult(RequestHandle& reqHandle) const;
212 : InfoCtxMap& GetInfoCtxMap(const TpProtocol tpProtocol);
213 : ReqCtxMap& GetReqCtxMap(const TpProtocol tpProtocol);
214 : std::mutex& GetInfoCtxMutex(const TpProtocol tpProtocol);
215 : std::mutex& GetReqCtxMutex(const TpProtocol tpProtocol);
216 : };
217 :
218 : /// UbConnection 释放 TpInfo：Release 键须与 GetTpInfo 时的业务 QoS 一致
219 : void ReleaseUbConnectionTp(
220 : int32_t devLogicId, const IpAddress& locAddr, const IpAddress& rmtAddr, TpProtocol tpProtocol, TpInfo& tpInfo,
221 : uint32_t requestQos);
222 :
223 : } // namespace Hccl
224 :
225 : #endif // HCCLV2_TP_MANAGER_H
|
