Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 : #ifndef HCCLV2_TP_MANAGER_H
11 : #define HCCLV2_TP_MANAGER_H
12 :
13 : #include <cstdint>
14 : #include <functional>
15 : #include <mutex>
16 : #include <vector>
17 : #include <unordered_map>
18 :
19 : #include "hccl_types.h"
20 : #include "ip_address.h"
21 : #include "orion_adapter_hccp.h"
22 :
23 : namespace Hccl {
24 :
25 : constexpr uint8_t TA_GEAR_INDEX_0 = 0;
26 : constexpr uint8_t TA_GEAR_INDEX_1 = 1;
27 : constexpr uint8_t TA_GEAR_INDEX_2 = 2;
28 : constexpr uint8_t TA_GEAR_INDEX_3 = 3;
29 :
30 : constexpr uint8_t TA_HW_GEAR0_BASE = 0;
31 : constexpr uint8_t TA_HW_GEAR1_BASE = 8;
32 : constexpr uint8_t TA_HW_GEAR2_BASE = 16;
33 : constexpr uint8_t TA_HW_GEAR3_BASE = 24;
34 :
35 : static constexpr uint32_t TA_TIMEOUT_MS_GEAR0 = 512;
36 : static constexpr uint32_t TA_TIMEOUT_MS_GEAR1 = 1000;
37 : static constexpr uint32_t TA_TIMEOUT_MS_GEAR2 = 8000;
38 : static constexpr uint32_t TA_TIMEOUT_MS_GEAR3 = 32000;
39 :
40 : constexpr uint8_t AT_GEAR_MIN = 0;
41 : constexpr uint8_t AT_GEAR_MAX = 3;
42 : constexpr uint8_t AT_GEAR_DEFAULT = 2;
43 : constexpr uint32_t AT_TIMEOUT_MAP[4] = {16, 128, 1000, 4000};
44 :
45 : /// 与 GetTpInfo / ReleaseTpInfo 中 info、req 两级 map 的 qos 键一致（param.qos 低 8 位）
46 : using QosKey = uint32_t;
47 :
48 : /*
49 : * TP信息，当前申请TpHandle，不感知具体TP信息，当前仅支持TP与CTP
50 : * tpHandle: 对应管控面的TPID与相关资源，URMA通过引用计数管理申请和销毁TP
51 : */
52 : using TpHandle = uint64_t;
53 : struct TpInfo {
54 : TpHandle tpHandle{0};
55 : uint32_t mappedJettyPriority{0};
56 : bool hasMappedJettyPriority{false};
57 :
58 530 : TpInfo() = default;
59 : TpInfo(const TpHandle handle) : tpHandle(handle) {}
60 : };
61 :
62 : struct TpAttrInfo {
63 : struct TpAttr tpAttr {};
64 :
65 20 : TpAttrInfo() = default;
66 4 : TpAttrInfo(const struct TpAttr& attr) : tpAttr(attr) {}
67 : };
68 :
69 : struct GetTpAttrParam {
70 : TpHandle tpHandle{0};
71 : uint32_t attrBitmap{0};
72 :
73 : GetTpAttrParam() = default;
74 14 : GetTpAttrParam(const TpHandle handle, const uint32_t bitmap) : tpHandle(handle), attrBitmap(bitmap) {}
75 :
76 4 : std::string Describe() const
77 : {
78 4 : return Hccl::StringFormat("GetTpAttrParam[tpHandle=0x%llx, attrBitmap=0x%x]", tpHandle, attrBitmap);
79 : }
80 : };
81 :
82 : class TpManager {
83 : public:
84 : static TpManager& GetInstance(const int32_t deviceLogicId);
85 : void Init();
86 : /// `isSync==true`：同步路径（HOST_NET ctx + HrtRa* 同步接口），一次返回 SUCCESS。
87 : /// `isSync==false`：异步三阶段，轮询返回 HCCL_E_AGAIN。
88 : HcclResult GetTpInfo(const RaUbGetTpInfoParam& param, TpInfo& tpInfo, bool isSync = false);
89 : // unimport jetty 会 URMA 销毁 tp 资源，hccl 配套删除记录
90 : HcclResult ReleaseTpInfo(const RaUbGetTpInfoParam& param, const TpInfo& tpInfo);
91 : HcclResult GetTpAttr(const GetTpAttrParam& param, TpAttrInfo& tpAttrInfo, RdmaHandle rdmaHandle);
92 : HcclResult ReleaseTpAttr(const TpHandle tpHandle, const TpAttrInfo& tpAttrInfo);
93 :
94 : static HcclResult GetTpTotalTimeout(const TpAttrInfo& tpAttrInfo, uint32_t& tpTimeOutMs);
95 : static uint8_t CalcTaTimeout(const TpAttrInfo& tpAttrInfo);
96 : static uint32_t TaHwValueToMs(uint8_t hwValue);
97 : static uint8_t FindMinTaHwValue(uint32_t tpTotalTimeoutMs);
98 :
99 : private:
100 : bool initFlag{false};
101 : uint32_t devLogicId{0};
102 : uint32_t devPhyId{0};
103 :
104 : struct TpInfoCtx {
105 : TpInfo tpInfo{};
106 : uint32_t useCnt{0};
107 :
108 17 : TpInfoCtx() = default;
109 17 : TpInfoCtx(const TpInfo& info, const uint32_t cnt) : tpInfo(info), useCnt(cnt) {}
110 : };
111 :
112 : struct TpAttrCtx {
113 : TpAttrInfo tpAttrInfo{};
114 : uint32_t useCnt{0};
115 :
116 4 : TpAttrCtx() = default;
117 4 : TpAttrCtx(const TpAttrInfo& info, const uint32_t cnt) : tpAttrInfo(info), useCnt(cnt) {}
118 : };
119 :
120 : /*
121 : * Request上下文，保存查询TP信息相关调用异步接口出参
122 : * handle: 异步接口调用handle，用于查询处理结果
123 : * tpInfoNum: 查询到的TP信息个数，当前为复用TP，只会申请1个
124 : * dataBuffer: 查询到的TP信息数据，原始数据保留缓冲区
125 : */
126 : struct RequestCtx {
127 : enum class ReqPhase : uint8_t { WAIT_LIST = 0, WAIT_TP_ATTR = 1 };
128 : ReqPhase phase{ReqPhase::WAIT_LIST};
129 : RequestHandle handle{0};
130 : uint32_t tpInfoNum{0};
131 : std::vector<char_t> dataBuffer;
132 : TpAttr tpAttr{};
133 : uint32_t tpAttrBitmap{0};
134 : };
135 :
136 : struct TpAttrRequestCtx {
137 : RequestHandle handle{0};
138 : struct TpAttr tpAttr {};
139 : uint32_t attrBitmap{0};
140 : };
141 :
142 : /// 三级索引：先按本端 IP，再按对端 IP，最后按 QoS 键（`QosKey`：`param.qos & 0xFF`，与 next `TpMgr` 一致）。
143 : using InfoQosMap = std::unordered_map<uint32_t, TpInfoCtx>;
144 : using InfoRmtMap = std::unordered_map<IpAddress, InfoQosMap>;
145 : using InfoCtxMap = std::unordered_map<IpAddress, InfoRmtMap>;
146 : using ReqQosMap = std::unordered_map<uint32_t, RequestCtx>;
147 : using ReqRmtMap = std::unordered_map<IpAddress, ReqQosMap>;
148 : using ReqCtxMap = std::unordered_map<IpAddress, ReqRmtMap>;
149 :
150 : using TpAttrCtxMap = std::unordered_map<TpHandle, TpAttrCtx>;
151 : using TpAttrReqCtxMap = std::unordered_map<TpHandle, TpAttrRequestCtx>;
152 :
153 : InfoCtxMap ctpInfoMap;
154 : ReqCtxMap ctpReqMap;
155 :
156 : InfoCtxMap tpInfoMap;
157 : ReqCtxMap tpReqMap;
158 :
159 : InfoCtxMap uboeInfoMap;
160 : ReqCtxMap uboeReqMap;
161 :
162 : InfoCtxMap ubgInfoMap;
163 : ReqCtxMap ubgReqMap;
164 :
165 : TpAttrCtxMap tpAttrCtxMap;
166 : TpAttrReqCtxMap tpAttrReqCtxMap;
167 :
168 : std::mutex ctpInfoMutex;
169 : std::mutex ctpReqMutex;
170 :
171 : std::mutex tpInfoMutex;
172 : std::mutex tpReqMutex;
173 :
174 : std::mutex uboeInfoMutex;
175 : std::mutex uboeReqMutex;
176 :
177 : std::mutex ubgInfoMutex;
178 : std::mutex ubgReqMutex;
179 :
180 : std::mutex tpAttrCtxMutex;
181 : std::mutex tpAttrReqMutex;
182 :
183 67 : TpManager() = default;
184 67 : ~TpManager() = default;
185 : TpManager(const TpManager& that) = delete;
186 : TpManager& operator=(const TpManager& that) = delete;
187 :
188 : HcclResult FindAndGetTpInfo(const RaUbGetTpInfoParam& param, TpInfo& tpInfo);
189 : HcclResult RunSyncGetTpInfo(const RaUbGetTpInfoParam& param, TpInfo& tpInfo);
190 : HcclResult RunAsyncGetTpInfo(const RaUbGetTpInfoParam& param, TpInfo& tpInfo);
191 : HcclResult StoreTpInfoResult(const RaUbGetTpInfoParam& param, TpInfo& tpInfo);
192 : HcclResult SyncGetFirstTpAttrForSlPolicy(
193 : const RaUbGetTpInfoParam& param, uint64_t firstTpHandle, TpAttr& tpAttr, uint32_t& attrBitmap) const;
194 : HcclResult AdvanceDeviceWaitListPhase(
195 : const RaUbGetTpInfoParam& param, RequestCtx& reqCtx, ReqQosMap& qosReqMap, ReqQosMap::iterator it,
196 : std::unique_lock<std::mutex>& reqCtxLock, TpInfo& tpInfo);
197 : void StartGetTpInfoListRequest(const RaUbGetTpInfoParam& param, RequestCtx& reqCtx, bool isSync) const;
198 : HcclResult FindAndGetTpAttr(const TpHandle tpHandle, TpAttrInfo& tpAttrInfo);
199 : HcclResult
200 : StartGetTpAttrRequest(const GetTpAttrParam& param, TpAttrRequestCtx& reqCtx, RdmaHandle rdmaHandle) const;
201 : HcclResult
202 : HandleCompletedTpAttrRequest(const TpAttrRequestCtx reqCtx, const TpHandle tpHandle, TpAttrInfo& tpAttrInfo);
203 :
204 : void StartGetTpAttrForFirstTpDevice(const RaUbGetTpInfoParam& param, RequestCtx& reqCtx) const;
205 : HcclResult
206 : HandleCompletedRequest(const RequestCtx reqCtx, const RaUbGetTpInfoParam& param, TpInfo& tpInfo, bool withSlPolicy);
207 : HcclResult
208 : MapTpInfoFromTpAttr(const RaUbGetTpInfoParam& param, const RequestCtx& reqCtx, TpInfo& outTpInfo, bool isSync);
209 :
210 : bool CheckRequestResult(RequestHandle& reqHandle) const;
211 : InfoCtxMap& GetInfoCtxMap(const TpProtocol tpProtocol);
212 : ReqCtxMap& GetReqCtxMap(const TpProtocol tpProtocol);
213 : std::mutex& GetInfoCtxMutex(const TpProtocol tpProtocol);
214 : std::mutex& GetReqCtxMutex(const TpProtocol tpProtocol);
215 : };
216 :
217 : /// UbConnection 释放 TpInfo：Release 键须与 GetTpInfo 时的业务 QoS 一致
218 : void ReleaseUbConnectionTp(
219 : int32_t devLogicId, const IpAddress& locAddr, const IpAddress& rmtAddr, TpProtocol tpProtocol, TpInfo& tpInfo,
220 : uint32_t requestQos);
221 :
222 : } // namespace Hccl
223 :
224 : #endif // HCCLV2_TP_MANAGER_H
|
