Line data Source code
1 : /**
2 : * Copyright (c) 2026 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #ifndef AICPU_TS_ROCE_CHANNEL_V2_H
12 : #define AICPU_TS_ROCE_CHANNEL_V2_H
13 :
14 : #include "../channel.h"
15 : #include "aicpu_ts_channel_helper.h"
16 : #include "enum_factory.h"
17 : #include "hccl_common.h"
18 : #include "../../sockets/socket_mgr.h"
19 : #include "mem_device_pub.h"
20 : #include <mutex>
21 : #include "hcomm/hcomm_res_entity_defs.h"
22 :
23 : // Orion
24 : #include "../../../../../../legacy/ascend950/unified_platform/resource/socket/socket.h"
25 : #include "../../../../../../legacy/ascend950/unified_platform/resource/buffer/local_rdma_rma_buffer_v2.h"
26 : #include "remote_rma_buffer.h"
27 : #include "./dev_rdma_connection_v2.h"
28 : #include "rdma_local_notify.h"
29 : #include "dev_buffer.h"
30 :
31 : namespace hcomm {
32 : /**
33 : * @note 职责：Channel的AicpuTs通信引擎、RoCE协议的类派生
34 : */
35 : constexpr u32 RDMA_NOTIFY_NUM = 3;
36 :
37 : typedef decltype(EndpointLoc::device) EndpointDeviceLoc;
38 :
39 : class AicpuTsRoceChannelV2 final : public Channel {
40 : public:
41 54 : MAKE_ENUM(RdmaStatus, INIT, SOCKET_OK, QP_CREATED, DATA_EXCHANGE, QP_MODIFIED, CONN_OK)
42 :
43 : AicpuTsRoceChannelV2(EndpointHandle endpointHandle, HcommChannelDesc channelDesc, CommEngine engine);
44 : ~AicpuTsRoceChannelV2() override;
45 :
46 : HcclResult Init() override;
47 : HcclResult GetNotifyNum(uint32_t* notifyNum) const override;
48 : HcclResult GetBufferNum(uint32_t* bufferNum) const;
49 : HcclResult GetQpNum(uint32_t* qpNum) const;
50 : HcclResult GetRemoteMems(uint32_t* memNum, CommMem** remoteMem, char*** memInfos) override;
51 : ChannelStatus GetStatus() override;
52 : HcclResult GetStatus(ChannelStatus& status);
53 : HcclResult ProcessStatus();
54 : HcommChannelKind GetChannelKind() const override;
55 0 : const HcommChannelDesc& GetChannelDesc() const override { return channelDesc_; }
56 :
57 : std::string Describe() const;
58 :
59 7 : CommEngine GetCommEngine() const { return engine_; }
60 7 : CommProtocol GetCommProtocol() const { return channelDesc_.remoteEndpoint.protocol; }
61 :
62 : HcclResult BuildAndGetDevChannelEntity(uint64_t* devChannelEntityPtr);
63 : HcclResult PreAllocDevChannelEntity(uint64_t* devChannelEntityPtr);
64 : HcclResult FillDevChannelEntity();
65 :
66 : HcclResult H2DResPack(std::vector<char>& buffer);
67 :
68 : HcclResult Serialize(std::shared_ptr<hccl::DeviceMem>& out) override;
69 :
70 : HcclResult Clean() override;
71 : HcclResult Resume() override;
72 :
73 0 : HcclResult NotifyRecord([[maybe_unused]] const uint32_t remoteNotifyIdx) override { return HCCL_SUCCESS; }
74 : HcclResult
75 0 : NotifyWait([[maybe_unused]] const uint32_t localNotifyIdx, [[maybe_unused]] const uint32_t timeout) override
76 : {
77 0 : return HCCL_SUCCESS;
78 : }
79 0 : HcclResult WriteWithNotify(
80 : [[maybe_unused]] void* dst, [[maybe_unused]] const void* src, [[maybe_unused]] const uint64_t len,
81 : [[maybe_unused]] uint32_t remoteNotifyIdx) override
82 : {
83 0 : return HCCL_SUCCESS;
84 : }
85 : HcclResult
86 0 : Write([[maybe_unused]] void* dst, [[maybe_unused]] const void* src, [[maybe_unused]] uint64_t len) override
87 : {
88 0 : return HCCL_SUCCESS;
89 : }
90 : HcclResult
91 0 : Read([[maybe_unused]] void* dst, [[maybe_unused]] const void* src, [[maybe_unused]] uint64_t len) override
92 : {
93 0 : return HCCL_SUCCESS;
94 : }
95 0 : HcclResult ChannelFence() override { return HCCL_SUCCESS; }
96 :
97 0 : AicpuTsChannelHelper* GetAicpuTsHelper() override { return &aicpuTsHelper_; }
98 :
99 : private:
100 : AicpuTsChannelHelper aicpuTsHelper_;
101 : HcclResult ParseInputParam();
102 : HcclResult BuildConnection();
103 : HcclResult BuildNotify();
104 : HcclResult BuildBuffer();
105 : HcclResult BuildNotifyValueBuffer();
106 : HcclResult BuildSocket();
107 : HcclResult StartListen();
108 :
109 : HcclResult CheckSocketStatus();
110 : HcclResult CreateQp();
111 : HcclResult ExchangeData();
112 : HcclResult ModifyQp();
113 :
114 : void NotifyVecPack(Hccl::BinaryStream& binaryStream);
115 : HcclResult BufferVecPack(Hccl::BinaryStream& binaryStream);
116 : HcclResult ConnVecPack(Hccl::BinaryStream& binaryStream);
117 :
118 : HcclResult NotifyVecUnpack(Hccl::BinaryStream& binaryStream);
119 : HcclResult RmtBufferVecUnpackProc(Hccl::BinaryStream& binaryStream);
120 : HcclResult ConnVecUnpackProc(Hccl::BinaryStream& binaryStream);
121 :
122 : HcclResult BuildAndGetLocNotifyInfo(RegedNotifyEntity** notify);
123 : HcclResult BuildAndGetRmtNotifyInfo(RegedNotifyEntity** notify);
124 : HcclResult BuildAndGetRmtBufInfo(std::vector<RegedBufferEntity>& bufList, RegedBufferEntity** bufferEntityPtr);
125 : HcclResult BuildAndGetLocBufInfo(std::vector<RegedBufferEntity>& bufList, RegedBufferEntity** bufferEntityPtr);
126 : HcclResult BuildAndGetSqContext(std::vector<SqContext>& sqList, SqContext** sqContextPtr);
127 : HcclResult BuildAndGetCqContext(std::vector<CqContext>& cqList, CqContext** cqContextPtr);
128 : HcclResult BuildHostEntity(
129 : ChannelEntity& hostEntity, std::vector<RegedBufferEntity>& locBufList,
130 : std::vector<RegedBufferEntity>& rmtBufList, std::vector<SqContext>& sqList, std::vector<CqContext>& cqList);
131 :
132 : void FreeDeviceMemories();
133 : void ReleaseDeviceEntitySlab();
134 :
135 : std::vector<char> GetLocalNotifyUniqueIds() const;
136 : std::vector<char> GetRemoteNotifyUniqueIds() const;
137 : std::vector<char> GetLocBufferUniqueIds() const;
138 : std::vector<char> GetRmtBufferUniqueIds() const;
139 : std::vector<char> GetNotifyValueBufferUniqueIds() const;
140 : std::vector<char> GetSingleRmaBufferUniqueId(u64 addr, u64 size, u32 key) const;
141 : std::vector<char> GetConnUniqueIds() const;
142 : std::vector<char> GetUniqueId() const;
143 : HcclResult PackOpData(std::vector<char>& data) const;
144 :
145 : // 入参
146 : EndpointHandle endpointHandle_;
147 : HcommChannelDesc channelDesc_;
148 : CommEngine engine_;
149 :
150 : // 转换参数
151 : EndpointDesc localEp_;
152 : EndpointDesc remoteEp_;
153 : uint32_t notifyNum_{0};
154 : Hccl::Socket* socket_{nullptr};
155 : const Hccl::SocketConfig* socketConfig_{nullptr};
156 : RdmaHandle rdmaHandle_{nullptr};
157 : uint32_t devicePhyId_{};
158 :
159 : std::vector<std::unique_ptr<DevRdmaConnectionV2>> connections_{};
160 : std::vector<Hccl::LocalRdmaRmaBuffer*> localRmaBuffers_{};
161 : std::vector<std::unique_ptr<Hccl::RdmaLocalNotify>> localNotifies_{};
162 : std::shared_ptr<Hccl::DevBuffer> notifyValueMem_{nullptr};
163 : std::unique_ptr<Hccl::LocalRdmaRmaBuffer> notifyValueBuffer_{nullptr};
164 : uint32_t bufferNum_{0};
165 : uint32_t connNum_{0};
166 : ChannelStatus channelStatus_{ChannelStatus::INIT};
167 : RdmaStatus rdmaStatus_{RdmaStatus::INIT};
168 : std::vector<std::unique_ptr<Hccl::RemoteRdmaRmaBuffer>> remoteNotifies_{};
169 : std::vector<std::unique_ptr<Hccl::RemoteRdmaRmaBuffer>> rmtRmaBuffers_{};
170 : ExchangeRdmaConnDto rmtConnDto_;
171 : std::mutex remoteMemsMutex_{};
172 : std::vector<CommMem> remoteUserMems_{};
173 : std::vector<std::string> memInfoCopies_{};
174 : std::vector<char*> memInfoPointers_{};
175 : bool cacheValid_{false};
176 :
177 : void* devChannelEntitySlab_{nullptr};
178 : size_t devChannelEntitySlabSize_{0};
179 : };
180 :
181 : } // namespace hcomm
182 :
183 : #endif // AICPU_TS_ROCE_CHANNEL_V2_H
|
