Line data Source code
1 : /**
2 : * Copyright (c) 2026 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 : #ifndef AICPU_TS_ROCE_CHANNEL_V2_H
11 : #define AICPU_TS_ROCE_CHANNEL_V2_H
12 :
13 : #include "../channel.h"
14 : #include "aicpu_ts_channel_helper.h"
15 : #include "enum_factory.h"
16 : #include "hccl_common.h"
17 : #include "../../sockets/socket_mgr.h"
18 : #include "mem_device_pub.h"
19 : #include <mutex>
20 : #include "hcomm/hcomm_res_entity_defs.h"
21 :
22 : // Orion
23 : #include "../../../../../../legacy/ascend950/unified_platform/resource/socket/socket.h"
24 : #include "../../../../../../legacy/ascend950/unified_platform/resource/buffer/local_rdma_rma_buffer_v2.h"
25 : #include "remote_rma_buffer.h"
26 : #include "./dev_rdma_connection_v2.h"
27 : #include "rdma_local_notify.h"
28 : #include "dev_buffer.h"
29 :
30 : namespace hcomm {
31 : /**
32 : * @note 职责：Channel的AicpuTs通信引擎、RoCE协议的类派生
33 : */
34 : constexpr u32 RDMA_NOTIFY_NUM = 3;
35 :
36 : typedef decltype(EndpointLoc::device) EndpointDeviceLoc;
37 :
38 : class AicpuTsRoceChannelV2 final : public Channel {
39 : public:
40 54 : MAKE_ENUM(RdmaStatus, INIT, SOCKET_OK, QP_CREATED, DATA_EXCHANGE, QP_MODIFIED, CONN_OK)
41 :
42 : AicpuTsRoceChannelV2(EndpointHandle endpointHandle, HcommChannelDesc channelDesc, CommEngine engine);
43 : ~AicpuTsRoceChannelV2() override;
44 :
45 : HcclResult Init() override;
46 : HcclResult GetNotifyNum(uint32_t* notifyNum) const override;
47 : HcclResult GetBufferNum(uint32_t* bufferNum) const;
48 : HcclResult GetQpNum(uint32_t* qpNum) const;
49 : HcclResult GetRemoteMems(uint32_t* memNum, CommMem** remoteMem, char*** memInfos) override;
50 : ChannelStatus GetStatus() override;
51 : HcclResult GetStatus(ChannelStatus& status);
52 : HcclResult ProcessStatus();
53 : HcommChannelKind GetChannelKind() const override;
54 0 : const HcommChannelDesc& GetChannelDesc() const override { return channelDesc_; }
55 :
56 : std::string Describe() const;
57 :
58 7 : CommEngine GetCommEngine() const { return engine_; }
59 7 : CommProtocol GetCommProtocol() const { return channelDesc_.remoteEndpoint.protocol; }
60 :
61 : HcclResult BuildAndGetDevChannelEntity(uint64_t* devChannelEntityPtr);
62 : HcclResult PreAllocDevChannelEntity(uint64_t* devChannelEntityPtr);
63 : HcclResult FillDevChannelEntity();
64 :
65 : HcclResult H2DResPack(std::vector<char>& buffer);
66 :
67 : HcclResult Serialize(std::shared_ptr<hccl::DeviceMem>& out) override;
68 :
69 : HcclResult Clean() override;
70 : HcclResult Resume() override;
71 :
72 0 : HcclResult NotifyRecord(const uint32_t remoteNotifyIdx) override { return HCCL_SUCCESS; }
73 0 : HcclResult NotifyWait(const uint32_t localNotifyIdx, const uint32_t timeout) override { return HCCL_SUCCESS; }
74 0 : HcclResult WriteWithNotify(void* dst, const void* src, const uint64_t len, uint32_t remoteNotifyIdx) override
75 : {
76 0 : return HCCL_SUCCESS;
77 : }
78 0 : HcclResult Write(void* dst, const void* src, uint64_t len) override { return HCCL_SUCCESS; }
79 0 : HcclResult Read(void* dst, const void* src, uint64_t len) override { return HCCL_SUCCESS; }
80 0 : HcclResult ChannelFence() override { return HCCL_SUCCESS; }
81 :
82 0 : AicpuTsChannelHelper* GetAicpuTsHelper() override { return &aicpuTsHelper_; }
83 :
84 : private:
85 : AicpuTsChannelHelper aicpuTsHelper_;
86 : HcclResult ParseInputParam();
87 : HcclResult BuildConnection();
88 : HcclResult BuildNotify();
89 : HcclResult BuildBuffer();
90 : HcclResult BuildNotifyValueBuffer();
91 : HcclResult BuildSocket();
92 : HcclResult StartListen();
93 :
94 : HcclResult CheckSocketStatus();
95 : HcclResult CreateQp();
96 : HcclResult ExchangeData();
97 : HcclResult ModifyQp();
98 :
99 : void NotifyVecPack(Hccl::BinaryStream& binaryStream);
100 : HcclResult BufferVecPack(Hccl::BinaryStream& binaryStream);
101 : HcclResult ConnVecPack(Hccl::BinaryStream& binaryStream);
102 :
103 : HcclResult NotifyVecUnpack(Hccl::BinaryStream& binaryStream);
104 : HcclResult RmtBufferVecUnpackProc(Hccl::BinaryStream& binaryStream);
105 : HcclResult ConnVecUnpackProc(Hccl::BinaryStream& binaryStream);
106 :
107 : HcclResult BuildAndGetLocNotifyInfo(RegedNotifyEntity** notify);
108 : HcclResult BuildAndGetRmtNotifyInfo(RegedNotifyEntity** notify);
109 : HcclResult BuildAndGetRmtBufInfo(std::vector<RegedBufferEntity>& bufList, RegedBufferEntity** bufferEntityPtr);
110 : HcclResult BuildAndGetLocBufInfo(std::vector<RegedBufferEntity>& bufList, RegedBufferEntity** bufferEntityPtr);
111 : HcclResult BuildAndGetSqContext(std::vector<SqContext>& sqList, SqContext** sqContextPtr);
112 : HcclResult BuildAndGetCqContext(std::vector<CqContext>& cqList, CqContext** cqContextPtr);
113 : HcclResult BuildHostEntity(
114 : ChannelEntity& hostEntity, std::vector<RegedBufferEntity>& locBufList,
115 : std::vector<RegedBufferEntity>& rmtBufList, std::vector<SqContext>& sqList, std::vector<CqContext>& cqList);
116 :
117 : void FreeDeviceMemories();
118 : void ReleaseDeviceEntitySlab();
119 :
120 : std::vector<char> GetLocalNotifyUniqueIds() const;
121 : std::vector<char> GetRemoteNotifyUniqueIds() const;
122 : std::vector<char> GetLocBufferUniqueIds() const;
123 : std::vector<char> GetRmtBufferUniqueIds() const;
124 : std::vector<char> GetNotifyValueBufferUniqueIds() const;
125 : std::vector<char> GetSingleRmaBufferUniqueId(u64 addr, u64 size, u32 key) const;
126 : std::vector<char> GetConnUniqueIds() const;
127 : std::vector<char> GetUniqueId() const;
128 : HcclResult PackOpData(std::vector<char>& data) const;
129 :
130 : // 入参
131 : EndpointHandle endpointHandle_;
132 : HcommChannelDesc channelDesc_;
133 : CommEngine engine_;
134 :
135 : // 转换参数
136 : EndpointDesc localEp_;
137 : EndpointDesc remoteEp_;
138 : uint32_t notifyNum_{0};
139 : Hccl::Socket* socket_{nullptr};
140 : const Hccl::SocketConfig* socketConfig_{nullptr};
141 : RdmaHandle rdmaHandle_{nullptr};
142 : uint32_t devicePhyId_{};
143 :
144 : std::vector<std::unique_ptr<DevRdmaConnectionV2>> connections_{};
145 : std::vector<Hccl::LocalRdmaRmaBuffer*> localRmaBuffers_{};
146 : std::vector<std::unique_ptr<Hccl::RdmaLocalNotify>> localNotifies_{};
147 : std::shared_ptr<Hccl::DevBuffer> notifyValueMem_{nullptr};
148 : std::unique_ptr<Hccl::LocalRdmaRmaBuffer> notifyValueBuffer_{nullptr};
149 : uint32_t bufferNum_{0};
150 : uint32_t connNum_{0};
151 : ChannelStatus channelStatus_{ChannelStatus::INIT};
152 : RdmaStatus rdmaStatus_{RdmaStatus::INIT};
153 : std::vector<std::unique_ptr<Hccl::RemoteRdmaRmaBuffer>> remoteNotifies_{};
154 : std::vector<std::unique_ptr<Hccl::RemoteRdmaRmaBuffer>> rmtRmaBuffers_{};
155 : ExchangeRdmaConnDto rmtConnDto_;
156 : std::mutex remoteMemsMutex_{};
157 : std::vector<CommMem> remoteUserMems_{};
158 : std::vector<std::string> memInfoCopies_{};
159 : std::vector<char*> memInfoPointers_{};
160 : bool cacheValid_{false};
161 :
162 : void* devChannelEntitySlab_{nullptr};
163 : size_t devChannelEntitySlabSize_{0};
164 : };
165 :
166 : } // namespace hcomm
167 :
168 : #endif // AICPU_TS_ROCE_CHANNEL_V2_H
|
