Line data Source code
1 : /**
2 : * Copyright (c) 2026 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 : #ifndef AICPU_TS_ROCE_CHANNEL_V2_H
11 : #define AICPU_TS_ROCE_CHANNEL_V2_H
12 :
13 : #include "../channel.h"
14 : #include "aicpu_ts_channel_helper.h"
15 : #include "enum_factory.h"
16 : #include "hccl_common.h"
17 : #include "../../sockets/socket_mgr.h"
18 : #include "mem_device_pub.h"
19 : #include <mutex>
20 : #include "hcomm/hcomm_res_entity_defs.h"
21 :
22 : // Orion
23 : #include "../../../../../../legacy/ascend950/unified_platform/resource/socket/socket.h"
24 : #include "../../../../../../legacy/ascend950/unified_platform/resource/buffer/local_rdma_rma_buffer.h"
25 : #include "remote_rma_buffer.h"
26 : #include "./dev_rdma_connection_v2.h"
27 : #include "rdma_local_notify.h"
28 : #include "dev_buffer.h"
29 :
30 : namespace hcomm {
31 : /**
32 : * @note 职责：Channel的AicpuTs通信引擎、RoCE协议的类派生
33 : */
34 : constexpr u32 RDMA_NOTIFY_NUM = 3;
35 :
36 : typedef decltype(EndpointLoc::device) EndpointDeviceLoc;
37 :
38 : class AicpuTsRoceChannelV2 final : public Channel {
39 : public:
40 54 : MAKE_ENUM(RdmaStatus, INIT, SOCKET_OK, QP_CREATED, DATA_EXCHANGE, QP_MODIFIED, CONN_OK)
41 :
42 : AicpuTsRoceChannelV2(EndpointHandle endpointHandle, HcommChannelDesc channelDesc, CommEngine engine);
43 : ~AicpuTsRoceChannelV2() override;
44 :
45 : HcclResult Init() override;
46 : HcclResult GetNotifyNum(uint32_t *notifyNum) const override;
47 : HcclResult GetBufferNum(uint32_t *bufferNum) const;
48 : HcclResult GetQpNum(uint32_t *qpNum) const;
49 : HcclResult GetRemoteMems(uint32_t *memNum, CommMem **remoteMem, char ***memInfos) override;
50 : ChannelStatus GetStatus() override;
51 : HcclResult GetStatus(ChannelStatus &status);
52 : HcclResult ProcessStatus();
53 : HcommChannelKind GetChannelKind() const override;
54 0 : const HcommChannelDesc& GetChannelDesc() const override { return channelDesc_; }
55 :
56 : std::string Describe() const;
57 :
58 7 : CommEngine GetCommEngine() const {
59 7 : return engine_;
60 : }
61 7 : CommProtocol GetCommProtocol() const {
62 7 : return channelDesc_.remoteEndpoint.protocol;
63 : }
64 :
65 : HcclResult BuildAndGetDevChannelEntity(uint64_t* devChannelEntityPtr);
66 : HcclResult PreAllocDevChannelEntity(uint64_t* devChannelEntityPtr);
67 : HcclResult FillDevChannelEntity();
68 :
69 : HcclResult H2DResPack(std::vector<char>& buffer);
70 :
71 : HcclResult Serialize(std::shared_ptr<hccl::DeviceMem> &out) override;
72 :
73 : HcclResult Clean() override;
74 : HcclResult Resume() override;
75 :
76 0 : HcclResult NotifyRecord(const uint32_t remoteNotifyIdx) override { return HCCL_SUCCESS; }
77 0 : HcclResult NotifyWait(const uint32_t localNotifyIdx, const uint32_t timeout) override { return HCCL_SUCCESS; }
78 0 : HcclResult WriteWithNotify(void *dst, const void *src, const uint64_t len, uint32_t remoteNotifyIdx) override { return HCCL_SUCCESS; }
79 0 : HcclResult Write(void *dst, const void *src, uint64_t len) override { return HCCL_SUCCESS; }
80 0 : HcclResult Read(void *dst, const void *src, uint64_t len) override { return HCCL_SUCCESS; }
81 0 : HcclResult ChannelFence() override { return HCCL_SUCCESS; }
82 :
83 0 : AicpuTsChannelHelper *GetAicpuTsHelper() override { return &aicpuTsHelper_; }
84 :
85 : private:
86 : AicpuTsChannelHelper aicpuTsHelper_;
87 : HcclResult ParseInputParam();
88 : HcclResult BuildConnection();
89 : HcclResult BuildNotify();
90 : HcclResult BuildBuffer();
91 : HcclResult BuildNotifyValueBuffer();
92 : HcclResult BuildSocket();
93 : HcclResult StartListen();
94 :
95 : HcclResult CheckSocketStatus();
96 : HcclResult CreateQp();
97 : HcclResult ExchangeData();
98 : HcclResult ModifyQp();
99 :
100 : void NotifyVecPack(Hccl::BinaryStream &binaryStream);
101 : HcclResult BufferVecPack(Hccl::BinaryStream &binaryStream);
102 : HcclResult ConnVecPack(Hccl::BinaryStream &binaryStream);
103 :
104 : HcclResult NotifyVecUnpack(Hccl::BinaryStream &binaryStream);
105 : HcclResult RmtBufferVecUnpackProc(Hccl::BinaryStream &binaryStream);
106 : HcclResult ConnVecUnpackProc(Hccl::BinaryStream &binaryStream);
107 :
108 : HcclResult BuildAndGetLocNotifyInfo(RegedNotifyEntity** notify);
109 : HcclResult BuildAndGetRmtNotifyInfo(RegedNotifyEntity** notify);
110 : HcclResult BuildAndGetRmtBufInfo(std::vector<RegedBufferEntity>& bufList, RegedBufferEntity** bufferEntityPtr);
111 : HcclResult BuildAndGetLocBufInfo(std::vector<RegedBufferEntity>& bufList, RegedBufferEntity** bufferEntityPtr);
112 : HcclResult BuildAndGetSqContext(std::vector<SqContext>& sqList, SqContext** sqContextPtr);
113 : HcclResult BuildAndGetCqContext(std::vector<CqContext>& cqList, CqContext** cqContextPtr);
114 : HcclResult BuildHostEntity(ChannelEntity &hostEntity,
115 : std::vector<RegedBufferEntity> &locBufList, std::vector<RegedBufferEntity> &rmtBufList,
116 : std::vector<SqContext> &sqList, std::vector<CqContext> &cqList);
117 :
118 : void FreeDeviceMemories();
119 : void ReleaseDeviceEntitySlab();
120 :
121 : std::vector<char> GetLocalNotifyUniqueIds() const;
122 : std::vector<char> GetRemoteNotifyUniqueIds() const;
123 : std::vector<char> GetLocBufferUniqueIds() const;
124 : std::vector<char> GetRmtBufferUniqueIds() const;
125 : std::vector<char> GetNotifyValueBufferUniqueIds() const;
126 : std::vector<char> GetSingleRmaBufferUniqueId(u64 addr, u64 size, u32 key) const;
127 : std::vector<char> GetConnUniqueIds() const;
128 : std::vector<char> GetUniqueId() const;
129 : HcclResult PackOpData(std::vector<char> &data) const;
130 :
131 : // 入参
132 : EndpointHandle endpointHandle_;
133 : HcommChannelDesc channelDesc_;
134 : CommEngine engine_;
135 :
136 : // 转换参数
137 : EndpointDesc localEp_;
138 : EndpointDesc remoteEp_;
139 : uint32_t notifyNum_{0};
140 : Hccl::Socket* socket_{nullptr};
141 : const Hccl::SocketConfig* socketConfig_{nullptr};
142 : RdmaHandle rdmaHandle_{nullptr};
143 : uint32_t devicePhyId_{};
144 :
145 : std::vector<std::unique_ptr<DevRdmaConnectionV2>> connections_{};
146 : std::vector<Hccl::LocalRdmaRmaBuffer *> localRmaBuffers_{};
147 : std::vector<std::unique_ptr<Hccl::RdmaLocalNotify>> localNotifies_{};
148 : std::shared_ptr<Hccl::DevBuffer> notifyValueMem_{nullptr};
149 : std::unique_ptr<Hccl::LocalRdmaRmaBuffer> notifyValueBuffer_{nullptr};
150 : uint32_t bufferNum_{0};
151 : uint32_t connNum_{0};
152 : ChannelStatus channelStatus_{ChannelStatus::INIT};
153 : RdmaStatus rdmaStatus_{RdmaStatus::INIT};
154 : std::vector<std::unique_ptr<Hccl::RemoteRdmaRmaBuffer>> remoteNotifies_{};
155 : std::vector<std::unique_ptr<Hccl::RemoteRdmaRmaBuffer>> rmtRmaBuffers_{};
156 : ExchangeRdmaConnDto rmtConnDto_;
157 : std::mutex remoteMemsMutex_{};
158 : std::vector<CommMem> remoteUserMems_{};
159 : std::vector<std::string> memInfoCopies_{};
160 : std::vector<char*> memInfoPointers_{};
161 : bool cacheValid_{false};
162 :
163 : void * devChannelEntitySlab_{nullptr};
164 : size_t devChannelEntitySlabSize_{0};
165 : };
166 :
167 : } // namespace hcomm
168 :
169 : #endif // AICPU_TS_ROCE_CHANNEL_V2_H
|
