Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 : #ifndef HOST_CPU_ROCE_CHANNEL_H
11 : #define HOST_CPU_ROCE_CHANNEL_H
12 :
13 : #include <mutex>
14 :
15 : #include "../channel.h"
16 : #include "enum_factory.h"
17 : #include "hccl_common.h"
18 : #include "../../sockets/socket_mgr.h"
19 : #include "infiniband/verbs.h"
20 :
21 : // Orion
22 : #include "../../../../../../legacy/ascend950/unified_platform/resource/socket/socket.h"
23 : #include "../../../../../../legacy/ascend950/unified_platform/resource/buffer/local_rdma_rma_buffer_v2.h"
24 : #include "remote_rma_buffer.h"
25 : #include "host_rdma_connection.h"
26 : #include "task_param.h"
27 :
28 : #include "exchange_data_format.h"
29 : #include "private_types.h"
30 :
31 : namespace hcomm {
32 :
33 : class HostCpuRoceChannel final : public Channel {
34 : public:
35 847 : MAKE_ENUM(RdmaStatus, INIT, SOCKET_OK, CAP_EXCHANGED, QP_CREATED, DATA_EXCHANGE, QP_MODIFIED, CONN_OK)
36 :
37 : HostCpuRoceChannel(EndpointHandle endpointHandle, HcommChannelDesc channelDesc);
38 : ~HostCpuRoceChannel();
39 :
40 : HcclResult Init() override;
41 : HcclResult GetNotifyNum(uint32_t* notifyNum) const override;
42 : HcclResult GetRemoteMems(uint32_t* memNum, CommMem** remoteMem, char*** memInfos) override;
43 : ChannelStatus GetStatus() override;
44 : HcclResult GetStatus(ChannelStatus& status);
45 : HcclResult ProcessStatus();
46 1 : const HcommChannelDesc& GetChannelDesc() const override { return channelDesc_; }
47 :
48 : std::string Describe() const;
49 :
50 : HcclResult SetDfxCallback(std::function<HcclResult(const Hccl::TaskParam&, u64)> callback);
51 :
52 : // 数据面调用verbs接口
53 : HcclResult NotifyRecord(const uint32_t remoteNotifyIdx) override;
54 : HcclResult NotifyWait(const uint32_t localNotifyIdx, const uint32_t timeout) override;
55 : HcclResult WriteWithNotify(void* dst, const void* src, const uint64_t len, uint32_t remoteNotifyIdx) override;
56 : HcclResult Write(void* dst, const void* src, uint64_t len) override;
57 : HcclResult Read(void* dst, const void* src, uint64_t len) override;
58 : HcclResult ChannelFence() override;
59 : HcclResult GetHcclBuffer(void*& addr, uint64_t& size);
60 :
61 : private:
62 : HcclResult WaitForFenceCompletion();
63 :
64 : virtual HcclResult Clean() override;
65 : HcclResult Resume() override;
66 : HcclResult ExchangeCapability();
67 : HcclResult ExchangeDataHybird();
68 : HcclResult GetRemoteAddrHybird(hccl::MemType memType, u8*& data, u64& size);
69 : HcclResult ParseRecvExchangeDataHybird();
70 : HcclResult BuildExchangeDataHybird();
71 : HcclResult BuildExchangeDataLengthHybird();
72 :
73 : HcclResult RegisterUserMemHybird();
74 : HcclResult BuildNotifyWrHybird(const uint32_t remoteNotifyIdx, struct ibv_send_wr& notifRecordWr);
75 : HcclResult WriteWithNotifyHybrid(void* dst, const void* src, uint64_t len, uint32_t remoteNotifyIdx);
76 : HcclResult NotifyWaitHybrid(uint32_t localNotifyIdx, uint32_t timeout);
77 :
78 : HcclResult CreateNotifyHybird(hccl::MemType notifyType, uint32_t notifyId);
79 : HcclResult CreateNotifyValueBufferHybird();
80 : HcclResult CreateNotifyBufferHybird(hccl::MemType notifyType, uint32_t notifyId, u8*& data, u64& size);
81 : hccl::MemType NotifyIdToMemtypeHybird(uint32_t remoteNotifyIdx);
82 : HcclResult ConnectSingleQpHybrid(std::function<bool()> needStop);
83 :
84 : private:
85 : HcclResult ParseInputParam();
86 : HcclResult StartListen();
87 : HcclResult BuildSocket();
88 : HcclResult BuildConnection();
89 : HcclResult BuildNotify();
90 : HcclResult BuildBuffer();
91 :
92 : HcclResult CheckSocketStatus();
93 : HcclResult CreateQp();
94 : HcclResult ExchangeData();
95 : HcclResult ModifyQp();
96 : HcclResult SyncAfterModifyQp();
97 :
98 : void NotifyVecPack(Hccl::BinaryStream& binaryStream);
99 : HcclResult BufferVecPack(Hccl::BinaryStream& binaryStream);
100 : HcclResult ConnVecPack(Hccl::BinaryStream& binaryStream);
101 : // void HandshakeMsgPack(Hccl::BinaryStream &binaryStream);
102 :
103 : // HcclResult HandshakeMsgUnpack(Hccl::BinaryStream &binaryStream);
104 : HcclResult NotifyVecUnpack(Hccl::BinaryStream& binaryStream);
105 : HcclResult RmtBufferVecUnpackProc(Hccl::BinaryStream& binaryStream);
106 : HcclResult ConnVecUnpackProc(Hccl::BinaryStream& binaryStream);
107 :
108 : std::vector<Hccl::QpInfo> GetQpInfos() const; // in Connection
109 :
110 : HcclResult IbvPostRecv() const;
111 : HcclResult PrepareNotifyWrResource(
112 : uint32_t qpIdx, const uint64_t len, const uint32_t remoteNotifyIdx, struct ibv_send_wr& notifyRecordWr,
113 : Hccl::TaskParam& taskParam) const;
114 : HcclResult PrepareWriteWrResource(
115 : const void* dst, const void* src, const uint64_t len, const uint32_t remoteNotifyIdx,
116 : struct ibv_send_wr& writeWithNotifyWr, Hccl::TaskParam& taskParam) const;
117 :
118 : HcclResult
119 : PostRdmaOp(const char* caller, ibv_wr_opcode opcode, void* localAddr, const void* remoteAddr, const uint64_t len);
120 : void BuildRdmaWr(
121 : const char* caller, ibv_wr_opcode opcode, void* localAddr, const void* remoteAddr, uint64_t len,
122 : size_t localIdx, size_t rmtIdx, struct ibv_send_wr& wr, struct ibv_sge& sg) const;
123 : HcclResult PostAndCheckSend(struct ibv_qp* qp, const uint32_t qpIdx, const char* caller, struct ibv_send_wr& wr);
124 : HcclResult FindLocalBuffer(const uint64_t addr, const uint64_t len, size_t& targetIdx) const;
125 : HcclResult FindRemoteBuffer(const uint64_t addr, const uint64_t len, size_t& targetIdx) const;
126 : HcclResult ReportWcStatusError(enum ibv_wc_status status);
127 :
128 : // Wrapper for stub
129 0 : int IbvPollCq(ibv_cq* sendCq, uint32_t numEntries, ibv_wc* wc) const { return ibv_poll_cq(sendCq, numEntries, wc); }
130 :
131 : // 入参
132 : EndpointHandle endpointHandle_;
133 : HcommChannelDesc channelDesc_;
134 :
135 : // 转换参数
136 : EndpointDesc localEp_;
137 : EndpointDesc remoteEp_;
138 : uint32_t notifyNum_{0};
139 : Hccl::Socket* socket_{nullptr};
140 : const Hccl::SocketConfig* socketConfig_{nullptr};
141 : RdmaHandle rdmaHandle_{nullptr};
142 :
143 : std::vector<std::unique_ptr<HostRdmaConnection>> connections_{};
144 : std::vector<Hccl::LocalRdmaRmaBuffer*> localRmaBuffers_{};
145 : std::vector<uint32_t> localDpuNotifyIds_{};
146 : uint32_t bufferNum_{0};
147 : uint32_t connNum_{0};
148 : // Hccl::BaseMemTransport::Attribution attr_;
149 : ChannelStatus channelStatus_{ChannelStatus::INIT};
150 : RdmaStatus rdmaStatus_{RdmaStatus::INIT};
151 : std::vector<uint32_t> remoteDpuNotifyIds_;
152 : std::vector<std::unique_ptr<Hccl::RemoteRdmaRmaBuffer>> rmtRmaBuffers_{};
153 : std::vector<ExchangeRdmaConnDto> rmtConnDtos_;
154 : std::vector<int> wqeNums_;
155 : bool fenceFlag_{false};
156 : std::mutex remoteMemsMutex_; // 远端内存列表互斥锁
157 :
158 : // GetRemoteMems 缓存
159 : bool cacheValid_{false};
160 : std::vector<CommMem> userRemoteMems_;
161 : std::vector<std::string> memInfoCopies_;
162 : std::vector<char*> memInfoPointers_;
163 :
164 : uint64_t maxMsgSize_{0};
165 : uint32_t lbMax_{0}; // 多QP负载均衡
166 :
167 : std::function<HcclResult(const Hccl::TaskParam&, u64)> dfxCallback_;
168 :
169 : std::mutex cq_mutex;
170 : std::mutex sendCq_mutex;
171 :
172 : // ========== 混合模式（RoCE Cross-Mode）成员变量 ==========
173 : // 1. 能力协商结果
174 : RoCECapability remoteCap_; // 对端能力
175 : bool isHybridMode_ = false; // 是否为混合模式
176 :
177 : uint32_t localNotifySize_;
178 : uint32_t localNotifyAccess_;
179 :
180 : std::array<hccl::MemMsg, static_cast<u32>(hccl::MemType::MEM_TYPE_RESERVED)> localMemMsg_;
181 : std::array<hccl::MemMsg, static_cast<u32>(hccl::MemType::MEM_TYPE_RESERVED)> remoteMemMsg_;
182 : uint64_t exchangeDataTotalSize_;
183 : std::vector<uint8_t> exchangeDataForSend_;
184 : std::vector<uint8_t> exchangeDataForRecv_;
185 :
186 : uint32_t devicePhyId_{};
187 : };
188 :
189 : } // namespace hcomm
190 :
191 : #endif // HOST_CPU_ROCE_CHANNEL_H
|
