Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #ifndef HOST_CPU_ROCE_CHANNEL_H
12 : #define HOST_CPU_ROCE_CHANNEL_H
13 :
14 : #include <mutex>
15 :
16 : #include "../channel.h"
17 : #include "enum_factory.h"
18 : #include "hccl_common.h"
19 : #include "../../sockets/socket_mgr.h"
20 : #include "infiniband/verbs.h"
21 :
22 : // Orion
23 : #include "../../../../../../legacy/ascend950/unified_platform/resource/socket/socket.h"
24 : #include "../../../../../../legacy/ascend950/unified_platform/resource/buffer/local_rdma_rma_buffer_v2.h"
25 : #include "remote_rma_buffer.h"
26 : #include "host_rdma_connection.h"
27 : #include "task_param.h"
28 :
29 : #include "exchange_data_format.h"
30 : #include "private_types.h"
31 :
32 : namespace hcomm {
33 :
34 : class HostCpuRoceChannel final : public Channel {
35 : public:
36 847 : MAKE_ENUM(RdmaStatus, INIT, SOCKET_OK, CAP_EXCHANGED, QP_CREATED, DATA_EXCHANGE, QP_MODIFIED, CONN_OK)
37 :
38 : HostCpuRoceChannel(EndpointHandle endpointHandle, HcommChannelDesc channelDesc);
39 : ~HostCpuRoceChannel();
40 :
41 : HcclResult Init() override;
42 : HcclResult GetNotifyNum(uint32_t* notifyNum) const override;
43 : HcclResult GetRemoteMems(uint32_t* memNum, CommMem** remoteMem, char*** memInfos) override;
44 : ChannelStatus GetStatus() override;
45 : HcclResult GetStatus(ChannelStatus& status);
46 : HcclResult ProcessStatus();
47 1 : const HcommChannelDesc& GetChannelDesc() const override { return channelDesc_; }
48 :
49 : std::string Describe() const;
50 :
51 : HcclResult SetDfxCallback(std::function<HcclResult(const Hccl::TaskParam&, u64)> callback);
52 :
53 : // 数据面调用verbs接口
54 : HcclResult NotifyRecord(const uint32_t remoteNotifyIdx) override;
55 : HcclResult NotifyWait(const uint32_t localNotifyIdx, const uint32_t timeout) override;
56 : HcclResult WriteWithNotify(void* dst, const void* src, const uint64_t len, uint32_t remoteNotifyIdx) override;
57 : HcclResult Write(void* dst, const void* src, uint64_t len) override;
58 : HcclResult Read(void* dst, const void* src, uint64_t len) override;
59 : HcclResult ChannelFence() override;
60 : HcclResult GetHcclBuffer(void*& addr, uint64_t& size);
61 :
62 : private:
63 : HcclResult WaitForFenceCompletion();
64 :
65 : virtual HcclResult Clean() override;
66 : HcclResult Resume() override;
67 : HcclResult ExchangeCapability();
68 : HcclResult ExchangeDataHybird();
69 : HcclResult GetRemoteAddrHybird(hccl::MemType memType, u8*& data, u64& size);
70 : HcclResult ParseRecvExchangeDataHybird();
71 : HcclResult BuildExchangeDataHybird();
72 : HcclResult BuildExchangeDataLengthHybird();
73 :
74 : HcclResult RegisterUserMemHybird();
75 : HcclResult BuildNotifyWrHybird(const uint32_t remoteNotifyIdx, struct ibv_send_wr& notifRecordWr);
76 : HcclResult WriteWithNotifyHybrid(void* dst, const void* src, uint64_t len, uint32_t remoteNotifyIdx);
77 : HcclResult NotifyWaitHybrid(uint32_t localNotifyIdx, uint32_t timeout);
78 :
79 : HcclResult CreateNotifyHybird(hccl::MemType notifyType, uint32_t notifyId);
80 : HcclResult CreateNotifyValueBufferHybird();
81 : HcclResult CreateNotifyBufferHybird(hccl::MemType notifyType, uint32_t notifyId, u8*& data, u64& size);
82 : hccl::MemType NotifyIdToMemtypeHybird(uint32_t remoteNotifyIdx);
83 : HcclResult ConnectSingleQpHybrid(std::function<bool()> needStop);
84 :
85 : private:
86 : HcclResult ParseInputParam();
87 : HcclResult StartListen();
88 : HcclResult BuildSocket();
89 : HcclResult BuildConnection();
90 : HcclResult BuildNotify();
91 : HcclResult BuildBuffer();
92 :
93 : HcclResult CheckSocketStatus();
94 : HcclResult CreateQp();
95 : HcclResult ExchangeData();
96 : HcclResult ModifyQp();
97 : HcclResult SyncAfterModifyQp();
98 :
99 : void NotifyVecPack(Hccl::BinaryStream& binaryStream);
100 : HcclResult BufferVecPack(Hccl::BinaryStream& binaryStream);
101 : HcclResult ConnVecPack(Hccl::BinaryStream& binaryStream);
102 : // void HandshakeMsgPack(Hccl::BinaryStream &binaryStream);
103 :
104 : // HcclResult HandshakeMsgUnpack(Hccl::BinaryStream &binaryStream);
105 : HcclResult NotifyVecUnpack(Hccl::BinaryStream& binaryStream);
106 : HcclResult RmtBufferVecUnpackProc(Hccl::BinaryStream& binaryStream);
107 : HcclResult ConnVecUnpackProc(Hccl::BinaryStream& binaryStream);
108 :
109 : std::vector<Hccl::QpInfo> GetQpInfos() const; // in Connection
110 :
111 : HcclResult IbvPostRecv() const;
112 : HcclResult PrepareNotifyWrResource(
113 : uint32_t qpIdx, const uint64_t len, const uint32_t remoteNotifyIdx, struct ibv_send_wr& notifyRecordWr,
114 : Hccl::TaskParam& taskParam) const;
115 : HcclResult PrepareWriteWrResource(
116 : const void* dst, const void* src, const uint64_t len, const uint32_t remoteNotifyIdx,
117 : struct ibv_send_wr& writeWithNotifyWr, Hccl::TaskParam& taskParam) const;
118 :
119 : HcclResult
120 : PostRdmaOp(const char* caller, ibv_wr_opcode opcode, void* localAddr, const void* remoteAddr, const uint64_t len);
121 : void BuildRdmaWr(
122 : const char* caller, ibv_wr_opcode opcode, void* localAddr, const void* remoteAddr, uint64_t len,
123 : size_t localIdx, size_t rmtIdx, struct ibv_send_wr& wr, struct ibv_sge& sg) const;
124 : HcclResult PostAndCheckSend(struct ibv_qp* qp, const uint32_t qpIdx, const char* caller, struct ibv_send_wr& wr);
125 : HcclResult FindLocalBuffer(const uint64_t addr, const uint64_t len, size_t& targetIdx) const;
126 : HcclResult FindRemoteBuffer(const uint64_t addr, const uint64_t len, size_t& targetIdx) const;
127 : HcclResult ReportWcStatusError(enum ibv_wc_status status);
128 :
129 : // Wrapper for stub
130 0 : int IbvPollCq(ibv_cq* sendCq, uint32_t numEntries, ibv_wc* wc) const { return ibv_poll_cq(sendCq, numEntries, wc); }
131 :
132 : // 入参
133 : EndpointHandle endpointHandle_;
134 : HcommChannelDesc channelDesc_;
135 :
136 : // 转换参数
137 : EndpointDesc localEp_;
138 : EndpointDesc remoteEp_;
139 : uint32_t notifyNum_{0};
140 : Hccl::Socket* socket_{nullptr};
141 : const Hccl::SocketConfig* socketConfig_{nullptr};
142 : RdmaHandle rdmaHandle_{nullptr};
143 :
144 : std::vector<std::unique_ptr<HostRdmaConnection>> connections_{};
145 : std::vector<Hccl::LocalRdmaRmaBuffer*> localRmaBuffers_{};
146 : std::vector<uint32_t> localDpuNotifyIds_{};
147 : uint32_t bufferNum_{0};
148 : uint32_t connNum_{0};
149 : // Hccl::BaseMemTransport::Attribution attr_;
150 : ChannelStatus channelStatus_{ChannelStatus::INIT};
151 : RdmaStatus rdmaStatus_{RdmaStatus::INIT};
152 : std::vector<uint32_t> remoteDpuNotifyIds_;
153 : std::vector<std::unique_ptr<Hccl::RemoteRdmaRmaBuffer>> rmtRmaBuffers_{};
154 : std::vector<ExchangeRdmaConnDto> rmtConnDtos_;
155 : std::vector<int> wqeNums_;
156 : bool fenceFlag_{false};
157 : std::mutex remoteMemsMutex_; // 远端内存列表互斥锁
158 :
159 : // GetRemoteMems 缓存
160 : bool cacheValid_{false};
161 : std::vector<CommMem> userRemoteMems_;
162 : std::vector<std::string> memInfoCopies_;
163 : std::vector<char*> memInfoPointers_;
164 :
165 : uint64_t maxMsgSize_{0};
166 : uint32_t lbMax_{0}; // 多QP负载均衡
167 :
168 : std::function<HcclResult(const Hccl::TaskParam&, u64)> dfxCallback_;
169 :
170 : std::mutex cq_mutex;
171 : std::mutex sendCq_mutex;
172 :
173 : // ========== 混合模式（RoCE Cross-Mode）成员变量 ==========
174 : // 1. 能力协商结果
175 : RoCECapability remoteCap_; // 对端能力
176 : bool isHybridMode_ = false; // 是否为混合模式
177 :
178 : uint32_t localNotifySize_;
179 : uint32_t localNotifyAccess_;
180 :
181 : std::array<hccl::MemMsg, static_cast<u32>(hccl::MemType::MEM_TYPE_RESERVED)> localMemMsg_;
182 : std::array<hccl::MemMsg, static_cast<u32>(hccl::MemType::MEM_TYPE_RESERVED)> remoteMemMsg_;
183 : uint64_t exchangeDataTotalSize_;
184 : std::vector<uint8_t> exchangeDataForSend_;
185 : std::vector<uint8_t> exchangeDataForRecv_;
186 :
187 : uint32_t devicePhyId_{};
188 : };
189 :
190 : } // namespace hcomm
191 :
192 : #endif // HOST_CPU_ROCE_CHANNEL_H
|
