Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #ifndef TRANSPORT_MEM_H
12 : #define TRANSPORT_MEM_H
13 :
14 : #include <hccl/hccl_types.h>
15 : #include <atomic>
16 : #include "dispatcher.h"
17 : #include "notify_pool.h"
18 : #include "hccl_socket.h"
19 : #include "hccl_network_pub.h"
20 : #include "hccl_common.h"
21 : #include "hccl_mem.h"
22 : #include "transport_pub.h"
23 :
24 : namespace hccl {
25 :
26 : enum class RmaMemType : int {
27 : DEVICE = 0, // device侧内存
28 : HOST = 1, // host侧内存
29 : TYPE_NUM
30 : };
31 :
32 : constexpr size_t TRANSPORT_EMD_ESC_SIZE = 512U - (sizeof(u32) * 2);
33 :
34 : class TransportMem {
35 : public:
36 : enum class TpType : int { IPC = 0, ROCE = 1, ROCE_DEVICE, TYPE_NUM };
37 :
38 : struct AttrInfo {
39 : u32 localRankId{INVALID_VALUE_RANKID};
40 : u32 remoteRankId{INVALID_VALUE_RANKID};
41 : u32 sdid{INVALID_UINT}; // 本端所属超节点
42 : u32 serverId{INVALID_UINT}; // 本端所属server
43 : u32 trafficClass{HCCL_COMM_TRAFFIC_CLASS_CONFIG_NOT_SET};
44 : u32 serviceLevel{HCCL_COMM_SERVICE_LEVEL_CONFIG_NOT_SET};
45 : u32 timeout{INVALID_UINT}; // 传输超时时间
46 : };
47 :
48 : struct RmaMemDesc {
49 : u32 localRankId;
50 : u32 remoteRankId;
51 : char memDesc[TRANSPORT_EMD_ESC_SIZE];
52 : };
53 :
54 : struct RmaMemDescs {
55 : RmaMemDesc* array;
56 : u32 arrayLength;
57 : };
58 :
59 : struct RmaOpMem {
60 : void* addr;
61 : u64 size;
62 : };
63 :
64 : struct RmaMem {
65 : RmaMemType type; // segment的内存类型
66 : void* addr; // segment的虚拟地址
67 : u64 size; // segment的size
68 : };
69 :
70 : static std::shared_ptr<TransportMem> Create(
71 : TpType tpType, const std::unique_ptr<NotifyPool>& notifyPool, const HcclNetDevCtx& netDevCtx,
72 : const HcclDispatcher& dispatcher, AttrInfo& attrInfo);
73 : static std::shared_ptr<TransportMem> Create(
74 : TpType tpType, const std::unique_ptr<NotifyPool>& notifyPool, const HcclNetDevCtx& netDevCtx,
75 : const HcclDispatcher& dispatcher, AttrInfo& attrInfo, bool aicpuUnfoldMode);
76 : // AICPU侧创建Transport
77 : static std::shared_ptr<TransportMem>
78 : Create(TpType tpType, const HcclQpInfoV2& qpInfo, const HcclDispatcher& dispatcher, AttrInfo& attrInfo);
79 :
80 : explicit TransportMem(
81 : const std::unique_ptr<NotifyPool>& notifyPool, const HcclNetDevCtx& netDevCtx, const HcclDispatcher& dispatcher,
82 : AttrInfo& attrInfo);
83 : TransportMem(
84 : const std::unique_ptr<NotifyPool>& notifyPool, const HcclNetDevCtx& netDevCtx, const HcclDispatcher& dispatcher,
85 : AttrInfo& attrInfo, bool aicpuUnfoldMode);
86 : virtual ~TransportMem();
87 : virtual HcclResult
88 : ExchangeMemDesc(const RmaMemDescs& localMemDescs, RmaMemDescs& remoteMemDescs, u32& actualNumOfRemote)
89 : = 0;
90 : virtual HcclResult EnableMemAccess(const RmaMemDesc& remoteMemDesc, RmaMem& remoteMem) = 0;
91 : virtual HcclResult DisableMemAccess(const RmaMemDesc& remoteMemDesc) = 0;
92 : virtual HcclResult SetDataSocket(const std::shared_ptr<HcclSocket>& socket);
93 :
94 : virtual HcclResult SetSocket(const std::shared_ptr<HcclSocket>& socket) = 0;
95 : virtual HcclResult Connect(s32 timeoutSec) = 0;
96 : virtual HcclResult Write(const HcclBuf& remoteMem, const HcclBuf& localMem, const rtStream_t& stream) = 0;
97 : virtual HcclResult Read(const HcclBuf& localMem, const HcclBuf& remoteMem, const rtStream_t& stream) = 0;
98 : /**
99 : * @brief 旧版Write
100 : * @deprecated 参数优化，改用 `Write(const HcclBuf &remoteMem, const HcclBuf &localMem, const rtStream_t &stream)`。
101 : */
102 : virtual HcclResult Write(const RmaOpMem& remoteMem, const RmaOpMem& localMem, const rtStream_t& stream) = 0;
103 : /**
104 : * @brief 旧版Read
105 : * @deprecated 参数优化，改用 `Read(const HcclBuf &localMem, const HcclBuf &remoteMem, const rtStream_t &stream)`。
106 : */
107 : virtual HcclResult Read(const RmaOpMem& localMem, const RmaOpMem& remoteMem, const rtStream_t& stream) = 0;
108 : virtual HcclResult AddOpFence(const rtStream_t& stream) = 0;
109 :
110 : virtual HcclResult
111 : GetTransInfo(HcclQpInfoV2& qpInfo, u32* lkey, u32* rkey, HcclBuf* localMem, HcclBuf* remoteMem, u32 num)
112 : = 0;
113 : virtual HcclResult WaitOpFence(const rtStream_t& stream) = 0;
114 :
115 : // AICPU侧批量下发读、写操作，下发wr后敲Doorbell
116 : virtual HcclResult
117 : BatchWrite(const std::vector<MemDetails>& remoteMems, const std::vector<MemDetails>& localMems, Stream& stream)
118 : = 0;
119 : virtual HcclResult
120 : BatchRead(const std::vector<MemDetails>& localMems, const std::vector<MemDetails>& remoteMems, Stream& stream)
121 : = 0;
122 : virtual HcclResult AddOpFence(const MemDetails& localFenceMem, const MemDetails& remoteFenceMem, Stream& stream)
123 : = 0;
124 :
125 : protected:
126 : // 从 string 拷贝到 memDesc
127 : HcclResult RmaMemDescCopyFromStr(RmaMemDesc& rmaMemDesc, const std::string& memDescStr) const
128 : {
129 : if (memcpy_s(rmaMemDesc.memDesc, TRANSPORT_EMD_ESC_SIZE, memDescStr.c_str(), memDescStr.size() + 1) != EOK) {
130 : return HCCL_E_INTERNAL;
131 : }
132 : return HCCL_SUCCESS;
133 : }
134 :
135 : // 从 memDesc 转换为 string
136 0 : std::string RmaMemDescCopyToStr(const RmaMemDesc& rmaMemDesc) const
137 : {
138 0 : return std::string(rmaMemDesc.memDesc, TRANSPORT_EMD_ESC_SIZE);
139 : }
140 :
141 : HcclResult DoExchangeMemDesc(const RmaMemDescs& localMemDescs, RmaMemDescs& remoteMemDescs, u32& actualNumOfRemote);
142 : HcclResult SendLocalMemDesc(const RmaMemDescs& localMemDescs);
143 : HcclResult ReceiveRemoteMemDesc(RmaMemDescs& remoteMemDescs, u32& actualNumOfRemote);
144 :
145 : const std::unique_ptr<NotifyPool>& notifyPool_;
146 : HcclNetDevCtx netDevCtx_{nullptr};
147 : HcclDispatcher dispatcher_{nullptr};
148 :
149 : u32 localRankId_{0};
150 : u32 remoteRankId_{0};
151 : std::shared_ptr<HcclSocket> socket_{nullptr};
152 :
153 : std::shared_ptr<HcclSocket> dataSocket_{nullptr};
154 :
155 : bool aicpuUnfoldMode_{false};
156 : };
157 : } // namespace hccl
158 : #endif
|
