Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #ifndef HCCLV2_RMA_CONN_LITE_H
12 : #define HCCLV2_RMA_CONN_LITE_H
13 : #include <memory>
14 : #include "rma_buf_slice_lite.h"
15 : #include "rmt_rma_buf_slice_lite.h"
16 : #include "log.h"
17 : #include "reduce_in.h"
18 : #include "data_type.h"
19 : #include "reduce_op.h"
20 : #include "ip_address.h"
21 : #include "ub_jetty_lite.h"
22 : #include "kernel_param_lite.h"
23 : #include "stream_lite.h"
24 : namespace Hccl {
25 : MAKE_ENUM(RmaConnLiteType, P2P, RDMA, UB, CCU) // 需要和RmaConnType一一对应
26 :
27 : struct SqeConfigLite {
28 39 : SqeConfigLite() : placeOdr(1), compOrder(1), fence(1) {}
29 : bool cqeEn{true};
30 : bool userConfig{false};
31 : u8 placeOdr : 2;
32 : u8 compOrder : 1;
33 : u8 fence : 1;
34 : };
35 :
36 : struct ConnLiteOperationOut {
37 : u16 pi{0};
38 : u8* data{};
39 : u8 dataSize{0};
40 : };
41 :
42 : class RmaConnLite {
43 : public:
44 53 : RmaConnLite() = default;
45 :
46 : RmaConnLite(const UbJettyLiteId& id, const UbJettyLiteAttr& attr, const Eid& rmtEid);
47 :
48 : explicit RmaConnLite(const u64 qpVa);
49 :
50 175 : virtual ~RmaConnLite() = default;
51 :
52 : static std::unique_ptr<RmaConnLite> Create(std::vector<char>& uniqueId);
53 :
54 : UbJettyLiteId GetUbJettyLiteId() const;
55 :
56 : UbJettyLiteAttr GetUbJettyLiteAttr() const;
57 :
58 : Eid GetRmtEid() const;
59 : Eid GetLocEid() const;
60 :
61 : u32 GetQpVa() const;
62 : uint64_t GetJettyHandle() const;
63 : uint32_t GetJettyId() const;
64 :
65 : virtual std::string Describe();
66 :
67 2 : virtual void Read(
68 : const RmaBufSliceLite& loc, const RmtRmaBufSliceLite& rmt, const SqeConfigLite& cfg,
69 : [[maybe_unused]] const StreamLite& stream, ConnLiteOperationOut& out)
70 : {
71 6 : HCCL_INFO(
72 : "RmaConnLite read start. loc.addr = %llx, rmt.addr = %llx, cfg.cqeEn = %u, out.pi = %u", loc.GetAddr(),
73 : rmt.GetAddr(), cfg.cqeEn, out.pi);
74 2 : }
75 :
76 1 : virtual void ReadReduce(
77 : ReduceIn reduceIn, const RmaBufSliceLite& loc, const RmtRmaBufSliceLite& rmt,
78 : [[maybe_unused]] const StreamLite& stream, const SqeConfigLite& cfg, ConnLiteOperationOut& out)
79 : {
80 3 : HCCL_INFO(
81 : "RmaConnLite ReadReduce start. dataType = %u, reduceOp %u, loc.addr = %llx, "
82 : "rmt.addr = %llx, cfg.cqeEn = %u, out.pi = %u",
83 : reduceIn.dataType, reduceIn.reduceOp, loc.GetAddr(), rmt.GetAddr(), cfg.cqeEn, out.pi);
84 1 : }
85 :
86 1 : virtual void Write(
87 : const RmaBufSliceLite& loc, const RmtRmaBufSliceLite& rmt, const SqeConfigLite& cfg,
88 : [[maybe_unused]] const StreamLite& stream, ConnLiteOperationOut& out)
89 : {
90 3 : HCCL_INFO(
91 : "RmaConnLite write start. loc.addr = %llx, rmt.addr = %llx, cfg.cqeEn = %u, out.pi = %u", loc.GetAddr(),
92 : rmt.GetAddr(), cfg.cqeEn, out.pi);
93 1 : }
94 :
95 1 : virtual void InlineWrite(
96 : const u8* data, u16 size, const RmtRmaBufSliceLite& rmt, const SqeConfigLite& cfg,
97 : [[maybe_unused]] const StreamLite& stream, ConnLiteOperationOut& out)
98 : {
99 3 : HCCL_INFO(
100 : "RmaConnLite InlineWrite start. data = %p, size = %hu, rmt.addr = %llx, cfg.cqeEn = %u, out.pi = %u", data,
101 : size, rmt.GetAddr(), cfg.cqeEn, out.pi);
102 1 : }
103 :
104 1 : virtual void WriteReduce(
105 : DataType dataType, ReduceOp reduceOp, const RmaBufSliceLite& loc, [[maybe_unused]] const StreamLite& stream,
106 : const RmtRmaBufSliceLite& rmt, const SqeConfigLite& cfg, ConnLiteOperationOut& out)
107 : {
108 3 : HCCL_INFO(
109 : "RmaConnLite WriteReduce start. dataType = %u, reduceOp %u, loc.addr = %llx, "
110 : "rmt.addr = %llx, cfg.cqeEn = %u, out.pi = %u",
111 : dataType, reduceOp, loc.GetAddr(), rmt.GetAddr(), cfg.cqeEn, out.pi);
112 1 : }
113 :
114 1 : virtual void WriteWithNotify(
115 : const RmaBufSliceLite& loc, const RmtRmaBufSliceLite& rmt, const SqeConfigLite& cfg, ConnLiteOperationOut& out,
116 : const RmtRmaBufSliceLite& notify, [[maybe_unused]] const StreamLite& stream, u64 notifyData)
117 : {
118 3 : HCCL_INFO(
119 : "RmaConnLite WriteWithNotify start. loc.addr = %llx, rmt.addr = %llx, cfg.cqeEn = %u, "
120 : "out.pi = %u, notify.addr = %llx, notifyData = %u",
121 : loc.GetAddr(), rmt.GetAddr(), cfg.cqeEn, out.pi, notify.GetAddr(), notifyData);
122 1 : }
123 :
124 0 : virtual void WriteReduceWithNotify(
125 : DataType dataType, ReduceOp reduceOp, const RmaBufSliceLite& loc, const RmtRmaBufSliceLite& rmt,
126 : const SqeConfigLite& cfg, [[maybe_unused]] const StreamLite& stream, ConnLiteOperationOut& out,
127 : const RmtRmaBufSliceLite& notify, u64 notifyData)
128 : {
129 0 : HCCL_INFO(
130 : "RmaConnLite WriteReduceWithNotify start. dataType = %d, , reduceOp %d, loc.addr = %llx, "
131 : "rmt.addr = %llx, cfg.cqeEn = %u, out.pi = %u, notify.addr = %llx, notifyData = %llu",
132 : dataType, reduceOp, loc.GetAddr(), rmt.GetAddr(), cfg.cqeEn, out.pi, notify.GetAddr(), notifyData);
133 0 : }
134 :
135 1 : virtual void BatchOneSidedRead(
136 : const vector<RmaBufSliceLite>& loc, const vector<RmtRmaBufSliceLite>& rmt, const SqeConfigLite& cfg,
137 : [[maybe_unused]] const StreamLite& stream, ConnLiteOperationOut& out)
138 : {
139 3 : HCCL_INFO(
140 : "RmaConnLite BatchOneSidedRead start. loc.size = %llu, rmt.size = %llu, cfg.cqeEn = %u, out.pi = %u",
141 : loc.size(), rmt.size(), cfg.cqeEn, out.pi);
142 1 : }
143 :
144 1 : virtual void BatchOneSidedWrite(
145 : const vector<RmaBufSliceLite>& loc, const vector<RmtRmaBufSliceLite>& rmt, const SqeConfigLite& cfg,
146 : [[maybe_unused]] const StreamLite& stream, ConnLiteOperationOut& out)
147 : {
148 3 : HCCL_INFO(
149 : "RmaConnLite BatchOneSidedWrite start. loc.size = %llu, rmt.size = %llu, cfg.cqeEn = %u, out.pi = %u",
150 : loc.size(), rmt.size(), cfg.cqeEn, out.pi);
151 1 : }
152 :
153 : protected:
154 : u32 qpVa_{0};
155 :
156 : u32 dieId_{0};
157 : u32 funcId_{0};
158 : u32 jettyId_{0};
159 : u64 dbAddr_{0};
160 : u64 sqVa_{0};
161 : u32 sqDepth_{0};
162 : bool dwqeCacheLocked_{false}; // direct WQE cache Lock
163 : u32 jfcPollMode_{0}; // 0代表STARS POLL， 1代表软件Poll
164 : u32 tpn_{0};
165 :
166 : Eid rmtEid_;
167 : Eid locEid_;
168 : uint64_t jettyHandle_{0};
169 : };
170 :
171 : } // namespace Hccl
172 : #endif
|
