Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 : #ifndef HCCLV2_RMA_CONN_LITE_H
11 : #define HCCLV2_RMA_CONN_LITE_H
12 : #include <memory>
13 : #include "rma_buf_slice_lite.h"
14 : #include "rmt_rma_buf_slice_lite.h"
15 : #include "log.h"
16 : #include "reduce_in.h"
17 : #include "data_type.h"
18 : #include "reduce_op.h"
19 : #include "ip_address.h"
20 : #include "ub_jetty_lite.h"
21 : #include "kernel_param_lite.h"
22 : #include "stream_lite.h"
23 : namespace Hccl {
24 : MAKE_ENUM(RmaConnLiteType, P2P, RDMA, UB, CCU) // 需要和RmaConnType一一对应
25 :
26 : struct SqeConfigLite {
27 39 : SqeConfigLite() : placeOdr(1), compOrder(1), fence(1) {}
28 : bool cqeEn{true};
29 : bool userConfig{false};
30 : u8 placeOdr : 2;
31 : u8 compOrder : 1;
32 : u8 fence : 1;
33 : };
34 :
35 : struct ConnLiteOperationOut {
36 : u16 pi{0};
37 : u8* data{};
38 : u8 dataSize{0};
39 : };
40 :
41 : class RmaConnLite {
42 : public:
43 53 : RmaConnLite() = default;
44 :
45 : RmaConnLite(const UbJettyLiteId& id, const UbJettyLiteAttr& attr, const Eid& rmtEid);
46 :
47 : explicit RmaConnLite(const u64 qpVa);
48 :
49 175 : virtual ~RmaConnLite() = default;
50 :
51 : static std::unique_ptr<RmaConnLite> Create(std::vector<char>& uniqueId);
52 :
53 : UbJettyLiteId GetUbJettyLiteId() const;
54 :
55 : UbJettyLiteAttr GetUbJettyLiteAttr() const;
56 :
57 : Eid GetRmtEid() const;
58 : Eid GetLocEid() const;
59 :
60 : u32 GetQpVa() const;
61 : uint64_t GetJettyHandle() const;
62 : uint32_t GetJettyId() const;
63 :
64 : virtual std::string Describe();
65 :
66 2 : virtual void Read(
67 : const RmaBufSliceLite& loc, const RmtRmaBufSliceLite& rmt, const SqeConfigLite& cfg, const StreamLite& stream,
68 : ConnLiteOperationOut& out)
69 : {
70 6 : HCCL_INFO(
71 : "RmaConnLite read start. loc.addr = %llx, rmt.addr = %llx, cfg.cqeEn = %u, out.pi = %u", loc.GetAddr(),
72 : rmt.GetAddr(), cfg.cqeEn, out.pi);
73 2 : }
74 :
75 1 : virtual void ReadReduce(
76 : ReduceIn reduceIn, const RmaBufSliceLite& loc, const RmtRmaBufSliceLite& rmt, const StreamLite& stream,
77 : const SqeConfigLite& cfg, ConnLiteOperationOut& out)
78 : {
79 3 : HCCL_INFO(
80 : "RmaConnLite ReadReduce start. dataType = %u, reduceOp %u, loc.addr = %llx, "
81 : "rmt.addr = %llx, cfg.cqeEn = %u, out.pi = %u",
82 : reduceIn.dataType, reduceIn.reduceOp, loc.GetAddr(), rmt.GetAddr(), cfg.cqeEn, out.pi);
83 1 : }
84 :
85 1 : virtual void Write(
86 : const RmaBufSliceLite& loc, const RmtRmaBufSliceLite& rmt, const SqeConfigLite& cfg, const StreamLite& stream,
87 : ConnLiteOperationOut& out)
88 : {
89 3 : HCCL_INFO(
90 : "RmaConnLite write start. loc.addr = %llx, rmt.addr = %llx, cfg.cqeEn = %u, out.pi = %u", loc.GetAddr(),
91 : rmt.GetAddr(), cfg.cqeEn, out.pi);
92 1 : }
93 :
94 1 : virtual void InlineWrite(
95 : const u8* data, u16 size, const RmtRmaBufSliceLite& rmt, const SqeConfigLite& cfg, const StreamLite& stream,
96 : ConnLiteOperationOut& out)
97 : {
98 3 : HCCL_INFO(
99 : "RmaConnLite InlineWrite start. data = %p, size = %hu, rmt.addr = %llx, cfg.cqeEn = %u, out.pi = %u", data,
100 : size, rmt.GetAddr(), cfg.cqeEn, out.pi);
101 1 : }
102 :
103 1 : virtual void WriteReduce(
104 : DataType dataType, ReduceOp reduceOp, const RmaBufSliceLite& loc, const StreamLite& stream,
105 : const RmtRmaBufSliceLite& rmt, const SqeConfigLite& cfg, ConnLiteOperationOut& out)
106 : {
107 3 : HCCL_INFO(
108 : "RmaConnLite WriteReduce start. dataType = %u, reduceOp %u, loc.addr = %llx, "
109 : "rmt.addr = %llx, cfg.cqeEn = %u, out.pi = %u",
110 : dataType, reduceOp, loc.GetAddr(), rmt.GetAddr(), cfg.cqeEn, out.pi);
111 1 : }
112 :
113 1 : virtual void WriteWithNotify(
114 : const RmaBufSliceLite& loc, const RmtRmaBufSliceLite& rmt, const SqeConfigLite& cfg, ConnLiteOperationOut& out,
115 : const RmtRmaBufSliceLite& notify, const StreamLite& stream, u64 notifyData)
116 : {
117 3 : HCCL_INFO(
118 : "RmaConnLite WriteWithNotify start. loc.addr = %llx, rmt.addr = %llx, cfg.cqeEn = %u, "
119 : "out.pi = %u, notify.addr = %llx, notifyData = %u",
120 : loc.GetAddr(), rmt.GetAddr(), cfg.cqeEn, out.pi, notify.GetAddr(), notifyData);
121 1 : }
122 :
123 0 : virtual void WriteReduceWithNotify(
124 : DataType dataType, ReduceOp reduceOp, const RmaBufSliceLite& loc, const RmtRmaBufSliceLite& rmt,
125 : const SqeConfigLite& cfg, const StreamLite& stream, ConnLiteOperationOut& out, const RmtRmaBufSliceLite& notify,
126 : u64 notifyData)
127 : {
128 0 : HCCL_INFO(
129 : "RmaConnLite WriteReduceWithNotify start. dataType = %d, , reduceOp %d, loc.addr = %llx, "
130 : "rmt.addr = %llx, cfg.cqeEn = %u, out.pi = %u, notify.addr = %llx, notifyData = %llu",
131 : dataType, reduceOp, loc.GetAddr(), rmt.GetAddr(), cfg.cqeEn, out.pi, notify.GetAddr(), notifyData);
132 0 : }
133 :
134 1 : virtual void BatchOneSidedRead(
135 : const vector<RmaBufSliceLite>& loc, const vector<RmtRmaBufSliceLite>& rmt, const SqeConfigLite& cfg,
136 : const StreamLite& stream, ConnLiteOperationOut& out)
137 : {
138 3 : HCCL_INFO(
139 : "RmaConnLite BatchOneSidedRead start. loc.size = %llu, rmt.size = %llu, cfg.cqeEn = %u, out.pi = %u",
140 : loc.size(), rmt.size(), cfg.cqeEn, out.pi);
141 1 : }
142 :
143 1 : virtual void BatchOneSidedWrite(
144 : const vector<RmaBufSliceLite>& loc, const vector<RmtRmaBufSliceLite>& rmt, const SqeConfigLite& cfg,
145 : const StreamLite& stream, ConnLiteOperationOut& out)
146 : {
147 3 : HCCL_INFO(
148 : "RmaConnLite BatchOneSidedWrite start. loc.size = %llu, rmt.size = %llu, cfg.cqeEn = %u, out.pi = %u",
149 : loc.size(), rmt.size(), cfg.cqeEn, out.pi);
150 1 : }
151 :
152 : protected:
153 : u32 qpVa_{0};
154 :
155 : u32 dieId_{0};
156 : u32 funcId_{0};
157 : u32 jettyId_{0};
158 : u64 dbAddr_{0};
159 : u64 sqVa_{0};
160 : u32 sqDepth_{0};
161 : bool dwqeCacheLocked_{false}; // direct WQE cache Lock
162 : u32 jfcPollMode_{0}; // 0代表STARS POLL， 1代表软件Poll
163 : u32 tpn_{0};
164 :
165 : Eid rmtEid_;
166 : Eid locEid_;
167 : uint64_t jettyHandle_{0};
168 : };
169 :
170 : } // namespace Hccl
171 : #endif
|
