Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #ifndef HCCL_NSLBDP_H
12 : #define HCCL_NSLBDP_H
13 :
14 : #include <vector>
15 : #include <memory>
16 : #include <map>
17 : #include <mutex>
18 :
19 : #include "hccl/base.h"
20 : #include "hccl_common.h"
21 : #include "hccl_comm_pub.h"
22 : #include "topoinfo_struct.h"
23 : #include "sal_pub.h"
24 : #include "comm.h"
25 : #include "coll_alg_param.h"
26 : #include "hccl_nslbdp_pub.h"
27 :
28 : namespace hccl {
29 :
30 : constexpr unsigned int NSLBDP_ILLEGAL_TLVBUFFERSIZE = 0;
31 : constexpr unsigned int NSLBDP_ILLEGAL_MSGLENGTH = 0;
32 : constexpr unsigned int NSLBDP_TYPE_TBL_COMM_INFO = 1001;
33 : constexpr unsigned int NSLBDP_TYPE_TBL_OPER = 1002;
34 : constexpr unsigned int NSLBDP_TYPE_TBL_ADJ = 1003;
35 : constexpr unsigned int NSLBDP_TYPE_TBL_RANK = 1004;
36 : constexpr unsigned int NSLBDP_TYPE_TBL_RANK_DIST = 1005;
37 : constexpr unsigned int NSLBDP_TYPE_TBL_ROOT_RANK = 1006;
38 : constexpr unsigned int NSLBDP_TYPE_INIT_NETCO = 9001;
39 : constexpr unsigned int NSLBDP_TYPE_DEINIT_NETCO = 9002;
40 :
41 : constexpr u32 NSLBDP_RANKTOTALNUM_BLOCK_FIR = 1024;
42 : constexpr u32 NSLBDP_RANKTOTALNUM_BLOCK_SEC = 2 * 1024;
43 : constexpr u32 NSLBDP_RANKTOTALNUM_BLOCK_THR = 3 * 1024;
44 : constexpr u32 NSLBDP_RANKTOTALNUM_BLOCK_FOU = 4 * 1024;
45 :
46 : constexpr u32 NSLBDP_HIGH_8BIT = 8;
47 :
48 : constexpr u16 NSLBDP_COMMINTERVAL_FLAG = 128;
49 : constexpr u16 NSLBDP_COMMINTERVAL_FLAGSEC = 256;
50 : constexpr u16 NSLBDP_COMMINTERVAL_FLAGTHR = 512;
51 : constexpr u16 NSLBDP_COMMINTERVAL_FLAGFOU = 1024;
52 : constexpr u16 NSLBDP_COMMINTERVAL_FLAGFIV = 2048;
53 : constexpr u16 NSLBDP_COMMINTERVAL_FLAGSIX = 4096;
54 :
55 : constexpr u32 NSLBDP_PKTNUM_FIR = 1;
56 : constexpr u32 NSLBDP_PKTNUM_SEC = 2;
57 : constexpr u32 NSLBDP_PKTNUM_THR = 3;
58 : constexpr u32 NSLBDP_PKTNUM_FOU = 4;
59 :
60 : constexpr u32 NSLBDP_TASKID_OFFSET = 32;
61 :
62 : constexpr u32 NSLBDP_INITIME_MILLISENDS = 1000;
63 :
64 : constexpr u32 NSLBDP_PAIRWISE = 5;
65 :
66 : constexpr u32 NSLBDP_BEGINFOURBIT = 2;
67 :
68 : constexpr u32 NSLBDP_SPLIT_SIZE = 3;
69 :
70 : constexpr u16 NSLBDP_PRIVATE_PORT = 3;
71 : constexpr u16 NSLBDP_RANGE_ID = 14;
72 : constexpr u16 NSLBDP_COMMON_RANGE = 11;
73 : constexpr u16 NSLBDP_ALGO_RANGE = 4;
74 :
75 : constexpr u32 NSLBDP_TRAFFICCONUT = 60;
76 :
77 : enum class NslbDpAlgType {
78 : NSLBDP_WHOLE_RING = 0, // 单层拓扑, 所有level均为Whole ring时，组成一个大环
79 : NSLBDP_HD, // HDR
80 : NSLBDP_RING, // Ring
81 : NSLBDP_PIPELINE, // Pipeline
82 : NSLBDP_STAR,
83 : NSLBDP_NHR, // NHR
84 : NSLBDP_NHR_V1, // NHR_V1
85 : NSLBDP_NB, // NB
86 : NSLBDP_AHC, // AHC
87 : NSLBDP_AHC_BROKE, // AHC_BROKE
88 : NSLBDP_RESERVED
89 : };
90 :
91 : constexpr u8 NSLBDP_CMD_INVALID = 0;
92 : constexpr u8 NSLBDP_CMD_BROADCAST = 1;
93 : constexpr u8 NSLBDP_CMD_ALLREDUCE = 2;
94 : constexpr u8 NSLBDP_CMD_REDUCE = 3;
95 : constexpr u8 NSLBDP_CMD_SEND = 4;
96 : constexpr u8 NSLBDP_CMD_RECEIVE = 5;
97 : constexpr u8 NSLBDP_CMD_ALLGATHER = 6;
98 : constexpr u8 NSLBDP_CMD_REDUCE_SCATTER = 7;
99 : constexpr u8 NSLBDP_CMD_ALLTOALLV = 8;
100 : constexpr u8 NSLBDP_CMD_ALLTOALLVC = 9;
101 : constexpr u8 NSLBDP_CMD_ALLTOALL = 10;
102 : constexpr u8 NSLBDP_CMD_GATHER = 11;
103 : constexpr u8 NSLBDP_CMD_SCATTER = 12;
104 : constexpr u8 NSLBDP_CMD_BATCH_SEND_RECV = 13;
105 : constexpr u8 NSLBDP_CMD_BATCH_PUT = 14;
106 : constexpr u8 NSLBDP_CMD_BATCH_GET = 15;
107 : constexpr u8 NSLBDP_CMD_ALL = 16;
108 :
109 : constexpr u8 NSLB_ALGO_TYPE_DEFAULT = 0;
110 : constexpr u8 NSLB_ALGO_TYPE_RING = 1;
111 : constexpr u8 NSLB_ALGO_TYPE_PIPELINE = 2;
112 : constexpr u8 NSLB_ALGO_TYPE_FULLMESH = 3;
113 : constexpr u8 NSLB_ALGO_TYPE_HDR = 4;
114 : constexpr u8 NSLB_ALGO_TYPE_PAIRWISE = 5;
115 : constexpr u8 NSLB_ALGO_TYPE_NHR = 6;
116 : constexpr u8 NSLB_ALGO_TYPE_NHR_V1 = 7;
117 : constexpr u8 NSLB_ALGO_TYPE_NB = 8;
118 : constexpr u8 NSLB_ALGO_TYPE_NULL = 9;
119 : constexpr u8 NSLB_ALGO_TYPE_NA = 10;
120 : constexpr u8 NSLB_ALGO_TYPE_FAST_DOUBLE_RING = 11;
121 : constexpr u8 NSLB_ALGO_TYPE_AHC = 12;
122 :
123 : constexpr u16 NSLB_COMM_INTERVAL_FLAG_BEGIN = 0;
124 : constexpr u16 NSLB_COMM_INTERVAL_FLAG_FIR = 1;
125 : constexpr u16 NSLB_COMM_INTERVAL_FLAG_SEC = 2;
126 : constexpr u16 NSLB_COMM_INTERVAL_FLAG_THR = 3;
127 : constexpr u16 NSLB_COMM_INTERVAL_FLAG_FOR = 4;
128 : constexpr u16 NSLB_COMM_INTERVAL_FLAG_FIV = 5;
129 : constexpr u16 NSLB_COMM_INTERVAL_FLAG_SIX = 6;
130 : constexpr u16 NSLB_COMM_INTERVAL_FLAG_SEV = 7;
131 : constexpr u16 NSLB_COMM_INTERVAL_FLAG_MAX = 8;
132 :
133 : constexpr unsigned int MODULE_TYPE_NSLB = 0;
134 : constexpr unsigned int MODULE_TYPE_MAX = 1;
135 :
136 : constexpr u32 NSLBDP_UNDERDCORES_COUNT = 3;
137 :
138 : using nslb_msg = struct nslb_msg {
139 : unsigned int type;
140 : unsigned int length;
141 : std::string data;
142 :
143 12 : nslb_msg() : type(INVALID_UINT), length(0), data("") {}
144 : };
145 :
146 : class hcclNslbDp {
147 : public:
148 : static hcclNslbDp& GetInstance();
149 : NslbDpCommConfigVal GetNslbDpCommConfig();
150 : HcclResult HcclSetGlobalRankTotalNum(u32 nRanks);
151 :
152 : bool GetInitNetCoFlag();
153 : HcclResult ClearInitNetCoFlag();
154 : void InitCmmDesc(std::string& identifier_nslb);
155 : std::string GetCmmDesc();
156 : void SetDeviceType();
157 : void SetGlobalCommTaskId(u64 taskId);
158 : void SetGlobalCommNodeId(u32 nodeId);
159 : void SetGlobalCommLocalRankNum(u32 localRankNum);
160 : void SetGlobalCommRankTotalNum(u32 rankTotalNum);
161 : bool GetDeviceType();
162 : u64 GetGlobalCommTaskId();
163 : u32 GetGlobalCommNodeId();
164 : u32 GetTlvInitBufferSize();
165 : u8 GetGlobalCommLocalRankNum();
166 : u8 GetNslbOpType(HcclCMDType opType);
167 : u8 GetNslbLevel1AlgType(AlgTypeLevel1 algValue);
168 : u8 GetNslbLevel2AlgType(AlgTypeLevel2 algValue);
169 : u32 GetGlobalCommRankTotalNum();
170 : u32 Getl4SPortId();
171 : u64 GetNslbDpFirstFourBit(u8 opType, u8 algType);
172 : bool CheckAlgoConsistency(HcclCMDType opType, std::string& algName);
173 : void SplitString(const std::string& identifier, std::vector<std::string>& splitInfo, const std::string& frag);
174 : void SetGlobalDisRankTable(const HcclBasicRankInfo& rankTable);
175 : HcclResult SetCommInfo_NoRankTable(const RankTable_t rankTable, std::string identifier);
176 : HcclResult SetCommInfo_RankTableExit(RankTable_t rankTable);
177 : HcclResult SetGlobalRank_RankTableExit(const RankTable_t rankTable);
178 : HcclResult GenerateOpAndAdjTable(
179 : HcclCMDType opType, u32 rootRank, u32 srcLocalRankId, u8 algType, std::string identifier, u64 count,
180 : u32 rankSize);
181 : HcclResult GetAlgAdjacencyTable(
182 : HcclCMDType opType, u32 srcLocalRankId, u32 rootRank, u8 algType, std::string identifier, AdjInfo nslbAdjInfo);
183 : HcclResult GetNslbDpl4SPortId(u32 rankSize, u8 algType, u16* l4SPortId);
184 : HcclResult SendCommRankTable(uint32_t rank, NslbDpCommConfigVal globalCommInfo);
185 : bool CheckMultiMachine(const RankTable_t rankTable);
186 : bool CheckSupportOptype(HcclCMDType opType);
187 : bool CheckCommDescExit(NslbDpOperatorInfo& OperatorInfo);
188 : bool CheckSameOperatorVal(size_t operSize, NslbDpOperatorInfo& OperatorInfo, u32 rootRank);
189 : void SetGlobalCommRankTable_RootInfo(
190 : const RankTable_t& rankTable, const HcclBasicRankInfo& localRankInfo, const std::vector<RankInfo>& rankLists,
191 : const std::string& identifier, u32 nRanks, u32 rank);
192 : void GetGlobalRankTable(const RankTable_t* rankTable, u32 nRanks, HcclUs startut);
193 : void fullCommonGlobalRankInfo(NslbDpGlobalRankInfo tab_f, NslbDpGlobalRankVal& cominfo);
194 : void fullCommConfigInfo(NslbDpCommConfigInfo& tab_f, NslbDpCommConfigVal cominfo, u32 packetNum);
195 : void fullcommDescInitTime(std::string identifier, NslbDpOperatorInfo& OperatorInfo);
196 : HcclResult SetNslbDpRootRank(HcclCMDType opType, u32 rootRank, std::string identifier, u8 algType);
197 : HcclResult SendRankTable(NslbDpCommConfigInfo tab_f);
198 : std::vector<uint8_t> serializeTLV_TableFir(NslbDpCommConfigInfo cominfo);
199 : HcclResult SendTableProc(u32 rank, u32 packetNum, NslbDpCommConfigVal cominfo);
200 : HcclResult SendTableFir(uint32_t rank);
201 : HcclResult SetH2DTlvInitInfo(u32 buffer_size, void* tlv_handle);
202 : u32 ipToUint32(const std::string& ipAddress);
203 : HcclResult SendOpAndAdjTable();
204 : HcclResult SendRankTableOpAndAdj(NslbDpOperatorInfo& tab_f);
205 : std::vector<uint8_t> serializeTLV_TableOpAndAdj(NslbDpOperatorInfo& info);
206 : HcclResult SendAlgorithmInfoTable();
207 : HcclResult SendRankTableAlgorithmInfo(NslbDpAlgorithmTlv& tab_f);
208 : std::vector<uint8_t> serializeTLV_TableAlgorithmInfo(NslbDpAlgorithmTlv& info);
209 : HcclResult SendGlobalRankTable(uint32_t rank);
210 : HcclResult SendTableGlobalRankProc(uint32_t rank, uint32_t packetNum, NslbDpGlobalRankVal& cominfo);
211 : std::vector<uint8_t> serializeTLV_TableGlobalRankInfo(NslbDpGlobalRankInfo& info);
212 : HcclResult SendRankTableGlobalRank(NslbDpGlobalRankInfo& tab_f);
213 : HcclResult SendGlobalDisRankTable();
214 : HcclResult SendRankTableGlobalDisRankVal(NslbDpGlobalDisRankVal& tab_f);
215 : std::vector<uint8_t> serializeTLV_TableGlobalDisRankVal(NslbDpGlobalDisRankVal& info);
216 : HcclResult SendRootRankTable();
217 : HcclResult SendRankTableRootRank(NslbDpRootRank& tab_f);
218 : std::vector<uint8_t> serializeTLV_TableRootRank(NslbDpRootRank& config);
219 : HcclResult InitNetCo();
220 : void DeinitNetCo();
221 : bool check910_93_ = false;
222 : std::atomic<bool> nslbdpIsInitNetCo_ = {false};
223 : void* nslbdp_handle_;
224 : unsigned int nslbdp_buffsize_;
225 : std::string nslbdp_identifier_;
226 : u32 hcclNslbDpL4SPortId_;
227 : // 上层调用数据存储关键字唯一标识的taskid等信息
228 : NslbDpGlobalCommInfo hcclNslbDpGlobalCommInfo_;
229 : // 分表1-基础数据. 承载通信与信息
230 : std::vector<NslbDpCommConfigVal> hcclNslbDpCommConfig_;
231 : // 分表2-基础数据，承载执行的算子算法信息
232 : std::vector<NslbDpOperatorInfo> hcclNslbDpOperatorVal_;
233 : // 分表3-基础数据，承载执行的算子算法的邻接信息
234 : std::vector<NslbDpAlgorithmInfo> hcclNslbDpAlgorithmInfo_;
235 : // 分表4-基础数据，在非ranktble场景下创建通信与场景承载全局通信域信息
236 : NslbDpGlobalRankVal hcclNslbDpGlobalRankVal_;
237 : // 分表5-基础数据，在非ranktble场景下创建通信与场景域场景下的分布式rank表
238 : NslbDpGlobalDisRankVal hcclNslbDpGlobalDisRankVal_;
239 : // 分表6-基础数据，非对称算子 statter， reduce，bcast 算子场景下会有rootrabnk表
240 : NslbDpRootRank hcclNslbDpRootRankVal_;
241 :
242 : private:
243 : hcclNslbDp();
244 : ~hcclNslbDp();
245 : bool CheckAhcCommInfo(NslbDpCommConfigVal comInfo);
246 : bool CheckAhcSupport(u8 algType, std::string identifier);
247 : bool InitAlgInfoCommDesc(NslbDpAlgorithmInfo& algorithmInfo, const std::string& identifier);
248 : bool FillAlgInfoCommMd5(NslbDpAlgorithmInfo& algorithmInfo);
249 : void FillAlgInfoBaseFields(
250 : NslbDpAlgorithmInfo& algorithmInfo, HcclCMDType opType, u32 srcLocalRankId, u32 rootRank, u8 algType);
251 : bool IsAlgAdjacencyDuplicated(const NslbDpAlgorithmInfo& algorithmInfo);
252 : bool FillAlgInfoAdjInfo(NslbDpAlgorithmInfo& algorithmInfo, const AdjInfo& nslbAdjInfo, u32 srcLocalRankId);
253 : };
254 :
255 : } // namespace hccl
256 :
257 : #endif /* HCCL_NSLB_DP_PUB_H */
|
