Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include <atomic>
12 : #include <fcntl.h>
13 : #include <unistd.h>
14 : #include <hccl/hccl_types.h>
15 : #include "topoinfo_struct.h"
16 : #include "adapter_hccp_common.h"
17 : #include "hccl_nslb_md5.h"
18 : #include "../common/src/h2d_tlv/hccl_h2dtlv.h"
19 : #include "hccl_nslbdp.h"
20 :
21 : namespace hccl {
22 :
23 : const u32 COMM_SLICING_LENGTH = 1024; // group name max length
24 :
25 459 : hcclNslbDp& hcclNslbDp::GetInstance()
26 : {
27 459 : static hcclNslbDp pNslbDp;
28 462 : return pNslbDp;
29 : }
30 :
31 8 : hcclNslbDp::hcclNslbDp() {}
32 :
33 8 : hcclNslbDp::~hcclNslbDp() {}
34 :
35 0 : void hcclNslbDp::InitCmmDesc(std::string& identifier_nslb)
36 : {
37 0 : nslbdp_identifier_ = identifier_nslb;
38 0 : HCCL_INFO("[NSLB-DP] Init CmmDesc nslbdp_identifier_[%s] .", nslbdp_identifier_.c_str());
39 0 : return;
40 : }
41 :
42 0 : std::string hcclNslbDp::GetCmmDesc()
43 : {
44 0 : HCCL_INFO("[NSLB-DP] Get CmmDesc nslbdp_identifier_[%s] .", nslbdp_identifier_.c_str());
45 0 : return nslbdp_identifier_;
46 : }
47 :
48 0 : void hcclNslbDp::SetDeviceType()
49 : {
50 0 : check910_93_ = true;
51 0 : return;
52 : }
53 :
54 2 : bool hcclNslbDp::GetDeviceType()
55 : {
56 2 : bool is91093 = check910_93_;
57 2 : return is91093;
58 : }
59 :
60 0 : HcclResult hcclNslbDp::SetH2DTlvInitInfo(u32 buffer_size, void* tlv_handle)
61 : {
62 0 : HCCL_DEBUG("[NSLB-DP] try to set tlvinit info.");
63 0 : nslbdp_buffsize_ = buffer_size;
64 0 : nslbdp_handle_ = tlv_handle;
65 0 : nslbdpIsInitNetCo_ = true;
66 0 : HCCL_INFO("[NSLB-DP] set tlvinit info buffer_size:[%u].", buffer_size);
67 0 : return HCCL_SUCCESS;
68 : }
69 :
70 12 : bool hcclNslbDp::GetInitNetCoFlag() { return nslbdpIsInitNetCo_; }
71 :
72 0 : HcclResult hcclNslbDp::ClearInitNetCoFlag()
73 : {
74 0 : nslbdpIsInitNetCo_ = false;
75 0 : return HCCL_SUCCESS;
76 : }
77 :
78 0 : u32 hcclNslbDp::GetTlvInitBufferSize()
79 : {
80 0 : u32 nslbdpBuffsize = nslbdp_buffsize_;
81 0 : return nslbdpBuffsize;
82 : }
83 :
84 0 : HcclResult hcclNslbDp::HcclSetGlobalRankTotalNum(u32 nRanks)
85 : {
86 : // 填充表4
87 0 : hcclNslbDpGlobalRankVal_.rankTotalNum = nRanks;
88 0 : return HCCL_SUCCESS;
89 : }
90 :
91 92 : void hcclNslbDp::SetGlobalCommTaskId(u64 taskId)
92 : {
93 92 : hcclNslbDpGlobalCommInfo_.taskId = taskId;
94 92 : return;
95 : }
96 :
97 0 : void hcclNslbDp::SetGlobalCommNodeId(u32 nodeId)
98 : {
99 0 : hcclNslbDpGlobalCommInfo_.nodeId = nodeId;
100 0 : return;
101 : }
102 :
103 0 : void hcclNslbDp::SetGlobalCommLocalRankNum(u32 localRankNum)
104 : {
105 0 : hcclNslbDpGlobalCommInfo_.localRankNum = localRankNum;
106 0 : return;
107 : }
108 :
109 0 : void hcclNslbDp::SetGlobalCommRankTotalNum(u32 rankTotalNum)
110 : {
111 0 : hcclNslbDpGlobalCommInfo_.rankTotalNum = rankTotalNum;
112 0 : return;
113 : }
114 :
115 416 : u64 hcclNslbDp::GetGlobalCommTaskId() { return hcclNslbDpGlobalCommInfo_.taskId; }
116 :
117 0 : u32 hcclNslbDp::GetGlobalCommNodeId() { return hcclNslbDpGlobalCommInfo_.nodeId; }
118 :
119 0 : u8 hcclNslbDp::GetGlobalCommLocalRankNum() { return hcclNslbDpGlobalCommInfo_.localRankNum; }
120 :
121 0 : u32 hcclNslbDp::GetGlobalCommRankTotalNum() { return hcclNslbDpGlobalCommInfo_.rankTotalNum; }
122 :
123 : /* 获取 l4SPortId */
124 0 : u32 hcclNslbDp::Getl4SPortId()
125 : {
126 0 : u32 NslbDpL4SPortId = hcclNslbDpL4SPortId_;
127 0 : return NslbDpL4SPortId;
128 : }
129 :
130 : /* 切分identifier字段 */
131 4 : void hcclNslbDp::SplitString(
132 : const std::string& identifier, std::vector<std::string>& splitInfo, const std::string& frag) const
133 : {
134 4 : std::string::size_type pos2 = identifier.find(frag);
135 4 : std::string::size_type pos1 = 0;
136 8 : while (std::string::npos != pos2) {
137 4 : splitInfo.push_back(identifier.substr(pos1, pos2 - pos1));
138 4 : pos1 = pos2 + frag.size();
139 4 : pos2 = identifier.find(frag, pos1);
140 : }
141 4 : if (pos1 != identifier.length()) {
142 3 : splitInfo.push_back(identifier.substr(pos1));
143 : }
144 4 : return;
145 : }
146 :
147 : /* 将IP转换成U32值 */
148 25 : u32 hcclNslbDp::ipToUint32(const std::string& ipAddress)
149 : {
150 : struct sockaddr_in sa;
151 25 : inet_pton(AF_INET, ipAddress.c_str(), &(sa.sin_addr));
152 25 : return ntohl(sa.sin_addr.s_addr); // Convert to host byte order
153 : }
154 :
155 : /* 执行send 流程 */
156 3 : HcclResult hcclNslbDp::SendCommRankTable(uint32_t rank, NslbDpCommConfigVal globalCommInfo)
157 : {
158 3 : HCCL_DEBUG("[NSLB-DP] entry to send TBL_COMM_INFO");
159 3 : u32 rankTotalNum = globalCommInfo.rankTotalNum;
160 :
161 3 : HCCL_INFO("[NSLB-DP] TBL_COMM_INFO rankTotalNum:[%u].", rankTotalNum);
162 3 : if (rankTotalNum > NSLBDP_RANKTOTALNUM_BLOCK_FOU) {
163 1 : return HCCL_SUCCESS;
164 : }
165 2 : u32 packetNum = CalcPacketNum(rankTotalNum);
166 2 : SendTableProc(rank, packetNum, globalCommInfo);
167 :
168 2 : HCCL_DEBUG("[NSLB-DP] entry send TBL_COMM_INFO end");
169 2 : return HCCL_SUCCESS;
170 : }
171 :
172 0 : bool hcclNslbDp::CheckAhcCommInfo(NslbDpCommConfigVal comInfo)
173 : {
174 0 : std::unordered_map<uint32_t, int> podToIpCount;
175 0 : for (const auto& info : comInfo.rankInfo) {
176 0 : podToIpCount[info.podId]++;
177 : }
178 : // 获取超节点对应的ip数量
179 0 : std::vector<int> ipCounts;
180 0 : for (const auto& entry : podToIpCount) {
181 0 : ipCounts.push_back(entry.second);
182 : }
183 0 : if (ipCounts.empty()) {
184 0 : return true;
185 : }
186 : // 找到最小的ip数量
187 0 : uint32_t minCount = *std::min_element(ipCounts.begin(), ipCounts.end());
188 0 : for (uint32_t count : ipCounts) {
189 0 : if (count % minCount != 0) {
190 0 : return false;
191 : }
192 : }
193 0 : return true;
194 0 : }
195 :
196 7 : bool hcclNslbDp::CheckAhcSupport(u8 algType, std::string identifier)
197 : {
198 7 : if (algType != NSLB_ALGO_TYPE_AHC) {
199 7 : return true;
200 : }
201 : char commDesc[COMM_DESC_MAX_LENGTH];
202 0 : s32 sRet = memset_s(commDesc, COMM_DESC_MAX_LENGTH, 0, sizeof(commDesc));
203 0 : if (sRet != EOK) {
204 0 : HCCL_ERROR("memset_s commDesc fail");
205 0 : return true;
206 : }
207 :
208 0 : s32 ret = strncpy_s(commDesc, COMM_DESC_MAX_LENGTH, identifier.c_str(), identifier.size());
209 0 : if (ret != EOK) {
210 0 : HCCL_INFO("strncpy_s commDesc fail");
211 0 : return true;
212 : }
213 0 : commDesc[COMM_DESC_MAX_LENGTH - 1] = '\0';
214 0 : for (const auto& info : hcclNslbDpCommConfig_) {
215 0 : if (strcmp(commDesc, info.commDesc) == 0) {
216 0 : return CheckAhcCommInfo(info);
217 : }
218 : }
219 0 : return true;
220 : }
221 :
222 : /* 判断指定 commDesc + taskId 的表一是否已存在(避免重复填充, inittime 必然不同) */
223 7 : bool hcclNslbDp::IsCommDescDuplicated(const char* commDesc, u64 taskId) const
224 : {
225 10 : for (size_t i = 0; i < hcclNslbDpCommConfig_.size(); i++) {
226 5 : if (hcclNslbDpCommConfig_[i].taskId == taskId && strcmp(hcclNslbDpCommConfig_[i].commDesc, commDesc) == 0) {
227 2 : return true;
228 : }
229 : }
230 5 : return false;
231 : }
232 :
233 : /* 从 rankTable.rankList 构建并填充 rankInfo 列表到 globalCommInfo */
234 4 : void hcclNslbDp::FillRankInfoFromRankTable(NslbDpCommConfigVal& globalCommInfo, const hccl::RankTable_t& rankTable)
235 : {
236 12 : for (u32 rankIndex = 0; rankIndex < rankTable.rankList.size(); rankIndex++) {
237 : NslbDpRankInfo dpRankInfo;
238 8 : HcclIpAddress tmpIp = rankTable.rankList[rankIndex].deviceInfo.deviceIp[0];
239 8 : std::string deviceIp = tmpIp.GetReadableAddress();
240 8 : dpRankInfo.deviceIp = ipToUint32(deviceIp);
241 8 : HCCL_INFO("[NSLB-DP] FillRankInfoFromRankTable deviceIp:[%u] success.", dpRankInfo.deviceIp);
242 8 : if (rankTable.rankList[rankIndex].superPodIdx == INVALID_UINT) {
243 1 : dpRankInfo.podId = 0;
244 : } else {
245 7 : dpRankInfo.podId = rankTable.rankList[rankIndex].superPodIdx;
246 : }
247 8 : dpRankInfo.rev = 0;
248 8 : globalCommInfo.rankInfo.push_back(dpRankInfo);
249 8 : }
250 4 : }
251 :
252 : /* 读配置文件的场景与正常创建通信域场景下填充通信域信息表（表一） */
253 0 : void hcclNslbDp::SetGlobalCommRankTable_RootInfo(
254 : const RankTable_t& rankTable, const HcclBasicRankInfo& localRankInfo, const std::vector<RankInfo>& rankLists,
255 : const std::string& identifier, u32 nRanks, u32 rank)
256 : {
257 0 : HCCL_INFO(
258 : "[NSLB-DP] Try to collect NSLBDP_TYPE_TBL_COMM_INFO commDesc[%s] - size = [%u].", identifier.c_str(),
259 : rankTable.rankList.size());
260 0 : u64 checkTaskId = GetGlobalCommTaskId();
261 0 : if (checkTaskId == 0 || nRanks == 1) {
262 0 : return;
263 : }
264 0 : if (nRanks > NSLBDP_RANKTOTALNUM_BLOCK_FOU) {
265 0 : HCCL_INFO("[NSLB-DP] nRanks[%u] exceeds limit[%u], skip TBL_COMM_INFO.", nRanks, NSLBDP_RANKTOTALNUM_BLOCK_FOU);
266 0 : return;
267 : }
268 :
269 : // 判断是否跨机
270 0 : if (CheckMultiMachine(rankTable) == false) {
271 0 : HCCL_INFO("[NSLB-DP] CheckMultiMachine is false");
272 0 : return;
273 : }
274 :
275 0 : NslbDpCommConfigVal globalCommInfo{};
276 0 : u32 sRet = memset_s(globalCommInfo.commDesc, COMM_DESC_MAX_LENGTH, 0, sizeof(globalCommInfo.commDesc));
277 0 : if (sRet != EOK) {
278 0 : HCCL_ERROR("memset_s commDesc fail sRet[%u]", sRet);
279 0 : return;
280 : }
281 : /* 获取通信域唯一标识 */
282 0 : sRet = strncpy_s(globalCommInfo.commDesc, COMM_DESC_MAX_LENGTH, identifier.c_str(), identifier.size());
283 0 : if (sRet != EOK) {
284 0 : HCCL_ERROR("strncpy_s commDesc fail sRet[%u]", sRet);
285 0 : return;
286 : }
287 0 : globalCommInfo.commDesc[COMM_DESC_MAX_LENGTH - 1] = '\0';
288 :
289 0 : if (IsCommDescDuplicated(globalCommInfo.commDesc, checkTaskId)) {
290 0 : HCCL_INFO(
291 : "[NSLB-DP] commDesc[%s] taskId[%llu] already exists, skip TBL_COMM_INFO.", identifier.c_str(), checkTaskId);
292 0 : return;
293 : }
294 :
295 : u64 utime
296 0 : = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now().time_since_epoch())
297 0 : .count();
298 0 : globalCommInfo.commInitTime = utime;
299 0 : globalCommInfo.taskId = checkTaskId;
300 0 : globalCommInfo.rankTotalNum = nRanks;
301 :
302 0 : for (u32 rankIndex = 0; rankIndex < rankTable.rankList.size(); rankIndex++) {
303 : NslbDpRankInfo dpRankInfo;
304 0 : HcclIpAddress tmpIp = rankTable.rankList[rankIndex].deviceInfo.deviceIp[0];
305 0 : std::string deviceIp = tmpIp.GetReadableAddress();
306 0 : dpRankInfo.deviceIp = ipToUint32(deviceIp);
307 0 : HCCL_INFO("[NSLB-DP] SetGlobalCommRankTable_RootInfo deviceIp:[%u] success.", dpRankInfo.deviceIp);
308 :
309 0 : std::string serverIp = rankTable.rankList[rankIndex].serverId;
310 0 : size_t underscorePos = serverIp.find('_');
311 0 : if (underscorePos != std::string::npos) {
312 0 : serverIp = serverIp.substr(0, underscorePos);
313 : }
314 0 : dpRankInfo.serverIp = ipToUint32(serverIp);
315 0 : HCCL_INFO("[NSLB-DP] SetGlobalCommRankTable_RootInfo serverIp:[%s] success.", serverIp.c_str());
316 0 : if (rankLists.size() < rankIndex) {
317 0 : return;
318 : }
319 0 : if (rankLists[rankIndex].superPodIdx == INVALID_UINT) {
320 0 : dpRankInfo.podId = 0;
321 : } else {
322 0 : dpRankInfo.podId = rankLists[rankIndex].superPodIdx;
323 : }
324 0 : dpRankInfo.rev = 0;
325 0 : globalCommInfo.rankInfo.push_back(dpRankInfo);
326 0 : }
327 0 : NSLBMD5::calculateRankInfoMd5(globalCommInfo.rankInfo, globalCommInfo.commMd5Sum);
328 0 : std::string nslbdpmd5 = NSLBMD5::md5ToString(globalCommInfo.commMd5Sum);
329 0 : HCCL_DEBUG("[NSLB-DP] NSLBDP-MD5 nslbdpmd5:[%s] success.", nslbdpmd5.c_str());
330 :
331 0 : std::string npuIp = localRankInfo.deviceIP[0].GetReadableIP();
332 0 : if (ipToUint32(npuIp) != 0) {
333 0 : HCCL_RUN_INFO(
334 : "[NSLB-DP] rank[%u]: identifier[%s] nslbdpmd5:[%s].", rank, identifier.c_str(), nslbdpmd5.c_str());
335 0 : SendCommRankTable(rank, globalCommInfo);
336 : }
337 0 : hcclNslbDpCommConfig_.push_back(globalCommInfo);
338 0 : HCCL_INFO("[NSLB-DP] Entry SetGlobalCommRankTable_RootInfo end size = [%zu]", hcclNslbDpCommConfig_.size());
339 0 : }
340 :
341 : /* 填充表5 */
342 0 : void hcclNslbDp::SetGlobalDisRankTable(const HcclBasicRankInfo& rankTable)
343 : {
344 0 : HCCL_INFO("[NSLB-DP] Try to collect NSLBDP_TYPE_TBL_RANK_DIST");
345 0 : u64 taskId = GetGlobalCommTaskId();
346 0 : if (taskId == 0) {
347 0 : return;
348 : }
349 0 : std::string npuIp = rankTable.deviceIP[0].GetReadableIP();
350 0 : if (ipToUint32(npuIp) == 0) {
351 0 : return;
352 : }
353 0 : hcclNslbDpGlobalDisRankVal_.taskId = taskId;
354 0 : hcclNslbDpGlobalDisRankVal_.nodeId = GetGlobalCommNodeId();
355 0 : hcclNslbDpGlobalDisRankVal_.localRankNum = GetGlobalCommLocalRankNum();
356 0 : hcclNslbDpGlobalDisRankVal_.rankTotalNum = GetGlobalCommRankTotalNum();
357 :
358 0 : std::string serverIp = rankTable.hostIP.GetReadableIP();
359 0 : hcclNslbDpGlobalDisRankVal_.serverIp = ipToUint32(serverIp);
360 0 : HCCL_DEBUG("[NSLB-DP] SetGlobalDisRankTable serverIp:[%u].", hcclNslbDpGlobalDisRankVal_.serverIp);
361 :
362 0 : hcclNslbDpGlobalDisRankVal_.npuIp = ipToUint32(npuIp);
363 0 : HCCL_DEBUG("[NSLB-DP] SetGlobalDisRankTable npuIp:[%u].", hcclNslbDpGlobalDisRankVal_.npuIp);
364 0 : }
365 :
366 : /* check 是否是多机场景 */
367 2 : bool hcclNslbDp::CheckMultiMachine(const RankTable_t rankTable)
368 : {
369 2 : HCCL_DEBUG("[NSLB-DP] check device is multi machine");
370 2 : u32 serverIpFir = 0;
371 2 : u16 podIdFir = 0;
372 2 : bool bIsMultiMachine = false;
373 2 : if (GetDeviceType() == true) {
374 0 : u16 podIdIndex = 0;
375 0 : for (u32 index = 0; index < rankTable.rankList.size(); index++) {
376 0 : if (rankTable.rankList[index].superPodIdx == INVALID_UINT) {
377 0 : podIdIndex = 0;
378 : } else {
379 0 : podIdIndex = rankTable.rankList[index].superPodIdx;
380 : }
381 0 : if (index == 0) {
382 0 : podIdFir = podIdIndex;
383 0 : continue;
384 : }
385 :
386 0 : if (podIdFir != podIdIndex) {
387 0 : return true;
388 : }
389 : }
390 0 : return false;
391 : }
392 4 : for (u32 rankIndex = 0; rankIndex < rankTable.rankList.size(); rankIndex++) {
393 4 : std::string serverIpInfo = rankTable.rankList[rankIndex].serverId;
394 4 : u32 serverIp = ipToUint32(serverIpInfo);
395 4 : u16 podIdInfo = 0;
396 4 : if (rankTable.rankList[rankIndex].superPodIdx == INVALID_UINT) {
397 0 : podIdInfo = 0;
398 : } else {
399 4 : podIdInfo = rankTable.rankList[rankIndex].superPodIdx;
400 : }
401 4 : if (rankIndex == 0) {
402 2 : serverIpFir = serverIp;
403 2 : podIdFir = podIdInfo;
404 2 : continue;
405 : }
406 :
407 2 : if (serverIp != serverIpFir || podIdFir != podIdInfo) {
408 2 : bIsMultiMachine = true;
409 2 : break;
410 : }
411 4 : }
412 2 : return bIsMultiMachine;
413 : }
414 :
415 : /* 无ranktable场景， 子通信域场景表1 赋值 */
416 : HcclResult
417 4 : hcclNslbDp::SetCommInfo_NoRankTable(const hccl::RankTable_t rankTable, std::string identifier, u32 subCommRankId)
418 : {
419 4 : HCCL_DEBUG("[NSLB-DP] Try to collect NSLBDP_TYPE_TBL_COMM_INFO for no RankTable");
420 4 : u64 taskId = GetGlobalCommTaskId();
421 4 : if (taskId == 0) {
422 1 : return HCCL_SUCCESS;
423 : }
424 3 : if (rankTable.rankNum > NSLBDP_RANKTOTALNUM_BLOCK_FOU) {
425 1 : HCCL_INFO(
426 : "[NSLB-DP] rankNum[%u] exceeds limit[%u], skip TBL_COMM_INFO.", rankTable.rankNum,
427 : NSLBDP_RANKTOTALNUM_BLOCK_FOU);
428 1 : return HCCL_SUCCESS;
429 : }
430 : // 判断是否跨机 false非错误场景
431 2 : if (CheckMultiMachine(rankTable) == false) {
432 0 : HCCL_INFO("[NSLB-DP] nslb-dp CheckMultiMachine is false");
433 0 : return HCCL_SUCCESS;
434 : }
435 :
436 2 : NslbDpCommConfigVal globalCommInfo;
437 2 : (void)memset_s(globalCommInfo.commDesc, COMM_DESC_MAX_LENGTH, 0, sizeof(globalCommInfo.commDesc));
438 :
439 : // 获取通信域唯一标识
440 2 : s32 ret = strncpy_s(globalCommInfo.commDesc, COMM_DESC_MAX_LENGTH, identifier.c_str(), identifier.size());
441 2 : CHK_PRT_RET(ret != EOK, HCCL_ERROR("[NSLB_DP]GetIdentifier str copy fail. return[%d]", ret), HCCL_E_INTERNAL);
442 :
443 2 : globalCommInfo.commDesc[COMM_DESC_MAX_LENGTH - 1] = '\0';
444 2 : if (IsCommDescDuplicated(globalCommInfo.commDesc, taskId)) {
445 1 : HCCL_INFO(
446 : "[NSLB-DP] commDesc[%s] taskId[%llu] already exists, skip TBL_COMM_INFO.", identifier.c_str(), taskId);
447 1 : return HCCL_SUCCESS;
448 : }
449 :
450 : // commInitTime在有ranktable的赋值
451 : u64 utime
452 1 : = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now().time_since_epoch())
453 1 : .count();
454 1 : globalCommInfo.commInitTime = utime;
455 1 : u16 nRanks = rankTable.rankNum;
456 1 : globalCommInfo.taskId = taskId;
457 1 : globalCommInfo.rankTotalNum = nRanks;
458 :
459 1 : FillRankInfoFromRankTable(globalCommInfo, rankTable);
460 1 : NSLBMD5::calculateRankInfoMd5(globalCommInfo.rankInfo, globalCommInfo.commMd5Sum);
461 1 : std::string nslbdpmd5 = NSLBMD5::md5ToString(globalCommInfo.commMd5Sum);
462 1 : HCCL_RUN_INFO(
463 : "[NSLB-DP] Subcomm rankId[%u] identifier[%s] nslbdpmd5:[%s].", subCommRankId, identifier.c_str(),
464 : nslbdpmd5.c_str());
465 :
466 1 : SendCommRankTable(subCommRankId, globalCommInfo);
467 1 : hcclNslbDpCommConfig_.push_back(globalCommInfo);
468 :
469 1 : return HCCL_SUCCESS;
470 2 : }
471 :
472 : /* 有ranktable场景， 表1 赋值 */
473 3 : HcclResult hcclNslbDp::SetCommInfo_RankTableExit(RankTable_t rankTable)
474 : {
475 3 : HCCL_DEBUG("[NSLB-DP] Entry SetCommInfo for RankTable exit");
476 3 : u16 nRanks = rankTable.rankNum;
477 3 : u64 taskId = GetGlobalCommTaskId();
478 3 : if (taskId == 0 || nRanks == 1) {
479 1 : return HCCL_SUCCESS;
480 : }
481 2 : if (nRanks > NSLBDP_RANKTOTALNUM_BLOCK_FOU) {
482 1 : HCCL_INFO("[NSLB-DP] nRanks[%u] exceeds limit[%u], skip TBL_COMM_INFO.", nRanks, NSLBDP_RANKTOTALNUM_BLOCK_FOU);
483 1 : return HCCL_SUCCESS;
484 : }
485 :
486 1 : NslbDpCommConfigVal globalCommInfo = {};
487 : // 获取通信域唯一标识
488 1 : (void)memset_s(
489 1 : hcclNslbDpGlobalRankVal_.commDesc, COMM_DESC_MAX_LENGTH, 0, sizeof(hcclNslbDpGlobalRankVal_.commDesc));
490 1 : char commDesc[COMM_DESC_MAX_LENGTH] = "HCCL_WORLD_GROUP";
491 1 : s32 sRet = memcpy_s(globalCommInfo.commDesc, sizeof(globalCommInfo.commDesc), commDesc, COMM_DESC_MAX_LENGTH);
492 1 : if (sRet != EOK) {
493 0 : HCCL_ERROR("memcpy_s commDesc fail");
494 0 : return HCCL_SUCCESS;
495 : }
496 :
497 1 : if (IsCommDescDuplicated(globalCommInfo.commDesc, taskId)) {
498 0 : HCCL_INFO(
499 : "[NSLB-DP] commDesc[%s] taskId[%llu] already exists, skip TBL_COMM_INFO.", globalCommInfo.commDesc, taskId);
500 0 : return HCCL_SUCCESS;
501 : }
502 1 : globalCommInfo.taskId = taskId;
503 1 : globalCommInfo.rankTotalNum = nRanks;
504 :
505 : u64 utime
506 1 : = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now().time_since_epoch())
507 1 : .count();
508 1 : globalCommInfo.commInitTime = utime;
509 :
510 1 : u32 size = rankTable.rankList.size();
511 1 : HCCL_INFO("[NSLB-DP] RankTableExit size:[%u] success.", size);
512 :
513 5 : for (size_t ranksize = 0; ranksize < rankTable.rankList.size(); ranksize++) {
514 20 : for (size_t rankIndex = 0; rankIndex < rankTable.rankList.size(); rankIndex++) {
515 16 : u16 podId = 0;
516 16 : if (rankTable.rankList[rankIndex].superPodIdx != INVALID_UINT) {
517 16 : podId = rankTable.rankList[rankIndex].superPodIdx;
518 : }
519 : // 按照rankid 排序
520 16 : if (ranksize == rankTable.rankList[rankIndex].rankId) {
521 : NslbDpRankInfo dpRankInfo;
522 4 : HcclIpAddress tmpIp = rankTable.rankList[rankIndex].deviceInfo.deviceIp[0];
523 4 : std::string deviceIp = tmpIp.GetReadableAddress();
524 4 : dpRankInfo.deviceIp = ipToUint32(deviceIp);
525 4 : HCCL_INFO("[NSLB-DP] SetCommInfo_RankTableExit deviceIp:[%s] success.", deviceIp.c_str());
526 4 : std::string serverIp = rankTable.rankList[rankIndex].serverId;
527 4 : dpRankInfo.serverIp = ipToUint32(serverIp);
528 4 : HCCL_INFO("[NSLB-DP] SetCommInfo_RankTableExit serverIp:[%s] success.", serverIp.c_str());
529 4 : dpRankInfo.podId = podId;
530 4 : dpRankInfo.rev = 0;
531 4 : globalCommInfo.rankInfo.push_back(dpRankInfo);
532 4 : }
533 : }
534 : }
535 1 : NSLBMD5::calculateRankInfoMd5(globalCommInfo.rankInfo, globalCommInfo.commMd5Sum);
536 1 : std::string nslbdpmd5 = NSLBMD5::md5ToString(globalCommInfo.commMd5Sum);
537 1 : HCCL_INFO("[NSLB-DP] check pmd5:[%s] success.", nslbdpmd5.c_str());
538 :
539 1 : hcclNslbDpCommConfig_.push_back(globalCommInfo);
540 1 : HCCL_DEBUG("[NSLB-DP] entry SetCommInfo_RankTableExit end");
541 :
542 1 : return HCCL_SUCCESS;
543 1 : }
544 :
545 : /* 有ranktable场景，表4赋值 */
546 0 : HcclResult hcclNslbDp::SetGlobalRank_RankTableExit(const hccl::RankTable_t rankTable)
547 : {
548 0 : u64 taskId = GetGlobalCommTaskId();
549 0 : HCCL_INFO("[NSLB-DP] set TBL_RANK for RankTableExit[%ull] success.", taskId);
550 :
551 0 : if (taskId == 0) {
552 0 : return HCCL_SUCCESS;
553 : }
554 0 : u16 nRanks = rankTable.rankNum;
555 0 : hcclNslbDpGlobalRankVal_.taskId = taskId;
556 0 : (void)memset_s(
557 0 : hcclNslbDpGlobalRankVal_.commDesc, COMM_DESC_MAX_LENGTH, 0, sizeof(hcclNslbDpGlobalRankVal_.commDesc));
558 0 : char commDesc[COMM_DESC_MAX_LENGTH] = "HCCL_WORLD_GROUP";
559 0 : s32 sRet = memcpy_s(
560 0 : hcclNslbDpGlobalRankVal_.commDesc, sizeof(hcclNslbDpGlobalRankVal_.commDesc), commDesc, COMM_DESC_MAX_LENGTH);
561 0 : if (sRet != EOK) {
562 0 : HCCL_ERROR("memcpy_s commDesc fail");
563 0 : return HCCL_SUCCESS;
564 : }
565 :
566 : hcclNslbDpGlobalRankVal_.commInitTime
567 0 : = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now().time_since_epoch())
568 0 : .count();
569 0 : hcclNslbDpGlobalRankVal_.rankTotalNum = nRanks;
570 :
571 0 : for (size_t ranksize = 0; ranksize < rankTable.rankList.size(); ranksize++) {
572 0 : for (size_t rankIndex = 0; rankIndex < rankTable.rankList.size(); rankIndex++) {
573 0 : if (ranksize == rankTable.rankList[rankIndex].rankId) {
574 : TableFourRankInfo dpGloRankInfo;
575 0 : HcclIpAddress tmpIp = rankTable.rankList[rankIndex].deviceInfo.deviceIp[0];
576 0 : std::string deviceIp = tmpIp.GetReadableAddress();
577 0 : dpGloRankInfo.deviceIp = ipToUint32(deviceIp);
578 :
579 0 : std::string serverIp = rankTable.rankList[rankIndex].serverId;
580 0 : dpGloRankInfo.serverIp = ipToUint32(serverIp);
581 :
582 0 : hcclNslbDpGlobalRankVal_.rankInfo.push_back(dpGloRankInfo);
583 0 : }
584 : }
585 : }
586 0 : NSLBMD5::calculateTableFourRankInfoMd5(hcclNslbDpGlobalRankVal_.rankInfo, hcclNslbDpGlobalRankVal_.commMd5Sum);
587 0 : std::string nslbdpmd5 = NSLBMD5::md5ToString(hcclNslbDpGlobalRankVal_.commMd5Sum);
588 0 : HCCL_INFO("[NSLB-DP] check pmd5:[%s] success.", nslbdpmd5.c_str());
589 0 : return HCCL_SUCCESS;
590 0 : }
591 :
592 : /* 拼接 l4SPortId */
593 1 : HcclResult hcclNslbDp::GetNslbDpl4SPortId(u32 rankSize, u8 algType, u16* l4SPortId)
594 : {
595 1 : u16 priFlag = NSLBDP_PRIVATE_PORT;
596 1 : u16 CommIntervalFlag = NSLB_COMM_INTERVAL_FLAG_BEGIN;
597 1 : if (rankSize > NSLBDP_COMMINTERVAL_FLAGSIX) {
598 0 : CommIntervalFlag = NSLB_COMM_INTERVAL_FLAG_SEV;
599 1 : } else if (rankSize > NSLBDP_COMMINTERVAL_FLAGFIV) {
600 0 : CommIntervalFlag = NSLB_COMM_INTERVAL_FLAG_SIX;
601 1 : } else if (rankSize > NSLBDP_COMMINTERVAL_FLAGFOU) {
602 0 : CommIntervalFlag = NSLB_COMM_INTERVAL_FLAG_FIV;
603 1 : } else if (rankSize > NSLBDP_COMMINTERVAL_FLAGTHR) {
604 0 : CommIntervalFlag = NSLB_COMM_INTERVAL_FLAG_FOR;
605 1 : } else if (rankSize > NSLBDP_COMMINTERVAL_FLAGSEC) {
606 0 : CommIntervalFlag = NSLB_COMM_INTERVAL_FLAG_THR;
607 1 : } else if (rankSize > NSLBDP_COMMINTERVAL_FLAG) {
608 0 : CommIntervalFlag = NSLB_COMM_INTERVAL_FLAG_SEC;
609 : } else {
610 1 : CommIntervalFlag = NSLB_COMM_INTERVAL_FLAG_FIR;
611 : }
612 1 : u16 CommPrecisely = rankSize % NSLBDP_COMMINTERVAL_FLAG;
613 1 : u8 CommalgType = algType;
614 :
615 1 : *l4SPortId = (priFlag << NSLBDP_RANGE_ID) + (CommIntervalFlag << NSLBDP_COMMON_RANGE)
616 1 : + (CommPrecisely << NSLBDP_ALGO_RANGE) + CommalgType;
617 1 : hcclNslbDpL4SPortId_ = *l4SPortId;
618 :
619 1 : HCCL_INFO("[NSLB-DP-L4PORT] get hcclNslbDpL4SPortId_[%u] success", hcclNslbDpL4SPortId_);
620 1 : return HCCL_SUCCESS;
621 : }
622 :
623 : /* 表6赋值 */
624 0 : HcclResult hcclNslbDp::SetNslbDpRootRank(HcclCMDType opType, u32 rootRank, std::string identifier, u8 algType)
625 : {
626 0 : HCCL_DEBUG("[NSLB-DP] try to collect NSLBDP_TYPE_TBL_ROOT_RANK");
627 0 : u64 taskId = GetGlobalCommTaskId();
628 0 : if (taskId == 0) {
629 0 : return HCCL_SUCCESS;
630 : }
631 0 : if (hcclNslbDpRootRankVal_.taskId == 0) {
632 0 : HCCL_DEBUG("[NSLB-DP-BEGIN] RootRank first entry");
633 : // 获取task id
634 0 : hcclNslbDpRootRankVal_.taskId = taskId;
635 0 : s32 sRet = memset_s(
636 0 : hcclNslbDpRootRankVal_.commDesc, COMM_DESC_MAX_LENGTH, 0, sizeof(hcclNslbDpRootRankVal_.commDesc));
637 0 : if (sRet != EOK) {
638 0 : HCCL_ERROR("memset_s commDesc fail");
639 0 : return HCCL_SUCCESS;
640 : }
641 : // 获取通信域唯一标识
642 : s32 ret
643 0 : = strncpy_s(hcclNslbDpRootRankVal_.commDesc, COMM_DESC_MAX_LENGTH, identifier.c_str(), identifier.size());
644 0 : if (ret != EOK) {
645 0 : HCCL_INFO("strncpy_s hcclNslbDpRootRankVal_.commDesc fail");
646 0 : return HCCL_SUCCESS;
647 : }
648 0 : hcclNslbDpRootRankVal_.commDesc[COMM_DESC_MAX_LENGTH - 1] = '\0';
649 :
650 : // commInitTime在有ranktable的赋值
651 : hcclNslbDpRootRankVal_.commInitTime
652 0 : = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now().time_since_epoch())
653 0 : .count();
654 :
655 : // 获取operator、algorithm
656 0 : hcclNslbDpRootRankVal_.oper = GetNslbOpType(opType);
657 0 : hcclNslbDpRootRankVal_.algorithm = algType;
658 0 : hcclNslbDpRootRankVal_.rootRankNum = hcclNslbDpRootRankVal_.rootRankNum + 1;
659 :
660 : NslbDpRankId rootRankId;
661 0 : rootRankId.rankID = rootRank;
662 0 : hcclNslbDpRootRankVal_.rankId.push_back(rootRankId);
663 :
664 0 : HCCL_INFO(
665 : "[NSLB-DP] Rootable rootRankNum[%u]-rootRank[%u]-algorithm[%u].", hcclNslbDpRootRankVal_.rootRankNum,
666 : rootRank, hcclNslbDpRootRankVal_.algorithm);
667 :
668 0 : HCCL_DEBUG("[NSLB-DP] Rootabl entry SetNslbDpRootRank end");
669 0 : SendRootRankTable();
670 0 : return HCCL_SUCCESS;
671 : }
672 0 : HCCL_INFO(
673 : "[NSLB-DP] Rootable rankId.size[%zu]-rootRankNum[%u]-rootRank[%u].", hcclNslbDpRootRankVal_.rankId.size(),
674 : hcclNslbDpRootRankVal_.rootRankNum, rootRank);
675 :
676 0 : for (const auto& rank : hcclNslbDpRootRankVal_.rankId) {
677 0 : HCCL_INFO("[NSLB-DP] Rootable rootRank exit rankID[%u]-rootRank[%u]", rank.rankID, rootRank);
678 0 : if (rank.rankID == rootRank) {
679 0 : return HCCL_SUCCESS;
680 : }
681 : }
682 :
683 : NslbDpRankId rootRankId;
684 0 : rootRankId.rankID = static_cast<u16>(rootRank);
685 0 : hcclNslbDpRootRankVal_.rankId.push_back(rootRankId);
686 0 : hcclNslbDpRootRankVal_.rootRankNum = hcclNslbDpRootRankVal_.rootRankNum + 1;
687 :
688 : // 新增场景下走send 流程
689 0 : SendRootRankTable();
690 :
691 0 : return HCCL_SUCCESS;
692 : }
693 :
694 : /* AlgType 的转换 */
695 9 : u8 hcclNslbDp::GetNslbLevel1AlgType(AlgTypeLevel1 algValue)
696 : {
697 9 : HCCL_DEBUG("[NSLB-DP] try to switch Level1 type to nslbtype");
698 9 : switch (algValue) {
699 1 : case AlgTypeLevel1::ALG_LEVEL1_RING: {
700 1 : return NSLB_ALGO_TYPE_RING;
701 : }
702 1 : case AlgTypeLevel1::ALG_LEVEL1_PIPELINE: {
703 1 : return NSLB_ALGO_TYPE_PIPELINE;
704 : }
705 1 : case AlgTypeLevel1::ALG_LEVEL1_HD: {
706 1 : return NSLB_ALGO_TYPE_HDR;
707 : }
708 1 : case AlgTypeLevel1::ALG_LEVEL1_NHR: {
709 1 : return NSLB_ALGO_TYPE_NHR;
710 : }
711 1 : case AlgTypeLevel1::ALG_LEVEL1_NHR_V1: {
712 1 : return NSLB_ALGO_TYPE_NHR_V1;
713 : }
714 1 : case AlgTypeLevel1::ALG_LEVEL1_NB: {
715 1 : return NSLB_ALGO_TYPE_NB;
716 : }
717 1 : case AlgTypeLevel1::ALG_LEVEL1_AHC: {
718 1 : return NSLB_ALGO_TYPE_AHC;
719 : }
720 1 : case AlgTypeLevel1::ALG_LEVEL1_AHC_BROKE: {
721 1 : return NSLB_ALGO_TYPE_AHC;
722 : }
723 1 : default:
724 1 : return NSLB_ALGO_TYPE_NA;
725 : }
726 : }
727 :
728 : /* Level2 的 AlgType 转换 */
729 6 : u8 hcclNslbDp::GetNslbLevel2AlgType(AlgTypeLevel2 algValue)
730 : {
731 6 : HCCL_DEBUG("[NSLB-DP] try to switch Level2 type to nslbtype");
732 6 : switch (algValue) {
733 1 : case AlgTypeLevel2::ALG_LEVEL2_RING: {
734 1 : return NSLB_ALGO_TYPE_RING;
735 : }
736 1 : case AlgTypeLevel2::ALG_LEVEL2_HD: {
737 1 : return NSLB_ALGO_TYPE_HDR;
738 : }
739 1 : case AlgTypeLevel2::ALG_LEVEL2_NHR: {
740 1 : return NSLB_ALGO_TYPE_NHR;
741 : }
742 1 : case AlgTypeLevel2::ALG_LEVEL2_NB: {
743 1 : return NSLB_ALGO_TYPE_NB;
744 : }
745 1 : case AlgTypeLevel2::ALG_LEVEL2_PIPELINE: {
746 1 : return NSLB_ALGO_TYPE_PIPELINE;
747 : }
748 1 : default:
749 1 : return NSLB_ALGO_TYPE_NA;
750 : }
751 : }
752 :
753 : /* opType 的转换 */
754 148 : u8 hcclNslbDp::GetNslbOpType(HcclCMDType opType)
755 : {
756 148 : HCCL_DEBUG("[NSLB-DP] try to get nslb optype");
757 148 : switch (opType) {
758 8 : case HcclCMDType::HCCL_CMD_BROADCAST: {
759 8 : return NSLBDP_CMD_BROADCAST;
760 : }
761 10 : case HcclCMDType::HCCL_CMD_ALLREDUCE: {
762 10 : return NSLBDP_CMD_ALLREDUCE;
763 : }
764 11 : case HcclCMDType::HCCL_CMD_REDUCE: {
765 11 : return NSLBDP_CMD_REDUCE;
766 : }
767 12 : case HcclCMDType::HCCL_CMD_SEND: {
768 12 : return NSLBDP_CMD_SEND;
769 : }
770 12 : case HcclCMDType::HCCL_CMD_RECEIVE: {
771 12 : return NSLBDP_CMD_RECEIVE;
772 : }
773 18 : case HcclCMDType::HCCL_CMD_ALLGATHER: {
774 18 : return NSLBDP_CMD_ALLGATHER;
775 : }
776 4 : case HcclCMDType::HCCL_CMD_REDUCE_SCATTER: {
777 4 : return NSLBDP_CMD_REDUCE_SCATTER;
778 : }
779 6 : case HcclCMDType::HCCL_CMD_ALLTOALLV: {
780 6 : return NSLBDP_CMD_ALLTOALLV;
781 : }
782 7 : case HcclCMDType::HCCL_CMD_ALLTOALLVC: {
783 7 : return NSLBDP_CMD_ALLTOALLVC;
784 : }
785 5 : case HcclCMDType::HCCL_CMD_ALLTOALL: {
786 5 : return NSLBDP_CMD_ALLTOALL;
787 : }
788 23 : case HcclCMDType::HCCL_CMD_GATHER: {
789 23 : return NSLBDP_CMD_GATHER;
790 : }
791 9 : case HcclCMDType::HCCL_CMD_SCATTER: {
792 9 : return NSLBDP_CMD_SCATTER;
793 : }
794 11 : case HcclCMDType::HCCL_CMD_BATCH_SEND_RECV: {
795 11 : return NSLBDP_CMD_BATCH_SEND_RECV;
796 : }
797 12 : default:
798 12 : return 0;
799 : }
800 : }
801 :
802 : /* 获取通信量的前4bit */
803 7 : u64 hcclNslbDp::GetNslbDpFirstFourBit(u8 opType, u8 algType)
804 : {
805 7 : u64 firstFourBit = 0;
806 7 : if (opType == NSLBDP_CMD_ALLREDUCE) {
807 2 : firstFourBit = 1 << NSLBDP_BEGINFOURBIT;
808 2 : firstFourBit = firstFourBit + 1;
809 : }
810 :
811 7 : if (opType == NSLBDP_CMD_ALLGATHER) {
812 1 : firstFourBit = 1 << 1;
813 : }
814 :
815 7 : HCCL_INFO("[NSLB-DP-FIRST4] try to get first FourBit[%llu]-algType[%u]", firstFourBit, algType);
816 7 : return firstFourBit;
817 : }
818 :
819 : /* 校验算法的一致性 */
820 12 : bool hcclNslbDp::CheckAlgoConsistency(HcclCMDType opType, std::string& algName)
821 : {
822 12 : if (opType == HcclCMDType::HCCL_CMD_ALLREDUCE) {
823 2 : if (algName.find("AllReduce") != std::string::npos) {
824 1 : return true;
825 : } else {
826 1 : return false;
827 : }
828 10 : } else if (opType == HcclCMDType::HCCL_CMD_ALLGATHER) {
829 2 : if (algName.find("AllGather") != std::string::npos) {
830 1 : return true;
831 : } else {
832 1 : return false;
833 : }
834 8 : } else if (opType == HcclCMDType::HCCL_CMD_REDUCE_SCATTER) {
835 2 : if (algName.find("ReduceScatter") != std::string::npos) {
836 1 : return true;
837 : } else {
838 1 : return false;
839 : }
840 6 : } else if (opType == HcclCMDType::HCCL_CMD_ALLTOALL) {
841 2 : if (algName != "RunAlltoAllVFullMesh") {
842 1 : return false;
843 : }
844 4 : } else if (opType == HcclCMDType::HCCL_CMD_BROADCAST) {
845 2 : if (algName.find("BroadCast") != std::string::npos) {
846 1 : return true;
847 : } else {
848 1 : return false;
849 : }
850 2 : } else if (opType == HcclCMDType::HCCL_CMD_SCATTER) {
851 2 : if (algName.find("Scatter") != std::string::npos) {
852 1 : return true;
853 : } else {
854 1 : return false;
855 : }
856 : }
857 1 : return true;
858 : }
859 :
860 22 : bool hcclNslbDp::CheckSupportOptype(HcclCMDType opType)
861 : {
862 22 : HCCL_DEBUG("[NSLB-DP] try to check support optype");
863 39 : if (GetNslbOpType(opType) != NSLBDP_CMD_ALLREDUCE && GetNslbOpType(opType) != NSLBDP_CMD_ALLGATHER
864 13 : && GetNslbOpType(opType) != NSLBDP_CMD_REDUCE_SCATTER && GetNslbOpType(opType) != NSLBDP_CMD_ALLTOALL
865 11 : && GetNslbOpType(opType) != NSLBDP_CMD_ALLTOALLV && GetNslbOpType(opType) != NSLBDP_CMD_ALLTOALLVC
866 9 : && GetNslbOpType(opType) != NSLBDP_CMD_BROADCAST && GetNslbOpType(opType) != NSLBDP_CMD_SCATTER
867 7 : && GetNslbOpType(opType) != NSLBDP_CMD_BATCH_SEND_RECV && GetNslbOpType(opType) != NSLBDP_CMD_REDUCE
868 39 : && GetNslbOpType(opType) != NSLBDP_CMD_SEND) {
869 4 : return false;
870 : }
871 18 : return true;
872 : }
873 :
874 : /* 写算法算子表--表3 */
875 7 : HcclResult hcclNslbDp::GetAlgAdjacencyTable(
876 : HcclCMDType opType, u32 srcLocalRankId, u32 rootRank, u8 algType, std::string identifier, AdjInfo nslbAdjInfo)
877 : {
878 7 : HCCL_DEBUG("[NSLB-DP] check table NSLBDP_TYPE_TBL_ADJ size:[%zu].", hcclNslbDpAlgorithmInfo_.size());
879 7 : if (CheckAhcSupport(algType, identifier) == false) {
880 0 : HCCL_RUN_INFO("[NSLB-DP-ADJ] Check AHC commoninfo is not support.");
881 0 : return HCCL_SUCCESS;
882 : }
883 7 : u64 taskId = GetGlobalCommTaskId();
884 7 : if (taskId == 0) {
885 1 : return HCCL_SUCCESS;
886 : }
887 6 : HCCL_INFO(
888 : "[NSLB-DP-ADJ] opType:[%u],srcLocalRankId[%u],rootRank[%u]-commDesc[%s],dstRankNum:[%u].", opType,
889 : srcLocalRankId, rootRank, identifier.c_str(), nslbAdjInfo.dstRankNum);
890 :
891 6 : if (CheckSupportOptype(opType) == false) {
892 1 : HCCL_INFO("[NSLB-DP-OPER] CheckSupportOptype false .");
893 1 : return HCCL_SUCCESS;
894 : }
895 :
896 5 : NslbDpAlgorithmInfo algorithmInfo;
897 5 : algorithmInfo.taskId = taskId;
898 5 : if (InitAlgInfoCommDesc(algorithmInfo, identifier) == false) {
899 1 : return HCCL_SUCCESS;
900 : }
901 4 : if (FillAlgInfoCommMd5(algorithmInfo) == false) {
902 0 : return HCCL_SUCCESS;
903 : }
904 4 : FillAlgInfoBaseFields(algorithmInfo, opType, srcLocalRankId, rootRank, algType);
905 :
906 4 : HCCL_RUN_INFO(
907 : "[NSLB-DP] add adjINfo:***[%llu]***[%u]***[%u]***[%u]***[%u]-[%zu] success.", taskId, srcLocalRankId, rootRank,
908 : GetNslbOpType(opType), algType, nslbAdjInfo.nsAdjInfo.size());
909 :
910 4 : if (IsAlgAdjacencyDuplicated(algorithmInfo) == true) {
911 1 : HCCL_INFO("[NSLB-DP] Deduplication hcclNslbDpAlgorithmInfo_");
912 1 : return HCCL_SUCCESS;
913 : }
914 :
915 3 : HCCL_INFO(
916 : "[NSLB-DP-ADJ] add adjINfo:***[%llu]***[%u]***[%u]***[%u]***[%u] success.", taskId, srcLocalRankId, rootRank,
917 : GetNslbOpType(opType), algType);
918 :
919 3 : if (FillAlgInfoAdjInfo(algorithmInfo, nslbAdjInfo, srcLocalRankId) == false) {
920 0 : return HCCL_SUCCESS;
921 : }
922 :
923 3 : algorithmInfo.sedFlag = 0;
924 3 : hcclNslbDpAlgorithmInfo_.push_back(algorithmInfo);
925 3 : HCCL_DEBUG("[NSLB-DP] entry GetAlgAdjacencyTable end");
926 3 : return HCCL_SUCCESS;
927 5 : }
928 :
929 : /* 初始化commDesc并校验通信域是否存在；存在返回true，不存在返回false */
930 8 : bool hcclNslbDp::InitAlgInfoCommDesc(NslbDpAlgorithmInfo& algorithmInfo, const std::string& identifier)
931 : {
932 8 : (void)memset_s(algorithmInfo.commDesc, COMM_DESC_MAX_LENGTH, 0, sizeof(algorithmInfo.commDesc));
933 8 : s32 ret = strncpy_s(algorithmInfo.commDesc, COMM_DESC_MAX_LENGTH, identifier.c_str(), identifier.size());
934 8 : if (ret != EOK) {
935 0 : HCCL_ERROR("[NSLB-DP] strncpy_s algorithmInfo.commDesc fail");
936 0 : return true;
937 : }
938 8 : algorithmInfo.commDesc[COMM_DESC_MAX_LENGTH - 1] = '\0';
939 :
940 8 : HCCL_INFO("[NSLB-DP] CheckCommDescExit algorithmInfo.commDesc[%s] .", algorithmInfo.commDesc);
941 :
942 : // 去除不存在的通信域信息
943 8 : bool commDescExit = false;
944 10 : for (const auto& info : hcclNslbDpCommConfig_) {
945 7 : if (strcmp(algorithmInfo.commDesc, info.commDesc) == 0) {
946 5 : commDescExit = true;
947 5 : break;
948 : }
949 : }
950 8 : return commDescExit;
951 : }
952 :
953 : /* 根据表一信息填充commMd5Sum；memcpy失败返回false */
954 6 : bool hcclNslbDp::FillAlgInfoCommMd5(NslbDpAlgorithmInfo& algorithmInfo)
955 : {
956 : // 根据表一信息填充MD5
957 7 : for (size_t comsize = 0; comsize < hcclNslbDpCommConfig_.size(); comsize++) {
958 6 : if (strcmp(algorithmInfo.commDesc, hcclNslbDpCommConfig_[comsize].commDesc) == 0) {
959 10 : s32 sRet = memcpy_s(
960 5 : algorithmInfo.commMd5Sum, sizeof(algorithmInfo.commMd5Sum), hcclNslbDpCommConfig_[comsize].commMd5Sum,
961 : sizeof(hcclNslbDpCommConfig_[comsize].commMd5Sum));
962 5 : if (sRet != EOK) {
963 0 : HCCL_ERROR("memcpy_s commMd5Sum fail");
964 0 : return false;
965 : }
966 5 : break;
967 : }
968 : }
969 6 : return true;
970 : }
971 :
972 : /* 填充srcLocalRankId/rootRank/oper/algorithm基础字段 */
973 6 : void hcclNslbDp::FillAlgInfoBaseFields(
974 : NslbDpAlgorithmInfo& algorithmInfo, HcclCMDType opType, u32 srcLocalRankId, u32 rootRank, u8 algType)
975 : {
976 6 : algorithmInfo.srcLocalRankId = srcLocalRankId;
977 6 : algorithmInfo.rootRank = static_cast<u16>(rootRank);
978 6 : algorithmInfo.oper = GetNslbOpType(opType);
979 6 : algorithmInfo.algorithm = algType;
980 6 : }
981 :
982 : /* 判断是否与已有邻接表项重复（去重） */
983 8 : bool hcclNslbDp::IsAlgAdjacencyDuplicated(const NslbDpAlgorithmInfo& algorithmInfo)
984 : {
985 11 : for (const auto& info : hcclNslbDpAlgorithmInfo_) {
986 5 : HCCL_INFO(
987 : "[NSLB-DP-ADJ] info: *[%llu]-[%u]-[%u]-[%u]-[%u]* success.", algorithmInfo.taskId, info.srcLocalRankId,
988 : info.rootRank, info.oper, info.algorithm);
989 5 : if (info.taskId == algorithmInfo.taskId && info.srcLocalRankId == algorithmInfo.srcLocalRankId
990 3 : && info.rootRank == algorithmInfo.rootRank && info.oper == algorithmInfo.oper
991 3 : && info.algorithm == algorithmInfo.algorithm && strcmp(algorithmInfo.commDesc, info.commDesc) == 0) {
992 2 : return true;
993 : }
994 : }
995 6 : return false;
996 : }
997 :
998 : /* 填充dstRankNum与AdjInfo邻接信息；nsAdjInfo为空返回false */
999 5 : bool hcclNslbDp::FillAlgInfoAdjInfo(NslbDpAlgorithmInfo& algorithmInfo, const AdjInfo& nslbAdjInfo, u32 srcLocalRankId)
1000 : {
1001 5 : algorithmInfo.dstRankNum = nslbAdjInfo.dstRankNum;
1002 5 : HCCL_INFO("[NSLB-DP-ADJ] nslbAdjInfo.dstRankNum:[%u].", nslbAdjInfo.dstRankNum);
1003 :
1004 5 : if (nslbAdjInfo.nsAdjInfo.size() == 0) {
1005 1 : algorithmInfo.dstRankNum = 0;
1006 1 : HCCL_INFO("[NSLB-DP] get nsAdjInfo fail dstRankNum:[%u]", algorithmInfo.dstRankNum);
1007 1 : return false;
1008 : }
1009 :
1010 14 : for (size_t rankIndex = 0; rankIndex < nslbAdjInfo.nsAdjInfo.size(); rankIndex++) {
1011 : NslbDpAdjInfo adjInfo;
1012 10 : adjInfo.dstLocalRankId = nslbAdjInfo.nsAdjInfo[rankIndex].dstLocalRankId;
1013 10 : adjInfo.phaseId = nslbAdjInfo.nsAdjInfo[rankIndex].phaseId;
1014 10 : adjInfo.rev = nslbAdjInfo.nsAdjInfo[rankIndex].rev;
1015 10 : HCCL_INFO(
1016 : "[NSLB-DP-ADJ] adjINfo:[%u]-[%u]-[%u] success.", srcLocalRankId,
1017 : nslbAdjInfo.nsAdjInfo[rankIndex].dstLocalRankId, nslbAdjInfo.nsAdjInfo[rankIndex].phaseId);
1018 10 : algorithmInfo.AdjInfo.push_back(adjInfo);
1019 : }
1020 4 : return true;
1021 : }
1022 :
1023 5 : bool hcclNslbDp::CheckCommDescExit(NslbDpOperatorInfo& OperatorInfo)
1024 : {
1025 5 : HCCL_DEBUG("[NSLB-DP-OPER] Check CommDescExit size:[%zu].", hcclNslbDpCommConfig_.size());
1026 7 : for (const auto& info : hcclNslbDpCommConfig_) {
1027 4 : HCCL_DEBUG("[NSLB-DP-OPER] CheckCommDescExit info.commDesc[%s] .", info.commDesc);
1028 4 : if (strcmp(OperatorInfo.commDesc, info.commDesc) == 0) {
1029 2 : return true;
1030 : }
1031 : }
1032 3 : return false;
1033 : }
1034 :
1035 4 : void hcclNslbDp::fullcommDescInitTime(std::string identifier, NslbDpOperatorInfo& OperatorInfo)
1036 : {
1037 : /* 获取通信域唯一标识 */
1038 4 : s32 ret = strncpy_s(OperatorInfo.commDesc, COMM_DESC_MAX_LENGTH, identifier.c_str(), identifier.size());
1039 4 : if (ret != EOK) {
1040 0 : HCCL_INFO("[NSLB-DP] strncpy_s OperatorInfo.commDesc fail");
1041 0 : return;
1042 : }
1043 4 : HCCL_DEBUG("[NSLB-DP-OPER] fullcommDescInitTime commDesc[%s] .", identifier.c_str());
1044 :
1045 4 : OperatorInfo.commDesc[COMM_DESC_MAX_LENGTH - 1] = '\0';
1046 :
1047 : // commInitTime在有ranktable的赋值
1048 : OperatorInfo.commInitTime
1049 4 : = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now().time_since_epoch())
1050 4 : .count();
1051 :
1052 4 : return;
1053 : }
1054 :
1055 4 : bool hcclNslbDp::CheckSameOperatorVal(size_t operSize, NslbDpOperatorInfo& OperatorInfo, u32 rootRank)
1056 : {
1057 4 : size_t num = hcclNslbDpOperatorVal_.size();
1058 4 : if (operSize >= num) {
1059 1 : return false;
1060 : }
1061 3 : if (hcclNslbDpOperatorVal_[operSize].taskId == OperatorInfo.taskId
1062 3 : && hcclNslbDpOperatorVal_[operSize].rootRank == rootRank
1063 2 : && hcclNslbDpOperatorVal_[operSize].oper == OperatorInfo.oper
1064 2 : && hcclNslbDpOperatorVal_[operSize].algorithm == OperatorInfo.algorithm
1065 6 : && strcmp(OperatorInfo.commDesc, hcclNslbDpOperatorVal_[operSize].commDesc) == 0) {
1066 1 : return true;
1067 : }
1068 2 : return false;
1069 : }
1070 :
1071 : /* 写算法算子表--表2 */
1072 4 : HcclResult hcclNslbDp::GenerateOpAndAdjTable(
1073 : HcclCMDType opType, u32 rootRank, u32 srcLocalRankId, u8 algType, std::string identifier, u64 count, u32 rankSize)
1074 : {
1075 4 : HCCL_INFO("[NSLB-DP-OPER] count=[%llu], hcclNslbDpOperatorVal_ size:[%zu].", count, hcclNslbDpOperatorVal_.size());
1076 4 : u64 taskId = GetGlobalCommTaskId();
1077 4 : if (taskId == 0) {
1078 1 : return HCCL_SUCCESS;
1079 : }
1080 :
1081 : // 获取，0号
1082 3 : if (rootRank != 0) {
1083 1 : return HCCL_SUCCESS;
1084 : }
1085 2 : u64 trafficNum = count;
1086 2 : if (CheckSupportOptype(opType) == false) {
1087 0 : trafficNum = 0;
1088 0 : HCCL_DEBUG("[NSLB-DP-OPER] CheckSupportOptype false trafficNum == 0.");
1089 : }
1090 :
1091 2 : NslbDpOperatorInfo OperatorInfo = {};
1092 : // 获取task id
1093 2 : OperatorInfo.taskId = taskId;
1094 2 : (void)memset_s(OperatorInfo.commDesc, COMM_DESC_MAX_LENGTH, 0, sizeof(OperatorInfo.commDesc));
1095 :
1096 2 : fullcommDescInitTime(identifier, OperatorInfo);
1097 :
1098 : // 去除不存在的通信域信息
1099 2 : if (CheckCommDescExit(OperatorInfo) == false) {
1100 1 : HCCL_INFO("[NSLB-DP-OPER] CheckCommDesc not exit ");
1101 1 : return HCCL_SUCCESS;
1102 : }
1103 :
1104 : // 获取operator、algorithm
1105 1 : OperatorInfo.oper = GetNslbOpType(opType);
1106 1 : OperatorInfo.algorithm = algType;
1107 1 : u64 trafficCount = GetNslbDpFirstFourBit(OperatorInfo.oper, OperatorInfo.algorithm);
1108 1 : trafficCount = (trafficCount << NSLBDP_TRAFFICCONUT) + trafficNum;
1109 :
1110 1 : for (size_t operSize = 0; operSize < hcclNslbDpOperatorVal_.size(); operSize++) {
1111 0 : if (CheckSameOperatorVal(operSize, OperatorInfo, rootRank) == true) {
1112 0 : if (hcclNslbDpOperatorVal_[operSize].trafficCnt < trafficCount && srcLocalRankId == 0) {
1113 0 : hcclNslbDpOperatorVal_[operSize].trafficCnt = trafficCount;
1114 0 : hcclNslbDpOperatorVal_[operSize].sedFlag = 0;
1115 0 : SendRankTableOpAndAdj(hcclNslbDpOperatorVal_[operSize]);
1116 0 : HCCL_RUN_INFO(
1117 : "[NSLB-DP-ADJ] commDesc[%s] try to update trafficCnt[%llu] success.", OperatorInfo.commDesc,
1118 : trafficCount);
1119 : }
1120 0 : return HCCL_SUCCESS;
1121 : }
1122 : }
1123 :
1124 1 : OperatorInfo.trafficCnt = trafficCount; // 判断变大
1125 1 : OperatorInfo.rootRank = rootRank;
1126 1 : HCCL_INFO(
1127 : "[NSLB-DP-OPER] add operInfo:***[%llu]***[%llu]***[%u]***[%u]***[%u] success.", taskId,
1128 : OperatorInfo.commInitTime, rootRank, OperatorInfo.oper, OperatorInfo.algorithm);
1129 :
1130 1 : GetNslbDpl4SPortId(rankSize, algType, &OperatorInfo.l4SPortId);
1131 1 : if (srcLocalRankId == 0) {
1132 1 : SendRankTableOpAndAdj(OperatorInfo);
1133 : }
1134 1 : hcclNslbDpOperatorVal_.push_back(OperatorInfo);
1135 :
1136 1 : return HCCL_SUCCESS;
1137 : }
1138 :
1139 : /* 根将表1 序列化处理 */
1140 0 : std::vector<uint8_t> hcclNslbDp::serializeTLV_TableFir(NslbDpCommConfigInfo cominfo)
1141 : {
1142 0 : HCCL_DEBUG("[NSLB-DP] entry serializeTLV TBL_COMM_INFO beg");
1143 :
1144 0 : std::vector<uint8_t> tlvData;
1145 :
1146 : // Task ID
1147 0 : uint64_t netTaskId = htobe64(cominfo.taskId);
1148 0 : tlvData.resize(tlvData.size() + sizeof(netTaskId));
1149 0 : (void)memcpy_s(
1150 0 : tlvData.data() + tlvData.size() - sizeof(netTaskId), sizeof(netTaskId), &netTaskId, sizeof(netTaskId));
1151 :
1152 : // CommDesc
1153 0 : tlvData.insert(tlvData.end(), cominfo.commDesc, cominfo.commDesc + COMM_DESC_MAX_LENGTH);
1154 :
1155 : // CommInitTime
1156 0 : uint64_t netCommInitTime = htobe64(cominfo.commInitTime);
1157 0 : tlvData.resize(tlvData.size() + sizeof(netCommInitTime));
1158 0 : (void)memcpy_s(
1159 0 : tlvData.data() + tlvData.size() - sizeof(netCommInitTime), sizeof(netCommInitTime), &netCommInitTime,
1160 : sizeof(netCommInitTime));
1161 :
1162 : // packetId
1163 0 : uint16_t netPacketId = htons(cominfo.packetId);
1164 0 : tlvData.resize(tlvData.size() + sizeof(netPacketId));
1165 0 : (void)memcpy_s(
1166 0 : tlvData.data() + tlvData.size() - sizeof(netPacketId), sizeof(netPacketId), &netPacketId, sizeof(netPacketId));
1167 :
1168 : // rev
1169 0 : uint16_t netRev = htons(cominfo.rev);
1170 0 : tlvData.resize(tlvData.size() + sizeof(netRev));
1171 0 : (void)memcpy_s(tlvData.data() + tlvData.size() - sizeof(netRev), sizeof(netRev), &netRev, sizeof(netRev));
1172 :
1173 : // packetNum
1174 0 : uint16_t netPacketNum = htons(cominfo.packetNum);
1175 0 : tlvData.resize(tlvData.size() + sizeof(netPacketNum));
1176 0 : (void)memcpy_s(
1177 0 : tlvData.data() + tlvData.size() - sizeof(netPacketNum), sizeof(netPacketNum), &netPacketNum,
1178 : sizeof(netPacketNum));
1179 :
1180 : // revSecond
1181 0 : uint16_t netRevSecond = htons(cominfo.revSecond);
1182 0 : tlvData.resize(tlvData.size() + sizeof(netRevSecond));
1183 0 : (void)memcpy_s(
1184 0 : tlvData.data() + tlvData.size() - sizeof(netRevSecond), sizeof(netRevSecond), &netRevSecond,
1185 : sizeof(netRevSecond));
1186 :
1187 : // sendRankInfo
1188 0 : for (const auto& rankInfo : cominfo.sendRankInfo) {
1189 0 : uint32_t netDeviceIp = htonl(rankInfo.deviceIp);
1190 0 : tlvData.resize(tlvData.size() + sizeof(netDeviceIp));
1191 0 : (void)memcpy_s(
1192 0 : tlvData.data() + tlvData.size() - sizeof(netDeviceIp), sizeof(netDeviceIp), &netDeviceIp,
1193 : sizeof(netDeviceIp));
1194 : }
1195 :
1196 : // CommMd5Sum
1197 0 : tlvData.insert(tlvData.end(), cominfo.commMd5Sum, cominfo.commMd5Sum + sizeof(cominfo.commMd5Sum));
1198 :
1199 : // RankTotalNum
1200 0 : uint16_t netRankTotalNum = htons(cominfo.rankTotalNum);
1201 0 : tlvData.resize(tlvData.size() + sizeof(netRankTotalNum));
1202 0 : (void)memcpy_s(
1203 0 : tlvData.data() + tlvData.size() - sizeof(netRankTotalNum), sizeof(netRankTotalNum), &netRankTotalNum,
1204 : sizeof(netRankTotalNum));
1205 :
1206 : // Rank Number
1207 0 : uint16_t netRankNum = htons(cominfo.rankNum);
1208 0 : tlvData.resize(tlvData.size() + sizeof(netRankNum));
1209 0 : (void)memcpy_s(
1210 0 : tlvData.data() + tlvData.size() - sizeof(netRankNum), sizeof(netRankNum), &netRankNum, sizeof(netRankNum));
1211 :
1212 : // Rank Info
1213 0 : for (const auto& rankInfo : cominfo.rankInfo) {
1214 0 : uint32_t netDeviceIp = htonl(rankInfo.deviceIp);
1215 0 : uint32_t netServerIp = htonl(rankInfo.serverIp);
1216 0 : uint16_t netPodId = htons(rankInfo.podId);
1217 0 : uint16_t netRev3 = htons(rankInfo.rev);
1218 0 : tlvData.resize(tlvData.size() + sizeof(netDeviceIp));
1219 0 : (void)memcpy_s(
1220 0 : tlvData.data() + tlvData.size() - sizeof(netDeviceIp), sizeof(netDeviceIp), &netDeviceIp,
1221 : sizeof(netDeviceIp));
1222 0 : tlvData.resize(tlvData.size() + sizeof(netServerIp));
1223 0 : (void)memcpy_s(
1224 0 : tlvData.data() + tlvData.size() - sizeof(netServerIp), sizeof(netServerIp), &netServerIp,
1225 : sizeof(netServerIp));
1226 0 : tlvData.resize(tlvData.size() + sizeof(netPodId));
1227 0 : (void)memcpy_s(
1228 0 : tlvData.data() + tlvData.size() - sizeof(netPodId), sizeof(netPodId), &netPodId, sizeof(netPodId));
1229 0 : tlvData.resize(tlvData.size() + sizeof(netRev3));
1230 0 : (void)memcpy_s(tlvData.data() + tlvData.size() - sizeof(netRev3), sizeof(netRev3), &netRev3, sizeof(netRev3));
1231 : }
1232 0 : HCCL_DEBUG("[NSLB-DP] entry serializeTLV TBL_COMM_INFO end");
1233 :
1234 0 : return tlvData;
1235 0 : }
1236 :
1237 12 : void hcclNslbDp::fullCommConfigInfo(NslbDpCommConfigInfo& tab_f, NslbDpCommConfigVal cominfo, u32 packetNum)
1238 : {
1239 12 : HCCL_DEBUG("[NSLB-DP] entry full CommConfigInfo");
1240 12 : tab_f.taskId = cominfo.taskId;
1241 12 : s32 sRet = memcpy_s(tab_f.commDesc, sizeof(tab_f.commDesc), cominfo.commDesc, COMM_DESC_MAX_LENGTH);
1242 12 : if (sRet != EOK) {
1243 0 : HCCL_INFO("memcpy_s commDesc fail");
1244 : }
1245 12 : tab_f.commInitTime = cominfo.commInitTime;
1246 :
1247 12 : tab_f.rev = 0;
1248 12 : tab_f.packetNum = packetNum;
1249 12 : tab_f.revSecond = 0;
1250 12 : u32 sendCnt = std::min(packetNum, static_cast<u32>(cominfo.rankInfo.size()));
1251 48 : for (u32 dip = 0; dip < sendCnt; dip++) {
1252 36 : tab_f.sendRankInfo[dip].deviceIp = cominfo.rankInfo[dip].deviceIp;
1253 : }
1254 12 : sRet = memcpy_s(tab_f.commMd5Sum, sizeof(tab_f.commMd5Sum), cominfo.commMd5Sum, sizeof(cominfo.commMd5Sum));
1255 12 : if (sRet != EOK) {
1256 0 : HCCL_INFO("memcpy_s commmd5 fail");
1257 : }
1258 12 : }
1259 :
1260 : /* 根据表1 中的rank数量进行分片处理 */
1261 7 : HcclResult hcclNslbDp::SendTableProc(u32 rank, u32 packetNum, NslbDpCommConfigVal cominfo)
1262 : {
1263 7 : if (packetNum - 1 < rank) {
1264 1 : HCCL_INFO("[NSLB-DP] rank [%u] no need send TableFir(packetNum[%u]).", rank, packetNum);
1265 1 : return HCCL_SUCCESS;
1266 : }
1267 6 : u32 packetIndex = rank;
1268 6 : NslbDpCommConfigInfo tab_f = {};
1269 6 : fullCommConfigInfo(tab_f, cominfo, packetNum);
1270 6 : tab_f.packetId = packetIndex;
1271 :
1272 6 : u32 start = packetIndex * NSLBDP_RANKTOTALNUM_BLOCK_FIR;
1273 6 : u32 end = (packetIndex + 1) * NSLBDP_RANKTOTALNUM_BLOCK_FIR;
1274 6 : u32 totalSize = static_cast<u32>(cominfo.rankInfo.size());
1275 6 : if (totalSize < start) {
1276 1 : HCCL_INFO("[NSLB-DP] Comm RankInfo not as expected, size[%u] < start[%u]", totalSize, start);
1277 1 : return HCCL_SUCCESS;
1278 : }
1279 5 : end = std::min(end, totalSize);
1280 :
1281 5 : u32 count = end - start;
1282 5 : tab_f.rankInfo.reserve(count);
1283 3078 : for (u32 i = start; i < end; i++) {
1284 3073 : tab_f.rankInfo.push_back(cominfo.rankInfo[i]);
1285 : }
1286 5 : tab_f.rankTotalNum = cominfo.rankTotalNum;
1287 5 : tab_f.rankNum = static_cast<u16>(tab_f.rankInfo.size());
1288 :
1289 5 : const char* tag = (packetIndex != packetNum - 1) ? "SendTableProc-F" : "SendTableProc-N";
1290 5 : HCCL_INFO(
1291 : "[NSLB-DP] %s info:[%u]-[%u]-[%u]-[%u].", tag, packetNum, packetIndex, cominfo.rankTotalNum,
1292 : tab_f.rankInfo.size());
1293 5 : HCCL_DEBUG(
1294 : "[NSLB-DP] SendRankTable-info:[%u]-[%u]-[%u]-[%u].", tab_f.rankTotalNum, tab_f.packetNum, tab_f.rankTotalNum,
1295 : tab_f.rankInfo.size());
1296 5 : SendRankTable(tab_f);
1297 :
1298 5 : return HCCL_SUCCESS;
1299 6 : }
1300 :
1301 : /* 遍历表1 执行send 流程 */
1302 1 : HcclResult hcclNslbDp::SendTableFir(uint32_t rank)
1303 : {
1304 1 : size_t size = hcclNslbDpCommConfig_.size();
1305 1 : HCCL_INFO("[NSLB-DP] SendTableFir size:[%u] success.", size);
1306 2 : for (size_t i = 0; i < hcclNslbDpCommConfig_.size(); i++) {
1307 1 : u32 rankTotalNum = hcclNslbDpCommConfig_[i].rankTotalNum;
1308 1 : if (rankTotalNum > NSLBDP_RANKTOTALNUM_BLOCK_FOU) {
1309 0 : return HCCL_SUCCESS;
1310 : }
1311 1 : u32 packetNum = CalcPacketNum(rankTotalNum);
1312 1 : SendTableProc(rank, packetNum, hcclNslbDpCommConfig_[i]);
1313 : }
1314 1 : HCCL_DEBUG("[NSLB-DP] entry SendTableFir end");
1315 1 : return HCCL_SUCCESS;
1316 : }
1317 :
1318 5 : HcclResult hcclNslbDp::SendRankTable(NslbDpCommConfigInfo tab_f)
1319 : {
1320 5 : if (GetInitNetCoFlag() == false) {
1321 5 : return HCCL_SUCCESS;
1322 : }
1323 :
1324 0 : u32 tablen = sizeof(tab_f);
1325 0 : HCCL_INFO("[NSLB-DP] SendRankTable NslbDpCommConfigInfo len:[%u].", tablen);
1326 :
1327 0 : std::vector<uint8_t> tlvData = serializeTLV_TableFir(tab_f);
1328 0 : u32 datlen = tlvData.size();
1329 0 : HCCL_INFO("[NSLB-DP] SendRankTable tlvData.len:[%u] success.", datlen);
1330 :
1331 0 : if (nslbdp_handle_ == nullptr) {
1332 0 : HCCL_INFO("[NSLB-DP] ndlbdp nslbdp_handle_ error SendRankTable.");
1333 0 : return HCCL_SUCCESS;
1334 : }
1335 :
1336 0 : nslb_msg sendMsg;
1337 0 : nslb_msg recvMsg;
1338 0 : sendMsg.type = NSLBDP_TYPE_TBL_COMM_INFO;
1339 0 : sendMsg.length = datlen;
1340 0 : sendMsg.data.assign(tlvData.begin(), tlvData.end());
1341 0 : s32 ret = H2DTlvRequest(
1342 0 : nslbdp_handle_, MODULE_TYPE_NSLB, reinterpret_cast<TlvMsg*>(&sendMsg), reinterpret_cast<TlvMsg*>(&recvMsg));
1343 :
1344 0 : HCCL_INFO("[NSLBDP-SENDTABLE] hccl send table NSLBDP_TYPE_TBL_COMM_INFO(1001) to hccp. ret(%d)\n", ret);
1345 0 : return HCCL_SUCCESS;
1346 0 : }
1347 :
1348 : /* 根将表2 序列化处理 */
1349 2 : std::vector<uint8_t> hcclNslbDp::serializeTLV_TableOpAndAdj(NslbDpOperatorInfo& info)
1350 : {
1351 2 : HCCL_DEBUG("[NSLB-DP] ndlbdp entry serializeTLV TableOpAndAdj.");
1352 2 : std::vector<uint8_t> tlvData;
1353 :
1354 : // 处理 taskId
1355 2 : uint64_t netTaskId = htobe64(info.taskId);
1356 2 : tlvData.resize(tlvData.size() + sizeof(netTaskId));
1357 2 : (void)memcpy_s(
1358 2 : tlvData.data() + tlvData.size() - sizeof(netTaskId), sizeof(netTaskId), &netTaskId, sizeof(netTaskId));
1359 :
1360 : // 处理 commDesc
1361 2 : tlvData.insert(tlvData.end(), info.commDesc, info.commDesc + COMM_DESC_MAX_LENGTH);
1362 :
1363 : // 处理 commInitTime
1364 2 : uint64_t netCommInitTime = htobe64(info.commInitTime);
1365 2 : tlvData.resize(tlvData.size() + sizeof(netCommInitTime));
1366 2 : (void)memcpy_s(
1367 2 : tlvData.data() + tlvData.size() - sizeof(netCommInitTime), sizeof(netCommInitTime), &netCommInitTime,
1368 : sizeof(netCommInitTime));
1369 :
1370 : // 处理 oper
1371 2 : tlvData.push_back(info.oper);
1372 :
1373 : // 处理 algorithm
1374 2 : tlvData.push_back(info.algorithm);
1375 :
1376 : // 处理 rootRank
1377 2 : uint16_t netRootRank = htons(info.rootRank);
1378 2 : tlvData.resize(tlvData.size() + sizeof(netRootRank));
1379 2 : (void)memcpy_s(
1380 2 : tlvData.data() + tlvData.size() - sizeof(netRootRank), sizeof(netRootRank), &netRootRank, sizeof(netRootRank));
1381 :
1382 : // 处理 trafficCnt
1383 2 : uint64_t netTrafficCnt = htobe64(info.trafficCnt);
1384 2 : tlvData.resize(tlvData.size() + sizeof(netTrafficCnt));
1385 2 : (void)memcpy_s(
1386 2 : tlvData.data() + tlvData.size() - sizeof(netTrafficCnt), sizeof(netTrafficCnt), &netTrafficCnt,
1387 : sizeof(netTrafficCnt));
1388 :
1389 : // 处理 l4SPortId
1390 2 : uint16_t netL4SPortId = htons(info.l4SPortId);
1391 2 : tlvData.resize(tlvData.size() + sizeof(netL4SPortId));
1392 2 : (void)memcpy_s(
1393 2 : tlvData.data() + tlvData.size() - sizeof(netL4SPortId), sizeof(netL4SPortId), &netL4SPortId,
1394 : sizeof(netL4SPortId));
1395 :
1396 : // 处理 maskLen
1397 2 : uint16_t netMaskLen = htons(info.maskLen);
1398 2 : tlvData.resize(tlvData.size() + sizeof(netMaskLen));
1399 2 : (void)memcpy_s(
1400 2 : tlvData.data() + tlvData.size() - sizeof(netMaskLen), sizeof(netMaskLen), &netMaskLen, sizeof(netMaskLen));
1401 2 : HCCL_INFO("[NSLB-DP] entry serializeTLV_TableOpAndAdj end.");
1402 2 : return tlvData;
1403 0 : }
1404 :
1405 : /* 执行send 流程 */
1406 2 : HcclResult hcclNslbDp::SendRankTableOpAndAdj(NslbDpOperatorInfo& tab_f)
1407 : {
1408 2 : if (GetInitNetCoFlag() == false) {
1409 1 : return HCCL_SUCCESS;
1410 : }
1411 1 : HCCL_DEBUG("[NSLB-DP] ndlbdp entry SendRankTableOpAndAdj.");
1412 1 : std::vector<uint8_t> tlvData = serializeTLV_TableOpAndAdj(tab_f);
1413 :
1414 1 : u32 datlen = tlvData.size();
1415 1 : HCCL_INFO("[NSLB-DP] SendRankTableOpAndAdj tlvData.len:[%u] success.", datlen);
1416 :
1417 1 : nslb_msg sendMsg;
1418 1 : nslb_msg recvMsg;
1419 1 : sendMsg.type = NSLBDP_TYPE_TBL_OPER;
1420 1 : sendMsg.length = datlen;
1421 1 : sendMsg.data.assign(tlvData.begin(), tlvData.end());
1422 1 : s32 ret = H2DTlvRequest(
1423 1 : nslbdp_handle_, MODULE_TYPE_NSLB, reinterpret_cast<TlvMsg*>(&sendMsg), reinterpret_cast<TlvMsg*>(&recvMsg));
1424 :
1425 1 : HCCL_INFO("[NSLBDP-SENDTABLE] hccl send table NSLBDP_TYPE_TBL_OPER(1002) to hccp. ret(%d)\n", ret);
1426 :
1427 1 : return HCCL_SUCCESS;
1428 1 : }
1429 :
1430 : /* 遍历表2 执行send 流程 */
1431 1 : HcclResult hcclNslbDp::SendOpAndAdjTable()
1432 : {
1433 1 : HCCL_DEBUG("[NSLB-DP] ndlbdp entry SendOpAndAdjTable.");
1434 2 : for (size_t i = 0; i < hcclNslbDpOperatorVal_.size(); i++) {
1435 1 : if (hcclNslbDpOperatorVal_[i].sedFlag == 1) {
1436 0 : continue;
1437 : }
1438 1 : NslbDpOperatorInfo tab_f = {};
1439 1 : tab_f.taskId = hcclNslbDpOperatorVal_[i].taskId;
1440 2 : s32 sRet = memcpy_s(
1441 1 : tab_f.commDesc, sizeof(tab_f.commDesc), hcclNslbDpOperatorVal_[i].commDesc, COMM_DESC_MAX_LENGTH);
1442 1 : if (sRet != EOK) {
1443 0 : HCCL_ERROR("memcpy_s commDesc fail");
1444 : }
1445 1 : tab_f.commInitTime = hcclNslbDpOperatorVal_[i].commInitTime;
1446 1 : tab_f.rootRank = hcclNslbDpOperatorVal_[i].rootRank;
1447 1 : tab_f.oper = hcclNslbDpOperatorVal_[i].oper;
1448 1 : tab_f.algorithm = hcclNslbDpOperatorVal_[i].algorithm;
1449 1 : tab_f.trafficCnt = hcclNslbDpOperatorVal_[i].trafficCnt;
1450 1 : tab_f.l4SPortId = hcclNslbDpOperatorVal_[i].l4SPortId;
1451 1 : tab_f.maskLen = hcclNslbDpOperatorVal_[i].maskLen;
1452 1 : SendRankTableOpAndAdj(tab_f);
1453 1 : hcclNslbDpOperatorVal_[i].sedFlag = 1;
1454 1 : HCCL_INFO("[NSLB-DP] try to sen RankTableOpAndAdj times:[%u].", i);
1455 : }
1456 1 : HCCL_INFO("[NSLB-DP] SendOpAndAdjTable end.");
1457 1 : return HCCL_SUCCESS;
1458 : }
1459 :
1460 : /* 根将表3 序列化处理 */
1461 2 : std::vector<uint8_t> hcclNslbDp::serializeTLV_TableAlgorithmInfo(NslbDpAlgorithmTlv& info)
1462 : {
1463 2 : HCCL_DEBUG("[NSLB-DP] entry serializeTLV_TableAlgorithmInfo.");
1464 2 : std::vector<uint8_t> tlvData;
1465 :
1466 : // 处理 taskId
1467 2 : uint64_t netTaskId = htobe64(info.taskId);
1468 2 : tlvData.resize(tlvData.size() + sizeof(netTaskId));
1469 2 : (void)memcpy_s(
1470 2 : tlvData.data() + tlvData.size() - sizeof(netTaskId), sizeof(netTaskId), &netTaskId, sizeof(netTaskId));
1471 :
1472 : // 处理 commDesc
1473 2 : tlvData.insert(tlvData.end(), info.commDesc, info.commDesc + COMM_DESC_MAX_LENGTH);
1474 :
1475 : // CommMd5Sum
1476 2 : tlvData.insert(tlvData.end(), info.commMd5Sum, info.commMd5Sum + sizeof(info.commMd5Sum));
1477 :
1478 : // 处理 srcLocalRankId
1479 2 : uint16_t netSrcLocalRankId = htons(info.srcLocalRankId);
1480 2 : tlvData.resize(tlvData.size() + sizeof(netSrcLocalRankId));
1481 2 : (void)memcpy_s(
1482 2 : tlvData.data() + tlvData.size() - sizeof(netSrcLocalRankId), sizeof(netSrcLocalRankId), &netSrcLocalRankId,
1483 : sizeof(netSrcLocalRankId));
1484 :
1485 : // 处理 oper
1486 2 : tlvData.push_back(info.oper);
1487 :
1488 : // 处理 algorithm
1489 2 : tlvData.push_back(info.algorithm);
1490 :
1491 : // 处理 rootRank
1492 2 : uint16_t netRootRank = htons(info.rootRank);
1493 2 : tlvData.resize(tlvData.size() + sizeof(netRootRank));
1494 2 : (void)memcpy_s(
1495 2 : tlvData.data() + tlvData.size() - sizeof(netRootRank), sizeof(netRootRank), &netRootRank, sizeof(netRootRank));
1496 :
1497 : // 处理 rev
1498 2 : uint16_t netRev = htons(info.rev);
1499 2 : tlvData.resize(tlvData.size() + sizeof(netRev));
1500 2 : (void)memcpy_s(tlvData.data() + tlvData.size() - sizeof(netRev), sizeof(netRev), &netRev, sizeof(netRev));
1501 :
1502 : // 处理 dstRankNum
1503 2 : uint16_t netDstRankNum = htons(info.dstRankNum);
1504 2 : tlvData.resize(tlvData.size() + sizeof(netDstRankNum));
1505 2 : (void)memcpy_s(
1506 2 : tlvData.data() + tlvData.size() - sizeof(netDstRankNum), sizeof(netDstRankNum), &netDstRankNum,
1507 : sizeof(netDstRankNum));
1508 :
1509 : // 处理 revSecond
1510 2 : uint16_t netRevSecond = htons(info.revsecond);
1511 2 : tlvData.resize(tlvData.size() + sizeof(netRevSecond));
1512 2 : (void)memcpy_s(
1513 2 : tlvData.data() + tlvData.size() - sizeof(netRevSecond), sizeof(netRevSecond), &netRevSecond,
1514 : sizeof(netRevSecond));
1515 :
1516 2 : HCCL_INFO("[NSLB-DP] serializeTLV_TableAlgorithmInfo proc adjInfo.");
1517 : // 处理 adjInfo
1518 5 : for (const auto& adj : info.AdjInfo) {
1519 3 : uint16_t netDstLocalRankId = htons(adj.dstLocalRankId);
1520 3 : tlvData.resize(tlvData.size() + sizeof(netDstLocalRankId));
1521 3 : (void)memcpy_s(
1522 3 : tlvData.data() + tlvData.size() - sizeof(netDstLocalRankId), sizeof(netDstLocalRankId), &netDstLocalRankId,
1523 : sizeof(netDstLocalRankId));
1524 3 : tlvData.push_back(adj.phaseId);
1525 3 : tlvData.push_back(adj.rev);
1526 3 : HCCL_INFO("[NSLB-DP] serializeTLV_TableAlgo adjInfo:***[%u]***[%c]***.", adj.dstLocalRankId, adj.phaseId);
1527 : }
1528 2 : HCCL_DEBUG("[NSLB-DP] serializeTLV_TableAlgo proc adjInfo end.");
1529 :
1530 2 : return tlvData;
1531 0 : }
1532 :
1533 1 : HcclResult hcclNslbDp::SendRankTableAlgorithmInfo(NslbDpAlgorithmTlv& tab_f)
1534 : {
1535 1 : if (GetInitNetCoFlag() == false) {
1536 0 : return HCCL_SUCCESS;
1537 : }
1538 1 : std::vector<uint8_t> tlvData = serializeTLV_TableAlgorithmInfo(tab_f);
1539 :
1540 1 : u32 datlen = tlvData.size();
1541 1 : HCCL_INFO("[NSLB-DP] SendRankTableAlgorithmInfo tlvData.len:[%u] success.", datlen);
1542 :
1543 1 : nslb_msg sendMsg;
1544 1 : nslb_msg recvMsg;
1545 1 : sendMsg.type = NSLBDP_TYPE_TBL_ADJ;
1546 1 : sendMsg.length = datlen;
1547 1 : sendMsg.data.assign(tlvData.begin(), tlvData.end());
1548 1 : s32 ret = H2DTlvRequest(
1549 1 : nslbdp_handle_, MODULE_TYPE_NSLB, reinterpret_cast<TlvMsg*>(&sendMsg), reinterpret_cast<TlvMsg*>(&recvMsg));
1550 :
1551 1 : HCCL_INFO("[NSLBDP-SENDTABLE] hccl send table NSLBDP_TYPE_TBL_ADJ(1003) to hccp. ret(%d)\n", ret);
1552 1 : return HCCL_SUCCESS;
1553 1 : }
1554 :
1555 : /* 遍历表3 执行send 流程 */
1556 1 : HcclResult hcclNslbDp::SendAlgorithmInfoTable()
1557 : {
1558 1 : u32 size = hcclNslbDpAlgorithmInfo_.size();
1559 1 : HCCL_INFO("[NSLB-DP] ndlbdp entry SendAlgorithmInfoTable size=[%u].", size);
1560 :
1561 2 : for (size_t i = 0; i < hcclNslbDpAlgorithmInfo_.size(); i++) {
1562 1 : if (hcclNslbDpAlgorithmInfo_[i].sedFlag == 1) {
1563 0 : continue;
1564 : }
1565 1 : NslbDpAlgorithmTlv tab_f = {};
1566 1 : tab_f.taskId = hcclNslbDpAlgorithmInfo_[i].taskId;
1567 2 : s32 sRet = memcpy_s(
1568 1 : tab_f.commDesc, sizeof(tab_f.commDesc), hcclNslbDpAlgorithmInfo_[i].commDesc, COMM_DESC_MAX_LENGTH);
1569 2 : sRet = memcpy_s(
1570 1 : tab_f.commMd5Sum, sizeof(tab_f.commMd5Sum), hcclNslbDpAlgorithmInfo_[i].commMd5Sum,
1571 : sizeof(hcclNslbDpAlgorithmInfo_[i].commMd5Sum));
1572 1 : if (sRet != EOK) {
1573 0 : HCCL_INFO("memcpy_s commDesc fail");
1574 : }
1575 1 : tab_f.srcLocalRankId = hcclNslbDpAlgorithmInfo_[i].srcLocalRankId;
1576 1 : tab_f.rootRank = hcclNslbDpAlgorithmInfo_[i].rootRank;
1577 1 : tab_f.oper = hcclNslbDpAlgorithmInfo_[i].oper;
1578 1 : tab_f.algorithm = hcclNslbDpAlgorithmInfo_[i].algorithm;
1579 :
1580 1 : tab_f.rev = 0;
1581 :
1582 1 : tab_f.dstRankNum = hcclNslbDpAlgorithmInfo_[i].dstRankNum;
1583 1 : tab_f.revsecond = 0;
1584 1 : tab_f.AdjInfo = hcclNslbDpAlgorithmInfo_[i].AdjInfo;
1585 :
1586 1 : SendRankTableAlgorithmInfo(tab_f);
1587 1 : HCCL_INFO("[NSLB-DP] try to sen AlgorithmInfoTable times:[%u].", i);
1588 :
1589 1 : hcclNslbDpAlgorithmInfo_[i].sedFlag = 1;
1590 1 : }
1591 1 : HCCL_DEBUG("[NSLB-DP] entry SendAlgorithmInfoTable end.");
1592 :
1593 1 : return HCCL_SUCCESS;
1594 : }
1595 :
1596 : /* 序列化表4 */
1597 0 : std::vector<uint8_t> hcclNslbDp::serializeTLV_TableGlobalRankInfo(NslbDpGlobalRankInfo& info)
1598 : {
1599 0 : HCCL_DEBUG("[NSLB-DP] entry serializeTLV_TableGlobalRankInfo.");
1600 0 : std::vector<uint8_t> tlvData;
1601 : // 处理 taskId
1602 0 : uint64_t netTaskId = htobe64(info.taskId);
1603 0 : tlvData.resize(tlvData.size() + sizeof(netTaskId));
1604 0 : (void)memcpy_s(
1605 0 : tlvData.data() + tlvData.size() - sizeof(netTaskId), sizeof(netTaskId), &netTaskId, sizeof(netTaskId));
1606 :
1607 : // 处理 commDesc
1608 0 : tlvData.insert(tlvData.end(), info.commDesc, info.commDesc + COMM_DESC_MAX_LENGTH);
1609 :
1610 : // 处理 commInitTime
1611 0 : uint64_t netCommInitTime = htobe64(info.commInitTime);
1612 0 : tlvData.resize(tlvData.size() + sizeof(netCommInitTime));
1613 0 : (void)memcpy_s(
1614 0 : tlvData.data() + tlvData.size() - sizeof(netCommInitTime), sizeof(netCommInitTime), &netCommInitTime,
1615 : sizeof(netCommInitTime));
1616 :
1617 : // 处理 packetId
1618 0 : uint16_t netPacketId = htons(info.packetId);
1619 0 : tlvData.resize(tlvData.size() + sizeof(netPacketId));
1620 0 : (void)memcpy_s(
1621 0 : tlvData.data() + tlvData.size() - sizeof(netPacketId), sizeof(netPacketId), &netPacketId, sizeof(netPacketId));
1622 :
1623 : // 处理 rev
1624 0 : uint16_t netRev = htons(info.rev);
1625 0 : tlvData.resize(tlvData.size() + sizeof(netRev));
1626 0 : (void)memcpy_s(tlvData.data() + tlvData.size() - sizeof(netRev), sizeof(netRev), &netRev, sizeof(netRev));
1627 :
1628 : // 处理 packetNum
1629 0 : uint16_t netPacketNum = htons(info.packetNum);
1630 0 : tlvData.resize(tlvData.size() + sizeof(netPacketNum));
1631 0 : (void)memcpy_s(
1632 0 : tlvData.data() + tlvData.size() - sizeof(netPacketNum), sizeof(netPacketNum), &netPacketNum,
1633 : sizeof(netPacketNum));
1634 0 : HCCL_INFO("[NSLB-DP] hcclNslbDpGlobalRankVal_ info.packetNum:[%u]", netPacketNum);
1635 :
1636 : // 处理 rev2
1637 0 : uint16_t netRev2 = htons(info.rev2);
1638 0 : tlvData.resize(tlvData.size() + sizeof(netRev2));
1639 0 : (void)memcpy_s(tlvData.data() + tlvData.size() - sizeof(netRev2), sizeof(netRev2), &netRev2, sizeof(netRev2));
1640 :
1641 : // 处理 sendRankInfo
1642 0 : for (const auto& rankInfo : info.sendRankInfo) {
1643 0 : uint32_t netDeviceIp = htonl(rankInfo.deviceIp);
1644 0 : tlvData.resize(tlvData.size() + sizeof(netDeviceIp));
1645 0 : (void)memcpy_s(
1646 0 : tlvData.data() + tlvData.size() - sizeof(netDeviceIp), sizeof(netDeviceIp), &netDeviceIp,
1647 : sizeof(netDeviceIp));
1648 : }
1649 :
1650 : // 处理 commMd5Sum
1651 0 : tlvData.insert(tlvData.end(), info.commMd5Sum, info.commMd5Sum + sizeof(info.commMd5Sum));
1652 :
1653 : // 处理 rankTotalNum
1654 0 : uint32_t netRankTotalNum = htonl(info.rankTotalNum);
1655 0 : tlvData.resize(tlvData.size() + sizeof(netRankTotalNum));
1656 0 : (void)memcpy_s(
1657 0 : tlvData.data() + tlvData.size() - sizeof(netRankTotalNum), sizeof(netRankTotalNum), &netRankTotalNum,
1658 : sizeof(netRankTotalNum));
1659 :
1660 : // 处理 rankNum
1661 0 : uint16_t netRankNum = htons(info.rankNum);
1662 0 : tlvData.resize(tlvData.size() + sizeof(netRankNum));
1663 0 : (void)memcpy_s(
1664 0 : tlvData.data() + tlvData.size() - sizeof(netRankNum), sizeof(netRankNum), &netRankNum, sizeof(netRankNum));
1665 :
1666 : // 处理 rev3
1667 0 : uint16_t netRev3 = htons(info.rev3);
1668 0 : tlvData.resize(tlvData.size() + sizeof(netRev3));
1669 0 : (void)memcpy_s(tlvData.data() + tlvData.size() - sizeof(netRev3), sizeof(netRev3), &netRev3, sizeof(netRev3));
1670 :
1671 : // 处理 rankInfo
1672 0 : for (const auto& rankInfo : info.rankInfo) {
1673 0 : uint32_t netDeviceIp = htonl(rankInfo.deviceIp);
1674 0 : uint32_t netServerIp = htonl(rankInfo.serverIp);
1675 0 : tlvData.resize(tlvData.size() + sizeof(netDeviceIp));
1676 0 : (void)memcpy_s(
1677 0 : tlvData.data() + tlvData.size() - sizeof(netDeviceIp), sizeof(netDeviceIp), &netDeviceIp,
1678 : sizeof(netDeviceIp));
1679 0 : tlvData.resize(tlvData.size() + sizeof(netServerIp));
1680 0 : (void)memcpy_s(
1681 0 : tlvData.data() + tlvData.size() - sizeof(netServerIp), sizeof(netServerIp), &netServerIp,
1682 : sizeof(netServerIp));
1683 : }
1684 0 : return tlvData;
1685 0 : }
1686 :
1687 4 : HcclResult hcclNslbDp::SendRankTableGlobalRank(NslbDpGlobalRankInfo& tab_f)
1688 : {
1689 4 : if (GetInitNetCoFlag() == false) {
1690 4 : return HCCL_SUCCESS;
1691 : }
1692 0 : std::vector<uint8_t> tlvData = serializeTLV_TableGlobalRankInfo(tab_f);
1693 0 : u32 datlen = tlvData.size();
1694 0 : HCCL_INFO("[NSLB-DP] SendRankTableGlobalRank tlvData.len:[%u] success.", datlen);
1695 :
1696 0 : nslb_msg sendMsg;
1697 0 : nslb_msg recvMsg;
1698 0 : sendMsg.type = NSLBDP_TYPE_TBL_RANK;
1699 0 : sendMsg.length = datlen;
1700 0 : sendMsg.data.assign(tlvData.begin(), tlvData.end());
1701 0 : s32 ret = H2DTlvRequest(
1702 0 : nslbdp_handle_, MODULE_TYPE_NSLB, reinterpret_cast<TlvMsg*>(&sendMsg), reinterpret_cast<TlvMsg*>(&recvMsg));
1703 :
1704 0 : HCCL_INFO("[NSLBDP-SENDTABLE] hccl send table NSLBDP_TYPE_TBL_RANK(1004) to hccp. ret(%d)\n", ret);
1705 :
1706 0 : return HCCL_SUCCESS;
1707 0 : }
1708 :
1709 6 : void hcclNslbDp::fullCommonGlobalRankInfo(NslbDpGlobalRankInfo& tab_f, NslbDpGlobalRankVal& cominfo)
1710 : {
1711 6 : tab_f.rev = 0;
1712 6 : tab_f.rev2 = 0;
1713 6 : tab_f.rev3 = 0;
1714 6 : tab_f.taskId = cominfo.taskId;
1715 6 : s32 sRet = memcpy_s(tab_f.commDesc, sizeof(tab_f.commDesc), cominfo.commDesc, sizeof(cominfo.commDesc));
1716 6 : if (sRet != EOK) {
1717 0 : HCCL_ERROR("memcpy_s commDesc fail");
1718 : }
1719 6 : tab_f.commInitTime = cominfo.commInitTime;
1720 6 : sRet = memcpy_s(tab_f.commMd5Sum, sizeof(tab_f.commMd5Sum), cominfo.commMd5Sum, sizeof(cominfo.commMd5Sum));
1721 6 : if (sRet != EOK) {
1722 0 : HCCL_ERROR("memcpy_s commMD5 fail");
1723 : }
1724 : /* 当前函数仅仅用于发送流程，在处理packetNum时需要根据rankNum进行处理，不能超过rankNum的范围，packetNum值由赋值处确定
1725 : */
1726 6 : u32 sendCnt = std::min(static_cast<u32>(tab_f.packetNum), static_cast<u32>(cominfo.rankInfo.size()));
1727 20 : for (u32 dip = 0; dip < sendCnt; dip++) {
1728 14 : tab_f.sendRankInfo[dip].deviceIp = cominfo.rankInfo[dip].deviceIp;
1729 : }
1730 6 : return;
1731 : }
1732 :
1733 : /* 根据表4 中的rank数量进行分片处理 */
1734 5 : HcclResult hcclNslbDp::SendTableGlobalRankProc(uint32_t rank, uint32_t packetNum, NslbDpGlobalRankVal& cominfo)
1735 : {
1736 5 : HCCL_INFO("[NSLB-DP] entry SendTableGlobalRankProc packetNum:[%u]", packetNum);
1737 5 : if (packetNum - 1 < rank) {
1738 1 : HCCL_INFO("[NSLB-DP] rank [%u] no need send TableFir(packetNum[%u]).", rank, packetNum);
1739 1 : return HCCL_SUCCESS;
1740 : }
1741 4 : u32 packetIndex = rank;
1742 4 : NslbDpGlobalRankInfo tab_f = {};
1743 4 : tab_f.packetId = packetIndex;
1744 4 : tab_f.packetNum = packetNum;
1745 4 : fullCommonGlobalRankInfo(tab_f, cominfo);
1746 :
1747 4 : uint32_t start = packetIndex * NSLBDP_RANKTOTALNUM_BLOCK_FIR;
1748 4 : uint32_t end = (packetIndex + 1) * NSLBDP_RANKTOTALNUM_BLOCK_FIR;
1749 4 : uint32_t totalSize = static_cast<uint32_t>(cominfo.rankInfo.size());
1750 4 : end = std::min(end, totalSize);
1751 4 : if (start >= end) {
1752 0 : HCCL_INFO("[NSLB-DP] GlobalRank RankInfo empty range[%u-%u], totalSize[%u]", start, end, totalSize);
1753 : } else {
1754 4 : tab_f.rankInfo.reserve(end - start);
1755 3121 : for (uint32_t j = start; j < end; ++j) {
1756 3117 : tab_f.rankInfo.push_back(cominfo.rankInfo[j]);
1757 : }
1758 : }
1759 4 : tab_f.rankTotalNum = cominfo.rankTotalNum;
1760 4 : tab_f.rankNum = static_cast<u16>(tab_f.rankInfo.size());
1761 4 : SendRankTableGlobalRank(tab_f);
1762 4 : return HCCL_SUCCESS;
1763 4 : }
1764 :
1765 : /* 根据rankNum的总数做分片发送流程 */
1766 2 : HcclResult hcclNslbDp::SendGlobalRankTable(uint32_t rank)
1767 : {
1768 2 : HCCL_DEBUG("[NSLB-DP] try to send GlobalRankTable.");
1769 2 : u32 rankTotalNum = hcclNslbDpGlobalRankVal_.rankTotalNum;
1770 2 : if (rankTotalNum > NSLBDP_RANKTOTALNUM_BLOCK_FOU) {
1771 1 : return HCCL_SUCCESS;
1772 : }
1773 1 : u32 packetNum = CalcPacketNum(rankTotalNum);
1774 1 : SendTableGlobalRankProc(rank, packetNum, hcclNslbDpGlobalRankVal_);
1775 :
1776 1 : return HCCL_SUCCESS;
1777 : }
1778 :
1779 : /* 序列化表5 */
1780 0 : std::vector<uint8_t> hcclNslbDp::serializeTLV_TableGlobalDisRankVal(NslbDpGlobalDisRankVal& info)
1781 : {
1782 0 : HCCL_DEBUG("[NSLB-DP] entry serializeTLV_TableGlobalDisRankVal.");
1783 0 : std::vector<uint8_t> tlvData;
1784 :
1785 : // 处理 taskId
1786 0 : uint64_t netTaskId = htobe64(info.taskId);
1787 0 : tlvData.resize(tlvData.size() + sizeof(netTaskId));
1788 0 : (void)memcpy_s(
1789 0 : tlvData.data() + tlvData.size() - sizeof(netTaskId), sizeof(netTaskId), &netTaskId, sizeof(netTaskId));
1790 :
1791 : // 处理 npuIp
1792 0 : uint32_t netNpuIp = htonl(info.npuIp);
1793 0 : tlvData.resize(tlvData.size() + sizeof(netNpuIp));
1794 0 : (void)memcpy_s(tlvData.data() + tlvData.size() - sizeof(netNpuIp), sizeof(netNpuIp), &netNpuIp, sizeof(netNpuIp));
1795 :
1796 : // 处理 serverIp
1797 0 : uint32_t netServerIp = htonl(info.serverIp);
1798 0 : tlvData.resize(tlvData.size() + sizeof(netServerIp));
1799 0 : (void)memcpy_s(
1800 0 : tlvData.data() + tlvData.size() - sizeof(netServerIp), sizeof(netServerIp), &netServerIp, sizeof(netServerIp));
1801 :
1802 : // 处理 nodeId
1803 0 : uint32_t netNodeId = htonl(info.nodeId);
1804 0 : tlvData.resize(tlvData.size() + sizeof(netNodeId));
1805 0 : (void)memcpy_s(
1806 0 : tlvData.data() + tlvData.size() - sizeof(netNodeId), sizeof(netNodeId), &netNodeId, sizeof(netNodeId));
1807 :
1808 : // 处理 localRankNum
1809 0 : tlvData.push_back(info.localRankNum);
1810 :
1811 : // 处理 rev
1812 0 : tlvData.insert(tlvData.end(), info.rev, info.rev + sizeof(info.rev));
1813 :
1814 : // 处理 rankTotalNum
1815 0 : uint32_t netRankTotalNum = htonl(info.rankTotalNum);
1816 0 : tlvData.resize(tlvData.size() + sizeof(netRankTotalNum));
1817 0 : (void)memcpy_s(
1818 0 : tlvData.data() + tlvData.size() - sizeof(netRankTotalNum), sizeof(netRankTotalNum), &netRankTotalNum,
1819 : sizeof(netRankTotalNum));
1820 :
1821 0 : return tlvData;
1822 0 : }
1823 :
1824 : /* 发送GlobalDisRank表信息 */
1825 0 : HcclResult hcclNslbDp::SendRankTableGlobalDisRankVal(NslbDpGlobalDisRankVal& tab_f)
1826 : {
1827 0 : if (GetInitNetCoFlag() == false) {
1828 0 : return HCCL_SUCCESS;
1829 : }
1830 0 : std::vector<uint8_t> tlvData = serializeTLV_TableGlobalDisRankVal(tab_f);
1831 0 : u32 datlen = tlvData.size();
1832 0 : HCCL_INFO("[NSLB-DP] SendRankTableGlobalDisRankVal tlvData.len:[%u] success.", datlen);
1833 :
1834 0 : nslb_msg sendMsg;
1835 0 : nslb_msg recvMsg;
1836 0 : sendMsg.type = NSLBDP_TYPE_TBL_RANK_DIST;
1837 0 : sendMsg.length = datlen;
1838 0 : sendMsg.data.assign(tlvData.begin(), tlvData.end());
1839 0 : s32 ret = H2DTlvRequest(
1840 0 : nslbdp_handle_, MODULE_TYPE_NSLB, reinterpret_cast<TlvMsg*>(&sendMsg), reinterpret_cast<TlvMsg*>(&recvMsg));
1841 :
1842 0 : HCCL_INFO("[NSLBDP-SENDTABLE] hccl send table NSLBDP_TYPE_TBL_RANK_DIST(1005) to hccp. ret(%d)\n", ret);
1843 0 : return HCCL_SUCCESS;
1844 0 : }
1845 :
1846 : /* 遍历表5 执行send流程 */
1847 0 : HcclResult hcclNslbDp::SendGlobalDisRankTable()
1848 : {
1849 0 : HCCL_DEBUG("[NSLB-DP] ndlbdp entry serializeTLV_TableGlobalDisRankVal.");
1850 0 : NslbDpGlobalDisRankVal tab_f = {};
1851 0 : tab_f.taskId = hcclNslbDpGlobalDisRankVal_.taskId;
1852 0 : tab_f.npuIp = hcclNslbDpGlobalDisRankVal_.npuIp;
1853 0 : tab_f.serverIp = hcclNslbDpGlobalDisRankVal_.serverIp;
1854 0 : tab_f.nodeId = hcclNslbDpGlobalDisRankVal_.nodeId;
1855 0 : tab_f.localRankNum = hcclNslbDpGlobalDisRankVal_.localRankNum;
1856 0 : s32 sRet = memcpy_s(
1857 0 : tab_f.rev, sizeof(tab_f.rev), hcclNslbDpGlobalDisRankVal_.rev, sizeof(hcclNslbDpGlobalDisRankVal_.rev));
1858 0 : if (sRet != EOK) {
1859 0 : HCCL_INFO("memcpy_s rev info fail");
1860 : }
1861 0 : tab_f.rankTotalNum = hcclNslbDpGlobalDisRankVal_.rankTotalNum;
1862 :
1863 0 : SendRankTableGlobalDisRankVal(tab_f);
1864 0 : return HCCL_SUCCESS;
1865 : }
1866 :
1867 : /* 序列化表6 */
1868 0 : std::vector<uint8_t> hcclNslbDp::serializeTLV_TableRootRank(NslbDpRootRank& config)
1869 : {
1870 0 : HCCL_DEBUG("[NSLB-DP] ndlbdp entry serializeTLV_TableRootRank.");
1871 0 : std::vector<uint8_t> tlvData;
1872 :
1873 : // 处理 taskId
1874 0 : uint64_t netTaskId = htobe64(config.taskId);
1875 0 : tlvData.resize(tlvData.size() + sizeof(netTaskId));
1876 0 : (void)memcpy_s(
1877 0 : tlvData.data() + tlvData.size() - sizeof(netTaskId), sizeof(netTaskId), &netTaskId, sizeof(netTaskId));
1878 :
1879 : // 处理 commDesc
1880 0 : tlvData.insert(tlvData.end(), config.commDesc, config.commDesc + COMM_DESC_MAX_LENGTH);
1881 :
1882 : // 处理 commInitTime
1883 0 : uint64_t netCommInitTime = htobe64(config.commInitTime);
1884 0 : tlvData.resize(tlvData.size() + sizeof(netCommInitTime));
1885 0 : (void)memcpy_s(
1886 0 : tlvData.data() + tlvData.size() - sizeof(netCommInitTime), sizeof(netCommInitTime), &netCommInitTime,
1887 : sizeof(netCommInitTime));
1888 :
1889 : // 处理 oper
1890 0 : tlvData.push_back(config.oper);
1891 :
1892 : // 处理 algorithm
1893 0 : tlvData.push_back(config.algorithm);
1894 :
1895 : // 处理 revfir
1896 0 : uint16_t netRevfir = htons(config.revfir);
1897 0 : tlvData.resize(tlvData.size() + sizeof(netRevfir));
1898 0 : (void)memcpy_s(
1899 0 : tlvData.data() + tlvData.size() - sizeof(netRevfir), sizeof(netRevfir), &netRevfir, sizeof(netRevfir));
1900 :
1901 : // 处理 rootRankNum
1902 0 : uint16_t netRootRankNum = htons(config.rootRankNum);
1903 0 : tlvData.resize(tlvData.size() + sizeof(netRootRankNum));
1904 0 : (void)memcpy_s(
1905 0 : tlvData.data() + tlvData.size() - sizeof(netRootRankNum), sizeof(netRootRankNum), &netRootRankNum,
1906 : sizeof(netRootRankNum));
1907 :
1908 : // 处理 revsec
1909 0 : uint16_t netRevsec = htons(config.revsec);
1910 0 : tlvData.resize(tlvData.size() + sizeof(netRevsec));
1911 0 : (void)memcpy_s(
1912 0 : tlvData.data() + tlvData.size() - sizeof(netRevsec), sizeof(netRevsec), &netRevsec, sizeof(netRevsec));
1913 :
1914 : // 处理 rankId
1915 0 : for (const auto& id : config.rankId) {
1916 0 : uint32_t netRankId = htonl(id.rankID);
1917 0 : tlvData.resize(tlvData.size() + sizeof(netRankId));
1918 0 : (void)memcpy_s(
1919 0 : tlvData.data() + tlvData.size() - sizeof(netRankId), sizeof(netRankId), &netRankId, sizeof(netRankId));
1920 : }
1921 0 : return tlvData;
1922 0 : }
1923 :
1924 : /* 发送Rank表到netco */
1925 0 : HcclResult hcclNslbDp::SendRankTableRootRank(NslbDpRootRank& tab_f)
1926 : {
1927 0 : if (GetInitNetCoFlag() == false) {
1928 0 : return HCCL_SUCCESS;
1929 : }
1930 0 : std::vector<uint8_t> tlvData = serializeTLV_TableRootRank(tab_f);
1931 :
1932 0 : u32 datlen = tlvData.size();
1933 0 : HCCL_DEBUG("[NSLB-DP] SendRankTableRootRank tlvData.len:[%u] success.", datlen);
1934 :
1935 0 : nslb_msg sendMsg;
1936 0 : nslb_msg recvMsg;
1937 0 : sendMsg.type = NSLBDP_TYPE_TBL_ROOT_RANK;
1938 0 : sendMsg.length = datlen;
1939 0 : sendMsg.data.assign(tlvData.begin(), tlvData.end());
1940 0 : s32 ret = H2DTlvRequest(
1941 0 : nslbdp_handle_, MODULE_TYPE_NSLB, reinterpret_cast<TlvMsg*>(&sendMsg), reinterpret_cast<TlvMsg*>(&recvMsg));
1942 :
1943 0 : HCCL_INFO("[NSLBDP-SENDTABLE] hccl send table NSLBDP_TYPE_TBL_ROOT_RANK(1006) to hccp. ret(%d)\n", ret);
1944 0 : return HCCL_SUCCESS;
1945 0 : }
1946 :
1947 : /* 遍历表6 执行send流程 */
1948 0 : HcclResult hcclNslbDp::SendRootRankTable()
1949 : {
1950 0 : HCCL_DEBUG("[NSLB-DP] entry SendRootRankTable.");
1951 0 : NslbDpRootRank tab_f = {};
1952 0 : tab_f.taskId = hcclNslbDpRootRankVal_.taskId;
1953 0 : s32 sRet = memcpy_s(
1954 0 : tab_f.commDesc, sizeof(tab_f.commDesc), hcclNslbDpRootRankVal_.commDesc,
1955 : sizeof(hcclNslbDpRootRankVal_.commDesc));
1956 0 : if (sRet != EOK) {
1957 0 : HCCL_INFO("memcpy_s commDesc info fail");
1958 : }
1959 0 : tab_f.commInitTime = hcclNslbDpRootRankVal_.commInitTime;
1960 0 : tab_f.oper = hcclNslbDpRootRankVal_.oper;
1961 0 : tab_f.algorithm = hcclNslbDpRootRankVal_.algorithm;
1962 0 : tab_f.revfir = hcclNslbDpRootRankVal_.revfir;
1963 0 : tab_f.rootRankNum = hcclNslbDpRootRankVal_.rootRankNum;
1964 0 : tab_f.revsec = hcclNslbDpRootRankVal_.revsec;
1965 0 : tab_f.rankId = hcclNslbDpRootRankVal_.rankId;
1966 :
1967 0 : for (const auto& rank : hcclNslbDpRootRankVal_.rankId) {
1968 0 : tab_f.rankId.push_back(rank);
1969 : }
1970 :
1971 0 : SendRankTableRootRank(tab_f);
1972 :
1973 0 : return HCCL_SUCCESS;
1974 0 : }
1975 :
1976 : /* 初始化NetCo通道 */
1977 0 : HcclResult hcclNslbDp::InitNetCo()
1978 : {
1979 0 : if (hcclH2dTlv::GetInstance().GetH2dTlvInitFlag() != true) {
1980 0 : HCCL_INFO("Check GetH2dTlvInitFlag is not success");
1981 0 : return HCCL_SUCCESS;
1982 : }
1983 : /* 避免二次初始化 */
1984 0 : if (GetInitNetCoFlag() == true) {
1985 0 : HCCL_INFO("Get getHccpInitFlag is true");
1986 0 : return HCCL_SUCCESS;
1987 : }
1988 0 : if (hcclH2dTlv::GetInstance().GetH2dTlvBufferSize() == NSLBDP_ILLEGAL_TLVBUFFERSIZE) {
1989 : /* 异常场景处理 */
1990 0 : HCCL_INFO("Check H2dTlvBufferSize equal 0.");
1991 0 : ClearInitNetCoFlag();
1992 0 : return HCCL_E_NOT_SUPPORT;
1993 : }
1994 0 : if (hcclH2dTlv::GetInstance().GetH2dTlvHandle() == nullptr) {
1995 0 : HCCL_ERROR("Check InitNetCo handle is null.");
1996 0 : ClearInitNetCoFlag();
1997 0 : return HCCL_E_NOT_SUPPORT;
1998 : }
1999 0 : u32 nslbBuffersize = hcclH2dTlv::GetInstance().GetH2dTlvBufferSize();
2000 0 : void* tlvHandle = hcclH2dTlv::GetInstance().GetH2dTlvHandle();
2001 0 : nslb_msg sendMsg;
2002 0 : nslb_msg recvMsg;
2003 0 : sendMsg.type = NSLBDP_TYPE_INIT_NETCO;
2004 0 : sendMsg.length = NSLBDP_ILLEGAL_MSGLENGTH;
2005 :
2006 : /* init/deinit 场景date信息默认填成数字0，hccp不关注此字段，但数据不能为NULL */
2007 0 : std::vector<uint8_t> tlvData;
2008 0 : tlvData.push_back(0);
2009 0 : sendMsg.data.assign(tlvData.begin(), tlvData.end());
2010 0 : s32 ret = H2DTlvRequest(
2011 0 : tlvHandle, MODULE_TYPE_NSLB, reinterpret_cast<TlvMsg*>(&sendMsg), reinterpret_cast<TlvMsg*>(&recvMsg));
2012 0 : HCCL_RUN_INFO("[NSLBDP-SENDTABLE] hccl send table NSLBDP_TYPE_INIT_NETCO(9001) to hccp. ret(%d)\n", ret);
2013 0 : if (ret != 0) {
2014 0 : return HCCL_E_NOT_SUPPORT;
2015 : }
2016 : /* 数据转存 */
2017 0 : SetH2DTlvInitInfo(nslbBuffersize, tlvHandle);
2018 0 : HCCL_DEBUG("Entry InitNetCo end");
2019 0 : return HCCL_SUCCESS;
2020 0 : }
2021 :
2022 : /* 去初始化NetCo通道 */
2023 0 : void hcclNslbDp::DeinitNetCo()
2024 : {
2025 0 : if (GetInitNetCoFlag() == false) {
2026 0 : return;
2027 : }
2028 0 : if (hcclH2dTlv::GetInstance().GetH2dTlvBufferSize() == NSLBDP_ILLEGAL_TLVBUFFERSIZE) {
2029 0 : return;
2030 : }
2031 0 : if (hcclH2dTlv::GetInstance().GetH2dTlvHandle() == nullptr) {
2032 0 : return;
2033 : }
2034 0 : nslb_msg sendMsg;
2035 0 : nslb_msg recvMsg;
2036 0 : sendMsg.type = NSLBDP_TYPE_DEINIT_NETCO;
2037 0 : sendMsg.length = NSLBDP_ILLEGAL_MSGLENGTH;
2038 : /* init/deinit 场景date信息默认填成数字0，hccp不关注此字段，但数据不能为NULL */
2039 0 : std::vector<uint8_t> tlvData;
2040 0 : tlvData.push_back(0);
2041 0 : sendMsg.data.assign(tlvData.begin(), tlvData.end());
2042 0 : s32 ret = H2DTlvRequest(
2043 0 : nslbdp_handle_, MODULE_TYPE_NSLB, reinterpret_cast<TlvMsg*>(&sendMsg), reinterpret_cast<TlvMsg*>(&recvMsg));
2044 0 : HCCL_RUN_INFO("[NSLBDP-SENDTABLE] hccl send table NSLBDP_TYPE_DEINIT_NETCO(9002) to hccp. ret(%d)\n", ret);
2045 0 : ClearInitNetCoFlag();
2046 0 : return;
2047 0 : }
2048 :
2049 : } // namespace hccl
|
