Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include <atomic>
12 : #include <fcntl.h>
13 : #include <unistd.h>
14 : #include <hccl/hccl_types.h>
15 : #include "topoinfo_struct.h"
16 : #include "adapter_hccp_common.h"
17 : #include "hccl_nslb_md5.h"
18 : #include "../common/src/h2d_tlv/hccl_h2dtlv.h"
19 : #include "hccl_nslbdp.h"
20 :
21 : namespace hccl {
22 :
23 : const u32 COMM_SLICING_LENGTH = 1024; // group name max length
24 :
25 396 : hcclNslbDp &hcclNslbDp::GetInstance()
26 : {
27 396 : static hcclNslbDp pNslbDp;
28 399 : return pNslbDp;
29 : }
30 :
31 :
32 7 : hcclNslbDp::hcclNslbDp()
33 : {
34 7 : }
35 :
36 7 : hcclNslbDp::~hcclNslbDp()
37 : {
38 7 : }
39 :
40 :
41 0 : void hcclNslbDp::InitCmmDesc(std::string &identifier_nslb)
42 : {
43 0 : nslbdp_identifier_ = identifier_nslb;
44 0 : HCCL_INFO("[NSLB-DP] Init CmmDesc nslbdp_identifier_[%s] .", nslbdp_identifier_.c_str());
45 0 : return;
46 : }
47 :
48 0 : std::string hcclNslbDp::GetCmmDesc()
49 : {
50 0 : HCCL_INFO("[NSLB-DP] Get CmmDesc nslbdp_identifier_[%s] .", nslbdp_identifier_.c_str());
51 0 : return nslbdp_identifier_;
52 : }
53 :
54 0 : void hcclNslbDp::SetDeviceType()
55 : {
56 0 : check910_93_ = true;
57 0 : return;
58 : }
59 :
60 0 : bool hcclNslbDp::GetDeviceType()
61 : {
62 0 : bool is91093 = check910_93_;
63 0 : return is91093;
64 : }
65 :
66 0 : HcclResult hcclNslbDp::SetH2DTlvInitInfo(u32 buffer_size, void* tlv_handle)
67 : {
68 0 : HCCL_DEBUG("[NSLB-DP] try to set tlvinit info.");
69 0 : nslbdp_buffsize_ = buffer_size;
70 0 : nslbdp_handle_ = tlv_handle;
71 0 : nslbdpIsInitNetCo_ = true;
72 0 : HCCL_INFO("[NSLB-DP] set tlvinit info buffer_size:[%u].", buffer_size);
73 0 : return HCCL_SUCCESS;
74 : }
75 :
76 0 : bool hcclNslbDp::GetInitNetCoFlag()
77 : {
78 0 : return nslbdpIsInitNetCo_;
79 : }
80 :
81 0 : HcclResult hcclNslbDp::ClearInitNetCoFlag()
82 : {
83 0 : nslbdpIsInitNetCo_ = false;
84 0 : return HCCL_SUCCESS;
85 : }
86 :
87 0 : u32 hcclNslbDp::GetTlvInitBufferSize()
88 : {
89 0 : u32 nslbdpBuffsize = nslbdp_buffsize_;
90 0 : return nslbdpBuffsize;
91 : }
92 :
93 0 : HcclResult hcclNslbDp::HcclSetGlobalRankTotalNum(u32 nRanks)
94 : {
95 : //填充表4
96 0 : hcclNslbDpGlobalRankVal_.rankTotalNum = nRanks;
97 0 : return HCCL_SUCCESS;
98 : }
99 :
100 0 : void hcclNslbDp::SetGlobalCommTaskId(u64 taskId)
101 : {
102 0 : hcclNslbDpGlobalCommInfo_.taskId = taskId;
103 0 : return;
104 : }
105 :
106 0 : void hcclNslbDp::SetGlobalCommNodeId(u32 nodeId)
107 : {
108 0 : hcclNslbDpGlobalCommInfo_.nodeId= nodeId;
109 0 : return;
110 : }
111 :
112 0 : void hcclNslbDp::SetGlobalCommLocalRankNum(u32 localRankNum)
113 : {
114 0 : hcclNslbDpGlobalCommInfo_.localRankNum= localRankNum;
115 0 : return;
116 : }
117 :
118 0 : void hcclNslbDp::SetGlobalCommRankTotalNum(u32 rankTotalNum)
119 : {
120 0 : hcclNslbDpGlobalCommInfo_.rankTotalNum= rankTotalNum;
121 0 : return;
122 : }
123 :
124 398 : u64 hcclNslbDp::GetGlobalCommTaskId()
125 : {
126 398 : return hcclNslbDpGlobalCommInfo_.taskId;
127 : }
128 :
129 0 : u32 hcclNslbDp::GetGlobalCommNodeId()
130 : {
131 0 : return hcclNslbDpGlobalCommInfo_.nodeId;
132 : }
133 :
134 0 : u8 hcclNslbDp::GetGlobalCommLocalRankNum()
135 : {
136 0 : return hcclNslbDpGlobalCommInfo_.localRankNum;
137 : }
138 :
139 0 : u32 hcclNslbDp::GetGlobalCommRankTotalNum()
140 : {
141 0 : return hcclNslbDpGlobalCommInfo_.rankTotalNum;
142 : }
143 :
144 : /* 获取 l4SPortId */
145 0 : u32 hcclNslbDp::Getl4SPortId()
146 : {
147 0 : u32 NslbDpL4SPortId = hcclNslbDpL4SPortId_;
148 0 : return NslbDpL4SPortId;
149 : }
150 :
151 : /* 切分identifier字段 */
152 0 : void hcclNslbDp::SplitString(const std::string& identifier, std::vector<std::string>& splitInfo,
153 : const std::string& frag)
154 : {
155 0 : std::string::size_type pos2 = identifier.find(frag);
156 0 : std::string::size_type pos1 = 0;
157 0 : while(std::string::npos != pos2)
158 : {
159 0 : splitInfo.push_back(identifier.substr(pos1, pos2-pos1));
160 0 : pos1 = pos2 + frag.size();
161 0 : pos2 = identifier.find(frag, pos1);
162 : }
163 0 : if(pos1 != identifier.length()) {
164 0 : splitInfo.push_back(identifier.substr(pos1));
165 : }
166 0 : return;
167 : }
168 :
169 : /* 将IP转换成U32值 */
170 0 : u32 hcclNslbDp::ipToUint32(const std::string& ipAddress)
171 : {
172 : struct sockaddr_in sa;
173 0 : inet_pton(AF_INET, ipAddress.c_str(), &(sa.sin_addr));
174 0 : return ntohl(sa.sin_addr.s_addr); // Convert to host byte order
175 : }
176 :
177 : /* 执行send 流程 */
178 0 : HcclResult hcclNslbDp::SendCommRankTable(uint32_t rank, NslbDpCommConfigVal globalCommInfo)
179 : {
180 0 : HCCL_DEBUG("[NSLB-DP] entry to send TBL_COMM_INFO");
181 0 : u32 rankTotalNum = globalCommInfo.rankTotalNum;
182 :
183 0 : HCCL_INFO("[NSLB-DP] TBL_COMM_INFO rankTotalNum:[%u].", rankTotalNum);
184 0 : if (rankTotalNum > NSLBDP_RANKTOTALNUM_BLOCK_FOU) {
185 0 : return HCCL_SUCCESS;
186 : }
187 0 : u32 packetNum = NSLBDP_PKTNUM_FIR;
188 0 : if (rankTotalNum <= NSLBDP_RANKTOTALNUM_BLOCK_FIR) {
189 0 : packetNum = NSLBDP_PKTNUM_FIR;
190 0 : } else if (rankTotalNum <= NSLBDP_RANKTOTALNUM_BLOCK_SEC) {
191 0 : packetNum = NSLBDP_PKTNUM_SEC;
192 0 : } else if (rankTotalNum <= NSLBDP_RANKTOTALNUM_BLOCK_THR) {
193 0 : packetNum = NSLBDP_PKTNUM_THR;
194 : } else {
195 0 : packetNum = NSLBDP_PKTNUM_FOU;
196 : }
197 0 : SendTableProc(rank, packetNum, globalCommInfo);
198 :
199 0 : HCCL_DEBUG("[NSLB-DP] entry send TBL_COMM_INFO end");
200 0 : return HCCL_SUCCESS;
201 : }
202 :
203 0 : bool hcclNslbDp::CheckAhcCommInfo(NslbDpCommConfigVal comInfo)
204 : {
205 0 : std::unordered_map<uint32_t, int> podToIpCount;
206 0 : for (const auto &info : comInfo.rankInfo) {
207 0 : podToIpCount[info.podId]++;
208 : }
209 : // 获取超节点对应的ip数量
210 0 : std::vector<int> ipCounts;
211 0 : for (const auto &entry : podToIpCount) {
212 0 : ipCounts.push_back(entry.second);
213 : }
214 0 : if (ipCounts.empty()) {
215 0 : return true;
216 : }
217 : // 找到最小的ip数量
218 0 : uint32_t minCount = *std::min_element(ipCounts.begin(), ipCounts.end());
219 0 : for (uint32_t count : ipCounts) {
220 0 : if (count % minCount != 0) {
221 0 : return false;
222 : }
223 : }
224 0 : return true;
225 0 : }
226 :
227 0 : bool hcclNslbDp::CheckAhcSupport(u8 algType, std::string identifier)
228 : {
229 0 : if (algType != NSLB_ALGO_TYPE_AHC) {
230 0 : return true;
231 : }
232 : char commDesc[COMM_DESC_MAX_LENGTH];
233 0 : s32 sRet = memset_s(commDesc, COMM_DESC_MAX_LENGTH, 0, sizeof(commDesc));
234 0 : if (sRet != EOK) {
235 0 : HCCL_ERROR("memset_s commDesc fail");
236 0 : return true;
237 : }
238 :
239 0 : s32 ret = strncpy_s(commDesc, COMM_DESC_MAX_LENGTH, identifier.c_str(), identifier.size());
240 0 : if (ret != EOK) {
241 0 : HCCL_INFO("strncpy_s commDesc fail");
242 0 : return true;
243 : }
244 0 : commDesc[COMM_DESC_MAX_LENGTH - 1] = '\0';
245 0 : for (const auto &info : hcclNslbDpCommConfig_) {
246 0 : if (strcmp(commDesc, info.commDesc) == 0) {
247 0 : return CheckAhcCommInfo(info);
248 : }
249 : }
250 0 : return true;
251 : }
252 :
253 : /* 读配置文件的场景与正常创建通信域场景下填充通信域信息表（表一） */
254 0 : void hcclNslbDp::SetGlobalCommRankTable_RootInfo(const RankTable_t &rankTable, const HcclBasicRankInfo &localRankInfo,
255 : const std::vector<RankInfo> &rankLists, const std::string& identifier, u32 nRanks, u32 rank)
256 : {
257 0 : HCCL_INFO("[NSLB-DP] Try to collect NSLBDP_TYPE_TBL_COMM_INFO commDesc[%s] - size = [%u].", identifier.c_str(), rankTable.rankList.size());
258 0 : u64 checkTaskId = GetGlobalCommTaskId();
259 0 : if (checkTaskId == 0 || nRanks == 1) {
260 0 : return;
261 : }
262 :
263 : //判断是否跨机
264 0 : if (CheckMultiMachine(rankTable) == false) {
265 0 : HCCL_INFO("[NSLB-DP] CheckMultiMachine is false");
266 0 : return;
267 : }
268 :
269 0 : NslbDpCommConfigVal globalCommInfo{};
270 0 : u32 sRet = memset_s(globalCommInfo.commDesc, COMM_DESC_MAX_LENGTH, 0, sizeof(globalCommInfo.commDesc));
271 0 : if (sRet != EOK) {
272 0 : HCCL_ERROR("memset_s commDesc fail sRet[%u]", sRet);
273 0 : return;
274 : }
275 0 : int countUnderScores = std::count(identifier.begin(), identifier.end(), '_');
276 0 : if (countUnderScores == NSLBDP_UNDERDCORES_COUNT) {
277 : /* 此时认为通信域描述信息中包含时间戳 */
278 0 : size_t lastUnderScoreIndex = identifier.rfind('_');
279 0 : std::string nslbIdentifier = identifier.substr(0, lastUnderScoreIndex);
280 0 : sRet = strncpy_s(globalCommInfo.commDesc, COMM_DESC_MAX_LENGTH, nslbIdentifier.c_str(), nslbIdentifier.size());
281 0 : if (sRet != EOK) { return; }
282 0 : for (size_t operSize = 0; operSize < hcclNslbDpCommConfig_.size(); operSize++) {
283 0 : if (strcmp(globalCommInfo.commDesc, hcclNslbDpCommConfig_[operSize].commDesc) == 0) {
284 0 : return;
285 : }
286 : }
287 0 : } else {
288 : /* 获取通信域唯一标识 */
289 0 : sRet = strncpy_s(globalCommInfo.commDesc, COMM_DESC_MAX_LENGTH,
290 : identifier.c_str(), identifier.size());
291 0 : if (sRet != EOK) { return; }
292 : }
293 0 : globalCommInfo.commDesc[COMM_DESC_MAX_LENGTH - 1] = '\0';
294 :
295 : u64 utime = std::chrono::duration_cast<std::chrono::milliseconds>
296 0 : (std::chrono::system_clock::now().time_since_epoch()).count();
297 0 : globalCommInfo.commInitTime = utime;
298 0 : globalCommInfo.taskId = checkTaskId;
299 0 : globalCommInfo.rankTotalNum = nRanks;
300 :
301 0 : for(u32 rankIndex = 0; rankIndex < rankTable.rankList.size(); rankIndex++) {
302 : NslbDpRankInfo dpRankInfo;
303 0 : HcclIpAddress tmpIp = rankTable.rankList[rankIndex].deviceInfo.deviceIp[0];
304 0 : std::string deviceIp = tmpIp.GetReadableAddress();
305 0 : dpRankInfo.deviceIp = ipToUint32(deviceIp);
306 0 : HCCL_INFO("[NSLB-DP] SetGlobalCommRankTable_RootInfo deviceIp:[%u] success.", dpRankInfo.deviceIp);
307 :
308 0 : std::string serverIp = rankTable.rankList[rankIndex].serverId;
309 0 : dpRankInfo.serverIp = ipToUint32(serverIp);
310 0 : HCCL_INFO("[NSLB-DP] SetGlobalCommRankTable_RootInfo serverIp:[%u] success.", dpRankInfo.serverIp);
311 0 : if (rankLists.size() < rankIndex) {
312 0 : return;
313 : }
314 0 : if (rankLists[rankIndex].superPodIdx == INVALID_UINT) {
315 0 : dpRankInfo.podId = 0;
316 : } else {
317 0 : dpRankInfo.podId = rankLists[rankIndex].superPodIdx;
318 : }
319 0 : dpRankInfo.rev = 0;
320 0 : globalCommInfo.rankInfo.push_back(dpRankInfo);
321 0 : }
322 0 : NSLBMD5::calculateRankInfoMd5(globalCommInfo.rankInfo, globalCommInfo.commMd5Sum);
323 0 : std::string nslbdpmd5 = NSLBMD5::md5ToString(globalCommInfo.commMd5Sum);
324 0 : HCCL_DEBUG("[NSLB-DP] NSLBDP-MD5 nslbdpmd5:[%s] success.", nslbdpmd5.c_str());
325 :
326 0 : std::string npuIp = localRankInfo.deviceIP[0].GetReadableIP();
327 0 : if (ipToUint32(npuIp) != 0) {
328 0 : SendCommRankTable(rank, globalCommInfo);
329 : }
330 0 : hcclNslbDpCommConfig_.push_back(globalCommInfo);
331 0 : HCCL_INFO("[NSLB-DP] Entry SetGlobalCommRankTable_RootInfo end size = [%zu]",
332 : hcclNslbDpCommConfig_.size());
333 0 : }
334 :
335 : /* 填充表5 */
336 0 : void hcclNslbDp::SetGlobalDisRankTable(const HcclBasicRankInfo &rankTable)
337 : {
338 0 : HCCL_INFO("[NSLB-DP] Try to collect NSLBDP_TYPE_TBL_RANK_DIST");
339 0 : u64 taskId = GetGlobalCommTaskId();
340 0 : if (taskId == 0) {
341 0 : return;
342 : }
343 0 : std::string npuIp = rankTable.deviceIP[0].GetReadableIP();
344 0 : if (ipToUint32(npuIp) == 0) {
345 0 : return;
346 : }
347 0 : hcclNslbDpGlobalDisRankVal_.taskId = taskId;
348 0 : hcclNslbDpGlobalDisRankVal_.nodeId = GetGlobalCommNodeId();
349 0 : hcclNslbDpGlobalDisRankVal_.localRankNum = GetGlobalCommLocalRankNum();
350 0 : hcclNslbDpGlobalDisRankVal_.rankTotalNum = GetGlobalCommRankTotalNum();
351 :
352 0 : std::string serverIp = rankTable.hostIP.GetReadableIP();
353 0 : hcclNslbDpGlobalDisRankVal_.serverIp = ipToUint32(serverIp);
354 0 : HCCL_DEBUG("[NSLB-DP] SetGlobalDisRankTable serverIp:[%u].", hcclNslbDpGlobalDisRankVal_.serverIp);
355 :
356 0 : hcclNslbDpGlobalDisRankVal_.npuIp = ipToUint32(npuIp);
357 0 : HCCL_DEBUG("[NSLB-DP] SetGlobalDisRankTable npuIp:[%u].", hcclNslbDpGlobalDisRankVal_.npuIp);
358 0 : }
359 :
360 : /* check 是否是多机场景 */
361 0 : bool hcclNslbDp::CheckMultiMachine(const RankTable_t rankTable)
362 : {
363 0 : HCCL_DEBUG("[NSLB-DP] check device is multi machine");
364 0 : u32 serverIpFir = 0;
365 0 : u16 podIdFir = 0;
366 0 : bool bIsMultiMachine = false;
367 0 : if (GetDeviceType() == true) {
368 0 : u16 podIdIndex = 0;
369 0 : for(u32 index = 0; index < rankTable.rankList.size(); index++) {
370 0 : if (rankTable.rankList[index].superPodIdx == INVALID_UINT) {
371 0 : podIdIndex = 0;
372 : } else {
373 0 : podIdIndex = rankTable.rankList[index].superPodIdx;
374 : }
375 0 : if (index == 0) {
376 0 : podIdFir = podIdIndex;
377 0 : continue;
378 : }
379 :
380 0 : if (podIdFir != podIdIndex) {
381 0 : return true;
382 : }
383 : }
384 0 : return false;
385 : }
386 0 : for(u32 rankIndex = 0; rankIndex < rankTable.rankList.size(); rankIndex++) {
387 0 : std::string serverIpInfo = rankTable.rankList[rankIndex].serverId;
388 0 : u32 serverIp = ipToUint32(serverIpInfo);
389 0 : u16 podIdInfo = 0;
390 0 : if (rankTable.rankList[rankIndex].superPodIdx == INVALID_UINT) {
391 0 : podIdInfo = 0;
392 : } else {
393 0 : podIdInfo = rankTable.rankList[rankIndex].superPodIdx;
394 : }
395 0 : if (rankIndex == 0) {
396 0 : serverIpFir = serverIp;
397 0 : podIdFir = podIdInfo;
398 0 : continue;
399 : }
400 :
401 0 : if (serverIp != serverIpFir || podIdFir != podIdInfo) {
402 0 : bIsMultiMachine = true;
403 0 : break;
404 : }
405 0 : }
406 0 : return bIsMultiMachine;
407 : }
408 :
409 : /* 无ranktable场景， 子通信域场景表1 赋值 */
410 1 : HcclResult hcclNslbDp::SetCommInfo_NoRankTable(const hccl::RankTable_t rankTable, std::string identifier)
411 : {
412 1 : HCCL_DEBUG("[NSLB-DP] Try to collect NSLBDP_TYPE_TBL_COMM_INFO for no RankTable");
413 1 : u64 taskId = GetGlobalCommTaskId();
414 1 : if (taskId == 0) {
415 1 : return HCCL_SUCCESS;
416 : }
417 : //判断是否跨机 false非错误场景
418 0 : if (CheckMultiMachine(rankTable) == false) {
419 0 : HCCL_INFO("[NSLB-DP] nslb-dp CheckMultiMachine is false");
420 0 : return HCCL_SUCCESS;
421 : }
422 :
423 0 : NslbDpCommConfigVal globalCommInfo;
424 0 : (void)memset_s(globalCommInfo.commDesc, COMM_DESC_MAX_LENGTH,
425 : 0, sizeof(globalCommInfo.commDesc));
426 :
427 : // 获取通信域唯一标识
428 0 : s32 ret = strncpy_s(globalCommInfo.commDesc, COMM_DESC_MAX_LENGTH,
429 : identifier.c_str(), identifier.size());
430 0 : if (ret != EOK) {
431 0 : HCCL_INFO("strncpy_s globalCommInfo.commDesc fail");
432 0 : return HCCL_E_MEMORY;
433 : }
434 0 : globalCommInfo.commDesc[COMM_DESC_MAX_LENGTH - 1] = '\0';
435 0 : CHK_PRT_RET(ret != EOK, HCCL_ERROR("[NSLB_DP]GetIdentifier str copy fail. return[%d]", ret), HCCL_E_INTERNAL);
436 :
437 : // commInitTime在有ranktable的赋值
438 0 : u64 utime = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now().time_since_epoch()).count();
439 0 : globalCommInfo.commInitTime = utime;
440 0 : u16 nRanks = rankTable.rankNum;
441 0 : globalCommInfo.taskId = taskId;
442 0 : globalCommInfo.rankTotalNum = nRanks;
443 :
444 0 : for(u32 rankIndex = 0; rankIndex < rankTable.rankList.size(); rankIndex++) {
445 : NslbDpRankInfo dpRankInfo;
446 0 : HcclIpAddress tmpIp = rankTable.rankList[rankIndex].deviceInfo.deviceIp[0];
447 0 : std::string deviceIp = tmpIp.GetReadableAddress();
448 0 : dpRankInfo.deviceIp = ipToUint32(deviceIp);
449 0 : HCCL_INFO("[NSLB-DP] SetCommInfo_RankTableExit deviceIp:[%u] success.", dpRankInfo.deviceIp);
450 0 : if (rankTable.rankList[rankIndex].superPodIdx == INVALID_UINT) {
451 0 : dpRankInfo.podId = 0;
452 : } else {
453 0 : dpRankInfo.podId = rankTable.rankList[rankIndex].superPodIdx;
454 : }
455 0 : dpRankInfo.rev = 0;
456 0 : globalCommInfo.rankInfo.push_back(dpRankInfo);
457 0 : }
458 0 : hcclNslbDpCommConfig_.push_back(globalCommInfo);
459 :
460 0 : return HCCL_SUCCESS;
461 0 : }
462 :
463 : /* 有ranktable场景， 表1 赋值 */
464 0 : HcclResult hcclNslbDp::SetCommInfo_RankTableExit(RankTable_t rankTable)
465 : {
466 0 : HCCL_DEBUG("[NSLB-DP] Entry SetCommInfo for RankTable exit");
467 0 : u16 nRanks = rankTable.rankNum;
468 0 : u64 taskId = GetGlobalCommTaskId();
469 0 : if (taskId == 0 || nRanks == 1) {
470 0 : return HCCL_SUCCESS;
471 : }
472 :
473 0 : NslbDpCommConfigVal globalCommInfo = {};
474 : // 获取通信域唯一标识
475 0 : (void)memset_s(hcclNslbDpGlobalRankVal_.commDesc, COMM_DESC_MAX_LENGTH,
476 : 0, sizeof(hcclNslbDpGlobalRankVal_.commDesc));
477 0 : char commDesc[COMM_DESC_MAX_LENGTH] = "HCCL_WORLD_GROUP";
478 0 : s32 sRet = memcpy_s(globalCommInfo.commDesc, sizeof(globalCommInfo.commDesc),
479 : commDesc, COMM_DESC_MAX_LENGTH);
480 0 : if (sRet != EOK) {
481 0 : HCCL_ERROR("memcpy_s commDesc fail");
482 0 : return HCCL_SUCCESS;;
483 : }
484 :
485 0 : globalCommInfo.taskId = taskId;
486 0 : globalCommInfo.rankTotalNum = nRanks;
487 :
488 0 : u64 utime = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now().time_since_epoch()).count();
489 0 : globalCommInfo.commInitTime = utime;
490 :
491 0 : u32 size = rankTable.rankList.size();
492 0 : HCCL_INFO("[NSLB-DP] RankTableExit size:[%u] success.", size);
493 :
494 0 : for(size_t ranksize = 0; ranksize < rankTable.rankList.size(); ranksize++) {
495 0 : for(size_t rankIndex = 0; rankIndex < rankTable.rankList.size(); rankIndex++) {
496 0 : u16 podId = 0;
497 0 : if (rankTable.rankList[rankIndex].superPodIdx != INVALID_UINT) {
498 0 : podId = rankTable.rankList[rankIndex].superPodIdx;
499 : }
500 : //按照rankid 排序
501 0 : if (ranksize == rankTable.rankList[rankIndex].rankId) {
502 : NslbDpRankInfo dpRankInfo;
503 0 : HcclIpAddress tmpIp = rankTable.rankList[rankIndex].deviceInfo.deviceIp[0];
504 0 : std::string deviceIp = tmpIp.GetReadableAddress();
505 0 : dpRankInfo.deviceIp = ipToUint32(deviceIp);
506 0 : HCCL_INFO("[NSLB-DP] SetCommInfo_RankTableExit deviceIp:[%s] success.", deviceIp.c_str());
507 0 : std::string serverIp = rankTable.rankList[rankIndex].serverId;
508 0 : dpRankInfo.serverIp = ipToUint32(serverIp);
509 0 : HCCL_INFO("[NSLB-DP] SetCommInfo_RankTableExit serverIp:[%s] success.", serverIp.c_str());
510 0 : dpRankInfo.podId = podId;
511 0 : dpRankInfo.rev = 0;
512 0 : globalCommInfo.rankInfo.push_back(dpRankInfo);
513 0 : }
514 : }
515 : }
516 0 : NSLBMD5::calculateRankInfoMd5(globalCommInfo.rankInfo, globalCommInfo.commMd5Sum);
517 0 : std::string nslbdpmd5 = NSLBMD5::md5ToString(globalCommInfo.commMd5Sum);
518 0 : HCCL_INFO("[NSLB-DP] check pmd5:[%s] success.", nslbdpmd5.c_str());
519 :
520 0 : hcclNslbDpCommConfig_.push_back(globalCommInfo);
521 0 : HCCL_DEBUG("[NSLB-DP] entry SetCommInfo_RankTableExit end");
522 :
523 0 : return HCCL_SUCCESS;
524 0 : }
525 :
526 : /* 有ranktable场景，表4赋值 */
527 0 : HcclResult hcclNslbDp::SetGlobalRank_RankTableExit(const hccl::RankTable_t rankTable)
528 : {
529 0 : u64 taskId = GetGlobalCommTaskId();
530 0 : HCCL_INFO("[NSLB-DP] set TBL_RANK for RankTableExit[%ull] success.", taskId);
531 :
532 0 : if (taskId == 0) {
533 0 : return HCCL_SUCCESS;
534 : }
535 0 : u16 nRanks = rankTable.rankNum;
536 0 : hcclNslbDpGlobalRankVal_.taskId = taskId;
537 0 : (void)memset_s(hcclNslbDpGlobalRankVal_.commDesc, COMM_DESC_MAX_LENGTH,
538 : 0, sizeof(hcclNslbDpGlobalRankVal_.commDesc));
539 0 : char commDesc[COMM_DESC_MAX_LENGTH] = "HCCL_WORLD_GROUP";
540 0 : s32 sRet = memcpy_s(hcclNslbDpGlobalRankVal_.commDesc, sizeof(hcclNslbDpGlobalRankVal_.commDesc),
541 : commDesc, COMM_DESC_MAX_LENGTH);
542 0 : if (sRet != EOK) {
543 0 : HCCL_ERROR("memcpy_s commDesc fail");
544 0 : return HCCL_SUCCESS;
545 : }
546 :
547 0 : hcclNslbDpGlobalRankVal_.commInitTime = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now().time_since_epoch()).count();
548 0 : hcclNslbDpGlobalRankVal_.rankTotalNum = nRanks;
549 :
550 0 : for (size_t ranksize = 0; ranksize < rankTable.rankList.size(); ranksize++) {
551 0 : for (size_t rankIndex = 0; rankIndex < rankTable.rankList.size(); rankIndex++) {
552 0 : if(ranksize == rankTable.rankList[rankIndex].rankId) {
553 : TableFourRankInfo dpGloRankInfo;
554 0 : HcclIpAddress tmpIp = rankTable.rankList[rankIndex].deviceInfo.deviceIp[0];
555 0 : std::string deviceIp = tmpIp.GetReadableAddress();
556 0 : dpGloRankInfo.deviceIp = ipToUint32(deviceIp);
557 :
558 0 : std::string serverIp = rankTable.rankList[rankIndex].serverId;
559 0 : dpGloRankInfo.serverIp = ipToUint32(serverIp);
560 :
561 0 : hcclNslbDpGlobalRankVal_.rankInfo.push_back(dpGloRankInfo);
562 0 : }
563 : }
564 : }
565 0 : NSLBMD5::calculateTableFourRankInfoMd5(hcclNslbDpGlobalRankVal_.rankInfo, hcclNslbDpGlobalRankVal_.commMd5Sum);
566 0 : std::string nslbdpmd5 = NSLBMD5::md5ToString(hcclNslbDpGlobalRankVal_.commMd5Sum);
567 0 : HCCL_INFO("[NSLB-DP] check pmd5:[%s] success.", nslbdpmd5.c_str());
568 0 : return HCCL_SUCCESS;
569 0 : }
570 :
571 : /* 拼接 l4SPortId */
572 0 : HcclResult hcclNslbDp::GetNslbDpl4SPortId(u32 rankSize, u8 algType, u16 *l4SPortId)
573 : {
574 0 : u16 priFlag = NSLBDP_PRIVATE_PORT;
575 0 : u16 CommIntervalFlag = NSLB_COMM_INTERVAL_FLAG_BEGIN;
576 0 : if (rankSize > NSLBDP_COMMINTERVAL_FLAGSIX) {
577 0 : CommIntervalFlag = NSLB_COMM_INTERVAL_FLAG_SEV;
578 0 : } else if (rankSize > NSLBDP_COMMINTERVAL_FLAGFIV) {
579 0 : CommIntervalFlag = NSLB_COMM_INTERVAL_FLAG_SIX;
580 0 : } else if (rankSize > NSLBDP_COMMINTERVAL_FLAGFOU) {
581 0 : CommIntervalFlag = NSLB_COMM_INTERVAL_FLAG_FIV;
582 0 : } else if (rankSize > NSLBDP_COMMINTERVAL_FLAGTHR) {
583 0 : CommIntervalFlag = NSLB_COMM_INTERVAL_FLAG_FOR;
584 0 : } else if (rankSize > NSLBDP_COMMINTERVAL_FLAGSEC) {
585 0 : CommIntervalFlag = NSLB_COMM_INTERVAL_FLAG_THR;
586 0 : } else if (rankSize > NSLBDP_COMMINTERVAL_FLAG) {
587 0 : CommIntervalFlag = NSLB_COMM_INTERVAL_FLAG_SEC;
588 : } else {
589 0 : CommIntervalFlag = NSLB_COMM_INTERVAL_FLAG_FIR;
590 : }
591 0 : u16 CommPrecisely = rankSize % NSLBDP_COMMINTERVAL_FLAG;
592 0 : u8 CommalgType = algType;
593 :
594 0 : *l4SPortId = (priFlag << NSLBDP_RANGE_ID) + (CommIntervalFlag << NSLBDP_COMMON_RANGE) + (CommPrecisely << NSLBDP_ALGO_RANGE) + CommalgType;
595 0 : hcclNslbDpL4SPortId_ = *l4SPortId;
596 :
597 0 : HCCL_INFO("[NSLB-DP-L4PORT] get hcclNslbDpL4SPortId_[%u] success", hcclNslbDpL4SPortId_);
598 0 : return HCCL_SUCCESS;
599 : }
600 :
601 : /* 表6赋值 */
602 0 : HcclResult hcclNslbDp::SetNslbDpRootRank(HcclCMDType opType, u32 rootRank, std::string identifier, u8 algType)
603 : {
604 0 : HCCL_DEBUG("[NSLB-DP] try to collect NSLBDP_TYPE_TBL_ROOT_RANK");
605 0 : u64 taskId = GetGlobalCommTaskId();
606 0 : if (taskId == 0) {
607 0 : return HCCL_SUCCESS;
608 : }
609 0 : if (hcclNslbDpRootRankVal_.taskId == 0) {
610 0 : HCCL_DEBUG("[NSLB-DP-BEGIN] RootRank first entry");
611 : // 获取task id
612 0 : hcclNslbDpRootRankVal_.taskId = taskId;
613 0 : s32 sRet = memset_s(hcclNslbDpRootRankVal_.commDesc, COMM_DESC_MAX_LENGTH,
614 : 0, sizeof(hcclNslbDpRootRankVal_.commDesc));
615 0 : if (sRet != EOK) {
616 0 : HCCL_ERROR("memset_s commDesc fail");
617 0 : return HCCL_SUCCESS;
618 : }
619 : // 获取通信域唯一标识
620 0 : s32 ret = strncpy_s(hcclNslbDpRootRankVal_.commDesc, COMM_DESC_MAX_LENGTH,
621 : identifier.c_str(), identifier.size());
622 0 : if (ret != EOK) {
623 0 : HCCL_INFO("strncpy_s hcclNslbDpRootRankVal_.commDesc fail");
624 0 : return HCCL_SUCCESS;
625 : }
626 0 : hcclNslbDpRootRankVal_.commDesc[COMM_DESC_MAX_LENGTH - 1] = '\0';
627 :
628 : // commInitTime在有ranktable的赋值
629 0 : hcclNslbDpRootRankVal_.commInitTime = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now().time_since_epoch()).count();
630 :
631 : // 获取operator、algorithm
632 0 : hcclNslbDpRootRankVal_.oper = GetNslbOpType(opType);;
633 :
634 0 : hcclNslbDpRootRankVal_.algorithm = algType;
635 :
636 0 : hcclNslbDpRootRankVal_.rootRankNum = hcclNslbDpRootRankVal_.rootRankNum + 1;
637 :
638 : NslbDpRankId rootRankId;
639 0 : rootRankId.rankID = rootRank;
640 0 : hcclNslbDpRootRankVal_.rankId.push_back(rootRankId);
641 :
642 0 : HCCL_INFO("[NSLB-DP] Rootable rootRankNum[%u]-rootRank[%u]-algorithm[%u].",
643 : hcclNslbDpRootRankVal_.rootRankNum, rootRank, hcclNslbDpRootRankVal_.algorithm);
644 :
645 0 : HCCL_DEBUG("[NSLB-DP] Rootabl entry SetNslbDpRootRank end");
646 0 : SendRootRankTable();
647 0 : return HCCL_SUCCESS;
648 : }
649 0 : HCCL_INFO("[NSLB-DP] Rootable rankId.size[%zu]-rootRankNum[%u]-rootRank[%u].",
650 : hcclNslbDpRootRankVal_.rankId.size(), hcclNslbDpRootRankVal_.rootRankNum, rootRank);
651 :
652 0 : for (const auto& rank : hcclNslbDpRootRankVal_.rankId) {
653 0 : HCCL_INFO("[NSLB-DP] Rootable rootRank exit rankID[%u]-rootRank[%u]",rank.rankID, rootRank);
654 0 : if (rank.rankID == rootRank) {
655 0 : return HCCL_SUCCESS;
656 : }
657 : }
658 :
659 : NslbDpRankId rootRankId;
660 0 : rootRankId.rankID = static_cast<u16>(rootRank);
661 0 : hcclNslbDpRootRankVal_.rankId.push_back(rootRankId);
662 0 : hcclNslbDpRootRankVal_.rootRankNum = hcclNslbDpRootRankVal_.rootRankNum + 1;
663 :
664 : // 新增场景下走send 流程
665 0 : SendRootRankTable();
666 :
667 0 : return HCCL_SUCCESS;
668 : }
669 :
670 : /* AlgType 的转换 */
671 0 : u8 hcclNslbDp::GetNslbLevel1AlgType(AlgTypeLevel1 algValue)
672 : {
673 0 : HCCL_DEBUG("[NSLB-DP] try to switch Level1 type to nslbtype");
674 0 : switch (algValue) {
675 0 : case AlgTypeLevel1::ALG_LEVEL1_RING: {
676 0 : return NSLB_ALGO_TYPE_RING;
677 : }
678 0 : case AlgTypeLevel1::ALG_LEVEL1_PIPELINE: {
679 0 : return NSLB_ALGO_TYPE_PIPELINE;
680 : }
681 0 : case AlgTypeLevel1::ALG_LEVEL1_HD: {
682 0 : return NSLB_ALGO_TYPE_HDR;
683 : }
684 0 : case AlgTypeLevel1::ALG_LEVEL1_NHR: {
685 0 : return NSLB_ALGO_TYPE_NHR;
686 : }
687 0 : case AlgTypeLevel1::ALG_LEVEL1_NHR_V1: {
688 0 : return NSLB_ALGO_TYPE_NHR_V1;
689 : }
690 0 : case AlgTypeLevel1::ALG_LEVEL1_NB: {
691 0 : return NSLB_ALGO_TYPE_NB;
692 : }
693 0 : case AlgTypeLevel1::ALG_LEVEL1_AHC: {
694 0 : return NSLB_ALGO_TYPE_AHC;
695 : }
696 0 : case AlgTypeLevel1::ALG_LEVEL1_AHC_BROKE: {
697 0 : return NSLB_ALGO_TYPE_AHC;
698 : }
699 0 : default:
700 0 : return NSLB_ALGO_TYPE_NA;
701 : }
702 : }
703 :
704 : /* Level2 的 AlgType 转换 */
705 0 : u8 hcclNslbDp::GetNslbLevel2AlgType(AlgTypeLevel2 algValue)
706 : {
707 0 : HCCL_DEBUG("[NSLB-DP] try to switch Level2 type to nslbtype");
708 0 : switch (algValue) {
709 0 : case AlgTypeLevel2::ALG_LEVEL2_RING: {
710 0 : return NSLB_ALGO_TYPE_RING;
711 : }
712 0 : case AlgTypeLevel2::ALG_LEVEL2_HD: {
713 0 : return NSLB_ALGO_TYPE_HDR;
714 : }
715 0 : case AlgTypeLevel2::ALG_LEVEL2_NHR: {
716 0 : return NSLB_ALGO_TYPE_NHR;
717 : }
718 0 : case AlgTypeLevel2::ALG_LEVEL2_NB: {
719 0 : return NSLB_ALGO_TYPE_NB;
720 : }
721 0 : case AlgTypeLevel2::ALG_LEVEL2_PIPELINE: {
722 0 : return NSLB_ALGO_TYPE_PIPELINE;
723 : }
724 0 : default:
725 0 : return NSLB_ALGO_TYPE_NA;
726 : }
727 : }
728 :
729 : /* opType 的转换 */
730 0 : u8 hcclNslbDp::GetNslbOpType(HcclCMDType opType)
731 : {
732 0 : HCCL_DEBUG("[NSLB-DP] try to get nslb optype");
733 0 : switch (opType) {
734 0 : case HcclCMDType::HCCL_CMD_BROADCAST: {
735 0 : return NSLBDP_CMD_BROADCAST;
736 : }
737 0 : case HcclCMDType::HCCL_CMD_ALLREDUCE: {
738 0 : return NSLBDP_CMD_ALLREDUCE;
739 : }
740 0 : case HcclCMDType::HCCL_CMD_REDUCE: {
741 0 : return NSLBDP_CMD_REDUCE;
742 : }
743 0 : case HcclCMDType::HCCL_CMD_SEND: {
744 0 : return NSLBDP_CMD_SEND;
745 : }
746 0 : case HcclCMDType::HCCL_CMD_RECEIVE: {
747 0 : return NSLBDP_CMD_RECEIVE;
748 : }
749 0 : case HcclCMDType::HCCL_CMD_ALLGATHER: {
750 0 : return NSLBDP_CMD_ALLGATHER;
751 : }
752 0 : case HcclCMDType::HCCL_CMD_REDUCE_SCATTER: {
753 0 : return NSLBDP_CMD_REDUCE_SCATTER;
754 : }
755 0 : case HcclCMDType::HCCL_CMD_ALLTOALLV: {
756 0 : return NSLBDP_CMD_ALLTOALLV;
757 : }
758 0 : case HcclCMDType::HCCL_CMD_ALLTOALLVC: {
759 0 : return NSLBDP_CMD_ALLTOALLVC;
760 : }
761 0 : case HcclCMDType::HCCL_CMD_ALLTOALL: {
762 0 : return NSLBDP_CMD_ALLTOALL;
763 : }
764 0 : case HcclCMDType::HCCL_CMD_GATHER: {
765 0 : return NSLBDP_CMD_GATHER;
766 : }
767 0 : case HcclCMDType::HCCL_CMD_SCATTER: {
768 0 : return NSLBDP_CMD_SCATTER;
769 : }
770 0 : case HcclCMDType::HCCL_CMD_BATCH_SEND_RECV: {
771 0 : return NSLBDP_CMD_BATCH_SEND_RECV;
772 : }
773 0 : default:
774 0 : return 0;
775 : }
776 : }
777 :
778 : /* 获取通信量的前4bit */
779 0 : u64 hcclNslbDp::GetNslbDpFirstFourBit(u8 opType, u8 algType)
780 : {
781 0 : u64 firstFourBit = 0;
782 0 : if (opType == NSLBDP_CMD_ALLREDUCE) {
783 0 : firstFourBit = 1 << NSLBDP_BEGINFOURBIT;
784 0 : firstFourBit = firstFourBit + 1;
785 : }
786 :
787 0 : if (opType == NSLBDP_CMD_ALLGATHER) {
788 0 : firstFourBit = 1 << 1;
789 : }
790 :
791 0 : HCCL_INFO("[NSLB-DP-FIRST4] try to get first FourBit[%llu]-algType[%u]", firstFourBit, algType);
792 0 : return firstFourBit;
793 : }
794 :
795 : /* 校验算法的一致性 */
796 0 : bool hcclNslbDp::CheckAlgoConsistency(HcclCMDType opType, std::string& algName)
797 : {
798 0 : if (opType == HcclCMDType::HCCL_CMD_ALLREDUCE) {
799 0 : if (algName.find("AllReduce") != std::string::npos) {
800 0 : return true;
801 : } else {
802 0 : return false;
803 : }
804 0 : } else if (opType == HcclCMDType::HCCL_CMD_ALLGATHER) {
805 0 : if (algName.find("AllGather") != std::string::npos) {
806 0 : return true;
807 : } else {
808 0 : return false;
809 : }
810 0 : } else if (opType == HcclCMDType::HCCL_CMD_REDUCE_SCATTER) {
811 0 : if (algName.find("ReduceScatter") != std::string::npos) {
812 0 : return true;
813 : } else {
814 0 : return false;
815 : }
816 0 : } else if (opType == HcclCMDType::HCCL_CMD_ALLTOALL) {
817 0 : if (algName != "RunAlltoAllVFullMesh") {
818 0 : return false;
819 : }
820 0 : } else if (opType == HcclCMDType::HCCL_CMD_BROADCAST) {
821 0 : if (algName.find("BroadCast") != std::string::npos) {
822 0 : return true;
823 : } else {
824 0 : return false;
825 : }
826 0 : } else if (opType == HcclCMDType::HCCL_CMD_SCATTER) {
827 0 : if (algName.find("Scatter") != std::string::npos) {
828 0 : return true;
829 : } else {
830 0 : return false;
831 : }
832 : }
833 0 : return true;
834 : }
835 :
836 :
837 0 : bool hcclNslbDp::CheckSupportOptype(HcclCMDType opType)
838 : {
839 0 : HCCL_DEBUG("[NSLB-DP] try to check support optype");
840 0 : if (GetNslbOpType(opType) != NSLBDP_CMD_ALLREDUCE && GetNslbOpType(opType) != NSLBDP_CMD_ALLGATHER &&
841 0 : GetNslbOpType(opType) != NSLBDP_CMD_REDUCE_SCATTER && GetNslbOpType(opType) != NSLBDP_CMD_ALLTOALL &&
842 0 : GetNslbOpType(opType) != NSLBDP_CMD_ALLTOALLV && GetNslbOpType(opType) != NSLBDP_CMD_ALLTOALLVC &&
843 0 : GetNslbOpType(opType) != NSLBDP_CMD_BROADCAST && GetNslbOpType(opType) != NSLBDP_CMD_SCATTER &&
844 0 : GetNslbOpType(opType) != NSLBDP_CMD_BATCH_SEND_RECV && GetNslbOpType(opType) != NSLBDP_CMD_REDUCE &&
845 0 : GetNslbOpType(opType) != NSLBDP_CMD_SEND) {
846 0 : return false;
847 : }
848 0 : return true;
849 : }
850 :
851 : /* 写算法算子表--表3 */
852 0 : HcclResult hcclNslbDp::GetAlgAdjacencyTable(HcclCMDType opType, u32 srcLocalRankId, u32 rootRank,
853 : u8 algType, std::string identifier, AdjInfo nslbAdjInfo)
854 : {
855 0 : std::string nslbIdentifier = identifier;
856 0 : int countUnderScores = std::count(identifier.begin(), identifier.end(), '_');
857 0 : if (countUnderScores == NSLBDP_UNDERDCORES_COUNT) {
858 0 : size_t lastUnderScoreIndex = identifier.rfind('_');
859 0 : nslbIdentifier = identifier.substr(0, lastUnderScoreIndex);
860 : }
861 0 : HCCL_DEBUG("[NSLB-DP] check table NSLBDP_TYPE_TBL_ADJ size:[%zu].", hcclNslbDpAlgorithmInfo_.size());
862 0 : if (CheckAhcSupport(algType, nslbIdentifier) == false) {
863 0 : HCCL_RUN_INFO("[NSLB-DP-ADJ] Check AHC commoninfo is not support.");
864 0 : return HCCL_SUCCESS;
865 : }
866 0 : u64 taskId = GetGlobalCommTaskId();
867 0 : if (taskId == 0) {
868 0 : return HCCL_SUCCESS;
869 : }
870 0 : HCCL_INFO("[NSLB-DP-ADJ] opType:[%u],srcLocalRankId[%u],rootRank[%u]-commDesc[%s],dstRankNum:[%u].",
871 : opType, srcLocalRankId, rootRank, nslbIdentifier.c_str(), nslbAdjInfo.dstRankNum);
872 :
873 0 : if (CheckSupportOptype(opType) == false) {
874 0 : HCCL_INFO("[NSLB-DP-OPER] CheckSupportOptype false .");
875 0 : return HCCL_SUCCESS;
876 : }
877 :
878 0 : NslbDpAlgorithmInfo algorithmInfo;
879 0 : algorithmInfo.taskId = taskId;
880 0 : (void)memset_s(algorithmInfo.commDesc, COMM_DESC_MAX_LENGTH,
881 : 0, sizeof(algorithmInfo.commDesc));
882 0 : s32 ret = strncpy_s(algorithmInfo.commDesc, COMM_DESC_MAX_LENGTH,
883 : nslbIdentifier.c_str(), nslbIdentifier.size());
884 0 : if (ret != EOK) {
885 0 : HCCL_INFO("[NSLB-DP] strncpy_s algorithmInfo.commDesc fail");
886 : }
887 0 : algorithmInfo.commDesc[COMM_DESC_MAX_LENGTH - 1] = '\0';
888 :
889 0 : HCCL_INFO("[NSLB-DP] CheckCommDescExit algorithmInfo.commDesc[%s] .", algorithmInfo.commDesc);
890 :
891 : // 去除不存在的通信域信息
892 0 : bool commDescExit = false;
893 0 : for (const auto& info : hcclNslbDpCommConfig_) {
894 0 : if (strcmp(algorithmInfo.commDesc, info.commDesc) == 0) {
895 0 : commDescExit = true;
896 0 : break;
897 : }
898 : }
899 0 : if (commDescExit == false) {
900 0 : return HCCL_SUCCESS;
901 : }
902 :
903 : // 根据表一信息填充MD5
904 0 : for (size_t comsize = 0; comsize < hcclNslbDpCommConfig_.size(); comsize++) {
905 0 : if(strcmp(algorithmInfo.commDesc, hcclNslbDpCommConfig_[comsize].commDesc) == 0) {
906 0 : s32 sRet = memcpy_s(algorithmInfo.commMd5Sum, sizeof(algorithmInfo.commMd5Sum),
907 0 : hcclNslbDpCommConfig_[comsize].commMd5Sum, sizeof(hcclNslbDpCommConfig_[comsize].commMd5Sum));
908 0 : if (sRet != EOK) {
909 0 : HCCL_ERROR("memcpy_s commMd5Sum fail");
910 0 : return HCCL_SUCCESS;
911 : }
912 0 : break;
913 : }
914 : }
915 0 : HCCL_RUN_INFO("[NSLB-DP] add adjINfo:***[%llu]***[%u]***[%u]***[%u]***[%u]-[%zu] success.",
916 : taskId, srcLocalRankId, rootRank, GetNslbOpType(opType),algType, nslbAdjInfo.nsAdjInfo.size());
917 :
918 0 : for (const auto& info : hcclNslbDpAlgorithmInfo_) {
919 0 : HCCL_INFO("[NSLB-DP-ADJ] info: *[%llu]-[%u]-[%u]-[%u]-[%u]* success.",
920 : taskId, info.srcLocalRankId, info.rootRank, info.oper, info.algorithm);
921 0 : if (info.taskId == taskId && info.srcLocalRankId == srcLocalRankId &&
922 0 : info.rootRank == static_cast<u16>(rootRank) && info.oper == GetNslbOpType(opType) && info.algorithm == algType &&
923 0 : strcmp(algorithmInfo.commDesc, info.commDesc) == 0) {
924 0 : HCCL_INFO("[NSLB-DP] Deduplication hcclNslbDpAlgorithmInfo_");
925 0 : return HCCL_SUCCESS;
926 : }
927 : }
928 :
929 0 : HCCL_INFO("[NSLB-DP-ADJ] add adjINfo:***[%llu]***[%u]***[%u]***[%u]***[%u] success.",
930 : taskId, srcLocalRankId, rootRank, GetNslbOpType(opType), algType);
931 :
932 0 : algorithmInfo.srcLocalRankId = srcLocalRankId;
933 0 : algorithmInfo.rootRank = static_cast<u16>(rootRank);
934 0 : algorithmInfo.oper = GetNslbOpType(opType);
935 0 : algorithmInfo.algorithm = algType;
936 :
937 0 : algorithmInfo.dstRankNum = nslbAdjInfo.dstRankNum;
938 0 : HCCL_INFO("[NSLB-DP-ADJ] nslbAdjInfo.dstRankNum:[%u].", nslbAdjInfo.dstRankNum);
939 :
940 0 : if (nslbAdjInfo.nsAdjInfo.size() == 0) {
941 0 : algorithmInfo.dstRankNum = 0;
942 0 : HCCL_INFO("[NSLB-DP] get nsAdjInfo fail dstRankNum:[%u]", algorithmInfo.dstRankNum);
943 0 : return HCCL_SUCCESS;
944 : } else {
945 0 : for (size_t rankIndex = 0; rankIndex < nslbAdjInfo.nsAdjInfo.size(); rankIndex++) {
946 : NslbDpAdjInfo adjInfo;
947 0 : adjInfo.dstLocalRankId = nslbAdjInfo.nsAdjInfo[rankIndex].dstLocalRankId;
948 0 : adjInfo.phaseId = nslbAdjInfo.nsAdjInfo[rankIndex].phaseId;
949 0 : adjInfo.rev = nslbAdjInfo.nsAdjInfo[rankIndex].rev;
950 0 : HCCL_INFO("[NSLB-DP-ADJ] adjINfo:[%u]-[%u]-[%u] success.",
951 : srcLocalRankId, nslbAdjInfo.nsAdjInfo[rankIndex].dstLocalRankId, nslbAdjInfo.nsAdjInfo[rankIndex].phaseId);
952 0 : algorithmInfo.AdjInfo.push_back(adjInfo);
953 : }
954 : }
955 :
956 0 : algorithmInfo.sedFlag = 0;
957 0 : hcclNslbDpAlgorithmInfo_.push_back(algorithmInfo);
958 0 : HCCL_DEBUG("[NSLB-DP] entry GetAlgAdjacencyTable end");
959 0 : return HCCL_SUCCESS;
960 0 : }
961 :
962 0 : bool hcclNslbDp::CheckCommDescExit(NslbDpOperatorInfo &OperatorInfo)
963 : {
964 0 : HCCL_DEBUG("[NSLB-DP-OPER] Check CommDescExit size:[%zu].", hcclNslbDpCommConfig_.size());
965 0 : for (const auto& info : hcclNslbDpCommConfig_) {
966 0 : HCCL_DEBUG("[NSLB-DP-OPER] CheckCommDescExit info.commDesc[%s] .", info.commDesc);
967 0 : if (strcmp(OperatorInfo.commDesc, info.commDesc) == 0) {
968 0 : return true;
969 : }
970 : }
971 0 : return false;
972 : }
973 :
974 0 : void hcclNslbDp::fullcommDescInitTime(std::string identifier, NslbDpOperatorInfo &OperatorInfo)
975 : {
976 0 : int countUnderScores = std::count(identifier.begin(), identifier.end(), '_');
977 0 : if (countUnderScores == NSLBDP_UNDERDCORES_COUNT) {
978 : /* 此时认为通信域描述信息中包含时间戳 */
979 0 : size_t lastUnderScoreIndex = identifier.rfind('_');
980 0 : std::string nslbIdentifier = identifier.substr(0, lastUnderScoreIndex);
981 0 : (void)strncpy_s(OperatorInfo.commDesc, COMM_DESC_MAX_LENGTH,
982 : nslbIdentifier.c_str(), nslbIdentifier.size());
983 0 : } else {
984 : /* 获取通信域唯一标识 */
985 0 : (void)strncpy_s(OperatorInfo.commDesc, COMM_DESC_MAX_LENGTH,
986 : identifier.c_str(), identifier.size());
987 : }
988 0 : HCCL_DEBUG("[NSLB-DP-OPER] fullcommDescInitTime commDesc[%s] .", identifier.c_str());
989 :
990 0 : OperatorInfo.commDesc[COMM_DESC_MAX_LENGTH - 1] = '\0';
991 :
992 : // commInitTime在有ranktable的赋值
993 0 : OperatorInfo.commInitTime = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now().time_since_epoch()).count();
994 :
995 0 : return;
996 : }
997 :
998 0 : bool hcclNslbDp::CheckSameOperatorVal(size_t operSize, NslbDpOperatorInfo &OperatorInfo, u32 rootRank)
999 : {
1000 0 : size_t num = hcclNslbDpOperatorVal_.size();
1001 0 : if (operSize >= num) {
1002 0 : return false;
1003 : }
1004 0 : if (hcclNslbDpOperatorVal_[operSize].taskId == OperatorInfo.taskId &&
1005 0 : hcclNslbDpOperatorVal_[operSize].rootRank == rootRank &&
1006 0 : hcclNslbDpOperatorVal_[operSize].oper == OperatorInfo.oper &&
1007 0 : hcclNslbDpOperatorVal_[operSize].algorithm == OperatorInfo.algorithm &&
1008 0 : strcmp(OperatorInfo.commDesc, hcclNslbDpOperatorVal_[operSize].commDesc) == 0) {
1009 0 : return true;
1010 : }
1011 0 : return false;
1012 : }
1013 :
1014 : /* 写算法算子表--表2 */
1015 0 : HcclResult hcclNslbDp::GenerateOpAndAdjTable(HcclCMDType opType, u32 rootRank, u32 srcLocalRankId, u8 algType,
1016 : std::string identifier, u64 count, u32 rankSize)
1017 : {
1018 0 : HCCL_INFO("[NSLB-DP-OPER] count=[%llu], hcclNslbDpOperatorVal_ size:[%zu].", count, hcclNslbDpOperatorVal_.size());
1019 0 : u64 taskId = GetGlobalCommTaskId();
1020 0 : if (taskId == 0) {
1021 0 : return HCCL_SUCCESS;
1022 : }
1023 :
1024 : // 获取，0号
1025 0 : if (rootRank != 0) {
1026 0 : return HCCL_SUCCESS;
1027 : }
1028 0 : u64 trafficNum = count;
1029 0 : if (CheckSupportOptype(opType) == false) {
1030 0 : trafficNum = 0;
1031 0 : HCCL_DEBUG("[NSLB-DP-OPER] CheckSupportOptype false trafficNum == 0.");
1032 : }
1033 :
1034 0 : NslbDpOperatorInfo OperatorInfo = {};
1035 : // 获取task id
1036 0 : OperatorInfo.taskId = taskId;
1037 0 : (void)memset_s(OperatorInfo.commDesc, COMM_DESC_MAX_LENGTH,
1038 : 0, sizeof(OperatorInfo.commDesc));
1039 :
1040 0 : fullcommDescInitTime(identifier, OperatorInfo);
1041 :
1042 : // 去除不存在的通信域信息
1043 0 : if (CheckCommDescExit(OperatorInfo) == false) {
1044 0 : HCCL_INFO("[NSLB-DP-OPER] CheckCommDesc not exit ");
1045 0 : return HCCL_SUCCESS;
1046 : }
1047 :
1048 : // 获取operator、algorithm
1049 0 : OperatorInfo.oper = GetNslbOpType(opType);
1050 0 : OperatorInfo.algorithm = algType;
1051 0 : u64 trafficCount = GetNslbDpFirstFourBit(OperatorInfo.oper, OperatorInfo.algorithm);
1052 0 : trafficCount = (trafficCount << NSLBDP_TRAFFICCONUT) + trafficNum;
1053 :
1054 0 : for (size_t operSize = 0; operSize < hcclNslbDpOperatorVal_.size(); operSize++) {
1055 0 : if (CheckSameOperatorVal(operSize, OperatorInfo, rootRank) == true) {
1056 0 : if (hcclNslbDpOperatorVal_[operSize].trafficCnt < trafficCount && srcLocalRankId == 0) {
1057 0 : hcclNslbDpOperatorVal_[operSize].trafficCnt = trafficCount;
1058 0 : hcclNslbDpOperatorVal_[operSize].sedFlag = 0;
1059 0 : SendRankTableOpAndAdj(hcclNslbDpOperatorVal_[operSize]);
1060 0 : HCCL_RUN_INFO("[NSLB-DP-ADJ] commDesc[%s] try to update trafficCnt[%llu] success.", OperatorInfo.commDesc, trafficCount);
1061 : }
1062 0 : return HCCL_SUCCESS;
1063 : }
1064 : }
1065 :
1066 0 : OperatorInfo.trafficCnt = trafficCount; // 判断变大
1067 0 : OperatorInfo.rootRank = rootRank;
1068 0 : HCCL_INFO("[NSLB-DP-OPER] add operInfo:***[%llu]***[%llu]***[%u]***[%u]***[%u] success.",
1069 : taskId, OperatorInfo.commInitTime, rootRank, OperatorInfo.oper, OperatorInfo.algorithm);
1070 :
1071 0 : GetNslbDpl4SPortId(rankSize, algType, &OperatorInfo.l4SPortId);
1072 0 : if (srcLocalRankId == 0) {
1073 0 : SendRankTableOpAndAdj(OperatorInfo);
1074 : }
1075 0 : hcclNslbDpOperatorVal_.push_back(OperatorInfo);
1076 :
1077 0 : return HCCL_SUCCESS;
1078 : }
1079 :
1080 : /* 根将表1 序列化处理 */
1081 0 : std::vector<uint8_t> hcclNslbDp::serializeTLV_TableFir(NslbDpCommConfigInfo cominfo)
1082 : {
1083 0 : HCCL_DEBUG("[NSLB-DP] entry serializeTLV TBL_COMM_INFO beg");
1084 :
1085 0 : std::vector<uint8_t> tlvData;
1086 :
1087 : // Task ID
1088 0 : uint64_t netTaskId = htobe64(cominfo.taskId);
1089 0 : tlvData.resize(tlvData.size() + sizeof(netTaskId));
1090 0 : (void)memcpy_s(tlvData.data() + tlvData.size() - sizeof(netTaskId), sizeof(netTaskId), &netTaskId, sizeof(netTaskId));
1091 :
1092 : // CommDesc
1093 0 : tlvData.insert(tlvData.end(), cominfo.commDesc, cominfo.commDesc + COMM_DESC_MAX_LENGTH);
1094 :
1095 : // CommInitTime
1096 0 : uint64_t netCommInitTime = htobe64(cominfo.commInitTime);
1097 0 : tlvData.resize(tlvData.size() + sizeof(netCommInitTime));
1098 0 : (void)memcpy_s(tlvData.data() + tlvData.size() - sizeof(netCommInitTime), sizeof(netCommInitTime), &netCommInitTime, sizeof(netCommInitTime));
1099 :
1100 : // packetId
1101 0 : uint16_t netPacketId = htons(cominfo.packetId);
1102 0 : tlvData.resize(tlvData.size() + sizeof(netPacketId));
1103 0 : (void)memcpy_s(tlvData.data() + tlvData.size() - sizeof(netPacketId), sizeof(netPacketId), &netPacketId, sizeof(netPacketId));
1104 :
1105 : // rev
1106 0 : uint16_t netRev = htons(cominfo.rev);
1107 0 : tlvData.resize(tlvData.size() + sizeof(netRev));
1108 0 : (void)memcpy_s(tlvData.data() + tlvData.size() - sizeof(netRev), sizeof(netRev), &netRev, sizeof(netRev));
1109 :
1110 : // packetNum
1111 0 : uint16_t netPacketNum = htons(cominfo.packetNum);
1112 0 : tlvData.resize(tlvData.size() + sizeof(netPacketNum));
1113 0 : (void)memcpy_s(tlvData.data() + tlvData.size() - sizeof(netPacketNum), sizeof(netPacketNum), &netPacketNum, sizeof(netPacketNum));
1114 :
1115 : // revSecond
1116 0 : uint16_t netRevSecond = htons(cominfo.revSecond);
1117 0 : tlvData.resize(tlvData.size() + sizeof(netRevSecond));
1118 0 : (void)memcpy_s(tlvData.data() + tlvData.size() - sizeof(netRevSecond), sizeof(netRevSecond), &netRevSecond, sizeof(netRevSecond));
1119 :
1120 : // sendRankInfo
1121 0 : for (const auto& rankInfo : cominfo.sendRankInfo) {
1122 0 : uint32_t netDeviceIp = htonl(rankInfo.deviceIp);
1123 0 : tlvData.resize(tlvData.size() + sizeof(netDeviceIp));
1124 0 : (void)memcpy_s(tlvData.data() + tlvData.size() - sizeof(netDeviceIp), sizeof(netDeviceIp), &netDeviceIp, sizeof(netDeviceIp));
1125 : }
1126 :
1127 : // CommMd5Sum
1128 0 : tlvData.insert(tlvData.end(), cominfo.commMd5Sum, cominfo.commMd5Sum + sizeof(cominfo.commMd5Sum));
1129 :
1130 : // RankTotalNum
1131 0 : uint16_t netRankTotalNum = htons(cominfo.rankTotalNum);
1132 0 : tlvData.resize(tlvData.size() + sizeof(netRankTotalNum));
1133 0 : (void)memcpy_s(tlvData.data() + tlvData.size() - sizeof(netRankTotalNum), sizeof(netRankTotalNum), &netRankTotalNum, sizeof(netRankTotalNum));
1134 :
1135 : // Rank Number
1136 0 : uint16_t netRankNum = htons(cominfo.rankNum);
1137 0 : tlvData.resize(tlvData.size() + sizeof(netRankNum));
1138 0 : (void)memcpy_s(tlvData.data() + tlvData.size() - sizeof(netRankNum), sizeof(netRankNum), &netRankNum, sizeof(netRankNum));
1139 :
1140 : // Rank Info
1141 0 : for (const auto& rankInfo : cominfo.rankInfo) {
1142 0 : uint32_t netDeviceIp = htonl(rankInfo.deviceIp);
1143 0 : uint32_t netServerIp = htonl(rankInfo.serverIp);
1144 0 : uint16_t netPodId = htons(rankInfo.podId);
1145 0 : uint16_t netRev3 = htons(rankInfo.rev);
1146 0 : tlvData.resize(tlvData.size() + sizeof(netDeviceIp));
1147 0 : (void)memcpy_s(tlvData.data() + tlvData.size() - sizeof(netDeviceIp), sizeof(netDeviceIp), &netDeviceIp, sizeof(netDeviceIp));
1148 0 : tlvData.resize(tlvData.size() + sizeof(netServerIp));
1149 0 : (void)memcpy_s(tlvData.data() + tlvData.size() - sizeof(netServerIp), sizeof(netServerIp), &netServerIp, sizeof(netServerIp));
1150 0 : tlvData.resize(tlvData.size() + sizeof(netPodId));
1151 0 : (void)memcpy_s(tlvData.data() + tlvData.size() - sizeof(netPodId), sizeof(netPodId), &netPodId, sizeof(netPodId));
1152 0 : tlvData.resize(tlvData.size() + sizeof(netRev3));
1153 0 : (void)memcpy_s(tlvData.data() + tlvData.size() - sizeof(netRev3), sizeof(netRev3), &netRev3, sizeof(netRev3));
1154 : }
1155 0 : HCCL_DEBUG("[NSLB-DP] entry serializeTLV TBL_COMM_INFO end");
1156 :
1157 0 : return tlvData;
1158 0 : }
1159 :
1160 0 : void hcclNslbDp::fullCommConfigInfo(NslbDpCommConfigInfo &tab_f, NslbDpCommConfigVal cominfo, u32 packetNum)
1161 : {
1162 0 : HCCL_DEBUG("[NSLB-DP] entry full CommConfigInfo");
1163 0 : tab_f.taskId = cominfo.taskId;
1164 0 : s32 sRet = memcpy_s(tab_f.commDesc, sizeof(tab_f.commDesc), cominfo.commDesc, COMM_DESC_MAX_LENGTH);
1165 0 : if (sRet != EOK) {
1166 0 : HCCL_INFO("memcpy_s commDesc fail");
1167 : }
1168 0 : tab_f.commInitTime = cominfo.commInitTime;
1169 :
1170 0 : tab_f.rev = 0;
1171 0 : tab_f.packetNum = packetNum;
1172 0 : tab_f.revSecond = 0;
1173 0 : for (u32 dip = 0; dip < packetNum; dip++) {
1174 0 : if(cominfo.rankInfo.size() == 0) {
1175 0 : break;
1176 : }
1177 0 : tab_f.sendRankInfo[dip].deviceIp = cominfo.rankInfo[dip].deviceIp;
1178 : }
1179 0 : sRet = memcpy_s(tab_f.commMd5Sum, sizeof(tab_f.commMd5Sum), cominfo.commMd5Sum, sizeof(cominfo.commMd5Sum));
1180 0 : if (sRet != EOK) {
1181 0 : HCCL_INFO("memcpy_s commmd5 fail");
1182 : }
1183 0 : }
1184 :
1185 : /* 根据表1 中的rank数量进行分片处理 */
1186 0 : HcclResult hcclNslbDp::SendTableProc(u32 rank, u32 packetNum, NslbDpCommConfigVal cominfo)
1187 : {
1188 0 : if (packetNum - 1 < rank) {
1189 0 : HCCL_INFO("[NSLB-DP] rank [%u] no need send TableFir(packetNum[%u]).", rank, packetNum);
1190 0 : return HCCL_SUCCESS;
1191 : }
1192 0 : u32 packetIndex = rank;
1193 0 : NslbDpCommConfigInfo tab_f = {};
1194 0 : fullCommConfigInfo(tab_f, cominfo, packetNum);
1195 0 : if (packetIndex != packetNum - 1) {
1196 0 : tab_f.packetId = packetIndex;
1197 0 : if(cominfo.rankInfo.size() < packetIndex * NSLBDP_RANKTOTALNUM_BLOCK_FIR) {
1198 0 : HCCL_INFO("[NSLB-DP] Comm RankInfo not as expected");
1199 0 : return HCCL_SUCCESS;
1200 : }
1201 0 : for (u32 rankindex = packetIndex * NSLBDP_RANKTOTALNUM_BLOCK_FIR; rankindex < (packetIndex + 1) * NSLBDP_RANKTOTALNUM_BLOCK_FIR; rankindex++) {
1202 : NslbDpRankInfo rankInfoTemp;
1203 0 : rankInfoTemp.deviceIp = cominfo.rankInfo[rankindex].deviceIp;
1204 0 : rankInfoTemp.serverIp = cominfo.rankInfo[rankindex].serverIp;
1205 0 : rankInfoTemp.podId = cominfo.rankInfo[rankindex].podId;
1206 0 : rankInfoTemp.rev = cominfo.rankInfo[rankindex].rev;
1207 0 : tab_f.rankInfo.push_back(rankInfoTemp);
1208 : }
1209 0 : tab_f.rankTotalNum = cominfo.rankTotalNum;
1210 0 : tab_f.rankNum = tab_f.rankInfo.size();
1211 0 : HCCL_INFO("[NSLB-DP] SendTableProc-F info:[%u]-[%u]-[%u]-[%u].",
1212 : packetNum, packetIndex, cominfo.rankTotalNum, tab_f.rankInfo.size());
1213 : } else {
1214 0 : tab_f.packetId = packetIndex;
1215 0 : for (u32 j = packetIndex * NSLBDP_RANKTOTALNUM_BLOCK_FIR; j < cominfo.rankInfo.size(); j++) {
1216 : NslbDpRankInfo rankInfoTemp;
1217 0 : rankInfoTemp.deviceIp = cominfo.rankInfo[j].deviceIp;
1218 0 : rankInfoTemp.serverIp = cominfo.rankInfo[j].serverIp;
1219 0 : rankInfoTemp.podId = cominfo.rankInfo[j].podId;
1220 0 : rankInfoTemp.rev = cominfo.rankInfo[j].rev;
1221 0 : tab_f.rankInfo.push_back(rankInfoTemp);
1222 : }
1223 0 : tab_f.rankTotalNum = cominfo.rankTotalNum;
1224 0 : tab_f.rankNum = tab_f.rankInfo.size();
1225 0 : HCCL_INFO("[NSLB-DP] SendTableProc-N info:[%u]-[%u]-[%u]-[%u].",
1226 : packetNum, packetIndex, cominfo.rankTotalNum, tab_f.rankInfo.size());
1227 : }
1228 0 : HCCL_DEBUG("[NSLB-DP] SendRankTable-info:[%u]-[%u]-[%u]-[%u].",
1229 : tab_f.rankTotalNum, tab_f.packetNum, tab_f.rankTotalNum, tab_f.rankInfo.size());
1230 0 : SendRankTable(tab_f);
1231 :
1232 0 : return HCCL_SUCCESS;
1233 0 : }
1234 :
1235 : /* 遍历表1 执行send 流程 */
1236 1 : HcclResult hcclNslbDp::SendTableFir(uint32_t rank)
1237 : {
1238 1 : size_t size = hcclNslbDpCommConfig_.size();
1239 1 : HCCL_INFO("[NSLB-DP] SendTableFir size:[%u] success.", size);
1240 1 : for (size_t i = 0; i < hcclNslbDpCommConfig_.size(); i++) {
1241 0 : u32 rankTotalNum = hcclNslbDpCommConfig_[i].rankTotalNum;
1242 0 : if (rankTotalNum > NSLBDP_RANKTOTALNUM_BLOCK_FOU) {
1243 0 : return HCCL_SUCCESS;
1244 : }
1245 0 : u32 packetNum = NSLBDP_PKTNUM_FIR;
1246 0 : if (rankTotalNum <= NSLBDP_RANKTOTALNUM_BLOCK_FIR) {
1247 0 : packetNum = NSLBDP_PKTNUM_FIR;
1248 0 : } else if (rankTotalNum <= NSLBDP_RANKTOTALNUM_BLOCK_SEC) {
1249 0 : packetNum = NSLBDP_PKTNUM_SEC;
1250 0 : } else if (rankTotalNum <= NSLBDP_RANKTOTALNUM_BLOCK_THR) {
1251 0 : packetNum = NSLBDP_PKTNUM_THR;
1252 : } else {
1253 0 : packetNum = NSLBDP_PKTNUM_FOU;
1254 : }
1255 0 : SendTableProc(rank, packetNum, hcclNslbDpCommConfig_[i]);
1256 : }
1257 1 : HCCL_DEBUG("[NSLB-DP] entry SendTableFir end");
1258 1 : return HCCL_SUCCESS;
1259 : }
1260 :
1261 :
1262 0 : HcclResult hcclNslbDp::SendRankTable(NslbDpCommConfigInfo tab_f)
1263 : {
1264 0 : if (GetInitNetCoFlag() == false) {
1265 0 : return HCCL_SUCCESS;
1266 : }
1267 :
1268 0 : u32 tablen = sizeof(tab_f);
1269 0 : HCCL_INFO("[NSLB-DP] SendRankTable NslbDpCommConfigInfo len:[%u].", tablen);
1270 :
1271 0 : std::vector<uint8_t> tlvData = serializeTLV_TableFir(tab_f);
1272 0 : u32 datlen = tlvData.size();
1273 0 : HCCL_INFO("[NSLB-DP] SendRankTable tlvData.len:[%u] success.", datlen);
1274 :
1275 0 : if(nslbdp_handle_ == nullptr) {
1276 0 : HCCL_INFO("[NSLB-DP] ndlbdp nslbdp_handle_ error SendRankTable.");
1277 0 : return HCCL_SUCCESS;
1278 : }
1279 :
1280 0 : nslb_msg sendMsg;
1281 0 : nslb_msg recvMsg;
1282 0 : sendMsg.type = NSLBDP_TYPE_TBL_COMM_INFO;
1283 0 : sendMsg.length = datlen;
1284 0 : sendMsg.data.assign(tlvData.begin(), tlvData.end());
1285 0 : s32 ret = H2DTlvRequest(nslbdp_handle_, MODULE_TYPE_NSLB,
1286 0 : reinterpret_cast<TlvMsg*>(&sendMsg), reinterpret_cast<TlvMsg*>(&recvMsg));
1287 :
1288 0 : HCCL_INFO("[NSLBDP-SENDTABLE] hccl send table NSLBDP_TYPE_TBL_COMM_INFO(1001) to hccp. ret(%d)\n", ret);
1289 0 : return HCCL_SUCCESS;
1290 0 : }
1291 :
1292 :
1293 : /* 根将表2 序列化处理 */
1294 0 : std::vector<uint8_t> hcclNslbDp::serializeTLV_TableOpAndAdj(NslbDpOperatorInfo &info)
1295 : {
1296 0 : HCCL_DEBUG("[NSLB-DP] ndlbdp entry serializeTLV TableOpAndAdj.");
1297 0 : std::vector<uint8_t> tlvData;
1298 :
1299 : // 处理 taskId
1300 0 : uint64_t netTaskId = htobe64(info.taskId);
1301 0 : tlvData.resize(tlvData.size() + sizeof(netTaskId));
1302 0 : (void)memcpy_s(tlvData.data() + tlvData.size() - sizeof(netTaskId), sizeof(netTaskId), &netTaskId, sizeof(netTaskId));
1303 :
1304 : // 处理 commDesc
1305 0 : tlvData.insert(tlvData.end(), info.commDesc, info.commDesc + COMM_DESC_MAX_LENGTH);
1306 :
1307 : // 处理 commInitTime
1308 0 : uint64_t netCommInitTime = htobe64(info.commInitTime);
1309 0 : tlvData.resize(tlvData.size() + sizeof(netCommInitTime));
1310 0 : (void)memcpy_s(tlvData.data() + tlvData.size() - sizeof(netCommInitTime), sizeof(netCommInitTime), &netCommInitTime, sizeof(netCommInitTime));
1311 :
1312 : // 处理 oper
1313 0 : tlvData.push_back(info.oper);
1314 :
1315 : // 处理 algorithm
1316 0 : tlvData.push_back(info.algorithm);
1317 :
1318 : // 处理 rootRank
1319 0 : uint16_t netRootRank = htons(info.rootRank);
1320 0 : tlvData.resize(tlvData.size() + sizeof(netRootRank));
1321 0 : (void)memcpy_s(tlvData.data() + tlvData.size() - sizeof(netRootRank), sizeof(netRootRank), &netRootRank, sizeof(netRootRank));
1322 :
1323 : // 处理 trafficCnt
1324 0 : uint64_t netTrafficCnt = htobe64(info.trafficCnt);
1325 0 : tlvData.resize(tlvData.size() + sizeof(netTrafficCnt));
1326 0 : (void)memcpy_s(tlvData.data() + tlvData.size() - sizeof(netTrafficCnt), sizeof(netTrafficCnt), &netTrafficCnt, sizeof(netTrafficCnt));
1327 :
1328 : // 处理 l4SPortId
1329 0 : uint16_t netL4SPortId = htons(info.l4SPortId);
1330 0 : tlvData.resize(tlvData.size() + sizeof(netL4SPortId));
1331 0 : (void)memcpy_s(tlvData.data() + tlvData.size() - sizeof(netL4SPortId), sizeof(netL4SPortId), &netL4SPortId, sizeof(netL4SPortId));
1332 :
1333 : // 处理 maskLen
1334 0 : uint16_t netMaskLen = htons(info.maskLen);
1335 0 : tlvData.resize(tlvData.size() + sizeof(netMaskLen));
1336 0 : (void)memcpy_s(tlvData.data() + tlvData.size() - sizeof(netMaskLen), sizeof(netMaskLen), &netMaskLen, sizeof(netMaskLen));
1337 0 : HCCL_INFO("[NSLB-DP] entry serializeTLV_TableOpAndAdj end.");
1338 0 : return tlvData;
1339 0 : }
1340 :
1341 :
1342 : /* 执行send 流程 */
1343 0 : HcclResult hcclNslbDp::SendRankTableOpAndAdj(NslbDpOperatorInfo &tab_f)
1344 : {
1345 0 : if (GetInitNetCoFlag() == false) {
1346 0 : return HCCL_SUCCESS;
1347 : }
1348 0 : HCCL_DEBUG("[NSLB-DP] ndlbdp entry SendRankTableOpAndAdj.");
1349 0 : std::vector<uint8_t> tlvData = serializeTLV_TableOpAndAdj(tab_f);
1350 :
1351 0 : u32 datlen = tlvData.size();
1352 0 : HCCL_INFO("[NSLB-DP] SendRankTableOpAndAdj tlvData.len:[%u] success.", datlen);
1353 :
1354 0 : nslb_msg sendMsg;
1355 0 : nslb_msg recvMsg;
1356 0 : sendMsg.type = NSLBDP_TYPE_TBL_OPER;
1357 0 : sendMsg.length = datlen;
1358 0 : sendMsg.data.assign(tlvData.begin(), tlvData.end());
1359 0 : s32 ret = H2DTlvRequest(nslbdp_handle_, MODULE_TYPE_NSLB,
1360 0 : reinterpret_cast<TlvMsg*>(&sendMsg), reinterpret_cast<TlvMsg*>(&recvMsg));
1361 :
1362 0 : HCCL_INFO("[NSLBDP-SENDTABLE] hccl send table NSLBDP_TYPE_TBL_OPER(1002) to hccp. ret(%d)\n", ret);
1363 :
1364 0 : return HCCL_SUCCESS;
1365 0 : }
1366 :
1367 :
1368 : /* 遍历表2 执行send 流程 */
1369 0 : HcclResult hcclNslbDp::SendOpAndAdjTable()
1370 : {
1371 0 : HCCL_DEBUG("[NSLB-DP] ndlbdp entry SendOpAndAdjTable.");
1372 0 : for (size_t i = 0; i < hcclNslbDpOperatorVal_.size(); i++) {
1373 0 : if (hcclNslbDpOperatorVal_[i].sedFlag == 1) {
1374 0 : continue;
1375 : }
1376 0 : NslbDpOperatorInfo tab_f = {};
1377 0 : tab_f.taskId = hcclNslbDpOperatorVal_[i].taskId;
1378 0 : s32 sRet = memcpy_s(tab_f.commDesc, sizeof(tab_f.commDesc), hcclNslbDpOperatorVal_[i].commDesc, COMM_DESC_MAX_LENGTH);
1379 0 : if (sRet != EOK) {
1380 0 : HCCL_ERROR("memcpy_s commDesc fail");
1381 : }
1382 0 : tab_f.commInitTime = hcclNslbDpOperatorVal_[i].commInitTime;
1383 0 : tab_f.rootRank = hcclNslbDpOperatorVal_[i].rootRank;
1384 0 : tab_f.oper = hcclNslbDpOperatorVal_[i].oper;
1385 0 : tab_f.algorithm = hcclNslbDpOperatorVal_[i].algorithm;
1386 0 : tab_f.trafficCnt = hcclNslbDpOperatorVal_[i].trafficCnt;
1387 0 : tab_f.l4SPortId = hcclNslbDpOperatorVal_[i].l4SPortId;
1388 0 : tab_f.maskLen = hcclNslbDpOperatorVal_[i].maskLen;
1389 0 : SendRankTableOpAndAdj(tab_f);
1390 0 : hcclNslbDpOperatorVal_[i].sedFlag = 1;
1391 0 : HCCL_INFO("[NSLB-DP] try to sen RankTableOpAndAdj times:[%u].", i);
1392 : }
1393 0 : HCCL_INFO("[NSLB-DP] SendOpAndAdjTable end.");
1394 0 : return HCCL_SUCCESS;
1395 : }
1396 :
1397 :
1398 : /* 根将表3 序列化处理 */
1399 0 : std::vector<uint8_t> hcclNslbDp::serializeTLV_TableAlgorithmInfo(NslbDpAlgorithmTlv &info)
1400 : {
1401 0 : HCCL_DEBUG("[NSLB-DP] entry serializeTLV_TableAlgorithmInfo.");
1402 0 : std::vector<uint8_t> tlvData;
1403 :
1404 : // 处理 taskId
1405 0 : uint64_t netTaskId = htobe64(info.taskId);
1406 0 : tlvData.resize(tlvData.size() + sizeof(netTaskId));
1407 0 : (void)memcpy_s(tlvData.data() + tlvData.size() - sizeof(netTaskId), sizeof(netTaskId), &netTaskId, sizeof(netTaskId));
1408 :
1409 : // 处理 commDesc
1410 0 : tlvData.insert(tlvData.end(), info.commDesc, info.commDesc + COMM_DESC_MAX_LENGTH);
1411 :
1412 : // CommMd5Sum
1413 0 : tlvData.insert(tlvData.end(), info.commMd5Sum, info.commMd5Sum + sizeof(info.commMd5Sum));
1414 :
1415 : // 处理 srcLocalRankId
1416 0 : uint16_t netSrcLocalRankId = htons(info.srcLocalRankId);
1417 0 : tlvData.resize(tlvData.size() + sizeof(netSrcLocalRankId));
1418 0 : (void)memcpy_s(tlvData.data() + tlvData.size() - sizeof(netSrcLocalRankId), sizeof(netSrcLocalRankId), &netSrcLocalRankId, sizeof(netSrcLocalRankId));
1419 :
1420 : // 处理 oper
1421 0 : tlvData.push_back(info.oper);
1422 :
1423 : // 处理 algorithm
1424 0 : tlvData.push_back(info.algorithm);
1425 :
1426 : // 处理 rootRank
1427 0 : uint16_t netRootRank = htons(info.rootRank);
1428 0 : tlvData.resize(tlvData.size() + sizeof(netRootRank));
1429 0 : (void)memcpy_s(tlvData.data() + tlvData.size() - sizeof(netRootRank), sizeof(netRootRank), &netRootRank, sizeof(netRootRank));
1430 :
1431 : // 处理 rev
1432 0 : uint16_t netRev = htons(info.rev);
1433 0 : tlvData.resize(tlvData.size() + sizeof(netRev));
1434 0 : (void)memcpy_s(tlvData.data() + tlvData.size() - sizeof(netRev), sizeof(netRev), &netRev, sizeof(netRev));
1435 :
1436 : // 处理 dstRankNum
1437 0 : uint16_t netDstRankNum = htons(info.dstRankNum);
1438 0 : tlvData.resize(tlvData.size() + sizeof(netDstRankNum));
1439 0 : (void)memcpy_s(tlvData.data() + tlvData.size() - sizeof(netDstRankNum), sizeof(netDstRankNum), &netDstRankNum, sizeof(netDstRankNum));
1440 :
1441 : // 处理 revSecond
1442 0 : uint16_t netRevSecond = htons(info.revsecond);
1443 0 : tlvData.resize(tlvData.size() + sizeof(netRevSecond));
1444 0 : (void)memcpy_s(tlvData.data() + tlvData.size() - sizeof(netRevSecond), sizeof(netRevSecond), &netRevSecond, sizeof(netRevSecond));
1445 :
1446 0 : HCCL_INFO("[NSLB-DP] serializeTLV_TableAlgorithmInfo proc adjInfo.");
1447 : // 处理 adjInfo
1448 0 : for (const auto& adj : info.AdjInfo) {
1449 0 : uint16_t netDstLocalRankId = htons(adj.dstLocalRankId);
1450 0 : tlvData.resize(tlvData.size() + sizeof(netDstLocalRankId));
1451 0 : (void)memcpy_s(tlvData.data() + tlvData.size() - sizeof(netDstLocalRankId), sizeof(netDstLocalRankId), &netDstLocalRankId, sizeof(netDstLocalRankId));
1452 0 : tlvData.push_back(adj.phaseId);
1453 0 : tlvData.push_back(adj.rev);
1454 0 : HCCL_INFO("[NSLB-DP] serializeTLV_TableAlgo adjInfo:***[%u]***[%c]***.", adj.dstLocalRankId, adj.phaseId);
1455 : }
1456 0 : HCCL_DEBUG("[NSLB-DP] serializeTLV_TableAlgo proc adjInfo end.");
1457 :
1458 0 : return tlvData;
1459 0 : }
1460 :
1461 :
1462 0 : HcclResult hcclNslbDp::SendRankTableAlgorithmInfo(NslbDpAlgorithmTlv &tab_f)
1463 : {
1464 0 : if (GetInitNetCoFlag() == false) {
1465 0 : return HCCL_SUCCESS;
1466 : }
1467 0 : std::vector<uint8_t> tlvData = serializeTLV_TableAlgorithmInfo(tab_f);
1468 :
1469 0 : u32 datlen = tlvData.size();
1470 0 : HCCL_INFO("[NSLB-DP] SendRankTableAlgorithmInfo tlvData.len:[%u] success.", datlen);
1471 :
1472 0 : nslb_msg sendMsg;
1473 0 : nslb_msg recvMsg;
1474 0 : sendMsg.type = NSLBDP_TYPE_TBL_ADJ;
1475 0 : sendMsg.length = datlen;
1476 0 : sendMsg.data.assign(tlvData.begin(), tlvData.end());
1477 0 : s32 ret = H2DTlvRequest(nslbdp_handle_, MODULE_TYPE_NSLB,
1478 0 : reinterpret_cast<TlvMsg*>(&sendMsg), reinterpret_cast<TlvMsg*>(&recvMsg));
1479 :
1480 0 : HCCL_INFO("[NSLBDP-SENDTABLE] hccl send table NSLBDP_TYPE_TBL_ADJ(1003) to hccp. ret(%d)\n", ret);
1481 0 : return HCCL_SUCCESS;
1482 0 : }
1483 :
1484 : /* 遍历表3 执行send 流程 */
1485 0 : HcclResult hcclNslbDp::SendAlgorithmInfoTable()
1486 : {
1487 0 : u32 size = hcclNslbDpAlgorithmInfo_.size();
1488 0 : HCCL_INFO("[NSLB-DP] ndlbdp entry SendAlgorithmInfoTable size=[%u].", size);
1489 :
1490 0 : for (size_t i = 0; i < hcclNslbDpAlgorithmInfo_.size(); i++) {
1491 0 : if (hcclNslbDpAlgorithmInfo_[i].sedFlag == 1) {
1492 0 : continue;
1493 : }
1494 0 : NslbDpAlgorithmTlv tab_f = {};
1495 0 : tab_f.taskId = hcclNslbDpAlgorithmInfo_[i].taskId;
1496 0 : s32 sRet = memcpy_s(tab_f.commDesc, sizeof(tab_f.commDesc), hcclNslbDpAlgorithmInfo_[i].commDesc, COMM_DESC_MAX_LENGTH);
1497 0 : sRet = memcpy_s(tab_f.commMd5Sum, sizeof(tab_f.commMd5Sum),
1498 0 : hcclNslbDpAlgorithmInfo_[i].commMd5Sum, sizeof(hcclNslbDpAlgorithmInfo_[i].commMd5Sum));
1499 0 : if (sRet != EOK) {
1500 0 : HCCL_INFO("memcpy_s commDesc fail");
1501 : }
1502 0 : tab_f.srcLocalRankId = hcclNslbDpAlgorithmInfo_[i].srcLocalRankId;
1503 0 : tab_f.rootRank = hcclNslbDpAlgorithmInfo_[i].rootRank;
1504 0 : tab_f.oper = hcclNslbDpAlgorithmInfo_[i].oper;
1505 0 : tab_f.algorithm = hcclNslbDpAlgorithmInfo_[i].algorithm;
1506 :
1507 0 : tab_f.rev = 0;
1508 :
1509 0 : tab_f.dstRankNum = hcclNslbDpAlgorithmInfo_[i].dstRankNum;
1510 0 : tab_f.revsecond = 0;
1511 0 : tab_f.AdjInfo = hcclNslbDpAlgorithmInfo_[i].AdjInfo;
1512 0 : for (const auto& adj : hcclNslbDpAlgorithmInfo_[i].AdjInfo) {
1513 0 : tab_f.AdjInfo.push_back(adj);
1514 : }
1515 0 : SendRankTableAlgorithmInfo(tab_f);
1516 0 : HCCL_INFO("[NSLB-DP] try to sen AlgorithmInfoTable times:[%u].", i);
1517 :
1518 0 : hcclNslbDpAlgorithmInfo_[i].sedFlag = 1;
1519 0 : }
1520 0 : HCCL_DEBUG("[NSLB-DP] entry SendAlgorithmInfoTable end.");
1521 :
1522 0 : return HCCL_SUCCESS;
1523 : }
1524 :
1525 : /* 序列化表4 */
1526 0 : std::vector<uint8_t> hcclNslbDp::serializeTLV_TableGlobalRankInfo(NslbDpGlobalRankInfo &info)
1527 : {
1528 0 : HCCL_DEBUG("[NSLB-DP] entry serializeTLV_TableGlobalRankInfo.");
1529 0 : std::vector<uint8_t> tlvData;
1530 : // 处理 taskId
1531 0 : uint64_t netTaskId = htobe64(info.taskId);
1532 0 : tlvData.resize(tlvData.size() + sizeof(netTaskId));
1533 0 : (void)memcpy_s(tlvData.data() + tlvData.size() - sizeof(netTaskId), sizeof(netTaskId), &netTaskId, sizeof(netTaskId));
1534 :
1535 : // 处理 commDesc
1536 0 : tlvData.insert(tlvData.end(), info.commDesc, info.commDesc + COMM_DESC_MAX_LENGTH);
1537 :
1538 : // 处理 commInitTime
1539 0 : uint64_t netCommInitTime = htobe64(info.commInitTime);
1540 0 : tlvData.resize(tlvData.size() + sizeof(netCommInitTime));
1541 0 : (void)memcpy_s(tlvData.data() + tlvData.size() - sizeof(netCommInitTime), sizeof(netCommInitTime), &netCommInitTime, sizeof(netCommInitTime));
1542 :
1543 : // 处理 packetId
1544 0 : uint16_t netPacketId = htons(info.packetId);
1545 0 : tlvData.resize(tlvData.size() + sizeof(netPacketId));
1546 0 : (void)memcpy_s(tlvData.data() + tlvData.size() - sizeof(netPacketId), sizeof(netPacketId), &netPacketId, sizeof(netPacketId));
1547 :
1548 : // 处理 rev
1549 0 : uint16_t netRev = htons(info.rev);
1550 0 : tlvData.resize(tlvData.size() + sizeof(netRev));
1551 0 : (void)memcpy_s(tlvData.data() + tlvData.size() - sizeof(netRev), sizeof(netRev), &netRev, sizeof(netRev));
1552 :
1553 : // 处理 packetNum
1554 0 : uint16_t netPacketNum = htons(info.packetNum);
1555 0 : tlvData.resize(tlvData.size() + sizeof(netPacketNum));
1556 0 : (void)memcpy_s(tlvData.data() + tlvData.size() - sizeof(netPacketNum), sizeof(netPacketNum), &netPacketNum, sizeof(netPacketNum));
1557 0 : HCCL_INFO("[NSLB-DP] hcclNslbDpGlobalRankVal_ info.packetNum:[%u]", netPacketNum);
1558 :
1559 : // 处理 rev2
1560 0 : uint16_t netRev2 = htons(info.rev2);
1561 0 : tlvData.resize(tlvData.size() + sizeof(netRev2));
1562 0 : (void)memcpy_s(tlvData.data() + tlvData.size() - sizeof(netRev2), sizeof(netRev2), &netRev2, sizeof(netRev2));
1563 :
1564 : // 处理 sendRankInfo
1565 0 : for (const auto& rankInfo : info.sendRankInfo) {
1566 0 : uint32_t netDeviceIp = htonl(rankInfo.deviceIp);
1567 0 : tlvData.resize(tlvData.size() + sizeof(netDeviceIp));
1568 0 : (void)memcpy_s(tlvData.data() + tlvData.size() - sizeof(netDeviceIp), sizeof(netDeviceIp), &netDeviceIp, sizeof(netDeviceIp));
1569 : }
1570 :
1571 : // 处理 commMd5Sum
1572 0 : tlvData.insert(tlvData.end(), info.commMd5Sum, info.commMd5Sum + sizeof(info.commMd5Sum));
1573 :
1574 : // 处理 rankTotalNum
1575 0 : uint32_t netRankTotalNum = htonl(info.rankTotalNum);
1576 0 : tlvData.resize(tlvData.size() + sizeof(netRankTotalNum));
1577 0 : (void)memcpy_s(tlvData.data() + tlvData.size() - sizeof(netRankTotalNum), sizeof(netRankTotalNum), &netRankTotalNum, sizeof(netRankTotalNum));
1578 :
1579 : // 处理 rankNum
1580 0 : uint16_t netRankNum = htons(info.rankNum);
1581 0 : tlvData.resize(tlvData.size() + sizeof(netRankNum));
1582 0 : (void)memcpy_s(tlvData.data() + tlvData.size() - sizeof(netRankNum), sizeof(netRankNum), &netRankNum, sizeof(netRankNum));
1583 :
1584 : // 处理 rev3
1585 0 : uint16_t netRev3 = htons(info.rev3);
1586 0 : tlvData.resize(tlvData.size() + sizeof(netRev3));
1587 0 : (void)memcpy_s(tlvData.data() + tlvData.size() - sizeof(netRev3), sizeof(netRev3), &netRev3, sizeof(netRev3));
1588 :
1589 : // 处理 rankInfo
1590 0 : for (const auto& rankInfo : info.rankInfo) {
1591 0 : uint32_t netDeviceIp = htonl(rankInfo.deviceIp);
1592 0 : uint32_t netServerIp = htonl(rankInfo.serverIp);
1593 0 : tlvData.resize(tlvData.size() + sizeof(netDeviceIp));
1594 0 : (void)memcpy_s(tlvData.data() + tlvData.size() - sizeof(netDeviceIp), sizeof(netDeviceIp), &netDeviceIp, sizeof(netDeviceIp));
1595 0 : tlvData.resize(tlvData.size() + sizeof(netServerIp));
1596 0 : (void)memcpy_s(tlvData.data() + tlvData.size() - sizeof(netServerIp), sizeof(netServerIp), &netServerIp, sizeof(netServerIp));
1597 : }
1598 0 : return tlvData;
1599 0 : }
1600 :
1601 :
1602 0 : HcclResult hcclNslbDp::SendRankTableGlobalRank(NslbDpGlobalRankInfo &tab_f)
1603 : {
1604 0 : if (GetInitNetCoFlag() == false) {
1605 0 : return HCCL_SUCCESS;
1606 : }
1607 0 : std::vector<uint8_t> tlvData = serializeTLV_TableGlobalRankInfo(tab_f);
1608 0 : u32 datlen = tlvData.size();
1609 0 : HCCL_INFO("[NSLB-DP] SendRankTableGlobalRank tlvData.len:[%u] success.", datlen);
1610 :
1611 0 : nslb_msg sendMsg;
1612 0 : nslb_msg recvMsg;
1613 0 : sendMsg.type = NSLBDP_TYPE_TBL_RANK;
1614 0 : sendMsg.length = datlen;
1615 0 : sendMsg.data.assign(tlvData.begin(), tlvData.end());
1616 0 : s32 ret = H2DTlvRequest(nslbdp_handle_, MODULE_TYPE_NSLB,
1617 0 : reinterpret_cast<TlvMsg*>(&sendMsg), reinterpret_cast<TlvMsg*>(&recvMsg));
1618 :
1619 0 : HCCL_INFO("[NSLBDP-SENDTABLE] hccl send table NSLBDP_TYPE_TBL_RANK(1004) to hccp. ret(%d)\n", ret);
1620 :
1621 0 : return HCCL_SUCCESS;
1622 0 : }
1623 :
1624 0 : void hcclNslbDp::fullCommonGlobalRankInfo(NslbDpGlobalRankInfo tab_f, NslbDpGlobalRankVal &cominfo)
1625 : {
1626 0 : tab_f.rev = 0;
1627 0 : tab_f.rev2 = 0;
1628 0 : tab_f.rev3 = 0;
1629 0 : tab_f.taskId = cominfo.taskId;
1630 0 : s32 sRet = memcpy_s(tab_f.commDesc, sizeof(tab_f.commDesc), cominfo.commDesc, sizeof(cominfo.commDesc));
1631 0 : if (sRet != EOK) {
1632 0 : HCCL_INFO("memcpy_s commDesc fail");
1633 : }
1634 0 : tab_f.commInitTime = cominfo.commInitTime;
1635 0 : sRet = memcpy_s(tab_f.commMd5Sum, sizeof(tab_f.commMd5Sum), cominfo.commMd5Sum, sizeof(cominfo.commMd5Sum));
1636 0 : if (sRet != EOK) {
1637 0 : HCCL_ERROR("memcpy_s commMD5 fail");
1638 : }
1639 :
1640 0 : for (u32 dip = 0; dip < tab_f.packetNum; dip++) {
1641 0 : if(cominfo.rankInfo.size() == 0) {
1642 0 : break;
1643 : }
1644 0 : tab_f.sendRankInfo[dip].deviceIp = cominfo.rankInfo[dip].deviceIp;
1645 : }
1646 0 : return;
1647 : }
1648 :
1649 : /* 根据表4 中的rank数量进行分片处理 */
1650 0 : HcclResult hcclNslbDp::SendTableGlobalRankProc(uint32_t rank, uint32_t packetNum, NslbDpGlobalRankVal &cominfo)
1651 : {
1652 0 : HCCL_INFO("[NSLB-DP] entry SendTableGlobalRankProc packetNum:[%u]", packetNum);
1653 0 : if (packetNum - 1 < rank) {
1654 0 : HCCL_INFO("[NSLB-DP] rank [%u] no need send TableFir(packetNum[%u]).", rank, packetNum);
1655 0 : return HCCL_SUCCESS;
1656 : }
1657 0 : u32 packetIndex = rank;
1658 0 : NslbDpGlobalRankInfo tab_f = {};
1659 0 : tab_f.packetId = packetIndex;
1660 0 : tab_f.packetNum = packetNum;
1661 0 : fullCommonGlobalRankInfo(tab_f, cominfo);
1662 0 : if (packetIndex != packetNum - 1) {
1663 0 : uint32_t start = packetIndex * NSLBDP_RANKTOTALNUM_BLOCK_FIR;
1664 0 : uint32_t end = (packetIndex + 1) * NSLBDP_RANKTOTALNUM_BLOCK_FIR;
1665 0 : end = (end > cominfo.rankInfo.size()) ? cominfo.rankInfo.size() : end;
1666 0 : for (uint32_t j = start; j < end; ++j) {
1667 : TableFourRankInfo rankInfoTemp;
1668 0 : rankInfoTemp.deviceIp = cominfo.rankInfo[j].deviceIp;
1669 0 : rankInfoTemp.serverIp = cominfo.rankInfo[j].serverIp;
1670 0 : tab_f.rankInfo.push_back(rankInfoTemp);
1671 : }
1672 : } else {
1673 0 : uint32_t rankNum = cominfo.rankInfo.size();
1674 0 : uint32_t start = packetIndex * NSLBDP_RANKTOTALNUM_BLOCK_FIR;
1675 :
1676 0 : for (uint32_t j = start; j < rankNum; j++) {
1677 : TableFourRankInfo rankInfoTemp;
1678 0 : rankInfoTemp.deviceIp = cominfo.rankInfo[j].deviceIp;
1679 0 : rankInfoTemp.serverIp = cominfo.rankInfo[j].serverIp;
1680 0 : tab_f.rankInfo.push_back(rankInfoTemp);
1681 : }
1682 : }
1683 0 : tab_f.rankTotalNum = cominfo.rankTotalNum;
1684 0 : tab_f.rankNum = tab_f.rankInfo.size();
1685 0 : SendRankTableGlobalRank(tab_f);
1686 0 : return HCCL_SUCCESS;
1687 0 : }
1688 :
1689 : /* 根据rankNum的总数做分片发送流程 */
1690 0 : HcclResult hcclNslbDp::SendGlobalRankTable(uint32_t rank)
1691 : {
1692 0 : HCCL_DEBUG("[NSLB-DP] try to send GlobalRankTable.");
1693 0 : u32 rankTotalNum = hcclNslbDpGlobalRankVal_.rankTotalNum;
1694 0 : if (rankTotalNum > NSLBDP_RANKTOTALNUM_BLOCK_FOU) {
1695 0 : return HCCL_SUCCESS;
1696 : }
1697 0 : u32 packetNum = rankTotalNum / NSLBDP_PKTNUM_FIR;
1698 0 : if (rankTotalNum <= NSLBDP_RANKTOTALNUM_BLOCK_FIR) {
1699 0 : packetNum = NSLBDP_PKTNUM_FIR;
1700 0 : }else if (rankTotalNum <= NSLBDP_RANKTOTALNUM_BLOCK_SEC) {
1701 0 : packetNum = NSLBDP_PKTNUM_SEC;
1702 0 : } else if (rankTotalNum <= NSLBDP_RANKTOTALNUM_BLOCK_THR) {
1703 0 : packetNum = NSLBDP_PKTNUM_THR;
1704 : } else {
1705 0 : packetNum = NSLBDP_PKTNUM_FOU;
1706 : }
1707 0 : SendTableGlobalRankProc(rank, packetNum, hcclNslbDpGlobalRankVal_);
1708 :
1709 0 : return HCCL_SUCCESS;
1710 : }
1711 :
1712 : /* 序列化表5 */
1713 0 : std::vector<uint8_t> hcclNslbDp::serializeTLV_TableGlobalDisRankVal(NslbDpGlobalDisRankVal &info)
1714 : {
1715 0 : HCCL_DEBUG("[NSLB-DP] entry serializeTLV_TableGlobalDisRankVal.");
1716 0 : std::vector<uint8_t> tlvData;
1717 :
1718 : // 处理 taskId
1719 0 : uint64_t netTaskId = htobe64(info.taskId);
1720 0 : tlvData.resize(tlvData.size() + sizeof(netTaskId));
1721 0 : (void)memcpy_s(tlvData.data() + tlvData.size() - sizeof(netTaskId), sizeof(netTaskId),
1722 : &netTaskId, sizeof(netTaskId));
1723 :
1724 : // 处理 npuIp
1725 0 : uint32_t netNpuIp = htonl(info.npuIp);
1726 0 : tlvData.resize(tlvData.size() + sizeof(netNpuIp));
1727 0 : (void)memcpy_s(tlvData.data() + tlvData.size() - sizeof(netNpuIp), sizeof(netNpuIp),
1728 : &netNpuIp, sizeof(netNpuIp));
1729 :
1730 : // 处理 serverIp
1731 0 : uint32_t netServerIp = htonl(info.serverIp);
1732 0 : tlvData.resize(tlvData.size() + sizeof(netServerIp));
1733 0 : (void)memcpy_s(tlvData.data() + tlvData.size() - sizeof(netServerIp), sizeof(netServerIp),
1734 : &netServerIp, sizeof(netServerIp));
1735 :
1736 : // 处理 nodeId
1737 0 : uint32_t netNodeId = htonl(info.nodeId);
1738 0 : tlvData.resize(tlvData.size() + sizeof(netNodeId));
1739 0 : (void)memcpy_s(tlvData.data() + tlvData.size() - sizeof(netNodeId), sizeof(netNodeId),
1740 : &netNodeId, sizeof(netNodeId));
1741 :
1742 : // 处理 localRankNum
1743 0 : tlvData.push_back(info.localRankNum);
1744 :
1745 : // 处理 rev
1746 0 : tlvData.insert(tlvData.end(), info.rev, info.rev + sizeof(info.rev));
1747 :
1748 : // 处理 rankTotalNum
1749 0 : uint32_t netRankTotalNum = htonl(info.rankTotalNum);
1750 0 : tlvData.resize(tlvData.size() + sizeof(netRankTotalNum));
1751 0 : (void)memcpy_s(tlvData.data() + tlvData.size() - sizeof(netRankTotalNum), sizeof(netRankTotalNum),
1752 : &netRankTotalNum, sizeof(netRankTotalNum));
1753 :
1754 0 : return tlvData;
1755 0 : }
1756 :
1757 : /* 发送GlobalDisRank表信息 */
1758 0 : HcclResult hcclNslbDp::SendRankTableGlobalDisRankVal(NslbDpGlobalDisRankVal &tab_f)
1759 : {
1760 0 : if (GetInitNetCoFlag() == false) {
1761 0 : return HCCL_SUCCESS;
1762 : }
1763 0 : std::vector<uint8_t> tlvData = serializeTLV_TableGlobalDisRankVal(tab_f);
1764 0 : u32 datlen = tlvData.size();
1765 0 : HCCL_INFO("[NSLB-DP] SendRankTableGlobalDisRankVal tlvData.len:[%u] success.", datlen);
1766 :
1767 0 : nslb_msg sendMsg;
1768 0 : nslb_msg recvMsg;
1769 0 : sendMsg.type = NSLBDP_TYPE_TBL_RANK_DIST;
1770 0 : sendMsg.length = datlen;
1771 0 : sendMsg.data.assign(tlvData.begin(), tlvData.end());
1772 0 : s32 ret = H2DTlvRequest(nslbdp_handle_, MODULE_TYPE_NSLB,
1773 0 : reinterpret_cast<TlvMsg*>(&sendMsg), reinterpret_cast<TlvMsg*>(&recvMsg));
1774 :
1775 0 : HCCL_INFO("[NSLBDP-SENDTABLE] hccl send table NSLBDP_TYPE_TBL_RANK_DIST(1005) to hccp. ret(%d)\n", ret);
1776 0 : return HCCL_SUCCESS;
1777 0 : }
1778 :
1779 :
1780 : /* 遍历表5 执行send流程 */
1781 0 : HcclResult hcclNslbDp::SendGlobalDisRankTable()
1782 : {
1783 0 : HCCL_DEBUG("[NSLB-DP] ndlbdp entry serializeTLV_TableGlobalDisRankVal.");
1784 0 : NslbDpGlobalDisRankVal tab_f = {};
1785 0 : tab_f.taskId = hcclNslbDpGlobalDisRankVal_.taskId;
1786 0 : tab_f.npuIp = hcclNslbDpGlobalDisRankVal_.npuIp;
1787 0 : tab_f.serverIp = hcclNslbDpGlobalDisRankVal_.serverIp;
1788 0 : tab_f.nodeId = hcclNslbDpGlobalDisRankVal_.nodeId;
1789 0 : tab_f.localRankNum = hcclNslbDpGlobalDisRankVal_.localRankNum;
1790 0 : s32 sRet = memcpy_s(tab_f.rev, sizeof(tab_f.rev), hcclNslbDpGlobalDisRankVal_.rev, sizeof(hcclNslbDpGlobalDisRankVal_.rev));
1791 0 : if (sRet != EOK) {
1792 0 : HCCL_INFO("memcpy_s rev info fail");
1793 : }
1794 0 : tab_f.rankTotalNum = hcclNslbDpGlobalDisRankVal_.rankTotalNum;
1795 :
1796 0 : SendRankTableGlobalDisRankVal(tab_f);
1797 0 : return HCCL_SUCCESS;
1798 : }
1799 :
1800 : /* 序列化表6 */
1801 0 : std::vector<uint8_t> hcclNslbDp::serializeTLV_TableRootRank(NslbDpRootRank &config)
1802 : {
1803 0 : HCCL_DEBUG("[NSLB-DP] ndlbdp entry serializeTLV_TableRootRank.");
1804 0 : std::vector<uint8_t> tlvData;
1805 :
1806 : // 处理 taskId
1807 0 : uint64_t netTaskId = htobe64(config.taskId);
1808 0 : tlvData.resize(tlvData.size() + sizeof(netTaskId));
1809 0 : (void)memcpy_s(tlvData.data() + tlvData.size() - sizeof(netTaskId), sizeof(netTaskId), &netTaskId, sizeof(netTaskId));
1810 :
1811 : // 处理 commDesc
1812 0 : tlvData.insert(tlvData.end(), config.commDesc, config.commDesc + COMM_DESC_MAX_LENGTH);
1813 :
1814 : // 处理 commInitTime
1815 0 : uint64_t netCommInitTime = htobe64(config.commInitTime);
1816 0 : tlvData.resize(tlvData.size() + sizeof(netCommInitTime));
1817 0 : (void)memcpy_s(tlvData.data() + tlvData.size() - sizeof(netCommInitTime), sizeof(netCommInitTime), &netCommInitTime, sizeof(netCommInitTime));
1818 :
1819 : // 处理 oper
1820 0 : tlvData.push_back(config.oper);
1821 :
1822 : // 处理 algorithm
1823 0 : tlvData.push_back(config.algorithm);
1824 :
1825 : // 处理 revfir
1826 0 : uint16_t netRevfir = htons(config.revfir);
1827 0 : tlvData.resize(tlvData.size() + sizeof(netRevfir));
1828 0 : (void)memcpy_s(tlvData.data() + tlvData.size() - sizeof(netRevfir), sizeof(netRevfir), &netRevfir, sizeof(netRevfir));
1829 :
1830 : // 处理 rootRankNum
1831 0 : uint16_t netRootRankNum = htons(config.rootRankNum);
1832 0 : tlvData.resize(tlvData.size() + sizeof(netRootRankNum));
1833 0 : (void)memcpy_s(tlvData.data() + tlvData.size() - sizeof(netRootRankNum), sizeof(netRootRankNum), &netRootRankNum, sizeof(netRootRankNum));
1834 :
1835 : // 处理 revsec
1836 0 : uint16_t netRevsec = htons(config.revsec);
1837 0 : tlvData.resize(tlvData.size() + sizeof(netRevsec));
1838 0 : (void)memcpy_s(tlvData.data() + tlvData.size() - sizeof(netRevsec), sizeof(netRevsec), &netRevsec, sizeof(netRevsec));
1839 :
1840 : // 处理 rankId
1841 0 : for (const auto& id : config.rankId) {
1842 0 : uint32_t netRankId = htonl(id.rankID);
1843 0 : tlvData.resize(tlvData.size() + sizeof(netRankId));
1844 0 : (void)memcpy_s(tlvData.data() + tlvData.size() - sizeof(netRankId), sizeof(netRankId), &netRankId, sizeof(netRankId));
1845 : }
1846 0 : return tlvData;
1847 0 : }
1848 :
1849 : /* 发送Rank表到netco */
1850 0 : HcclResult hcclNslbDp::SendRankTableRootRank(NslbDpRootRank &tab_f)
1851 : {
1852 0 : if (GetInitNetCoFlag() == false) {
1853 0 : return HCCL_SUCCESS;
1854 : }
1855 0 : std::vector<uint8_t> tlvData = serializeTLV_TableRootRank(tab_f);
1856 :
1857 0 : u32 datlen = tlvData.size();
1858 0 : HCCL_DEBUG("[NSLB-DP] SendRankTableRootRank tlvData.len:[%u] success.", datlen);
1859 :
1860 0 : nslb_msg sendMsg;
1861 0 : nslb_msg recvMsg;
1862 0 : sendMsg.type = NSLBDP_TYPE_TBL_ROOT_RANK;
1863 0 : sendMsg.length = datlen;
1864 0 : sendMsg.data.assign(tlvData.begin(), tlvData.end());
1865 0 : s32 ret = H2DTlvRequest(nslbdp_handle_, MODULE_TYPE_NSLB,
1866 0 : reinterpret_cast<TlvMsg*>(&sendMsg), reinterpret_cast<TlvMsg*>(&recvMsg));
1867 :
1868 0 : HCCL_INFO("[NSLBDP-SENDTABLE] hccl send table NSLBDP_TYPE_TBL_ROOT_RANK(1006) to hccp. ret(%d)\n", ret);
1869 0 : return HCCL_SUCCESS;
1870 0 : }
1871 :
1872 :
1873 : /* 遍历表6 执行send流程 */
1874 0 : HcclResult hcclNslbDp::SendRootRankTable()
1875 : {
1876 0 : HCCL_DEBUG("[NSLB-DP] entry SendRootRankTable.");
1877 0 : NslbDpRootRank tab_f = {};
1878 0 : tab_f.taskId = hcclNslbDpRootRankVal_.taskId;
1879 0 : s32 sRet = memcpy_s(tab_f.commDesc, sizeof(tab_f.commDesc), hcclNslbDpRootRankVal_.commDesc, sizeof(hcclNslbDpRootRankVal_.commDesc));
1880 0 : if (sRet != EOK) {
1881 0 : HCCL_INFO("memcpy_s commDesc info fail");
1882 : }
1883 0 : tab_f.commInitTime = hcclNslbDpRootRankVal_.commInitTime;
1884 0 : tab_f.oper = hcclNslbDpRootRankVal_.oper;
1885 0 : tab_f.algorithm = hcclNslbDpRootRankVal_.algorithm;
1886 0 : tab_f.revfir = hcclNslbDpRootRankVal_.revfir;
1887 0 : tab_f.rootRankNum = hcclNslbDpRootRankVal_.rootRankNum;
1888 0 : tab_f.revsec = hcclNslbDpRootRankVal_.revsec;
1889 0 : tab_f.rankId = hcclNslbDpRootRankVal_.rankId;
1890 :
1891 0 : for (const auto& rank : hcclNslbDpRootRankVal_.rankId) {
1892 0 : tab_f.rankId.push_back(rank);
1893 : }
1894 :
1895 0 : SendRankTableRootRank(tab_f);
1896 :
1897 0 : return HCCL_SUCCESS;
1898 0 : }
1899 :
1900 : /* 初始化NetCo通道 */
1901 0 : HcclResult hcclNslbDp::InitNetCo()
1902 : {
1903 0 : if (hcclH2dTlv::GetInstance().GetH2dTlvInitFlag() != true) {
1904 0 : HCCL_INFO("Check GetH2dTlvInitFlag is not success");
1905 0 : return HCCL_SUCCESS;
1906 : }
1907 : /* 避免二次初始化 */
1908 0 : if (GetInitNetCoFlag() == true) {
1909 0 : HCCL_INFO("Get getHccpInitFlag is true");
1910 0 : return HCCL_SUCCESS;
1911 : }
1912 0 : if (hcclH2dTlv::GetInstance().GetH2dTlvBufferSize() == NSLBDP_ILLEGAL_TLVBUFFERSIZE) {
1913 : /* 异常场景处理 */
1914 0 : HCCL_INFO("Check H2dTlvBufferSize equal 0.");
1915 0 : ClearInitNetCoFlag();
1916 0 : return HCCL_E_NOT_SUPPORT;
1917 : }
1918 0 : if (hcclH2dTlv::GetInstance().GetH2dTlvHandle() == nullptr) {
1919 0 : HCCL_ERROR("Check InitNetCo handle is null.");
1920 0 : ClearInitNetCoFlag();
1921 0 : return HCCL_E_NOT_SUPPORT;
1922 : }
1923 0 : u32 nslbBuffersize = hcclH2dTlv::GetInstance().GetH2dTlvBufferSize();
1924 0 : void *tlvHandle = hcclH2dTlv::GetInstance().GetH2dTlvHandle();
1925 0 : nslb_msg sendMsg;
1926 0 : nslb_msg recvMsg;
1927 0 : sendMsg.type = NSLBDP_TYPE_INIT_NETCO;
1928 0 : sendMsg.length = NSLBDP_ILLEGAL_MSGLENGTH;
1929 :
1930 : /* init/deinit 场景date信息默认填成数字0，hccp不关注此字段，但数据不能为NULL */
1931 0 : std::vector<uint8_t> tlvData;
1932 0 : tlvData.push_back(0);
1933 0 : sendMsg.data.assign(tlvData.begin(), tlvData.end());
1934 0 : s32 ret = H2DTlvRequest(tlvHandle, MODULE_TYPE_NSLB,
1935 0 : reinterpret_cast<TlvMsg*>(&sendMsg), reinterpret_cast<TlvMsg*>(&recvMsg));
1936 0 : HCCL_RUN_INFO("[NSLBDP-SENDTABLE] hccl send table NSLBDP_TYPE_INIT_NETCO(9001) to hccp. ret(%d)\n", ret);
1937 0 : if (ret != 0) {
1938 0 : return HCCL_E_NOT_SUPPORT;
1939 : }
1940 : /* 数据转存 */
1941 0 : SetH2DTlvInitInfo(nslbBuffersize, tlvHandle);
1942 0 : HCCL_DEBUG("Entry InitNetCo end");
1943 0 : return HCCL_SUCCESS;
1944 0 : }
1945 :
1946 : /* 去初始化NetCo通道 */
1947 0 : void hcclNslbDp::DeinitNetCo()
1948 : {
1949 0 : if (GetInitNetCoFlag() == false) {
1950 0 : return;
1951 : }
1952 0 : if (hcclH2dTlv::GetInstance().GetH2dTlvBufferSize() == NSLBDP_ILLEGAL_TLVBUFFERSIZE) {
1953 0 : return;
1954 : }
1955 0 : if (hcclH2dTlv::GetInstance().GetH2dTlvHandle() == nullptr) {
1956 0 : return;
1957 : }
1958 0 : nslb_msg sendMsg;
1959 0 : nslb_msg recvMsg;
1960 0 : sendMsg.type = NSLBDP_TYPE_DEINIT_NETCO;
1961 0 : sendMsg.length = NSLBDP_ILLEGAL_MSGLENGTH;
1962 : /* init/deinit 场景date信息默认填成数字0，hccp不关注此字段，但数据不能为NULL */
1963 0 : std::vector<uint8_t> tlvData;
1964 0 : tlvData.push_back(0);
1965 0 : sendMsg.data.assign(tlvData.begin(), tlvData.end());
1966 0 : s32 ret = H2DTlvRequest(nslbdp_handle_, MODULE_TYPE_NSLB,
1967 0 : reinterpret_cast<TlvMsg*>(&sendMsg), reinterpret_cast<TlvMsg*>(&recvMsg));
1968 0 : HCCL_RUN_INFO("[NSLBDP-SENDTABLE] hccl send table NSLBDP_TYPE_DEINIT_NETCO(9002) to hccp. ret(%d)\n", ret);
1969 0 : ClearInitNetCoFlag();
1970 0 : return;
1971 0 : }
1972 :
1973 : }
|
