Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include <atomic>
12 : #include <fcntl.h>
13 : #include <unistd.h>
14 : #include <hccl/hccl_types.h>
15 : #include "topoinfo_struct.h"
16 : #include "adapter_hccp_common.h"
17 : #include "hccl_nslb_md5.h"
18 : #include "../common/src/h2d_tlv/hccl_h2dtlv.h"
19 : #include "hccl_nslbdp.h"
20 :
21 : namespace hccl {
22 :
23 : const u32 COMM_SLICING_LENGTH = 1024; // group name max length
24 :
25 425 : hcclNslbDp& hcclNslbDp::GetInstance()
26 : {
27 425 : static hcclNslbDp pNslbDp;
28 428 : return pNslbDp;
29 : }
30 :
31 8 : hcclNslbDp::hcclNslbDp() {}
32 :
33 8 : hcclNslbDp::~hcclNslbDp() {}
34 :
35 0 : void hcclNslbDp::InitCmmDesc(std::string& identifier_nslb)
36 : {
37 0 : nslbdp_identifier_ = identifier_nslb;
38 0 : HCCL_INFO("[NSLB-DP] Init CmmDesc nslbdp_identifier_[%s] .", nslbdp_identifier_.c_str());
39 0 : return;
40 : }
41 :
42 0 : std::string hcclNslbDp::GetCmmDesc()
43 : {
44 0 : HCCL_INFO("[NSLB-DP] Get CmmDesc nslbdp_identifier_[%s] .", nslbdp_identifier_.c_str());
45 0 : return nslbdp_identifier_;
46 : }
47 :
48 0 : void hcclNslbDp::SetDeviceType()
49 : {
50 0 : check910_93_ = true;
51 0 : return;
52 : }
53 :
54 0 : bool hcclNslbDp::GetDeviceType()
55 : {
56 0 : bool is91093 = check910_93_;
57 0 : return is91093;
58 : }
59 :
60 0 : HcclResult hcclNslbDp::SetH2DTlvInitInfo(u32 buffer_size, void* tlv_handle)
61 : {
62 0 : HCCL_DEBUG("[NSLB-DP] try to set tlvinit info.");
63 0 : nslbdp_buffsize_ = buffer_size;
64 0 : nslbdp_handle_ = tlv_handle;
65 0 : nslbdpIsInitNetCo_ = true;
66 0 : HCCL_INFO("[NSLB-DP] set tlvinit info buffer_size:[%u].", buffer_size);
67 0 : return HCCL_SUCCESS;
68 : }
69 :
70 3 : bool hcclNslbDp::GetInitNetCoFlag() { return nslbdpIsInitNetCo_; }
71 :
72 0 : HcclResult hcclNslbDp::ClearInitNetCoFlag()
73 : {
74 0 : nslbdpIsInitNetCo_ = false;
75 0 : return HCCL_SUCCESS;
76 : }
77 :
78 0 : u32 hcclNslbDp::GetTlvInitBufferSize()
79 : {
80 0 : u32 nslbdpBuffsize = nslbdp_buffsize_;
81 0 : return nslbdpBuffsize;
82 : }
83 :
84 0 : HcclResult hcclNslbDp::HcclSetGlobalRankTotalNum(u32 nRanks)
85 : {
86 : // 填充表4
87 0 : hcclNslbDpGlobalRankVal_.rankTotalNum = nRanks;
88 0 : return HCCL_SUCCESS;
89 : }
90 :
91 22 : void hcclNslbDp::SetGlobalCommTaskId(u64 taskId)
92 : {
93 22 : hcclNslbDpGlobalCommInfo_.taskId = taskId;
94 22 : return;
95 : }
96 :
97 0 : void hcclNslbDp::SetGlobalCommNodeId(u32 nodeId)
98 : {
99 0 : hcclNslbDpGlobalCommInfo_.nodeId = nodeId;
100 0 : return;
101 : }
102 :
103 0 : void hcclNslbDp::SetGlobalCommLocalRankNum(u32 localRankNum)
104 : {
105 0 : hcclNslbDpGlobalCommInfo_.localRankNum = localRankNum;
106 0 : return;
107 : }
108 :
109 0 : void hcclNslbDp::SetGlobalCommRankTotalNum(u32 rankTotalNum)
110 : {
111 0 : hcclNslbDpGlobalCommInfo_.rankTotalNum = rankTotalNum;
112 0 : return;
113 : }
114 :
115 410 : u64 hcclNslbDp::GetGlobalCommTaskId() { return hcclNslbDpGlobalCommInfo_.taskId; }
116 :
117 0 : u32 hcclNslbDp::GetGlobalCommNodeId() { return hcclNslbDpGlobalCommInfo_.nodeId; }
118 :
119 0 : u8 hcclNslbDp::GetGlobalCommLocalRankNum() { return hcclNslbDpGlobalCommInfo_.localRankNum; }
120 :
121 0 : u32 hcclNslbDp::GetGlobalCommRankTotalNum() { return hcclNslbDpGlobalCommInfo_.rankTotalNum; }
122 :
123 : /* 获取 l4SPortId */
124 0 : u32 hcclNslbDp::Getl4SPortId()
125 : {
126 0 : u32 NslbDpL4SPortId = hcclNslbDpL4SPortId_;
127 0 : return NslbDpL4SPortId;
128 : }
129 :
130 : /* 切分identifier字段 */
131 4 : void hcclNslbDp::SplitString(
132 : const std::string& identifier, std::vector<std::string>& splitInfo, const std::string& frag)
133 : {
134 4 : std::string::size_type pos2 = identifier.find(frag);
135 4 : std::string::size_type pos1 = 0;
136 8 : while (std::string::npos != pos2) {
137 4 : splitInfo.push_back(identifier.substr(pos1, pos2 - pos1));
138 4 : pos1 = pos2 + frag.size();
139 4 : pos2 = identifier.find(frag, pos1);
140 : }
141 4 : if (pos1 != identifier.length()) {
142 3 : splitInfo.push_back(identifier.substr(pos1));
143 : }
144 4 : return;
145 : }
146 :
147 : /* 将IP转换成U32值 */
148 5 : u32 hcclNslbDp::ipToUint32(const std::string& ipAddress)
149 : {
150 : struct sockaddr_in sa;
151 5 : inet_pton(AF_INET, ipAddress.c_str(), &(sa.sin_addr));
152 5 : return ntohl(sa.sin_addr.s_addr); // Convert to host byte order
153 : }
154 :
155 : /* 执行send 流程 */
156 0 : HcclResult hcclNslbDp::SendCommRankTable(uint32_t rank, NslbDpCommConfigVal globalCommInfo)
157 : {
158 0 : HCCL_DEBUG("[NSLB-DP] entry to send TBL_COMM_INFO");
159 0 : u32 rankTotalNum = globalCommInfo.rankTotalNum;
160 :
161 0 : HCCL_INFO("[NSLB-DP] TBL_COMM_INFO rankTotalNum:[%u].", rankTotalNum);
162 0 : if (rankTotalNum > NSLBDP_RANKTOTALNUM_BLOCK_FOU) {
163 0 : return HCCL_SUCCESS;
164 : }
165 0 : u32 packetNum = NSLBDP_PKTNUM_FIR;
166 0 : if (rankTotalNum <= NSLBDP_RANKTOTALNUM_BLOCK_FIR) {
167 0 : packetNum = NSLBDP_PKTNUM_FIR;
168 0 : } else if (rankTotalNum <= NSLBDP_RANKTOTALNUM_BLOCK_SEC) {
169 0 : packetNum = NSLBDP_PKTNUM_SEC;
170 0 : } else if (rankTotalNum <= NSLBDP_RANKTOTALNUM_BLOCK_THR) {
171 0 : packetNum = NSLBDP_PKTNUM_THR;
172 : } else {
173 0 : packetNum = NSLBDP_PKTNUM_FOU;
174 : }
175 0 : SendTableProc(rank, packetNum, globalCommInfo);
176 :
177 0 : HCCL_DEBUG("[NSLB-DP] entry send TBL_COMM_INFO end");
178 0 : return HCCL_SUCCESS;
179 : }
180 :
181 0 : bool hcclNslbDp::CheckAhcCommInfo(NslbDpCommConfigVal comInfo)
182 : {
183 0 : std::unordered_map<uint32_t, int> podToIpCount;
184 0 : for (const auto& info : comInfo.rankInfo) {
185 0 : podToIpCount[info.podId]++;
186 : }
187 : // 获取超节点对应的ip数量
188 0 : std::vector<int> ipCounts;
189 0 : for (const auto& entry : podToIpCount) {
190 0 : ipCounts.push_back(entry.second);
191 : }
192 0 : if (ipCounts.empty()) {
193 0 : return true;
194 : }
195 : // 找到最小的ip数量
196 0 : uint32_t minCount = *std::min_element(ipCounts.begin(), ipCounts.end());
197 0 : for (uint32_t count : ipCounts) {
198 0 : if (count % minCount != 0) {
199 0 : return false;
200 : }
201 : }
202 0 : return true;
203 0 : }
204 :
205 7 : bool hcclNslbDp::CheckAhcSupport(u8 algType, std::string identifier)
206 : {
207 7 : if (algType != NSLB_ALGO_TYPE_AHC) {
208 7 : return true;
209 : }
210 : char commDesc[COMM_DESC_MAX_LENGTH];
211 0 : s32 sRet = memset_s(commDesc, COMM_DESC_MAX_LENGTH, 0, sizeof(commDesc));
212 0 : if (sRet != EOK) {
213 0 : HCCL_ERROR("memset_s commDesc fail");
214 0 : return true;
215 : }
216 :
217 0 : s32 ret = strncpy_s(commDesc, COMM_DESC_MAX_LENGTH, identifier.c_str(), identifier.size());
218 0 : if (ret != EOK) {
219 0 : HCCL_INFO("strncpy_s commDesc fail");
220 0 : return true;
221 : }
222 0 : commDesc[COMM_DESC_MAX_LENGTH - 1] = '\0';
223 0 : for (const auto& info : hcclNslbDpCommConfig_) {
224 0 : if (strcmp(commDesc, info.commDesc) == 0) {
225 0 : return CheckAhcCommInfo(info);
226 : }
227 : }
228 0 : return true;
229 : }
230 :
231 : /* 读配置文件的场景与正常创建通信域场景下填充通信域信息表（表一） */
232 0 : void hcclNslbDp::SetGlobalCommRankTable_RootInfo(
233 : const RankTable_t& rankTable, const HcclBasicRankInfo& localRankInfo, const std::vector<RankInfo>& rankLists,
234 : const std::string& identifier, u32 nRanks, u32 rank)
235 : {
236 0 : HCCL_INFO(
237 : "[NSLB-DP] Try to collect NSLBDP_TYPE_TBL_COMM_INFO commDesc[%s] - size = [%u].", identifier.c_str(),
238 : rankTable.rankList.size());
239 0 : u64 checkTaskId = GetGlobalCommTaskId();
240 0 : if (checkTaskId == 0 || nRanks == 1) {
241 0 : return;
242 : }
243 :
244 : // 判断是否跨机
245 0 : if (CheckMultiMachine(rankTable) == false) {
246 0 : HCCL_INFO("[NSLB-DP] CheckMultiMachine is false");
247 0 : return;
248 : }
249 :
250 0 : NslbDpCommConfigVal globalCommInfo{};
251 0 : u32 sRet = memset_s(globalCommInfo.commDesc, COMM_DESC_MAX_LENGTH, 0, sizeof(globalCommInfo.commDesc));
252 0 : if (sRet != EOK) {
253 0 : HCCL_ERROR("memset_s commDesc fail sRet[%u]", sRet);
254 0 : return;
255 : }
256 : /* 获取通信域唯一标识 */
257 0 : sRet = strncpy_s(globalCommInfo.commDesc, COMM_DESC_MAX_LENGTH, identifier.c_str(), identifier.size());
258 0 : if (sRet != EOK) {
259 0 : HCCL_ERROR("strncpy_s commDesc fail sRet[%u]", sRet);
260 0 : return;
261 : }
262 0 : for (size_t operSize = 0; operSize < hcclNslbDpCommConfig_.size(); operSize++) {
263 0 : if (strcmp(globalCommInfo.commDesc, hcclNslbDpCommConfig_[operSize].commDesc) == 0) {
264 : /* 此时认为是已存在的相同通信域，无需处理直接返回 */
265 0 : return;
266 : }
267 : }
268 :
269 0 : globalCommInfo.commDesc[COMM_DESC_MAX_LENGTH - 1] = '\0';
270 :
271 : u64 utime
272 0 : = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now().time_since_epoch())
273 0 : .count();
274 0 : globalCommInfo.commInitTime = utime;
275 0 : globalCommInfo.taskId = checkTaskId;
276 0 : globalCommInfo.rankTotalNum = nRanks;
277 :
278 0 : for (u32 rankIndex = 0; rankIndex < rankTable.rankList.size(); rankIndex++) {
279 : NslbDpRankInfo dpRankInfo;
280 0 : HcclIpAddress tmpIp = rankTable.rankList[rankIndex].deviceInfo.deviceIp[0];
281 0 : std::string deviceIp = tmpIp.GetReadableAddress();
282 0 : dpRankInfo.deviceIp = ipToUint32(deviceIp);
283 0 : HCCL_INFO("[NSLB-DP] SetGlobalCommRankTable_RootInfo deviceIp:[%u] success.", dpRankInfo.deviceIp);
284 :
285 0 : std::string serverIp = rankTable.rankList[rankIndex].serverId;
286 0 : dpRankInfo.serverIp = ipToUint32(serverIp);
287 0 : HCCL_INFO("[NSLB-DP] SetGlobalCommRankTable_RootInfo serverIp:[%u] success.", dpRankInfo.serverIp);
288 0 : if (rankLists.size() < rankIndex) {
289 0 : return;
290 : }
291 0 : if (rankLists[rankIndex].superPodIdx == INVALID_UINT) {
292 0 : dpRankInfo.podId = 0;
293 : } else {
294 0 : dpRankInfo.podId = rankLists[rankIndex].superPodIdx;
295 : }
296 0 : dpRankInfo.rev = 0;
297 0 : globalCommInfo.rankInfo.push_back(dpRankInfo);
298 0 : }
299 0 : NSLBMD5::calculateRankInfoMd5(globalCommInfo.rankInfo, globalCommInfo.commMd5Sum);
300 0 : std::string nslbdpmd5 = NSLBMD5::md5ToString(globalCommInfo.commMd5Sum);
301 0 : HCCL_DEBUG("[NSLB-DP] NSLBDP-MD5 nslbdpmd5:[%s] success.", nslbdpmd5.c_str());
302 :
303 0 : std::string npuIp = localRankInfo.deviceIP[0].GetReadableIP();
304 0 : if (ipToUint32(npuIp) != 0) {
305 0 : SendCommRankTable(rank, globalCommInfo);
306 : }
307 0 : hcclNslbDpCommConfig_.push_back(globalCommInfo);
308 0 : HCCL_INFO("[NSLB-DP] Entry SetGlobalCommRankTable_RootInfo end size = [%zu]", hcclNslbDpCommConfig_.size());
309 0 : }
310 :
311 : /* 填充表5 */
312 0 : void hcclNslbDp::SetGlobalDisRankTable(const HcclBasicRankInfo& rankTable)
313 : {
314 0 : HCCL_INFO("[NSLB-DP] Try to collect NSLBDP_TYPE_TBL_RANK_DIST");
315 0 : u64 taskId = GetGlobalCommTaskId();
316 0 : if (taskId == 0) {
317 0 : return;
318 : }
319 0 : std::string npuIp = rankTable.deviceIP[0].GetReadableIP();
320 0 : if (ipToUint32(npuIp) == 0) {
321 0 : return;
322 : }
323 0 : hcclNslbDpGlobalDisRankVal_.taskId = taskId;
324 0 : hcclNslbDpGlobalDisRankVal_.nodeId = GetGlobalCommNodeId();
325 0 : hcclNslbDpGlobalDisRankVal_.localRankNum = GetGlobalCommLocalRankNum();
326 0 : hcclNslbDpGlobalDisRankVal_.rankTotalNum = GetGlobalCommRankTotalNum();
327 :
328 0 : std::string serverIp = rankTable.hostIP.GetReadableIP();
329 0 : hcclNslbDpGlobalDisRankVal_.serverIp = ipToUint32(serverIp);
330 0 : HCCL_DEBUG("[NSLB-DP] SetGlobalDisRankTable serverIp:[%u].", hcclNslbDpGlobalDisRankVal_.serverIp);
331 :
332 0 : hcclNslbDpGlobalDisRankVal_.npuIp = ipToUint32(npuIp);
333 0 : HCCL_DEBUG("[NSLB-DP] SetGlobalDisRankTable npuIp:[%u].", hcclNslbDpGlobalDisRankVal_.npuIp);
334 0 : }
335 :
336 : /* check 是否是多机场景 */
337 0 : bool hcclNslbDp::CheckMultiMachine(const RankTable_t rankTable)
338 : {
339 0 : HCCL_DEBUG("[NSLB-DP] check device is multi machine");
340 0 : u32 serverIpFir = 0;
341 0 : u16 podIdFir = 0;
342 0 : bool bIsMultiMachine = false;
343 0 : if (GetDeviceType() == true) {
344 0 : u16 podIdIndex = 0;
345 0 : for (u32 index = 0; index < rankTable.rankList.size(); index++) {
346 0 : if (rankTable.rankList[index].superPodIdx == INVALID_UINT) {
347 0 : podIdIndex = 0;
348 : } else {
349 0 : podIdIndex = rankTable.rankList[index].superPodIdx;
350 : }
351 0 : if (index == 0) {
352 0 : podIdFir = podIdIndex;
353 0 : continue;
354 : }
355 :
356 0 : if (podIdFir != podIdIndex) {
357 0 : return true;
358 : }
359 : }
360 0 : return false;
361 : }
362 0 : for (u32 rankIndex = 0; rankIndex < rankTable.rankList.size(); rankIndex++) {
363 0 : std::string serverIpInfo = rankTable.rankList[rankIndex].serverId;
364 0 : u32 serverIp = ipToUint32(serverIpInfo);
365 0 : u16 podIdInfo = 0;
366 0 : if (rankTable.rankList[rankIndex].superPodIdx == INVALID_UINT) {
367 0 : podIdInfo = 0;
368 : } else {
369 0 : podIdInfo = rankTable.rankList[rankIndex].superPodIdx;
370 : }
371 0 : if (rankIndex == 0) {
372 0 : serverIpFir = serverIp;
373 0 : podIdFir = podIdInfo;
374 0 : continue;
375 : }
376 :
377 0 : if (serverIp != serverIpFir || podIdFir != podIdInfo) {
378 0 : bIsMultiMachine = true;
379 0 : break;
380 : }
381 0 : }
382 0 : return bIsMultiMachine;
383 : }
384 :
385 : /* 无ranktable场景， 子通信域场景表1 赋值 */
386 1 : HcclResult hcclNslbDp::SetCommInfo_NoRankTable(const hccl::RankTable_t rankTable, std::string identifier)
387 : {
388 1 : HCCL_DEBUG("[NSLB-DP] Try to collect NSLBDP_TYPE_TBL_COMM_INFO for no RankTable");
389 1 : u64 taskId = GetGlobalCommTaskId();
390 1 : if (taskId == 0) {
391 1 : return HCCL_SUCCESS;
392 : }
393 : // 判断是否跨机 false非错误场景
394 0 : if (CheckMultiMachine(rankTable) == false) {
395 0 : HCCL_INFO("[NSLB-DP] nslb-dp CheckMultiMachine is false");
396 0 : return HCCL_SUCCESS;
397 : }
398 :
399 0 : NslbDpCommConfigVal globalCommInfo;
400 0 : (void)memset_s(globalCommInfo.commDesc, COMM_DESC_MAX_LENGTH, 0, sizeof(globalCommInfo.commDesc));
401 :
402 : // 获取通信域唯一标识
403 0 : s32 ret = strncpy_s(globalCommInfo.commDesc, COMM_DESC_MAX_LENGTH, identifier.c_str(), identifier.size());
404 0 : if (ret != EOK) {
405 0 : HCCL_INFO("strncpy_s globalCommInfo.commDesc fail");
406 0 : return HCCL_E_MEMORY;
407 : }
408 0 : globalCommInfo.commDesc[COMM_DESC_MAX_LENGTH - 1] = '\0';
409 0 : CHK_PRT_RET(ret != EOK, HCCL_ERROR("[NSLB_DP]GetIdentifier str copy fail. return[%d]", ret), HCCL_E_INTERNAL);
410 :
411 : // commInitTime在有ranktable的赋值
412 : u64 utime
413 0 : = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now().time_since_epoch())
414 0 : .count();
415 0 : globalCommInfo.commInitTime = utime;
416 0 : u16 nRanks = rankTable.rankNum;
417 0 : globalCommInfo.taskId = taskId;
418 0 : globalCommInfo.rankTotalNum = nRanks;
419 :
420 0 : for (u32 rankIndex = 0; rankIndex < rankTable.rankList.size(); rankIndex++) {
421 : NslbDpRankInfo dpRankInfo;
422 0 : HcclIpAddress tmpIp = rankTable.rankList[rankIndex].deviceInfo.deviceIp[0];
423 0 : std::string deviceIp = tmpIp.GetReadableAddress();
424 0 : dpRankInfo.deviceIp = ipToUint32(deviceIp);
425 0 : HCCL_INFO("[NSLB-DP] SetCommInfo_RankTableExit deviceIp:[%u] success.", dpRankInfo.deviceIp);
426 0 : if (rankTable.rankList[rankIndex].superPodIdx == INVALID_UINT) {
427 0 : dpRankInfo.podId = 0;
428 : } else {
429 0 : dpRankInfo.podId = rankTable.rankList[rankIndex].superPodIdx;
430 : }
431 0 : dpRankInfo.rev = 0;
432 0 : globalCommInfo.rankInfo.push_back(dpRankInfo);
433 0 : }
434 0 : hcclNslbDpCommConfig_.push_back(globalCommInfo);
435 :
436 0 : return HCCL_SUCCESS;
437 0 : }
438 :
439 : /* 有ranktable场景， 表1 赋值 */
440 0 : HcclResult hcclNslbDp::SetCommInfo_RankTableExit(RankTable_t rankTable)
441 : {
442 0 : HCCL_DEBUG("[NSLB-DP] Entry SetCommInfo for RankTable exit");
443 0 : u16 nRanks = rankTable.rankNum;
444 0 : u64 taskId = GetGlobalCommTaskId();
445 0 : if (taskId == 0 || nRanks == 1) {
446 0 : return HCCL_SUCCESS;
447 : }
448 :
449 0 : NslbDpCommConfigVal globalCommInfo = {};
450 : // 获取通信域唯一标识
451 0 : (void)memset_s(
452 0 : hcclNslbDpGlobalRankVal_.commDesc, COMM_DESC_MAX_LENGTH, 0, sizeof(hcclNslbDpGlobalRankVal_.commDesc));
453 0 : char commDesc[COMM_DESC_MAX_LENGTH] = "HCCL_WORLD_GROUP";
454 0 : s32 sRet = memcpy_s(globalCommInfo.commDesc, sizeof(globalCommInfo.commDesc), commDesc, COMM_DESC_MAX_LENGTH);
455 0 : if (sRet != EOK) {
456 0 : HCCL_ERROR("memcpy_s commDesc fail");
457 0 : return HCCL_SUCCESS;
458 : ;
459 : }
460 :
461 0 : globalCommInfo.taskId = taskId;
462 0 : globalCommInfo.rankTotalNum = nRanks;
463 :
464 : u64 utime
465 0 : = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now().time_since_epoch())
466 0 : .count();
467 0 : globalCommInfo.commInitTime = utime;
468 :
469 0 : u32 size = rankTable.rankList.size();
470 0 : HCCL_INFO("[NSLB-DP] RankTableExit size:[%u] success.", size);
471 :
472 0 : for (size_t ranksize = 0; ranksize < rankTable.rankList.size(); ranksize++) {
473 0 : for (size_t rankIndex = 0; rankIndex < rankTable.rankList.size(); rankIndex++) {
474 0 : u16 podId = 0;
475 0 : if (rankTable.rankList[rankIndex].superPodIdx != INVALID_UINT) {
476 0 : podId = rankTable.rankList[rankIndex].superPodIdx;
477 : }
478 : // 按照rankid 排序
479 0 : if (ranksize == rankTable.rankList[rankIndex].rankId) {
480 : NslbDpRankInfo dpRankInfo;
481 0 : HcclIpAddress tmpIp = rankTable.rankList[rankIndex].deviceInfo.deviceIp[0];
482 0 : std::string deviceIp = tmpIp.GetReadableAddress();
483 0 : dpRankInfo.deviceIp = ipToUint32(deviceIp);
484 0 : HCCL_INFO("[NSLB-DP] SetCommInfo_RankTableExit deviceIp:[%s] success.", deviceIp.c_str());
485 0 : std::string serverIp = rankTable.rankList[rankIndex].serverId;
486 0 : dpRankInfo.serverIp = ipToUint32(serverIp);
487 0 : HCCL_INFO("[NSLB-DP] SetCommInfo_RankTableExit serverIp:[%s] success.", serverIp.c_str());
488 0 : dpRankInfo.podId = podId;
489 0 : dpRankInfo.rev = 0;
490 0 : globalCommInfo.rankInfo.push_back(dpRankInfo);
491 0 : }
492 : }
493 : }
494 0 : NSLBMD5::calculateRankInfoMd5(globalCommInfo.rankInfo, globalCommInfo.commMd5Sum);
495 0 : std::string nslbdpmd5 = NSLBMD5::md5ToString(globalCommInfo.commMd5Sum);
496 0 : HCCL_INFO("[NSLB-DP] check pmd5:[%s] success.", nslbdpmd5.c_str());
497 :
498 0 : hcclNslbDpCommConfig_.push_back(globalCommInfo);
499 0 : HCCL_DEBUG("[NSLB-DP] entry SetCommInfo_RankTableExit end");
500 :
501 0 : return HCCL_SUCCESS;
502 0 : }
503 :
504 : /* 有ranktable场景，表4赋值 */
505 0 : HcclResult hcclNslbDp::SetGlobalRank_RankTableExit(const hccl::RankTable_t rankTable)
506 : {
507 0 : u64 taskId = GetGlobalCommTaskId();
508 0 : HCCL_INFO("[NSLB-DP] set TBL_RANK for RankTableExit[%ull] success.", taskId);
509 :
510 0 : if (taskId == 0) {
511 0 : return HCCL_SUCCESS;
512 : }
513 0 : u16 nRanks = rankTable.rankNum;
514 0 : hcclNslbDpGlobalRankVal_.taskId = taskId;
515 0 : (void)memset_s(
516 0 : hcclNslbDpGlobalRankVal_.commDesc, COMM_DESC_MAX_LENGTH, 0, sizeof(hcclNslbDpGlobalRankVal_.commDesc));
517 0 : char commDesc[COMM_DESC_MAX_LENGTH] = "HCCL_WORLD_GROUP";
518 0 : s32 sRet = memcpy_s(
519 0 : hcclNslbDpGlobalRankVal_.commDesc, sizeof(hcclNslbDpGlobalRankVal_.commDesc), commDesc, COMM_DESC_MAX_LENGTH);
520 0 : if (sRet != EOK) {
521 0 : HCCL_ERROR("memcpy_s commDesc fail");
522 0 : return HCCL_SUCCESS;
523 : }
524 :
525 : hcclNslbDpGlobalRankVal_.commInitTime
526 0 : = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now().time_since_epoch())
527 0 : .count();
528 0 : hcclNslbDpGlobalRankVal_.rankTotalNum = nRanks;
529 :
530 0 : for (size_t ranksize = 0; ranksize < rankTable.rankList.size(); ranksize++) {
531 0 : for (size_t rankIndex = 0; rankIndex < rankTable.rankList.size(); rankIndex++) {
532 0 : if (ranksize == rankTable.rankList[rankIndex].rankId) {
533 : TableFourRankInfo dpGloRankInfo;
534 0 : HcclIpAddress tmpIp = rankTable.rankList[rankIndex].deviceInfo.deviceIp[0];
535 0 : std::string deviceIp = tmpIp.GetReadableAddress();
536 0 : dpGloRankInfo.deviceIp = ipToUint32(deviceIp);
537 :
538 0 : std::string serverIp = rankTable.rankList[rankIndex].serverId;
539 0 : dpGloRankInfo.serverIp = ipToUint32(serverIp);
540 :
541 0 : hcclNslbDpGlobalRankVal_.rankInfo.push_back(dpGloRankInfo);
542 0 : }
543 : }
544 : }
545 0 : NSLBMD5::calculateTableFourRankInfoMd5(hcclNslbDpGlobalRankVal_.rankInfo, hcclNslbDpGlobalRankVal_.commMd5Sum);
546 0 : std::string nslbdpmd5 = NSLBMD5::md5ToString(hcclNslbDpGlobalRankVal_.commMd5Sum);
547 0 : HCCL_INFO("[NSLB-DP] check pmd5:[%s] success.", nslbdpmd5.c_str());
548 0 : return HCCL_SUCCESS;
549 0 : }
550 :
551 : /* 拼接 l4SPortId */
552 1 : HcclResult hcclNslbDp::GetNslbDpl4SPortId(u32 rankSize, u8 algType, u16* l4SPortId)
553 : {
554 1 : u16 priFlag = NSLBDP_PRIVATE_PORT;
555 1 : u16 CommIntervalFlag = NSLB_COMM_INTERVAL_FLAG_BEGIN;
556 1 : if (rankSize > NSLBDP_COMMINTERVAL_FLAGSIX) {
557 0 : CommIntervalFlag = NSLB_COMM_INTERVAL_FLAG_SEV;
558 1 : } else if (rankSize > NSLBDP_COMMINTERVAL_FLAGFIV) {
559 0 : CommIntervalFlag = NSLB_COMM_INTERVAL_FLAG_SIX;
560 1 : } else if (rankSize > NSLBDP_COMMINTERVAL_FLAGFOU) {
561 0 : CommIntervalFlag = NSLB_COMM_INTERVAL_FLAG_FIV;
562 1 : } else if (rankSize > NSLBDP_COMMINTERVAL_FLAGTHR) {
563 0 : CommIntervalFlag = NSLB_COMM_INTERVAL_FLAG_FOR;
564 1 : } else if (rankSize > NSLBDP_COMMINTERVAL_FLAGSEC) {
565 0 : CommIntervalFlag = NSLB_COMM_INTERVAL_FLAG_THR;
566 1 : } else if (rankSize > NSLBDP_COMMINTERVAL_FLAG) {
567 0 : CommIntervalFlag = NSLB_COMM_INTERVAL_FLAG_SEC;
568 : } else {
569 1 : CommIntervalFlag = NSLB_COMM_INTERVAL_FLAG_FIR;
570 : }
571 1 : u16 CommPrecisely = rankSize % NSLBDP_COMMINTERVAL_FLAG;
572 1 : u8 CommalgType = algType;
573 :
574 1 : *l4SPortId = (priFlag << NSLBDP_RANGE_ID) + (CommIntervalFlag << NSLBDP_COMMON_RANGE)
575 1 : + (CommPrecisely << NSLBDP_ALGO_RANGE) + CommalgType;
576 1 : hcclNslbDpL4SPortId_ = *l4SPortId;
577 :
578 1 : HCCL_INFO("[NSLB-DP-L4PORT] get hcclNslbDpL4SPortId_[%u] success", hcclNslbDpL4SPortId_);
579 1 : return HCCL_SUCCESS;
580 : }
581 :
582 : /* 表6赋值 */
583 0 : HcclResult hcclNslbDp::SetNslbDpRootRank(HcclCMDType opType, u32 rootRank, std::string identifier, u8 algType)
584 : {
585 0 : HCCL_DEBUG("[NSLB-DP] try to collect NSLBDP_TYPE_TBL_ROOT_RANK");
586 0 : u64 taskId = GetGlobalCommTaskId();
587 0 : if (taskId == 0) {
588 0 : return HCCL_SUCCESS;
589 : }
590 0 : if (hcclNslbDpRootRankVal_.taskId == 0) {
591 0 : HCCL_DEBUG("[NSLB-DP-BEGIN] RootRank first entry");
592 : // 获取task id
593 0 : hcclNslbDpRootRankVal_.taskId = taskId;
594 0 : s32 sRet = memset_s(
595 0 : hcclNslbDpRootRankVal_.commDesc, COMM_DESC_MAX_LENGTH, 0, sizeof(hcclNslbDpRootRankVal_.commDesc));
596 0 : if (sRet != EOK) {
597 0 : HCCL_ERROR("memset_s commDesc fail");
598 0 : return HCCL_SUCCESS;
599 : }
600 : // 获取通信域唯一标识
601 : s32 ret
602 0 : = strncpy_s(hcclNslbDpRootRankVal_.commDesc, COMM_DESC_MAX_LENGTH, identifier.c_str(), identifier.size());
603 0 : if (ret != EOK) {
604 0 : HCCL_INFO("strncpy_s hcclNslbDpRootRankVal_.commDesc fail");
605 0 : return HCCL_SUCCESS;
606 : }
607 0 : hcclNslbDpRootRankVal_.commDesc[COMM_DESC_MAX_LENGTH - 1] = '\0';
608 :
609 : // commInitTime在有ranktable的赋值
610 : hcclNslbDpRootRankVal_.commInitTime
611 0 : = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now().time_since_epoch())
612 0 : .count();
613 :
614 : // 获取operator、algorithm
615 0 : hcclNslbDpRootRankVal_.oper = GetNslbOpType(opType);
616 : ;
617 :
618 0 : hcclNslbDpRootRankVal_.algorithm = algType;
619 :
620 0 : hcclNslbDpRootRankVal_.rootRankNum = hcclNslbDpRootRankVal_.rootRankNum + 1;
621 :
622 : NslbDpRankId rootRankId;
623 0 : rootRankId.rankID = rootRank;
624 0 : hcclNslbDpRootRankVal_.rankId.push_back(rootRankId);
625 :
626 0 : HCCL_INFO(
627 : "[NSLB-DP] Rootable rootRankNum[%u]-rootRank[%u]-algorithm[%u].", hcclNslbDpRootRankVal_.rootRankNum,
628 : rootRank, hcclNslbDpRootRankVal_.algorithm);
629 :
630 0 : HCCL_DEBUG("[NSLB-DP] Rootabl entry SetNslbDpRootRank end");
631 0 : SendRootRankTable();
632 0 : return HCCL_SUCCESS;
633 : }
634 0 : HCCL_INFO(
635 : "[NSLB-DP] Rootable rankId.size[%zu]-rootRankNum[%u]-rootRank[%u].", hcclNslbDpRootRankVal_.rankId.size(),
636 : hcclNslbDpRootRankVal_.rootRankNum, rootRank);
637 :
638 0 : for (const auto& rank : hcclNslbDpRootRankVal_.rankId) {
639 0 : HCCL_INFO("[NSLB-DP] Rootable rootRank exit rankID[%u]-rootRank[%u]", rank.rankID, rootRank);
640 0 : if (rank.rankID == rootRank) {
641 0 : return HCCL_SUCCESS;
642 : }
643 : }
644 :
645 : NslbDpRankId rootRankId;
646 0 : rootRankId.rankID = static_cast<u16>(rootRank);
647 0 : hcclNslbDpRootRankVal_.rankId.push_back(rootRankId);
648 0 : hcclNslbDpRootRankVal_.rootRankNum = hcclNslbDpRootRankVal_.rootRankNum + 1;
649 :
650 : // 新增场景下走send 流程
651 0 : SendRootRankTable();
652 :
653 0 : return HCCL_SUCCESS;
654 : }
655 :
656 : /* AlgType 的转换 */
657 9 : u8 hcclNslbDp::GetNslbLevel1AlgType(AlgTypeLevel1 algValue)
658 : {
659 9 : HCCL_DEBUG("[NSLB-DP] try to switch Level1 type to nslbtype");
660 9 : switch (algValue) {
661 1 : case AlgTypeLevel1::ALG_LEVEL1_RING: {
662 1 : return NSLB_ALGO_TYPE_RING;
663 : }
664 1 : case AlgTypeLevel1::ALG_LEVEL1_PIPELINE: {
665 1 : return NSLB_ALGO_TYPE_PIPELINE;
666 : }
667 1 : case AlgTypeLevel1::ALG_LEVEL1_HD: {
668 1 : return NSLB_ALGO_TYPE_HDR;
669 : }
670 1 : case AlgTypeLevel1::ALG_LEVEL1_NHR: {
671 1 : return NSLB_ALGO_TYPE_NHR;
672 : }
673 1 : case AlgTypeLevel1::ALG_LEVEL1_NHR_V1: {
674 1 : return NSLB_ALGO_TYPE_NHR_V1;
675 : }
676 1 : case AlgTypeLevel1::ALG_LEVEL1_NB: {
677 1 : return NSLB_ALGO_TYPE_NB;
678 : }
679 1 : case AlgTypeLevel1::ALG_LEVEL1_AHC: {
680 1 : return NSLB_ALGO_TYPE_AHC;
681 : }
682 1 : case AlgTypeLevel1::ALG_LEVEL1_AHC_BROKE: {
683 1 : return NSLB_ALGO_TYPE_AHC;
684 : }
685 1 : default:
686 1 : return NSLB_ALGO_TYPE_NA;
687 : }
688 : }
689 :
690 : /* Level2 的 AlgType 转换 */
691 6 : u8 hcclNslbDp::GetNslbLevel2AlgType(AlgTypeLevel2 algValue)
692 : {
693 6 : HCCL_DEBUG("[NSLB-DP] try to switch Level2 type to nslbtype");
694 6 : switch (algValue) {
695 1 : case AlgTypeLevel2::ALG_LEVEL2_RING: {
696 1 : return NSLB_ALGO_TYPE_RING;
697 : }
698 1 : case AlgTypeLevel2::ALG_LEVEL2_HD: {
699 1 : return NSLB_ALGO_TYPE_HDR;
700 : }
701 1 : case AlgTypeLevel2::ALG_LEVEL2_NHR: {
702 1 : return NSLB_ALGO_TYPE_NHR;
703 : }
704 1 : case AlgTypeLevel2::ALG_LEVEL2_NB: {
705 1 : return NSLB_ALGO_TYPE_NB;
706 : }
707 1 : case AlgTypeLevel2::ALG_LEVEL2_PIPELINE: {
708 1 : return NSLB_ALGO_TYPE_PIPELINE;
709 : }
710 1 : default:
711 1 : return NSLB_ALGO_TYPE_NA;
712 : }
713 : }
714 :
715 : /* opType 的转换 */
716 148 : u8 hcclNslbDp::GetNslbOpType(HcclCMDType opType)
717 : {
718 148 : HCCL_DEBUG("[NSLB-DP] try to get nslb optype");
719 148 : switch (opType) {
720 8 : case HcclCMDType::HCCL_CMD_BROADCAST: {
721 8 : return NSLBDP_CMD_BROADCAST;
722 : }
723 10 : case HcclCMDType::HCCL_CMD_ALLREDUCE: {
724 10 : return NSLBDP_CMD_ALLREDUCE;
725 : }
726 11 : case HcclCMDType::HCCL_CMD_REDUCE: {
727 11 : return NSLBDP_CMD_REDUCE;
728 : }
729 12 : case HcclCMDType::HCCL_CMD_SEND: {
730 12 : return NSLBDP_CMD_SEND;
731 : }
732 12 : case HcclCMDType::HCCL_CMD_RECEIVE: {
733 12 : return NSLBDP_CMD_RECEIVE;
734 : }
735 18 : case HcclCMDType::HCCL_CMD_ALLGATHER: {
736 18 : return NSLBDP_CMD_ALLGATHER;
737 : }
738 4 : case HcclCMDType::HCCL_CMD_REDUCE_SCATTER: {
739 4 : return NSLBDP_CMD_REDUCE_SCATTER;
740 : }
741 6 : case HcclCMDType::HCCL_CMD_ALLTOALLV: {
742 6 : return NSLBDP_CMD_ALLTOALLV;
743 : }
744 7 : case HcclCMDType::HCCL_CMD_ALLTOALLVC: {
745 7 : return NSLBDP_CMD_ALLTOALLVC;
746 : }
747 5 : case HcclCMDType::HCCL_CMD_ALLTOALL: {
748 5 : return NSLBDP_CMD_ALLTOALL;
749 : }
750 23 : case HcclCMDType::HCCL_CMD_GATHER: {
751 23 : return NSLBDP_CMD_GATHER;
752 : }
753 9 : case HcclCMDType::HCCL_CMD_SCATTER: {
754 9 : return NSLBDP_CMD_SCATTER;
755 : }
756 11 : case HcclCMDType::HCCL_CMD_BATCH_SEND_RECV: {
757 11 : return NSLBDP_CMD_BATCH_SEND_RECV;
758 : }
759 12 : default:
760 12 : return 0;
761 : }
762 : }
763 :
764 : /* 获取通信量的前4bit */
765 7 : u64 hcclNslbDp::GetNslbDpFirstFourBit(u8 opType, u8 algType)
766 : {
767 7 : u64 firstFourBit = 0;
768 7 : if (opType == NSLBDP_CMD_ALLREDUCE) {
769 2 : firstFourBit = 1 << NSLBDP_BEGINFOURBIT;
770 2 : firstFourBit = firstFourBit + 1;
771 : }
772 :
773 7 : if (opType == NSLBDP_CMD_ALLGATHER) {
774 1 : firstFourBit = 1 << 1;
775 : }
776 :
777 7 : HCCL_INFO("[NSLB-DP-FIRST4] try to get first FourBit[%llu]-algType[%u]", firstFourBit, algType);
778 7 : return firstFourBit;
779 : }
780 :
781 : /* 校验算法的一致性 */
782 12 : bool hcclNslbDp::CheckAlgoConsistency(HcclCMDType opType, std::string& algName)
783 : {
784 12 : if (opType == HcclCMDType::HCCL_CMD_ALLREDUCE) {
785 2 : if (algName.find("AllReduce") != std::string::npos) {
786 1 : return true;
787 : } else {
788 1 : return false;
789 : }
790 10 : } else if (opType == HcclCMDType::HCCL_CMD_ALLGATHER) {
791 2 : if (algName.find("AllGather") != std::string::npos) {
792 1 : return true;
793 : } else {
794 1 : return false;
795 : }
796 8 : } else if (opType == HcclCMDType::HCCL_CMD_REDUCE_SCATTER) {
797 2 : if (algName.find("ReduceScatter") != std::string::npos) {
798 1 : return true;
799 : } else {
800 1 : return false;
801 : }
802 6 : } else if (opType == HcclCMDType::HCCL_CMD_ALLTOALL) {
803 2 : if (algName != "RunAlltoAllVFullMesh") {
804 1 : return false;
805 : }
806 4 : } else if (opType == HcclCMDType::HCCL_CMD_BROADCAST) {
807 2 : if (algName.find("BroadCast") != std::string::npos) {
808 1 : return true;
809 : } else {
810 1 : return false;
811 : }
812 2 : } else if (opType == HcclCMDType::HCCL_CMD_SCATTER) {
813 2 : if (algName.find("Scatter") != std::string::npos) {
814 1 : return true;
815 : } else {
816 1 : return false;
817 : }
818 : }
819 1 : return true;
820 : }
821 :
822 22 : bool hcclNslbDp::CheckSupportOptype(HcclCMDType opType)
823 : {
824 22 : HCCL_DEBUG("[NSLB-DP] try to check support optype");
825 39 : if (GetNslbOpType(opType) != NSLBDP_CMD_ALLREDUCE && GetNslbOpType(opType) != NSLBDP_CMD_ALLGATHER
826 13 : && GetNslbOpType(opType) != NSLBDP_CMD_REDUCE_SCATTER && GetNslbOpType(opType) != NSLBDP_CMD_ALLTOALL
827 11 : && GetNslbOpType(opType) != NSLBDP_CMD_ALLTOALLV && GetNslbOpType(opType) != NSLBDP_CMD_ALLTOALLVC
828 9 : && GetNslbOpType(opType) != NSLBDP_CMD_BROADCAST && GetNslbOpType(opType) != NSLBDP_CMD_SCATTER
829 7 : && GetNslbOpType(opType) != NSLBDP_CMD_BATCH_SEND_RECV && GetNslbOpType(opType) != NSLBDP_CMD_REDUCE
830 39 : && GetNslbOpType(opType) != NSLBDP_CMD_SEND) {
831 4 : return false;
832 : }
833 18 : return true;
834 : }
835 :
836 : /* 写算法算子表--表3 */
837 7 : HcclResult hcclNslbDp::GetAlgAdjacencyTable(
838 : HcclCMDType opType, u32 srcLocalRankId, u32 rootRank, u8 algType, std::string identifier, AdjInfo nslbAdjInfo)
839 : {
840 7 : HCCL_DEBUG("[NSLB-DP] check table NSLBDP_TYPE_TBL_ADJ size:[%zu].", hcclNslbDpAlgorithmInfo_.size());
841 7 : if (CheckAhcSupport(algType, identifier) == false) {
842 0 : HCCL_RUN_INFO("[NSLB-DP-ADJ] Check AHC commoninfo is not support.");
843 0 : return HCCL_SUCCESS;
844 : }
845 7 : u64 taskId = GetGlobalCommTaskId();
846 7 : if (taskId == 0) {
847 1 : return HCCL_SUCCESS;
848 : }
849 6 : HCCL_INFO(
850 : "[NSLB-DP-ADJ] opType:[%u],srcLocalRankId[%u],rootRank[%u]-commDesc[%s],dstRankNum:[%u].", opType,
851 : srcLocalRankId, rootRank, identifier.c_str(), nslbAdjInfo.dstRankNum);
852 :
853 6 : if (CheckSupportOptype(opType) == false) {
854 1 : HCCL_INFO("[NSLB-DP-OPER] CheckSupportOptype false .");
855 1 : return HCCL_SUCCESS;
856 : }
857 :
858 5 : NslbDpAlgorithmInfo algorithmInfo;
859 5 : algorithmInfo.taskId = taskId;
860 5 : if (InitAlgInfoCommDesc(algorithmInfo, identifier) == false) {
861 1 : return HCCL_SUCCESS;
862 : }
863 4 : if (FillAlgInfoCommMd5(algorithmInfo) == false) {
864 0 : return HCCL_SUCCESS;
865 : }
866 4 : FillAlgInfoBaseFields(algorithmInfo, opType, srcLocalRankId, rootRank, algType);
867 :
868 4 : HCCL_RUN_INFO(
869 : "[NSLB-DP] add adjINfo:***[%llu]***[%u]***[%u]***[%u]***[%u]-[%zu] success.", taskId, srcLocalRankId, rootRank,
870 : GetNslbOpType(opType), algType, nslbAdjInfo.nsAdjInfo.size());
871 :
872 4 : if (IsAlgAdjacencyDuplicated(algorithmInfo) == true) {
873 1 : HCCL_INFO("[NSLB-DP] Deduplication hcclNslbDpAlgorithmInfo_");
874 1 : return HCCL_SUCCESS;
875 : }
876 :
877 3 : HCCL_INFO(
878 : "[NSLB-DP-ADJ] add adjINfo:***[%llu]***[%u]***[%u]***[%u]***[%u] success.", taskId, srcLocalRankId, rootRank,
879 : GetNslbOpType(opType), algType);
880 :
881 3 : if (FillAlgInfoAdjInfo(algorithmInfo, nslbAdjInfo, srcLocalRankId) == false) {
882 0 : return HCCL_SUCCESS;
883 : }
884 :
885 3 : algorithmInfo.sedFlag = 0;
886 3 : hcclNslbDpAlgorithmInfo_.push_back(algorithmInfo);
887 3 : HCCL_DEBUG("[NSLB-DP] entry GetAlgAdjacencyTable end");
888 3 : return HCCL_SUCCESS;
889 5 : }
890 :
891 : /* 初始化commDesc并校验通信域是否存在；存在返回true，不存在返回false */
892 8 : bool hcclNslbDp::InitAlgInfoCommDesc(NslbDpAlgorithmInfo& algorithmInfo, const std::string& identifier)
893 : {
894 8 : (void)memset_s(algorithmInfo.commDesc, COMM_DESC_MAX_LENGTH, 0, sizeof(algorithmInfo.commDesc));
895 8 : s32 ret = strncpy_s(algorithmInfo.commDesc, COMM_DESC_MAX_LENGTH, identifier.c_str(), identifier.size());
896 8 : if (ret != EOK) {
897 0 : HCCL_INFO("[NSLB-DP] strncpy_s algorithmInfo.commDesc fail");
898 0 : return false;
899 : }
900 8 : algorithmInfo.commDesc[COMM_DESC_MAX_LENGTH - 1] = '\0';
901 :
902 8 : HCCL_INFO("[NSLB-DP] CheckCommDescExit algorithmInfo.commDesc[%s] .", algorithmInfo.commDesc);
903 :
904 : // 去除不存在的通信域信息
905 8 : bool commDescExit = false;
906 10 : for (const auto& info : hcclNslbDpCommConfig_) {
907 7 : if (strcmp(algorithmInfo.commDesc, info.commDesc) == 0) {
908 5 : commDescExit = true;
909 5 : break;
910 : }
911 : }
912 8 : return commDescExit;
913 : }
914 :
915 : /* 根据表一信息填充commMd5Sum；memcpy失败返回false */
916 6 : bool hcclNslbDp::FillAlgInfoCommMd5(NslbDpAlgorithmInfo& algorithmInfo)
917 : {
918 : // 根据表一信息填充MD5
919 7 : for (size_t comsize = 0; comsize < hcclNslbDpCommConfig_.size(); comsize++) {
920 6 : if (strcmp(algorithmInfo.commDesc, hcclNslbDpCommConfig_[comsize].commDesc) == 0) {
921 10 : s32 sRet = memcpy_s(
922 5 : algorithmInfo.commMd5Sum, sizeof(algorithmInfo.commMd5Sum), hcclNslbDpCommConfig_[comsize].commMd5Sum,
923 : sizeof(hcclNslbDpCommConfig_[comsize].commMd5Sum));
924 5 : if (sRet != EOK) {
925 0 : HCCL_ERROR("memcpy_s commMd5Sum fail");
926 0 : return false;
927 : }
928 5 : break;
929 : }
930 : }
931 6 : return true;
932 : }
933 :
934 : /* 填充srcLocalRankId/rootRank/oper/algorithm基础字段 */
935 6 : void hcclNslbDp::FillAlgInfoBaseFields(
936 : NslbDpAlgorithmInfo& algorithmInfo, HcclCMDType opType, u32 srcLocalRankId, u32 rootRank, u8 algType)
937 : {
938 6 : algorithmInfo.srcLocalRankId = srcLocalRankId;
939 6 : algorithmInfo.rootRank = static_cast<u16>(rootRank);
940 6 : algorithmInfo.oper = GetNslbOpType(opType);
941 6 : algorithmInfo.algorithm = algType;
942 6 : }
943 :
944 : /* 判断是否与已有邻接表项重复（去重） */
945 8 : bool hcclNslbDp::IsAlgAdjacencyDuplicated(const NslbDpAlgorithmInfo& algorithmInfo)
946 : {
947 11 : for (const auto& info : hcclNslbDpAlgorithmInfo_) {
948 5 : HCCL_INFO(
949 : "[NSLB-DP-ADJ] info: *[%llu]-[%u]-[%u]-[%u]-[%u]* success.", algorithmInfo.taskId, info.srcLocalRankId,
950 : info.rootRank, info.oper, info.algorithm);
951 5 : if (info.taskId == algorithmInfo.taskId && info.srcLocalRankId == algorithmInfo.srcLocalRankId
952 3 : && info.rootRank == algorithmInfo.rootRank && info.oper == algorithmInfo.oper
953 3 : && info.algorithm == algorithmInfo.algorithm && strcmp(algorithmInfo.commDesc, info.commDesc) == 0) {
954 2 : return true;
955 : }
956 : }
957 6 : return false;
958 : }
959 :
960 : /* 填充dstRankNum与AdjInfo邻接信息；nsAdjInfo为空返回false */
961 5 : bool hcclNslbDp::FillAlgInfoAdjInfo(NslbDpAlgorithmInfo& algorithmInfo, const AdjInfo& nslbAdjInfo, u32 srcLocalRankId)
962 : {
963 5 : algorithmInfo.dstRankNum = nslbAdjInfo.dstRankNum;
964 5 : HCCL_INFO("[NSLB-DP-ADJ] nslbAdjInfo.dstRankNum:[%u].", nslbAdjInfo.dstRankNum);
965 :
966 5 : if (nslbAdjInfo.nsAdjInfo.size() == 0) {
967 1 : algorithmInfo.dstRankNum = 0;
968 1 : HCCL_INFO("[NSLB-DP] get nsAdjInfo fail dstRankNum:[%u]", algorithmInfo.dstRankNum);
969 1 : return false;
970 : }
971 :
972 14 : for (size_t rankIndex = 0; rankIndex < nslbAdjInfo.nsAdjInfo.size(); rankIndex++) {
973 : NslbDpAdjInfo adjInfo;
974 10 : adjInfo.dstLocalRankId = nslbAdjInfo.nsAdjInfo[rankIndex].dstLocalRankId;
975 10 : adjInfo.phaseId = nslbAdjInfo.nsAdjInfo[rankIndex].phaseId;
976 10 : adjInfo.rev = nslbAdjInfo.nsAdjInfo[rankIndex].rev;
977 10 : HCCL_INFO(
978 : "[NSLB-DP-ADJ] adjINfo:[%u]-[%u]-[%u] success.", srcLocalRankId,
979 : nslbAdjInfo.nsAdjInfo[rankIndex].dstLocalRankId, nslbAdjInfo.nsAdjInfo[rankIndex].phaseId);
980 10 : algorithmInfo.AdjInfo.push_back(adjInfo);
981 : }
982 4 : return true;
983 : }
984 :
985 5 : bool hcclNslbDp::CheckCommDescExit(NslbDpOperatorInfo& OperatorInfo)
986 : {
987 5 : HCCL_DEBUG("[NSLB-DP-OPER] Check CommDescExit size:[%zu].", hcclNslbDpCommConfig_.size());
988 7 : for (const auto& info : hcclNslbDpCommConfig_) {
989 4 : HCCL_DEBUG("[NSLB-DP-OPER] CheckCommDescExit info.commDesc[%s] .", info.commDesc);
990 4 : if (strcmp(OperatorInfo.commDesc, info.commDesc) == 0) {
991 2 : return true;
992 : }
993 : }
994 3 : return false;
995 : }
996 :
997 4 : void hcclNslbDp::fullcommDescInitTime(std::string identifier, NslbDpOperatorInfo& OperatorInfo)
998 : {
999 : /* 获取通信域唯一标识 */
1000 4 : s32 ret = strncpy_s(OperatorInfo.commDesc, COMM_DESC_MAX_LENGTH, identifier.c_str(), identifier.size());
1001 4 : if (ret != EOK) {
1002 0 : HCCL_INFO("[NSLB-DP] strncpy_s OperatorInfo.commDesc fail");
1003 0 : return;
1004 : }
1005 4 : HCCL_DEBUG("[NSLB-DP-OPER] fullcommDescInitTime commDesc[%s] .", identifier.c_str());
1006 :
1007 4 : OperatorInfo.commDesc[COMM_DESC_MAX_LENGTH - 1] = '\0';
1008 :
1009 : // commInitTime在有ranktable的赋值
1010 : OperatorInfo.commInitTime
1011 4 : = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now().time_since_epoch())
1012 4 : .count();
1013 :
1014 4 : return;
1015 : }
1016 :
1017 4 : bool hcclNslbDp::CheckSameOperatorVal(size_t operSize, NslbDpOperatorInfo& OperatorInfo, u32 rootRank)
1018 : {
1019 4 : size_t num = hcclNslbDpOperatorVal_.size();
1020 4 : if (operSize >= num) {
1021 1 : return false;
1022 : }
1023 3 : if (hcclNslbDpOperatorVal_[operSize].taskId == OperatorInfo.taskId
1024 3 : && hcclNslbDpOperatorVal_[operSize].rootRank == rootRank
1025 2 : && hcclNslbDpOperatorVal_[operSize].oper == OperatorInfo.oper
1026 2 : && hcclNslbDpOperatorVal_[operSize].algorithm == OperatorInfo.algorithm
1027 6 : && strcmp(OperatorInfo.commDesc, hcclNslbDpOperatorVal_[operSize].commDesc) == 0) {
1028 1 : return true;
1029 : }
1030 2 : return false;
1031 : }
1032 :
1033 : /* 写算法算子表--表2 */
1034 4 : HcclResult hcclNslbDp::GenerateOpAndAdjTable(
1035 : HcclCMDType opType, u32 rootRank, u32 srcLocalRankId, u8 algType, std::string identifier, u64 count, u32 rankSize)
1036 : {
1037 4 : HCCL_INFO("[NSLB-DP-OPER] count=[%llu], hcclNslbDpOperatorVal_ size:[%zu].", count, hcclNslbDpOperatorVal_.size());
1038 4 : u64 taskId = GetGlobalCommTaskId();
1039 4 : if (taskId == 0) {
1040 1 : return HCCL_SUCCESS;
1041 : }
1042 :
1043 : // 获取，0号
1044 3 : if (rootRank != 0) {
1045 1 : return HCCL_SUCCESS;
1046 : }
1047 2 : u64 trafficNum = count;
1048 2 : if (CheckSupportOptype(opType) == false) {
1049 0 : trafficNum = 0;
1050 0 : HCCL_DEBUG("[NSLB-DP-OPER] CheckSupportOptype false trafficNum == 0.");
1051 : }
1052 :
1053 2 : NslbDpOperatorInfo OperatorInfo = {};
1054 : // 获取task id
1055 2 : OperatorInfo.taskId = taskId;
1056 2 : (void)memset_s(OperatorInfo.commDesc, COMM_DESC_MAX_LENGTH, 0, sizeof(OperatorInfo.commDesc));
1057 :
1058 2 : fullcommDescInitTime(identifier, OperatorInfo);
1059 :
1060 : // 去除不存在的通信域信息
1061 2 : if (CheckCommDescExit(OperatorInfo) == false) {
1062 1 : HCCL_INFO("[NSLB-DP-OPER] CheckCommDesc not exit ");
1063 1 : return HCCL_SUCCESS;
1064 : }
1065 :
1066 : // 获取operator、algorithm
1067 1 : OperatorInfo.oper = GetNslbOpType(opType);
1068 1 : OperatorInfo.algorithm = algType;
1069 1 : u64 trafficCount = GetNslbDpFirstFourBit(OperatorInfo.oper, OperatorInfo.algorithm);
1070 1 : trafficCount = (trafficCount << NSLBDP_TRAFFICCONUT) + trafficNum;
1071 :
1072 1 : for (size_t operSize = 0; operSize < hcclNslbDpOperatorVal_.size(); operSize++) {
1073 0 : if (CheckSameOperatorVal(operSize, OperatorInfo, rootRank) == true) {
1074 0 : if (hcclNslbDpOperatorVal_[operSize].trafficCnt < trafficCount && srcLocalRankId == 0) {
1075 0 : hcclNslbDpOperatorVal_[operSize].trafficCnt = trafficCount;
1076 0 : hcclNslbDpOperatorVal_[operSize].sedFlag = 0;
1077 0 : SendRankTableOpAndAdj(hcclNslbDpOperatorVal_[operSize]);
1078 0 : HCCL_RUN_INFO(
1079 : "[NSLB-DP-ADJ] commDesc[%s] try to update trafficCnt[%llu] success.", OperatorInfo.commDesc,
1080 : trafficCount);
1081 : }
1082 0 : return HCCL_SUCCESS;
1083 : }
1084 : }
1085 :
1086 1 : OperatorInfo.trafficCnt = trafficCount; // 判断变大
1087 1 : OperatorInfo.rootRank = rootRank;
1088 1 : HCCL_INFO(
1089 : "[NSLB-DP-OPER] add operInfo:***[%llu]***[%llu]***[%u]***[%u]***[%u] success.", taskId,
1090 : OperatorInfo.commInitTime, rootRank, OperatorInfo.oper, OperatorInfo.algorithm);
1091 :
1092 1 : GetNslbDpl4SPortId(rankSize, algType, &OperatorInfo.l4SPortId);
1093 1 : if (srcLocalRankId == 0) {
1094 1 : SendRankTableOpAndAdj(OperatorInfo);
1095 : }
1096 1 : hcclNslbDpOperatorVal_.push_back(OperatorInfo);
1097 :
1098 1 : return HCCL_SUCCESS;
1099 : }
1100 :
1101 : /* 根将表1 序列化处理 */
1102 0 : std::vector<uint8_t> hcclNslbDp::serializeTLV_TableFir(NslbDpCommConfigInfo cominfo)
1103 : {
1104 0 : HCCL_DEBUG("[NSLB-DP] entry serializeTLV TBL_COMM_INFO beg");
1105 :
1106 0 : std::vector<uint8_t> tlvData;
1107 :
1108 : // Task ID
1109 0 : uint64_t netTaskId = htobe64(cominfo.taskId);
1110 0 : tlvData.resize(tlvData.size() + sizeof(netTaskId));
1111 0 : (void)memcpy_s(
1112 0 : tlvData.data() + tlvData.size() - sizeof(netTaskId), sizeof(netTaskId), &netTaskId, sizeof(netTaskId));
1113 :
1114 : // CommDesc
1115 0 : tlvData.insert(tlvData.end(), cominfo.commDesc, cominfo.commDesc + COMM_DESC_MAX_LENGTH);
1116 :
1117 : // CommInitTime
1118 0 : uint64_t netCommInitTime = htobe64(cominfo.commInitTime);
1119 0 : tlvData.resize(tlvData.size() + sizeof(netCommInitTime));
1120 0 : (void)memcpy_s(
1121 0 : tlvData.data() + tlvData.size() - sizeof(netCommInitTime), sizeof(netCommInitTime), &netCommInitTime,
1122 : sizeof(netCommInitTime));
1123 :
1124 : // packetId
1125 0 : uint16_t netPacketId = htons(cominfo.packetId);
1126 0 : tlvData.resize(tlvData.size() + sizeof(netPacketId));
1127 0 : (void)memcpy_s(
1128 0 : tlvData.data() + tlvData.size() - sizeof(netPacketId), sizeof(netPacketId), &netPacketId, sizeof(netPacketId));
1129 :
1130 : // rev
1131 0 : uint16_t netRev = htons(cominfo.rev);
1132 0 : tlvData.resize(tlvData.size() + sizeof(netRev));
1133 0 : (void)memcpy_s(tlvData.data() + tlvData.size() - sizeof(netRev), sizeof(netRev), &netRev, sizeof(netRev));
1134 :
1135 : // packetNum
1136 0 : uint16_t netPacketNum = htons(cominfo.packetNum);
1137 0 : tlvData.resize(tlvData.size() + sizeof(netPacketNum));
1138 0 : (void)memcpy_s(
1139 0 : tlvData.data() + tlvData.size() - sizeof(netPacketNum), sizeof(netPacketNum), &netPacketNum,
1140 : sizeof(netPacketNum));
1141 :
1142 : // revSecond
1143 0 : uint16_t netRevSecond = htons(cominfo.revSecond);
1144 0 : tlvData.resize(tlvData.size() + sizeof(netRevSecond));
1145 0 : (void)memcpy_s(
1146 0 : tlvData.data() + tlvData.size() - sizeof(netRevSecond), sizeof(netRevSecond), &netRevSecond,
1147 : sizeof(netRevSecond));
1148 :
1149 : // sendRankInfo
1150 0 : for (const auto& rankInfo : cominfo.sendRankInfo) {
1151 0 : uint32_t netDeviceIp = htonl(rankInfo.deviceIp);
1152 0 : tlvData.resize(tlvData.size() + sizeof(netDeviceIp));
1153 0 : (void)memcpy_s(
1154 0 : tlvData.data() + tlvData.size() - sizeof(netDeviceIp), sizeof(netDeviceIp), &netDeviceIp,
1155 : sizeof(netDeviceIp));
1156 : }
1157 :
1158 : // CommMd5Sum
1159 0 : tlvData.insert(tlvData.end(), cominfo.commMd5Sum, cominfo.commMd5Sum + sizeof(cominfo.commMd5Sum));
1160 :
1161 : // RankTotalNum
1162 0 : uint16_t netRankTotalNum = htons(cominfo.rankTotalNum);
1163 0 : tlvData.resize(tlvData.size() + sizeof(netRankTotalNum));
1164 0 : (void)memcpy_s(
1165 0 : tlvData.data() + tlvData.size() - sizeof(netRankTotalNum), sizeof(netRankTotalNum), &netRankTotalNum,
1166 : sizeof(netRankTotalNum));
1167 :
1168 : // Rank Number
1169 0 : uint16_t netRankNum = htons(cominfo.rankNum);
1170 0 : tlvData.resize(tlvData.size() + sizeof(netRankNum));
1171 0 : (void)memcpy_s(
1172 0 : tlvData.data() + tlvData.size() - sizeof(netRankNum), sizeof(netRankNum), &netRankNum, sizeof(netRankNum));
1173 :
1174 : // Rank Info
1175 0 : for (const auto& rankInfo : cominfo.rankInfo) {
1176 0 : uint32_t netDeviceIp = htonl(rankInfo.deviceIp);
1177 0 : uint32_t netServerIp = htonl(rankInfo.serverIp);
1178 0 : uint16_t netPodId = htons(rankInfo.podId);
1179 0 : uint16_t netRev3 = htons(rankInfo.rev);
1180 0 : tlvData.resize(tlvData.size() + sizeof(netDeviceIp));
1181 0 : (void)memcpy_s(
1182 0 : tlvData.data() + tlvData.size() - sizeof(netDeviceIp), sizeof(netDeviceIp), &netDeviceIp,
1183 : sizeof(netDeviceIp));
1184 0 : tlvData.resize(tlvData.size() + sizeof(netServerIp));
1185 0 : (void)memcpy_s(
1186 0 : tlvData.data() + tlvData.size() - sizeof(netServerIp), sizeof(netServerIp), &netServerIp,
1187 : sizeof(netServerIp));
1188 0 : tlvData.resize(tlvData.size() + sizeof(netPodId));
1189 0 : (void)memcpy_s(
1190 0 : tlvData.data() + tlvData.size() - sizeof(netPodId), sizeof(netPodId), &netPodId, sizeof(netPodId));
1191 0 : tlvData.resize(tlvData.size() + sizeof(netRev3));
1192 0 : (void)memcpy_s(tlvData.data() + tlvData.size() - sizeof(netRev3), sizeof(netRev3), &netRev3, sizeof(netRev3));
1193 : }
1194 0 : HCCL_DEBUG("[NSLB-DP] entry serializeTLV TBL_COMM_INFO end");
1195 :
1196 0 : return tlvData;
1197 0 : }
1198 :
1199 0 : void hcclNslbDp::fullCommConfigInfo(NslbDpCommConfigInfo& tab_f, NslbDpCommConfigVal cominfo, u32 packetNum)
1200 : {
1201 0 : HCCL_DEBUG("[NSLB-DP] entry full CommConfigInfo");
1202 0 : tab_f.taskId = cominfo.taskId;
1203 0 : s32 sRet = memcpy_s(tab_f.commDesc, sizeof(tab_f.commDesc), cominfo.commDesc, COMM_DESC_MAX_LENGTH);
1204 0 : if (sRet != EOK) {
1205 0 : HCCL_INFO("memcpy_s commDesc fail");
1206 : }
1207 0 : tab_f.commInitTime = cominfo.commInitTime;
1208 :
1209 0 : tab_f.rev = 0;
1210 0 : tab_f.packetNum = packetNum;
1211 0 : tab_f.revSecond = 0;
1212 0 : for (u32 dip = 0; dip < packetNum; dip++) {
1213 0 : if (cominfo.rankInfo.size() == 0) {
1214 0 : break;
1215 : }
1216 0 : tab_f.sendRankInfo[dip].deviceIp = cominfo.rankInfo[dip].deviceIp;
1217 : }
1218 0 : sRet = memcpy_s(tab_f.commMd5Sum, sizeof(tab_f.commMd5Sum), cominfo.commMd5Sum, sizeof(cominfo.commMd5Sum));
1219 0 : if (sRet != EOK) {
1220 0 : HCCL_INFO("memcpy_s commmd5 fail");
1221 : }
1222 0 : }
1223 :
1224 : /* 根据表1 中的rank数量进行分片处理 */
1225 0 : HcclResult hcclNslbDp::SendTableProc(u32 rank, u32 packetNum, NslbDpCommConfigVal cominfo)
1226 : {
1227 0 : if (packetNum - 1 < rank) {
1228 0 : HCCL_INFO("[NSLB-DP] rank [%u] no need send TableFir(packetNum[%u]).", rank, packetNum);
1229 0 : return HCCL_SUCCESS;
1230 : }
1231 0 : u32 packetIndex = rank;
1232 0 : NslbDpCommConfigInfo tab_f = {};
1233 0 : fullCommConfigInfo(tab_f, cominfo, packetNum);
1234 0 : if (packetIndex != packetNum - 1) {
1235 0 : tab_f.packetId = packetIndex;
1236 0 : if (cominfo.rankInfo.size() < packetIndex * NSLBDP_RANKTOTALNUM_BLOCK_FIR) {
1237 0 : HCCL_INFO("[NSLB-DP] Comm RankInfo not as expected");
1238 0 : return HCCL_SUCCESS;
1239 : }
1240 0 : for (u32 rankindex = packetIndex * NSLBDP_RANKTOTALNUM_BLOCK_FIR;
1241 0 : rankindex < (packetIndex + 1) * NSLBDP_RANKTOTALNUM_BLOCK_FIR; rankindex++) {
1242 : NslbDpRankInfo rankInfoTemp;
1243 0 : rankInfoTemp.deviceIp = cominfo.rankInfo[rankindex].deviceIp;
1244 0 : rankInfoTemp.serverIp = cominfo.rankInfo[rankindex].serverIp;
1245 0 : rankInfoTemp.podId = cominfo.rankInfo[rankindex].podId;
1246 0 : rankInfoTemp.rev = cominfo.rankInfo[rankindex].rev;
1247 0 : tab_f.rankInfo.push_back(rankInfoTemp);
1248 : }
1249 0 : tab_f.rankTotalNum = cominfo.rankTotalNum;
1250 0 : tab_f.rankNum = tab_f.rankInfo.size();
1251 0 : HCCL_INFO(
1252 : "[NSLB-DP] SendTableProc-F info:[%u]-[%u]-[%u]-[%u].", packetNum, packetIndex, cominfo.rankTotalNum,
1253 : tab_f.rankInfo.size());
1254 : } else {
1255 0 : tab_f.packetId = packetIndex;
1256 0 : for (u32 j = packetIndex * NSLBDP_RANKTOTALNUM_BLOCK_FIR; j < cominfo.rankInfo.size(); j++) {
1257 : NslbDpRankInfo rankInfoTemp;
1258 0 : rankInfoTemp.deviceIp = cominfo.rankInfo[j].deviceIp;
1259 0 : rankInfoTemp.serverIp = cominfo.rankInfo[j].serverIp;
1260 0 : rankInfoTemp.podId = cominfo.rankInfo[j].podId;
1261 0 : rankInfoTemp.rev = cominfo.rankInfo[j].rev;
1262 0 : tab_f.rankInfo.push_back(rankInfoTemp);
1263 : }
1264 0 : tab_f.rankTotalNum = cominfo.rankTotalNum;
1265 0 : tab_f.rankNum = tab_f.rankInfo.size();
1266 0 : HCCL_INFO(
1267 : "[NSLB-DP] SendTableProc-N info:[%u]-[%u]-[%u]-[%u].", packetNum, packetIndex, cominfo.rankTotalNum,
1268 : tab_f.rankInfo.size());
1269 : }
1270 0 : HCCL_DEBUG(
1271 : "[NSLB-DP] SendRankTable-info:[%u]-[%u]-[%u]-[%u].", tab_f.rankTotalNum, tab_f.packetNum, tab_f.rankTotalNum,
1272 : tab_f.rankInfo.size());
1273 0 : SendRankTable(tab_f);
1274 :
1275 0 : return HCCL_SUCCESS;
1276 0 : }
1277 :
1278 : /* 遍历表1 执行send 流程 */
1279 1 : HcclResult hcclNslbDp::SendTableFir(uint32_t rank)
1280 : {
1281 1 : size_t size = hcclNslbDpCommConfig_.size();
1282 1 : HCCL_INFO("[NSLB-DP] SendTableFir size:[%u] success.", size);
1283 1 : for (size_t i = 0; i < hcclNslbDpCommConfig_.size(); i++) {
1284 0 : u32 rankTotalNum = hcclNslbDpCommConfig_[i].rankTotalNum;
1285 0 : if (rankTotalNum > NSLBDP_RANKTOTALNUM_BLOCK_FOU) {
1286 0 : return HCCL_SUCCESS;
1287 : }
1288 0 : u32 packetNum = NSLBDP_PKTNUM_FIR;
1289 0 : if (rankTotalNum <= NSLBDP_RANKTOTALNUM_BLOCK_FIR) {
1290 0 : packetNum = NSLBDP_PKTNUM_FIR;
1291 0 : } else if (rankTotalNum <= NSLBDP_RANKTOTALNUM_BLOCK_SEC) {
1292 0 : packetNum = NSLBDP_PKTNUM_SEC;
1293 0 : } else if (rankTotalNum <= NSLBDP_RANKTOTALNUM_BLOCK_THR) {
1294 0 : packetNum = NSLBDP_PKTNUM_THR;
1295 : } else {
1296 0 : packetNum = NSLBDP_PKTNUM_FOU;
1297 : }
1298 0 : SendTableProc(rank, packetNum, hcclNslbDpCommConfig_[i]);
1299 : }
1300 1 : HCCL_DEBUG("[NSLB-DP] entry SendTableFir end");
1301 1 : return HCCL_SUCCESS;
1302 : }
1303 :
1304 0 : HcclResult hcclNslbDp::SendRankTable(NslbDpCommConfigInfo tab_f)
1305 : {
1306 0 : if (GetInitNetCoFlag() == false) {
1307 0 : return HCCL_SUCCESS;
1308 : }
1309 :
1310 0 : u32 tablen = sizeof(tab_f);
1311 0 : HCCL_INFO("[NSLB-DP] SendRankTable NslbDpCommConfigInfo len:[%u].", tablen);
1312 :
1313 0 : std::vector<uint8_t> tlvData = serializeTLV_TableFir(tab_f);
1314 0 : u32 datlen = tlvData.size();
1315 0 : HCCL_INFO("[NSLB-DP] SendRankTable tlvData.len:[%u] success.", datlen);
1316 :
1317 0 : if (nslbdp_handle_ == nullptr) {
1318 0 : HCCL_INFO("[NSLB-DP] ndlbdp nslbdp_handle_ error SendRankTable.");
1319 0 : return HCCL_SUCCESS;
1320 : }
1321 :
1322 0 : nslb_msg sendMsg;
1323 0 : nslb_msg recvMsg;
1324 0 : sendMsg.type = NSLBDP_TYPE_TBL_COMM_INFO;
1325 0 : sendMsg.length = datlen;
1326 0 : sendMsg.data.assign(tlvData.begin(), tlvData.end());
1327 0 : s32 ret = H2DTlvRequest(
1328 0 : nslbdp_handle_, MODULE_TYPE_NSLB, reinterpret_cast<TlvMsg*>(&sendMsg), reinterpret_cast<TlvMsg*>(&recvMsg));
1329 :
1330 0 : HCCL_INFO("[NSLBDP-SENDTABLE] hccl send table NSLBDP_TYPE_TBL_COMM_INFO(1001) to hccp. ret(%d)\n", ret);
1331 0 : return HCCL_SUCCESS;
1332 0 : }
1333 :
1334 : /* 根将表2 序列化处理 */
1335 2 : std::vector<uint8_t> hcclNslbDp::serializeTLV_TableOpAndAdj(NslbDpOperatorInfo& info)
1336 : {
1337 2 : HCCL_DEBUG("[NSLB-DP] ndlbdp entry serializeTLV TableOpAndAdj.");
1338 2 : std::vector<uint8_t> tlvData;
1339 :
1340 : // 处理 taskId
1341 2 : uint64_t netTaskId = htobe64(info.taskId);
1342 2 : tlvData.resize(tlvData.size() + sizeof(netTaskId));
1343 2 : (void)memcpy_s(
1344 2 : tlvData.data() + tlvData.size() - sizeof(netTaskId), sizeof(netTaskId), &netTaskId, sizeof(netTaskId));
1345 :
1346 : // 处理 commDesc
1347 2 : tlvData.insert(tlvData.end(), info.commDesc, info.commDesc + COMM_DESC_MAX_LENGTH);
1348 :
1349 : // 处理 commInitTime
1350 2 : uint64_t netCommInitTime = htobe64(info.commInitTime);
1351 2 : tlvData.resize(tlvData.size() + sizeof(netCommInitTime));
1352 2 : (void)memcpy_s(
1353 2 : tlvData.data() + tlvData.size() - sizeof(netCommInitTime), sizeof(netCommInitTime), &netCommInitTime,
1354 : sizeof(netCommInitTime));
1355 :
1356 : // 处理 oper
1357 2 : tlvData.push_back(info.oper);
1358 :
1359 : // 处理 algorithm
1360 2 : tlvData.push_back(info.algorithm);
1361 :
1362 : // 处理 rootRank
1363 2 : uint16_t netRootRank = htons(info.rootRank);
1364 2 : tlvData.resize(tlvData.size() + sizeof(netRootRank));
1365 2 : (void)memcpy_s(
1366 2 : tlvData.data() + tlvData.size() - sizeof(netRootRank), sizeof(netRootRank), &netRootRank, sizeof(netRootRank));
1367 :
1368 : // 处理 trafficCnt
1369 2 : uint64_t netTrafficCnt = htobe64(info.trafficCnt);
1370 2 : tlvData.resize(tlvData.size() + sizeof(netTrafficCnt));
1371 2 : (void)memcpy_s(
1372 2 : tlvData.data() + tlvData.size() - sizeof(netTrafficCnt), sizeof(netTrafficCnt), &netTrafficCnt,
1373 : sizeof(netTrafficCnt));
1374 :
1375 : // 处理 l4SPortId
1376 2 : uint16_t netL4SPortId = htons(info.l4SPortId);
1377 2 : tlvData.resize(tlvData.size() + sizeof(netL4SPortId));
1378 2 : (void)memcpy_s(
1379 2 : tlvData.data() + tlvData.size() - sizeof(netL4SPortId), sizeof(netL4SPortId), &netL4SPortId,
1380 : sizeof(netL4SPortId));
1381 :
1382 : // 处理 maskLen
1383 2 : uint16_t netMaskLen = htons(info.maskLen);
1384 2 : tlvData.resize(tlvData.size() + sizeof(netMaskLen));
1385 2 : (void)memcpy_s(
1386 2 : tlvData.data() + tlvData.size() - sizeof(netMaskLen), sizeof(netMaskLen), &netMaskLen, sizeof(netMaskLen));
1387 2 : HCCL_INFO("[NSLB-DP] entry serializeTLV_TableOpAndAdj end.");
1388 2 : return tlvData;
1389 0 : }
1390 :
1391 : /* 执行send 流程 */
1392 2 : HcclResult hcclNslbDp::SendRankTableOpAndAdj(NslbDpOperatorInfo& tab_f)
1393 : {
1394 2 : if (GetInitNetCoFlag() == false) {
1395 1 : return HCCL_SUCCESS;
1396 : }
1397 1 : HCCL_DEBUG("[NSLB-DP] ndlbdp entry SendRankTableOpAndAdj.");
1398 1 : std::vector<uint8_t> tlvData = serializeTLV_TableOpAndAdj(tab_f);
1399 :
1400 1 : u32 datlen = tlvData.size();
1401 1 : HCCL_INFO("[NSLB-DP] SendRankTableOpAndAdj tlvData.len:[%u] success.", datlen);
1402 :
1403 1 : nslb_msg sendMsg;
1404 1 : nslb_msg recvMsg;
1405 1 : sendMsg.type = NSLBDP_TYPE_TBL_OPER;
1406 1 : sendMsg.length = datlen;
1407 1 : sendMsg.data.assign(tlvData.begin(), tlvData.end());
1408 1 : s32 ret = H2DTlvRequest(
1409 1 : nslbdp_handle_, MODULE_TYPE_NSLB, reinterpret_cast<TlvMsg*>(&sendMsg), reinterpret_cast<TlvMsg*>(&recvMsg));
1410 :
1411 1 : HCCL_INFO("[NSLBDP-SENDTABLE] hccl send table NSLBDP_TYPE_TBL_OPER(1002) to hccp. ret(%d)\n", ret);
1412 :
1413 1 : return HCCL_SUCCESS;
1414 1 : }
1415 :
1416 : /* 遍历表2 执行send 流程 */
1417 1 : HcclResult hcclNslbDp::SendOpAndAdjTable()
1418 : {
1419 1 : HCCL_DEBUG("[NSLB-DP] ndlbdp entry SendOpAndAdjTable.");
1420 2 : for (size_t i = 0; i < hcclNslbDpOperatorVal_.size(); i++) {
1421 1 : if (hcclNslbDpOperatorVal_[i].sedFlag == 1) {
1422 0 : continue;
1423 : }
1424 1 : NslbDpOperatorInfo tab_f = {};
1425 1 : tab_f.taskId = hcclNslbDpOperatorVal_[i].taskId;
1426 2 : s32 sRet = memcpy_s(
1427 1 : tab_f.commDesc, sizeof(tab_f.commDesc), hcclNslbDpOperatorVal_[i].commDesc, COMM_DESC_MAX_LENGTH);
1428 1 : if (sRet != EOK) {
1429 0 : HCCL_ERROR("memcpy_s commDesc fail");
1430 : }
1431 1 : tab_f.commInitTime = hcclNslbDpOperatorVal_[i].commInitTime;
1432 1 : tab_f.rootRank = hcclNslbDpOperatorVal_[i].rootRank;
1433 1 : tab_f.oper = hcclNslbDpOperatorVal_[i].oper;
1434 1 : tab_f.algorithm = hcclNslbDpOperatorVal_[i].algorithm;
1435 1 : tab_f.trafficCnt = hcclNslbDpOperatorVal_[i].trafficCnt;
1436 1 : tab_f.l4SPortId = hcclNslbDpOperatorVal_[i].l4SPortId;
1437 1 : tab_f.maskLen = hcclNslbDpOperatorVal_[i].maskLen;
1438 1 : SendRankTableOpAndAdj(tab_f);
1439 1 : hcclNslbDpOperatorVal_[i].sedFlag = 1;
1440 1 : HCCL_INFO("[NSLB-DP] try to sen RankTableOpAndAdj times:[%u].", i);
1441 : }
1442 1 : HCCL_INFO("[NSLB-DP] SendOpAndAdjTable end.");
1443 1 : return HCCL_SUCCESS;
1444 : }
1445 :
1446 : /* 根将表3 序列化处理 */
1447 2 : std::vector<uint8_t> hcclNslbDp::serializeTLV_TableAlgorithmInfo(NslbDpAlgorithmTlv& info)
1448 : {
1449 2 : HCCL_DEBUG("[NSLB-DP] entry serializeTLV_TableAlgorithmInfo.");
1450 2 : std::vector<uint8_t> tlvData;
1451 :
1452 : // 处理 taskId
1453 2 : uint64_t netTaskId = htobe64(info.taskId);
1454 2 : tlvData.resize(tlvData.size() + sizeof(netTaskId));
1455 2 : (void)memcpy_s(
1456 2 : tlvData.data() + tlvData.size() - sizeof(netTaskId), sizeof(netTaskId), &netTaskId, sizeof(netTaskId));
1457 :
1458 : // 处理 commDesc
1459 2 : tlvData.insert(tlvData.end(), info.commDesc, info.commDesc + COMM_DESC_MAX_LENGTH);
1460 :
1461 : // CommMd5Sum
1462 2 : tlvData.insert(tlvData.end(), info.commMd5Sum, info.commMd5Sum + sizeof(info.commMd5Sum));
1463 :
1464 : // 处理 srcLocalRankId
1465 2 : uint16_t netSrcLocalRankId = htons(info.srcLocalRankId);
1466 2 : tlvData.resize(tlvData.size() + sizeof(netSrcLocalRankId));
1467 2 : (void)memcpy_s(
1468 2 : tlvData.data() + tlvData.size() - sizeof(netSrcLocalRankId), sizeof(netSrcLocalRankId), &netSrcLocalRankId,
1469 : sizeof(netSrcLocalRankId));
1470 :
1471 : // 处理 oper
1472 2 : tlvData.push_back(info.oper);
1473 :
1474 : // 处理 algorithm
1475 2 : tlvData.push_back(info.algorithm);
1476 :
1477 : // 处理 rootRank
1478 2 : uint16_t netRootRank = htons(info.rootRank);
1479 2 : tlvData.resize(tlvData.size() + sizeof(netRootRank));
1480 2 : (void)memcpy_s(
1481 2 : tlvData.data() + tlvData.size() - sizeof(netRootRank), sizeof(netRootRank), &netRootRank, sizeof(netRootRank));
1482 :
1483 : // 处理 rev
1484 2 : uint16_t netRev = htons(info.rev);
1485 2 : tlvData.resize(tlvData.size() + sizeof(netRev));
1486 2 : (void)memcpy_s(tlvData.data() + tlvData.size() - sizeof(netRev), sizeof(netRev), &netRev, sizeof(netRev));
1487 :
1488 : // 处理 dstRankNum
1489 2 : uint16_t netDstRankNum = htons(info.dstRankNum);
1490 2 : tlvData.resize(tlvData.size() + sizeof(netDstRankNum));
1491 2 : (void)memcpy_s(
1492 2 : tlvData.data() + tlvData.size() - sizeof(netDstRankNum), sizeof(netDstRankNum), &netDstRankNum,
1493 : sizeof(netDstRankNum));
1494 :
1495 : // 处理 revSecond
1496 2 : uint16_t netRevSecond = htons(info.revsecond);
1497 2 : tlvData.resize(tlvData.size() + sizeof(netRevSecond));
1498 2 : (void)memcpy_s(
1499 2 : tlvData.data() + tlvData.size() - sizeof(netRevSecond), sizeof(netRevSecond), &netRevSecond,
1500 : sizeof(netRevSecond));
1501 :
1502 2 : HCCL_INFO("[NSLB-DP] serializeTLV_TableAlgorithmInfo proc adjInfo.");
1503 : // 处理 adjInfo
1504 7 : for (const auto& adj : info.AdjInfo) {
1505 5 : uint16_t netDstLocalRankId = htons(adj.dstLocalRankId);
1506 5 : tlvData.resize(tlvData.size() + sizeof(netDstLocalRankId));
1507 5 : (void)memcpy_s(
1508 5 : tlvData.data() + tlvData.size() - sizeof(netDstLocalRankId), sizeof(netDstLocalRankId), &netDstLocalRankId,
1509 : sizeof(netDstLocalRankId));
1510 5 : tlvData.push_back(adj.phaseId);
1511 5 : tlvData.push_back(adj.rev);
1512 5 : HCCL_INFO("[NSLB-DP] serializeTLV_TableAlgo adjInfo:***[%u]***[%c]***.", adj.dstLocalRankId, adj.phaseId);
1513 : }
1514 2 : HCCL_DEBUG("[NSLB-DP] serializeTLV_TableAlgo proc adjInfo end.");
1515 :
1516 2 : return tlvData;
1517 0 : }
1518 :
1519 1 : HcclResult hcclNslbDp::SendRankTableAlgorithmInfo(NslbDpAlgorithmTlv& tab_f)
1520 : {
1521 1 : if (GetInitNetCoFlag() == false) {
1522 0 : return HCCL_SUCCESS;
1523 : }
1524 1 : std::vector<uint8_t> tlvData = serializeTLV_TableAlgorithmInfo(tab_f);
1525 :
1526 1 : u32 datlen = tlvData.size();
1527 1 : HCCL_INFO("[NSLB-DP] SendRankTableAlgorithmInfo tlvData.len:[%u] success.", datlen);
1528 :
1529 1 : nslb_msg sendMsg;
1530 1 : nslb_msg recvMsg;
1531 1 : sendMsg.type = NSLBDP_TYPE_TBL_ADJ;
1532 1 : sendMsg.length = datlen;
1533 1 : sendMsg.data.assign(tlvData.begin(), tlvData.end());
1534 1 : s32 ret = H2DTlvRequest(
1535 1 : nslbdp_handle_, MODULE_TYPE_NSLB, reinterpret_cast<TlvMsg*>(&sendMsg), reinterpret_cast<TlvMsg*>(&recvMsg));
1536 :
1537 1 : HCCL_INFO("[NSLBDP-SENDTABLE] hccl send table NSLBDP_TYPE_TBL_ADJ(1003) to hccp. ret(%d)\n", ret);
1538 1 : return HCCL_SUCCESS;
1539 1 : }
1540 :
1541 : /* 遍历表3 执行send 流程 */
1542 1 : HcclResult hcclNslbDp::SendAlgorithmInfoTable()
1543 : {
1544 1 : u32 size = hcclNslbDpAlgorithmInfo_.size();
1545 1 : HCCL_INFO("[NSLB-DP] ndlbdp entry SendAlgorithmInfoTable size=[%u].", size);
1546 :
1547 2 : for (size_t i = 0; i < hcclNslbDpAlgorithmInfo_.size(); i++) {
1548 1 : if (hcclNslbDpAlgorithmInfo_[i].sedFlag == 1) {
1549 0 : continue;
1550 : }
1551 1 : NslbDpAlgorithmTlv tab_f = {};
1552 1 : tab_f.taskId = hcclNslbDpAlgorithmInfo_[i].taskId;
1553 2 : s32 sRet = memcpy_s(
1554 1 : tab_f.commDesc, sizeof(tab_f.commDesc), hcclNslbDpAlgorithmInfo_[i].commDesc, COMM_DESC_MAX_LENGTH);
1555 2 : sRet = memcpy_s(
1556 1 : tab_f.commMd5Sum, sizeof(tab_f.commMd5Sum), hcclNslbDpAlgorithmInfo_[i].commMd5Sum,
1557 : sizeof(hcclNslbDpAlgorithmInfo_[i].commMd5Sum));
1558 1 : if (sRet != EOK) {
1559 0 : HCCL_INFO("memcpy_s commDesc fail");
1560 : }
1561 1 : tab_f.srcLocalRankId = hcclNslbDpAlgorithmInfo_[i].srcLocalRankId;
1562 1 : tab_f.rootRank = hcclNslbDpAlgorithmInfo_[i].rootRank;
1563 1 : tab_f.oper = hcclNslbDpAlgorithmInfo_[i].oper;
1564 1 : tab_f.algorithm = hcclNslbDpAlgorithmInfo_[i].algorithm;
1565 :
1566 1 : tab_f.rev = 0;
1567 :
1568 1 : tab_f.dstRankNum = hcclNslbDpAlgorithmInfo_[i].dstRankNum;
1569 1 : tab_f.revsecond = 0;
1570 1 : tab_f.AdjInfo = hcclNslbDpAlgorithmInfo_[i].AdjInfo;
1571 3 : for (const auto& adj : hcclNslbDpAlgorithmInfo_[i].AdjInfo) {
1572 2 : tab_f.AdjInfo.push_back(adj);
1573 : }
1574 1 : SendRankTableAlgorithmInfo(tab_f);
1575 1 : HCCL_INFO("[NSLB-DP] try to sen AlgorithmInfoTable times:[%u].", i);
1576 :
1577 1 : hcclNslbDpAlgorithmInfo_[i].sedFlag = 1;
1578 1 : }
1579 1 : HCCL_DEBUG("[NSLB-DP] entry SendAlgorithmInfoTable end.");
1580 :
1581 1 : return HCCL_SUCCESS;
1582 : }
1583 :
1584 : /* 序列化表4 */
1585 0 : std::vector<uint8_t> hcclNslbDp::serializeTLV_TableGlobalRankInfo(NslbDpGlobalRankInfo& info)
1586 : {
1587 0 : HCCL_DEBUG("[NSLB-DP] entry serializeTLV_TableGlobalRankInfo.");
1588 0 : std::vector<uint8_t> tlvData;
1589 : // 处理 taskId
1590 0 : uint64_t netTaskId = htobe64(info.taskId);
1591 0 : tlvData.resize(tlvData.size() + sizeof(netTaskId));
1592 0 : (void)memcpy_s(
1593 0 : tlvData.data() + tlvData.size() - sizeof(netTaskId), sizeof(netTaskId), &netTaskId, sizeof(netTaskId));
1594 :
1595 : // 处理 commDesc
1596 0 : tlvData.insert(tlvData.end(), info.commDesc, info.commDesc + COMM_DESC_MAX_LENGTH);
1597 :
1598 : // 处理 commInitTime
1599 0 : uint64_t netCommInitTime = htobe64(info.commInitTime);
1600 0 : tlvData.resize(tlvData.size() + sizeof(netCommInitTime));
1601 0 : (void)memcpy_s(
1602 0 : tlvData.data() + tlvData.size() - sizeof(netCommInitTime), sizeof(netCommInitTime), &netCommInitTime,
1603 : sizeof(netCommInitTime));
1604 :
1605 : // 处理 packetId
1606 0 : uint16_t netPacketId = htons(info.packetId);
1607 0 : tlvData.resize(tlvData.size() + sizeof(netPacketId));
1608 0 : (void)memcpy_s(
1609 0 : tlvData.data() + tlvData.size() - sizeof(netPacketId), sizeof(netPacketId), &netPacketId, sizeof(netPacketId));
1610 :
1611 : // 处理 rev
1612 0 : uint16_t netRev = htons(info.rev);
1613 0 : tlvData.resize(tlvData.size() + sizeof(netRev));
1614 0 : (void)memcpy_s(tlvData.data() + tlvData.size() - sizeof(netRev), sizeof(netRev), &netRev, sizeof(netRev));
1615 :
1616 : // 处理 packetNum
1617 0 : uint16_t netPacketNum = htons(info.packetNum);
1618 0 : tlvData.resize(tlvData.size() + sizeof(netPacketNum));
1619 0 : (void)memcpy_s(
1620 0 : tlvData.data() + tlvData.size() - sizeof(netPacketNum), sizeof(netPacketNum), &netPacketNum,
1621 : sizeof(netPacketNum));
1622 0 : HCCL_INFO("[NSLB-DP] hcclNslbDpGlobalRankVal_ info.packetNum:[%u]", netPacketNum);
1623 :
1624 : // 处理 rev2
1625 0 : uint16_t netRev2 = htons(info.rev2);
1626 0 : tlvData.resize(tlvData.size() + sizeof(netRev2));
1627 0 : (void)memcpy_s(tlvData.data() + tlvData.size() - sizeof(netRev2), sizeof(netRev2), &netRev2, sizeof(netRev2));
1628 :
1629 : // 处理 sendRankInfo
1630 0 : for (const auto& rankInfo : info.sendRankInfo) {
1631 0 : uint32_t netDeviceIp = htonl(rankInfo.deviceIp);
1632 0 : tlvData.resize(tlvData.size() + sizeof(netDeviceIp));
1633 0 : (void)memcpy_s(
1634 0 : tlvData.data() + tlvData.size() - sizeof(netDeviceIp), sizeof(netDeviceIp), &netDeviceIp,
1635 : sizeof(netDeviceIp));
1636 : }
1637 :
1638 : // 处理 commMd5Sum
1639 0 : tlvData.insert(tlvData.end(), info.commMd5Sum, info.commMd5Sum + sizeof(info.commMd5Sum));
1640 :
1641 : // 处理 rankTotalNum
1642 0 : uint32_t netRankTotalNum = htonl(info.rankTotalNum);
1643 0 : tlvData.resize(tlvData.size() + sizeof(netRankTotalNum));
1644 0 : (void)memcpy_s(
1645 0 : tlvData.data() + tlvData.size() - sizeof(netRankTotalNum), sizeof(netRankTotalNum), &netRankTotalNum,
1646 : sizeof(netRankTotalNum));
1647 :
1648 : // 处理 rankNum
1649 0 : uint16_t netRankNum = htons(info.rankNum);
1650 0 : tlvData.resize(tlvData.size() + sizeof(netRankNum));
1651 0 : (void)memcpy_s(
1652 0 : tlvData.data() + tlvData.size() - sizeof(netRankNum), sizeof(netRankNum), &netRankNum, sizeof(netRankNum));
1653 :
1654 : // 处理 rev3
1655 0 : uint16_t netRev3 = htons(info.rev3);
1656 0 : tlvData.resize(tlvData.size() + sizeof(netRev3));
1657 0 : (void)memcpy_s(tlvData.data() + tlvData.size() - sizeof(netRev3), sizeof(netRev3), &netRev3, sizeof(netRev3));
1658 :
1659 : // 处理 rankInfo
1660 0 : for (const auto& rankInfo : info.rankInfo) {
1661 0 : uint32_t netDeviceIp = htonl(rankInfo.deviceIp);
1662 0 : uint32_t netServerIp = htonl(rankInfo.serverIp);
1663 0 : tlvData.resize(tlvData.size() + sizeof(netDeviceIp));
1664 0 : (void)memcpy_s(
1665 0 : tlvData.data() + tlvData.size() - sizeof(netDeviceIp), sizeof(netDeviceIp), &netDeviceIp,
1666 : sizeof(netDeviceIp));
1667 0 : tlvData.resize(tlvData.size() + sizeof(netServerIp));
1668 0 : (void)memcpy_s(
1669 0 : tlvData.data() + tlvData.size() - sizeof(netServerIp), sizeof(netServerIp), &netServerIp,
1670 : sizeof(netServerIp));
1671 : }
1672 0 : return tlvData;
1673 0 : }
1674 :
1675 0 : HcclResult hcclNslbDp::SendRankTableGlobalRank(NslbDpGlobalRankInfo& tab_f)
1676 : {
1677 0 : if (GetInitNetCoFlag() == false) {
1678 0 : return HCCL_SUCCESS;
1679 : }
1680 0 : std::vector<uint8_t> tlvData = serializeTLV_TableGlobalRankInfo(tab_f);
1681 0 : u32 datlen = tlvData.size();
1682 0 : HCCL_INFO("[NSLB-DP] SendRankTableGlobalRank tlvData.len:[%u] success.", datlen);
1683 :
1684 0 : nslb_msg sendMsg;
1685 0 : nslb_msg recvMsg;
1686 0 : sendMsg.type = NSLBDP_TYPE_TBL_RANK;
1687 0 : sendMsg.length = datlen;
1688 0 : sendMsg.data.assign(tlvData.begin(), tlvData.end());
1689 0 : s32 ret = H2DTlvRequest(
1690 0 : nslbdp_handle_, MODULE_TYPE_NSLB, reinterpret_cast<TlvMsg*>(&sendMsg), reinterpret_cast<TlvMsg*>(&recvMsg));
1691 :
1692 0 : HCCL_INFO("[NSLBDP-SENDTABLE] hccl send table NSLBDP_TYPE_TBL_RANK(1004) to hccp. ret(%d)\n", ret);
1693 :
1694 0 : return HCCL_SUCCESS;
1695 0 : }
1696 :
1697 0 : void hcclNslbDp::fullCommonGlobalRankInfo(NslbDpGlobalRankInfo tab_f, NslbDpGlobalRankVal& cominfo)
1698 : {
1699 0 : tab_f.rev = 0;
1700 0 : tab_f.rev2 = 0;
1701 0 : tab_f.rev3 = 0;
1702 0 : tab_f.taskId = cominfo.taskId;
1703 0 : s32 sRet = memcpy_s(tab_f.commDesc, sizeof(tab_f.commDesc), cominfo.commDesc, sizeof(cominfo.commDesc));
1704 0 : if (sRet != EOK) {
1705 0 : HCCL_INFO("memcpy_s commDesc fail");
1706 : }
1707 0 : tab_f.commInitTime = cominfo.commInitTime;
1708 0 : sRet = memcpy_s(tab_f.commMd5Sum, sizeof(tab_f.commMd5Sum), cominfo.commMd5Sum, sizeof(cominfo.commMd5Sum));
1709 0 : if (sRet != EOK) {
1710 0 : HCCL_ERROR("memcpy_s commMD5 fail");
1711 : }
1712 :
1713 0 : for (u32 dip = 0; dip < tab_f.packetNum; dip++) {
1714 0 : if (cominfo.rankInfo.size() == 0) {
1715 0 : break;
1716 : }
1717 0 : tab_f.sendRankInfo[dip].deviceIp = cominfo.rankInfo[dip].deviceIp;
1718 : }
1719 0 : return;
1720 : }
1721 :
1722 : /* 根据表4 中的rank数量进行分片处理 */
1723 0 : HcclResult hcclNslbDp::SendTableGlobalRankProc(uint32_t rank, uint32_t packetNum, NslbDpGlobalRankVal& cominfo)
1724 : {
1725 0 : HCCL_INFO("[NSLB-DP] entry SendTableGlobalRankProc packetNum:[%u]", packetNum);
1726 0 : if (packetNum - 1 < rank) {
1727 0 : HCCL_INFO("[NSLB-DP] rank [%u] no need send TableFir(packetNum[%u]).", rank, packetNum);
1728 0 : return HCCL_SUCCESS;
1729 : }
1730 0 : u32 packetIndex = rank;
1731 0 : NslbDpGlobalRankInfo tab_f = {};
1732 0 : tab_f.packetId = packetIndex;
1733 0 : tab_f.packetNum = packetNum;
1734 0 : fullCommonGlobalRankInfo(tab_f, cominfo);
1735 0 : if (packetIndex != packetNum - 1) {
1736 0 : uint32_t start = packetIndex * NSLBDP_RANKTOTALNUM_BLOCK_FIR;
1737 0 : uint32_t end = (packetIndex + 1) * NSLBDP_RANKTOTALNUM_BLOCK_FIR;
1738 0 : end = (end > cominfo.rankInfo.size()) ? cominfo.rankInfo.size() : end;
1739 0 : for (uint32_t j = start; j < end; ++j) {
1740 : TableFourRankInfo rankInfoTemp;
1741 0 : rankInfoTemp.deviceIp = cominfo.rankInfo[j].deviceIp;
1742 0 : rankInfoTemp.serverIp = cominfo.rankInfo[j].serverIp;
1743 0 : tab_f.rankInfo.push_back(rankInfoTemp);
1744 : }
1745 : } else {
1746 0 : uint32_t rankNum = cominfo.rankInfo.size();
1747 0 : uint32_t start = packetIndex * NSLBDP_RANKTOTALNUM_BLOCK_FIR;
1748 :
1749 0 : for (uint32_t j = start; j < rankNum; j++) {
1750 : TableFourRankInfo rankInfoTemp;
1751 0 : rankInfoTemp.deviceIp = cominfo.rankInfo[j].deviceIp;
1752 0 : rankInfoTemp.serverIp = cominfo.rankInfo[j].serverIp;
1753 0 : tab_f.rankInfo.push_back(rankInfoTemp);
1754 : }
1755 : }
1756 0 : tab_f.rankTotalNum = cominfo.rankTotalNum;
1757 0 : tab_f.rankNum = tab_f.rankInfo.size();
1758 0 : SendRankTableGlobalRank(tab_f);
1759 0 : return HCCL_SUCCESS;
1760 0 : }
1761 :
1762 : /* 根据rankNum的总数做分片发送流程 */
1763 0 : HcclResult hcclNslbDp::SendGlobalRankTable(uint32_t rank)
1764 : {
1765 0 : HCCL_DEBUG("[NSLB-DP] try to send GlobalRankTable.");
1766 0 : u32 rankTotalNum = hcclNslbDpGlobalRankVal_.rankTotalNum;
1767 0 : if (rankTotalNum > NSLBDP_RANKTOTALNUM_BLOCK_FOU) {
1768 0 : return HCCL_SUCCESS;
1769 : }
1770 0 : u32 packetNum = rankTotalNum / NSLBDP_PKTNUM_FIR;
1771 0 : if (rankTotalNum <= NSLBDP_RANKTOTALNUM_BLOCK_FIR) {
1772 0 : packetNum = NSLBDP_PKTNUM_FIR;
1773 0 : } else if (rankTotalNum <= NSLBDP_RANKTOTALNUM_BLOCK_SEC) {
1774 0 : packetNum = NSLBDP_PKTNUM_SEC;
1775 0 : } else if (rankTotalNum <= NSLBDP_RANKTOTALNUM_BLOCK_THR) {
1776 0 : packetNum = NSLBDP_PKTNUM_THR;
1777 : } else {
1778 0 : packetNum = NSLBDP_PKTNUM_FOU;
1779 : }
1780 0 : SendTableGlobalRankProc(rank, packetNum, hcclNslbDpGlobalRankVal_);
1781 :
1782 0 : return HCCL_SUCCESS;
1783 : }
1784 :
1785 : /* 序列化表5 */
1786 0 : std::vector<uint8_t> hcclNslbDp::serializeTLV_TableGlobalDisRankVal(NslbDpGlobalDisRankVal& info)
1787 : {
1788 0 : HCCL_DEBUG("[NSLB-DP] entry serializeTLV_TableGlobalDisRankVal.");
1789 0 : std::vector<uint8_t> tlvData;
1790 :
1791 : // 处理 taskId
1792 0 : uint64_t netTaskId = htobe64(info.taskId);
1793 0 : tlvData.resize(tlvData.size() + sizeof(netTaskId));
1794 0 : (void)memcpy_s(
1795 0 : tlvData.data() + tlvData.size() - sizeof(netTaskId), sizeof(netTaskId), &netTaskId, sizeof(netTaskId));
1796 :
1797 : // 处理 npuIp
1798 0 : uint32_t netNpuIp = htonl(info.npuIp);
1799 0 : tlvData.resize(tlvData.size() + sizeof(netNpuIp));
1800 0 : (void)memcpy_s(tlvData.data() + tlvData.size() - sizeof(netNpuIp), sizeof(netNpuIp), &netNpuIp, sizeof(netNpuIp));
1801 :
1802 : // 处理 serverIp
1803 0 : uint32_t netServerIp = htonl(info.serverIp);
1804 0 : tlvData.resize(tlvData.size() + sizeof(netServerIp));
1805 0 : (void)memcpy_s(
1806 0 : tlvData.data() + tlvData.size() - sizeof(netServerIp), sizeof(netServerIp), &netServerIp, sizeof(netServerIp));
1807 :
1808 : // 处理 nodeId
1809 0 : uint32_t netNodeId = htonl(info.nodeId);
1810 0 : tlvData.resize(tlvData.size() + sizeof(netNodeId));
1811 0 : (void)memcpy_s(
1812 0 : tlvData.data() + tlvData.size() - sizeof(netNodeId), sizeof(netNodeId), &netNodeId, sizeof(netNodeId));
1813 :
1814 : // 处理 localRankNum
1815 0 : tlvData.push_back(info.localRankNum);
1816 :
1817 : // 处理 rev
1818 0 : tlvData.insert(tlvData.end(), info.rev, info.rev + sizeof(info.rev));
1819 :
1820 : // 处理 rankTotalNum
1821 0 : uint32_t netRankTotalNum = htonl(info.rankTotalNum);
1822 0 : tlvData.resize(tlvData.size() + sizeof(netRankTotalNum));
1823 0 : (void)memcpy_s(
1824 0 : tlvData.data() + tlvData.size() - sizeof(netRankTotalNum), sizeof(netRankTotalNum), &netRankTotalNum,
1825 : sizeof(netRankTotalNum));
1826 :
1827 0 : return tlvData;
1828 0 : }
1829 :
1830 : /* 发送GlobalDisRank表信息 */
1831 0 : HcclResult hcclNslbDp::SendRankTableGlobalDisRankVal(NslbDpGlobalDisRankVal& tab_f)
1832 : {
1833 0 : if (GetInitNetCoFlag() == false) {
1834 0 : return HCCL_SUCCESS;
1835 : }
1836 0 : std::vector<uint8_t> tlvData = serializeTLV_TableGlobalDisRankVal(tab_f);
1837 0 : u32 datlen = tlvData.size();
1838 0 : HCCL_INFO("[NSLB-DP] SendRankTableGlobalDisRankVal tlvData.len:[%u] success.", datlen);
1839 :
1840 0 : nslb_msg sendMsg;
1841 0 : nslb_msg recvMsg;
1842 0 : sendMsg.type = NSLBDP_TYPE_TBL_RANK_DIST;
1843 0 : sendMsg.length = datlen;
1844 0 : sendMsg.data.assign(tlvData.begin(), tlvData.end());
1845 0 : s32 ret = H2DTlvRequest(
1846 0 : nslbdp_handle_, MODULE_TYPE_NSLB, reinterpret_cast<TlvMsg*>(&sendMsg), reinterpret_cast<TlvMsg*>(&recvMsg));
1847 :
1848 0 : HCCL_INFO("[NSLBDP-SENDTABLE] hccl send table NSLBDP_TYPE_TBL_RANK_DIST(1005) to hccp. ret(%d)\n", ret);
1849 0 : return HCCL_SUCCESS;
1850 0 : }
1851 :
1852 : /* 遍历表5 执行send流程 */
1853 0 : HcclResult hcclNslbDp::SendGlobalDisRankTable()
1854 : {
1855 0 : HCCL_DEBUG("[NSLB-DP] ndlbdp entry serializeTLV_TableGlobalDisRankVal.");
1856 0 : NslbDpGlobalDisRankVal tab_f = {};
1857 0 : tab_f.taskId = hcclNslbDpGlobalDisRankVal_.taskId;
1858 0 : tab_f.npuIp = hcclNslbDpGlobalDisRankVal_.npuIp;
1859 0 : tab_f.serverIp = hcclNslbDpGlobalDisRankVal_.serverIp;
1860 0 : tab_f.nodeId = hcclNslbDpGlobalDisRankVal_.nodeId;
1861 0 : tab_f.localRankNum = hcclNslbDpGlobalDisRankVal_.localRankNum;
1862 0 : s32 sRet = memcpy_s(
1863 0 : tab_f.rev, sizeof(tab_f.rev), hcclNslbDpGlobalDisRankVal_.rev, sizeof(hcclNslbDpGlobalDisRankVal_.rev));
1864 0 : if (sRet != EOK) {
1865 0 : HCCL_INFO("memcpy_s rev info fail");
1866 : }
1867 0 : tab_f.rankTotalNum = hcclNslbDpGlobalDisRankVal_.rankTotalNum;
1868 :
1869 0 : SendRankTableGlobalDisRankVal(tab_f);
1870 0 : return HCCL_SUCCESS;
1871 : }
1872 :
1873 : /* 序列化表6 */
1874 0 : std::vector<uint8_t> hcclNslbDp::serializeTLV_TableRootRank(NslbDpRootRank& config)
1875 : {
1876 0 : HCCL_DEBUG("[NSLB-DP] ndlbdp entry serializeTLV_TableRootRank.");
1877 0 : std::vector<uint8_t> tlvData;
1878 :
1879 : // 处理 taskId
1880 0 : uint64_t netTaskId = htobe64(config.taskId);
1881 0 : tlvData.resize(tlvData.size() + sizeof(netTaskId));
1882 0 : (void)memcpy_s(
1883 0 : tlvData.data() + tlvData.size() - sizeof(netTaskId), sizeof(netTaskId), &netTaskId, sizeof(netTaskId));
1884 :
1885 : // 处理 commDesc
1886 0 : tlvData.insert(tlvData.end(), config.commDesc, config.commDesc + COMM_DESC_MAX_LENGTH);
1887 :
1888 : // 处理 commInitTime
1889 0 : uint64_t netCommInitTime = htobe64(config.commInitTime);
1890 0 : tlvData.resize(tlvData.size() + sizeof(netCommInitTime));
1891 0 : (void)memcpy_s(
1892 0 : tlvData.data() + tlvData.size() - sizeof(netCommInitTime), sizeof(netCommInitTime), &netCommInitTime,
1893 : sizeof(netCommInitTime));
1894 :
1895 : // 处理 oper
1896 0 : tlvData.push_back(config.oper);
1897 :
1898 : // 处理 algorithm
1899 0 : tlvData.push_back(config.algorithm);
1900 :
1901 : // 处理 revfir
1902 0 : uint16_t netRevfir = htons(config.revfir);
1903 0 : tlvData.resize(tlvData.size() + sizeof(netRevfir));
1904 0 : (void)memcpy_s(
1905 0 : tlvData.data() + tlvData.size() - sizeof(netRevfir), sizeof(netRevfir), &netRevfir, sizeof(netRevfir));
1906 :
1907 : // 处理 rootRankNum
1908 0 : uint16_t netRootRankNum = htons(config.rootRankNum);
1909 0 : tlvData.resize(tlvData.size() + sizeof(netRootRankNum));
1910 0 : (void)memcpy_s(
1911 0 : tlvData.data() + tlvData.size() - sizeof(netRootRankNum), sizeof(netRootRankNum), &netRootRankNum,
1912 : sizeof(netRootRankNum));
1913 :
1914 : // 处理 revsec
1915 0 : uint16_t netRevsec = htons(config.revsec);
1916 0 : tlvData.resize(tlvData.size() + sizeof(netRevsec));
1917 0 : (void)memcpy_s(
1918 0 : tlvData.data() + tlvData.size() - sizeof(netRevsec), sizeof(netRevsec), &netRevsec, sizeof(netRevsec));
1919 :
1920 : // 处理 rankId
1921 0 : for (const auto& id : config.rankId) {
1922 0 : uint32_t netRankId = htonl(id.rankID);
1923 0 : tlvData.resize(tlvData.size() + sizeof(netRankId));
1924 0 : (void)memcpy_s(
1925 0 : tlvData.data() + tlvData.size() - sizeof(netRankId), sizeof(netRankId), &netRankId, sizeof(netRankId));
1926 : }
1927 0 : return tlvData;
1928 0 : }
1929 :
1930 : /* 发送Rank表到netco */
1931 0 : HcclResult hcclNslbDp::SendRankTableRootRank(NslbDpRootRank& tab_f)
1932 : {
1933 0 : if (GetInitNetCoFlag() == false) {
1934 0 : return HCCL_SUCCESS;
1935 : }
1936 0 : std::vector<uint8_t> tlvData = serializeTLV_TableRootRank(tab_f);
1937 :
1938 0 : u32 datlen = tlvData.size();
1939 0 : HCCL_DEBUG("[NSLB-DP] SendRankTableRootRank tlvData.len:[%u] success.", datlen);
1940 :
1941 0 : nslb_msg sendMsg;
1942 0 : nslb_msg recvMsg;
1943 0 : sendMsg.type = NSLBDP_TYPE_TBL_ROOT_RANK;
1944 0 : sendMsg.length = datlen;
1945 0 : sendMsg.data.assign(tlvData.begin(), tlvData.end());
1946 0 : s32 ret = H2DTlvRequest(
1947 0 : nslbdp_handle_, MODULE_TYPE_NSLB, reinterpret_cast<TlvMsg*>(&sendMsg), reinterpret_cast<TlvMsg*>(&recvMsg));
1948 :
1949 0 : HCCL_INFO("[NSLBDP-SENDTABLE] hccl send table NSLBDP_TYPE_TBL_ROOT_RANK(1006) to hccp. ret(%d)\n", ret);
1950 0 : return HCCL_SUCCESS;
1951 0 : }
1952 :
1953 : /* 遍历表6 执行send流程 */
1954 0 : HcclResult hcclNslbDp::SendRootRankTable()
1955 : {
1956 0 : HCCL_DEBUG("[NSLB-DP] entry SendRootRankTable.");
1957 0 : NslbDpRootRank tab_f = {};
1958 0 : tab_f.taskId = hcclNslbDpRootRankVal_.taskId;
1959 0 : s32 sRet = memcpy_s(
1960 0 : tab_f.commDesc, sizeof(tab_f.commDesc), hcclNslbDpRootRankVal_.commDesc,
1961 : sizeof(hcclNslbDpRootRankVal_.commDesc));
1962 0 : if (sRet != EOK) {
1963 0 : HCCL_INFO("memcpy_s commDesc info fail");
1964 : }
1965 0 : tab_f.commInitTime = hcclNslbDpRootRankVal_.commInitTime;
1966 0 : tab_f.oper = hcclNslbDpRootRankVal_.oper;
1967 0 : tab_f.algorithm = hcclNslbDpRootRankVal_.algorithm;
1968 0 : tab_f.revfir = hcclNslbDpRootRankVal_.revfir;
1969 0 : tab_f.rootRankNum = hcclNslbDpRootRankVal_.rootRankNum;
1970 0 : tab_f.revsec = hcclNslbDpRootRankVal_.revsec;
1971 0 : tab_f.rankId = hcclNslbDpRootRankVal_.rankId;
1972 :
1973 0 : for (const auto& rank : hcclNslbDpRootRankVal_.rankId) {
1974 0 : tab_f.rankId.push_back(rank);
1975 : }
1976 :
1977 0 : SendRankTableRootRank(tab_f);
1978 :
1979 0 : return HCCL_SUCCESS;
1980 0 : }
1981 :
1982 : /* 初始化NetCo通道 */
1983 0 : HcclResult hcclNslbDp::InitNetCo()
1984 : {
1985 0 : if (hcclH2dTlv::GetInstance().GetH2dTlvInitFlag() != true) {
1986 0 : HCCL_INFO("Check GetH2dTlvInitFlag is not success");
1987 0 : return HCCL_SUCCESS;
1988 : }
1989 : /* 避免二次初始化 */
1990 0 : if (GetInitNetCoFlag() == true) {
1991 0 : HCCL_INFO("Get getHccpInitFlag is true");
1992 0 : return HCCL_SUCCESS;
1993 : }
1994 0 : if (hcclH2dTlv::GetInstance().GetH2dTlvBufferSize() == NSLBDP_ILLEGAL_TLVBUFFERSIZE) {
1995 : /* 异常场景处理 */
1996 0 : HCCL_INFO("Check H2dTlvBufferSize equal 0.");
1997 0 : ClearInitNetCoFlag();
1998 0 : return HCCL_E_NOT_SUPPORT;
1999 : }
2000 0 : if (hcclH2dTlv::GetInstance().GetH2dTlvHandle() == nullptr) {
2001 0 : HCCL_ERROR("Check InitNetCo handle is null.");
2002 0 : ClearInitNetCoFlag();
2003 0 : return HCCL_E_NOT_SUPPORT;
2004 : }
2005 0 : u32 nslbBuffersize = hcclH2dTlv::GetInstance().GetH2dTlvBufferSize();
2006 0 : void* tlvHandle = hcclH2dTlv::GetInstance().GetH2dTlvHandle();
2007 0 : nslb_msg sendMsg;
2008 0 : nslb_msg recvMsg;
2009 0 : sendMsg.type = NSLBDP_TYPE_INIT_NETCO;
2010 0 : sendMsg.length = NSLBDP_ILLEGAL_MSGLENGTH;
2011 :
2012 : /* init/deinit 场景date信息默认填成数字0，hccp不关注此字段，但数据不能为NULL */
2013 0 : std::vector<uint8_t> tlvData;
2014 0 : tlvData.push_back(0);
2015 0 : sendMsg.data.assign(tlvData.begin(), tlvData.end());
2016 0 : s32 ret = H2DTlvRequest(
2017 0 : tlvHandle, MODULE_TYPE_NSLB, reinterpret_cast<TlvMsg*>(&sendMsg), reinterpret_cast<TlvMsg*>(&recvMsg));
2018 0 : HCCL_RUN_INFO("[NSLBDP-SENDTABLE] hccl send table NSLBDP_TYPE_INIT_NETCO(9001) to hccp. ret(%d)\n", ret);
2019 0 : if (ret != 0) {
2020 0 : return HCCL_E_NOT_SUPPORT;
2021 : }
2022 : /* 数据转存 */
2023 0 : SetH2DTlvInitInfo(nslbBuffersize, tlvHandle);
2024 0 : HCCL_DEBUG("Entry InitNetCo end");
2025 0 : return HCCL_SUCCESS;
2026 0 : }
2027 :
2028 : /* 去初始化NetCo通道 */
2029 0 : void hcclNslbDp::DeinitNetCo()
2030 : {
2031 0 : if (GetInitNetCoFlag() == false) {
2032 0 : return;
2033 : }
2034 0 : if (hcclH2dTlv::GetInstance().GetH2dTlvBufferSize() == NSLBDP_ILLEGAL_TLVBUFFERSIZE) {
2035 0 : return;
2036 : }
2037 0 : if (hcclH2dTlv::GetInstance().GetH2dTlvHandle() == nullptr) {
2038 0 : return;
2039 : }
2040 0 : nslb_msg sendMsg;
2041 0 : nslb_msg recvMsg;
2042 0 : sendMsg.type = NSLBDP_TYPE_DEINIT_NETCO;
2043 0 : sendMsg.length = NSLBDP_ILLEGAL_MSGLENGTH;
2044 : /* init/deinit 场景date信息默认填成数字0，hccp不关注此字段，但数据不能为NULL */
2045 0 : std::vector<uint8_t> tlvData;
2046 0 : tlvData.push_back(0);
2047 0 : sendMsg.data.assign(tlvData.begin(), tlvData.end());
2048 0 : s32 ret = H2DTlvRequest(
2049 0 : nslbdp_handle_, MODULE_TYPE_NSLB, reinterpret_cast<TlvMsg*>(&sendMsg), reinterpret_cast<TlvMsg*>(&recvMsg));
2050 0 : HCCL_RUN_INFO("[NSLBDP-SENDTABLE] hccl send table NSLBDP_TYPE_DEINIT_NETCO(9002) to hccp. ret(%d)\n", ret);
2051 0 : ClearInitNetCoFlag();
2052 0 : return;
2053 0 : }
2054 :
2055 : } // namespace hccl
|
