Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "diff_rank_updater.h"
12 : #include <fstream>
13 : #include <sstream>
14 : #include <vector>
15 : #include <string>
16 : #include <linux/limits.h>
17 : #include "changed_rank_info.h"
18 : #include "rank_table_info.h"
19 : #include "json_parser.h"
20 :
21 : using namespace Hccl;
22 :
23 : // 64+1替换前校验
24 1 : static HcclResult Check64Plus1Replace(const NewRankInfo &changeRank, const NewRankInfo &snapshotRank)
25 : {
26 1 : if (changeRank.replacedLocalId == snapshotRank.localId && changeRank.rankId == snapshotRank.rankId) {
27 1 : if ((changeRank.localId == BACKUP_LOCAL_ID && snapshotRank.localId != BACKUP_LOCAL_ID) ||
28 0 : (changeRank.localId != BACKUP_LOCAL_ID && snapshotRank.localId == BACKUP_LOCAL_ID)) {
29 1 : return HCCL_SUCCESS;
30 : }
31 : }
32 0 : HCCL_ERROR("[%s] 64+1 replacement check failed.", __func__);
33 0 : return HCCL_E_PARA;
34 : }
35 :
36 : // 整框替换前校验,传入全量的Rank数组changeRanks和snapshotRanks和下标数组changeRankIndex和snapshotRankIndex
37 : // 通过下标数组里的下标，可以找到对应R0Id的newRankInfo
38 1 : static HcclResult CheckPodReplace(vector<NewRankInfo> &changeRanks, const vector<u32> &changeRankIndex,
39 : vector<NewRankInfo> &snapshotRanks, const vector<u32> &snapshotRankIndex)
40 : {
41 : // 将changeRankId存入map，key为rankId，value为下标
42 1 : unordered_map<u32, u32> changeRankMap;
43 5 : for (u32 i = 0; i < changeRankIndex.size(); ++i) {
44 4 : u32 changeRankId = changeRanks[changeRankIndex[i]].rankId;
45 4 : changeRankMap[changeRankId] = changeRankIndex[i];
46 : }
47 :
48 : // 根据替换前后的rankId相同，校验rankId是否全部存在
49 : // 再将替换前后的下标对存入rankPair，校验是否满足整框替换的条件
50 1 : vector<pair<u32, u32>> rankPair;
51 5 : for (u32 i = 0; i < snapshotRankIndex.size(); ++i) {
52 4 : u32 snapshotRankId = snapshotRanks[snapshotRankIndex[i]].rankId;
53 4 : if (changeRankMap.find(snapshotRankId) == changeRankMap.end()) {
54 0 : HCCL_ERROR("[%s] full frame replacement check failed, rankId[%d] not found.", __func__, snapshotRanks[snapshotRankIndex[i]].rankId);
55 0 : return HCCL_E_PARA;
56 : } else {
57 4 : rankPair.push_back({snapshotRankIndex[i], changeRankMap[snapshotRankId]});
58 : }
59 : }
60 :
61 : // 每组rank替换，需要校验以下四种情况
62 : // 1、正常场景 整框替换为 备份场景
63 : // 2、备份场景 整框替换为 正常场景
64 : // 3、正常场景 整框替换为 正常场景
65 : // 4、备份场景 整框替换为 备份场景
66 5 : for (u32 i = 0; i < rankPair.size(); ++i) {
67 4 : NewRankInfo& snapshotRank = snapshotRanks[rankPair[i].first];
68 4 : NewRankInfo& changeRank = changeRanks[rankPair[i].second];
69 4 : bool isChangeReplace = false;
70 4 : bool isSnapshotReplace = false;
71 :
72 4 : if (snapshotRank.localId == changeRank.localId) {
73 4 : continue;
74 0 : } else if (changeRank.localId == BACKUP_LOCAL_ID && changeRank.replacedLocalId == snapshotRank.localId && !isChangeReplace) {
75 0 : isChangeReplace = true;
76 0 : continue;
77 0 : } else if (snapshotRank.localId == BACKUP_LOCAL_ID && snapshotRank.replacedLocalId == changeRank.localId && !isSnapshotReplace) {
78 0 : isSnapshotReplace = true;
79 0 : continue;
80 : } else {
81 0 : HCCL_ERROR("[%s] full frame replacement check failed, localId[%d] not match, rankId[%d].", __func__, snapshotRank.localId, snapshotRank.rankId);
82 0 : return HCCL_E_PARA;
83 : }
84 : }
85 1 : return HCCL_SUCCESS;
86 1 : }
87 :
88 4 : static HcclResult ParseChangeInfo(const char *changeInfo, ChangedRankInfo &changeTable)
89 : {
90 12 : HCCL_INFO("[%s] start.", __func__);
91 : // 获取真实路径
92 4 : char resolvedPath[PATH_MAX] = {0};
93 4 : if (realpath(changeInfo, resolvedPath) == nullptr) {
94 6 : HCCL_ERROR("RanktableRealPath: %s is not a valid real path.", changeInfo);
95 2 : return HCCL_E_PARA;
96 : }
97 :
98 : // 读取文件、校验文件是否存在
99 6 : HCCL_INFO("[%s] waiting for json file load complete", __func__);
100 2 : ifstream infoFile(resolvedPath, ifstream::in);
101 2 : if (!infoFile) {
102 0 : HCCL_ERROR("open file %s failed.", resolvedPath);
103 0 : return HCCL_E_OPEN_FILE_FAILURE;
104 : }
105 :
106 : // 获取ranktableM
107 2 : stringstream rankTableStr;
108 2 : rankTableStr << infoFile.rdbuf();
109 2 : string ranktableM = rankTableStr.str();
110 :
111 : // 解析ranktable
112 : JsonParser rankTableParser;
113 2 : rankTableParser.ParseString(ranktableM, changeTable);
114 :
115 6 : HCCL_INFO("[%s] end.", __func__);
116 2 : return HCCL_SUCCESS;
117 2 : }
118 2 : static HcclResult GetRankMapAndChangeMap(unordered_map<string, vector<u32>> &rankTableMap,
119 : unordered_map<string, vector<u32>> &changedRankMap, RankTableInfo &rankTableInfo,
120 : ChangedRankInfo &changeTable)
121 : {
122 6 : HCCL_INFO("[%s] start.", __func__);
123 : // 将changeTable和rankTable按R0 id分组
124 7 : for (u32 i = 0; i < changeTable.ranks.size(); ++i) {
125 5 : const auto &rank = changeTable.ranks[i];
126 : // 遍历rankLevelInfos中的每个RankLevelInfo
127 5 : for (const auto &levelInfo : rank.rankLevelInfos) {
128 5 : if (levelInfo.netLayer == 0) {
129 5 : changedRankMap[levelInfo.netInstId].push_back(i);
130 5 : break;
131 : }
132 : }
133 : }
134 :
135 10 : for (u32 i = 0; i < rankTableInfo.ranks.size(); ++i) {
136 8 : const auto &rank = rankTableInfo.ranks[i];
137 : // 遍历rankLevelInfos中的每个RankLevelInfo
138 8 : for (const auto &levelInfo : rank.rankLevelInfos) {
139 8 : if (levelInfo.netLayer == 0) {
140 8 : rankTableMap[levelInfo.netInstId].push_back(i);
141 8 : break;
142 : }
143 : }
144 : }
145 :
146 6 : HCCL_INFO("[%s] end.", __func__);
147 2 : return HCCL_SUCCESS;
148 : }
149 4 : HcclResult Hccl::DiffRankUpdater(const char *changeInfo, RankTableInfo &rankTableInfo)
150 : {
151 4 : CHK_PTR_NULL(changeInfo);
152 12 : HCCL_INFO("[%s] Start to update rankTableInfo by changeInfo changeInfo[%s]", __func__, changeInfo);
153 4 : rankTableInfo.Dump();
154 :
155 4 : ChangedRankInfo changeTable;
156 10 : CHK_RET(ParseChangeInfo(changeInfo, changeTable));
157 :
158 : // 将changeTable和rankTable按R0 id分组
159 2 : unordered_map<string, vector<u32>> changedRankMap;
160 2 : unordered_map<string, vector<u32>> rankTableMap;
161 2 : CHK_RET(GetRankMapAndChangeMap(rankTableMap, changedRankMap, rankTableInfo, changeTable));
162 :
163 2 : vector<pair<u32, u32>> needChangeRank;
164 4 : for (auto &rankInfo : rankTableMap) {
165 : // 获取R0 id
166 2 : std::string levelZeroId = rankInfo.first;
167 :
168 : // 获取对应的changedRankInfo的Rank数量
169 2 : u32 changeCount = changedRankMap[levelZeroId].size();
170 2 : u32 snapshotCount = rankTableMap[levelZeroId].size();
171 :
172 : // 确定替换策略
173 2 : if (changeCount == 0) {
174 0 : HCCL_INFO("[%s] Level[%s] No changes to apply.", __func__, levelZeroId.c_str());
175 0 : continue;
176 2 : } else if (changeCount == snapshotCount) {
177 : // 整框替换
178 3 : HCCL_INFO("[%s] Level[%s] Performing full frame replacement.", __func__, levelZeroId.c_str());
179 1 : CHK_RET(CheckPodReplace(changeTable.ranks, changedRankMap[levelZeroId], rankTableInfo.ranks,
180 : rankTableMap[levelZeroId]));
181 :
182 5 : for (u32 i = 0; i < snapshotCount; ++i) {
183 4 : needChangeRank.push_back({changedRankMap[levelZeroId][i], rankTableMap[levelZeroId][i]});
184 : }
185 1 : } else if (changeCount == 1) {
186 : // 64+1替换
187 3 : HCCL_INFO("[%s] Level[%s] Performing 64+1 replacement.", __func__, levelZeroId.c_str());
188 1 : auto changeRankId = changeTable.ranks[changedRankMap[levelZeroId][0]].rankId;
189 2 : for (u32 i = 0; i < snapshotCount; ++i) {
190 2 : if (changeRankId == rankTableInfo.ranks[rankTableMap[levelZeroId][i]].rankId) {
191 1 : CHK_RET(Check64Plus1Replace(changeTable.ranks[changedRankMap[levelZeroId][0]],
192 : rankTableInfo.ranks[rankTableMap[levelZeroId][i]]));
193 :
194 1 : needChangeRank.push_back({changedRankMap[levelZeroId][0], rankTableMap[levelZeroId][i]});
195 1 : break;
196 : }
197 : }
198 : } else {
199 0 : HCCL_ERROR(
200 : "[%s] Level[%s] Invalid number of changed ranks. Must be 0, 1, or equal to the snapshot count (%u).",
201 : __func__, levelZeroId.c_str(), snapshotCount);
202 0 : return HCCL_E_PARA;
203 : }
204 2 : }
205 :
206 : // 更新rankTableInfo
207 7 : for (auto &rankInfo : needChangeRank) {
208 5 : auto &rank = rankTableInfo.ranks[rankInfo.second];
209 5 : rank = changeTable.ranks[rankInfo.first];
210 : }
211 :
212 2 : rankTableInfo.Check();
213 :
214 6 : HCCL_INFO("Update rankTableInfo by changeInfo success");
215 2 : rankTableInfo.Dump();
216 2 : return HCCL_SUCCESS;
217 4 : }
|
