Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "diff_rank_updater.h"
12 : #include <fstream>
13 : #include <sstream>
14 : #include <vector>
15 : #include <string>
16 : #include <linux/limits.h>
17 : #include "changed_rank_info.h"
18 : #include "rank_table_info.h"
19 : #include "json_parser.h"
20 :
21 : using namespace Hccl;
22 :
23 : // 64+1替换前校验
24 0 : static HcclResult Check64Plus1Replace(const NewRankInfo& changeRank, const NewRankInfo& snapshotRank)
25 : {
26 0 : if (changeRank.replacedLocalId == snapshotRank.localId && changeRank.rankId == snapshotRank.rankId) {
27 0 : if ((changeRank.localId == BACKUP_LOCAL_ID && snapshotRank.localId != BACKUP_LOCAL_ID)
28 0 : || (changeRank.localId != BACKUP_LOCAL_ID && snapshotRank.localId == BACKUP_LOCAL_ID)) {
29 0 : return HCCL_SUCCESS;
30 : }
31 : }
32 0 : HCCL_ERROR("[%s] 64+1 replacement check failed.", __func__);
33 0 : return HCCL_E_PARA;
34 : }
35 :
36 : // 整框替换前校验,传入全量的Rank数组changeRanks和snapshotRanks和下标数组changeRankIndex和snapshotRankIndex
37 : // 通过下标数组里的下标，可以找到对应R0Id的newRankInfo
38 0 : static HcclResult CheckPodReplace(
39 : vector<NewRankInfo>& changeRanks, const vector<u32>& changeRankIndex, vector<NewRankInfo>& snapshotRanks,
40 : const vector<u32>& snapshotRankIndex)
41 : {
42 : // 将changeRankId存入map，key为rankId，value为下标
43 0 : unordered_map<u32, u32> changeRankMap;
44 0 : for (u32 i = 0; i < changeRankIndex.size(); ++i) {
45 0 : u32 changeRankId = changeRanks[changeRankIndex[i]].rankId;
46 0 : changeRankMap[changeRankId] = changeRankIndex[i];
47 : }
48 :
49 : // 根据替换前后的rankId相同，校验rankId是否全部存在
50 : // 再将替换前后的下标对存入rankPair，校验是否满足整框替换的条件
51 0 : vector<pair<u32, u32>> rankPair;
52 0 : for (u32 i = 0; i < snapshotRankIndex.size(); ++i) {
53 0 : u32 snapshotRankId = snapshotRanks[snapshotRankIndex[i]].rankId;
54 0 : if (changeRankMap.find(snapshotRankId) == changeRankMap.end()) {
55 0 : HCCL_ERROR(
56 : "[%s] full frame replacement check failed, rankId[%d] not found.", __func__,
57 : snapshotRanks[snapshotRankIndex[i]].rankId);
58 0 : return HCCL_E_PARA;
59 : } else {
60 0 : rankPair.push_back({snapshotRankIndex[i], changeRankMap[snapshotRankId]});
61 : }
62 : }
63 :
64 : // 每组rank替换，需要校验以下四种情况
65 : // 1、正常场景 整框替换为 备份场景
66 : // 2、备份场景 整框替换为 正常场景
67 : // 3、正常场景 整框替换为 正常场景
68 : // 4、备份场景 整框替换为 备份场景
69 0 : for (u32 i = 0; i < rankPair.size(); ++i) {
70 0 : NewRankInfo& snapshotRank = snapshotRanks[rankPair[i].first];
71 0 : NewRankInfo& changeRank = changeRanks[rankPair[i].second];
72 0 : bool isChangeReplace = false;
73 0 : bool isSnapshotReplace = false;
74 :
75 0 : if (snapshotRank.localId == changeRank.localId) {
76 0 : continue;
77 0 : } else if (
78 0 : changeRank.localId == BACKUP_LOCAL_ID && changeRank.replacedLocalId == snapshotRank.localId
79 0 : && !isChangeReplace) {
80 0 : isChangeReplace = true;
81 0 : continue;
82 0 : } else if (
83 0 : snapshotRank.localId == BACKUP_LOCAL_ID && snapshotRank.replacedLocalId == changeRank.localId
84 0 : && !isSnapshotReplace) {
85 0 : isSnapshotReplace = true;
86 0 : continue;
87 : } else {
88 0 : HCCL_ERROR(
89 : "[%s] full frame replacement check failed, localId[%d] not match, rankId[%d].", __func__,
90 : snapshotRank.localId, snapshotRank.rankId);
91 0 : return HCCL_E_PARA;
92 : }
93 : }
94 0 : return HCCL_SUCCESS;
95 0 : }
96 :
97 2 : static HcclResult ParseChangeInfo(const char* changeInfo, ChangedRankInfo& changeTable)
98 : {
99 6 : HCCL_INFO("[%s] start.", __func__);
100 : // 获取真实路径
101 2 : char resolvedPath[PATH_MAX] = {0};
102 2 : if (realpath(changeInfo, resolvedPath) == nullptr) {
103 6 : HCCL_ERROR("RanktableRealPath: %s is not a valid real path.", changeInfo);
104 2 : return HCCL_E_PARA;
105 : }
106 :
107 : // 读取文件、校验文件是否存在
108 0 : HCCL_INFO("[%s] waiting for json file load complete", __func__);
109 0 : ifstream infoFile(resolvedPath, ifstream::in);
110 0 : if (!infoFile) {
111 0 : HCCL_ERROR("open file %s failed.", resolvedPath);
112 0 : return HCCL_E_OPEN_FILE_FAILURE;
113 : }
114 :
115 : // 获取ranktableM
116 0 : stringstream rankTableStr;
117 0 : rankTableStr << infoFile.rdbuf();
118 0 : string ranktableM = rankTableStr.str();
119 :
120 : // 解析ranktable
121 : JsonParser rankTableParser;
122 0 : rankTableParser.ParseString(ranktableM, changeTable);
123 :
124 0 : HCCL_INFO("[%s] end.", __func__);
125 0 : return HCCL_SUCCESS;
126 0 : }
127 0 : static HcclResult GetRankMapAndChangeMap(
128 : unordered_map<string, vector<u32>>& rankTableMap, unordered_map<string, vector<u32>>& changedRankMap,
129 : RankTableInfo& rankTableInfo, ChangedRankInfo& changeTable)
130 : {
131 0 : HCCL_INFO("[%s] start.", __func__);
132 : // 将changeTable和rankTable按R0 id分组
133 0 : for (u32 i = 0; i < changeTable.ranks.size(); ++i) {
134 0 : const auto& rank = changeTable.ranks[i];
135 : // 遍历rankLevelInfos中的每个RankLevelInfo
136 0 : for (const auto& levelInfo : rank.rankLevelInfos) {
137 0 : if (levelInfo.netLayer == 0) {
138 0 : changedRankMap[levelInfo.netInstId].push_back(i);
139 0 : break;
140 : }
141 : }
142 : }
143 :
144 0 : for (u32 i = 0; i < rankTableInfo.ranks.size(); ++i) {
145 0 : const auto& rank = rankTableInfo.ranks[i];
146 : // 遍历rankLevelInfos中的每个RankLevelInfo
147 0 : for (const auto& levelInfo : rank.rankLevelInfos) {
148 0 : if (levelInfo.netLayer == 0) {
149 0 : rankTableMap[levelInfo.netInstId].push_back(i);
150 0 : break;
151 : }
152 : }
153 : }
154 :
155 0 : HCCL_INFO("[%s] end.", __func__);
156 0 : return HCCL_SUCCESS;
157 : }
158 2 : HcclResult Hccl::DiffRankUpdater(const char* changeInfo, RankTableInfo& rankTableInfo)
159 : {
160 2 : CHK_PTR_NULL(changeInfo);
161 6 : HCCL_INFO("[%s] Start to update rankTableInfo by changeInfo changeInfo[%s]", __func__, changeInfo);
162 2 : rankTableInfo.Dump();
163 :
164 2 : ChangedRankInfo changeTable;
165 8 : CHK_RET(ParseChangeInfo(changeInfo, changeTable));
166 :
167 : // 将changeTable和rankTable按R0 id分组
168 0 : unordered_map<string, vector<u32>> changedRankMap;
169 0 : unordered_map<string, vector<u32>> rankTableMap;
170 0 : CHK_RET(GetRankMapAndChangeMap(rankTableMap, changedRankMap, rankTableInfo, changeTable));
171 :
172 0 : vector<pair<u32, u32>> needChangeRank;
173 0 : for (auto& rankInfo : rankTableMap) {
174 : // 获取R0 id
175 0 : std::string levelZeroId = rankInfo.first;
176 :
177 : // 获取对应的changedRankInfo的Rank数量
178 0 : u32 changeCount = changedRankMap[levelZeroId].size();
179 0 : u32 snapshotCount = rankTableMap[levelZeroId].size();
180 :
181 : // 确定替换策略
182 0 : if (changeCount == 0) {
183 0 : HCCL_INFO("[%s] Level[%s] No changes to apply.", __func__, levelZeroId.c_str());
184 0 : continue;
185 0 : } else if (changeCount == snapshotCount) {
186 : // 整框替换
187 0 : HCCL_INFO("[%s] Level[%s] Performing full frame replacement.", __func__, levelZeroId.c_str());
188 0 : CHK_RET(CheckPodReplace(
189 : changeTable.ranks, changedRankMap[levelZeroId], rankTableInfo.ranks, rankTableMap[levelZeroId]));
190 :
191 0 : for (u32 i = 0; i < snapshotCount; ++i) {
192 0 : needChangeRank.push_back({changedRankMap[levelZeroId][i], rankTableMap[levelZeroId][i]});
193 : }
194 0 : } else if (changeCount == 1) {
195 : // 64+1替换
196 0 : HCCL_INFO("[%s] Level[%s] Performing 64+1 replacement.", __func__, levelZeroId.c_str());
197 0 : auto changeRankId = changeTable.ranks[changedRankMap[levelZeroId][0]].rankId;
198 0 : for (u32 i = 0; i < snapshotCount; ++i) {
199 0 : if (changeRankId == rankTableInfo.ranks[rankTableMap[levelZeroId][i]].rankId) {
200 0 : CHK_RET(Check64Plus1Replace(
201 : changeTable.ranks[changedRankMap[levelZeroId][0]],
202 : rankTableInfo.ranks[rankTableMap[levelZeroId][i]]));
203 :
204 0 : needChangeRank.push_back({changedRankMap[levelZeroId][0], rankTableMap[levelZeroId][i]});
205 0 : break;
206 : }
207 : }
208 : } else {
209 0 : HCCL_ERROR(
210 : "[%s] Level[%s] Invalid number of changed ranks. Must be 0, 1, or equal to the snapshot count (%u).",
211 : __func__, levelZeroId.c_str(), snapshotCount);
212 0 : return HCCL_E_PARA;
213 : }
214 0 : }
215 :
216 : // 更新rankTableInfo
217 0 : for (auto& rankInfo : needChangeRank) {
218 0 : auto& rank = rankTableInfo.ranks[rankInfo.second];
219 0 : rank = changeTable.ranks[rankInfo.first];
220 : }
221 :
222 0 : rankTableInfo.Check();
223 :
224 0 : HCCL_INFO("Update rankTableInfo by changeInfo success");
225 0 : rankTableInfo.Dump();
226 0 : return HCCL_SUCCESS;
227 2 : }
|
