Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "collected_manifest_repository.h"
12 :
13 : #include <algorithm>
14 : #include <cstdint>
15 : #include <cstring>
16 : #include <utility>
17 :
18 : #include <boost/filesystem.hpp>
19 : #include <boost/system/error_code.hpp>
20 :
21 : #include "ascendc_tool_log.h"
22 : #include "file_utils.h"
23 : #include "nlohmann/json.hpp"
24 : #include "resource_manifest_validator.h"
25 :
26 : using Json = nlohmann::json;
27 :
28 : namespace ascendc {
29 : namespace manifest_generator {
30 : namespace {
31 :
32 : namespace fs = boost::filesystem;
33 :
34 : constexpr uint32_t MAX_MANIFEST_BYTES = 8U * 1024U * 1024U;
35 : constexpr uint32_t MAX_RESOURCE_FILE_BYTES = 1U * 1024U * 1024U;
36 : constexpr uint32_t MAX_BUNDLE_BYTES = 256U * 1024U * 1024U;
37 : constexpr size_t MAX_MANIFEST_COUNT = 4096U;
38 : constexpr size_t MAX_FILES_PER_MANIFEST = 4096U;
39 : constexpr size_t MAX_FILES_PER_BUNDLE = 65536U;
40 : constexpr size_t MAX_DIRECTORY_DEPTH = 64U;
41 :
42 : class ResourceBudget final {
43 : public:
44 : bool AddResource(uint32_t size, const std::string& field)
45 : {
46 : if (fileCount_ >= MAX_FILES_PER_BUNDLE) {
47 : ASCENDLOGE("Bundle exceeds resource file count limit of %zu", MAX_FILES_PER_BUNDLE);
48 : return false;
49 : }
50 : if (!AddPayload(size, field)) {
51 : return false;
52 : }
53 : ++fileCount_;
54 : return true;
55 : }
56 :
57 : bool AddPayload(uint32_t size, const std::string& field)
58 : {
59 : if (size > MAX_BUNDLE_BYTES - totalBytes_) {
60 : ASCENDLOGE("Bundle exceeds raw payload limit while adding %s", field.c_str());
61 : return false;
62 : }
63 : totalBytes_ += size;
64 : return true;
65 : }
66 :
67 : private:
68 : uint32_t totalBytes_{0U};
69 : size_t fileCount_{0U};
70 : };
71 :
72 29 : bool ValidateAndExtractResourcePath(std::string& text, const std::string& manifestPath, std::string& resourcePath)
73 : {
74 29 : Json manifest = Json::parse(text, nullptr, false);
75 : if (manifest.is_discarded()) {
76 : ASCENDLOGE("Manifest is not valid JSON: %s", manifestPath.c_str());
77 : return false;
78 : }
79 27 : if (manifest.contains("schema_version")) {
80 1 : ASCENDLOGE("Manifest must not contain schema_version: %s", manifestPath.c_str());
81 1 : return false;
82 : }
83 :
84 : if (!ValidateResourceManifest(manifest, manifestPath)) {
85 : return false;
86 : }
87 :
88 : const Json::const_iterator resourcePathValue = manifest.find("resource_path");
89 : if (resourcePathValue == manifest.end() || !resourcePathValue->is_string()) {
90 : ASCENDLOGE("Manifest resource_path is unavailable: %s", manifestPath.c_str());
91 : return false;
92 : }
93 : resourcePath = resourcePathValue->get<std::string>();
94 22 : manifest["schema_version"] = "1.0";
95 22 : text = manifest.dump();
96 : return true;
97 : }
98 :
99 : bool IsManifestFile(const fs::path& path)
100 : {
101 : constexpr const char* suffix = "_manifest.json";
102 : const std::string name = path.filename().string();
103 : return name.size() > std::strlen(suffix) &&
104 : name.compare(name.size() - std::strlen(suffix), std::strlen(suffix), suffix) == 0;
105 : }
106 :
107 : bool AdvanceIterator(fs::recursive_directory_iterator& iterator, const std::string& root)
108 : {
109 : boost::system::error_code error;
110 : iterator.increment(error);
111 : if (!error) {
112 : return true;
113 : }
114 : ASCENDLOGE("Failed to enumerate %s: %s", root.c_str(), error.message().c_str());
115 : return false;
116 : }
117 :
118 : bool DiscoverManifestFiles(const std::string& manifestSearchRoot, std::vector<std::string>& manifests)
119 : {
120 : boost::system::error_code error;
121 : fs::recursive_directory_iterator iterator(fs::path(manifestSearchRoot), fs::directory_options::none, error);
122 : const fs::recursive_directory_iterator end;
123 : if (error) {
124 : ASCENDLOGE("Failed to enumerate %s: %s", manifestSearchRoot.c_str(), error.message().c_str());
125 : return false;
126 : }
127 : while (iterator != end) {
128 : const fs::path path = iterator->path();
129 : const fs::file_status status = iterator->symlink_status(error);
130 : if (error) {
131 : ASCENDLOGE("Failed to inspect collection entry %s: %s", path.c_str(), error.message().c_str());
132 : return false;
133 : }
134 : if (fs::is_symlink(status)) {
135 : iterator.disable_recursion_pending();
136 : } else if (fs::is_directory(status)) {
137 : if (static_cast<size_t>(iterator.depth()) >= MAX_DIRECTORY_DEPTH) {
138 : ASCENDLOGE("Collection exceeds directory depth limit of %zu: %s", MAX_DIRECTORY_DEPTH, path.c_str());
139 : return false;
140 : }
141 : } else if (fs::is_regular_file(status) && IsManifestFile(path)) {
142 : if (manifests.size() >= MAX_MANIFEST_COUNT) {
143 : ASCENDLOGE("Collection exceeds manifest count limit of %zu", MAX_MANIFEST_COUNT);
144 : return false;
145 : }
146 : manifests.push_back(path.string());
147 : }
148 : if (!AdvanceIterator(iterator, manifestSearchRoot)) {
149 : return false;
150 : }
151 : }
152 : std::sort(manifests.begin(), manifests.end());
153 : if (manifests.empty()) {
154 : ASCENDLOGE("Collected resources contain no manifest units");
155 : return false;
156 : }
157 : return true;
158 : }
159 :
160 : bool LoadResourceFile(
161 : const fs::path& source, const std::string& manifestRoot, const std::string& manifestPath, ResourceBudget& budget,
162 : ManifestUnit& unit)
163 : {
164 : std::vector<uint8_t> data;
165 : if (!FileUtils::ReadRegularFile(source.string(), MAX_RESOURCE_FILE_BYTES, data)) {
166 : ASCENDLOGE("Failed to read permitted manifest resource: %s", source.c_str());
167 : return false;
168 : }
169 : if (data.empty()) {
170 : ASCENDLOGD("Skipping empty manifest resource %s", source.c_str());
171 : return true;
172 : }
173 : if (unit.files.size() >= MAX_FILES_PER_MANIFEST) {
174 : ASCENDLOGE("Manifest exceeds resource file count limit: %s", manifestPath.c_str());
175 : return false;
176 : }
177 : if (!budget.AddResource(static_cast<uint32_t>(data.size()), "manifest resource " + source.string())) {
178 : return false;
179 : }
180 : boost::system::error_code error;
181 : const fs::path relative = fs::relative(source, fs::path(manifestRoot), error);
182 : if (error || relative.empty()) {
183 : ASCENDLOGE(
184 : "Failed to make manifest resource path relative: path=%s root=%s error=%s", source.c_str(),
185 : manifestRoot.c_str(), error ? error.message().c_str() : "empty relative path");
186 : return false;
187 : }
188 : ResourceFile file;
189 : file.fileName = source.filename().string();
190 : file.filePath = relative.generic_string();
191 : file.data = std::move(data);
192 : unit.files.push_back(std::move(file));
193 : return true;
194 : }
195 :
196 : bool LoadResourceFiles(
197 : const std::string& directory, const std::string& manifestRoot, const std::string& manifestPath,
198 : ResourceBudget& budget, ManifestUnit& unit)
199 : {
200 : boost::system::error_code error;
201 : fs::recursive_directory_iterator iterator(fs::path(directory), fs::directory_options::none, error);
202 : const fs::recursive_directory_iterator end;
203 : if (error) {
204 : ASCENDLOGE("Failed to enumerate manifest resource_path %s: %s", directory.c_str(), error.message().c_str());
205 : return false;
206 : }
207 : while (iterator != end) {
208 : const fs::path source = iterator->path();
209 : const fs::file_status status = iterator->symlink_status(error);
210 : if (error) {
211 : ASCENDLOGE("Failed to inspect manifest resource %s: %s", source.c_str(), error.message().c_str());
212 : return false;
213 : }
214 : if (fs::is_symlink(status)) {
215 : ASCENDLOGE("Manifest resource_path contains a symlink: %s", source.c_str());
216 : return false;
217 : }
218 : if (fs::is_directory(status)) {
219 : if (static_cast<size_t>(iterator.depth()) >= MAX_DIRECTORY_DEPTH) {
220 : ASCENDLOGE(
221 : "Manifest resources exceed directory depth limit of %zu: %s", MAX_DIRECTORY_DEPTH, source.c_str());
222 : return false;
223 : }
224 : } else if (!fs::is_regular_file(status)) {
225 : ASCENDLOGE("Manifest resource_path contains a non-regular file: %s", source.c_str());
226 : return false;
227 : } else if (!LoadResourceFile(source, manifestRoot, manifestPath, budget, unit)) {
228 : return false;
229 : }
230 : if (!AdvanceIterator(iterator, directory)) {
231 : return false;
232 : }
233 : }
234 : return true;
235 : }
236 :
237 : bool LoadUnit(const std::string& manifestPath, ResourceBudget& budget, ManifestUnit& unit)
238 : {
239 : ASCENDLOGD("Loading manifest %s", manifestPath.c_str());
240 : std::vector<uint8_t> manifestBytes;
241 : if (!FileUtils::ReadRegularFile(manifestPath, MAX_MANIFEST_BYTES, manifestBytes)) {
242 : ASCENDLOGE("Failed to read permitted manifest: %s", manifestPath.c_str());
243 : return false;
244 : }
245 : if (!budget.AddPayload(static_cast<uint32_t>(manifestBytes.size()), "manifest " + manifestPath)) {
246 : return false;
247 : }
248 : unit.json.assign(manifestBytes.begin(), manifestBytes.end());
249 : std::string resourcePath;
250 : if (!ValidateAndExtractResourcePath(unit.json, manifestPath, resourcePath)) {
251 : return false;
252 : }
253 : if (resourcePath.empty()) {
254 : ASCENDLOGD("Loaded manifest %s with no resource files", manifestPath.c_str());
255 : return true;
256 : }
257 :
258 : const std::string manifestRoot = FileUtils::ParentPath(manifestPath);
259 : const fs::path resourcePathValue(resourcePath);
260 : const std::string resourceInput =
261 : resourcePathValue.is_absolute() ? resourcePath : FileUtils::JoinPath(manifestRoot, resourcePath);
262 : std::string resourceRoot;
263 : if (!FileUtils::ResolveSubdirectory(resourceInput, manifestRoot, resourceRoot)) {
264 : ASCENDLOGE(
265 : "Manifest resource_path must be a directory below the manifest root and must not be a symlink: %s",
266 : resourceInput.c_str());
267 : return false;
268 : }
269 : if (!LoadResourceFiles(resourceRoot, manifestRoot, manifestPath, budget, unit)) {
270 : return false;
271 : }
272 : std::sort(unit.files.begin(), unit.files.end(), [](const ResourceFile& left, const ResourceFile& right) {
273 : return left.filePath < right.filePath;
274 : });
275 : ASCENDLOGD("Loaded manifest %s with %zu resource file(s)", manifestPath.c_str(), unit.files.size());
276 : return true;
277 : }
278 :
279 : } // namespace
280 :
281 : CollectedManifestRepository::CollectedManifestRepository(std::string manifestSearchRoot) noexcept
282 : : manifestSearchRoot_(std::move(manifestSearchRoot))
283 : {}
284 :
285 : bool CollectedManifestRepository::Load(std::vector<ManifestUnit>& units) const
286 : {
287 : ASCENDLOGD("Discovering manifests under %s", manifestSearchRoot_.c_str());
288 : ResourceBudget budget;
289 : std::vector<std::string> manifests;
290 : if (!DiscoverManifestFiles(manifestSearchRoot_, manifests)) {
291 : return false;
292 : }
293 : ASCENDLOGI("Discovered %zu manifest(s)", manifests.size());
294 : units.clear();
295 : units.reserve(manifests.size());
296 : for (const std::string& manifest : manifests) {
297 : ManifestUnit unit;
298 : if (!LoadUnit(manifest, budget, unit)) {
299 : return false;
300 : }
301 : units.push_back(std::move(unit));
302 : }
303 : return true;
304 : }
305 :
306 : } // namespace manifest_generator
307 : } // namespace ascendc
|
