Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "adx_dump_record.h"
12 : #include <map>
13 : #include <cinttypes>
14 : #include <functional>
15 : #include <pthread.h>
16 : #include "mmpa_api.h"
17 : #include "adx_log.h"
18 : #include "file_utils.h"
19 : #include "string_utils.h"
20 : #include "memory_utils.h"
21 : #include "common_utils.h"
22 : #include "adx_dump_process.h"
23 : #include "ide_os_type.h"
24 : namespace Adx {
25 : static const std::size_t MAX_IP_LENGTH = 16;
26 : #if !defined(ADUMP_SOC_HOST) || ADUMP_SOC_HOST == 1
27 : constexpr char STRING_BIN[] = ".bin";
28 : constexpr char STRING_CSV[] = ".csv";
29 : constexpr char CSV_HEADER[] = "Input/Output,Index,Data Size,Data Type,Format,Shape";
30 :
31 : static const std::map<uint64_t, std::string> STATS_ITEM_MAP = {
32 : {DUMP_STATS_MAX, ",Max Value"},
33 : {DUMP_STATS_MIN, ",Min Value"},
34 : {DUMP_STATS_AVG, ",Avg Value"},
35 : {DUMP_STATS_NAN, ",Nan Count"},
36 : {DUMP_STATS_NEG_INF, ",Negative Inf Count"},
37 : {DUMP_STATS_POS_INF, ",Positive Inf Count"},
38 : {DUMP_STATS_L2NORM, ",l2norm"}};
39 :
40 : static const std::map<toolkit::dump::OutputDataType, std::string> DT_STRING_MAP = {
41 : {toolkit::dump::DT_UNDEFINED, "DT_UNDEFINED"},
42 : {toolkit::dump::DT_FLOAT, "DT_FLOAT"},
43 : {toolkit::dump::DT_FLOAT16, "DT_FLOAT16"},
44 : {toolkit::dump::DT_INT8, "DT_INT8"},
45 : {toolkit::dump::DT_UINT8, "DT_UINT8"},
46 : {toolkit::dump::DT_INT16, "DT_INT16"},
47 : {toolkit::dump::DT_UINT16, "DT_UINT16"},
48 : {toolkit::dump::DT_INT32, "DT_INT32"},
49 : {toolkit::dump::DT_INT64, "DT_INT64"},
50 : {toolkit::dump::DT_UINT32, "DT_UINT32"},
51 : {toolkit::dump::DT_UINT64, "DT_UINT64"},
52 : {toolkit::dump::DT_BOOL, "DT_BOOL"},
53 : {toolkit::dump::DT_DOUBLE, "DT_DOUBLE"},
54 : {toolkit::dump::DT_STRING, "DT_STRING"},
55 : {toolkit::dump::DT_DUAL_SUB_INT8, "DT_DUAL_SUB_INT8"},
56 : {toolkit::dump::DT_DUAL_SUB_UINT8, "DT_DUAL_SUB_UINT8"},
57 : {toolkit::dump::DT_COMPLEX64, "DT_COMPLEX64"},
58 : {toolkit::dump::DT_COMPLEX128, "DT_COMPLEX128"},
59 : {toolkit::dump::DT_QINT8, "DT_QINT8"},
60 : {toolkit::dump::DT_QINT16, "DT_QINT16"},
61 : {toolkit::dump::DT_QINT32, "DT_QINT32"},
62 : {toolkit::dump::DT_QUINT8, "DT_QUINT8"},
63 : {toolkit::dump::DT_QUINT16, "DT_QUINT16"},
64 : {toolkit::dump::DT_RESOURCE, "DT_RESOURCE"},
65 : {toolkit::dump::DT_STRING_REF, "DT_STRING_REF"},
66 : {toolkit::dump::DT_DUAL, "DT_DUAL"},
67 : {toolkit::dump::DT_VARIANT, "DT_VARIANT"},
68 : {toolkit::dump::DT_BF16, "DT_BF16"},
69 : {toolkit::dump::DT_INT4, "DT_INT4"},
70 : {toolkit::dump::DT_UINT1, "DT_UINT1"},
71 : {toolkit::dump::DT_INT2, "DT_INT2"},
72 : {toolkit::dump::DT_UINT2, "DT_UINT2"},
73 : {toolkit::dump::DT_HIFLOAT8, "DT_HIFLOAT8"},
74 : {toolkit::dump::DT_FLOAT8_E5M2, "DT_FLOAT8_E5M2"},
75 : {toolkit::dump::DT_FLOAT8_E4M3FN, "DT_FLOAT8_E4M3FN"},
76 : {toolkit::dump::DT_FLOAT8_E8M0, "DT_FLOAT8_E8M0"},
77 : {toolkit::dump::DT_FLOAT6_E3M2, "DT_FLOAT6_E3M2"},
78 : {toolkit::dump::DT_FLOAT6_E2M3, "DT_FLOAT6_E2M3"},
79 : {toolkit::dump::DT_FLOAT4_E2M1, "DT_FLOAT4_E2M1"},
80 : {toolkit::dump::DT_FLOAT4_E1M2, "DT_FLOAT4_E1M2"},
81 : };
82 :
83 : static const std::map<toolkit::dump::OutputFormat, std::string> FORMAT_STRING_MAP = {
84 : {toolkit::dump::FORMAT_NCHW, "NCHW"},
85 : {toolkit::dump::FORMAT_NHWC, "NHWC"},
86 : {toolkit::dump::FORMAT_ND, "ND"},
87 : {toolkit::dump::FORMAT_NC1HWC0, "NC1HWC0"},
88 : {toolkit::dump::FORMAT_FRACTAL_Z, "FRACTAL_Z"},
89 : {toolkit::dump::FORMAT_NC1C0HWPAD, "NC1C0HWPAD"},
90 : {toolkit::dump::FORMAT_NHWC1C0, "NHWC1C0"},
91 : {toolkit::dump::FORMAT_FSR_NCHW, "FSR_NCHW"},
92 : {toolkit::dump::FORMAT_FRACTAL_DECONV, "FRACTAL_DECONV"},
93 : {toolkit::dump::FORMAT_C1HWNC0, "C1HWNC0"},
94 : {toolkit::dump::FORMAT_FRACTAL_DECONV_TRANSPOSE, "FRACTAL_DECONV_TRANSPOSE"},
95 : {toolkit::dump::FORMAT_FRACTAL_DECONV_SP_STRIDE_TRANS, "FRACTAL_DECONV_SP_STRIDE_TRANS"},
96 : {toolkit::dump::FORMAT_NC1HWC0_C04, "NC1HWC0_C04"},
97 : {toolkit::dump::FORMAT_FRACTAL_Z_C04, "FRACTAL_Z_C04"},
98 : {toolkit::dump::FORMAT_CHWN, "CHWN"},
99 : {toolkit::dump::FORMAT_FRACTAL_DECONV_SP_STRIDE8_TRANS, "FRACTAL_DECONV_SP_STRIDE8_TRANS"},
100 : {toolkit::dump::FORMAT_HWCN, "HWCN"},
101 : {toolkit::dump::FORMAT_NC1KHKWHWC0, "NC1KHKWHWC0"},
102 : {toolkit::dump::FORMAT_BN_WEIGHT, "BN_WEIGHT"},
103 : {toolkit::dump::FORMAT_FILTER_HWCK, "FILTER_HWCK"},
104 : {toolkit::dump::FORMAT_HASHTABLE_LOOKUP_LOOKUPS, "HASHTABLE_LOOKUP_LOOKUPS"},
105 : {toolkit::dump::FORMAT_HASHTABLE_LOOKUP_KEYS, "HASHTABLE_LOOKUP_KEYS"},
106 : {toolkit::dump::FORMAT_HASHTABLE_LOOKUP_VALUE, "HASHTABLE_LOOKUP_VALUE"},
107 : {toolkit::dump::FORMAT_HASHTABLE_LOOKUP_OUTPUT, "HASHTABLE_LOOKUP_OUTPUT"},
108 : {toolkit::dump::FORMAT_HASHTABLE_LOOKUP_HITS, "HASHTABLE_LOOKUP_HITS"},
109 : {toolkit::dump::FORMAT_C1HWNCoC0, "C1HWNCoC0"},
110 : {toolkit::dump::FORMAT_MD, "MD"},
111 : {toolkit::dump::FORMAT_NDHWC, "NDHWC"},
112 : {toolkit::dump::FORMAT_FRACTAL_ZZ, "FRACTAL_ZZ"},
113 : {toolkit::dump::FORMAT_FRACTAL_NZ, "FRACTAL_NZ"},
114 : {toolkit::dump::FORMAT_NCDHW, "NCDHW"},
115 : {toolkit::dump::FORMAT_DHWCH, "DHWCH"},
116 : {toolkit::dump::FORMAT_NDC1HWC0, "NDC1HWC0"},
117 : {toolkit::dump::FORMAT_FRACTAL_Z_3D, "FRACTAL_Z_3D"},
118 : {toolkit::dump::FORMAT_CN, "CN"},
119 : {toolkit::dump::FORMAT_NC, "NC"},
120 : {toolkit::dump::FORMAT_DHWNC, "DHWNC"},
121 : {toolkit::dump::FORMAT_FRACTAL_Z_3D_TRANSPOSE, "FRACTAL_Z_3D_TRANSPOSE"},
122 : {toolkit::dump::FORMAT_FRACTAL_ZN_LSTM, "FRACTAL_ZN_LSTM"},
123 : {toolkit::dump::FORMAT_FRACTAL_Z_G, "FRACTAL_Z_G"},
124 : {toolkit::dump::FORMAT_RESERVED, "RESERVED"},
125 : {toolkit::dump::FORMAT_ALL, "ALL"},
126 : {toolkit::dump::FORMAT_NULL, "NULL"},
127 : {toolkit::dump::FORMAT_ND_RNN_BIAS, "ND_RNN_BIAS"},
128 : {toolkit::dump::FORMAT_FRACTAL_ZN_RNN, "FRACTAL_ZN_RNN"},
129 : {toolkit::dump::FORMAT_NYUV, "NYUV"},
130 : {toolkit::dump::FORMAT_NYUV_A, "NYUV_A"},
131 : {toolkit::dump::FORMAT_NCL, "NCL"},
132 : {toolkit::dump::FORMAT_FRACTAL_Z_WINO, "FRACTAL_Z_WINO"},
133 : {toolkit::dump::FORMAT_C1HWC0, "C1HWC0"}};
134 : #endif
135 :
136 5 : AdxDumpRecord::AdxDumpRecord() : dumpRecordFlag_(true), dumpInitNum_(0)
137 : {
138 : int32_t ret =
139 5 : pthread_atfork(AdxDumpRecord::PrepareFork, AdxDumpRecord::PostForkParent, AdxDumpRecord::PostForkChild);
140 5 : if (ret != 0) {
141 0 : IDE_LOGW("call pthread_atfork failed, ret: %d", ret);
142 : }
143 : #if !defined(ADUMP_SOC_HOST) || ADUMP_SOC_HOST == 1
144 5 : fileNameStatus_.reserve(FILENAME_CHECK_SIZE_MAX);
145 55 : for (size_t idx = 0; idx < FILENAME_CHECK_SIZE_MAX; ++idx) {
146 150 : fileNameStatus_.push_back("");
147 : }
148 : funcMap_ = {
149 0 : {0, std::bind(&AdxDumpRecord::TypeDataHandle, this, std::placeholders::_1, std::placeholders::_2)}, // max
150 0 : {1, std::bind(&AdxDumpRecord::TypeDataHandle, this, std::placeholders::_1, std::placeholders::_2)}, // min
151 0 : {2, std::bind(&AdxDumpRecord::FloatDataHandle, this, std::placeholders::_1)}, // avg
152 0 : {3, std::bind(&AdxDumpRecord::Int32DataHandle, this, std::placeholders::_1)}, // nan
153 0 : {4, std::bind(&AdxDumpRecord::Int32DataHandle, this, std::placeholders::_1)}, // neg inf
154 0 : {5, std::bind(&AdxDumpRecord::Int32DataHandle, this, std::placeholders::_1)}, // pos inf
155 0 : {6, std::bind(&AdxDumpRecord::FloatDataHandle, this, std::placeholders::_1)}, // l2norm
156 40 : };
157 : #endif
158 10 : }
159 :
160 5 : AdxDumpRecord::~AdxDumpRecord() { UnInit(); }
161 :
162 391 : int32_t AdxDumpRecord::GetDumpInitNum() const { return dumpInitNum_; }
163 :
164 112 : void AdxDumpRecord::UpdateDumpInitNum(bool isPlus)
165 : {
166 112 : if (isPlus) {
167 52 : dumpInitNum_++;
168 60 : } else if (dumpInitNum_ > 0) {
169 52 : dumpInitNum_--;
170 : }
171 112 : IDE_LOGI("dump init number: %d", dumpInitNum_);
172 112 : }
173 :
174 55 : bool AdxDumpRecord::HasStartedServer() const { return dumpInitNum_ > 0; }
175 :
176 62 : bool AdxDumpRecord::CanShutdownServer() const { return dumpInitNum_ <= 1; }
177 :
178 65 : int32_t AdxDumpRecord::Init(const std::string& hostPid)
179 : {
180 : // non-soc case
181 65 : if (hostPid.empty()) {
182 61 : char dumpPath[MAX_FILE_PATH_LENGTH] = {0};
183 61 : if (mmGetCwd(dumpPath, sizeof(dumpPath)) != EN_OK) {
184 1 : IDE_LOGE("get current dir failed ");
185 1 : return IDE_DAEMON_ERROR;
186 : }
187 60 : dumpPath_ = dumpPath;
188 : } else {
189 : #if (OS_TYPE == LINUX)
190 : // soc case
191 4 : std::string appBin = "/proc/" + hostPid + "/exe";
192 4 : errno = 0;
193 4 : if (!FileUtils::IsFileExist(appBin) && errno != EACCES) {
194 0 : appBin = "/local/proc/" + hostPid + "/exe"; // aoscore
195 : }
196 4 : uint32_t pathSize = MMPA_MAX_PATH + 1;
197 4 : IdeStringBuffer curPath = reinterpret_cast<IdeStringBuffer>(IdeXmalloc(pathSize));
198 4 : IDE_CTRL_VALUE_FAILED(curPath != nullptr, return IDE_DAEMON_ERROR, "malloc failed");
199 4 : errno = 0;
200 4 : int32_t len = readlink(appBin.c_str(), curPath, MMPA_MAX_PATH); // read self path of store
201 4 : if (len < 0 || len > MMPA_MAX_PATH) {
202 2 : IDE_LOGE(
203 : "Can't get app bin directory, strerr: %s, length: %d bytes, path: %s", strerror(errno), len,
204 : appBin.c_str());
205 2 : IDE_XFREE_AND_SET_NULL(curPath);
206 2 : return IDE_DAEMON_ERROR;
207 : }
208 2 : IDE_LOGI("get app bin path: %s", curPath);
209 2 : curPath[len] = '\0'; // add string end char
210 2 : dumpPath_ = curPath;
211 2 : std::string::size_type idx = dumpPath_.find_last_of(OS_SPLIT_STR);
212 2 : dumpPath_ = dumpPath_.substr(0, idx);
213 2 : IDE_XFREE_AND_SET_NULL(curPath);
214 : #endif
215 4 : }
216 62 : IDE_LOGI("dumpPath prefix is %s", dumpPath_.c_str());
217 62 : if (!dumpPath_.empty() && dumpPath_.back() != OS_SPLIT_CHAR) {
218 39 : dumpPath_ += OS_SPLIT_CHAR;
219 : }
220 :
221 62 : std::lock_guard<std::mutex> lock(recordMutex_);
222 62 : if (hostDumpDataInfoQueue_ == nullptr) {
223 6 : hostDumpDataInfoQueue_.reset(new (std::nothrow) BoundQueueMemory<HostDumpDataInfo>());
224 6 : IDE_CTRL_VALUE_FAILED(
225 : hostDumpDataInfoQueue_ != nullptr, return IDE_DAEMON_ERROR, "Failed to new hostDumpDataInfoQueue");
226 : }
227 62 : hostDumpDataInfoQueue_->Init();
228 62 : IDE_LOGI("record remote dump temp path: %s", dumpPath_.c_str());
229 62 : hostDumpDataInfoQueue_->SetPath(dumpPath_);
230 62 : dumpRecordFlag_ = true;
231 62 : return IDE_DAEMON_OK;
232 62 : }
233 :
234 57 : int32_t AdxDumpRecord::StartRecord()
235 : {
236 57 : std::lock_guard<std::mutex> lock(recordMutex_);
237 57 : if (recordThread_.joinable()) {
238 4 : IDE_LOGI("dump record thread has been started, no need to start again");
239 4 : return IDE_DAEMON_OK;
240 : }
241 53 : dumpRecordFlag_ = true;
242 : try {
243 53 : recordThread_ = std::thread(&AdxDumpRecord::RecordDumpInfo, this);
244 0 : } catch (const std::exception& ex) {
245 0 : dumpRecordFlag_ = false;
246 0 : IDE_LOGE("Create the dump record thread failed, message: %s", ex.what());
247 0 : return IDE_DAEMON_ERROR;
248 0 : }
249 53 : return IDE_DAEMON_OK;
250 57 : }
251 :
252 : /**
253 : * @brief initialize record file
254 : * @param [in] recordPath : record file Path
255 : * @return
256 : * IDE_DAEMON_ERROR : falied
257 : * IDE_DAEMON_OK : success
258 : */
259 67 : int32_t AdxDumpRecord::UnInit()
260 : {
261 67 : std::lock_guard<std::mutex> lock(recordMutex_);
262 67 : IDE_LOGI("start to dump uninit");
263 67 : dumpRecordFlag_ = false;
264 67 : if (hostDumpDataInfoQueue_ != nullptr) {
265 64 : hostDumpDataInfoQueue_->Quit();
266 : }
267 67 : if (recordThread_.joinable()) {
268 52 : recordThread_.join();
269 : }
270 67 : IDE_LOGI("dump uninit success");
271 67 : return IDE_DAEMON_OK;
272 67 : }
273 :
274 8 : void AdxDumpRecord::PrepareFork() { Instance().recordMutex_.lock(); }
275 :
276 5 : void AdxDumpRecord::PostForkParent() { Instance().recordMutex_.unlock(); }
277 :
278 3 : void AdxDumpRecord::PostForkChild()
279 : {
280 3 : auto& instance = Instance();
281 3 : instance.dumpRecordFlag_ = false;
282 3 : if (instance.recordThread_.joinable()) {
283 1 : instance.recordThread_.detach();
284 : }
285 :
286 3 : (void)instance.hostDumpDataInfoQueue_.release();
287 3 : instance.recordMutex_.unlock();
288 3 : }
289 :
290 : /**
291 : * @brief record dump data to disk
292 : * @param [in] dumpChunk : dump chunk
293 : * @return
294 : * true : record dump data to disk success
295 : * false : record dump data to disk failed
296 : */
297 8 : bool AdxDumpRecord::RecordDumpDataToDisk(const DumpChunk& dumpChunk) const
298 : {
299 : // dump file aging
300 8 : std::string filePath = dumpChunk.fileName;
301 8 : if (filePath.empty()) {
302 2 : IDE_LOGE("filepath of received dump chunk is empty");
303 2 : return false;
304 : }
305 6 : if (JudgeRemoteFalg(filePath)) {
306 0 : auto pos = filePath.find_first_of(":");
307 0 : filePath = filePath.substr(pos + 1);
308 0 : filePath = dumpPath_ + filePath;
309 : } else {
310 6 : if (!FileUtils::IsAbsolutePath(filePath)) {
311 6 : filePath = dumpPath_ + filePath;
312 : }
313 : }
314 :
315 : #if (OS_TYPE != LINUX)
316 : filePath = FileUtils::ReplaceAll(filePath, "/", "\\");
317 : #endif
318 :
319 6 : IDE_LOGI("start to record dump data to disk path: %s", filePath.c_str());
320 :
321 6 : std::string saveDirName = FileUtils::GetFileDir(filePath);
322 6 : if (!FileUtils::IsFileExist(saveDirName)) {
323 1 : if (FileUtils::CreateDir(saveDirName) != IDE_DAEMON_NONE_ERROR) {
324 1 : IDE_LOGE("create dir failed path: %s", filePath.c_str());
325 1 : return false;
326 : }
327 : }
328 :
329 5 : while (FileUtils::IsDiskFull(saveDirName, dumpChunk.bufLen)) {
330 1 : IDE_LOGE("don't have enough free disk %u bytes", dumpChunk.bufLen);
331 1 : return false;
332 : }
333 :
334 4 : std::string realPath;
335 4 : if (FileUtils::FileNameIsReal(filePath, realPath) != IDE_DAEMON_OK) {
336 1 : IDE_LOGE("real path: %s", filePath.c_str());
337 1 : return false;
338 : }
339 :
340 6 : IdeErrorT err = FileUtils::WriteFile(realPath.c_str(), dumpChunk.dataBuf, dumpChunk.bufLen, dumpChunk.offset);
341 3 : if (err != IDE_DAEMON_NONE_ERROR) {
342 1 : (void)remove(realPath.c_str());
343 1 : IDE_LOGE("WriteFile failed, fileName: %s, err: %d", realPath.c_str(), err);
344 1 : return false;
345 : }
346 2 : IDE_LOGI("record dump data success: %s", realPath.c_str());
347 2 : return true;
348 8 : }
349 :
350 : /**
351 : * @brief judge if is remote case based on flag
352 : * @param [in] msg : flag msg
353 : * @return
354 : * true : is remote case
355 : * false : not remote case
356 : */
357 6 : bool AdxDumpRecord::JudgeRemoteFalg(const std::string& msg) const
358 : {
359 6 : std::size_t len = msg.find_first_of(":");
360 6 : if (len != std::string::npos && len < MAX_IP_LENGTH) {
361 0 : std::string ipStr = msg.substr(0, len);
362 0 : if (StringUtils::IpValid(ipStr)) {
363 0 : IDE_LOGD("remote ip info check pass: %s", ipStr.c_str());
364 0 : return true;
365 : }
366 0 : }
367 6 : IDE_LOGD("non remote ip case checked.");
368 :
369 6 : return false;
370 : }
371 :
372 0 : void AdxDumpRecord::SetWorkPath(const std::string& path) { workPath_ = path; }
373 :
374 : #if !defined(ADUMP_SOC_HOST) || ADUMP_SOC_HOST == 1
375 13 : void AdxDumpRecord::SetOptimizationMode(uint64_t statsItem)
376 : {
377 13 : if (dumpStatsItem_ == statsItem) {
378 10 : return;
379 : }
380 3 : dumpStatsItem_ = statsItem;
381 3 : IDE_LOGI("SetOptimizationMode success with stats items:[0x%llx]", dumpStatsItem_);
382 3 : uint16_t statsNum = 0;
383 3 : statsHeader_ = CSV_HEADER;
384 3 : statsList_.clear();
385 :
386 3 : if (dumpStatsItem_ ==
387 : (DUMP_STATS_MAX | DUMP_STATS_MIN | DUMP_STATS_AVG | DUMP_STATS_NAN | DUMP_STATS_NEG_INF | DUMP_STATS_POS_INF)) {
388 1 : compatible_ = true;
389 : } else {
390 2 : statsHeader_ += ",Count";
391 : }
392 24 : for (auto it = STATS_ITEM_MAP.begin(); it != STATS_ITEM_MAP.cend(); ++it) {
393 21 : if ((dumpStatsItem_ & it->first) != 0) {
394 9 : if (compatible_ && statsNum == COUNT_HEADER_COMPATIBLE) {
395 1 : statsHeader_ += ",Count";
396 : }
397 9 : statsHeader_ += it->second;
398 9 : statsList_.push_back(statsNum);
399 : }
400 21 : ++statsNum;
401 : }
402 3 : IDE_LOGI("SetOptimizationMode generate table header [%s].", statsHeader_.c_str());
403 : }
404 :
405 10 : std::string AdxDumpRecord::GetShapeString(const uint32_t shape[], int32_t shapeSize, int64_t& count) const
406 : {
407 10 : IDE_CTRL_VALUE_FAILED(
408 : shapeSize <= MAX_SHAPE_SIZE, return "NA", "Shape size %d is out of range %d", shapeSize, MAX_SHAPE_SIZE);
409 10 : if (shapeSize == 0) { // 没有shape信息时，保持原有方式用-占位
410 0 : return "-";
411 : }
412 10 : std::ostringstream oss;
413 40 : for (int32_t idx = 0; idx < shapeSize; ++idx) {
414 30 : count *= shape[idx];
415 30 : oss << shape[idx];
416 30 : if (idx != shapeSize - 1) {
417 20 : oss << "x";
418 : }
419 : }
420 10 : return oss.str();
421 10 : }
422 :
423 : template <typename T>
424 20 : std::string AdxDumpRecord::GetStringName(T key, const std::map<T, std::string>& stringMap) const
425 : {
426 20 : auto it = stringMap.find(key);
427 20 : if (it != stringMap.cend()) {
428 19 : return it->second;
429 : }
430 1 : IDE_LOGW("Cannot find [%d] type name in string map.", key);
431 2 : return "UNKNOW";
432 : }
433 :
434 28 : std::string AdxDumpRecord::Int32DataHandle(const int64_t& data) const
435 : {
436 : DataTypeUnion dtUnion;
437 28 : dtUnion.longIntValue = data;
438 56 : return std::to_string(dtUnion.intValue[0]);
439 : }
440 :
441 19 : std::string AdxDumpRecord::FloatDataHandle(const int64_t& data) const
442 : {
443 : DataTypeUnion dtUnion;
444 19 : dtUnion.longIntValue = data;
445 19 : std::ostringstream oss;
446 19 : oss << dtUnion.floatValue;
447 38 : return oss.str();
448 19 : }
449 :
450 18 : std::string AdxDumpRecord::TypeDataHandle(const int64_t& data, toolkit::dump::OutputDataType dType) const
451 : {
452 : // 如果dt是int64数据读取为int64；dt是非int64数据读取为int32，dt是float数据读取为float32
453 18 : if (dType == toolkit::dump::DT_FLOAT || dType == toolkit::dump::DT_FLOAT16 || dType == toolkit::dump::DT_BF16 ||
454 12 : dType == toolkit::dump::DT_HIFLOAT8 || dType == toolkit::dump::DT_FLOAT8_E5M2 ||
455 : dType == toolkit::dump::DT_FLOAT8_E4M3FN) {
456 10 : return FloatDataHandle(data);
457 8 : } else if (dType == toolkit::dump::DT_INT64) {
458 4 : return std::to_string(data);
459 : } else {
460 4 : return Int32DataHandle(data);
461 : }
462 : }
463 :
464 52 : std::string AdxDumpRecord::GetStatsString(toolkit::dump::OutputDataType dType, uint16_t pos, const int64_t& data)
465 : {
466 52 : auto it = funcMap_.find(pos);
467 52 : if (it != funcMap_.cend()) {
468 51 : std::string result = it->second(data, dType);
469 51 : return result;
470 0 : } else {
471 1 : IDE_LOGW("Dump stats [%hu] is not supported.", pos);
472 2 : return "NA";
473 : }
474 : }
475 :
476 3 : bool AdxDumpRecord::CheckFileNameExist(const std::string& filename)
477 : {
478 3 : auto it = std::find(fileNameStatus_.begin(), fileNameStatus_.end(), filename);
479 6 : if (it != fileNameStatus_.end()) {
480 2 : return true;
481 : }
482 1 : return false;
483 : }
484 :
485 14 : void AdxDumpRecord::AppendFileName(const std::string& filename)
486 : {
487 : // Stores the names of processed files.
488 14 : fileNameStatus_[filenameIndex_] = filename;
489 14 : filenameIndex_ = (filenameIndex_ + 1) % fileNameStatus_.size();
490 14 : }
491 :
492 10 : void AdxDumpRecord::StatisticsData(
493 : std::stringstream& strStream, std::shared_ptr<OpStatsResult> statsResult, const int64_t& count, const int32_t& idx)
494 : {
495 10 : if (statsResult->stat[idx].result != 0) {
496 1 : IDE_LOGW(
497 : "Stats data is unsupported, index is %d, size is %" PRId64 ", result is %u.", statsResult->stat[idx].index,
498 : statsResult->stat[idx].size, statsResult->stat[idx].result);
499 7 : for (size_t i = 0; i < statsList_.size(); ++i) {
500 6 : if (compatible_ && i == COUNT_HEADER_COMPATIBLE) {
501 1 : strStream << "," << count;
502 : }
503 : strStream << ","
504 6 : << "NA";
505 : }
506 : } else {
507 9 : size_t statsSize = sizeof(statsResult->stat[idx].stats) / sizeof(statsResult->stat[idx].stats[0]);
508 9 : if (!compatible_) {
509 0 : strStream << "," << count;
510 : }
511 69 : for (const uint16_t& pos : statsList_) {
512 51 : IDE_CTRL_VALUE_FAILED_NODO(
513 : pos < statsSize, continue, "Index %hu is out of range %" PRIu64 ".", pos, statsSize);
514 51 : if (compatible_ && pos == COUNT_HEADER_COMPATIBLE) {
515 8 : strStream << "," << count;
516 : }
517 : strStream << ","
518 51 : << GetStatsString(
519 51 : static_cast<toolkit::dump::OutputDataType>(statsResult->stat[idx].dType), pos,
520 102 : statsResult->stat[idx].stats[pos]);
521 : }
522 : }
523 10 : }
524 :
525 10 : bool AdxDumpRecord::GenerateFileData(
526 : std::stringstream& strStream, const std::string& filename, std::shared_ptr<OpStatsResult> statsResult)
527 : {
528 10 : IDE_CTRL_VALUE_FAILED(statsResult->statItem != 0, return false, "Dump stats is empty, nothing need to do.");
529 10 : IDE_CTRL_VALUE_FAILED(
530 : statsResult->tensorNum <= MAX_STATS_RESULT_NUM, return false, "Dump stats number %d is bigger than %d.",
531 : statsResult->tensorNum, MAX_STATS_RESULT_NUM);
532 10 : SetOptimizationMode(statsResult->statItem);
533 : // If the file name is processed for the first time, add the table header.
534 10 : if (!CheckFileNameExist(filename)) {
535 9 : strStream << statsHeader_;
536 9 : strStream << "\n";
537 : }
538 :
539 20 : for (int32_t idx = 0; idx < statsResult->tensorNum; ++idx) {
540 10 : IDE_LOGI("Process the data in file %s for the %d times.", filename.c_str(), idx);
541 10 : int64_t count = 1; // 默认count值为1
542 10 : std::string ioString = statsResult->stat[idx].io == 0 ? "Input" : "Output";
543 10 : strStream << ioString;
544 10 : strStream << "," << statsResult->stat[idx].index;
545 10 : strStream << "," << statsResult->stat[idx].size;
546 : strStream << ","
547 10 : << GetStringName(
548 10 : static_cast<toolkit::dump::OutputDataType>(statsResult->stat[idx].dType), DT_STRING_MAP);
549 : strStream << ","
550 10 : << GetStringName(
551 10 : static_cast<toolkit::dump::OutputFormat>(statsResult->stat[idx].format), FORMAT_STRING_MAP);
552 10 : strStream << "," << GetShapeString(statsResult->stat[idx].shape, statsResult->stat[idx].shapeSize, count);
553 10 : IDE_LOGD(
554 : "Process the data with index[%d], io[%d], statsLen[%u], result[%u], shapeSize[%d], size[%" PRId64 "].",
555 : statsResult->stat[idx].index, statsResult->stat[idx].io, statsResult->stat[idx].statsLen,
556 : statsResult->stat[idx].result, statsResult->stat[idx].shapeSize, statsResult->stat[idx].size);
557 :
558 10 : StatisticsData(strStream, statsResult, count, idx);
559 10 : strStream << "\n";
560 10 : }
561 10 : return true;
562 : }
563 :
564 1 : bool AdxDumpRecord::DumpDataToCallback(
565 : const std::string& filename, const std::string& dumpData, int64_t offSet, int32_t flag)
566 : {
567 1 : IDE_LOGD("Ready to send file %s to mindspore session!", filename.c_str());
568 1 : IDE_CTRL_VALUE_FAILED(!dumpData.empty(), return false, "Dump data in DumpDataToCallback is empty.");
569 :
570 : std::function<int32_t(const struct DumpChunk*, int32_t)> messageCallback =
571 1 : AdxDumpProcess::Instance().GetCallbackFun();
572 1 : IDE_CTRL_VALUE_FAILED(
573 : messageCallback, return false, "Registered messageCallback function is not callable,\
574 : drop this data packet, filename:%s",
575 : filename.c_str());
576 :
577 1 : auto dumpChunkLen = sizeof(DumpChunk) + dumpData.size() + 1;
578 : std::unique_ptr<DumpChunk, void (*)(DumpChunk*)> dumpChunk(
579 2 : static_cast<DumpChunk*>(IdeXmalloc(dumpChunkLen)), [](DumpChunk* p) { IdeXfree(p); });
580 1 : IDE_CTRL_VALUE_FAILED(dumpChunk != nullptr, return IDE_DAEMON_ERROR, "Failed to malloc for dump chunk.");
581 :
582 1 : errno_t err = strncpy_s(dumpChunk->fileName, MAX_FILE_PATH_LENGTH, filename.c_str(), filename.size());
583 1 : IDE_CTRL_VALUE_FAILED(err == EOK, return false, "Filename string copy failed, err: %d", err);
584 1 : dumpChunk->bufLen = dumpData.size() + 1;
585 1 : dumpChunk->offset = offSet;
586 1 : dumpChunk->flag = flag;
587 1 : dumpChunk->isLastChunk = 1;
588 :
589 1 : err = strncpy_s(
590 1 : reinterpret_cast<AdxStringBuffer>(dumpChunk->dataBuf), dumpChunk->bufLen, dumpData.c_str(), dumpData.size());
591 1 : IDE_CTRL_VALUE_FAILED(err == EOK, return false, "DataBuf string copy failed, err: %d", err);
592 :
593 1 : int32_t ret = messageCallback(dumpChunk.get(), dumpChunkLen);
594 1 : IDE_CTRL_VALUE_FAILED(
595 : ret == IDE_DAEMON_NONE_ERROR, return false, "Failed to transmission dump data to mindspore. err = %d", ret);
596 1 : IDE_LOGI("Send dump data to mindspore success: %s", filename.c_str());
597 1 : AppendFileName(filename);
598 1 : return true;
599 1 : }
600 :
601 6 : bool AdxDumpRecord::FileNameCheck(const DumpChunk& dumpChunk) const
602 : {
603 6 : size_t binSize = strlen(STRING_BIN);
604 6 : std::string filename = dumpChunk.fileName;
605 6 : size_t nameSize = filename.size();
606 9 : if (nameSize > binSize && filename.compare(nameSize - binSize, binSize, STRING_BIN) == 0 &&
607 3 : dumpChunk.bufLen == sizeof(OpStatsResult)) {
608 1 : IDE_LOGI("Received dump buffer length %u bytes, file name [%s].", dumpChunk.bufLen, dumpChunk.fileName);
609 1 : return true;
610 : }
611 5 : IDE_LOGD(
612 : "File name [%s] of dump chunk is not end with .bin, or data size %u bytes is different from %" PRIu64 ".",
613 : dumpChunk.fileName, dumpChunk.bufLen, sizeof(OpStatsResult));
614 5 : return false;
615 6 : }
616 :
617 7 : bool AdxDumpRecord::StatsDataParsing(const DumpChunk& dumpChunk)
618 : {
619 7 : std::string filename = dumpChunk.fileName;
620 7 : filename.replace(filename.size() - strlen(STRING_BIN), strlen(STRING_BIN), STRING_CSV);
621 :
622 7 : std::shared_ptr<OpStatsResult> statsResult;
623 : try {
624 7 : statsResult = std::make_shared<OpStatsResult>();
625 0 : } catch (std::exception& ex) {
626 0 : IDE_LOGE("Make shared failed, message: %s", ex.what());
627 0 : return false;
628 0 : }
629 7 : auto err = memcpy_s(statsResult.get(), dumpChunk.bufLen, dumpChunk.dataBuf, dumpChunk.bufLen);
630 7 : IDE_CTRL_VALUE_FAILED(err == EOK, return false, "Filename string copy failed, err: %d", err);
631 :
632 7 : std::stringstream strStream;
633 7 : IDE_CTRL_VALUE_FAILED(GenerateFileData(strStream, filename, statsResult), return false, "Failed to export data.");
634 :
635 7 : if (AdxDumpProcess::Instance().IsRegistered()) { // dump data to mindspore and return
636 1 : return DumpDataToCallback(filename, strStream.str(), dumpChunk.offset, dumpChunk.flag);
637 : }
638 :
639 6 : if (JudgeRemoteFalg(filename)) {
640 1 : auto pos = filename.find_first_of(":");
641 1 : filename = filename.substr(pos + 1);
642 1 : filename = dumpPath_ + filename;
643 : } else {
644 5 : if (!FileUtils::IsAbsolutePath(filename)) {
645 5 : filename = dumpPath_ + filename;
646 : }
647 : }
648 :
649 : #if (OS_TYPE != LINUX)
650 : filename = FileUtils::ReplaceAll(filename, "/", "\\");
651 : #endif
652 :
653 6 : std::string dirName = FileUtils::GetFileDir(filename);
654 6 : if (!FileUtils::IsFileExist(dirName)) {
655 1 : IDE_CTRL_VALUE_FAILED(
656 : FileUtils::CreateDir(dirName) == IDE_DAEMON_NONE_ERROR, return false, "Create dir failed path: %s",
657 : dirName.c_str());
658 : }
659 5 : while (FileUtils::IsDiskFull(dirName, dumpChunk.bufLen)) {
660 1 : IDE_LOGE("Don't have enough free disk %u bytes", dumpChunk.bufLen);
661 1 : return false;
662 : }
663 :
664 4 : std::string realPath;
665 4 : IDE_CTRL_VALUE_FAILED(
666 : FileUtils::FileNameIsReal(filename, realPath) == IDE_DAEMON_OK, return false, "File name is not real: %s",
667 : filename.c_str());
668 :
669 3 : IDE_CTRL_VALUE_FAILED(
670 : FileUtils::WriteFile(filename, strStream.str().c_str(), strStream.str().size(), dumpChunk.offset) ==
671 : IDE_DAEMON_NONE_ERROR,
672 : return false, "Failed to dump file %s to path %s", filename.c_str(), dirName.c_str());
673 2 : IDE_LOGI("Record dump stats data success: %s", filename.c_str());
674 2 : AppendFileName(filename);
675 2 : return true;
676 7 : }
677 : #endif
678 :
679 : /**
680 : * @brief record dump info to the file
681 : * @param [in] data : record data
682 : * @return
683 : * IDE_DAEMON_ERROR : falied
684 : * IDE_DAEMON_OK : success
685 : */
686 58 : void AdxDumpRecord::RecordDumpInfo()
687 : {
688 58 : IDE_RUN_LOGI("start dump thread, remote dump record temp path : %s.", dumpPath_.c_str());
689 58 : uint32_t chunkHeaderLen = static_cast<uint32_t>(sizeof(DumpChunk));
690 111 : while (hostDumpDataInfoQueue_ != nullptr && (dumpRecordFlag_ || !DumpDataQueueIsEmpty())) {
691 53 : HostDumpDataInfo data = {nullptr, 0};
692 53 : if (!hostDumpDataInfoQueue_->Pop(data)) {
693 47 : continue;
694 : }
695 :
696 6 : if (data.msg == nullptr) {
697 0 : continue;
698 : }
699 :
700 6 : SharedPtr<MsgProto> msgPtr = data.msg;
701 6 : IDE_CTRL_VALUE_FAILED_NODO(
702 : data.recvLen >= chunkHeaderLen, continue, "recvLen(%u) too small for DumpChunk header(%zu bytes)",
703 : data.recvLen, chunkHeaderLen);
704 :
705 6 : DumpChunk* dumpChunk = reinterpret_cast<DumpChunk*>(msgPtr->data);
706 :
707 6 : IDE_CTRL_VALUE_FAILED_NODO(
708 : dumpChunk->bufLen <= data.recvLen - chunkHeaderLen, continue,
709 : "bufLen(%u) exceeds actual data buffer size(%u bytes), fileName: %s", dumpChunk->bufLen,
710 : data.recvLen - chunkHeaderLen, dumpChunk->fileName);
711 :
712 6 : IDE_LOGI(
713 : "Queue pop data success! filename: %s, offset: %" PRId64 ", bufLen: %u bytes, "
714 : "isLast: %u, flag: %d, remaining queue size: %u.",
715 : dumpChunk->fileName, dumpChunk->offset, dumpChunk->bufLen, dumpChunk->isLastChunk, dumpChunk->flag,
716 : hostDumpDataInfoQueue_->Size());
717 : #if !defined(ADUMP_SOC_HOST) || ADUMP_SOC_HOST == 1
718 6 : if (FileNameCheck(*dumpChunk)) {
719 1 : IDE_CTRL_VALUE_FAILED_NODO(
720 : StatsDataParsing(*dumpChunk), continue, "Failed to parse dump data with file name %s",
721 : dumpChunk->fileName);
722 1 : continue;
723 : }
724 : #endif
725 5 : if (AdxDumpProcess::Instance().IsRegistered()) {
726 3 : IDE_LOGI("mindspore session!");
727 : std::function<int32_t(const struct DumpChunk*, int32_t)> messageCallback =
728 3 : AdxDumpProcess::Instance().GetCallbackFun();
729 3 : if (!messageCallback) {
730 0 : IDE_LOGE(
731 : "Registered messageCallback function is not callable,\
732 : drop this data packet, fileName:%s",
733 : dumpChunk->fileName);
734 0 : continue;
735 : }
736 3 : int32_t dumpChunkLen = sizeof(struct DumpChunk) + dumpChunk->bufLen;
737 3 : int32_t ret = messageCallback(dumpChunk, dumpChunkLen);
738 3 : if (ret != IDE_DAEMON_NONE_ERROR) {
739 0 : IDE_LOGE("failed to transmission dump data to mindspore. err = %d", ret);
740 : }
741 5 : } else if (!RecordDumpDataToDisk(*dumpChunk)) {
742 2 : IDE_LOGE("failed to record dump data to disk.");
743 : }
744 54 : }
745 58 : IDE_LOGI("exit record file thread");
746 58 : }
747 :
748 : /**
749 : * @brief record dump info to the file
750 : * @param [in] data : record data
751 : * @return
752 : * IDE_DAEMON_ERROR : falied
753 : * IDE_DAEMON_OK : success
754 : */
755 8 : bool AdxDumpRecord::RecordDumpDataToQueue(HostDumpDataInfo& info)
756 : {
757 8 : if (hostDumpDataInfoQueue_ == nullptr) {
758 1 : IDE_LOGW("dump data queue is not initialized, drop this data packet.");
759 1 : return false;
760 : }
761 7 : if (hostDumpDataInfoQueue_->IsFull()) {
762 2 : const std::string tipFull = "Memory usage exceeds 85%, the dump data queue is full";
763 2 : const std::string tipReduce = "Please reduce model batches, images or dump layers";
764 1 : const std::string tipMemory = "Or clear the used memory or increase the maximum memory";
765 1 : IDE_LOGW("%s. %s. %s.", tipFull.c_str(), tipReduce.c_str(), tipMemory.c_str());
766 1 : return false;
767 1 : } else {
768 6 : if (!hostDumpDataInfoQueue_->Push(info)) {
769 0 : IDE_LOGW("dump data queue has quit, drop this data packet.");
770 0 : return false;
771 : }
772 6 : IDE_LOGI("Insert dump data to queue success, queue size: %u.", hostDumpDataInfoQueue_->Size());
773 : }
774 :
775 6 : return true;
776 : }
777 :
778 : /**
779 : * @brief get dump size from the record lists
780 : * @param [in] tag : tag of record lists
781 : * @return
782 : * true : success
783 : * false : false
784 : */
785 62 : bool AdxDumpRecord::DumpDataQueueIsEmpty() const
786 : {
787 62 : return hostDumpDataInfoQueue_ == nullptr || hostDumpDataInfoQueue_->IsEmpty();
788 : }
789 :
790 0 : void AdxDumpRecord::SetDumpPath(const std::string& dumpPath) { dumpPath_ = dumpPath; }
791 : } // namespace Adx
|
