Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "adx_dump_record.h"
12 : #include <map>
13 : #include <cinttypes>
14 : #include <functional>
15 : #include <pthread.h>
16 : #include "mmpa_api.h"
17 : #include "adx_log.h"
18 : #include "file_utils.h"
19 : #include "string_utils.h"
20 : #include "memory_utils.h"
21 : #include "common_utils.h"
22 : #include "adx_dump_process.h"
23 : #include "ide_os_type.h"
24 : namespace Adx {
25 : static const std::size_t MAX_IP_LENGTH = 16;
26 : #if !defined(ADUMP_SOC_HOST) || ADUMP_SOC_HOST == 1
27 : constexpr char STRING_BIN[] = ".bin";
28 : constexpr char STRING_CSV[] = ".csv";
29 : constexpr char CSV_HEADER[] = "Input/Output,Index,Data Size,Data Type,Format,Shape";
30 :
31 : static const std::map<uint64_t, std::string> STATS_ITEM_MAP = {
32 : {DUMP_STATS_MAX, ",Max Value"},
33 : {DUMP_STATS_MIN, ",Min Value"},
34 : {DUMP_STATS_AVG, ",Avg Value"},
35 : {DUMP_STATS_NAN, ",Nan Count"},
36 : {DUMP_STATS_NEG_INF, ",Negative Inf Count"},
37 : {DUMP_STATS_POS_INF, ",Positive Inf Count"},
38 : {DUMP_STATS_L2NORM, ",l2norm"}
39 : };
40 :
41 : static const std::map<toolkit::dump::OutputDataType, std::string> DT_STRING_MAP = {
42 : {toolkit::dump::DT_UNDEFINED, "DT_UNDEFINED"},
43 : {toolkit::dump::DT_FLOAT, "DT_FLOAT"},
44 : {toolkit::dump::DT_FLOAT16, "DT_FLOAT16"},
45 : {toolkit::dump::DT_INT8, "DT_INT8"},
46 : {toolkit::dump::DT_UINT8, "DT_UINT8"},
47 : {toolkit::dump::DT_INT16, "DT_INT16"},
48 : {toolkit::dump::DT_UINT16, "DT_UINT16"},
49 : {toolkit::dump::DT_INT32, "DT_INT32"},
50 : {toolkit::dump::DT_INT64, "DT_INT64"},
51 : {toolkit::dump::DT_UINT32, "DT_UINT32"},
52 : {toolkit::dump::DT_UINT64, "DT_UINT64"},
53 : {toolkit::dump::DT_BOOL, "DT_BOOL"},
54 : {toolkit::dump::DT_DOUBLE, "DT_DOUBLE"},
55 : {toolkit::dump::DT_STRING, "DT_STRING"},
56 : {toolkit::dump::DT_DUAL_SUB_INT8, "DT_DUAL_SUB_INT8"},
57 : {toolkit::dump::DT_DUAL_SUB_UINT8, "DT_DUAL_SUB_UINT8"},
58 : {toolkit::dump::DT_COMPLEX64, "DT_COMPLEX64"},
59 : {toolkit::dump::DT_COMPLEX128, "DT_COMPLEX128"},
60 : {toolkit::dump::DT_QINT8, "DT_QINT8"},
61 : {toolkit::dump::DT_QINT16, "DT_QINT16"},
62 : {toolkit::dump::DT_QINT32, "DT_QINT32"},
63 : {toolkit::dump::DT_QUINT8, "DT_QUINT8"},
64 : {toolkit::dump::DT_QUINT16, "DT_QUINT16"},
65 : {toolkit::dump::DT_RESOURCE, "DT_RESOURCE"},
66 : {toolkit::dump::DT_STRING_REF, "DT_STRING_REF"},
67 : {toolkit::dump::DT_DUAL, "DT_DUAL"},
68 : {toolkit::dump::DT_VARIANT, "DT_VARIANT"},
69 : {toolkit::dump::DT_BF16, "DT_BF16"},
70 : {toolkit::dump::DT_INT4, "DT_INT4"},
71 : {toolkit::dump::DT_UINT1, "DT_UINT1"},
72 : {toolkit::dump::DT_INT2, "DT_INT2"},
73 : {toolkit::dump::DT_UINT2, "DT_UINT2"},
74 : {toolkit::dump::DT_HIFLOAT8, "DT_HIFLOAT8"},
75 : {toolkit::dump::DT_FLOAT8_E5M2, "DT_FLOAT8_E5M2"},
76 : {toolkit::dump::DT_FLOAT8_E4M3FN, "DT_FLOAT8_E4M3FN"},
77 : {toolkit::dump::DT_FLOAT8_E8M0, "DT_FLOAT8_E8M0"},
78 : {toolkit::dump::DT_FLOAT6_E3M2, "DT_FLOAT6_E3M2"},
79 : {toolkit::dump::DT_FLOAT6_E2M3, "DT_FLOAT6_E2M3"},
80 : {toolkit::dump::DT_FLOAT4_E2M1, "DT_FLOAT4_E2M1"},
81 : {toolkit::dump::DT_FLOAT4_E1M2, "DT_FLOAT4_E1M2"},
82 : };
83 :
84 : static const std::map<toolkit::dump::OutputFormat, std::string> FORMAT_STRING_MAP = {
85 : {toolkit::dump::FORMAT_NCHW, "NCHW"},
86 : {toolkit::dump::FORMAT_NHWC, "NHWC"},
87 : {toolkit::dump::FORMAT_ND, "ND"},
88 : {toolkit::dump::FORMAT_NC1HWC0, "NC1HWC0"},
89 : {toolkit::dump::FORMAT_FRACTAL_Z, "FRACTAL_Z"},
90 : {toolkit::dump::FORMAT_NC1C0HWPAD, "NC1C0HWPAD"},
91 : {toolkit::dump::FORMAT_NHWC1C0, "NHWC1C0"},
92 : {toolkit::dump::FORMAT_FSR_NCHW, "FSR_NCHW"},
93 : {toolkit::dump::FORMAT_FRACTAL_DECONV, "FRACTAL_DECONV"},
94 : {toolkit::dump::FORMAT_C1HWNC0, "C1HWNC0"},
95 : {toolkit::dump::FORMAT_FRACTAL_DECONV_TRANSPOSE, "FRACTAL_DECONV_TRANSPOSE"},
96 : {toolkit::dump::FORMAT_FRACTAL_DECONV_SP_STRIDE_TRANS, "FRACTAL_DECONV_SP_STRIDE_TRANS"},
97 : {toolkit::dump::FORMAT_NC1HWC0_C04, "NC1HWC0_C04"},
98 : {toolkit::dump::FORMAT_FRACTAL_Z_C04, "FRACTAL_Z_C04"},
99 : {toolkit::dump::FORMAT_CHWN, "CHWN"},
100 : {toolkit::dump::FORMAT_FRACTAL_DECONV_SP_STRIDE8_TRANS, "FRACTAL_DECONV_SP_STRIDE8_TRANS"},
101 : {toolkit::dump::FORMAT_HWCN, "HWCN"},
102 : {toolkit::dump::FORMAT_NC1KHKWHWC0, "NC1KHKWHWC0"},
103 : {toolkit::dump::FORMAT_BN_WEIGHT, "BN_WEIGHT"},
104 : {toolkit::dump::FORMAT_FILTER_HWCK, "FILTER_HWCK"},
105 : {toolkit::dump::FORMAT_HASHTABLE_LOOKUP_LOOKUPS, "HASHTABLE_LOOKUP_LOOKUPS"},
106 : {toolkit::dump::FORMAT_HASHTABLE_LOOKUP_KEYS, "HASHTABLE_LOOKUP_KEYS"},
107 : {toolkit::dump::FORMAT_HASHTABLE_LOOKUP_VALUE, "HASHTABLE_LOOKUP_VALUE"},
108 : {toolkit::dump::FORMAT_HASHTABLE_LOOKUP_OUTPUT, "HASHTABLE_LOOKUP_OUTPUT"},
109 : {toolkit::dump::FORMAT_HASHTABLE_LOOKUP_HITS, "HASHTABLE_LOOKUP_HITS"},
110 : {toolkit::dump::FORMAT_C1HWNCoC0, "C1HWNCoC0"},
111 : {toolkit::dump::FORMAT_MD, "MD"},
112 : {toolkit::dump::FORMAT_NDHWC, "NDHWC"},
113 : {toolkit::dump::FORMAT_FRACTAL_ZZ, "FRACTAL_ZZ"},
114 : {toolkit::dump::FORMAT_FRACTAL_NZ, "FRACTAL_NZ"},
115 : {toolkit::dump::FORMAT_NCDHW, "NCDHW"},
116 : {toolkit::dump::FORMAT_DHWCH, "DHWCH"},
117 : {toolkit::dump::FORMAT_NDC1HWC0, "NDC1HWC0"},
118 : {toolkit::dump::FORMAT_FRACTAL_Z_3D, "FRACTAL_Z_3D"},
119 : {toolkit::dump::FORMAT_CN, "CN"},
120 : {toolkit::dump::FORMAT_NC, "NC"},
121 : {toolkit::dump::FORMAT_DHWNC, "DHWNC"},
122 : {toolkit::dump::FORMAT_FRACTAL_Z_3D_TRANSPOSE, "FRACTAL_Z_3D_TRANSPOSE"},
123 : {toolkit::dump::FORMAT_FRACTAL_ZN_LSTM, "FRACTAL_ZN_LSTM"},
124 : {toolkit::dump::FORMAT_FRACTAL_Z_G, "FRACTAL_Z_G"},
125 : {toolkit::dump::FORMAT_RESERVED, "RESERVED"},
126 : {toolkit::dump::FORMAT_ALL, "ALL"},
127 : {toolkit::dump::FORMAT_NULL, "NULL"},
128 : {toolkit::dump::FORMAT_ND_RNN_BIAS, "ND_RNN_BIAS"},
129 : {toolkit::dump::FORMAT_FRACTAL_ZN_RNN, "FRACTAL_ZN_RNN"},
130 : {toolkit::dump::FORMAT_NYUV, "NYUV"},
131 : {toolkit::dump::FORMAT_NYUV_A, "NYUV_A"},
132 : {toolkit::dump::FORMAT_NCL, "NCL"},
133 : {toolkit::dump::FORMAT_FRACTAL_Z_WINO, "FRACTAL_Z_WINO"},
134 : {toolkit::dump::FORMAT_C1HWC0, "C1HWC0"}
135 : };
136 : #endif
137 :
138 5 : AdxDumpRecord::AdxDumpRecord()
139 5 : : dumpRecordFlag_(true),
140 5 : dumpInitNum_(0)
141 : {
142 5 : int32_t ret = pthread_atfork(
143 : AdxDumpRecord::PrepareFork, AdxDumpRecord::PostForkParent, AdxDumpRecord::PostForkChild);
144 5 : if (ret != 0) {
145 0 : IDE_LOGW("call pthread_atfork failed, ret: %d", ret);
146 : }
147 : #if !defined(ADUMP_SOC_HOST) || ADUMP_SOC_HOST == 1
148 5 : fileNameStatus_.reserve(FILENAME_CHECK_SIZE_MAX);
149 55 : for (size_t idx = 0; idx < FILENAME_CHECK_SIZE_MAX; ++idx) {
150 150 : fileNameStatus_.push_back("");
151 : }
152 40 : funcMap_ = {
153 0 : {0, std::bind(&AdxDumpRecord::TypeDataHandle, this, std::placeholders::_1, std::placeholders::_2)}, // max
154 0 : {1, std::bind(&AdxDumpRecord::TypeDataHandle, this, std::placeholders::_1, std::placeholders::_2)}, // min
155 0 : {2, std::bind(&AdxDumpRecord::FloatDataHandle, this, std::placeholders::_1)}, // avg
156 0 : {3, std::bind(&AdxDumpRecord::Int32DataHandle, this, std::placeholders::_1)}, // nan
157 0 : {4, std::bind(&AdxDumpRecord::Int32DataHandle, this, std::placeholders::_1)}, // neg inf
158 0 : {5, std::bind(&AdxDumpRecord::Int32DataHandle, this, std::placeholders::_1)}, // pos inf
159 0 : {6, std::bind(&AdxDumpRecord::FloatDataHandle, this, std::placeholders::_1)}, // l2norm
160 40 : };
161 : #endif
162 10 : }
163 :
164 5 : AdxDumpRecord::~AdxDumpRecord()
165 : {
166 5 : UnInit();
167 5 : }
168 :
169 391 : int32_t AdxDumpRecord::GetDumpInitNum() const
170 : {
171 391 : return dumpInitNum_;
172 : }
173 :
174 112 : void AdxDumpRecord::UpdateDumpInitNum(bool isPlus)
175 : {
176 112 : if (isPlus) {
177 52 : dumpInitNum_++;
178 60 : } else if (dumpInitNum_ > 0) {
179 52 : dumpInitNum_--;
180 : }
181 112 : IDE_LOGI("dump init number: %d", dumpInitNum_);
182 112 : }
183 :
184 55 : bool AdxDumpRecord::HasStartedServer() const
185 : {
186 55 : return dumpInitNum_ > 0;
187 : }
188 :
189 62 : bool AdxDumpRecord::CanShutdownServer() const
190 : {
191 62 : return dumpInitNum_ <= 1;
192 : }
193 :
194 65 : int32_t AdxDumpRecord::Init(const std::string &hostPid)
195 : {
196 : // non-soc case
197 65 : if (hostPid.empty()) {
198 60 : char dumpPath[MAX_FILE_PATH_LENGTH] = {0};
199 60 : if (mmGetCwd(dumpPath, sizeof(dumpPath)) != EN_OK) {
200 0 : IDE_LOGE("get current dir failed ");
201 0 : return IDE_DAEMON_ERROR;
202 : }
203 60 : dumpPath_ = dumpPath;
204 : } else {
205 : #if (OS_TYPE == LINUX)
206 : // soc case
207 5 : std::string appBin = "/proc/" + hostPid + "/exe";
208 5 : errno = 0;
209 5 : if (!FileUtils::IsFileExist(appBin) && errno != EACCES) {
210 0 : appBin = "/local/proc/" + hostPid + "/exe"; // aoscore
211 : }
212 5 : uint32_t pathSize = MMPA_MAX_PATH + 1;
213 5 : IdeStringBuffer curPath = reinterpret_cast<IdeStringBuffer>(IdeXmalloc(pathSize));
214 5 : IDE_CTRL_VALUE_FAILED(curPath != nullptr, return IDE_DAEMON_ERROR, "malloc failed");
215 5 : errno = 0;
216 5 : int32_t len = readlink(appBin.c_str(), curPath, MMPA_MAX_PATH); // read self path of store
217 5 : if (len < 0 || len > MMPA_MAX_PATH) {
218 3 : IDE_LOGE("Can't get app bin directory, strerr: %s, length: %d bytes, path: %s",
219 : strerror(errno), len, appBin.c_str());
220 3 : IDE_XFREE_AND_SET_NULL(curPath);
221 3 : return IDE_DAEMON_ERROR;
222 : }
223 2 : IDE_LOGI("get app bin path: %s", curPath);
224 2 : curPath[len] = '\0'; // add string end char
225 2 : dumpPath_ = curPath;
226 2 : std::string::size_type idx = dumpPath_.find_last_of(OS_SPLIT_STR);
227 2 : dumpPath_ = dumpPath_.substr(0, idx);
228 2 : IDE_XFREE_AND_SET_NULL(curPath);
229 : #endif
230 5 : }
231 62 : IDE_LOGI("dumpPath prefix is %s", dumpPath_.c_str());
232 62 : if (!dumpPath_.empty() && dumpPath_.back() != OS_SPLIT_CHAR) {
233 40 : dumpPath_ += OS_SPLIT_CHAR;
234 : }
235 :
236 62 : std::lock_guard<std::mutex> lock(recordMutex_);
237 62 : if (hostDumpDataInfoQueue_ == nullptr) {
238 6 : hostDumpDataInfoQueue_.reset(new(std::nothrow) BoundQueueMemory<HostDumpDataInfo>());
239 6 : IDE_CTRL_VALUE_FAILED(hostDumpDataInfoQueue_ != nullptr, return IDE_DAEMON_ERROR,
240 : "Failed to new hostDumpDataInfoQueue");
241 : }
242 62 : hostDumpDataInfoQueue_->Init();
243 62 : IDE_LOGI("record remote dump temp path: %s", dumpPath_.c_str());
244 62 : hostDumpDataInfoQueue_->SetPath(dumpPath_);
245 62 : dumpRecordFlag_ = true;
246 62 : return IDE_DAEMON_OK;
247 62 : }
248 :
249 57 : int32_t AdxDumpRecord::StartRecord()
250 : {
251 57 : std::lock_guard<std::mutex> lock(recordMutex_);
252 57 : if (recordThread_.joinable()) {
253 4 : IDE_LOGI("dump record thread has been started, no need to start again");
254 4 : return IDE_DAEMON_OK;
255 : }
256 53 : dumpRecordFlag_ = true;
257 : try {
258 53 : recordThread_ = std::thread(&AdxDumpRecord::RecordDumpInfo, this);
259 0 : } catch (const std::exception &ex) {
260 0 : dumpRecordFlag_ = false;
261 0 : IDE_LOGE("Create the dump record thread failed, message: %s", ex.what());
262 0 : return IDE_DAEMON_ERROR;
263 0 : }
264 53 : return IDE_DAEMON_OK;
265 57 : }
266 :
267 : /**
268 : * @brief initialize record file
269 : * @param [in] recordPath : record file Path
270 : * @return
271 : * IDE_DAEMON_ERROR : falied
272 : * IDE_DAEMON_OK : success
273 : */
274 67 : int32_t AdxDumpRecord::UnInit()
275 : {
276 67 : std::lock_guard<std::mutex> lock(recordMutex_);
277 67 : IDE_LOGI("start to dump uninit");
278 67 : dumpRecordFlag_ = false;
279 67 : if (hostDumpDataInfoQueue_ != nullptr) {
280 64 : hostDumpDataInfoQueue_->Quit();
281 : }
282 67 : if (recordThread_.joinable()) {
283 52 : recordThread_.join();
284 : }
285 67 : IDE_LOGI("dump uninit success");
286 67 : return IDE_DAEMON_OK;
287 67 : }
288 :
289 8 : void AdxDumpRecord::PrepareFork()
290 : {
291 8 : Instance().recordMutex_.lock();
292 8 : }
293 :
294 5 : void AdxDumpRecord::PostForkParent()
295 : {
296 5 : Instance().recordMutex_.unlock();
297 5 : }
298 :
299 3 : void AdxDumpRecord::PostForkChild()
300 : {
301 3 : auto &instance = Instance();
302 3 : instance.dumpRecordFlag_ = false;
303 3 : if (instance.recordThread_.joinable()) {
304 1 : instance.recordThread_.detach();
305 : }
306 :
307 3 : (void)instance.hostDumpDataInfoQueue_.release();
308 3 : instance.recordMutex_.unlock();
309 3 : }
310 :
311 : /**
312 : * @brief record dump data to disk
313 : * @param [in] dumpChunk : dump chunk
314 : * @return
315 : * true : record dump data to disk success
316 : * false : record dump data to disk failed
317 : */
318 8 : bool AdxDumpRecord::RecordDumpDataToDisk(const DumpChunk &dumpChunk) const
319 : {
320 : // dump file aging
321 8 : std::string filePath = dumpChunk.fileName;
322 8 : if (filePath.empty()) {
323 2 : IDE_LOGE("filepath of received dump chunk is empty");
324 2 : return false;
325 : }
326 6 : if (JudgeRemoteFalg(filePath)) {
327 0 : auto pos = filePath.find_first_of(":");
328 0 : filePath = filePath.substr(pos + 1);
329 0 : filePath = dumpPath_ + filePath;
330 : } else {
331 6 : if (!FileUtils::IsAbsolutePath(filePath)) {
332 6 : filePath = dumpPath_ + filePath;
333 : }
334 : }
335 :
336 : #if (OS_TYPE != LINUX)
337 : filePath = FileUtils::ReplaceAll(filePath, "/", "\\");
338 : #endif
339 :
340 6 : IDE_LOGI("start to record dump data to disk path: %s", filePath.c_str());
341 :
342 6 : std::string saveDirName = FileUtils::GetFileDir(filePath);
343 6 : if (!FileUtils::IsFileExist(saveDirName)) {
344 1 : if (FileUtils::CreateDir(saveDirName) != IDE_DAEMON_NONE_ERROR) {
345 1 : IDE_LOGE("create dir failed path: %s", filePath.c_str());
346 1 : return false;
347 : }
348 : }
349 :
350 5 : while (FileUtils::IsDiskFull(saveDirName, dumpChunk.bufLen)) {
351 1 : IDE_LOGE("don't have enough free disk %u bytes", dumpChunk.bufLen);
352 1 : return false;
353 : }
354 :
355 4 : std::string realPath;
356 4 : if (FileUtils::FileNameIsReal(filePath, realPath) != IDE_DAEMON_OK) {
357 1 : IDE_LOGE("real path: %s", filePath.c_str());
358 1 : return false;
359 : }
360 :
361 3 : IdeErrorT err = FileUtils::WriteFile(realPath.c_str(),
362 3 : dumpChunk.dataBuf, dumpChunk.bufLen, dumpChunk.offset);
363 3 : if (err != IDE_DAEMON_NONE_ERROR) {
364 1 : (void)remove(realPath.c_str());
365 1 : IDE_LOGE("WriteFile failed, fileName: %s, err: %d", realPath.c_str(), err);
366 1 : return false;
367 : }
368 2 : IDE_LOGI("record dump data success: %s", realPath.c_str());
369 2 : return true;
370 8 : }
371 :
372 : /**
373 : * @brief judge if is remote case based on flag
374 : * @param [in] msg : flag msg
375 : * @return
376 : * true : is remote case
377 : * false : not remote case
378 : */
379 6 : bool AdxDumpRecord::JudgeRemoteFalg(const std::string &msg) const
380 : {
381 6 : std::size_t len = msg.find_first_of (":");
382 6 : if (len != std::string::npos && len < MAX_IP_LENGTH) {
383 0 : std::string ipStr = msg.substr(0, len);
384 0 : if (StringUtils::IpValid(ipStr)) {
385 0 : IDE_LOGD("remote ip info check pass: %s", ipStr.c_str());
386 0 : return true;
387 : }
388 0 : }
389 6 : IDE_LOGD("non remote ip case checked.");
390 :
391 6 : return false;
392 : }
393 :
394 0 : void AdxDumpRecord::SetWorkPath(const std::string &path)
395 : {
396 0 : workPath_ = path;
397 0 : }
398 :
399 : #if !defined(ADUMP_SOC_HOST) || ADUMP_SOC_HOST == 1
400 13 : void AdxDumpRecord::SetOptimizationMode(uint64_t statsItem)
401 : {
402 13 : if (dumpStatsItem_ == statsItem) {
403 10 : return;
404 : }
405 3 : dumpStatsItem_ = statsItem;
406 3 : IDE_LOGI("SetOptimizationMode success with stats items:[0x%llx]", dumpStatsItem_);
407 3 : uint16_t statsNum = 0;
408 3 : statsHeader_ = CSV_HEADER;
409 3 : statsList_.clear();
410 :
411 3 : if (dumpStatsItem_ == (DUMP_STATS_MAX | DUMP_STATS_MIN | DUMP_STATS_AVG | DUMP_STATS_NAN | DUMP_STATS_NEG_INF |
412 : DUMP_STATS_POS_INF)) {
413 1 : compatible_ = true;
414 : } else {
415 2 : statsHeader_ += ",Count";
416 : }
417 24 : for (auto it = STATS_ITEM_MAP.begin(); it != STATS_ITEM_MAP.cend(); ++it) {
418 21 : if ((dumpStatsItem_ & it->first) != 0) {
419 9 : if (compatible_ && statsNum == COUNT_HEADER_COMPATIBLE) {
420 1 : statsHeader_ += ",Count";
421 : }
422 9 : statsHeader_ += it->second;
423 9 : statsList_.push_back(statsNum);
424 : }
425 21 : ++statsNum;
426 : }
427 3 : IDE_LOGI("SetOptimizationMode generate table header [%s].", statsHeader_.c_str());
428 : }
429 :
430 10 : std::string AdxDumpRecord::GetShapeString(const uint32_t shape[], int32_t shapeSize, int64_t &count) const
431 : {
432 10 : IDE_CTRL_VALUE_FAILED(shapeSize <= MAX_SHAPE_SIZE, return "NA", "Shape size %d is out of range %d",
433 : shapeSize, MAX_SHAPE_SIZE);
434 10 : if (shapeSize == 0) { // 没有shape信息时，保持原有方式用-占位
435 0 : return "-";
436 : }
437 10 : std::ostringstream oss;
438 40 : for (int32_t idx = 0; idx < shapeSize; ++idx) {
439 30 : count *= shape[idx];
440 30 : oss << shape[idx];
441 30 : if (idx != shapeSize - 1) {
442 20 : oss << "x";
443 : }
444 : }
445 10 : return oss.str();
446 10 : }
447 :
448 : template <typename T>
449 20 : std::string AdxDumpRecord::GetStringName(T key, const std::map<T, std::string> &stringMap) const
450 : {
451 20 : auto it = stringMap.find(key);
452 20 : if (it != stringMap.cend()) {
453 19 : return it->second;
454 : }
455 1 : IDE_LOGW("Cannot find [%d] type name in string map.", key);
456 2 : return "UNKNOW";
457 : }
458 :
459 28 : std::string AdxDumpRecord::Int32DataHandle(const int64_t &data) const
460 : {
461 : DataTypeUnion dtUnion;
462 28 : dtUnion.longIntValue = data;
463 56 : return std::to_string(dtUnion.intValue[0]);
464 : }
465 :
466 19 : std::string AdxDumpRecord::FloatDataHandle(const int64_t &data) const
467 : {
468 : DataTypeUnion dtUnion;
469 19 : dtUnion.longIntValue = data;
470 19 : std::ostringstream oss;
471 19 : oss << dtUnion.floatValue;
472 38 : return oss.str();
473 19 : }
474 :
475 18 : std::string AdxDumpRecord::TypeDataHandle(const int64_t &data, toolkit::dump::OutputDataType dType) const
476 : {
477 : // 如果dt是int64数据读取为int64；dt是非int64数据读取为int32，dt是float数据读取为float32
478 18 : if (dType == toolkit::dump::DT_FLOAT || dType == toolkit::dump::DT_FLOAT16 || dType == toolkit::dump::DT_BF16 ||
479 12 : dType == toolkit::dump::DT_HIFLOAT8 || dType == toolkit::dump::DT_FLOAT8_E5M2 ||
480 : dType == toolkit::dump::DT_FLOAT8_E4M3FN) {
481 10 : return FloatDataHandle(data);
482 8 : } else if (dType == toolkit::dump::DT_INT64) {
483 4 : return std::to_string(data);
484 : } else {
485 4 : return Int32DataHandle(data);
486 : }
487 : }
488 :
489 52 : std::string AdxDumpRecord::GetStatsString(toolkit::dump::OutputDataType dType, uint16_t pos, const int64_t &data)
490 : {
491 52 : auto it = funcMap_.find(pos);
492 52 : if (it != funcMap_.cend()) {
493 51 : std::string result = it->second(data, dType);
494 51 : return result;
495 0 : } else {
496 1 : IDE_LOGW("Dump stats [%hu] is not supported.", pos);
497 2 : return "NA";
498 : }
499 : }
500 :
501 3 : bool AdxDumpRecord::CheckFileNameExist(const std::string &filename)
502 : {
503 3 : auto it = std::find(fileNameStatus_.begin(), fileNameStatus_.end(), filename);
504 3 : if (it != fileNameStatus_.end()) {
505 2 : return true;
506 : }
507 1 : return false;
508 : }
509 :
510 14 : void AdxDumpRecord::AppendFileName(const std::string &filename)
511 : {
512 : // Stores the names of processed files.
513 14 : fileNameStatus_[filenameIndex_] = filename;
514 14 : filenameIndex_ = (filenameIndex_ + 1) % fileNameStatus_.size();
515 14 : }
516 :
517 10 : void AdxDumpRecord::StatisticsData(std::stringstream &strStream, std::shared_ptr<OpStatsResult> statsResult,
518 : const int64_t &count, const int32_t &idx)
519 : {
520 10 : if (statsResult->stat[idx].result != 0) {
521 1 : IDE_LOGW("Stats data is unsupported, index is %d, size is %" PRId64 ", result is %u.",
522 : statsResult->stat[idx].index, statsResult->stat[idx].size, statsResult->stat[idx].result);
523 7 : for (size_t i = 0; i < statsList_.size(); ++i) {
524 6 : if (compatible_ && i == COUNT_HEADER_COMPATIBLE) {
525 1 : strStream << "," << count;
526 : }
527 6 : strStream << "," << "NA";
528 : }
529 : } else {
530 9 : size_t statsSize = sizeof(statsResult->stat[idx].stats) / sizeof(statsResult->stat[idx].stats[0]);
531 9 : if (!compatible_) {
532 0 : strStream << "," << count;
533 : }
534 60 : for (const uint16_t &pos : statsList_) {
535 51 : IDE_CTRL_VALUE_FAILED_NODO(pos < statsSize, continue, "Index %hu is out of range %" PRIu64 ".",
536 : pos, statsSize);
537 51 : if (compatible_ && pos == COUNT_HEADER_COMPATIBLE) {
538 8 : strStream << "," << count;
539 : }
540 51 : strStream << "," << GetStatsString(
541 51 : static_cast<toolkit::dump::OutputDataType>(statsResult->stat[idx].dType), pos,
542 102 : statsResult->stat[idx].stats[pos]);
543 : }
544 : }
545 10 : }
546 :
547 10 : bool AdxDumpRecord::GenerateFileData(std::stringstream &strStream, const std::string &filename,
548 : std::shared_ptr<OpStatsResult> statsResult)
549 : {
550 10 : IDE_CTRL_VALUE_FAILED(statsResult->statItem != 0, return false, "Dump stats is empty, nothing need to do.");
551 10 : IDE_CTRL_VALUE_FAILED(statsResult->tensorNum <= MAX_STATS_RESULT_NUM, return false,
552 : "Dump stats number %d is bigger than %d.", statsResult->tensorNum, MAX_STATS_RESULT_NUM);
553 10 : SetOptimizationMode(statsResult->statItem);
554 : // If the file name is processed for the first time, add the table header.
555 10 : if (!CheckFileNameExist(filename)) {
556 9 : strStream << statsHeader_;
557 9 : strStream << "\n";
558 : }
559 :
560 20 : for (int32_t idx = 0; idx < statsResult->tensorNum; ++idx) {
561 10 : IDE_LOGI("Process the data in file %s for the %d times.", filename.c_str(), idx);
562 10 : int64_t count = 1; // 默认count值为1
563 10 : std::string ioString = statsResult->stat[idx].io == 0 ? "Input" : "Output";
564 10 : strStream << ioString;
565 10 : strStream << "," << statsResult->stat[idx].index;
566 10 : strStream << "," << statsResult->stat[idx].size;
567 20 : strStream << "," << GetStringName(static_cast<toolkit::dump::OutputDataType>(statsResult->stat[idx].dType),
568 20 : DT_STRING_MAP);
569 20 : strStream << "," << GetStringName(static_cast<toolkit::dump::OutputFormat>(statsResult->stat[idx].format),
570 20 : FORMAT_STRING_MAP);
571 10 : strStream << "," << GetShapeString(statsResult->stat[idx].shape, statsResult->stat[idx].shapeSize, count);
572 10 : IDE_LOGD("Process the data with index[%d], io[%d], statsLen[%u], result[%u], shapeSize[%d], size[%" PRId64 "].",
573 : statsResult->stat[idx].index, statsResult->stat[idx].io, statsResult->stat[idx].statsLen,
574 : statsResult->stat[idx].result, statsResult->stat[idx].shapeSize, statsResult->stat[idx].size);
575 :
576 10 : StatisticsData(strStream, statsResult, count, idx);
577 10 : strStream << "\n";
578 10 : }
579 10 : return true;
580 : }
581 :
582 1 : bool AdxDumpRecord::DumpDataToCallback(const std::string &filename, const std::string &dumpData, int64_t offSet,
583 : int32_t flag)
584 : {
585 1 : IDE_LOGD("Ready to send file %s to mindspore session!", filename.c_str());
586 1 : IDE_CTRL_VALUE_FAILED(!dumpData.empty(), return false, "Dump data in DumpDataToCallback is empty.");
587 :
588 1 : std::function<int32_t(const struct DumpChunk *, int32_t)> messageCallback = AdxDumpProcess::Instance().GetCallbackFun();
589 1 : IDE_CTRL_VALUE_FAILED(messageCallback, return false, "Registered messageCallback function is not callable,\
590 : drop this data packet, filename:%s", filename.c_str());
591 :
592 1 : auto dumpChunkLen = sizeof(DumpChunk) + dumpData.size() + 1;
593 2 : std::unique_ptr<DumpChunk, void(*)(DumpChunk*)> dumpChunk(static_cast<DumpChunk*>(IdeXmalloc(dumpChunkLen)),
594 2 : [](DumpChunk *p) { IdeXfree(p); });
595 1 : IDE_CTRL_VALUE_FAILED(dumpChunk != nullptr, return IDE_DAEMON_ERROR, "Failed to malloc for dump chunk.");
596 :
597 1 : errno_t err = strncpy_s(dumpChunk->fileName, MAX_FILE_PATH_LENGTH, filename.c_str(), filename.size());
598 1 : IDE_CTRL_VALUE_FAILED(err == EOK, return false, "Filename string copy failed, err: %d", err);
599 1 : dumpChunk->bufLen = dumpData.size() + 1;
600 1 : dumpChunk->offset = offSet;
601 1 : dumpChunk->flag = flag;
602 1 : dumpChunk->isLastChunk = 1;
603 :
604 1 : err = strncpy_s(reinterpret_cast<AdxStringBuffer>(dumpChunk->dataBuf), dumpChunk->bufLen,
605 : dumpData.c_str(), dumpData.size());
606 1 : IDE_CTRL_VALUE_FAILED(err == EOK, return false, "DataBuf string copy failed, err: %d", err);
607 :
608 1 : int32_t ret = messageCallback(dumpChunk.get(), dumpChunkLen);
609 1 : IDE_CTRL_VALUE_FAILED(ret == IDE_DAEMON_NONE_ERROR, return false,
610 : "Failed to transmission dump data to mindspore. err = %d", ret);
611 1 : IDE_LOGI("Send dump data to mindspore success: %s", filename.c_str());
612 1 : AppendFileName(filename);
613 1 : return true;
614 1 : }
615 :
616 6 : bool AdxDumpRecord::FileNameCheck(const DumpChunk &dumpChunk) const
617 : {
618 6 : size_t binSize = strlen(STRING_BIN);
619 6 : std::string filename = dumpChunk.fileName;
620 6 : size_t nameSize = filename.size();
621 9 : if (nameSize > binSize && filename.compare(nameSize - binSize, binSize, STRING_BIN) == 0 &&
622 3 : dumpChunk.bufLen == sizeof(OpStatsResult)) {
623 1 : IDE_LOGI("Received dump buffer length %u bytes, file name [%s].", dumpChunk.bufLen, dumpChunk.fileName);
624 1 : return true;
625 : }
626 5 : IDE_LOGD("File name [%s] of dump chunk is not end with .bin, or data size %u bytes is different from %" PRIu64 ".",
627 : dumpChunk.fileName, dumpChunk.bufLen, sizeof(OpStatsResult));
628 5 : return false;
629 6 : }
630 :
631 7 : bool AdxDumpRecord::StatsDataParsing(const DumpChunk &dumpChunk)
632 : {
633 7 : std::string filename = dumpChunk.fileName;
634 7 : filename.replace(filename.size() - strlen(STRING_BIN), strlen(STRING_BIN), STRING_CSV);
635 :
636 7 : std::shared_ptr<OpStatsResult> statsResult;
637 : try {
638 7 : statsResult = std::make_shared<OpStatsResult>();
639 0 : } catch (std::exception &ex) {
640 0 : IDE_LOGE("Make shared failed, message: %s", ex.what());
641 0 : return false;
642 0 : }
643 7 : auto err = memcpy_s(statsResult.get(), dumpChunk.bufLen, dumpChunk.dataBuf, dumpChunk.bufLen);
644 7 : IDE_CTRL_VALUE_FAILED(err == EOK, return false, "Filename string copy failed, err: %d", err);
645 :
646 7 : std::stringstream strStream;
647 7 : IDE_CTRL_VALUE_FAILED(GenerateFileData(strStream, filename, statsResult), return false, "Failed to export data.");
648 :
649 7 : if (AdxDumpProcess::Instance().IsRegistered()) { // dump data to mindspore and return
650 1 : return DumpDataToCallback(filename, strStream.str(), dumpChunk.offset, dumpChunk.flag);
651 : }
652 :
653 6 : if (JudgeRemoteFalg(filename)) {
654 1 : auto pos = filename.find_first_of(":");
655 1 : filename = filename.substr(pos + 1);
656 1 : filename = dumpPath_ + filename;
657 : } else {
658 5 : if (!FileUtils::IsAbsolutePath(filename)) {
659 5 : filename = dumpPath_ + filename;
660 : }
661 : }
662 :
663 : #if (OS_TYPE != LINUX)
664 : filename = FileUtils::ReplaceAll(filename, "/", "\\");
665 : #endif
666 :
667 6 : std::string dirName = FileUtils::GetFileDir(filename);
668 6 : if (!FileUtils::IsFileExist(dirName)) {
669 1 : IDE_CTRL_VALUE_FAILED(FileUtils::CreateDir(dirName) == IDE_DAEMON_NONE_ERROR, return false,
670 : "Create dir failed path: %s", dirName.c_str());
671 : }
672 5 : while (FileUtils::IsDiskFull(dirName, dumpChunk.bufLen)) {
673 1 : IDE_LOGE("Don't have enough free disk %u bytes", dumpChunk.bufLen);
674 1 : return false;
675 : }
676 :
677 4 : std::string realPath;
678 4 : IDE_CTRL_VALUE_FAILED(FileUtils::FileNameIsReal(filename, realPath) == IDE_DAEMON_OK, return false,
679 : "File name is not real: %s", filename.c_str());
680 :
681 3 : IDE_CTRL_VALUE_FAILED(FileUtils::WriteFile(filename, strStream.str().c_str(), strStream.str().size(),
682 : dumpChunk.offset) == IDE_DAEMON_NONE_ERROR, return false, "Failed to dump file %s to path %s",
683 : filename.c_str(), dirName.c_str());
684 2 : IDE_LOGI("Record dump stats data success: %s", filename.c_str());
685 2 : AppendFileName(filename);
686 2 : return true;
687 7 : }
688 : #endif
689 :
690 : /**
691 : * @brief record dump info to the file
692 : * @param [in] data : record data
693 : * @return
694 : * IDE_DAEMON_ERROR : falied
695 : * IDE_DAEMON_OK : success
696 : */
697 58 : void AdxDumpRecord::RecordDumpInfo()
698 : {
699 58 : IDE_RUN_LOGI("start dump thread, remote dump record temp path : %s.", dumpPath_.c_str());
700 58 : uint32_t chunkHeaderLen = static_cast<uint32_t>(sizeof(DumpChunk));
701 110 : while (hostDumpDataInfoQueue_ != nullptr && (dumpRecordFlag_ || !DumpDataQueueIsEmpty())) {
702 52 : HostDumpDataInfo data = {nullptr, 0};
703 52 : if (!hostDumpDataInfoQueue_->Pop(data)) {
704 46 : continue;
705 : }
706 :
707 6 : if (data.msg == nullptr) {
708 0 : continue;
709 : }
710 :
711 6 : SharedPtr<MsgProto> msgPtr = data.msg;
712 6 : IDE_CTRL_VALUE_FAILED_NODO(data.recvLen >= chunkHeaderLen, continue,
713 : "recvLen(%u) too small for DumpChunk header(%zu bytes)", data.recvLen, chunkHeaderLen);
714 :
715 6 : DumpChunk* dumpChunk = reinterpret_cast<DumpChunk*>(msgPtr->data);
716 :
717 6 : IDE_CTRL_VALUE_FAILED_NODO(dumpChunk->bufLen <= data.recvLen - chunkHeaderLen, continue,
718 : "bufLen(%u) exceeds actual data buffer size(%u bytes), fileName: %s",
719 : dumpChunk->bufLen, data.recvLen - chunkHeaderLen, dumpChunk->fileName);
720 :
721 6 : IDE_LOGI("Queue pop data success! filename: %s, offset: %" PRId64 ", bufLen: %u bytes, "
722 : "isLast: %u, flag: %d, remaining queue size: %u.",
723 : dumpChunk->fileName, dumpChunk->offset, dumpChunk->bufLen, dumpChunk->isLastChunk,
724 : dumpChunk->flag, hostDumpDataInfoQueue_->Size());
725 : #if !defined(ADUMP_SOC_HOST) || ADUMP_SOC_HOST == 1
726 6 : if (FileNameCheck(*dumpChunk)) {
727 1 : IDE_CTRL_VALUE_FAILED_NODO(StatsDataParsing(*dumpChunk), continue,
728 : "Failed to parse dump data with file name %s", dumpChunk->fileName);
729 1 : continue;
730 : }
731 : #endif
732 5 : if (AdxDumpProcess::Instance().IsRegistered()) {
733 3 : IDE_LOGI("mindspore session!");
734 : std::function<int32_t(const struct DumpChunk *, int32_t)> messageCallback =
735 3 : AdxDumpProcess::Instance().GetCallbackFun();
736 3 : if (!messageCallback) {
737 0 : IDE_LOGE("Registered messageCallback function is not callable,\
738 : drop this data packet, fileName:%s", dumpChunk->fileName);
739 0 : continue;
740 : }
741 3 : int32_t dumpChunkLen = sizeof(struct DumpChunk) + dumpChunk->bufLen;
742 3 : int32_t ret = messageCallback(dumpChunk, dumpChunkLen);
743 3 : if (ret != IDE_DAEMON_NONE_ERROR) {
744 0 : IDE_LOGE("failed to transmission dump data to mindspore. err = %d", ret);
745 : }
746 5 : } else if (!RecordDumpDataToDisk(*dumpChunk)) {
747 2 : IDE_LOGE("failed to record dump data to disk.");
748 : }
749 53 : }
750 58 : IDE_LOGI("exit record file thread");
751 58 : }
752 :
753 : /**
754 : * @brief record dump info to the file
755 : * @param [in] data : record data
756 : * @return
757 : * IDE_DAEMON_ERROR : falied
758 : * IDE_DAEMON_OK : success
759 : */
760 8 : bool AdxDumpRecord::RecordDumpDataToQueue(HostDumpDataInfo &info)
761 : {
762 8 : if (hostDumpDataInfoQueue_ == nullptr) {
763 1 : IDE_LOGW("dump data queue is not initialized, drop this data packet.");
764 1 : return false;
765 : }
766 7 : if (hostDumpDataInfoQueue_->IsFull()) {
767 2 : const std::string tipFull = "Memory usage exceeds 85%, the dump data queue is full";
768 2 : const std::string tipReduce = "Please reduce model batches, images or dump layers";
769 1 : const std::string tipMemory = "Or clear the used memory or increase the maximum memory";
770 1 : IDE_LOGW("%s. %s. %s.", tipFull.c_str(), tipReduce.c_str(), tipMemory.c_str());
771 1 : return false;
772 1 : } else {
773 6 : if (!hostDumpDataInfoQueue_->Push(info)) {
774 0 : IDE_LOGW("dump data queue has quit, drop this data packet.");
775 0 : return false;
776 : }
777 6 : IDE_LOGI("Insert dump data to queue success, queue size: %u.", hostDumpDataInfoQueue_->Size());
778 : }
779 :
780 6 : return true;
781 : }
782 :
783 : /**
784 : * @brief get dump size from the record lists
785 : * @param [in] tag : tag of record lists
786 : * @return
787 : * true : success
788 : * false : false
789 : */
790 62 : bool AdxDumpRecord::DumpDataQueueIsEmpty() const
791 : {
792 62 : return hostDumpDataInfoQueue_ == nullptr || hostDumpDataInfoQueue_->IsEmpty();
793 : }
794 :
795 0 : void AdxDumpRecord::SetDumpPath(const std::string &dumpPath)
796 : {
797 0 : dumpPath_ = dumpPath;
798 0 : }
799 : }
|
