Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "host_executor_dumper.h"
12 :
13 : #include <utility>
14 : #include <aicore/launch_kernel/ai_core_launch_kernel.h>
15 : #include <anchor_utils.h>
16 : #include <tuning_utils.h>
17 : #include <dlog_pub.h>
18 : #include <anchor.h>
19 : #include <iostream>
20 : #include <string>
21 : #include <vector>
22 : #include <regex>
23 : #include "common/checker.h"
24 : #include "common/ge_inner_error_codes.h"
25 : #include "mmpa/mmpa_api.h"
26 : #include "common/debug/memory_dumper.h"
27 : #include "graph/utils/tensor_utils.h"
28 : #include "framework/common/string_util.h"
29 : #include "runtime/subscriber/global_dumper.h"
30 : #include "runtime/subscriber/built_in_subscriber_definitions.h"
31 : #include "framework/common/util.h"
32 : #include "common/aclrt_malloc_helper.h"
33 :
34 : namespace {
35 : constexpr int32_t kDecimal = 10;
36 : }
37 :
38 : namespace gert {
39 : std::mutex HostExecutorDumper::mutex_;
40 :
41 : HostExecutorDumper::HostExecutorDumper(const std::shared_ptr<const SubscriberExtendInfo> &extend_info)
42 : : ExecutorDumper(extend_info), extend_info_(std::move(extend_info)) {
43 : if ((extend_info_ == nullptr) || (extend_info_->exe_graph == nullptr) || (extend_info_->executor == nullptr)) {
44 : GELOGE(ge::FAILED, "Exe graph is nullptr, dumper will do nothing.");
45 : return;
46 : }
47 : if (IsEnable(DumpType::kHostDump)) {
48 : if (Init() != ge::SUCCESS) {
49 : GELOGE(ge::FAILED, "Init dumper failed, dumper does nothing.");
50 : return;
51 : }
52 : }
53 : }
54 :
55 : ge::Status HostExecutorDumper::GetDumpAddrFromChainAddrOnHost(const NodeDumpUnit &dump_unit, bool is_input,
56 : std::vector<uintptr_t> &dump_addrs) const {
57 : const auto &chain_addrs = is_input ? dump_unit.input_addrs : dump_unit.output_addrs;
58 : for (size_t i = 0UL; i < chain_addrs.size(); ++i) {
59 : if (chain_addrs[i] == nullptr) {
60 : dump_addrs.emplace_back(reinterpret_cast<uintptr_t>(chain_addrs[i]));
61 : continue;
62 : }
63 :
64 : const auto tensor_data = chain_addrs[i]->GetPointer<TensorData>();
65 : GE_ASSERT_NOTNULL(tensor_data);
66 : auto host_addr = tensor_data->GetAddr();
67 : dump_addrs.emplace_back(reinterpret_cast<uintptr_t>(host_addr));
68 : }
69 : return ge::SUCCESS;
70 : }
71 :
72 : void HostExecutorDumper::SetOpDescInfo(NodeDumpUnit &dump_unit, ge::OpDescPtr &op_desc, ge::OpDescInfo &op_desc_info,
73 : const std::vector<uintptr_t> &input_addrs,
74 : const std::vector<uintptr_t> &output_addrs) const {
75 : op_desc_info.op_name = dump_unit.node->GetName();
76 : op_desc_info.op_type = dump_unit.node->GetType();
77 : int64_t desc_size = 0;
78 : for (size_t in_index = 0UL; in_index < dump_unit.input_addrs.size(); ++in_index) {
79 : const auto &in_tensor_desc = op_desc->MutableInputDesc(in_index);
80 : if ((in_tensor_desc == nullptr) || (input_addrs[in_index] == 0U)) {
81 : continue;
82 : }
83 : op_desc_info.input_data_type.emplace_back(in_tensor_desc->GetDataType());
84 : op_desc_info.input_format.emplace_back(in_tensor_desc->GetFormat());
85 : GE_CHK_STATUS(ge::TensorUtils::GetTensorSizeInBytes(*in_tensor_desc, desc_size), "op %s input %zu",
86 : dump_unit.node->GetName().c_str(), in_index);
87 : op_desc_info.input_size.emplace_back(desc_size);
88 : op_desc_info.input_shape.emplace_back(in_tensor_desc->GetShape().GetDims());
89 : op_desc_info.input_addrs.emplace_back(reinterpret_cast<void *>(input_addrs[in_index]));
90 : }
91 :
92 : for (size_t out_index = 0UL; out_index < dump_unit.output_addrs.size(); ++out_index) {
93 : const auto &out_tensor_desc = op_desc->MutableOutputDesc(out_index);
94 : if ((out_tensor_desc == nullptr) || (output_addrs[out_index] == 0U)) {
95 : continue;
96 : }
97 : op_desc_info.output_data_type.emplace_back(out_tensor_desc->GetDataType());
98 : op_desc_info.output_format.emplace_back(out_tensor_desc->GetFormat());
99 : GE_CHK_STATUS(ge::TensorUtils::GetTensorSizeInBytes(*out_tensor_desc, desc_size), "op %s output %zu",
100 : dump_unit.node->GetName().c_str(), out_index);
101 : op_desc_info.output_size.emplace_back(desc_size);
102 : op_desc_info.output_shape.emplace_back(out_tensor_desc->GetShape().GetDims());
103 : op_desc_info.output_addrs.emplace_back(reinterpret_cast<void *>(output_addrs[out_index]));
104 : }
105 :
106 : const auto workspace_size = op_desc->GetWorkspaceBytes();
107 : for (size_t i = 0U; (i < workspace_size.size()) && (i < dump_unit.workspace_info.size()); ++i) {
108 : op_desc_info.space_addrs.emplace_back(reinterpret_cast<void *>(dump_unit.workspace_info[i].first));
109 : op_desc_info.workspace_bytes.emplace_back(workspace_size[i]);
110 : }
111 : }
112 :
113 11 : bool HostExecutorDumper::IsInDumpStep(const int64_t step_id, const std::string &dump_step) const {
114 : if (!dump_step.empty()) {
115 : const auto step = step_set_.find(step_id);
116 : if (step != step_set_.end()) {
117 : return true;
118 : }
119 : for (size_t i = 0UL; i < step_range_.size(); i++) {
120 : if ((step_id >= step_range_[i].first) && (step_id <= step_range_[i].second)) {
121 : return true;
122 : }
123 : }
124 : return false;
125 : }
126 : return true;
127 : }
128 :
129 : ge::Status HostExecutorDumper::DoHostDataDump(NodeDumpUnit &dump_unit, const ge::DumpProperties &dump_properties) {
130 : // todo default session id
131 : const auto name = dump_unit.node->GetName();
132 : const auto type = dump_unit.node->GetType();
133 : GELOGI("[Dumper] Start to dump, op name: %s, type: %s", name.c_str(), type.c_str());
134 : // todo change with DoDataDump
135 : if (type == "Identity" || type == "MemcpyAsync") {
136 : return ge::SUCCESS;
137 : }
138 :
139 : void *step_id = nullptr;
140 : GE_ASSERT_ACL_OK(ge::AclrtMalloc(&step_id, sizeof(uint64_t), RT_MEMORY_HBM, GE_MODULE_NAME_U16));
141 : const auto callback = [&dump_unit, &step_id]() {
142 : GE_CHK_RT(aclrtFree(step_id));
143 : dump_unit.Clear();
144 : };
145 : GE_MAKE_GUARD(dump_release, callback);
146 :
147 : // todo single_op and graph have different switch, which will be normalized
148 : const std::string model_name = extend_info_->model_name;
149 : if (!dump_properties.IsOpDebugOpen() &&
150 : !dump_properties.IsLayerNeedDump(model_name, extend_info_->model_data.om_name, name) &&
151 : !dump_properties.IsSingleOpNeedDump()) {
152 : GELOGI("[Dumper] [%s] is not in dump list, no need to dump", name.c_str());
153 : return ge::SUCCESS;
154 : }
155 :
156 : // todo model_id does not matter
157 : GELOGD("[Data][Dumper]model name[%s], model id[%u].", model_name.c_str(), extend_info_->model_id);
158 : const auto op_desc = dump_unit.node->GetOpDescBarePtr();
159 : GE_ASSERT_NOTNULL(op_desc);
160 : ge::OpDescPtr op_desc_dump = nullptr;
161 : GE_MAKE_SHARED(op_desc_dump = std::make_shared<ge::OpDesc>(*op_desc), return ge::FAILED);
162 : dump_unit.UpdateInputShapes(op_desc_dump);
163 : dump_unit.UpdateOutputShapes(op_desc_dump);
164 : std::vector<uintptr_t> input_addrs;
165 : if (GetDumpAddrFromChainAddrOnHost(dump_unit, true, input_addrs) != ge::SUCCESS) {
166 : // skip and continue dump other nodes
167 : return ge::SUCCESS;
168 : }
169 : std::vector<uintptr_t> output_addrs;
170 : if (GetDumpAddrFromChainAddrOnHost(dump_unit, false, output_addrs) != ge::SUCCESS) {
171 : // skip and continue dump other nodes
172 : return ge::SUCCESS;
173 : }
174 :
175 : if ((output_addrs.size() != op_desc->GetAllOutputsDescSize()) ||
176 : (input_addrs.size() != op_desc->GetAllInputsSize())) {
177 : GELOGW(
178 : "[Dumper] Node %s input addr or output addr size is invalid, input addr size is %zu, output addr size is %zu, "
179 : "op desc input size is %zu, output size is %zu.",
180 : name.c_str(), input_addrs.size(), output_addrs.size(), op_desc->GetInputsSize(), op_desc->GetOutputsSize());
181 : // skip and continue dump
182 : return ge::SUCCESS;
183 : }
184 :
185 : ge::OpDescInfo op_desc_info;
186 : ge::ExceptionDumper exception_dumper;
187 : SetOpDescInfo(dump_unit, op_desc_dump, op_desc_info, input_addrs, output_addrs);
188 : auto iteration_num = GetIterationNum();
189 : GE_ASSERT_RT_OK(aclrtMemcpy(step_id, sizeof(uint64_t), &iteration_num, sizeof(uint64_t), ACL_MEMCPY_HOST_TO_DEVICE));
190 : GELOGD("[Dumper] Is single op %d", static_cast<int32_t>(dump_properties.IsSingleOpNeedDump()));
191 : const auto &dump_step = dump_properties.GetDumpStep();
192 : if (!IsInDumpStep(static_cast<int64_t>(iteration_num), dump_step)) {
193 : GELOGW("[Dumper] The current step is %lld, not in the dump step", iteration_num);
194 : return ge::SUCCESS;
195 : }
196 : const auto dump_path = dump_properties.GetDumpPath();
197 : const auto worker_id = dump_properties.GetDumpWorkerId();
198 : GELOGD("[Dumper] Get worker id succeed, value is %s.", worker_id.c_str());
199 : std::string file_name = model_name;
200 : replace(file_name.begin(), file_name.end(), '/', '_');
201 : replace(file_name.begin(), file_name.end(), '\\', '_');
202 : replace(file_name.begin(), file_name.end(), '.', '_');
203 : std::string file_path = "/";
204 : if (!dump_path.empty()) {
205 : const std::lock_guard<std::mutex> lock(mutex_);
206 : // multiprocess shared directory creates first to avoid fuilure of creating full path
207 : const std::string shared_path = dump_path + "/0/";
208 : (void)ge::CreateDirectory(shared_path);
209 : file_path = shared_path + "pid" + std::to_string(mmGetPid()) + "/" + file_name + "/" +
210 : std::to_string(extend_info_->model_id) + "/" + worker_id + "/" + std::to_string(iteration_num) + "/";
211 : const int32_t directory_ret = ge::CreateDirectory(file_path);
212 : if (directory_ret != 0) {
213 : GELOGW("Cannot create directory[%s].", file_path.c_str());
214 : return ge::SUCCESS;
215 : }
216 : }
217 : GE_CHK_STATUS_RET(exception_dumper.DumpNodeInfo(op_desc_info, file_path, false, false, dump_properties),
218 : "Dump op %s failed", name.c_str());
219 : GELOGI("[Dumper] Launch dump op:%s Successfully", name.c_str());
220 : return ge::SUCCESS;
221 : }
222 :
223 : ge::Status HostExecutorDumper::OnUpdateDumpUnitForHostDump(const Node &node) {
224 : for (auto &dump_unit : kernel_idxes_to_dump_units_[node.node_id]) {
225 : ++dump_unit->cur_update_count;
226 : GE_ASSERT_SUCCESS(SaveWorkSpaceAddrForAiCpuLaunchCCNode(node));
227 :
228 : if (dump_unit->cur_update_count == dump_unit->total_update_count) {
229 : const auto &dump_properties = ge::DumpManager::GetInstance().GetDumpProperties(GetSessionId());
230 : GE_ASSERT_SUCCESS(DoHostDataDump(*dump_unit, dump_properties));
231 : }
232 : }
233 : return ge::SUCCESS;
234 : }
235 :
236 : void HostExecutorDumper::ParseDumpStep() {
237 : if (is_parsed_) {
238 : return;
239 : }
240 : const auto &dump_properties = ge::DumpManager::GetInstance().GetDumpProperties(GetSessionId());
241 : const auto &dump_step = dump_properties.GetDumpStep();
242 : if (!dump_step.empty()) {
243 : auto match_vecs = ge::StringUtils::Split(dump_step, '|');
244 : const std::regex pattern(R"(\d{1,})");
245 : std::smatch result;
246 : for (const auto &match_vec : match_vecs) {
247 : if (regex_match(match_vec, result, pattern)) {
248 : step_set_.insert(static_cast<int64_t>(std::strtol(match_vec.c_str(), nullptr, kDecimal)));
249 : GELOGI("[HostDumper] Insert one step:%d", std::strtol(match_vec.c_str(), nullptr, kDecimal));
250 : } else {
251 : auto vec_split = ge::StringUtils::Split(match_vec, '-');
252 : const auto lower_range = static_cast<int64_t>(std::strtol(vec_split[0UL].c_str(), nullptr, kDecimal));
253 : const auto higher_range = static_cast<int64_t>(std::strtol(vec_split[1UL].c_str(), nullptr, kDecimal));
254 : step_range_.emplace_back(lower_range, higher_range);
255 : GELOGI("[HostDumper] Insert step range from %d to %d", lower_range, higher_range);
256 : }
257 : }
258 : }
259 : is_parsed_ = true;
260 : return;
261 : }
262 :
263 : ge::Status HostExecutorDumper::HostDataDump(const Node *node, ExecutorEvent event) {
264 : if (event == ExecutorEvent::kModelStart) {
265 : GE_ASSERT_SUCCESS(Init());
266 : SaveSessionId();
267 : ParseDumpStep();
268 : }
269 : if (event == ExecutorEvent::kExecuteEnd) {
270 : GE_ASSERT_SUCCESS(OnUpdateDumpUnitForHostDump(*node));
271 : }
272 : if (event == ExecutorEvent::kModelEnd) {
273 : CountIterNum();
274 : }
275 : return ge::SUCCESS;
276 : }
277 :
278 : void HostExecutorDumper::OnExecuteEvent(int32_t sub_exe_graph_type, HostExecutorDumper *dumper, ExecutorEvent event,
279 : const void *node, KernelStatus result) {
280 : (void)sub_exe_graph_type;
281 : (void)result;
282 : if (dumper == nullptr) {
283 : return;
284 : }
285 : if (dumper->IsEnable(DumpType::kHostDump)) {
286 : (void)dumper->HostDataDump(static_cast<const Node *>(node), event);
287 : return;
288 : }
289 : }
290 : } // namespace gert
|
