Coverage for /opt/cloud/slavespace/usr1/096471637100f3de0fcfc01072822a80/ut/src/asys/collect/stacktrace/stacktrace_collect.py: 71%
204 statements
« prev ^ index » next coverage.py v7.15.3, created at 2026-08-11 18:39 +0800
1#!/usr/bin/env python3
2# -*- coding: utf-8 -*-
3# ----------------------------------------------------------------------------
4# Copyright (c) 2025 Huawei Technologies Co., Ltd.
5#
6# Licensed under the Apache License, Version 2.0 (the "License");
7# you may not use this file except in compliance with the License.
8# You may obtain a copy of the License at
9#
10# http://www.apache.org/licenses/LICENSE-2.0
11#
12# Unless required by applicable law or agreed to in writing, software
13# distributed under the License is distributed on an "AS IS" BASIS,
14# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15# See the License for the specific language governing permissions and
16# limitations under the License.
17# ----------------------------------------------------------------------------
19import os
20import re
21import time
23from common import get_cann_log_path
24from common import log_error, popen_run_cmd, log_warning, log_info
25from common import FileOperate as f
26from common.file_operate import COPY_MODE
27from common.const import ATRACE_LOG_NAME, RetCode, CHECK_BIN_MAX_TIMEOUT, CHECK_BIN_DEFAULT_TIMEOUT
28from params import ParamDict
29from collect.stacktrace import AscendTraceDll
30from drv import EnvVarName
32EVERY_ROUND_TIME = 0.5
35class AsysStackTrace(AscendTraceDll):
36 """
37 Send signal to export stackcore
38 """
39 def __init__(self):
40 super(AsysStackTrace, self).__init__()
41 self.run_mode = ParamDict().get_arg("run_mode")
42 self.remote_id = ParamDict().get_arg("remote")
43 self.is_all_task = ParamDict().get_arg("all")
44 self.quiet = ParamDict().get_arg("quiet")
45 self.timeout = ParamDict().get_arg("timeout")
46 self.output = ParamDict().asys_output_timestamp_dir
47 self.trace_work_path = ""
49 def _get_target_work_path(self):
50 target_env_file = os.path.join("/proc", str(self.remote_id), "environ")
51 try:
52 with open(target_env_file, "r") as target_env:
53 env_content = target_env.read()
54 env_list = env_content.split('\0')
55 env_name = "ASCEND_WORK_PATH"
56 for env in env_list:
57 if not env:
58 continue
60 env_info = env.split("=", 1)
61 if len(env_info) >= 2 and env_info[0] == env_name:
62 return env_info[1]
63 return None
64 except PermissionError:
65 log_warning(f"permission denied: cannot read env of process {self.remote_id}.")
66 return None
67 except FileNotFoundError:
68 log_warning(f"process {self.remote_id} does not exist: {target_env_file}.")
69 return None
70 except Exception as e:
71 log_warning(f"failed to get env for process {self.remote_id}：{str(e)}.")
72 return None
74 def _set_trace_work_path(self):
75 asys_env_var = EnvVarName()
76 target_work_path = self._get_target_work_path()
77 if target_work_path:
78 self.trace_work_path = os.path.join(target_work_path, ATRACE_LOG_NAME)
79 log_info(f"bin file generate path is {os.path.abspath(self.trace_work_path)}, "
80 f"get from environment variables of process {self.remote_id}.")
81 else:
82 self.trace_work_path = os.path.join(asys_env_var.home_path, "ascend", ATRACE_LOG_NAME)
83 log_info(f"bin file generate path is {os.path.abspath(self.trace_work_path)}, "
84 f"get from default path.")
85 return
87 def _get_bin_file_path(self, all_exists_bin):
88 for path, _, files in os.walk(os.path.abspath(self.trace_work_path)):
89 for file in files:
90 if not (file.startswith(f"stackcore_tracer_35_{self.remote_id}_") and file.endswith(".bin")):
91 continue
92 bin_file_path = os.path.join(path, file)
93 if bin_file_path in all_exists_bin:
94 continue
95 return bin_file_path
97 def _get_exists_bin_file_num(self):
98 cmd = f"ls -lt {os.path.abspath(self.trace_work_path)}/trace_*/stackcore_event_{self.remote_id}_*/" \
99 f"stackcore_tracer_35_{self.remote_id}_*.bin | wc -l"
100 ret = popen_run_cmd(cmd).replace("\n", "")
101 if not ret.isdigit():
102 return 0
103 return int(ret)
105 def _get_last_bin_file_name(self):
106 cmd = f"ls -lt {os.path.abspath(self.trace_work_path)}/trace_*/stackcore_event_{self.remote_id}_*/" \
107 f"stackcore_tracer_35_{self.remote_id}_*.bin | head -n 1 | awk \'{{print $9}}\'"
108 return popen_run_cmd(cmd).replace("\n", "")
110 def _wait_bin_file_generate(self, exists_bin_file_num):
111 bin_file_name = None
112 for _ in range(int(self.timeout // EVERY_ROUND_TIME)): # 20 * 0.5 = 10s
113 if not bin_file_name:
114 current_bin_file_num = self._get_exists_bin_file_num()
115 if current_bin_file_num == exists_bin_file_num:
116 time.sleep(EVERY_ROUND_TIME)
117 continue
118 if current_bin_file_num > exists_bin_file_num:
119 bin_file_name = self._get_last_bin_file_name()
120 log_info("bin file generated, awaiting stack trace completion.")
121 continue
123 if popen_run_cmd(f"lsof {bin_file_name}"):
124 time.sleep(EVERY_ROUND_TIME)
125 continue
126 return bin_file_name
127 log_error(f"get the stackcore bin file in path {os.path.abspath(self.trace_work_path)} timeout.")
128 return None
130 def _check_other_param(self):
131 task_dir = ParamDict().get_arg("task_dir")
132 tar = ParamDict().get_arg("tar")
133 if task_dir or tar:
134 log_error("'--task_dir', and '--tar' can be used only when '-r' is not used.")
135 return False
136 if isinstance(self.timeout, int) and not isinstance(self.timeout, bool):
137 if self.timeout <= 0 or self.timeout > CHECK_BIN_MAX_TIMEOUT:
138 log_error("The value of timeout must in the range [1,60]")
139 return False
140 else:
141 self.timeout = CHECK_BIN_DEFAULT_TIMEOUT
142 return True
144 def _check_remote_id_validity(self):
145 if self.remote_id < 2:
146 log_error(f'The value of "--remote" must be greater than 1, input: {self.remote_id}.')
147 return False
149 try:
150 os.kill(self.remote_id, 0)
151 except Exception:
152 log_error(f'No such process, id: {self.remote_id}.')
153 return False
155 # check remote pid ?
156 cmd = f"ps -p {self.remote_id}"
157 ret = popen_run_cmd(cmd)[:-1].split("\n")
158 if len(ret) != 2:
159 log_error("The remote parameter must be set to the PID of the process.")
160 return False
161 return True
163 def _get_all_tid_of_process(self, current_pid):
164 cmd = fr"ps -efT | grep ' {self.remote_id} ' | grep -v {current_pid} | awk '{{print $2}}' | xargs ps -Lf \
165 | awk '{{print $4}}'"
166 ret = popen_run_cmd(cmd).split("\n")
167 ret = [i for i in ret if i.isdigit()]
168 if len(ret) < 2:
169 log_error(f'Get pid failed by remote: {self.remote_id}.')
170 return []
171 return ret
173 @staticmethod
174 def _get_other_stacktrace_remote_id(current_pid):
175 all_remote_id = []
176 cmd = rf"ps -ef | grep -E asys[\.py]{{0\,3}}\ collect | grep stacktrace | grep -v ' {current_pid} '"
177 ret = popen_run_cmd(cmd).split("\n")
178 ret = [i for i in ret if i]
179 if not ret:
180 return all_remote_id
182 p_pid = os.getppid()
183 for process in ret:
184 process_info_list = [i for i in process.split(" ") if i]
185 _pid = process_info_list[1]
186 # exclude current process ppid is other process pid
187 if _pid.isdigit() and int(_pid) == p_pid:
188 continue
189 _remote_id = re.search(r" --remote[ =](\d+)", process)
190 if _remote_id:
191 all_remote_id.append(_remote_id.group(1))
192 return all_remote_id
194 def _check_collect_stacktrace_parallel(self):
195 current_pid = os.getpid()
196 all_remote_id = self._get_other_stacktrace_remote_id(current_pid)
197 if not all_remote_id:
198 return True
199 # other running remote_id contains the current remote_id.
200 if str(self.remote_id) in all_remote_id:
201 return False
203 all_tid_of_process = self._get_all_tid_of_process(current_pid)
204 # abnormal state
205 if not all_tid_of_process:
206 return False
208 all_tid_remote_id = all_remote_id + all_tid_of_process
209 # tid contained in the current remote_id is running
210 if len(all_tid_remote_id) > len(set(all_tid_remote_id)):
211 return False
212 return True
214 @staticmethod
215 def _clear_dfx_log(folder_path):
216 for file in os.listdir(folder_path):
217 if file.endswith(".log") and file.startswith("stackcore_tracer_35_"):
218 log_path = os.path.join(folder_path, file)
219 try:
220 os.remove(log_path)
221 except OSError as e:
222 continue
224 def run(self):
225 """
226 send signals to export stackcore files.
227 """
228 f.remove_dir(self.output)
229 param_ret = self._check_other_param()
230 if not param_ret:
231 return False
233 if self.remote_id is False or not self.is_all_task:
234 log_error('"-r=stacktrace" must be used together with "--remote" and "--all".')
235 return False
237 if self.trace_dll == RetCode.FAILED:
238 return False
240 if not self._check_remote_id_validity():
241 return False
242 log_warning(f"This command sends signal 35 to the process:{self.remote_id}. "
243 "If the process is executed to disable signal receiving through the environment variable "
244 f"ASCEND_COREDUMP_SIGNAL=none, the process:{self.remote_id} will be killed. ")
245 if not self.quiet:
246 log_warning("Are you sure that signal reception is not disabled? (Y/N)")
247 if input().upper() != "Y":
248 return True
250 self._set_trace_work_path()
252 if not self._check_collect_stacktrace_parallel():
253 log_error('Collect stacktrace not support Parallelism.')
254 return False
256 exists_bin_num = self._get_exists_bin_file_num()
257 signal_ret = self.send_signal_to_pid(self.is_all_task, self.remote_id)
258 if not signal_ret:
259 return False
261 bin_file_path = self._wait_bin_file_generate(exists_bin_num)
262 if not bin_file_path:
263 return False
265 parse_ret = self.parse_stackcore_bin_to_txt(bin_file_path)
266 if not parse_ret:
267 return False
269 folder_path = os.path.dirname(bin_file_path)
270 self._clear_dfx_log(folder_path)
271 ret = f.collect_dir(folder_path, self.output, COPY_MODE)
272 if not ret:
273 log_warning(f"Copy output file from {folder_path} to {self.output} failed.")
275 log_info(f"Stacktrace output directory: {self.output}")
276 return True