Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 : #include "aicpusd_monitor.h"
11 :
12 : #include <mutex>
13 : #include <cstdlib>
14 : #include <string>
15 : #include <unistd.h>
16 : #include <cstring>
17 : #include <cerrno>
18 : #include "securec.h"
19 : #include "aicpusd_common.h"
20 : #include "profiling_adp.h"
21 : #include "aicpusd_drv_manager.h"
22 : #include "tsd.h"
23 : #include "aicpu_context.h"
24 : #include "aicpu_pulse.h"
25 : #include "aicpusd_info.h"
26 : #include "status.h"
27 : #include "feature_ctrl.h"
28 :
29 : namespace {
30 : // aicpu task timeout
31 : const uint64_t AICPU_TASK_TIMEOUT = 28LU;
32 : const uint64_t AICPU_TASK_TIMEOUT_LONG = 60UL;
33 : } // namespace
34 :
35 : namespace AicpuSchedule {
36 : /**
37 : * @ingroup AicpuMonitor
38 : * @brief it is used to construct a object of AicpuMonitor.
39 : * using default value to construct
40 : */
41 13 : AicpuMonitor::AicpuMonitor()
42 13 : : deviceId_(0U),
43 13 : taskTimeoutFlag_(false),
44 13 : taskInfo_(nullptr),
45 13 : done_(false),
46 13 : taskTimeout_(UINT64_MAX),
47 13 : taskTimeoutTick_(UINT64_MAX),
48 13 : taskTimer_(nullptr),
49 13 : running_(false),
50 13 : aicpuCoreNum_(0U),
51 26 : online_(false)
52 13 : {}
53 :
54 : /**
55 : * @ingroup AicpuMonitor
56 : * @brief it is used to destructor a object of AicpuMonitor.
57 : * it is neccessary to set flag false to make sure thread to be stopped
58 : */
59 13 : AicpuMonitor::~AicpuMonitor()
60 : {
61 13 : if (!done_) {
62 11 : Stop();
63 : }
64 13 : }
65 :
66 55 : AicpuMonitor& AicpuMonitor::GetInstance()
67 : {
68 55 : static AicpuMonitor instance;
69 55 : return instance;
70 : }
71 :
72 5 : int32_t AicpuMonitor::InitAicpuMonitor(const uint32_t deviceId, const bool online)
73 : {
74 5 : aicpusd_info("Begin to init aicpu monitor");
75 5 : online_ = online;
76 5 : if (!online_) {
77 1 : aicpusd_info("End to init aicpu monitor, offline mode");
78 1 : return AICPU_SCHEDULE_OK;
79 : }
80 4 : deviceId_ = deviceId;
81 4 : aicpuCoreNum_ = AicpuSchedule::AicpuDrvManager::GetInstance().GetAicpuNum();
82 4 : if (aicpuCoreNum_ != 0U) {
83 4 : taskInfo_.reset(new (std::nothrow) TaskInfoForMonitor[aicpuCoreNum_]);
84 4 : if (taskInfo_ == nullptr) {
85 0 : aicpusd_err("malloc task info memory for monitor failed");
86 0 : return AICPU_SCHEDULE_ERROR_COMMON_ERROR;
87 : }
88 8 : for (size_t i = 0UL; i < aicpuCoreNum_; i++) {
89 4 : taskInfo_[i] = {UINT64_MAX, UINT64_MAX, UINT32_MAX, false};
90 : }
91 : }
92 :
93 4 : const int32_t ret = SetTaskTimeoutFlag();
94 4 : if (ret != AICPU_SCHEDULE_OK) {
95 1 : aicpusd_err("set task timeout flag failed, ret[%d]", ret);
96 1 : return ret;
97 : }
98 :
99 3 : if (taskTimeoutFlag_ && (aicpuCoreNum_ != 0U)) {
100 6 : taskTimer_.reset(new (std::nothrow) TaskTimer[aicpuCoreNum_]);
101 3 : if (taskTimer_ == nullptr) {
102 0 : aicpusd_err("malloc memory for task timer failed");
103 0 : return AICPU_SCHEDULE_ERROR_COMMON_ERROR;
104 : }
105 : }
106 :
107 3 : aicpusd_run_info(
108 : "Init aicpu monitor success, taskTimeout=%lus, tickFreq=%lu", taskTimeout_, aicpu::GetSystemTickFreq());
109 3 : return AICPU_SCHEDULE_OK;
110 : }
111 :
112 2 : void AicpuMonitor::SendKillMsgToTsd() const
113 : {
114 2 : aicpusd_run_info(
115 : "dev[%u] send msg to tsdaemon, tsdaemon will kill aicpu-custom-sd process[%u]", deviceId_,
116 : static_cast<uint32_t>(getpid()));
117 6 : const int32_t ret = TsdDestroy(
118 2 : deviceId_, TSD_CUSTOM_COMPUTE, static_cast<uint32_t>(AicpuDrvManager::GetInstance().GetHostPid()),
119 2 : AicpuDrvManager::GetInstance().GetVfId());
120 2 : if (ret != static_cast<int32_t>(tsd::TSD_OK)) {
121 1 : aicpusd_err("dev[%u] send abnormal msg to tsdaemon failed, ret[%d]", deviceId_, ret);
122 : }
123 2 : }
124 :
125 10 : void AicpuMonitor::SetTaskInfo(const uint64_t taskIndex, const TaskInfoForMonitor& taskInfo)
126 : {
127 10 : if ((taskIndex < aicpuCoreNum_) && online_) {
128 2 : taskInfo_[taskIndex] = taskInfo;
129 : }
130 10 : }
131 :
132 8 : int32_t AicpuMonitor::SetTaskTimeoutFlag()
133 : {
134 8 : taskTimeout_ = FeatureCtrl::IsDoubleDieProduct() ? AICPU_TASK_TIMEOUT_LONG : AICPU_TASK_TIMEOUT;
135 8 : taskTimeoutTick_ = taskTimeout_ * aicpu::GetSystemTickFreq();
136 8 : taskTimeoutFlag_ = true;
137 8 : return AICPU_SCHEDULE_OK;
138 : }
139 :
140 10 : void AicpuMonitor::SetTaskStartTime(const uint64_t taskIndex)
141 : {
142 10 : if ((!IsMonitorClosed()) && (taskTimeoutFlag_) && (online_)) {
143 1 : taskTimer_[taskIndex].SetStartTick(aicpu::GetSystemTick());
144 1 : taskTimer_[taskIndex].SetRunFlag(true);
145 : }
146 10 : }
147 :
148 10 : void AicpuMonitor::SetTaskEndTime(const uint64_t taskIndex)
149 : {
150 10 : if ((taskTimeoutFlag_) && (online_)) {
151 2 : taskTimer_[taskIndex].SetRunFlag(false);
152 2 : taskInfo_[taskIndex].isHwts = false;
153 : }
154 10 : }
155 :
156 2 : void AicpuMonitor::Work(AicpuMonitor* const monitor)
157 : {
158 2 : std::once_flag flag;
159 3 : while ((monitor != nullptr) && (!(monitor->done_))) {
160 3 : if (monitor->online_) {
161 : // check and handle task timeout, for aicpu task of ts stream and aicpu stream
162 0 : monitor->HandleTaskTimeout();
163 : }
164 : // call pulseNotifyFuncMap
165 3 : AicpuPulseNotify();
166 3 : std::call_once(flag, [&]() {
167 2 : monitor->running_ = true;
168 2 : if (sem_post(&monitor->sem_) != 0) {
169 0 : aicpusd_err("sem post failed, %s", strerror(errno));
170 0 : return;
171 : }
172 : });
173 3 : (void)sleep(1U);
174 : }
175 0 : }
176 :
177 4 : int32_t AicpuMonitor::Run()
178 : {
179 4 : if (done_) {
180 1 : aicpusd_err("Fail to run monitor for it has been stopped");
181 1 : return AICPU_SCHEDULE_ERROR_COMMON_ERROR;
182 : }
183 3 : int32_t semRet = sem_init(&sem_, 0, 0U);
184 3 : if (semRet == -1) {
185 1 : aicpusd_err("sem init failed, %s.", strerror(errno));
186 1 : return AICPU_SCHEDULE_ERROR_COMMON_ERROR;
187 : }
188 : try {
189 2 : std::thread th(&AicpuMonitor::Work, this);
190 2 : th.detach();
191 2 : } catch (std::exception& e) {
192 0 : (void)sem_destroy(&sem_);
193 0 : aicpusd_err("create aicpu monitor thread object failed, %s", e.what());
194 0 : return AICPU_SCHEDULE_ERROR_COMMON_ERROR;
195 0 : }
196 :
197 2 : semRet = sem_wait(&sem_);
198 2 : if (semRet == -1) {
199 0 : (void)sem_destroy(&sem_);
200 0 : aicpusd_err("sem wait failed, %s.", strerror(errno));
201 0 : return AICPU_SCHEDULE_ERROR_COMMON_ERROR;
202 : }
203 2 : (void)sem_destroy(&sem_);
204 2 : if (!running_) {
205 0 : aicpusd_err("create aicpu monitor thread failed");
206 0 : return AICPU_SCHEDULE_ERROR_COMMON_ERROR;
207 : }
208 2 : aicpusd_info("aicpu monitor thread running");
209 :
210 2 : return AICPU_SCHEDULE_OK;
211 : }
212 :
213 30 : void AicpuMonitor::Stop()
214 : {
215 30 : const std::unique_lock<std::mutex> stopLock(mutex_);
216 30 : done_ = true;
217 30 : }
218 :
219 3 : void AicpuMonitor::HandleTaskTimeout()
220 : {
221 3 : if (!taskTimeoutFlag_ || IsMonitorClosed()) {
222 2 : return;
223 : }
224 :
225 1 : const uint64_t nowTick = aicpu::GetSystemTick();
226 1 : for (size_t coreIndex = 0UL; coreIndex < aicpuCoreNum_; ++coreIndex) {
227 1 : TaskTimer taskTimer;
228 1 : taskTimer.SetRunFlag(taskTimer_[coreIndex].GetRunFlag());
229 1 : taskTimer.SetStartTick(taskTimer_[coreIndex].GetStartTick());
230 2 : if (taskTimer.GetRunFlag() && (nowTick > taskTimer.GetStartTick()) &&
231 1 : ((nowTick - taskTimer.GetStartTick()) >= taskTimeoutTick_)) {
232 : // handle task timeout
233 1 : std::string opname;
234 1 : (void)aicpu::GetOpname(static_cast<uint32_t>(coreIndex), opname);
235 1 : std::ostringstream oss;
236 1 : oss << "Send timeout to tsdaemon, tsdaemon will kill aicpu-custom-sd process, thread index[" << coreIndex
237 1 : << "], op name[" << opname << "]";
238 1 : if (taskInfo_[coreIndex].isHwts) {
239 0 : oss << ", " << taskInfo_[coreIndex].DebugString();
240 : }
241 1 : aicpusd_err("%s.", oss.str().c_str());
242 1 : SendKillMsgToTsd();
243 1 : break;
244 1 : }
245 : }
246 : }
247 : } // namespace AicpuSchedule
|
