Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 : #include "aicpusd_monitor.h"
11 :
12 : #include <mutex>
13 : #include <cstdlib>
14 : #include <string>
15 : #include <unistd.h>
16 : #include <cstring>
17 : #include <cerrno>
18 : #include "securec.h"
19 : #include "aicpusd_common.h"
20 : #include "profiling_adp.h"
21 : #include "aicpusd_drv_manager.h"
22 : #include "tsd.h"
23 : #include "aicpu_context.h"
24 : #include "aicpu_pulse.h"
25 : #include "aicpusd_info.h"
26 : #include "status.h"
27 : #include "feature_ctrl.h"
28 : #include "aicpusd_worker.h"
29 :
30 : namespace {
31 : // aicpu task timeout
32 : const uint64_t AICPU_TASK_TIMEOUT = 28LU;
33 : const uint64_t AICPU_TASK_TIMEOUT_LONG = 60UL;
34 : } // namespace
35 :
36 : namespace AicpuSchedule {
37 : /**
38 : * @ingroup AicpuMonitor
39 : * @brief it is used to construct a object of AicpuMonitor.
40 : * using default value to construct
41 : */
42 15 : AicpuMonitor::AicpuMonitor()
43 15 : : deviceId_(0U),
44 15 : taskTimeoutFlag_(false),
45 15 : taskInfo_(nullptr),
46 15 : done_(false),
47 15 : taskTimeout_(UINT64_MAX),
48 15 : taskTimeoutTick_(UINT64_MAX),
49 15 : taskTimer_(nullptr),
50 15 : running_(false),
51 15 : aicpuCoreNum_(0U),
52 30 : online_(false)
53 15 : {}
54 :
55 : /**
56 : * @ingroup AicpuMonitor
57 : * @brief it is used to destructor a object of AicpuMonitor.
58 : * it is neccessary to set flag false to make sure thread to be stopped
59 : */
60 15 : AicpuMonitor::~AicpuMonitor()
61 : {
62 15 : if (!done_) {
63 12 : Stop();
64 : }
65 15 : }
66 :
67 54 : AicpuMonitor& AicpuMonitor::GetInstance()
68 : {
69 54 : static AicpuMonitor instance;
70 54 : return instance;
71 : }
72 :
73 7 : int32_t AicpuMonitor::InitAicpuMonitor(const uint32_t deviceId, const bool online)
74 : {
75 7 : aicpusd_info("Begin to init aicpu monitor");
76 7 : online_ = online;
77 7 : if (!online_) {
78 1 : aicpusd_info("End to init aicpu monitor, offline mode");
79 1 : return AICPU_SCHEDULE_OK;
80 : }
81 6 : deviceId_ = deviceId;
82 6 : aicpuCoreNum_ = static_cast<uint32_t>(AicpuSchedule::ThreadPool::GetWorkerNum());
83 6 : if (aicpuCoreNum_ != 0U) {
84 6 : taskInfo_.reset(new (std::nothrow) TaskInfoForMonitor[aicpuCoreNum_]);
85 6 : if (taskInfo_ == nullptr) {
86 0 : aicpusd_err("malloc task info memory for monitor failed");
87 0 : return AICPU_SCHEDULE_ERROR_COMMON_ERROR;
88 : }
89 16 : for (size_t i = 0UL; i < aicpuCoreNum_; i++) {
90 10 : taskInfo_[i] = {UINT64_MAX, UINT64_MAX, UINT32_MAX, false};
91 : }
92 : }
93 :
94 6 : const int32_t ret = SetTaskTimeoutFlag();
95 6 : if (ret != AICPU_SCHEDULE_OK) {
96 1 : aicpusd_err("set task timeout flag failed, ret[%d]", ret);
97 1 : return ret;
98 : }
99 :
100 5 : if (taskTimeoutFlag_ && (aicpuCoreNum_ != 0U)) {
101 14 : taskTimer_.reset(new (std::nothrow) TaskTimer[aicpuCoreNum_]);
102 5 : if (taskTimer_ == nullptr) {
103 0 : aicpusd_err("malloc memory for task timer failed");
104 0 : return AICPU_SCHEDULE_ERROR_COMMON_ERROR;
105 : }
106 : }
107 :
108 5 : aicpusd_run_info(
109 : "Init aicpu monitor success, taskTimeout=%lus, tickFreq=%lu", taskTimeout_, aicpu::GetSystemTickFreq());
110 5 : return AICPU_SCHEDULE_OK;
111 : }
112 :
113 2 : void AicpuMonitor::SendKillMsgToTsd() const
114 : {
115 2 : aicpusd_run_info(
116 : "dev[%u] send msg to tsdaemon, tsdaemon will kill aicpu-custom-sd process[%u]", deviceId_,
117 : static_cast<uint32_t>(getpid()));
118 6 : const int32_t ret = TsdDestroy(
119 2 : deviceId_, TSD_CUSTOM_COMPUTE, static_cast<uint32_t>(AicpuDrvManager::GetInstance().GetHostPid()),
120 2 : AicpuDrvManager::GetInstance().GetVfId());
121 2 : if (ret != static_cast<int32_t>(tsd::TSD_OK)) {
122 1 : aicpusd_err("dev[%u] send abnormal msg to tsdaemon failed, ret[%d]", deviceId_, ret);
123 : }
124 2 : }
125 :
126 10 : void AicpuMonitor::SetTaskInfo(const uint64_t taskIndex, const TaskInfoForMonitor& taskInfo)
127 : {
128 10 : if ((taskIndex < aicpuCoreNum_) && online_) {
129 2 : taskInfo_[taskIndex] = taskInfo;
130 : }
131 10 : }
132 :
133 10 : int32_t AicpuMonitor::SetTaskTimeoutFlag()
134 : {
135 10 : taskTimeout_ = FeatureCtrl::IsDoubleDieProduct() ? AICPU_TASK_TIMEOUT_LONG : AICPU_TASK_TIMEOUT;
136 10 : taskTimeoutTick_ = taskTimeout_ * aicpu::GetSystemTickFreq();
137 10 : taskTimeoutFlag_ = true;
138 10 : return AICPU_SCHEDULE_OK;
139 : }
140 :
141 10 : void AicpuMonitor::SetTaskStartTime(const uint64_t taskIndex)
142 : {
143 10 : if ((!IsMonitorClosed()) && (taskTimeoutFlag_) && (online_)) {
144 1 : taskTimer_[taskIndex].SetStartTick(aicpu::GetSystemTick());
145 1 : taskTimer_[taskIndex].SetRunFlag(true);
146 : }
147 10 : }
148 :
149 10 : void AicpuMonitor::SetTaskEndTime(const uint64_t taskIndex)
150 : {
151 10 : if ((taskTimeoutFlag_) && (online_)) {
152 2 : taskTimer_[taskIndex].SetRunFlag(false);
153 2 : taskInfo_[taskIndex].isHwts = false;
154 : }
155 10 : }
156 :
157 1 : void AicpuMonitor::Work(AicpuMonitor* const monitor)
158 : {
159 1 : std::once_flag flag;
160 2 : while ((monitor != nullptr) && (!(monitor->done_))) {
161 1 : if (monitor->online_) {
162 : // check and handle task timeout, for aicpu task of ts stream and aicpu stream
163 0 : monitor->HandleTaskTimeout();
164 : }
165 : // call pulseNotifyFuncMap
166 1 : AicpuPulseNotify();
167 1 : std::call_once(flag, [&]() {
168 1 : monitor->running_ = true;
169 1 : if (sem_post(&monitor->sem_) != 0) {
170 0 : aicpusd_err("sem post failed, %s", strerror(errno));
171 0 : return;
172 : }
173 : });
174 1 : (void)sleep(1U);
175 : }
176 1 : }
177 :
178 3 : int32_t AicpuMonitor::Run()
179 : {
180 3 : if (done_) {
181 1 : aicpusd_err("Fail to run monitor for it has been stopped");
182 1 : return AICPU_SCHEDULE_ERROR_COMMON_ERROR;
183 : }
184 2 : int32_t semRet = sem_init(&sem_, 0, 0U);
185 2 : if (semRet == -1) {
186 1 : aicpusd_err("sem init failed, %s.", strerror(errno));
187 1 : return AICPU_SCHEDULE_ERROR_COMMON_ERROR;
188 : }
189 : try {
190 1 : std::thread th(&AicpuMonitor::Work, this);
191 1 : th.detach();
192 1 : } catch (std::exception& e) {
193 0 : (void)sem_destroy(&sem_);
194 0 : aicpusd_err("create aicpu monitor thread object failed, %s", e.what());
195 0 : return AICPU_SCHEDULE_ERROR_COMMON_ERROR;
196 0 : }
197 :
198 1 : semRet = sem_wait(&sem_);
199 1 : if (semRet == -1) {
200 0 : (void)sem_destroy(&sem_);
201 0 : aicpusd_err("sem wait failed, %s.", strerror(errno));
202 0 : return AICPU_SCHEDULE_ERROR_COMMON_ERROR;
203 : }
204 1 : (void)sem_destroy(&sem_);
205 1 : if (!running_) {
206 0 : aicpusd_err("create aicpu monitor thread failed");
207 0 : return AICPU_SCHEDULE_ERROR_COMMON_ERROR;
208 : }
209 1 : aicpusd_info("aicpu monitor thread running");
210 :
211 1 : return AICPU_SCHEDULE_OK;
212 : }
213 :
214 31 : void AicpuMonitor::Stop()
215 : {
216 31 : const std::unique_lock<std::mutex> stopLock(mutex_);
217 31 : done_ = true;
218 31 : }
219 :
220 3 : void AicpuMonitor::HandleTaskTimeout()
221 : {
222 3 : if (!taskTimeoutFlag_ || IsMonitorClosed()) {
223 2 : return;
224 : }
225 :
226 1 : const uint64_t nowTick = aicpu::GetSystemTick();
227 1 : for (size_t coreIndex = 0UL; coreIndex < aicpuCoreNum_; ++coreIndex) {
228 1 : TaskTimer taskTimer;
229 1 : taskTimer.SetRunFlag(taskTimer_[coreIndex].GetRunFlag());
230 1 : taskTimer.SetStartTick(taskTimer_[coreIndex].GetStartTick());
231 2 : if (taskTimer.GetRunFlag() && (nowTick > taskTimer.GetStartTick()) &&
232 1 : ((nowTick - taskTimer.GetStartTick()) >= taskTimeoutTick_)) {
233 : // handle task timeout
234 1 : std::string opname;
235 1 : (void)aicpu::GetOpname(static_cast<uint32_t>(coreIndex), opname);
236 1 : std::ostringstream oss;
237 1 : oss << "Send timeout to tsdaemon, tsdaemon will kill aicpu-custom-sd process, thread index[" << coreIndex
238 1 : << "], op name[" << opname << "]";
239 1 : if (taskInfo_[coreIndex].isHwts) {
240 0 : oss << ", " << taskInfo_[coreIndex].DebugString();
241 : }
242 1 : aicpusd_err("%s.", oss.str().c_str());
243 1 : SendKillMsgToTsd();
244 1 : break;
245 1 : }
246 : }
247 : }
248 : } // namespace AicpuSchedule
|
