Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 : #include "aicpusd_monitor.h"
11 :
12 : #include <mutex>
13 : #include <cstdlib>
14 : #include <string>
15 : #include <unistd.h>
16 : #include <cstring>
17 : #include <cerrno>
18 : #include "securec.h"
19 : #include "aicpusd_common.h"
20 : #include "profiling_adp.h"
21 : #include "aicpusd_drv_manager.h"
22 : #include "tsd.h"
23 : #include "aicpu_context.h"
24 : #include "aicpu_pulse.h"
25 : #include "aicpusd_info.h"
26 : #include "status.h"
27 : #include "feature_ctrl.h"
28 :
29 : namespace {
30 : // aicpu task timeout
31 : const uint64_t AICPU_TASK_TIMEOUT = 28LU;
32 : const uint64_t AICPU_TASK_TIMEOUT_LONG = 60UL;
33 : }
34 :
35 : namespace AicpuSchedule {
36 : /**
37 : * @ingroup AicpuMonitor
38 : * @brief it is used to construct a object of AicpuMonitor.
39 : * using default value to construct
40 : */
41 11 : AicpuMonitor::AicpuMonitor()
42 11 : : deviceId_(0U),
43 11 : taskTimeoutFlag_(false),
44 11 : taskInfo_(nullptr),
45 11 : done_(false),
46 11 : taskTimeout_(UINT64_MAX),
47 11 : taskTimeoutTick_(UINT64_MAX),
48 11 : taskTimer_(nullptr),
49 11 : running_(false),
50 11 : aicpuCoreNum_(0U),
51 11 : online_(false)
52 11 : {}
53 :
54 : /**
55 : * @ingroup AicpuMonitor
56 : * @brief it is used to destructor a object of AicpuMonitor.
57 : * it is neccessary to set flag false to make sure thread to be stopped
58 : */
59 11 : AicpuMonitor::~AicpuMonitor()
60 : {
61 11 : if (!done_) {
62 9 : Stop();
63 : }
64 11 : }
65 :
66 49 : AicpuMonitor &AicpuMonitor::GetInstance()
67 : {
68 49 : static AicpuMonitor instance;
69 49 : return instance;
70 : }
71 :
72 5 : int32_t AicpuMonitor::InitAicpuMonitor(const uint32_t deviceId, const bool online)
73 : {
74 5 : aicpusd_info("Begin to init aicpu monitor");
75 5 : online_ = online;
76 5 : if (!online_) {
77 1 : aicpusd_info("End to init aicpu monitor, offline mode");
78 1 : return AICPU_SCHEDULE_OK;
79 : }
80 4 : deviceId_ = deviceId;
81 4 : aicpuCoreNum_ = AicpuSchedule::AicpuDrvManager::GetInstance().GetAicpuNum();
82 4 : if (aicpuCoreNum_ != 0U) {
83 4 : taskInfo_.reset(new (std::nothrow) TaskInfoForMonitor[aicpuCoreNum_]);
84 4 : if (taskInfo_ == nullptr) {
85 0 : aicpusd_err("malloc task info memory for monitor failed");
86 0 : return AICPU_SCHEDULE_ERROR_COMMON_ERROR;
87 : }
88 8 : for (size_t i = 0UL; i < aicpuCoreNum_; i++) {
89 4 : taskInfo_[i] = {UINT64_MAX, UINT64_MAX, UINT32_MAX, false};
90 : }
91 : }
92 :
93 4 : const int32_t ret = SetTaskTimeoutFlag();
94 4 : if (ret != AICPU_SCHEDULE_OK) {
95 1 : aicpusd_err("set task timeout flag failed, ret[%d]", ret);
96 1 : return ret;
97 : }
98 :
99 3 : if (taskTimeoutFlag_ && (aicpuCoreNum_ != 0U)) {
100 6 : taskTimer_.reset(new (std::nothrow) TaskTimer[aicpuCoreNum_]);
101 3 : if (taskTimer_ == nullptr) {
102 0 : aicpusd_err("malloc memory for task timer failed");
103 0 : return AICPU_SCHEDULE_ERROR_COMMON_ERROR;
104 : }
105 : }
106 :
107 3 : aicpusd_run_info("Init aicpu monitor success, taskTimeout=%lus, tickFreq=%lu",
108 : taskTimeout_, aicpu::GetSystemTickFreq());
109 3 : return AICPU_SCHEDULE_OK;
110 : }
111 :
112 2 : void AicpuMonitor::SendKillMsgToTsd() const
113 : {
114 2 : aicpusd_run_info("dev[%u] send msg to tsdaemon, tsdaemon will kill aicpu-custom-sd process[%u]",
115 : deviceId_, static_cast<uint32_t>(getpid()));
116 6 : const int32_t ret = TsdDestroy(deviceId_, TSD_CUSTOM_COMPUTE,
117 2 : static_cast<uint32_t>(AicpuDrvManager::GetInstance().GetHostPid()),
118 2 : AicpuDrvManager::GetInstance().GetVfId());
119 2 : if (ret != static_cast<int32_t>(tsd::TSD_OK)) {
120 1 : aicpusd_err("dev[%u] send abnormal msg to tsdaemon failed, ret[%d]", deviceId_, ret);
121 : }
122 2 : }
123 :
124 10 : void AicpuMonitor::SetTaskInfo(const uint64_t taskIndex, const TaskInfoForMonitor &taskInfo)
125 : {
126 10 : if ((taskIndex < aicpuCoreNum_) && online_) {
127 2 : taskInfo_[taskIndex] = taskInfo;
128 : }
129 10 : }
130 :
131 8 : int32_t AicpuMonitor::SetTaskTimeoutFlag()
132 : {
133 8 : taskTimeout_ = FeatureCtrl::IsDoubleDieProduct() ?
134 : AICPU_TASK_TIMEOUT_LONG : AICPU_TASK_TIMEOUT;
135 8 : taskTimeoutTick_ = taskTimeout_ * aicpu::GetSystemTickFreq();
136 8 : taskTimeoutFlag_ = true;
137 8 : return AICPU_SCHEDULE_OK;
138 : }
139 :
140 9 : void AicpuMonitor::SetTaskStartTime(const uint64_t taskIndex)
141 : {
142 9 : if ((taskTimeoutFlag_) && (online_)) {
143 1 : taskTimer_[taskIndex].SetStartTick(aicpu::GetSystemTick());
144 1 : taskTimer_[taskIndex].SetRunFlag(true);
145 : }
146 9 : }
147 :
148 9 : void AicpuMonitor::SetTaskEndTime(const uint64_t taskIndex)
149 : {
150 9 : if ((taskTimeoutFlag_) && (online_)) {
151 1 : taskTimer_[taskIndex].SetRunFlag(false);
152 1 : taskInfo_[taskIndex].isHwts = false;
153 : }
154 9 : }
155 :
156 2 : void AicpuMonitor::Work(AicpuMonitor *const monitor)
157 : {
158 2 : std::once_flag flag;
159 3 : while ((monitor != nullptr) && (!(monitor->done_))) {
160 3 : if (monitor->online_) {
161 : // check and handle task timeout, for aicpu task of ts stream and aicpu stream
162 0 : monitor->HandleTaskTimeout();
163 : }
164 : // call pulseNotifyFuncMap
165 3 : AicpuPulseNotify();
166 3 : std::call_once(flag, [&]() {
167 2 : monitor->running_ = true;
168 2 : if (sem_post(&monitor->sem_) != 0) {
169 0 : aicpusd_err("sem post failed, %s", strerror(errno));
170 0 : return;
171 : }
172 : });
173 3 : (void)sleep(1U);
174 : }
175 0 : }
176 :
177 4 : int32_t AicpuMonitor::Run()
178 : {
179 4 : if (done_) {
180 1 : aicpusd_err("Fail to run monitor for it has been stopped");
181 1 : return AICPU_SCHEDULE_ERROR_COMMON_ERROR;
182 : }
183 3 : int32_t semRet = sem_init(&sem_, 0, 0U);
184 3 : if (semRet == -1) {
185 1 : aicpusd_err("sem init failed, %s.", strerror(errno));
186 1 : return AICPU_SCHEDULE_ERROR_COMMON_ERROR;
187 : }
188 : try {
189 2 : std::thread th(&AicpuMonitor::Work, this);
190 2 : th.detach();
191 2 : } catch(std::exception &e) {
192 0 : (void)sem_destroy(&sem_);
193 0 : aicpusd_err("create aicpu monitor thread object failed, %s", e.what());
194 0 : return AICPU_SCHEDULE_ERROR_COMMON_ERROR;
195 0 : }
196 :
197 2 : semRet = sem_wait(&sem_);
198 2 : if (semRet == -1) {
199 0 : (void)sem_destroy(&sem_);
200 0 : aicpusd_err("sem wait failed, %s.", strerror(errno));
201 0 : return AICPU_SCHEDULE_ERROR_COMMON_ERROR;
202 : }
203 2 : (void)sem_destroy(&sem_);
204 2 : if (!running_) {
205 0 : aicpusd_err("create aicpu monitor thread failed");
206 0 : return AICPU_SCHEDULE_ERROR_COMMON_ERROR;
207 : }
208 2 : aicpusd_info("aicpu monitor thread running");
209 :
210 2 : return AICPU_SCHEDULE_OK;
211 : }
212 :
213 28 : void AicpuMonitor::Stop()
214 : {
215 28 : const std::unique_lock<std::mutex> stopLock(mutex_);
216 28 : done_ = true;
217 28 : }
218 :
219 2 : void AicpuMonitor::HandleTaskTimeout()
220 : {
221 2 : if (!taskTimeoutFlag_) {
222 1 : return;
223 : }
224 :
225 1 : const uint64_t nowTick = aicpu::GetSystemTick();
226 1 : for (size_t coreIndex = 0UL; coreIndex < aicpuCoreNum_; ++coreIndex) {
227 1 : TaskTimer taskTimer;
228 1 : taskTimer.SetRunFlag(taskTimer_[coreIndex].GetRunFlag());
229 1 : taskTimer.SetStartTick(taskTimer_[coreIndex].GetStartTick());
230 2 : if (taskTimer.GetRunFlag() && (nowTick > taskTimer.GetStartTick()) &&
231 1 : ((nowTick - taskTimer.GetStartTick()) >= taskTimeoutTick_)) {
232 : // handle task timeout
233 1 : std::string opname;
234 1 : (void)aicpu::GetOpname(static_cast<uint32_t>(coreIndex), opname);
235 1 : std::ostringstream oss;
236 1 : oss << "Send timeout to tsdaemon, tsdaemon will kill aicpu-custom-sd process, thread index["
237 1 : << coreIndex << "], op name[" << opname << "]";
238 1 : if (taskInfo_[coreIndex].isHwts) {
239 0 : oss << ", " << taskInfo_[coreIndex].DebugString();
240 : }
241 1 : aicpusd_err("%s.", oss.str().c_str());
242 1 : SendKillMsgToTsd();
243 1 : break;
244 1 : }
245 : }
246 : }
247 : } // namespace AicpuSchedule
|
