Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 : #include "aicpusd_monitor.h"
11 :
12 : #include <new>
13 : #include <mutex>
14 : #include <string>
15 : #include <unistd.h>
16 : #include "aicpusd_info.h"
17 : #include "aicpusd_util.h"
18 : #include "aicpusd_common.h"
19 : #include "aicpusd_drv_manager.h"
20 : #include "tsd.h"
21 : #include "aicpu_context.h"
22 : #include "aicpu_pulse.h"
23 : #include "aicpusd_mpi_mgr.h"
24 : #include "aicpusd_lastword.h"
25 : #include "aicpusd_resource_manager.h"
26 : #include "aicpusd_threads_process.h"
27 : #include "aicpusd_worker.h"
28 :
29 : namespace {
30 : // aicpu task timeout
31 : constexpr uint64_t AICPU_TASK_TIMEOUT = 28UL;
32 : constexpr uint64_t AICPU_TASK_TIMEOUT_LONG = 60UL;
33 : constexpr uint64_t AICPU_TASK_TIMEOUT_FPGA = 2800UL;
34 : constexpr const uint32_t MONITOR_SLEEP_INTERVAL = 100000U; // 100ms
35 : #ifndef aicpusd_UT
36 : constexpr const uint32_t MONITOR_TIMEOUT_COUNT = 10U; // 1s
37 : constexpr const uint32_t MONITOR_SUPPLYENQUE_COUNT = 20U; // 2s
38 : #else
39 : constexpr const uint32_t MONITOR_TIMEOUT_COUNT = 1U;
40 : constexpr const uint32_t MONITOR_SUPPLYENQUE_COUNT = 1U;
41 : #endif
42 : }
43 :
44 : namespace AicpuSchedule {
45 : /**
46 : * @ingroup AicpuMonitor
47 : * @brief it is used to construct a object of AicpuMonitor.
48 : */
49 28 : AicpuMonitor::AicpuMonitor()
50 28 : : deviceId_(0U),
51 28 : taskTimeoutFlag_(false),
52 28 : modelTimeoutFlag_(false),
53 28 : monitorTaskInfo_(nullptr),
54 28 : done_(false),
55 28 : tsTimeoutEnable_(false),
56 28 : tsOpTimeOut_(0U),
57 28 : taskTimeout_(UINT64_MAX),
58 28 : taskTimeoutTick_(UINT64_MAX),
59 28 : modelTimeoutTick_(UINT64_MAX),
60 28 : aicpuTaskTimer_(nullptr),
61 28 : modelTimer_(nullptr),
62 28 : running_(false),
63 28 : aicpuCoreNum_(0U),
64 28 : online_(false),
65 112 : opTimeoutFlag_(false)
66 28 : {}
67 :
68 : /**
69 : * @ingroup AicpuMonitor
70 : * @brief it is used to destructor a object of AicpuMonitor.
71 : */
72 28 : AicpuMonitor::~AicpuMonitor()
73 : {
74 28 : if (!done_) {
75 28 : StopMonitor();
76 : }
77 28 : aicpusd_info("AicpuMonitor join begin");
78 28 : for (auto &monitorThread : th_) {
79 0 : if (monitorThread.joinable()) {
80 0 : monitorThread.join();
81 : }
82 : }
83 28 : aicpusd_info("AicpuMonitor join end");
84 28 : }
85 :
86 635 : AicpuMonitor &AicpuMonitor::GetInstance()
87 : {
88 635 : static AicpuMonitor instance;
89 635 : return instance;
90 : }
91 :
92 21 : int32_t AicpuMonitor::InitTimer()
93 : {
94 21 : if (taskTimeoutFlag_ && (aicpuCoreNum_ != 0U)) {
95 55 : aicpuTaskTimer_.reset(new (std::nothrow) TaskTimer[aicpuCoreNum_]);
96 21 : if (aicpuTaskTimer_ == nullptr) {
97 0 : aicpusd_err("malloc memory for task timer failed");
98 0 : return AICPU_SCHEDULE_ERROR_COMMON_ERROR;
99 : }
100 : }
101 21 : if (taskTimeoutFlag_) {
102 21525 : aicpuStreamTaskTimer_.reset(new (std::nothrow) TaskTimer[MAX_MODEL_COUNT]);
103 21 : if (aicpuStreamTaskTimer_ == nullptr) {
104 0 : aicpusd_err("malloc memory for aicpu stream task timer failed");
105 0 : return AICPU_SCHEDULE_ERROR_COMMON_ERROR;
106 : }
107 : }
108 21 : if (modelTimeoutFlag_) {
109 3075 : modelTimer_.reset(new (std::nothrow) TaskTimer[MAX_MODEL_COUNT]);
110 3 : if (modelTimer_ == nullptr) {
111 0 : aicpusd_err("malloc memory for model timer failed");
112 0 : return AICPU_SCHEDULE_ERROR_COMMON_ERROR;
113 : }
114 : }
115 21 : return AICPU_SCHEDULE_OK;
116 : }
117 :
118 25 : int32_t AicpuMonitor::InitMonitor(const uint32_t devId, const bool isOnline)
119 : {
120 25 : aicpusd_info("Begin to init aicpu monitor.");
121 25 : online_ = isOnline;
122 25 : if (!online_) {
123 1 : aicpusd_info("End to init aicpu monitor, offline mode");
124 1 : return AICPU_SCHEDULE_OK;
125 : }
126 24 : deviceId_ = devId;
127 24 : aicpuCoreNum_ = static_cast<uint32_t>(AicpuSchedule::ThreadPool::GetWorkerNum());
128 24 : if (aicpuCoreNum_ != 0U) {
129 24 : monitorTaskInfo_.reset(new (std::nothrow) TaskInfoForMonitor[aicpuCoreNum_]);
130 24 : if (monitorTaskInfo_ == nullptr) {
131 0 : aicpusd_err("Malloc task info memory for monitor failed");
132 0 : return AICPU_SCHEDULE_ERROR_COMMON_ERROR;
133 : }
134 61 : for (uint32_t i = 0U; i < aicpuCoreNum_; i++) {
135 37 : monitorTaskInfo_[static_cast<uint64_t>(i)] = {UINT64_MAX, UINT64_MAX, UINT32_MAX, false};
136 : }
137 : }
138 24 : int32_t ret = SetTaskTimeoutFlag();
139 24 : if (ret != AICPU_SCHEDULE_OK) {
140 1 : aicpusd_err("Set task timeout flag failed, ret[%d]", ret);
141 1 : return ret;
142 : }
143 23 : ret = SetModelTimeoutFlag();
144 23 : if (ret != AICPU_SCHEDULE_OK) {
145 1 : aicpusd_err("Set model timeout flag failed, ret[%d]", ret);
146 1 : return ret;
147 : }
148 :
149 22 : ret = InitTimer();
150 22 : if (ret != AICPU_SCHEDULE_OK) {
151 1 : aicpusd_err("Init timer failed, ret[%d]", ret);
152 1 : return ret;
153 : }
154 :
155 21 : InitAsyncOpTimer();
156 :
157 21 : aicpusd_run_info("Init aicpu monitor successfully, taskTimeout=%lus, tickFreq=%lu.",
158 : taskTimeout_, aicpu::GetSystemTickFreq());
159 21 : return AICPU_SCHEDULE_OK;
160 : }
161 :
162 21 : void AicpuMonitor::InitAsyncOpTimer()
163 : {
164 21 : aicpusd_info("Start initializing async op timer in monitor");
165 :
166 1 : const auto startTimerCbk = [this](const aicpu::TimerHandle timerHandle, const uint32_t timeInS) {
167 1 : return SetOpTimerStartTime(timerHandle, timeInS);
168 21 : };
169 :
170 1 : const auto stopTimerCbk = [this](const aicpu::TimerHandle timerHandle) {
171 1 : return SetOpTimerEndTime(timerHandle);
172 21 : };
173 :
174 21 : aicpu::AicpuTimer::GetInstance().RegistMonitorFunc(startTimerCbk, stopTimerCbk);
175 21 : }
176 :
177 15 : void AicpuMonitor::SendKillMsgToTsd() const
178 : {
179 15 : aicpusd_run_info("dev[%u] send msg to tsdaemon, tsdaemon will kill aicpu-sd process[%u]", deviceId_,
180 : static_cast<uint32_t>(getpid()));
181 15 : AicpusdLastword::GetInstance().LastwordCallback();
182 : // flush cache log to slogd
183 15 : DlogFlushAicpu();
184 15 : (void)AicpuSchedule::mpi::MpiDvppStatisticManager::Instance().PrintStatisticInfo();
185 :
186 : // thread node ppid is not tsd, don't need to destroy
187 15 : if (!online_) {
188 3 : aicpusd_run_info("offline mode no need send msg to tsd");
189 3 : return;
190 : }
191 36 : const int32_t ret = TsdDestroy(deviceId_, TSD_COMPUTE,
192 12 : static_cast<uint32_t>(AicpuDrvManager::GetInstance().GetHostPid()), AicpuDrvManager::GetInstance().GetVfId());
193 12 : if (ret != 0) {
194 1 : aicpusd_err("dev[%u] send abnormal msg to tsdaemon failed, ret[%d]", deviceId_, ret);
195 : }
196 : }
197 :
198 127 : void AicpuMonitor::SetTaskInfo(const uint32_t threadIndex, const TaskInfoForMonitor &taskInfo) const
199 : {
200 127 : if ((threadIndex < aicpuCoreNum_) && online_) {
201 125 : monitorTaskInfo_[static_cast<uint64_t>(threadIndex)] = taskInfo;
202 : }
203 127 : }
204 :
205 24 : int32_t AicpuMonitor::SetTaskTimeoutFlag()
206 : {
207 24 : taskTimeout_ = FeatureCtrl::IsDoubleDieProduct() ?
208 : AICPU_TASK_TIMEOUT_LONG : AICPU_TASK_TIMEOUT;
209 24 : taskTimeoutTick_ = taskTimeout_ * aicpu::GetSystemTickFreq();
210 119 : if (AicpuUtil::IsEnvValEqual(ENV_NAME_DATAMASTER_RUN_MODE, "1") ||
211 70 : AicpuUtil::IsEnvValEqual(ENV_NAME_DATAMASTER_RUN_MODE, "2")) {
212 1 : aicpusd_warn("Set task timeout to FPGA mode.");
213 1 : taskTimeout_ = AICPU_TASK_TIMEOUT_FPGA;
214 1 : taskTimeoutTick_ = AICPU_TASK_TIMEOUT_FPGA * aicpu::GetSystemTickFreq();
215 : }
216 :
217 24 : taskTimeoutFlag_ = true;
218 24 : SetOpTimeoutFlag(true);
219 :
220 24 : return AICPU_SCHEDULE_OK;
221 : }
222 :
223 24 : int32_t AicpuMonitor::SetModelTimeoutFlag()
224 : {
225 24 : std::string timeoutFlag;
226 24 : const bool ret = AicpuUtil::GetEnvVal(ENV_NAME_AICPU_MODEL_TIMEOUT, timeoutFlag);
227 24 : if (!ret) {
228 19 : aicpusd_run_info("Not set aicpu model timeout. Env is not set.");
229 19 : return AICPU_SCHEDULE_OK;
230 : }
231 :
232 5 : aicpusd_run_info("Get model timeout is %ss", timeoutFlag.c_str());
233 : try {
234 5 : const uint64_t modelTimeout = std::stoul(timeoutFlag);
235 4 : const uint64_t frep = aicpu::GetSystemTickFreq();
236 4 : if (AicpuUtil::IsUint64MulOverflow(modelTimeout, frep)) {
237 1 : aicpusd_err("Overflow occurred when set model timeout, timeout=%lus, freq=%lu.", modelTimeout, frep);
238 1 : return AICPU_SCHEDULE_ERROR_PARAMETER_NOT_VALID;
239 : }
240 3 : modelTimeoutTick_ = modelTimeout * frep;
241 1 : } catch (std::exception &e) {
242 1 : aicpusd_err("Convert AICPU_MODEL_TIMEOUT[%s]s to number failed, %s", timeoutFlag.c_str(), e.what());
243 1 : return AICPU_SCHEDULE_ERROR_COMMON_ERROR;
244 1 : }
245 :
246 3 : modelTimeoutFlag_ = true;
247 3 : return AICPU_SCHEDULE_OK;
248 24 : }
249 :
250 : // index used inner, it is valid
251 130 : void AicpuMonitor::SetTaskStartTime(const uint32_t taskId)
252 : {
253 130 : if ((taskTimeoutFlag_) && (online_)) {
254 130 : aicpuTaskTimer_[static_cast<uint64_t>(taskId)].SetStartTick(aicpu::GetSystemTick());
255 130 : aicpuTaskTimer_[static_cast<uint64_t>(taskId)].SetRunFlag(true);
256 : }
257 130 : }
258 :
259 128 : void AicpuMonitor::SetTaskEndTime(const uint32_t taskId)
260 : {
261 128 : if ((taskTimeoutFlag_) && (online_)) {
262 128 : aicpuTaskTimer_[static_cast<uint64_t>(taskId)].SetRunFlag(false);
263 130 : monitorTaskInfo_[static_cast<uint64_t>(taskId)].isHwts = false;
264 : }
265 128 : }
266 :
267 66 : void AicpuMonitor::SetAicpuStreamTaskStartTime(const uint32_t taskId)
268 : {
269 66 : if ((taskTimeoutFlag_) && (taskId < MAX_MODEL_COUNT) && (online_)) {
270 3 : aicpuStreamTaskTimer_[static_cast<uint64_t>(taskId)].SetStartTick(aicpu::GetSystemTick());
271 3 : aicpuStreamTaskTimer_[static_cast<uint64_t>(taskId)].SetRunFlag(true);
272 : }
273 66 : }
274 :
275 66 : void AicpuMonitor::SetAicpuStreamTaskEndTime(const uint32_t taskId)
276 : {
277 66 : if ((taskTimeoutFlag_) && (taskId < MAX_MODEL_COUNT) && (online_)) {
278 3 : aicpuStreamTaskTimer_[static_cast<uint64_t>(taskId)].SetRunFlag(false);
279 : }
280 66 : }
281 :
282 8 : void AicpuMonitor::SetModelStartTime(const uint32_t modelId)
283 : {
284 8 : if ((modelTimeoutFlag_) && (modelId < MAX_MODEL_COUNT) && (online_)) {
285 1 : modelTimer_[static_cast<uint64_t>(modelId)].SetStartTick(aicpu::GetSystemTick());
286 1 : modelTimer_[static_cast<uint64_t>(modelId)].SetRunFlag(true);
287 : }
288 8 : }
289 :
290 8 : void AicpuMonitor::SetModelEndTime(const uint32_t modelId)
291 : {
292 8 : if ((modelTimeoutFlag_) && (modelId < MAX_MODEL_COUNT) && (online_)) {
293 1 : modelTimer_[static_cast<uint64_t>(modelId)].SetRunFlag(false);
294 : }
295 8 : }
296 :
297 2 : void AicpuMonitor::SetOpTimerStartTime(const aicpu::TimerHandle timerId, const uint32_t timeInS)
298 : {
299 2 : if ((opTimeoutFlag_) && (online_)) {
300 2 : std::shared_ptr<TaskTimer> taskTimerPtr = std::make_shared<TaskTimer>(aicpu::GetSystemTick(), true);
301 2 : if (taskTimerPtr == nullptr) {
302 0 : aicpusd_err("alloc taskTimerPtr fail");
303 0 : return;
304 : }
305 2 : taskTimerPtr->SetTimeTick(timeInS);
306 : {
307 2 : const std::lock_guard<std::mutex> lk(opTimerMapMutex_);
308 2 : (void)opTimer_.emplace(timerId, taskTimerPtr);
309 2 : }
310 2 : }
311 : }
312 :
313 2 : void AicpuMonitor::SetOpTimerEndTime(const aicpu::TimerHandle timerId)
314 : {
315 2 : if ((opTimeoutFlag_ && online_) || (!opTimer_.empty())) {
316 2 : const std::lock_guard<std::mutex> lk(opTimerMapMutex_);
317 2 : (void)opTimer_.erase(timerId);
318 2 : }
319 2 : }
320 :
321 5 : void AicpuMonitor::Work(AicpuMonitor *const monitor)
322 : {
323 5 : std::once_flag exeOnce;
324 5 : uint32_t count = 0;
325 5 : AICPUSubEventInfo subEventInfo = {};
326 5 : event_summary eventInfoSummary = {};
327 5 : eventInfoSummary.pid = getpid();
328 5 : eventInfoSummary.event_id = EVENT_AICPU_MSG;
329 5 : eventInfoSummary.subevent_id = AICPU_SUB_EVENT_SUPPLY_ENQUEUE;
330 5 : eventInfoSummary.msg = PtrToPtr<AICPUSubEventInfo, char_t>(&subEventInfo);
331 5 : eventInfoSummary.msg_len = static_cast<uint32_t>(sizeof(AICPUSubEventInfo));
332 5 : eventInfoSummary.grp_id = CP_DEFAULT_GROUP_ID;
333 5 : std::vector<size_t> eventWaitIds;
334 9 : while ((monitor != nullptr) && (!monitor->done_)) {
335 : if (count % MONITOR_TIMEOUT_COUNT == 0) {
336 4 : if (monitor->online_) {
337 : // check and handle task timeout, for aicpu task of ts stream and aicpu stream
338 4 : monitor->HandleTaskTimeout();
339 : // check and handle model timeout
340 4 : monitor->HandleModelTimeout();
341 4 : monitor->HandleOpTimeout();
342 : }
343 4 : if (FeatureCtrl::ShouldMonitorWork() ) {
344 4 : AicpuPulseNotify();
345 : }
346 4 : std::call_once(exeOnce, [&]() {
347 4 : monitor->running_ = true;
348 4 : if (sem_post(&monitor->sem_) != 0) {
349 0 : aicpusd_err("sem post failed, %s", strerror(errno));
350 0 : return;
351 : }
352 : });
353 : }
354 8 : if ((count % MONITOR_SUPPLYENQUE_COUNT == 0) && (FeatureCtrl::ShouldMonitorWork()) &&
355 4 : (!FeatureCtrl::ShouldSkipSupplyEvent())) {
356 4 : EventWaitManager::AnyQueNotEmptyWaitManager().GetWaitingEvent(eventWaitIds);
357 5 : for (const auto waitId : eventWaitIds) {
358 1 : subEventInfo.modelId = waitId;
359 1 : const drvError_t ret = halEschedSubmitEvent(monitor->deviceId_, &eventInfoSummary);
360 1 : if (ret != DRV_ERROR_NONE) {
361 1 : aicpusd_err("Failed to supply enque event for model[%u]. ret is %d.", waitId, ret);
362 : }
363 1 : aicpusd_info("supply enque event for model[%u].", waitId);
364 : }
365 4 : eventWaitIds.clear();
366 : }
367 4 : count++;
368 4 : if (count == MONITOR_SUPPLYENQUE_COUNT) {
369 4 : count = 0;
370 : }
371 4 : (void)usleep(MONITOR_SLEEP_INTERVAL);
372 : }
373 5 : }
374 :
375 8 : int32_t AicpuMonitor::Run()
376 : {
377 8 : if (done_) {
378 2 : aicpusd_err("Fail to run monitor for it has been stopped");
379 2 : return AICPU_SCHEDULE_ERROR_COMMON_ERROR;
380 : }
381 6 : int32_t semRet = sem_init(&sem_, 0, 0U);
382 6 : if (semRet == -1) {
383 1 : aicpusd_err("sem init failed, %s.", strerror(errno));
384 1 : return AICPU_SCHEDULE_ERROR_COMMON_ERROR;
385 : }
386 :
387 : try {
388 5 : std::thread monitorThread(&AicpuMonitor::Work, this);
389 5 : th_.emplace_back(std::move(monitorThread));
390 5 : } catch (std::exception &threadException) {
391 0 : (void)sem_destroy(&sem_);
392 0 : aicpusd_err("create aicpu monitor thread object failed, %s", threadException.what());
393 0 : return AICPU_SCHEDULE_ERROR_COMMON_ERROR;
394 0 : }
395 :
396 5 : semRet = sem_wait(&sem_);
397 5 : if (semRet == -1) {
398 1 : (void)sem_destroy(&sem_);
399 1 : aicpusd_err("sem wait failed, %s.", strerror(errno));
400 1 : return AICPU_SCHEDULE_ERROR_COMMON_ERROR;
401 : }
402 4 : (void)sem_destroy(&sem_);
403 4 : if (!running_) {
404 0 : aicpusd_err("create aicpu monitor thread failed");
405 0 : return AICPU_SCHEDULE_ERROR_COMMON_ERROR;
406 : }
407 4 : aicpusd_info("Aicpu monitor thread running");
408 :
409 4 : return AICPU_SCHEDULE_OK;
410 : }
411 :
412 56 : void AicpuMonitor::StopMonitor()
413 : {
414 56 : aicpusd_info("Begin to stop aicpu monitor");
415 56 : const std::unique_lock<std::mutex> lk(mutex_);
416 56 : done_ = true;
417 61 : for (auto &thead : th_) {
418 5 : if (thead.joinable()) {
419 5 : thead.join();
420 : }
421 : }
422 :
423 56 : done_ = false;
424 56 : th_.clear();
425 56 : }
426 :
427 10 : void AicpuMonitor::SetOpExecuteTimeOut(const uint32_t timeOutEn, const uint32_t opExecuteTimeOut)
428 : {
429 10 : const std::unique_lock<std::mutex> lk(setTimeOutMut_);
430 10 : if (timeOutEn == 1U) {
431 9 : tsTimeoutEnable_ = true;
432 9 : tsOpTimeOut_ = opExecuteTimeOut;
433 9 : taskTimeoutTick_ = opExecuteTimeOut * aicpu::GetSystemTickFreq();
434 9 : taskTimeoutFlag_ = true;
435 27 : aicpusd_run_info("Get enable op execute timeout config from ts. timeout[%u]s tickFrequency[%lu]",
436 : tsOpTimeOut_.load(), taskTimeoutTick_.load());
437 : } else {
438 1 : tsTimeoutEnable_ = false;
439 1 : (void)SetTaskTimeoutFlag();
440 2 : aicpusd_run_info("Get disable op execute timeout config from ts. tickFrequency[%lu]", taskTimeoutTick_.load());
441 : }
442 10 : }
443 :
444 30 : void AicpuMonitor::SetOpTimeoutFlag(const bool flag)
445 : {
446 30 : aicpusd_info("Set op timer flag to %d", flag);
447 30 : opTimeoutFlag_ = flag;
448 30 : aicpu::AicpuTimer::GetInstance().SetSupportTimer(flag);
449 30 : }
450 :
451 7 : void AicpuMonitor::HandleTaskTimeout()
452 : {
453 7 : if (!taskTimeoutFlag_) {
454 1 : return;
455 : }
456 :
457 6 : const uint64_t nowTick = aicpu::GetSystemTick();
458 20 : for (uint32_t i = 0U; i < aicpuCoreNum_; ++i) {
459 15 : const auto runFlag = aicpuTaskTimer_[static_cast<uint64_t>(i)].GetRunFlag();
460 15 : const auto startTick = aicpuTaskTimer_[static_cast<uint64_t>(i)].GetStartTick();
461 16 : if (runFlag && (nowTick > startTick) && ((nowTick - startTick) >= taskTimeoutTick_.load())) {
462 : // handle task timeout
463 1 : std::string opname;
464 1 : (void)aicpu::GetOpname(i, opname);
465 1 : std::ostringstream oss;
466 1 : oss << "Send timeout to tsdaemon, tsdaemon will kill aicpu-sd process, thread index[" << i <<
467 1 : "], op name[" << opname << "]";
468 1 : if (monitorTaskInfo_[static_cast<uint64_t>(i)].isHwts) {
469 0 : oss << ", " << MonitorDebug::MonitorDebugString(monitorTaskInfo_[static_cast<uint64_t>(i)]);
470 : }
471 2 : aicpusd_err("%s, nowTick:%llu, startTick:%llu, timeOut:%llu, tickFreq:%llu.", oss.str().c_str(),
472 : nowTick, startTick, taskTimeoutTick_.load(), aicpu::GetSystemTickFreq());
473 1 : aicpusd_run_info("%s", ComputeProcess::GetInstance().DebugString().c_str());
474 1 : SendKillMsgToTsd();
475 1 : return;
476 1 : }
477 : }
478 4101 : for (uint32_t modelId = 0U; modelId < MAX_MODEL_COUNT; ++modelId) {
479 4097 : const auto runFlag = aicpuStreamTaskTimer_[static_cast<uint64_t>(modelId)].GetRunFlag();
480 4097 : const auto startTick = aicpuStreamTaskTimer_[static_cast<uint64_t>(modelId)].GetStartTick();
481 4098 : if (runFlag && (nowTick > startTick) && ((nowTick - startTick) >= taskTimeoutTick_.load())) {
482 : // handle aicpu stream task timeout
483 1 : aicpusd_err("Send stream task timeout, tsdaemon will kill aicpu-sd process, model id[%u].", modelId);
484 1 : SendKillMsgToTsd();
485 1 : break;
486 : }
487 : }
488 : }
489 :
490 5 : void AicpuMonitor::HandleOpTimeout()
491 : {
492 5 : if ((!opTimeoutFlag_) && (opTimer_.empty())) {
493 0 : return;
494 : }
495 :
496 5 : aicpu::TimerHandle timerId = 0;
497 5 : bool isTimeout = false;
498 : {
499 5 : const std::lock_guard<std::mutex> lk(opTimerMapMutex_);
500 5 : const uint64_t nowTick = aicpu::GetSystemTick();
501 6 : for (auto &timer : opTimer_) {
502 1 : const auto runFlag = timer.second->GetRunFlag();
503 1 : const auto startTick = timer.second->GetStartTick();
504 1 : const auto timeoutTick = timer.second->GetTimeTick() == 0 ? taskTimeoutTick_.load() :
505 1 : timer.second->GetTimeTick();
506 1 : if (runFlag && (nowTick > startTick) && ((nowTick - startTick) >= timeoutTick)) {
507 1 : timerId = timer.first;
508 1 : isTimeout = true;
509 1 : std::ostringstream oss;
510 1 : oss << "Op timeout occurred, timer id[" << timerId << "]";
511 2 : aicpusd_err("%s, nowTick:%llu, startTick:%llu, timeOut:%llu, tickFreq:%llu.", oss.str().c_str(),
512 : nowTick, startTick, taskTimeoutTick_.load(), aicpu::GetSystemTickFreq());
513 1 : aicpu::AicpuTimer::GetInstance().CallTimeoutCallback(timerId);
514 1 : }
515 : }
516 5 : }
517 :
518 5 : if (isTimeout) {
519 1 : aicpusd_err("Timeout occurred, start to stop timer. TimerId=%lu", timerId);
520 1 : SetOpTimerEndTime(timerId);
521 1 : DlogFlushAicpu();
522 : }
523 : }
524 :
525 5 : void AicpuMonitor::HandleModelTimeout()
526 : {
527 5 : if (modelTimeoutFlag_) {
528 2 : const uint64_t nowTick = aicpu::GetSystemTick();
529 1026 : for (uint32_t modelId = 0U; modelId < MAX_MODEL_COUNT; ++modelId) {
530 1025 : const TaskTimer timer(modelTimer_[static_cast<uint64_t>(modelId)].GetStartTick(),
531 1025 : modelTimer_[static_cast<uint64_t>(modelId)].GetRunFlag());
532 1026 : if (timer.GetRunFlag() && (nowTick > timer.GetStartTick()) &&
533 1 : ((nowTick - timer.GetStartTick()) >= modelTimeoutTick_)) {
534 : // handle model timeout
535 1 : aicpusd_err("Send model timeout, tsdaemon will kill aicpu-sd process, model id[%u].", modelId);
536 1 : SendKillMsgToTsd();
537 1 : break;
538 : }
539 : }
540 : }
541 5 : }
542 :
543 28 : void AicpuMonitor::DisableModelTimeout()
544 : {
545 28 : modelTimeoutFlag_ = false;
546 28 : }
547 :
548 14 : uint32_t AicpuMonitor::GetTaskDefaultTimeout() const
549 : {
550 27 : return tsTimeoutEnable_ ? tsOpTimeOut_.load() : taskTimeout_;
551 : }
552 : } // namespace AicpuSchedule
|
