Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 : #include "aicpusd_monitor.h"
11 :
12 : #include <new>
13 : #include <mutex>
14 : #include <string>
15 : #include <unistd.h>
16 : #include "aicpusd_info.h"
17 : #include "aicpusd_util.h"
18 : #include "aicpusd_common.h"
19 : #include "aicpusd_drv_manager.h"
20 : #include "tsd.h"
21 : #include "aicpu_context.h"
22 : #include "aicpu_pulse.h"
23 : #include "aicpusd_mpi_mgr.h"
24 : #include "aicpusd_lastword.h"
25 : #include "aicpusd_resource_manager.h"
26 : #include "aicpusd_threads_process.h"
27 : #include "aicpusd_worker.h"
28 : #include "aicpusd_feature_ctrl.h"
29 :
30 : namespace {
31 : // aicpu task timeout
32 : constexpr uint64_t AICPU_TASK_TIMEOUT = 28UL;
33 : constexpr uint64_t AICPU_TASK_TIMEOUT_LONG = 60UL;
34 : constexpr uint64_t AICPU_TASK_TIMEOUT_FPGA = 2800UL;
35 : constexpr const uint32_t MONITOR_SLEEP_INTERVAL = 100000U; // 100ms
36 : #ifndef aicpusd_UT
37 : constexpr const uint32_t MONITOR_TIMEOUT_COUNT = 10U; // 1s
38 : constexpr const uint32_t MONITOR_SUPPLYENQUE_COUNT = 20U; // 2s
39 : #else
40 : constexpr const uint32_t MONITOR_TIMEOUT_COUNT = 1U;
41 : constexpr const uint32_t MONITOR_SUPPLYENQUE_COUNT = 1U;
42 : #endif
43 : } // namespace
44 :
45 : namespace AicpuSchedule {
46 : /**
47 : * @ingroup AicpuMonitor
48 : * @brief it is used to construct a object of AicpuMonitor.
49 : */
50 30 : AicpuMonitor::AicpuMonitor()
51 30 : : deviceId_(0U),
52 30 : taskTimeoutFlag_(false),
53 30 : modelTimeoutFlag_(false),
54 30 : monitorTaskInfo_(nullptr),
55 30 : done_(false),
56 30 : tsTimeoutEnable_(false),
57 30 : tsOpTimeOut_(0U),
58 30 : taskTimeout_(UINT64_MAX),
59 30 : taskTimeoutTick_(UINT64_MAX),
60 30 : modelTimeoutTick_(UINT64_MAX),
61 30 : aicpuTaskTimer_(nullptr),
62 30 : modelTimer_(nullptr),
63 30 : running_(false),
64 30 : aicpuCoreNum_(0U),
65 30 : online_(false),
66 120 : opTimeoutFlag_(false)
67 30 : {}
68 :
69 : /**
70 : * @ingroup AicpuMonitor
71 : * @brief it is used to destructor a object of AicpuMonitor.
72 : */
73 30 : AicpuMonitor::~AicpuMonitor()
74 : {
75 30 : if (!done_) {
76 30 : StopMonitor();
77 : }
78 30 : aicpusd_info("AicpuMonitor join begin");
79 30 : for (auto& monitorThread : th_) {
80 0 : if (monitorThread.joinable()) {
81 0 : monitorThread.join();
82 : }
83 : }
84 30 : aicpusd_info("AicpuMonitor join end");
85 30 : }
86 :
87 648 : AicpuMonitor& AicpuMonitor::GetInstance()
88 : {
89 648 : static AicpuMonitor instance;
90 648 : return instance;
91 : }
92 :
93 21 : int32_t AicpuMonitor::InitTimer()
94 : {
95 21 : if (taskTimeoutFlag_ && (aicpuCoreNum_ != 0U)) {
96 55 : aicpuTaskTimer_.reset(new (std::nothrow) TaskTimer[aicpuCoreNum_]);
97 21 : if (aicpuTaskTimer_ == nullptr) {
98 0 : aicpusd_err("malloc memory for task timer failed");
99 0 : return AICPU_SCHEDULE_ERROR_COMMON_ERROR;
100 : }
101 : }
102 21 : if (taskTimeoutFlag_) {
103 21525 : aicpuStreamTaskTimer_.reset(new (std::nothrow) TaskTimer[MAX_MODEL_COUNT]);
104 21 : if (aicpuStreamTaskTimer_ == nullptr) {
105 0 : aicpusd_err("malloc memory for aicpu stream task timer failed");
106 0 : return AICPU_SCHEDULE_ERROR_COMMON_ERROR;
107 : }
108 : }
109 21 : if (modelTimeoutFlag_) {
110 3075 : modelTimer_.reset(new (std::nothrow) TaskTimer[MAX_MODEL_COUNT]);
111 3 : if (modelTimer_ == nullptr) {
112 0 : aicpusd_err("malloc memory for model timer failed");
113 0 : return AICPU_SCHEDULE_ERROR_COMMON_ERROR;
114 : }
115 : }
116 21 : return AICPU_SCHEDULE_OK;
117 : }
118 :
119 25 : int32_t AicpuMonitor::InitMonitor(const uint32_t devId, const bool isOnline)
120 : {
121 25 : aicpusd_info("Begin to init aicpu monitor.");
122 25 : online_ = isOnline;
123 25 : if (!online_) {
124 1 : aicpusd_info("End to init aicpu monitor, offline mode");
125 1 : return AICPU_SCHEDULE_OK;
126 : }
127 24 : deviceId_ = devId;
128 24 : aicpuCoreNum_ = static_cast<uint32_t>(AicpuSchedule::ThreadPool::GetWorkerNum());
129 24 : if (aicpuCoreNum_ != 0U) {
130 24 : monitorTaskInfo_.reset(new (std::nothrow) TaskInfoForMonitor[aicpuCoreNum_]);
131 24 : if (monitorTaskInfo_ == nullptr) {
132 0 : aicpusd_err("Malloc task info memory for monitor failed");
133 0 : return AICPU_SCHEDULE_ERROR_COMMON_ERROR;
134 : }
135 61 : for (uint32_t i = 0U; i < aicpuCoreNum_; i++) {
136 37 : monitorTaskInfo_[static_cast<uint64_t>(i)] = {UINT64_MAX, UINT64_MAX, UINT32_MAX, false};
137 : }
138 : }
139 24 : int32_t ret = SetTaskTimeoutFlag();
140 24 : if (ret != AICPU_SCHEDULE_OK) {
141 1 : aicpusd_err("Set task timeout flag failed, ret[%d]", ret);
142 1 : return ret;
143 : }
144 23 : ret = SetModelTimeoutFlag();
145 23 : if (ret != AICPU_SCHEDULE_OK) {
146 1 : aicpusd_err("Set model timeout flag failed, ret[%d]", ret);
147 1 : return ret;
148 : }
149 :
150 22 : ret = InitTimer();
151 22 : if (ret != AICPU_SCHEDULE_OK) {
152 1 : aicpusd_err("Init timer failed, ret[%d]", ret);
153 1 : return ret;
154 : }
155 :
156 21 : InitAsyncOpTimer();
157 :
158 21 : aicpusd_run_info(
159 : "Init aicpu monitor successfully, taskTimeout=%lus, tickFreq=%lu.", taskTimeout_, aicpu::GetSystemTickFreq());
160 21 : return AICPU_SCHEDULE_OK;
161 : }
162 :
163 21 : void AicpuMonitor::InitAsyncOpTimer()
164 : {
165 21 : aicpusd_info("Start initializing async op timer in monitor");
166 :
167 1 : const auto startTimerCbk = [this](const aicpu::TimerHandle timerHandle, const uint32_t timeInS) {
168 1 : return SetOpTimerStartTime(timerHandle, timeInS);
169 21 : };
170 :
171 1 : const auto stopTimerCbk = [this](const aicpu::TimerHandle timerHandle) { return SetOpTimerEndTime(timerHandle); };
172 :
173 21 : aicpu::AicpuTimer::GetInstance().RegistMonitorFunc(startTimerCbk, stopTimerCbk);
174 21 : }
175 :
176 15 : void AicpuMonitor::SendKillMsgToTsd() const
177 : {
178 15 : aicpusd_run_info(
179 : "dev[%u] send msg to tsdaemon, tsdaemon will kill aicpu-sd process[%u]", deviceId_,
180 : static_cast<uint32_t>(getpid()));
181 15 : AicpusdLastword::GetInstance().LastwordCallback();
182 : // flush cache log to slogd
183 15 : DlogFlushAicpu();
184 15 : (void)AicpuSchedule::mpi::MpiDvppStatisticManager::Instance().PrintStatisticInfo();
185 :
186 : // thread node ppid is not tsd, don't need to destroy
187 15 : if (!online_) {
188 3 : aicpusd_run_info("offline mode no need send msg to tsd");
189 3 : return;
190 : }
191 36 : const int32_t ret = TsdDestroy(
192 12 : deviceId_, TSD_COMPUTE, static_cast<uint32_t>(AicpuDrvManager::GetInstance().GetHostPid()),
193 12 : AicpuDrvManager::GetInstance().GetVfId());
194 12 : if (ret != 0) {
195 1 : aicpusd_err("dev[%u] send abnormal msg to tsdaemon failed, ret[%d]", deviceId_, ret);
196 : }
197 : }
198 :
199 126 : void AicpuMonitor::SetTaskInfo(const uint32_t threadIndex, const TaskInfoForMonitor& taskInfo) const
200 : {
201 126 : if ((threadIndex < aicpuCoreNum_) && online_) {
202 124 : monitorTaskInfo_[static_cast<uint64_t>(threadIndex)] = taskInfo;
203 : }
204 126 : }
205 :
206 24 : int32_t AicpuMonitor::SetTaskTimeoutFlag()
207 : {
208 24 : taskTimeout_ = FeatureCtrl::IsDoubleDieProduct() ? AICPU_TASK_TIMEOUT_LONG : AICPU_TASK_TIMEOUT;
209 24 : taskTimeoutTick_ = taskTimeout_ * aicpu::GetSystemTickFreq();
210 119 : if (AicpuUtil::IsEnvValEqual(ENV_NAME_DATAMASTER_RUN_MODE, "1") ||
211 70 : AicpuUtil::IsEnvValEqual(ENV_NAME_DATAMASTER_RUN_MODE, "2")) {
212 1 : aicpusd_warn("Set task timeout to FPGA mode.");
213 1 : taskTimeout_ = AICPU_TASK_TIMEOUT_FPGA;
214 1 : taskTimeoutTick_ = AICPU_TASK_TIMEOUT_FPGA * aicpu::GetSystemTickFreq();
215 : }
216 :
217 24 : taskTimeoutFlag_ = true;
218 24 : SetOpTimeoutFlag(true);
219 :
220 24 : return AICPU_SCHEDULE_OK;
221 : }
222 :
223 24 : int32_t AicpuMonitor::SetModelTimeoutFlag()
224 : {
225 24 : std::string timeoutFlag;
226 24 : const bool ret = AicpuUtil::GetEnvVal(ENV_NAME_AICPU_MODEL_TIMEOUT, timeoutFlag);
227 24 : if (!ret) {
228 19 : aicpusd_run_info("Not set aicpu model timeout. Env is not set.");
229 19 : return AICPU_SCHEDULE_OK;
230 : }
231 :
232 5 : aicpusd_run_info("Get model timeout is %ss", timeoutFlag.c_str());
233 : try {
234 5 : const uint64_t modelTimeout = std::stoul(timeoutFlag);
235 4 : const uint64_t frep = aicpu::GetSystemTickFreq();
236 4 : if (AicpuUtil::IsUint64MulOverflow(modelTimeout, frep)) {
237 1 : aicpusd_err("Overflow occurred when set model timeout, timeout=%lus, freq=%lu.", modelTimeout, frep);
238 1 : return AICPU_SCHEDULE_ERROR_PARAMETER_NOT_VALID;
239 : }
240 3 : modelTimeoutTick_ = modelTimeout * frep;
241 1 : } catch (std::exception& e) {
242 1 : aicpusd_err("Convert AICPU_MODEL_TIMEOUT[%s]s to number failed, %s", timeoutFlag.c_str(), e.what());
243 1 : return AICPU_SCHEDULE_ERROR_COMMON_ERROR;
244 1 : }
245 :
246 3 : modelTimeoutFlag_ = true;
247 3 : return AICPU_SCHEDULE_OK;
248 24 : }
249 :
250 : // index used inner, it is valid
251 131 : void AicpuMonitor::SetTaskStartTime(const uint32_t taskId)
252 : {
253 131 : if ((!IsMonitorClosed()) && (taskTimeoutFlag_) && (online_)) {
254 130 : aicpuTaskTimer_[static_cast<uint64_t>(taskId)].SetStartTick(aicpu::GetSystemTick());
255 130 : aicpuTaskTimer_[static_cast<uint64_t>(taskId)].SetRunFlag(true);
256 : }
257 131 : }
258 :
259 129 : void AicpuMonitor::SetTaskEndTime(const uint32_t taskId)
260 : {
261 129 : if ((taskTimeoutFlag_) && (online_)) {
262 129 : aicpuTaskTimer_[static_cast<uint64_t>(taskId)].SetRunFlag(false);
263 131 : monitorTaskInfo_[static_cast<uint64_t>(taskId)].isHwts = false;
264 : }
265 131 : }
266 :
267 66 : void AicpuMonitor::SetAicpuStreamTaskStartTime(const uint32_t taskId)
268 : {
269 66 : if ((taskTimeoutFlag_) && (taskId < MAX_MODEL_COUNT) && (online_)) {
270 3 : aicpuStreamTaskTimer_[static_cast<uint64_t>(taskId)].SetStartTick(aicpu::GetSystemTick());
271 3 : aicpuStreamTaskTimer_[static_cast<uint64_t>(taskId)].SetRunFlag(true);
272 : }
273 66 : }
274 :
275 66 : void AicpuMonitor::SetAicpuStreamTaskEndTime(const uint32_t taskId)
276 : {
277 66 : if ((taskTimeoutFlag_) && (taskId < MAX_MODEL_COUNT) && (online_)) {
278 3 : aicpuStreamTaskTimer_[static_cast<uint64_t>(taskId)].SetRunFlag(false);
279 : }
280 66 : }
281 :
282 8 : void AicpuMonitor::SetModelStartTime(const uint32_t modelId)
283 : {
284 8 : if ((modelTimeoutFlag_) && (modelId < MAX_MODEL_COUNT) && (online_)) {
285 1 : modelTimer_[static_cast<uint64_t>(modelId)].SetStartTick(aicpu::GetSystemTick());
286 1 : modelTimer_[static_cast<uint64_t>(modelId)].SetRunFlag(true);
287 : }
288 8 : }
289 :
290 8 : void AicpuMonitor::SetModelEndTime(const uint32_t modelId)
291 : {
292 8 : if ((modelTimeoutFlag_) && (modelId < MAX_MODEL_COUNT) && (online_)) {
293 1 : modelTimer_[static_cast<uint64_t>(modelId)].SetRunFlag(false);
294 : }
295 8 : }
296 :
297 2 : void AicpuMonitor::SetOpTimerStartTime(const aicpu::TimerHandle timerId, const uint32_t timeInS)
298 : {
299 2 : if ((opTimeoutFlag_) && (online_)) {
300 2 : std::shared_ptr<TaskTimer> taskTimerPtr = std::make_shared<TaskTimer>(aicpu::GetSystemTick(), true);
301 2 : if (taskTimerPtr == nullptr) {
302 0 : aicpusd_err("alloc taskTimerPtr fail");
303 0 : return;
304 : }
305 2 : taskTimerPtr->SetTimeTick(timeInS);
306 : {
307 2 : const std::lock_guard<std::mutex> lk(opTimerMapMutex_);
308 2 : (void)opTimer_.emplace(timerId, taskTimerPtr);
309 2 : }
310 2 : }
311 : }
312 :
313 2 : void AicpuMonitor::SetOpTimerEndTime(const aicpu::TimerHandle timerId)
314 : {
315 2 : if ((opTimeoutFlag_ && online_) || (!opTimer_.empty())) {
316 2 : const std::lock_guard<std::mutex> lk(opTimerMapMutex_);
317 2 : (void)opTimer_.erase(timerId);
318 2 : }
319 2 : }
320 :
321 5 : void AicpuMonitor::Work(AicpuMonitor* const monitor)
322 : {
323 5 : std::once_flag exeOnce;
324 5 : uint32_t count = 0;
325 5 : AICPUSubEventInfo subEventInfo = {};
326 5 : event_summary eventInfoSummary = {};
327 5 : eventInfoSummary.pid = getpid();
328 5 : eventInfoSummary.event_id = EVENT_AICPU_MSG;
329 5 : eventInfoSummary.subevent_id = AICPU_SUB_EVENT_SUPPLY_ENQUEUE;
330 5 : eventInfoSummary.msg = PtrToPtr<AICPUSubEventInfo, char_t>(&subEventInfo);
331 5 : eventInfoSummary.msg_len = static_cast<uint32_t>(sizeof(AICPUSubEventInfo));
332 5 : eventInfoSummary.grp_id = CP_DEFAULT_GROUP_ID;
333 5 : std::vector<size_t> eventWaitIds;
334 9 : while ((monitor != nullptr) && (!monitor->done_)) {
335 : if (count % MONITOR_TIMEOUT_COUNT == 0) {
336 4 : if (monitor->online_) {
337 : // check and handle task timeout, for aicpu task of ts stream and aicpu stream
338 4 : monitor->HandleTaskTimeout();
339 : // check and handle model timeout
340 4 : monitor->HandleModelTimeout();
341 4 : monitor->HandleOpTimeout();
342 : }
343 4 : if (FeatureCtrl::ShouldMonitorWork()) {
344 4 : AicpuPulseNotify();
345 : }
346 4 : std::call_once(exeOnce, [&]() {
347 4 : monitor->running_ = true;
348 4 : if (sem_post(&monitor->sem_) != 0) {
349 0 : aicpusd_err("sem post failed, %s", strerror(errno));
350 0 : return;
351 : }
352 : });
353 : }
354 8 : if ((count % MONITOR_SUPPLYENQUE_COUNT == 0) && (FeatureCtrl::ShouldMonitorWork()) &&
355 4 : (!FeatureCtrl::ShouldSkipSupplyEvent())) {
356 4 : EventWaitManager::AnyQueNotEmptyWaitManager().GetWaitingEvent(eventWaitIds);
357 5 : for (const auto waitId : eventWaitIds) {
358 1 : subEventInfo.modelId = waitId;
359 1 : const drvError_t ret = halEschedSubmitEvent(monitor->deviceId_, &eventInfoSummary);
360 1 : if (ret != DRV_ERROR_NONE) {
361 1 : aicpusd_err("Failed to supply enque event for model[%u]. ret is %d.", waitId, ret);
362 : }
363 1 : aicpusd_info("supply enque event for model[%u].", waitId);
364 : }
365 4 : eventWaitIds.clear();
366 : }
367 4 : count++;
368 4 : if (count == MONITOR_SUPPLYENQUE_COUNT) {
369 4 : count = 0;
370 : }
371 4 : (void)usleep(MONITOR_SLEEP_INTERVAL);
372 : }
373 5 : }
374 :
375 8 : int32_t AicpuMonitor::Run()
376 : {
377 8 : if (done_) {
378 2 : aicpusd_err("Fail to run monitor for it has been stopped");
379 2 : return AICPU_SCHEDULE_ERROR_COMMON_ERROR;
380 : }
381 6 : int32_t semRet = sem_init(&sem_, 0, 0U);
382 6 : if (semRet == -1) {
383 1 : aicpusd_err("sem init failed, %s.", strerror(errno));
384 1 : return AICPU_SCHEDULE_ERROR_COMMON_ERROR;
385 : }
386 :
387 : try {
388 5 : std::thread monitorThread(&AicpuMonitor::Work, this);
389 5 : th_.emplace_back(std::move(monitorThread));
390 5 : } catch (std::exception& threadException) {
391 0 : (void)sem_destroy(&sem_);
392 0 : aicpusd_err("create aicpu monitor thread object failed, %s", threadException.what());
393 0 : return AICPU_SCHEDULE_ERROR_COMMON_ERROR;
394 0 : }
395 :
396 5 : semRet = sem_wait(&sem_);
397 5 : if (semRet == -1) {
398 1 : (void)sem_destroy(&sem_);
399 1 : aicpusd_err("sem wait failed, %s.", strerror(errno));
400 1 : return AICPU_SCHEDULE_ERROR_COMMON_ERROR;
401 : }
402 4 : (void)sem_destroy(&sem_);
403 4 : if (!running_) {
404 0 : aicpusd_err("create aicpu monitor thread failed");
405 0 : return AICPU_SCHEDULE_ERROR_COMMON_ERROR;
406 : }
407 4 : aicpusd_info("Aicpu monitor thread running");
408 :
409 4 : return AICPU_SCHEDULE_OK;
410 : }
411 :
412 58 : void AicpuMonitor::StopMonitor()
413 : {
414 58 : aicpusd_info("Begin to stop aicpu monitor");
415 58 : const std::unique_lock<std::mutex> lk(mutex_);
416 58 : done_ = true;
417 63 : for (auto& thead : th_) {
418 5 : if (thead.joinable()) {
419 5 : thead.join();
420 : }
421 : }
422 :
423 58 : done_ = false;
424 58 : th_.clear();
425 58 : }
426 :
427 10 : void AicpuMonitor::SetOpExecuteTimeOut(const uint32_t timeOutEn, const uint32_t opExecuteTimeOut)
428 : {
429 10 : const std::unique_lock<std::mutex> lk(setTimeOutMut_);
430 10 : if (timeOutEn == 1U) {
431 9 : tsTimeoutEnable_ = true;
432 9 : tsOpTimeOut_ = opExecuteTimeOut;
433 9 : taskTimeoutTick_ = opExecuteTimeOut * aicpu::GetSystemTickFreq();
434 9 : taskTimeoutFlag_ = true;
435 27 : aicpusd_run_info(
436 : "Get enable op execute timeout config from ts. timeout[%u]s tickFrequency[%lu]", tsOpTimeOut_.load(),
437 : taskTimeoutTick_.load());
438 : } else {
439 1 : tsTimeoutEnable_ = false;
440 1 : (void)SetTaskTimeoutFlag();
441 2 : aicpusd_run_info("Get disable op execute timeout config from ts. tickFrequency[%lu]", taskTimeoutTick_.load());
442 : }
443 10 : }
444 :
445 30 : void AicpuMonitor::SetOpTimeoutFlag(const bool flag)
446 : {
447 30 : aicpusd_info("Set op timer flag to %d", flag);
448 30 : opTimeoutFlag_ = flag;
449 30 : aicpu::AicpuTimer::GetInstance().SetSupportTimer(flag);
450 30 : }
451 :
452 8 : void AicpuMonitor::HandleTaskTimeout()
453 : {
454 8 : if (!taskTimeoutFlag_ || IsMonitorClosed()) {
455 2 : return;
456 : }
457 :
458 6 : const uint64_t nowTick = aicpu::GetSystemTick();
459 20 : for (uint32_t i = 0U; i < aicpuCoreNum_; ++i) {
460 15 : const auto runFlag = aicpuTaskTimer_[static_cast<uint64_t>(i)].GetRunFlag();
461 15 : const auto startTick = aicpuTaskTimer_[static_cast<uint64_t>(i)].GetStartTick();
462 16 : if (runFlag && (nowTick > startTick) && ((nowTick - startTick) >= taskTimeoutTick_.load())) {
463 : // handle task timeout
464 1 : std::string opname;
465 1 : (void)aicpu::GetOpname(i, opname);
466 1 : std::ostringstream oss;
467 1 : oss << "Send timeout to tsdaemon, tsdaemon will kill aicpu-sd process, thread index[" << i << "], op name["
468 1 : << opname << "]";
469 1 : if (monitorTaskInfo_[static_cast<uint64_t>(i)].isHwts) {
470 0 : oss << ", " << MonitorDebug::MonitorDebugString(monitorTaskInfo_[static_cast<uint64_t>(i)]);
471 : }
472 2 : aicpusd_err(
473 : "%s, nowTick:%llu, startTick:%llu, timeOut:%llu, tickFreq:%llu.", oss.str().c_str(), nowTick, startTick,
474 : taskTimeoutTick_.load(), aicpu::GetSystemTickFreq());
475 1 : aicpusd_run_info("%s", ComputeProcess::GetInstance().DebugString().c_str());
476 1 : SendKillMsgToTsd();
477 1 : return;
478 1 : }
479 : }
480 4101 : for (uint32_t modelId = 0U; modelId < MAX_MODEL_COUNT; ++modelId) {
481 4097 : const auto runFlag = aicpuStreamTaskTimer_[static_cast<uint64_t>(modelId)].GetRunFlag();
482 4097 : const auto startTick = aicpuStreamTaskTimer_[static_cast<uint64_t>(modelId)].GetStartTick();
483 4098 : if (runFlag && (nowTick > startTick) && ((nowTick - startTick) >= taskTimeoutTick_.load())) {
484 : // handle aicpu stream task timeout
485 1 : aicpusd_err("Send stream task timeout, tsdaemon will kill aicpu-sd process, model id[%u].", modelId);
486 1 : SendKillMsgToTsd();
487 1 : break;
488 : }
489 : }
490 : }
491 :
492 5 : void AicpuMonitor::HandleOpTimeout()
493 : {
494 5 : if ((!opTimeoutFlag_) && (opTimer_.empty())) {
495 0 : return;
496 : }
497 :
498 5 : aicpu::TimerHandle timerId = 0;
499 5 : bool isTimeout = false;
500 : {
501 5 : const std::lock_guard<std::mutex> lk(opTimerMapMutex_);
502 5 : const uint64_t nowTick = aicpu::GetSystemTick();
503 6 : for (auto& timer : opTimer_) {
504 1 : const auto runFlag = timer.second->GetRunFlag();
505 1 : const auto startTick = timer.second->GetStartTick();
506 : const auto timeoutTick =
507 1 : timer.second->GetTimeTick() == 0 ? taskTimeoutTick_.load() : timer.second->GetTimeTick();
508 1 : if (runFlag && (nowTick > startTick) && ((nowTick - startTick) >= timeoutTick)) {
509 1 : timerId = timer.first;
510 1 : isTimeout = true;
511 1 : std::ostringstream oss;
512 1 : oss << "Op timeout occurred, timer id[" << timerId << "]";
513 2 : aicpusd_err(
514 : "%s, nowTick:%llu, startTick:%llu, timeOut:%llu, tickFreq:%llu.", oss.str().c_str(), nowTick,
515 : startTick, taskTimeoutTick_.load(), aicpu::GetSystemTickFreq());
516 1 : aicpu::AicpuTimer::GetInstance().CallTimeoutCallback(timerId);
517 1 : }
518 : }
519 5 : }
520 :
521 5 : if (isTimeout) {
522 1 : aicpusd_err("Timeout occurred, start to stop timer. TimerId=%lu", timerId);
523 1 : SetOpTimerEndTime(timerId);
524 1 : DlogFlushAicpu();
525 : }
526 : }
527 :
528 5 : void AicpuMonitor::HandleModelTimeout()
529 : {
530 5 : if (modelTimeoutFlag_) {
531 2 : const uint64_t nowTick = aicpu::GetSystemTick();
532 1026 : for (uint32_t modelId = 0U; modelId < MAX_MODEL_COUNT; ++modelId) {
533 : const TaskTimer timer(
534 1025 : modelTimer_[static_cast<uint64_t>(modelId)].GetStartTick(),
535 1025 : modelTimer_[static_cast<uint64_t>(modelId)].GetRunFlag());
536 1026 : if (timer.GetRunFlag() && (nowTick > timer.GetStartTick()) &&
537 1 : ((nowTick - timer.GetStartTick()) >= modelTimeoutTick_)) {
538 : // handle model timeout
539 1 : aicpusd_err("Send model timeout, tsdaemon will kill aicpu-sd process, model id[%u].", modelId);
540 1 : SendKillMsgToTsd();
541 1 : break;
542 : }
543 : }
544 : }
545 5 : }
546 :
547 28 : void AicpuMonitor::DisableModelTimeout() { modelTimeoutFlag_ = false; }
548 :
549 27 : uint32_t AicpuMonitor::GetTaskDefaultTimeout() const { return tsTimeoutEnable_ ? tsOpTimeOut_.load() : taskTimeout_; }
550 : } // namespace AicpuSchedule
|
