Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 : #include "awatchdog_monitor.h"
11 : #include "adiag_list.h"
12 : #include "adiag_print.h"
13 : #include "adiag_utils.h"
14 : #include "awatchdog.h"
15 : #include "awatchdog_core.h"
16 : #include "mmpa_api.h"
17 :
18 : #define MAX_TIMEOUT_COUNT 0xFFFFFFF
19 : #define AWD_DEFAULT_MONITOR_PERIOD 200000U // 200ms, must be smaller than 1s
20 : #define MAX_THREAD_TASK_PATH_LENGTH 50U
21 : #define ADIAG_NS_TO_US 1000U
22 : #define ADIAG_US_TO_MS 1000U
23 : #define ADIAG_MS_TO_S 1000U
24 : #define AWD_THREAD_STACK_SIZE (128 * 1024)
25 :
26 : typedef struct AwdMonitorInfo {
27 : uint32_t watchdogCount; // monitor loop count
28 : uint32_t watchdogPeriod; // watchdogCount * 最大公约数 <= watchdogPeriod，相等时，watchdogCount 归零，执行一次监控
29 : void (*watchdogFunc)(void);
30 : } AwdMonitorInfo;
31 :
32 : #define THREAD_STATUS_INIT 0
33 : #define THREAD_STATUS_RUN 1
34 : #define THREAD_STATUS_WAIT_EXIT 2
35 :
36 : static AwdMonitorInfo g_awdMonitorInfo[AWD_WATCHDOG_TYPE_MAX] = {{0}};
37 : static int32_t g_awdMonitorThreadStatus = THREAD_STATUS_INIT;
38 : static uint32_t g_awdMonitorResPeriod = AWD_DEFAULT_MONITOR_PERIOD;
39 : static mmThread g_awdMonitorThreadId = 0;
40 :
41 1 : STATIC INLINE uint32_t AwdGetWatchDogModuleId(uint32_t dogId)
42 : {
43 1 : return (dogId & (((uint32_t)1U << WATCHDOG_TYPE_BIT) - 1U));
44 : }
45 :
46 : // Maximum common divisor
47 1 : STATIC INLINE uint32_t AwdMonitorGetCommonDivisor(uint32_t x, uint32_t y)
48 : {
49 1 : uint32_t a = x;
50 1 : uint32_t b = y;
51 1 : while (a != b) {
52 0 : if (a > b) {
53 0 : a = a - b;
54 : } else {
55 0 : b = b - a;
56 : }
57 : }
58 1 : return a;
59 : }
60 :
61 1 : STATIC INLINE void AwdMonitorGetPeriod(void)
62 : {
63 1 : uint32_t temp = g_awdMonitorInfo[0].watchdogPeriod;
64 2 : for (int32_t i = 0; i < (int32_t)AWD_WATCHDOG_TYPE_MAX; i++) {
65 1 : temp = AwdMonitorGetCommonDivisor(temp, g_awdMonitorInfo[i].watchdogPeriod);
66 1 : ADIAG_DBG("watchdog %d, period %u, resPeriod %u", i, g_awdMonitorInfo[i].watchdogPeriod, temp);
67 : }
68 1 : g_awdMonitorResPeriod = temp;
69 1 : }
70 :
71 0 : STATIC INLINE void AwdMonitorResItem(void)
72 : {
73 0 : for (int32_t i = 0; i < (int32_t)AWD_WATCHDOG_TYPE_MAX; i++) {
74 0 : g_awdMonitorInfo[i].watchdogCount++;
75 0 : if (g_awdMonitorInfo[i].watchdogCount * g_awdMonitorResPeriod >= g_awdMonitorInfo[i].watchdogPeriod) {
76 0 : g_awdMonitorInfo[i].watchdogFunc();
77 0 : g_awdMonitorInfo[i].watchdogCount = 0;
78 : }
79 : }
80 0 : }
81 :
82 1 : STATIC void* AwdMonitorProcess(void* arg)
83 : {
84 : (void)arg;
85 1 : AwdMonitorGetPeriod();
86 1 : const uint32_t period = g_awdMonitorResPeriod;
87 1 : uint64_t start = GetRealTime() / ADIAG_NS_TO_US;
88 : uint64_t end;
89 : uint32_t sleepTime;
90 1 : (void)mmSetCurrentThreadName("WatchdogMonitor");
91 1 : while (g_awdMonitorThreadStatus == THREAD_STATUS_RUN) {
92 0 : AwdMonitorResItem();
93 0 : end = GetRealTime() / ADIAG_NS_TO_US;
94 0 : uint32_t duration = (uint32_t)(end - start);
95 0 : if (duration >= period) {
96 : static bool warningFlag = true;
97 0 : if (warningFlag) {
98 0 : ADIAG_WAR(
99 : "awd monitor res period %uus is less than process duration %uus, may need to be increased", period,
100 : duration);
101 0 : warningFlag = false;
102 : }
103 0 : start = start + duration;
104 0 : continue;
105 : }
106 0 : sleepTime = period - duration;
107 0 : (void)usleep(sleepTime);
108 0 : start = start + sleepTime;
109 : }
110 1 : AwdWatchDog* awd = AwdGetWatchDog(AWD_WATCHDOG_TYPE_THREAD);
111 1 : (void)AdiagListDestroy(&awd->runList);
112 1 : (void)AdiagListDestroy(&awd->newList);
113 1 : return NULL;
114 : }
115 :
116 3 : STATIC INLINE AwdStatus AwdCreateMonitorThread(void)
117 : {
118 : mmUserBlock_t thread;
119 3 : thread.procFunc = AwdMonitorProcess;
120 3 : thread.pulArg = NULL;
121 3 : mmThreadAttr attr = {0, 0, 0, 0, 0, 0, AWD_THREAD_STACK_SIZE};
122 3 : mmThread tid = 0;
123 3 : if (mmCreateTaskWithThreadAttr(&tid, &thread, &attr) != 0) {
124 2 : ADIAG_ERR("create task failed, strerr=%s.", strerror(errno));
125 2 : return AWD_FAILURE;
126 : }
127 1 : g_awdMonitorThreadId = tid;
128 1 : ADIAG_INF("create monitor thread successfully, tid :%ld ", tid);
129 1 : return AWD_SUCCESS;
130 : }
131 :
132 1 : STATIC INLINE bool CheckThreadExist(AwdThreadWatchdog* node)
133 : {
134 1 : char dir[MAX_THREAD_TASK_PATH_LENGTH] = {0};
135 1 : int ret = sprintf_s(dir, MAX_THREAD_TASK_PATH_LENGTH, "/proc/%d/task/%d", node->pid, node->tid);
136 1 : if (ret == -1) {
137 0 : ADIAG_ERR("sprintf thread dir failed", strerror(errno));
138 0 : return true;
139 : }
140 1 : if (access(dir, F_OK) != 0) {
141 0 : return false;
142 : }
143 1 : return true;
144 : }
145 :
146 1 : STATIC void AwdProcessTimeout(AwdThreadWatchdog* node)
147 : {
148 1 : AWD_ATOMIC_TEST_AND_SET(&node->startCount, AWD_STATUS_INIT);
149 : // record timeout log to run info
150 1 : ADIAG_RUN_INF("watchdog timeout, moduleId %u, timeout : %us", AwdGetWatchDogModuleId(node->dogId), node->timeout);
151 1 : if (node->callback != NULL) {
152 1 : node->callback(NULL);
153 : }
154 1 : }
155 :
156 1 : STATIC AdiagStatus AwdMonitorThreadWatchdogPro(void* data)
157 : {
158 1 : AwdThreadWatchdog* node = (AwdThreadWatchdog*)data;
159 1 : int32_t startCount = node->startCount;
160 1 : if (!CheckThreadExist(node)) {
161 0 : ADIAG_DBG("thread %d does not exist", node->tid);
162 0 : AWD_ATOMIC_TEST_AND_SET(&node->startCount, AWD_STATUS_DESTROYED);
163 0 : return AWD_SUCCESS;
164 : }
165 1 : if (startCount == AWD_STATUS_INIT) {
166 0 : return AWD_SUCCESS;
167 : }
168 : // process runCount
169 1 : int32_t runCount = node->runCount + 1;
170 1 : if (runCount == MAX_TIMEOUT_COUNT) {
171 0 : runCount = 0;
172 : }
173 1 : AWD_ATOMIC_TEST_AND_SET(&node->runCount, runCount);
174 : // calculate duration time
175 : uint64_t duration;
176 1 : if (runCount < startCount) {
177 0 : duration = (uint64_t)runCount + (uint64_t)MAX_TIMEOUT_COUNT - (uint64_t)startCount;
178 : } else {
179 1 : duration = (uint64_t)runCount - (uint64_t)startCount;
180 : }
181 : // add 1 second to compensate the time of start
182 1 : duration = (duration + 1U) * (uint64_t)g_awdMonitorInfo[AWD_WATCHDOG_TYPE_THREAD].watchdogPeriod / ADIAG_US_TO_MS;
183 1 : if (duration / ADIAG_MS_TO_S >= node->timeout) {
184 1 : AwdProcessTimeout(node);
185 : }
186 1 : return AWD_SUCCESS;
187 : }
188 :
189 1 : STATIC INLINE AdiagStatus CheckNodeDestroyed(const void* nodeData, const void* data)
190 : {
191 : (void)data;
192 1 : const AwdThreadWatchdog* dog = (const AwdThreadWatchdog*)nodeData;
193 1 : if (dog->startCount == AWD_STATUS_DESTROYED) {
194 1 : return ADIAG_SUCCESS;
195 : }
196 0 : return ADIAG_FAILURE;
197 : }
198 :
199 1 : STATIC INLINE void AwdMonitorThreadWatchdog(void)
200 : {
201 1 : AwdWatchDog* awd = AwdGetWatchDog(AWD_WATCHDOG_TYPE_THREAD);
202 : // move watchdog node from newList to runList
203 1 : AdiagListMove(&awd->newList, &awd->runList);
204 :
205 1 : (void)AdiagListClearAndProcessNoLock(&awd->runList, CheckNodeDestroyed, NULL, AwdMonitorThreadWatchdogPro);
206 1 : }
207 :
208 4 : void AwdMonitorReset(void) { g_awdMonitorThreadStatus = THREAD_STATUS_WAIT_EXIT; }
209 :
210 3 : AwdStatus AwdMonitorInit(void)
211 : {
212 3 : g_awdMonitorInfo[AWD_WATCHDOG_TYPE_THREAD].watchdogPeriod = AWD_DEFAULT_MONITOR_PERIOD;
213 3 : g_awdMonitorInfo[AWD_WATCHDOG_TYPE_THREAD].watchdogCount = 0;
214 3 : g_awdMonitorInfo[AWD_WATCHDOG_TYPE_THREAD].watchdogFunc = AwdMonitorThreadWatchdog;
215 :
216 3 : g_awdMonitorThreadStatus = THREAD_STATUS_RUN;
217 3 : AwdStatus ret = AwdCreateMonitorThread();
218 3 : if (ret != AWD_SUCCESS) {
219 2 : g_awdMonitorThreadId = 0;
220 2 : g_awdMonitorThreadStatus = THREAD_STATUS_INIT;
221 2 : ADIAG_ERR("Create monitor thread failed");
222 2 : return AWD_FAILURE;
223 : }
224 1 : ADIAG_INF("AwdMonitorInit");
225 1 : return AWD_SUCCESS;
226 : }
227 :
228 20 : void AwdMonitorExit(void)
229 : {
230 20 : if (g_awdMonitorThreadStatus != THREAD_STATUS_RUN) {
231 19 : return;
232 : }
233 1 : ADIAG_INF("AwdMonitorExit");
234 1 : g_awdMonitorThreadStatus = THREAD_STATUS_WAIT_EXIT;
235 1 : if (g_awdMonitorThreadId != 0) {
236 1 : (void)mmJoinTask(&g_awdMonitorThreadId);
237 1 : g_awdMonitorThreadId = 0;
238 : }
239 : }
|
