Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 : #include "awatchdog_monitor.h"
11 : #include "adiag_list.h"
12 : #include "adiag_print.h"
13 : #include "adiag_utils.h"
14 : #include "awatchdog.h"
15 : #include "awatchdog_core.h"
16 : #include "mmpa_api.h"
17 :
18 : #define MAX_TIMEOUT_COUNT 0xFFFFFFF
19 : #define AWD_DEFAULT_MONITOR_PERIOD 200000U // 200ms, must be smaller than 1s
20 : #define MAX_THREAD_TASK_PATH_LENGTH 50U
21 : #define ADIAG_NS_TO_US 1000U
22 : #define ADIAG_US_TO_MS 1000U
23 : #define ADIAG_MS_TO_S 1000U
24 : #define AWD_THREAD_STACK_SIZE 128 * 1024
25 :
26 : typedef struct AwdMonitorInfo {
27 : uint32_t watchdogCount; // monitor loop count
28 : uint32_t watchdogPeriod; // watchdogCount * 最大公约数 <= watchdogPeriod，相等时，watchdogCount 归零，执行一次监控
29 : void (*watchdogFunc)(void);
30 : } AwdMonitorInfo;
31 :
32 : #define THREAD_STATUS_INIT 0
33 : #define THREAD_STATUS_RUN 1
34 : #define THREAD_STATUS_WAIT_EXIT 2
35 :
36 : static AwdMonitorInfo g_awdMonitorInfo[AWD_WATCHDOG_TYPE_MAX] = {{0}};
37 : static int32_t g_awdMonitorThreadStatus = THREAD_STATUS_INIT;
38 : static uint32_t g_awdMonitorResPeriod = AWD_DEFAULT_MONITOR_PERIOD;
39 : static mmThread g_awdMonitorThreadId = 0;
40 :
41 0 : STATIC INLINE uint32_t AwdGetWatchDogModuleId(uint32_t dogId)
42 : {
43 0 : return (dogId & (((uint32_t)1U << WATCHDOG_TYPE_BIT) - 1U));
44 : }
45 :
46 : // Maximum common divisor
47 0 : STATIC INLINE uint32_t AwdMonitorGetCommonDivisor(uint32_t x, uint32_t y)
48 : {
49 0 : uint32_t a = x;
50 0 : uint32_t b = y;
51 0 : while (a != b) {
52 0 : if (a > b) {
53 0 : a = a - b;
54 : } else {
55 0 : b = b - a;
56 : }
57 : }
58 0 : return a;
59 : }
60 :
61 0 : STATIC INLINE void AwdMonitorGetPeriod(void)
62 : {
63 0 : uint32_t temp = g_awdMonitorInfo[0].watchdogPeriod;
64 0 : for (int32_t i = 0; i < (int32_t)AWD_WATCHDOG_TYPE_MAX; i++) {
65 0 : temp = AwdMonitorGetCommonDivisor(temp, g_awdMonitorInfo[i].watchdogPeriod);
66 0 : ADIAG_DBG("watchdog %d, period %u, resPeriod %u", i, g_awdMonitorInfo[i].watchdogPeriod, temp);
67 : }
68 0 : g_awdMonitorResPeriod = temp;
69 0 : }
70 :
71 0 : STATIC INLINE void AwdMonitorResItem(void)
72 : {
73 0 : for (int32_t i = 0; i < (int32_t)AWD_WATCHDOG_TYPE_MAX; i++) {
74 0 : g_awdMonitorInfo[i].watchdogCount++;
75 0 : if (g_awdMonitorInfo[i].watchdogCount * g_awdMonitorResPeriod >= g_awdMonitorInfo[i].watchdogPeriod) {
76 0 : g_awdMonitorInfo[i].watchdogFunc();
77 0 : g_awdMonitorInfo[i].watchdogCount = 0;
78 : }
79 : }
80 0 : }
81 :
82 0 : STATIC void *AwdMonitorProcess(void *arg)
83 : {
84 : (void)arg;
85 0 : AwdMonitorGetPeriod();
86 0 : const uint32_t period = g_awdMonitorResPeriod;
87 0 : uint64_t start = GetRealTime() / ADIAG_NS_TO_US;
88 : uint64_t end;
89 : uint32_t sleepTime;
90 0 : (void)mmSetCurrentThreadName("WatchdogMonitor");
91 0 : while (g_awdMonitorThreadStatus == THREAD_STATUS_RUN) {
92 0 : AwdMonitorResItem();
93 0 : end = GetRealTime() / ADIAG_NS_TO_US;
94 0 : uint32_t duration = (uint32_t)(end - start);
95 0 : if (duration >= period) {
96 : static bool warningFlag = true;
97 0 : if (warningFlag) {
98 0 : ADIAG_WAR("awd monitor res period %uus is less than process duration %uus, may need to be increased",
99 : period, duration);
100 0 : warningFlag = false;
101 : }
102 0 : start = start + duration;
103 0 : continue;
104 : }
105 0 : sleepTime = period - duration;
106 0 : (void)usleep(sleepTime);
107 0 : start = start + sleepTime;
108 : }
109 0 : AwdWatchDog *awd = AwdGetWatchDog(AWD_WATCHDOG_TYPE_THREAD);
110 0 : (void)AdiagListDestroy(&awd->runList);
111 0 : (void)AdiagListDestroy(&awd->newList);
112 0 : return NULL;
113 : }
114 :
115 2 : STATIC INLINE AwdStatus AwdCreateMonitorThread(void)
116 : {
117 : mmUserBlock_t thread;
118 2 : thread.procFunc = AwdMonitorProcess;
119 2 : thread.pulArg = NULL;
120 2 : mmThreadAttr attr = { 0, 0, 0, 0, 0, 0, AWD_THREAD_STACK_SIZE };
121 2 : mmThread tid = 0;
122 2 : if (mmCreateTaskWithThreadAttr(&tid, &thread, &attr) != 0) {
123 1 : ADIAG_ERR("create task failed, strerr=%s.", strerror(errno));
124 1 : return AWD_FAILURE;
125 : }
126 1 : g_awdMonitorThreadId = tid;
127 1 : ADIAG_INF("create monitor thread successfully, tid :%ld ", tid);
128 1 : return AWD_SUCCESS;
129 : }
130 :
131 0 : STATIC INLINE bool CheckThreadExist(AwdThreadWatchdog *node)
132 : {
133 0 : char dir[MAX_THREAD_TASK_PATH_LENGTH] = {0};
134 0 : int ret = sprintf_s(dir, MAX_THREAD_TASK_PATH_LENGTH, "/proc/%d/task/%d", node->pid, node->tid);
135 0 : if (ret == -1) {
136 0 : ADIAG_ERR("sprintf thread dir failed", strerror(errno));
137 0 : return true;
138 : }
139 0 : if (access(dir, F_OK) != 0) {
140 0 : return false;
141 : }
142 0 : return true;
143 : }
144 :
145 0 : STATIC void AwdProcessTimeout(AwdThreadWatchdog *node)
146 : {
147 0 : AWD_ATOMIC_TEST_AND_SET(&node->startCount, AWD_STATUS_INIT);
148 : // record timeout log to run info
149 0 : ADIAG_RUN_INF("watchdog timeout, moduleId %u, timeout : %us", AwdGetWatchDogModuleId(node->dogId), node->timeout);
150 0 : if (node->callback != NULL) {
151 0 : node->callback(NULL);
152 : }
153 0 : }
154 :
155 0 : STATIC AdiagStatus AwdMonitorThreadWatchdogPro(void *data)
156 : {
157 0 : AwdThreadWatchdog *node = (AwdThreadWatchdog *)data;
158 0 : int32_t startCount = node->startCount;
159 0 : if (!CheckThreadExist(node)) {
160 0 : ADIAG_DBG("thread %d does not exist", node->tid);
161 0 : AWD_ATOMIC_TEST_AND_SET(&node->startCount, AWD_STATUS_DESTROYED);
162 0 : return AWD_SUCCESS;
163 : }
164 0 : if (startCount == AWD_STATUS_INIT) {
165 0 : return AWD_SUCCESS;
166 : }
167 : // process runCount
168 0 : int32_t runCount = node->runCount + 1;
169 0 : if (runCount == MAX_TIMEOUT_COUNT) {
170 0 : runCount = 0;
171 : }
172 0 : AWD_ATOMIC_TEST_AND_SET(&node->runCount, runCount);
173 : // calculate duration time
174 : uint64_t duration;
175 0 : if (runCount < startCount) {
176 0 : duration = (uint64_t)runCount + (uint64_t)MAX_TIMEOUT_COUNT - (uint64_t)startCount;
177 : } else {
178 0 : duration = (uint64_t)runCount - (uint64_t)startCount;
179 : }
180 : // add 1 second to compensate the time of start
181 0 : duration = (duration + 1U) * (uint64_t)g_awdMonitorInfo[AWD_WATCHDOG_TYPE_THREAD].watchdogPeriod / ADIAG_US_TO_MS;
182 0 : if (duration / ADIAG_MS_TO_S >= node->timeout) {
183 0 : AwdProcessTimeout(node);
184 : }
185 0 : return AWD_SUCCESS;
186 : }
187 :
188 0 : STATIC INLINE AdiagStatus CheckNodeDestroyed(const void *nodeData, const void *data)
189 : {
190 : (void)data;
191 0 : const AwdThreadWatchdog *dog = (const AwdThreadWatchdog *)nodeData;
192 0 : if (dog->startCount == AWD_STATUS_DESTROYED) {
193 0 : return ADIAG_SUCCESS;
194 : }
195 0 : return ADIAG_FAILURE;
196 : }
197 :
198 0 : STATIC INLINE void AwdMonitorThreadWatchdog(void)
199 : {
200 0 : AwdWatchDog *awd = AwdGetWatchDog(AWD_WATCHDOG_TYPE_THREAD);
201 : // move watchdog node from newList to runList
202 0 : AdiagListMove(&awd->newList, &awd->runList);
203 :
204 0 : (void)AdiagListClearAndProcessNoLock(&awd->runList, CheckNodeDestroyed, NULL, AwdMonitorThreadWatchdogPro);
205 0 : }
206 :
207 2 : void AwdMonitorReset(void)
208 : {
209 2 : g_awdMonitorThreadStatus = THREAD_STATUS_WAIT_EXIT;
210 2 : }
211 :
212 2 : AwdStatus AwdMonitorInit(void)
213 : {
214 2 : g_awdMonitorInfo[AWD_WATCHDOG_TYPE_THREAD].watchdogPeriod = AWD_DEFAULT_MONITOR_PERIOD;
215 2 : g_awdMonitorInfo[AWD_WATCHDOG_TYPE_THREAD].watchdogCount = 0;
216 2 : g_awdMonitorInfo[AWD_WATCHDOG_TYPE_THREAD].watchdogFunc = AwdMonitorThreadWatchdog;
217 :
218 2 : g_awdMonitorThreadStatus = THREAD_STATUS_RUN;
219 2 : AwdStatus ret = AwdCreateMonitorThread();
220 2 : if (ret != AWD_SUCCESS) {
221 1 : g_awdMonitorThreadId = 0;
222 1 : g_awdMonitorThreadStatus = THREAD_STATUS_INIT;
223 1 : ADIAG_ERR("Create monitor thread failed");
224 1 : return AWD_FAILURE;
225 : }
226 1 : ADIAG_INF("AwdMonitorInit");
227 1 : return AWD_SUCCESS;
228 : }
229 :
230 19 : void AwdMonitorExit(void)
231 : {
232 19 : if (g_awdMonitorThreadStatus != THREAD_STATUS_RUN) {
233 18 : return;
234 : }
235 1 : ADIAG_INF("AwdMonitorExit");
236 1 : g_awdMonitorThreadStatus = THREAD_STATUS_WAIT_EXIT;
237 1 : if (g_awdMonitorThreadId != 0) {
238 1 : (void)mmJoinTask(&g_awdMonitorThreadId);
239 1 : g_awdMonitorThreadId = 0;
240 : }
241 : }
|
