Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 : #include "ktrace_ts.h"
11 : #include "adiag_lock.h"
12 : #include "adiag_print.h"
13 : #include "adiag_utils.h"
14 : #include "ascend_hal.h"
15 : #include "trace_msg.h"
16 : #include "trace_node.h"
17 : #include "trace_server_mgr.h"
18 : #include "trace_system_api.h"
19 : #include "trace_types.h"
20 :
21 : #define MAX_DEV_NUM 64
22 : #define DRV_RECV_MAX_LEN 524288
23 : #define TRACE_RECV_TIMEOUT 500
24 : #define GET_DEV_ID_RETRY_INTERVAL 1000 * 1000
25 : #define GET_DEV_ID_RETRY_TIMES 10
26 :
27 : typedef struct {
28 : TraceThread tid;
29 : uint32_t devId;
30 : int32_t status;
31 : } ThreadInfo;
32 :
33 : typedef struct {
34 : uint32_t pid;
35 : uint32_t logSize;
36 : uint8_t endFlag; // 0 start; 1 mid; 2 end
37 : uint8_t reserve[23];
38 : } TraceInfoHead;
39 :
40 : STATIC ThreadInfo** g_ktraceTsThread = NULL;
41 : STATIC AdiagLock g_ktraceTsLock = TRACE_MUTEX_INITIALIZER;
42 : STATIC int32_t g_ktraceTsStatus = 0;
43 : STATIC uint32_t g_devNum = 0;
44 :
45 : /**
46 : * @brief get event time when each event start.
47 : * @param [in] sessionNode: sessionNode
48 : * @param [in] endFlag: event flag
49 : * @param [out] time: event time
50 : * @param [in] len: time length
51 : * @return TraStatus
52 : */
53 23 : STATIC TraStatus KtraceGetEventTime(SessionNode* sessionNode, uint8_t endFlag, char* time, uint32_t len)
54 : {
55 23 : if ((strlen(sessionNode->eventTime) == 0) || (endFlag == ADIAG_INFO_FLAG_START)) {
56 21 : (void)memset_s(sessionNode->eventTime, TIMESTAMP_MAX_LENGTH, 0, TIMESTAMP_MAX_LENGTH);
57 21 : TraStatus ret = TimestampToFileStr(GetRealTime(), sessionNode->eventTime, TIMESTAMP_MAX_LENGTH);
58 21 : if (ret != TRACE_SUCCESS) {
59 1 : ADIAG_ERR("get timestamp failed, ret = %d.", ret);
60 1 : return TRACE_FAILURE;
61 : }
62 : }
63 22 : errno_t err = strncpy_s(time, len, sessionNode->eventTime, TIMESTAMP_MAX_LENGTH);
64 22 : if (err != EOK) {
65 1 : ADIAG_ERR("strncpy timestamp failed, err = %d, strerr = %s", (int32_t)err, strerror(AdiagGetErrorCode()));
66 1 : return TRACE_FAILURE;
67 : }
68 21 : if (endFlag == ADIAG_INFO_FLAG_END) {
69 19 : (void)memset_s(sessionNode->eventTime, TIMESTAMP_MAX_LENGTH, 0, TIMESTAMP_MAX_LENGTH);
70 : }
71 21 : return TRACE_SUCCESS;
72 : }
73 :
74 397 : STATIC TraStatus KtracePushDataToNode(TraceInfoHead* head, TraceEventMsg* eventMsg)
75 : {
76 397 : SessionNode* sessionNode = TraceServerGetSessionNode((int32_t)head->pid, (int32_t)eventMsg->devId);
77 397 : if (sessionNode == NULL) {
78 374 : ADIAG_WAR("no session node is valid, pid = %u.", head->pid);
79 374 : return TRACE_FAILURE;
80 : }
81 23 : TraStatus ret = KtraceGetEventTime(sessionNode, head->endFlag, eventMsg->eventTime, TIMESTAMP_MAX_LENGTH);
82 23 : if (ret != TRACE_SUCCESS) {
83 2 : ADIAG_ERR("get event time failed, ret = %d, pid = %u.", ret, head->pid);
84 2 : return TRACE_FAILURE;
85 : }
86 :
87 21 : ret = TraceTsPushNode(sessionNode, head->endFlag, (void*)eventMsg, eventMsg->bufLen + sizeof(TraceEventMsg));
88 21 : if (ret != TRACE_SUCCESS) {
89 2 : if (ret == TRACE_QUEUE_FULL) {
90 2 : ADIAG_WAR("queue is full.");
91 : } else {
92 0 : ADIAG_ERR("push node failed, ret = %d, pid = %u.", ret, head->pid);
93 : }
94 2 : return TRACE_FAILURE;
95 : }
96 19 : return TRACE_SUCCESS;
97 : }
98 :
99 : /**
100 : * @brief parse data from ts and construct to event msg.
101 : * @param [in] traceInfo: data from ts
102 : * @param [in] len: length of data from ts
103 : * @param [in] devId: device id
104 : * @return NA
105 : */
106 400 : STATIC void KtraceDataProcess(char* traceInfo, uint32_t len, uint32_t localDevId, uint32_t devId)
107 : {
108 400 : ADIAG_CHK_EXPR_ACTION(
109 : (len > DRV_RECV_MAX_LEN) || (len < sizeof(TraceInfoHead)), return, "invalid length[%u] from driver", len);
110 400 : TraceInfoHead* head = (TraceInfoHead*)traceInfo;
111 400 : ADIAG_CHK_EXPR_ACTION(
112 : head->logSize >= len, return, "data len[%u] is over max len[%u] from ts.", head->logSize, len);
113 :
114 399 : TraceEventMsg* eventMsg = (TraceEventMsg*)AdiagMalloc(sizeof(TraceEventMsg) + head->logSize + 1U);
115 399 : if (eventMsg == NULL) {
116 0 : ADIAG_ERR("malloc for event msg failed.");
117 0 : return;
118 : }
119 399 : eventMsg->msgType = TRACE_EVENT_MSG;
120 399 : eventMsg->eventType = 0;
121 399 : eventMsg->devId = localDevId;
122 399 : eventMsg->pid = (int32_t)head->pid;
123 399 : eventMsg->seqFlag = head->endFlag;
124 399 : eventMsg->sequence = 0;
125 399 : eventMsg->bufLen = head->logSize;
126 399 : errno_t err = memcpy_s(eventMsg->buf, head->logSize + 1U, traceInfo + sizeof(TraceInfoHead), head->logSize);
127 399 : if (err != EOK) {
128 1 : ADIAG_ERR("memcpy failed, err = %d, strerr = %s.", (int32_t)err, strerror(AdiagGetErrorCode()));
129 1 : ADIAG_SAFE_FREE(eventMsg);
130 1 : return;
131 : }
132 398 : int32_t ret = snprintf_s(eventMsg->eventName, EVENT_NAME_MAX_LENGTH, EVENT_NAME_MAX_LENGTH - 1U, "ts_%u", devId);
133 398 : if (ret == -1) {
134 1 : ADIAG_ERR("snprintf failed, ret = %d, strerr = %s.", ret, strerror(AdiagGetErrorCode()));
135 1 : ADIAG_SAFE_FREE(eventMsg);
136 1 : return;
137 : }
138 397 : TraceServerSessionLock();
139 397 : ret = KtracePushDataToNode(head, eventMsg);
140 397 : TraceServerSessionUnlock();
141 397 : if (ret != TRACE_SUCCESS) {
142 378 : ADIAG_SAFE_FREE(eventMsg);
143 378 : return;
144 : }
145 19 : ADIAG_INF(
146 : "log read by type successfully, pid = %d, data len = %u bytes, end flag = %hhu.", eventMsg->pid,
147 : eventMsg->bufLen, eventMsg->seqFlag);
148 19 : return;
149 : }
150 :
151 : /**
152 : * @brief convert device-side devId to host-side devId
153 : * @param [in] localDeviceId: chip ID
154 : * @return host side devId (physical id)
155 : */
156 26 : STATIC uint32_t KtraceGetPhysicalDeviceID(uint32_t localDeviceId)
157 : {
158 26 : uint32_t phyDeviceId = 0;
159 26 : drvError_t ret = DRV_ERROR_NONE;
160 26 : uint32_t retryTime = 0;
161 : do {
162 36 : ret = drvGetDevIDByLocalDevID(localDeviceId, &phyDeviceId);
163 36 : if (ret == DRV_ERROR_NONE) {
164 25 : break;
165 : }
166 11 : usleep(GET_DEV_ID_RETRY_INTERVAL);
167 11 : retryTime++;
168 11 : } while (retryTime < GET_DEV_ID_RETRY_TIMES);
169 26 : if (ret != DRV_ERROR_NONE) {
170 1 : ADIAG_WAR("get physical device-id by local device-id=%u, result=%d", localDeviceId, ret);
171 1 : return localDeviceId;
172 : }
173 25 : return phyDeviceId;
174 : }
175 :
176 : /**
177 : * @brief thread to get data from ts.
178 : * @param [in] arg: thread info
179 : * @return NULL
180 : */
181 26 : STATIC void* KtraceTsThread(void* arg)
182 : {
183 26 : ADIAG_CHK_NULL_PTR(arg, return NULL);
184 26 : ThreadInfo* info = (ThreadInfo*)arg;
185 26 : uint32_t phyDeviceId = KtraceGetPhysicalDeviceID(info->devId);
186 26 : ADIAG_RUN_INF("ktrace ts thread start, local device id = %u, physical device id = %u.", info->devId, phyDeviceId);
187 26 : if (TraceSetThreadName("TraceServerRecv") != TRACE_SUCCESS) {
188 3 : ADIAG_WAR("can not set thread name(TraceServerRecv) but continue.");
189 : }
190 26 : char* traceInfo = (char*)AdiagMalloc(DRV_RECV_MAX_LEN);
191 26 : if (traceInfo == NULL) {
192 0 : ADIAG_ERR("trace info malloc failed, strerr=%s.", strerror(AdiagGetErrorCode()));
193 0 : return NULL;
194 : }
195 37876 : while (info->status != 0) {
196 37851 : (void)memset_s(traceInfo, DRV_RECV_MAX_LEN, 0, DRV_RECV_MAX_LEN);
197 37851 : uint32_t len = DRV_RECV_MAX_LEN;
198 : int32_t ret =
199 37851 : log_read_by_type((int32_t)info->devId, traceInfo, &len, TRACE_RECV_TIMEOUT, LOG_CHANNEL_TYPE_TS_PROC);
200 37851 : if (ret == (int32_t)LOG_NOT_SUPPORT) {
201 1 : ADIAG_RUN_INF("ts channel is not supported.");
202 1 : break;
203 : }
204 37850 : if (ret == (int32_t)LOG_NOT_READY) {
205 37450 : continue;
206 : }
207 406 : if (ret != TRACE_SUCCESS) {
208 6 : ADIAG_WAR("can not read trace data from driver, ret = %d, strerr=%s.", ret, strerror(AdiagGetErrorCode()));
209 6 : usleep(TRACE_RECV_TIMEOUT * TIME_ONE_THOUSAND_MS);
210 6 : continue;
211 : }
212 400 : KtraceDataProcess(traceInfo, len, info->devId, phyDeviceId);
213 : }
214 26 : ADIAG_SAFE_FREE(traceInfo);
215 26 : return NULL;
216 : }
217 :
218 29 : STATIC TraStatus KtraceTsRecvThread(uint32_t devIndex, uint32_t devId)
219 : {
220 29 : g_ktraceTsThread[devIndex] = (ThreadInfo*)AdiagMalloc(sizeof(ThreadInfo));
221 29 : if (g_ktraceTsThread[devIndex] == NULL) {
222 0 : ADIAG_ERR("malloc ktrace ts thread failed, device id = %u.", devId);
223 0 : return TRACE_FAILURE;
224 : }
225 29 : g_ktraceTsThread[devIndex]->status = 1;
226 29 : g_ktraceTsThread[devIndex]->devId = devId;
227 :
228 : TraceUserBlock thread;
229 29 : thread.procFunc = KtraceTsThread;
230 29 : thread.pulArg = (void*)g_ktraceTsThread[devIndex];
231 29 : TraceThreadAttr attr = {0, 0, 0, 0, 0, 0, TRACE_THREAD_STACK_SIZE};
232 29 : TraceThread tid = 0;
233 29 : if (TraceCreateTaskWithThreadAttr(&tid, &thread, &attr) != TRACE_SUCCESS) {
234 3 : ADIAG_ERR("create task failed, strerr=%s.", strerror(AdiagGetErrorCode()));
235 3 : return TRACE_FAILURE;
236 : }
237 26 : g_ktraceTsThread[devIndex]->tid = tid;
238 26 : return TRACE_SUCCESS;
239 : }
240 :
241 : /**
242 : * @brief create thread to data from ts for each device.
243 : * @param [in] devNum: device number
244 : * @param [in] devIdArray: device id array
245 : * @return TraStatus
246 : */
247 33 : TraStatus KtraceTsCreateThread(uint32_t devNum, uint32_t* devIdArray)
248 : {
249 33 : if ((devNum > MAX_DEV_NUM) || (devIdArray == NULL)) {
250 2 : ADIAG_ERR("ktrace ts receive thread init failed.");
251 2 : return TRACE_FAILURE;
252 : }
253 :
254 31 : if (g_ktraceTsStatus != 0) {
255 1 : ADIAG_ERR("ktrace ts receive thread has already existed.");
256 1 : return TRACE_FAILURE;
257 : }
258 :
259 30 : (void)AdiagLockInit(&g_ktraceTsLock);
260 30 : (void)AdiagLockGet(&g_ktraceTsLock);
261 30 : g_ktraceTsStatus = 1;
262 30 : g_devNum = devNum;
263 30 : g_ktraceTsThread = (ThreadInfo**)AdiagMalloc(sizeof(ThreadInfo*) * (size_t)devNum);
264 30 : if (g_ktraceTsThread == NULL) {
265 2 : ADIAG_ERR("malloc ktrace ts thread failed.");
266 2 : (void)AdiagLockRelease(&g_ktraceTsLock);
267 2 : KtraceTsDestroyThread();
268 2 : return TRACE_FAILURE;
269 : }
270 :
271 28 : TraStatus ret = TRACE_SUCCESS;
272 54 : for (uint32_t i = 0; i < devNum; i++) {
273 29 : ret = KtraceTsRecvThread(i, devIdArray[i]);
274 29 : if (ret != TRACE_SUCCESS) {
275 3 : ADIAG_ERR("ktrace create ts receive thread failed, device id = %u.", i);
276 3 : (void)AdiagLockRelease(&g_ktraceTsLock);
277 3 : KtraceTsDestroyThread();
278 3 : return TRACE_FAILURE;
279 : }
280 : }
281 :
282 25 : (void)AdiagLockRelease(&g_ktraceTsLock);
283 25 : ADIAG_RUN_INF("create ts ktrace thread successfully.");
284 25 : return TRACE_SUCCESS;
285 : }
286 :
287 52 : void KtraceTsDestroyThread(void)
288 : {
289 52 : if (g_ktraceTsStatus == 0) {
290 22 : return;
291 : }
292 30 : (void)AdiagLockGet(&g_ktraceTsLock);
293 30 : g_ktraceTsStatus = 0;
294 30 : if (g_ktraceTsThread == NULL) {
295 2 : (void)AdiagLockRelease(&g_ktraceTsLock);
296 2 : (void)AdiagLockDestroy(&g_ktraceTsLock);
297 2 : return;
298 : }
299 58 : for (uint32_t i = 0; i < g_devNum; i++) {
300 30 : if (g_ktraceTsThread[i] != NULL) {
301 29 : g_ktraceTsThread[i]->status = 0;
302 : }
303 : }
304 58 : for (uint32_t i = 0; i < g_devNum; i++) {
305 30 : if ((g_ktraceTsThread[i] != NULL) && (g_ktraceTsThread[i]->tid != 0)) {
306 26 : (void)TraceJoinTask(&g_ktraceTsThread[i]->tid);
307 : }
308 30 : ADIAG_SAFE_FREE(g_ktraceTsThread[i]);
309 : }
310 28 : ADIAG_SAFE_FREE(g_ktraceTsThread);
311 28 : (void)AdiagLockRelease(&g_ktraceTsLock);
312 28 : (void)AdiagLockDestroy(&g_ktraceTsLock);
313 28 : ADIAG_RUN_INF("destroy ts ktrace thread successfully.");
314 : }
|
