Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 : #include "ktrace_ts.h"
11 : #include "adiag_lock.h"
12 : #include "adiag_print.h"
13 : #include "adiag_utils.h"
14 : #include "ascend_hal.h"
15 : #include "trace_msg.h"
16 : #include "trace_node.h"
17 : #include "trace_server_mgr.h"
18 : #include "trace_system_api.h"
19 :
20 : #define MAX_DEV_NUM 64
21 : #define DRV_RECV_MAX_LEN 524288
22 : #define TRACE_RECV_TIMEOUT 500
23 : #define GET_DEV_ID_RETRY_INTERVAL 1000 * 1000
24 : #define GET_DEV_ID_RETRY_TIMES 10
25 :
26 : typedef struct {
27 : TraceThread tid;
28 : uint32_t devId;
29 : int32_t status;
30 : } ThreadInfo;
31 :
32 : typedef struct {
33 : uint32_t pid;
34 : uint32_t logSize;
35 : uint8_t endFlag; // 0 start; 1 mid; 2 end
36 : uint8_t reserve[23];
37 : } TraceInfoHead;
38 :
39 : STATIC ThreadInfo **g_ktraceTsThread = NULL;
40 : STATIC AdiagLock g_ktraceTsLock = TRACE_MUTEX_INITIALIZER;
41 : STATIC int32_t g_ktraceTsStatus = 0;
42 : STATIC uint32_t g_devNum = 0;
43 :
44 : /**
45 : * @brief get event time when each event start.
46 : * @param [in] sessionNode: sessionNode
47 : * @param [in] endFlag: event flag
48 : * @param [out] time: event time
49 : * @param [in] len: time length
50 : * @return TraStatus
51 : */
52 23 : STATIC TraStatus KtraceGetEventTime(SessionNode *sessionNode, uint8_t endFlag, char *time, uint32_t len)
53 : {
54 23 : if ((strlen(sessionNode->eventTime) == 0) || (endFlag == ADIAG_INFO_FLAG_START)) {
55 21 : (void)memset_s(sessionNode->eventTime, TIMESTAMP_MAX_LENGTH, 0, TIMESTAMP_MAX_LENGTH);
56 21 : TraStatus ret = TimestampToFileStr(GetRealTime(), sessionNode->eventTime, TIMESTAMP_MAX_LENGTH);
57 21 : if (ret != TRACE_SUCCESS) {
58 1 : ADIAG_ERR("get timestamp failed, ret = %d.", ret);
59 1 : return TRACE_FAILURE;
60 : }
61 : }
62 22 : errno_t err = strncpy_s(time, len, sessionNode->eventTime, TIMESTAMP_MAX_LENGTH);
63 22 : if (err != EOK) {
64 1 : ADIAG_ERR("strncpy timestamp failed, err = %d, strerr = %s", (int32_t)err, strerror(AdiagGetErrorCode()));
65 1 : return TRACE_FAILURE;
66 : }
67 21 : if (endFlag == ADIAG_INFO_FLAG_END) {
68 19 : (void)memset_s(sessionNode->eventTime, TIMESTAMP_MAX_LENGTH, 0, TIMESTAMP_MAX_LENGTH);
69 : }
70 21 : return TRACE_SUCCESS;
71 : }
72 :
73 41655 : STATIC TraStatus KtracePushDataToNode(TraceInfoHead *head, TraceEventMsg *eventMsg)
74 : {
75 41655 : SessionNode *sessionNode = TraceServerGetSessionNode((int32_t)head->pid, (int32_t)eventMsg->devId);
76 41655 : if (sessionNode == NULL) {
77 41632 : ADIAG_WAR("no session node is valid, pid = %u.", head->pid);
78 41632 : return TRACE_FAILURE;
79 : }
80 23 : TraStatus ret = KtraceGetEventTime(sessionNode, head->endFlag, eventMsg->eventTime, TIMESTAMP_MAX_LENGTH);
81 23 : if (ret != TRACE_SUCCESS) {
82 2 : ADIAG_ERR("get event time failed, ret = %d, pid = %u.", ret, head->pid);
83 2 : return TRACE_FAILURE;
84 : }
85 :
86 21 : ret = TraceTsPushNode(sessionNode, head->endFlag, (void *)eventMsg, eventMsg->bufLen + sizeof(TraceEventMsg));
87 21 : if (ret != TRACE_SUCCESS) {
88 2 : ADIAG_ERR("push node failed, ret = %d, pid = %u.", ret, head->pid);
89 2 : return TRACE_FAILURE;
90 : }
91 19 : return TRACE_SUCCESS;
92 : }
93 :
94 : /**
95 : * @brief parse data from ts and construct to event msg.
96 : * @param [in] traceInfo: data from ts
97 : * @param [in] len: length of data from ts
98 : * @param [in] devId: device id
99 : * @return NA
100 : */
101 41658 : STATIC void KtraceDataProcess(char *traceInfo, uint32_t len, uint32_t localDevId, uint32_t devId)
102 : {
103 41658 : ADIAG_CHK_EXPR_ACTION((len > DRV_RECV_MAX_LEN) || (len < sizeof(TraceInfoHead)),
104 : return, "invalid length[%u] from driver", len);
105 41658 : TraceInfoHead *head = (TraceInfoHead *)traceInfo;
106 41658 : ADIAG_CHK_EXPR_ACTION(head->logSize >= len, return,
107 : "data len[%u] is over max len[%u] from ts.", head->logSize, len);
108 :
109 41657 : TraceEventMsg *eventMsg = (TraceEventMsg *)AdiagMalloc(sizeof(TraceEventMsg) + head->logSize + 1U);
110 41657 : if (eventMsg == NULL) {
111 0 : ADIAG_ERR("malloc for event msg failed.");
112 0 : return;
113 : }
114 41657 : eventMsg->msgType = TRACE_EVENT_MSG;
115 41657 : eventMsg->devId = localDevId;
116 41657 : eventMsg->pid = (int32_t)head->pid;
117 41657 : eventMsg->seqFlag = head->endFlag;
118 41657 : eventMsg->bufLen = head->logSize;
119 41657 : errno_t err = memcpy_s(eventMsg->buf, head->logSize + 1U, traceInfo + sizeof(TraceInfoHead), head->logSize);
120 41657 : if (err != EOK) {
121 1 : ADIAG_ERR("memcpy failed, err = %d, strerr = %s.", (int32_t)err, strerror(AdiagGetErrorCode()));
122 1 : ADIAG_SAFE_FREE(eventMsg);
123 1 : return;
124 : }
125 41656 : int32_t ret = snprintf_s(eventMsg->eventName, EVENT_NAME_MAX_LENGTH, EVENT_NAME_MAX_LENGTH - 1U, "ts_%u", devId);
126 41656 : if (ret == -1) {
127 1 : ADIAG_ERR("snprintf failed, ret = %d, strerr = %s.", ret, strerror(AdiagGetErrorCode()));
128 1 : ADIAG_SAFE_FREE(eventMsg);
129 1 : return;
130 : }
131 41655 : TraceServerSessionLock();
132 41655 : ret = KtracePushDataToNode(head, eventMsg);
133 41655 : TraceServerSessionUnlock();
134 41655 : if (ret != TRACE_SUCCESS) {
135 41636 : ADIAG_SAFE_FREE(eventMsg);
136 41636 : return;
137 : }
138 19 : ADIAG_INF("log read by type successfully, pid = %d, data len = %u bytes, end flag = %hhu.",
139 : eventMsg->pid, eventMsg->bufLen, eventMsg->seqFlag);
140 19 : return;
141 : }
142 :
143 : /**
144 : * @brief convert device-side devId to host-side devId
145 : * @param [in] localDeviceId: chip ID
146 : * @return host side devId (physical id)
147 : */
148 24 : STATIC uint32_t KtraceGetPhysicalDeviceID(uint32_t localDeviceId)
149 : {
150 24 : uint32_t phyDeviceId = 0;
151 24 : drvError_t ret = DRV_ERROR_NONE;
152 24 : uint32_t retryTime = 0;
153 : do {
154 34 : ret = drvGetDevIDByLocalDevID(localDeviceId, &phyDeviceId);
155 34 : if (ret == DRV_ERROR_NONE) {
156 23 : break;
157 : }
158 11 : usleep(GET_DEV_ID_RETRY_INTERVAL);
159 11 : retryTime++;
160 11 : } while (retryTime < GET_DEV_ID_RETRY_TIMES);
161 24 : if (ret != DRV_ERROR_NONE) {
162 1 : ADIAG_WAR("get physical device-id by local device-id=%u, result=%d", localDeviceId, ret);
163 1 : return localDeviceId;
164 : }
165 23 : return phyDeviceId;
166 : }
167 :
168 : /**
169 : * @brief thread to get data from ts.
170 : * @param [in] arg: thread info
171 : * @return NULL
172 : */
173 24 : STATIC void *KtraceTsThread(void *arg)
174 : {
175 24 : ADIAG_CHK_NULL_PTR(arg, return NULL);
176 24 : ThreadInfo *info = (ThreadInfo *)arg;
177 24 : uint32_t phyDeviceId = KtraceGetPhysicalDeviceID(info->devId);
178 24 : ADIAG_RUN_INF("ktrace ts thread start, local device id = %u, physical device id = %u.", info->devId, phyDeviceId);
179 24 : if (TraceSetThreadName("TraceServerRecv") != TRACE_SUCCESS) {
180 3 : ADIAG_WAR("can not set thread name(TraceServerRecv) but continue.");
181 : }
182 24 : char *traceInfo = (char *)AdiagMalloc(DRV_RECV_MAX_LEN);
183 24 : if (traceInfo == NULL) {
184 0 : ADIAG_ERR("trace info malloc failed, strerr=%s.", strerror(AdiagGetErrorCode()));
185 0 : return NULL;
186 : }
187 79172 : while (info->status != 0) {
188 79149 : (void)memset_s(traceInfo, DRV_RECV_MAX_LEN, 0, DRV_RECV_MAX_LEN);
189 79149 : uint32_t len = DRV_RECV_MAX_LEN;
190 79149 : int32_t ret = log_read_by_type((int32_t)info->devId, traceInfo, &len,
191 : TRACE_RECV_TIMEOUT, LOG_CHANNEL_TYPE_TS_PROC);
192 79149 : if (ret == (int32_t)LOG_NOT_SUPPORT) {
193 1 : ADIAG_RUN_INF("ts channel is not supported.");
194 1 : break;
195 : }
196 79148 : if (ret == (int32_t)LOG_NOT_READY) {
197 37490 : continue;
198 : }
199 41664 : if (ret != TRACE_SUCCESS) {
200 6 : ADIAG_WAR("can not read trace data from driver, ret = %d, strerr=%s.", ret, strerror(AdiagGetErrorCode()));
201 6 : usleep(TRACE_RECV_TIMEOUT * TIME_ONE_THOUSAND_MS);
202 6 : continue;
203 : }
204 41658 : KtraceDataProcess(traceInfo, len, info->devId, phyDeviceId);
205 : }
206 24 : ADIAG_SAFE_FREE(traceInfo);
207 24 : return NULL;
208 : }
209 :
210 27 : STATIC TraStatus KtraceTsRecvThread(uint32_t devIndex, uint32_t devId)
211 : {
212 27 : g_ktraceTsThread[devIndex] = (ThreadInfo *)AdiagMalloc(sizeof(ThreadInfo));
213 27 : if (g_ktraceTsThread[devIndex] == NULL) {
214 0 : ADIAG_ERR("malloc ktrace ts thread failed, device id = %u.", devId);
215 0 : return TRACE_FAILURE;
216 : }
217 27 : g_ktraceTsThread[devIndex]->status = 1;
218 27 : g_ktraceTsThread[devIndex]->devId = devId;
219 :
220 : TraceUserBlock thread;
221 27 : thread.procFunc = KtraceTsThread;
222 27 : thread.pulArg = (void *)g_ktraceTsThread[devIndex];
223 27 : TraceThreadAttr attr = { 0, 0, 0, 0, 0, 0, TRACE_THREAD_STACK_SIZE };
224 27 : TraceThread tid = 0;
225 27 : if (TraceCreateTaskWithThreadAttr(&tid, &thread, &attr) != TRACE_SUCCESS) {
226 3 : ADIAG_ERR("create task failed, strerr=%s.", strerror(AdiagGetErrorCode()));
227 3 : return TRACE_FAILURE;
228 : }
229 24 : g_ktraceTsThread[devIndex]->tid = tid;
230 24 : return TRACE_SUCCESS;
231 : }
232 :
233 : /**
234 : * @brief create thread to data from ts for each device.
235 : * @param [in] devNum: device number
236 : * @param [in] devIdArray: device id array
237 : * @return TraStatus
238 : */
239 31 : TraStatus KtraceTsCreateThread(uint32_t devNum, uint32_t *devIdArray)
240 : {
241 31 : if ((devNum > MAX_DEV_NUM) || (devIdArray == NULL)) {
242 2 : ADIAG_ERR("ktrace ts receive thread init failed.");
243 2 : return TRACE_FAILURE;
244 : }
245 :
246 29 : if (g_ktraceTsStatus != 0) {
247 1 : ADIAG_ERR("ktrace ts receive thread has already existed.");
248 1 : return TRACE_FAILURE;
249 : }
250 :
251 28 : (void)AdiagLockInit(&g_ktraceTsLock);
252 28 : (void)AdiagLockGet(&g_ktraceTsLock);
253 28 : g_ktraceTsStatus = 1;
254 28 : g_devNum = devNum;
255 28 : g_ktraceTsThread = (ThreadInfo **)AdiagMalloc(sizeof(ThreadInfo *) * (size_t)devNum);
256 28 : if (g_ktraceTsThread == NULL) {
257 2 : ADIAG_ERR("malloc ktrace ts thread failed.");
258 2 : (void)AdiagLockRelease(&g_ktraceTsLock);
259 2 : KtraceTsDestroyThread();
260 2 : return TRACE_FAILURE;
261 : }
262 :
263 26 : TraStatus ret = TRACE_SUCCESS;
264 50 : for (uint32_t i = 0; i < devNum; i++) {
265 27 : ret = KtraceTsRecvThread(i, devIdArray[i]);
266 27 : if (ret != TRACE_SUCCESS) {
267 3 : ADIAG_ERR("ktrace create ts receive thread failed, device id = %u.", i);
268 3 : (void)AdiagLockRelease(&g_ktraceTsLock);
269 3 : KtraceTsDestroyThread();
270 3 : return TRACE_FAILURE;
271 : }
272 : }
273 :
274 23 : (void)AdiagLockRelease(&g_ktraceTsLock);
275 23 : ADIAG_RUN_INF("create ts ktrace thread successfully.");
276 23 : return TRACE_SUCCESS;
277 : }
278 :
279 50 : void KtraceTsDestroyThread(void)
280 : {
281 50 : if (g_ktraceTsStatus == 0) {
282 22 : return;
283 : }
284 28 : (void)AdiagLockGet(&g_ktraceTsLock);
285 28 : g_ktraceTsStatus = 0;
286 28 : if (g_ktraceTsThread == NULL) {
287 2 : (void)AdiagLockRelease(&g_ktraceTsLock);
288 2 : (void)AdiagLockDestroy(&g_ktraceTsLock);
289 2 : return;
290 : }
291 54 : for (uint32_t i = 0; i < g_devNum; i++) {
292 28 : if (g_ktraceTsThread[i] != NULL) {
293 27 : g_ktraceTsThread[i]->status = 0;
294 : }
295 : }
296 54 : for (uint32_t i = 0; i < g_devNum; i++) {
297 28 : if ((g_ktraceTsThread[i] != NULL) && (g_ktraceTsThread[i]->tid != 0)) {
298 24 : (void)TraceJoinTask(&g_ktraceTsThread[i]->tid);
299 : }
300 28 : ADIAG_SAFE_FREE(g_ktraceTsThread[i]);
301 : }
302 26 : ADIAG_SAFE_FREE(g_ktraceTsThread);
303 26 : (void)AdiagLockRelease(&g_ktraceTsLock);
304 26 : (void)AdiagLockDestroy(&g_ktraceTsLock);
305 26 : ADIAG_RUN_INF("destroy ts ktrace thread successfully.");
306 : }
|
