Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "mc2_trace_utils.h"
12 :
13 : #include <cmath>
14 : #include "log.h"
15 : #include "mmpa_api.h"
16 : #include "atrace_pub.h"
17 : #include "common/aicpu_hccl_common.h"
18 : #include "framework/aicpu_kfc_prof.h"
19 : #include "common/aicpu_kfc_utils.h"
20 : #include "adapter_hal_pub.h"
21 :
22 : namespace {
23 : constexpr uint32_t TRACE_RING_BUFF_SIZE = 128 * 1024;
24 : constexpr uint32_t MAX_SQE_SUBMIT_NUM = 256U;
25 : constexpr char UTRACE_SO[] = "libutrace.so";
26 : constexpr char ATRACE_SO[] = "libascend_trace.so";
27 :
28 : using TraceCreateWithAttrFunc = TraHandle (*)(TracerType, const char*, const TraceAttr*);
29 : using TraceGetHandleFunc = TraHandle (*)(TracerType, const char*);
30 : using TraceSubmitFunc = TraStatus (*)(TraHandle handle, const void*, uint32_t);
31 : using TraceEventCreateFunc = TraEventHandle (*)(const char*);
32 : using TraceEventDestroyFunc = void (*)(TraEventHandle);
33 : using TraceHandleDestroyFunc = void (*)(TraHandle);
34 : using TraceEventBindTraceFunc = TraStatus (*)(TraEventHandle, TraHandle);
35 : using TraceEventReportFunc = TraStatus (*)(TraEventHandle);
36 : using TraceSetGlobalAttrFunc = TraStatus (*)(const TraceGlobalAttr*);
37 : using TraceStructEntryCreateFunc = TraceStructEntry* (*)(const char*);
38 : using TraceStructItemFieldSetFunc = void (*)(TraceStructEntry*, const char*, uint8_t, uint8_t, uint64_t);
39 : using TraceStructItemArraySetFunc = void (*)(TraceStructEntry*, const char*, uint8_t, uint8_t, uint64_t);
40 : using TraceStructSetAttrFunc = void (*)(TraceStructEntry*, uint8_t, TraceAttr*);
41 : using TraceStructEntryDestroyFunc = void (*)(TraceStructEntry*);
42 :
43 : void* g_soHandle = nullptr;
44 : bool g_isAtrace = true;
45 : TraceCreateWithAttrFunc g_traceCreateWithAttr = nullptr;
46 : TraceGetHandleFunc g_traceGetHandle = nullptr;
47 : TraceSubmitFunc g_traceSubmit = nullptr;
48 : TraceEventCreateFunc g_traceEventCreate = nullptr;
49 : TraceEventDestroyFunc g_traceEventDestroy = nullptr;
50 : TraceHandleDestroyFunc g_traceHandleDestroy = nullptr;
51 : TraceEventBindTraceFunc g_traceEventBindTrace = nullptr;
52 : TraceEventReportFunc g_traceEventReport = nullptr;
53 : TraceSetGlobalAttrFunc g_traceSetGlobalAttr = nullptr;
54 : TraceStructEntryCreateFunc g_traceStructEntryCreate = nullptr;
55 : TraceStructItemFieldSetFunc g_traceStructItemFieldSet = nullptr;
56 : TraceStructItemArraySetFunc g_traceStructItemArraySet = nullptr;
57 : TraceStructSetAttrFunc g_traceStructSetAttr = nullptr;
58 : TraceStructEntryDestroyFunc g_traceStructEntryDestroy = nullptr;
59 :
60 : TraEventHandle g_eventHandle = -1;
61 : TraHandle g_traceStrHandle = -1;
62 : TraHandle g_traceTaskAndTilingDataHandle = -1;
63 : TraHandle g_traceAicpuComDataHandle = -1;
64 : TraHandle g_traceMsgInfoHandle = -1;
65 : TraHandle g_traceSqeBatchInfoHandle = -1;
66 : TraceStructEntry* g_traceStrSt = nullptr;
67 : TraceStructEntry* g_traceAicpuComTraceSt = nullptr;
68 : TraceStructEntry* g_traceKFCtaskAndTilingTraceDataSt = nullptr;
69 : TraceStructEntry* g_traceMsgInfoSt = nullptr;
70 : TraceStructEntry* g_traceSqeBatchInfoSt = nullptr;
71 : } // namespace
72 :
73 : // 当msgNum不为2的幂时，会自动向上取2的幂
74 10 : uint16_t MC2TraceUtils::GetMsgNum(size_t msgSize)
75 : {
76 10 : float logValue = log(TRACE_RING_BUFF_SIZE / (msgSize + 16)) / log(2); // 16 预留字节 2 幂次
77 10 : return static_cast<uint16_t>(pow(2, floor(logValue))); // 2 幂次
78 : }
79 :
80 2 : HcclResult MC2TraceUtils::InitTraceStrHandle()
81 : {
82 : // trace api存在限制: msgNum * msgSize <= 128k
83 2 : uint16_t strMsgNum = GetMsgNum(sizeof(TraceStr));
84 2 : TraceAttr attr = {};
85 2 : attr = {false, strMsgNum, sizeof(TraceStr), {nullptr}, 0, {0}};
86 2 : g_traceStrSt = g_traceStructEntryCreate("TraceStr");
87 2 : g_traceStructItemArraySet(
88 : g_traceStrSt, "transmit", TRACE_STRUCT_ARRAY_TYPE_CHAR, TRACE_STRUCT_SHOW_MODE_CHAR,
89 : sizeof(TraceStr::transmit));
90 2 : g_traceStructSetAttr(g_traceStrSt, 0, &attr);
91 2 : g_traceStrHandle = g_traceCreateWithAttr(TRACER_TYPE_SCHEDULE, "TraceStr", &attr);
92 2 : if (g_traceStrHandle < 0) {
93 1 : HCCL_ERROR("Create g_traceStrHandle failed, ret:%d", g_traceStrHandle);
94 1 : g_traceStructEntryDestroy(g_traceStrSt);
95 1 : return HCCL_E_INTERNAL;
96 : }
97 1 : TraStatus ret = g_traceEventBindTrace(g_eventHandle, g_traceStrHandle);
98 1 : if (ret != TRACE_SUCCESS) {
99 0 : HCCL_ERROR("Bind g_traceStrHandle failed, ret:%d", ret);
100 0 : return HCCL_E_INTERNAL;
101 : }
102 1 : return HCCL_SUCCESS;
103 : }
104 :
105 2 : void MC2TraceUtils::SetHcclKFCTilingDataOne()
106 : {
107 2 : g_traceStructItemFieldSet(
108 : g_traceKFCtaskAndTilingTraceDataSt, "sendOff", TRACE_STRUCT_FIELD_TYPE_UINT64, TRACE_STRUCT_SHOW_MODE_HEX,
109 : 8); // 8 uint64
110 2 : g_traceStructItemFieldSet(
111 : g_traceKFCtaskAndTilingTraceDataSt, "recvOff", TRACE_STRUCT_FIELD_TYPE_UINT64, TRACE_STRUCT_SHOW_MODE_HEX,
112 : 8); // 8 uint64
113 2 : g_traceStructItemFieldSet(
114 : g_traceKFCtaskAndTilingTraceDataSt, "tailSendOff", TRACE_STRUCT_FIELD_TYPE_UINT64, TRACE_STRUCT_SHOW_MODE_HEX,
115 : 8); // 8 uint64
116 2 : g_traceStructItemFieldSet(
117 : g_traceKFCtaskAndTilingTraceDataSt, "tailRecvOff", TRACE_STRUCT_FIELD_TYPE_UINT64, TRACE_STRUCT_SHOW_MODE_HEX,
118 : 8); // 8 uint64
119 2 : g_traceStructItemFieldSet(
120 : g_traceKFCtaskAndTilingTraceDataSt, "sendCnt", TRACE_STRUCT_FIELD_TYPE_UINT64, TRACE_STRUCT_SHOW_MODE_HEX,
121 : 8); // 8 uint64
122 2 : g_traceStructItemFieldSet(
123 : g_traceKFCtaskAndTilingTraceDataSt, "recvCnt", TRACE_STRUCT_FIELD_TYPE_UINT64, TRACE_STRUCT_SHOW_MODE_HEX,
124 : 8); // 8 uint64
125 2 : g_traceStructItemFieldSet(
126 : g_traceKFCtaskAndTilingTraceDataSt, "tailSendCnt", TRACE_STRUCT_FIELD_TYPE_UINT64, TRACE_STRUCT_SHOW_MODE_HEX,
127 : 8); // 8 uint64
128 2 : g_traceStructItemFieldSet(
129 : g_traceKFCtaskAndTilingTraceDataSt, "tailRecvCnt", TRACE_STRUCT_FIELD_TYPE_UINT64, TRACE_STRUCT_SHOW_MODE_HEX,
130 : 8); // 8 uint64
131 2 : g_traceStructItemFieldSet(
132 : g_traceKFCtaskAndTilingTraceDataSt, "totalCnt", TRACE_STRUCT_FIELD_TYPE_UINT64, TRACE_STRUCT_SHOW_MODE_HEX,
133 : 8); // 8 uint64
134 2 : g_traceStructItemFieldSet(
135 : g_traceKFCtaskAndTilingTraceDataSt, "turnNum", TRACE_STRUCT_FIELD_TYPE_UINT32, TRACE_STRUCT_SHOW_MODE_DEC,
136 : 4); // 4 uint32
137 2 : g_traceStructItemFieldSet(
138 : g_traceKFCtaskAndTilingTraceDataSt, "tailNum", TRACE_STRUCT_FIELD_TYPE_UINT32, TRACE_STRUCT_SHOW_MODE_DEC,
139 : 4); // 4 uint32
140 2 : g_traceStructItemFieldSet(
141 : g_traceKFCtaskAndTilingTraceDataSt, "stride", TRACE_STRUCT_FIELD_TYPE_UINT32, TRACE_STRUCT_SHOW_MODE_DEC,
142 : 4); // 4 uint32
143 2 : g_traceStructItemFieldSet(
144 : g_traceKFCtaskAndTilingTraceDataSt, "workspaceOff", TRACE_STRUCT_FIELD_TYPE_UINT32, TRACE_STRUCT_SHOW_MODE_DEC,
145 : 4); // 4 uint32
146 2 : g_traceStructItemFieldSet(
147 : g_traceKFCtaskAndTilingTraceDataSt, "notifyOff", TRACE_STRUCT_FIELD_TYPE_UINT32, TRACE_STRUCT_SHOW_MODE_DEC,
148 : 4); // 4 uint32
149 2 : g_traceStructItemFieldSet(
150 : g_traceKFCtaskAndTilingTraceDataSt, "notifyBeginCnt", TRACE_STRUCT_FIELD_TYPE_UINT16,
151 : TRACE_STRUCT_SHOW_MODE_DEC, 2); // 2 uint16
152 2 : g_traceStructItemFieldSet(
153 : g_traceKFCtaskAndTilingTraceDataSt, "notifyEndCnt", TRACE_STRUCT_FIELD_TYPE_UINT16, TRACE_STRUCT_SHOW_MODE_DEC,
154 : 2); // 2 uint16
155 2 : g_traceStructItemFieldSet(
156 : g_traceKFCtaskAndTilingTraceDataSt, "useBufferType", TRACE_STRUCT_FIELD_TYPE_UINT8, TRACE_STRUCT_SHOW_MODE_DEC,
157 : 1); // 1 uint8
158 2 : g_traceStructItemFieldSet(
159 : g_traceKFCtaskAndTilingTraceDataSt, "funID", TRACE_STRUCT_FIELD_TYPE_UINT8, TRACE_STRUCT_SHOW_MODE_DEC,
160 : 1); // 1 uint8
161 2 : g_traceStructItemFieldSet(
162 : g_traceKFCtaskAndTilingTraceDataSt, "dataType", TRACE_STRUCT_FIELD_TYPE_UINT8, TRACE_STRUCT_SHOW_MODE_DEC,
163 : 1); // 1 uint8
164 2 : g_traceStructItemFieldSet(
165 : g_traceKFCtaskAndTilingTraceDataSt, "groupNum", TRACE_STRUCT_FIELD_TYPE_UINT8, TRACE_STRUCT_SHOW_MODE_DEC,
166 : 1); // 1 uint8
167 2 : g_traceStructItemFieldSet(
168 : g_traceKFCtaskAndTilingTraceDataSt, "reuseMode", TRACE_STRUCT_FIELD_TYPE_UINT8, TRACE_STRUCT_SHOW_MODE_DEC,
169 : 1); // 1 uint8
170 2 : }
171 :
172 2 : void MC2TraceUtils::SetHcclKFCTilingDataTwo()
173 : {
174 2 : g_traceStructItemFieldSet(
175 : g_traceKFCtaskAndTilingTraceDataSt, "commType", TRACE_STRUCT_FIELD_TYPE_UINT8, TRACE_STRUCT_SHOW_MODE_DEC,
176 : 1); // 1 uint8
177 2 : g_traceStructItemFieldSet(
178 : g_traceKFCtaskAndTilingTraceDataSt, "reduceOp", TRACE_STRUCT_FIELD_TYPE_UINT8, TRACE_STRUCT_SHOW_MODE_DEC,
179 : 1); // 1 uint8
180 2 : g_traceStructItemFieldSet(
181 : g_traceKFCtaskAndTilingTraceDataSt, "commOrder", TRACE_STRUCT_FIELD_TYPE_UINT8, TRACE_STRUCT_SHOW_MODE_DEC,
182 : 1); // 1 uint8
183 2 : g_traceStructItemFieldSet(
184 : g_traceKFCtaskAndTilingTraceDataSt, "waitPolicy", TRACE_STRUCT_FIELD_TYPE_UINT8, TRACE_STRUCT_SHOW_MODE_DEC,
185 : 1); // 1 uint8
186 2 : g_traceStructItemFieldSet(
187 : g_traceKFCtaskAndTilingTraceDataSt, "rspPolicy", TRACE_STRUCT_FIELD_TYPE_UINT8, TRACE_STRUCT_SHOW_MODE_DEC,
188 : 1); // 1 uint8
189 2 : g_traceStructItemFieldSet(
190 : g_traceKFCtaskAndTilingTraceDataSt, "exitPolicy", TRACE_STRUCT_FIELD_TYPE_UINT8, TRACE_STRUCT_SHOW_MODE_DEC,
191 : 1); // 1 uint8
192 2 : g_traceStructItemFieldSet(
193 : g_traceKFCtaskAndTilingTraceDataSt, "commAlg", TRACE_STRUCT_FIELD_TYPE_UINT8, TRACE_STRUCT_SHOW_MODE_DEC,
194 : 1); // 1 uint8
195 2 : g_traceStructItemFieldSet(
196 : g_traceKFCtaskAndTilingTraceDataSt, "taskType", TRACE_STRUCT_FIELD_TYPE_UINT8, TRACE_STRUCT_SHOW_MODE_DEC,
197 : 1); // 1 uint8
198 2 : g_traceStructItemFieldSet(
199 : g_traceKFCtaskAndTilingTraceDataSt, "debugMode", TRACE_STRUCT_FIELD_TYPE_UINT8, TRACE_STRUCT_SHOW_MODE_DEC,
200 : 1); // 1 uint8
201 2 : g_traceStructItemFieldSet(
202 : g_traceKFCtaskAndTilingTraceDataSt, "stepSize", TRACE_STRUCT_FIELD_TYPE_UINT8, TRACE_STRUCT_SHOW_MODE_DEC,
203 : 1); // 1 uint8
204 2 : g_traceStructItemFieldSet(
205 : g_traceKFCtaskAndTilingTraceDataSt, "sendArgIndex", TRACE_STRUCT_FIELD_TYPE_UINT8, TRACE_STRUCT_SHOW_MODE_DEC,
206 : 1); // 1 uint8
207 2 : g_traceStructItemFieldSet(
208 : g_traceKFCtaskAndTilingTraceDataSt, "recvArgIndex", TRACE_STRUCT_FIELD_TYPE_UINT8, TRACE_STRUCT_SHOW_MODE_DEC,
209 : 1); // 1 uint8
210 2 : g_traceStructItemFieldSet(
211 : g_traceKFCtaskAndTilingTraceDataSt, "commOutArgIndex", TRACE_STRUCT_FIELD_TYPE_UINT8,
212 : TRACE_STRUCT_SHOW_MODE_DEC, 1); // 1 uint8
213 2 : g_traceStructItemFieldSet(
214 : g_traceKFCtaskAndTilingTraceDataSt, "hasCommOut", TRACE_STRUCT_FIELD_TYPE_UINT8, TRACE_STRUCT_SHOW_MODE_DEC,
215 : 1); // 1 uint8
216 2 : g_traceStructItemFieldSet(
217 : g_traceKFCtaskAndTilingTraceDataSt, "reverse", TRACE_STRUCT_FIELD_TYPE_UINT8, TRACE_STRUCT_SHOW_MODE_DEC,
218 : 1); // 1 uint8
219 2 : g_traceStructItemFieldSet(
220 : g_traceKFCtaskAndTilingTraceDataSt, "reserve2", TRACE_STRUCT_FIELD_TYPE_UINT32, TRACE_STRUCT_SHOW_MODE_DEC,
221 : 4); // 4 uint32
222 2 : }
223 :
224 2 : HcclResult MC2TraceUtils::InitTaskAndTilingDataHandle()
225 : {
226 2 : uint16_t taskAndTilingDataMsgNum = GetMsgNum(sizeof(KFCtaskAndTilingTraceData));
227 2 : TraceAttr attr = {};
228 2 : attr = {false, taskAndTilingDataMsgNum, sizeof(KFCtaskAndTilingTraceData), {nullptr}, 0, {0}};
229 2 : g_traceKFCtaskAndTilingTraceDataSt = g_traceStructEntryCreate("KFCtaskAndTilingTraceData");
230 2 : g_traceStructItemFieldSet(
231 : g_traceKFCtaskAndTilingTraceDataSt, "inputA", TRACE_STRUCT_FIELD_TYPE_UINT64, TRACE_STRUCT_SHOW_MODE_HEX,
232 : 8); // 8 uint64
233 2 : g_traceStructItemFieldSet(
234 : g_traceKFCtaskAndTilingTraceDataSt, "outputC", TRACE_STRUCT_FIELD_TYPE_UINT64, TRACE_STRUCT_SHOW_MODE_HEX,
235 : 8); // 8 uint64
236 2 : g_traceStructItemFieldSet(
237 : g_traceKFCtaskAndTilingTraceDataSt, "commOut", TRACE_STRUCT_FIELD_TYPE_UINT64, TRACE_STRUCT_SHOW_MODE_HEX,
238 : 8); // 8 uint64
239 2 : g_traceStructItemFieldSet(
240 : g_traceKFCtaskAndTilingTraceDataSt, "context", TRACE_STRUCT_FIELD_TYPE_UINT64, TRACE_STRUCT_SHOW_MODE_HEX,
241 : 8); // 8 uint64
242 2 : g_traceStructItemFieldSet(
243 : g_traceKFCtaskAndTilingTraceDataSt, "workSpace", TRACE_STRUCT_FIELD_TYPE_UINT64, TRACE_STRUCT_SHOW_MODE_HEX,
244 : 8); // 8 uint64
245 2 : g_traceStructItemFieldSet(
246 : g_traceKFCtaskAndTilingTraceDataSt, "tilingData", TRACE_STRUCT_FIELD_TYPE_UINT64, TRACE_STRUCT_SHOW_MODE_HEX,
247 : 8); // 8 uint64
248 2 : SetHcclKFCTilingDataOne();
249 2 : SetHcclKFCTilingDataTwo();
250 2 : g_traceStructSetAttr(g_traceKFCtaskAndTilingTraceDataSt, 0, &attr);
251 2 : g_traceTaskAndTilingDataHandle = g_traceCreateWithAttr(TRACER_TYPE_SCHEDULE, "KFCtaskAndTilingTraceData", &attr);
252 2 : if (g_traceTaskAndTilingDataHandle < 0) {
253 1 : HCCL_ERROR("Create g_traceTaskAndTilingDataHandle failed, ret:%d", g_traceTaskAndTilingDataHandle);
254 1 : g_traceStructEntryDestroy(g_traceKFCtaskAndTilingTraceDataSt);
255 1 : return HCCL_E_INTERNAL;
256 : }
257 1 : TraStatus ret = g_traceEventBindTrace(g_eventHandle, g_traceTaskAndTilingDataHandle);
258 1 : if (ret != TRACE_SUCCESS) {
259 0 : HCCL_ERROR("Bind g_traceTaskAndTilingDataHandle failed, ret:%d", ret);
260 0 : return HCCL_E_INTERNAL;
261 : }
262 1 : return HCCL_SUCCESS;
263 : }
264 :
265 2 : HcclResult MC2TraceUtils::InitAicpuComDataHandle()
266 : {
267 2 : uint16_t aicpuComMsgNum = GetMsgNum(sizeof(AicpuComTraceData));
268 2 : TraceAttr attr = {};
269 2 : attr = {false, aicpuComMsgNum, sizeof(AicpuComTraceData), {nullptr}, 0, {0}};
270 2 : g_traceAicpuComTraceSt = g_traceStructEntryCreate("AicpuComTraceData");
271 2 : g_traceStructItemFieldSet(
272 : g_traceAicpuComTraceSt, "devId", TRACE_STRUCT_FIELD_TYPE_UINT32, TRACE_STRUCT_SHOW_MODE_DEC, 4); // 4 uint32
273 2 : g_traceStructItemFieldSet(
274 : g_traceAicpuComTraceSt, "ssid", TRACE_STRUCT_FIELD_TYPE_UINT32, TRACE_STRUCT_SHOW_MODE_DEC, 4); // 4 uint32
275 2 : g_traceStructItemFieldSet(
276 : g_traceAicpuComTraceSt, "rankId", TRACE_STRUCT_FIELD_TYPE_UINT32, TRACE_STRUCT_SHOW_MODE_DEC, 4); // 4 uint32
277 2 : g_traceStructItemFieldSet(
278 : g_traceAicpuComTraceSt, "rankNum", TRACE_STRUCT_FIELD_TYPE_UINT32, TRACE_STRUCT_SHOW_MODE_DEC, 4); // 4 uint32
279 2 : g_traceStructItemFieldSet(
280 : g_traceAicpuComTraceSt, "windowSize", TRACE_STRUCT_FIELD_TYPE_UINT64, TRACE_STRUCT_SHOW_MODE_DEC,
281 : 8); // 8 uint64
282 2 : g_traceStructItemFieldSet(
283 : g_traceAicpuComTraceSt, "workSpaceAddr", TRACE_STRUCT_FIELD_TYPE_UINT64, TRACE_STRUCT_SHOW_MODE_HEX,
284 : 8); // 8 uint64
285 2 : g_traceStructItemFieldSet(
286 : g_traceAicpuComTraceSt, "kfcNotifyId", TRACE_STRUCT_FIELD_TYPE_UINT64, TRACE_STRUCT_SHOW_MODE_DEC,
287 : 8); // 8 uint64
288 2 : g_traceStructItemArraySet(
289 : g_traceAicpuComTraceSt, "eventIds", TRACE_STRUCT_ARRAY_TYPE_UINT32, TRACE_STRUCT_SHOW_MODE_DEC,
290 : 128); // 4 uint32 128 AC_MAX_RANK_NUM*uint32
291 2 : g_traceStructItemArraySet(
292 : g_traceAicpuComTraceSt, "windowIn", TRACE_STRUCT_ARRAY_TYPE_UINT64, TRACE_STRUCT_SHOW_MODE_HEX,
293 : 256); // 8 uint64 256 AC_MAX_RANK_NUM*uint64
294 2 : g_traceStructItemArraySet(
295 : g_traceAicpuComTraceSt, "windowOut", TRACE_STRUCT_ARRAY_TYPE_UINT64, TRACE_STRUCT_SHOW_MODE_HEX,
296 : 256); // 8 uint64 256 AC_MAX_RANK_NUM*uint64
297 2 : g_traceStructItemArraySet(
298 : g_traceAicpuComTraceSt, "actualStreamId", TRACE_STRUCT_ARRAY_TYPE_INT32, TRACE_STRUCT_SHOW_MODE_DEC,
299 : 128); // 4 uint32 128 AC_MAX_RANK_NUM*uint32
300 2 : g_traceStructItemArraySet(
301 : g_traceAicpuComTraceSt, "sqId", TRACE_STRUCT_ARRAY_TYPE_INT32, TRACE_STRUCT_SHOW_MODE_DEC,
302 : 128); // 4 uint32 128 AC_MAX_RANK_NUM*uint32
303 2 : g_traceStructItemArraySet(
304 : g_traceAicpuComTraceSt, "aicpuOpNotifyAddress", TRACE_STRUCT_ARRAY_TYPE_UINT64, TRACE_STRUCT_SHOW_MODE_HEX,
305 : 16); // 8 uint64 16 2*uint64
306 2 : g_traceStructItemArraySet(
307 : g_traceAicpuComTraceSt, "aicpuOpNotifyActualNotifyId", TRACE_STRUCT_ARRAY_TYPE_INT32,
308 : TRACE_STRUCT_SHOW_MODE_DEC, 8); // 4 uint32 8 2*uint32
309 2 : g_traceStructItemFieldSet(
310 : g_traceAicpuComTraceSt, "clusterId", TRACE_STRUCT_FIELD_TYPE_INT32, TRACE_STRUCT_SHOW_MODE_DEC, 4); // 4 uint32
311 2 : g_traceStructSetAttr(g_traceAicpuComTraceSt, 0, &attr);
312 2 : g_traceAicpuComDataHandle = g_traceCreateWithAttr(TRACER_TYPE_SCHEDULE, "AicpuComTraceData", &attr);
313 2 : if (g_traceAicpuComDataHandle < 0) {
314 1 : HCCL_ERROR("Create g_traceAicpuComDataHandle failed, ret:%d", g_traceAicpuComDataHandle);
315 1 : g_traceStructEntryDestroy(g_traceAicpuComTraceSt);
316 1 : return HCCL_E_INTERNAL;
317 : }
318 1 : TraStatus ret = g_traceEventBindTrace(g_eventHandle, g_traceAicpuComDataHandle);
319 1 : if (ret != TRACE_SUCCESS) {
320 0 : HCCL_ERROR("Bind g_traceAicpuComDataHandle failed, ret:%d", ret);
321 0 : return HCCL_E_INTERNAL;
322 : }
323 1 : return HCCL_SUCCESS;
324 : }
325 :
326 2 : void MC2TraceUtils::SetTraceMsgInfo()
327 : {
328 2 : g_traceStructItemFieldSet(
329 : g_traceMsgInfoSt, "commType", TRACE_STRUCT_FIELD_TYPE_UINT32, TRACE_STRUCT_SHOW_MODE_DEC, 4); // 4 uint32
330 2 : g_traceStructItemFieldSet(
331 : g_traceMsgInfoSt, "opType", TRACE_STRUCT_FIELD_TYPE_UINT32, TRACE_STRUCT_SHOW_MODE_DEC, 4); // 4 uint32
332 2 : g_traceStructItemFieldSet(
333 : g_traceMsgInfoSt, "sendBuffer", TRACE_STRUCT_FIELD_TYPE_UINT64, TRACE_STRUCT_SHOW_MODE_HEX, 8); // 8 uint64
334 2 : g_traceStructItemFieldSet(
335 : g_traceMsgInfoSt, "recvBuffer", TRACE_STRUCT_FIELD_TYPE_UINT64, TRACE_STRUCT_SHOW_MODE_HEX, 8); // 8 uint64
336 2 : g_traceStructItemFieldSet(
337 : g_traceMsgInfoSt, "count", TRACE_STRUCT_FIELD_TYPE_UINT64, TRACE_STRUCT_SHOW_MODE_DEC, 8); // 8 uint64
338 2 : g_traceStructItemFieldSet(
339 : g_traceMsgInfoSt, "strideLen", TRACE_STRUCT_FIELD_TYPE_UINT64, TRACE_STRUCT_SHOW_MODE_DEC, 8); // 8 uint64
340 2 : g_traceStructItemFieldSet(
341 : g_traceMsgInfoSt, "hcclDataType", TRACE_STRUCT_FIELD_TYPE_UINT32, TRACE_STRUCT_SHOW_MODE_DEC, 4); // 4 uint32
342 2 : g_traceStructItemFieldSet(
343 : g_traceMsgInfoSt, "valid", TRACE_STRUCT_FIELD_TYPE_UINT32, TRACE_STRUCT_SHOW_MODE_DEC, 4); // 4 uint32
344 2 : g_traceStructItemFieldSet(
345 : g_traceMsgInfoSt, "isLast", TRACE_STRUCT_FIELD_TYPE_UINT8, TRACE_STRUCT_SHOW_MODE_DEC, 1); // 1 uint8
346 2 : g_traceStructItemFieldSet(
347 : g_traceMsgInfoSt, "funID", TRACE_STRUCT_FIELD_TYPE_UINT8, TRACE_STRUCT_SHOW_MODE_DEC, 1); // 1 uint8
348 2 : g_traceStructItemFieldSet(
349 : g_traceMsgInfoSt, "sendCnt", TRACE_STRUCT_FIELD_TYPE_UINT8, TRACE_STRUCT_SHOW_MODE_DEC, 1); // 1 uint8
350 2 : g_traceStructItemFieldSet(
351 : g_traceMsgInfoSt, "rcvCnt", TRACE_STRUCT_FIELD_TYPE_UINT8, TRACE_STRUCT_SHOW_MODE_DEC, 1); // 1 uint8
352 2 : g_traceStructItemFieldSet(
353 : g_traceMsgInfoSt, "everyTurnRsp", TRACE_STRUCT_FIELD_TYPE_UINT8, TRACE_STRUCT_SHOW_MODE_DEC, 1); // 1 uint8
354 2 : g_traceStructItemFieldSet(
355 : g_traceMsgInfoSt, "everyTurnWait", TRACE_STRUCT_FIELD_TYPE_UINT8, TRACE_STRUCT_SHOW_MODE_DEC, 1); // 1 uint8
356 2 : g_traceStructItemFieldSet(
357 : g_traceMsgInfoSt, "totalTurnCnt", TRACE_STRUCT_FIELD_TYPE_UINT8, TRACE_STRUCT_SHOW_MODE_DEC, 1); // 1 uint8
358 2 : g_traceStructItemFieldSet(
359 : g_traceMsgInfoSt, "useBufferType", TRACE_STRUCT_FIELD_TYPE_UINT8, TRACE_STRUCT_SHOW_MODE_DEC, 1); // 1 uint8
360 2 : g_traceStructItemFieldSet(
361 : g_traceMsgInfoSt, "winOffset", TRACE_STRUCT_ARRAY_TYPE_UINT64, TRACE_STRUCT_SHOW_MODE_DEC, 8); // 8 uint64
362 2 : g_traceStructItemArraySet(
363 : g_traceMsgInfoSt, "res", TRACE_STRUCT_ARRAY_TYPE_UINT8, TRACE_STRUCT_SHOW_MODE_DEC, 2); // 1 uint8 2 2*uint8
364 2 : }
365 :
366 2 : HcclResult MC2TraceUtils::InitMsgInfoHandle()
367 : {
368 2 : uint16_t msgInfoNum = GetMsgNum(sizeof(AivAicpuOpParam));
369 2 : TraceAttr attr = {};
370 2 : attr = {false, msgInfoNum, sizeof(AivAicpuOpParam), {nullptr}, 0, {0}};
371 2 : g_traceMsgInfoSt = g_traceStructEntryCreate("AivAicpuOpParam");
372 2 : MC2TraceUtils::SetTraceMsgInfo();
373 2 : g_traceStructSetAttr(g_traceMsgInfoSt, 0, &attr);
374 2 : g_traceMsgInfoHandle = g_traceCreateWithAttr(TRACER_TYPE_SCHEDULE, "AivAicpuOpParam", &attr);
375 2 : if (g_traceMsgInfoHandle < 0) {
376 1 : HCCL_ERROR("Create g_traceMsgInfoHandle failed, ret:%d", g_traceMsgInfoHandle);
377 1 : g_traceStructEntryDestroy(g_traceMsgInfoSt);
378 1 : return HCCL_E_INTERNAL;
379 : }
380 1 : TraStatus ret = g_traceEventBindTrace(g_eventHandle, g_traceMsgInfoHandle);
381 1 : if (ret != TRACE_SUCCESS) {
382 0 : HCCL_ERROR("Bind g_traceMsgInfoHandle failed, ret:%d", ret);
383 0 : return HCCL_E_INTERNAL;
384 : }
385 1 : return HCCL_SUCCESS;
386 : }
387 :
388 32 : void MC2TraceUtils::SetTraceSqeBatchInfo()
389 : {
390 32 : g_traceStructItemFieldSet(
391 : g_traceSqeBatchInfoSt, "addr1High", TRACE_STRUCT_FIELD_TYPE_UINT32, TRACE_STRUCT_SHOW_MODE_HEX, 4); // 4 uint32
392 32 : g_traceStructItemFieldSet(
393 : g_traceSqeBatchInfoSt, "addr1Low", TRACE_STRUCT_FIELD_TYPE_UINT32, TRACE_STRUCT_SHOW_MODE_HEX, 4); // 4 uint32
394 32 : g_traceStructItemFieldSet(
395 : g_traceSqeBatchInfoSt, "addr2High", TRACE_STRUCT_FIELD_TYPE_UINT32, TRACE_STRUCT_SHOW_MODE_HEX, 4); // 4 uint32
396 32 : g_traceStructItemFieldSet(
397 : g_traceSqeBatchInfoSt, "addr2Low", TRACE_STRUCT_FIELD_TYPE_UINT32, TRACE_STRUCT_SHOW_MODE_HEX, 4); // 4 uint32
398 32 : g_traceStructItemFieldSet(
399 : g_traceSqeBatchInfoSt, "sqeHeadIdx", TRACE_STRUCT_FIELD_TYPE_UINT32, TRACE_STRUCT_SHOW_MODE_DEC, 4); // 4 uint32
400 32 : g_traceStructItemFieldSet(
401 : g_traceSqeBatchInfoSt, "notifyId", TRACE_STRUCT_FIELD_TYPE_UINT32, TRACE_STRUCT_SHOW_MODE_DEC, 4); // 4 uint32
402 32 : g_traceStructItemFieldSet(
403 : g_traceSqeBatchInfoSt, "length", TRACE_STRUCT_FIELD_TYPE_UINT32, TRACE_STRUCT_SHOW_MODE_DEC, 4); // 4 uint32
404 32 : g_traceStructItemFieldSet(
405 : g_traceSqeBatchInfoSt, "partId", TRACE_STRUCT_FIELD_TYPE_UINT32, TRACE_STRUCT_SHOW_MODE_DEC, 4); // 4 uint32
406 32 : g_traceStructItemFieldSet(
407 : g_traceSqeBatchInfoSt, "remoteRank", TRACE_STRUCT_FIELD_TYPE_UINT32, TRACE_STRUCT_SHOW_MODE_DEC, 4); // 4 uint32
408 32 : g_traceStructItemFieldSet(
409 : g_traceSqeBatchInfoSt, "dataType", TRACE_STRUCT_FIELD_TYPE_UINT32, TRACE_STRUCT_SHOW_MODE_DEC, 4); // 4 uint32
410 32 : g_traceStructItemFieldSet(
411 : g_traceSqeBatchInfoSt, "streamId", TRACE_STRUCT_FIELD_TYPE_UINT16, TRACE_STRUCT_SHOW_MODE_DEC, 2); // 2 uint16
412 32 : g_traceStructItemFieldSet(
413 : g_traceSqeBatchInfoSt, "eventId", TRACE_STRUCT_FIELD_TYPE_UINT16, TRACE_STRUCT_SHOW_MODE_DEC, 2); // 2 uint16
414 32 : g_traceStructItemFieldSet(
415 : g_traceSqeBatchInfoSt, "taskId", TRACE_STRUCT_FIELD_TYPE_UINT16, TRACE_STRUCT_SHOW_MODE_DEC, 2); // 2 uint16
416 32 : g_traceStructItemFieldSet(
417 : g_traceSqeBatchInfoSt, "condValue", TRACE_STRUCT_FIELD_TYPE_UINT16, TRACE_STRUCT_SHOW_MODE_DEC, 2); // 2 uint16
418 32 : g_traceStructItemFieldSet(
419 : g_traceSqeBatchInfoSt, "isLast", TRACE_STRUCT_FIELD_TYPE_UINT8, TRACE_STRUCT_SHOW_MODE_DEC, 1); // 1 uint8
420 32 : g_traceStructItemFieldSet(
421 : g_traceSqeBatchInfoSt, "opCode", TRACE_STRUCT_FIELD_TYPE_UINT8, TRACE_STRUCT_SHOW_MODE_DEC, 1); // 1 uint8
422 32 : g_traceStructItemFieldSet(
423 : g_traceSqeBatchInfoSt, "sqeNum", TRACE_STRUCT_FIELD_TYPE_UINT8, TRACE_STRUCT_SHOW_MODE_DEC, 1); // 1 uint8
424 32 : g_traceStructItemFieldSet(
425 : g_traceSqeBatchInfoSt, "type", TRACE_STRUCT_FIELD_TYPE_UINT8, TRACE_STRUCT_SHOW_MODE_DEC, 1); // 1 uint8
426 32 : g_traceStructItemFieldSet(
427 : g_traceSqeBatchInfoSt, "subType", TRACE_STRUCT_FIELD_TYPE_UINT8, TRACE_STRUCT_SHOW_MODE_DEC, 1); // 1 uint8
428 32 : g_traceStructItemFieldSet(
429 : g_traceSqeBatchInfoSt, "valid", TRACE_STRUCT_FIELD_TYPE_UINT8, TRACE_STRUCT_SHOW_MODE_DEC, 1); // 1 uint8
430 32 : g_traceStructItemArraySet(
431 : g_traceSqeBatchInfoSt, "reverse", TRACE_STRUCT_ARRAY_TYPE_UINT8, TRACE_STRUCT_SHOW_MODE_DEC,
432 : 10); // 1 uint8 10 10*uint8
433 32 : }
434 :
435 2 : HcclResult MC2TraceUtils::InitSqeBatchInfoHandle()
436 : {
437 2 : uint16_t sqeBatchInfoMsgNum = GetMsgNum(sizeof(SqeBatchInfo));
438 2 : TraceAttr attr = {};
439 2 : attr = {false, sqeBatchInfoMsgNum, sizeof(SqeBatchInfo), {nullptr}, 0, {0}};
440 2 : g_traceSqeBatchInfoSt = g_traceStructEntryCreate("SqeBatchInfo");
441 34 : for (uint32_t i = 0; i < MAX_SQE_BATCH_SIZE; i++) {
442 32 : SetTraceSqeBatchInfo();
443 : }
444 2 : g_traceStructSetAttr(g_traceSqeBatchInfoSt, 0, &attr);
445 2 : g_traceSqeBatchInfoHandle = g_traceCreateWithAttr(TRACER_TYPE_SCHEDULE, "SqeBatchInfo", &attr);
446 2 : if (g_traceSqeBatchInfoHandle < 0) {
447 1 : HCCL_ERROR("Create g_traceSqeBatchInfoHandle failed, ret:%d", g_traceSqeBatchInfoHandle);
448 1 : g_traceStructEntryDestroy(g_traceSqeBatchInfoSt);
449 1 : return HCCL_E_INTERNAL;
450 : }
451 1 : TraStatus ret = g_traceEventBindTrace(g_eventHandle, g_traceSqeBatchInfoHandle);
452 1 : if (ret != TRACE_SUCCESS) {
453 0 : HCCL_ERROR("Bind g_traceSqeBatchInfoHandle failed, ret:%d", ret);
454 0 : return HCCL_E_INTERNAL;
455 : }
456 1 : return HCCL_SUCCESS;
457 : }
458 :
459 1 : HcclResult MC2TraceUtils::InitTraceHandle()
460 : {
461 1 : g_eventHandle = g_traceEventCreate("mc2_dfx");
462 1 : if (g_eventHandle < 0) {
463 0 : HCCL_ERROR("Create mc2_dfx event handle failed");
464 0 : return HCCL_E_INTERNAL;
465 : }
466 1 : CHK_RET(InitTaskAndTilingDataHandle());
467 1 : CHK_RET(InitAicpuComDataHandle());
468 1 : CHK_RET(InitTraceStrHandle());
469 1 : CHK_RET(InitMsgInfoHandle());
470 1 : CHK_RET(InitSqeBatchInfoHandle());
471 1 : return HCCL_SUCCESS;
472 : }
473 :
474 2 : HcclResult MC2TraceUtils::GetTraceFunc(const std::string& traceName)
475 : {
476 2 : g_traceEventBindTrace = reinterpret_cast<TraceEventBindTraceFunc>(
477 2 : mmDlsym(g_soHandle, std::string(traceName + "EventBindTrace").c_str()));
478 2 : CHK_PRT_RET(
479 : g_traceEventBindTrace == nullptr, HCCL_ERROR("Get g_traceEventBindTrace error: %s", mmDlerror()),
480 : HCCL_E_SYSCALL);
481 : g_traceEventReport
482 2 : = reinterpret_cast<TraceEventReportFunc>(mmDlsym(g_soHandle, std::string(traceName + "EventReport").c_str()));
483 2 : CHK_PRT_RET(
484 : g_traceEventReport == nullptr, HCCL_ERROR("Get g_traceEventReport error: %s", mmDlerror()), HCCL_E_SYSCALL);
485 2 : g_traceSetGlobalAttr = reinterpret_cast<TraceSetGlobalAttrFunc>(
486 2 : mmDlsym(g_soHandle, std::string(traceName + "SetGlobalAttr").c_str()));
487 2 : CHK_PRT_RET(
488 : g_traceSetGlobalAttr == nullptr, HCCL_ERROR("Get g_traceSetGlobalAttr error: %s", mmDlerror()), HCCL_E_SYSCALL);
489 2 : g_traceStructEntryCreate = reinterpret_cast<TraceStructEntryCreateFunc>(
490 2 : mmDlsym(g_soHandle, std::string(traceName + "StructEntryCreate").c_str()));
491 2 : CHK_PRT_RET(
492 : g_traceStructEntryCreate == nullptr, HCCL_ERROR("Get g_traceStructEntryCreate error: %s", mmDlerror()),
493 : HCCL_E_SYSCALL);
494 2 : g_traceStructItemFieldSet = reinterpret_cast<TraceStructItemFieldSetFunc>(
495 2 : mmDlsym(g_soHandle, std::string(traceName + "StructItemFieldSet").c_str()));
496 2 : CHK_PRT_RET(
497 : g_traceStructItemFieldSet == nullptr, HCCL_ERROR("Get g_traceStructItemFieldSet error: %s", mmDlerror()),
498 : HCCL_E_SYSCALL);
499 2 : g_traceStructItemArraySet = reinterpret_cast<TraceStructItemArraySetFunc>(
500 2 : mmDlsym(g_soHandle, std::string(traceName + "StructItemArraySet").c_str()));
501 2 : CHK_PRT_RET(
502 : g_traceStructItemArraySet == nullptr, HCCL_ERROR("Get g_traceStructItemArraySet error: %s", mmDlerror()),
503 : HCCL_E_SYSCALL);
504 2 : g_traceStructSetAttr = reinterpret_cast<TraceStructSetAttrFunc>(
505 2 : mmDlsym(g_soHandle, std::string(traceName + "StructSetAttr").c_str()));
506 2 : CHK_PRT_RET(
507 : g_traceStructSetAttr == nullptr, HCCL_ERROR("Get g_traceStructSetAttr error: %s", mmDlerror()), HCCL_E_SYSCALL);
508 2 : g_traceStructEntryDestroy = reinterpret_cast<TraceStructEntryDestroyFunc>(
509 2 : mmDlsym(g_soHandle, std::string(traceName + "StructEntryDestroy").c_str()));
510 2 : CHK_PRT_RET(
511 : g_traceStructEntryDestroy == nullptr, HCCL_ERROR("Get g_traceStructEntryDestroy error: %s", mmDlerror()),
512 : HCCL_E_SYSCALL);
513 2 : return HCCL_SUCCESS;
514 : }
515 :
516 2 : HcclResult MC2TraceUtils::InitFuncHandle()
517 : {
518 2 : std::string traceName = g_isAtrace ? "Atrace" : "Utrace";
519 2 : HCCL_INFO("Start mmDlsym %s funcHandle", traceName.c_str());
520 2 : g_traceCreateWithAttr = reinterpret_cast<TraceCreateWithAttrFunc>(
521 2 : mmDlsym(g_soHandle, std::string(traceName + "CreateWithAttr").c_str()));
522 2 : CHK_PRT_RET(
523 : g_traceCreateWithAttr == nullptr, HCCL_ERROR("Get g_traceCreateWithAttr error: %s", mmDlerror()),
524 : HCCL_E_SYSCALL);
525 : g_traceGetHandle
526 2 : = reinterpret_cast<TraceGetHandleFunc>(mmDlsym(g_soHandle, std::string(traceName + "GetHandle").c_str()));
527 2 : CHK_PRT_RET(g_traceGetHandle == nullptr, HCCL_ERROR("Get g_traceGetHandle error: %s", mmDlerror()), HCCL_E_SYSCALL);
528 2 : g_traceSubmit = reinterpret_cast<TraceSubmitFunc>(mmDlsym(g_soHandle, std::string(traceName + "Submit").c_str()));
529 2 : CHK_PRT_RET(g_traceSubmit == nullptr, HCCL_ERROR("Get g_traceSubmit error: %s", mmDlerror()), HCCL_E_SYSCALL);
530 : g_traceEventCreate
531 2 : = reinterpret_cast<TraceEventCreateFunc>(mmDlsym(g_soHandle, std::string(traceName + "EventCreate").c_str()));
532 2 : CHK_PRT_RET(
533 : g_traceEventCreate == nullptr, HCCL_ERROR("Get g_traceEventCreate error: %s", mmDlerror()), HCCL_E_SYSCALL);
534 : g_traceEventDestroy
535 2 : = reinterpret_cast<TraceEventDestroyFunc>(mmDlsym(g_soHandle, std::string(traceName + "EventDestroy").c_str()));
536 2 : CHK_PRT_RET(
537 : g_traceEventDestroy == nullptr, HCCL_ERROR("Get g_traceEventDestroy error: %s", mmDlerror()), HCCL_E_SYSCALL);
538 : g_traceHandleDestroy
539 2 : = reinterpret_cast<TraceHandleDestroyFunc>(mmDlsym(g_soHandle, std::string(traceName + "Destroy").c_str()));
540 2 : CHK_PRT_RET(
541 : g_traceHandleDestroy == nullptr, HCCL_ERROR("Get g_traceHandleDestroy error: %s", mmDlerror()), HCCL_E_SYSCALL);
542 2 : CHK_RET(GetTraceFunc(traceName));
543 2 : return HCCL_SUCCESS;
544 2 : }
545 :
546 47 : __attribute__((constructor)) void DlopenTraceSo()
547 : {
548 47 : if (g_soHandle == nullptr) {
549 47 : g_soHandle = mmDlopen(ATRACE_SO, RTLD_NOW | RTLD_GLOBAL);
550 47 : if (g_soHandle == nullptr) {
551 47 : g_soHandle = mmDlopen(UTRACE_SO, RTLD_NOW | RTLD_GLOBAL);
552 47 : g_isAtrace = false;
553 : }
554 47 : HCCL_INFO("mmDlopen success");
555 : }
556 47 : }
557 :
558 84 : HcclResult MC2TraceUtils::Init()
559 : {
560 84 : if (g_soHandle == nullptr) {
561 84 : HCCL_WARNING("Cannot find so file: %s", UTRACE_SO);
562 84 : return HCCL_SUCCESS;
563 : }
564 0 : CHK_RET(InitFuncHandle());
565 0 : CHK_RET(InitTraceHandle());
566 0 : unsigned int hostpid = 0;
567 0 : unsigned int cpType = DEVDRV_PROCESS_CPTYPE_MAX;
568 0 : CHK_RET(HrtHalDrvQueryProcessHostPid(getpid(), nullptr, nullptr, &hostpid, &cpType));
569 :
570 0 : TraceGlobalAttr traceGlobalAttr = {};
571 0 : traceGlobalAttr.saveMode = 1;
572 0 : traceGlobalAttr.deviceId = AicpuGetComContext()->devId;
573 0 : traceGlobalAttr.pid = hostpid;
574 0 : TraStatus ret = g_traceSetGlobalAttr(&traceGlobalAttr);
575 0 : if (ret != TRACE_SUCCESS) {
576 0 : HCCL_ERROR("TraceSetGlobalAttrFunc failed, ret:%d", ret);
577 0 : return HCCL_E_INTERNAL;
578 : }
579 0 : return HCCL_SUCCESS;
580 : }
581 :
582 : template <typename T>
583 1 : HcclResult MC2TraceUtils::Submit(const T* const traceData)
584 : {
585 1 : if (g_soHandle == nullptr || g_traceGetHandle == nullptr || g_traceSubmit == nullptr) {
586 1 : return HCCL_SUCCESS;
587 : }
588 0 : CHK_PTR_NULL(traceData);
589 0 : std::string typeName;
590 : if (std::is_same<T, KFCtaskAndTilingTraceData>::value) {
591 0 : typeName = "KFCtaskAndTilingTraceData";
592 : } else if (std::is_same<T, AicpuComTraceData>::value) {
593 0 : typeName = "AicpuComTraceData";
594 : } else if (std::is_same<T, AivAicpuOpParam>::value) {
595 : typeName = "AivAicpuOpParam";
596 : } else if (std::is_same<T, SqeBatchInfo>::value) {
597 0 : typeName = "SqeBatchInfo";
598 : } else {
599 : HCCL_ERROR("find typename: %s failed", typeid(T).name());
600 : return HCCL_E_INTERNAL;
601 : }
602 :
603 0 : auto traceHandle = g_traceGetHandle(TRACER_TYPE_SCHEDULE, typeName.c_str());
604 0 : if (traceHandle < 0) {
605 0 : HCCL_ERROR("getHandle %s failed, ret:%d", typeName.c_str(), traceHandle);
606 0 : return HCCL_E_INTERNAL;
607 : }
608 0 : auto traceRet = g_traceSubmit(traceHandle, reinterpret_cast<const void*>(traceData), sizeof(T));
609 0 : if (traceRet != TRACE_SUCCESS) {
610 0 : HCCL_ERROR("submit %s failed, ret:%d", typeName.c_str(), traceRet);
611 0 : return HCCL_E_INTERNAL;
612 : }
613 0 : return HCCL_SUCCESS;
614 0 : }
615 :
616 64 : HcclResult MC2TraceUtils::Submit(AicpuComContext* ctx)
617 : {
618 64 : if (g_soHandle == nullptr) {
619 64 : return HCCL_SUCCESS;
620 : }
621 0 : CHK_PTR_NULL(ctx);
622 0 : uint64_t startUsec = GetCurCpuTimestamp();
623 : AicpuComTraceData comTraceData;
624 0 : comTraceData.devId = ctx->devId;
625 0 : comTraceData.ssid = ctx->ssid;
626 0 : comTraceData.rankId = ctx->rankId;
627 0 : comTraceData.rankNum = ctx->rankNum;
628 0 : comTraceData.windowSize = ctx->windowSize;
629 0 : comTraceData.workSpaceAddr = ctx->workSpaceAddr;
630 0 : comTraceData.kfcNotifyId = ctx->kfcNotifyId;
631 0 : for (uint32_t i = 0; i < AC_MAX_RANK_NUM; i++) {
632 0 : comTraceData.eventIds[i] = ctx->eventIds[i];
633 0 : comTraceData.windowIn[i] = ctx->rankInfo[i].window;
634 0 : comTraceData.windowOut[i] = ctx->rankInfo[i].windowOut;
635 0 : comTraceData.actualStreamId[i] = ctx->streamInfo[i].actualStreamId;
636 0 : comTraceData.sqId[i] = ctx->streamInfo[i].sqId;
637 : }
638 0 : for (uint32_t i = 0; i < 2; i++) { // 2 aicpuOpNotify size
639 0 : comTraceData.aicpuOpNotifyAddress[i] = ctx->aicpuOpNotify[i].address;
640 0 : comTraceData.aicpuOpNotifyActualNotifyId[i] = ctx->aicpuOpNotify[i].actualNotifyId;
641 : }
642 0 : comTraceData.clusterId = ctx->clusterId;
643 0 : CHK_RET(Submit<AicpuComTraceData>(&comTraceData));
644 0 : if (AicpuKfcUtils::IsDebugModeEquals(*ctx, MC2_DEBUG_TIME_TAKEN)) {
645 0 : AicpuKfcProf::GetProInst(*ctx).traceCtxTime += GetCurCpuTimestamp() - startUsec;
646 : }
647 0 : return HCCL_SUCCESS;
648 : }
649 :
650 61 : HcclResult MC2TraceUtils::Submit(const KFCTask* const task, const HcclKFCTilingData* const tilingData)
651 : {
652 61 : if (g_soHandle == nullptr) {
653 61 : return HCCL_SUCCESS;
654 : }
655 0 : CHK_PTR_NULL(task);
656 0 : CHK_PTR_NULL(tilingData);
657 0 : uint64_t startUsec = GetCurCpuTimestamp();
658 : KFCtaskAndTilingTraceData taskAndTilingData;
659 0 : error_t ret = memcpy_s(&taskAndTilingData, sizeof(KFCtaskAndTilingTraceData), task, sizeof(KFCTask));
660 0 : if (ret != EOK) {
661 0 : HCCL_ERROR("task memcpy_s fail ret:%d", ret);
662 0 : return HCCL_E_MEMORY;
663 : }
664 0 : ret = memcpy_s(
665 : reinterpret_cast<uint8_t*>(&taskAndTilingData) + sizeof(KFCTask),
666 : sizeof(KFCtaskAndTilingTraceData) - sizeof(KFCTask), tilingData, sizeof(HcclKFCTilingData));
667 0 : if (ret != EOK) {
668 0 : HCCL_ERROR("tilingData memcpy_s fail ret:%d", ret);
669 0 : return HCCL_E_MEMORY;
670 : }
671 0 : CHK_RET(Submit<KFCtaskAndTilingTraceData>(&taskAndTilingData));
672 0 : auto ctx = AicpuGetComContext();
673 0 : if (AicpuKfcUtils::IsDebugModeEquals(*ctx, MC2_DEBUG_TIME_TAKEN)) {
674 0 : AicpuKfcProf::GetProInst(*ctx).traceSubmitTime += GetCurCpuTimestamp() - startUsec;
675 : }
676 0 : return HCCL_SUCCESS;
677 : }
678 :
679 1 : HcclResult MC2TraceUtils::Submit(const std::string& traceStr)
680 : {
681 1 : if (g_soHandle == nullptr) {
682 1 : return HCCL_SUCCESS;
683 : }
684 0 : return Submit(traceStr.c_str());
685 : }
686 :
687 1 : HcclResult MC2TraceUtils::Submit(const char* traceStr)
688 : {
689 1 : if (g_soHandle == nullptr || g_traceSubmit == nullptr) {
690 1 : return HCCL_SUCCESS;
691 : }
692 0 : uint64_t startUsec = GetCurCpuTimestamp();
693 0 : uint32_t strSize = strlen(traceStr);
694 0 : uint32_t posSize = 0U;
695 0 : TraceStr singleTraceStr;
696 0 : while (posSize < strSize) {
697 0 : uint32_t traceStrLen = std::min<uint32_t>(strSize - posSize, sizeof(singleTraceStr.transmit) - 1);
698 0 : CHK_SAFETY_FUNC_RET(memset_s(
699 : reinterpret_cast<void*>(&singleTraceStr.transmit), sizeof(singleTraceStr.transmit), 0,
700 : sizeof(singleTraceStr.transmit)));
701 0 : error_t ret = memcpy_s(singleTraceStr.transmit, traceStrLen, traceStr + posSize, traceStrLen);
702 0 : if (ret != EOK) {
703 0 : HCCL_ERROR("traceStr memcpy_s fail ret:%d", ret);
704 0 : return HCCL_E_MEMORY;
705 : }
706 0 : posSize += traceStrLen;
707 : auto traceStrRet
708 0 : = g_traceSubmit(g_traceStrHandle, reinterpret_cast<const void*>(&singleTraceStr), sizeof(singleTraceStr));
709 0 : if (traceStrRet != TRACE_SUCCESS) {
710 0 : HCCL_ERROR("submittraceStr failed, ret:%d", traceStrRet);
711 0 : return HCCL_E_INTERNAL;
712 : }
713 0 : HCCL_INFO("Submit string: %s", singleTraceStr.transmit);
714 : }
715 0 : auto ctx = AicpuGetComContext();
716 0 : if (AicpuKfcUtils::IsDebugModeEquals(*ctx, MC2_DEBUG_TIME_TAKEN)) {
717 0 : AicpuKfcProf::GetProInst(*ctx).traceSubmitTime += GetCurCpuTimestamp() - startUsec;
718 : }
719 0 : return HCCL_SUCCESS;
720 : }
721 :
722 0 : HcclResult MC2TraceUtils::SubmitBatchSqeInfo()
723 : {
724 0 : if (g_soHandle == nullptr) {
725 0 : return HCCL_SUCCESS;
726 : }
727 0 : uint64_t startUsec = GetCurCpuTimestamp();
728 0 : SqeContext* context = GetSqeContext();
729 0 : CHK_PTR_NULL(context->buffPtr);
730 0 : for (uint32_t streamId = 0U; streamId < AC_MAX_RANK_NUM; streamId++) {
731 0 : auto& buff = context->buffPtr[streamId];
732 0 : HCCL_INFO(
733 : "sqTail: %u, sqHead: %u, sqeCnt: %u, tailSqeTaskId: %u, tailSqeIdx: %u", buff.sqTail, buff.sqHead,
734 : buff.sqeCnt, buff.tailSqeTaskId, buff.tailSqeIdx);
735 0 : auto submitNum = std::min<uint32_t>(buff.tailSqeIdx - buff.sqeCnt, MAX_SQE_SUBMIT_NUM);
736 0 : for (uint32_t i = 0U; i <= submitNum / MAX_SQE_BATCH_SIZE; i++) {
737 0 : SqeBatchInfo sqeBatchInfo;
738 0 : for (uint32_t j = 0U; j < MAX_SQE_BATCH_SIZE; j++) {
739 0 : const auto idx = i * MAX_SQE_BATCH_SIZE + j;
740 0 : if (idx >= submitNum) {
741 0 : break;
742 : }
743 0 : if (AicpuSqeContext::QuerySqeInfoByTaskId(
744 0 : streamId, buff.tailSqeTaskId - submitNum + idx, &sqeBatchInfo.sqeInfos[j])
745 0 : != HCCL_SUCCESS) {
746 0 : HCCL_WARNING("Query sqe info failed, stream:%u, id:%u", streamId, idx);
747 0 : break;
748 : }
749 : }
750 0 : CHK_RET(Submit<SqeBatchInfo>(&sqeBatchInfo));
751 : }
752 : }
753 0 : auto ctx = AicpuGetComContext();
754 0 : if (AicpuKfcUtils::IsDebugModeEquals(*ctx, MC2_DEBUG_TIME_TAKEN)) {
755 0 : AicpuKfcProf::GetProInst(*ctx).traceSqeTime += GetCurCpuTimestamp() - startUsec;
756 : }
757 0 : return HCCL_SUCCESS;
758 : }
759 :
760 10 : HcclResult MC2TraceUtils::Save()
761 : {
762 10 : if (g_soHandle == nullptr) {
763 10 : return HCCL_SUCCESS;
764 : }
765 0 : TraStatus ret = g_traceEventReport(g_eventHandle);
766 0 : if (ret != TRACE_SUCCESS) {
767 0 : HCCL_ERROR("TraceEventReport failed, ret:%d", ret);
768 0 : return HCCL_E_INTERNAL;
769 : }
770 0 : return HCCL_SUCCESS;
771 : }
772 :
773 2 : HcclResult MC2TraceUtils::DestoryHandles()
774 : {
775 2 : if (g_soHandle == nullptr) {
776 2 : return HCCL_SUCCESS;
777 : }
778 0 : g_traceStructEntryDestroy(g_traceStrSt);
779 0 : g_traceStructEntryDestroy(g_traceAicpuComTraceSt);
780 0 : g_traceStructEntryDestroy(g_traceKFCtaskAndTilingTraceDataSt);
781 0 : g_traceStructEntryDestroy(g_traceMsgInfoSt);
782 0 : g_traceStructEntryDestroy(g_traceSqeBatchInfoSt);
783 0 : g_traceHandleDestroy(g_traceStrHandle);
784 0 : g_traceHandleDestroy(g_traceTaskAndTilingDataHandle);
785 0 : g_traceHandleDestroy(g_traceAicpuComDataHandle);
786 0 : g_traceHandleDestroy(g_traceMsgInfoHandle);
787 0 : g_traceHandleDestroy(g_traceSqeBatchInfoHandle);
788 0 : g_traceEventDestroy(g_eventHandle);
789 :
790 0 : return HCCL_SUCCESS;
791 : }
792 :
793 47 : __attribute__((destructor)) void Finalize()
794 : {
795 47 : if (g_soHandle != nullptr) {
796 0 : (void)mmDlclose(g_soHandle);
797 : }
798 47 : g_soHandle = nullptr;
799 47 : }
800 :
801 : // 显式实例化
802 : template HcclResult MC2TraceUtils::Submit(const SqeBatchInfo* const);
|
