Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "mc2_trace_utils.h"
12 :
13 : #include <cmath>
14 : #include "log.h"
15 : #include "mmpa_api.h"
16 : #include "atrace_pub.h"
17 : #include "common/aicpu_hccl_common.h"
18 : #include "framework/aicpu_kfc_prof.h"
19 : #include "common/aicpu_kfc_utils.h"
20 : #include "adapter_hal_pub.h"
21 :
22 : namespace {
23 : constexpr uint32_t TRACE_RING_BUFF_SIZE = 128 * 1024;
24 : constexpr uint32_t MAX_SQE_SUBMIT_NUM = 256U;
25 : constexpr char UTRACE_SO[] = "libutrace.so";
26 : constexpr char ATRACE_SO[] = "libascend_trace.so";
27 :
28 : using TraceCreateWithAttrFunc = TraHandle (*)(TracerType, const char *, const TraceAttr *);
29 : using TraceGetHandleFunc = TraHandle (*)(TracerType, const char *);
30 : using TraceSubmitFunc = TraStatus (*)(TraHandle handle, const void *, uint32_t);
31 : using TraceEventCreateFunc = TraEventHandle (*)(const char *);
32 : using TraceEventDestroyFunc = void (*)(TraEventHandle);
33 : using TraceHandleDestroyFunc = void (*)(TraHandle);
34 : using TraceEventBindTraceFunc = TraStatus (*)(TraEventHandle, TraHandle);
35 : using TraceEventReportFunc = TraStatus (*)(TraEventHandle);
36 : using TraceSetGlobalAttrFunc = TraStatus (*)(const TraceGlobalAttr *);
37 : using TraceStructEntryCreateFunc = TraceStructEntry *(*)(const char *);
38 : using TraceStructItemFieldSetFunc = void (*)(TraceStructEntry *, const char *, uint8_t, uint8_t, uint64_t);
39 : using TraceStructItemArraySetFunc = void (*)(TraceStructEntry *, const char *, uint8_t, uint8_t, uint64_t);
40 : using TraceStructSetAttrFunc = void (*)(TraceStructEntry *, uint8_t, TraceAttr *);
41 : using TraceStructEntryDestroyFunc = void (*)(TraceStructEntry *);
42 :
43 : void *g_soHandle = nullptr;
44 : bool g_isAtrace = true;
45 : TraceCreateWithAttrFunc g_traceCreateWithAttr = nullptr;
46 : TraceGetHandleFunc g_traceGetHandle = nullptr;
47 : TraceSubmitFunc g_traceSubmit = nullptr;
48 : TraceEventCreateFunc g_traceEventCreate = nullptr;
49 : TraceEventDestroyFunc g_traceEventDestroy = nullptr;
50 : TraceHandleDestroyFunc g_traceHandleDestroy = nullptr;
51 : TraceEventBindTraceFunc g_traceEventBindTrace = nullptr;
52 : TraceEventReportFunc g_traceEventReport = nullptr;
53 : TraceSetGlobalAttrFunc g_traceSetGlobalAttr = nullptr;
54 : TraceStructEntryCreateFunc g_traceStructEntryCreate = nullptr;
55 : TraceStructItemFieldSetFunc g_traceStructItemFieldSet = nullptr;
56 : TraceStructItemArraySetFunc g_traceStructItemArraySet = nullptr;
57 : TraceStructSetAttrFunc g_traceStructSetAttr = nullptr;
58 : TraceStructEntryDestroyFunc g_traceStructEntryDestroy = nullptr;
59 :
60 : TraEventHandle g_eventHandle = -1;
61 : TraHandle g_traceStrHandle = -1;
62 : TraHandle g_traceTaskAndTilingDataHandle = -1;
63 : TraHandle g_traceAicpuComDataHandle = -1;
64 : TraHandle g_traceMsgInfoHandle = -1;
65 : TraHandle g_traceSqeBatchInfoHandle = -1;
66 : TraceStructEntry *g_traceStrSt = nullptr;
67 : TraceStructEntry *g_traceAicpuComTraceSt = nullptr;
68 : TraceStructEntry *g_traceKFCtaskAndTilingTraceDataSt = nullptr;
69 : TraceStructEntry *g_traceMsgInfoSt = nullptr;
70 : TraceStructEntry *g_traceSqeBatchInfoSt = nullptr;
71 : }
72 :
73 : // 当msgNum不为2的幂时，会自动向上取2的幂
74 10 : uint16_t MC2TraceUtils::GetMsgNum(size_t msgSize)
75 : {
76 10 : float logValue = log(TRACE_RING_BUFF_SIZE / (msgSize + 16)) / log(2); // 16 预留字节 2 幂次
77 10 : return static_cast<uint16_t>(pow(2, floor(logValue))); // 2 幂次
78 : }
79 :
80 2 : HcclResult MC2TraceUtils::InitTraceStrHandle()
81 : {
82 : // trace api存在限制: msgNum * msgSize <= 128k
83 2 : uint16_t strMsgNum = GetMsgNum(sizeof(TraceStr));
84 2 : TraceAttr attr = {0};
85 2 : attr = {false, strMsgNum, sizeof(TraceStr), nullptr};
86 2 : g_traceStrSt = g_traceStructEntryCreate("TraceStr");
87 2 : g_traceStructItemArraySet(g_traceStrSt, "transmit", TRACE_STRUCT_ARRAY_TYPE_CHAR, TRACE_STRUCT_SHOW_MODE_CHAR,
88 : sizeof(TraceStr::transmit));
89 2 : g_traceStructSetAttr(g_traceStrSt, 0, &attr);
90 2 : g_traceStrHandle = g_traceCreateWithAttr(TRACER_TYPE_SCHEDULE, "TraceStr", &attr);
91 2 : if (g_traceStrHandle < 0) {
92 1 : HCCL_ERROR("Create g_traceStrHandle failed, ret:%d", g_traceStrHandle);
93 1 : g_traceStructEntryDestroy(g_traceStrSt);
94 1 : return HCCL_E_INTERNAL;
95 : }
96 1 : TraStatus ret = g_traceEventBindTrace(g_eventHandle, g_traceStrHandle);
97 1 : if (ret != TRACE_SUCCESS) {
98 0 : HCCL_ERROR("Bind g_traceStrHandle failed, ret:%d", ret);
99 0 : return HCCL_E_INTERNAL;
100 : }
101 1 : return HCCL_SUCCESS;
102 : }
103 :
104 2 : void MC2TraceUtils::SetHcclKFCTilingDataOne()
105 : {
106 2 : g_traceStructItemFieldSet(g_traceKFCtaskAndTilingTraceDataSt, "sendOff", TRACE_STRUCT_FIELD_TYPE_UINT64,
107 : TRACE_STRUCT_SHOW_MODE_HEX, 8); // 8 uint64
108 2 : g_traceStructItemFieldSet(g_traceKFCtaskAndTilingTraceDataSt, "recvOff", TRACE_STRUCT_FIELD_TYPE_UINT64,
109 : TRACE_STRUCT_SHOW_MODE_HEX, 8); // 8 uint64
110 2 : g_traceStructItemFieldSet(g_traceKFCtaskAndTilingTraceDataSt, "tailSendOff", TRACE_STRUCT_FIELD_TYPE_UINT64,
111 : TRACE_STRUCT_SHOW_MODE_HEX, 8); // 8 uint64
112 2 : g_traceStructItemFieldSet(g_traceKFCtaskAndTilingTraceDataSt, "tailRecvOff", TRACE_STRUCT_FIELD_TYPE_UINT64,
113 : TRACE_STRUCT_SHOW_MODE_HEX, 8); // 8 uint64
114 2 : g_traceStructItemFieldSet(g_traceKFCtaskAndTilingTraceDataSt, "sendCnt", TRACE_STRUCT_FIELD_TYPE_UINT64,
115 : TRACE_STRUCT_SHOW_MODE_HEX, 8); // 8 uint64
116 2 : g_traceStructItemFieldSet(g_traceKFCtaskAndTilingTraceDataSt, "recvCnt", TRACE_STRUCT_FIELD_TYPE_UINT64,
117 : TRACE_STRUCT_SHOW_MODE_HEX, 8); // 8 uint64
118 2 : g_traceStructItemFieldSet(g_traceKFCtaskAndTilingTraceDataSt, "tailSendCnt", TRACE_STRUCT_FIELD_TYPE_UINT64,
119 : TRACE_STRUCT_SHOW_MODE_HEX, 8); // 8 uint64
120 2 : g_traceStructItemFieldSet(g_traceKFCtaskAndTilingTraceDataSt, "tailRecvCnt", TRACE_STRUCT_FIELD_TYPE_UINT64,
121 : TRACE_STRUCT_SHOW_MODE_HEX, 8); // 8 uint64
122 2 : g_traceStructItemFieldSet(g_traceKFCtaskAndTilingTraceDataSt, "totalCnt", TRACE_STRUCT_FIELD_TYPE_UINT64,
123 : TRACE_STRUCT_SHOW_MODE_HEX, 8); // 8 uint64
124 2 : g_traceStructItemFieldSet(g_traceKFCtaskAndTilingTraceDataSt, "turnNum", TRACE_STRUCT_FIELD_TYPE_UINT32,
125 : TRACE_STRUCT_SHOW_MODE_DEC, 4); // 4 uint32
126 2 : g_traceStructItemFieldSet(g_traceKFCtaskAndTilingTraceDataSt, "tailNum", TRACE_STRUCT_FIELD_TYPE_UINT32,
127 : TRACE_STRUCT_SHOW_MODE_DEC, 4); // 4 uint32
128 2 : g_traceStructItemFieldSet(g_traceKFCtaskAndTilingTraceDataSt, "stride", TRACE_STRUCT_FIELD_TYPE_UINT32,
129 : TRACE_STRUCT_SHOW_MODE_DEC, 4); // 4 uint32
130 2 : g_traceStructItemFieldSet(g_traceKFCtaskAndTilingTraceDataSt, "workspaceOff", TRACE_STRUCT_FIELD_TYPE_UINT32,
131 : TRACE_STRUCT_SHOW_MODE_DEC, 4); // 4 uint32
132 2 : g_traceStructItemFieldSet(g_traceKFCtaskAndTilingTraceDataSt, "notifyOff", TRACE_STRUCT_FIELD_TYPE_UINT32,
133 : TRACE_STRUCT_SHOW_MODE_DEC, 4); // 4 uint32
134 2 : g_traceStructItemFieldSet(g_traceKFCtaskAndTilingTraceDataSt, "notifyBeginCnt", TRACE_STRUCT_FIELD_TYPE_UINT16,
135 : TRACE_STRUCT_SHOW_MODE_DEC, 2); // 2 uint16
136 2 : g_traceStructItemFieldSet(g_traceKFCtaskAndTilingTraceDataSt, "notifyEndCnt", TRACE_STRUCT_FIELD_TYPE_UINT16,
137 : TRACE_STRUCT_SHOW_MODE_DEC, 2); // 2 uint16
138 2 : g_traceStructItemFieldSet(g_traceKFCtaskAndTilingTraceDataSt, "useBufferType", TRACE_STRUCT_FIELD_TYPE_UINT8,
139 : TRACE_STRUCT_SHOW_MODE_DEC, 1); // 1 uint8
140 2 : g_traceStructItemFieldSet(g_traceKFCtaskAndTilingTraceDataSt, "funID", TRACE_STRUCT_FIELD_TYPE_UINT8,
141 : TRACE_STRUCT_SHOW_MODE_DEC, 1); // 1 uint8
142 2 : g_traceStructItemFieldSet(g_traceKFCtaskAndTilingTraceDataSt, "dataType", TRACE_STRUCT_FIELD_TYPE_UINT8,
143 : TRACE_STRUCT_SHOW_MODE_DEC, 1); // 1 uint8
144 2 : g_traceStructItemFieldSet(g_traceKFCtaskAndTilingTraceDataSt, "groupNum", TRACE_STRUCT_FIELD_TYPE_UINT8,
145 : TRACE_STRUCT_SHOW_MODE_DEC, 1); // 1 uint8
146 2 : g_traceStructItemFieldSet(g_traceKFCtaskAndTilingTraceDataSt, "reuseMode", TRACE_STRUCT_FIELD_TYPE_UINT8,
147 : TRACE_STRUCT_SHOW_MODE_DEC, 1); // 1 uint8
148 2 : }
149 :
150 2 : void MC2TraceUtils::SetHcclKFCTilingDataTwo()
151 : {
152 2 : g_traceStructItemFieldSet(g_traceKFCtaskAndTilingTraceDataSt, "commType", TRACE_STRUCT_FIELD_TYPE_UINT8,
153 : TRACE_STRUCT_SHOW_MODE_DEC, 1); // 1 uint8
154 2 : g_traceStructItemFieldSet(g_traceKFCtaskAndTilingTraceDataSt, "reduceOp", TRACE_STRUCT_FIELD_TYPE_UINT8,
155 : TRACE_STRUCT_SHOW_MODE_DEC, 1); // 1 uint8
156 2 : g_traceStructItemFieldSet(g_traceKFCtaskAndTilingTraceDataSt, "commOrder", TRACE_STRUCT_FIELD_TYPE_UINT8,
157 : TRACE_STRUCT_SHOW_MODE_DEC, 1); // 1 uint8
158 2 : g_traceStructItemFieldSet(g_traceKFCtaskAndTilingTraceDataSt, "waitPolicy", TRACE_STRUCT_FIELD_TYPE_UINT8,
159 : TRACE_STRUCT_SHOW_MODE_DEC, 1); // 1 uint8
160 2 : g_traceStructItemFieldSet(g_traceKFCtaskAndTilingTraceDataSt, "rspPolicy", TRACE_STRUCT_FIELD_TYPE_UINT8,
161 : TRACE_STRUCT_SHOW_MODE_DEC, 1); // 1 uint8
162 2 : g_traceStructItemFieldSet(g_traceKFCtaskAndTilingTraceDataSt, "exitPolicy", TRACE_STRUCT_FIELD_TYPE_UINT8,
163 : TRACE_STRUCT_SHOW_MODE_DEC, 1); // 1 uint8
164 2 : g_traceStructItemFieldSet(g_traceKFCtaskAndTilingTraceDataSt, "commAlg", TRACE_STRUCT_FIELD_TYPE_UINT8,
165 : TRACE_STRUCT_SHOW_MODE_DEC, 1); // 1 uint8
166 2 : g_traceStructItemFieldSet(g_traceKFCtaskAndTilingTraceDataSt, "taskType", TRACE_STRUCT_FIELD_TYPE_UINT8,
167 : TRACE_STRUCT_SHOW_MODE_DEC, 1); // 1 uint8
168 2 : g_traceStructItemFieldSet(g_traceKFCtaskAndTilingTraceDataSt, "debugMode", TRACE_STRUCT_FIELD_TYPE_UINT8,
169 : TRACE_STRUCT_SHOW_MODE_DEC, 1); // 1 uint8
170 2 : g_traceStructItemFieldSet(g_traceKFCtaskAndTilingTraceDataSt, "stepSize", TRACE_STRUCT_FIELD_TYPE_UINT8,
171 : TRACE_STRUCT_SHOW_MODE_DEC, 1); // 1 uint8
172 2 : g_traceStructItemFieldSet(g_traceKFCtaskAndTilingTraceDataSt, "sendArgIndex", TRACE_STRUCT_FIELD_TYPE_UINT8,
173 : TRACE_STRUCT_SHOW_MODE_DEC, 1); // 1 uint8
174 2 : g_traceStructItemFieldSet(g_traceKFCtaskAndTilingTraceDataSt, "recvArgIndex", TRACE_STRUCT_FIELD_TYPE_UINT8,
175 : TRACE_STRUCT_SHOW_MODE_DEC, 1); // 1 uint8
176 2 : g_traceStructItemFieldSet(g_traceKFCtaskAndTilingTraceDataSt, "commOutArgIndex", TRACE_STRUCT_FIELD_TYPE_UINT8,
177 : TRACE_STRUCT_SHOW_MODE_DEC, 1); // 1 uint8
178 2 : g_traceStructItemFieldSet(g_traceKFCtaskAndTilingTraceDataSt, "hasCommOut", TRACE_STRUCT_FIELD_TYPE_UINT8,
179 : TRACE_STRUCT_SHOW_MODE_DEC, 1); // 1 uint8
180 2 : g_traceStructItemFieldSet(g_traceKFCtaskAndTilingTraceDataSt, "reverse", TRACE_STRUCT_FIELD_TYPE_UINT8,
181 : TRACE_STRUCT_SHOW_MODE_DEC, 1); // 1 uint8
182 2 : g_traceStructItemFieldSet(g_traceKFCtaskAndTilingTraceDataSt, "reserve2", TRACE_STRUCT_FIELD_TYPE_UINT32,
183 : TRACE_STRUCT_SHOW_MODE_DEC, 4); // 4 uint32
184 2 : }
185 :
186 2 : HcclResult MC2TraceUtils::InitTaskAndTilingDataHandle()
187 : {
188 2 : uint16_t taskAndTilingDataMsgNum = GetMsgNum(sizeof(KFCtaskAndTilingTraceData));
189 2 : TraceAttr attr = {0};
190 2 : attr = {false, taskAndTilingDataMsgNum, sizeof(KFCtaskAndTilingTraceData), nullptr};
191 2 : g_traceKFCtaskAndTilingTraceDataSt = g_traceStructEntryCreate("KFCtaskAndTilingTraceData");
192 2 : g_traceStructItemFieldSet(g_traceKFCtaskAndTilingTraceDataSt, "inputA", TRACE_STRUCT_FIELD_TYPE_UINT64,
193 : TRACE_STRUCT_SHOW_MODE_HEX, 8); // 8 uint64
194 2 : g_traceStructItemFieldSet(g_traceKFCtaskAndTilingTraceDataSt, "outputC", TRACE_STRUCT_FIELD_TYPE_UINT64,
195 : TRACE_STRUCT_SHOW_MODE_HEX, 8); // 8 uint64
196 2 : g_traceStructItemFieldSet(g_traceKFCtaskAndTilingTraceDataSt, "commOut", TRACE_STRUCT_FIELD_TYPE_UINT64,
197 : TRACE_STRUCT_SHOW_MODE_HEX, 8); // 8 uint64
198 2 : g_traceStructItemFieldSet(g_traceKFCtaskAndTilingTraceDataSt, "context", TRACE_STRUCT_FIELD_TYPE_UINT64,
199 : TRACE_STRUCT_SHOW_MODE_HEX, 8); // 8 uint64
200 2 : g_traceStructItemFieldSet(g_traceKFCtaskAndTilingTraceDataSt, "workSpace", TRACE_STRUCT_FIELD_TYPE_UINT64,
201 : TRACE_STRUCT_SHOW_MODE_HEX, 8); // 8 uint64
202 2 : g_traceStructItemFieldSet(g_traceKFCtaskAndTilingTraceDataSt, "tilingData", TRACE_STRUCT_FIELD_TYPE_UINT64,
203 : TRACE_STRUCT_SHOW_MODE_HEX, 8); // 8 uint64
204 2 : SetHcclKFCTilingDataOne();
205 2 : SetHcclKFCTilingDataTwo();
206 2 : g_traceStructSetAttr(g_traceKFCtaskAndTilingTraceDataSt, 0, &attr);
207 2 : g_traceTaskAndTilingDataHandle = g_traceCreateWithAttr(TRACER_TYPE_SCHEDULE, "KFCtaskAndTilingTraceData", &attr);
208 2 : if (g_traceTaskAndTilingDataHandle < 0) {
209 1 : HCCL_ERROR("Create g_traceTaskAndTilingDataHandle failed, ret:%d", g_traceTaskAndTilingDataHandle);
210 1 : g_traceStructEntryDestroy(g_traceKFCtaskAndTilingTraceDataSt);
211 1 : return HCCL_E_INTERNAL;
212 : }
213 1 : TraStatus ret = g_traceEventBindTrace(g_eventHandle, g_traceTaskAndTilingDataHandle);
214 1 : if (ret != TRACE_SUCCESS) {
215 0 : HCCL_ERROR("Bind g_traceTaskAndTilingDataHandle failed, ret:%d", ret);
216 0 : return HCCL_E_INTERNAL;
217 : }
218 1 : return HCCL_SUCCESS;
219 : }
220 :
221 2 : HcclResult MC2TraceUtils::InitAicpuComDataHandle()
222 : {
223 2 : uint16_t aicpuComMsgNum = GetMsgNum(sizeof(AicpuComTraceData));
224 2 : TraceAttr attr = {0};
225 2 : attr = {false, aicpuComMsgNum, sizeof(AicpuComTraceData), nullptr};
226 2 : g_traceAicpuComTraceSt = g_traceStructEntryCreate("AicpuComTraceData");
227 2 : g_traceStructItemFieldSet(g_traceAicpuComTraceSt, "devId", TRACE_STRUCT_FIELD_TYPE_UINT32,
228 : TRACE_STRUCT_SHOW_MODE_DEC, 4); // 4 uint32
229 2 : g_traceStructItemFieldSet(g_traceAicpuComTraceSt, "ssid", TRACE_STRUCT_FIELD_TYPE_UINT32,
230 : TRACE_STRUCT_SHOW_MODE_DEC, 4); // 4 uint32
231 2 : g_traceStructItemFieldSet(g_traceAicpuComTraceSt, "rankId", TRACE_STRUCT_FIELD_TYPE_UINT32,
232 : TRACE_STRUCT_SHOW_MODE_DEC, 4); // 4 uint32
233 2 : g_traceStructItemFieldSet(g_traceAicpuComTraceSt, "rankNum", TRACE_STRUCT_FIELD_TYPE_UINT32,
234 : TRACE_STRUCT_SHOW_MODE_DEC, 4); // 4 uint32
235 2 : g_traceStructItemFieldSet(g_traceAicpuComTraceSt, "windowSize", TRACE_STRUCT_FIELD_TYPE_UINT64,
236 : TRACE_STRUCT_SHOW_MODE_DEC, 8); // 8 uint64
237 2 : g_traceStructItemFieldSet(g_traceAicpuComTraceSt, "workSpaceAddr", TRACE_STRUCT_FIELD_TYPE_UINT64,
238 : TRACE_STRUCT_SHOW_MODE_HEX, 8); // 8 uint64
239 2 : g_traceStructItemFieldSet(g_traceAicpuComTraceSt, "kfcNotifyId", TRACE_STRUCT_FIELD_TYPE_UINT64,
240 : TRACE_STRUCT_SHOW_MODE_DEC, 8); // 8 uint64
241 2 : g_traceStructItemArraySet(g_traceAicpuComTraceSt, "eventIds", TRACE_STRUCT_ARRAY_TYPE_UINT32,
242 : TRACE_STRUCT_SHOW_MODE_DEC, 128); // 4 uint32 128 AC_MAX_RANK_NUM*uint32
243 2 : g_traceStructItemArraySet(g_traceAicpuComTraceSt, "windowIn", TRACE_STRUCT_ARRAY_TYPE_UINT64,
244 : TRACE_STRUCT_SHOW_MODE_HEX, 256); // 8 uint64 256 AC_MAX_RANK_NUM*uint64
245 2 : g_traceStructItemArraySet(g_traceAicpuComTraceSt, "windowOut", TRACE_STRUCT_ARRAY_TYPE_UINT64,
246 : TRACE_STRUCT_SHOW_MODE_HEX, 256); // 8 uint64 256 AC_MAX_RANK_NUM*uint64
247 2 : g_traceStructItemArraySet(g_traceAicpuComTraceSt, "actualStreamId", TRACE_STRUCT_ARRAY_TYPE_INT32,
248 : TRACE_STRUCT_SHOW_MODE_DEC, 128); // 4 uint32 128 AC_MAX_RANK_NUM*uint32
249 2 : g_traceStructItemArraySet(g_traceAicpuComTraceSt, "sqId", TRACE_STRUCT_ARRAY_TYPE_INT32,
250 : TRACE_STRUCT_SHOW_MODE_DEC, 128); // 4 uint32 128 AC_MAX_RANK_NUM*uint32
251 2 : g_traceStructItemArraySet(g_traceAicpuComTraceSt, "aicpuOpNotifyAddress", TRACE_STRUCT_ARRAY_TYPE_UINT64,
252 : TRACE_STRUCT_SHOW_MODE_HEX, 16); // 8 uint64 16 2*uint64
253 2 : g_traceStructItemArraySet(g_traceAicpuComTraceSt, "aicpuOpNotifyActualNotifyId", TRACE_STRUCT_ARRAY_TYPE_INT32,
254 : TRACE_STRUCT_SHOW_MODE_DEC, 8); // 4 uint32 8 2*uint32
255 2 : g_traceStructItemFieldSet(g_traceAicpuComTraceSt, "clusterId", TRACE_STRUCT_FIELD_TYPE_INT32,
256 : TRACE_STRUCT_SHOW_MODE_DEC, 4); // 4 uint32
257 2 : g_traceStructSetAttr(g_traceAicpuComTraceSt, 0, &attr);
258 2 : g_traceAicpuComDataHandle = g_traceCreateWithAttr(TRACER_TYPE_SCHEDULE, "AicpuComTraceData", &attr);
259 2 : if (g_traceAicpuComDataHandle < 0) {
260 1 : HCCL_ERROR("Create g_traceAicpuComDataHandle failed, ret:%d", g_traceAicpuComDataHandle);
261 1 : g_traceStructEntryDestroy(g_traceAicpuComTraceSt);
262 1 : return HCCL_E_INTERNAL;
263 : }
264 1 : TraStatus ret = g_traceEventBindTrace(g_eventHandle, g_traceAicpuComDataHandle);
265 1 : if (ret != TRACE_SUCCESS) {
266 0 : HCCL_ERROR("Bind g_traceAicpuComDataHandle failed, ret:%d", ret);
267 0 : return HCCL_E_INTERNAL;
268 : }
269 1 : return HCCL_SUCCESS;
270 : }
271 :
272 2 : void MC2TraceUtils::SetTraceMsgInfo()
273 : {
274 2 : g_traceStructItemFieldSet(g_traceMsgInfoSt, "commType", TRACE_STRUCT_FIELD_TYPE_UINT32,
275 : TRACE_STRUCT_SHOW_MODE_DEC, 4); // 4 uint32
276 2 : g_traceStructItemFieldSet(g_traceMsgInfoSt, "opType", TRACE_STRUCT_FIELD_TYPE_UINT32,
277 : TRACE_STRUCT_SHOW_MODE_DEC, 4); // 4 uint32
278 2 : g_traceStructItemFieldSet(g_traceMsgInfoSt, "sendBuffer", TRACE_STRUCT_FIELD_TYPE_UINT64,
279 : TRACE_STRUCT_SHOW_MODE_HEX, 8); // 8 uint64
280 2 : g_traceStructItemFieldSet(g_traceMsgInfoSt, "recvBuffer", TRACE_STRUCT_FIELD_TYPE_UINT64,
281 : TRACE_STRUCT_SHOW_MODE_HEX, 8); // 8 uint64
282 2 : g_traceStructItemFieldSet(g_traceMsgInfoSt, "count", TRACE_STRUCT_FIELD_TYPE_UINT64,
283 : TRACE_STRUCT_SHOW_MODE_DEC, 8); // 8 uint64
284 2 : g_traceStructItemFieldSet(g_traceMsgInfoSt, "strideLen", TRACE_STRUCT_FIELD_TYPE_UINT64,
285 : TRACE_STRUCT_SHOW_MODE_DEC, 8); // 8 uint64
286 2 : g_traceStructItemFieldSet(g_traceMsgInfoSt, "hcclDataType", TRACE_STRUCT_FIELD_TYPE_UINT32,
287 : TRACE_STRUCT_SHOW_MODE_DEC, 4); // 4 uint32
288 2 : g_traceStructItemFieldSet(g_traceMsgInfoSt, "valid", TRACE_STRUCT_FIELD_TYPE_UINT32,
289 : TRACE_STRUCT_SHOW_MODE_DEC, 4); // 4 uint32
290 2 : g_traceStructItemFieldSet(g_traceMsgInfoSt, "isLast", TRACE_STRUCT_FIELD_TYPE_UINT8,
291 : TRACE_STRUCT_SHOW_MODE_DEC, 1); // 1 uint8
292 2 : g_traceStructItemFieldSet(g_traceMsgInfoSt, "funID", TRACE_STRUCT_FIELD_TYPE_UINT8,
293 : TRACE_STRUCT_SHOW_MODE_DEC, 1); // 1 uint8
294 2 : g_traceStructItemFieldSet(g_traceMsgInfoSt, "sendCnt", TRACE_STRUCT_FIELD_TYPE_UINT8,
295 : TRACE_STRUCT_SHOW_MODE_DEC, 1); // 1 uint8
296 2 : g_traceStructItemFieldSet(g_traceMsgInfoSt, "rcvCnt", TRACE_STRUCT_FIELD_TYPE_UINT8,
297 : TRACE_STRUCT_SHOW_MODE_DEC, 1); // 1 uint8
298 2 : g_traceStructItemFieldSet(g_traceMsgInfoSt, "everyTurnRsp", TRACE_STRUCT_FIELD_TYPE_UINT8,
299 : TRACE_STRUCT_SHOW_MODE_DEC, 1); // 1 uint8
300 2 : g_traceStructItemFieldSet(g_traceMsgInfoSt, "everyTurnWait", TRACE_STRUCT_FIELD_TYPE_UINT8,
301 : TRACE_STRUCT_SHOW_MODE_DEC, 1); // 1 uint8
302 2 : g_traceStructItemFieldSet(g_traceMsgInfoSt, "totalTurnCnt", TRACE_STRUCT_FIELD_TYPE_UINT8,
303 : TRACE_STRUCT_SHOW_MODE_DEC, 1); // 1 uint8
304 2 : g_traceStructItemFieldSet(g_traceMsgInfoSt, "useBufferType", TRACE_STRUCT_FIELD_TYPE_UINT8,
305 : TRACE_STRUCT_SHOW_MODE_DEC, 1); // 1 uint8
306 2 : g_traceStructItemFieldSet(g_traceMsgInfoSt, "winOffset", TRACE_STRUCT_ARRAY_TYPE_UINT64,
307 : TRACE_STRUCT_SHOW_MODE_DEC, 8); // 8 uint64
308 2 : g_traceStructItemArraySet(g_traceMsgInfoSt, "res", TRACE_STRUCT_ARRAY_TYPE_UINT8,
309 : TRACE_STRUCT_SHOW_MODE_DEC, 2); // 1 uint8 2 2*uint8
310 2 : }
311 :
312 2 : HcclResult MC2TraceUtils::InitMsgInfoHandle()
313 : {
314 2 : uint16_t msgInfoNum = GetMsgNum(sizeof(AivAicpuOpParam));
315 2 : TraceAttr attr = {0};
316 2 : attr = {false, msgInfoNum, sizeof(AivAicpuOpParam), nullptr};
317 2 : g_traceMsgInfoSt = g_traceStructEntryCreate("AivAicpuOpParam");
318 2 : MC2TraceUtils::SetTraceMsgInfo();
319 2 : g_traceStructSetAttr(g_traceMsgInfoSt, 0, &attr);
320 2 : g_traceMsgInfoHandle = g_traceCreateWithAttr(TRACER_TYPE_SCHEDULE, "AivAicpuOpParam", &attr);
321 2 : if (g_traceMsgInfoHandle < 0) {
322 1 : HCCL_ERROR("Create g_traceMsgInfoHandle failed, ret:%d", g_traceMsgInfoHandle);
323 1 : g_traceStructEntryDestroy(g_traceMsgInfoSt);
324 1 : return HCCL_E_INTERNAL;
325 : }
326 1 : TraStatus ret = g_traceEventBindTrace(g_eventHandle, g_traceMsgInfoHandle);
327 1 : if (ret != TRACE_SUCCESS) {
328 0 : HCCL_ERROR("Bind g_traceMsgInfoHandle failed, ret:%d", ret);
329 0 : return HCCL_E_INTERNAL;
330 : }
331 1 : return HCCL_SUCCESS;
332 : }
333 :
334 32 : void MC2TraceUtils::SetTraceSqeBatchInfo()
335 : {
336 32 : g_traceStructItemFieldSet(g_traceSqeBatchInfoSt, "addr1High", TRACE_STRUCT_FIELD_TYPE_UINT32,
337 : TRACE_STRUCT_SHOW_MODE_HEX, 4); // 4 uint32
338 32 : g_traceStructItemFieldSet(g_traceSqeBatchInfoSt, "addr1Low", TRACE_STRUCT_FIELD_TYPE_UINT32,
339 : TRACE_STRUCT_SHOW_MODE_HEX, 4); // 4 uint32
340 32 : g_traceStructItemFieldSet(g_traceSqeBatchInfoSt, "addr2High", TRACE_STRUCT_FIELD_TYPE_UINT32,
341 : TRACE_STRUCT_SHOW_MODE_HEX, 4); // 4 uint32
342 32 : g_traceStructItemFieldSet(g_traceSqeBatchInfoSt, "addr2Low", TRACE_STRUCT_FIELD_TYPE_UINT32,
343 : TRACE_STRUCT_SHOW_MODE_HEX, 4); // 4 uint32
344 32 : g_traceStructItemFieldSet(g_traceSqeBatchInfoSt, "sqeHeadIdx", TRACE_STRUCT_FIELD_TYPE_UINT32,
345 : TRACE_STRUCT_SHOW_MODE_DEC, 4); // 4 uint32
346 32 : g_traceStructItemFieldSet(g_traceSqeBatchInfoSt, "notifyId", TRACE_STRUCT_FIELD_TYPE_UINT32,
347 : TRACE_STRUCT_SHOW_MODE_DEC, 4); // 4 uint32
348 32 : g_traceStructItemFieldSet(g_traceSqeBatchInfoSt, "length", TRACE_STRUCT_FIELD_TYPE_UINT32,
349 : TRACE_STRUCT_SHOW_MODE_DEC, 4); // 4 uint32
350 32 : g_traceStructItemFieldSet(g_traceSqeBatchInfoSt, "partId", TRACE_STRUCT_FIELD_TYPE_UINT32,
351 : TRACE_STRUCT_SHOW_MODE_DEC, 4); // 4 uint32
352 32 : g_traceStructItemFieldSet(g_traceSqeBatchInfoSt, "remoteRank", TRACE_STRUCT_FIELD_TYPE_UINT32,
353 : TRACE_STRUCT_SHOW_MODE_DEC, 4); // 4 uint32
354 32 : g_traceStructItemFieldSet(g_traceSqeBatchInfoSt, "dataType", TRACE_STRUCT_FIELD_TYPE_UINT32,
355 : TRACE_STRUCT_SHOW_MODE_DEC, 4); // 4 uint32
356 32 : g_traceStructItemFieldSet(g_traceSqeBatchInfoSt, "streamId", TRACE_STRUCT_FIELD_TYPE_UINT16,
357 : TRACE_STRUCT_SHOW_MODE_DEC, 2); // 2 uint16
358 32 : g_traceStructItemFieldSet(g_traceSqeBatchInfoSt, "eventId", TRACE_STRUCT_FIELD_TYPE_UINT16,
359 : TRACE_STRUCT_SHOW_MODE_DEC, 2); // 2 uint16
360 32 : g_traceStructItemFieldSet(g_traceSqeBatchInfoSt, "taskId", TRACE_STRUCT_FIELD_TYPE_UINT16,
361 : TRACE_STRUCT_SHOW_MODE_DEC, 2); // 2 uint16
362 32 : g_traceStructItemFieldSet(g_traceSqeBatchInfoSt, "condValue", TRACE_STRUCT_FIELD_TYPE_UINT16,
363 : TRACE_STRUCT_SHOW_MODE_DEC, 2); // 2 uint16
364 32 : g_traceStructItemFieldSet(g_traceSqeBatchInfoSt, "isLast", TRACE_STRUCT_FIELD_TYPE_UINT8,
365 : TRACE_STRUCT_SHOW_MODE_DEC, 1); // 1 uint8
366 32 : g_traceStructItemFieldSet(g_traceSqeBatchInfoSt, "opCode", TRACE_STRUCT_FIELD_TYPE_UINT8,
367 : TRACE_STRUCT_SHOW_MODE_DEC, 1); // 1 uint8
368 32 : g_traceStructItemFieldSet(g_traceSqeBatchInfoSt, "sqeNum", TRACE_STRUCT_FIELD_TYPE_UINT8,
369 : TRACE_STRUCT_SHOW_MODE_DEC, 1); // 1 uint8
370 32 : g_traceStructItemFieldSet(g_traceSqeBatchInfoSt, "type", TRACE_STRUCT_FIELD_TYPE_UINT8,
371 : TRACE_STRUCT_SHOW_MODE_DEC, 1); // 1 uint8
372 32 : g_traceStructItemFieldSet(g_traceSqeBatchInfoSt, "subType", TRACE_STRUCT_FIELD_TYPE_UINT8,
373 : TRACE_STRUCT_SHOW_MODE_DEC, 1); // 1 uint8
374 32 : g_traceStructItemFieldSet(g_traceSqeBatchInfoSt, "valid", TRACE_STRUCT_FIELD_TYPE_UINT8,
375 : TRACE_STRUCT_SHOW_MODE_DEC, 1); // 1 uint8
376 32 : g_traceStructItemArraySet(g_traceSqeBatchInfoSt, "reverse", TRACE_STRUCT_ARRAY_TYPE_UINT8,
377 : TRACE_STRUCT_SHOW_MODE_DEC, 10); // 1 uint8 10 10*uint8
378 32 : }
379 :
380 2 : HcclResult MC2TraceUtils::InitSqeBatchInfoHandle()
381 : {
382 2 : uint16_t sqeBatchInfoMsgNum = GetMsgNum(sizeof(SqeBatchInfo));
383 2 : TraceAttr attr = {0};
384 2 : attr = {false, sqeBatchInfoMsgNum, sizeof(SqeBatchInfo), nullptr};
385 2 : g_traceSqeBatchInfoSt = g_traceStructEntryCreate("SqeBatchInfo");
386 34 : for (uint32_t i = 0; i < MAX_SQE_BATCH_SIZE; i++) {
387 32 : SetTraceSqeBatchInfo();
388 : }
389 2 : g_traceStructSetAttr(g_traceSqeBatchInfoSt, 0, &attr);
390 2 : g_traceSqeBatchInfoHandle = g_traceCreateWithAttr(TRACER_TYPE_SCHEDULE, "SqeBatchInfo", &attr);
391 2 : if (g_traceSqeBatchInfoHandle < 0) {
392 1 : HCCL_ERROR("Create g_traceSqeBatchInfoHandle failed, ret:%d", g_traceSqeBatchInfoHandle);
393 1 : g_traceStructEntryDestroy(g_traceSqeBatchInfoSt);
394 1 : return HCCL_E_INTERNAL;
395 : }
396 1 : TraStatus ret = g_traceEventBindTrace(g_eventHandle, g_traceSqeBatchInfoHandle);
397 1 : if (ret != TRACE_SUCCESS) {
398 0 : HCCL_ERROR("Bind g_traceSqeBatchInfoHandle failed, ret:%d", ret);
399 0 : return HCCL_E_INTERNAL;
400 : }
401 1 : return HCCL_SUCCESS;
402 : }
403 :
404 1 : HcclResult MC2TraceUtils::InitTraceHandle()
405 : {
406 1 : g_eventHandle = g_traceEventCreate("mc2_dfx");
407 1 : if (g_eventHandle < 0) {
408 0 : HCCL_ERROR("Create mc2_dfx event handle failed");
409 0 : return HCCL_E_INTERNAL;
410 : }
411 1 : CHK_RET(InitTaskAndTilingDataHandle());
412 1 : CHK_RET(InitAicpuComDataHandle());
413 1 : CHK_RET(InitTraceStrHandle());
414 1 : CHK_RET(InitMsgInfoHandle());
415 1 : CHK_RET(InitSqeBatchInfoHandle());
416 1 : return HCCL_SUCCESS;
417 : }
418 :
419 2 : HcclResult MC2TraceUtils::GetTraceFunc(const std::string &traceName)
420 : {
421 2 : g_traceEventBindTrace = reinterpret_cast<TraceEventBindTraceFunc>(
422 2 : mmDlsym(g_soHandle, std::string(traceName + "EventBindTrace").c_str()));
423 2 : CHK_PRT_RET(g_traceEventBindTrace == nullptr, HCCL_ERROR("Get g_traceEventBindTrace error: %s", mmDlerror()),
424 : HCCL_E_SYSCALL);
425 2 : g_traceEventReport = reinterpret_cast<TraceEventReportFunc>(
426 2 : mmDlsym(g_soHandle, std::string(traceName + "EventReport").c_str()));
427 2 : CHK_PRT_RET(g_traceEventReport == nullptr, HCCL_ERROR("Get g_traceEventReport error: %s", mmDlerror()),
428 : HCCL_E_SYSCALL);
429 2 : g_traceSetGlobalAttr = reinterpret_cast<TraceSetGlobalAttrFunc>(
430 2 : mmDlsym(g_soHandle, std::string(traceName + "SetGlobalAttr").c_str()));
431 2 : CHK_PRT_RET(g_traceSetGlobalAttr == nullptr, HCCL_ERROR("Get g_traceSetGlobalAttr error: %s", mmDlerror()),
432 : HCCL_E_SYSCALL);
433 2 : g_traceStructEntryCreate = reinterpret_cast<TraceStructEntryCreateFunc>(
434 2 : mmDlsym(g_soHandle, std::string(traceName + "StructEntryCreate").c_str()));
435 2 : CHK_PRT_RET(g_traceStructEntryCreate == nullptr,
436 : HCCL_ERROR("Get g_traceStructEntryCreate error: %s", mmDlerror()), HCCL_E_SYSCALL);
437 2 : g_traceStructItemFieldSet = reinterpret_cast<TraceStructItemFieldSetFunc>(
438 2 : mmDlsym(g_soHandle, std::string(traceName + "StructItemFieldSet").c_str()));
439 2 : CHK_PRT_RET(g_traceStructItemFieldSet == nullptr,
440 : HCCL_ERROR("Get g_traceStructItemFieldSet error: %s", mmDlerror()), HCCL_E_SYSCALL);
441 2 : g_traceStructItemArraySet = reinterpret_cast<TraceStructItemArraySetFunc>(
442 2 : mmDlsym(g_soHandle, std::string(traceName + "StructItemArraySet").c_str()));
443 2 : CHK_PRT_RET(g_traceStructItemArraySet == nullptr,
444 : HCCL_ERROR("Get g_traceStructItemArraySet error: %s", mmDlerror()), HCCL_E_SYSCALL);
445 2 : g_traceStructSetAttr = reinterpret_cast<TraceStructSetAttrFunc>(
446 2 : mmDlsym(g_soHandle, std::string(traceName + "StructSetAttr").c_str()));
447 2 : CHK_PRT_RET(g_traceStructSetAttr == nullptr, HCCL_ERROR("Get g_traceStructSetAttr error: %s", mmDlerror()),
448 : HCCL_E_SYSCALL);
449 2 : g_traceStructEntryDestroy = reinterpret_cast<TraceStructEntryDestroyFunc>(
450 2 : mmDlsym(g_soHandle, std::string(traceName + "StructEntryDestroy").c_str()));
451 2 : CHK_PRT_RET(g_traceStructEntryDestroy == nullptr,
452 : HCCL_ERROR("Get g_traceStructEntryDestroy error: %s", mmDlerror()), HCCL_E_SYSCALL);
453 2 : return HCCL_SUCCESS;
454 : }
455 :
456 2 : HcclResult MC2TraceUtils::InitFuncHandle()
457 : {
458 2 : std::string traceName = g_isAtrace ? "Atrace" : "Utrace";
459 2 : HCCL_INFO("Start mmDlsym %s funcHandle", traceName.c_str());
460 2 : g_traceCreateWithAttr = reinterpret_cast<TraceCreateWithAttrFunc>(
461 2 : mmDlsym(g_soHandle, std::string(traceName + "CreateWithAttr").c_str()));
462 2 : CHK_PRT_RET(g_traceCreateWithAttr == nullptr, HCCL_ERROR("Get g_traceCreateWithAttr error: %s", mmDlerror()),
463 : HCCL_E_SYSCALL);
464 2 : g_traceGetHandle = reinterpret_cast<TraceGetHandleFunc>(
465 2 : mmDlsym(g_soHandle, std::string(traceName + "GetHandle").c_str()));
466 2 : CHK_PRT_RET(g_traceGetHandle == nullptr, HCCL_ERROR("Get g_traceGetHandle error: %s", mmDlerror()),
467 : HCCL_E_SYSCALL);
468 2 : g_traceSubmit = reinterpret_cast<TraceSubmitFunc>(
469 2 : mmDlsym(g_soHandle, std::string(traceName + "Submit").c_str()));
470 2 : CHK_PRT_RET(g_traceSubmit == nullptr, HCCL_ERROR("Get g_traceSubmit error: %s", mmDlerror()), HCCL_E_SYSCALL);
471 2 : g_traceEventCreate = reinterpret_cast<TraceEventCreateFunc>(
472 2 : mmDlsym(g_soHandle, std::string(traceName + "EventCreate").c_str()));
473 2 : CHK_PRT_RET(g_traceEventCreate == nullptr, HCCL_ERROR("Get g_traceEventCreate error: %s", mmDlerror()),
474 : HCCL_E_SYSCALL);
475 2 : g_traceEventDestroy = reinterpret_cast<TraceEventDestroyFunc>(
476 2 : mmDlsym(g_soHandle, std::string(traceName + "EventDestroy").c_str()));
477 2 : CHK_PRT_RET(g_traceEventDestroy == nullptr, HCCL_ERROR("Get g_traceEventDestroy error: %s", mmDlerror()),
478 : HCCL_E_SYSCALL);
479 2 : g_traceHandleDestroy = reinterpret_cast<TraceHandleDestroyFunc>(
480 2 : mmDlsym(g_soHandle, std::string(traceName + "Destroy").c_str()));
481 2 : CHK_PRT_RET(g_traceHandleDestroy == nullptr, HCCL_ERROR("Get g_traceHandleDestroy error: %s", mmDlerror()),
482 : HCCL_E_SYSCALL);
483 2 : CHK_RET(GetTraceFunc(traceName));
484 2 : return HCCL_SUCCESS;
485 2 : }
486 :
487 43 : __attribute__ ((constructor)) void DlopenTraceSo()
488 : {
489 43 : if (g_soHandle == nullptr) {
490 43 : g_soHandle = mmDlopen(ATRACE_SO, RTLD_NOW | RTLD_GLOBAL);
491 43 : if (g_soHandle == nullptr) {
492 43 : g_soHandle = mmDlopen(UTRACE_SO, RTLD_NOW | RTLD_GLOBAL);
493 43 : g_isAtrace = false;
494 : }
495 43 : HCCL_INFO("mmDlopen success");
496 : }
497 43 : }
498 :
499 84 : HcclResult MC2TraceUtils::Init()
500 : {
501 84 : if (g_soHandle == nullptr) {
502 84 : HCCL_WARNING("Cannot find so file: %s", UTRACE_SO);
503 84 : return HCCL_SUCCESS;
504 : }
505 0 : CHK_RET(InitFuncHandle());
506 0 : CHK_RET(InitTraceHandle());
507 0 : unsigned int hostpid = 0;
508 0 : unsigned int cpType = DEVDRV_PROCESS_CPTYPE_MAX;
509 0 : CHK_RET(HrtHalDrvQueryProcessHostPid(getpid(), nullptr, nullptr, &hostpid, &cpType));
510 :
511 0 : TraceGlobalAttr traceGlobalAttr = { 0 };
512 0 : traceGlobalAttr.saveMode = 1;
513 0 : traceGlobalAttr.deviceId = AicpuGetComContext()->devId;
514 0 : traceGlobalAttr.pid = hostpid;
515 0 : TraStatus ret = g_traceSetGlobalAttr(&traceGlobalAttr);
516 0 : if (ret != TRACE_SUCCESS) {
517 0 : HCCL_ERROR("TraceSetGlobalAttrFunc failed, ret:%d", ret);
518 0 : return HCCL_E_INTERNAL;
519 : }
520 0 : return HCCL_SUCCESS;
521 : }
522 :
523 1 : template <typename T> HcclResult MC2TraceUtils::Submit(const T * const traceData)
524 : {
525 1 : if (g_soHandle == nullptr || g_traceGetHandle == nullptr || g_traceSubmit == nullptr) {
526 1 : return HCCL_SUCCESS;
527 : }
528 0 : CHK_PTR_NULL(traceData);
529 0 : std::string typeName;
530 : if (std::is_same<T, KFCtaskAndTilingTraceData>::value) {
531 0 : typeName = "KFCtaskAndTilingTraceData";
532 : } else if (std::is_same<T, AicpuComTraceData>::value) {
533 0 : typeName = "AicpuComTraceData";
534 : } else if (std::is_same<T, AivAicpuOpParam>::value) {
535 : typeName = "AivAicpuOpParam";
536 : } else if (std::is_same<T, SqeBatchInfo>::value) {
537 0 : typeName = "SqeBatchInfo";
538 : } else {
539 : HCCL_ERROR("find typename: %s failed", typeid(T).name());
540 : return HCCL_E_INTERNAL;
541 : }
542 :
543 0 : auto traceHandle = g_traceGetHandle(TRACER_TYPE_SCHEDULE, typeName.c_str());
544 0 : if (traceHandle < 0) {
545 0 : HCCL_ERROR("getHandle %s failed, ret:%d", typeName.c_str(), traceHandle);
546 0 : return HCCL_E_INTERNAL;
547 : }
548 0 : auto traceRet = g_traceSubmit(traceHandle, reinterpret_cast<const void *>(traceData), sizeof(T));
549 0 : if (traceRet != TRACE_SUCCESS) {
550 0 : HCCL_ERROR("submit %s failed, ret:%d", typeName.c_str(), traceRet);
551 0 : return HCCL_E_INTERNAL;
552 : }
553 0 : return HCCL_SUCCESS;
554 0 : }
555 :
556 64 : HcclResult MC2TraceUtils::Submit(AicpuComContext *ctx)
557 : {
558 64 : if (g_soHandle == nullptr) {
559 64 : return HCCL_SUCCESS;
560 : }
561 0 : CHK_PTR_NULL(ctx);
562 0 : uint64_t startUsec = GetCurCpuTimestamp();
563 : AicpuComTraceData comTraceData;
564 0 : comTraceData.devId = ctx->devId;
565 0 : comTraceData.ssid = ctx->ssid;
566 0 : comTraceData.rankId = ctx->rankId;
567 0 : comTraceData.rankNum = ctx->rankNum;
568 0 : comTraceData.windowSize = ctx->windowSize;
569 0 : comTraceData.workSpaceAddr = ctx->workSpaceAddr;
570 0 : comTraceData.kfcNotifyId = ctx->kfcNotifyId;
571 0 : for (uint32_t i = 0; i < AC_MAX_RANK_NUM; i++) {
572 0 : comTraceData.eventIds[i] = ctx->eventIds[i];
573 0 : comTraceData.windowIn[i] = ctx->rankInfo[i].window;
574 0 : comTraceData.windowOut[i] = ctx->rankInfo[i].windowOut;
575 0 : comTraceData.actualStreamId[i] = ctx->streamInfo[i].actualStreamId;
576 0 : comTraceData.sqId[i] = ctx->streamInfo[i].sqId;
577 : }
578 0 : for (uint32_t i = 0; i < 2; i++) { // 2 aicpuOpNotify size
579 0 : comTraceData.aicpuOpNotifyAddress[i] = ctx->aicpuOpNotify[i].address;
580 0 : comTraceData.aicpuOpNotifyActualNotifyId[i] = ctx->aicpuOpNotify[i].actualNotifyId;
581 : }
582 0 : comTraceData.clusterId = ctx->clusterId;
583 0 : CHK_RET(Submit<AicpuComTraceData>(&comTraceData));
584 0 : if (AicpuKfcUtils::IsDebugModeEquals(*ctx, MC2_DEBUG_TIME_TAKEN)) {
585 0 : AicpuKfcProf::GetProInst(*ctx).traceCtxTime += GetCurCpuTimestamp() - startUsec;
586 : }
587 0 : return HCCL_SUCCESS;
588 : }
589 :
590 61 : HcclResult MC2TraceUtils::Submit(const KFCTask * const task, const HcclKFCTilingData * const tilingData)
591 : {
592 61 : if (g_soHandle == nullptr) {
593 61 : return HCCL_SUCCESS;
594 : }
595 0 : CHK_PTR_NULL(task);
596 0 : CHK_PTR_NULL(tilingData);
597 0 : uint64_t startUsec = GetCurCpuTimestamp();
598 : KFCtaskAndTilingTraceData taskAndTilingData;
599 0 : error_t ret = memcpy_s(&taskAndTilingData, sizeof(KFCtaskAndTilingTraceData), task, sizeof(KFCTask));
600 0 : if (ret != EOK) {
601 0 : HCCL_ERROR("task memcpy_s fail ret:%d", ret);
602 0 : return HCCL_E_MEMORY;
603 : }
604 0 : ret = memcpy_s(reinterpret_cast<uint8_t *>(&taskAndTilingData) + sizeof(KFCTask),
605 : sizeof(KFCtaskAndTilingTraceData) - sizeof(KFCTask), tilingData, sizeof(HcclKFCTilingData));
606 0 : if (ret != EOK) {
607 0 : HCCL_ERROR("tilingData memcpy_s fail ret:%d", ret);
608 0 : return HCCL_E_MEMORY;
609 : }
610 0 : CHK_RET(Submit<KFCtaskAndTilingTraceData>(&taskAndTilingData));
611 0 : auto ctx = AicpuGetComContext();
612 0 : if (AicpuKfcUtils::IsDebugModeEquals(*ctx, MC2_DEBUG_TIME_TAKEN)) {
613 0 : AicpuKfcProf::GetProInst(*ctx).traceSubmitTime += GetCurCpuTimestamp() - startUsec;
614 : }
615 0 : return HCCL_SUCCESS;
616 : }
617 :
618 1 : HcclResult MC2TraceUtils::Submit(const std::string &traceStr)
619 : {
620 1 : if (g_soHandle == nullptr) {
621 1 : return HCCL_SUCCESS;
622 : }
623 0 : return Submit(traceStr.c_str());
624 : }
625 :
626 1 : HcclResult MC2TraceUtils::Submit(const char *traceStr)
627 : {
628 1 : if (g_soHandle == nullptr || g_traceSubmit == nullptr) {
629 1 : return HCCL_SUCCESS;
630 : }
631 0 : uint64_t startUsec = GetCurCpuTimestamp();
632 0 : uint32_t strSize = strlen(traceStr);
633 0 : uint32_t posSize = 0U;
634 0 : TraceStr singleTraceStr;
635 0 : while (posSize < strSize) {
636 0 : uint32_t traceStrLen = std::min<uint32_t>(strSize - posSize, sizeof(singleTraceStr.transmit) - 1);
637 0 : CHK_SAFETY_FUNC_RET(
638 : memset_s(reinterpret_cast<void *>(&singleTraceStr.transmit), sizeof(singleTraceStr.transmit), 0,
639 : sizeof(singleTraceStr.transmit)));
640 0 : error_t ret = memcpy_s(singleTraceStr.transmit, traceStrLen, traceStr + posSize, traceStrLen);
641 0 : if (ret != EOK) {
642 0 : HCCL_ERROR("traceStr memcpy_s fail ret:%d", ret);
643 0 : return HCCL_E_MEMORY;
644 : }
645 0 : posSize += traceStrLen;
646 : auto traceStrRet =
647 0 : g_traceSubmit(g_traceStrHandle, reinterpret_cast<const void *>(&singleTraceStr), sizeof(singleTraceStr));
648 0 : if (traceStrRet != TRACE_SUCCESS) {
649 0 : HCCL_ERROR("submittraceStr failed, ret:%d", traceStrRet);
650 0 : return HCCL_E_INTERNAL;
651 : }
652 0 : HCCL_INFO("Submit string: %s", singleTraceStr.transmit);
653 : }
654 0 : auto ctx = AicpuGetComContext();
655 0 : if (AicpuKfcUtils::IsDebugModeEquals(*ctx, MC2_DEBUG_TIME_TAKEN)) {
656 0 : AicpuKfcProf::GetProInst(*ctx).traceSubmitTime += GetCurCpuTimestamp() - startUsec;
657 : }
658 0 : return HCCL_SUCCESS;
659 : }
660 :
661 0 : HcclResult MC2TraceUtils::SubmitBatchSqeInfo()
662 : {
663 0 : if (g_soHandle == nullptr) {
664 0 : return HCCL_SUCCESS;
665 : }
666 0 : uint64_t startUsec = GetCurCpuTimestamp();
667 0 : SqeContext *context = GetSqeContext();
668 0 : CHK_PTR_NULL(context->buffPtr);
669 0 : for (uint32_t streamId = 0U; streamId < AC_MAX_RANK_NUM; streamId++) {
670 0 : auto &buff = context->buffPtr[streamId];
671 0 : HCCL_INFO("sqTail: %u, sqHead: %u, sqeCnt: %u, tailSqeTaskId: %u, tailSqeIdx: %u",
672 : buff.sqTail, buff.sqHead, buff.sqeCnt, buff.tailSqeTaskId, buff.tailSqeIdx);
673 0 : auto submitNum = std::min<uint32_t>(buff.tailSqeIdx - buff.sqeCnt, MAX_SQE_SUBMIT_NUM);
674 0 : for (uint32_t i = 0U; i <= submitNum / MAX_SQE_BATCH_SIZE; i++) {
675 0 : SqeBatchInfo sqeBatchInfo;
676 0 : for (uint32_t j = 0U; j < MAX_SQE_BATCH_SIZE; j++) {
677 0 : const auto idx = i * MAX_SQE_BATCH_SIZE + j;
678 0 : if (idx >= submitNum) {
679 0 : break;
680 : }
681 0 : if (AicpuSqeContext::QuerySqeInfoByTaskId(streamId, buff.tailSqeTaskId - submitNum + idx,
682 0 : &sqeBatchInfo.sqeInfos[j]) != HCCL_SUCCESS) {
683 0 : HCCL_WARNING("Query sqe info failed, stream:%u, id:%u", streamId, idx);
684 0 : break;
685 : }
686 : }
687 0 : CHK_RET(Submit<SqeBatchInfo>(&sqeBatchInfo));
688 : }
689 : }
690 0 : auto ctx = AicpuGetComContext();
691 0 : if (AicpuKfcUtils::IsDebugModeEquals(*ctx, MC2_DEBUG_TIME_TAKEN)) {
692 0 : AicpuKfcProf::GetProInst(*ctx).traceSqeTime += GetCurCpuTimestamp() - startUsec;
693 : }
694 0 : return HCCL_SUCCESS;
695 : }
696 :
697 10 : HcclResult MC2TraceUtils::Save()
698 : {
699 10 : if (g_soHandle == nullptr) {
700 10 : return HCCL_SUCCESS;
701 : }
702 0 : TraStatus ret = g_traceEventReport(g_eventHandle);
703 0 : if (ret != TRACE_SUCCESS) {
704 0 : HCCL_ERROR("TraceEventReport failed, ret:%d", ret);
705 0 : return HCCL_E_INTERNAL;
706 : }
707 0 : return HCCL_SUCCESS;
708 : }
709 :
710 2 : HcclResult MC2TraceUtils::DestoryHandles()
711 : {
712 2 : if (g_soHandle == nullptr) {
713 2 : return HCCL_SUCCESS;
714 : }
715 0 : g_traceStructEntryDestroy(g_traceStrSt);
716 0 : g_traceStructEntryDestroy(g_traceAicpuComTraceSt);
717 0 : g_traceStructEntryDestroy(g_traceKFCtaskAndTilingTraceDataSt);
718 0 : g_traceStructEntryDestroy(g_traceMsgInfoSt);
719 0 : g_traceStructEntryDestroy(g_traceSqeBatchInfoSt);
720 0 : g_traceHandleDestroy(g_traceStrHandle);
721 0 : g_traceHandleDestroy(g_traceTaskAndTilingDataHandle);
722 0 : g_traceHandleDestroy(g_traceAicpuComDataHandle);
723 0 : g_traceHandleDestroy(g_traceMsgInfoHandle);
724 0 : g_traceHandleDestroy(g_traceSqeBatchInfoHandle);
725 0 : g_traceEventDestroy(g_eventHandle);
726 :
727 0 : return HCCL_SUCCESS;
728 : }
729 :
730 43 : __attribute__ ((destructor)) void Finalize()
731 : {
732 43 : if (g_soHandle != nullptr) {
733 0 : (void)mmDlclose(g_soHandle);
734 : }
735 43 : g_soHandle = nullptr;
736 43 : }
737 :
738 : // 显式实例化
739 : template HcclResult MC2TraceUtils::Submit(const SqeBatchInfo *const);
|
