Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "profiling_manager.h"
12 : #include <string>
13 :
14 : #include "adapter_prof.h"
15 : #include "adapter_rts_common.h"
16 : #include "workflow_pub.h"
17 : #include "sal_pub.h"
18 :
19 : namespace hccl {
20 : std::queue<MsprofApi> ProfilingManager::storageTaskApi_;
21 : std::queue<MsprofApi> ProfilingManager::storageOpApi_;
22 : std::array<std::queue<MsprofAdditionalInfo>, MAX_MODULE_DEVICE_NUM> ProfilingManager::storageAdditionInfo_;
23 : std::array<std::mutex, MAX_MODULE_DEVICE_NUM> ProfilingManager::reportAddInfoMutex_;
24 : std::array<std::queue<MsprofCompactInfo>, MAX_MODULE_DEVICE_NUM> ProfilingManager::storageCompactInfo_;
25 : std::array<std::mutex, MAX_MODULE_DEVICE_NUM> ProfilingManager::reportCompactInfoMutex_;
26 : std::array<std::queue<MsprofAdditionalInfo>, MAX_MODULE_DEVICE_NUM> ProfilingManager::storageAdditionInfoFftsCapture_;
27 : std::array<std::mutex, MAX_MODULE_DEVICE_NUM> ProfilingManager::reportAddInfoFftsCaptureMutex_;
28 : std::mutex ProfilingManager::reportDataQueueMutex_;
29 : std::unordered_map<s32, bool> ProfilingManager::captureStatusThreadIDMap_;
30 : std::mutex ProfilingManager::captureStatusMapMutex_;
31 : std::mutex ProfilingManager::reportDataOpQueueMutex_;
32 :
33 7 : ProfilingManager::ProfilingManager()
34 7 : : reporterCallback_(nullptr), isHostApiSubscribe_(false),
35 7 : isTaskApiSubscribe_(false), isAdditionInfoSubscribe_(false)
36 7 : {}
37 :
38 7 : ProfilingManager::~ProfilingManager()
39 7 : {}
40 :
41 1623 : ProfilingManager &ProfilingManager::Instance()
42 : {
43 1623 : static ProfilingManager profilingManager;
44 1623 : return profilingManager;
45 : }
46 :
47 0 : Prof_Status ProfilingManager::CallMsprofReport(ReporterData &reporterData) const
48 : {
49 0 : CHK_PRT_RET((reporterCallback_ == nullptr),
50 : HCCL_ERROR("[ProfilingManager][CallMsprofReport] MsprofReporterCallback callback is nullptr."),
51 : FAILED);
52 0 : return reporterCallback_(static_cast<uint32_t>(MsprofReporterModuleId::MSPROF_MODULE_HCCL),
53 : static_cast<uint32_t>(MsprofReporterCallbackType::MSPROF_REPORTER_REPORT),
54 : static_cast<void *>(&reporterData),
55 0 : sizeof(ReporterData));
56 : }
57 :
58 2 : HcclResult ProfilingManager::CallMsprofRegFftsLaunch() const
59 : {
60 2 : if (!isFftsLaunchSubscribe_) {
61 0 : return HCCL_SUCCESS;
62 : }
63 :
64 2 : CHK_RET(hrtMsprofRegTypeInfo(MSPROF_REPORT_HCCL_NODE_LEVEL,
65 : MSPROF_REPORT_NODE_CONTEXT_ID_INFO_TYPE, "context_id_info"));
66 :
67 2 : CHK_RET(hrtMsprofRegTypeInfo(MSPROF_REPORT_NODE_LEVEL,
68 : MSPROF_REPORT_NODE_MC2_COMMINFO_TYPE, "mc2_comm_info"));
69 :
70 2 : return HCCL_SUCCESS;
71 : }
72 :
73 3 : HcclResult ProfilingManager::CallMsprofRegHcclOpApi() const
74 : {
75 5 : if (!isHostHcclOpSubscribe_ || GetExternalInputHcclAicpuUnfold() ||
76 2 : GetExternalInputHcclEnableFfts()) {
77 3 : return HCCL_SUCCESS;
78 : }
79 :
80 0 : for (const auto &name_to_type : PROF_OP_NAME) {
81 0 : CHK_RET(hrtMsprofRegTypeInfo(MSPROF_REPORT_HCCL_NODE_LEVEL,
82 : static_cast<uint32_t>(name_to_type.first) + MSPROF_REPORT_ACL_HOST_HCCL_BASE_TYPE,
83 : name_to_type.second.c_str()));
84 : }
85 :
86 0 : CHK_RET(hrtMsprofRegTypeInfo(MSPROF_REPORT_NODE_LEVEL,
87 : MSPROF_REPORT_NODE_MC2_COMMINFO_TYPE, "mc2_comm_info"));
88 :
89 0 : return HCCL_SUCCESS;
90 : }
91 :
92 0 : HcclResult ProfilingManager::CallMsprofRegHostApi() const
93 : {
94 0 : if (!isHostApiSubscribe_) {
95 0 : return HCCL_SUCCESS;
96 : }
97 0 : for (const auto &name_to_type : PROF_OP_NAME) {
98 0 : CHK_RET(hrtMsprofRegTypeInfo(MSPROF_REPORT_ACL_LEVEL,
99 : static_cast<uint32_t>(name_to_type.first) + MSPROF_REPORT_ACL_HOST_HCCL_BASE_TYPE,
100 : name_to_type.second.c_str()));
101 : }
102 :
103 0 : for (const auto &name_to_type : PROF_OP_NAME) {
104 0 : CHK_RET(hrtMsprofRegTypeInfo(MSPROF_REPORT_NODE_LEVEL,
105 : static_cast<uint32_t>(name_to_type.first) + MSPROF_REPORT_NODE_HCCL_BASE_TYPE,
106 : name_to_type.second.c_str()));
107 : }
108 :
109 0 : const std::string hcclType("hccl_op_info");
110 0 : CHK_RET(hrtMsprofRegTypeInfo(MSPROF_REPORT_NODE_LEVEL, MSPROF_REPORT_NODE_HCCL_OP_INFO_TYPE, hcclType.c_str()));
111 0 : return HCCL_SUCCESS;
112 0 : }
113 :
114 14 : HcclResult ProfilingManager::CallMsprofReportHostNodeApi(
115 : uint64_t beginTime, uint64_t endTime, uint64_t itemId, uint32_t threadId) const
116 : {
117 14 : MsprofApi reporterData{};
118 14 : reporterData.level = MSPROF_REPORT_NODE_LEVEL;
119 14 : reporterData.type = MSPROF_REPORT_NODE_LAUNCH_TYPE;
120 14 : reporterData.threadId = threadId;
121 14 : reporterData.beginTime = beginTime;
122 14 : reporterData.endTime = endTime;
123 14 : reporterData.itemId = itemId;
124 :
125 : // 静态图场景或者acl graph场景, 一次下发，多次执行; 缓存对应数据
126 14 : auto mode = GetWorkflowMode();
127 14 : if ((mode == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OPS_KERNEL_INFO_LIB) || GetThreadCaptureStatus()) {
128 14 : HCCL_INFO("CallMsprofReportTaskApi, storageTaskApi");
129 14 : std::unique_lock<std::mutex> lock(reportDataQueueMutex_);
130 14 : storageTaskApi_.push(reporterData);
131 14 : return HCCL_SUCCESS;
132 14 : }
133 :
134 0 : HCCL_INFO("CallMsprofReportHostNodeApi, HostNodeApiType[%u]", MSPROF_REPORT_NODE_LAUNCH_TYPE);
135 0 : CHK_RET(hrtMsprofReportApi(1, &reporterData));
136 0 : return HCCL_SUCCESS;
137 : }
138 :
139 3 : HcclResult ProfilingManager::CallMsprofReportNodeInfo(uint64_t beginTime, uint64_t endTime,
140 : const std::string profName, uint32_t threadId)
141 : {
142 3 : uint64_t itemId = hrtMsprofGetHashId(profName.c_str(), profName.length());
143 3 : auto mode = GetWorkflowMode();
144 : // hostapi开关 1) 开启: 单算子、静态图模式均上报; 关闭: 静态图模式或者acl graph场景缓存, 单算子不上报
145 3 : if (isHostApiSubscribe_ || mode == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OPS_KERNEL_INFO_LIB || GetThreadCaptureStatus()) {
146 1 : CHK_RET(CallMsprofReportHostNodeApi(beginTime, endTime, itemId, threadId));
147 : }
148 : // additionInfo开关 1) 开启: 单算子、静态图模式均上报; 关闭: 静态图或者acl graph场景模式缓存, 单算子不上报
149 3 : if (isAdditionInfoSubscribe_ || mode == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OPS_KERNEL_INFO_LIB || GetThreadCaptureStatus()) {
150 1 : CHK_RET(CallMsprofReportHostNodeBasicInfo(endTime, itemId, threadId));
151 : }
152 3 : return HCCL_SUCCESS;
153 : }
154 :
155 13 : HcclResult ProfilingManager::CallMsprofReportHostApi(HcclCMDType cmdType, uint64_t beginTime, u64 count,
156 : HcclDataType dataType, AlgType algType, uint64_t groupName, u32 numBlocks) const
157 : {
158 26 : if (!isHostApiSubscribe_ && GetWorkflowMode() == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE &&
159 13 : !GetThreadCaptureStatus()) {
160 0 : return HCCL_SUCCESS;
161 : }
162 13 : uint64_t endTime = hrtMsprofSysCycleTime();
163 13 : uint32_t threadId = SalGetTid();
164 13 : uint32_t type = static_cast<int32_t>(cmdType);
165 13 : const std::string profName(GetProfOpName(cmdType));
166 13 : uint64_t itemId = hrtMsprofGetHashId(profName.c_str(), profName.length());
167 13 : if (GetWorkflowMode() == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE && IsLaunchKernelMode() != true) {
168 13 : CHK_RET(CallMsprofReportHostAclApi(type, beginTime, endTime, itemId, threadId));
169 13 : CHK_RET(CallMsprofReportHostNodeApi(beginTime, endTime, itemId, threadId));
170 13 : if (isAdditionInfoSubscribe_ || GetThreadCaptureStatus()) {
171 13 : CHK_RET(CallMsprofReportHostNodeBasicInfo(endTime, itemId, threadId, numBlocks));
172 : }
173 : }
174 13 : std::string algTypeStr = TransferAlgType(algType);
175 13 : CHK_RET(CallMsprofReportHostHcclOpInfo(endTime, threadId, count, dataType, algTypeStr, groupName));
176 13 : CHK_RET(CallMsprofReportHostHcclOpApi(beginTime, endTime, itemId, threadId));
177 13 : return HCCL_SUCCESS;
178 13 : }
179 :
180 70 : inline HcclResult ProfilingManager::RegEsTaskType(ProfTaskType taskType) const
181 : {
182 70 : const std::string str(GetProfTaskOpName(taskType));
183 70 : CHK_RET(hrtMsprofRegTypeInfo(MSPROF_REPORT_HCCL_NODE_LEVEL, static_cast<uint32_t>(taskType), str.c_str()));
184 70 : return HCCL_SUCCESS;
185 70 : }
186 :
187 46 : HcclResult ProfilingManager::ReportTaskApi(
188 : bool isMainStrem, uint64_t beginTime, ProfTaskType taskType, uint32_t agingFlag) const
189 : {
190 46 : HcclWorkflowMode mode = GetWorkflowMode();
191 : // 1、单算子场景，如果订阅开关没有开，直接退出
192 : // 2、l0 l1 级别时, 子图 launch都需要上报
193 46 : if ((!isTaskApiSubscribe_) &&
194 22 : (mode == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE) &&
195 109 : (taskType != ProfTaskType::TASK_LAUNCH_FFTS_TASK) && (taskType != ProfTaskType::TASK_AIV) &&
196 17 : !GetThreadCaptureStatus()){
197 17 : return HCCL_SUCCESS;
198 : }
199 29 : MsprofApi reporterData{};
200 29 : reporterData.level = MSPROF_REPORT_HCCL_NODE_LEVEL;
201 29 : reporterData.type = (isMainStrem == true) ? MSPROF_REPORT_HCCL_MASTER_TYPE : MSPROF_REPORT_HCCL_SLAVE_TYPE;
202 29 : reporterData.threadId = SalGetTid();
203 29 : reporterData.beginTime = beginTime;
204 29 : reporterData.endTime = hrtMsprofSysCycleTime();
205 29 : const std::string taskName(GetProfTaskOpName(taskType));
206 29 : reporterData.itemId = hrtMsprofGetHashId(taskName.c_str(), taskName.length());
207 :
208 : // 2、图下沉场景或者acl graph场景，缓存对应数据
209 29 : if ((mode == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OPS_KERNEL_INFO_LIB) || GetThreadCaptureStatus()) {
210 : // 缓存对应数据
211 24 : HCCL_INFO("CallMsprofReportTaskApi, storageTaskApi");
212 24 : std::unique_lock<std::mutex> lock(reportDataQueueMutex_);
213 24 : storageTaskApi_.push(reporterData);
214 24 : return HCCL_SUCCESS;
215 24 : }
216 :
217 5 : HCCL_INFO("CallMsprofReportTaskApi, isMainStrem[%u], taskType[%d], taskName[%s]",
218 : isMainStrem,
219 : static_cast<int32_t>(taskType),
220 : taskName.c_str());
221 5 : CHK_RET(hrtMsprofReportApi(agingFlag, &reporterData));
222 5 : return HCCL_SUCCESS;
223 29 : }
224 :
225 13 : HcclResult ProfilingManager::CallMsprofReportHostAclApi(
226 : uint32_t type, uint64_t beginTime, uint64_t endTime, uint64_t itemId, uint32_t threadId) const
227 : {
228 13 : if (!isHostApiSubscribe_) {
229 13 : return HCCL_SUCCESS;
230 : }
231 0 : MsprofApi reporterData{};
232 0 : reporterData.level = MSPROF_REPORT_ACL_LEVEL;
233 0 : reporterData.type = static_cast<int32_t>(type) + MSPROF_REPORT_ACL_HOST_HCCL_BASE_TYPE;
234 0 : reporterData.threadId = threadId;
235 0 : reporterData.beginTime = beginTime;
236 0 : reporterData.endTime = endTime;
237 0 : reporterData.itemId = itemId;
238 :
239 0 : HCCL_INFO("CallMsprofReportHostHcclOpApi, HcclOpApiType[%u]", reporterData.type);
240 0 : CHK_RET(hrtMsprofReportApi(1, &reporterData));
241 0 : return HCCL_SUCCESS;
242 : }
243 :
244 14 : HcclResult ProfilingManager::CallMsprofReportHostNodeBasicInfo(
245 : uint64_t timeStamp, uint64_t itemId, uint32_t threadId, u32 numBlocks) const
246 : {
247 14 : MsprofCompactInfo reporterData{};
248 :
249 14 : reporterData.level = MSPROF_REPORT_NODE_LEVEL;
250 14 : reporterData.type = MSPROF_REPORT_NODE_BASIC_INFO_TYPE;
251 14 : reporterData.threadId = threadId;
252 14 : reporterData.dataLen = sizeof(MsprofNodeBasicInfo);
253 14 : reporterData.timeStamp = timeStamp;
254 :
255 14 : reporterData.data.nodeBasicInfo.opName = itemId;
256 14 : reporterData.data.nodeBasicInfo.taskType = MSPROF_GE_TASK_TYPE_HCCL;
257 14 : reporterData.data.nodeBasicInfo.opType = itemId;
258 14 : reporterData.data.nodeBasicInfo.blockDim = numBlocks;
259 14 : reporterData.data.nodeBasicInfo.opFlag = 0;
260 :
261 : // 图下沉场景或者acl graph场景，缓存对应数据
262 14 : auto mode = GetWorkflowMode();
263 14 : if ((mode == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OPS_KERNEL_INFO_LIB) || GetThreadCaptureStatus()) {
264 14 : s32 deviceLogicId = -1;
265 28 : CHK_RET(hrtGetDevice(&deviceLogicId));
266 14 : HCCL_INFO("CallMsprofReportHostNodeBasicInfo, storageCompactInfo, The used deviceLogicId is [%d]", deviceLogicId);
267 : u32 maxDeviceNum;
268 14 : CHK_RET(GetMaxDevNum(maxDeviceNum));
269 14 : CHK_PRT_RET(static_cast<u32>(deviceLogicId) >= maxDeviceNum,
270 : HCCL_ERROR("[ReportHostNodeBasicInfo]deviceLogicId_[%u] is bigger than maxDeviceNum[%u]",
271 : static_cast<u32>(deviceLogicId), maxDeviceNum), HCCL_E_INTERNAL);
272 14 : std::unique_lock<std::mutex> lock(reportCompactInfoMutex_[deviceLogicId]);
273 14 : storageCompactInfo_[deviceLogicId].push(reporterData);
274 14 : if (!isAdditionInfoSubscribe_) {
275 14 : return HCCL_SUCCESS;
276 : }
277 14 : }
278 0 : HCCL_INFO("CallMsprofReportHostNodeBasicInfo, HostNodeBasicInfoType[%u]", MSPROF_REPORT_NODE_BASIC_INFO_TYPE);
279 0 : CHK_RET(hrtMsprofReportCompactInfo(1, &reporterData, sizeof(MsprofCompactInfo)));
280 0 : return HCCL_SUCCESS;
281 : }
282 :
283 13 : HcclResult ProfilingManager::CallMsprofReportHostHcclOpApi(
284 : uint64_t beginTime, uint64_t endTime, uint64_t itemId, uint32_t threadId) const
285 : {
286 13 : auto mode = GetWorkflowMode();
287 0 : if ((!isHostHcclOpSubscribe_ || GetExternalInputHcclAicpuUnfold() ||
288 13 : GetExternalInputHcclEnableFfts()) &&
289 26 : (mode == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE) &&
290 13 : !GetThreadCaptureStatus()) {
291 0 : return HCCL_SUCCESS;
292 : }
293 :
294 13 : MsprofApi reporterData{};
295 13 : reporterData.level = MSPROF_REPORT_HCCL_NODE_LEVEL;
296 : // 集合通信算子粒度的都是主流
297 13 : reporterData.type = MSPROF_REPORT_HCCL_MASTER_TYPE;
298 13 : reporterData.threadId = threadId;
299 13 : reporterData.beginTime = beginTime;
300 13 : reporterData.endTime = endTime;
301 13 : reporterData.itemId = itemId;
302 :
303 : // 静态图场景或者acl graph场景, 一次下发，多次执行; 缓存对应数据
304 13 : if ((mode == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OPS_KERNEL_INFO_LIB) || GetThreadCaptureStatus()) {
305 13 : HCCL_INFO("CallMsprofReportOpApi, storageOpApi");
306 13 : std::unique_lock<std::mutex> lock(reportDataOpQueueMutex_);
307 13 : storageOpApi_.push(reporterData);
308 13 : return HCCL_SUCCESS;
309 13 : }
310 :
311 0 : HCCL_INFO("CallMsprofReportHostHcclOpApi, HcclOpApiType[%u]", MSPROF_REPORT_HCCL_MASTER_TYPE);
312 0 : CHK_RET(hrtMsprofReportApi(1, &reporterData));
313 0 : return HCCL_SUCCESS;
314 : }
315 :
316 46 : HcclResult ProfilingManager::CallMsprofReportTaskApi(
317 : bool isMainStrem, uint64_t beginTime, ProfTaskType taskType) const
318 : {
319 46 : uint32_t agingFlag = 0;
320 46 : if (GetWorkflowMode() == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE) {
321 22 : agingFlag = aging;
322 : }
323 46 : CHK_RET(ReportTaskApi(isMainStrem, beginTime, taskType, agingFlag));
324 :
325 46 : return HCCL_SUCCESS;
326 : }
327 :
328 13 : HcclResult ProfilingManager::CallMsprofReportHostHcclOpInfo(
329 : uint64_t timeStamp, uint32_t threadId, u64 count, HcclDataType dataType, std::string &algTypeStr, uint64_t groupName) const
330 : {
331 13 : MsprofCompactInfo reporterData{};
332 :
333 13 : reporterData.level = MSPROF_REPORT_NODE_LEVEL;
334 13 : reporterData.type = MSPROF_REPORT_NODE_HCCL_OP_INFO_TYPE;
335 13 : reporterData.threadId = threadId;
336 13 : reporterData.dataLen = sizeof(MsprofHCCLOPInfo);
337 13 : reporterData.timeStamp = timeStamp;
338 :
339 13 : reporterData.data.hcclopInfo.relay = 0;
340 13 : reporterData.data.hcclopInfo.retry = 0;
341 13 : reporterData.data.hcclopInfo.dataType = dataType;
342 13 : reporterData.data.hcclopInfo.algType = hrtMsprofGetHashId(algTypeStr.c_str(), algTypeStr.length());
343 13 : reporterData.data.hcclopInfo.count = count;
344 13 : reporterData.data.hcclopInfo.groupName = groupName;
345 :
346 : // 图下沉场景或者acl graph场景，缓存对应数据
347 13 : if ( (GetWorkflowMode() == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OPS_KERNEL_INFO_LIB) || GetThreadCaptureStatus()) {
348 : // 缓存对应数据
349 13 : s32 deviceLogicId = -1;
350 26 : CHK_RET(hrtGetDevice(&deviceLogicId));
351 13 : HCCL_INFO("CallMsprofReportHostHcclOpInfo, storageCompactInfo, The used deviceLogicId is [%d]", deviceLogicId);
352 : u32 maxDeviceNum;
353 13 : CHK_RET(GetMaxDevNum(maxDeviceNum));
354 13 : CHK_PRT_RET(static_cast<u32>(deviceLogicId) >= maxDeviceNum,
355 : HCCL_ERROR("[ReportAdditionInfo]deviceLogicId_[%u] is bigger than maxDeviceNum[%u]",
356 : static_cast<u32>(deviceLogicId), maxDeviceNum), HCCL_E_INTERNAL);
357 13 : std::unique_lock<std::mutex> lock(reportCompactInfoMutex_[deviceLogicId]);
358 13 : storageCompactInfo_[deviceLogicId].push(reporterData);
359 13 : if (!isHostApiSubscribe_) {
360 13 : return HCCL_SUCCESS;
361 : }
362 13 : }
363 :
364 0 : HCCL_INFO("CallMsprofReportHostHcclOpInfo, hcclopInfoType[%u]", MSPROF_REPORT_NODE_HCCL_OP_INFO_TYPE);
365 0 : CHK_RET(hrtMsprofReportCompactInfo(1, &reporterData, sizeof(MsprofCompactInfo)));
366 0 : return HCCL_SUCCESS;
367 : }
368 :
369 1 : HcclResult ProfilingManager::CallMsprofReportMc2CommInfo(uint64_t timeStamp, const void *data, int len)
370 : {
371 1 : uint32_t agingFlag = 0;
372 1 : auto mode = GetWorkflowMode();
373 1 : if (mode == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE) {
374 1 : agingFlag = 1;
375 : }
376 :
377 1 : MsprofAdditionalInfo reporterData{};
378 1 : reporterData.level = MSPROF_REPORT_NODE_LEVEL;
379 1 : reporterData.type = MSPROF_REPORT_NODE_MC2_COMMINFO_TYPE;
380 1 : reporterData.threadId = SalGetTid();
381 1 : reporterData.dataLen = len;
382 1 : reporterData.timeStamp = timeStamp;
383 :
384 1 : s32 sret = memcpy_s(reporterData.data, MSPROF_ADDTIONAL_INFO_DATA_LENGTH, data, len);
385 1 : CHK_PRT_RET(sret != EOK, HCCL_ERROR("memcpy failed. errorno[%d]:", sret), HCCL_E_MEMORY);
386 :
387 1 : if (!isHostApiSubscribe_ || GetThreadCaptureStatus()) {
388 : // 缓存对应数据
389 1 : s32 deviceLogicId = -1;
390 1 : CHK_RET(hrtGetDevice(&deviceLogicId));
391 1 : HCCL_INFO("CallMsprofReportAdditionInfo, storageAdditionInfo, The used deviceLogicId is [%d]", deviceLogicId);
392 : u32 maxDeviceNum;
393 1 : CHK_RET(GetMaxDevNum(maxDeviceNum));
394 1 : CHK_PRT_RET(static_cast<u32>(deviceLogicId) >= maxDeviceNum,
395 : HCCL_ERROR("[ReportAdditionInfo]deviceLogicId_[%u] is bigger than maxDeviceNum[%u]",
396 : static_cast<u32>(deviceLogicId), maxDeviceNum), HCCL_E_INTERNAL);
397 1 : std::unique_lock<std::mutex> lock(reportAddInfoMutex_[deviceLogicId]);
398 1 : storageAdditionInfo_[deviceLogicId].push(reporterData);
399 1 : return HCCL_SUCCESS;
400 1 : }
401 0 : HCCL_INFO("CallMsprofReportMc2CommInfo, Mc2CommInfoType[%u]", MSPROF_REPORT_NODE_MC2_COMMINFO_TYPE);
402 0 : CHK_RET(hrtMsprofReportAdditionalInfo(agingFlag, &reporterData, sizeof(MsprofAdditionalInfo)));
403 0 : return HCCL_SUCCESS;
404 : }
405 :
406 0 : HcclResult ProfilingManager::CallEsMsprofReportTaskApi(
407 : bool isMainStrem, uint64_t beginTime, ProfTaskType taskType) const
408 : {
409 0 : if (!isTaskApiSubscribe_) {
410 0 : return HCCL_SUCCESS;
411 : }
412 0 : MsprofApi reporterData{};
413 0 : reporterData.level = MSPROF_REPORT_HCCL_NODE_LEVEL;
414 0 : reporterData.type = (isMainStrem == true) ? MSPROF_REPORT_HCCL_MASTER_TYPE : MSPROF_REPORT_HCCL_SLAVE_TYPE;
415 0 : reporterData.threadId = SalGetTid();
416 0 : reporterData.beginTime = beginTime;
417 0 : reporterData.endTime = hrtMsprofSysCycleTime();
418 0 : const std::string taskName(GetProfTaskOpName(taskType));
419 0 : reporterData.itemId = hrtMsprofGetHashId(taskName.c_str(), taskName.length());
420 0 : HCCL_INFO("ReportTaskApi, isMainStrem[%u], taskType[%d], taskName[%s]",
421 : isMainStrem,
422 : static_cast<int32_t>(taskType),
423 : taskName.c_str());
424 0 : CHK_RET(hrtMsprofReportApi(aging, &reporterData));
425 0 : return HCCL_SUCCESS;
426 0 : }
427 :
428 1 : HcclResult ProfilingManager::CallMsprofReportMultiThreadInfo(const std::vector<uint32_t> &tidInfo) const
429 : {
430 1 : if (!isTaskApiSubscribe_ && !GetThreadCaptureStatus()) {
431 1 : return HCCL_SUCCESS;
432 : }
433 :
434 : struct MsprofMultiThread threadInfo;
435 0 : uint64_t timeStamp = hrtMsprofSysCycleTime();
436 0 : uint32_t totalSize = tidInfo.size();
437 : uint32_t currentSize;
438 0 : uint32_t sendNum = totalSize / MSPROF_MULTI_THREAD_MAX_NUM + 1;
439 :
440 0 : for (uint32_t j = 0; j < sendNum; j++) {
441 0 : currentSize = totalSize - j * MSPROF_MULTI_THREAD_MAX_NUM;
442 0 : threadInfo.threadNum = currentSize > MSPROF_MULTI_THREAD_MAX_NUM ? MSPROF_MULTI_THREAD_MAX_NUM : currentSize;
443 0 : for (uint32_t i = 0; i < threadInfo.threadNum; i++) {
444 0 : threadInfo.threadId[i] = tidInfo[i + j * MSPROF_MULTI_THREAD_MAX_NUM];
445 : }
446 0 : HCCL_INFO("CallMsprofReportMultiThreadInfo");
447 0 : CHK_RET(CallMsprofReportAdditionInfo(static_cast<int32_t>(ProfTaskType::TASK_MULTI_THREAD),
448 : timeStamp,
449 : &threadInfo,
450 : sizeof(struct MsprofMultiThread)));
451 : }
452 :
453 0 : return HCCL_SUCCESS;
454 : }
455 :
456 0 : HcclResult ProfilingManager::CallMsprofReportContextIdInfo(u32 ctxIdMax) const
457 : {
458 : struct MsprofContextIdInfo ctxInfo;
459 0 : ctxInfo.ctxIdNum = 2; // 因HCCL ctxId连续，固定上报2个：开始：0； 结束：ctxIdMax
460 0 : ctxInfo.ctxIds[0] = 0;
461 0 : ctxInfo.ctxIds[1] = ctxIdMax;
462 :
463 0 : uint64_t timeStamp = hrtMsprofSysCycleTime();
464 0 : HCCL_INFO("CallMsprofReportContextIdInfo, ctxIdNum[%u]", ctxInfo.ctxIdNum);
465 0 : CHK_RET(CallMsprofReportAdditionInfo(MSPROF_REPORT_NODE_CONTEXT_ID_INFO_TYPE,
466 : timeStamp,
467 : &ctxInfo,
468 : sizeof(struct MsprofContextIdInfo)));
469 0 : return HCCL_SUCCESS;
470 : }
471 :
472 234 : HcclResult ProfilingManager::ClearStoragedProfilingInfo()
473 : {
474 234 : std::unique_lock<std::mutex> lockTaskApi(reportDataQueueMutex_);
475 234 : HCCL_INFO("[ClearStoragedProfilingInfo] taskApiQueueSize is [%u]", storageTaskApi_.size());
476 :
477 234 : std::queue<MsprofApi> emptyTaskApi;
478 234 : std::swap(storageTaskApi_, emptyTaskApi);
479 :
480 234 : std::unique_lock<std::mutex> lockOpApi(reportDataOpQueueMutex_);
481 234 : HCCL_INFO("[ClearStoragedProfilingInfo] opApiQueueSize is [%u]", storageOpApi_.size());
482 :
483 234 : std::queue<MsprofApi> emptyOpApi;
484 234 : std::swap(storageOpApi_, emptyOpApi);
485 :
486 234 : s32 deviceLogicId = -1;
487 234 : CHK_RET(hrtGetDevice(&deviceLogicId));
488 234 : HCCL_INFO("[ClearStoragedAdditionInfo] The size of the storageAdditionInfo_[%d] is [%u]",
489 : deviceLogicId, storageAdditionInfo_[deviceLogicId].size());
490 : u32 maxDeviceNum;
491 234 : CHK_RET(GetMaxDevNum(maxDeviceNum));
492 234 : CHK_PRT_RET(static_cast<u32>(deviceLogicId) >= maxDeviceNum,
493 : HCCL_ERROR("[ReportStoragedAdditionInfo]deviceLogicId_[%u] is bigger than maxDeviceNum[%u]",
494 : static_cast<u32>(deviceLogicId), maxDeviceNum), HCCL_E_INTERNAL);
495 234 : std::unique_lock<std::mutex> lockAddInfo(reportAddInfoMutex_[deviceLogicId]);
496 234 : std::queue<MsprofAdditionalInfo> emptyAddition;
497 234 : std::swap(storageAdditionInfo_[deviceLogicId], emptyAddition);
498 :
499 234 : HCCL_INFO("[ClearStoragedCompactInfo] The size of the storageCompactInfo_[%d] is [%u]",
500 : deviceLogicId, storageCompactInfo_[deviceLogicId].size());
501 234 : std::unique_lock<std::mutex> lockCompactInfo(reportCompactInfoMutex_[deviceLogicId]);
502 234 : std::queue<MsprofCompactInfo> emptyCompactInfo;
503 234 : std::swap(storageCompactInfo_[deviceLogicId], emptyCompactInfo);
504 :
505 234 : HCCL_INFO("[ClearStorageAdditionInfoFftsCapture_] The size of the storageAdditionInfoFftsCapture_[%d] is [%u]",
506 : deviceLogicId, storageAdditionInfoFftsCapture_[deviceLogicId].size());
507 234 : std::unique_lock<std::mutex> lockAddInfoCapture(reportAddInfoFftsCaptureMutex_[deviceLogicId]);
508 234 : std::queue<MsprofAdditionalInfo> emptyAdditionCapture;
509 234 : std::swap(storageAdditionInfoFftsCapture_[deviceLogicId], emptyAdditionCapture);
510 234 : return HCCL_SUCCESS;
511 234 : }
512 :
513 24 : HcclResult ProfilingManager::ReportAdditionInfo(
514 : uint32_t type, uint64_t timeStamp, const void *data, int len, uint32_t agingFlag) const
515 : {
516 24 : HcclWorkflowMode mode = GetWorkflowMode();
517 : // 1、单算子场景，如果订阅开关没有开, 且上报的不是contextID,直接退出;
518 24 : if ((!isAdditionInfoSubscribe_) &&
519 0 : (mode == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE) &&
520 48 : (type != MSPROF_REPORT_NODE_CONTEXT_ID_INFO_TYPE) &&
521 0 : !GetThreadCaptureStatus()) {
522 0 : return HCCL_SUCCESS;
523 : }
524 24 : MsprofAdditionalInfo reporterData{};
525 :
526 24 : reporterData.level = MSPROF_REPORT_HCCL_NODE_LEVEL;
527 24 : reporterData.type = type;
528 24 : reporterData.threadId = SalGetTid();
529 24 : reporterData.dataLen = len;
530 24 : reporterData.timeStamp = timeStamp;
531 24 : s32 sret = memcpy_s(reporterData.data, sizeof(reporterData.data), data, len);
532 24 : CHK_PRT_RET(sret != EOK, HCCL_ERROR("memcpy failed. errorno[%d]:", sret), HCCL_E_MEMORY);
533 :
534 : // 2、图下沉场景或者acl graph场景，缓存对应数据
535 24 : if ((mode == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OPS_KERNEL_INFO_LIB) || GetThreadCaptureStatus() ||
536 0 : (isFftsDispatcher_.load() && // 3、FFTS+下发场景，addition开关打开，缓存对应数据
537 0 : (mode == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE) &&
538 0 : (isAdditionInfoSubscribe_))) {
539 : // 缓存对应数据
540 24 : s32 deviceLogicId = -1;
541 48 : CHK_RET(hrtGetDevice(&deviceLogicId));
542 24 : HCCL_INFO("CallMsprofReportAdditionInfo, storageAdditionInfo, The used deviceLogicId is [%d]", deviceLogicId);
543 : u32 maxDeviceNum;
544 24 : CHK_RET(GetMaxDevNum(maxDeviceNum));
545 24 : CHK_PRT_RET(static_cast<u32>(deviceLogicId) >= maxDeviceNum,
546 : HCCL_ERROR("[ReportAdditionInfo]deviceLogicId_[%u] is bigger than maxDeviceNum[%u]",
547 : static_cast<u32>(deviceLogicId), maxDeviceNum), HCCL_E_INTERNAL);
548 24 : std::unique_lock<std::mutex> lock(reportAddInfoMutex_[deviceLogicId]);
549 24 : storageAdditionInfo_[deviceLogicId].push(reporterData);
550 24 : if (isFftsDispatcher_ || !isAdditionInfoSubscribe_) {
551 24 : return HCCL_SUCCESS;
552 : }
553 24 : }
554 :
555 : // 4、开关开启，非子图下发场景，直接上报对应数据
556 0 : HCCL_INFO("CallMsprofReportAdditionInfo, AdditionInfoType[%u]", type);
557 0 : CHK_RET(hrtMsprofReportAdditionalInfo(agingFlag, &reporterData, sizeof(MsprofAdditionalInfo)));
558 0 : return HCCL_SUCCESS;
559 : }
560 :
561 24 : HcclResult ProfilingManager::CallMsprofReportAdditionInfo(
562 : uint32_t type, uint64_t timeStamp, const void *data, int len) const
563 : {
564 24 : uint32_t agingFlag = 0;
565 24 : if (GetWorkflowMode() == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE) {
566 0 : agingFlag = 1;
567 : }
568 24 : CHK_RET(ReportAdditionInfo(type, timeStamp, data, len, agingFlag));
569 24 : return HCCL_SUCCESS;
570 : }
571 :
572 1 : HcclResult ProfilingManager::CallMsprofReportEsAdditionInfo(
573 : uint32_t type, uint64_t timeStamp, const void *data, int len) const
574 : {
575 1 : if (!isAdditionInfoSubscribe_) {
576 1 : return HCCL_SUCCESS;
577 : }
578 0 : MsprofAdditionalInfo reporterData{};
579 :
580 0 : reporterData.level = MSPROF_REPORT_HCCL_NODE_LEVEL;
581 0 : reporterData.type = type;
582 0 : reporterData.threadId = SalGetTid();
583 0 : reporterData.dataLen = len;
584 0 : reporterData.timeStamp = timeStamp;
585 :
586 0 : s32 sret = memcpy_s(reporterData.data, sizeof(reporterData.data), data, len);
587 0 : CHK_PRT_RET(sret != EOK, HCCL_ERROR("memcpy failed. errorno[%d]:", sret), HCCL_E_MEMORY);
588 0 : HCCL_INFO("CallMsprofReportAdditionInfo, AdditionInfoType[%u]", type);
589 0 : CHK_RET(hrtMsprofReportAdditionalInfo(aging, &reporterData, sizeof(MsprofAdditionalInfo)));
590 0 : return HCCL_SUCCESS;
591 : }
592 :
593 3 : Prof_Status ProfilingManager::PluginInit() const
594 : {
595 3 : CHK_PRT_RET((reporterCallback_ == nullptr),
596 : HCCL_ERROR("[ProfilingManager][PluginInit] MsprofReporterCallback callback is nullptr."),
597 : FAILED);
598 :
599 0 : int32_t cb_ret = reporterCallback_(static_cast<uint32_t>(MsprofReporterModuleId::MSPROF_MODULE_HCCL),
600 : static_cast<uint32_t>(MsprofReporterCallbackType::MSPROF_REPORTER_INIT),
601 : nullptr,
602 : 0);
603 0 : CHK_PRT_RET(
604 : (cb_ret != MSPROF_ERROR_NONE), HCCL_ERROR("[ProfilingManager][PluginInit] Reporter init failed."), FAILED);
605 :
606 0 : HCCL_INFO("[ProfilingManager][PluginInit] Reporter init success.");
607 :
608 0 : return SUCCESS;
609 : }
610 :
611 2 : Prof_Status ProfilingManager::PluginUnInit() const
612 : {
613 2 : CHK_PRT_RET((reporterCallback_ == nullptr),
614 : HCCL_ERROR("[ProfilingManager][PluginUnInit] MsprofReporterCallback callback is nullptr."),
615 : FAILED);
616 :
617 0 : int32_t cb_ret = reporterCallback_(static_cast<uint32_t>(MsprofReporterModuleId::MSPROF_MODULE_HCCL),
618 : static_cast<uint32_t>(MsprofReporterCallbackType::MSPROF_REPORTER_UNINIT),
619 : nullptr,
620 : 0);
621 0 : CHK_PRT_RET((cb_ret != MSPROF_ERROR_NONE),
622 : HCCL_ERROR("[ProfilingManager][PluginUnInit] Profiling reporter uinit failed."),
623 : FAILED);
624 :
625 0 : HCCL_INFO("[ProfilingManager][PluginUnInit] Profiling reporter uinit success.");
626 :
627 0 : return SUCCESS;
628 : }
629 :
630 0 : HcclResult ProfilingManager::CallMsprofReportAdditionInfoForEsLookup(EsLoopUpPara ¶, ProfTaskType type)
631 : {
632 0 : HCCL_INFO("Entry CallMsprofReportAdditionInfoForEsLookup");
633 0 : HCCLReportData hcclReportData{};
634 0 : hcclReportData.ts = hrtMsprofSysCycleTime();
635 0 : std::string nameInfo = GetProfTaskOpName(type);
636 0 : hcclReportData.profInfo.itemId = hrtMsprofGetHashId(nameInfo.c_str(), nameInfo.length());
637 0 : std::string cclTag = std::to_string(para.tag);
638 0 : hcclReportData.profInfo.cclTag = hrtMsprofGetHashId(cclTag.c_str(), cclTag.length());
639 0 : hcclReportData.profInfo.groupName = static_cast<u64>(reinterpret_cast<uintptr_t>("unknown"));
640 0 : hcclReportData.profInfo.rankSize = 0;
641 0 : hcclReportData.profInfo.workFlowMode = static_cast<u32>(HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE);
642 0 : hcclReportData.profInfo.planeID = 0;
643 0 : hcclReportData.profInfo.notifyID = 0;
644 0 : hcclReportData.profInfo.stage = 0;
645 0 : hcclReportData.profInfo.role = static_cast<uint32_t>(TaskRole::DST);
646 0 : hcclReportData.profInfo.durationEstimated = 0;
647 0 : hcclReportData.profInfo.srcAddr = static_cast<u64>(reinterpret_cast<uintptr_t>(para.srcAddr));
648 0 : hcclReportData.profInfo.dstAddr = static_cast<u64>(reinterpret_cast<uintptr_t>(para.dstAddr));
649 0 : hcclReportData.profInfo.dataSize = static_cast<u32>(para.dataSize);
650 0 : hcclReportData.profInfo.opType = 0;
651 0 : hcclReportData.profInfo.dataType = HCCL_DATA_TYPE_FP32;
652 0 : hcclReportData.profInfo.linkType = static_cast<u32>(LinkType::LINK_ONCHIP);
653 0 : hcclReportData.profInfo.transportType = static_cast<int32_t>(SimpleTaskType::RDMA);
654 :
655 0 : int32_t ret = CallMsprofReportEsAdditionInfo(static_cast<uint32_t>(ProfTaskType::TASK_HCCL_INFO),
656 : hcclReportData.ts,
657 : &hcclReportData.profInfo,
658 0 : sizeof(hcclReportData.profInfo));
659 0 : CHK_PRT_RET((ret != 0), HCCL_ERROR("[TaskProfiling] CallMsprofReportAdditionInfoForEsLookup failed."), HCCL_E_PARA);
660 0 : return HCCL_SUCCESS;
661 0 : }
662 :
663 0 : Prof_Status ProfilingManager::GetHashKey(MsprofHashData &data) const
664 : {
665 0 : CHK_PRT_RET((reporterCallback_ == nullptr),
666 : HCCL_ERROR("[ProfilingManager][GetHashKey] MsprofReporterCallback callback is nullptr."),
667 : FAILED);
668 0 : if (data.dataLen == 0) {
669 0 : data.hashId = 0;
670 0 : HCCL_INFO("[Check][Param]PluginUnInit MsprofReporterCallback GetHashKey in default.");
671 : } else {
672 0 : int32_t cb_ret = reporterCallback_(static_cast<uint32_t>(MsprofReporterModuleId::MSPROF_MODULE_HCCL),
673 : static_cast<uint32_t>(MsprofReporterCallbackType::MSPROF_REPORTER_HASH),
674 : static_cast<void *>(&data),
675 : sizeof(MsprofHashData));
676 :
677 0 : CHK_PRT_RET((cb_ret != MSPROF_ERROR_NONE),
678 : HCCL_ERROR("[ProfilingManager][GetHashKey] Profiling reporter GetHashKey failed."),
679 : FAILED);
680 :
681 0 : HCCL_INFO("[ProfilingManager][GetHashKey] Profiling reporter GetHashKey success.");
682 : }
683 :
684 0 : return SUCCESS;
685 : }
686 :
687 3 : HcclResult ProfilingManager::ReportStoragedTaskApi()
688 : {
689 3 : if (!isTaskApiSubscribe_ && !isHostApiSubscribe_) {
690 1 : return HCCL_SUCCESS;
691 : }
692 2 : std::unique_lock<std::mutex> lock(reportDataQueueMutex_);
693 2 : HCCL_INFO("[ReportStoragedTaskApi] taskApiQueueSize is [%u]", storageTaskApi_.size());
694 2 : if (!storageTaskApi_.empty()) {
695 2 : std::queue<MsprofApi> tempTaskApi = storageTaskApi_;
696 2 : lock.unlock();
697 50 : while (!tempTaskApi.empty()) {
698 48 : MsprofApi reportData = tempTaskApi.front();
699 48 : tempTaskApi.pop();
700 48 : CHK_RET(hrtMsprofReportApi(0, &reportData));
701 : }
702 2 : }
703 2 : return HCCL_SUCCESS;
704 2 : }
705 :
706 3 : HcclResult ProfilingManager::ReportStoragedOpApi()
707 : {
708 5 : if (!isHostHcclOpSubscribe_ || GetExternalInputHcclAicpuUnfold() ||
709 2 : GetExternalInputHcclEnableFfts()) {
710 3 : return HCCL_SUCCESS;
711 : }
712 :
713 0 : std::unique_lock<std::mutex> lock(reportDataOpQueueMutex_);
714 0 : HCCL_INFO("[ReportStoragedOpApi] opApiQueueSize is [%u]", storageOpApi_.size());
715 0 : if (!storageOpApi_.empty()) {
716 0 : std::queue<MsprofApi> tempOpApi = storageOpApi_;
717 0 : lock.unlock();
718 0 : while (!tempOpApi.empty()) {
719 0 : MsprofApi reportData = tempOpApi.front();
720 0 : tempOpApi.pop();
721 0 : CHK_RET(hrtMsprofReportApi(0, &reportData));
722 : }
723 0 : }
724 0 : return HCCL_SUCCESS;
725 0 : }
726 :
727 2 : HcclResult ProfilingManager::ReportStoragedAdditionInfo()
728 : {
729 132 : for (u32 i = 0; i < MAX_MODULE_DEVICE_NUM; i++) {
730 130 : std::unique_lock<std::mutex> lock(reportAddInfoMutex_[i]);
731 130 : HCCL_INFO("[ReportStoragedAdditionInfo] The size of the storageAdditionInfo_[%u] is [%u]",
732 : i, storageAdditionInfo_[i].size());
733 130 : if (!storageAdditionInfo_[i].empty()) {
734 2 : std::queue<MsprofAdditionalInfo> tempTaskAdditionalInfo = storageAdditionInfo_[i];
735 2 : lock.unlock();
736 50 : while (!tempTaskAdditionalInfo.empty()) {
737 48 : MsprofAdditionalInfo reportData = tempTaskAdditionalInfo.front();
738 48 : tempTaskAdditionalInfo.pop();
739 48 : CHK_RET(hrtMsprofReportAdditionalInfo(0, &reportData, sizeof(MsprofAdditionalInfo)));
740 : }
741 2 : }
742 : // acl graph ffts+场景下， 一次下发多次执行， 执行时上报保存的task信息
743 130 : std::unique_lock<std::mutex> lockCapture(reportAddInfoFftsCaptureMutex_[i]);
744 130 : HCCL_INFO("[ReportStoragedAdditionInfo] The size of the storageAdditionInfoFftsCapture_[%u] is [%u]",
745 : i, storageAdditionInfoFftsCapture_[i].size());
746 130 : if (!storageAdditionInfoFftsCapture_[i].empty()) {
747 0 : std::queue<MsprofAdditionalInfo> tempTaskAdditionalInfo = storageAdditionInfoFftsCapture_[i];
748 0 : lockCapture.unlock();
749 0 : while (!tempTaskAdditionalInfo.empty()) {
750 0 : MsprofAdditionalInfo reportData = tempTaskAdditionalInfo.front();
751 0 : tempTaskAdditionalInfo.pop();
752 0 : CHK_RET(hrtMsprofReportAdditionalInfo(0, &reportData, sizeof(MsprofAdditionalInfo)));
753 : }
754 0 : }
755 130 : }
756 2 : return HCCL_SUCCESS;
757 : }
758 :
759 1 : HcclResult ProfilingManager::ReportStoragedCompactInfo()
760 : {
761 66 : for (u32 i = 0; i < MAX_MODULE_DEVICE_NUM; i++) {
762 65 : std::unique_lock<std::mutex> lock(reportCompactInfoMutex_[i]);
763 65 : HCCL_INFO("[ReportStoragedCompactInfo] The size of the storageCompactInfo_[%u] is [%u]",
764 : i, storageCompactInfo_[i].size());
765 65 : if (!storageCompactInfo_[i].empty()) {
766 1 : std::queue<MsprofCompactInfo> tempCompactInfo = storageCompactInfo_[i];
767 1 : lock.unlock();
768 2 : while (!tempCompactInfo.empty()) {
769 1 : MsprofCompactInfo reportData = tempCompactInfo.front();
770 1 : tempCompactInfo.pop();
771 1 : CHK_RET(hrtMsprofReportCompactInfo(0, &reportData, sizeof(MsprofCompactInfo)));
772 : }
773 1 : }
774 65 : }
775 1 : return HCCL_SUCCESS;
776 : }
777 :
778 0 : HcclResult ProfilingManager::ReportStoragedFftsInfo()
779 : {
780 0 : uint64_t ts = hrtMsprofSysCycleTime();
781 :
782 0 : s32 deviceLogicId = -1;
783 0 : if (hrtGetDevice(&deviceLogicId) != HCCL_SUCCESS) {
784 0 : deviceLogicId = 0;
785 0 : HCCL_WARNING("[ReportStoragedAdditionInfo]deviceLogicId[%d]", deviceLogicId);
786 : }
787 : u32 maxDeviceNum;
788 0 : CHK_RET(GetMaxDevNum(maxDeviceNum));
789 0 : CHK_PRT_RET(static_cast<u32>(deviceLogicId) >= maxDeviceNum,
790 : HCCL_ERROR("[ReportStoragedAdditionInfo]deviceLogicId_[%u] is bigger than maxDeviceNum[%u]",
791 : static_cast<u32>(deviceLogicId), maxDeviceNum), HCCL_E_INTERNAL);
792 0 : std::unique_lock<std::mutex> lock(reportAddInfoMutex_[deviceLogicId]);
793 0 : HCCL_INFO("[ReportStoragedFftsInfo] The size of the storageAdditionInfo_[%d] is [%u] ", deviceLogicId,
794 : storageAdditionInfo_[deviceLogicId].size());
795 :
796 0 : while (!storageAdditionInfo_[deviceLogicId].empty()) {
797 0 : MsprofAdditionalInfo reportData = storageAdditionInfo_[deviceLogicId].front();
798 0 : storageAdditionInfo_[deviceLogicId].pop();
799 0 : reportData.timeStamp = ts;
800 0 : if (GetThreadCaptureStatus()) {
801 : // acl graph ffts+ 场景下， 下发的task信息进行保存以便后续多次使用
802 0 : std::unique_lock<std::mutex> lockCapture(reportAddInfoFftsCaptureMutex_[deviceLogicId]);
803 0 : storageAdditionInfoFftsCapture_[deviceLogicId].push(reportData);
804 0 : lockCapture.unlock();
805 0 : }
806 0 : if (isFftsLaunchSubscribe_ && isTaskApiSubscribe_) {
807 0 : CHK_RET(hrtMsprofReportAdditionalInfo(0, &reportData, sizeof(MsprofAdditionalInfo)));
808 : }
809 : }
810 0 : return HCCL_SUCCESS;
811 0 : }
812 :
813 1 : HcclResult ProfilingManager::CallMsprofReportAdditionInfoForEsUpdate(const EsUpdatePara ¶,
814 : ProfTaskType type)
815 : {
816 1 : HCCLReportData hcclReportData{};
817 1 : hcclReportData.ts = hrtMsprofSysCycleTime();
818 1 : std::string nameInfo = GetProfTaskOpName(type);
819 1 : hcclReportData.profInfo.itemId = hrtMsprofGetHashId(nameInfo.c_str(), nameInfo.length());
820 1 : std::string cclTag = std::to_string(para.tag);
821 1 : hcclReportData.profInfo.cclTag = hrtMsprofGetHashId(cclTag.c_str(), cclTag.length());
822 1 : hcclReportData.profInfo.groupName = static_cast<u64>(reinterpret_cast<uintptr_t>(para.groupName));
823 1 : hcclReportData.profInfo.localRank = INVALID_VALUE_RANKID;
824 1 : hcclReportData.profInfo.remoteRank = INVALID_VALUE_RANKID;
825 1 : hcclReportData.profInfo.rankSize = PARSE_RANK_SIZE(0);
826 1 : hcclReportData.profInfo.workFlowMode = static_cast<u32>(HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE);
827 1 : hcclReportData.profInfo.planeID = 0;
828 1 : hcclReportData.profInfo.notifyID = INVALID_U64;
829 1 : hcclReportData.profInfo.stage = 0;
830 1 : hcclReportData.profInfo.role = static_cast<uint32_t>(TaskRole::DST);
831 1 : hcclReportData.profInfo.durationEstimated = 0;
832 1 : hcclReportData.profInfo.srcAddr = static_cast<u64>(reinterpret_cast<uintptr_t>(para.srcAddr));
833 1 : hcclReportData.profInfo.dstAddr = static_cast<u64>(reinterpret_cast<uintptr_t>(para.dstAddr));
834 1 : hcclReportData.profInfo.dataSize = static_cast<u32>(para.dataSize);
835 1 : hcclReportData.profInfo.opType = 0;
836 1 : hcclReportData.profInfo.dataType = HCCL_DATA_TYPE_FP32;
837 1 : hcclReportData.profInfo.linkType = static_cast<u32>(LinkType::LINK_ONCHIP);
838 1 : hcclReportData.profInfo.transportType = static_cast<int32_t>(SimpleTaskType::RDMA);
839 1 : hcclReportData.profInfo.rdmaType = static_cast<u32>(RdmaType::RDMA_SEND_PAYLOAD);
840 :
841 1 : int32_t ret = CallMsprofReportEsAdditionInfo(static_cast<uint32_t>(ProfTaskType::TASK_HCCL_INFO),
842 : hcclReportData.ts,
843 : &hcclReportData.profInfo,
844 1 : sizeof(hcclReportData.profInfo));
845 1 : CHK_PRT_RET((ret != 0), HCCL_ERROR("[TaskProfiling] CallMsprofReportAdditionInfoForEsUpdate failed."), HCCL_E_PARA);
846 1 : return HCCL_SUCCESS;
847 1 : }
848 :
849 2 : HcclResult ProfilingManager::CallMsprofRegEsTaskTypeApi() const
850 : {
851 2 : if (!isTaskApiSubscribe_) {
852 0 : return HCCL_SUCCESS;
853 : }
854 :
855 2 : HCCL_INFO("[ProfilingManager][CallMsprofRegEsTaskTypeApi] ready to register task types");
856 :
857 : // new
858 2 : CHK_RET(RegEsTaskType(ProfTaskType::TASK_LOOKUP_RESPONSE_MEMCPY));
859 2 : CHK_RET(RegEsTaskType(ProfTaskType::TASK_LOOKUP_RESPONSE_ISEND));
860 2 : CHK_RET(RegEsTaskType(ProfTaskType::TASK_SHARE_MEMORY_ISEND_RECORD));
861 :
862 2 : CHK_RET(RegEsTaskType(ProfTaskType::TASK_ABORT_SELF));
863 2 : CHK_RET(RegEsTaskType(ProfTaskType::TASK_SERVICE_CANCEL));
864 2 : CHK_RET(RegEsTaskType(ProfTaskType::TASK_DESTROY_RESOURCE));
865 2 : CHK_RET(RegEsTaskType(ProfTaskType::TASK_EVENT_WAIT));
866 2 : CHK_RET(RegEsTaskType(ProfTaskType::TASK_ISET_LOOKUP_RESPONSE));
867 2 : CHK_RET(RegEsTaskType(ProfTaskType::TASK_WAIT_SOME));
868 2 : CHK_RET(RegEsTaskType(ProfTaskType::TASK_GET_LOOKUP_REQUEST));
869 2 : CHK_RET(RegEsTaskType(ProfTaskType::TASK_ISEND_UPDATE_RESPONSE));
870 2 : CHK_RET(RegEsTaskType(ProfTaskType::TASK_ISEND_LOOKUP_RESPONSE));
871 :
872 2 : CHK_RET(RegEsTaskType(ProfTaskType::TASK_HCCL_INFO));
873 2 : CHK_RET(RegEsTaskType(ProfTaskType::TASK_COLL_RECV_LOOKUP_REQUEST));
874 2 : CHK_RET(RegEsTaskType(ProfTaskType::TASK_COLL_RECV_UPDATE_REQUEST));
875 2 : CHK_RET(RegEsTaskType(ProfTaskType::TASK_REMOTE_UPDATE_SEND_REQUEST));
876 :
877 2 : CHK_RET(RegEsTaskType(ProfTaskType::TASK_KEY_DROP_DUPLICATES));
878 2 : CHK_RET(RegEsTaskType(ProfTaskType::TASK_SEND_KEYS));
879 2 : CHK_RET(RegEsTaskType(ProfTaskType::TASK_SEND_KEYS_RECORD));
880 2 : CHK_RET(RegEsTaskType(ProfTaskType::TASK_EVENT_WAIT_RECV_DONE));
881 2 : CHK_RET(RegEsTaskType(ProfTaskType::TASK_RESET_UNIQUE_HANDLE));
882 2 : CHK_RET(RegEsTaskType(ProfTaskType::TASK_EVENT_WAIT_SEND_DONE));
883 2 : CHK_RET(RegEsTaskType(ProfTaskType::TASK_RECV_VALUES));
884 2 : CHK_RET(RegEsTaskType(ProfTaskType::TASK_RECOVER_VALUE_AICORE));
885 2 : CHK_RET(RegEsTaskType(ProfTaskType::TASK_GATHER_FINISH));
886 :
887 2 : CHK_RET(RegEsTaskType(ProfTaskType::TASK_REMOTE_UPDATE_KEY_REDUCE));
888 2 : CHK_RET(RegEsTaskType(ProfTaskType::TASK_VALUE_CLEAR_AICORE));
889 2 : CHK_RET(RegEsTaskType(ProfTaskType::TASK_VALUE_REDUCE_SUM_AICORE));
890 2 : CHK_RET(RegEsTaskType(ProfTaskType::TASK_UPDATE_RESET_UNIQUE_HANDLE));
891 2 : CHK_RET(RegEsTaskType(ProfTaskType::TASK_NOTIFY_REMOTE_IMRECV_DONE_SIGNAL_KEY));
892 2 : CHK_RET(RegEsTaskType(ProfTaskType::TASK_NOTIFY_REMOTE_IMRECV_DONE_SIGNAL_VALUE));
893 2 : CHK_RET(RegEsTaskType(ProfTaskType::TASK_REMOTE_UPDATE_RECV_RESPONSE));
894 :
895 2 : CHK_RET(RegEsTaskType(ProfTaskType::TASK_BUILD_CS_TRANSPORT));
896 2 : CHK_RET(RegEsTaskType(ProfTaskType::TASK_UPDATE_ALG_GLOBAL_REDUCE));
897 2 : CHK_RET(RegEsTaskType(ProfTaskType::TASK_AIV));
898 :
899 2 : return HCCL_SUCCESS;
900 : }
901 :
902 2 : HcclResult ProfilingManager::CallMsprofRegTaskTypeApi() const
903 : {
904 2 : if (!isTaskApiSubscribe_) {
905 0 : return HCCL_SUCCESS;
906 : }
907 :
908 2 : const std::string taskType(GetProfTaskOpName(ProfTaskType::TASK_HCCL_INFO));
909 2 : CHK_RET(hrtMsprofRegTypeInfo(
910 : MSPROF_REPORT_HCCL_NODE_LEVEL, static_cast<uint32_t>(ProfTaskType::TASK_HCCL_INFO), taskType.c_str()));
911 :
912 2 : const std::string multiThreadType(GetProfTaskOpName(ProfTaskType::TASK_MULTI_THREAD));
913 2 : CHK_RET(hrtMsprofRegTypeInfo(MSPROF_REPORT_HCCL_NODE_LEVEL,
914 : static_cast<uint32_t>(ProfTaskType::TASK_MULTI_THREAD),
915 : multiThreadType.c_str()));
916 :
917 2 : const std::string ctxIdInfo("context_id_info");
918 2 : CHK_RET(hrtMsprofRegTypeInfo(MSPROF_REPORT_HCCL_NODE_LEVEL,
919 : MSPROF_REPORT_NODE_CONTEXT_ID_INFO_TYPE,
920 : ctxIdInfo.c_str()));
921 :
922 2 : const std::string type("node_basic_info");
923 2 : CHK_RET(hrtMsprofRegTypeInfo(MSPROF_REPORT_NODE_LEVEL, MSPROF_REPORT_NODE_BASIC_INFO_TYPE, type.c_str()));
924 :
925 2 : const std::string mc2Type("mc2_comm_info");
926 2 : CHK_RET(hrtMsprofRegTypeInfo(MSPROF_REPORT_NODE_LEVEL, MSPROF_REPORT_NODE_MC2_COMMINFO_TYPE, mc2Type.c_str()));
927 2 : return HCCL_SUCCESS;
928 2 : }
929 :
930 0 : void ProfilingManager::SetFftsDispatcherMode()
931 : {
932 0 : isFftsDispatcher_.store(true);
933 0 : }
934 :
935 0 : void ProfilingManager::ReSetFftsDispatcherMode()
936 : {
937 0 : isFftsDispatcher_.store(false);
938 0 : }
939 :
940 330 : void ProfilingManager::SetThreadCaptureStatus(s32 threadID, bool isCapture)
941 : {
942 330 : HCCL_DEBUG("[SetThreadCaptureStatus] threadID[%d], captureStatus[%d]", threadID, isCapture);
943 330 : std::unique_lock<std::mutex> lockMap(captureStatusMapMutex_);
944 330 : captureStatusThreadIDMap_.insert(std::make_pair(threadID, isCapture));
945 330 : }
946 :
947 118 : bool ProfilingManager::GetThreadCaptureStatus()
948 : {
949 : // 返回当前线程的capture状态
950 : DevType devType;
951 118 : CHK_RET(hrtGetDeviceType(devType));
952 118 : if (devType == DevType::DEV_TYPE_310P1 || devType == DevType::DEV_TYPE_310P3) {
953 0 : return false;
954 : }
955 :
956 118 : s32 threadID = SalGetTid();
957 118 : std::unique_lock<std::mutex> lockMap(captureStatusMapMutex_);
958 118 : if (captureStatusThreadIDMap_.count(threadID) == 0) {
959 27 : HCCL_DEBUG("[GetThreadCaptureStatus] threadID[%d] not in map", threadID);\
960 27 : return false;
961 : } else {
962 91 : return captureStatusThreadIDMap_[threadID];
963 : }
964 118 : }
965 :
966 302 : void ProfilingManager::DeleteThreadCaptureStatus(s32 threadID)
967 : {
968 302 : std::unique_lock<std::mutex> lockMap(captureStatusMapMutex_);
969 302 : captureStatusThreadIDMap_.erase(threadID);
970 302 : }
971 :
972 : } // namespace hccl
|
