Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "profiling_manager.h"
12 : #include <string>
13 :
14 : #include "adapter_prof.h"
15 : #include "adapter_rts_common.h"
16 : #include "workflow_pub.h"
17 : #include "sal_pub.h"
18 :
19 : namespace hccl {
20 : std::queue<MsprofApi> ProfilingManager::storageTaskApi_;
21 : std::queue<MsprofApi> ProfilingManager::storageOpApi_;
22 : std::array<std::queue<MsprofAdditionalInfo>, MAX_MODULE_DEVICE_NUM> ProfilingManager::storageAdditionInfo_;
23 : std::array<std::mutex, MAX_MODULE_DEVICE_NUM> ProfilingManager::reportAddInfoMutex_;
24 : std::array<std::queue<MsprofCompactInfo>, MAX_MODULE_DEVICE_NUM> ProfilingManager::storageCompactInfo_;
25 : std::array<std::mutex, MAX_MODULE_DEVICE_NUM> ProfilingManager::reportCompactInfoMutex_;
26 : std::array<std::queue<MsprofAdditionalInfo>, MAX_MODULE_DEVICE_NUM> ProfilingManager::storageAdditionInfoFftsCapture_;
27 : std::array<std::mutex, MAX_MODULE_DEVICE_NUM> ProfilingManager::reportAddInfoFftsCaptureMutex_;
28 : std::mutex ProfilingManager::reportDataQueueMutex_;
29 : std::unordered_map<s32, bool> ProfilingManager::captureStatusThreadIDMap_;
30 : std::mutex ProfilingManager::captureStatusMapMutex_;
31 : std::mutex ProfilingManager::reportDataOpQueueMutex_;
32 :
33 7 : ProfilingManager::ProfilingManager()
34 7 : : reporterCallback_(nullptr),
35 7 : isHostApiSubscribe_(false),
36 7 : isTaskApiSubscribe_(false),
37 7 : isAdditionInfoSubscribe_(false)
38 7 : {}
39 :
40 7 : ProfilingManager::~ProfilingManager() {}
41 :
42 1625 : ProfilingManager& ProfilingManager::Instance()
43 : {
44 1625 : static ProfilingManager profilingManager;
45 1625 : return profilingManager;
46 : }
47 :
48 0 : Prof_Status ProfilingManager::CallMsprofReport(ReporterData& reporterData) const
49 : {
50 0 : CHK_PRT_RET(
51 : (reporterCallback_ == nullptr),
52 : HCCL_ERROR("[ProfilingManager][CallMsprofReport] MsprofReporterCallback callback is nullptr."), FAILED);
53 0 : return reporterCallback_(
54 : static_cast<uint32_t>(MsprofReporterModuleId::MSPROF_MODULE_HCCL),
55 : static_cast<uint32_t>(MsprofReporterCallbackType::MSPROF_REPORTER_REPORT), static_cast<void*>(&reporterData),
56 0 : sizeof(ReporterData));
57 : }
58 :
59 2 : HcclResult ProfilingManager::CallMsprofRegFftsLaunch() const
60 : {
61 2 : if (!isFftsLaunchSubscribe_) {
62 0 : return HCCL_SUCCESS;
63 : }
64 :
65 2 : CHK_RET(hrtMsprofRegTypeInfo(
66 : MSPROF_REPORT_HCCL_NODE_LEVEL, MSPROF_REPORT_NODE_CONTEXT_ID_INFO_TYPE, "context_id_info"));
67 :
68 2 : CHK_RET(hrtMsprofRegTypeInfo(MSPROF_REPORT_NODE_LEVEL, MSPROF_REPORT_NODE_MC2_COMMINFO_TYPE, "mc2_comm_info"));
69 :
70 2 : return HCCL_SUCCESS;
71 : }
72 :
73 3 : HcclResult ProfilingManager::CallMsprofRegHcclOpApi() const
74 : {
75 3 : if (!isHostHcclOpSubscribe_ || GetExternalInputHcclAicpuUnfold() || GetExternalInputHcclEnableFfts()) {
76 3 : return HCCL_SUCCESS;
77 : }
78 :
79 0 : for (const auto& name_to_type : PROF_OP_NAME) {
80 0 : CHK_RET(hrtMsprofRegTypeInfo(
81 : MSPROF_REPORT_HCCL_NODE_LEVEL,
82 : static_cast<uint32_t>(name_to_type.first) + MSPROF_REPORT_ACL_HOST_HCCL_BASE_TYPE,
83 : name_to_type.second.c_str()));
84 : }
85 :
86 0 : CHK_RET(hrtMsprofRegTypeInfo(MSPROF_REPORT_NODE_LEVEL, MSPROF_REPORT_NODE_MC2_COMMINFO_TYPE, "mc2_comm_info"));
87 :
88 0 : return HCCL_SUCCESS;
89 : }
90 :
91 0 : HcclResult ProfilingManager::CallMsprofRegHostApi() const
92 : {
93 0 : if (!isHostApiSubscribe_) {
94 0 : return HCCL_SUCCESS;
95 : }
96 0 : for (const auto& name_to_type : PROF_OP_NAME) {
97 0 : CHK_RET(hrtMsprofRegTypeInfo(
98 : MSPROF_REPORT_ACL_LEVEL, static_cast<uint32_t>(name_to_type.first) + MSPROF_REPORT_ACL_HOST_HCCL_BASE_TYPE,
99 : name_to_type.second.c_str()));
100 : }
101 :
102 0 : for (const auto& name_to_type : PROF_OP_NAME) {
103 0 : CHK_RET(hrtMsprofRegTypeInfo(
104 : MSPROF_REPORT_NODE_LEVEL, static_cast<uint32_t>(name_to_type.first) + MSPROF_REPORT_NODE_HCCL_BASE_TYPE,
105 : name_to_type.second.c_str()));
106 : }
107 :
108 0 : const std::string hcclType("hccl_op_info");
109 0 : CHK_RET(hrtMsprofRegTypeInfo(MSPROF_REPORT_NODE_LEVEL, MSPROF_REPORT_NODE_HCCL_OP_INFO_TYPE, hcclType.c_str()));
110 0 : return HCCL_SUCCESS;
111 0 : }
112 :
113 14 : HcclResult ProfilingManager::CallMsprofReportHostNodeApi(
114 : uint64_t beginTime, uint64_t endTime, uint64_t itemId, uint32_t threadId) const
115 : {
116 14 : MsprofApi reporterData{};
117 14 : reporterData.level = MSPROF_REPORT_NODE_LEVEL;
118 14 : reporterData.type = MSPROF_REPORT_NODE_LAUNCH_TYPE;
119 14 : reporterData.threadId = threadId;
120 14 : reporterData.beginTime = beginTime;
121 14 : reporterData.endTime = endTime;
122 14 : reporterData.itemId = itemId;
123 :
124 : // 静态图场景或者acl graph场景, 一次下发，多次执行; 缓存对应数据
125 14 : auto mode = GetWorkflowMode();
126 14 : if ((mode == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OPS_KERNEL_INFO_LIB) || GetThreadCaptureStatus()) {
127 14 : HCCL_INFO("CallMsprofReportTaskApi, storageTaskApi");
128 14 : std::unique_lock<std::mutex> lock(reportDataQueueMutex_);
129 14 : storageTaskApi_.push(reporterData);
130 14 : return HCCL_SUCCESS;
131 14 : }
132 :
133 0 : HCCL_INFO("CallMsprofReportHostNodeApi, HostNodeApiType[%u]", MSPROF_REPORT_NODE_LAUNCH_TYPE);
134 0 : CHK_RET(hrtMsprofReportApi(1, &reporterData));
135 0 : return HCCL_SUCCESS;
136 : }
137 :
138 3 : HcclResult ProfilingManager::CallMsprofReportNodeInfo(
139 : uint64_t beginTime, uint64_t endTime, const std::string profName, uint32_t threadId)
140 : {
141 3 : uint64_t itemId = hrtMsprofGetHashId(profName.c_str(), profName.length());
142 3 : auto mode = GetWorkflowMode();
143 : // hostapi开关 1) 开启: 单算子、静态图模式均上报; 关闭: 静态图模式或者acl graph场景缓存, 单算子不上报
144 3 : if (isHostApiSubscribe_ || mode == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OPS_KERNEL_INFO_LIB
145 6 : || GetThreadCaptureStatus()) {
146 1 : CHK_RET(CallMsprofReportHostNodeApi(beginTime, endTime, itemId, threadId));
147 : }
148 : // additionInfo开关 1) 开启: 单算子、静态图模式均上报; 关闭: 静态图或者acl graph场景模式缓存, 单算子不上报
149 3 : if (isAdditionInfoSubscribe_ || mode == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OPS_KERNEL_INFO_LIB
150 6 : || GetThreadCaptureStatus()) {
151 1 : CHK_RET(CallMsprofReportHostNodeBasicInfo(endTime, itemId, threadId));
152 : }
153 3 : return HCCL_SUCCESS;
154 : }
155 :
156 13 : HcclResult ProfilingManager::CallMsprofReportHostApi(
157 : HcclCMDType cmdType, uint64_t beginTime, u64 count, HcclDataType dataType, AlgType algType, uint64_t groupName,
158 : u32 numBlocks) const
159 : {
160 13 : if (!isHostApiSubscribe_ && GetWorkflowMode() == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE
161 26 : && !GetThreadCaptureStatus()) {
162 0 : return HCCL_SUCCESS;
163 : }
164 13 : uint64_t endTime = hrtMsprofSysCycleTime();
165 13 : uint32_t threadId = SalGetTid();
166 13 : uint32_t type = static_cast<int32_t>(cmdType);
167 13 : const std::string profName(GetProfOpName(cmdType));
168 13 : uint64_t itemId = hrtMsprofGetHashId(profName.c_str(), profName.length());
169 13 : if (GetWorkflowMode() == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE && IsLaunchKernelMode() != true) {
170 13 : CHK_RET(CallMsprofReportHostAclApi(type, beginTime, endTime, itemId, threadId));
171 13 : CHK_RET(CallMsprofReportHostNodeApi(beginTime, endTime, itemId, threadId));
172 13 : if (isAdditionInfoSubscribe_ || GetThreadCaptureStatus()) {
173 13 : CHK_RET(CallMsprofReportHostNodeBasicInfo(endTime, itemId, threadId, numBlocks));
174 : }
175 : }
176 13 : std::string algTypeStr = TransferAlgType(algType);
177 13 : CHK_RET(CallMsprofReportHostHcclOpInfo(endTime, threadId, count, dataType, algTypeStr, groupName));
178 13 : CHK_RET(CallMsprofReportHostHcclOpApi(beginTime, endTime, itemId, threadId));
179 13 : return HCCL_SUCCESS;
180 13 : }
181 :
182 70 : inline HcclResult ProfilingManager::RegEsTaskType(ProfTaskType taskType) const
183 : {
184 70 : const std::string str(GetProfTaskOpName(taskType));
185 70 : CHK_RET(hrtMsprofRegTypeInfo(MSPROF_REPORT_HCCL_NODE_LEVEL, static_cast<uint32_t>(taskType), str.c_str()));
186 70 : return HCCL_SUCCESS;
187 70 : }
188 :
189 : HcclResult
190 46 : ProfilingManager::ReportTaskApi(bool isMainStrem, uint64_t beginTime, ProfTaskType taskType, uint32_t agingFlag) const
191 : {
192 46 : HcclWorkflowMode mode = GetWorkflowMode();
193 : // 1、单算子场景，如果订阅开关没有开，直接退出
194 : // 2、l0 l1 级别时, 子图 launch都需要上报
195 46 : if ((!isTaskApiSubscribe_) && (mode == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE)
196 22 : && (taskType != ProfTaskType::TASK_LAUNCH_FFTS_TASK) && (taskType != ProfTaskType::TASK_AIV)
197 92 : && !GetThreadCaptureStatus()) {
198 17 : return HCCL_SUCCESS;
199 : }
200 29 : MsprofApi reporterData{};
201 29 : reporterData.level = MSPROF_REPORT_HCCL_NODE_LEVEL;
202 29 : reporterData.type = (isMainStrem == true) ? MSPROF_REPORT_HCCL_MASTER_TYPE : MSPROF_REPORT_HCCL_SLAVE_TYPE;
203 29 : reporterData.threadId = SalGetTid();
204 29 : reporterData.beginTime = beginTime;
205 29 : reporterData.endTime = hrtMsprofSysCycleTime();
206 29 : const std::string taskName(GetProfTaskOpName(taskType));
207 29 : reporterData.itemId = hrtMsprofGetHashId(taskName.c_str(), taskName.length());
208 :
209 : // 2、图下沉场景或者acl graph场景，缓存对应数据
210 29 : if ((mode == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OPS_KERNEL_INFO_LIB) || GetThreadCaptureStatus()) {
211 : // 缓存对应数据
212 24 : HCCL_INFO("CallMsprofReportTaskApi, storageTaskApi");
213 24 : std::unique_lock<std::mutex> lock(reportDataQueueMutex_);
214 24 : storageTaskApi_.push(reporterData);
215 24 : return HCCL_SUCCESS;
216 24 : }
217 :
218 5 : HCCL_INFO(
219 : "CallMsprofReportTaskApi, isMainStrem[%u], taskType[%d], taskName[%s]", isMainStrem,
220 : static_cast<int32_t>(taskType), taskName.c_str());
221 5 : CHK_RET(hrtMsprofReportApi(agingFlag, &reporterData));
222 5 : return HCCL_SUCCESS;
223 29 : }
224 :
225 13 : HcclResult ProfilingManager::CallMsprofReportHostAclApi(
226 : uint32_t type, uint64_t beginTime, uint64_t endTime, uint64_t itemId, uint32_t threadId) const
227 : {
228 13 : if (!isHostApiSubscribe_) {
229 13 : return HCCL_SUCCESS;
230 : }
231 0 : MsprofApi reporterData{};
232 0 : reporterData.level = MSPROF_REPORT_ACL_LEVEL;
233 0 : reporterData.type = static_cast<int32_t>(type) + MSPROF_REPORT_ACL_HOST_HCCL_BASE_TYPE;
234 0 : reporterData.threadId = threadId;
235 0 : reporterData.beginTime = beginTime;
236 0 : reporterData.endTime = endTime;
237 0 : reporterData.itemId = itemId;
238 :
239 0 : HCCL_INFO("CallMsprofReportHostHcclOpApi, HcclOpApiType[%u]", reporterData.type);
240 0 : CHK_RET(hrtMsprofReportApi(1, &reporterData));
241 0 : return HCCL_SUCCESS;
242 : }
243 :
244 14 : HcclResult ProfilingManager::CallMsprofReportHostNodeBasicInfo(
245 : uint64_t timeStamp, uint64_t itemId, uint32_t threadId, u32 numBlocks) const
246 : {
247 14 : MsprofCompactInfo reporterData{};
248 :
249 14 : reporterData.level = MSPROF_REPORT_NODE_LEVEL;
250 14 : reporterData.type = MSPROF_REPORT_NODE_BASIC_INFO_TYPE;
251 14 : reporterData.threadId = threadId;
252 14 : reporterData.dataLen = sizeof(MsprofNodeBasicInfo);
253 14 : reporterData.timeStamp = timeStamp;
254 :
255 14 : reporterData.data.nodeBasicInfo.opName = itemId;
256 14 : reporterData.data.nodeBasicInfo.taskType = MSPROF_GE_TASK_TYPE_HCCL;
257 14 : reporterData.data.nodeBasicInfo.opType = itemId;
258 14 : reporterData.data.nodeBasicInfo.blockDim = numBlocks;
259 14 : reporterData.data.nodeBasicInfo.opFlag = 0;
260 :
261 : // 图下沉场景或者acl graph场景，缓存对应数据
262 14 : auto mode = GetWorkflowMode();
263 14 : if ((mode == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OPS_KERNEL_INFO_LIB) || GetThreadCaptureStatus()) {
264 14 : s32 deviceLogicId = -1;
265 28 : CHK_RET(hrtGetDevice(&deviceLogicId));
266 14 : HCCL_INFO(
267 : "CallMsprofReportHostNodeBasicInfo, storageCompactInfo, The used deviceLogicId is [%d]", deviceLogicId);
268 : u32 maxDeviceNum;
269 14 : CHK_RET(GetMaxDevNum(maxDeviceNum));
270 14 : CHK_PRT_RET(
271 : static_cast<u32>(deviceLogicId) >= maxDeviceNum,
272 : HCCL_ERROR(
273 : "[ReportHostNodeBasicInfo]deviceLogicId_[%u] is bigger than maxDeviceNum[%u]",
274 : static_cast<u32>(deviceLogicId), maxDeviceNum),
275 : HCCL_E_INTERNAL);
276 14 : std::unique_lock<std::mutex> lock(reportCompactInfoMutex_[deviceLogicId]);
277 14 : storageCompactInfo_[deviceLogicId].push(reporterData);
278 14 : if (!isAdditionInfoSubscribe_) {
279 14 : return HCCL_SUCCESS;
280 : }
281 14 : }
282 0 : HCCL_INFO("CallMsprofReportHostNodeBasicInfo, HostNodeBasicInfoType[%u]", MSPROF_REPORT_NODE_BASIC_INFO_TYPE);
283 0 : CHK_RET(hrtMsprofReportCompactInfo(1, &reporterData, sizeof(MsprofCompactInfo)));
284 0 : return HCCL_SUCCESS;
285 : }
286 :
287 13 : HcclResult ProfilingManager::CallMsprofReportHostHcclOpApi(
288 : uint64_t beginTime, uint64_t endTime, uint64_t itemId, uint32_t threadId) const
289 : {
290 13 : auto mode = GetWorkflowMode();
291 0 : if ((!isHostHcclOpSubscribe_ || GetExternalInputHcclAicpuUnfold() || GetExternalInputHcclEnableFfts())
292 13 : && (mode == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE) && !GetThreadCaptureStatus()) {
293 0 : return HCCL_SUCCESS;
294 : }
295 :
296 13 : MsprofApi reporterData{};
297 13 : reporterData.level = MSPROF_REPORT_HCCL_NODE_LEVEL;
298 : // 集合通信算子粒度的都是主流
299 13 : reporterData.type = MSPROF_REPORT_HCCL_MASTER_TYPE;
300 13 : reporterData.threadId = threadId;
301 13 : reporterData.beginTime = beginTime;
302 13 : reporterData.endTime = endTime;
303 13 : reporterData.itemId = itemId;
304 :
305 : // 静态图场景或者acl graph场景, 一次下发，多次执行; 缓存对应数据
306 13 : if ((mode == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OPS_KERNEL_INFO_LIB) || GetThreadCaptureStatus()) {
307 13 : HCCL_INFO("CallMsprofReportOpApi, storageOpApi");
308 13 : std::unique_lock<std::mutex> lock(reportDataOpQueueMutex_);
309 13 : storageOpApi_.push(reporterData);
310 13 : return HCCL_SUCCESS;
311 13 : }
312 :
313 0 : HCCL_INFO("CallMsprofReportHostHcclOpApi, HcclOpApiType[%u]", MSPROF_REPORT_HCCL_MASTER_TYPE);
314 0 : CHK_RET(hrtMsprofReportApi(1, &reporterData));
315 0 : return HCCL_SUCCESS;
316 : }
317 :
318 46 : HcclResult ProfilingManager::CallMsprofReportTaskApi(bool isMainStrem, uint64_t beginTime, ProfTaskType taskType) const
319 : {
320 46 : uint32_t agingFlag = 0;
321 46 : if (GetWorkflowMode() == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE) {
322 22 : agingFlag = aging;
323 : }
324 46 : CHK_RET(ReportTaskApi(isMainStrem, beginTime, taskType, agingFlag));
325 :
326 46 : return HCCL_SUCCESS;
327 : }
328 :
329 13 : HcclResult ProfilingManager::CallMsprofReportHostHcclOpInfo(
330 : uint64_t timeStamp, uint32_t threadId, u64 count, HcclDataType dataType, std::string& algTypeStr,
331 : uint64_t groupName) const
332 : {
333 13 : MsprofCompactInfo reporterData{};
334 :
335 13 : reporterData.level = MSPROF_REPORT_NODE_LEVEL;
336 13 : reporterData.type = MSPROF_REPORT_NODE_HCCL_OP_INFO_TYPE;
337 13 : reporterData.threadId = threadId;
338 13 : reporterData.dataLen = sizeof(MsprofHCCLOPInfo);
339 13 : reporterData.timeStamp = timeStamp;
340 :
341 13 : reporterData.data.hcclopInfo.relay = 0;
342 13 : reporterData.data.hcclopInfo.retry = 0;
343 13 : reporterData.data.hcclopInfo.dataType = dataType;
344 13 : reporterData.data.hcclopInfo.algType = hrtMsprofGetHashId(algTypeStr.c_str(), algTypeStr.length());
345 13 : reporterData.data.hcclopInfo.count = count;
346 13 : reporterData.data.hcclopInfo.groupName = groupName;
347 :
348 : // 图下沉场景或者acl graph场景，缓存对应数据
349 13 : if ((GetWorkflowMode() == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OPS_KERNEL_INFO_LIB) || GetThreadCaptureStatus()) {
350 : // 缓存对应数据
351 13 : s32 deviceLogicId = -1;
352 26 : CHK_RET(hrtGetDevice(&deviceLogicId));
353 13 : HCCL_INFO("CallMsprofReportHostHcclOpInfo, storageCompactInfo, The used deviceLogicId is [%d]", deviceLogicId);
354 : u32 maxDeviceNum;
355 13 : CHK_RET(GetMaxDevNum(maxDeviceNum));
356 13 : CHK_PRT_RET(
357 : static_cast<u32>(deviceLogicId) >= maxDeviceNum,
358 : HCCL_ERROR(
359 : "[ReportAdditionInfo]deviceLogicId_[%u] is bigger than maxDeviceNum[%u]",
360 : static_cast<u32>(deviceLogicId), maxDeviceNum),
361 : HCCL_E_INTERNAL);
362 13 : std::unique_lock<std::mutex> lock(reportCompactInfoMutex_[deviceLogicId]);
363 13 : storageCompactInfo_[deviceLogicId].push(reporterData);
364 13 : if (!isHostApiSubscribe_) {
365 13 : return HCCL_SUCCESS;
366 : }
367 13 : }
368 :
369 0 : HCCL_INFO("CallMsprofReportHostHcclOpInfo, hcclopInfoType[%u]", MSPROF_REPORT_NODE_HCCL_OP_INFO_TYPE);
370 0 : CHK_RET(hrtMsprofReportCompactInfo(1, &reporterData, sizeof(MsprofCompactInfo)));
371 0 : return HCCL_SUCCESS;
372 : }
373 :
374 1 : HcclResult ProfilingManager::CallMsprofReportMc2CommInfo(uint64_t timeStamp, const void* data, int len)
375 : {
376 1 : uint32_t agingFlag = 0;
377 1 : auto mode = GetWorkflowMode();
378 1 : if (mode == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE) {
379 1 : agingFlag = 1;
380 : }
381 :
382 1 : MsprofAdditionalInfo reporterData{};
383 1 : reporterData.level = MSPROF_REPORT_NODE_LEVEL;
384 1 : reporterData.type = MSPROF_REPORT_NODE_MC2_COMMINFO_TYPE;
385 1 : reporterData.threadId = SalGetTid();
386 1 : reporterData.dataLen = len;
387 1 : reporterData.timeStamp = timeStamp;
388 :
389 1 : s32 sret = memcpy_s(reporterData.data, MSPROF_ADDTIONAL_INFO_DATA_LENGTH, data, len);
390 1 : CHK_PRT_RET(sret != EOK, HCCL_ERROR("memcpy failed. errorno[%d]:", sret), HCCL_E_MEMORY);
391 :
392 1 : if (!isHostApiSubscribe_ || GetThreadCaptureStatus()) {
393 : // 缓存对应数据
394 1 : s32 deviceLogicId = -1;
395 1 : CHK_RET(hrtGetDevice(&deviceLogicId));
396 1 : HCCL_INFO("CallMsprofReportAdditionInfo, storageAdditionInfo, The used deviceLogicId is [%d]", deviceLogicId);
397 : u32 maxDeviceNum;
398 1 : CHK_RET(GetMaxDevNum(maxDeviceNum));
399 1 : CHK_PRT_RET(
400 : static_cast<u32>(deviceLogicId) >= maxDeviceNum,
401 : HCCL_ERROR(
402 : "[ReportAdditionInfo]deviceLogicId_[%u] is bigger than maxDeviceNum[%u]",
403 : static_cast<u32>(deviceLogicId), maxDeviceNum),
404 : HCCL_E_INTERNAL);
405 1 : std::unique_lock<std::mutex> lock(reportAddInfoMutex_[deviceLogicId]);
406 1 : storageAdditionInfo_[deviceLogicId].push(reporterData);
407 1 : return HCCL_SUCCESS;
408 1 : }
409 0 : HCCL_INFO("CallMsprofReportMc2CommInfo, Mc2CommInfoType[%u]", MSPROF_REPORT_NODE_MC2_COMMINFO_TYPE);
410 0 : CHK_RET(hrtMsprofReportAdditionalInfo(agingFlag, &reporterData, sizeof(MsprofAdditionalInfo)));
411 0 : return HCCL_SUCCESS;
412 : }
413 :
414 : HcclResult
415 0 : ProfilingManager::CallEsMsprofReportTaskApi(bool isMainStrem, uint64_t beginTime, ProfTaskType taskType) const
416 : {
417 0 : if (!isTaskApiSubscribe_) {
418 0 : return HCCL_SUCCESS;
419 : }
420 0 : MsprofApi reporterData{};
421 0 : reporterData.level = MSPROF_REPORT_HCCL_NODE_LEVEL;
422 0 : reporterData.type = (isMainStrem == true) ? MSPROF_REPORT_HCCL_MASTER_TYPE : MSPROF_REPORT_HCCL_SLAVE_TYPE;
423 0 : reporterData.threadId = SalGetTid();
424 0 : reporterData.beginTime = beginTime;
425 0 : reporterData.endTime = hrtMsprofSysCycleTime();
426 0 : const std::string taskName(GetProfTaskOpName(taskType));
427 0 : reporterData.itemId = hrtMsprofGetHashId(taskName.c_str(), taskName.length());
428 0 : HCCL_INFO(
429 : "ReportTaskApi, isMainStrem[%u], taskType[%d], taskName[%s]", isMainStrem, static_cast<int32_t>(taskType),
430 : taskName.c_str());
431 0 : CHK_RET(hrtMsprofReportApi(aging, &reporterData));
432 0 : return HCCL_SUCCESS;
433 0 : }
434 :
435 1 : HcclResult ProfilingManager::CallMsprofReportMultiThreadInfo(const std::vector<uint32_t>& tidInfo) const
436 : {
437 1 : if (!isTaskApiSubscribe_ && !GetThreadCaptureStatus()) {
438 1 : return HCCL_SUCCESS;
439 : }
440 :
441 : struct MsprofMultiThread threadInfo;
442 0 : uint64_t timeStamp = hrtMsprofSysCycleTime();
443 0 : uint32_t totalSize = tidInfo.size();
444 : uint32_t currentSize;
445 0 : uint32_t sendNum = totalSize / MSPROF_MULTI_THREAD_MAX_NUM + 1;
446 :
447 0 : for (uint32_t j = 0; j < sendNum; j++) {
448 0 : currentSize = totalSize - j * MSPROF_MULTI_THREAD_MAX_NUM;
449 0 : threadInfo.threadNum = currentSize > MSPROF_MULTI_THREAD_MAX_NUM ? MSPROF_MULTI_THREAD_MAX_NUM : currentSize;
450 0 : for (uint32_t i = 0; i < threadInfo.threadNum; i++) {
451 0 : threadInfo.threadId[i] = tidInfo[i + j * MSPROF_MULTI_THREAD_MAX_NUM];
452 : }
453 0 : HCCL_INFO("CallMsprofReportMultiThreadInfo");
454 0 : CHK_RET(CallMsprofReportAdditionInfo(
455 : static_cast<int32_t>(ProfTaskType::TASK_MULTI_THREAD), timeStamp, &threadInfo,
456 : sizeof(struct MsprofMultiThread)));
457 : }
458 :
459 0 : return HCCL_SUCCESS;
460 : }
461 :
462 0 : HcclResult ProfilingManager::CallMsprofReportContextIdInfo(u32 ctxIdMax) const
463 : {
464 : struct MsprofContextIdInfo ctxInfo;
465 0 : ctxInfo.ctxIdNum = 2; // 因HCCL ctxId连续，固定上报2个：开始：0； 结束：ctxIdMax
466 0 : ctxInfo.ctxIds[0] = 0;
467 0 : ctxInfo.ctxIds[1] = ctxIdMax;
468 :
469 0 : uint64_t timeStamp = hrtMsprofSysCycleTime();
470 0 : HCCL_INFO("CallMsprofReportContextIdInfo, ctxIdNum[%u]", ctxInfo.ctxIdNum);
471 0 : CHK_RET(CallMsprofReportAdditionInfo(
472 : MSPROF_REPORT_NODE_CONTEXT_ID_INFO_TYPE, timeStamp, &ctxInfo, sizeof(struct MsprofContextIdInfo)));
473 0 : return HCCL_SUCCESS;
474 : }
475 :
476 235 : HcclResult ProfilingManager::ClearStoragedProfilingInfo()
477 : {
478 235 : std::unique_lock<std::mutex> lockTaskApi(reportDataQueueMutex_);
479 235 : HCCL_INFO("[ClearStoragedProfilingInfo] taskApiQueueSize is [%u]", storageTaskApi_.size());
480 :
481 235 : std::queue<MsprofApi> emptyTaskApi;
482 235 : std::swap(storageTaskApi_, emptyTaskApi);
483 :
484 235 : std::unique_lock<std::mutex> lockOpApi(reportDataOpQueueMutex_);
485 235 : HCCL_INFO("[ClearStoragedProfilingInfo] opApiQueueSize is [%u]", storageOpApi_.size());
486 :
487 235 : std::queue<MsprofApi> emptyOpApi;
488 235 : std::swap(storageOpApi_, emptyOpApi);
489 :
490 235 : s32 deviceLogicId = -1;
491 235 : CHK_RET(hrtGetDevice(&deviceLogicId));
492 235 : HCCL_INFO(
493 : "[ClearStoragedAdditionInfo] The size of the storageAdditionInfo_[%d] is [%u]", deviceLogicId,
494 : storageAdditionInfo_[deviceLogicId].size());
495 : u32 maxDeviceNum;
496 235 : CHK_RET(GetMaxDevNum(maxDeviceNum));
497 235 : CHK_PRT_RET(
498 : static_cast<u32>(deviceLogicId) >= maxDeviceNum,
499 : HCCL_ERROR(
500 : "[ReportStoragedAdditionInfo]deviceLogicId_[%u] is bigger than maxDeviceNum[%u]",
501 : static_cast<u32>(deviceLogicId), maxDeviceNum),
502 : HCCL_E_INTERNAL);
503 235 : std::unique_lock<std::mutex> lockAddInfo(reportAddInfoMutex_[deviceLogicId]);
504 235 : std::queue<MsprofAdditionalInfo> emptyAddition;
505 235 : std::swap(storageAdditionInfo_[deviceLogicId], emptyAddition);
506 :
507 235 : HCCL_INFO(
508 : "[ClearStoragedCompactInfo] The size of the storageCompactInfo_[%d] is [%u]", deviceLogicId,
509 : storageCompactInfo_[deviceLogicId].size());
510 235 : std::unique_lock<std::mutex> lockCompactInfo(reportCompactInfoMutex_[deviceLogicId]);
511 235 : std::queue<MsprofCompactInfo> emptyCompactInfo;
512 235 : std::swap(storageCompactInfo_[deviceLogicId], emptyCompactInfo);
513 :
514 235 : HCCL_INFO(
515 : "[ClearStorageAdditionInfoFftsCapture_] The size of the storageAdditionInfoFftsCapture_[%d] is [%u]",
516 : deviceLogicId, storageAdditionInfoFftsCapture_[deviceLogicId].size());
517 235 : std::unique_lock<std::mutex> lockAddInfoCapture(reportAddInfoFftsCaptureMutex_[deviceLogicId]);
518 235 : std::queue<MsprofAdditionalInfo> emptyAdditionCapture;
519 235 : std::swap(storageAdditionInfoFftsCapture_[deviceLogicId], emptyAdditionCapture);
520 235 : return HCCL_SUCCESS;
521 235 : }
522 :
523 24 : HcclResult ProfilingManager::ReportAdditionInfo(
524 : uint32_t type, uint64_t timeStamp, const void* data, int len, uint32_t agingFlag) const
525 : {
526 24 : HcclWorkflowMode mode = GetWorkflowMode();
527 : // 1、单算子场景，如果订阅开关没有开, 且上报的不是contextID,直接退出;
528 24 : if ((!isAdditionInfoSubscribe_) && (mode == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE)
529 48 : && (type != MSPROF_REPORT_NODE_CONTEXT_ID_INFO_TYPE) && !GetThreadCaptureStatus()) {
530 0 : return HCCL_SUCCESS;
531 : }
532 24 : MsprofAdditionalInfo reporterData{};
533 :
534 24 : reporterData.level = MSPROF_REPORT_HCCL_NODE_LEVEL;
535 24 : reporterData.type = type;
536 24 : reporterData.threadId = SalGetTid();
537 24 : reporterData.dataLen = len;
538 24 : reporterData.timeStamp = timeStamp;
539 24 : s32 sret = memcpy_s(reporterData.data, sizeof(reporterData.data), data, len);
540 24 : CHK_PRT_RET(sret != EOK, HCCL_ERROR("memcpy failed. errorno[%d]:", sret), HCCL_E_MEMORY);
541 :
542 : // 2、图下沉场景或者acl graph场景，缓存对应数据
543 0 : if ((mode == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OPS_KERNEL_INFO_LIB) || GetThreadCaptureStatus()
544 24 : || (isFftsDispatcher_.load() && // 3、FFTS+下发场景，addition开关打开，缓存对应数据
545 0 : (mode == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE) && (isAdditionInfoSubscribe_))) {
546 : // 缓存对应数据
547 24 : s32 deviceLogicId = -1;
548 48 : CHK_RET(hrtGetDevice(&deviceLogicId));
549 24 : HCCL_INFO("CallMsprofReportAdditionInfo, storageAdditionInfo, The used deviceLogicId is [%d]", deviceLogicId);
550 : u32 maxDeviceNum;
551 24 : CHK_RET(GetMaxDevNum(maxDeviceNum));
552 24 : CHK_PRT_RET(
553 : static_cast<u32>(deviceLogicId) >= maxDeviceNum,
554 : HCCL_ERROR(
555 : "[ReportAdditionInfo]deviceLogicId_[%u] is bigger than maxDeviceNum[%u]",
556 : static_cast<u32>(deviceLogicId), maxDeviceNum),
557 : HCCL_E_INTERNAL);
558 24 : std::unique_lock<std::mutex> lock(reportAddInfoMutex_[deviceLogicId]);
559 24 : storageAdditionInfo_[deviceLogicId].push(reporterData);
560 24 : if (isFftsDispatcher_ || !isAdditionInfoSubscribe_) {
561 24 : return HCCL_SUCCESS;
562 : }
563 24 : }
564 :
565 : // 4、开关开启，非子图下发场景，直接上报对应数据
566 0 : HCCL_INFO("CallMsprofReportAdditionInfo, AdditionInfoType[%u]", type);
567 0 : CHK_RET(hrtMsprofReportAdditionalInfo(agingFlag, &reporterData, sizeof(MsprofAdditionalInfo)));
568 0 : return HCCL_SUCCESS;
569 : }
570 :
571 : HcclResult
572 24 : ProfilingManager::CallMsprofReportAdditionInfo(uint32_t type, uint64_t timeStamp, const void* data, int len) const
573 : {
574 24 : uint32_t agingFlag = 0;
575 24 : if (GetWorkflowMode() == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE) {
576 0 : agingFlag = 1;
577 : }
578 24 : CHK_RET(ReportAdditionInfo(type, timeStamp, data, len, agingFlag));
579 24 : return HCCL_SUCCESS;
580 : }
581 :
582 : HcclResult
583 1 : ProfilingManager::CallMsprofReportEsAdditionInfo(uint32_t type, uint64_t timeStamp, const void* data, int len) const
584 : {
585 1 : if (!isAdditionInfoSubscribe_) {
586 1 : return HCCL_SUCCESS;
587 : }
588 0 : MsprofAdditionalInfo reporterData{};
589 :
590 0 : reporterData.level = MSPROF_REPORT_HCCL_NODE_LEVEL;
591 0 : reporterData.type = type;
592 0 : reporterData.threadId = SalGetTid();
593 0 : reporterData.dataLen = len;
594 0 : reporterData.timeStamp = timeStamp;
595 :
596 0 : s32 sret = memcpy_s(reporterData.data, sizeof(reporterData.data), data, len);
597 0 : CHK_PRT_RET(sret != EOK, HCCL_ERROR("memcpy failed. errorno[%d]:", sret), HCCL_E_MEMORY);
598 0 : HCCL_INFO("CallMsprofReportAdditionInfo, AdditionInfoType[%u]", type);
599 0 : CHK_RET(hrtMsprofReportAdditionalInfo(aging, &reporterData, sizeof(MsprofAdditionalInfo)));
600 0 : return HCCL_SUCCESS;
601 : }
602 :
603 3 : Prof_Status ProfilingManager::PluginInit() const
604 : {
605 3 : CHK_PRT_RET(
606 : (reporterCallback_ == nullptr),
607 : HCCL_ERROR("[ProfilingManager][PluginInit] MsprofReporterCallback callback is nullptr."), FAILED);
608 :
609 0 : int32_t cb_ret = reporterCallback_(
610 : static_cast<uint32_t>(MsprofReporterModuleId::MSPROF_MODULE_HCCL),
611 : static_cast<uint32_t>(MsprofReporterCallbackType::MSPROF_REPORTER_INIT), nullptr, 0);
612 0 : CHK_PRT_RET(
613 : (cb_ret != MSPROF_ERROR_NONE), HCCL_ERROR("[ProfilingManager][PluginInit] Reporter init failed."), FAILED);
614 :
615 0 : HCCL_INFO("[ProfilingManager][PluginInit] Reporter init success.");
616 :
617 0 : return SUCCESS;
618 : }
619 :
620 2 : Prof_Status ProfilingManager::PluginUnInit() const
621 : {
622 2 : CHK_PRT_RET(
623 : (reporterCallback_ == nullptr),
624 : HCCL_ERROR("[ProfilingManager][PluginUnInit] MsprofReporterCallback callback is nullptr."), FAILED);
625 :
626 0 : int32_t cb_ret = reporterCallback_(
627 : static_cast<uint32_t>(MsprofReporterModuleId::MSPROF_MODULE_HCCL),
628 : static_cast<uint32_t>(MsprofReporterCallbackType::MSPROF_REPORTER_UNINIT), nullptr, 0);
629 0 : CHK_PRT_RET(
630 : (cb_ret != MSPROF_ERROR_NONE), HCCL_ERROR("[ProfilingManager][PluginUnInit] Profiling reporter uinit failed."),
631 : FAILED);
632 :
633 0 : HCCL_INFO("[ProfilingManager][PluginUnInit] Profiling reporter uinit success.");
634 :
635 0 : return SUCCESS;
636 : }
637 :
638 0 : HcclResult ProfilingManager::CallMsprofReportAdditionInfoForEsLookup(EsLoopUpPara& para, ProfTaskType type)
639 : {
640 0 : HCCL_INFO("Entry CallMsprofReportAdditionInfoForEsLookup");
641 0 : HCCLReportData hcclReportData{};
642 0 : hcclReportData.ts = hrtMsprofSysCycleTime();
643 0 : std::string nameInfo = GetProfTaskOpName(type);
644 0 : hcclReportData.profInfo.itemId = hrtMsprofGetHashId(nameInfo.c_str(), nameInfo.length());
645 0 : std::string cclTag = std::to_string(para.tag);
646 0 : hcclReportData.profInfo.cclTag = hrtMsprofGetHashId(cclTag.c_str(), cclTag.length());
647 0 : hcclReportData.profInfo.groupName = static_cast<u64>(reinterpret_cast<uintptr_t>("unknown"));
648 0 : hcclReportData.profInfo.rankSize = 0;
649 0 : hcclReportData.profInfo.workFlowMode = static_cast<u32>(HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE);
650 0 : hcclReportData.profInfo.planeID = 0;
651 0 : hcclReportData.profInfo.notifyID = 0;
652 0 : hcclReportData.profInfo.stage = 0;
653 0 : hcclReportData.profInfo.role = static_cast<uint32_t>(TaskRole::DST);
654 0 : hcclReportData.profInfo.durationEstimated = 0;
655 0 : hcclReportData.profInfo.srcAddr = static_cast<u64>(reinterpret_cast<uintptr_t>(para.srcAddr));
656 0 : hcclReportData.profInfo.dstAddr = static_cast<u64>(reinterpret_cast<uintptr_t>(para.dstAddr));
657 0 : hcclReportData.profInfo.dataSize = static_cast<u32>(para.dataSize);
658 0 : hcclReportData.profInfo.opType = 0;
659 0 : hcclReportData.profInfo.dataType = HCCL_DATA_TYPE_FP32;
660 0 : hcclReportData.profInfo.linkType = static_cast<u32>(LinkType::LINK_ONCHIP);
661 0 : hcclReportData.profInfo.transportType = static_cast<int32_t>(SimpleTaskType::RDMA);
662 :
663 0 : int32_t ret = CallMsprofReportEsAdditionInfo(
664 : static_cast<uint32_t>(ProfTaskType::TASK_HCCL_INFO), hcclReportData.ts, &hcclReportData.profInfo,
665 0 : sizeof(hcclReportData.profInfo));
666 0 : CHK_PRT_RET((ret != 0), HCCL_ERROR("[TaskProfiling] CallMsprofReportAdditionInfoForEsLookup failed."), HCCL_E_PARA);
667 0 : return HCCL_SUCCESS;
668 0 : }
669 :
670 0 : Prof_Status ProfilingManager::GetHashKey(MsprofHashData& data) const
671 : {
672 0 : CHK_PRT_RET(
673 : (reporterCallback_ == nullptr),
674 : HCCL_ERROR("[ProfilingManager][GetHashKey] MsprofReporterCallback callback is nullptr."), FAILED);
675 0 : if (data.dataLen == 0) {
676 0 : data.hashId = 0;
677 0 : HCCL_INFO("[Check][Param]PluginUnInit MsprofReporterCallback GetHashKey in default.");
678 : } else {
679 0 : int32_t cb_ret = reporterCallback_(
680 : static_cast<uint32_t>(MsprofReporterModuleId::MSPROF_MODULE_HCCL),
681 : static_cast<uint32_t>(MsprofReporterCallbackType::MSPROF_REPORTER_HASH), static_cast<void*>(&data),
682 : sizeof(MsprofHashData));
683 :
684 0 : CHK_PRT_RET(
685 : (cb_ret != MSPROF_ERROR_NONE),
686 : HCCL_ERROR("[ProfilingManager][GetHashKey] Profiling reporter GetHashKey failed."), FAILED);
687 :
688 0 : HCCL_INFO("[ProfilingManager][GetHashKey] Profiling reporter GetHashKey success.");
689 : }
690 :
691 0 : return SUCCESS;
692 : }
693 :
694 3 : HcclResult ProfilingManager::ReportStoragedTaskApi()
695 : {
696 3 : if (!isTaskApiSubscribe_ && !isHostApiSubscribe_) {
697 1 : return HCCL_SUCCESS;
698 : }
699 2 : std::unique_lock<std::mutex> lock(reportDataQueueMutex_);
700 2 : HCCL_INFO("[ReportStoragedTaskApi] taskApiQueueSize is [%u]", storageTaskApi_.size());
701 2 : if (!storageTaskApi_.empty()) {
702 2 : std::queue<MsprofApi> tempTaskApi = storageTaskApi_;
703 2 : lock.unlock();
704 50 : while (!tempTaskApi.empty()) {
705 48 : MsprofApi reportData = tempTaskApi.front();
706 48 : tempTaskApi.pop();
707 48 : CHK_RET(hrtMsprofReportApi(0, &reportData));
708 : }
709 2 : }
710 2 : return HCCL_SUCCESS;
711 2 : }
712 :
713 3 : HcclResult ProfilingManager::ReportStoragedOpApi()
714 : {
715 3 : if (!isHostHcclOpSubscribe_ || GetExternalInputHcclAicpuUnfold() || GetExternalInputHcclEnableFfts()) {
716 3 : return HCCL_SUCCESS;
717 : }
718 :
719 0 : std::unique_lock<std::mutex> lock(reportDataOpQueueMutex_);
720 0 : HCCL_INFO("[ReportStoragedOpApi] opApiQueueSize is [%u]", storageOpApi_.size());
721 0 : if (!storageOpApi_.empty()) {
722 0 : std::queue<MsprofApi> tempOpApi = storageOpApi_;
723 0 : lock.unlock();
724 0 : while (!tempOpApi.empty()) {
725 0 : MsprofApi reportData = tempOpApi.front();
726 0 : tempOpApi.pop();
727 0 : CHK_RET(hrtMsprofReportApi(0, &reportData));
728 : }
729 0 : }
730 0 : return HCCL_SUCCESS;
731 0 : }
732 :
733 2 : HcclResult ProfilingManager::ReportStoragedAdditionInfo()
734 : {
735 132 : for (u32 i = 0; i < MAX_MODULE_DEVICE_NUM; i++) {
736 130 : std::unique_lock<std::mutex> lock(reportAddInfoMutex_[i]);
737 130 : HCCL_INFO(
738 : "[ReportStoragedAdditionInfo] The size of the storageAdditionInfo_[%u] is [%u]", i,
739 : storageAdditionInfo_[i].size());
740 130 : if (!storageAdditionInfo_[i].empty()) {
741 2 : std::queue<MsprofAdditionalInfo> tempTaskAdditionalInfo = storageAdditionInfo_[i];
742 2 : lock.unlock();
743 50 : while (!tempTaskAdditionalInfo.empty()) {
744 48 : MsprofAdditionalInfo reportData = tempTaskAdditionalInfo.front();
745 48 : tempTaskAdditionalInfo.pop();
746 48 : CHK_RET(hrtMsprofReportAdditionalInfo(0, &reportData, sizeof(MsprofAdditionalInfo)));
747 : }
748 2 : }
749 : // acl graph ffts+场景下， 一次下发多次执行， 执行时上报保存的task信息
750 130 : std::unique_lock<std::mutex> lockCapture(reportAddInfoFftsCaptureMutex_[i]);
751 130 : HCCL_INFO(
752 : "[ReportStoragedAdditionInfo] The size of the storageAdditionInfoFftsCapture_[%u] is [%u]", i,
753 : storageAdditionInfoFftsCapture_[i].size());
754 130 : if (!storageAdditionInfoFftsCapture_[i].empty()) {
755 0 : std::queue<MsprofAdditionalInfo> tempTaskAdditionalInfo = storageAdditionInfoFftsCapture_[i];
756 0 : lockCapture.unlock();
757 0 : while (!tempTaskAdditionalInfo.empty()) {
758 0 : MsprofAdditionalInfo reportData = tempTaskAdditionalInfo.front();
759 0 : tempTaskAdditionalInfo.pop();
760 0 : CHK_RET(hrtMsprofReportAdditionalInfo(0, &reportData, sizeof(MsprofAdditionalInfo)));
761 : }
762 0 : }
763 130 : }
764 2 : return HCCL_SUCCESS;
765 : }
766 :
767 1 : HcclResult ProfilingManager::ReportStoragedCompactInfo()
768 : {
769 66 : for (u32 i = 0; i < MAX_MODULE_DEVICE_NUM; i++) {
770 65 : std::unique_lock<std::mutex> lock(reportCompactInfoMutex_[i]);
771 65 : HCCL_INFO(
772 : "[ReportStoragedCompactInfo] The size of the storageCompactInfo_[%u] is [%u]", i,
773 : storageCompactInfo_[i].size());
774 65 : if (!storageCompactInfo_[i].empty()) {
775 1 : std::queue<MsprofCompactInfo> tempCompactInfo = storageCompactInfo_[i];
776 1 : lock.unlock();
777 2 : while (!tempCompactInfo.empty()) {
778 1 : MsprofCompactInfo reportData = tempCompactInfo.front();
779 1 : tempCompactInfo.pop();
780 1 : CHK_RET(hrtMsprofReportCompactInfo(0, &reportData, sizeof(MsprofCompactInfo)));
781 : }
782 1 : }
783 65 : }
784 1 : return HCCL_SUCCESS;
785 : }
786 :
787 0 : HcclResult ProfilingManager::ReportStoragedFftsInfo()
788 : {
789 0 : uint64_t ts = hrtMsprofSysCycleTime();
790 :
791 0 : s32 deviceLogicId = -1;
792 0 : if (hrtGetDevice(&deviceLogicId) != HCCL_SUCCESS) {
793 0 : deviceLogicId = 0;
794 0 : HCCL_WARNING("[ReportStoragedAdditionInfo]deviceLogicId[%d]", deviceLogicId);
795 : }
796 : u32 maxDeviceNum;
797 0 : CHK_RET(GetMaxDevNum(maxDeviceNum));
798 0 : CHK_PRT_RET(
799 : static_cast<u32>(deviceLogicId) >= maxDeviceNum,
800 : HCCL_ERROR(
801 : "[ReportStoragedAdditionInfo]deviceLogicId_[%u] is bigger than maxDeviceNum[%u]",
802 : static_cast<u32>(deviceLogicId), maxDeviceNum),
803 : HCCL_E_INTERNAL);
804 0 : std::unique_lock<std::mutex> lock(reportAddInfoMutex_[deviceLogicId]);
805 0 : HCCL_INFO(
806 : "[ReportStoragedFftsInfo] The size of the storageAdditionInfo_[%d] is [%u] ", deviceLogicId,
807 : storageAdditionInfo_[deviceLogicId].size());
808 :
809 0 : while (!storageAdditionInfo_[deviceLogicId].empty()) {
810 0 : MsprofAdditionalInfo reportData = storageAdditionInfo_[deviceLogicId].front();
811 0 : storageAdditionInfo_[deviceLogicId].pop();
812 0 : reportData.timeStamp = ts;
813 0 : if (GetThreadCaptureStatus()) {
814 : // acl graph ffts+ 场景下， 下发的task信息进行保存以便后续多次使用
815 0 : std::unique_lock<std::mutex> lockCapture(reportAddInfoFftsCaptureMutex_[deviceLogicId]);
816 0 : storageAdditionInfoFftsCapture_[deviceLogicId].push(reportData);
817 0 : lockCapture.unlock();
818 0 : }
819 0 : if (isFftsLaunchSubscribe_ && isTaskApiSubscribe_) {
820 0 : CHK_RET(hrtMsprofReportAdditionalInfo(0, &reportData, sizeof(MsprofAdditionalInfo)));
821 : }
822 : }
823 0 : return HCCL_SUCCESS;
824 0 : }
825 :
826 1 : HcclResult ProfilingManager::CallMsprofReportAdditionInfoForEsUpdate(const EsUpdatePara& para, ProfTaskType type)
827 : {
828 1 : HCCLReportData hcclReportData{};
829 1 : hcclReportData.ts = hrtMsprofSysCycleTime();
830 1 : std::string nameInfo = GetProfTaskOpName(type);
831 1 : hcclReportData.profInfo.itemId = hrtMsprofGetHashId(nameInfo.c_str(), nameInfo.length());
832 1 : std::string cclTag = std::to_string(para.tag);
833 1 : hcclReportData.profInfo.cclTag = hrtMsprofGetHashId(cclTag.c_str(), cclTag.length());
834 1 : hcclReportData.profInfo.groupName = static_cast<u64>(reinterpret_cast<uintptr_t>(para.groupName));
835 1 : hcclReportData.profInfo.localRank = INVALID_VALUE_RANKID;
836 1 : hcclReportData.profInfo.remoteRank = INVALID_VALUE_RANKID;
837 1 : hcclReportData.profInfo.rankSize = PARSE_RANK_SIZE(0);
838 1 : hcclReportData.profInfo.workFlowMode = static_cast<u32>(HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE);
839 1 : hcclReportData.profInfo.planeID = 0;
840 1 : hcclReportData.profInfo.notifyID = INVALID_U64;
841 1 : hcclReportData.profInfo.stage = 0;
842 1 : hcclReportData.profInfo.role = static_cast<uint32_t>(TaskRole::DST);
843 1 : hcclReportData.profInfo.durationEstimated = 0;
844 1 : hcclReportData.profInfo.srcAddr = static_cast<u64>(reinterpret_cast<uintptr_t>(para.srcAddr));
845 1 : hcclReportData.profInfo.dstAddr = static_cast<u64>(reinterpret_cast<uintptr_t>(para.dstAddr));
846 1 : hcclReportData.profInfo.dataSize = static_cast<u32>(para.dataSize);
847 1 : hcclReportData.profInfo.opType = 0;
848 1 : hcclReportData.profInfo.dataType = HCCL_DATA_TYPE_FP32;
849 1 : hcclReportData.profInfo.linkType = static_cast<u32>(LinkType::LINK_ONCHIP);
850 1 : hcclReportData.profInfo.transportType = static_cast<int32_t>(SimpleTaskType::RDMA);
851 1 : hcclReportData.profInfo.rdmaType = static_cast<u32>(RdmaType::RDMA_SEND_PAYLOAD);
852 :
853 1 : int32_t ret = CallMsprofReportEsAdditionInfo(
854 : static_cast<uint32_t>(ProfTaskType::TASK_HCCL_INFO), hcclReportData.ts, &hcclReportData.profInfo,
855 1 : sizeof(hcclReportData.profInfo));
856 1 : CHK_PRT_RET((ret != 0), HCCL_ERROR("[TaskProfiling] CallMsprofReportAdditionInfoForEsUpdate failed."), HCCL_E_PARA);
857 1 : return HCCL_SUCCESS;
858 1 : }
859 :
860 2 : HcclResult ProfilingManager::CallMsprofRegEsTaskTypeApi() const
861 : {
862 2 : if (!isTaskApiSubscribe_) {
863 0 : return HCCL_SUCCESS;
864 : }
865 :
866 2 : HCCL_INFO("[ProfilingManager][CallMsprofRegEsTaskTypeApi] ready to register task types");
867 :
868 : // new
869 2 : CHK_RET(RegEsTaskType(ProfTaskType::TASK_LOOKUP_RESPONSE_MEMCPY));
870 2 : CHK_RET(RegEsTaskType(ProfTaskType::TASK_LOOKUP_RESPONSE_ISEND));
871 2 : CHK_RET(RegEsTaskType(ProfTaskType::TASK_SHARE_MEMORY_ISEND_RECORD));
872 :
873 2 : CHK_RET(RegEsTaskType(ProfTaskType::TASK_ABORT_SELF));
874 2 : CHK_RET(RegEsTaskType(ProfTaskType::TASK_SERVICE_CANCEL));
875 2 : CHK_RET(RegEsTaskType(ProfTaskType::TASK_DESTROY_RESOURCE));
876 2 : CHK_RET(RegEsTaskType(ProfTaskType::TASK_EVENT_WAIT));
877 2 : CHK_RET(RegEsTaskType(ProfTaskType::TASK_ISET_LOOKUP_RESPONSE));
878 2 : CHK_RET(RegEsTaskType(ProfTaskType::TASK_WAIT_SOME));
879 2 : CHK_RET(RegEsTaskType(ProfTaskType::TASK_GET_LOOKUP_REQUEST));
880 2 : CHK_RET(RegEsTaskType(ProfTaskType::TASK_ISEND_UPDATE_RESPONSE));
881 2 : CHK_RET(RegEsTaskType(ProfTaskType::TASK_ISEND_LOOKUP_RESPONSE));
882 :
883 2 : CHK_RET(RegEsTaskType(ProfTaskType::TASK_HCCL_INFO));
884 2 : CHK_RET(RegEsTaskType(ProfTaskType::TASK_COLL_RECV_LOOKUP_REQUEST));
885 2 : CHK_RET(RegEsTaskType(ProfTaskType::TASK_COLL_RECV_UPDATE_REQUEST));
886 2 : CHK_RET(RegEsTaskType(ProfTaskType::TASK_REMOTE_UPDATE_SEND_REQUEST));
887 :
888 2 : CHK_RET(RegEsTaskType(ProfTaskType::TASK_KEY_DROP_DUPLICATES));
889 2 : CHK_RET(RegEsTaskType(ProfTaskType::TASK_SEND_KEYS));
890 2 : CHK_RET(RegEsTaskType(ProfTaskType::TASK_SEND_KEYS_RECORD));
891 2 : CHK_RET(RegEsTaskType(ProfTaskType::TASK_EVENT_WAIT_RECV_DONE));
892 2 : CHK_RET(RegEsTaskType(ProfTaskType::TASK_RESET_UNIQUE_HANDLE));
893 2 : CHK_RET(RegEsTaskType(ProfTaskType::TASK_EVENT_WAIT_SEND_DONE));
894 2 : CHK_RET(RegEsTaskType(ProfTaskType::TASK_RECV_VALUES));
895 2 : CHK_RET(RegEsTaskType(ProfTaskType::TASK_RECOVER_VALUE_AICORE));
896 2 : CHK_RET(RegEsTaskType(ProfTaskType::TASK_GATHER_FINISH));
897 :
898 2 : CHK_RET(RegEsTaskType(ProfTaskType::TASK_REMOTE_UPDATE_KEY_REDUCE));
899 2 : CHK_RET(RegEsTaskType(ProfTaskType::TASK_VALUE_CLEAR_AICORE));
900 2 : CHK_RET(RegEsTaskType(ProfTaskType::TASK_VALUE_REDUCE_SUM_AICORE));
901 2 : CHK_RET(RegEsTaskType(ProfTaskType::TASK_UPDATE_RESET_UNIQUE_HANDLE));
902 2 : CHK_RET(RegEsTaskType(ProfTaskType::TASK_NOTIFY_REMOTE_IMRECV_DONE_SIGNAL_KEY));
903 2 : CHK_RET(RegEsTaskType(ProfTaskType::TASK_NOTIFY_REMOTE_IMRECV_DONE_SIGNAL_VALUE));
904 2 : CHK_RET(RegEsTaskType(ProfTaskType::TASK_REMOTE_UPDATE_RECV_RESPONSE));
905 :
906 2 : CHK_RET(RegEsTaskType(ProfTaskType::TASK_BUILD_CS_TRANSPORT));
907 2 : CHK_RET(RegEsTaskType(ProfTaskType::TASK_UPDATE_ALG_GLOBAL_REDUCE));
908 2 : CHK_RET(RegEsTaskType(ProfTaskType::TASK_AIV));
909 :
910 2 : return HCCL_SUCCESS;
911 : }
912 :
913 2 : HcclResult ProfilingManager::CallMsprofRegTaskTypeApi() const
914 : {
915 2 : if (!isTaskApiSubscribe_) {
916 0 : return HCCL_SUCCESS;
917 : }
918 :
919 2 : const std::string taskType(GetProfTaskOpName(ProfTaskType::TASK_HCCL_INFO));
920 2 : CHK_RET(hrtMsprofRegTypeInfo(
921 : MSPROF_REPORT_HCCL_NODE_LEVEL, static_cast<uint32_t>(ProfTaskType::TASK_HCCL_INFO), taskType.c_str()));
922 :
923 2 : const std::string multiThreadType(GetProfTaskOpName(ProfTaskType::TASK_MULTI_THREAD));
924 2 : CHK_RET(hrtMsprofRegTypeInfo(
925 : MSPROF_REPORT_HCCL_NODE_LEVEL, static_cast<uint32_t>(ProfTaskType::TASK_MULTI_THREAD),
926 : multiThreadType.c_str()));
927 :
928 2 : const std::string ctxIdInfo("context_id_info");
929 2 : CHK_RET(hrtMsprofRegTypeInfo(
930 : MSPROF_REPORT_HCCL_NODE_LEVEL, MSPROF_REPORT_NODE_CONTEXT_ID_INFO_TYPE, ctxIdInfo.c_str()));
931 :
932 2 : const std::string type("node_basic_info");
933 2 : CHK_RET(hrtMsprofRegTypeInfo(MSPROF_REPORT_NODE_LEVEL, MSPROF_REPORT_NODE_BASIC_INFO_TYPE, type.c_str()));
934 :
935 2 : const std::string mc2Type("mc2_comm_info");
936 2 : CHK_RET(hrtMsprofRegTypeInfo(MSPROF_REPORT_NODE_LEVEL, MSPROF_REPORT_NODE_MC2_COMMINFO_TYPE, mc2Type.c_str()));
937 2 : return HCCL_SUCCESS;
938 2 : }
939 :
940 0 : void ProfilingManager::SetFftsDispatcherMode() { isFftsDispatcher_.store(true); }
941 :
942 0 : void ProfilingManager::ReSetFftsDispatcherMode() { isFftsDispatcher_.store(false); }
943 :
944 330 : void ProfilingManager::SetThreadCaptureStatus(s32 threadID, bool isCapture)
945 : {
946 330 : HCCL_DEBUG("[SetThreadCaptureStatus] threadID[%d], captureStatus[%d]", threadID, isCapture);
947 330 : std::unique_lock<std::mutex> lockMap(captureStatusMapMutex_);
948 330 : captureStatusThreadIDMap_.insert(std::make_pair(threadID, isCapture));
949 330 : }
950 :
951 118 : bool ProfilingManager::GetThreadCaptureStatus()
952 : {
953 : // 返回当前线程的capture状态
954 : DevType devType;
955 118 : CHK_RET(hrtGetDeviceType(devType));
956 118 : if (devType == DevType::DEV_TYPE_310P1 || devType == DevType::DEV_TYPE_310P3) {
957 0 : return false;
958 : }
959 :
960 118 : s32 threadID = SalGetTid();
961 118 : std::unique_lock<std::mutex> lockMap(captureStatusMapMutex_);
962 118 : if (captureStatusThreadIDMap_.count(threadID) == 0) {
963 27 : HCCL_DEBUG("[GetThreadCaptureStatus] threadID[%d] not in map", threadID);
964 27 : return false;
965 : } else {
966 91 : return captureStatusThreadIDMap_[threadID];
967 : }
968 118 : }
969 :
970 302 : void ProfilingManager::DeleteThreadCaptureStatus(s32 threadID)
971 : {
972 302 : std::unique_lock<std::mutex> lockMap(captureStatusMapMutex_);
973 302 : captureStatusThreadIDMap_.erase(threadID);
974 302 : }
975 :
976 : } // namespace hccl
|
