Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "profiling_manager_device.h"
12 : #include "aicpu_schedule/aicpu_context.h"
13 : #include "prof_common.h"
14 : #include "log.h"
15 : #include "sal_pub.h"
16 : #include "common/aicpu_sqe_context.h"
17 : #include "common/aicpu_hccl_common.h"
18 : #include "dlprof_function.h"
19 :
20 : namespace {
21 : static constexpr u32 aging = 1;
22 : }
23 : namespace dfx {
24 :
25 : std::mutex ProfilingManager::streamMutex_;
26 : std::unordered_map<std::string, ProfCommInfo> ProfilingManager::tagOpInfoMap_;
27 : std::unordered_map<s32, std::string> ProfilingManager::streamToTagMap_;
28 :
29 : std::mutex ProfilingManager::startReportSqeIdxMutex_;
30 : std::unordered_map<s32, u32> ProfilingManager::streamToSqeIdxMap_;
31 :
32 : bool ProfilingManager::isL0Open_ = false;
33 : bool ProfilingManager::isL1Open_ = false;
34 :
35 : constexpr std::uint32_t HCCLINFO_REPORT_BATCH_NUM = 2;
36 :
37 46 : bool ProfilingManager::IsProfOn(uint64_t feature)
38 : {
39 46 : if (MsprofReportBatchAdditionalInfo == nullptr) {
40 46 : if (AdprofCheckFeatureIsOn == nullptr) {
41 0 : return false;
42 : }
43 46 : return AdprofCheckFeatureIsOn(feature) > 0;
44 : } else {
45 0 : if (feature == ADPROF_TASK_TIME_L1) {
46 0 : return isL1Open_;
47 0 : } else if (feature == ADPROF_TASK_TIME_L0) {
48 0 : return isL0Open_;
49 : }
50 : }
51 0 : return false;
52 : }
53 :
54 0 : bool ProfilingManager::IsL1fromOffToOn()
55 : {
56 0 : if (((!GetProfL1State()) && ProfilingManager::IsProfL1On())) {
57 0 : HCCL_INFO("Profiling L1 switch from off to on.");
58 0 : return true;
59 : }
60 0 : return false;
61 : }
62 :
63 23 : bool ProfilingManager::IsProfL1On()
64 : {
65 23 : if (IsProfOn(ADPROF_TASK_TIME_L1)) {
66 8 : isL1Open_ = true;
67 8 : return true;
68 : }
69 15 : isL1Open_ = false;
70 15 : return false;
71 : }
72 :
73 23 : bool ProfilingManager::IsProfL0On()
74 : {
75 23 : if (IsProfOn(ADPROF_TASK_TIME_L0)) {
76 8 : isL0Open_ = true;
77 8 : return true;
78 : }
79 15 : isL0Open_ = false;
80 15 : return false;
81 : }
82 :
83 0 : void ProfilingManager::SetProL1On(bool val)
84 : {
85 0 : HCCL_INFO("[%s] val = [%d]", __func__, val);
86 0 : isL1Open_ = val;
87 0 : }
88 :
89 0 : void ProfilingManager::SetProL0On(bool val)
90 : {
91 0 : HCCL_INFO("[%s] val = [%d]", __func__, val);
92 0 : isL0Open_ = val;
93 0 : }
94 :
95 424 : bool ProfilingManager::GetProfL0State()
96 : {
97 424 : if (!isL0Open_) {
98 381 : return false;
99 : }
100 43 : return true;
101 : }
102 :
103 30101750 : bool ProfilingManager::GetProfL1State()
104 : {
105 30101750 : if (!isL1Open_) {
106 30101627 : return false;
107 : }
108 123 : return true;
109 : }
110 :
111 1161 : HcclResult ProfilingManager::CallMsprofReportAdditionInfo(uint32_t type, uint64_t timeStamp, const void* data, int len)
112 : {
113 1161 : MsprofAdditionalInfo reporterData{};
114 1161 : reporterData.level = MSPROF_REPORT_AICPU_LEVEL;
115 1161 : reporterData.type = type;
116 1161 : reporterData.threadId = SalGetTid();
117 1161 : reporterData.dataLen = len;
118 1161 : reporterData.timeStamp = timeStamp;
119 1161 : s32 sret = memcpy_s(reporterData.data, sizeof(reporterData.data), data, len);
120 1161 : CHK_PRT_RET(
121 : sret != EOK,
122 : HCCL_ERROR(
123 : "memcpy failed. errorno:[%d] level:[%hu] type:[%u] threadId:[%u] sizeof_data[%zu] len:[%u] "
124 : "timeStamp:[%llu]",
125 : sret, reporterData.level, type, reporterData.threadId, sizeof(reporterData.data), len, timeStamp),
126 : HCCL_E_MEMORY);
127 1161 : HCCL_DEBUG("CallMsprofReportAdditionInfo, AdditionInfoType[%u]", type);
128 1161 : if (MsprofReportBatchAdditionalInfo == nullptr) {
129 1161 : CHK_PTR_NULL(AdprofReportAdditionalInfo);
130 1161 : int32_t ret = AdprofReportAdditionalInfo(aging, &reporterData, sizeof(MsprofAdditionalInfo));
131 1161 : CHK_PRT_RET(ret != 0, HCCL_ERROR("AdprofReportAdditionalInfo failed. ret = [%d]", ret), HCCL_E_INTERNAL);
132 : } else {
133 0 : CHK_PTR_NULL(MsprofReportAdditionalInfo);
134 0 : int32_t ret = MsprofReportAdditionalInfo(aging, &reporterData, sizeof(MsprofAdditionalInfo));
135 0 : CHK_PRT_RET(ret != 0, HCCL_ERROR("MsprofReportAdditionalInfo failed. ret = [%d]", ret), HCCL_E_INTERNAL);
136 : }
137 :
138 1161 : HCCL_DEBUG("CallMsprofReportAdditionInfo with additionInfoType[%u] successfully", type);
139 1161 : return HCCL_SUCCESS;
140 : }
141 :
142 0 : HcclResult ProfilingManager::TaskInfo2Addition(const void* data, int len, MsprofAdditionalInfo& reporterData)
143 : {
144 0 : reporterData.level = MSPROF_REPORT_AICPU_LEVEL;
145 0 : reporterData.type = MSPROF_REPORT_AICPU_MC2_BATCH_HCCL_INFO;
146 0 : reporterData.threadId = SalGetTid();
147 0 : reporterData.dataLen = len;
148 0 : reporterData.timeStamp = 0;
149 0 : s32 sret = memcpy_s(reporterData.data, sizeof(reporterData.data), data, len);
150 0 : CHK_PRT_RET(sret != EOK, HCCL_ERROR("memcpy failed. errorno[%d]:", sret), HCCL_E_MEMORY);
151 0 : return HCCL_SUCCESS;
152 : }
153 :
154 8 : HcclResult ProfilingManager::ReportTaskInfo(s32 streamId, void* ctxPtr)
155 : {
156 8 : if (!GetProfL1State()) {
157 8 : return HCCL_SUCCESS;
158 : }
159 0 : ProfCommInfo profInfo;
160 : // 通信域中获取rankId, groupHashId等信息
161 0 : CHK_RET(GetProfInfoByStreamId(streamId, profInfo));
162 0 : CHK_PTR_NULL(ctxPtr);
163 0 : hccl::HcclSqeContext* sqeContext = reinterpret_cast<hccl::HcclSqeContext*>(ctxPtr);
164 0 : hccl::SqeRingBuffer* sqeContextBuffer = &(sqeContext->buffer);
165 0 : CHK_PTR_NULL(sqeContextBuffer);
166 0 : u32 startSqeIdx = GetStartReportSqeIdx(streamId);
167 0 : HCCL_INFO(
168 : "[ReportTaskInfo] Rank:%u, stream:%d, sqeNum:%u, startSqeIdx:%u, curSqeTailIdx: %u", profInfo.rankId, streamId,
169 : sqeContextBuffer->tailSqeIdx - startSqeIdx, startSqeIdx, sqeContextBuffer->tailSqeIdx);
170 0 : MsprofAicpuHcclTaskInfo taskInfos[HCCLINFO_REPORT_BATCH_NUM] = {};
171 0 : auto endIdx = static_cast<uint32_t>(sqeContextBuffer->tailSqeIdx);
172 0 : bool isSupportBatchReport
173 0 : = (AdprofReportBatchAdditionalInfo != nullptr || MsprofReportBatchAdditionalInfo != nullptr);
174 0 : HCCL_INFO(
175 : "AdprofReportBatchAdditionalInfo != nullptr || MsprofReportBatchAdditionalInfo != nullptr: %s",
176 : isSupportBatchReport ? "true" : "false");
177 0 : constexpr int32_t MAX_BATCH_REPORT_NUM
178 : = 512; // 最大支持批量上报的MsprofAdditionalInfo个数, 需要与接口实现侧保持一致
179 0 : MsprofAdditionalInfo addInfoVec[MAX_BATCH_REPORT_NUM] = {};
180 0 : uint32_t addInfoIndx = 0;
181 0 : for (uint32_t idx = startSqeIdx, batchId = 0; idx < endIdx; ++idx) {
182 : // 获取SqeInfo
183 0 : SqeInfo sqeInfo{};
184 0 : SqeContextUtils::QuerySqeInfo(
185 0 : sqeContextBuffer->localBuff + idx * hccl::HCCL_SQE_SIZE, sqeContextBuffer->sqeType[idx],
186 : sqeContextBuffer->addInfo[idx], &sqeInfo);
187 0 : sqeInfo.remoteRank = sqeContextBuffer->dfxInfo[idx].remoteRank;
188 : // 转换为MsprofAicpuHcclTaskInfo
189 0 : auto& taskInfo = taskInfos[batchId++];
190 0 : dfx::ProfilingExtendInfoHelper::InitHcclInfo(taskInfo);
191 0 : CommInfo2HcclInfo(profInfo, taskInfo);
192 0 : dfx::ProfilingExtendInfoHelper::SqeInfo2MsprofAicpuMC2HcclInfo(sqeInfo, taskInfo);
193 0 : taskInfo.timeStamp = sqeContextBuffer->profTimestap[idx]; // 时间戳
194 0 : DumpHcclInfo(taskInfo, batchId, idx);
195 : // 上报信息
196 0 : if (batchId == HCCLINFO_REPORT_BATCH_NUM || idx == (endIdx - 1)) {
197 0 : if (!isSupportBatchReport) {
198 0 : CHK_PRT(dfx::ProfilingManager::CallMsprofReportAdditionInfo(
199 : MSPROF_REPORT_AICPU_MC2_BATCH_HCCL_INFO, 0, taskInfos, sizeof(MsprofAicpuHcclTaskInfo) * batchId));
200 : } else {
201 0 : CHK_PRT(
202 : TaskInfo2Addition(taskInfos, sizeof(MsprofAicpuHcclTaskInfo) * batchId, addInfoVec[addInfoIndx++]));
203 0 : if (addInfoIndx == MAX_BATCH_REPORT_NUM || idx == (endIdx - 1)) {
204 0 : if (MsprofReportBatchAdditionalInfo == nullptr) {
205 0 : CHK_PRT_RET(
206 : AdprofReportBatchAdditionalInfo(
207 : aging, addInfoVec, addInfoIndx * sizeof(MsprofAdditionalInfo)),
208 : HCCL_ERROR("AdprofReportBatchAdditionalInfo failed"), HCCL_E_INTERNAL);
209 0 : addInfoIndx = 0; // 后面直接覆盖就行不需要清零;
210 : } else {
211 0 : CHK_PRT_RET(
212 : MsprofReportBatchAdditionalInfo(
213 : aging, addInfoVec, addInfoIndx * sizeof(MsprofAdditionalInfo)),
214 : HCCL_ERROR("MsprofReportBatchAdditionalInfo failed"), HCCL_E_INTERNAL);
215 0 : addInfoIndx = 0; // 后面直接覆盖就行不需要清零;
216 : }
217 : }
218 : }
219 0 : batchId = 0;
220 0 : memset_s(taskInfos, sizeof(taskInfos), 0, sizeof(taskInfos));
221 : }
222 : }
223 0 : CHK_RET(UpdateStartReportSqeIdx(streamId, sqeContextBuffer->tailSqeIdx));
224 0 : return HCCL_SUCCESS;
225 : }
226 :
227 2237 : void ProfilingManager::DumpHcclInfo(const MsprofAicpuHcclTaskInfo& taskInfo, u32 batchId, u32 idx)
228 : {
229 2237 : HCCL_DEBUG(
230 : "[ReportTaskInfo] batchId:%u, idx:%u, itemId:%llu, groupName:%llu, localRank:%u, remoteRank:%u, "
231 : "rankSize:%u, timeStamp:%llu, srcAddr:0x%llx, dstAddr:0x%llx, dataSize:%llu, taskId:%u, streamId:%u, "
232 : "planeID:%u,"
233 : "opType:%u, dataType:%u, linkType:%u, transportType:%u, rdmaType:%u, role:%u",
234 : batchId, idx, taskInfo.itemId, taskInfo.groupName, taskInfo.localRank, taskInfo.remoteRank, taskInfo.rankSize,
235 : taskInfo.timeStamp, taskInfo.srcAddr, taskInfo.dstAddr, taskInfo.dataSize, taskInfo.taskId, taskInfo.streamId,
236 : taskInfo.planeID, taskInfo.opType, taskInfo.dataType, taskInfo.linkType, taskInfo.transportType,
237 : taskInfo.rdmaType, taskInfo.role);
238 2237 : }
239 :
240 0 : void ProfilingManager::CommInfo2HcclInfo(const dfx::ProfCommInfo& profInfo, MsprofAicpuHcclTaskInfo& taskInfo)
241 : {
242 0 : taskInfo.groupName = profInfo.groupNameHashId;
243 0 : taskInfo.localRank = profInfo.rankId;
244 0 : taskInfo.rankSize = profInfo.rankNum;
245 0 : }
246 :
247 0 : HcclResult ProfilingManager::ReportHcclOpInfo(MsprofAicpuHCCLOPInfo& hcclOpInfo, std::string& algTypeStr)
248 : {
249 0 : if (!GetProfL0State()) {
250 0 : return HCCL_SUCCESS;
251 : }
252 0 : CHK_PTR_NULL(aicpu::GetTaskAndStreamId);
253 0 : uint64_t taskId = 0U;
254 0 : uint32_t streamId = 0;
255 0 : if (AicpuGetStreamId == nullptr || AicpuGetTaskId == nullptr) {
256 0 : CHK_PTR_NULL(aicpu::GetTaskAndStreamId);
257 0 : CHK_PRT_RET(
258 : aicpu::GetTaskAndStreamId(taskId, streamId) != aicpu::status_t::AICPU_ERROR_NONE,
259 : HCCL_ERROR("Failed to get task id and stream id."), HCCL_E_PARA);
260 : } else {
261 0 : streamId = AicpuGetStreamId();
262 0 : taskId = AicpuGetTaskId();
263 : }
264 :
265 0 : HCCL_INFO("[ProfilingManager] ReportHcclOpInfo streamId = %u, taskId = %llu", streamId, taskId);
266 :
267 0 : hcclOpInfo.algType = GetProfHashId(algTypeStr.c_str(), algTypeStr.length());
268 0 : hcclOpInfo.taskId = taskId;
269 0 : hcclOpInfo.streamId = streamId;
270 0 : HCCL_INFO(
271 : "[ReportHcclOpInfo] relay:%u, retry:%u, dataType:%u, algType:%llu, count:%llu, groupHashId:%llu",
272 : hcclOpInfo.relay, hcclOpInfo.retry, hcclOpInfo.dataType, hcclOpInfo.algType, hcclOpInfo.count,
273 : hcclOpInfo.groupName);
274 0 : CHK_PRT(dfx::ProfilingManager::CallMsprofReportAdditionInfo(
275 : MSPROF_REPORT_AICPU_HCCL_OP_INFO, ProfGetCurCpuTimestamp(), &hcclOpInfo, sizeof(MsprofAicpuHCCLOPInfo)));
276 0 : return HCCL_SUCCESS;
277 : }
278 :
279 0 : HcclResult ProfilingManager::ReportMainStreamTask(hccl::Stream& stream, uint16_t taskId, uint16_t type)
280 : {
281 0 : if (!GetProfL0State()) {
282 0 : return HCCL_SUCCESS;
283 : }
284 0 : CHK_PTR_NULL(aicpu::GetTaskAndStreamId);
285 0 : uint64_t aicpuKernelTaskId = 0U;
286 0 : uint32_t aicpuKernelStreamId = 0;
287 0 : if (AicpuGetStreamId == nullptr || AicpuGetTaskId == nullptr) {
288 0 : CHK_PTR_NULL(aicpu::GetTaskAndStreamId);
289 0 : CHK_PRT_RET(
290 : aicpu::GetTaskAndStreamId(aicpuKernelTaskId, aicpuKernelStreamId) != aicpu::status_t::AICPU_ERROR_NONE,
291 : HCCL_ERROR("Failed to get task id and stream id."), HCCL_E_PARA);
292 : } else {
293 0 : aicpuKernelTaskId = AicpuGetTaskId();
294 0 : aicpuKernelStreamId = AicpuGetStreamId();
295 : }
296 0 : HCCL_INFO(
297 : "[ReportMainStreamTask] aicpuKernelStreamId = %u, aicpuKernelTaskId = %llu", aicpuKernelStreamId,
298 : aicpuKernelTaskId);
299 0 : MsprofAicpuHcclMainStreamTask flagtask{};
300 0 : flagtask.streamId = stream.id();
301 0 : flagtask.taskId = taskId;
302 0 : flagtask.type = type;
303 0 : flagtask.aicpuStreamId = aicpuKernelStreamId;
304 0 : flagtask.aicpuTaskId = aicpuKernelTaskId;
305 0 : HCCL_INFO(
306 : "[ReportMainStreamTask] streamId:%u, taskId:%u, type:%u", flagtask.streamId, flagtask.taskId, flagtask.type);
307 :
308 0 : CHK_PRT(dfx::ProfilingManager::CallMsprofReportAdditionInfo(
309 : MSPROF_REPORT_AICPU_HCCL_FLAG_TASK, ProfGetCurCpuTimestamp(), &flagtask,
310 : sizeof(MsprofAicpuHcclMainStreamTask)));
311 :
312 0 : return HCCL_SUCCESS;
313 : }
314 :
315 411 : HcclResult ProfilingManager::ReportFilpTask(s32 streamId, uint16_t taskId, uint32_t flipNum)
316 : {
317 411 : if (!GetProfL0State()) {
318 369 : return HCCL_SUCCESS;
319 : }
320 42 : MsporfAicpuFlipTask flipTaskInfo{};
321 42 : flipTaskInfo.streamId = streamId;
322 42 : flipTaskInfo.taskId = taskId;
323 42 : flipTaskInfo.flipNum = flipNum;
324 42 : HCCL_INFO(
325 : "[ReportFlipTask] streamId:%u, taskId:%u, flipNum:%u", flipTaskInfo.streamId, flipTaskInfo.taskId,
326 : flipTaskInfo.flipNum);
327 42 : CHK_PRT(dfx::ProfilingManager::CallMsprofReportAdditionInfo(
328 : MSPROF_REPORT_AICPU_FILP_TASK, ProfGetCurCpuTimestamp(), &flipTaskInfo, sizeof(MsporfAicpuFlipTask)));
329 :
330 42 : return HCCL_SUCCESS;
331 : }
332 :
333 11 : uint64_t ProfilingManager::GetProfHashId(const char* name, uint32_t len)
334 : {
335 11 : if (name == nullptr || len == 0) {
336 0 : HCCL_WARNING("HashData is empty.");
337 0 : return INVALID_U64;
338 : }
339 11 : if (MsprofReportBatchAdditionalInfo == nullptr) {
340 11 : CHK_PRT_RET((AdprofGetHashId == nullptr), HCCL_WARNING("AdprofGetHashId is null, just return"), INVALID_U64);
341 11 : return AdprofGetHashId(name, len);
342 : } else {
343 0 : CHK_PRT_RET((MsprofStr2Id == nullptr), HCCL_WARNING("MsprofStr2Id is null, just return"), INVALID_U64);
344 0 : return MsprofStr2Id(name, len);
345 : }
346 : }
347 :
348 0 : uint32_t ProfilingManager::GetStartReportSqeIdx(s32 streamId)
349 : {
350 0 : std::unique_lock<std::mutex> lock(startReportSqeIdxMutex_);
351 0 : u32 lastSqeTailIdx = 0;
352 0 : auto iter = streamToSqeIdxMap_.find(streamId);
353 0 : if (iter == streamToSqeIdxMap_.end()) {
354 0 : HCCL_INFO("[GetProfInfoByStreamId]streamId:%d is not found", streamId);
355 0 : streamToSqeIdxMap_.insert({streamId, 0});
356 : } else {
357 0 : lastSqeTailIdx = iter->second;
358 : }
359 0 : return lastSqeTailIdx;
360 0 : }
361 :
362 5 : HcclResult ProfilingManager::UpdateStartReportSqeIdx(s32 streamId, u32 newSqeTailIdx)
363 : {
364 5 : std::unique_lock<std::mutex> lock(startReportSqeIdxMutex_);
365 5 : auto iter = streamToSqeIdxMap_.find(streamId);
366 5 : if (iter == streamToSqeIdxMap_.end()) {
367 2 : streamToSqeIdxMap_.insert({streamId, newSqeTailIdx});
368 2 : HCCL_INFO("[UpdateStartReportSqeIdx]streamId:%d is not found, newSqeTailIdx:%u", streamId, newSqeTailIdx);
369 : } else {
370 : // 到2048时，更新成0;
371 3 : newSqeTailIdx = (newSqeTailIdx == hccl::HCCL_SQE_MAX_CNT) ? 0 : newSqeTailIdx;
372 3 : HCCL_INFO(
373 : "[UpdateStartReportSqeIdx] streamId:%d, lastSqeTailIdx:%u, newSqeTailIdx:%u", streamId, iter->second,
374 : newSqeTailIdx);
375 3 : iter->second = newSqeTailIdx;
376 : }
377 5 : return HCCL_SUCCESS;
378 5 : }
379 :
380 0 : HcclResult ProfilingManager::GetProfInfoByStreamId(s32 streamId, ProfCommInfo& profInfo)
381 : {
382 0 : std::string tag = "unknown";
383 0 : std::unique_lock<std::mutex> lock(streamMutex_);
384 0 : auto iter = streamToTagMap_.find(streamId);
385 0 : if (iter == streamToTagMap_.end()) {
386 0 : HCCL_INFO("[GetProfInfoByStreamId]streamId:%d is not found", streamId);
387 : } else {
388 0 : tag = iter->second;
389 0 : auto opInfoIter = tagOpInfoMap_.find(tag);
390 0 : if (opInfoIter == tagOpInfoMap_.end()) {
391 0 : HCCL_INFO("[GetProfInfoByStreamId]tag:%s is not found", tag.c_str());
392 : } else {
393 0 : profInfo = opInfoIter->second;
394 : }
395 : }
396 0 : return HCCL_SUCCESS;
397 0 : }
398 :
399 451 : HcclResult ProfilingManager::AddProfInfoByStreamId(s32 streamId, const std::string& tag, const ProfCommInfo& profInfo)
400 : {
401 451 : std::unique_lock<std::mutex> lock(streamMutex_);
402 451 : auto tagMapIter = streamToTagMap_.find(streamId);
403 451 : if (tagMapIter == streamToTagMap_.end()) {
404 41 : streamToTagMap_.insert({streamId, tag});
405 : } else {
406 410 : tagMapIter->second = tag;
407 : // streamId之前存在说明流被销毁了，被其它通信域复用了, 上报一条信息告知profiling taskid发生翻转
408 410 : CHK_RET(ReportFilpTask(streamId, UINT16_MAX, UINT16_MAX));
409 410 : HCCL_INFO("[AddProfInfoByStreamId] streamId:%d content:%s update", streamId, tag.c_str());
410 : }
411 :
412 451 : auto opInfoMapIter = tagOpInfoMap_.find(tag);
413 451 : if (opInfoMapIter == tagOpInfoMap_.end()) {
414 11 : tagOpInfoMap_.insert(std::make_pair(tag, profInfo));
415 : } else {
416 440 : opInfoMapIter->second = profInfo;
417 : }
418 451 : return HCCL_SUCCESS;
419 451 : }
420 :
421 0 : void TaskProfilingCallBack([[maybe_unused]] void* userPtr, void* param, u32 length)
422 : {
423 0 : if (UNLIKELY(param == nullptr)) {
424 0 : HCCL_ERROR("[ProfilingManager][%s]param is nullptr.", __func__);
425 0 : return;
426 : }
427 0 : struct hccl::TaskPara* taskPara = (struct hccl::TaskPara*)param;
428 :
429 0 : if (UNLIKELY(sizeof(hccl::TaskPara) < length)) {
430 0 : return;
431 : }
432 0 : HCCL_INFO(
433 : "[ProfilingManager][%s]Start handle task profiler, taskType[%d], profilerType[%d]", __func__,
434 : static_cast<int>(taskPara->type), static_cast<int>(taskPara->profilerType));
435 0 : switch (taskPara->type) {
436 0 : case hccl::TaskType::TASK_BATCH_REPORT:
437 0 : dfx::ProfilingManager::ReportTaskInfo(taskPara->streamTasks.streamID, taskPara->streamTasks.ctxPtr);
438 0 : break;
439 :
440 0 : case hccl::TaskType::TASK_FLIP:
441 0 : dfx::ProfilingManager::ReportFilpTask(
442 0 : taskPara->flipTask.streamID, taskPara->flipTask.taskID, taskPara->flipTask.flipNum);
443 0 : break;
444 :
445 0 : default:
446 0 : return;
447 : }
448 0 : return;
449 : }
450 : } // namespace dfx
|
