Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 : #include <cstring>
11 : #include <algorithm>
12 : #include "profiling_handler.h"
13 : #include "dlprof_function_v2.h"
14 : #include "exception_util.h"
15 : #include "internal_exception.h"
16 : #include "sal.h"
17 : #include "task_param.h"
18 : #include "aprof_pub.h"
19 : #include "orion_adapter_rts.h"
20 : #include "communicator_impl.h"
21 : #include "data_type.h"
22 : namespace Hccl {
23 : #define UNUSED(x) (void)(x)
24 :
25 : constexpr uint16_t CCU_TYPE = 2;
26 : constexpr uint32_t DPU_DEV_ID_MASK = (1U << 12);
27 :
28 : ProfilingHandler ProfilingHandler::instance_;
29 :
30 1 : ProfilingHandler::ProfilingHandler() {}
31 :
32 1 : ProfilingHandler::~ProfilingHandler() {}
33 :
34 749 : ProfilingHandler& ProfilingHandler::GetInstance() { return instance_; }
35 :
36 1 : void ProfilingHandler::InitLog() const
37 : {
38 1 : if (LIKELY(HcclCheckLogLevel(HCCL_LOG_INFO) == 0)) {
39 0 : return;
40 : }
41 24 : for (auto i = 0; i < TaskParamType::__COUNT__; ++i) {
42 23 : TaskParamType type(static_cast<TaskParamType::Value>(i));
43 23 : std::string nameInfo = type.Describe();
44 23 : uint64_t hashId = GetProfHashId(nameInfo.c_str(), nameInfo.length());
45 69 : HCCL_INFO("[TaskParamType] nameInfo[%s] ret[%llu]", nameInfo.c_str(), hashId);
46 23 : }
47 : }
48 :
49 258 : HcclResult ProfilingHandler::Init()
50 : {
51 258 : if (initializedFlag_) {
52 257 : return HCCL_SUCCESS;
53 : }
54 1 : if (Hccl::DlProfFunction::GetInstance().DlProfFunctionInit() != HCCL_SUCCESS) {
55 0 : HCCL_ERROR("[ProfilingHandler][Init] DlProfFunctionInit failed.");
56 0 : return HCCL_E_INTERNAL;
57 : }
58 1 : ProfCommandHandle callback = CommandHandleWrapper;
59 1 : auto ret = DlProfFunction::GetInstance().dlMsprofRegisterCallback(HCCL, callback);
60 1 : if (ret != 0) {
61 0 : HCCL_ERROR(
62 : "[ProfilingHandler][Init]errNo[0x%016llx] Prof Register CtrlCallback fail, return[%d]",
63 : HCCL_ERROR_CODE(HCCL_E_RUNTIME), ret);
64 0 : return HCCL_E_RUNTIME;
65 : }
66 1 : cachedAlgTypeHashId_.store(GetProfHashId("AlgType::NHR", strlen("AlgType::NHR")));
67 1 : initializedFlag_ = true;
68 1 : InitLog();
69 1 : return HCCL_SUCCESS;
70 : }
71 :
72 0 : int32_t ProfilingHandler::CommandHandleWrapper(uint32_t rtType, void* data, uint32_t len)
73 : {
74 0 : return instance_.CommandHandle(rtType, data, len);
75 : }
76 :
77 0 : void ProfilingHandler::ReportKernel() const {}
78 :
79 3 : void ProfilingHandler::ReportHostApi(OpType opType, uint64_t beginTime, uint64_t endTime, bool cachedReq, bool isAiCpu)
80 : {
81 3 : uint32_t threadId = SalGetTid();
82 3 : std::string profName(GetProfOpName(opType));
83 3 : if (isAiCpu) {
84 1 : profName += "AicpuKernel";
85 : }
86 3 : uint64_t cmdItemId = DlProfFunction::GetInstance().dlMsprofStr2Id(profName.c_str(), profName.length());
87 3 : if (enableHostApi_) {
88 0 : ReportAclApi(opType, beginTime, endTime, cmdItemId, threadId, cachedReq);
89 : }
90 3 : ReportNodeApi(beginTime, endTime, cmdItemId, threadId, cachedReq);
91 3 : ReportNodeBasicInfo(endTime, cmdItemId, threadId, cachedReq);
92 3 : }
93 :
94 3 : void ProfilingHandler::ReportHcclOp(const DfxOpInfo& opInfo, bool cachedReq)
95 : {
96 3 : uint32_t threadId = SalGetTid();
97 3 : ReportHcclOpInfo(opInfo.endTime_, opInfo, threadId, cachedReq);
98 3 : }
99 :
100 5 : void ProfilingHandler::ReportHcclTaskApi(
101 : TaskParamType taskType, uint64_t beginTime, uint64_t endTime, bool isMasterStream, bool cachedReq, bool ignoreLevel)
102 : {
103 5 : MsprofApi reporterData{};
104 5 : reporterData.level = MSPROF_REPORT_HCCL_NODE_LEVEL;
105 5 : reporterData.type = isMasterStream ? MSPROF_REPORT_HCCL_MASTER_TYPE : MSPROF_REPORT_HCCL_SLAVE_TYPE;
106 5 : reporterData.threadId = SalGetTid();
107 5 : reporterData.beginTime = beginTime;
108 5 : reporterData.endTime = endTime;
109 5 : const std::string proName(GetProfTaskOpNameV2(taskType));
110 5 : reporterData.itemId = GetProfHashId(proName.c_str(), proName.length());
111 5 : if (taskType == TaskParamType::TASK_AICPU_KERNEL) {
112 0 : return;
113 : }
114 5 : if (cachedReq) {
115 5 : std::lock_guard<std::mutex> lock(cachedTaskApiInfoMutex_);
116 5 : cachedTaskApiInfo_.push(reporterData);
117 5 : }
118 5 : if ((!enableHcclNode_) || (!ignoreLevel && !enableHcclL1_)) {
119 3 : return;
120 : }
121 6 : HCCL_INFO(
122 : "ReportHcclTaskApi, enableHcclNode_[%d],ignoreLevel[%d],enableHcclL1_[%d],taskType[%s], beginTime[%llu], "
123 : "endTime[%llu], isMasterStream[%d], cachedReq[%d]",
124 : enableHcclNode_, ignoreLevel, enableHcclL1_, proName.c_str(), beginTime, endTime, isMasterStream, cachedReq);
125 2 : s32 ret = DlProfFunction::GetInstance().dlMsprofReportApi(1, &reporterData);
126 2 : if (ret != 0) {
127 0 : THROW<InternalException>("Call MsprofReportApi fail, return[%d]", ret);
128 : }
129 5 : }
130 :
131 5 : void ProfilingHandler::SetCachedCclTag()
132 : {
133 5 : cachedNewCclTag_.clear();
134 100 : for (const auto& item : CMD_OP_TYPE_INFO_MAP) {
135 95 : const std::string& cclTag = item.second.second;
136 95 : cachedNewCclTag_[item.second.first] = GetProfHashId(cclTag.c_str(), cclTag.length());
137 : }
138 5 : }
139 :
140 6 : uint32_t ProfilingHandler::GetTaskTypeValue(TaskParamType taskType) const
141 : {
142 6 : switch (taskType) {
143 0 : case TaskParamType::TASK_DPU_INLINE_WRITE:
144 : case TaskParamType::TASK_DPU_NOTIFY_WAIT:
145 : case TaskParamType::TASK_DPU_WRITE_WITH_NOTIFY:
146 : case TaskParamType::TASK_DPU_CHANNEL_FENCE:
147 0 : return static_cast<uint32_t>(ProfTaskType::TASK_DPU_HCCL_INFO);
148 6 : default:
149 6 : return static_cast<uint32_t>(ProfTaskType::TASK_HCCL_INFO);
150 : }
151 : }
152 :
153 7 : void ProfilingHandler::FillTaskAdditionInfo(const TaskInfo& taskInfo, MsprofAdditionalInfo& reporterData) const
154 : {
155 7 : reporterData.level = MSPROF_REPORT_HCCL_NODE_LEVEL;
156 7 : reporterData.threadId = SalGetTid();
157 7 : FillProfCommonInfo(taskInfo, reporterData);
158 :
159 7 : switch (taskInfo.taskParam_.taskType) {
160 0 : case TaskParamType::TASK_DPU_INLINE_WRITE:
161 : case TaskParamType::TASK_DPU_NOTIFY_WAIT:
162 : case TaskParamType::TASK_DPU_WRITE_WITH_NOTIFY:
163 : case TaskParamType::TASK_DPU_CHANNEL_FENCE:
164 0 : reporterData.type = static_cast<uint32_t>(ProfTaskType::TASK_DPU_HCCL_INFO);
165 0 : FillDpuProfInfo(taskInfo, reporterData);
166 0 : break;
167 7 : default:
168 7 : reporterData.type = static_cast<uint32_t>(ProfTaskType::TASK_HCCL_INFO);
169 7 : FillProfTaskSpecificInfo(taskInfo, reinterpret_cast<MsprofHcclInfo*>(reporterData.data));
170 7 : break;
171 : }
172 7 : }
173 :
174 3 : void ProfilingHandler::ReportHcclTaskDetails(const TaskInfo& taskInfo, bool cachedReq)
175 : {
176 3 : if (enableHcclL1_ == false && !cachedReq) {
177 2 : return;
178 : }
179 3 : if (taskInfo.dfxOpInfo_ == nullptr) {
180 3 : HCCL_WARNING("[%s] dfxOpInfo_ is nullptr, skip ReportHcclTaskDetails!", __func__);
181 1 : return;
182 : }
183 2 : if (taskInfo.dfxOpInfo_->comm_ == nullptr) {
184 3 : HCCL_WARNING("[%s] comm_ is nullptr, skip ReportHcclTaskDetails!", __func__);
185 1 : return;
186 : }
187 1 : if (cachedReq) {
188 1 : std::lock_guard<std::mutex> lock(cacheTaskInfosMutex_);
189 1 : cacheTaskInfos_.push_back(taskInfo);
190 1 : }
191 1 : if (enableHcclL1_ == false) {
192 0 : return;
193 : }
194 1 : MsprofAdditionalInfo reporterData{};
195 1 : FillTaskAdditionInfo(taskInfo, reporterData);
196 1 : DumpHCCLReportData(taskInfo, reporterData);
197 1 : CallAdditionInfo(reporterData);
198 : }
199 :
200 2 : void ProfilingHandler::ReportHcclTaskDetailsBatchLog(const std::vector<TaskInfo*>& taskInfos) const
201 : {
202 2 : if (LIKELY(HcclCheckLogLevel(HCCL_LOG_INFO) == 0)) {
203 0 : return;
204 : }
205 6 : for (const auto& taskInfo : taskInfos) {
206 12 : HCCL_INFO("ReportHcclTaskDetailsBatchLog, taskType[%u]", GetTaskTypeValue(taskInfo->taskParam_.taskType));
207 : }
208 : }
209 :
210 7 : void ProfilingHandler::ReportHcclTaskDetailsBatch(const std::vector<TaskInfo*>& taskInfos, bool cachedReq)
211 : {
212 7 : if (taskInfos.empty()) {
213 3 : HCCL_WARNING("[ProfilingHandler::ReportHcclTaskDetailsBatch] taskInfos is empty!");
214 1 : return;
215 : }
216 6 : if (taskInfos[0]->dfxOpInfo_ == nullptr) {
217 3 : HCCL_WARNING("[ProfilingHandler::ReportHcclTaskDetailsBatch] taskInfo.dfxOpInfo_ is nullptr, skip!");
218 1 : return;
219 : }
220 5 : if (taskInfos[0]->dfxOpInfo_->comm_ == nullptr) {
221 3 : HCCL_WARNING("[ProfilingHandler::ReportHcclTaskDetailsBatch] taskInfo.dfxOpInfo_->comm_ is nullptr, skip!");
222 1 : return;
223 : }
224 4 : if (enableHcclL1_ == false && !cachedReq) {
225 1 : return;
226 : }
227 3 : if (cachedReq) {
228 3 : std::lock_guard<std::mutex> lock(cacheTaskInfosMutex_);
229 8 : for (const auto& taskInfo : taskInfos) {
230 5 : cacheTaskInfos_.push_back(*taskInfo);
231 : }
232 3 : }
233 3 : if (enableHcclL1_ == false) {
234 1 : return;
235 : }
236 2 : ReportHcclTaskDetailsBatchLog(taskInfos);
237 6 : for (const auto& taskInfo : taskInfos) {
238 4 : MsprofAdditionalInfo reporterData{};
239 4 : FillTaskAdditionInfo(*taskInfo, reporterData);
240 4 : CallAdditionInfo(reporterData);
241 : }
242 : }
243 :
244 5 : void ProfilingHandler::CallAdditionInfo(MsprofAdditionalInfo& reporterData) const
245 : {
246 : s32 ret
247 5 : = DlProfFunction::GetInstance().dlMsprofReportAdditionalInfo(1, &reporterData, sizeof(MsprofAdditionalInfo));
248 5 : if (ret != 0) {
249 0 : THROW<InternalException>("Call MsprofReportAdditionalInfo failed, return[%d]", ret);
250 : }
251 5 : }
252 :
253 7 : void ProfilingHandler::FillProfCommonInfo(const TaskInfo& taskInfo, MsprofAdditionalInfo& reporterData) const
254 : {
255 7 : if (taskInfo.dfxOpInfo_ == nullptr) {
256 0 : HCCL_WARNING("[ProfilingHandler][%s]taskInfo.dfxOpInfo_ is nullptr, skip GetProfCommonInfo.", __func__);
257 0 : return;
258 : }
259 7 : reporterData.timeStamp = taskInfo.taskParam_.endTime;
260 7 : reporterData.dataLen = sizeof(MsprofHcclInfo);
261 7 : auto* profInfo = reinterpret_cast<MsprofHcclInfo*>(reporterData.data);
262 7 : *profInfo = MsprofHcclInfo();
263 :
264 7 : const auto& profName = GetProfTaskOpNameV2(taskInfo.taskParam_.taskType);
265 7 : profInfo->itemId = GetProfHashId(profName.c_str(), profName.length());
266 7 : auto newCclTagIt = cachedNewCclTag_.find(taskInfo.dfxOpInfo_->op_.opType);
267 7 : profInfo->cclTag = (newCclTagIt != cachedNewCclTag_.end()) ? newCclTagIt->second : INVALID_U64;
268 7 : const auto& opTag = taskInfo.dfxOpInfo_->op_.opTag;
269 7 : uint64_t groupName = GetProfHashId(opTag.c_str(), opTag.length());
270 7 : if (taskInfo.dfxOpInfo_->comm_ == nullptr) {
271 0 : HCCL_WARNING("[ProfilingHandler][%s]taskInfo.dfxOpInfo_->comm_ is nullptr, skip GetProfCommonInfo.", __func__);
272 0 : return;
273 : }
274 7 : if (taskInfo.dfxOpInfo_->isIndop_ == true) {
275 0 : profInfo->groupName = groupName;
276 0 : profInfo->rankSize = taskInfo.dfxOpInfo_->rankSize_;
277 : } else {
278 7 : CommunicatorImpl* commImp = static_cast<CommunicatorImpl*>(taskInfo.dfxOpInfo_->comm_);
279 7 : if (commImp == nullptr) {
280 0 : HCCL_WARNING("[ProfilingHandler][%s]commImp is nullptr, skip GetProfCommonInfo.", __func__);
281 0 : return;
282 : }
283 7 : profInfo->groupName = groupName;
284 7 : profInfo->rankSize = commImp->GetRankSize();
285 : }
286 7 : profInfo->workFlowMode = static_cast<u32>(HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE);
287 7 : profInfo->planeID = 0;
288 7 : profInfo->stage = 0;
289 7 : profInfo->role = static_cast<uint32_t>(TaskRole::DST);
290 7 : profInfo->durationEstimated = 0;
291 7 : profInfo->localRank = taskInfo.dfxOpInfo_->op_.myRank;
292 7 : profInfo->remoteRank = taskInfo.remoteRank_;
293 7 : profInfo->dataType = taskInfo.dfxOpInfo_->op_.dataType;
294 7 : profInfo->opType = taskInfo.dfxOpInfo_->op_.opType;
295 7 : profInfo->transportType = static_cast<int32_t>(SimpleTaskType::UB);
296 7 : if (profInfo->remoteRank == INVALID_VALUE_RANKID) {
297 0 : profInfo->transportType = static_cast<int32_t>(SimpleTaskType::LOCAL);
298 0 : profInfo->remoteRank = profInfo->localRank;
299 0 : profInfo->linkType = 0;
300 : }
301 7 : }
302 :
303 7 : void ProfilingHandler::FillProfTaskSpecificInfo(const TaskInfo& taskInfo, MsprofHcclInfo* profInfo) const
304 : {
305 7 : const auto& taskType = taskInfo.taskParam_.taskType;
306 7 : const auto& taskPara = taskInfo.taskParam_.taskPara;
307 7 : switch (taskType) {
308 0 : case TaskParamType::TASK_SDMA:
309 : case TaskParamType::TASK_RDMA:
310 0 : profInfo->srcAddr = static_cast<u64>(reinterpret_cast<uintptr_t>(taskPara.DMA.src));
311 0 : profInfo->dstAddr = static_cast<u64>(reinterpret_cast<uintptr_t>(taskPara.DMA.dst));
312 0 : profInfo->dataSize = static_cast<u32>(taskPara.DMA.size);
313 0 : profInfo->notifyID = taskPara.DMA.notifyID;
314 0 : profInfo->linkType = static_cast<uint16_t>(taskPara.DMA.linkType);
315 0 : break;
316 0 : case TaskParamType::TASK_REDUCE_INLINE:
317 : case TaskParamType::TASK_REDUCE_TBE:
318 0 : profInfo->srcAddr = static_cast<u64>(reinterpret_cast<uintptr_t>(taskPara.Reduce.src));
319 0 : profInfo->dstAddr = static_cast<u64>(reinterpret_cast<uintptr_t>(taskPara.Reduce.dst));
320 0 : profInfo->dataSize = static_cast<u32>(taskPara.Reduce.size);
321 0 : profInfo->notifyID = taskPara.Reduce.notifyID;
322 0 : profInfo->linkType = static_cast<uint16_t>(taskPara.Reduce.linkType);
323 0 : break;
324 7 : case TaskParamType::TASK_NOTIFY_RECORD:
325 : case TaskParamType::TASK_NOTIFY_WAIT:
326 7 : profInfo->notifyID = taskPara.Notify.notifyID;
327 7 : break;
328 0 : case TaskParamType::TASK_CCU:
329 0 : HCCL_INFO("current taskType is TASK_CCU");
330 0 : ReportCcuInfo(taskInfo);
331 0 : break;
332 0 : default:
333 0 : break;
334 : }
335 7 : }
336 :
337 0 : void ProfilingHandler::ConvertHcclInfoToDpuTrack(MsprofAdditionalInfo& reporterData) const
338 : {
339 0 : auto* profInfo = reinterpret_cast<MsprofHcclInfo*>(reporterData.data);
340 0 : uint64_t itemId = profInfo->itemId;
341 0 : uint64_t cclTag = profInfo->cclTag;
342 0 : uint64_t groupName = profInfo->groupName;
343 0 : uint32_t localRank = profInfo->localRank;
344 0 : uint32_t remoteRank = profInfo->remoteRank;
345 0 : uint32_t rankSize = profInfo->rankSize;
346 0 : uint32_t workFlowMode = profInfo->workFlowMode;
347 0 : uint16_t planeID = static_cast<uint16_t>(profInfo->planeID);
348 0 : uint32_t stage = profInfo->stage;
349 0 : uint32_t role = profInfo->role;
350 0 : double durationEstimated = profInfo->durationEstimated;
351 :
352 0 : reporterData.dataLen = sizeof(MsprofDpuHcclTrack);
353 0 : auto* dpuProfInfo = reinterpret_cast<MsprofDpuHcclTrack*>(reporterData.data);
354 0 : *dpuProfInfo = MsprofDpuHcclTrack();
355 :
356 0 : dpuProfInfo->itemId = itemId;
357 0 : dpuProfInfo->cclTag = cclTag;
358 0 : dpuProfInfo->groupName = groupName;
359 0 : dpuProfInfo->localRank = localRank;
360 0 : dpuProfInfo->remoteRank = remoteRank;
361 0 : dpuProfInfo->rankSize = rankSize;
362 0 : dpuProfInfo->workFlowMode = static_cast<uint8_t>(workFlowMode);
363 0 : dpuProfInfo->planeID = planeID;
364 0 : dpuProfInfo->stage = stage;
365 0 : dpuProfInfo->role = static_cast<uint8_t>(role);
366 0 : dpuProfInfo->durationEstimated = durationEstimated;
367 0 : }
368 :
369 0 : void ProfilingHandler::FillDpuTaskParaDetails(const TaskInfo& taskInfo, MsprofDpuHcclTrack* dpuProfInfo) const
370 : {
371 0 : const auto& taskPara = taskInfo.taskParam_.taskPara;
372 0 : switch (taskInfo.taskParam_.taskType) {
373 0 : case TaskParamType::TASK_DPU_INLINE_WRITE:
374 : case TaskParamType::TASK_DPU_WRITE_WITH_NOTIFY:
375 0 : dpuProfInfo->srcAddr = static_cast<u64>(reinterpret_cast<uintptr_t>(taskPara.DMA.src));
376 0 : dpuProfInfo->dstAddr = static_cast<u64>(reinterpret_cast<uintptr_t>(taskPara.DMA.dst));
377 0 : dpuProfInfo->dataSize = static_cast<u32>(taskPara.DMA.size);
378 0 : dpuProfInfo->notifyID = taskPara.DMA.notifyID;
379 0 : break;
380 0 : case TaskParamType::TASK_DPU_NOTIFY_WAIT:
381 : case TaskParamType::TASK_DPU_CHANNEL_FENCE:
382 0 : dpuProfInfo->notifyID = taskPara.Notify.notifyID;
383 0 : break;
384 0 : default:
385 0 : break;
386 : }
387 0 : }
388 :
389 0 : void ProfilingHandler::FillDpuProfInfo(const TaskInfo& taskInfo, MsprofAdditionalInfo& reporterData) const
390 : {
391 0 : if (taskInfo.dfxOpInfo_ == nullptr) {
392 0 : HCCL_WARNING("[ProfilingHandler::FillDpuProfInfo] taskInfo.dfxOpInfo_ is nullptr, skip GetDpuProfInfo!");
393 0 : return;
394 : }
395 0 : ConvertHcclInfoToDpuTrack(reporterData);
396 0 : auto* dpuProfInfo = reinterpret_cast<MsprofDpuHcclTrack*>(reporterData.data);
397 0 : dpuProfInfo->dataType = static_cast<uint32_t>(taskInfo.dfxOpInfo_->op_.dataType);
398 0 : dpuProfInfo->opType = static_cast<uint32_t>(taskInfo.dfxOpInfo_->op_.opType);
399 0 : dpuProfInfo->transportType = static_cast<uint32_t>(SimpleTaskType::ROCE);
400 0 : dpuProfInfo->aicpu_task_id = taskInfo.taskParam_.aicpuTaskId;
401 0 : dpuProfInfo->npuDevId = taskInfo.taskParam_.npuDevId;
402 0 : dpuProfInfo->dpuDevId = DPU_DEV_ID_MASK;
403 0 : const auto& taskPara = taskInfo.taskParam_.taskPara;
404 0 : dpuProfInfo->linkType = static_cast<uint16_t>(taskPara.DMA.linkType);
405 0 : if (dpuProfInfo->remoteRank == INVALID_VALUE_RANKID) {
406 0 : dpuProfInfo->transportType = static_cast<uint32_t>(SimpleTaskType::LOCAL);
407 0 : dpuProfInfo->remoteRank = dpuProfInfo->localRank;
408 0 : dpuProfInfo->linkType = 0;
409 : }
410 0 : dpuProfInfo->taskId = taskInfo.taskId_;
411 0 : dpuProfInfo->streamId = taskInfo.streamId_;
412 0 : dpuProfInfo->timeStamp = taskInfo.taskParam_.beginTime;
413 0 : HCCL_INFO(
414 : "[FillDpuProfInfo]taskId[%u], streamId[%u], npuDevId[%u], dpuDevId[%u], "
415 : "starttime[%llu], endtime[%llu], aicputaskId[%u].",
416 : dpuProfInfo->taskId, dpuProfInfo->streamId, dpuProfInfo->npuDevId, dpuProfInfo->dpuDevId,
417 : dpuProfInfo->timeStamp, reporterData.timeStamp, dpuProfInfo->aicpu_task_id);
418 0 : FillDpuTaskParaDetails(taskInfo, dpuProfInfo);
419 : }
420 :
421 9 : void ProfilingHandler::GetHCCLReportData(const TaskInfo& taskInfo, HCCLReportData& hcclReportData) const
422 : {
423 9 : if (taskInfo.dfxOpInfo_ == nullptr) {
424 3 : HCCL_ERROR("[ProfilingHandler]taskInfo.dfxOpInfo_ is nullptr");
425 1 : return;
426 : }
427 8 : hcclReportData.ts = taskInfo.taskParam_.endTime;
428 8 : const auto& profName = GetProfTaskOpNameV2(taskInfo.taskParam_.taskType);
429 8 : hcclReportData.profInfo.itemId = GetProfHashId(profName.c_str(), profName.length());
430 8 : auto newCclTagIt = cachedNewCclTag_.find(taskInfo.dfxOpInfo_->op_.opType);
431 8 : hcclReportData.profInfo.cclTag = (newCclTagIt != cachedNewCclTag_.end()) ? newCclTagIt->second : INVALID_U64;
432 8 : const auto& opTag = taskInfo.dfxOpInfo_->op_.opTag;
433 8 : uint64_t groupName = GetProfHashId(opTag.c_str(), opTag.length());
434 8 : if (taskInfo.dfxOpInfo_->comm_ == nullptr) {
435 0 : HCCL_ERROR("[ProfilingHandler]taskInfo.dfxOpInfo_->comm_ is nullptr");
436 0 : return;
437 : }
438 8 : if (taskInfo.dfxOpInfo_->isIndop_ == true) {
439 0 : hcclReportData.profInfo.groupName = groupName;
440 0 : hcclReportData.profInfo.rankSize = taskInfo.dfxOpInfo_->rankSize_;
441 : } else {
442 8 : CommunicatorImpl* commImp = static_cast<CommunicatorImpl*>(taskInfo.dfxOpInfo_->comm_);
443 8 : if (commImp == nullptr) {
444 0 : HCCL_ERROR("[ProfilingHandler]commImp is nullptr");
445 0 : return;
446 : }
447 8 : hcclReportData.profInfo.groupName = groupName;
448 8 : hcclReportData.profInfo.rankSize = commImp->GetRankSize();
449 : }
450 8 : hcclReportData.profInfo.workFlowMode = static_cast<u32>(HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE);
451 8 : hcclReportData.profInfo.planeID = 0;
452 8 : hcclReportData.profInfo.stage = 0;
453 8 : hcclReportData.profInfo.role = static_cast<uint32_t>(TaskRole::DST);
454 8 : hcclReportData.profInfo.durationEstimated = 0;
455 8 : hcclReportData.profInfo.localRank = taskInfo.dfxOpInfo_->op_.myRank;
456 8 : hcclReportData.profInfo.remoteRank = taskInfo.remoteRank_;
457 8 : hcclReportData.profInfo.dataType = taskInfo.dfxOpInfo_->op_.dataType;
458 8 : hcclReportData.profInfo.opType = taskInfo.dfxOpInfo_->op_.opType;
459 8 : hcclReportData.profInfo.transportType = static_cast<int32_t>(SimpleTaskType::UB);
460 8 : if (hcclReportData.profInfo.remoteRank == INVALID_VALUE_RANKID) {
461 0 : hcclReportData.profInfo.transportType = static_cast<int32_t>(SimpleTaskType::LOCAL);
462 0 : hcclReportData.profInfo.remoteRank = hcclReportData.profInfo.localRank;
463 0 : hcclReportData.profInfo.linkType = 0;
464 : }
465 8 : const auto& taskType = taskInfo.taskParam_.taskType;
466 8 : const auto& taskPara = taskInfo.taskParam_.taskPara;
467 8 : switch (taskType) {
468 4 : case TaskParamType::TASK_DPU_INLINE_WRITE:
469 : case TaskParamType::TASK_DPU_NOTIFY_WAIT:
470 : case TaskParamType::TASK_DPU_WRITE_WITH_NOTIFY:
471 : case TaskParamType::TASK_DPU_CHANNEL_FENCE:
472 4 : hcclReportData.dpuProfInfo.itemId = hcclReportData.profInfo.itemId;
473 4 : hcclReportData.dpuProfInfo.cclTag = hcclReportData.profInfo.cclTag;
474 4 : hcclReportData.dpuProfInfo.groupName = hcclReportData.profInfo.groupName;
475 4 : hcclReportData.dpuProfInfo.localRank = hcclReportData.profInfo.localRank;
476 4 : hcclReportData.dpuProfInfo.remoteRank = hcclReportData.profInfo.remoteRank;
477 4 : hcclReportData.dpuProfInfo.rankSize = hcclReportData.profInfo.rankSize;
478 4 : hcclReportData.dpuProfInfo.workFlowMode = hcclReportData.profInfo.workFlowMode;
479 4 : hcclReportData.dpuProfInfo.planeID = hcclReportData.profInfo.planeID;
480 4 : hcclReportData.dpuProfInfo.stage = hcclReportData.profInfo.stage;
481 4 : hcclReportData.dpuProfInfo.role = hcclReportData.profInfo.role;
482 4 : hcclReportData.dpuProfInfo.durationEstimated = hcclReportData.profInfo.durationEstimated;
483 4 : hcclReportData.dpuProfInfo.dataType = static_cast<uint32_t>(taskInfo.dfxOpInfo_->op_.dataType);
484 4 : hcclReportData.dpuProfInfo.opType = static_cast<uint32_t>(taskInfo.dfxOpInfo_->op_.opType);
485 4 : hcclReportData.dpuProfInfo.transportType = static_cast<uint32_t>(SimpleTaskType::ROCE);
486 4 : hcclReportData.dpuProfInfo.aicpu_task_id = taskInfo.taskParam_.aicpuTaskId;
487 4 : hcclReportData.dpuProfInfo.npuDevId = taskInfo.taskParam_.npuDevId;
488 4 : hcclReportData.dpuProfInfo.dpuDevId = DPU_DEV_ID_MASK;
489 4 : hcclReportData.dpuProfInfo.linkType = static_cast<uint16_t>(taskPara.DMA.linkType);
490 4 : if (hcclReportData.dpuProfInfo.remoteRank == INVALID_VALUE_RANKID) {
491 0 : hcclReportData.dpuProfInfo.transportType = static_cast<uint32_t>(SimpleTaskType::LOCAL);
492 0 : hcclReportData.dpuProfInfo.remoteRank = hcclReportData.dpuProfInfo.localRank;
493 0 : hcclReportData.dpuProfInfo.linkType = 0;
494 : }
495 4 : hcclReportData.dpuProfInfo.taskId = taskInfo.taskId_;
496 4 : hcclReportData.dpuProfInfo.streamId = taskInfo.streamId_;
497 4 : hcclReportData.dpuProfInfo.timeStamp = taskInfo.taskParam_.beginTime;
498 4 : switch (taskType) {
499 2 : case TaskParamType::TASK_DPU_INLINE_WRITE:
500 : case TaskParamType::TASK_DPU_WRITE_WITH_NOTIFY:
501 : hcclReportData.dpuProfInfo.srcAddr
502 2 : = static_cast<u64>(reinterpret_cast<uintptr_t>(taskPara.DMA.src));
503 : hcclReportData.dpuProfInfo.dstAddr
504 2 : = static_cast<u64>(reinterpret_cast<uintptr_t>(taskPara.DMA.dst));
505 2 : hcclReportData.dpuProfInfo.dataSize = static_cast<u32>(taskPara.DMA.size);
506 2 : hcclReportData.dpuProfInfo.notifyID = taskPara.DMA.notifyID;
507 2 : break;
508 2 : case TaskParamType::TASK_DPU_NOTIFY_WAIT:
509 : case TaskParamType::TASK_DPU_CHANNEL_FENCE:
510 2 : hcclReportData.dpuProfInfo.notifyID = taskPara.Notify.notifyID;
511 2 : break;
512 0 : default:
513 0 : break;
514 : }
515 4 : break;
516 4 : default:
517 4 : switch (taskType) {
518 1 : case TaskParamType::TASK_SDMA:
519 : case TaskParamType::TASK_RDMA:
520 1 : hcclReportData.profInfo.srcAddr = static_cast<u64>(reinterpret_cast<uintptr_t>(taskPara.DMA.src));
521 1 : hcclReportData.profInfo.dstAddr = static_cast<u64>(reinterpret_cast<uintptr_t>(taskPara.DMA.dst));
522 1 : hcclReportData.profInfo.dataSize = static_cast<u32>(taskPara.DMA.size);
523 1 : hcclReportData.profInfo.notifyID = taskPara.DMA.notifyID;
524 1 : hcclReportData.profInfo.linkType = static_cast<uint16_t>(taskPara.DMA.linkType);
525 1 : break;
526 1 : case TaskParamType::TASK_REDUCE_INLINE:
527 : case TaskParamType::TASK_REDUCE_TBE:
528 : hcclReportData.profInfo.srcAddr
529 1 : = static_cast<u64>(reinterpret_cast<uintptr_t>(taskPara.Reduce.src));
530 : hcclReportData.profInfo.dstAddr
531 1 : = static_cast<u64>(reinterpret_cast<uintptr_t>(taskPara.Reduce.dst));
532 1 : hcclReportData.profInfo.dataSize = static_cast<u32>(taskPara.Reduce.size);
533 1 : hcclReportData.profInfo.notifyID = taskPara.Reduce.notifyID;
534 1 : hcclReportData.profInfo.linkType = static_cast<uint16_t>(taskPara.Reduce.linkType);
535 1 : break;
536 1 : case TaskParamType::TASK_NOTIFY_RECORD:
537 : case TaskParamType::TASK_NOTIFY_WAIT:
538 1 : hcclReportData.profInfo.notifyID = taskPara.Notify.notifyID;
539 1 : break;
540 1 : default:
541 1 : break;
542 : }
543 4 : break;
544 : }
545 8 : }
546 :
547 1 : void ProfilingHandler::DumpHCCLReportData(const TaskInfo& taskInfo, const MsprofAdditionalInfo& reporterData) const
548 : {
549 1 : if (reporterData.type == static_cast<uint32_t>(ProfTaskType::TASK_HCCL_INFO)) {
550 1 : auto* profInfo = reinterpret_cast<const MsprofHcclInfo*>(reporterData.data);
551 3 : HCCL_INFO(
552 : "MsprofAdditionalInfo profInfo: timeStamp[%llu], itemId[%llu], cclTag[%llu], groupName[%llu], "
553 : "localRank[%u], remoteRank[%u], rankSize[%u], workFlowMode[%u], planeID[%u], ctxId[%u], "
554 : "stage[%u], role[%u], durationEstimated[%f], taskType[%d]",
555 : reporterData.timeStamp, profInfo->itemId, profInfo->cclTag, profInfo->groupName, profInfo->localRank,
556 : profInfo->remoteRank, profInfo->rankSize, profInfo->workFlowMode, profInfo->planeID, profInfo->ctxId,
557 : profInfo->stage, profInfo->role, profInfo->durationEstimated,
558 : static_cast<int>(taskInfo.taskParam_.taskType));
559 3 : HCCL_INFO(
560 : "MsprofAdditionalInfo profInfo detail: srcAddr[%llu], dstAddr[%llu], dataSize[%llu], notifyID[%llu], "
561 : "linkType[%u], opType[%s], transportType[%u], dataType[%s], rdmaType[%u]",
562 : profInfo->srcAddr, profInfo->dstAddr, profInfo->dataSize, profInfo->notifyID, profInfo->linkType,
563 : OpTypeToSerialString(profInfo->opType).c_str(), profInfo->transportType,
564 : DataTypeToSerialString(profInfo->dataType).c_str(), profInfo->rdmaType);
565 : } else {
566 0 : auto* dpuProfInfo = reinterpret_cast<const MsprofDpuHcclTrack*>(reporterData.data);
567 0 : HCCL_INFO(
568 : "MsprofAdditionalInfo dpuProfInfo: timeStamp[%llu], itemId[%llu], cclTag[%llu], groupName[%llu], "
569 : "localRank[%u], remoteRank[%u], rankSize[%u], workFlowMode[%u], planeID[%u], "
570 : "stage[%u], role[%u], durationEstimated[%f], taskType[%d]",
571 : reporterData.timeStamp, dpuProfInfo->itemId, dpuProfInfo->cclTag, dpuProfInfo->groupName,
572 : dpuProfInfo->localRank, dpuProfInfo->remoteRank, dpuProfInfo->rankSize, dpuProfInfo->workFlowMode,
573 : dpuProfInfo->planeID, dpuProfInfo->stage, dpuProfInfo->role, dpuProfInfo->durationEstimated,
574 : static_cast<int>(taskInfo.taskParam_.taskType));
575 0 : HCCL_INFO(
576 : "MsprofAdditionalInfo dpuProfInfo detail: srcAddr[%llu], dstAddr[%llu], dataSize[%llu], notifyID[%llu], "
577 : "linkType[%u], opType[%s], transportType[%u], dataType[%s], rdmaType[%u], "
578 : "taskId[%u], aicpu_task_id[%u], streamId[%u], npuDevId[%u], dpuDevId[%u], timeStamp[%llu]",
579 : dpuProfInfo->srcAddr, dpuProfInfo->dstAddr, dpuProfInfo->dataSize, dpuProfInfo->notifyID,
580 : dpuProfInfo->linkType, OpTypeToSerialString(dpuProfInfo->opType).c_str(), dpuProfInfo->transportType,
581 : DataTypeToSerialString(dpuProfInfo->dataType).c_str(), dpuProfInfo->rdmaType, dpuProfInfo->taskId,
582 : dpuProfInfo->aicpu_task_id, dpuProfInfo->streamId, dpuProfInfo->npuDevId, dpuProfInfo->dpuDevId,
583 : dpuProfInfo->timeStamp);
584 : }
585 1 : }
586 :
587 0 : void ProfilingHandler::LogCcuTaskInfo(
588 : const CcuProfilingInfo& info, const TaskInfo& taskInfo, uint64_t itemId, uint64_t groupName, u32 rankId,
589 : u32 ranksize) const
590 : {
591 0 : HCCL_INFO(
592 : "[ProfilingHandler]GetCcuTaskInfo, ccuTaskInfo data is: version[%u], workFlowMode[%u], itemId[%llu], "
593 : "groupName[%llu], rankId[%u], ranksize[%u], streamId[%u], taskId[%u], dieId[%u], "
594 : "missionId[%u],instrId[%u]",
595 : 0, static_cast<u32>(HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE), itemId, groupName, rankId, ranksize,
596 : taskInfo.streamId_, taskInfo.taskId_, info.dieId, info.missionId, info.instrId);
597 0 : }
598 :
599 0 : void ProfilingHandler::LogCcuWaitSignalInfo(
600 : const CcuProfilingInfo& info, const TaskInfo& taskInfo, uint64_t itemId, uint64_t groupName, u32 rankId,
601 : u32 ranksize) const
602 : {
603 0 : HCCL_INFO(
604 : "[ProfilingHandler]GetCcuWaitSignalInfo, waitSignalInfo data is: version[%u], itemId[%llu], groupName[%llu], "
605 : "rankId[%u], ranksize[%u], workFlowMode[%u], streamId[%u], taskId[%u], dieId[%u],instrId[%u],missionId[%u], "
606 : "ckeId[%u],mask[%u]",
607 : 0, itemId, groupName, rankId, ranksize, static_cast<u32>(HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE),
608 : taskInfo.streamId_, taskInfo.taskId_, info.dieId, info.instrId, info.missionId, info.ckeId, info.mask);
609 0 : }
610 :
611 0 : void ProfilingHandler::LogCcuGroupInfo(
612 : const CcuProfilingInfo& info, const TaskInfo& taskInfo, uint64_t itemId, uint64_t groupName, u32 rankId,
613 : u32 ranksize) const
614 : {
615 0 : HCCL_INFO(
616 : "[ProfilingHandler]GetCcuGroupInfo, ccuGroupInfo data is: version[%u], itemId[%llu], "
617 : "groupName[%llu], rankId[%u], ranksize[%u], workFlowMode[%u], streamId[%u], taskId[%u], "
618 : "dieId[%u],instrId[%u],missionId[%u], dataSize[%llu]",
619 : 0, itemId, groupName, rankId, ranksize, static_cast<u32>(HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE),
620 : taskInfo.streamId_, taskInfo.taskId_, info.dieId, info.instrId, info.missionId, info.dataSize);
621 0 : if (info.reduceOpType != INVALID_TYPE_VALUE) {
622 0 : HCCL_INFO("ccuGroupInfo reduceOpType is [%d]", static_cast<int>(info.reduceOpType));
623 : }
624 0 : if (info.inputDataType != INVALID_TYPE_VALUE) {
625 0 : HCCL_INFO("ccuGroupInfo inputDataType is [%d]", static_cast<int>(info.inputDataType));
626 : }
627 0 : if (info.outputDataType != INVALID_TYPE_VALUE) {
628 0 : HCCL_INFO("ccuGroupInfo outputDataType is [%d]", static_cast<int>(info.outputDataType));
629 : }
630 0 : for (auto i = 0; i < CCU_MAX_CHANNEL_NUM; i++) {
631 0 : if (info.channelId[i] != INVALID_VALUE_CHANNELID && info.remoteRankId[i] != INVALID_VALUE_RANKID) {
632 0 : HCCL_INFO(
633 : "[ProfilingHandler]GetCcuGroupInfo, ccuGroupInfo data is: channelId[%d] = %u, "
634 : "remoteRankId[%d] = %u",
635 : i, info.channelId[i], i, info.remoteRankId[i]);
636 : }
637 : }
638 0 : }
639 :
640 4 : void ProfilingHandler::ReportCcuInfoLog(const TaskInfo& taskInfo) const
641 : {
642 4 : if (LIKELY(HcclCheckLogLevel(HCCL_LOG_INFO) == 0)) {
643 4 : return;
644 : }
645 4 : if (!enableHcclL1_) {
646 0 : return;
647 : }
648 4 : if (taskInfo.taskParam_.ccuDetailInfo == nullptr || taskInfo.dfxOpInfo_ == nullptr) {
649 4 : return;
650 : }
651 0 : auto ccuDetailInfo = taskInfo.taskParam_.ccuDetailInfo;
652 0 : for (const auto& info : *ccuDetailInfo) {
653 0 : uint64_t itemId = GetProfHashId(info.name.c_str(), info.name.length());
654 : uint64_t groupName
655 0 : = GetProfHashId(taskInfo.dfxOpInfo_->op_.opTag.c_str(), taskInfo.dfxOpInfo_->op_.opTag.length());
656 0 : u32 rankId = 0;
657 0 : u32 ranksize = 0;
658 0 : if (taskInfo.dfxOpInfo_->isIndop_ == true) {
659 0 : rankId = taskInfo.dfxOpInfo_->op_.myRank;
660 0 : ranksize = taskInfo.dfxOpInfo_->rankSize_;
661 : } else {
662 0 : CommunicatorImpl* commImp = static_cast<CommunicatorImpl*>(taskInfo.dfxOpInfo_->comm_);
663 0 : rankId = commImp->GetIdIndex();
664 0 : ranksize = commImp->GetRankSize();
665 : }
666 0 : if (info.type == 0) {
667 0 : LogCcuTaskInfo(info, taskInfo, itemId, groupName, rankId, ranksize);
668 0 : } else if (info.type == 1) {
669 0 : LogCcuWaitSignalInfo(info, taskInfo, itemId, groupName, rankId, ranksize);
670 0 : } else if (info.type == CCU_TYPE) {
671 0 : LogCcuGroupInfo(info, taskInfo, itemId, groupName, rankId, ranksize);
672 : }
673 : }
674 0 : }
675 :
676 4 : void ProfilingHandler::ReportCcuInfo(const TaskInfo& taskInfo) const
677 : {
678 4 : ReportCcuInfoLog(taskInfo);
679 4 : if (taskInfo.taskParam_.ccuDetailInfo == nullptr) {
680 9 : HCCL_ERROR("[ProfilingHandler]ReportCcuInfo ccuDetailInfo is nullptr.");
681 4 : return;
682 : }
683 1 : if (taskInfo.dfxOpInfo_ == nullptr) {
684 3 : HCCL_WARNING("[ProfilingHandler]ReportCcuInfo dfxOpInfo_ is nullptr, skip ReportCcuInfo.");
685 1 : return;
686 : }
687 0 : auto ccuDetailInfo = taskInfo.taskParam_.ccuDetailInfo;
688 0 : for (const auto& info : *ccuDetailInfo) {
689 0 : if (info.type == 0 && enableHcclL1_) {
690 0 : GetCcuTaskInfo(taskInfo, info);
691 0 : } else if (info.type == 1 && enableHcclL1_) {
692 0 : GetCcuWaitSignalInfo(taskInfo, info);
693 0 : } else if (info.type == CCU_TYPE && enableHcclL1_) {
694 0 : GetCcuGroupInfo(taskInfo, info);
695 : }
696 : }
697 0 : }
698 :
699 0 : void ProfilingHandler::GetCcuTaskInfo(const TaskInfo& taskInfo, const CcuProfilingInfo& info) const
700 : {
701 0 : uint64_t timestamp = DlProfFunction::GetInstance().dlMsprofSysCycleTime();
702 0 : MsprofAdditionalInfo reporterData{};
703 0 : reporterData.level = MSPROF_REPORT_HCCL_NODE_LEVEL;
704 0 : reporterData.type = MSPROF_REPORT_CCU_TASK_INFO;
705 0 : reporterData.threadId = SalGetTid();
706 0 : reporterData.dataLen = sizeof(MsprofCcuTaskInfo);
707 0 : reporterData.timeStamp = timestamp;
708 0 : auto* ccuTaskInfo = reinterpret_cast<MsprofCcuTaskInfo*>(reporterData.data);
709 0 : ccuTaskInfo->version = 0;
710 0 : ccuTaskInfo->workFlowMode = static_cast<u32>(HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE);
711 0 : ccuTaskInfo->itemId = GetProfHashId(info.name.c_str(), info.name.length());
712 0 : uint64_t groupName = GetProfHashId(taskInfo.dfxOpInfo_->op_.opTag.c_str(), taskInfo.dfxOpInfo_->op_.opTag.length());
713 0 : ccuTaskInfo->groupName = groupName;
714 0 : if (taskInfo.dfxOpInfo_->isIndop_ == true) {
715 0 : ccuTaskInfo->rankId = taskInfo.dfxOpInfo_->op_.myRank;
716 0 : ccuTaskInfo->ranksize = taskInfo.dfxOpInfo_->rankSize_;
717 : } else {
718 0 : CommunicatorImpl* commImp = static_cast<CommunicatorImpl*>(taskInfo.dfxOpInfo_->comm_);
719 0 : ccuTaskInfo->rankId = commImp->GetIdIndex();
720 0 : ccuTaskInfo->ranksize = commImp->GetRankSize();
721 : }
722 :
723 0 : ccuTaskInfo->streamId = taskInfo.streamId_;
724 0 : ccuTaskInfo->taskId = taskInfo.taskId_;
725 0 : ccuTaskInfo->dieId = info.dieId;
726 0 : ccuTaskInfo->missionId = info.missionId;
727 0 : ccuTaskInfo->instrId = info.instrId;
728 0 : ReportAdditionInfo(reporterData);
729 0 : }
730 :
731 1 : void ProfilingHandler::GetCcuGroupInfo(const TaskInfo& taskInfo, const CcuProfilingInfo& info) const
732 : {
733 1 : MsprofAdditionalInfo reporterData{};
734 1 : reporterData.level = MSPROF_REPORT_HCCL_NODE_LEVEL;
735 1 : reporterData.type = MSPROF_REPORT_CCU_GROUP_INFO;
736 1 : reporterData.threadId = SalGetTid();
737 1 : reporterData.dataLen = sizeof(MsprofCcuGroupInfo);
738 1 : reporterData.timeStamp = DlProfFunction::GetInstance().dlMsprofSysCycleTime();
739 1 : auto* ccuGroupInfo = reinterpret_cast<MsprofCcuGroupInfo*>(reporterData.data);
740 1 : ccuGroupInfo->version = 0;
741 1 : ccuGroupInfo->itemId = GetProfHashId(info.name.c_str(), info.name.length());
742 1 : uint64_t groupName = GetProfHashId(taskInfo.dfxOpInfo_->op_.opTag.c_str(), taskInfo.dfxOpInfo_->op_.opTag.length());
743 1 : ccuGroupInfo->groupName = groupName;
744 1 : if (taskInfo.dfxOpInfo_->isIndop_ == true) {
745 0 : ccuGroupInfo->rankId = taskInfo.dfxOpInfo_->op_.myRank;
746 0 : ccuGroupInfo->ranksize = taskInfo.dfxOpInfo_->rankSize_;
747 : } else {
748 1 : CommunicatorImpl* commImp = static_cast<CommunicatorImpl*>(taskInfo.dfxOpInfo_->comm_);
749 1 : ccuGroupInfo->rankId = commImp->GetIdIndex();
750 1 : ccuGroupInfo->ranksize = commImp->GetRankSize();
751 : }
752 1 : ccuGroupInfo->workFlowMode = static_cast<u32>(HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE);
753 1 : ccuGroupInfo->streamId = taskInfo.streamId_;
754 1 : ccuGroupInfo->taskId = taskInfo.taskId_;
755 1 : ccuGroupInfo->dieId = info.dieId;
756 1 : ccuGroupInfo->instrId = info.instrId;
757 1 : ccuGroupInfo->missionId = info.missionId;
758 1 : ccuGroupInfo->reduceOpType = info.reduceOpType;
759 1 : ccuGroupInfo->inputDataType = info.inputDataType;
760 1 : ccuGroupInfo->outputDataType = info.outputDataType;
761 1 : ccuGroupInfo->dataSize = info.dataSize;
762 1 : std::copy(info.channelId, info.channelId + CCU_MAX_CHANNEL_NUM, ccuGroupInfo->channelId);
763 1 : std::copy(info.remoteRankId, info.remoteRankId + CCU_MAX_CHANNEL_NUM, ccuGroupInfo->remoteRankId);
764 1 : ReportAdditionInfo(reporterData);
765 1 : }
766 :
767 0 : void ProfilingHandler::DumpCcuGroupInfo([[maybe_unused]] const MsprofCcuGroupInfo& ccuGroupInfo) const
768 : {
769 0 : if (LIKELY(HcclCheckLogLevel(HCCL_LOG_INFO) == 0)) {
770 0 : return;
771 : }
772 : // HCCL_INFOs migrated to ReportCcuInfoLog
773 : }
774 :
775 1 : void ProfilingHandler::GetCcuWaitSignalInfo(const TaskInfo& taskInfo, const CcuProfilingInfo& info) const
776 : {
777 1 : uint64_t timestamp = DlProfFunction::GetInstance().dlMsprofSysCycleTime();
778 1 : MsprofAdditionalInfo reporterData{};
779 1 : reporterData.level = MSPROF_REPORT_HCCL_NODE_LEVEL;
780 1 : reporterData.type = MSPROF_REPORT_CCU_WAIT_SIGNAL_INFO;
781 1 : reporterData.threadId = SalGetTid();
782 1 : reporterData.dataLen = sizeof(MsprofCcuWaitSignalInfo);
783 1 : reporterData.timeStamp = timestamp;
784 1 : auto* waitSignalInfo = reinterpret_cast<MsprofCcuWaitSignalInfo*>(reporterData.data);
785 1 : waitSignalInfo->version = 0;
786 1 : waitSignalInfo->itemId = GetProfHashId(info.name.c_str(), info.name.length());
787 1 : uint64_t groupName = GetProfHashId(taskInfo.dfxOpInfo_->op_.opTag.c_str(), taskInfo.dfxOpInfo_->op_.opTag.length());
788 1 : waitSignalInfo->groupName = groupName;
789 1 : if (taskInfo.dfxOpInfo_->isIndop_ == true) {
790 0 : waitSignalInfo->rankId = taskInfo.dfxOpInfo_->op_.myRank;
791 0 : waitSignalInfo->ranksize = taskInfo.dfxOpInfo_->rankSize_;
792 : } else {
793 1 : CommunicatorImpl* commImp = static_cast<CommunicatorImpl*>(taskInfo.dfxOpInfo_->comm_);
794 1 : waitSignalInfo->rankId = commImp->GetIdIndex();
795 1 : waitSignalInfo->ranksize = commImp->GetRankSize();
796 : }
797 1 : waitSignalInfo->workFlowMode = static_cast<u32>(HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE);
798 1 : waitSignalInfo->streamId = taskInfo.streamId_;
799 1 : waitSignalInfo->taskId = taskInfo.taskId_;
800 1 : waitSignalInfo->dieId = info.dieId;
801 1 : waitSignalInfo->instrId = info.instrId;
802 1 : waitSignalInfo->missionId = info.missionId;
803 1 : waitSignalInfo->ckeId = info.ckeId;
804 1 : waitSignalInfo->mask = info.mask;
805 1 : std::copy(info.channelId, info.channelId + CCU_MAX_CHANNEL_NUM, waitSignalInfo->channelId);
806 1 : std::copy(info.remoteRankId, info.remoteRankId + CCU_MAX_CHANNEL_NUM, waitSignalInfo->remoteRankId);
807 1 : ReportAdditionInfo(reporterData);
808 1 : }
809 :
810 2 : void ProfilingHandler::ReportAclApi(
811 : uint32_t cmdType, uint64_t beginTime, uint64_t endTime, uint64_t cmdItemId, uint32_t threadId, bool cachedReq)
812 : {
813 2 : MsprofApi reporterData{};
814 2 : reporterData.level = MSPROF_REPORT_ACL_LEVEL;
815 2 : reporterData.type = static_cast<int32_t>(cmdType) + MSPROF_REPORT_ACL_HOST_HCCL_BASE_TYPE;
816 2 : reporterData.threadId = threadId;
817 2 : reporterData.beginTime = beginTime;
818 2 : reporterData.endTime = endTime;
819 2 : reporterData.itemId = cmdItemId;
820 :
821 2 : if (cachedReq) {
822 1 : std::lock_guard<std::mutex> lock(cachedAclApiInfoMutex_);
823 3 : HCCL_INFO(
824 : "ReportAclApi MSPROF_REPORT_ACL_LEVEL type[%u] beginTime[%llu]", reporterData.type, reporterData.beginTime);
825 1 : cachedAclApiInfo_.push(reporterData);
826 1 : }
827 :
828 2 : s32 ret = DlProfFunction::GetInstance().dlMsprofReportApi(1, &reporterData);
829 6 : HCCL_INFO(
830 : "[ProfilingHandler][ReportAclApi], reporterData data is: level[%u], type[%u], threadId[%u], "
831 : "beginTime[%llu], endTime[%llu], itemId[%llu], return value[%d]",
832 : reporterData.level, reporterData.type, reporterData.threadId, reporterData.beginTime, reporterData.endTime,
833 : reporterData.itemId, ret);
834 2 : if (ret != 0) {
835 0 : THROW<InternalException>("Call dlMsprofReportApi failed, return[%d]", ret);
836 : }
837 2 : }
838 :
839 4 : void ProfilingHandler::ReportNodeApi(
840 : uint64_t beginTime, uint64_t endTime, uint64_t cmdItemId, uint32_t threadId, bool cachedReq)
841 : {
842 4 : MsprofApi reporterData{};
843 4 : reporterData.level = MSPROF_REPORT_NODE_LEVEL;
844 4 : reporterData.type = MSPROF_REPORT_NODE_LAUNCH_TYPE;
845 4 : reporterData.threadId = threadId;
846 4 : reporterData.beginTime = beginTime;
847 4 : reporterData.endTime = endTime;
848 4 : reporterData.itemId = cmdItemId;
849 :
850 4 : if (cachedReq) {
851 2 : std::lock_guard<std::mutex> lock(cachedTaskApiInfoMutex_);
852 2 : cachedTaskApiInfo_.push(reporterData);
853 2 : }
854 4 : if (!enableHostApi_) {
855 3 : return;
856 : }
857 :
858 1 : s32 ret = DlProfFunction::GetInstance().dlMsprofReportApi(1, &reporterData);
859 3 : HCCL_INFO(
860 : "[ProfilingHandler][ReportNodeApi], reporterData data is: level[%u], type[%u], threadId[%u], "
861 : "beginTime[%llu], endTime[%llu], itemId[%llu], return value[%d]",
862 : reporterData.level, reporterData.type, reporterData.threadId, reporterData.beginTime, reporterData.endTime,
863 : reporterData.itemId, ret);
864 1 : if (ret != 0) {
865 0 : THROW<InternalException>("Call MsprofReportApi failed, return[%d]", ret);
866 : }
867 : }
868 :
869 4 : void ProfilingHandler::ReportNodeBasicInfo(uint64_t timeStamp, uint64_t cmdItemId, uint32_t threadId, bool cachedReq)
870 : {
871 4 : MsprofCompactInfo reporterData{};
872 4 : reporterData.level = MSPROF_REPORT_NODE_LEVEL;
873 4 : reporterData.type = MSPROF_REPORT_NODE_BASIC_INFO_TYPE;
874 4 : reporterData.threadId = threadId;
875 4 : reporterData.dataLen = sizeof(MsprofNodeBasicInfo);
876 4 : reporterData.timeStamp = timeStamp;
877 4 : reporterData.data.nodeBasicInfo.opName = cmdItemId;
878 4 : reporterData.data.nodeBasicInfo.taskType = MSPROF_GE_TASK_TYPE_HCCL;
879 4 : reporterData.data.nodeBasicInfo.opType = cmdItemId;
880 4 : reporterData.data.nodeBasicInfo.opFlag = 0;
881 12 : HCCL_INFO(
882 : "[ProfilingHandler][ReportNodeBasicInfo], reporterData data is: level[%u], type[%u], threadId[%u], "
883 : "dataLen[%u], taskType[%u], opFlag[%u]",
884 : reporterData.level, reporterData.type, reporterData.threadId, reporterData.dataLen,
885 : reporterData.data.nodeBasicInfo.taskType, reporterData.data.nodeBasicInfo.opFlag);
886 4 : if (cachedReq) {
887 2 : std::lock_guard<std::mutex> lock(cacheHcclOpInfoMutex_);
888 2 : cacheHcclOpInfo_.push(reporterData);
889 2 : }
890 4 : if (!enableHcclL1_) {
891 2 : return;
892 : }
893 2 : s32 ret = DlProfFunction::GetInstance().dlMsprofReportCompactInfo(1, &reporterData, sizeof(MsprofCompactInfo));
894 6 : HCCL_INFO("Call MsprofReportCompactInfo, return value[%d]", ret);
895 2 : if (ret != 0) {
896 0 : THROW<InternalException>("Call MsprofReportCompactInfo failed, return[%d]", ret);
897 : }
898 : }
899 :
900 5 : void ProfilingHandler::ReportHcclOpInfo(uint64_t timeStamp, const DfxOpInfo& opInfo, uint32_t threadId, bool cachedReq)
901 : {
902 5 : MsprofCompactInfo reporterData{};
903 5 : reporterData.level = MSPROF_REPORT_NODE_LEVEL;
904 5 : reporterData.type = MSPROF_REPORT_NODE_HCCL_OP_INFO_TYPE;
905 5 : reporterData.threadId = threadId;
906 5 : reporterData.dataLen = sizeof(MsprofHCCLOPInfo);
907 5 : reporterData.timeStamp = timeStamp;
908 5 : reporterData.data.hcclopInfo.relay = 0;
909 5 : reporterData.data.hcclopInfo.retry = 0;
910 5 : reporterData.data.hcclopInfo.dataType = opInfo.op_.dataType;
911 5 : reporterData.data.hcclopInfo.algType = cachedAlgTypeHashId_.load();
912 5 : uint64_t groupName = GetProfHashId(opInfo.op_.opTag.c_str(), opInfo.op_.opTag.length());
913 5 : reporterData.data.hcclopInfo.groupName = groupName;
914 5 : u32 ranksize{0};
915 5 : if (opInfo.isIndop_ == true) {
916 0 : ranksize = opInfo.rankSize_;
917 0 : reporterData.data.hcclopInfo.count = opInfo.op_.dataCount;
918 : } else {
919 5 : CommunicatorImpl* commImp = static_cast<CommunicatorImpl*>(opInfo.comm_);
920 5 : ranksize = commImp->GetRankSize();
921 5 : if (opInfo.op_.opType == OpType::ALLTOALLV) {
922 1 : u64 sendCount = 0;
923 3 : for (u64 i = 0; i < ranksize; i++) {
924 2 : sendCount += *(static_cast<const u64*>(opInfo.op_.all2AllVDataDes.sendCounts) + i);
925 : }
926 1 : reporterData.data.hcclopInfo.count = sendCount;
927 4 : } else if (opInfo.op_.opType == OpType::ALLTOALL) {
928 0 : reporterData.data.hcclopInfo.count = opInfo.op_.all2AllDataDes.sendCount;
929 : } else {
930 4 : reporterData.data.hcclopInfo.count = opInfo.op_.dataCount;
931 : }
932 : }
933 5 : if (cachedReq) {
934 2 : std::lock_guard<std::mutex> lock(cacheHcclOpInfoMutex_);
935 2 : cacheHcclOpInfo_.push(reporterData);
936 2 : }
937 5 : if (!enableHostApi_) {
938 3 : return;
939 : }
940 6 : HCCL_INFO(
941 : "[ProfilingHandler][ReportHcclOpInfo], data is: level[%u], type[%u], threadId[%u], dataLen[%u], "
942 : "timeStamp[%llu], relay [%u], retry[%u], dataType[%s], algType[%llu], groupName[%llu], count[%llu]",
943 : reporterData.level, reporterData.type, reporterData.threadId, reporterData.dataLen, reporterData.timeStamp,
944 : reporterData.data.hcclopInfo.relay, reporterData.data.hcclopInfo.retry,
945 : DataTypeToSerialString(reporterData.data.hcclopInfo.dataType).c_str(), reporterData.data.hcclopInfo.algType,
946 : reporterData.data.hcclopInfo.groupName, reporterData.data.hcclopInfo.count);
947 2 : s32 ret = DlProfFunction::GetInstance().dlMsprofReportCompactInfo(1, &reporterData, sizeof(MsprofCompactInfo));
948 2 : if (ret != 0) {
949 0 : THROW<InternalException>("[ProfilingHandler] Call dlMsprofReportCompactInfo failed, return[%d]", ret);
950 : }
951 : }
952 :
953 2 : void ProfilingHandler::ReportAdditionInfo(MsprofAdditionalInfo& reporterData) const
954 : {
955 : s32 ret
956 2 : = DlProfFunction::GetInstance().dlMsprofReportAdditionalInfo(0, &reporterData, sizeof(MsprofAdditionalInfo));
957 6 : HCCL_INFO(
958 : "[ProfilingHandler][ReportAdditionInfo], level[%u], type[%u], threadId[%u], dataLen[%u], "
959 : "timeStamp[%llu], return value[%d]",
960 : reporterData.level, reporterData.type, reporterData.threadId, reporterData.dataLen, reporterData.timeStamp,
961 : ret);
962 2 : if (ret != 0) {
963 0 : THROW<InternalException>("Call MsprofReportAdditionalInfo failed, return[%d]", ret);
964 : }
965 2 : }
966 :
967 2 : int32_t ProfilingHandler::CommandHandle(uint32_t rtType, void* data, uint32_t len) const
968 : {
969 : (void)len;
970 2 : if (data == nullptr || rtType != rtProfCtrlType_t::RT_PROF_CTRL_SWITCH) {
971 3 : HCCL_ERROR("[ProfilingHandler][CommandHandle] data is nullptr or rtType is invalid, rtType[%u]", rtType);
972 1 : return HCCL_E_PARA;
973 : }
974 1 : rtProfCommandHandle_t* profConfigParam = static_cast<rtProfCommandHandle_t*>(data);
975 1 : auto type = profConfigParam->type;
976 1 : auto profconfig = profConfigParam->profSwitch;
977 3 : HCCL_RUN_INFO(
978 : "[Profiling][CommandHandle] CommandHandle's rtType is %u. CommandHandle_switch type[%u], "
979 : "profconfig[%llu], deviceLogicId[%u]",
980 : rtType, type, profconfig, profConfigParam->devIdList[0]);
981 1 : switch (type) {
982 0 : case PROF_COMMANDHANDLE_TYPE_START:
983 0 : instance_.StartSubscribe(profconfig);
984 0 : break;
985 0 : case PROF_COMMANDHANDLE_TYPE_STOP:
986 0 : instance_.StopSubscribe();
987 0 : break;
988 1 : default:
989 3 : HCCL_RUN_INFO("[Profiling][CommandHandle] Unexpected behaviour.");
990 : }
991 1 : return HCCL_SUCCESS;
992 : }
993 :
994 5 : void ProfilingHandler::StartSubscribe(uint64_t profconfig)
995 : {
996 15 : HCCL_RUN_INFO("[Profiling][CommandHandle] profSwitch is[%llu]", profconfig);
997 5 : SetCachedCclTag();
998 5 : cachedAlgTypeHashId_ = GetProfHashId("AlgType::NHR", strlen("AlgType::NHR"));
999 5 : if ((profconfig & PROF_ACL_API_MASK) != 0) {
1000 1 : StartHostApiSubscribe();
1001 : }
1002 5 : if ((profconfig & PROF_TASK_TIME_MASK) != 0 && (profconfig & PROF_TASK_TIME_L1_MASK) == 0) {
1003 1 : StartHostHcclOpSubscribe();
1004 : }
1005 5 : if ((profconfig & PROF_TASK_TIME_L1_MASK) != 0) {
1006 1 : StartTaskApiSubscribe();
1007 1 : StartAdditionInfoSubscribe();
1008 1 : StartCcuSubscribe();
1009 : }
1010 15 : HCCL_RUN_INFO("[Profiling][CommandHandle] profSwitch is[%llu]", profconfig);
1011 5 : }
1012 :
1013 2 : void ProfilingHandler::StartHostApiSubscribe()
1014 : {
1015 2 : enableHostApi_ = true;
1016 2 : CallProfRegHostApi();
1017 2 : ReportStoragedCompactInfo();
1018 2 : ReportMc2AdditionInfo();
1019 2 : ReportStoragedAclApi();
1020 6 : HCCL_RUN_INFO("SetHostApiSubscribe:[%d]", enableHostApi_);
1021 2 : }
1022 :
1023 2 : void ProfilingHandler::CallProfRegHostApi() const
1024 : {
1025 2 : if (!enableHostApi_) {
1026 0 : return;
1027 : }
1028 2 : auto& profFunction = DlProfFunction::GetInstance();
1029 44 : for (auto i = 0; i < OpType::__COUNT__; ++i) {
1030 42 : OpType type(static_cast<OpType::Value>(i));
1031 42 : s32 ret = profFunction.dlMsprofRegTypeInfo(
1032 42 : MSPROF_REPORT_ACL_LEVEL, static_cast<uint32_t>(type) + MSPROF_REPORT_ACL_HOST_HCCL_BASE_TYPE,
1033 84 : type.Describe().c_str());
1034 42 : if (ret != 0) {
1035 0 : THROW<InternalException>("Call MsprofRegTypeInfo fail, return[%d]", ret);
1036 : }
1037 : }
1038 44 : for (auto i = 0; i < OpType::__COUNT__; ++i) {
1039 42 : OpType type(static_cast<OpType::Value>(i));
1040 42 : s32 ret = profFunction.dlMsprofRegTypeInfo(
1041 42 : MSPROF_REPORT_NODE_LEVEL, static_cast<uint32_t>(type) + MSPROF_REPORT_NODE_HCCL_BASE_TYPE,
1042 84 : type.Describe().c_str());
1043 42 : if (ret != 0) {
1044 0 : THROW<InternalException>("Call MsprofRegTypeInfo fail, return[%d]", ret);
1045 : }
1046 : }
1047 2 : const std::string hcclType("hccl_op_info");
1048 2 : s32 ret = profFunction.dlMsprofRegTypeInfo(
1049 : MSPROF_REPORT_NODE_LEVEL, MSPROF_REPORT_NODE_HCCL_OP_INFO_TYPE, hcclType.c_str());
1050 2 : if (ret != 0) {
1051 0 : THROW<InternalException>("Call MsprofRegTypeInfo fail, return[%d]", ret);
1052 : }
1053 2 : }
1054 :
1055 4 : void ProfilingHandler::ReportStoragedCompactInfo()
1056 : {
1057 4 : std::lock_guard<std::mutex> lock(cacheHcclOpInfoMutex_);
1058 12 : HCCL_INFO("[ReportStoragedCompactInfo] The size of the storageCompactInfo_ is [%zu]", cacheHcclOpInfo_.size());
1059 4 : std::queue<MsprofCompactInfo> tempCompactInfo = cacheHcclOpInfo_;
1060 12 : while (!tempCompactInfo.empty()) {
1061 8 : MsprofCompactInfo reportData = tempCompactInfo.front();
1062 8 : tempCompactInfo.pop();
1063 8 : s32 ret = DlProfFunction::GetInstance().dlMsprofReportCompactInfo(0, &reportData, sizeof(MsprofCompactInfo));
1064 8 : if (ret != 0) {
1065 0 : THROW<InternalException>("Call MsprofRegTypeInfo failed, return[%d]", ret);
1066 : }
1067 : }
1068 4 : }
1069 :
1070 2 : void ProfilingHandler::ReportMc2AdditionInfo()
1071 : {
1072 2 : std::lock_guard<std::mutex> lock(cacheHcclAdditionInfoMutex_);
1073 6 : HCCL_INFO("[ReportMc2AdditionInfo] The size of the storageCompactInfo_ is [%zu]", cacheHcclAdditionInfo_.size());
1074 2 : std::queue<MsprofAdditionalInfo> tempCompactInfo = cacheHcclAdditionInfo_;
1075 6 : while (!tempCompactInfo.empty()) {
1076 4 : MsprofAdditionalInfo reportData = tempCompactInfo.front();
1077 4 : tempCompactInfo.pop();
1078 : s32 ret
1079 4 : = DlProfFunction::GetInstance().dlMsprofReportAdditionalInfo(1, &reportData, sizeof(MsprofAdditionalInfo));
1080 4 : if (ret != 0) {
1081 0 : THROW<InternalException>("Call MsprofRegTypeInfo failed, return[%d]", ret);
1082 : }
1083 : }
1084 2 : }
1085 :
1086 2 : void ProfilingHandler::StartTaskApiSubscribe()
1087 : {
1088 2 : enableHcclNode_ = true;
1089 2 : CallProfRegTaskTypeApi();
1090 2 : ReportStoragedTaskApi();
1091 6 : HCCL_INFO("SetTaskApiSubscribe:[%d]", enableHcclNode_);
1092 2 : }
1093 :
1094 2 : void ProfilingHandler::CallProfRegTaskTypeApi() const
1095 : {
1096 2 : if (!enableHcclNode_) {
1097 0 : HCCL_INFO("[ProfilingHandler] enableHostApi_ is false.");
1098 0 : return;
1099 : }
1100 2 : const std::string hcclType("hccl_info");
1101 2 : s32 sret = DlProfFunction::GetInstance().dlMsprofRegTypeInfo(
1102 : MSPROF_REPORT_HCCL_NODE_LEVEL, static_cast<uint32_t>(ProfTaskType::TASK_HCCL_INFO), hcclType.c_str());
1103 2 : if (sret != 0) {
1104 0 : THROW<InternalException>("Call MsprofRegTypeInfo fail, return[%d]", sret);
1105 : }
1106 :
1107 2 : const std::string dpuhcclType("dpu_hccl_info");
1108 2 : sret = DlProfFunction::GetInstance().dlMsprofRegTypeInfo(
1109 : MSPROF_REPORT_HCCL_NODE_LEVEL, static_cast<uint32_t>(ProfTaskType::TASK_DPU_HCCL_INFO), dpuhcclType.c_str());
1110 2 : if (sret != 0) {
1111 0 : THROW<InternalException>("Call MsprofRegTypeInfo fail, return[%d]", sret);
1112 : }
1113 : const std::vector<std::pair<uint32_t, std::string>> taskTypes
1114 6 : = {{MSPROF_REPORT_NODE_CONTEXT_ID_INFO_TYPE, "context_id_info"}};
1115 : const std::vector<std::pair<uint32_t, std::string>> taskOtherTypes
1116 0 : = {{MSPROF_REPORT_NODE_BASIC_INFO_TYPE, "node_basic_info"},
1117 8 : {MSPROF_REPORT_NODE_MC2_COMMINFO_TYPE, "mc2_comm_info"}};
1118 :
1119 4 : for (auto& it : taskTypes) {
1120 2 : s32 ret = DlProfFunction::GetInstance().dlMsprofRegTypeInfo(
1121 2 : MSPROF_REPORT_HCCL_NODE_LEVEL, it.first, it.second.c_str());
1122 2 : if (ret != 0) {
1123 0 : THROW<InternalException>("Call dlMsprofRegTypeInfo failed, return[%d]", ret);
1124 : }
1125 : }
1126 6 : for (auto& it : taskOtherTypes) {
1127 : s32 ret
1128 4 : = DlProfFunction::GetInstance().dlMsprofRegTypeInfo(MSPROF_REPORT_NODE_LEVEL, it.first, it.second.c_str());
1129 4 : if (ret != 0) {
1130 0 : THROW<InternalException>("Call dlMsprofRegTypeInfo failed, return[%d]", ret);
1131 : }
1132 : }
1133 6 : }
1134 :
1135 2 : void ProfilingHandler::ReportStoragedTaskApi()
1136 : {
1137 2 : std::lock_guard<std::mutex> lock(cachedTaskApiInfoMutex_);
1138 6 : HCCL_INFO("[ReportStoragedTaskApi] taskApiQueueSize is [%zu]", cachedTaskApiInfo_.size());
1139 2 : if (!cachedTaskApiInfo_.empty()) {
1140 2 : std::queue<MsprofApi> tempTaskApi = cachedTaskApiInfo_;
1141 10 : while (!tempTaskApi.empty()) {
1142 8 : MsprofApi reportData = tempTaskApi.front();
1143 8 : tempTaskApi.pop();
1144 8 : s32 ret = DlProfFunction::GetInstance().dlMsprofReportApi(0, &reportData);
1145 8 : if (ret != 0) {
1146 0 : THROW<InternalException>("Call dlMsprofReportApi failed, return[%d]", ret);
1147 : }
1148 : }
1149 2 : }
1150 2 : }
1151 :
1152 2 : void ProfilingHandler::ReportStoragedAclApi()
1153 : {
1154 2 : std::lock_guard<std::mutex> lock(cachedAclApiInfoMutex_);
1155 6 : HCCL_INFO("[ReportStoragedAclApi] aclApiQueueSize is [%u]", cachedAclApiInfo_.size());
1156 2 : if (!cachedAclApiInfo_.empty()) {
1157 2 : std::queue<MsprofApi> tempAclApi = cachedAclApiInfo_;
1158 4 : while (!tempAclApi.empty()) {
1159 2 : MsprofApi reportData = tempAclApi.front();
1160 2 : tempAclApi.pop();
1161 2 : s32 ret = DlProfFunction::GetInstance().dlMsprofReportApi(0, &reportData);
1162 2 : if (ret != 0) {
1163 0 : THROW<InternalException>("Call dlMsprofReportApi failed, return[%d]", ret);
1164 : }
1165 : }
1166 2 : }
1167 2 : }
1168 :
1169 2 : void ProfilingHandler::StartHostHcclOpSubscribe()
1170 : {
1171 2 : enableHcclNode_ = true;
1172 2 : enableHcclL0_ = true;
1173 2 : CallProfRegHcclOpApi();
1174 2 : ReportStoragedCompactInfo();
1175 6 : HCCL_RUN_INFO("StartHostHcclOpSubscribe:enableHcclNode_[%d],enableHcclL0_[%d]", enableHcclNode_, enableHcclL0_);
1176 2 : }
1177 :
1178 2 : void ProfilingHandler::CallProfRegHcclOpApi() const
1179 : {
1180 2 : if (enableHcclL0_ == false) {
1181 0 : HCCL_INFO("[ProfilingHandler] enableHcclNode_ is false.");
1182 0 : return;
1183 : }
1184 44 : for (auto i = 0; i < OpType::__COUNT__; ++i) {
1185 42 : OpType type(static_cast<OpType::Value>(i));
1186 84 : s32 ret = DlProfFunction::GetInstance().dlMsprofRegTypeInfo(
1187 42 : MSPROF_REPORT_HCCL_NODE_LEVEL, static_cast<uint32_t>(type) + MSPROF_REPORT_ACL_HOST_HCCL_BASE_TYPE,
1188 84 : type.Describe().c_str());
1189 42 : if (ret != 0) {
1190 0 : THROW<InternalException>("[ProfilingHandler]Call MsprofReportApi fail, return[%d]", ret);
1191 : }
1192 : }
1193 2 : s32 ret = DlProfFunction::GetInstance().dlMsprofRegTypeInfo(
1194 : MSPROF_REPORT_NODE_LEVEL, MSPROF_REPORT_NODE_MC2_COMMINFO_TYPE, "mc2_comm_info");
1195 2 : if (ret != 0) {
1196 0 : THROW<InternalException>("[ProfilingHandler]Call MsprofRegTypeInfo fail, return[%d]", ret);
1197 : }
1198 : }
1199 :
1200 2 : void ProfilingHandler::StartAdditionInfoSubscribe()
1201 : {
1202 2 : enableHcclL1_ = true;
1203 2 : ReportStoragedAdditionInfo();
1204 6 : HCCL_RUN_INFO("StartAdditionInfoSubscribe:enableHcclL1_[%d]", enableHcclL1_);
1205 2 : }
1206 :
1207 2 : void ProfilingHandler::ReportStoragedAdditionInfoLog() const
1208 : {
1209 2 : if (LIKELY(HcclCheckLogLevel(HCCL_LOG_INFO) == 0)) {
1210 0 : return;
1211 : }
1212 2 : if (cacheTaskInfos_.empty()) {
1213 0 : return;
1214 : }
1215 4 : for (const auto& taskInfo : cacheTaskInfos_) {
1216 6 : HCCL_INFO("ReportStoragedAdditionInfoLog, taskType[%u]", GetTaskTypeValue(taskInfo.taskParam_.taskType));
1217 : }
1218 : }
1219 :
1220 2 : void ProfilingHandler::ReportStoragedAdditionInfo()
1221 : {
1222 2 : std::lock_guard<std::mutex> lock(cacheTaskInfosMutex_);
1223 2 : ReportStoragedAdditionInfoLog();
1224 2 : if (cacheTaskInfos_.empty()) {
1225 0 : HCCL_INFO("[ProfilingHandler]ReportStoragedAdditionInfo cacheTaskInfos_ is empty.");
1226 0 : return;
1227 : }
1228 4 : for (auto& taskInfo : cacheTaskInfos_) {
1229 2 : MsprofAdditionalInfo reporterData{};
1230 2 : FillTaskAdditionInfo(taskInfo, reporterData);
1231 2 : CallAdditionInfo(reporterData);
1232 : }
1233 2 : }
1234 :
1235 2 : void ProfilingHandler::StartCcuSubscribe()
1236 : {
1237 2 : enableHcclNode_ = true;
1238 2 : enableHcclL1_ = true;
1239 6 : HCCL_INFO("ProfilingHandler StartCcuSubscribe");
1240 : const std::vector<std::pair<uint32_t, std::string>> ccuInfoTypes
1241 : = {{MSPROF_REPORT_CCU_TASK_INFO, "ccu_task_info"},
1242 : {MSPROF_REPORT_CCU_WAIT_SIGNAL_INFO, "ccu_wait_signal_info"},
1243 10 : {MSPROF_REPORT_CCU_GROUP_INFO, "ccu_group_info"}};
1244 8 : for (auto& it : ccuInfoTypes) {
1245 6 : s32 ret = DlProfFunction::GetInstance().dlMsprofRegTypeInfo(
1246 6 : MSPROF_REPORT_HCCL_NODE_LEVEL, it.first, it.second.c_str());
1247 6 : if (ret != 0) {
1248 0 : THROW<InternalException>("Call dlMsprofRegTypeInfo failed, return[%d]", ret);
1249 : }
1250 : }
1251 2 : std::lock_guard<std::mutex> lock(cacheTaskInfosMutex_);
1252 2 : if (cacheTaskInfos_.empty()) {
1253 0 : HCCL_INFO("[ProfilingHandler]StartL2Subscribe cacheTaskInfos_ is empty.");
1254 0 : return;
1255 : }
1256 4 : for (auto& taskInfo : cacheTaskInfos_) {
1257 2 : ReportCcuInfo(taskInfo);
1258 : }
1259 4 : }
1260 :
1261 1 : void ProfilingHandler::ProfilingHandler::StopSubscribe()
1262 : {
1263 1 : enableHostApi_ = false;
1264 1 : enableHcclNode_ = false;
1265 1 : enableHcclL0_ = false;
1266 1 : enableHcclL1_ = false;
1267 3 : HCCL_RUN_INFO("[ProfilingHandler]StopSubscribe.");
1268 1 : }
1269 :
1270 48 : bool ProfilingHandler::GetHostApiState() const { return enableHostApi_; }
1271 48 : bool ProfilingHandler::GetHcclNodeState() const { return enableHcclNode_; }
1272 50 : bool ProfilingHandler::GetHcclL0State() const { return enableHcclL0_; }
1273 :
1274 54 : bool ProfilingHandler::GetHcclL1State() const { return enableHcclL1_; }
1275 :
1276 171 : uint64_t ProfilingHandler::GetProfHashId(const char* name, uint32_t len) const
1277 : {
1278 171 : if (name == nullptr || len == 0) {
1279 63 : HCCL_WARNING("HashData is empty. name:%s, len:%u", name, len);
1280 21 : return INVALID_U64;
1281 : }
1282 150 : if (DlProfFunction::GetInstance().dlMsprofStr2Id == nullptr) {
1283 0 : HCCL_WARNING("HashFunc is nullPtr");
1284 0 : return INVALID_U64;
1285 : }
1286 150 : return DlProfFunction::GetInstance().dlMsprofStr2Id(name, len);
1287 : }
1288 :
1289 2 : void ProfilingHandler::ReportHcclMC2CommInfoLog(
1290 : [[maybe_unused]] const Stream& kfcStream, const Stream& stream, const std::vector<Stream*>& aicpuStreams,
1291 : [[maybe_unused]] const std::string& id, [[maybe_unused]] RankId myRank, [[maybe_unused]] u32 rankSize,
1292 : [[maybe_unused]] RankId rankInParentComm) const
1293 : {
1294 2 : if (LIKELY(HcclCheckLogLevel(HCCL_LOG_INFO) == 0)) {
1295 0 : return;
1296 : }
1297 2 : if (aicpuStreams.empty()) {
1298 3 : HCCL_INFO("only exist main stream, streamId(sqId):%u", stream.GetSqId());
1299 1 : return;
1300 : }
1301 1 : uint32_t reportId = 0;
1302 2 : for (uint32_t streamIndex = 0; streamIndex < aicpuStreams.size(); streamIndex++) {
1303 3 : HCCL_INFO(
1304 : "streamIndex:%u, reportId:%u, streamId(sqId):%u", streamIndex, reportId,
1305 : aicpuStreams[streamIndex]->GetSqId());
1306 1 : reportId++;
1307 : }
1308 : }
1309 :
1310 2 : void ProfilingHandler::ReportHcclMC2CommInfo(
1311 : const Stream& kfcStream, const Stream& stream, const std::vector<Stream*>& aicpuStreams, const std::string& id,
1312 : RankId myRank, u32 rankSize, RankId rankInParentComm)
1313 : {
1314 2 : ReportHcclMC2CommInfoLog(kfcStream, stream, aicpuStreams, id, myRank, rankSize, rankInParentComm);
1315 : ProfilingDeviceCommResInfo hcclMc2Info;
1316 2 : hcclMc2Info.groupName = GetProfHashId(id.c_str(), id.length());
1317 2 : hcclMc2Info.rankSize = rankSize;
1318 2 : hcclMc2Info.rankId = myRank;
1319 2 : hcclMc2Info.usrRankId = rankInParentComm;
1320 2 : hcclMc2Info.aicpuKfcStreamId = static_cast<uint32_t>(kfcStream.GetSqId());
1321 2 : hcclMc2Info.reserve = 0;
1322 2 : const uint32_t ONCE_REPORT_STREAM_NUM_MAX = 8;
1323 3 : for (uint32_t streamIndex = 0, reportId = 0; streamIndex < aicpuStreams.size(); streamIndex++) {
1324 1 : hcclMc2Info.commStreamIds[reportId++] = aicpuStreams[streamIndex]->GetSqId();
1325 1 : if (reportId == ONCE_REPORT_STREAM_NUM_MAX) {
1326 0 : hcclMc2Info.commStreamSize = reportId;
1327 0 : ReportMc2AdditionInfo(
1328 0 : DlProfFunction::GetInstance().dlMsprofSysCycleTime(), &hcclMc2Info, sizeof(hcclMc2Info));
1329 0 : reportId = 0;
1330 : }
1331 1 : if (streamIndex == (aicpuStreams.size() - 1)) {
1332 1 : hcclMc2Info.commStreamIds[reportId++] = stream.GetSqId();
1333 1 : hcclMc2Info.commStreamSize = reportId;
1334 1 : ReportMc2AdditionInfo(
1335 1 : DlProfFunction::GetInstance().dlMsprofSysCycleTime(), &hcclMc2Info, sizeof(hcclMc2Info));
1336 1 : reportId = 0;
1337 : }
1338 : }
1339 2 : if (aicpuStreams.empty()) {
1340 1 : hcclMc2Info.commStreamIds[0] = stream.GetSqId();
1341 1 : hcclMc2Info.commStreamSize = 1;
1342 1 : ReportMc2AdditionInfo(DlProfFunction::GetInstance().dlMsprofSysCycleTime(), &hcclMc2Info, sizeof(hcclMc2Info));
1343 : }
1344 2 : }
1345 :
1346 1 : void ProfilingHandler::ReportHcclMC2CommInfoLog(
1347 : [[maybe_unused]] const u32 kfcStreamId, const std::vector<u32>& aicpuStreamsId, const std::string& id,
1348 : [[maybe_unused]] RankId myRank, [[maybe_unused]] u32 rankSize, [[maybe_unused]] RankId rankInParentComm) const
1349 : {
1350 1 : if (LIKELY(HcclCheckLogLevel(HCCL_LOG_INFO) == 0)) {
1351 0 : return;
1352 : }
1353 1 : uint64_t groupName = GetProfHashId(id.c_str(), id.length());
1354 1 : uint32_t reportId = 0;
1355 4 : for (uint32_t streamIndex = 0; streamIndex < aicpuStreamsId.size(); streamIndex++) {
1356 9 : HCCL_INFO(
1357 : "streamIndex:[%u], reportId:[%u], streamId:[%u] id [%s] hcclMC2Info.groupName:[%llu]", streamIndex,
1358 : reportId, aicpuStreamsId[streamIndex], id.c_str(), groupName);
1359 3 : reportId++;
1360 : }
1361 : }
1362 :
1363 0 : void ProfilingHandler::ReportHcclMC2CommInfo(
1364 : const u32 kfcStreamId, const std::vector<u32>& aicpuStreamsId, const std::string& id, RankId myRank, u32 rankSize,
1365 : RankId rankInParentComm)
1366 : {
1367 0 : ReportHcclMC2CommInfoLog(kfcStreamId, aicpuStreamsId, id, myRank, rankSize, rankInParentComm);
1368 : ProfilingDeviceCommResInfo hcclMc2Info;
1369 0 : hcclMc2Info.groupName = GetProfHashId(id.c_str(), id.length());
1370 0 : hcclMc2Info.rankSize = rankSize;
1371 0 : hcclMc2Info.rankId = myRank;
1372 0 : hcclMc2Info.usrRankId = rankInParentComm;
1373 0 : hcclMc2Info.aicpuKfcStreamId = static_cast<uint32_t>(kfcStreamId);
1374 0 : hcclMc2Info.reserve = 0;
1375 :
1376 0 : const uint32_t ONCE_REPORT_STREAM_NUM_MAX = 8;
1377 0 : uint32_t reportId = 0;
1378 0 : for (uint32_t streamIndex = 0; streamIndex < aicpuStreamsId.size(); streamIndex++) {
1379 0 : hcclMc2Info.commStreamIds[reportId++] = aicpuStreamsId[streamIndex];
1380 0 : if (reportId == ONCE_REPORT_STREAM_NUM_MAX) {
1381 0 : hcclMc2Info.commStreamSize = reportId;
1382 0 : ReportMc2AdditionInfo(
1383 0 : DlProfFunction::GetInstance().dlMsprofSysCycleTime(), &hcclMc2Info, sizeof(hcclMc2Info));
1384 0 : reportId = 0;
1385 : }
1386 : }
1387 0 : if (reportId > 0) {
1388 0 : hcclMc2Info.commStreamSize = reportId;
1389 0 : ReportMc2AdditionInfo(DlProfFunction::GetInstance().dlMsprofSysCycleTime(), &hcclMc2Info, sizeof(hcclMc2Info));
1390 0 : reportId = 0;
1391 : }
1392 0 : }
1393 2 : void ProfilingHandler::ReportMc2AdditionInfo(uint64_t timeStamp, const void* data, int len)
1394 : {
1395 2 : MsprofAdditionalInfo reporterData{};
1396 2 : reporterData.level = MSPROF_REPORT_NODE_LEVEL;
1397 2 : reporterData.type = MSPROF_REPORT_NODE_MC2_COMMINFO_TYPE;
1398 2 : reporterData.threadId = SalGetTid();
1399 2 : reporterData.dataLen = len;
1400 2 : reporterData.timeStamp = timeStamp;
1401 2 : s32 sret = memcpy_s(reporterData.data, sizeof(reporterData.data), data, len);
1402 2 : if (sret != EOK) {
1403 0 : THROW<InternalException>("Call memcpy_s failed, errorno[%d]", sret);
1404 : }
1405 6 : HCCL_INFO(
1406 : "[ProfilingHandler][ReportMc2CommInfo], level [%u], type[%u], threadId[%u], dataLen[%u], timeStamp[%llu]",
1407 : reporterData.level, reporterData.type, reporterData.threadId, reporterData.dataLen, reporterData.timeStamp);
1408 2 : if (!enableHostApi_) {
1409 2 : std::lock_guard<std::mutex> lock(cacheHcclAdditionInfoMutex_);
1410 2 : cacheHcclAdditionInfo_.push(reporterData);
1411 2 : return;
1412 2 : }
1413 : s32 ret
1414 0 : = DlProfFunction::GetInstance().dlMsprofReportAdditionalInfo(1, &reporterData, sizeof(MsprofAdditionalInfo));
1415 0 : HCCL_INFO("Call MsprofReportAdditionalInfo, return value[%d]", ret);
1416 0 : if (ret != 0) {
1417 0 : THROW<InternalException>("Call MsprofReportAdditionalInfo failed, return[%d]", ret);
1418 : }
1419 : }
1420 :
1421 : } // namespace Hccl
|
