Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include <cstring>
12 : #include <algorithm>
13 : #include "profiling_handler.h"
14 : #include "dlprof_function_v2.h"
15 : #include "exception_util.h"
16 : #include "internal_exception.h"
17 : #include "sal.h"
18 : #include "task_param.h"
19 : #include "aprof_pub.h"
20 : #include "orion_adapter_rts.h"
21 : #include "communicator_impl.h"
22 : #include "data_type.h"
23 : namespace Hccl {
24 : #define UNUSED(x) (void)(x)
25 :
26 : constexpr uint16_t CCU_TYPE = 2;
27 : constexpr uint32_t DPU_DEV_ID_MASK = (1U << 12);
28 :
29 : ProfilingHandler ProfilingHandler::instance_;
30 :
31 1 : ProfilingHandler::ProfilingHandler() {}
32 :
33 1 : ProfilingHandler::~ProfilingHandler() {}
34 :
35 749 : ProfilingHandler& ProfilingHandler::GetInstance() { return instance_; }
36 :
37 1 : void ProfilingHandler::InitLog() const
38 : {
39 1 : if (LIKELY(HcclCheckLogLevel(HCCL_LOG_INFO) == 0)) {
40 0 : return;
41 : }
42 24 : for (auto i = 0; i < TaskParamType::__COUNT__; ++i) {
43 23 : TaskParamType type(static_cast<TaskParamType::Value>(i));
44 23 : std::string nameInfo = type.Describe();
45 23 : uint64_t hashId = GetProfHashId(nameInfo.c_str(), nameInfo.length());
46 69 : HCCL_INFO("[TaskParamType] nameInfo[%s] ret[%llu]", nameInfo.c_str(), hashId);
47 23 : }
48 : }
49 :
50 258 : HcclResult ProfilingHandler::Init()
51 : {
52 258 : if (initializedFlag_) {
53 257 : return HCCL_SUCCESS;
54 : }
55 1 : if (Hccl::DlProfFunction::GetInstance().DlProfFunctionInit() != HCCL_SUCCESS) {
56 0 : HCCL_ERROR("[ProfilingHandler][Init] DlProfFunctionInit failed.");
57 0 : return HCCL_E_INTERNAL;
58 : }
59 1 : ProfCommandHandle callback = CommandHandleWrapper;
60 1 : auto ret = DlProfFunction::GetInstance().dlMsprofRegisterCallback(HCCL, callback);
61 1 : if (ret != 0) {
62 0 : HCCL_ERROR(
63 : "[ProfilingHandler][Init]errNo[0x%016llx] Prof Register CtrlCallback fail, return[%d]",
64 : HCCL_ERROR_CODE(HCCL_E_RUNTIME), ret);
65 0 : return HCCL_E_RUNTIME;
66 : }
67 1 : SetCachedCclTag();
68 1 : cachedAlgTypeHashId_.store(GetProfHashId("AlgType::NHR", strlen("AlgType::NHR")));
69 1 : initializedFlag_ = true;
70 1 : InitLog();
71 1 : return HCCL_SUCCESS;
72 : }
73 :
74 0 : int32_t ProfilingHandler::CommandHandleWrapper(uint32_t rtType, void* data, uint32_t len)
75 : {
76 0 : return instance_.CommandHandle(rtType, data, len);
77 : }
78 :
79 0 : void ProfilingHandler::ReportKernel() const {}
80 :
81 3 : void ProfilingHandler::ReportHostApi(OpType opType, uint64_t beginTime, uint64_t endTime, bool cachedReq, bool isAiCpu)
82 : {
83 3 : uint32_t threadId = SalGetTid();
84 3 : std::string profName(GetProfOpName(opType));
85 3 : if (isAiCpu) {
86 1 : profName += "AicpuKernel";
87 : }
88 3 : uint64_t cmdItemId = DlProfFunction::GetInstance().dlMsprofStr2Id(profName.c_str(), profName.length());
89 3 : if (enableHostApi_) {
90 0 : ReportAclApi(opType, beginTime, endTime, cmdItemId, threadId, cachedReq);
91 : }
92 3 : ReportNodeApi(beginTime, endTime, cmdItemId, threadId, cachedReq);
93 3 : ReportNodeBasicInfo(endTime, cmdItemId, threadId, cachedReq);
94 3 : }
95 :
96 3 : void ProfilingHandler::ReportHcclOp(const DfxOpInfo& opInfo, bool cachedReq)
97 : {
98 3 : uint32_t threadId = SalGetTid();
99 3 : ReportHcclOpInfo(opInfo.endTime_, opInfo, threadId, cachedReq);
100 3 : }
101 :
102 5 : void ProfilingHandler::ReportHcclTaskApi(
103 : TaskParamType taskType, uint64_t beginTime, uint64_t endTime, bool isMasterStream, bool cachedReq, bool ignoreLevel)
104 : {
105 5 : MsprofApi reporterData{};
106 5 : reporterData.level = MSPROF_REPORT_HCCL_NODE_LEVEL;
107 5 : reporterData.type = isMasterStream ? MSPROF_REPORT_HCCL_MASTER_TYPE : MSPROF_REPORT_HCCL_SLAVE_TYPE;
108 5 : reporterData.threadId = SalGetTid();
109 5 : reporterData.beginTime = beginTime;
110 5 : reporterData.endTime = endTime;
111 5 : const std::string proName(GetProfTaskOpNameV2(taskType));
112 5 : reporterData.itemId = GetProfHashId(proName.c_str(), proName.length());
113 5 : if (taskType == TaskParamType::TASK_AICPU_KERNEL) {
114 0 : return;
115 : }
116 5 : if (cachedReq) {
117 5 : std::lock_guard<std::mutex> lock(cachedTaskApiInfoMutex_);
118 5 : cachedTaskApiInfo_.push(reporterData);
119 5 : }
120 5 : if ((!enableHcclNode_) || (!ignoreLevel && !enableHcclL1_)) {
121 3 : return;
122 : }
123 6 : HCCL_INFO(
124 : "ReportHcclTaskApi, enableHcclNode_[%d],ignoreLevel[%d],enableHcclL1_[%d],taskType[%s], beginTime[%llu], "
125 : "endTime[%llu], isMasterStream[%d], cachedReq[%d]",
126 : enableHcclNode_, ignoreLevel, enableHcclL1_, proName.c_str(), beginTime, endTime, isMasterStream, cachedReq);
127 2 : s32 ret = DlProfFunction::GetInstance().dlMsprofReportApi(1, &reporterData);
128 2 : if (ret != 0) {
129 0 : THROW<InternalException>("Call MsprofReportApi fail, return[%d]", ret);
130 : }
131 5 : }
132 :
133 6 : void ProfilingHandler::SetCachedCclTag()
134 : {
135 6 : cachedNewCclTag_.clear();
136 120 : for (const auto& item : CMD_OP_TYPE_INFO_MAP) {
137 114 : const std::string& cclTag = item.second.second;
138 114 : cachedNewCclTag_[item.second.first] = GetProfHashId(cclTag.c_str(), cclTag.length());
139 : }
140 6 : if (cachedNewCclTag_.find(static_cast<uint32_t>(OpType::HCCLGROUPOP)) == cachedNewCclTag_.end()) {
141 6 : const std::string cclTag = "OpType::HCCLGROUPOP";
142 6 : cachedNewCclTag_[OpType::HCCLGROUPOP] = GetProfHashId(cclTag.c_str(), cclTag.length());
143 6 : }
144 6 : }
145 :
146 6 : uint32_t ProfilingHandler::GetTaskTypeValue(TaskParamType taskType) const
147 : {
148 6 : switch (taskType) {
149 0 : case TaskParamType::TASK_DPU_INLINE_WRITE:
150 : case TaskParamType::TASK_DPU_NOTIFY_WAIT:
151 : case TaskParamType::TASK_DPU_WRITE_WITH_NOTIFY:
152 : case TaskParamType::TASK_DPU_CHANNEL_FENCE:
153 0 : return static_cast<uint32_t>(ProfTaskType::TASK_DPU_HCCL_INFO);
154 6 : default:
155 6 : return static_cast<uint32_t>(ProfTaskType::TASK_HCCL_INFO);
156 : }
157 : }
158 :
159 7 : void ProfilingHandler::FillTaskAdditionInfo(const TaskInfo& taskInfo, MsprofAdditionalInfo& reporterData) const
160 : {
161 7 : reporterData.level = MSPROF_REPORT_HCCL_NODE_LEVEL;
162 7 : reporterData.threadId = SalGetTid();
163 7 : FillProfCommonInfo(taskInfo, reporterData);
164 :
165 7 : switch (taskInfo.taskParam_.taskType) {
166 0 : case TaskParamType::TASK_DPU_INLINE_WRITE:
167 : case TaskParamType::TASK_DPU_NOTIFY_WAIT:
168 : case TaskParamType::TASK_DPU_WRITE_WITH_NOTIFY:
169 : case TaskParamType::TASK_DPU_CHANNEL_FENCE:
170 0 : reporterData.type = static_cast<uint32_t>(ProfTaskType::TASK_DPU_HCCL_INFO);
171 0 : FillDpuProfInfo(taskInfo, reporterData);
172 0 : break;
173 7 : default:
174 7 : reporterData.type = static_cast<uint32_t>(ProfTaskType::TASK_HCCL_INFO);
175 7 : FillProfTaskSpecificInfo(taskInfo, reinterpret_cast<MsprofHcclInfo*>(reporterData.data));
176 7 : break;
177 : }
178 7 : }
179 :
180 3 : void ProfilingHandler::ReportHcclTaskDetails(const TaskInfo& taskInfo, bool cachedReq)
181 : {
182 3 : if (enableHcclL1_ == false && !cachedReq) {
183 2 : return;
184 : }
185 3 : if (taskInfo.dfxOpInfo_ == nullptr) {
186 3 : HCCL_WARNING("[%s] dfxOpInfo_ is nullptr, skip ReportHcclTaskDetails!", __func__);
187 1 : return;
188 : }
189 2 : if (taskInfo.dfxOpInfo_->comm_ == nullptr) {
190 3 : HCCL_WARNING("[%s] comm_ is nullptr, skip ReportHcclTaskDetails!", __func__);
191 1 : return;
192 : }
193 1 : if (cachedReq) {
194 1 : std::lock_guard<std::mutex> lock(cacheTaskInfosMutex_);
195 1 : cacheTaskInfos_.push_back(taskInfo);
196 1 : }
197 1 : if (enableHcclL1_ == false) {
198 0 : return;
199 : }
200 1 : MsprofAdditionalInfo reporterData{};
201 1 : FillTaskAdditionInfo(taskInfo, reporterData);
202 1 : DumpHCCLReportData(taskInfo, reporterData);
203 1 : CallAdditionInfo(reporterData);
204 : }
205 :
206 2 : void ProfilingHandler::ReportHcclTaskDetailsBatchLog(const std::vector<TaskInfo*>& taskInfos) const
207 : {
208 2 : if (LIKELY(HcclCheckLogLevel(HCCL_LOG_INFO) == 0)) {
209 0 : return;
210 : }
211 6 : for (const auto& taskInfo : taskInfos) {
212 12 : HCCL_INFO("ReportHcclTaskDetailsBatchLog, taskType[%u]", GetTaskTypeValue(taskInfo->taskParam_.taskType));
213 : }
214 : }
215 :
216 7 : void ProfilingHandler::ReportHcclTaskDetailsBatch(const std::vector<TaskInfo*>& taskInfos, bool cachedReq)
217 : {
218 7 : if (taskInfos.empty()) {
219 3 : HCCL_WARNING("[ProfilingHandler::ReportHcclTaskDetailsBatch] taskInfos is empty!");
220 1 : return;
221 : }
222 6 : if (taskInfos[0]->dfxOpInfo_ == nullptr) {
223 3 : HCCL_WARNING("[ProfilingHandler::ReportHcclTaskDetailsBatch] taskInfo.dfxOpInfo_ is nullptr, skip!");
224 1 : return;
225 : }
226 5 : if (taskInfos[0]->dfxOpInfo_->comm_ == nullptr) {
227 3 : HCCL_WARNING("[ProfilingHandler::ReportHcclTaskDetailsBatch] taskInfo.dfxOpInfo_->comm_ is nullptr, skip!");
228 1 : return;
229 : }
230 4 : if (enableHcclL1_ == false && !cachedReq) {
231 1 : return;
232 : }
233 3 : if (cachedReq) {
234 3 : std::lock_guard<std::mutex> lock(cacheTaskInfosMutex_);
235 8 : for (const auto& taskInfo : taskInfos) {
236 5 : cacheTaskInfos_.push_back(*taskInfo);
237 : }
238 3 : }
239 3 : if (enableHcclL1_ == false) {
240 1 : return;
241 : }
242 2 : ReportHcclTaskDetailsBatchLog(taskInfos);
243 6 : for (const auto& taskInfo : taskInfos) {
244 4 : MsprofAdditionalInfo reporterData{};
245 4 : FillTaskAdditionInfo(*taskInfo, reporterData);
246 4 : CallAdditionInfo(reporterData);
247 : }
248 : }
249 :
250 5 : void ProfilingHandler::CallAdditionInfo(MsprofAdditionalInfo& reporterData) const
251 : {
252 : s32 ret
253 5 : = DlProfFunction::GetInstance().dlMsprofReportAdditionalInfo(1, &reporterData, sizeof(MsprofAdditionalInfo));
254 5 : if (ret != 0) {
255 0 : THROW<InternalException>("Call MsprofReportAdditionalInfo failed, return[%d]", ret);
256 : }
257 5 : }
258 :
259 7 : void ProfilingHandler::FillProfCommonInfo(const TaskInfo& taskInfo, MsprofAdditionalInfo& reporterData) const
260 : {
261 7 : if (taskInfo.dfxOpInfo_ == nullptr) {
262 0 : HCCL_WARNING("[ProfilingHandler][%s]taskInfo.dfxOpInfo_ is nullptr, skip GetProfCommonInfo.", __func__);
263 0 : return;
264 : }
265 7 : reporterData.timeStamp = taskInfo.taskParam_.endTime;
266 7 : reporterData.dataLen = sizeof(MsprofHcclInfo);
267 7 : auto* profInfo = reinterpret_cast<MsprofHcclInfo*>(reporterData.data);
268 7 : *profInfo = MsprofHcclInfo();
269 :
270 7 : const auto& profName = GetProfTaskOpNameV2(taskInfo.taskParam_.taskType);
271 7 : profInfo->itemId = GetProfHashId(profName.c_str(), profName.length());
272 7 : auto newCclTagIt = cachedNewCclTag_.find(taskInfo.dfxOpInfo_->op_.opType);
273 7 : profInfo->cclTag = (newCclTagIt != cachedNewCclTag_.end()) ? newCclTagIt->second : INVALID_U64;
274 7 : const auto& opTag = taskInfo.dfxOpInfo_->op_.opTag;
275 7 : uint64_t groupName = GetProfHashId(opTag.c_str(), opTag.length());
276 7 : if (taskInfo.dfxOpInfo_->comm_ == nullptr) {
277 0 : HCCL_WARNING("[ProfilingHandler][%s]taskInfo.dfxOpInfo_->comm_ is nullptr, skip GetProfCommonInfo.", __func__);
278 0 : return;
279 : }
280 7 : if (taskInfo.dfxOpInfo_->isIndop_ == true) {
281 0 : profInfo->groupName = groupName;
282 0 : profInfo->rankSize = taskInfo.dfxOpInfo_->rankSize_;
283 : } else {
284 7 : CommunicatorImpl* commImp = static_cast<CommunicatorImpl*>(taskInfo.dfxOpInfo_->comm_);
285 7 : if (commImp == nullptr) {
286 0 : HCCL_WARNING("[ProfilingHandler][%s]commImp is nullptr, skip GetProfCommonInfo.", __func__);
287 0 : return;
288 : }
289 7 : profInfo->groupName = groupName;
290 7 : profInfo->rankSize = commImp->GetRankSize();
291 : }
292 7 : profInfo->workFlowMode = static_cast<u32>(HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE);
293 7 : profInfo->planeID = 0;
294 7 : profInfo->stage = 0;
295 7 : profInfo->role = static_cast<uint32_t>(TaskRole::DST);
296 7 : profInfo->durationEstimated = 0;
297 7 : profInfo->localRank = taskInfo.dfxOpInfo_->op_.myRank;
298 7 : profInfo->remoteRank = taskInfo.remoteRank_;
299 7 : profInfo->dataType = taskInfo.dfxOpInfo_->op_.dataType;
300 7 : profInfo->opType = taskInfo.dfxOpInfo_->op_.opType;
301 7 : profInfo->transportType = static_cast<int32_t>(SimpleTaskType::UB);
302 7 : if (profInfo->remoteRank == INVALID_VALUE_RANKID) {
303 0 : profInfo->transportType = static_cast<int32_t>(SimpleTaskType::LOCAL);
304 0 : profInfo->remoteRank = profInfo->localRank;
305 0 : profInfo->linkType = 0;
306 : }
307 7 : }
308 :
309 7 : void ProfilingHandler::FillProfTaskSpecificInfo(const TaskInfo& taskInfo, MsprofHcclInfo* profInfo) const
310 : {
311 7 : const auto& taskType = taskInfo.taskParam_.taskType;
312 7 : const auto& taskPara = taskInfo.taskParam_.taskPara;
313 7 : switch (taskType) {
314 0 : case TaskParamType::TASK_SDMA:
315 : case TaskParamType::TASK_RDMA:
316 0 : profInfo->srcAddr = static_cast<u64>(reinterpret_cast<uintptr_t>(taskPara.DMA.src));
317 0 : profInfo->dstAddr = static_cast<u64>(reinterpret_cast<uintptr_t>(taskPara.DMA.dst));
318 0 : profInfo->dataSize = static_cast<u32>(taskPara.DMA.size);
319 0 : profInfo->notifyID = taskPara.DMA.notifyID;
320 0 : profInfo->linkType = static_cast<uint16_t>(taskPara.DMA.linkType);
321 0 : break;
322 0 : case TaskParamType::TASK_REDUCE_INLINE:
323 : case TaskParamType::TASK_REDUCE_TBE:
324 0 : profInfo->srcAddr = static_cast<u64>(reinterpret_cast<uintptr_t>(taskPara.Reduce.src));
325 0 : profInfo->dstAddr = static_cast<u64>(reinterpret_cast<uintptr_t>(taskPara.Reduce.dst));
326 0 : profInfo->dataSize = static_cast<u32>(taskPara.Reduce.size);
327 0 : profInfo->notifyID = taskPara.Reduce.notifyID;
328 0 : profInfo->linkType = static_cast<uint16_t>(taskPara.Reduce.linkType);
329 0 : break;
330 7 : case TaskParamType::TASK_NOTIFY_RECORD:
331 : case TaskParamType::TASK_NOTIFY_WAIT:
332 7 : profInfo->notifyID = taskPara.Notify.notifyID;
333 7 : break;
334 0 : case TaskParamType::TASK_CCU:
335 0 : HCCL_INFO("current taskType is TASK_CCU");
336 0 : ReportCcuInfo(taskInfo);
337 0 : break;
338 0 : default:
339 0 : break;
340 : }
341 7 : }
342 :
343 0 : void ProfilingHandler::ConvertHcclInfoToDpuTrack(MsprofAdditionalInfo& reporterData) const
344 : {
345 0 : auto* profInfo = reinterpret_cast<MsprofHcclInfo*>(reporterData.data);
346 0 : uint64_t itemId = profInfo->itemId;
347 0 : uint64_t cclTag = profInfo->cclTag;
348 0 : uint64_t groupName = profInfo->groupName;
349 0 : uint32_t localRank = profInfo->localRank;
350 0 : uint32_t remoteRank = profInfo->remoteRank;
351 0 : uint32_t rankSize = profInfo->rankSize;
352 0 : uint32_t workFlowMode = profInfo->workFlowMode;
353 0 : uint16_t planeID = static_cast<uint16_t>(profInfo->planeID);
354 0 : uint32_t stage = profInfo->stage;
355 0 : uint32_t role = profInfo->role;
356 0 : double durationEstimated = profInfo->durationEstimated;
357 :
358 0 : reporterData.dataLen = sizeof(MsprofDpuHcclTrack);
359 0 : auto* dpuProfInfo = reinterpret_cast<MsprofDpuHcclTrack*>(reporterData.data);
360 0 : *dpuProfInfo = MsprofDpuHcclTrack();
361 :
362 0 : dpuProfInfo->itemId = itemId;
363 0 : dpuProfInfo->cclTag = cclTag;
364 0 : dpuProfInfo->groupName = groupName;
365 0 : dpuProfInfo->localRank = localRank;
366 0 : dpuProfInfo->remoteRank = remoteRank;
367 0 : dpuProfInfo->rankSize = rankSize;
368 0 : dpuProfInfo->workFlowMode = static_cast<uint8_t>(workFlowMode);
369 0 : dpuProfInfo->planeID = planeID;
370 0 : dpuProfInfo->stage = stage;
371 0 : dpuProfInfo->role = static_cast<uint8_t>(role);
372 0 : dpuProfInfo->durationEstimated = durationEstimated;
373 0 : }
374 :
375 0 : void ProfilingHandler::FillDpuTaskParaDetails(const TaskInfo& taskInfo, MsprofDpuHcclTrack* dpuProfInfo) const
376 : {
377 0 : const auto& taskPara = taskInfo.taskParam_.taskPara;
378 0 : switch (taskInfo.taskParam_.taskType) {
379 0 : case TaskParamType::TASK_DPU_INLINE_WRITE:
380 : case TaskParamType::TASK_DPU_WRITE_WITH_NOTIFY:
381 0 : dpuProfInfo->srcAddr = static_cast<u64>(reinterpret_cast<uintptr_t>(taskPara.DMA.src));
382 0 : dpuProfInfo->dstAddr = static_cast<u64>(reinterpret_cast<uintptr_t>(taskPara.DMA.dst));
383 0 : dpuProfInfo->dataSize = static_cast<u32>(taskPara.DMA.size);
384 0 : dpuProfInfo->notifyID = taskPara.DMA.notifyID;
385 0 : break;
386 0 : case TaskParamType::TASK_DPU_NOTIFY_WAIT:
387 : case TaskParamType::TASK_DPU_CHANNEL_FENCE:
388 0 : dpuProfInfo->notifyID = taskPara.Notify.notifyID;
389 0 : break;
390 0 : default:
391 0 : break;
392 : }
393 0 : }
394 :
395 0 : void ProfilingHandler::FillDpuProfInfo(const TaskInfo& taskInfo, MsprofAdditionalInfo& reporterData) const
396 : {
397 0 : if (taskInfo.dfxOpInfo_ == nullptr) {
398 0 : HCCL_WARNING("[ProfilingHandler::FillDpuProfInfo] taskInfo.dfxOpInfo_ is nullptr, skip GetDpuProfInfo!");
399 0 : return;
400 : }
401 0 : ConvertHcclInfoToDpuTrack(reporterData);
402 0 : auto* dpuProfInfo = reinterpret_cast<MsprofDpuHcclTrack*>(reporterData.data);
403 0 : dpuProfInfo->dataType = static_cast<uint32_t>(taskInfo.dfxOpInfo_->op_.dataType);
404 0 : dpuProfInfo->opType = static_cast<uint32_t>(taskInfo.dfxOpInfo_->op_.opType);
405 0 : dpuProfInfo->transportType = static_cast<uint32_t>(SimpleTaskType::ROCE);
406 0 : dpuProfInfo->aicpu_task_id = taskInfo.taskParam_.aicpuTaskId;
407 0 : dpuProfInfo->npuDevId = taskInfo.taskParam_.npuDevId;
408 0 : dpuProfInfo->dpuDevId = DPU_DEV_ID_MASK;
409 0 : const auto& taskPara = taskInfo.taskParam_.taskPara;
410 0 : dpuProfInfo->linkType = static_cast<uint16_t>(taskPara.DMA.linkType);
411 0 : if (dpuProfInfo->remoteRank == INVALID_VALUE_RANKID) {
412 0 : dpuProfInfo->transportType = static_cast<uint32_t>(SimpleTaskType::LOCAL);
413 0 : dpuProfInfo->remoteRank = dpuProfInfo->localRank;
414 0 : dpuProfInfo->linkType = 0;
415 : }
416 0 : dpuProfInfo->taskId = taskInfo.taskId_;
417 0 : dpuProfInfo->streamId = taskInfo.streamId_;
418 0 : dpuProfInfo->timeStamp = taskInfo.taskParam_.beginTime;
419 0 : HCCL_INFO(
420 : "[FillDpuProfInfo]taskId[%u], streamId[%u], npuDevId[%u], dpuDevId[%u], "
421 : "starttime[%llu], endtime[%llu], aicputaskId[%u].",
422 : dpuProfInfo->taskId, dpuProfInfo->streamId, dpuProfInfo->npuDevId, dpuProfInfo->dpuDevId,
423 : dpuProfInfo->timeStamp, reporterData.timeStamp, dpuProfInfo->aicpu_task_id);
424 0 : FillDpuTaskParaDetails(taskInfo, dpuProfInfo);
425 : }
426 :
427 9 : void ProfilingHandler::GetHCCLReportData(const TaskInfo& taskInfo, HCCLReportData& hcclReportData) const
428 : {
429 9 : if (taskInfo.dfxOpInfo_ == nullptr) {
430 3 : HCCL_ERROR("[ProfilingHandler]taskInfo.dfxOpInfo_ is nullptr");
431 1 : return;
432 : }
433 8 : hcclReportData.ts = taskInfo.taskParam_.endTime;
434 8 : const auto& profName = GetProfTaskOpNameV2(taskInfo.taskParam_.taskType);
435 8 : hcclReportData.profInfo.itemId = GetProfHashId(profName.c_str(), profName.length());
436 8 : auto newCclTagIt = cachedNewCclTag_.find(taskInfo.dfxOpInfo_->op_.opType);
437 8 : hcclReportData.profInfo.cclTag = (newCclTagIt != cachedNewCclTag_.end()) ? newCclTagIt->second : INVALID_U64;
438 8 : const auto& opTag = taskInfo.dfxOpInfo_->op_.opTag;
439 8 : uint64_t groupName = GetProfHashId(opTag.c_str(), opTag.length());
440 8 : if (taskInfo.dfxOpInfo_->comm_ == nullptr) {
441 0 : HCCL_ERROR("[ProfilingHandler]taskInfo.dfxOpInfo_->comm_ is nullptr");
442 0 : return;
443 : }
444 8 : if (taskInfo.dfxOpInfo_->isIndop_ == true) {
445 0 : hcclReportData.profInfo.groupName = groupName;
446 0 : hcclReportData.profInfo.rankSize = taskInfo.dfxOpInfo_->rankSize_;
447 : } else {
448 8 : CommunicatorImpl* commImp = static_cast<CommunicatorImpl*>(taskInfo.dfxOpInfo_->comm_);
449 8 : if (commImp == nullptr) {
450 0 : HCCL_ERROR("[ProfilingHandler]commImp is nullptr");
451 0 : return;
452 : }
453 8 : hcclReportData.profInfo.groupName = groupName;
454 8 : hcclReportData.profInfo.rankSize = commImp->GetRankSize();
455 : }
456 8 : hcclReportData.profInfo.workFlowMode = static_cast<u32>(HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE);
457 8 : hcclReportData.profInfo.planeID = 0;
458 8 : hcclReportData.profInfo.stage = 0;
459 8 : hcclReportData.profInfo.role = static_cast<uint32_t>(TaskRole::DST);
460 8 : hcclReportData.profInfo.durationEstimated = 0;
461 8 : hcclReportData.profInfo.localRank = taskInfo.dfxOpInfo_->op_.myRank;
462 8 : hcclReportData.profInfo.remoteRank = taskInfo.remoteRank_;
463 8 : hcclReportData.profInfo.dataType = taskInfo.dfxOpInfo_->op_.dataType;
464 8 : hcclReportData.profInfo.opType = taskInfo.dfxOpInfo_->op_.opType;
465 8 : hcclReportData.profInfo.transportType = static_cast<int32_t>(SimpleTaskType::UB);
466 8 : if (hcclReportData.profInfo.remoteRank == INVALID_VALUE_RANKID) {
467 0 : hcclReportData.profInfo.transportType = static_cast<int32_t>(SimpleTaskType::LOCAL);
468 0 : hcclReportData.profInfo.remoteRank = hcclReportData.profInfo.localRank;
469 0 : hcclReportData.profInfo.linkType = 0;
470 : }
471 8 : const auto& taskType = taskInfo.taskParam_.taskType;
472 8 : const auto& taskPara = taskInfo.taskParam_.taskPara;
473 8 : switch (taskType) {
474 4 : case TaskParamType::TASK_DPU_INLINE_WRITE:
475 : case TaskParamType::TASK_DPU_NOTIFY_WAIT:
476 : case TaskParamType::TASK_DPU_WRITE_WITH_NOTIFY:
477 : case TaskParamType::TASK_DPU_CHANNEL_FENCE:
478 4 : hcclReportData.dpuProfInfo.itemId = hcclReportData.profInfo.itemId;
479 4 : hcclReportData.dpuProfInfo.cclTag = hcclReportData.profInfo.cclTag;
480 4 : hcclReportData.dpuProfInfo.groupName = hcclReportData.profInfo.groupName;
481 4 : hcclReportData.dpuProfInfo.localRank = hcclReportData.profInfo.localRank;
482 4 : hcclReportData.dpuProfInfo.remoteRank = hcclReportData.profInfo.remoteRank;
483 4 : hcclReportData.dpuProfInfo.rankSize = hcclReportData.profInfo.rankSize;
484 4 : hcclReportData.dpuProfInfo.workFlowMode = hcclReportData.profInfo.workFlowMode;
485 4 : hcclReportData.dpuProfInfo.planeID = hcclReportData.profInfo.planeID;
486 4 : hcclReportData.dpuProfInfo.stage = hcclReportData.profInfo.stage;
487 4 : hcclReportData.dpuProfInfo.role = hcclReportData.profInfo.role;
488 4 : hcclReportData.dpuProfInfo.durationEstimated = hcclReportData.profInfo.durationEstimated;
489 4 : hcclReportData.dpuProfInfo.dataType = static_cast<uint32_t>(taskInfo.dfxOpInfo_->op_.dataType);
490 4 : hcclReportData.dpuProfInfo.opType = static_cast<uint32_t>(taskInfo.dfxOpInfo_->op_.opType);
491 4 : hcclReportData.dpuProfInfo.transportType = static_cast<uint32_t>(SimpleTaskType::ROCE);
492 4 : hcclReportData.dpuProfInfo.aicpu_task_id = taskInfo.taskParam_.aicpuTaskId;
493 4 : hcclReportData.dpuProfInfo.npuDevId = taskInfo.taskParam_.npuDevId;
494 4 : hcclReportData.dpuProfInfo.dpuDevId = DPU_DEV_ID_MASK;
495 4 : hcclReportData.dpuProfInfo.linkType = static_cast<uint16_t>(taskPara.DMA.linkType);
496 4 : if (hcclReportData.dpuProfInfo.remoteRank == INVALID_VALUE_RANKID) {
497 0 : hcclReportData.dpuProfInfo.transportType = static_cast<uint32_t>(SimpleTaskType::LOCAL);
498 0 : hcclReportData.dpuProfInfo.remoteRank = hcclReportData.dpuProfInfo.localRank;
499 0 : hcclReportData.dpuProfInfo.linkType = 0;
500 : }
501 4 : hcclReportData.dpuProfInfo.taskId = taskInfo.taskId_;
502 4 : hcclReportData.dpuProfInfo.streamId = taskInfo.streamId_;
503 4 : hcclReportData.dpuProfInfo.timeStamp = taskInfo.taskParam_.beginTime;
504 4 : switch (taskType) {
505 2 : case TaskParamType::TASK_DPU_INLINE_WRITE:
506 : case TaskParamType::TASK_DPU_WRITE_WITH_NOTIFY:
507 : hcclReportData.dpuProfInfo.srcAddr
508 2 : = static_cast<u64>(reinterpret_cast<uintptr_t>(taskPara.DMA.src));
509 : hcclReportData.dpuProfInfo.dstAddr
510 2 : = static_cast<u64>(reinterpret_cast<uintptr_t>(taskPara.DMA.dst));
511 2 : hcclReportData.dpuProfInfo.dataSize = static_cast<u32>(taskPara.DMA.size);
512 2 : hcclReportData.dpuProfInfo.notifyID = taskPara.DMA.notifyID;
513 2 : break;
514 2 : case TaskParamType::TASK_DPU_NOTIFY_WAIT:
515 : case TaskParamType::TASK_DPU_CHANNEL_FENCE:
516 2 : hcclReportData.dpuProfInfo.notifyID = taskPara.Notify.notifyID;
517 2 : break;
518 0 : default:
519 0 : break;
520 : }
521 4 : break;
522 4 : default:
523 4 : switch (taskType) {
524 1 : case TaskParamType::TASK_SDMA:
525 : case TaskParamType::TASK_RDMA:
526 1 : hcclReportData.profInfo.srcAddr = static_cast<u64>(reinterpret_cast<uintptr_t>(taskPara.DMA.src));
527 1 : hcclReportData.profInfo.dstAddr = static_cast<u64>(reinterpret_cast<uintptr_t>(taskPara.DMA.dst));
528 1 : hcclReportData.profInfo.dataSize = static_cast<u32>(taskPara.DMA.size);
529 1 : hcclReportData.profInfo.notifyID = taskPara.DMA.notifyID;
530 1 : hcclReportData.profInfo.linkType = static_cast<uint16_t>(taskPara.DMA.linkType);
531 1 : break;
532 1 : case TaskParamType::TASK_REDUCE_INLINE:
533 : case TaskParamType::TASK_REDUCE_TBE:
534 : hcclReportData.profInfo.srcAddr
535 1 : = static_cast<u64>(reinterpret_cast<uintptr_t>(taskPara.Reduce.src));
536 : hcclReportData.profInfo.dstAddr
537 1 : = static_cast<u64>(reinterpret_cast<uintptr_t>(taskPara.Reduce.dst));
538 1 : hcclReportData.profInfo.dataSize = static_cast<u32>(taskPara.Reduce.size);
539 1 : hcclReportData.profInfo.notifyID = taskPara.Reduce.notifyID;
540 1 : hcclReportData.profInfo.linkType = static_cast<uint16_t>(taskPara.Reduce.linkType);
541 1 : break;
542 1 : case TaskParamType::TASK_NOTIFY_RECORD:
543 : case TaskParamType::TASK_NOTIFY_WAIT:
544 1 : hcclReportData.profInfo.notifyID = taskPara.Notify.notifyID;
545 1 : break;
546 1 : default:
547 1 : break;
548 : }
549 4 : break;
550 : }
551 8 : }
552 :
553 1 : void ProfilingHandler::DumpHCCLReportData(const TaskInfo& taskInfo, const MsprofAdditionalInfo& reporterData) const
554 : {
555 1 : if (reporterData.type == static_cast<uint32_t>(ProfTaskType::TASK_HCCL_INFO)) {
556 1 : auto* profInfo = reinterpret_cast<const MsprofHcclInfo*>(reporterData.data);
557 3 : HCCL_INFO(
558 : "MsprofAdditionalInfo profInfo: timeStamp[%llu], itemId[%llu], cclTag[%llu], groupName[%llu], "
559 : "localRank[%u], remoteRank[%u], rankSize[%u], workFlowMode[%u], planeID[%u], ctxId[%u], "
560 : "stage[%u], role[%u], durationEstimated[%f], taskType[%d]",
561 : reporterData.timeStamp, profInfo->itemId, profInfo->cclTag, profInfo->groupName, profInfo->localRank,
562 : profInfo->remoteRank, profInfo->rankSize, profInfo->workFlowMode, profInfo->planeID, profInfo->ctxId,
563 : profInfo->stage, profInfo->role, profInfo->durationEstimated,
564 : static_cast<int>(taskInfo.taskParam_.taskType));
565 3 : HCCL_INFO(
566 : "MsprofAdditionalInfo profInfo detail: srcAddr[%llu], dstAddr[%llu], dataSize[%llu], notifyID[%llu], "
567 : "linkType[%u], opType[%s], transportType[%u], dataType[%s], rdmaType[%u]",
568 : profInfo->srcAddr, profInfo->dstAddr, profInfo->dataSize, profInfo->notifyID, profInfo->linkType,
569 : OpTypeToSerialString(profInfo->opType).c_str(), profInfo->transportType,
570 : DataTypeToSerialString(profInfo->dataType).c_str(), profInfo->rdmaType);
571 : } else {
572 0 : auto* dpuProfInfo = reinterpret_cast<const MsprofDpuHcclTrack*>(reporterData.data);
573 0 : HCCL_INFO(
574 : "MsprofAdditionalInfo dpuProfInfo: timeStamp[%llu], itemId[%llu], cclTag[%llu], groupName[%llu], "
575 : "localRank[%u], remoteRank[%u], rankSize[%u], workFlowMode[%u], planeID[%u], "
576 : "stage[%u], role[%u], durationEstimated[%f], taskType[%d]",
577 : reporterData.timeStamp, dpuProfInfo->itemId, dpuProfInfo->cclTag, dpuProfInfo->groupName,
578 : dpuProfInfo->localRank, dpuProfInfo->remoteRank, dpuProfInfo->rankSize, dpuProfInfo->workFlowMode,
579 : dpuProfInfo->planeID, dpuProfInfo->stage, dpuProfInfo->role, dpuProfInfo->durationEstimated,
580 : static_cast<int>(taskInfo.taskParam_.taskType));
581 0 : HCCL_INFO(
582 : "MsprofAdditionalInfo dpuProfInfo detail: srcAddr[%llu], dstAddr[%llu], dataSize[%llu], notifyID[%llu], "
583 : "linkType[%u], opType[%s], transportType[%u], dataType[%s], rdmaType[%u], "
584 : "taskId[%u], aicpu_task_id[%u], streamId[%u], npuDevId[%u], dpuDevId[%u], timeStamp[%llu]",
585 : dpuProfInfo->srcAddr, dpuProfInfo->dstAddr, dpuProfInfo->dataSize, dpuProfInfo->notifyID,
586 : dpuProfInfo->linkType, OpTypeToSerialString(dpuProfInfo->opType).c_str(), dpuProfInfo->transportType,
587 : DataTypeToSerialString(dpuProfInfo->dataType).c_str(), dpuProfInfo->rdmaType, dpuProfInfo->taskId,
588 : dpuProfInfo->aicpu_task_id, dpuProfInfo->streamId, dpuProfInfo->npuDevId, dpuProfInfo->dpuDevId,
589 : dpuProfInfo->timeStamp);
590 : }
591 1 : }
592 :
593 0 : void ProfilingHandler::LogCcuTaskInfo(
594 : const CcuProfilingInfo& info, const TaskInfo& taskInfo, uint64_t itemId, uint64_t groupName, u32 rankId,
595 : u32 ranksize) const
596 : {
597 0 : HCCL_INFO(
598 : "[ProfilingHandler]GetCcuTaskInfo, ccuTaskInfo data is: version[%u], workFlowMode[%u], itemId[%llu], "
599 : "groupName[%llu], rankId[%u], ranksize[%u], streamId[%u], taskId[%u], dieId[%u], "
600 : "missionId[%u],instrId[%u]",
601 : 0, static_cast<u32>(HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE), itemId, groupName, rankId, ranksize,
602 : taskInfo.streamId_, taskInfo.taskId_, info.dieId, info.missionId, info.instrId);
603 0 : }
604 :
605 0 : void ProfilingHandler::LogCcuWaitSignalInfo(
606 : const CcuProfilingInfo& info, const TaskInfo& taskInfo, uint64_t itemId, uint64_t groupName, u32 rankId,
607 : u32 ranksize) const
608 : {
609 0 : HCCL_INFO(
610 : "[ProfilingHandler]GetCcuWaitSignalInfo, waitSignalInfo data is: version[%u], itemId[%llu], groupName[%llu], "
611 : "rankId[%u], ranksize[%u], workFlowMode[%u], streamId[%u], taskId[%u], dieId[%u],instrId[%u],missionId[%u], "
612 : "ckeId[%u],mask[%u]",
613 : 0, itemId, groupName, rankId, ranksize, static_cast<u32>(HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE),
614 : taskInfo.streamId_, taskInfo.taskId_, info.dieId, info.instrId, info.missionId, info.ckeId, info.mask);
615 0 : }
616 :
617 0 : void ProfilingHandler::LogCcuGroupInfo(
618 : const CcuProfilingInfo& info, const TaskInfo& taskInfo, uint64_t itemId, uint64_t groupName, u32 rankId,
619 : u32 ranksize) const
620 : {
621 0 : HCCL_INFO(
622 : "[ProfilingHandler]GetCcuGroupInfo, ccuGroupInfo data is: version[%u], itemId[%llu], "
623 : "groupName[%llu], rankId[%u], ranksize[%u], workFlowMode[%u], streamId[%u], taskId[%u], "
624 : "dieId[%u],instrId[%u],missionId[%u], dataSize[%llu]",
625 : 0, itemId, groupName, rankId, ranksize, static_cast<u32>(HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE),
626 : taskInfo.streamId_, taskInfo.taskId_, info.dieId, info.instrId, info.missionId, info.dataSize);
627 0 : if (info.reduceOpType != INVALID_TYPE_VALUE) {
628 0 : HCCL_INFO("ccuGroupInfo reduceOpType is [%d]", static_cast<int>(info.reduceOpType));
629 : }
630 0 : if (info.inputDataType != INVALID_TYPE_VALUE) {
631 0 : HCCL_INFO("ccuGroupInfo inputDataType is [%d]", static_cast<int>(info.inputDataType));
632 : }
633 0 : if (info.outputDataType != INVALID_TYPE_VALUE) {
634 0 : HCCL_INFO("ccuGroupInfo outputDataType is [%d]", static_cast<int>(info.outputDataType));
635 : }
636 0 : for (auto i = 0; i < CCU_MAX_CHANNEL_NUM; i++) {
637 0 : if (info.channelId[i] != INVALID_VALUE_CHANNELID && info.remoteRankId[i] != INVALID_VALUE_RANKID) {
638 0 : HCCL_INFO(
639 : "[ProfilingHandler]GetCcuGroupInfo, ccuGroupInfo data is: channelId[%d] = %u, "
640 : "remoteRankId[%d] = %u",
641 : i, info.channelId[i], i, info.remoteRankId[i]);
642 : }
643 : }
644 0 : }
645 :
646 4 : void ProfilingHandler::ReportCcuInfoLog(const TaskInfo& taskInfo) const
647 : {
648 4 : if (LIKELY(HcclCheckLogLevel(HCCL_LOG_INFO) == 0)) {
649 4 : return;
650 : }
651 4 : if (!enableHcclL1_) {
652 0 : return;
653 : }
654 4 : if (taskInfo.taskParam_.ccuDetailInfo == nullptr || taskInfo.dfxOpInfo_ == nullptr) {
655 4 : return;
656 : }
657 0 : auto ccuDetailInfo = taskInfo.taskParam_.ccuDetailInfo;
658 0 : for (const auto& info : *ccuDetailInfo) {
659 0 : uint64_t itemId = GetProfHashId(info.name.c_str(), info.name.length());
660 : uint64_t groupName
661 0 : = GetProfHashId(taskInfo.dfxOpInfo_->op_.opTag.c_str(), taskInfo.dfxOpInfo_->op_.opTag.length());
662 0 : u32 rankId = 0;
663 0 : u32 ranksize = 0;
664 0 : if (taskInfo.dfxOpInfo_->isIndop_ == true) {
665 0 : rankId = taskInfo.dfxOpInfo_->op_.myRank;
666 0 : ranksize = taskInfo.dfxOpInfo_->rankSize_;
667 : } else {
668 0 : CommunicatorImpl* commImp = static_cast<CommunicatorImpl*>(taskInfo.dfxOpInfo_->comm_);
669 0 : rankId = commImp->GetIdIndex();
670 0 : ranksize = commImp->GetRankSize();
671 : }
672 0 : if (info.type == 0) {
673 0 : LogCcuTaskInfo(info, taskInfo, itemId, groupName, rankId, ranksize);
674 0 : } else if (info.type == 1) {
675 0 : LogCcuWaitSignalInfo(info, taskInfo, itemId, groupName, rankId, ranksize);
676 0 : } else if (info.type == CCU_TYPE) {
677 0 : LogCcuGroupInfo(info, taskInfo, itemId, groupName, rankId, ranksize);
678 : }
679 : }
680 0 : }
681 :
682 4 : void ProfilingHandler::ReportCcuInfo(const TaskInfo& taskInfo) const
683 : {
684 4 : ReportCcuInfoLog(taskInfo);
685 4 : if (taskInfo.taskParam_.ccuDetailInfo == nullptr) {
686 9 : HCCL_ERROR("[ProfilingHandler]ReportCcuInfo ccuDetailInfo is nullptr.");
687 4 : return;
688 : }
689 1 : if (taskInfo.dfxOpInfo_ == nullptr) {
690 3 : HCCL_WARNING("[ProfilingHandler]ReportCcuInfo dfxOpInfo_ is nullptr, skip ReportCcuInfo.");
691 1 : return;
692 : }
693 0 : auto ccuDetailInfo = taskInfo.taskParam_.ccuDetailInfo;
694 0 : for (const auto& info : *ccuDetailInfo) {
695 0 : if (info.type == 0 && enableHcclL1_) {
696 0 : GetCcuTaskInfo(taskInfo, info);
697 0 : } else if (info.type == 1 && enableHcclL1_) {
698 0 : GetCcuWaitSignalInfo(taskInfo, info);
699 0 : } else if (info.type == CCU_TYPE && enableHcclL1_) {
700 0 : GetCcuGroupInfo(taskInfo, info);
701 : }
702 : }
703 0 : }
704 :
705 0 : void ProfilingHandler::GetCcuTaskInfo(const TaskInfo& taskInfo, const CcuProfilingInfo& info) const
706 : {
707 0 : uint64_t timestamp = DlProfFunction::GetInstance().dlMsprofSysCycleTime();
708 0 : MsprofAdditionalInfo reporterData{};
709 0 : reporterData.level = MSPROF_REPORT_HCCL_NODE_LEVEL;
710 0 : reporterData.type = MSPROF_REPORT_CCU_TASK_INFO;
711 0 : reporterData.threadId = SalGetTid();
712 0 : reporterData.dataLen = sizeof(MsprofCcuTaskInfo);
713 0 : reporterData.timeStamp = timestamp;
714 0 : auto* ccuTaskInfo = reinterpret_cast<MsprofCcuTaskInfo*>(reporterData.data);
715 0 : ccuTaskInfo->version = 0;
716 0 : ccuTaskInfo->workFlowMode = static_cast<u32>(HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE);
717 0 : ccuTaskInfo->itemId = GetProfHashId(info.name.c_str(), info.name.length());
718 0 : uint64_t groupName = GetProfHashId(taskInfo.dfxOpInfo_->op_.opTag.c_str(), taskInfo.dfxOpInfo_->op_.opTag.length());
719 0 : ccuTaskInfo->groupName = groupName;
720 0 : if (taskInfo.dfxOpInfo_->isIndop_ == true) {
721 0 : ccuTaskInfo->rankId = taskInfo.dfxOpInfo_->op_.myRank;
722 0 : ccuTaskInfo->ranksize = taskInfo.dfxOpInfo_->rankSize_;
723 : } else {
724 0 : CommunicatorImpl* commImp = static_cast<CommunicatorImpl*>(taskInfo.dfxOpInfo_->comm_);
725 0 : ccuTaskInfo->rankId = commImp->GetIdIndex();
726 0 : ccuTaskInfo->ranksize = commImp->GetRankSize();
727 : }
728 :
729 0 : ccuTaskInfo->streamId = taskInfo.streamId_;
730 0 : ccuTaskInfo->taskId = taskInfo.taskId_;
731 0 : ccuTaskInfo->dieId = info.dieId;
732 0 : ccuTaskInfo->missionId = info.missionId;
733 0 : ccuTaskInfo->instrId = info.instrId;
734 0 : ReportAdditionInfo(reporterData);
735 0 : }
736 :
737 1 : void ProfilingHandler::GetCcuGroupInfo(const TaskInfo& taskInfo, const CcuProfilingInfo& info) const
738 : {
739 1 : MsprofAdditionalInfo reporterData{};
740 1 : reporterData.level = MSPROF_REPORT_HCCL_NODE_LEVEL;
741 1 : reporterData.type = MSPROF_REPORT_CCU_GROUP_INFO;
742 1 : reporterData.threadId = SalGetTid();
743 1 : reporterData.dataLen = sizeof(MsprofCcuGroupInfo);
744 1 : reporterData.timeStamp = DlProfFunction::GetInstance().dlMsprofSysCycleTime();
745 1 : auto* ccuGroupInfo = reinterpret_cast<MsprofCcuGroupInfo*>(reporterData.data);
746 1 : ccuGroupInfo->version = 0;
747 1 : ccuGroupInfo->itemId = GetProfHashId(info.name.c_str(), info.name.length());
748 1 : uint64_t groupName = GetProfHashId(taskInfo.dfxOpInfo_->op_.opTag.c_str(), taskInfo.dfxOpInfo_->op_.opTag.length());
749 1 : ccuGroupInfo->groupName = groupName;
750 1 : if (taskInfo.dfxOpInfo_->isIndop_ == true) {
751 0 : ccuGroupInfo->rankId = taskInfo.dfxOpInfo_->op_.myRank;
752 0 : ccuGroupInfo->ranksize = taskInfo.dfxOpInfo_->rankSize_;
753 : } else {
754 1 : CommunicatorImpl* commImp = static_cast<CommunicatorImpl*>(taskInfo.dfxOpInfo_->comm_);
755 1 : ccuGroupInfo->rankId = commImp->GetIdIndex();
756 1 : ccuGroupInfo->ranksize = commImp->GetRankSize();
757 : }
758 1 : ccuGroupInfo->workFlowMode = static_cast<u32>(HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE);
759 1 : ccuGroupInfo->streamId = taskInfo.streamId_;
760 1 : ccuGroupInfo->taskId = taskInfo.taskId_;
761 1 : ccuGroupInfo->dieId = info.dieId;
762 1 : ccuGroupInfo->instrId = info.instrId;
763 1 : ccuGroupInfo->missionId = info.missionId;
764 1 : ccuGroupInfo->reduceOpType = info.reduceOpType;
765 1 : ccuGroupInfo->inputDataType = info.inputDataType;
766 1 : ccuGroupInfo->outputDataType = info.outputDataType;
767 1 : ccuGroupInfo->dataSize = info.dataSize;
768 1 : std::copy(info.channelId, info.channelId + CCU_MAX_CHANNEL_NUM, ccuGroupInfo->channelId);
769 1 : std::copy(info.remoteRankId, info.remoteRankId + CCU_MAX_CHANNEL_NUM, ccuGroupInfo->remoteRankId);
770 1 : ReportAdditionInfo(reporterData);
771 1 : }
772 :
773 0 : void ProfilingHandler::DumpCcuGroupInfo([[maybe_unused]] const MsprofCcuGroupInfo& ccuGroupInfo) const
774 : {
775 0 : if (LIKELY(HcclCheckLogLevel(HCCL_LOG_INFO) == 0)) {
776 0 : return;
777 : }
778 : // HCCL_INFOs migrated to ReportCcuInfoLog
779 : }
780 :
781 1 : void ProfilingHandler::GetCcuWaitSignalInfo(const TaskInfo& taskInfo, const CcuProfilingInfo& info) const
782 : {
783 1 : uint64_t timestamp = DlProfFunction::GetInstance().dlMsprofSysCycleTime();
784 1 : MsprofAdditionalInfo reporterData{};
785 1 : reporterData.level = MSPROF_REPORT_HCCL_NODE_LEVEL;
786 1 : reporterData.type = MSPROF_REPORT_CCU_WAIT_SIGNAL_INFO;
787 1 : reporterData.threadId = SalGetTid();
788 1 : reporterData.dataLen = sizeof(MsprofCcuWaitSignalInfo);
789 1 : reporterData.timeStamp = timestamp;
790 1 : auto* waitSignalInfo = reinterpret_cast<MsprofCcuWaitSignalInfo*>(reporterData.data);
791 1 : waitSignalInfo->version = 0;
792 1 : waitSignalInfo->itemId = GetProfHashId(info.name.c_str(), info.name.length());
793 1 : uint64_t groupName = GetProfHashId(taskInfo.dfxOpInfo_->op_.opTag.c_str(), taskInfo.dfxOpInfo_->op_.opTag.length());
794 1 : waitSignalInfo->groupName = groupName;
795 1 : if (taskInfo.dfxOpInfo_->isIndop_ == true) {
796 0 : waitSignalInfo->rankId = taskInfo.dfxOpInfo_->op_.myRank;
797 0 : waitSignalInfo->ranksize = taskInfo.dfxOpInfo_->rankSize_;
798 : } else {
799 1 : CommunicatorImpl* commImp = static_cast<CommunicatorImpl*>(taskInfo.dfxOpInfo_->comm_);
800 1 : waitSignalInfo->rankId = commImp->GetIdIndex();
801 1 : waitSignalInfo->ranksize = commImp->GetRankSize();
802 : }
803 1 : waitSignalInfo->workFlowMode = static_cast<u32>(HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE);
804 1 : waitSignalInfo->streamId = taskInfo.streamId_;
805 1 : waitSignalInfo->taskId = taskInfo.taskId_;
806 1 : waitSignalInfo->dieId = info.dieId;
807 1 : waitSignalInfo->instrId = info.instrId;
808 1 : waitSignalInfo->missionId = info.missionId;
809 1 : waitSignalInfo->ckeId = info.ckeId;
810 1 : waitSignalInfo->mask = info.mask;
811 1 : std::copy(info.channelId, info.channelId + CCU_MAX_CHANNEL_NUM, waitSignalInfo->channelId);
812 1 : std::copy(info.remoteRankId, info.remoteRankId + CCU_MAX_CHANNEL_NUM, waitSignalInfo->remoteRankId);
813 1 : ReportAdditionInfo(reporterData);
814 1 : }
815 :
816 2 : void ProfilingHandler::ReportAclApi(
817 : uint32_t cmdType, uint64_t beginTime, uint64_t endTime, uint64_t cmdItemId, uint32_t threadId, bool cachedReq)
818 : {
819 2 : MsprofApi reporterData{};
820 2 : reporterData.level = MSPROF_REPORT_ACL_LEVEL;
821 2 : reporterData.type = static_cast<int32_t>(cmdType) + MSPROF_REPORT_ACL_HOST_HCCL_BASE_TYPE;
822 2 : reporterData.threadId = threadId;
823 2 : reporterData.beginTime = beginTime;
824 2 : reporterData.endTime = endTime;
825 2 : reporterData.itemId = cmdItemId;
826 :
827 2 : if (cachedReq) {
828 1 : std::lock_guard<std::mutex> lock(cachedAclApiInfoMutex_);
829 3 : HCCL_INFO(
830 : "ReportAclApi MSPROF_REPORT_ACL_LEVEL type[%u] beginTime[%llu]", reporterData.type, reporterData.beginTime);
831 1 : cachedAclApiInfo_.push(reporterData);
832 1 : }
833 :
834 2 : s32 ret = DlProfFunction::GetInstance().dlMsprofReportApi(1, &reporterData);
835 6 : HCCL_INFO(
836 : "[ProfilingHandler][ReportAclApi], reporterData data is: level[%u], type[%u], threadId[%u], "
837 : "beginTime[%llu], endTime[%llu], itemId[%llu], return value[%d]",
838 : reporterData.level, reporterData.type, reporterData.threadId, reporterData.beginTime, reporterData.endTime,
839 : reporterData.itemId, ret);
840 2 : if (ret != 0) {
841 0 : THROW<InternalException>("Call dlMsprofReportApi failed, return[%d]", ret);
842 : }
843 2 : }
844 :
845 4 : void ProfilingHandler::ReportNodeApi(
846 : uint64_t beginTime, uint64_t endTime, uint64_t cmdItemId, uint32_t threadId, bool cachedReq)
847 : {
848 4 : MsprofApi reporterData{};
849 4 : reporterData.level = MSPROF_REPORT_NODE_LEVEL;
850 4 : reporterData.type = MSPROF_REPORT_NODE_LAUNCH_TYPE;
851 4 : reporterData.threadId = threadId;
852 4 : reporterData.beginTime = beginTime;
853 4 : reporterData.endTime = endTime;
854 4 : reporterData.itemId = cmdItemId;
855 :
856 4 : if (cachedReq) {
857 2 : std::lock_guard<std::mutex> lock(cachedTaskApiInfoMutex_);
858 2 : cachedTaskApiInfo_.push(reporterData);
859 2 : }
860 4 : if (!enableHostApi_) {
861 3 : return;
862 : }
863 :
864 1 : s32 ret = DlProfFunction::GetInstance().dlMsprofReportApi(1, &reporterData);
865 3 : HCCL_INFO(
866 : "[ProfilingHandler][ReportNodeApi], reporterData data is: level[%u], type[%u], threadId[%u], "
867 : "beginTime[%llu], endTime[%llu], itemId[%llu], return value[%d]",
868 : reporterData.level, reporterData.type, reporterData.threadId, reporterData.beginTime, reporterData.endTime,
869 : reporterData.itemId, ret);
870 1 : if (ret != 0) {
871 0 : THROW<InternalException>("Call MsprofReportApi failed, return[%d]", ret);
872 : }
873 : }
874 :
875 4 : void ProfilingHandler::ReportNodeBasicInfo(uint64_t timeStamp, uint64_t cmdItemId, uint32_t threadId, bool cachedReq)
876 : {
877 4 : MsprofCompactInfo reporterData{};
878 4 : reporterData.level = MSPROF_REPORT_NODE_LEVEL;
879 4 : reporterData.type = MSPROF_REPORT_NODE_BASIC_INFO_TYPE;
880 4 : reporterData.threadId = threadId;
881 4 : reporterData.dataLen = sizeof(MsprofNodeBasicInfo);
882 4 : reporterData.timeStamp = timeStamp;
883 4 : reporterData.data.nodeBasicInfo.opName = cmdItemId;
884 4 : reporterData.data.nodeBasicInfo.taskType = MSPROF_GE_TASK_TYPE_HCCL;
885 4 : reporterData.data.nodeBasicInfo.opType = cmdItemId;
886 4 : reporterData.data.nodeBasicInfo.opFlag = 0;
887 12 : HCCL_INFO(
888 : "[ProfilingHandler][ReportNodeBasicInfo], reporterData data is: level[%u], type[%u], threadId[%u], "
889 : "dataLen[%u], taskType[%u], opFlag[%u]",
890 : reporterData.level, reporterData.type, reporterData.threadId, reporterData.dataLen,
891 : reporterData.data.nodeBasicInfo.taskType, reporterData.data.nodeBasicInfo.opFlag);
892 4 : if (cachedReq) {
893 2 : std::lock_guard<std::mutex> lock(cacheHcclOpInfoMutex_);
894 2 : cacheHcclOpInfo_.push(reporterData);
895 2 : }
896 4 : if (!enableHcclL1_) {
897 2 : return;
898 : }
899 2 : s32 ret = DlProfFunction::GetInstance().dlMsprofReportCompactInfo(1, &reporterData, sizeof(MsprofCompactInfo));
900 6 : HCCL_INFO("Call MsprofReportCompactInfo, return value[%d]", ret);
901 2 : if (ret != 0) {
902 0 : THROW<InternalException>("Call MsprofReportCompactInfo failed, return[%d]", ret);
903 : }
904 : }
905 :
906 5 : void ProfilingHandler::ReportHcclOpInfo(uint64_t timeStamp, const DfxOpInfo& opInfo, uint32_t threadId, bool cachedReq)
907 : {
908 5 : MsprofCompactInfo reporterData{};
909 5 : reporterData.level = MSPROF_REPORT_NODE_LEVEL;
910 5 : reporterData.type = MSPROF_REPORT_NODE_HCCL_OP_INFO_TYPE;
911 5 : reporterData.threadId = threadId;
912 5 : reporterData.dataLen = sizeof(MsprofHCCLOPInfo);
913 5 : reporterData.timeStamp = timeStamp;
914 5 : reporterData.data.hcclopInfo.relay = 0;
915 5 : reporterData.data.hcclopInfo.retry = 0;
916 5 : reporterData.data.hcclopInfo.dataType = opInfo.op_.dataType;
917 5 : reporterData.data.hcclopInfo.algType = cachedAlgTypeHashId_.load();
918 5 : uint64_t groupName = GetProfHashId(opInfo.op_.opTag.c_str(), opInfo.op_.opTag.length());
919 5 : reporterData.data.hcclopInfo.groupName = groupName;
920 5 : u32 ranksize{0};
921 5 : if (opInfo.isIndop_ == true) {
922 0 : ranksize = opInfo.rankSize_;
923 0 : reporterData.data.hcclopInfo.count = opInfo.op_.dataCount;
924 : } else {
925 5 : CommunicatorImpl* commImp = static_cast<CommunicatorImpl*>(opInfo.comm_);
926 5 : ranksize = commImp->GetRankSize();
927 5 : if (opInfo.op_.opType == OpType::ALLTOALLV) {
928 1 : u64 sendCount = 0;
929 3 : for (u64 i = 0; i < ranksize; i++) {
930 2 : sendCount += *(static_cast<const u64*>(opInfo.op_.all2AllVDataDes.sendCounts) + i);
931 : }
932 1 : reporterData.data.hcclopInfo.count = sendCount;
933 4 : } else if (opInfo.op_.opType == OpType::ALLTOALL) {
934 0 : reporterData.data.hcclopInfo.count = opInfo.op_.all2AllDataDes.sendCount;
935 : } else {
936 4 : reporterData.data.hcclopInfo.count = opInfo.op_.dataCount;
937 : }
938 : }
939 5 : if (cachedReq) {
940 2 : std::lock_guard<std::mutex> lock(cacheHcclOpInfoMutex_);
941 2 : cacheHcclOpInfo_.push(reporterData);
942 2 : }
943 5 : if (!enableHostApi_) {
944 3 : return;
945 : }
946 6 : HCCL_INFO(
947 : "[ProfilingHandler][ReportHcclOpInfo], data is: level[%u], type[%u], threadId[%u], dataLen[%u], "
948 : "timeStamp[%llu], relay [%u], retry[%u], dataType[%s], algType[%llu], groupName[%llu], count[%llu]",
949 : reporterData.level, reporterData.type, reporterData.threadId, reporterData.dataLen, reporterData.timeStamp,
950 : reporterData.data.hcclopInfo.relay, reporterData.data.hcclopInfo.retry,
951 : DataTypeToSerialString(reporterData.data.hcclopInfo.dataType).c_str(), reporterData.data.hcclopInfo.algType,
952 : reporterData.data.hcclopInfo.groupName, reporterData.data.hcclopInfo.count);
953 2 : s32 ret = DlProfFunction::GetInstance().dlMsprofReportCompactInfo(1, &reporterData, sizeof(MsprofCompactInfo));
954 2 : if (ret != 0) {
955 0 : THROW<InternalException>("[ProfilingHandler] Call dlMsprofReportCompactInfo failed, return[%d]", ret);
956 : }
957 : }
958 :
959 2 : void ProfilingHandler::ReportAdditionInfo(MsprofAdditionalInfo& reporterData) const
960 : {
961 : s32 ret
962 2 : = DlProfFunction::GetInstance().dlMsprofReportAdditionalInfo(0, &reporterData, sizeof(MsprofAdditionalInfo));
963 6 : HCCL_INFO(
964 : "[ProfilingHandler][ReportAdditionInfo], level[%u], type[%u], threadId[%u], dataLen[%u], "
965 : "timeStamp[%llu], return value[%d]",
966 : reporterData.level, reporterData.type, reporterData.threadId, reporterData.dataLen, reporterData.timeStamp,
967 : ret);
968 2 : if (ret != 0) {
969 0 : THROW<InternalException>("Call MsprofReportAdditionalInfo failed, return[%d]", ret);
970 : }
971 2 : }
972 :
973 2 : int32_t ProfilingHandler::CommandHandle(uint32_t rtType, void* data, uint32_t len) const
974 : {
975 : (void)len;
976 2 : if (data == nullptr || rtType != rtProfCtrlType_t::RT_PROF_CTRL_SWITCH) {
977 3 : HCCL_ERROR("[ProfilingHandler][CommandHandle] data is nullptr or rtType is invalid, rtType[%u]", rtType);
978 1 : return HCCL_E_PARA;
979 : }
980 1 : rtProfCommandHandle_t* profConfigParam = static_cast<rtProfCommandHandle_t*>(data);
981 1 : auto type = profConfigParam->type;
982 1 : auto profconfig = profConfigParam->profSwitch;
983 3 : HCCL_RUN_INFO(
984 : "[Profiling][CommandHandle] CommandHandle's rtType is %u. CommandHandle_switch type[%u], "
985 : "profconfig[%llu], deviceLogicId[%u]",
986 : rtType, type, profconfig, profConfigParam->devIdList[0]);
987 1 : switch (type) {
988 0 : case PROF_COMMANDHANDLE_TYPE_START:
989 0 : instance_.StartSubscribe(profconfig);
990 0 : break;
991 0 : case PROF_COMMANDHANDLE_TYPE_STOP:
992 0 : instance_.StopSubscribe();
993 0 : break;
994 1 : default:
995 3 : HCCL_RUN_INFO("[Profiling][CommandHandle] Unexpected behaviour.");
996 : }
997 1 : return HCCL_SUCCESS;
998 : }
999 :
1000 5 : void ProfilingHandler::StartSubscribe(uint64_t profconfig)
1001 : {
1002 15 : HCCL_RUN_INFO("[Profiling][CommandHandle] profSwitch is[%llu]", profconfig);
1003 5 : SetCachedCclTag();
1004 5 : cachedAlgTypeHashId_ = GetProfHashId("AlgType::NHR", strlen("AlgType::NHR"));
1005 5 : if ((profconfig & PROF_ACL_API_MASK) != 0) {
1006 1 : StartHostApiSubscribe();
1007 : }
1008 5 : if ((profconfig & PROF_TASK_TIME_MASK) != 0 && (profconfig & PROF_TASK_TIME_L1_MASK) == 0) {
1009 1 : StartHostHcclOpSubscribe();
1010 : }
1011 5 : if ((profconfig & PROF_TASK_TIME_L1_MASK) != 0) {
1012 1 : StartTaskApiSubscribe();
1013 1 : StartAdditionInfoSubscribe();
1014 1 : StartCcuSubscribe();
1015 : }
1016 15 : HCCL_RUN_INFO("[Profiling][CommandHandle] profSwitch is[%llu]", profconfig);
1017 5 : }
1018 :
1019 2 : void ProfilingHandler::StartHostApiSubscribe()
1020 : {
1021 2 : enableHostApi_ = true;
1022 2 : CallProfRegHostApi();
1023 2 : ReportStoragedCompactInfo();
1024 2 : ReportMc2AdditionInfo();
1025 2 : ReportStoragedAclApi();
1026 6 : HCCL_RUN_INFO("SetHostApiSubscribe:[%d]", enableHostApi_);
1027 2 : }
1028 :
1029 2 : void ProfilingHandler::CallProfRegHostApi() const
1030 : {
1031 2 : if (!enableHostApi_) {
1032 0 : return;
1033 : }
1034 2 : auto& profFunction = DlProfFunction::GetInstance();
1035 46 : for (auto i = 0; i < OpType::__COUNT__; ++i) {
1036 44 : OpType type(static_cast<OpType::Value>(i));
1037 44 : s32 ret = profFunction.dlMsprofRegTypeInfo(
1038 44 : MSPROF_REPORT_ACL_LEVEL, static_cast<uint32_t>(type) + MSPROF_REPORT_ACL_HOST_HCCL_BASE_TYPE,
1039 88 : type.Describe().c_str());
1040 44 : if (ret != 0) {
1041 0 : THROW<InternalException>("Call MsprofRegTypeInfo fail, return[%d]", ret);
1042 : }
1043 : }
1044 46 : for (auto i = 0; i < OpType::__COUNT__; ++i) {
1045 44 : OpType type(static_cast<OpType::Value>(i));
1046 44 : s32 ret = profFunction.dlMsprofRegTypeInfo(
1047 44 : MSPROF_REPORT_NODE_LEVEL, static_cast<uint32_t>(type) + MSPROF_REPORT_NODE_HCCL_BASE_TYPE,
1048 88 : type.Describe().c_str());
1049 44 : if (ret != 0) {
1050 0 : THROW<InternalException>("Call MsprofRegTypeInfo fail, return[%d]", ret);
1051 : }
1052 : }
1053 2 : const std::string hcclType("hccl_op_info");
1054 2 : s32 ret = profFunction.dlMsprofRegTypeInfo(
1055 : MSPROF_REPORT_NODE_LEVEL, MSPROF_REPORT_NODE_HCCL_OP_INFO_TYPE, hcclType.c_str());
1056 2 : if (ret != 0) {
1057 0 : THROW<InternalException>("Call MsprofRegTypeInfo fail, return[%d]", ret);
1058 : }
1059 2 : }
1060 :
1061 4 : void ProfilingHandler::ReportStoragedCompactInfo()
1062 : {
1063 4 : std::lock_guard<std::mutex> lock(cacheHcclOpInfoMutex_);
1064 12 : HCCL_INFO("[ReportStoragedCompactInfo] The size of the storageCompactInfo_ is [%zu]", cacheHcclOpInfo_.size());
1065 4 : std::queue<MsprofCompactInfo> tempCompactInfo = cacheHcclOpInfo_;
1066 12 : while (!tempCompactInfo.empty()) {
1067 8 : MsprofCompactInfo reportData = tempCompactInfo.front();
1068 8 : tempCompactInfo.pop();
1069 8 : s32 ret = DlProfFunction::GetInstance().dlMsprofReportCompactInfo(0, &reportData, sizeof(MsprofCompactInfo));
1070 8 : if (ret != 0) {
1071 0 : THROW<InternalException>("Call MsprofRegTypeInfo failed, return[%d]", ret);
1072 : }
1073 : }
1074 4 : }
1075 :
1076 2 : void ProfilingHandler::ReportMc2AdditionInfo()
1077 : {
1078 2 : std::lock_guard<std::mutex> lock(cacheHcclAdditionInfoMutex_);
1079 6 : HCCL_INFO("[ReportMc2AdditionInfo] The size of the storageCompactInfo_ is [%zu]", cacheHcclAdditionInfo_.size());
1080 2 : std::queue<MsprofAdditionalInfo> tempCompactInfo = cacheHcclAdditionInfo_;
1081 6 : while (!tempCompactInfo.empty()) {
1082 4 : MsprofAdditionalInfo reportData = tempCompactInfo.front();
1083 4 : tempCompactInfo.pop();
1084 : s32 ret
1085 4 : = DlProfFunction::GetInstance().dlMsprofReportAdditionalInfo(1, &reportData, sizeof(MsprofAdditionalInfo));
1086 4 : if (ret != 0) {
1087 0 : THROW<InternalException>("Call MsprofRegTypeInfo failed, return[%d]", ret);
1088 : }
1089 : }
1090 2 : }
1091 :
1092 2 : void ProfilingHandler::StartTaskApiSubscribe()
1093 : {
1094 2 : enableHcclNode_ = true;
1095 2 : CallProfRegTaskTypeApi();
1096 2 : ReportStoragedTaskApi();
1097 6 : HCCL_INFO("SetTaskApiSubscribe:[%d]", enableHcclNode_);
1098 2 : }
1099 :
1100 2 : void ProfilingHandler::CallProfRegTaskTypeApi() const
1101 : {
1102 2 : if (!enableHcclNode_) {
1103 0 : HCCL_INFO("[ProfilingHandler] enableHostApi_ is false.");
1104 0 : return;
1105 : }
1106 2 : const std::string hcclType("hccl_info");
1107 2 : s32 sret = DlProfFunction::GetInstance().dlMsprofRegTypeInfo(
1108 : MSPROF_REPORT_HCCL_NODE_LEVEL, static_cast<uint32_t>(ProfTaskType::TASK_HCCL_INFO), hcclType.c_str());
1109 2 : if (sret != 0) {
1110 0 : THROW<InternalException>("Call MsprofRegTypeInfo fail, return[%d]", sret);
1111 : }
1112 :
1113 2 : const std::string dpuhcclType("dpu_hccl_info");
1114 2 : sret = DlProfFunction::GetInstance().dlMsprofRegTypeInfo(
1115 : MSPROF_REPORT_HCCL_NODE_LEVEL, static_cast<uint32_t>(ProfTaskType::TASK_DPU_HCCL_INFO), dpuhcclType.c_str());
1116 2 : if (sret != 0) {
1117 0 : THROW<InternalException>("Call MsprofRegTypeInfo fail, return[%d]", sret);
1118 : }
1119 : const std::vector<std::pair<uint32_t, std::string>> taskTypes
1120 6 : = {{MSPROF_REPORT_NODE_CONTEXT_ID_INFO_TYPE, "context_id_info"}};
1121 : const std::vector<std::pair<uint32_t, std::string>> taskOtherTypes
1122 0 : = {{MSPROF_REPORT_NODE_BASIC_INFO_TYPE, "node_basic_info"},
1123 8 : {MSPROF_REPORT_NODE_MC2_COMMINFO_TYPE, "mc2_comm_info"}};
1124 :
1125 4 : for (auto& it : taskTypes) {
1126 2 : s32 ret = DlProfFunction::GetInstance().dlMsprofRegTypeInfo(
1127 2 : MSPROF_REPORT_HCCL_NODE_LEVEL, it.first, it.second.c_str());
1128 2 : if (ret != 0) {
1129 0 : THROW<InternalException>("Call dlMsprofRegTypeInfo failed, return[%d]", ret);
1130 : }
1131 : }
1132 6 : for (auto& it : taskOtherTypes) {
1133 : s32 ret
1134 4 : = DlProfFunction::GetInstance().dlMsprofRegTypeInfo(MSPROF_REPORT_NODE_LEVEL, it.first, it.second.c_str());
1135 4 : if (ret != 0) {
1136 0 : THROW<InternalException>("Call dlMsprofRegTypeInfo failed, return[%d]", ret);
1137 : }
1138 : }
1139 6 : }
1140 :
1141 2 : void ProfilingHandler::ReportStoragedTaskApi()
1142 : {
1143 2 : std::lock_guard<std::mutex> lock(cachedTaskApiInfoMutex_);
1144 6 : HCCL_INFO("[ReportStoragedTaskApi] taskApiQueueSize is [%zu]", cachedTaskApiInfo_.size());
1145 2 : if (!cachedTaskApiInfo_.empty()) {
1146 2 : std::queue<MsprofApi> tempTaskApi = cachedTaskApiInfo_;
1147 10 : while (!tempTaskApi.empty()) {
1148 8 : MsprofApi reportData = tempTaskApi.front();
1149 8 : tempTaskApi.pop();
1150 8 : s32 ret = DlProfFunction::GetInstance().dlMsprofReportApi(0, &reportData);
1151 8 : if (ret != 0) {
1152 0 : THROW<InternalException>("Call dlMsprofReportApi failed, return[%d]", ret);
1153 : }
1154 : }
1155 2 : }
1156 2 : }
1157 :
1158 2 : void ProfilingHandler::ReportStoragedAclApi()
1159 : {
1160 2 : std::lock_guard<std::mutex> lock(cachedAclApiInfoMutex_);
1161 6 : HCCL_INFO("[ReportStoragedAclApi] aclApiQueueSize is [%u]", cachedAclApiInfo_.size());
1162 2 : if (!cachedAclApiInfo_.empty()) {
1163 2 : std::queue<MsprofApi> tempAclApi = cachedAclApiInfo_;
1164 4 : while (!tempAclApi.empty()) {
1165 2 : MsprofApi reportData = tempAclApi.front();
1166 2 : tempAclApi.pop();
1167 2 : s32 ret = DlProfFunction::GetInstance().dlMsprofReportApi(0, &reportData);
1168 2 : if (ret != 0) {
1169 0 : THROW<InternalException>("Call dlMsprofReportApi failed, return[%d]", ret);
1170 : }
1171 : }
1172 2 : }
1173 2 : }
1174 :
1175 2 : void ProfilingHandler::StartHostHcclOpSubscribe()
1176 : {
1177 2 : enableHcclNode_ = true;
1178 2 : enableHcclL0_ = true;
1179 2 : CallProfRegHcclOpApi();
1180 2 : ReportStoragedCompactInfo();
1181 6 : HCCL_RUN_INFO("StartHostHcclOpSubscribe:enableHcclNode_[%d],enableHcclL0_[%d]", enableHcclNode_, enableHcclL0_);
1182 2 : }
1183 :
1184 2 : void ProfilingHandler::CallProfRegHcclOpApi() const
1185 : {
1186 2 : if (enableHcclL0_ == false) {
1187 0 : HCCL_INFO("[ProfilingHandler] enableHcclNode_ is false.");
1188 0 : return;
1189 : }
1190 46 : for (auto i = 0; i < OpType::__COUNT__; ++i) {
1191 44 : OpType type(static_cast<OpType::Value>(i));
1192 88 : s32 ret = DlProfFunction::GetInstance().dlMsprofRegTypeInfo(
1193 44 : MSPROF_REPORT_HCCL_NODE_LEVEL, static_cast<uint32_t>(type) + MSPROF_REPORT_ACL_HOST_HCCL_BASE_TYPE,
1194 88 : type.Describe().c_str());
1195 44 : if (ret != 0) {
1196 0 : THROW<InternalException>("[ProfilingHandler]Call MsprofReportApi fail, return[%d]", ret);
1197 : }
1198 : }
1199 2 : s32 ret = DlProfFunction::GetInstance().dlMsprofRegTypeInfo(
1200 : MSPROF_REPORT_NODE_LEVEL, MSPROF_REPORT_NODE_MC2_COMMINFO_TYPE, "mc2_comm_info");
1201 2 : if (ret != 0) {
1202 0 : THROW<InternalException>("[ProfilingHandler]Call MsprofRegTypeInfo fail, return[%d]", ret);
1203 : }
1204 : }
1205 :
1206 2 : void ProfilingHandler::StartAdditionInfoSubscribe()
1207 : {
1208 2 : enableHcclL1_ = true;
1209 2 : ReportStoragedAdditionInfo();
1210 6 : HCCL_RUN_INFO("StartAdditionInfoSubscribe:enableHcclL1_[%d]", enableHcclL1_);
1211 2 : }
1212 :
1213 2 : void ProfilingHandler::ReportStoragedAdditionInfoLog() const
1214 : {
1215 2 : if (LIKELY(HcclCheckLogLevel(HCCL_LOG_INFO) == 0)) {
1216 0 : return;
1217 : }
1218 2 : if (cacheTaskInfos_.empty()) {
1219 0 : return;
1220 : }
1221 4 : for (const auto& taskInfo : cacheTaskInfos_) {
1222 6 : HCCL_INFO("ReportStoragedAdditionInfoLog, taskType[%u]", GetTaskTypeValue(taskInfo.taskParam_.taskType));
1223 : }
1224 : }
1225 :
1226 2 : void ProfilingHandler::ReportStoragedAdditionInfo()
1227 : {
1228 2 : std::lock_guard<std::mutex> lock(cacheTaskInfosMutex_);
1229 2 : ReportStoragedAdditionInfoLog();
1230 2 : if (cacheTaskInfos_.empty()) {
1231 0 : HCCL_INFO("[ProfilingHandler]ReportStoragedAdditionInfo cacheTaskInfos_ is empty.");
1232 0 : return;
1233 : }
1234 4 : for (auto& taskInfo : cacheTaskInfos_) {
1235 2 : MsprofAdditionalInfo reporterData{};
1236 2 : FillTaskAdditionInfo(taskInfo, reporterData);
1237 2 : CallAdditionInfo(reporterData);
1238 : }
1239 2 : }
1240 :
1241 2 : void ProfilingHandler::StartCcuSubscribe()
1242 : {
1243 2 : enableHcclNode_ = true;
1244 2 : enableHcclL1_ = true;
1245 6 : HCCL_INFO("ProfilingHandler StartCcuSubscribe");
1246 : const std::vector<std::pair<uint32_t, std::string>> ccuInfoTypes
1247 : = {{MSPROF_REPORT_CCU_TASK_INFO, "ccu_task_info"},
1248 : {MSPROF_REPORT_CCU_WAIT_SIGNAL_INFO, "ccu_wait_signal_info"},
1249 10 : {MSPROF_REPORT_CCU_GROUP_INFO, "ccu_group_info"}};
1250 8 : for (auto& it : ccuInfoTypes) {
1251 6 : s32 ret = DlProfFunction::GetInstance().dlMsprofRegTypeInfo(
1252 6 : MSPROF_REPORT_HCCL_NODE_LEVEL, it.first, it.second.c_str());
1253 6 : if (ret != 0) {
1254 0 : THROW<InternalException>("Call dlMsprofRegTypeInfo failed, return[%d]", ret);
1255 : }
1256 : }
1257 2 : std::lock_guard<std::mutex> lock(cacheTaskInfosMutex_);
1258 2 : if (cacheTaskInfos_.empty()) {
1259 0 : HCCL_INFO("[ProfilingHandler]StartL2Subscribe cacheTaskInfos_ is empty.");
1260 0 : return;
1261 : }
1262 4 : for (auto& taskInfo : cacheTaskInfos_) {
1263 2 : ReportCcuInfo(taskInfo);
1264 : }
1265 4 : }
1266 :
1267 1 : void ProfilingHandler::ProfilingHandler::StopSubscribe()
1268 : {
1269 1 : enableHostApi_ = false;
1270 1 : enableHcclNode_ = false;
1271 1 : enableHcclL0_ = false;
1272 1 : enableHcclL1_ = false;
1273 3 : HCCL_RUN_INFO("[ProfilingHandler]StopSubscribe.");
1274 1 : }
1275 :
1276 48 : bool ProfilingHandler::GetHostApiState() const { return enableHostApi_; }
1277 48 : bool ProfilingHandler::GetHcclNodeState() const { return enableHcclNode_; }
1278 50 : bool ProfilingHandler::GetHcclL0State() const { return enableHcclL0_; }
1279 :
1280 54 : bool ProfilingHandler::GetHcclL1State() const { return enableHcclL1_; }
1281 :
1282 196 : uint64_t ProfilingHandler::GetProfHashId(const char* name, uint32_t len) const
1283 : {
1284 196 : if (name == nullptr || len == 0) {
1285 63 : HCCL_WARNING("HashData is empty. name:%s, len:%u", name, len);
1286 21 : return INVALID_U64;
1287 : }
1288 175 : if (DlProfFunction::GetInstance().dlMsprofStr2Id == nullptr) {
1289 0 : HCCL_WARNING("HashFunc is nullPtr");
1290 0 : return INVALID_U64;
1291 : }
1292 175 : return DlProfFunction::GetInstance().dlMsprofStr2Id(name, len);
1293 : }
1294 :
1295 2 : void ProfilingHandler::ReportHcclMC2CommInfoLog(
1296 : [[maybe_unused]] const Stream& kfcStream, const Stream& stream, const std::vector<Stream*>& aicpuStreams,
1297 : [[maybe_unused]] const std::string& id, [[maybe_unused]] RankId myRank, [[maybe_unused]] u32 rankSize,
1298 : [[maybe_unused]] RankId rankInParentComm) const
1299 : {
1300 2 : if (LIKELY(HcclCheckLogLevel(HCCL_LOG_INFO) == 0)) {
1301 0 : return;
1302 : }
1303 2 : if (aicpuStreams.empty()) {
1304 3 : HCCL_INFO("only exist main stream, streamId(sqId):%u", stream.GetSqId());
1305 1 : return;
1306 : }
1307 1 : uint32_t reportId = 0;
1308 2 : for (uint32_t streamIndex = 0; streamIndex < aicpuStreams.size(); streamIndex++) {
1309 3 : HCCL_INFO(
1310 : "streamIndex:%u, reportId:%u, streamId(sqId):%u", streamIndex, reportId,
1311 : aicpuStreams[streamIndex]->GetSqId());
1312 1 : reportId++;
1313 : }
1314 : }
1315 :
1316 2 : void ProfilingHandler::ReportHcclMC2CommInfo(
1317 : const Stream& kfcStream, const Stream& stream, const std::vector<Stream*>& aicpuStreams, const std::string& id,
1318 : RankId myRank, u32 rankSize, RankId rankInParentComm)
1319 : {
1320 2 : ReportHcclMC2CommInfoLog(kfcStream, stream, aicpuStreams, id, myRank, rankSize, rankInParentComm);
1321 : ProfilingDeviceCommResInfo hcclMc2Info;
1322 2 : hcclMc2Info.groupName = GetProfHashId(id.c_str(), id.length());
1323 2 : hcclMc2Info.rankSize = rankSize;
1324 2 : hcclMc2Info.rankId = myRank;
1325 2 : hcclMc2Info.usrRankId = rankInParentComm;
1326 2 : hcclMc2Info.aicpuKfcStreamId = static_cast<uint32_t>(kfcStream.GetSqId());
1327 2 : hcclMc2Info.reserve = 0;
1328 2 : const uint32_t ONCE_REPORT_STREAM_NUM_MAX = 8;
1329 3 : for (uint32_t streamIndex = 0, reportId = 0; streamIndex < aicpuStreams.size(); streamIndex++) {
1330 1 : hcclMc2Info.commStreamIds[reportId++] = aicpuStreams[streamIndex]->GetSqId();
1331 1 : if (reportId == ONCE_REPORT_STREAM_NUM_MAX) {
1332 0 : hcclMc2Info.commStreamSize = reportId;
1333 0 : ReportMc2AdditionInfo(
1334 0 : DlProfFunction::GetInstance().dlMsprofSysCycleTime(), &hcclMc2Info, sizeof(hcclMc2Info));
1335 0 : reportId = 0;
1336 : }
1337 1 : if (streamIndex == (aicpuStreams.size() - 1)) {
1338 1 : hcclMc2Info.commStreamIds[reportId++] = stream.GetSqId();
1339 1 : hcclMc2Info.commStreamSize = reportId;
1340 1 : ReportMc2AdditionInfo(
1341 1 : DlProfFunction::GetInstance().dlMsprofSysCycleTime(), &hcclMc2Info, sizeof(hcclMc2Info));
1342 1 : reportId = 0;
1343 : }
1344 : }
1345 2 : if (aicpuStreams.empty()) {
1346 1 : hcclMc2Info.commStreamIds[0] = stream.GetSqId();
1347 1 : hcclMc2Info.commStreamSize = 1;
1348 1 : ReportMc2AdditionInfo(DlProfFunction::GetInstance().dlMsprofSysCycleTime(), &hcclMc2Info, sizeof(hcclMc2Info));
1349 : }
1350 2 : }
1351 :
1352 1 : void ProfilingHandler::ReportHcclMC2CommInfoLog(
1353 : [[maybe_unused]] const u32 kfcStreamId, const std::vector<u32>& aicpuStreamsId, const std::string& id,
1354 : [[maybe_unused]] RankId myRank, [[maybe_unused]] u32 rankSize, [[maybe_unused]] RankId rankInParentComm) const
1355 : {
1356 1 : if (LIKELY(HcclCheckLogLevel(HCCL_LOG_INFO) == 0)) {
1357 0 : return;
1358 : }
1359 1 : uint64_t groupName = GetProfHashId(id.c_str(), id.length());
1360 1 : uint32_t reportId = 0;
1361 4 : for (uint32_t streamIndex = 0; streamIndex < aicpuStreamsId.size(); streamIndex++) {
1362 9 : HCCL_INFO(
1363 : "streamIndex:[%u], reportId:[%u], streamId:[%u] id [%s] hcclMC2Info.groupName:[%llu]", streamIndex,
1364 : reportId, aicpuStreamsId[streamIndex], id.c_str(), groupName);
1365 3 : reportId++;
1366 : }
1367 : }
1368 :
1369 0 : void ProfilingHandler::ReportHcclMC2CommInfo(
1370 : const u32 kfcStreamId, const std::vector<u32>& aicpuStreamsId, const std::string& id, RankId myRank, u32 rankSize,
1371 : RankId rankInParentComm)
1372 : {
1373 0 : ReportHcclMC2CommInfoLog(kfcStreamId, aicpuStreamsId, id, myRank, rankSize, rankInParentComm);
1374 : ProfilingDeviceCommResInfo hcclMc2Info;
1375 0 : hcclMc2Info.groupName = GetProfHashId(id.c_str(), id.length());
1376 0 : hcclMc2Info.rankSize = rankSize;
1377 0 : hcclMc2Info.rankId = myRank;
1378 0 : hcclMc2Info.usrRankId = rankInParentComm;
1379 0 : hcclMc2Info.aicpuKfcStreamId = static_cast<uint32_t>(kfcStreamId);
1380 0 : hcclMc2Info.reserve = 0;
1381 :
1382 0 : const uint32_t ONCE_REPORT_STREAM_NUM_MAX = 8;
1383 0 : uint32_t reportId = 0;
1384 0 : for (uint32_t streamIndex = 0; streamIndex < aicpuStreamsId.size(); streamIndex++) {
1385 0 : hcclMc2Info.commStreamIds[reportId++] = aicpuStreamsId[streamIndex];
1386 0 : if (reportId == ONCE_REPORT_STREAM_NUM_MAX) {
1387 0 : hcclMc2Info.commStreamSize = reportId;
1388 0 : ReportMc2AdditionInfo(
1389 0 : DlProfFunction::GetInstance().dlMsprofSysCycleTime(), &hcclMc2Info, sizeof(hcclMc2Info));
1390 0 : reportId = 0;
1391 : }
1392 : }
1393 0 : if (reportId > 0) {
1394 0 : hcclMc2Info.commStreamSize = reportId;
1395 0 : ReportMc2AdditionInfo(DlProfFunction::GetInstance().dlMsprofSysCycleTime(), &hcclMc2Info, sizeof(hcclMc2Info));
1396 0 : reportId = 0;
1397 : }
1398 0 : }
1399 2 : void ProfilingHandler::ReportMc2AdditionInfo(uint64_t timeStamp, const void* data, int len)
1400 : {
1401 2 : MsprofAdditionalInfo reporterData{};
1402 2 : reporterData.level = MSPROF_REPORT_NODE_LEVEL;
1403 2 : reporterData.type = MSPROF_REPORT_NODE_MC2_COMMINFO_TYPE;
1404 2 : reporterData.threadId = SalGetTid();
1405 2 : reporterData.dataLen = len;
1406 2 : reporterData.timeStamp = timeStamp;
1407 2 : s32 sret = memcpy_s(reporterData.data, sizeof(reporterData.data), data, len);
1408 2 : if (sret != EOK) {
1409 0 : THROW<InternalException>("Call memcpy_s failed, errorno[%d]", sret);
1410 : }
1411 6 : HCCL_INFO(
1412 : "[ProfilingHandler][ReportMc2CommInfo], level [%u], type[%u], threadId[%u], dataLen[%u], timeStamp[%llu]",
1413 : reporterData.level, reporterData.type, reporterData.threadId, reporterData.dataLen, reporterData.timeStamp);
1414 2 : if (!enableHostApi_) {
1415 2 : std::lock_guard<std::mutex> lock(cacheHcclAdditionInfoMutex_);
1416 2 : cacheHcclAdditionInfo_.push(reporterData);
1417 2 : return;
1418 2 : }
1419 : s32 ret
1420 0 : = DlProfFunction::GetInstance().dlMsprofReportAdditionalInfo(1, &reporterData, sizeof(MsprofAdditionalInfo));
1421 0 : HCCL_INFO("Call MsprofReportAdditionalInfo, return value[%d]", ret);
1422 0 : if (ret != 0) {
1423 0 : THROW<InternalException>("Call MsprofReportAdditionalInfo failed, return[%d]", ret);
1424 : }
1425 : }
1426 :
1427 : } // namespace Hccl
|
