Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include <cstring>
12 : #include <algorithm>
13 : #include "profiling_handler.h"
14 : #include "dlprof_function_v2.h"
15 : #include "exception_util.h"
16 : #include "internal_exception.h"
17 : #include "sal.h"
18 : #include "task_param.h"
19 : #include "aprof_pub.h"
20 : #include "orion_adapter_rts.h"
21 : #include "communicator_impl.h"
22 : #include "data_type.h"
23 : namespace Hccl {
24 : #define UNUSED(x) (void)(x)
25 :
26 : constexpr uint16_t CCU_TYPE = 2;
27 : constexpr uint32_t DPU_DEV_ID_MASK = (1U << 12);
28 :
29 : ProfilingHandler ProfilingHandler::instance_;
30 :
31 1 : ProfilingHandler::ProfilingHandler() {}
32 :
33 1 : ProfilingHandler::~ProfilingHandler() {}
34 :
35 769 : ProfilingHandler& ProfilingHandler::GetInstance() { return instance_; }
36 :
37 1 : void ProfilingHandler::InitLog() const
38 : {
39 1 : if (LIKELY(HcclCheckLogLevel(HCCL_LOG_INFO) == 0)) {
40 0 : return;
41 : }
42 24 : for (auto i = 0; i < TaskParamType::__COUNT__; ++i) {
43 23 : TaskParamType type(static_cast<TaskParamType::Value>(i));
44 23 : std::string nameInfo = type.Describe();
45 23 : uint64_t hashId = GetProfHashId(nameInfo.c_str(), nameInfo.length());
46 69 : HCCL_INFO("[TaskParamType] nameInfo[%s] ret[%llu]", nameInfo.c_str(), hashId);
47 23 : }
48 : }
49 :
50 258 : HcclResult ProfilingHandler::Init()
51 : {
52 258 : if (initializedFlag_) {
53 257 : return HCCL_SUCCESS;
54 : }
55 1 : if (Hccl::DlProfFunction::GetInstance().DlProfFunctionInit() != HCCL_SUCCESS) {
56 0 : HCCL_ERROR("[ProfilingHandler][Init] DlProfFunctionInit failed.");
57 0 : return HCCL_E_INTERNAL;
58 : }
59 1 : ProfCommandHandle callback = CommandHandleWrapper;
60 1 : auto ret = DlProfFunction::GetInstance().dlMsprofRegisterCallback(HCCL, callback);
61 1 : if (ret != 0) {
62 0 : HCCL_ERROR(
63 : "[ProfilingHandler][Init]errNo[0x%016llx] Prof Register CtrlCallback fail, return[%d]",
64 : HCCL_ERROR_CODE(HCCL_E_RUNTIME), ret);
65 0 : return HCCL_E_RUNTIME;
66 : }
67 1 : cachedAlgTypeHashId_.store(GetProfHashId("AlgType::NHR", strlen("AlgType::NHR")));
68 1 : initializedFlag_ = true;
69 1 : InitLog();
70 1 : return HCCL_SUCCESS;
71 : }
72 :
73 0 : int32_t ProfilingHandler::CommandHandleWrapper(uint32_t rtType, void* data, uint32_t len)
74 : {
75 0 : return instance_.CommandHandle(rtType, data, len);
76 : }
77 :
78 0 : void ProfilingHandler::ReportKernel() const {}
79 :
80 8 : void ProfilingHandler::ReportHostApi(OpType opType, uint64_t beginTime, uint64_t endTime, bool cachedReq, bool isAiCpu)
81 : {
82 8 : uint32_t threadId = SalGetTid();
83 8 : std::string profName(GetProfOpName(opType));
84 8 : if (isAiCpu) {
85 1 : profName += "AicpuKernel";
86 : }
87 8 : uint64_t cmdItemId = DlProfFunction::GetInstance().dlMsprofStr2Id(profName.c_str(), profName.length());
88 8 : if (enableHostApi_) {
89 0 : ReportAclApi(opType, beginTime, endTime, cmdItemId, threadId, cachedReq);
90 : }
91 8 : ReportNodeApi(beginTime, endTime, cmdItemId, threadId, cachedReq);
92 8 : ReportNodeBasicInfo(endTime, cmdItemId, threadId, cachedReq);
93 8 : }
94 :
95 8 : void ProfilingHandler::ReportHcclOp(const DfxOpInfo& opInfo, bool cachedReq)
96 : {
97 8 : uint32_t threadId = SalGetTid();
98 8 : ReportHcclOpInfo(opInfo.endTime_, opInfo, threadId, cachedReq);
99 8 : }
100 :
101 5 : void ProfilingHandler::ReportHcclTaskApi(
102 : TaskParamType taskType, uint64_t beginTime, uint64_t endTime, bool isMasterStream, bool cachedReq, bool ignoreLevel)
103 : {
104 5 : MsprofApi reporterData{};
105 5 : reporterData.level = MSPROF_REPORT_HCCL_NODE_LEVEL;
106 5 : reporterData.type = isMasterStream ? MSPROF_REPORT_HCCL_MASTER_TYPE : MSPROF_REPORT_HCCL_SLAVE_TYPE;
107 5 : reporterData.threadId = SalGetTid();
108 5 : reporterData.beginTime = beginTime;
109 5 : reporterData.endTime = endTime;
110 5 : const std::string proName(GetProfTaskOpNameV2(taskType));
111 5 : reporterData.itemId = GetProfHashId(proName.c_str(), proName.length());
112 5 : if (taskType == TaskParamType::TASK_AICPU_KERNEL) {
113 0 : return;
114 : }
115 5 : if (cachedReq) {
116 5 : std::lock_guard<std::mutex> lock(cachedTaskApiInfoMutex_);
117 5 : cachedTaskApiInfo_.push(reporterData);
118 5 : }
119 5 : if ((!enableHcclNode_) || (!ignoreLevel && !enableHcclL1_)) {
120 3 : return;
121 : }
122 6 : HCCL_INFO(
123 : "ReportHcclTaskApi, enableHcclNode_[%d],ignoreLevel[%d],enableHcclL1_[%d],taskType[%s], beginTime[%llu], "
124 : "endTime[%llu], isMasterStream[%d], cachedReq[%d]",
125 : enableHcclNode_, ignoreLevel, enableHcclL1_, proName.c_str(), beginTime, endTime, isMasterStream, cachedReq);
126 2 : s32 ret = DlProfFunction::GetInstance().dlMsprofReportApi(1, &reporterData);
127 2 : if (ret != 0) {
128 0 : THROW<InternalException>("Call MsprofReportApi fail, return[%d]", ret);
129 : }
130 5 : }
131 :
132 5 : void ProfilingHandler::SetCachedCclTag()
133 : {
134 5 : cachedNewCclTag_.clear();
135 100 : for (const auto& item : CMD_OP_TYPE_INFO_MAP) {
136 95 : const std::string& cclTag = item.second.second;
137 95 : cachedNewCclTag_[item.second.first] = GetProfHashId(cclTag.c_str(), cclTag.length());
138 : }
139 5 : if (cachedNewCclTag_.find(static_cast<uint32_t>(OpType::HCCLGROUPOP)) == cachedNewCclTag_.end()) {
140 5 : const std::string cclTag = "OpType::HCCLGROUPOP";
141 5 : cachedNewCclTag_[OpType::HCCLGROUPOP] = GetProfHashId(cclTag.c_str(), cclTag.length());
142 5 : }
143 5 : }
144 :
145 6 : uint32_t ProfilingHandler::GetTaskTypeValue(TaskParamType taskType) const
146 : {
147 6 : switch (taskType) {
148 0 : case TaskParamType::TASK_DPU_INLINE_WRITE:
149 : case TaskParamType::TASK_DPU_NOTIFY_WAIT:
150 : case TaskParamType::TASK_DPU_WRITE_WITH_NOTIFY:
151 : case TaskParamType::TASK_DPU_CHANNEL_FENCE:
152 0 : return static_cast<uint32_t>(ProfTaskType::TASK_DPU_HCCL_INFO);
153 6 : default:
154 6 : return static_cast<uint32_t>(ProfTaskType::TASK_HCCL_INFO);
155 : }
156 : }
157 :
158 7 : void ProfilingHandler::FillTaskAdditionInfo(const TaskInfo& taskInfo, MsprofAdditionalInfo& reporterData) const
159 : {
160 7 : reporterData.level = MSPROF_REPORT_HCCL_NODE_LEVEL;
161 7 : reporterData.threadId = SalGetTid();
162 7 : FillProfCommonInfo(taskInfo, reporterData);
163 :
164 7 : switch (taskInfo.taskParam_.taskType) {
165 0 : case TaskParamType::TASK_DPU_INLINE_WRITE:
166 : case TaskParamType::TASK_DPU_NOTIFY_WAIT:
167 : case TaskParamType::TASK_DPU_WRITE_WITH_NOTIFY:
168 : case TaskParamType::TASK_DPU_CHANNEL_FENCE:
169 0 : reporterData.type = static_cast<uint32_t>(ProfTaskType::TASK_DPU_HCCL_INFO);
170 0 : FillDpuProfInfo(taskInfo, reporterData);
171 0 : break;
172 7 : default:
173 7 : reporterData.type = static_cast<uint32_t>(ProfTaskType::TASK_HCCL_INFO);
174 7 : FillProfTaskSpecificInfo(taskInfo, reinterpret_cast<MsprofHcclInfo*>(reporterData.data));
175 7 : break;
176 : }
177 7 : }
178 :
179 3 : void ProfilingHandler::ReportHcclTaskDetails(const TaskInfo& taskInfo, bool cachedReq)
180 : {
181 3 : if (enableHcclL1_ == false && !cachedReq) {
182 2 : return;
183 : }
184 3 : if (taskInfo.dfxOpInfo_ == nullptr) {
185 3 : HCCL_WARNING("[%s] dfxOpInfo_ is nullptr, skip ReportHcclTaskDetails!", __func__);
186 1 : return;
187 : }
188 2 : if (taskInfo.dfxOpInfo_->comm_ == nullptr) {
189 3 : HCCL_WARNING("[%s] comm_ is nullptr, skip ReportHcclTaskDetails!", __func__);
190 1 : return;
191 : }
192 1 : if (cachedReq) {
193 1 : std::lock_guard<std::mutex> lock(cacheTaskInfosMutex_);
194 1 : cacheTaskInfos_.push_back(taskInfo);
195 1 : }
196 1 : if (enableHcclL1_ == false) {
197 0 : return;
198 : }
199 1 : MsprofAdditionalInfo reporterData{};
200 1 : FillTaskAdditionInfo(taskInfo, reporterData);
201 1 : DumpHCCLReportData(taskInfo, reporterData);
202 1 : CallAdditionInfo(reporterData);
203 : }
204 :
205 2 : void ProfilingHandler::ReportHcclTaskDetailsBatchLog(const std::vector<TaskInfo*>& taskInfos) const
206 : {
207 2 : if (LIKELY(HcclCheckLogLevel(HCCL_LOG_INFO) == 0)) {
208 0 : return;
209 : }
210 6 : for (const auto& taskInfo : taskInfos) {
211 12 : HCCL_INFO("ReportHcclTaskDetailsBatchLog, taskType[%u]", GetTaskTypeValue(taskInfo->taskParam_.taskType));
212 : }
213 : }
214 :
215 7 : void ProfilingHandler::ReportHcclTaskDetailsBatch(const std::vector<TaskInfo*>& taskInfos, bool cachedReq)
216 : {
217 7 : if (taskInfos.empty()) {
218 3 : HCCL_WARNING("[ProfilingHandler::ReportHcclTaskDetailsBatch] taskInfos is empty!");
219 1 : return;
220 : }
221 6 : if (taskInfos[0]->dfxOpInfo_ == nullptr) {
222 3 : HCCL_WARNING("[ProfilingHandler::ReportHcclTaskDetailsBatch] taskInfo.dfxOpInfo_ is nullptr, skip!");
223 1 : return;
224 : }
225 5 : if (taskInfos[0]->dfxOpInfo_->comm_ == nullptr) {
226 3 : HCCL_WARNING("[ProfilingHandler::ReportHcclTaskDetailsBatch] taskInfo.dfxOpInfo_->comm_ is nullptr, skip!");
227 1 : return;
228 : }
229 4 : if (enableHcclL1_ == false && !cachedReq) {
230 1 : return;
231 : }
232 3 : if (cachedReq) {
233 3 : std::lock_guard<std::mutex> lock(cacheTaskInfosMutex_);
234 8 : for (const auto& taskInfo : taskInfos) {
235 5 : cacheTaskInfos_.push_back(*taskInfo);
236 : }
237 3 : }
238 3 : if (enableHcclL1_ == false) {
239 1 : return;
240 : }
241 2 : ReportHcclTaskDetailsBatchLog(taskInfos);
242 6 : for (const auto& taskInfo : taskInfos) {
243 4 : MsprofAdditionalInfo reporterData{};
244 4 : FillTaskAdditionInfo(*taskInfo, reporterData);
245 4 : CallAdditionInfo(reporterData);
246 : }
247 : }
248 :
249 5 : void ProfilingHandler::CallAdditionInfo(MsprofAdditionalInfo& reporterData) const
250 : {
251 : s32 ret
252 5 : = DlProfFunction::GetInstance().dlMsprofReportAdditionalInfo(1, &reporterData, sizeof(MsprofAdditionalInfo));
253 5 : if (ret != 0) {
254 0 : THROW<InternalException>("Call MsprofReportAdditionalInfo failed, return[%d]", ret);
255 : }
256 5 : }
257 :
258 7 : void ProfilingHandler::FillProfCommonInfo(const TaskInfo& taskInfo, MsprofAdditionalInfo& reporterData) const
259 : {
260 7 : if (taskInfo.dfxOpInfo_ == nullptr) {
261 0 : HCCL_WARNING("[ProfilingHandler][%s]taskInfo.dfxOpInfo_ is nullptr, skip GetProfCommonInfo.", __func__);
262 0 : return;
263 : }
264 7 : reporterData.timeStamp = taskInfo.taskParam_.endTime;
265 7 : reporterData.dataLen = sizeof(MsprofHcclInfo);
266 7 : auto* profInfo = reinterpret_cast<MsprofHcclInfo*>(reporterData.data);
267 7 : *profInfo = MsprofHcclInfo();
268 :
269 7 : const auto& profName = GetProfTaskOpNameV2(taskInfo.taskParam_.taskType);
270 7 : profInfo->itemId = GetProfHashId(profName.c_str(), profName.length());
271 7 : auto newCclTagIt = cachedNewCclTag_.find(taskInfo.dfxOpInfo_->op_.opType);
272 7 : profInfo->cclTag = (newCclTagIt != cachedNewCclTag_.end()) ? newCclTagIt->second : INVALID_U64;
273 7 : const auto& opTag = taskInfo.dfxOpInfo_->op_.opTag;
274 7 : uint64_t groupName = GetProfHashId(opTag.c_str(), opTag.length());
275 7 : if (taskInfo.dfxOpInfo_->comm_ == nullptr) {
276 0 : HCCL_WARNING("[ProfilingHandler][%s]taskInfo.dfxOpInfo_->comm_ is nullptr, skip GetProfCommonInfo.", __func__);
277 0 : return;
278 : }
279 7 : if (taskInfo.dfxOpInfo_->isIndop_ == true) {
280 0 : profInfo->groupName = groupName;
281 0 : profInfo->rankSize = taskInfo.dfxOpInfo_->rankSize_;
282 : } else {
283 7 : CommunicatorImpl* commImp = static_cast<CommunicatorImpl*>(taskInfo.dfxOpInfo_->comm_);
284 7 : if (commImp == nullptr) {
285 0 : HCCL_WARNING("[ProfilingHandler][%s]commImp is nullptr, skip GetProfCommonInfo.", __func__);
286 0 : return;
287 : }
288 7 : profInfo->groupName = groupName;
289 7 : profInfo->rankSize = commImp->GetRankSize();
290 : }
291 7 : profInfo->workFlowMode = static_cast<u32>(HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE);
292 7 : profInfo->planeID = 0;
293 7 : profInfo->stage = 0;
294 7 : profInfo->role = static_cast<uint32_t>(TaskRole::DST);
295 7 : profInfo->durationEstimated = 0;
296 7 : profInfo->localRank = taskInfo.dfxOpInfo_->op_.myRank;
297 7 : profInfo->remoteRank = taskInfo.remoteRank_;
298 7 : profInfo->dataType = taskInfo.dfxOpInfo_->op_.dataType;
299 7 : profInfo->opType = taskInfo.dfxOpInfo_->op_.opType;
300 7 : profInfo->transportType = static_cast<int32_t>(SimpleTaskType::UB);
301 7 : if (profInfo->remoteRank == INVALID_VALUE_RANKID) {
302 0 : profInfo->transportType = static_cast<int32_t>(SimpleTaskType::LOCAL);
303 0 : profInfo->remoteRank = profInfo->localRank;
304 0 : profInfo->linkType = 0;
305 : }
306 7 : }
307 :
308 7 : void ProfilingHandler::FillProfTaskSpecificInfo(const TaskInfo& taskInfo, MsprofHcclInfo* profInfo) const
309 : {
310 7 : const auto& taskType = taskInfo.taskParam_.taskType;
311 7 : const auto& taskPara = taskInfo.taskParam_.taskPara;
312 7 : switch (taskType) {
313 0 : case TaskParamType::TASK_SDMA:
314 : case TaskParamType::TASK_RDMA:
315 0 : profInfo->srcAddr = static_cast<u64>(reinterpret_cast<uintptr_t>(taskPara.DMA.src));
316 0 : profInfo->dstAddr = static_cast<u64>(reinterpret_cast<uintptr_t>(taskPara.DMA.dst));
317 0 : profInfo->dataSize = static_cast<u32>(taskPara.DMA.size);
318 0 : profInfo->notifyID = taskPara.DMA.notifyID;
319 0 : profInfo->linkType = static_cast<uint16_t>(taskPara.DMA.linkType);
320 0 : break;
321 0 : case TaskParamType::TASK_REDUCE_INLINE:
322 : case TaskParamType::TASK_REDUCE_TBE:
323 0 : profInfo->srcAddr = static_cast<u64>(reinterpret_cast<uintptr_t>(taskPara.Reduce.src));
324 0 : profInfo->dstAddr = static_cast<u64>(reinterpret_cast<uintptr_t>(taskPara.Reduce.dst));
325 0 : profInfo->dataSize = static_cast<u32>(taskPara.Reduce.size);
326 0 : profInfo->notifyID = taskPara.Reduce.notifyID;
327 0 : profInfo->linkType = static_cast<uint16_t>(taskPara.Reduce.linkType);
328 0 : break;
329 7 : case TaskParamType::TASK_NOTIFY_RECORD:
330 : case TaskParamType::TASK_NOTIFY_WAIT:
331 7 : profInfo->notifyID = taskPara.Notify.notifyID;
332 7 : break;
333 0 : case TaskParamType::TASK_CCU:
334 0 : HCCL_INFO("current taskType is TASK_CCU");
335 0 : ReportCcuInfo(taskInfo);
336 0 : break;
337 0 : default:
338 0 : break;
339 : }
340 7 : }
341 :
342 0 : void ProfilingHandler::ConvertHcclInfoToDpuTrack(MsprofAdditionalInfo& reporterData) const
343 : {
344 0 : auto* profInfo = reinterpret_cast<MsprofHcclInfo*>(reporterData.data);
345 0 : uint64_t itemId = profInfo->itemId;
346 0 : uint64_t cclTag = profInfo->cclTag;
347 0 : uint64_t groupName = profInfo->groupName;
348 0 : uint32_t localRank = profInfo->localRank;
349 0 : uint32_t remoteRank = profInfo->remoteRank;
350 0 : uint32_t rankSize = profInfo->rankSize;
351 0 : uint32_t workFlowMode = profInfo->workFlowMode;
352 0 : uint16_t planeID = static_cast<uint16_t>(profInfo->planeID);
353 0 : uint32_t stage = profInfo->stage;
354 0 : uint32_t role = profInfo->role;
355 0 : double durationEstimated = profInfo->durationEstimated;
356 :
357 0 : reporterData.dataLen = sizeof(MsprofDpuHcclTrack);
358 0 : auto* dpuProfInfo = reinterpret_cast<MsprofDpuHcclTrack*>(reporterData.data);
359 0 : *dpuProfInfo = MsprofDpuHcclTrack();
360 :
361 0 : dpuProfInfo->itemId = itemId;
362 0 : dpuProfInfo->cclTag = cclTag;
363 0 : dpuProfInfo->groupName = groupName;
364 0 : dpuProfInfo->localRank = localRank;
365 0 : dpuProfInfo->remoteRank = remoteRank;
366 0 : dpuProfInfo->rankSize = rankSize;
367 0 : dpuProfInfo->workFlowMode = static_cast<uint8_t>(workFlowMode);
368 0 : dpuProfInfo->planeID = planeID;
369 0 : dpuProfInfo->stage = stage;
370 0 : dpuProfInfo->role = static_cast<uint8_t>(role);
371 0 : dpuProfInfo->durationEstimated = durationEstimated;
372 0 : }
373 :
374 0 : void ProfilingHandler::FillDpuTaskParaDetails(const TaskInfo& taskInfo, MsprofDpuHcclTrack* dpuProfInfo) const
375 : {
376 0 : const auto& taskPara = taskInfo.taskParam_.taskPara;
377 0 : switch (taskInfo.taskParam_.taskType) {
378 0 : case TaskParamType::TASK_DPU_INLINE_WRITE:
379 : case TaskParamType::TASK_DPU_WRITE_WITH_NOTIFY:
380 0 : dpuProfInfo->srcAddr = static_cast<u64>(reinterpret_cast<uintptr_t>(taskPara.DMA.src));
381 0 : dpuProfInfo->dstAddr = static_cast<u64>(reinterpret_cast<uintptr_t>(taskPara.DMA.dst));
382 0 : dpuProfInfo->dataSize = static_cast<u32>(taskPara.DMA.size);
383 0 : dpuProfInfo->notifyID = taskPara.DMA.notifyID;
384 0 : break;
385 0 : case TaskParamType::TASK_DPU_NOTIFY_WAIT:
386 : case TaskParamType::TASK_DPU_CHANNEL_FENCE:
387 0 : dpuProfInfo->notifyID = taskPara.Notify.notifyID;
388 0 : break;
389 0 : default:
390 0 : break;
391 : }
392 0 : }
393 :
394 0 : void ProfilingHandler::FillDpuProfInfo(const TaskInfo& taskInfo, MsprofAdditionalInfo& reporterData) const
395 : {
396 0 : if (taskInfo.dfxOpInfo_ == nullptr) {
397 0 : HCCL_WARNING("[ProfilingHandler::FillDpuProfInfo] taskInfo.dfxOpInfo_ is nullptr, skip GetDpuProfInfo!");
398 0 : return;
399 : }
400 0 : ConvertHcclInfoToDpuTrack(reporterData);
401 0 : auto* dpuProfInfo = reinterpret_cast<MsprofDpuHcclTrack*>(reporterData.data);
402 0 : dpuProfInfo->dataType = static_cast<uint32_t>(taskInfo.dfxOpInfo_->op_.dataType);
403 0 : dpuProfInfo->opType = static_cast<uint32_t>(taskInfo.dfxOpInfo_->op_.opType);
404 0 : dpuProfInfo->transportType = static_cast<uint32_t>(SimpleTaskType::ROCE);
405 0 : dpuProfInfo->aicpu_task_id = taskInfo.taskParam_.aicpuTaskId;
406 0 : dpuProfInfo->npuDevId = taskInfo.taskParam_.npuDevId;
407 0 : dpuProfInfo->dpuDevId = DPU_DEV_ID_MASK;
408 0 : const auto& taskPara = taskInfo.taskParam_.taskPara;
409 0 : dpuProfInfo->linkType = static_cast<uint16_t>(taskPara.DMA.linkType);
410 0 : if (dpuProfInfo->remoteRank == INVALID_VALUE_RANKID) {
411 0 : dpuProfInfo->transportType = static_cast<uint32_t>(SimpleTaskType::LOCAL);
412 0 : dpuProfInfo->remoteRank = dpuProfInfo->localRank;
413 0 : dpuProfInfo->linkType = 0;
414 : }
415 0 : dpuProfInfo->taskId = taskInfo.taskId_;
416 0 : dpuProfInfo->streamId = taskInfo.streamId_;
417 0 : dpuProfInfo->timeStamp = taskInfo.taskParam_.beginTime;
418 0 : HCCL_INFO(
419 : "[FillDpuProfInfo]taskId[%u], streamId[%u], npuDevId[%u], dpuDevId[%u], "
420 : "starttime[%llu], endtime[%llu], aicputaskId[%u].",
421 : dpuProfInfo->taskId, dpuProfInfo->streamId, dpuProfInfo->npuDevId, dpuProfInfo->dpuDevId,
422 : dpuProfInfo->timeStamp, reporterData.timeStamp, dpuProfInfo->aicpu_task_id);
423 0 : FillDpuTaskParaDetails(taskInfo, dpuProfInfo);
424 : }
425 :
426 9 : void ProfilingHandler::GetHCCLReportData(const TaskInfo& taskInfo, HCCLReportData& hcclReportData) const
427 : {
428 9 : if (taskInfo.dfxOpInfo_ == nullptr) {
429 3 : HCCL_ERROR("[ProfilingHandler]taskInfo.dfxOpInfo_ is nullptr");
430 1 : return;
431 : }
432 8 : hcclReportData.ts = taskInfo.taskParam_.endTime;
433 8 : const auto& profName = GetProfTaskOpNameV2(taskInfo.taskParam_.taskType);
434 8 : hcclReportData.profInfo.itemId = GetProfHashId(profName.c_str(), profName.length());
435 8 : auto newCclTagIt = cachedNewCclTag_.find(taskInfo.dfxOpInfo_->op_.opType);
436 8 : hcclReportData.profInfo.cclTag = (newCclTagIt != cachedNewCclTag_.end()) ? newCclTagIt->second : INVALID_U64;
437 8 : const auto& opTag = taskInfo.dfxOpInfo_->op_.opTag;
438 8 : uint64_t groupName = GetProfHashId(opTag.c_str(), opTag.length());
439 8 : if (taskInfo.dfxOpInfo_->comm_ == nullptr) {
440 0 : HCCL_ERROR("[ProfilingHandler]taskInfo.dfxOpInfo_->comm_ is nullptr");
441 0 : return;
442 : }
443 8 : if (taskInfo.dfxOpInfo_->isIndop_ == true) {
444 0 : hcclReportData.profInfo.groupName = groupName;
445 0 : hcclReportData.profInfo.rankSize = taskInfo.dfxOpInfo_->rankSize_;
446 : } else {
447 8 : CommunicatorImpl* commImp = static_cast<CommunicatorImpl*>(taskInfo.dfxOpInfo_->comm_);
448 8 : if (commImp == nullptr) {
449 0 : HCCL_ERROR("[ProfilingHandler]commImp is nullptr");
450 0 : return;
451 : }
452 8 : hcclReportData.profInfo.groupName = groupName;
453 8 : hcclReportData.profInfo.rankSize = commImp->GetRankSize();
454 : }
455 8 : hcclReportData.profInfo.workFlowMode = static_cast<u32>(HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE);
456 8 : hcclReportData.profInfo.planeID = 0;
457 8 : hcclReportData.profInfo.stage = 0;
458 8 : hcclReportData.profInfo.role = static_cast<uint32_t>(TaskRole::DST);
459 8 : hcclReportData.profInfo.durationEstimated = 0;
460 8 : hcclReportData.profInfo.localRank = taskInfo.dfxOpInfo_->op_.myRank;
461 8 : hcclReportData.profInfo.remoteRank = taskInfo.remoteRank_;
462 8 : hcclReportData.profInfo.dataType = taskInfo.dfxOpInfo_->op_.dataType;
463 8 : hcclReportData.profInfo.opType = taskInfo.dfxOpInfo_->op_.opType;
464 8 : hcclReportData.profInfo.transportType = static_cast<int32_t>(SimpleTaskType::UB);
465 8 : if (hcclReportData.profInfo.remoteRank == INVALID_VALUE_RANKID) {
466 0 : hcclReportData.profInfo.transportType = static_cast<int32_t>(SimpleTaskType::LOCAL);
467 0 : hcclReportData.profInfo.remoteRank = hcclReportData.profInfo.localRank;
468 0 : hcclReportData.profInfo.linkType = 0;
469 : }
470 8 : const auto& taskType = taskInfo.taskParam_.taskType;
471 8 : const auto& taskPara = taskInfo.taskParam_.taskPara;
472 8 : switch (taskType) {
473 4 : case TaskParamType::TASK_DPU_INLINE_WRITE:
474 : case TaskParamType::TASK_DPU_NOTIFY_WAIT:
475 : case TaskParamType::TASK_DPU_WRITE_WITH_NOTIFY:
476 : case TaskParamType::TASK_DPU_CHANNEL_FENCE:
477 4 : hcclReportData.dpuProfInfo.itemId = hcclReportData.profInfo.itemId;
478 4 : hcclReportData.dpuProfInfo.cclTag = hcclReportData.profInfo.cclTag;
479 4 : hcclReportData.dpuProfInfo.groupName = hcclReportData.profInfo.groupName;
480 4 : hcclReportData.dpuProfInfo.localRank = hcclReportData.profInfo.localRank;
481 4 : hcclReportData.dpuProfInfo.remoteRank = hcclReportData.profInfo.remoteRank;
482 4 : hcclReportData.dpuProfInfo.rankSize = hcclReportData.profInfo.rankSize;
483 4 : hcclReportData.dpuProfInfo.workFlowMode = hcclReportData.profInfo.workFlowMode;
484 4 : hcclReportData.dpuProfInfo.planeID = hcclReportData.profInfo.planeID;
485 4 : hcclReportData.dpuProfInfo.stage = hcclReportData.profInfo.stage;
486 4 : hcclReportData.dpuProfInfo.role = hcclReportData.profInfo.role;
487 4 : hcclReportData.dpuProfInfo.durationEstimated = hcclReportData.profInfo.durationEstimated;
488 4 : hcclReportData.dpuProfInfo.dataType = static_cast<uint32_t>(taskInfo.dfxOpInfo_->op_.dataType);
489 4 : hcclReportData.dpuProfInfo.opType = static_cast<uint32_t>(taskInfo.dfxOpInfo_->op_.opType);
490 4 : hcclReportData.dpuProfInfo.transportType = static_cast<uint32_t>(SimpleTaskType::ROCE);
491 4 : hcclReportData.dpuProfInfo.aicpu_task_id = taskInfo.taskParam_.aicpuTaskId;
492 4 : hcclReportData.dpuProfInfo.npuDevId = taskInfo.taskParam_.npuDevId;
493 4 : hcclReportData.dpuProfInfo.dpuDevId = DPU_DEV_ID_MASK;
494 4 : hcclReportData.dpuProfInfo.linkType = static_cast<uint16_t>(taskPara.DMA.linkType);
495 4 : if (hcclReportData.dpuProfInfo.remoteRank == INVALID_VALUE_RANKID) {
496 0 : hcclReportData.dpuProfInfo.transportType = static_cast<uint32_t>(SimpleTaskType::LOCAL);
497 0 : hcclReportData.dpuProfInfo.remoteRank = hcclReportData.dpuProfInfo.localRank;
498 0 : hcclReportData.dpuProfInfo.linkType = 0;
499 : }
500 4 : hcclReportData.dpuProfInfo.taskId = taskInfo.taskId_;
501 4 : hcclReportData.dpuProfInfo.streamId = taskInfo.streamId_;
502 4 : hcclReportData.dpuProfInfo.timeStamp = taskInfo.taskParam_.beginTime;
503 4 : switch (taskType) {
504 2 : case TaskParamType::TASK_DPU_INLINE_WRITE:
505 : case TaskParamType::TASK_DPU_WRITE_WITH_NOTIFY:
506 : hcclReportData.dpuProfInfo.srcAddr
507 2 : = static_cast<u64>(reinterpret_cast<uintptr_t>(taskPara.DMA.src));
508 : hcclReportData.dpuProfInfo.dstAddr
509 2 : = static_cast<u64>(reinterpret_cast<uintptr_t>(taskPara.DMA.dst));
510 2 : hcclReportData.dpuProfInfo.dataSize = static_cast<u32>(taskPara.DMA.size);
511 2 : hcclReportData.dpuProfInfo.notifyID = taskPara.DMA.notifyID;
512 2 : break;
513 2 : case TaskParamType::TASK_DPU_NOTIFY_WAIT:
514 : case TaskParamType::TASK_DPU_CHANNEL_FENCE:
515 2 : hcclReportData.dpuProfInfo.notifyID = taskPara.Notify.notifyID;
516 2 : break;
517 0 : default:
518 0 : break;
519 : }
520 4 : break;
521 4 : default:
522 4 : switch (taskType) {
523 1 : case TaskParamType::TASK_SDMA:
524 : case TaskParamType::TASK_RDMA:
525 1 : hcclReportData.profInfo.srcAddr = static_cast<u64>(reinterpret_cast<uintptr_t>(taskPara.DMA.src));
526 1 : hcclReportData.profInfo.dstAddr = static_cast<u64>(reinterpret_cast<uintptr_t>(taskPara.DMA.dst));
527 1 : hcclReportData.profInfo.dataSize = static_cast<u32>(taskPara.DMA.size);
528 1 : hcclReportData.profInfo.notifyID = taskPara.DMA.notifyID;
529 1 : hcclReportData.profInfo.linkType = static_cast<uint16_t>(taskPara.DMA.linkType);
530 1 : break;
531 1 : case TaskParamType::TASK_REDUCE_INLINE:
532 : case TaskParamType::TASK_REDUCE_TBE:
533 : hcclReportData.profInfo.srcAddr
534 1 : = static_cast<u64>(reinterpret_cast<uintptr_t>(taskPara.Reduce.src));
535 : hcclReportData.profInfo.dstAddr
536 1 : = static_cast<u64>(reinterpret_cast<uintptr_t>(taskPara.Reduce.dst));
537 1 : hcclReportData.profInfo.dataSize = static_cast<u32>(taskPara.Reduce.size);
538 1 : hcclReportData.profInfo.notifyID = taskPara.Reduce.notifyID;
539 1 : hcclReportData.profInfo.linkType = static_cast<uint16_t>(taskPara.Reduce.linkType);
540 1 : break;
541 1 : case TaskParamType::TASK_NOTIFY_RECORD:
542 : case TaskParamType::TASK_NOTIFY_WAIT:
543 1 : hcclReportData.profInfo.notifyID = taskPara.Notify.notifyID;
544 1 : break;
545 1 : default:
546 1 : break;
547 : }
548 4 : break;
549 : }
550 8 : }
551 :
552 1 : void ProfilingHandler::DumpHCCLReportData(const TaskInfo& taskInfo, const MsprofAdditionalInfo& reporterData) const
553 : {
554 1 : if (reporterData.type == static_cast<uint32_t>(ProfTaskType::TASK_HCCL_INFO)) {
555 1 : auto* profInfo = reinterpret_cast<const MsprofHcclInfo*>(reporterData.data);
556 3 : HCCL_INFO(
557 : "MsprofAdditionalInfo profInfo: timeStamp[%llu], itemId[%llu], cclTag[%llu], groupName[%llu], "
558 : "localRank[%u], remoteRank[%u], rankSize[%u], workFlowMode[%u], planeID[%u], ctxId[%u], "
559 : "stage[%u], role[%u], durationEstimated[%f], taskType[%d]",
560 : reporterData.timeStamp, profInfo->itemId, profInfo->cclTag, profInfo->groupName, profInfo->localRank,
561 : profInfo->remoteRank, profInfo->rankSize, profInfo->workFlowMode, profInfo->planeID, profInfo->ctxId,
562 : profInfo->stage, profInfo->role, profInfo->durationEstimated,
563 : static_cast<int>(taskInfo.taskParam_.taskType));
564 3 : HCCL_INFO(
565 : "MsprofAdditionalInfo profInfo detail: srcAddr[%llu], dstAddr[%llu], dataSize[%llu], notifyID[%llu], "
566 : "linkType[%u], opType[%s], transportType[%u], dataType[%s], rdmaType[%u]",
567 : profInfo->srcAddr, profInfo->dstAddr, profInfo->dataSize, profInfo->notifyID, profInfo->linkType,
568 : OpTypeToSerialString(profInfo->opType).c_str(), profInfo->transportType,
569 : DataTypeToSerialString(profInfo->dataType).c_str(), profInfo->rdmaType);
570 : } else {
571 0 : auto* dpuProfInfo = reinterpret_cast<const MsprofDpuHcclTrack*>(reporterData.data);
572 0 : HCCL_INFO(
573 : "MsprofAdditionalInfo dpuProfInfo: timeStamp[%llu], itemId[%llu], cclTag[%llu], groupName[%llu], "
574 : "localRank[%u], remoteRank[%u], rankSize[%u], workFlowMode[%u], planeID[%u], "
575 : "stage[%u], role[%u], durationEstimated[%f], taskType[%d]",
576 : reporterData.timeStamp, dpuProfInfo->itemId, dpuProfInfo->cclTag, dpuProfInfo->groupName,
577 : dpuProfInfo->localRank, dpuProfInfo->remoteRank, dpuProfInfo->rankSize, dpuProfInfo->workFlowMode,
578 : dpuProfInfo->planeID, dpuProfInfo->stage, dpuProfInfo->role, dpuProfInfo->durationEstimated,
579 : static_cast<int>(taskInfo.taskParam_.taskType));
580 0 : HCCL_INFO(
581 : "MsprofAdditionalInfo dpuProfInfo detail: srcAddr[%llu], dstAddr[%llu], dataSize[%llu], notifyID[%llu], "
582 : "linkType[%u], opType[%s], transportType[%u], dataType[%s], rdmaType[%u], "
583 : "taskId[%u], aicpu_task_id[%u], streamId[%u], npuDevId[%u], dpuDevId[%u], timeStamp[%llu]",
584 : dpuProfInfo->srcAddr, dpuProfInfo->dstAddr, dpuProfInfo->dataSize, dpuProfInfo->notifyID,
585 : dpuProfInfo->linkType, OpTypeToSerialString(dpuProfInfo->opType).c_str(), dpuProfInfo->transportType,
586 : DataTypeToSerialString(dpuProfInfo->dataType).c_str(), dpuProfInfo->rdmaType, dpuProfInfo->taskId,
587 : dpuProfInfo->aicpu_task_id, dpuProfInfo->streamId, dpuProfInfo->npuDevId, dpuProfInfo->dpuDevId,
588 : dpuProfInfo->timeStamp);
589 : }
590 1 : }
591 :
592 0 : void ProfilingHandler::LogCcuTaskInfo(
593 : const CcuProfilingInfo& info, const TaskInfo& taskInfo, uint64_t itemId, uint64_t groupName, u32 rankId,
594 : u32 ranksize) const
595 : {
596 0 : HCCL_INFO(
597 : "[ProfilingHandler]GetCcuTaskInfo, ccuTaskInfo data is: version[%u], workFlowMode[%u], itemId[%llu], "
598 : "groupName[%llu], rankId[%u], ranksize[%u], streamId[%u], taskId[%u], dieId[%u], "
599 : "missionId[%u],instrId[%u]",
600 : 0, static_cast<u32>(HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE), itemId, groupName, rankId, ranksize,
601 : taskInfo.streamId_, taskInfo.taskId_, info.dieId, info.missionId, info.instrId);
602 0 : }
603 :
604 0 : void ProfilingHandler::LogCcuWaitSignalInfo(
605 : const CcuProfilingInfo& info, const TaskInfo& taskInfo, uint64_t itemId, uint64_t groupName, u32 rankId,
606 : u32 ranksize) const
607 : {
608 0 : HCCL_INFO(
609 : "[ProfilingHandler]GetCcuWaitSignalInfo, waitSignalInfo data is: version[%u], itemId[%llu], groupName[%llu], "
610 : "rankId[%u], ranksize[%u], workFlowMode[%u], streamId[%u], taskId[%u], dieId[%u],instrId[%u],missionId[%u], "
611 : "ckeId[%u],mask[%u]",
612 : 0, itemId, groupName, rankId, ranksize, static_cast<u32>(HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE),
613 : taskInfo.streamId_, taskInfo.taskId_, info.dieId, info.instrId, info.missionId, info.ckeId, info.mask);
614 0 : }
615 :
616 0 : void ProfilingHandler::LogCcuGroupInfo(
617 : const CcuProfilingInfo& info, const TaskInfo& taskInfo, uint64_t itemId, uint64_t groupName, u32 rankId,
618 : u32 ranksize) const
619 : {
620 0 : HCCL_INFO(
621 : "[ProfilingHandler]GetCcuGroupInfo, ccuGroupInfo data is: version[%u], itemId[%llu], "
622 : "groupName[%llu], rankId[%u], ranksize[%u], workFlowMode[%u], streamId[%u], taskId[%u], "
623 : "dieId[%u],instrId[%u],missionId[%u], dataSize[%llu]",
624 : 0, itemId, groupName, rankId, ranksize, static_cast<u32>(HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE),
625 : taskInfo.streamId_, taskInfo.taskId_, info.dieId, info.instrId, info.missionId, info.dataSize);
626 0 : if (info.reduceOpType != INVALID_TYPE_VALUE) {
627 0 : HCCL_INFO("ccuGroupInfo reduceOpType is [%d]", static_cast<int>(info.reduceOpType));
628 : }
629 0 : if (info.inputDataType != INVALID_TYPE_VALUE) {
630 0 : HCCL_INFO("ccuGroupInfo inputDataType is [%d]", static_cast<int>(info.inputDataType));
631 : }
632 0 : if (info.outputDataType != INVALID_TYPE_VALUE) {
633 0 : HCCL_INFO("ccuGroupInfo outputDataType is [%d]", static_cast<int>(info.outputDataType));
634 : }
635 0 : for (auto i = 0; i < CCU_MAX_CHANNEL_NUM; i++) {
636 0 : if (info.channelId[i] != INVALID_VALUE_CHANNELID && info.remoteRankId[i] != INVALID_VALUE_RANKID) {
637 0 : HCCL_INFO(
638 : "[ProfilingHandler]GetCcuGroupInfo, ccuGroupInfo data is: channelId[%d] = %u, "
639 : "remoteRankId[%d] = %u",
640 : i, info.channelId[i], i, info.remoteRankId[i]);
641 : }
642 : }
643 0 : }
644 :
645 4 : void ProfilingHandler::ReportCcuInfoLog(const TaskInfo& taskInfo) const
646 : {
647 4 : if (LIKELY(HcclCheckLogLevel(HCCL_LOG_INFO) == 0)) {
648 4 : return;
649 : }
650 4 : if (!enableHcclL1_) {
651 0 : return;
652 : }
653 4 : if (taskInfo.taskParam_.ccuDetailInfo == nullptr || taskInfo.dfxOpInfo_ == nullptr) {
654 4 : return;
655 : }
656 0 : auto ccuDetailInfo = taskInfo.taskParam_.ccuDetailInfo;
657 0 : for (const auto& info : *ccuDetailInfo) {
658 0 : uint64_t itemId = GetProfHashId(info.name.c_str(), info.name.length());
659 : uint64_t groupName
660 0 : = GetProfHashId(taskInfo.dfxOpInfo_->op_.opTag.c_str(), taskInfo.dfxOpInfo_->op_.opTag.length());
661 0 : u32 rankId = 0;
662 0 : u32 ranksize = 0;
663 0 : if (taskInfo.dfxOpInfo_->isIndop_ == true) {
664 0 : rankId = taskInfo.dfxOpInfo_->op_.myRank;
665 0 : ranksize = taskInfo.dfxOpInfo_->rankSize_;
666 : } else {
667 0 : CommunicatorImpl* commImp = static_cast<CommunicatorImpl*>(taskInfo.dfxOpInfo_->comm_);
668 0 : rankId = commImp->GetIdIndex();
669 0 : ranksize = commImp->GetRankSize();
670 : }
671 0 : if (info.type == 0) {
672 0 : LogCcuTaskInfo(info, taskInfo, itemId, groupName, rankId, ranksize);
673 0 : } else if (info.type == 1) {
674 0 : LogCcuWaitSignalInfo(info, taskInfo, itemId, groupName, rankId, ranksize);
675 0 : } else if (info.type == CCU_TYPE) {
676 0 : LogCcuGroupInfo(info, taskInfo, itemId, groupName, rankId, ranksize);
677 : }
678 : }
679 0 : }
680 :
681 4 : void ProfilingHandler::ReportCcuInfo(const TaskInfo& taskInfo) const
682 : {
683 4 : ReportCcuInfoLog(taskInfo);
684 4 : if (taskInfo.taskParam_.ccuDetailInfo == nullptr) {
685 9 : HCCL_WARNING("[ProfilingHandler]ReportCcuInfo ccuDetailInfo is nullptr.");
686 4 : return;
687 : }
688 1 : if (taskInfo.dfxOpInfo_ == nullptr) {
689 3 : HCCL_WARNING("[ProfilingHandler]ReportCcuInfo dfxOpInfo_ is nullptr, skip ReportCcuInfo.");
690 1 : return;
691 : }
692 0 : auto ccuDetailInfo = taskInfo.taskParam_.ccuDetailInfo;
693 0 : for (const auto& info : *ccuDetailInfo) {
694 0 : if (info.type == 0 && enableHcclL1_) {
695 0 : GetCcuTaskInfo(taskInfo, info);
696 0 : } else if (info.type == 1 && enableHcclL1_) {
697 0 : GetCcuWaitSignalInfo(taskInfo, info);
698 0 : } else if (info.type == CCU_TYPE && enableHcclL1_) {
699 0 : GetCcuGroupInfo(taskInfo, info);
700 : }
701 : }
702 0 : }
703 :
704 0 : void ProfilingHandler::GetCcuTaskInfo(const TaskInfo& taskInfo, const CcuProfilingInfo& info) const
705 : {
706 0 : uint64_t timestamp = DlProfFunction::GetInstance().dlMsprofSysCycleTime();
707 0 : MsprofAdditionalInfo reporterData{};
708 0 : reporterData.level = MSPROF_REPORT_HCCL_NODE_LEVEL;
709 0 : reporterData.type = MSPROF_REPORT_CCU_TASK_INFO;
710 0 : reporterData.threadId = SalGetTid();
711 0 : reporterData.dataLen = sizeof(MsprofCcuTaskInfo);
712 0 : reporterData.timeStamp = timestamp;
713 0 : auto* ccuTaskInfo = reinterpret_cast<MsprofCcuTaskInfo*>(reporterData.data);
714 0 : ccuTaskInfo->version = 0;
715 0 : ccuTaskInfo->workFlowMode = static_cast<u32>(HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE);
716 0 : ccuTaskInfo->itemId = GetProfHashId(info.name.c_str(), info.name.length());
717 0 : uint64_t groupName = GetProfHashId(taskInfo.dfxOpInfo_->op_.opTag.c_str(), taskInfo.dfxOpInfo_->op_.opTag.length());
718 0 : ccuTaskInfo->groupName = groupName;
719 0 : if (taskInfo.dfxOpInfo_->isIndop_ == true) {
720 0 : ccuTaskInfo->rankId = taskInfo.dfxOpInfo_->op_.myRank;
721 0 : ccuTaskInfo->ranksize = taskInfo.dfxOpInfo_->rankSize_;
722 : } else {
723 0 : CommunicatorImpl* commImp = static_cast<CommunicatorImpl*>(taskInfo.dfxOpInfo_->comm_);
724 0 : ccuTaskInfo->rankId = commImp->GetIdIndex();
725 0 : ccuTaskInfo->ranksize = commImp->GetRankSize();
726 : }
727 :
728 0 : ccuTaskInfo->streamId = taskInfo.streamId_;
729 0 : ccuTaskInfo->taskId = taskInfo.taskId_;
730 0 : ccuTaskInfo->dieId = info.dieId;
731 0 : ccuTaskInfo->missionId = info.missionId;
732 0 : ccuTaskInfo->instrId = info.instrId;
733 0 : ReportAdditionInfo(reporterData);
734 0 : }
735 :
736 1 : void ProfilingHandler::GetCcuGroupInfo(const TaskInfo& taskInfo, const CcuProfilingInfo& info) const
737 : {
738 1 : MsprofAdditionalInfo reporterData{};
739 1 : reporterData.level = MSPROF_REPORT_HCCL_NODE_LEVEL;
740 1 : reporterData.type = MSPROF_REPORT_CCU_GROUP_INFO;
741 1 : reporterData.threadId = SalGetTid();
742 1 : reporterData.dataLen = sizeof(MsprofCcuGroupInfo);
743 1 : reporterData.timeStamp = DlProfFunction::GetInstance().dlMsprofSysCycleTime();
744 1 : auto* ccuGroupInfo = reinterpret_cast<MsprofCcuGroupInfo*>(reporterData.data);
745 1 : ccuGroupInfo->version = 0;
746 1 : ccuGroupInfo->itemId = GetProfHashId(info.name.c_str(), info.name.length());
747 1 : uint64_t groupName = GetProfHashId(taskInfo.dfxOpInfo_->op_.opTag.c_str(), taskInfo.dfxOpInfo_->op_.opTag.length());
748 1 : ccuGroupInfo->groupName = groupName;
749 1 : if (taskInfo.dfxOpInfo_->isIndop_ == true) {
750 0 : ccuGroupInfo->rankId = taskInfo.dfxOpInfo_->op_.myRank;
751 0 : ccuGroupInfo->ranksize = taskInfo.dfxOpInfo_->rankSize_;
752 : } else {
753 1 : CommunicatorImpl* commImp = static_cast<CommunicatorImpl*>(taskInfo.dfxOpInfo_->comm_);
754 1 : ccuGroupInfo->rankId = commImp->GetIdIndex();
755 1 : ccuGroupInfo->ranksize = commImp->GetRankSize();
756 : }
757 1 : ccuGroupInfo->workFlowMode = static_cast<u32>(HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE);
758 1 : ccuGroupInfo->streamId = taskInfo.streamId_;
759 1 : ccuGroupInfo->taskId = taskInfo.taskId_;
760 1 : ccuGroupInfo->dieId = info.dieId;
761 1 : ccuGroupInfo->instrId = info.instrId;
762 1 : ccuGroupInfo->missionId = info.missionId;
763 1 : ccuGroupInfo->reduceOpType = info.reduceOpType;
764 1 : ccuGroupInfo->inputDataType = info.inputDataType;
765 1 : ccuGroupInfo->outputDataType = info.outputDataType;
766 1 : ccuGroupInfo->dataSize = info.dataSize;
767 1 : std::copy(info.channelId, info.channelId + CCU_MAX_CHANNEL_NUM, ccuGroupInfo->channelId);
768 1 : std::copy(info.remoteRankId, info.remoteRankId + CCU_MAX_CHANNEL_NUM, ccuGroupInfo->remoteRankId);
769 1 : ReportAdditionInfo(reporterData);
770 1 : }
771 :
772 0 : void ProfilingHandler::DumpCcuGroupInfo([[maybe_unused]] const MsprofCcuGroupInfo& ccuGroupInfo) const
773 : {
774 0 : if (LIKELY(HcclCheckLogLevel(HCCL_LOG_INFO) == 0)) {
775 0 : return;
776 : }
777 : // HCCL_INFOs migrated to ReportCcuInfoLog
778 : }
779 :
780 1 : void ProfilingHandler::GetCcuWaitSignalInfo(const TaskInfo& taskInfo, const CcuProfilingInfo& info) const
781 : {
782 1 : uint64_t timestamp = DlProfFunction::GetInstance().dlMsprofSysCycleTime();
783 1 : MsprofAdditionalInfo reporterData{};
784 1 : reporterData.level = MSPROF_REPORT_HCCL_NODE_LEVEL;
785 1 : reporterData.type = MSPROF_REPORT_CCU_WAIT_SIGNAL_INFO;
786 1 : reporterData.threadId = SalGetTid();
787 1 : reporterData.dataLen = sizeof(MsprofCcuWaitSignalInfo);
788 1 : reporterData.timeStamp = timestamp;
789 1 : auto* waitSignalInfo = reinterpret_cast<MsprofCcuWaitSignalInfo*>(reporterData.data);
790 1 : waitSignalInfo->version = 0;
791 1 : waitSignalInfo->itemId = GetProfHashId(info.name.c_str(), info.name.length());
792 1 : uint64_t groupName = GetProfHashId(taskInfo.dfxOpInfo_->op_.opTag.c_str(), taskInfo.dfxOpInfo_->op_.opTag.length());
793 1 : waitSignalInfo->groupName = groupName;
794 1 : if (taskInfo.dfxOpInfo_->isIndop_ == true) {
795 0 : waitSignalInfo->rankId = taskInfo.dfxOpInfo_->op_.myRank;
796 0 : waitSignalInfo->ranksize = taskInfo.dfxOpInfo_->rankSize_;
797 : } else {
798 1 : CommunicatorImpl* commImp = static_cast<CommunicatorImpl*>(taskInfo.dfxOpInfo_->comm_);
799 1 : waitSignalInfo->rankId = commImp->GetIdIndex();
800 1 : waitSignalInfo->ranksize = commImp->GetRankSize();
801 : }
802 1 : waitSignalInfo->workFlowMode = static_cast<u32>(HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE);
803 1 : waitSignalInfo->streamId = taskInfo.streamId_;
804 1 : waitSignalInfo->taskId = taskInfo.taskId_;
805 1 : waitSignalInfo->dieId = info.dieId;
806 1 : waitSignalInfo->instrId = info.instrId;
807 1 : waitSignalInfo->missionId = info.missionId;
808 1 : waitSignalInfo->ckeId = info.ckeId;
809 1 : waitSignalInfo->mask = info.mask;
810 1 : std::copy(info.channelId, info.channelId + CCU_MAX_CHANNEL_NUM, waitSignalInfo->channelId);
811 1 : std::copy(info.remoteRankId, info.remoteRankId + CCU_MAX_CHANNEL_NUM, waitSignalInfo->remoteRankId);
812 1 : ReportAdditionInfo(reporterData);
813 1 : }
814 :
815 2 : void ProfilingHandler::ReportAclApi(
816 : uint32_t cmdType, uint64_t beginTime, uint64_t endTime, uint64_t cmdItemId, uint32_t threadId, bool cachedReq)
817 : {
818 2 : MsprofApi reporterData{};
819 2 : reporterData.level = MSPROF_REPORT_ACL_LEVEL;
820 2 : reporterData.type = static_cast<int32_t>(cmdType) + MSPROF_REPORT_ACL_HOST_HCCL_BASE_TYPE;
821 2 : reporterData.threadId = threadId;
822 2 : reporterData.beginTime = beginTime;
823 2 : reporterData.endTime = endTime;
824 2 : reporterData.itemId = cmdItemId;
825 :
826 2 : if (cachedReq) {
827 1 : std::lock_guard<std::mutex> lock(cachedAclApiInfoMutex_);
828 3 : HCCL_INFO(
829 : "ReportAclApi MSPROF_REPORT_ACL_LEVEL type[%u] beginTime[%llu]", reporterData.type, reporterData.beginTime);
830 1 : cachedAclApiInfo_.push(reporterData);
831 1 : }
832 :
833 2 : s32 ret = DlProfFunction::GetInstance().dlMsprofReportApi(1, &reporterData);
834 6 : HCCL_INFO(
835 : "[ProfilingHandler][ReportAclApi], reporterData data is: level[%u], type[%u], threadId[%u], "
836 : "beginTime[%llu], endTime[%llu], itemId[%llu], return value[%d]",
837 : reporterData.level, reporterData.type, reporterData.threadId, reporterData.beginTime, reporterData.endTime,
838 : reporterData.itemId, ret);
839 2 : if (ret != 0) {
840 0 : THROW<InternalException>("Call dlMsprofReportApi failed, return[%d]", ret);
841 : }
842 2 : }
843 :
844 9 : void ProfilingHandler::ReportNodeApi(
845 : uint64_t beginTime, uint64_t endTime, uint64_t cmdItemId, uint32_t threadId, bool cachedReq)
846 : {
847 9 : MsprofApi reporterData{};
848 9 : reporterData.level = MSPROF_REPORT_NODE_LEVEL;
849 9 : reporterData.type = MSPROF_REPORT_NODE_LAUNCH_TYPE;
850 9 : reporterData.threadId = threadId;
851 9 : reporterData.beginTime = beginTime;
852 9 : reporterData.endTime = endTime;
853 9 : reporterData.itemId = cmdItemId;
854 :
855 9 : if (cachedReq) {
856 7 : std::lock_guard<std::mutex> lock(cachedTaskApiInfoMutex_);
857 7 : cachedTaskApiInfo_.push(reporterData);
858 7 : }
859 9 : if (!enableHostApi_) {
860 8 : return;
861 : }
862 :
863 1 : s32 ret = DlProfFunction::GetInstance().dlMsprofReportApi(1, &reporterData);
864 3 : HCCL_INFO(
865 : "[ProfilingHandler][ReportNodeApi], reporterData data is: level[%u], type[%u], threadId[%u], "
866 : "beginTime[%llu], endTime[%llu], itemId[%llu], return value[%d]",
867 : reporterData.level, reporterData.type, reporterData.threadId, reporterData.beginTime, reporterData.endTime,
868 : reporterData.itemId, ret);
869 1 : if (ret != 0) {
870 0 : THROW<InternalException>("Call MsprofReportApi failed, return[%d]", ret);
871 : }
872 : }
873 :
874 9 : void ProfilingHandler::ReportNodeBasicInfo(uint64_t timeStamp, uint64_t cmdItemId, uint32_t threadId, bool cachedReq)
875 : {
876 9 : MsprofCompactInfo reporterData{};
877 9 : reporterData.level = MSPROF_REPORT_NODE_LEVEL;
878 9 : reporterData.type = MSPROF_REPORT_NODE_BASIC_INFO_TYPE;
879 9 : reporterData.threadId = threadId;
880 9 : reporterData.dataLen = sizeof(MsprofNodeBasicInfo);
881 9 : reporterData.timeStamp = timeStamp;
882 9 : reporterData.data.nodeBasicInfo.opName = cmdItemId;
883 9 : reporterData.data.nodeBasicInfo.taskType = MSPROF_GE_TASK_TYPE_HCCL;
884 9 : reporterData.data.nodeBasicInfo.opType = cmdItemId;
885 9 : reporterData.data.nodeBasicInfo.opFlag = 0;
886 27 : HCCL_INFO(
887 : "[ProfilingHandler][ReportNodeBasicInfo], reporterData data is: level[%u], type[%u], threadId[%u], "
888 : "dataLen[%u], taskType[%u], opFlag[%u]",
889 : reporterData.level, reporterData.type, reporterData.threadId, reporterData.dataLen,
890 : reporterData.data.nodeBasicInfo.taskType, reporterData.data.nodeBasicInfo.opFlag);
891 9 : if (cachedReq) {
892 7 : std::lock_guard<std::mutex> lock(cacheHcclOpInfoMutex_);
893 7 : cacheHcclOpInfo_.push(reporterData);
894 7 : }
895 9 : if (!enableHcclL1_) {
896 7 : return;
897 : }
898 2 : s32 ret = DlProfFunction::GetInstance().dlMsprofReportCompactInfo(1, &reporterData, sizeof(MsprofCompactInfo));
899 6 : HCCL_INFO("Call MsprofReportCompactInfo, return value[%d]", ret);
900 2 : if (ret != 0) {
901 0 : THROW<InternalException>("Call MsprofReportCompactInfo failed, return[%d]", ret);
902 : }
903 : }
904 :
905 10 : void ProfilingHandler::ReportHcclOpInfo(uint64_t timeStamp, const DfxOpInfo& opInfo, uint32_t threadId, bool cachedReq)
906 : {
907 10 : MsprofCompactInfo reporterData{};
908 10 : reporterData.level = MSPROF_REPORT_NODE_LEVEL;
909 10 : reporterData.type = MSPROF_REPORT_NODE_HCCL_OP_INFO_TYPE;
910 10 : reporterData.threadId = threadId;
911 10 : reporterData.dataLen = sizeof(MsprofHCCLOPInfo);
912 10 : reporterData.timeStamp = timeStamp;
913 10 : reporterData.data.hcclopInfo.relay = 0;
914 10 : reporterData.data.hcclopInfo.retry = 0;
915 10 : reporterData.data.hcclopInfo.dataType = opInfo.op_.dataType;
916 10 : reporterData.data.hcclopInfo.algType = cachedAlgTypeHashId_.load();
917 10 : uint64_t groupName = GetProfHashId(opInfo.op_.opTag.c_str(), opInfo.op_.opTag.length());
918 10 : reporterData.data.hcclopInfo.groupName = groupName;
919 10 : u32 ranksize{0};
920 10 : if (opInfo.isIndop_ == true) {
921 0 : ranksize = opInfo.rankSize_;
922 0 : reporterData.data.hcclopInfo.count = opInfo.op_.dataCount;
923 : } else {
924 10 : CommunicatorImpl* commImp = static_cast<CommunicatorImpl*>(opInfo.comm_);
925 10 : ranksize = commImp->GetRankSize();
926 10 : if (opInfo.op_.opType == OpType::ALLTOALLV) {
927 1 : u64 sendCount = 0;
928 3 : for (u64 i = 0; i < ranksize; i++) {
929 2 : sendCount += *(static_cast<const u64*>(opInfo.op_.all2AllVDataDes.sendCounts) + i);
930 : }
931 1 : reporterData.data.hcclopInfo.count = sendCount;
932 9 : } else if (opInfo.op_.opType == OpType::ALLTOALL) {
933 0 : reporterData.data.hcclopInfo.count = opInfo.op_.all2AllDataDes.sendCount;
934 : } else {
935 9 : reporterData.data.hcclopInfo.count = opInfo.op_.dataCount;
936 : }
937 : }
938 10 : if (cachedReq) {
939 7 : std::lock_guard<std::mutex> lock(cacheHcclOpInfoMutex_);
940 7 : cacheHcclOpInfo_.push(reporterData);
941 7 : }
942 10 : if (!enableHostApi_) {
943 8 : return;
944 : }
945 6 : HCCL_INFO(
946 : "[ProfilingHandler][ReportHcclOpInfo], data is: level[%u], type[%u], threadId[%u], dataLen[%u], "
947 : "timeStamp[%llu], relay [%u], retry[%u], dataType[%s], algType[%llu], groupName[%llu], count[%llu]",
948 : reporterData.level, reporterData.type, reporterData.threadId, reporterData.dataLen, reporterData.timeStamp,
949 : reporterData.data.hcclopInfo.relay, reporterData.data.hcclopInfo.retry,
950 : DataTypeToSerialString(reporterData.data.hcclopInfo.dataType).c_str(), reporterData.data.hcclopInfo.algType,
951 : reporterData.data.hcclopInfo.groupName, reporterData.data.hcclopInfo.count);
952 2 : s32 ret = DlProfFunction::GetInstance().dlMsprofReportCompactInfo(1, &reporterData, sizeof(MsprofCompactInfo));
953 2 : if (ret != 0) {
954 0 : THROW<InternalException>("[ProfilingHandler] Call dlMsprofReportCompactInfo failed, return[%d]", ret);
955 : }
956 : }
957 :
958 2 : void ProfilingHandler::ReportAdditionInfo(MsprofAdditionalInfo& reporterData) const
959 : {
960 : s32 ret
961 2 : = DlProfFunction::GetInstance().dlMsprofReportAdditionalInfo(0, &reporterData, sizeof(MsprofAdditionalInfo));
962 6 : HCCL_INFO(
963 : "[ProfilingHandler][ReportAdditionInfo], level[%u], type[%u], threadId[%u], dataLen[%u], "
964 : "timeStamp[%llu], return value[%d]",
965 : reporterData.level, reporterData.type, reporterData.threadId, reporterData.dataLen, reporterData.timeStamp,
966 : ret);
967 2 : if (ret != 0) {
968 0 : THROW<InternalException>("Call MsprofReportAdditionalInfo failed, return[%d]", ret);
969 : }
970 2 : }
971 :
972 2 : int32_t ProfilingHandler::CommandHandle(uint32_t rtType, void* data, uint32_t len) const
973 : {
974 : (void)len;
975 2 : if (data == nullptr || rtType != rtProfCtrlType_t::RT_PROF_CTRL_SWITCH) {
976 3 : HCCL_ERROR("[ProfilingHandler][CommandHandle] data is nullptr or rtType is invalid, rtType[%u]", rtType);
977 1 : return HCCL_E_PARA;
978 : }
979 1 : rtProfCommandHandle_t* profConfigParam = static_cast<rtProfCommandHandle_t*>(data);
980 1 : auto type = profConfigParam->type;
981 1 : auto profconfig = profConfigParam->profSwitch;
982 3 : HCCL_RUN_INFO(
983 : "[Profiling][CommandHandle] CommandHandle's rtType is %u. CommandHandle_switch type[%u], "
984 : "profconfig[%llu], deviceLogicId[%u]",
985 : rtType, type, profconfig, profConfigParam->devIdList[0]);
986 1 : switch (type) {
987 0 : case PROF_COMMANDHANDLE_TYPE_START:
988 0 : instance_.StartSubscribe(profconfig);
989 0 : break;
990 0 : case PROF_COMMANDHANDLE_TYPE_STOP:
991 0 : instance_.StopSubscribe();
992 0 : break;
993 1 : default:
994 3 : HCCL_RUN_INFO("[Profiling][CommandHandle] Unexpected behaviour.");
995 : }
996 1 : return HCCL_SUCCESS;
997 : }
998 :
999 5 : void ProfilingHandler::StartSubscribe(uint64_t profconfig)
1000 : {
1001 15 : HCCL_RUN_INFO("[Profiling][CommandHandle] profSwitch is[%llu]", profconfig);
1002 5 : SetCachedCclTag();
1003 5 : cachedAlgTypeHashId_ = GetProfHashId("AlgType::NHR", strlen("AlgType::NHR"));
1004 5 : if ((profconfig & PROF_ACL_API_MASK) != 0) {
1005 1 : StartHostApiSubscribe();
1006 : }
1007 5 : if ((profconfig & PROF_TASK_TIME_MASK) != 0 && (profconfig & PROF_TASK_TIME_L1_MASK) == 0) {
1008 1 : StartHostHcclOpSubscribe();
1009 : }
1010 5 : if ((profconfig & PROF_TASK_TIME_L1_MASK) != 0) {
1011 1 : StartTaskApiSubscribe();
1012 1 : StartAdditionInfoSubscribe();
1013 1 : StartCcuSubscribe();
1014 : }
1015 15 : HCCL_RUN_INFO("[Profiling][CommandHandle] profSwitch is[%llu]", profconfig);
1016 5 : }
1017 :
1018 2 : void ProfilingHandler::StartHostApiSubscribe()
1019 : {
1020 2 : enableHostApi_ = true;
1021 2 : CallProfRegHostApi();
1022 2 : ReportStoragedCompactInfo();
1023 2 : ReportMc2AdditionInfo();
1024 2 : ReportStoragedAclApi();
1025 6 : HCCL_RUN_INFO("SetHostApiSubscribe:[%d]", enableHostApi_);
1026 2 : }
1027 :
1028 2 : void ProfilingHandler::CallProfRegHostApi() const
1029 : {
1030 2 : if (!enableHostApi_) {
1031 0 : return;
1032 : }
1033 2 : auto& profFunction = DlProfFunction::GetInstance();
1034 46 : for (auto i = 0; i < OpType::__COUNT__; ++i) {
1035 44 : OpType type(static_cast<OpType::Value>(i));
1036 44 : s32 ret = profFunction.dlMsprofRegTypeInfo(
1037 44 : MSPROF_REPORT_ACL_LEVEL, static_cast<uint32_t>(type) + MSPROF_REPORT_ACL_HOST_HCCL_BASE_TYPE,
1038 88 : type.Describe().c_str());
1039 44 : if (ret != 0) {
1040 0 : THROW<InternalException>("Call MsprofRegTypeInfo fail, return[%d]", ret);
1041 : }
1042 : }
1043 46 : for (auto i = 0; i < OpType::__COUNT__; ++i) {
1044 44 : OpType type(static_cast<OpType::Value>(i));
1045 44 : s32 ret = profFunction.dlMsprofRegTypeInfo(
1046 44 : MSPROF_REPORT_NODE_LEVEL, static_cast<uint32_t>(type) + MSPROF_REPORT_NODE_HCCL_BASE_TYPE,
1047 88 : type.Describe().c_str());
1048 44 : if (ret != 0) {
1049 0 : THROW<InternalException>("Call MsprofRegTypeInfo fail, return[%d]", ret);
1050 : }
1051 : }
1052 2 : const std::string hcclType("hccl_op_info");
1053 2 : s32 ret = profFunction.dlMsprofRegTypeInfo(
1054 : MSPROF_REPORT_NODE_LEVEL, MSPROF_REPORT_NODE_HCCL_OP_INFO_TYPE, hcclType.c_str());
1055 2 : if (ret != 0) {
1056 0 : THROW<InternalException>("Call MsprofRegTypeInfo fail, return[%d]", ret);
1057 : }
1058 2 : }
1059 :
1060 4 : void ProfilingHandler::ReportStoragedCompactInfo()
1061 : {
1062 4 : std::lock_guard<std::mutex> lock(cacheHcclOpInfoMutex_);
1063 12 : HCCL_INFO("[ReportStoragedCompactInfo] The size of the storageCompactInfo_ is [%zu]", cacheHcclOpInfo_.size());
1064 4 : std::queue<MsprofCompactInfo> tempCompactInfo = cacheHcclOpInfo_;
1065 52 : while (!tempCompactInfo.empty()) {
1066 48 : MsprofCompactInfo reportData = tempCompactInfo.front();
1067 48 : tempCompactInfo.pop();
1068 48 : s32 ret = DlProfFunction::GetInstance().dlMsprofReportCompactInfo(0, &reportData, sizeof(MsprofCompactInfo));
1069 48 : if (ret != 0) {
1070 0 : THROW<InternalException>("Call MsprofRegTypeInfo failed, return[%d]", ret);
1071 : }
1072 : }
1073 4 : }
1074 :
1075 2 : void ProfilingHandler::ReportMc2AdditionInfo()
1076 : {
1077 2 : std::lock_guard<std::mutex> lock(cacheHcclAdditionInfoMutex_);
1078 6 : HCCL_INFO("[ReportMc2AdditionInfo] The size of the storageCompactInfo_ is [%zu]", cacheHcclAdditionInfo_.size());
1079 2 : std::queue<MsprofAdditionalInfo> tempCompactInfo = cacheHcclAdditionInfo_;
1080 6 : while (!tempCompactInfo.empty()) {
1081 4 : MsprofAdditionalInfo reportData = tempCompactInfo.front();
1082 4 : tempCompactInfo.pop();
1083 : s32 ret
1084 4 : = DlProfFunction::GetInstance().dlMsprofReportAdditionalInfo(1, &reportData, sizeof(MsprofAdditionalInfo));
1085 4 : if (ret != 0) {
1086 0 : THROW<InternalException>("Call MsprofRegTypeInfo failed, return[%d]", ret);
1087 : }
1088 : }
1089 2 : }
1090 :
1091 2 : void ProfilingHandler::StartTaskApiSubscribe()
1092 : {
1093 2 : enableHcclNode_ = true;
1094 2 : CallProfRegTaskTypeApi();
1095 2 : ReportStoragedTaskApi();
1096 6 : HCCL_INFO("SetTaskApiSubscribe:[%d]", enableHcclNode_);
1097 2 : }
1098 :
1099 2 : void ProfilingHandler::CallProfRegTaskTypeApi() const
1100 : {
1101 2 : if (!enableHcclNode_) {
1102 0 : HCCL_INFO("[ProfilingHandler] enableHostApi_ is false.");
1103 0 : return;
1104 : }
1105 2 : const std::string hcclType("hccl_info");
1106 2 : s32 sret = DlProfFunction::GetInstance().dlMsprofRegTypeInfo(
1107 : MSPROF_REPORT_HCCL_NODE_LEVEL, static_cast<uint32_t>(ProfTaskType::TASK_HCCL_INFO), hcclType.c_str());
1108 2 : if (sret != 0) {
1109 0 : THROW<InternalException>("Call MsprofRegTypeInfo fail, return[%d]", sret);
1110 : }
1111 :
1112 2 : const std::string dpuhcclType("dpu_hccl_info");
1113 2 : sret = DlProfFunction::GetInstance().dlMsprofRegTypeInfo(
1114 : MSPROF_REPORT_HCCL_NODE_LEVEL, static_cast<uint32_t>(ProfTaskType::TASK_DPU_HCCL_INFO), dpuhcclType.c_str());
1115 2 : if (sret != 0) {
1116 0 : THROW<InternalException>("Call MsprofRegTypeInfo fail, return[%d]", sret);
1117 : }
1118 : const std::vector<std::pair<uint32_t, std::string>> taskTypes
1119 6 : = {{MSPROF_REPORT_NODE_CONTEXT_ID_INFO_TYPE, "context_id_info"}};
1120 : const std::vector<std::pair<uint32_t, std::string>> taskOtherTypes
1121 0 : = {{MSPROF_REPORT_NODE_BASIC_INFO_TYPE, "node_basic_info"},
1122 8 : {MSPROF_REPORT_NODE_MC2_COMMINFO_TYPE, "mc2_comm_info"}};
1123 :
1124 4 : for (auto& it : taskTypes) {
1125 2 : s32 ret = DlProfFunction::GetInstance().dlMsprofRegTypeInfo(
1126 2 : MSPROF_REPORT_HCCL_NODE_LEVEL, it.first, it.second.c_str());
1127 2 : if (ret != 0) {
1128 0 : THROW<InternalException>("Call dlMsprofRegTypeInfo failed, return[%d]", ret);
1129 : }
1130 : }
1131 6 : for (auto& it : taskOtherTypes) {
1132 : s32 ret
1133 4 : = DlProfFunction::GetInstance().dlMsprofRegTypeInfo(MSPROF_REPORT_NODE_LEVEL, it.first, it.second.c_str());
1134 4 : if (ret != 0) {
1135 0 : THROW<InternalException>("Call dlMsprofRegTypeInfo failed, return[%d]", ret);
1136 : }
1137 : }
1138 6 : }
1139 :
1140 2 : void ProfilingHandler::ReportStoragedTaskApi()
1141 : {
1142 2 : std::lock_guard<std::mutex> lock(cachedTaskApiInfoMutex_);
1143 6 : HCCL_INFO("[ReportStoragedTaskApi] taskApiQueueSize is [%zu]", cachedTaskApiInfo_.size());
1144 2 : if (!cachedTaskApiInfo_.empty()) {
1145 2 : std::queue<MsprofApi> tempTaskApi = cachedTaskApiInfo_;
1146 20 : while (!tempTaskApi.empty()) {
1147 18 : MsprofApi reportData = tempTaskApi.front();
1148 18 : tempTaskApi.pop();
1149 18 : s32 ret = DlProfFunction::GetInstance().dlMsprofReportApi(0, &reportData);
1150 18 : if (ret != 0) {
1151 0 : THROW<InternalException>("Call dlMsprofReportApi failed, return[%d]", ret);
1152 : }
1153 : }
1154 2 : }
1155 2 : }
1156 :
1157 2 : void ProfilingHandler::ReportStoragedAclApi()
1158 : {
1159 2 : std::lock_guard<std::mutex> lock(cachedAclApiInfoMutex_);
1160 6 : HCCL_INFO("[ReportStoragedAclApi] aclApiQueueSize is [%u]", cachedAclApiInfo_.size());
1161 2 : if (!cachedAclApiInfo_.empty()) {
1162 2 : std::queue<MsprofApi> tempAclApi = cachedAclApiInfo_;
1163 4 : while (!tempAclApi.empty()) {
1164 2 : MsprofApi reportData = tempAclApi.front();
1165 2 : tempAclApi.pop();
1166 2 : s32 ret = DlProfFunction::GetInstance().dlMsprofReportApi(0, &reportData);
1167 2 : if (ret != 0) {
1168 0 : THROW<InternalException>("Call dlMsprofReportApi failed, return[%d]", ret);
1169 : }
1170 : }
1171 2 : }
1172 2 : }
1173 :
1174 2 : void ProfilingHandler::StartHostHcclOpSubscribe()
1175 : {
1176 2 : enableHcclNode_ = true;
1177 2 : enableHcclL0_ = true;
1178 2 : CallProfRegHcclOpApi();
1179 2 : ReportStoragedCompactInfo();
1180 6 : HCCL_RUN_INFO("StartHostHcclOpSubscribe:enableHcclNode_[%d],enableHcclL0_[%d]", enableHcclNode_, enableHcclL0_);
1181 2 : }
1182 :
1183 2 : void ProfilingHandler::CallProfRegHcclOpApi() const
1184 : {
1185 2 : if (enableHcclL0_ == false) {
1186 0 : HCCL_INFO("[ProfilingHandler] enableHcclNode_ is false.");
1187 0 : return;
1188 : }
1189 46 : for (auto i = 0; i < OpType::__COUNT__; ++i) {
1190 44 : OpType type(static_cast<OpType::Value>(i));
1191 88 : s32 ret = DlProfFunction::GetInstance().dlMsprofRegTypeInfo(
1192 44 : MSPROF_REPORT_HCCL_NODE_LEVEL, static_cast<uint32_t>(type) + MSPROF_REPORT_ACL_HOST_HCCL_BASE_TYPE,
1193 88 : type.Describe().c_str());
1194 44 : if (ret != 0) {
1195 0 : THROW<InternalException>("[ProfilingHandler]Call MsprofReportApi fail, return[%d]", ret);
1196 : }
1197 : }
1198 2 : s32 ret = DlProfFunction::GetInstance().dlMsprofRegTypeInfo(
1199 : MSPROF_REPORT_NODE_LEVEL, MSPROF_REPORT_NODE_MC2_COMMINFO_TYPE, "mc2_comm_info");
1200 2 : if (ret != 0) {
1201 0 : THROW<InternalException>("[ProfilingHandler]Call MsprofRegTypeInfo fail, return[%d]", ret);
1202 : }
1203 : }
1204 :
1205 2 : void ProfilingHandler::StartAdditionInfoSubscribe()
1206 : {
1207 2 : enableHcclL1_ = true;
1208 2 : ReportStoragedAdditionInfo();
1209 6 : HCCL_RUN_INFO("StartAdditionInfoSubscribe:enableHcclL1_[%d]", enableHcclL1_);
1210 2 : }
1211 :
1212 2 : void ProfilingHandler::ReportStoragedAdditionInfoLog() const
1213 : {
1214 2 : if (LIKELY(HcclCheckLogLevel(HCCL_LOG_INFO) == 0)) {
1215 0 : return;
1216 : }
1217 2 : if (cacheTaskInfos_.empty()) {
1218 0 : return;
1219 : }
1220 4 : for (const auto& taskInfo : cacheTaskInfos_) {
1221 6 : HCCL_INFO("ReportStoragedAdditionInfoLog, taskType[%u]", GetTaskTypeValue(taskInfo.taskParam_.taskType));
1222 : }
1223 : }
1224 :
1225 2 : void ProfilingHandler::ReportStoragedAdditionInfo()
1226 : {
1227 2 : std::lock_guard<std::mutex> lock(cacheTaskInfosMutex_);
1228 2 : ReportStoragedAdditionInfoLog();
1229 2 : if (cacheTaskInfos_.empty()) {
1230 0 : HCCL_INFO("[ProfilingHandler]ReportStoragedAdditionInfo cacheTaskInfos_ is empty.");
1231 0 : return;
1232 : }
1233 4 : for (auto& taskInfo : cacheTaskInfos_) {
1234 2 : MsprofAdditionalInfo reporterData{};
1235 2 : FillTaskAdditionInfo(taskInfo, reporterData);
1236 2 : CallAdditionInfo(reporterData);
1237 : }
1238 2 : }
1239 :
1240 2 : void ProfilingHandler::StartCcuSubscribe()
1241 : {
1242 2 : enableHcclNode_ = true;
1243 2 : enableHcclL1_ = true;
1244 6 : HCCL_INFO("ProfilingHandler StartCcuSubscribe");
1245 : const std::vector<std::pair<uint32_t, std::string>> ccuInfoTypes
1246 : = {{MSPROF_REPORT_CCU_TASK_INFO, "ccu_task_info"},
1247 : {MSPROF_REPORT_CCU_WAIT_SIGNAL_INFO, "ccu_wait_signal_info"},
1248 10 : {MSPROF_REPORT_CCU_GROUP_INFO, "ccu_group_info"}};
1249 8 : for (auto& it : ccuInfoTypes) {
1250 6 : s32 ret = DlProfFunction::GetInstance().dlMsprofRegTypeInfo(
1251 6 : MSPROF_REPORT_HCCL_NODE_LEVEL, it.first, it.second.c_str());
1252 6 : if (ret != 0) {
1253 0 : THROW<InternalException>("Call dlMsprofRegTypeInfo failed, return[%d]", ret);
1254 : }
1255 : }
1256 2 : std::lock_guard<std::mutex> lock(cacheTaskInfosMutex_);
1257 2 : if (cacheTaskInfos_.empty()) {
1258 0 : HCCL_INFO("[ProfilingHandler]StartL2Subscribe cacheTaskInfos_ is empty.");
1259 0 : return;
1260 : }
1261 4 : for (auto& taskInfo : cacheTaskInfos_) {
1262 2 : ReportCcuInfo(taskInfo);
1263 : }
1264 4 : }
1265 :
1266 1 : void ProfilingHandler::ProfilingHandler::StopSubscribe()
1267 : {
1268 1 : enableHostApi_ = false;
1269 1 : enableHcclNode_ = false;
1270 1 : enableHcclL0_ = false;
1271 1 : enableHcclL1_ = false;
1272 3 : HCCL_RUN_INFO("[ProfilingHandler]StopSubscribe.");
1273 1 : }
1274 :
1275 53 : bool ProfilingHandler::GetHostApiState() const { return enableHostApi_; }
1276 53 : bool ProfilingHandler::GetHcclNodeState() const { return enableHcclNode_; }
1277 55 : bool ProfilingHandler::GetHcclL0State() const { return enableHcclL0_; }
1278 :
1279 64 : bool ProfilingHandler::GetHcclL1State() const { return enableHcclL1_; }
1280 :
1281 181 : uint64_t ProfilingHandler::GetProfHashId(const char* name, uint32_t len) const
1282 : {
1283 181 : if (name == nullptr || len == 0) {
1284 78 : HCCL_WARNING("HashData is empty. name:%s, len:%u", name, len);
1285 26 : return INVALID_U64;
1286 : }
1287 155 : if (DlProfFunction::GetInstance().dlMsprofStr2Id == nullptr) {
1288 0 : HCCL_WARNING("HashFunc is nullPtr");
1289 0 : return INVALID_U64;
1290 : }
1291 155 : return DlProfFunction::GetInstance().dlMsprofStr2Id(name, len);
1292 : }
1293 :
1294 2 : void ProfilingHandler::ReportHcclMC2CommInfoLog(
1295 : [[maybe_unused]] const Stream& kfcStream, const Stream& stream, const std::vector<Stream*>& aicpuStreams,
1296 : [[maybe_unused]] const std::string& id, [[maybe_unused]] RankId myRank, [[maybe_unused]] u32 rankSize,
1297 : [[maybe_unused]] RankId rankInParentComm) const
1298 : {
1299 2 : if (LIKELY(HcclCheckLogLevel(HCCL_LOG_INFO) == 0)) {
1300 0 : return;
1301 : }
1302 2 : if (aicpuStreams.empty()) {
1303 3 : HCCL_INFO("only exist main stream, streamId(sqId):%u", stream.GetSqId());
1304 1 : return;
1305 : }
1306 1 : uint32_t reportId = 0;
1307 2 : for (uint32_t streamIndex = 0; streamIndex < aicpuStreams.size(); streamIndex++) {
1308 3 : HCCL_INFO(
1309 : "streamIndex:%u, reportId:%u, streamId(sqId):%u", streamIndex, reportId,
1310 : aicpuStreams[streamIndex]->GetSqId());
1311 1 : reportId++;
1312 : }
1313 : }
1314 :
1315 2 : void ProfilingHandler::ReportHcclMC2CommInfo(
1316 : const Stream& kfcStream, const Stream& stream, const std::vector<Stream*>& aicpuStreams, const std::string& id,
1317 : RankId myRank, u32 rankSize, RankId rankInParentComm)
1318 : {
1319 2 : ReportHcclMC2CommInfoLog(kfcStream, stream, aicpuStreams, id, myRank, rankSize, rankInParentComm);
1320 : ProfilingDeviceCommResInfo hcclMc2Info;
1321 2 : hcclMc2Info.groupName = GetProfHashId(id.c_str(), id.length());
1322 2 : hcclMc2Info.rankSize = rankSize;
1323 2 : hcclMc2Info.rankId = myRank;
1324 2 : hcclMc2Info.usrRankId = rankInParentComm;
1325 2 : hcclMc2Info.aicpuKfcStreamId = static_cast<uint32_t>(kfcStream.GetSqId());
1326 2 : hcclMc2Info.reserve = 0;
1327 2 : const uint32_t ONCE_REPORT_STREAM_NUM_MAX = 8;
1328 3 : for (uint32_t streamIndex = 0, reportId = 0; streamIndex < aicpuStreams.size(); streamIndex++) {
1329 1 : hcclMc2Info.commStreamIds[reportId++] = aicpuStreams[streamIndex]->GetSqId();
1330 1 : if (reportId == ONCE_REPORT_STREAM_NUM_MAX) {
1331 0 : hcclMc2Info.commStreamSize = reportId;
1332 0 : ReportMc2AdditionInfo(
1333 0 : DlProfFunction::GetInstance().dlMsprofSysCycleTime(), &hcclMc2Info, sizeof(hcclMc2Info));
1334 0 : reportId = 0;
1335 : }
1336 1 : if (streamIndex == (aicpuStreams.size() - 1)) {
1337 1 : hcclMc2Info.commStreamIds[reportId++] = stream.GetSqId();
1338 1 : hcclMc2Info.commStreamSize = reportId;
1339 1 : ReportMc2AdditionInfo(
1340 1 : DlProfFunction::GetInstance().dlMsprofSysCycleTime(), &hcclMc2Info, sizeof(hcclMc2Info));
1341 1 : reportId = 0;
1342 : }
1343 : }
1344 2 : if (aicpuStreams.empty()) {
1345 1 : hcclMc2Info.commStreamIds[0] = stream.GetSqId();
1346 1 : hcclMc2Info.commStreamSize = 1;
1347 1 : ReportMc2AdditionInfo(DlProfFunction::GetInstance().dlMsprofSysCycleTime(), &hcclMc2Info, sizeof(hcclMc2Info));
1348 : }
1349 2 : }
1350 :
1351 1 : void ProfilingHandler::ReportHcclMC2CommInfoLog(
1352 : [[maybe_unused]] const u32 kfcStreamId, const std::vector<u32>& aicpuStreamsId, const std::string& id,
1353 : [[maybe_unused]] RankId myRank, [[maybe_unused]] u32 rankSize, [[maybe_unused]] RankId rankInParentComm) const
1354 : {
1355 1 : if (LIKELY(HcclCheckLogLevel(HCCL_LOG_INFO) == 0)) {
1356 0 : return;
1357 : }
1358 1 : uint64_t groupName = GetProfHashId(id.c_str(), id.length());
1359 1 : uint32_t reportId = 0;
1360 4 : for (uint32_t streamIndex = 0; streamIndex < aicpuStreamsId.size(); streamIndex++) {
1361 9 : HCCL_INFO(
1362 : "streamIndex:[%u], reportId:[%u], streamId:[%u] id [%s] hcclMC2Info.groupName:[%llu]", streamIndex,
1363 : reportId, aicpuStreamsId[streamIndex], id.c_str(), groupName);
1364 3 : reportId++;
1365 : }
1366 : }
1367 :
1368 0 : void ProfilingHandler::ReportHcclMC2CommInfo(
1369 : const u32 kfcStreamId, const std::vector<u32>& aicpuStreamsId, const std::string& id, RankId myRank, u32 rankSize,
1370 : RankId rankInParentComm)
1371 : {
1372 0 : ReportHcclMC2CommInfoLog(kfcStreamId, aicpuStreamsId, id, myRank, rankSize, rankInParentComm);
1373 : ProfilingDeviceCommResInfo hcclMc2Info;
1374 0 : hcclMc2Info.groupName = GetProfHashId(id.c_str(), id.length());
1375 0 : hcclMc2Info.rankSize = rankSize;
1376 0 : hcclMc2Info.rankId = myRank;
1377 0 : hcclMc2Info.usrRankId = rankInParentComm;
1378 0 : hcclMc2Info.aicpuKfcStreamId = static_cast<uint32_t>(kfcStreamId);
1379 0 : hcclMc2Info.reserve = 0;
1380 :
1381 0 : const uint32_t ONCE_REPORT_STREAM_NUM_MAX = 8;
1382 0 : uint32_t reportId = 0;
1383 0 : for (uint32_t streamIndex = 0; streamIndex < aicpuStreamsId.size(); streamIndex++) {
1384 0 : hcclMc2Info.commStreamIds[reportId++] = aicpuStreamsId[streamIndex];
1385 0 : if (reportId == ONCE_REPORT_STREAM_NUM_MAX) {
1386 0 : hcclMc2Info.commStreamSize = reportId;
1387 0 : ReportMc2AdditionInfo(
1388 0 : DlProfFunction::GetInstance().dlMsprofSysCycleTime(), &hcclMc2Info, sizeof(hcclMc2Info));
1389 0 : reportId = 0;
1390 : }
1391 : }
1392 0 : if (reportId > 0) {
1393 0 : hcclMc2Info.commStreamSize = reportId;
1394 0 : ReportMc2AdditionInfo(DlProfFunction::GetInstance().dlMsprofSysCycleTime(), &hcclMc2Info, sizeof(hcclMc2Info));
1395 0 : reportId = 0;
1396 : }
1397 0 : }
1398 2 : void ProfilingHandler::ReportMc2AdditionInfo(uint64_t timeStamp, const void* data, int len)
1399 : {
1400 2 : MsprofAdditionalInfo reporterData{};
1401 2 : reporterData.level = MSPROF_REPORT_NODE_LEVEL;
1402 2 : reporterData.type = MSPROF_REPORT_NODE_MC2_COMMINFO_TYPE;
1403 2 : reporterData.threadId = SalGetTid();
1404 2 : reporterData.dataLen = len;
1405 2 : reporterData.timeStamp = timeStamp;
1406 2 : s32 sret = memcpy_s(reporterData.data, sizeof(reporterData.data), data, len);
1407 2 : if (sret != EOK) {
1408 0 : THROW<InternalException>("Call memcpy_s failed, errorno[%d]", sret);
1409 : }
1410 6 : HCCL_INFO(
1411 : "[ProfilingHandler][ReportMc2CommInfo], level [%u], type[%u], threadId[%u], dataLen[%u], timeStamp[%llu]",
1412 : reporterData.level, reporterData.type, reporterData.threadId, reporterData.dataLen, reporterData.timeStamp);
1413 2 : if (!enableHostApi_) {
1414 2 : std::lock_guard<std::mutex> lock(cacheHcclAdditionInfoMutex_);
1415 2 : cacheHcclAdditionInfo_.push(reporterData);
1416 2 : return;
1417 2 : }
1418 : s32 ret
1419 0 : = DlProfFunction::GetInstance().dlMsprofReportAdditionalInfo(1, &reporterData, sizeof(MsprofAdditionalInfo));
1420 0 : HCCL_INFO("Call MsprofReportAdditionalInfo, return value[%d]", ret);
1421 0 : if (ret != 0) {
1422 0 : THROW<InternalException>("Call MsprofReportAdditionalInfo failed, return[%d]", ret);
1423 : }
1424 : }
1425 :
1426 : } // namespace Hccl
|
