Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 : #include <cstring>
11 : #include <algorithm>
12 : #include "profiling_handler.h"
13 : #include "dlprof_function.h"
14 : #include "exception_util.h"
15 : #include "internal_exception.h"
16 : #include "sal.h"
17 : #include "task_param.h"
18 : #include "aprof_pub.h"
19 : #include "orion_adapter_rts.h"
20 : #include "communicator_impl.h"
21 : #include "data_type.h"
22 : namespace Hccl {
23 : #define UNUSED(x) (void)(x)
24 :
25 : constexpr uint16_t CCU_TYPE = 2;
26 : constexpr uint32_t DPU_DEV_ID_MASK = (1U << 12);
27 :
28 : ProfilingHandler ProfilingHandler::instance_;
29 :
30 1 : ProfilingHandler::ProfilingHandler()
31 : {
32 1 : }
33 :
34 1 : ProfilingHandler::~ProfilingHandler()
35 : {
36 1 : }
37 :
38 764 : ProfilingHandler &ProfilingHandler::GetInstance()
39 : {
40 764 : return instance_;
41 : }
42 :
43 1 : void ProfilingHandler::InitLog() const
44 : {
45 1 : if (LIKELY(HcclCheckLogLevel(HCCL_LOG_INFO) == 0)) {
46 0 : return;
47 : }
48 24 : for (auto i = 0; i < TaskParamType::__COUNT__; ++i) {
49 23 : TaskParamType type(static_cast<TaskParamType::Value>(i));
50 23 : std::string nameInfo = type.Describe();
51 23 : uint64_t hashId = GetProfHashId(nameInfo.c_str(), nameInfo.length());
52 69 : HCCL_INFO("[TaskParamType] nameInfo[%s] ret[%llu]", nameInfo.c_str(), hashId);
53 23 : }
54 : }
55 :
56 257 : HcclResult ProfilingHandler::Init()
57 : {
58 257 : if (initializedFlag_) {
59 256 : return HCCL_SUCCESS;
60 : }
61 1 : InitLog();
62 1 : if (Hccl::DlProfFunction::GetInstance().DlProfFunctionInit() != HCCL_SUCCESS) {
63 0 : HCCL_ERROR("[ProfilingHandler][Init] DlProfFunctionInit failed.");
64 0 : return HCCL_E_INTERNAL;
65 : }
66 1 : ProfCommandHandle callback = CommandHandleWrapper;
67 1 : auto ret = DlProfFunction::GetInstance().dlMsprofRegisterCallback(HCCL, callback);
68 1 : if (ret != 0) {
69 0 : HCCL_ERROR("[ProfilingHandler][Init]errNo[0x%016llx] Prof Register CtrlCallback fail, return[%d]",
70 : HCCL_ERROR_CODE(HCCL_E_RUNTIME), ret);
71 0 : return HCCL_E_RUNTIME;
72 : }
73 24 : for (auto i = 0; i < TaskParamType::__COUNT__; ++i) {
74 23 : TaskParamType type(static_cast<TaskParamType::Value>(i));
75 23 : std::string nameInfo = type.Describe();
76 23 : uint64_t hashId = GetProfHashId(nameInfo.c_str(), nameInfo.length());
77 23 : str2HashId_[nameInfo] = hashId;
78 23 : }
79 1 : SetCachedCclTag();
80 1 : cachedAlgTypeHashId_ = GetProfHashId("AlgType::NHR", strlen("AlgType::NHR"));
81 1 : initializedFlag_ = true;
82 1 : return HCCL_SUCCESS;
83 : }
84 :
85 0 : int32_t ProfilingHandler::CommandHandleWrapper(uint32_t rtType, void *data, uint32_t len)
86 : {
87 0 : return instance_.CommandHandle(rtType, data, len);
88 : }
89 :
90 0 : void ProfilingHandler::ReportKernel() const
91 : {
92 0 : }
93 :
94 8 : void ProfilingHandler::ReportHostApi(OpType opType, uint64_t beginTime, uint64_t endTime, bool cachedReq, bool isAiCpu)
95 : {
96 8 : uint32_t threadId = SalGetTid();
97 8 : std::string profName(GetProfOpName(opType));
98 8 : if (isAiCpu) {
99 1 : profName += "AicpuKernel";
100 : }
101 8 : uint64_t cmdItemId = DlProfFunction::GetInstance().dlMsprofStr2Id(profName.c_str(), profName.length());
102 8 : if (enableHostApi_) {
103 0 : ReportAclApi(opType, beginTime, endTime, cmdItemId, threadId);
104 : }
105 8 : ReportNodeApi(beginTime, endTime, cmdItemId, threadId, cachedReq);
106 8 : ReportNodeBasicInfo(endTime, cmdItemId, threadId, cachedReq);
107 8 : }
108 :
109 8 : void ProfilingHandler::ReportHcclOp(const DfxOpInfo &opInfo, bool cachedReq)
110 : {
111 8 : uint32_t threadId = SalGetTid();
112 8 : ReportHcclOpInfo(opInfo.endTime_, opInfo, threadId, cachedReq);
113 8 : }
114 :
115 5 : void ProfilingHandler::ReportHcclTaskApi(TaskParamType taskType, uint64_t beginTime, uint64_t endTime, bool isMasterStream, bool cachedReq, bool ignoreLevel)
116 : {
117 5 : MsprofApi reporterData{};
118 5 : reporterData.level = MSPROF_REPORT_HCCL_NODE_LEVEL;
119 5 : reporterData.type = isMasterStream ? MSPROF_REPORT_HCCL_MASTER_TYPE : MSPROF_REPORT_HCCL_SLAVE_TYPE;
120 5 : reporterData.threadId = SalGetTid();
121 5 : reporterData.beginTime = beginTime;
122 5 : reporterData.endTime = endTime;
123 5 : const std::string proName(GetProfTaskOpNameV2(taskType));
124 5 : reporterData.itemId = GetProfHashId(proName.c_str(), proName.length());
125 5 : if (taskType == TaskParamType::TASK_AICPU_KERNEL) {
126 0 : return;
127 : }
128 5 : if (cachedReq) {
129 5 : std::lock_guard<std::mutex> lock(cachedTaskApiInfoMutex_);
130 5 : cachedTaskApiInfo_.push(reporterData);
131 5 : }
132 5 : if ((!enableHcclNode_) || (!ignoreLevel && !enableHcclL1_)) {
133 3 : return;
134 : }
135 6 : HCCL_INFO("ReportHcclTaskApi, enableHcclNode_[%d],ignoreLevel[%d],enableHcclL1_[%d],taskType[%s], beginTime[%llu], endTime[%llu], isMasterStream[%d], cachedReq[%d]",
136 : enableHcclNode_, ignoreLevel, enableHcclL1_, proName.c_str(), beginTime, endTime, isMasterStream, cachedReq);
137 2 : s32 ret = DlProfFunction::GetInstance().dlMsprofReportApi(1, &reporterData);
138 2 : if (ret != 0) {
139 0 : THROW<InternalException>("Call MsprofReportApi fail, return[%d]", ret);
140 : }
141 5 : }
142 :
143 1 : void ProfilingHandler::SetCachedCclTag()
144 : {
145 20 : for (const auto &item : CMD_OP_TYPE_INFO_MAP) {
146 19 : uint32_t tag = static_cast<uint32_t>(item.first);
147 19 : if (cachedNewCclTag_.find(tag) == cachedNewCclTag_.end()) {
148 15 : const std::string &cclTag = item.second.second;
149 15 : auto hashId = GetProfHashId(cclTag.c_str(), cclTag.length());
150 15 : cachedNewCclTag_[item.second.first] = hashId;
151 : }
152 : }
153 1 : }
154 :
155 6 : uint32_t ProfilingHandler::GetTaskTypeValue(TaskParamType taskType) const
156 : {
157 6 : switch (taskType) {
158 0 : case TaskParamType::TASK_DPU_INLINE_WRITE:
159 : case TaskParamType::TASK_DPU_NOTIFY_WAIT:
160 : case TaskParamType::TASK_DPU_WRITE_WITH_NOTIFY:
161 : case TaskParamType::TASK_DPU_CHANNEL_FENCE:
162 0 : return static_cast<uint32_t>(ProfTaskType::TASK_DPU_HCCL_INFO);
163 6 : default:
164 6 : return static_cast<uint32_t>(ProfTaskType::TASK_HCCL_INFO);
165 : }
166 : }
167 :
168 7 : void ProfilingHandler::FillTaskAdditionInfo(const TaskInfo &taskInfo, MsprofAdditionalInfo &reporterData) const
169 : {
170 7 : reporterData.level = MSPROF_REPORT_HCCL_NODE_LEVEL;
171 7 : reporterData.threadId = SalGetTid();
172 7 : FillProfCommonInfo(taskInfo, reporterData);
173 :
174 7 : switch (taskInfo.taskParam_.taskType) {
175 0 : case TaskParamType::TASK_DPU_INLINE_WRITE:
176 : case TaskParamType::TASK_DPU_NOTIFY_WAIT:
177 : case TaskParamType::TASK_DPU_WRITE_WITH_NOTIFY:
178 : case TaskParamType::TASK_DPU_CHANNEL_FENCE:
179 0 : reporterData.type = static_cast<uint32_t>(ProfTaskType::TASK_DPU_HCCL_INFO);
180 0 : FillDpuProfInfo(taskInfo, reporterData);
181 0 : break;
182 7 : default:
183 7 : reporterData.type = static_cast<uint32_t>(ProfTaskType::TASK_HCCL_INFO);
184 7 : FillProfTaskSpecificInfo(taskInfo, reinterpret_cast<MsprofHcclInfo *>(reporterData.data));
185 7 : break;
186 : }
187 7 : }
188 :
189 3 : void ProfilingHandler::ReportHcclTaskDetails(const TaskInfo &taskInfo, bool cachedReq)
190 : {
191 3 : if (enableHcclL1_ == false && !cachedReq) {
192 2 : return;
193 : }
194 3 : if (taskInfo.dfxOpInfo_ == nullptr) {
195 3 : HCCL_WARNING("[%s] dfxOpInfo_ is nullptr, skip ReportHcclTaskDetails!", __func__);
196 1 : return;
197 : }
198 2 : if (taskInfo.dfxOpInfo_->comm_ == nullptr) {
199 3 : HCCL_WARNING("[%s] comm_ is nullptr, skip ReportHcclTaskDetails!", __func__);
200 1 : return;
201 : }
202 1 : if (cachedReq) {
203 1 : std::lock_guard<std::mutex> lock(cacheTaskInfosMutex_);
204 1 : cacheTaskInfos_.push_back(taskInfo);
205 1 : }
206 1 : if (enableHcclL1_ == false) {
207 0 : return;
208 : }
209 1 : MsprofAdditionalInfo reporterData{};
210 1 : FillTaskAdditionInfo(taskInfo, reporterData);
211 1 : DumpHCCLReportData(taskInfo, reporterData);
212 1 : CallAdditionInfo(reporterData);
213 : }
214 :
215 2 : void ProfilingHandler::ReportHcclTaskDetailsBatchLog(const std::vector<TaskInfo*> &taskInfos) const
216 : {
217 2 : if (LIKELY(HcclCheckLogLevel(HCCL_LOG_INFO) == 0)) {
218 0 : return;
219 : }
220 6 : for (const auto &taskInfo : taskInfos) {
221 12 : HCCL_INFO("ReportHcclTaskDetailsBatchLog, taskType[%u]", GetTaskTypeValue(taskInfo->taskParam_.taskType));
222 : }
223 : }
224 :
225 7 : void ProfilingHandler::ReportHcclTaskDetailsBatch(const std::vector<TaskInfo*> &taskInfos, bool cachedReq)
226 : {
227 7 : if (taskInfos.empty()) {
228 3 : HCCL_WARNING("[ProfilingHandler::ReportHcclTaskDetailsBatch] taskInfos is empty!");
229 1 : return;
230 : }
231 6 : if (taskInfos[0]->dfxOpInfo_ == nullptr) {
232 3 : HCCL_WARNING("[ProfilingHandler::ReportHcclTaskDetailsBatch] taskInfo.dfxOpInfo_ is nullptr, skip!");
233 1 : return;
234 : }
235 5 : if (taskInfos[0]->dfxOpInfo_->comm_ == nullptr) {
236 3 : HCCL_WARNING("[ProfilingHandler::ReportHcclTaskDetailsBatch] taskInfo.dfxOpInfo_->comm_ is nullptr, skip!");
237 1 : return;
238 : }
239 4 : if (enableHcclL1_ == false && !cachedReq) {
240 1 : return;
241 : }
242 3 : if (cachedReq) {
243 3 : std::lock_guard<std::mutex> lock(cacheTaskInfosMutex_);
244 8 : for (const auto &taskInfo : taskInfos) {
245 5 : cacheTaskInfos_.push_back(*taskInfo);
246 : }
247 3 : }
248 3 : if (enableHcclL1_ == false) {
249 1 : return;
250 : }
251 2 : ReportHcclTaskDetailsBatchLog(taskInfos);
252 6 : for (const auto &taskInfo : taskInfos) {
253 4 : MsprofAdditionalInfo reporterData{};
254 4 : FillTaskAdditionInfo(*taskInfo, reporterData);
255 4 : CallAdditionInfo(reporterData);
256 : }
257 : }
258 :
259 5 : void ProfilingHandler::CallAdditionInfo(MsprofAdditionalInfo &reporterData) const
260 : {
261 5 : s32 ret = DlProfFunction::GetInstance().dlMsprofReportAdditionalInfo(
262 : 1, &reporterData, sizeof(MsprofAdditionalInfo));
263 5 : if (ret != 0) {
264 0 : THROW<InternalException>("Call MsprofReportAdditionalInfo failed, return[%d]", ret);
265 : }
266 5 : }
267 :
268 7 : void ProfilingHandler::FillProfCommonInfo(const TaskInfo &taskInfo, MsprofAdditionalInfo &reporterData) const
269 : {
270 7 : if (taskInfo.dfxOpInfo_ == nullptr) {
271 0 : HCCL_WARNING("[ProfilingHandler][%s]taskInfo.dfxOpInfo_ is nullptr, skip GetProfCommonInfo.", __func__);
272 0 : return;
273 : }
274 7 : reporterData.timeStamp = taskInfo.taskParam_.endTime;
275 7 : reporterData.dataLen = sizeof(MsprofHcclInfo);
276 7 : auto *profInfo = reinterpret_cast<MsprofHcclInfo *>(reporterData.data);
277 7 : *profInfo = MsprofHcclInfo();
278 :
279 7 : const auto &profName = GetProfTaskOpNameV2(taskInfo.taskParam_.taskType);
280 7 : profInfo->itemId = GetProfHashId(profName.c_str(), profName.length());
281 7 : auto newCclTagIt = cachedNewCclTag_.find(taskInfo.dfxOpInfo_->op_.opType);
282 7 : profInfo->cclTag = (newCclTagIt != cachedNewCclTag_.end()) ? newCclTagIt->second : INVALID_U64;
283 7 : const auto &opTag = taskInfo.dfxOpInfo_->op_.opTag;
284 7 : uint64_t groupName = GetProfHashId(opTag.c_str(), opTag.length());
285 7 : if (taskInfo.dfxOpInfo_->comm_ == nullptr) {
286 0 : HCCL_WARNING("[ProfilingHandler][%s]taskInfo.dfxOpInfo_->comm_ is nullptr, skip GetProfCommonInfo.", __func__);
287 0 : return;
288 : }
289 7 : if (taskInfo.dfxOpInfo_->isIndop_ == true) {
290 0 : profInfo->groupName = groupName;
291 0 : profInfo->rankSize = taskInfo.dfxOpInfo_->rankSize_;
292 : } else {
293 7 : CommunicatorImpl *commImp = static_cast<CommunicatorImpl *>(taskInfo.dfxOpInfo_->comm_);
294 7 : if (commImp == nullptr) {
295 0 : HCCL_WARNING("[ProfilingHandler][%s]commImp is nullptr, skip GetProfCommonInfo.", __func__);
296 0 : return;
297 : }
298 7 : profInfo->groupName = groupName;
299 7 : profInfo->rankSize = commImp->GetRankSize();
300 : }
301 7 : profInfo->workFlowMode = static_cast<u32>(HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE);
302 7 : profInfo->planeID = 0;
303 7 : profInfo->stage = 0;
304 7 : profInfo->role = static_cast<uint32_t>(TaskRole::DST);
305 7 : profInfo->durationEstimated = 0;
306 7 : profInfo->localRank = taskInfo.dfxOpInfo_->op_.myRank;
307 7 : profInfo->remoteRank = taskInfo.remoteRank_;
308 7 : profInfo->dataType = taskInfo.dfxOpInfo_->op_.dataType;
309 7 : profInfo->opType = taskInfo.dfxOpInfo_->op_.opType;
310 7 : profInfo->transportType = static_cast<int32_t>(SimpleTaskType::UB);
311 7 : if (profInfo->remoteRank == INVALID_VALUE_RANKID) {
312 0 : profInfo->transportType = static_cast<int32_t>(SimpleTaskType::LOCAL);
313 0 : profInfo->remoteRank = profInfo->localRank;
314 0 : profInfo->linkType = 0;
315 : }
316 7 : }
317 :
318 7 : void ProfilingHandler::FillProfTaskSpecificInfo(const TaskInfo &taskInfo, MsprofHcclInfo *profInfo) const
319 : {
320 7 : const auto &taskType = taskInfo.taskParam_.taskType;
321 7 : const auto &taskPara = taskInfo.taskParam_.taskPara;
322 7 : switch (taskType) {
323 0 : case TaskParamType::TASK_SDMA:
324 : case TaskParamType::TASK_RDMA:
325 0 : profInfo->srcAddr = static_cast<u64>(reinterpret_cast<uintptr_t>(taskPara.DMA.src));
326 0 : profInfo->dstAddr = static_cast<u64>(reinterpret_cast<uintptr_t>(taskPara.DMA.dst));
327 0 : profInfo->dataSize = static_cast<u32>(taskPara.DMA.size);
328 0 : profInfo->notifyID = taskPara.DMA.notifyID;
329 0 : profInfo->linkType = static_cast<uint16_t>(taskPara.DMA.linkType);
330 0 : break;
331 0 : case TaskParamType::TASK_REDUCE_INLINE:
332 : case TaskParamType::TASK_REDUCE_TBE:
333 0 : profInfo->srcAddr = static_cast<u64>(reinterpret_cast<uintptr_t>(taskPara.Reduce.src));
334 0 : profInfo->dstAddr = static_cast<u64>(reinterpret_cast<uintptr_t>(taskPara.Reduce.dst));
335 0 : profInfo->dataSize = static_cast<u32>(taskPara.Reduce.size);
336 0 : profInfo->notifyID = taskPara.Reduce.notifyID;
337 0 : profInfo->linkType = static_cast<uint16_t>(taskPara.Reduce.linkType);
338 0 : break;
339 7 : case TaskParamType::TASK_NOTIFY_RECORD:
340 : case TaskParamType::TASK_NOTIFY_WAIT:
341 7 : profInfo->notifyID = taskPara.Notify.notifyID;
342 7 : break;
343 0 : case TaskParamType::TASK_CCU:
344 0 : HCCL_INFO("current taskType is TASK_CCU");
345 0 : ReportCcuInfo(taskInfo);
346 0 : break;
347 0 : default:
348 0 : break;
349 : }
350 7 : }
351 :
352 0 : void ProfilingHandler::ConvertHcclInfoToDpuTrack(MsprofAdditionalInfo &reporterData) const
353 : {
354 0 : auto *profInfo = reinterpret_cast<MsprofHcclInfo *>(reporterData.data);
355 0 : uint64_t itemId = profInfo->itemId;
356 0 : uint64_t cclTag = profInfo->cclTag;
357 0 : uint64_t groupName = profInfo->groupName;
358 0 : uint32_t localRank = profInfo->localRank;
359 0 : uint32_t remoteRank = profInfo->remoteRank;
360 0 : uint32_t rankSize = profInfo->rankSize;
361 0 : uint32_t workFlowMode = profInfo->workFlowMode;
362 0 : uint16_t planeID = static_cast<uint16_t>(profInfo->planeID);
363 0 : uint32_t stage = profInfo->stage;
364 0 : uint32_t role = profInfo->role;
365 0 : double durationEstimated = profInfo->durationEstimated;
366 :
367 0 : reporterData.dataLen = sizeof(MsprofDpuHcclTrack);
368 0 : auto *dpuProfInfo = reinterpret_cast<MsprofDpuHcclTrack *>(reporterData.data);
369 0 : *dpuProfInfo = MsprofDpuHcclTrack();
370 :
371 0 : dpuProfInfo->itemId = itemId;
372 0 : dpuProfInfo->cclTag = cclTag;
373 0 : dpuProfInfo->groupName = groupName;
374 0 : dpuProfInfo->localRank = localRank;
375 0 : dpuProfInfo->remoteRank = remoteRank;
376 0 : dpuProfInfo->rankSize = rankSize;
377 0 : dpuProfInfo->workFlowMode = static_cast<uint8_t>(workFlowMode);
378 0 : dpuProfInfo->planeID = planeID;
379 0 : dpuProfInfo->stage = stage;
380 0 : dpuProfInfo->role = static_cast<uint8_t>(role);
381 0 : dpuProfInfo->durationEstimated = durationEstimated;
382 0 : }
383 :
384 0 : void ProfilingHandler::FillDpuTaskParaDetails(const TaskInfo &taskInfo, MsprofDpuHcclTrack *dpuProfInfo) const
385 : {
386 0 : const auto &taskPara = taskInfo.taskParam_.taskPara;
387 0 : switch (taskInfo.taskParam_.taskType) {
388 0 : case TaskParamType::TASK_DPU_INLINE_WRITE:
389 : case TaskParamType::TASK_DPU_WRITE_WITH_NOTIFY:
390 0 : dpuProfInfo->srcAddr = static_cast<u64>(reinterpret_cast<uintptr_t>(taskPara.DMA.src));
391 0 : dpuProfInfo->dstAddr = static_cast<u64>(reinterpret_cast<uintptr_t>(taskPara.DMA.dst));
392 0 : dpuProfInfo->dataSize = static_cast<u32>(taskPara.DMA.size);
393 0 : dpuProfInfo->notifyID = taskPara.DMA.notifyID;
394 0 : break;
395 0 : case TaskParamType::TASK_DPU_NOTIFY_WAIT:
396 : case TaskParamType::TASK_DPU_CHANNEL_FENCE:
397 0 : dpuProfInfo->notifyID = taskPara.Notify.notifyID;
398 0 : break;
399 0 : default:
400 0 : break;
401 : }
402 0 : }
403 :
404 0 : void ProfilingHandler::FillDpuProfInfo(const TaskInfo &taskInfo, MsprofAdditionalInfo &reporterData) const
405 : {
406 0 : if (taskInfo.dfxOpInfo_ == nullptr) {
407 0 : HCCL_WARNING("[ProfilingHandler::FillDpuProfInfo] taskInfo.dfxOpInfo_ is nullptr, skip GetDpuProfInfo!");
408 0 : return;
409 : }
410 0 : ConvertHcclInfoToDpuTrack(reporterData);
411 0 : auto *dpuProfInfo = reinterpret_cast<MsprofDpuHcclTrack *>(reporterData.data);
412 0 : dpuProfInfo->dataType = static_cast<uint32_t>(taskInfo.dfxOpInfo_->op_.dataType);
413 0 : dpuProfInfo->opType = static_cast<uint32_t>(taskInfo.dfxOpInfo_->op_.opType);
414 0 : dpuProfInfo->transportType = static_cast<uint32_t>(SimpleTaskType::ROCE);
415 0 : dpuProfInfo->aicpu_task_id = taskInfo.taskParam_.aicpuTaskId;
416 0 : dpuProfInfo->npuDevId = taskInfo.taskParam_.npuDevId;
417 0 : dpuProfInfo->dpuDevId = DPU_DEV_ID_MASK;
418 0 : const auto &taskPara = taskInfo.taskParam_.taskPara;
419 0 : dpuProfInfo->linkType = static_cast<uint16_t>(taskPara.DMA.linkType);
420 0 : if (dpuProfInfo->remoteRank == INVALID_VALUE_RANKID) {
421 0 : dpuProfInfo->transportType = static_cast<uint32_t>(SimpleTaskType::LOCAL);
422 0 : dpuProfInfo->remoteRank = dpuProfInfo->localRank;
423 0 : dpuProfInfo->linkType = 0;
424 : }
425 0 : dpuProfInfo->taskId = taskInfo.taskId_;
426 0 : dpuProfInfo->streamId = taskInfo.streamId_;
427 0 : dpuProfInfo->timeStamp = taskInfo.taskParam_.beginTime;
428 0 : HCCL_INFO("[FillDpuProfInfo]taskId[%u], streamId[%u], npuDevId[%u], dpuDevId[%u], "
429 : "starttime[%llu], endtime[%llu], aicputaskId[%u].",
430 : dpuProfInfo->taskId, dpuProfInfo->streamId,
431 : dpuProfInfo->npuDevId, dpuProfInfo->dpuDevId,
432 : dpuProfInfo->timeStamp, reporterData.timeStamp, dpuProfInfo->aicpu_task_id);
433 0 : FillDpuTaskParaDetails(taskInfo, dpuProfInfo);
434 : }
435 :
436 9 : void ProfilingHandler::GetHCCLReportData(const TaskInfo &taskInfo, HCCLReportData &hcclReportData) const
437 : {
438 9 : if (taskInfo.dfxOpInfo_ == nullptr) {
439 3 : HCCL_ERROR("[ProfilingHandler]taskInfo.dfxOpInfo_ is nullptr");
440 1 : return;
441 : }
442 8 : hcclReportData.ts = taskInfo.taskParam_.endTime;
443 8 : const auto &profName = GetProfTaskOpNameV2(taskInfo.taskParam_.taskType);
444 8 : hcclReportData.profInfo.itemId = GetProfHashId(profName.c_str(), profName.length());
445 8 : auto newCclTagIt = cachedNewCclTag_.find(taskInfo.dfxOpInfo_->op_.opType);
446 8 : hcclReportData.profInfo.cclTag = (newCclTagIt != cachedNewCclTag_.end()) ? newCclTagIt->second : INVALID_U64;
447 8 : const auto &opTag = taskInfo.dfxOpInfo_->op_.opTag;
448 8 : uint64_t groupName = GetProfHashId(opTag.c_str(), opTag.length());
449 8 : if (taskInfo.dfxOpInfo_->comm_ == nullptr) {
450 0 : HCCL_ERROR("[ProfilingHandler]taskInfo.dfxOpInfo_->comm_ is nullptr");
451 0 : return;
452 : }
453 8 : if (taskInfo.dfxOpInfo_->isIndop_ == true) {
454 0 : hcclReportData.profInfo.groupName = groupName;
455 0 : hcclReportData.profInfo.rankSize = taskInfo.dfxOpInfo_->rankSize_;
456 : } else {
457 8 : CommunicatorImpl *commImp = static_cast<CommunicatorImpl *>(taskInfo.dfxOpInfo_->comm_);
458 8 : if (commImp == nullptr) {
459 0 : HCCL_ERROR("[ProfilingHandler]commImp is nullptr");
460 0 : return;
461 : }
462 8 : hcclReportData.profInfo.groupName = groupName;
463 8 : hcclReportData.profInfo.rankSize = commImp->GetRankSize();
464 : }
465 8 : hcclReportData.profInfo.workFlowMode = static_cast<u32>(HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE);
466 8 : hcclReportData.profInfo.planeID = 0;
467 8 : hcclReportData.profInfo.stage = 0;
468 8 : hcclReportData.profInfo.role = static_cast<uint32_t>(TaskRole::DST);
469 8 : hcclReportData.profInfo.durationEstimated = 0;
470 8 : hcclReportData.profInfo.localRank = taskInfo.dfxOpInfo_->op_.myRank;
471 8 : hcclReportData.profInfo.remoteRank = taskInfo.remoteRank_;
472 8 : hcclReportData.profInfo.dataType = taskInfo.dfxOpInfo_->op_.dataType;
473 8 : hcclReportData.profInfo.opType = taskInfo.dfxOpInfo_->op_.opType;
474 8 : hcclReportData.profInfo.transportType = static_cast<int32_t>(SimpleTaskType::UB);
475 8 : if (hcclReportData.profInfo.remoteRank == INVALID_VALUE_RANKID) {
476 0 : hcclReportData.profInfo.transportType = static_cast<int32_t>(SimpleTaskType::LOCAL);
477 0 : hcclReportData.profInfo.remoteRank = hcclReportData.profInfo.localRank;
478 0 : hcclReportData.profInfo.linkType = 0;
479 : }
480 8 : const auto &taskType = taskInfo.taskParam_.taskType;
481 8 : const auto &taskPara = taskInfo.taskParam_.taskPara;
482 8 : switch (taskType) {
483 4 : case TaskParamType::TASK_DPU_INLINE_WRITE:
484 : case TaskParamType::TASK_DPU_NOTIFY_WAIT:
485 : case TaskParamType::TASK_DPU_WRITE_WITH_NOTIFY:
486 : case TaskParamType::TASK_DPU_CHANNEL_FENCE:
487 4 : hcclReportData.dpuProfInfo.itemId = hcclReportData.profInfo.itemId;
488 4 : hcclReportData.dpuProfInfo.cclTag = hcclReportData.profInfo.cclTag;
489 4 : hcclReportData.dpuProfInfo.groupName = hcclReportData.profInfo.groupName;
490 4 : hcclReportData.dpuProfInfo.localRank = hcclReportData.profInfo.localRank;
491 4 : hcclReportData.dpuProfInfo.remoteRank = hcclReportData.profInfo.remoteRank;
492 4 : hcclReportData.dpuProfInfo.rankSize = hcclReportData.profInfo.rankSize;
493 4 : hcclReportData.dpuProfInfo.workFlowMode = hcclReportData.profInfo.workFlowMode;
494 4 : hcclReportData.dpuProfInfo.planeID = hcclReportData.profInfo.planeID;
495 4 : hcclReportData.dpuProfInfo.stage = hcclReportData.profInfo.stage;
496 4 : hcclReportData.dpuProfInfo.role = hcclReportData.profInfo.role;
497 4 : hcclReportData.dpuProfInfo.durationEstimated = hcclReportData.profInfo.durationEstimated;
498 4 : hcclReportData.dpuProfInfo.dataType = static_cast<uint32_t>(taskInfo.dfxOpInfo_->op_.dataType);
499 4 : hcclReportData.dpuProfInfo.opType = static_cast<uint32_t>(taskInfo.dfxOpInfo_->op_.opType);
500 4 : hcclReportData.dpuProfInfo.transportType = static_cast<uint32_t>(SimpleTaskType::ROCE);
501 4 : hcclReportData.dpuProfInfo.aicpu_task_id = taskInfo.taskParam_.aicpuTaskId;
502 4 : hcclReportData.dpuProfInfo.npuDevId = taskInfo.taskParam_.npuDevId;
503 4 : hcclReportData.dpuProfInfo.dpuDevId = DPU_DEV_ID_MASK;
504 4 : hcclReportData.dpuProfInfo.linkType = static_cast<uint16_t>(taskPara.DMA.linkType);
505 4 : if (hcclReportData.dpuProfInfo.remoteRank == INVALID_VALUE_RANKID) {
506 0 : hcclReportData.dpuProfInfo.transportType = static_cast<uint32_t>(SimpleTaskType::LOCAL);
507 0 : hcclReportData.dpuProfInfo.remoteRank = hcclReportData.dpuProfInfo.localRank;
508 0 : hcclReportData.dpuProfInfo.linkType = 0;
509 : }
510 4 : hcclReportData.dpuProfInfo.taskId = taskInfo.taskId_;
511 4 : hcclReportData.dpuProfInfo.streamId = taskInfo.streamId_;
512 4 : hcclReportData.dpuProfInfo.timeStamp = taskInfo.taskParam_.beginTime;
513 4 : switch (taskType) {
514 2 : case TaskParamType::TASK_DPU_INLINE_WRITE:
515 : case TaskParamType::TASK_DPU_WRITE_WITH_NOTIFY:
516 2 : hcclReportData.dpuProfInfo.srcAddr = static_cast<u64>(reinterpret_cast<uintptr_t>(taskPara.DMA.src));
517 2 : hcclReportData.dpuProfInfo.dstAddr = static_cast<u64>(reinterpret_cast<uintptr_t>(taskPara.DMA.dst));
518 2 : hcclReportData.dpuProfInfo.dataSize = static_cast<u32>(taskPara.DMA.size);
519 2 : hcclReportData.dpuProfInfo.notifyID = taskPara.DMA.notifyID;
520 2 : break;
521 2 : case TaskParamType::TASK_DPU_NOTIFY_WAIT:
522 : case TaskParamType::TASK_DPU_CHANNEL_FENCE:
523 2 : hcclReportData.dpuProfInfo.notifyID = taskPara.Notify.notifyID;
524 2 : break;
525 0 : default:
526 0 : break;
527 : }
528 4 : break;
529 4 : default:
530 4 : switch (taskType) {
531 1 : case TaskParamType::TASK_SDMA:
532 : case TaskParamType::TASK_RDMA:
533 1 : hcclReportData.profInfo.srcAddr = static_cast<u64>(reinterpret_cast<uintptr_t>(taskPara.DMA.src));
534 1 : hcclReportData.profInfo.dstAddr = static_cast<u64>(reinterpret_cast<uintptr_t>(taskPara.DMA.dst));
535 1 : hcclReportData.profInfo.dataSize = static_cast<u32>(taskPara.DMA.size);
536 1 : hcclReportData.profInfo.notifyID = taskPara.DMA.notifyID;
537 1 : hcclReportData.profInfo.linkType = static_cast<uint16_t>(taskPara.DMA.linkType);
538 1 : break;
539 1 : case TaskParamType::TASK_REDUCE_INLINE:
540 : case TaskParamType::TASK_REDUCE_TBE:
541 1 : hcclReportData.profInfo.srcAddr = static_cast<u64>(reinterpret_cast<uintptr_t>(taskPara.Reduce.src));
542 1 : hcclReportData.profInfo.dstAddr = static_cast<u64>(reinterpret_cast<uintptr_t>(taskPara.Reduce.dst));
543 1 : hcclReportData.profInfo.dataSize = static_cast<u32>(taskPara.Reduce.size);
544 1 : hcclReportData.profInfo.notifyID = taskPara.Reduce.notifyID;
545 1 : hcclReportData.profInfo.linkType = static_cast<uint16_t>(taskPara.Reduce.linkType);
546 1 : break;
547 1 : case TaskParamType::TASK_NOTIFY_RECORD:
548 : case TaskParamType::TASK_NOTIFY_WAIT:
549 1 : hcclReportData.profInfo.notifyID = taskPara.Notify.notifyID;
550 1 : break;
551 1 : default:
552 1 : break;
553 : }
554 4 : break;
555 : }
556 8 : }
557 :
558 1 : void ProfilingHandler::DumpHCCLReportData(const TaskInfo &taskInfo, const MsprofAdditionalInfo &reporterData) const
559 : {
560 1 : if (reporterData.type == static_cast<uint32_t>(ProfTaskType::TASK_HCCL_INFO)) {
561 1 : auto *profInfo = reinterpret_cast<const MsprofHcclInfo *>(reporterData.data);
562 3 : HCCL_INFO(
563 : "MsprofAdditionalInfo profInfo: timeStamp[%llu], itemId[%llu], cclTag[%llu], groupName[%llu], "
564 : "localRank[%u], remoteRank[%u], rankSize[%u], workFlowMode[%u], planeID[%u], ctxId[%u], "
565 : "stage[%u], role[%u], durationEstimated[%f], taskType[%d]",
566 : reporterData.timeStamp, profInfo->itemId, profInfo->cclTag,
567 : profInfo->groupName, profInfo->localRank, profInfo->remoteRank,
568 : profInfo->rankSize, profInfo->workFlowMode, profInfo->planeID,
569 : profInfo->ctxId, profInfo->stage, profInfo->role,
570 : profInfo->durationEstimated, static_cast<int>(taskInfo.taskParam_.taskType));
571 3 : HCCL_INFO(
572 : "MsprofAdditionalInfo profInfo detail: srcAddr[%llu], dstAddr[%llu], dataSize[%llu], notifyID[%llu], "
573 : "linkType[%u], opType[%s], transportType[%u], dataType[%s], rdmaType[%u]",
574 : profInfo->srcAddr, profInfo->dstAddr, profInfo->dataSize,
575 : profInfo->notifyID, profInfo->linkType,
576 : OpTypeToSerialString(profInfo->opType).c_str(), profInfo->transportType,
577 : DataTypeToSerialString(profInfo->dataType).c_str(), profInfo->rdmaType);
578 : } else {
579 0 : auto *dpuProfInfo = reinterpret_cast<const MsprofDpuHcclTrack *>(reporterData.data);
580 0 : HCCL_INFO(
581 : "MsprofAdditionalInfo dpuProfInfo: timeStamp[%llu], itemId[%llu], cclTag[%llu], groupName[%llu], "
582 : "localRank[%u], remoteRank[%u], rankSize[%u], workFlowMode[%u], planeID[%u], "
583 : "stage[%u], role[%u], durationEstimated[%f], taskType[%d]",
584 : reporterData.timeStamp, dpuProfInfo->itemId, dpuProfInfo->cclTag,
585 : dpuProfInfo->groupName, dpuProfInfo->localRank, dpuProfInfo->remoteRank,
586 : dpuProfInfo->rankSize, dpuProfInfo->workFlowMode, dpuProfInfo->planeID,
587 : dpuProfInfo->stage, dpuProfInfo->role, dpuProfInfo->durationEstimated,
588 : static_cast<int>(taskInfo.taskParam_.taskType));
589 0 : HCCL_INFO(
590 : "MsprofAdditionalInfo dpuProfInfo detail: srcAddr[%llu], dstAddr[%llu], dataSize[%llu], notifyID[%llu], "
591 : "linkType[%u], opType[%s], transportType[%u], dataType[%s], rdmaType[%u], "
592 : "taskId[%u], aicpu_task_id[%u], streamId[%u], npuDevId[%u], dpuDevId[%u], timeStamp[%llu]",
593 : dpuProfInfo->srcAddr, dpuProfInfo->dstAddr, dpuProfInfo->dataSize,
594 : dpuProfInfo->notifyID, dpuProfInfo->linkType,
595 : OpTypeToSerialString(dpuProfInfo->opType).c_str(), dpuProfInfo->transportType,
596 : DataTypeToSerialString(dpuProfInfo->dataType).c_str(), dpuProfInfo->rdmaType,
597 : dpuProfInfo->taskId, dpuProfInfo->aicpu_task_id, dpuProfInfo->streamId,
598 : dpuProfInfo->npuDevId, dpuProfInfo->dpuDevId, dpuProfInfo->timeStamp);
599 : }
600 1 : }
601 :
602 0 : void ProfilingHandler::LogCcuTaskInfo(const CcuProfilingInfo &info, const TaskInfo &taskInfo,
603 : uint64_t itemId, uint64_t groupName, u32 rankId, u32 ranksize) const
604 : {
605 0 : HCCL_INFO("[ProfilingHandler]GetCcuTaskInfo, ccuTaskInfo data is: version[%u], workFlowMode[%u], itemId[%llu], "
606 : "groupName[%llu], rankId[%u], ranksize[%u], streamId[%u], taskId[%u], dieId[%u], "
607 : "missionId[%u],instrId[%u]",
608 : 0, static_cast<u32>(HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE), itemId, groupName,
609 : rankId, ranksize, taskInfo.streamId_, taskInfo.taskId_, info.dieId,
610 : info.missionId, info.instrId);
611 0 : }
612 :
613 0 : void ProfilingHandler::LogCcuWaitSignalInfo(const CcuProfilingInfo &info, const TaskInfo &taskInfo,
614 : uint64_t itemId, uint64_t groupName, u32 rankId, u32 ranksize) const
615 : {
616 0 : HCCL_INFO(
617 : "[ProfilingHandler]GetCcuWaitSignalInfo, waitSignalInfo data is: version[%u], itemId[%llu], groupName[%llu], "
618 : "rankId[%u], ranksize[%u], workFlowMode[%u], streamId[%u], taskId[%u], dieId[%u],instrId[%u],missionId[%u], "
619 : "ckeId[%u],mask[%u]",
620 : 0, itemId, groupName, rankId, ranksize,
621 : static_cast<u32>(HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE), taskInfo.streamId_, taskInfo.taskId_, info.dieId, info.instrId,
622 : info.missionId, info.ckeId, info.mask);
623 0 : }
624 :
625 0 : void ProfilingHandler::LogCcuGroupInfo(const CcuProfilingInfo &info, const TaskInfo &taskInfo,
626 : uint64_t itemId, uint64_t groupName, u32 rankId, u32 ranksize) const
627 : {
628 0 : HCCL_INFO("[ProfilingHandler]GetCcuGroupInfo, ccuGroupInfo data is: version[%u], itemId[%llu], "
629 : "groupName[%llu], rankId[%u], ranksize[%u], workFlowMode[%u], streamId[%u], taskId[%u], "
630 : "dieId[%u],instrId[%u],missionId[%u], dataSize[%llu]",
631 : 0, itemId, groupName,
632 : rankId, ranksize, static_cast<u32>(HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE),
633 : taskInfo.streamId_, taskInfo.taskId_, info.dieId, info.instrId,
634 : info.missionId, info.dataSize);
635 0 : if (info.reduceOpType != INVALID_TYPE_VALUE) {
636 0 : HCCL_INFO("ccuGroupInfo reduceOpType is [%d]", static_cast<int>(info.reduceOpType));
637 : }
638 0 : if (info.inputDataType != INVALID_TYPE_VALUE) {
639 0 : HCCL_INFO("ccuGroupInfo inputDataType is [%d]", static_cast<int>(info.inputDataType));
640 : }
641 0 : if (info.outputDataType != INVALID_TYPE_VALUE) {
642 0 : HCCL_INFO("ccuGroupInfo outputDataType is [%d]", static_cast<int>(info.outputDataType));
643 : }
644 0 : for (auto i = 0; i < CCU_MAX_CHANNEL_NUM; i++) {
645 0 : if (info.channelId[i] != INVALID_VALUE_CHANNELID
646 0 : && info.remoteRankId[i] != INVALID_VALUE_RANKID) {
647 0 : HCCL_INFO("[ProfilingHandler]GetCcuGroupInfo, ccuGroupInfo data is: channelId[%d] = %u, "
648 : "remoteRankId[%d] = %u",
649 : i, info.channelId[i], i, info.remoteRankId[i]);
650 : }
651 : }
652 0 : }
653 :
654 4 : void ProfilingHandler::ReportCcuInfoLog(const TaskInfo &taskInfo) const
655 : {
656 4 : if (LIKELY(HcclCheckLogLevel(HCCL_LOG_INFO) == 0)) {
657 4 : return;
658 : }
659 4 : if (!enableHcclL1_) {
660 0 : return;
661 : }
662 4 : if (taskInfo.taskParam_.ccuDetailInfo == nullptr || taskInfo.dfxOpInfo_ == nullptr) {
663 4 : return;
664 : }
665 0 : auto ccuDetailInfo = taskInfo.taskParam_.ccuDetailInfo;
666 0 : for (const auto &info : *ccuDetailInfo) {
667 0 : uint64_t itemId = GetProfHashId(info.name.c_str(), info.name.length());
668 0 : uint64_t groupName = GetProfHashId(taskInfo.dfxOpInfo_->op_.opTag.c_str(),
669 0 : taskInfo.dfxOpInfo_->op_.opTag.length());
670 0 : u32 rankId = 0;
671 0 : u32 ranksize = 0;
672 0 : if (taskInfo.dfxOpInfo_->isIndop_ == true) {
673 0 : rankId = taskInfo.dfxOpInfo_->op_.myRank;
674 0 : ranksize = taskInfo.dfxOpInfo_->rankSize_;
675 : } else {
676 0 : CommunicatorImpl *commImp = static_cast<CommunicatorImpl *>(taskInfo.dfxOpInfo_->comm_);
677 0 : rankId = commImp->GetIdIndex();
678 0 : ranksize = commImp->GetRankSize();
679 : }
680 0 : if (info.type == 0) {
681 0 : LogCcuTaskInfo(info, taskInfo, itemId, groupName, rankId, ranksize);
682 0 : } else if (info.type == 1) {
683 0 : LogCcuWaitSignalInfo(info, taskInfo, itemId, groupName, rankId, ranksize);
684 0 : } else if (info.type == CCU_TYPE) {
685 0 : LogCcuGroupInfo(info, taskInfo, itemId, groupName, rankId, ranksize);
686 : }
687 : }
688 0 : }
689 :
690 4 : void ProfilingHandler::ReportCcuInfo(const TaskInfo &taskInfo) const
691 : {
692 4 : ReportCcuInfoLog(taskInfo);
693 4 : if (taskInfo.taskParam_.ccuDetailInfo == nullptr) {
694 9 : HCCL_ERROR("[ProfilingHandler]ReportCcuInfo ccuDetailInfo is nullptr.");
695 4 : return;
696 : }
697 1 : if (taskInfo.dfxOpInfo_ == nullptr) {
698 3 : HCCL_WARNING("[ProfilingHandler]ReportCcuInfo dfxOpInfo_ is nullptr, skip ReportCcuInfo.");
699 1 : return;
700 : }
701 0 : auto ccuDetailInfo = taskInfo.taskParam_.ccuDetailInfo;
702 0 : for (const auto &info : *ccuDetailInfo) {
703 0 : if (info.type == 0 && enableHcclL1_) {
704 0 : GetCcuTaskInfo(taskInfo, info);
705 0 : } else if (info.type == 1 && enableHcclL1_) {
706 0 : GetCcuWaitSignalInfo(taskInfo, info);
707 0 : } else if (info.type == CCU_TYPE && enableHcclL1_) {
708 0 : GetCcuGroupInfo(taskInfo, info);
709 : }
710 : }
711 0 : }
712 :
713 0 : void ProfilingHandler::GetCcuTaskInfo(const TaskInfo &taskInfo, const CcuProfilingInfo &info) const
714 : {
715 0 : uint64_t timestamp = DlProfFunction::GetInstance().dlMsprofSysCycleTime();
716 0 : MsprofAdditionalInfo reporterData{};
717 0 : reporterData.level = MSPROF_REPORT_HCCL_NODE_LEVEL;
718 0 : reporterData.type = MSPROF_REPORT_CCU_TASK_INFO;
719 0 : reporterData.threadId = SalGetTid();
720 0 : reporterData.dataLen = sizeof(MsprofCcuTaskInfo);
721 0 : reporterData.timeStamp = timestamp;
722 0 : auto *ccuTaskInfo = reinterpret_cast<MsprofCcuTaskInfo *>(reporterData.data);
723 0 : ccuTaskInfo->version = 0;
724 0 : ccuTaskInfo->workFlowMode = static_cast<u32>(HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE);
725 0 : ccuTaskInfo->itemId = GetProfHashId(info.name.c_str(), info.name.length());
726 0 : uint64_t groupName = GetProfHashId(taskInfo.dfxOpInfo_->op_.opTag.c_str(),
727 0 : taskInfo.dfxOpInfo_->op_.opTag.length());
728 0 : ccuTaskInfo->groupName = groupName;
729 0 : if (taskInfo.dfxOpInfo_->isIndop_ == true) {
730 0 : ccuTaskInfo->rankId = taskInfo.dfxOpInfo_->op_.myRank;
731 0 : ccuTaskInfo->ranksize = taskInfo.dfxOpInfo_->rankSize_;
732 : } else {
733 0 : CommunicatorImpl *commImp = static_cast<CommunicatorImpl *>(taskInfo.dfxOpInfo_->comm_);
734 0 : ccuTaskInfo->rankId = commImp->GetIdIndex();
735 0 : ccuTaskInfo->ranksize = commImp->GetRankSize();
736 : }
737 :
738 0 : ccuTaskInfo->streamId = taskInfo.streamId_;
739 0 : ccuTaskInfo->taskId = taskInfo.taskId_;
740 0 : ccuTaskInfo->dieId = info.dieId;
741 0 : ccuTaskInfo->missionId = info.missionId;
742 0 : ccuTaskInfo->instrId = info.instrId;
743 0 : ReportAdditionInfo(reporterData);
744 0 : }
745 :
746 1 : void ProfilingHandler::GetCcuGroupInfo(const TaskInfo &taskInfo, const CcuProfilingInfo &info) const
747 : {
748 1 : MsprofAdditionalInfo reporterData{};
749 1 : reporterData.level = MSPROF_REPORT_HCCL_NODE_LEVEL;
750 1 : reporterData.type = MSPROF_REPORT_CCU_GROUP_INFO;
751 1 : reporterData.threadId = SalGetTid();
752 1 : reporterData.dataLen = sizeof(MsprofCcuGroupInfo);
753 1 : reporterData.timeStamp = DlProfFunction::GetInstance().dlMsprofSysCycleTime();
754 1 : auto *ccuGroupInfo = reinterpret_cast<MsprofCcuGroupInfo *>(reporterData.data);
755 1 : ccuGroupInfo->version = 0;
756 1 : ccuGroupInfo->itemId = GetProfHashId(info.name.c_str(), info.name.length());
757 1 : uint64_t groupName = GetProfHashId(taskInfo.dfxOpInfo_->op_.opTag.c_str(), taskInfo.dfxOpInfo_->op_.opTag.length());
758 1 : ccuGroupInfo->groupName = groupName;
759 1 : if (taskInfo.dfxOpInfo_->isIndop_ == true) {
760 0 : ccuGroupInfo->rankId = taskInfo.dfxOpInfo_->op_.myRank;
761 0 : ccuGroupInfo->ranksize = taskInfo.dfxOpInfo_->rankSize_;
762 : } else {
763 1 : CommunicatorImpl *commImp = static_cast<CommunicatorImpl *>(taskInfo.dfxOpInfo_->comm_);
764 1 : ccuGroupInfo->rankId = commImp->GetIdIndex();
765 1 : ccuGroupInfo->ranksize = commImp->GetRankSize();
766 : }
767 1 : ccuGroupInfo->workFlowMode = static_cast<u32>(HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE);
768 1 : ccuGroupInfo->streamId = taskInfo.streamId_;
769 1 : ccuGroupInfo->taskId = taskInfo.taskId_;
770 1 : ccuGroupInfo->dieId = info.dieId;
771 1 : ccuGroupInfo->instrId = info.instrId;
772 1 : ccuGroupInfo->missionId = info.missionId;
773 1 : ccuGroupInfo->reduceOpType = info.reduceOpType;
774 1 : ccuGroupInfo->inputDataType = info.inputDataType;
775 1 : ccuGroupInfo->outputDataType = info.outputDataType;
776 1 : ccuGroupInfo->dataSize = info.dataSize;
777 1 : std::copy(info.channelId, info.channelId + CCU_MAX_CHANNEL_NUM, ccuGroupInfo->channelId);
778 1 : std::copy(info.remoteRankId, info.remoteRankId + CCU_MAX_CHANNEL_NUM, ccuGroupInfo->remoteRankId);
779 1 : ReportAdditionInfo(reporterData);
780 1 : }
781 :
782 0 : void ProfilingHandler::DumpCcuGroupInfo([[maybe_unused]] const MsprofCcuGroupInfo &ccuGroupInfo) const
783 : {
784 0 : if (LIKELY(HcclCheckLogLevel(HCCL_LOG_INFO) == 0)) {
785 0 : return;
786 : }
787 : // HCCL_INFOs migrated to ReportCcuInfoLog
788 : }
789 :
790 1 : void ProfilingHandler::GetCcuWaitSignalInfo(const TaskInfo &taskInfo, const CcuProfilingInfo &info) const
791 : {
792 1 : uint64_t timestamp = DlProfFunction::GetInstance().dlMsprofSysCycleTime();
793 1 : MsprofAdditionalInfo reporterData{};
794 1 : reporterData.level = MSPROF_REPORT_HCCL_NODE_LEVEL;
795 1 : reporterData.type = MSPROF_REPORT_CCU_WAIT_SIGNAL_INFO;
796 1 : reporterData.threadId = SalGetTid();
797 1 : reporterData.dataLen = sizeof(MsprofCcuWaitSignalInfo);
798 1 : reporterData.timeStamp = timestamp;
799 1 : auto *waitSignalInfo = reinterpret_cast<MsprofCcuWaitSignalInfo *>(reporterData.data);
800 1 : waitSignalInfo->version = 0;
801 1 : waitSignalInfo->itemId = GetProfHashId(info.name.c_str(), info.name.length());
802 1 : uint64_t groupName = GetProfHashId(taskInfo.dfxOpInfo_->op_.opTag.c_str(), taskInfo.dfxOpInfo_->op_.opTag.length());
803 1 : waitSignalInfo->groupName = groupName;
804 1 : if (taskInfo.dfxOpInfo_->isIndop_ == true) {
805 0 : waitSignalInfo->rankId = taskInfo.dfxOpInfo_->op_.myRank;
806 0 : waitSignalInfo->ranksize = taskInfo.dfxOpInfo_->rankSize_;
807 : } else {
808 1 : CommunicatorImpl *commImp = static_cast<CommunicatorImpl *>(taskInfo.dfxOpInfo_->comm_);
809 1 : waitSignalInfo->rankId = commImp->GetIdIndex();
810 1 : waitSignalInfo->ranksize = commImp->GetRankSize();
811 : }
812 1 : waitSignalInfo->workFlowMode = static_cast<u32>(HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE);
813 1 : waitSignalInfo->streamId = taskInfo.streamId_;
814 1 : waitSignalInfo->taskId = taskInfo.taskId_;
815 1 : waitSignalInfo->dieId = info.dieId;
816 1 : waitSignalInfo->instrId = info.instrId;
817 1 : waitSignalInfo->missionId = info.missionId;
818 1 : waitSignalInfo->ckeId = info.ckeId;
819 1 : waitSignalInfo->mask = info.mask;
820 1 : std::copy(info.channelId, info.channelId + CCU_MAX_CHANNEL_NUM, waitSignalInfo->channelId);
821 1 : std::copy(info.remoteRankId, info.remoteRankId + CCU_MAX_CHANNEL_NUM, waitSignalInfo->remoteRankId);
822 1 : ReportAdditionInfo(reporterData);
823 1 : }
824 :
825 1 : void ProfilingHandler::ReportAclApi(uint32_t cmdType, uint64_t beginTime, uint64_t endTime, uint64_t cmdItemId, uint32_t threadId) const
826 : {
827 1 : MsprofApi reporterData{};
828 1 : reporterData.level = MSPROF_REPORT_ACL_LEVEL;
829 1 : reporterData.type = static_cast<int32_t>(cmdType) + MSPROF_REPORT_ACL_HOST_HCCL_BASE_TYPE;
830 1 : reporterData.threadId = threadId;
831 1 : reporterData.beginTime = beginTime;
832 1 : reporterData.endTime = endTime;
833 1 : reporterData.itemId = cmdItemId;
834 :
835 1 : s32 ret = DlProfFunction::GetInstance().dlMsprofReportApi(1, &reporterData);
836 3 : HCCL_INFO("[ProfilingHandler][ReportAclApi], reporterData data is: level[%u], type[%u], threadId[%u], "
837 : "beginTime[%llu], endTime[%llu], itemId[%llu], return value[%d]",
838 : reporterData.level, reporterData.type, reporterData.threadId, reporterData.beginTime,
839 : reporterData.endTime, reporterData.itemId, ret);
840 1 : if (ret != 0) {
841 0 : THROW<InternalException>("Call dlMsprofReportApi failed, return[%d]", ret);
842 : }
843 1 : }
844 :
845 9 : void ProfilingHandler::ReportNodeApi(uint64_t beginTime, uint64_t endTime, uint64_t cmdItemId, uint32_t threadId,
846 : bool cachedReq)
847 : {
848 9 : MsprofApi reporterData{};
849 9 : reporterData.level = MSPROF_REPORT_NODE_LEVEL;
850 9 : reporterData.type = MSPROF_REPORT_NODE_LAUNCH_TYPE;
851 9 : reporterData.threadId = threadId;
852 9 : reporterData.beginTime = beginTime;
853 9 : reporterData.endTime = endTime;
854 9 : reporterData.itemId = cmdItemId;
855 :
856 9 : if (cachedReq) {
857 1 : std::lock_guard<std::mutex> lock(cachedTaskApiInfoMutex_);
858 1 : cachedTaskApiInfo_.push(reporterData);
859 1 : }
860 9 : if (!enableHostApi_) {
861 8 : return;
862 : }
863 :
864 1 : s32 ret = DlProfFunction::GetInstance().dlMsprofReportApi(1, &reporterData);
865 3 : HCCL_INFO("[ProfilingHandler][ReportNodeApi], reporterData data is: level[%u], type[%u], threadId[%u], "
866 : "beginTime[%llu], endTime[%llu], itemId[%llu], return value[%d]",
867 : reporterData.level, reporterData.type, reporterData.threadId, reporterData.beginTime,
868 : reporterData.endTime, reporterData.itemId, ret);
869 1 : if (ret != 0) {
870 0 : THROW<InternalException>("Call MsprofReportApi failed, return[%d]", ret);
871 : }
872 : }
873 :
874 9 : void ProfilingHandler::ReportNodeBasicInfo(uint64_t timeStamp, uint64_t cmdItemId, uint32_t threadId, bool cachedReq)
875 : {
876 9 : MsprofCompactInfo reporterData{};
877 9 : reporterData.level = MSPROF_REPORT_NODE_LEVEL;
878 9 : reporterData.type = MSPROF_REPORT_NODE_BASIC_INFO_TYPE;
879 9 : reporterData.threadId = threadId;
880 9 : reporterData.dataLen = sizeof(MsprofNodeBasicInfo);
881 9 : reporterData.timeStamp = timeStamp;
882 9 : reporterData.data.nodeBasicInfo.opName = cmdItemId;
883 9 : reporterData.data.nodeBasicInfo.taskType = MSPROF_GE_TASK_TYPE_HCCL;
884 9 : reporterData.data.nodeBasicInfo.opType = cmdItemId;
885 9 : reporterData.data.nodeBasicInfo.opFlag = 0;
886 27 : HCCL_INFO("[ProfilingHandler][ReportNodeBasicInfo], reporterData data is: level[%u], type[%u], threadId[%u], "
887 : "dataLen[%u], taskType[%u], opFlag[%u]", reporterData.level, reporterData.type, reporterData.threadId,
888 : reporterData.dataLen, reporterData.data.nodeBasicInfo.taskType, reporterData.data.nodeBasicInfo.opFlag);
889 9 : if (cachedReq) {
890 1 : std::lock_guard<std::mutex> lock(cacheHcclOpInfoMutex_);
891 1 : cacheHcclOpInfo_.push(reporterData);
892 1 : }
893 9 : if (!enableHcclL1_) {
894 7 : return;
895 : }
896 2 : s32 ret = DlProfFunction::GetInstance().dlMsprofReportCompactInfo(1, &reporterData, sizeof(MsprofCompactInfo));
897 6 : HCCL_INFO("Call MsprofReportCompactInfo, return value[%d]", ret);
898 2 : if (ret != 0) {
899 0 : THROW<InternalException>("Call MsprofReportCompactInfo failed, return[%d]", ret);
900 : }
901 : }
902 :
903 10 : void ProfilingHandler::ReportHcclOpInfo(uint64_t timeStamp, const DfxOpInfo &opInfo, uint32_t threadId, bool cachedReq)
904 : {
905 10 : MsprofCompactInfo reporterData{};
906 10 : reporterData.level = MSPROF_REPORT_NODE_LEVEL;
907 10 : reporterData.type = MSPROF_REPORT_NODE_HCCL_OP_INFO_TYPE;
908 10 : reporterData.threadId = threadId;
909 10 : reporterData.dataLen = sizeof(MsprofHCCLOPInfo);
910 10 : reporterData.timeStamp = timeStamp;
911 10 : reporterData.data.hcclopInfo.relay = 0;
912 10 : reporterData.data.hcclopInfo.retry = 0;
913 10 : reporterData.data.hcclopInfo.dataType = opInfo.op_.dataType;
914 10 : reporterData.data.hcclopInfo.algType = cachedAlgTypeHashId_;
915 10 : uint64_t groupName = GetProfHashId(opInfo.op_.opTag.c_str(), opInfo.op_.opTag.length());
916 10 : reporterData.data.hcclopInfo.groupName = groupName;
917 10 : u32 ranksize{0};
918 10 : if (opInfo.isIndop_ == true) {
919 0 : ranksize = opInfo.rankSize_;
920 0 : reporterData.data.hcclopInfo.count = opInfo.op_.dataCount;
921 : } else {
922 10 : CommunicatorImpl *commImp = static_cast<CommunicatorImpl *>(opInfo.comm_);
923 10 : ranksize = commImp->GetRankSize();
924 10 : if (opInfo.op_.opType == OpType::ALLTOALLV) {
925 1 : u64 sendCount = 0;
926 3 : for (u64 i = 0; i < ranksize; i++) {
927 2 : sendCount += *(static_cast<const u64 *>(opInfo.op_.all2AllVDataDes.sendCounts) + i);
928 : }
929 1 : reporterData.data.hcclopInfo.count = sendCount;
930 9 : } else if (opInfo.op_.opType == OpType::ALLTOALL) {
931 0 : reporterData.data.hcclopInfo.count = opInfo.op_.all2AllDataDes.sendCount;
932 : } else {
933 9 : reporterData.data.hcclopInfo.count = opInfo.op_.dataCount;
934 : }
935 : }
936 10 : if (cachedReq) {
937 7 : std::lock_guard<std::mutex> lock(cacheHcclOpInfoMutex_);
938 7 : cacheHcclOpInfo_.push(reporterData);
939 7 : }
940 10 : if (!enableHostApi_) {
941 8 : return;
942 : }
943 6 : HCCL_INFO("[ProfilingHandler][ReportHcclOpInfo], data is: level[%u], type[%u], threadId[%u], dataLen[%u], "
944 : "timeStamp[%llu], relay [%u], retry[%u], dataType[%s], algType[%llu], groupName[%llu], count[%llu]",
945 : reporterData.level, reporterData.type, reporterData.threadId, reporterData.dataLen,
946 : reporterData.timeStamp, reporterData.data.hcclopInfo.relay, reporterData.data.hcclopInfo.retry,
947 : DataTypeToSerialString(reporterData.data.hcclopInfo.dataType).c_str(), reporterData.data.hcclopInfo.algType,
948 : reporterData.data.hcclopInfo.groupName, reporterData.data.hcclopInfo.count);
949 2 : s32 ret = DlProfFunction::GetInstance().dlMsprofReportCompactInfo(1, &reporterData, sizeof(MsprofCompactInfo));
950 2 : if (ret != 0) {
951 0 : THROW<InternalException>("[ProfilingHandler] Call dlMsprofReportCompactInfo failed, return[%d]", ret);
952 : }
953 : }
954 :
955 2 : void ProfilingHandler::ReportAdditionInfo(MsprofAdditionalInfo& reporterData) const
956 : {
957 : s32 ret
958 2 : = DlProfFunction::GetInstance().dlMsprofReportAdditionalInfo(0, &reporterData, sizeof(MsprofAdditionalInfo));
959 6 : HCCL_INFO("[ProfilingHandler][ReportAdditionInfo], level[%u], type[%u], threadId[%u], dataLen[%u], "
960 : "timeStamp[%llu], return value[%d]",
961 : reporterData.level, reporterData.type, reporterData.threadId, reporterData.dataLen,
962 : reporterData.timeStamp, ret);
963 2 : if (ret != 0) {
964 0 : THROW<InternalException>("Call MsprofReportAdditionalInfo failed, return[%d]", ret);
965 : }
966 2 : }
967 :
968 2 : int32_t ProfilingHandler::CommandHandle(uint32_t rtType, void *data, uint32_t len) const
969 : {
970 : (void)len;
971 2 : if (data == nullptr || rtType != rtProfCtrlType_t::RT_PROF_CTRL_SWITCH) {
972 3 : HCCL_ERROR("[ProfilingHandler][CommandHandle] data is nullptr or rtType is invalid, rtType[%u]", rtType);
973 1 : return HCCL_E_PARA;
974 : }
975 1 : rtProfCommandHandle_t *profConfigParam = static_cast<rtProfCommandHandle_t *>(data);
976 1 : auto type = profConfigParam->type;
977 1 : auto profconfig = profConfigParam->profSwitch;
978 3 : HCCL_RUN_INFO("[Profiling][CommandHandle] CommandHandle's rtType is %u. CommandHandle_switch type[%u], " \
979 : "profconfig[%llu], deviceLogicId[%u]", rtType, type, profconfig, profConfigParam->devIdList[0]);
980 1 : switch (type) {
981 0 : case PROF_COMMANDHANDLE_TYPE_START:
982 0 : instance_.StartSubscribe(profconfig);
983 0 : break;
984 0 : case PROF_COMMANDHANDLE_TYPE_STOP:
985 0 : instance_.StopSubscribe();
986 0 : break;
987 1 : default:
988 3 : HCCL_RUN_INFO("[Profiling][CommandHandle] Unexpected behaviour.");
989 : }
990 1 : return HCCL_SUCCESS;
991 : }
992 :
993 5 : void ProfilingHandler::StartSubscribe(uint64_t profconfig)
994 : {
995 15 : HCCL_RUN_INFO("[Profiling][CommandHandle] profSwitch is[%llu]", profconfig);
996 5 : if ((profconfig & PROF_ACL_API_MASK) != 0) {
997 1 : StartHostApiSubscribe();
998 : }
999 5 : if ((profconfig & PROF_TASK_TIME_MASK) != 0 && (profconfig & PROF_TASK_TIME_L1_MASK) == 0) {
1000 1 : StartHostHcclOpSubscribe();
1001 : }
1002 5 : if ((profconfig & PROF_TASK_TIME_L1_MASK) != 0) {
1003 1 : StartTaskApiSubscribe();
1004 1 : StartAdditionInfoSubscribe();
1005 1 : StartCcuSubscribe();
1006 : }
1007 15 : HCCL_RUN_INFO("[Profiling][CommandHandle] profSwitch is[%llu]", profconfig);
1008 5 : }
1009 :
1010 2 : void ProfilingHandler::StartHostApiSubscribe()
1011 : {
1012 2 : enableHostApi_ = true;
1013 2 : CallProfRegHostApi();
1014 2 : ReportStoragedCompactInfo();
1015 2 : ReportMc2AdditionInfo();
1016 6 : HCCL_RUN_INFO("SetHostApiSubscribe:[%d]", enableHostApi_);
1017 2 : }
1018 :
1019 2 : void ProfilingHandler::CallProfRegHostApi() const
1020 : {
1021 2 : if (!enableHostApi_) {
1022 0 : return;
1023 : }
1024 2 : auto &profFunction = DlProfFunction::GetInstance();
1025 44 : for (auto i = 0; i < OpType::__COUNT__; ++i) {
1026 42 : OpType type(static_cast<OpType::Value>(i));
1027 42 : s32 ret = profFunction.dlMsprofRegTypeInfo(MSPROF_REPORT_ACL_LEVEL,
1028 42 : static_cast<uint32_t>(type) + MSPROF_REPORT_ACL_HOST_HCCL_BASE_TYPE,
1029 84 : type.Describe().c_str());
1030 42 : if (ret != 0) {
1031 0 : THROW<InternalException>("Call MsprofRegTypeInfo fail, return[%d]", ret);
1032 : }
1033 : }
1034 44 : for (auto i = 0; i < OpType::__COUNT__; ++i) {
1035 42 : OpType type(static_cast<OpType::Value>(i));
1036 42 : s32 ret = profFunction.dlMsprofRegTypeInfo(MSPROF_REPORT_NODE_LEVEL,
1037 42 : static_cast<uint32_t>(type) + MSPROF_REPORT_NODE_HCCL_BASE_TYPE,
1038 84 : type.Describe().c_str());
1039 42 : if (ret != 0) {
1040 0 : THROW<InternalException>("Call MsprofRegTypeInfo fail, return[%d]", ret);
1041 : }
1042 : }
1043 2 : const std::string hcclType("hccl_op_info");
1044 2 : s32 ret = profFunction.dlMsprofRegTypeInfo(MSPROF_REPORT_NODE_LEVEL, MSPROF_REPORT_NODE_HCCL_OP_INFO_TYPE,
1045 : hcclType.c_str());
1046 2 : if (ret != 0) {
1047 0 : THROW<InternalException>("Call MsprofRegTypeInfo fail, return[%d]", ret);
1048 : }
1049 2 : }
1050 :
1051 4 : void ProfilingHandler::ReportStoragedCompactInfo()
1052 : {
1053 4 : std::lock_guard<std::mutex> lock(cacheHcclOpInfoMutex_);
1054 12 : HCCL_INFO("[ReportStoragedCompactInfo] The size of the storageCompactInfo_ is [%zu]", cacheHcclOpInfo_.size());
1055 4 : std::queue<MsprofCompactInfo> tempCompactInfo = cacheHcclOpInfo_;
1056 32 : while (!tempCompactInfo.empty()) {
1057 28 : MsprofCompactInfo reportData = tempCompactInfo.front();
1058 28 : tempCompactInfo.pop();
1059 28 : s32 ret = DlProfFunction::GetInstance().dlMsprofReportCompactInfo(0, &reportData, sizeof(MsprofCompactInfo));
1060 28 : if (ret != 0) {
1061 0 : THROW<InternalException>("Call MsprofRegTypeInfo failed, return[%d]", ret);
1062 : }
1063 : }
1064 4 : }
1065 :
1066 2 : void ProfilingHandler::ReportMc2AdditionInfo()
1067 : {
1068 2 : std::lock_guard<std::mutex> lock(cacheHcclAdditionInfoMutex_);
1069 6 : HCCL_INFO("[ReportMc2AdditionInfo] The size of the storageCompactInfo_ is [%zu]", cacheHcclAdditionInfo_.size());
1070 2 : std::queue<MsprofAdditionalInfo> tempCompactInfo = cacheHcclAdditionInfo_;
1071 6 : while (!tempCompactInfo.empty()) {
1072 4 : MsprofAdditionalInfo reportData = tempCompactInfo.front();
1073 4 : tempCompactInfo.pop();
1074 4 : s32 ret = DlProfFunction::GetInstance().dlMsprofReportAdditionalInfo(1, &reportData,
1075 : sizeof(MsprofAdditionalInfo));
1076 4 : if (ret != 0) {
1077 0 : THROW<InternalException>("Call MsprofRegTypeInfo failed, return[%d]", ret);
1078 : }
1079 : }
1080 2 : }
1081 :
1082 2 : void ProfilingHandler::StartTaskApiSubscribe()
1083 : {
1084 2 : enableHcclNode_ = true;
1085 2 : CallProfRegTaskTypeApi();
1086 2 : ReportStoragedTaskApi();
1087 6 : HCCL_INFO("SetTaskApiSubscribe:[%d]", enableHcclNode_);
1088 2 : }
1089 :
1090 2 : void ProfilingHandler::CallProfRegTaskTypeApi() const
1091 : {
1092 2 : if (!enableHcclNode_) {
1093 0 : HCCL_INFO("[ProfilingHandler] enableHostApi_ is false.");
1094 0 : return;
1095 : }
1096 2 : const std::string hcclType("hccl_info");
1097 2 : s32 sret = DlProfFunction::GetInstance().dlMsprofRegTypeInfo(
1098 : MSPROF_REPORT_HCCL_NODE_LEVEL, static_cast<uint32_t>(ProfTaskType::TASK_HCCL_INFO), hcclType.c_str());
1099 2 : if (sret != 0) {
1100 0 : THROW<InternalException>("Call MsprofRegTypeInfo fail, return[%d]", sret);
1101 : }
1102 :
1103 2 : const std::string dpuhcclType("dpu_hccl_info");
1104 2 : sret = DlProfFunction::GetInstance().dlMsprofRegTypeInfo(
1105 : MSPROF_REPORT_HCCL_NODE_LEVEL, static_cast<uint32_t>(ProfTaskType::TASK_DPU_HCCL_INFO), dpuhcclType.c_str());
1106 2 : if (sret != 0) {
1107 0 : THROW<InternalException>("Call MsprofRegTypeInfo fail, return[%d]", sret);
1108 : }
1109 : const std::vector<std::pair<uint32_t, std::string>> taskTypes
1110 6 : = {{MSPROF_REPORT_NODE_CONTEXT_ID_INFO_TYPE, "context_id_info"}};
1111 : const std::vector<std::pair<uint32_t, std::string>> taskOtherTypes
1112 0 : = {{MSPROF_REPORT_NODE_BASIC_INFO_TYPE, "node_basic_info"},
1113 8 : {MSPROF_REPORT_NODE_MC2_COMMINFO_TYPE, "mc2_comm_info"}};
1114 :
1115 4 : for (auto &it : taskTypes) {
1116 2 : s32 ret = DlProfFunction::GetInstance().dlMsprofRegTypeInfo(MSPROF_REPORT_HCCL_NODE_LEVEL, it.first,
1117 : it.second.c_str());
1118 2 : if (ret != 0) {
1119 0 : THROW<InternalException>("Call dlMsprofRegTypeInfo failed, return[%d]", ret);
1120 : }
1121 : }
1122 6 : for (auto &it : taskOtherTypes) {
1123 : s32 ret
1124 4 : = DlProfFunction::GetInstance().dlMsprofRegTypeInfo(MSPROF_REPORT_NODE_LEVEL, it.first, it.second.c_str());
1125 4 : if (ret != 0) {
1126 0 : THROW<InternalException>("Call dlMsprofRegTypeInfo failed, return[%d]", ret);
1127 : }
1128 : }
1129 6 : }
1130 :
1131 2 : void ProfilingHandler::ReportStoragedTaskApi()
1132 : {
1133 2 : std::lock_guard<std::mutex> lock(cachedTaskApiInfoMutex_);
1134 6 : HCCL_INFO("[ReportStoragedTaskApi] taskApiQueueSize is [%zu]", cachedTaskApiInfo_.size());
1135 2 : if (!cachedTaskApiInfo_.empty()) {
1136 2 : std::queue<MsprofApi> tempTaskApi = cachedTaskApiInfo_;
1137 10 : while (!tempTaskApi.empty()) {
1138 8 : MsprofApi reportData = tempTaskApi.front();
1139 8 : tempTaskApi.pop();
1140 8 : s32 ret = DlProfFunction::GetInstance().dlMsprofReportApi(0, &reportData);
1141 8 : if (ret != 0) {
1142 0 : THROW<InternalException>("Call dlMsprofReportApi failed, return[%d]", ret);
1143 : }
1144 : }
1145 2 : }
1146 2 : }
1147 :
1148 2 : void ProfilingHandler::StartHostHcclOpSubscribe() {
1149 2 : enableHcclNode_ = true;
1150 2 : enableHcclL0_ = true;
1151 2 : CallProfRegHcclOpApi();
1152 2 : ReportStoragedCompactInfo();
1153 6 : HCCL_RUN_INFO("StartHostHcclOpSubscribe:enableHcclNode_[%d],enableHcclL0_[%d]", enableHcclNode_, enableHcclL0_);
1154 2 : }
1155 :
1156 2 : void ProfilingHandler::CallProfRegHcclOpApi() const
1157 : {
1158 2 : if (enableHcclL0_ == false) {
1159 0 : HCCL_INFO("[ProfilingHandler] enableHcclNode_ is false.");
1160 0 : return;
1161 : }
1162 44 : for (auto i = 0; i < OpType::__COUNT__; ++i) {
1163 42 : OpType type(static_cast<OpType::Value>(i));
1164 84 : s32 ret = DlProfFunction::GetInstance().dlMsprofRegTypeInfo(
1165 42 : MSPROF_REPORT_HCCL_NODE_LEVEL, static_cast<uint32_t>(type) + MSPROF_REPORT_ACL_HOST_HCCL_BASE_TYPE,
1166 84 : type.Describe().c_str());
1167 42 : if (ret != 0) {
1168 0 : THROW<InternalException>("[ProfilingHandler]Call MsprofReportApi fail, return[%d]", ret);
1169 : }
1170 : }
1171 2 : s32 ret = DlProfFunction::GetInstance().dlMsprofRegTypeInfo(MSPROF_REPORT_NODE_LEVEL,
1172 : MSPROF_REPORT_NODE_MC2_COMMINFO_TYPE, "mc2_comm_info");
1173 2 : if (ret != 0) {
1174 0 : THROW<InternalException>("[ProfilingHandler]Call MsprofRegTypeInfo fail, return[%d]", ret);
1175 : }
1176 : }
1177 :
1178 2 : void ProfilingHandler::StartAdditionInfoSubscribe()
1179 : {
1180 2 : enableHcclL1_ = true;
1181 2 : ReportStoragedAdditionInfo();
1182 6 : HCCL_RUN_INFO("StartAdditionInfoSubscribe:enableHcclL1_[%d]", enableHcclL1_);
1183 2 : }
1184 :
1185 2 : void ProfilingHandler::ReportStoragedAdditionInfoLog() const
1186 : {
1187 2 : if (LIKELY(HcclCheckLogLevel(HCCL_LOG_INFO) == 0)) {
1188 0 : return;
1189 : }
1190 2 : if (cacheTaskInfos_.empty()) {
1191 0 : return;
1192 : }
1193 4 : for (const auto &taskInfo : cacheTaskInfos_) {
1194 6 : HCCL_INFO("ReportStoragedAdditionInfoLog, taskType[%u]", GetTaskTypeValue(taskInfo.taskParam_.taskType));
1195 : }
1196 : }
1197 :
1198 2 : void ProfilingHandler::ReportStoragedAdditionInfo()
1199 : {
1200 2 : std::lock_guard<std::mutex> lock(cacheTaskInfosMutex_);
1201 2 : ReportStoragedAdditionInfoLog();
1202 2 : if (cacheTaskInfos_.empty()) {
1203 0 : HCCL_INFO("[ProfilingHandler]ReportStoragedAdditionInfo cacheTaskInfos_ is empty.");
1204 0 : return;
1205 : }
1206 4 : for (auto &taskInfo : cacheTaskInfos_) {
1207 2 : MsprofAdditionalInfo reporterData{};
1208 2 : FillTaskAdditionInfo(taskInfo, reporterData);
1209 2 : CallAdditionInfo(reporterData);
1210 : }
1211 2 : }
1212 :
1213 2 : void ProfilingHandler::StartCcuSubscribe()
1214 : {
1215 2 : enableHcclNode_ = true;
1216 2 : enableHcclL1_ = true;
1217 6 : HCCL_INFO("ProfilingHandler StartCcuSubscribe");
1218 : const std::vector<std::pair<uint32_t, std::string>> ccuInfoTypes
1219 : = {{MSPROF_REPORT_CCU_TASK_INFO, "ccu_task_info"},
1220 : {MSPROF_REPORT_CCU_WAIT_SIGNAL_INFO, "ccu_wait_signal_info"},
1221 10 : {MSPROF_REPORT_CCU_GROUP_INFO, "ccu_group_info"}};
1222 8 : for (auto &it : ccuInfoTypes) {
1223 6 : s32 ret = DlProfFunction::GetInstance().dlMsprofRegTypeInfo(MSPROF_REPORT_HCCL_NODE_LEVEL, it.first,
1224 : it.second.c_str());
1225 6 : if (ret != 0) {
1226 0 : THROW<InternalException>("Call dlMsprofRegTypeInfo failed, return[%d]", ret);
1227 : }
1228 : }
1229 2 : std::lock_guard<std::mutex> lock(cacheTaskInfosMutex_);
1230 2 : if (cacheTaskInfos_.empty()) {
1231 0 : HCCL_INFO("[ProfilingHandler]StartL2Subscribe cacheTaskInfos_ is empty.");
1232 0 : return;
1233 : }
1234 4 : for (auto &taskInfo : cacheTaskInfos_) {
1235 2 : ReportCcuInfo(taskInfo);
1236 : }
1237 4 : }
1238 :
1239 1 : void ProfilingHandler::ProfilingHandler::StopSubscribe()
1240 : {
1241 1 : enableHostApi_ = false;
1242 1 : enableHcclNode_ = false;
1243 1 : enableHcclL0_ = false;
1244 1 : enableHcclL1_ = false;
1245 3 : HCCL_RUN_INFO("[ProfilingHandler]StopSubscribe.");
1246 1 : }
1247 :
1248 53 : bool ProfilingHandler::GetHostApiState() const
1249 : {
1250 53 : return enableHostApi_;
1251 : }
1252 53 : bool ProfilingHandler::GetHcclNodeState() const
1253 : {
1254 53 : return enableHcclNode_;
1255 : }
1256 53 : bool ProfilingHandler::GetHcclL0State() const
1257 : {
1258 53 : return enableHcclL0_;
1259 : }
1260 :
1261 62 : bool ProfilingHandler::GetHcclL1State() const
1262 : {
1263 62 : return enableHcclL1_;
1264 : }
1265 :
1266 114 : uint64_t ProfilingHandler::GetProfHashId(const char *name, uint32_t len) const
1267 : {
1268 114 : if (name == nullptr || len == 0) {
1269 78 : HCCL_WARNING("HashData is empty. name:%s, len:%u", name, len);
1270 26 : return INVALID_U64;
1271 : }
1272 88 : if (DlProfFunction::GetInstance().dlMsprofStr2Id == nullptr) {
1273 0 : HCCL_WARNING("HashFunc is nullPtr");
1274 0 : return INVALID_U64;
1275 : }
1276 88 : return DlProfFunction::GetInstance().dlMsprofStr2Id(name, len);
1277 : }
1278 :
1279 2 : void ProfilingHandler::ReportHcclMC2CommInfoLog([[maybe_unused]] const Stream &kfcStream, const Stream &stream,
1280 : const std::vector<Stream *> &aicpuStreams, [[maybe_unused]] const std::string &id,
1281 : [[maybe_unused]] RankId myRank, [[maybe_unused]] u32 rankSize, [[maybe_unused]] RankId rankInParentComm) const
1282 : {
1283 2 : if (LIKELY(HcclCheckLogLevel(HCCL_LOG_INFO) == 0)) {
1284 0 : return;
1285 : }
1286 2 : if (aicpuStreams.empty()) {
1287 3 : HCCL_INFO("only exist main stream, streamId(sqId):%u", stream.GetSqId());
1288 1 : return;
1289 : }
1290 1 : uint32_t reportId = 0;
1291 2 : for (uint32_t streamIndex = 0; streamIndex < aicpuStreams.size(); streamIndex++) {
1292 3 : HCCL_INFO("streamIndex:%u, reportId:%u, streamId(sqId):%u", streamIndex, reportId, aicpuStreams[streamIndex]->GetSqId());
1293 1 : reportId++;
1294 : }
1295 : }
1296 :
1297 2 : void ProfilingHandler::ReportHcclMC2CommInfo(const Stream &kfcStream, const Stream &stream,
1298 : const std::vector<Stream *> &aicpuStreams, const std::string &id,
1299 : RankId myRank, u32 rankSize, RankId rankInParentComm)
1300 : {
1301 2 : ReportHcclMC2CommInfoLog(kfcStream, stream, aicpuStreams, id, myRank, rankSize, rankInParentComm);
1302 : ProfilingDeviceCommResInfo hcclMc2Info;
1303 2 : hcclMc2Info.groupName = GetProfHashId(id.c_str(), id.length());
1304 2 : hcclMc2Info.rankSize = rankSize;
1305 2 : hcclMc2Info.rankId = myRank;
1306 2 : hcclMc2Info.usrRankId = rankInParentComm;
1307 2 : hcclMc2Info.aicpuKfcStreamId = static_cast<uint32_t>(kfcStream.GetSqId());
1308 2 : hcclMc2Info.reserve = 0;
1309 2 : const uint32_t ONCE_REPORT_STREAM_NUM_MAX = 8;
1310 3 : for (uint32_t streamIndex = 0, reportId = 0; streamIndex < aicpuStreams.size(); streamIndex++) {
1311 1 : hcclMc2Info.commStreamIds[reportId++] = aicpuStreams[streamIndex]->GetSqId();
1312 1 : if (reportId == ONCE_REPORT_STREAM_NUM_MAX) {
1313 0 : hcclMc2Info.commStreamSize = reportId;
1314 0 : ReportMc2AdditionInfo(DlProfFunction::GetInstance().dlMsprofSysCycleTime(), &hcclMc2Info, sizeof(hcclMc2Info));
1315 0 : reportId = 0;
1316 : }
1317 1 : if (streamIndex == (aicpuStreams.size() - 1)) {
1318 1 : hcclMc2Info.commStreamIds[reportId++] = stream.GetSqId();
1319 1 : hcclMc2Info.commStreamSize = reportId;
1320 1 : ReportMc2AdditionInfo(DlProfFunction::GetInstance().dlMsprofSysCycleTime(), &hcclMc2Info,
1321 : sizeof(hcclMc2Info));
1322 1 : reportId = 0;
1323 : }
1324 : }
1325 2 : if (aicpuStreams.empty()) {
1326 1 : hcclMc2Info.commStreamIds[0] = stream.GetSqId();
1327 1 : hcclMc2Info.commStreamSize = 1;
1328 1 : ReportMc2AdditionInfo(DlProfFunction::GetInstance().dlMsprofSysCycleTime(), &hcclMc2Info, sizeof(hcclMc2Info));
1329 : }
1330 2 : }
1331 :
1332 1 : void ProfilingHandler::ReportHcclMC2CommInfoLog([[maybe_unused]] const u32 kfcStreamId,
1333 : const std::vector<u32> &aicpuStreamsId, const std::string &id,
1334 : [[maybe_unused]] RankId myRank, [[maybe_unused]] u32 rankSize, [[maybe_unused]] RankId rankInParentComm) const
1335 : {
1336 1 : if (LIKELY(HcclCheckLogLevel(HCCL_LOG_INFO) == 0)) {
1337 0 : return;
1338 : }
1339 1 : uint64_t groupName = GetProfHashId(id.c_str(), id.length());
1340 1 : uint32_t reportId = 0;
1341 4 : for (uint32_t streamIndex = 0; streamIndex < aicpuStreamsId.size(); streamIndex++) {
1342 9 : HCCL_INFO("streamIndex:[%u], reportId:[%u], streamId:[%u] id [%s] hcclMC2Info.groupName:[%llu]", streamIndex,
1343 : reportId, aicpuStreamsId[streamIndex], id.c_str(), groupName);
1344 3 : reportId++;
1345 : }
1346 : }
1347 :
1348 0 : void ProfilingHandler::ReportHcclMC2CommInfo(const u32 kfcStreamId,
1349 : const std::vector<u32> &aicpuStreamsId, const std::string &id,
1350 : RankId myRank, u32 rankSize, RankId rankInParentComm)
1351 : {
1352 0 : ReportHcclMC2CommInfoLog(kfcStreamId, aicpuStreamsId, id, myRank, rankSize, rankInParentComm);
1353 : ProfilingDeviceCommResInfo hcclMc2Info;
1354 0 : hcclMc2Info.groupName = GetProfHashId(id.c_str(),id.length());
1355 0 : hcclMc2Info.rankSize = rankSize;
1356 0 : hcclMc2Info.rankId = myRank;
1357 0 : hcclMc2Info.usrRankId = rankInParentComm;
1358 0 : hcclMc2Info.aicpuKfcStreamId = static_cast<uint32_t>(kfcStreamId);
1359 0 : hcclMc2Info.reserve = 0;
1360 :
1361 0 : const uint32_t ONCE_REPORT_STREAM_NUM_MAX = 8;
1362 0 : uint32_t reportId = 0;
1363 0 : for (uint32_t streamIndex = 0; streamIndex < aicpuStreamsId.size(); streamIndex++) {
1364 0 : hcclMc2Info.commStreamIds[reportId++] = aicpuStreamsId[streamIndex];
1365 0 : if (reportId == ONCE_REPORT_STREAM_NUM_MAX) {
1366 0 : hcclMc2Info.commStreamSize = reportId;
1367 0 : ReportMc2AdditionInfo(DlProfFunction::GetInstance().dlMsprofSysCycleTime(), &hcclMc2Info, sizeof(hcclMc2Info));
1368 0 : reportId = 0;
1369 : }
1370 : }
1371 0 : if (reportId > 0) {
1372 0 : hcclMc2Info.commStreamSize = reportId;
1373 0 : ReportMc2AdditionInfo(DlProfFunction::GetInstance().dlMsprofSysCycleTime(), &hcclMc2Info,
1374 : sizeof(hcclMc2Info));
1375 0 : reportId = 0;
1376 : }
1377 0 : }
1378 2 : void ProfilingHandler::ReportMc2AdditionInfo(uint64_t timeStamp, const void *data, int len)
1379 : {
1380 2 : MsprofAdditionalInfo reporterData{};
1381 2 : reporterData.level = MSPROF_REPORT_NODE_LEVEL;
1382 2 : reporterData.type = MSPROF_REPORT_NODE_MC2_COMMINFO_TYPE;
1383 2 : reporterData.threadId = SalGetTid();
1384 2 : reporterData.dataLen = len;
1385 2 : reporterData.timeStamp = timeStamp;
1386 2 : s32 sret = memcpy_s(reporterData.data, sizeof(reporterData.data), data, len);
1387 2 : if (sret != EOK) {
1388 0 : THROW<InternalException>("Call memcpy_s failed, errorno[%d]", sret);
1389 : }
1390 6 : HCCL_INFO("[ProfilingHandler][ReportMc2CommInfo], level [%u], type[%u], threadId[%u], dataLen[%u], timeStamp[%llu]",
1391 : reporterData.level, reporterData.type, reporterData.threadId, reporterData.dataLen,
1392 : reporterData.timeStamp);
1393 2 : if (!enableHostApi_) {
1394 2 : std::lock_guard<std::mutex> lock(cacheHcclAdditionInfoMutex_);
1395 2 : cacheHcclAdditionInfo_.push(reporterData);
1396 2 : return;
1397 2 : }
1398 : s32 ret
1399 0 : = DlProfFunction::GetInstance().dlMsprofReportAdditionalInfo(1, &reporterData, sizeof(MsprofAdditionalInfo));
1400 0 : HCCL_INFO("Call MsprofReportAdditionalInfo, return value[%d]", ret);
1401 0 : if (ret != 0) {
1402 0 : THROW<InternalException>("Call MsprofReportAdditionalInfo failed, return[%d]", ret);
1403 : }
1404 : }
1405 :
1406 : } // namespace Hccl
|
