Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 : #include "profiling_handler_lite.h"
11 : #include "exception_util.h"
12 : #include "internal_exception.h"
13 : #include "sal.h"
14 : #include "task_info.h"
15 : #include "task_param.h"
16 : #include "communicator_impl_lite.h"
17 : #include "dlprof_function.h"
18 : namespace Hccl {
19 :
20 : static constexpr u32 aging = 1;
21 : constexpr std::uint32_t HCCLINFO_REPORT_BATCH_NUM = 2;
22 : ProfilingHandlerLite ProfilingHandlerLite::instance_;
23 :
24 1 : ProfilingHandlerLite::ProfilingHandlerLite()
25 : {
26 1 : }
27 :
28 1 : ProfilingHandlerLite::~ProfilingHandlerLite()
29 : {
30 1 : }
31 :
32 122 : ProfilingHandlerLite &ProfilingHandlerLite::GetInstance()
33 : {
34 122 : return instance_;
35 : }
36 :
37 0 : void ProfilingHandlerLite::SetCachedCclTag()
38 : {
39 0 : for (const auto &item : CMD_OP_TYPE_INFO_MAP) {
40 0 : const string &tag = item.second.second;
41 0 : if (cachedCclTag_.find(tag) == cachedCclTag_.end()) {
42 0 : const std::string &cclTag = item.second.second;
43 0 : cachedCclTag_[tag] = GetProfHashId(cclTag.c_str(), cclTag.length());
44 : }
45 : }
46 0 : }
47 :
48 0 : void ProfilingHandlerLite::SetCachedGroupName(const std::string &groupName, u32 rankSize)
49 : {
50 0 : cachedGroupName_ = GetProfHashId(groupName.c_str(), groupName.length());
51 0 : cachedRankSize_ = rankSize;
52 0 : }
53 :
54 5 : HcclResult ProfilingHandlerLite::Init()
55 : {
56 5 : if (initializedFlag_) {
57 0 : return HCCL_SUCCESS;
58 : }
59 5 : cachedTid_ = SalGetTid();
60 5 : if (MsprofReportBatchAdditionalInfo == nullptr) {
61 5 : if (AdprofReportAdditionalInfo != nullptr) {
62 5 : reportAdditionalInfo_ = AdprofReportAdditionalInfo;
63 : }
64 5 : if (AdprofGetHashId != nullptr) {
65 0 : getProfHashId_ = AdprofGetHashId;
66 : }
67 5 : if (AdprofReportBatchAdditionalInfo != nullptr) {
68 0 : reportBatchAdditionalInfo_ = AdprofReportBatchAdditionalInfo;
69 : }
70 : } else {
71 0 : if (MsprofReportAdditionalInfo != nullptr) {
72 0 : reportAdditionalInfo_ = [](uint32_t flag, const void* data, uint32_t len) -> int32_t {
73 0 : return MsprofReportAdditionalInfo(flag, const_cast<void*>(data), len);
74 : };
75 : }
76 0 : if (MsprofStr2Id != nullptr) {
77 0 : getProfHashId_ = MsprofStr2Id;
78 : }
79 0 : reportBatchAdditionalInfo_ = [](uint32_t flag, const void* data, uint32_t len) -> int32_t {
80 0 : return MsprofReportBatchAdditionalInfo(flag, const_cast<void*>(data), len);
81 : };
82 : }
83 5 : if (reportAdditionalInfo_ == nullptr) {
84 0 : HCCL_ERROR("[ProfilingHandlerLite][Init] reportAdditionalInfo_ is nullptr, profiling report will be skipped");
85 0 : return HCCL_E_PROFILING;
86 : }
87 5 : if (getProfHashId_ == nullptr) {
88 15 : HCCL_ERROR("[ProfilingHandlerLite][Init] getProfHashId_ is nullptr, profiling hash will be invalid");
89 5 : return HCCL_E_PROFILING;
90 : }
91 0 : for (auto i = 0; i < static_cast<int>(TaskParamType::__COUNT__); i++) {
92 0 : TaskParamType type(static_cast<TaskParamType::Value>(i));
93 0 : std::string nameInfo = GetProfTaskOpNameV2(type);
94 0 : taskTypeHashCache_[static_cast<uint32_t>(i)] = GetProfHashId(nameInfo.c_str(), nameInfo.length());
95 0 : }
96 0 : cachedAlgTypeHashId_ = GetProfHashId("AlgType::NHR", strlen("AlgType::NHR"));
97 0 : SetCachedCclTag();
98 0 : initializedFlag_ = true;
99 0 : return HCCL_SUCCESS;
100 : }
101 :
102 9 : void ProfilingHandlerLite::ReportHcclOpInfo(const DfxOpInfo &opInfo) const
103 : {
104 9 : if (!GetProfL0State()) {
105 15 : HCCL_INFO("[ProfilingHandlerLite][ReportHcclOpInfo] l0 is false.");
106 5 : return;
107 : }
108 4 : if (aicpu::GetTaskAndStreamId == nullptr) {
109 0 : HCCL_WARNING("[ProfilingHandlerLite][ReportHcclOpInfo] GetTaskAndStreamId is nullptr.");
110 0 : return;
111 : }
112 4 : uint64_t taskId = 0U;
113 4 : uint32_t streamId = 0;
114 4 : if (aicpu::GetTaskAndStreamId(taskId, streamId) != aicpu::status_t::AICPU_ERROR_NONE) {
115 0 : THROW<InternalException>("[ProfilingHandler] Failed to get task id and stream id.");
116 : }
117 4 : if (taskId > static_cast<uint64_t>(std::numeric_limits<uint32_t>::max())) {
118 0 : THROW<InvalidParamsException>("[ProfilingHandler] taskId is larger than u32.");
119 : }
120 4 : MsprofAdditionalInfo reporterData{};
121 4 : reporterData.level = MSPROF_REPORT_AICPU_LEVEL;
122 4 : reporterData.type = MSPROF_REPORT_AICPU_HCCL_OP_INFO;
123 4 : reporterData.threadId = cachedTid_;
124 4 : reporterData.dataLen = sizeof(MsprofAicpuHCCLOPInfo);
125 4 : reporterData.timeStamp = ProfGetCurCpuTimestamp();
126 4 : auto *hcclOpInfo = reinterpret_cast<MsprofAicpuHCCLOPInfo *>(reporterData.data);
127 4 : hcclOpInfo->algType = cachedAlgTypeHashId_;
128 :
129 4 : hcclOpInfo->taskId = static_cast<uint32_t>(taskId);
130 4 : hcclOpInfo->streamId = streamId;
131 4 : hcclOpInfo->count = opInfo.op_.dataCount;
132 4 : hcclOpInfo->dataType = opInfo.op_.dataType;
133 4 : hcclOpInfo->groupName = cachedGroupName_;
134 4 : hcclOpInfo->ranksize = cachedRankSize_;
135 12 : HCCL_INFO("[ProfilingHandlerLite][ReportHcclOpInfo] relay:%d, retry:%d, dataType:%s, algType:%llu, count:%llu, "
136 : "groupName:%llu, ranksize:%u, taskId:%u, streamId:%u",
137 : hcclOpInfo->relay, hcclOpInfo->retry, DataTypeToSerialString(hcclOpInfo->dataType).c_str(), hcclOpInfo->algType, hcclOpInfo->count,
138 : hcclOpInfo->groupName, hcclOpInfo->ranksize, hcclOpInfo->taskId, hcclOpInfo->streamId);
139 4 : ReportAdditionInfo(reporterData);
140 : }
141 :
142 5 : void ProfilingHandlerLite::ReportHcclTaskDetailsLog(const std::vector<TaskInfo *> &taskInfo) const
143 : {
144 5 : if (LIKELY(HcclCheckLogLevel(HCCL_LOG_INFO) == 0)) {
145 0 : return;
146 : }
147 13 : for (std::vector<Hccl::TaskInfo *>::size_type i = 0; i < taskInfo.size(); i++) {
148 8 : DumpTaskDetails(MsprofAicpuHcclTaskInfo{}, taskInfo[i]);
149 : }
150 : }
151 :
152 6 : bool ProfilingHandlerLite::FillBatchReporterData(uint32_t batchId, const MsprofAicpuHcclTaskInfo *taskInfos,
153 : MsprofAdditionalInfo &addInfo) const
154 : {
155 6 : addInfo.level = MSPROF_REPORT_AICPU_LEVEL;
156 6 : addInfo.type = MSPROF_REPORT_AICPU_MC2_BATCH_HCCL_INFO;
157 6 : addInfo.threadId = cachedTid_;
158 6 : addInfo.timeStamp = 0;
159 6 : addInfo.dataLen = sizeof(MsprofAicpuHcclTaskInfo) * batchId;
160 6 : s32 sret = memcpy_s(addInfo.data, sizeof(addInfo.data), taskInfos, addInfo.dataLen);
161 6 : if (sret != 0) {
162 0 : HCCL_WARNING("[ProfilingHandlerLite][FillBatchReporterData] memcpy failed, sret[%d]", sret);
163 0 : return false;
164 : }
165 6 : return true;
166 : }
167 :
168 0 : bool ProfilingHandlerLite::ReportBatchAddInfo(uint32_t batchId, const MsprofAicpuHcclTaskInfo *taskInfos,
169 : MsprofAdditionalInfo *addInfoVec, uint32_t &addInfoIndx, uint32_t maxBatchNum, bool isLastBatch) const
170 : {
171 0 : if (!FillBatchReporterData(batchId, taskInfos, addInfoVec[addInfoIndx])) { return false; }
172 0 : addInfoIndx++;
173 0 : if (addInfoIndx == maxBatchNum || isLastBatch) {
174 0 : if (reportBatchAdditionalInfo_(aging, addInfoVec, addInfoIndx * sizeof(MsprofAdditionalInfo)) != 0) {
175 0 : HCCL_WARNING("[ProfilingHandlerLite][ReportHcclTaskDetails] reportBatchAdditionalInfo failed");
176 0 : return false;
177 : }
178 0 : addInfoIndx = 0;
179 : }
180 0 : return true;
181 : }
182 :
183 5 : void ProfilingHandlerLite::ReportHcclTaskDetails(const std::vector<TaskInfo *> &taskInfo) const
184 : {
185 5 : ReportHcclTaskDetailsLog(taskInfo);
186 5 : MsprofAicpuHcclTaskInfo taskInfos[HCCLINFO_REPORT_BATCH_NUM] = {};
187 5 : bool isSupportBatchReport = (reportBatchAdditionalInfo_ != nullptr);
188 15 : HCCL_INFO("[ProfilingHandlerLite][ReportHcclTaskDetails] isSupportBatchReport: %s, taskInfo.size: %zu",
189 : isSupportBatchReport ? "true" : "false", taskInfo.size());
190 5 : constexpr int32_t MAX_BATCH_REPORT_NUM = 512;
191 2560 : MsprofAdditionalInfo addInfoVec[MAX_BATCH_REPORT_NUM] = {};
192 5 : uint32_t addInfoIndx = 0;
193 5 : uint32_t batchId = 0;
194 13 : for (std::vector<Hccl::TaskInfo *>::size_type i = 0; i < taskInfo.size(); i++) {
195 8 : GetTaskDetailInfos(taskInfo[i], taskInfos[batchId++]);
196 8 : if (batchId == HCCLINFO_REPORT_BATCH_NUM || i == taskInfo.size() - 1) {
197 6 : if (!isSupportBatchReport) {
198 6 : MsprofAdditionalInfo reporterData{};
199 6 : if (!FillBatchReporterData(batchId, taskInfos, reporterData)) { return; }
200 6 : ReportAdditionInfo(reporterData);
201 : } else {
202 0 : if (!ReportBatchAddInfo(batchId, taskInfos, addInfoVec, addInfoIndx,
203 0 : MAX_BATCH_REPORT_NUM, i == taskInfo.size() - 1)) { return; }
204 : }
205 6 : batchId = 0;
206 6 : memset_s(taskInfos, sizeof(taskInfos), 0, sizeof(taskInfos));
207 : }
208 : }
209 : }
210 :
211 8 : void ProfilingHandlerLite::FillTaskParamDetails(const TaskInfo *it, MsprofAicpuHcclTaskInfo &taskDetailsInfos) const
212 : {
213 8 : const auto &taskPara = it->taskParam_.taskPara;
214 8 : switch (it->taskParam_.taskType) {
215 1 : case TaskParamType::TASK_SDMA:
216 : case TaskParamType::TASK_RDMA:
217 : case TaskParamType::TASK_UB_INLINE_WRITE:
218 : case TaskParamType::TASK_WRITE_WITH_NOTIFY:
219 : case TaskParamType::TASK_UB:
220 1 : taskDetailsInfos.srcAddr = static_cast<u64>(reinterpret_cast<uintptr_t>(taskPara.DMA.src));
221 1 : taskDetailsInfos.dstAddr = static_cast<u64>(reinterpret_cast<uintptr_t>(taskPara.DMA.dst));
222 1 : taskDetailsInfos.dataSize = static_cast<u32>(taskPara.DMA.size);
223 1 : taskDetailsInfos.notifyID = taskPara.DMA.notifyID;
224 1 : taskDetailsInfos.linkType = static_cast<uint16_t>(taskPara.DMA.linkType);
225 1 : break;
226 1 : case TaskParamType::TASK_REDUCE_INLINE:
227 : case TaskParamType::TASK_REDUCE_TBE:
228 : case TaskParamType::TASK_UB_REDUCE_INLINE:
229 : case TaskParamType::TASK_WRITE_REDUCE_WITH_NOTIFY:
230 1 : taskDetailsInfos.srcAddr = static_cast<u64>(reinterpret_cast<uintptr_t>(taskPara.Reduce.src));
231 1 : taskDetailsInfos.dstAddr = static_cast<u64>(reinterpret_cast<uintptr_t>(taskPara.Reduce.dst));
232 1 : taskDetailsInfos.dataSize = static_cast<u32>(taskPara.Reduce.size);
233 1 : taskDetailsInfos.notifyID = taskPara.Reduce.notifyID;
234 1 : taskDetailsInfos.dataType = static_cast<uint16_t>(taskPara.Reduce.dataType);
235 1 : taskDetailsInfos.linkType = static_cast<uint16_t>(taskPara.Reduce.linkType);
236 1 : taskDetailsInfos.opType = taskPara.Reduce.reduceOp;
237 1 : break;
238 5 : case TaskParamType::TASK_NOTIFY_RECORD:
239 : case TaskParamType::TASK_NOTIFY_WAIT:
240 5 : taskDetailsInfos.notifyID = taskPara.Notify.notifyID;
241 5 : break;
242 1 : default:
243 1 : break;
244 : }
245 8 : }
246 :
247 8 : void ProfilingHandlerLite::GetTaskDetailInfos(const TaskInfo *it, MsprofAicpuHcclTaskInfo &taskDetailsInfos) const
248 : {
249 8 : constexpr u32 INVALID_LOCAL_RANK = UINT32_MAX;
250 8 : constexpr u32 INVALID_REMOTE_RANK = UINT32_MAX;
251 8 : taskDetailsInfos.localRank = INVALID_LOCAL_RANK;
252 8 : taskDetailsInfos.remoteRank = INVALID_REMOTE_RANK;
253 8 : if (it->dfxOpInfo_ != nullptr) {
254 7 : auto cacheIt = taskTypeHashCache_.find(static_cast<uint32_t>(it->taskParam_.taskType));
255 7 : taskDetailsInfos.itemId = (cacheIt != taskTypeHashCache_.end()) ? cacheIt->second : INVALID_U64;
256 7 : auto cclTagIt = cachedCclTag_.find(it->dfxOpInfo_->tag_);
257 7 : taskDetailsInfos.cclTag = (cclTagIt != cachedCclTag_.end()) ? cclTagIt->second : INVALID_U64;
258 7 : taskDetailsInfos.remoteRank = it->GetRemoteRankId();
259 7 : taskDetailsInfos.groupName = cachedGroupName_;
260 7 : taskDetailsInfos.rankSize = cachedRankSize_;
261 7 : taskDetailsInfos.localRank = it->dfxOpInfo_->op_.myRank;
262 7 : taskDetailsInfos.stage = 0;
263 : } else {
264 3 : HCCL_WARNING("[ProfilingHandlerLite::GetTaskDetailInfos] dfxOpInfo_ is nullptr!");
265 1 : taskDetailsInfos.remoteRank = taskDetailsInfos.localRank;
266 : }
267 8 : taskDetailsInfos.stage = 0;
268 8 : FillTaskParamDetails(it, taskDetailsInfos);
269 8 : taskDetailsInfos.timeStamp = ProfGetCurCpuTimestamp();
270 8 : taskDetailsInfos.durationEstimated = 0;
271 8 : taskDetailsInfos.taskId = it->taskId_;
272 8 : taskDetailsInfos.streamId = it->streamId_;
273 8 : taskDetailsInfos.planeID = 0;
274 8 : taskDetailsInfos.transportType = static_cast<int32_t>(SimpleTaskType::UB);
275 8 : if (taskDetailsInfos.remoteRank == INVALID_VALUE_RANKID) {
276 1 : taskDetailsInfos.transportType = static_cast<int32_t>(SimpleTaskType::LOCAL);
277 1 : taskDetailsInfos.remoteRank = taskDetailsInfos.localRank;
278 : }
279 8 : taskDetailsInfos.role = static_cast<uint32_t>(TaskRole::DST);
280 8 : taskDetailsInfos.workFlowMode = static_cast<uint32_t>(HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE);
281 8 : }
282 :
283 8 : void ProfilingHandlerLite::DumpTaskDetails([[maybe_unused]] const MsprofAicpuHcclTaskInfo &taskDetailsInfos, const TaskInfo *taskInfo) const
284 : {
285 24 : HCCL_INFO("[ProfilingHandlerLite] DumpTaskDetails %s", taskInfo->Describe().c_str());
286 8 : }
287 :
288 9 : void ProfilingHandlerLite::ReportMainStreamTask(const FlagTaskInfo &flagTaskInfo) const
289 : {
290 9 : if (!GetProfL0State()) {
291 24 : HCCL_INFO("[ProfilingHandlerLite][ReportMainStreamTask] l0 is false.");
292 8 : return;
293 : }
294 1 : if (aicpu::GetTaskAndStreamId == nullptr) {
295 0 : HCCL_WARNING("[ProfilingHandlerLite][ReportMainStreamTask] aicpu::GetTaskAndStreamId is nullptr.");
296 0 : return;
297 : }
298 1 : uint64_t aicpuKernelTaskId = 0U;
299 1 : uint32_t aicpuKernelStreamId = 0;
300 1 : if (aicpu::GetTaskAndStreamId(aicpuKernelTaskId, aicpuKernelStreamId) != aicpu::status_t::AICPU_ERROR_NONE) {
301 0 : THROW<InternalException>("[ProfilingHandler] Failed to get task id and stream id.");
302 : }
303 1 : if (aicpuKernelTaskId > static_cast<uint64_t>(std::numeric_limits<uint32_t>::max())) {
304 0 : THROW<InvalidParamsException>("[ProfilingHandler] aicpuKernelTaskId is larger than u32.");
305 : }
306 1 : MsprofAdditionalInfo reporterData{};
307 1 : reporterData.level = MSPROF_REPORT_AICPU_LEVEL;
308 1 : reporterData.type = MSPROF_REPORT_AICPU_HCCL_FLAG_TASK;
309 1 : reporterData.threadId = cachedTid_;
310 1 : reporterData.dataLen = sizeof(MsprofAicpuHcclMainStreamTask);
311 1 : reporterData.timeStamp = ProfGetCurCpuTimestamp();
312 1 : constexpr uint32_t UINT16_BIT_WIDTH = std::numeric_limits<uint16_t>::digits;
313 1 : auto *flagtask = reinterpret_cast<MsprofAicpuHcclMainStreamTask *>(reporterData.data);
314 1 : flagtask->taskId = static_cast<uint16_t>(flagTaskInfo.taskId >> UINT16_BIT_WIDTH);
315 1 : flagtask->streamId = static_cast<uint16_t>(flagTaskInfo.taskId);
316 1 : flagtask->type = flagTaskInfo.type;
317 1 : uint32_t aicpuKernelTaskIdLow32 = static_cast<uint32_t>(aicpuKernelTaskId);
318 1 : flagtask->aicpuTaskId = static_cast<uint16_t>(aicpuKernelTaskIdLow32 >> UINT16_BIT_WIDTH);
319 1 : flagtask->aicpuStreamId = static_cast<uint16_t>(aicpuKernelTaskIdLow32);
320 3 : HCCL_INFO("[ProfilingHandlerLite][ReportMainStreamTask] streamId:%u, taskId:%u, type:%u,"
321 : "aicpuStreamId:%u, aicpuTaskId:%u",
322 : flagtask->streamId, flagtask->taskId, flagtask->type, flagtask->aicpuStreamId, flagtask->aicpuTaskId);
323 1 : ReportAdditionInfo(reporterData);
324 : }
325 :
326 11 : void ProfilingHandlerLite::ReportAdditionInfo(const MsprofAdditionalInfo& reporterData) const
327 : {
328 11 : if (reportAdditionalInfo_(aging, &reporterData, sizeof(MsprofAdditionalInfo)) != 0) {
329 1 : THROW<InternalException>("[ProfilingHandler] ReportAdditionalInfo failed.");
330 : }
331 10 : }
332 :
333 5 : void ProfilingHandlerLite::UpdateProfSwitch()
334 : {
335 5 : IsL1fromOffToOn();
336 5 : IsProfSwitchOn(ProfilingLevel::L0);
337 5 : IsProfSwitchOn(ProfilingLevel::L1);
338 5 : }
339 :
340 18 : bool ProfilingHandlerLite::IsProfOn(uint64_t feature) const
341 : {
342 18 : if (MsprofReportBatchAdditionalInfo == nullptr) {
343 18 : if (AdprofCheckFeatureIsOn == nullptr) {
344 18 : return false;
345 : }
346 0 : return AdprofCheckFeatureIsOn(feature) > 0;
347 : } else {
348 0 : if (feature == ADPROF_TASK_TIME_L1) {
349 0 : return enableHcclL1_;
350 0 : } else if (feature == ADPROF_TASK_TIME_L0) {
351 0 : return enableHcclL0_;
352 : }
353 : }
354 :
355 0 : return false;
356 : }
357 :
358 17 : bool ProfilingHandlerLite::IsProfSwitchOn(ProfilingLevel level)
359 : {
360 17 : bool res = false;
361 17 : if (level == ProfilingLevel::L0) {
362 6 : res = IsProfOn(ADPROF_TASK_TIME_L0);
363 6 : enableHcclL0_ = res;
364 11 : } else if (level == ProfilingLevel::L1) {
365 11 : res = IsProfOn(ADPROF_TASK_TIME_L1);
366 11 : enableHcclL1_ = res;
367 : }
368 17 : return res;
369 : }
370 :
371 6 : bool ProfilingHandlerLite::IsL1fromOffToOn()
372 : {
373 6 : if (((!GetProfL1State()) && IsProfSwitchOn(ProfilingLevel::L1))) {
374 0 : HCCL_INFO("Profiling L1 switch form off to on.");
375 0 : return true;
376 : }
377 6 : return false;
378 : }
379 :
380 1 : void ProfilingHandlerLite::SetProL1On(bool val)
381 : {
382 3 : HCCL_INFO("[%s] val = [%d]", __func__, val);
383 1 : enableHcclL1_ = val;
384 1 : }
385 :
386 1 : void ProfilingHandlerLite::SetProL0On(bool val)
387 : {
388 3 : HCCL_INFO("[%s] val = [%d]", __func__, val);
389 1 : enableHcclL0_ = val;
390 1 : }
391 :
392 0 : uint64_t ProfilingHandlerLite::GetProfHashId(const char *name, uint32_t len) const
393 : {
394 0 : if (name == nullptr || len == 0) {
395 0 : HCCL_WARNING("HashData is empty. name:%s, len:%u", name, len);
396 0 : return INVALID_U64;
397 : }
398 0 : return getProfHashId_(name, len);
399 : }
400 :
401 : } // namespace Hccl
|
