Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include <dlog_pub.h>
12 : #include <hccl/hccl_types.h>
13 : #include "hccl/base.h"
14 : #include "adapter_prof.h"
15 : #include "profiling_manager.h"
16 : #include "profiler_manager_impl.h"
17 : #include "stream_utils.h"
18 :
19 : namespace hccl {
20 312 : ProfilerManagerImpl::ProfilerManagerImpl(s32 devicePhyId, s32 deviceLogicId, u32 realUserRank, u32 rankSize)
21 312 : : devicePhyId_(devicePhyId),
22 312 : deviceLogicId_(deviceLogicId),
23 312 : realUserRank_(realUserRank),
24 312 : rankSize_(rankSize),
25 312 : profiler_(nullptr),
26 312 : taskExceptionHandler_(nullptr),
27 312 : taskOverflowHandler_(nullptr)
28 312 : {}
29 312 : ProfilerManagerImpl::~ProfilerManagerImpl() {}
30 :
31 936 : void ProfilerManagerImpl::RegisterCallBack(ProfilerType name, hccl::PluginRunner& callback)
32 : {
33 : // 容器操作并非线程安全的, 加锁
34 936 : std::unique_lock<std::mutex> lock(mutex_);
35 :
36 936 : if (callbacks_.find(name) != callbacks_.end()) {
37 0 : HCCL_WARNING("callback[%d] already registered", name);
38 0 : return;
39 : }
40 :
41 936 : callbacks_.insert(std::make_pair<ProfilerType&, hccl::PluginRunner&>(name, callback));
42 936 : }
43 :
44 312 : HcclResult ProfilerManagerImpl::InitProfiler()
45 : {
46 312 : if (static_cast<s32>(devicePhyId_) == HOST_DEVICE_ID) {
47 0 : return HCCL_SUCCESS;
48 : }
49 312 : CHK_RET(DlProfFunction::GetInstance().DlProfFunctionInit());
50 312 : CHK_RET(DlRtFunction::GetInstance().DlRtFunctionInit());
51 312 : profiler_.reset(new (std::nothrow) TaskProfiling(deviceLogicId_, realUserRank_, rankSize_, true));
52 312 : CHK_SMART_PTR_NULL(profiler_);
53 312 : PluginRunner profrunner(profiler_.get());
54 312 : RegisterCallBack(ProfilerType::TASK_PROFILING, profrunner);
55 :
56 312 : taskExceptionHandler_.reset(new (std::nothrow) TaskExceptionHandler(static_cast<u32>(deviceLogicId_)));
57 312 : CHK_SMART_PTR_NULL(taskExceptionHandler_);
58 312 : PluginRunner runner(taskExceptionHandler_.get());
59 312 : RegisterCallBack(ProfilerType::TASK_EXCEPTION, runner);
60 :
61 : // 记录可能导致算子溢出的task信息
62 312 : taskOverflowHandler_.reset(new (std::nothrow) TaskOverflow(static_cast<u32>(deviceLogicId_)));
63 312 : CHK_SMART_PTR_NULL(taskOverflowHandler_);
64 312 : PluginRunner dumprunner(taskOverflowHandler_.get());
65 312 : RegisterCallBack(ProfilerType::TASK_OVERFLOW, dumprunner);
66 :
67 312 : rtProfCtrlHandle callback = CommandHandle;
68 312 : HcclResult ret = hrtProfRegisterCtrlCallback(HCCL, callback);
69 312 : CHK_PRT_RET(
70 : (ret != HCCL_SUCCESS), HCCL_ERROR("[ProfilerManager][InitProfiler]Register CtrlCallBack failed."), HCCL_E_PARA);
71 :
72 15912 : for (const auto& it : PROF_TASK_OP_NAME) {
73 15600 : std::string nameInfo = it.second;
74 15600 : uint64_t ret = hrtMsprofGetHashId(nameInfo.c_str(), nameInfo.length());
75 15600 : HCCL_DEBUG("[PROF_TASK_OP_NAME] nameInfo[%s] ret[%llu]", nameInfo.c_str(), ret);
76 15600 : }
77 :
78 312 : HCCL_INFO("[ProfilerManager][InitProfiler]Register CtrlCallBack success");
79 :
80 312 : return HCCL_SUCCESS;
81 312 : }
82 :
83 0 : HcclResult ProfilerManagerImpl::GetandClearOverFlowTasks(std::vector<HcclDumpInfo>& hcclDumpInfo)
84 : {
85 0 : if (taskOverflowHandler_ != nullptr) {
86 0 : CHK_RET(taskOverflowHandler_->GetandClearOverFlowTasks(hcclDumpInfo));
87 : } else {
88 0 : HCCL_WARNING("[ProfilerManager][GetDumpTask] taskOverflowHandler_ not set");
89 : }
90 0 : return HCCL_SUCCESS;
91 : }
92 :
93 32 : void ProfilerManagerImpl::TaskSdmaProfiler(ProfilerType profilerType, HcclRtStream stream, TaskParaDMA& para)
94 : {
95 32 : if (!callbacks_.empty()) {
96 128 : for (auto& callback : callbacks_) {
97 96 : if (profilerType != ProfilerType::TASK_ALL && callback.first != profilerType) {
98 0 : continue;
99 : }
100 96 : callback.second(stream, hccl::TaskType::TASK_SDMA, para);
101 : }
102 : }
103 32 : }
104 :
105 0 : void ProfilerManagerImpl::TaskRdmaProfiler(ProfilerType profilerType, HcclRtStream stream, TaskParaDMA& para)
106 : {
107 0 : if (!callbacks_.empty()) {
108 0 : for (auto& callback : callbacks_) {
109 0 : if (profilerType != ProfilerType::TASK_ALL && callback.first != profilerType) {
110 0 : continue;
111 : }
112 0 : callback.second(stream, hccl::TaskType::TASK_RDMA, para);
113 : }
114 : }
115 0 : }
116 :
117 9 : void ProfilerManagerImpl::TaskReduceInlineProfiler(ProfilerType profilerType, HcclRtStream stream, TaskParaReduce& para)
118 : {
119 9 : if (!callbacks_.empty()) {
120 36 : for (auto& callback : callbacks_) {
121 27 : if (profilerType != ProfilerType::TASK_ALL && callback.first != profilerType) {
122 0 : continue;
123 : }
124 27 : callback.second(stream, hccl::TaskType::TASK_REDUCE_INLINE, para);
125 : }
126 : }
127 9 : }
128 :
129 0 : void ProfilerManagerImpl::TaskReduceTbeProfiler(ProfilerType profilerType, HcclRtStream stream, TaskParaReduce& para)
130 : {
131 0 : if (!callbacks_.empty()) {
132 0 : for (auto& callback : callbacks_) {
133 0 : if (profilerType != ProfilerType::TASK_ALL && callback.first != profilerType) {
134 0 : continue;
135 : }
136 0 : callback.second(stream, hccl::TaskType::TASK_REDUCE_TBE, para);
137 : }
138 : }
139 0 : }
140 :
141 0 : void ProfilerManagerImpl::TaskRecordProfiler(ProfilerType profilerType, HcclRtStream stream, TaskParaNotify& para)
142 : {
143 0 : if (!callbacks_.empty()) {
144 0 : for (auto& callback : callbacks_) {
145 0 : if (profilerType != ProfilerType::TASK_ALL && callback.first != profilerType) {
146 0 : continue;
147 : }
148 0 : callback.second(stream, hccl::TaskType::TASK_NOTIFY_RECORD, para);
149 : }
150 : }
151 0 : }
152 :
153 0 : void ProfilerManagerImpl::TaskWaitProfiler(ProfilerType profilerType, HcclRtStream stream, TaskParaNotify& para)
154 : {
155 0 : if (!callbacks_.empty()) {
156 0 : for (auto& callback : callbacks_) {
157 0 : if (profilerType != ProfilerType::TASK_ALL && callback.first != profilerType) {
158 0 : continue;
159 : }
160 0 : callback.second(stream, hccl::TaskType::TASK_NOTIFY_WAIT, para);
161 : }
162 : }
163 0 : }
164 :
165 5 : void ProfilerManagerImpl::TaskAivProfiler(ProfilerType profilerType, HcclRtStream stream, TaskParaAiv& para)
166 : {
167 5 : if (!callbacks_.empty()) {
168 20 : for (auto& callback : callbacks_) {
169 15 : if (profilerType != ProfilerType::TASK_ALL && callback.first != profilerType) {
170 0 : continue;
171 : }
172 15 : callback.second(stream, para);
173 : }
174 : }
175 5 : }
176 :
177 0 : void ProfilerManagerImpl::TaskProfiler(
178 : ProfilerType profilerType, HcclRtStream stream, const void* descBuf, size_t descBufLen)
179 : {
180 0 : if (!callbacks_.empty()) {
181 0 : for (auto& callback : callbacks_) {
182 0 : if (profilerType != ProfilerType::TASK_ALL && callback.first != profilerType) {
183 0 : continue;
184 : }
185 0 : if (profilerType == ProfilerType::TASK_EXCEPTION) {
186 0 : callback.second(stream, descBuf, descBufLen);
187 : } else {
188 0 : callback.second(stream);
189 : }
190 : }
191 : }
192 0 : }
193 :
194 0 : void ProfilerManagerImpl::TaskProfiler(ProfilerType profilerType, TaskParaHost& para)
195 : {
196 0 : if (!callbacks_.empty()) {
197 0 : for (auto& callback : callbacks_) {
198 0 : if (profilerType != ProfilerType::TASK_ALL && callback.first != profilerType) {
199 0 : continue;
200 : }
201 0 : callback.second(para);
202 : }
203 : }
204 0 : }
205 :
206 41 : void ProfilerManagerImpl::TaskProfilerHandle(void* param, u32 length)
207 : {
208 41 : if (UNLIKELY(param == nullptr)) {
209 0 : HCCL_ERROR("[ProfilerManagerImpl][%s]param is nullptr.", __func__);
210 0 : return;
211 : }
212 41 : struct TaskPara* taskPara = (struct TaskPara*)param;
213 :
214 41 : if (sizeof(TaskPara) < length) {
215 0 : return;
216 : }
217 41 : HCCL_INFO(
218 : "[ProfilerManagerImpl][%s]Start handle task profiler, taskType[%d], profilerType[%d]", __func__, taskPara->type,
219 : taskPara->profilerType);
220 :
221 41 : u32 ctxId = 0;
222 : ProfTaskType profTaskType;
223 41 : auto& profilingManager = hccl::ProfilingManager::Instance();
224 41 : if (taskPara->isFftsDispatcher) {
225 0 : profilingManager.SetFftsDispatcherMode();
226 : }
227 :
228 41 : HandleTask(taskPara, ctxId, profTaskType);
229 :
230 41 : if (taskPara->isFftsDispatcher) {
231 0 : profilingManager.ReSetFftsDispatcherMode();
232 : }
233 :
234 41 : if (GetIfProfile() && ctxId == INVALID_UINT) {
235 41 : (void)profilingManager.CallMsprofReportTaskApi(taskPara->isMainStream, taskPara->beginTime, profTaskType);
236 : }
237 : }
238 :
239 5 : void ProfilerManagerImpl::TaskAivProfilerHandle(void* param, u32 length)
240 : {
241 5 : if (UNLIKELY(param == nullptr)) {
242 0 : HCCL_ERROR("[ProfilerManagerImpl][%s]param is nullptr.", __func__);
243 0 : return;
244 : }
245 :
246 5 : struct TaskParaGeneral* taskParaGeneral = static_cast<struct TaskParaGeneral*>(param);
247 :
248 5 : if (sizeof(TaskParaGeneral) < length) {
249 0 : return;
250 : }
251 :
252 5 : TaskAivProfiler(ProfilerType::TASK_ALL, taskParaGeneral->stream, taskParaGeneral->aiv);
253 :
254 5 : if (GetIfProfile()) {
255 5 : auto& profilingManager = hccl::ProfilingManager::Instance();
256 5 : (void)profilingManager.CallMsprofReportTaskApi(
257 5 : taskParaGeneral->isMainStream, taskParaGeneral->beginTime, ProfTaskType::TASK_AIV);
258 : }
259 : }
260 :
261 41 : void ProfilerManagerImpl::HandleTask(struct TaskPara* taskPara, u32& ctxId, ProfTaskType& profTaskType)
262 : {
263 41 : switch (taskPara->type) {
264 0 : case TaskType::TASK_NOTIFY_RECORD:
265 0 : TaskRecordProfiler(taskPara->profilerType, taskPara->stream, taskPara->notify);
266 0 : ctxId = taskPara->notify.ctxId;
267 0 : profTaskType = ProfTaskType::TASK_NOTIFY_RECORD;
268 0 : break;
269 :
270 0 : case TaskType::TASK_NOTIFY_WAIT:
271 0 : TaskWaitProfiler(taskPara->profilerType, taskPara->stream, taskPara->notify);
272 0 : ctxId = taskPara->notify.ctxId;
273 0 : profTaskType = ProfTaskType::TASK_NOTIFY_WAIT;
274 0 : break;
275 :
276 32 : case TaskType::TASK_SDMA:
277 32 : TaskSdmaProfiler(taskPara->profilerType, taskPara->stream, taskPara->dma);
278 32 : ctxId = taskPara->dma.ctxId;
279 32 : profTaskType = ProfTaskType::TASK_SDMA;
280 32 : break;
281 :
282 0 : case TaskType::TASK_RDMA:
283 0 : TaskRdmaProfiler(taskPara->profilerType, taskPara->stream, taskPara->dma);
284 0 : ctxId = taskPara->dma.ctxId;
285 0 : profTaskType = ProfTaskType::TASK_RDMA;
286 0 : break;
287 :
288 0 : case TaskType::TASK_REDUCE_TBE:
289 0 : TaskReduceTbeProfiler(taskPara->profilerType, taskPara->stream, taskPara->reduce);
290 0 : ctxId = taskPara->reduce.ctxId;
291 0 : profTaskType = ProfTaskType::TASK_REDUCE_TBE;
292 0 : break;
293 :
294 9 : case TaskType::TASK_REDUCE_INLINE:
295 9 : TaskReduceInlineProfiler(taskPara->profilerType, taskPara->stream, taskPara->reduce);
296 9 : ctxId = taskPara->reduce.ctxId;
297 9 : profTaskType = ProfTaskType::TASK_REDUCE_INLINE;
298 9 : break;
299 :
300 0 : case TaskType::TASK_HOST:
301 0 : (void)ProfilerBase::GetTagByStream(taskPara->host.streamID, taskPara->host.tag);
302 0 : TaskProfiler(ProfilerType::TASK_PROFILING, taskPara->host);
303 0 : break;
304 :
305 0 : case TaskType::TASK_GRAPH_LAUNCH:
306 0 : HandleGraphLaunchTask(taskPara);
307 0 : break;
308 :
309 0 : default:
310 0 : return;
311 : }
312 : }
313 :
314 0 : void ProfilerManagerImpl::HandleGraphLaunchTask(struct TaskPara* taskPara)
315 : {
316 0 : if (GetIfProfile()) {
317 0 : auto& profilingManager = hccl::ProfilingManager::Instance();
318 0 : rtModel_t rtModel = nullptr;
319 0 : bool isCapture = false;
320 0 : HcclResult retCapture = GetStreamCaptureInfo(taskPara->stream, rtModel, isCapture);
321 0 : CHK_PRT_CONT(
322 : retCapture != HCCL_SUCCESS, HCCL_ERROR("Get capture status error. return[%d], capture model", retCapture));
323 0 : if (profilingManager.GetFftsLaunchApiState() || isCapture) {
324 : // 上报批量下发的ContextId信息
325 0 : (void)profilingManager.CallMsprofReportContextIdInfo((taskPara->graphLaunch.ctxNum - 1));
326 :
327 0 : if (profilingManager.GetTaskApiState() || isCapture) {
328 : // 上报编排的task(memcpy\notify等) addition Info
329 0 : profilingManager.ReportStoragedFftsInfo();
330 : }
331 :
332 0 : (void)profilingManager.CallMsprofReportTaskApi(
333 0 : taskPara->isMainStream, taskPara->beginTime, ProfTaskType::TASK_LAUNCH_FFTS_TASK);
334 : }
335 0 : TaskProfiler(ProfilerType::TASK_PROFILING, taskPara->stream);
336 : }
337 0 : TaskProfiler(
338 : ProfilerType::TASK_EXCEPTION, taskPara->stream, taskPara->graphLaunch.descBuf,
339 : taskPara->graphLaunch.descBufLen);
340 0 : TaskProfiler(ProfilerType::TASK_OVERFLOW, taskPara->stream);
341 0 : }
342 : } // namespace hccl
|
