Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include <dlog_pub.h>
12 : #include <hccl/hccl_types.h>
13 : #include "hccl/base.h"
14 : #include "adapter_prof.h"
15 : #include "profiling_manager.h"
16 : #include "profiler_manager_impl.h"
17 : #include "stream_utils.h"
18 :
19 : namespace hccl {
20 309 : ProfilerManagerImpl::ProfilerManagerImpl(s32 devicePhyId, s32 deviceLogicId, u32 realUserRank, u32 rankSize)
21 309 : : devicePhyId_(devicePhyId), deviceLogicId_(deviceLogicId), realUserRank_(realUserRank), rankSize_(rankSize),
22 309 : profiler_(nullptr), taskExceptionHandler_(nullptr), taskOverflowHandler_(nullptr)
23 : {
24 309 : }
25 309 : ProfilerManagerImpl::~ProfilerManagerImpl()
26 : {
27 309 : }
28 :
29 927 : void ProfilerManagerImpl::RegisterCallBack(ProfilerType name, hccl::PluginRunner &callback)
30 : {
31 : // 容器操作并非线程安全的, 加锁
32 927 : std::unique_lock<std::mutex> lock(mutex_);
33 :
34 927 : if (callbacks_.find(name) != callbacks_.end()) {
35 0 : HCCL_WARNING("callback[%d] already registered", name);
36 0 : return;
37 : }
38 :
39 927 : callbacks_.insert(std::make_pair<ProfilerType &, hccl::PluginRunner &>(name, callback));
40 927 : }
41 :
42 309 : HcclResult ProfilerManagerImpl::InitProfiler()
43 : {
44 309 : if (static_cast<s32>(devicePhyId_) == HOST_DEVICE_ID) {
45 0 : return HCCL_SUCCESS;
46 : }
47 309 : CHK_RET(DlProfFunction::GetInstance().DlProfFunctionInit());
48 309 : CHK_RET(DlRtFunction::GetInstance().DlRtFunctionInit());
49 309 : profiler_.reset(new (std::nothrow) TaskProfiling(deviceLogicId_, realUserRank_, rankSize_, true));
50 309 : CHK_SMART_PTR_NULL(profiler_);
51 309 : PluginRunner profrunner(profiler_.get());
52 309 : RegisterCallBack(ProfilerType::TASK_PROFILING, profrunner);
53 :
54 309 : taskExceptionHandler_.reset(new (std::nothrow) TaskExceptionHandler(static_cast<u32>(deviceLogicId_)));
55 309 : CHK_SMART_PTR_NULL(taskExceptionHandler_);
56 309 : PluginRunner runner(taskExceptionHandler_.get());
57 309 : RegisterCallBack(ProfilerType::TASK_EXCEPTION, runner);
58 :
59 : // 记录可能导致算子溢出的task信息
60 309 : taskOverflowHandler_.reset(new (std::nothrow) TaskOverflow(static_cast<u32>(deviceLogicId_)));
61 309 : CHK_SMART_PTR_NULL(taskOverflowHandler_);
62 309 : PluginRunner dumprunner(taskOverflowHandler_.get());
63 309 : RegisterCallBack(ProfilerType::TASK_OVERFLOW, dumprunner);
64 :
65 309 : rtProfCtrlHandle callback = CommandHandle;
66 309 : HcclResult ret = hrtProfRegisterCtrlCallback(HCCL, callback);
67 309 : CHK_PRT_RET((ret != HCCL_SUCCESS), HCCL_ERROR("[ProfilerManager][InitProfiler]Register CtrlCallBack failed."),
68 : HCCL_E_PARA);
69 :
70 15759 : for (const auto& it : PROF_TASK_OP_NAME) {
71 15450 : std::string nameInfo = it.second;
72 15450 : uint64_t ret = hrtMsprofGetHashId(nameInfo.c_str(), nameInfo.length());
73 15450 : HCCL_DEBUG("[PROF_TASK_OP_NAME] nameInfo[%s] ret[%llu]", nameInfo.c_str(), ret);
74 15450 : }
75 :
76 309 : HCCL_INFO("[ProfilerManager][InitProfiler]Register CtrlCallBack success");
77 :
78 309 : return HCCL_SUCCESS;
79 309 : }
80 :
81 0 : HcclResult ProfilerManagerImpl::GetandClearOverFlowTasks(std::vector<HcclDumpInfo> &hcclDumpInfo)
82 : {
83 0 : if (taskOverflowHandler_ != nullptr) {
84 0 : CHK_RET(taskOverflowHandler_->GetandClearOverFlowTasks(hcclDumpInfo));
85 : } else {
86 0 : HCCL_WARNING("[ProfilerManager][GetDumpTask] taskOverflowHandler_ not set");
87 : }
88 0 : return HCCL_SUCCESS;
89 : }
90 :
91 32 : void ProfilerManagerImpl::TaskSdmaProfiler(ProfilerType profilerType, HcclRtStream stream, TaskParaDMA ¶)
92 : {
93 32 : if (!callbacks_.empty()) {
94 128 : for (auto &callback : callbacks_) {
95 96 : if (profilerType != ProfilerType::TASK_ALL && callback.first != profilerType) {
96 0 : continue;
97 : }
98 96 : callback.second(stream, hccl::TaskType::TASK_SDMA, para);
99 : }
100 : }
101 32 : }
102 :
103 0 : void ProfilerManagerImpl::TaskRdmaProfiler(ProfilerType profilerType, HcclRtStream stream, TaskParaDMA ¶)
104 : {
105 0 : if (!callbacks_.empty()) {
106 0 : for (auto &callback : callbacks_) {
107 0 : if (profilerType != ProfilerType::TASK_ALL && callback.first != profilerType) {
108 0 : continue;
109 : }
110 0 : callback.second(stream, hccl::TaskType::TASK_RDMA, para);
111 : }
112 : }
113 0 : }
114 :
115 9 : void ProfilerManagerImpl::TaskReduceInlineProfiler(ProfilerType profilerType, HcclRtStream stream, TaskParaReduce ¶)
116 : {
117 9 : if (!callbacks_.empty()) {
118 36 : for (auto &callback : callbacks_) {
119 27 : if (profilerType != ProfilerType::TASK_ALL && callback.first != profilerType) {
120 0 : continue;
121 : }
122 27 : callback.second(stream, hccl::TaskType::TASK_REDUCE_INLINE, para);
123 : }
124 : }
125 9 : }
126 :
127 0 : void ProfilerManagerImpl::TaskReduceTbeProfiler(ProfilerType profilerType, HcclRtStream stream, TaskParaReduce ¶)
128 : {
129 0 : if (!callbacks_.empty()) {
130 0 : for (auto &callback : callbacks_) {
131 0 : if (profilerType != ProfilerType::TASK_ALL && callback.first != profilerType) {
132 0 : continue;
133 : }
134 0 : callback.second(stream, hccl::TaskType::TASK_REDUCE_TBE, para);
135 : }
136 : }
137 0 : }
138 :
139 0 : void ProfilerManagerImpl::TaskRecordProfiler(ProfilerType profilerType, HcclRtStream stream, TaskParaNotify ¶)
140 : {
141 0 : if (!callbacks_.empty()) {
142 0 : for (auto &callback : callbacks_) {
143 0 : if (profilerType != ProfilerType::TASK_ALL && callback.first != profilerType) {
144 0 : continue;
145 : }
146 0 : callback.second(stream, hccl::TaskType::TASK_NOTIFY_RECORD, para);
147 : }
148 : }
149 0 : }
150 :
151 0 : void ProfilerManagerImpl::TaskWaitProfiler(ProfilerType profilerType, HcclRtStream stream, TaskParaNotify ¶)
152 : {
153 0 : if (!callbacks_.empty()) {
154 0 : for (auto &callback : callbacks_) {
155 0 : if (profilerType != ProfilerType::TASK_ALL && callback.first != profilerType) {
156 0 : continue;
157 : }
158 0 : callback.second(stream, hccl::TaskType::TASK_NOTIFY_WAIT, para);
159 : }
160 : }
161 0 : }
162 :
163 5 : void ProfilerManagerImpl::TaskAivProfiler(ProfilerType profilerType, HcclRtStream stream, TaskParaAiv ¶)
164 : {
165 5 : if (!callbacks_.empty()) {
166 20 : for (auto &callback : callbacks_) {
167 15 : if (profilerType != ProfilerType::TASK_ALL && callback.first != profilerType) {
168 0 : continue;
169 : }
170 15 : callback.second(stream, para);
171 : }
172 : }
173 5 : }
174 :
175 0 : void ProfilerManagerImpl::TaskProfiler(ProfilerType profilerType, HcclRtStream stream, const void *descBuf, size_t descBufLen)
176 : {
177 0 : if (!callbacks_.empty()) {
178 0 : for (auto &callback : callbacks_) {
179 0 : if (profilerType != ProfilerType::TASK_ALL && callback.first != profilerType) {
180 0 : continue;
181 : }
182 0 : if (profilerType == ProfilerType::TASK_EXCEPTION) {
183 0 : callback.second(stream, descBuf, descBufLen);
184 : } else {
185 0 : callback.second(stream);
186 : }
187 : }
188 : }
189 0 : }
190 :
191 0 : void ProfilerManagerImpl::TaskProfiler(ProfilerType profilerType, TaskParaHost ¶)
192 : {
193 0 : if (!callbacks_.empty()) {
194 0 : for (auto &callback : callbacks_) {
195 0 : if (profilerType != ProfilerType::TASK_ALL && callback.first != profilerType) {
196 0 : continue;
197 : }
198 0 : callback.second(para);
199 : }
200 : }
201 0 : }
202 :
203 41 : void ProfilerManagerImpl::TaskProfilerHandle(void *param, u32 length)
204 : {
205 41 : if (UNLIKELY(param == nullptr)) {
206 0 : HCCL_ERROR("[ProfilerManagerImpl][%s]param is nullptr.", __func__);
207 0 : return;
208 : }
209 41 : struct TaskPara *taskPara = (struct TaskPara *)param;
210 :
211 41 : if (sizeof(TaskPara) < length) {
212 0 : return;
213 : }
214 41 : HCCL_INFO("[ProfilerManagerImpl][%s]Start handle task profiler, taskType[%d], profilerType[%d]", __func__,
215 : taskPara->type, taskPara->profilerType);
216 :
217 41 : u32 ctxId = 0;
218 : ProfTaskType profTaskType;
219 41 : auto &profilingManager = hccl::ProfilingManager::Instance();
220 41 : if (taskPara->isFftsDispatcher) {
221 0 : profilingManager.SetFftsDispatcherMode();
222 : }
223 :
224 41 : HandleTask(taskPara, ctxId, profTaskType);
225 :
226 41 : if (taskPara->isFftsDispatcher) {
227 0 : profilingManager.ReSetFftsDispatcherMode();
228 : }
229 :
230 41 : if (GetIfProfile() && ctxId == INVALID_UINT) {
231 41 : (void)profilingManager.CallMsprofReportTaskApi(taskPara->isMainStream, taskPara->beginTime, profTaskType);
232 : }
233 : }
234 :
235 5 : void ProfilerManagerImpl::TaskAivProfilerHandle(void *param, u32 length)
236 : {
237 5 : if (UNLIKELY(param == nullptr)) {
238 0 : HCCL_ERROR("[ProfilerManagerImpl][%s]param is nullptr.", __func__);
239 0 : return;
240 : }
241 :
242 5 : struct TaskParaGeneral* taskParaGeneral = static_cast<struct TaskParaGeneral *>(param);
243 :
244 5 : if(sizeof(TaskParaGeneral) < length){
245 0 : return;
246 : }
247 :
248 5 : TaskAivProfiler(ProfilerType::TASK_ALL, taskParaGeneral->stream, taskParaGeneral->aiv);
249 :
250 5 : if (GetIfProfile()){
251 5 : auto &profilingManager = hccl::ProfilingManager::Instance();
252 5 : (void)profilingManager.CallMsprofReportTaskApi(taskParaGeneral->isMainStream, taskParaGeneral->beginTime, ProfTaskType::TASK_AIV);
253 : }
254 : }
255 :
256 41 : void ProfilerManagerImpl::HandleTask(struct TaskPara *taskPara, u32 &ctxId, ProfTaskType &profTaskType)
257 : {
258 41 : switch (taskPara->type) {
259 0 : case TaskType::TASK_NOTIFY_RECORD:
260 0 : TaskRecordProfiler(taskPara->profilerType, taskPara->stream, taskPara->notify);
261 0 : ctxId = taskPara->notify.ctxId;
262 0 : profTaskType = ProfTaskType::TASK_NOTIFY_RECORD;
263 0 : break;
264 :
265 0 : case TaskType::TASK_NOTIFY_WAIT:
266 0 : TaskWaitProfiler(taskPara->profilerType, taskPara->stream, taskPara->notify);
267 0 : ctxId = taskPara->notify.ctxId;
268 0 : profTaskType = ProfTaskType::TASK_NOTIFY_WAIT;
269 0 : break;
270 :
271 32 : case TaskType::TASK_SDMA:
272 32 : TaskSdmaProfiler(taskPara->profilerType, taskPara->stream, taskPara->dma);
273 32 : ctxId = taskPara->dma.ctxId;
274 32 : profTaskType = ProfTaskType::TASK_SDMA;
275 32 : break;
276 :
277 0 : case TaskType::TASK_RDMA:
278 0 : TaskRdmaProfiler(taskPara->profilerType, taskPara->stream, taskPara->dma);
279 0 : ctxId = taskPara->dma.ctxId;
280 0 : profTaskType = ProfTaskType::TASK_RDMA;
281 0 : break;
282 :
283 0 : case TaskType::TASK_REDUCE_TBE:
284 0 : TaskReduceTbeProfiler(taskPara->profilerType, taskPara->stream, taskPara->reduce);
285 0 : ctxId = taskPara->reduce.ctxId;
286 0 : profTaskType = ProfTaskType::TASK_REDUCE_TBE;
287 0 : break;
288 :
289 9 : case TaskType::TASK_REDUCE_INLINE:
290 9 : TaskReduceInlineProfiler(taskPara->profilerType, taskPara->stream, taskPara->reduce);
291 9 : ctxId = taskPara->reduce.ctxId;
292 9 : profTaskType = ProfTaskType::TASK_REDUCE_INLINE;
293 9 : break;
294 :
295 0 : case TaskType::TASK_HOST:
296 0 : (void)ProfilerBase::GetTagByStream(taskPara->host.streamID, taskPara->host.tag);
297 0 : TaskProfiler(ProfilerType::TASK_PROFILING, taskPara->host);
298 0 : break;
299 :
300 0 : case TaskType::TASK_GRAPH_LAUNCH:
301 0 : HandleGraphLaunchTask(taskPara);
302 0 : break;
303 :
304 0 : default:
305 0 : return;
306 : }
307 : }
308 :
309 0 : void ProfilerManagerImpl::HandleGraphLaunchTask(struct TaskPara *taskPara)
310 : {
311 0 : if (GetIfProfile()) {
312 0 : auto &profilingManager = hccl::ProfilingManager::Instance();
313 0 : rtModel_t rtModel = nullptr;
314 0 : bool isCapture = false;
315 0 : HcclResult retCapture = GetStreamCaptureInfo(taskPara->stream, rtModel, isCapture);
316 0 : CHK_PRT_CONT(retCapture != HCCL_SUCCESS,
317 : HCCL_ERROR("Get capture status error. return[%d], capture model", retCapture));
318 0 : if (profilingManager.GetFftsLaunchApiState() || isCapture) {
319 : // 上报批量下发的ContextId信息
320 0 : (void)profilingManager.CallMsprofReportContextIdInfo((taskPara->graphLaunch.ctxNum - 1));
321 :
322 0 : if (profilingManager.GetTaskApiState() || isCapture) {
323 : // 上报编排的task(memcpy\notify等) addition Info
324 0 : profilingManager.ReportStoragedFftsInfo();
325 : }
326 :
327 0 : (void)profilingManager.CallMsprofReportTaskApi(taskPara->isMainStream, taskPara->beginTime,
328 : ProfTaskType::TASK_LAUNCH_FFTS_TASK);
329 : }
330 0 : TaskProfiler(ProfilerType::TASK_PROFILING, taskPara->stream);
331 : }
332 0 : TaskProfiler(ProfilerType::TASK_EXCEPTION, taskPara->stream, taskPara->graphLaunch.descBuf, taskPara->graphLaunch.descBufLen);
333 0 : TaskProfiler(ProfilerType::TASK_OVERFLOW, taskPara->stream);
334 0 : }
335 : } // namespace hccl
|
