Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include <securec.h>
12 : #include "profiling_manager.h"
13 : #include "adapter_prof.h"
14 : #include "task_profiling.h"
15 :
16 : namespace hccl {
17 : std::mutex TaskProfiling::mutex_;
18 :
19 315 : TaskProfiling::TaskProfiling(u32 deviceLogicId_, u32 localRank_, u32 rankSize_, bool profilingOn)
20 : : ProfilerBase(deviceLogicId_),
21 315 : localRank_(localRank_),
22 315 : rankSize_(rankSize_),
23 315 : profilingOn_(profilingOn)
24 315 : {}
25 :
26 627 : TaskProfiling::~TaskProfiling() {}
27 :
28 0 : u64 TaskProfiling::TimestampNanosecond() const
29 : {
30 : // 此时间戳获取方式需要与runtime保持一致
31 : struct timespec ts;
32 0 : clock_gettime(CLOCK_MONOTONIC_RAW, &ts);
33 0 : return static_cast<u64>(ts.tv_sec * MULTIPLIER_S2NS + ts.tv_nsec);
34 : }
35 :
36 24 : double TaskProfiling::GetTaskTime(TaskType taskType, const TaskData& taskData) const
37 : {
38 24 : double estimatedUs = DURATION_INIT_VALUE;
39 24 : double fixedUs = DURATION_INIT_VALUE;
40 :
41 24 : switch (taskType) {
42 15 : case TaskType::TASK_SDMA:
43 : /* * 统一按照PCIe的带宽来计算, SDMA的固定开销按照0.6us(<512KB), 1.5us(>512KB)计算
44 : PCIe DMA实测带宽为19.3GB */
45 15 : fixedUs = (taskData.DMA.size > DURATION_SDMA_FIXED_THRESHOLD) ? DURATION_SDMA_FIXED_THRESHOLD_ABOVE :
46 : DURATION_SDMA_FIXED_THRESHOLD_BELOW;
47 15 : estimatedUs = (taskData.DMA.size / DURATION_SDMA_BANDWIDTH_MB) + fixedUs;
48 15 : break;
49 :
50 9 : case TaskType::TASK_REDUCE_INLINE:
51 : /* * 统一按照PCIe的带宽来计算, SDMA的固定开销按照0.6us(<512KB), 1.5us(>512KB)计算
52 : PCIe DMA实测带宽为19.3GB */
53 9 : fixedUs = (taskData.DMA.size > DURATION_SDMA_FIXED_THRESHOLD) ? DURATION_SDMA_FIXED_THRESHOLD_ABOVE :
54 : DURATION_SDMA_FIXED_THRESHOLD_BELOW;
55 9 : estimatedUs = (taskData.Reduce.size / DURATION_SDMA_BANDWIDTH_MB) + fixedUs;
56 9 : break;
57 :
58 0 : case TaskType::TASK_RDMA:
59 : /* * RDMA的固定开销按照7us计算(实测7us)
60 : RDMA实测大包单流带宽为 12.5 GB(刨去协议头, 取12GB) */
61 0 : estimatedUs = (taskData.DMA.size / DURATION_RDMA_BANDWIDTH_MB) + DURATION_RDMA_FIXED;
62 0 : break;
63 :
64 0 : case TaskType::TASK_REDUCE_TBE:
65 : // 统一按照10GB计算, CCE reduce的固定开销按照0.6us计算(理论值0.5 + dim * 8)
66 0 : estimatedUs = (taskData.Reduce.size / DURATION_CCE_BANDWIDTH_MB) + DURATION_CCE_FIXED;
67 0 : break;
68 :
69 0 : case TaskType::TASK_NOTIFY_RECORD:
70 : // 统一按1us算(片间1us, 片内0.5us)
71 0 : estimatedUs = DURATION_NOTIFY_RECORD;
72 0 : break;
73 :
74 0 : case TaskType::TASK_NOTIFY_WAIT:
75 : // Notify Wait按0.02us估计
76 0 : estimatedUs = DURATION_NOTIFY_WAIT;
77 0 : break;
78 :
79 0 : default:
80 0 : break;
81 : }
82 :
83 24 : return estimatedUs;
84 : }
85 :
86 24 : ProfTaskType TaskProfiling::GetProfTaskType(TaskType taskType) const
87 : {
88 24 : ProfTaskType type = ProfTaskType::TASK_INVALID;
89 24 : switch (taskType) {
90 15 : case TaskType::TASK_SDMA:
91 : case TaskType::TASK_RDMA:
92 15 : type = (taskType == TaskType::TASK_SDMA) ? (ProfTaskType::TASK_SDMA) : (ProfTaskType::TASK_RDMA);
93 15 : break;
94 9 : case TaskType::TASK_REDUCE_INLINE:
95 : case TaskType::TASK_REDUCE_TBE:
96 9 : type = (taskType == TaskType::TASK_REDUCE_INLINE) ? (ProfTaskType::TASK_REDUCE_INLINE) :
97 : (ProfTaskType::TASK_REDUCE_TBE);
98 9 : break;
99 0 : case TaskType::TASK_NOTIFY_RECORD:
100 : case TaskType::TASK_NOTIFY_WAIT:
101 0 : type = (taskType == TaskType::TASK_NOTIFY_RECORD) ? (ProfTaskType::TASK_NOTIFY_RECORD) :
102 : (ProfTaskType::TASK_NOTIFY_WAIT);
103 0 : break;
104 0 : default:
105 0 : break;
106 : }
107 :
108 24 : return type;
109 : }
110 :
111 24 : uint32_t TaskProfiling::GetTransportType(TaskType taskType) const
112 : {
113 24 : if (taskType == TaskType::TASK_SDMA || taskType == TaskType::TASK_REDUCE_INLINE
114 0 : || taskType == TaskType::TASK_NOTIFY_RECORD) {
115 24 : return static_cast<int32_t>(SimpleTaskType::SDMA);
116 0 : } else if (taskType == TaskType::TASK_RDMA) {
117 0 : return static_cast<int32_t>(SimpleTaskType::RDMA);
118 : } else {
119 0 : return static_cast<int32_t>(SimpleTaskType::LOCAL);
120 : }
121 : }
122 :
123 24 : uint32_t TaskProfiling::GetTaskRole(TaskType taskType) const
124 : {
125 24 : if (taskType == TaskType::TASK_SDMA || taskType == TaskType::TASK_REDUCE_INLINE
126 0 : || taskType == TaskType::TASK_REDUCE_TBE || taskType == TaskType::TASK_NOTIFY_WAIT) {
127 24 : return static_cast<uint32_t>(TaskRole::DST);
128 : } else {
129 0 : return static_cast<uint32_t>(TaskRole::SRC);
130 : }
131 : }
132 :
133 24 : void TaskProfiling::GetTaskData(TaskType taskType, const TaskData& taskData, struct MsprofHcclInfo& taskInfo)
134 : {
135 24 : switch (taskType) {
136 15 : case TaskType::TASK_SDMA:
137 15 : GetSdmaTaskData(taskType, taskData, taskInfo);
138 15 : break;
139 0 : case TaskType::TASK_RDMA:
140 0 : GetRdmaTaskData(taskType, taskData, taskInfo);
141 0 : break;
142 9 : case TaskType::TASK_REDUCE_INLINE:
143 : case TaskType::TASK_REDUCE_TBE:
144 9 : GetReduceTaskData(taskType, taskData, taskInfo);
145 9 : break;
146 0 : case TaskType::TASK_NOTIFY_RECORD:
147 : case TaskType::TASK_NOTIFY_WAIT:
148 0 : GetNotifyTaskData(taskType, taskData, taskInfo);
149 0 : break;
150 0 : default:
151 0 : break;
152 : }
153 24 : }
154 :
155 15 : void TaskProfiling::GetSdmaTaskData(TaskType taskType, const TaskData& taskData, struct MsprofHcclInfo& taskInfo) const
156 : {
157 15 : taskInfo.srcAddr = static_cast<u64>(reinterpret_cast<uintptr_t>(taskData.DMA.src));
158 15 : taskInfo.dstAddr = static_cast<u64>(reinterpret_cast<uintptr_t>(taskData.DMA.dst));
159 15 : taskInfo.dataSize = static_cast<u64>(taskData.DMA.size);
160 15 : taskInfo.notifyID = taskData.DMA.notifyID;
161 15 : taskInfo.linkType = static_cast<u32>(taskData.DMA.linkType);
162 : taskInfo.remoteRank
163 15 : = (taskData.DMA.remoteUserRank == INVALID_VALUE_RANKID) ? localRank_ : taskData.DMA.remoteUserRank;
164 15 : taskInfo.transportType = GetTransportType(taskType);
165 15 : taskInfo.role = GetTaskRole(taskType);
166 15 : taskInfo.durationEstimated = GetTaskTime(taskData.taskType, taskData);
167 15 : taskInfo.ctxId = taskData.DMA.ctxId;
168 15 : }
169 :
170 0 : void TaskProfiling::GetRdmaTaskData(TaskType taskType, const TaskData& taskData, struct MsprofHcclInfo& taskInfo) const
171 : {
172 0 : taskInfo.srcAddr = static_cast<u64>(reinterpret_cast<uintptr_t>(taskData.DMA.src));
173 0 : taskInfo.dstAddr = static_cast<u64>(reinterpret_cast<uintptr_t>(taskData.DMA.dst));
174 0 : taskInfo.dataSize = static_cast<u64>(taskData.DMA.size);
175 0 : taskInfo.notifyID = taskData.DMA.notifyID;
176 0 : taskInfo.linkType = static_cast<u32>(taskData.DMA.linkType);
177 : taskInfo.remoteRank
178 0 : = (taskData.DMA.remoteUserRank == INVALID_VALUE_RANKID) ? localRank_ : taskData.DMA.remoteUserRank;
179 0 : taskInfo.transportType = GetTransportType(taskType);
180 0 : taskInfo.role = GetTaskRole(taskType);
181 0 : taskInfo.rdmaType = static_cast<u32>(taskData.DMA.rdmaType);
182 0 : taskInfo.durationEstimated = GetTaskTime(taskData.taskType, taskData);
183 0 : taskInfo.ctxId = taskData.DMA.ctxId;
184 0 : }
185 :
186 9 : void TaskProfiling::GetReduceTaskData(
187 : TaskType taskType, const TaskData& taskData, struct MsprofHcclInfo& taskInfo) const
188 : {
189 9 : taskInfo.srcAddr = static_cast<u64>(reinterpret_cast<uintptr_t>(taskData.DMA.src));
190 9 : taskInfo.dstAddr = static_cast<u64>(reinterpret_cast<uintptr_t>(taskData.DMA.dst));
191 9 : taskInfo.dataSize = static_cast<u64>(taskData.Reduce.size);
192 9 : taskInfo.opType = opString[taskData.Reduce.op];
193 9 : taskInfo.dataType = dataTypeString[taskData.Reduce.dataType];
194 9 : taskInfo.linkType = static_cast<u32>(taskData.Reduce.linkType);
195 9 : taskInfo.remoteRank = taskData.Reduce.remoteUserRank;
196 9 : taskInfo.transportType = GetTransportType(taskType);
197 9 : taskInfo.role = GetTaskRole(taskType);
198 9 : taskInfo.durationEstimated = GetTaskTime(taskData.taskType, taskData);
199 9 : taskInfo.ctxId = taskData.Reduce.ctxId;
200 9 : }
201 :
202 0 : void TaskProfiling::GetNotifyTaskData(
203 : TaskType taskType, const TaskData& taskData, struct MsprofHcclInfo& taskInfo) const
204 : {
205 0 : taskInfo.notifyID = taskData.Notify.notifyID;
206 0 : taskInfo.stage = taskData.Notify.stage;
207 0 : taskInfo.remoteRank = taskData.Notify.remoteUserRank;
208 0 : taskInfo.transportType = GetTransportType(taskType);
209 0 : taskInfo.role = GetTaskRole(taskType);
210 0 : taskInfo.durationEstimated = GetTaskTime(taskData.taskType, taskData);
211 0 : taskInfo.ctxId = taskData.Notify.ctxId;
212 0 : }
213 :
214 0 : HcclResult TaskProfiling::Run([[maybe_unused]] const StepData& stepData) { return HCCL_SUCCESS; }
215 :
216 41 : HcclResult TaskProfiling::Run(const TaskData& taskData, bool isCapture)
217 : {
218 41 : HCCLReportData hcclReportData{};
219 41 : auto& profilingManager = hccl::ProfilingManager::Instance();
220 41 : bool isSubscribed = profilingManager.GetAdditionInfoState();
221 41 : if (!isSubscribed && GetWorkflowMode() == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE && !isCapture) {
222 17 : return HCCL_SUCCESS;
223 : }
224 24 : std::unique_lock<std::mutex> lock(mutex_);
225 :
226 24 : std::unique_lock<std::mutex> streamLock(streamMutex_[deviceLogicId_]);
227 24 : if (streamRecordInfoMap_[deviceLogicId_].find(taskData.streamID) == streamRecordInfoMap_[deviceLogicId_].end()) {
228 : // 找不到对应的tag则认为该stream不参与profiling, 返回SUCCESS
229 12 : HCCL_DEBUG("streamID[%u] not found in profiler", taskData.streamID);
230 12 : hcclReportData.tag = "unknown";
231 12 : hcclReportData.profInfo.planeID = 0;
232 12 : hcclReportData.groupName = "unknown";
233 12 : hcclReportData.profInfo.workFlowMode = 0;
234 : } else {
235 12 : StreamRecordInfo& streamInfo = streamRecordInfoMap_[deviceLogicId_][taskData.streamID];
236 12 : hcclReportData.tag = streamInfo.tag;
237 12 : hcclReportData.profInfo.planeID = streamInfo.planeId;
238 12 : hcclReportData.groupName = tagGroupMap_[deviceLogicId_][hcclReportData.tag];
239 12 : hcclReportData.profInfo.workFlowMode = static_cast<uint32_t>(tagModeMap_[deviceLogicId_][hcclReportData.tag]);
240 : }
241 24 : streamLock.unlock();
242 :
243 24 : hcclReportData.ts = hrtMsprofSysCycleTime();
244 24 : ProfTaskType type = GetProfTaskType(taskData.taskType);
245 24 : hcclReportData.type = static_cast<int32_t>(type);
246 24 : std::string nameInfo = GetProfTaskOpName(type);
247 24 : hcclReportData.profInfo.itemId = hrtMsprofGetHashId(nameInfo.c_str(), nameInfo.length());
248 24 : hcclReportData.profInfo.localRank = localRank_;
249 24 : hcclReportData.profInfo.rankSize = rankSize_;
250 24 : GetTaskData(taskData.taskType, taskData, hcclReportData.profInfo);
251 :
252 24 : hcclReportData.profInfo.cclTag = hrtMsprofGetHashId(hcclReportData.tag.c_str(), hcclReportData.tag.length());
253 : hcclReportData.profInfo.groupName
254 24 : = hrtMsprofGetHashId(hcclReportData.groupName.c_str(), hcclReportData.groupName.length());
255 :
256 24 : HCCL_DEBUG(
257 : "ReportMsprofData:streamID[%u] tag[%s][%llu] group[%s][%llu]", taskData.streamID, hcclReportData.tag.c_str(),
258 : hcclReportData.profInfo.cclTag, hcclReportData.groupName.c_str(), hcclReportData.profInfo.groupName);
259 :
260 24 : DumpReportDataInfo(hcclReportData.type, hcclReportData.profInfo);
261 24 : CHK_RET(ReportMsprofData(hcclReportData));
262 24 : return HCCL_SUCCESS;
263 41 : }
264 :
265 0 : HcclResult TaskProfiling::Run([[maybe_unused]] const std::string& opName, [[maybe_unused]] const std::string& tag) const
266 : {
267 0 : return HCCL_SUCCESS;
268 : }
269 :
270 0 : HcclResult TaskProfiling::Save(
271 : u32 captureStreamID, [[maybe_unused]] u32 streamID, u32 taskID, TaskType& taskType,
272 : const TaskParaReduce& paraReduce)
273 : {
274 0 : TaskData taskData(captureStreamID, taskID, taskType, paraReduce);
275 0 : Run(taskData, true);
276 0 : return HCCL_SUCCESS;
277 : }
278 :
279 9 : HcclResult TaskProfiling::Save(u32& streamID, u32& taskID, TaskType& taskType, const TaskParaReduce& paraReduce)
280 : {
281 9 : TaskData taskData(streamID, taskID, taskType, paraReduce);
282 9 : Run(taskData);
283 9 : return HCCL_SUCCESS;
284 : }
285 :
286 0 : HcclResult TaskProfiling::Save(
287 : u32 captureStreamID, [[maybe_unused]] u32 streamID, u32 taskID, TaskType& taskType, const TaskParaDMA& paraDMA)
288 : {
289 0 : TaskData taskData(captureStreamID, taskID, taskType, paraDMA);
290 0 : Run(taskData, true);
291 0 : return HCCL_SUCCESS;
292 : }
293 :
294 32 : HcclResult TaskProfiling::Save(u32& streamID, u32& taskID, TaskType& taskType, const TaskParaDMA& paraDMA)
295 : {
296 32 : TaskData taskData(streamID, taskID, taskType, paraDMA);
297 32 : Run(taskData);
298 32 : return HCCL_SUCCESS;
299 : }
300 :
301 0 : HcclResult TaskProfiling::Save(
302 : u32 captureStreamID, [[maybe_unused]] u32 streamID, u32 taskID, TaskType& taskType,
303 : const TaskParaNotify& paraNotify)
304 : {
305 0 : TaskData taskData(captureStreamID, taskID, taskType, paraNotify);
306 0 : Run(taskData, true);
307 0 : return HCCL_SUCCESS;
308 : }
309 :
310 0 : HcclResult TaskProfiling::Save(u32& streamID, u32& taskID, TaskType& taskType, const TaskParaNotify& paraNotify)
311 : {
312 0 : TaskData taskData(streamID, taskID, taskType, paraNotify);
313 0 : Run(taskData);
314 0 : return HCCL_SUCCESS;
315 : }
316 :
317 0 : HcclResult TaskProfiling::Save(
318 : [[maybe_unused]] u32 captureStreamID, [[maybe_unused]] u32 streamID, [[maybe_unused]] u32 taskID,
319 : [[maybe_unused]] const void* descBuf, [[maybe_unused]] size_t descBufLen)
320 : {
321 0 : return HCCL_SUCCESS;
322 : }
323 :
324 : HcclResult
325 8 : TaskProfiling::Save(u32 captureStreamID, u32 streamID, [[maybe_unused]] u32 taskID, const TaskParaAiv& paraAiv)
326 : {
327 8 : HCCLReportData hcclReportData{};
328 8 : auto& profilingManager = hccl::ProfilingManager::Instance();
329 8 : bool isSubscribed = profilingManager.GetAdditionInfoState();
330 8 : bool isCapture = (captureStreamID != streamID);
331 8 : if (!isSubscribed && GetWorkflowMode() == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE && !isCapture) {
332 5 : return HCCL_SUCCESS;
333 : }
334 3 : std::unique_lock<std::mutex> lock(mutex_);
335 :
336 3 : std::unique_lock<std::mutex> streamLock(streamMutex_[deviceLogicId_]);
337 :
338 3 : if (streamRecordInfoMap_[deviceLogicId_].find(captureStreamID) == streamRecordInfoMap_[deviceLogicId_].end()) {
339 : // 找不到对应的tag则认为该stream不参与profiling, 返回SUCCESS
340 1 : HCCL_DEBUG("streamID[%u] not found in profiler", captureStreamID);
341 1 : hcclReportData.tag = "unknown";
342 1 : hcclReportData.profInfo.planeID = 0;
343 1 : hcclReportData.groupName = "unknown";
344 1 : hcclReportData.profInfo.workFlowMode = 0;
345 : } else {
346 2 : StreamRecordInfo& streamInfo = streamRecordInfoMap_[deviceLogicId_][captureStreamID];
347 2 : hcclReportData.tag = streamInfo.tag;
348 2 : hcclReportData.profInfo.planeID = streamInfo.planeId;
349 2 : hcclReportData.groupName = tagGroupMap_[deviceLogicId_][hcclReportData.tag];
350 2 : hcclReportData.profInfo.workFlowMode = static_cast<uint32_t>(tagModeMap_[deviceLogicId_][hcclReportData.tag]);
351 : }
352 3 : streamLock.unlock();
353 :
354 3 : hcclReportData.ts = hrtMsprofSysCycleTime();
355 :
356 3 : hcclReportData.type = static_cast<int32_t>(ProfTaskType::TASK_AIV);
357 3 : std::string nameInfo = GetCMDTypeEnumStr(paraAiv.cmdType) + "AivKernel";
358 3 : hcclReportData.profInfo.itemId = hrtMsprofGetHashId(nameInfo.c_str(), nameInfo.length());
359 3 : hcclReportData.profInfo.localRank = localRank_;
360 3 : hcclReportData.profInfo.rankSize = rankSize_;
361 :
362 3 : hcclReportData.profInfo.dataSize = paraAiv.size;
363 3 : hcclReportData.profInfo.cclTag = hrtMsprofGetHashId(hcclReportData.tag.c_str(), hcclReportData.tag.length());
364 : hcclReportData.profInfo.groupName
365 3 : = hrtMsprofGetHashId(hcclReportData.groupName.c_str(), hcclReportData.groupName.length());
366 :
367 3 : HCCL_DEBUG(
368 : "ReportMsprofData:streamID[%u] tag[%s][%llu] group[%s][%llu]", captureStreamID, hcclReportData.tag.c_str(),
369 : hcclReportData.profInfo.cclTag, hcclReportData.groupName.c_str(), hcclReportData.profInfo.groupName);
370 :
371 3 : DumpReportDataInfo(hcclReportData.type, hcclReportData.profInfo);
372 3 : CHK_RET(ReportMsprofData(hcclReportData));
373 3 : return HCCL_SUCCESS;
374 8 : }
375 :
376 2 : HcclResult TaskProfiling::Save(u32 streamID, u32 taskID, const TaskParaAiv& paraAiv)
377 : {
378 2 : return Save(streamID, streamID, taskID, paraAiv);
379 : }
380 :
381 0 : HcclResult TaskProfiling::Save(
382 : [[maybe_unused]] u32& streamID, [[maybe_unused]] u32& taskID, [[maybe_unused]] const void* descBuf,
383 : [[maybe_unused]] size_t descBufLen)
384 : {
385 0 : return HCCL_SUCCESS;
386 : }
387 :
388 0 : HcclResult TaskProfiling::SaveToLog(const TaskParaHost& paraHost)
389 : {
390 0 : auto& profilingManager = hccl::ProfilingManager::Instance();
391 0 : bool isSubscribed = profilingManager.GetAdditionInfoState();
392 0 : if (!isSubscribed) {
393 0 : return HCCL_SUCCESS;
394 : }
395 :
396 0 : u32 streamID = paraHost.streamID;
397 0 : u32 taskID = paraHost.taskID;
398 0 : u64 len = paraHost.len;
399 0 : std::string tag = paraHost.tag;
400 0 : std::chrono::microseconds duration = paraHost.duration;
401 :
402 0 : double bandWidth = 0;
403 0 : if (duration.count() > 0) {
404 0 : double ratio = 1.0 * std::chrono::microseconds::period::num / std::chrono::microseconds::period::den;
405 0 : bandWidth = len / (duration.count() * ratio);
406 0 : double BytetoGB = 1 << 30;
407 0 : bandWidth /= BytetoGB;
408 : }
409 :
410 0 : HCCL_INFO(
411 : "[profiling][host TCP/RDMA] tag[%s], streamId[%u], taskId[%u], bandWidth[%f GB / s]", tag.c_str(), streamID,
412 : taskID, bandWidth);
413 :
414 0 : return HCCL_SUCCESS;
415 0 : }
416 :
417 27 : void TaskProfiling::DumpReportDataInfo(uint32_t type, const MsprofHcclInfo& profInfo)
418 : {
419 27 : HCCL_DEBUG(
420 : "[DumpReportDataInfo] type[%u], itemId[%llu], cclTag[%llu], groupName[%llu], localRank[%u], remoteRank[%u], "
421 : "rankSize[%u], workFlowMode[%u], planeID[%u], ctxId[%u], notifyID[%llu], stage[%u], role[%u], "
422 : "durationEstimated[%f], srcAddr[%llu], dstAddr[%llu], dataSize[%llu Byte], opType[%u], dataType[%u], "
423 : "linkType[%u], "
424 : "transportType[%u], rdmaType[%u]",
425 : type, profInfo.itemId, profInfo.cclTag, profInfo.groupName, profInfo.localRank, profInfo.remoteRank,
426 : profInfo.rankSize, profInfo.workFlowMode, profInfo.planeID, profInfo.ctxId, profInfo.notifyID, profInfo.stage,
427 : profInfo.role, profInfo.durationEstimated, profInfo.srcAddr, profInfo.dstAddr, profInfo.dataSize,
428 : profInfo.opType, profInfo.dataType, profInfo.linkType, profInfo.transportType, profInfo.rdmaType);
429 :
430 27 : return;
431 : }
432 :
433 24 : HcclResult TaskProfiling::ReportMsprofData(HCCLReportData& hcclReportData)
434 : {
435 : int32_t cb_ret;
436 24 : auto& profilingManager = hccl::ProfilingManager::Instance();
437 48 : cb_ret = profilingManager.CallMsprofReportAdditionInfo(
438 24 : static_cast<uint32_t>(ProfTaskType::TASK_HCCL_INFO), hcclReportData.ts, &hcclReportData.profInfo,
439 : sizeof(hcclReportData.profInfo));
440 24 : CHK_PRT_RET((cb_ret != 0), HCCL_ERROR("[TaskProfiling] ReportData profiling failed."), HCCL_E_PARA);
441 :
442 24 : return HCCL_SUCCESS;
443 : }
444 :
445 0 : HcclResult TaskProfiling::Flush() { return HCCL_SUCCESS; }
446 : } // namespace hccl
|
