Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #ifndef HCCL_PROFILING_HANDLER_H
12 : #define HCCL_PROFILING_HANDLER_H
13 : #include <unordered_map>
14 : #include <queue>
15 : #include <mutex>
16 : #include <atomic>
17 : #include "hccl/hccl_types.h"
18 : #include "task_info.h"
19 : #include "rt_external.h"
20 : #include "profiling_common.h"
21 : #include "stream_manager.h"
22 : #include "task_param.h"
23 :
24 : namespace Hccl {
25 : MAKE_ENUM(kernelType, AICPU_KERNEL = 0, CCU_KERNEL);
26 :
27 : // ccu 上报数据结构
28 : constexpr unsigned int MSPROF_REPORT_CCU_TASK_INFO = 14U;
29 : constexpr unsigned int MSPROF_REPORT_CCU_WAIT_SIGNAL_INFO = 15U;
30 : constexpr unsigned int MSPROF_REPORT_CCU_GROUP_INFO = 16U;
31 : constexpr uint8_t INVALID_TYPE_VALUE = 0xFF; // reduceOpType、inputDataType、outputDataType非法值
32 :
33 : MAKE_ENUM(ProfTaskType, TASK_HCCL_INFO, TASK_DPU_HCCL_INFO);
34 :
35 : struct MsprofCcuTaskInfo {
36 : uint8_t version;
37 : uint8_t workFlowMode;
38 : uint64_t itemId; // CCU任务名 hash id
39 : uint64_t groupName; // 通信域 hash id
40 : uint32_t rankId;
41 : uint32_t ranksize; // CCU任务设计的Chip数目
42 :
43 : uint16_t streamId;
44 : uint32_t taskId;
45 : uint8_t dieId; // CCU任务执行的DieId
46 : uint8_t missionId; // CCU任务执行的MissionId
47 : uint16_t instrId;
48 : };
49 :
50 : struct MsprofCcuGroupInfo {
51 : uint8_t version;
52 : uint64_t itemId; // CCU任务名 hash id
53 : uint64_t groupName; // 通信域 hash id
54 : uint32_t rankId;
55 : uint32_t ranksize; // CCU任务设计的Chip数目
56 : uint8_t workFlowMode;
57 :
58 : uint16_t streamId;
59 : uint32_t taskId;
60 : uint8_t dieId; // CCU任务执行的DieId
61 : uint16_t instrId;
62 : uint8_t missionId; // CCU任务执行的MissionId
63 :
64 : uint8_t reduceOpType; // 与HcclReduceOp类型保持一致
65 : uint8_t inputDataType; // 与HcclDataType类型保持一致
66 : uint8_t outputDataType; // 与HcclDataType类型保持一致
67 : uint64_t dataSize; // 输入数据大小
68 :
69 : uint16_t channelId[CCU_MAX_CHANNEL_NUM]; // LoopGroup所包含的搬运指令使用的ChannelId
70 : uint32_t remoteRankId[CCU_MAX_CHANNEL_NUM]; // LoopGroup所包含的搬运指令的对端
71 : };
72 :
73 : struct MsprofCcuWaitSignalInfo {
74 : uint8_t version;
75 : uint64_t itemId; // CCU任务名 hash id
76 : uint64_t groupName; // 通信域 hash id
77 : uint32_t rankId;
78 : uint32_t ranksize; // CCU任务设计的Chip数目
79 : uint8_t workFlowMode;
80 :
81 : uint16_t streamId;
82 : uint32_t taskId;
83 : uint8_t dieId; // CCU任务执行的DieId
84 : uint16_t instrId;
85 : uint8_t missionId; // CCU任务执行的MissionId
86 :
87 : uint32_t ckeId;
88 : uint32_t mask;
89 : uint16_t channelId[CCU_MAX_CHANNEL_NUM]; // LoopGroup所包含的搬运指令使用的ChannelId
90 : uint32_t remoteRankId[CCU_MAX_CHANNEL_NUM]; // LoopGroup所包含的搬运指令的对端
91 : };
92 :
93 : struct HCCLReportData {
94 : std::string fileTag;
95 : uint64_t ts;
96 : uint32_t type;
97 : MsprofHcclInfo profInfo;
98 : MsprofDpuHcclTrack dpuProfInfo;
99 : std::string tag;
100 : std::string groupName;
101 : };
102 :
103 : const std::map<OpType, std::string> PROF_OP_NAME_V2
104 : = {{OpType::INVALID, "hcom_invalid_"},
105 : {OpType::ALLREDUCE, "hcom_allReduce_"},
106 : {OpType::BROADCAST, "hcom_broadcast_"},
107 : {OpType::REDUCE, "hcom_reduce_"},
108 : {OpType::SEND, "hcom_send_"},
109 : {OpType::RECV, "hcom_receive_"},
110 : {OpType::ALLGATHER, "hcom_allGather_"},
111 : {OpType::REDUCESCATTER, "hcom_reduceScatter_"},
112 : {OpType::SCATTER, "hcom_scatter_"},
113 : {OpType::ALLTOALL, "hcom_alltoall_"},
114 : {OpType::ALLTOALLV, "hcom_alltoallv_"},
115 : {OpType::ALLGATHERV, "hcom_allGatherv_"},
116 : {OpType::REDUCESCATTERV, "hcom_reduceScatterv_"},
117 : {OpType::ALLTOALLVC, "hcom_alltoallvc_"},
118 : {OpType::BATCHSENDRECV, "hcom_batchSendRecv_"},
119 : {OpType::BATCHPUT, "hccl_batchPut_"},
120 : {OpType::BATCHGET, "hccl_batchGet_"},
121 : {OpType::DEBUGCASE, "hccl_debugCase_"},
122 : {OpType::BARRIER, "hccl_barrier_"},
123 : {OpType::HALFALLTOALLV, "hccl_halfAlltoallv_"},
124 : {OpType::HCCLGROUPOP, "hccl_groupOp_"}};
125 :
126 3 : inline std::string GetProfOpName(OpType opType)
127 : {
128 3 : CHK_PRT_RET(PROF_OP_NAME_V2.empty(), HCCL_ERROR("PROF_OP_NAME_V2 has not inited."), "hcom_invalid_");
129 3 : auto it = PROF_OP_NAME_V2.find(opType);
130 3 : if (it != PROF_OP_NAME_V2.end()) {
131 3 : return it->second;
132 : }
133 0 : return PROF_OP_NAME_V2.begin()->second;
134 : }
135 :
136 : class ProfilingHandler {
137 : public:
138 : ~ProfilingHandler();
139 :
140 : ProfilingHandler(const ProfilingHandler& that) = delete;
141 :
142 : ProfilingHandler& operator=(const ProfilingHandler& that) = delete;
143 :
144 : static ProfilingHandler& GetInstance();
145 :
146 : static int32_t CommandHandleWrapper(uint32_t rtType, void* data, uint32_t len);
147 :
148 : void ReportKernel() const;
149 :
150 : void ReportHostApi(OpType opType, uint64_t beginTime, uint64_t endTime, bool cachedReq, bool isAiCpu);
151 :
152 : void ReportHcclOp(const DfxOpInfo& opInfo, bool cachedReq);
153 :
154 : void ReportHcclTaskApi(
155 : TaskParamType taskType, uint64_t beginTime, uint64_t endTime, bool isMasterStream, bool cachedReq,
156 : bool ignoreLevel = false);
157 :
158 : void ReportHcclTaskDetails(const TaskInfo& taskInfo, bool cachedReq);
159 : void ReportHcclTaskDetailsBatch(const std::vector<TaskInfo*>& taskInfos, bool cachedReq);
160 :
161 : bool GetHostApiState() const;
162 : bool GetHcclNodeState() const;
163 : bool GetHcclL0State() const;
164 : bool GetHcclL1State() const;
165 0 : inline void SetOpModeFlags(bool isOpBase, bool isCached)
166 : {
167 0 : isOpbase_ = isOpBase;
168 0 : isCached_ = isCached;
169 0 : }
170 : inline bool GetOpBaseFlag() const { return isOpbase_; }
171 5 : inline bool GetCachedFlag() const { return isCached_; }
172 : int32_t CommandHandle(uint32_t rtType, void* data, uint32_t len) const;
173 : HcclResult Init();
174 : void ReportHcclMC2CommInfo(
175 : const Stream& kfcStream, const Stream& stream, const std::vector<Stream*>& aicpuStreams, const std::string& id,
176 : RankId myRank, u32 rankSize, RankId rankInParentComm);
177 : void ReportHcclMC2CommInfo(
178 : const u32 kfcStreamId, const std::vector<u32>& aicpuStreamsId, const std::string& id, RankId myRank,
179 : u32 rankSize, RankId rankInParentComm);
180 : void ReportNodeApi(uint64_t beginTime, uint64_t endTime, uint64_t cmdItemId, uint32_t threadId, bool cachedReq);
181 : void ReportNodeBasicInfo(uint64_t timeStamp, uint64_t cmdItemId, uint32_t threadId, bool cachedReq);
182 : uint64_t GetProfHashId(const char* name, uint32_t len) const;
183 : uint64_t GetCachedAlgTypeHashId() const { return cachedAlgTypeHashId_.load(); }
184 :
185 : private:
186 : explicit ProfilingHandler();
187 :
188 : void ReportAclApi(
189 : uint32_t cmdType, uint64_t beginTime, uint64_t endTime, uint64_t cmdItemId, uint32_t threadId, bool cachedReq);
190 :
191 : void ReportHcclOpInfo(uint64_t timeStamp, const DfxOpInfo& opInfo, uint32_t threadId, bool cachedReq);
192 : void ReportAdditionInfo(MsprofAdditionalInfo& reporterData) const;
193 :
194 : void StartSubscribe(uint64_t profconfig);
195 : void StartTaskApiSubscribe();
196 : void StartHostApiSubscribe();
197 : void StartAdditionInfoSubscribe();
198 : void StartHostHcclOpSubscribe();
199 : void StartCcuSubscribe();
200 : void StopSubscribe();
201 :
202 : void CallProfRegHostApi() const;
203 : void ReportStoragedCompactInfo();
204 : void ReportMc2AdditionInfo();
205 :
206 : void CallProfRegTaskTypeApi() const;
207 : void ReportStoragedTaskApi();
208 : void ReportStoragedAclApi();
209 :
210 : void CallProfRegHcclOpApi() const;
211 :
212 : void ReportStoragedAdditionInfo();
213 :
214 : void GetHCCLReportData(const TaskInfo& taskInfo, HCCLReportData& hcclReportData) const;
215 : void FillProfCommonInfo(const TaskInfo& taskInfo, MsprofAdditionalInfo& reporterData) const;
216 : void FillProfTaskSpecificInfo(const TaskInfo& taskInfo, MsprofHcclInfo* profInfo) const;
217 : void FillDpuProfInfo(const TaskInfo& taskInfo, MsprofAdditionalInfo& reporterData) const;
218 : void FillDpuTaskParaDetails(const TaskInfo& taskInfo, MsprofDpuHcclTrack* dpuProfInfo) const;
219 : void ConvertHcclInfoToDpuTrack(MsprofAdditionalInfo& reporterData) const;
220 : void FillTaskAdditionInfo(const TaskInfo& taskInfo, MsprofAdditionalInfo& reporterData) const;
221 : uint32_t GetTaskTypeValue(TaskParamType taskType) const;
222 : void CallAdditionInfo(MsprofAdditionalInfo& reporterData) const;
223 :
224 : void ReportCcuInfo(const TaskInfo& taskInfo) const;
225 : void GetCcuTaskInfo(const TaskInfo& taskInfo, const CcuProfilingInfo& info) const;
226 : void GetCcuWaitSignalInfo(const TaskInfo& taskInfo, const CcuProfilingInfo& info) const;
227 : void GetCcuGroupInfo(const TaskInfo& taskInfo, const CcuProfilingInfo& info) const;
228 :
229 : void DumpHCCLReportData(const TaskInfo& taskInfo, const MsprofAdditionalInfo& reporterData) const;
230 : void DumpCcuGroupInfo(const MsprofCcuGroupInfo& ccuGroupInfo) const;
231 : void ReportMc2AdditionInfo(uint64_t timeStamp, const void* data, int len);
232 : void SetCachedCclTag();
233 : void InitLog() const;
234 : void ReportHcclMC2CommInfoLog(
235 : const Stream& kfcStream, const Stream& stream, const std::vector<Stream*>& aicpuStreams, const std::string& id,
236 : RankId myRank, u32 rankSize, RankId rankInParentComm) const;
237 : void ReportHcclMC2CommInfoLog(
238 : const u32 kfcStreamId, const std::vector<u32>& aicpuStreamsId, const std::string& id, RankId myRank,
239 : u32 rankSize, RankId rankInParentComm) const;
240 : void ReportCcuInfoLog(const TaskInfo& taskInfo) const;
241 : void LogCcuTaskInfo(
242 : const CcuProfilingInfo& info, const TaskInfo& taskInfo, uint64_t itemId, uint64_t groupName, u32 rankId,
243 : u32 ranksize) const;
244 : void LogCcuWaitSignalInfo(
245 : const CcuProfilingInfo& info, const TaskInfo& taskInfo, uint64_t itemId, uint64_t groupName, u32 rankId,
246 : u32 ranksize) const;
247 : void LogCcuGroupInfo(
248 : const CcuProfilingInfo& info, const TaskInfo& taskInfo, uint64_t itemId, uint64_t groupName, u32 rankId,
249 : u32 ranksize) const;
250 : void ReportHcclTaskDetailsBatchLog(const std::vector<TaskInfo*>& taskInfos) const;
251 : void ReportStoragedAdditionInfoLog() const;
252 :
253 : private:
254 : static ProfilingHandler instance_;
255 : bool initializedFlag_{false};
256 : bool enableHostApi_{false};
257 : bool enableHcclNode_{false};
258 : bool enableHcclL0_{false};
259 : bool enableHcclL1_{false};
260 : bool isOpbase_{false};
261 : bool isCached_{false};
262 :
263 : std::vector<TaskInfo> cacheTaskInfos_{};
264 : std::queue<MsprofApi> cachedTaskApiInfo_{};
265 : std::queue<MsprofApi> cachedAclApiInfo_{};
266 : std::queue<MsprofCompactInfo> cacheHcclOpInfo_{};
267 : std::queue<MsprofAdditionalInfo> cacheHcclAdditionInfo_{};
268 : std::atomic<uint64_t> cachedAlgTypeHashId_{0};
269 :
270 : std::map<uint32_t, uint64_t> cachedNewCclTag_{};
271 : mutable std::mutex cacheTaskInfosMutex_;
272 : std::mutex cachedTaskApiInfoMutex_;
273 : std::mutex cachedAclApiInfoMutex_;
274 : std::mutex cacheHcclOpInfoMutex_;
275 : std::mutex cacheHcclAdditionInfoMutex_;
276 : };
277 : } // namespace Hccl
278 :
279 : #endif // HCCL_PROFILING_HANDLER_H
|
