Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 : #ifndef HCCL_PROFILING_HANDLER_H
11 : #define HCCL_PROFILING_HANDLER_H
12 : #include <unordered_map>
13 : #include <queue>
14 : #include <mutex>
15 : #include <atomic>
16 : #include "hccl/hccl_types.h"
17 : #include "task_info.h"
18 : #include "rt_external.h"
19 : #include "profiling_common.h"
20 : #include "stream_manager.h"
21 : #include "task_param.h"
22 :
23 : namespace Hccl {
24 : MAKE_ENUM(kernelType, AICPU_KERNEL = 0, CCU_KERNEL);
25 :
26 : // ccu 上报数据结构
27 : constexpr unsigned int MSPROF_REPORT_CCU_TASK_INFO = 14U;
28 : constexpr unsigned int MSPROF_REPORT_CCU_WAIT_SIGNAL_INFO = 15U;
29 : constexpr unsigned int MSPROF_REPORT_CCU_GROUP_INFO = 16U;
30 : constexpr uint8_t INVALID_TYPE_VALUE = 0xFF; // reduceOpType、inputDataType、outputDataType非法值
31 :
32 : MAKE_ENUM(ProfTaskType, TASK_HCCL_INFO, TASK_DPU_HCCL_INFO);
33 :
34 : struct MsprofCcuTaskInfo {
35 : uint8_t version;
36 : uint8_t workFlowMode;
37 : uint64_t itemId; // CCU任务名 hash id
38 : uint64_t groupName; // 通信域 hash id
39 : uint32_t rankId;
40 : uint32_t ranksize; // CCU任务设计的Chip数目
41 :
42 : uint16_t streamId;
43 : uint32_t taskId;
44 : uint8_t dieId; // CCU任务执行的DieId
45 : uint8_t missionId; // CCU任务执行的MissionId
46 : uint16_t instrId;
47 : };
48 :
49 : struct MsprofCcuGroupInfo {
50 : uint8_t version;
51 : uint64_t itemId; // CCU任务名 hash id
52 : uint64_t groupName; // 通信域 hash id
53 : uint32_t rankId;
54 : uint32_t ranksize; // CCU任务设计的Chip数目
55 : uint8_t workFlowMode;
56 :
57 : uint16_t streamId;
58 : uint32_t taskId;
59 : uint8_t dieId; // CCU任务执行的DieId
60 : uint16_t instrId;
61 : uint8_t missionId; // CCU任务执行的MissionId
62 :
63 : uint8_t reduceOpType; // 与HcclReduceOp类型保持一致
64 : uint8_t inputDataType; // 与HcclDataType类型保持一致
65 : uint8_t outputDataType; // 与HcclDataType类型保持一致
66 : uint64_t dataSize; // 输入数据大小
67 :
68 : uint16_t channelId[CCU_MAX_CHANNEL_NUM]; // LoopGroup所包含的搬运指令使用的ChannelId
69 : uint32_t remoteRankId[CCU_MAX_CHANNEL_NUM]; // LoopGroup所包含的搬运指令的对端
70 : };
71 :
72 : struct MsprofCcuWaitSignalInfo {
73 : uint8_t version;
74 : uint64_t itemId; // CCU任务名 hash id
75 : uint64_t groupName; // 通信域 hash id
76 : uint32_t rankId;
77 : uint32_t ranksize; // CCU任务设计的Chip数目
78 : uint8_t workFlowMode;
79 :
80 : uint16_t streamId;
81 : uint32_t taskId;
82 : uint8_t dieId; // CCU任务执行的DieId
83 : uint16_t instrId;
84 : uint8_t missionId; // CCU任务执行的MissionId
85 :
86 : uint32_t ckeId;
87 : uint32_t mask;
88 : uint16_t channelId[CCU_MAX_CHANNEL_NUM]; // LoopGroup所包含的搬运指令使用的ChannelId
89 : uint32_t remoteRankId[CCU_MAX_CHANNEL_NUM]; // LoopGroup所包含的搬运指令的对端
90 : };
91 :
92 : struct HCCLReportData {
93 : std::string fileTag;
94 : uint64_t ts;
95 : uint32_t type;
96 : MsprofHcclInfo profInfo;
97 : MsprofDpuHcclTrack dpuProfInfo;
98 : std::string tag;
99 : std::string groupName;
100 : };
101 :
102 : const std::map<OpType, std::string> PROF_OP_NAME_V2
103 : = {{OpType::INVALID, "hcom_invalid_"},
104 : {OpType::ALLREDUCE, "hcom_allReduce_"},
105 : {OpType::BROADCAST, "hcom_broadcast_"},
106 : {OpType::REDUCE, "hcom_reduce_"},
107 : {OpType::SEND, "hcom_send_"},
108 : {OpType::RECV, "hcom_receive_"},
109 : {OpType::ALLGATHER, "hcom_allGather_"},
110 : {OpType::REDUCESCATTER, "hcom_reduceScatter_"},
111 : {OpType::SCATTER, "hcom_scatter_"},
112 : {OpType::ALLTOALL, "hcom_alltoall_"},
113 : {OpType::ALLTOALLV, "hcom_alltoallv_"},
114 : {OpType::ALLGATHERV, "hcom_allGatherv_"},
115 : {OpType::REDUCESCATTERV, "hcom_reduceScatterv_"},
116 : {OpType::ALLTOALLVC, "hcom_alltoallvc_"},
117 : {OpType::BATCHSENDRECV, "hcom_batchSendRecv_"},
118 : {OpType::BATCHPUT, "hccl_batchPut_"},
119 : {OpType::BATCHGET, "hccl_batchGet_"},
120 : {OpType::DEBUGCASE, "hccl_debugCase_"},
121 : {OpType::BARRIER, "hccl_barrier_"},
122 : {OpType::HALFALLTOALLV, "hccl_halfAlltoallv_"}};
123 :
124 3 : inline std::string GetProfOpName(OpType opType)
125 : {
126 3 : CHK_PRT_RET(PROF_OP_NAME_V2.empty(), HCCL_ERROR("PROF_OP_NAME_V2 has not inited."), "hcom_invalid_");
127 3 : auto it = PROF_OP_NAME_V2.find(opType);
128 3 : if (it != PROF_OP_NAME_V2.end()) {
129 3 : return it->second;
130 : }
131 0 : return PROF_OP_NAME_V2.begin()->second;
132 : }
133 :
134 : class ProfilingHandler {
135 : public:
136 : ~ProfilingHandler();
137 :
138 : ProfilingHandler(const ProfilingHandler& that) = delete;
139 :
140 : ProfilingHandler& operator=(const ProfilingHandler& that) = delete;
141 :
142 : static ProfilingHandler& GetInstance();
143 :
144 : static int32_t CommandHandleWrapper(uint32_t rtType, void* data, uint32_t len);
145 :
146 : void ReportKernel() const;
147 :
148 : void ReportHostApi(OpType opType, uint64_t beginTime, uint64_t endTime, bool cachedReq, bool isAiCpu);
149 :
150 : void ReportHcclOp(const DfxOpInfo& opInfo, bool cachedReq);
151 :
152 : void ReportHcclTaskApi(
153 : TaskParamType taskType, uint64_t beginTime, uint64_t endTime, bool isMasterStream, bool cachedReq,
154 : bool ignoreLevel = false);
155 :
156 : void ReportHcclTaskDetails(const TaskInfo& taskInfo, bool cachedReq);
157 : void ReportHcclTaskDetailsBatch(const std::vector<TaskInfo*>& taskInfos, bool cachedReq);
158 :
159 : bool GetHostApiState() const;
160 : bool GetHcclNodeState() const;
161 : bool GetHcclL0State() const;
162 : bool GetHcclL1State() const;
163 0 : inline void SetOpModeFlags(bool isOpBase, bool isCached)
164 : {
165 0 : isOpbase_ = isOpBase;
166 0 : isCached_ = isCached;
167 0 : }
168 : inline bool GetOpBaseFlag() const { return isOpbase_; }
169 5 : inline bool GetCachedFlag() const { return isCached_; }
170 : int32_t CommandHandle(uint32_t rtType, void* data, uint32_t len) const;
171 : HcclResult Init();
172 : void ReportHcclMC2CommInfo(
173 : const Stream& kfcStream, const Stream& stream, const std::vector<Stream*>& aicpuStreams, const std::string& id,
174 : RankId myRank, u32 rankSize, RankId rankInParentComm);
175 : void ReportHcclMC2CommInfo(
176 : const u32 kfcStreamId, const std::vector<u32>& aicpuStreamsId, const std::string& id, RankId myRank,
177 : u32 rankSize, RankId rankInParentComm);
178 : void ReportNodeApi(uint64_t beginTime, uint64_t endTime, uint64_t cmdItemId, uint32_t threadId, bool cachedReq);
179 : void ReportNodeBasicInfo(uint64_t timeStamp, uint64_t cmdItemId, uint32_t threadId, bool cachedReq);
180 : uint64_t GetProfHashId(const char* name, uint32_t len) const;
181 : uint64_t GetCachedAlgTypeHashId() const { return cachedAlgTypeHashId_.load(); }
182 :
183 : private:
184 : explicit ProfilingHandler();
185 :
186 : void ReportAclApi(
187 : uint32_t cmdType, uint64_t beginTime, uint64_t endTime, uint64_t cmdItemId, uint32_t threadId, bool cachedReq);
188 :
189 : void ReportHcclOpInfo(uint64_t timeStamp, const DfxOpInfo& opInfo, uint32_t threadId, bool cachedReq);
190 : void ReportAdditionInfo(MsprofAdditionalInfo& reporterData) const;
191 :
192 : void StartSubscribe(uint64_t profconfig);
193 : void StartTaskApiSubscribe();
194 : void StartHostApiSubscribe();
195 : void StartAdditionInfoSubscribe();
196 : void StartHostHcclOpSubscribe();
197 : void StartCcuSubscribe();
198 : void StopSubscribe();
199 :
200 : void CallProfRegHostApi() const;
201 : void ReportStoragedCompactInfo();
202 : void ReportMc2AdditionInfo();
203 :
204 : void CallProfRegTaskTypeApi() const;
205 : void ReportStoragedTaskApi();
206 : void ReportStoragedAclApi();
207 :
208 : void CallProfRegHcclOpApi() const;
209 :
210 : void ReportStoragedAdditionInfo();
211 :
212 : void GetHCCLReportData(const TaskInfo& taskInfo, HCCLReportData& hcclReportData) const;
213 : void FillProfCommonInfo(const TaskInfo& taskInfo, MsprofAdditionalInfo& reporterData) const;
214 : void FillProfTaskSpecificInfo(const TaskInfo& taskInfo, MsprofHcclInfo* profInfo) const;
215 : void FillDpuProfInfo(const TaskInfo& taskInfo, MsprofAdditionalInfo& reporterData) const;
216 : void FillDpuTaskParaDetails(const TaskInfo& taskInfo, MsprofDpuHcclTrack* dpuProfInfo) const;
217 : void ConvertHcclInfoToDpuTrack(MsprofAdditionalInfo& reporterData) const;
218 : void FillTaskAdditionInfo(const TaskInfo& taskInfo, MsprofAdditionalInfo& reporterData) const;
219 : uint32_t GetTaskTypeValue(TaskParamType taskType) const;
220 : void CallAdditionInfo(MsprofAdditionalInfo& reporterData) const;
221 :
222 : void ReportCcuInfo(const TaskInfo& taskInfo) const;
223 : void GetCcuTaskInfo(const TaskInfo& taskInfo, const CcuProfilingInfo& info) const;
224 : void GetCcuWaitSignalInfo(const TaskInfo& taskInfo, const CcuProfilingInfo& info) const;
225 : void GetCcuGroupInfo(const TaskInfo& taskInfo, const CcuProfilingInfo& info) const;
226 :
227 : void DumpHCCLReportData(const TaskInfo& taskInfo, const MsprofAdditionalInfo& reporterData) const;
228 : void DumpCcuGroupInfo(const MsprofCcuGroupInfo& ccuGroupInfo) const;
229 : void ReportMc2AdditionInfo(uint64_t timeStamp, const void* data, int len);
230 : void SetCachedCclTag();
231 : void InitLog() const;
232 : void ReportHcclMC2CommInfoLog(
233 : const Stream& kfcStream, const Stream& stream, const std::vector<Stream*>& aicpuStreams, const std::string& id,
234 : RankId myRank, u32 rankSize, RankId rankInParentComm) const;
235 : void ReportHcclMC2CommInfoLog(
236 : const u32 kfcStreamId, const std::vector<u32>& aicpuStreamsId, const std::string& id, RankId myRank,
237 : u32 rankSize, RankId rankInParentComm) const;
238 : void ReportCcuInfoLog(const TaskInfo& taskInfo) const;
239 : void LogCcuTaskInfo(
240 : const CcuProfilingInfo& info, const TaskInfo& taskInfo, uint64_t itemId, uint64_t groupName, u32 rankId,
241 : u32 ranksize) const;
242 : void LogCcuWaitSignalInfo(
243 : const CcuProfilingInfo& info, const TaskInfo& taskInfo, uint64_t itemId, uint64_t groupName, u32 rankId,
244 : u32 ranksize) const;
245 : void LogCcuGroupInfo(
246 : const CcuProfilingInfo& info, const TaskInfo& taskInfo, uint64_t itemId, uint64_t groupName, u32 rankId,
247 : u32 ranksize) const;
248 : void ReportHcclTaskDetailsBatchLog(const std::vector<TaskInfo*>& taskInfos) const;
249 : void ReportStoragedAdditionInfoLog() const;
250 :
251 : private:
252 : static ProfilingHandler instance_;
253 : bool initializedFlag_{false};
254 : bool enableHostApi_{false};
255 : bool enableHcclNode_{false};
256 : bool enableHcclL0_{false};
257 : bool enableHcclL1_{false};
258 : bool isOpbase_{false};
259 : bool isCached_{false};
260 :
261 : std::vector<TaskInfo> cacheTaskInfos_{};
262 : std::queue<MsprofApi> cachedTaskApiInfo_{};
263 : std::queue<MsprofApi> cachedAclApiInfo_{};
264 : std::queue<MsprofCompactInfo> cacheHcclOpInfo_{};
265 : std::queue<MsprofAdditionalInfo> cacheHcclAdditionInfo_{};
266 : std::atomic<uint64_t> cachedAlgTypeHashId_{0};
267 :
268 : std::map<uint32_t, uint64_t> cachedNewCclTag_{};
269 : mutable std::mutex cacheTaskInfosMutex_;
270 : std::mutex cachedTaskApiInfoMutex_;
271 : std::mutex cachedAclApiInfoMutex_;
272 : std::mutex cacheHcclOpInfoMutex_;
273 : std::mutex cacheHcclAdditionInfoMutex_;
274 : };
275 : } // namespace Hccl
276 :
277 : #endif // HCCL_PROFILING_HANDLER_H
|
