Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 : #ifndef HCCL_PROFILING_HANDLER_H
11 : #define HCCL_PROFILING_HANDLER_H
12 : #include <unordered_map>
13 : #include <queue>
14 : #include <mutex>
15 : #include "hccl/hccl_types.h"
16 : #include "task_info.h"
17 : #include "rt_external.h"
18 : #include "profiling_common.h"
19 : #include "stream_manager.h"
20 : #include "task_param.h"
21 :
22 : namespace Hccl {
23 : MAKE_ENUM(kernelType, AICPU_KERNEL = 0, CCU_KERNEL);
24 :
25 : // ccu 上报数据结构
26 : constexpr unsigned int MSPROF_REPORT_CCU_TASK_INFO = 14U;
27 : constexpr unsigned int MSPROF_REPORT_CCU_WAIT_SIGNAL_INFO = 15U;
28 : constexpr unsigned int MSPROF_REPORT_CCU_GROUP_INFO = 16U;
29 : constexpr uint8_t INVALID_TYPE_VALUE = 0xFF; // reduceOpType、inputDataType、outputDataType非法值
30 :
31 : MAKE_ENUM(ProfTaskType, TASK_HCCL_INFO, TASK_DPU_HCCL_INFO);
32 :
33 : struct MsprofCcuTaskInfo {
34 : uint8_t version;
35 : uint8_t workFlowMode;
36 : uint64_t itemId; // CCU任务名 hash id
37 : uint64_t groupName; // 通信域 hash id
38 : uint32_t rankId;
39 : uint32_t ranksize; // CCU任务设计的Chip数目
40 :
41 : uint16_t streamId;
42 : uint32_t taskId;
43 : uint8_t dieId; // CCU任务执行的DieId
44 : uint8_t missionId; // CCU任务执行的MissionId
45 : uint16_t instrId;
46 : };
47 :
48 : struct MsprofCcuGroupInfo {
49 : uint8_t version;
50 : uint64_t itemId; // CCU任务名 hash id
51 : uint64_t groupName; // 通信域 hash id
52 : uint32_t rankId;
53 : uint32_t ranksize; // CCU任务设计的Chip数目
54 : uint8_t workFlowMode;
55 :
56 : uint16_t streamId;
57 : uint32_t taskId;
58 : uint8_t dieId; // CCU任务执行的DieId
59 : uint16_t instrId;
60 : uint8_t missionId; // CCU任务执行的MissionId
61 :
62 : uint8_t reduceOpType; // 与HcclReduceOp类型保持一致
63 : uint8_t inputDataType; // 与HcclDataType类型保持一致
64 : uint8_t outputDataType; // 与HcclDataType类型保持一致
65 : uint64_t dataSize; // 输入数据大小
66 :
67 : uint16_t channelId[CCU_MAX_CHANNEL_NUM]; // LoopGroup所包含的搬运指令使用的ChannelId
68 : uint32_t remoteRankId[CCU_MAX_CHANNEL_NUM]; // LoopGroup所包含的搬运指令的对端
69 : };
70 :
71 : struct MsprofCcuWaitSignalInfo {
72 : uint8_t version;
73 : uint64_t itemId; // CCU任务名 hash id
74 : uint64_t groupName; // 通信域 hash id
75 : uint32_t rankId;
76 : uint32_t ranksize; // CCU任务设计的Chip数目
77 : uint8_t workFlowMode;
78 :
79 : uint16_t streamId;
80 : uint32_t taskId;
81 : uint8_t dieId; // CCU任务执行的DieId
82 : uint16_t instrId;
83 : uint8_t missionId; // CCU任务执行的MissionId
84 :
85 : uint32_t ckeId;
86 : uint32_t mask;
87 : uint16_t channelId[CCU_MAX_CHANNEL_NUM]; // LoopGroup所包含的搬运指令使用的ChannelId
88 : uint32_t remoteRankId[CCU_MAX_CHANNEL_NUM]; // LoopGroup所包含的搬运指令的对端
89 : };
90 :
91 : struct HCCLReportData {
92 : std::string fileTag;
93 : uint64_t ts;
94 : uint32_t type;
95 : MsprofHcclInfo profInfo;
96 : MsprofDpuHcclTrack dpuProfInfo;
97 : std::string tag;
98 : std::string groupName;
99 : };
100 :
101 : const std::map<OpType, std::string> PROF_OP_NAME_V2 = {{OpType::INVALID, "hcom_invalid_"},
102 : {OpType::ALLREDUCE, "hcom_allReduce_"}, {OpType::BROADCAST, "hcom_broadcast_"},
103 : {OpType::REDUCE, "hcom_reduce_"}, {OpType::SEND, "hcom_send_"},
104 : {OpType::RECV, "hcom_receive_"}, {OpType::ALLGATHER, "hcom_allGather_"},
105 : {OpType::REDUCESCATTER, "hcom_reduceScatter_"}, {OpType::SCATTER, "hcom_scatter_"},
106 : {OpType::ALLTOALL, "hcom_alltoall_"}, {OpType::ALLTOALLV, "hcom_alltoallv_"},
107 : {OpType::ALLGATHERV, "hcom_allGatherv_"}, {OpType::REDUCESCATTERV, "hcom_reduceScatterv_"},
108 : {OpType::ALLTOALLVC, "hcom_alltoallvc_"}, {OpType::BATCHSENDRECV, "hcom_batchSendRecv_"},
109 : {OpType::BATCHPUT, "hccl_batchPut_"}, {OpType::BATCHGET, "hccl_batchGet_"},
110 : {OpType::DEBUGCASE, "hccl_debugCase_"}, {OpType::BARRIER, "hccl_barrier_"},
111 : {OpType::HALFALLTOALLV, "hccl_halfAlltoallv_"}
112 : };
113 :
114 3 : inline std::string GetProfOpName(OpType opType)
115 : {
116 3 : CHK_PRT_RET(PROF_OP_NAME_V2.empty(), HCCL_ERROR("PROF_OP_NAME_V2 has not inited."), "hcom_invalid_");
117 3 : auto it = PROF_OP_NAME_V2.find(opType);
118 3 : if (it != PROF_OP_NAME_V2.end()) {
119 3 : return it->second;
120 : }
121 0 : return PROF_OP_NAME_V2.begin()->second;
122 : }
123 :
124 : class ProfilingHandler {
125 : public:
126 : ~ProfilingHandler();
127 :
128 : ProfilingHandler(const ProfilingHandler &that) = delete;
129 :
130 : ProfilingHandler &operator=(const ProfilingHandler &that) = delete;
131 :
132 : static ProfilingHandler &GetInstance();
133 :
134 : static int32_t CommandHandleWrapper(uint32_t rtType, void *data, uint32_t len);
135 :
136 : void ReportKernel() const;
137 :
138 : void ReportHostApi(OpType opType, uint64_t beginTime, uint64_t endTime, bool cachedReq, bool isAiCpu);
139 :
140 : void ReportHcclOp(const DfxOpInfo &opInfo, bool cachedReq);
141 :
142 : void ReportHcclTaskApi(TaskParamType taskType, uint64_t beginTime, uint64_t endTime, bool isMasterStream,bool cachedReq,
143 : bool ignoreLevel = false);
144 :
145 : void ReportHcclTaskDetails(const TaskInfo &taskInfo, bool cachedReq);
146 : void ReportHcclTaskDetailsBatch(const std::vector<TaskInfo*> &taskInfos, bool cachedReq);
147 :
148 : bool GetHostApiState() const;
149 : bool GetHcclNodeState() const;
150 : bool GetHcclL0State() const;
151 : bool GetHcclL1State() const;
152 0 : inline void SetIsOpbase(bool val) { isOpbase_ = val; }
153 3 : inline bool GetIsOpbase() const { return isOpbase_; }
154 : int32_t CommandHandle(uint32_t rtType, void *data, uint32_t len) const;
155 : HcclResult Init();
156 : void ReportHcclMC2CommInfo(const Stream &kfcStream, const Stream &stream, const std::vector<Stream *> &aicpuStreams,
157 : const std::string &id, RankId myRank, u32 rankSize, RankId rankInParentComm);
158 : void ReportHcclMC2CommInfo(const u32 kfcStreamId, const std::vector<u32> &aicpuStreamsId, const std::string &id,
159 : RankId myRank, u32 rankSize, RankId rankInParentComm);
160 : void ReportNodeApi(uint64_t beginTime, uint64_t endTime, uint64_t cmdItemId, uint32_t threadId, bool cachedReq);
161 : void ReportNodeBasicInfo(uint64_t timeStamp, uint64_t cmdItemId, uint32_t threadId, bool cachedReq);
162 : uint64_t GetProfHashId(const char *name, uint32_t len) const;
163 : uint64_t GetCachedAlgTypeHashId() const { return cachedAlgTypeHashId_; }
164 : private:
165 : explicit ProfilingHandler();
166 :
167 : void ReportAclApi(uint32_t cmdType, uint64_t beginTime, uint64_t endTime, uint64_t cmdItemId,
168 : uint32_t threadId) const;
169 :
170 : void ReportHcclOpInfo(uint64_t timeStamp, const DfxOpInfo &opInfo, uint32_t threadId, bool cachedReq);
171 : void ReportAdditionInfo(MsprofAdditionalInfo& reporterData) const;
172 :
173 : void StartSubscribe(uint64_t profconfig);
174 : void StartTaskApiSubscribe();
175 : void StartHostApiSubscribe();
176 : void StartAdditionInfoSubscribe();
177 : void StartHostHcclOpSubscribe();
178 : void StartCcuSubscribe();
179 : void StopSubscribe();
180 :
181 : void CallProfRegHostApi() const;
182 : void ReportStoragedCompactInfo();
183 : void ReportMc2AdditionInfo();
184 :
185 : void CallProfRegTaskTypeApi() const;
186 : void ReportStoragedTaskApi();
187 :
188 : void CallProfRegHcclOpApi() const;
189 :
190 : void ReportStoragedAdditionInfo();
191 :
192 : void GetHCCLReportData(const TaskInfo &taskInfo, HCCLReportData &hcclReportData) const;
193 : void FillProfCommonInfo(const TaskInfo &taskInfo, MsprofAdditionalInfo &reporterData) const;
194 : void FillProfTaskSpecificInfo(const TaskInfo &taskInfo, MsprofHcclInfo *profInfo) const;
195 : void FillDpuProfInfo(const TaskInfo &taskInfo, MsprofAdditionalInfo &reporterData) const;
196 : void FillDpuTaskParaDetails(const TaskInfo &taskInfo, MsprofDpuHcclTrack *dpuProfInfo) const;
197 : void ConvertHcclInfoToDpuTrack(MsprofAdditionalInfo &reporterData) const;
198 : void FillTaskAdditionInfo(const TaskInfo &taskInfo, MsprofAdditionalInfo &reporterData) const;
199 : uint32_t GetTaskTypeValue(TaskParamType taskType) const;
200 : void CallAdditionInfo(MsprofAdditionalInfo &reporterData) const;
201 :
202 : void ReportCcuInfo(const TaskInfo &taskInfo) const;
203 : void GetCcuTaskInfo(const TaskInfo &taskInfo, const CcuProfilingInfo &info) const;
204 : void GetCcuWaitSignalInfo(const TaskInfo &taskInfo, const CcuProfilingInfo &info) const;
205 : void GetCcuGroupInfo(const TaskInfo &taskInfo, const CcuProfilingInfo &info) const;
206 :
207 : void DumpHCCLReportData(const TaskInfo &taskInfo, const MsprofAdditionalInfo &reporterData) const;
208 : void DumpCcuGroupInfo(const MsprofCcuGroupInfo& ccuGroupInfo) const;
209 : void ReportMc2AdditionInfo(uint64_t timeStamp, const void* data, int len);
210 : void SetCachedCclTag();
211 : void InitLog() const;
212 : void ReportHcclMC2CommInfoLog(const Stream &kfcStream, const Stream &stream,
213 : const std::vector<Stream *> &aicpuStreams, const std::string &id,
214 : RankId myRank, u32 rankSize, RankId rankInParentComm) const;
215 : void ReportHcclMC2CommInfoLog(const u32 kfcStreamId, const std::vector<u32> &aicpuStreamsId,
216 : const std::string &id, RankId myRank, u32 rankSize,
217 : RankId rankInParentComm) const;
218 : void ReportCcuInfoLog(const TaskInfo &taskInfo) const;
219 : void LogCcuTaskInfo(const CcuProfilingInfo &info, const TaskInfo &taskInfo,
220 : uint64_t itemId, uint64_t groupName, u32 rankId, u32 ranksize) const;
221 : void LogCcuWaitSignalInfo(const CcuProfilingInfo &info, const TaskInfo &taskInfo,
222 : uint64_t itemId, uint64_t groupName, u32 rankId, u32 ranksize) const;
223 : void LogCcuGroupInfo(const CcuProfilingInfo &info, const TaskInfo &taskInfo,
224 : uint64_t itemId, uint64_t groupName, u32 rankId, u32 ranksize) const;
225 : void ReportHcclTaskDetailsBatchLog(const std::vector<TaskInfo*> &taskInfos) const;
226 : void ReportStoragedAdditionInfoLog() const;
227 :
228 : private:
229 : static ProfilingHandler instance_;
230 : bool initializedFlag_{false};
231 : bool enableHostApi_{false};
232 : bool enableHcclNode_{false};
233 : bool enableHcclL0_{false};
234 : bool enableHcclL1_{false};
235 : bool isOpbase_{false};
236 :
237 : std::vector<TaskInfo> cacheTaskInfos_{};
238 : std::queue<MsprofApi> cachedTaskApiInfo_{};
239 : std::queue<MsprofCompactInfo> cacheHcclOpInfo_{};
240 : std::queue<MsprofAdditionalInfo> cacheHcclAdditionInfo_{};
241 : std::unordered_map<std::string, uint64_t> str2HashId_{};
242 : uint64_t cachedAlgTypeHashId_{0};
243 : std::map<uint32_t, uint64_t> cachedNewCclTag_{};
244 : mutable std::mutex cacheTaskInfosMutex_;
245 : std::mutex cachedTaskApiInfoMutex_;
246 : std::mutex cacheHcclOpInfoMutex_;
247 : std::mutex cacheHcclAdditionInfoMutex_;
248 : };
249 : } // namespace Hccl
250 :
251 : #endif // HCCL_PROFILING_HANDLER_H
|
