Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include <sstream>
12 : #include "task_info.h"
13 : #include "log.h"
14 : #include "unified_platform/pub_inc/config_plf_log.h"
15 : #include "string_util.h"
16 : #include "const_val.h"
17 : #include "reduce_op.h"
18 : #include "data_type.h"
19 :
20 : namespace Hccl {
21 : using namespace std;
22 :
23 10229 : TaskInfo::TaskInfo(u32 streamId, u32 taskId, u32 remoteRank, const TaskParam& taskParam, const std::shared_ptr<DfxOpInfo>& dfxOpInfo, bool isMaster)
24 10229 : : streamId_(streamId), taskId_(taskId), remoteRank_(remoteRank), taskParam_(taskParam), dfxOpInfo_(dfxOpInfo), isMaster_(isMaster)
25 10229 : {}
26 :
27 16 : std::string TaskInfo::Describe() const
28 : {
29 : return StringFormat("TaskInfo[streamId(sqId):[%u], taskId(sqeId):[%u], remoteRank:[%u], taskParam:[%s], dfxOpInfo:[%s], isMaster[%d]]",
30 32 : streamId_, taskId_, remoteRank_, taskParam_.Describe().c_str(),
31 48 : dfxOpInfo_ == nullptr ? "nullptr" : dfxOpInfo_->Describe().c_str(), isMaster_);
32 : }
33 :
34 7 : string TaskInfo::GetAlgTypeName() const
35 : {
36 7 : if (this->dfxOpInfo_ == nullptr) {
37 3 : HCCL_ERROR("[TaskInfo][%s]dfxOpInfo is nullptr, no op info registered, return default value. "
38 : "streamId(sqId)[%u], taskId(sqeId)[%u].", __func__, streamId_, taskId_);
39 2 : return "NULL";
40 : }
41 6 : return this->dfxOpInfo_->algType_;
42 : }
43 :
44 5 : string TaskInfo::GetBaseInfo() const
45 : {
46 5 : if (this->dfxOpInfo_ == nullptr) {
47 3 : HCCL_ERROR("[TaskInfo][%s]dfxOpInfo is nullptr, no op info registered, return default value. "
48 : "streamId(sqId)[%u], taskId(sqeId)[%u].", __func__, streamId_, taskId_);
49 2 : return "";
50 : }
51 : return StringFormat("streamID(sqId):[%u], taskID(sqeId):[%u], taskType:[%s]",
52 4 : this->streamId_,
53 4 : this->taskId_,
54 4 : this->taskParam_.taskType.Describe().c_str());
55 : }
56 :
57 6 : string TaskInfo::GetParaInfo() const
58 : {
59 6 : switch (this->taskParam_.taskType) {
60 1 : case TaskParamType::TASK_SDMA:
61 : case TaskParamType::TASK_RDMA:
62 : case TaskParamType::TASK_SEND_PAYLOAD:
63 : case TaskParamType::TASK_UB_INLINE_WRITE:
64 : case TaskParamType::TASK_UB:
65 : case TaskParamType::TASK_WRITE_WITH_NOTIFY:
66 : case TaskParamType::TASK_WRITE_REDUCE_WITH_NOTIFY:
67 : case TaskParamType::TASK_DPU_INLINE_WRITE:
68 : case TaskParamType::TASK_DPU_WRITE_WITH_NOTIFY:
69 1 : return GetParaDMA();
70 1 : case TaskParamType::TASK_REDUCE_INLINE:
71 : case TaskParamType::TASK_UB_REDUCE_INLINE:
72 : case TaskParamType::TASK_REDUCE_TBE:
73 1 : return GetParaReduce();
74 3 : case TaskParamType::TASK_NOTIFY_RECORD:
75 : case TaskParamType::TASK_NOTIFY_WAIT:
76 : case TaskParamType::TASK_SEND_NOTIFY:
77 : case TaskParamType::TASK_DPU_NOTIFY_WAIT:
78 : case TaskParamType::TASK_DPU_CHANNEL_FENCE:
79 3 : return GetParaNotify();
80 0 : case TaskParamType::TASK_AIV:
81 0 : return GetParaAiv();
82 1 : default:
83 1 : return this->taskParam_.taskType.Describe();
84 : }
85 : }
86 :
87 1 : string TaskInfo::GetParaDMA() const
88 : {
89 1 : const auto& taskPara = this->taskParam_.taskPara;
90 : return StringFormat("src:[0x%llx], dst:[0x%llx], size:[0x%llx], notify id:[0x%016llx], "
91 : "link type:[%s], remote rank:[%s]",
92 1 : static_cast<u64>(reinterpret_cast<uintptr_t>(taskPara.DMA.src)),
93 1 : static_cast<u64>(reinterpret_cast<uintptr_t>(taskPara.DMA.dst)),
94 1 : static_cast<u64>(taskPara.DMA.size),
95 1 : taskPara.DMA.notifyID,
96 2 : taskPara.DMA.linkType.Describe().c_str(),
97 3 : this->GetRemoteRankInfo().c_str());
98 : }
99 :
100 1 : string TaskInfo::GetParaReduce() const
101 : {
102 1 : const auto& taskPara = this->taskParam_.taskPara;
103 : return StringFormat("src:[0x%llx], dst:[0x%llx], size:[0x%llx], notify id:[0x%016llx], "
104 : "op:[%u], data type:[%u], link type:[%s], remote rank:[%s]",
105 1 : static_cast<u64>(reinterpret_cast<uintptr_t>(taskPara.Reduce.src)),
106 1 : static_cast<u64>(reinterpret_cast<uintptr_t>(taskPara.Reduce.dst)),
107 1 : static_cast<u64>(taskPara.Reduce.size),
108 1 : taskPara.Reduce.notifyID,
109 1 : static_cast<u32>(taskPara.Reduce.reduceOp),
110 1 : static_cast<u32>(taskPara.Reduce.dataType),
111 2 : taskPara.Reduce.linkType.Describe().c_str(),
112 3 : this->GetRemoteRankInfo().c_str());
113 : }
114 :
115 3 : string TaskInfo::GetParaNotify() const
116 : {
117 3 : const auto& taskPara = this->taskParam_.taskPara;
118 : return StringFormat("notify id:[0x%016llx], value:[%u], remote rank:[%s]",
119 3 : taskPara.Notify.notifyID,
120 3 : taskPara.Notify.value,
121 3 : this->GetRemoteRankInfo().c_str());
122 : }
123 :
124 5 : string TaskInfo::GetOpInfo() const
125 : {
126 5 : if (this->dfxOpInfo_ == nullptr) {
127 3 : HCCL_ERROR("[TaskInfo][%s]dfxOpInfo is nullptr, no op info registered, return default value. "
128 : "streamId(sqId)[%u], taskId(sqeId)[%u].", __func__, streamId_, taskId_);
129 2 : return "";
130 : }
131 4 : const auto opInfo = this->dfxOpInfo_;
132 : return StringFormat("commIndex[%u], opType[%s], commId[%s], count[%llu], reduceType[%s], dataType[%s]",
133 4 : opInfo->commIndex_,
134 8 : opInfo->op_.opType.Describe().c_str(),
135 4 : opInfo->commId_.c_str(),
136 4 : opInfo->op_.dataCount,
137 8 : opInfo->op_.reduceOp.Describe().c_str(),
138 24 : opInfo->op_.dataType.Describe().c_str());
139 4 : }
140 :
141 66 : string TaskInfo::GetRemoteRankInfo(bool needConcise) const
142 : {
143 66 : string invRank = needConcise ? "/" : "local";
144 132 : return (this->GetRemoteRankId() == INVALID_VALUE_RANKID) ? invRank : to_string(this->GetRemoteRankId());
145 66 : }
146 :
147 73 : string TaskInfo::GetTaskConciseName() const
148 : {
149 : static const map<TaskParamType, string> taskConciseNameMap {
150 0 : {TaskParamType::TASK_SDMA, "M"},
151 0 : {TaskParamType::TASK_RDMA, "RS"},
152 0 : {TaskParamType::TASK_SEND_PAYLOAD, "SP"},
153 0 : {TaskParamType::TASK_REDUCE_INLINE, "IR"},
154 0 : {TaskParamType::TASK_UB_REDUCE_INLINE, "IR"},
155 0 : {TaskParamType::TASK_UB, "WorR"},
156 0 : {TaskParamType::TASK_REDUCE_TBE, "R"},
157 0 : {TaskParamType::TASK_NOTIFY_RECORD, "NR"},
158 0 : {TaskParamType::TASK_NOTIFY_WAIT, "NW"},
159 0 : {TaskParamType::TASK_SEND_NOTIFY, "SN"},
160 0 : {TaskParamType::TASK_WRITE_WITH_NOTIFY, "WN"},
161 0 : {TaskParamType::TASK_UB_INLINE_WRITE, "IW"},
162 0 : {TaskParamType::TASK_WRITE_REDUCE_WITH_NOTIFY, "WRN"},
163 0 : {TaskParamType::TASK_CCU, "CCU"},
164 90 : {TaskParamType::TASK_AICPU_KERNEL, "AIK"}};
165 :
166 73 : const auto taskName = taskConciseNameMap.find(this->taskParam_.taskType);
167 73 : if (taskName == taskConciseNameMap.end()) {
168 0 : return "UNKNOWN";
169 : } else {
170 73 : return taskName->second;
171 : }
172 1 : }
173 :
174 60 : string TaskInfo::GetNotifyInfo() const
175 : {
176 60 : const auto& taskPara = this->taskParam_.taskPara;
177 60 : u64 notifyInfo = INVALID_U64;
178 60 : switch (this->taskParam_.taskType) {
179 2 : case TaskParamType::TASK_RDMA:
180 : case TaskParamType::TASK_UB_INLINE_WRITE:
181 2 : notifyInfo = taskPara.DMA.notifyID;
182 2 : break;
183 57 : case TaskParamType::TASK_NOTIFY_RECORD:
184 : case TaskParamType::TASK_NOTIFY_WAIT:
185 : case TaskParamType::TASK_SEND_NOTIFY:
186 : case TaskParamType::TASK_WRITE_WITH_NOTIFY:
187 : case TaskParamType::TASK_WRITE_REDUCE_WITH_NOTIFY:
188 57 : notifyInfo = taskPara.Notify.notifyID;
189 57 : break;
190 1 : default:
191 2 : return "/";
192 : }
193 59 : if (notifyInfo == INVALID_U64) {
194 2 : return "/";
195 : } else {
196 58 : stringstream paraStr;
197 58 : paraStr << notifyInfo;
198 58 : return paraStr.str();
199 58 : }
200 : }
201 :
202 61 : string TaskInfo::GetConciseBaseInfo() const
203 : {
204 61 : stringstream taskConciseInfo;
205 61 : taskConciseInfo << this->GetTaskConciseName();
206 61 : taskConciseInfo << "(";
207 61 : taskConciseInfo << this->GetRemoteRankInfo(true);
208 61 : const auto taskType = this->taskParam_.taskType;
209 179 : if (taskType == TaskParamType::TASK_RDMA || taskType == TaskParamType::TASK_NOTIFY_RECORD ||
210 73 : taskType == TaskParamType::TASK_NOTIFY_WAIT || taskType == TaskParamType::TASK_SEND_NOTIFY ||
211 133 : taskType == TaskParamType::TASK_WRITE_WITH_NOTIFY || taskType == TaskParamType::TASK_WRITE_REDUCE_WITH_NOTIFY ||
212 5 : taskType == TaskParamType::TASK_UB_INLINE_WRITE) {
213 56 : taskConciseInfo << "," << this->GetNotifyInfo();
214 : }
215 61 : taskConciseInfo << ")";
216 122 : return taskConciseInfo.str();
217 61 : }
218 :
219 1 : string TaskInfo::GetIndopBaseInfo() const
220 : {
221 : return Hccl::StringFormat("streamID(sqId):[%u], taskID(sqeId):[%u], taskType:[%s]",
222 1 : this->streamId_, this->taskId_, this->taskParam_.taskType.Describe().c_str());
223 : }
224 :
225 2 : string TaskInfo::GetIndopDataInfo() const
226 : {
227 2 : if (this->dfxOpInfo_ == nullptr) {
228 3 : HCCL_ERROR("[TaskInfo][%s]dfxOpInfo is nullptr, no op info registered, return default value. "
229 : "streamId(sqId)[%u], taskId(sqeId)[%u].", __func__, streamId_, taskId_);
230 2 : return "";
231 : }
232 :
233 1 : const auto &opInfo = this->dfxOpInfo_;
234 :
235 1 : ReduceOp reduceOp = opInfo->op_.reduceOp;
236 1 : auto reduceOpIt = REDUCE_OP_MAP.find(static_cast<HcclReduceOp>(opInfo->op_.oldReduceOp));
237 1 : if (reduceOpIt != REDUCE_OP_MAP.end()) {
238 1 : reduceOp = reduceOpIt->second;
239 : }
240 :
241 1 : DataType dataType = opInfo->op_.dataType;
242 1 : auto datatypeIt = DATA_TYPE_MAP.find(static_cast<HcclDataType>(opInfo->op_.oldDataType));
243 1 : if (datatypeIt != DATA_TYPE_MAP.end()) {
244 1 : dataType = datatypeIt->second;
245 : }
246 :
247 : return Hccl::StringFormat("opIndex[%u], algTag[%s], count[%llu], reduceType[%s], dataType[%s], "\
248 : "input: ptr[0x%llx] size[%llu], output: ptr[0x%llx] size[%llu]",
249 1 : opInfo->opIndex_,
250 1 : opInfo->algTag_.c_str(),
251 1 : opInfo->op_.dataCount,
252 2 : reduceOp.Describe().c_str(),
253 1 : dataType.Describe().c_str(),
254 1 : opInfo->op_.inputAddr,
255 1 : opInfo->op_.inputSize,
256 1 : opInfo->op_.outputAddr,
257 4 : opInfo->op_.outputSize);
258 : }
259 :
260 0 : string TaskInfo::GetParaAiv() const
261 : {
262 0 : const auto &taskPara = this->taskParam_.taskPara;
263 : return StringFormat("cmdType:[%d], tag:[%u], count:[%llu], numBlocks:[%u], rankSize:[%u], "
264 : "rank:[%u], sendRecvRemoteRank:[%u], dataType:[%d], remote rank:[%s]",
265 0 : static_cast<int>(taskPara.Aiv.cmdType), taskPara.Aiv.tag,
266 0 : taskPara.Aiv.count, taskPara.Aiv.numBlocks, taskPara.Aiv.rankSize,
267 0 : taskPara.Aiv.rank, taskPara.Aiv.sendRecvRemoteRank,
268 0 : static_cast<int>(taskPara.Aiv.dataType),
269 0 : this->GetRemoteRankInfo().c_str());
270 : }
271 :
272 137 : u32 TaskInfo::GetRemoteRankId() const
273 : {
274 137 : return getRemoteRankByHandle_ ? getRemoteRankByHandle_(channelHandle_) : remoteRank_;
275 : }
276 :
277 3 : void PrintTaskLog(u32 streamId, u32 taskId, const TaskParam &taskParam, u32 remoteRankId)
278 : {
279 3 : switch (taskParam.taskType) {
280 1 : case TaskParamType::TASK_SDMA:
281 : case TaskParamType::TASK_RDMA:
282 : case TaskParamType::TASK_UB:
283 3 : PLF_CONFIG_INFO(PLF_TASK, "[AddTaskInfo] streamId=%u, taskId=%u, taskType=%s, "
284 : "remoteRank=%u, src=%p, dst=%p, size=%llu",
285 : streamId, taskId, taskParam.taskType.Describe().c_str(), remoteRankId,
286 : taskParam.taskPara.DMA.src, taskParam.taskPara.DMA.dst,
287 : static_cast<u64>(taskParam.taskPara.DMA.size));
288 1 : break;
289 0 : case TaskParamType::TASK_UB_INLINE_WRITE:
290 : case TaskParamType::TASK_WRITE_WITH_NOTIFY:
291 : case TaskParamType::TASK_WRITE_REDUCE_WITH_NOTIFY:
292 : case TaskParamType::TASK_DPU_INLINE_WRITE:
293 : case TaskParamType::TASK_DPU_WRITE_WITH_NOTIFY:
294 0 : PLF_CONFIG_INFO(PLF_TASK, "[AddTaskInfo] streamId=%u, taskId=%u, taskType=%s, "
295 : "remoteRank=%u, src=%p, dst=%p, size=%llu, notifyId=%llu",
296 : streamId, taskId, taskParam.taskType.Describe().c_str(), remoteRankId,
297 : taskParam.taskPara.DMA.src, taskParam.taskPara.DMA.dst,
298 : static_cast<u64>(taskParam.taskPara.DMA.size), taskParam.taskPara.DMA.notifyID);
299 0 : break;
300 1 : case TaskParamType::TASK_REDUCE_INLINE:
301 : case TaskParamType::TASK_UB_REDUCE_INLINE:
302 : case TaskParamType::TASK_REDUCE_TBE:
303 3 : PLF_CONFIG_INFO(PLF_TASK, "[AddTaskInfo] streamId=%u, taskId=%u, taskType=%s, "
304 : "remoteRank=%u, src=%p, dst=%p, size=%llu, notifyId=%llu, "
305 : "dataType=%d, reduceOp=%d",
306 : streamId, taskId, taskParam.taskType.Describe().c_str(), remoteRankId,
307 : taskParam.taskPara.Reduce.src, taskParam.taskPara.Reduce.dst,
308 : static_cast<u64>(taskParam.taskPara.Reduce.size), taskParam.taskPara.Reduce.notifyID,
309 : taskParam.taskPara.Reduce.dataType, taskParam.taskPara.Reduce.reduceOp);
310 1 : break;
311 1 : case TaskParamType::TASK_NOTIFY_RECORD:
312 : case TaskParamType::TASK_NOTIFY_WAIT:
313 : case TaskParamType::TASK_SEND_NOTIFY:
314 : case TaskParamType::TASK_DPU_NOTIFY_WAIT:
315 : case TaskParamType::TASK_DPU_CHANNEL_FENCE:
316 3 : PLF_CONFIG_INFO(PLF_TASK, "[AddTaskInfo] streamId=%u, taskId=%u, taskType=%s, "
317 : "remoteRank=%u, notifyId=%llu",
318 : streamId, taskId, taskParam.taskType.Describe().c_str(), remoteRankId,
319 : taskParam.taskPara.Notify.notifyID);
320 1 : break;
321 0 : default:
322 0 : PLF_CONFIG_INFO(PLF_TASK, "[AddTaskInfo] streamId=%u, taskId=%u, taskType=%s",
323 : streamId, taskId, taskParam.taskType.Describe().c_str());
324 0 : break;
325 : }
326 3 : }
327 :
328 : } // namespace Hccl
|
