Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include <sstream>
12 : #include "task_info.h"
13 : #include "log.h"
14 : #include "unified_platform/pub_inc/config_plf_log.h"
15 : #include "string_util.h"
16 : #include "const_val.h"
17 : #include "reduce_op.h"
18 : #include "data_type.h"
19 :
20 : namespace Hccl {
21 : using namespace std;
22 :
23 10232 : TaskInfo::TaskInfo(
24 : u32 streamId, u32 taskId, u32 remoteRank, const TaskParam& taskParam, const std::shared_ptr<DfxOpInfo>& dfxOpInfo,
25 10232 : bool isMaster)
26 10232 : : streamId_(streamId),
27 10232 : taskId_(taskId),
28 10232 : remoteRank_(remoteRank),
29 10232 : taskParam_(taskParam),
30 10232 : dfxOpInfo_(dfxOpInfo),
31 10232 : isMaster_(isMaster)
32 10232 : {}
33 :
34 16 : std::string TaskInfo::Describe() const
35 : {
36 : return StringFormat(
37 : "TaskInfo[streamId(sqId):[%u], taskId(sqeId):[%u], remoteRank:[%u], taskParam:[%s], dfxOpInfo:[%s], "
38 : "isMaster[%d]]",
39 32 : streamId_, taskId_, remoteRank_, taskParam_.Describe().c_str(),
40 48 : dfxOpInfo_ == nullptr ? "nullptr" : dfxOpInfo_->Describe().c_str(), isMaster_);
41 : }
42 :
43 7 : string TaskInfo::GetAlgTypeName() const
44 : {
45 7 : if (this->dfxOpInfo_ == nullptr) {
46 3 : HCCL_ERROR(
47 : "[TaskInfo][%s]dfxOpInfo is nullptr, no op info registered, return default value. "
48 : "streamId(sqId)[%u], taskId(sqeId)[%u].",
49 : __func__, streamId_, taskId_);
50 2 : return "NULL";
51 : }
52 6 : return this->dfxOpInfo_->algType_;
53 : }
54 :
55 8 : string TaskInfo::GetBaseInfo() const
56 : {
57 8 : if (this->dfxOpInfo_ == nullptr) {
58 3 : HCCL_ERROR(
59 : "[TaskInfo][%s]dfxOpInfo is nullptr, no op info registered, return default value. "
60 : "streamId(sqId)[%u], taskId(sqeId)[%u].",
61 : __func__, streamId_, taskId_);
62 2 : return "";
63 : }
64 : return StringFormat(
65 7 : "streamID(sqId):[%u], taskID(sqeId):[%u], taskType:[%s]", this->streamId_, this->taskId_,
66 7 : this->taskParam_.taskType.Describe().c_str());
67 : }
68 :
69 9 : string TaskInfo::GetParaInfo() const
70 : {
71 9 : switch (this->taskParam_.taskType) {
72 1 : case TaskParamType::TASK_SDMA:
73 : case TaskParamType::TASK_RDMA:
74 : case TaskParamType::TASK_SEND_PAYLOAD:
75 : case TaskParamType::TASK_UB_INLINE_WRITE:
76 : case TaskParamType::TASK_UB:
77 : case TaskParamType::TASK_WRITE_WITH_NOTIFY:
78 : case TaskParamType::TASK_WRITE_REDUCE_WITH_NOTIFY:
79 : case TaskParamType::TASK_DPU_INLINE_WRITE:
80 : case TaskParamType::TASK_DPU_WRITE_WITH_NOTIFY:
81 1 : return GetParaDMA();
82 1 : case TaskParamType::TASK_REDUCE_INLINE:
83 : case TaskParamType::TASK_UB_REDUCE_INLINE:
84 : case TaskParamType::TASK_REDUCE_TBE:
85 1 : return GetParaReduce();
86 6 : case TaskParamType::TASK_NOTIFY_RECORD:
87 : case TaskParamType::TASK_NOTIFY_WAIT:
88 : case TaskParamType::TASK_SEND_NOTIFY:
89 : case TaskParamType::TASK_DPU_NOTIFY_WAIT:
90 : case TaskParamType::TASK_DPU_CHANNEL_FENCE:
91 6 : return GetParaNotify();
92 0 : case TaskParamType::TASK_AIV:
93 0 : return GetParaAiv();
94 1 : default:
95 1 : return this->taskParam_.taskType.Describe();
96 : }
97 : }
98 :
99 1 : string TaskInfo::GetParaDMA() const
100 : {
101 1 : const auto& taskPara = this->taskParam_.taskPara;
102 : return StringFormat(
103 : "src:[0x%llx], dst:[0x%llx], size:[0x%llx], notify id:[0x%016llx], "
104 : "link type:[%s], remote rank:[%s]",
105 1 : static_cast<u64>(reinterpret_cast<uintptr_t>(taskPara.DMA.src)),
106 1 : static_cast<u64>(reinterpret_cast<uintptr_t>(taskPara.DMA.dst)), static_cast<u64>(taskPara.DMA.size),
107 1 : taskPara.DMA.notifyID, taskPara.DMA.linkType.Describe().c_str(), this->GetRemoteRankInfo().c_str());
108 : }
109 :
110 1 : string TaskInfo::GetParaReduce() const
111 : {
112 1 : const auto& taskPara = this->taskParam_.taskPara;
113 : return StringFormat(
114 : "src:[0x%llx], dst:[0x%llx], size:[0x%llx], notify id:[0x%016llx], "
115 : "op:[%u], data type:[%u], link type:[%s], remote rank:[%s]",
116 1 : static_cast<u64>(reinterpret_cast<uintptr_t>(taskPara.Reduce.src)),
117 1 : static_cast<u64>(reinterpret_cast<uintptr_t>(taskPara.Reduce.dst)), static_cast<u64>(taskPara.Reduce.size),
118 1 : taskPara.Reduce.notifyID, static_cast<u32>(taskPara.Reduce.reduceOp),
119 2 : static_cast<u32>(taskPara.Reduce.dataType), taskPara.Reduce.linkType.Describe().c_str(),
120 3 : this->GetRemoteRankInfo().c_str());
121 : }
122 :
123 6 : string TaskInfo::GetParaNotify() const
124 : {
125 6 : const auto& taskPara = this->taskParam_.taskPara;
126 : return StringFormat(
127 6 : "notify id:[0x%016llx], value:[%u], remote rank:[%s]", taskPara.Notify.notifyID, taskPara.Notify.value,
128 6 : this->GetRemoteRankInfo().c_str());
129 : }
130 :
131 5 : string TaskInfo::GetOpInfo() const
132 : {
133 5 : if (this->dfxOpInfo_ == nullptr) {
134 3 : HCCL_ERROR(
135 : "[TaskInfo][%s]dfxOpInfo is nullptr, no op info registered, return default value. "
136 : "streamId(sqId)[%u], taskId(sqeId)[%u].",
137 : __func__, streamId_, taskId_);
138 2 : return "";
139 : }
140 4 : const auto opInfo = this->dfxOpInfo_;
141 : return StringFormat(
142 4 : "commIndex[%u], opType[%s], commId[%s], count[%llu], reduceType[%s], dataType[%s]", opInfo->commIndex_,
143 12 : opInfo->op_.opType.Describe().c_str(), opInfo->commId_.c_str(), opInfo->op_.dataCount,
144 20 : opInfo->op_.reduceOp.Describe().c_str(), opInfo->op_.dataType.Describe().c_str());
145 4 : }
146 :
147 69 : string TaskInfo::GetRemoteRankInfo(bool needConcise) const
148 : {
149 69 : string invRank = needConcise ? "/" : "local";
150 138 : return (this->GetRemoteRankId() == INVALID_VALUE_RANKID) ? invRank : to_string(this->GetRemoteRankId());
151 69 : }
152 :
153 73 : string TaskInfo::GetTaskConciseName() const
154 : {
155 : static const map<TaskParamType, string> taskConciseNameMap{
156 0 : {TaskParamType::TASK_SDMA, "M"},
157 0 : {TaskParamType::TASK_RDMA, "RS"},
158 0 : {TaskParamType::TASK_SEND_PAYLOAD, "SP"},
159 0 : {TaskParamType::TASK_REDUCE_INLINE, "IR"},
160 0 : {TaskParamType::TASK_UB_REDUCE_INLINE, "IR"},
161 0 : {TaskParamType::TASK_UB, "WorR"},
162 0 : {TaskParamType::TASK_REDUCE_TBE, "R"},
163 0 : {TaskParamType::TASK_NOTIFY_RECORD, "NR"},
164 0 : {TaskParamType::TASK_NOTIFY_WAIT, "NW"},
165 0 : {TaskParamType::TASK_SEND_NOTIFY, "SN"},
166 0 : {TaskParamType::TASK_WRITE_WITH_NOTIFY, "WN"},
167 0 : {TaskParamType::TASK_UB_INLINE_WRITE, "IW"},
168 0 : {TaskParamType::TASK_WRITE_REDUCE_WITH_NOTIFY, "WRN"},
169 0 : {TaskParamType::TASK_CCU, "CCU"},
170 90 : {TaskParamType::TASK_AICPU_KERNEL, "AIK"}};
171 :
172 73 : const auto taskName = taskConciseNameMap.find(this->taskParam_.taskType);
173 73 : if (taskName == taskConciseNameMap.end()) {
174 0 : return "UNKNOWN";
175 : } else {
176 73 : return taskName->second;
177 : }
178 1 : }
179 :
180 60 : string TaskInfo::GetNotifyInfo() const
181 : {
182 60 : const auto& taskPara = this->taskParam_.taskPara;
183 60 : u64 notifyInfo = INVALID_U64;
184 60 : switch (this->taskParam_.taskType) {
185 2 : case TaskParamType::TASK_RDMA:
186 : case TaskParamType::TASK_UB_INLINE_WRITE:
187 2 : notifyInfo = taskPara.DMA.notifyID;
188 2 : break;
189 57 : case TaskParamType::TASK_NOTIFY_RECORD:
190 : case TaskParamType::TASK_NOTIFY_WAIT:
191 : case TaskParamType::TASK_SEND_NOTIFY:
192 : case TaskParamType::TASK_WRITE_WITH_NOTIFY:
193 : case TaskParamType::TASK_WRITE_REDUCE_WITH_NOTIFY:
194 57 : notifyInfo = taskPara.Notify.notifyID;
195 57 : break;
196 1 : default:
197 2 : return "/";
198 : }
199 59 : if (notifyInfo == INVALID_U64) {
200 2 : return "/";
201 : } else {
202 58 : stringstream paraStr;
203 58 : paraStr << notifyInfo;
204 58 : return paraStr.str();
205 58 : }
206 : }
207 :
208 61 : string TaskInfo::GetConciseBaseInfo() const
209 : {
210 61 : stringstream taskConciseInfo;
211 61 : taskConciseInfo << this->GetTaskConciseName();
212 61 : taskConciseInfo << "(";
213 61 : taskConciseInfo << this->GetRemoteRankInfo(true);
214 61 : const auto taskType = this->taskParam_.taskType;
215 121 : if (taskType == TaskParamType::TASK_RDMA || taskType == TaskParamType::TASK_NOTIFY_RECORD
216 58 : || taskType == TaskParamType::TASK_NOTIFY_WAIT || taskType == TaskParamType::TASK_SEND_NOTIFY
217 7 : || taskType == TaskParamType::TASK_WRITE_WITH_NOTIFY || taskType == TaskParamType::TASK_WRITE_REDUCE_WITH_NOTIFY
218 121 : || taskType == TaskParamType::TASK_UB_INLINE_WRITE) {
219 56 : taskConciseInfo << "," << this->GetNotifyInfo();
220 : }
221 61 : taskConciseInfo << ")";
222 122 : return taskConciseInfo.str();
223 61 : }
224 :
225 1 : string TaskInfo::GetIndopBaseInfo() const
226 : {
227 : return Hccl::StringFormat(
228 1 : "streamID(sqId):[%u], taskID(sqeId):[%u], taskType:[%s]", this->streamId_, this->taskId_,
229 1 : this->taskParam_.taskType.Describe().c_str());
230 : }
231 :
232 2 : string TaskInfo::GetIndopDataInfo() const
233 : {
234 2 : if (this->dfxOpInfo_ == nullptr) {
235 3 : HCCL_ERROR(
236 : "[TaskInfo][%s]dfxOpInfo is nullptr, no op info registered, return default value. "
237 : "streamId(sqId)[%u], taskId(sqeId)[%u].",
238 : __func__, streamId_, taskId_);
239 2 : return "";
240 : }
241 :
242 1 : const auto& opInfo = this->dfxOpInfo_;
243 :
244 1 : ReduceOp reduceOp = opInfo->op_.reduceOp;
245 1 : auto reduceOpIt = REDUCE_OP_MAP.find(static_cast<HcclReduceOp>(opInfo->op_.oldReduceOp));
246 1 : if (reduceOpIt != REDUCE_OP_MAP.end()) {
247 1 : reduceOp = reduceOpIt->second;
248 : }
249 :
250 1 : DataType dataType = opInfo->op_.dataType;
251 1 : auto datatypeIt = DATA_TYPE_MAP.find(static_cast<HcclDataType>(opInfo->op_.oldDataType));
252 1 : if (datatypeIt != DATA_TYPE_MAP.end()) {
253 1 : dataType = datatypeIt->second;
254 : }
255 :
256 : return Hccl::StringFormat(
257 : "opIndex[%u], algTag[%s], count[%llu], reduceType[%s], dataType[%s], "
258 : "input: ptr[0x%llx] size[%llu], output: ptr[0x%llx] size[%llu]",
259 4 : opInfo->opIndex_, opInfo->algTag_.c_str(), opInfo->op_.dataCount, reduceOp.Describe().c_str(),
260 1 : dataType.Describe().c_str(), opInfo->op_.inputAddr, opInfo->op_.inputSize, opInfo->op_.outputAddr,
261 4 : opInfo->op_.outputSize);
262 : }
263 :
264 0 : string TaskInfo::GetParaAiv() const
265 : {
266 0 : const auto& taskPara = this->taskParam_.taskPara;
267 : return StringFormat(
268 : "cmdType:[%d], tag:[%u], count:[%llu], numBlocks:[%u], rankSize:[%u], "
269 : "rank:[%u], sendRecvRemoteRank:[%u], dataType:[%d], remote rank:[%s]",
270 0 : static_cast<int>(taskPara.Aiv.cmdType), taskPara.Aiv.tag, taskPara.Aiv.count, taskPara.Aiv.numBlocks,
271 0 : taskPara.Aiv.rankSize, taskPara.Aiv.rank, taskPara.Aiv.sendRecvRemoteRank,
272 0 : static_cast<int>(taskPara.Aiv.dataType), this->GetRemoteRankInfo().c_str());
273 : }
274 :
275 143 : u32 TaskInfo::GetRemoteRankId() const
276 : {
277 143 : return getRemoteRankByHandle_ ? getRemoteRankByHandle_(channelHandle_) : remoteRank_;
278 : }
279 :
280 3 : void PrintTaskLog(u32 streamId, u32 taskId, const TaskParam& taskParam, u32 remoteRankId)
281 : {
282 3 : switch (taskParam.taskType) {
283 1 : case TaskParamType::TASK_SDMA:
284 : case TaskParamType::TASK_RDMA:
285 : case TaskParamType::TASK_UB:
286 3 : PLF_CONFIG_INFO(
287 : PLF_TASK,
288 : "[AddTaskInfo] streamId=%u, taskId=%u, taskType=%s, "
289 : "remoteRank=%u, src=%p, dst=%p, size=%llu",
290 : streamId, taskId, taskParam.taskType.Describe().c_str(), remoteRankId, taskParam.taskPara.DMA.src,
291 : taskParam.taskPara.DMA.dst, static_cast<u64>(taskParam.taskPara.DMA.size));
292 1 : break;
293 0 : case TaskParamType::TASK_UB_INLINE_WRITE:
294 : case TaskParamType::TASK_WRITE_WITH_NOTIFY:
295 : case TaskParamType::TASK_WRITE_REDUCE_WITH_NOTIFY:
296 : case TaskParamType::TASK_DPU_INLINE_WRITE:
297 : case TaskParamType::TASK_DPU_WRITE_WITH_NOTIFY:
298 0 : PLF_CONFIG_INFO(
299 : PLF_TASK,
300 : "[AddTaskInfo] streamId=%u, taskId=%u, taskType=%s, "
301 : "remoteRank=%u, src=%p, dst=%p, size=%llu, notifyId=%llu",
302 : streamId, taskId, taskParam.taskType.Describe().c_str(), remoteRankId, taskParam.taskPara.DMA.src,
303 : taskParam.taskPara.DMA.dst, static_cast<u64>(taskParam.taskPara.DMA.size),
304 : taskParam.taskPara.DMA.notifyID);
305 0 : break;
306 1 : case TaskParamType::TASK_REDUCE_INLINE:
307 : case TaskParamType::TASK_UB_REDUCE_INLINE:
308 : case TaskParamType::TASK_REDUCE_TBE:
309 3 : PLF_CONFIG_INFO(
310 : PLF_TASK,
311 : "[AddTaskInfo] streamId=%u, taskId=%u, taskType=%s, "
312 : "remoteRank=%u, src=%p, dst=%p, size=%llu, notifyId=%llu, "
313 : "dataType=%d, reduceOp=%d",
314 : streamId, taskId, taskParam.taskType.Describe().c_str(), remoteRankId, taskParam.taskPara.Reduce.src,
315 : taskParam.taskPara.Reduce.dst, static_cast<u64>(taskParam.taskPara.Reduce.size),
316 : taskParam.taskPara.Reduce.notifyID, taskParam.taskPara.Reduce.dataType,
317 : taskParam.taskPara.Reduce.reduceOp);
318 1 : break;
319 1 : case TaskParamType::TASK_NOTIFY_RECORD:
320 : case TaskParamType::TASK_NOTIFY_WAIT:
321 : case TaskParamType::TASK_SEND_NOTIFY:
322 : case TaskParamType::TASK_DPU_NOTIFY_WAIT:
323 : case TaskParamType::TASK_DPU_CHANNEL_FENCE:
324 3 : PLF_CONFIG_INFO(
325 : PLF_TASK,
326 : "[AddTaskInfo] streamId=%u, taskId=%u, taskType=%s, "
327 : "remoteRank=%u, notifyId=%llu",
328 : streamId, taskId, taskParam.taskType.Describe().c_str(), remoteRankId,
329 : taskParam.taskPara.Notify.notifyID);
330 1 : break;
331 0 : default:
332 0 : PLF_CONFIG_INFO(
333 : PLF_TASK, "[AddTaskInfo] streamId=%u, taskId=%u, taskType=%s", streamId, taskId,
334 : taskParam.taskType.Describe().c_str());
335 0 : break;
336 : }
337 3 : }
338 :
339 : } // namespace Hccl
|
