Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "hcclCommTaskExceptionLite.h"
12 : #include "stream_lite.h"
13 : #include "hcomm_task_scheduler_error.h"
14 : #include "task_struct_v2.h"
15 : #include "dlhal_function_v2.h"
16 : #include <shared_mutex>
17 : #include "aicpu_indop_env.h"
18 : #include "kernel_entrance.h"
19 : #include "ub_transport_lite_impl.h"
20 : #include "res_TE.h"
21 :
22 : namespace hcomm {
23 : constexpr u32 RT_SDMA_COMPERR = 0x9; // A3 sdma error类型为0x9时，表示写拷贝发生超时代答，或者数据搬移时地址译码错误
24 : constexpr u32 RT_SDMA_COMPDATAERR = 0xa; // A3 sdma error类型为0xa时，表示读拷贝发生超时代答，或者读HBM返回ERROR
25 : constexpr u32 RT_SDMA_DATAERR = 0x8; // A3 sdma error类型为0x8时，表示读HBM返回ERROR
26 : constexpr u32 RT_UB_LOCAL_OPERATIOINERR = 0x2; // A5 ub error类型为0x2时，表示UB本端返回ERROR
27 : constexpr u32 RT_UB_REMOTE_OPERATIOINERR = 0x3; // A5 ub error类型为0x3时，表示UB远端返回ERROR
28 : constexpr u32 RT_UB_LINK_FAILEDERR = 0x5; // A5 ub error类型为0x5时，表示网络异常，taack超时
29 : constexpr uint8_t ubSqeType = 9; // A5 sqeType为9表示UBDMA任务
30 : constexpr uint8_t sdmaSqeType = 11; // A5 sqeType为11表示SDMA任务
31 :
32 : constexpr uint32_t TASK_CONTEXT_SIZE = 50; // task 执行失败时打印前序task信息的数量
33 : constexpr uint32_t TASK_CONTEXT_INFO_SIZE
34 : = LOG_TMPBUF_SIZE - TASK_CONTEXT_SIZE; // task 执行失败时打印前序task信息的长度限制
35 : constexpr u32 MAX_NAME_LEN = 64;
36 : constexpr u32 TASK_ID_SHIFT_BITS = 16;
37 :
38 113 : HcclCommTaskExceptionLite& HcclCommTaskExceptionLite::GetInstance()
39 : {
40 113 : static HcclCommTaskExceptionLite instance; // aicpu侧一个dev一个进程，不需要按dev区分单例对象
41 113 : return instance;
42 : }
43 :
44 59 : void HcclCommTaskExceptionLite::Init(u32 devId)
45 : {
46 59 : devId_ = devId;
47 59 : HCCL_INFO("[%s]success, devId_[%u]", __func__, devId_);
48 59 : }
49 :
50 1 : void HcclCommTaskExceptionLite::Call()
51 : {
52 1 : if (stopCall_ == true) {
53 0 : return;
54 : }
55 :
56 1 : HcclResult ret = HandleExceptionCqe();
57 1 : if (ret != HCCL_SUCCESS) {
58 0 : stopCall_ = true;
59 0 : HCCL_ERROR(
60 : "[%s]HandleExceptionCqe fail, set stopCall_[%d]", __func__, stopCall_); // 函数调用失败，停止调用避免刷屏
61 : }
62 : }
63 :
64 6 : HcclResult HcclCommTaskExceptionLite::IsHandleDpuStop(uint8_t* taskexceptionVa, bool& isStop)
65 : {
66 6 : uint8_t stopSignal = 0;
67 6 : errno_t ret = memcpy_s(
68 : &stopSignal, sizeof(stopSignal), taskexceptionVa,
69 : sizeof(stopSignal)); // 读标志位,第1字节，存放host侧发送是否停止的信号。
70 6 : if (ret != EOK) {
71 0 : HCCL_ERROR("[HcclCommTaskExceptionLite::%s] memcpy_s failed on flag, return[%d].", __func__, ret);
72 0 : return HCCL_E_MEMORY;
73 : }
74 6 : if (stopSignal == 1) {
75 1 : isStop = true;
76 1 : stopSignal = 0;
77 1 : ret = memcpy_s(
78 : taskexceptionVa, sizeof(stopSignal), &stopSignal,
79 : sizeof(stopSignal)); // 读标志位,第1字节，存放host侧发送是否停止的信号。
80 1 : if (ret != EOK) {
81 0 : HCCL_ERROR("[HcclCommTaskExceptionLite::%s] memcpy_s failed on flag, return[%d].", __func__, ret);
82 0 : return HCCL_E_MEMORY;
83 : }
84 : }
85 6 : return HCCL_SUCCESS;
86 : }
87 :
88 8 : HcclResult HcclCommTaskExceptionLite::HandleDpuTaskexception(CollCommAicpu* aicpuComm)
89 : {
90 : // 轮询taskexception共享内存
91 8 : auto commId = aicpuComm->GetIdentifier();
92 8 : std::lock_guard<std::mutex> lock(g_taskExpDevMemMapMutex);
93 8 : auto it = g_taskExpDevMemMap.find(commId);
94 8 : if (it == g_taskExpDevMemMap.end()) {
95 1 : return HCCL_SUCCESS; // 非dpu场景，map为空
96 : }
97 :
98 7 : auto taskexceptionVa = reinterpret_cast<uint8_t*>(it->second);
99 7 : if (taskexceptionVa == nullptr) {
100 1 : return HCCL_SUCCESS;
101 : }
102 : // 查是否要停止
103 6 : bool isStop = false;
104 6 : CHK_RET(IsHandleDpuStop(taskexceptionVa, isStop));
105 6 : if (isStop) {
106 1 : it->second = nullptr;
107 1 : return HCCL_SUCCESS;
108 : }
109 : // 查是否有错误
110 5 : uint16_t flag = 0;
111 10 : errno_t ret = memcpy_s(
112 5 : &flag, sizeof(flag), taskexceptionVa + sizeof(uint8_t), sizeof(flag)); // 读标志位,第2-3字节，存放HcclResult。
113 5 : if (ret != EOK) {
114 0 : HCCL_ERROR("[HcclCommTaskExceptionLite::%s] memcpy_s failed on flag, return[%d].", __func__, ret);
115 0 : return HCCL_E_MEMORY;
116 : }
117 5 : if (flag != 0) {
118 : // 触发taskexception
119 4 : HCCL_ERROR(
120 : "[HcclCommTaskExceptionLite][DPU] taskexceptionVa[%p], errorCode[%d], devId[%u], commId[%s]",
121 : taskexceptionVa, flag, aicpuComm->GetDevId(), commId.c_str());
122 : // 1、取notify，并构造rtLogicCqReport_t
123 4 : auto* hcclCommDfxLite = aicpuComm->GetHcclCommDfxLite();
124 7 : CHK_PTR_NULL(hcclCommDfxLite);
125 4 : const auto curDfxOpInfo = static_cast<const Hccl::DfxDfxOpInfo*>(hcclCommDfxLite->GetLatestDfxOpInfo());
126 4 : CHK_PTR_NULL(curDfxOpInfo);
127 1 : u32 notifyId = curDfxOpInfo->cpuWaitAicpuNotifyId;
128 1 : rtLogicCqReport_t exceptionInfo{};
129 : // 2、调用SendTaskExceptionByMBox触发taskexception回调
130 1 : CHK_RET(SendTaskExceptionByMBox(notifyId, 0, exceptionInfo));
131 : // 3、标志位置0
132 1 : ret = memset_s(taskexceptionVa + sizeof(uint8_t), sizeof(uint16_t), 0, sizeof(uint16_t)); // 标志位置0
133 1 : if (ret != EOK) {
134 0 : HCCL_ERROR("[HcclCommTaskExceptionLite::%s] memset_s failed on flag, return[%d].", __func__, ret);
135 0 : return HCCL_E_MEMORY;
136 : }
137 : }
138 2 : return HCCL_SUCCESS;
139 8 : }
140 :
141 1 : HcclResult HcclCommTaskExceptionLite::HandleExceptionCqe()
142 : {
143 1 : std::shared_lock<std::shared_mutex> rwlock(CollCommAicpuMgr::GetInstance().GetMutex());
144 :
145 1 : std::vector<std::pair<std::string, CollCommAicpu*>> aicpuCommInfo;
146 1 : CHK_RET(CollCommAicpuMgr::GetInstance().GetAllComms(aicpuCommInfo));
147 :
148 2 : for (auto& commInfo : aicpuCommInfo) {
149 1 : CollCommAicpu* aicpuComm = commInfo.second;
150 1 : CHK_PTR_NULL(aicpuComm);
151 :
152 1 : if ((aicpuComm->GetCommmStatus() == HcclCommStatus::HCCL_COMM_STATUS_INVALID)
153 1 : || (aicpuComm->GetCommmStatus() == HcclCommStatus::HCCL_COMM_STATUS_SUSPENDING)) {
154 1 : continue;
155 : }
156 0 : CHK_RET(HandleDpuTaskexception(aicpuComm)); // dpu taskexception
157 :
158 0 : std::shared_lock<std::shared_mutex> threadRwlock(aicpuComm->GetCommEngineResMgr()->GetThreadMutex());
159 0 : const std::vector<std::shared_ptr<hccl::Thread>> threads = aicpuComm->GetCommEngineResMgr()->GetAllThread();
160 0 : for (auto thread : threads) {
161 : rtLogicCqReport_t cqeException;
162 0 : dfx::CqeStatus cqeStatus = dfx::CqeStatus::kDefault;
163 0 : Hccl::StreamLite* streamLite = static_cast<Hccl::StreamLite*>(thread->GetStreamLitePtr());
164 0 : CHK_PTR_NULL(streamLite);
165 :
166 0 : HcclResult ret = GetThreadCqe(thread.get(), cqeException, cqeStatus);
167 0 : CHK_PRT_RET(
168 : ret != HCCL_SUCCESS,
169 : HCCL_ERROR(
170 : "[%s]GetThreadCqe fail, aicpuComm[%s], streamId[%u]", __func__, aicpuComm->GetIdentifier().c_str(),
171 : streamLite->GetId()),
172 : ret);
173 :
174 0 : ret = ProcessCqe(aicpuComm, cqeException, cqeStatus, aicpuCommInfo);
175 0 : CHK_PRT_RET(
176 : ret != HCCL_SUCCESS,
177 : HCCL_ERROR(
178 : "[%s]ProcessCqe fail, aicpuComm[%s], streamId[%u], "
179 : "cqeStatus[%lld]",
180 : __func__, aicpuComm->GetIdentifier().c_str(), streamLite->GetId(),
181 : static_cast<long long>(cqeStatus)),
182 : ret);
183 0 : }
184 0 : }
185 1 : return HCCL_SUCCESS;
186 1 : }
187 :
188 1 : HcclResult HcclCommTaskExceptionLite::PrintAllCommTaskException()
189 : {
190 1 : std::shared_lock<std::shared_mutex> rwlock(CollCommAicpuMgr::GetInstance().GetMutex());
191 :
192 1 : std::vector<std::pair<std::string, CollCommAicpu*>> aicpuCommInfo;
193 1 : CHK_RET(CollCommAicpuMgr::GetInstance().GetAllComms(aicpuCommInfo));
194 :
195 1 : HCCL_RUN_INFO("[TaskException][AICPU]%s start, comm size[%u]", __func__, aicpuCommInfo.size());
196 1 : HcclResult ret = HCCL_SUCCESS;
197 2 : for (auto& commInfo : aicpuCommInfo) {
198 1 : CollCommAicpu* aicpuComm = commInfo.second;
199 1 : HcclResult pRet = PrintCommTaskException(aicpuComm);
200 1 : CHK_PRT_CONT(
201 : pRet != HCCL_SUCCESS,
202 : HCCL_ERROR("PrintCommTaskException fail, comm[%s]", aicpuComm->GetIdentifier().c_str()));
203 1 : ret = (pRet != HCCL_SUCCESS) ? pRet : ret;
204 : }
205 1 : HCCL_RUN_INFO("[TaskException][AICPU]%s end, ret[%d]", __func__, ret);
206 1 : return ret;
207 1 : }
208 :
209 2 : HcclResult HcclCommTaskExceptionLite::PrintCommTaskException(CollCommAicpu* aicpuComm)
210 : {
211 2 : CHK_PTR_NULL(aicpuComm);
212 2 : HcclResult ret = HCCL_SUCCESS;
213 2 : HCCL_RUN_INFO("[TaskException][AICPU]%s comm[%s] start", __func__, aicpuComm->GetIdentifier().c_str());
214 2 : std::shared_lock<std::shared_mutex> threadRwlock(aicpuComm->GetCommEngineResMgr()->GetThreadMutex());
215 2 : const std::vector<std::shared_ptr<hccl::Thread>> threads = aicpuComm->GetCommEngineResMgr()->GetAllThread();
216 3 : for (auto thread : threads) {
217 1 : CHK_SMART_PTR_NULL(thread);
218 1 : Hccl::StreamLite* streamLite = static_cast<Hccl::StreamLite*>(thread->GetStreamLitePtr());
219 1 : CHK_PTR_NULL(streamLite);
220 1 : u32 sqHead = 0U;
221 1 : u32 sqTail = 0U;
222 1 : ret = QuerySqStatus(devId_, streamLite->GetSqId(), sqHead, sqTail);
223 1 : if (ret != HCCL_SUCCESS || sqHead == sqTail) { // 此流为空时，不打印
224 0 : HCCL_RUN_INFO(
225 : "[TaskException][AICPU]PrintTaskExceptionBySqeId skip, "
226 : "QuerySqStatus ret[%d], aicpuComm[%s], sqId[%u], sqHead[%u], sqTail[%u]",
227 : ret, aicpuComm->GetIdentifier().c_str(), streamLite->GetSqId(), sqHead, sqTail);
228 0 : continue;
229 : }
230 1 : uint16_t streamId = 0;
231 1 : uint16_t taskId = 0;
232 1 : streamLite->GetRtsq()->GetStreamIdAndTaskIdBySqIdx(sqHead, streamId, taskId);
233 1 : const u32 sqeId = GetSqeId(taskId, streamId);
234 1 : HcclResult pRet = PrintTaskExceptionBySqeId(aicpuComm, streamLite->GetSqId(), sqeId);
235 1 : CHK_PRT_CONT(
236 : pRet != HCCL_SUCCESS, HCCL_ERROR(
237 : "PrintTaskExceptionBySqeId fail, comm[%s], sqId[%u], sqeId[%u]",
238 : aicpuComm->GetIdentifier().c_str(), streamLite->GetSqId(), sqeId));
239 1 : ret = (pRet != HCCL_SUCCESS) ? pRet : ret;
240 1 : }
241 2 : HCCL_RUN_INFO("[TaskException][AICPU]%s comm[%s] end, ret[%d]", __func__, aicpuComm->GetIdentifier().c_str(), ret);
242 2 : return ret;
243 2 : }
244 :
245 0 : HcclResult HcclCommTaskExceptionLite::GetThreadCqe(
246 : hccl::Thread* thread, rtLogicCqReport_t& cqeException, dfx::CqeStatus& cqeStatus)
247 : {
248 0 : CHK_SMART_PTR_NULL(thread);
249 0 : Hccl::StreamLite* streamLite = static_cast<Hccl::StreamLite*>(thread->GetStreamLitePtr());
250 0 : CHK_PTR_NULL(streamLite);
251 :
252 0 : constexpr u32 reportSize = MAX_REPORT_CNT;
253 : rtLogicCqReport_t streamReport[reportSize];
254 :
255 : CqeQueryInput cqeQueryInput;
256 0 : cqeQueryInput.devId = devId_;
257 0 : cqeQueryInput.streamId = streamLite->GetId();
258 0 : cqeQueryInput.sqId = streamLite->GetSqId();
259 0 : cqeQueryInput.cqId = streamLite->GetCqId();
260 0 : cqeQueryInput.type = static_cast<uint32_t>(DRV_LOGIC_TYPE);
261 0 : cqeQueryInput.cqeAddr = reinterpret_cast<uint8_t*>(streamReport);
262 :
263 0 : cqeStatus = CqReportRecv(cqeQueryInput, cqeException);
264 0 : if (cqeStatus == dfx::CqeStatus::kCqeInnerError) {
265 0 : HCCL_ERROR("[%s]CqReportRecv fail, CqeQueryInput:%s", __func__, cqeQueryInput.ToString().c_str());
266 0 : return HCCL_E_INTERNAL;
267 : }
268 0 : return HCCL_SUCCESS;
269 : }
270 :
271 1 : HcclResult HcclCommTaskExceptionLite::ProcessCqe(
272 : CollCommAicpu* aicpuComm, const rtLogicCqReport_t& exceptionInfo, const CqeStatus& cqeStatus,
273 : const std::vector<std::pair<std::string, CollCommAicpu*>>& aicpuCommInfo)
274 : {
275 1 : if (cqeStatus == dfx::CqeStatus::kDefault) {
276 1 : return HCCL_SUCCESS;
277 : }
278 :
279 0 : if (hcomm::GetTaskExceptionEnable() == false) {
280 0 : HCCL_ERROR("[TaskException][AICPU]taskException enable is false, skip print taskException");
281 0 : return HCCL_SUCCESS;
282 : }
283 :
284 0 : HcclResult ret = HCCL_SUCCESS;
285 0 : const u32 sqeId = GetSqeId(exceptionInfo.taskId, exceptionInfo.streamId);
286 0 : ret = PrintTaskExceptionBySqeId(aicpuComm, exceptionInfo.sqId, sqeId);
287 0 : CHK_PRT_CONT(
288 : ret != HCCL_SUCCESS, HCCL_ERROR(
289 : "[PrintTaskExceptionBySqeId]fail, ret[%d], group[%s], sqId[%u], taskId[%u]", ret,
290 : aicpuComm->GetIdentifier().c_str(), exceptionInfo.sqId,
291 : exceptionInfo.taskId)); // 如果上报失败，继续打印taskException
292 :
293 0 : ret = ReportErrMsg(aicpuComm, exceptionInfo);
294 0 : CHK_PRT_CONT(
295 : ret != HCCL_SUCCESS,
296 : HCCL_ERROR(
297 : "[ReportErrMsg]fail, ret[%d], group[%s], sqId[%u], taskId[%u]", ret, aicpuComm->GetIdentifier().c_str(),
298 : exceptionInfo.sqId, exceptionInfo.taskId)); // 如果上报失败，继续打印taskException
299 :
300 : // notify超时场景：step1 打印当前流信息；step2 打印当前通信域信息；step3 打印其他通信域信息
301 0 : if (cqeStatus == dfx::CqeStatus::kCqeException && exceptionInfo.sqeType == RT_STARS_SQE_TYPE_PLACE_HOLDER) {
302 0 : CHK_RET(PrintCommTaskException(aicpuComm));
303 0 : for (auto& commInfo : aicpuCommInfo) {
304 0 : CollCommAicpu* comm = commInfo.second;
305 0 : if (comm != nullptr && comm->GetIdentifier() != aicpuComm->GetIdentifier()) {
306 0 : CHK_RET(PrintCommTaskException(comm));
307 : }
308 : }
309 : }
310 0 : return ret;
311 : }
312 :
313 4 : u32 HcclCommTaskExceptionLite::GetSqeId(uint16_t taskId, uint16_t streamId)
314 : {
315 4 : return (static_cast<u32>(taskId) << TASK_ID_SHIFT_BITS) | static_cast<u32>(streamId);
316 : }
317 :
318 4 : HcclResult HcclCommTaskExceptionLite::ReportErrMsg(CollCommAicpu* aicpuComm, const rtLogicCqReport_t& exceptionInfo)
319 : {
320 4 : CHK_PTR_NULL(aicpuComm);
321 :
322 3 : const u32 sqeId = GetSqeId(exceptionInfo.taskId, exceptionInfo.streamId);
323 3 : HCCL_INFO(
324 : "[%s]group[%s], sqeId[0x%x], taskId[%u], streamId[%u].", __func__, aicpuComm->GetIdentifier().c_str(), sqeId,
325 : exceptionInfo.taskId, exceptionInfo.streamId);
326 :
327 3 : Hccl::DfxTaskInfo* curTask = FindDfxTaskInfo(aicpuComm, exceptionInfo.sqId, sqeId);
328 3 : CHK_PTR_NULL(curTask);
329 :
330 2 : const Hccl::DfxDfxOpInfo* opInfo = (curTask->dfxOpInfo != DFX_INVALID_U64) ?
331 1 : reinterpret_cast<const Hccl::DfxDfxOpInfo*>(curTask->dfxOpInfo) :
332 : nullptr;
333 2 : CHK_PTR_NULL(opInfo);
334 :
335 1 : if (!aicpuComm->IsErrorReported()) {
336 0 : Hccl::ErrorMessageReport errMsgInfo{};
337 0 : CHK_RET(GenerateErrorMessageReport(aicpuComm, *curTask, exceptionInfo, errMsgInfo));
338 0 : CHK_RET(aicpuComm->SendErrorMessageReportToHost(errMsgInfo));
339 :
340 0 : u32 notifyId = opInfo->cpuWaitAicpuNotifyId;
341 0 : CHK_RET(SendTaskExceptionByMBox(notifyId, 0, exceptionInfo));
342 0 : aicpuComm->SetErrorReported(true);
343 : }
344 1 : return HCCL_SUCCESS;
345 : }
346 :
347 1 : HcclResult HcclCommTaskExceptionLite::PrintTaskExceptionBySqeId(CollCommAicpu* aicpuComm, u32 sqId, u32 sqeId)
348 : {
349 1 : CHK_PTR_NULL(aicpuComm);
350 :
351 : // 已经打印过的不再重复打印
352 1 : Hccl::DfxTaskInfo* curTask = FindDfxTaskInfo(aicpuComm, sqId, sqeId);
353 1 : CHK_PTR_NULL(curTask);
354 :
355 1 : auto it = threadsPrinted_.find(sqId);
356 1 : if (it != threadsPrinted_.end() && it->second == sqeId) {
357 0 : HCCL_RUN_INFO("[TaskException][AICPU]sqId:%u, sqeId:%u has been printed, skip", sqId, sqeId);
358 0 : return HCCL_SUCCESS;
359 : }
360 1 : threadsPrinted_[sqId] = sqeId;
361 :
362 1 : u32 sqHead = 0U;
363 1 : u32 sqTail = 0U;
364 1 : (void)QuerySqStatus(devId_, sqId, sqHead, sqTail);
365 :
366 1 : HCCL_ERROR(
367 : "[TaskException][AICPU]base information is streamID(sqId):[%u], taskID(sqeId):[%u], taskType:[%u], "
368 : "sqHead:%u, sqTail:%u",
369 : curTask->sqId, curTask->taskId, curTask->taskType, sqHead, sqTail);
370 :
371 1 : PrintEid(*curTask);
372 :
373 1 : HCCL_ERROR("[TaskException][AICPU]group information is %s.", GetGroupInfo(aicpuComm).c_str());
374 :
375 1 : if (curTask->taskType != static_cast<u8>(Hccl::TaskParamTypeVal::TASK_NOTIFY_WAIT)) {
376 1 : PrintOpDataInfo(curTask);
377 : } else {
378 0 : CHK_RET(PrintTaskContextInfo(aicpuComm, sqId, sqeId));
379 : }
380 1 : return HCCL_SUCCESS;
381 : }
382 :
383 2 : HcclResult HcclCommTaskExceptionLite::GenerateErrorMessageReport(
384 : CollCommAicpu* aicpuComm, const Hccl::DfxTaskInfo& taskInfo, const rtLogicCqReport_t& exceptionInfo,
385 : Hccl::ErrorMessageReport& errMsgInfo)
386 : {
387 2 : const Hccl::DfxDfxOpInfo* opInfo = (taskInfo.dfxOpInfo != DFX_INVALID_U64) ?
388 1 : reinterpret_cast<const Hccl::DfxDfxOpInfo*>(taskInfo.dfxOpInfo) :
389 : nullptr;
390 2 : CHK_PTR_NULL(opInfo);
391 :
392 1 : errMsgInfo.remoteUserRank = GetRemoteRankId(taskInfo);
393 1 : errMsgInfo.streamId = taskInfo.sqId;
394 1 : errMsgInfo.taskId = taskInfo.taskId;
395 1 : errMsgInfo.rankId = aicpuComm->GetTopoInfo().userRank;
396 1 : errMsgInfo.rankSize = aicpuComm->GetTopoInfo().userRankSize;
397 :
398 1 : errMsgInfo.opIndex = opInfo->opIndex;
399 1 : errMsgInfo.opType = opInfo->opType;
400 1 : errMsgInfo.count = opInfo->count;
401 1 : errMsgInfo.dataType = opInfo->dataType;
402 1 : errMsgInfo.srcAddr = opInfo->srcAddr;
403 1 : errMsgInfo.dstAddr = opInfo->dstAddr;
404 1 : CHK_SAFETY_FUNC_RET(memcpy_s(
405 : errMsgInfo.tag, sizeof(errMsgInfo.tag), opInfo->algTag, strnlen(opInfo->algTag, sizeof(opInfo->algTag))));
406 :
407 1 : errMsgInfo.taskType = Hccl::TaskParamType(static_cast<Hccl::TaskParamType::Value>(taskInfo.taskType));
408 1 : errMsgInfo.rtCqErrorType = exceptionInfo.errorType;
409 1 : errMsgInfo.rtCqErrorCode = exceptionInfo.errorCode;
410 :
411 1 : errMsgInfo.jettyHandle = taskInfo.taskPara.ubDma.jettyHandle;
412 1 : errMsgInfo.jettyId = taskInfo.taskPara.ubDma.jettyId;
413 :
414 1 : CHK_SAFETY_FUNC_RET(memcpy_s(
415 : errMsgInfo.group, sizeof(errMsgInfo.group), aicpuComm->GetIdentifier().c_str(),
416 : aicpuComm->GetIdentifier().size() + 1));
417 :
418 1 : GenerateTaskErrMsg(taskInfo, errMsgInfo, exceptionInfo);
419 1 : return HCCL_SUCCESS;
420 : }
421 :
422 10 : void HcclCommTaskExceptionLite::GenerateTaskErrMsg(
423 : const Hccl::DfxTaskInfo& taskInfo, Hccl::ErrorMessageReport& errMsgInfo, const rtLogicCqReport_t& exceptionInfo)
424 : {
425 10 : auto taskType = static_cast<Hccl::TaskParamTypeVal>(taskInfo.taskType);
426 10 : switch (taskType) {
427 2 : case Hccl::TaskParamTypeVal::TASK_NOTIFY_WAIT:
428 : case Hccl::TaskParamTypeVal::TASK_NOTIFY_RECORD:
429 2 : FillNotifyErrMsg(taskInfo, errMsgInfo);
430 2 : break;
431 2 : case Hccl::TaskParamTypeVal::TASK_UB_REDUCE_INLINE:
432 : case Hccl::TaskParamTypeVal::TASK_WRITE_REDUCE_WITH_NOTIFY:
433 2 : FillReduceErrMsg(taskInfo, errMsgInfo, exceptionInfo);
434 2 : break;
435 1 : case Hccl::TaskParamTypeVal::TASK_REDUCE_INLINE:
436 1 : FillReduceInlineErrMsg(taskInfo, errMsgInfo);
437 1 : break;
438 2 : case Hccl::TaskParamTypeVal::TASK_UB_INLINE_WRITE:
439 : case Hccl::TaskParamTypeVal::TASK_WRITE_WITH_NOTIFY:
440 2 : FillDmaErrMsg(taskInfo, errMsgInfo, exceptionInfo);
441 2 : break;
442 2 : case Hccl::TaskParamTypeVal::TASK_UB:
443 2 : FillUbErrMsg(taskInfo, errMsgInfo, exceptionInfo);
444 2 : break;
445 1 : case Hccl::TaskParamTypeVal::TASK_SDMA:
446 1 : FillSdmaErrMsg(taskInfo, errMsgInfo);
447 1 : break;
448 0 : default:
449 0 : HCCL_ERROR("[TaskException][AICPU]%s taskType[%d] is not support", __func__, taskInfo.taskType);
450 0 : return;
451 : }
452 : }
453 :
454 2 : void HcclCommTaskExceptionLite::FillNotifyErrMsg(
455 : const Hccl::DfxTaskInfo& taskInfo, Hccl::ErrorMessageReport& errMsgInfo)
456 : {
457 2 : void* sqePtr = reinterpret_cast<void*>(taskInfo.taskPara.Notify.sqeAddr);
458 2 : if (sqePtr != nullptr) {
459 2 : auto* header = reinterpret_cast<Hccl::Rt91095StarsSqeHeader*>(sqePtr);
460 2 : if (static_cast<Hccl::Rt91095StarsSqeType>(header->type)
461 2 : == Hccl::Rt91095StarsSqeType::RT_91095_SQE_TYPE_NOTIFY_RECORD
462 1 : || static_cast<Hccl::Rt91095StarsSqeType>(header->type)
463 1 : == Hccl::Rt91095StarsSqeType::RT_91095_SQE_TYPE_NOTIFY_WAIT) {
464 2 : auto* notifySqe = reinterpret_cast<Hccl::Rt91095StarsNotifySqe*>(sqePtr);
465 2 : errMsgInfo.notifyId = notifySqe->notifyId;
466 2 : errMsgInfo.notifyValue = notifySqe->cntValue;
467 : }
468 : }
469 2 : }
470 :
471 2 : void HcclCommTaskExceptionLite::FillReduceErrMsg(
472 : const Hccl::DfxTaskInfo& taskInfo, Hccl::ErrorMessageReport& errMsgInfo, const rtLogicCqReport_t& exceptionInfo)
473 : {
474 2 : errMsgInfo.reduceType = taskInfo.taskPara.Reduce.reduceOp;
475 2 : errMsgInfo.notifyId = taskInfo.taskPara.Reduce.notifyId;
476 2 : errMsgInfo.notifyValue = INVALID_U32;
477 2 : GetEidFromChannelHandle(taskInfo, errMsgInfo.locEid, errMsgInfo.rmtEid);
478 2 : errMsgInfo.ubCqeStatus = exceptionInfo.errorCode & 0xFF;
479 2 : errMsgInfo.linkType = Hccl::DfxLinkType(static_cast<Hccl::DfxLinkType::Value>(taskInfo.linkType));
480 2 : errMsgInfo.size = taskInfo.taskPara.Reduce.size;
481 2 : errMsgInfo.taskSrcAddr = taskInfo.taskPara.Reduce.srcAddr;
482 2 : errMsgInfo.taskDstAddr = taskInfo.taskPara.Reduce.dstAddr;
483 2 : HCCL_ERROR(
484 : "[TaskException][AICPU]ubCqeStatus[%u], localEid[%s], remoteEid[%s]. ", errMsgInfo.ubCqeStatus,
485 : errMsgInfo.locEid.Describe().c_str(), errMsgInfo.rmtEid.Describe().c_str());
486 2 : }
487 :
488 2 : void HcclCommTaskExceptionLite::FillDmaErrMsg(
489 : const Hccl::DfxTaskInfo& taskInfo, Hccl::ErrorMessageReport& errMsgInfo, const rtLogicCqReport_t& exceptionInfo)
490 : {
491 2 : errMsgInfo.notifyId = taskInfo.taskPara.ubDma.notifyId;
492 2 : errMsgInfo.notifyValue = INVALID_U32;
493 2 : FillUbErrMsg(taskInfo, errMsgInfo, exceptionInfo);
494 2 : }
495 :
496 1 : void HcclCommTaskExceptionLite::FillSdmaErrMsg(const Hccl::DfxTaskInfo& taskInfo, Hccl::ErrorMessageReport& errMsgInfo)
497 : {
498 1 : errMsgInfo.linkType = Hccl::DfxLinkType(static_cast<Hccl::DfxLinkType::Value>(taskInfo.linkType));
499 1 : void* sqePtr = reinterpret_cast<void*>(taskInfo.taskPara.Dma.sqeAddr);
500 1 : if (sqePtr != nullptr) {
501 0 : auto* header = reinterpret_cast<Hccl::Rt91095StarsSqeHeader*>(sqePtr);
502 0 : if (static_cast<Hccl::Rt91095StarsSqeType>(header->type) == Hccl::Rt91095StarsSqeType::RT_91095_SQE_TYPE_SDMA) {
503 0 : auto* dmaSqe = reinterpret_cast<Hccl::Rt91095StarsMemcpySqe*>(sqePtr);
504 : errMsgInfo.taskSrcAddr
505 0 : = (static_cast<u64>(dmaSqe->u.strideMode0.srcAddrHigh) << 32) | dmaSqe->u.strideMode0.srcAddrLow;
506 : errMsgInfo.taskDstAddr
507 0 : = (static_cast<u64>(dmaSqe->u.strideMode0.dstAddrHigh) << 32) | dmaSqe->u.strideMode0.dstAddrLow;
508 0 : errMsgInfo.size = dmaSqe->u.strideMode0.lengthMove;
509 : }
510 : }
511 1 : }
512 :
513 4 : void HcclCommTaskExceptionLite::FillUbErrMsg(
514 : const Hccl::DfxTaskInfo& taskInfo, Hccl::ErrorMessageReport& errMsgInfo, const rtLogicCqReport_t& exceptionInfo)
515 : {
516 4 : GetEidFromChannelHandle(taskInfo, errMsgInfo.locEid, errMsgInfo.rmtEid);
517 4 : errMsgInfo.ubCqeStatus = exceptionInfo.errorCode & 0xFF;
518 4 : errMsgInfo.linkType = Hccl::DfxLinkType(static_cast<Hccl::DfxLinkType::Value>(taskInfo.linkType));
519 4 : errMsgInfo.size = taskInfo.taskPara.ubDma.size;
520 4 : errMsgInfo.taskSrcAddr = taskInfo.taskPara.ubDma.srcAddr;
521 4 : errMsgInfo.taskDstAddr = taskInfo.taskPara.ubDma.dstAddr;
522 4 : HCCL_ERROR(
523 : "[TaskException][AICPU]ubCqeStatus[%u], localEid[%s], remoteEid[%s]. ", errMsgInfo.ubCqeStatus,
524 : errMsgInfo.locEid.Describe().c_str(), errMsgInfo.rmtEid.Describe().c_str());
525 4 : }
526 :
527 1 : void HcclCommTaskExceptionLite::FillReduceInlineErrMsg(
528 : const Hccl::DfxTaskInfo& taskInfo, Hccl::ErrorMessageReport& errMsgInfo)
529 : {
530 1 : errMsgInfo.reduceType = taskInfo.taskPara.Reduce.reduceOp;
531 1 : }
532 :
533 4 : HcclResult HcclCommTaskExceptionLite::SendTaskExceptionByMBox(
534 : const u32 notifyId, const u32 tsId, const rtLogicCqReport_t& exceptionInfo)
535 : {
536 4 : ts_aicpu_msg_info_t aicpuSqe = {};
537 4 : u32 hostpid = 0;
538 4 : u32 vfId = 0;
539 4 : int pid = getpid();
540 4 : HCCL_INFO("[%s]getpid[%d]", __func__, pid);
541 : // 调整drvQueryProcessHostPid获取pid和vf_id的值
542 4 : CHK_RET(HrtHalDrvQueryProcessHostPid(pid, nullptr, &vfId, &hostpid, nullptr));
543 :
544 4 : aicpuSqe.pid = hostpid;
545 4 : aicpuSqe.cmd_type = TS_AICPU_RECORD;
546 4 : aicpuSqe.vf_id = vfId;
547 4 : aicpuSqe.tid = 0U; // notify is no need tid
548 4 : aicpuSqe.u.aicpu_record.record_type = AICPU_MSG_NOTIFY_RECORD_V2;
549 4 : aicpuSqe.u.aicpu_record.record_id = notifyId;
550 4 : aicpuSqe.ts_id = static_cast<uint8_t>(tsId);
551 4 : aicpuSqe.u.aicpu_record.fault_task_id = 0xffffffff;
552 :
553 4 : if (exceptionInfo.sqeType == ubSqeType) {
554 1 : aicpuSqe.u.aicpu_record.ret_code = SwitchUBCqeErrCodeToTsErrCode(exceptionInfo.errorCode & 0xFF);
555 3 : } else if (exceptionInfo.sqeType == sdmaSqeType) {
556 1 : aicpuSqe.u.aicpu_record.ret_code = SwitchSdmaCqeErrCodeToTsErrCode(exceptionInfo.errorCode);
557 : } else {
558 2 : aicpuSqe.u.aicpu_record.ret_code = TS_ERROR_HCCL_OTHER_ERROR;
559 : }
560 :
561 : struct event_summary event;
562 4 : event.dst_engine = TS_CPU;
563 4 : event.policy = ONLY;
564 4 : event.pid = 0;
565 4 : event.grp_id = 0;
566 4 : event.event_id = EVENT_TS_CTRL_MSG;
567 4 : event.subevent_id = 0U;
568 4 : event.msg_len = static_cast<uint32_t>(sizeof(ts_aicpu_msg_info_t));
569 4 : event.msg = reinterpret_cast<char_t*>(&aicpuSqe);
570 4 : drvError_t ret = Hccl::DlHalFunctionV2::GetInstance().dlHalEschedSubmitEvent(devId_, &event);
571 4 : if (ret != DRV_ERROR_NONE) {
572 0 : HCCL_ERROR(
573 : "[%s]dlHalEschedSubmitEvent failed, ret=%d, notifyId=%u, hostpid=%u, vfId=%u, tsId=%u", __func__, ret,
574 : notifyId, hostpid, vfId, tsId);
575 0 : return HCCL_E_DRV;
576 : }
577 4 : HCCL_RUN_INFO(
578 : "[%s]finished, notifyId=%u, hostpid=%u, vfId=%u, tsId=%u, errorType=%u, errorCode=%u, ret_code=%u", __func__,
579 : notifyId, hostpid, vfId, tsId, exceptionInfo.errorType, exceptionInfo.errorCode,
580 : aicpuSqe.u.aicpu_record.ret_code);
581 4 : return HCCL_SUCCESS;
582 : }
583 :
584 : // 把UB类错误码转换成Ts对应的错误码
585 5 : uint16_t HcclCommTaskExceptionLite::SwitchUBCqeErrCodeToTsErrCode(u32 cqeErrCode)
586 : {
587 5 : switch (cqeErrCode) {
588 2 : case RT_UB_LOCAL_OPERATIOINERR:
589 2 : return TS_ERROR_HCCL_OP_UB_DDRC_FAILED;
590 1 : case RT_UB_REMOTE_OPERATIOINERR:
591 1 : return TS_ERROR_HCCL_OP_UB_POISON_FAILED;
592 1 : case RT_UB_LINK_FAILEDERR:
593 1 : return TS_ERROR_HCCL_OP_UB_LINK_FAILED;
594 1 : default:
595 1 : return TS_ERROR_HCCL_OTHER_ERROR;
596 : }
597 : }
598 :
599 : // 把SDMA类错误码转换成Ts对应的错误码
600 5 : uint16_t HcclCommTaskExceptionLite::SwitchSdmaCqeErrCodeToTsErrCode(u32 cqeErrCode)
601 : {
602 5 : switch (cqeErrCode) {
603 2 : case RT_SDMA_COMPERR:
604 2 : return TS_ERROR_SDMA_LINK_ERROR;
605 1 : case RT_SDMA_COMPDATAERR:
606 1 : return TS_ERROR_SDMA_POISON_ERROR;
607 1 : case RT_SDMA_DATAERR:
608 1 : return TS_ERROR_SDMA_DDRC_ERROR;
609 1 : default:
610 1 : return TS_ERROR_HCCL_OTHER_ERROR;
611 : }
612 : }
613 :
614 2 : HcclResult HcclCommTaskExceptionLite::CollectTaskContext(
615 : CollCommAicpu* aicpuComm, u32 sqId, u32 taskId, std::vector<Hccl::DfxTaskInfo*>& taskContext)
616 : {
617 2 : Hccl::TaskInfoCircularQueue* queue = GetTaskQueueBySqId(aicpuComm, sqId);
618 2 : CHK_PRT_RET(
619 : queue == nullptr, HCCL_ERROR("[%s]GetTaskQueueBySqId nullptr, devId[%u], sqId[%u].", __func__, devId_, sqId),
620 : HCCL_E_PARA);
621 :
622 0 : if (queue->IsEmpty()) {
623 0 : HCCL_ERROR("[%s]queue is empty, devId[%u], sqId[%u].", __func__, devId_, sqId);
624 0 : return HCCL_E_PARA;
625 : }
626 :
627 0 : u32 targetTaskId = taskId;
628 0 : u16 begin = queue->GetBegin();
629 0 : Hccl::DfxTaskInfo* found = nullptr;
630 0 : u16 foundIdx = 0;
631 0 : for (u16 idx = 0; idx < queue->GetCapacity(); idx++) {
632 0 : Hccl::DfxTaskInfo* slot = queue->GetSlot(idx);
633 0 : if (slot != nullptr && slot->taskId == targetTaskId) {
634 0 : found = slot;
635 0 : foundIdx = idx;
636 0 : break;
637 : }
638 : }
639 0 : CHK_PRT_RET(
640 : found == nullptr,
641 : HCCL_ERROR("[%s]exception task not found, devId[%u], sqId[%u], taskId[%u]", __func__, devId_, sqId, taskId),
642 : HCCL_E_PARA);
643 :
644 0 : u32 ctxCount = 0;
645 0 : for (u16 idx = foundIdx; ctxCount < TASK_CONTEXT_SIZE; ++ctxCount) {
646 0 : if (idx == begin) {
647 0 : break;
648 : }
649 0 : idx = (idx == 0) ? static_cast<u16>(queue->GetCapacity() - 1) : idx - 1;
650 0 : Hccl::DfxTaskInfo* slot = queue->GetSlot(idx);
651 0 : if (slot == nullptr || slot->taskId > targetTaskId) {
652 : break;
653 : }
654 0 : taskContext.push_back(slot);
655 : }
656 0 : return HCCL_SUCCESS;
657 : }
658 :
659 1 : HcclResult HcclCommTaskExceptionLite::PrintTaskContextInfo(CollCommAicpu* aicpuComm, u32 sqId, u32 taskId)
660 : {
661 1 : std::vector<Hccl::DfxTaskInfo*> taskContext{};
662 1 : CHK_PRT_RET(
663 : CollectTaskContext(aicpuComm, sqId, taskId, taskContext) != HCCL_SUCCESS,
664 : HCCL_ERROR("[%s]CollectTaskContext failed, devId[%u], sqId[%u], taskId[%u]", __func__, devId_, sqId, taskId),
665 : HCCL_E_PARA);
666 :
667 0 : std::string taskContextInfo = "";
668 0 : Hccl::DfxTaskInfo* lastTask = nullptr;
669 0 : for (u32 i = 0; i < taskContext.size(); ++i) {
670 0 : if (taskContext[i] == nullptr) {
671 0 : continue;
672 : }
673 0 : if (lastTask == nullptr) {
674 0 : lastTask = taskContext[i];
675 : }
676 0 : std::string conciseInfo = GetConciseTaskName(*taskContext[i]) + ",";
677 0 : u32 lastOpIndex = GetOpIndex(lastTask);
678 0 : u32 curOpIndex = GetOpIndex(taskContext[i]);
679 0 : bool overSize = (taskContextInfo.size() + conciseInfo.size()) >= TASK_CONTEXT_INFO_SIZE;
680 0 : if (overSize || (lastOpIndex != curOpIndex)) {
681 0 : PrintOpDataInfo(lastTask);
682 0 : HCCL_ERROR("[TaskException][AICPU]task sequence is OP(%u): %s", lastOpIndex, taskContextInfo.c_str());
683 0 : taskContextInfo = "";
684 0 : lastTask = taskContext[i];
685 : }
686 0 : taskContextInfo += conciseInfo;
687 0 : }
688 :
689 0 : if (!taskContextInfo.empty() && lastTask != nullptr) {
690 0 : u32 lastOpIndex = GetOpIndex(lastTask);
691 0 : PrintOpDataInfo(lastTask);
692 0 : HCCL_ERROR("[TaskException][AICPU]task sequence is OP(%u): %s", lastOpIndex, taskContextInfo.c_str());
693 : }
694 0 : HCCL_ERROR("[TaskException][AICPU]task sequence end.");
695 0 : return HCCL_SUCCESS;
696 1 : }
697 :
698 3 : std::string HcclCommTaskExceptionLite::GetGroupInfo(CollCommAicpu* aicpuComm)
699 : {
700 3 : if (aicpuComm == nullptr) {
701 1 : HCCL_ERROR("[%s]aicpuComm is nullptr, return empty string.", __func__);
702 2 : return "";
703 : }
704 : return Hccl::StringFormat(
705 2 : "group:[%s], rankSize:[%u], localRank:[%u]", aicpuComm->GetIdentifier().c_str(),
706 2 : aicpuComm->GetTopoInfo().userRankSize, aicpuComm->GetTopoInfo().userRank);
707 : }
708 :
709 3 : void HcclCommTaskExceptionLite::PrintEid(const Hccl::DfxTaskInfo& taskInfo)
710 : {
711 3 : auto taskType = static_cast<Hccl::TaskParamTypeVal>(taskInfo.taskType);
712 3 : if (taskType == Hccl::TaskParamTypeVal::TASK_UB_REDUCE_INLINE
713 3 : || taskType == Hccl::TaskParamTypeVal::TASK_WRITE_REDUCE_WITH_NOTIFY
714 3 : || taskType == Hccl::TaskParamTypeVal::TASK_UB_INLINE_WRITE
715 3 : || taskType == Hccl::TaskParamTypeVal::TASK_WRITE_WITH_NOTIFY || taskType == Hccl::TaskParamTypeVal::TASK_UB) {
716 1 : Hccl::Eid locEid;
717 1 : Hccl::Eid rmtEid;
718 1 : GetEidFromChannelHandle(taskInfo, locEid, rmtEid);
719 1 : HCCL_ERROR(
720 : "[TaskException][AICPU][%s]Error UB link info: localEid[%s], remoteEid[%s].", __func__,
721 : locEid.Describe().c_str(), rmtEid.Describe().c_str());
722 : }
723 3 : }
724 :
725 4 : Hccl::DfxTaskInfo* HcclCommTaskExceptionLite::FindDfxTaskInfo(CollCommAicpu* aicpuComm, u32 sqId, u32 sqeId)
726 : {
727 4 : Hccl::TaskInfoCircularQueue* queue = GetTaskQueueBySqId(aicpuComm, sqId);
728 4 : if (queue == nullptr || queue->IsEmpty()) {
729 1 : HCCL_ERROR("[%s]GetTaskQueueBySqId nullptr or queue is empty, devId[%u], sqId[%u].", __func__, devId_, sqId);
730 1 : return nullptr;
731 : }
732 3 : u32 targetTaskId = sqeId;
733 3 : for (u16 idx = 0; idx < queue->GetCapacity(); idx++) {
734 3 : Hccl::DfxTaskInfo* slot = queue->GetSlot(idx);
735 3 : if (slot != nullptr && slot->taskId == targetTaskId) {
736 3 : return slot;
737 : }
738 : }
739 0 : HCCL_ERROR("[%s]exception task not found, devId[%u], sqId[%u], sqeId[%u]", __func__, devId_, sqId, sqeId);
740 0 : return nullptr;
741 : }
742 :
743 6 : Hccl::TaskInfoCircularQueue* HcclCommTaskExceptionLite::GetTaskQueueBySqId(CollCommAicpu* aicpuComm, u32 sqId)
744 : {
745 6 : std::shared_lock<std::shared_mutex> threadRwlock(aicpuComm->GetCommEngineResMgr()->GetThreadMutex());
746 6 : const std::vector<std::shared_ptr<hccl::Thread>> threads = aicpuComm->GetCommEngineResMgr()->GetAllThread();
747 6 : for (auto& thread : threads) {
748 3 : Hccl::StreamLite* streamLite = static_cast<Hccl::StreamLite*>(thread->GetStreamLitePtr());
749 3 : if (streamLite != nullptr && streamLite->GetSqId() == sqId) {
750 3 : return streamLite->GetTaskInfos();
751 : }
752 : }
753 3 : return nullptr;
754 6 : }
755 :
756 7 : void HcclCommTaskExceptionLite::GetEidFromChannelHandle(
757 : const Hccl::DfxTaskInfo& taskInfo, Hccl::Eid& locEid, Hccl::Eid& rmtEid)
758 : {
759 7 : if (taskInfo.channelHandle != DFX_INVALID_U64) {
760 0 : auto* transport = reinterpret_cast<Hccl::UbTransportLiteImpl*>(taskInfo.channelHandle);
761 0 : locEid = transport->GetLocEid();
762 0 : rmtEid = transport->GetRmtEid();
763 : }
764 7 : }
765 :
766 5 : u32 HcclCommTaskExceptionLite::GetRemoteRankId(const Hccl::DfxTaskInfo& taskInfo)
767 : {
768 5 : if (taskInfo.dfxOpInfo != DFX_INVALID_U64) {
769 2 : auto* opInfo = reinterpret_cast<const Hccl::DfxDfxOpInfo*>(taskInfo.dfxOpInfo);
770 2 : if (opInfo->hcclCommDfxLite != nullptr) {
771 0 : return static_cast<hccl::HcclCommDfxLite*>(opInfo->hcclCommDfxLite)
772 0 : ->GetChannelRemoteRankId(taskInfo.channelHandle);
773 : }
774 : }
775 5 : return Hccl::DFX_INVALID_RANKID;
776 : }
777 :
778 1 : void HcclCommTaskExceptionLite::GetNotifyIdFromSqe(u64 sqeAddr, u32& notifyId)
779 : {
780 1 : void* sqePtr = reinterpret_cast<void*>(sqeAddr);
781 1 : if (sqePtr != nullptr) {
782 0 : auto* header = reinterpret_cast<Hccl::Rt91095StarsSqeHeader*>(sqePtr);
783 0 : if (static_cast<Hccl::Rt91095StarsSqeType>(header->type)
784 0 : == Hccl::Rt91095StarsSqeType::RT_91095_SQE_TYPE_NOTIFY_RECORD
785 0 : || static_cast<Hccl::Rt91095StarsSqeType>(header->type)
786 0 : == Hccl::Rt91095StarsSqeType::RT_91095_SQE_TYPE_NOTIFY_WAIT) {
787 0 : auto* notifySqe = reinterpret_cast<Hccl::Rt91095StarsNotifySqe*>(sqePtr);
788 0 : notifyId = notifySqe->notifyId;
789 : }
790 : }
791 1 : }
792 :
793 4 : std::string HcclCommTaskExceptionLite::GetNotifyInfo(const Hccl::DfxTaskInfo& taskInfo)
794 : {
795 4 : auto taskType = static_cast<Hccl::TaskParamTypeVal>(taskInfo.taskType);
796 4 : u32 notifyId = INVALID_U32;
797 4 : switch (taskType) {
798 1 : case Hccl::TaskParamTypeVal::TASK_UB_INLINE_WRITE:
799 : case Hccl::TaskParamTypeVal::TASK_WRITE_WITH_NOTIFY:
800 1 : notifyId = taskInfo.taskPara.ubDma.notifyId;
801 1 : break;
802 1 : case Hccl::TaskParamTypeVal::TASK_WRITE_REDUCE_WITH_NOTIFY:
803 1 : notifyId = taskInfo.taskPara.Reduce.notifyId;
804 1 : break;
805 1 : case Hccl::TaskParamTypeVal::TASK_NOTIFY_RECORD:
806 : case Hccl::TaskParamTypeVal::TASK_NOTIFY_WAIT:
807 : case Hccl::TaskParamTypeVal::TASK_SEND_NOTIFY: {
808 1 : GetNotifyIdFromSqe(taskInfo.taskPara.Notify.sqeAddr, notifyId);
809 1 : break;
810 : }
811 0 : case Hccl::TaskParamTypeVal::TASK_RDMA: {
812 0 : GetNotifyIdFromSqe(taskInfo.taskPara.Dma.sqeAddr, notifyId);
813 0 : break;
814 : }
815 1 : default:
816 2 : return "/";
817 : }
818 5 : return (notifyId == INVALID_U32) ? "/" : std::to_string(notifyId);
819 : }
820 :
821 2 : std::string HcclCommTaskExceptionLite::GetConciseTaskName(const Hccl::DfxTaskInfo& taskInfo)
822 : {
823 2 : const auto& taskConciseNameMap = Hccl::GetTaskConciseNameMap();
824 2 : auto it = taskConciseNameMap.find(taskInfo.taskType);
825 2 : std::string name = (it != taskConciseNameMap.end()) ? it->second : "UNKNOWN";
826 2 : u32 remoteRank = GetRemoteRankId(taskInfo);
827 4 : std::string rankStr = (remoteRank == Hccl::DFX_INVALID_RANKID) ? "/" : std::to_string(remoteRank);
828 2 : auto taskType = static_cast<Hccl::TaskParamTypeVal>(taskInfo.taskType);
829 2 : if (taskType == Hccl::TaskParamTypeVal::TASK_RDMA || taskType == Hccl::TaskParamTypeVal::TASK_NOTIFY_RECORD
830 2 : || taskType == Hccl::TaskParamTypeVal::TASK_NOTIFY_WAIT || taskType == Hccl::TaskParamTypeVal::TASK_SEND_NOTIFY
831 1 : || taskType == Hccl::TaskParamTypeVal::TASK_WRITE_WITH_NOTIFY
832 1 : || taskType == Hccl::TaskParamTypeVal::TASK_WRITE_REDUCE_WITH_NOTIFY
833 1 : || taskType == Hccl::TaskParamTypeVal::TASK_UB_INLINE_WRITE) {
834 1 : return name + "(" + rankStr + "," + GetNotifyInfo(taskInfo) + ")";
835 : }
836 1 : return name + "(" + rankStr + ")";
837 2 : }
838 :
839 0 : u32 HcclCommTaskExceptionLite::GetOpIndex(const Hccl::DfxTaskInfo* taskInfo)
840 : {
841 0 : if (taskInfo == nullptr || taskInfo->dfxOpInfo == DFX_INVALID_U64) {
842 0 : return UINT32_MAX;
843 : }
844 0 : return reinterpret_cast<const Hccl::DfxDfxOpInfo*>(taskInfo->dfxOpInfo)->opIndex;
845 : }
846 :
847 1 : void HcclCommTaskExceptionLite::PrintOpDataInfo(const Hccl::DfxTaskInfo* taskInfo)
848 : {
849 1 : if (taskInfo == nullptr || taskInfo->dfxOpInfo == DFX_INVALID_U64) {
850 1 : HCCL_ERROR("[TaskException][AICPU]opData information is (dfxOpInfo unavailable).");
851 1 : return;
852 : }
853 0 : const Hccl::DfxDfxOpInfo* opInfo = reinterpret_cast<const Hccl::DfxDfxOpInfo*>(taskInfo->dfxOpInfo);
854 0 : HCCL_ERROR(
855 : "[TaskException][AICPU]opData information is opIndex[%u], algTag[%s], count[%llu], "
856 : "dataType[%u], input: ptr[0x%llx] size[%llu], output: ptr[0x%llx] size[%llu].",
857 : opInfo->opIndex, opInfo->algTag, opInfo->count, opInfo->dataType, opInfo->srcAddr, opInfo->srcSize,
858 : opInfo->dstAddr, opInfo->dstSize);
859 : }
860 : } // namespace hcomm
|
