Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 : #include "hcclCommTaskExceptionLite.h"
11 : #include "stream_lite.h"
12 : #include "hcomm_task_scheduler_error.h"
13 : #include "task_struct_v2.h"
14 : #include "dlhal_function_v2.h"
15 : #include <shared_mutex>
16 : #include "aicpu_indop_env.h"
17 : #include "kernel_entrance.h"
18 :
19 : namespace hcomm {
20 : constexpr u32 RT_SDMA_COMPERR = 0x9; // A3 sdma error类型为0x9时，表示写拷贝发生超时代答，或者数据搬移时地址译码错误
21 : constexpr u32 RT_SDMA_COMPDATAERR = 0xa; // A3 sdma error类型为0xa时，表示读拷贝发生超时代答，或者读HBM返回ERROR
22 : constexpr u32 RT_SDMA_DATAERR = 0x8; // A3 sdma error类型为0x8时，表示读HBM返回ERROR
23 : constexpr u32 RT_UB_LOCAL_OPERATIOINERR = 0x2; // A5 ub error类型为0x2时，表示UB本端返回ERROR
24 : constexpr u32 RT_UB_REMOTE_OPERATIOINERR = 0x3; // A5 ub error类型为0x3时，表示UB远端返回ERROR
25 : constexpr u32 RT_UB_LINK_FAILEDERR = 0x5; // A5 ub error类型为0x5时，表示网络异常，taack超时
26 : constexpr uint8_t ubSqeType = 9; // A5 sqeType为9表示UBDMA任务
27 : constexpr uint8_t sdmaSqeType = 11; // A5 sqeType为11表示SDMA任务
28 :
29 : constexpr uint32_t TASK_CONTEXT_SIZE = 50; // task 执行失败时打印前序task信息的数量
30 : constexpr uint32_t TASK_CONTEXT_INFO_SIZE = LOG_TMPBUF_SIZE - TASK_CONTEXT_SIZE; // task 执行失败时打印前序task信息的长度限制
31 : constexpr u32 MAX_NAME_LEN = 64;
32 : constexpr u32 TASK_ID_SHIFT_BITS = 16;
33 :
34 71 : HcclCommTaskExceptionLite &HcclCommTaskExceptionLite::GetInstance()
35 : {
36 71 : static HcclCommTaskExceptionLite instance; // aicpu侧一个dev一个进程，不需要按dev区分单例对象
37 71 : return instance;
38 : }
39 :
40 37 : void HcclCommTaskExceptionLite::Init(u32 devId)
41 : {
42 37 : devId_ = devId;
43 37 : HCCL_INFO("[%s]success, devId_[%u]", __func__, devId_);
44 37 : }
45 :
46 1 : void HcclCommTaskExceptionLite::Call()
47 : {
48 1 : if (stopCall_ == true) {
49 0 : return;
50 : }
51 :
52 1 : HcclResult ret = HandleExceptionCqe();
53 1 : if (ret != HCCL_SUCCESS) {
54 0 : stopCall_ = true;
55 0 : HCCL_ERROR("[%s]HandleExceptionCqe fail, set stopCall_[%d]", __func__, stopCall_); // 函数调用失败，停止调用避免刷屏
56 : }
57 : }
58 :
59 6 : HcclResult HcclCommTaskExceptionLite::IsHandleDpuStop(uint8_t *taskexceptionVa, bool &isStop)
60 : {
61 6 : uint8_t stopSignal = 0;
62 6 : errno_t ret = memcpy_s(&stopSignal, sizeof(stopSignal), taskexceptionVa, sizeof(stopSignal)); // 读标志位,第1字节，存放host侧发送是否停止的信号。
63 6 : if (ret != EOK) {
64 0 : HCCL_ERROR("[HcclCommTaskExceptionLite::%s] memcpy_s failed on flag, return[%d].", __func__, ret);
65 0 : return HCCL_E_MEMORY;
66 : }
67 6 : if (stopSignal == 1) {
68 1 : isStop = true;
69 1 : stopSignal = 0;
70 1 : ret = memcpy_s(taskexceptionVa, sizeof(stopSignal), &stopSignal, sizeof(stopSignal)); // 读标志位,第1字节，存放host侧发送是否停止的信号。
71 1 : if (ret != EOK) {
72 0 : HCCL_ERROR("[HcclCommTaskExceptionLite::%s] memcpy_s failed on flag, return[%d].", __func__, ret);
73 0 : return HCCL_E_MEMORY;
74 : }
75 : }
76 6 : return HCCL_SUCCESS;
77 : }
78 :
79 8 : HcclResult HcclCommTaskExceptionLite::HandleDpuTaskexception(CollCommAicpu *aicpuComm)
80 : {
81 : // 轮询taskexception共享内存
82 8 : auto commId = aicpuComm->GetIdentifier();
83 8 : std::lock_guard<std::mutex> lock(g_taskExpDevMemMapMutex);
84 8 : auto it = g_taskExpDevMemMap.find(commId);
85 8 : if (it == g_taskExpDevMemMap.end()) {
86 1 : return HCCL_SUCCESS; // 非dpu场景，map为空
87 : }
88 :
89 7 : auto taskexceptionVa = reinterpret_cast<uint8_t *>(it->second);
90 7 : if (taskexceptionVa == nullptr) {
91 1 : return HCCL_SUCCESS;
92 : }
93 : // 查是否要停止
94 6 : bool isStop = false;
95 6 : CHK_RET(IsHandleDpuStop(taskexceptionVa, isStop));
96 6 : if (isStop) {
97 1 : it->second = nullptr;
98 1 : return HCCL_SUCCESS;
99 : }
100 : // 查是否有错误
101 5 : uint16_t flag = 0;
102 5 : errno_t ret = memcpy_s(&flag, sizeof(flag), taskexceptionVa + sizeof(uint8_t), sizeof(flag)); // 读标志位,第2-3字节，存放HcclResult。
103 5 : if (ret != EOK) {
104 0 : HCCL_ERROR("[HcclCommTaskExceptionLite::%s] memcpy_s failed on flag, return[%d].", __func__, ret);
105 0 : return HCCL_E_MEMORY;
106 : }
107 5 : if (flag != 0) {
108 : // 触发taskexception
109 4 : HCCL_ERROR("[HcclCommTaskExceptionLite][DPU] taskexceptionVa[%p], errorCode[%d], devId[%u], commId[%s]", taskexceptionVa, flag, aicpuComm->GetDevId(), commId.c_str());
110 : // 1、取notify，并构造rtLogicCqReport_t
111 7 : CHK_PTR_NULL(aicpuComm->GetHcclCommDfxLite());
112 4 : CHK_PTR_NULL(aicpuComm->GetHcclCommDfxLite()->GetMirrorTaskManagerLite());
113 2 : CHK_PTR_NULL(aicpuComm->GetHcclCommDfxLite()->GetMirrorTaskManagerLite()->GetCurrDfxOpInfo());
114 1 : auto curDfxOpInfo = aicpuComm->GetHcclCommDfxLite()->GetMirrorTaskManagerLite()->GetCurrDfxOpInfo();
115 1 : u32 notifyId = curDfxOpInfo->cpuWaitAicpuNotifyId_;
116 1 : rtLogicCqReport_t exceptionInfo{};
117 : // 2、调用SendTaskExceptionByMBox触发taskexception回调
118 1 : CHK_RET(SendTaskExceptionByMBox(notifyId, 0, exceptionInfo));
119 : // 3、标志位置0
120 1 : ret = memset_s(taskexceptionVa + sizeof(uint8_t), sizeof(uint16_t), 0, sizeof(uint16_t)); // 标志位置0
121 1 : if (ret != EOK) {
122 0 : HCCL_ERROR("[HcclCommTaskExceptionLite::%s] memset_s failed on flag, return[%d].", __func__, ret);
123 0 : return HCCL_E_MEMORY;
124 : }
125 1 : }
126 2 : return HCCL_SUCCESS;
127 8 : }
128 :
129 1 : HcclResult HcclCommTaskExceptionLite::HandleExceptionCqe()
130 : {
131 1 : std::shared_lock<std::shared_mutex> rwlock(AicpuIndopProcess::AicpuGetCommMutex());
132 :
133 1 : std::vector<std::pair<std::string, CollCommAicpuMgr *>> aicpuCommInfo;
134 1 : CHK_RET(AicpuIndopProcess::AicpuGetCommAll(aicpuCommInfo));
135 :
136 2 : for (auto &commInfo : aicpuCommInfo) {
137 1 : CollCommAicpu *aicpuComm = commInfo.second->GetCollCommAicpu();
138 1 : CHK_PTR_NULL(aicpuComm);
139 :
140 2 : if ((aicpuComm->GetCommmStatus() == HcclCommStatus::HCCL_COMM_STATUS_INVALID) ||
141 1 : (aicpuComm->GetCommmStatus() == HcclCommStatus::HCCL_COMM_STATUS_SUSPENDING)) {
142 1 : continue;
143 : }
144 0 : CHK_RET(HandleDpuTaskexception(aicpuComm)); // dpu taskexception
145 :
146 0 : std::shared_lock<std::shared_mutex> threadRwlock(aicpuComm->GetThreadMutex());
147 0 : const std::vector<std::shared_ptr<hccl::Thread>> threads = aicpuComm->GetAllThread();
148 0 : for (auto thread : threads) {
149 : rtLogicCqReport_t cqeException;
150 0 : dfx::CqeStatus cqeStatus = dfx::CqeStatus::kDefault;
151 0 : Hccl::StreamLite *streamLite = static_cast<Hccl::StreamLite *>(thread->GetStreamLitePtr());
152 0 : CHK_PTR_NULL(streamLite);
153 :
154 0 : HcclResult ret = GetThreadCqe(thread.get(), cqeException, cqeStatus);
155 0 : CHK_PRT_RET(ret != HCCL_SUCCESS, HCCL_ERROR("[%s]GetThreadCqe fail, aicpuComm[%s], streamId[%u]",
156 : __func__, aicpuComm->GetIdentifier().c_str(), streamLite->GetId()), ret);
157 :
158 0 : ret = ProcessCqe(aicpuComm, cqeException, cqeStatus, aicpuCommInfo);
159 0 : CHK_PRT_RET(ret != HCCL_SUCCESS, HCCL_ERROR("[%s]ProcessCqe fail, aicpuComm[%s], streamId[%u], "
160 : "cqeStatus[%lld]", __func__, aicpuComm->GetIdentifier().c_str(), streamLite->GetId(), static_cast<long long>(cqeStatus)), ret);
161 0 : }
162 0 : }
163 1 : return HCCL_SUCCESS;
164 1 : }
165 :
166 1 : HcclResult HcclCommTaskExceptionLite::PrintAllCommTaskException()
167 : {
168 1 : std::shared_lock<std::shared_mutex> rwlock(AicpuIndopProcess::AicpuGetCommMutex());
169 :
170 1 : std::vector<std::pair<std::string, CollCommAicpuMgr *>> aicpuCommInfo;
171 1 : CHK_RET(AicpuIndopProcess::AicpuGetCommAll(aicpuCommInfo));
172 :
173 1 : HCCL_RUN_INFO("[TaskException][AICPU]%s start, comm size[%u]", __func__, aicpuCommInfo.size());
174 1 : HcclResult ret = HCCL_SUCCESS;
175 2 : for (auto &commInfo : aicpuCommInfo) {
176 1 : CollCommAicpu *aicpuComm = commInfo.second->GetCollCommAicpu();
177 1 : HcclResult pRet = PrintCommTaskException(aicpuComm);
178 1 : CHK_PRT_CONT(pRet != HCCL_SUCCESS, HCCL_ERROR("PrintCommTaskException fail, comm[%s]",
179 : aicpuComm->GetIdentifier().c_str()));
180 1 : ret = (pRet != HCCL_SUCCESS) ? pRet : ret;
181 : }
182 1 : HCCL_RUN_INFO("[TaskException][AICPU]%s end, ret[%d]", __func__, ret);
183 1 : return ret;
184 1 : }
185 :
186 2 : HcclResult HcclCommTaskExceptionLite::PrintCommTaskException(CollCommAicpu *aicpuComm)
187 : {
188 2 : CHK_PTR_NULL(aicpuComm);
189 2 : HcclResult ret = HCCL_SUCCESS;
190 2 : HCCL_RUN_INFO("[TaskException][AICPU]%s comm[%s] start", __func__, aicpuComm->GetIdentifier().c_str());
191 2 : std::shared_lock<std::shared_mutex> threadRwlock(aicpuComm->GetThreadMutex());
192 2 : const std::vector<std::shared_ptr<hccl::Thread>> threads = aicpuComm->GetAllThread();
193 3 : for (auto thread : threads) {
194 1 : CHK_SMART_PTR_NULL(thread);
195 1 : Hccl::StreamLite *streamLite = static_cast<Hccl::StreamLite *>(thread->GetStreamLitePtr());
196 1 : CHK_PTR_NULL(streamLite);
197 1 : u32 sqHead = 0U;
198 1 : u32 sqTail = 0U;
199 1 : HcclResult ret = QuerySqStatus(devId_, streamLite->GetSqId(), sqHead, sqTail);
200 1 : if (ret != HCCL_SUCCESS || sqHead == sqTail) { // 此流为空时，不打印
201 0 : HCCL_RUN_INFO("[TaskException][AICPU]PrintTaskExceptionBySqeId skip, "
202 : "QuerySqStatus ret[%d], aicpuComm[%s], sqId[%u], sqHead[%u], sqTail[%u]",
203 : ret, aicpuComm->GetIdentifier().c_str(), streamLite->GetSqId(), sqHead, sqTail);
204 0 : continue;
205 : }
206 1 : uint16_t streamId = 0;
207 1 : uint16_t taskId = 0;
208 1 : streamLite->GetRtsq()->GetStreamIdAndTaskIdBySqIdx(sqHead, streamId, taskId);
209 1 : const u32 sqeId = GetSqeId(taskId, streamId);
210 1 : HcclResult pRet = PrintTaskExceptionBySqeId(aicpuComm, streamLite->GetSqId(), sqeId);
211 1 : CHK_PRT_CONT(pRet != HCCL_SUCCESS, HCCL_ERROR("PrintTaskExceptionBySqeId fail, comm[%s], sqId[%u], sqeId[%u]",
212 : aicpuComm->GetIdentifier().c_str(), streamLite->GetSqId(), sqeId));
213 1 : ret = (pRet != HCCL_SUCCESS) ? pRet : ret;
214 1 : }
215 2 : HCCL_RUN_INFO("[TaskException][AICPU]%s comm[%s] end, ret[%d]", __func__, aicpuComm->GetIdentifier().c_str(), ret);
216 2 : return ret;
217 2 : }
218 :
219 0 : HcclResult HcclCommTaskExceptionLite::GetThreadCqe(hccl::Thread* thread, rtLogicCqReport_t &cqeException,
220 : dfx::CqeStatus &cqeStatus)
221 : {
222 0 : CHK_SMART_PTR_NULL(thread);
223 0 : Hccl::StreamLite *streamLite = static_cast<Hccl::StreamLite *>(thread->GetStreamLitePtr());
224 0 : CHK_PTR_NULL(streamLite);
225 :
226 0 : constexpr u32 reportSize = MAX_REPORT_CNT;
227 : rtLogicCqReport_t streamReport[reportSize];
228 :
229 : CqeQueryInput cqeQueryInput;
230 0 : cqeQueryInput.devId = devId_;
231 0 : cqeQueryInput.streamId = streamLite->GetId();
232 0 : cqeQueryInput.sqId = streamLite->GetSqId();
233 0 : cqeQueryInput.cqId = streamLite->GetCqId();
234 0 : cqeQueryInput.type = static_cast<uint32_t>(DRV_LOGIC_TYPE);
235 0 : cqeQueryInput.cqeAddr = reinterpret_cast<uint8_t *>(streamReport);
236 :
237 0 : cqeStatus = CqReportRecv(cqeQueryInput, cqeException);
238 0 : if (cqeStatus == dfx::CqeStatus::kCqeInnerError) {
239 0 : HCCL_ERROR("[%s]CqReportRecv fail, CqeQueryInput:%s", __func__, cqeQueryInput.ToString().c_str());
240 0 : return HCCL_E_INTERNAL;
241 : }
242 0 : return HCCL_SUCCESS;
243 : }
244 :
245 1 : HcclResult HcclCommTaskExceptionLite::ProcessCqe(CollCommAicpu *aicpuComm, const rtLogicCqReport_t &exceptionInfo,
246 : const CqeStatus &cqeStatus, const std::vector<std::pair<std::string, CollCommAicpuMgr *>> &aicpuCommInfo)
247 : {
248 1 : if (cqeStatus == dfx::CqeStatus::kDefault) {
249 1 : return HCCL_SUCCESS;
250 : }
251 :
252 0 : if (hcomm::GetTaskExceptionEnable() == false) {
253 0 : HCCL_ERROR("[TaskException][AICPU]taskException enable is false, skip print taskException");
254 0 : return HCCL_SUCCESS;
255 : }
256 :
257 0 : HcclResult ret = HCCL_SUCCESS;
258 0 : const u32 sqeId = GetSqeId(exceptionInfo.taskId, exceptionInfo.streamId);
259 0 : ret = PrintTaskExceptionBySqeId(aicpuComm, exceptionInfo.sqId, sqeId);
260 0 : CHK_PRT_CONT(ret != HCCL_SUCCESS, HCCL_ERROR("[PrintTaskExceptionBySqeId]fail, ret[%d], group[%s], sqId[%u], taskId[%u]",
261 : ret, aicpuComm->GetIdentifier().c_str(), exceptionInfo.sqId, exceptionInfo.taskId)); // 如果上报失败，继续打印taskException
262 :
263 0 : ret = ReportErrMsg(aicpuComm, exceptionInfo);
264 0 : CHK_PRT_CONT(ret != HCCL_SUCCESS, HCCL_ERROR("[ReportErrMsg]fail, ret[%d], group[%s], sqId[%u], taskId[%u]",
265 : ret, aicpuComm->GetIdentifier().c_str(), exceptionInfo.sqId, exceptionInfo.taskId)); // 如果上报失败，继续打印taskException
266 :
267 : // notify超时场景：step1 打印当前流信息；step2 打印当前通信域信息；step3 打印其他通信域信息
268 0 : if (cqeStatus == dfx::CqeStatus::kCqeException && exceptionInfo.sqeType == RT_STARS_SQE_TYPE_PLACE_HOLDER) {
269 0 : CHK_RET(PrintCommTaskException(aicpuComm));
270 0 : for (auto &commInfo : aicpuCommInfo) {
271 0 : CollCommAicpu *comm = commInfo.second->GetCollCommAicpu();
272 0 : if (comm != nullptr && comm->GetIdentifier() != aicpuComm->GetIdentifier()) {
273 0 : CHK_RET(PrintCommTaskException(comm));
274 : }
275 : }
276 : }
277 0 : return ret;
278 : }
279 :
280 1 : u32 HcclCommTaskExceptionLite::GetSqeId(uint16_t taskId, uint16_t streamId)
281 : {
282 1 : return (static_cast<u32>(taskId) << TASK_ID_SHIFT_BITS) | static_cast<u32>(streamId);
283 : }
284 :
285 0 : HcclResult HcclCommTaskExceptionLite::ReportErrMsg(CollCommAicpu *aicpuComm, const rtLogicCqReport_t &exceptionInfo)
286 : {
287 0 : CHK_PTR_NULL(aicpuComm);
288 0 : CHK_PTR_NULL(aicpuComm->GetHcclCommDfxLite());
289 0 : CHK_PTR_NULL(aicpuComm->GetHcclCommDfxLite()->GetMirrorTaskManagerLite());
290 :
291 0 : const u32 sqeId = GetSqeId(exceptionInfo.taskId, exceptionInfo.streamId);
292 0 : HCCL_INFO("[%s]group[%s], sqeId[0x%x], taskId[%u], streamId[%u].",
293 : __func__, aicpuComm->GetIdentifier().c_str(), sqeId, exceptionInfo.taskId, exceptionInfo.streamId);
294 :
295 0 : const auto curTask = aicpuComm->GetHcclCommDfxLite()->GetMirrorTaskManagerLite()->GetTaskInfo(exceptionInfo.sqId, sqeId);
296 0 : CHK_SMART_PTR_NULL(curTask);
297 0 : CHK_SMART_PTR_NULL(curTask->dfxOpInfo_);
298 :
299 0 : if (!aicpuComm->IsErrorReported()) {
300 : // 1) errorMessage上报
301 0 : Hccl::ErrorMessageReport errMsgInfo{};
302 0 : CHK_RET(GenerateErrorMessageReport(aicpuComm, *curTask, exceptionInfo, errMsgInfo));
303 0 : CHK_RET(aicpuComm->SendErrorMessageReportToHost(errMsgInfo));
304 :
305 : // 2) send mbox to tsfw
306 0 : u32 notifyId = curTask->dfxOpInfo_->cpuWaitAicpuNotifyId_;
307 0 : CHK_RET(SendTaskExceptionByMBox(notifyId, 0, exceptionInfo));
308 0 : aicpuComm->SetErrorReported(true);
309 : }
310 0 : return HCCL_SUCCESS;
311 : }
312 :
313 1 : HcclResult HcclCommTaskExceptionLite::PrintTaskExceptionBySqeId(CollCommAicpu *aicpuComm, u32 sqId, u32 sqeId)
314 : {
315 1 : CHK_PTR_NULL(aicpuComm);
316 1 : CHK_PTR_NULL(aicpuComm->GetHcclCommDfxLite());
317 1 : CHK_PTR_NULL(aicpuComm->GetHcclCommDfxLite()->GetMirrorTaskManagerLite());
318 :
319 : // 已经打印过的不再重复打印
320 1 : auto it = threadsPrinted_.find(sqId);
321 1 : if (it != threadsPrinted_.end() && it->second == sqeId) {
322 0 : HCCL_RUN_INFO("[TaskException][AICPU]sqId:%u, sqeId:%u has been printed, skip", sqId, sqeId);
323 0 : return HCCL_SUCCESS;
324 : }
325 1 : threadsPrinted_[sqId] = sqeId;
326 :
327 1 : const auto curTask = aicpuComm->GetHcclCommDfxLite()->GetMirrorTaskManagerLite()->GetTaskInfo(sqId, sqeId);
328 1 : CHK_SMART_PTR_NULL(curTask);
329 1 : CHK_SMART_PTR_NULL(curTask->dfxOpInfo_);
330 :
331 1 : u32 sqHead = 0U;
332 1 : u32 sqTail = 0U;
333 1 : (void)QuerySqStatus(devId_, sqId, sqHead, sqTail);
334 :
335 : // 1. 打印task信息
336 1 : HCCL_ERROR("[TaskException][AICPU]base information is %s, %s, sqHead:%u, sqTail:%u",
337 : curTask->GetIndopBaseInfo().c_str(), curTask->GetParaInfo().c_str(), sqHead, sqTail);
338 : // 2. UB任务打印EID信息
339 1 : PrintEid(*curTask);
340 : // 3. 打印group信息
341 1 : HCCL_ERROR("[TaskException][AICPU]group information is %s.", GetGroupInfo(aicpuComm).c_str());
342 : // 4. 打印算子信息和task序列
343 1 : if (curTask->taskParam_.taskType != Hccl::TaskParamType::TASK_NOTIFY_WAIT) { // 非notify场景，仅打印算子信息
344 1 : HCCL_ERROR("[TaskException][AICPU]opData information is %s.", curTask->GetIndopDataInfo().c_str());
345 : } else {
346 0 : CHK_RET(PrintTaskContextInfo(aicpuComm, sqId, sqeId)); // notify场景打印算子信息和task序列
347 : }
348 1 : return HCCL_SUCCESS;
349 : }
350 :
351 0 : HcclResult HcclCommTaskExceptionLite::GenerateErrorMessageReport(CollCommAicpu *aicpuComm,
352 : const Hccl::TaskInfo& taskInfo, const rtLogicCqReport_t &exceptionInfo, Hccl::ErrorMessageReport &errMsgInfo)
353 : {
354 : // 获取需要上报的关键信息
355 0 : errMsgInfo.remoteUserRank = taskInfo.GetRemoteRankId();
356 0 : errMsgInfo.streamId = taskInfo.streamId_;
357 0 : errMsgInfo.taskId = taskInfo.taskId_;
358 0 : errMsgInfo.rankId = aicpuComm->GetTopoInfo().userRank;
359 0 : errMsgInfo.rankSize = aicpuComm->GetTopoInfo().userRankSize;
360 0 : errMsgInfo.opIndex = taskInfo.dfxOpInfo_->opIndex_;
361 0 : errMsgInfo.opType = taskInfo.dfxOpInfo_->op_.opType;
362 0 : errMsgInfo.count = taskInfo.dfxOpInfo_->op_.dataCount;
363 0 : errMsgInfo.dataType = taskInfo.dfxOpInfo_->op_.dataType;
364 0 : errMsgInfo.srcAddr = taskInfo.dfxOpInfo_->op_.inputAddr;
365 0 : errMsgInfo.dstAddr = taskInfo.dfxOpInfo_->op_.outputAddr;
366 0 : errMsgInfo.taskType = taskInfo.taskParam_.taskType;
367 :
368 0 : errMsgInfo.rtCqErrorType = exceptionInfo.errorType;
369 0 : errMsgInfo.rtCqErrorCode = exceptionInfo.errorCode;
370 :
371 0 : errMsgInfo.jettyHandle = taskInfo.taskParam_.taskPara.DMA.jettyHandle;
372 0 : errMsgInfo.jettyId = taskInfo.taskParam_.taskPara.DMA.jettyId;
373 :
374 0 : CHK_SAFETY_FUNC_RET(memcpy_s(errMsgInfo.tag, sizeof(errMsgInfo.tag),
375 : taskInfo.dfxOpInfo_->algTag_.c_str(), taskInfo.dfxOpInfo_->algTag_.size() + 1));
376 0 : CHK_SAFETY_FUNC_RET(memcpy_s(errMsgInfo.group, sizeof(errMsgInfo.group),
377 : aicpuComm->GetIdentifier().c_str(), aicpuComm->GetIdentifier().size() + 1));
378 :
379 0 : GenerateTaskErrMsg(taskInfo, errMsgInfo, exceptionInfo);
380 0 : return HCCL_SUCCESS;
381 : }
382 :
383 9 : void HcclCommTaskExceptionLite::GenerateTaskErrMsg(const Hccl::TaskInfo& taskInfo, Hccl::ErrorMessageReport &errMsgInfo,
384 : const rtLogicCqReport_t &exceptionInfo)
385 : {
386 9 : switch (taskInfo.taskParam_.taskType) {
387 2 : case Hccl::TaskParamType::TASK_NOTIFY_WAIT:
388 : case Hccl::TaskParamType::TASK_NOTIFY_RECORD:
389 2 : FillNotifyErrMsg(taskInfo, errMsgInfo);
390 2 : break;
391 2 : case Hccl::TaskParamType::TASK_UB_REDUCE_INLINE:
392 : case Hccl::TaskParamType::TASK_WRITE_REDUCE_WITH_NOTIFY:
393 2 : FillReduceErrMsg(taskInfo, errMsgInfo, exceptionInfo);
394 2 : break;
395 1 : case Hccl::TaskParamType::TASK_REDUCE_INLINE:
396 1 : FillReduceInlineErrMsg(taskInfo, errMsgInfo);
397 1 : break;
398 2 : case Hccl::TaskParamType::TASK_UB_INLINE_WRITE:
399 : case Hccl::TaskParamType::TASK_WRITE_WITH_NOTIFY:
400 2 : FillDmaErrMsg(taskInfo, errMsgInfo, exceptionInfo);
401 2 : break;
402 1 : case Hccl::TaskParamType::TASK_UB:
403 1 : FillUbErrMsg(taskInfo, errMsgInfo, exceptionInfo);
404 1 : break;
405 1 : case Hccl::TaskParamType::TASK_SDMA:
406 1 : FillSdmaErrMsg(taskInfo, errMsgInfo);
407 1 : break;
408 0 : default:
409 0 : HCCL_ERROR("[TaskException][AICPU]%s taskType[%d] is not support", __func__, taskInfo.taskParam_.taskType);
410 0 : return;
411 : }
412 : }
413 :
414 2 : void HcclCommTaskExceptionLite::FillNotifyErrMsg(const Hccl::TaskInfo& taskInfo, Hccl::ErrorMessageReport &errMsgInfo)
415 : {
416 2 : errMsgInfo.notifyId = taskInfo.taskParam_.taskPara.Notify.notifyID;
417 2 : errMsgInfo.notifyValue = taskInfo.taskParam_.taskPara.Notify.value;
418 2 : }
419 :
420 2 : void HcclCommTaskExceptionLite::FillReduceErrMsg(const Hccl::TaskInfo& taskInfo, Hccl::ErrorMessageReport &errMsgInfo,
421 : const rtLogicCqReport_t &exceptionInfo)
422 : {
423 2 : errMsgInfo.reduceType = taskInfo.taskParam_.taskPara.Reduce.reduceOp;
424 2 : errMsgInfo.notifyId = taskInfo.taskParam_.taskPara.Reduce.notifyID;
425 2 : errMsgInfo.notifyValue = taskInfo.taskParam_.taskPara.Reduce.notifyValue;
426 2 : errMsgInfo.locEid = taskInfo.taskParam_.taskPara.Reduce.locEid;
427 2 : errMsgInfo.rmtEid = taskInfo.taskParam_.taskPara.Reduce.rmtEid;
428 2 : errMsgInfo.ubCqeStatus = exceptionInfo.errorCode & 0xFF;
429 2 : errMsgInfo.linkType = taskInfo.taskParam_.taskPara.Reduce.linkType;
430 2 : errMsgInfo.size = taskInfo.taskParam_.taskPara.Reduce.size;
431 2 : errMsgInfo.taskSrcAddr = reinterpret_cast<u64>(taskInfo.taskParam_.taskPara.Reduce.src);
432 2 : errMsgInfo.taskDstAddr = reinterpret_cast<u64>(taskInfo.taskParam_.taskPara.Reduce.dst);
433 2 : HCCL_ERROR("[TaskException][AICPU]ubCqeStatus[%u], localEid[%s], remoteEid[%s]. ",
434 : errMsgInfo.ubCqeStatus, errMsgInfo.locEid.Describe().c_str(), errMsgInfo.rmtEid.Describe().c_str());
435 2 : }
436 :
437 2 : void HcclCommTaskExceptionLite::FillDmaErrMsg(const Hccl::TaskInfo& taskInfo, Hccl::ErrorMessageReport &errMsgInfo,
438 : const rtLogicCqReport_t &exceptionInfo)
439 : {
440 2 : errMsgInfo.notifyId = taskInfo.taskParam_.taskPara.DMA.notifyID;
441 2 : errMsgInfo.notifyValue = taskInfo.taskParam_.taskPara.DMA.notifyValue;
442 2 : FillUbErrMsg(taskInfo, errMsgInfo, exceptionInfo);
443 2 : }
444 :
445 1 : void HcclCommTaskExceptionLite::FillSdmaErrMsg(const Hccl::TaskInfo& taskInfo, Hccl::ErrorMessageReport &errMsgInfo)
446 : {
447 1 : errMsgInfo.linkType = taskInfo.taskParam_.taskPara.DMA.linkType;
448 1 : errMsgInfo.size = taskInfo.taskParam_.taskPara.DMA.size;
449 1 : errMsgInfo.taskSrcAddr = reinterpret_cast<u64>(taskInfo.taskParam_.taskPara.DMA.src);
450 1 : errMsgInfo.taskDstAddr = reinterpret_cast<u64>(taskInfo.taskParam_.taskPara.DMA.dst);
451 1 : }
452 :
453 3 : void HcclCommTaskExceptionLite::FillUbErrMsg(const Hccl::TaskInfo& taskInfo, Hccl::ErrorMessageReport &errMsgInfo,
454 : const rtLogicCqReport_t &exceptionInfo)
455 : {
456 3 : errMsgInfo.locEid = taskInfo.taskParam_.taskPara.DMA.locEid;
457 3 : errMsgInfo.rmtEid = taskInfo.taskParam_.taskPara.DMA.rmtEid;
458 3 : errMsgInfo.ubCqeStatus = exceptionInfo.errorCode & 0xFF;
459 3 : errMsgInfo.linkType = taskInfo.taskParam_.taskPara.DMA.linkType;
460 3 : errMsgInfo.size = taskInfo.taskParam_.taskPara.DMA.size;
461 3 : errMsgInfo.taskSrcAddr = reinterpret_cast<u64>(taskInfo.taskParam_.taskPara.DMA.src);
462 3 : errMsgInfo.taskDstAddr = reinterpret_cast<u64>(taskInfo.taskParam_.taskPara.DMA.dst);
463 3 : HCCL_ERROR("[TaskException][AICPU]ubCqeStatus[%u], localEid[%s], remoteEid[%s]. ",
464 : errMsgInfo.ubCqeStatus, errMsgInfo.locEid.Describe().c_str(), errMsgInfo.rmtEid.Describe().c_str());
465 3 : }
466 :
467 1 : void HcclCommTaskExceptionLite::FillReduceInlineErrMsg(const Hccl::TaskInfo& taskInfo,
468 : Hccl::ErrorMessageReport &errMsgInfo)
469 : {
470 1 : errMsgInfo.reduceType = taskInfo.taskParam_.taskPara.Reduce.reduceOp;
471 1 : }
472 :
473 4 : HcclResult HcclCommTaskExceptionLite::SendTaskExceptionByMBox(const u32 notifyId, const u32 tsId,
474 : const rtLogicCqReport_t &exceptionInfo)
475 : {
476 4 : ts_aicpu_msg_info_t aicpuSqe = {};
477 4 : u32 hostpid = 0;
478 4 : u32 vfId = 0;
479 4 : int pid = getpid();
480 4 : HCCL_INFO("[%s]getpid[%d]", __func__, pid);
481 : // 调整drvQueryProcessHostPid获取pid和vf_id的值
482 4 : CHK_RET(HrtHalDrvQueryProcessHostPid(pid, nullptr, &vfId, &hostpid, nullptr));
483 :
484 4 : aicpuSqe.pid = hostpid;
485 4 : aicpuSqe.cmd_type = TS_AICPU_RECORD;
486 4 : aicpuSqe.vf_id = vfId;
487 4 : aicpuSqe.tid = 0U; // notify is no need tid
488 4 : aicpuSqe.u.aicpu_record.record_type = AICPU_MSG_NOTIFY_RECORD_V2;
489 4 : aicpuSqe.u.aicpu_record.record_id = notifyId;
490 4 : aicpuSqe.ts_id = static_cast<uint8_t>(tsId);
491 4 : aicpuSqe.u.aicpu_record.fault_task_id = 0xffffffff;
492 :
493 4 : if (exceptionInfo.sqeType == ubSqeType) {
494 1 : aicpuSqe.u.aicpu_record.ret_code = SwitchUBCqeErrCodeToTsErrCode(exceptionInfo.errorCode & 0xFF);
495 3 : } else if (exceptionInfo.sqeType == sdmaSqeType) {
496 1 : aicpuSqe.u.aicpu_record.ret_code = SwitchSdmaCqeErrCodeToTsErrCode(exceptionInfo.errorCode);
497 : } else {
498 2 : aicpuSqe.u.aicpu_record.ret_code = TS_ERROR_HCCL_OTHER_ERROR;
499 : }
500 :
501 : struct event_summary event;
502 4 : event.dst_engine = TS_CPU;
503 4 : event.policy = ONLY;
504 4 : event.pid = 0;
505 4 : event.grp_id = 0;
506 4 : event.event_id = EVENT_TS_CTRL_MSG;
507 4 : event.subevent_id = 0U;
508 4 : event.msg_len = static_cast<uint32_t>(sizeof(ts_aicpu_msg_info_t));
509 4 : event.msg = reinterpret_cast<char_t *>(&aicpuSqe);
510 4 : drvError_t ret = Hccl::DlHalFunctionV2::GetInstance().dlHalEschedSubmitEvent(devId_, &event);
511 4 : if (ret != DRV_ERROR_NONE) {
512 0 : HCCL_ERROR("[%s]dlHalEschedSubmitEvent failed, ret=%d, notifyId=%u, hostpid=%u, vfId=%u, tsId=%u",
513 : __func__, ret, notifyId, hostpid, vfId, tsId);
514 0 : return HCCL_E_DRV;
515 : }
516 4 : HCCL_RUN_INFO("[%s]finished, notifyId=%u, hostpid=%u, vfId=%u, tsId=%u, errorType=%u, errorCode=%u, ret_code=%u",
517 : __func__, notifyId, hostpid, vfId, tsId, exceptionInfo.errorType, exceptionInfo.errorCode,
518 : aicpuSqe.u.aicpu_record.ret_code);
519 4 : return HCCL_SUCCESS;
520 : }
521 :
522 : // 把UB类错误码转换成Ts对应的错误码
523 5 : uint16_t HcclCommTaskExceptionLite::SwitchUBCqeErrCodeToTsErrCode(u32 cqeErrCode) {
524 5 : switch (cqeErrCode) {
525 2 : case RT_UB_LOCAL_OPERATIOINERR:
526 2 : return TS_ERROR_HCCL_OP_UB_DDRC_FAILED;
527 1 : case RT_UB_REMOTE_OPERATIOINERR:
528 1 : return TS_ERROR_HCCL_OP_UB_POISON_FAILED;
529 1 : case RT_UB_LINK_FAILEDERR:
530 1 : return TS_ERROR_HCCL_OP_UB_LINK_FAILED;
531 1 : default:
532 1 : return TS_ERROR_HCCL_OTHER_ERROR;
533 : }
534 : }
535 :
536 : // 把SDMA类错误码转换成Ts对应的错误码
537 5 : uint16_t HcclCommTaskExceptionLite::SwitchSdmaCqeErrCodeToTsErrCode(u32 cqeErrCode) {
538 5 : switch (cqeErrCode) {
539 2 : case RT_SDMA_COMPERR:
540 2 : return TS_ERROR_SDMA_LINK_ERROR;
541 1 : case RT_SDMA_COMPDATAERR:
542 1 : return TS_ERROR_SDMA_POISON_ERROR;
543 1 : case RT_SDMA_DATAERR:
544 1 : return TS_ERROR_SDMA_DDRC_ERROR;
545 1 : default:
546 1 : return TS_ERROR_HCCL_OTHER_ERROR;
547 : }
548 : }
549 :
550 0 : HcclResult HcclCommTaskExceptionLite::CollectTaskContext(CollCommAicpu *aicpuComm, u32 sqId, u32 taskId,
551 : std::vector<Hccl::TaskInfo*> &taskContext)
552 : {
553 0 : auto queue = aicpuComm->GetHcclCommDfxLite()->GetMirrorTaskManagerLite()->GetQueue(sqId);
554 0 : CHK_PRT_RET(queue == nullptr,
555 : HCCL_ERROR("[%s]GetQueue nullptr, devId[%u], sqId[%u].", __func__, devId_, sqId), HCCL_E_PARA);
556 :
557 0 : auto func = [taskId] (const std::unique_ptr<Hccl::TaskInfo>& task) { return task->taskId_ == taskId; };
558 0 : auto taskIterPtr = queue->Find(func);
559 0 : CHK_PRT_RET(taskIterPtr == nullptr || *taskIterPtr == *queue->End(),
560 : HCCL_ERROR("[%s]exception task not found, devId[%u], sqId[%u], taskId[%u]", __func__, devId_, sqId, taskId),
561 : HCCL_E_PARA);
562 :
563 : // 找到当前异常task的前50个task(至多)
564 0 : for (uint32_t i = 0; i < TASK_CONTEXT_SIZE && !queue->IsEmpty(); ++i, --(*taskIterPtr)) {
565 0 : if ((**taskIterPtr)->taskId_ > taskId) { // 回绕中止
566 0 : HCCL_ERROR("[%s]prev taskId[%u] is bigger than err taskId[%u], taskNum[%u], stop traversal",
567 : __func__, (**taskIterPtr)->taskId_, taskId, taskContext.size());
568 0 : break;
569 : }
570 0 : taskContext.emplace_back((**taskIterPtr).get());
571 :
572 0 : if (*taskIterPtr == *queue->Begin()) { // 遍历到起始位置中止
573 0 : HCCL_ERROR("[%s]taskId[%u] is queue Begin, taskNum[%u], stop traversal",
574 : __func__, (**taskIterPtr)->taskId_, taskContext.size());
575 0 : break;
576 : }
577 : }
578 0 : return HCCL_SUCCESS;
579 0 : }
580 :
581 0 : HcclResult HcclCommTaskExceptionLite::PrintTaskContextInfo(CollCommAicpu *aicpuComm, u32 sqId, u32 taskId)
582 : {
583 0 : std::vector<Hccl::TaskInfo*> taskContext {};
584 0 : CHK_PRT_RET(CollectTaskContext(aicpuComm, sqId, taskId, taskContext) != HCCL_SUCCESS,
585 : HCCL_ERROR("[%s]CollectTaskContext failed, devId[%u], sqId[%u], taskId[%u]", __func__, devId_, sqId, taskId),
586 : HCCL_E_PARA);
587 :
588 0 : std::string taskContextInfo = "";
589 0 : Hccl::TaskInfo* lastTask = nullptr;
590 0 : for (u32 i = 0; i < taskContext.size(); ++i) {
591 0 : if (taskContext[i] == nullptr) {
592 0 : HCCL_ERROR("[%s]taskContext[%u] is nullptr, skip!", __func__, i);
593 0 : continue;
594 : }
595 0 : if (lastTask == nullptr) {
596 0 : lastTask = taskContext[i];
597 : }
598 0 : std::string conciseInfo = taskContext[i]->GetConciseBaseInfo() + ",";
599 :
600 0 : u32 lastOpIndex = ((lastTask->dfxOpInfo_ == nullptr) ? UINT32_MAX : lastTask->dfxOpInfo_->opIndex_);
601 0 : u32 curOpIndex = ((taskContext[i]->dfxOpInfo_ == nullptr) ? UINT32_MAX : taskContext[i]->dfxOpInfo_->opIndex_);
602 0 : bool overSize = (taskContextInfo.size() + conciseInfo.size()) >= TASK_CONTEXT_INFO_SIZE; // 1. 字符串超过一定长度时，打印一次
603 : // 2. 不同算子，新起一行打印
604 0 : if (overSize || (lastOpIndex != curOpIndex)) {
605 0 : HCCL_ERROR("[TaskException][AICPU]opData information is %s.", lastTask->GetIndopDataInfo().c_str());
606 0 : HCCL_ERROR("[TaskException][AICPU]task sequence is OP(%u): %s", lastOpIndex, taskContextInfo.c_str());
607 0 : taskContextInfo = "";
608 0 : lastTask = taskContext[i];
609 : }
610 0 : taskContextInfo += conciseInfo;
611 0 : }
612 :
613 : // 3. 最后一个task，打印一次
614 0 : if (!taskContextInfo.empty() && lastTask != nullptr) {
615 0 : u32 lastOpIndex = (lastTask->dfxOpInfo_ == nullptr) ? UINT32_MAX : lastTask->dfxOpInfo_->opIndex_;
616 0 : HCCL_ERROR("[TaskException][AICPU]opData information is %s.", lastTask->GetIndopDataInfo().c_str());
617 0 : HCCL_ERROR("[TaskException][AICPU]task sequence is OP(%u): %s", lastOpIndex, taskContextInfo.c_str());
618 : }
619 0 : HCCL_ERROR("[TaskException][AICPU]task sequence end.");
620 0 : return HCCL_SUCCESS;
621 0 : }
622 :
623 3 : std::string HcclCommTaskExceptionLite::GetGroupInfo(CollCommAicpu *aicpuComm)
624 : {
625 3 : if (aicpuComm == nullptr) {
626 1 : HCCL_ERROR("[%s]aicpuComm is nullptr, return empty string.", __func__);
627 2 : return "";
628 : }
629 : return Hccl::StringFormat("group:[%s], rankSize:[%u], localRank:[%u]",
630 2 : aicpuComm->GetIdentifier().c_str(), aicpuComm->GetTopoInfo().userRankSize, aicpuComm->GetTopoInfo().userRank);
631 : }
632 :
633 1 : void HcclCommTaskExceptionLite::PrintEid(const Hccl::TaskInfo& taskInfo)
634 : {
635 2 : if (taskInfo.taskParam_.taskType == Hccl::TaskParamType::TASK_UB_REDUCE_INLINE ||
636 1 : taskInfo.taskParam_.taskType == Hccl::TaskParamType::TASK_WRITE_REDUCE_WITH_NOTIFY) {
637 0 : HCCL_ERROR("[TaskException][AICPU][%s]Error UB link info: localEid[%s], remoteEid[%s].", __func__,
638 : taskInfo.taskParam_.taskPara.Reduce.locEid.Describe().c_str(),
639 : taskInfo.taskParam_.taskPara.Reduce.rmtEid.Describe().c_str());
640 2 : } else if (taskInfo.taskParam_.taskType == Hccl::TaskParamType::TASK_UB_INLINE_WRITE ||
641 2 : taskInfo.taskParam_.taskType == Hccl::TaskParamType::TASK_WRITE_WITH_NOTIFY ||
642 1 : taskInfo.taskParam_.taskType == Hccl::TaskParamType::TASK_UB) {
643 0 : HCCL_ERROR("[TaskException][AICPU][%s]Error UB link info: localEid[%s], remoteEid[%s].", __func__,
644 : taskInfo.taskParam_.taskPara.DMA.locEid.Describe().c_str(),
645 : taskInfo.taskParam_.taskPara.DMA.rmtEid.Describe().c_str());
646 : }
647 1 : }
648 : }
|
