Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 : #include "hcclCommTaskExceptionLite.h"
11 : #include "stream_lite.h"
12 : #include "hcomm_task_scheduler_error.h"
13 : #include "task_struct_v2.h"
14 : #include "dlhal_function_v2.h"
15 : #include <shared_mutex>
16 : #include "aicpu_indop_env.h"
17 : #include "kernel_entrance.h"
18 : #include "ub_transport_lite_impl.h"
19 : #include "res_TE.h"
20 :
21 : namespace hcomm {
22 : constexpr u32 RT_SDMA_COMPERR = 0x9; // A3 sdma error类型为0x9时，表示写拷贝发生超时代答，或者数据搬移时地址译码错误
23 : constexpr u32 RT_SDMA_COMPDATAERR = 0xa; // A3 sdma error类型为0xa时，表示读拷贝发生超时代答，或者读HBM返回ERROR
24 : constexpr u32 RT_SDMA_DATAERR = 0x8; // A3 sdma error类型为0x8时，表示读HBM返回ERROR
25 : constexpr u32 RT_UB_LOCAL_OPERATIOINERR = 0x2; // A5 ub error类型为0x2时，表示UB本端返回ERROR
26 : constexpr u32 RT_UB_REMOTE_OPERATIOINERR = 0x3; // A5 ub error类型为0x3时，表示UB远端返回ERROR
27 : constexpr u32 RT_UB_LINK_FAILEDERR = 0x5; // A5 ub error类型为0x5时，表示网络异常，taack超时
28 : constexpr uint8_t ubSqeType = 9; // A5 sqeType为9表示UBDMA任务
29 : constexpr uint8_t sdmaSqeType = 11; // A5 sqeType为11表示SDMA任务
30 :
31 : constexpr uint32_t TASK_CONTEXT_SIZE = 50; // task 执行失败时打印前序task信息的数量
32 : constexpr uint32_t TASK_CONTEXT_INFO_SIZE
33 : = LOG_TMPBUF_SIZE - TASK_CONTEXT_SIZE; // task 执行失败时打印前序task信息的长度限制
34 : constexpr u32 MAX_NAME_LEN = 64;
35 : constexpr u32 TASK_ID_SHIFT_BITS = 16;
36 :
37 109 : HcclCommTaskExceptionLite& HcclCommTaskExceptionLite::GetInstance()
38 : {
39 109 : static HcclCommTaskExceptionLite instance; // aicpu侧一个dev一个进程，不需要按dev区分单例对象
40 109 : return instance;
41 : }
42 :
43 56 : void HcclCommTaskExceptionLite::Init(u32 devId)
44 : {
45 56 : devId_ = devId;
46 56 : HCCL_INFO("[%s]success, devId_[%u]", __func__, devId_);
47 56 : }
48 :
49 1 : void HcclCommTaskExceptionLite::Call()
50 : {
51 1 : if (stopCall_ == true) {
52 0 : return;
53 : }
54 :
55 1 : HcclResult ret = HandleExceptionCqe();
56 1 : if (ret != HCCL_SUCCESS) {
57 0 : stopCall_ = true;
58 0 : HCCL_ERROR(
59 : "[%s]HandleExceptionCqe fail, set stopCall_[%d]", __func__, stopCall_); // 函数调用失败，停止调用避免刷屏
60 : }
61 : }
62 :
63 6 : HcclResult HcclCommTaskExceptionLite::IsHandleDpuStop(uint8_t* taskexceptionVa, bool& isStop)
64 : {
65 6 : uint8_t stopSignal = 0;
66 6 : errno_t ret = memcpy_s(
67 : &stopSignal, sizeof(stopSignal), taskexceptionVa,
68 : sizeof(stopSignal)); // 读标志位,第1字节，存放host侧发送是否停止的信号。
69 6 : if (ret != EOK) {
70 0 : HCCL_ERROR("[HcclCommTaskExceptionLite::%s] memcpy_s failed on flag, return[%d].", __func__, ret);
71 0 : return HCCL_E_MEMORY;
72 : }
73 6 : if (stopSignal == 1) {
74 1 : isStop = true;
75 1 : stopSignal = 0;
76 1 : ret = memcpy_s(
77 : taskexceptionVa, sizeof(stopSignal), &stopSignal,
78 : sizeof(stopSignal)); // 读标志位,第1字节，存放host侧发送是否停止的信号。
79 1 : if (ret != EOK) {
80 0 : HCCL_ERROR("[HcclCommTaskExceptionLite::%s] memcpy_s failed on flag, return[%d].", __func__, ret);
81 0 : return HCCL_E_MEMORY;
82 : }
83 : }
84 6 : return HCCL_SUCCESS;
85 : }
86 :
87 8 : HcclResult HcclCommTaskExceptionLite::HandleDpuTaskexception(CollCommAicpu* aicpuComm)
88 : {
89 : // 轮询taskexception共享内存
90 8 : auto commId = aicpuComm->GetIdentifier();
91 8 : std::lock_guard<std::mutex> lock(g_taskExpDevMemMapMutex);
92 8 : auto it = g_taskExpDevMemMap.find(commId);
93 8 : if (it == g_taskExpDevMemMap.end()) {
94 1 : return HCCL_SUCCESS; // 非dpu场景，map为空
95 : }
96 :
97 7 : auto taskexceptionVa = reinterpret_cast<uint8_t*>(it->second);
98 7 : if (taskexceptionVa == nullptr) {
99 1 : return HCCL_SUCCESS;
100 : }
101 : // 查是否要停止
102 6 : bool isStop = false;
103 6 : CHK_RET(IsHandleDpuStop(taskexceptionVa, isStop));
104 6 : if (isStop) {
105 1 : it->second = nullptr;
106 1 : return HCCL_SUCCESS;
107 : }
108 : // 查是否有错误
109 5 : uint16_t flag = 0;
110 10 : errno_t ret = memcpy_s(
111 5 : &flag, sizeof(flag), taskexceptionVa + sizeof(uint8_t), sizeof(flag)); // 读标志位,第2-3字节，存放HcclResult。
112 5 : if (ret != EOK) {
113 0 : HCCL_ERROR("[HcclCommTaskExceptionLite::%s] memcpy_s failed on flag, return[%d].", __func__, ret);
114 0 : return HCCL_E_MEMORY;
115 : }
116 5 : if (flag != 0) {
117 : // 触发taskexception
118 4 : HCCL_ERROR(
119 : "[HcclCommTaskExceptionLite][DPU] taskexceptionVa[%p], errorCode[%d], devId[%u], commId[%s]",
120 : taskexceptionVa, flag, aicpuComm->GetDevId(), commId.c_str());
121 : // 1、取notify，并构造rtLogicCqReport_t
122 4 : auto* hcclCommDfxLite = aicpuComm->GetHcclCommDfxLite();
123 7 : CHK_PTR_NULL(hcclCommDfxLite);
124 4 : const auto curDfxOpInfo = static_cast<const Hccl::DfxDfxOpInfo*>(hcclCommDfxLite->GetLatestDfxOpInfo());
125 4 : CHK_PTR_NULL(curDfxOpInfo);
126 1 : u32 notifyId = curDfxOpInfo->cpuWaitAicpuNotifyId;
127 1 : rtLogicCqReport_t exceptionInfo{};
128 : // 2、调用SendTaskExceptionByMBox触发taskexception回调
129 1 : CHK_RET(SendTaskExceptionByMBox(notifyId, 0, exceptionInfo));
130 : // 3、标志位置0
131 1 : ret = memset_s(taskexceptionVa + sizeof(uint8_t), sizeof(uint16_t), 0, sizeof(uint16_t)); // 标志位置0
132 1 : if (ret != EOK) {
133 0 : HCCL_ERROR("[HcclCommTaskExceptionLite::%s] memset_s failed on flag, return[%d].", __func__, ret);
134 0 : return HCCL_E_MEMORY;
135 : }
136 : }
137 2 : return HCCL_SUCCESS;
138 8 : }
139 :
140 1 : HcclResult HcclCommTaskExceptionLite::HandleExceptionCqe()
141 : {
142 1 : std::shared_lock<std::shared_mutex> rwlock(CollCommAicpuMgr::GetInstance().GetMutex());
143 :
144 1 : std::vector<std::pair<std::string, CollCommAicpu*>> aicpuCommInfo;
145 1 : CHK_RET(CollCommAicpuMgr::GetInstance().GetAllComms(aicpuCommInfo));
146 :
147 2 : for (auto& commInfo : aicpuCommInfo) {
148 1 : CollCommAicpu* aicpuComm = commInfo.second;
149 1 : CHK_PTR_NULL(aicpuComm);
150 :
151 1 : if ((aicpuComm->GetCommmStatus() == HcclCommStatus::HCCL_COMM_STATUS_INVALID)
152 1 : || (aicpuComm->GetCommmStatus() == HcclCommStatus::HCCL_COMM_STATUS_SUSPENDING)) {
153 1 : continue;
154 : }
155 0 : CHK_RET(HandleDpuTaskexception(aicpuComm)); // dpu taskexception
156 :
157 0 : std::shared_lock<std::shared_mutex> threadRwlock(aicpuComm->GetCommEngineResMgr()->GetThreadMutex());
158 0 : const std::vector<std::shared_ptr<hccl::Thread>> threads = aicpuComm->GetCommEngineResMgr()->GetAllThread();
159 0 : for (auto thread : threads) {
160 : rtLogicCqReport_t cqeException;
161 0 : dfx::CqeStatus cqeStatus = dfx::CqeStatus::kDefault;
162 0 : Hccl::StreamLite* streamLite = static_cast<Hccl::StreamLite*>(thread->GetStreamLitePtr());
163 0 : CHK_PTR_NULL(streamLite);
164 :
165 0 : HcclResult ret = GetThreadCqe(thread.get(), cqeException, cqeStatus);
166 0 : CHK_PRT_RET(
167 : ret != HCCL_SUCCESS,
168 : HCCL_ERROR(
169 : "[%s]GetThreadCqe fail, aicpuComm[%s], streamId[%u]", __func__, aicpuComm->GetIdentifier().c_str(),
170 : streamLite->GetId()),
171 : ret);
172 :
173 0 : ret = ProcessCqe(aicpuComm, cqeException, cqeStatus, aicpuCommInfo);
174 0 : CHK_PRT_RET(
175 : ret != HCCL_SUCCESS,
176 : HCCL_ERROR(
177 : "[%s]ProcessCqe fail, aicpuComm[%s], streamId[%u], "
178 : "cqeStatus[%lld]",
179 : __func__, aicpuComm->GetIdentifier().c_str(), streamLite->GetId(),
180 : static_cast<long long>(cqeStatus)),
181 : ret);
182 0 : }
183 0 : }
184 1 : return HCCL_SUCCESS;
185 1 : }
186 :
187 1 : HcclResult HcclCommTaskExceptionLite::PrintAllCommTaskException()
188 : {
189 1 : std::shared_lock<std::shared_mutex> rwlock(CollCommAicpuMgr::GetInstance().GetMutex());
190 :
191 1 : std::vector<std::pair<std::string, CollCommAicpu*>> aicpuCommInfo;
192 1 : CHK_RET(CollCommAicpuMgr::GetInstance().GetAllComms(aicpuCommInfo));
193 :
194 1 : HCCL_RUN_INFO("[TaskException][AICPU]%s start, comm size[%u]", __func__, aicpuCommInfo.size());
195 1 : HcclResult ret = HCCL_SUCCESS;
196 2 : for (auto& commInfo : aicpuCommInfo) {
197 1 : CollCommAicpu* aicpuComm = commInfo.second;
198 1 : HcclResult pRet = PrintCommTaskException(aicpuComm);
199 1 : CHK_PRT_CONT(
200 : pRet != HCCL_SUCCESS,
201 : HCCL_ERROR("PrintCommTaskException fail, comm[%s]", aicpuComm->GetIdentifier().c_str()));
202 1 : ret = (pRet != HCCL_SUCCESS) ? pRet : ret;
203 : }
204 1 : HCCL_RUN_INFO("[TaskException][AICPU]%s end, ret[%d]", __func__, ret);
205 1 : return ret;
206 1 : }
207 :
208 2 : HcclResult HcclCommTaskExceptionLite::PrintCommTaskException(CollCommAicpu* aicpuComm)
209 : {
210 2 : CHK_PTR_NULL(aicpuComm);
211 2 : HcclResult ret = HCCL_SUCCESS;
212 2 : HCCL_RUN_INFO("[TaskException][AICPU]%s comm[%s] start", __func__, aicpuComm->GetIdentifier().c_str());
213 2 : std::shared_lock<std::shared_mutex> threadRwlock(aicpuComm->GetCommEngineResMgr()->GetThreadMutex());
214 2 : const std::vector<std::shared_ptr<hccl::Thread>> threads = aicpuComm->GetCommEngineResMgr()->GetAllThread();
215 3 : for (auto thread : threads) {
216 1 : CHK_SMART_PTR_NULL(thread);
217 1 : Hccl::StreamLite* streamLite = static_cast<Hccl::StreamLite*>(thread->GetStreamLitePtr());
218 1 : CHK_PTR_NULL(streamLite);
219 1 : u32 sqHead = 0U;
220 1 : u32 sqTail = 0U;
221 1 : ret = QuerySqStatus(devId_, streamLite->GetSqId(), sqHead, sqTail);
222 1 : if (ret != HCCL_SUCCESS || sqHead == sqTail) { // 此流为空时，不打印
223 0 : HCCL_RUN_INFO(
224 : "[TaskException][AICPU]PrintTaskExceptionBySqeId skip, "
225 : "QuerySqStatus ret[%d], aicpuComm[%s], sqId[%u], sqHead[%u], sqTail[%u]",
226 : ret, aicpuComm->GetIdentifier().c_str(), streamLite->GetSqId(), sqHead, sqTail);
227 0 : continue;
228 : }
229 1 : uint16_t streamId = 0;
230 1 : uint16_t taskId = 0;
231 1 : streamLite->GetRtsq()->GetStreamIdAndTaskIdBySqIdx(sqHead, streamId, taskId);
232 1 : const u32 sqeId = GetSqeId(taskId, streamId);
233 1 : HcclResult pRet = PrintTaskExceptionBySqeId(aicpuComm, streamLite->GetSqId(), sqeId);
234 1 : CHK_PRT_CONT(
235 : pRet != HCCL_SUCCESS, HCCL_ERROR(
236 : "PrintTaskExceptionBySqeId fail, comm[%s], sqId[%u], sqeId[%u]",
237 : aicpuComm->GetIdentifier().c_str(), streamLite->GetSqId(), sqeId));
238 1 : ret = (pRet != HCCL_SUCCESS) ? pRet : ret;
239 1 : }
240 2 : HCCL_RUN_INFO("[TaskException][AICPU]%s comm[%s] end, ret[%d]", __func__, aicpuComm->GetIdentifier().c_str(), ret);
241 2 : return ret;
242 2 : }
243 :
244 0 : HcclResult HcclCommTaskExceptionLite::GetThreadCqe(
245 : hccl::Thread* thread, rtLogicCqReport_t& cqeException, dfx::CqeStatus& cqeStatus)
246 : {
247 0 : CHK_SMART_PTR_NULL(thread);
248 0 : Hccl::StreamLite* streamLite = static_cast<Hccl::StreamLite*>(thread->GetStreamLitePtr());
249 0 : CHK_PTR_NULL(streamLite);
250 :
251 0 : constexpr u32 reportSize = MAX_REPORT_CNT;
252 : rtLogicCqReport_t streamReport[reportSize];
253 :
254 : CqeQueryInput cqeQueryInput;
255 0 : cqeQueryInput.devId = devId_;
256 0 : cqeQueryInput.streamId = streamLite->GetId();
257 0 : cqeQueryInput.sqId = streamLite->GetSqId();
258 0 : cqeQueryInput.cqId = streamLite->GetCqId();
259 0 : cqeQueryInput.type = static_cast<uint32_t>(DRV_LOGIC_TYPE);
260 0 : cqeQueryInput.cqeAddr = reinterpret_cast<uint8_t*>(streamReport);
261 :
262 0 : cqeStatus = CqReportRecv(cqeQueryInput, cqeException);
263 0 : if (cqeStatus == dfx::CqeStatus::kCqeInnerError) {
264 0 : HCCL_ERROR("[%s]CqReportRecv fail, CqeQueryInput:%s", __func__, cqeQueryInput.ToString().c_str());
265 0 : return HCCL_E_INTERNAL;
266 : }
267 0 : return HCCL_SUCCESS;
268 : }
269 :
270 1 : HcclResult HcclCommTaskExceptionLite::ProcessCqe(
271 : CollCommAicpu* aicpuComm, const rtLogicCqReport_t& exceptionInfo, const CqeStatus& cqeStatus,
272 : const std::vector<std::pair<std::string, CollCommAicpu*>>& aicpuCommInfo)
273 : {
274 1 : if (cqeStatus == dfx::CqeStatus::kDefault) {
275 1 : return HCCL_SUCCESS;
276 : }
277 :
278 0 : if (hcomm::GetTaskExceptionEnable() == false) {
279 0 : HCCL_ERROR("[TaskException][AICPU]taskException enable is false, skip print taskException");
280 0 : return HCCL_SUCCESS;
281 : }
282 :
283 0 : HcclResult ret = HCCL_SUCCESS;
284 0 : const u32 sqeId = GetSqeId(exceptionInfo.taskId, exceptionInfo.streamId);
285 0 : ret = PrintTaskExceptionBySqeId(aicpuComm, exceptionInfo.sqId, sqeId);
286 0 : CHK_PRT_CONT(
287 : ret != HCCL_SUCCESS, HCCL_ERROR(
288 : "[PrintTaskExceptionBySqeId]fail, ret[%d], group[%s], sqId[%u], taskId[%u]", ret,
289 : aicpuComm->GetIdentifier().c_str(), exceptionInfo.sqId,
290 : exceptionInfo.taskId)); // 如果上报失败，继续打印taskException
291 :
292 0 : ret = ReportErrMsg(aicpuComm, exceptionInfo);
293 0 : CHK_PRT_CONT(
294 : ret != HCCL_SUCCESS,
295 : HCCL_ERROR(
296 : "[ReportErrMsg]fail, ret[%d], group[%s], sqId[%u], taskId[%u]", ret, aicpuComm->GetIdentifier().c_str(),
297 : exceptionInfo.sqId, exceptionInfo.taskId)); // 如果上报失败，继续打印taskException
298 :
299 : // notify超时场景：step1 打印当前流信息；step2 打印当前通信域信息；step3 打印其他通信域信息
300 0 : if (cqeStatus == dfx::CqeStatus::kCqeException && exceptionInfo.sqeType == RT_STARS_SQE_TYPE_PLACE_HOLDER) {
301 0 : CHK_RET(PrintCommTaskException(aicpuComm));
302 0 : for (auto& commInfo : aicpuCommInfo) {
303 0 : CollCommAicpu* comm = commInfo.second;
304 0 : if (comm != nullptr && comm->GetIdentifier() != aicpuComm->GetIdentifier()) {
305 0 : CHK_RET(PrintCommTaskException(comm));
306 : }
307 : }
308 : }
309 0 : return ret;
310 : }
311 :
312 4 : u32 HcclCommTaskExceptionLite::GetSqeId(uint16_t taskId, uint16_t streamId)
313 : {
314 4 : return (static_cast<u32>(taskId) << TASK_ID_SHIFT_BITS) | static_cast<u32>(streamId);
315 : }
316 :
317 4 : HcclResult HcclCommTaskExceptionLite::ReportErrMsg(CollCommAicpu* aicpuComm, const rtLogicCqReport_t& exceptionInfo)
318 : {
319 4 : CHK_PTR_NULL(aicpuComm);
320 :
321 3 : const u32 sqeId = GetSqeId(exceptionInfo.taskId, exceptionInfo.streamId);
322 3 : HCCL_INFO(
323 : "[%s]group[%s], sqeId[0x%x], taskId[%u], streamId[%u].", __func__, aicpuComm->GetIdentifier().c_str(), sqeId,
324 : exceptionInfo.taskId, exceptionInfo.streamId);
325 :
326 3 : Hccl::DfxTaskInfo* curTask = FindDfxTaskInfo(aicpuComm, exceptionInfo.sqId, sqeId);
327 3 : CHK_PTR_NULL(curTask);
328 :
329 2 : const Hccl::DfxDfxOpInfo* opInfo = (curTask->dfxOpInfo != DFX_INVALID_U64) ?
330 1 : reinterpret_cast<const Hccl::DfxDfxOpInfo*>(curTask->dfxOpInfo) :
331 : nullptr;
332 2 : CHK_PTR_NULL(opInfo);
333 :
334 1 : if (!aicpuComm->IsErrorReported()) {
335 0 : Hccl::ErrorMessageReport errMsgInfo{};
336 0 : CHK_RET(GenerateErrorMessageReport(aicpuComm, *curTask, exceptionInfo, errMsgInfo));
337 0 : CHK_RET(aicpuComm->SendErrorMessageReportToHost(errMsgInfo));
338 :
339 0 : u32 notifyId = opInfo->cpuWaitAicpuNotifyId;
340 0 : CHK_RET(SendTaskExceptionByMBox(notifyId, 0, exceptionInfo));
341 0 : aicpuComm->SetErrorReported(true);
342 : }
343 1 : return HCCL_SUCCESS;
344 : }
345 :
346 1 : HcclResult HcclCommTaskExceptionLite::PrintTaskExceptionBySqeId(CollCommAicpu* aicpuComm, u32 sqId, u32 sqeId)
347 : {
348 1 : CHK_PTR_NULL(aicpuComm);
349 :
350 : // 已经打印过的不再重复打印
351 1 : Hccl::DfxTaskInfo* curTask = FindDfxTaskInfo(aicpuComm, sqId, sqeId);
352 1 : CHK_PTR_NULL(curTask);
353 :
354 1 : auto it = threadsPrinted_.find(sqId);
355 1 : if (it != threadsPrinted_.end() && it->second == sqeId) {
356 0 : HCCL_RUN_INFO("[TaskException][AICPU]sqId:%u, sqeId:%u has been printed, skip", sqId, sqeId);
357 0 : return HCCL_SUCCESS;
358 : }
359 1 : threadsPrinted_[sqId] = sqeId;
360 :
361 1 : u32 sqHead = 0U;
362 1 : u32 sqTail = 0U;
363 1 : (void)QuerySqStatus(devId_, sqId, sqHead, sqTail);
364 :
365 1 : HCCL_ERROR(
366 : "[TaskException][AICPU]base information is streamID(sqId):[%u], taskID(sqeId):[%u], taskType:[%u], "
367 : "sqHead:%u, sqTail:%u",
368 : curTask->sqId, curTask->taskId, curTask->taskType, sqHead, sqTail);
369 :
370 1 : PrintEid(*curTask);
371 :
372 1 : HCCL_ERROR("[TaskException][AICPU]group information is %s.", GetGroupInfo(aicpuComm).c_str());
373 :
374 1 : if (curTask->taskType != static_cast<u8>(Hccl::TaskParamTypeVal::TASK_NOTIFY_WAIT)) {
375 1 : PrintOpDataInfo(curTask);
376 : } else {
377 0 : CHK_RET(PrintTaskContextInfo(aicpuComm, sqId, sqeId));
378 : }
379 1 : return HCCL_SUCCESS;
380 : }
381 :
382 1 : HcclResult HcclCommTaskExceptionLite::GenerateErrorMessageReport(
383 : CollCommAicpu* aicpuComm, const Hccl::DfxTaskInfo& taskInfo, const rtLogicCqReport_t& exceptionInfo,
384 : Hccl::ErrorMessageReport& errMsgInfo)
385 : {
386 1 : const Hccl::DfxDfxOpInfo* opInfo = (taskInfo.dfxOpInfo != DFX_INVALID_U64) ?
387 0 : reinterpret_cast<const Hccl::DfxDfxOpInfo*>(taskInfo.dfxOpInfo) :
388 : nullptr;
389 1 : CHK_PTR_NULL(opInfo);
390 :
391 0 : errMsgInfo.remoteUserRank = GetRemoteRankId(taskInfo);
392 0 : errMsgInfo.streamId = taskInfo.sqId;
393 0 : errMsgInfo.taskId = taskInfo.taskId;
394 0 : errMsgInfo.rankId = aicpuComm->GetTopoInfo().userRank;
395 0 : errMsgInfo.rankSize = aicpuComm->GetTopoInfo().userRankSize;
396 :
397 0 : errMsgInfo.opIndex = opInfo->opIndex;
398 0 : errMsgInfo.opType = opInfo->opType;
399 0 : errMsgInfo.count = opInfo->count;
400 0 : errMsgInfo.dataType = opInfo->dataType;
401 0 : errMsgInfo.srcAddr = opInfo->srcAddr;
402 0 : errMsgInfo.dstAddr = opInfo->dstAddr;
403 0 : CHK_SAFETY_FUNC_RET(memcpy_s(
404 : errMsgInfo.tag, sizeof(errMsgInfo.tag), opInfo->algTag, strnlen(opInfo->algTag, sizeof(opInfo->algTag))));
405 :
406 0 : errMsgInfo.taskType = Hccl::TaskParamType(static_cast<Hccl::TaskParamType::Value>(taskInfo.taskType));
407 0 : errMsgInfo.rtCqErrorType = exceptionInfo.errorType;
408 0 : errMsgInfo.rtCqErrorCode = exceptionInfo.errorCode;
409 :
410 0 : CHK_SAFETY_FUNC_RET(memcpy_s(
411 : errMsgInfo.group, sizeof(errMsgInfo.group), aicpuComm->GetIdentifier().c_str(),
412 : aicpuComm->GetIdentifier().size() + 1));
413 :
414 0 : GenerateTaskErrMsg(taskInfo, errMsgInfo, exceptionInfo);
415 0 : return HCCL_SUCCESS;
416 : }
417 :
418 9 : void HcclCommTaskExceptionLite::GenerateTaskErrMsg(
419 : const Hccl::DfxTaskInfo& taskInfo, Hccl::ErrorMessageReport& errMsgInfo, const rtLogicCqReport_t& exceptionInfo)
420 : {
421 9 : auto taskType = static_cast<Hccl::TaskParamTypeVal>(taskInfo.taskType);
422 9 : switch (taskType) {
423 2 : case Hccl::TaskParamTypeVal::TASK_NOTIFY_WAIT:
424 : case Hccl::TaskParamTypeVal::TASK_NOTIFY_RECORD:
425 2 : FillNotifyErrMsg(taskInfo, errMsgInfo);
426 2 : break;
427 2 : case Hccl::TaskParamTypeVal::TASK_UB_REDUCE_INLINE:
428 : case Hccl::TaskParamTypeVal::TASK_WRITE_REDUCE_WITH_NOTIFY:
429 2 : FillReduceErrMsg(taskInfo, errMsgInfo, exceptionInfo);
430 2 : break;
431 1 : case Hccl::TaskParamTypeVal::TASK_REDUCE_INLINE:
432 1 : FillReduceInlineErrMsg(taskInfo, errMsgInfo);
433 1 : break;
434 2 : case Hccl::TaskParamTypeVal::TASK_UB_INLINE_WRITE:
435 : case Hccl::TaskParamTypeVal::TASK_WRITE_WITH_NOTIFY:
436 2 : FillDmaErrMsg(taskInfo, errMsgInfo, exceptionInfo);
437 2 : break;
438 1 : case Hccl::TaskParamTypeVal::TASK_UB:
439 1 : FillUbErrMsg(taskInfo, errMsgInfo, exceptionInfo);
440 1 : break;
441 1 : case Hccl::TaskParamTypeVal::TASK_SDMA:
442 1 : FillSdmaErrMsg(taskInfo, errMsgInfo);
443 1 : break;
444 0 : default:
445 0 : HCCL_ERROR("[TaskException][AICPU]%s taskType[%d] is not support", __func__, taskInfo.taskType);
446 0 : return;
447 : }
448 : }
449 :
450 2 : void HcclCommTaskExceptionLite::FillNotifyErrMsg(
451 : const Hccl::DfxTaskInfo& taskInfo, Hccl::ErrorMessageReport& errMsgInfo)
452 : {
453 2 : void* sqePtr = reinterpret_cast<void*>(taskInfo.taskPara.Notify.sqeAddr);
454 2 : if (sqePtr != nullptr) {
455 2 : auto* header = reinterpret_cast<Hccl::Rt91095StarsSqeHeader*>(sqePtr);
456 2 : if (static_cast<Hccl::Rt91095StarsSqeType>(header->type)
457 2 : == Hccl::Rt91095StarsSqeType::RT_91095_SQE_TYPE_NOTIFY_RECORD
458 1 : || static_cast<Hccl::Rt91095StarsSqeType>(header->type)
459 1 : == Hccl::Rt91095StarsSqeType::RT_91095_SQE_TYPE_NOTIFY_WAIT) {
460 2 : auto* notifySqe = reinterpret_cast<Hccl::Rt91095StarsNotifySqe*>(sqePtr);
461 2 : errMsgInfo.notifyId = notifySqe->notifyId;
462 2 : errMsgInfo.notifyValue = notifySqe->cntValue;
463 : }
464 : }
465 2 : }
466 :
467 2 : void HcclCommTaskExceptionLite::FillReduceErrMsg(
468 : const Hccl::DfxTaskInfo& taskInfo, Hccl::ErrorMessageReport& errMsgInfo, const rtLogicCqReport_t& exceptionInfo)
469 : {
470 2 : errMsgInfo.reduceType = taskInfo.taskPara.Reduce.reduceOp;
471 2 : errMsgInfo.notifyId = taskInfo.taskPara.Reduce.notifyId;
472 2 : errMsgInfo.notifyValue = INVALID_U32;
473 2 : GetEidFromChannelHandle(taskInfo, errMsgInfo.locEid, errMsgInfo.rmtEid);
474 2 : errMsgInfo.ubCqeStatus = exceptionInfo.errorCode & 0xFF;
475 2 : errMsgInfo.linkType = Hccl::DfxLinkType(static_cast<Hccl::DfxLinkType::Value>(taskInfo.linkType));
476 2 : errMsgInfo.size = taskInfo.taskPara.Reduce.size;
477 2 : errMsgInfo.taskSrcAddr = taskInfo.taskPara.Reduce.srcAddr;
478 2 : errMsgInfo.taskDstAddr = taskInfo.taskPara.Reduce.dstAddr;
479 2 : HCCL_ERROR(
480 : "[TaskException][AICPU]ubCqeStatus[%u], localEid[%s], remoteEid[%s]. ", errMsgInfo.ubCqeStatus,
481 : errMsgInfo.locEid.Describe().c_str(), errMsgInfo.rmtEid.Describe().c_str());
482 2 : }
483 :
484 2 : void HcclCommTaskExceptionLite::FillDmaErrMsg(
485 : const Hccl::DfxTaskInfo& taskInfo, Hccl::ErrorMessageReport& errMsgInfo, const rtLogicCqReport_t& exceptionInfo)
486 : {
487 2 : errMsgInfo.notifyId = taskInfo.taskPara.ubDma.notifyId;
488 2 : errMsgInfo.notifyValue = INVALID_U32;
489 2 : FillUbErrMsg(taskInfo, errMsgInfo, exceptionInfo);
490 2 : }
491 :
492 1 : void HcclCommTaskExceptionLite::FillSdmaErrMsg(const Hccl::DfxTaskInfo& taskInfo, Hccl::ErrorMessageReport& errMsgInfo)
493 : {
494 1 : errMsgInfo.linkType = Hccl::DfxLinkType(static_cast<Hccl::DfxLinkType::Value>(taskInfo.linkType));
495 1 : void* sqePtr = reinterpret_cast<void*>(taskInfo.taskPara.Dma.sqeAddr);
496 1 : if (sqePtr != nullptr) {
497 0 : auto* header = reinterpret_cast<Hccl::Rt91095StarsSqeHeader*>(sqePtr);
498 0 : if (static_cast<Hccl::Rt91095StarsSqeType>(header->type) == Hccl::Rt91095StarsSqeType::RT_91095_SQE_TYPE_SDMA) {
499 0 : auto* dmaSqe = reinterpret_cast<Hccl::Rt91095StarsMemcpySqe*>(sqePtr);
500 : errMsgInfo.taskSrcAddr
501 0 : = (static_cast<u64>(dmaSqe->u.strideMode0.srcAddrHigh) << 32) | dmaSqe->u.strideMode0.srcAddrLow;
502 : errMsgInfo.taskDstAddr
503 0 : = (static_cast<u64>(dmaSqe->u.strideMode0.dstAddrHigh) << 32) | dmaSqe->u.strideMode0.dstAddrLow;
504 0 : errMsgInfo.size = dmaSqe->u.strideMode0.lengthMove;
505 : }
506 : }
507 1 : }
508 :
509 3 : void HcclCommTaskExceptionLite::FillUbErrMsg(
510 : const Hccl::DfxTaskInfo& taskInfo, Hccl::ErrorMessageReport& errMsgInfo, const rtLogicCqReport_t& exceptionInfo)
511 : {
512 3 : GetEidFromChannelHandle(taskInfo, errMsgInfo.locEid, errMsgInfo.rmtEid);
513 3 : errMsgInfo.ubCqeStatus = exceptionInfo.errorCode & 0xFF;
514 3 : errMsgInfo.linkType = Hccl::DfxLinkType(static_cast<Hccl::DfxLinkType::Value>(taskInfo.linkType));
515 3 : errMsgInfo.size = taskInfo.taskPara.ubDma.size;
516 3 : errMsgInfo.taskSrcAddr = taskInfo.taskPara.ubDma.srcAddr;
517 3 : errMsgInfo.taskDstAddr = taskInfo.taskPara.ubDma.dstAddr;
518 3 : HCCL_ERROR(
519 : "[TaskException][AICPU]ubCqeStatus[%u], localEid[%s], remoteEid[%s]. ", errMsgInfo.ubCqeStatus,
520 : errMsgInfo.locEid.Describe().c_str(), errMsgInfo.rmtEid.Describe().c_str());
521 3 : }
522 :
523 1 : void HcclCommTaskExceptionLite::FillReduceInlineErrMsg(
524 : const Hccl::DfxTaskInfo& taskInfo, Hccl::ErrorMessageReport& errMsgInfo)
525 : {
526 1 : errMsgInfo.reduceType = taskInfo.taskPara.Reduce.reduceOp;
527 1 : }
528 :
529 4 : HcclResult HcclCommTaskExceptionLite::SendTaskExceptionByMBox(
530 : const u32 notifyId, const u32 tsId, const rtLogicCqReport_t& exceptionInfo)
531 : {
532 4 : ts_aicpu_msg_info_t aicpuSqe = {};
533 4 : u32 hostpid = 0;
534 4 : u32 vfId = 0;
535 4 : int pid = getpid();
536 4 : HCCL_INFO("[%s]getpid[%d]", __func__, pid);
537 : // 调整drvQueryProcessHostPid获取pid和vf_id的值
538 4 : CHK_RET(HrtHalDrvQueryProcessHostPid(pid, nullptr, &vfId, &hostpid, nullptr));
539 :
540 4 : aicpuSqe.pid = hostpid;
541 4 : aicpuSqe.cmd_type = TS_AICPU_RECORD;
542 4 : aicpuSqe.vf_id = vfId;
543 4 : aicpuSqe.tid = 0U; // notify is no need tid
544 4 : aicpuSqe.u.aicpu_record.record_type = AICPU_MSG_NOTIFY_RECORD_V2;
545 4 : aicpuSqe.u.aicpu_record.record_id = notifyId;
546 4 : aicpuSqe.ts_id = static_cast<uint8_t>(tsId);
547 4 : aicpuSqe.u.aicpu_record.fault_task_id = 0xffffffff;
548 :
549 4 : if (exceptionInfo.sqeType == ubSqeType) {
550 1 : aicpuSqe.u.aicpu_record.ret_code = SwitchUBCqeErrCodeToTsErrCode(exceptionInfo.errorCode & 0xFF);
551 3 : } else if (exceptionInfo.sqeType == sdmaSqeType) {
552 1 : aicpuSqe.u.aicpu_record.ret_code = SwitchSdmaCqeErrCodeToTsErrCode(exceptionInfo.errorCode);
553 : } else {
554 2 : aicpuSqe.u.aicpu_record.ret_code = TS_ERROR_HCCL_OTHER_ERROR;
555 : }
556 :
557 : struct event_summary event;
558 4 : event.dst_engine = TS_CPU;
559 4 : event.policy = ONLY;
560 4 : event.pid = 0;
561 4 : event.grp_id = 0;
562 4 : event.event_id = EVENT_TS_CTRL_MSG;
563 4 : event.subevent_id = 0U;
564 4 : event.msg_len = static_cast<uint32_t>(sizeof(ts_aicpu_msg_info_t));
565 4 : event.msg = reinterpret_cast<char_t*>(&aicpuSqe);
566 4 : drvError_t ret = Hccl::DlHalFunctionV2::GetInstance().dlHalEschedSubmitEvent(devId_, &event);
567 4 : if (ret != DRV_ERROR_NONE) {
568 0 : HCCL_ERROR(
569 : "[%s]dlHalEschedSubmitEvent failed, ret=%d, notifyId=%u, hostpid=%u, vfId=%u, tsId=%u", __func__, ret,
570 : notifyId, hostpid, vfId, tsId);
571 0 : return HCCL_E_DRV;
572 : }
573 4 : HCCL_RUN_INFO(
574 : "[%s]finished, notifyId=%u, hostpid=%u, vfId=%u, tsId=%u, errorType=%u, errorCode=%u, ret_code=%u", __func__,
575 : notifyId, hostpid, vfId, tsId, exceptionInfo.errorType, exceptionInfo.errorCode,
576 : aicpuSqe.u.aicpu_record.ret_code);
577 4 : return HCCL_SUCCESS;
578 : }
579 :
580 : // 把UB类错误码转换成Ts对应的错误码
581 5 : uint16_t HcclCommTaskExceptionLite::SwitchUBCqeErrCodeToTsErrCode(u32 cqeErrCode)
582 : {
583 5 : switch (cqeErrCode) {
584 2 : case RT_UB_LOCAL_OPERATIOINERR:
585 2 : return TS_ERROR_HCCL_OP_UB_DDRC_FAILED;
586 1 : case RT_UB_REMOTE_OPERATIOINERR:
587 1 : return TS_ERROR_HCCL_OP_UB_POISON_FAILED;
588 1 : case RT_UB_LINK_FAILEDERR:
589 1 : return TS_ERROR_HCCL_OP_UB_LINK_FAILED;
590 1 : default:
591 1 : return TS_ERROR_HCCL_OTHER_ERROR;
592 : }
593 : }
594 :
595 : // 把SDMA类错误码转换成Ts对应的错误码
596 5 : uint16_t HcclCommTaskExceptionLite::SwitchSdmaCqeErrCodeToTsErrCode(u32 cqeErrCode)
597 : {
598 5 : switch (cqeErrCode) {
599 2 : case RT_SDMA_COMPERR:
600 2 : return TS_ERROR_SDMA_LINK_ERROR;
601 1 : case RT_SDMA_COMPDATAERR:
602 1 : return TS_ERROR_SDMA_POISON_ERROR;
603 1 : case RT_SDMA_DATAERR:
604 1 : return TS_ERROR_SDMA_DDRC_ERROR;
605 1 : default:
606 1 : return TS_ERROR_HCCL_OTHER_ERROR;
607 : }
608 : }
609 :
610 2 : HcclResult HcclCommTaskExceptionLite::CollectTaskContext(
611 : CollCommAicpu* aicpuComm, u32 sqId, u32 taskId, std::vector<Hccl::DfxTaskInfo*>& taskContext)
612 : {
613 2 : Hccl::TaskInfoCircularQueue* queue = GetTaskQueueBySqId(aicpuComm, sqId);
614 2 : CHK_PRT_RET(
615 : queue == nullptr, HCCL_ERROR("[%s]GetTaskQueueBySqId nullptr, devId[%u], sqId[%u].", __func__, devId_, sqId),
616 : HCCL_E_PARA);
617 :
618 0 : if (queue->IsEmpty()) {
619 0 : HCCL_ERROR("[%s]queue is empty, devId[%u], sqId[%u].", __func__, devId_, sqId);
620 0 : return HCCL_E_PARA;
621 : }
622 :
623 0 : u32 targetTaskId = taskId;
624 0 : u16 begin = queue->GetBegin();
625 0 : Hccl::DfxTaskInfo* found = nullptr;
626 0 : u16 foundIdx = 0;
627 0 : for (u16 idx = 0; idx < queue->GetCapacity(); idx++) {
628 0 : Hccl::DfxTaskInfo* slot = queue->GetSlot(idx);
629 0 : if (slot != nullptr && slot->taskId == targetTaskId) {
630 0 : found = slot;
631 0 : foundIdx = idx;
632 0 : break;
633 : }
634 : }
635 0 : CHK_PRT_RET(
636 : found == nullptr,
637 : HCCL_ERROR("[%s]exception task not found, devId[%u], sqId[%u], taskId[%u]", __func__, devId_, sqId, taskId),
638 : HCCL_E_PARA);
639 :
640 0 : u32 ctxCount = 0;
641 0 : for (u16 idx = foundIdx; ctxCount < TASK_CONTEXT_SIZE; ++ctxCount) {
642 0 : if (idx == begin) {
643 0 : break;
644 : }
645 0 : idx = (idx == 0) ? static_cast<u16>(queue->GetCapacity() - 1) : idx - 1;
646 0 : Hccl::DfxTaskInfo* slot = queue->GetSlot(idx);
647 0 : if (slot == nullptr || slot->taskId > targetTaskId) {
648 : break;
649 : }
650 0 : taskContext.push_back(slot);
651 : }
652 0 : return HCCL_SUCCESS;
653 : }
654 :
655 1 : HcclResult HcclCommTaskExceptionLite::PrintTaskContextInfo(CollCommAicpu* aicpuComm, u32 sqId, u32 taskId)
656 : {
657 1 : std::vector<Hccl::DfxTaskInfo*> taskContext{};
658 1 : CHK_PRT_RET(
659 : CollectTaskContext(aicpuComm, sqId, taskId, taskContext) != HCCL_SUCCESS,
660 : HCCL_ERROR("[%s]CollectTaskContext failed, devId[%u], sqId[%u], taskId[%u]", __func__, devId_, sqId, taskId),
661 : HCCL_E_PARA);
662 :
663 0 : std::string taskContextInfo = "";
664 0 : Hccl::DfxTaskInfo* lastTask = nullptr;
665 0 : for (u32 i = 0; i < taskContext.size(); ++i) {
666 0 : if (taskContext[i] == nullptr) {
667 0 : continue;
668 : }
669 0 : if (lastTask == nullptr) {
670 0 : lastTask = taskContext[i];
671 : }
672 0 : std::string conciseInfo = GetConciseTaskName(*taskContext[i]) + ",";
673 0 : u32 lastOpIndex = GetOpIndex(lastTask);
674 0 : u32 curOpIndex = GetOpIndex(taskContext[i]);
675 0 : bool overSize = (taskContextInfo.size() + conciseInfo.size()) >= TASK_CONTEXT_INFO_SIZE;
676 0 : if (overSize || (lastOpIndex != curOpIndex)) {
677 0 : PrintOpDataInfo(lastTask);
678 0 : HCCL_ERROR("[TaskException][AICPU]task sequence is OP(%u): %s", lastOpIndex, taskContextInfo.c_str());
679 0 : taskContextInfo = "";
680 0 : lastTask = taskContext[i];
681 : }
682 0 : taskContextInfo += conciseInfo;
683 0 : }
684 :
685 0 : if (!taskContextInfo.empty() && lastTask != nullptr) {
686 0 : u32 lastOpIndex = GetOpIndex(lastTask);
687 0 : PrintOpDataInfo(lastTask);
688 0 : HCCL_ERROR("[TaskException][AICPU]task sequence is OP(%u): %s", lastOpIndex, taskContextInfo.c_str());
689 : }
690 0 : HCCL_ERROR("[TaskException][AICPU]task sequence end.");
691 0 : return HCCL_SUCCESS;
692 1 : }
693 :
694 3 : std::string HcclCommTaskExceptionLite::GetGroupInfo(CollCommAicpu* aicpuComm)
695 : {
696 3 : if (aicpuComm == nullptr) {
697 1 : HCCL_ERROR("[%s]aicpuComm is nullptr, return empty string.", __func__);
698 2 : return "";
699 : }
700 : return Hccl::StringFormat(
701 2 : "group:[%s], rankSize:[%u], localRank:[%u]", aicpuComm->GetIdentifier().c_str(),
702 2 : aicpuComm->GetTopoInfo().userRankSize, aicpuComm->GetTopoInfo().userRank);
703 : }
704 :
705 3 : void HcclCommTaskExceptionLite::PrintEid(const Hccl::DfxTaskInfo& taskInfo)
706 : {
707 3 : auto taskType = static_cast<Hccl::TaskParamTypeVal>(taskInfo.taskType);
708 3 : if (taskType == Hccl::TaskParamTypeVal::TASK_UB_REDUCE_INLINE
709 3 : || taskType == Hccl::TaskParamTypeVal::TASK_WRITE_REDUCE_WITH_NOTIFY
710 3 : || taskType == Hccl::TaskParamTypeVal::TASK_UB_INLINE_WRITE
711 3 : || taskType == Hccl::TaskParamTypeVal::TASK_WRITE_WITH_NOTIFY || taskType == Hccl::TaskParamTypeVal::TASK_UB) {
712 1 : Hccl::Eid locEid;
713 1 : Hccl::Eid rmtEid;
714 1 : GetEidFromChannelHandle(taskInfo, locEid, rmtEid);
715 1 : HCCL_ERROR(
716 : "[TaskException][AICPU][%s]Error UB link info: localEid[%s], remoteEid[%s].", __func__,
717 : locEid.Describe().c_str(), rmtEid.Describe().c_str());
718 : }
719 3 : }
720 :
721 4 : Hccl::DfxTaskInfo* HcclCommTaskExceptionLite::FindDfxTaskInfo(CollCommAicpu* aicpuComm, u32 sqId, u32 sqeId)
722 : {
723 4 : Hccl::TaskInfoCircularQueue* queue = GetTaskQueueBySqId(aicpuComm, sqId);
724 4 : if (queue == nullptr || queue->IsEmpty()) {
725 1 : HCCL_ERROR("[%s]GetTaskQueueBySqId nullptr or queue is empty, devId[%u], sqId[%u].", __func__, devId_, sqId);
726 1 : return nullptr;
727 : }
728 3 : u32 targetTaskId = sqeId;
729 3 : for (u16 idx = 0; idx < queue->GetCapacity(); idx++) {
730 3 : Hccl::DfxTaskInfo* slot = queue->GetSlot(idx);
731 3 : if (slot != nullptr && slot->taskId == targetTaskId) {
732 3 : return slot;
733 : }
734 : }
735 0 : HCCL_ERROR("[%s]exception task not found, devId[%u], sqId[%u], sqeId[%u]", __func__, devId_, sqId, sqeId);
736 0 : return nullptr;
737 : }
738 :
739 6 : Hccl::TaskInfoCircularQueue* HcclCommTaskExceptionLite::GetTaskQueueBySqId(CollCommAicpu* aicpuComm, u32 sqId)
740 : {
741 6 : std::shared_lock<std::shared_mutex> threadRwlock(aicpuComm->GetCommEngineResMgr()->GetThreadMutex());
742 6 : const std::vector<std::shared_ptr<hccl::Thread>> threads = aicpuComm->GetCommEngineResMgr()->GetAllThread();
743 6 : for (auto& thread : threads) {
744 3 : Hccl::StreamLite* streamLite = static_cast<Hccl::StreamLite*>(thread->GetStreamLitePtr());
745 3 : if (streamLite != nullptr && streamLite->GetSqId() == sqId) {
746 3 : return streamLite->GetTaskInfos();
747 : }
748 : }
749 3 : return nullptr;
750 6 : }
751 :
752 6 : void HcclCommTaskExceptionLite::GetEidFromChannelHandle(
753 : const Hccl::DfxTaskInfo& taskInfo, Hccl::Eid& locEid, Hccl::Eid& rmtEid)
754 : {
755 6 : if (taskInfo.channelHandle != DFX_INVALID_U64) {
756 0 : auto* transport = reinterpret_cast<Hccl::UbTransportLiteImpl*>(taskInfo.channelHandle);
757 0 : locEid = transport->GetLocEid();
758 0 : rmtEid = transport->GetRmtEid();
759 : }
760 6 : }
761 :
762 4 : u32 HcclCommTaskExceptionLite::GetRemoteRankId(const Hccl::DfxTaskInfo& taskInfo)
763 : {
764 4 : if (taskInfo.dfxOpInfo != DFX_INVALID_U64) {
765 1 : auto* opInfo = reinterpret_cast<const Hccl::DfxDfxOpInfo*>(taskInfo.dfxOpInfo);
766 1 : if (opInfo->hcclCommDfxLite != nullptr) {
767 0 : return static_cast<hccl::HcclCommDfxLite*>(opInfo->hcclCommDfxLite)
768 0 : ->GetChannelRemoteRankId(taskInfo.channelHandle);
769 : }
770 : }
771 4 : return Hccl::DFX_INVALID_RANKID;
772 : }
773 :
774 1 : void HcclCommTaskExceptionLite::GetNotifyIdFromSqe(u64 sqeAddr, u32& notifyId)
775 : {
776 1 : void* sqePtr = reinterpret_cast<void*>(sqeAddr);
777 1 : if (sqePtr != nullptr) {
778 0 : auto* header = reinterpret_cast<Hccl::Rt91095StarsSqeHeader*>(sqePtr);
779 0 : if (static_cast<Hccl::Rt91095StarsSqeType>(header->type)
780 0 : == Hccl::Rt91095StarsSqeType::RT_91095_SQE_TYPE_NOTIFY_RECORD
781 0 : || static_cast<Hccl::Rt91095StarsSqeType>(header->type)
782 0 : == Hccl::Rt91095StarsSqeType::RT_91095_SQE_TYPE_NOTIFY_WAIT) {
783 0 : auto* notifySqe = reinterpret_cast<Hccl::Rt91095StarsNotifySqe*>(sqePtr);
784 0 : notifyId = notifySqe->notifyId;
785 : }
786 : }
787 1 : }
788 :
789 4 : std::string HcclCommTaskExceptionLite::GetNotifyInfo(const Hccl::DfxTaskInfo& taskInfo)
790 : {
791 4 : auto taskType = static_cast<Hccl::TaskParamTypeVal>(taskInfo.taskType);
792 4 : u32 notifyId = INVALID_U32;
793 4 : switch (taskType) {
794 1 : case Hccl::TaskParamTypeVal::TASK_UB_INLINE_WRITE:
795 : case Hccl::TaskParamTypeVal::TASK_WRITE_WITH_NOTIFY:
796 1 : notifyId = taskInfo.taskPara.ubDma.notifyId;
797 1 : break;
798 1 : case Hccl::TaskParamTypeVal::TASK_WRITE_REDUCE_WITH_NOTIFY:
799 1 : notifyId = taskInfo.taskPara.Reduce.notifyId;
800 1 : break;
801 1 : case Hccl::TaskParamTypeVal::TASK_NOTIFY_RECORD:
802 : case Hccl::TaskParamTypeVal::TASK_NOTIFY_WAIT:
803 : case Hccl::TaskParamTypeVal::TASK_SEND_NOTIFY: {
804 1 : GetNotifyIdFromSqe(taskInfo.taskPara.Notify.sqeAddr, notifyId);
805 1 : break;
806 : }
807 0 : case Hccl::TaskParamTypeVal::TASK_RDMA: {
808 0 : GetNotifyIdFromSqe(taskInfo.taskPara.Dma.sqeAddr, notifyId);
809 0 : break;
810 : }
811 1 : default:
812 2 : return "/";
813 : }
814 5 : return (notifyId == INVALID_U32) ? "/" : std::to_string(notifyId);
815 : }
816 :
817 2 : std::string HcclCommTaskExceptionLite::GetConciseTaskName(const Hccl::DfxTaskInfo& taskInfo)
818 : {
819 2 : const auto& taskConciseNameMap = Hccl::GetTaskConciseNameMap();
820 2 : auto it = taskConciseNameMap.find(taskInfo.taskType);
821 2 : std::string name = (it != taskConciseNameMap.end()) ? it->second : "UNKNOWN";
822 2 : u32 remoteRank = GetRemoteRankId(taskInfo);
823 4 : std::string rankStr = (remoteRank == Hccl::DFX_INVALID_RANKID) ? "/" : std::to_string(remoteRank);
824 2 : auto taskType = static_cast<Hccl::TaskParamTypeVal>(taskInfo.taskType);
825 2 : if (taskType == Hccl::TaskParamTypeVal::TASK_RDMA || taskType == Hccl::TaskParamTypeVal::TASK_NOTIFY_RECORD
826 2 : || taskType == Hccl::TaskParamTypeVal::TASK_NOTIFY_WAIT || taskType == Hccl::TaskParamTypeVal::TASK_SEND_NOTIFY
827 1 : || taskType == Hccl::TaskParamTypeVal::TASK_WRITE_WITH_NOTIFY
828 1 : || taskType == Hccl::TaskParamTypeVal::TASK_WRITE_REDUCE_WITH_NOTIFY
829 1 : || taskType == Hccl::TaskParamTypeVal::TASK_UB_INLINE_WRITE) {
830 1 : return name + "(" + rankStr + "," + GetNotifyInfo(taskInfo) + ")";
831 : }
832 1 : return name + "(" + rankStr + ")";
833 2 : }
834 :
835 0 : u32 HcclCommTaskExceptionLite::GetOpIndex(const Hccl::DfxTaskInfo* taskInfo)
836 : {
837 0 : if (taskInfo == nullptr || taskInfo->dfxOpInfo == DFX_INVALID_U64) {
838 0 : return UINT32_MAX;
839 : }
840 0 : return reinterpret_cast<const Hccl::DfxDfxOpInfo*>(taskInfo->dfxOpInfo)->opIndex;
841 : }
842 :
843 1 : void HcclCommTaskExceptionLite::PrintOpDataInfo(const Hccl::DfxTaskInfo* taskInfo)
844 : {
845 1 : if (taskInfo == nullptr || taskInfo->dfxOpInfo == DFX_INVALID_U64) {
846 1 : HCCL_ERROR("[TaskException][AICPU]opData information is (dfxOpInfo unavailable).");
847 1 : return;
848 : }
849 0 : const Hccl::DfxDfxOpInfo* opInfo = reinterpret_cast<const Hccl::DfxDfxOpInfo*>(taskInfo->dfxOpInfo);
850 0 : HCCL_ERROR(
851 : "[TaskException][AICPU]opData information is opIndex[%u], algTag[%s], count[%llu], "
852 : "dataType[%u], input: ptr[0x%llx] size[%llu], output: ptr[0x%llx] size[%llu].",
853 : opInfo->opIndex, opInfo->algTag, opInfo->count, opInfo->dataType, opInfo->srcAddr, opInfo->srcSize,
854 : opInfo->dstAddr, opInfo->dstSize);
855 : }
856 : } // namespace hcomm
|
