Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "hcclCommTaskExceptionLite.h"
12 : #include "stream_lite.h"
13 : #include "hcomm_task_scheduler_error.h"
14 : #include "task_struct_v2.h"
15 : #include "dlhal_function_v2.h"
16 : #include <shared_mutex>
17 : #include "aicpu_indop_env.h"
18 : #include "kernel_entrance.h"
19 : #include "ub_transport_lite_impl.h"
20 : #include "res_TE.h"
21 :
22 : namespace hcomm {
23 : constexpr u32 RT_SDMA_COMPERR = 0x9; // A3 sdma error类型为0x9时，表示写拷贝发生超时代答，或者数据搬移时地址译码错误
24 : constexpr u32 RT_SDMA_COMPDATAERR = 0xa; // A3 sdma error类型为0xa时，表示读拷贝发生超时代答，或者读HBM返回ERROR
25 : constexpr u32 RT_SDMA_DATAERR = 0x8; // A3 sdma error类型为0x8时，表示读HBM返回ERROR
26 : constexpr u32 RT_UB_LOCAL_OPERATIOINERR = 0x2; // A5 ub error类型为0x2时，表示UB本端返回ERROR
27 : constexpr u32 RT_UB_REMOTE_OPERATIOINERR = 0x3; // A5 ub error类型为0x3时，表示UB远端返回ERROR
28 : constexpr u32 RT_UB_LINK_FAILEDERR = 0x5; // A5 ub error类型为0x5时，表示网络异常，taack超时
29 : constexpr uint8_t ubSqeType = 9; // A5 sqeType为9表示UBDMA任务
30 : constexpr uint8_t sdmaSqeType = 11; // A5 sqeType为11表示SDMA任务
31 :
32 : constexpr uint32_t TASK_CONTEXT_SIZE = 50; // task 执行失败时打印前序task信息的数量
33 : constexpr uint32_t TASK_CONTEXT_INFO_SIZE
34 : = LOG_TMPBUF_SIZE - TASK_CONTEXT_SIZE; // task 执行失败时打印前序task信息的长度限制
35 : constexpr u32 MAX_NAME_LEN = 64;
36 : constexpr u32 TASK_ID_SHIFT_BITS = 16;
37 :
38 109 : HcclCommTaskExceptionLite& HcclCommTaskExceptionLite::GetInstance()
39 : {
40 109 : static HcclCommTaskExceptionLite instance; // aicpu侧一个dev一个进程，不需要按dev区分单例对象
41 109 : return instance;
42 : }
43 :
44 56 : void HcclCommTaskExceptionLite::Init(u32 devId)
45 : {
46 56 : devId_ = devId;
47 56 : HCCL_INFO("[%s]success, devId_[%u]", __func__, devId_);
48 56 : }
49 :
50 1 : void HcclCommTaskExceptionLite::Call()
51 : {
52 1 : if (stopCall_ == true) {
53 0 : return;
54 : }
55 :
56 1 : HcclResult ret = HandleExceptionCqe();
57 1 : if (ret != HCCL_SUCCESS) {
58 0 : stopCall_ = true;
59 0 : HCCL_ERROR(
60 : "[%s]HandleExceptionCqe fail, set stopCall_[%d]", __func__, stopCall_); // 函数调用失败，停止调用避免刷屏
61 : }
62 : }
63 :
64 6 : HcclResult HcclCommTaskExceptionLite::IsHandleDpuStop(uint8_t* taskexceptionVa, bool& isStop)
65 : {
66 6 : uint8_t stopSignal = 0;
67 6 : errno_t ret = memcpy_s(
68 : &stopSignal, sizeof(stopSignal), taskexceptionVa,
69 : sizeof(stopSignal)); // 读标志位,第1字节，存放host侧发送是否停止的信号。
70 6 : if (ret != EOK) {
71 0 : HCCL_ERROR("[HcclCommTaskExceptionLite::%s] memcpy_s failed on flag, return[%d].", __func__, ret);
72 0 : return HCCL_E_MEMORY;
73 : }
74 6 : if (stopSignal == 1) {
75 1 : isStop = true;
76 1 : stopSignal = 0;
77 1 : ret = memcpy_s(
78 : taskexceptionVa, sizeof(stopSignal), &stopSignal,
79 : sizeof(stopSignal)); // 读标志位,第1字节，存放host侧发送是否停止的信号。
80 1 : if (ret != EOK) {
81 0 : HCCL_ERROR("[HcclCommTaskExceptionLite::%s] memcpy_s failed on flag, return[%d].", __func__, ret);
82 0 : return HCCL_E_MEMORY;
83 : }
84 : }
85 6 : return HCCL_SUCCESS;
86 : }
87 :
88 8 : HcclResult HcclCommTaskExceptionLite::HandleDpuTaskexception(CollCommAicpu* aicpuComm)
89 : {
90 : // 轮询taskexception共享内存
91 8 : auto commId = aicpuComm->GetIdentifier();
92 8 : std::lock_guard<std::mutex> lock(g_taskExpDevMemMapMutex);
93 8 : auto it = g_taskExpDevMemMap.find(commId);
94 8 : if (it == g_taskExpDevMemMap.end()) {
95 1 : return HCCL_SUCCESS; // 非dpu场景，map为空
96 : }
97 :
98 7 : auto taskexceptionVa = reinterpret_cast<uint8_t*>(it->second);
99 7 : if (taskexceptionVa == nullptr) {
100 1 : return HCCL_SUCCESS;
101 : }
102 : // 查是否要停止
103 6 : bool isStop = false;
104 6 : CHK_RET(IsHandleDpuStop(taskexceptionVa, isStop));
105 6 : if (isStop) {
106 1 : it->second = nullptr;
107 1 : return HCCL_SUCCESS;
108 : }
109 : // 查是否有错误
110 5 : uint16_t flag = 0;
111 10 : errno_t ret = memcpy_s(
112 5 : &flag, sizeof(flag), taskexceptionVa + sizeof(uint8_t), sizeof(flag)); // 读标志位,第2-3字节，存放HcclResult。
113 5 : if (ret != EOK) {
114 0 : HCCL_ERROR("[HcclCommTaskExceptionLite::%s] memcpy_s failed on flag, return[%d].", __func__, ret);
115 0 : return HCCL_E_MEMORY;
116 : }
117 5 : if (flag != 0) {
118 : // 触发taskexception
119 4 : HCCL_ERROR(
120 : "[HcclCommTaskExceptionLite][DPU] taskexceptionVa[%p], errorCode[%d], devId[%u], commId[%s]",
121 : taskexceptionVa, flag, aicpuComm->GetDevId(), commId.c_str());
122 : // 1、取notify，并构造rtLogicCqReport_t
123 4 : auto* hcclCommDfxLite = aicpuComm->GetHcclCommDfxLite();
124 7 : CHK_PTR_NULL(hcclCommDfxLite);
125 4 : const auto curDfxOpInfo = static_cast<const Hccl::DfxDfxOpInfo*>(hcclCommDfxLite->GetLatestDfxOpInfo());
126 4 : CHK_PTR_NULL(curDfxOpInfo);
127 1 : u32 notifyId = curDfxOpInfo->cpuWaitAicpuNotifyId;
128 1 : rtLogicCqReport_t exceptionInfo{};
129 : // 2、调用SendTaskExceptionByMBox触发taskexception回调
130 1 : CHK_RET(SendTaskExceptionByMBox(notifyId, 0, exceptionInfo));
131 : // 3、标志位置0
132 1 : ret = memset_s(taskexceptionVa + sizeof(uint8_t), sizeof(uint16_t), 0, sizeof(uint16_t)); // 标志位置0
133 1 : if (ret != EOK) {
134 0 : HCCL_ERROR("[HcclCommTaskExceptionLite::%s] memset_s failed on flag, return[%d].", __func__, ret);
135 0 : return HCCL_E_MEMORY;
136 : }
137 : }
138 2 : return HCCL_SUCCESS;
139 8 : }
140 :
141 1 : HcclResult HcclCommTaskExceptionLite::HandleExceptionCqe()
142 : {
143 1 : std::shared_lock<std::shared_mutex> rwlock(CollCommAicpuMgr::GetInstance().GetMutex());
144 :
145 1 : std::vector<std::pair<std::string, CollCommAicpu*>> aicpuCommInfo;
146 1 : CHK_RET(CollCommAicpuMgr::GetInstance().GetAllComms(aicpuCommInfo));
147 :
148 2 : for (auto& commInfo : aicpuCommInfo) {
149 1 : CollCommAicpu* aicpuComm = commInfo.second;
150 1 : CHK_PTR_NULL(aicpuComm);
151 :
152 1 : if ((aicpuComm->GetCommmStatus() == HcclCommStatus::HCCL_COMM_STATUS_INVALID)
153 1 : || (aicpuComm->GetCommmStatus() == HcclCommStatus::HCCL_COMM_STATUS_SUSPENDING)) {
154 1 : continue;
155 : }
156 0 : CHK_RET(HandleDpuTaskexception(aicpuComm)); // dpu taskexception
157 :
158 0 : std::shared_lock<std::shared_mutex> threadRwlock(aicpuComm->GetCommEngineResMgr()->GetThreadMutex());
159 0 : const std::vector<std::shared_ptr<hccl::Thread>> threads = aicpuComm->GetCommEngineResMgr()->GetAllThread();
160 0 : for (auto thread : threads) {
161 : rtLogicCqReport_t cqeException;
162 0 : dfx::CqeStatus cqeStatus = dfx::CqeStatus::kDefault;
163 0 : Hccl::StreamLite* streamLite = static_cast<Hccl::StreamLite*>(thread->GetStreamLitePtr());
164 0 : CHK_PTR_NULL(streamLite);
165 :
166 0 : HcclResult ret = GetThreadCqe(thread.get(), cqeException, cqeStatus);
167 0 : CHK_PRT_RET(
168 : ret != HCCL_SUCCESS,
169 : HCCL_ERROR(
170 : "[%s]GetThreadCqe fail, aicpuComm[%s], streamId[%u]", __func__, aicpuComm->GetIdentifier().c_str(),
171 : streamLite->GetId()),
172 : ret);
173 :
174 0 : ret = ProcessCqe(aicpuComm, cqeException, cqeStatus, aicpuCommInfo);
175 0 : CHK_PRT_RET(
176 : ret != HCCL_SUCCESS,
177 : HCCL_ERROR(
178 : "[%s]ProcessCqe fail, aicpuComm[%s], streamId[%u], "
179 : "cqeStatus[%lld]",
180 : __func__, aicpuComm->GetIdentifier().c_str(), streamLite->GetId(),
181 : static_cast<long long>(cqeStatus)),
182 : ret);
183 0 : }
184 0 : }
185 1 : return HCCL_SUCCESS;
186 1 : }
187 :
188 1 : HcclResult HcclCommTaskExceptionLite::PrintAllCommTaskException()
189 : {
190 1 : std::shared_lock<std::shared_mutex> rwlock(CollCommAicpuMgr::GetInstance().GetMutex());
191 :
192 1 : std::vector<std::pair<std::string, CollCommAicpu*>> aicpuCommInfo;
193 1 : CHK_RET(CollCommAicpuMgr::GetInstance().GetAllComms(aicpuCommInfo));
194 :
195 1 : HCCL_RUN_INFO("[TaskException][AICPU]%s start, comm size[%u]", __func__, aicpuCommInfo.size());
196 1 : HcclResult ret = HCCL_SUCCESS;
197 2 : for (auto& commInfo : aicpuCommInfo) {
198 1 : CollCommAicpu* aicpuComm = commInfo.second;
199 1 : HcclResult pRet = PrintCommTaskException(aicpuComm);
200 1 : CHK_PRT_CONT(
201 : pRet != HCCL_SUCCESS,
202 : HCCL_ERROR("PrintCommTaskException fail, comm[%s]", aicpuComm->GetIdentifier().c_str()));
203 1 : ret = (pRet != HCCL_SUCCESS) ? pRet : ret;
204 : }
205 1 : HCCL_RUN_INFO("[TaskException][AICPU]%s end, ret[%d]", __func__, ret);
206 1 : return ret;
207 1 : }
208 :
209 2 : HcclResult HcclCommTaskExceptionLite::PrintCommTaskException(CollCommAicpu* aicpuComm)
210 : {
211 2 : CHK_PTR_NULL(aicpuComm);
212 2 : HcclResult ret = HCCL_SUCCESS;
213 2 : HCCL_RUN_INFO("[TaskException][AICPU]%s comm[%s] start", __func__, aicpuComm->GetIdentifier().c_str());
214 2 : std::shared_lock<std::shared_mutex> threadRwlock(aicpuComm->GetCommEngineResMgr()->GetThreadMutex());
215 2 : const std::vector<std::shared_ptr<hccl::Thread>> threads = aicpuComm->GetCommEngineResMgr()->GetAllThread();
216 3 : for (auto thread : threads) {
217 1 : CHK_SMART_PTR_NULL(thread);
218 1 : Hccl::StreamLite* streamLite = static_cast<Hccl::StreamLite*>(thread->GetStreamLitePtr());
219 1 : CHK_PTR_NULL(streamLite);
220 1 : u32 sqHead = 0U;
221 1 : u32 sqTail = 0U;
222 1 : ret = QuerySqStatus(devId_, streamLite->GetSqId(), sqHead, sqTail);
223 1 : if (ret != HCCL_SUCCESS || sqHead == sqTail) { // 此流为空时，不打印
224 0 : HCCL_RUN_INFO(
225 : "[TaskException][AICPU]PrintTaskExceptionBySqeId skip, "
226 : "QuerySqStatus ret[%d], aicpuComm[%s], sqId[%u], sqHead[%u], sqTail[%u]",
227 : ret, aicpuComm->GetIdentifier().c_str(), streamLite->GetSqId(), sqHead, sqTail);
228 0 : continue;
229 : }
230 1 : uint16_t streamId = 0;
231 1 : uint16_t taskId = 0;
232 1 : streamLite->GetRtsq()->GetStreamIdAndTaskIdBySqIdx(sqHead, streamId, taskId);
233 1 : const u32 sqeId = GetSqeId(taskId, streamId);
234 1 : HcclResult pRet = PrintTaskExceptionBySqeId(aicpuComm, streamLite->GetSqId(), sqeId);
235 1 : CHK_PRT_CONT(
236 : pRet != HCCL_SUCCESS, HCCL_ERROR(
237 : "PrintTaskExceptionBySqeId fail, comm[%s], sqId[%u], sqeId[%u]",
238 : aicpuComm->GetIdentifier().c_str(), streamLite->GetSqId(), sqeId));
239 1 : ret = (pRet != HCCL_SUCCESS) ? pRet : ret;
240 1 : }
241 2 : HCCL_RUN_INFO("[TaskException][AICPU]%s comm[%s] end, ret[%d]", __func__, aicpuComm->GetIdentifier().c_str(), ret);
242 2 : return ret;
243 2 : }
244 :
245 0 : HcclResult HcclCommTaskExceptionLite::GetThreadCqe(
246 : hccl::Thread* thread, rtLogicCqReport_t& cqeException, dfx::CqeStatus& cqeStatus)
247 : {
248 0 : CHK_SMART_PTR_NULL(thread);
249 0 : Hccl::StreamLite* streamLite = static_cast<Hccl::StreamLite*>(thread->GetStreamLitePtr());
250 0 : CHK_PTR_NULL(streamLite);
251 :
252 0 : constexpr u32 reportSize = MAX_REPORT_CNT;
253 : rtLogicCqReport_t streamReport[reportSize];
254 :
255 : CqeQueryInput cqeQueryInput;
256 0 : cqeQueryInput.devId = devId_;
257 0 : cqeQueryInput.streamId = streamLite->GetId();
258 0 : cqeQueryInput.sqId = streamLite->GetSqId();
259 0 : cqeQueryInput.cqId = streamLite->GetCqId();
260 0 : cqeQueryInput.type = static_cast<uint32_t>(DRV_LOGIC_TYPE);
261 0 : cqeQueryInput.cqeAddr = reinterpret_cast<uint8_t*>(streamReport);
262 :
263 0 : cqeStatus = CqReportRecv(cqeQueryInput, cqeException);
264 0 : if (cqeStatus == dfx::CqeStatus::kCqeInnerError) {
265 0 : HCCL_ERROR("[%s]CqReportRecv fail, CqeQueryInput:%s", __func__, cqeQueryInput.ToString().c_str());
266 0 : return HCCL_E_INTERNAL;
267 : }
268 0 : return HCCL_SUCCESS;
269 : }
270 :
271 1 : HcclResult HcclCommTaskExceptionLite::ProcessCqe(
272 : CollCommAicpu* aicpuComm, const rtLogicCqReport_t& exceptionInfo, const CqeStatus& cqeStatus,
273 : const std::vector<std::pair<std::string, CollCommAicpu*>>& aicpuCommInfo)
274 : {
275 1 : if (cqeStatus == dfx::CqeStatus::kDefault) {
276 1 : return HCCL_SUCCESS;
277 : }
278 :
279 0 : if (hcomm::GetTaskExceptionEnable() == false) {
280 0 : HCCL_ERROR("[TaskException][AICPU]taskException enable is false, skip print taskException");
281 0 : return HCCL_SUCCESS;
282 : }
283 :
284 0 : HcclResult ret = HCCL_SUCCESS;
285 0 : const u32 sqeId = GetSqeId(exceptionInfo.taskId, exceptionInfo.streamId);
286 0 : ret = PrintTaskExceptionBySqeId(aicpuComm, exceptionInfo.sqId, sqeId);
287 0 : CHK_PRT_CONT(
288 : ret != HCCL_SUCCESS, HCCL_ERROR(
289 : "[PrintTaskExceptionBySqeId]fail, ret[%d], group[%s], sqId[%u], taskId[%u]", ret,
290 : aicpuComm->GetIdentifier().c_str(), exceptionInfo.sqId,
291 : exceptionInfo.taskId)); // 如果上报失败，继续打印taskException
292 :
293 0 : ret = ReportErrMsg(aicpuComm, exceptionInfo);
294 0 : CHK_PRT_CONT(
295 : ret != HCCL_SUCCESS,
296 : HCCL_ERROR(
297 : "[ReportErrMsg]fail, ret[%d], group[%s], sqId[%u], taskId[%u]", ret, aicpuComm->GetIdentifier().c_str(),
298 : exceptionInfo.sqId, exceptionInfo.taskId)); // 如果上报失败，继续打印taskException
299 :
300 : // notify超时场景：step1 打印当前流信息；step2 打印当前通信域信息；step3 打印其他通信域信息
301 0 : if (cqeStatus == dfx::CqeStatus::kCqeException && exceptionInfo.sqeType == RT_STARS_SQE_TYPE_PLACE_HOLDER) {
302 0 : CHK_RET(PrintCommTaskException(aicpuComm));
303 0 : for (auto& commInfo : aicpuCommInfo) {
304 0 : CollCommAicpu* comm = commInfo.second;
305 0 : if (comm != nullptr && comm->GetIdentifier() != aicpuComm->GetIdentifier()) {
306 0 : CHK_RET(PrintCommTaskException(comm));
307 : }
308 : }
309 : }
310 0 : return ret;
311 : }
312 :
313 4 : u32 HcclCommTaskExceptionLite::GetSqeId(uint16_t taskId, uint16_t streamId)
314 : {
315 4 : return (static_cast<u32>(taskId) << TASK_ID_SHIFT_BITS) | static_cast<u32>(streamId);
316 : }
317 :
318 4 : HcclResult HcclCommTaskExceptionLite::ReportErrMsg(CollCommAicpu* aicpuComm, const rtLogicCqReport_t& exceptionInfo)
319 : {
320 4 : CHK_PTR_NULL(aicpuComm);
321 :
322 3 : const u32 sqeId = GetSqeId(exceptionInfo.taskId, exceptionInfo.streamId);
323 3 : HCCL_INFO(
324 : "[%s]group[%s], sqeId[0x%x], taskId[%u], streamId[%u].", __func__, aicpuComm->GetIdentifier().c_str(), sqeId,
325 : exceptionInfo.taskId, exceptionInfo.streamId);
326 :
327 3 : Hccl::DfxTaskInfo* curTask = FindDfxTaskInfo(aicpuComm, exceptionInfo.sqId, sqeId);
328 3 : CHK_PTR_NULL(curTask);
329 :
330 2 : const Hccl::DfxDfxOpInfo* opInfo = (curTask->dfxOpInfo != DFX_INVALID_U64) ?
331 1 : reinterpret_cast<const Hccl::DfxDfxOpInfo*>(curTask->dfxOpInfo) :
332 : nullptr;
333 2 : CHK_PTR_NULL(opInfo);
334 :
335 1 : if (!aicpuComm->IsErrorReported()) {
336 0 : Hccl::ErrorMessageReport errMsgInfo{};
337 0 : CHK_RET(GenerateErrorMessageReport(aicpuComm, *curTask, exceptionInfo, errMsgInfo));
338 0 : CHK_RET(aicpuComm->SendErrorMessageReportToHost(errMsgInfo));
339 :
340 0 : u32 notifyId = opInfo->cpuWaitAicpuNotifyId;
341 0 : CHK_RET(SendTaskExceptionByMBox(notifyId, 0, exceptionInfo));
342 0 : aicpuComm->SetErrorReported(true);
343 : }
344 1 : return HCCL_SUCCESS;
345 : }
346 :
347 1 : HcclResult HcclCommTaskExceptionLite::PrintTaskExceptionBySqeId(CollCommAicpu* aicpuComm, u32 sqId, u32 sqeId)
348 : {
349 1 : CHK_PTR_NULL(aicpuComm);
350 :
351 : // 已经打印过的不再重复打印
352 1 : Hccl::DfxTaskInfo* curTask = FindDfxTaskInfo(aicpuComm, sqId, sqeId);
353 1 : CHK_PTR_NULL(curTask);
354 :
355 1 : auto it = threadsPrinted_.find(sqId);
356 1 : if (it != threadsPrinted_.end() && it->second == sqeId) {
357 0 : HCCL_RUN_INFO("[TaskException][AICPU]sqId:%u, sqeId:%u has been printed, skip", sqId, sqeId);
358 0 : return HCCL_SUCCESS;
359 : }
360 1 : threadsPrinted_[sqId] = sqeId;
361 :
362 1 : u32 sqHead = 0U;
363 1 : u32 sqTail = 0U;
364 1 : (void)QuerySqStatus(devId_, sqId, sqHead, sqTail);
365 :
366 1 : HCCL_ERROR(
367 : "[TaskException][AICPU]base information is streamID(sqId):[%u], taskID(sqeId):[%u], taskType:[%u], "
368 : "sqHead:%u, sqTail:%u",
369 : curTask->sqId, curTask->taskId, curTask->taskType, sqHead, sqTail);
370 :
371 1 : PrintEid(*curTask);
372 :
373 1 : HCCL_ERROR("[TaskException][AICPU]group information is %s.", GetGroupInfo(aicpuComm).c_str());
374 :
375 1 : if (curTask->taskType != static_cast<u8>(Hccl::TaskParamTypeVal::TASK_NOTIFY_WAIT)) {
376 1 : PrintOpDataInfo(curTask);
377 : } else {
378 0 : CHK_RET(PrintTaskContextInfo(aicpuComm, sqId, sqeId));
379 : }
380 1 : return HCCL_SUCCESS;
381 : }
382 :
383 1 : HcclResult HcclCommTaskExceptionLite::GenerateErrorMessageReport(
384 : CollCommAicpu* aicpuComm, const Hccl::DfxTaskInfo& taskInfo, const rtLogicCqReport_t& exceptionInfo,
385 : Hccl::ErrorMessageReport& errMsgInfo)
386 : {
387 1 : const Hccl::DfxDfxOpInfo* opInfo = (taskInfo.dfxOpInfo != DFX_INVALID_U64) ?
388 0 : reinterpret_cast<const Hccl::DfxDfxOpInfo*>(taskInfo.dfxOpInfo) :
389 : nullptr;
390 1 : CHK_PTR_NULL(opInfo);
391 :
392 0 : errMsgInfo.remoteUserRank = GetRemoteRankId(taskInfo);
393 0 : errMsgInfo.streamId = taskInfo.sqId;
394 0 : errMsgInfo.taskId = taskInfo.taskId;
395 0 : errMsgInfo.rankId = aicpuComm->GetTopoInfo().userRank;
396 0 : errMsgInfo.rankSize = aicpuComm->GetTopoInfo().userRankSize;
397 :
398 0 : errMsgInfo.opIndex = opInfo->opIndex;
399 0 : errMsgInfo.opType = opInfo->opType;
400 0 : errMsgInfo.count = opInfo->count;
401 0 : errMsgInfo.dataType = opInfo->dataType;
402 0 : errMsgInfo.srcAddr = opInfo->srcAddr;
403 0 : errMsgInfo.dstAddr = opInfo->dstAddr;
404 0 : CHK_SAFETY_FUNC_RET(memcpy_s(
405 : errMsgInfo.tag, sizeof(errMsgInfo.tag), opInfo->algTag, strnlen(opInfo->algTag, sizeof(opInfo->algTag))));
406 :
407 0 : errMsgInfo.taskType = Hccl::TaskParamType(static_cast<Hccl::TaskParamType::Value>(taskInfo.taskType));
408 0 : errMsgInfo.rtCqErrorType = exceptionInfo.errorType;
409 0 : errMsgInfo.rtCqErrorCode = exceptionInfo.errorCode;
410 :
411 0 : CHK_SAFETY_FUNC_RET(memcpy_s(
412 : errMsgInfo.group, sizeof(errMsgInfo.group), aicpuComm->GetIdentifier().c_str(),
413 : aicpuComm->GetIdentifier().size() + 1));
414 :
415 0 : GenerateTaskErrMsg(taskInfo, errMsgInfo, exceptionInfo);
416 0 : return HCCL_SUCCESS;
417 : }
418 :
419 9 : void HcclCommTaskExceptionLite::GenerateTaskErrMsg(
420 : const Hccl::DfxTaskInfo& taskInfo, Hccl::ErrorMessageReport& errMsgInfo, const rtLogicCqReport_t& exceptionInfo)
421 : {
422 9 : auto taskType = static_cast<Hccl::TaskParamTypeVal>(taskInfo.taskType);
423 9 : switch (taskType) {
424 2 : case Hccl::TaskParamTypeVal::TASK_NOTIFY_WAIT:
425 : case Hccl::TaskParamTypeVal::TASK_NOTIFY_RECORD:
426 2 : FillNotifyErrMsg(taskInfo, errMsgInfo);
427 2 : break;
428 2 : case Hccl::TaskParamTypeVal::TASK_UB_REDUCE_INLINE:
429 : case Hccl::TaskParamTypeVal::TASK_WRITE_REDUCE_WITH_NOTIFY:
430 2 : FillReduceErrMsg(taskInfo, errMsgInfo, exceptionInfo);
431 2 : break;
432 1 : case Hccl::TaskParamTypeVal::TASK_REDUCE_INLINE:
433 1 : FillReduceInlineErrMsg(taskInfo, errMsgInfo);
434 1 : break;
435 2 : case Hccl::TaskParamTypeVal::TASK_UB_INLINE_WRITE:
436 : case Hccl::TaskParamTypeVal::TASK_WRITE_WITH_NOTIFY:
437 2 : FillDmaErrMsg(taskInfo, errMsgInfo, exceptionInfo);
438 2 : break;
439 1 : case Hccl::TaskParamTypeVal::TASK_UB:
440 1 : FillUbErrMsg(taskInfo, errMsgInfo, exceptionInfo);
441 1 : break;
442 1 : case Hccl::TaskParamTypeVal::TASK_SDMA:
443 1 : FillSdmaErrMsg(taskInfo, errMsgInfo);
444 1 : break;
445 0 : default:
446 0 : HCCL_ERROR("[TaskException][AICPU]%s taskType[%d] is not support", __func__, taskInfo.taskType);
447 0 : return;
448 : }
449 : }
450 :
451 2 : void HcclCommTaskExceptionLite::FillNotifyErrMsg(
452 : const Hccl::DfxTaskInfo& taskInfo, Hccl::ErrorMessageReport& errMsgInfo)
453 : {
454 2 : void* sqePtr = reinterpret_cast<void*>(taskInfo.taskPara.Notify.sqeAddr);
455 2 : if (sqePtr != nullptr) {
456 2 : auto* header = reinterpret_cast<Hccl::Rt91095StarsSqeHeader*>(sqePtr);
457 2 : if (static_cast<Hccl::Rt91095StarsSqeType>(header->type)
458 2 : == Hccl::Rt91095StarsSqeType::RT_91095_SQE_TYPE_NOTIFY_RECORD
459 1 : || static_cast<Hccl::Rt91095StarsSqeType>(header->type)
460 1 : == Hccl::Rt91095StarsSqeType::RT_91095_SQE_TYPE_NOTIFY_WAIT) {
461 2 : auto* notifySqe = reinterpret_cast<Hccl::Rt91095StarsNotifySqe*>(sqePtr);
462 2 : errMsgInfo.notifyId = notifySqe->notifyId;
463 2 : errMsgInfo.notifyValue = notifySqe->cntValue;
464 : }
465 : }
466 2 : }
467 :
468 2 : void HcclCommTaskExceptionLite::FillReduceErrMsg(
469 : const Hccl::DfxTaskInfo& taskInfo, Hccl::ErrorMessageReport& errMsgInfo, const rtLogicCqReport_t& exceptionInfo)
470 : {
471 2 : errMsgInfo.reduceType = taskInfo.taskPara.Reduce.reduceOp;
472 2 : errMsgInfo.notifyId = taskInfo.taskPara.Reduce.notifyId;
473 2 : errMsgInfo.notifyValue = INVALID_U32;
474 2 : GetEidFromChannelHandle(taskInfo, errMsgInfo.locEid, errMsgInfo.rmtEid);
475 2 : errMsgInfo.ubCqeStatus = exceptionInfo.errorCode & 0xFF;
476 2 : errMsgInfo.linkType = Hccl::DfxLinkType(static_cast<Hccl::DfxLinkType::Value>(taskInfo.linkType));
477 2 : errMsgInfo.size = taskInfo.taskPara.Reduce.size;
478 2 : errMsgInfo.taskSrcAddr = taskInfo.taskPara.Reduce.srcAddr;
479 2 : errMsgInfo.taskDstAddr = taskInfo.taskPara.Reduce.dstAddr;
480 2 : HCCL_ERROR(
481 : "[TaskException][AICPU]ubCqeStatus[%u], localEid[%s], remoteEid[%s]. ", errMsgInfo.ubCqeStatus,
482 : errMsgInfo.locEid.Describe().c_str(), errMsgInfo.rmtEid.Describe().c_str());
483 2 : }
484 :
485 2 : void HcclCommTaskExceptionLite::FillDmaErrMsg(
486 : const Hccl::DfxTaskInfo& taskInfo, Hccl::ErrorMessageReport& errMsgInfo, const rtLogicCqReport_t& exceptionInfo)
487 : {
488 2 : errMsgInfo.notifyId = taskInfo.taskPara.ubDma.notifyId;
489 2 : errMsgInfo.notifyValue = INVALID_U32;
490 2 : FillUbErrMsg(taskInfo, errMsgInfo, exceptionInfo);
491 2 : }
492 :
493 1 : void HcclCommTaskExceptionLite::FillSdmaErrMsg(const Hccl::DfxTaskInfo& taskInfo, Hccl::ErrorMessageReport& errMsgInfo)
494 : {
495 1 : errMsgInfo.linkType = Hccl::DfxLinkType(static_cast<Hccl::DfxLinkType::Value>(taskInfo.linkType));
496 1 : void* sqePtr = reinterpret_cast<void*>(taskInfo.taskPara.Dma.sqeAddr);
497 1 : if (sqePtr != nullptr) {
498 0 : auto* header = reinterpret_cast<Hccl::Rt91095StarsSqeHeader*>(sqePtr);
499 0 : if (static_cast<Hccl::Rt91095StarsSqeType>(header->type) == Hccl::Rt91095StarsSqeType::RT_91095_SQE_TYPE_SDMA) {
500 0 : auto* dmaSqe = reinterpret_cast<Hccl::Rt91095StarsMemcpySqe*>(sqePtr);
501 : errMsgInfo.taskSrcAddr
502 0 : = (static_cast<u64>(dmaSqe->u.strideMode0.srcAddrHigh) << 32) | dmaSqe->u.strideMode0.srcAddrLow;
503 : errMsgInfo.taskDstAddr
504 0 : = (static_cast<u64>(dmaSqe->u.strideMode0.dstAddrHigh) << 32) | dmaSqe->u.strideMode0.dstAddrLow;
505 0 : errMsgInfo.size = dmaSqe->u.strideMode0.lengthMove;
506 : }
507 : }
508 1 : }
509 :
510 3 : void HcclCommTaskExceptionLite::FillUbErrMsg(
511 : const Hccl::DfxTaskInfo& taskInfo, Hccl::ErrorMessageReport& errMsgInfo, const rtLogicCqReport_t& exceptionInfo)
512 : {
513 3 : GetEidFromChannelHandle(taskInfo, errMsgInfo.locEid, errMsgInfo.rmtEid);
514 3 : errMsgInfo.ubCqeStatus = exceptionInfo.errorCode & 0xFF;
515 3 : errMsgInfo.linkType = Hccl::DfxLinkType(static_cast<Hccl::DfxLinkType::Value>(taskInfo.linkType));
516 3 : errMsgInfo.size = taskInfo.taskPara.ubDma.size;
517 3 : errMsgInfo.taskSrcAddr = taskInfo.taskPara.ubDma.srcAddr;
518 3 : errMsgInfo.taskDstAddr = taskInfo.taskPara.ubDma.dstAddr;
519 3 : HCCL_ERROR(
520 : "[TaskException][AICPU]ubCqeStatus[%u], localEid[%s], remoteEid[%s]. ", errMsgInfo.ubCqeStatus,
521 : errMsgInfo.locEid.Describe().c_str(), errMsgInfo.rmtEid.Describe().c_str());
522 3 : }
523 :
524 1 : void HcclCommTaskExceptionLite::FillReduceInlineErrMsg(
525 : const Hccl::DfxTaskInfo& taskInfo, Hccl::ErrorMessageReport& errMsgInfo)
526 : {
527 1 : errMsgInfo.reduceType = taskInfo.taskPara.Reduce.reduceOp;
528 1 : }
529 :
530 4 : HcclResult HcclCommTaskExceptionLite::SendTaskExceptionByMBox(
531 : const u32 notifyId, const u32 tsId, const rtLogicCqReport_t& exceptionInfo)
532 : {
533 4 : ts_aicpu_msg_info_t aicpuSqe = {};
534 4 : u32 hostpid = 0;
535 4 : u32 vfId = 0;
536 4 : int pid = getpid();
537 4 : HCCL_INFO("[%s]getpid[%d]", __func__, pid);
538 : // 调整drvQueryProcessHostPid获取pid和vf_id的值
539 4 : CHK_RET(HrtHalDrvQueryProcessHostPid(pid, nullptr, &vfId, &hostpid, nullptr));
540 :
541 4 : aicpuSqe.pid = hostpid;
542 4 : aicpuSqe.cmd_type = TS_AICPU_RECORD;
543 4 : aicpuSqe.vf_id = vfId;
544 4 : aicpuSqe.tid = 0U; // notify is no need tid
545 4 : aicpuSqe.u.aicpu_record.record_type = AICPU_MSG_NOTIFY_RECORD_V2;
546 4 : aicpuSqe.u.aicpu_record.record_id = notifyId;
547 4 : aicpuSqe.ts_id = static_cast<uint8_t>(tsId);
548 4 : aicpuSqe.u.aicpu_record.fault_task_id = 0xffffffff;
549 :
550 4 : if (exceptionInfo.sqeType == ubSqeType) {
551 1 : aicpuSqe.u.aicpu_record.ret_code = SwitchUBCqeErrCodeToTsErrCode(exceptionInfo.errorCode & 0xFF);
552 3 : } else if (exceptionInfo.sqeType == sdmaSqeType) {
553 1 : aicpuSqe.u.aicpu_record.ret_code = SwitchSdmaCqeErrCodeToTsErrCode(exceptionInfo.errorCode);
554 : } else {
555 2 : aicpuSqe.u.aicpu_record.ret_code = TS_ERROR_HCCL_OTHER_ERROR;
556 : }
557 :
558 : struct event_summary event;
559 4 : event.dst_engine = TS_CPU;
560 4 : event.policy = ONLY;
561 4 : event.pid = 0;
562 4 : event.grp_id = 0;
563 4 : event.event_id = EVENT_TS_CTRL_MSG;
564 4 : event.subevent_id = 0U;
565 4 : event.msg_len = static_cast<uint32_t>(sizeof(ts_aicpu_msg_info_t));
566 4 : event.msg = reinterpret_cast<char_t*>(&aicpuSqe);
567 4 : drvError_t ret = Hccl::DlHalFunctionV2::GetInstance().dlHalEschedSubmitEvent(devId_, &event);
568 4 : if (ret != DRV_ERROR_NONE) {
569 0 : HCCL_ERROR(
570 : "[%s]dlHalEschedSubmitEvent failed, ret=%d, notifyId=%u, hostpid=%u, vfId=%u, tsId=%u", __func__, ret,
571 : notifyId, hostpid, vfId, tsId);
572 0 : return HCCL_E_DRV;
573 : }
574 4 : HCCL_RUN_INFO(
575 : "[%s]finished, notifyId=%u, hostpid=%u, vfId=%u, tsId=%u, errorType=%u, errorCode=%u, ret_code=%u", __func__,
576 : notifyId, hostpid, vfId, tsId, exceptionInfo.errorType, exceptionInfo.errorCode,
577 : aicpuSqe.u.aicpu_record.ret_code);
578 4 : return HCCL_SUCCESS;
579 : }
580 :
581 : // 把UB类错误码转换成Ts对应的错误码
582 5 : uint16_t HcclCommTaskExceptionLite::SwitchUBCqeErrCodeToTsErrCode(u32 cqeErrCode)
583 : {
584 5 : switch (cqeErrCode) {
585 2 : case RT_UB_LOCAL_OPERATIOINERR:
586 2 : return TS_ERROR_HCCL_OP_UB_DDRC_FAILED;
587 1 : case RT_UB_REMOTE_OPERATIOINERR:
588 1 : return TS_ERROR_HCCL_OP_UB_POISON_FAILED;
589 1 : case RT_UB_LINK_FAILEDERR:
590 1 : return TS_ERROR_HCCL_OP_UB_LINK_FAILED;
591 1 : default:
592 1 : return TS_ERROR_HCCL_OTHER_ERROR;
593 : }
594 : }
595 :
596 : // 把SDMA类错误码转换成Ts对应的错误码
597 5 : uint16_t HcclCommTaskExceptionLite::SwitchSdmaCqeErrCodeToTsErrCode(u32 cqeErrCode)
598 : {
599 5 : switch (cqeErrCode) {
600 2 : case RT_SDMA_COMPERR:
601 2 : return TS_ERROR_SDMA_LINK_ERROR;
602 1 : case RT_SDMA_COMPDATAERR:
603 1 : return TS_ERROR_SDMA_POISON_ERROR;
604 1 : case RT_SDMA_DATAERR:
605 1 : return TS_ERROR_SDMA_DDRC_ERROR;
606 1 : default:
607 1 : return TS_ERROR_HCCL_OTHER_ERROR;
608 : }
609 : }
610 :
611 2 : HcclResult HcclCommTaskExceptionLite::CollectTaskContext(
612 : CollCommAicpu* aicpuComm, u32 sqId, u32 taskId, std::vector<Hccl::DfxTaskInfo*>& taskContext)
613 : {
614 2 : Hccl::TaskInfoCircularQueue* queue = GetTaskQueueBySqId(aicpuComm, sqId);
615 2 : CHK_PRT_RET(
616 : queue == nullptr, HCCL_ERROR("[%s]GetTaskQueueBySqId nullptr, devId[%u], sqId[%u].", __func__, devId_, sqId),
617 : HCCL_E_PARA);
618 :
619 0 : if (queue->IsEmpty()) {
620 0 : HCCL_ERROR("[%s]queue is empty, devId[%u], sqId[%u].", __func__, devId_, sqId);
621 0 : return HCCL_E_PARA;
622 : }
623 :
624 0 : u32 targetTaskId = taskId;
625 0 : u16 begin = queue->GetBegin();
626 0 : Hccl::DfxTaskInfo* found = nullptr;
627 0 : u16 foundIdx = 0;
628 0 : for (u16 idx = 0; idx < queue->GetCapacity(); idx++) {
629 0 : Hccl::DfxTaskInfo* slot = queue->GetSlot(idx);
630 0 : if (slot != nullptr && slot->taskId == targetTaskId) {
631 0 : found = slot;
632 0 : foundIdx = idx;
633 0 : break;
634 : }
635 : }
636 0 : CHK_PRT_RET(
637 : found == nullptr,
638 : HCCL_ERROR("[%s]exception task not found, devId[%u], sqId[%u], taskId[%u]", __func__, devId_, sqId, taskId),
639 : HCCL_E_PARA);
640 :
641 0 : u32 ctxCount = 0;
642 0 : for (u16 idx = foundIdx; ctxCount < TASK_CONTEXT_SIZE; ++ctxCount) {
643 0 : if (idx == begin) {
644 0 : break;
645 : }
646 0 : idx = (idx == 0) ? static_cast<u16>(queue->GetCapacity() - 1) : idx - 1;
647 0 : Hccl::DfxTaskInfo* slot = queue->GetSlot(idx);
648 0 : if (slot == nullptr || slot->taskId > targetTaskId) {
649 : break;
650 : }
651 0 : taskContext.push_back(slot);
652 : }
653 0 : return HCCL_SUCCESS;
654 : }
655 :
656 1 : HcclResult HcclCommTaskExceptionLite::PrintTaskContextInfo(CollCommAicpu* aicpuComm, u32 sqId, u32 taskId)
657 : {
658 1 : std::vector<Hccl::DfxTaskInfo*> taskContext{};
659 1 : CHK_PRT_RET(
660 : CollectTaskContext(aicpuComm, sqId, taskId, taskContext) != HCCL_SUCCESS,
661 : HCCL_ERROR("[%s]CollectTaskContext failed, devId[%u], sqId[%u], taskId[%u]", __func__, devId_, sqId, taskId),
662 : HCCL_E_PARA);
663 :
664 0 : std::string taskContextInfo = "";
665 0 : Hccl::DfxTaskInfo* lastTask = nullptr;
666 0 : for (u32 i = 0; i < taskContext.size(); ++i) {
667 0 : if (taskContext[i] == nullptr) {
668 0 : continue;
669 : }
670 0 : if (lastTask == nullptr) {
671 0 : lastTask = taskContext[i];
672 : }
673 0 : std::string conciseInfo = GetConciseTaskName(*taskContext[i]) + ",";
674 0 : u32 lastOpIndex = GetOpIndex(lastTask);
675 0 : u32 curOpIndex = GetOpIndex(taskContext[i]);
676 0 : bool overSize = (taskContextInfo.size() + conciseInfo.size()) >= TASK_CONTEXT_INFO_SIZE;
677 0 : if (overSize || (lastOpIndex != curOpIndex)) {
678 0 : PrintOpDataInfo(lastTask);
679 0 : HCCL_ERROR("[TaskException][AICPU]task sequence is OP(%u): %s", lastOpIndex, taskContextInfo.c_str());
680 0 : taskContextInfo = "";
681 0 : lastTask = taskContext[i];
682 : }
683 0 : taskContextInfo += conciseInfo;
684 0 : }
685 :
686 0 : if (!taskContextInfo.empty() && lastTask != nullptr) {
687 0 : u32 lastOpIndex = GetOpIndex(lastTask);
688 0 : PrintOpDataInfo(lastTask);
689 0 : HCCL_ERROR("[TaskException][AICPU]task sequence is OP(%u): %s", lastOpIndex, taskContextInfo.c_str());
690 : }
691 0 : HCCL_ERROR("[TaskException][AICPU]task sequence end.");
692 0 : return HCCL_SUCCESS;
693 1 : }
694 :
695 3 : std::string HcclCommTaskExceptionLite::GetGroupInfo(CollCommAicpu* aicpuComm)
696 : {
697 3 : if (aicpuComm == nullptr) {
698 1 : HCCL_ERROR("[%s]aicpuComm is nullptr, return empty string.", __func__);
699 2 : return "";
700 : }
701 : return Hccl::StringFormat(
702 2 : "group:[%s], rankSize:[%u], localRank:[%u]", aicpuComm->GetIdentifier().c_str(),
703 2 : aicpuComm->GetTopoInfo().userRankSize, aicpuComm->GetTopoInfo().userRank);
704 : }
705 :
706 3 : void HcclCommTaskExceptionLite::PrintEid(const Hccl::DfxTaskInfo& taskInfo)
707 : {
708 3 : auto taskType = static_cast<Hccl::TaskParamTypeVal>(taskInfo.taskType);
709 3 : if (taskType == Hccl::TaskParamTypeVal::TASK_UB_REDUCE_INLINE
710 3 : || taskType == Hccl::TaskParamTypeVal::TASK_WRITE_REDUCE_WITH_NOTIFY
711 3 : || taskType == Hccl::TaskParamTypeVal::TASK_UB_INLINE_WRITE
712 3 : || taskType == Hccl::TaskParamTypeVal::TASK_WRITE_WITH_NOTIFY || taskType == Hccl::TaskParamTypeVal::TASK_UB) {
713 1 : Hccl::Eid locEid;
714 1 : Hccl::Eid rmtEid;
715 1 : GetEidFromChannelHandle(taskInfo, locEid, rmtEid);
716 1 : HCCL_ERROR(
717 : "[TaskException][AICPU][%s]Error UB link info: localEid[%s], remoteEid[%s].", __func__,
718 : locEid.Describe().c_str(), rmtEid.Describe().c_str());
719 : }
720 3 : }
721 :
722 4 : Hccl::DfxTaskInfo* HcclCommTaskExceptionLite::FindDfxTaskInfo(CollCommAicpu* aicpuComm, u32 sqId, u32 sqeId)
723 : {
724 4 : Hccl::TaskInfoCircularQueue* queue = GetTaskQueueBySqId(aicpuComm, sqId);
725 4 : if (queue == nullptr || queue->IsEmpty()) {
726 1 : HCCL_ERROR("[%s]GetTaskQueueBySqId nullptr or queue is empty, devId[%u], sqId[%u].", __func__, devId_, sqId);
727 1 : return nullptr;
728 : }
729 3 : u32 targetTaskId = sqeId;
730 3 : for (u16 idx = 0; idx < queue->GetCapacity(); idx++) {
731 3 : Hccl::DfxTaskInfo* slot = queue->GetSlot(idx);
732 3 : if (slot != nullptr && slot->taskId == targetTaskId) {
733 3 : return slot;
734 : }
735 : }
736 0 : HCCL_ERROR("[%s]exception task not found, devId[%u], sqId[%u], sqeId[%u]", __func__, devId_, sqId, sqeId);
737 0 : return nullptr;
738 : }
739 :
740 6 : Hccl::TaskInfoCircularQueue* HcclCommTaskExceptionLite::GetTaskQueueBySqId(CollCommAicpu* aicpuComm, u32 sqId)
741 : {
742 6 : std::shared_lock<std::shared_mutex> threadRwlock(aicpuComm->GetCommEngineResMgr()->GetThreadMutex());
743 6 : const std::vector<std::shared_ptr<hccl::Thread>> threads = aicpuComm->GetCommEngineResMgr()->GetAllThread();
744 6 : for (auto& thread : threads) {
745 3 : Hccl::StreamLite* streamLite = static_cast<Hccl::StreamLite*>(thread->GetStreamLitePtr());
746 3 : if (streamLite != nullptr && streamLite->GetSqId() == sqId) {
747 3 : return streamLite->GetTaskInfos();
748 : }
749 : }
750 3 : return nullptr;
751 6 : }
752 :
753 6 : void HcclCommTaskExceptionLite::GetEidFromChannelHandle(
754 : const Hccl::DfxTaskInfo& taskInfo, Hccl::Eid& locEid, Hccl::Eid& rmtEid)
755 : {
756 6 : if (taskInfo.channelHandle != DFX_INVALID_U64) {
757 0 : auto* transport = reinterpret_cast<Hccl::UbTransportLiteImpl*>(taskInfo.channelHandle);
758 0 : locEid = transport->GetLocEid();
759 0 : rmtEid = transport->GetRmtEid();
760 : }
761 6 : }
762 :
763 4 : u32 HcclCommTaskExceptionLite::GetRemoteRankId(const Hccl::DfxTaskInfo& taskInfo)
764 : {
765 4 : if (taskInfo.dfxOpInfo != DFX_INVALID_U64) {
766 1 : auto* opInfo = reinterpret_cast<const Hccl::DfxDfxOpInfo*>(taskInfo.dfxOpInfo);
767 1 : if (opInfo->hcclCommDfxLite != nullptr) {
768 0 : return static_cast<hccl::HcclCommDfxLite*>(opInfo->hcclCommDfxLite)
769 0 : ->GetChannelRemoteRankId(taskInfo.channelHandle);
770 : }
771 : }
772 4 : return Hccl::DFX_INVALID_RANKID;
773 : }
774 :
775 1 : void HcclCommTaskExceptionLite::GetNotifyIdFromSqe(u64 sqeAddr, u32& notifyId)
776 : {
777 1 : void* sqePtr = reinterpret_cast<void*>(sqeAddr);
778 1 : if (sqePtr != nullptr) {
779 0 : auto* header = reinterpret_cast<Hccl::Rt91095StarsSqeHeader*>(sqePtr);
780 0 : if (static_cast<Hccl::Rt91095StarsSqeType>(header->type)
781 0 : == Hccl::Rt91095StarsSqeType::RT_91095_SQE_TYPE_NOTIFY_RECORD
782 0 : || static_cast<Hccl::Rt91095StarsSqeType>(header->type)
783 0 : == Hccl::Rt91095StarsSqeType::RT_91095_SQE_TYPE_NOTIFY_WAIT) {
784 0 : auto* notifySqe = reinterpret_cast<Hccl::Rt91095StarsNotifySqe*>(sqePtr);
785 0 : notifyId = notifySqe->notifyId;
786 : }
787 : }
788 1 : }
789 :
790 4 : std::string HcclCommTaskExceptionLite::GetNotifyInfo(const Hccl::DfxTaskInfo& taskInfo)
791 : {
792 4 : auto taskType = static_cast<Hccl::TaskParamTypeVal>(taskInfo.taskType);
793 4 : u32 notifyId = INVALID_U32;
794 4 : switch (taskType) {
795 1 : case Hccl::TaskParamTypeVal::TASK_UB_INLINE_WRITE:
796 : case Hccl::TaskParamTypeVal::TASK_WRITE_WITH_NOTIFY:
797 1 : notifyId = taskInfo.taskPara.ubDma.notifyId;
798 1 : break;
799 1 : case Hccl::TaskParamTypeVal::TASK_WRITE_REDUCE_WITH_NOTIFY:
800 1 : notifyId = taskInfo.taskPara.Reduce.notifyId;
801 1 : break;
802 1 : case Hccl::TaskParamTypeVal::TASK_NOTIFY_RECORD:
803 : case Hccl::TaskParamTypeVal::TASK_NOTIFY_WAIT:
804 : case Hccl::TaskParamTypeVal::TASK_SEND_NOTIFY: {
805 1 : GetNotifyIdFromSqe(taskInfo.taskPara.Notify.sqeAddr, notifyId);
806 1 : break;
807 : }
808 0 : case Hccl::TaskParamTypeVal::TASK_RDMA: {
809 0 : GetNotifyIdFromSqe(taskInfo.taskPara.Dma.sqeAddr, notifyId);
810 0 : break;
811 : }
812 1 : default:
813 2 : return "/";
814 : }
815 5 : return (notifyId == INVALID_U32) ? "/" : std::to_string(notifyId);
816 : }
817 :
818 2 : std::string HcclCommTaskExceptionLite::GetConciseTaskName(const Hccl::DfxTaskInfo& taskInfo)
819 : {
820 2 : const auto& taskConciseNameMap = Hccl::GetTaskConciseNameMap();
821 2 : auto it = taskConciseNameMap.find(taskInfo.taskType);
822 2 : std::string name = (it != taskConciseNameMap.end()) ? it->second : "UNKNOWN";
823 2 : u32 remoteRank = GetRemoteRankId(taskInfo);
824 4 : std::string rankStr = (remoteRank == Hccl::DFX_INVALID_RANKID) ? "/" : std::to_string(remoteRank);
825 2 : auto taskType = static_cast<Hccl::TaskParamTypeVal>(taskInfo.taskType);
826 2 : if (taskType == Hccl::TaskParamTypeVal::TASK_RDMA || taskType == Hccl::TaskParamTypeVal::TASK_NOTIFY_RECORD
827 2 : || taskType == Hccl::TaskParamTypeVal::TASK_NOTIFY_WAIT || taskType == Hccl::TaskParamTypeVal::TASK_SEND_NOTIFY
828 1 : || taskType == Hccl::TaskParamTypeVal::TASK_WRITE_WITH_NOTIFY
829 1 : || taskType == Hccl::TaskParamTypeVal::TASK_WRITE_REDUCE_WITH_NOTIFY
830 1 : || taskType == Hccl::TaskParamTypeVal::TASK_UB_INLINE_WRITE) {
831 1 : return name + "(" + rankStr + "," + GetNotifyInfo(taskInfo) + ")";
832 : }
833 1 : return name + "(" + rankStr + ")";
834 2 : }
835 :
836 0 : u32 HcclCommTaskExceptionLite::GetOpIndex(const Hccl::DfxTaskInfo* taskInfo)
837 : {
838 0 : if (taskInfo == nullptr || taskInfo->dfxOpInfo == DFX_INVALID_U64) {
839 0 : return UINT32_MAX;
840 : }
841 0 : return reinterpret_cast<const Hccl::DfxDfxOpInfo*>(taskInfo->dfxOpInfo)->opIndex;
842 : }
843 :
844 1 : void HcclCommTaskExceptionLite::PrintOpDataInfo(const Hccl::DfxTaskInfo* taskInfo)
845 : {
846 1 : if (taskInfo == nullptr || taskInfo->dfxOpInfo == DFX_INVALID_U64) {
847 1 : HCCL_ERROR("[TaskException][AICPU]opData information is (dfxOpInfo unavailable).");
848 1 : return;
849 : }
850 0 : const Hccl::DfxDfxOpInfo* opInfo = reinterpret_cast<const Hccl::DfxDfxOpInfo*>(taskInfo->dfxOpInfo);
851 0 : HCCL_ERROR(
852 : "[TaskException][AICPU]opData information is opIndex[%u], algTag[%s], count[%llu], "
853 : "dataType[%u], input: ptr[0x%llx] size[%llu], output: ptr[0x%llx] size[%llu].",
854 : opInfo->opIndex, opInfo->algTag, opInfo->count, opInfo->dataType, opInfo->srcAddr, opInfo->srcSize,
855 : opInfo->dstAddr, opInfo->dstSize);
856 : }
857 : } // namespace hcomm
|
