Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "task_exception_handler_lite.h"
12 : #include "task_exception_func.h"
13 : #include "communicator_impl_lite.h"
14 : #include "error_message_v2.h"
15 : #include "dlhal_function_v2.h"
16 : #include "task_param.h"
17 : #include "adapter_error_manager_pub.h"
18 :
19 : #include "task_struct_v2.h"
20 : #include "hcomm_task_scheduler_error.h"
21 : #include "orion_adapter_hal.h"
22 : #include "internal_exception.h"
23 :
24 : namespace Hccl {
25 :
26 : using namespace std;
27 :
28 : constexpr uint32_t TASK_CONTEXT_SIZE = 50;
29 : constexpr uint32_t TASK_CONTEXT_INFO_SIZE = LOG_TMPBUF_SIZE - 50; // task 执行失败时打印前序task信息的长度限制
30 :
31 7 : TaskExceptionHandlerLite& TaskExceptionHandlerLite::GetInstance()
32 : {
33 7 : static TaskExceptionHandlerLite instance;
34 7 : return instance;
35 : }
36 :
37 1 : TaskExceptionHandlerLite::TaskExceptionHandlerLite() { Register(); }
38 :
39 1 : TaskExceptionHandlerLite::~TaskExceptionHandlerLite() {}
40 :
41 1 : void TaskExceptionHandlerLite::Register() const { TaskExceptionFunc::GetInstance().RegisterCallback(Process); }
42 :
43 0 : void GetUbErrMsgInfo(TaskInfo* taskInfo, ErrorMessageReport& errMsgInfo, const rtLogicCqReport_t* exceptionInfo)
44 : {
45 0 : if (taskInfo->taskParam_.taskType == TaskParamType::TASK_WRITE_WITH_NOTIFY
46 0 : || taskInfo->taskParam_.taskType == TaskParamType::TASK_UB_INLINE_WRITE
47 0 : || taskInfo->taskParam_.taskType == TaskParamType::TASK_UB) {
48 0 : errMsgInfo.locEid = taskInfo->taskParam_.taskPara.DMA.locEid;
49 0 : errMsgInfo.rmtEid = taskInfo->taskParam_.taskPara.DMA.rmtEid;
50 0 : errMsgInfo.ubCqeStatus = exceptionInfo->errorCode & 0xFF;
51 0 : errMsgInfo.linkType = taskInfo->taskParam_.taskPara.DMA.linkType;
52 0 : errMsgInfo.size = taskInfo->taskParam_.taskPara.DMA.size;
53 0 : errMsgInfo.taskSrcAddr = reinterpret_cast<u64>(taskInfo->taskParam_.taskPara.DMA.src);
54 0 : errMsgInfo.taskDstAddr = reinterpret_cast<u64>(taskInfo->taskParam_.taskPara.DMA.dst);
55 0 : } else if (
56 0 : taskInfo->taskParam_.taskType == TaskParamType::TASK_UB_REDUCE_INLINE
57 0 : || taskInfo->taskParam_.taskType == TaskParamType::TASK_WRITE_REDUCE_WITH_NOTIFY) {
58 0 : errMsgInfo.locEid = taskInfo->taskParam_.taskPara.Reduce.locEid;
59 0 : errMsgInfo.rmtEid = taskInfo->taskParam_.taskPara.Reduce.rmtEid;
60 0 : errMsgInfo.ubCqeStatus = exceptionInfo->errorCode & 0xFF;
61 0 : errMsgInfo.linkType = taskInfo->taskParam_.taskPara.Reduce.linkType;
62 0 : errMsgInfo.size = taskInfo->taskParam_.taskPara.Reduce.size;
63 0 : errMsgInfo.taskSrcAddr = reinterpret_cast<u64>(taskInfo->taskParam_.taskPara.Reduce.src);
64 0 : errMsgInfo.taskDstAddr = reinterpret_cast<u64>(taskInfo->taskParam_.taskPara.Reduce.dst);
65 : }
66 0 : errMsgInfo.rtCqErrorType = exceptionInfo->errorType;
67 0 : errMsgInfo.rtCqErrorCode = exceptionInfo->errorCode;
68 0 : }
69 :
70 0 : HcclResult GenerateErrorMessageReport(
71 : const CommunicatorImplLite* aicpuComm, TaskInfo* taskInfo, ErrorMessageReport& errMsgInfo,
72 : const rtLogicCqReport_t* exceptionInfo)
73 : {
74 : // 获取需要上报的关键信息
75 0 : errMsgInfo.remoteUserRank = taskInfo->remoteRank_;
76 0 : errMsgInfo.streamId = taskInfo->streamId_;
77 0 : errMsgInfo.taskId = taskInfo->taskId_;
78 0 : errMsgInfo.rankId = aicpuComm->GetMyRank();
79 0 : errMsgInfo.rankSize = aicpuComm->GetRankSize();
80 0 : errMsgInfo.opIndex = taskInfo->dfxOpInfo_ == nullptr ? 0 : taskInfo->dfxOpInfo_->commIndex_;
81 0 : errMsgInfo.opType = taskInfo->dfxOpInfo_->op_.opType;
82 0 : errMsgInfo.count = taskInfo->dfxOpInfo_->op_.dataCount;
83 0 : errMsgInfo.dataType = taskInfo->dfxOpInfo_->op_.dataType;
84 0 : errMsgInfo.srcAddr = static_cast<u64>(taskInfo->dfxOpInfo_->op_.inputMem->GetAddr());
85 0 : errMsgInfo.dstAddr = static_cast<u64>(taskInfo->dfxOpInfo_->op_.outputMem->GetAddr());
86 0 : errMsgInfo.taskType = taskInfo->taskParam_.taskType;
87 :
88 0 : if (taskInfo->taskParam_.taskType == TaskParamType::TASK_NOTIFY_WAIT) {
89 0 : errMsgInfo.notifyId = taskInfo->taskParam_.taskPara.Notify.notifyID;
90 0 : errMsgInfo.notifyValue = taskInfo->taskParam_.taskPara.Notify.value;
91 0 : } else if (
92 0 : taskInfo->taskParam_.taskType == TaskParamType::TASK_UB_REDUCE_INLINE
93 0 : || taskInfo->taskParam_.taskType == TaskParamType::TASK_WRITE_REDUCE_WITH_NOTIFY) {
94 0 : errMsgInfo.notifyId = taskInfo->taskParam_.taskPara.Reduce.notifyID;
95 0 : errMsgInfo.notifyValue = taskInfo->taskParam_.taskPara.Reduce.notifyValue;
96 0 : } else if (
97 0 : taskInfo->taskParam_.taskType == TaskParamType::TASK_UB_INLINE_WRITE
98 0 : || taskInfo->taskParam_.taskType == TaskParamType::TASK_WRITE_WITH_NOTIFY) {
99 0 : errMsgInfo.notifyId = taskInfo->taskParam_.taskPara.DMA.notifyID;
100 0 : errMsgInfo.notifyValue = taskInfo->taskParam_.taskPara.DMA.notifyValue;
101 : }
102 :
103 0 : if (taskInfo->taskParam_.taskType == TaskParamType::TASK_UB_REDUCE_INLINE
104 0 : || taskInfo->taskParam_.taskType == TaskParamType::TASK_WRITE_REDUCE_WITH_NOTIFY
105 0 : || taskInfo->taskParam_.taskType == TaskParamType::TASK_REDUCE_INLINE) {
106 0 : errMsgInfo.reduceType = taskInfo->taskParam_.taskPara.Reduce.reduceOp;
107 : }
108 :
109 0 : memcpy_s(errMsgInfo.tag, sizeof(errMsgInfo.tag), aicpuComm->GetId().c_str(), aicpuComm->GetId().size());
110 0 : memcpy_s(errMsgInfo.group, sizeof(errMsgInfo.group), aicpuComm->GetId().c_str(), aicpuComm->GetId().size());
111 :
112 0 : GetUbErrMsgInfo(taskInfo, errMsgInfo, exceptionInfo);
113 :
114 0 : return HCCL_SUCCESS;
115 : }
116 :
117 0 : HcclResult SendErrorMessageReportToHost(CommunicatorImplLite* aicpuComm, ErrorMessageReport& errMsgInfo)
118 : {
119 0 : CHK_RET(aicpuComm->SendErrorMessageReportToHost(errMsgInfo));
120 0 : return HCCL_SUCCESS;
121 : }
122 :
123 : constexpr u32 RT_SDMA_COMPERR = 0x9; // A3 sdma error类型为0x9时，表示写拷贝发生超时代答，或者数据搬移时地址译码错误
124 : constexpr u32 RT_SDMA_COMPDATAERR = 0xa; // A3 sdma error类型为0xa时，表示读拷贝发生超时代答，或者读HBM返回ERROR
125 : constexpr u32 RT_SDMA_DATAERR = 0x8; // A3 sdma error类型为0x8时，表示读HBM返回ERROR
126 : constexpr u32 RT_UB_LOCAL_OPERATIOINERR = 0x2; // A5 ub error类型为0x2时，表示UB本端返回ERROR
127 : constexpr u32 RT_UB_REMOTE_OPERATIOINERR = 0x3; // A5 ub error类型为0x3时，表示UB远端返回ERROR
128 : constexpr u32 RT_UB_LINK_FAILEDERR = 0x5; // A5 ub error类型为0x5时，表示网络异常，taack超时
129 :
130 : // 把SDMA类错误码转换成Ts对应的错误码
131 0 : uint16_t SwitchSdmaCqeErrCodeToTsErrCode(u32 cqeErrCode)
132 : {
133 0 : switch (cqeErrCode) {
134 0 : case RT_SDMA_COMPERR:
135 0 : return TS_ERROR_SDMA_LINK_ERROR;
136 0 : case RT_SDMA_COMPDATAERR:
137 0 : return TS_ERROR_SDMA_POISON_ERROR;
138 0 : case RT_SDMA_DATAERR:
139 0 : return TS_ERROR_SDMA_DDRC_ERROR;
140 0 : default:
141 0 : return TS_ERROR_HCCL_OTHER_ERROR;
142 : }
143 : }
144 :
145 : // 把UB类错误码转换成Ts对应的错误码
146 0 : uint16_t SwitchUBCqeErrCodeToTsErrCode(u32 cqeErrCode)
147 : {
148 0 : switch (cqeErrCode) {
149 0 : case RT_UB_LOCAL_OPERATIOINERR:
150 0 : return TS_ERROR_HCCL_OP_UB_DDRC_FAILED;
151 0 : case RT_UB_REMOTE_OPERATIOINERR:
152 0 : return TS_ERROR_HCCL_OP_UB_POISON_FAILED;
153 0 : case RT_UB_LINK_FAILEDERR:
154 0 : return TS_ERROR_HCCL_OP_UB_LINK_FAILED;
155 0 : default:
156 0 : return TS_ERROR_HCCL_OTHER_ERROR;
157 : }
158 : }
159 :
160 0 : static std::string GetSdmaErrorDesc(u32 errorCode)
161 : {
162 0 : switch (errorCode) {
163 0 : case RT_SDMA_DATAERR:
164 0 : return "read HBM return ERROR";
165 0 : case RT_SDMA_COMPERR:
166 0 : return "write copy timeout or address decode ERROR";
167 0 : case RT_SDMA_COMPDATAERR:
168 0 : return "read copy timeout or read HBM return ERROR";
169 0 : default:
170 0 : return "unknown SDMA ERROR";
171 : }
172 : }
173 :
174 0 : static void ReportSdmaError(
175 : u32 localDeviceId, u32 notifyId, u32 tsId, s32 userStreamId, u32 errorCode, const std::string& remoteRankId,
176 : const std::string& groupRankContent)
177 : {
178 0 : std::string baseInfo = "localDeviceId=" + std::to_string(localDeviceId);
179 0 : std::string taskInfo = "notifyId= " + std::to_string(notifyId) + ", tsId=" + std::to_string(tsId)
180 0 : + ", streamId=" + std::to_string(userStreamId) + ", error_code=" + std::to_string(errorCode)
181 0 : + ", description=" + GetSdmaErrorDesc(errorCode);
182 0 : RPT_INPUT_ERR(
183 : true, "EI0012",
184 : std::vector<std::string>({"remote_rankid", "base_information", "task_information", "group_rank_content"}),
185 : std::vector<std::string>({remoteRankId, baseInfo, taskInfo, groupRankContent}));
186 0 : }
187 :
188 0 : HcclResult SendTaskExceptionByMBox(
189 : const u32 localDeviceId, const u32 notifyId, const u32 tsId, const s32 userStreamId,
190 : const rtLogicCqReport_t* exceptionInfo)
191 : {
192 0 : HCCL_INFO("[SendTaskExceptionByMBox] SendTaskExceptionByMBox start.");
193 0 : ts_aicpu_msg_info_t aicpuSqe = {};
194 0 : u32 hostpid = 0;
195 0 : u32 vf_id = 0;
196 0 : int pid = getpid();
197 0 : HCCL_INFO("SendTaskExceptionByMBox getpid[%d]", pid);
198 : // 调整drvQueryProcessHostPid获取pid和vf_id的值
199 0 : CHK_RET(HrtHalDrvQueryProcessHostPid(pid, nullptr, &vf_id, &hostpid, nullptr));
200 :
201 0 : aicpuSqe.pid = hostpid;
202 0 : aicpuSqe.cmd_type = TS_AICPU_RECORD;
203 0 : aicpuSqe.vf_id = vf_id;
204 0 : aicpuSqe.tid = 0U; // notify is no need tid
205 0 : aicpuSqe.u.aicpu_record.record_type = AICPU_MSG_NOTIFY_RECORD_V2;
206 0 : aicpuSqe.u.aicpu_record.record_id = notifyId;
207 :
208 0 : aicpuSqe.ts_id = static_cast<uint8_t>(tsId);
209 :
210 0 : aicpuSqe.u.aicpu_record.fault_task_id = 0xffffffff;
211 :
212 0 : HCCL_ERROR(
213 : "[SendTaskExceptionByMBox] exceptionInfo errorType[%u], errorCode[%u]",
214 : static_cast<u32>(exceptionInfo->errorType), exceptionInfo->errorCode);
215 0 : if (exceptionInfo->errorType == 1) { // ub类型为1
216 0 : aicpuSqe.u.aicpu_record.ret_code = SwitchUBCqeErrCodeToTsErrCode(exceptionInfo->errorCode & 0xFF);
217 : } else {
218 0 : aicpuSqe.u.aicpu_record.ret_code = SwitchSdmaCqeErrCodeToTsErrCode(exceptionInfo->errorCode);
219 : }
220 :
221 : struct event_summary event;
222 0 : event.dst_engine = TS_CPU;
223 0 : event.policy = ONLY;
224 0 : event.pid = 0;
225 0 : event.grp_id = 0;
226 0 : event.event_id = EVENT_TS_CTRL_MSG;
227 0 : event.subevent_id = 0U;
228 0 : event.msg_len = static_cast<uint32_t>(sizeof(ts_aicpu_msg_info_t));
229 0 : event.msg = reinterpret_cast<char_t*>(&aicpuSqe);
230 0 : auto ret = DlHalFunctionV2::GetInstance().dlHalEschedSubmitEvent(localDeviceId, &event);
231 0 : if (ret != static_cast<int32_t>(DRV_ERROR_NONE)) {
232 0 : HCCL_ERROR(
233 : "[SendTaskExceptionByMBox]Send msg async to ts failed. ret=%d, streamId=%d, "
234 : "notifyId=%u.",
235 : ret, userStreamId, notifyId);
236 0 : return HCCL_E_DRV;
237 : }
238 0 : HCCL_RUN_INFO(
239 : "[SendTaskExceptionByMBox]Send msg async to ts finished. streamId=%d, notifyId=%u, msg_size=%u, "
240 : "hostpid=%u, vf_id=%u, errCode=%u.",
241 : userStreamId, notifyId, static_cast<uint32_t>(sizeof(ts_aicpu_msg_info_t)), hostpid, vf_id,
242 : exceptionInfo->errorCode);
243 0 : return HCCL_SUCCESS;
244 : }
245 :
246 0 : HcclResult SendTaskExceptionByMBox(CommunicatorImplLite* aicpuComm, const rtLogicCqReport_t* exceptionInfo)
247 : {
248 0 : u32 localDeviceId = 0;
249 0 : u32 notifyId = aicpuComm->GetHostDeviceSyncNotifyLiteMgr()->GetHostWaitNotify()->GetId();
250 0 : u32 devPhyId = aicpuComm->GetHostDeviceSyncNotifyLiteMgr()->GetHostWaitNotify()->GetDevPhyId();
251 :
252 0 : HCCL_INFO("[TaskExceptionHandlerLite][SendTaskExceptionByMBox] HostToDeviceLogicId[%u]", devPhyId);
253 0 : auto ret = drvGetLocalDevIDByHostDevID(devPhyId, &localDeviceId);
254 0 : if (ret != 0) {
255 0 : HCCL_ERROR("[TaskExceptionHandlerLite][SendTaskExceptionByMBox] HostToDeviceLogicId[%u] failed.", devPhyId);
256 0 : return HCCL_E_DRV;
257 : }
258 0 : HCCL_INFO(
259 : "[TaskExceptionHandlerLite][SendTaskExceptionByMBox] HostToDeviceLogicId[%u], localDeviceid[%u]", devPhyId,
260 : localDeviceId);
261 :
262 0 : CHK_RET(SendTaskExceptionByMBox(localDeviceId, notifyId, 0, aicpuComm->GetUserStreamId(), exceptionInfo));
263 :
264 0 : if (exceptionInfo->errorType != 1) {
265 0 : std::string remoteRankId = std::to_string(aicpuComm->GetMyRank());
266 0 : std::string groupRankContent = "group:[" + aicpuComm->GetId() + "], rankSize["
267 0 : + std::to_string(aicpuComm->GetRankSize()) + "], rankId["
268 0 : + std::to_string(aicpuComm->GetMyRank()) + "]";
269 0 : ReportSdmaError(
270 0 : localDeviceId, notifyId, 0, aicpuComm->GetUserStreamId(), exceptionInfo->errorCode, remoteRankId,
271 : groupRankContent);
272 0 : }
273 0 : return HCCL_SUCCESS;
274 : }
275 :
276 0 : string GetOpDataInfo(const TaskInfo& taskInfo)
277 : {
278 0 : if (taskInfo.dfxOpInfo_ == nullptr || taskInfo.dfxOpInfo_->comm_ == nullptr) {
279 0 : HCCL_ERROR("[GetOpDataInfo]TaskInfo communicator is nullptr.");
280 0 : return "";
281 : }
282 0 : CommunicatorImplLite* aicpuComm = static_cast<CommunicatorImplLite*>(taskInfo.dfxOpInfo_->comm_);
283 0 : u32 localDeviceId = 0;
284 0 : u32 devPhyId = aicpuComm->GetDevPhyId();
285 :
286 0 : HCCL_INFO("[GetOpDataInfo] HostToDeviceLogicId[%u]", devPhyId);
287 0 : auto ret = drvGetLocalDevIDByHostDevID(devPhyId, &localDeviceId);
288 0 : if (ret != 0) {
289 0 : HCCL_ERROR("[GetOpDataInfo] HostToDeviceLogicId[%u] failed.", devPhyId);
290 0 : return "";
291 : }
292 0 : return StringFormat("deviceId[%u], %s", localDeviceId, taskInfo.GetOpInfo().c_str());
293 : }
294 :
295 0 : void PrintEid(TaskInfo* taskInfo)
296 : {
297 0 : if (taskInfo->taskParam_.taskType == TaskParamType::TASK_UB_REDUCE_INLINE
298 0 : || taskInfo->taskParam_.taskType == TaskParamType::TASK_WRITE_REDUCE_WITH_NOTIFY) {
299 0 : HCCL_ERROR(
300 : "[PrintEid] Error UB link info: localEid[%s], remoteEid[%s]. ",
301 : taskInfo->taskParam_.taskPara.Reduce.locEid.Describe().c_str(),
302 : taskInfo->taskParam_.taskPara.Reduce.rmtEid.Describe().c_str());
303 0 : } else if (
304 0 : taskInfo->taskParam_.taskType == TaskParamType::TASK_UB_INLINE_WRITE
305 0 : || taskInfo->taskParam_.taskType == TaskParamType::TASK_WRITE_WITH_NOTIFY
306 0 : || taskInfo->taskParam_.taskType == TaskParamType::TASK_UB) {
307 0 : HCCL_ERROR(
308 : "[PrintEid] Error UB link info: localEid[%s], remoteEid[%s]. ",
309 : taskInfo->taskParam_.taskPara.DMA.locEid.Describe().c_str(),
310 : taskInfo->taskParam_.taskPara.DMA.rmtEid.Describe().c_str());
311 : }
312 0 : }
313 :
314 0 : void TaskExceptionHandlerLite::Process(CommunicatorImplLite* aicpuComm, rtLogicCqReport_t* exceptionInfo)
315 : {
316 0 : if (exceptionInfo == nullptr) {
317 0 : HCCL_ERROR("Exception process failed, rtExceptionInfo is nullptr.");
318 0 : return;
319 : }
320 : // exceptionInfo->taskId和exceptionInfo->streamId拼成sqeId
321 : const u32 sqeId
322 0 : = static_cast<uint32_t>(exceptionInfo->taskId << 16) | static_cast<uint32_t>(exceptionInfo->streamId);
323 0 : const auto curTask = aicpuComm->GetMirrorTaskMgrLite()->GetTaskInfo(exceptionInfo->sqId, sqeId);
324 0 : if (curTask == nullptr) {
325 : // 未找到异常对应的TaskInfo
326 0 : HCCL_ERROR("Exception task not found. streamId(sqId)[%u], taskId(sqeId)[%u].", exceptionInfo->sqId, sqeId);
327 0 : return;
328 : }
329 :
330 : // 每个通信域仅首次上报（N秒快恢时重置）
331 0 : if (!aicpuComm->IsErrorReported()) {
332 : // 1) errorMessage上报
333 0 : ErrorMessageReport errMsgInfo{};
334 0 : auto ret = GenerateErrorMessageReport(aicpuComm, curTask, errMsgInfo, exceptionInfo);
335 0 : if (ret != HCCL_SUCCESS) {
336 0 : THROW<InvalidParamsException>("GenerateErrorMessageReport failed.");
337 : }
338 0 : ret = SendErrorMessageReportToHost(aicpuComm, errMsgInfo);
339 0 : if (ret != HCCL_SUCCESS) {
340 0 : THROW<InvalidParamsException>("SendErrorMessageReportToHost Failed.");
341 : }
342 :
343 : // 2) send mbox to tsfw
344 0 : CHK_RET_THROW(
345 : InternalException,
346 : StringFormat(
347 : "[TaskExceptionHandlerLite][SendTaskExceptionByMBox]group[%s]:"
348 : "Send task exception by mailBox failed, streamId[%d]",
349 : aicpuComm->GetId().c_str(), exceptionInfo->streamId),
350 : SendTaskExceptionByMBox(aicpuComm, exceptionInfo));
351 :
352 0 : aicpuComm->SetErrorReported();
353 : }
354 :
355 0 : HCCL_ERROR("[TaskExceptionHandlerLite][%s]Task from HCCL run failed.", __func__);
356 0 : if (curTask->taskParam_.taskType == TaskParamType::TASK_NOTIFY_WAIT) {
357 0 : PrintTaskContextInfo(aicpuComm, exceptionInfo->sqId, sqeId);
358 : }
359 0 : if (curTask->taskParam_.taskType == TaskParamType::TASK_WRITE_WITH_NOTIFY
360 0 : || curTask->taskParam_.taskType == TaskParamType::TASK_WRITE_REDUCE_WITH_NOTIFY
361 0 : || curTask->taskParam_.taskType == TaskParamType::TASK_UB_INLINE_WRITE
362 0 : || curTask->taskParam_.taskType == TaskParamType::TASK_UB_REDUCE_INLINE
363 0 : || curTask->taskParam_.taskType == TaskParamType::TASK_UB) {
364 0 : HCCL_ERROR("[TaskExceptionHandlerLite] ubCqeStatus[%u]", static_cast<u32>(exceptionInfo->errorCode & 0xFF));
365 : }
366 0 : PrintEid(curTask);
367 0 : HCCL_ERROR("[TaskExceptionHandlerLite]Task run failed, base information is %s.", curTask->GetBaseInfo().c_str());
368 0 : HCCL_ERROR("[TaskExceptionHandlerLite]Task run failed, para information is %s.", curTask->GetParaInfo().c_str());
369 0 : HCCL_ERROR(
370 : "[TaskExceptionHandlerLite]Task run failed, groupRank information is %s.", GetGroupRankInfo(*curTask).c_str());
371 0 : if (curTask->dfxOpInfo_ != nullptr && curTask->dfxOpInfo_->headOpCounterAddr_ != 0
372 0 : && curTask->dfxOpInfo_->tailOpCounterAddr_ != 0) {
373 0 : HCCL_ERROR(
374 : "[TaskExceptionHandlerLite]Task run failed, headOpCounter[%u] tailOpCounter[%u] opIndex[%u].",
375 : static_cast<u32>(*(reinterpret_cast<float*>(curTask->dfxOpInfo_->headOpCounterAddr_))),
376 : static_cast<u32>(*(reinterpret_cast<float*>(curTask->dfxOpInfo_->tailOpCounterAddr_))),
377 : curTask->dfxOpInfo_->opIndex_);
378 : }
379 0 : HCCL_ERROR("[TaskExceptionHandlerLite]Task run failed, opData information is %s.", GetOpDataInfo(*curTask).c_str());
380 : }
381 :
382 0 : string TaskExceptionHandlerLite::GetGroupRankInfo(const TaskInfo& taskInfo)
383 : {
384 0 : if (taskInfo.dfxOpInfo_ == nullptr || taskInfo.dfxOpInfo_->comm_ == nullptr) {
385 0 : HCCL_ERROR("[TaskInfo][%s]TaskInfo communicator is nullptr.", __func__);
386 0 : return "";
387 : }
388 0 : const CommunicatorImplLite* commImplLite = static_cast<CommunicatorImplLite*>(taskInfo.dfxOpInfo_->comm_);
389 : return StringFormat(
390 0 : "group:[%s], rankSize[%u], localRank[%d], remoteRank[%u]", commImplLite->GetId().c_str(),
391 0 : commImplLite->GetRankSize(), commImplLite->GetMyRank(), taskInfo.remoteRank_);
392 : }
393 :
394 0 : void TaskExceptionHandlerLite::PrintTaskContextInfo(CommunicatorImplLite* aicpuComm, uint32_t sqId, uint32_t taskId)
395 : {
396 0 : auto queue = aicpuComm->GetMirrorTaskMgrLite()->GetQueue(sqId);
397 0 : if (queue == nullptr) {
398 : // 未找到异常对应的TaskQueue
399 0 : HCCL_ERROR("Exception task queue not found. streamId(sqId)[%u].", sqId);
400 0 : return;
401 : }
402 :
403 0 : auto func = [taskId](const unique_ptr<TaskInfo>& task) {
404 0 : return task->taskId_ == taskId;
405 0 : };
406 0 : auto taskItorPtr = queue->Find(func);
407 0 : if (taskItorPtr == nullptr || *taskItorPtr == *queue->End()) {
408 : // 在队列中未找到异常对应的TaskInfo
409 0 : HCCL_ERROR("Exception task not found. streamId(sqId)[%u], taskId(sqeId)[%u].", sqId, taskId);
410 0 : return;
411 : }
412 :
413 : // 找到当前异常task的前50个task(至多)
414 0 : vector<TaskInfo*> taskContext{};
415 0 : for (uint32_t i = 0; i < TASK_CONTEXT_SIZE && *taskItorPtr != *queue->Begin(); ++i, --(*taskItorPtr)) {
416 0 : if ((**taskItorPtr)->taskId_ > taskId) {
417 0 : break;
418 : }
419 0 : if ((**taskItorPtr)->taskId_ != taskId) {
420 0 : taskContext.emplace_back((**taskItorPtr).get());
421 : }
422 : }
423 :
424 0 : if (taskContext.empty()) {
425 0 : return;
426 : }
427 :
428 0 : HCCL_ERROR("[TaskExceptionHandlerLite]Task run failed, context sequence before error task is "
429 : "[SDMA:M(rank), RDMA:RS(rank,id), SendPayload:SP(rank), InlineReduce:IR(rank), Reduce:R(rank), "
430 : "NotifyRecord:NR(rank,id), NotifyWait:NW(rank,id), SendNotify:SN(rank,id), "
431 : "WriteWithNotify:WN(rank,id), WriteReduceWithNotify:WRN(rank,id)]:");
432 :
433 0 : string taskContextInfo = "";
434 0 : for (auto it = taskContext.rbegin(); it != taskContext.rend(); ++it) {
435 0 : string conciseInfo = (*it)->GetConciseBaseInfo();
436 0 : conciseInfo += ",";
437 :
438 0 : if (taskContextInfo.size() + conciseInfo.size() >= TASK_CONTEXT_INFO_SIZE) {
439 0 : HCCL_ERROR("[TaskExceptionHandlerLite]%s", taskContextInfo.c_str());
440 0 : taskContextInfo = "";
441 : }
442 :
443 0 : taskContextInfo += conciseInfo;
444 0 : }
445 0 : HCCL_ERROR("[TaskExceptionHandlerLite]%s end.", taskContextInfo.c_str());
446 0 : }
447 :
448 : } // namespace Hccl
|
