Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "task_exception_handler_lite.h"
12 : #include "task_exception_func.h"
13 : #include "communicator_impl_lite.h"
14 : #include "error_message_v2.h"
15 : #include "dlhal_function_v2.h"
16 : #include "task_param.h"
17 : #include "adapter_error_manager_pub.h"
18 :
19 : #include "task_struct_v2.h"
20 : #include "hcomm_task_scheduler_error.h"
21 : #include "orion_adapter_hal.h"
22 : #include "internal_exception.h"
23 :
24 : namespace Hccl {
25 :
26 : using namespace std;
27 :
28 : constexpr uint32_t TASK_CONTEXT_SIZE = 50;
29 : constexpr uint32_t TASK_CONTEXT_INFO_SIZE = LOG_TMPBUF_SIZE - 50; // task 执行失败时打印前序task信息的长度限制
30 :
31 7 : TaskExceptionHandlerLite &TaskExceptionHandlerLite::GetInstance()
32 : {
33 7 : static TaskExceptionHandlerLite instance;
34 7 : return instance;
35 : }
36 :
37 1 : TaskExceptionHandlerLite::TaskExceptionHandlerLite()
38 : {
39 1 : Register();
40 1 : }
41 :
42 1 : TaskExceptionHandlerLite::~TaskExceptionHandlerLite()
43 : {
44 1 : }
45 :
46 1 : void TaskExceptionHandlerLite::Register() const
47 : {
48 1 : TaskExceptionFunc::GetInstance().RegisterCallback(Process);
49 1 : }
50 :
51 0 : void GetUbErrMsgInfo(TaskInfo* taskInfo, ErrorMessageReport &errMsgInfo, const rtLogicCqReport_t* exceptionInfo) {
52 0 : if (taskInfo->taskParam_.taskType == TaskParamType::TASK_WRITE_WITH_NOTIFY
53 0 : || taskInfo->taskParam_.taskType == TaskParamType::TASK_UB_INLINE_WRITE
54 0 : || taskInfo->taskParam_.taskType == TaskParamType::TASK_UB) {
55 0 : errMsgInfo.locEid = taskInfo->taskParam_.taskPara.DMA.locEid;
56 0 : errMsgInfo.rmtEid = taskInfo->taskParam_.taskPara.DMA.rmtEid;
57 0 : errMsgInfo.ubCqeStatus = exceptionInfo->errorCode & 0xFF;
58 0 : errMsgInfo.linkType = taskInfo->taskParam_.taskPara.DMA.linkType;
59 0 : errMsgInfo.size = taskInfo->taskParam_.taskPara.DMA.size;
60 0 : errMsgInfo.taskSrcAddr = reinterpret_cast<u64>(taskInfo->taskParam_.taskPara.DMA.src);
61 0 : errMsgInfo.taskDstAddr = reinterpret_cast<u64>(taskInfo->taskParam_.taskPara.DMA.dst);
62 0 : } else if (taskInfo->taskParam_.taskType == TaskParamType::TASK_UB_REDUCE_INLINE
63 0 : || taskInfo->taskParam_.taskType == TaskParamType::TASK_WRITE_REDUCE_WITH_NOTIFY) {
64 0 : errMsgInfo.locEid = taskInfo->taskParam_.taskPara.Reduce.locEid;
65 0 : errMsgInfo.rmtEid = taskInfo->taskParam_.taskPara.Reduce.rmtEid;
66 0 : errMsgInfo.ubCqeStatus = exceptionInfo->errorCode & 0xFF;
67 0 : errMsgInfo.linkType = taskInfo->taskParam_.taskPara.Reduce.linkType;
68 0 : errMsgInfo.size = taskInfo->taskParam_.taskPara.Reduce.size;
69 0 : errMsgInfo.taskSrcAddr = reinterpret_cast<u64>(taskInfo->taskParam_.taskPara.Reduce.src);
70 0 : errMsgInfo.taskDstAddr = reinterpret_cast<u64>(taskInfo->taskParam_.taskPara.Reduce.dst);
71 : }
72 0 : errMsgInfo.rtCqErrorType = exceptionInfo->errorType;
73 0 : errMsgInfo.rtCqErrorCode = exceptionInfo->errorCode;
74 0 : }
75 :
76 0 : HcclResult GenerateErrorMessageReport(const CommunicatorImplLite *aicpuComm, TaskInfo* taskInfo, ErrorMessageReport &errMsgInfo, const rtLogicCqReport_t* exceptionInfo)
77 : {
78 : // 获取需要上报的关键信息
79 0 : errMsgInfo.remoteUserRank = taskInfo->remoteRank_;
80 0 : errMsgInfo.streamId = taskInfo->streamId_;
81 0 : errMsgInfo.taskId = taskInfo->taskId_;
82 0 : errMsgInfo.rankId = aicpuComm->GetMyRank();
83 0 : errMsgInfo.rankSize = aicpuComm->GetRankSize();
84 0 : errMsgInfo.opIndex = taskInfo->dfxOpInfo_ == nullptr ? 0 : taskInfo->dfxOpInfo_->commIndex_;
85 0 : errMsgInfo.opType = taskInfo->dfxOpInfo_->op_.opType;
86 0 : errMsgInfo.count = taskInfo->dfxOpInfo_->op_.dataCount;
87 0 : errMsgInfo.dataType = taskInfo->dfxOpInfo_->op_.dataType;
88 0 : errMsgInfo.srcAddr = static_cast<u64>(taskInfo->dfxOpInfo_->op_.inputMem->GetAddr());
89 0 : errMsgInfo.dstAddr = static_cast<u64>(taskInfo->dfxOpInfo_->op_.outputMem->GetAddr());
90 0 : errMsgInfo.taskType = taskInfo->taskParam_.taskType;
91 :
92 0 : if (taskInfo->taskParam_.taskType == TaskParamType::TASK_NOTIFY_WAIT) {
93 0 : errMsgInfo.notifyId = taskInfo->taskParam_.taskPara.Notify.notifyID;
94 0 : errMsgInfo.notifyValue = taskInfo->taskParam_.taskPara.Notify.value;
95 0 : } else if (taskInfo->taskParam_.taskType == TaskParamType::TASK_UB_REDUCE_INLINE
96 0 : || taskInfo->taskParam_.taskType == TaskParamType::TASK_WRITE_REDUCE_WITH_NOTIFY) {
97 0 : errMsgInfo.notifyId = taskInfo->taskParam_.taskPara.Reduce.notifyID;
98 0 : errMsgInfo.notifyValue = taskInfo->taskParam_.taskPara.Reduce.notifyValue;
99 0 : } else if (taskInfo->taskParam_.taskType == TaskParamType::TASK_UB_INLINE_WRITE
100 0 : || taskInfo->taskParam_.taskType == TaskParamType::TASK_WRITE_WITH_NOTIFY) {
101 0 : errMsgInfo.notifyId = taskInfo->taskParam_.taskPara.DMA.notifyID;
102 0 : errMsgInfo.notifyValue = taskInfo->taskParam_.taskPara.DMA.notifyValue;
103 : }
104 :
105 0 : if (taskInfo->taskParam_.taskType == TaskParamType::TASK_UB_REDUCE_INLINE
106 0 : || taskInfo->taskParam_.taskType == TaskParamType::TASK_WRITE_REDUCE_WITH_NOTIFY
107 0 : || taskInfo->taskParam_.taskType == TaskParamType::TASK_REDUCE_INLINE) {
108 0 : errMsgInfo.reduceType = taskInfo->taskParam_.taskPara.Reduce.reduceOp;
109 : }
110 :
111 0 : memcpy_s(errMsgInfo.tag, sizeof(errMsgInfo.tag), aicpuComm->GetId().c_str(), aicpuComm->GetId().size());
112 0 : memcpy_s(errMsgInfo.group, sizeof(errMsgInfo.group), aicpuComm->GetId().c_str(), aicpuComm->GetId().size());
113 :
114 0 : GetUbErrMsgInfo(taskInfo, errMsgInfo, exceptionInfo);
115 :
116 0 : return HCCL_SUCCESS;
117 : }
118 :
119 0 : HcclResult SendErrorMessageReportToHost(CommunicatorImplLite *aicpuComm, ErrorMessageReport &errMsgInfo)
120 : {
121 0 : CHK_RET(aicpuComm->SendErrorMessageReportToHost(errMsgInfo));
122 0 : return HCCL_SUCCESS;
123 : }
124 :
125 : constexpr u32 RT_SDMA_COMPERR = 0x9; // A3 sdma error类型为0x9时，表示写拷贝发生超时代答，或者数据搬移时地址译码错误
126 : constexpr u32 RT_SDMA_COMPDATAERR = 0xa; // A3 sdma error类型为0xa时，表示读拷贝发生超时代答，或者读HBM返回ERROR
127 : constexpr u32 RT_SDMA_DATAERR = 0x8; // A3 sdma error类型为0x8时，表示读HBM返回ERROR
128 : constexpr u32 RT_UB_LOCAL_OPERATIOINERR = 0x2; // A5 ub error类型为0x2时，表示UB本端返回ERROR
129 : constexpr u32 RT_UB_REMOTE_OPERATIOINERR = 0x3; // A5 ub error类型为0x3时，表示UB远端返回ERROR
130 : constexpr u32 RT_UB_LINK_FAILEDERR = 0x5; // A5 ub error类型为0x5时，表示网络异常，taack超时
131 :
132 : // 把SDMA类错误码转换成Ts对应的错误码
133 0 : uint16_t SwitchSdmaCqeErrCodeToTsErrCode(u32 cqeErrCode){
134 0 : switch (cqeErrCode) {
135 0 : case RT_SDMA_COMPERR:
136 0 : return TS_ERROR_SDMA_LINK_ERROR;
137 0 : case RT_SDMA_COMPDATAERR:
138 0 : return TS_ERROR_SDMA_POISON_ERROR;
139 0 : case RT_SDMA_DATAERR:
140 0 : return TS_ERROR_SDMA_DDRC_ERROR;
141 0 : default:
142 0 : return TS_ERROR_HCCL_OTHER_ERROR;
143 : }
144 : }
145 :
146 : // 把UB类错误码转换成Ts对应的错误码
147 0 : uint16_t SwitchUBCqeErrCodeToTsErrCode(u32 cqeErrCode){
148 0 : switch (cqeErrCode) {
149 0 : case RT_UB_LOCAL_OPERATIOINERR:
150 0 : return TS_ERROR_HCCL_OP_UB_DDRC_FAILED;
151 0 : case RT_UB_REMOTE_OPERATIOINERR:
152 0 : return TS_ERROR_HCCL_OP_UB_POISON_FAILED;
153 0 : case RT_UB_LINK_FAILEDERR:
154 0 : return TS_ERROR_HCCL_OP_UB_LINK_FAILED;
155 0 : default:
156 0 : return TS_ERROR_HCCL_OTHER_ERROR;
157 : }
158 : }
159 :
160 0 : static std::string GetSdmaErrorDesc(u32 errorCode)
161 : {
162 0 : switch (errorCode) {
163 0 : case RT_SDMA_DATAERR:
164 0 : return "read HBM return ERROR";
165 0 : case RT_SDMA_COMPERR:
166 0 : return "write copy timeout or address decode ERROR";
167 0 : case RT_SDMA_COMPDATAERR:
168 0 : return "read copy timeout or read HBM return ERROR";
169 0 : default:
170 0 : return "unknown SDMA ERROR";
171 : }
172 : }
173 :
174 0 : static void ReportSdmaError(u32 localDeviceId, u32 notifyId, u32 tsId, s32 userStreamId, u32 errorCode,
175 : const std::string &remoteRankId, const std::string &groupRankContent)
176 : {
177 0 : std::string baseInfo = "localDeviceId=" + std::to_string(localDeviceId);
178 0 : std::string taskInfo = "notifyId= " + std::to_string(notifyId) + ", tsId=" + std::to_string(tsId) +
179 0 : ", streamId=" + std::to_string(userStreamId) + ", error_code=" + std::to_string(errorCode) +
180 0 : ", description=" + GetSdmaErrorDesc(errorCode);
181 0 : RPT_INPUT_ERR(true, "EI0012", std::vector<std::string>({"remote_rankid", "base_information",
182 : "task_information", "group_rank_content"}),
183 : std::vector<std::string>({remoteRankId, baseInfo, taskInfo, groupRankContent}));
184 0 : }
185 :
186 0 : HcclResult SendTaskExceptionByMBox(const u32 localDeviceId, const u32 notifyId, const u32 tsId,
187 : const s32 userStreamId, const rtLogicCqReport_t* exceptionInfo)
188 : {
189 0 : HCCL_INFO("[SendTaskExceptionByMBox] SendTaskExceptionByMBox start.");
190 0 : ts_aicpu_msg_info_t aicpuSqe = {};
191 0 : u32 hostpid = 0;
192 0 : u32 vf_id = 0;
193 0 : int pid = getpid();
194 0 : HCCL_INFO("SendTaskExceptionByMBox getpid[%d]", pid);
195 : // 调整drvQueryProcessHostPid获取pid和vf_id的值
196 0 : CHK_RET(HrtHalDrvQueryProcessHostPid(pid, nullptr, &vf_id, &hostpid, nullptr));
197 :
198 0 : aicpuSqe.pid = hostpid;
199 0 : aicpuSqe.cmd_type = TS_AICPU_RECORD;
200 0 : aicpuSqe.vf_id = vf_id;
201 0 : aicpuSqe.tid = 0U; // notify is no need tid
202 0 : aicpuSqe.u.aicpu_record.record_type = AICPU_MSG_NOTIFY_RECORD_V2;
203 0 : aicpuSqe.u.aicpu_record.record_id = notifyId;
204 :
205 0 : aicpuSqe.ts_id = static_cast<uint8_t>(tsId);
206 :
207 0 : aicpuSqe.u.aicpu_record.fault_task_id = 0xffffffff;
208 :
209 0 : HCCL_ERROR("[SendTaskExceptionByMBox] exceptionInfo errorType[%u], errorCode[%u]", static_cast<u32>(exceptionInfo->errorType), exceptionInfo->errorCode);
210 0 : if (exceptionInfo->errorType == 1) { // ub类型为1
211 0 : aicpuSqe.u.aicpu_record.ret_code = SwitchUBCqeErrCodeToTsErrCode(exceptionInfo->errorCode & 0xFF);
212 : } else {
213 0 : aicpuSqe.u.aicpu_record.ret_code = SwitchSdmaCqeErrCodeToTsErrCode(exceptionInfo->errorCode);
214 : }
215 :
216 : struct event_summary event;
217 0 : event.dst_engine = TS_CPU;
218 0 : event.policy = ONLY;
219 0 : event.pid = 0;
220 0 : event.grp_id = 0;
221 0 : event.event_id = EVENT_TS_CTRL_MSG;
222 0 : event.subevent_id = 0U;
223 0 : event.msg_len = static_cast<uint32_t>(sizeof(ts_aicpu_msg_info_t));
224 0 : event.msg = reinterpret_cast<char_t *>(&aicpuSqe);
225 0 : auto ret = DlHalFunctionV2::GetInstance().dlHalEschedSubmitEvent(localDeviceId, &event);
226 0 : if (ret != static_cast<int32_t>(DRV_ERROR_NONE)) {
227 0 : HCCL_ERROR("[SendTaskExceptionByMBox]Send msg async to ts failed. ret=%d, streamId=%d, "
228 : "notifyId=%u.", ret, userStreamId, notifyId);
229 0 : return HCCL_E_DRV;
230 : }
231 0 : HCCL_RUN_INFO("[SendTaskExceptionByMBox]Send msg async to ts finished. streamId=%d, notifyId=%u, msg_size=%u, "
232 : "hostpid=%u, vf_id=%u, errCode=%u.", userStreamId, notifyId,
233 : static_cast<uint32_t>(sizeof(ts_aicpu_msg_info_t)), hostpid, vf_id, exceptionInfo->errorCode);
234 0 : return HCCL_SUCCESS;
235 : }
236 :
237 0 : HcclResult SendTaskExceptionByMBox(CommunicatorImplLite *aicpuComm, const rtLogicCqReport_t* exceptionInfo)
238 : {
239 0 : u32 localDeviceId = 0;
240 0 : u32 notifyId = aicpuComm->GetHostDeviceSyncNotifyLiteMgr()->GetHostWaitNotify()->GetId();
241 0 : u32 devPhyId = aicpuComm->GetHostDeviceSyncNotifyLiteMgr()->GetHostWaitNotify()->GetDevPhyId();
242 :
243 0 : HCCL_INFO("[TaskExceptionHandlerLite][SendTaskExceptionByMBox] HostToDeviceLogicId[%u]", devPhyId);
244 0 : auto ret = drvGetLocalDevIDByHostDevID(devPhyId, &localDeviceId);
245 0 : if (ret != 0) {
246 0 : HCCL_ERROR("[TaskExceptionHandlerLite][SendTaskExceptionByMBox] HostToDeviceLogicId[%u] failed.",
247 : devPhyId);
248 0 : return HCCL_E_DRV;
249 : }
250 0 : HCCL_INFO("[TaskExceptionHandlerLite][SendTaskExceptionByMBox] HostToDeviceLogicId[%u], localDeviceid[%u]",
251 : devPhyId, localDeviceId);
252 :
253 0 : CHK_RET(SendTaskExceptionByMBox(localDeviceId, notifyId, 0, aicpuComm->GetUserStreamId(), exceptionInfo));
254 :
255 0 : if (exceptionInfo->errorType != 1) {
256 0 : std::string remoteRankId = std::to_string(aicpuComm->GetMyRank());
257 0 : std::string groupRankContent = "group:[" + aicpuComm->GetId() + "], rankSize[" +
258 0 : std::to_string(aicpuComm->GetRankSize()) + "], rankId[" + std::to_string(aicpuComm->GetMyRank()) + "]";
259 0 : ReportSdmaError(localDeviceId, notifyId, 0, aicpuComm->GetUserStreamId(), exceptionInfo->errorCode,
260 : remoteRankId, groupRankContent);
261 0 : }
262 0 : return HCCL_SUCCESS;
263 : }
264 :
265 0 : string GetOpDataInfo(const TaskInfo& taskInfo)
266 : {
267 0 : if (taskInfo.dfxOpInfo_ == nullptr || taskInfo.dfxOpInfo_->comm_ == nullptr) {
268 0 : HCCL_ERROR("[GetOpDataInfo]TaskInfo communicator is nullptr.");
269 0 : return "";
270 : }
271 0 : CommunicatorImplLite* aicpuComm = static_cast<CommunicatorImplLite*>(taskInfo.dfxOpInfo_->comm_);
272 0 : u32 localDeviceId = 0;
273 0 : u32 devPhyId = aicpuComm->GetDevPhyId();
274 :
275 0 : HCCL_INFO("[GetOpDataInfo] HostToDeviceLogicId[%u]", devPhyId);
276 0 : auto ret = drvGetLocalDevIDByHostDevID(devPhyId, &localDeviceId);
277 0 : if (ret != 0) {
278 0 : HCCL_ERROR("[GetOpDataInfo] HostToDeviceLogicId[%u] failed.", devPhyId);
279 0 : return "";
280 : }
281 0 : return StringFormat("deviceId[%u], %s", localDeviceId, taskInfo.GetOpInfo().c_str());
282 : }
283 :
284 0 : void PrintEid(TaskInfo* taskInfo) {
285 0 : if (taskInfo->taskParam_.taskType == TaskParamType::TASK_UB_REDUCE_INLINE || taskInfo->taskParam_.taskType == TaskParamType::TASK_WRITE_REDUCE_WITH_NOTIFY) {
286 0 : HCCL_ERROR("[PrintEid] Error UB link info: localEid[%s], remoteEid[%s]. ", taskInfo->taskParam_.taskPara.Reduce.locEid.Describe().c_str(),
287 : taskInfo->taskParam_.taskPara.Reduce.rmtEid.Describe().c_str());
288 0 : } else if (taskInfo->taskParam_.taskType == TaskParamType::TASK_UB_INLINE_WRITE || taskInfo->taskParam_.taskType == TaskParamType::TASK_WRITE_WITH_NOTIFY
289 0 : || taskInfo->taskParam_.taskType == TaskParamType::TASK_UB) {
290 0 : HCCL_ERROR("[PrintEid] Error UB link info: localEid[%s], remoteEid[%s]. ", taskInfo->taskParam_.taskPara.DMA.locEid.Describe().c_str(),
291 : taskInfo->taskParam_.taskPara.DMA.rmtEid.Describe().c_str());
292 : }
293 0 : }
294 :
295 0 : void TaskExceptionHandlerLite::Process(CommunicatorImplLite *aicpuComm, rtLogicCqReport_t* exceptionInfo)
296 : {
297 0 : if (exceptionInfo == nullptr) {
298 0 : HCCL_ERROR("Exception process failed, rtExceptionInfo is nullptr.");
299 0 : return;
300 : }
301 : // exceptionInfo->taskId和exceptionInfo->streamId拼成sqeId
302 0 : const u32 sqeId = static_cast<uint32_t>(exceptionInfo->taskId << 16) | static_cast<uint32_t>(exceptionInfo->streamId);
303 0 : const auto curTask = aicpuComm->GetMirrorTaskMgrLite()->GetTaskInfo(exceptionInfo->sqId, sqeId);
304 0 : if (curTask == nullptr) {
305 : // 未找到异常对应的TaskInfo
306 0 : HCCL_ERROR("Exception task not found. streamId(sqId)[%u], taskId(sqeId)[%u].", exceptionInfo->sqId, sqeId);
307 0 : return;
308 : }
309 :
310 : // 每个通信域仅首次上报（N秒快恢时重置）
311 0 : if (!aicpuComm->IsErrorReported()) {
312 : // 1) errorMessage上报
313 0 : ErrorMessageReport errMsgInfo{};
314 0 : auto ret = GenerateErrorMessageReport(aicpuComm, curTask, errMsgInfo, exceptionInfo);
315 0 : if (ret != HCCL_SUCCESS) {
316 0 : THROW<InvalidParamsException>("GenerateErrorMessageReport failed.");
317 : }
318 0 : ret = SendErrorMessageReportToHost(aicpuComm, errMsgInfo);
319 0 : if (ret != HCCL_SUCCESS) {
320 0 : THROW<InvalidParamsException>("SendErrorMessageReportToHost Failed.");
321 : }
322 :
323 : // 2) send mbox to tsfw
324 0 : CHK_RET_THROW(InternalException,
325 : StringFormat("[TaskExceptionHandlerLite][SendTaskExceptionByMBox]group[%s]:"
326 : "Send task exception by mailBox failed, streamId[%d]",
327 : aicpuComm->GetId().c_str(), exceptionInfo->streamId), SendTaskExceptionByMBox(aicpuComm, exceptionInfo));
328 :
329 0 : aicpuComm->SetErrorReported();
330 : }
331 :
332 0 : HCCL_ERROR("[TaskExceptionHandlerLite][%s]Task from HCCL run failed.", __func__);
333 0 : if (curTask->taskParam_.taskType == TaskParamType::TASK_NOTIFY_WAIT) {
334 0 : PrintTaskContextInfo(aicpuComm, exceptionInfo->sqId, sqeId);
335 : }
336 0 : if (curTask->taskParam_.taskType == TaskParamType::TASK_WRITE_WITH_NOTIFY || curTask->taskParam_.taskType == TaskParamType::TASK_WRITE_REDUCE_WITH_NOTIFY
337 0 : || curTask->taskParam_.taskType == TaskParamType::TASK_UB_INLINE_WRITE || curTask->taskParam_.taskType == TaskParamType::TASK_UB_REDUCE_INLINE
338 0 : || curTask->taskParam_.taskType == TaskParamType::TASK_UB) {
339 0 : HCCL_ERROR("[TaskExceptionHandlerLite] ubCqeStatus[%u]", static_cast<u32>(exceptionInfo->errorCode & 0xFF));
340 : }
341 0 : PrintEid(curTask);
342 0 : HCCL_ERROR("[TaskExceptionHandlerLite]Task run failed, base information is %s.", curTask->GetBaseInfo().c_str());
343 0 : HCCL_ERROR("[TaskExceptionHandlerLite]Task run failed, para information is %s.", curTask->GetParaInfo().c_str());
344 0 : HCCL_ERROR("[TaskExceptionHandlerLite]Task run failed, groupRank information is %s.", GetGroupRankInfo(*curTask).c_str());
345 0 : if(curTask->dfxOpInfo_ != nullptr && curTask->dfxOpInfo_->headOpCounterAddr_ != 0 && curTask->dfxOpInfo_->tailOpCounterAddr_ != 0) {
346 0 : HCCL_ERROR("[TaskExceptionHandlerLite]Task run failed, headOpCounter[%u] tailOpCounter[%u] opIndex[%u].",
347 : static_cast<u32>(*(reinterpret_cast<float *>(curTask->dfxOpInfo_->headOpCounterAddr_))),
348 : static_cast<u32>(*(reinterpret_cast<float *>(curTask->dfxOpInfo_->tailOpCounterAddr_))),
349 : curTask->dfxOpInfo_->opIndex_);
350 : }
351 0 : HCCL_ERROR("[TaskExceptionHandlerLite]Task run failed, opData information is %s.", GetOpDataInfo(*curTask).c_str());
352 : }
353 :
354 0 : string TaskExceptionHandlerLite::GetGroupRankInfo(const TaskInfo& taskInfo)
355 : {
356 0 : if (taskInfo.dfxOpInfo_ == nullptr || taskInfo.dfxOpInfo_->comm_ == nullptr) {
357 0 : HCCL_ERROR("[TaskInfo][%s]TaskInfo communicator is nullptr.", __func__);
358 0 : return "";
359 : }
360 0 : const CommunicatorImplLite* commImplLite = static_cast<CommunicatorImplLite*>(taskInfo.dfxOpInfo_->comm_);
361 : return StringFormat("group:[%s], rankSize[%u], localRank[%d], remoteRank[%u]",
362 0 : commImplLite->GetId().c_str(), commImplLite->GetRankSize(), commImplLite->GetMyRank(), taskInfo.remoteRank_);
363 : }
364 :
365 0 : void TaskExceptionHandlerLite::PrintTaskContextInfo(CommunicatorImplLite *aicpuComm, uint32_t sqId, uint32_t taskId)
366 : {
367 0 : auto queue = aicpuComm->GetMirrorTaskMgrLite()->GetQueue(sqId);
368 0 : if (queue == nullptr) {
369 : // 未找到异常对应的TaskQueue
370 0 : HCCL_ERROR("Exception task queue not found. streamId(sqId)[%u].", sqId);
371 0 : return;
372 : }
373 :
374 0 : auto func = [taskId] (const unique_ptr<TaskInfo>& task) { return task->taskId_ == taskId; };
375 0 : auto taskItorPtr = queue->Find(func);
376 0 : if (taskItorPtr == nullptr || *taskItorPtr == *queue->End()) {
377 : // 在队列中未找到异常对应的TaskInfo
378 0 : HCCL_ERROR("Exception task not found. streamId(sqId)[%u], taskId(sqeId)[%u].", sqId, taskId);
379 0 : return;
380 : }
381 :
382 : // 找到当前异常task的前50个task(至多)
383 0 : vector<TaskInfo*> taskContext {};
384 0 : for (uint32_t i = 0; i < TASK_CONTEXT_SIZE && *taskItorPtr != *queue->Begin(); ++i, --(*taskItorPtr)) {
385 0 : if ((**taskItorPtr)->taskId_ > taskId) {
386 0 : break;
387 : }
388 0 : if ((**taskItorPtr)->taskId_ != taskId) {
389 0 : taskContext.emplace_back((**taskItorPtr).get());
390 : }
391 : }
392 :
393 0 : if (taskContext.empty()) {
394 0 : return;
395 : }
396 :
397 0 : HCCL_ERROR("[TaskExceptionHandlerLite]Task run failed, context sequence before error task is "
398 : "[SDMA:M(rank), RDMA:RS(rank,id), SendPayload:SP(rank), InlineReduce:IR(rank), Reduce:R(rank), "
399 : "NotifyRecord:NR(rank,id), NotifyWait:NW(rank,id), SendNotify:SN(rank,id), "
400 : "WriteWithNotify:WN(rank,id), WriteReduceWithNotify:WRN(rank,id)]:");
401 :
402 0 : string taskContextInfo = "";
403 0 : for (auto it = taskContext.rbegin(); it != taskContext.rend(); ++it) {
404 0 : string conciseInfo = (*it)->GetConciseBaseInfo();
405 0 : conciseInfo += ",";
406 :
407 0 : if (taskContextInfo.size() + conciseInfo.size() >= TASK_CONTEXT_INFO_SIZE) {
408 0 : HCCL_ERROR("[TaskExceptionHandlerLite]%s", taskContextInfo.c_str());
409 0 : taskContextInfo = "";
410 : }
411 :
412 0 : taskContextInfo += conciseInfo;
413 0 : }
414 0 : HCCL_ERROR("[TaskExceptionHandlerLite]%s end.", taskContextInfo.c_str());
415 0 : }
416 :
417 : } // namespace Hccl
|
