Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 : #include "hcclCommTaskException.h"
11 : #include <memory>
12 : #include "log.h"
13 : #include "coll_comm.h"
14 : #include "acl/acl_rt.h"
15 : #include "orion_adapter_hccp.h"
16 : #include "hcomm_adapter_hccp.h"
17 : #include <adapter_error_manager_pub.h>
18 : #include "op_type.h"
19 : #include "task_exception_handler.h"
20 : #include "ccuTaskException.h"
21 : #include "hccl_types.h"
22 : #include "dpu_kernel_entrance.h"
23 :
24 : namespace hcomm {
25 :
26 : using namespace std;
27 :
28 : constexpr u32 MAX_MODULE_DEVICE_NUM_V2 = 65;
29 : constexpr uint32_t TASK_CONTEXT_SIZE = 50;
30 : constexpr uint32_t TASK_CONTEXT_INFO_SIZE = LOG_TMPBUF_SIZE - 50; // task 执行失败时打印前序task信息的长度限制
31 :
32 : static TaskExceptionHost handlers_[MAX_MODULE_DEVICE_NUM_V2];
33 : GetAicpuCqeErrInfoCallBackHcomm g_getAicpuCqeErrInfoCallBack = nullptr;
34 : AicpuGetErrStatusVecCallBack g_AicpuGetErrStatusVecCallBack = nullptr;
35 :
36 43 : void RegisterGetAicpuCqeErrInfoCallBackHcomm(GetAicpuCqeErrInfoCallBackHcomm p1)
37 : {
38 43 : g_getAicpuCqeErrInfoCallBack = p1;
39 43 : return;
40 : }
41 :
42 0 : void TaskExceptionHost::ClusterMoniterGetAicpuCqeErrInfo(u32 remoteLocalId, u32 locDeviceId, uint16_t status, string localEid, string remoteEid, string remoteInsId) const
43 : {
44 0 : if (g_getAicpuCqeErrInfoCallBack != nullptr) {
45 0 : g_getAicpuCqeErrInfoCallBack(remoteLocalId, locDeviceId, status, localEid, remoteEid, remoteInsId);
46 : } else {
47 0 : HCCL_RUN_WARNING("[ClusterMoniterGetAicpuCqeErrInfo]g_getAicpuCqeErrInfoCallBack is nullptr.");
48 : }
49 0 : return;
50 : }
51 :
52 43 : void RegisterAicpuGetErrStatusVecCallBack(AicpuGetErrStatusVecCallBack p1)
53 : {
54 43 : g_AicpuGetErrStatusVecCallBack = p1;
55 43 : return;
56 : }
57 :
58 0 : std::vector<std::string> AicpuGetErrStatusVec(s32 deviceLogicID)
59 : {
60 0 : if (g_AicpuGetErrStatusVecCallBack != nullptr) {
61 0 : return g_AicpuGetErrStatusVecCallBack(deviceLogicID);
62 : } else {
63 0 : HCCL_RUN_WARNING("[AicpuGetErrStatusVec]g_AicpuGetErrStatusVecCallBack is nullptr.");
64 : }
65 0 : return std::vector<std::string>();
66 : }
67 :
68 0 : std::string AicpuGetAndPrintClusterMonitorErr(const rtExceptionInfo *exceptionInfo)
69 : {
70 0 : auto errStatusVec = AicpuGetErrStatusVec(exceptionInfo->deviceid);
71 0 : std::string errMsg = "";
72 0 : int errSize = errStatusVec.size();
73 0 : if (errSize > 0) {
74 0 : int maxListSize = 3; // 放入errMsg中的异常事件最多只有3个
75 0 : if (errSize <= maxListSize) {
76 0 : errMsg = "\nthere are(is) " + std::to_string(errSize) + " abnormal device(s):\n";
77 : } else {
78 0 : errMsg = "\nthere are " + std::to_string(errSize) + " abnormal device(s), " +
79 0 : "only the first 3 devices are listed:\n";
80 : }
81 :
82 0 : for (int i = 0; i < errSize; i++) {
83 0 : HCCL_ERROR("%s", errStatusVec[i].c_str());
84 0 : if (i < maxListSize) {
85 0 : errMsg += ("\t" + errStatusVec[i] + "\n");
86 : }
87 : }
88 : }
89 0 : return errMsg;
90 0 : }
91 :
92 244 : TaskExceptionHost *TaskExceptionHost::GetInstance(s32 deviceLogicID)
93 : {
94 244 : if (deviceLogicID < 0 || static_cast<u32>(deviceLogicID) >= MAX_MODULE_DEVICE_NUM_V2) {
95 0 : HCCL_ERROR("[TaskExceptionHost][GetInstance] deviceLogicID[%d] is invalid", deviceLogicID);
96 0 : return nullptr;
97 : }
98 244 : return &handlers_[deviceLogicID];
99 : }
100 :
101 2795 : TaskExceptionHost::~TaskExceptionHost()
102 : {
103 2795 : std::unique_lock<std::mutex> lock(taskExceptionMutex_);
104 2795 : if (!CommRegisterMap_.empty()) {
105 0 : CommRegisterMap_.clear();
106 0 : aclError ret = aclrtSetExceptionInfoCallback(nullptr); // 把注册给rts的TaskException回调函数指针置空
107 0 : HCCL_RUN_INFO("[%s]aclrtSetExceptionInfoCallback set nullptr, ret[%d]", __func__, ret);
108 : }
109 2795 : }
110 :
111 1 : void TaskExceptionHost::ProcessCallback(rtExceptionInfo_t *exceptionInfo)
112 : {
113 1 : CHK_PRT_RET(exceptionInfo == nullptr, HCCL_ERROR("[%s]fail, exceptionInfo is nullptr", __func__),);
114 :
115 0 : TaskExceptionHost *handler = TaskExceptionHost::GetInstance(exceptionInfo->deviceid);
116 0 : CHK_PRT_RET(handler == nullptr, HCCL_ERROR("[%s]fail, TaskExceptionHost is nullptr", __func__),);
117 0 : handler->Process(exceptionInfo);
118 : }
119 :
120 98 : HcclResult TaskExceptionHost::Register(u64 commHandle)
121 : {
122 98 : std::unique_lock<std::mutex> lock(taskExceptionMutex_);
123 98 : if (CommRegisterMap_.empty()) {
124 97 : aclError ret = aclrtSetExceptionInfoCallback(ProcessCallback);
125 97 : CHK_PRT_RET(ret != ACL_SUCCESS,
126 : HCCL_ERROR("[%s]aclrtSetExceptionInfoCallback failed, ret[%d]", __func__, ret), HCCL_E_RUNTIME);
127 97 : HCCL_RUN_INFO("[%s]aclrtSetExceptionInfoCallback set ProcessCallback success", __func__);
128 : }
129 :
130 98 : CommRegisterMap_.insert(commHandle);
131 98 : HCCL_INFO("[%s]success, commHandle[0x%llx]", __func__, commHandle);
132 98 : return HCCL_SUCCESS;
133 98 : }
134 :
135 134 : HcclResult TaskExceptionHost::UnRegister(u64 commHandle)
136 : {
137 134 : std::unique_lock<std::mutex> lock(taskExceptionMutex_);
138 134 : if (CommRegisterMap_.find(commHandle) == CommRegisterMap_.end()) {
139 36 : HCCL_INFO("[%s] commHandle[0x%llx] has not registered, skip", __func__, commHandle);
140 36 : return HCCL_SUCCESS;
141 : }
142 :
143 98 : CommRegisterMap_.erase(commHandle);
144 98 : if (CommRegisterMap_.empty()) {
145 97 : aclError ret = aclrtSetExceptionInfoCallback(nullptr); // 把注册给rts的TaskException回调函数指针置空
146 97 : CHK_PRT_RET(ret != ACL_SUCCESS,
147 : HCCL_ERROR("[%s]aclrtSetExceptionInfoCallback failed, ret[%d]", __func__, ret), HCCL_E_RUNTIME);
148 97 : HCCL_RUN_INFO("[%s]aclrtSetExceptionInfoCallback set nullptr success", __func__);
149 : }
150 :
151 98 : HCCL_INFO("[%s]success, commHandle[0x%llx]", __func__, commHandle);
152 98 : return HCCL_SUCCESS;
153 134 : }
154 :
155 0 : HcclResult TaskExceptionHost::PrintUbRegisters(s32 devLogicId, const RdmaHandle rdmaHandle) const
156 : {
157 0 : HCCL_INFO("[PrintUbRegister] start, devLogicId[%d], rdmaHandle[%p]", devLogicId, rdmaHandle);
158 0 : Hccl::AuxInfoIn in;
159 0 : in.cqe.status = 0xffffffff; // 0xffffffff代表查询所有寄存器
160 0 : in.auxInfoInType = Hccl::AuxInfoInType::AUX_INFO_IN_TYPE_CQE;
161 0 : in.cqe.sR = 0;
162 0 : Hccl::AuxInfoOut auxInfo;
163 0 : auto ret = Hccl::RaGetAuxInfo(rdmaHandle, in, auxInfo);
164 0 : if (ret != HCCL_SUCCESS) {
165 0 : HCCL_ERROR("[PrintUbRegister]GetUbRegisterInfo failed, devLogicId[%d], rdmaHandle[%p]", devLogicId, rdmaHandle);
166 0 : return ret;
167 : }
168 :
169 0 : bool isAuxInfoExisted{false};
170 0 : for (u32 i = 0; i < auxInfo.auxInfoNum; i++) {
171 0 : if (auxInfo.auxInfoValues[i] != 0) { // 非零进行打印
172 0 : isAuxInfoExisted = true;
173 0 : HCCL_ERROR("devLogicId[%d], cqe_aux_info_type[%u], cqe_aux_info_value[0x%x]",
174 : devLogicId, auxInfo.auxInfoTypes[i], auxInfo.auxInfoValues[i]);
175 : } else {
176 0 : HCCL_INFO("devLogicId[%d], cqe_aux_info_type[%u], cqe_aux_info_value[0x%x]",
177 : devLogicId, auxInfo.auxInfoTypes[i], auxInfo.auxInfoValues[i]);
178 : }
179 : }
180 0 : if (!isAuxInfoExisted) {
181 0 : HCCL_ERROR("devLogicId[%d], all aux_info values are zero.", devLogicId);
182 : }
183 0 : return HCCL_SUCCESS;
184 : }
185 :
186 7 : bool IsMC2Exception(const rtExceptionInfo_t* exceptionInfo)
187 : {
188 7 : return exceptionInfo->expandInfo.type == RT_EXCEPTION_FUSION &&
189 7 : exceptionInfo->expandInfo.u.fusionInfo.type == RT_FUSION_AICORE_CCU;
190 : }
191 :
192 7 : bool TaskExceptionHost::ProcessDpuException(const rtExceptionInfo_t* exceptionInfo) const
193 : {
194 7 : HCCL_RUN_INFO("[TaskExceptionHost][%s]begin to execute hccl task exception callback function.", __func__);
195 7 : bool isExce = false;
196 7 : errno_t ret = EOK;
197 7 : uint16_t hcclRet = 0;
198 7 : for (auto pairMap : g_taskExpMemMap) {
199 0 : std::string commId = pairMap.first;
200 : // 读取共享内存内容并打印
201 0 : ret = memcpy_s(&hcclRet, sizeof(uint16_t), reinterpret_cast<uint8_t *>(pairMap.second[exceptionInfo->deviceid]) + sizeof(uint8_t) + sizeof(uint16_t), sizeof(uint16_t));
202 0 : if (ret != EOK) {
203 0 : HCCL_ERROR("memcpy_s get dpu taskexception failed: %d", ret);
204 0 : return isExce;
205 : }
206 0 : if (hcclRet != 0) { // 有dpu任务出错
207 0 : HCCL_ERROR("[TaskExceptionHost][ProcessDpuException] Task from HCCL run failed.");
208 0 : HCCL_ERROR("[TaskExceptionHost][ProcessDpuException] errorCode[%d], devId[%u], commId[%s]", hcclRet, exceptionInfo->deviceid, commId.c_str());
209 0 : ret = memset_s(reinterpret_cast<uint8_t *>(pairMap.second[exceptionInfo->deviceid]) + sizeof(uint8_t) + sizeof(uint16_t), sizeof(uint16_t), 0, sizeof(uint16_t)); // 清空 dpu taskexception共享内存内容
210 0 : if (ret != EOK) {
211 0 : HCCL_ERROR("memset_s clean dpu taskexception failed: %d", ret);
212 0 : return isExce;
213 : }
214 0 : isExce = true;
215 0 : break;
216 : }
217 0 : }
218 7 : return isExce;
219 : }
220 :
221 7 : void TaskExceptionHost::Process(rtExceptionInfo_t* exceptionInfo)
222 : {
223 7 : HCCL_RUN_INFO("[TaskExceptionHost][%s], taskid[%u], streamid[%u], tid[%u], deviceid[%u], retcode[%u], type[%d]",
224 : __func__, exceptionInfo->taskid, exceptionInfo->streamid, exceptionInfo->tid, exceptionInfo->deviceid,
225 : exceptionInfo->retcode, exceptionInfo->expandInfo.type);
226 :
227 7 : if (IsMC2Exception(exceptionInfo)) { // MC2 taskException 新流程暂未支持，回退到老流程
228 0 : Hccl::TaskExceptionHandler::Process(exceptionInfo);
229 7 : return;
230 : }
231 :
232 : // dpu taskexception
233 7 : if (ProcessDpuException(exceptionInfo)) {
234 0 : HCCL_ERROR("[TaskExceptionHost][ProcessDpuException] end.");
235 0 : return;
236 : };
237 :
238 7 : Hccl::TaskInfo* curTask = nullptr;
239 7 : HcclResult ret = Hccl::GlobalMirrorTasks::Instance().FindTaskInfo(exceptionInfo->deviceid, exceptionInfo->streamid,
240 : exceptionInfo->taskid, curTask);
241 7 : CHK_PRT_RET(ret == HCCL_E_NOT_FOUND, HCCL_RUN_WARNING("[%s]FindTaskInfo not found, deviceid[%u] streamid[%u] taskid[%u].",
242 : __func__, exceptionInfo->deviceid, exceptionInfo->streamid, exceptionInfo->taskid),);
243 :
244 0 : CHK_PRT_RET(ret != HCCL_SUCCESS || curTask == nullptr,
245 : HCCL_ERROR("[%s]FindTaskInfo fail, ret[%d], deviceid[%u], streamid[%u], taskid[%u].",
246 : __func__, ret, exceptionInfo->deviceid, exceptionInfo->streamid, exceptionInfo->taskid),);
247 :
248 0 : CHK_PRT_RET(curTask->dfxOpInfo_ == nullptr, HCCL_ERROR("[%s]fail, dfxOpInfo is nullptr", __func__),);
249 0 : bool isIndop_ = curTask->dfxOpInfo_->isIndop_;
250 0 : HCCL_INFO("[%s]isIndop_[%d], taskType[%s]", __func__, isIndop_, curTask->taskParam_.taskType.Describe().c_str());
251 :
252 : // 老流程TaskException打印
253 0 : if (!isIndop_) {
254 0 : Hccl::TaskExceptionHandler::Process(exceptionInfo);
255 0 : return;
256 : }
257 :
258 : // 新流程支持的TaskException打印
259 0 : std::unique_lock<std::mutex> lock(taskExceptionMutex_);
260 0 : u64 commHandle = reinterpret_cast<u64>(curTask->dfxOpInfo_->comm_);
261 0 : if (curTask->dfxOpInfo_->comm_ == nullptr || CommRegisterMap_.find(commHandle) == CommRegisterMap_.end()) {
262 0 : HCCL_ERROR("[TaskExceptionHost][%s] commHandle[0x%llx] not exist, "
263 : "the comm may have been destroyed or was never registered", __func__, commHandle);
264 0 : return;
265 : }
266 :
267 0 : if (curTask->taskParam_.taskType == Hccl::TaskParamType::TASK_CCU) {
268 0 : CcuTaskException::ProcessCcuException(exceptionInfo, *curTask);
269 : } else {
270 0 : ProcessException(exceptionInfo, *curTask);
271 : }
272 0 : }
273 :
274 0 : std::string TaskExceptionHost::GetGroupRankInfo(const Hccl::TaskInfo& taskInfo) const
275 : {
276 0 : if (taskInfo.dfxOpInfo_ == nullptr || taskInfo.dfxOpInfo_->comm_ == nullptr) {
277 0 : HCCL_ERROR("[TaskInfo][%s]TaskInfo communicator is nullptr.", __func__);
278 0 : return "";
279 : }
280 :
281 0 : hccl::CollComm *communicator = static_cast<hccl::CollComm*>(taskInfo.dfxOpInfo_->comm_);
282 : return Hccl::StringFormat("group:[%s], rankSize[%u], rankId[%u]",
283 0 : communicator->GetCommId().c_str(), communicator->GetRankSize(), communicator->GetMyRankId());
284 : }
285 :
286 0 : void TaskExceptionHost::GetAicpuCqeErrRemoteLocalIdByRankId(hccl::CollComm* collComm, uint32_t rankid, u32 &remoteLocalId) const
287 : {
288 0 : if (collComm == nullptr || rankid == INVALID_VALUE_RANKID) {
289 0 : HCCL_ERROR("[GetAicpuCqeErrRemoteLocalIdByRankId]collComm is nullptr or rankId is invalid, rankId[%u]", rankid);
290 0 : remoteLocalId = INVALID_VALUE_RANKID;
291 0 : return;
292 : }
293 :
294 0 : Hccl::HcclCommunicator *commV2 = static_cast<Hccl::HcclCommunicator *>(collComm->GetCommunicatorV2());
295 0 : if (commV2 == nullptr) {
296 0 : HCCL_ERROR("[GetAicpuCqeErrRemoteLocalIdByRankId]commV2 is nullptr, rankId[%u]", rankid);
297 0 : remoteLocalId = INVALID_VALUE_RANKID;
298 0 : return;
299 : }
300 0 : void *rankGraph = nullptr;
301 0 : HcclResult ret =commV2->GetRankGraphV2(rankGraph);
302 0 : if (ret != HCCL_SUCCESS) {
303 0 : HCCL_ERROR("[GetAicpuCqeErrRemoteLocalIdByRankId]GetRankGraphV2 failed, rankId[%u], ret[%d]", rankid, ret);
304 0 : remoteLocalId = INVALID_VALUE_RANKID;
305 0 : return;
306 : }
307 0 : Hccl::RankGraph *rankGraphv2 = static_cast<Hccl::RankGraph *>(rankGraph);
308 0 : u32 LocalId = rankGraphv2->GetLocalId(rankid);
309 0 : remoteLocalId = LocalId;
310 0 : return;
311 : }
312 :
313 0 : void TaskExceptionHost::GetAicpuCqeErrNetInstanceByRankId(hccl::CollComm* collComm, uint32_t rankid, std::string &netInstanceId) const
314 : {
315 0 : if (collComm == nullptr || rankid == INVALID_VALUE_RANKID) {
316 0 : HCCL_ERROR("[GetAicpuCqeErrNetInstanceByRankId]collComm is nullptr or rankId is invalid, rankId[%u]", rankid);
317 0 : netInstanceId = "";
318 0 : return;
319 : }
320 :
321 0 : Hccl::HcclCommunicator *commV2 = static_cast<Hccl::HcclCommunicator *>(collComm->GetCommunicatorV2());
322 0 : if (commV2 == nullptr) {
323 0 : HCCL_ERROR("[GetAicpuCqeErrNetInstanceByRankId]commV2 is nullptr, rankId[%u]", rankid);
324 0 : netInstanceId = "";
325 0 : return;
326 : }
327 0 : void *rankGraph = nullptr;
328 0 : HcclResult ret = commV2->GetRankGraphV2(rankGraph);
329 0 : if (ret != HCCL_SUCCESS) {
330 0 : HCCL_ERROR("[GetAicpuCqeErrNetInstanceByRankId]GetRankGraphV2 failed, rankId[%u], ret[%d]", rankid, ret);
331 0 : netInstanceId = "";
332 0 : return;
333 : }
334 0 : Hccl::RankGraph *rankGraphv2 = static_cast<Hccl::RankGraph *>(rankGraph);
335 0 : const Hccl::NetInstance *netInstance = rankGraphv2->GetNetInstanceByRankId(0, rankid);
336 0 : if (netInstance == nullptr) {
337 0 : HCCL_ERROR("[GetAicpuCqeErrNetInstanceByRankId]netInstance is nullptr, rankId[%u]", rankid);
338 0 : netInstanceId = "";
339 0 : return;
340 : }
341 0 : std::string netInsId = netInstance->GetNetInstId();
342 0 : netInstanceId = netInsId;
343 0 : return;
344 0 : }
345 :
346 0 : void TaskExceptionHost::GetAicpuCqeErrInfo(rtExceptionInfo_t* exceptionInfo, const Hccl::ErrorMessageReport &errorMessage, const Hccl::TaskInfo& taskInfo) const
347 : {
348 0 : hccl::CollComm *collComm = static_cast<hccl::CollComm*>(taskInfo.dfxOpInfo_->comm_);
349 0 : u32 remoteLocalId = INVALID_VALUE_RANKID;
350 0 : GetAicpuCqeErrRemoteLocalIdByRankId(collComm, errorMessage.remoteUserRank, remoteLocalId);
351 0 : std::string netInstanceId = "";
352 0 : GetAicpuCqeErrNetInstanceByRankId(collComm, errorMessage.remoteUserRank, netInstanceId);
353 0 : ClusterMoniterGetAicpuCqeErrInfo(remoteLocalId, exceptionInfo->deviceid, errorMessage.ubCqeStatus, errorMessage.locEid.Describe(), errorMessage.rmtEid.Describe(), netInstanceId); // 上报AICPU CQE错误信息到集群监控
354 0 : return;
355 0 : }
356 :
357 0 : void TaskExceptionHost::ProcessException(rtExceptionInfo_t* exceptionInfo, const Hccl::TaskInfo& taskInfo)
358 : {
359 0 : HCCL_RUN_INFO("[TaskExceptionHost][%s]begin to execute hccl task exception callback function.", __func__);
360 0 : Hccl::ErrorMessageReport errorMessage;
361 0 : if (!hasAicpuReport_) { // 防止aicpu task exception重复上报
362 0 : hccl::CollComm *communicator = static_cast<hccl::CollComm*>(taskInfo.dfxOpInfo_->comm_);
363 0 : errorMessage = communicator->GetAicpuTaskException();
364 : }
365 :
366 0 : if (strlen(errorMessage.tag) > 0) {
367 0 : hasAicpuReport_ = true;
368 0 : HandleAicpuErrorReport(exceptionInfo, errorMessage, taskInfo);
369 : } else {
370 0 : HandleHostErrorReport(exceptionInfo, taskInfo);
371 : }
372 0 : }
373 :
374 0 : void TaskExceptionHost::HandleHostErrorReport(rtExceptionInfo_t *exceptionInfo, const Hccl::TaskInfo &taskInfo) const
375 : {
376 0 : HCCL_ERROR("[TaskExceptionHost][%s]Task from HCCL run failed.", __func__);
377 0 : if (taskInfo.taskParam_.taskType == Hccl::TaskParamType::TASK_NOTIFY_WAIT) {
378 0 : PrintTaskContextInfo(exceptionInfo->deviceid, exceptionInfo->streamid, exceptionInfo->taskid);
379 0 : RPT_INPUT_ERR(true,
380 : "EI0002",
381 : std::vector<std::string>({"remote_rankid", "base_information", "task_information", "group_rank_content"}),
382 : std::vector<std::string>({
383 : std::to_string(taskInfo.remoteRank_),
384 : taskInfo.GetBaseInfo(), (taskInfo.GetParaInfo()),
385 : ""})
386 : );
387 : }
388 0 : std::string clusterMonitorErrMsg = AicpuGetAndPrintClusterMonitorErr(exceptionInfo);
389 0 : HCCL_ERROR("[TaskExceptionHost]Task run failed, base information is deviceID:[%u], %s.",
390 : exceptionInfo->deviceid, taskInfo.GetIndopBaseInfo().c_str());
391 0 : HCCL_ERROR("[TaskExceptionHost]Task run failed, para information is %s.", taskInfo.GetParaInfo().c_str());
392 0 : HCCL_ERROR("[TaskExceptionHost]Task run failed, groupRank information is %s.",
393 : GetGroupRankInfo(taskInfo).c_str());
394 0 : HCCL_ERROR("[TaskExceptionHost]Task run failed, opData information is %s.", taskInfo.GetIndopDataInfo().c_str());
395 0 : }
396 :
397 4 : void TaskExceptionHost::PrintTaskContextInfo(uint32_t deviceId, uint32_t streamId, uint32_t taskId) const
398 : {
399 4 : Hccl::TaskInfoQueue *queue = nullptr;
400 : try {
401 4 : queue = Hccl::GlobalMirrorTasks::Instance().GetQueue(deviceId, streamId);
402 0 : } catch (Hccl::HcclException &e) {
403 0 : HCCL_ERROR("Exception task queue not found. deviceId[%u], streamId[%u].", deviceId, streamId);
404 0 : return ;
405 0 : }
406 :
407 4 : if (queue == nullptr) {
408 : // 未找到异常对应的TaskQueue
409 0 : HCCL_ERROR("Exception task queue not found. deviceId[%u], streamId[%u].", deviceId, streamId);
410 0 : return;
411 : }
412 :
413 31 : auto func = [taskId] (const unique_ptr<Hccl::TaskInfo>& task) { return task->taskId_ == taskId; };
414 4 : auto taskIterPtr = queue->Find(func);
415 4 : if (taskIterPtr == nullptr || *taskIterPtr == *queue->End()) {
416 : // 在队列中未找到异常对应的TaskInfo
417 1 : HCCL_ERROR("Exception task not found. deviceId[%u], streamId[%u], taskId[%u].", deviceId, streamId, taskId);
418 1 : return;
419 : }
420 :
421 : // 找到当前异常task的前50个task(至多)
422 3 : vector<Hccl::TaskInfo*> taskContext {};
423 10 : for (uint32_t i = 0; i < TASK_CONTEXT_SIZE && *taskIterPtr != *queue->Begin(); ++i, --(*taskIterPtr)) {
424 8 : if ((**taskIterPtr)->taskId_ > taskId) {
425 1 : HCCL_ERROR("[%s]prev taskId[%u]is bigger than err taskId[%u], traversal end.",
426 : __func__, (**taskIterPtr)->taskId_, taskId);
427 1 : break;
428 : }
429 7 : taskContext.emplace_back((**taskIterPtr).get());
430 : }
431 :
432 3 : HCCL_ERROR("[TaskExceptionHost]Task run failed, context sequence before error task is "
433 : "[SDMA:M(rank), RDMA:RS(rank,id), SendPayload:SP(rank), InlineReduce:IR(rank), Reduce:R(rank), "
434 : "NotifyRecord:NR(rank,id), NotifyWait:NW(rank,id), SendNotify:SN(rank,id), "
435 : "WriteWithNotify:WN(rank,id), WriteReduceWithNotify:WRN(rank,id)]:");
436 :
437 3 : std::string taskContextInfo = "";
438 10 : for (auto it = taskContext.rbegin(); it != taskContext.rend(); ++it) {
439 7 : std::string conciseInfo = (*it)->GetConciseBaseInfo();
440 7 : conciseInfo += ",";
441 :
442 7 : if (taskContextInfo.size() + conciseInfo.size() >= TASK_CONTEXT_INFO_SIZE) {
443 0 : HCCL_ERROR("[TaskExceptionHost]%s", taskContextInfo.c_str());
444 0 : taskContextInfo = "";
445 : }
446 :
447 7 : taskContextInfo += conciseInfo;
448 7 : }
449 3 : HCCL_ERROR("[TaskExceptionHost]%s end.", taskContextInfo.c_str());
450 4 : }
451 :
452 :
453 0 : inline void PrintBaseErrorLog(const std::string &stageErrInfo, const std::string &baseInfo)
454 : {
455 0 : HCCL_ERROR("%sTask run failed, base information is %s", stageErrInfo.c_str(), baseInfo.c_str());
456 0 : }
457 :
458 0 : inline void PrintParaErrorLog(const std::string &stageErrInfo, const std::string ¶InfoStr)
459 : {
460 0 : HCCL_ERROR("%sTask run failed, para information is %s.", stageErrInfo.c_str(), paraInfoStr.c_str());
461 0 : }
462 :
463 0 : inline void PrintOpDataErrorLog(const std::string &stageErrInfo, const std::string &opDataContent)
464 : {
465 0 : HCCL_ERROR("%sTask run failed, opData information is %s", stageErrInfo.c_str(), opDataContent.c_str());
466 0 : }
467 :
468 0 : inline void PrintGroupErrorLog(const std::string &stageErrInfo, const std::string &groupRankContent)
469 : {
470 0 : HCCL_ERROR("%sTask run failed, groupRank information is %s.", stageErrInfo.c_str(), groupRankContent.c_str());
471 0 : }
472 :
473 0 : void TaskExceptionHost::PrintGroupErrorMessage(const Hccl::ErrorMessageReport &errorMessage, [[maybe_unused]] const Hccl::TaskInfo &exceptionTaskInfo,
474 : std::string &groupRankContent, std::string &stageErrInfo) const
475 : {
476 0 : groupRankContent += "group:[";
477 0 : groupRankContent += std::string(errorMessage.group);
478 0 : groupRankContent += "], rankSize[";
479 0 : groupRankContent += std::to_string(errorMessage.rankSize);
480 0 : groupRankContent += "], localRank[";
481 0 : groupRankContent += std::to_string(errorMessage.rankId);
482 0 : groupRankContent += "], remoteRank[";
483 0 : groupRankContent += std::to_string(errorMessage.remoteUserRank);
484 0 : groupRankContent += "]";
485 :
486 0 : PrintGroupErrorLog(stageErrInfo, groupRankContent);
487 0 : return;
488 : }
489 :
490 : const std::map<HcclReduceOp, std::string> HCOM_REDUCE_OP_STR_MAP{
491 : {HcclReduceOp::HCCL_REDUCE_SUM, "sum"},
492 : {HcclReduceOp::HCCL_REDUCE_PROD, "prod"},
493 : {HcclReduceOp::HCCL_REDUCE_MAX, "max"},
494 : {HcclReduceOp::HCCL_REDUCE_MIN, "min"},
495 : {HcclReduceOp::HCCL_REDUCE_RESERVED, "invalid"}
496 : };
497 :
498 0 : inline std::string GetReduceOpEnumStr2(HcclReduceOp reduceOp)
499 : {
500 0 : auto iter = HCOM_REDUCE_OP_STR_MAP.find(reduceOp);
501 0 : if (iter == HCOM_REDUCE_OP_STR_MAP.end()) {
502 0 : return "HcclReduceOp(" + std::to_string(reduceOp) + ")";
503 : } else {
504 0 : return iter->second;
505 : }
506 : }
507 :
508 : const std::map<HcclDataType, std::string> HCOM_DATA_TYPE_STR_MAP{
509 : {HcclDataType::HCCL_DATA_TYPE_INT8, "int8"},
510 : {HcclDataType::HCCL_DATA_TYPE_INT16, "int16"},
511 : {HcclDataType::HCCL_DATA_TYPE_INT32, "int32"},
512 : {HcclDataType::HCCL_DATA_TYPE_INT64, "int64"},
513 : {HcclDataType::HCCL_DATA_TYPE_UINT64, "uint64"},
514 : {HcclDataType::HCCL_DATA_TYPE_FP16, "float16"},
515 : {HcclDataType::HCCL_DATA_TYPE_FP32, "float32"},
516 : {HcclDataType::HCCL_DATA_TYPE_UINT8, "uint8"},
517 : {HcclDataType::HCCL_DATA_TYPE_UINT16, "uint16"},
518 : {HcclDataType::HCCL_DATA_TYPE_UINT32, "uint32"},
519 : {HcclDataType::HCCL_DATA_TYPE_FP64, "float64"},
520 : {HcclDataType::HCCL_DATA_TYPE_BFP16, "bfloat16"},
521 : {HcclDataType::HCCL_DATA_TYPE_INT128, "int128"},
522 : {HcclDataType::HCCL_DATA_TYPE_FP8E4M3, "fp8e4m3"},
523 : {HcclDataType::HCCL_DATA_TYPE_FP8E5M2, "fp8e5m2"},
524 : {HcclDataType::HCCL_DATA_TYPE_RESERVED, "reserved"}
525 : };
526 :
527 0 : inline std::string GetDataTypeEnumStr2(HcclDataType dataType)
528 : {
529 0 : auto iter = HCOM_DATA_TYPE_STR_MAP.find(dataType);
530 0 : if (iter == HCOM_DATA_TYPE_STR_MAP.end()) {
531 0 : return "HcclDataType(" + std::to_string(dataType) + ")";
532 : } else {
533 0 : return iter->second;
534 : }
535 : }
536 :
537 0 : inline std::string GetDataTypeEnumStr(u32 dataType)
538 : {
539 0 : auto hcclDataType = static_cast<HcclDataType>(dataType);
540 0 : return GetDataTypeEnumStr2(hcclDataType);
541 : }
542 : inline std::string GetOpTypeEnumStr(u32 opType)
543 : {
544 : Hccl::OpType hcclOpType = static_cast<Hccl::OpType::Value>(opType);
545 : return hcclOpType.Describe();
546 : }
547 :
548 0 : void TaskExceptionHost::PrintOpDataErrorMessage(u32 deviceId, const Hccl::ErrorMessageReport &errorMessage,
549 : std::string &stageErrInfo) const
550 : {
551 0 : std::stringstream opDataStr;
552 0 : opDataStr << "src" << "[0x"
553 0 : << std::hex << errorMessage.srcAddr << "], dst[0x"
554 0 : << std::hex << errorMessage.dstAddr << "], ";
555 :
556 0 : std::string opStr;
557 0 : if (errorMessage.reduceType != HcclReduceOp::HCCL_REDUCE_RESERVED) {
558 0 : opStr += "reduceType[";
559 0 : opStr += GetReduceOpEnumStr2(static_cast<HcclReduceOp>(errorMessage.reduceType));
560 0 : opStr += "], ";
561 : }
562 :
563 0 : std::string opDataContent;
564 0 : opDataContent += "deviceId:[";
565 0 : opDataContent += std::to_string(deviceId);
566 0 : opDataContent += "], index[";
567 0 : opDataContent += std::to_string(errorMessage.opIndex);
568 0 : opDataContent += "], count[";
569 0 : opDataContent += std::to_string(errorMessage.count);
570 0 : opDataContent += "], ";
571 0 : opDataContent += opStr;
572 0 : opDataContent += opDataStr.str();
573 0 : opDataContent += "dataType[";
574 0 : opDataContent += GetDataTypeEnumStr(errorMessage.dataType);
575 0 : opDataContent += "].";
576 :
577 0 : PrintOpDataErrorLog(stageErrInfo, opDataContent);
578 0 : return;
579 0 : }
580 :
581 0 : void TaskExceptionHost::ReportErrorMsg(const Hccl::TaskInfo &exceptionTaskInfo, [[maybe_unused]] const std::string &groupRankContent,
582 : const Hccl::ErrorMessageReport &errorMessage, rtExceptionInfo_t *exceptionInfo) const
583 : {
584 0 : HCCL_RUN_INFO("[ReportErrorMsg] start, taskType[%d]", exceptionTaskInfo.taskParam_.taskType);
585 :
586 0 : std::string clusterMonitorErrMsg = AicpuGetAndPrintClusterMonitorErr(exceptionInfo);
587 :
588 0 : if (exceptionTaskInfo.taskParam_.taskType == Hccl::TaskParamType::TASK_NOTIFY_WAIT) {
589 0 : HCCL_ERROR("[ReportErrorMsg] EI0002");
590 0 : RPT_INPUT_ERR(true,
591 : "EI0002",
592 : std::vector<std::string>({"remote_rankid", "base_information", "task_information", "group_rank_content"}),
593 : std::vector<std::string>({
594 : std::to_string(exceptionTaskInfo.remoteRank_),
595 : exceptionTaskInfo.GetIndopBaseInfo().c_str(), (exceptionTaskInfo.GetParaInfo() + clusterMonitorErrMsg).c_str(),
596 : ""})
597 : );
598 0 : } else if (exceptionTaskInfo.taskParam_.taskType == Hccl::TaskParamType::TASK_WRITE_REDUCE_WITH_NOTIFY
599 0 : || exceptionTaskInfo.taskParam_.taskType == Hccl::TaskParamType::TASK_WRITE_WITH_NOTIFY
600 0 : || exceptionTaskInfo.taskParam_.taskType == Hccl::TaskParamType::TASK_UB_INLINE_WRITE
601 0 : || exceptionTaskInfo.taskParam_.taskType == Hccl::TaskParamType::TASK_UB_REDUCE_INLINE
602 0 : || exceptionTaskInfo.taskParam_.taskType == Hccl::TaskParamType::TASK_UB) {
603 0 : hccl::CollComm *collComm = static_cast<hccl::CollComm*>(exceptionTaskInfo.dfxOpInfo_->comm_);
604 0 : std::string localServerId = "";
605 0 : GetAicpuCqeErrNetInstanceByRankId(collComm, errorMessage.rankId, localServerId);
606 0 : u32 localDeviceId = INVALID_VALUE_RANKID;
607 0 : GetAicpuCqeErrRemoteLocalIdByRankId(collComm, errorMessage.rankId, localDeviceId);
608 0 : std::string remoteServerId = "";
609 0 : GetAicpuCqeErrNetInstanceByRankId(collComm, errorMessage.remoteUserRank, remoteServerId);
610 0 : u32 remoteDeviceId = INVALID_VALUE_RANKID;
611 0 : GetAicpuCqeErrRemoteLocalIdByRankId(collComm, errorMessage.remoteUserRank, remoteDeviceId);
612 0 : Hccl::IpAddress localAddr(errorMessage.locEid);
613 0 : Hccl::IpAddress remoteAddr(errorMessage.rmtEid);
614 0 : HCCL_ERROR("[ReportErrorMsg] EI0018");
615 0 : RPT_INPUT_ERR(true,
616 : "EI0018",
617 : std::vector<std::string>({"localServerId", "localDeviceId", "localDeviceIp", "remoteServerId", "remoteDeviceId", "remoteDeviceIp"}),
618 : std::vector<std::string>({
619 : localServerId, std::to_string(localDeviceId), localAddr.GetReverseEid().Describe().c_str(),
620 : remoteServerId, std::to_string(remoteDeviceId), remoteAddr.GetReverseEid().Describe().c_str()})
621 : );
622 0 : }
623 0 : }
624 :
625 9 : void GetTaskParam(Hccl::TaskParam &taskParam, const Hccl::ErrorMessageReport &errMsgInfo) {
626 9 : switch (errMsgInfo.taskType) {
627 2 : case Hccl::TaskParamType::TASK_NOTIFY_WAIT:
628 : case Hccl::TaskParamType::TASK_NOTIFY_RECORD:
629 2 : taskParam.taskPara.Notify.notifyID = errMsgInfo.notifyId;
630 2 : taskParam.taskPara.Notify.value = errMsgInfo.notifyValue;
631 2 : break;
632 2 : case Hccl::TaskParamType::TASK_UB_REDUCE_INLINE:
633 : case Hccl::TaskParamType::TASK_WRITE_REDUCE_WITH_NOTIFY:
634 2 : taskParam.taskPara.Reduce.notifyID = errMsgInfo.notifyId;
635 2 : taskParam.taskPara.Reduce.notifyValue = errMsgInfo.notifyValue;
636 2 : taskParam.taskPara.Reduce.locEid = errMsgInfo.locEid;
637 2 : taskParam.taskPara.Reduce.rmtEid = errMsgInfo.rmtEid;
638 2 : taskParam.taskPara.Reduce.linkType = errMsgInfo.linkType;
639 2 : taskParam.taskPara.Reduce.size = errMsgInfo.size;
640 2 : taskParam.taskPara.Reduce.src = reinterpret_cast<void *>(errMsgInfo.taskSrcAddr);
641 2 : taskParam.taskPara.Reduce.dst = reinterpret_cast<void *>(errMsgInfo.taskDstAddr);
642 2 : break;
643 1 : case Hccl::TaskParamType::TASK_REDUCE_INLINE:
644 1 : taskParam.taskPara.Reduce.reduceOp = static_cast<HcclReduceOp>(errMsgInfo.reduceType);
645 1 : break;
646 2 : case Hccl::TaskParamType::TASK_UB_INLINE_WRITE:
647 : case Hccl::TaskParamType::TASK_WRITE_WITH_NOTIFY:
648 2 : taskParam.taskPara.DMA.notifyID = errMsgInfo.notifyId;
649 2 : taskParam.taskPara.DMA.notifyValue = errMsgInfo.notifyValue;
650 2 : taskParam.taskPara.DMA.locEid = errMsgInfo.locEid;
651 2 : taskParam.taskPara.DMA.rmtEid = errMsgInfo.rmtEid;
652 2 : taskParam.taskPara.DMA.linkType = errMsgInfo.linkType;
653 2 : taskParam.taskPara.DMA.size = errMsgInfo.size;
654 2 : taskParam.taskPara.DMA.src = reinterpret_cast<void *>(errMsgInfo.taskSrcAddr);
655 2 : taskParam.taskPara.DMA.dst = reinterpret_cast<void *>(errMsgInfo.taskDstAddr);
656 2 : break;
657 1 : case Hccl::TaskParamType::TASK_UB:
658 1 : taskParam.taskPara.DMA.locEid = errMsgInfo.locEid;
659 1 : taskParam.taskPara.DMA.rmtEid = errMsgInfo.rmtEid;
660 1 : taskParam.taskPara.DMA.linkType = errMsgInfo.linkType;
661 1 : taskParam.taskPara.DMA.size = errMsgInfo.size;
662 1 : taskParam.taskPara.DMA.src = reinterpret_cast<void *>(errMsgInfo.taskSrcAddr);
663 1 : taskParam.taskPara.DMA.dst = reinterpret_cast<void *>(errMsgInfo.taskDstAddr);
664 1 : break;
665 1 : case Hccl::TaskParamType::TASK_SDMA:
666 1 : taskParam.taskPara.DMA.linkType = errMsgInfo.linkType;
667 1 : taskParam.taskPara.DMA.size = errMsgInfo.size;
668 1 : taskParam.taskPara.DMA.src = reinterpret_cast<void *>(errMsgInfo.taskSrcAddr);
669 1 : taskParam.taskPara.DMA.dst = reinterpret_cast<void *>(errMsgInfo.taskDstAddr);
670 1 : break;
671 0 : default:
672 0 : HCCL_ERROR("[TaskException][HOST]%s taskType[%d] is not support", __func__, taskParam.taskType);
673 0 : return;
674 : }
675 : }
676 :
677 0 : void TaskExceptionHost::HandleAicpuErrorReport(rtExceptionInfo_t *exceptionInfo,
678 : const Hccl::ErrorMessageReport &errorMessage, const Hccl::TaskInfo &taskInfo) const
679 : {
680 0 : std::string groupRankContent;
681 0 : u32 streamId = static_cast<u32>(errorMessage.streamId);
682 0 : std::string tag = std::string(errorMessage.tag);
683 0 : Hccl::TaskParam taskParam{};
684 0 : taskParam.taskType = errorMessage.taskType;
685 0 : GetTaskParam(taskParam, errorMessage);
686 0 : std::shared_ptr<Hccl::DfxOpInfo> dfxOpInfo = std::make_shared<Hccl::DfxOpInfo>();
687 0 : dfxOpInfo->tag_ = tag;
688 0 : dfxOpInfo->comm_ = taskInfo.dfxOpInfo_->comm_;
689 0 : Hccl::TaskInfo exceptionTaskInfo(streamId, errorMessage.taskId, errorMessage.remoteUserRank, taskParam, dfxOpInfo);
690 0 : auto logKeywordL2 = exceptionTaskInfo.taskParam_.taskType ==
691 0 : Hccl::TaskParamType::TASK_NOTIFY_WAIT ? Hccl::LOG_KEYWORDS_TIMEOUT : Hccl::LOG_KEYWORDS_RUN_FAILED;
692 0 : auto stageErrInfo = "[" + Hccl::LOG_KEYWORDS_TASK_EXEC + "][" + logKeywordL2 + "][" + Hccl::LOG_KEYWORDS_AICPU + "]";
693 0 : HCCL_ERROR("%sTask from HCCL run failed.", stageErrInfo.c_str());
694 : // 防止tag字符串过长， 信息分开打印
695 0 : PrintBaseErrorLog(stageErrInfo, exceptionTaskInfo.GetIndopBaseInfo());
696 0 : PrintParaErrorLog(stageErrInfo, exceptionTaskInfo.GetParaInfo());
697 0 : PrintGroupErrorMessage(errorMessage, exceptionTaskInfo, groupRankContent, stageErrInfo);
698 0 : PrintOpDataErrorMessage(exceptionInfo->deviceid, errorMessage, stageErrInfo);
699 0 : HCCL_ERROR("errorMessage taskType[%s], rtCqErrorType[%u], rtCqErrorCode[%u]. ",
700 : errorMessage.taskType.Describe().c_str(), static_cast<u32>(errorMessage.rtCqErrorType), errorMessage.rtCqErrorCode);
701 :
702 : // 打印UB DFX寄存器信息
703 0 : PrintUbDfxInfo(exceptionInfo, errorMessage);
704 0 : ReportErrorMsg(exceptionTaskInfo, groupRankContent, errorMessage, exceptionInfo);
705 0 : if (errorMessage.ubCqeStatus != 0) {
706 0 : GetAicpuCqeErrInfo(exceptionInfo, errorMessage, taskInfo);
707 : }
708 0 : }
709 :
710 0 : void TaskExceptionHost::PrintUbDfxInfo(rtExceptionInfo_t *exceptionInfo, const Hccl::ErrorMessageReport &errorMessage) const
711 : {
712 0 : if (errorMessage.taskType == Hccl::TaskParamType::TASK_WRITE_WITH_NOTIFY ||
713 0 : errorMessage.taskType == Hccl::TaskParamType::TASK_WRITE_REDUCE_WITH_NOTIFY ||
714 0 : errorMessage.taskType == Hccl::TaskParamType::TASK_UB_INLINE_WRITE ||
715 0 : errorMessage.taskType == Hccl::TaskParamType::TASK_UB_REDUCE_INLINE ||
716 0 : errorMessage.taskType == Hccl::TaskParamType::TASK_UB) {
717 0 : HCCL_ERROR("errorMessage ubCqeStatus[%u], localEid[%s], remoteEid[%s]. ", static_cast<u32>(errorMessage.ubCqeStatus),
718 : errorMessage.locEid.Describe().c_str(), errorMessage.rmtEid.Describe().c_str());
719 0 : auto reverseAddr = Hccl::IpAddress(errorMessage.locEid);
720 0 : auto addr = Hccl::IpAddress(reverseAddr.GetReverseEid());
721 0 : u32 devPhyId = Hccl::HrtGetDevicePhyIdByIndex(exceptionInfo->deviceid);
722 0 : auto rdmaHandle = Hccl::RdmaHandleManager::GetInstance().GetByIp(devPhyId, addr);
723 0 : HrtRaDumpJettyContext(reinterpret_cast<void*>(errorMessage.jettyHandle), errorMessage.jettyId);
724 0 : PrintUbRegisters(static_cast<s32>(exceptionInfo->deviceid), rdmaHandle);
725 : }
726 0 : }
727 : } // namespace Hccl
|
