Line data Source code
1 : /**
2 : * Copyright (c) 2026 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "exception_handle.h"
12 : #include "log.h"
13 : #include "thread.h"
14 : #include "stream_lite.h"
15 : #include "aicpu_thread_process.h"
16 : #include "exception_callback_mgr.h"
17 : #include "exception_util.h"
18 : #include "ascend_hal.h"
19 :
20 : using Hccl::HcclException;
21 : using std::exception;
22 : using std::string;
23 :
24 : namespace hcomm {
25 : constexpr u32 URMA_STARTS_CQE_FAILEDERR = 0x5;
26 :
27 112 : ExceptionHandle& ExceptionHandle::GetInstance()
28 : {
29 112 : static ExceptionHandle instance;
30 112 : return instance;
31 : }
32 :
33 14 : uint32_t ExceptionHandle::GetSqeId(uint16_t taskId, uint16_t streamId)
34 : {
35 14 : constexpr uint32_t TASK_ID_SHIFT_BITS = 16;
36 14 : return (static_cast<uint32_t>(taskId) << TASK_ID_SHIFT_BITS) | static_cast<uint32_t>(streamId);
37 : }
38 :
39 8 : HcclResult ExceptionHandle::CheckRepeatBySqeId(
40 : Hccl::StreamLite* streamLite, [[maybe_unused]] uint32_t devId, uint16_t taskId, uint16_t streamId)
41 : {
42 8 : CHK_PTR_NULL(streamLite);
43 :
44 7 : const uint32_t sqId = streamLite->GetSqId();
45 7 : const uint32_t sqeId = GetSqeId(taskId, streamId);
46 :
47 7 : auto it = threadsPrinted_.find(sqId);
48 7 : if (it != threadsPrinted_.end() && it->second == sqeId) {
49 2 : return HCCL_E_AGAIN;
50 : }
51 5 : threadsPrinted_[sqId] = sqeId;
52 5 : return HCCL_SUCCESS;
53 : }
54 :
55 : dfx::CqeStatus
56 11 : ExceptionHandle::ReceiveCqeReport(uint32_t devId, Hccl::StreamLite* streamLite, rtLogicCqReport_t& cqeException)
57 : {
58 11 : constexpr uint32_t REPORT_SIZE = 1;
59 11 : rtLogicCqReport_t streamReport[REPORT_SIZE] = {};
60 :
61 11 : CqeQueryInput cqeQueryInput{};
62 11 : cqeQueryInput.devId = devId;
63 11 : cqeQueryInput.streamId = streamLite->GetId();
64 11 : cqeQueryInput.sqId = streamLite->GetSqId();
65 11 : cqeQueryInput.cqId = streamLite->GetCqId();
66 11 : cqeQueryInput.type = static_cast<uint32_t>(DRV_LOGIC_TYPE);
67 11 : cqeQueryInput.cqeAddr = reinterpret_cast<uint8_t*>(streamReport);
68 :
69 22 : return CqReportRecv(cqeQueryInput, cqeException);
70 : }
71 :
72 10 : HcclResult ExceptionHandle::CheckExceptionCqe(hccl::Thread* thread, uint32_t devId)
73 : {
74 10 : CHK_PTR_NULL(thread);
75 :
76 9 : Hccl::StreamLite* streamLite = static_cast<Hccl::StreamLite*>(thread->GetStreamLitePtr());
77 9 : CHK_PTR_NULL(streamLite);
78 :
79 8 : uint32_t sqId = streamLite->GetSqId();
80 8 : if (sqCqeErrorSet_.count(sqId) > 0) {
81 1 : return HCCL_SUCCESS;
82 : }
83 :
84 7 : rtLogicCqReport_t cqeException{};
85 7 : dfx::CqeStatus cqeStatus = ReceiveCqeReport(devId, streamLite, cqeException);
86 :
87 7 : if (cqeStatus == dfx::CqeStatus::kCqeTimeOut) {
88 1 : cqeException.taskId = 0xFFFF;
89 : }
90 :
91 7 : if (cqeStatus == dfx::CqeStatus::kCqeInnerError) {
92 1 : HCCL_ERROR(
93 : "[ExceptionHandle][CheckExceptionCqe] CqReportRecv internal error, "
94 : "devId[%u], streamId[%u], sqId[%u], skip this sq in future polls",
95 : devId, streamLite->GetId(), sqId);
96 1 : sqCqeErrorSet_.insert(sqId);
97 1 : return HCCL_E_INTERNAL;
98 : }
99 :
100 6 : if (cqeStatus == dfx::CqeStatus::kDefault) {
101 2 : return HCCL_SUCCESS;
102 : }
103 :
104 4 : if (cqeException.sqeType != DFX_SQE_TYPE_UDMA) {
105 2 : return HCCL_SUCCESS;
106 : }
107 :
108 2 : if (CheckRepeatBySqeId(streamLite, devId, cqeException.taskId, cqeException.streamId) != HCCL_SUCCESS) {
109 1 : return HCCL_SUCCESS;
110 : }
111 :
112 : // 前6位为0，则表示无错误
113 1 : if ((cqeException.errorType & RT_STARS_EXIST_ERROR) == 0U) {
114 0 : return HCCL_SUCCESS;
115 : }
116 :
117 1 : HCCL_ERROR(
118 : "[ExceptionHandle][CheckExceptionCqe] CQE exception detected, "
119 : "devId[%u], streamId[%u], taskId[%u], errorCode[0x%x], errorType[0x%x]",
120 : devId, cqeException.streamId, cqeException.taskId, cqeException.errorCode, cqeException.errorType);
121 1 : HcommExceptionInfo exceptionInfo{};
122 1 : CHK_RET(FillExceptionInfo(exceptionInfo, thread, cqeException.taskId, cqeException));
123 1 : ExceptionCallbackMgr::GetInstance().NotifyAll(exceptionInfo);
124 :
125 1 : return HCCL_E_ROCE_TRANSFER;
126 : }
127 :
128 : // 把UB类错误码转换成hccl result对应的错误码
129 9 : uint32_t ExceptionHandle::SwitchCqeErrCodeToHcclErrCode(uint32_t cqeErrCode)
130 : {
131 9 : switch (cqeErrCode) {
132 3 : case URMA_STARTS_CQE_FAILEDERR:
133 3 : return HCCL_E_INTERNAL;
134 6 : default:
135 6 : return HCCL_E_ROCE_TRANSFER;
136 : }
137 : }
138 :
139 6 : HcclResult ExceptionHandle::FillExceptionInfo(
140 : HcommExceptionInfo& exceptionInfo, hccl::Thread* thread, uint32_t taskId, const rtLogicCqReport_t& cqeException)
141 : {
142 6 : CHK_PTR_NULL(thread);
143 :
144 5 : exceptionInfo.thread = reinterpret_cast<uint64_t>(thread);
145 5 : exceptionInfo.channel = 0;
146 5 : exceptionInfo.taskId = taskId;
147 5 : exceptionInfo.retCode = SwitchCqeErrCodeToHcclErrCode(cqeException.errorCode & 0xFF);
148 :
149 5 : exceptionInfo.expandInfo.type = HCOMM_EXCEPTION_STARS;
150 5 : exceptionInfo.expandInfo.detail.starsInfo.starsErrcode = cqeException.errorType;
151 5 : exceptionInfo.expandInfo.detail.starsInfo.sqeType = cqeException.sqeType;
152 5 : exceptionInfo.expandInfo.detail.starsInfo.statusMerged = cqeException.errorCode & 0xFF;
153 :
154 5 : return HCCL_SUCCESS;
155 : }
156 :
157 1 : HcclResult ExceptionHandle::HandleExceptionCqe()
158 : {
159 1 : std::shared_lock<std::shared_mutex> rwlock(AicpuThreadProcess::GetMutex());
160 1 : std::vector<std::shared_ptr<hccl::Thread>> threads = AicpuThreadProcess::GetThreads();
161 :
162 1 : for (auto& thread : threads) {
163 0 : if (thread == nullptr) {
164 0 : continue;
165 : }
166 0 : Hccl::StreamLite* streamLite = static_cast<Hccl::StreamLite*>(thread->GetStreamLitePtr());
167 0 : if (streamLite == nullptr) {
168 0 : continue;
169 : }
170 :
171 0 : uint32_t phyId = streamLite->GetDevPhyId();
172 0 : uint32_t localDevId = 0;
173 0 : drvError_t drvRet = drvGetLocalDevIDByHostDevID(phyId, &localDevId);
174 0 : if (drvRet != DRV_ERROR_NONE) {
175 0 : HCCL_ERROR(
176 : "[ExceptionHandle][HandleExceptionCqe] drvGetLocalDevIDByHostDevID failed, "
177 : "phyId=%u, ret=%d",
178 : phyId, drvRet);
179 0 : continue;
180 : }
181 :
182 0 : CheckExceptionCqe(thread.get(), localDevId);
183 : }
184 1 : return HCCL_SUCCESS;
185 1 : }
186 :
187 2 : void ExceptionHandle::Call()
188 : {
189 2 : if (ExceptionCallbackMgr::GetInstance().IsEmpty()) {
190 2 : return;
191 : }
192 0 : TRY_CATCH_PRINT_ERROR(HandleExceptionCqe());
193 : }
194 :
195 4 : void ExceptionHandle::ClearStreamState(uint32_t sqId)
196 : {
197 4 : threadsPrinted_.erase(sqId);
198 4 : sqCqeErrorSet_.erase(sqId);
199 4 : HCCL_INFO("[ExceptionHandle][%s] sqId[%u] state cleared", __func__, sqId);
200 4 : }
201 :
202 : } // namespace hcomm
|
