Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "task_exception_func.h"
12 : #include "communicator_impl_lite_manager.h"
13 : #include "log.h"
14 : #include <map>
15 : #include <vector>
16 : #include <memory>
17 : #include <string>
18 :
19 : namespace Hccl {
20 46 : TaskExceptionFunc &TaskExceptionFunc::GetInstance()
21 : {
22 46 : static TaskExceptionFunc instance;
23 46 : return instance;
24 : }
25 :
26 6 : void TaskExceptionFunc::SetEnable(bool isEnable)
27 : {
28 6 : isEnable_ = isEnable;
29 6 : if (isEnable_) {
30 18 : HCCL_INFO("[TaskExceptionFunc] Task Exception enabled.");
31 : } else {
32 0 : HCCL_INFO("[TaskExceptionFunc] Task Exception disabled.");
33 : }
34 6 : }
35 :
36 1 : void TaskExceptionFunc::SetDevId(uint32_t devId)
37 : {
38 1 : devId_ = devId;
39 1 : }
40 :
41 2 : void TaskExceptionFunc::RegisterCallback(const Callback &callback)
42 : {
43 2 : callback_ = callback;
44 2 : }
45 :
46 8 : void TaskExceptionFunc::Register(StreamLite *streamLite)
47 : {
48 8 : streamLiteMap_[streamLite->GetSqId()] = streamLite;
49 8 : }
50 :
51 19 : void TaskExceptionFunc::UnRegister(StreamLite *streamLite)
52 : {
53 19 : streamLiteMap_.erase(streamLite->GetSqId());
54 19 : }
55 :
56 14 : std::string TaskExceptionFunc::ErrorType2Str(uint8_t errorType) const
57 : {
58 : static const std::map<uint8_t, std::string> errorType2Str{
59 0 : {0b1, "exception"}, // bit0:代表是否有exception
60 0 : {0b11, "bus error"}, // bit1:代表是否为bus_error
61 0 : {0b101, "rsv"}, // bit2:代表为rsv域段（该域段硬件值可能为0或1）
62 0 : {0b1001, "sqe error"}, // bit3:代表是否为sqe_error
63 0 : {0b10001, "res conflict error"}, // bit4:代表是否为res_conflict_error
64 0 : {0b100001, "pre_p/post_p error"}, // bit5:表示软件在pre_p或post_p过程中发生了错误，写入了sq_sw_status
65 22 : };
66 14 : const auto res = errorType2Str.find(errorType);
67 24 : return res == errorType2Str.end() ? "" : res->second;
68 1 : }
69 :
70 2 : std::string TaskExceptionFunc::CqeStatus2Str(uint32_t errorCode) const
71 : {
72 2 : uint8_t status = errorCode & 0xFF; // errorCode低8bit是UB CQE的status
73 : static const std::map<uint8_t, std::string> code2Str{
74 0 : {0x00, "OK"},
75 0 : {0x01, "Unsupported OpCode"},
76 0 : {0x02, "Local Operation Error"},
77 0 : {0x03, "Remote Operation Error"},
78 0 : {0x04, "Transaction Retry Counter Exceeded"},
79 0 : {0x05, "Transaction ACK Timeout"},
80 0 : {0x06, "Jetty work Request Flushed"},
81 11 : };
82 2 : const auto res = code2Str.find(status);
83 4 : return res == code2Str.end() ? "Reserved" : res->second;
84 1 : }
85 :
86 14 : std::string TaskExceptionFunc::StringLogicCqReportInfo(const rtLogicCqReport_t &reportOfOne) const
87 : {
88 14 : std::stringstream ss;
89 14 : ss << "streamId :" << reportOfOne.streamId;
90 14 : ss << " taskId :" << reportOfOne.taskId;
91 14 : ss << " errorCode :" << reportOfOne.errorCode;
92 14 : if (reportOfOne.sqeType == 9) { // sqeType等于9时, errorCode才按照UB的格式解析, 不等于9时可不关注errorCode
93 2 : const std::string errorStatus = CqeStatus2Str(reportOfOne.errorCode);
94 2 : ss << "(" << errorStatus << ")";
95 2 : }
96 14 : ss << " errorType :" << static_cast<uint32_t>(reportOfOne.errorType);
97 14 : const std::string errorTypeStr = ErrorType2Str(reportOfOne.errorType);
98 14 : if (!errorTypeStr.empty()) {
99 9 : ss << "(" << errorTypeStr << ")";
100 : }
101 14 : ss << " sqeType :" << static_cast<uint32_t>(reportOfOne.sqeType);
102 14 : ss << " sqId :" << reportOfOne.sqId;
103 14 : ss << " sqHead :" << reportOfOne.sqHead;
104 14 : ss << " matchFlag :" << reportOfOne.matchFlag;
105 14 : ss << " dropFlag :" << reportOfOne.dropFlag;
106 14 : ss << " errorBit :" << reportOfOne.errorBit;
107 14 : ss << " accError :" << reportOfOne.accError;
108 28 : return ss.str();
109 14 : }
110 :
111 1 : unsigned int TaskExceptionFunc::GetTrailingZeros(uint8_t num) const
112 : {
113 1 : uint8_t count = 0;
114 4 : while ((num & 1U) == 0) {
115 4 : count++;
116 4 : num >>= 1;
117 4 : if (num == 1U) {
118 1 : break;
119 : }
120 : }
121 1 : return count;
122 : }
123 :
124 : constexpr uint8_t RT_STARS_EXIST_ERROR = 0x3FU;
125 :
126 6 : bool TaskExceptionFunc::IsExceptionCqe(const rtLogicCqReport_t &reportOfOne) const
127 : {
128 6 : if ((reportOfOne.errorType & RT_STARS_EXIST_ERROR) == 0U) { // 取低6位
129 9 : HCCL_INFO("ReportOfOne info [%s]", StringLogicCqReportInfo(reportOfOne).c_str());
130 3 : return false;
131 : }
132 9 : HCCL_ERROR("ReportOfOne error info [%s]", StringLogicCqReportInfo(reportOfOne).c_str());
133 3 : return true;
134 : }
135 :
136 : constexpr uint32_t MAX_REPORT_CNT = 256U;
137 : constexpr uint32_t AC_SQE_REV_MAX_CNT = 32U;
138 : enum class CqeStatus : int64_t {
139 : kDefault = 0,
140 : kCqeException,
141 : kCqeTimeOut,
142 : kCqeInnerError,
143 : kCqeUnknown,
144 : };
145 :
146 0 : uint32_t TaskExceptionFunc::GetReporterInfo(const StreamLite *curStream, std::shared_ptr<halReportRecvInfo> recvInfo)
147 : {
148 0 : recvInfo->type = static_cast<drvSqCqType_t>(DRV_LOGIC_TYPE);
149 0 : recvInfo->tsId = 0;
150 0 : recvInfo->report_cqe_num = 0;
151 0 : recvInfo->timeout = 0; // 不设置超时时间，非阻塞
152 0 : recvInfo->task_id = 0xFFFF; // 接收所有类型
153 0 : recvInfo->cqe_num = MAX_REPORT_CNT; // 单次接收的最大cqe数量
154 0 : recvInfo->stream_id = curStream->GetId();
155 0 : recvInfo->cqId = curStream->GetCqId();
156 0 : auto exceptionInfo = reinterpret_cast<rtLogicCqReport_t *>(recvInfo->cqe_addr);
157 :
158 : // 接收错误信息
159 0 : drvError_t ret = halCqReportRecv(devId_, recvInfo.get());
160 0 : if (recvInfo->report_cqe_num != 0) {
161 0 : HCCL_INFO("[TaskExceptionFunc]after exceptionInfo deviceId[%u], streamId[%u], taskId[%u], recvInfo->report_cqe_num[%u].",
162 : devId_, exceptionInfo->streamId, exceptionInfo->taskId, recvInfo->report_cqe_num);
163 : }
164 :
165 0 : if (ret == DRV_ERROR_WAIT_TIMEOUT) {
166 0 : HCCL_INFO("[TaskExceptionFunc]halCqReportRecv has found nothing, ret:%d", ret);
167 0 : return 1;
168 : }
169 0 : if (ret != DRV_ERROR_NONE) {
170 0 : HCCL_WARNING("[TaskExceptionFunc]halCqReportRecv failed, ret:%d", ret);
171 0 : return 1;
172 : }
173 0 : if (recvInfo->type != DRV_LOGIC_TYPE) { // 非DRV_LOGIC_TYPE不支持解析
174 0 : HCCL_WARNING("[TaskExceptionFunc]halCqReportRecv type is not %d, recvInfo->type:%d", DRV_LOGIC_TYPE, recvInfo->type);
175 0 : return 1;
176 : }
177 0 : return 0;
178 : }
179 :
180 4 : void TaskExceptionFunc::Call()
181 : {
182 12 : TRY_CATCH_PRINT_ERROR(
183 : if (!isEnable_) {
184 : return;
185 : }
186 : auto recvInfo = std::make_shared<halReportRecvInfo>();
187 : constexpr uint32_t cqeSize = MAX_REPORT_CNT * sizeof(rtLogicCqReport_t);
188 : uint8_t tmpAddr[cqeSize] = {}; // cqe byte size
189 : recvInfo->cqe_addr = tmpAddr; // 外部保证是有效的地址
190 :
191 : std::vector<CommunicatorImplLite *> aicpuComms = CommunicatorImplLiteMgr::GetInstance().GetAll();
192 : for (auto aicpuComm : aicpuComms) {
193 : std::vector<StreamLite *> aicpuStreams = aicpuComm-> GetStreamLiteMgr()->GetAllStreams();
194 : for (auto &aicpuStream : aicpuStreams) {
195 : if (aicpuStream == nullptr) {
196 : HCCL_ERROR("[TaskExceptionFunc]stream of in aicpuComm[%s] is nullptr", aicpuComm->GetId().c_str());
197 : continue;
198 : }
199 : if (GetReporterInfo(aicpuStream, recvInfo) != 0) {
200 : continue;
201 : }
202 : uint32_t reportNum = recvInfo->report_cqe_num;
203 : if (reportNum > MAX_REPORT_CNT) {
204 : HCCL_ERROR("[TaskExceptionFunc]report cqe num %u should not big than %u", reportNum, MAX_REPORT_CNT);
205 : continue;
206 : }
207 : for (uint32_t idx = 0U; idx < reportNum; ++idx) {
208 : auto &reportOfOne
209 : = *((reinterpret_cast<rtLogicCqReport_t *>(recvInfo->cqe_addr)) + idx); // 外部保证是有效的地址
210 : if (IsExceptionCqe(reportOfOne)) {
211 : callback_(aicpuComm, &reportOfOne);
212 : }
213 : }
214 : }
215 : }
216 : );
217 : }
218 : } // namespace Hccl
|
