Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "task_exception_func.h"
12 : #include "communicator_impl_lite_manager.h"
13 : #include "log.h"
14 : #include <map>
15 : #include <vector>
16 : #include <memory>
17 : #include <string>
18 :
19 : namespace Hccl {
20 46 : TaskExceptionFunc& TaskExceptionFunc::GetInstance()
21 : {
22 46 : static TaskExceptionFunc instance;
23 46 : return instance;
24 : }
25 :
26 6 : void TaskExceptionFunc::SetEnable(bool isEnable)
27 : {
28 6 : isEnable_ = isEnable;
29 6 : if (isEnable_) {
30 18 : HCCL_INFO("[TaskExceptionFunc] Task Exception enabled.");
31 : } else {
32 0 : HCCL_INFO("[TaskExceptionFunc] Task Exception disabled.");
33 : }
34 6 : }
35 :
36 1 : void TaskExceptionFunc::SetDevId(uint32_t devId) { devId_ = devId; }
37 :
38 2 : void TaskExceptionFunc::RegisterCallback(const Callback& callback) { callback_ = callback; }
39 :
40 8 : void TaskExceptionFunc::Register(StreamLite* streamLite) { streamLiteMap_[streamLite->GetSqId()] = streamLite; }
41 :
42 19 : void TaskExceptionFunc::UnRegister(StreamLite* streamLite) { streamLiteMap_.erase(streamLite->GetSqId()); }
43 :
44 14 : std::string TaskExceptionFunc::ErrorType2Str(uint8_t errorType) const
45 : {
46 : static const std::map<uint8_t, std::string> errorType2Str{
47 0 : {0b1, "exception"}, // bit0:代表是否有exception
48 0 : {0b11, "bus error"}, // bit1:代表是否为bus_error
49 0 : {0b101, "rsv"}, // bit2:代表为rsv域段（该域段硬件值可能为0或1）
50 0 : {0b1001, "sqe error"}, // bit3:代表是否为sqe_error
51 0 : {0b10001, "res conflict error"}, // bit4:代表是否为res_conflict_error
52 0 : {0b100001, "pre_p/post_p error"}, // bit5:表示软件在pre_p或post_p过程中发生了错误，写入了sq_sw_status
53 22 : };
54 14 : const auto res = errorType2Str.find(errorType);
55 24 : return res == errorType2Str.end() ? "" : res->second;
56 1 : }
57 :
58 2 : std::string TaskExceptionFunc::CqeStatus2Str(uint32_t errorCode) const
59 : {
60 2 : uint8_t status = errorCode & 0xFF; // errorCode低8bit是UB CQE的status
61 : static const std::map<uint8_t, std::string> code2Str{
62 0 : {0x00, "OK"},
63 0 : {0x01, "Unsupported OpCode"},
64 0 : {0x02, "Local Operation Error"},
65 0 : {0x03, "Remote Operation Error"},
66 0 : {0x04, "Transaction Retry Counter Exceeded"},
67 0 : {0x05, "Transaction ACK Timeout"},
68 0 : {0x06, "Jetty work Request Flushed"},
69 11 : };
70 2 : const auto res = code2Str.find(status);
71 4 : return res == code2Str.end() ? "Reserved" : res->second;
72 1 : }
73 :
74 14 : std::string TaskExceptionFunc::StringLogicCqReportInfo(const rtLogicCqReport_t& reportOfOne) const
75 : {
76 14 : std::stringstream ss;
77 14 : ss << "streamId :" << reportOfOne.streamId;
78 14 : ss << " taskId :" << reportOfOne.taskId;
79 14 : ss << " errorCode :" << reportOfOne.errorCode;
80 14 : if (reportOfOne.sqeType == 9) { // sqeType等于9时, errorCode才按照UB的格式解析, 不等于9时可不关注errorCode
81 2 : const std::string errorStatus = CqeStatus2Str(reportOfOne.errorCode);
82 2 : ss << "(" << errorStatus << ")";
83 2 : }
84 14 : ss << " errorType :" << static_cast<uint32_t>(reportOfOne.errorType);
85 14 : const std::string errorTypeStr = ErrorType2Str(reportOfOne.errorType);
86 14 : if (!errorTypeStr.empty()) {
87 9 : ss << "(" << errorTypeStr << ")";
88 : }
89 14 : ss << " sqeType :" << static_cast<uint32_t>(reportOfOne.sqeType);
90 14 : ss << " sqId :" << reportOfOne.sqId;
91 14 : ss << " sqHead :" << reportOfOne.sqHead;
92 14 : ss << " matchFlag :" << reportOfOne.matchFlag;
93 14 : ss << " dropFlag :" << reportOfOne.dropFlag;
94 14 : ss << " errorBit :" << reportOfOne.errorBit;
95 14 : ss << " accError :" << reportOfOne.accError;
96 28 : return ss.str();
97 14 : }
98 :
99 1 : unsigned int TaskExceptionFunc::GetTrailingZeros(uint8_t num) const
100 : {
101 1 : uint8_t count = 0;
102 4 : while ((num & 1U) == 0) {
103 4 : count++;
104 4 : num >>= 1;
105 4 : if (num == 1U) {
106 1 : break;
107 : }
108 : }
109 1 : return count;
110 : }
111 :
112 : constexpr uint8_t RT_STARS_EXIST_ERROR = 0x3FU;
113 :
114 6 : bool TaskExceptionFunc::IsExceptionCqe(const rtLogicCqReport_t& reportOfOne) const
115 : {
116 6 : if ((reportOfOne.errorType & RT_STARS_EXIST_ERROR) == 0U) { // 取低6位
117 9 : HCCL_INFO("ReportOfOne info [%s]", StringLogicCqReportInfo(reportOfOne).c_str());
118 3 : return false;
119 : }
120 9 : HCCL_ERROR("ReportOfOne error info [%s]", StringLogicCqReportInfo(reportOfOne).c_str());
121 3 : return true;
122 : }
123 :
124 : constexpr uint32_t MAX_REPORT_CNT = 256U;
125 : constexpr uint32_t AC_SQE_REV_MAX_CNT = 32U;
126 : enum class CqeStatus : int64_t {
127 : kDefault = 0,
128 : kCqeException,
129 : kCqeTimeOut,
130 : kCqeInnerError,
131 : kCqeUnknown,
132 : };
133 :
134 0 : uint32_t TaskExceptionFunc::GetReporterInfo(const StreamLite* curStream, std::shared_ptr<halReportRecvInfo> recvInfo)
135 : {
136 0 : recvInfo->type = static_cast<drvSqCqType_t>(DRV_LOGIC_TYPE);
137 0 : recvInfo->tsId = 0;
138 0 : recvInfo->report_cqe_num = 0;
139 0 : recvInfo->timeout = 0; // 不设置超时时间，非阻塞
140 0 : recvInfo->task_id = 0xFFFF; // 接收所有类型
141 0 : recvInfo->cqe_num = MAX_REPORT_CNT; // 单次接收的最大cqe数量
142 0 : recvInfo->stream_id = curStream->GetId();
143 0 : recvInfo->cqId = curStream->GetCqId();
144 0 : auto exceptionInfo = reinterpret_cast<rtLogicCqReport_t*>(recvInfo->cqe_addr);
145 :
146 : // 接收错误信息
147 0 : drvError_t ret = halCqReportRecv(devId_, recvInfo.get());
148 0 : if (recvInfo->report_cqe_num != 0) {
149 0 : HCCL_INFO(
150 : "[TaskExceptionFunc]after exceptionInfo deviceId[%u], streamId[%u], taskId[%u], "
151 : "recvInfo->report_cqe_num[%u].",
152 : devId_, exceptionInfo->streamId, exceptionInfo->taskId, recvInfo->report_cqe_num);
153 : }
154 :
155 0 : if (ret == DRV_ERROR_WAIT_TIMEOUT) {
156 0 : HCCL_INFO("[TaskExceptionFunc]halCqReportRecv has found nothing, ret:%d", ret);
157 0 : return 1;
158 : }
159 0 : if (ret != DRV_ERROR_NONE) {
160 0 : HCCL_WARNING("[TaskExceptionFunc]halCqReportRecv failed, ret:%d", ret);
161 0 : return 1;
162 : }
163 0 : if (recvInfo->type != DRV_LOGIC_TYPE) { // 非DRV_LOGIC_TYPE不支持解析
164 0 : HCCL_WARNING(
165 : "[TaskExceptionFunc]halCqReportRecv type is not %d, recvInfo->type:%d", DRV_LOGIC_TYPE, recvInfo->type);
166 0 : return 1;
167 : }
168 0 : return 0;
169 : }
170 :
171 4 : void TaskExceptionFunc::Call()
172 : {
173 12 : TRY_CATCH_PRINT_ERROR(
174 : if (!isEnable_) { return; } auto recvInfo = std::make_shared<halReportRecvInfo>();
175 : constexpr uint32_t cqeSize = MAX_REPORT_CNT * sizeof(rtLogicCqReport_t);
176 : uint8_t tmpAddr[cqeSize] = {}; // cqe byte size
177 : recvInfo->cqe_addr = tmpAddr; // 外部保证是有效的地址
178 :
179 : std::vector<CommunicatorImplLite*> aicpuComms = CommunicatorImplLiteMgr::GetInstance().GetAll();
180 : for (auto aicpuComm
181 : : aicpuComms) {
182 : std::vector<StreamLite*> aicpuStreams = aicpuComm->GetStreamLiteMgr()->GetAllStreams();
183 : for (auto& aicpuStream : aicpuStreams) {
184 : if (aicpuStream == nullptr) {
185 : HCCL_ERROR("[TaskExceptionFunc]stream of in aicpuComm[%s] is nullptr", aicpuComm->GetId().c_str());
186 : continue;
187 : }
188 : if (GetReporterInfo(aicpuStream, recvInfo) != 0) {
189 : continue;
190 : }
191 : uint32_t reportNum = recvInfo->report_cqe_num;
192 : if (reportNum > MAX_REPORT_CNT) {
193 : HCCL_ERROR(
194 : "[TaskExceptionFunc]report cqe num %u should not big than %u", reportNum, MAX_REPORT_CNT);
195 : continue;
196 : }
197 : for (uint32_t idx = 0U; idx < reportNum; ++idx) {
198 : auto& reportOfOne
199 : = *((reinterpret_cast<rtLogicCqReport_t*>(recvInfo->cqe_addr)) + idx); // 外部保证是有效的地址
200 : if (IsExceptionCqe(reportOfOne)) {
201 : callback_(aicpuComm, &reportOfOne);
202 : }
203 : }
204 : }
205 : });
206 : }
207 : } // namespace Hccl
|
