Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 : #include "ccuTaskException.h"
11 : #include "ccu_dfx_schema.h"
12 : #include <memory>
13 : #include "log.h"
14 : #include "comm_addr_logger.h"
15 : #include "coll_comm.h"
16 : #include <adapter_error_manager_pub.h>
17 : #include "task_param.h"
18 : #include "ccu_kernel_mgr.h"
19 : #include "hcomm_c_adpt.h"
20 : #include "../../endpoint_pairs/channels/ccu/ccu_urma_channel.h"
21 : #include "orion_adpt_utils.h"
22 : #include "hcomm_adapter_hccp.h"
23 : #include "hccp_tlv_hdc_manager.h"
24 : #include "adapter_rts_common.h"
25 : #include "ccu_rep_loopgroup_bundle_v1.h"
26 : #include "ccu_rep_type_v1.h"
27 : #include "ccu_rep_loc_record_event.h"
28 : #include "ccu_rep_v1.h"
29 : #include "ccu_comp.h"
30 : #include "string_util.h"
31 :
32 : #include "net_instance.h"
33 : #include "hccl_communicator.h"
34 :
35 : #include "sal.h"
36 :
37 : namespace hcomm {
38 :
39 : using namespace std;
40 : constexpr int BYTE = 8;
41 : constexpr size_t CCU_CTX_RAW_CAPACITY = 64;
42 : constexpr uint64_t CCU_MSG_256MB_LEN = 256ULL * 1024 * 1024; // CCU消息长度不能大于256MB
43 : constexpr uint16_t INVALID_U16 = 65535;
44 : constexpr uint8_t CCUM_EXECUTE_ERROR = 0X09;
45 : constexpr uint8_t CCU_MISSION_TASK_KILLED = 0X02;
46 :
47 : const map<uint8_t, string> MISSION_STATUS_MAP {
48 : {0x01, "Unsupported Opcode(0x01)"}, {0x02, "Local Operation Error(0x02)"},
49 : {0x03, "Remote Operation Error(0x03)"}, {0x04, "Transaction Retry Counter Exceeded(0x04)"},
50 : {0x05, "Transaction ACK Timeout(0x05)"}, {0x06, "Jetty Work Request Flushed(0x06)"},
51 : {0x07, "CCUA Alg Task Error(0x07)"}, {0x08, "Memory ECC Error(0x08)"},
52 : {0x09, "CCUM Execute Error(0x09)"}, {0x0A, "CCUA Execute Error(0x0A)"},
53 : };
54 :
55 : const map<uint8_t, map<uint8_t, string>> MISSION_SUB_STATUS_MAP {
56 : {0x02,
57 : {{0x01, "Local Length Error(0x01)"},
58 : {0x02, "Local Access Error(0x02)"},
59 : {0x03, "Remote Response Length Error(0x03)"},
60 : {0x04, "Local Data Poison(0x04)"}}},
61 : {0x03,
62 : {{0x01, "Remote Unsupported Request(0x01)"},
63 : {0x02, "Remote Access Abort(0x02)"},
64 : {0x04, "Remote Data Poison(0x04)"}}},
65 : {0x07, {{0x01, "Overflow(0x01)"}, {0x02, "Underflow(0x02)"}, {0x04, "NaN(0x04)"}, {0x08, "Inf(0x08)"},
66 : {0x09, "Inf Overflow(0x09)"}}},
67 : {0x09, {{0x01, "SQE instr and key not match(0x01)"}, {0x02, "CCU Mission Task Killed(0x02)"}}},
68 : {0x0A,
69 : {{0x01, "EXOKAY(0x01)"},
70 : {0x11, "EXOKAY(0x11)"},
71 : {0x02, "SLVERR(0x02)"},
72 : {0x12, "SLVERR(0x12)"},
73 : {0x03, "DECERR(0x03)"},
74 : {0x13, "DECERR(0x13)"},
75 : {0x04, "Abort(0x04)"},
76 : {0x14, "Abort(0x14)"},
77 : {0x05, "Write Permission Err(0x05)"},
78 : {0x15, "Write Permission Err(0x15)"},
79 : {0x06, "Read Permission Err(0x06)"},
80 : {0x16, "Read Permission Err(0x16)"},
81 : {0x07, "Atomic Permission Err(0x07)"},
82 : {0x17, "Atomic Permission Err(0x17)"},
83 : {0x08, "Tokenval Err(0x08)"},
84 : {0x18, "Tokenval Err(0x18)"},
85 : {0x09, "Page Fault(0x09)"},
86 : {0x0a, "Page Fault(0x0A)"},
87 : {0x0b, "Page Fault(0x0B)"},
88 : {0x19, "Page Fault(0x19)"},
89 : {0x1a, "Page Fault(0x1A)"},
90 : {0x1b, "Page Fault(0x1B)"},
91 : {0x0c, "Read Local Mem Poison(0x0C)"}}},
92 : };
93 :
94 : const unordered_set<CcuRep::CcuRepType> REP_WITH_CHANNEL = {
95 : {CcuRep::CcuRepType::REM_POST_SEM},
96 : {CcuRep::CcuRepType::REM_WAIT_SEM},
97 : {CcuRep::CcuRepType::REM_POST_VAR},
98 : {CcuRep::CcuRepType::READ},
99 : {CcuRep::CcuRepType::WRITE},
100 : {CcuRep::CcuRepType::BUF_READ},
101 : {CcuRep::CcuRepType::BUF_WRITE},
102 : };
103 :
104 0 : MAKE_ENUM(AuxInfoInType, AUX_INFO_IN_TYPE_CQE, AUX_INFO_IN_TYPE_AE, AUX_INFO_IN_TYPE_MAX);
105 : struct AuxInfoIn {
106 : AuxInfoInType auxInfoInType;
107 : union {
108 : struct {
109 : uint32_t status;
110 : uint8_t sR;
111 : } cqe;
112 : struct {
113 : uint32_t eventType;
114 : } ae;
115 : };
116 : u8 resv[7];
117 : };
118 :
119 : constexpr u32 MAX_AUX_INFO_NUM = 256;
120 : struct AuxInfoOut {
121 : uint32_t auxInfoTypes[MAX_AUX_INFO_NUM];
122 : uint32_t auxInfoValues[MAX_AUX_INFO_NUM];
123 : uint32_t auxInfoNum{0};
124 : };
125 :
126 : std::mutex g_channelMapMutex;
127 : std::unordered_map<uint16_t, uint64_t> g_channelIdToHandle;
128 : std::atomic<bool> isGetCqeErrInfo(false); // 是否已获取过CQE错误信息，获取过后再次获取的概率较小，且获取过程可能较慢，因此设置标志位避免重复获取
129 : CcuGetErrStatusVecCallBack g_CcuGetErrStatusVecCallBack = nullptr;
130 : GetCcuCqeErrInfoCallBackHcomm g_getCqeErrInfoCallBack = nullptr;
131 :
132 41 : void RegisterCcuGetErrStatusVecCallBack(CcuGetErrStatusVecCallBack callback)
133 : {
134 41 : g_CcuGetErrStatusVecCallBack = callback;
135 41 : return;
136 : }
137 :
138 12 : std::vector<std::string> CcuGetErrStatusVec(s32 deviceLogicID)
139 : {
140 12 : if (g_CcuGetErrStatusVecCallBack != nullptr) {
141 12 : return g_CcuGetErrStatusVecCallBack(deviceLogicID);
142 : } else {
143 0 : HCCL_RUN_WARNING("[GetErrStatusVec]g_CcuGetErrStatusVecCallBack is nullptr.");
144 : }
145 0 : return std::vector<std::string>();
146 : }
147 :
148 12 : std::string CcuGetAndPrintClusterMonitorErr(const u32 deviceId)
149 : {
150 12 : auto errStatusVec = CcuGetErrStatusVec(deviceId);
151 12 : std::string errMsg = "";
152 12 : int errSize = errStatusVec.size();
153 12 : if (errSize > 0) {
154 0 : int maxListSize = 3; // 放入errMsg中的异常事件最多只有3个
155 0 : if (errSize <= maxListSize) {
156 0 : errMsg = "\nthere are(is) " + std::to_string(errSize) + " abnormal device(s):\n";
157 : } else {
158 0 : errMsg = "\nthere are " + std::to_string(errSize) + " abnormal device(s), " +
159 0 : "only the first 3 devices are listed:\n";
160 : }
161 :
162 0 : for (int i = 0; i < errSize; i++) {
163 0 : HCCL_ERROR("%s", errStatusVec[i].c_str());
164 0 : if (i < maxListSize) {
165 0 : errMsg += ("\t" + errStatusVec[i] + "\n");
166 : }
167 : }
168 : }
169 12 : return errMsg;
170 12 : }
171 :
172 41 : void RegisterGetCcuCqeErrInfoCallBackHcomm(GetCcuCqeErrInfoCallBackHcomm p1)
173 : {
174 41 : g_getCqeErrInfoCallBack = p1;
175 41 : return;
176 : }
177 0 : void CcuTaskException::ClusterMoniterGetCcuCqeErrInfo(u32 RemoteDeviceId, u32 locDeviceId, uint16_t status, std::string LocalEid, std::string RemoteEid, std::string RemoteInsId)
178 : {
179 0 : if (g_getCqeErrInfoCallBack != nullptr) {
180 0 : g_getCqeErrInfoCallBack(RemoteDeviceId, locDeviceId, status, LocalEid, RemoteEid, RemoteInsId);
181 : }
182 0 : return;
183 : }
184 :
185 :
186 5 : static void PrintPanicLogWithOps(const uint8_t *panicLog, const CcuVersionOps *ops)
187 : {
188 5 : if (panicLog == nullptr) {
189 0 : HCCL_ERROR("[CcuTaskException][PrintPanicLogWithOps] panicLog is nullptr.");
190 5 : return;
191 : }
192 5 : if (ops == nullptr || ops->printCcumDfxInfo == nullptr) {
193 5 : HCCL_ERROR("[CcuTaskException][PrintPanicLogWithOps] ops or printCcumDfxInfo is nullptr.");
194 5 : return;
195 : }
196 0 : std::ostringstream oss;
197 0 : oss << "[CCU DFX][ops=" << ops->name << "] CCU DFX INFO:";
198 0 : ops->printCcumDfxInfo(panicLog, oss);
199 0 : std::string logStr = oss.str();
200 0 : HCCL_ERROR("%s", logStr.c_str());
201 0 : }
202 :
203 4 : void CcuTaskException::ProcessCcuException(const rtExceptionInfo_t* exceptionInfo, const Hccl::TaskInfo& taskInfo)
204 : {
205 4 : auto deviceId = exceptionInfo->deviceid;
206 4 : HCCL_ERROR("[CcuTaskException][%s]Task from HCCL run failed.", __func__);
207 4 : HCCL_ERROR("[CcuTaskException]Task run failed, base information is deviceID:[%u], %s.",
208 : deviceId, taskInfo.GetIndopBaseInfo().c_str());
209 4 : HCCL_ERROR("[CcuTaskException]Task run failed, groupRank information is %s.",
210 : GetGroupRankInfo(taskInfo).c_str());
211 4 : HCCL_ERROR("[CcuTaskException]Task run failed, opData information is %s.", taskInfo.GetIndopDataInfo().c_str());
212 4 : CHK_PRT(InitChannelMap(deviceId, taskInfo.taskParam_.taskPara.Ccu.ccuKernelHandle));
213 4 : auto& ccuExDetailInfo = exceptionInfo->expandInfo.u.ccuInfo;
214 4 : isGetCqeErrInfo = true;
215 :
216 4 : const CcuVersionOps *ops = nullptr;
217 4 : if (GetCcuOps(ops) != HCCL_SUCCESS) {
218 4 : ops = nullptr;
219 : }
220 :
221 9 : for (uint32_t i = 0; i < ccuExDetailInfo.ccuMissionNum; ++i) { // ccuExDetailInfo.ccuMissionNum为1
222 5 : const auto& missionInfo = ccuExDetailInfo.missionInfo[i]; // 异常mission
223 5 : uint16_t status = static_cast<uint16_t>(missionInfo.status) << BYTE | missionInfo.subStatus;
224 5 : PrintCcuErrorInfo(deviceId, status, taskInfo);
225 : // 打印寄存器信息
226 5 : PrintPanicLogWithOps(missionInfo.panicLog, ops);
227 : }
228 :
229 4 : const int32_t devLogicId = static_cast<int32_t>(deviceId);
230 4 : if (CcuComponent::GetInstance(devLogicId).CleanTaskKillState() != HcclResult::HCCL_SUCCESS) {
231 1 : HCCL_ERROR("[CcuTaskException][%s] failed to clean ccu task kill state, devLogicId[%d].", __func__, devLogicId);
232 : }
233 :
234 4 : const uint8_t dieId = taskInfo.taskParam_.taskPara.Ccu.dieId;
235 4 : if (CcuComponent::GetInstance(devLogicId).CleanDieCkes(dieId) != HcclResult::HCCL_SUCCESS) {
236 1 : HCCL_ERROR("[CcuTaskException][%s] failed to clean ccu die ckes, dieId[%u], devLogicId[%d].",
237 : __func__, dieId, devLogicId);
238 : }
239 4 : }
240 :
241 10 : HcclResult CcuTaskException::InitChannelMap(s32 deviceId, u64 ccuKernelHandle)
242 : {
243 10 : std::lock_guard<std::mutex> lock(g_channelMapMutex);
244 :
245 10 : auto &kernelMgr = hcomm::CcuKernelMgr::GetInstance(deviceId);
246 10 : auto *kernel = kernelMgr.GetKernel(ccuKernelHandle);
247 10 : CHK_PRT_RET(kernel == nullptr, HCCL_ERROR("[%s]GetKernel nullptr, deviceId[%u], ccuKernelHandle[0x%llx]",
248 : __func__, deviceId, ccuKernelHandle), HCCL_E_PARA);
249 :
250 5 : const auto& ccuChannels = kernel->GetChannels();
251 8 : for (const auto& it : ccuChannels) {
252 5 : void *channelPtr = nullptr;
253 6 : CHK_RET(static_cast<HcclResult>(HcommChannelGet(it, &channelPtr)));
254 4 : auto *channelImpl = dynamic_cast<CcuUrmaChannel *>(static_cast<Channel *>(channelPtr));
255 4 : CHK_PTR_NULL(channelImpl);
256 3 : g_channelIdToHandle[channelImpl->GetChannelId()] = it;
257 3 : HCCL_RUN_INFO("[%s]deviceId[%d], ccuKernelHandle[0x%llx], channelId[%u], channelHandle[0x%llx]",
258 : __func__, deviceId, ccuKernelHandle, channelImpl->GetChannelId(), it);
259 : }
260 3 : return HCCL_SUCCESS;
261 10 : }
262 :
263 7 : std::string CcuTaskException::GetGroupRankInfo(const Hccl::TaskInfo& taskInfo)
264 : {
265 7 : if (taskInfo.dfxOpInfo_ == nullptr || taskInfo.dfxOpInfo_->comm_ == nullptr) {
266 5 : HCCL_ERROR("[TaskInfo][%s]TaskInfo communicator is nullptr.", __func__);
267 10 : return "";
268 : }
269 :
270 2 : hccl::CollComm *communicator = static_cast<hccl::CollComm*>(taskInfo.dfxOpInfo_->comm_);
271 : return Hccl::StringFormat("group:[%s], rankSize[%u], rankId[%d]",
272 2 : communicator->GetCommId().c_str(), communicator->GetRankSize(), communicator->GetMyRankId());
273 : }
274 :
275 4 : void CcuTaskException::PrintPanicLogInfo(const uint8_t *panicLog)
276 : {
277 4 : if (panicLog == nullptr) {
278 2 : HCCL_ERROR("[CcuTaskException][%s] panicLog is nullptr.", __func__);
279 4 : return;
280 : }
281 :
282 2 : const CcuVersionOps *ops = nullptr;
283 2 : if (GetCcuOps(ops) != HCCL_SUCCESS) {
284 2 : HCCL_ERROR("[CcuTaskException][%s] Failed to get printer ops for ccum_dfxInfo", __func__);
285 2 : return;
286 : }
287 0 : PrintPanicLogWithOps(panicLog, ops);
288 : }
289 :
290 : namespace {
291 19 : HcclResult QueryCcuCtxRaw(int32_t deviceId, uint32_t dieId, uint32_t ctxId, CcuOpcodeType op, const char *tag,
292 : uint8_t *buf, size_t bufLen, size_t &copiedLen)
293 : {
294 19 : copiedLen = 0;
295 19 : if (buf == nullptr || bufLen < CCU_CTX_RAW_CAPACITY) {
296 3 : HCCL_ERROR("[%s] invalid buffer: buf=%p, bufLen=%zu, required=%zu", tag, buf, bufLen, CCU_CTX_RAW_CAPACITY);
297 3 : return HCCL_E_PARA;
298 : }
299 16 : const auto memsetRet = memset_s(buf, bufLen, 0, bufLen);
300 16 : if (memsetRet != EOK) {
301 0 : HCCL_ERROR("[%s] memset_s failed, ret[%d]", tag, memsetRet);
302 0 : return HCCL_E_INTERNAL;
303 : }
304 :
305 16 : auto tlvHandle = Hccl::HccpTlvHdcManager::GetInstance().GetTlvHandle(deviceId);
306 16 : CHK_PRT_RET(tlvHandle == nullptr,
307 : HCCL_ERROR("[%s]tlvHandle is null, deviceId[%d]", __func__, deviceId), HCCL_E_PTR);
308 :
309 16 : CustomChannelInfoIn inBuff{};
310 16 : CustomChannelInfoOut outBuff{};
311 16 : inBuff.op = op;
312 16 : inBuff.data.dataInfo.udieIdx = dieId;
313 16 : inBuff.offsetStartIdx = ctxId;
314 16 : inBuff.data.dataInfo.dataArraySize = 1; // 读1个MissionContext
315 16 : inBuff.data.dataInfo.dataLen = CCU_CTX_RAW_CAPACITY;
316 :
317 16 : HcclResult ret = HccpRaTlvRequestForCustomChannel(tlvHandle, MSG_TYPE_CCU_DISPATCH_CMD, static_cast<void*>(&inBuff), static_cast<void*>(&outBuff));
318 :
319 16 : CHK_PRT_RET(ret != HCCL_SUCCESS, HCCL_ERROR("[%s]HccpRaTlvRequestForCustomChannel fail, ret[%u]", tag, ret), ret);
320 7 : const auto sret = memcpy_s(buf, bufLen, outBuff.data.dataInfo.dataArray, inBuff.data.dataInfo.dataLen);
321 7 : CHK_PRT_RET(sret != EOK, HCCL_ERROR("[%s]memcpy_s failed, ret[%d]", tag, sret), HCCL_E_MEMORY);
322 7 : copiedLen = inBuff.data.dataInfo.dataLen;
323 7 : return HCCL_SUCCESS;
324 : }
325 : } // namespace
326 :
327 : using GetMissionRawFunc = HcclResult (*)(
328 : int32_t deviceId, uint32_t dieId, uint32_t missionId, uint8_t *buf, size_t bufLen, size_t &copiedLen);
329 : using GenErrorInfoLoopGroupFunc = HcclResult (*)(const ErrorInfoBase &baseInfo,
330 : std::shared_ptr<CcuRep::CcuRepBase> repBase, CcuRep::CcuRepContext &ctx, std::vector<CcuErrorInfo> &errorInfo);
331 : using GenErrorInfoByRepTypeFunc = void (*)(
332 : const ErrorInfoBase &baseInfo, std::shared_ptr<CcuRep::CcuRepBase> repBase, std::vector<CcuErrorInfo> &errorInfo);
333 :
334 5 : static HcclResult DecodeRawMissionInfo(const uint8_t *raw, CcuMissionInfo &out)
335 : {
336 5 : const CcuVersionOps *ops = nullptr;
337 5 : if (GetCcuOps(ops) != HCCL_SUCCESS) {
338 5 : HCCL_ERROR("[DecodeRawMissionInfo] Failed to get ops for mission context");
339 5 : return HCCL_E_INTERNAL;
340 : }
341 0 : if (ops->getMissionInfo == nullptr) {
342 0 : HCCL_ERROR("[DecodeRawMissionInfo] getMissionInfo is nullptr, ops[%s]", ops->name);
343 0 : return HCCL_E_INTERNAL;
344 : }
345 0 : return ops->getMissionInfo(raw, &out);
346 : }
347 :
348 0 : static HcclResult FetchAndDecodeLoopInfo(int32_t deviceId, uint32_t dieId, uint32_t loopCtxId, CcuLoopInfo &out)
349 : {
350 0 : uint8_t loopRaw[CCU_CTX_RAW_CAPACITY] = {0};
351 0 : size_t loopRawLen = 0;
352 0 : if (QueryCcuCtxRaw(deviceId, dieId, loopCtxId, CcuOpcodeType::CCU_U_OP_GET_LOOP_CTX, "FetchAndDecodeLoopInfo",
353 : loopRaw, sizeof(loopRaw), loopRawLen)
354 0 : != HCCL_SUCCESS) {
355 0 : HCCL_ERROR("[FetchAndDecodeLoopInfo] Failed to fetch loop raw context, deviceId[%d]", deviceId);
356 0 : return HCCL_E_INTERNAL;
357 : }
358 0 : const CcuVersionOps *ops = nullptr;
359 0 : if (GetCcuOps(ops) != HCCL_SUCCESS) {
360 0 : HCCL_ERROR("[FetchAndDecodeLoopInfo] Failed to get ops for loop context");
361 0 : return HCCL_E_INTERNAL;
362 : }
363 0 : if (ops->getLoopInfo == nullptr) {
364 0 : HCCL_ERROR("[FetchAndDecodeLoopInfo] getLoopInfo is nullptr, ops[%s]", ops->name);
365 0 : return HCCL_E_INTERNAL;
366 : }
367 0 : return ops->getLoopInfo(loopRaw, &out);
368 : }
369 :
370 : // 计算报错指令附近可用 Rep 的起始指令，提取自 GetCcuErrorMsg 以降低其圈复杂度
371 0 : static uint16_t FindSurroundingBeginInstr(CcuRep::CcuRepContext &ctx, uint16_t startIns, uint16_t currIns)
372 : {
373 0 : uint16_t loopUpInstrNum = 10; // 出错指令前 10 条
374 0 : uint16_t beginIns = (currIns < loopUpInstrNum)
375 : ? startIns
376 0 : : ((currIns - loopUpInstrNum) > startIns ? (currIns - loopUpInstrNum) : startIns);
377 : // 从第一个非空 rep 开始，使用有符号变量避免 uint16_t 下溢无限循环
378 0 : for (int32_t instrId = static_cast<int32_t>(currIns); instrId >= static_cast<int32_t>(beginIns); instrId--) {
379 0 : if (ctx.GetRepByInstrId(static_cast<uint16_t>(instrId)) == nullptr) {
380 0 : beginIns = static_cast<uint16_t>(instrId) + 1U;
381 0 : break;
382 : }
383 : }
384 0 : return beginIns;
385 : }
386 :
387 5 : static HcclResult ValidateAndDecodeMissionContext(int32_t deviceId, uint16_t missionStatus,
388 : const Hccl::ParaCcu &ccuTaskParam, GetMissionRawFunc getMissionRaw, CcuMissionInfo &missionInfo)
389 : {
390 5 : CHK_PRT_RET((deviceId < 0 || static_cast<u32>(deviceId) >= MAX_MODULE_DEVICE_NUM),
391 : HCCL_ERROR("[CcuTaskException][GetCcuErrorMsg]deviceId[%d] error.", deviceId), HcclResult::HCCL_E_PARA);
392 :
393 5 : if (missionStatus == 0) {
394 0 : HCCL_ERROR(
395 : "[CcuErrorHandler][%s] no err found, mission status is 0, deviceId[%d], dieId[%u], execMissionId[%u]",
396 : __func__, deviceId, static_cast<u32>(ccuTaskParam.dieId), static_cast<u32>(ccuTaskParam.execMissionId));
397 0 : return HCCL_E_PARA;
398 : }
399 :
400 5 : uint8_t missionRaw[CCU_CTX_RAW_CAPACITY] = {0};
401 5 : size_t missionRawLen = 0;
402 10 : if (getMissionRaw(
403 5 : deviceId, ccuTaskParam.dieId, ccuTaskParam.execMissionId, missionRaw, sizeof(missionRaw), missionRawLen)
404 5 : != HCCL_SUCCESS) {
405 0 : HCCL_ERROR("[CcuErrorHandler][%s] Failed to fetch mission raw context, deviceId[%d]", __func__, deviceId);
406 0 : return HCCL_E_INTERNAL;
407 : }
408 5 : if (DecodeRawMissionInfo(missionRaw, missionInfo) != HCCL_SUCCESS) {
409 5 : HCCL_ERROR("[CcuErrorHandler][%s] Failed to decode mission info, deviceId[%d]", __func__, deviceId);
410 5 : return HCCL_E_INTERNAL;
411 : }
412 0 : return HCCL_SUCCESS;
413 : }
414 :
415 0 : static HcclResult ResolveRepContextAndCurrentRep(int32_t deviceId, const Hccl::ParaCcu &ccuTaskParam, uint16_t currIns,
416 : CcuRep::CcuRepContext *&ctx, std::shared_ptr<CcuRep::CcuRepBase> &rep, std::shared_ptr<CcuRep::CcuRepBase> &prevRep)
417 : {
418 0 : auto &kernelMgr = hcomm::CcuKernelMgr::GetInstance(deviceId);
419 0 : auto *kernel = kernelMgr.GetKernel(ccuTaskParam.ccuKernelHandle);
420 0 : CHK_PRT_RET(kernel == nullptr,
421 : HCCL_ERROR("[%s]GetKernel nullptr, deviceId[%u], ccuKernelHandle[0x%llx]", __func__, deviceId,
422 : ccuTaskParam.ccuKernelHandle),
423 : HCCL_E_PARA);
424 :
425 0 : ctx = reinterpret_cast<CcuRep::CcuRepContext *>(kernel);
426 0 : CHK_PRT_RET(ctx == nullptr,
427 : HCCL_ERROR("CcuContext not found, deviceId[%d], dieId[%u], missionId[%u], executeId[%llu]", deviceId,
428 : static_cast<u32>(ccuTaskParam.dieId), static_cast<u32>(ccuTaskParam.missionId), ccuTaskParam.executeId),
429 : HCCL_E_PARA);
430 :
431 0 : rep = ctx->GetRepByInstrId(currIns);
432 0 : CHK_PRT_RET(rep == nullptr,
433 : HCCL_ERROR("[CcuErrorHandler][%s] cannot find REP from current CcuContext, instrId[%u]", __func__, currIns),
434 : HCCL_E_PARA);
435 :
436 0 : prevRep = nullptr;
437 0 : if (currIns > 0) {
438 0 : prevRep = ctx->GetRepByInstrId(currIns - 1);
439 : }
440 :
441 0 : while (rep->Type() == CcuRep::CcuRepType::FUNC_BLOCK) {
442 0 : auto blockRep = static_pointer_cast<CcuRepBlock>(rep);
443 0 : rep = blockRep->GetRepByInstrId(currIns);
444 0 : CHK_PRT_RET(rep == nullptr,
445 : HCCL_ERROR(
446 : "Failed to find REP from FuncBlock, instrId[%u], FuncBlock[%s]", currIns, blockRep->GetLabel().c_str()),
447 : HcclResult::HCCL_E_PARA);
448 0 : }
449 0 : return HCCL_SUCCESS;
450 : }
451 :
452 0 : static void HandleSurroundingRepErrorInfo(int32_t deviceId, uint32_t execMissionId, const ErrorInfoBase &baseInfo,
453 : CcuRep::CcuRepContext &ctx, uint16_t startIns, uint16_t endIns, uint16_t currIns,
454 : GenErrorInfoByRepTypeFunc genByRepType, std::vector<CcuErrorInfo> &errorInfo)
455 : {
456 0 : HCCL_ERROR("[CcuErrorHandler]device %d, execMissionId[%u], startIns[%u], endIns[%u], currIns[%u]", deviceId,
457 : execMissionId, startIns, endIns, currIns);
458 0 : if (endIns == currIns) {
459 0 : HCCL_ERROR("[CcuErrorHandler]device %d SQE != CQE, endIns[%u], currIns[%u]", deviceId, endIns, currIns);
460 : }
461 :
462 0 : const uint16_t beginIns = FindSurroundingBeginInstr(ctx, startIns, currIns);
463 0 : for (uint16_t instrId = beginIns; instrId <= currIns; instrId++) {
464 0 : auto surroundingRep = ctx.GetRepByInstrId(instrId);
465 0 : if (surroundingRep == nullptr) {
466 0 : HCCL_WARNING(
467 : "[CcuErrorHandler][%s] cannot find REP from current CcuContext, instrId[%u]", __func__, instrId);
468 0 : continue;
469 : }
470 0 : genByRepType(baseInfo, surroundingRep, errorInfo);
471 0 : }
472 0 : }
473 :
474 13 : HcclResult CcuTaskException::GetCcuMissionContextRaw(
475 : int32_t deviceId, uint32_t dieId, uint32_t missionId, uint8_t *buf, size_t bufLen, size_t &copiedLen)
476 : {
477 13 : return QueryCcuCtxRaw(deviceId, dieId, missionId, CcuOpcodeType::CCU_U_OP_GET_MISSION_CTX,
478 13 : "GetCcuMissionContextRaw", buf, bufLen, copiedLen);
479 : }
480 :
481 12 : static string StatusCode2Str(uint8_t highPart, uint8_t lowPart)
482 : {
483 12 : HCCL_INFO("Mission Status Code: highPart[0x%02x], lowPart[0x%02x]", highPart, lowPart);
484 12 : const auto status = MISSION_STATUS_MAP.find(highPart);
485 12 : if (status == MISSION_STATUS_MAP.end()) {
486 2 : return "Unknown Status";
487 : }
488 11 : stringstream result;
489 11 : result << status->second;
490 :
491 11 : const auto lowMap = MISSION_SUB_STATUS_MAP.find(highPart);
492 11 : if (lowMap == MISSION_SUB_STATUS_MAP.end()) {
493 6 : return result.str();
494 : }
495 :
496 5 : const auto subStatus = lowMap->second.find(lowPart);
497 6 : const string subStatusMsg = subStatus == lowMap->second.end() ? "Unknown Status" : subStatus->second;
498 5 : result << ", " << subStatusMsg;
499 5 : return result.str();
500 11 : }
501 :
502 12 : void CcuTaskException::GenStatusInfo(const ErrorInfoBase &baseInfo, vector<CcuErrorInfo> &errorInfo)
503 : {
504 12 : CcuErrorInfo errorMsg{};
505 12 : errorMsg.type = CcuErrorType::MISSION;
506 12 : errorMsg.SetBaseInfo(CcuRep::CcuRepType::BASE, baseInfo.dieId, baseInfo.missionId, baseInfo.currentInsId);
507 12 : const auto baseInformation = Hccl::StringFormat("dieId[%u], missionId[%u]", baseInfo.dieId, baseInfo.missionId);
508 12 : const auto taskInformation = Hccl::StringFormat("currentInsId[%u], status[%u]", baseInfo.currentInsId, baseInfo.status);
509 12 : const uint8_t highPart = (baseInfo.status >> 8) & 0xFF; // 高8位
510 12 : const uint8_t lowPart = baseInfo.status & 0xFF; // 低8位
511 12 : string clusterMonitorErrMsg = CcuGetAndPrintClusterMonitorErr(baseInfo.deviceId);
512 :
513 12 : if (highPart == CCUM_EXECUTE_ERROR && lowPart == CCU_MISSION_TASK_KILLED) {
514 0 : RPT_INPUT_ERR(true,
515 : "EI0002",
516 : std::vector<std::string>({"remote_rankid", "base_information", "task_information", "group_rank_content"}),
517 : std::vector<std::string>({
518 : std::to_string(baseInfo.deviceId),
519 : baseInformation.c_str(), (taskInformation + clusterMonitorErrMsg).c_str(),
520 : "none"})
521 : );
522 : }
523 12 : const string statusMsg = StatusCode2Str(highPart, lowPart);
524 : const auto sRet
525 12 : = strncpy_s(errorMsg.msg.mission.missionError, MISSION_STATUS_MSG_LEN, statusMsg.c_str(), statusMsg.length());
526 12 : if (sRet != EOK) {
527 0 : HCCL_ERROR("[CcuErrorHandler][%s] strcpy failed, statusMsg: [%s].sRet:[%d]", __func__, statusMsg.c_str(), sRet);
528 : }
529 :
530 12 : errorInfo.push_back(errorMsg);
531 12 : }
532 :
533 3 : uint16_t CcuTaskException::GetCcuCKEValue(int32_t deviceId, uint32_t dieId, uint32_t ckeId)
534 : {
535 3 : CustomChannelInfoIn inBuff{};
536 3 : CustomChannelInfoOut outBuff{};
537 3 : auto tlvHandle = Hccl::HccpTlvHdcManager::GetInstance().GetTlvHandle(deviceId);
538 3 : CHK_PRT_RET(tlvHandle == nullptr,
539 : HCCL_ERROR("[%s]tlvHandle is null, deviceId[%d]", __func__, deviceId), INVALID_U16);
540 :
541 3 : inBuff.op = CcuOpcodeType::CCU_U_OP_GET_CKE;
542 3 : inBuff.data.dataInfo.udieIdx = dieId;
543 3 : inBuff.offsetStartIdx = ckeId;
544 3 : inBuff.data.dataInfo.dataArraySize = 1; // 读1个CKE
545 3 : inBuff.data.dataInfo.dataLen = sizeof(uint64_t) * inBuff.data.dataInfo.dataArraySize;
546 :
547 3 : HcclResult ret = HccpRaTlvRequestForCustomChannel(tlvHandle, MSG_TYPE_CCU_DISPATCH_CMD, static_cast<void*>(&inBuff), static_cast<void*>(&outBuff));
548 3 : CHK_PRT_RET(ret != HCCL_SUCCESS, HCCL_ERROR("[%s]HccpRaTlvRequestForCustomChannel fail, ret[%d]", __func__, ret), INVALID_U16);
549 :
550 1 : uint64_t ckeVal{0};
551 1 : s32 sret = memcpy_s(&ckeVal, sizeof(ckeVal), outBuff.data.dataInfo.dataArray, inBuff.data.dataInfo.dataLen);
552 1 : CHK_PRT_RET(sret != EOK, HCCL_ERROR("[%s]memcpy failed. errorno[%d]:", __func__, sret), INVALID_U16);
553 1 : return static_cast<uint16_t>(ckeVal);
554 : }
555 :
556 0 : void CcuTaskException::GenErrorInfoLocRecordEvent(const ErrorInfoBase &baseInfo, shared_ptr<CcuRep::CcuRepBase> repBase,
557 : vector<CcuErrorInfo> &errorInfo)
558 : {
559 0 : CcuErrorInfo errorMsg{};
560 0 : errorMsg.type = CcuErrorType::WAIT_SIGNAL;
561 0 : errorMsg.SetBaseInfo(repBase->Type(), baseInfo.dieId, baseInfo.missionId, repBase->StartInstrId());
562 :
563 0 : const auto rep = static_pointer_cast<CcuRep::CcuRepLocRecordEvent>(repBase);
564 0 : errorMsg.msg.waitSignal.signalId = rep->GetEventId();
565 0 : errorMsg.msg.waitSignal.signalValue = GetCcuCKEValue(baseInfo.deviceId, baseInfo.dieId, rep->GetEventId());
566 0 : errorMsg.msg.waitSignal.signalMask = rep->GetMask();
567 :
568 0 : errorInfo.push_back(errorMsg);
569 0 : }
570 :
571 0 : void CcuTaskException::GenErrorInfoLocWaitEvent(const ErrorInfoBase &baseInfo, shared_ptr<CcuRep::CcuRepBase> repBase,
572 : vector<CcuErrorInfo> &errorInfo)
573 : {
574 0 : CcuErrorInfo errorMsg{};
575 0 : errorMsg.type = CcuErrorType::WAIT_SIGNAL;
576 0 : errorMsg.SetBaseInfo(repBase->Type(), baseInfo.dieId, baseInfo.missionId, repBase->StartInstrId());
577 :
578 0 : const auto rep = static_pointer_cast<CcuRep::CcuRepLocWaitEvent>(repBase);
579 0 : errorMsg.msg.waitSignal.signalId = rep->GetEventId();
580 0 : errorMsg.msg.waitSignal.signalValue = GetCcuCKEValue(baseInfo.deviceId, baseInfo.dieId, rep->GetEventId());
581 0 : errorMsg.msg.waitSignal.signalMask = rep->GetMask();
582 :
583 0 : errorInfo.push_back(errorMsg);
584 0 : }
585 :
586 0 : void CcuTaskException::GenErrorInfoLocWaitNotify(const ErrorInfoBase &baseInfo, shared_ptr<CcuRep::CcuRepBase> repBase,
587 : vector<CcuErrorInfo> &errorInfo)
588 : {
589 0 : CcuErrorInfo errorMsg{};
590 0 : errorMsg.type = CcuErrorType::WAIT_SIGNAL;
591 0 : errorMsg.SetBaseInfo(repBase->Type(), baseInfo.dieId, baseInfo.missionId, repBase->StartInstrId());
592 :
593 0 : const auto rep = static_pointer_cast<CcuRep::CcuRepLocWaitNotify>(repBase);
594 0 : errorMsg.msg.waitSignal.signalId = rep->GetNotifyId();
595 0 : errorMsg.msg.waitSignal.signalValue = GetCcuCKEValue(baseInfo.deviceId, baseInfo.dieId, rep->GetNotifyId());
596 0 : errorMsg.msg.waitSignal.signalMask = rep->GetMask();
597 :
598 0 : errorInfo.push_back(errorMsg);
599 0 : }
600 :
601 3 : uint64_t CcuTaskException::GetCcuGSAValue(int32_t deviceId, uint32_t dieId, uint32_t gsaId)
602 : {
603 3 : uint64_t gsaVal{0};
604 :
605 3 : auto tlvHandle = Hccl::HccpTlvHdcManager::GetInstance().GetTlvHandle(deviceId);
606 3 : CHK_PRT_RET(tlvHandle == nullptr,
607 : HCCL_ERROR("[%s]tlvHandle is null, deviceId[%d]", __func__, deviceId), INVALID_U64);
608 :
609 3 : CustomChannelInfoIn inBuff{};
610 3 : CustomChannelInfoOut outBuff{};
611 :
612 3 : inBuff.op = CcuOpcodeType::CCU_U_OP_GET_GSA;
613 3 : inBuff.data.dataInfo.udieIdx = dieId;
614 3 : inBuff.offsetStartIdx = gsaId;
615 3 : inBuff.data.dataInfo.dataArraySize = 1; // 读1个GSA
616 3 : inBuff.data.dataInfo.dataLen = sizeof(uint64_t) * inBuff.data.dataInfo.dataArraySize;
617 :
618 3 : HcclResult ret = HccpRaTlvRequestForCustomChannel(tlvHandle, MSG_TYPE_CCU_DISPATCH_CMD, static_cast<void*>(&inBuff), static_cast<void*>(&outBuff));
619 3 : CHK_PRT_RET(ret != HCCL_SUCCESS, HCCL_ERROR("[%s]HccpRaTlvRequestForCustomChannel fail, ret[%d]", __func__, ret), INVALID_U64);
620 :
621 1 : auto sret = memcpy_s(&gsaVal, sizeof(gsaVal), outBuff.data.dataInfo.dataArray, inBuff.data.dataInfo.dataLen);
622 1 : CHK_PRT_RET(sret != EOK, HCCL_ERROR("[%s]memcpy failed. errorno[%d]:", __func__, sret), INVALID_U64);
623 1 : return gsaVal;
624 : }
625 :
626 3 : uint64_t CcuTaskException::GetCcuXnValue(int32_t deviceId, uint32_t dieId, uint32_t xnId)
627 : {
628 3 : auto tlvHandle = Hccl::HccpTlvHdcManager::GetInstance().GetTlvHandle(deviceId);
629 3 : CHK_PRT_RET(tlvHandle == nullptr,
630 : HCCL_ERROR("[%s]tlvHandle is null, deviceId[%d]", __func__, deviceId), INVALID_U64);
631 3 : uint64_t xnVal{0};
632 :
633 3 : CustomChannelInfoIn inBuff{};
634 3 : CustomChannelInfoOut outBuff{};
635 :
636 3 : inBuff.op = CcuOpcodeType::CCU_U_OP_GET_XN;
637 3 : inBuff.data.dataInfo.udieIdx = dieId;
638 3 : inBuff.offsetStartIdx = xnId;
639 3 : inBuff.data.dataInfo.dataArraySize = 1; // 读1个Xn
640 3 : inBuff.data.dataInfo.dataLen = sizeof(uint64_t) * inBuff.data.dataInfo.dataArraySize;
641 :
642 3 : HcclResult ret = HccpRaTlvRequestForCustomChannel(tlvHandle, MSG_TYPE_CCU_DISPATCH_CMD, static_cast<void*>(&inBuff), static_cast<void*>(&outBuff));
643 :
644 3 : CHK_PRT_RET(ret != HCCL_SUCCESS, HCCL_ERROR("[%s]HccpRaTlvRequestForCustomChannel fail, ret[%d]", __func__, ret), INVALID_U64);
645 :
646 1 : auto sret = memcpy_s(&xnVal, sizeof(xnVal), outBuff.data.dataInfo.dataArray, inBuff.data.dataInfo.dataLen);
647 1 : CHK_PRT_RET(sret != EOK, HCCL_ERROR("[%s]memcpy failed. errorno[%d]:", __func__, sret), INVALID_U64);
648 1 : return xnVal;
649 : }
650 :
651 0 : void CcuTaskException::GenErrorInfoRemPostSem(const ErrorInfoBase &baseInfo, shared_ptr<CcuRep::CcuRepBase> repBase,
652 : vector<CcuErrorInfo> &errorInfo)
653 : {
654 0 : CcuErrorInfo errorMsg{};
655 0 : errorMsg.type = CcuErrorType::WAIT_SIGNAL;
656 0 : errorMsg.SetBaseInfo(repBase->Type(), baseInfo.dieId, baseInfo.missionId, repBase->StartInstrId());
657 :
658 0 : const auto rep = static_pointer_cast<CcuRep::CcuRepRemPostSem>(repBase);
659 0 : errorMsg.msg.waitSignal.signalId = rep->GetId();
660 0 : errorMsg.msg.waitSignal.signalMask = rep->GetMask();
661 0 : auto sret = memset_s(errorMsg.msg.waitSignal.channelId, sizeof(errorMsg.msg.waitSignal.channelId), 0xFF,
662 : sizeof(errorMsg.msg.waitSignal.channelId));
663 0 : CHK_PRT_RET(sret != EOK, HCCL_ERROR("[%s]memset_s failed. errorno[%d]:", __func__, sret),);
664 0 : errorMsg.msg.waitSignal.channelId[0] = rep->GetChannelId();
665 :
666 0 : errorInfo.push_back(errorMsg);
667 0 : }
668 :
669 0 : void CcuTaskException::GenErrorInfoRemWaitSem(const ErrorInfoBase &baseInfo, shared_ptr<CcuRep::CcuRepBase> repBase,
670 : vector<CcuErrorInfo> &errorInfo)
671 : {
672 0 : CcuErrorInfo errorMsg{};
673 0 : errorMsg.type = CcuErrorType::WAIT_SIGNAL;
674 0 : errorMsg.SetBaseInfo(repBase->Type(), baseInfo.dieId, baseInfo.missionId, repBase->StartInstrId());
675 :
676 0 : const auto rep = static_pointer_cast<CcuRep::CcuRepRemWaitSem>(repBase);
677 0 : errorMsg.msg.waitSignal.signalId = rep->GetId();
678 0 : errorMsg.msg.waitSignal.signalValue = GetCcuCKEValue(baseInfo.deviceId, baseInfo.dieId,
679 0 : errorMsg.msg.waitSignal.signalId);
680 0 : errorMsg.msg.waitSignal.signalMask = rep->GetMask();
681 0 : auto sret = memset_s(errorMsg.msg.waitSignal.channelId, sizeof(errorMsg.msg.waitSignal.channelId), 0xFF,
682 : sizeof(errorMsg.msg.waitSignal.channelId));
683 0 : CHK_PRT_RET(sret != EOK, HCCL_ERROR("[%s]memset_s failed. errorno[%d]:", __func__, sret),);
684 0 : errorMsg.msg.waitSignal.channelId[0] = rep->GetChannelId();
685 :
686 0 : errorInfo.push_back(errorMsg);
687 0 : }
688 :
689 0 : void CcuTaskException::GenErrorInfoRemPostVar(const ErrorInfoBase &baseInfo, shared_ptr<CcuRep::CcuRepBase> repBase,
690 : vector<CcuErrorInfo> &errorInfo)
691 : {
692 0 : CcuErrorInfo errorMsg{};
693 0 : errorMsg.type = CcuErrorType::WAIT_SIGNAL;
694 0 : errorMsg.SetBaseInfo(repBase->Type(), baseInfo.dieId, baseInfo.missionId, repBase->StartInstrId());
695 :
696 0 : const auto rep = static_pointer_cast<CcuRep::CcuRepRemPostVar>(repBase);
697 0 : errorMsg.msg.waitSignal.signalId = rep->GetRmtCkeId();
698 0 : errorMsg.msg.waitSignal.signalMask = rep->GetMask();
699 0 : auto sret = memset_s(errorMsg.msg.waitSignal.channelId, sizeof(errorMsg.msg.waitSignal.channelId), 0xFF,
700 : sizeof(errorMsg.msg.waitSignal.channelId));
701 0 : CHK_PRT_RET(sret != EOK, HCCL_ERROR("[%s]memset_s failed. errorno[%d]:", __func__, sret),);
702 :
703 0 : errorMsg.msg.waitSignal.channelId[0] = rep->GetChannelId();
704 0 : errorMsg.msg.waitSignal.paramId = rep->GetRmtXnId();
705 0 : errorMsg.msg.waitSignal.paramValue = GetCcuXnValue(baseInfo.deviceId, baseInfo.dieId, rep->GetParam().Id());
706 :
707 0 : errorInfo.push_back(errorMsg);
708 0 : }
709 :
710 0 : void CcuTaskException::GenErrorInfoPostSharedSem(const ErrorInfoBase &baseInfo, shared_ptr<CcuRep::CcuRepBase> repBase,
711 : vector<CcuErrorInfo> &errorInfo)
712 : {
713 0 : CcuErrorInfo errorMsg{};
714 0 : errorMsg.type = CcuErrorType::WAIT_SIGNAL;
715 0 : errorMsg.SetBaseInfo(repBase->Type(), baseInfo.dieId, baseInfo.missionId, repBase->StartInstrId());
716 :
717 0 : const auto rep = static_pointer_cast<CcuRep::CcuRepRecordSharedNotify>(repBase);
718 0 : errorMsg.msg.waitSignal.signalId = rep->GetNotifyId();
719 0 : errorMsg.msg.waitSignal.signalMask = rep->GetMask();
720 :
721 0 : errorInfo.push_back(errorMsg);
722 0 : }
723 :
724 0 : void CcuTaskException::GenErrorInfoRead(const ErrorInfoBase &baseInfo, shared_ptr<CcuRep::CcuRepBase> repBase,
725 : vector<CcuErrorInfo> &errorInfo)
726 : {
727 0 : CcuErrorInfo errorMsg{};
728 0 : errorMsg.type = CcuErrorType::TRANS_MEM;
729 0 : errorMsg.SetBaseInfo(repBase->Type(), baseInfo.dieId, baseInfo.missionId, repBase->StartInstrId());
730 :
731 0 : const auto rep = static_pointer_cast<CcuRep::CcuRepRead>(repBase);
732 0 : errorMsg.msg.transMem.locAddr = GetCcuGSAValue(baseInfo.deviceId, baseInfo.dieId, rep->GetLocAddrId());
733 0 : errorMsg.msg.transMem.locToken = GetCcuXnValue(baseInfo.deviceId, baseInfo.dieId, rep->GetLocTokenId());
734 0 : errorMsg.msg.transMem.rmtAddr = GetCcuGSAValue(baseInfo.deviceId, baseInfo.dieId, rep->GetRemAddrId());
735 0 : errorMsg.msg.transMem.rmtToken = GetCcuXnValue(baseInfo.deviceId, baseInfo.dieId, rep->GetRemTokenId());
736 0 : errorMsg.msg.transMem.len = GetCcuXnValue(baseInfo.deviceId, baseInfo.dieId, rep->GetLenId());
737 0 : errorMsg.msg.transMem.signalMask = rep->GetMask();
738 0 : errorMsg.msg.transMem.signalId = rep->GetSemId();
739 0 : errorMsg.msg.transMem.channelId = rep->GetChannelId();
740 0 : errorInfo.push_back(errorMsg);
741 0 : }
742 :
743 0 : void CcuTaskException::GenErrorInfoWrite(const ErrorInfoBase &baseInfo, shared_ptr<CcuRep::CcuRepBase> repBase,
744 : vector<CcuErrorInfo> &errorInfo)
745 : {
746 0 : CcuErrorInfo errorMsg{};
747 0 : errorMsg.type = CcuErrorType::TRANS_MEM;
748 0 : errorMsg.SetBaseInfo(repBase->Type(), baseInfo.dieId, baseInfo.missionId, repBase->StartInstrId());
749 :
750 0 : const auto rep = static_pointer_cast<CcuRep::CcuRepWrite>(repBase);
751 0 : errorMsg.msg.transMem.locAddr = GetCcuGSAValue(baseInfo.deviceId, baseInfo.dieId, rep->GetLocAddrId());
752 0 : errorMsg.msg.transMem.locToken = GetCcuXnValue(baseInfo.deviceId, baseInfo.dieId, rep->GetLocTokenId());
753 0 : errorMsg.msg.transMem.rmtAddr = GetCcuGSAValue(baseInfo.deviceId, baseInfo.dieId, rep->GetRemAddrId());
754 0 : errorMsg.msg.transMem.rmtToken = GetCcuXnValue(baseInfo.deviceId, baseInfo.dieId, rep->GetRemTokenId());
755 0 : errorMsg.msg.transMem.len = GetCcuXnValue(baseInfo.deviceId, baseInfo.dieId, rep->GetLenId());
756 0 : errorMsg.msg.transMem.signalId = rep->GetSemId();
757 0 : errorMsg.msg.transMem.signalMask = rep->GetMask();
758 0 : errorMsg.msg.transMem.channelId = rep->GetChannelId();
759 :
760 0 : errorInfo.push_back(errorMsg);
761 0 : }
762 :
763 0 : void CcuTaskException::GenErrorInfoLocalCpy(const ErrorInfoBase &baseInfo, shared_ptr<CcuRep::CcuRepBase> repBase,
764 : vector<CcuErrorInfo> &errorInfo)
765 : {
766 0 : CcuErrorInfo errorMsg{};
767 0 : errorMsg.type = CcuErrorType::TRANS_MEM;
768 0 : errorMsg.SetBaseInfo(repBase->Type(), baseInfo.dieId, baseInfo.missionId, repBase->StartInstrId());
769 :
770 0 : const auto rep = static_pointer_cast<CcuRep::CcuRepLocCpy>(repBase);
771 0 : errorMsg.msg.transMem.locAddr = GetCcuGSAValue(baseInfo.deviceId, baseInfo.dieId, rep->GetSrcAddrId());
772 0 : errorMsg.msg.transMem.rmtAddr = GetCcuGSAValue(baseInfo.deviceId, baseInfo.dieId, rep->GetDstAddrId());
773 0 : errorMsg.msg.transMem.locToken = GetCcuXnValue(baseInfo.deviceId, baseInfo.dieId, rep->GetSrcTokenId());
774 0 : errorMsg.msg.transMem.rmtToken = GetCcuXnValue(baseInfo.deviceId, baseInfo.dieId, rep->GetDstTokenId());
775 0 : errorMsg.msg.transMem.len = GetCcuXnValue(baseInfo.deviceId, baseInfo.dieId, rep->GetLenId());
776 0 : errorMsg.msg.transMem.signalId = rep->GetSemId();
777 0 : errorMsg.msg.transMem.signalMask = rep->GetMask();
778 :
779 0 : errorInfo.push_back(errorMsg);
780 0 : }
781 :
782 0 : void CcuTaskException::GenErrorInfoLocalReduce(const ErrorInfoBase &baseInfo, shared_ptr<CcuRep::CcuRepBase> repBase,
783 : vector<CcuErrorInfo> &errorInfo)
784 : {
785 0 : CcuErrorInfo errorMsg{};
786 0 : errorMsg.type = CcuErrorType::TRANS_MEM;
787 0 : errorMsg.SetBaseInfo(repBase->Type(), baseInfo.dieId, baseInfo.missionId, repBase->StartInstrId());
788 :
789 0 : const auto rep = static_pointer_cast<CcuRep::CcuRepLocCpy>(repBase);
790 0 : errorMsg.msg.transMem.locAddr = GetCcuGSAValue(baseInfo.deviceId, baseInfo.dieId, rep->GetSrcAddrId());
791 0 : errorMsg.msg.transMem.locToken = GetCcuXnValue(baseInfo.deviceId, baseInfo.dieId, rep->GetSrcTokenId());
792 0 : errorMsg.msg.transMem.rmtAddr = GetCcuGSAValue(baseInfo.deviceId, baseInfo.dieId, rep->GetDstAddrId());
793 0 : errorMsg.msg.transMem.rmtToken = GetCcuXnValue(baseInfo.deviceId, baseInfo.dieId, rep->GetDstTokenId());
794 0 : errorMsg.msg.transMem.len = GetCcuXnValue(baseInfo.deviceId, baseInfo.dieId, rep->GetLenId());
795 0 : errorMsg.msg.transMem.signalId = rep->GetSemId();
796 0 : errorMsg.msg.transMem.signalMask = rep->GetMask();
797 0 : errorMsg.msg.transMem.opType = rep->GetOpType();
798 0 : errorMsg.msg.transMem.dataType = rep->GetDataType();
799 :
800 0 : errorInfo.push_back(errorMsg);
801 0 : }
802 :
803 0 : void CcuTaskException::GenErrorInfoBufRead(const ErrorInfoBase &baseInfo, shared_ptr<CcuRep::CcuRepBase> repBase,
804 : vector<CcuErrorInfo> &errorInfo)
805 : {
806 0 : CcuErrorInfo errorMsg{};
807 0 : errorMsg.type = CcuErrorType::BUF_TRANS_MEM;
808 0 : errorMsg.SetBaseInfo(repBase->Type(), baseInfo.dieId, baseInfo.missionId, repBase->StartInstrId());
809 :
810 0 : const auto rep = static_pointer_cast<CcuRep::CcuRepBufRead>(repBase);
811 0 : errorMsg.msg.bufTransMem.bufId = GetMSIdPerDie(rep->GetDstAddrId());
812 0 : errorMsg.msg.bufTransMem.addr = GetCcuGSAValue(baseInfo.deviceId, baseInfo.dieId, rep->GetSrcAddrId());
813 0 : errorMsg.msg.bufTransMem.token = GetCcuXnValue(baseInfo.deviceId, baseInfo.dieId, rep->GetSrcTokenId());
814 0 : errorMsg.msg.bufTransMem.len = GetCcuXnValue(baseInfo.deviceId, baseInfo.dieId, rep->GetLenId());
815 0 : errorMsg.msg.bufTransMem.signalId = rep->GetSemId();
816 0 : errorMsg.msg.bufTransMem.signalMask = rep->GetMask();
817 0 : errorMsg.msg.bufTransMem.channelId = rep->GetChannelId();
818 0 : errorInfo.push_back(errorMsg);
819 0 : }
820 :
821 0 : void CcuTaskException::GenErrorInfoBufWrite(const ErrorInfoBase &baseInfo, shared_ptr<CcuRep::CcuRepBase> repBase,
822 : vector<CcuErrorInfo> &errorInfo)
823 : {
824 0 : CcuErrorInfo errorMsg{};
825 0 : errorMsg.type = CcuErrorType::BUF_TRANS_MEM;
826 0 : errorMsg.SetBaseInfo(repBase->Type(), baseInfo.dieId, baseInfo.missionId, repBase->StartInstrId());
827 :
828 0 : const auto rep = static_pointer_cast<CcuRep::CcuRepBufWrite>(repBase);
829 0 : errorMsg.msg.bufTransMem.bufId = GetMSIdPerDie(rep->GetSrcId());
830 0 : errorMsg.msg.bufTransMem.addr = GetCcuGSAValue(baseInfo.deviceId, baseInfo.dieId, rep->GetDstAddrId());
831 0 : errorMsg.msg.bufTransMem.token = GetCcuXnValue(baseInfo.deviceId, baseInfo.dieId, rep->GetDstTokenId());
832 0 : errorMsg.msg.bufTransMem.len = GetCcuXnValue(baseInfo.deviceId, baseInfo.dieId, rep->GetLenId());
833 0 : errorMsg.msg.bufTransMem.signalId = rep->GetSemId();
834 0 : errorMsg.msg.bufTransMem.signalMask = rep->GetMask();
835 0 : errorMsg.msg.bufTransMem.channelId = rep->GetChannelId();
836 :
837 0 : errorInfo.push_back(errorMsg);
838 0 : }
839 :
840 0 : void CcuTaskException::GenErrorInfoBufLocRead(const ErrorInfoBase &baseInfo, shared_ptr<CcuRep::CcuRepBase> repBase,
841 : vector<CcuErrorInfo> &errorInfo)
842 : {
843 0 : CcuErrorInfo errorMsg{};
844 0 : errorMsg.type = CcuErrorType::BUF_TRANS_MEM;
845 0 : errorMsg.SetBaseInfo(repBase->Type(), baseInfo.dieId, baseInfo.missionId, repBase->StartInstrId());
846 :
847 0 : const auto rep = static_pointer_cast<CcuRep::CcuRepBufLocRead>(repBase);
848 0 : errorMsg.msg.bufTransMem.bufId = GetMSIdPerDie(rep->GetDstId());
849 0 : errorMsg.msg.bufTransMem.addr = GetCcuGSAValue(baseInfo.deviceId, baseInfo.dieId, rep->GetSrcAddrId());
850 0 : errorMsg.msg.bufTransMem.token = GetCcuXnValue(baseInfo.deviceId, baseInfo.dieId, rep->GetSrcTokenId());
851 0 : errorMsg.msg.bufTransMem.len = GetCcuXnValue(baseInfo.deviceId, baseInfo.dieId, rep->GetLenId());
852 0 : errorMsg.msg.bufTransMem.signalId = rep->GetSemId();
853 0 : errorMsg.msg.bufTransMem.signalMask = rep->GetMask();
854 :
855 0 : errorInfo.push_back(errorMsg);
856 0 : }
857 :
858 0 : void CcuTaskException::GenErrorInfoBufLocWrite(const ErrorInfoBase &baseInfo, shared_ptr<CcuRep::CcuRepBase> repBase,
859 : vector<CcuErrorInfo> &errorInfo)
860 : {
861 0 : CcuErrorInfo errorMsg{};
862 0 : errorMsg.type = CcuErrorType::BUF_TRANS_MEM;
863 0 : errorMsg.SetBaseInfo(repBase->Type(), baseInfo.dieId, baseInfo.missionId, repBase->StartInstrId());
864 :
865 0 : const auto rep = static_pointer_cast<CcuRep::CcuRepBufLocWrite>(repBase);
866 0 : errorMsg.msg.bufTransMem.bufId = GetMSIdPerDie(rep->GetSrcAddrId());
867 0 : errorMsg.msg.bufTransMem.addr = GetCcuGSAValue(baseInfo.deviceId, baseInfo.dieId, rep->GetDstAddrId());
868 0 : errorMsg.msg.bufTransMem.token = GetCcuXnValue(baseInfo.deviceId, baseInfo.dieId, rep->GetDstTokenId());
869 0 : errorMsg.msg.bufTransMem.len = GetCcuXnValue(baseInfo.deviceId, baseInfo.dieId, rep->GetLenId());
870 0 : errorMsg.msg.bufTransMem.signalId = rep->GetSemId();
871 0 : errorMsg.msg.bufTransMem.signalMask = rep->GetMask();
872 :
873 0 : errorInfo.push_back(errorMsg);
874 0 : }
875 :
876 0 : void CcuTaskException::GenErrorInfoBufReduce(const ErrorInfoBase &baseInfo, shared_ptr<CcuRep::CcuRepBase> repBase,
877 : vector<CcuErrorInfo> &errorInfo)
878 : {
879 0 : CcuErrorInfo errorMsg{};
880 0 : errorMsg.type = CcuErrorType::BUF_REDUCE;
881 0 : errorMsg.SetBaseInfo(repBase->Type(), baseInfo.dieId, baseInfo.missionId, repBase->StartInstrId());
882 :
883 0 : const auto rep = static_pointer_cast<CcuRep::CcuRepBufReduce>(repBase);
884 0 : errorMsg.msg.bufReduce.count = rep->GetCount();
885 0 : errorMsg.msg.bufReduce.dataType = rep->GetDataType();
886 0 : errorMsg.msg.bufReduce.outputDataType = rep->GetOutputDataType();
887 0 : errorMsg.msg.bufReduce.opType = rep->GetOpType();
888 0 : errorMsg.msg.bufReduce.signalId = rep->GetSemId();
889 0 : errorMsg.msg.bufReduce.signalMask = rep->GetMask();
890 0 : errorMsg.msg.bufReduce.xnIdLength = rep->GetXnLengthId();
891 0 : const auto &buffs = rep->GetMem();
892 0 : auto sret = memset_s(errorMsg.msg.bufReduce.bufIds, sizeof(errorMsg.msg.bufReduce.bufIds), 0xFF,
893 : sizeof(errorMsg.msg.bufReduce.bufIds));
894 0 : CHK_PRT_RET(sret != EOK, HCCL_ERROR("[%s]memset failed. errorno[%d]:", __func__, sret), );
895 0 : for (uint32_t i = 0; i < buffs.size() && i < BUF_REDUCE_ID_SIZE; ++i) {
896 0 : errorMsg.msg.bufReduce.bufIds[i] = GetMSIdPerDie(buffs[i].Id());
897 : }
898 :
899 0 : errorInfo.push_back(errorMsg);
900 0 : }
901 0 : void CcuTaskException::GenErrorInfoDefault(const ErrorInfoBase &baseInfo, shared_ptr<CcuRep::CcuRepBase> repBase,
902 : vector<CcuErrorInfo> &errorInfo)
903 : {
904 0 : CcuErrorInfo errorMsg{};
905 0 : errorMsg.type = CcuErrorType::DEFAULT;
906 0 : errorMsg.SetBaseInfo(repBase->Type(), baseInfo.dieId, baseInfo.missionId, repBase->StartInstrId());
907 0 : errorInfo.push_back(errorMsg);
908 0 : }
909 :
910 0 : void CcuTaskException::GenErrorInfoByRepType(const ErrorInfoBase &baseInfo, shared_ptr<CcuRep::CcuRepBase> repBase,
911 : vector<CcuErrorInfo> &errorInfo)
912 : {
913 : using GenErrorInfoFunc = void (*)(const ErrorInfoBase &baseInfo, shared_ptr<CcuRep::CcuRepBase> repBase,
914 : vector<CcuErrorInfo> &errorInfo);
915 : static const map<CcuRep::CcuRepType, GenErrorInfoFunc> HANDLER_MAP {
916 : // WAIT_SIGNAL
917 : {CcuRep::CcuRepType::LOC_RECORD_EVENT, &CcuTaskException::GenErrorInfoLocRecordEvent},
918 : {CcuRep::CcuRepType::LOC_WAIT_EVENT, &CcuTaskException::GenErrorInfoLocWaitEvent},
919 : {CcuRep::CcuRepType::LOC_WAIT_NOTIFY, &CcuTaskException::GenErrorInfoLocWaitNotify},
920 : {CcuRep::CcuRepType::REM_POST_SEM, &CcuTaskException::GenErrorInfoRemPostSem},
921 : {CcuRep::CcuRepType::REM_WAIT_SEM, &CcuTaskException::GenErrorInfoRemWaitSem},
922 : {CcuRep::CcuRepType::REM_POST_VAR, &CcuTaskException::GenErrorInfoRemPostVar},
923 : {CcuRep::CcuRepType::RECORD_SHARED_NOTIFY, &CcuTaskException::GenErrorInfoPostSharedSem},
924 : // TRANS_MEM
925 : {CcuRep::CcuRepType::READ, &CcuTaskException::GenErrorInfoRead},
926 : {CcuRep::CcuRepType::WRITE, &CcuTaskException::GenErrorInfoWrite},
927 : {CcuRep::CcuRepType::LOCAL_CPY, &CcuTaskException::GenErrorInfoLocalCpy},
928 : {CcuRep::CcuRepType::LOCAL_REDUCE, &CcuTaskException::GenErrorInfoLocalReduce},
929 : // BUF_TRANS_MEM
930 : {CcuRep::CcuRepType::BUF_READ, &CcuTaskException::GenErrorInfoBufRead},
931 : {CcuRep::CcuRepType::BUF_WRITE, &CcuTaskException::GenErrorInfoBufWrite},
932 : {CcuRep::CcuRepType::BUF_LOC_READ, &CcuTaskException::GenErrorInfoBufLocRead},
933 : {CcuRep::CcuRepType::BUF_LOC_WRITE, &CcuTaskException::GenErrorInfoBufLocWrite},
934 : // BUF_REDUCE
935 : {CcuRep::CcuRepType::BUF_REDUCE, &CcuTaskException::GenErrorInfoBufReduce}
936 0 : };
937 0 : const auto funcIt = HANDLER_MAP.find(repBase->Type());
938 0 : HCCL_INFO("[%s]type[%d]", __func__, repBase->Type());
939 0 : if (funcIt == HANDLER_MAP.end()) {
940 : // DEFAULT, chip error
941 0 : GenErrorInfoDefault(baseInfo, repBase, errorInfo);
942 : } else {
943 0 : (funcIt->second)(baseInfo, repBase, errorInfo);
944 : }
945 0 : }
946 :
947 6 : HcclResult CcuTaskException::GetCcuLoopContextRaw(
948 : int32_t deviceId, uint32_t dieId, uint32_t loopCtxId, uint8_t *buf, size_t bufLen, size_t &copiedLen)
949 : {
950 6 : return QueryCcuCtxRaw(deviceId, dieId, loopCtxId, CcuOpcodeType::CCU_U_OP_GET_LOOP_CTX, "GetCcuLoopContextRaw", buf,
951 6 : bufLen, copiedLen);
952 : }
953 :
954 0 : HcclResult CcuTaskException::GenErrorInfoLoop(const ErrorInfoBase &baseInfo, CcuRep::CcuRepContext &ctx,
955 : vector<CcuErrorInfo> &errorInfo)
956 : {
957 : // 找LoopRep
958 0 : auto repBase = ctx.GetRepByInstrId(baseInfo.currentInsId);
959 0 : if (repBase == nullptr || repBase->Type() != CcuRep::CcuRepType::LOOP) {
960 0 : HCCL_ERROR("Failed to find Loop REP from CcuContext, instrId[%u]", baseInfo.currentInsId);
961 0 : return HCCL_E_PARA;
962 : }
963 0 : const auto rep = static_pointer_cast<CcuRep::CcuRepLoop>(repBase);
964 :
965 0 : CcuErrorInfo errorMsg{};
966 0 : errorMsg.type = CcuErrorType::LOOP;
967 0 : errorMsg.SetBaseInfo(repBase->Type(), baseInfo.dieId, baseInfo.missionId, baseInfo.currentInsId);
968 :
969 0 : LoopXm loopXm{};
970 0 : loopXm.value = GetCcuXnValue(baseInfo.deviceId, baseInfo.dieId, rep->GetLoopParam()->Id());
971 0 : CcuLoopInfo loopInfo{};
972 0 : if (FetchAndDecodeLoopInfo(baseInfo.deviceId, baseInfo.dieId, loopXm.loopCtxId, loopInfo) != HCCL_SUCCESS) {
973 0 : HCCL_ERROR("[GenErrorInfoLoop] Failed to fetch/decode loop info, deviceId[%d]", baseInfo.deviceId);
974 0 : return HCCL_E_INTERNAL;
975 : }
976 0 : errorMsg.msg.loop.startInstrId = rep->GetLoopBlock()->StartInstrId();
977 0 : errorMsg.msg.loop.endInstrId = rep->GetLoopBlock()->StartInstrId() + rep->GetLoopBlock()->InstrCount() - 1;
978 0 : errorMsg.msg.loop.loopEngineId = loopXm.loopCtxId;
979 0 : errorMsg.msg.loop.loopCnt = static_cast<uint16_t>(loopXm.loopCnt);
980 0 : errorMsg.msg.loop.loopCurrentCnt = loopInfo.currentCnt;
981 0 : errorMsg.msg.loop.addrStride = loopInfo.addrStride;
982 :
983 0 : errorInfo.push_back(errorMsg);
984 :
985 : // 解析Loop内的异常Rep
986 0 : for (uint16_t loopCurrentIns = errorMsg.msg.loop.startInstrId; loopCurrentIns <= errorMsg.msg.loop.endInstrId;
987 : loopCurrentIns++) {
988 0 : auto inLoopExRep = rep->GetLoopBlock()->GetRepByInstrId(loopCurrentIns);
989 0 : if (inLoopExRep == nullptr) {
990 0 : HCCL_ERROR("Failed to find REP from Loop, instrId[%u], Loop[%s]", loopCurrentIns, rep->GetLabel().c_str());
991 0 : return HCCL_E_PARA;
992 : }
993 0 : ErrorInfoBase loopErrBase{baseInfo.deviceId, baseInfo.dieId, baseInfo.missionId, loopCurrentIns,
994 0 : baseInfo.status};
995 0 : GenErrorInfoByRepType(loopErrBase, inLoopExRep, errorInfo);
996 0 : }
997 0 : return HCCL_SUCCESS;
998 0 : }
999 :
1000 0 : HcclResult CcuTaskException::GenErrorInfoLoopGroup(const ErrorInfoBase &baseInfo, shared_ptr<CcuRep::CcuRepBase> repBase,
1001 : CcuRep::CcuRepContext &ctx, vector<CcuErrorInfo> &errorInfo)
1002 : {
1003 0 : CcuErrorInfo errorMsg{};
1004 0 : errorMsg.type = CcuErrorType::LOOP_GROUP;
1005 0 : errorMsg.SetBaseInfo(repBase->Type(), baseInfo.dieId, baseInfo.missionId, repBase->StartInstrId());
1006 :
1007 0 : const auto rep = static_pointer_cast<CcuRep::CcuRepLoopGroupBundle>(repBase);
1008 0 : const auto startLoopInstrId = rep->GetStartLoopInstrId();
1009 0 : LoopGroupXn loopGroupXn{};
1010 0 : loopGroupXn.value = GetCcuXnValue(baseInfo.deviceId, baseInfo.dieId, rep->GetOffsetParam().Id());
1011 0 : errorMsg.msg.loopGroup.startLoopInsId = startLoopInstrId;
1012 0 : errorMsg.msg.loopGroup.loopInsCnt = static_cast<uint16_t>(loopGroupXn.loopInsCnt);
1013 0 : errorMsg.msg.loopGroup.expandOffset = static_cast<uint16_t>(loopGroupXn.expandOffset);
1014 0 : errorMsg.msg.loopGroup.expandCnt = static_cast<uint16_t>(loopGroupXn.expandCnt);
1015 :
1016 0 : errorInfo.push_back(errorMsg);
1017 :
1018 : // 处理loop
1019 0 : for (uint16_t i = 0; i < loopGroupXn.loopInsCnt; ++i) {
1020 0 : uint16_t loopInsId = startLoopInstrId + i;
1021 0 : ErrorInfoBase loopErrInfoBase{baseInfo.deviceId, baseInfo.dieId, baseInfo.missionId, loopInsId,
1022 0 : baseInfo.status};
1023 0 : CHK_RET(GenErrorInfoLoop(loopErrInfoBase, ctx, errorInfo));
1024 : }
1025 0 : return HCCL_SUCCESS;
1026 0 : }
1027 :
1028 0 : static HcclResult HandleCurrentRepErrorInfo(const ErrorInfoBase &baseInfo, CcuRep::CcuRepContext &ctx,
1029 : const std::shared_ptr<CcuRep::CcuRepBase> &rep, const std::shared_ptr<CcuRep::CcuRepBase> &prevRep,
1030 : GenErrorInfoLoopGroupFunc genLoopGroup, GenErrorInfoByRepTypeFunc genByRepType,
1031 : std::vector<CcuErrorInfo> &errorInfo)
1032 : {
1033 0 : if ((prevRep != nullptr && prevRep->Type() == CcuRep::CcuRepType::LOOPGROUP)
1034 0 : || (rep->Type() == CcuRep::CcuRepType::LOOPGROUP)) {
1035 0 : CHK_RET(genLoopGroup(baseInfo, prevRep, ctx, errorInfo));
1036 0 : return HCCL_SUCCESS;
1037 : }
1038 :
1039 0 : if (rep->Type() == CcuRep::CcuRepType::LOC_WAIT_EVENT || rep->Type() == CcuRep::CcuRepType::LOC_WAIT_NOTIFY) {
1040 0 : genByRepType(baseInfo, rep, errorInfo);
1041 0 : uint16_t actValue = errorInfo.back().msg.waitSignal.signalValue;
1042 0 : uint16_t expValue = errorInfo.back().msg.waitSignal.signalMask;
1043 0 : for (uint16_t i = 0; i < 16; ++i) { // CKE的bit数最多为16
1044 0 : uint16_t mask = 1 << i; // 创建一个用于检查第 i 位的掩码
1045 0 : if ((expValue & mask) != 0 && (actValue & mask) == 0) {
1046 0 : auto depRepVec = static_pointer_cast<CcuRep::CcuRepLocWaitEvent>(rep)->GetDependencyInfo(mask);
1047 0 : for (const auto& depRep : depRepVec) {
1048 0 : genByRepType(baseInfo, depRep, errorInfo);
1049 : }
1050 0 : }
1051 : }
1052 0 : return HCCL_SUCCESS;
1053 : }
1054 :
1055 0 : genByRepType(baseInfo, rep, errorInfo);
1056 0 : return HCCL_SUCCESS;
1057 : }
1058 :
1059 5 : HcclResult CcuTaskException::GetCcuErrorMsg(int32_t deviceId, uint16_t missionStatus, const Hccl::ParaCcu &ccuTaskParam,
1060 : std::vector<CcuErrorInfo> &errorInfo)
1061 : {
1062 5 : HCCL_INFO("[CcuTaskException]%s: deviceId[%d], dieId[%u], missionId[%u], execMissionId[%u], executeId[%llu].",
1063 : __func__, deviceId, static_cast<u32>(ccuTaskParam.dieId), static_cast<u32>(ccuTaskParam.missionId),
1064 : static_cast<u32>(ccuTaskParam.execMissionId), ccuTaskParam.executeId);
1065 :
1066 5 : CcuMissionInfo missionInfo{};
1067 5 : CHK_RET(
1068 : ValidateAndDecodeMissionContext(deviceId, missionStatus, ccuTaskParam, GetCcuMissionContextRaw, missionInfo));
1069 :
1070 0 : const uint16_t currIns = missionInfo.currentIns;
1071 0 : CcuRep::CcuRepContext *ctx = nullptr;
1072 0 : std::shared_ptr<CcuRep::CcuRepBase> rep = nullptr;
1073 0 : std::shared_ptr<CcuRep::CcuRepBase> prevRep = nullptr;
1074 0 : CHK_RET(ResolveRepContextAndCurrentRep(deviceId, ccuTaskParam, currIns, ctx, rep, prevRep));
1075 :
1076 : // 分类处理Rep, 返回异常信息
1077 0 : ErrorInfoBase baseInfo{deviceId, ccuTaskParam.dieId, ccuTaskParam.missionId, currIns, missionStatus};
1078 0 : GenStatusInfo(baseInfo, errorInfo);
1079 0 : CHK_RET(HandleCurrentRepErrorInfo(
1080 : baseInfo, *ctx, rep, prevRep, GenErrorInfoLoopGroup, GenErrorInfoByRepType, errorInfo));
1081 :
1082 0 : const uint16_t endIns = missionInfo.endIns;
1083 0 : const uint16_t startIns = missionInfo.startIns;
1084 0 : HandleSurroundingRepErrorInfo(deviceId, ccuTaskParam.execMissionId, baseInfo, *ctx, startIns, endIns, currIns,
1085 : GenErrorInfoByRepType, errorInfo);
1086 0 : return HCCL_SUCCESS;
1087 0 : }
1088 :
1089 0 : void CcuTaskException::GetCcuCqeErrRemoteLocalIdByRankId(hccl::CollComm* collComm, uint32_t rankid, u32 &remoteLocalId)
1090 : {
1091 0 : if (collComm == nullptr) {
1092 0 : HCCL_ERROR("[GetCcuCqeErrRemoteLocalIdByRankId] collComm is nullptr");
1093 0 : return;
1094 : }
1095 :
1096 0 : if (rankid == INVALID_VALUE_RANKID) {
1097 0 : HCCL_ERROR("[GetCcuCqeErrRemoteLocalIdByRankId] RemoteLocalId is already set, rankId[%u]", rankid);
1098 0 : return;
1099 : }
1100 :
1101 0 : Hccl::HcclCommunicator* commV2 = static_cast<Hccl::HcclCommunicator *>(collComm->GetCommunicatorV2());
1102 0 : if (commV2 == nullptr) {
1103 0 : HCCL_ERROR("[GetCcuCqeErrRemoteLocalIdByRankId] commV2 is nullptr");
1104 0 : return;
1105 : }
1106 0 : void *rankGraph = nullptr;
1107 0 : HcclResult ret = commV2->GetRankGraphV2(rankGraph);
1108 0 : if (ret != HCCL_SUCCESS) {
1109 0 : HCCL_ERROR("[GetCcuCqeErrRemoteLocalIdByRankId]GetRankGraphV2 failed, rankId[%u], ret[%d]", rankid, ret);
1110 0 : return;
1111 : }
1112 0 : if (rankGraph == nullptr) {
1113 0 : HCCL_ERROR("[GetCcuCqeErrRemoteLocalIdByRankId] rankGraph is nullptr for rankid[%u]", rankid);
1114 0 : return;
1115 : }
1116 :
1117 0 : Hccl::RankGraph *rankGraphv2 = static_cast<Hccl::RankGraph *>(rankGraph);
1118 0 : if (rankGraphv2 == nullptr) {
1119 0 : HCCL_ERROR("[GetCcuCqeErrRemoteLocalIdByRankId] rankGraphv2 is nullptr for rankid[%u]", rankid);
1120 0 : return;
1121 : }
1122 0 : u32 LocalId = rankGraphv2->GetLocalId(rankid);
1123 0 : remoteLocalId = LocalId;
1124 0 : return;
1125 : }
1126 :
1127 0 : void CcuTaskException::GetCcuCqeErrNetInstanceByRankId(hccl::CollComm* collComm, uint32_t rankid, std::string &netInstanceId)
1128 : {
1129 0 : if (collComm == nullptr) {
1130 0 : HCCL_ERROR("[GetCcuCqeErrNetInstanceByRankId] collComm is nullptr");
1131 0 : return;
1132 : }
1133 0 : if (rankid == INVALID_VALUE_RANKID) {
1134 0 : HCCL_ERROR("[GetCcuCqeErrNetInstanceByRankId] RemoteLocalId is already set, rankId[%u]", rankid);
1135 0 : return;
1136 : }
1137 0 : Hccl::HcclCommunicator* commV2 = static_cast<Hccl::HcclCommunicator *>(collComm->GetCommunicatorV2());
1138 0 : if (commV2 == nullptr) {
1139 0 : HCCL_ERROR("[GetCcuCqeErrNetInstanceByRankId] commV2 is nullptr");
1140 0 : return;
1141 : }
1142 0 : void *rankGraph = nullptr;
1143 0 : HcclResult ret = commV2->GetRankGraphV2(rankGraph);
1144 0 : if (ret != HCCL_SUCCESS) {
1145 0 : HCCL_ERROR("[GetCcuCqeErrNetInstanceByRankId]GetRankGraphV2 failed, rankId[%u], ret[%d]", rankid, ret);
1146 0 : return;
1147 : }
1148 0 : Hccl::RankGraph *rankGraphv2 = static_cast<Hccl::RankGraph *>(rankGraph);
1149 0 : const Hccl::NetInstance *netInstance = rankGraphv2->GetNetInstanceByRankId(0, rankid);
1150 0 : if (netInstance == nullptr) {
1151 0 : HCCL_ERROR("[GetCcuCqeErrNetInstanceByRankId] netInstance is nullptr for rankid[%u]", rankid);
1152 0 : return;
1153 : }
1154 0 : std::string netInsId = netInstance->GetNetInstId();
1155 0 : netInstanceId = netInsId;
1156 0 : return;
1157 0 : }
1158 :
1159 0 : void CcuTaskException::GetCcuCqeErrorInfo(const CcuErrorInfo &ccuErrorInfo, const Hccl::TaskInfo &taskInfo, u32 locDeviceId, uint8_t missionStatus)
1160 : {
1161 0 : uint16_t channelId = GetChannleIdByCcuErrorInfo(ccuErrorInfo);
1162 0 : if (channelId == INVALID_U16) {
1163 0 : HCCL_ERROR("[%s]cannot get channelId from repType[%d]", __func__, ccuErrorInfo.repType);
1164 0 : return;
1165 : }
1166 0 : auto pair = GetAddrPairByChannelId(channelId, taskInfo, locDeviceId);
1167 0 : RankId remoteRankId = GetRankIdByChannelId(channelId, taskInfo, locDeviceId);
1168 0 : hccl::CollComm *collComm = static_cast<hccl::CollComm*>(taskInfo.dfxOpInfo_->comm_);
1169 0 : u32 remoteLocalId = INVALID_VALUE_RANKID;
1170 0 : GetCcuCqeErrRemoteLocalIdByRankId(collComm, remoteRankId, remoteLocalId);
1171 0 : std::string netInstanceId = "";
1172 0 : GetCcuCqeErrNetInstanceByRankId(collComm, remoteRankId, netInstanceId);
1173 0 : std::string srcEid = pair.first.Describe();
1174 0 : std::string dstEid = pair.second.Describe();
1175 0 : ClusterMoniterGetCcuCqeErrInfo(remoteLocalId, locDeviceId, missionStatus, srcEid, dstEid, netInstanceId);
1176 0 : return;
1177 0 : }
1178 :
1179 :
1180 5 : void CcuTaskException::PrintCcuErrorInfo(uint32_t deviceId, uint16_t status, const Hccl::TaskInfo& taskInfo)
1181 : {
1182 5 : const Hccl::ParaCcu& ccuTaskParam = taskInfo.taskParam_.taskPara.Ccu;
1183 5 : vector<CcuErrorInfo> errorInfos {};
1184 5 : HcclResult ret = GetCcuErrorMsg(deviceId, status, ccuTaskParam, errorInfos);
1185 5 : if (ret != HcclResult::HCCL_SUCCESS || errorInfos.empty()) {
1186 5 : HCCL_ERROR("Get CCU error info failed. deviceId[%u], dieId[%u], missionId[%u], executeId[%llu].",
1187 : deviceId, ccuTaskParam.dieId, ccuTaskParam.missionId, ccuTaskParam.executeId);
1188 5 : return;
1189 : }
1190 0 : PrintCcuErrorLog(errorInfos, taskInfo, deviceId);
1191 0 : const uint8_t missionStatus = (status >> 8) & 0xFF;
1192 0 : if (missionStatus >= 0x01 && missionStatus <= 0x05) { // 如果是UB错误(missionStatus为[0x01, 0x05])，打印Ub Dfx寄存器信息
1193 0 : PrintCcuUbRegisters(errorInfos, static_cast<s32>(deviceId), taskInfo);
1194 0 : if (isGetCqeErrInfo) {
1195 0 : isGetCqeErrInfo = false; // 只获取一次CQE错误信息，避免重复获取
1196 0 : for (const auto& errorInfo : errorInfos) {
1197 0 : if (errorInfo.repType == CcuRep::CcuRepType::READ || errorInfo.repType == CcuRep::CcuRepType::WRITE || errorInfo.repType == CcuRep::CcuRepType::BUF_READ || errorInfo.repType == CcuRep::CcuRepType::BUF_WRITE) {
1198 0 : GetCcuCqeErrorInfo(errorInfo, taskInfo, deviceId, missionStatus);//添加注释errorInfos[0]对应missionStatus异常
1199 : }
1200 : }
1201 : }
1202 : }
1203 5 : }
1204 :
1205 0 : void CcuTaskException::PrintCcuErrorLog(const std::vector<CcuErrorInfo>& errorInfos, const Hccl::TaskInfo& taskInfo,
1206 : u32 deviceId)
1207 : {
1208 0 : if (errorInfos.empty()) {
1209 0 : return;
1210 : }
1211 0 : HCCL_ERROR("[CcuTaskException]Task run failed, ccu runtime information is: %s", __func__);
1212 0 : for (const auto& errorInfo : errorInfos) {
1213 0 : HCCL_ERROR("[CcuTaskException][%s]", GetCcuErrorMsgByType(errorInfo, taskInfo, deviceId).c_str());
1214 : }
1215 : }
1216 :
1217 0 : HcclResult CcuTaskException::PrintCcuUbRegisters(const std::vector<CcuErrorInfo>& errorInfos, s32 devLogicId,
1218 : const Hccl::TaskInfo& taskInfo)
1219 : {
1220 0 : std::vector<CcuJetty *> ccuJettys;
1221 0 : for (const CcuErrorInfo& errorInfo : errorInfos) {
1222 0 : if (REP_WITH_CHANNEL.find(errorInfo.repType) == REP_WITH_CHANNEL.end()) {
1223 0 : HCCL_INFO("[%s]repType[%d] not found in REP_WITH_CHANNEL, skip", __func__, errorInfo.repType);
1224 0 : continue;
1225 : }
1226 :
1227 0 : std::pair<CcuChannelInfo, std::vector<CcuJetty *>> ctx;
1228 0 : (void)GetCcuJettys(errorInfo, ctx);
1229 0 : ccuJettys.insert(ccuJettys.end(), ctx.second.begin(), ctx.second.end());
1230 0 : }
1231 :
1232 0 : u32 jettyNum = ccuJettys.size();
1233 0 : CHK_PRT_RET(jettyNum == 0, HCCL_RUN_INFO("[%s]jettyNum[%u], skip", __func__, jettyNum), HCCL_SUCCESS);
1234 :
1235 0 : std::vector<JettyHandle> jettyHandles;
1236 0 : jettyHandles.reserve(jettyNum);
1237 0 : for (auto &ccuJetty : ccuJettys) {
1238 0 : jettyHandles.push_back(ccuJetty->GetJettyHandle());
1239 : }
1240 :
1241 0 : std::vector<JettyStatus> jettyStatusVec;
1242 0 : CHK_RET(RaBatchQueryJettyStatus(jettyHandles, jettyStatusVec, jettyNum));
1243 :
1244 0 : for (u32 i = 0; i < jettyNum; ++i) {
1245 0 : if (jettyStatusVec[i] == JettyStatus::ERROR) {
1246 0 : auto rdmaHandle = ccuJettys[i]->GetRdmaHandle();
1247 0 : HCCL_ERROR("[%s]jettyId[%u]", __func__, ccuJettys[i]->GetJettyId());
1248 0 : PrintUbRegisters(devLogicId, rdmaHandle);
1249 0 : break;
1250 : }
1251 : }
1252 0 : return HCCL_SUCCESS;
1253 0 : }
1254 :
1255 4 : uint16_t CcuTaskException::GetChannleIdByCcuErrorInfo(const CcuErrorInfo& errorInfo)
1256 : {
1257 4 : uint16_t channelId = INVALID_U16;
1258 4 : switch (errorInfo.repType) {
1259 1 : case CcuRep::CcuRepType::REM_POST_SEM:
1260 : case CcuRep::CcuRepType::REM_WAIT_SEM:
1261 : case CcuRep::CcuRepType::REM_POST_VAR:
1262 1 : channelId = errorInfo.msg.waitSignal.channelId[0];
1263 1 : break;
1264 1 : case CcuRep::CcuRepType::READ:
1265 : case CcuRep::CcuRepType::WRITE:
1266 1 : channelId = errorInfo.msg.transMem.channelId;
1267 1 : break;
1268 1 : case CcuRep::CcuRepType::BUF_READ:
1269 : case CcuRep::CcuRepType::BUF_WRITE:
1270 1 : channelId = errorInfo.msg.bufTransMem.channelId;
1271 1 : break;
1272 1 : default:
1273 1 : HCCL_ERROR("[%s]repType[%d] does not have jetty", __func__, errorInfo.repType);
1274 1 : break;
1275 : }
1276 4 : HCCL_INFO("[%s]repType[%d], channelId[%u]", __func__, errorInfo.repType, channelId);
1277 4 : return channelId;
1278 : }
1279 :
1280 0 : HcclResult CcuTaskException::GetCcuJettys(const CcuErrorInfo& errorInfo, std::pair<CcuChannelInfo, std::vector<CcuJetty *>> &ctx)
1281 : {
1282 0 : if (REP_WITH_CHANNEL.find(errorInfo.repType) == REP_WITH_CHANNEL.end()) {
1283 0 : HCCL_INFO("[%s]repType[%d] does not need to get jettys, skip", __func__, errorInfo.repType);
1284 0 : return HCCL_SUCCESS;
1285 : }
1286 0 : uint16_t channelId = GetChannleIdByCcuErrorInfo(errorInfo);
1287 :
1288 : // channelId -> channelHandle
1289 0 : u64 channelHandle = INVALID_U64;
1290 0 : CHK_RET(GetCcuChannelHandleById(channelId, channelHandle));
1291 :
1292 : // channelHandle -> CcuUrmaChannel
1293 0 : void *channelPtr{nullptr};
1294 0 : CHK_RET(static_cast<HcclResult>(HcommChannelGet(channelHandle, &channelPtr)));
1295 0 : CHK_PTR_NULL(channelPtr);
1296 0 : auto *channelImpl = dynamic_cast<CcuUrmaChannel *>(static_cast<Channel *>(channelPtr));
1297 :
1298 : // CcuUrmaChannel -> UrmaEndpoint
1299 0 : EndpointHandle locEndPointHandle = channelImpl->GetlocEndPointHandle();
1300 0 : void *endpoint{nullptr};
1301 0 : CHK_RET(static_cast<HcclResult>(HcommEndpointGet(locEndPointHandle, &endpoint)));
1302 0 : CHK_PTR_NULL(endpoint);
1303 0 : UrmaEndpoint *ccuEndpoint = dynamic_cast<UrmaEndpoint *>(static_cast<Endpoint *>(endpoint));
1304 :
1305 : // 打印localEid和remoteEid
1306 0 : CHK_PTR_NULL(ccuEndpoint);
1307 0 : HCCL_ERROR("[GetCcuJettys]localEid[%s], remoteEid[%s]",
1308 : hcomm::logger::CommAddrLogger::ToString(ccuEndpoint->GetEndpointDesc().commAddr).c_str(),
1309 : hcomm::logger::CommAddrLogger::ToString(channelImpl->GetChannelDesc().remoteEndpoint.commAddr).c_str());
1310 :
1311 : // UrmaEndpoint -> CcuChannelCtxPool
1312 0 : CcuChannelCtxPool *ccuChannelCtxPool = ccuEndpoint->GetCcuChannelCtxPool();
1313 0 : CHK_PTR_NULL(ccuChannelCtxPool);
1314 :
1315 : // CcuChannelCtxPool -> CcuJetty
1316 0 : auto channelIdKey = std::make_pair(errorInfo.dieId, channelId);
1317 0 : CHK_RET(ccuChannelCtxPool->GetCcuChannelCtxById(channelIdKey, ctx));
1318 0 : return HCCL_SUCCESS;
1319 : }
1320 :
1321 0 : HcclResult RaGetAuxInfo(const RdmaHandle rdmaHandle, AuxInfoIn auxInfoIn, AuxInfoOut &auxInfoOut)
1322 : {
1323 : HccpAuxInfoIn in;
1324 0 : in.type = static_cast<HccpAuxInfoInType>(static_cast<int>(auxInfoIn.auxInfoInType));
1325 0 : if (auxInfoIn.auxInfoInType == AuxInfoInType::AUX_INFO_IN_TYPE_CQE) {
1326 0 : in.cqe.status = auxInfoIn.cqe.status;
1327 0 : in.cqe.sR = auxInfoIn.cqe.sR;
1328 0 : } else if (auxInfoIn.auxInfoInType == AuxInfoInType::AUX_INFO_IN_TYPE_AE) {
1329 0 : in.ae.eventType = auxInfoIn.ae.eventType;
1330 : }
1331 :
1332 : HccpAuxInfoOut out;
1333 0 : auto ret = RaCtxGetAuxInfo(rdmaHandle, &in, &out);
1334 0 : if (ret != 0) {
1335 0 : HCCL_ERROR("RaGetAuxInfo failed.ret=[%d]", ret);
1336 0 : return HCCL_E_NETWORK;
1337 : }
1338 :
1339 0 : auxInfoOut.auxInfoNum = out.auxInfoNum;
1340 0 : for (uint32_t i = 0; i < out.auxInfoNum; i++) {
1341 0 : auxInfoOut.auxInfoTypes[i] = out.auxInfoType[i];
1342 0 : auxInfoOut.auxInfoValues[i] = out.auxInfoValue[i];
1343 : }
1344 0 : return HCCL_SUCCESS;
1345 : }
1346 0 : HcclResult CcuTaskException::PrintUbRegisters(s32 devLogicId, RdmaHandle rdmaHandle)
1347 : {
1348 0 : HCCL_INFO("[PrintUbRegister] start, devLogicId[%d], rdmaHandle[%p]", devLogicId, rdmaHandle);
1349 0 : AuxInfoIn in;
1350 0 : in.cqe.status = 0xffffffff; // 0xffffffff代表查询所有寄存器
1351 0 : in.auxInfoInType = AuxInfoInType::AUX_INFO_IN_TYPE_CQE;
1352 0 : in.cqe.sR = 0;
1353 0 : AuxInfoOut auxInfo;
1354 0 : auto ret = RaGetAuxInfo(rdmaHandle, in, auxInfo);
1355 0 : if (ret != HCCL_SUCCESS) {
1356 0 : HCCL_ERROR("[PrintUbRegister] RaGetAuxInfo failed, devLogicId[%d], rdmaHandle[%p], ret[%d]", devLogicId,
1357 : rdmaHandle, ret);
1358 0 : return ret;
1359 : }
1360 :
1361 0 : uint16_t isAuxInfoExisted{false};
1362 0 : for (u32 i = 0; i < auxInfo.auxInfoNum; i++) {
1363 0 : if (auxInfo.auxInfoValues[i]) { // 非零进行打印
1364 0 : isAuxInfoExisted = true;
1365 0 : HCCL_ERROR("devLogicId[%d], cqe_aux_info_type[%u], cqe_aux_info_value[0x%x]", devLogicId,
1366 : auxInfo.auxInfoTypes[i], auxInfo.auxInfoValues[i]);
1367 : } else {
1368 0 : HCCL_INFO("devLogicId[%d], cqe_aux_info_type[%u], cqe_aux_info_value[0x%x]",
1369 : devLogicId, auxInfo.auxInfoTypes[i], auxInfo.auxInfoValues[i]);
1370 : }
1371 : }
1372 0 : if (!isAuxInfoExisted) {
1373 0 : HCCL_ERROR("devLogicId[%d], all aux_info values are zero.", devLogicId);
1374 : }
1375 0 : return HCCL_SUCCESS;
1376 : }
1377 :
1378 14 : string CcuTaskException::GetCcuLenErrorMsg(const uint64_t len)
1379 : {
1380 14 : if ((0 < len) && (len <= CCU_MSG_256MB_LEN)) {
1381 14 : return "";
1382 : }
1383 7 : return Hccl::StringFormat("ccu transMem Len[%llu]B > 256MB or is zero, not support!", len);
1384 : }
1385 :
1386 1 : string CcuTaskException::GetCcuErrorMsgLoop(const CcuErrorInfo &ccuErrorInfo, const Hccl::TaskInfo &taskInfo, u32 deviceId)
1387 : {
1388 : (void)taskInfo;
1389 : (void)deviceId;
1390 : return Hccl::StringFormat("InstrId[%u]: Loop startInstrId[%u], endInstrId[%u], executorId[%u], "
1391 : "totalIter[%u], curIter[%u], addressStride[0x%llx]",
1392 1 : ccuErrorInfo.instrId, ccuErrorInfo.msg.loop.startInstrId, ccuErrorInfo.msg.loop.endInstrId,
1393 1 : ccuErrorInfo.msg.loop.loopEngineId, ccuErrorInfo.msg.loop.loopCnt,
1394 1 : ccuErrorInfo.msg.loop.loopCurrentCnt, ccuErrorInfo.msg.loop.addrStride);
1395 : }
1396 :
1397 1 : string CcuTaskException::GetCcuErrorMsgLoopGroup(const CcuErrorInfo &ccuErrorInfo, const Hccl::TaskInfo &taskInfo, u32 deviceId)
1398 : {
1399 : (void)taskInfo;
1400 : (void)deviceId;
1401 : return Hccl::StringFormat("InstrId[%u]: LoopGroup startLoopInsId[%u], loopInsCnt[%u], "
1402 : "expandOffset[%u], expandCnt[%u]",
1403 1 : ccuErrorInfo.instrId, ccuErrorInfo.msg.loopGroup.startLoopInsId,
1404 1 : ccuErrorInfo.msg.loopGroup.loopInsCnt, ccuErrorInfo.msg.loopGroup.expandOffset,
1405 1 : ccuErrorInfo.msg.loopGroup.expandCnt);
1406 : }
1407 :
1408 1 : string CcuTaskException::GetCcuErrorMsgLocPostSem(const CcuErrorInfo &ccuErrorInfo, const Hccl::TaskInfo &taskInfo, u32 deviceId)
1409 : {
1410 : (void)taskInfo;
1411 : (void)deviceId;
1412 1 : return Hccl::StringFormat("InstrId[%u]: Set sem[%u], semValue[0x%04x], mask[0x%04x]", ccuErrorInfo.instrId,
1413 1 : ccuErrorInfo.msg.waitSignal.signalId, ccuErrorInfo.msg.waitSignal.signalValue,
1414 1 : ccuErrorInfo.msg.waitSignal.signalMask);
1415 : }
1416 :
1417 1 : string CcuTaskException::GetCcuErrorMsgLocWaitEvent(const CcuErrorInfo &ccuErrorInfo, const Hccl::TaskInfo &taskInfo, u32 deviceId)
1418 : {
1419 : (void)taskInfo;
1420 : (void)deviceId;
1421 1 : return Hccl::StringFormat("InstrId[%u]: LocWaitEvent[%u], semValue[0x%04x], mask[0x%04x]", ccuErrorInfo.instrId,
1422 1 : ccuErrorInfo.msg.waitSignal.signalId, ccuErrorInfo.msg.waitSignal.signalValue,
1423 1 : ccuErrorInfo.msg.waitSignal.signalMask);
1424 : }
1425 :
1426 1 : string CcuTaskException::GetCcuErrorMsgLocWaitNotify(const CcuErrorInfo &ccuErrorInfo, const Hccl::TaskInfo &taskInfo, u32 deviceId)
1427 : {
1428 : (void)taskInfo;
1429 : (void)deviceId;
1430 1 : return Hccl::StringFormat("InstrId[%u]: LocWaitNotify[%u], semValue[0x%04x], mask[0x%04x]", ccuErrorInfo.instrId,
1431 1 : ccuErrorInfo.msg.waitSignal.signalId, ccuErrorInfo.msg.waitSignal.signalValue,
1432 1 : ccuErrorInfo.msg.waitSignal.signalMask);
1433 : }
1434 :
1435 1 : string CcuTaskException::GetCcuErrorMsgRemPostSem(const CcuErrorInfo &ccuErrorInfo, const Hccl::TaskInfo &taskInfo, u32 deviceId)
1436 : {
1437 1 : return Hccl::StringFormat("InstrId[%u]: Post, Use sem[%u], mask[0x%04x], rankId[%d]", ccuErrorInfo.instrId,
1438 1 : ccuErrorInfo.msg.waitSignal.signalId, ccuErrorInfo.msg.waitSignal.signalMask,
1439 1 : GetRankIdByChannelId(ccuErrorInfo.msg.waitSignal.channelId[0], taskInfo, deviceId));
1440 : }
1441 :
1442 1 : string CcuTaskException::GetCcuErrorMsgRemWaitSem(const CcuErrorInfo &ccuErrorInfo, const Hccl::TaskInfo &taskInfo, u32 deviceId)
1443 : {
1444 : return Hccl::StringFormat("InstrId[%u]: Wait, Use sem[%u], semValue[0x%04x], mask[0x%04x], rankId[%d]",
1445 1 : ccuErrorInfo.instrId, ccuErrorInfo.msg.waitSignal.signalId,
1446 1 : ccuErrorInfo.msg.waitSignal.signalValue, ccuErrorInfo.msg.waitSignal.signalMask,
1447 1 : GetRankIdByChannelId(ccuErrorInfo.msg.waitSignal.channelId[0], taskInfo, deviceId));
1448 : }
1449 :
1450 1 : string CcuTaskException::GetCcuErrorMsgRemPostVar(const CcuErrorInfo &ccuErrorInfo, const Hccl::TaskInfo &taskInfo, u32 deviceId)
1451 : {
1452 : return Hccl::StringFormat("InstrId[%u]: Post Variable[0x%016llx] To Param[%u], Use sem[%u], mask[0x%04x], rankId[%d]",
1453 1 : ccuErrorInfo.instrId, ccuErrorInfo.msg.waitSignal.paramValue,
1454 1 : ccuErrorInfo.msg.waitSignal.paramId, ccuErrorInfo.msg.waitSignal.signalId,
1455 1 : ccuErrorInfo.msg.waitSignal.signalMask,
1456 1 : GetRankIdByChannelId(ccuErrorInfo.msg.waitSignal.channelId[0], taskInfo, deviceId));
1457 : }
1458 :
1459 1 : string CcuTaskException::GetCcuErrorMsgPostSharedSem(const CcuErrorInfo &ccuErrorInfo, const Hccl::TaskInfo &taskInfo, u32 deviceId)
1460 : {
1461 : (void)taskInfo;
1462 : (void)deviceId;
1463 1 : return Hccl::StringFormat("InstrId[%u]: Post, Use sem[%u], mask[0x%04x]", ccuErrorInfo.instrId,
1464 1 : ccuErrorInfo.msg.waitSignal.signalId, ccuErrorInfo.msg.waitSignal.signalMask);
1465 : }
1466 :
1467 1 : string CcuTaskException::GetCcuErrorMsgRead(const CcuErrorInfo &ccuErrorInfo, const Hccl::TaskInfo &taskInfo, u32 deviceId)
1468 : {
1469 1 : auto pair = GetAddrPairByChannelId(ccuErrorInfo.msg.transMem.channelId, taskInfo, deviceId);
1470 1 : string printMsg = GetCcuLenErrorMsg(ccuErrorInfo.msg.transMem.len);
1471 : return Hccl::StringFormat(
1472 : "InstrId[%u]: Read Memory[0x%016llx] To Memory[0x%016llx], Len[%llu], "
1473 : "Set sem[%u] with mask[0x%04x], remoteRankId[%d], srcEID[%s], dstEID[%s] %s",
1474 1 : ccuErrorInfo.instrId, ccuErrorInfo.msg.transMem.rmtAddr, ccuErrorInfo.msg.transMem.locAddr,
1475 1 : ccuErrorInfo.msg.transMem.len, ccuErrorInfo.msg.transMem.signalId, ccuErrorInfo.msg.transMem.signalMask,
1476 1 : GetRankIdByChannelId(ccuErrorInfo.msg.transMem.channelId, taskInfo, deviceId),
1477 2 : pair.first.Describe().c_str(),
1478 4 : pair.second.Describe().c_str(), printMsg.c_str());
1479 1 : }
1480 :
1481 1 : string CcuTaskException::GetCcuErrorMsgWrite(const CcuErrorInfo &ccuErrorInfo, const Hccl::TaskInfo &taskInfo, u32 deviceId)
1482 : {
1483 1 : auto pair = GetAddrPairByChannelId(ccuErrorInfo.msg.transMem.channelId, taskInfo, deviceId);
1484 1 : string printMsg = GetCcuLenErrorMsg(ccuErrorInfo.msg.transMem.len);
1485 : return Hccl::StringFormat(
1486 : "InstrId[%u]: Write Memory[0x%016llx] to Memory[0x%016llx], Len[%llu], "
1487 : "Set sem[%u] with mask[0x%04x], remoteRankId[%d], srcEID[%s], dstEID[%s] %s",
1488 1 : ccuErrorInfo.instrId, ccuErrorInfo.msg.transMem.locAddr, ccuErrorInfo.msg.transMem.rmtAddr,
1489 1 : ccuErrorInfo.msg.transMem.len, ccuErrorInfo.msg.transMem.signalId, ccuErrorInfo.msg.transMem.signalMask,
1490 1 : GetRankIdByChannelId(ccuErrorInfo.msg.transMem.channelId, taskInfo, deviceId),
1491 2 : pair.first.Describe().c_str(),
1492 4 : pair.second.Describe().c_str(), printMsg.c_str());
1493 1 : }
1494 :
1495 1 : string CcuTaskException::GetCcuErrorMsgLocalCpy(const CcuErrorInfo &ccuErrorInfo, const Hccl::TaskInfo &taskInfo, u32 deviceId)
1496 : {
1497 : (void)taskInfo;
1498 : (void)deviceId;
1499 1 : string printMsg = GetCcuLenErrorMsg(ccuErrorInfo.msg.transMem.len);
1500 : return Hccl::StringFormat("InstrId[%u]: Read Memory[0x%016llx] to Memory[0x%016llx], Len[%llu], "
1501 : "Set sem[%u] with mask[0x%04x] %s",
1502 1 : ccuErrorInfo.instrId, ccuErrorInfo.msg.transMem.locAddr, ccuErrorInfo.msg.transMem.rmtAddr,
1503 1 : ccuErrorInfo.msg.transMem.len, ccuErrorInfo.msg.transMem.signalId,
1504 2 : ccuErrorInfo.msg.transMem.signalMask, printMsg.c_str());
1505 1 : }
1506 :
1507 1 : string CcuTaskException::GetCcuErrorMsgLocalReduce(const CcuErrorInfo &ccuErrorInfo, const Hccl::TaskInfo &taskInfo, u32 deviceId)
1508 : {
1509 : (void)taskInfo;
1510 : (void)deviceId;
1511 1 : string printMsg = GetCcuLenErrorMsg(ccuErrorInfo.msg.transMem.len);
1512 : return Hccl::StringFormat("InstrId[%u]: Read Memory[0x%016llx] to Memory[0x%016llx], Len[%llu], "
1513 : "Set sem[%u] with mask[0x%04x], dataType[%u], opType[%u] %s",
1514 1 : ccuErrorInfo.instrId, ccuErrorInfo.msg.transMem.locAddr, ccuErrorInfo.msg.transMem.rmtAddr,
1515 1 : ccuErrorInfo.msg.transMem.len, ccuErrorInfo.msg.transMem.signalId,
1516 1 : ccuErrorInfo.msg.transMem.signalMask, ccuErrorInfo.msg.transMem.dataType,
1517 2 : ccuErrorInfo.msg.transMem.opType, printMsg.c_str());
1518 1 : }
1519 :
1520 1 : string CcuTaskException::GetCcuErrorMsgBufRead(const CcuErrorInfo &ccuErrorInfo, const Hccl::TaskInfo &taskInfo, u32 deviceId)
1521 : {
1522 1 : auto pair = GetAddrPairByChannelId(ccuErrorInfo.msg.bufTransMem.channelId, taskInfo, deviceId);
1523 1 : string printMsg = GetCcuLenErrorMsg(ccuErrorInfo.msg.bufTransMem.len);
1524 : return Hccl::StringFormat(
1525 : "InstrId[%u]: Read Rmt Mem[0x%016llx] To CcuBuffer[%u], Len[%llu], "
1526 : "sem[%u], mask[0x%04x], remoteRankId[%d], srcEID[%s], dstEID[%s] %s",
1527 1 : ccuErrorInfo.instrId, ccuErrorInfo.msg.bufTransMem.addr, ccuErrorInfo.msg.bufTransMem.bufId,
1528 1 : ccuErrorInfo.msg.bufTransMem.len, ccuErrorInfo.msg.bufTransMem.signalId, ccuErrorInfo.msg.bufTransMem.signalMask,
1529 1 : GetRankIdByChannelId(ccuErrorInfo.msg.bufTransMem.channelId, taskInfo, deviceId),
1530 2 : pair.first.Describe().c_str(),
1531 4 : pair.second.Describe().c_str(), printMsg.c_str());
1532 1 : }
1533 :
1534 1 : string CcuTaskException::GetCcuErrorMsgBufWrite(const CcuErrorInfo &ccuErrorInfo, const Hccl::TaskInfo &taskInfo, u32 deviceId)
1535 : {
1536 1 : auto pair = GetAddrPairByChannelId(ccuErrorInfo.msg.bufTransMem.channelId, taskInfo, deviceId);
1537 1 : string printMsg = GetCcuLenErrorMsg(ccuErrorInfo.msg.bufTransMem.len);
1538 : return Hccl::StringFormat(
1539 : "InstrId[%u]: Write CcuBuffer[%u] To Rmt Mem[0x%016llx], Len[%llu], "
1540 : "sem[%u], mask[0x%04x], remoteRankId[%d], srcEID[%s], dstEID[%s] %s",
1541 1 : ccuErrorInfo.instrId, ccuErrorInfo.msg.bufTransMem.bufId, ccuErrorInfo.msg.bufTransMem.addr,
1542 1 : ccuErrorInfo.msg.bufTransMem.len, ccuErrorInfo.msg.bufTransMem.signalId, ccuErrorInfo.msg.bufTransMem.signalMask,
1543 1 : GetRankIdByChannelId(ccuErrorInfo.msg.bufTransMem.channelId, taskInfo, deviceId),
1544 2 : pair.first.Describe().c_str(),
1545 4 : pair.second.Describe().c_str(), printMsg.c_str());
1546 1 : }
1547 :
1548 1 : string CcuTaskException::GetCcuErrorMsgBufLocRead(const CcuErrorInfo &ccuErrorInfo, const Hccl::TaskInfo &taskInfo, u32 deviceId)
1549 : {
1550 : (void)taskInfo;
1551 : (void)deviceId;
1552 1 : string printMsg = GetCcuLenErrorMsg(ccuErrorInfo.msg.bufTransMem.len);
1553 : return Hccl::StringFormat("InstrId[%u]: Read Loc Mem[0x%016llx] To CcuBuffer[%u], Len[%llu], sem[%u], mask[0x%04x] %s",
1554 1 : ccuErrorInfo.instrId, ccuErrorInfo.msg.bufTransMem.addr, ccuErrorInfo.msg.bufTransMem.bufId,
1555 1 : ccuErrorInfo.msg.bufTransMem.len, ccuErrorInfo.msg.bufTransMem.signalId,
1556 2 : ccuErrorInfo.msg.bufTransMem.signalMask, printMsg.c_str());
1557 1 : }
1558 :
1559 1 : string CcuTaskException::GetCcuErrorMsgBufLocWrite(const CcuErrorInfo &ccuErrorInfo, const Hccl::TaskInfo &taskInfo, u32 deviceId)
1560 : {
1561 : (void)taskInfo;
1562 : (void)deviceId;
1563 1 : string printMsg = GetCcuLenErrorMsg(ccuErrorInfo.msg.bufTransMem.len);
1564 : return Hccl::StringFormat("InstrId[%u]: Write CcuBuffer[%u] To Loc Mem[0x%016llx], Len[%llu], sem[%u], mask[0x%04x] %s",
1565 1 : ccuErrorInfo.instrId, ccuErrorInfo.msg.bufTransMem.bufId, ccuErrorInfo.msg.bufTransMem.addr,
1566 1 : ccuErrorInfo.msg.bufTransMem.len, ccuErrorInfo.msg.bufTransMem.signalId,
1567 2 : ccuErrorInfo.msg.bufTransMem.signalMask, printMsg.c_str());
1568 1 : }
1569 :
1570 1 : string CcuTaskException::GetCcuErrorMsgBufReduce(const CcuErrorInfo &ccuErrorInfo, const Hccl::TaskInfo &taskInfo, u32 deviceId)
1571 : {
1572 : (void)taskInfo;
1573 : (void)deviceId;
1574 1 : stringstream buffIds;
1575 9 : for (uint32_t i = 0; i < BUF_REDUCE_ID_SIZE; ++i) {
1576 8 : const auto buffId = ccuErrorInfo.msg.bufReduce.bufIds[i];
1577 8 : if (buffId == UINT16_MAX) {
1578 0 : break;
1579 : }
1580 8 : if (i != 0) {
1581 7 : buffIds << ", ";
1582 : }
1583 8 : buffIds << to_string(buffId);
1584 : }
1585 :
1586 : return Hccl::StringFormat("InstrId[%u]: Buffer Reduce count[%u], dataType[%u], outputDataType[%u], opType[%u], "
1587 : "sem[%u], mask[0x%04x], CcuBuffers[%s]",
1588 1 : ccuErrorInfo.instrId, ccuErrorInfo.msg.bufReduce.count, ccuErrorInfo.msg.bufReduce.dataType,
1589 1 : ccuErrorInfo.msg.bufReduce.outputDataType, ccuErrorInfo.msg.bufReduce.opType,
1590 1 : ccuErrorInfo.msg.bufReduce.signalId, ccuErrorInfo.msg.bufReduce.signalMask,
1591 2 : buffIds.str().c_str());
1592 1 : }
1593 :
1594 2 : string CcuTaskException::GetCcuErrorMsgDefault(const CcuErrorInfo &ccuErrorInfo)
1595 : {
1596 : return Hccl::StringFormat("InstrId[%u]: CcuErrorType[%s]",
1597 2 : ccuErrorInfo.instrId, ccuErrorInfo.type.Describe().c_str());
1598 : }
1599 :
1600 2 : string CcuTaskException::GetCcuErrorMsgMission(const CcuErrorInfo &ccuErrorInfo)
1601 : {
1602 : return Hccl::StringFormat("InstrId[%u]: dieId[%u], missionId[%u], missionError[%s]",
1603 2 : ccuErrorInfo.instrId, ccuErrorInfo.dieId, ccuErrorInfo.missionId,
1604 2 : ccuErrorInfo.msg.mission.missionError);
1605 : }
1606 :
1607 0 : HcclResult CcuTaskException::GetCcuChannelHandleById(u16 channelId, u64& channelHandle)
1608 : {
1609 0 : std::lock_guard<std::mutex> lock(g_channelMapMutex);
1610 0 : auto it = g_channelIdToHandle.find(channelId);
1611 0 : if (it == g_channelIdToHandle.end()) {
1612 0 : HCCL_ERROR("[%s]channelId[%u] not found", __func__, channelId);
1613 0 : for (auto info : g_channelIdToHandle) {
1614 0 : HCCL_ERROR("[%s]g_channelIdToHandle[%u]=[0x%llx]", __func__, info.first, info.second);
1615 : }
1616 0 : return HCCL_E_NOT_FOUND;
1617 : }
1618 0 : channelHandle = it->second;
1619 0 : return HCCL_SUCCESS;
1620 0 : }
1621 :
1622 8 : RankId CcuTaskException::GetRankIdByChannelId(uint16_t channelId, const Hccl::TaskInfo &taskInfo, u32 deviceId)
1623 : {
1624 8 : if (taskInfo.taskParam_.taskType != Hccl::TaskParamType::TASK_CCU) {
1625 0 : HCCL_ERROR("[%s]taskType[%s] is not CCU.", __func__, taskInfo.taskParam_.taskType.Describe().c_str());
1626 0 : return INVALID_UINT;
1627 : }
1628 8 : if (taskInfo.dfxOpInfo_ == nullptr || taskInfo.dfxOpInfo_->comm_ == nullptr) {
1629 8 : HCCL_ERROR("[%s]dfxOpInfo[%p] or comm is nullptr.", __func__, taskInfo.dfxOpInfo_);
1630 8 : return INVALID_UINT;
1631 : }
1632 :
1633 0 : u64 channelHandle = INVALID_U64;
1634 0 : if (GetCcuChannelHandleById(channelId, channelHandle) != HCCL_SUCCESS) {
1635 0 : HCCL_ERROR("[%s]GetCcuChannelHandleById fail, deviceId[%u], channelId[%u], channelHandle[0x%llx]",
1636 : __func__, deviceId, channelId, channelHandle);
1637 0 : return INVALID_UINT;
1638 : }
1639 :
1640 0 : hccl::CollComm *collComm = static_cast<hccl::CollComm*>(taskInfo.dfxOpInfo_->comm_);
1641 0 : u32 remoteRank = INVALID_UINT;
1642 0 : if (hccl::HcclCommDfx::GetChannelRemoteRankId(collComm->GetCommId(), channelHandle, remoteRank) != HCCL_SUCCESS) {
1643 0 : HCCL_ERROR("[%s]GetChannelRemoteRankId fail, channelHandle[0x%llx], commId[%s]",
1644 : __func__, channelHandle, collComm->GetCommId().c_str());
1645 0 : return INVALID_UINT;
1646 : }
1647 :
1648 0 : HCCL_INFO("[%s]channelId[%u], deviceId[%u], channelHandle[0x%llx], commId[%s], remoteRank[%u]",
1649 : __func__, channelId, deviceId, channelHandle, collComm->GetCommId().c_str(), remoteRank);
1650 0 : return remoteRank;
1651 : }
1652 :
1653 5 : std::pair<Hccl::IpAddress, Hccl::IpAddress> CcuTaskException::GetAddrPairByChannelId(uint16_t channelId,
1654 : const Hccl::TaskInfo &taskInfo, u32 deviceId)
1655 : {
1656 5 : std::pair<Hccl::IpAddress, Hccl::IpAddress> dummy = {Hccl::IpAddress(), Hccl::IpAddress()};
1657 5 : if (taskInfo.taskParam_.taskType != Hccl::TaskParamType::TASK_CCU) {
1658 0 : HCCL_ERROR("[TaskException][%s]Get AddrPair failed, task type error[%s]", __func__,
1659 : taskInfo.taskParam_.Describe().c_str());
1660 0 : return dummy;
1661 : }
1662 5 : if (taskInfo.dfxOpInfo_ == nullptr || taskInfo.dfxOpInfo_->comm_ == nullptr) {
1663 5 : HCCL_ERROR("[TaskException][%s]Get AddrPair failed, communicator is nullptr.", __func__);
1664 5 : return dummy;
1665 : }
1666 :
1667 0 : u64 channelHandle = INVALID_U64;
1668 0 : if (GetCcuChannelHandleById(channelId, channelHandle) != HCCL_SUCCESS) {
1669 0 : HCCL_ERROR("[%s]GetCcuChannelHandleById fail, deviceId[%u], channelId[%u], channelHandle[0x%llx]",
1670 : __func__, deviceId, channelId, channelHandle);
1671 0 : return dummy;
1672 : }
1673 :
1674 0 : void *channelPtr{nullptr};
1675 0 : HcclResult ret = static_cast<HcclResult>(HcommChannelGet(channelHandle, &channelPtr));
1676 0 : if (ret != HCCL_SUCCESS || channelPtr == nullptr) {
1677 0 : HCCL_ERROR("[%s]HcommChannelGet failed, ret[%d], channelHandle[0x%llx], channelPtr[%p]",
1678 : __func__, ret, channelHandle, channelPtr);
1679 0 : return dummy;
1680 : }
1681 0 : auto *channelImpl = dynamic_cast<CcuUrmaChannel *>(static_cast<Channel *>(channelPtr));
1682 :
1683 : // 获取locAddr
1684 0 : EndpointHandle locEndPointHandle = channelImpl->GetlocEndPointHandle();
1685 0 : void *endpoint{nullptr};
1686 0 : ret = static_cast<HcclResult>(HcommEndpointGet(locEndPointHandle, &endpoint));
1687 0 : if (ret != HCCL_SUCCESS || endpoint == nullptr) {
1688 0 : HCCL_ERROR("[%s]HcommEndpointGet failed, ret[%d], locEndPointHandle[%p], endpoint[%p], channelId[%u]",
1689 : __func__, ret, locEndPointHandle, endpoint, channelId);
1690 0 : return dummy;
1691 : }
1692 0 : UrmaEndpoint *ccuEndpoint = dynamic_cast<UrmaEndpoint *>(static_cast<Endpoint *>(endpoint));
1693 0 : const EndpointDesc &locEndpointDesc = ccuEndpoint->GetEndpointDesc();
1694 0 : HcclResult locRet = CommAddrToIpAddress(locEndpointDesc.commAddr, dummy.first);
1695 :
1696 : // 获取remoteAddr
1697 0 : HcommChannelDesc remoteChannelDesc = channelImpl->GetChannelDesc();
1698 0 : HcclResult remRet = CommAddrToIpAddress(remoteChannelDesc.remoteEndpoint.commAddr, dummy.second);
1699 0 : HCCL_INFO("[%s]channelId[%u], channelHandle[0x%llx], locRet[%d], locIpAddr[%s], remRet[%d], remIpAddr[%s]",
1700 : __func__, channelId, channelHandle, locRet, dummy.first.Describe().c_str(),
1701 : remRet, dummy.second.Describe().c_str());
1702 0 : return dummy;
1703 : }
1704 :
1705 2 : string CcuTaskException::GetCcuErrorMsgByType(const CcuErrorInfo &ccuErrorInfo, const Hccl::TaskInfo &taskInfo, u32 deviceId)
1706 : {
1707 2 : if (ccuErrorInfo.type == CcuErrorType::MISSION) {
1708 1 : return GetCcuErrorMsgMission(ccuErrorInfo);
1709 : }
1710 :
1711 : using GetCcuErrorMsgFunc = string (*)(const CcuErrorInfo &ccuErrorInfo, const Hccl::TaskInfo &taskInfo, u32 deviceId);
1712 : static const map<CcuRep::CcuRepType, GetCcuErrorMsgFunc> HANDLER_MAP {
1713 : {CcuRep::CcuRepType::LOOP, &CcuTaskException::GetCcuErrorMsgLoop},
1714 : {CcuRep::CcuRepType::LOOPGROUP, &CcuTaskException::GetCcuErrorMsgLoopGroup},
1715 : {CcuRep::CcuRepType::LOC_RECORD_EVENT, &CcuTaskException::GetCcuErrorMsgLocPostSem},
1716 : {CcuRep::CcuRepType::LOC_WAIT_EVENT, &CcuTaskException::GetCcuErrorMsgLocWaitEvent},
1717 : {CcuRep::CcuRepType::LOC_WAIT_NOTIFY, &CcuTaskException::GetCcuErrorMsgLocWaitNotify},
1718 : {CcuRep::CcuRepType::REM_POST_SEM, &CcuTaskException::GetCcuErrorMsgRemPostSem},
1719 : {CcuRep::CcuRepType::REM_WAIT_SEM, &CcuTaskException::GetCcuErrorMsgRemWaitSem},
1720 : {CcuRep::CcuRepType::REM_POST_VAR, &CcuTaskException::GetCcuErrorMsgRemPostVar},
1721 : {CcuRep::CcuRepType::RECORD_SHARED_NOTIFY, &CcuTaskException::GetCcuErrorMsgPostSharedSem},
1722 : {CcuRep::CcuRepType::READ, &CcuTaskException::GetCcuErrorMsgRead},
1723 : {CcuRep::CcuRepType::WRITE, &CcuTaskException::GetCcuErrorMsgWrite},
1724 : {CcuRep::CcuRepType::LOCAL_CPY, &CcuTaskException::GetCcuErrorMsgLocalCpy},
1725 : {CcuRep::CcuRepType::LOCAL_REDUCE, &CcuTaskException::GetCcuErrorMsgLocalReduce},
1726 : {CcuRep::CcuRepType::BUF_READ, &CcuTaskException::GetCcuErrorMsgBufRead},
1727 : {CcuRep::CcuRepType::BUF_WRITE, &CcuTaskException::GetCcuErrorMsgBufWrite},
1728 : {CcuRep::CcuRepType::BUF_LOC_READ, &CcuTaskException::GetCcuErrorMsgBufLocRead},
1729 : {CcuRep::CcuRepType::BUF_LOC_WRITE, &CcuTaskException::GetCcuErrorMsgBufLocWrite},
1730 : {CcuRep::CcuRepType::BUF_REDUCE, &CcuTaskException::GetCcuErrorMsgBufReduce}
1731 3 : };
1732 :
1733 1 : const auto funcIt = HANDLER_MAP.find(ccuErrorInfo.repType);
1734 1 : if (funcIt == HANDLER_MAP.end()) {
1735 1 : return GetCcuErrorMsgDefault(ccuErrorInfo);
1736 : } else {
1737 0 : return funcIt->second(ccuErrorInfo, taskInfo, deviceId);
1738 : }
1739 : }
1740 : }
|
