Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "aicpu_executor_tracer.h"
12 :
13 : #include "hccl_msg.h"
14 : #include "common/aicpu_hccl_common.h"
15 : #include "mc2_trace_utils.h"
16 : #include "utils/aicpu_hdc_utils.h"
17 : #include "framework/aicpu_kfc_process.h"
18 :
19 : using HcclApi::HcclMsgArea;
20 : using HcclApi::HCCL_MSG_CNT;
21 : namespace dfx_tracer {
22 :
23 : // recv host stop command
24 8101 : void AicpuExecutorTracer::HandleBackGround(AicpuComContext *const ctx)
25 : {
26 : BackgroundCommand bgCmd;
27 8101 : if (ctx->commOpenStatus) {
28 8101 : (void)AicpuHdcUtils::GetBackGroundCommand(ctx->kfcControlTransferH2D, bgCmd);
29 8101 : if (bgCmd == BackgroundCommand::kStop) {
30 1 : HCCL_RUN_INFO("ctx stop back ground");
31 1 : StopKfcThread(ctx, {});
32 1 : KfcExecStatus bgresponse;
33 1 : bgresponse.execStatus.backgroundStatus = BackgroundStatus::kStop;
34 1 : ctx->commOpenStatus = false;
35 1 : AicpuUpdatComContextMumber(offsetof(AicpuComContext, alreadyInit), false);
36 1 : (void)MC2TraceUtils::DestoryHandles();
37 1 : (void)AicpuHdcUtils::ResponseBackGroundStatus(ctx->kfcStatusTransferD2H, bgresponse);
38 1 : bgCmd = BackgroundCommand::kNone;
39 : }
40 : }
41 8101 : }
42 :
43 : // handle StopLaunch Command
44 8100 : void AicpuExecutorTracer::StopLaunchCommandHandle(AicpuComContext *const ctx)
45 : {
46 8100 : if (ctx->commOpenStatus) {
47 8100 : if (!ctx->endStopLaunch) {
48 8100 : KfcCommand cmd = KfcCommand::kNone;
49 8100 : (void)AicpuHdcUtils::GetOpExecCtrlCmd(ctx->kfcControlTransferH2D, cmd);
50 8100 : if (cmd == KfcCommand::NsStopLaunch) {
51 0 : if (!ctx->isOpLaunch) {
52 0 : (void)AicpuHdcUtils::SetOpExecStatus(
53 0 : ctx->kfcStatusTransferD2H, KfcStatus::kStoplaunch, KfcError::kNone, 0);
54 0 : AicpuUpdatComContextMumber(offsetof(AicpuComContext, endStopLaunch), true);
55 0 : HCCL_INFO("[NsRecovery][backGround]send in mc2 environment");
56 : }
57 : }
58 : }
59 : }
60 8100 : }
61 :
62 : // handle StopExec and Clean Command
63 8106 : void AicpuExecutorTracer::KfcCommandHandle(AicpuComContext *const ctx)
64 : {
65 8106 : if (ctx->commOpenStatus) {
66 : using CommandCall = std::function<void(AicpuComContext *const ctx)>;
67 : static std::map<KfcCommand, CommandCall> commandHandles = {
68 0 : {KfcCommand::NsStopExec, KfcCommandHandles::StopFunc},
69 8114 : {KfcCommand::NsClear, KfcCommandHandles::ClearFunc}};
70 :
71 8106 : KfcCommand cmd = KfcCommand::kNone;
72 8106 : (void) AicpuHdcUtils::GetOpExecCtrlCmd(ctx->kfcControlTransferH2D, cmd);
73 8106 : auto iter = commandHandles.find(cmd);
74 8106 : if (iter == commandHandles.cend()) {
75 8100 : return;
76 : }
77 6 : HCCL_INFO("Start to run command %ld", cmd);
78 6 : iter->second(ctx);
79 : }
80 2 : }
81 :
82 8100 : void AicpuExecutorTracer::HandleCqeStatus(AicpuComContext *const ctx)
83 : {
84 8100 : if (ctx == nullptr || ctx->alreadyInit == false) {
85 0 : return;
86 : }
87 :
88 8100 : if (ctx->dfxExtendInfo.cqeStatus != dfx::CqeStatus::kDefault) {
89 99 : return;
90 : }
91 :
92 8001 : if (ctx->dfxExtendInfo.kfcStatus != DfxKfcStatus::kOneStart) {
93 8000 : return;
94 : }
95 :
96 : // 遍历每个卡的cqe状态, 如果一个卡有异常，所有的卡的ctx里的轮询状态都会被标记为异常
97 : // rankNum前面的逻辑保证了是大于0的, 多机场景取1，非多机场景取rankNum
98 1 : const u32 streamNum = ctx->multiServerFlag ? 1U : ctx->rankNum;
99 9 : for (uint32_t rank = 0U; rank <= streamNum - 1; ++rank) {
100 8 : HandleCqeStatusByRank(ctx, rank);
101 : }
102 : }
103 :
104 : // stop 主线程
105 1 : void AicpuExecutorTracer::StopKfcThread(AicpuComContext *const ctx,
106 : std::vector<std::pair<std::string, hccl::HcclCommAicpu *>> aicpuCommInfo)
107 : {
108 1 : const uint64_t waitTime = static_cast<uint64_t>(NSEC_PER_SEC) * 10U; // 10s
109 1 : uint64_t startTime = GetCurCpuTimestamp();
110 231003541 : while (ctx->isRunning) {
111 231003541 : if (!aicpuCommInfo.empty()) {
112 0 : for (auto &commInfo : aicpuCommInfo) {
113 0 : hccl::HcclCommAicpu *hcclAicpu = commInfo.second;
114 0 : DfxExtendInfo* dfxInfo = hcclAicpu->GetDfxExtendInfo();
115 0 : dfxInfo->pollStatus = PollStatus::kStopAsException;
116 : }
117 : } else {
118 231003541 : AicpuUpdatComContextMumber(offsetof(AicpuComContext, dfxExtendInfo.pollStatus),
119 : PollStatus::kStopAsException);
120 : }
121 :
122 231003541 : if ((GetCurCpuTimestamp() - startTime) >= waitTime) {
123 1 : HCCL_ERROR("stop kfc thread timeout [10s]");
124 1 : break;
125 : }
126 : }
127 1 : }
128 :
129 8 : void AicpuExecutorTracer::HandleCqeStatusByRank(AicpuComContext *const ctx, uint32_t rank)
130 : {
131 8 : const HcclComStreamInfo &streamInfo = ctx->streamInfo[rank];
132 : CqeQueryInput cqeQueryInput;
133 8 : SetCqeQueryInput(ctx->devId, streamInfo, cqeQueryInput);
134 : rtLogicCqReport_t report[AC_SQE_REV_MAX_CNT];
135 8 : cqeQueryInput.cqeAddr = reinterpret_cast<uint8_t *>(report); // 用于存放接收到的cq
136 : rtLogicCqReport_t cqeException;
137 8 : CqeStatus cqeStatus = CqReportRecv(cqeQueryInput, cqeException);
138 8 : if (cqeStatus != dfx::CqeStatus::kDefault) {
139 8 : (void)MC2TraceUtils::Save();
140 8 : AicpuUpdatComContextMumber(offsetof(AicpuComContext, dfxExtendInfo.cqeStatus), cqeStatus);
141 8 : AicpuUpdatComContextMumber(offsetof(AicpuComContext, dfxExtendInfo.pollStatus), PollStatus::kStopAsException);
142 8 : AicpuUpdatComContextMumber(offsetof(AicpuComContext, dfxExtendInfo.cqeException.sqeType), cqeException.sqeType);
143 8 : HCCL_ERROR("After send sqe:%d, exception happened on rank %u, cqeStatus[%d], sqetype[%u]",
144 : streamInfo.sqId, rank, cqeStatus, cqeException.sqeType);
145 : }
146 :
147 8 : if (cqeStatus == dfx::CqeStatus::kCqeException) {
148 8 : PrintTaskException(cqeException);
149 : }
150 8 : }
151 :
152 8 : void AicpuExecutorTracer::PrintTaskException(const rtLogicCqReport_t &reportOfOne)
153 : {
154 : const std::vector<std::string> StarsCqeErrorDesc = {
155 : "task exception",
156 : "task trap",
157 : "task timeout",
158 : "sqe error",
159 : "resource conflict error",
160 : "sq sw status error",
161 : "warning"
162 16 : };
163 8 : uint32_t errBit = static_cast<uint32_t>(getTrailingZeros(reportOfOne.errorType));
164 8 : const char *const errMsg = errBit < StarsCqeErrorDesc.size() ? StarsCqeErrorDesc[errBit].c_str() : "unknown";
165 8 : uint32_t idx = AicpuKfcProcess::GetStreamRankIdx(reportOfOne.streamId);
166 8 : SqeInfo sqeInfo;
167 8 : (void)AicpuSqeContext::QuerySqeInfoByTaskId(idx, reportOfOne.taskId, &sqeInfo);
168 8 : HCCL_ERROR("Task run failed of exception, errorType [%u] error msg:[%s] sqe info:[%s]",
169 : errBit,
170 : errMsg,
171 : AicpuSqeContext::GetString(sqeInfo).c_str());
172 8 : }
173 :
174 8 : uint8_t AicpuExecutorTracer::getTrailingZeros(uint8_t num)
175 : {
176 8 : uint8_t count = 0;
177 40 : while ((num & 1U) == 0) {
178 40 : count++;
179 40 : num >>= 1;
180 40 : if (num == 1U) {
181 8 : break;
182 : }
183 : }
184 8 : return count;
185 : }
186 :
187 16 : void AicpuExecutorTracer::SetCqeQueryInput(const uint32_t devId, const HcclComStreamInfo &streamInfo,
188 : CqeQueryInput &cqeQueryInput)
189 : {
190 16 : cqeQueryInput.devId = devId;
191 16 : cqeQueryInput.streamId = streamInfo.actualStreamId;
192 16 : cqeQueryInput.sqId = streamInfo.sqId;
193 16 : cqeQueryInput.cqId = streamInfo.logicCqId;
194 16 : cqeQueryInput.type = static_cast<uint32_t>(DRV_LOGIC_TYPE);
195 16 : }
196 :
197 3 : void KfcCommandHandles::StopFunc(AicpuComContext *const ctx)
198 : {
199 3 : HCCL_INFO("StopFunc, current clusterId:%d", ctx->clusterId);
200 3 : if (ctx->isStopLaunch) {
201 : // kill 流
202 3 : if ((StreamsKill(ctx->devId) != HCCL_SUCCESS) ||
203 1 : (DeviceQuery(ctx->devId, ts::APP_ABORT_KILL_FINISH, 0U) != HCCL_SUCCESS)) {
204 1 : (void)AicpuHdcUtils::SetOpExecStatus(ctx->kfcStatusTransferD2H, KfcStatus::kError, KfcError::kExec, 0);
205 1 : AicpuUpdatComContextMumber(offsetof(AicpuComContext, isStopLaunch), false);
206 1 : return;
207 : }
208 :
209 : // 停止条件算子
210 1 : HcclMsgArea *hcclMsgArea = reinterpret_cast<HcclMsgArea *>(ctx->workSpaceAddr);
211 65 : for (uint32_t i = 0; i < HCCL_MSG_CNT; i++) {
212 64 : hcclMsgArea->commMsg.singleMsg.commitTurnCnt[i].cnt = 0xFF;
213 : }
214 1 : (void)AicpuHdcUtils::SetOpExecStatus(ctx->kfcStatusTransferD2H, KfcStatus::kStopExec, KfcError::kNone, 0);
215 : } else {
216 1 : (void)AicpuHdcUtils::SetOpExecStatus(ctx->kfcStatusTransferD2H, KfcStatus::kEnd, KfcError::kNone, 0);
217 : }
218 2 : HCCL_INFO("StopFunc Finish");
219 : }
220 :
221 1 : void KfcCommandHandles::ClearCq(AicpuComContext *const ctx)
222 : {
223 1 : const u32 streamNum = ctx->multiServerFlag ? 1U : ctx->rankNum;
224 9 : for (u32 i = 0; i < streamNum; i++) {
225 8 : const HcclComStreamInfo &streamInfo = ctx->streamInfo[i];
226 8 : HCCL_INFO("ClearFunc, sqid:%d", streamInfo.sqId);
227 8 : if (ConfigSqStatusByType(ctx->devId, streamInfo.sqId, DRV_SQCQ_PROP_SQ_DISABLE_TO_ENABLE, 1) != HCCL_SUCCESS) {
228 0 : (void)AicpuHdcUtils::SetOpExecStatus(ctx->kfcStatusTransferD2H, KfcStatus::kError, KfcError::kExec, 0);
229 0 : return;
230 : }
231 : CqeQueryInput cqeQueryInput;
232 8 : AicpuExecutorTracer::SetCqeQueryInput(ctx->devId, streamInfo, cqeQueryInput);
233 : rtLogicCqReport_t report[AC_SQE_REV_MAX_CNT];
234 8 : cqeQueryInput.cqeAddr = reinterpret_cast<uint8_t *>(report); // 用于存放接收到的cq
235 : rtLogicCqReport_t cqeException;
236 8 : (void)CqReportRecv(cqeQueryInput, cqeException);
237 : }
238 : }
239 :
240 3 : void KfcCommandHandles::ClearFunc(AicpuComContext *const ctx)
241 : {
242 3 : if (ctx->isStopLaunch) {
243 : // 等待drv任务停止
244 2 : if (DeviceQuery(ctx->devId, ts::APP_ABORT_TERMINATE_FINISH, 0U) != HCCL_SUCCESS) {
245 1 : (void)AicpuHdcUtils::SetOpExecStatus(ctx->kfcStatusTransferD2H, KfcStatus::kError, KfcError::kExec, 0);
246 1 : return;
247 : }
248 1 : HCCL_INFO("ClearFunc, after APP_ABORT_TERMINATE_FINISH");
249 : // 使能sq,读清cq
250 1 : ClearCq(ctx);
251 :
252 : // 清理dfxExtendInfo状态
253 1 : AicpuUpdatComContextMumber(offsetof(AicpuComContext, dfxExtendInfo.kfcStatus), DfxKfcStatus::kDefault);
254 1 : AicpuUpdatComContextMumber(offsetof(AicpuComContext, dfxExtendInfo.cqeStatus), dfx::CqeStatus::kDefault);
255 1 : AicpuUpdatComContextMumber(offsetof(AicpuComContext, dfxExtendInfo.pollStatus), PollStatus::kDefault);
256 :
257 : // 清理SqeContext
258 1 : AicpuSqeContext::SyncVariable();
259 1 : AicpuSqeContext::SaveVariable();
260 1 : if (AicpuSqeContext::ClearLocalBuff() != HCCL_SUCCESS) {
261 0 : (void)AicpuHdcUtils::SetOpExecStatus(ctx->kfcStatusTransferD2H, KfcStatus::kError, KfcError::kExec, 0);
262 0 : return;
263 : }
264 1 : SqeContext *sqeContext = GetSqeContext();
265 1 : const u32 streamNum = ctx->multiServerFlag ? 1U : ctx->rankNum;
266 9 : for (u32 i = 0; i < streamNum; i++) {
267 8 : auto &buff = sqeContext->buffPtr[i];
268 8 : if ((QuerySqStatusByType(ctx->devId, ctx->streamInfo[i].sqId, DRV_SQCQ_PROP_SQ_TAIL, buff.sqTail) !=
269 16 : HCCL_SUCCESS) ||
270 8 : (QuerySqStatusByType(ctx->devId, ctx->streamInfo[i].sqId, DRV_SQCQ_PROP_SQ_HEAD, buff.sqHead) !=
271 : HCCL_SUCCESS)) {
272 0 : (void)AicpuHdcUtils::SetOpExecStatus(ctx->kfcStatusTransferD2H, KfcStatus::kError, KfcError::kExec, 0);
273 0 : return;
274 : }
275 8 : HCCL_INFO("hccl aicpu reset stream buffer, sqid:%d head:%u tail:%u.", ctx->streamInfo[i].sqId, buff.sqHead,
276 : buff.sqTail);
277 : }
278 : }
279 2 : if (!ctx->ibversData.empty()) {
280 0 : (void)memset_s(reinterpret_cast<void *>(ctx->ibversData[ctx->rankId].localInputMem.addr),
281 0 : ctx->ibversData[ctx->rankId].localInputMem.size, 0, ctx->ibversData[ctx->rankId].localInputMem.size);
282 0 : (void)memset_s(reinterpret_cast<void *>(ctx->ibversData[ctx->rankId].localOutputMem.addr),
283 0 : ctx->ibversData[ctx->rankId].localOutputMem.size, 0, ctx->ibversData[ctx->rankId].localOutputMem.size);
284 : }
285 : // 清理共享内存
286 2 : (void)memset_s(reinterpret_cast<void *>(ctx->workSpaceAddr), sizeof(HcclMsgArea), 0, sizeof(HcclMsgArea));
287 :
288 2 : AicpuUpdatComContextMumber(offsetof(AicpuComContext, isOpLaunch), false);
289 2 : AicpuUpdatComContextMumber(offsetof(AicpuComContext, endStopLaunch), false);
290 2 : if (ctx->isStopLaunch) {
291 1 : AicpuUpdatComContextMumber(offsetof(AicpuComContext, isStopLaunch), false);
292 1 : (void)AicpuHdcUtils::SetOpExecStatus(ctx->kfcStatusTransferD2H, KfcStatus::kClear, KfcError::kNone, 0);
293 : } else {
294 1 : (void)AicpuHdcUtils::SetOpExecStatus(ctx->kfcStatusTransferD2H, KfcStatus::kEnd, KfcError::kNone, 0);
295 : }
296 2 : HCCL_INFO("ClearFunc Finish");
297 : }
298 : } // namespace dfx_tracer
|
