Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "aicpu_executor_tracer.h"
12 :
13 : #include "hccl_msg.h"
14 : #include "common/aicpu_hccl_common.h"
15 : #include "mc2_trace_utils.h"
16 : #include "utils/aicpu_hdc_utils.h"
17 : #include "framework/aicpu_kfc_process.h"
18 :
19 : using HcclApi::HCCL_MSG_CNT;
20 : using HcclApi::HcclMsgArea;
21 : namespace dfx_tracer {
22 :
23 : // recv host stop command
24 8101 : void AicpuExecutorTracer::HandleBackGround(AicpuComContext* const ctx)
25 : {
26 : BackgroundCommand bgCmd;
27 8101 : if (ctx->commOpenStatus) {
28 8101 : (void)AicpuHdcUtils::GetBackGroundCommand(ctx->kfcControlTransferH2D, bgCmd);
29 8101 : if (bgCmd == BackgroundCommand::kStop) {
30 1 : HCCL_RUN_INFO("ctx stop back ground");
31 1 : StopKfcThread(ctx, {});
32 1 : KfcExecStatus bgresponse;
33 1 : bgresponse.execStatus.backgroundStatus = BackgroundStatus::kStop;
34 1 : ctx->commOpenStatus = false;
35 1 : AicpuUpdatComContextMumber(offsetof(AicpuComContext, alreadyInit), false);
36 1 : (void)MC2TraceUtils::DestoryHandles();
37 1 : (void)AicpuHdcUtils::ResponseBackGroundStatus(ctx->kfcStatusTransferD2H, bgresponse);
38 1 : bgCmd = BackgroundCommand::kNone;
39 : }
40 : }
41 8101 : }
42 :
43 : // handle StopLaunch Command
44 8100 : void AicpuExecutorTracer::StopLaunchCommandHandle(AicpuComContext* const ctx)
45 : {
46 8100 : if (ctx->commOpenStatus) {
47 8100 : if (!ctx->endStopLaunch) {
48 8100 : KfcCommand cmd = KfcCommand::kNone;
49 8100 : (void)AicpuHdcUtils::GetOpExecCtrlCmd(ctx->kfcControlTransferH2D, cmd);
50 8100 : if (cmd == KfcCommand::NsStopLaunch) {
51 0 : if (!ctx->isOpLaunch) {
52 0 : (void)AicpuHdcUtils::SetOpExecStatus(
53 0 : ctx->kfcStatusTransferD2H, KfcStatus::kStoplaunch, KfcError::kNone, 0);
54 0 : AicpuUpdatComContextMumber(offsetof(AicpuComContext, endStopLaunch), true);
55 0 : HCCL_INFO("[NsRecovery][backGround]send in mc2 environment");
56 : }
57 : }
58 : }
59 : }
60 8100 : }
61 :
62 : // handle StopExec and Clean Command
63 8106 : void AicpuExecutorTracer::KfcCommandHandle(AicpuComContext* const ctx)
64 : {
65 8106 : if (ctx->commOpenStatus) {
66 : using CommandCall = std::function<void(AicpuComContext* const ctx)>;
67 : static std::map<KfcCommand, CommandCall> commandHandles = {
68 8114 : {KfcCommand::NsStopExec, KfcCommandHandles::StopFunc}, {KfcCommand::NsClear, KfcCommandHandles::ClearFunc}};
69 :
70 8106 : KfcCommand cmd = KfcCommand::kNone;
71 8106 : (void)AicpuHdcUtils::GetOpExecCtrlCmd(ctx->kfcControlTransferH2D, cmd);
72 8106 : auto iter = commandHandles.find(cmd);
73 8106 : if (iter == commandHandles.cend()) {
74 8100 : return;
75 : }
76 6 : HCCL_INFO("Start to run command %ld", cmd);
77 6 : iter->second(ctx);
78 : }
79 2 : }
80 :
81 8100 : void AicpuExecutorTracer::HandleCqeStatus(AicpuComContext* const ctx)
82 : {
83 8100 : if (ctx == nullptr || ctx->alreadyInit == false) {
84 0 : return;
85 : }
86 :
87 8100 : if (ctx->dfxExtendInfo.cqeStatus != dfx::CqeStatus::kDefault) {
88 99 : return;
89 : }
90 :
91 8001 : if (ctx->dfxExtendInfo.kfcStatus != DfxKfcStatus::kOneStart) {
92 8000 : return;
93 : }
94 :
95 : // 遍历每个卡的cqe状态, 如果一个卡有异常，所有的卡的ctx里的轮询状态都会被标记为异常
96 : // rankNum前面的逻辑保证了是大于0的, 多机场景取1，非多机场景取rankNum
97 1 : const u32 streamNum = ctx->multiServerFlag ? 1U : ctx->rankNum;
98 9 : for (uint32_t rank = 0U; rank <= streamNum - 1; ++rank) {
99 8 : HandleCqeStatusByRank(ctx, rank);
100 : }
101 : }
102 :
103 : // stop 主线程
104 1 : void AicpuExecutorTracer::StopKfcThread(
105 : AicpuComContext* const ctx, std::vector<std::pair<std::string, hccl::HcclCommAicpu*>> aicpuCommInfo)
106 : {
107 1 : const uint64_t waitTime = static_cast<uint64_t>(NSEC_PER_SEC) * 10U; // 10s
108 1 : uint64_t startTime = GetCurCpuTimestamp();
109 229257176 : while (ctx->isRunning) {
110 229257176 : if (!aicpuCommInfo.empty()) {
111 0 : for (auto& commInfo : aicpuCommInfo) {
112 0 : hccl::HcclCommAicpu* hcclAicpu = commInfo.second;
113 0 : DfxExtendInfo* dfxInfo = hcclAicpu->GetDfxExtendInfo();
114 0 : dfxInfo->pollStatus = PollStatus::kStopAsException;
115 : }
116 : } else {
117 229257176 : AicpuUpdatComContextMumber(
118 : offsetof(AicpuComContext, dfxExtendInfo.pollStatus), PollStatus::kStopAsException);
119 : }
120 :
121 229257176 : if ((GetCurCpuTimestamp() - startTime) >= waitTime) {
122 1 : HCCL_ERROR("stop kfc thread timeout [10s]");
123 1 : break;
124 : }
125 : }
126 1 : }
127 :
128 8 : void AicpuExecutorTracer::HandleCqeStatusByRank(AicpuComContext* const ctx, uint32_t rank)
129 : {
130 8 : const HcclComStreamInfo& streamInfo = ctx->streamInfo[rank];
131 : CqeQueryInput cqeQueryInput;
132 8 : SetCqeQueryInput(ctx->devId, streamInfo, cqeQueryInput);
133 : rtLogicCqReport_t report[AC_SQE_REV_MAX_CNT];
134 8 : cqeQueryInput.cqeAddr = reinterpret_cast<uint8_t*>(report); // 用于存放接收到的cq
135 : rtLogicCqReport_t cqeException;
136 8 : CqeStatus cqeStatus = CqReportRecv(cqeQueryInput, cqeException);
137 8 : if (cqeStatus != dfx::CqeStatus::kDefault) {
138 8 : (void)MC2TraceUtils::Save();
139 8 : AicpuUpdatComContextMumber(offsetof(AicpuComContext, dfxExtendInfo.cqeStatus), cqeStatus);
140 8 : AicpuUpdatComContextMumber(offsetof(AicpuComContext, dfxExtendInfo.pollStatus), PollStatus::kStopAsException);
141 8 : AicpuUpdatComContextMumber(offsetof(AicpuComContext, dfxExtendInfo.cqeException.sqeType), cqeException.sqeType);
142 8 : HCCL_ERROR(
143 : "After send sqe:%d, exception happened on rank %u, cqeStatus[%d], sqetype[%u]", streamInfo.sqId, rank,
144 : cqeStatus, cqeException.sqeType);
145 : }
146 :
147 8 : if (cqeStatus == dfx::CqeStatus::kCqeException) {
148 8 : PrintTaskException(cqeException);
149 : }
150 8 : }
151 :
152 8 : void AicpuExecutorTracer::PrintTaskException(const rtLogicCqReport_t& reportOfOne)
153 : {
154 : const std::vector<std::string> StarsCqeErrorDesc
155 : = {"task exception", "task trap", "task timeout", "sqe error",
156 16 : "resource conflict error", "sq sw status error", "warning"};
157 8 : uint32_t errBit = static_cast<uint32_t>(getTrailingZeros(reportOfOne.errorType));
158 8 : const char* const errMsg = errBit < StarsCqeErrorDesc.size() ? StarsCqeErrorDesc[errBit].c_str() : "unknown";
159 8 : uint32_t idx = AicpuKfcProcess::GetStreamRankIdx(reportOfOne.streamId);
160 8 : SqeInfo sqeInfo;
161 8 : (void)AicpuSqeContext::QuerySqeInfoByTaskId(idx, reportOfOne.taskId, &sqeInfo);
162 8 : HCCL_ERROR(
163 : "Task run failed of exception, errorType [%u] error msg:[%s] sqe info:[%s]", errBit, errMsg,
164 : AicpuSqeContext::GetString(sqeInfo).c_str());
165 8 : }
166 :
167 8 : uint8_t AicpuExecutorTracer::getTrailingZeros(uint8_t num)
168 : {
169 8 : uint8_t count = 0;
170 40 : while ((num & 1U) == 0) {
171 40 : count++;
172 40 : num >>= 1;
173 40 : if (num == 1U) {
174 8 : break;
175 : }
176 : }
177 8 : return count;
178 : }
179 :
180 16 : void AicpuExecutorTracer::SetCqeQueryInput(
181 : const uint32_t devId, const HcclComStreamInfo& streamInfo, CqeQueryInput& cqeQueryInput)
182 : {
183 16 : cqeQueryInput.devId = devId;
184 16 : cqeQueryInput.streamId = streamInfo.actualStreamId;
185 16 : cqeQueryInput.sqId = streamInfo.sqId;
186 16 : cqeQueryInput.cqId = streamInfo.logicCqId;
187 16 : cqeQueryInput.type = static_cast<uint32_t>(DRV_LOGIC_TYPE);
188 16 : }
189 :
190 3 : void KfcCommandHandles::StopFunc(AicpuComContext* const ctx)
191 : {
192 3 : HCCL_INFO("StopFunc, current clusterId:%d", ctx->clusterId);
193 3 : if (ctx->isStopLaunch) {
194 : // kill 流
195 2 : if ((StreamsKill(ctx->devId) != HCCL_SUCCESS)
196 2 : || (DeviceQuery(ctx->devId, ts::APP_ABORT_KILL_FINISH, 0U) != HCCL_SUCCESS)) {
197 1 : (void)AicpuHdcUtils::SetOpExecStatus(ctx->kfcStatusTransferD2H, KfcStatus::kError, KfcError::kExec, 0);
198 1 : AicpuUpdatComContextMumber(offsetof(AicpuComContext, isStopLaunch), false);
199 1 : return;
200 : }
201 :
202 : // 停止条件算子
203 1 : HcclMsgArea* hcclMsgArea = reinterpret_cast<HcclMsgArea*>(ctx->workSpaceAddr);
204 65 : for (uint32_t i = 0; i < HCCL_MSG_CNT; i++) {
205 64 : hcclMsgArea->commMsg.singleMsg.commitTurnCnt[i].cnt = 0xFF;
206 : }
207 1 : (void)AicpuHdcUtils::SetOpExecStatus(ctx->kfcStatusTransferD2H, KfcStatus::kStopExec, KfcError::kNone, 0);
208 : } else {
209 1 : (void)AicpuHdcUtils::SetOpExecStatus(ctx->kfcStatusTransferD2H, KfcStatus::kEnd, KfcError::kNone, 0);
210 : }
211 2 : HCCL_INFO("StopFunc Finish");
212 : }
213 :
214 1 : void KfcCommandHandles::ClearCq(AicpuComContext* const ctx)
215 : {
216 1 : const u32 streamNum = ctx->multiServerFlag ? 1U : ctx->rankNum;
217 9 : for (u32 i = 0; i < streamNum; i++) {
218 8 : const HcclComStreamInfo& streamInfo = ctx->streamInfo[i];
219 8 : HCCL_INFO("ClearFunc, sqid:%d", streamInfo.sqId);
220 8 : if (ConfigSqStatusByType(ctx->devId, streamInfo.sqId, DRV_SQCQ_PROP_SQ_DISABLE_TO_ENABLE, 1) != HCCL_SUCCESS) {
221 0 : (void)AicpuHdcUtils::SetOpExecStatus(ctx->kfcStatusTransferD2H, KfcStatus::kError, KfcError::kExec, 0);
222 0 : return;
223 : }
224 : CqeQueryInput cqeQueryInput;
225 8 : AicpuExecutorTracer::SetCqeQueryInput(ctx->devId, streamInfo, cqeQueryInput);
226 : rtLogicCqReport_t report[AC_SQE_REV_MAX_CNT];
227 8 : cqeQueryInput.cqeAddr = reinterpret_cast<uint8_t*>(report); // 用于存放接收到的cq
228 : rtLogicCqReport_t cqeException;
229 8 : (void)CqReportRecv(cqeQueryInput, cqeException);
230 : }
231 : }
232 :
233 3 : void KfcCommandHandles::ClearFunc(AicpuComContext* const ctx)
234 : {
235 3 : if (ctx->isStopLaunch) {
236 : // 等待drv任务停止
237 2 : if (DeviceQuery(ctx->devId, ts::APP_ABORT_TERMINATE_FINISH, 0U) != HCCL_SUCCESS) {
238 1 : (void)AicpuHdcUtils::SetOpExecStatus(ctx->kfcStatusTransferD2H, KfcStatus::kError, KfcError::kExec, 0);
239 1 : return;
240 : }
241 1 : HCCL_INFO("ClearFunc, after APP_ABORT_TERMINATE_FINISH");
242 : // 使能sq,读清cq
243 1 : ClearCq(ctx);
244 :
245 : // 清理dfxExtendInfo状态
246 1 : AicpuUpdatComContextMumber(offsetof(AicpuComContext, dfxExtendInfo.kfcStatus), DfxKfcStatus::kDefault);
247 1 : AicpuUpdatComContextMumber(offsetof(AicpuComContext, dfxExtendInfo.cqeStatus), dfx::CqeStatus::kDefault);
248 1 : AicpuUpdatComContextMumber(offsetof(AicpuComContext, dfxExtendInfo.pollStatus), PollStatus::kDefault);
249 :
250 : // 清理SqeContext
251 1 : AicpuSqeContext::SyncVariable();
252 1 : AicpuSqeContext::SaveVariable();
253 1 : if (AicpuSqeContext::ClearLocalBuff() != HCCL_SUCCESS) {
254 0 : (void)AicpuHdcUtils::SetOpExecStatus(ctx->kfcStatusTransferD2H, KfcStatus::kError, KfcError::kExec, 0);
255 0 : return;
256 : }
257 1 : SqeContext* sqeContext = GetSqeContext();
258 1 : const u32 streamNum = ctx->multiServerFlag ? 1U : ctx->rankNum;
259 9 : for (u32 i = 0; i < streamNum; i++) {
260 8 : auto& buff = sqeContext->buffPtr[i];
261 8 : if ((QuerySqStatusByType(ctx->devId, ctx->streamInfo[i].sqId, DRV_SQCQ_PROP_SQ_TAIL, buff.sqTail)
262 : != HCCL_SUCCESS)
263 8 : || (QuerySqStatusByType(ctx->devId, ctx->streamInfo[i].sqId, DRV_SQCQ_PROP_SQ_HEAD, buff.sqHead)
264 : != HCCL_SUCCESS)) {
265 0 : (void)AicpuHdcUtils::SetOpExecStatus(ctx->kfcStatusTransferD2H, KfcStatus::kError, KfcError::kExec, 0);
266 0 : return;
267 : }
268 8 : HCCL_INFO(
269 : "hccl aicpu reset stream buffer, sqid:%d head:%u tail:%u.", ctx->streamInfo[i].sqId, buff.sqHead,
270 : buff.sqTail);
271 : }
272 : }
273 2 : if (!ctx->ibversData.empty()) {
274 0 : (void)memset_s(
275 0 : reinterpret_cast<void*>(ctx->ibversData[ctx->rankId].localInputMem.addr),
276 0 : ctx->ibversData[ctx->rankId].localInputMem.size, 0, ctx->ibversData[ctx->rankId].localInputMem.size);
277 0 : (void)memset_s(
278 0 : reinterpret_cast<void*>(ctx->ibversData[ctx->rankId].localOutputMem.addr),
279 0 : ctx->ibversData[ctx->rankId].localOutputMem.size, 0, ctx->ibversData[ctx->rankId].localOutputMem.size);
280 : }
281 : // 清理共享内存
282 2 : (void)memset_s(reinterpret_cast<void*>(ctx->workSpaceAddr), sizeof(HcclMsgArea), 0, sizeof(HcclMsgArea));
283 :
284 2 : AicpuUpdatComContextMumber(offsetof(AicpuComContext, isOpLaunch), false);
285 2 : AicpuUpdatComContextMumber(offsetof(AicpuComContext, endStopLaunch), false);
286 2 : if (ctx->isStopLaunch) {
287 1 : AicpuUpdatComContextMumber(offsetof(AicpuComContext, isStopLaunch), false);
288 1 : (void)AicpuHdcUtils::SetOpExecStatus(ctx->kfcStatusTransferD2H, KfcStatus::kClear, KfcError::kNone, 0);
289 : } else {
290 1 : (void)AicpuHdcUtils::SetOpExecStatus(ctx->kfcStatusTransferD2H, KfcStatus::kEnd, KfcError::kNone, 0);
291 : }
292 2 : HCCL_INFO("ClearFunc Finish");
293 : }
294 : } // namespace dfx_tracer
|
