Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "aicpu_kfc_interface.h"
12 :
13 : #include "common/aicpu_hccl_common.h"
14 : #include "common/aicpu_hccl_def.h"
15 : #include "common/aicpu_sqe_context.h"
16 : #include "dfx/mc2_trace_utils.h"
17 : #include "dfx/aicpu_profiling_manager.h"
18 : #include "framework/aicpu_hccl_process.h"
19 : #include "aicpu_kfc/framework/aicpu_kfc_process.h"
20 : #include "aicpu_kfc/decoupler/comm_kfc_dispatcher.h"
21 : #include "aicpu_kfc/framework/aicpu_kfc_deprecated_process.h"
22 : #include "aicpu_kfc/framework/aicpu_kfc_batchwrite_process.h"
23 : #include "utils/hccl_aicpu_utils.h"
24 : #include "common/aicpu_kfc_utils.h"
25 : #include "aicpu_kfc/common/aicpu_kfc_tiling_utils.h"
26 : #include "aicpu_kfc/framework/aicpu_kfc_prof.h"
27 : #include "utils/aicpu_hdc_utils.h"
28 :
29 : using namespace HcclApi;
30 : namespace {
31 2 : u64 GetTensorAddr(uint16_t index, uint8_t* tensorPtr)
32 : {
33 2 : uint64_t* dataAddr = reinterpret_cast<uint64_t*>(tensorPtr);
34 2 : uint64_t tensorPtrOffset = *dataAddr;
35 : // Moving 3 bits to the right means dividing by sizeof(uint64 t).
36 2 : uint64_t* resPtr = dataAddr + (tensorPtrOffset >> 3);
37 2 : return u64(*(resPtr + index));
38 : }
39 :
40 5 : u64 GetUpdatedOpIdx()
41 : {
42 : static uint64_t aicpuOpIdx[MAX_AICPU_NUM_BLOCKS] = {0UL};
43 5 : const u32 blockNum = HcclAicpuUtils::GetBlockNum();
44 5 : const u32 blockIdx = HcclAicpuUtils::GetBlockIdx();
45 5 : ++aicpuOpIdx[blockIdx];
46 5 : if (blockIdx == blockNum - 1U) {
47 30 : for (u32 i = blockIdx + 1U; i < MAX_AICPU_NUM_BLOCKS; ++i) {
48 25 : ++aicpuOpIdx[i];
49 : }
50 : }
51 5 : return aicpuOpIdx[blockIdx];
52 : }
53 :
54 60 : HcclResult AicpuRunRpcServer(AicpuComContext* ctx, KFCTask* taskInfo)
55 : {
56 : // 启动RPC服务
57 60 : static AicpuKfcRpcServer rpc;
58 60 : rpc.Init(ctx->workSpaceAddr, ctx->notifyOff, ctx->notifyBeginCnt, taskInfo);
59 60 : AicpuKfcProf::GetProInst(*ctx).commInitEndTime = GetCurCpuTimestamp(true);
60 60 : if (rpc.GetPreparePosition() == TASK_PREPARE_KERNEL) {
61 23 : auto ret = AicpuKfcProcess::RunRpcServerApi(ctx, rpc);
62 23 : AicpuUpdatComContextMumber(offsetof(AicpuComContext, isOpLaunch), false);
63 23 : if (ret != HCCL_SUCCESS) {
64 8 : return AicpuKfcProcess::DealReturnValue(ctx, ret);
65 : } else {
66 15 : CHK_RET(AicpuHdcUtils::SetOpExecStatus(ctx->kfcStatusTransferD2H, KfcStatus::kEnd, KfcError::kNone, 0));
67 15 : return ret;
68 : }
69 : }
70 :
71 37 : if (rpc.GetTaskType() == HCCL_KFC_TASK_HCCL_ONLY_EXE) {
72 15 : auto ret = AicpuKfcDeprecatedProcess::AICPU_RpcServerUnfoldStageWait(ctx, rpc);
73 15 : if ((ret != HCCL_SUCCESS) && (ret != HCCL_E_SUSPENDING)) {
74 11 : HCCL_ERROR(
75 : "AicpuRpcStageWait failed, commType:%d, reducekind:%d, totalCnt:%lu, totalTurnCnt:%u", ctx->commType,
76 : ctx->reducekind, ctx->totalCnt, ctx->totalTurnCnt);
77 11 : return ret;
78 : }
79 4 : if (ret == HCCL_E_SUSPENDING) {
80 0 : HCCL_RUN_INFO("[NsRecovery][AICPU] Suspending");
81 0 : return ret;
82 : }
83 22 : } else if (ctx->devType == DevType::DEV_TYPE_310P1 || ctx->devType == DevType::DEV_TYPE_310P3) {
84 1 : auto ret = AicpuKfcDeprecatedProcess::RunRpcServerTwoStageWait(ctx, rpc);
85 1 : if ((ret != HCCL_SUCCESS) && (ret != HCCL_E_SUSPENDING)) {
86 1 : return ret;
87 : }
88 0 : if (ret == HCCL_E_SUSPENDING) {
89 0 : HCCL_RUN_INFO("[NsRecovery][MC2] Suspending");
90 0 : return ret;
91 : }
92 0 : } else {
93 21 : auto ret = AicpuKfcDeprecatedProcess::TryRunRpcServerOneStageWait(ctx, rpc);
94 21 : if ((ret != HCCL_SUCCESS) && (ret != HCCL_E_SUSPENDING)) {
95 1 : return ret;
96 : }
97 20 : if (ret == HCCL_E_SUSPENDING) {
98 0 : HCCL_RUN_INFO("[NsRecovery][MC2] Suspending");
99 0 : return ret;
100 : }
101 : }
102 :
103 24 : return HCCL_SUCCESS;
104 : }
105 :
106 3 : u32 RunAicpuInnerRpcSrvGroupLaunch(void* args[], KFCGroupTilingDataAuto* tilingData, CommKfcParamDesc* desc)
107 : {
108 3 : constexpr int DESC_POS = 0;
109 :
110 3 : if (tilingData == nullptr || tilingData->groupNum == 0) {
111 1 : HCCL_ERROR("tilingData is nullptr or groupNum is 0.");
112 1 : return HCCL_E_PARA;
113 : }
114 3 : for (uint32_t i = 0; i < tilingData->groupNum; ++i) {
115 : KFCTask singleTask;
116 2 : singleTask.inputA = u64(args[tilingData->msg[i].sendArgIndex + desc->hasFfts + desc->itemNum + 1]);
117 4 : singleTask.outputC = GetTensorAddr(
118 2 : i, reinterpret_cast<uint8_t*>(args[tilingData->msg[i].recvArgIndex + desc->hasFfts + desc->itemNum + 1]));
119 2 : singleTask.commOut = 0;
120 2 : singleTask.context = u64(args[DESC_POS + desc->hasFfts + desc->itemNum]);
121 2 : singleTask.workSpace = u64(args[desc->tilingOff - 1]);
122 2 : singleTask.tilingData = u64(&tilingData->msg[i]);
123 2 : uint32_t ret = RunAicpuRpcSrvLaunch(&singleTask);
124 2 : if (ret != 0) {
125 1 : HCCL_ERROR("RunAicpuRpcSrvGroupLaunch runs failed.");
126 1 : return HCCL_E_PARA;
127 : }
128 : }
129 1 : return 0;
130 : }
131 :
132 2 : u32 RunKernelAicpuServerV1(void* args[], CommKfcParamDesc* desc)
133 : {
134 2 : HcclKFCTilingData* tilingData = static_cast<HcclKFCTilingData*>(args[desc->tilingOff]);
135 2 : HCCL_INFO("RunAicpuKfcSrvLaunch, tiling.sendArgIndex %lu", tilingData->sendArgIndex);
136 2 : HCCL_INFO("RunAicpuKfcSrvLaunch, tiling.recvArgIndex %lu", tilingData->recvArgIndex);
137 2 : HCCL_INFO("RunAicpuKfcSrvLaunch, tiling.commOutArgIndex %lu", tilingData->commOutArgIndex);
138 2 : HCCL_INFO("RunAicpuKfcSrvLaunch, tiling.hasCommOut %lu", tilingData->hasCommOut);
139 : KFCTask task;
140 2 : task.tilingData = u64(tilingData);
141 2 : task.inputA = u64(args[tilingData->sendArgIndex + desc->hasFfts + desc->itemNum + 1]);
142 2 : task.outputC = u64(args[tilingData->recvArgIndex + desc->hasFfts + desc->itemNum + 1]);
143 2 : task.context = u64(args[desc->hasFfts + desc->itemNum]);
144 2 : task.workSpace = u64(args[desc->tilingOff - 1]);
145 2 : if (tilingData->commOutArgIndex != u64(0xff)) {
146 2 : task.commOut = u64(args[tilingData->commOutArgIndex + desc->hasFfts + desc->itemNum + 1]);
147 : } else {
148 0 : task.commOut = 0;
149 : }
150 2 : AicpuKfcUtils::PrintKFCTask(task);
151 2 : HCCL_INFO("Task Assembled. Start to launch RunAicpuRpcSrvLaunch");
152 2 : const uint32_t ret = RunAicpuRpcSrvLaunch(&task);
153 2 : HCCL_INFO("RunAicpuKfcSrvLaunch ends with result %lu.", ret);
154 2 : return ret;
155 : }
156 :
157 6 : HcclResult KfcProf(u64 launchEntryTime, KFCTaskV2& task, u32 turnOffset = 0U)
158 : {
159 6 : if (AicpuKfcProf::NeedRecordTimeTaken()) {
160 2 : AicpuKfcProf::SetCurrentProf(launchEntryTime);
161 2 : HcclOpResParam* commParam = reinterpret_cast<HcclOpResParam*>(task.context[0]);
162 2 : AicpuKfcProf::GetCurrentAicpuProf()->rankId = commParam->topoInfo.userRank;
163 2 : AicpuKfcProf::GetCurrentAicpuProf()->endTime = GetCurCpuTimestamp(true);
164 : }
165 :
166 6 : CHK_RET(dfx::AicpuProfilingManager::ReportTaskExecTimeLine(AicpuKfcProf::GetCurrentAicpuProf(), turnOffset));
167 6 : AicpuKfcProf::OutputProfLog(
168 6 : AicpuKfcProf::IsDebugModeEquals(MC2_DEBUG_TIME_TAKEN), AicpuKfcProf::GetaicpuProfInst());
169 6 : AicpuKfcProf::AddProfLoopCnt();
170 6 : return HCCL_SUCCESS;
171 : }
172 :
173 4 : u32 RunKernelAicpuServerV2(void* args[], CommKfcParamDesc* desc, void* tilingData)
174 : {
175 4 : u64 launchEntryTime = GetCurCpuTimestamp(true);
176 : // MC2目前只支持OP_BASE
177 4 : SetWorkflowMode(HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE);
178 : static uint64_t aicpuOpIdx = 0;
179 :
180 4 : Mc2ServerCfg* cfg = MC2TilingGetServerCfg(tilingData);
181 4 : AicpuKfcProf::SetDebugMode(cfg->debugMode);
182 4 : if (AicpuKfcProf::IsDebugModeEquals(MC2_DEBUG_ONLY_CUBE)) {
183 1 : HCCL_INFO("[%s]DebugMode is set to be 1 (i.e. computation only).", __func__);
184 1 : return HCCL_SUCCESS;
185 : }
186 :
187 : KFCTaskV2 task;
188 3 : task.tilingData = reinterpret_cast<u64>(tilingData);
189 3 : task.ctxNum = desc->itemNum;
190 3 : if (task.ctxNum > MAX_COMM_CTX_NUM) {
191 0 : HCCL_ERROR("group num must be smaller than %u.", MAX_COMM_CTX_NUM);
192 0 : return HCCL_E_PARA;
193 : }
194 7 : for (int i = 0; i < desc->itemNum; i++) {
195 5 : task.context[i] = reinterpret_cast<u64>(args[desc->hasFfts + i + 1]);
196 5 : if (task.context[i] == 0) {
197 1 : HCCL_ERROR("idx %d ctx is null, please check the input ctx.", i);
198 1 : return HCCL_E_PARA;
199 : }
200 4 : HcclAicpuUtils::PrintHcclOpResParam(reinterpret_cast<HcclOpResParam*>(task.context[i]));
201 : }
202 2 : task.workSpace = reinterpret_cast<u64>(args[desc->tilingOff - 1]);
203 2 : aicpuOpIdx++;
204 2 : if (AicpuKfcProf::IsDebugModeEquals(MC2_DEBUG_PRINT_MSG) || AicpuKfcProf::IsDebugModeEquals(MC2_DEBUG_PRINT_BUFF)) {
205 1 : HCCL_RUN_INFO("Server start, MC2 opIdx:%lu", aicpuOpIdx);
206 : }
207 2 : HCCL_INFO("Start launch RunAicpuInnerKfcSrvLaunch");
208 2 : AicpuKfcProf::GetCurrentAicpuProf()->workCnt = 0;
209 2 : const u32 ret = AicpuKfcProcess::AicpuRunRpcServerForMC2(&task);
210 2 : if (ret != HCCL_SUCCESS) {
211 0 : HCCL_ERROR("Server runs failed, error code %u.", ret);
212 0 : return ret;
213 : }
214 2 : CHK_RET(KfcProf(launchEntryTime, task));
215 2 : HCCL_INFO("end kfc server.");
216 2 : return 0;
217 : }
218 :
219 6 : u32 RunAicpuApiRpcSrvLaunchV1(void* args[], CommKfcParamDesc* desc)
220 : {
221 : static u32 aicpuOpIdx = 0;
222 6 : u64 launchEntryTime = GetCurCpuTimestamp(true);
223 :
224 6 : HccCommResParamTask* contextParam = nullptr;
225 6 : for (uint64_t i = 1; i <= desc->itemNum; ++i) {
226 6 : contextParam = reinterpret_cast<HccCommResParamTask*>(args[desc->hasFfts + i]);
227 6 : if (contextParam != nullptr) {
228 6 : HCCL_INFO("Idx %llu ctx addr %p.", i, contextParam);
229 6 : break;
230 : }
231 : }
232 6 : if (contextParam == nullptr) {
233 0 : HCCL_ERROR("Context args is null.");
234 0 : return HCCL_E_PARA;
235 : }
236 :
237 6 : AicpuComContext* ctx = AicpuGetComContext();
238 6 : if (ctx == nullptr || !ctx->alreadyInit || strcmp(ctx->hcomId, contextParam->hcomId) != 0) {
239 0 : HCCL_ERROR("The comm domain %s have not exist.", contextParam->hcomId);
240 0 : return HCCL_E_PARA;
241 : }
242 6 : AicpuServerRole role = AicpuKfcBatchwriteProcess::GetVerifiedServerRole(*ctx);
243 6 : if (role == AicpuServerRole::INVALID) {
244 1 : HCCL_INFO("aicpu server role is invalid, return");
245 1 : return HCCL_SUCCESS;
246 5 : } else if (role == AicpuServerRole::SLAVE) {
247 1 : return AicpuKfcBatchwriteProcess::RunSlaveRpcServerForApi(ctx);
248 : }
249 4 : if (ctx->dfxExtendInfo.cqeStatus != dfx::CqeStatus::kDefault
250 4 : || ctx->dfxExtendInfo.pollStatus == PollStatus::kStopAsException) {
251 0 : HCCL_ERROR(
252 : "Exist errors before, cqeStatus:%d, pollStatus:%d, group[%s]", ctx->dfxExtendInfo.cqeStatus,
253 : ctx->dfxExtendInfo.pollStatus, contextParam->hcomId);
254 0 : return HCCL_E_INTERNAL;
255 : }
256 :
257 4 : Mc2InitTilingInner* tilingData = reinterpret_cast<Mc2InitTilingInner*>(args[desc->tilingOff]);
258 4 : ctx->debugMode = tilingData->debugMode;
259 4 : if (ctx->debugMode == MC2_DEBUG_ONLY_CUBE) {
260 1 : HCCL_INFO("[%s]DebugMode is set to be 1 (i.e. computation only).", __func__);
261 1 : return HCCL_SUCCESS;
262 : }
263 :
264 : HcclComSuspendingFlag kfcFlag;
265 3 : CHK_RET(AicpuHdcUtils::GetSuspendingStatus(ctx->kfcControlTransferH2D, kfcFlag));
266 3 : if (kfcFlag == HcclComSuspendingFlag::isSuspending) {
267 0 : HCCL_WARNING("[NsRecovery] the op should not be launched in the suspending status");
268 0 : return HCCL_SUCCESS;
269 : }
270 3 : AicpuUpdatComContextMumber(offsetof(AicpuComContext, endStopLaunch), false);
271 3 : AicpuUpdatComContextMumber(offsetof(AicpuComContext, isStopLaunch), false);
272 3 : AicpuSqeContext::SyncVariable();
273 3 : auto profInst = AicpuKfcProf::GetProInst(*ctx);
274 3 : if (AicpuKfcUtils::NeedRecordTimeTaken(*ctx)) {
275 2 : profInst.tid = syscall(__NR_gettid);
276 2 : profInst.clusterId = ctx->clusterId;
277 2 : profInst.rankId = ctx->rankId;
278 2 : profInst.launchEntryTime = launchEntryTime;
279 : }
280 :
281 3 : ctx->preparePosition = TASK_PREPARE_KERNEL;
282 3 : ctx->notifyOff = 0;
283 3 : ctx->notifyBeginCnt = 0;
284 3 : ctx->notifyEndCnt = 0;
285 3 : ctx->totalCnt = 0;
286 3 : u64 newAddr = ctx->workSpaceAddr;
287 3 : if (newAddr & 0x1ff) {
288 3 : newAddr = (newAddr & (~((uint64_t)0x1ff))) + 0x200;
289 3 : HCCL_INFO("Align hcclmsgarea from %p to %p", ctx->workSpaceAddr, newAddr);
290 : }
291 3 : ctx->workSpaceAddr = newAddr;
292 3 : ctx->commAlg = COMM_ALG_FULL_MESH;
293 3 : ctx->curTurnCnt = 0;
294 3 : profInst.workCnt = 0;
295 3 : ctx->msgPosForKernel = 0;
296 3 : ctx->curTurnCntForKernel = 0;
297 3 : ctx->totalTurnCntForKernel = 0;
298 3 : ctx->gatherOut = 0UL;
299 3 : AicpuKfcUtils::PrintTilingData(*tilingData);
300 3 : AicpuKfcUtils::PrintMC2AicpuContext(*ctx);
301 :
302 3 : CHK_RET(MC2TraceUtils::Submit(ctx)); // 上报ctx消息
303 :
304 3 : aicpuOpIdx++;
305 3 : if (ctx->debugMode == MC2_DEBUG_PRINT_MSG || ctx->debugMode == MC2_DEBUG_PRINT_BUFF) {
306 0 : HCCL_RUN_INFO("Server start, MC2 opIdx:%u", aicpuOpIdx);
307 : }
308 3 : HcclResult ret = AicpuKfcProcess::AicpuRunRpcServerForApi(ctx, reinterpret_cast<u64>(tilingData));
309 3 : if (ret == HCCL_E_SUSPENDING) {
310 0 : HCCL_INFO("mc2 opp is suspended");
311 0 : return AICPUSUSPENDING_ERROR;
312 3 : } else if (ret != HCCL_SUCCESS) {
313 0 : AicpuKfcUtils::PrintTilingData(*tilingData, true);
314 0 : AicpuKfcUtils::PrintMC2AicpuContext(*ctx, true);
315 0 : HCCL_ERROR("Server failed, MC2 opIdx:%u", aicpuOpIdx);
316 0 : return ret;
317 : }
318 3 : if (AicpuKfcUtils::NeedRecordTimeTaken(*ctx)) {
319 2 : profInst.endTime = GetCurCpuTimestamp(true);
320 : }
321 3 : CHK_RET(dfx::AicpuProfilingManager::ReportTaskExecTimeLine(&profInst));
322 3 : AicpuComContext* contextBase = nullptr;
323 3 : u32 contextNum = 0;
324 3 : AicpuGetAllComContext(contextBase, contextNum);
325 3 : AicpuKfcProf::OutputProfLog(
326 3 : AicpuKfcUtils::IsDebugModeEquals(*ctx, MC2_DEBUG_TIME_TAKEN), contextBase[0].acprof, contextBase[1].acprof);
327 3 : AicpuKfcProf::AddProfLoopCnt();
328 3 : AicpuSqeContext::SaveVariable();
329 3 : CHK_RET(AicpuSqeContext::ClearLocalBuff());
330 3 : HCCL_INFO("Kfc server ends successfully.");
331 3 : return HCCL_SUCCESS;
332 : }
333 :
334 6 : u32 RunAicpuApiRpcSrvLaunchV2(void* args[], CommKfcParamDesc* desc)
335 : {
336 : // MC2目前只支持OP_BASE
337 6 : SetWorkflowMode(HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE);
338 6 : u64 launchEntryTime = GetCurCpuTimestamp(true);
339 :
340 6 : const Mc2InitTilingInner* tilingData = static_cast<const Mc2InitTilingInner*>(args[desc->tilingOff]);
341 6 : AicpuKfcProf::SetDebugMode(tilingData->debugMode);
342 6 : if (AicpuKfcProf::IsDebugModeEquals(MC2_DEBUG_ONLY_CUBE)) {
343 1 : HCCL_INFO("[%s]DebugMode is set to be 1 (i.e. computation only).", __func__);
344 1 : return HCCL_SUCCESS;
345 : }
346 :
347 5 : KFCTaskV2 task{};
348 12 : for (uint64_t i = 0; i < desc->itemNum; i++) {
349 7 : u64 arg = reinterpret_cast<u64>(args[desc->hasFfts + i + 1]);
350 7 : if (arg != 0UL) {
351 6 : HCCL_INFO("Ctx idx %u, addr %#llx.", task.ctxNum, arg);
352 6 : task.context[task.ctxNum++] = arg;
353 : }
354 : }
355 5 : CHK_PRT_RET(
356 : task.ctxNum == 0 || task.ctxNum > MAX_COMM_CTX_NUM, HCCL_ERROR("Invalid ctx number %u.", task.ctxNum),
357 : HCCL_E_PARA);
358 5 : task.workSpace = 0;
359 5 : AicpuKfcProf::GetCurrentAicpuProf()->workCnt = 0;
360 5 : const u64 aicpuOpIdx = GetUpdatedOpIdx();
361 5 : if (AicpuKfcProf::IsDebugModeEquals(MC2_DEBUG_PRINT_MSG) || AicpuKfcProf::IsDebugModeEquals(MC2_DEBUG_PRINT_BUFF)) {
362 3 : HCCL_RUN_INFO("Server start, MC2 opIdx:%lu", aicpuOpIdx);
363 : }
364 5 : HCCL_INFO("Start %s, aicpuOpIdx %lu", __func__, aicpuOpIdx);
365 5 : const u32 ret = AicpuKfcProcess::AicpuRunRpcServerForMC2V2(&task, tilingData);
366 5 : if (ret != HCCL_SUCCESS) {
367 1 : AicpuKfcUtils::PrintTilingData(*tilingData, true);
368 1 : HCCL_ERROR("[%s] aicpuOpIdx %lu", __func__, aicpuOpIdx);
369 1 : return ret;
370 : }
371 :
372 4 : const u32 blockNum = HcclAicpuUtils::GetBlockNum();
373 4 : const u32 blockIdx = HcclAicpuUtils::GetBlockIdx();
374 4 : const u32 totalQueueNum = tilingData->commBlockNum * tilingData->queueNum;
375 4 : const u32 turnOffset = blockIdx * (totalQueueNum / blockNum) + std::min(blockIdx, totalQueueNum % blockNum);
376 4 : CHK_RET(KfcProf(launchEntryTime, task, turnOffset));
377 4 : HCCL_INFO("End %s, aicpuOpIdx %lu", __func__, aicpuOpIdx);
378 4 : return 0;
379 : }
380 :
381 12 : u32 RunKernelAicpuServerForTilingApi(void* args[], CommKfcParamDesc* desc)
382 : {
383 12 : if (AicpuHcclProcess::AicpuGetInnerDevType() == DevType::DEV_TYPE_910_93) {
384 6 : return RunAicpuApiRpcSrvLaunchV2(args, desc);
385 : } else {
386 6 : return RunAicpuApiRpcSrvLaunchV1(args, desc);
387 : }
388 : }
389 : } // namespace
390 :
391 : extern "C" {
392 81 : __attribute__((visibility("default"))) uint32_t RunAicpuKfcResInit(void* args)
393 : {
394 81 : if (args == nullptr) {
395 1 : HCCL_ERROR("args is null.");
396 1 : return HCCL_E_PARA;
397 : }
398 :
399 80 : KFCResInitTask* ctxArgs = static_cast<KFCResInitTask*>(args);
400 80 : return AicpuKfcProcess::AicpuRpcResInit(reinterpret_cast<HccCommResParamTask*>(ctxArgs->context));
401 : }
402 :
403 : // aclgraph 销毁触发的 tag 清理入口。host 端走 KFCResInitTask{context, isCustom} 模式投递，这里从 context 反解 payload
404 : // 交给 AicpuHcclProcess
405 0 : __attribute__((visibility("default"))) uint32_t RunAicpuKfcClearOpRes(void* args)
406 : {
407 0 : if (args == nullptr) {
408 0 : HCCL_ERROR("[RunAicpuKfcClearOpRes] args is null.");
409 0 : return HCCL_E_PARA;
410 : }
411 0 : KFCResInitTask* ctxArgs = static_cast<KFCResInitTask*>(args);
412 0 : if (ctxArgs->context == 0) {
413 0 : HCCL_ERROR("[RunAicpuKfcClearOpRes] ctxArgs->context is null.");
414 0 : return HCCL_E_PARA;
415 : }
416 0 : const HcclKfcClearOpResTilingData* tilingData
417 0 : = reinterpret_cast<const HcclKfcClearOpResTilingData*>(ctxArgs->context);
418 0 : return AicpuHcclProcess::AicpuRpcClearOpRes(tilingData);
419 : }
420 :
421 68 : __attribute__((visibility("default"))) uint32_t RunAicpuRpcSrvLaunch(void* args)
422 : {
423 68 : KfcState state;
424 : static uint32_t aicpuOpIdx = 0;
425 68 : u64 launchEntryTime = GetCurCpuTimestamp(true);
426 :
427 68 : if (args == nullptr) {
428 1 : HCCL_ERROR("args is null.");
429 1 : return HCCL_E_PARA;
430 : }
431 :
432 67 : KFCTask* task = reinterpret_cast<KFCTask*>(args);
433 67 : HCCL_INFO(
434 : "KFCTask inputA %p, outputC %p, commOut %p, context %p, workSpace %p, tilingData %p", task->inputA,
435 : task->outputC, task->commOut, task->context, task->workSpace, task->tilingData);
436 67 : HcclKFCTilingData* tilingData = reinterpret_cast<HcclKFCTilingData*>(task->tilingData);
437 67 : HccCommResParamTask* contextParam = reinterpret_cast<HccCommResParamTask*>(task->context);
438 67 : if (tilingData == nullptr || contextParam == nullptr) {
439 2 : HCCL_ERROR("tilingData or context args is null.");
440 2 : return HCCL_E_PARA;
441 : }
442 :
443 65 : AicpuComContext* ctx = AicpuGetComContext();
444 65 : if (ctx == nullptr || !ctx->alreadyInit || strcmp(ctx->hcomId, contextParam->hcomId) != 0) {
445 1 : HCCL_ERROR("The comm domain %s have not exist.", contextParam->hcomId);
446 1 : return HCCL_E_PARA;
447 : }
448 64 : if ((ctx->dfxExtendInfo.cqeStatus != dfx::CqeStatus::kDefault)
449 63 : || (ctx->dfxExtendInfo.pollStatus == PollStatus::kStopAsException)) {
450 1 : HCCL_ERROR(
451 : "Exist errors before, cqeStatus:%d, pollStatus:%d, group[%s]", ctx->dfxExtendInfo.cqeStatus,
452 : ctx->dfxExtendInfo.pollStatus, contextParam->hcomId);
453 1 : return HCCL_E_INTERNAL;
454 : }
455 63 : ctx->debugMode = tilingData->debugMode;
456 63 : if (ctx->debugMode == MC2_DEBUG_ONLY_CUBE) {
457 2 : HCCL_INFO("[%s]DebugMode is set to be 1 (i.e. computation only).", __func__);
458 2 : return HCCL_SUCCESS;
459 : }
460 61 : if (!tilingData->useBufferType) {
461 55 : ctx->gatherOut = task->commOut;
462 : } else {
463 6 : ctx->gatherOut = task->outputC;
464 : }
465 : // 这里就是判断Suspending通道的内容
466 61 : HCCL_DEBUG("[NsRecovery]check the suspending status");
467 61 : HcclComSuspendingFlag kfcFlag = HcclComSuspendingFlag ::isNull;
468 61 : CHK_RET(AicpuHdcUtils::GetSuspendingStatus(ctx->kfcControlTransferH2D, kfcFlag));
469 61 : if (kfcFlag == HcclComSuspendingFlag::isSuspending) {
470 0 : HCCL_WARNING("[NsRecovery] the op should not be launched in the suspending status");
471 0 : return 0;
472 : }
473 61 : AicpuUpdatComContextMumber(offsetof(AicpuComContext, endStopLaunch), false);
474 61 : AicpuUpdatComContextMumber(offsetof(AicpuComContext, isStopLaunch), false);
475 61 : AicpuSqeContext::SyncVariable();
476 61 : auto profInst = AicpuKfcProf::GetProInst(*ctx);
477 61 : if (AicpuKfcUtils::NeedRecordTimeTaken(*ctx)) {
478 5 : profInst.tid = syscall(__NR_gettid);
479 5 : profInst.clusterId = ctx->clusterId;
480 5 : profInst.rankId = ctx->rankId;
481 5 : profInst.launchEntryTime = launchEntryTime;
482 : }
483 61 : HCCL_INFO("RunAicpuRpcSrvLaunch, preparePosition %u", tilingData->preparePosition);
484 61 : if (tilingData->preparePosition > 1) {
485 1 : HCCL_ERROR("invalid preparePosition %u", tilingData->preparePosition);
486 1 : return HCCL_E_PARA;
487 : }
488 60 : ctx->preparePosition = static_cast<TASK_PREPARE_POSITION>(tilingData->preparePosition);
489 60 : if (ctx->preparePosition == TASK_PREPARE_HOST) {
490 37 : ctx->notifyOff = tilingData->notifyOff;
491 37 : ctx->notifyBeginCnt = tilingData->notifyBeginCnt;
492 37 : ctx->notifyEndCnt = tilingData->notifyEndCnt;
493 37 : ctx->totalCnt = tilingData->totalCnt;
494 : } else {
495 23 : ctx->notifyOff = 0;
496 23 : ctx->notifyBeginCnt = 0;
497 23 : ctx->notifyEndCnt = 0;
498 23 : ctx->totalCnt = 0;
499 23 : u64 newAddr = ctx->workSpaceAddr;
500 23 : if (newAddr & 0x1ff) {
501 22 : newAddr = (newAddr & (~((uint64_t)0x1ff))) + 0x200;
502 22 : HCCL_INFO("Align hcclmsgarea from %p to %p", ctx->workSpaceAddr, newAddr);
503 : }
504 23 : ctx->workSpaceAddr = newAddr;
505 : }
506 : tilingData->commAlg
507 60 : = (ctx->devType == DevType::DEV_TYPE_910B) ? static_cast<u8>(COMM_ALG_FULL_MESH) : tilingData->commAlg;
508 60 : ctx->commAlg = tilingData->commAlg;
509 60 : ctx->skipLocalDataCopy = tilingData->hasCommOut ? false : true;
510 60 : ctx->curTurnCnt = 0;
511 60 : profInst.workCnt = 0;
512 60 : ctx->msgPosForKernel = 0;
513 60 : ctx->curTurnCntForKernel = 0;
514 60 : ctx->sendCntRecord[0] = AicpuKfcUtils::GetSendCnt(ctx);
515 60 : ctx->recvCntRecord[0] = AicpuKfcUtils::GetRecvCnt(ctx);
516 60 : ctx->totalTurnCntForKernel = 0;
517 60 : AicpuKfcUtils::PrintTilingData(*tilingData);
518 60 : AicpuKfcUtils::PrintMC2AicpuContext(*ctx);
519 :
520 60 : CHK_RET(MC2TraceUtils::Submit(task, tilingData));
521 60 : CHK_RET(MC2TraceUtils::Submit(ctx)); // 上报ctx消息
522 :
523 60 : aicpuOpIdx++;
524 60 : if (ctx->debugMode == MC2_DEBUG_PRINT_MSG || ctx->debugMode == MC2_DEBUG_PRINT_BUFF) {
525 2 : HCCL_RUN_INFO("Server start, MC2 opIdx:%u", aicpuOpIdx);
526 : }
527 60 : auto ret = AicpuRunRpcServer(ctx, task);
528 60 : ctx->sendCntRecord[3] = AicpuKfcUtils::GetSendCnt(ctx); // 3 记录执行结束时的sendCnt
529 60 : ctx->recvCntRecord[3] = AicpuKfcUtils::GetRecvCnt(ctx); // 3 记录执行结束时的recvCnt
530 60 : if ((ret != HCCL_SUCCESS) && (ret != HCCL_E_SUSPENDING)) {
531 20 : AicpuKfcUtils::PrintTilingData(*tilingData, true);
532 20 : AicpuKfcUtils::PrintMC2AicpuContext(*ctx, true);
533 20 : if (ctx->preparePosition == TASK_PREPARE_HOST) { // host展开时aicore会通过workspace回传维测信息, 解析后打印
534 13 : HCCL_ERROR(
535 : "Run rpc error, opIdx:%u, sndCnt:%d %d %d %d, rcvCnt:%d %d %d %d", aicpuOpIdx, ctx->sendCntRecord[0],
536 : ctx->sendCntRecord[1], ctx->sendCntRecord[2], ctx->sendCntRecord[3], ctx->recvCntRecord[0],
537 : ctx->recvCntRecord[1], ctx->recvCntRecord[2], ctx->recvCntRecord[3]);
538 : }
539 20 : HCCL_ERROR("Failed to run aicpu server failed, MC2 opIdx:%u", aicpuOpIdx);
540 20 : return ret;
541 40 : } else if (ret == HCCL_E_SUSPENDING) {
542 1 : HCCL_INFO("mc2 opp is suspended");
543 1 : return AICPUSUSPENDING_ERROR;
544 : }
545 39 : if (AicpuKfcUtils::NeedRecordTimeTaken(*ctx)) {
546 4 : profInst.endTime = GetCurCpuTimestamp(true);
547 : }
548 39 : CHK_RET(dfx::AicpuProfilingManager::ReportTaskExecTimeLine(&profInst));
549 39 : AicpuComContext* contextBase = nullptr;
550 39 : u32 contextNum = 0;
551 39 : AicpuGetAllComContext(contextBase, contextNum);
552 39 : AicpuKfcProf::OutputProfLog(
553 39 : AicpuKfcUtils::IsDebugModeEquals(*ctx, MC2_DEBUG_TIME_TAKEN), contextBase[0].acprof, contextBase[1].acprof);
554 39 : AicpuKfcProf::AddProfLoopCnt();
555 39 : AicpuSqeContext::SaveVariable();
556 39 : CHK_RET(AicpuSqeContext::ClearLocalBuff());
557 39 : HCCL_INFO("end RunAicpuRpcSrvLaunch");
558 39 : return 0;
559 68 : }
560 :
561 4 : __attribute__((visibility("default"))) uint32_t RunAicpuRpcSrvGroupLaunch(void* args)
562 : {
563 4 : KfcState state;
564 4 : if (args == nullptr) {
565 1 : HCCL_ERROR("args is null.");
566 1 : return HCCL_E_PARA;
567 : }
568 :
569 3 : KFCTask* task = reinterpret_cast<KFCTask*>(args);
570 3 : HCCL_INFO(
571 : "KFCTask inputA %p, outputC %p, commOut %p, context %p, workSpace %p, tilingData %p", task->inputA,
572 : task->outputC, task->commOut, task->context, task->workSpace, task->tilingData);
573 3 : KFCGroupTilingData* tilingData = reinterpret_cast<KFCGroupTilingData*>(task->tilingData);
574 :
575 3 : if (tilingData == nullptr || tilingData->groupNum == 0) {
576 2 : HCCL_ERROR("tilingData is nullptr or groupNum is 0.");
577 2 : return HCCL_E_PARA;
578 : }
579 :
580 2 : for (uint32_t i = 0; i < tilingData->groupNum; ++i) {
581 : KFCTask singleTask;
582 1 : singleTask.inputA = task->inputA;
583 1 : singleTask.outputC = *reinterpret_cast<u64*>(task->outputC + sizeof(void*) * i);
584 1 : singleTask.commOut = task->commOut;
585 1 : singleTask.context = task->context;
586 1 : singleTask.workSpace = task->workSpace;
587 1 : singleTask.tilingData = reinterpret_cast<u64>(&tilingData->msg[i]);
588 1 : uint32_t ret = RunAicpuRpcSrvLaunch(&singleTask);
589 1 : if (ret != 0) {
590 0 : HCCL_ERROR("RunAicpuRpcSrvGroupLaunch runs failed.");
591 0 : return HCCL_E_PARA;
592 : }
593 : }
594 :
595 1 : return 0;
596 4 : }
597 :
598 : constexpr u32 GROUP_DYN_FLAG = 23U;
599 : constexpr u32 GROUP_TILING_MAGIC_NUM = 99U;
600 36 : __attribute__((visibility("default"))) uint32_t RunAicpuKfcSrvLaunch(void* args[])
601 : {
602 36 : if (args == nullptr) {
603 2 : HCCL_ERROR("args is null.");
604 2 : return HCCL_E_PARA;
605 : }
606 34 : constexpr int DESC_POS = 0;
607 34 : uint64_t desc_value = u64(args[DESC_POS]);
608 34 : uint64_t* desc_addr = &desc_value;
609 34 : CommKfcParamDesc* desc = reinterpret_cast<CommKfcParamDesc*>(desc_addr);
610 34 : AicpuKfcUtils::PrintHcclCommParamDesc(*desc);
611 34 : if (desc->version == DECOUPLED_CTX_VER) {
612 12 : return CommKfcDispatcher::Run(&(args[1]), desc->itemNum);
613 : }
614 22 : void* tiling = reinterpret_cast<void*>(args[desc->tilingOff]);
615 22 : if (tiling == nullptr) {
616 0 : HCCL_ERROR("tiling is null.");
617 0 : return HCCL_E_PARA;
618 : }
619 22 : KfcState state;
620 22 : bool profL1Open = dfx::ProfilingManager::IsProfL1On();
621 22 : bool profL0Open = dfx::ProfilingManager::IsProfL0On();
622 22 : HCCL_INFO("profL1Open:%d, profL0Open:%d", profL1Open, profL0Open);
623 22 : const uint32_t ver = MC2TilingGetVer(tiling);
624 22 : HCCL_INFO("Start RunAicpuKfcSrvLaunch with tiling version %u.", ver);
625 : uint32_t ret;
626 22 : switch (ver) {
627 4 : case TILING_DATA_VER_OLD_FOR_HOST: {
628 4 : KFCGroupTilingDataAuto* tilingData = static_cast<KFCGroupTilingDataAuto*>(tiling);
629 4 : if (desc->isDyn == GROUP_DYN_FLAG && tilingData->groupTilingMagicNum == GROUP_TILING_MAGIC_NUM) {
630 3 : ret = RunAicpuInnerRpcSrvGroupLaunch(args, tilingData, desc);
631 : } else {
632 1 : ret = RunKernelAicpuServerV1(args, desc);
633 : }
634 4 : break;
635 : }
636 1 : case TILING_DATA_VER_OLD_FOR_KERNEL:
637 1 : ret = RunKernelAicpuServerV1(args, desc);
638 1 : break;
639 4 : case TILING_DATA_VER_OLD_FOR_KERNEL_V2:
640 4 : ret = RunKernelAicpuServerV2(args, desc, tiling);
641 4 : break;
642 12 : case TILING_DATA_VER_FOR_TILING_API:
643 12 : ret = RunKernelAicpuServerForTilingApi(args, desc);
644 12 : break;
645 1 : default:
646 1 : HCCL_ERROR("Invalid tiling version %u.", ver);
647 1 : ret = HCCL_E_PARA;
648 : }
649 22 : return ret;
650 22 : }
651 : }
|
