Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "aicpu_kfc_interface.h"
12 :
13 : #include "common/aicpu_hccl_common.h"
14 : #include "common/aicpu_hccl_def.h"
15 : #include "common/aicpu_sqe_context.h"
16 : #include "dfx/mc2_trace_utils.h"
17 : #include "dfx/aicpu_profiling_manager.h"
18 : #include "framework/aicpu_hccl_process.h"
19 : #include "aicpu_kfc/framework/aicpu_kfc_process.h"
20 : #include "aicpu_kfc/decoupler/comm_kfc_dispatcher.h"
21 : #include "aicpu_kfc/framework/aicpu_kfc_deprecated_process.h"
22 : #include "aicpu_kfc/framework/aicpu_kfc_batchwrite_process.h"
23 : #include "utils/hccl_aicpu_utils.h"
24 : #include "common/aicpu_kfc_utils.h"
25 : #include "aicpu_kfc/common/aicpu_kfc_tiling_utils.h"
26 : #include "aicpu_kfc/framework/aicpu_kfc_prof.h"
27 : #include "utils/aicpu_hdc_utils.h"
28 :
29 : using namespace HcclApi;
30 : namespace {
31 2 : u64 GetTensorAddr(uint16_t index, uint8_t *tensorPtr) {
32 2 : uint64_t* dataAddr = reinterpret_cast<uint64_t*>(tensorPtr);
33 2 : uint64_t tensorPtrOffset = *dataAddr;
34 : // Moving 3 bits to the right means dividing by sizeof(uint64 t).
35 2 : uint64_t* resPtr = dataAddr + (tensorPtrOffset >> 3);
36 2 : return u64(*(resPtr + index));
37 : }
38 :
39 5 : u64 GetUpdatedOpIdx()
40 : {
41 : static uint64_t aicpuOpIdx[MAX_AICPU_NUM_BLOCKS] = {0UL};
42 5 : const u32 blockNum = HcclAicpuUtils::GetBlockNum();
43 5 : const u32 blockIdx = HcclAicpuUtils::GetBlockIdx();
44 5 : ++aicpuOpIdx[blockIdx];
45 5 : if (blockIdx == blockNum - 1U) {
46 30 : for (u32 i = blockIdx + 1U; i < MAX_AICPU_NUM_BLOCKS; ++i) {
47 25 : ++aicpuOpIdx[i];
48 : }
49 : }
50 5 : return aicpuOpIdx[blockIdx];
51 : }
52 :
53 60 : HcclResult AicpuRunRpcServer(AicpuComContext *ctx, KFCTask *taskInfo)
54 : {
55 : // 启动RPC服务
56 60 : static AicpuKfcRpcServer rpc;
57 60 : rpc.Init(ctx->workSpaceAddr, ctx->notifyOff, ctx->notifyBeginCnt, taskInfo);
58 60 : AicpuKfcProf::GetProInst(*ctx).commInitEndTime = GetCurCpuTimestamp(true);
59 60 : if (rpc.GetPreparePosition() == TASK_PREPARE_KERNEL) {
60 23 : auto ret = AicpuKfcProcess::RunRpcServerApi(ctx, rpc);
61 23 : AicpuUpdatComContextMumber(offsetof(AicpuComContext, isOpLaunch), false);
62 23 : if (ret != HCCL_SUCCESS) {
63 8 : return AicpuKfcProcess::DealReturnValue(ctx, ret);
64 : } else {
65 15 : CHK_RET(AicpuHdcUtils::SetOpExecStatus(ctx->kfcStatusTransferD2H, KfcStatus::kEnd, KfcError::kNone, 0));
66 15 : return ret;
67 : }
68 : }
69 :
70 37 : if (rpc.GetTaskType() == HCCL_KFC_TASK_HCCL_ONLY_EXE) {
71 15 : auto ret = AicpuKfcDeprecatedProcess::AICPU_RpcServerUnfoldStageWait(ctx, rpc);
72 15 : if ((ret != HCCL_SUCCESS) && (ret != HCCL_E_SUSPENDING)) {
73 11 : HCCL_ERROR("AicpuRpcStageWait failed, commType:%d, reducekind:%d, totalCnt:%lu, totalTurnCnt:%u",
74 : ctx->commType, ctx->reducekind, ctx->totalCnt, ctx->totalTurnCnt);
75 11 : return ret;
76 : }
77 4 : if (ret == HCCL_E_SUSPENDING) {
78 0 : HCCL_RUN_INFO("[NsRecovery][AICPU] Suspending");
79 0 : return ret;
80 : }
81 22 : } else if (ctx->devType == DevType::DEV_TYPE_310P1 || ctx->devType == DevType::DEV_TYPE_310P3) {
82 1 : auto ret = AicpuKfcDeprecatedProcess::RunRpcServerTwoStageWait(ctx, rpc);
83 1 : if ((ret != HCCL_SUCCESS) && (ret != HCCL_E_SUSPENDING)) {
84 1 : return ret;
85 : }
86 0 : if (ret == HCCL_E_SUSPENDING) {
87 0 : HCCL_RUN_INFO("[NsRecovery][MC2] Suspending");
88 0 : return ret;
89 : }
90 0 : } else {
91 21 : auto ret = AicpuKfcDeprecatedProcess::TryRunRpcServerOneStageWait(ctx, rpc);
92 21 : if ((ret != HCCL_SUCCESS) && (ret != HCCL_E_SUSPENDING)) {
93 1 : return ret;
94 : }
95 20 : if (ret == HCCL_E_SUSPENDING) {
96 0 : HCCL_RUN_INFO("[NsRecovery][MC2] Suspending");
97 0 : return ret;
98 : }
99 : }
100 :
101 24 : return HCCL_SUCCESS;
102 : }
103 :
104 3 : u32 RunAicpuInnerRpcSrvGroupLaunch(void *args[], KFCGroupTilingDataAuto *tilingData, CommKfcParamDesc* desc)
105 : {
106 3 : constexpr int DESC_POS = 0;
107 :
108 3 : if (tilingData == nullptr || tilingData->groupNum == 0) {
109 1 : HCCL_ERROR("tilingData is nullptr or groupNum is 0.");
110 1 : return HCCL_E_PARA;
111 : }
112 3 : for (uint32_t i = 0; i < tilingData->groupNum; ++i) {
113 : KFCTask singleTask;
114 2 : singleTask.inputA = u64(args[tilingData->msg[i].sendArgIndex + desc->hasFfts + desc->itemNum + 1]);
115 4 : singleTask.outputC = GetTensorAddr(i,
116 2 : reinterpret_cast<uint8_t*>(args[tilingData->msg[i].recvArgIndex + desc->hasFfts + desc->itemNum + 1]));
117 2 : singleTask.commOut = 0;
118 2 : singleTask.context = u64(args[DESC_POS + desc->hasFfts + desc->itemNum]);
119 2 : singleTask.workSpace = u64(args[desc->tilingOff - 1]);
120 2 : singleTask.tilingData = u64(&tilingData->msg[i]);
121 2 : uint32_t ret = RunAicpuRpcSrvLaunch(&singleTask);
122 2 : if (ret != 0) {
123 1 : HCCL_ERROR("RunAicpuRpcSrvGroupLaunch runs failed.");
124 1 : return HCCL_E_PARA;
125 : }
126 : }
127 1 : return 0;
128 : }
129 :
130 2 : u32 RunKernelAicpuServerV1(void *args[], CommKfcParamDesc *desc)
131 : {
132 2 : HcclKFCTilingData *tilingData = static_cast<HcclKFCTilingData *>(args[desc->tilingOff]);
133 2 : HCCL_INFO("RunAicpuKfcSrvLaunch, tiling.sendArgIndex %lu", tilingData->sendArgIndex);
134 2 : HCCL_INFO("RunAicpuKfcSrvLaunch, tiling.recvArgIndex %lu", tilingData->recvArgIndex);
135 2 : HCCL_INFO("RunAicpuKfcSrvLaunch, tiling.commOutArgIndex %lu", tilingData->commOutArgIndex);
136 2 : HCCL_INFO("RunAicpuKfcSrvLaunch, tiling.hasCommOut %lu", tilingData->hasCommOut);
137 : KFCTask task;
138 2 : task.tilingData = u64(tilingData);
139 2 : task.inputA = u64(args[tilingData->sendArgIndex + desc->hasFfts + desc->itemNum + 1]);
140 2 : task.outputC = u64(args[tilingData->recvArgIndex + desc->hasFfts + desc->itemNum + 1]);
141 2 : task.context = u64(args[desc->hasFfts + desc->itemNum]);
142 2 : task.workSpace = u64(args[desc->tilingOff - 1]);
143 2 : if (tilingData->commOutArgIndex != u64(0xff)) {
144 2 : task.commOut = u64(args[tilingData->commOutArgIndex + desc->hasFfts + desc->itemNum + 1]);
145 : } else {
146 0 : task.commOut = 0;
147 : }
148 2 : AicpuKfcUtils::PrintKFCTask(task);
149 2 : HCCL_INFO("Task Assembled. Start to launch RunAicpuRpcSrvLaunch");
150 2 : const uint32_t ret = RunAicpuRpcSrvLaunch(&task);
151 2 : HCCL_INFO("RunAicpuKfcSrvLaunch ends with result %lu.", ret);
152 2 : return ret;
153 : }
154 :
155 6 : HcclResult KfcProf(u64 launchEntryTime, KFCTaskV2 &task, u32 turnOffset = 0U)
156 : {
157 6 : if (AicpuKfcProf::NeedRecordTimeTaken()) {
158 2 : AicpuKfcProf::SetCurrentProf(launchEntryTime);
159 2 : HcclOpResParam *commParam = reinterpret_cast<HcclOpResParam *>(task.context[0]);
160 2 : AicpuKfcProf::GetCurrentAicpuProf()->rankId = commParam->topoInfo.userRank;
161 2 : AicpuKfcProf::GetCurrentAicpuProf()->endTime = GetCurCpuTimestamp(true);
162 : }
163 :
164 6 : CHK_RET(dfx::AicpuProfilingManager::ReportTaskExecTimeLine(AicpuKfcProf::GetCurrentAicpuProf(), turnOffset));
165 6 : AicpuKfcProf::OutputProfLog(AicpuKfcProf::IsDebugModeEquals(MC2_DEBUG_TIME_TAKEN),
166 : AicpuKfcProf::GetaicpuProfInst());
167 6 : AicpuKfcProf::AddProfLoopCnt();
168 6 : return HCCL_SUCCESS;
169 : }
170 :
171 4 : u32 RunKernelAicpuServerV2(void *args[], CommKfcParamDesc *desc, void *tilingData)
172 : {
173 4 : u64 launchEntryTime = GetCurCpuTimestamp(true);
174 : // MC2目前只支持OP_BASE
175 4 : SetWorkflowMode(HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE);
176 : static uint64_t aicpuOpIdx = 0;
177 :
178 4 : Mc2ServerCfg *cfg = MC2TilingGetServerCfg(tilingData);
179 4 : AicpuKfcProf::SetDebugMode(cfg->debugMode);
180 4 : if (AicpuKfcProf::IsDebugModeEquals(MC2_DEBUG_ONLY_CUBE)) {
181 1 : HCCL_INFO("[%s]DebugMode is set to be 1 (i.e. computation only).", __func__);
182 1 : return HCCL_SUCCESS;
183 : }
184 :
185 : KFCTaskV2 task;
186 3 : task.tilingData = reinterpret_cast<u64>(tilingData);
187 3 : task.ctxNum = desc->itemNum;
188 3 : if (task.ctxNum > MAX_COMM_CTX_NUM) {
189 0 : HCCL_ERROR("group num must be smaller than %u.", MAX_COMM_CTX_NUM);
190 0 : return HCCL_E_PARA;
191 : }
192 7 : for (int i = 0; i < desc->itemNum; i++) {
193 5 : task.context[i] = reinterpret_cast<u64>(args[desc->hasFfts + i + 1]);
194 5 : if (task.context[i] == 0) {
195 1 : HCCL_ERROR("idx %d ctx is null, please check the input ctx.", i);
196 1 : return HCCL_E_PARA;
197 : }
198 4 : HcclAicpuUtils::PrintHcclOpResParam(reinterpret_cast<HcclOpResParam *>(task.context[i]));
199 : }
200 2 : task.workSpace = reinterpret_cast<u64>(args[desc->tilingOff - 1]);
201 2 : aicpuOpIdx++;
202 2 : if (AicpuKfcProf::IsDebugModeEquals(MC2_DEBUG_PRINT_MSG) || AicpuKfcProf::IsDebugModeEquals(MC2_DEBUG_PRINT_BUFF)) {
203 1 : HCCL_RUN_INFO("Server start, MC2 opIdx:%lu", aicpuOpIdx);
204 : }
205 2 : HCCL_INFO("Start launch RunAicpuInnerKfcSrvLaunch");
206 2 : AicpuKfcProf::GetCurrentAicpuProf()->workCnt = 0;
207 2 : const u32 ret = AicpuKfcProcess::AicpuRunRpcServerForMC2(&task);
208 2 : if (ret != HCCL_SUCCESS) {
209 0 : HCCL_ERROR("Server runs failed, error code %u.", ret);
210 0 : return ret;
211 : }
212 2 : CHK_RET(KfcProf(launchEntryTime, task));
213 2 : HCCL_INFO("end kfc server.");
214 2 : return 0;
215 : }
216 :
217 6 : u32 RunAicpuApiRpcSrvLaunchV1(void *args[], CommKfcParamDesc *desc)
218 : {
219 : static u32 aicpuOpIdx = 0;
220 6 : u64 launchEntryTime = GetCurCpuTimestamp(true);
221 :
222 6 : HccCommResParamTask *contextParam = nullptr;
223 6 : for (uint64_t i = 1; i <= desc->itemNum; ++i) {
224 6 : contextParam = reinterpret_cast<HccCommResParamTask *>(args[desc->hasFfts + i]);
225 6 : if (contextParam != nullptr) {
226 6 : HCCL_INFO("Idx %llu ctx addr %p.", i, contextParam);
227 6 : break;
228 : }
229 : }
230 6 : if (contextParam == nullptr) {
231 0 : HCCL_ERROR("Context args is null.");
232 0 : return HCCL_E_PARA;
233 : }
234 :
235 6 : AicpuComContext *ctx = AicpuGetComContext();
236 6 : if (ctx == nullptr || !ctx->alreadyInit || strcmp(ctx->hcomId, contextParam->hcomId) != 0) {
237 0 : HCCL_ERROR("The comm domain %s have not exist.", contextParam->hcomId);
238 0 : return HCCL_E_PARA;
239 : }
240 6 : AicpuServerRole role = AicpuKfcBatchwriteProcess::GetVerifiedServerRole(*ctx);
241 6 : if (role == AicpuServerRole::INVALID) {
242 1 : HCCL_INFO("aicpu server role is invalid, return");
243 1 : return HCCL_SUCCESS;
244 5 : } else if (role == AicpuServerRole::SLAVE) {
245 1 : return AicpuKfcBatchwriteProcess::RunSlaveRpcServerForApi(ctx);
246 : }
247 4 : if (ctx->dfxExtendInfo.cqeStatus != dfx::CqeStatus::kDefault ||
248 4 : ctx->dfxExtendInfo.pollStatus == PollStatus::kStopAsException) {
249 0 : HCCL_ERROR("Exist errors before, cqeStatus:%d, pollStatus:%d, group[%s]", ctx->dfxExtendInfo.cqeStatus,
250 : ctx->dfxExtendInfo.pollStatus, contextParam->hcomId);
251 0 : return HCCL_E_INTERNAL;
252 : }
253 :
254 4 : Mc2InitTilingInner *tilingData = reinterpret_cast<Mc2InitTilingInner *>(args[desc->tilingOff]);
255 4 : ctx->debugMode = tilingData->debugMode;
256 4 : if (ctx->debugMode == MC2_DEBUG_ONLY_CUBE) {
257 1 : HCCL_INFO("[%s]DebugMode is set to be 1 (i.e. computation only).", __func__);
258 1 : return HCCL_SUCCESS;
259 : }
260 :
261 : HcclComSuspendingFlag kfcFlag;
262 3 : CHK_RET(AicpuHdcUtils::GetSuspendingStatus(ctx->kfcControlTransferH2D, kfcFlag));
263 3 : if (kfcFlag == HcclComSuspendingFlag::isSuspending) {
264 0 : HCCL_WARNING("[NsRecovery] the op should not be launched in the suspending status");
265 0 : return HCCL_SUCCESS;
266 : }
267 3 : AicpuUpdatComContextMumber(offsetof(AicpuComContext, endStopLaunch), false);
268 3 : AicpuUpdatComContextMumber(offsetof(AicpuComContext, isStopLaunch), false);
269 3 : AicpuSqeContext::SyncVariable();
270 3 : auto profInst = AicpuKfcProf::GetProInst(*ctx);
271 3 : if (AicpuKfcUtils::NeedRecordTimeTaken(*ctx)) {
272 2 : profInst.tid = syscall(__NR_gettid);
273 2 : profInst.clusterId = ctx->clusterId;
274 2 : profInst.rankId = ctx->rankId;
275 2 : profInst.launchEntryTime = launchEntryTime;
276 : }
277 :
278 3 : ctx->preparePosition = TASK_PREPARE_KERNEL;
279 3 : ctx->notifyOff = 0;
280 3 : ctx->notifyBeginCnt = 0;
281 3 : ctx->notifyEndCnt = 0;
282 3 : ctx->totalCnt = 0;
283 3 : u64 newAddr = ctx->workSpaceAddr;
284 3 : if (newAddr & 0x1ff) {
285 3 : newAddr = (newAddr & (~((uint64_t)0x1ff))) + 0x200;
286 3 : HCCL_INFO("Align hcclmsgarea from %p to %p", ctx->workSpaceAddr, newAddr);
287 : }
288 3 : ctx->workSpaceAddr = newAddr;
289 3 : ctx->commAlg = COMM_ALG_FULL_MESH;
290 3 : ctx->curTurnCnt = 0;
291 3 : profInst.workCnt = 0;
292 3 : ctx->msgPosForKernel = 0;
293 3 : ctx->curTurnCntForKernel = 0;
294 3 : ctx->totalTurnCntForKernel = 0;
295 3 : ctx->gatherOut = 0UL;
296 3 : AicpuKfcUtils::PrintTilingData(*tilingData);
297 3 : AicpuKfcUtils::PrintMC2AicpuContext(*ctx);
298 :
299 3 : CHK_RET(MC2TraceUtils::Submit(ctx)); // 上报ctx消息
300 :
301 3 : aicpuOpIdx++;
302 3 : if (ctx->debugMode == MC2_DEBUG_PRINT_MSG || ctx->debugMode == MC2_DEBUG_PRINT_BUFF) {
303 0 : HCCL_RUN_INFO("Server start, MC2 opIdx:%u", aicpuOpIdx);
304 : }
305 3 : HcclResult ret = AicpuKfcProcess::AicpuRunRpcServerForApi(ctx, reinterpret_cast<u64>(tilingData));
306 3 : if (ret == HCCL_E_SUSPENDING) {
307 0 : HCCL_INFO("mc2 opp is suspended");
308 0 : return AICPUSUSPENDING_ERROR;
309 3 : } else if (ret != HCCL_SUCCESS) {
310 0 : AicpuKfcUtils::PrintTilingData(*tilingData, true);
311 0 : AicpuKfcUtils::PrintMC2AicpuContext(*ctx, true);
312 0 : HCCL_ERROR("Server failed, MC2 opIdx:%u", aicpuOpIdx);
313 0 : return ret;
314 : }
315 3 : if (AicpuKfcUtils::NeedRecordTimeTaken(*ctx)) {
316 2 : profInst.endTime = GetCurCpuTimestamp(true);
317 : }
318 3 : CHK_RET(dfx::AicpuProfilingManager::ReportTaskExecTimeLine(&profInst));
319 3 : AicpuComContext *contextBase = nullptr;
320 3 : u32 contextNum = 0;
321 3 : AicpuGetAllComContext(contextBase, contextNum);
322 3 : AicpuKfcProf::OutputProfLog(AicpuKfcUtils::IsDebugModeEquals(*ctx, MC2_DEBUG_TIME_TAKEN), contextBase[0].acprof,
323 3 : contextBase[1].acprof);
324 3 : AicpuKfcProf::AddProfLoopCnt();
325 3 : AicpuSqeContext::SaveVariable();
326 3 : CHK_RET(AicpuSqeContext::ClearLocalBuff());
327 3 : HCCL_INFO("Kfc server ends successfully.");
328 3 : return HCCL_SUCCESS;
329 : }
330 :
331 6 : u32 RunAicpuApiRpcSrvLaunchV2(void *args[], CommKfcParamDesc *desc)
332 : {
333 : // MC2目前只支持OP_BASE
334 6 : SetWorkflowMode(HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE);
335 6 : u64 launchEntryTime = GetCurCpuTimestamp(true);
336 :
337 6 : const Mc2InitTilingInner *tilingData = static_cast<const Mc2InitTilingInner *>(args[desc->tilingOff]);
338 6 : AicpuKfcProf::SetDebugMode(tilingData->debugMode);
339 6 : if (AicpuKfcProf::IsDebugModeEquals(MC2_DEBUG_ONLY_CUBE)) {
340 1 : HCCL_INFO("[%s]DebugMode is set to be 1 (i.e. computation only).", __func__);
341 1 : return HCCL_SUCCESS;
342 : }
343 :
344 5 : KFCTaskV2 task{};
345 12 : for (uint64_t i = 0; i < desc->itemNum; i++) {
346 7 : u64 arg = reinterpret_cast<u64>(args[desc->hasFfts + i + 1]);
347 7 : if (arg != 0UL) {
348 6 : HCCL_INFO("Ctx idx %u, addr %#llx.", task.ctxNum, arg);
349 6 : task.context[task.ctxNum++] = arg;
350 : }
351 : }
352 5 : CHK_PRT_RET(task.ctxNum == 0 || task.ctxNum > MAX_COMM_CTX_NUM, HCCL_ERROR("Invalid ctx number %u.", task.ctxNum),
353 : HCCL_E_PARA);
354 5 : task.workSpace = 0;
355 5 : AicpuKfcProf::GetCurrentAicpuProf()->workCnt = 0;
356 5 : const u64 aicpuOpIdx = GetUpdatedOpIdx();
357 5 : if (AicpuKfcProf::IsDebugModeEquals(MC2_DEBUG_PRINT_MSG) || AicpuKfcProf::IsDebugModeEquals(MC2_DEBUG_PRINT_BUFF)) {
358 3 : HCCL_RUN_INFO("Server start, MC2 opIdx:%lu", aicpuOpIdx);
359 : }
360 5 : HCCL_INFO("Start %s, aicpuOpIdx %lu", __func__, aicpuOpIdx);
361 5 : const u32 ret = AicpuKfcProcess::AicpuRunRpcServerForMC2V2(&task, tilingData);
362 5 : if (ret != HCCL_SUCCESS) {
363 1 : AicpuKfcUtils::PrintTilingData(*tilingData, true);
364 1 : HCCL_ERROR("[%s] aicpuOpIdx %lu", __func__, aicpuOpIdx);
365 1 : return ret;
366 : }
367 :
368 4 : const u32 blockNum = HcclAicpuUtils::GetBlockNum();
369 4 : const u32 blockIdx = HcclAicpuUtils::GetBlockIdx();
370 4 : const u32 totalQueueNum = tilingData->commBlockNum * tilingData->queueNum;
371 4 : const u32 turnOffset = blockIdx * (totalQueueNum / blockNum) + std::min(blockIdx, totalQueueNum % blockNum);
372 4 : CHK_RET(KfcProf(launchEntryTime, task, turnOffset));
373 4 : HCCL_INFO("End %s, aicpuOpIdx %lu", __func__, aicpuOpIdx);
374 4 : return 0;
375 : }
376 :
377 12 : u32 RunKernelAicpuServerForTilingApi(void *args[], CommKfcParamDesc* desc)
378 : {
379 12 : if (AicpuHcclProcess::AicpuGetInnerDevType() == DevType::DEV_TYPE_910_93) {
380 6 : return RunAicpuApiRpcSrvLaunchV2(args, desc);
381 : } else {
382 6 : return RunAicpuApiRpcSrvLaunchV1(args, desc);
383 : }
384 : }
385 : }
386 :
387 : extern "C" {
388 81 : __attribute__((visibility("default"))) uint32_t RunAicpuKfcResInit(void *args) {
389 81 : if (args == nullptr) {
390 1 : HCCL_ERROR("args is null.");
391 1 : return HCCL_E_PARA;
392 : }
393 :
394 80 : KFCResInitTask *ctxArgs = static_cast<KFCResInitTask *>(args);
395 80 : return AicpuKfcProcess::AicpuRpcResInit(reinterpret_cast<HccCommResParamTask *>(ctxArgs->context));
396 : }
397 :
398 : // aclgraph 销毁触发的 tag 清理入口。host 端走 KFCResInitTask{context, isCustom} 模式投递，这里从 context 反解 payload 交给 AicpuHcclProcess
399 0 : __attribute__((visibility("default"))) uint32_t RunAicpuKfcClearOpRes(void *args) {
400 0 : if (args == nullptr) {
401 0 : HCCL_ERROR("[RunAicpuKfcClearOpRes] args is null.");
402 0 : return HCCL_E_PARA;
403 : }
404 0 : KFCResInitTask *ctxArgs = static_cast<KFCResInitTask *>(args);
405 0 : if (ctxArgs->context == 0) {
406 0 : HCCL_ERROR("[RunAicpuKfcClearOpRes] ctxArgs->context is null.");
407 0 : return HCCL_E_PARA;
408 : }
409 0 : const HcclKfcClearOpResTilingData *tilingData =
410 0 : reinterpret_cast<const HcclKfcClearOpResTilingData *>(ctxArgs->context);
411 0 : return AicpuHcclProcess::AicpuRpcClearOpRes(tilingData);
412 : }
413 :
414 68 : __attribute__((visibility("default"))) uint32_t RunAicpuRpcSrvLaunch(void *args)
415 : {
416 68 : KfcState state;
417 : static uint32_t aicpuOpIdx = 0;
418 68 : u64 launchEntryTime = GetCurCpuTimestamp(true);
419 :
420 68 : if (args == nullptr) {
421 1 : HCCL_ERROR("args is null.");
422 1 : return HCCL_E_PARA;
423 : }
424 :
425 67 : KFCTask *task = reinterpret_cast<KFCTask *>(args);
426 67 : HCCL_INFO("KFCTask inputA %p, outputC %p, commOut %p, context %p, workSpace %p, tilingData %p",
427 : task->inputA, task->outputC, task->commOut, task->context, task->workSpace, task->tilingData);
428 67 : HcclKFCTilingData *tilingData = reinterpret_cast<HcclKFCTilingData *>(task->tilingData);
429 67 : HccCommResParamTask *contextParam = reinterpret_cast<HccCommResParamTask *>(task->context);
430 67 : if (tilingData == nullptr || contextParam == nullptr) {
431 2 : HCCL_ERROR("tilingData or context args is null.");
432 2 : return HCCL_E_PARA;
433 : }
434 :
435 65 : AicpuComContext *ctx = AicpuGetComContext();
436 65 : if (ctx == nullptr || !ctx->alreadyInit || strcmp(ctx->hcomId, contextParam->hcomId) != 0) {
437 1 : HCCL_ERROR("The comm domain %s have not exist.", contextParam->hcomId);
438 1 : return HCCL_E_PARA;
439 : }
440 64 : if ((ctx->dfxExtendInfo.cqeStatus != dfx::CqeStatus::kDefault) ||
441 63 : (ctx->dfxExtendInfo.pollStatus == PollStatus::kStopAsException)) {
442 1 : HCCL_ERROR("Exist errors before, cqeStatus:%d, pollStatus:%d, group[%s]", ctx->dfxExtendInfo.cqeStatus,
443 : ctx->dfxExtendInfo.pollStatus, contextParam->hcomId);
444 1 : return HCCL_E_INTERNAL;
445 : }
446 63 : ctx->debugMode = tilingData->debugMode;
447 63 : if (ctx->debugMode == MC2_DEBUG_ONLY_CUBE) {
448 2 : HCCL_INFO("[%s]DebugMode is set to be 1 (i.e. computation only).", __func__);
449 2 : return HCCL_SUCCESS;
450 : }
451 61 : if (!tilingData->useBufferType) {
452 55 : ctx->gatherOut = task->commOut;
453 : } else {
454 6 : ctx->gatherOut = task->outputC;
455 : }
456 : // 这里就是判断Suspending通道的内容
457 61 : HCCL_DEBUG("[NsRecovery]check the suspending status");
458 61 : HcclComSuspendingFlag kfcFlag = HcclComSuspendingFlag ::isNull;
459 61 : CHK_RET(AicpuHdcUtils::GetSuspendingStatus(ctx->kfcControlTransferH2D, kfcFlag));
460 61 : if (kfcFlag == HcclComSuspendingFlag::isSuspending) {
461 0 : HCCL_WARNING("[NsRecovery] the op should not be launched in the suspending status");
462 0 : return 0;
463 : }
464 61 : AicpuUpdatComContextMumber(offsetof(AicpuComContext, endStopLaunch), false);
465 61 : AicpuUpdatComContextMumber(offsetof(AicpuComContext, isStopLaunch), false);
466 61 : AicpuSqeContext::SyncVariable();
467 61 : auto profInst = AicpuKfcProf::GetProInst(*ctx);
468 61 : if (AicpuKfcUtils::NeedRecordTimeTaken(*ctx)) {
469 5 : profInst.tid = syscall(__NR_gettid);
470 5 : profInst.clusterId = ctx->clusterId;
471 5 : profInst.rankId = ctx->rankId;
472 5 : profInst.launchEntryTime = launchEntryTime;
473 : }
474 61 : HCCL_INFO("RunAicpuRpcSrvLaunch, preparePosition %u", tilingData->preparePosition);
475 61 : if (tilingData->preparePosition > 1) {
476 1 : HCCL_ERROR("invalid preparePosition %u", tilingData->preparePosition);
477 1 : return HCCL_E_PARA;
478 : }
479 60 : ctx->preparePosition = static_cast<TASK_PREPARE_POSITION>(tilingData->preparePosition);
480 60 : if (ctx->preparePosition == TASK_PREPARE_HOST) {
481 37 : ctx->notifyOff = tilingData->notifyOff;
482 37 : ctx->notifyBeginCnt = tilingData->notifyBeginCnt;
483 37 : ctx->notifyEndCnt = tilingData->notifyEndCnt;
484 37 : ctx->totalCnt = tilingData->totalCnt;
485 : } else {
486 23 : ctx->notifyOff = 0;
487 23 : ctx->notifyBeginCnt = 0;
488 23 : ctx->notifyEndCnt = 0;
489 23 : ctx->totalCnt = 0;
490 23 : u64 newAddr = ctx->workSpaceAddr;
491 23 : if (newAddr & 0x1ff) {
492 22 : newAddr = (newAddr & (~((uint64_t)0x1ff))) + 0x200;
493 22 : HCCL_INFO("Align hcclmsgarea from %p to %p", ctx->workSpaceAddr, newAddr);
494 : }
495 23 : ctx->workSpaceAddr = newAddr;
496 : }
497 60 : tilingData->commAlg = (ctx->devType == DevType::DEV_TYPE_910B) ? static_cast<u8>(COMM_ALG_FULL_MESH) : tilingData->commAlg;
498 60 : ctx->commAlg = tilingData->commAlg;
499 60 : ctx->skipLocalDataCopy = tilingData->hasCommOut ? false : true;
500 60 : ctx->curTurnCnt = 0;
501 60 : profInst.workCnt = 0;
502 60 : ctx->msgPosForKernel = 0;
503 60 : ctx->curTurnCntForKernel = 0;
504 60 : ctx->sendCntRecord[0] = AicpuKfcUtils::GetSendCnt(ctx);
505 60 : ctx->recvCntRecord[0] = AicpuKfcUtils::GetRecvCnt(ctx);
506 60 : ctx->totalTurnCntForKernel = 0;
507 60 : AicpuKfcUtils::PrintTilingData(*tilingData);
508 60 : AicpuKfcUtils::PrintMC2AicpuContext(*ctx);
509 :
510 60 : CHK_RET(MC2TraceUtils::Submit(task, tilingData));
511 60 : CHK_RET(MC2TraceUtils::Submit(ctx)); // 上报ctx消息
512 :
513 60 : aicpuOpIdx++;
514 60 : if (ctx->debugMode == MC2_DEBUG_PRINT_MSG || ctx->debugMode == MC2_DEBUG_PRINT_BUFF) {
515 2 : HCCL_RUN_INFO("Server start, MC2 opIdx:%u", aicpuOpIdx);
516 : }
517 60 : auto ret = AicpuRunRpcServer(ctx, task);
518 60 : ctx->sendCntRecord[3] = AicpuKfcUtils::GetSendCnt(ctx); // 3 记录执行结束时的sendCnt
519 60 : ctx->recvCntRecord[3] = AicpuKfcUtils::GetRecvCnt(ctx); // 3 记录执行结束时的recvCnt
520 60 : if ((ret != HCCL_SUCCESS) && (ret != HCCL_E_SUSPENDING)) {
521 20 : AicpuKfcUtils::PrintTilingData(*tilingData, true);
522 20 : AicpuKfcUtils::PrintMC2AicpuContext(*ctx, true);
523 20 : if (ctx->preparePosition == TASK_PREPARE_HOST) { // host展开时aicore会通过workspace回传维测信息, 解析后打印
524 13 : HCCL_ERROR("Run rpc error, opIdx:%u, sndCnt:%d %d %d %d, rcvCnt:%d %d %d %d", aicpuOpIdx,
525 : ctx->sendCntRecord[0], ctx->sendCntRecord[1], ctx->sendCntRecord[2], ctx->sendCntRecord[3],
526 : ctx->recvCntRecord[0], ctx->recvCntRecord[1], ctx->recvCntRecord[2], ctx->recvCntRecord[3]);
527 : }
528 20 : HCCL_ERROR("Failed to run aicpu server failed, MC2 opIdx:%u", aicpuOpIdx);
529 20 : return ret;
530 40 : } else if (ret == HCCL_E_SUSPENDING) {
531 1 : HCCL_INFO("mc2 opp is suspended");
532 1 : return AICPUSUSPENDING_ERROR;
533 : }
534 39 : if (AicpuKfcUtils::NeedRecordTimeTaken(*ctx)) {
535 4 : profInst.endTime = GetCurCpuTimestamp(true);
536 : }
537 39 : CHK_RET(dfx::AicpuProfilingManager::ReportTaskExecTimeLine(&profInst));
538 39 : AicpuComContext *contextBase = nullptr;
539 39 : u32 contextNum = 0;
540 39 : AicpuGetAllComContext(contextBase, contextNum);
541 39 : AicpuKfcProf::OutputProfLog(AicpuKfcUtils::IsDebugModeEquals(*ctx, MC2_DEBUG_TIME_TAKEN), contextBase[0].acprof,
542 39 : contextBase[1].acprof);
543 39 : AicpuKfcProf::AddProfLoopCnt();
544 39 : AicpuSqeContext::SaveVariable();
545 39 : CHK_RET(AicpuSqeContext::ClearLocalBuff());
546 39 : HCCL_INFO("end RunAicpuRpcSrvLaunch");
547 39 : return 0;
548 68 : }
549 :
550 4 : __attribute__((visibility("default"))) uint32_t RunAicpuRpcSrvGroupLaunch(void *args)
551 : {
552 4 : KfcState state;
553 4 : if (args == nullptr) {
554 1 : HCCL_ERROR("args is null.");
555 1 : return HCCL_E_PARA;
556 : }
557 :
558 3 : KFCTask *task = reinterpret_cast<KFCTask *>(args);
559 3 : HCCL_INFO("KFCTask inputA %p, outputC %p, commOut %p, context %p, workSpace %p, tilingData %p",
560 : task->inputA, task->outputC, task->commOut, task->context, task->workSpace, task->tilingData);
561 3 : KFCGroupTilingData *tilingData = reinterpret_cast<KFCGroupTilingData *>(task->tilingData);
562 :
563 3 : if (tilingData == nullptr || tilingData->groupNum == 0) {
564 2 : HCCL_ERROR("tilingData is nullptr or groupNum is 0.");
565 2 : return HCCL_E_PARA;
566 : }
567 :
568 2 : for (uint32_t i = 0; i < tilingData->groupNum; ++i) {
569 : KFCTask singleTask;
570 1 : singleTask.inputA = task->inputA;
571 1 : singleTask.outputC = *reinterpret_cast<u64*>(task->outputC + sizeof(void*) * i);
572 1 : singleTask.commOut = task->commOut;
573 1 : singleTask.context = task->context;
574 1 : singleTask.workSpace = task->workSpace;
575 1 : singleTask.tilingData = reinterpret_cast<u64>(&tilingData->msg[i]);
576 1 : uint32_t ret = RunAicpuRpcSrvLaunch(&singleTask);
577 1 : if (ret != 0) {
578 0 : HCCL_ERROR("RunAicpuRpcSrvGroupLaunch runs failed.");
579 0 : return HCCL_E_PARA;
580 : }
581 : }
582 :
583 1 : return 0;
584 4 : }
585 :
586 : constexpr u32 GROUP_DYN_FLAG = 23U;
587 : constexpr u32 GROUP_TILING_MAGIC_NUM = 99U;
588 36 : __attribute__((visibility("default"))) uint32_t RunAicpuKfcSrvLaunch(void *args[])
589 : {
590 36 : if (args == nullptr) {
591 2 : HCCL_ERROR("args is null.");
592 2 : return HCCL_E_PARA;
593 : }
594 34 : constexpr int DESC_POS = 0;
595 34 : uint64_t desc_value = u64(args[DESC_POS]);
596 34 : uint64_t *desc_addr = &desc_value;
597 34 : CommKfcParamDesc *desc = reinterpret_cast<CommKfcParamDesc*>(desc_addr);
598 34 : AicpuKfcUtils::PrintHcclCommParamDesc(*desc);
599 34 : if (desc->version == DECOUPLED_CTX_VER) {
600 12 : return CommKfcDispatcher::Run(&(args[1]), desc->itemNum);
601 : }
602 22 : void *tiling = reinterpret_cast<void *>(args[desc->tilingOff]);
603 22 : if (tiling == nullptr) {
604 0 : HCCL_ERROR("tiling is null.");
605 0 : return HCCL_E_PARA;
606 : }
607 22 : KfcState state;
608 22 : bool profL1Open = dfx::ProfilingManager::IsProfL1On();
609 22 : bool profL0Open = dfx::ProfilingManager::IsProfL0On();
610 22 : HCCL_INFO("profL1Open:%d, profL0Open:%d", profL1Open, profL0Open);
611 22 : const uint32_t ver = MC2TilingGetVer(tiling);
612 22 : HCCL_INFO("Start RunAicpuKfcSrvLaunch with tiling version %u.", ver);
613 : uint32_t ret;
614 22 : switch (ver) {
615 4 : case TILING_DATA_VER_OLD_FOR_HOST: {
616 4 : KFCGroupTilingDataAuto *tilingData = static_cast<KFCGroupTilingDataAuto *>(tiling);
617 4 : if (desc->isDyn == GROUP_DYN_FLAG && tilingData->groupTilingMagicNum == GROUP_TILING_MAGIC_NUM) {
618 3 : ret = RunAicpuInnerRpcSrvGroupLaunch(args, tilingData, desc);
619 : } else {
620 1 : ret = RunKernelAicpuServerV1(args, desc);
621 : }
622 4 : break;
623 : }
624 1 : case TILING_DATA_VER_OLD_FOR_KERNEL:
625 1 : ret = RunKernelAicpuServerV1(args, desc);
626 1 : break;
627 4 : case TILING_DATA_VER_OLD_FOR_KERNEL_V2:
628 4 : ret = RunKernelAicpuServerV2(args, desc, tiling);
629 4 : break;
630 12 : case TILING_DATA_VER_FOR_TILING_API:
631 12 : ret = RunKernelAicpuServerForTilingApi(args, desc);
632 12 : break;
633 1 : default:
634 1 : HCCL_ERROR("Invalid tiling version %u.", ver);
635 1 : ret = HCCL_E_PARA;
636 : }
637 22 : return ret;
638 22 : }
639 : }
|
