Line data    Source code

       1              : /**
       2              :  * Copyright (c) 2025 Huawei Technologies Co., Ltd.
       3              :  * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
       4              :  * CANN Open Software License Agreement Version 2.0 (the "License").
       5              :  * Please refer to the License for details. You may not use this file except in compliance with the License.
       6              :  * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
       7              :  * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
       8              :  * See LICENSE in the root of the software repository for the full text of the License.
       9              :  */
      10              : 
      11              : #include "aicpu_kernel_launcher.h"
      12              : #include <memory>
      13              : #include "internal_exception.h"
      14              : #include "coll_service_device_mode.h"
      15              : #include "aicpu_ins_preprocessor.h"
      16              : #include "coll_service_ai_cpu_impl.h"
      17              : #include "exception_util.h"
      18              : 
      19              : namespace Hccl {
      20              : 
      21              : template <class T, class U> u16 CalcFieldOffset(T *target, U *base)
      22              : {
      23              :     return static_cast<u16>(reinterpret_cast<const char *>(target) - reinterpret_cast<const char *>(base));
      24              : }
      25              : 
      26              : 
      27            0 : void AicpuKernelLauncher::AicpuKernelLaunch(const Stream &stream, const string &algName) const
      28              : {
      29            0 :     HCCL_INFO("[AicpuKernelLauncher::%s] start.", __func__);
      30              : 
      31            0 :     auto                  op = comm->GetCurrentCollOperator();
      32            0 :     HcclKernelLaunchParam param;
      33              : 
      34            0 :     s32 ret = strcpy_s(param.kernel.algName, sizeof(param.kernel.algName), algName.data());
      35            0 :     if (ret != EOK) {
      36            0 :         THROW<InternalException>(StringFormat("AicpuKernelLaunch, strcpy_s algName failed!"));
      37              :     }
      38              : 
      39            0 :     ret = strcpy_s(param.kernel.opTag, sizeof(param.kernel.opTag), op->opTag.data());
      40            0 :     if (ret != EOK) {
      41            0 :         THROW<InternalException>(StringFormat("AicpuKernelLaunch, strcpy_s opTag failed!"));
      42              :     }
      43              : 
      44            0 :     HCCL_INFO("AicpuKernelLauncher::AicpuKernelLaunch param.kernel.algName: %s, opTag %s", param.kernel.algName,
      45              :                op->opTag.c_str());
      46              : 
      47              :     auto aicpuInsPreprocessor
      48            0 :         = dynamic_cast<CollServiceDeviceMode *>(comm->GetCollService())->GetAicpuInsPreprocessor();
      49            0 :     DevBuffer *mem             = aicpuInsPreprocessor->GetAicpuResBuffer(algName);
      50            0 :     param.kernel.binaryResAddr = mem->GetAddr();
      51            0 :     param.kernel.binaryResSize = mem->GetSize();
      52            0 :     aicpuInsPreprocessor->SetAicpuResExisted(algName);
      53              : 
      54            0 :     SetHcclKernelLaunchParam(param);
      55              : 
      56            0 :     AddPostToUserStream(stream);
      57            0 :     const auto funcHandle = comm->GetAicpuKernelFuncHandle(param.kernelName);
      58              : 
      59              :     aclrtLaunchKernelAttr attr;
      60            0 :     attr.id = ACL_RT_LAUNCH_KERNEL_ATTR_TIMEOUT;
      61            0 :     attr.value.timeout = comm->GetNotifyTimeoutCfg().GetNotifyTimeout();
      62              :     aclrtLaunchKernelCfg cfg;
      63            0 :     cfg.attrs = &attr;
      64            0 :     cfg.numAttrs = 1;
      65            0 :     constexpr u32 numBlocks = 1;
      66            0 :     if (op->opMode == OpMode::OPBASE) {
      67            0 :         HrtAicpuLaunchKernelWithHostArgs(funcHandle, numBlocks, comm->GetAicpuStreamManager().GetFreeStream()->GetPtr(), &cfg,
      68              :           &param.kernel, sizeof(HcclKernelParamLite));
      69            0 :         HCCL_INFO("[AicpuKernelLauncher][AicpuKernelLaunch] param.kernel.algName: %s OPBASE mode "
      70              :                    "HrtAicpuLaunchKernelWithHostArgs end!", param.kernel.algName);
      71            0 :     } else if (op->opMode == OpMode::OFFLOAD) {
      72            0 :         HrtAicpuLaunchKernelWithHostArgs(funcHandle, numBlocks, stream.GetPtr(), &cfg,
      73              :             &param.kernel, sizeof(HcclKernelParamLite));
      74            0 :         HCCL_INFO("[AicpuKernelLauncher][AicpuKernelLaunch] param.kernel.algName: %s OFFLOAD mode "
      75              :                    "HrtAicpuLaunchKernelWithHostArgs end!", param.kernel.algName);
      76              :     }
      77            0 :     AddWaitToUserStream(stream);
      78            0 :     HCCL_INFO("[AicpuKernelLauncher::%s] end.", __func__);
      79            0 : }
      80              : 
      81            1 : void AicpuKernelLauncher::SetOpbaseBufferParam(HcclKernelLaunchParam &param, CollOperator &op) const
      82              : {
      83            3 :     HCCL_INFO("[AicpuKernelLauncher::%s] start.", __func__);
      84              : 
      85            1 :     auto buffer                          = comm->GetCclBuffer();
      86            1 :     param.kernel.comm.opBaseScratch.addr = buffer->GetAddr();
      87            1 :     param.kernel.comm.opBaseScratch.size = buffer->GetSize();
      88            1 :     InitAicpuLocBufLite(param.kernel.comm.opBaseScratch, buffer->GetAddr(), buffer->GetSize(), "opBaseScratch");
      89            1 :     if (op.inputMem != nullptr) {
      90            2 :         InitAicpuLocBufLite(param.kernel.op.input, op.inputMem->GetAddr(), op.inputMem->GetSize(), "inputMem");
      91              :     }
      92              : 
      93            1 :     if (op.outputMem != nullptr) {
      94            2 :         InitAicpuLocBufLite(param.kernel.op.output, op.outputMem->GetAddr(), op.outputMem->GetSize(), "outputMem");
      95              :     }
      96            3 :     HCCL_INFO("[AicpuKernelLauncher::%s] end, SetOpbaseBufferParam param.kernel.comm.opBaseScratch.addr %llu, "
      97              :                "param.kernel.comm.opBaseScratch.size %llu",
      98              :                __func__, param.kernel.comm.opBaseScratch.addr, param.kernel.comm.opBaseScratch.size);
      99            1 : }
     100              : 
     101            1 : void AicpuKernelLauncher::SetOffloadBufferParam(HcclKernelLaunchParam &param, CollOperator &op) const
     102              : {
     103            3 :     HCCL_INFO("[AicpuKernelLauncher::%s] start.", __func__);
     104              : 
     105            1 :     auto offloadInput = comm->GetDataBufferManager().Get(op.opTag, BufferType::INPUT);
     106            1 :     if (offloadInput != nullptr) {
     107            0 :         InitAicpuLocBufLite(param.kernel.op.input, op.inputMem->GetAddr(), op.inputMem->GetSize(), "inputMem");
     108              :     }
     109              : 
     110            1 :     auto offloadOuput = comm->GetDataBufferManager().Get(op.opTag, BufferType::OUTPUT);
     111            1 :     if (offloadOuput != nullptr) {
     112            0 :         InitAicpuLocBufLite(param.kernel.op.output, op.outputMem->GetAddr(), op.outputMem->GetSize(), "outputMem");
     113              :     }
     114              : 
     115            1 :     auto offloadScartch = comm->GetDataBufferManager().Get(op.opTag, BufferType::SCRATCH);
     116            1 :     if (offloadScartch != nullptr) {
     117            0 :         InitAicpuLocBufLite(param.kernel.op.scratch, op.scratchMem->GetAddr(), op.scratchMem->GetSize(), "scratchMem");
     118              :     }
     119              : 
     120            3 :     HCCL_INFO("[AicpuKernelLauncher::%s] end.", __func__);
     121            1 : }
     122              : 
     123            2 : void AicpuKernelLauncher::SetHcclKernelLaunchParam(HcclKernelLaunchParam &param) const
     124              : {
     125            6 :     HCCL_INFO("[AicpuKernelLauncher::%s] start.", __func__);
     126              : 
     127            2 :     CollOperator op = *comm->GetCurrentCollOperator();
     128              : 
     129            2 :     param.kernel.comm.idIndex       = comm->GetIdIndex();
     130            2 :     param.kernel.comm.myRank        = comm->GetMyRank();
     131            2 :     param.kernel.comm.rankSize       = comm->GetRankSize();
     132            2 :     param.kernel.comm.devType       = comm->GetDevType();
     133            2 :     param.kernel.comm.devPhyId      = comm->GetDevicePhyId();
     134            2 :     auto collService                = comm->GetCollService();
     135            2 :     param.kernel.comm.opCounterAddr = static_cast<u64>(collService->GetOpCounterBuf()->GetAddr());
     136              : 
     137            2 :     if (op.opMode == OpMode::OPBASE) {
     138            1 :         SetOpbaseBufferParam(param, op);
     139              :     } else {
     140            1 :         SetOffloadBufferParam(param, op);
     141              :     }
     142              : 
     143            2 :     param.kernel.op.algOperator.opMode    = op.opMode;
     144            2 :     param.kernel.op.algOperator.opType    = op.opType;
     145            2 :     param.kernel.op.algOperator.reduceOp  = op.reduceOp;
     146            2 :     param.kernel.op.algOperator.dataType  = op.dataType;
     147            2 :     param.kernel.op.algOperator.dataCount = op.dataCount;
     148            2 :     param.kernel.op.algOperator.root      = op.root;
     149            2 :     if (op.opType == OpType::ALLTOALL) {
     150            0 :         param.kernel.op.algOperator.all2AllDataDes = op.all2AllDataDes;
     151            2 :     } else if (op.opType == OpType::ALLTOALLV) {
     152              :         auto aicpuInsPreprocessor
     153            0 :             = dynamic_cast<CollServiceDeviceMode *>(comm->GetCollService())->GetAicpuInsPreprocessor();
     154            0 :         aicpuInsPreprocessor->SetAicpuKernelLaunchParam(param);
     155              :     }
     156              : 
     157            2 :     param.kernel.op.sendRecvRemoteRank = op.sendRecvRemoteRank;
     158              : 
     159            6 :     HCCL_INFO("[AicpuKernelLauncher::%s] end.", __func__);
     160            2 : }
     161              : 
     162              : // 待解决：当前方案未确定此处的取值
     163              : constexpr u32 HOST_DEVICE_SYNC_TIMEOUT = 1000;
     164              : 
     165            0 : void AicpuKernelLauncher::AddPostToUserStream(const Stream &stream) const
     166              : {
     167            0 :     auto postNotify = comm->GetHostDeviceSyncNotifyManager().GetDeviceWaitNotify();
     168              : 
     169            0 :     postNotify->Post(stream);
     170            0 : }
     171              : 
     172            0 : void AicpuKernelLauncher::AddWaitToUserStream(const Stream &stream) const
     173              : {
     174            0 :     auto waitNotify = comm->GetHostDeviceSyncNotifyManager().GetHostWaitNotify();
     175              : 
     176            0 :     waitNotify->Wait(stream, HOST_DEVICE_SYNC_TIMEOUT);
     177            0 : }
     178              : 
     179              : } // namespace Hccl