Line data    Source code

       1              : /**
       2              :  * Copyright (c) 2025 Huawei Technologies Co., Ltd.
       3              :  * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
       4              :  * CANN Open Software License Agreement Version 2.0 (the "License").
       5              :  * Please refer to the License for details. You may not use this file except in compliance with the License.
       6              :  * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
       7              :  * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
       8              :  * See LICENSE in the root of the software repository for the full text of the License.
       9              :  */
      10              : 
      11              : #include "rdma_local_notify.h"
      12              : #include "not_support_exception.h"
      13              : #include "dev_capability.h"
      14              : #include "hccp.h"
      15              : #include "exchange_rdma_buffer_dto.h"
      16              : 
      17              : namespace Hccl {
      18              : 
      19            1 : RdmaLocalNotify::RdmaLocalNotify(RdmaHandle rdmaHandle, bool devUsed)
      20            1 :     : BaseLocalNotify(RmaType::RDMA, devUsed), rdmaHandle(rdmaHandle)
      21              : {
      22            1 :     auto devType = HrtGetDeviceType(); // 先查询，避免后续失败资源泄露
      23            1 :     HrtDevResInfo devResInfo;
      24            1 :     devResInfo.dieId            = 0;
      25            1 :     devResInfo.procType         = HrtDevResProcType::PROCESS_CP1;
      26            1 :     devResInfo.resType          = HrtDevResType::RES_TYPE_STARS_NOTIFY_RECORD;
      27            1 :     devResInfo.resId            = GetNotify()->GetId();
      28            1 :     devResInfo.flag             = HRT_DEV_RES_FLAG_USE_UNIQUE_VA;
      29            1 :     auto resAddrInfo            = HrtGetDevResAddress(devResInfo);
      30            1 :     addr                        = resAddrInfo.address;
      31            1 :     DevCapability::GetInstance().Init(devType); // 单例初始化
      32            1 :     size                        = DevCapability::GetInstance().GetNotifySize();
      33              :     // 注册内存
      34              :     struct MrInfoT mrInfo;
      35            1 :     addr = addr & ~(4096 - 1ULL); // 临时规避，待ubdevmem适配后修改
      36            1 :     size = 4096;
      37            1 :     mrInfo.addr   = reinterpret_cast<void *>(addr);
      38            1 :     mrInfo.size   = size;
      39            1 :     mrInfo.access = RA_ACCESS_REMOTE_WRITE | RA_ACCESS_LOCAL_WRITE | RA_ACCESS_REMOTE_READ;
      40            1 :     s32 ret = RaRegisterMr(rdmaHandle, &mrInfo, &mrHandle);
      41            1 :     if (ret != 0 || mrHandle == nullptr) {
      42            0 :         HCCL_ERROR("[RdmaLocalNotify] RaRegisterMr failed, call interface error[%d] mrHandle[%p]", ret, mrHandle);
      43            0 :         THROW<InternalException>("[%s] failed, call interface error[%d].", __func__, ret);
      44              :     }
      45            1 :     lkey = mrInfo.lkey;
      46            1 :     rkey = mrInfo.rkey;
      47            1 : }
      48              : 
      49            1 : RdmaLocalNotify::~RdmaLocalNotify()
      50              : {
      51            1 :     if (mrHandle) {
      52            1 :         s32 ret = RaDeregisterMr(rdmaHandle, mrHandle);
      53            1 :         if (ret != 0) {
      54            0 :             HCCL_ERROR("[~RdmaLocalNotify]errNo[0x%016llx] RaDeregisterMr failed, return[%d]",
      55              :                 HCCL_ERROR_CODE(HCCL_E_NETWORK), ret);
      56              :         }
      57            1 :         mrHandle = nullptr;
      58              :     }
      59              : 
      60            1 :     HrtDevResInfo devResInfo;
      61            1 :     devResInfo.dieId    = 0;
      62            1 :     devResInfo.procType = HrtDevResProcType::PROCESS_CP1;
      63            1 :     devResInfo.resType  = HrtDevResType::RES_TYPE_STARS_NOTIFY_RECORD;
      64            1 :     devResInfo.resId    = GetNotify()->GetId();
      65            1 :     devResInfo.flag     = HRT_DEV_RES_FLAG_USE_UNIQUE_VA;
      66            1 :     HrtReleaseDevResAddress(devResInfo);
      67            1 : }
      68              : 
      69            1 : void RdmaLocalNotify::Wait(const Stream &stream, u32 timeout) const
      70              : {
      71            1 :     GetNotify()->Wait(stream, timeout);
      72            1 : }
      73              : 
      74            1 : void RdmaLocalNotify::Post(const Stream &stream) const
      75              : {
      76            3 :     HCCL_ERROR("RdmaLocalNotify does not support submit record task");
      77            3 :     throw NotSupportException("RdmaLocalNotify does not support submit record task");
      78              : }
      79              : 
      80            1 : string RdmaLocalNotify::Describe() const
      81              : {
      82            1 :     return StringFormat("RdmaLocalNotify[notify=%s, addr=0x%llx, size=%u]", GetNotify()->Describe().c_str(), addr, size);
      83              : }
      84              : 
      85            0 : std::unique_ptr<Serializable> RdmaLocalNotify::GetExchangeDto()
      86              : {
      87              :     std::unique_ptr<ExchangeRdmaBufferDto> dto
      88            0 :         = make_unique<ExchangeRdmaBufferDto>(addr, size, rkey, "RdmaNotify");
      89            0 :     return std::unique_ptr<Serializable>(dto.release());
      90            0 : }
      91              : 
      92              : } // namesapce Hccl
      93              :