Line data    Source code

       1              : /**
       2              :  * Copyright (c) 2025 Huawei Technologies Co., Ltd.
       3              :  * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
       4              :  * CANN Open Software License Agreement Version 2.0 (the "License").
       5              :  * Please refer to the License for details. You may not use this file except in compliance with the License.
       6              :  * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
       7              :  * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
       8              :  * See LICENSE in the root of the software repository for the full text of the License.
       9              :  */
      10              : 
      11              : #include "rdma_local_notify.h"
      12              : #include "not_support_exception.h"
      13              : #include "dev_capability.h"
      14              : #include "hccp.h"
      15              : #include "exchange_rdma_buffer_dto.h"
      16              : 
      17              : namespace Hccl {
      18              : 
      19            1 : RdmaLocalNotify::RdmaLocalNotify(RdmaHandle rdmaHandle, bool devUsed)
      20              :     : BaseLocalNotify(RmaType::RDMA, devUsed),
      21            1 :       rdmaHandle(rdmaHandle)
      22              : {
      23            1 :     auto devType = HrtGetDeviceType(); // 先查询，避免后续失败资源泄露
      24            1 :     HrtDevResInfo devResInfo;
      25            1 :     devResInfo.dieId = 0;
      26            1 :     devResInfo.procType = HrtDevResProcType::PROCESS_CP1;
      27            1 :     devResInfo.resType = HrtDevResType::RES_TYPE_STARS_NOTIFY_RECORD;
      28            1 :     devResInfo.resId = GetNotify()->GetId();
      29            1 :     devResInfo.flag = HRT_DEV_RES_FLAG_USE_UNIQUE_VA;
      30            1 :     auto resAddrInfo = HrtGetDevResAddress(devResInfo);
      31            1 :     addr = resAddrInfo.address;
      32            1 :     DevCapability::GetInstance().Init(devType); // 单例初始化
      33            1 :     size = DevCapability::GetInstance().GetNotifySize();
      34            3 :     HCCL_DEBUG("[RdmaLocalNotify] addr=0x%llx, size=%u", addr, size);
      35              :     // 注册内存
      36              :     struct MrInfoT mrInfo;
      37            1 :     mrInfo.addr = reinterpret_cast<void*>(addr);
      38            1 :     mrInfo.size = size;
      39            1 :     mrInfo.access = RA_ACCESS_REMOTE_WRITE | RA_ACCESS_LOCAL_WRITE | RA_ACCESS_REMOTE_READ;
      40            1 :     s32 ret = RaRegisterMr(rdmaHandle, &mrInfo, &mrHandle);
      41            1 :     if (ret != 0 || mrHandle == nullptr) {
      42            0 :         HCCL_ERROR("[RdmaLocalNotify] RaRegisterMr failed, call interface error[%d] mrHandle[%p]", ret, mrHandle);
      43            0 :         THROW<InternalException>("[%s] failed, call interface error[%d].", __func__, ret);
      44              :     }
      45            1 :     lkey = mrInfo.lkey;
      46            1 :     rkey = mrInfo.rkey;
      47            1 : }
      48              : 
      49            1 : RdmaLocalNotify::~RdmaLocalNotify()
      50              : {
      51            1 :     if (mrHandle) {
      52            1 :         s32 ret = RaDeregisterMr(rdmaHandle, mrHandle);
      53            1 :         if (ret != 0) {
      54            0 :             HCCL_ERROR(
      55              :                 "[~RdmaLocalNotify]errNo[0x%016llx] RaDeregisterMr failed, return[%d]", HCCL_ERROR_CODE(HCCL_E_NETWORK),
      56              :                 ret);
      57              :         }
      58            1 :         mrHandle = nullptr;
      59              :     }
      60              : 
      61            1 :     HrtDevResInfo devResInfo;
      62            1 :     devResInfo.dieId = 0;
      63            1 :     devResInfo.procType = HrtDevResProcType::PROCESS_CP1;
      64            1 :     devResInfo.resType = HrtDevResType::RES_TYPE_STARS_NOTIFY_RECORD;
      65            1 :     devResInfo.resId = GetNotify()->GetId();
      66            1 :     devResInfo.flag = HRT_DEV_RES_FLAG_USE_UNIQUE_VA;
      67            1 :     HrtReleaseDevResAddress(devResInfo);
      68            1 : }
      69              : 
      70            1 : void RdmaLocalNotify::Wait(const Stream& stream, u32 timeout) const { GetNotify()->Wait(stream, timeout); }
      71              : 
      72            1 : void RdmaLocalNotify::Post([[maybe_unused]] const Stream& stream) const
      73              : {
      74            3 :     HCCL_ERROR("RdmaLocalNotify does not support submit record task");
      75            3 :     throw NotSupportException("RdmaLocalNotify does not support submit record task");
      76              : }
      77              : 
      78            1 : string RdmaLocalNotify::Describe() const
      79              : {
      80              :     return StringFormat(
      81            1 :         "RdmaLocalNotify[notify=%s, addr=0x%llx, size=%u]", GetNotify()->Describe().c_str(), addr, size);
      82              : }
      83              : 
      84            0 : std::unique_ptr<Serializable> RdmaLocalNotify::GetExchangeDto()
      85              : {
      86            0 :     std::unique_ptr<ExchangeRdmaBufferDto> dto = make_unique<ExchangeRdmaBufferDto>(addr, size, rkey, "RdmaNotify");
      87            0 :     return std::unique_ptr<Serializable>(dto.release());
      88            0 : }
      89              : 
      90              : } // namespace Hccl