Line data    Source code

       1              : 
       2              : /**
       3              :  * Copyright (c) 2025 Huawei Technologies Co., Ltd.
       4              :  * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
       5              :  * CANN Open Software License Agreement Version 2.0 (the "License").
       6              :  * Please refer to the License for details. You may not use this file except in compliance with the License.
       7              :  * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
       8              :  * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
       9              :  * See LICENSE in the root of the software repository for the full text of the License.
      10              :  */
      11              : 
      12              : #include "local_rdma_rma_buffer.h"
      13              : #include "hccp.h"
      14              : #include "exchange_rdma_buffer_dto.h"
      15              : 
      16              : namespace Hccl {
      17              : 
      18            4 : LocalRdmaRmaBuffer::LocalRdmaRmaBuffer(std::shared_ptr<Buffer> buf, RdmaHandle rdmaHandle)
      19            4 :     : LocalRmaBuffer(buf, RmaType::RDMA), rdmaHandle(rdmaHandle)
      20              : {
      21            4 :     if (rdmaHandle == nullptr || buf == nullptr) {
      22            1 :         string nullParam = rdmaHandle == nullptr ? "rdmaHandle" : "buf";
      23            1 :         THROW<NullPtrException>("LocalRdmaRmaBuffer's %s is nullptr", nullParam.c_str());
      24            1 :     }
      25            3 :     const uintptr_t bufAddr = buf->GetAddr();
      26            3 :     size_t bufSize = buf->GetSize();
      27            3 :     if (bufAddr == 0 || bufSize <= 0) {
      28            3 :         HCCL_ERROR("[LocalRdmaRmaBuffer]buffer size[%zu Byte] and addr[%zu] should be greater than 0.", bufAddr,
      29              :                    bufSize);
      30            1 :         THROW<InvalidParamsException>("[%s] failed, param error.", __func__);
      31              :     }
      32              :     // 注册内存
      33              :     struct MrInfoT mrInfo;
      34            2 :     mrInfo.addr   = reinterpret_cast<void *>(bufAddr);
      35            2 :     mrInfo.size   = bufSize;
      36            2 :     mrInfo.access = RA_ACCESS_REMOTE_WRITE | RA_ACCESS_LOCAL_WRITE | RA_ACCESS_REMOTE_READ | RA_ACCESS_REMOTE_ATOMIC;
      37              : 
      38            2 :     s32 ret = RaRegisterMr(rdmaHandle, &mrInfo, &mrHandle);
      39            2 :     if (ret != 0 || mrHandle == nullptr) {
      40            0 :         HCCL_ERROR("[HrtRaRegisterMr] RaRegisterMr failed, call interface error[%d]", ret);
      41            0 :         THROW<InternalException>("[%s] failed, call interface error[%d].", __func__, ret);
      42              :     }
      43            2 :     lkey = mrInfo.lkey;
      44            2 :     rkey = mrInfo.rkey;
      45            6 :     HCCL_INFO("LocalRdmaRmaBuffer[rdmaHandle=%p, mrHandle = %p, buf=%s]",
      46              :             rdmaHandle, mrHandle, buf->Describe().c_str());
      47            6 : }
      48              : 
      49            0 : LocalRdmaRmaBuffer::LocalRdmaRmaBuffer(std::shared_ptr<Buffer> buf, RdmaHandle rdmaHandle, u32 lkey, u32 rkey, MrHandle mrHandle)
      50            0 :     : LocalRmaBuffer(buf, RmaType::RDMA, true), rdmaHandle(rdmaHandle), lkey(lkey), rkey(rkey), mrHandle(mrHandle)
      51              : {
      52            0 :     if (rdmaHandle == nullptr || buf == nullptr) {
      53            0 :         string nullParam = rdmaHandle == nullptr ? "rdmaHandle" : "buf";
      54            0 :         THROW<NullPtrException>("LocalRdmaRmaBuffer alias: %s is nullptr", nullParam.c_str());
      55            0 :     }
      56            0 :     const uintptr_t bufAddr = buf->GetAddr();
      57            0 :     size_t bufSize = buf->GetSize();
      58            0 :     if (bufAddr == 0 || bufSize <= 0) {
      59            0 :         HCCL_ERROR("[LocalRdmaRmaBuffer] alias buffer addr[%p] and size[%llu Byte] should be greater than 0.",
      60              :             reinterpret_cast<void *>(bufAddr), static_cast<unsigned long long>(bufSize));
      61            0 :         THROW<InvalidParamsException>("[%s] alias failed, param error.", __func__);
      62              :     }
      63            0 :     if (mrHandle == nullptr) {
      64            0 :         THROW<NullPtrException>("LocalRdmaRmaBuffer alias: mrHandle is nullptr");
      65              :     }
      66            0 :     HCCL_INFO("LocalRdmaRmaBuffer alias[rdmaHandle=%p, mrHandle=%p, lkey=%u, buf=%s]",
      67              :             rdmaHandle, mrHandle, lkey, buf->Describe().c_str());
      68            0 : }
      69              : 
      70            2 : LocalRdmaRmaBuffer::~LocalRdmaRmaBuffer()
      71              : {
      72            2 :     if (mrHandle && !isAlias_) {
      73            2 :         s32 ret = RaDeregisterMr(rdmaHandle, mrHandle);
      74            2 :         if (ret != 0) {
      75            0 :             HCCL_ERROR("[HrtRaDeRegisterMr]errNo[0x%016llx] RaDeregisterMr failed, return[%d]",
      76              :                 HCCL_ERROR_CODE(HCCL_E_NETWORK), ret);
      77              :             // THROW<InternalException>("[%s] failed, call interface error[%d].", __func__, ret);
      78              :         }
      79            2 :         mrHandle = nullptr;
      80              :     }
      81            2 : }
      82              : 
      83            2 : string LocalRdmaRmaBuffer::Describe() const
      84              : {
      85            2 :     return StringFormat("LocalRdmaRmaBuffer[rdmaHandle=%p, mrHandle = %p, buf=%s]", rdmaHandle, mrHandle,
      86            2 :                         buf->Describe().c_str());
      87              : }
      88              : 
      89            0 : std::unique_ptr<Serializable> LocalRdmaRmaBuffer::GetExchangeDto()
      90              : {
      91              :     std::unique_ptr<ExchangeRdmaBufferDto> dto = make_unique<ExchangeRdmaBufferDto>(
      92            0 :         buf->GetAddr(), buf->GetSize(), this->rkey, buf->GetMemInfo().c_str());
      93            0 :     return std::unique_ptr<Serializable>(dto.release());
      94            0 : }
      95              : 
      96              : } // namespace Hccl