Line data    Source code

       1              : 
       2              : /**
       3              :  * Copyright (c) 2025 Huawei Technologies Co., Ltd.
       4              :  * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
       5              :  * CANN Open Software License Agreement Version 2.0 (the "License").
       6              :  * Please refer to the License for details. You may not use this file except in compliance with the License.
       7              :  * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
       8              :  * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
       9              :  * See LICENSE in the root of the software repository for the full text of the License.
      10              :  */
      11              : 
      12              : #include "local_rdma_rma_buffer_v2.h"
      13              : #include "hccp.h"
      14              : #include "exchange_rdma_buffer_dto.h"
      15              : 
      16              : namespace Hccl {
      17              : 
      18            4 : LocalRdmaRmaBuffer::LocalRdmaRmaBuffer(std::shared_ptr<Buffer> buf, RdmaHandle rdmaHandle)
      19              :     : LocalRmaBuffer(buf, RmaType::RDMA),
      20            4 :       rdmaHandle(rdmaHandle)
      21              : {
      22            4 :     if (rdmaHandle == nullptr || buf == nullptr) {
      23            1 :         string nullParam = rdmaHandle == nullptr ? "rdmaHandle" : "buf";
      24            1 :         THROW<NullPtrException>("LocalRdmaRmaBuffer's %s is nullptr", nullParam.c_str());
      25            1 :     }
      26            3 :     const uintptr_t bufAddr = buf->GetAddr();
      27            3 :     size_t bufSize = buf->GetSize();
      28            3 :     if (bufAddr == 0 || bufSize <= 0) {
      29            3 :         HCCL_ERROR(
      30              :             "[LocalRdmaRmaBuffer]buffer size[%zu Byte] and addr[%zu] should be greater than 0.", bufAddr, bufSize);
      31            1 :         THROW<InvalidParamsException>("[%s] failed, param error.", __func__);
      32              :     }
      33              :     // 注册内存
      34              :     struct MrInfoT mrInfo;
      35            2 :     mrInfo.addr = reinterpret_cast<void*>(bufAddr);
      36            2 :     mrInfo.size = bufSize;
      37            2 :     mrInfo.access = RA_ACCESS_REMOTE_WRITE | RA_ACCESS_LOCAL_WRITE | RA_ACCESS_REMOTE_READ | RA_ACCESS_REMOTE_ATOMIC;
      38              : 
      39            2 :     s32 ret = RaRegisterMr(rdmaHandle, &mrInfo, &mrHandle);
      40            2 :     if (ret != 0 || mrHandle == nullptr) {
      41            0 :         HCCL_ERROR("[HrtRaRegisterMr] RaRegisterMr failed, call interface error[%d]", ret);
      42            0 :         THROW<InternalException>("[%s] failed, call interface error[%d].", __func__, ret);
      43              :     }
      44            2 :     lkey = mrInfo.lkey;
      45            2 :     rkey = mrInfo.rkey;
      46            6 :     HCCL_INFO(
      47              :         "LocalRdmaRmaBuffer[rdmaHandle=%p, mrHandle = %p, buf=%s]", rdmaHandle, mrHandle, buf->Describe().c_str());
      48            6 : }
      49              : 
      50            0 : LocalRdmaRmaBuffer::LocalRdmaRmaBuffer(
      51            0 :     std::shared_ptr<Buffer> buf, RdmaHandle rdmaHandle, u32 lkey, u32 rkey, MrHandle mrHandle)
      52              :     : LocalRmaBuffer(buf, RmaType::RDMA, true),
      53            0 :       rdmaHandle(rdmaHandle),
      54            0 :       lkey(lkey),
      55            0 :       rkey(rkey),
      56            0 :       mrHandle(mrHandle)
      57              : {
      58            0 :     if (rdmaHandle == nullptr || buf == nullptr) {
      59            0 :         string nullParam = rdmaHandle == nullptr ? "rdmaHandle" : "buf";
      60            0 :         THROW<NullPtrException>("LocalRdmaRmaBuffer alias: %s is nullptr", nullParam.c_str());
      61            0 :     }
      62            0 :     const uintptr_t bufAddr = buf->GetAddr();
      63            0 :     size_t bufSize = buf->GetSize();
      64            0 :     if (bufAddr == 0 || bufSize <= 0) {
      65            0 :         HCCL_ERROR(
      66              :             "[LocalRdmaRmaBuffer] alias buffer addr[%p] and size[%llu Byte] should be greater than 0.",
      67              :             reinterpret_cast<void*>(bufAddr), static_cast<unsigned long long>(bufSize));
      68            0 :         THROW<InvalidParamsException>("[%s] alias failed, param error.", __func__);
      69              :     }
      70            0 :     if (mrHandle == nullptr) {
      71            0 :         THROW<NullPtrException>("LocalRdmaRmaBuffer alias: mrHandle is nullptr");
      72              :     }
      73            0 :     HCCL_INFO(
      74              :         "LocalRdmaRmaBuffer alias[rdmaHandle=%p, mrHandle=%p, lkey=%u, buf=%s]", rdmaHandle, mrHandle, lkey,
      75              :         buf->Describe().c_str());
      76            0 : }
      77              : 
      78            2 : LocalRdmaRmaBuffer::~LocalRdmaRmaBuffer()
      79              : {
      80            2 :     if (mrHandle && !isAlias_) {
      81            2 :         s32 ret = RaDeregisterMr(rdmaHandle, mrHandle);
      82            2 :         if (ret != 0) {
      83            0 :             HCCL_ERROR(
      84              :                 "[HrtRaDeRegisterMr]errNo[0x%016llx] RaDeregisterMr failed, return[%d]",
      85              :                 HCCL_ERROR_CODE(HCCL_E_NETWORK), ret);
      86              :             // THROW<InternalException>("[%s] failed, call interface error[%d].", __func__, ret);
      87              :         }
      88            2 :         mrHandle = nullptr;
      89              :     }
      90            2 : }
      91              : 
      92            2 : string LocalRdmaRmaBuffer::Describe() const
      93              : {
      94              :     return StringFormat(
      95            2 :         "LocalRdmaRmaBuffer[rdmaHandle=%p, mrHandle = %p, buf=%s]", rdmaHandle, mrHandle, buf->Describe().c_str());
      96              : }
      97              : 
      98            0 : std::unique_ptr<Serializable> LocalRdmaRmaBuffer::GetExchangeDto()
      99              : {
     100              :     std::unique_ptr<ExchangeRdmaBufferDto> dto
     101            0 :         = make_unique<ExchangeRdmaBufferDto>(buf->GetAddr(), buf->GetSize(), this->rkey, buf->GetMemInfo().c_str());
     102            0 :     return std::unique_ptr<Serializable>(dto.release());
     103            0 : }
     104              : 
     105              : } // namespace Hccl