Line data    Source code

       1              : /**
       2              :  * Copyright (c) 2025 Huawei Technologies Co., Ltd.
       3              :  * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
       4              :  * CANN Open Software License Agreement Version 2.0 (the "License").
       5              :  * Please refer to the License for details. You may not use this file except in compliance with the License.
       6              :  * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
       7              :  * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
       8              :  * See LICENSE in the root of the software repository for the full text of the License.
       9              :  */
      10              : 
      11              : #include "local_rdma_rma_buffer_v2.h"
      12              : #include "hccp.h"
      13              : #include "exchange_rdma_buffer_dto.h"
      14              : 
      15              : namespace Hccl {
      16              : 
      17            4 : LocalRdmaRmaBuffer::LocalRdmaRmaBuffer(std::shared_ptr<Buffer> buf, RdmaHandle rdmaHandle)
      18              :     : LocalRmaBuffer(buf, RmaType::RDMA),
      19            4 :       rdmaHandle(rdmaHandle)
      20              : {
      21            4 :     if (rdmaHandle == nullptr || buf == nullptr) {
      22            1 :         string nullParam = rdmaHandle == nullptr ? "rdmaHandle" : "buf";
      23            1 :         THROW<NullPtrException>("LocalRdmaRmaBuffer's %s is nullptr", nullParam.c_str());
      24            1 :     }
      25            3 :     const uintptr_t bufAddr = buf->GetAddr();
      26            3 :     size_t bufSize = buf->GetSize();
      27            3 :     if (bufAddr == 0 || bufSize <= 0) {
      28            3 :         HCCL_ERROR(
      29              :             "[LocalRdmaRmaBuffer]buffer size[%zu Byte] and addr[%zu] should be greater than 0.", bufAddr, bufSize);
      30            1 :         THROW<InvalidParamsException>("[%s] failed, param error.", __func__);
      31              :     }
      32              :     // 注册内存
      33              :     struct MrInfoT mrInfo;
      34            2 :     mrInfo.addr = reinterpret_cast<void*>(bufAddr);
      35            2 :     mrInfo.size = bufSize;
      36            2 :     mrInfo.access = RA_ACCESS_REMOTE_WRITE | RA_ACCESS_LOCAL_WRITE | RA_ACCESS_REMOTE_READ | RA_ACCESS_REMOTE_ATOMIC;
      37              : 
      38            2 :     s32 ret = RaRegisterMr(rdmaHandle, &mrInfo, &mrHandle);
      39            2 :     if (ret != 0 || mrHandle == nullptr) {
      40            0 :         HCCL_ERROR("[HrtRaRegisterMr] RaRegisterMr failed, call interface error[%d]", ret);
      41            0 :         THROW<InternalException>("[%s] failed, call interface error[%d].", __func__, ret);
      42              :     }
      43            2 :     lkey = mrInfo.lkey;
      44            2 :     rkey = mrInfo.rkey;
      45            6 :     HCCL_INFO(
      46              :         "LocalRdmaRmaBuffer[rdmaHandle=%p, mrHandle = %p, buf=%s]", rdmaHandle, mrHandle, buf->Describe().c_str());
      47            6 : }
      48              : 
      49            0 : LocalRdmaRmaBuffer::LocalRdmaRmaBuffer(
      50            0 :     std::shared_ptr<Buffer> buf, RdmaHandle rdmaHandle, u32 lkey, u32 rkey, MrHandle mrHandle)
      51              :     : LocalRmaBuffer(buf, RmaType::RDMA, true),
      52            0 :       rdmaHandle(rdmaHandle),
      53            0 :       lkey(lkey),
      54            0 :       rkey(rkey),
      55            0 :       mrHandle(mrHandle)
      56              : {
      57            0 :     if (rdmaHandle == nullptr || buf == nullptr) {
      58            0 :         string nullParam = rdmaHandle == nullptr ? "rdmaHandle" : "buf";
      59            0 :         THROW<NullPtrException>("LocalRdmaRmaBuffer alias: %s is nullptr", nullParam.c_str());
      60            0 :     }
      61            0 :     const uintptr_t bufAddr = buf->GetAddr();
      62            0 :     size_t bufSize = buf->GetSize();
      63            0 :     if (bufAddr == 0 || bufSize <= 0) {
      64            0 :         HCCL_ERROR(
      65              :             "[LocalRdmaRmaBuffer] alias buffer addr[%p] and size[%llu Byte] should be greater than 0.",
      66              :             reinterpret_cast<void*>(bufAddr), static_cast<unsigned long long>(bufSize));
      67            0 :         THROW<InvalidParamsException>("[%s] alias failed, param error.", __func__);
      68              :     }
      69            0 :     if (mrHandle == nullptr) {
      70            0 :         THROW<NullPtrException>("LocalRdmaRmaBuffer alias: mrHandle is nullptr");
      71              :     }
      72            0 :     HCCL_INFO(
      73              :         "LocalRdmaRmaBuffer alias[rdmaHandle=%p, mrHandle=%p, lkey=%u, buf=%s]", rdmaHandle, mrHandle, lkey,
      74              :         buf->Describe().c_str());
      75            0 : }
      76              : 
      77            2 : LocalRdmaRmaBuffer::~LocalRdmaRmaBuffer()
      78              : {
      79            2 :     if (mrHandle && !isAlias_) {
      80            2 :         s32 ret = RaDeregisterMr(rdmaHandle, mrHandle);
      81            2 :         if (ret != 0) {
      82            0 :             HCCL_ERROR(
      83              :                 "[HrtRaDeRegisterMr]errNo[0x%016llx] RaDeregisterMr failed, return[%d]",
      84              :                 HCCL_ERROR_CODE(HCCL_E_NETWORK), ret);
      85              :             // THROW<InternalException>("[%s] failed, call interface error[%d].", __func__, ret);
      86              :         }
      87            2 :         mrHandle = nullptr;
      88              :     }
      89            2 : }
      90              : 
      91            2 : string LocalRdmaRmaBuffer::Describe() const
      92              : {
      93              :     return StringFormat(
      94            2 :         "LocalRdmaRmaBuffer[rdmaHandle=%p, mrHandle = %p, buf=%s]", rdmaHandle, mrHandle, buf->Describe().c_str());
      95              : }
      96              : 
      97            0 : std::unique_ptr<Serializable> LocalRdmaRmaBuffer::GetExchangeDto()
      98              : {
      99              :     std::unique_ptr<ExchangeRdmaBufferDto> dto
     100            0 :         = make_unique<ExchangeRdmaBufferDto>(buf->GetAddr(), buf->GetSize(), this->rkey, buf->GetMemInfo().c_str());
     101            0 :     return std::unique_ptr<Serializable>(dto.release());
     102            0 : }
     103              : 
     104              : } // namespace Hccl