Line data    Source code

       1              : /**
       2              :  * Copyright (c) 2025 Huawei Technologies Co., Ltd.
       3              :  * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
       4              :  * CANN Open Software License Agreement Version 2.0 (the "License").
       5              :  * Please refer to the License for details. You may not use this file except in compliance with the License.
       6              :  * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
       7              :  * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
       8              :  * See LICENSE in the root of the software repository for the full text of the License.
       9              :  */
      10              : 
      11              : #include "typical_mr_manager.h"
      12              : #include "adapter_rts_common.h"
      13              : #include "rdma_resource_manager.h"
      14              : 
      15              : namespace hccl {
      16              : 
      17           64 : TypicalMrManager& TypicalMrManager::GetInstance()
      18              : {
      19          130 :     static TypicalMrManager typicalMrManager[MAX_MODULE_DEVICE_NUM + 1];
      20           64 :     s32 deviceLogicId = INVALID_INT;
      21           64 :     HcclResult ret = hrtGetDevice(&deviceLogicId);
      22           64 :     if (ret == HCCL_SUCCESS && (static_cast<u32>(deviceLogicId) < MAX_MODULE_DEVICE_NUM)) {
      23           64 :         HCCL_INFO("[TypicalMrManager::GetInstance]deviceLogicID[%d]", deviceLogicId);
      24           64 :         return typicalMrManager[deviceLogicId];
      25              :     }
      26            0 :     HCCL_WARNING("[TypicalWindowMem::GetInstance]deviceLogicID[%d] is invalid, ret[%d]", deviceLogicId, ret);
      27            0 :     return typicalMrManager[MAX_MODULE_DEVICE_NUM];
      28              : }
      29              : 
      30           66 : TypicalMrManager::TypicalMrManager() : rdmaHandle_(nullptr) {}
      31              : 
      32           66 : TypicalMrManager::~TypicalMrManager()
      33              : {
      34           66 :     if (rdmaHandle_ != nullptr) {
      35            8 :         ReleaseMrResource();
      36              :     }
      37           66 : }
      38              : 
      39           30 : HcclResult TypicalMrManager::RegisterMem(struct MrInfoT& mrInfo)
      40              : {
      41           30 :     HCCL_DEBUG(
      42              :         "[TypicalMrManager][RegisterMem]MR register start, addr[%llu], MR size[%llu].", mrInfo.addr, mrInfo.size);
      43           30 :     CHK_RET(RdmaResourceManager::GetInstance().GetRdmaHandle(rdmaHandle_));
      44           30 :     CHK_PTR_NULL(rdmaHandle_);
      45              :     // Check whether the key is the default value or already registered in map
      46           30 :     std::unique_lock<std::mutex> lockMrMap(mrMapMutex_);
      47           30 :     if (mrInfo.lkey != DEFAULT_MR_KEY) {
      48            0 :         auto mrIter = regedMrMap_.find(mrInfo.lkey);
      49            0 :         if (mrIter != regedMrMap_.end()) {
      50            0 :             HCCL_WARNING(
      51              :                 "[TypicalMrManager][RegisterMem]MR key[%lu] already registered, "
      52              :                 "MR size[%llu], regedMrMap size[%u].",
      53              :                 mrInfo.lkey, mrIter->second.first.size, regedMrMap_.size());
      54            0 :             return HCCL_E_PARA;
      55              :         }
      56            0 :         HCCL_ERROR(
      57              :             "[TypicalMrManager][RegisterMem]invalid MR info for register, addr[%llu], key[%lu].", mrInfo.addr,
      58              :             mrInfo.lkey);
      59            0 :         return HCCL_E_PARA;
      60              :     }
      61           30 :     MrHandle mrHandle = nullptr;
      62              :     // Register MR
      63           30 :     CHK_RET(hrtRaRegGlobalMr(rdmaHandle_, mrInfo, mrHandle));
      64           30 :     if (mrHandle == nullptr) {
      65            0 :         HCCL_WARNING(
      66              :             "[TypicalMrManager][RegisterMem]MR register not success, addr[%llu], MR size[%llu].", mrInfo.addr,
      67              :             mrInfo.size);
      68            0 :         return HCCL_E_INTERNAL;
      69              :     }
      70           30 :     regedMrMap_[mrInfo.lkey].first = mrInfo;
      71           30 :     regedMrMap_[mrInfo.lkey].second = mrHandle;
      72           30 :     HCCL_INFO(
      73              :         "[TypicalMrManager][RegisterMem]MR register success, "
      74              :         "MR key[%llu], addr[%llu], size[%llu], mrHandle[%p], regedMrMap size[%u].",
      75              :         mrInfo.lkey, mrInfo.addr, mrInfo.size, mrHandle, regedMrMap_.size());
      76           30 :     return HCCL_SUCCESS;
      77           30 : }
      78              : 
      79           30 : HcclResult TypicalMrManager::DeRegisterMem(struct MrInfoT& mrInfo)
      80              : {
      81           30 :     HCCL_DEBUG(
      82              :         "[TypicalMrManager][DeRegisterMem]MR deregister start, addr[%llu], size[%llu], key[%lu].", mrInfo.addr,
      83              :         mrInfo.size, mrInfo.lkey);
      84           30 :     CHK_RET(RdmaResourceManager::GetInstance().GetRdmaHandle(rdmaHandle_));
      85           30 :     CHK_PTR_NULL(rdmaHandle_);
      86              :     // Check whether the mem key exists. If exists，remove from MR map; else return error.
      87           30 :     std::unique_lock<std::mutex> lockMrMap(mrMapMutex_);
      88           30 :     auto mrIter = regedMrMap_.find(mrInfo.lkey);
      89           30 :     if (mrIter == regedMrMap_.end()) {
      90            0 :         HCCL_ERROR(
      91              :             "[TypicalMrManager][DeRegisterMem]no match MR info in MR map, "
      92              :             "MR key[%llu], addr[%llu], size[%llu], regedMrMap size[%u].",
      93              :             mrInfo.lkey, mrInfo.addr, mrInfo.size, regedMrMap_.size());
      94            0 :         return HCCL_E_PARA;
      95              :     }
      96              :     // Unregister MR
      97           30 :     MrHandle mrHandle = mrIter->second.second;
      98           30 :     HCCL_DEBUG("[TypicalMrManager][DeRegisterMem]rdma handle[%p], mr handle [%p].", rdmaHandle_, mrHandle);
      99           30 :     CHK_RET(hrtRaDeRegGlobalMr(rdmaHandle_, mrHandle));
     100           30 :     regedMrMap_.erase(mrIter);
     101           30 :     HCCL_INFO("[TypicalMrManager][DeRegisterMem]MR unregister success, regedMrMap size[%u].", regedMrMap_.size());
     102           30 :     mrInfo.lkey = DEFAULT_MR_KEY;
     103           30 :     return HCCL_SUCCESS;
     104           30 : }
     105              : 
     106            8 : HcclResult TypicalMrManager::ReleaseMrResource()
     107              : {
     108            8 :     CHK_PTR_NULL(rdmaHandle_);
     109            8 :     std::unique_lock<std::mutex> lockMrMap(mrMapMutex_);
     110            8 :     if (!regedMrMap_.empty()) {
     111            0 :         for (auto& mrIter : regedMrMap_) {
     112            0 :             MrHandle mrHandle = mrIter.second.second;
     113            0 :             if (mrHandle != nullptr) {
     114            0 :                 CHK_RET(hrtRaDeRegGlobalMr(rdmaHandle_, mrHandle));
     115              :             }
     116              :         }
     117            0 :         regedMrMap_.clear();
     118              :     }
     119            8 :     HCCL_INFO("[TypicalMrManager][ReleaseMrResource]release mr resources success.");
     120            8 :     return HCCL_SUCCESS;
     121            8 : }
     122              : } // namespace hccl