Line data    Source code

       1              : /**
       2              :  * Copyright (c) 2025 Huawei Technologies Co., Ltd.
       3              :  * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
       4              :  * CANN Open Software License Agreement Version 2.0 (the "License").
       5              :  * Please refer to the License for details. You may not use this file except in compliance with the License.
       6              :  * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
       7              :  * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
       8              :  * See LICENSE in the root of the software repository for the full text of the License.
       9              :  */
      10              : #include "typical_mr_manager.h"
      11              : #include "adapter_rts_common.h"
      12              : #include "rdma_resource_manager.h"
      13              : 
      14              : namespace hccl {
      15              : 
      16           64 : TypicalMrManager& TypicalMrManager::GetInstance()
      17              : {
      18          130 :     static TypicalMrManager typicalMrManager[MAX_MODULE_DEVICE_NUM + 1];
      19           64 :     s32 deviceLogicId = INVALID_INT;
      20           64 :     HcclResult ret = hrtGetDevice(&deviceLogicId);
      21           64 :     if (ret == HCCL_SUCCESS && (static_cast<u32>(deviceLogicId) < MAX_MODULE_DEVICE_NUM)) {
      22           64 :         HCCL_INFO("[TypicalMrManager::GetInstance]deviceLogicID[%d]", deviceLogicId);
      23           64 :         return typicalMrManager[deviceLogicId];
      24              :     }
      25            0 :     HCCL_WARNING("[TypicalWindowMem::GetInstance]deviceLogicID[%d] is invalid, ret[%d]", deviceLogicId, ret);
      26            0 :     return typicalMrManager[MAX_MODULE_DEVICE_NUM];
      27              : }
      28              : 
      29           66 : TypicalMrManager::TypicalMrManager() : rdmaHandle_(nullptr) {}
      30              : 
      31           66 : TypicalMrManager::~TypicalMrManager()
      32              : {
      33           66 :     if (rdmaHandle_ != nullptr) {
      34            8 :         ReleaseMrResource();
      35              :     }
      36           66 : }
      37              : 
      38           30 : HcclResult TypicalMrManager::RegisterMem(struct MrInfoT& mrInfo)
      39              : {
      40           30 :     HCCL_DEBUG(
      41              :         "[TypicalMrManager][RegisterMem]MR register start, addr[%llu], MR size[%llu].", mrInfo.addr, mrInfo.size);
      42           30 :     CHK_RET(RdmaResourceManager::GetInstance().GetRdmaHandle(rdmaHandle_));
      43           30 :     CHK_PTR_NULL(rdmaHandle_);
      44              :     // Check whether the key is the default value or already registered in map
      45           30 :     std::unique_lock<std::mutex> lockMrMap(mrMapMutex_);
      46           30 :     if (mrInfo.lkey != DEFAULT_MR_KEY) {
      47            0 :         auto mrIter = regedMrMap_.find(mrInfo.lkey);
      48            0 :         if (mrIter != regedMrMap_.end()) {
      49            0 :             HCCL_WARNING(
      50              :                 "[TypicalMrManager][RegisterMem]MR key[%lu] already registered, "
      51              :                 "MR size[%llu], regedMrMap size[%u].",
      52              :                 mrInfo.lkey, mrIter->second.first.size, regedMrMap_.size());
      53            0 :             return HCCL_E_PARA;
      54              :         }
      55            0 :         HCCL_ERROR(
      56              :             "[TypicalMrManager][RegisterMem]invalid MR info for register, addr[%llu], key[%lu].", mrInfo.addr,
      57              :             mrInfo.lkey);
      58            0 :         return HCCL_E_PARA;
      59              :     }
      60           30 :     MrHandle mrHandle = nullptr;
      61              :     // Register MR
      62           30 :     CHK_RET(hrtRaRegGlobalMr(rdmaHandle_, mrInfo, mrHandle));
      63           30 :     if (mrHandle == nullptr) {
      64            0 :         HCCL_WARNING(
      65              :             "[TypicalMrManager][RegisterMem]MR register not success, addr[%llu], MR size[%llu].", mrInfo.addr,
      66              :             mrInfo.size);
      67            0 :         return HCCL_E_INTERNAL;
      68              :     }
      69           30 :     regedMrMap_[mrInfo.lkey].first = mrInfo;
      70           30 :     regedMrMap_[mrInfo.lkey].second = mrHandle;
      71           30 :     HCCL_INFO(
      72              :         "[TypicalMrManager][RegisterMem]MR register success, "
      73              :         "MR key[%llu], addr[%llu], size[%llu], mrHandle[%p], regedMrMap size[%u].",
      74              :         mrInfo.lkey, mrInfo.addr, mrInfo.size, mrHandle, regedMrMap_.size());
      75           30 :     return HCCL_SUCCESS;
      76           30 : }
      77              : 
      78           30 : HcclResult TypicalMrManager::DeRegisterMem(struct MrInfoT& mrInfo)
      79              : {
      80           30 :     HCCL_DEBUG(
      81              :         "[TypicalMrManager][DeRegisterMem]MR deregister start, addr[%llu], size[%llu], key[%lu].", mrInfo.addr,
      82              :         mrInfo.size, mrInfo.lkey);
      83           30 :     CHK_RET(RdmaResourceManager::GetInstance().GetRdmaHandle(rdmaHandle_));
      84           30 :     CHK_PTR_NULL(rdmaHandle_);
      85              :     // Check whether the mem key exists. If exists，remove from MR map; else return error.
      86           30 :     std::unique_lock<std::mutex> lockMrMap(mrMapMutex_);
      87           30 :     auto mrIter = regedMrMap_.find(mrInfo.lkey);
      88           30 :     if (mrIter == regedMrMap_.end()) {
      89            0 :         HCCL_ERROR(
      90              :             "[TypicalMrManager][DeRegisterMem]no match MR info in MR map, "
      91              :             "MR key[%llu], addr[%llu], size[%llu], regedMrMap size[%u].",
      92              :             mrInfo.lkey, mrInfo.addr, mrInfo.size, regedMrMap_.size());
      93            0 :         return HCCL_E_PARA;
      94              :     }
      95              :     // Unregister MR
      96           30 :     MrHandle mrHandle = mrIter->second.second;
      97           30 :     HCCL_DEBUG("[TypicalMrManager][DeRegisterMem]rdma handle[%p], mr handle [%p].", rdmaHandle_, mrHandle);
      98           30 :     CHK_RET(hrtRaDeRegGlobalMr(rdmaHandle_, mrHandle));
      99           30 :     regedMrMap_.erase(mrIter);
     100           30 :     HCCL_INFO("[TypicalMrManager][DeRegisterMem]MR unregister success, regedMrMap size[%u].", regedMrMap_.size());
     101           30 :     mrInfo.lkey = DEFAULT_MR_KEY;
     102           30 :     return HCCL_SUCCESS;
     103           30 : }
     104              : 
     105            8 : HcclResult TypicalMrManager::ReleaseMrResource()
     106              : {
     107            8 :     CHK_PTR_NULL(rdmaHandle_);
     108            8 :     std::unique_lock<std::mutex> lockMrMap(mrMapMutex_);
     109            8 :     if (!regedMrMap_.empty()) {
     110            0 :         for (auto& mrIter : regedMrMap_) {
     111            0 :             MrHandle mrHandle = mrIter.second.second;
     112            0 :             if (mrHandle != nullptr) {
     113            0 :                 CHK_RET(hrtRaDeRegGlobalMr(rdmaHandle_, mrHandle));
     114              :             }
     115              :         }
     116            0 :         regedMrMap_.clear();
     117              :     }
     118            8 :     HCCL_INFO("[TypicalMrManager][ReleaseMrResource]release mr resources success.");
     119            8 :     return HCCL_SUCCESS;
     120            8 : }
     121              : } // namespace hccl