Line data    Source code

       1              : /**
       2              :  * Copyright (c) 2025 Huawei Technologies Co., Ltd.
       3              :  * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
       4              :  * CANN Open Software License Agreement Version 2.0 (the "License").
       5              :  * Please refer to the License for details. You may not use this file except in compliance with the License.
       6              :  * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
       7              :  * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
       8              :  * See LICENSE in the root of the software repository for the full text of the License.
       9              :  */
      10              : #include "typical_mr_manager.h"
      11              : #include "adapter_rts_common.h"
      12              : #include "rdma_resource_manager.h"
      13              : 
      14              : namespace hccl {
      15              : 
      16           64 : TypicalMrManager &TypicalMrManager::GetInstance()
      17              : {
      18          130 :     static TypicalMrManager typicalMrManager[MAX_MODULE_DEVICE_NUM + 1];
      19           64 :     s32 deviceLogicId = INVALID_INT;
      20           64 :     HcclResult ret = hrtGetDevice(&deviceLogicId);
      21           64 :     if (ret == HCCL_SUCCESS && (static_cast<u32>(deviceLogicId) < MAX_MODULE_DEVICE_NUM)) {
      22           64 :         HCCL_INFO("[TypicalMrManager::GetInstance]deviceLogicID[%d]", deviceLogicId);
      23           64 :         return typicalMrManager[deviceLogicId];
      24              :     }
      25            0 :     HCCL_WARNING("[TypicalWindowMem::GetInstance]deviceLogicID[%d] is invalid, ret[%d]", deviceLogicId, ret);
      26            0 :     return typicalMrManager[MAX_MODULE_DEVICE_NUM];
      27              : }
      28              : 
      29           66 : TypicalMrManager::TypicalMrManager()
      30           66 :     : rdmaHandle_(nullptr)
      31              : {
      32           66 : }
      33              : 
      34           66 : TypicalMrManager::~TypicalMrManager()
      35              : {
      36           66 :     if (rdmaHandle_ != nullptr) {
      37            8 :         ReleaseMrResource();
      38              :     }
      39           66 : }
      40              : 
      41           30 : HcclResult TypicalMrManager::RegisterMem(struct MrInfoT &mrInfo)
      42              : {
      43           30 :     HCCL_DEBUG("[TypicalMrManager][RegisterMem]MR register start, addr[%llu], MR size[%llu].",
      44              :         mrInfo.addr, mrInfo.size);
      45           30 :     CHK_RET(RdmaResourceManager::GetInstance().GetRdmaHandle(rdmaHandle_));
      46           30 :     CHK_PTR_NULL(rdmaHandle_);
      47              :     // Check whether the key is the default value or already registered in map
      48           30 :     std::unique_lock<std::mutex> lockMrMap(mrMapMutex_);
      49           30 :     if (mrInfo.lkey != DEFAULT_MR_KEY) {
      50            0 :         auto mrIter = regedMrMap_.find(mrInfo.lkey);
      51            0 :         if (mrIter != regedMrMap_.end()) {
      52            0 :             HCCL_WARNING("[TypicalMrManager][RegisterMem]MR key[%lu] already registered, " \
      53              :                 "MR size[%llu], regedMrMap size[%u].",
      54              :                 mrInfo.lkey, mrIter->second.first.size, regedMrMap_.size());
      55            0 :             return HCCL_E_PARA;
      56              :         }
      57            0 :         HCCL_ERROR("[TypicalMrManager][RegisterMem]invalid MR info for register, addr[%llu], key[%lu].",
      58              :             mrInfo.addr, mrInfo.lkey);
      59            0 :         return HCCL_E_PARA;
      60              :     }
      61           30 :     MrHandle mrHandle = nullptr;
      62              :     // Register MR
      63           30 :     CHK_RET(hrtRaRegGlobalMr(rdmaHandle_, mrInfo, mrHandle));
      64           30 :     if (mrHandle == nullptr) {
      65            0 :         HCCL_WARNING("[TypicalMrManager][RegisterMem]MR register not success, addr[%llu], MR size[%llu].",
      66              :             mrInfo.addr, mrInfo.size);
      67            0 :         return HCCL_E_INTERNAL;
      68              :     }
      69           30 :     regedMrMap_[mrInfo.lkey].first = mrInfo;
      70           30 :     regedMrMap_[mrInfo.lkey].second = mrHandle;
      71           30 :     HCCL_INFO("[TypicalMrManager][RegisterMem]MR register success, " \
      72              :         "MR key[%llu], addr[%llu], size[%llu], mrHandle[%p], regedMrMap size[%u].",
      73              :         mrInfo.lkey, mrInfo.addr, mrInfo.size, mrHandle, regedMrMap_.size());
      74           30 :     return HCCL_SUCCESS;
      75           30 : }
      76              : 
      77           30 : HcclResult TypicalMrManager::DeRegisterMem(struct MrInfoT &mrInfo)
      78              : {
      79           30 :     HCCL_DEBUG("[TypicalMrManager][DeRegisterMem]MR deregister start, addr[%llu], size[%llu], key[%lu].",
      80              :         mrInfo.addr, mrInfo.size, mrInfo.lkey);
      81           30 :     CHK_RET(RdmaResourceManager::GetInstance().GetRdmaHandle(rdmaHandle_));
      82           30 :     CHK_PTR_NULL(rdmaHandle_);
      83              :     // Check whether the mem key exists. If exists，remove from MR map; else return error.
      84           30 :     std::unique_lock<std::mutex> lockMrMap(mrMapMutex_);
      85           30 :     auto mrIter = regedMrMap_.find(mrInfo.lkey);
      86           30 :     if (mrIter == regedMrMap_.end()) {
      87            0 :         HCCL_ERROR("[TypicalMrManager][DeRegisterMem]no match MR info in MR map, " \
      88              :             "MR key[%llu], addr[%llu], size[%llu], regedMrMap size[%u].",
      89              :             mrInfo.lkey, mrInfo.addr, mrInfo.size, regedMrMap_.size());
      90            0 :         return HCCL_E_PARA;
      91              :     }
      92              :     // Unregister MR
      93           30 :     MrHandle mrHandle = mrIter->second.second;
      94           30 :     HCCL_DEBUG("[TypicalMrManager][DeRegisterMem]rdma handle[%p], mr handle [%p].", rdmaHandle_, mrHandle);
      95           30 :     CHK_RET(hrtRaDeRegGlobalMr(rdmaHandle_, mrHandle));
      96           30 :     regedMrMap_.erase(mrIter);
      97           30 :     HCCL_INFO("[TypicalMrManager][DeRegisterMem]MR unregister success, regedMrMap size[%u].", regedMrMap_.size());
      98           30 :     mrInfo.lkey = DEFAULT_MR_KEY;
      99           30 :     return HCCL_SUCCESS;
     100           30 : }
     101              : 
     102            8 : HcclResult TypicalMrManager::ReleaseMrResource()
     103              : {
     104            8 :     CHK_PTR_NULL(rdmaHandle_);
     105            8 :     std::unique_lock<std::mutex> lockMrMap(mrMapMutex_);
     106            8 :     if (!regedMrMap_.empty()) {
     107            0 :         for (auto &mrIter : regedMrMap_) {
     108            0 :             MrHandle mrHandle = mrIter.second.second;
     109            0 :             if (mrHandle != nullptr) {
     110            0 :                 CHK_RET(hrtRaDeRegGlobalMr(rdmaHandle_, mrHandle));
     111              :             }
     112              :         }
     113            0 :         regedMrMap_.clear();
     114              :     }
     115            8 :     HCCL_INFO("[TypicalMrManager][ReleaseMrResource]release mr resources success.");
     116            8 :     return HCCL_SUCCESS;
     117            8 : }
     118              : }  // namespace hccl