Line data    Source code

       1              : /**
       2              :  * Copyright (c) 2025 Huawei Technologies Co., Ltd.
       3              :  * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
       4              :  * CANN Open Software License Agreement Version 2.0 (the "License").
       5              :  * Please refer to the License for details. You may not use this file except in compliance with the License.
       6              :  * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
       7              :  * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
       8              :  * See LICENSE in the root of the software repository for the full text of the License.
       9              :  */
      10              : #ifndef USER_REMOTE_MEM_GETTER_H
      11              : #define USER_REMOTE_MEM_GETTER_H
      12              : 
      13              : #include <cstdint>
      14              : #include <vector>
      15              : #include <array>
      16              : #include <string>
      17              : #include <functional>
      18              : #include "hccl_mem_defs.h"
      19              : #include "hccl/hccl_types.h"
      20              : #include "log.h"
      21              : 
      22              : constexpr uint32_t MAX_BUFFER_NUM = 30000;
      23              : 
      24              : namespace Hccl {
      25              : 
      26              : template <typename T>
      27              : struct RemoteMemCtx {
      28              :     bool& cacheValid;
      29              :     std::vector<T>& rmtBufferVec;
      30              :     std::vector<CommMem>& remoteUserMems;
      31              :     std::vector<std::string>& memInfoCopies;
      32              :     std::vector<char*>& memInfoPointers;
      33              :     CommMem** remoteMem;
      34              :     char*** memInfos;
      35              :     uint32_t* memNum;
      36              : 
      37           28 :     RemoteMemCtx(
      38              :         bool& cacheValid, std::vector<T>& rmtBufferVec, std::vector<CommMem>& remoteUserMems,
      39              :         std::vector<std::string>& memInfoCopies, std::vector<char*>& memInfoPointers, CommMem** remoteMem,
      40              :         char*** memInfos, uint32_t* memNum)
      41           28 :         : cacheValid(cacheValid),
      42           28 :           rmtBufferVec(rmtBufferVec),
      43           28 :           remoteUserMems(remoteUserMems),
      44           28 :           memInfoCopies(memInfoCopies),
      45           28 :           memInfoPointers(memInfoPointers),
      46           28 :           remoteMem(remoteMem),
      47           28 :           memInfos(memInfos),
      48           28 :           memNum(memNum) {};
      49              : };
      50              : 
      51            0 : inline HcclMemType CommMemTypeToHcclMemType(CommMemType type)
      52              : {
      53            0 :     switch (type) {
      54            0 :         case COMM_MEM_TYPE_DEVICE:
      55            0 :             return HCCL_MEM_TYPE_DEVICE;
      56            0 :         case COMM_MEM_TYPE_HOST:
      57            0 :             return HCCL_MEM_TYPE_HOST;
      58            0 :         default:
      59            0 :             return HCCL_MEM_TYPE_NUM;
      60              :     }
      61              : }
      62              : 
      63           19 : inline CommMemType HcclMemTypeToCommMemType(HcclMemType type)
      64              : {
      65           19 :     switch (type) {
      66           10 :         case HCCL_MEM_TYPE_DEVICE:
      67           10 :             return COMM_MEM_TYPE_DEVICE;
      68            4 :         case HCCL_MEM_TYPE_HOST:
      69            4 :             return COMM_MEM_TYPE_HOST;
      70            5 :         default:
      71            5 :             return COMM_MEM_TYPE_INVALID;
      72              :     }
      73              : }
      74              : 
      75              : template <typename T>
      76           28 : HcclResult GetRemoteUserMems(RemoteMemCtx<T>& remoteMemCtx)
      77              : {
      78           28 :     CHK_PRT_RET(!remoteMemCtx.remoteMem, HCCL_ERROR("[GetRemoteUserMems] remoteMem is nullptr"), HCCL_E_PARA);
      79           25 :     CHK_PRT_RET(!remoteMemCtx.memInfos, HCCL_ERROR("[GetRemoteUserMems] memInfos is nullptr"), HCCL_E_PARA);
      80           22 :     CHK_PRT_RET(!remoteMemCtx.memNum, HCCL_ERROR("[GetRemoteUserMems] memNum is nullptr"), HCCL_E_PARA);
      81           18 :     *(remoteMemCtx.remoteMem) = nullptr;
      82           18 :     *(remoteMemCtx.memInfos) = nullptr;
      83           18 :     *(remoteMemCtx.memNum) = 0;
      84           18 :     uint32_t userMemCount = remoteMemCtx.rmtBufferVec.size();
      85           18 :     if (userMemCount == 0) {
      86            8 :         HCCL_WARNING("[%s] bufferNum is 0.", __func__);
      87            6 :         return HCCL_SUCCESS;
      88              :     }
      89              :     // 检查是否有缓存
      90           12 :     if (!remoteMemCtx.cacheValid) {
      91           11 :         remoteMemCtx.remoteUserMems.clear();
      92           11 :         remoteMemCtx.memInfoCopies.clear();
      93           11 :         remoteMemCtx.memInfoCopies.reserve(userMemCount);
      94           11 :         remoteMemCtx.memInfoPointers.clear();
      95           11 :         remoteMemCtx.memInfoPointers.reserve(userMemCount);
      96           32 :         for (uint32_t i = 0; i < userMemCount; ++i) {
      97           19 :             auto& rmtBuffer = remoteMemCtx.rmtBufferVec[i];
      98           19 :             if (rmtBuffer == nullptr) {
      99            0 :                 return HCCL_E_PTR;
     100              :             }
     101           19 :             CommMem mem{};
     102           19 :             mem.type = HcclMemTypeToCommMemType(rmtBuffer->GetMemType());
     103           19 :             mem.addr = reinterpret_cast<void*>(rmtBuffer->GetAddr());
     104           19 :             mem.size = rmtBuffer->GetSize();
     105           19 :             remoteMemCtx.remoteUserMems.push_back(mem);
     106           19 :             std::string memInfoCopy = rmtBuffer->GetMemInfo();
     107           19 :             remoteMemCtx.memInfoCopies.push_back(std::move(memInfoCopy));
     108           19 :             remoteMemCtx.memInfoPointers.push_back(const_cast<char*>(remoteMemCtx.memInfoCopies.back().c_str()));
     109           23 :             HCCL_INFO(
     110              :                 "[%s] Found buffer[addr:%p, size:%llu, memInfo:%s]", __func__, mem.addr, mem.size,
     111              :                 remoteMemCtx.memInfoCopies.back().c_str());
     112              :         }
     113           11 :         remoteMemCtx.cacheValid = true;
     114              :     }
     115           12 :     *(remoteMemCtx.remoteMem) = remoteMemCtx.remoteUserMems.data();
     116           12 :     *(remoteMemCtx.memInfos) = remoteMemCtx.memInfoPointers.data();
     117           12 :     *(remoteMemCtx.memNum) = userMemCount;
     118           12 :     return HCCL_SUCCESS;
     119              : }
     120              : } // namespace Hccl
     121              : 
     122              : #endif // USER_REMOTE_MEM_GETTER_H