Line data    Source code

       1              : /**
       2              :  * Copyright (c) 2026 Huawei Technologies Co., Ltd.
       3              :  * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
       4              :  * CANN Open Software License Agreement Version 2.0 (the "License").
       5              :  * Please refer to the License for details. You may not use this file except in compliance with the License.
       6              :  * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
       7              :  * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
       8              :  * See LICENSE in the root of the software repository for the full text of the License.
       9              :  */
      10              : 
      11              : #ifndef AICPU_TS_ROCE_MEM_H
      12              : #define AICPU_TS_ROCE_MEM_H
      13              : 
      14              : #include <cstring>
      15              : #include <map>
      16              : #include <memory>
      17              : #include <string>
      18              : #include <unordered_map>
      19              : #include <vector>
      20              : #include "reged_mem_mgr.h"
      21              : #include "hccl_mem_defs.h"
      22              : #include "hccl_network.h"
      23              : #include "transport_pub.h"
      24              : #include "local_rdma_rma_buffer.h"
      25              : #include "remote_rdma_rma_buffer.h"
      26              : 
      27              : namespace hcomm {
      28              : class AicpuTsRoceRegedMemMgr : public RegedMemMgr {
      29              : public:
      30              :     AicpuTsRoceRegedMemMgr(HcclNetDev netDev, RdmaHandle rdmaHandle);
      31           25 :     ~AicpuTsRoceRegedMemMgr() override = default;
      32              : 
      33              :     HcclResult RegisterMemory(HcommMem mem, const char* memTag, void** memHandle) override;
      34              :     HcclResult UnregisterMemory(void* memHandle) override;
      35              :     HcclResult
      36              :     MemoryExport(const EndpointDesc endpointDesc, void* memHandle, void** memDesc, uint32_t* memDescLen) override;
      37              :     HcclResult MemoryImport(const void* memDesc, uint32_t descLen, HcommMem* outMem) override;
      38              :     HcclResult MemoryUnimport(const void* memDesc, uint32_t descLen) override;
      39              :     HcclResult GetAllMemHandles(void** memHandles, uint32_t* memHandleNum) override;
      40              : 
      41              :     HcclResult GetAllMemDetails(std::vector<RoceMemDetails>& localOut, std::vector<RoceMemDetails>& remoteOut) const;
      42              : 
      43              : private:
      44              :     using RemoteRdmaRmaBufferMgr
      45              :         = hcomm::RmaBufferMgr<hccl::BufferKey<uintptr_t, u64>, std::shared_ptr<hccl::RemoteRdmaRmaBuffer>>;
      46              : 
      47              :     struct EndpointDescLess {
      48            0 :         bool operator()(const EndpointDesc& a, const EndpointDesc& b) const noexcept
      49              :         {
      50            0 :             return std::memcmp(&a, &b, sizeof(EndpointDesc)) < 0;
      51              :         }
      52              :     };
      53              : 
      54              :     HcclResult
      55              :     GetParamsFromMemDesc(const void* memDesc, uint32_t descLen, EndpointDesc& endpointDesc, std::string& rdmaBlob);
      56              :     void TrackRegisteredBuffer(const std::shared_ptr<hccl::LocalRdmaRmaBuffer>& localBuffer);
      57              : 
      58              :     HcclResult GatherLocalMemDetails(std::vector<RoceMemDetails>& localOut) const;
      59              :     HcclResult AppendLocalNotifyMemDetails(std::vector<RoceMemDetails>& localOut) const;
      60              :     void GatherRemoteMemDetails(std::vector<RoceMemDetails>& remoteOut) const;
      61              : 
      62              :     HcclNetDev netDev_{nullptr};
      63              :     std::shared_ptr<hccl::NetDevContext::LocalRdmaRmaBufferMgr> localRdmaRmaBufferMgr_{nullptr};
      64              :     std::vector<RegedBufferEntry<hccl::LocalRdmaRmaBuffer>> allRegisteredBuffers_;
      65              :     std::vector<HcclBuf> hcclBufRecords_;
      66              :     std::unordered_map<hccl::LocalRdmaRmaBuffer*, std::vector<char>> exportDescByBuffer_;
      67              :     std::map<EndpointDesc, std::unique_ptr<RemoteRdmaRmaBufferMgr>, EndpointDescLess> remoteRdmaRmaBufferMgrs_;
      68              : };
      69              : 
      70              : } // namespace hcomm
      71              : #endif // AICPU_TS_ROCE_MEM_H