Line data    Source code

       1              : /**
       2              :  * Copyright (c) 2025 Huawei Technologies Co., Ltd.
       3              :  * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
       4              :  * CANN Open Software License Agreement Version 2.0 (the "License").
       5              :  * Please refer to the License for details. You may not use this file except in compliance with the License.
       6              :  * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
       7              :  * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
       8              :  * See LICENSE in the root of the software repository for the full text of the License.
       9              :  */
      10              : 
      11              : #ifndef TRANSPORT_REMOTE_ACCESS_H
      12              : #define TRANSPORT_REMOTE_ACCESS_H
      13              : 
      14              : #include "transport_ibverbs.h"
      15              : #include "dispatcher_pub.h"
      16              : 
      17              : namespace hccl {
      18              : constexpr u32 MAX_HDC_CHANEL_NUM = 1024;
      19              : constexpr u32 NOTIFY_BUFFER_SIZE = 8;
      20              : constexpr u32 SEND_WRLIST_MAX_COUNT = 100;
      21              : 
      22              : using RemoteAccessPara = struct TagRemoteAccessParaS {
      23              : public:
      24              :     u32 role{0};                          // server:0 client:1
      25              :     HcclIpAddress localIp;                // 本端rank ip
      26              :     u32 localRank{INVALID_VALUE_RANKID};  // 本端rankuser rank
      27              :     u32 remoteRank{INVALID_VALUE_RANKID}; // 对端rankuser rank
      28              :     FdHandle socketFdhandle{nullptr};
      29              :     SocketHandle nicSocketHandle{nullptr};
      30              :     RdmaHandle nicRdmaHandle{nullptr};
      31              :     RaResourceInfo raResourceInfo;
      32              : 
      33            0 :     TagRemoteAccessParaS() {}
      34              : 
      35            0 :     TagRemoteAccessParaS(const struct TagRemoteAccessParaS& that)
      36            0 :     {
      37            0 :         role = (that.role);
      38            0 :         localIp = (that.localIp);
      39            0 :         localRank = (that.localRank);
      40            0 :         remoteRank = (that.remoteRank);
      41            0 :         socketFdhandle = (that.socketFdhandle);
      42            0 :         nicSocketHandle = (that.nicSocketHandle);
      43            0 :         nicRdmaHandle = (that.nicRdmaHandle);
      44            0 :         raResourceInfo = (that.raResourceInfo);
      45            0 :     }
      46              : 
      47              :     struct TagRemoteAccessParaS& operator=(struct TagRemoteAccessParaS& that)
      48              :     {
      49              :         if (&that != this) {
      50              :             role = (that.role);
      51              :             localIp = (that.localIp);
      52              :             localRank = (that.localRank);
      53              :             remoteRank = (that.remoteRank);
      54              :             socketFdhandle = (that.socketFdhandle);
      55              :             nicSocketHandle = (that.nicSocketHandle);
      56              :             nicRdmaHandle = (that.nicRdmaHandle);
      57              :             raResourceInfo = (that.raResourceInfo);
      58              :         }
      59              : 
      60              :         return *this;
      61              :     }
      62              : };
      63              : 
      64              : struct NotifyMsg {
      65              :     void* addr{nullptr};
      66              :     u64 len{0};
      67              :     s32 mrRegFlag{0};
      68              :     u64 offset{0};
      69              : };
      70              : 
      71              : class TransportRemoteAccess {
      72              : public:
      73              :     explicit TransportRemoteAccess(
      74              :         const std::string tag, const HcclDispatcher dispatcher, const std::unique_ptr<NotifyPool>& notifyPool_,
      75              :         const RemoteAccessPara& remoteAccessPara, const std::vector<MemRegisterAddr>& memRegistInfos,
      76              :         s32 deviceLogicId);
      77              :     ~TransportRemoteAccess();
      78              :     HcclResult Init();
      79              :     HcclResult RemoteRead(const std::vector<HcomRemoteAccessAddrInfo>& addrInfos, Stream& stream);
      80              :     HcclResult RemoteWrite(const std::vector<HcomRemoteAccessAddrInfo>& addrInfos, Stream& stream);
      81              : 
      82              : private:
      83              :     HcclResult CreateQp();
      84              :     HcclResult MrRegister();
      85              :     HcclResult NotifyRegister();
      86              :     HcclResult GetRemoteNotifyInfo();
      87              :     HcclResult SetLocalNotify();
      88              :     HcclResult CreateNotify();
      89              :     HcclResult CreateNotifyValueBuffer();
      90              :     HcclResult RdmaDataTransport(const std::vector<HcomRemoteAccessAddrInfo>& addrInfos, s32 rdmaOp);
      91              :     HcclResult ReadRemoteNotifyBuffer();
      92              :     HcclResult ConnectQp();
      93              :     const HcclDispatcher dispatcher_;
      94              :     const std::unique_ptr<NotifyPool>& notifyPool_;
      95              :     const std::vector<MemRegisterAddr> MemRegistInfos_; // 本端注册地址
      96              :     RemoteAccessPara RemoteAccessPara_;
      97              :     QpHandle handle_; // QP handle
      98              :     std::shared_ptr<LocalIpcNotify> ackNotify_;
      99              :     s32 access_;                     // rdma 访问权限，支持本端/远端写数据、远端读数据
     100              :     u32 notifySize_;                 // notify buffer size，8字节
     101              :     std::vector<void*> localRegMem_; // 本端注册成功的地址信息，用于解注册
     102              :     NotifyMsg ackNotifyMsg_;         // 本端notify的信息
     103              :     NotifyMsg remoteNotifyDataMsg_;
     104              :     static std::array<DeviceMem, MAX_MODULE_DEVICE_NUM> notifyValueMem_;
     105              :     static std::array<std::mutex, MAX_MODULE_DEVICE_NUM> notifyValueMutex_;
     106              :     const u64 notifyValueSize_{LARGE_PAGE_MEMORY_MIN_SIZE};            // 避免申请小页内存。最小2*1024*1024
     107              :     static std::array<Referenced, MAX_MODULE_DEVICE_NUM> instanceRef_; // 实例计数，用于释放静态资源
     108              :     const std::string tag_;
     109              :     const std::chrono::seconds timeout_;
     110              :     s32 deviceLogicId_;
     111              : };
     112              : } // namespace hccl
     113              : 
     114              : #endif // TRANSPORT_REMOTE_ACCESS_H