Line data    Source code

       1              : /**
       2              :  * Copyright (c) 2025 Huawei Technologies Co., Ltd.
       3              :  * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
       4              :  * CANN Open Software License Agreement Version 2.0 (the "License").
       5              :  * Please refer to the License for details. You may not use this file except in compliance with the License.
       6              :  * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
       7              :  * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
       8              :  * See LICENSE in the root of the software repository for the full text of the License.
       9              :  */
      10              : 
      11              : #ifndef HCCLV2_INS_V2_BATCH_SEND_RECV_EXECUTOR_H
      12              : #define HCCLV2_INS_V2_BATCH_SEND_RECV_EXECUTOR_H
      13              : 
      14              : #include <unordered_set>
      15              : #include <algorithm>
      16              : 
      17              : #include "ins_coll_alg_base.h"
      18              : #include "ins_temp_all_gather_mesh.h"
      19              : #include "topo_match_partial_mesh.h"
      20              : #include "instruction.h"
      21              : #include "data_buffer.h"
      22              : #include "rmt_data_buffer_mgr.h"
      23              : 
      24              : namespace Hccl {
      25              : constexpr u32 MULTIPLY_TWO = 2;
      26              : constexpr u64 HCCL_CHUNK_SIZE = 1024 * 1024 * 1024; // 1024*1024*1024的size
      27              : 
      28              : template <typename AlgTopoMatch> class InsV2BatchSendRecvExecutor : public InsCollAlgBase {
      29              : public:
      30              :     explicit InsV2BatchSendRecvExecutor();
      31              :     ~InsV2BatchSendRecvExecutor() override;
      32              : 
      33            0 :     std::string Describe() const override
      34              :     {
      35            0 :         return "Instruction based Send Executor.";
      36              :     }
      37              : 
      38              :     HcclResult Orchestrate(const RankGraph *rankGraph, const CollAlgOperator &op, const CollAlgParams &params,
      39              :                           InsQuePtr insQue) override;
      40              : 
      41              :     HcclResult CalcResOffload(const RankGraph *rankGraph, const u64 &dataSize,
      42              :                               CollOffloadOpResReq &resReq) override;
      43              : 
      44              :     HcclResult CalcRes(const RankGraph *rankGraph, CollAlgResReq &algResReq) override;
      45              : 
      46              :     HcclResult Orchestrate(const AlgTopoInfo &topoInfo, const CollAlgOperator &op, const CollAlgParams &params,
      47              :                              ConnectedLinkMgr *linkMgr, InsQuePtr insQue) override;
      48              : 
      49              :     void SetRmaDataBufferMgr(const RmtDataBufferMgr* rmaDataBufferMgr) override;
      50              :     HcclResult CalNumBlocks(u32& blockDim, u64 dataSize, u32 blockDimLimit) override;
      51              : 
      52              : protected:
      53              :     struct SendRecvSlice {
      54              :         uintptr_t addr_;
      55              :         u64 size_;
      56              :         u32 remoteRank_;
      57              :         SendRecvSlice(uintptr_t addr, u64 size, u32 remoteRank) :
      58              :             addr_(addr), size_(size), remoteRank_(remoteRank) {}
      59              :     };
      60              : 
      61              :     u32 remoteUserRank_ = 0;
      62              :     const u32 MAX_LOOP_IN_ONCE_LAUNCH = 200;
      63              :     std::deque<SendRecvSlice> sendDataSilces_;
      64              :     std::deque<SendRecvSlice> recvDataSilces_;
      65              : 
      66              : private:
      67              :     bool SortSendItems(HcclSendRecvItem* a, HcclSendRecvItem* b) const;
      68              :     bool SortRecvItems(HcclSendRecvItem* a, HcclSendRecvItem* b) const;
      69              :     HcclResult InitParams(const CollAlgOperator &op, const CollAlgParams &params) override;
      70              :     HcclResult GetPairWiseList();
      71              : 
      72              :     // 收发数据准备
      73              :     HcclResult CalcSendSlices(u64 maxRoundTransferSize);
      74              :     HcclResult CalcRecvSlices(u64 maxRoundTransferSize);
      75              :     HcclResult GenSendSlicesMapRank();
      76              :     HcclResult GenRecvSlicesMapRank();
      77              : 
      78              :     // 实现自发自收
      79              :     HcclResult ProcessSelfSendRecvTasks(InsQuePtr& queue);
      80              : 
      81              :     // 实现数据发送&接收
      82              :     HcclResult ProcessSendRecv(const CollAlgOperator &op, InsQuePtr& queue, u32 remoteRank,
      83              :         std::vector<SendRecvSlice>& sendRemoteSlices,
      84              :         std::vector<SendRecvSlice>& recvRemoteSlices, LinkData& link) const;
      85              :     HcclResult ProcessSendDataSlice(InsQuePtr& queue, SendRecvSlice& sendRemoteSlice,
      86              :         u32 remoteRank, uint64_t scratchBufferAddr, LinkData& link) const;
      87              :     HcclResult SendRun(DataBuffer &execBufferSlice, u32 remoteUserRank, InsQuePtr& queue, LinkData& link) const;
      88              :     HcclResult CopyRecvDataSliceToUsrOut(InsQuePtr& queue, SendRecvSlice& slice,
      89              :         u32 remoteRank, uint64_t scratchBufferAddr) const;
      90              : 
      91              :     HcclResult RunLoopSendRecv(const CollAlgOperator &op, std::vector<InsQuePtr>& queues, InsTempAllGatherMesh1D& tempAlg);
      92              :     HcclResult CalcResLinksPartialMesh(const RankId myRank, const std::vector<std::vector<RankId>> &tempVTopo,
      93              :         const u32 linkNumBtwPeers, AlgTempResReq &tempResReq);
      94              :     HcclResult CalcRes(AlgTempResReq &tempResReq);
      95              :     HcclResult ExecAiv();
      96              : 
      97              :     HcclResult InitCommInfo(const RankGraph *rankGraph);
      98              :     HcclResult InitCommInfo(const AlgTopoInfo &topoInfo);
      99              : 
     100              :     std::set<u32> commTargetUserRankSet_;
     101              :     std::deque<HcclSendRecvItem*> sendToSelfDeque_;
     102              :     std::deque<HcclSendRecvItem*> recvFromSelfDeque_;
     103              :     std::deque<HcclSendRecvItem*> sendDeque_;
     104              :     std::deque<HcclSendRecvItem*> recvDeque_;
     105              :     BuffInfo buffInfo_;
     106              : 
     107              :     std::vector<RankId>              virtRanks_;
     108              :     std::map<RankId, u32>            virtRankMap_; // map<virtRank, virtRankOrder>
     109              :     std::vector<std::vector<RankId>> vTopo_;
     110              : 
     111              :     std::vector<InsQuePtr> requiredQue_;
     112              :     ResLinks               tempResLinks_;
     113              : 
     114              :     std::map<u32, std::vector<SendRecvSlice>> SendSliceMapByRemoteRank_;
     115              :     std::map<u32, std::vector<SendRecvSlice>> RecvSliceMapByRemoteRank_;
     116              :     u64 maxRoundTransferSize_ = 0; // 单轮最多能够传输的size
     117              :     u32 sliceId_{0};  // 用于组装aivTag
     118              : };
     119              : } // namespace Hccl
     120              : #endif // !HCCLV2_INS_V2_BATCH_SEND_RECV_EXECUTOR_H