Line data    Source code

       1              : /**
       2              :  * Copyright (c) 2025 Huawei Technologies Co., Ltd.
       3              :  * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
       4              :  * CANN Open Software License Agreement Version 2.0 (the "License").
       5              :  * Please refer to the License for details. You may not use this file except in compliance with the License.
       6              :  * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
       7              :  * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
       8              :  * See LICENSE in the root of the software repository for the full text of the License.
       9              :  */
      10              : 
      11              : #include "all_gather_v_graph_pipeline.h"
      12              : #include "alg_template_register.h"
      13              : 
      14              : constexpr u32 STEP_OFFSET_TWO = 2;
      15              : 
      16              : namespace hccl
      17              : {
      18            0 : AllGatherVGraphPipeline::AllGatherVGraphPipeline(const HcclDispatcher dispatcher) : AllGatherVPipeline(dispatcher) {}
      19              : 
      20            0 : HcclResult AllGatherVGraphPipeline::RunAsync()
      21              : {
      22            0 :     HCCL_INFO("[AllGatherVGraphPipeline][RunAsync]AllGatherRingMesh starts groupRankId[%u] ", userRank_);
      23              : 
      24            0 :     u32 unitSize = SIZE_TABLE[opInfo_->dataType];
      25              : 
      26              :     // step 0前置操作 : 所有卡本地数据从userIn-->userout
      27            0 :     DeviceMem locSrc = DeviceMem::create(usrInMemAddr_, memSliceCount_ * unitSize);
      28            0 :     u64 localOffset = userMemSlice_[userRank_].offset;
      29            0 :     DeviceMem locDst = DeviceMem::create(static_cast<void *>(static_cast<u8 *>(dmaMem_[1].ptr()) + localOffset),
      30            0 :                                                 memSliceCount_ * unitSize);
      31            0 :     CHK_RET(HcclD2DMemcpyAsync(dispatcher_, locDst, locSrc, stream_));
      32              : 
      33            0 :     for (u32 step = 0; step < interRankSize_; step++) {
      34            0 :         CHK_RET(MainRecordSub());
      35            0 :         CHK_RET(SubWaitMain());
      36            0 :         CHK_RET(ExecInterServer(step));
      37            0 :         CHK_RET(ExecIntraServer(step));
      38            0 :         CHK_RET(SubRecordMain());
      39            0 :         CHK_RET(MainWaitSub());
      40              :     }
      41              : 
      42            0 :     HCCL_INFO("[AllGatherVGraphPipeline][RunAsync]AllGatherVGraphPipeline finished groupRankId[%u] ", userRank_);
      43            0 :     return HCCL_SUCCESS;
      44            0 : }
      45              : 
      46            0 : HcclResult AllGatherVGraphPipeline::ExecInterServer(u32 step)
      47              : {
      48              :     // inter ring algo
      49            0 :     u32 prevInterRankId = (interRankId_ - 1 + interRankSize_) % interRankSize_;
      50            0 :     u32 nextInterRankId = (interRankId_ + 1) % interRankSize_;
      51            0 :     LINK prevInterLink = interLinks_[prevInterRankId];
      52            0 :     LINK nextInterLink = interLinks_[nextInterRankId];
      53            0 :     u64 serverRankOffset = intraRankId_ + (interRankId_ + interRankSize_ - step) % interRankSize_ * intraRankSize_;
      54            0 :     u64 serverOffsetByte = userMemSlice_[serverRankOffset].offset;
      55            0 :     u64 readRemoteOffset = intraRankId_ + (prevInterRankId + interRankSize_ - step) % interRankSize_ *
      56            0 :                                                 intraRankSize_; // sever间前通信rank偏移
      57            0 :     u64 readRemoteOffsetByte = userMemSlice_[readRemoteOffset].offset;
      58            0 :     if (step < interRankSize_ - 1) {
      59            0 :         CHK_RET(prevInterLink->TxAck(subStream_[0])); // AckRecord
      60            0 :         CHK_RET(nextInterLink->RxAck(subStream_[0])); // AckWait
      61              :         // RdmaSend + Record 或 PCIE::Record
      62            0 :         CHK_RET(nextInterLink->TxAsync(UserMemType::OUTPUT_MEM,
      63              :                                         serverOffsetByte, static_cast<u8 *>(dmaMem_[1].ptr()) + serverOffsetByte,
      64              :                                         userMemSlice_[serverRankOffset].size, subStream_[0]));
      65            0 :         HCCL_DEBUG("[AllGatherVGraphPipeline][RunAsync] local rank[%u] localOffset[%llu]tx with remoteRank[%u],"
      66              :                     "remoteOffset[%llu] with slice[%llu]",
      67              :                     userRank_, serverOffsetByte, nextInterRankId,
      68              :                     serverOffsetByte, userMemSlice_[serverRankOffset].size);
      69              :         // 对于RDMA RxAsync，内存属性入参无效 RDMA::Wait
      70              :         // 对于PCIE，需设置内存属性 PCIE::Read + Record
      71            0 :         CHK_RET(prevInterLink->RxAsync(UserMemType::OUTPUT_MEM,
      72              :                                         readRemoteOffsetByte, static_cast<u8 *>(dmaMem_[1].ptr()) + readRemoteOffsetByte,
      73              :                                         userMemSlice_[readRemoteOffset].size, subStream_[0])) ; // wait
      74            0 :         HCCL_DEBUG("[AllGatherVGraphPipeline][RunAsync]read local rank[%u] localOffset[%llu]tx with remoteRank[%u],"
      75              :                     "remoteOffset[%llu] with slice[%llu]",
      76              :                     userRank_, readRemoteOffsetByte, readRemoteOffset,
      77              :                     readRemoteOffsetByte, userMemSlice_[readRemoteOffset].size);
      78            0 :         CHK_RET(prevInterLink->PostFinAck(subStream_[0]));
      79            0 :         CHK_RET(nextInterLink->WaitFinAck(subStream_[0]));
      80              :         // inter的最后一步需要barrier确保数据发完
      81            0 :         if (step == interRankSize_ - STEP_OFFSET_TWO) {
      82            0 :             CHK_RET(ExecuteBarrier(prevInterLink, nextInterLink, subStream_[0]));
      83              :         }
      84              :     }
      85            0 :     return HCCL_SUCCESS;
      86            0 : }
      87              : 
      88            0 : HcclResult AllGatherVGraphPipeline::ExecIntraServer(u32 step)
      89              : {
      90            0 :     for (u32 i = 1; i < intraRankSize_; i++) {
      91            0 :         u32 remIntraRankId = (intraRankId_ + i) % intraRankSize_;
      92            0 :         CHK_RET(intraLinks_[remIntraRankId]->TxAck(subStream_[i]));  // ackrecord
      93            0 :         CHK_RET(intraLinks_[remIntraRankId]->RxAck(subStream_[i]));  // ackwait
      94              : 
      95            0 :         void *remDMAMemPtr = nullptr;
      96            0 :         CHK_RET(intraLinks_[remIntraRankId]->GetRemoteMem(UserMemType::OUTPUT_MEM, &remDMAMemPtr));
      97            0 :         u64 remoteRankOffset =
      98            0 :             (interRankId_ - step + interRankSize_) % interRankSize_ * intraRankSize_ + remIntraRankId;
      99            0 :         u64 remoteOffsetByte = userMemSlice_[remoteRankOffset].offset;
     100            0 :         void *dstAddr = static_cast<u8 *>(usrOutMemAddr_) + remoteOffsetByte;
     101              : 
     102            0 :         DeviceMem src = DeviceMem::create(static_cast<u8 *>(remDMAMemPtr) + remoteOffsetByte, userMemSlice_[remoteRankOffset].size);
     103            0 :         DeviceMem dst = DeviceMem::create(dstAddr, userMemSlice_[remoteRankOffset].size);
     104            0 :         CHK_RET(HcclD2DMemcpyAsync(dispatcher_, dst, src, subStream_[i]));
     105              : 
     106            0 :         CHK_RET(intraLinks_[remIntraRankId]->TxDataSignal(subStream_[i]));  // data record
     107            0 :         CHK_RET(intraLinks_[remIntraRankId]->RxDataSignal(subStream_[i]));  // data wait
     108            0 :     }
     109            0 :     return HCCL_SUCCESS;
     110              : }
     111              : 
     112              : REGISTER_TEMPLATE(TemplateType::TEMPLATE_ALL_GATHER_V_GRAPH_PIPELINE, AllGatherVGraphPipeline);
     113              : } // namespace hccl