Line data    Source code

       1              : /**
       2              :  * Copyright (c) 2025 Huawei Technologies Co., Ltd.
       3              :  * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
       4              :  * CANN Open Software License Agreement Version 2.0 (the "License").
       5              :  * Please refer to the License for details. You may not use this file except in compliance with the License.
       6              :  * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
       7              :  * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
       8              :  * See LICENSE in the root of the software repository for the full text of the License.
       9              :  */
      10              : #include "conn_local_notify_manager.h"
      11              : 
      12              : #include "rdma_handle_manager.h"
      13              : #include "communicator_impl.h"
      14              : #include "invalid_params_exception.h"
      15              : 
      16              : #include "ipc_local_notify.h"
      17              : #include "rdma_local_notify.h"
      18              : #include "ub_local_notify.h"
      19              : 
      20              : namespace Hccl {
      21              : 
      22          488 : ConnLocalNotifyManager::ConnLocalNotifyManager(CommunicatorImpl *communicator) : comm(communicator)
      23              : {
      24          488 : }
      25              : 
      26          488 : ConnLocalNotifyManager::~ConnLocalNotifyManager()
      27              : {
      28          488 :     DECTOR_TRY_CATCH("ConnLocalNotifyManager", Destroy());
      29          488 : }
      30              : 
      31           12 : bool ConnLocalNotifyManager::IsExist(RankId remoteRankId, const LinkData &linkData)
      32              : {
      33           12 :     return notifyPool.count(remoteRankId) != 0 && notifyPool[remoteRankId].count(linkData) != 0;
      34              : }
      35              : 
      36            6 : void ConnLocalNotifyManager::ApplyFor(RankId remoteRankId, const LinkData &linkData)
      37              : {
      38           18 :     HCCL_INFO("Local notify for remoteRankId[%d] and linkData[%s] alloc.",
      39              :                      remoteRankId, linkData.Describe().c_str());
      40            6 :     if (IsExist(remoteRankId, linkData)) {
      41            3 :         HCCL_WARNING("Local notify for remoteRankId[%d] and linkData[%s] already exists, no need to alloc.",
      42              :                      remoteRankId, linkData.Describe().c_str());
      43            1 :         return;
      44              :     }
      45              :         
      46            5 :     u32 count = 3; // 待修改: 需要定义GetCount()
      47            5 :     notifyPool[remoteRankId][linkData].resize(count);
      48              : 
      49           20 :     for (u32 i = 0; i < count; ++i) {
      50           15 :         if (linkData.GetType() == PortDeploymentType::P2P) {
      51           12 :             notifyPool[remoteRankId][linkData][i] = make_unique<IpcLocalNotify>(comm->GetOpAiCpuTSFeatureFlag());// 算子粒度
      52           12 :             continue;
      53            3 :         } else if (linkData.GetType() == PortDeploymentType::DEV_NET) {
      54            3 :             auto linkProtocol = linkData.GetLinkProtocol();
      55            3 :             if (linkProtocol == LinkProtocol::ROCE) {
      56            0 :                 RdmaHandle rdmaHandle = RdmaHandleManager::GetInstance().Get(
      57            0 :                     comm->GetDevicePhyId(), linkData.GetLocalPort(), linkProtocol);
      58            0 :                 notifyPool[remoteRankId][linkData][i] = make_unique<RdmaLocalNotify>(rdmaHandle, comm->GetOpAiCpuTSFeatureFlag()); // 算子粒度
      59            0 :                 continue;
      60            3 :             } else if (linkProtocol == LinkProtocol::UB_CTP || linkProtocol == LinkProtocol::UB_TP ||
      61            3 :                        linkProtocol == LinkProtocol::UBOE || linkProtocol == LinkProtocol::UBG) {
      62              :                 RdmaHandle rdmaHandle = 
      63            3 :                     RdmaHandleManager::GetInstance().Get(comm->GetDevicePhyId(), linkData.GetLocalPort(), linkProtocol);
      64            3 :                 notifyPool[remoteRankId][linkData][i] = make_unique<UbLocalNotify>(rdmaHandle, comm->GetOpAiCpuTSFeatureFlag()); // 算子粒度
      65              :             } else {
      66              :                 // 待修改: 仅支持 P2P 和 RDMA 申请 notify
      67            0 :                 string msg = StringFormat("Unsupported %s of link %s", linkProtocol.Describe().c_str(),
      68            0 :                                           linkData.Describe().c_str());
      69            0 :                 THROW<InvalidParamsException>(msg);
      70            0 :             }
      71              :         } else {
      72              :             // 待修改: 仅支持 P2P 和 RDMA 申请 notify
      73            0 :             string msg = StringFormat("Unsupported %s of link %s", linkData.GetType().Describe().c_str(),
      74            0 :                                       linkData.Describe().c_str());
      75            0 :             THROW<InvalidParamsException>(msg);
      76            0 :         }
      77              :     };
      78              : }
      79              : 
      80            1 : bool ConnLocalNotifyManager::Release(RankId remoteRankId, const LinkData &linkData)
      81              : {
      82            1 :     if (!IsExist(remoteRankId, linkData)) {
      83            0 :         HCCL_WARNING("Notify for remoteRankId[%d] and linkData[%p] does not exist, no need to release.", remoteRankId,
      84              :                      &linkData);
      85            0 :         return true;
      86              :     }
      87              : 
      88            1 :     notifyPool[remoteRankId].erase(linkData);
      89            1 :     if (notifyPool[remoteRankId].empty()) {
      90            1 :         notifyPool.erase(remoteRankId);
      91              :     }
      92            1 :     return true;
      93              : }
      94              : 
      95            5 : vector<BaseLocalNotify *> ConnLocalNotifyManager::Get(RankId remoteRankId, const LinkData &linkData)
      96              : {
      97            5 :     vector<BaseLocalNotify *> v;
      98              : 
      99            5 :     if (!IsExist(remoteRankId, linkData)) {
     100           15 :         HCCL_WARNING("Get Local notify for remoteRankId[%d] and linkData[%s] does not exist",
     101              :                      remoteRankId, linkData.Describe().c_str());
     102            5 :         return v;
     103              :     }
     104              : 
     105            0 :     for (const auto &i : notifyPool[remoteRankId][linkData]) {
     106            0 :         v.emplace_back(i.get());
     107              :     }
     108              : 
     109            0 :     return v;
     110            0 : }
     111              : 
     112          489 : bool ConnLocalNotifyManager::Destroy()
     113              : {
     114          489 :     notifyPool.clear();
     115          489 :     return true;
     116              : }
     117              : 
     118              : } // namespace Hccl