Line data    Source code

       1              : /**
       2              :  * Copyright (c) 2025 Huawei Technologies Co., Ltd.
       3              :  * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
       4              :  * CANN Open Software License Agreement Version 2.0 (the "License").
       5              :  * Please refer to the License for details. You may not use this file except in compliance with the License.
       6              :  * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
       7              :  * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
       8              :  * See LICENSE in the root of the software repository for the full text of the License.
       9              :  */
      10              : 
      11              : #ifndef HCOM_COMMOM_H
      12              : #define HCOM_COMMOM_H
      13              : 
      14              : #include "hccl_comm_pub.h"
      15              : #include "hcom_common_v2.h"
      16              : #include "../common/src/topo/topoinfo_detect.h"
      17              : 
      18              : #include "topoinfo_struct.h"
      19              : 
      20              : // Ge适配的类
      21              : constexpr u32 SEND_RECEIVE_TASK_NUM = 20;
      22              : constexpr u32 OP_DEFAULT_TASK_NUM = 245;
      23              : constexpr u32 AIV_DEFAULT_TASK_NUM = 4; // 在AIV且非RDMA场景下，task数量固定为4
      24              : constexpr u32 DFX_DEFAULT_TASK_NUM = 16; // 2个计数 + 2个其它dfx + 2个memcpy_async + 预留10个
      25              : constexpr u32 DFX_PADDING_TASK_NUM = 4;
      26              : constexpr u32 MASTER_STREAM_EVENT_NUM = 3;
      27              : constexpr u32 SLAVE_STREAM_EVENT_NUM = 2;
      28              : constexpr u32 COM_STEP_NUM = 2;
      29              : constexpr s32 SERVER_NUM_ONE = 1;
      30              : constexpr s32 SERVER_NUM_EIGHT = 8;
      31              : constexpr u32 ALLREDUCE_DEFAULT_COM_STEP = 19;  // allgather + reducescatter
      32              : constexpr u32 ALLGATHER_DEFAULT_COM_STEP = 9;  // 5个通信 + 4个barrier
      33              : constexpr u32 REDUCESCATTER_DEFAULT_COM_STEP = 11;  // 6个通信 + 5个barrier
      34              : constexpr u32 ALLTOALL_DEFAULT_COM_STEP = 14; // alltoall taskNum, 每个对端通信的最大task数量(图模式, NA+pairwise)
      35              : constexpr u32 TASK_NUM_DEVICE_FOUR = 4;
      36              : constexpr s32 TASK_NUM_DEVICE_ONE = 1;
      37              : constexpr u32 ALG_8P_RING_COMM_STEP = 7;
      38              : constexpr u32 PIPLINE_STREAM_EVENT_NUM = 2;
      39              : constexpr u32 MINUS_MESH_STREAM_NUM = 2;
      40              : 
      41              : enum class GeDeterministicOption {
      42              :     DISABLE = 0,
      43              :     ENABLE = 1,
      44              :     STRICT = 2
      45              : };
      46              : 
      47              : enum class RankInfoType {
      48              :     RANK_SIZE_IN_GROUP,
      49              :     RANK_ID_IN_GROUP,
      50              :     WORLD_RANK_ID_BY_GROUP,
      51              :     GROUP_RANK_ID_BY_WORLD,
      52              :     SERVER_NUM_IN_GROUP
      53              : };
      54              : 
      55              : static std::unordered_map<s32, u64> OFFLINE_BUILD_SUB_STEAM_NUM = {
      56              :     {HCCL_DEVICE_NUM_EIGHT, HCCL_SUB_STREAM_NUM_8P_RING},
      57              :     {HCCL_DEVICE_NUM_FOUR, HCCL_SUB_STREAM_NUM_4P_MESH},
      58              :     {HCCL_DEVICE_NUM_TWO, HCCL_SUB_STREAM_NUM_ZERO},
      59              :     {HCCL_DEVICE_MINNUM, HCCL_SUB_STREAM_NUM_ZERO},
      60              : };
      61              : 
      62              : constexpr u32 SINGLE_SERVER_NUM = 1;
      63              : using HcomOpTagInfo = struct HcomOpTagInfoCtx {
      64              :     std::map<std::string, u32> opIndex; // key: (group name) or (identifier), value: op index
      65              : };
      66              : 
      67              : using HcclGroupParams = struct TagHcclGroupParamsInfo {
      68              :     /* * group的基本构建信息，节点数及本节点在group中的编号、
      69              :     本节点在worldgroup中的编号、group的所有ranks */
      70              :     u32 worldRank;                /* * 用于标识world内不同节点 */
      71              :     u32 groupRank;                /* * 用于标识group内不同节点 */
      72              :     u32 serverNum;                /* * 用于标识group内服务器总数 */
      73              :     u32 totalRanks;              /* * 用于指示group内的节点总数, rank范围[0, totalRanks-1] */
      74              :     std::vector<u32> groupRanks;  // 内部存储wordrankid，其下标表示groupid
      75              :     HcclCommPtr pSubComm;
      76              :     u32 refCounter = 0;
      77              :     bool destroyFlag = false;
      78              : };
      79              : 
      80              : using HcomInfo = struct HcomInfoTag {
      81              :     HcclCommPtr pComm;
      82              :     void *psComm;
      83              :     hccl::HcclCommParams params;
      84              :     std::unordered_map<std::string, HcclGroupParams> hcomGroupMap;  // 每个group的信息(kname为服务器的server_id,按照服务器区分)
      85              :     std::mutex groupParamsLock;
      86              :     hccl::RankTable_t rankTable;
      87              :     s32 devId;
      88              :     bool cloudFlag;  // cloudFlag为0即实验室场景,cloudFlag为1则为云场景
      89              :     bool isHcomInit; // 标识是否为pytorch单算子通信域复用场景
      90              :     std::mutex backloggedGroupLock;
      91              :     std::map<std::string, std::vector<u32>> backloggedGroup;     // 待创建的group
      92              :     std::map<std::string, std::shared_ptr<hccl::TopoInfoDetect>> hcclCommTopoInfoDetectServer;
      93              :     std::map<std::string, std::shared_ptr<hccl::TopoInfoDetect>> hcclCommTopoInfoDetectAgent;
      94              :     std::mutex groupRankNumMapLock;
      95              :     std::unordered_map<std::string, u32> groupRankNumMap; // 记录每个group的rank数量，用于topoInfo设置
      96         2859 :     HcomInfoTag()
      97         2859 :         :pComm(nullptr), devId(-1), cloudFlag(false), isHcomInit(false)
      98              :     {
      99         2859 :     }
     100              : 
     101         2793 :     ~HcomInfoTag()
     102              :     {
     103         2793 :         pComm = nullptr;
     104         2792 :         hcclCommTopoInfoDetectServer.clear();
     105         2790 :         hcclCommTopoInfoDetectAgent.clear();
     106         2790 :     }
     107              : };
     108              : 
     109              : HcclResult HcomSetGroupTopoInfo(const char *group, uint32_t rankSize);
     110              : void HcomUnSetGroupTopoInfo(const char *group);
     111              : HcclResult HcomGetCommByGroup(const char *group, std::shared_ptr<hccl::hcclComm> &hcclComm);
     112              : HcclResult HcomGetTopoDesc(const char *group, HcclTopoDescs *topoDescs, uint32_t topoSize);
     113              : s32 HcclGetThreadDeviceId();
     114              : void HcomGroupCallbackFuncInstall(HcclResult (*p1)(const std::string &, const std::vector<u32> &),
     115              :     bool (*p2)(HcomInfo &), HcclResult (*p3)(const std::string &), HcclResult (*p4)(HcomInfo &));
     116              : HcclResult DestroyFlag(const char *group, bool flag);
     117              : HcclResult HcomQueryGroupRef(const char *group, u32 &groupRef);
     118              : bool HcomCheckrtMemcpyAddrAsync(const std::string& group = HCCL_WORLD_GROUP);
     119              : HcclResult HcomGetbackloggedByGroup(const char *group, std::vector<u32> &groupRanks, s32 &groupSize);
     120              : HcomInfo& HcomGetCtxHomInfo(void);
     121              : HcclResult HcomInitCollComm(uint32_t rank, void **commV2, HcclCommPtr &comm);
     122              : 
     123              : #ifdef __cplusplus
     124              : extern "C" {
     125              : #endif
     126              : HcclResult HcomInitByFile(const char *rankTablePath, const char *identify);
     127              : #ifdef __cplusplus
     128              : }
     129              : #endif
     130              : #endif /* HCCL_COMM_PUB_H */