Line data    Source code

       1              : /**
       2              :  * Copyright (c) 2025 Huawei Technologies Co., Ltd.
       3              :  * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
       4              :  * CANN Open Software License Agreement Version 2.0 (the "License").
       5              :  * Please refer to the License for details. You may not use this file except in compliance with the License.
       6              :  * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
       7              :  * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
       8              :  * See LICENSE in the root of the software repository for the full text of the License.
       9              :  */
      10              : 
      11              : #ifndef HCOM_COMMOM_H
      12              : #define HCOM_COMMOM_H
      13              : 
      14              : #include "hccl_comm_pub.h"
      15              : #include "hcom_common_v2.h"
      16              : #include "../common/src/topo/topoinfo_detect.h"
      17              : 
      18              : #include "topoinfo_struct.h"
      19              : 
      20              : HcclResult HcclGetDeviceId(void);
      21              : 
      22              : // Ge适配的类
      23              : constexpr u32 SEND_RECEIVE_TASK_NUM = 20;
      24              : constexpr u32 OP_DEFAULT_TASK_NUM = 245;
      25              : constexpr u32 AIV_DEFAULT_TASK_NUM = 4;  // 在AIV且非RDMA场景下，task数量固定为4
      26              : constexpr u32 DFX_DEFAULT_TASK_NUM = 16; // 2个计数 + 2个其它dfx + 2个memcpy_async + 预留10个
      27              : constexpr u32 DFX_PADDING_TASK_NUM = 4;
      28              : constexpr u32 MASTER_STREAM_EVENT_NUM = 3;
      29              : constexpr u32 SLAVE_STREAM_EVENT_NUM = 2;
      30              : constexpr u32 COM_STEP_NUM = 2;
      31              : constexpr s32 SERVER_NUM_ONE = 1;
      32              : constexpr s32 SERVER_NUM_EIGHT = 8;
      33              : constexpr u32 ALLREDUCE_DEFAULT_COM_STEP = 19;     // allgather + reducescatter
      34              : constexpr u32 ALLGATHER_DEFAULT_COM_STEP = 9;      // 5个通信 + 4个barrier
      35              : constexpr u32 REDUCESCATTER_DEFAULT_COM_STEP = 11; // 6个通信 + 5个barrier
      36              : constexpr u32 ALLTOALL_DEFAULT_COM_STEP = 14; // alltoall taskNum, 每个对端通信的最大task数量(图模式, NA+pairwise)
      37              : constexpr u32 TASK_NUM_DEVICE_FOUR = 4;
      38              : constexpr s32 TASK_NUM_DEVICE_ONE = 1;
      39              : constexpr u32 ALG_8P_RING_COMM_STEP = 7;
      40              : constexpr u32 PIPLINE_STREAM_EVENT_NUM = 2;
      41              : constexpr u32 MINUS_MESH_STREAM_NUM = 2;
      42              : 
      43              : enum class GeDeterministicOption { DISABLE = 0, ENABLE = 1, STRICT = 2 };
      44              : 
      45              : enum class RankInfoType {
      46              :     RANK_SIZE_IN_GROUP,
      47              :     RANK_ID_IN_GROUP,
      48              :     WORLD_RANK_ID_BY_GROUP,
      49              :     GROUP_RANK_ID_BY_WORLD,
      50              :     SERVER_NUM_IN_GROUP
      51              : };
      52              : 
      53              : static std::unordered_map<s32, u64> OFFLINE_BUILD_SUB_STEAM_NUM = {
      54              :     {HCCL_DEVICE_NUM_EIGHT, HCCL_SUB_STREAM_NUM_8P_RING},
      55              :     {HCCL_DEVICE_NUM_FOUR, HCCL_SUB_STREAM_NUM_4P_MESH},
      56              :     {HCCL_DEVICE_NUM_TWO, HCCL_SUB_STREAM_NUM_ZERO},
      57              :     {HCCL_DEVICE_MINNUM, HCCL_SUB_STREAM_NUM_ZERO},
      58              : };
      59              : 
      60              : constexpr u32 SINGLE_SERVER_NUM = 1;
      61              : using HcomOpTagInfo = struct HcomOpTagInfoCtx {
      62              :     std::map<std::string, u32> opIndex; // key: (group name) or (identifier), value: op index
      63              : };
      64              : 
      65              : using HcclGroupParams = struct TagHcclGroupParamsInfo {
      66              :     /* * group的基本构建信息，节点数及本节点在group中的编号、
      67              :     本节点在worldgroup中的编号、group的所有ranks */
      68              :     u32 worldRank;               /* * 用于标识world内不同节点 */
      69              :     u32 groupRank;               /* * 用于标识group内不同节点 */
      70              :     u32 serverNum;               /* * 用于标识group内服务器总数 */
      71              :     u32 totalRanks;              /* * 用于指示group内的节点总数, rank范围[0, totalRanks-1] */
      72              :     std::vector<u32> groupRanks; // 内部存储wordrankid，其下标表示groupid
      73              :     HcclCommPtr pSubComm;
      74              :     u32 refCounter = 0;
      75              :     bool destroyFlag = false;
      76              : };
      77              : 
      78              : using HcomInfo = struct HcomInfoTag {
      79              :     HcclCommPtr pComm;
      80              :     void* psComm;
      81              :     hccl::HcclCommParams params;
      82              :     std::unordered_map<std::string, HcclGroupParams>
      83              :         hcomGroupMap; // 每个group的信息(kname为服务器的server_id,按照服务器区分)
      84              :     std::mutex groupParamsLock;
      85              :     hccl::RankTable_t rankTable;
      86              :     s32 devId;
      87              :     bool cloudFlag;  // cloudFlag为0即实验室场景,cloudFlag为1则为云场景
      88              :     bool isHcomInit; // 标识是否为pytorch单算子通信域复用场景
      89              :     std::mutex backloggedGroupLock;
      90              :     std::map<std::string, std::vector<u32>> backloggedGroup; // 待创建的group
      91              :     std::map<std::string, std::shared_ptr<hccl::TopoInfoDetect>> hcclCommTopoInfoDetectServer;
      92              :     std::map<std::string, std::shared_ptr<hccl::TopoInfoDetect>> hcclCommTopoInfoDetectAgent;
      93              :     std::mutex groupRankNumMapLock;
      94              :     std::unordered_map<std::string, u32> groupRankNumMap; // 记录每个group的rank数量，用于topoInfo设置
      95         3255 :     HcomInfoTag() : pComm(nullptr), devId(-1), cloudFlag(false), isHcomInit(false) {}
      96              : 
      97         3189 :     ~HcomInfoTag()
      98              :     {
      99         3189 :         pComm = nullptr;
     100         3188 :         hcclCommTopoInfoDetectServer.clear();
     101         3187 :         hcclCommTopoInfoDetectAgent.clear();
     102         3186 :     }
     103              : };
     104              : 
     105              : HcclResult HcomSetGroupTopoInfo(const char* group, uint32_t rankSize);
     106              : void HcomUnSetGroupTopoInfo(const char* group);
     107              : HcclResult HcomGetCommByGroup(const char* group, std::shared_ptr<hccl::hcclComm>& hcclComm);
     108              : HcclResult HcomGetTopoDesc(const char* group, HcclTopoDescs* topoDescs, uint32_t topoSize);
     109              : s32 HcclGetThreadDeviceId();
     110              : void HcomGroupCallbackFuncInstall(
     111              :     HcclResult (*p1)(const std::string&, const std::vector<u32>&), bool (*p2)(HcomInfo&),
     112              :     HcclResult (*p3)(const std::string&), HcclResult (*p4)(HcomInfo&));
     113              : HcclResult DestroyFlag(const char* group, bool flag);
     114              : HcclResult HcomQueryGroupRef(const char* group, u32& groupRef);
     115              : bool HcomCheckrtMemcpyAddrAsync(const std::string& group = HCCL_WORLD_GROUP);
     116              : HcclResult HcomGetbackloggedByGroup(const char* group, std::vector<u32>& groupRanks, s32& groupSize);
     117              : HcomInfo& HcomGetCtxHomInfo(void);
     118              : HcclResult HcomInitCollComm(uint32_t rank, void** commV2, HcclCommPtr& comm);
     119              : 
     120              : #ifdef __cplusplus
     121              : extern "C" {
     122              : #endif
     123              : HcclResult HcomInitByFile(const char* rankTablePath, const char* identify);
     124              : #ifdef __cplusplus
     125              : }
     126              : #endif
     127              : #endif /* HCCL_COMM_PUB_H */