Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 : #ifndef MY_RANK_H
11 : #define MY_RANK_H
12 :
13 : #include <cstdint>
14 : #include <memory>
15 : #include <string>
16 : #include <unordered_map>
17 : #include <utility>
18 : #include <vector>
19 :
20 : #include "hccl/hccl_types.h"
21 : #include "hccl/base.h"
22 : #include "hccl/hccl_res.h"
23 : #include "hccl_mem_defs.h"
24 : #include "acl/acl_rt.h"
25 : #include "socket_manager.h"
26 : #include "hcomm_res_defs.h"
27 : #include "mem_host_pub.h"
28 : #include "rank_pair_mgr.h"
29 : #include "endpoint_mgr.h"
30 : #include "comm_config_pub.h"
31 : #include "manager_common.h"
32 : #include "common.h"
33 : #include "comm_mems/comm_mems.h"
34 : #include "engine_ctxs.h"
35 : #include "../../../dfx/ns_recovery/ns_recovery.h"
36 : #include "hdc_pub.h"
37 : #include "rank_graph.h"
38 : #include "orion_adapter_hccp.h"
39 : #include "coll_comm_config_consistency.h"
40 : #include "exchange_info_mgr.h"
41 :
42 : #include "ccu_types.h"
43 :
44 : namespace hccl {
45 :
46 : /**
47 : * @note 职责：管理当前通信域下本Rank的信息和通信资源
48 : */
49 : class MyRank {
50 : public:
51 : MyRank(aclrtBinHandle binHandle, uint32_t rankId, const CommConfig& config,
52 : const ManagerCallbacks& callbacks, RankGraph* rankGraph,
53 : const Hccl::RankIpPortMapPtr& rankIpPortMap);
54 : ~MyRank();
55 :
56 : HcclResult Init(HcclMem cclBuffer, const uint32_t opExpansionMode, uint32_t rankNum);
57 :
58 6 : CommMems* GetCommMems() const { return commMems_.get(); }
59 :
60 17 : EngineCtxs* GetEngineCtxs() const { return engineCtxs_.get(); }
61 :
62 103 : uint32_t GetOpExpansionMode() {
63 103 : return opExpansionMode_;
64 : }
65 0 : CcuInsHandle GetCcuInstance() const {
66 0 : return ccuInsHandle_;
67 : }
68 :
69 : CollCommConfigConsistency &GetCollCommConfigConsistency();
70 :
71 : HcclResult CreateChannels(CommEngine engine, const std::string &commTag,
72 : const HcclChannelDesc* channelDescs, uint32_t channelNum, ChannelHandle *channels);
73 :
74 : HcclResult ChannelGetHcclBuffer(ChannelHandle channel, void **buffer, uint64_t *size);
75 : HcclResult ChannelGetRemoteMems(ChannelHandle channel, uint32_t *memNum, CommMem **remoteMem, char ***memTags) const;
76 : HcclResult ChannelGetRemoteMems(ChannelHandle channel, uint32_t *memNum, CommMem **remoteMem,
77 : std::vector<std::string> &memTags) const;
78 :
79 : // Ns recovery
80 : void SetKfcControlTransfer(std::shared_ptr<HDCommunicate> kfcControlTransferH2D,
81 : std::shared_ptr<HDCommunicate> kfcStatusTransferD2H);
82 : std::vector<ChannelHandle> GetAllChannelList();
83 : HcclResult StopLaunch();
84 : HcclResult Clean();
85 : HcclResult Resume();
86 :
87 : private:
88 : using ReuseSocketIdxMap = std::unordered_map<RankPair*, std::unordered_map<hcomm::EndpointPair*, u32>>;
89 : HcclResult GetEndpointPairFromChannel(const HcclChannelDesc &channelDesc, uint32_t channelIndex, uint32_t channelNum,
90 : uint32_t &remoteRank, hcomm::EndpointPair* &endpointPair, RankPair* &rankPair);
91 : HcclResult BatchServerInitForChannels(const HcclChannelDesc* channelDescs, uint32_t channelNum,
92 : const std::string &socketTag, ReuseSocketIdxMap &reuseSocketIdxMap);
93 : HcclResult BatchGetSocketsForChannels(const HcclChannelDesc* channelDescs, uint32_t channelNum,
94 : const std::string &socketTag, std::vector<HcommChannelDesc> &hcommDescs,
95 : ReuseSocketIdxMap &reuseSocketIdxMap);
96 : HcclResult BatchCreateSockets(const HcclChannelDesc* channelDescs, uint32_t channelNum,
97 : const std::string &socketTag, std::vector<HcommChannelDesc> &hcommDescs);
98 : HcclResult BatchCreateChannels(CommEngine engine, const HcclChannelDesc* channelDescs, uint32_t channelNum,
99 : std::vector<HcommChannelDesc> &hcommDescs, ChannelHandle *channelHandles);
100 : HcclResult BatchConnectChannels(const HcclChannelDesc* channelDescs, ChannelHandle *channelHandles, uint32_t channelNum);
101 : HcclResult CheckChannelParam(CommEngine engine, const HcclChannelDesc* channelDesc, uint32_t channelNum) const;
102 : HcclResult QueryListenPort(uint32_t localRank, uint32_t remoteRank, const EndpointDesc &localEndpointDesc,
103 : const EndpointDesc &remoteEndpointDesc, uint32_t &listenPort, HcommChannelDesc &hcommDesc);
104 : HcclResult GetLocalTlsStatus(Hccl::TlsStatus &tlsStatus) const;
105 :
106 : HcclResult TryInitCcuInstance();
107 : HcclResult ConfigSqDepthByExpansionMode(CommEngine engine, HcommChannelDesc& hcommDesc) const;
108 : HcclResult DestroyNewChannels(CommEngine engine, const HcclChannelDesc* channelDescs);
109 : // 获取port
110 : HcclResult GetListenPortInternal(uint32_t rank, uint32_t *devPort, EndpointLocType locType);
111 :
112 : aclrtBinHandle binHandle_{nullptr};
113 : uint32_t rankId_{};
114 : int32_t devLogicId_{};
115 : CommConfig config_{};
116 :
117 : // 当前通信域初始化没有处理CommConfig，暂时只使用展开模式
118 : uint32_t opExpansionMode_{0};
119 :
120 : std::unique_ptr<RankPairMgr> rankPairMgr_{nullptr};
121 : std::unique_ptr<hcomm::EndpointMgr> endpointMgr_{nullptr};
122 : std::unique_ptr<CommMems> commMems_{nullptr};
123 : std::unique_ptr<EngineCtxs> engineCtxs_{nullptr};
124 :
125 : CcuInsHandle ccuInsHandle_{0};
126 :
127 : ManagerCallbacks callbacks_;
128 :
129 : // RankGraph (临时放在myRank里面，后面会随着createchannel整体迁移到RankPairMgr上)
130 : RankGraph* rankGraph_{nullptr};
131 :
132 : // 记录每次调用BatchCreateChannels时新增的channelIndex, reuseIdx
133 : std::vector<std::pair<u32, u32>> newChannels_{};
134 :
135 : // Ns recovery
136 : std::unique_ptr<NsRecoveryProcessor> nsRecoveryProcessor_{nullptr};
137 : // 内部获取 port 的方法，根据 mode_ 区分 v1/v2
138 : HcclResult GetDevicePortInternal(uint32_t rank, uint32_t *devPort, EndpointLocType locType);
139 :
140 : Hccl::RankIpPortMapPtr rankIpPortMap_;
141 :
142 : CollCommConfigConsistency collCommConfigConsistency_;
143 : ExchangeInfoMgr exchangeInfoMgr_;
144 : };
145 :
146 : } // namespace hccl
147 :
148 : namespace MyRankUtils {
149 :
150 : HcommChannelDesc ChannelDescHccl2Hcomm(const HcclChannelDesc &hcclDesc, const hccl::CommConfig &commConfig);
151 :
152 : } // namespace MyRankUtils
153 :
154 : #endif // MY_RANK_H
|
