Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 : #ifndef MY_RANK_H
11 : #define MY_RANK_H
12 :
13 : #include <cstdint>
14 : #include <memory>
15 : #include <string>
16 : #include <unordered_map>
17 : #include <utility>
18 : #include <vector>
19 :
20 : #include "hccl/hccl_types.h"
21 : #include "hccl/base.h"
22 : #include "hccl/hccl_res.h"
23 : #include "hccl_mem_defs.h"
24 : #include "acl/acl_rt.h"
25 : #include "socket_manager.h"
26 : #include "hcomm_res_defs.h"
27 : #include "mem_host_pub.h"
28 : #include "rank_pair_mgr.h"
29 : #include "endpoint_mgr.h"
30 : #include "comm_config_pub.h"
31 : #include "manager_common.h"
32 : #include "common.h"
33 : #include "comm_mems/comm_mems.h"
34 : #include "engine_ctxs.h"
35 : #include "../../../dfx/ns_recovery/ns_recovery.h"
36 : #include "hdc_pub.h"
37 : #include "rank_graph.h"
38 : #include "orion_adapter_hccp.h"
39 : #include "coll_comm_config_consistency.h"
40 : #include "exchange_info_mgr.h"
41 :
42 : #include "ccu_types.h"
43 : #include "ccu_drv_handle.h"
44 : #include "dev_type.h"
45 :
46 : namespace hccl {
47 :
48 : /**
49 : * @note 职责：管理当前通信域下本Rank的信息和通信资源
50 : */
51 : class MyRank {
52 : public:
53 : MyRank(aclrtBinHandle binHandle, uint32_t rankId, const CommConfig& config,
54 : const ManagerCallbacks& callbacks, RankGraph* rankGraph,
55 : const Hccl::RankIpPortMapPtr& rankIpPortMap);
56 : ~MyRank();
57 :
58 : HcclResult Init(HcclMem cclBuffer, const uint32_t opExpansionMode, uint32_t rankNum);
59 :
60 6 : CommMems* GetCommMems() const { return commMems_.get(); }
61 :
62 17 : EngineCtxs* GetEngineCtxs() const { return engineCtxs_.get(); }
63 :
64 105 : uint32_t GetOpExpansionMode() {
65 105 : return opExpansionMode_;
66 : }
67 17 : CcuInsHandle GetCcuInstance() const {
68 17 : return ccuInsHandle_;
69 : }
70 3 : void SetCcuInstance(CcuInsHandle ccuInsHandle) {
71 3 : ccuInsHandle_ = ccuInsHandle;
72 3 : }
73 :
74 : CollCommConfigConsistency &GetCollCommConfigConsistency();
75 :
76 : HcclResult CreateChannels(CommEngine engine, const std::string &commTag,
77 : const HcclChannelDesc* channelDescs, uint32_t channelNum, ChannelHandle *channels);
78 :
79 : HcclResult ChannelGetHcclBuffer(ChannelHandle channel, void **buffer, uint64_t *size);
80 : HcclResult ChannelGetRemoteMems(ChannelHandle channel, uint32_t *memNum, CommMem **remoteMem, char ***memTags) const;
81 : HcclResult ChannelGetRemoteMems(ChannelHandle channel, uint32_t *memNum, CommMem **remoteMem,
82 : std::vector<std::string> &memTags) const;
83 :
84 : // Ns recovery
85 : void SetKfcControlTransfer(std::shared_ptr<HDCommunicate> kfcControlTransferH2D,
86 : std::shared_ptr<HDCommunicate> kfcStatusTransferD2H);
87 : std::vector<ChannelHandle> GetAllChannelList();
88 : HcclResult StopLaunch();
89 : HcclResult Clean();
90 : HcclResult Resume();
91 :
92 : private:
93 : using ReuseSocketIdxMap = std::unordered_map<RankPair*, std::unordered_map<hcomm::EndpointPair*, u32>>;
94 : HcclResult GetEndpointPairFromChannel(const HcclChannelDesc &channelDesc, uint32_t channelIndex, uint32_t channelNum,
95 : uint32_t &remoteRank, hcomm::EndpointPair* &endpointPair, RankPair* &rankPair);
96 : HcclResult BatchServerInitForChannels(const HcclChannelDesc* channelDescs, uint32_t channelNum,
97 : const std::string &socketTag, ReuseSocketIdxMap &reuseSocketIdxMap);
98 : HcclResult BatchGetSocketsForChannels(const HcclChannelDesc* channelDescs, uint32_t channelNum,
99 : const std::string &socketTag, std::vector<HcommChannelDesc> &hcommDescs,
100 : ReuseSocketIdxMap &reuseSocketIdxMap);
101 : HcclResult BatchCreateSockets(const HcclChannelDesc* channelDescs, uint32_t channelNum,
102 : const std::string &socketTag, std::vector<HcommChannelDesc> &hcommDescs);
103 : HcclResult BatchCreateChannels(CommEngine engine, const HcclChannelDesc* channelDescs, uint32_t channelNum,
104 : std::vector<HcommChannelDesc> &hcommDescs, ChannelHandle *channelHandles);
105 : HcclResult BatchConnectChannels(const HcclChannelDesc* channelDescs, ChannelHandle *channelHandles, uint32_t channelNum);
106 : HcclResult CheckChannelParam(CommEngine engine, const HcclChannelDesc* channelDesc, uint32_t channelNum) const;
107 : HcclResult QueryListenPort(uint32_t localRank, uint32_t remoteRank, const EndpointDesc &localEndpointDesc,
108 : const EndpointDesc &remoteEndpointDesc, uint32_t &listenPort, HcommChannelDesc &hcommDesc);
109 : HcclResult GetLocalTlsStatus(Hccl::TlsStatus &tlsStatus) const;
110 :
111 : HcclResult TryInitCcuInstanceLegacy();
112 : HcclResult TryInitCcuInstance();
113 : HcclResult ReserveCcuMsCommOrFallback();
114 : HcclResult TryInitCcuInstanceOnDemand();
115 : void ReconcileCcuMsCommReservation(HcclResult initRet);
116 : void ReleaseCcuMsCommReservation();
117 : HcclResult ConfigSqDepthByExpansionMode(CommEngine engine, HcommChannelDesc& hcommDesc) const;
118 : HcclResult DestroyNewChannels(CommEngine engine, const HcclChannelDesc* channelDescs);
119 : // 获取port
120 : HcclResult GetListenPortInternal(uint32_t rank, uint32_t *devPort, EndpointLocType locType);
121 :
122 : aclrtBinHandle binHandle_{nullptr};
123 : uint32_t rankId_{};
124 : int32_t devLogicId_{};
125 : CommConfig config_{};
126 :
127 : // 当前通信域初始化没有处理CommConfig，暂时只使用展开模式
128 : uint32_t opExpansionMode_{0};
129 :
130 : std::unique_ptr<RankPairMgr> rankPairMgr_{nullptr};
131 : std::unique_ptr<hcomm::EndpointMgr> endpointMgr_{nullptr};
132 : std::unique_ptr<CommMems> commMems_{nullptr};
133 : std::unique_ptr<EngineCtxs> engineCtxs_{nullptr};
134 :
135 : CcuInsHandle ccuInsHandle_{0};
136 :
137 : ManagerCallbacks callbacks_;
138 :
139 : // RankGraph (临时放在myRank里面，后面会随着createchannel整体迁移到RankPairMgr上)
140 : RankGraph* rankGraph_{nullptr};
141 :
142 : // 记录每次调用BatchCreateChannels时新增的channelIndex, reuseIdx
143 : std::vector<std::pair<u32, u32>> newChannels_{};
144 :
145 : // Ns recovery
146 : std::unique_ptr<NsRecoveryProcessor> nsRecoveryProcessor_{nullptr};
147 : // 内部获取 port 的方法，根据 mode_ 区分 v1/v2
148 : HcclResult GetDevicePortInternal(uint32_t rank, uint32_t *devPort, EndpointLocType locType);
149 :
150 : Hccl::RankIpPortMapPtr rankIpPortMap_;
151 :
152 : CollCommConfigConsistency collCommConfigConsistency_;
153 : ExchangeInfoMgr exchangeInfoMgr_;
154 : std::shared_ptr<hcomm::CcuDrvHandle> ccuDrvHandle_{};
155 : bool useCcuResStaticAlloc_{false}; // HCCL版本不支持CCU资源按需申请
156 : bool ccuMsCommReserved_{false};
157 : Hccl::HcclMainboardId mainBoardType_{Hccl::HcclMainboardId::MAINBOARD_OTHERS};
158 : };
159 :
160 : } // namespace hccl
161 :
162 : namespace MyRankUtils {
163 :
164 : HcommChannelDesc ChannelDescHccl2Hcomm(const HcclChannelDesc &hcclDesc, const hccl::CommConfig &commConfig);
165 :
166 : } // namespace MyRankUtils
167 :
168 : #endif // MY_RANK_H
|
