Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 : #ifndef MY_RANK_H
11 : #define MY_RANK_H
12 :
13 : #include <cstdint>
14 : #include <memory>
15 : #include <string>
16 : #include <unordered_map>
17 : #include <utility>
18 : #include <vector>
19 :
20 : #include "hccl/hccl_types.h"
21 : #include "hccl/base.h"
22 : #include "hccl/hccl_res.h"
23 : #include "hccl_mem_defs.h"
24 : #include "acl/acl_rt.h"
25 : #include "socket_manager.h"
26 : #include "hcomm_res_defs.h"
27 : #include "hcomm_channel.h"
28 : #include "mem_host_pub.h"
29 : #include "rank_pair_mgr.h"
30 : #include "endpoint_mgr.h"
31 : #include "comm_config_pub.h"
32 : #include "manager_common.h"
33 : #include "common.h"
34 : #include "comm_mems/comm_mems.h"
35 : #include "engine_ctxs.h"
36 : #include "../../../dfx/ns_recovery/ns_recovery.h"
37 : #include "hdc_pub.h"
38 : #include "rank_graph.h"
39 : #include "orion_adapter_hccp.h"
40 : #include "coll_comm_config_consistency.h"
41 : #include "exchange_info_mgr.h"
42 :
43 : #include "ccu_types.h"
44 : #include "ccu_drv_handle.h"
45 : #include "dev_type.h"
46 :
47 : namespace hccl {
48 :
49 : /**
50 : * @note 职责：管理当前通信域下本Rank的信息和通信资源
51 : */
52 : class MyRank {
53 : public:
54 : MyRank(
55 : aclrtBinHandle binHandle, uint32_t rankId, const CommConfig& config, const ManagerCallbacks& callbacks,
56 : RankGraph* rankGraph, const Hccl::RankIpPortMapPtr& rankIpPortMap);
57 : ~MyRank();
58 :
59 : HcclResult Init(HcclMem cclBuffer, const uint32_t opExpansionMode, uint32_t rankNum);
60 :
61 36 : CommMems* GetCommMems() const { return commMems_.get(); }
62 :
63 17 : EngineCtxs* GetEngineCtxs() const { return engineCtxs_.get(); }
64 :
65 : HcclResult UnregMemByTag(const std::string& tag);
66 153 : uint32_t GetOpExpansionMode() { return opExpansionMode_; }
67 17 : CcuInsHandle GetCcuInstance() const { return ccuInsHandle_; }
68 3 : void SetCcuInstance(CcuInsHandle ccuInsHandle) { ccuInsHandle_ = ccuInsHandle; }
69 :
70 : CollCommConfigConsistency& GetCollCommConfigConsistency();
71 :
72 0 : hcomm::EndpointMgr* GetEndpointMgr() const { return endpointMgr_.get(); }
73 :
74 : HcclResult CreateChannels(
75 : CommEngine engine, const std::string& commTag, const HcclChannelDesc* channelDescs, uint32_t channelNum,
76 : ChannelHandle* channels);
77 :
78 : HcclResult ChannelGetHcclBuffer(ChannelHandle channel, void** buffer, uint64_t* size);
79 : HcclResult
80 : ChannelGetRemoteMems(ChannelHandle channel, uint32_t* memNum, CommMem** remoteMem, char*** memTags) const;
81 : HcclResult ChannelGetRemoteMems(
82 : ChannelHandle channel, uint32_t* memNum, CommMem** remoteMem, std::vector<std::string>& memTags) const;
83 :
84 : // Ns recovery
85 : void SetKfcControlTransfer(
86 : std::shared_ptr<HDCommunicate> kfcControlTransferH2D, std::shared_ptr<HDCommunicate> kfcStatusTransferD2H);
87 : std::vector<ChannelHandle> GetAllChannelList();
88 : HcclResult StopLaunch();
89 : HcclResult Clean();
90 : HcclResult Resume();
91 :
92 : /**
93 : * @brief 批量预建 socket（server 监听 + client 连接），与非共享路径 MyRank::CreateChannels 一致。
94 : * 共享 jetty 路径在 createFunc 中调用，确保两端 socket 体系一致、可对接。
95 : * @param[in] channelDescs HCCL 层 channel desc 数组
96 : * @param[in] channelNum 数量
97 : * @param[in] socketTag socket 标签（commTag + "_engine_" + engine）
98 : * @param[out] hcommDescs 输出的 HcommChannelDesc 数组（调用前需用 ChannelDescHccl2Hcomm 填充基础字段，
99 : * 本方法补上 socket/role/port 字段）
100 : */
101 : HcclResult BatchCreateSockets(
102 : const HcclChannelDesc* channelDescs, uint32_t channelNum, const std::string& socketTag,
103 : std::vector<HcommChannelDesc>& hcommDescs);
104 :
105 : /**
106 : * @brief 在已建好的 socket 上执行通信域一致性校验交换（CheckFrameV2 + 用户信息）。
107 : * 与非共享路径 MyRank::CreateChannels 内部调用 exchangeInfoMgr_ 的逻辑一致，
108 : * 供共享 jetty 路径在 createFunc 中调用，确保两端无论 shared/非shared 配置是否对称，
109 : * 都会在 socket 上完成定长 CheckFrameV2（120B）的对称收发，避免一端死等。
110 : * @param[in] channelDescs HCCL 层 channel desc 数组
111 : * @param[in] hcommDescs 已通过 BatchCreateSockets 填充 socket 字段的 HcommChannelDesc 数组
112 : * @param[in] channelNum 数量
113 : * @param[in] newChannels 新建 channel 的 (idx, reuseIdx) 列表；空表示全部按新建处理
114 : * @param[in] engine 通信引擎
115 : * @note 仅 DEV_TYPE_950 实际执行交换，其它设备类型直接返回 SUCCESS，与非共享路径保持一致。
116 : */
117 : HcclResult BatchExchangeAndCheckConsistency(
118 : const HcclChannelDesc* channelDescs, const std::vector<HcommChannelDesc>& hcommDescs, uint32_t channelNum,
119 : const std::vector<std::pair<u32, u32>>& newChannels, CommEngine engine);
120 : HcclResult PrepareMemHandles(
121 : EndpointHandle epHandle, void** memHandles, uint32_t memHandleNum, std::vector<MemHandle>& memHandleVec);
122 :
123 : private:
124 : using ReuseSocketIdxMap = std::unordered_map<RankPair*, std::unordered_map<hcomm::EndpointPair*, u32>>;
125 : HcclResult GetEndpointPairFromChannel(
126 : const HcclChannelDesc& channelDesc, uint32_t channelIndex, uint32_t channelNum, uint32_t& remoteRank,
127 : hcomm::EndpointPair*& endpointPair, RankPair*& rankPair);
128 : HcclResult BatchServerInitForChannels(
129 : const HcclChannelDesc* channelDescs, uint32_t channelNum, const std::string& socketTag,
130 : ReuseSocketIdxMap& reuseSocketIdxMap);
131 : HcclResult BatchGetSocketsForChannels(
132 : const HcclChannelDesc* channelDescs, uint32_t channelNum, const std::string& socketTag,
133 : std::vector<HcommChannelDesc>& hcommDescs, ReuseSocketIdxMap& reuseSocketIdxMap);
134 : HcclResult BatchCreateChannels(
135 : CommEngine engine, const HcclChannelDesc* channelDescs, uint32_t channelNum,
136 : std::vector<HcommChannelDesc>& hcommDescs, ChannelHandle* channelHandles,
137 : std::vector<std::vector<MemHandle>>& allHandles);
138 : HcclResult
139 : BatchConnectChannels(const HcclChannelDesc* channelDescs, ChannelHandle* channelHandles, uint32_t channelNum);
140 : void LogChannelCreationInfo(
141 : CommEngine engine, const std::string& commTag, const HcclChannelDesc* channelDescs, uint32_t channelNum,
142 : ChannelHandle* hostChannelHandleList);
143 : HcclResult FinalizeChannelsByEngine(
144 : CommEngine engine, const std::string& commTag, const HcclChannelDesc* channelDescs, uint32_t channelNum,
145 : std::vector<HcommChannelDesc>& hcommDescs, ChannelHandle* hostChannelHandleList, ChannelHandle* channelHandles);
146 : HcclResult CheckChannelParam(CommEngine engine, const HcclChannelDesc* channelDesc, uint32_t channelNum) const;
147 : HcclResult QueryListenPort(
148 : uint32_t localRank, uint32_t remoteRank, const EndpointDesc& localEndpointDesc,
149 : const EndpointDesc& remoteEndpointDesc, uint32_t& listenPort, HcommChannelDesc& hcommDesc);
150 : HcclResult GetLocalTlsStatus(Hccl::TlsStatus& tlsStatus) const;
151 : HcclResult RegisterCommMemsToEndpoint(EndpointHandle epHandle);
152 : HcclResult TryInitCcuInstanceLegacy();
153 : HcclResult TryInitCcuInstance();
154 : HcclResult ReserveCcuMsCommOrFallback();
155 : HcclResult TryInitCcuInstanceOnDemand();
156 : void ReconcileCcuMsCommReservation(HcclResult initRet);
157 : void ReleaseCcuMsCommReservation();
158 : HcclResult ConfigSqDepthByExpansionMode(CommEngine engine, HcommChannelDesc& hcommDesc) const;
159 : HcclResult DestroyNewChannels(CommEngine engine, const HcclChannelDesc* channelDescs);
160 :
161 : aclrtBinHandle binHandle_{nullptr};
162 : uint32_t rankId_{};
163 : int32_t devLogicId_{};
164 : CommConfig config_{};
165 :
166 : // 当前通信域初始化没有处理CommConfig，暂时只使用展开模式
167 : uint32_t opExpansionMode_{0};
168 :
169 : std::unique_ptr<RankPairMgr> rankPairMgr_{nullptr};
170 : std::unique_ptr<hcomm::EndpointMgr> endpointMgr_{nullptr};
171 : std::unique_ptr<CommMems> commMems_{nullptr};
172 : std::unique_ptr<EngineCtxs> engineCtxs_{nullptr};
173 :
174 : CcuInsHandle ccuInsHandle_{0};
175 :
176 : ManagerCallbacks callbacks_;
177 :
178 : // RankGraph (临时放在myRank里面，后面会随着createchannel整体迁移到RankPairMgr上)
179 : RankGraph* rankGraph_{nullptr};
180 :
181 : // 记录每次调用BatchCreateChannels时新增的channelIndex, reuseIdx
182 : std::vector<std::pair<u32, u32>> newChannels_{};
183 :
184 : // Ns recovery
185 : std::unique_ptr<NsRecoveryProcessor> nsRecoveryProcessor_{nullptr};
186 : // 内部获取 port 的方法，根据 mode_ 区分 v1/v2
187 : HcclResult GetDevicePortInternal(uint32_t rank, uint32_t* devPort, EndpointLocType locType);
188 :
189 : Hccl::RankIpPortMapPtr rankIpPortMap_;
190 :
191 : CollCommConfigConsistency collCommConfigConsistency_;
192 : ExchangeInfoMgr exchangeInfoMgr_;
193 : std::shared_ptr<hcomm::CcuDrvHandle> ccuDrvHandle_{};
194 : bool useCcuResStaticAlloc_{false}; // HCCL版本不支持CCU资源按需申请
195 : bool ccuMsCommReserved_{false};
196 : Hccl::HcclMainboardId mainBoardType_{Hccl::HcclMainboardId::MAINBOARD_OTHERS};
197 : };
198 :
199 : } // namespace hccl
200 :
201 : namespace MyRankUtils {
202 :
203 : HcommChannelDesc ChannelDescHccl2Hcomm(const HcclChannelDesc& hcclDesc, const hccl::CommConfig& commConfig);
204 :
205 : } // namespace MyRankUtils
206 :
207 : #endif // MY_RANK_H
|
