Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #ifndef MY_RANK_H
12 : #define MY_RANK_H
13 :
14 : #include <cstdint>
15 : #include <memory>
16 : #include <string>
17 : #include <unordered_map>
18 : #include <utility>
19 : #include <vector>
20 :
21 : #include "hccl/hccl_types.h"
22 : #include "hccl/base.h"
23 : #include "hccl/hccl_res.h"
24 : #include "hccl_mem_defs.h"
25 : #include "acl/acl_rt.h"
26 : #include "socket_manager.h"
27 : #include "hcomm_res_defs.h"
28 : #include "hcomm_channel.h"
29 : #include "mem_host_pub.h"
30 : #include "rank_pair_mgr.h"
31 : #include "endpoint_mgr.h"
32 : #include "comm_config_pub.h"
33 : #include "manager_common.h"
34 : #include "common.h"
35 : #include "comm_mems/comm_mems.h"
36 : #include "engine_ctxs.h"
37 : #include "../../../dfx/ns_recovery/ns_recovery.h"
38 : #include "hdc_pub.h"
39 : #include "rank_graph.h"
40 : #include "orion_adapter_hccp.h"
41 : #include "coll_comm_config_consistency.h"
42 : #include "exchange_info_mgr.h"
43 :
44 : #include "ccu_types.h"
45 : #include "ccu_drv_handle.h"
46 : #include "dev_type.h"
47 :
48 : namespace hccl {
49 :
50 : /**
51 : * @note 职责：管理当前通信域下本Rank的信息和通信资源
52 : */
53 : class MyRank {
54 : public:
55 : MyRank(
56 : aclrtBinHandle binHandle, uint32_t rankId, const CommConfig& config, const ManagerCallbacks& callbacks,
57 : RankGraph* rankGraph, const Hccl::RankIpPortMapPtr& rankIpPortMap);
58 : ~MyRank();
59 :
60 : HcclResult Init(HcclMem cclBuffer, const uint32_t opExpansionMode, uint32_t rankNum);
61 :
62 36 : CommMems* GetCommMems() const { return commMems_.get(); }
63 :
64 17 : EngineCtxs* GetEngineCtxs() const { return engineCtxs_.get(); }
65 :
66 : HcclResult UnregMemByTag(const std::string& tag);
67 153 : uint32_t GetOpExpansionMode() { return opExpansionMode_; }
68 17 : CcuInsHandle GetCcuInstance() const { return ccuInsHandle_; }
69 3 : void SetCcuInstance(CcuInsHandle ccuInsHandle) { ccuInsHandle_ = ccuInsHandle; }
70 :
71 : CollCommConfigConsistency& GetCollCommConfigConsistency();
72 :
73 0 : hcomm::EndpointMgr* GetEndpointMgr() const { return endpointMgr_.get(); }
74 :
75 : HcclResult CreateChannels(
76 : CommEngine engine, const std::string& commTag, const HcclChannelDesc* channelDescs, uint32_t channelNum,
77 : ChannelHandle* channels);
78 :
79 : HcclResult ChannelGetHcclBuffer(ChannelHandle channel, void** buffer, uint64_t* size);
80 : HcclResult
81 : ChannelGetRemoteMems(ChannelHandle channel, uint32_t* memNum, CommMem** remoteMem, char*** memTags) const;
82 : HcclResult ChannelGetRemoteMems(
83 : ChannelHandle channel, uint32_t* memNum, CommMem** remoteMem, std::vector<std::string>& memTags) const;
84 :
85 : // Ns recovery
86 : void SetKfcControlTransfer(
87 : std::shared_ptr<HDCommunicate> kfcControlTransferH2D, std::shared_ptr<HDCommunicate> kfcStatusTransferD2H);
88 : std::vector<ChannelHandle> GetAllChannelList();
89 : HcclResult StopLaunch();
90 : HcclResult Clean();
91 : HcclResult Resume();
92 :
93 : /**
94 : * @brief 批量预建 socket（server 监听 + client 连接），与非共享路径 MyRank::CreateChannels 一致。
95 : * 共享 jetty 路径在 createFunc 中调用，确保两端 socket 体系一致、可对接。
96 : * @param[in] channelDescs HCCL 层 channel desc 数组
97 : * @param[in] channelNum 数量
98 : * @param[in] socketTag socket 标签（commTag + "_engine_" + engine）
99 : * @param[out] hcommDescs 输出的 HcommChannelDesc 数组（调用前需用 ChannelDescHccl2Hcomm 填充基础字段，
100 : * 本方法补上 socket/role/port 字段）
101 : */
102 : HcclResult BatchCreateSockets(
103 : const HcclChannelDesc* channelDescs, uint32_t channelNum, const std::string& socketTag,
104 : std::vector<HcommChannelDesc>& hcommDescs);
105 :
106 : /**
107 : * @brief 在已建好的 socket 上执行通信域一致性校验交换（CheckFrameV2 + 用户信息）。
108 : * 与非共享路径 MyRank::CreateChannels 内部调用 exchangeInfoMgr_ 的逻辑一致，
109 : * 供共享 jetty 路径在 createFunc 中调用，确保两端无论 shared/非shared 配置是否对称，
110 : * 都会在 socket 上完成定长 CheckFrameV2（120B）的对称收发，避免一端死等。
111 : * @param[in] channelDescs HCCL 层 channel desc 数组
112 : * @param[in] hcommDescs 已通过 BatchCreateSockets 填充 socket 字段的 HcommChannelDesc 数组
113 : * @param[in] channelNum 数量
114 : * @param[in] newChannels 新建 channel 的 (idx, reuseIdx) 列表；空表示全部按新建处理
115 : * @param[in] engine 通信引擎
116 : * @note 仅 DEV_TYPE_950 实际执行交换，其它设备类型直接返回 SUCCESS，与非共享路径保持一致。
117 : */
118 : HcclResult BatchExchangeAndCheckConsistency(
119 : const HcclChannelDesc* channelDescs, const std::vector<HcommChannelDesc>& hcommDescs, uint32_t channelNum,
120 : const std::vector<std::pair<u32, u32>>& newChannels, CommEngine engine);
121 : HcclResult PrepareMemHandles(
122 : EndpointHandle epHandle, void** memHandles, uint32_t memHandleNum, std::vector<MemHandle>& memHandleVec);
123 :
124 : private:
125 : using ReuseSocketIdxMap = std::unordered_map<RankPair*, std::unordered_map<hcomm::EndpointPair*, u32>>;
126 : HcclResult GetEndpointPairFromChannel(
127 : const HcclChannelDesc& channelDesc, uint32_t channelIndex, uint32_t channelNum, uint32_t& remoteRank,
128 : hcomm::EndpointPair*& endpointPair, RankPair*& rankPair);
129 : HcclResult BatchServerInitForChannels(
130 : const HcclChannelDesc* channelDescs, uint32_t channelNum, const std::string& socketTag,
131 : ReuseSocketIdxMap& reuseSocketIdxMap);
132 : HcclResult BatchGetSocketsForChannels(
133 : const HcclChannelDesc* channelDescs, uint32_t channelNum, const std::string& socketTag,
134 : std::vector<HcommChannelDesc>& hcommDescs, ReuseSocketIdxMap& reuseSocketIdxMap);
135 : HcclResult BatchCreateChannels(
136 : CommEngine engine, const HcclChannelDesc* channelDescs, uint32_t channelNum,
137 : std::vector<HcommChannelDesc>& hcommDescs, ChannelHandle* channelHandles,
138 : std::vector<std::vector<MemHandle>>& allHandles);
139 : HcclResult
140 : BatchConnectChannels(const HcclChannelDesc* channelDescs, ChannelHandle* channelHandles, uint32_t channelNum);
141 : void LogChannelCreationInfo(
142 : CommEngine engine, const std::string& commTag, const HcclChannelDesc* channelDescs, uint32_t channelNum,
143 : ChannelHandle* hostChannelHandleList);
144 : HcclResult FinalizeChannelsByEngine(
145 : CommEngine engine, const std::string& commTag, const HcclChannelDesc* channelDescs, uint32_t channelNum,
146 : std::vector<HcommChannelDesc>& hcommDescs, ChannelHandle* hostChannelHandleList, ChannelHandle* channelHandles);
147 : HcclResult CheckChannelParam(CommEngine engine, const HcclChannelDesc* channelDesc, uint32_t channelNum) const;
148 : HcclResult QueryListenPort(
149 : uint32_t localRank, uint32_t remoteRank, const EndpointDesc& localEndpointDesc,
150 : const EndpointDesc& remoteEndpointDesc, uint32_t& listenPort, HcommChannelDesc& hcommDesc);
151 : HcclResult GetLocalTlsStatus(Hccl::TlsStatus& tlsStatus) const;
152 : HcclResult RegisterCommMemsToEndpoint(EndpointHandle epHandle);
153 : HcclResult TryInitCcuInstanceLegacy();
154 : HcclResult TryInitCcuInstance();
155 : HcclResult ReserveCcuMsCommOrFallback();
156 : HcclResult TryInitCcuInstanceOnDemand();
157 : void ReconcileCcuMsCommReservation(HcclResult initRet);
158 : void ReleaseCcuMsCommReservation();
159 : HcclResult ConfigSqDepthByExpansionMode(CommEngine engine, HcommChannelDesc& hcommDesc) const;
160 : HcclResult DestroyNewChannels(CommEngine engine, const HcclChannelDesc* channelDescs);
161 :
162 : aclrtBinHandle binHandle_{nullptr};
163 : uint32_t rankId_{};
164 : int32_t devLogicId_{};
165 : CommConfig config_{};
166 :
167 : // 当前通信域初始化没有处理CommConfig，暂时只使用展开模式
168 : uint32_t opExpansionMode_{0};
169 :
170 : std::unique_ptr<RankPairMgr> rankPairMgr_{nullptr};
171 : std::unique_ptr<hcomm::EndpointMgr> endpointMgr_{nullptr};
172 : std::unique_ptr<CommMems> commMems_{nullptr};
173 : std::unique_ptr<EngineCtxs> engineCtxs_{nullptr};
174 :
175 : CcuInsHandle ccuInsHandle_{0};
176 :
177 : ManagerCallbacks callbacks_;
178 :
179 : // RankGraph (临时放在myRank里面，后面会随着createchannel整体迁移到RankPairMgr上)
180 : RankGraph* rankGraph_{nullptr};
181 :
182 : // 记录每次调用BatchCreateChannels时新增的channelIndex, reuseIdx
183 : std::vector<std::pair<u32, u32>> newChannels_{};
184 :
185 : // Ns recovery
186 : std::unique_ptr<NsRecoveryProcessor> nsRecoveryProcessor_{nullptr};
187 : // 内部获取 port 的方法，根据 mode_ 区分 v1/v2
188 : HcclResult GetDevicePortInternal(uint32_t rank, uint32_t* devPort, EndpointLocType locType);
189 :
190 : Hccl::RankIpPortMapPtr rankIpPortMap_;
191 :
192 : CollCommConfigConsistency collCommConfigConsistency_;
193 : ExchangeInfoMgr exchangeInfoMgr_;
194 : std::shared_ptr<hcomm::CcuDrvHandle> ccuDrvHandle_{};
195 : bool useCcuResStaticAlloc_{false}; // HCCL版本不支持CCU资源按需申请
196 : bool ccuMsCommReserved_{false};
197 : Hccl::HcclMainboardId mainBoardType_{Hccl::HcclMainboardId::MAINBOARD_OTHERS};
198 : };
199 :
200 : } // namespace hccl
201 :
202 : namespace MyRankUtils {
203 :
204 : HcommChannelDesc ChannelDescHccl2Hcomm(const HcclChannelDesc& hcclDesc, const hccl::CommConfig& commConfig);
205 :
206 : } // namespace MyRankUtils
207 :
208 : #endif // MY_RANK_H
|
