Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 : #include "channel_manager.h"
11 : #include "adapter_rts_common.h"
12 : #include "log.h"
13 : #include "comm_configer.h"
14 : #include "launch_aicpu.h"
15 : #include "comm_engine_utils.h"
16 : #include <unordered_set>
17 : #include <string>
18 : #include "adapter_prof.h"
19 : #include "hcom_host_profiling.h"
20 :
21 : namespace hccl {
22 :
23 : constexpr u32 RDMA_NOTIFY_MIN_NUM = 3;
24 : constexpr u32 NOTIFY_NUM_MAX = 64; // HcclChannelDesc 中 notifynum 的默认限制最大为64
25 :
26 234 : HcclResult ChannelManager::Init(aclrtBinHandle binHandle, u32 userRank, const ManagerCallbacks& callbacks)
27 : {
28 234 : binHandle_ = binHandle;
29 234 : userRank_ = userRank;
30 234 : callbacks_ = callbacks;
31 234 : return HCCL_SUCCESS;
32 : }
33 :
34 403 : HcclResult ChannelManager::SetChannelCallbacks(const ChannelManagerCallbacks& channelCallbacks)
35 : {
36 403 : channelCallbacks_ = channelCallbacks;
37 403 : rankInfoList_ = channelCallbacks_.getRankLists();
38 403 : return HCCL_SUCCESS;
39 : }
40 :
41 1 : HcclResult ChannelManager::CheckChannelParam(CommEngine engine,
42 : const HcclChannelDesc *channelDesc, uint32_t descNum)
43 : {
44 1 : std::unordered_set<HcclChannelDesc, std::hash<HcclChannelDesc>, HcclChannelDescEqual> descSet;
45 :
46 1 : for (uint32_t descIdx = 0; descIdx < descNum; ++descIdx) {
47 : // 检查notifyNum
48 1 : CHK_PRT_RET(channelDesc[descIdx].notifyNum > NOTIFY_NUM_MAX,
49 : HCCL_ERROR("[%s]Channeldesc[%u] invalid notifyNum, notifyNum[%u], max notify num[%u]",
50 : __func__, descIdx, channelDesc[descIdx].notifyNum, NOTIFY_NUM_MAX), HCCL_E_PARA);
51 : // 检查memHandleNum是否大于0
52 0 : if (channelDesc[descIdx].memHandleNum != 0) {
53 0 : HCCL_WARNING("[%s]Channeldesc[%u] memHandleNum[%u] is non-zero, memHandle exchange is not supported.",
54 : __func__, descIdx, channelDesc[descIdx].memHandleNum);
55 : }
56 : // 检查HcclChannelDesc是否有重复元素
57 0 : CHK_PRT_RET(descSet.find(channelDesc[descIdx]) != descSet.end(),
58 : HCCL_ERROR("[%s]Duplicate item found in hcclchanneldesc.", __func__), HCCL_E_PARA);
59 0 : descSet.insert(channelDesc[descIdx]);
60 : // 检查RemoteRank有效性
61 0 : CHK_PRT_RET(channelDesc[descIdx].remoteRank == userRank_,
62 : HCCL_ERROR("[%s]Local rank found in channeldesc, userRank_ = %u.", __func__, userRank_),
63 : HCCL_E_PARA);
64 : // 检查是否有不支持协议
65 0 : CHK_PRT_RET(channelDesc[descIdx].channelProtocol != COMM_PROTOCOL_HCCS &&
66 : channelDesc[descIdx].channelProtocol != COMM_PROTOCOL_ROCE &&
67 : channelDesc[descIdx].channelProtocol != COMM_PROTOCOL_HCCS_ONLY &&
68 : channelDesc[descIdx].channelProtocol != COMM_PROTOCOL_SIO,
69 : HCCL_ERROR("[%s]Unsupported protocol[%d] found in channeldesc, protocol: %d.", __func__,
70 : descIdx, channelDesc[descIdx].channelProtocol), HCCL_E_PARA);
71 :
72 : // 检查engine支持情况
73 0 : if (engine != COMM_ENGINE_CPU && engine != COMM_ENGINE_CPU_TS &&
74 0 : engine != COMM_ENGINE_AICPU && engine != COMM_ENGINE_AICPU_TS) {
75 0 : HCCL_ERROR("[%s]Unsupported engine for channel, engine: %s.", __func__, GetEnumToString(GetCommEngineStatusStrMap(), engine).c_str());
76 0 : return HCCL_E_PARA;
77 : }
78 : }
79 0 : return HCCL_SUCCESS;
80 1 : }
81 :
82 0 : HcclResult ChannelManager::RegisterHandle(const std::string &tag, CommEngine engine,
83 : const HcclChannelDesc &channelDesc, ChannelHandle channelHandle)
84 : {
85 0 : std::string channelKey = tag + ":" + std::to_string(engine) + ":" + std::to_string(channelDesc.remoteRank) +
86 0 : ":" + std::to_string(channelDesc.channelProtocol);
87 :
88 0 : CHK_PRT_RET((channelHandleMap_.find(channelKey) != channelHandleMap_.end()),
89 : HCCL_ERROR("[%s]Channel already exists, tag[%s], engine[%s], remoteRank[%d], channelProtocol[%d].",
90 : __func__, tag.c_str(), GetEnumToString(GetCommEngineStatusStrMap(), engine).c_str(), channelDesc.remoteRank, channelDesc.channelProtocol), HCCL_E_PARA);
91 0 : channelHandleMap_[channelKey] = channelHandle;
92 0 : keyMap_[channelHandle] = channelKey;
93 0 : engineMap_[channelHandle] = engine;
94 0 : HCCL_INFO("[%s]Register channel handle[%llu], channelKey[%s]", __func__, channelHandle, channelKey.c_str());
95 0 : return HCCL_SUCCESS;
96 0 : }
97 :
98 0 : HcclResult ChannelManager::PrepareHandleArray(const std::string& tag, CommEngine engine, const HcclChannelDesc *channelDesc,
99 : uint32_t descNum, ChannelHandle* channelHandleArray, std::vector<HcclChannelDesc>& needCreateDescs,
100 : std::vector<uint32_t>& needCreateIndices)
101 : {
102 0 : needCreateDescs.clear();
103 0 : needCreateIndices.clear();
104 :
105 0 : for (uint32_t descIdx = 0; descIdx < descNum; descIdx++) {
106 : // 组合channelKey
107 0 : std::string channelKey = tag + ":" + std::to_string(engine) + ":" + std::to_string(channelDesc[descIdx].remoteRank) +
108 0 : ":" + std::to_string(channelDesc[descIdx].channelProtocol);
109 0 : if (channelHandleMap_.find(channelKey) != channelHandleMap_.end()) {
110 0 : channelHandleArray[descIdx] = channelHandleMap_[channelKey];
111 0 : continue;
112 : }
113 0 : channelHandleArray[descIdx] = 0;
114 0 : needCreateDescs.push_back(channelDesc[descIdx]);
115 0 : needCreateIndices.push_back(descIdx);
116 0 : }
117 :
118 0 : return HCCL_SUCCESS;
119 : }
120 :
121 0 : HcclResult ChannelManager::IsChannelExist(ChannelHandle channel)
122 : {
123 0 : CHK_PRT_RET((keyMap_.find(channel) == keyMap_.end()),
124 : HCCL_ERROR("[%s]ChannelHandle is not exist.", __func__), HCCL_E_PARA);
125 0 : HCCL_INFO("[%s]ChannelHandle exist, ChannelHandle[%llu], channelKey[%s]", __func__, channel, keyMap_[channel].c_str());
126 0 : return HCCL_SUCCESS;
127 : }
128 :
129 0 : HcclResult ChannelManager::UnregisterHandle(ChannelHandle channel)
130 : {
131 0 : CHK_PRT_RET((keyMap_.find(channel) == keyMap_.end()),
132 : HCCL_ERROR("[%s]ChannelHandle is not exist.", __func__), HCCL_E_PARA);
133 :
134 0 : channelHandleMap_.erase(keyMap_[channel]);
135 0 : keyMap_.erase(channel);
136 0 : if (engineMap_[channel] == COMM_ENGINE_AICPU ||
137 0 : engineMap_[channel] == COMM_ENGINE_AICPU_TS) {
138 0 : channelD2HMap_.erase(channel);
139 : }
140 0 : engineMap_.erase(channel);
141 :
142 0 : HCCL_INFO("[%s]Unregister channel handle success.", __func__);
143 0 : return HCCL_SUCCESS;
144 : }
145 :
146 0 : HcclResult ChannelManager::RegisterHandleHDPair(ChannelHandle deviceChannelHandle, ChannelHandle hostChannelHandle)
147 : {
148 0 : CHK_PRT_RET((deviceChannelHandle == 0 || hostChannelHandle == 0),
149 : HCCL_ERROR("[%s]ChannelHandle is 0.", __func__), HCCL_E_PARA);
150 0 : CHK_PRT_RET((channelD2HMap_.find(deviceChannelHandle) != channelD2HMap_.end()),
151 : HCCL_ERROR("[%s]deviceChannelHandle has existed in channelD2HMap_.", __func__), HCCL_E_PARA);
152 :
153 0 : channelD2HMap_[deviceChannelHandle] = hostChannelHandle;
154 0 : return HCCL_SUCCESS;
155 : }
156 :
157 0 : HcclResult ChannelManager::GetHostChannel(ChannelHandle channel, ChannelHandle &hostChannel)
158 : {
159 0 : if (engineMap_[channel] == COMM_ENGINE_AICPU ||
160 0 : engineMap_[channel] == COMM_ENGINE_AICPU_TS) {
161 0 : CHK_PRT_RET((channelD2HMap_.find(channel) == channelD2HMap_.end()),
162 : HCCL_ERROR("[%s]device channel handle has not existed in channelD2HMap_.", __func__), HCCL_E_PARA);
163 0 : hostChannel = channelD2HMap_[channel];
164 : } else {
165 0 : hostChannel = channel;
166 : }
167 0 : return HCCL_SUCCESS;
168 : }
169 :
170 1 : void ChannelManager::ClearOpTransportResponseLinks(OpCommTransport &opTransportResponse)
171 : {
172 2 : for (auto &levelNSubCommTransport : opTransportResponse)
173 : {
174 2 : for (auto &singleSubCommTransport : levelNSubCommTransport)
175 : {
176 1 : u32 size = singleSubCommTransport.transportRequests.size();
177 1 : singleSubCommTransport.links.resize(size, nullptr);
178 1 : singleSubCommTransport.status.resize(size, TransportStatus::INIT);
179 1 : HCCL_INFO("[%s] size[%u], linksSize[%zu]", __func__, size, singleSubCommTransport.links.size());
180 : }
181 : }
182 1 : }
183 :
184 0 : HcclResult ChannelManager::CheckNotifyOrQPMaxNum(u64 &existNum, const u64 &MaxNum, const bool &isNotifyRes)
185 : {
186 0 : std::string resType = isNotifyRes ? "Notify" : "QP";
187 0 : if (existNum + 1 > MaxNum)
188 : {
189 0 : HCCL_ERROR("[%s]%s resources are insufficient, existNum[%llu], MaxNum is [%llu]",
190 : __func__, resType.c_str(), existNum, MaxNum);
191 0 : return HCCL_E_INTERNAL;
192 : }
193 0 : HCCL_DEBUG("[%s]%s resources are sufficient, existNum[%llu], MaxNum is [%llu]",
194 : __func__, resType.c_str(), existNum, MaxNum);
195 0 : return HCCL_SUCCESS;
196 0 : }
197 :
198 :
199 0 : HcclResult ChannelManager::CreateWorkSpace(u64 size, DeviceMem &buffer) const
200 : {
201 0 : CHK_PRT_RET(size == 0, HCCL_INFO("[Create][WorkSpace]work space size is zero. not need to malloc memory"),
202 : HCCL_SUCCESS);
203 :
204 : CHK_PRT_RET((size > ULONG_MAX),
205 : HCCL_ERROR("[Create][WorkSpace]work space size is greater than %llu",
206 : ULONG_MAX),
207 : HCCL_E_PARA);
208 :
209 0 : u64 memSize = size;
210 0 : buffer = DeviceMem::alloc(memSize);
211 0 : CHK_PRT_RET(size > 0 && !buffer, HCCL_ERROR("[Create][WorkSpace]Create work space size[%llu] fail,"
212 : "please check workspace size.",
213 : size),
214 : HCCL_E_PTR);
215 0 : CHK_RET(hrtMemSet(buffer.ptr(), size, size));
216 0 : return HCCL_SUCCESS;
217 : }
218 :
219 0 : HcclResult ChannelManager::AllocAndClearHostMem(u64 size, std::shared_ptr<HostMem> &bufferPtr) const
220 : {
221 0 : CHK_PRT_RET(size == 0,
222 : HCCL_INFO("[ChannelManager][AllocAndClearHostMem] host memory size is zero. not need to malloc memory"),
223 : HCCL_SUCCESS);
224 :
225 : CHK_PRT_RET((size > ULONG_MAX),
226 : HCCL_ERROR("[ChannelManager][AllocAndClearHostMem] host memory size is greater than %llu", ULONG_MAX),
227 : HCCL_E_PARA);
228 :
229 0 : HostMem tmpBuffer = HostMem::alloc(size);
230 0 : EXCEPTION_CATCH((bufferPtr = std::make_shared<HostMem>(std::move(tmpBuffer))), return HCCL_E_PTR);
231 :
232 0 : CHK_PRT_RET(size > 0 && !bufferPtr.get()->ptr(),
233 : HCCL_ERROR("[ChannelManager][AllocAndClearHostMem]host memory space size[%llu] fail,"
234 : "please check workspace size.",
235 : size),
236 : HCCL_E_PTR);
237 0 : CHK_SAFETY_FUNC_RET(memset_s(bufferPtr.get()->ptr(), size, 0, size));
238 0 : return HCCL_SUCCESS;
239 0 : }
240 :
241 : template <typename T>
242 0 : HcclResult ChannelManager::CopyVectorToDeviceMem(const u64 len, DeviceMem &dstDeviceMem, const std::vector<T> &srcVec)
243 : {
244 0 : CHK_PRT_RET(len == 0,
245 : HCCL_INFO("[ChannelManager][CopyVectorToDeviceMem] space size is zero. not need to malloc memory"),
246 : HCCL_SUCCESS);
247 :
248 : CHK_PRT_RET((len > ULONG_MAX),
249 : HCCL_ERROR("[ChannelManager][CopyVectorToDeviceMem] space size is greater than %llu", ULONG_MAX),
250 : HCCL_E_PARA);
251 :
252 0 : CHK_RET(CreateWorkSpace(len, dstDeviceMem));
253 0 : std::shared_ptr<HostMem> srcHostMem;
254 0 : CHK_RET(AllocAndClearHostMem(len, srcHostMem));
255 0 : std::copy(srcVec.begin(), srcVec.end(), static_cast<T *>(srcHostMem.get()->ptr()));
256 0 : CHK_RET(hrtMemSyncCopy(
257 : dstDeviceMem.ptr(), len, srcHostMem.get()->ptr(), len, HcclRtMemcpyKind::HCCL_RT_MEMCPY_KIND_HOST_TO_DEVICE));
258 0 : return HCCL_SUCCESS;
259 0 : }
260 :
261 1 : OpCommTransport ChannelManager::BuildChannelRequests(const std::vector<HcclChannelDesc> &descs)
262 : {
263 1 : OpCommTransport opCommTransport;
264 1 : LevelNSubCommTransport level0Transport;
265 1 : SingleSubCommTransport commTransport;
266 :
267 3 : for (auto desc : descs) {
268 2 : TransportRequest tmpTransport;
269 2 : tmpTransport.isValid = true;
270 2 : tmpTransport.localUserRank = userRank_;
271 2 : tmpTransport.remoteUserRank = desc.remoteRank;
272 2 : tmpTransport.notifyNum = desc.notifyNum;
273 2 : tmpTransport.inputMemType = TransportMemType::CCL_INPUT;
274 2 : tmpTransport.outputMemType = TransportMemType::CCL_OUTPUT;
275 2 : tmpTransport.isUsedRdma = (desc.channelProtocol == CommProtocol::COMM_PROTOCOL_ROCE);
276 2 : TransportLinkType linkType = TransportLinkType::RESERVED;
277 2 : if (desc.channelProtocol == CommProtocol::COMM_PROTOCOL_HCCS_ONLY) {
278 0 : linkType = TransportLinkType::HCCS;
279 2 : } else if (desc.channelProtocol == CommProtocol::COMM_PROTOCOL_SIO) {
280 1 : linkType = TransportLinkType::SIO;
281 : }
282 2 : tmpTransport.linkType = linkType;
283 2 : commTransport.transportRequests.push_back(tmpTransport);
284 : }
285 :
286 1 : level0Transport.push_back(commTransport);
287 1 : opCommTransport.push_back(level0Transport);
288 1 : ClearOpTransportResponseLinks(opCommTransport);
289 :
290 1 : return opCommTransport;
291 1 : }
292 :
293 :
294 0 : HcclResult ChannelManager::ParseChannelRemoteDataToMem(const OpCommTransport &opTransportResponse,
295 : HcclIndOpChannelRemoteResV3 &channelParam)
296 : {
297 0 : uint32_t level0 = 0;
298 0 : auto &singleSubCommTransport = opTransportResponse[level0][level0];
299 0 : CHK_PRT_RET(channelParam.listNum == 0,
300 : HCCL_ERROR("[%s]invalid listNum, listNum[%u]", __func__, channelParam.listNum), HCCL_E_PARA);
301 0 : CHK_PRT_RET((channelParam.listNum != singleSubCommTransport.links.size()),
302 : HCCL_ERROR("[%s]invalid listNum, listNum[%u] but links size is [%zu]",
303 : __func__, channelParam.listNum, singleSubCommTransport.links.size()), HCCL_E_PARA);
304 : // 分配 HcclIndOpChannelRemoteResV2 内存，需要手动释放
305 0 : channelParam.remoteResV2 = static_cast<HcclIndOpChannelRemoteResV2*>(malloc(channelParam.listNum * sizeof(HcclIndOpChannelRemoteResV2)));
306 0 : CHK_PRT_RET(channelParam.remoteResV2 == nullptr,
307 : HCCL_ERROR("[%s]channelParam.remoteResV2 is null.", __func__), HCCL_E_MEMORY);
308 0 : u32 linkIdx = 0;
309 0 : for (auto &transportRequest : singleSubCommTransport.transportRequests) {
310 0 : auto &tempLink = singleSubCommTransport.links[linkIdx];
311 0 : channelParam.remoteResV2[linkIdx].remoteWorldRank = rankInfoList_[transportRequest.remoteUserRank].worldRank;
312 0 : channelParam.remoteResV2[linkIdx].remoteRank = transportRequest.remoteUserRank;
313 : // transport信息保存（notify、qp）
314 0 : if (!transportRequest.isUsedRdma) {
315 : // sdma -> P2P
316 0 : CHK_RET(BuildOpRemoteChannelP2pResParam(tempLink, channelParam.remoteResV2[linkIdx]));
317 0 : channelParam.remoteResV2[linkIdx].channelP2p.qos = hcclQos_;
318 0 : HCCL_INFO("[ChannelManager] [ParseChannelRemoteDataToMem] hcclQos[%u]", channelParam.remoteResV2[linkIdx].channelP2p.qos);
319 : } else {
320 : // rdma -> roce
321 0 : CHK_RET(BuildOpRemoteChannelRoceResParam(tempLink, channelParam.remoteResV2[linkIdx]));
322 : }
323 0 : linkIdx++;
324 : }
325 0 : return HCCL_SUCCESS;
326 : }
327 :
328 0 : HcclResult ChannelManager::BuildOpRemoteChannelP2pResParam(const LINK &link, HcclIndOpChannelRemoteResV2 &remoteRes)
329 : {
330 0 : remoteRes.isUsedRdma = false;
331 0 : HcclChannelP2p &linkp2p = remoteRes.channelP2p;
332 : // remoteMem, 独立算子localmem是否需要传待确认
333 0 : void *bufferPtr = nullptr;
334 0 : CHK_RET(link->GetRemoteMem(UserMemType::INPUT_MEM, &bufferPtr));
335 0 : linkp2p.remoteHcclbuffer.addr = reinterpret_cast<void*>(bufferPtr);
336 : u64 remotebufferSize;
337 0 : CHK_RET(link->GetRemoteMemSize(UserMemType::INPUT_MEM, remotebufferSize));
338 0 : linkp2p.remoteHcclbuffer.size = remotebufferSize;
339 : // 独立算子远端用户内存，linkp2p.remoteUserMem需要手动释放内存
340 0 : CHK_RET(link->GetIndOpRemoteMem(&linkp2p.remoteUserMem, &linkp2p.remoteUserMemCount));
341 0 : HCCL_DEBUG("[%s] finish set remoteMem info", __func__);
342 :
343 : // localnotify & remotenotify
344 0 : u64 notifyNum = 0;
345 0 : std::vector<HcclSignalInfo> locIpcSignals;
346 0 : std::vector<HcclSignalInfo> rmtIpcSignals;
347 0 : CHK_RET(link->GetLocalNotify(locIpcSignals));
348 0 : CHK_RET(link->GetRemoteNotify(rmtIpcSignals));
349 :
350 0 : for (size_t i = 0; i < locIpcSignals.size(); i++) {
351 0 : linkp2p.localIpcSignal[notifyNum] = locIpcSignals[i];
352 0 : linkp2p.remoteIpcSignal[notifyNum] = rmtIpcSignals[i];
353 0 : notifyNum++;
354 : }
355 0 : remoteRes.p2pNotifyNum = link->GetNotifyNum();
356 0 : HCCL_DEBUG("[%s] finish set localnotify & remotenotify info, notifyNum[%llu], p2pNotifyNum[%llu]",
357 : __func__, notifyNum, remoteRes.p2pNotifyNum);
358 : // transportAttr
359 0 : CHK_RET(link->GetTransportAttr(linkp2p.transportAttr));
360 0 : HCCL_DEBUG("[%s] finish set RemoteChannelP2pResParam info", __func__);
361 0 : return HCCL_SUCCESS;
362 0 : }
363 :
364 0 : HcclResult ChannelManager::BuildOpRemoteChannelRoceResParam(const LINK &link, HcclIndOpChannelRemoteResV2 &remoteRes)
365 : {
366 0 : remoteRes.isUsedRdma = true;
367 0 : HcclChannelRoce &linkRoce = remoteRes.channelRoce;
368 : // 填充localMem信息到linkRoce中
369 0 : CHK_RET(link->GetLocalMemDetails(UserMemType::INPUT_MEM, linkRoce.localHcclbuffer));
370 : // 填充remoteMem信息到linkRoce中
371 0 : void *bufferPtr = nullptr;
372 0 : CHK_RET(link->GetRemoteMem(UserMemType::INPUT_MEM, &bufferPtr));
373 0 : linkRoce.remoteHcclbuffer.addr = reinterpret_cast<u64>(bufferPtr);
374 0 : CHK_RET(link->GetRemoteMemKey(UserMemType::INPUT_MEM, &(linkRoce.remoteHcclbuffer.key)));
375 0 : CHK_RET(link->GetRemoteMemSize(UserMemType::INPUT_MEM, linkRoce.remoteHcclbuffer.size));
376 : // 独立算子远端用户内存，linkRoce.remoteUserHostMem和remoteUserDeviceMem需要手动释放内存
377 0 : CHK_RET(link->GetIndOpRemoteMemDetails(&linkRoce.remoteUserHostMem, &linkRoce.remoteUserHostMemCount, HcclMemType::HCCL_MEM_TYPE_HOST));
378 0 : CHK_RET(link->GetIndOpRemoteMemDetails(&linkRoce.remoteUserDeviceMem, &linkRoce.remoteUserHostMemCount, HcclMemType::HCCL_MEM_TYPE_DEVICE));
379 0 : HCCL_DEBUG("[%s] finish set remoteMem info", __func__);
380 :
381 : // 填充notifyValue和notifyValueKey信息到linkRoce中
382 0 : std::vector<AddrKey> notifyValueAddrKey;
383 0 : CHK_RET(link->GetLocalNotifyValueAddrKey(notifyValueAddrKey));
384 0 : linkRoce.notifyValue = notifyValueAddrKey[0].addr;
385 0 : linkRoce.notifyValueKey = notifyValueAddrKey[0].key;
386 :
387 : // 填充QP信息到linkRoce中
388 0 : std::vector<HcclQpInfoV2> aiQpInfos;
389 0 : CHK_RET(link->GetAiQpInfo(aiQpInfos));
390 0 : u32 qpNum = aiQpInfos.size();
391 0 : if (qpNum > RDMA_QP_MAX_NUM || qpNum < 1) {
392 0 : return HCCL_E_INTERNAL;
393 : }
394 0 : std::copy_n(aiQpInfos.begin(), qpNum, linkRoce.QpInfo);
395 0 : linkRoce.qpsPerConnection = qpNum - static_cast<u32>(qpNum > 1); // 多QP数量或单QP模式
396 :
397 : // 填充localNotify和remoteNotify信息到linkRoce中
398 0 : std::vector<AddrKey> notifyAddrKey;
399 0 : std::vector<HcclSignalInfo> signalInfos;
400 0 : CHK_RET(link->GetLocalRdmaNotify(signalInfos));
401 0 : CHK_RET(link->GetRemoteRdmaNotifyAddrKey(notifyAddrKey));
402 0 : if ((signalInfos.size() != notifyAddrKey.size()) || (signalInfos.size() < RDMA_NOTIFY_MIN_NUM) ||
403 0 : (signalInfos.size() > RDMA_NOTIFY_MAX_NUM) || (notifyAddrKey.size() < RDMA_NOTIFY_MIN_NUM) ||
404 0 : (notifyAddrKey.size() > RDMA_NOTIFY_MAX_NUM) ||
405 0 : ((signalInfos.size() - RDMA_NOTIFY_MIN_NUM) % linkRoce.qpsPerConnection) != 0 ||
406 0 : ((notifyAddrKey.size() - RDMA_NOTIFY_MIN_NUM) % linkRoce.qpsPerConnection) != 0) {
407 0 : return HCCL_E_INTERNAL;
408 : }
409 0 : u64 notifyNum = (notifyAddrKey.size() - RDMA_NOTIFY_MIN_NUM) / linkRoce.qpsPerConnection - static_cast<u32>(linkRoce.qpsPerConnection > 1);
410 0 : linkRoce.singleQPNotifyNum = notifyNum;
411 :
412 0 : u64 len = signalInfos.size() * sizeof(HcclSignalInfo);
413 0 : DeviceMem localNotifyListMem;
414 0 : CHK_RET(CopyVectorToDeviceMem(len, localNotifyListMem, signalInfos));
415 0 : linkRoce.localNotifyList = reinterpret_cast<u64>(localNotifyListMem.ptr());
416 0 : channelParamMemList_.emplace_back(std::move(localNotifyListMem));
417 :
418 0 : len = notifyAddrKey.size() * sizeof(AddrKey);
419 0 : DeviceMem remoteNotifyListMem;
420 0 : CHK_RET(CopyVectorToDeviceMem(len, remoteNotifyListMem, notifyAddrKey));
421 0 : linkRoce.remoteNotifyList = reinterpret_cast<u64>(remoteNotifyListMem.ptr());
422 0 : channelParamMemList_.emplace_back(std::move(remoteNotifyListMem));
423 :
424 0 : remoteRes.roceNotifyNum = linkRoce.singleQPNotifyNum;
425 0 : remoteRes.qpNum = linkRoce.qpsPerConnection;
426 :
427 0 : return HCCL_SUCCESS;
428 0 : }
429 :
430 0 : HcclResult ChannelManager::DeepCopyH2DchannelParam(const HcclIndOpChannelRemoteResV3 &hostChannelParam,
431 : HcclIndOpChannelRemoteResV3 &deviceChannelParam)
432 : {
433 0 : deviceChannelParam = hostChannelParam;
434 : // 拷贝remoteResV2
435 :
436 0 : if (hostChannelParam.remoteResV2 != nullptr && hostChannelParam.listNum > 0) {
437 : // 为设备端的remoteResV2数组分配内存（注意：这个数组存放的是HcclIndOpChannelRemoteResV2结构体）
438 0 : size_t remoteResV2ArraySize = sizeof(HcclIndOpChannelRemoteResV2) * hostChannelParam.listNum;
439 0 : std::shared_ptr<DeviceMem> deviceRemoteResV2Array;
440 0 : EXCEPTION_CATCH(
441 : (deviceRemoteResV2Array = std::make_shared<DeviceMem>(DeviceMem::alloc(remoteResV2ArraySize))),
442 : return HCCL_E_PTR);
443 :
444 : // 为每个数组元素进行深度拷贝，并保存设备内存和主机结构体（指针已调整）
445 0 : std::vector<DeviceMem> elementMemories; // 保存每个元素分配的设备内存（包括内部指针数据）
446 0 : std::vector<HcclIndOpChannelRemoteResV2> hostRemoteResV2Array(hostChannelParam.listNum);
447 :
448 0 : for (uint32_t i = 0; i < hostChannelParam.listNum; ++i) {
449 0 : HcclIndOpChannelRemoteResV2 hostElement = hostChannelParam.remoteResV2[i];
450 0 : HcclIndOpChannelRemoteResV2 deviceElement;
451 : // 深度拷贝一个元素到设备内存，并返回设备内存中的结构体布局（host端）
452 0 : CHK_RET(DeepCopyH2DChannelRemoteResV2(hostElement, deviceElement));
453 : // 保存调整后的主机端结构体（其指针指向设备内存）
454 0 : hostRemoteResV2Array[i] = deviceElement;
455 : }
456 :
457 : // 将主机端的结构体数组（指针已调整）拷贝到设备内存数组
458 0 : CHK_RET(hrtMemSyncCopy(deviceRemoteResV2Array.get()->ptr(), remoteResV2ArraySize, hostRemoteResV2Array.data(),
459 : remoteResV2ArraySize, HcclRtMemcpyKind::HCCL_RT_MEMCPY_KIND_HOST_TO_DEVICE));
460 :
461 : // 更新设备端参数中的remoteResV2指针
462 0 : deviceChannelParam.remoteResV2 = reinterpret_cast<HcclIndOpChannelRemoteResV2*>(deviceRemoteResV2Array.get()->ptr());
463 0 : channelParamMemVector_.push_back(std::move(deviceRemoteResV2Array));
464 0 : } else {
465 0 : HCCL_ERROR("[%s]invalid hostChannelParam", __func__);
466 0 : return HCCL_E_INTERNAL;
467 : }
468 0 : return HCCL_SUCCESS;
469 : }
470 :
471 0 : HcclResult ChannelManager::DeepCopyH2DChannelRemoteResV2(const HcclIndOpChannelRemoteResV2 &hostRemoteResV2,
472 : HcclIndOpChannelRemoteResV2 &deviceRemoteResV2)
473 : {
474 : // 复制基本成员
475 0 : deviceRemoteResV2 = hostRemoteResV2;
476 : // 根据通信类型处理不同的通道
477 0 : if (hostRemoteResV2.isUsedRdma) {
478 : // 处理RoCE通道
479 0 : CHK_RET(DeepCopyH2DChannelRoce(
480 : hostRemoteResV2.channelRoce,
481 : deviceRemoteResV2.channelRoce));
482 : } else {
483 : // 处理P2P通道
484 0 : CHK_RET(DeepCopyH2DChannelP2p(
485 : hostRemoteResV2.channelP2p,
486 : deviceRemoteResV2.channelP2p));
487 : }
488 0 : return HCCL_SUCCESS;
489 : }
490 :
491 0 : HcclResult ChannelManager::DeepCopyH2DChannelRoce(const HcclChannelRoce &hostChannelRoce,
492 : HcclChannelRoce &deviceChannelRoce)
493 : {
494 : // 复制基本成员
495 0 : deviceChannelRoce = hostChannelRoce;
496 : // 处理remoteUserHostMem
497 0 : if (hostChannelRoce.remoteUserHostMem != nullptr && hostChannelRoce.remoteUserHostMemCount > 0) {
498 0 : size_t remoteUserHostMemSize = hostChannelRoce.remoteUserHostMemCount * sizeof(MemDetails);
499 0 : std::shared_ptr<DeviceMem> deviceMem;
500 0 : EXCEPTION_CATCH((deviceMem = std::make_shared<DeviceMem>(DeviceMem::alloc(remoteUserHostMemSize))),
501 : return HCCL_E_PTR);
502 0 : CHK_RET(hrtMemSyncCopy(deviceMem.get()->ptr(), remoteUserHostMemSize, hostChannelRoce.remoteUserHostMem,
503 : remoteUserHostMemSize, HcclRtMemcpyKind::HCCL_RT_MEMCPY_KIND_HOST_TO_DEVICE));
504 0 : deviceChannelRoce.remoteUserHostMem = reinterpret_cast<MemDetails*>(deviceMem.get()->ptr());
505 0 : channelParamMemVector_.push_back(std::move(deviceMem));
506 0 : } else {
507 0 : deviceChannelRoce.remoteUserHostMem = nullptr;
508 : }
509 : // 处理remoteUserDeviceMem
510 0 : if (hostChannelRoce.remoteUserDeviceMem != nullptr && hostChannelRoce.remoteUserDeviceMemCount > 0) {
511 0 : size_t remoteUserDeviceMemSize = hostChannelRoce.remoteUserDeviceMemCount * sizeof(MemDetails);
512 0 : std::shared_ptr<DeviceMem> deviceMem;
513 0 : EXCEPTION_CATCH((deviceMem = std::make_shared<DeviceMem>(DeviceMem::alloc(remoteUserDeviceMemSize))),
514 : return HCCL_E_PTR);
515 0 : CHK_RET(hrtMemSyncCopy(deviceMem.get()->ptr(), remoteUserDeviceMemSize, hostChannelRoce.remoteUserDeviceMem,
516 : remoteUserDeviceMemSize, HcclRtMemcpyKind::HCCL_RT_MEMCPY_KIND_HOST_TO_DEVICE));
517 0 : deviceChannelRoce.remoteUserDeviceMem = reinterpret_cast<MemDetails*>(deviceMem.get()->ptr());
518 0 : channelParamMemVector_.push_back(std::move(deviceMem));
519 0 : } else {
520 0 : deviceChannelRoce.remoteUserDeviceMem = nullptr;
521 : }
522 :
523 0 : return HCCL_SUCCESS;
524 : }
525 :
526 0 : HcclResult ChannelManager::DeepCopyH2DChannelP2p(const HcclChannelP2p &hostChannelP2p,
527 : HcclChannelP2p &deviceChannelP2p)
528 : {
529 : // 复制基本成员
530 0 : deviceChannelP2p = hostChannelP2p;
531 : // 处理remoteUserMem
532 0 : if (hostChannelP2p.remoteUserMem != nullptr && hostChannelP2p.remoteUserMemCount > 0) {
533 0 : size_t remoteUserMemSize = hostChannelP2p.remoteUserMemCount * sizeof(HcclMem);
534 0 : std::shared_ptr<DeviceMem> deviceMem;
535 0 : EXCEPTION_CATCH((deviceMem = std::make_shared<DeviceMem>(DeviceMem::alloc(remoteUserMemSize))),
536 : return HCCL_E_PTR);
537 0 : CHK_RET(hrtMemSyncCopy(deviceMem.get()->ptr(), remoteUserMemSize, hostChannelP2p.remoteUserMem,
538 : remoteUserMemSize, HcclRtMemcpyKind::HCCL_RT_MEMCPY_KIND_HOST_TO_DEVICE));
539 0 : deviceChannelP2p.remoteUserMem = reinterpret_cast<HcclMem*>(deviceMem.get()->ptr());
540 0 : channelParamMemVector_.push_back(std::move(deviceMem));
541 0 : } else {
542 0 : deviceChannelP2p.remoteUserMem = nullptr;
543 : }
544 0 : return HCCL_SUCCESS;
545 : }
546 :
547 0 : HcclResult ChannelManager::ReleaseChannelParam(HcclIndOpChannelRemoteResV3 &channelParam) {
548 : // 释放remoteResV2
549 0 : if (channelParam.remoteResV2 != nullptr) {
550 0 : for (uint32_t i = 0; i < channelParam.listNum; ++i) {
551 0 : HcclIndOpChannelRemoteResV2 &remoteRes = channelParam.remoteResV2[i];
552 0 : if (remoteRes.isUsedRdma) {
553 0 : if (remoteRes.channelRoce.remoteUserHostMem != nullptr) {
554 0 : free(remoteRes.channelRoce.remoteUserHostMem);
555 : }
556 0 : if (remoteRes.channelRoce.remoteUserDeviceMem != nullptr) {
557 0 : free(remoteRes.channelRoce.remoteUserDeviceMem);
558 : }
559 : } else {
560 0 : if (remoteRes.channelP2p.remoteUserMem != nullptr) {
561 0 : free(remoteRes.channelP2p.remoteUserMem);
562 : }
563 : }
564 : }
565 : }
566 0 : free(channelParam.remoteResV2);
567 0 : channelParam.remoteResV2 = nullptr;
568 :
569 : // 将kernel下发时临时分配的deviceMem一起销毁
570 0 : channelParamMemVector_.clear();
571 0 : channelParamMemList_.clear();
572 0 : return HCCL_SUCCESS;
573 : }
574 :
575 0 : HcclResult ChannelManager::AicpuChannelInit(const std::string &commId, const std::string &tag, CommEngine engine,
576 : const OpCommTransport &opTransportResponse, ChannelHandle *channelList, uint32_t listNum)
577 : {
578 0 : HcclIndOpChannelRemoteResV3 channelParam{};
579 0 : CHK_SAFETY_FUNC_RET(memset_s(&channelParam, sizeof(channelParam), 0, sizeof(channelParam)));
580 0 : uint64_t beginTime = hrtMsprofSysCycleTime();
581 : // channelParam资源参数填充
582 0 : strncpy_s(channelParam.hcomId, HCOMID_MAX_LENGTH, commId.c_str(), HCOMID_MAX_LENGTH - 1);
583 0 : strncpy_s(channelParam.channelTag, TAG_MAX_LENGTH, tag.c_str(), TAG_MAX_LENGTH - 1);
584 0 : channelParam.engine = engine;
585 0 : channelParam.localUserRank = userRank_;
586 0 : channelParam.multiQpThreshold = GetExternalInputMultiQpThreshold();
587 :
588 : // 为device侧的channelList分配内存
589 0 : DeviceMem deviceChannelList = DeviceMem::alloc(listNum * sizeof(ChannelHandle));
590 0 : CHK_PTR_NULL(deviceChannelList.ptr());
591 0 : channelParam.channelList = static_cast<void*>(deviceChannelList.ptr());
592 0 : channelParam.listNum = listNum;
593 :
594 : // 将建链获取的远端数据填充到channelParam
595 0 : HcclResult ret = ParseChannelRemoteDataToMem(opTransportResponse, channelParam);
596 0 : if (ret != HCCL_SUCCESS) {
597 0 : HCCL_ERROR("[%s] ParseChannelRemoteDataToMem failed, return [%d].", __func__, ret);
598 0 : ReleaseChannelParam(channelParam);
599 0 : return ret;
600 : }
601 :
602 : // 创建局部流
603 0 : Stream localStream(StreamType::STREAM_TYPE_ONLINE);
604 0 : constexpr u32 aicpuStreamMode = 1;
605 0 : CHK_RET(hrtStreamSetMode(localStream.ptr(), aicpuStreamMode));
606 :
607 : // 将channelParam内部的host内存拷贝成device内存
608 0 : HcclIndOpChannelRemoteResV3 deviceChannelParam = channelParam;
609 0 : CHK_RET(DeepCopyH2DchannelParam(channelParam, deviceChannelParam));
610 :
611 0 : DeviceMem addr = DeviceMem::alloc(sizeof(deviceChannelParam));
612 0 : CHK_PTR_NULL(addr.ptr());
613 0 : CHK_RET(hrtMemSyncCopy(addr.ptr(), sizeof(deviceChannelParam), &deviceChannelParam, sizeof(deviceChannelParam),
614 : HcclRtMemcpyKind::HCCL_RT_MEMCPY_KIND_HOST_TO_DEVICE));
615 :
616 : // 下kernel
617 0 : std::string kernelName = "RunAicpuIndOpChannelInit";
618 : struct InitTask {
619 : u64 context;
620 : bool isCustom;
621 : };
622 0 : InitTask customInitTask = {0};
623 0 : customInitTask.context = reinterpret_cast<u64>(addr.ptr());
624 0 : customInitTask.isCustom = false;
625 :
626 0 : u16 timeOut = NOTIFY_DEFAULT_WAIT_TIME > std::numeric_limits<uint16_t>::max() ?
627 : std::numeric_limits<uint16_t>::max() : NOTIFY_DEFAULT_WAIT_TIME;
628 0 : CHK_RET(AicpuAclKernelLaunch(localStream.ptr(), reinterpret_cast<void *>(&customInitTask),
629 : sizeof(customInitTask), binHandle_, kernelName, true, timeOut));
630 0 : CHK_RET(hcclStreamSynchronize(localStream.ptr(), CommConfiger::GetInstance().GetCommConfigExecTimeOut(tag)));
631 :
632 : // 将device侧的channelList拷贝回host侧的channelList
633 0 : CHK_RET(hrtMemSyncCopy(channelList, listNum * sizeof(ChannelHandle),
634 : deviceChannelList.ptr(), listNum * sizeof(ChannelHandle),
635 : HcclRtMemcpyKind::HCCL_RT_MEMCPY_KIND_DEVICE_TO_HOST));
636 :
637 : // 手动释放channelParam中申请的内存
638 0 : CHK_RET(ReleaseChannelParam(channelParam));
639 0 : const std::string profName = "RunAicpuIndOpChannelInit";
640 0 : HCCL_DEBUG("[%s] RunAicpuIndOpChannelInit",__func__);
641 : // 上报初始化kernel的时间
642 0 : HcommProfilingReportKernel(beginTime, profName.c_str());
643 0 : return HCCL_SUCCESS;
644 0 : }
645 :
646 : const std::map<CommEngine, std::string> COMM_ENGINE_TYPE_STR_MAP {
647 : {CommEngine::COMM_ENGINE_CPU, "host_cpu"},
648 : {CommEngine::COMM_ENGINE_CPU_TS, "host_cpu_ts"},
649 : {CommEngine::COMM_ENGINE_AICPU, "aicpu"},
650 : {CommEngine::COMM_ENGINE_AICPU_TS, "aicpu_ts"},
651 : {CommEngine::COMM_ENGINE_AIV, "aiv"},
652 : {CommEngine::COMM_ENGINE_CCU, "ccu"},
653 : {CommEngine::COMM_ENGINE_RESERVED, "reserved"}
654 : };
655 :
656 0 : std::string GetCommEngineEnumStr(CommEngine engine)
657 : {
658 0 : auto iter = COMM_ENGINE_TYPE_STR_MAP.find(engine);
659 0 : if (iter == COMM_ENGINE_TYPE_STR_MAP.end()) {
660 0 : return "CommEngine=" + std::to_string(engine);
661 : } else {
662 0 : return iter->second;
663 : }
664 : }
665 :
666 1 : HcclResult ChannelManager::ChannelCommCreate(const std::string &commId, CommEngine engine,
667 : const HcclChannelDesc *channelDescList, uint32_t listNum, ChannelHandle *channelList)
668 : {
669 1 : CHK_RET(CheckChannelParam(engine, channelDescList, listNum));
670 :
671 : // channel复用，以tag + engine + remoterank + channelProtocol 作为channel标识
672 0 : std::vector<HcclChannelDesc> needCreateDescs;
673 0 : std::vector<uint32_t> needCreateIndices;
674 0 : std::string tag = commId;
675 0 : CHK_RET(PrepareHandleArray(tag, engine, channelDescList, listNum, channelList, needCreateDescs, needCreateIndices));
676 :
677 : // 对未复用的channelDesc进行建链
678 0 : if (needCreateDescs.size() > 0) {
679 : // 构造建链param
680 0 : OpCommTransport opCommTransport = BuildChannelRequests(needCreateDescs);
681 0 : std::string linkTag = commId + "_" + GetCommEngineEnumStr(engine);
682 0 : bool isAicpuModeEn = false;
683 0 : if (engine == COMM_ENGINE_AICPU || engine == COMM_ENGINE_AICPU_TS) {
684 0 : isAicpuModeEn = true;
685 : }
686 0 : CHK_RET(channelCallbacks_.indOpTransportAlloc(linkTag, opCommTransport, isAicpuModeEn));
687 :
688 0 : uint32_t level0 = 0;
689 0 : std::vector<LINK> links = opCommTransport[level0][level0].links;
690 0 : uint32_t newDescNum = needCreateDescs.size();
691 : // 创建host或device侧channel句柄
692 0 : if (isAicpuModeEn) {
693 : //Kernel下发恢复
694 0 : if (!callbacks_.getAicpuCommState()) {
695 0 : HcclResult ret = callbacks_.kernelLaunchAicpuCommInit();
696 0 : CHK_PRT_RET(ret != HCCL_SUCCESS,
697 : HCCL_ERROR("[%s] kernelLaunchAicpuCommInit failed, return [%d].", __func__, ret), ret);
698 0 : callbacks_.setAicpuCommState(true);
699 : }
700 0 : std::unique_ptr<ChannelHandle[]> tmpChannelList = std::make_unique<ChannelHandle[]>(newDescNum);
701 0 : CHK_RET(AicpuChannelInit(commId, tag, engine, opCommTransport, tmpChannelList.get(), newDescNum));
702 0 : for (u32 i = 0; i < newDescNum; i++) {
703 0 : uint32_t arrayIndex = needCreateIndices[i];
704 0 : channelList[arrayIndex] = tmpChannelList[i];
705 0 : CHK_RET(RegisterHandle(tag, engine, needCreateDescs[i], tmpChannelList[i]));
706 0 : ChannelHandle channelHandle = reinterpret_cast<ChannelHandle>(links[i].get());
707 0 : CHK_RET(RegisterHandleHDPair(tmpChannelList[i], channelHandle));
708 : }
709 0 : } else {
710 0 : for (u32 i = 0; i < newDescNum; i++) {
711 0 : uint32_t arrayIndex = needCreateIndices[i];
712 0 : ChannelHandle channelHandle = reinterpret_cast<ChannelHandle>(links[i].get());
713 0 : channelList[arrayIndex] = channelHandle;
714 0 : CHK_RET(RegisterHandle(tag, engine, needCreateDescs[i], channelHandle));
715 : }
716 : }
717 : // 保存link
718 0 : for (auto& link : links) {
719 : // 设置成员变量保存link
720 0 : channelLinks_.push_back(link);
721 : }
722 0 : }
723 0 : return HCCL_SUCCESS;
724 0 : }
725 :
726 0 : HcclResult ChannelManager::ChannelCommGetNotifyNum(ChannelHandle channel, uint32_t *notifyNum)
727 : {
728 0 : CHK_RET(IsChannelExist(channel));
729 : ChannelHandle hostchannel;
730 0 : CHK_RET(GetHostChannel(channel, hostchannel));
731 :
732 0 : Transport* transportPtr = reinterpret_cast<Transport*>(hostchannel);
733 0 : *notifyNum = transportPtr->GetNotifyNum();
734 0 : return HCCL_SUCCESS;
735 : }
736 :
737 0 : HcclResult ChannelManager::ChannelCommDestroy(ChannelHandle *channelList, uint32_t channelNum)
738 : {
739 0 : for (uint32_t i = 0; i < channelNum; ++i) {
740 0 : UnregisterHandle(channelList[i]);
741 0 : channelList[i] = 0;
742 : }
743 0 : return HCCL_SUCCESS;
744 : }
745 :
746 0 : HcclResult ChannelManager::ChannelCommGetHcclBuffer(ChannelHandle channel, CommBuffer *buffer)
747 : {
748 : ChannelHandle hostchannel;
749 0 : CHK_RET(IsChannelExist(channel));
750 0 : CHK_RET(GetHostChannel(channel, hostchannel));
751 0 : Transport* transportPtr = reinterpret_cast<Transport*>(hostchannel);
752 :
753 0 : buffer->addr = nullptr;
754 0 : CHK_RET(transportPtr->GetRemoteMem(UserMemType::INPUT_MEM, &buffer->addr));
755 0 : CHK_PTR_NULL(buffer->addr);
756 0 : u64 tempSize = 0;
757 0 : CHK_RET(transportPtr->GetRemoteMemSize(UserMemType::INPUT_MEM, tempSize));
758 0 : buffer->size = static_cast<uint64_t>(tempSize);
759 0 : buffer->type = HCCL_MEM_TYPE_DEVICE;
760 0 : HCCL_INFO("[%s]channel[%llu] channelKey[%s] get remote hccl buffer success, remote addr[%p], size[%llu]",
761 : __func__, channel, keyMap_[channel].c_str(), buffer->addr, buffer->size);
762 0 : return HCCL_SUCCESS;
763 : }
764 :
765 0 : HcclResult ChannelManager::ChannelCommGetRemoteMem(ChannelHandle channel, HcclMem **remoteMem, uint32_t *memNum)
766 : {
767 0 : CHK_RET(IsChannelExist(channel));
768 : ChannelHandle hostchannel;
769 0 : CHK_RET(GetHostChannel(channel, hostchannel));
770 0 : Transport* transportPtr = reinterpret_cast<Transport*>(hostchannel);
771 :
772 0 : CHK_RET(transportPtr->GetIndOpRemoteMem(remoteMem, memNum));
773 0 : HCCL_INFO("[%s]get remote mem success, mem num[%u]", __func__, *memNum);
774 0 : return HCCL_SUCCESS;
775 : }
776 :
777 401 : HcclResult ChannelManager::ReleaseChannel()
778 : {
779 401 : for (auto &link : channelLinks_) {
780 0 : if (link != nullptr) {
781 0 : if (link->DeInit() != HCCL_SUCCESS) {
782 0 : HCCL_ERROR("[%s]transport[%p] deinit failed.", __func__, link.get());
783 : }
784 : }
785 : }
786 402 : channelLinks_.clear();
787 402 : return HCCL_SUCCESS;
788 : }
789 :
790 234 : HcclResult ChannelManager::SetHcclQos(u32 hcclQos)
791 : {
792 234 : HCCL_INFO("[ChannelManager] [SetHcclQos] hcclQos[%u]", hcclQos);
793 234 : hcclQos_ = hcclQos;
794 234 : return HCCL_SUCCESS;
795 : }
796 : } // namespace hccl
|
