Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "mr_manager.h"
12 : #include "adapter_hal.h"
13 : #include "adapter_hccp.h"
14 : #include "network_manager_pub.h"
15 : #include "../resource/socket/hccl_network.h"
16 :
17 : namespace hccl {
18 : using namespace std;
19 : u64 MrManager::g_devAddr = 0;
20 : map<HostMappingKey, HostMappingInfo> MrManager::mappedHostToDevMap_ = {};
21 : std::mutex MrManager::mappedHostToDevMutex_;
22 :
23 0 : MrManager &MrManager::GetInstance()
24 : {
25 0 : static MrManager hcclMrManager;
26 0 : return hcclMrManager;
27 : }
28 :
29 0 : MrManager::MrManager()
30 0 : : rdmaHandle_(nullptr), count_(0)
31 : {
32 0 : }
33 :
34 0 : MrManager::MrManager(HcclNetDevCtx netDevCtx)
35 0 : : rdmaHandle_(nullptr), count_(0), netDevCtx_(netDevCtx)
36 : {
37 0 : }
38 :
39 0 : MrManager::~MrManager()
40 : {
41 0 : }
42 :
43 0 : HcclResult MrManager::Init(QpHandle qpHandle, u32 devId, bool isHostMem, map<MrMapKey, MrInfo>& unRegMrMap)
44 : {
45 0 : CHK_PTR_NULL(qpHandle);
46 0 : unique_lock<std::mutex> lockUnMrMap(unMrMapSpinMutex_);
47 0 : unRegMrMap_ = unRegMrMap;
48 0 : lockUnMrMap.unlock();
49 0 : SetHdcPara(devId, isHostMem, true);
50 0 : CHK_RET(InitMrManager(qpHandle));
51 0 : return HCCL_SUCCESS;
52 0 : }
53 :
54 0 : HcclResult MrManager::Init(RdmaHandle rdmaHandle, u32 devId, bool isHostMem)
55 : {
56 0 : CHK_PTR_NULL(rdmaHandle);
57 0 : SetHdcPara(devId, isHostMem, false);
58 0 : CHK_RET(InitMrManager(rdmaHandle));
59 0 : return HCCL_SUCCESS;
60 : }
61 :
62 0 : HcclResult MrManager::Init(RdmaHandle rdmaHandle)
63 : {
64 0 : CHK_PTR_NULL(rdmaHandle);
65 0 : return InitMrManager(rdmaHandle);
66 : }
67 :
68 0 : HcclResult MrManager::Init()
69 : {
70 0 : CHK_PTR_NULL(netDevCtx_);
71 0 : RaResourceInfo raResourceInfo;
72 0 : s32 deviceLogicId = (static_cast<hccl::NetDevContext *>(netDevCtx_))->GetLogicId();
73 0 : HcclIpAddress localIp = (static_cast<hccl::NetDevContext *>(netDevCtx_))->GetLocalIp();
74 0 : CHK_RET(NetworkManager::GetInstance(deviceLogicId).GetRaResourceInfo(raResourceInfo));
75 0 : void *nicRdmaHandle = raResourceInfo.nicSocketMap[localIp].nicRdmaHandle;
76 0 : return InitMrManager(nicRdmaHandle);
77 0 : }
78 :
79 0 : HcclResult MrManager::InitUnRegMrMap()
80 : {
81 0 : unique_lock<std::mutex> lockUnMrMap(unMrMapSpinMutex_);
82 0 : for (auto &iter : unRegMrMap_) {
83 : // 目前全局内存由于地址非法注册失败返回成功，需要driver修复进程退出不通知通信库解注册内存问题
84 0 : CHK_RET(RegMr(iter.second.addr, iter.second.size));
85 : // 内存注册失败，mrHandl为空，不用记录
86 0 : MrMapKey mrMapKey(reinterpret_cast<u64>(iter.second.addr), iter.second.size);
87 0 : unique_lock<std::mutex> lockMrMap(mrMapSpinMutex_);
88 0 : if (regedMrMap_.find(mrMapKey) != regedMrMap_.end()) {
89 0 : auto iterator = regedMrMap_.find(iter.first);
90 0 : if (iterator != regedMrMap_.end()) {
91 0 : iterator->second.gloMemRef = iter.second.gloMemRef;
92 : }
93 : }
94 0 : lockMrMap.unlock();
95 0 : }
96 :
97 0 : unRegMrMap_.clear();
98 0 : lockUnMrMap.unlock();
99 0 : return HCCL_SUCCESS;
100 0 : }
101 :
102 0 : HcclResult MrManager::InitUnRegMrMap(map<MrMapKey, MrInfo>& unRegMrMap)
103 : {
104 0 : unique_lock<std::mutex> lockUnMrMap(unMrMapSpinMutex_);
105 0 : unRegMrMap_ = unRegMrMap;
106 0 : lockUnMrMap.unlock();
107 0 : CHK_RET(InitUnRegMrMap());
108 0 : return HCCL_SUCCESS;
109 0 : }
110 :
111 0 : HcclResult MrManager::InitMrManager(void *handle)
112 : {
113 0 : CHK_PTR_NULL(handle);
114 0 : if (++count_ == COUNT_ONE) {
115 0 : if (isUseQPHandle_) {
116 0 : qpHandle_ = handle;
117 : } else {
118 0 : rdmaHandle_ = handle;
119 : }
120 0 : CHK_RET(InitUnRegMrMap());
121 0 : } else if (count_ > COUNT_ONE) {
122 0 : if (rdmaHandle_ != handle && qpHandle_ != handle) {
123 0 : HCCL_ERROR("[MrManager][Init]mr manager init failed, count[%d].", count_.load());
124 0 : return HCCL_E_PARA;
125 : }
126 : }
127 0 : HCCL_INFO("[MrManager][Init]mr manager init success, count[%d]", count_.load());
128 0 : return HCCL_SUCCESS;
129 : }
130 :
131 0 : HcclResult MrManager::DeInit()
132 : {
133 0 : RaResourceInfo raResourceInfo;
134 0 : s32 deviceLogicId = (static_cast<hccl::NetDevContext *>(netDevCtx_))->GetLogicId();
135 0 : HcclIpAddress localIp = (static_cast<hccl::NetDevContext *>(netDevCtx_))->GetLocalIp();
136 0 : CHK_RET(NetworkManager::GetInstance(deviceLogicId).GetRaResourceInfo(raResourceInfo));
137 0 : void *nicRdmaHandle = raResourceInfo.nicSocketMap[localIp].nicRdmaHandle;
138 0 : return DeInit(nicRdmaHandle);
139 0 : }
140 :
141 0 : HcclResult MrManager::DeInit(const void *handle)
142 : {
143 0 : CHK_PTR_NULL(handle);
144 0 : if (rdmaHandle_ == handle || qpHandle_ == handle) {
145 0 : --count_;
146 0 : if (count_ > 0) {
147 0 : HCCL_INFO("[MrManager][DeInit]mr manager deinit success, count[%d].", count_.load());
148 0 : return HCCL_SUCCESS;
149 0 : } else if (count_ == 0) {
150 0 : ReleaseMrResource();
151 0 : if (isUseQPHandle_) {
152 0 : qpHandle_ = nullptr;
153 : } else {
154 0 : rdmaHandle_ = nullptr;
155 : }
156 : }
157 : } else {
158 0 : HCCL_ERROR("[MrManager][DeInit]count[%d]", count_.load());
159 0 : return HCCL_E_PARA;
160 : }
161 0 : HCCL_INFO("[MrManager][DeInit]mr manager deinit success, count[%d].", count_.load());
162 0 : return HCCL_SUCCESS;
163 : }
164 :
165 0 : bool MrManager::IsRequireMapping(void *addr, u64 size, void *&devVirAddr)
166 : {
167 0 : u64 userAddr = reinterpret_cast<u64>(addr);
168 0 : u64 userSize = size;
169 0 : if (mappedHostToDevMap_.size() == 0) {
170 0 : return true;
171 : }
172 :
173 0 : auto iter = SearchMappingMap(userAddr, userSize);
174 0 : if (iter != mappedHostToDevMap_.end()) {
175 0 : u64 tmpDva = reinterpret_cast<u64>(iter->second.devVirAddr) + userAddr - iter->first.addr;
176 0 : devVirAddr = reinterpret_cast<void*>(static_cast<uintptr_t>(tmpDva));
177 0 : iter->second.mappingRef++;
178 0 : return false;
179 : }
180 :
181 0 : return true;
182 : }
183 :
184 0 : map<MrMapKey, MrInfo> MrManager::GetUnregMap()
185 : {
186 0 : unique_lock<std::mutex> lockUnMrMap(unMrMapSpinMutex_);
187 0 : return unRegMrMap_;
188 0 : }
189 :
190 0 : std::map<HostMappingKey, HostMappingInfo>::iterator MrManager::SearchMappingMap(u64 userAddr, u64 userSize)
191 : {
192 0 : for (auto iter = mappedHostToDevMap_.begin(); iter != mappedHostToDevMap_.end(); ++iter) {
193 0 : if ((userAddr >= iter->first.addr) &&
194 0 : (userAddr + userSize <= iter->first.size + iter->first.addr) &&
195 0 : (iter->first.devId == curDevId_)) {
196 0 : return iter;
197 : }
198 : }
199 0 : return mappedHostToDevMap_.end();
200 : }
201 :
202 0 : HcclResult MrManager::RegMrImpl(void *addr, u64 size, HcclMrInfo &mrInfo, MrHandle &mrHandle, void *&devVirAddr)
203 : {
204 0 : MrInfoT info = {};
205 0 : info.addr = mrInfo.addr;
206 0 : info.size = mrInfo.size;
207 0 : info.access = mrInfo.access;
208 :
209 0 : if (IsHostMem_) {
210 0 : unique_lock<std::mutex> lockMapping(mappedHostToDevMutex_);
211 0 : CHK_RET(MapMem(addr, size, devVirAddr));
212 0 : lockMapping.unlock();
213 0 : info.addr = devVirAddr;
214 0 : }
215 :
216 0 : if (isUseQPHandle_) {
217 0 : CHK_RET(HrtRaMrReg(qpHandle_, &info));
218 : } else {
219 0 : CHK_RET(hrtRaRegGlobalMr(rdmaHandle_, info, mrHandle));
220 : }
221 :
222 0 : mrInfo.addr = addr;
223 0 : mrInfo.lkey = info.lkey;
224 0 : return HCCL_SUCCESS;
225 : }
226 :
227 0 : HcclResult MrManager::MapMem(void *addr, u64 size, void *&devVirAddr)
228 : {
229 0 : CHK_PTR_NULL(addr);
230 0 : if (IsRequireMapping(addr, size, devVirAddr)) {
231 : DevType devType;
232 0 : CHK_RET(hrtHalGetDeviceType(curDevId_, devType));
233 0 : if ((devType == DevType::DEV_TYPE_910B) || (devType == DevType::DEV_TYPE_910_93)) {
234 : // 910B环境传参要特殊处理
235 0 : HCCL_INFO("[MrManager][MapMem]hrtHalHostRegister addr[%p], size[%llu Byte], flag[%u], devId[%u]",
236 : addr, size, HOST_MEM_MAP_DEV_PCIE_TH, curDevId_);
237 0 : CHK_RET(hrtHalHostRegister(addr, size, HOST_MEM_MAP_DEV_PCIE_TH, curDevId_, devVirAddr));
238 0 : } else {
239 0 : CHK_RET(hrtHalHostRegister(addr, size, HOST_MEM_MAP_DEV, curDevId_, devVirAddr));
240 : }
241 0 : HostMappingKey hostMappingKey(reinterpret_cast<u64>(addr), size, curDevId_);
242 0 : mappedHostToDevMap_[hostMappingKey].devVirAddr = devVirAddr;
243 : }
244 0 : return HCCL_SUCCESS;
245 : }
246 :
247 0 : HcclResult MrManager::DeRegMrImpl(MrInfo mrInfo)
248 : {
249 : HcclMrInfo mrInfoTmp;
250 0 : if (isUseQPHandle_) {
251 : // 注销MR
252 0 : TransMrInfo((IsHostMem_) ? mrInfo.devVirAddr : mrInfo.addr, mrInfo.size, mrInfoTmp);
253 0 : MrInfoT hccpMrInfoTmp = {};
254 0 : hccpMrInfoTmp.addr = mrInfoTmp.addr;
255 0 : hccpMrInfoTmp.size = mrInfoTmp.size;
256 0 : hccpMrInfoTmp.access = mrInfoTmp.access;
257 0 : hccpMrInfoTmp.lkey = mrInfoTmp.lkey;
258 0 : CHK_RET(HrtRaMrDereg(qpHandle_, &hccpMrInfoTmp));
259 : } else {
260 0 : CHK_RET(hrtRaDeRegGlobalMr(rdmaHandle_, mrInfo.mrHandle));
261 : }
262 0 : if (IsHostMem_) {
263 0 : CHK_RET(UnmapMem(mrInfo));
264 : }
265 0 : return HCCL_SUCCESS;
266 : }
267 :
268 0 : HcclResult MrManager::DelayedReg(void *addr, u64 size)
269 : {
270 0 : CHK_PTR_NULL(addr);
271 0 : unique_lock<std::mutex> lockUnMrMap(unMrMapSpinMutex_);
272 0 : MrMapKey key(reinterpret_cast<u64>(addr), size);
273 0 : MrInfo info(addr, size);
274 0 : auto iter = unRegMrMap_.find(key);
275 0 : if (iter == unRegMrMap_.end()) {
276 0 : info.gloMemRef++;
277 0 : unRegMrMap_.emplace(key, info);
278 : } else {
279 0 : iter->second.gloMemRef++;
280 : }
281 :
282 0 : unique_lock<std::mutex> lock(addrSizeMutex_);
283 0 : globalAddrSizeMap_[addr] = size;
284 0 : lock.unlock();
285 :
286 0 : HCCL_INFO("[MrManager][RecordMr]record mr info success, size[%llu Byte], unRegMrMap size[%u].",
287 : size, unRegMrMap_.size());
288 0 : return HCCL_SUCCESS;
289 0 : }
290 :
291 0 : HcclResult MrManager::RegGlobalMr(void *addr, u64 size)
292 : {
293 0 : CHK_PTR_NULL(addr);
294 :
295 : // count = 0时表示没有初始化通信域,只需将内存信息记录到未注册内存unRegMrMap_中，无需注册MR等动作
296 0 : if (count_ == 0) {
297 0 : CHK_RET(DelayedReg(addr, size));
298 : } else {
299 0 : CHK_RET(RegMr(addr, size));
300 : }
301 :
302 0 : return HCCL_SUCCESS;
303 : }
304 :
305 0 : HcclResult MrManager::RegMr(void *addr, u64 size)
306 : {
307 0 : CHK_PTR_NULL(addr);
308 0 : CHK_PRT_RET((size == 0), HCCL_ERROR("[MrManager][RegTmpMr]memory size[%llu Byte] should be greater than 0.", size),
309 : HCCL_E_PARA);
310 : HcclMrInfo mrInfo;
311 0 : mrInfo.addr = addr;
312 0 : mrInfo.size = size;
313 0 : mrInfo.access = RA_ACCESS_REMOTE_WRITE | RA_ACCESS_LOCAL_WRITE | RA_ACCESS_REMOTE_READ;
314 :
315 0 : unique_lock<std::mutex> lockMrMap(mrMapSpinMutex_);
316 0 : MrMapKey mrMapKey(reinterpret_cast<u64>(addr), size);
317 0 : auto iter = regedMrMap_.find(mrMapKey);
318 : // 防止重复注册
319 0 : if (iter != regedMrMap_.end()) {
320 0 : HCCL_WARNING("[MrManager][RegMr]mr map addr is already exists, size[%llu Byte].", iter->second.size);
321 0 : iter->second.gloMemRef++;
322 0 : unique_lock<std::mutex> lock(addrSizeMutex_);
323 0 : globalAddrSizeMap_[addr] = size;
324 0 : lock.unlock();
325 0 : return HCCL_SUCCESS;
326 0 : }
327 :
328 0 : lockMrMap.unlock();
329 0 : MrHandle mrHandle = nullptr;
330 0 : void *devVirAddr = nullptr;
331 0 : CHK_RET(RegMrImpl(addr, size, mrInfo, mrHandle, devVirAddr));
332 0 : if (!isUseQPHandle_ && mrHandle == nullptr) {
333 0 : HCCL_WARNING("[MrManager][RegMr]global mr register not success, addr[%p], size[%u Byte]", addr, size);
334 0 : return HCCL_SUCCESS;
335 : }
336 :
337 0 : MrInfo tmpMrInfo{};
338 0 : tmpMrInfo = mrInfo;
339 0 : if (!isUseQPHandle_) {
340 0 : tmpMrInfo.mrHandle = mrHandle;
341 : }
342 :
343 0 : tmpMrInfo.gloMemRef++;
344 0 : tmpMrInfo.devVirAddr = devVirAddr;
345 :
346 0 : lockMrMap.lock();
347 0 : regedMrMap_.emplace(mrMapKey, tmpMrInfo);
348 0 : lockMrMap.unlock();
349 :
350 0 : unique_lock<std::mutex> lock(addrSizeMutex_);
351 0 : globalAddrSizeMap_[addr] = size;
352 0 : lock.unlock();
353 :
354 0 : HCCL_INFO("[MrManager][RegGlobalMr]global mr register success, size[%llu Byte], regMrMap size[%u].", size,
355 : regedMrMap_.size());
356 0 : return HCCL_SUCCESS;
357 0 : }
358 :
359 0 : HcclResult MrManager::RegTmpMr(void *addr, u64 size, u32 &lkey) // 注册临时MR
360 : {
361 0 : CHK_PTR_NULL(addr);
362 0 : CHK_PRT_RET((size == 0), HCCL_ERROR("[MrManager][RegTmpMr]memory size[%llu Byte] should be greater than 0.",
363 : size), HCCL_E_PARA);
364 :
365 : HcclMrInfo mrInfo;
366 0 : mrInfo.addr = addr;
367 0 : mrInfo.size = size;
368 0 : mrInfo.access = RA_ACCESS_REMOTE_WRITE | RA_ACCESS_LOCAL_WRITE | RA_ACCESS_REMOTE_READ;
369 0 : u64 uAddr = reinterpret_cast<u64>(addr);
370 0 : MrMapKey tmpMrMapKey(uAddr, size);
371 :
372 0 : unique_lock<std::mutex> lockMrMap(mrMapSpinMutex_);
373 0 : auto iter = regedMrMap_.find(tmpMrMapKey);
374 0 : if (iter != regedMrMap_.end()) {
375 0 : iter->second.tmpMemRef++;
376 0 : lkey = iter->second.lkey;
377 0 : HCCL_INFO("[MrManager][RegTmpMr]temp mr find success, size[%llu Byte], temp mr map size[%u], "
378 : "glo count[%d].", size, regedMrMap_.size(), iter->second.gloMemRef);
379 0 : return HCCL_SUCCESS;
380 : }
381 :
382 0 : lockMrMap.unlock();
383 0 : MrHandle mrHandle = nullptr;
384 0 : void *devVirAddr = nullptr;
385 0 : CHK_RET(RegMrImpl(addr, size, mrInfo, mrHandle, devVirAddr));
386 0 : if (!isUseQPHandle_ && mrHandle == nullptr) {
387 0 : HCCL_ERROR("[MrManager][RegTmpMr]temp mr register failed, size[%u Byte]", size);
388 0 : return HCCL_E_NETWORK;
389 : }
390 :
391 0 : MrInfo tmpMrInfo{};
392 0 : tmpMrInfo = mrInfo;
393 0 : if (!isUseQPHandle_) {
394 0 : tmpMrInfo.mrHandle = mrHandle;
395 : }
396 0 : tmpMrInfo.devVirAddr = devVirAddr;
397 : // 目前这个全局地址只有hdc模式下用，而hdc模式可能以qpHandle与rdmaHandle两种粒度去注册MR
398 0 : g_devAddr = (u64)devVirAddr;
399 0 : tmpMrInfo.tmpMemRef++;
400 :
401 0 : lockMrMap.lock();
402 0 : regedMrMap_.emplace(tmpMrMapKey, tmpMrInfo);
403 0 : lockMrMap.unlock();
404 :
405 0 : lkey = mrInfo.lkey;
406 0 : HCCL_INFO("[MrManager][RegTmpMr]temp mr register success, size[%llu Byte], temp mr map size[%u]",
407 : size, regedMrMap_.size());
408 0 : return HCCL_SUCCESS;
409 0 : }
410 :
411 0 : HcclResult MrManager::DeRegGlobalMr(void *addr)
412 : {
413 0 : CHK_PTR_NULL(addr);
414 0 : HCCL_INFO("[MrManager][DeRegGlobalMr] addr[%p]", hash<void *>{}(addr));
415 0 : unique_lock<std::mutex> lock(addrSizeMutex_);
416 0 : if (globalAddrSizeMap_.find(addr) == globalAddrSizeMap_.end()) {
417 0 : HCCL_ERROR("[MrManager][DeRegGlobalMr] is not found");
418 0 : return HCCL_E_PARA;
419 : }
420 :
421 0 : MrMapKey key(reinterpret_cast<u64>(addr), globalAddrSizeMap_[addr]);
422 0 : lock.unlock();
423 :
424 0 : unique_lock<std::mutex> lockUnMrMap(unMrMapSpinMutex_);
425 0 : auto aiter = unRegMrMap_.find(key);
426 0 : if (aiter != unRegMrMap_.end()) {
427 0 : aiter->second.gloMemRef--;
428 0 : if (aiter->second.gloMemRef == 0) {
429 0 : unRegMrMap_.erase(key);
430 : }
431 :
432 0 : HCCL_INFO("[MrManager][DeRecordMr]derecord global mr info success, unRegMrMap size[%u]", unRegMrMap_.size());
433 0 : return HCCL_SUCCESS;
434 : }
435 :
436 0 : lockUnMrMap.unlock();
437 0 : unique_lock<std::mutex> lockMrMap(mrMapSpinMutex_);
438 0 : auto iter = regedMrMap_.find(key);
439 0 : if (iter != regedMrMap_.end()) {
440 0 : iter->second.gloMemRef--;
441 0 : if (iter->second.gloMemRef > 0 || iter->second.tmpMemRef > 0) {
442 0 : HCCL_INFO("[MrManager][DeRegGlobalMr] minus count[%d] tmp count[%d] success, regMrMap size[%u].",
443 : iter->second.gloMemRef, iter->second.tmpMemRef, regedMrMap_.size());
444 0 : return HCCL_SUCCESS;
445 : }
446 :
447 0 : if (iter->second.size > 0) {
448 0 : CHK_RET(DeRegMrImpl(iter->second));
449 : }
450 :
451 0 : regedMrMap_.erase(key);
452 0 : lockMrMap.unlock();
453 0 : HCCL_INFO("[MrManager][DeRegGlobalMr]addr deregister success, regMrMap size[%u].",
454 : regedMrMap_.size());
455 : } else {
456 0 : HCCL_ERROR("[MrManager][DeRegGlobalMr]addr was not found, unRegMrMap size[%u], regMrMap size[%u].",
457 : unRegMrMap_.size(), regedMrMap_.size());
458 0 : return HCCL_E_MEMORY;
459 : }
460 0 : HCCL_INFO("[MrManager][DeRegGlobalMr] DeReg GlobalMr end");
461 0 : return HCCL_SUCCESS;
462 0 : }
463 :
464 0 : HcclResult MrManager::UnmapMem(MrInfo mrInfo)
465 : {
466 0 : unique_lock<std::mutex> lockMapping(mappedHostToDevMutex_);
467 0 : u64 userAddr = reinterpret_cast<u64>(mrInfo.addr);
468 0 : auto iter = SearchMappingMap(userAddr, mrInfo.size);
469 0 : CHK_PRT_RET((iter == mappedHostToDevMap_.end()),
470 : HCCL_ERROR("[MrManager][UnmapMem]the memory dereged isn't been reged"), HCCL_E_PARA);
471 0 : if (iter->second.mappingRef == 0) {
472 : // 解除内存映射
473 0 : CHK_RET(hrtHalHostUnregister(mrInfo.addr, curDevId_));
474 0 : mappedHostToDevMap_.erase(iter->first);
475 : } else {
476 0 : iter->second.mappingRef--;
477 : }
478 0 : return HCCL_SUCCESS;
479 0 : }
480 :
481 0 : HcclResult MrManager::GetKey(void *addr, u64 size, u32 &lkey) // 获取内存的lkey
482 : {
483 0 : CHK_PTR_NULL(addr);
484 0 : CHK_PRT_RET((size == 0), HCCL_ERROR("[MrManager][GetKey]memory size[%llu Byte] should be greater than 0.",
485 : size), HCCL_E_PARA);
486 :
487 0 : MrInfo mrInfo(addr, size);
488 0 : unique_lock<std::mutex> lockMrMap(mrMapSpinMutex_);
489 0 : bool isEmpty = regedMrMap_.empty();
490 0 : lockMrMap.unlock();
491 0 : if (isEmpty) {
492 0 : CHK_PRT_RET((RegTmpMr(addr, size, lkey) != HCCL_SUCCESS),
493 : HCCL_ERROR("[MrManager][GetKey]register temp memory error, size[%llu Byte].", size),
494 : HCCL_E_INTERNAL);
495 : } else {
496 0 : bool isInfoNotFound = false;
497 0 : CHK_PRT_RET((GetMrInfo(mrInfo, isInfoNotFound) != HCCL_SUCCESS),
498 : HCCL_ERROR("[MrManager][GetKey]get memory info error, size[%llu Byte].", size),
499 : HCCL_E_INTERNAL);
500 0 : if (isInfoNotFound) {
501 0 : CHK_PRT_RET((RegTmpMr(addr, size, lkey) != HCCL_SUCCESS),
502 : HCCL_ERROR("[MrManager][GetKey]register temp memory error, size[%llu Byte].", size),
503 : HCCL_E_INTERNAL);
504 : } else {
505 0 : lockMrMap.lock();
506 0 : MrMapKey key(reinterpret_cast<u64>(mrInfo.addr), mrInfo.size);
507 0 : auto iter = regedMrMap_.find(key);
508 0 : iter->second.tmpMemRef++;
509 0 : lkey = mrInfo.lkey;
510 0 : HCCL_INFO("[MrManager][GetKey]get memory lkey success, size[%llu Byte], regMrMap size[%u], "
511 : "temp mr map size[%u].", size, regedMrMap_.size(), regedMrMap_.size());
512 : }
513 : }
514 0 : return HCCL_SUCCESS;
515 0 : }
516 :
517 0 : HcclResult MrManager::ReleaseKey(void *addr, u64 size) // 释放临时MR
518 : {
519 0 : CHK_PTR_NULL(addr);
520 0 : CHK_PRT_RET((size == 0), HCCL_ERROR("[MrManager][ReleaseKey]memory size[%llu Byte] should be greater than 0.",
521 : size), HCCL_E_PARA);
522 :
523 : HcclResult ret;
524 0 : MrInfo mrInfo;
525 0 : mrInfo.addr = addr;
526 0 : mrInfo.size = size;
527 0 : bool isInfoNotFound = false;
528 0 : ret = GetMrInfo(mrInfo, isInfoNotFound);
529 0 : if (ret || isInfoNotFound) {
530 0 : HCCL_ERROR("[MrManager][ReleaseKey]get memory info error, size[%llu Byte].", size);
531 0 : return HCCL_E_INTERNAL;
532 : }
533 :
534 0 : MrMapKey tmpMrMapKey;
535 0 : tmpMrMapKey.addr = reinterpret_cast<u64>(mrInfo.addr);
536 0 : tmpMrMapKey.size = mrInfo.size;
537 :
538 0 : unique_lock<std::mutex> lockMrMap(mrMapSpinMutex_);
539 0 : auto iter = regedMrMap_.find(tmpMrMapKey);
540 0 : CHK_PRT_RET((iter == regedMrMap_.end()),
541 : HCCL_ERROR("[MrManager][ReleaseKey] release key failed, size[%llu Byte]"
542 : "size[%llu], regMrMap size[%u].", size, mrInfo.size, regedMrMap_.size()), HCCL_E_INTERNAL);
543 :
544 0 : --iter->second.tmpMemRef;
545 0 : if (iter->second.tmpMemRef > 0 || iter->second.gloMemRef > 0) {
546 0 : HCCL_INFO("[MrManager][ReleaseKey]release key success, size[%llu Byte], tmpMrMap size[%u], count[%d] "
547 : "tmp count[%d].", size, regedMrMap_.size(), iter->second.gloMemRef, iter->second.tmpMemRef);
548 0 : return HCCL_SUCCESS;
549 0 : } else if (iter->second.tmpMemRef < 0) {
550 0 : HCCL_ERROR("[MrManager][ReleaseKey]release key error, size[%llu Byte], count[%d].",
551 : size, iter->second.tmpMemRef);
552 0 : return HCCL_E_MEMORY;
553 : }
554 :
555 0 : CHK_RET(DeRegMrImpl(iter->second));
556 0 : HCCL_INFO("[MrManager][ReleaseKey] deregister success, size[%llu Byte], "
557 : "temp mr map size[%u].", size, regedMrMap_.size());
558 0 : regedMrMap_.erase(iter);
559 0 : lockMrMap.unlock();
560 0 : return HCCL_SUCCESS;
561 0 : }
562 :
563 0 : HcclResult MrManager::GetMrInfo(MrInfo &mrInfo, bool &isInfoNotFound)
564 : {
565 0 : CHK_PRT_RET(regedMrMap_.empty(), HCCL_ERROR("[MrManager][GetMrInfo]get mr info failed, mr map is empty"),
566 : HCCL_E_PARA);
567 :
568 0 : isInfoNotFound = false;
569 0 : unique_lock<std::mutex> lockMrMap(mrMapSpinMutex_);
570 :
571 0 : u64 uAddr = reinterpret_cast<u64>(mrInfo.addr);
572 0 : u64 size = mrInfo.size;
573 0 : MrMapKey key(uAddr, size);
574 0 : auto iter = regedMrMap_.find(key);
575 0 : if (iter != regedMrMap_.end()) {
576 0 : if (iter->second.size >= size) {
577 0 : mrInfo = iter->second;
578 0 : HCCL_DEBUG("[MrManager][GetMrInfo]get memory info success, size[%llu].", iter->second.size);
579 : } else {
580 0 : isInfoNotFound = true;
581 0 : HCCL_WARNING("[MrManager][GetMrInfo]mr addr size[%llu], but required addr size[%llu].",
582 : iter->second.size, mrInfo.size);
583 : }
584 :
585 0 : return HCCL_SUCCESS;
586 : }
587 :
588 0 : iter = regedMrMap_.upper_bound(key);
589 0 : if (iter != regedMrMap_.begin() &&
590 0 : !(iter != regedMrMap_.end() && iter->first.addr == uAddr && iter->first.size >= size)) {
591 0 : iter--;
592 : }
593 :
594 0 : u64 uTmpAddr = iter->first.addr;
595 0 : u64 tmpSize = iter->second.size;
596 0 : if (((uTmpAddr <= uAddr) && (uAddr < (uTmpAddr + tmpSize))) &&
597 0 : ((uTmpAddr < (uAddr + size)) && ((uAddr + size) <= (uTmpAddr + tmpSize)))) {
598 0 : mrInfo = iter->second;
599 : } else {
600 0 : HCCL_WARNING("[MrManager][GetMrInfo] size[%llu] was not found.", mrInfo.size);
601 0 : isInfoNotFound = true;
602 0 : return HCCL_SUCCESS;
603 : }
604 0 : HCCL_DEBUG("[MrManager][GetMrInfo]get memory info success, size[%llu]", mrInfo.size);
605 0 : return HCCL_SUCCESS;
606 0 : }
607 :
608 0 : HcclResult MrManager::GetDevVirAddr(void *addr, u64 size, u64 &devVirAddr)
609 : {
610 0 : CHK_PTR_NULL(addr);
611 0 : CHK_PRT_RET((size == 0), HCCL_ERROR("[MrManager][GetDevVirAddr]memory size[%llu Byte] should be greater than 0.",
612 : size), HCCL_E_PARA);
613 0 : MrInfo mrInfo(addr, size);
614 0 : bool isInfoNotFound = false;
615 0 : CHK_PRT_RET((GetMrInfo(mrInfo, isInfoNotFound) != HCCL_SUCCESS),
616 : HCCL_ERROR("[MrManager][GetDevVirAddr]get memory info error, size[%llu Byte].", size),
617 : HCCL_E_INTERNAL);
618 0 : CHK_PRT_RET(isInfoNotFound, HCCL_ERROR("[MrManager][GetDevVirAddr]get memory info fail, addr[%p], size[%llu Byte].",
619 : addr, size), HCCL_E_PARA);
620 0 : devVirAddr = reinterpret_cast<u64>(mrInfo.devVirAddr) + reinterpret_cast<u64>(addr) -
621 0 : reinterpret_cast<u64>(mrInfo.addr);
622 :
623 0 : return HCCL_SUCCESS;
624 : }
625 :
626 0 : void MrManager::TransMrInfo(void* addr, u64 size, HcclMrInfo& mrInfo)
627 : {
628 0 : mrInfo.addr = addr;
629 0 : mrInfo.size = size;
630 0 : mrInfo.access = RA_ACCESS_REMOTE_WRITE | RA_ACCESS_LOCAL_WRITE | RA_ACCESS_REMOTE_READ;
631 0 : }
632 :
633 0 : HcclResult MrManager::ReleaseMrResource()
634 : {
635 0 : HCCL_INFO("[MrManager][ReleaseMrResource]start release mr resource");
636 0 : unique_lock<std::mutex> lockMrMap(mrMapSpinMutex_);
637 0 : if (!regedMrMap_.empty()) {
638 0 : unique_lock<std::mutex> lockUnMrMap(unMrMapSpinMutex_);
639 0 : unRegMrMap_ = regedMrMap_;
640 0 : lockUnMrMap.unlock();
641 0 : u64 bound = regedMrMap_.begin()->first.addr;
642 0 : for (auto &iter : regedMrMap_) {
643 0 : if (iter.first.addr >= bound && iter.second.size != 0) {
644 0 : HCCL_DEBUG("deinit addr[%llu], size[%llu]", hash<void *>{}(iter.second.addr), iter.second.size);
645 0 : CHK_RET(DeRegMrImpl(iter.second));
646 0 : bound = iter.first.addr + iter.first.size;
647 : }
648 : }
649 :
650 0 : regedMrMap_.clear();
651 0 : }
652 :
653 0 : HCCL_INFO("[MrManager][ReleaseMrResource]release memory resource success.");
654 0 : return HCCL_SUCCESS;
655 0 : }
656 :
657 0 : void MrManager::SetHdcPara(u32 devId, bool isHostMem, bool isUseQPHandle)
658 : {
659 0 : isUseQPHandle_ = isUseQPHandle;
660 0 : curDevId_ = devId;
661 0 : IsHostMem_ = isHostMem;
662 0 : }
663 :
664 : }
|
