Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "mr_manager.h"
12 : #include "adapter_hal.h"
13 : #include "adapter_hccp.h"
14 : #include "network_manager_pub.h"
15 : #include "../resource/socket/hccl_network.h"
16 :
17 : namespace hccl {
18 : using namespace std;
19 : u64 MrManager::g_devAddr = 0;
20 : map<HostMappingKey, HostMappingInfo> MrManager::mappedHostToDevMap_ = {};
21 : std::mutex MrManager::mappedHostToDevMutex_;
22 :
23 0 : MrManager& MrManager::GetInstance()
24 : {
25 0 : static MrManager hcclMrManager;
26 0 : return hcclMrManager;
27 : }
28 :
29 0 : MrManager::MrManager() : rdmaHandle_(nullptr), count_(0) {}
30 :
31 0 : MrManager::MrManager(HcclNetDevCtx netDevCtx) : rdmaHandle_(nullptr), count_(0), netDevCtx_(netDevCtx) {}
32 :
33 0 : MrManager::~MrManager() {}
34 :
35 0 : HcclResult MrManager::Init(QpHandle qpHandle, u32 devId, bool isHostMem, map<MrMapKey, MrInfo>& unRegMrMap)
36 : {
37 0 : CHK_PTR_NULL(qpHandle);
38 0 : unique_lock<std::mutex> lockUnMrMap(unMrMapSpinMutex_);
39 0 : unRegMrMap_ = unRegMrMap;
40 0 : lockUnMrMap.unlock();
41 0 : SetHdcPara(devId, isHostMem, true);
42 0 : CHK_RET(InitMrManager(qpHandle));
43 0 : return HCCL_SUCCESS;
44 0 : }
45 :
46 0 : HcclResult MrManager::Init(RdmaHandle rdmaHandle, u32 devId, bool isHostMem)
47 : {
48 0 : CHK_PTR_NULL(rdmaHandle);
49 0 : SetHdcPara(devId, isHostMem, false);
50 0 : CHK_RET(InitMrManager(rdmaHandle));
51 0 : return HCCL_SUCCESS;
52 : }
53 :
54 0 : HcclResult MrManager::Init(RdmaHandle rdmaHandle)
55 : {
56 0 : CHK_PTR_NULL(rdmaHandle);
57 0 : return InitMrManager(rdmaHandle);
58 : }
59 :
60 0 : HcclResult MrManager::Init()
61 : {
62 0 : CHK_PTR_NULL(netDevCtx_);
63 0 : RaResourceInfo raResourceInfo;
64 0 : s32 deviceLogicId = (static_cast<hccl::NetDevContext*>(netDevCtx_))->GetLogicId();
65 0 : HcclIpAddress localIp = (static_cast<hccl::NetDevContext*>(netDevCtx_))->GetLocalIp();
66 0 : CHK_RET(NetworkManager::GetInstance(deviceLogicId).GetRaResourceInfo(raResourceInfo));
67 0 : void* nicRdmaHandle = raResourceInfo.nicSocketMap[localIp].nicRdmaHandle;
68 0 : return InitMrManager(nicRdmaHandle);
69 0 : }
70 :
71 0 : HcclResult MrManager::InitUnRegMrMap()
72 : {
73 0 : unique_lock<std::mutex> lockUnMrMap(unMrMapSpinMutex_);
74 0 : for (auto& iter : unRegMrMap_) {
75 : // 目前全局内存由于地址非法注册失败返回成功，需要driver修复进程退出不通知通信库解注册内存问题
76 0 : CHK_RET(RegMr(iter.second.addr, iter.second.size));
77 : // 内存注册失败，mrHandl为空，不用记录
78 0 : MrMapKey mrMapKey(reinterpret_cast<u64>(iter.second.addr), iter.second.size);
79 0 : unique_lock<std::mutex> lockMrMap(mrMapSpinMutex_);
80 0 : if (regedMrMap_.find(mrMapKey) != regedMrMap_.end()) {
81 0 : auto iterator = regedMrMap_.find(iter.first);
82 0 : if (iterator != regedMrMap_.end()) {
83 0 : iterator->second.gloMemRef = iter.second.gloMemRef;
84 : }
85 : }
86 0 : lockMrMap.unlock();
87 0 : }
88 :
89 0 : unRegMrMap_.clear();
90 0 : lockUnMrMap.unlock();
91 0 : return HCCL_SUCCESS;
92 0 : }
93 :
94 0 : HcclResult MrManager::InitUnRegMrMap(map<MrMapKey, MrInfo>& unRegMrMap)
95 : {
96 0 : unique_lock<std::mutex> lockUnMrMap(unMrMapSpinMutex_);
97 0 : unRegMrMap_ = unRegMrMap;
98 0 : lockUnMrMap.unlock();
99 0 : CHK_RET(InitUnRegMrMap());
100 0 : return HCCL_SUCCESS;
101 0 : }
102 :
103 0 : HcclResult MrManager::InitMrManager(void* handle)
104 : {
105 0 : CHK_PTR_NULL(handle);
106 0 : if (++count_ == COUNT_ONE) {
107 0 : if (isUseQPHandle_) {
108 0 : qpHandle_ = handle;
109 : } else {
110 0 : rdmaHandle_ = handle;
111 : }
112 0 : CHK_RET(InitUnRegMrMap());
113 0 : } else if (count_ > COUNT_ONE) {
114 0 : if (rdmaHandle_ != handle && qpHandle_ != handle) {
115 0 : HCCL_ERROR("[MrManager][Init]mr manager init failed, count[%d].", count_.load());
116 0 : return HCCL_E_PARA;
117 : }
118 : }
119 0 : HCCL_INFO("[MrManager][Init]mr manager init success, count[%d]", count_.load());
120 0 : return HCCL_SUCCESS;
121 : }
122 :
123 0 : HcclResult MrManager::DeInit()
124 : {
125 0 : RaResourceInfo raResourceInfo;
126 0 : s32 deviceLogicId = (static_cast<hccl::NetDevContext*>(netDevCtx_))->GetLogicId();
127 0 : HcclIpAddress localIp = (static_cast<hccl::NetDevContext*>(netDevCtx_))->GetLocalIp();
128 0 : CHK_RET(NetworkManager::GetInstance(deviceLogicId).GetRaResourceInfo(raResourceInfo));
129 0 : void* nicRdmaHandle = raResourceInfo.nicSocketMap[localIp].nicRdmaHandle;
130 0 : return DeInit(nicRdmaHandle);
131 0 : }
132 :
133 0 : HcclResult MrManager::DeInit(const void* handle)
134 : {
135 0 : CHK_PTR_NULL(handle);
136 0 : if (rdmaHandle_ == handle || qpHandle_ == handle) {
137 0 : --count_;
138 0 : if (count_ > 0) {
139 0 : HCCL_INFO("[MrManager][DeInit]mr manager deinit success, count[%d].", count_.load());
140 0 : return HCCL_SUCCESS;
141 0 : } else if (count_ == 0) {
142 0 : ReleaseMrResource();
143 0 : if (isUseQPHandle_) {
144 0 : qpHandle_ = nullptr;
145 : } else {
146 0 : rdmaHandle_ = nullptr;
147 : }
148 : }
149 : } else {
150 0 : HCCL_ERROR("[MrManager][DeInit]count[%d]", count_.load());
151 0 : return HCCL_E_PARA;
152 : }
153 0 : HCCL_INFO("[MrManager][DeInit]mr manager deinit success, count[%d].", count_.load());
154 0 : return HCCL_SUCCESS;
155 : }
156 :
157 0 : bool MrManager::IsRequireMapping(void* addr, u64 size, void*& devVirAddr)
158 : {
159 0 : u64 userAddr = reinterpret_cast<u64>(addr);
160 0 : u64 userSize = size;
161 0 : if (mappedHostToDevMap_.size() == 0) {
162 0 : return true;
163 : }
164 :
165 0 : auto iter = SearchMappingMap(userAddr, userSize);
166 0 : if (iter != mappedHostToDevMap_.end()) {
167 0 : u64 tmpDva = reinterpret_cast<u64>(iter->second.devVirAddr) + userAddr - iter->first.addr;
168 0 : devVirAddr = reinterpret_cast<void*>(static_cast<uintptr_t>(tmpDva));
169 0 : iter->second.mappingRef++;
170 0 : return false;
171 : }
172 :
173 0 : return true;
174 : }
175 :
176 0 : map<MrMapKey, MrInfo> MrManager::GetUnregMap()
177 : {
178 0 : unique_lock<std::mutex> lockUnMrMap(unMrMapSpinMutex_);
179 0 : return unRegMrMap_;
180 0 : }
181 :
182 0 : std::map<HostMappingKey, HostMappingInfo>::iterator MrManager::SearchMappingMap(u64 userAddr, u64 userSize)
183 : {
184 0 : for (auto iter = mappedHostToDevMap_.begin(); iter != mappedHostToDevMap_.end(); ++iter) {
185 0 : if ((userAddr >= iter->first.addr) && (userAddr + userSize <= iter->first.size + iter->first.addr)
186 0 : && (iter->first.devId == curDevId_)) {
187 0 : return iter;
188 : }
189 : }
190 0 : return mappedHostToDevMap_.end();
191 : }
192 :
193 0 : HcclResult MrManager::RegMrImpl(void* addr, u64 size, HcclMrInfo& mrInfo, MrHandle& mrHandle, void*& devVirAddr)
194 : {
195 0 : MrInfoT info = {};
196 0 : info.addr = mrInfo.addr;
197 0 : info.size = mrInfo.size;
198 0 : info.access = mrInfo.access;
199 :
200 0 : if (IsHostMem_) {
201 0 : unique_lock<std::mutex> lockMapping(mappedHostToDevMutex_);
202 0 : CHK_RET(MapMem(addr, size, devVirAddr));
203 0 : lockMapping.unlock();
204 0 : info.addr = devVirAddr;
205 0 : }
206 :
207 0 : if (isUseQPHandle_) {
208 0 : CHK_RET(HrtRaMrReg(qpHandle_, &info));
209 : } else {
210 0 : CHK_RET(hrtRaRegGlobalMr(rdmaHandle_, info, mrHandle));
211 : }
212 :
213 0 : mrInfo.addr = addr;
214 0 : mrInfo.lkey = info.lkey;
215 0 : return HCCL_SUCCESS;
216 : }
217 :
218 0 : HcclResult MrManager::MapMem(void* addr, u64 size, void*& devVirAddr)
219 : {
220 0 : CHK_PTR_NULL(addr);
221 0 : if (IsRequireMapping(addr, size, devVirAddr)) {
222 : DevType devType;
223 0 : CHK_RET(hrtHalGetDeviceType(curDevId_, devType));
224 0 : if ((devType == DevType::DEV_TYPE_910B) || (devType == DevType::DEV_TYPE_910_93)) {
225 : // 910B环境传参要特殊处理
226 0 : HCCL_INFO(
227 : "[MrManager][MapMem]hrtHalHostRegister addr[%p], size[%llu Byte], flag[%u], devId[%u]", addr, size,
228 : HOST_MEM_MAP_DEV_PCIE_TH, curDevId_);
229 0 : CHK_RET(hrtHalHostRegister(addr, size, HOST_MEM_MAP_DEV_PCIE_TH, curDevId_, devVirAddr));
230 0 : } else {
231 0 : CHK_RET(hrtHalHostRegister(addr, size, HOST_MEM_MAP_DEV, curDevId_, devVirAddr));
232 : }
233 0 : HostMappingKey hostMappingKey(reinterpret_cast<u64>(addr), size, curDevId_);
234 0 : mappedHostToDevMap_[hostMappingKey].devVirAddr = devVirAddr;
235 : }
236 0 : return HCCL_SUCCESS;
237 : }
238 :
239 0 : HcclResult MrManager::DeRegMrImpl(MrInfo mrInfo)
240 : {
241 : HcclMrInfo mrInfoTmp;
242 0 : if (isUseQPHandle_) {
243 : // 注销MR
244 0 : TransMrInfo((IsHostMem_) ? mrInfo.devVirAddr : mrInfo.addr, mrInfo.size, mrInfoTmp);
245 0 : MrInfoT hccpMrInfoTmp = {};
246 0 : hccpMrInfoTmp.addr = mrInfoTmp.addr;
247 0 : hccpMrInfoTmp.size = mrInfoTmp.size;
248 0 : hccpMrInfoTmp.access = mrInfoTmp.access;
249 0 : hccpMrInfoTmp.lkey = mrInfoTmp.lkey;
250 0 : CHK_RET(HrtRaMrDereg(qpHandle_, &hccpMrInfoTmp));
251 : } else {
252 0 : CHK_RET(hrtRaDeRegGlobalMr(rdmaHandle_, mrInfo.mrHandle));
253 : }
254 0 : if (IsHostMem_) {
255 0 : CHK_RET(UnmapMem(mrInfo));
256 : }
257 0 : return HCCL_SUCCESS;
258 : }
259 :
260 0 : HcclResult MrManager::DelayedReg(void* addr, u64 size)
261 : {
262 0 : CHK_PTR_NULL(addr);
263 0 : unique_lock<std::mutex> lockUnMrMap(unMrMapSpinMutex_);
264 0 : MrMapKey key(reinterpret_cast<u64>(addr), size);
265 0 : MrInfo info(addr, size);
266 0 : auto iter = unRegMrMap_.find(key);
267 0 : if (iter == unRegMrMap_.end()) {
268 0 : info.gloMemRef++;
269 0 : unRegMrMap_.emplace(key, info);
270 : } else {
271 0 : iter->second.gloMemRef++;
272 : }
273 :
274 0 : unique_lock<std::mutex> lock(addrSizeMutex_);
275 0 : globalAddrSizeMap_[addr] = size;
276 0 : lock.unlock();
277 :
278 0 : HCCL_INFO(
279 : "[MrManager][RecordMr]record mr info success, size[%llu Byte], unRegMrMap size[%u].", size, unRegMrMap_.size());
280 0 : return HCCL_SUCCESS;
281 0 : }
282 :
283 0 : HcclResult MrManager::RegGlobalMr(void* addr, u64 size)
284 : {
285 0 : CHK_PTR_NULL(addr);
286 :
287 : // count = 0时表示没有初始化通信域,只需将内存信息记录到未注册内存unRegMrMap_中，无需注册MR等动作
288 0 : if (count_ == 0) {
289 0 : CHK_RET(DelayedReg(addr, size));
290 : } else {
291 0 : CHK_RET(RegMr(addr, size));
292 : }
293 :
294 0 : return HCCL_SUCCESS;
295 : }
296 :
297 0 : HcclResult MrManager::RegMr(void* addr, u64 size)
298 : {
299 0 : CHK_PTR_NULL(addr);
300 0 : CHK_PRT_RET(
301 : (size == 0), HCCL_ERROR("[MrManager][RegTmpMr]memory size[%llu Byte] should be greater than 0.", size),
302 : HCCL_E_PARA);
303 : HcclMrInfo mrInfo;
304 0 : mrInfo.addr = addr;
305 0 : mrInfo.size = size;
306 0 : mrInfo.access = RA_ACCESS_REMOTE_WRITE | RA_ACCESS_LOCAL_WRITE | RA_ACCESS_REMOTE_READ;
307 :
308 0 : unique_lock<std::mutex> lockMrMap(mrMapSpinMutex_);
309 0 : MrMapKey mrMapKey(reinterpret_cast<u64>(addr), size);
310 0 : auto iter = regedMrMap_.find(mrMapKey);
311 : // 防止重复注册
312 0 : if (iter != regedMrMap_.end()) {
313 0 : HCCL_WARNING("[MrManager][RegMr]mr map addr is already exists, size[%llu Byte].", iter->second.size);
314 0 : iter->second.gloMemRef++;
315 0 : unique_lock<std::mutex> lock(addrSizeMutex_);
316 0 : globalAddrSizeMap_[addr] = size;
317 0 : lock.unlock();
318 0 : return HCCL_SUCCESS;
319 0 : }
320 :
321 0 : lockMrMap.unlock();
322 0 : MrHandle mrHandle = nullptr;
323 0 : void* devVirAddr = nullptr;
324 0 : CHK_RET(RegMrImpl(addr, size, mrInfo, mrHandle, devVirAddr));
325 0 : if (!isUseQPHandle_ && mrHandle == nullptr) {
326 0 : HCCL_WARNING("[MrManager][RegMr]global mr register not success, addr[%p], size[%u Byte]", addr, size);
327 0 : return HCCL_SUCCESS;
328 : }
329 :
330 0 : MrInfo tmpMrInfo{};
331 0 : tmpMrInfo = mrInfo;
332 0 : if (!isUseQPHandle_) {
333 0 : tmpMrInfo.mrHandle = mrHandle;
334 : }
335 :
336 0 : tmpMrInfo.gloMemRef++;
337 0 : tmpMrInfo.devVirAddr = devVirAddr;
338 :
339 0 : lockMrMap.lock();
340 0 : regedMrMap_.emplace(mrMapKey, tmpMrInfo);
341 0 : lockMrMap.unlock();
342 :
343 0 : unique_lock<std::mutex> lock(addrSizeMutex_);
344 0 : globalAddrSizeMap_[addr] = size;
345 0 : lock.unlock();
346 :
347 0 : HCCL_INFO(
348 : "[MrManager][RegGlobalMr]global mr register success, size[%llu Byte], regMrMap size[%u].", size,
349 : regedMrMap_.size());
350 0 : return HCCL_SUCCESS;
351 0 : }
352 :
353 0 : HcclResult MrManager::RegTmpMr(void* addr, u64 size, u32& lkey) // 注册临时MR
354 : {
355 0 : CHK_PTR_NULL(addr);
356 0 : CHK_PRT_RET(
357 : (size == 0), HCCL_ERROR("[MrManager][RegTmpMr]memory size[%llu Byte] should be greater than 0.", size),
358 : HCCL_E_PARA);
359 :
360 : HcclMrInfo mrInfo;
361 0 : mrInfo.addr = addr;
362 0 : mrInfo.size = size;
363 0 : mrInfo.access = RA_ACCESS_REMOTE_WRITE | RA_ACCESS_LOCAL_WRITE | RA_ACCESS_REMOTE_READ;
364 0 : u64 uAddr = reinterpret_cast<u64>(addr);
365 0 : MrMapKey tmpMrMapKey(uAddr, size);
366 :
367 0 : unique_lock<std::mutex> lockMrMap(mrMapSpinMutex_);
368 0 : auto iter = regedMrMap_.find(tmpMrMapKey);
369 0 : if (iter != regedMrMap_.end()) {
370 0 : iter->second.tmpMemRef++;
371 0 : lkey = iter->second.lkey;
372 0 : HCCL_INFO(
373 : "[MrManager][RegTmpMr]temp mr find success, size[%llu Byte], temp mr map size[%u], "
374 : "glo count[%d].",
375 : size, regedMrMap_.size(), iter->second.gloMemRef);
376 0 : return HCCL_SUCCESS;
377 : }
378 :
379 0 : lockMrMap.unlock();
380 0 : MrHandle mrHandle = nullptr;
381 0 : void* devVirAddr = nullptr;
382 0 : CHK_RET(RegMrImpl(addr, size, mrInfo, mrHandle, devVirAddr));
383 0 : if (!isUseQPHandle_ && mrHandle == nullptr) {
384 0 : HCCL_ERROR("[MrManager][RegTmpMr]temp mr register failed, size[%u Byte]", size);
385 0 : return HCCL_E_NETWORK;
386 : }
387 :
388 0 : MrInfo tmpMrInfo{};
389 0 : tmpMrInfo = mrInfo;
390 0 : if (!isUseQPHandle_) {
391 0 : tmpMrInfo.mrHandle = mrHandle;
392 : }
393 0 : tmpMrInfo.devVirAddr = devVirAddr;
394 : // 目前这个全局地址只有hdc模式下用，而hdc模式可能以qpHandle与rdmaHandle两种粒度去注册MR
395 0 : g_devAddr = (u64)devVirAddr;
396 0 : tmpMrInfo.tmpMemRef++;
397 :
398 0 : lockMrMap.lock();
399 0 : regedMrMap_.emplace(tmpMrMapKey, tmpMrInfo);
400 0 : lockMrMap.unlock();
401 :
402 0 : lkey = mrInfo.lkey;
403 0 : HCCL_INFO(
404 : "[MrManager][RegTmpMr]temp mr register success, size[%llu Byte], temp mr map size[%u]", size,
405 : regedMrMap_.size());
406 0 : return HCCL_SUCCESS;
407 0 : }
408 :
409 0 : HcclResult MrManager::DeRegGlobalMr(void* addr)
410 : {
411 0 : CHK_PTR_NULL(addr);
412 0 : HCCL_INFO("[MrManager][DeRegGlobalMr] addr[%p]", hash<void*>{}(addr));
413 0 : unique_lock<std::mutex> lock(addrSizeMutex_);
414 0 : if (globalAddrSizeMap_.find(addr) == globalAddrSizeMap_.end()) {
415 0 : HCCL_ERROR("[MrManager][DeRegGlobalMr] is not found");
416 0 : return HCCL_E_PARA;
417 : }
418 :
419 0 : MrMapKey key(reinterpret_cast<u64>(addr), globalAddrSizeMap_[addr]);
420 0 : lock.unlock();
421 :
422 0 : unique_lock<std::mutex> lockUnMrMap(unMrMapSpinMutex_);
423 0 : auto aiter = unRegMrMap_.find(key);
424 0 : if (aiter != unRegMrMap_.end()) {
425 0 : aiter->second.gloMemRef--;
426 0 : if (aiter->second.gloMemRef == 0) {
427 0 : unRegMrMap_.erase(key);
428 : }
429 :
430 0 : HCCL_INFO("[MrManager][DeRecordMr]derecord global mr info success, unRegMrMap size[%u]", unRegMrMap_.size());
431 0 : return HCCL_SUCCESS;
432 : }
433 :
434 0 : lockUnMrMap.unlock();
435 0 : unique_lock<std::mutex> lockMrMap(mrMapSpinMutex_);
436 0 : auto iter = regedMrMap_.find(key);
437 0 : if (iter != regedMrMap_.end()) {
438 0 : iter->second.gloMemRef--;
439 0 : if (iter->second.gloMemRef > 0 || iter->second.tmpMemRef > 0) {
440 0 : HCCL_INFO(
441 : "[MrManager][DeRegGlobalMr] minus count[%d] tmp count[%d] success, regMrMap size[%u].",
442 : iter->second.gloMemRef, iter->second.tmpMemRef, regedMrMap_.size());
443 0 : return HCCL_SUCCESS;
444 : }
445 :
446 0 : if (iter->second.size > 0) {
447 0 : CHK_RET(DeRegMrImpl(iter->second));
448 : }
449 :
450 0 : regedMrMap_.erase(key);
451 0 : lockMrMap.unlock();
452 0 : HCCL_INFO("[MrManager][DeRegGlobalMr]addr deregister success, regMrMap size[%u].", regedMrMap_.size());
453 : } else {
454 0 : HCCL_ERROR(
455 : "[MrManager][DeRegGlobalMr]addr was not found, unRegMrMap size[%u], regMrMap size[%u].", unRegMrMap_.size(),
456 : regedMrMap_.size());
457 0 : return HCCL_E_MEMORY;
458 : }
459 0 : HCCL_INFO("[MrManager][DeRegGlobalMr] DeReg GlobalMr end");
460 0 : return HCCL_SUCCESS;
461 0 : }
462 :
463 0 : HcclResult MrManager::UnmapMem(MrInfo mrInfo)
464 : {
465 0 : unique_lock<std::mutex> lockMapping(mappedHostToDevMutex_);
466 0 : u64 userAddr = reinterpret_cast<u64>(mrInfo.addr);
467 0 : auto iter = SearchMappingMap(userAddr, mrInfo.size);
468 0 : CHK_PRT_RET(
469 : (iter == mappedHostToDevMap_.end()), HCCL_ERROR("[MrManager][UnmapMem]the memory dereged isn't been reged"),
470 : HCCL_E_PARA);
471 0 : if (iter->second.mappingRef == 0) {
472 : // 解除内存映射
473 0 : CHK_RET(hrtHalHostUnregister(mrInfo.addr, curDevId_));
474 0 : mappedHostToDevMap_.erase(iter->first);
475 : } else {
476 0 : iter->second.mappingRef--;
477 : }
478 0 : return HCCL_SUCCESS;
479 0 : }
480 :
481 0 : HcclResult MrManager::GetKey(void* addr, u64 size, u32& lkey) // 获取内存的lkey
482 : {
483 0 : CHK_PTR_NULL(addr);
484 0 : CHK_PRT_RET(
485 : (size == 0), HCCL_ERROR("[MrManager][GetKey]memory size[%llu Byte] should be greater than 0.", size),
486 : HCCL_E_PARA);
487 :
488 0 : MrInfo mrInfo(addr, size);
489 0 : unique_lock<std::mutex> lockMrMap(mrMapSpinMutex_);
490 0 : bool isEmpty = regedMrMap_.empty();
491 0 : lockMrMap.unlock();
492 0 : if (isEmpty) {
493 0 : CHK_PRT_RET(
494 : (RegTmpMr(addr, size, lkey) != HCCL_SUCCESS),
495 : HCCL_ERROR("[MrManager][GetKey]register temp memory error, size[%llu Byte].", size), HCCL_E_INTERNAL);
496 : } else {
497 0 : bool isInfoNotFound = false;
498 0 : CHK_PRT_RET(
499 : (GetMrInfo(mrInfo, isInfoNotFound) != HCCL_SUCCESS),
500 : HCCL_ERROR("[MrManager][GetKey]get memory info error, size[%llu Byte].", size), HCCL_E_INTERNAL);
501 0 : if (isInfoNotFound) {
502 0 : CHK_PRT_RET(
503 : (RegTmpMr(addr, size, lkey) != HCCL_SUCCESS),
504 : HCCL_ERROR("[MrManager][GetKey]register temp memory error, size[%llu Byte].", size), HCCL_E_INTERNAL);
505 : } else {
506 0 : lockMrMap.lock();
507 0 : MrMapKey key(reinterpret_cast<u64>(mrInfo.addr), mrInfo.size);
508 0 : auto iter = regedMrMap_.find(key);
509 0 : iter->second.tmpMemRef++;
510 0 : lkey = mrInfo.lkey;
511 0 : HCCL_INFO(
512 : "[MrManager][GetKey]get memory lkey success, size[%llu Byte], regMrMap size[%u], "
513 : "temp mr map size[%u].",
514 : size, regedMrMap_.size(), regedMrMap_.size());
515 : }
516 : }
517 0 : return HCCL_SUCCESS;
518 0 : }
519 :
520 0 : HcclResult MrManager::ReleaseKey(void* addr, u64 size) // 释放临时MR
521 : {
522 0 : CHK_PTR_NULL(addr);
523 0 : CHK_PRT_RET(
524 : (size == 0), HCCL_ERROR("[MrManager][ReleaseKey]memory size[%llu Byte] should be greater than 0.", size),
525 : HCCL_E_PARA);
526 :
527 : HcclResult ret;
528 0 : MrInfo mrInfo;
529 0 : mrInfo.addr = addr;
530 0 : mrInfo.size = size;
531 0 : bool isInfoNotFound = false;
532 0 : ret = GetMrInfo(mrInfo, isInfoNotFound);
533 0 : if (ret || isInfoNotFound) {
534 0 : HCCL_ERROR("[MrManager][ReleaseKey]get memory info error, size[%llu Byte].", size);
535 0 : return HCCL_E_INTERNAL;
536 : }
537 :
538 0 : MrMapKey tmpMrMapKey;
539 0 : tmpMrMapKey.addr = reinterpret_cast<u64>(mrInfo.addr);
540 0 : tmpMrMapKey.size = mrInfo.size;
541 :
542 0 : unique_lock<std::mutex> lockMrMap(mrMapSpinMutex_);
543 0 : auto iter = regedMrMap_.find(tmpMrMapKey);
544 0 : CHK_PRT_RET(
545 : (iter == regedMrMap_.end()),
546 : HCCL_ERROR(
547 : "[MrManager][ReleaseKey] release key failed, size[%llu Byte]"
548 : "size[%llu], regMrMap size[%u].",
549 : size, mrInfo.size, regedMrMap_.size()),
550 : HCCL_E_INTERNAL);
551 :
552 0 : --iter->second.tmpMemRef;
553 0 : if (iter->second.tmpMemRef > 0 || iter->second.gloMemRef > 0) {
554 0 : HCCL_INFO(
555 : "[MrManager][ReleaseKey]release key success, size[%llu Byte], tmpMrMap size[%u], count[%d] "
556 : "tmp count[%d].",
557 : size, regedMrMap_.size(), iter->second.gloMemRef, iter->second.tmpMemRef);
558 0 : return HCCL_SUCCESS;
559 0 : } else if (iter->second.tmpMemRef < 0) {
560 0 : HCCL_ERROR(
561 : "[MrManager][ReleaseKey]release key error, size[%llu Byte], count[%d].", size, iter->second.tmpMemRef);
562 0 : return HCCL_E_MEMORY;
563 : }
564 :
565 0 : CHK_RET(DeRegMrImpl(iter->second));
566 0 : HCCL_INFO(
567 : "[MrManager][ReleaseKey] deregister success, size[%llu Byte], "
568 : "temp mr map size[%u].",
569 : size, regedMrMap_.size());
570 0 : regedMrMap_.erase(iter);
571 0 : lockMrMap.unlock();
572 0 : return HCCL_SUCCESS;
573 0 : }
574 :
575 0 : HcclResult MrManager::GetMrInfo(MrInfo& mrInfo, bool& isInfoNotFound)
576 : {
577 0 : CHK_PRT_RET(
578 : regedMrMap_.empty(), HCCL_ERROR("[MrManager][GetMrInfo]get mr info failed, mr map is empty"), HCCL_E_PARA);
579 :
580 0 : isInfoNotFound = false;
581 0 : unique_lock<std::mutex> lockMrMap(mrMapSpinMutex_);
582 :
583 0 : u64 uAddr = reinterpret_cast<u64>(mrInfo.addr);
584 0 : u64 size = mrInfo.size;
585 0 : MrMapKey key(uAddr, size);
586 0 : auto iter = regedMrMap_.find(key);
587 0 : if (iter != regedMrMap_.end()) {
588 0 : if (iter->second.size >= size) {
589 0 : mrInfo = iter->second;
590 0 : HCCL_DEBUG("[MrManager][GetMrInfo]get memory info success, size[%llu].", iter->second.size);
591 : } else {
592 0 : isInfoNotFound = true;
593 0 : HCCL_WARNING(
594 : "[MrManager][GetMrInfo]mr addr size[%llu], but required addr size[%llu].", iter->second.size,
595 : mrInfo.size);
596 : }
597 :
598 0 : return HCCL_SUCCESS;
599 : }
600 :
601 0 : iter = regedMrMap_.upper_bound(key);
602 0 : if (iter != regedMrMap_.begin()
603 0 : && !(iter != regedMrMap_.end() && iter->first.addr == uAddr && iter->first.size >= size)) {
604 0 : iter--;
605 : }
606 :
607 0 : u64 uTmpAddr = iter->first.addr;
608 0 : u64 tmpSize = iter->second.size;
609 0 : if (((uTmpAddr <= uAddr) && (uAddr < (uTmpAddr + tmpSize)))
610 0 : && ((uTmpAddr < (uAddr + size)) && ((uAddr + size) <= (uTmpAddr + tmpSize)))) {
611 0 : mrInfo = iter->second;
612 : } else {
613 0 : HCCL_WARNING("[MrManager][GetMrInfo] size[%llu] was not found.", mrInfo.size);
614 0 : isInfoNotFound = true;
615 0 : return HCCL_SUCCESS;
616 : }
617 0 : HCCL_DEBUG("[MrManager][GetMrInfo]get memory info success, size[%llu]", mrInfo.size);
618 0 : return HCCL_SUCCESS;
619 0 : }
620 :
621 0 : HcclResult MrManager::GetDevVirAddr(void* addr, u64 size, u64& devVirAddr)
622 : {
623 0 : CHK_PTR_NULL(addr);
624 0 : CHK_PRT_RET(
625 : (size == 0), HCCL_ERROR("[MrManager][GetDevVirAddr]memory size[%llu Byte] should be greater than 0.", size),
626 : HCCL_E_PARA);
627 0 : MrInfo mrInfo(addr, size);
628 0 : bool isInfoNotFound = false;
629 0 : CHK_PRT_RET(
630 : (GetMrInfo(mrInfo, isInfoNotFound) != HCCL_SUCCESS),
631 : HCCL_ERROR("[MrManager][GetDevVirAddr]get memory info error, size[%llu Byte].", size), HCCL_E_INTERNAL);
632 0 : CHK_PRT_RET(
633 : isInfoNotFound,
634 : HCCL_ERROR("[MrManager][GetDevVirAddr]get memory info fail, addr[%p], size[%llu Byte].", addr, size),
635 : HCCL_E_PARA);
636 : devVirAddr
637 0 : = reinterpret_cast<u64>(mrInfo.devVirAddr) + reinterpret_cast<u64>(addr) - reinterpret_cast<u64>(mrInfo.addr);
638 :
639 0 : return HCCL_SUCCESS;
640 : }
641 :
642 0 : void MrManager::TransMrInfo(void* addr, u64 size, HcclMrInfo& mrInfo)
643 : {
644 0 : mrInfo.addr = addr;
645 0 : mrInfo.size = size;
646 0 : mrInfo.access = RA_ACCESS_REMOTE_WRITE | RA_ACCESS_LOCAL_WRITE | RA_ACCESS_REMOTE_READ;
647 0 : }
648 :
649 0 : HcclResult MrManager::ReleaseMrResource()
650 : {
651 0 : HCCL_INFO("[MrManager][ReleaseMrResource]start release mr resource");
652 0 : unique_lock<std::mutex> lockMrMap(mrMapSpinMutex_);
653 0 : if (!regedMrMap_.empty()) {
654 0 : unique_lock<std::mutex> lockUnMrMap(unMrMapSpinMutex_);
655 0 : unRegMrMap_ = regedMrMap_;
656 0 : lockUnMrMap.unlock();
657 0 : u64 bound = regedMrMap_.begin()->first.addr;
658 0 : for (auto& iter : regedMrMap_) {
659 0 : if (iter.first.addr >= bound && iter.second.size != 0) {
660 0 : HCCL_DEBUG("deinit addr[%llu], size[%llu]", hash<void*>{}(iter.second.addr), iter.second.size);
661 0 : CHK_RET(DeRegMrImpl(iter.second));
662 0 : bound = iter.first.addr + iter.first.size;
663 : }
664 : }
665 :
666 0 : regedMrMap_.clear();
667 0 : }
668 :
669 0 : HCCL_INFO("[MrManager][ReleaseMrResource]release memory resource success.");
670 0 : return HCCL_SUCCESS;
671 0 : }
672 :
673 0 : void MrManager::SetHdcPara(u32 devId, bool isHostMem, bool isUseQPHandle)
674 : {
675 0 : isUseQPHandle_ = isUseQPHandle;
676 0 : curDevId_ = devId;
677 0 : IsHostMem_ = isHostMem;
678 0 : }
679 :
680 : } // namespace hccl
|
