Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "zero_copy_address_mgr.h"
12 :
13 : namespace hccl {
14 :
15 0 : HcclResult ZeroCopyAddressMgr::SetMemoryRange(u32 devicePhyId, void* baseAddr, u64 length)
16 : {
17 0 : if (baseAddr == nullptr || length == 0) {
18 0 : HCCL_ERROR("[ZeroCopyAddressMgr][SetMemoryRange] invalid input params addr[%p] len[%lu]", baseAddr, length);
19 0 : return HCCL_E_PARA;
20 : }
21 :
22 0 : CHK_PRT_RET(
23 : AddLocalIpc2RemoteAddr(devicePhyId, baseAddr, baseAddr, length),
24 : HCCL_ERROR("[ZeroCopyAddressMgr][SetMemoryRange] dev[%u] set addr [%p] failed", devicePhyId, baseAddr),
25 : HCCL_E_PARA);
26 :
27 0 : return HCCL_SUCCESS;
28 : }
29 :
30 0 : HcclResult ZeroCopyAddressMgr::UnsetMemoryRange(u32 devicePhyId, void* baseAddr)
31 : {
32 0 : if (baseAddr == nullptr) {
33 0 : HCCL_ERROR("[ZeroCopyAddressMgr][UnsetMemoryRange] invalid input params");
34 0 : return HCCL_E_PARA;
35 : }
36 :
37 0 : CHK_PRT_RET(
38 : DelLocalIpc2RemoteAddr(devicePhyId, baseAddr),
39 : HCCL_ERROR("[ZeroCopyAddressMgr][UnsetMemoryRange] dev[%u] unset [%p] addr failed", devicePhyId, baseAddr),
40 : HCCL_E_PARA);
41 :
42 0 : return HCCL_SUCCESS;
43 : }
44 :
45 0 : bool ZeroCopyAddressMgr::IsAddressSet(u32 devicePhyId, void* baseAddr)
46 : {
47 0 : std::lock_guard<std::mutex> guard(lock_);
48 0 : auto& addrMapping = reserveAddrMappings_[devicePhyId];
49 0 : return addrMapping.find(baseAddr) != addrMapping.end();
50 0 : }
51 :
52 : HcclResult
53 0 : ZeroCopyAddressMgr::AddLocalIpc2RemoteAddr(u32 devicePhyId, void* localIpcBase, void* remoteAddrBase, u64 length)
54 : {
55 : u32 maxDeviceNum;
56 0 : CHK_RET(GetMaxDevNum(maxDeviceNum));
57 0 : if (devicePhyId >= maxDeviceNum || localIpcBase == nullptr || remoteAddrBase == nullptr) {
58 0 : HCCL_ERROR(
59 : "[ZeroCopyAddressMgr][AddLocalIpc2RemoteAddr] devPhyId [%u] localIpc[%p] remoteAddr[%p] invalid params",
60 : devicePhyId, localIpcBase, remoteAddrBase);
61 0 : return HCCL_E_PARA;
62 : }
63 :
64 0 : ZeroCopyRingBufferItem item;
65 : {
66 0 : std::lock_guard<std::mutex> guard(lock_);
67 0 : auto& addrMapping = reserveAddrMappings_[devicePhyId];
68 0 : auto& addrRange = reserveRanges_[devicePhyId];
69 :
70 : // 检查地址是否已经reserve过
71 0 : CHK_PRT_RET(
72 : addrMapping.find(remoteAddrBase) != addrMapping.end(),
73 : HCCL_ERROR(
74 : "[ZeroCopyAddressMgr][AddLocalIpc2RemoteAddr] dev[%u] remote addr %p had set", devicePhyId,
75 : remoteAddrBase),
76 : HCCL_E_PARA);
77 :
78 : // 检查地址reserve的地址区间是否与之前的有交叠
79 0 : AddressRange range(remoteAddrBase, length);
80 0 : CHK_PRT_RET(
81 : addrRange.find(range) != addrRange.end(),
82 : HCCL_ERROR(
83 : "[ZeroCopyAddressMgr][AddLocalIpc2RemoteAddr] dev[%u] remote addr %p had set with overlap range",
84 : devicePhyId, remoteAddrBase),
85 : HCCL_E_PARA);
86 :
87 0 : item.type = ZeroCopyItemType::SET_MEMORY;
88 0 : item.addr.devicePhyId = devicePhyId;
89 0 : item.addr.localIpcAddr = reinterpret_cast<u64>(localIpcBase);
90 0 : item.addr.remoteAddr = reinterpret_cast<u64>(remoteAddrBase);
91 0 : item.addr.length = length;
92 0 : addrMapping.insert({remoteAddrBase, item.addr});
93 0 : addrRange.insert(range);
94 0 : }
95 0 : CHK_RET(PushOne(item));
96 0 : HCCL_INFO(
97 : "[ZeroCopyAddressMgr][AddLocalIpc2RemoteAddr] dev[%u] add set localIpc[%p] remote[%p] length[%lu]", devicePhyId,
98 : localIpcBase, remoteAddrBase, length);
99 0 : return HCCL_SUCCESS;
100 : }
101 :
102 0 : HcclResult ZeroCopyAddressMgr::DelLocalIpc2RemoteAddr(u32 devicePhyId, void* remoteAddrBase)
103 : {
104 : u32 maxDeviceNum;
105 0 : CHK_RET(GetMaxDevNum(maxDeviceNum));
106 0 : if (devicePhyId >= maxDeviceNum || remoteAddrBase == nullptr) {
107 0 : HCCL_ERROR("[ZeroCopyAddressMgr][DelLocalIpc2RemoteAddr] devPhyId [%u] invalid params", devicePhyId);
108 0 : return HCCL_E_PARA;
109 : }
110 :
111 : u64 length;
112 0 : ZeroCopyRingBufferItem item;
113 : {
114 0 : std::lock_guard<std::mutex> guard(lock_);
115 0 : auto& addrMapping = reserveAddrMappings_[devicePhyId];
116 0 : auto& addrRange = reserveRanges_[devicePhyId];
117 :
118 : // 检查地址是否已经reserve过
119 0 : auto mappingIt = addrMapping.find(remoteAddrBase);
120 0 : CHK_PRT_RET(
121 : mappingIt == addrMapping.end(),
122 : HCCL_ERROR(
123 : "[ZeroCopyAddressMgr][DelLocalIpc2RemoteAddr] dev[%u] addr %p not set", devicePhyId, remoteAddrBase),
124 : HCCL_E_PARA);
125 :
126 0 : length = mappingIt->second.length;
127 0 : AddressRange range(remoteAddrBase, length);
128 0 : auto rangeIt = addrRange.find(range);
129 0 : CHK_PRT_RET(
130 : rangeIt == addrRange.end(),
131 : HCCL_ERROR(
132 : "[ZeroCopyAddressMgr][DelLocalIpc2RemoteAddr] dev[%u] addr %p not set", devicePhyId, remoteAddrBase),
133 : HCCL_E_PARA);
134 :
135 : // 检查是否仍存在Activate的内存
136 0 : AddressRange localRange(mappingIt->second.localIpcAddr, length);
137 0 : auto activateIt = validAddressRanges_.find(localRange);
138 0 : CHK_PRT_RET(
139 : activateIt != validAddressRanges_.end(),
140 : HCCL_ERROR(
141 : "[ZeroCopyAddressMgr][DelLocalIpc2RemoteAddr] dev[%u] remoteAddr %p localAddr 0x%lx still have "
142 : "activate memory [0x%lx, 0x%lx)",
143 : devicePhyId, remoteAddrBase, mappingIt->second.localIpcAddr, activateIt->start, activateIt->end),
144 : HCCL_E_PARA);
145 :
146 0 : item.type = ZeroCopyItemType::UNSET_MEMORY;
147 0 : item.addr = mappingIt->second;
148 0 : addrMapping.erase(mappingIt);
149 0 : addrRange.erase(rangeIt);
150 0 : }
151 0 : CHK_RET(PushOne(item));
152 0 : HCCL_INFO(
153 : "[ZeroCopyAddressMgr][DelLocalIpc2RemoteAddr] dev[%u] del set localIpc[0x%lx] remote[0x%lx] length[%lu]",
154 : devicePhyId, item.addr.localIpcAddr, item.addr.remoteAddr, length);
155 0 : return HCCL_SUCCESS;
156 : }
157 :
158 0 : HcclResult ZeroCopyAddressMgr::GetLocalIpc2RemoteAddr(u32 devicePhyId, void* remoteAddr, LocalIpc2RemoteAddr& addr)
159 : {
160 : u32 maxDeviceNum;
161 0 : CHK_RET(GetMaxDevNum(maxDeviceNum));
162 0 : if (devicePhyId >= maxDeviceNum || remoteAddr == nullptr) {
163 0 : HCCL_ERROR("[ZeroCopyAddressMgr][GetLocalIpc2RemoteAddr] devPhyId [%u] invalid params", devicePhyId);
164 0 : return HCCL_E_PARA;
165 : }
166 :
167 0 : std::lock_guard<std::mutex> guard(lock_);
168 0 : auto& addrMapping = reserveAddrMappings_[devicePhyId];
169 0 : auto& addrRange = reserveRanges_[devicePhyId];
170 :
171 0 : AddressRange range(remoteAddr, 1);
172 0 : auto rangeIt = addrRange.find(range);
173 0 : CHK_PRT_RET(
174 : rangeIt == addrRange.end(),
175 : HCCL_ERROR("[ZeroCopyAddressMgr][GetLocalIpc2RemoteAddr] dev[%u] addr %p not set", devicePhyId, remoteAddr),
176 : HCCL_E_PARA);
177 :
178 0 : void* remoteAddrBase = reinterpret_cast<void*>(rangeIt->start);
179 0 : auto mapIt = addrMapping.find(remoteAddrBase);
180 0 : CHK_PRT_RET(
181 : mapIt == addrMapping.end(),
182 : HCCL_ERROR(
183 : "[ZeroCopyAddressMgr][GetLocalIpc2RemoteAddr] dev[%u] baseAddr %p not mapped", devicePhyId, remoteAddrBase),
184 : HCCL_E_PARA);
185 :
186 0 : addr = mapIt->second;
187 :
188 0 : return HCCL_SUCCESS;
189 0 : }
190 :
191 0 : HcclResult ZeroCopyAddressMgr::ActivateCommMemoryAddr(void* startPtr, u64 length)
192 : {
193 0 : CHK_PRT_RET(
194 : (startPtr == nullptr || length == 0), HCCL_ERROR("[ZeroCopyAddressMgr][ActivateCommMemoryAddr] Invalid params"),
195 : HCCL_E_PARA);
196 :
197 0 : AddressRange range(startPtr, length);
198 :
199 : {
200 0 : std::lock_guard<std::mutex> guard(lock_);
201 0 : auto it = validAddressRanges_.find(range);
202 0 : CHK_PRT_RET(
203 : (it != validAddressRanges_.end()),
204 : HCCL_ERROR(
205 : "[ZeroCopyAddressMgr][ActivateCommMemoryAddr] overlap address exist:[0x%lx, 0x%lx) valid:[0x%lx, "
206 : "0x%lx)",
207 : it->start, it->end, range.start, range.end),
208 : HCCL_E_PARA);
209 :
210 0 : validAddressRanges_.insert(range);
211 0 : }
212 0 : ZeroCopyRingBufferItem item;
213 0 : item.type = ZeroCopyItemType::ACTIVATE_MEMORY;
214 0 : item.addr.localIpcAddr = reinterpret_cast<u64>(startPtr);
215 0 : item.addr.length = length;
216 0 : CHK_RET(PushOne(item));
217 :
218 0 : HCCL_INFO(
219 : "[ZeroCopyAddressMgr][ActivateCommMemoryAddr] activate address [0x%lx, 0x%lx) success", range.start, range.end);
220 0 : return HCCL_SUCCESS;
221 : }
222 :
223 0 : HcclResult ZeroCopyAddressMgr::DeactivateCommMemoryAddr(void* startPtr)
224 : {
225 0 : CHK_PRT_RET(
226 : (startPtr == nullptr), HCCL_ERROR("[ZeroCopyAddressMgr][DeactivateCommMemoryAddr] Invalid params"),
227 : HCCL_E_PARA);
228 :
229 : // 我们构造一个最小的交叠区间去做比较
230 0 : u64 litteLen = 1;
231 0 : AddressRange range(startPtr, litteLen);
232 :
233 : {
234 0 : std::lock_guard<std::mutex> guard(lock_);
235 0 : auto it = validAddressRanges_.find(range);
236 0 : CHK_PRT_RET(
237 : (it == validAddressRanges_.end() || it->start != range.start),
238 : HCCL_ERROR("[ZeroCopyAddressMgr][DeactivateCommMemoryAddr] address %p is not activate", startPtr),
239 : HCCL_E_PARA);
240 :
241 0 : HCCL_INFO(
242 : "[ZeroCopyAddressMgr][DeactivateCommMemoryAddr] deactivate address [0x%lx, 0x%lx) success", it->start,
243 : it->end);
244 0 : validAddressRanges_.erase(it);
245 0 : }
246 :
247 0 : ZeroCopyRingBufferItem item;
248 0 : item.type = ZeroCopyItemType::DEACTIVATE_MEMORY;
249 0 : item.addr.localIpcAddr = reinterpret_cast<u64>(startPtr);
250 0 : CHK_RET(PushOne(item));
251 :
252 0 : return HCCL_SUCCESS;
253 : }
254 :
255 0 : HcclResult ZeroCopyAddressMgr::AddRemoteImportAddr(void* devPtr, void* handle)
256 : {
257 0 : CHK_PRT_RET(
258 : (devPtr == nullptr || handle == nullptr),
259 : HCCL_ERROR("[ZeroCopyAddressMgr][AddRemoteImportAddr] invalid devPtr[%p] handle[%p]", devPtr, handle),
260 : HCCL_E_PARA);
261 :
262 0 : std::lock_guard<std::mutex> guard(lock_);
263 0 : auto it = importAddrs_.find(devPtr);
264 0 : CHK_PRT_RET(
265 : it != importAddrs_.end(), HCCL_ERROR("[ZeroCopyAddressMgr][AddRemoteImportAddr] devPtr[%p] has import", devPtr),
266 : HCCL_E_PARA);
267 :
268 0 : HCCL_INFO("[ZeroCopyAddressMgr][AddRemoteImportAddr] add devPtr[%p] handle[%p]", devPtr, handle);
269 0 : importAddrs_.insert({devPtr, handle});
270 0 : return HCCL_SUCCESS;
271 0 : }
272 :
273 0 : HcclResult ZeroCopyAddressMgr::GetRemoteImportAddr(void* devPtr, void*& handle)
274 : {
275 0 : CHK_PRT_RET(
276 : (devPtr == nullptr), HCCL_ERROR("[ZeroCopyAddressMgr][GetRemoteImportAddr] invalid devPtr[%p]", devPtr),
277 : HCCL_E_PARA);
278 :
279 0 : std::lock_guard<std::mutex> guard(lock_);
280 0 : auto it = importAddrs_.find(devPtr);
281 0 : CHK_PRT_RET(
282 : it == importAddrs_.end(), HCCL_ERROR("[ZeroCopyAddressMgr][GetRemoteImportAddr] devPtr[%p] not import", devPtr),
283 : HCCL_E_PARA);
284 :
285 0 : handle = importAddrs_[devPtr];
286 0 : HCCL_INFO("[ZeroCopyAddressMgr][GetRemoteImportAddr] get devPtr[%p] handle[%p]", devPtr, handle);
287 :
288 0 : return HCCL_SUCCESS;
289 0 : }
290 :
291 0 : HcclResult ZeroCopyAddressMgr::DelRemoteImportAddr(void* devPtr)
292 : {
293 0 : CHK_PRT_RET(
294 : (devPtr == nullptr), HCCL_ERROR("[ZeroCopyAddressMgr][DelRemoteImportAddr] invalid devPtr[%p]", devPtr),
295 : HCCL_E_PARA);
296 :
297 0 : std::lock_guard<std::mutex> guard(lock_);
298 0 : auto it = importAddrs_.find(devPtr);
299 0 : CHK_PRT_RET(
300 : it == importAddrs_.end(), HCCL_ERROR("[ZeroCopyAddressMgr][GetRemoteImportAddr] devPtr[%p] not import", devPtr),
301 : HCCL_E_PARA);
302 :
303 0 : void* handle = importAddrs_[devPtr];
304 0 : HCCL_INFO("[ZeroCopyAddressMgr][GetRemoteImportAddr] del devPtr[%p] handle[%p]", devPtr, handle);
305 0 : importAddrs_.erase(it);
306 :
307 0 : return HCCL_SUCCESS;
308 0 : }
309 :
310 0 : bool ZeroCopyAddressMgr::IsActivateCommMemoryAddr(void* startPtr, u64 length)
311 : {
312 0 : if (startPtr == nullptr || length == 0) {
313 0 : return false;
314 : }
315 :
316 0 : AddressRange range(startPtr, length);
317 :
318 0 : std::lock_guard<std::mutex> guard(lock_);
319 : // 我们先用最小区间去查找最前面匹配的valid内存块，后续的就依次遍历即可
320 0 : AddressRange litteRange(startPtr, 1);
321 0 : auto beginIt = validAddressRanges_.find(litteRange);
322 0 : while (beginIt != validAddressRanges_.end()) {
323 : // 此片内存已经是valid内存的子集了，那么是有效的
324 0 : if (range.end <= beginIt->end) {
325 0 : return true;
326 : }
327 :
328 : // 前一片内存已经匹配到了小块，把前面的内存切分掉，继续去匹配更后面的数据
329 0 : range.start = beginIt->end;
330 0 : beginIt++;
331 : }
332 :
333 : // 输入区间内有部分没有查找到，所以认为是无效的
334 0 : return false;
335 0 : }
336 :
337 0 : bool ZeroCopyAddressMgr::IsOverlapWithActivateAddr(void* startPtr, u64 length)
338 : {
339 0 : if (startPtr == nullptr || length == 0) {
340 0 : return false;
341 : }
342 :
343 0 : AddressRange range(startPtr, length);
344 0 : std::lock_guard<std::mutex> guard(lock_);
345 0 : return validAddressRanges_.find(range) != validAddressRanges_.end();
346 0 : }
347 :
348 0 : bool ZeroCopyAddressMgr::IsInSetAddressRange(u32 devicePhyId, void* startPtr, u64 length)
349 : {
350 0 : std::lock_guard<std::mutex> guard(lock_);
351 0 : auto& addrRange = reserveRanges_[devicePhyId];
352 :
353 : // 构造最小的数据块去寻找，如果没找到肯定没有交集
354 0 : AddressRange range(startPtr, 1);
355 0 : auto rangeIt = addrRange.find(range);
356 0 : if (rangeIt == addrRange.end()) {
357 0 : HCCL_INFO("[ZeroCopyAddressMgr][IsInSetAddressRange] not in reserve range");
358 0 : return false;
359 : }
360 :
361 : // 判断尾巴是否在当前匹配内存块中，如果不在那么不在范围内
362 0 : if (range.start + length > rangeIt->end) {
363 0 : HCCL_INFO("[ZeroCopyAddressMgr][IsInSetAddressRange] exceed reserve range");
364 0 : return false;
365 : }
366 :
367 0 : return true;
368 0 : }
369 :
370 0 : HcclResult ZeroCopyAddressMgr::ProcessRingBuffer(ZeroCopyRingBufferItem* ringBuffer, u32* head, u32* tail)
371 : {
372 0 : if (ringBuffer == nullptr || head == nullptr || tail == nullptr) {
373 0 : HCCL_ERROR(
374 : "[ZeroCopyAddressMgr][ProcessRingBuffer] invalid param ringBuff[%p] head[%p] tail[%p]", ringBuffer, head,
375 : tail);
376 0 : return HCCL_E_PARA;
377 : }
378 :
379 0 : std::lock_guard<std::mutex> guard(processRingBufferLock_);
380 0 : needPushOne = false;
381 0 : if (*head == *tail) {
382 0 : HCCL_INFO(
383 : "[ZeroCopyAddressMgr][ProcessRingBuffer] ring buffer is empty, so do nothing, head[%u] tail[%u]", *head,
384 : *tail);
385 0 : return HCCL_SUCCESS;
386 : }
387 :
388 0 : if (*tail >= ZERO_COPY_BUFFER_MAX_MAP_COUNT || *head >= ZERO_COPY_BUFFER_MAX_MAP_COUNT) {
389 0 : HCCL_ERROR("[ZeroCopyAddressMgr][ProcessRingBuffer] invalid head/tail, head[%u] tail[%u]", *head, *tail);
390 0 : return HCCL_E_PARA;
391 : }
392 :
393 0 : u32 now = *head;
394 0 : while (now != *tail) {
395 0 : HCCL_INFO(
396 : "[ZeroCopyAddressMgr][ProcessRingBuffer] process ringbuffer now[%u] ptr[%p] tail[%u] type[%d]", now,
397 : ringBuffer + now, *tail, ringBuffer[now].type);
398 0 : CHK_RET(ProcessOneAddrMap(ringBuffer[now]));
399 0 : now = (now + 1) % ZERO_COPY_BUFFER_MAX_MAP_COUNT;
400 : }
401 :
402 : // 更新所有的值
403 0 : *head = *tail;
404 0 : HCCL_INFO("[ZeroCopyAddressMgr][ProcessRingBuffer] ringbuffer head[%u] tail[%u]", *head, *tail);
405 0 : return HCCL_SUCCESS;
406 0 : }
407 :
408 0 : HcclResult ZeroCopyAddressMgr::ProcessOneAddrMap(const ZeroCopyRingBufferItem& item)
409 : {
410 0 : HCCL_INFO(
411 : "[ZeroCopyAddressMgr][ProcessOneAddrMap] Item info: type[%d] dev[%u] local[0x%lx] remote[0x%lx] len[%lu]",
412 : item.type, item.addr.devicePhyId, item.addr.localIpcAddr, item.addr.remoteAddr, item.addr.length);
413 0 : switch (item.type) {
414 0 : case ZeroCopyItemType::SET_MEMORY:
415 0 : return AddLocalIpc2RemoteAddr(
416 0 : item.addr.devicePhyId, reinterpret_cast<void*>(item.addr.localIpcAddr),
417 0 : reinterpret_cast<void*>(item.addr.remoteAddr), item.addr.length);
418 0 : case ZeroCopyItemType::UNSET_MEMORY:
419 0 : return DelLocalIpc2RemoteAddr(item.addr.devicePhyId, reinterpret_cast<void*>(item.addr.remoteAddr));
420 0 : case ZeroCopyItemType::ACTIVATE_MEMORY:
421 0 : return ActivateCommMemoryAddr(reinterpret_cast<void*>(item.addr.localIpcAddr), item.addr.length);
422 0 : case ZeroCopyItemType::DEACTIVATE_MEMORY:
423 0 : return DeactivateCommMemoryAddr(reinterpret_cast<void*>(item.addr.localIpcAddr));
424 0 : default:
425 0 : HCCL_ERROR("[ZeroCopyAddressMgr][ProcessOneAddrMap] invalid type[%d]", item.type);
426 0 : return HCCL_E_PARA;
427 : }
428 :
429 : return HCCL_SUCCESS;
430 : }
431 :
432 0 : u32 ZeroCopyAddressMgr::GetCommRefCnt() const { return commRefCnt_; }
433 :
434 0 : HcclResult ZeroCopyAddressMgr::IncreCommRefCnt()
435 : {
436 0 : commRefCnt_++;
437 0 : return HCCL_SUCCESS;
438 : }
439 :
440 0 : HcclResult ZeroCopyAddressMgr::DecreCommRefCnt()
441 : {
442 0 : if (commRefCnt_ == 0) {
443 0 : HCCL_WARNING("[ZeroCopyAddressMgr][%s]commRefCnt_ is 0, cannot decrement", __func__);
444 0 : return HCCL_SUCCESS;
445 : }
446 0 : commRefCnt_--;
447 0 : return HCCL_SUCCESS;
448 : }
449 :
450 : } // namespace hccl
|
