Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "zero_copy_address_mgr.h"
12 :
13 : namespace hccl {
14 :
15 0 : HcclResult ZeroCopyAddressMgr::SetMemoryRange(u32 devicePhyId, void* baseAddr, u64 length)
16 : {
17 0 : if (baseAddr == nullptr || length == 0) {
18 0 : HCCL_ERROR("[ZeroCopyAddressMgr][SetMemoryRange] invalid input params addr[%p] len[%lu]", baseAddr, length);
19 0 : return HCCL_E_PARA;
20 : }
21 :
22 0 : CHK_PRT_RET(
23 : AddLocalIpc2RemoteAddr(devicePhyId, baseAddr, baseAddr, length),
24 : HCCL_ERROR("[ZeroCopyAddressMgr][SetMemoryRange] dev[%u] set addr [%p] failed", devicePhyId, baseAddr),
25 : HCCL_E_PARA);
26 :
27 0 : return HCCL_SUCCESS;
28 : }
29 :
30 0 : HcclResult ZeroCopyAddressMgr::UnsetMemoryRange(u32 devicePhyId, void* baseAddr)
31 : {
32 0 : if (baseAddr == nullptr) {
33 0 : HCCL_ERROR("[ZeroCopyAddressMgr][UnsetMemoryRange] invalid input params");
34 0 : return HCCL_E_PARA;
35 : }
36 :
37 0 : CHK_PRT_RET(
38 : DelLocalIpc2RemoteAddr(devicePhyId, baseAddr),
39 : HCCL_ERROR("[ZeroCopyAddressMgr][UnsetMemoryRange] dev[%u] unset [%p] addr failed", devicePhyId, baseAddr),
40 : HCCL_E_PARA);
41 :
42 0 : return HCCL_SUCCESS;
43 : }
44 :
45 0 : bool ZeroCopyAddressMgr::IsAddressSet(u32 devicePhyId, void* baseAddr)
46 : {
47 0 : std::lock_guard<std::mutex> guard(lock_);
48 0 : auto& addrMapping = reserveAddrMappings_[devicePhyId];
49 0 : return addrMapping.find(baseAddr) != addrMapping.end();
50 0 : }
51 :
52 : HcclResult
53 0 : ZeroCopyAddressMgr::AddLocalIpc2RemoteAddr(u32 devicePhyId, void* localIpcBase, void* remoteAddrBase, u64 length)
54 : {
55 : u32 maxDeviceNum;
56 0 : CHK_RET(GetMaxDevNum(maxDeviceNum));
57 0 : if (devicePhyId >= maxDeviceNum || localIpcBase == nullptr || remoteAddrBase == nullptr) {
58 0 : HCCL_ERROR(
59 : "[ZeroCopyAddressMgr][AddLocalIpc2RemoteAddr] devPhyId [%u] localIpc[%p] remoteAddr[%p] invalid params",
60 : devicePhyId, localIpcBase, remoteAddrBase);
61 0 : return HCCL_E_PARA;
62 : }
63 :
64 0 : ZeroCopyRingBufferItem item;
65 : {
66 0 : std::lock_guard<std::mutex> guard(lock_);
67 0 : auto& addrMapping = reserveAddrMappings_[devicePhyId];
68 0 : auto& addrRange = reserveRanges_[devicePhyId];
69 :
70 : // 检查地址是否已经reserve过
71 0 : CHK_PRT_RET(
72 : addrMapping.find(remoteAddrBase) != addrMapping.end(),
73 : HCCL_ERROR(
74 : "[ZeroCopyAddressMgr][AddLocalIpc2RemoteAddr] dev[%u] remote addr %p had set", devicePhyId,
75 : remoteAddrBase),
76 : HCCL_E_PARA);
77 :
78 : // 检查地址reserve的地址区间是否与之前的有交叠
79 0 : AddressRange range(remoteAddrBase, length);
80 0 : CHK_PRT_RET(
81 : addrRange.find(range) != addrRange.end(),
82 : HCCL_ERROR(
83 : "[ZeroCopyAddressMgr][AddLocalIpc2RemoteAddr] dev[%u] remote addr %p had set with overlap range",
84 : devicePhyId, remoteAddrBase),
85 : HCCL_E_PARA);
86 :
87 0 : item.type = ZeroCopyItemType::SET_MEMORY;
88 0 : item.addr.devicePhyId = devicePhyId;
89 0 : item.addr.localIpcAddr = reinterpret_cast<u64>(localIpcBase);
90 0 : item.addr.remoteAddr = reinterpret_cast<u64>(remoteAddrBase);
91 0 : item.addr.length = length;
92 0 : addrMapping.insert({remoteAddrBase, item.addr});
93 0 : addrRange.insert(range);
94 0 : }
95 0 : CHK_RET(PushOne(item));
96 0 : HCCL_INFO(
97 : "[ZeroCopyAddressMgr][AddLocalIpc2RemoteAddr] dev[%u] add set localIpc[%p] remote[%p] length[%lu]", devicePhyId,
98 : localIpcBase, remoteAddrBase, length);
99 0 : return HCCL_SUCCESS;
100 : }
101 :
102 0 : HcclResult ZeroCopyAddressMgr::DelLocalIpc2RemoteAddr(u32 devicePhyId, void* remoteAddrBase)
103 : {
104 : u32 maxDeviceNum;
105 0 : CHK_RET(GetMaxDevNum(maxDeviceNum));
106 0 : if (devicePhyId >= maxDeviceNum || remoteAddrBase == nullptr) {
107 0 : HCCL_ERROR("[ZeroCopyAddressMgr][DelLocalIpc2RemoteAddr] devPhyId [%u] invalid params", devicePhyId);
108 0 : return HCCL_E_PARA;
109 : }
110 :
111 : u64 length;
112 0 : ZeroCopyRingBufferItem item;
113 : {
114 0 : std::lock_guard<std::mutex> guard(lock_);
115 0 : auto& addrMapping = reserveAddrMappings_[devicePhyId];
116 0 : auto& addrRange = reserveRanges_[devicePhyId];
117 :
118 : // 检查地址是否已经reserve过
119 0 : auto mappingIt = addrMapping.find(remoteAddrBase);
120 0 : CHK_PRT_RET(
121 : mappingIt == addrMapping.end(),
122 : HCCL_ERROR(
123 : "[ZeroCopyAddressMgr][DelLocalIpc2RemoteAddr] dev[%u] addr %p not set", devicePhyId, remoteAddrBase),
124 : HCCL_E_PARA);
125 :
126 0 : length = mappingIt->second.length;
127 0 : AddressRange range(remoteAddrBase, length);
128 0 : auto rangeIt = addrRange.find(range);
129 0 : CHK_PRT_RET(
130 : rangeIt == addrRange.end(),
131 : HCCL_ERROR(
132 : "[ZeroCopyAddressMgr][DelLocalIpc2RemoteAddr] dev[%u] addr %p not set", devicePhyId, remoteAddrBase),
133 : HCCL_E_PARA);
134 :
135 : // 检查是否仍存在Activate的内存
136 0 : AddressRange localRange(mappingIt->second.localIpcAddr, length);
137 0 : auto activateIt = validAddressRanges_.find(localRange);
138 0 : CHK_PRT_RET(
139 : activateIt != validAddressRanges_.end(),
140 : HCCL_ERROR(
141 : "[ZeroCopyAddressMgr][DelLocalIpc2RemoteAddr] dev[%u] remoteAddr %p localAddr 0x%lx still have "
142 : "activate memory [0x%lx, 0x%lx)",
143 : devicePhyId, remoteAddrBase, mappingIt->second.localIpcAddr, activateIt->start, activateIt->end),
144 : HCCL_E_PARA);
145 :
146 0 : item.type = ZeroCopyItemType::UNSET_MEMORY;
147 0 : item.addr = mappingIt->second;
148 0 : addrMapping.erase(mappingIt);
149 0 : addrRange.erase(rangeIt);
150 0 : }
151 0 : CHK_RET(PushOne(item));
152 0 : HCCL_INFO(
153 : "[ZeroCopyAddressMgr][DelLocalIpc2RemoteAddr] dev[%u] del set localIpc[0x%lx] remote[0x%lx] length[%lu]",
154 : devicePhyId, item.addr.localIpcAddr, item.addr.remoteAddr, length);
155 0 : return HCCL_SUCCESS;
156 : }
157 :
158 0 : HcclResult ZeroCopyAddressMgr::GetLocalIpc2RemoteAddr(u32 devicePhyId, void* remoteAddr, LocalIpc2RemoteAddr& addr)
159 : {
160 : u32 maxDeviceNum;
161 0 : CHK_RET(GetMaxDevNum(maxDeviceNum));
162 0 : if (devicePhyId >= maxDeviceNum || remoteAddr == nullptr) {
163 0 : HCCL_ERROR("[ZeroCopyAddressMgr][GetLocalIpc2RemoteAddr] devPhyId [%u] invalid params", devicePhyId);
164 0 : return HCCL_E_PARA;
165 : }
166 :
167 0 : std::lock_guard<std::mutex> guard(lock_);
168 0 : auto& addrMapping = reserveAddrMappings_[devicePhyId];
169 0 : auto& addrRange = reserveRanges_[devicePhyId];
170 :
171 0 : AddressRange range(remoteAddr, 1);
172 0 : auto rangeIt = addrRange.find(range);
173 0 : CHK_PRT_RET(
174 : rangeIt == addrRange.end(),
175 : HCCL_ERROR("[ZeroCopyAddressMgr][GetLocalIpc2RemoteAddr] dev[%u] addr %p not set", devicePhyId, remoteAddr),
176 : HCCL_E_PARA);
177 :
178 0 : void* remoteAddrBase = reinterpret_cast<void*>(rangeIt->start);
179 0 : auto mapIt = addrMapping.find(remoteAddrBase);
180 0 : CHK_PRT_RET(
181 : rangeIt == addrRange.end(),
182 : HCCL_ERROR("[ZeroCopyAddressMgr][GetLocalIpc2RemoteAddr] dev[%u] addr %p not set", devicePhyId, remoteAddr),
183 : HCCL_E_PARA);
184 :
185 0 : addr = mapIt->second;
186 :
187 0 : return HCCL_SUCCESS;
188 0 : }
189 :
190 0 : HcclResult ZeroCopyAddressMgr::ActivateCommMemoryAddr(void* startPtr, u64 length)
191 : {
192 0 : CHK_PRT_RET(
193 : (startPtr == nullptr || length == 0), HCCL_ERROR("[ZeroCopyAddressMgr][ActivateCommMemoryAddr] Invalid params"),
194 : HCCL_E_PARA);
195 :
196 0 : AddressRange range(startPtr, length);
197 :
198 : {
199 0 : std::lock_guard<std::mutex> guard(lock_);
200 0 : auto it = validAddressRanges_.find(range);
201 0 : CHK_PRT_RET(
202 : (it != validAddressRanges_.end()),
203 : HCCL_ERROR(
204 : "[ZeroCopyAddressMgr][ActivateCommMemoryAddr] overlap address exist:[0x%lx, 0x%lx) valid:[0x%lx, "
205 : "0x%lx)",
206 : it->start, it->end, range.start, range.end),
207 : HCCL_E_PARA);
208 :
209 0 : validAddressRanges_.insert(range);
210 0 : }
211 0 : ZeroCopyRingBufferItem item;
212 0 : item.type = ZeroCopyItemType::ACTIVATE_MEMORY;
213 0 : item.addr.localIpcAddr = reinterpret_cast<u64>(startPtr);
214 0 : item.addr.length = length;
215 0 : CHK_RET(PushOne(item));
216 :
217 0 : HCCL_INFO(
218 : "[ZeroCopyAddressMgr][ActivateCommMemoryAddr] activate address [0x%lx, 0x%lx) success", range.start, range.end);
219 0 : return HCCL_SUCCESS;
220 : }
221 :
222 0 : HcclResult ZeroCopyAddressMgr::DeactivateCommMemoryAddr(void* startPtr)
223 : {
224 0 : CHK_PRT_RET(
225 : (startPtr == nullptr), HCCL_ERROR("[ZeroCopyAddressMgr][DeactivateCommMemoryAddr] Invalid params"),
226 : HCCL_E_PARA);
227 :
228 : // 我们构造一个最小的交叠区间去做比较
229 0 : u64 litteLen = 1;
230 0 : AddressRange range(startPtr, litteLen);
231 :
232 : {
233 0 : std::lock_guard<std::mutex> guard(lock_);
234 0 : auto it = validAddressRanges_.find(range);
235 0 : CHK_PRT_RET(
236 : (it == validAddressRanges_.end() || it->start != range.start),
237 : HCCL_ERROR("[ZeroCopyAddressMgr][DeactivateCommMemoryAddr] address %p is not activate", startPtr),
238 : HCCL_E_PARA);
239 :
240 0 : HCCL_INFO(
241 : "[ZeroCopyAddressMgr][DeactivateCommMemoryAddr] deactivate address [0x%lx, 0x%lx) success", it->start,
242 : it->end);
243 0 : validAddressRanges_.erase(it);
244 0 : }
245 :
246 0 : ZeroCopyRingBufferItem item;
247 0 : item.type = ZeroCopyItemType::DEACTIVATE_MEMORY;
248 0 : item.addr.localIpcAddr = reinterpret_cast<u64>(startPtr);
249 0 : CHK_RET(PushOne(item));
250 :
251 0 : return HCCL_SUCCESS;
252 : }
253 :
254 0 : HcclResult ZeroCopyAddressMgr::AddRemoteImportAddr(void* devPtr, void* handle)
255 : {
256 0 : CHK_PRT_RET(
257 : (devPtr == nullptr || handle == nullptr),
258 : HCCL_ERROR("[ZeroCopyAddressMgr][AddRemoteImportAddr] invalid devPtr[%p] handle[%p]", devPtr, handle),
259 : HCCL_E_PARA);
260 :
261 0 : std::lock_guard<std::mutex> guard(lock_);
262 0 : auto it = importAddrs_.find(devPtr);
263 0 : CHK_PRT_RET(
264 : it != importAddrs_.end(), HCCL_ERROR("[ZeroCopyAddressMgr][AddRemoteImportAddr] devPtr[%p] has import", devPtr),
265 : HCCL_E_PARA);
266 :
267 0 : HCCL_INFO("[ZeroCopyAddressMgr][AddRemoteImportAddr] add devPtr[%p] handle[%p]", devPtr, handle);
268 0 : importAddrs_.insert({devPtr, handle});
269 0 : return HCCL_SUCCESS;
270 0 : }
271 :
272 0 : HcclResult ZeroCopyAddressMgr::GetRemoteImportAddr(void* devPtr, void*& handle)
273 : {
274 0 : CHK_PRT_RET(
275 : (devPtr == nullptr), HCCL_ERROR("[ZeroCopyAddressMgr][GetRemoteImportAddr] invalid devPtr[%p]", devPtr),
276 : HCCL_E_PARA);
277 :
278 0 : std::lock_guard<std::mutex> guard(lock_);
279 0 : auto it = importAddrs_.find(devPtr);
280 0 : CHK_PRT_RET(
281 : it == importAddrs_.end(), HCCL_ERROR("[ZeroCopyAddressMgr][GetRemoteImportAddr] devPtr[%p] not import", devPtr),
282 : HCCL_E_PARA);
283 :
284 0 : handle = importAddrs_[devPtr];
285 0 : HCCL_INFO("[ZeroCopyAddressMgr][GetRemoteImportAddr] get devPtr[%p] handle[%p]", devPtr, handle);
286 :
287 0 : return HCCL_SUCCESS;
288 0 : }
289 :
290 0 : HcclResult ZeroCopyAddressMgr::DelRemoteImportAddr(void* devPtr)
291 : {
292 0 : CHK_PRT_RET(
293 : (devPtr == nullptr), HCCL_ERROR("[ZeroCopyAddressMgr][DelRemoteImportAddr] invalid devPtr[%p]", devPtr),
294 : HCCL_E_PARA);
295 :
296 0 : std::lock_guard<std::mutex> guard(lock_);
297 0 : auto it = importAddrs_.find(devPtr);
298 0 : CHK_PRT_RET(
299 : it == importAddrs_.end(), HCCL_ERROR("[ZeroCopyAddressMgr][GetRemoteImportAddr] devPtr[%p] not import", devPtr),
300 : HCCL_E_PARA);
301 :
302 0 : void* handle = importAddrs_[devPtr];
303 0 : HCCL_INFO("[ZeroCopyAddressMgr][GetRemoteImportAddr] del devPtr[%p] handle[%p]", devPtr, handle);
304 0 : importAddrs_.erase(it);
305 :
306 0 : return HCCL_SUCCESS;
307 0 : }
308 :
309 0 : bool ZeroCopyAddressMgr::IsActivateCommMemoryAddr(void* startPtr, u64 length)
310 : {
311 0 : if (startPtr == nullptr || length == 0) {
312 0 : return false;
313 : }
314 :
315 0 : AddressRange range(startPtr, length);
316 :
317 0 : std::lock_guard<std::mutex> guard(lock_);
318 : // 我们先用最小区间去查找最前面匹配的valid内存块，后续的就依次遍历即可
319 0 : AddressRange litteRange(startPtr, 1);
320 0 : auto beginIt = validAddressRanges_.find(litteRange);
321 0 : while (beginIt != validAddressRanges_.end()) {
322 : // 此片内存已经是valid内存的子集了，那么是有效的
323 0 : if (range.end <= beginIt->end) {
324 0 : return true;
325 : }
326 :
327 : // 前一片内存已经匹配到了小块，把前面的内存切分掉，继续去匹配更后面的数据
328 0 : range.start = beginIt->end;
329 0 : beginIt++;
330 : }
331 :
332 : // 输入区间内有部分没有查找到，所以认为是无效的
333 0 : return false;
334 0 : }
335 :
336 0 : bool ZeroCopyAddressMgr::IsOverlapWithActivateAddr(void* startPtr, u64 length)
337 : {
338 0 : if (startPtr == nullptr || length == 0) {
339 0 : return false;
340 : }
341 :
342 0 : AddressRange range(startPtr, length);
343 0 : std::lock_guard<std::mutex> guard(lock_);
344 0 : return validAddressRanges_.find(range) != validAddressRanges_.end();
345 0 : }
346 :
347 0 : bool ZeroCopyAddressMgr::IsInSetAddressRange(u32 devicePhyId, void* startPtr, u64 length)
348 : {
349 0 : std::lock_guard<std::mutex> guard(lock_);
350 0 : auto& addrRange = reserveRanges_[devicePhyId];
351 :
352 : // 构造最小的数据块去寻找，如果没找到肯定没有交集
353 0 : AddressRange range(startPtr, 1);
354 0 : auto rangeIt = addrRange.find(range);
355 0 : if (rangeIt == addrRange.end()) {
356 0 : HCCL_INFO("[ZeroCopyAddressMgr][IsInSetAddressRange] not in reserve range");
357 0 : return false;
358 : }
359 :
360 : // 判断尾巴是否在当前匹配内存块中，如果不在那么不在范围内
361 0 : if (range.start + length > rangeIt->end) {
362 0 : HCCL_INFO("[ZeroCopyAddressMgr][IsInSetAddressRange] exceed reserve range");
363 0 : return false;
364 : }
365 :
366 0 : return true;
367 0 : }
368 :
369 0 : HcclResult ZeroCopyAddressMgr::ProcessRingBuffer(ZeroCopyRingBufferItem* ringBuffer, u32* head, u32* tail)
370 : {
371 0 : if (ringBuffer == nullptr || head == nullptr || tail == nullptr) {
372 0 : HCCL_ERROR(
373 : "[ZeroCopyAddressMgr][ProcessRingBuffer] invalid param ringBuff[%p] head[%p] tail[%p]", ringBuffer, head,
374 : tail);
375 0 : return HCCL_E_PARA;
376 : }
377 :
378 0 : std::lock_guard<std::mutex> guard(processRingBufferLock_);
379 0 : needPushOne = false;
380 0 : if (*head == *tail) {
381 0 : HCCL_INFO(
382 : "[ZeroCopyAddressMgr][ProcessRingBuffer] ring buffer is empty, so do nothing, head[%u] tail[%u]", *head,
383 : *tail);
384 0 : return HCCL_SUCCESS;
385 : }
386 :
387 0 : if (*tail >= ZERO_COPY_BUFFER_MAX_MAP_COUNT || *head >= ZERO_COPY_BUFFER_MAX_MAP_COUNT) {
388 0 : HCCL_ERROR("[ZeroCopyAddressMgr][ProcessRingBuffer] invalid head/tail, head[%u] tail[%u]", *head, *tail);
389 0 : return HCCL_E_PARA;
390 : }
391 :
392 0 : u32 now = *head;
393 0 : while (now != *tail) {
394 0 : HCCL_INFO(
395 : "[ZeroCopyAddressMgr][ProcessRingBuffer] process ringbuffer now[%u] ptr[%p] tail[%u] type[%d]", now,
396 : ringBuffer + now, *tail, ringBuffer[now].type);
397 0 : CHK_RET(ProcessOneAddrMap(ringBuffer[now]));
398 0 : now = (now + 1) % ZERO_COPY_BUFFER_MAX_MAP_COUNT;
399 : }
400 :
401 : // 更新所有的值
402 0 : *head = *tail;
403 0 : HCCL_INFO("[ZeroCopyAddressMgr][ProcessRingBuffer] ringbuffer head[%u] tail[%u]", *head, *tail);
404 0 : return HCCL_SUCCESS;
405 0 : }
406 :
407 0 : HcclResult ZeroCopyAddressMgr::ProcessOneAddrMap(const ZeroCopyRingBufferItem& item)
408 : {
409 0 : HCCL_INFO(
410 : "[ZeroCopyAddressMgr][ProcessOneAddrMap] Item info: type[%d] dev[%u] local[0x%lx] remote[0x%lx] len[%lu]",
411 : item.type, item.addr.devicePhyId, item.addr.localIpcAddr, item.addr.remoteAddr, item.addr.length);
412 0 : switch (item.type) {
413 0 : case ZeroCopyItemType::SET_MEMORY:
414 0 : return AddLocalIpc2RemoteAddr(
415 0 : item.addr.devicePhyId, reinterpret_cast<void*>(item.addr.localIpcAddr),
416 0 : reinterpret_cast<void*>(item.addr.remoteAddr), item.addr.length);
417 0 : case ZeroCopyItemType::UNSET_MEMORY:
418 0 : return DelLocalIpc2RemoteAddr(item.addr.devicePhyId, reinterpret_cast<void*>(item.addr.remoteAddr));
419 0 : case ZeroCopyItemType::ACTIVATE_MEMORY:
420 0 : return ActivateCommMemoryAddr(reinterpret_cast<void*>(item.addr.localIpcAddr), item.addr.length);
421 0 : case ZeroCopyItemType::DEACTIVATE_MEMORY:
422 0 : return DeactivateCommMemoryAddr(reinterpret_cast<void*>(item.addr.localIpcAddr));
423 0 : default:
424 0 : HCCL_ERROR("[ZeroCopyAddressMgr][ProcessOneAddrMap] invalid type[%d]", item.type);
425 0 : return HCCL_E_PARA;
426 : }
427 :
428 : return HCCL_SUCCESS;
429 : }
430 :
431 0 : u32 ZeroCopyAddressMgr::GetCommRefCnt() { return commRefCnt_; }
432 :
433 0 : HcclResult ZeroCopyAddressMgr::IncreCommRefCnt()
434 : {
435 0 : commRefCnt_++;
436 0 : return HCCL_SUCCESS;
437 : }
438 :
439 0 : HcclResult ZeroCopyAddressMgr::DecreCommRefCnt()
440 : {
441 0 : if (commRefCnt_ == 0) {
442 0 : HCCL_WARNING("[ZeroCopyAddressMgr][%s]commRefCnt_ is 0, cannot decrement", __func__);
443 0 : return HCCL_SUCCESS;
444 : }
445 0 : commRefCnt_--;
446 0 : return HCCL_SUCCESS;
447 : }
448 :
449 : } // namespace hccl
|
