Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "zero_copy_address_mgr.h"
12 :
13 : namespace hccl {
14 :
15 0 : HcclResult ZeroCopyAddressMgr::SetMemoryRange(u32 devicePhyId, void *baseAddr, u64 length)
16 : {
17 0 : if (baseAddr == nullptr || length == 0) {
18 0 : HCCL_ERROR("[ZeroCopyAddressMgr][SetMemoryRange] invalid input params addr[%p] len[%lu]", baseAddr, length);
19 0 : return HCCL_E_PARA;
20 : }
21 :
22 0 : CHK_PRT_RET(AddLocalIpc2RemoteAddr(devicePhyId, baseAddr, baseAddr, length),
23 : HCCL_ERROR("[ZeroCopyAddressMgr][SetMemoryRange] dev[%u] set addr [%p] failed", devicePhyId, baseAddr),
24 : HCCL_E_PARA);
25 :
26 0 : return HCCL_SUCCESS;
27 : }
28 :
29 0 : HcclResult ZeroCopyAddressMgr::UnsetMemoryRange(u32 devicePhyId, void *baseAddr)
30 : {
31 0 : if (baseAddr == nullptr) {
32 0 : HCCL_ERROR("[ZeroCopyAddressMgr][UnsetMemoryRange] invalid input params");
33 0 : return HCCL_E_PARA;
34 : }
35 :
36 0 : CHK_PRT_RET(DelLocalIpc2RemoteAddr(devicePhyId, baseAddr),
37 : HCCL_ERROR("[ZeroCopyAddressMgr][UnsetMemoryRange] dev[%u] unset [%p] addr failed", devicePhyId, baseAddr),
38 : HCCL_E_PARA);
39 :
40 0 : return HCCL_SUCCESS;
41 : }
42 :
43 0 : bool ZeroCopyAddressMgr::IsAddressSet(u32 devicePhyId, void *baseAddr)
44 : {
45 0 : std::lock_guard<std::mutex> guard(lock_);
46 0 : auto &addrMapping = reserveAddrMappings_[devicePhyId];
47 0 : return addrMapping.find(baseAddr) != addrMapping.end();
48 0 : }
49 :
50 0 : HcclResult ZeroCopyAddressMgr::AddLocalIpc2RemoteAddr(u32 devicePhyId, void *localIpcBase, void *remoteAddrBase, u64 length)
51 : {
52 : u32 maxDeviceNum;
53 0 : CHK_RET(GetMaxDevNum(maxDeviceNum));
54 0 : if (devicePhyId >= maxDeviceNum || localIpcBase == nullptr || remoteAddrBase == nullptr) {
55 0 : HCCL_ERROR("[ZeroCopyAddressMgr][AddLocalIpc2RemoteAddr] devPhyId [%u] localIpc[%p] remoteAddr[%p] invalid params",
56 : devicePhyId, localIpcBase, remoteAddrBase);
57 0 : return HCCL_E_PARA;
58 : }
59 :
60 0 : ZeroCopyRingBufferItem item;
61 : {
62 0 : std::lock_guard<std::mutex> guard(lock_);
63 0 : auto &addrMapping = reserveAddrMappings_[devicePhyId];
64 0 : auto &addrRange = reserveRanges_[devicePhyId];
65 :
66 : // 检查地址是否已经reserve过
67 0 : CHK_PRT_RET(addrMapping.find(remoteAddrBase) != addrMapping.end(),
68 : HCCL_ERROR("[ZeroCopyAddressMgr][AddLocalIpc2RemoteAddr] dev[%u] remote addr %p had set", devicePhyId, remoteAddrBase), HCCL_E_PARA);
69 :
70 : // 检查地址reserve的地址区间是否与之前的有交叠
71 0 : AddressRange range(remoteAddrBase, length);
72 0 : CHK_PRT_RET(addrRange.find(range) != addrRange.end(),
73 : HCCL_ERROR("[ZeroCopyAddressMgr][AddLocalIpc2RemoteAddr] dev[%u] remote addr %p had set with overlap range",
74 : devicePhyId, remoteAddrBase), HCCL_E_PARA);
75 :
76 0 : item.type = ZeroCopyItemType::SET_MEMORY;
77 0 : item.addr.devicePhyId = devicePhyId;
78 0 : item.addr.localIpcAddr = reinterpret_cast<u64>(localIpcBase);
79 0 : item.addr.remoteAddr = reinterpret_cast<u64>(remoteAddrBase);
80 0 : item.addr.length = length;
81 0 : addrMapping.insert({remoteAddrBase, item.addr});
82 0 : addrRange.insert(range);
83 0 : }
84 0 : CHK_RET(PushOne(item));
85 0 : HCCL_INFO("[ZeroCopyAddressMgr][AddLocalIpc2RemoteAddr] dev[%u] add set localIpc[%p] remote[%p] length[%lu]",
86 : devicePhyId, localIpcBase, remoteAddrBase, length);
87 0 : return HCCL_SUCCESS;
88 : }
89 :
90 0 : HcclResult ZeroCopyAddressMgr::DelLocalIpc2RemoteAddr(u32 devicePhyId, void *remoteAddrBase)
91 : {
92 : u32 maxDeviceNum;
93 0 : CHK_RET(GetMaxDevNum(maxDeviceNum));
94 0 : if (devicePhyId >= maxDeviceNum || remoteAddrBase == nullptr) {
95 0 : HCCL_ERROR("[ZeroCopyAddressMgr][DelLocalIpc2RemoteAddr] devPhyId [%u] invalid params", devicePhyId);
96 0 : return HCCL_E_PARA;
97 : }
98 :
99 : u64 length;
100 0 : ZeroCopyRingBufferItem item;
101 : {
102 0 : std::lock_guard<std::mutex> guard(lock_);
103 0 : auto &addrMapping = reserveAddrMappings_[devicePhyId];
104 0 : auto &addrRange = reserveRanges_[devicePhyId];
105 :
106 : // 检查地址是否已经reserve过
107 0 : auto mappingIt = addrMapping.find(remoteAddrBase);
108 0 : CHK_PRT_RET(mappingIt == addrMapping.end(),
109 : HCCL_ERROR("[ZeroCopyAddressMgr][DelLocalIpc2RemoteAddr] dev[%u] addr %p not set", devicePhyId, remoteAddrBase), HCCL_E_PARA);
110 :
111 0 : length = mappingIt->second.length;
112 0 : AddressRange range(remoteAddrBase, length);
113 0 : auto rangeIt = addrRange.find(range);
114 0 : CHK_PRT_RET(rangeIt == addrRange.end(),
115 : HCCL_ERROR("[ZeroCopyAddressMgr][DelLocalIpc2RemoteAddr] dev[%u] addr %p not set", devicePhyId, remoteAddrBase), HCCL_E_PARA);
116 :
117 : // 检查是否仍存在Activate的内存
118 0 : AddressRange localRange(mappingIt->second.localIpcAddr, length);
119 0 : auto activateIt = validAddressRanges_.find(localRange);
120 0 : CHK_PRT_RET(activateIt != validAddressRanges_.end(),
121 : HCCL_ERROR("[ZeroCopyAddressMgr][DelLocalIpc2RemoteAddr] dev[%u] remoteAddr %p localAddr 0x%lx still have activate memory [0x%lx, 0x%lx)",
122 : devicePhyId, remoteAddrBase, mappingIt->second.localIpcAddr, activateIt->start, activateIt->end), HCCL_E_PARA);
123 :
124 0 : item.type = ZeroCopyItemType::UNSET_MEMORY;
125 0 : item.addr = mappingIt->second;
126 0 : addrMapping.erase(mappingIt);
127 0 : addrRange.erase(rangeIt);
128 0 : }
129 0 : CHK_RET(PushOne(item));
130 0 : HCCL_INFO("[ZeroCopyAddressMgr][DelLocalIpc2RemoteAddr] dev[%u] del set localIpc[0x%lx] remote[0x%lx] length[%lu]",
131 : devicePhyId, item.addr.localIpcAddr, item.addr.remoteAddr, length);
132 0 : return HCCL_SUCCESS;
133 : }
134 :
135 0 : HcclResult ZeroCopyAddressMgr::GetLocalIpc2RemoteAddr(u32 devicePhyId, void *remoteAddr, LocalIpc2RemoteAddr &addr)
136 : {
137 : u32 maxDeviceNum;
138 0 : CHK_RET(GetMaxDevNum(maxDeviceNum));
139 0 : if (devicePhyId >= maxDeviceNum || remoteAddr == nullptr) {
140 0 : HCCL_ERROR("[ZeroCopyAddressMgr][GetLocalIpc2RemoteAddr] devPhyId [%u] invalid params", devicePhyId);
141 0 : return HCCL_E_PARA;
142 : }
143 :
144 0 : std::lock_guard<std::mutex> guard(lock_);
145 0 : auto &addrMapping = reserveAddrMappings_[devicePhyId];
146 0 : auto &addrRange = reserveRanges_[devicePhyId];
147 :
148 0 : AddressRange range(remoteAddr, 1);
149 0 : auto rangeIt = addrRange.find(range);
150 0 : CHK_PRT_RET(rangeIt == addrRange.end(),
151 : HCCL_ERROR("[ZeroCopyAddressMgr][GetLocalIpc2RemoteAddr] dev[%u] addr %p not set", devicePhyId, remoteAddr), HCCL_E_PARA);
152 :
153 0 : void *remoteAddrBase = reinterpret_cast<void *>(rangeIt->start);
154 0 : auto mapIt = addrMapping.find(remoteAddrBase);
155 0 : CHK_PRT_RET(rangeIt == addrRange.end(),
156 : HCCL_ERROR("[ZeroCopyAddressMgr][GetLocalIpc2RemoteAddr] dev[%u] addr %p not set", devicePhyId, remoteAddr), HCCL_E_PARA);
157 :
158 0 : addr = mapIt->second;
159 :
160 0 : return HCCL_SUCCESS;
161 0 : }
162 :
163 0 : HcclResult ZeroCopyAddressMgr::ActivateCommMemoryAddr(void *startPtr, u64 length)
164 : {
165 0 : CHK_PRT_RET((startPtr == nullptr || length == 0),
166 : HCCL_ERROR("[ZeroCopyAddressMgr][ActivateCommMemoryAddr] Invalid params"), HCCL_E_PARA);
167 :
168 0 : AddressRange range(startPtr, length);
169 :
170 : {
171 0 : std::lock_guard<std::mutex> guard(lock_);
172 0 : auto it = validAddressRanges_.find(range);
173 0 : CHK_PRT_RET((it != validAddressRanges_.end()),
174 : HCCL_ERROR("[ZeroCopyAddressMgr][ActivateCommMemoryAddr] overlap address exist:[0x%lx, 0x%lx) valid:[0x%lx, 0x%lx)",
175 : it->start, it->end, range.start, range.end), HCCL_E_PARA);
176 :
177 0 : validAddressRanges_.insert(range);
178 0 : }
179 0 : ZeroCopyRingBufferItem item;
180 0 : item.type = ZeroCopyItemType::ACTIVATE_MEMORY;
181 0 : item.addr.localIpcAddr = reinterpret_cast<u64>(startPtr);
182 0 : item.addr.length = length;
183 0 : CHK_RET(PushOne(item));
184 :
185 0 : HCCL_INFO("[ZeroCopyAddressMgr][ActivateCommMemoryAddr] activate address [0x%lx, 0x%lx) success", range.start, range.end);
186 0 : return HCCL_SUCCESS;
187 : }
188 :
189 0 : HcclResult ZeroCopyAddressMgr::DeactivateCommMemoryAddr(void *startPtr)
190 : {
191 0 : CHK_PRT_RET((startPtr == nullptr),
192 : HCCL_ERROR("[ZeroCopyAddressMgr][DeactivateCommMemoryAddr] Invalid params"), HCCL_E_PARA);
193 :
194 : // 我们构造一个最小的交叠区间去做比较
195 0 : u64 litteLen = 1;
196 0 : AddressRange range(startPtr, litteLen);
197 :
198 : {
199 0 : std::lock_guard<std::mutex> guard(lock_);
200 0 : auto it = validAddressRanges_.find(range);
201 0 : CHK_PRT_RET((it == validAddressRanges_.end() || it->start != range.start),
202 : HCCL_ERROR("[ZeroCopyAddressMgr][DeactivateCommMemoryAddr] address %p is not activate", startPtr), HCCL_E_PARA);
203 :
204 0 : HCCL_INFO("[ZeroCopyAddressMgr][DeactivateCommMemoryAddr] deactivate address [0x%lx, 0x%lx) success", it->start, it->end);
205 0 : validAddressRanges_.erase(it);
206 0 : }
207 :
208 0 : ZeroCopyRingBufferItem item;
209 0 : item.type = ZeroCopyItemType::DEACTIVATE_MEMORY;
210 0 : item.addr.localIpcAddr = reinterpret_cast<u64>(startPtr);
211 0 : CHK_RET(PushOne(item));
212 :
213 0 : return HCCL_SUCCESS;
214 : }
215 :
216 0 : HcclResult ZeroCopyAddressMgr::AddRemoteImportAddr(void *devPtr, void *handle)
217 : {
218 0 : CHK_PRT_RET((devPtr == nullptr || handle == nullptr),
219 : HCCL_ERROR("[ZeroCopyAddressMgr][AddRemoteImportAddr] invalid devPtr[%p] handle[%p]",
220 : devPtr, handle), HCCL_E_PARA);
221 :
222 0 : std::lock_guard<std::mutex> guard(lock_);
223 0 : auto it = importAddrs_.find(devPtr);
224 0 : CHK_PRT_RET(it != importAddrs_.end(),
225 : HCCL_ERROR("[ZeroCopyAddressMgr][AddRemoteImportAddr] devPtr[%p] has import", devPtr), HCCL_E_PARA);
226 :
227 0 : HCCL_INFO("[ZeroCopyAddressMgr][AddRemoteImportAddr] add devPtr[%p] handle[%p]", devPtr, handle);
228 0 : importAddrs_.insert({devPtr, handle});
229 0 : return HCCL_SUCCESS;
230 0 : }
231 :
232 0 : HcclResult ZeroCopyAddressMgr::GetRemoteImportAddr(void *devPtr, void *&handle)
233 : {
234 0 : CHK_PRT_RET((devPtr == nullptr),
235 : HCCL_ERROR("[ZeroCopyAddressMgr][GetRemoteImportAddr] invalid devPtr[%p]", devPtr), HCCL_E_PARA);
236 :
237 0 : std::lock_guard<std::mutex> guard(lock_);
238 0 : auto it = importAddrs_.find(devPtr);
239 0 : CHK_PRT_RET(it == importAddrs_.end(),
240 : HCCL_ERROR("[ZeroCopyAddressMgr][GetRemoteImportAddr] devPtr[%p] not import", devPtr), HCCL_E_PARA);
241 :
242 0 : handle = importAddrs_[devPtr];
243 0 : HCCL_INFO("[ZeroCopyAddressMgr][GetRemoteImportAddr] get devPtr[%p] handle[%p]", devPtr, handle);
244 :
245 0 : return HCCL_SUCCESS;
246 0 : }
247 :
248 0 : HcclResult ZeroCopyAddressMgr::DelRemoteImportAddr(void *devPtr)
249 : {
250 0 : CHK_PRT_RET((devPtr == nullptr),
251 : HCCL_ERROR("[ZeroCopyAddressMgr][DelRemoteImportAddr] invalid devPtr[%p]", devPtr), HCCL_E_PARA);
252 :
253 0 : std::lock_guard<std::mutex> guard(lock_);
254 0 : auto it = importAddrs_.find(devPtr);
255 0 : CHK_PRT_RET(it == importAddrs_.end(),
256 : HCCL_ERROR("[ZeroCopyAddressMgr][GetRemoteImportAddr] devPtr[%p] not import", devPtr), HCCL_E_PARA);
257 :
258 0 : void *handle = importAddrs_[devPtr];
259 0 : HCCL_INFO("[ZeroCopyAddressMgr][GetRemoteImportAddr] del devPtr[%p] handle[%p]", devPtr, handle);
260 0 : importAddrs_.erase(it);
261 :
262 0 : return HCCL_SUCCESS;
263 0 : }
264 :
265 0 : bool ZeroCopyAddressMgr::IsActivateCommMemoryAddr(void *startPtr, u64 length)
266 : {
267 0 : if (startPtr == nullptr || length == 0) {
268 0 : return false;
269 : }
270 :
271 0 : AddressRange range(startPtr, length);
272 :
273 0 : std::lock_guard<std::mutex> guard(lock_);
274 : // 我们先用最小区间去查找最前面匹配的valid内存块，后续的就依次遍历即可
275 0 : AddressRange litteRange(startPtr, 1);
276 0 : auto beginIt = validAddressRanges_.find(litteRange);
277 0 : while (beginIt != validAddressRanges_.end()) {
278 : // 此片内存已经是valid内存的子集了，那么是有效的
279 0 : if (range.end <= beginIt->end) {
280 0 : return true;
281 : }
282 :
283 : // 前一片内存已经匹配到了小块，把前面的内存切分掉，继续去匹配更后面的数据
284 0 : range.start = beginIt->end;
285 0 : beginIt++;
286 : }
287 :
288 : // 输入区间内有部分没有查找到，所以认为是无效的
289 0 : return false;
290 0 : }
291 :
292 0 : bool ZeroCopyAddressMgr::IsOverlapWithActivateAddr(void *startPtr, u64 length)
293 : {
294 0 : if (startPtr == nullptr || length == 0) {
295 0 : return false;
296 : }
297 :
298 0 : AddressRange range(startPtr, length);
299 0 : std::lock_guard<std::mutex> guard(lock_);
300 0 : return validAddressRanges_.find(range) != validAddressRanges_.end();
301 0 : }
302 :
303 0 : bool ZeroCopyAddressMgr::IsInSetAddressRange(u32 devicePhyId, void *startPtr, u64 length)
304 : {
305 0 : std::lock_guard<std::mutex> guard(lock_);
306 0 : auto &addrRange = reserveRanges_[devicePhyId];
307 :
308 : // 构造最小的数据块去寻找，如果没找到肯定没有交集
309 0 : AddressRange range(startPtr, 1);
310 0 : auto rangeIt = addrRange.find(range);
311 0 : if (rangeIt == addrRange.end()) {
312 0 : HCCL_INFO("[ZeroCopyAddressMgr][IsInSetAddressRange] not in reserve range");
313 0 : return false;
314 : }
315 :
316 : // 判断尾巴是否在当前匹配内存块中，如果不在那么不在范围内
317 0 : if (range.start + length > rangeIt->end) {
318 0 : HCCL_INFO("[ZeroCopyAddressMgr][IsInSetAddressRange] exceed reserve range");
319 0 : return false;
320 : }
321 :
322 0 : return true;
323 0 : }
324 :
325 0 : HcclResult ZeroCopyAddressMgr::ProcessRingBuffer(ZeroCopyRingBufferItem *ringBuffer, u32 *head, u32 *tail)
326 : {
327 0 : if (ringBuffer == nullptr || head == nullptr || tail == nullptr) {
328 0 : HCCL_ERROR("[ZeroCopyAddressMgr][ProcessRingBuffer] invalid param ringBuff[%p] head[%p] tail[%p]",
329 : ringBuffer, head, tail);
330 0 : return HCCL_E_PARA;
331 : }
332 :
333 0 : std::lock_guard<std::mutex> guard(processRingBufferLock_);
334 0 : needPushOne = false;
335 0 : if (*head == *tail) {
336 0 : HCCL_INFO("[ZeroCopyAddressMgr][ProcessRingBuffer] ring buffer is empty, so do nothing, head[%u] tail[%u]", *head, *tail);
337 0 : return HCCL_SUCCESS;
338 : }
339 :
340 0 : if (*tail >= ZERO_COPY_BUFFER_MAX_MAP_COUNT || *head >= ZERO_COPY_BUFFER_MAX_MAP_COUNT) {
341 0 : HCCL_ERROR("[ZeroCopyAddressMgr][ProcessRingBuffer] invalid head/tail, head[%u] tail[%u]", *head, *tail);
342 0 : return HCCL_E_PARA;
343 : }
344 :
345 0 : u32 now = *head;
346 0 : while (now != *tail) {
347 0 : HCCL_INFO("[ZeroCopyAddressMgr][ProcessRingBuffer] process ringbuffer now[%u] ptr[%p] tail[%u] type[%d]",
348 : now, ringBuffer + now, *tail, ringBuffer[now].type);
349 0 : CHK_RET(ProcessOneAddrMap(ringBuffer[now]));
350 0 : now = (now + 1) % ZERO_COPY_BUFFER_MAX_MAP_COUNT;
351 : }
352 :
353 : // 更新所有的值
354 0 : *head = *tail;
355 0 : HCCL_INFO("[ZeroCopyAddressMgr][ProcessRingBuffer] ringbuffer head[%u] tail[%u]", *head, *tail);
356 0 : return HCCL_SUCCESS;
357 0 : }
358 :
359 0 : HcclResult ZeroCopyAddressMgr::ProcessOneAddrMap(const ZeroCopyRingBufferItem &item)
360 : {
361 0 : HCCL_INFO("[ZeroCopyAddressMgr][ProcessOneAddrMap] Item info: type[%d] dev[%u] local[0x%lx] remote[0x%lx] len[%lu]",
362 : item.type, item.addr.devicePhyId, item.addr.localIpcAddr, item.addr.remoteAddr, item.addr.length);
363 0 : switch (item.type) {
364 0 : case ZeroCopyItemType::SET_MEMORY:
365 0 : return AddLocalIpc2RemoteAddr(item.addr.devicePhyId, reinterpret_cast<void *>(item.addr.localIpcAddr),
366 0 : reinterpret_cast<void *>(item.addr.remoteAddr), item.addr.length);
367 0 : case ZeroCopyItemType::UNSET_MEMORY:
368 0 : return DelLocalIpc2RemoteAddr(item.addr.devicePhyId, reinterpret_cast<void *>(item.addr.remoteAddr));
369 0 : case ZeroCopyItemType::ACTIVATE_MEMORY:
370 0 : return ActivateCommMemoryAddr(reinterpret_cast<void *>(item.addr.localIpcAddr), item.addr.length);
371 0 : case ZeroCopyItemType::DEACTIVATE_MEMORY:
372 0 : return DeactivateCommMemoryAddr(reinterpret_cast<void *>(item.addr.localIpcAddr));
373 0 : default:
374 0 : HCCL_ERROR("[ZeroCopyAddressMgr][ProcessOneAddrMap] invalid type[%d]", item.type);
375 0 : return HCCL_E_PARA;
376 : }
377 :
378 : return HCCL_SUCCESS;
379 : }
380 :
381 0 : u32 ZeroCopyAddressMgr::GetCommRefCnt()
382 : {
383 0 : return commRefCnt_;
384 : }
385 :
386 0 : HcclResult ZeroCopyAddressMgr::IncreCommRefCnt()
387 : {
388 0 : commRefCnt_++;
389 0 : return HCCL_SUCCESS;
390 : }
391 :
392 0 : HcclResult ZeroCopyAddressMgr::DecreCommRefCnt()
393 : {
394 0 : if (commRefCnt_ == 0) {
395 0 : HCCL_WARNING("[ZeroCopyAddressMgr][%s]commRefCnt_ is 0, cannot decrement", __func__);
396 0 : return HCCL_SUCCESS;
397 : }
398 0 : commRefCnt_--;
399 0 : return HCCL_SUCCESS;
400 : }
401 :
402 : }
|
