Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 : #include "typical_sync_mem.h"
11 : #include "adapter_rts_common.h"
12 : #include "adapter_rts.h"
13 : #include "rdma_resource_manager.h"
14 :
15 : namespace hccl {
16 304 : TypicalSyncMem &TypicalSyncMem::GetInstance()
17 : {
18 370 : static TypicalSyncMem typicalSyncMem[MAX_MODULE_DEVICE_NUM + 1];
19 304 : s32 deviceLogicId = INVALID_INT;
20 304 : s32 ret = hrtGetDevice(&deviceLogicId);
21 304 : if (ret == HCCL_SUCCESS && (static_cast<u32>(deviceLogicId) < MAX_MODULE_DEVICE_NUM)) {
22 304 : HCCL_INFO("[TypicalSyncMem::GetInstance]deviceLogicID[%d]", deviceLogicId);
23 304 : return typicalSyncMem[deviceLogicId];
24 : }
25 0 : HCCL_WARNING("[TypicalSyncMem::GetInstance]deviceLogicID[%d] is invalid, ret[%d].", deviceLogicId, ret);
26 0 : return typicalSyncMem[MAX_MODULE_DEVICE_NUM];
27 : }
28 :
29 66 : TypicalSyncMem::TypicalSyncMem()
30 : {
31 66 : }
32 :
33 66 : TypicalSyncMem::~TypicalSyncMem()
34 : {
35 66 : (void)FreeAllSyncMem();
36 66 : (void)DeInitNotifySrcMem();
37 66 : }
38 :
39 30 : HcclResult TypicalSyncMem::InitNotifySrcMem()
40 : {
41 30 : CHK_RET(RdmaResourceManager::GetInstance().GetRdmaHandle(rdmaHandle_));
42 30 : CHK_PTR_NULL(rdmaHandle_);
43 30 : HCCL_DEBUG("[TypicalSyncMem][InitNotifySrcMem]start init notify source mem.");
44 30 : u32 notifyVaule = 1; // notify值写1表示record
45 30 : u32 notifySize = 0;
46 30 : CHK_RET(hrtGetNotifySize(notifySize));
47 :
48 30 : CHK_RET(DeviceMem::alloc(srcDevMem_, notifySize));
49 30 : HCCL_DEBUG("[TypicalSyncMem][InitNotifySrcMem]Create notify src buffer[%p], size[%u].",
50 : srcDevMem_.ptr(), notifySize);
51 :
52 30 : CHK_RET(hrtMemSyncCopy(srcDevMem_.ptr(), notifySize, ¬ifyVaule, notifySize, HcclRtMemcpyKind::HCCL_RT_MEMCPY_KIND_HOST_TO_DEVICE));
53 :
54 30 : notifySrcMrInfo_.addr = srcDevMem_.ptr();
55 30 : notifySrcMrInfo_.size = notifySize;
56 30 : notifySrcMrInfo_.access = RA_ACCESS_LOCAL_WRITE | RA_ACCESS_REMOTE_WRITE;
57 30 : notifySrcMrHandle_ = nullptr;
58 30 : CHK_RET(hrtRaRegGlobalMr(rdmaHandle_, notifySrcMrInfo_, notifySrcMrHandle_));
59 :
60 30 : HCCL_INFO("[TypicalSyncMem][InitNotifySrcMem]Init notifySrcMem_=%p success, mr lkey is [%u].",
61 : notifySrcMrInfo_.addr, notifySrcMrInfo_.lkey);
62 30 : return HCCL_SUCCESS;
63 : }
64 :
65 96 : HcclResult TypicalSyncMem::DeInitNotifySrcMem()
66 : {
67 96 : if (notifySrcMrHandle_ == nullptr) {
68 66 : HCCL_INFO("[TypicalSyncMem][InitNotifySrcMem] NotifySrcMem has been DeInit.");
69 66 : return HCCL_SUCCESS;
70 : }
71 30 : CHK_RET(RdmaResourceManager::GetInstance().GetRdmaHandle(rdmaHandle_));
72 30 : CHK_PTR_NULL(rdmaHandle_);
73 30 : HCCL_INFO("[TypicalSyncMem][InitNotifySrcMem] DeRegister notifySrcMem_=%p.", notifySrcMrInfo_.addr);
74 30 : CHK_RET(hrtRaDeRegGlobalMr(rdmaHandle_, notifySrcMrHandle_));
75 30 : notifySrcMrHandle_ = nullptr;
76 30 : return HCCL_SUCCESS;
77 : }
78 :
79 90 : HcclResult TypicalSyncMem::AllocSyncMem(int32_t **ptr)
80 : {
81 90 : HCCL_DEBUG("[TypicalSyncMem][AllocSyncMem]start alloc sync mem on [%p].", ptr);
82 90 : std::unique_lock<std::mutex> lockSyncMemMap(syncMemMapMutex_);
83 90 : if (syncMemMap_.empty()) {
84 30 : HCCL_INFO("[TypicalSyncMem][AllocSyncMem] syncMem has not inited. Start to init notify src mem.");
85 30 : CHK_RET(InitNotifySrcMem());
86 : }
87 90 : CHK_PTR_NULL(ptr);
88 90 : CHK_RET(RdmaResourceManager::GetInstance().GetRdmaHandle(rdmaHandle_));
89 90 : CHK_PTR_NULL(rdmaHandle_);
90 :
91 90 : u64 offset = 0;
92 90 : u64 notifyBaseVa = 0;
93 90 : u64 notifyTotalSize = 0;
94 :
95 : // Create an empty notify and get it's handle
96 90 : HcclRtSignal notify = nullptr;
97 90 : CHK_RET(CreateEmptyNotify(notify));
98 90 : HCCL_DEBUG("[TypicalSyncMem][AllocSyncMem]create an empty notify success.");
99 :
100 : // Get the base virtual address and the size of notify register.
101 90 : u64 notifyBaseVaTmp = 0;
102 90 : notifyBaseVaTmp = notifyBaseVa;
103 90 : CHK_RET(HrtRaGetNotifyBaseAddr(rdmaHandle_, ¬ifyBaseVa, ¬ifyTotalSize));
104 :
105 90 : CHK_PRT_RET(((notifyBaseVaTmp != 0) && (notifyBaseVaTmp != notifyBaseVa)),
106 : HCCL_ERROR("[TypicalSyncMem][AllocSyncMem]get base addr failed, notify base va has changed."),
107 : HCCL_E_INTERNAL);
108 :
109 : // Get the offset to the base address for the created notify,
110 : // which is same for both physical address and virtual address.
111 : // Here we use physical address to calculate the offset.
112 90 : CHK_RET(hrtNotifyGetOffset(notify, offset));
113 :
114 : // notify寄存器的虚拟地址与物理地址偏移相同，所以虚拟地址为虚拟基地址加偏移
115 90 : u64 notifyVa = notifyBaseVa + offset;
116 :
117 90 : HCCL_INFO("[TypicalSyncMem][AllocSyncMem]notifyBaseVa=0x%llx," \
118 : "notifyTotalSize=0x%x, offset=0x%llx, notifyVa=0x%llx notify=%p.",
119 : notifyBaseVa, notifyTotalSize, offset, notifyVa, notify);
120 : // Store the notifyVa to set
121 90 : syncMemMap_[notifyVa] = notify;
122 : // Assign the notify virtual address to *ptr.
123 90 : *ptr = reinterpret_cast<int32_t *>(static_cast<uintptr_t>(notifyVa));
124 90 : HCCL_RUN_INFO("[TypicalSyncMem][AllocSyncMem]alloc an empty sync mem success, notifyVa[%p]. " \
125 : "please register mr before use.", *ptr);
126 90 : return HCCL_SUCCESS;
127 90 : }
128 :
129 90 : HcclResult TypicalSyncMem::FreeSyncMem(int32_t *ptr)
130 : {
131 90 : HCCL_DEBUG("[TypicalSyncMem][FreeSyncMem]start free sync mem[%p], please deregister mr before free.", ptr);
132 90 : CHK_PTR_NULL(ptr);
133 90 : u64 notifyVa = reinterpret_cast<uintptr_t>(ptr);
134 90 : std::unique_lock<std::mutex> lockSyncMemMap(syncMemMapMutex_);
135 90 : auto smIter = syncMemMap_.find(notifyVa);
136 90 : if (smIter == syncMemMap_.end()) {
137 0 : HCCL_WARNING("[TypicalSyncMem][FreeSyncMem]No notifyVa match the given ptr[%p] in sync mem map.", ptr);
138 0 : return HCCL_SUCCESS;
139 : }
140 90 : CHK_RET(DestroyNotify(syncMemMap_[notifyVa]));
141 90 : syncMemMap_.erase(smIter);
142 90 : if (syncMemMap_.empty()) {
143 30 : HCCL_INFO("[TypicalSyncMem][FreeSyncMem] syncMem all deinit. Start to deinit notify src mem.");
144 30 : CHK_RET(DeInitNotifySrcMem());
145 : }
146 90 : HCCL_INFO("[TypicalSyncMem][FreeSyncMem] Free [%p] success.", ptr);
147 90 : return HCCL_SUCCESS;
148 90 : }
149 :
150 86 : HcclResult TypicalSyncMem::GetNotifyHandle(u64 notifyVa, HcclRtNotify ¬ifyHandle)
151 : {
152 86 : std::unique_lock<std::mutex> lockSyncMemMap(syncMemMapMutex_);
153 86 : auto smIter = syncMemMap_.find(notifyVa);
154 86 : if (smIter != syncMemMap_.end()) {
155 86 : notifyHandle = smIter->second;
156 86 : return HCCL_SUCCESS;
157 : }
158 0 : HCCL_ERROR("[TypicalSyncMem][GetNotifyHandle]invalid notifyVa[%llu].", notifyVa);
159 0 : return HCCL_E_PARA;
160 86 : }
161 :
162 38 : HcclResult TypicalSyncMem::GetNotifySrcMem(struct MrInfoT &mrInfo)
163 : {
164 38 : CHK_PTR_NULL(notifySrcMrInfo_.addr);
165 38 : mrInfo.addr = notifySrcMrInfo_.addr;
166 38 : mrInfo.size = notifySrcMrInfo_.size;
167 38 : mrInfo.access = notifySrcMrInfo_.access;
168 38 : mrInfo.lkey = notifySrcMrInfo_.lkey;
169 38 : return HCCL_SUCCESS;
170 : }
171 :
172 90 : HcclResult TypicalSyncMem::CreateEmptyNotify(HcclRtNotify ¬ifyHandle)
173 : {
174 90 : s32 deviceId = 0;
175 90 : CHK_RET(hrtGetDevice(&deviceId));
176 90 : HcclResult ret = hrtNotifyCreate(deviceId, ¬ifyHandle);
177 90 : CHK_PRT_RET(ret != HCCL_SUCCESS,
178 : HCCL_ERROR("[TypicalSyncMem][CreateNotify]errNo[0x%016llx] Notify create failed. return[%d], deviceLogicId[%d]",
179 : HCCL_ERROR_CODE(HCCL_E_RUNTIME), ret, deviceId), HCCL_E_RUNTIME);
180 90 : CHK_PRT_RET(notifyHandle == nullptr,
181 : HCCL_ERROR("[TypicalSyncMem][CreateNotify]errNo[0x%016llx] Notify create failed. notifyHandle is NULL",
182 : HCCL_ERROR_CODE(HCCL_E_RUNTIME)), HCCL_E_RUNTIME);
183 :
184 90 : HCCL_INFO("[TypicalSyncMem][CreateNotify]create notify success, deviceId[%d], notify handle[%p].",
185 : deviceId, notifyHandle);
186 90 : return HCCL_SUCCESS;
187 : }
188 :
189 90 : HcclResult TypicalSyncMem::DestroyNotify(HcclRtNotify notifyHandle)
190 : {
191 90 : HCCL_DEBUG("[TypicalSyncMem][DestroyNotify]start destroy notify[%p].", notifyHandle);
192 90 : CHK_PTR_NULL(notifyHandle);
193 90 : HcclResult ret = hrtNotifyDestroy(notifyHandle);
194 90 : CHK_PRT_RET(ret != RT_ERROR_NONE,
195 : HCCL_ERROR("[TypicalSyncMem][DestroyNotify]errNo[0x%016llx] rt notify destroy fail, return[%d].",
196 : HCCL_ERROR_CODE(HCCL_E_RUNTIME), ret), HCCL_E_RUNTIME);
197 90 : HCCL_INFO("[TypicalSyncMem][DestroyNotify]destroy notify success.");
198 90 : return HCCL_SUCCESS;
199 : }
200 :
201 66 : HcclResult TypicalSyncMem::FreeAllSyncMem()
202 : {
203 66 : std::unique_lock<std::mutex> lockSyncMemMap(syncMemMapMutex_);
204 66 : if (!syncMemMap_.empty()) {
205 0 : for (auto &smIter : syncMemMap_) {
206 0 : if (smIter.second != nullptr) {
207 0 : CHK_RET(DestroyNotify(smIter.second));
208 : }
209 : }
210 0 : syncMemMap_.clear();
211 : }
212 66 : HCCL_INFO("[TypicalSyncMem][FreeAllSyncMem]free all sync memory success.");
213 66 : return HCCL_SUCCESS;
214 66 : }
215 : } // namespace hccl
|
