Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "typical_sync_mem.h"
12 : #include "adapter_rts_common.h"
13 : #include "adapter_rts.h"
14 : #include "rdma_resource_manager.h"
15 :
16 : namespace hccl {
17 304 : TypicalSyncMem& TypicalSyncMem::GetInstance()
18 : {
19 370 : static TypicalSyncMem typicalSyncMem[MAX_MODULE_DEVICE_NUM + 1];
20 304 : s32 deviceLogicId = INVALID_INT;
21 304 : s32 ret = hrtGetDevice(&deviceLogicId);
22 304 : if (ret == HCCL_SUCCESS && (static_cast<u32>(deviceLogicId) < MAX_MODULE_DEVICE_NUM)) {
23 304 : HCCL_INFO("[TypicalSyncMem::GetInstance]deviceLogicID[%d]", deviceLogicId);
24 304 : return typicalSyncMem[deviceLogicId];
25 : }
26 0 : HCCL_WARNING("[TypicalSyncMem::GetInstance]deviceLogicID[%d] is invalid, ret[%d].", deviceLogicId, ret);
27 0 : return typicalSyncMem[MAX_MODULE_DEVICE_NUM];
28 : }
29 :
30 66 : TypicalSyncMem::TypicalSyncMem() {}
31 :
32 66 : TypicalSyncMem::~TypicalSyncMem()
33 : {
34 66 : (void)FreeAllSyncMem();
35 66 : (void)DeInitNotifySrcMem();
36 66 : }
37 :
38 30 : HcclResult TypicalSyncMem::InitNotifySrcMem()
39 : {
40 30 : CHK_RET(RdmaResourceManager::GetInstance().GetRdmaHandle(rdmaHandle_));
41 30 : CHK_PTR_NULL(rdmaHandle_);
42 30 : HCCL_DEBUG("[TypicalSyncMem][InitNotifySrcMem]start init notify source mem.");
43 30 : u32 notifyVaule = 1; // notify值写1表示record
44 30 : u32 notifySize = 0;
45 30 : CHK_RET(hrtGetNotifySize(notifySize));
46 :
47 30 : CHK_RET(DeviceMem::alloc(srcDevMem_, notifySize));
48 30 : HCCL_DEBUG(
49 : "[TypicalSyncMem][InitNotifySrcMem]Create notify src buffer[%p], size[%u].", srcDevMem_.ptr(), notifySize);
50 :
51 30 : CHK_RET(hrtMemSyncCopy(
52 : srcDevMem_.ptr(), notifySize, ¬ifyVaule, notifySize, HcclRtMemcpyKind::HCCL_RT_MEMCPY_KIND_HOST_TO_DEVICE));
53 :
54 30 : notifySrcMrInfo_.addr = srcDevMem_.ptr();
55 30 : notifySrcMrInfo_.size = notifySize;
56 30 : notifySrcMrInfo_.access = RA_ACCESS_LOCAL_WRITE | RA_ACCESS_REMOTE_WRITE;
57 30 : notifySrcMrHandle_ = nullptr;
58 30 : CHK_RET(hrtRaRegGlobalMr(rdmaHandle_, notifySrcMrInfo_, notifySrcMrHandle_));
59 :
60 30 : HCCL_INFO(
61 : "[TypicalSyncMem][InitNotifySrcMem]Init notifySrcMem_=%p success, mr lkey is [%u].", notifySrcMrInfo_.addr,
62 : notifySrcMrInfo_.lkey);
63 30 : return HCCL_SUCCESS;
64 : }
65 :
66 96 : HcclResult TypicalSyncMem::DeInitNotifySrcMem()
67 : {
68 96 : if (notifySrcMrHandle_ == nullptr) {
69 66 : HCCL_INFO("[TypicalSyncMem][InitNotifySrcMem] NotifySrcMem has been DeInit.");
70 66 : return HCCL_SUCCESS;
71 : }
72 30 : CHK_RET(RdmaResourceManager::GetInstance().GetRdmaHandle(rdmaHandle_));
73 30 : CHK_PTR_NULL(rdmaHandle_);
74 30 : HCCL_INFO("[TypicalSyncMem][InitNotifySrcMem] DeRegister notifySrcMem_=%p.", notifySrcMrInfo_.addr);
75 30 : CHK_RET(hrtRaDeRegGlobalMr(rdmaHandle_, notifySrcMrHandle_));
76 30 : notifySrcMrHandle_ = nullptr;
77 30 : return HCCL_SUCCESS;
78 : }
79 :
80 90 : HcclResult TypicalSyncMem::AllocSyncMem(int32_t** ptr)
81 : {
82 90 : HCCL_DEBUG("[TypicalSyncMem][AllocSyncMem]start alloc sync mem on [%p].", ptr);
83 90 : std::unique_lock<std::mutex> lockSyncMemMap(syncMemMapMutex_);
84 90 : if (syncMemMap_.empty()) {
85 30 : HCCL_INFO("[TypicalSyncMem][AllocSyncMem] syncMem has not inited. Start to init notify src mem.");
86 30 : CHK_RET(InitNotifySrcMem());
87 : }
88 90 : CHK_PTR_NULL(ptr);
89 90 : CHK_RET(RdmaResourceManager::GetInstance().GetRdmaHandle(rdmaHandle_));
90 90 : CHK_PTR_NULL(rdmaHandle_);
91 :
92 90 : u64 offset = 0;
93 90 : u64 notifyBaseVa = 0;
94 90 : u64 notifyTotalSize = 0;
95 :
96 : // Create an empty notify and get it's handle
97 90 : HcclRtSignal notify = nullptr;
98 90 : CHK_RET(CreateEmptyNotify(notify));
99 90 : HCCL_DEBUG("[TypicalSyncMem][AllocSyncMem]create an empty notify success.");
100 :
101 : // Get the base virtual address and the size of notify register.
102 90 : u64 notifyBaseVaTmp = 0;
103 90 : notifyBaseVaTmp = notifyBaseVa;
104 90 : CHK_RET(HrtRaGetNotifyBaseAddr(rdmaHandle_, ¬ifyBaseVa, ¬ifyTotalSize));
105 :
106 90 : CHK_PRT_RET(
107 : ((notifyBaseVaTmp != 0) && (notifyBaseVaTmp != notifyBaseVa)),
108 : HCCL_ERROR("[TypicalSyncMem][AllocSyncMem]get base addr failed, notify base va has changed."), HCCL_E_INTERNAL);
109 :
110 : // Get the offset to the base address for the created notify,
111 : // which is same for both physical address and virtual address.
112 : // Here we use physical address to calculate the offset.
113 90 : CHK_RET(hrtNotifyGetOffset(notify, offset));
114 :
115 : // notify寄存器的虚拟地址与物理地址偏移相同，所以虚拟地址为虚拟基地址加偏移
116 90 : u64 notifyVa = notifyBaseVa + offset;
117 :
118 90 : HCCL_INFO(
119 : "[TypicalSyncMem][AllocSyncMem]notifyBaseVa=0x%llx,"
120 : "notifyTotalSize=0x%x, offset=0x%llx, notifyVa=0x%llx notify=%p.",
121 : notifyBaseVa, notifyTotalSize, offset, notifyVa, notify);
122 : // Store the notifyVa to set
123 90 : syncMemMap_[notifyVa] = notify;
124 : // Assign the notify virtual address to *ptr.
125 90 : *ptr = reinterpret_cast<int32_t*>(static_cast<uintptr_t>(notifyVa));
126 90 : HCCL_RUN_INFO(
127 : "[TypicalSyncMem][AllocSyncMem]alloc an empty sync mem success, notifyVa[%p]. "
128 : "please register mr before use.",
129 : *ptr);
130 90 : return HCCL_SUCCESS;
131 90 : }
132 :
133 90 : HcclResult TypicalSyncMem::FreeSyncMem(int32_t* ptr)
134 : {
135 90 : HCCL_DEBUG("[TypicalSyncMem][FreeSyncMem]start free sync mem[%p], please deregister mr before free.", ptr);
136 90 : CHK_PTR_NULL(ptr);
137 90 : u64 notifyVa = reinterpret_cast<uintptr_t>(ptr);
138 90 : std::unique_lock<std::mutex> lockSyncMemMap(syncMemMapMutex_);
139 90 : auto smIter = syncMemMap_.find(notifyVa);
140 90 : if (smIter == syncMemMap_.end()) {
141 0 : HCCL_WARNING("[TypicalSyncMem][FreeSyncMem]No notifyVa match the given ptr[%p] in sync mem map.", ptr);
142 0 : return HCCL_SUCCESS;
143 : }
144 90 : CHK_RET(DestroyNotify(syncMemMap_[notifyVa]));
145 90 : syncMemMap_.erase(smIter);
146 90 : if (syncMemMap_.empty()) {
147 30 : HCCL_INFO("[TypicalSyncMem][FreeSyncMem] syncMem all deinit. Start to deinit notify src mem.");
148 30 : CHK_RET(DeInitNotifySrcMem());
149 : }
150 90 : HCCL_INFO("[TypicalSyncMem][FreeSyncMem] Free [%p] success.", ptr);
151 90 : return HCCL_SUCCESS;
152 90 : }
153 :
154 86 : HcclResult TypicalSyncMem::GetNotifyHandle(u64 notifyVa, HcclRtNotify& notifyHandle)
155 : {
156 86 : std::unique_lock<std::mutex> lockSyncMemMap(syncMemMapMutex_);
157 86 : auto smIter = syncMemMap_.find(notifyVa);
158 86 : if (smIter != syncMemMap_.end()) {
159 86 : notifyHandle = smIter->second;
160 86 : return HCCL_SUCCESS;
161 : }
162 0 : HCCL_ERROR("[TypicalSyncMem][GetNotifyHandle]invalid notifyVa[%llu].", notifyVa);
163 0 : return HCCL_E_PARA;
164 86 : }
165 :
166 38 : HcclResult TypicalSyncMem::GetNotifySrcMem(struct MrInfoT& mrInfo)
167 : {
168 38 : CHK_PTR_NULL(notifySrcMrInfo_.addr);
169 38 : mrInfo.addr = notifySrcMrInfo_.addr;
170 38 : mrInfo.size = notifySrcMrInfo_.size;
171 38 : mrInfo.access = notifySrcMrInfo_.access;
172 38 : mrInfo.lkey = notifySrcMrInfo_.lkey;
173 38 : return HCCL_SUCCESS;
174 : }
175 :
176 90 : HcclResult TypicalSyncMem::CreateEmptyNotify(HcclRtNotify& notifyHandle)
177 : {
178 90 : s32 deviceId = 0;
179 90 : CHK_RET(hrtGetDevice(&deviceId));
180 90 : HcclResult ret = hrtNotifyCreate(deviceId, ¬ifyHandle);
181 90 : CHK_PRT_RET(
182 : ret != HCCL_SUCCESS,
183 : HCCL_ERROR(
184 : "[TypicalSyncMem][CreateNotify]errNo[0x%016llx] Notify create failed. return[%d], deviceLogicId[%d]",
185 : HCCL_ERROR_CODE(HCCL_E_RUNTIME), ret, deviceId),
186 : HCCL_E_RUNTIME);
187 90 : CHK_PRT_RET(
188 : notifyHandle == nullptr,
189 : HCCL_ERROR(
190 : "[TypicalSyncMem][CreateNotify]errNo[0x%016llx] Notify create failed. notifyHandle is NULL",
191 : HCCL_ERROR_CODE(HCCL_E_RUNTIME)),
192 : HCCL_E_RUNTIME);
193 :
194 90 : HCCL_INFO(
195 : "[TypicalSyncMem][CreateNotify]create notify success, deviceId[%d], notify handle[%p].", deviceId,
196 : notifyHandle);
197 90 : return HCCL_SUCCESS;
198 : }
199 :
200 90 : HcclResult TypicalSyncMem::DestroyNotify(HcclRtNotify notifyHandle)
201 : {
202 90 : HCCL_DEBUG("[TypicalSyncMem][DestroyNotify]start destroy notify[%p].", notifyHandle);
203 90 : CHK_PTR_NULL(notifyHandle);
204 90 : HcclResult ret = hrtNotifyDestroy(notifyHandle);
205 90 : CHK_PRT_RET(
206 : ret != RT_ERROR_NONE,
207 : HCCL_ERROR(
208 : "[TypicalSyncMem][DestroyNotify]errNo[0x%016llx] rt notify destroy fail, return[%d].",
209 : HCCL_ERROR_CODE(HCCL_E_RUNTIME), ret),
210 : HCCL_E_RUNTIME);
211 90 : HCCL_INFO("[TypicalSyncMem][DestroyNotify]destroy notify success.");
212 90 : return HCCL_SUCCESS;
213 : }
214 :
215 66 : HcclResult TypicalSyncMem::FreeAllSyncMem()
216 : {
217 66 : std::unique_lock<std::mutex> lockSyncMemMap(syncMemMapMutex_);
218 66 : if (!syncMemMap_.empty()) {
219 0 : for (auto& smIter : syncMemMap_) {
220 0 : if (smIter.second != nullptr) {
221 0 : CHK_RET(DestroyNotify(smIter.second));
222 : }
223 : }
224 0 : syncMemMap_.clear();
225 : }
226 66 : HCCL_INFO("[TypicalSyncMem][FreeAllSyncMem]free all sync memory success.");
227 66 : return HCCL_SUCCESS;
228 66 : }
229 : } // namespace hccl
|
