Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 : #include "typical_sync_mem.h"
11 : #include "adapter_rts_common.h"
12 : #include "adapter_rts.h"
13 : #include "rdma_resource_manager.h"
14 :
15 : namespace hccl {
16 304 : TypicalSyncMem& TypicalSyncMem::GetInstance()
17 : {
18 370 : static TypicalSyncMem typicalSyncMem[MAX_MODULE_DEVICE_NUM + 1];
19 304 : s32 deviceLogicId = INVALID_INT;
20 304 : s32 ret = hrtGetDevice(&deviceLogicId);
21 304 : if (ret == HCCL_SUCCESS && (static_cast<u32>(deviceLogicId) < MAX_MODULE_DEVICE_NUM)) {
22 304 : HCCL_INFO("[TypicalSyncMem::GetInstance]deviceLogicID[%d]", deviceLogicId);
23 304 : return typicalSyncMem[deviceLogicId];
24 : }
25 0 : HCCL_WARNING("[TypicalSyncMem::GetInstance]deviceLogicID[%d] is invalid, ret[%d].", deviceLogicId, ret);
26 0 : return typicalSyncMem[MAX_MODULE_DEVICE_NUM];
27 : }
28 :
29 66 : TypicalSyncMem::TypicalSyncMem() {}
30 :
31 66 : TypicalSyncMem::~TypicalSyncMem()
32 : {
33 66 : (void)FreeAllSyncMem();
34 66 : (void)DeInitNotifySrcMem();
35 66 : }
36 :
37 30 : HcclResult TypicalSyncMem::InitNotifySrcMem()
38 : {
39 30 : CHK_RET(RdmaResourceManager::GetInstance().GetRdmaHandle(rdmaHandle_));
40 30 : CHK_PTR_NULL(rdmaHandle_);
41 30 : HCCL_DEBUG("[TypicalSyncMem][InitNotifySrcMem]start init notify source mem.");
42 30 : u32 notifyVaule = 1; // notify值写1表示record
43 30 : u32 notifySize = 0;
44 30 : CHK_RET(hrtGetNotifySize(notifySize));
45 :
46 30 : CHK_RET(DeviceMem::alloc(srcDevMem_, notifySize));
47 30 : HCCL_DEBUG(
48 : "[TypicalSyncMem][InitNotifySrcMem]Create notify src buffer[%p], size[%u].", srcDevMem_.ptr(), notifySize);
49 :
50 30 : CHK_RET(hrtMemSyncCopy(
51 : srcDevMem_.ptr(), notifySize, ¬ifyVaule, notifySize, HcclRtMemcpyKind::HCCL_RT_MEMCPY_KIND_HOST_TO_DEVICE));
52 :
53 30 : notifySrcMrInfo_.addr = srcDevMem_.ptr();
54 30 : notifySrcMrInfo_.size = notifySize;
55 30 : notifySrcMrInfo_.access = RA_ACCESS_LOCAL_WRITE | RA_ACCESS_REMOTE_WRITE;
56 30 : notifySrcMrHandle_ = nullptr;
57 30 : CHK_RET(hrtRaRegGlobalMr(rdmaHandle_, notifySrcMrInfo_, notifySrcMrHandle_));
58 :
59 30 : HCCL_INFO(
60 : "[TypicalSyncMem][InitNotifySrcMem]Init notifySrcMem_=%p success, mr lkey is [%u].", notifySrcMrInfo_.addr,
61 : notifySrcMrInfo_.lkey);
62 30 : return HCCL_SUCCESS;
63 : }
64 :
65 96 : HcclResult TypicalSyncMem::DeInitNotifySrcMem()
66 : {
67 96 : if (notifySrcMrHandle_ == nullptr) {
68 66 : HCCL_INFO("[TypicalSyncMem][InitNotifySrcMem] NotifySrcMem has been DeInit.");
69 66 : return HCCL_SUCCESS;
70 : }
71 30 : CHK_RET(RdmaResourceManager::GetInstance().GetRdmaHandle(rdmaHandle_));
72 30 : CHK_PTR_NULL(rdmaHandle_);
73 30 : HCCL_INFO("[TypicalSyncMem][InitNotifySrcMem] DeRegister notifySrcMem_=%p.", notifySrcMrInfo_.addr);
74 30 : CHK_RET(hrtRaDeRegGlobalMr(rdmaHandle_, notifySrcMrHandle_));
75 30 : notifySrcMrHandle_ = nullptr;
76 30 : return HCCL_SUCCESS;
77 : }
78 :
79 90 : HcclResult TypicalSyncMem::AllocSyncMem(int32_t** ptr)
80 : {
81 90 : HCCL_DEBUG("[TypicalSyncMem][AllocSyncMem]start alloc sync mem on [%p].", ptr);
82 90 : std::unique_lock<std::mutex> lockSyncMemMap(syncMemMapMutex_);
83 90 : if (syncMemMap_.empty()) {
84 30 : HCCL_INFO("[TypicalSyncMem][AllocSyncMem] syncMem has not inited. Start to init notify src mem.");
85 30 : CHK_RET(InitNotifySrcMem());
86 : }
87 90 : CHK_PTR_NULL(ptr);
88 90 : CHK_RET(RdmaResourceManager::GetInstance().GetRdmaHandle(rdmaHandle_));
89 90 : CHK_PTR_NULL(rdmaHandle_);
90 :
91 90 : u64 offset = 0;
92 90 : u64 notifyBaseVa = 0;
93 90 : u64 notifyTotalSize = 0;
94 :
95 : // Create an empty notify and get it's handle
96 90 : HcclRtSignal notify = nullptr;
97 90 : CHK_RET(CreateEmptyNotify(notify));
98 90 : HCCL_DEBUG("[TypicalSyncMem][AllocSyncMem]create an empty notify success.");
99 :
100 : // Get the base virtual address and the size of notify register.
101 90 : u64 notifyBaseVaTmp = 0;
102 90 : notifyBaseVaTmp = notifyBaseVa;
103 90 : CHK_RET(HrtRaGetNotifyBaseAddr(rdmaHandle_, ¬ifyBaseVa, ¬ifyTotalSize));
104 :
105 90 : CHK_PRT_RET(
106 : ((notifyBaseVaTmp != 0) && (notifyBaseVaTmp != notifyBaseVa)),
107 : HCCL_ERROR("[TypicalSyncMem][AllocSyncMem]get base addr failed, notify base va has changed."), HCCL_E_INTERNAL);
108 :
109 : // Get the offset to the base address for the created notify,
110 : // which is same for both physical address and virtual address.
111 : // Here we use physical address to calculate the offset.
112 90 : CHK_RET(hrtNotifyGetOffset(notify, offset));
113 :
114 : // notify寄存器的虚拟地址与物理地址偏移相同，所以虚拟地址为虚拟基地址加偏移
115 90 : u64 notifyVa = notifyBaseVa + offset;
116 :
117 90 : HCCL_INFO(
118 : "[TypicalSyncMem][AllocSyncMem]notifyBaseVa=0x%llx,"
119 : "notifyTotalSize=0x%x, offset=0x%llx, notifyVa=0x%llx notify=%p.",
120 : notifyBaseVa, notifyTotalSize, offset, notifyVa, notify);
121 : // Store the notifyVa to set
122 90 : syncMemMap_[notifyVa] = notify;
123 : // Assign the notify virtual address to *ptr.
124 90 : *ptr = reinterpret_cast<int32_t*>(static_cast<uintptr_t>(notifyVa));
125 90 : HCCL_RUN_INFO(
126 : "[TypicalSyncMem][AllocSyncMem]alloc an empty sync mem success, notifyVa[%p]. "
127 : "please register mr before use.",
128 : *ptr);
129 90 : return HCCL_SUCCESS;
130 90 : }
131 :
132 90 : HcclResult TypicalSyncMem::FreeSyncMem(int32_t* ptr)
133 : {
134 90 : HCCL_DEBUG("[TypicalSyncMem][FreeSyncMem]start free sync mem[%p], please deregister mr before free.", ptr);
135 90 : CHK_PTR_NULL(ptr);
136 90 : u64 notifyVa = reinterpret_cast<uintptr_t>(ptr);
137 90 : std::unique_lock<std::mutex> lockSyncMemMap(syncMemMapMutex_);
138 90 : auto smIter = syncMemMap_.find(notifyVa);
139 90 : if (smIter == syncMemMap_.end()) {
140 0 : HCCL_WARNING("[TypicalSyncMem][FreeSyncMem]No notifyVa match the given ptr[%p] in sync mem map.", ptr);
141 0 : return HCCL_SUCCESS;
142 : }
143 90 : CHK_RET(DestroyNotify(syncMemMap_[notifyVa]));
144 90 : syncMemMap_.erase(smIter);
145 90 : if (syncMemMap_.empty()) {
146 30 : HCCL_INFO("[TypicalSyncMem][FreeSyncMem] syncMem all deinit. Start to deinit notify src mem.");
147 30 : CHK_RET(DeInitNotifySrcMem());
148 : }
149 90 : HCCL_INFO("[TypicalSyncMem][FreeSyncMem] Free [%p] success.", ptr);
150 90 : return HCCL_SUCCESS;
151 90 : }
152 :
153 86 : HcclResult TypicalSyncMem::GetNotifyHandle(u64 notifyVa, HcclRtNotify& notifyHandle)
154 : {
155 86 : std::unique_lock<std::mutex> lockSyncMemMap(syncMemMapMutex_);
156 86 : auto smIter = syncMemMap_.find(notifyVa);
157 86 : if (smIter != syncMemMap_.end()) {
158 86 : notifyHandle = smIter->second;
159 86 : return HCCL_SUCCESS;
160 : }
161 0 : HCCL_ERROR("[TypicalSyncMem][GetNotifyHandle]invalid notifyVa[%llu].", notifyVa);
162 0 : return HCCL_E_PARA;
163 86 : }
164 :
165 38 : HcclResult TypicalSyncMem::GetNotifySrcMem(struct MrInfoT& mrInfo)
166 : {
167 38 : CHK_PTR_NULL(notifySrcMrInfo_.addr);
168 38 : mrInfo.addr = notifySrcMrInfo_.addr;
169 38 : mrInfo.size = notifySrcMrInfo_.size;
170 38 : mrInfo.access = notifySrcMrInfo_.access;
171 38 : mrInfo.lkey = notifySrcMrInfo_.lkey;
172 38 : return HCCL_SUCCESS;
173 : }
174 :
175 90 : HcclResult TypicalSyncMem::CreateEmptyNotify(HcclRtNotify& notifyHandle)
176 : {
177 90 : s32 deviceId = 0;
178 90 : CHK_RET(hrtGetDevice(&deviceId));
179 90 : HcclResult ret = hrtNotifyCreate(deviceId, ¬ifyHandle);
180 90 : CHK_PRT_RET(
181 : ret != HCCL_SUCCESS,
182 : HCCL_ERROR(
183 : "[TypicalSyncMem][CreateNotify]errNo[0x%016llx] Notify create failed. return[%d], deviceLogicId[%d]",
184 : HCCL_ERROR_CODE(HCCL_E_RUNTIME), ret, deviceId),
185 : HCCL_E_RUNTIME);
186 90 : CHK_PRT_RET(
187 : notifyHandle == nullptr,
188 : HCCL_ERROR(
189 : "[TypicalSyncMem][CreateNotify]errNo[0x%016llx] Notify create failed. notifyHandle is NULL",
190 : HCCL_ERROR_CODE(HCCL_E_RUNTIME)),
191 : HCCL_E_RUNTIME);
192 :
193 90 : HCCL_INFO(
194 : "[TypicalSyncMem][CreateNotify]create notify success, deviceId[%d], notify handle[%p].", deviceId,
195 : notifyHandle);
196 90 : return HCCL_SUCCESS;
197 : }
198 :
199 90 : HcclResult TypicalSyncMem::DestroyNotify(HcclRtNotify notifyHandle)
200 : {
201 90 : HCCL_DEBUG("[TypicalSyncMem][DestroyNotify]start destroy notify[%p].", notifyHandle);
202 90 : CHK_PTR_NULL(notifyHandle);
203 90 : HcclResult ret = hrtNotifyDestroy(notifyHandle);
204 90 : CHK_PRT_RET(
205 : ret != RT_ERROR_NONE,
206 : HCCL_ERROR(
207 : "[TypicalSyncMem][DestroyNotify]errNo[0x%016llx] rt notify destroy fail, return[%d].",
208 : HCCL_ERROR_CODE(HCCL_E_RUNTIME), ret),
209 : HCCL_E_RUNTIME);
210 90 : HCCL_INFO("[TypicalSyncMem][DestroyNotify]destroy notify success.");
211 90 : return HCCL_SUCCESS;
212 : }
213 :
214 66 : HcclResult TypicalSyncMem::FreeAllSyncMem()
215 : {
216 66 : std::unique_lock<std::mutex> lockSyncMemMap(syncMemMapMutex_);
217 66 : if (!syncMemMap_.empty()) {
218 0 : for (auto& smIter : syncMemMap_) {
219 0 : if (smIter.second != nullptr) {
220 0 : CHK_RET(DestroyNotify(smIter.second));
221 : }
222 : }
223 0 : syncMemMap_.clear();
224 : }
225 66 : HCCL_INFO("[TypicalSyncMem][FreeAllSyncMem]free all sync memory success.");
226 66 : return HCCL_SUCCESS;
227 66 : }
228 : } // namespace hccl
|
