Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "hdc_pub.h"
12 : #include "log.h"
13 : #include "ascend_hal.h"
14 : #include "adapter_hal.h"
15 : #include "adapter_rts.h"
16 : #include <atomic>
17 : #include <chrono>
18 :
19 : namespace hccl {
20 978 : HDCommunicate::HDCommunicate(u32 deviceLogicId, u32 flag, u32 buffLen)
21 978 : : deviceLogicId_(deviceLogicId),
22 978 : flag_(flag),
23 978 : buffLen_(buffLen)
24 978 : {}
25 198 : HDCommunicate::HDCommunicate() : deviceLogicId_(INVALID_UINT), flag_(0), buffLen_(0) {}
26 :
27 1175 : HDCommunicate::~HDCommunicate()
28 : {
29 1175 : if (isHost_ && devMem_.ptr() && supportDevMemReg_) {
30 965 : (void)hrtHalHostUnregister(devMem_.ptr(), deviceLogicId_);
31 : }
32 1174 : }
33 :
34 : // 在device中申请共享内存，其数据格式如下所示:
35 : // +---------------------+
36 : // | |
37 : // | content |
38 : // | |
39 : // +---------------------+
40 : // | head_cnt[u32] |
41 : // +---------------------+
42 : // | tail_cnt[u32] |
43 : // +---------------------+
44 : // 发送方更新content前，需要将head_cnt加1,更新数据后需要将tail_cnt加1;
45 : // 接收方在读取数据前判断共享内存中的tail_cnt和本地cache中的tailcnt是否一致，如不一致则需要更新本地cache;
46 : // 更新本地cache时，需要确保cache中head_cnt和tail_cnt一致，否则舍弃本次数据，继续更新cache直至一致;
47 :
48 : #define HCCL_SHM_ALIGN 4096
49 : #define HCCL_HDC_CONTROL_WORDS 2
50 : #define HCCL_HDC_HEAD_POS 2
51 : #define HCCL_HDC_TAIL_POS 1
52 :
53 361787528 : inline u32* HcclHdcGetControlWordAddr(void* base, u64 size, u32 pos)
54 : {
55 361787528 : return reinterpret_cast<u32*>(reinterpret_cast<u8*>((base)) + size - pos * sizeof(pos));
56 : }
57 :
58 966 : HcclResult HDCommunicate::InitHost()
59 : {
60 966 : CHK_RET(VerifyDeviceMemoryRegisterSupport());
61 :
62 966 : CHK_RET(AllocShm(deviceLogicId_, devMem_, hostMem_));
63 966 : CHK_RET(AllocReadCache(flag_, readCacheAddr_));
64 :
65 966 : headCntAddr_ = HcclHdcGetControlWordAddr(hostMem_.ptr(), hostMem_.size(), HCCL_HDC_HEAD_POS);
66 966 : tailCntAddr_ = HcclHdcGetControlWordAddr(hostMem_.ptr(), hostMem_.size(), HCCL_HDC_TAIL_POS);
67 :
68 966 : devHeadCntAddr_ = HcclHdcGetControlWordAddr(devMem_.ptr(), devMem_.size(), HCCL_HDC_HEAD_POS);
69 966 : devTailCntAddr_ = HcclHdcGetControlWordAddr(devMem_.ptr(), devMem_.size(), HCCL_HDC_TAIL_POS);
70 966 : return HCCL_SUCCESS;
71 : }
72 :
73 966 : HcclResult HDCommunicate::VerifyDeviceMemoryRegisterSupport()
74 : {
75 966 : supportDevMemReg_ = false;
76 966 : size_t outputLen = 0;
77 966 : struct supportFeaturePara input = {};
78 966 : struct supportFeaturePara output = {};
79 966 : s32 deviceId = 0;
80 966 : CHK_RET(hrtGetDevice(&deviceId));
81 966 : s32 logicDevId = 0;
82 : // 调用驱动接口前需将userDevId转换为logicDevId
83 966 : CHK_RET(hrtGetLogicDevIdByUserDevId(deviceId, logicDevId));
84 966 : deviceLogicId_ = static_cast<u32>(logicDevId);
85 966 : input.support_feature = CTRL_SUPPORT_PCIE_BAR_MEM_MASK;
86 966 : input.devid = static_cast<unsigned int>(logicDevId);
87 966 : CHK_RET(hrtHalMemCtl(CTRL_TYPE_SUPPORT_FEATURE, &input, sizeof(struct supportFeaturePara), &output, &outputLen));
88 :
89 966 : if ((output.support_feature & CTRL_SUPPORT_PCIE_BAR_MEM_MASK) != 0) {
90 966 : supportDevMemReg_ = true;
91 : }
92 966 : HCCL_INFO("[HDCommunicate]supportDevMemReg_ %d deviceId %d", supportDevMemReg_, input.devid);
93 966 : return HCCL_SUCCESS;
94 : }
95 :
96 448 : struct HDCommunicateParams HDCommunicate::GetCommunicateParams()
97 : {
98 448 : struct HDCommunicateParams params;
99 448 : params.hostAddr = reinterpret_cast<u64>(hostMem_.ptr());
100 448 : params.deviceAddr = reinterpret_cast<u64>(devMem_.ptr());
101 448 : params.readCacheAddr = reinterpret_cast<u64>(readCacheAddr_);
102 448 : params.devMemSize = devMem_.size();
103 448 : params.buffLen = buffLen_;
104 448 : params.flag = flag_;
105 448 : HCCL_DEBUG(
106 : "[HDCommunicate][GetCommunicateParams] hostAddr %p deviceAddr %p readCacheAddr %p devMemSize %u "
107 : "buffLen %u flag %u",
108 : params.hostAddr, params.deviceAddr, params.readCacheAddr, devMem_.size(), buffLen_, flag_);
109 448 : return params;
110 : }
111 :
112 154 : HcclResult HDCommunicate::InitDevice(const struct HDCommunicateParams& params)
113 : {
114 154 : CHK_PRT_RET(
115 : (params.devMemSize == 0), HCCL_ERROR("[HDCommunicate][InitDevice]Invalid devMemSize=%u", params.devMemSize),
116 : HCCL_E_PARA);
117 154 : void* deviceAddr = reinterpret_cast<void*>(params.deviceAddr);
118 154 : CHK_PTR_NULL(deviceAddr);
119 154 : readCacheAddr_ = reinterpret_cast<void*>(params.readCacheAddr);
120 154 : CHK_PTR_NULL(readCacheAddr_);
121 154 : devMem_ = DeviceMem::create(deviceAddr, params.devMemSize);
122 154 : buffLen_ = params.buffLen;
123 154 : flag_ = params.flag;
124 :
125 154 : headCntAddr_ = HcclHdcGetControlWordAddr(devMem_.ptr(), devMem_.size(), HCCL_HDC_HEAD_POS);
126 154 : tailCntAddr_ = HcclHdcGetControlWordAddr(devMem_.ptr(), devMem_.size(), HCCL_HDC_TAIL_POS);
127 154 : isHost_ = false;
128 :
129 154 : HCCL_DEBUG(
130 : "[debug HDCommunicate][InitDevice] buffLen_=%u, flag_=%u, readCacheAddr_=%p, headCntAddr_=%p, "
131 : "tailCntAddr_=%p, deviceAddr %p",
132 : buffLen_, flag_, readCacheAddr_, headCntAddr_, tailCntAddr_, devMem_.ptr());
133 154 : return HCCL_SUCCESS;
134 : }
135 :
136 193 : HcclResult HDCommunicate::Put(u32 offset, u32 length, u8* value)
137 : {
138 193 : if (length == 0) {
139 0 : return HCCL_SUCCESS;
140 : }
141 193 : CHK_PTR_NULL(value);
142 193 : if (((flag_ == HCCL_HDC_TYPE_D2H) && isHost_) || ((flag_ == HCCL_HDC_TYPE_H2D) && !isHost_)) {
143 1 : HCCL_ERROR("[HDCommunicate][Put]Invalid usage, flag=%u, isHost=%d", flag_, isHost_);
144 1 : return HCCL_E_PARA;
145 : }
146 192 : CHK_PRT_RET(
147 : (offset + length > buffLen_),
148 : HCCL_ERROR("[HDCommunicate][Put]Invalid length, offset=%u, length=%u", offset, length), HCCL_E_PARA);
149 192 : std::unique_lock<std::shared_mutex> lock(lock_);
150 192 : HcclResult ret = Write(offset, length, value);
151 192 : if (ret != HCCL_SUCCESS) {
152 0 : HCCL_ERROR("[HDCommunicate][Put]Write failed, offset=%u, length=%u", offset, length);
153 0 : return ret;
154 : }
155 192 : return HCCL_SUCCESS;
156 192 : }
157 :
158 361782853 : HcclResult HDCommunicate::Get(u32 offset, u32 length, u8* value)
159 : {
160 361782853 : if (length == 0) {
161 0 : return HCCL_SUCCESS;
162 : }
163 361782853 : CHK_PTR_NULL(value);
164 361782853 : CHK_PRT_RET(
165 : (offset + length > buffLen_),
166 : HCCL_ERROR("[HDCommunicate][Get]Invalid length, offset=%u, length=%u, befferLen=%u", offset, length, buffLen_),
167 : HCCL_E_PARA);
168 361782852 : std::shared_lock<std::shared_mutex> lock(lock_);
169 361782850 : HcclResult ret = Read(offset, length, value);
170 361782835 : if (ret != HCCL_SUCCESS) {
171 0 : HCCL_ERROR("[HDCommunicate][Get]Read failed, offset=%u, length=%u", offset, length);
172 0 : return ret;
173 : }
174 361782835 : return HCCL_SUCCESS;
175 361782835 : }
176 :
177 : #pragma GCC push_options
178 : #pragma GCC optimize("O0")
179 192 : HcclResult HDCommunicate::Write(u32 offset, u32 length, u8* value)
180 : {
181 192 : if (length == 0) {
182 0 : return HCCL_SUCCESS;
183 : }
184 192 : CHK_PTR_NULL(value);
185 192 : u32 head = *headCntAddr_;
186 192 : head++;
187 192 : *headCntAddr_ = head;
188 192 : if (isHost_ && !supportDevMemReg_) {
189 0 : CHK_RET(hrtDrvMemCpy(devHeadCntAddr_, sizeof(u32), headCntAddr_, sizeof(u32)));
190 : }
191 :
192 192 : if (isHost_) {
193 126 : auto ret = memcpy_s(
194 42 : reinterpret_cast<u8*>(hostMem_.ptr()) + offset, hostMem_.size() - HCCL_HDC_CONTROL_WORDS * sizeof(u32),
195 : value, length);
196 42 : CHK_PRT_RET(ret != EOK, HCCL_ERROR("[HDCommunicate][Write]memcpy_s failed, return[%d].", ret), HCCL_E_INTERNAL);
197 : } else {
198 450 : auto ret = memcpy_s(
199 150 : reinterpret_cast<u8*>(devMem_.ptr()) + offset, devMem_.size() - HCCL_HDC_CONTROL_WORDS * sizeof(u32), value,
200 : length);
201 150 : CHK_PRT_RET(ret != EOK, HCCL_ERROR("[HDCommunicate][Write]memcpy_s failed, return[%d].", ret), HCCL_E_INTERNAL);
202 : }
203 192 : if (isHost_ && !supportDevMemReg_) {
204 0 : CHK_RET(hrtDrvMemCpy(
205 : reinterpret_cast<u8*>(devMem_.ptr()) + offset, hostMem_.size() - HCCL_HDC_CONTROL_WORDS * sizeof(u32),
206 : value, length));
207 : }
208 :
209 : std::atomic_thread_fence(std::memory_order_seq_cst);
210 :
211 192 : u32 tail = *tailCntAddr_;
212 192 : tail++;
213 192 : *tailCntAddr_ = tail;
214 192 : if (isHost_ && !supportDevMemReg_) {
215 0 : CHK_RET(hrtDrvMemCpy(devTailCntAddr_, sizeof(u32), tailCntAddr_, sizeof(u32)));
216 : }
217 192 : return HCCL_SUCCESS;
218 : }
219 :
220 361782853 : HcclResult HDCommunicate::Read(u32 offset, u32 length, u8* value)
221 : {
222 361782853 : if (length == 0) {
223 0 : return HCCL_SUCCESS;
224 : }
225 361782853 : CHK_PTR_NULL(value);
226 361782853 : u32* cachedTailCntAddr = HcclHdcGetControlWordAddr(readCacheAddr_, devMem_.size(), HCCL_HDC_TAIL_POS);
227 361782840 : volatile u32 cachedTailCnt = *cachedTailCntAddr;
228 361782840 : volatile u32 tailCnt = 0;
229 361782840 : if (isHost_ && !supportDevMemReg_) {
230 0 : u32 tempTailCnt = 0;
231 0 : u32* devSrcTailCntAddr = HcclHdcGetControlWordAddr(devMem_.ptr(), devMem_.size(), HCCL_HDC_TAIL_POS);
232 0 : CHK_RET(hrtDrvMemCpy(&tempTailCnt, sizeof(u32), devSrcTailCntAddr, sizeof(u32)));
233 0 : tailCnt = tempTailCnt;
234 0 : } else {
235 361782840 : tailCnt = *tailCntAddr_;
236 : }
237 361782840 : if (cachedTailCnt != tailCnt) {
238 : // 默认HDC超时时间为10s
239 94 : CHK_RET(UpdateCache(10));
240 : }
241 361782840 : auto ret = memcpy_s(value, length, static_cast<u8*>(readCacheAddr_) + offset, length);
242 361782835 : CHK_PRT_RET(ret != EOK, HCCL_ERROR("[HDCommunicate][Read]memcpy_s failed, return[%d].", ret), HCCL_E_INTERNAL);
243 361782835 : return HCCL_SUCCESS;
244 : }
245 :
246 94 : HcclResult HDCommunicate::UpdateCache(u32 timeoutSec)
247 : {
248 94 : void* srcBaseAddr = isHost_ ? hostMem_.ptr() : devMem_.ptr();
249 94 : u32* srcHeadCntAddr = HcclHdcGetControlWordAddr(srcBaseAddr, devMem_.size(), HCCL_HDC_HEAD_POS);
250 94 : u32* srcTailCntAddr = HcclHdcGetControlWordAddr(srcBaseAddr, devMem_.size(), HCCL_HDC_TAIL_POS);
251 94 : u32* devSrcHeadCntAddr = HcclHdcGetControlWordAddr(devMem_.ptr(), devMem_.size(), HCCL_HDC_HEAD_POS);
252 94 : u32* devSrcTailCntAddr = HcclHdcGetControlWordAddr(devMem_.ptr(), devMem_.size(), HCCL_HDC_TAIL_POS);
253 94 : u32* cachedHeadCntAddr = HcclHdcGetControlWordAddr(readCacheAddr_, devMem_.size(), HCCL_HDC_HEAD_POS);
254 94 : u32* cachedTailCntAddr = HcclHdcGetControlWordAddr(readCacheAddr_, devMem_.size(), HCCL_HDC_TAIL_POS);
255 :
256 94 : s32 ret = 0;
257 94 : auto startTime = std::chrono::steady_clock::now();
258 94 : auto timeout = std::chrono::seconds(timeoutSec);
259 : while (1) {
260 94 : if (isHost_ && !supportDevMemReg_) {
261 : // step1: cache尾计数
262 0 : CHK_RET(hrtDrvMemCpy(cachedTailCntAddr, sizeof(u32), devSrcTailCntAddr, sizeof(u32)));
263 :
264 : // step2: cache数据
265 0 : CHK_RET(hrtDrvMemCpy(
266 : readCacheAddr_, devMem_.size() - HCCL_HDC_CONTROL_WORDS * sizeof(u32), devMem_.ptr(),
267 : devMem_.size() - HCCL_HDC_CONTROL_WORDS * sizeof(u32)));
268 :
269 : // step3：cache头计数
270 0 : CHK_RET(hrtDrvMemCpy(cachedHeadCntAddr, sizeof(u32), devSrcHeadCntAddr, sizeof(u32)));
271 0 : } else {
272 : // step1: cache尾计数
273 94 : ret = memcpy_s(cachedTailCntAddr, sizeof(u32), srcTailCntAddr, sizeof(u32));
274 94 : CHK_PRT_RET(
275 : ret != EOK, HCCL_ERROR("[HDCommunicate][UpdateCache]memcpy_s failed, return[%d].", ret),
276 : HCCL_E_INTERNAL);
277 :
278 : // step2: cache数据
279 188 : ret = memcpy_s(
280 94 : readCacheAddr_, devMem_.size() - HCCL_HDC_CONTROL_WORDS * sizeof(u32), srcBaseAddr,
281 94 : devMem_.size() - HCCL_HDC_CONTROL_WORDS * sizeof(u32));
282 94 : CHK_PRT_RET(
283 : ret != EOK, HCCL_ERROR("[HDCommunicate][UpdateCache]memcpy_s failed, return[%d].", ret),
284 : HCCL_E_INTERNAL);
285 :
286 : // step3：cache头计数
287 94 : ret = memcpy_s(cachedHeadCntAddr, sizeof(u32), srcHeadCntAddr, sizeof(u32));
288 94 : CHK_PRT_RET(
289 : ret != EOK, HCCL_ERROR("[HDCommunicate][UpdateCache]memcpy_s failed, return[%d].", ret),
290 : HCCL_E_INTERNAL);
291 : }
292 :
293 94 : volatile u32 cachedHeadCnt = *cachedHeadCntAddr;
294 94 : volatile u32 cachedTailCnt = *cachedTailCntAddr;
295 :
296 94 : if (cachedHeadCnt == cachedTailCnt) {
297 94 : break;
298 : }
299 0 : CHK_PRT_RET(
300 : ((std::chrono::steady_clock::now() - startTime) >= timeout),
301 : HCCL_WARNING("[HDCommunicate][UpdateCache]get remote data timeout[%u s].", timeoutSec), HCCL_E_AGAIN);
302 0 : }
303 94 : return HCCL_SUCCESS;
304 : }
305 : #pragma GCC pop_options
306 :
307 966 : HcclResult HDCommunicate::AllocShm(u32 devid, DeviceMem& devShm, HostMem& hostShm)
308 : {
309 : // 共享内存size需要按照4K(4*1024=4096)对齐
310 966 : u32 size = (buffLen_ + HCCL_HDC_CONTROL_WORDS * sizeof(u32) + HCCL_SHM_ALIGN - 1) / HCCL_SHM_ALIGN * HCCL_SHM_ALIGN;
311 966 : CHK_RET(DeviceMem::alloc(devShm, size));
312 966 : CHK_RET(hrtMemSet(devShm.ptr(), size, size));
313 :
314 966 : if (supportDevMemReg_) {
315 966 : void* hostAddr = nullptr;
316 966 : CHK_RET(hrtHalHostRegister(devShm.ptr(), devShm.size(), DEV_SVM_MAP_HOST, devid, hostAddr));
317 :
318 966 : hostShm = HostMem::create(hostAddr, devShm.size());
319 : } else {
320 0 : hostShm = HostMem::alloc(devShm.size());
321 : }
322 966 : CHK_PTR_NULL(hostShm.ptr());
323 :
324 966 : auto ret = memset_s(hostShm.ptr(), hostShm.size(), 0, hostShm.size());
325 966 : CHK_PRT_RET(ret != EOK, HCCL_ERROR("[HDCommunicate][AllocShm]memset_s failed, return[%d].", ret), HCCL_E_INTERNAL);
326 :
327 966 : return HCCL_SUCCESS;
328 : }
329 :
330 966 : HcclResult HDCommunicate::AllocReadCache(u32 flag, void*& readCacheAddr)
331 : {
332 966 : if (flag == HCCL_HDC_TYPE_D2H) {
333 483 : hostCache_ = HostMem::alloc(hostMem_.size());
334 483 : CHK_PTR_NULL(hostCache_.ptr());
335 483 : auto ret = memset_s(hostCache_.ptr(), hostCache_.size(), 0, hostCache_.size());
336 483 : CHK_PRT_RET(
337 : ret != EOK, HCCL_ERROR("[HDCommunicate][AllocReadCache]memset_s failed, return[%d].", ret),
338 : HCCL_E_INTERNAL);
339 483 : readCacheAddr = hostCache_.ptr();
340 : } else {
341 483 : CHK_RET(DeviceMem::alloc(devCache_, devMem_.size()));
342 483 : CHK_RET(hrtMemSet(devCache_.ptr(), devCache_.size(), devCache_.size()));
343 483 : readCacheAddr = devCache_.ptr();
344 : }
345 966 : return HCCL_SUCCESS;
346 : }
347 : } // namespace hccl
|
