Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "hdc_pub.h"
12 : #include "log.h"
13 : #include "ascend_hal.h"
14 : #include "adapter_hal.h"
15 : #include "adapter_rts.h"
16 : #include <atomic>
17 : #include <chrono>
18 :
19 : namespace hccl {
20 878 : HDCommunicate::HDCommunicate(u32 deviceLogicId, u32 flag, u32 buffLen)
21 878 : : deviceLogicId_(deviceLogicId), flag_(flag), buffLen_(buffLen)
22 878 : {}
23 198 : HDCommunicate::HDCommunicate() : deviceLogicId_(INVALID_UINT), flag_(0), buffLen_(0) {}
24 :
25 1075 : HDCommunicate::~HDCommunicate()
26 : {
27 1075 : if (isHost_ && devMem_.ptr() && supportDevMemReg_) {
28 866 : (void)hrtHalHostUnregister(devMem_.ptr(), deviceLogicId_);
29 : }
30 1070 : }
31 :
32 : // 在device中申请共享内存，其数据格式如下所示:
33 : // +---------------------+
34 : // | |
35 : // | content |
36 : // | |
37 : // +---------------------+
38 : // | head_cnt[u32] |
39 : // +---------------------+
40 : // | tail_cnt[u32] |
41 : // +---------------------+
42 : // 发送方更新content前，需要将head_cnt加1,更新数据后需要将tail_cnt加1;
43 : // 接收方在读取数据前判断共享内存中的tail_cnt和本地cache中的tailcnt是否一致，如不一致则需要更新本地cache;
44 : // 更新本地cache时，需要确保cache中head_cnt和tail_cnt一致，否则舍弃本次数据，继续更新cache直至一致;
45 :
46 :
47 : #define HCCL_SHM_ALIGN 4096
48 : #define HCCL_HDC_CONTROL_WORDS 2
49 : #define HCCL_HDC_HEAD_POS 2
50 : #define HCCL_HDC_TAIL_POS 1
51 :
52 359282832 : inline u32* HcclHdcGetControlWordAddr(void *base, u64 size, u32 pos)
53 : {
54 359282832 : return reinterpret_cast<u32 *>(reinterpret_cast<u8 *>((base)) + size - pos * sizeof(pos));
55 : }
56 :
57 866 : HcclResult HDCommunicate::InitHost()
58 : {
59 866 : CHK_RET(VerifyDeviceMemoryRegisterSupport());
60 :
61 866 : CHK_RET(AllocShm(deviceLogicId_, devMem_, hostMem_));
62 866 : CHK_RET(AllocReadCache(flag_, readCacheAddr_));
63 :
64 866 : headCntAddr_ = HcclHdcGetControlWordAddr(hostMem_.ptr(), hostMem_.size(), HCCL_HDC_HEAD_POS);
65 866 : tailCntAddr_ = HcclHdcGetControlWordAddr(hostMem_.ptr(), hostMem_.size(), HCCL_HDC_TAIL_POS);
66 :
67 866 : devHeadCntAddr_ = HcclHdcGetControlWordAddr(devMem_.ptr(), devMem_.size(), HCCL_HDC_HEAD_POS);
68 866 : devTailCntAddr_ = HcclHdcGetControlWordAddr(devMem_.ptr(), devMem_.size(), HCCL_HDC_TAIL_POS);
69 866 : return HCCL_SUCCESS;
70 : }
71 :
72 :
73 866 : HcclResult HDCommunicate::VerifyDeviceMemoryRegisterSupport()
74 : {
75 866 : supportDevMemReg_ = false;
76 866 : size_t outputLen = 0;
77 866 : struct supportFeaturePara input = { 0 };
78 866 : struct supportFeaturePara output = { 0 };
79 866 : s32 deviceId = 0;
80 866 : CHK_RET(hrtGetDevice(&deviceId));
81 866 : s32 logicDevId = 0;
82 : // 调用驱动接口前需将userDevId转换为logicDevId
83 866 : CHK_RET(hrtGetLogicDevIdByUserDevId(deviceId, logicDevId));
84 866 : deviceLogicId_ = static_cast<u32>(logicDevId);
85 866 : input.support_feature = CTRL_SUPPORT_PCIE_BAR_MEM_MASK;
86 866 : input.devid = static_cast<unsigned int>(logicDevId);
87 866 : CHK_RET(hrtHalMemCtl(CTRL_TYPE_SUPPORT_FEATURE, &input, sizeof(struct supportFeaturePara), &output, &outputLen));
88 :
89 866 : if ((output.support_feature & CTRL_SUPPORT_PCIE_BAR_MEM_MASK) != 0) {
90 866 : supportDevMemReg_ = true;
91 : }
92 866 : HCCL_INFO("[HDCommunicate]supportDevMemReg_ %d deviceId %d", supportDevMemReg_, input.devid);
93 866 : return HCCL_SUCCESS;
94 : }
95 :
96 352 : struct HDCommunicateParams HDCommunicate::GetCommunicateParams()
97 : {
98 352 : struct HDCommunicateParams params;
99 352 : params.hostAddr = reinterpret_cast<u64>(hostMem_.ptr());
100 352 : params.deviceAddr = reinterpret_cast<u64>(devMem_.ptr());
101 352 : params.readCacheAddr = reinterpret_cast<u64>(readCacheAddr_);
102 352 : params.devMemSize = devMem_.size();
103 352 : params.buffLen = buffLen_;
104 352 : params.flag = flag_;
105 352 : HCCL_DEBUG("[HDCommunicate][GetCommunicateParams] hostAddr %p deviceAddr %p readCacheAddr %p devMemSize %u " \
106 : "buffLen %u flag %u", params.hostAddr, params.deviceAddr, params.readCacheAddr, devMem_.size(), buffLen_,
107 : flag_);
108 352 : return params;
109 : }
110 :
111 154 : HcclResult HDCommunicate::InitDevice(const struct HDCommunicateParams ¶ms)
112 : {
113 154 : CHK_PRT_RET((params.devMemSize == 0),
114 : HCCL_ERROR("[HDCommunicate][InitDevice]Invalid devMemSize=%u", params.devMemSize), HCCL_E_PARA);
115 154 : void *deviceAddr = reinterpret_cast<void *>(params.deviceAddr);
116 154 : CHK_PTR_NULL(deviceAddr);
117 154 : readCacheAddr_ = reinterpret_cast<void *>(params.readCacheAddr);
118 154 : CHK_PTR_NULL(readCacheAddr_);
119 154 : devMem_ = DeviceMem::create(deviceAddr, params.devMemSize);
120 154 : buffLen_ = params.buffLen;
121 154 : flag_ = params.flag;
122 :
123 154 : headCntAddr_ = HcclHdcGetControlWordAddr(devMem_.ptr(), devMem_.size(), HCCL_HDC_HEAD_POS);
124 154 : tailCntAddr_ = HcclHdcGetControlWordAddr(devMem_.ptr(), devMem_.size(), HCCL_HDC_TAIL_POS);
125 154 : isHost_ = false;
126 :
127 154 : HCCL_DEBUG(
128 : "[debug HDCommunicate][InitDevice] buffLen_=%u, flag_=%u, readCacheAddr_=%p, headCntAddr_=%p, " \
129 : "tailCntAddr_=%p, deviceAddr %p", buffLen_, flag_, readCacheAddr_, headCntAddr_, tailCntAddr_, devMem_.ptr());
130 154 : return HCCL_SUCCESS;
131 : }
132 :
133 193 : HcclResult HDCommunicate::Put(u32 offset, u32 length, u8 *value)
134 : {
135 193 : if (length == 0) {
136 0 : return HCCL_SUCCESS;
137 : }
138 193 : CHK_PTR_NULL(value);
139 193 : if (((flag_ == HCCL_HDC_TYPE_D2H) && isHost_) || ((flag_ == HCCL_HDC_TYPE_H2D) && !isHost_)) {
140 1 : HCCL_ERROR("[HDCommunicate][Put]Invalid usage, flag=%u, isHost=%d", flag_, isHost_);
141 1 : return HCCL_E_PARA;
142 : }
143 192 : CHK_PRT_RET((offset + length > buffLen_),
144 : HCCL_ERROR("[HDCommunicate][Put]Invalid length, offset=%u, length=%u", offset, length), HCCL_E_PARA);
145 192 : std::unique_lock<std::shared_mutex> lock(lock_);
146 192 : HcclResult ret = Write(offset, length, value);
147 192 : if (ret != HCCL_SUCCESS) {
148 0 : HCCL_ERROR("[HDCommunicate][Put]Write failed, offset=%u, length=%u", offset, length);
149 0 : return ret;
150 : }
151 192 : return HCCL_SUCCESS;
152 192 : }
153 :
154 359278547 : HcclResult HDCommunicate::Get(u32 offset, u32 length, u8 *value)
155 : {
156 359278547 : if (length == 0) {
157 0 : return HCCL_SUCCESS;
158 : }
159 359278547 : CHK_PTR_NULL(value);
160 359278547 : CHK_PRT_RET((offset + length > buffLen_),
161 : HCCL_ERROR("[HDCommunicate][Get]Invalid length, offset=%u, length=%u, befferLen=%u", offset, length, buffLen_),
162 : HCCL_E_PARA);
163 359278546 : std::shared_lock<std::shared_mutex> lock(lock_);
164 359278547 : HcclResult ret = Read(offset, length, value);
165 359278532 : if (ret != HCCL_SUCCESS) {
166 0 : HCCL_ERROR("[HDCommunicate][Get]Read failed, offset=%u, length=%u", offset, length);
167 0 : return ret;
168 : }
169 359278532 : return HCCL_SUCCESS;
170 359278532 : }
171 :
172 : #pragma GCC push_options
173 : #pragma GCC optimize("O0")
174 192 : HcclResult HDCommunicate::Write(u32 offset, u32 length, u8 *value)
175 : {
176 192 : if (length == 0) {
177 0 : return HCCL_SUCCESS;
178 : }
179 192 : CHK_PTR_NULL(value);
180 192 : u32 head = *headCntAddr_;
181 192 : head++;
182 192 : *headCntAddr_ = head;
183 192 : if (isHost_ && !supportDevMemReg_) {
184 0 : CHK_RET(hrtDrvMemCpy(devHeadCntAddr_, sizeof(u32), headCntAddr_, sizeof(u32)));
185 : }
186 :
187 192 : if (isHost_) {
188 42 : auto ret = memcpy_s(reinterpret_cast<u8 *>(hostMem_.ptr()) + offset,
189 42 : hostMem_.size() - HCCL_HDC_CONTROL_WORDS * sizeof(u32), value, length);
190 42 : CHK_PRT_RET(ret != EOK, HCCL_ERROR("[HDCommunicate][Write]memcpy_s failed, return[%d].", ret), HCCL_E_INTERNAL);
191 : } else {
192 150 : auto ret = memcpy_s(reinterpret_cast<u8 *>(devMem_.ptr()) + offset,
193 150 : devMem_.size() - HCCL_HDC_CONTROL_WORDS * sizeof(u32), value, length);
194 150 : CHK_PRT_RET(ret != EOK, HCCL_ERROR("[HDCommunicate][Write]memcpy_s failed, return[%d].", ret), HCCL_E_INTERNAL);
195 : }
196 192 : if (isHost_ && !supportDevMemReg_) {
197 0 : CHK_RET(hrtDrvMemCpy(reinterpret_cast<u8 *>(devMem_.ptr()) + offset,
198 : hostMem_.size() - HCCL_HDC_CONTROL_WORDS * sizeof(u32), value, length));
199 : }
200 :
201 : std::atomic_thread_fence(std::memory_order_seq_cst);
202 :
203 192 : u32 tail = *tailCntAddr_;
204 192 : tail++;
205 192 : *tailCntAddr_ = tail;
206 192 : if (isHost_ && !supportDevMemReg_) {
207 0 : CHK_RET(hrtDrvMemCpy(devTailCntAddr_, sizeof(u32), tailCntAddr_, sizeof(u32)));
208 : }
209 192 : return HCCL_SUCCESS;
210 : }
211 :
212 359278548 : HcclResult HDCommunicate::Read(u32 offset, u32 length, u8 *value)
213 : {
214 359278548 : if (length == 0) {
215 0 : return HCCL_SUCCESS;
216 : }
217 359278548 : CHK_PTR_NULL(value);
218 359278548 : u32 *cachedTailCntAddr = HcclHdcGetControlWordAddr(readCacheAddr_, devMem_.size(), HCCL_HDC_TAIL_POS);
219 359278540 : volatile u32 cachedTailCnt = *cachedTailCntAddr;
220 359278540 : volatile u32 tailCnt = 0;
221 359278540 : if (isHost_ && !supportDevMemReg_) {
222 0 : u32 tempTailCnt = 0;
223 0 : u32 *devSrcTailCntAddr = HcclHdcGetControlWordAddr(devMem_.ptr(), devMem_.size(), HCCL_HDC_TAIL_POS);
224 0 : CHK_RET(hrtDrvMemCpy(&tempTailCnt, sizeof(u32), devSrcTailCntAddr, sizeof(u32)));
225 0 : tailCnt = tempTailCnt;
226 0 : } else {
227 359278540 : tailCnt = *tailCntAddr_;
228 : }
229 359278540 : if (cachedTailCnt != tailCnt) {
230 : // 默认HDC超时时间为10s
231 94 : CHK_RET(UpdateCache(10));
232 : }
233 359278540 : auto ret = memcpy_s(value, length, static_cast<u8 *>(readCacheAddr_) + offset, length);
234 359278536 : CHK_PRT_RET(ret != EOK, HCCL_ERROR("[HDCommunicate][Read]memcpy_s failed, return[%d].", ret), HCCL_E_INTERNAL);
235 359278534 : return HCCL_SUCCESS;
236 : }
237 :
238 94 : HcclResult HDCommunicate::UpdateCache(u32 timeoutSec)
239 : {
240 94 : void *srcBaseAddr = isHost_ ? hostMem_.ptr() : devMem_.ptr();
241 94 : u32 *srcHeadCntAddr = HcclHdcGetControlWordAddr(srcBaseAddr, devMem_.size(), HCCL_HDC_HEAD_POS);
242 94 : u32 *srcTailCntAddr = HcclHdcGetControlWordAddr(srcBaseAddr, devMem_.size(), HCCL_HDC_TAIL_POS);
243 94 : u32 *devSrcHeadCntAddr = HcclHdcGetControlWordAddr(devMem_.ptr(), devMem_.size(), HCCL_HDC_HEAD_POS);
244 94 : u32 *devSrcTailCntAddr = HcclHdcGetControlWordAddr(devMem_.ptr(), devMem_.size(), HCCL_HDC_TAIL_POS);
245 94 : u32 *cachedHeadCntAddr = HcclHdcGetControlWordAddr(readCacheAddr_, devMem_.size(), HCCL_HDC_HEAD_POS);
246 94 : u32 *cachedTailCntAddr = HcclHdcGetControlWordAddr(readCacheAddr_, devMem_.size(), HCCL_HDC_TAIL_POS);
247 :
248 94 : s32 ret = 0;
249 94 : auto startTime = std::chrono::steady_clock::now();
250 94 : auto timeout = std::chrono::seconds(timeoutSec);
251 : while (1) {
252 94 : if (isHost_ && !supportDevMemReg_) {
253 : // step1: cache尾计数
254 0 : CHK_RET(hrtDrvMemCpy(cachedTailCntAddr, sizeof(u32), devSrcTailCntAddr, sizeof(u32)));
255 :
256 : // step2: cache数据
257 0 : CHK_RET(hrtDrvMemCpy(readCacheAddr_, devMem_.size() - HCCL_HDC_CONTROL_WORDS * sizeof(u32), devMem_.ptr(),
258 : devMem_.size() - HCCL_HDC_CONTROL_WORDS * sizeof(u32)));
259 :
260 : // step3：cache头计数
261 0 : CHK_RET(hrtDrvMemCpy(cachedHeadCntAddr, sizeof(u32), devSrcHeadCntAddr, sizeof(u32)));
262 0 : } else {
263 : // step1: cache尾计数
264 94 : ret = memcpy_s(cachedTailCntAddr, sizeof(u32), srcTailCntAddr, sizeof(u32));
265 94 : CHK_PRT_RET(ret != EOK, HCCL_ERROR("[HDCommunicate][UpdateCache]memcpy_s failed, return[%d].", ret),
266 : HCCL_E_INTERNAL);
267 :
268 : // step2: cache数据
269 94 : ret = memcpy_s(readCacheAddr_, devMem_.size() - HCCL_HDC_CONTROL_WORDS * sizeof(u32), srcBaseAddr,
270 94 : devMem_.size() - HCCL_HDC_CONTROL_WORDS * sizeof(u32));
271 94 : CHK_PRT_RET(ret != EOK, HCCL_ERROR("[HDCommunicate][UpdateCache]memcpy_s failed, return[%d].", ret),
272 : HCCL_E_INTERNAL);
273 :
274 : // step3：cache头计数
275 94 : ret = memcpy_s(cachedHeadCntAddr, sizeof(u32), srcHeadCntAddr, sizeof(u32));
276 94 : CHK_PRT_RET(ret != EOK, HCCL_ERROR("[HDCommunicate][UpdateCache]memcpy_s failed, return[%d].", ret),
277 : HCCL_E_INTERNAL);
278 : }
279 :
280 94 : volatile u32 cachedHeadCnt = *cachedHeadCntAddr;
281 94 : volatile u32 cachedTailCnt = *cachedTailCntAddr;
282 :
283 94 : if (cachedHeadCnt == cachedTailCnt) {
284 94 : break;
285 : }
286 0 : CHK_PRT_RET(((std::chrono::steady_clock::now() - startTime) >= timeout),
287 : HCCL_WARNING("[HDCommunicate][UpdateCache]get remote data timeout[%u s].", timeoutSec), HCCL_E_AGAIN);
288 0 : }
289 94 : return HCCL_SUCCESS;
290 : }
291 : #pragma GCC pop_options
292 :
293 866 : HcclResult HDCommunicate::AllocShm(u32 devid, DeviceMem &devShm, HostMem &hostShm)
294 : {
295 : // 共享内存size需要按照4K(4*1024=4096)对齐
296 866 : u32 size = (buffLen_ + HCCL_HDC_CONTROL_WORDS * sizeof(u32) + HCCL_SHM_ALIGN - 1) / HCCL_SHM_ALIGN * HCCL_SHM_ALIGN;
297 866 : CHK_RET(DeviceMem::alloc(devShm, size));
298 866 : CHK_RET(hrtMemSet(devShm.ptr(), size, size));
299 :
300 866 : if (supportDevMemReg_) {
301 866 : void *hostAddr = nullptr;
302 866 : CHK_RET(hrtHalHostRegister(devShm.ptr(), devShm.size(), DEV_SVM_MAP_HOST, devid, hostAddr));
303 :
304 866 : hostShm = HostMem::create(hostAddr, devShm.size());
305 : } else {
306 0 : hostShm = HostMem::alloc(devShm.size());
307 : }
308 866 : CHK_PTR_NULL(hostShm.ptr());
309 :
310 866 : auto ret = memset_s(hostShm.ptr(), hostShm.size(), 0, hostShm.size());
311 866 : CHK_PRT_RET(ret != EOK, HCCL_ERROR("[HDCommunicate][AllocShm]memset_s failed, return[%d].", ret), HCCL_E_INTERNAL);
312 :
313 866 : return HCCL_SUCCESS;
314 : }
315 :
316 866 : HcclResult HDCommunicate::AllocReadCache(u32 flag, void *&readCacheAddr)
317 : {
318 866 : if (flag == HCCL_HDC_TYPE_D2H) {
319 433 : hostCache_ = HostMem::alloc(hostMem_.size());
320 433 : CHK_PTR_NULL(hostCache_.ptr());
321 433 : auto ret = memset_s(hostCache_.ptr(), hostCache_.size(), 0, hostCache_.size());
322 433 : CHK_PRT_RET(ret != EOK, HCCL_ERROR("[HDCommunicate][AllocReadCache]memset_s failed, return[%d].", ret),
323 : HCCL_E_INTERNAL);
324 433 : readCacheAddr = hostCache_.ptr();
325 : } else {
326 433 : CHK_RET(DeviceMem::alloc(devCache_, devMem_.size()));
327 433 : CHK_RET(hrtMemSet(devCache_.ptr(), devCache_.size(), devCache_.size()));
328 433 : readCacheAddr = devCache_.ptr();
329 : }
330 866 : return HCCL_SUCCESS;
331 : }
332 : }
|
