Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "hdc_pub.h"
12 : #include "log.h"
13 : #include "ascend_hal.h"
14 : #include "adapter_hal.h"
15 : #include "adapter_rts.h"
16 : #include <atomic>
17 : #include <chrono>
18 :
19 : namespace hccl {
20 876 : HDCommunicate::HDCommunicate(u32 deviceLogicId, u32 flag, u32 buffLen)
21 876 : : deviceLogicId_(deviceLogicId), flag_(flag), buffLen_(buffLen)
22 876 : {}
23 192 : HDCommunicate::HDCommunicate() : deviceLogicId_(INVALID_UINT), flag_(0), buffLen_(0) {}
24 :
25 1068 : HDCommunicate::~HDCommunicate()
26 : {
27 1068 : if (isHost_ && devMem_.ptr() && supportDevMemReg_) {
28 864 : (void)hrtHalHostUnregister(devMem_.ptr(), deviceLogicId_);
29 : }
30 1067 : }
31 :
32 : // 在device中申请共享内存，其数据格式如下所示:
33 : // +---------------------+
34 : // | |
35 : // | content |
36 : // | |
37 : // +---------------------+
38 : // | head_cnt[u32] |
39 : // +---------------------+
40 : // | tail_cnt[u32] |
41 : // +---------------------+
42 : // 发送方更新content前，需要将head_cnt加1,更新数据后需要将tail_cnt加1;
43 : // 接收方在读取数据前判断共享内存中的tail_cnt和本地cache中的tailcnt是否一致，如不一致则需要更新本地cache;
44 : // 更新本地cache时，需要确保cache中head_cnt和tail_cnt一致，否则舍弃本次数据，继续更新cache直至一致;
45 :
46 :
47 : #define HCCL_SHM_ALIGN 4096
48 : #define HCCL_HDC_CONTROL_WORDS 2
49 : #define HCCL_HDC_HEAD_POS 2
50 : #define HCCL_HDC_TAIL_POS 1
51 :
52 362166360 : inline u32* HcclHdcGetControlWordAddr(void *base, u64 size, u32 pos)
53 : {
54 362166360 : return reinterpret_cast<u32 *>(reinterpret_cast<u8 *>((base)) + size - pos * sizeof(pos));
55 : }
56 :
57 864 : HcclResult HDCommunicate::InitHost()
58 : {
59 864 : CHK_RET(VerifyDeviceMemoryRegisterSupport());
60 :
61 864 : CHK_RET(AllocShm(deviceLogicId_, devMem_, hostMem_));
62 864 : CHK_RET(AllocReadCache(flag_, readCacheAddr_));
63 :
64 864 : headCntAddr_ = HcclHdcGetControlWordAddr(hostMem_.ptr(), hostMem_.size(), HCCL_HDC_HEAD_POS);
65 864 : tailCntAddr_ = HcclHdcGetControlWordAddr(hostMem_.ptr(), hostMem_.size(), HCCL_HDC_TAIL_POS);
66 :
67 864 : devHeadCntAddr_ = HcclHdcGetControlWordAddr(devMem_.ptr(), devMem_.size(), HCCL_HDC_HEAD_POS);
68 864 : devTailCntAddr_ = HcclHdcGetControlWordAddr(devMem_.ptr(), devMem_.size(), HCCL_HDC_TAIL_POS);
69 864 : return HCCL_SUCCESS;
70 : }
71 :
72 :
73 864 : HcclResult HDCommunicate::VerifyDeviceMemoryRegisterSupport()
74 : {
75 864 : supportDevMemReg_ = false;
76 864 : size_t outputLen = 0;
77 864 : struct supportFeaturePara input = { 0 };
78 864 : struct supportFeaturePara output = { 0 };
79 864 : s32 deviceId = 0;
80 864 : CHK_RET(hrtGetDevice(&deviceId));
81 864 : input.support_feature = CTRL_SUPPORT_PCIE_BAR_MEM_MASK;
82 864 : input.devid = static_cast<unsigned int>(deviceId);
83 864 : CHK_RET(hrtHalMemCtl(CTRL_TYPE_SUPPORT_FEATURE, &input, sizeof(struct supportFeaturePara), &output, &outputLen));
84 :
85 864 : if ((output.support_feature & CTRL_SUPPORT_PCIE_BAR_MEM_MASK) != 0) {
86 864 : supportDevMemReg_ = true;
87 : }
88 864 : HCCL_INFO("[HDCommunicate]supportDevMemReg_ %d deviceId %d", supportDevMemReg_, input.devid);
89 864 : return HCCL_SUCCESS;
90 : }
91 :
92 350 : struct HDCommunicateParams HDCommunicate::GetCommunicateParams()
93 : {
94 350 : struct HDCommunicateParams params;
95 350 : params.hostAddr = reinterpret_cast<u64>(hostMem_.ptr());
96 350 : params.deviceAddr = reinterpret_cast<u64>(devMem_.ptr());
97 350 : params.readCacheAddr = reinterpret_cast<u64>(readCacheAddr_);
98 350 : params.devMemSize = devMem_.size();
99 350 : params.buffLen = buffLen_;
100 350 : params.flag = flag_;
101 350 : HCCL_DEBUG("[HDCommunicate][GetCommunicateParams] hostAddr %p deviceAddr %p readCacheAddr %p devMemSize %u " \
102 : "buffLen %u flag %u", params.hostAddr, params.deviceAddr, params.readCacheAddr, devMem_.size(), buffLen_,
103 : flag_);
104 350 : return params;
105 : }
106 :
107 154 : HcclResult HDCommunicate::InitDevice(const struct HDCommunicateParams ¶ms)
108 : {
109 154 : CHK_PRT_RET((params.devMemSize == 0),
110 : HCCL_ERROR("[HDCommunicate][InitDevice]Invalid devMemSize=%u", params.devMemSize), HCCL_E_PARA);
111 154 : void *deviceAddr = reinterpret_cast<void *>(params.deviceAddr);
112 154 : CHK_PTR_NULL(deviceAddr);
113 154 : readCacheAddr_ = reinterpret_cast<void *>(params.readCacheAddr);
114 154 : CHK_PTR_NULL(readCacheAddr_);
115 154 : devMem_ = DeviceMem::create(deviceAddr, params.devMemSize);
116 154 : buffLen_ = params.buffLen;
117 154 : flag_ = params.flag;
118 :
119 154 : headCntAddr_ = HcclHdcGetControlWordAddr(devMem_.ptr(), devMem_.size(), HCCL_HDC_HEAD_POS);
120 154 : tailCntAddr_ = HcclHdcGetControlWordAddr(devMem_.ptr(), devMem_.size(), HCCL_HDC_TAIL_POS);
121 154 : isHost_ = false;
122 :
123 154 : HCCL_DEBUG(
124 : "[debug HDCommunicate][InitDevice] buffLen_=%u, flag_=%u, readCacheAddr_=%p, headCntAddr_=%p, " \
125 : "tailCntAddr_=%p, deviceAddr %p", buffLen_, flag_, readCacheAddr_, headCntAddr_, tailCntAddr_, devMem_.ptr());
126 154 : return HCCL_SUCCESS;
127 : }
128 :
129 193 : HcclResult HDCommunicate::Put(u32 offset, u32 length, u8 *value)
130 : {
131 193 : if (length == 0) {
132 0 : return HCCL_SUCCESS;
133 : }
134 193 : CHK_PTR_NULL(value);
135 193 : if (((flag_ == HCCL_HDC_TYPE_D2H) && isHost_) || ((flag_ == HCCL_HDC_TYPE_H2D) && !isHost_)) {
136 1 : HCCL_ERROR("[HDCommunicate][Put]Invalid usage, flag=%u, isHost=%d", flag_, isHost_);
137 1 : return HCCL_E_PARA;
138 : }
139 192 : CHK_PRT_RET((offset + length > buffLen_),
140 : HCCL_ERROR("[HDCommunicate][Put]Invalid length, offset=%u, length=%u", offset, length), HCCL_E_PARA);
141 192 : std::unique_lock<std::shared_mutex> lock(lock_);
142 192 : HcclResult ret = Write(offset, length, value);
143 192 : if (ret != HCCL_SUCCESS) {
144 0 : HCCL_ERROR("[HDCommunicate][Put]Write failed, offset=%u, length=%u", offset, length);
145 0 : return ret;
146 : }
147 192 : return HCCL_SUCCESS;
148 192 : }
149 :
150 362162094 : HcclResult HDCommunicate::Get(u32 offset, u32 length, u8 *value)
151 : {
152 362162094 : if (length == 0) {
153 0 : return HCCL_SUCCESS;
154 : }
155 362162094 : CHK_PTR_NULL(value);
156 362162094 : CHK_PRT_RET((offset + length > buffLen_),
157 : HCCL_ERROR("[HDCommunicate][Get]Invalid length, offset=%u, length=%u, befferLen=%u", offset, length, buffLen_),
158 : HCCL_E_PARA);
159 362162093 : std::shared_lock<std::shared_mutex> lock(lock_);
160 362162090 : HcclResult ret = Read(offset, length, value);
161 362162075 : if (ret != HCCL_SUCCESS) {
162 0 : HCCL_ERROR("[HDCommunicate][Get]Read failed, offset=%u, length=%u", offset, length);
163 0 : return ret;
164 : }
165 362162075 : return HCCL_SUCCESS;
166 362162075 : }
167 :
168 : #pragma GCC push_options
169 : #pragma GCC optimize("O0")
170 192 : HcclResult HDCommunicate::Write(u32 offset, u32 length, u8 *value)
171 : {
172 192 : if (length == 0) {
173 0 : return HCCL_SUCCESS;
174 : }
175 192 : CHK_PTR_NULL(value);
176 192 : u32 head = *headCntAddr_;
177 192 : head++;
178 192 : *headCntAddr_ = head;
179 192 : if (isHost_ && !supportDevMemReg_) {
180 0 : CHK_RET(hrtDrvMemCpy(devHeadCntAddr_, sizeof(u32), headCntAddr_, sizeof(u32)));
181 : }
182 :
183 192 : if (isHost_) {
184 42 : auto ret = memcpy_s(reinterpret_cast<u8 *>(hostMem_.ptr()) + offset,
185 42 : hostMem_.size() - HCCL_HDC_CONTROL_WORDS * sizeof(u32), value, length);
186 42 : CHK_PRT_RET(ret != EOK, HCCL_ERROR("[HDCommunicate][Write]memcpy_s failed, return[%d].", ret), HCCL_E_INTERNAL);
187 : } else {
188 150 : auto ret = memcpy_s(reinterpret_cast<u8 *>(devMem_.ptr()) + offset,
189 150 : devMem_.size() - HCCL_HDC_CONTROL_WORDS * sizeof(u32), value, length);
190 150 : CHK_PRT_RET(ret != EOK, HCCL_ERROR("[HDCommunicate][Write]memcpy_s failed, return[%d].", ret), HCCL_E_INTERNAL);
191 : }
192 192 : if (isHost_ && !supportDevMemReg_) {
193 0 : CHK_RET(hrtDrvMemCpy(reinterpret_cast<u8 *>(devMem_.ptr()) + offset,
194 : hostMem_.size() - HCCL_HDC_CONTROL_WORDS * sizeof(u32), value, length));
195 : }
196 :
197 : std::atomic_thread_fence(std::memory_order_seq_cst);
198 :
199 192 : u32 tail = *tailCntAddr_;
200 192 : tail++;
201 192 : *tailCntAddr_ = tail;
202 192 : if (isHost_ && !supportDevMemReg_) {
203 0 : CHK_RET(hrtDrvMemCpy(devTailCntAddr_, sizeof(u32), tailCntAddr_, sizeof(u32)));
204 : }
205 192 : return HCCL_SUCCESS;
206 : }
207 :
208 362162086 : HcclResult HDCommunicate::Read(u32 offset, u32 length, u8 *value)
209 : {
210 362162086 : if (length == 0) {
211 0 : return HCCL_SUCCESS;
212 : }
213 362162086 : CHK_PTR_NULL(value);
214 362162086 : u32 *cachedTailCntAddr = HcclHdcGetControlWordAddr(readCacheAddr_, devMem_.size(), HCCL_HDC_TAIL_POS);
215 362162079 : volatile u32 cachedTailCnt = *cachedTailCntAddr;
216 362162079 : volatile u32 tailCnt = 0;
217 362162079 : if (isHost_ && !supportDevMemReg_) {
218 0 : u32 tempTailCnt = 0;
219 0 : u32 *devSrcTailCntAddr = HcclHdcGetControlWordAddr(devMem_.ptr(), devMem_.size(), HCCL_HDC_TAIL_POS);
220 0 : CHK_RET(hrtDrvMemCpy(&tempTailCnt, sizeof(u32), devSrcTailCntAddr, sizeof(u32)));
221 0 : tailCnt = tempTailCnt;
222 0 : } else {
223 362162079 : tailCnt = *tailCntAddr_;
224 : }
225 362162079 : if (cachedTailCnt != tailCnt) {
226 : // 默认HDC超时时间为10s
227 94 : CHK_RET(UpdateCache(10));
228 : }
229 362162079 : auto ret = memcpy_s(value, length, static_cast<u8 *>(readCacheAddr_) + offset, length);
230 362162080 : CHK_PRT_RET(ret != EOK, HCCL_ERROR("[HDCommunicate][Read]memcpy_s failed, return[%d].", ret), HCCL_E_INTERNAL);
231 362162080 : return HCCL_SUCCESS;
232 : }
233 :
234 94 : HcclResult HDCommunicate::UpdateCache(u32 timeoutSec)
235 : {
236 94 : void *srcBaseAddr = isHost_ ? hostMem_.ptr() : devMem_.ptr();
237 94 : u32 *srcHeadCntAddr = HcclHdcGetControlWordAddr(srcBaseAddr, devMem_.size(), HCCL_HDC_HEAD_POS);
238 94 : u32 *srcTailCntAddr = HcclHdcGetControlWordAddr(srcBaseAddr, devMem_.size(), HCCL_HDC_TAIL_POS);
239 94 : u32 *devSrcHeadCntAddr = HcclHdcGetControlWordAddr(devMem_.ptr(), devMem_.size(), HCCL_HDC_HEAD_POS);
240 94 : u32 *devSrcTailCntAddr = HcclHdcGetControlWordAddr(devMem_.ptr(), devMem_.size(), HCCL_HDC_TAIL_POS);
241 94 : u32 *cachedHeadCntAddr = HcclHdcGetControlWordAddr(readCacheAddr_, devMem_.size(), HCCL_HDC_HEAD_POS);
242 94 : u32 *cachedTailCntAddr = HcclHdcGetControlWordAddr(readCacheAddr_, devMem_.size(), HCCL_HDC_TAIL_POS);
243 :
244 94 : s32 ret = 0;
245 94 : auto startTime = std::chrono::steady_clock::now();
246 94 : auto timeout = std::chrono::seconds(timeoutSec);
247 : while (1) {
248 94 : if (isHost_ && !supportDevMemReg_) {
249 : // step1: cache尾计数
250 0 : CHK_RET(hrtDrvMemCpy(cachedTailCntAddr, sizeof(u32), devSrcTailCntAddr, sizeof(u32)));
251 :
252 : // step2: cache数据
253 0 : CHK_RET(hrtDrvMemCpy(readCacheAddr_, devMem_.size() - HCCL_HDC_CONTROL_WORDS * sizeof(u32), devMem_.ptr(),
254 : devMem_.size() - HCCL_HDC_CONTROL_WORDS * sizeof(u32)));
255 :
256 : // step3：cache头计数
257 0 : CHK_RET(hrtDrvMemCpy(cachedHeadCntAddr, sizeof(u32), devSrcHeadCntAddr, sizeof(u32)));
258 0 : } else {
259 : // step1: cache尾计数
260 94 : ret = memcpy_s(cachedTailCntAddr, sizeof(u32), srcTailCntAddr, sizeof(u32));
261 94 : CHK_PRT_RET(ret != EOK, HCCL_ERROR("[HDCommunicate][UpdateCache]memcpy_s failed, return[%d].", ret),
262 : HCCL_E_INTERNAL);
263 :
264 : // step2: cache数据
265 94 : ret = memcpy_s(readCacheAddr_, devMem_.size() - HCCL_HDC_CONTROL_WORDS * sizeof(u32), srcBaseAddr,
266 94 : devMem_.size() - HCCL_HDC_CONTROL_WORDS * sizeof(u32));
267 94 : CHK_PRT_RET(ret != EOK, HCCL_ERROR("[HDCommunicate][UpdateCache]memcpy_s failed, return[%d].", ret),
268 : HCCL_E_INTERNAL);
269 :
270 : // step3：cache头计数
271 94 : ret = memcpy_s(cachedHeadCntAddr, sizeof(u32), srcHeadCntAddr, sizeof(u32));
272 94 : CHK_PRT_RET(ret != EOK, HCCL_ERROR("[HDCommunicate][UpdateCache]memcpy_s failed, return[%d].", ret),
273 : HCCL_E_INTERNAL);
274 : }
275 :
276 94 : volatile u32 cachedHeadCnt = *cachedHeadCntAddr;
277 94 : volatile u32 cachedTailCnt = *cachedTailCntAddr;
278 :
279 94 : if (cachedHeadCnt == cachedTailCnt) {
280 94 : break;
281 : }
282 0 : CHK_PRT_RET(((std::chrono::steady_clock::now() - startTime) >= timeout),
283 : HCCL_WARNING("[HDCommunicate][UpdateCache]get remote data timeout[%u s].", timeoutSec), HCCL_E_AGAIN);
284 0 : }
285 94 : return HCCL_SUCCESS;
286 : }
287 : #pragma GCC pop_options
288 :
289 864 : HcclResult HDCommunicate::AllocShm(u32 devid, DeviceMem &devShm, HostMem &hostShm)
290 : {
291 : // 共享内存size需要按照4K(4*1024=4096)对齐
292 864 : u32 size = (buffLen_ + HCCL_HDC_CONTROL_WORDS * sizeof(u32) + HCCL_SHM_ALIGN - 1) / HCCL_SHM_ALIGN * HCCL_SHM_ALIGN;
293 864 : CHK_RET(DeviceMem::alloc(devShm, size));
294 864 : CHK_RET(hrtMemSet(devShm.ptr(), size, size));
295 :
296 864 : if (supportDevMemReg_) {
297 864 : void *hostAddr = nullptr;
298 864 : CHK_RET(hrtHalHostRegister(devShm.ptr(), devShm.size(), DEV_SVM_MAP_HOST, devid, hostAddr));
299 :
300 864 : hostShm = HostMem::create(hostAddr, devShm.size());
301 : } else {
302 0 : hostShm = HostMem::alloc(devShm.size());
303 : }
304 864 : CHK_PTR_NULL(hostShm.ptr());
305 :
306 864 : auto ret = memset_s(hostShm.ptr(), hostShm.size(), 0, hostShm.size());
307 864 : CHK_PRT_RET(ret != EOK, HCCL_ERROR("[HDCommunicate][AllocShm]memset_s failed, return[%d].", ret), HCCL_E_INTERNAL);
308 :
309 864 : return HCCL_SUCCESS;
310 : }
311 :
312 864 : HcclResult HDCommunicate::AllocReadCache(u32 flag, void *&readCacheAddr)
313 : {
314 864 : if (flag == HCCL_HDC_TYPE_D2H) {
315 432 : hostCache_ = HostMem::alloc(hostMem_.size());
316 432 : CHK_PTR_NULL(hostCache_.ptr());
317 432 : auto ret = memset_s(hostCache_.ptr(), hostCache_.size(), 0, hostCache_.size());
318 432 : CHK_PRT_RET(ret != EOK, HCCL_ERROR("[HDCommunicate][AllocReadCache]memset_s failed, return[%d].", ret),
319 : HCCL_E_INTERNAL);
320 432 : readCacheAddr = hostCache_.ptr();
321 : } else {
322 432 : CHK_RET(DeviceMem::alloc(devCache_, devMem_.size()));
323 432 : CHK_RET(hrtMemSet(devCache_.ptr(), devCache_.size(), devCache_.size()));
324 432 : readCacheAddr = devCache_.ptr();
325 : }
326 864 : return HCCL_SUCCESS;
327 : }
328 : }
|
