Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "hdc.h"
12 : #include <chrono>
13 : #include "log.h"
14 : #include "ascend_hal.h"
15 : #include "orion_adapter_rts.h"
16 :
17 : namespace Hccl {
18 :
19 117 : HDCommunicate::HDCommunicate(u32 deviceLogicId, u32 flag, u32 buffLen)
20 117 : : deviceLogicId(deviceLogicId), flag(flag), buffLen(buffLen)
21 117 : {}
22 :
23 117 : HDCommunicate::~HDCommunicate()
24 : {
25 351 : HCCL_INFO("[~HDCommunicate]start hdc destroy");
26 117 : if ((devMem->GetAddr() != 0) && supportDevMemReg) {
27 2 : (void)halHostUnregister(reinterpret_cast<void *>(devMem->GetAddr()), deviceLogicId);
28 : }
29 117 : }
30 :
31 : // 在device中申请共享内存，其数据格式如下所示:
32 : // +---------------------+
33 : // | |
34 : // | content |
35 : // | |
36 : // +---------------------+
37 : // | head_cnt[u32] |
38 : // +---------------------+
39 : // | tail_cnt[u32] |
40 : // +---------------------+
41 : // 发送方更新content前，需要将head_cnt加1,更新数据后需要将tail_cnt加1;
42 : // 接收方在读取数据前判断共享内存中的tail_cnt和本地cache中的tailcnt是否一致，如不一致则需要更新本地cache;
43 : // 更新本地cache时，需要确保cache中head_cnt和tail_cnt一致，否则舍弃本次数据，继续更新cache直至一致;
44 :
45 :
46 : constexpr u32 HCCL_SHM_ALIGN = 4096;
47 : constexpr u32 HCCL_HDC_CONTROL_WORDS = 2;
48 : constexpr u32 HCCL_HDC_HEAD_POS = 2;
49 : constexpr u32 HCCL_HDC_TAIL_POS = 1;
50 :
51 9233707 : inline u32* HcclHdcGetControlWordAddr(void *base, u64 size, u32 pos)
52 : {
53 9233707 : return reinterpret_cast<u32 *>(reinterpret_cast<u8 *>((base)) + size - pos * sizeof(pos));
54 : }
55 :
56 115 : HcclResult HDCommunicate::Init()
57 : {
58 115 : CHK_RET(VerifyDeviceMemoryRegisterSupport());
59 :
60 115 : CHK_RET(AllocShm());
61 115 : CHK_RET(AllocReadCache());
62 :
63 115 : headCntAddr = HcclHdcGetControlWordAddr(reinterpret_cast<void *>(hostMem->GetAddr()), hostMem->GetSize(), HCCL_HDC_HEAD_POS);
64 115 : tailCntAddr = HcclHdcGetControlWordAddr(reinterpret_cast<void *>(hostMem->GetAddr()), hostMem->GetSize(), HCCL_HDC_TAIL_POS);
65 :
66 115 : devHeadCntAddr = HcclHdcGetControlWordAddr(reinterpret_cast<void *>(devMem->GetAddr()), devMem->GetSize(), HCCL_HDC_HEAD_POS);
67 115 : devTailCntAddr = HcclHdcGetControlWordAddr(reinterpret_cast<void *>(devMem->GetAddr()), devMem->GetSize(), HCCL_HDC_TAIL_POS);
68 345 : HCCL_INFO(
69 : "[HDCommunicate][Init] buffLen=%u, flag=%u, readCacheAddr=%p, devHeadCntAddr=%p, devTailCntAddr=%p",
70 : buffLen, flag, readCacheAddr, devHeadCntAddr, devTailCntAddr);
71 115 : return HCCL_SUCCESS;
72 : }
73 :
74 115 : HcclResult HDCommunicate::VerifyDeviceMemoryRegisterSupport()
75 : {
76 115 : supportDevMemReg = false;
77 115 : size_t outputLen = 0;
78 115 : struct supportFeaturePara input = { 0 };
79 115 : struct supportFeaturePara output = { 0 };
80 115 : s32 deviceId = HrtGetDevice();
81 115 : s32 logicDevId = 0;
82 : // 调用驱动接口前需将userDevId转换为logicDevId
83 115 : CHK_RET(HrtGetLogicDevIdByUserDevId(deviceId, logicDevId));
84 115 : deviceLogicId = static_cast<u32>(logicDevId);
85 115 : input.support_feature = CTRL_SUPPORT_PCIE_BAR_MEM_MASK;
86 115 : input.devid = static_cast<unsigned int>(logicDevId);
87 115 : halMemCtl(CTRL_TYPE_SUPPORT_FEATURE, &input, sizeof(struct supportFeaturePara), &output, &outputLen);
88 :
89 115 : if ((output.support_feature & CTRL_SUPPORT_PCIE_BAR_MEM_MASK) != 0) {
90 0 : supportDevMemReg = true;
91 : }
92 345 : HCCL_INFO("[HDCommunicate]supportDevMemReg[%d]", supportDevMemReg);
93 115 : return HCCL_SUCCESS;
94 : }
95 :
96 33 : struct HDCommunicateParams HDCommunicate::GetCommunicateParams() const
97 : {
98 33 : struct HDCommunicateParams params;
99 33 : params.hostAddr = reinterpret_cast<u64>(reinterpret_cast<void *>(hostMem->GetAddr()));
100 33 : params.deviceAddr = reinterpret_cast<u64>(reinterpret_cast<void *>(devMem->GetAddr()));
101 33 : params.readCacheAddr = reinterpret_cast<u64>(readCacheAddr);
102 33 : params.devMemSize = devMem->GetSize();
103 33 : params.buffLen = buffLen;
104 33 : params.flag = flag;
105 33 : return params;
106 : }
107 : // 为了按照调用顺序执行，防止编译器优化导致产生异常行为
108 : #pragma GCC push_options
109 : #pragma GCC optimize("O0")
110 15 : HcclResult HDCommunicate::Put(u32 offset, u32 length, u8 *value)
111 : {
112 15 : if (length == 0) {
113 0 : return HCCL_SUCCESS;
114 : }
115 15 : CHK_PTR_NULL(value);
116 :
117 15 : if (flag == HCCLV2_HDC_TYPE_D2H) {
118 0 : HCCL_ERROR("[HDCommunicate][Put]Invalid usage, flag=%u", flag);
119 0 : return HCCL_E_PARA;
120 : }
121 15 : CHK_PRT_RET((offset + length > buffLen),
122 : HCCL_ERROR("[HDCommunicate][Put]Invalid length, offset=%u, length=%u", offset, length), HCCL_E_PARA);
123 15 : std::lock_guard<std::mutex> lock(shmLock);
124 15 : return Write(offset, length, value);
125 15 : }
126 :
127 4606791 : HcclResult HDCommunicate::Get(u32 offset, u32 length, u8 *value)
128 : {
129 4606791 : if (length == 0) {
130 0 : return HCCL_SUCCESS;
131 : }
132 4606791 : CHK_PTR_NULL(value);
133 4606791 : CHK_PRT_RET((offset + length > buffLen),
134 : HCCL_ERROR("[HDCommunicate][Get]Invalid length, offset=%u, length=%u, befferLen=%u", offset, length, buffLen),
135 : HCCL_E_PARA);
136 4606791 : std::lock_guard<std::mutex> lock(shmLock);
137 4606791 : return Read(offset, length, value);
138 4606791 : }
139 :
140 0 : HcclResult HrtDrvMemCpy(void *dst, uint64_t destMax, const void *src, uint64_t count)
141 : {
142 : // 参数有效性检查
143 0 : CHK_PTR_NULL(dst);
144 0 : CHK_PTR_NULL(src);
145 :
146 0 : uint64_t dstAddr = reinterpret_cast<uintptr_t>(dst);
147 0 : uint64_t srcAddr = reinterpret_cast<uintptr_t>(const_cast<void *>(src));
148 0 : drvError_t ret = drvMemcpy(dstAddr, destMax, srcAddr, count);
149 0 : CHK_PRT_RET(ret != DRV_ERROR_NONE, HCCL_ERROR("errNo[0x%016llx] hrtDrvMemCpy fail,"
150 : "return[%d].", HCCL_ERROR_CODE(HCCL_E_DRV), ret), HCCL_E_DRV);
151 :
152 0 : return HCCL_SUCCESS;
153 : }
154 :
155 15 : HcclResult HDCommunicate::Write(u32 offset, u32 length, u8 *value)
156 : {
157 15 : if (length == 0) {
158 0 : return HCCL_SUCCESS;
159 : }
160 15 : CHK_PTR_NULL(value);
161 :
162 15 : u32 head = *headCntAddr;
163 15 : head++;
164 15 : *headCntAddr = head;
165 15 : if (!supportDevMemReg) {
166 15 : CHK_RET(HrtDrvMemCpy(devHeadCntAddr, sizeof(u32), headCntAddr, sizeof(u32)));
167 : }
168 :
169 15 : auto ret = memcpy_s(reinterpret_cast<u8 *>(hostMem->GetAddr()) + offset,
170 15 : hostMem->GetSize() - HCCL_HDC_CONTROL_WORDS * sizeof(u32), value, length);
171 15 : CHK_PRT_RET(ret != EOK, HCCL_ERROR("[HDCommunicate][Write]memcpy_s failed, return[%d].", ret), HCCL_E_INTERNAL);
172 :
173 15 : if (!supportDevMemReg) {
174 15 : CHK_RET(HrtDrvMemCpy(reinterpret_cast<u8 *>(devMem->GetAddr()) + offset,
175 : hostMem->GetSize() - HCCL_HDC_CONTROL_WORDS * sizeof(u32), value, length));
176 : }
177 :
178 15 : u32 tail = *tailCntAddr;
179 15 : tail++;
180 15 : *tailCntAddr = tail;
181 15 : if (!supportDevMemReg) {
182 15 : CHK_RET(HrtDrvMemCpy(devTailCntAddr, sizeof(u32), tailCntAddr, sizeof(u32)));
183 : }
184 15 : return HCCL_SUCCESS;
185 : }
186 :
187 4606791 : HcclResult HDCommunicate::Read(u32 offset, u32 length, u8 *value)
188 : {
189 4606791 : if (length == 0) {
190 0 : return HCCL_SUCCESS;
191 : }
192 4606791 : CHK_PTR_NULL(value);
193 4606791 : u32 *cachedTailCntAddr = HcclHdcGetControlWordAddr(readCacheAddr, devMem->GetSize(), HCCL_HDC_TAIL_POS);
194 4606791 : volatile u32 cachedTailCnt = *cachedTailCntAddr;
195 4606791 : volatile u32 tailCnt = 0;
196 4606791 : if (!supportDevMemReg) {
197 4606791 : u32 tempTailCnt = 0;
198 4606791 : u32 *devSrcTailCntAddr = HcclHdcGetControlWordAddr(reinterpret_cast<void *>(devMem->GetAddr()), devMem->GetSize(), HCCL_HDC_TAIL_POS);
199 4606791 : CHK_RET(HrtDrvMemCpy(&tempTailCnt, sizeof(u32), devSrcTailCntAddr, sizeof(u32)));
200 4606791 : tailCnt = tempTailCnt;
201 : } else {
202 0 : tailCnt = *tailCntAddr;
203 : }
204 4606791 : if (cachedTailCnt != tailCnt) {
205 : // 默认HDC超时时间为10s
206 11 : CHK_RET(UpdateCache(10));
207 : }
208 4606791 : auto ret = memcpy_s(value, length, static_cast<u8 *>(readCacheAddr) + offset, length);
209 4606791 : CHK_PRT_RET(ret != EOK, HCCL_ERROR("[HDCommunicate][Read]memcpy_s failed, return[%d].", ret), HCCL_E_INTERNAL);
210 4606791 : return HCCL_SUCCESS;
211 : }
212 :
213 11 : HcclResult HDCommunicate::UpdateCache(u32 timeoutSec)
214 : {
215 11 : void *srcBaseAddr = reinterpret_cast<void *>(hostMem->GetAddr());
216 11 : u32 *srcHeadCntAddr = HcclHdcGetControlWordAddr(srcBaseAddr, devMem->GetSize(), HCCL_HDC_HEAD_POS);
217 11 : u32 *srcTailCntAddr = HcclHdcGetControlWordAddr(srcBaseAddr, devMem->GetSize(), HCCL_HDC_TAIL_POS);
218 11 : u32 *devSrcHeadCntAddr = HcclHdcGetControlWordAddr(reinterpret_cast<void *>(devMem->GetAddr()), devMem->GetSize(), HCCL_HDC_HEAD_POS);
219 11 : u32 *devSrcTailCntAddr = HcclHdcGetControlWordAddr(reinterpret_cast<void *>(devMem->GetAddr()), devMem->GetSize(), HCCL_HDC_TAIL_POS);
220 11 : u32 *cachedHeadCntAddr = HcclHdcGetControlWordAddr(readCacheAddr, devMem->GetSize(), HCCL_HDC_HEAD_POS);
221 11 : u32 *cachedTailCntAddr = HcclHdcGetControlWordAddr(readCacheAddr, devMem->GetSize(), HCCL_HDC_TAIL_POS);
222 :
223 11 : s32 ret = 0;
224 11 : auto startTime = std::chrono::steady_clock::now();
225 11 : auto timeout = std::chrono::seconds(timeoutSec);
226 : while (1) {
227 11 : if (!supportDevMemReg) {
228 : // step1: cache尾计数
229 11 : CHK_RET(HrtDrvMemCpy(cachedTailCntAddr, sizeof(u32), devSrcTailCntAddr, sizeof(u32)));
230 :
231 : // step2: cache数据
232 11 : CHK_RET(HrtDrvMemCpy(readCacheAddr, devMem->GetSize() - HCCL_HDC_CONTROL_WORDS * sizeof(u32), reinterpret_cast<void *>(devMem->GetAddr()),
233 : devMem->GetSize() - HCCL_HDC_CONTROL_WORDS * sizeof(u32)));
234 :
235 : // step3：cache头计数
236 11 : CHK_RET(HrtDrvMemCpy(cachedHeadCntAddr, sizeof(u32), devSrcHeadCntAddr, sizeof(u32)));
237 : } else {
238 : // step1: cache尾计数
239 0 : ret = memcpy_s(cachedTailCntAddr, sizeof(u32), srcTailCntAddr, sizeof(u32));
240 0 : CHK_PRT_RET(ret != EOK, HCCL_ERROR("[HDCommunicate][UpdateCache]memcpy_s failed, return[%d].", ret),
241 : HCCL_E_INTERNAL);
242 :
243 : // step2: cache数据
244 0 : ret = memcpy_s(readCacheAddr, devMem->GetSize() - HCCL_HDC_CONTROL_WORDS * sizeof(u32), srcBaseAddr,
245 0 : devMem->GetSize() - HCCL_HDC_CONTROL_WORDS * sizeof(u32));
246 0 : CHK_PRT_RET(ret != EOK, HCCL_ERROR("[HDCommunicate][UpdateCache]memcpy_s failed, return[%d].", ret),
247 : HCCL_E_INTERNAL);
248 :
249 : // step3：cache头计数
250 0 : ret = memcpy_s(cachedHeadCntAddr, sizeof(u32), srcHeadCntAddr, sizeof(u32));
251 0 : CHK_PRT_RET(ret != EOK, HCCL_ERROR("[HDCommunicate][UpdateCache]memcpy_s failed, return[%d].", ret),
252 : HCCL_E_INTERNAL);
253 : }
254 :
255 11 : volatile u32 cachedHeadCnt = *cachedHeadCntAddr;
256 11 : volatile u32 cachedTailCnt = *cachedTailCntAddr;
257 :
258 11 : if (cachedHeadCnt == cachedTailCnt) {
259 11 : break;
260 : }
261 0 : CHK_PRT_RET(((std::chrono::steady_clock::now() - startTime) >= timeout),
262 : HCCL_WARNING("[HDCommunicate][UpdateCache]get remote data timeout[%u s].", timeoutSec), HCCL_E_AGAIN);
263 0 : }
264 11 : return HCCL_SUCCESS;
265 : }
266 : #pragma GCC pop_options
267 :
268 115 : HcclResult HDCommunicate::AllocShm()
269 : {
270 : // 共享内存size需要按照4K(4*1024=4096)对齐
271 115 : size_t size = (buffLen + HCCL_HDC_CONTROL_WORDS * sizeof(u32) + HCCL_SHM_ALIGN - 1) / HCCL_SHM_ALIGN * HCCL_SHM_ALIGN;
272 115 : devMem = std::make_unique<DevBuffer>(size);
273 115 : HrtMemset(reinterpret_cast<void *>(devMem->GetAddr()), devMem->GetSize(), devMem->GetSize());
274 :
275 115 : if (supportDevMemReg) {
276 0 : void *hostAddr = nullptr;
277 0 : halHostRegister(reinterpret_cast<void *>(devMem->GetAddr()), devMem->GetSize(), DEV_SVM_MAP_HOST, deviceLogicId, &hostAddr);
278 :
279 0 : hostMem = std::make_unique<HostBuffer>(reinterpret_cast<uintptr_t>(hostAddr), devMem->GetSize());
280 : } else {
281 115 : hostMem = std::make_unique<HostBuffer>(devMem->GetSize());
282 : }
283 :
284 115 : auto ret = memset_s(reinterpret_cast<void *>(hostMem->GetAddr()), hostMem->GetSize(), 0, hostMem->GetSize());
285 115 : CHK_PRT_RET(ret != EOK, HCCL_ERROR("[HDCommunicate][AllocShm]memset_s failed, return[%d].", ret), HCCL_E_INTERNAL);
286 :
287 115 : return HCCL_SUCCESS;
288 : }
289 :
290 115 : HcclResult HDCommunicate::AllocReadCache()
291 : {
292 115 : if (flag == HCCLV2_HDC_TYPE_D2H) {
293 57 : hostCache = std::make_unique<HostBuffer>(hostMem->GetSize());
294 57 : auto ret = memset_s(reinterpret_cast<void *>(hostCache->GetAddr()), hostCache->GetSize(), 0, hostCache->GetSize());
295 57 : CHK_PRT_RET(ret != EOK, HCCL_ERROR("[HDCommunicate][AllocReadCache]memset_s failed, return[%d].", ret),
296 : HCCL_E_INTERNAL);
297 57 : readCacheAddr = reinterpret_cast<void *>(hostCache->GetAddr());
298 : } else {
299 58 : devCache = std::make_unique<DevBuffer>(devMem->GetSize());
300 58 : HrtMemset(reinterpret_cast<void *>(devCache->GetAddr()), devCache->GetSize(), devCache->GetSize());
301 58 : readCacheAddr = reinterpret_cast<void *>(devCache->GetAddr());
302 : }
303 115 : return HCCL_SUCCESS;
304 : }
305 : }
|
