Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "hdc.h"
12 : #include <chrono>
13 : #include "log.h"
14 : #include "ascend_hal.h"
15 : #include "orion_adapter_rts.h"
16 :
17 : namespace Hccl {
18 :
19 117 : HDCommunicate::HDCommunicate(u32 deviceLogicId, u32 flag, u32 buffLen)
20 117 : : deviceLogicId(deviceLogicId), flag(flag), buffLen(buffLen)
21 117 : {}
22 :
23 117 : HDCommunicate::~HDCommunicate()
24 : {
25 351 : HCCL_INFO("[~HDCommunicate]start hdc destroy");
26 117 : if ((devMem->GetAddr() != 0) && supportDevMemReg) {
27 2 : (void)halHostUnregister(reinterpret_cast<void *>(devMem->GetAddr()), deviceLogicId);
28 : }
29 117 : }
30 :
31 : // 在device中申请共享内存，其数据格式如下所示:
32 : // +---------------------+
33 : // | |
34 : // | content |
35 : // | |
36 : // +---------------------+
37 : // | head_cnt[u32] |
38 : // +---------------------+
39 : // | tail_cnt[u32] |
40 : // +---------------------+
41 : // 发送方更新content前，需要将head_cnt加1,更新数据后需要将tail_cnt加1;
42 : // 接收方在读取数据前判断共享内存中的tail_cnt和本地cache中的tailcnt是否一致，如不一致则需要更新本地cache;
43 : // 更新本地cache时，需要确保cache中head_cnt和tail_cnt一致，否则舍弃本次数据，继续更新cache直至一致;
44 :
45 :
46 : constexpr u32 HCCL_SHM_ALIGN = 4096;
47 : constexpr u32 HCCL_HDC_CONTROL_WORDS = 2;
48 : constexpr u32 HCCL_HDC_HEAD_POS = 2;
49 : constexpr u32 HCCL_HDC_TAIL_POS = 1;
50 :
51 9527812 : inline u32* HcclHdcGetControlWordAddr(void *base, u64 size, u32 pos)
52 : {
53 9527812 : return reinterpret_cast<u32 *>(reinterpret_cast<u8 *>((base)) + size - pos * sizeof(pos));
54 : }
55 :
56 115 : HcclResult HDCommunicate::Init()
57 : {
58 115 : CHK_RET(VerifyDeviceMemoryRegisterSupport());
59 :
60 115 : CHK_RET(AllocShm());
61 115 : CHK_RET(AllocReadCache());
62 :
63 115 : headCntAddr = HcclHdcGetControlWordAddr(reinterpret_cast<void *>(hostMem->GetAddr()), hostMem->GetSize(), HCCL_HDC_HEAD_POS);
64 115 : tailCntAddr = HcclHdcGetControlWordAddr(reinterpret_cast<void *>(hostMem->GetAddr()), hostMem->GetSize(), HCCL_HDC_TAIL_POS);
65 :
66 115 : devHeadCntAddr = HcclHdcGetControlWordAddr(reinterpret_cast<void *>(devMem->GetAddr()), devMem->GetSize(), HCCL_HDC_HEAD_POS);
67 115 : devTailCntAddr = HcclHdcGetControlWordAddr(reinterpret_cast<void *>(devMem->GetAddr()), devMem->GetSize(), HCCL_HDC_TAIL_POS);
68 345 : HCCL_INFO(
69 : "[HDCommunicate][Init] buffLen=%u, flag=%u, readCacheAddr=%p, devHeadCntAddr=%p, devTailCntAddr=%p",
70 : buffLen, flag, readCacheAddr, devHeadCntAddr, devTailCntAddr);
71 115 : return HCCL_SUCCESS;
72 : }
73 :
74 115 : HcclResult HDCommunicate::VerifyDeviceMemoryRegisterSupport()
75 : {
76 115 : supportDevMemReg = false;
77 115 : size_t outputLen = 0;
78 115 : struct supportFeaturePara input = { 0 };
79 115 : struct supportFeaturePara output = { 0 };
80 115 : s32 deviceId = HrtGetDevice();
81 115 : input.support_feature = CTRL_SUPPORT_PCIE_BAR_MEM_MASK;
82 115 : input.devid = static_cast<unsigned int>(deviceId);
83 115 : halMemCtl(CTRL_TYPE_SUPPORT_FEATURE, &input, sizeof(struct supportFeaturePara), &output, &outputLen);
84 :
85 115 : if ((output.support_feature & CTRL_SUPPORT_PCIE_BAR_MEM_MASK) != 0) {
86 0 : supportDevMemReg = true;
87 : }
88 345 : HCCL_INFO("[HDCommunicate]supportDevMemReg[%d]", supportDevMemReg);
89 115 : return HCCL_SUCCESS;
90 : }
91 :
92 33 : struct HDCommunicateParams HDCommunicate::GetCommunicateParams() const
93 : {
94 33 : struct HDCommunicateParams params;
95 33 : params.hostAddr = reinterpret_cast<u64>(reinterpret_cast<void *>(hostMem->GetAddr()));
96 33 : params.deviceAddr = reinterpret_cast<u64>(reinterpret_cast<void *>(devMem->GetAddr()));
97 33 : params.readCacheAddr = reinterpret_cast<u64>(readCacheAddr);
98 33 : params.devMemSize = devMem->GetSize();
99 33 : params.buffLen = buffLen;
100 33 : params.flag = flag;
101 33 : return params;
102 : }
103 : // 为了按照调用顺序执行，防止编译器优化导致产生异常行为
104 : #pragma GCC push_options
105 : #pragma GCC optimize("O0")
106 15 : HcclResult HDCommunicate::Put(u32 offset, u32 length, u8 *value)
107 : {
108 15 : if (length == 0) {
109 0 : return HCCL_SUCCESS;
110 : }
111 15 : CHK_PTR_NULL(value);
112 :
113 15 : if (flag == HCCLV2_HDC_TYPE_D2H) {
114 0 : HCCL_ERROR("[HDCommunicate][Put]Invalid usage, flag=%u", flag);
115 0 : return HCCL_E_PARA;
116 : }
117 15 : CHK_PRT_RET((offset + length > buffLen),
118 : HCCL_ERROR("[HDCommunicate][Put]Invalid length, offset=%u, length=%u", offset, length), HCCL_E_PARA);
119 15 : std::lock_guard<std::mutex> lock(shmLock);
120 15 : return Write(offset, length, value);
121 15 : }
122 :
123 4753682 : HcclResult HDCommunicate::Get(u32 offset, u32 length, u8 *value)
124 : {
125 4753682 : if (length == 0) {
126 0 : return HCCL_SUCCESS;
127 : }
128 4753682 : CHK_PTR_NULL(value);
129 4753682 : CHK_PRT_RET((offset + length > buffLen),
130 : HCCL_ERROR("[HDCommunicate][Get]Invalid length, offset=%u, length=%u, befferLen=%u", offset, length, buffLen),
131 : HCCL_E_PARA);
132 4753682 : std::lock_guard<std::mutex> lock(shmLock);
133 4753682 : return Read(offset, length, value);
134 4753682 : }
135 :
136 0 : HcclResult HrtDrvMemCpy(void *dst, uint64_t destMax, const void *src, uint64_t count)
137 : {
138 : // 参数有效性检查
139 0 : CHK_PTR_NULL(dst);
140 0 : CHK_PTR_NULL(src);
141 :
142 0 : uint64_t dstAddr = reinterpret_cast<uintptr_t>(dst);
143 0 : uint64_t srcAddr = reinterpret_cast<uintptr_t>(const_cast<void *>(src));
144 0 : drvError_t ret = drvMemcpy(dstAddr, destMax, srcAddr, count);
145 0 : CHK_PRT_RET(ret != DRV_ERROR_NONE, HCCL_ERROR("errNo[0x%016llx] hrtDrvMemCpy fail,"
146 : "return[%d].", HCCL_ERROR_CODE(HCCL_E_DRV), ret), HCCL_E_DRV);
147 :
148 0 : return HCCL_SUCCESS;
149 : }
150 :
151 15 : HcclResult HDCommunicate::Write(u32 offset, u32 length, u8 *value)
152 : {
153 15 : if (length == 0) {
154 0 : return HCCL_SUCCESS;
155 : }
156 15 : CHK_PTR_NULL(value);
157 :
158 15 : u32 head = *headCntAddr;
159 15 : head++;
160 15 : *headCntAddr = head;
161 15 : if (!supportDevMemReg) {
162 15 : CHK_RET(HrtDrvMemCpy(devHeadCntAddr, sizeof(u32), headCntAddr, sizeof(u32)));
163 : }
164 :
165 15 : auto ret = memcpy_s(reinterpret_cast<u8 *>(hostMem->GetAddr()) + offset,
166 15 : hostMem->GetSize() - HCCL_HDC_CONTROL_WORDS * sizeof(u32), value, length);
167 15 : CHK_PRT_RET(ret != EOK, HCCL_ERROR("[HDCommunicate][Write]memcpy_s failed, return[%d].", ret), HCCL_E_INTERNAL);
168 :
169 15 : if (!supportDevMemReg) {
170 15 : CHK_RET(HrtDrvMemCpy(reinterpret_cast<u8 *>(devMem->GetAddr()) + offset,
171 : hostMem->GetSize() - HCCL_HDC_CONTROL_WORDS * sizeof(u32), value, length));
172 : }
173 :
174 15 : u32 tail = *tailCntAddr;
175 15 : tail++;
176 15 : *tailCntAddr = tail;
177 15 : if (!supportDevMemReg) {
178 15 : CHK_RET(HrtDrvMemCpy(devTailCntAddr, sizeof(u32), tailCntAddr, sizeof(u32)));
179 : }
180 15 : return HCCL_SUCCESS;
181 : }
182 :
183 4753682 : HcclResult HDCommunicate::Read(u32 offset, u32 length, u8 *value)
184 : {
185 4753682 : if (length == 0) {
186 0 : return HCCL_SUCCESS;
187 : }
188 4753682 : CHK_PTR_NULL(value);
189 4753682 : u32 *cachedTailCntAddr = HcclHdcGetControlWordAddr(readCacheAddr, devMem->GetSize(), HCCL_HDC_TAIL_POS);
190 4753682 : volatile u32 cachedTailCnt = *cachedTailCntAddr;
191 4753682 : volatile u32 tailCnt = 0;
192 4753682 : if (!supportDevMemReg) {
193 4753682 : u32 tempTailCnt = 0;
194 4753682 : u32 *devSrcTailCntAddr = HcclHdcGetControlWordAddr(reinterpret_cast<void *>(devMem->GetAddr()), devMem->GetSize(), HCCL_HDC_TAIL_POS);
195 4753682 : CHK_RET(HrtDrvMemCpy(&tempTailCnt, sizeof(u32), devSrcTailCntAddr, sizeof(u32)));
196 4753682 : tailCnt = tempTailCnt;
197 : } else {
198 0 : tailCnt = *tailCntAddr;
199 : }
200 4753682 : if (cachedTailCnt != tailCnt) {
201 : // 默认HDC超时时间为10s
202 11 : CHK_RET(UpdateCache(10));
203 : }
204 4753682 : auto ret = memcpy_s(value, length, static_cast<u8 *>(readCacheAddr) + offset, length);
205 4753682 : CHK_PRT_RET(ret != EOK, HCCL_ERROR("[HDCommunicate][Read]memcpy_s failed, return[%d].", ret), HCCL_E_INTERNAL);
206 4753682 : return HCCL_SUCCESS;
207 : }
208 :
209 11 : HcclResult HDCommunicate::UpdateCache(u32 timeoutSec)
210 : {
211 11 : void *srcBaseAddr = reinterpret_cast<void *>(hostMem->GetAddr());
212 11 : u32 *srcHeadCntAddr = HcclHdcGetControlWordAddr(srcBaseAddr, devMem->GetSize(), HCCL_HDC_HEAD_POS);
213 11 : u32 *srcTailCntAddr = HcclHdcGetControlWordAddr(srcBaseAddr, devMem->GetSize(), HCCL_HDC_TAIL_POS);
214 11 : u32 *devSrcHeadCntAddr = HcclHdcGetControlWordAddr(reinterpret_cast<void *>(devMem->GetAddr()), devMem->GetSize(), HCCL_HDC_HEAD_POS);
215 11 : u32 *devSrcTailCntAddr = HcclHdcGetControlWordAddr(reinterpret_cast<void *>(devMem->GetAddr()), devMem->GetSize(), HCCL_HDC_TAIL_POS);
216 11 : u32 *cachedHeadCntAddr = HcclHdcGetControlWordAddr(readCacheAddr, devMem->GetSize(), HCCL_HDC_HEAD_POS);
217 11 : u32 *cachedTailCntAddr = HcclHdcGetControlWordAddr(readCacheAddr, devMem->GetSize(), HCCL_HDC_TAIL_POS);
218 :
219 11 : s32 ret = 0;
220 11 : auto startTime = std::chrono::steady_clock::now();
221 11 : auto timeout = std::chrono::seconds(timeoutSec);
222 : while (1) {
223 11 : if (!supportDevMemReg) {
224 : // step1: cache尾计数
225 11 : CHK_RET(HrtDrvMemCpy(cachedTailCntAddr, sizeof(u32), devSrcTailCntAddr, sizeof(u32)));
226 :
227 : // step2: cache数据
228 11 : CHK_RET(HrtDrvMemCpy(readCacheAddr, devMem->GetSize() - HCCL_HDC_CONTROL_WORDS * sizeof(u32), reinterpret_cast<void *>(devMem->GetAddr()),
229 : devMem->GetSize() - HCCL_HDC_CONTROL_WORDS * sizeof(u32)));
230 :
231 : // step3：cache头计数
232 11 : CHK_RET(HrtDrvMemCpy(cachedHeadCntAddr, sizeof(u32), devSrcHeadCntAddr, sizeof(u32)));
233 : } else {
234 : // step1: cache尾计数
235 0 : ret = memcpy_s(cachedTailCntAddr, sizeof(u32), srcTailCntAddr, sizeof(u32));
236 0 : CHK_PRT_RET(ret != EOK, HCCL_ERROR("[HDCommunicate][UpdateCache]memcpy_s failed, return[%d].", ret),
237 : HCCL_E_INTERNAL);
238 :
239 : // step2: cache数据
240 0 : ret = memcpy_s(readCacheAddr, devMem->GetSize() - HCCL_HDC_CONTROL_WORDS * sizeof(u32), srcBaseAddr,
241 0 : devMem->GetSize() - HCCL_HDC_CONTROL_WORDS * sizeof(u32));
242 0 : CHK_PRT_RET(ret != EOK, HCCL_ERROR("[HDCommunicate][UpdateCache]memcpy_s failed, return[%d].", ret),
243 : HCCL_E_INTERNAL);
244 :
245 : // step3：cache头计数
246 0 : ret = memcpy_s(cachedHeadCntAddr, sizeof(u32), srcHeadCntAddr, sizeof(u32));
247 0 : CHK_PRT_RET(ret != EOK, HCCL_ERROR("[HDCommunicate][UpdateCache]memcpy_s failed, return[%d].", ret),
248 : HCCL_E_INTERNAL);
249 : }
250 :
251 11 : volatile u32 cachedHeadCnt = *cachedHeadCntAddr;
252 11 : volatile u32 cachedTailCnt = *cachedTailCntAddr;
253 :
254 11 : if (cachedHeadCnt == cachedTailCnt) {
255 11 : break;
256 : }
257 0 : CHK_PRT_RET(((std::chrono::steady_clock::now() - startTime) >= timeout),
258 : HCCL_WARNING("[HDCommunicate][UpdateCache]get remote data timeout[%u s].", timeoutSec), HCCL_E_AGAIN);
259 0 : }
260 11 : return HCCL_SUCCESS;
261 : }
262 : #pragma GCC pop_options
263 :
264 115 : HcclResult HDCommunicate::AllocShm()
265 : {
266 : // 共享内存size需要按照4K(4*1024=4096)对齐
267 115 : size_t size = (buffLen + HCCL_HDC_CONTROL_WORDS * sizeof(u32) + HCCL_SHM_ALIGN - 1) / HCCL_SHM_ALIGN * HCCL_SHM_ALIGN;
268 115 : devMem = std::make_unique<DevBuffer>(size);
269 115 : HrtMemset(reinterpret_cast<void *>(devMem->GetAddr()), devMem->GetSize(), devMem->GetSize());
270 :
271 115 : if (supportDevMemReg) {
272 0 : void *hostAddr = nullptr;
273 0 : halHostRegister(reinterpret_cast<void *>(devMem->GetAddr()), devMem->GetSize(), DEV_SVM_MAP_HOST, deviceLogicId, &hostAddr);
274 :
275 0 : hostMem = std::make_unique<HostBuffer>(reinterpret_cast<uintptr_t>(hostAddr), devMem->GetSize());
276 : } else {
277 115 : hostMem = std::make_unique<HostBuffer>(devMem->GetSize());
278 : }
279 :
280 115 : auto ret = memset_s(reinterpret_cast<void *>(hostMem->GetAddr()), hostMem->GetSize(), 0, hostMem->GetSize());
281 115 : CHK_PRT_RET(ret != EOK, HCCL_ERROR("[HDCommunicate][AllocShm]memset_s failed, return[%d].", ret), HCCL_E_INTERNAL);
282 :
283 115 : return HCCL_SUCCESS;
284 : }
285 :
286 115 : HcclResult HDCommunicate::AllocReadCache()
287 : {
288 115 : if (flag == HCCLV2_HDC_TYPE_D2H) {
289 57 : hostCache = std::make_unique<HostBuffer>(hostMem->GetSize());
290 57 : auto ret = memset_s(reinterpret_cast<void *>(hostCache->GetAddr()), hostCache->GetSize(), 0, hostCache->GetSize());
291 57 : CHK_PRT_RET(ret != EOK, HCCL_ERROR("[HDCommunicate][AllocReadCache]memset_s failed, return[%d].", ret),
292 : HCCL_E_INTERNAL);
293 57 : readCacheAddr = reinterpret_cast<void *>(hostCache->GetAddr());
294 : } else {
295 58 : devCache = std::make_unique<DevBuffer>(devMem->GetSize());
296 58 : HrtMemset(reinterpret_cast<void *>(devCache->GetAddr()), devCache->GetSize(), devCache->GetSize());
297 58 : readCacheAddr = reinterpret_cast<void *>(devCache->GetAddr());
298 : }
299 115 : return HCCL_SUCCESS;
300 : }
301 : }
|
