Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "hccl_sqe_v82.h"
12 : #include "exception_util.h"
13 : #include "not_support_exception.h"
14 :
15 : namespace Hccl {
16 :
17 1 : HcclUBDmaDBSqe::HcclUBDmaDBSqe()
18 : {
19 1 : sqe = std::make_unique<Rt91095StarsUbdmaDBmodeSqe>();
20 1 : (void)memset_s(sqe.get(), sizeof(Rt91095StarsUbdmaDBmodeSqe), 0, sizeof(Rt91095StarsUbdmaDBmodeSqe));
21 1 : sqe->header.wrCqe = 0U;
22 1 : sqe->header.type = static_cast<uint8_t>(Rt91095StarsSqeType::RT_91095_SQE_TYPE_UBDMA);
23 1 : sqe->header.lock = 0U;
24 1 : sqe->header.unlock = 0U;
25 1 : sqe->header.ie = RtStarsSqeIntDirType::RT_STARS_SQE_INT_DIR_NO;
26 1 : sqe->header.preP = RtStarsSqeIntDirType::RT_STARS_SQE_INT_DIR_NO;
27 1 : sqe->header.postP = RtStarsSqeIntDirType::RT_STARS_SQE_INT_DIR_NO;
28 1 : sqe->mode = Rt91095UbDmaSqeMode::RT_91095_SQE_DOORBELL_MODE;
29 1 : sqe->kernelCredit = RT_STARS_DEFAULT_KERNEL_CREDIT;
30 1 : sqe->sqeLength = 0U;
31 1 : }
32 :
33 1 : void HcclUBDmaDBSqe::Config(u16 streamId, u16 taskId, u16 jettyid, u8 funcId, u16 piValue, u16 dieId)
34 : {
35 1 : sqe->header.rtStreamId = streamId;
36 1 : sqe->header.taskId = taskId;
37 :
38 1 : sqe->doorbellNum = 1U;
39 1 : sqe->jettyId1 = jettyid;
40 1 : sqe->funcId1 = funcId;
41 1 : sqe->piValue1 = piValue;
42 1 : sqe->dieId1 = dieId;
43 3 : HCCL_INFO("[SQE]HcclUBDmaDBSqe streamId=%u, taskId=%u, jettyid=%u, funcId=%u, dieId=%u, piValue=%u\n",
44 : streamId, taskId, jettyid, funcId, dieId, piValue);
45 1 : }
46 :
47 1 : u64 HcclUBDmaDBSqe::GetSqe()
48 : {
49 1 : return reinterpret_cast<u64>(sqe.get());
50 : }
51 :
52 1 : HcclUBNotifyWaitSqe::HcclUBNotifyWaitSqe()
53 : {
54 1 : sqe = std::make_unique<Rt91095StarsNotifySqe>();
55 1 : (void)memset_s(sqe.get(), sizeof(Rt91095StarsNotifySqe), 0, sizeof(Rt91095StarsNotifySqe));
56 1 : sqe->header.ie = RtStarsSqeIntDirType::RT_STARS_SQE_INT_DIR_NO;
57 1 : sqe->header.preP = RtStarsSqeIntDirType::RT_STARS_SQE_INT_DIR_NO;
58 1 : sqe->header.postP = RtStarsSqeIntDirType::RT_STARS_SQE_INT_DIR_NO;
59 1 : sqe->header.wrCqe = 0U;
60 1 : sqe->header.headUpdate = RtStarsSqeIntDirType::RT_STARS_SQE_INT_DIR_NO;
61 :
62 1 : sqe->kernelCredit = RT_STARS_NEVER_TIMEOUT_KERNEL_CREDIT;
63 1 : sqe->header.type = static_cast<uint8_t>(Rt91095StarsSqeType::RT_91095_SQE_TYPE_NOTIFY_WAIT);
64 1 : sqe->cntFlag = false;
65 1 : sqe->clrFlag = true;
66 1 : sqe->waitModeBit = 0U;
67 1 : sqe->recordModeBit = 0U;
68 1 : sqe->cntValue = 0U;
69 1 : sqe->subType = static_cast<uint16_t>(Rt91095NotifySubType::NOTIFY_SUB_TYPE_SINGLE_NOTIFY_WAIT);
70 1 : }
71 :
72 1 : void HcclUBNotifyWaitSqe::Config(u16 streamId, u16 taskId, u64 notifyId)
73 : {
74 1 : sqe->header.rtStreamId = streamId;
75 1 : sqe->header.taskId = taskId;
76 1 : sqe->notifyId = notifyId;
77 :
78 3 : HCCL_INFO("[SQE]HcclUBNotifyWaitSqe streamId=%u, taskId=%hu, notifyId:%llu\n", streamId, taskId, notifyId);
79 1 : }
80 :
81 1 : u64 HcclUBNotifyWaitSqe::GetSqe()
82 : {
83 1 : return reinterpret_cast<u64>(sqe.get());
84 : }
85 :
86 1 : HcclUBNotifyRecordSqe::HcclUBNotifyRecordSqe()
87 : {
88 1 : sqe = std::make_unique<Rt91095StarsNotifySqe>();
89 1 : (void)memset_s(sqe.get(), sizeof(Rt91095StarsNotifySqe), 0, sizeof(Rt91095StarsNotifySqe));
90 1 : sqe->header.ie = RtStarsSqeIntDirType::RT_STARS_SQE_INT_DIR_NO;
91 1 : sqe->header.preP = RtStarsSqeIntDirType::RT_STARS_SQE_INT_DIR_NO;
92 1 : sqe->header.postP = RtStarsSqeIntDirType::RT_STARS_SQE_INT_DIR_NO;
93 1 : sqe->header.wrCqe = 0U;
94 1 : sqe->header.headUpdate = RtStarsSqeIntDirType::RT_STARS_SQE_INT_DIR_NO;
95 1 : sqe->header.type = static_cast<uint8_t>(Rt91095StarsSqeType::RT_91095_SQE_TYPE_NOTIFY_RECORD);
96 1 : sqe->kernelCredit = RT_STARS_DEFAULT_KERNEL_CREDIT;
97 1 : sqe->clrFlag = 0U;
98 1 : sqe->cntFlag = 0U;
99 1 : sqe->cntValue = 0U;
100 1 : sqe->waitModeBit = 0U;
101 1 : sqe->recordModeBit = 0U;
102 1 : sqe->subType = static_cast<uint16_t>(Rt91095NotifySubType::NOTIFY_SUB_TYPE_SINGLE_NOTIFY_RECORD);
103 1 : }
104 :
105 1 : void HcclUBNotifyRecordSqe::Config(u16 streamId, u16 taskId, u64 notifyId)
106 : {
107 1 : sqe->header.rtStreamId = streamId;
108 1 : sqe->header.taskId = taskId;
109 1 : sqe->notifyId = notifyId;
110 3 : HCCL_INFO("[SQE]HcclUBNotifyRecordSqe streamId=%u, taskId=%hu, notifyId:%llu\n", streamId, taskId, notifyId);
111 1 : }
112 :
113 1 : u64 HcclUBNotifyRecordSqe::GetSqe()
114 : {
115 1 : return reinterpret_cast<u64>(sqe.get());
116 : }
117 :
118 1 : HcclUBCntNotifyNto1RecordSqe::HcclUBCntNotifyNto1RecordSqe()
119 : {
120 1 : sqe = std::make_unique<Rt91095StarsNotifySqe>();
121 1 : (void)memset_s(sqe.get(), sizeof(Rt91095StarsNotifySqe), 0, sizeof(Rt91095StarsNotifySqe));
122 1 : sqe->header.type = static_cast<uint8_t>(Rt91095StarsSqeType::RT_91095_SQE_TYPE_NOTIFY_RECORD);
123 1 : sqe->kernelCredit = RT_STARS_DEFAULT_KERNEL_CREDIT;
124 1 : sqe->clrFlag = false;
125 1 : sqe->cntFlag = true;
126 1 : sqe->recordModeBit = 0x2U; // rtCntNotifyRecordMode_t::RECORD_WRITE_BIT_MODE
127 1 : sqe->subType = static_cast<uint16_t>(Rt91095NotifySubType::NOTIFY_SUB_TYPE_COUNT_NOTIFY_RECORD);
128 1 : }
129 :
130 1 : void HcclUBCntNotifyNto1RecordSqe::Config(u16 streamId, u16 taskId, u64 notifyId, u32 cntValue)
131 : {
132 1 : sqe->header.rtStreamId = streamId;
133 1 : sqe->header.taskId = taskId;
134 1 : sqe->notifyId = notifyId;
135 1 : sqe->cntValue = cntValue;
136 3 : HCCL_INFO("[SQE]HcclUBCntNotifyNto1RecordSqe streamId=%u, taskId=%u, notifyId=%u, cntValue=%u\n",
137 : streamId, taskId, notifyId, cntValue);
138 1 : }
139 :
140 1 : u64 HcclUBCntNotifyNto1RecordSqe::GetSqe()
141 : {
142 1 : return reinterpret_cast<u64>(sqe.get());
143 : }
144 :
145 1 : HcclUBCntNotify1toNWaitSqe::HcclUBCntNotify1toNWaitSqe()
146 : {
147 1 : sqe = std::make_unique<Rt91095StarsNotifySqe>();
148 1 : (void)memset_s(sqe.get(), sizeof(Rt91095StarsNotifySqe), 0, sizeof(Rt91095StarsNotifySqe));
149 1 : sqe->kernelCredit = RT_STARS_NEVER_TIMEOUT_KERNEL_CREDIT;
150 1 : sqe->header.type = static_cast<uint8_t>(Rt91095StarsSqeType::RT_91095_SQE_TYPE_NOTIFY_WAIT);
151 1 : sqe->cntFlag = true;
152 1 : sqe->clrFlag = true;
153 1 : sqe->bitmap = 1U;
154 1 : sqe->subType = static_cast<uint16_t>(Rt91095NotifySubType::NOTIFY_SUB_TYPE_COUNT_NOTIFY_WAIT);
155 1 : }
156 :
157 1 : void HcclUBCntNotify1toNWaitSqe::Config(u16 streamId, u16 taskId, u64 notifyId, u32 cntValue)
158 : {
159 1 : sqe->header.rtStreamId = streamId;
160 1 : sqe->header.taskId = taskId;
161 1 : sqe->notifyId = notifyId;
162 1 : sqe->cntValue = cntValue;
163 3 : HCCL_INFO("[SQE]HcclUBCntNotify1toNWaitSqe streamId=%u, taskId=%u, notifyId=%u, cntValue=%u\n",
164 : streamId, taskId, notifyId, cntValue);
165 1 : }
166 :
167 1 : u64 HcclUBCntNotify1toNWaitSqe::GetSqe()
168 : {
169 1 : return reinterpret_cast<u64>(sqe.get());
170 : }
171 :
172 1 : HcclUBCntNotifyNto1WaitSqe::HcclUBCntNotifyNto1WaitSqe()
173 : {
174 1 : sqe = std::make_unique<Rt91095StarsNotifySqe>();
175 1 : (void)memset_s(sqe.get(), sizeof(Rt91095StarsNotifySqe), 0, sizeof(Rt91095StarsNotifySqe));
176 1 : sqe->kernelCredit = RT_STARS_NEVER_TIMEOUT_KERNEL_CREDIT;
177 1 : sqe->header.type = static_cast<uint8_t>(Rt91095StarsSqeType::RT_91095_SQE_TYPE_NOTIFY_WAIT);
178 1 : sqe->cntFlag = true;
179 1 : sqe->clrFlag = true;
180 1 : sqe->waitModeBit = 0x1U; // rtCntNotifyWaitMode_t::WAIT_EQUAL_MODE
181 1 : sqe->subType = static_cast<uint16_t>(Rt91095NotifySubType::NOTIFY_SUB_TYPE_COUNT_NOTIFY_WAIT);
182 1 : }
183 :
184 1 : void HcclUBCntNotifyNto1WaitSqe::Config(u16 streamId, u16 taskId, u64 notifyId, u32 cntValue)
185 : {
186 1 : sqe->header.rtStreamId = streamId;
187 1 : sqe->header.taskId = taskId;
188 1 : sqe->notifyId = notifyId;
189 1 : sqe->cntValue = cntValue;
190 3 : HCCL_INFO("[SQE]HcclUBCntNotifyNto1WaitSqe streamId=%u, taskId=%u, notifyId=%u, cntValue=%u\n",
191 : streamId, taskId, notifyId, cntValue);
192 1 : }
193 :
194 1 : u64 HcclUBCntNotifyNto1WaitSqe::GetSqe()
195 : {
196 1 : return reinterpret_cast<u64>(sqe.get());
197 : }
198 :
199 1 : HcclUBCntNotify1toNRecordSqe::HcclUBCntNotify1toNRecordSqe()
200 : {
201 1 : sqe = std::make_unique<Rt91095StarsNotifySqe>();
202 1 : (void)memset_s(sqe.get(), sizeof(Rt91095StarsNotifySqe), 0, sizeof(Rt91095StarsNotifySqe));
203 1 : sqe->header.type = static_cast<uint8_t>(Rt91095StarsSqeType::RT_91095_SQE_TYPE_NOTIFY_RECORD);
204 1 : sqe->kernelCredit = RT_STARS_DEFAULT_KERNEL_CREDIT;
205 1 : sqe->clrFlag = false;
206 1 : sqe->cntFlag = true;
207 1 : sqe->recordModeBit = 0x0U; // rtCntNotifyRecordMode_t::RECORD_STORE_MODE
208 1 : sqe->subType = static_cast<uint16_t>(Rt91095NotifySubType::NOTIFY_SUB_TYPE_COUNT_NOTIFY_RECORD);
209 1 : }
210 :
211 1 : void HcclUBCntNotify1toNRecordSqe::Config(u16 streamId, u16 taskId, u64 notifyId, u32 cntValue)
212 : {
213 1 : sqe->header.rtStreamId = streamId;
214 1 : sqe->header.taskId = taskId;
215 1 : sqe->notifyId = notifyId;
216 1 : sqe->cntValue = cntValue;
217 3 : HCCL_INFO("[SQE]HcclUBCntNotify1toNRecordSqe streamId=%u, taskId=%u, notifyId=%u, cntValue=%u\n",
218 : streamId, taskId, notifyId, cntValue);
219 1 : }
220 :
221 1 : u64 HcclUBCntNotify1toNRecordSqe::GetSqe()
222 : {
223 1 : return reinterpret_cast<u64>(sqe.get());
224 : }
225 :
226 1 : HcclUBMemcpySqe::HcclUBMemcpySqe()
227 : {
228 1 : sqe = std::make_unique<Rt91095StarsMemcpySqe>();
229 1 : (void)memset_s(sqe.get(), sizeof(Rt91095StarsMemcpySqe), 0, sizeof(Rt91095StarsMemcpySqe));
230 1 : sqe->header.type = static_cast<uint8_t>(Rt91095StarsSqeType::RT_91095_SQE_TYPE_SDMA);
231 1 : sqe->header.lock = 0U;
232 1 : sqe->header.unlock = 0U;
233 1 : sqe->header.ie = 0U;
234 1 : sqe->header.wrCqe = 0U;
235 1 : sqe->header.ptrMode = 0U;
236 1 : sqe->header.rttMode = 0U;
237 1 : sqe->header.headUpdate = 0U;
238 1 : sqe->header.reserved = 0U;
239 1 : sqe->header.numBlocks = 0U;
240 :
241 1 : sqe->kernelCredit = RT_STARS_DEFAULT_KERNEL_CREDIT;
242 :
243 1 : sqe->ie2 = 0U;
244 1 : sqe->sssv = 1U;
245 1 : sqe->dssv = 1U;
246 1 : sqe->sns = 1U;
247 1 : sqe->qos = 0U;
248 1 : sqe->dns = 1U;
249 1 : sqe->sro = 0U;
250 1 : sqe->dro = 0U;
251 1 : sqe->mapamPartId = 0U; // 这里走的memcpy，如果走withcfg,需要传入qoscfg
252 1 : sqe->mpamns = 0U;
253 1 : sqe->stride = 0U;
254 1 : sqe->compEn = 0U;
255 1 : sqe->pmg = 0U;
256 1 : sqe->res1 = 0U;
257 1 : sqe->res2 = 0U;
258 1 : sqe->res3 = 0U;
259 1 : sqe->res4 = 0U;
260 :
261 1 : sqe->d2dOffsetFlag = 0U;
262 1 : sqe->u.strideMode0.srcOffsetLow = 0U;
263 1 : sqe->u.strideMode0.dstOffsetLow = 0U;
264 1 : sqe->u.strideMode0.srcOffsetHigh = 0U;
265 3 : HCCL_INFO("[SQE]HcclUBMemcpySqe construct end");
266 1 : }
267 :
268 1 : void HcclUBMemcpySqe::Config(u16 streamId, u16 taskId, RtDataType rtDataType, RtReduceKind rtReduceOp,
269 : u64 count, const u64 *src, const u64 *dst, u32 partId)
270 : {
271 1 : sqe->header.rtStreamId = streamId;
272 1 : sqe->header.taskId = taskId;
273 1 : const bool isReduce
274 0 : = ((rtReduceOp == RtReduceKind::RT_MEMCPY_SDMA_AUTOMATIC_ADD) || (rtReduceOp == RtReduceKind::RT_MEMCPY_SDMA_AUTOMATIC_MAX)
275 1 : || (rtReduceOp == RtReduceKind::RT_MEMCPY_SDMA_AUTOMATIC_MIN) || (rtReduceOp == RtReduceKind::RT_MEMCPY_SDMA_AUTOMATIC_EQUAL));
276 1 : sqe->opcode = isReduce ? GetUBOpCode( static_cast<u32>(rtReduceOp), static_cast<u8>(rtDataType)) : 0U;
277 :
278 1 : sqe->u.strideMode0.lengthMove = count;
279 2 : sqe->u.strideMode0.srcAddrLow =
280 1 : static_cast<uint32_t>(static_cast<uint64_t>(*src) & 0x00000000ffffffffU);
281 2 : sqe->u.strideMode0.srcAddrHigh =
282 1 : static_cast<uint32_t>((static_cast<uint64_t>(*src) & 0xffffffff00000000U) >> UINT32_BIT_NUM);
283 2 : sqe->u.strideMode0.dstAddrLow =
284 1 : static_cast<uint32_t>(static_cast<uint64_t>(*dst) & 0x00000000ffffffffU);
285 2 : sqe->u.strideMode0.dstAddrHigh =
286 1 : static_cast<uint32_t>((static_cast<uint64_t>(*dst) & 0xffffffff00000000U) >> UINT32_BIT_NUM);
287 1 : sqe->mapamPartId = partId;
288 :
289 3 : HCCL_INFO("[SQE]HcclUBMemcpySqe dataType=%u,rtReduceOp =%u, count=%lu, src=%p, dst=%p, partId=%u, streamId=%u, \
290 : taskId=%u\n", rtDataType, rtReduceOp, count, src, dst, partId, streamId, taskId);
291 3 : HCCL_INFO("[SQE]HcclUBMemcpySqe sqe->opcode=%u sqe->u.strideMode0.srcAddrLow=0x%x, \
292 : sqe->u.strideMode0.srcAddrHigh=0x%x,sqe->u.strideMode0.dstAddrLow=0x%x, \
293 : sqe->u.strideMode0.dstAddrHigh=0x%x\n", sqe->opcode,
294 : sqe->u.strideMode0.srcAddrLow, sqe->u.strideMode0.srcAddrHigh,
295 : sqe->u.strideMode0.dstAddrLow, sqe->u.strideMode0.dstAddrHigh);
296 1 : }
297 :
298 1 : u64 HcclUBMemcpySqe::GetSqe()
299 : {
300 1 : return reinterpret_cast<u64>(sqe.get());
301 : }
302 :
303 : // change name: convert
304 1 : u8 HcclUBMemcpySqe::ConvertToMemcpyDataType(u8 copyDataType) const
305 : {
306 : u8 opcode;
307 1 : switch (static_cast<RtDataType>(copyDataType)) {
308 0 : case RtDataType::RT_DATA_TYPE_INT8: {
309 0 : opcode = static_cast<u8>(RtStarsMemcpyAsyncDataType::RT_STARS_MEMCPY_ASYNC_DATA_TYPE_INT8);
310 0 : break;
311 : }
312 1 : case RtDataType::RT_DATA_TYPE_INT16: {
313 1 : opcode = static_cast<u8>(RtStarsMemcpyAsyncDataType::RT_STARS_MEMCPY_ASYNC_DATA_TYPE_INT16);
314 1 : break;
315 : }
316 0 : case RtDataType::RT_DATA_TYPE_INT32: {
317 0 : opcode = static_cast<u8>(RtStarsMemcpyAsyncDataType::RT_STARS_MEMCPY_ASYNC_DATA_TYPE_INT32);
318 0 : break;
319 : }
320 0 : case RtDataType::RT_DATA_TYPE_FP16: {
321 0 : opcode = static_cast<u8>(RtStarsMemcpyAsyncDataType::RT_STARS_MEMCPY_ASYNC_DATA_TYPE_FP16);
322 0 : break;
323 : }
324 0 : case RtDataType::RT_DATA_TYPE_FP32: {
325 0 : opcode = static_cast<u8>(RtStarsMemcpyAsyncDataType::RT_STARS_MEMCPY_ASYNC_DATA_TYPE_FP32);
326 0 : break;
327 : }
328 0 : case RtDataType::RT_DATA_TYPE_BFP16: {
329 0 : opcode = static_cast<u8>(RtStarsMemcpyAsyncDataType::RT_STARS_MEMCPY_ASYNC_DATA_TYPE_BFP16);
330 0 : break;
331 : }
332 0 : default: {
333 : // Should not run here.
334 : // Only for code style, 0x80 is reserved value of STRAS opcode.
335 0 : MACRO_THROW(NotSupportException,
336 : StringFormat("DataType=%u do not support.", static_cast<u32>(copyDataType)));
337 : break;
338 : }
339 : }
340 1 : return opcode;
341 : }
342 :
343 1 : u8 HcclUBMemcpySqe::ConvertToMemcpyOpType(u32 copyKind) const
344 : {
345 : u8 opcode;
346 1 : switch (static_cast<RtReduceKind>(copyKind)) {
347 1 : case RtReduceKind::RT_MEMCPY_SDMA_AUTOMATIC_ADD: {
348 1 : opcode = static_cast<u8>(RtStarsMemcpyAsyncOperationKind::RT_STARS_MEMCPY_ASYNC_OP_KIND_ADD);
349 1 : break;
350 : }
351 0 : case RtReduceKind::RT_MEMCPY_SDMA_AUTOMATIC_MAX: {
352 0 : opcode = static_cast<u8>(RtStarsMemcpyAsyncOperationKind::RT_STARS_MEMCPY_ASYNC_OP_KIND_MAX);
353 0 : break;
354 : }
355 0 : case RtReduceKind::RT_MEMCPY_SDMA_AUTOMATIC_MIN: {
356 0 : opcode = static_cast<u8>(RtStarsMemcpyAsyncOperationKind::RT_STARS_MEMCPY_ASYNC_OP_KIND_MIN);
357 0 : break;
358 : }
359 0 : case RtReduceKind::RT_MEMCPY_SDMA_AUTOMATIC_EQUAL: {
360 0 : opcode = static_cast<u8>(RtStarsMemcpyAsyncOperationKind::RT_STARS_MEMCPY_ASYNC_OP_KIND_EQUAL);
361 0 : break;
362 : }
363 0 : default: {
364 0 : MACRO_THROW(NotSupportException, StringFormat("Type out of range: copyKind=%u", copyKind));
365 : break;
366 : }
367 : }
368 1 : return opcode;
369 : }
370 :
371 1 : u8 HcclUBMemcpySqe::GetUBOpCode(u32 copyKind, u8 copyDataType) const
372 : {
373 1 : const u8 memcpyDataType = ConvertToMemcpyDataType(copyDataType);
374 1 : const u8 opType = ConvertToMemcpyOpType(copyKind);
375 : // opcode: 高4bit为datatype，低4bit为optype
376 1 : return memcpyDataType | opType;
377 : }
378 :
379 : } // namespace Hccl
|
