Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "hccl_sqe_v82.h"
12 : #include "exception_util.h"
13 : #include "not_support_exception.h"
14 :
15 : namespace Hccl {
16 :
17 1 : HcclUBDmaDBSqe::HcclUBDmaDBSqe()
18 : {
19 1 : sqe = std::make_unique<Rt91095StarsUbdmaDBmodeSqe>();
20 1 : (void)memset_s(sqe.get(), sizeof(Rt91095StarsUbdmaDBmodeSqe), 0, sizeof(Rt91095StarsUbdmaDBmodeSqe));
21 1 : sqe->header.wrCqe = 0U;
22 1 : sqe->header.type = static_cast<uint8_t>(Rt91095StarsSqeType::RT_91095_SQE_TYPE_UBDMA);
23 1 : sqe->header.lock = 0U;
24 1 : sqe->header.unlock = 0U;
25 1 : sqe->header.ie = RtStarsSqeIntDirType::RT_STARS_SQE_INT_DIR_NO;
26 1 : sqe->header.preP = RtStarsSqeIntDirType::RT_STARS_SQE_INT_DIR_NO;
27 1 : sqe->header.postP = RtStarsSqeIntDirType::RT_STARS_SQE_INT_DIR_NO;
28 1 : sqe->mode = Rt91095UbDmaSqeMode::RT_91095_SQE_DOORBELL_MODE;
29 1 : sqe->kernelCredit = RT_STARS_DEFAULT_KERNEL_CREDIT;
30 1 : sqe->sqeLength = 0U;
31 1 : }
32 :
33 1 : void HcclUBDmaDBSqe::Config(u16 streamId, u16 taskId, u16 jettyid, u8 funcId, u16 piValue, u16 dieId)
34 : {
35 1 : sqe->header.rtStreamId = streamId;
36 1 : sqe->header.taskId = taskId;
37 :
38 1 : sqe->doorbellNum = 1U;
39 1 : sqe->jettyId1 = jettyid;
40 1 : sqe->funcId1 = funcId;
41 1 : sqe->piValue1 = piValue;
42 1 : sqe->dieId1 = dieId;
43 3 : HCCL_INFO(
44 : "[SQE]HcclUBDmaDBSqe streamId=%u, taskId=%u, jettyid=%u, funcId=%u, dieId=%u, piValue=%u", streamId, taskId,
45 : jettyid, funcId, dieId, piValue);
46 1 : }
47 :
48 1 : u64 HcclUBDmaDBSqe::GetSqe() { return reinterpret_cast<u64>(sqe.get()); }
49 :
50 1 : HcclUBNotifyWaitSqe::HcclUBNotifyWaitSqe()
51 : {
52 1 : sqe = std::make_unique<Rt91095StarsNotifySqe>();
53 1 : (void)memset_s(sqe.get(), sizeof(Rt91095StarsNotifySqe), 0, sizeof(Rt91095StarsNotifySqe));
54 1 : sqe->header.ie = RtStarsSqeIntDirType::RT_STARS_SQE_INT_DIR_NO;
55 1 : sqe->header.preP = RtStarsSqeIntDirType::RT_STARS_SQE_INT_DIR_NO;
56 1 : sqe->header.postP = RtStarsSqeIntDirType::RT_STARS_SQE_INT_DIR_NO;
57 1 : sqe->header.wrCqe = 0U;
58 1 : sqe->header.headUpdate = RtStarsSqeIntDirType::RT_STARS_SQE_INT_DIR_NO;
59 :
60 1 : sqe->kernelCredit = RT_STARS_NEVER_TIMEOUT_KERNEL_CREDIT;
61 1 : sqe->header.type = static_cast<uint8_t>(Rt91095StarsSqeType::RT_91095_SQE_TYPE_NOTIFY_WAIT);
62 1 : sqe->cntFlag = false;
63 1 : sqe->clrFlag = true;
64 1 : sqe->waitModeBit = 0U;
65 1 : sqe->recordModeBit = 0U;
66 1 : sqe->cntValue = 0U;
67 1 : sqe->subType = static_cast<uint16_t>(Rt91095NotifySubType::NOTIFY_SUB_TYPE_SINGLE_NOTIFY_WAIT);
68 1 : }
69 :
70 1 : void HcclUBNotifyWaitSqe::Config(u16 streamId, u16 taskId, u64 notifyId)
71 : {
72 1 : sqe->header.rtStreamId = streamId;
73 1 : sqe->header.taskId = taskId;
74 1 : sqe->notifyId = notifyId;
75 :
76 3 : HCCL_INFO("[SQE]HcclUBNotifyWaitSqe streamId=%u, taskId=%u, notifyId=%llu", streamId, taskId, notifyId);
77 1 : }
78 :
79 1 : u64 HcclUBNotifyWaitSqe::GetSqe() { return reinterpret_cast<u64>(sqe.get()); }
80 :
81 1 : HcclUBNotifyRecordSqe::HcclUBNotifyRecordSqe()
82 : {
83 1 : sqe = std::make_unique<Rt91095StarsNotifySqe>();
84 1 : (void)memset_s(sqe.get(), sizeof(Rt91095StarsNotifySqe), 0, sizeof(Rt91095StarsNotifySqe));
85 1 : sqe->header.ie = RtStarsSqeIntDirType::RT_STARS_SQE_INT_DIR_NO;
86 1 : sqe->header.preP = RtStarsSqeIntDirType::RT_STARS_SQE_INT_DIR_NO;
87 1 : sqe->header.postP = RtStarsSqeIntDirType::RT_STARS_SQE_INT_DIR_NO;
88 1 : sqe->header.wrCqe = 0U;
89 1 : sqe->header.headUpdate = RtStarsSqeIntDirType::RT_STARS_SQE_INT_DIR_NO;
90 1 : sqe->header.type = static_cast<uint8_t>(Rt91095StarsSqeType::RT_91095_SQE_TYPE_NOTIFY_RECORD);
91 1 : sqe->kernelCredit = RT_STARS_DEFAULT_KERNEL_CREDIT;
92 1 : sqe->clrFlag = 0U;
93 1 : sqe->cntFlag = 0U;
94 1 : sqe->cntValue = 0U;
95 1 : sqe->waitModeBit = 0U;
96 1 : sqe->recordModeBit = 0U;
97 1 : sqe->subType = static_cast<uint16_t>(Rt91095NotifySubType::NOTIFY_SUB_TYPE_SINGLE_NOTIFY_RECORD);
98 1 : }
99 :
100 1 : void HcclUBNotifyRecordSqe::Config(u16 streamId, u16 taskId, u64 notifyId)
101 : {
102 1 : sqe->header.rtStreamId = streamId;
103 1 : sqe->header.taskId = taskId;
104 1 : sqe->notifyId = notifyId;
105 3 : HCCL_INFO("[SQE]HcclUBNotifyRecordSqe streamId=%u, taskId=%u, notifyId=%llu", streamId, taskId, notifyId);
106 1 : }
107 :
108 1 : u64 HcclUBNotifyRecordSqe::GetSqe() { return reinterpret_cast<u64>(sqe.get()); }
109 :
110 1 : HcclUBCntNotifyNto1RecordSqe::HcclUBCntNotifyNto1RecordSqe()
111 : {
112 1 : sqe = std::make_unique<Rt91095StarsNotifySqe>();
113 1 : (void)memset_s(sqe.get(), sizeof(Rt91095StarsNotifySqe), 0, sizeof(Rt91095StarsNotifySqe));
114 1 : sqe->header.type = static_cast<uint8_t>(Rt91095StarsSqeType::RT_91095_SQE_TYPE_NOTIFY_RECORD);
115 1 : sqe->kernelCredit = RT_STARS_DEFAULT_KERNEL_CREDIT;
116 1 : sqe->clrFlag = false;
117 1 : sqe->cntFlag = true;
118 1 : sqe->recordModeBit = 0x2U; // rtCntNotifyRecordMode_t::RECORD_WRITE_BIT_MODE
119 1 : sqe->subType = static_cast<uint16_t>(Rt91095NotifySubType::NOTIFY_SUB_TYPE_COUNT_NOTIFY_RECORD);
120 1 : }
121 :
122 1 : void HcclUBCntNotifyNto1RecordSqe::Config(u16 streamId, u16 taskId, u64 notifyId, u32 cntValue)
123 : {
124 1 : sqe->header.rtStreamId = streamId;
125 1 : sqe->header.taskId = taskId;
126 1 : sqe->notifyId = notifyId;
127 1 : sqe->cntValue = cntValue;
128 3 : HCCL_INFO(
129 : "[SQE]HcclUBCntNotifyNto1RecordSqe streamId=%u, taskId=%u, notifyId=%llu, cntValue=%u", streamId, taskId,
130 : notifyId, cntValue);
131 1 : }
132 :
133 1 : u64 HcclUBCntNotifyNto1RecordSqe::GetSqe() { return reinterpret_cast<u64>(sqe.get()); }
134 :
135 1 : HcclUBCntNotify1toNWaitSqe::HcclUBCntNotify1toNWaitSqe()
136 : {
137 1 : sqe = std::make_unique<Rt91095StarsNotifySqe>();
138 1 : (void)memset_s(sqe.get(), sizeof(Rt91095StarsNotifySqe), 0, sizeof(Rt91095StarsNotifySqe));
139 1 : sqe->kernelCredit = RT_STARS_NEVER_TIMEOUT_KERNEL_CREDIT;
140 1 : sqe->header.type = static_cast<uint8_t>(Rt91095StarsSqeType::RT_91095_SQE_TYPE_NOTIFY_WAIT);
141 1 : sqe->cntFlag = true;
142 1 : sqe->clrFlag = true;
143 1 : sqe->bitmap = 1U;
144 1 : sqe->subType = static_cast<uint16_t>(Rt91095NotifySubType::NOTIFY_SUB_TYPE_COUNT_NOTIFY_WAIT);
145 1 : }
146 :
147 1 : void HcclUBCntNotify1toNWaitSqe::Config(u16 streamId, u16 taskId, u64 notifyId, u32 cntValue)
148 : {
149 1 : sqe->header.rtStreamId = streamId;
150 1 : sqe->header.taskId = taskId;
151 1 : sqe->notifyId = notifyId;
152 1 : sqe->cntValue = cntValue;
153 3 : HCCL_INFO(
154 : "[SQE]HcclUBCntNotify1toNWaitSqe streamId=%u, taskId=%u, notifyId=%llu, cntValue=%u", streamId, taskId,
155 : notifyId, cntValue);
156 1 : }
157 :
158 1 : u64 HcclUBCntNotify1toNWaitSqe::GetSqe() { return reinterpret_cast<u64>(sqe.get()); }
159 :
160 1 : HcclUBCntNotifyNto1WaitSqe::HcclUBCntNotifyNto1WaitSqe()
161 : {
162 1 : sqe = std::make_unique<Rt91095StarsNotifySqe>();
163 1 : (void)memset_s(sqe.get(), sizeof(Rt91095StarsNotifySqe), 0, sizeof(Rt91095StarsNotifySqe));
164 1 : sqe->kernelCredit = RT_STARS_NEVER_TIMEOUT_KERNEL_CREDIT;
165 1 : sqe->header.type = static_cast<uint8_t>(Rt91095StarsSqeType::RT_91095_SQE_TYPE_NOTIFY_WAIT);
166 1 : sqe->cntFlag = true;
167 1 : sqe->clrFlag = true;
168 1 : sqe->waitModeBit = 0x1U; // rtCntNotifyWaitMode_t::WAIT_EQUAL_MODE
169 1 : sqe->subType = static_cast<uint16_t>(Rt91095NotifySubType::NOTIFY_SUB_TYPE_COUNT_NOTIFY_WAIT);
170 1 : }
171 :
172 1 : void HcclUBCntNotifyNto1WaitSqe::Config(u16 streamId, u16 taskId, u64 notifyId, u32 cntValue)
173 : {
174 1 : sqe->header.rtStreamId = streamId;
175 1 : sqe->header.taskId = taskId;
176 1 : sqe->notifyId = notifyId;
177 1 : sqe->cntValue = cntValue;
178 3 : HCCL_INFO(
179 : "[SQE]HcclUBCntNotifyNto1WaitSqe streamId=%u, taskId=%u, notifyId=%llu, cntValue=%u", streamId, taskId,
180 : notifyId, cntValue);
181 1 : }
182 :
183 1 : u64 HcclUBCntNotifyNto1WaitSqe::GetSqe() { return reinterpret_cast<u64>(sqe.get()); }
184 :
185 1 : HcclUBCntNotify1toNRecordSqe::HcclUBCntNotify1toNRecordSqe()
186 : {
187 1 : sqe = std::make_unique<Rt91095StarsNotifySqe>();
188 1 : (void)memset_s(sqe.get(), sizeof(Rt91095StarsNotifySqe), 0, sizeof(Rt91095StarsNotifySqe));
189 1 : sqe->header.type = static_cast<uint8_t>(Rt91095StarsSqeType::RT_91095_SQE_TYPE_NOTIFY_RECORD);
190 1 : sqe->kernelCredit = RT_STARS_DEFAULT_KERNEL_CREDIT;
191 1 : sqe->clrFlag = false;
192 1 : sqe->cntFlag = true;
193 1 : sqe->recordModeBit = 0x0U; // rtCntNotifyRecordMode_t::RECORD_STORE_MODE
194 1 : sqe->subType = static_cast<uint16_t>(Rt91095NotifySubType::NOTIFY_SUB_TYPE_COUNT_NOTIFY_RECORD);
195 1 : }
196 :
197 1 : void HcclUBCntNotify1toNRecordSqe::Config(u16 streamId, u16 taskId, u64 notifyId, u32 cntValue)
198 : {
199 1 : sqe->header.rtStreamId = streamId;
200 1 : sqe->header.taskId = taskId;
201 1 : sqe->notifyId = notifyId;
202 1 : sqe->cntValue = cntValue;
203 3 : HCCL_INFO(
204 : "[SQE]HcclUBCntNotify1toNRecordSqe streamId=%u, taskId=%u, notifyId=%llu, cntValue=%u", streamId, taskId,
205 : notifyId, cntValue);
206 1 : }
207 :
208 1 : u64 HcclUBCntNotify1toNRecordSqe::GetSqe() { return reinterpret_cast<u64>(sqe.get()); }
209 :
210 1 : HcclUBMemcpySqe::HcclUBMemcpySqe()
211 : {
212 1 : sqe = std::make_unique<Rt91095StarsMemcpySqe>();
213 1 : (void)memset_s(sqe.get(), sizeof(Rt91095StarsMemcpySqe), 0, sizeof(Rt91095StarsMemcpySqe));
214 1 : sqe->header.type = static_cast<uint8_t>(Rt91095StarsSqeType::RT_91095_SQE_TYPE_SDMA);
215 1 : sqe->header.lock = 0U;
216 1 : sqe->header.unlock = 0U;
217 1 : sqe->header.ie = 0U;
218 1 : sqe->header.wrCqe = 0U;
219 1 : sqe->header.ptrMode = 0U;
220 1 : sqe->header.rttMode = 0U;
221 1 : sqe->header.headUpdate = 0U;
222 1 : sqe->header.reserved = 0U;
223 1 : sqe->header.numBlocks = 0U;
224 :
225 1 : sqe->kernelCredit = RT_STARS_DEFAULT_KERNEL_CREDIT;
226 :
227 1 : sqe->ie2 = 0U;
228 1 : sqe->sssv = 1U;
229 1 : sqe->dssv = 1U;
230 1 : sqe->sns = 1U;
231 1 : sqe->qos = 0U;
232 1 : sqe->dns = 1U;
233 1 : sqe->sro = 0U;
234 1 : sqe->dro = 0U;
235 1 : sqe->mapamPartId = 0U; // 这里走的memcpy，如果走withcfg,需要传入qoscfg
236 1 : sqe->mpamns = 0U;
237 1 : sqe->stride = 0U;
238 1 : sqe->compEn = 0U;
239 1 : sqe->pmg = 0U;
240 1 : sqe->res1 = 0U;
241 1 : sqe->res2 = 0U;
242 1 : sqe->res3 = 0U;
243 1 : sqe->res4 = 0U;
244 :
245 1 : sqe->d2dOffsetFlag = 0U;
246 1 : sqe->u.strideMode0.srcOffsetLow = 0U;
247 1 : sqe->u.strideMode0.dstOffsetLow = 0U;
248 1 : sqe->u.strideMode0.srcOffsetHigh = 0U;
249 3 : HCCL_INFO("[SQE]HcclUBMemcpySqe construct end");
250 1 : }
251 :
252 1 : void HcclUBMemcpySqe::Config(
253 : u16 streamId, u16 taskId, RtDataType rtDataType, RtReduceKind rtReduceOp, u64 count, const u64* src, const u64* dst,
254 : u32 partId)
255 : {
256 1 : sqe->header.rtStreamId = streamId;
257 1 : sqe->header.taskId = taskId;
258 1 : const bool isReduce
259 : = ((rtReduceOp == RtReduceKind::RT_MEMCPY_SDMA_AUTOMATIC_ADD)
260 0 : || (rtReduceOp == RtReduceKind::RT_MEMCPY_SDMA_AUTOMATIC_MAX)
261 0 : || (rtReduceOp == RtReduceKind::RT_MEMCPY_SDMA_AUTOMATIC_MIN)
262 1 : || (rtReduceOp == RtReduceKind::RT_MEMCPY_SDMA_AUTOMATIC_EQUAL));
263 1 : sqe->opcode = isReduce ? GetUBOpCode(static_cast<u32>(rtReduceOp), static_cast<u8>(rtDataType)) : 0U;
264 :
265 1 : sqe->u.strideMode0.lengthMove = count;
266 1 : sqe->u.strideMode0.srcAddrLow = static_cast<uint32_t>(static_cast<uint64_t>(*src) & 0x00000000ffffffffU);
267 1 : sqe->u.strideMode0.srcAddrHigh
268 1 : = static_cast<uint32_t>((static_cast<uint64_t>(*src) & 0xffffffff00000000U) >> UINT32_BIT_NUM);
269 1 : sqe->u.strideMode0.dstAddrLow = static_cast<uint32_t>(static_cast<uint64_t>(*dst) & 0x00000000ffffffffU);
270 1 : sqe->u.strideMode0.dstAddrHigh
271 1 : = static_cast<uint32_t>((static_cast<uint64_t>(*dst) & 0xffffffff00000000U) >> UINT32_BIT_NUM);
272 1 : sqe->mapamPartId = partId;
273 :
274 3 : HCCL_INFO(
275 : "[SQE]HcclUBMemcpySqe dataType=%u,rtReduceOp =%u, count=%llu, src=%p, dst=%p, partId=%u, streamId=%u, "
276 : "taskId=%u",
277 : rtDataType, rtReduceOp, count, src, dst, partId, streamId, taskId);
278 3 : HCCL_INFO(
279 : "[SQE]HcclUBMemcpySqe sqe->opcode=%u sqe->u.strideMode0.srcAddrLow=0x%x, "
280 : "sqe->u.strideMode0.srcAddrHigh=0x%x,sqe->u.strideMode0.dstAddrLow=0x%x, "
281 : "sqe->u.strideMode0.dstAddrHigh=0x%x",
282 : sqe->opcode, sqe->u.strideMode0.srcAddrLow, sqe->u.strideMode0.srcAddrHigh, sqe->u.strideMode0.dstAddrLow,
283 : sqe->u.strideMode0.dstAddrHigh);
284 1 : }
285 :
286 1 : u64 HcclUBMemcpySqe::GetSqe() { return reinterpret_cast<u64>(sqe.get()); }
287 :
288 : // change name: convert
289 1 : u8 HcclUBMemcpySqe::ConvertToMemcpyDataType(u8 copyDataType) const
290 : {
291 : u8 opcode;
292 1 : switch (static_cast<RtDataType>(copyDataType)) {
293 0 : case RtDataType::RT_DATA_TYPE_INT8: {
294 0 : opcode = static_cast<u8>(RtStarsMemcpyAsyncDataType::RT_STARS_MEMCPY_ASYNC_DATA_TYPE_INT8);
295 0 : break;
296 : }
297 1 : case RtDataType::RT_DATA_TYPE_INT16: {
298 1 : opcode = static_cast<u8>(RtStarsMemcpyAsyncDataType::RT_STARS_MEMCPY_ASYNC_DATA_TYPE_INT16);
299 1 : break;
300 : }
301 0 : case RtDataType::RT_DATA_TYPE_INT32: {
302 0 : opcode = static_cast<u8>(RtStarsMemcpyAsyncDataType::RT_STARS_MEMCPY_ASYNC_DATA_TYPE_INT32);
303 0 : break;
304 : }
305 0 : case RtDataType::RT_DATA_TYPE_FP16: {
306 0 : opcode = static_cast<u8>(RtStarsMemcpyAsyncDataType::RT_STARS_MEMCPY_ASYNC_DATA_TYPE_FP16);
307 0 : break;
308 : }
309 0 : case RtDataType::RT_DATA_TYPE_FP32: {
310 0 : opcode = static_cast<u8>(RtStarsMemcpyAsyncDataType::RT_STARS_MEMCPY_ASYNC_DATA_TYPE_FP32);
311 0 : break;
312 : }
313 0 : case RtDataType::RT_DATA_TYPE_BFP16: {
314 0 : opcode = static_cast<u8>(RtStarsMemcpyAsyncDataType::RT_STARS_MEMCPY_ASYNC_DATA_TYPE_BFP16);
315 0 : break;
316 : }
317 0 : default: {
318 : // Should not run here.
319 : // Only for code style, 0x80 is reserved value of STRAS opcode.
320 0 : MACRO_THROW(
321 : NotSupportException, StringFormat("DataType=%u do not support.", static_cast<u32>(copyDataType)));
322 : break;
323 : }
324 : }
325 1 : return opcode;
326 : }
327 :
328 1 : u8 HcclUBMemcpySqe::ConvertToMemcpyOpType(u32 copyKind) const
329 : {
330 : u8 opcode;
331 1 : switch (static_cast<RtReduceKind>(copyKind)) {
332 1 : case RtReduceKind::RT_MEMCPY_SDMA_AUTOMATIC_ADD: {
333 1 : opcode = static_cast<u8>(RtStarsMemcpyAsyncOperationKind::RT_STARS_MEMCPY_ASYNC_OP_KIND_ADD);
334 1 : break;
335 : }
336 0 : case RtReduceKind::RT_MEMCPY_SDMA_AUTOMATIC_MAX: {
337 0 : opcode = static_cast<u8>(RtStarsMemcpyAsyncOperationKind::RT_STARS_MEMCPY_ASYNC_OP_KIND_MAX);
338 0 : break;
339 : }
340 0 : case RtReduceKind::RT_MEMCPY_SDMA_AUTOMATIC_MIN: {
341 0 : opcode = static_cast<u8>(RtStarsMemcpyAsyncOperationKind::RT_STARS_MEMCPY_ASYNC_OP_KIND_MIN);
342 0 : break;
343 : }
344 0 : case RtReduceKind::RT_MEMCPY_SDMA_AUTOMATIC_EQUAL: {
345 0 : opcode = static_cast<u8>(RtStarsMemcpyAsyncOperationKind::RT_STARS_MEMCPY_ASYNC_OP_KIND_EQUAL);
346 0 : break;
347 : }
348 0 : default: {
349 0 : MACRO_THROW(NotSupportException, StringFormat("Type out of range: copyKind=%u", copyKind));
350 : break;
351 : }
352 : }
353 1 : return opcode;
354 : }
355 :
356 1 : u8 HcclUBMemcpySqe::GetUBOpCode(u32 copyKind, u8 copyDataType) const
357 : {
358 1 : const u8 memcpyDataType = ConvertToMemcpyDataType(copyDataType);
359 1 : const u8 opType = ConvertToMemcpyOpType(copyKind);
360 : // opcode: 高4bit为datatype，低4bit为optype
361 1 : return memcpyDataType | opType;
362 : }
363 :
364 : } // namespace Hccl
|
