Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "hccl_sqe.h"
12 : #include "exception_util.h"
13 : #include "not_support_exception.h"
14 :
15 : namespace Hccl {
16 :
17 7 : u32 GetAddrLow(u64 addr)
18 : {
19 7 : return static_cast<u32>(addr & LOW32_BIT_MASK);
20 : }
21 :
22 7 : u32 GetAddrHigh(u64 addr)
23 : {
24 7 : return static_cast<u32>((addr & HIGH32_BIT_MASK) >> UINT32_BIT_NUM);
25 : }
26 :
27 8 : HcclNotifyWaitSqe::HcclNotifyWaitSqe()
28 : {
29 8 : sqe = std::make_unique<RtStarsNotifySqe>();
30 8 : sqe->header.type = static_cast<uint8_t>(RtStarsSqeType::RT_STARS_SQE_TYPE_NOTIFY_WAIT);
31 8 : sqe->kernelCredit = RT_STARS_NEVER_TIMEOUT_KERNEL_CREDIT;
32 8 : }
33 :
34 6 : void HcclNotifyWaitSqe::Config(u16 streamId, u16 taskId, u64 notifyId)
35 : {
36 6 : sqe->header.rtStreamId = streamId;
37 6 : sqe->camelBack = notifyId;
38 6 : sqe->header.taskId = taskId;
39 18 : HCCL_INFO("[SQE] notify wait: notifyId=%llu, streamId=%u, taskId=%u.", notifyId, streamId, taskId);
40 6 : }
41 :
42 10 : u64 HcclNotifyWaitSqe::GetSqe()
43 : {
44 10 : return reinterpret_cast<u64>(sqe.get());
45 : }
46 :
47 2 : HcclNotifyRecordSqe::HcclNotifyRecordSqe()
48 : {
49 2 : sqe = std::make_unique<RtStarsNotifySqe>();
50 2 : sqe->header.type = static_cast<uint8_t>(RtStarsSqeType::RT_STARS_SQE_TYPE_NOTIFY_RECORD);
51 2 : sqe->kernelCredit = RT_STARS_NEVER_TIMEOUT_KERNEL_CREDIT;
52 2 : }
53 1 : void HcclNotifyRecordSqe::Config(u16 streamId, u16 taskId, u64 notifyId)
54 : {
55 1 : sqe->header.rtStreamId = streamId;
56 1 : sqe->camelBack = notifyId;
57 1 : sqe->header.taskId = taskId;
58 3 : HCCL_INFO("[SQE] notify record: notifyId=%llu, streamId=%u, taskId=%u.", notifyId, streamId, taskId);
59 1 : }
60 2 : u64 HcclNotifyRecordSqe::GetSqe()
61 : {
62 2 : return reinterpret_cast<u64>(sqe.get());
63 : }
64 :
65 2 : HcclWriteValueSqe::HcclWriteValueSqe()
66 : {
67 2 : sqe = std::make_unique<RtStarsWriteValueSqe>();
68 2 : sqe->header.type = static_cast<uint8_t>(RtStarsSqeType::RT_STARS_SQE_TYPE_WRITE_VALUE);
69 2 : sqe->kernelCredit = RT_STARS_DEFAULT_KERNEL_CREDIT;
70 2 : sqe->awsize = RtStarsWriteValueSizeType::RT_STARS_WRITE_VALUE_SIZE_TYPE_32BIT;
71 2 : sqe->writeValuePart0 = 1U;
72 2 : sqe->subType = RtStarsWriteValueSubType::RT_STARS_WRITE_VALUE_SUB_TYPE_NOTIFY_RECORD_IPC_NO_PCIE;
73 2 : }
74 :
75 1 : void HcclWriteValueSqe::Config(u16 streamId, u16 taskId, u64 notifyWRAddr)
76 : {
77 1 : sqe->header.rtStreamId = streamId;
78 1 : sqe->header.taskId = taskId;
79 1 : sqe->writeAddrLow = GetAddrLow(notifyWRAddr);
80 1 : sqe->writeAddrHigh = GetAddrHigh(notifyWRAddr) & MASK_17_BIT;
81 3 : HCCL_INFO("[SQE] write value: writePtr=0x%llx, streamId=%u, taskId=%u.", notifyWRAddr, streamId, taskId);
82 1 : }
83 :
84 2 : u64 HcclWriteValueSqe::GetSqe()
85 : {
86 2 : return reinterpret_cast<u64>(sqe.get());
87 : }
88 :
89 2 : HcclSdmaSqe::HcclSdmaSqe()
90 : {
91 2 : sqe = std::make_unique<RtStarsMemcpyAsyncSqe>();
92 2 : sqe->header.type = static_cast<uint8_t>(RtStarsSqeType::RT_STARS_SQE_TYPE_SDMA);
93 2 : sqe->kernelCredit = RT_STARS_DEFAULT_KERNEL_CREDIT;
94 2 : sqe->sssv = 1U;
95 2 : sqe->dssv = 1U;
96 2 : sqe->sns = 1U;
97 2 : sqe->dns = 1U;
98 2 : sqe->qos = 6; // 6 is HCCL QoS
99 2 : }
100 :
101 1 : void HcclSdmaSqe::Config(u16 streamId, u16 taskId, const u64 src, u32 length, RtDataType rtDataType,
102 : RtReduceKind rtReduceOp, const u64 dst, u32 partId)
103 : {
104 1 : sqe->header.rtStreamId = streamId;
105 1 : sqe->header.taskId = taskId;
106 1 : const bool isReduce
107 0 : = ((rtReduceOp == RtReduceKind::RT_MEMCPY_SDMA_AUTOMATIC_ADD) || (rtReduceOp == RtReduceKind::RT_MEMCPY_SDMA_AUTOMATIC_MAX)
108 1 : || (rtReduceOp == RtReduceKind::RT_MEMCPY_SDMA_AUTOMATIC_MIN) || (rtReduceOp == RtReduceKind::RT_MEMCPY_SDMA_AUTOMATIC_EQUAL));
109 1 : sqe->opcode = isReduce ? GetSdmaOpCode(static_cast<u32>(rtReduceOp), static_cast<u8>(rtDataType)) : 0U;
110 3 : HCCL_INFO("[SQE]MemcpySqe copyKind=%u,Opcode=0x%x, streamId=%u, len=%u, src:0x%llx, dst:0x%llx",
111 : static_cast<u32>(rtReduceOp), static_cast<u32>(sqe->opcode), streamId, length, src, dst);
112 1 : sqe->length = length;
113 :
114 : // extract common function for this
115 1 : sqe->srcAddrLow = GetAddrLow(src);
116 1 : sqe->srcAddrHigh = GetAddrHigh(src);
117 1 : sqe->dstAddrLow = GetAddrLow(dst);
118 1 : sqe->dstAddrHigh = GetAddrHigh(dst);
119 :
120 1 : sqe->partid = partId;
121 1 : }
122 :
123 1 : u8 HcclSdmaSqe::GetSdmaOpCode(u32 copyKind, u8 copyDataType) const
124 : {
125 1 : const u8 memcpyDataType = ConvertToMemcpyDataType(copyDataType);
126 1 : const u8 opType = ConvertToMemcpyOpType(copyKind);
127 : // opcode: 高4bit为datatype，低4bit为optype
128 1 : return memcpyDataType | opType;
129 : }
130 :
131 : // change name: convert
132 1 : u8 HcclSdmaSqe::ConvertToMemcpyDataType(u8 copyDataType) const
133 : {
134 : u8 opcode;
135 1 : switch (static_cast<RtDataType>(copyDataType)) {
136 0 : case RtDataType::RT_DATA_TYPE_INT8: {
137 0 : opcode = static_cast<u8>(RtStarsMemcpyAsyncDataType::RT_STARS_MEMCPY_ASYNC_DATA_TYPE_INT8);
138 0 : break;
139 : }
140 0 : case RtDataType::RT_DATA_TYPE_INT16: {
141 0 : opcode = static_cast<u8>(RtStarsMemcpyAsyncDataType::RT_STARS_MEMCPY_ASYNC_DATA_TYPE_INT16);
142 0 : break;
143 : }
144 0 : case RtDataType::RT_DATA_TYPE_INT32: {
145 0 : opcode = static_cast<u8>(RtStarsMemcpyAsyncDataType::RT_STARS_MEMCPY_ASYNC_DATA_TYPE_INT32);
146 0 : break;
147 : }
148 0 : case RtDataType::RT_DATA_TYPE_FP16: {
149 0 : opcode = static_cast<u8>(RtStarsMemcpyAsyncDataType::RT_STARS_MEMCPY_ASYNC_DATA_TYPE_FP16);
150 0 : break;
151 : }
152 1 : case RtDataType::RT_DATA_TYPE_FP32: {
153 1 : opcode = static_cast<u8>(RtStarsMemcpyAsyncDataType::RT_STARS_MEMCPY_ASYNC_DATA_TYPE_FP32);
154 1 : break;
155 : }
156 0 : case RtDataType::RT_DATA_TYPE_BFP16: {
157 0 : opcode = static_cast<u8>(RtStarsMemcpyAsyncDataType::RT_STARS_MEMCPY_ASYNC_DATA_TYPE_BFP16);
158 0 : break;
159 : }
160 0 : default: {
161 : // Should not run here.
162 : // Only for code style, 0x80 is reserved value of STRAS opcode.
163 0 : MACRO_THROW(NotSupportException,
164 : StringFormat("DataType=%u do not support.", static_cast<u32>(copyDataType)));
165 : break;
166 : }
167 : }
168 1 : return opcode;
169 : }
170 :
171 1 : u8 HcclSdmaSqe::ConvertToMemcpyOpType(u32 copyKind) const
172 : {
173 : u8 opcode;
174 1 : switch (static_cast<RtReduceKind>(copyKind)) {
175 1 : case RtReduceKind::RT_MEMCPY_SDMA_AUTOMATIC_ADD: {
176 1 : opcode = static_cast<u8>(RtStarsMemcpyAsyncOperationKind::RT_STARS_MEMCPY_ASYNC_OP_KIND_ADD);
177 1 : break;
178 : }
179 0 : case RtReduceKind::RT_MEMCPY_SDMA_AUTOMATIC_MAX: {
180 0 : opcode = static_cast<u8>(RtStarsMemcpyAsyncOperationKind::RT_STARS_MEMCPY_ASYNC_OP_KIND_MAX);
181 0 : break;
182 : }
183 0 : case RtReduceKind::RT_MEMCPY_SDMA_AUTOMATIC_MIN: {
184 0 : opcode = static_cast<u8>(RtStarsMemcpyAsyncOperationKind::RT_STARS_MEMCPY_ASYNC_OP_KIND_MIN);
185 0 : break;
186 : }
187 0 : case RtReduceKind::RT_MEMCPY_SDMA_AUTOMATIC_EQUAL: {
188 0 : opcode = static_cast<u8>(RtStarsMemcpyAsyncOperationKind::RT_STARS_MEMCPY_ASYNC_OP_KIND_EQUAL);
189 0 : break;
190 : }
191 0 : default: {
192 0 : MACRO_THROW(NotSupportException, StringFormat("Type out of range: copyKind=%u", copyKind));
193 : break;
194 : }
195 : }
196 1 : return opcode;
197 : }
198 :
199 2 : u64 HcclSdmaSqe::GetSqe()
200 : {
201 2 : return reinterpret_cast<u64>(sqe.get());
202 : }
203 :
204 : } // namespace Hccl
|
