Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "hccl_sqe.h"
12 : #include "exception_util.h"
13 : #include "not_support_exception.h"
14 :
15 : namespace Hccl {
16 :
17 7 : u32 GetAddrLow(u64 addr) { return static_cast<u32>(addr & LOW32_BIT_MASK); }
18 :
19 7 : u32 GetAddrHigh(u64 addr) { return static_cast<u32>((addr & HIGH32_BIT_MASK) >> UINT32_BIT_NUM); }
20 :
21 8 : HcclNotifyWaitSqe::HcclNotifyWaitSqe()
22 : {
23 8 : sqe = std::make_unique<RtStarsNotifySqe>();
24 8 : sqe->header.type = static_cast<uint8_t>(RtStarsSqeType::RT_STARS_SQE_TYPE_NOTIFY_WAIT);
25 8 : sqe->kernelCredit = RT_STARS_NEVER_TIMEOUT_KERNEL_CREDIT;
26 8 : }
27 :
28 6 : void HcclNotifyWaitSqe::Config(u16 streamId, u16 taskId, u64 notifyId)
29 : {
30 6 : sqe->header.rtStreamId = streamId;
31 6 : sqe->camelBack = notifyId;
32 6 : sqe->header.taskId = taskId;
33 18 : HCCL_INFO("[SQE] notify wait: notifyId=%llu, streamId=%u, taskId=%u.", notifyId, streamId, taskId);
34 6 : }
35 :
36 10 : u64 HcclNotifyWaitSqe::GetSqe() { return reinterpret_cast<u64>(sqe.get()); }
37 :
38 2 : HcclNotifyRecordSqe::HcclNotifyRecordSqe()
39 : {
40 2 : sqe = std::make_unique<RtStarsNotifySqe>();
41 2 : sqe->header.type = static_cast<uint8_t>(RtStarsSqeType::RT_STARS_SQE_TYPE_NOTIFY_RECORD);
42 2 : sqe->kernelCredit = RT_STARS_NEVER_TIMEOUT_KERNEL_CREDIT;
43 2 : }
44 1 : void HcclNotifyRecordSqe::Config(u16 streamId, u16 taskId, u64 notifyId)
45 : {
46 1 : sqe->header.rtStreamId = streamId;
47 1 : sqe->camelBack = notifyId;
48 1 : sqe->header.taskId = taskId;
49 3 : HCCL_INFO("[SQE] notify record: notifyId=%llu, streamId=%u, taskId=%u.", notifyId, streamId, taskId);
50 1 : }
51 2 : u64 HcclNotifyRecordSqe::GetSqe() { return reinterpret_cast<u64>(sqe.get()); }
52 :
53 2 : HcclWriteValueSqe::HcclWriteValueSqe()
54 : {
55 2 : sqe = std::make_unique<RtStarsWriteValueSqe>();
56 2 : sqe->header.type = static_cast<uint8_t>(RtStarsSqeType::RT_STARS_SQE_TYPE_WRITE_VALUE);
57 2 : sqe->kernelCredit = RT_STARS_DEFAULT_KERNEL_CREDIT;
58 2 : sqe->awsize = RtStarsWriteValueSizeType::RT_STARS_WRITE_VALUE_SIZE_TYPE_32BIT;
59 2 : sqe->writeValuePart0 = 1U;
60 2 : sqe->subType = RtStarsWriteValueSubType::RT_STARS_WRITE_VALUE_SUB_TYPE_NOTIFY_RECORD_IPC_NO_PCIE;
61 2 : }
62 :
63 1 : void HcclWriteValueSqe::Config(u16 streamId, u16 taskId, u64 notifyWRAddr)
64 : {
65 1 : sqe->header.rtStreamId = streamId;
66 1 : sqe->header.taskId = taskId;
67 1 : sqe->writeAddrLow = GetAddrLow(notifyWRAddr);
68 1 : sqe->writeAddrHigh = GetAddrHigh(notifyWRAddr) & MASK_17_BIT;
69 3 : HCCL_INFO("[SQE] write value: writePtr=0x%llx, streamId=%u, taskId=%u.", notifyWRAddr, streamId, taskId);
70 1 : }
71 :
72 2 : u64 HcclWriteValueSqe::GetSqe() { return reinterpret_cast<u64>(sqe.get()); }
73 :
74 2 : HcclSdmaSqe::HcclSdmaSqe()
75 : {
76 2 : sqe = std::make_unique<RtStarsMemcpyAsyncSqe>();
77 2 : sqe->header.type = static_cast<uint8_t>(RtStarsSqeType::RT_STARS_SQE_TYPE_SDMA);
78 2 : sqe->kernelCredit = RT_STARS_DEFAULT_KERNEL_CREDIT;
79 2 : sqe->sssv = 1U;
80 2 : sqe->dssv = 1U;
81 2 : sqe->sns = 1U;
82 2 : sqe->dns = 1U;
83 2 : sqe->qos = 6; // 6 is HCCL QoS
84 2 : }
85 :
86 1 : void HcclSdmaSqe::Config(
87 : u16 streamId, u16 taskId, const u64 src, u32 length, RtDataType rtDataType, RtReduceKind rtReduceOp, const u64 dst,
88 : u32 partId)
89 : {
90 1 : sqe->header.rtStreamId = streamId;
91 1 : sqe->header.taskId = taskId;
92 1 : const bool isReduce
93 : = ((rtReduceOp == RtReduceKind::RT_MEMCPY_SDMA_AUTOMATIC_ADD)
94 0 : || (rtReduceOp == RtReduceKind::RT_MEMCPY_SDMA_AUTOMATIC_MAX)
95 0 : || (rtReduceOp == RtReduceKind::RT_MEMCPY_SDMA_AUTOMATIC_MIN)
96 1 : || (rtReduceOp == RtReduceKind::RT_MEMCPY_SDMA_AUTOMATIC_EQUAL));
97 1 : sqe->opcode = isReduce ? GetSdmaOpCode(static_cast<u32>(rtReduceOp), static_cast<u8>(rtDataType)) : 0U;
98 3 : HCCL_INFO(
99 : "[SQE]MemcpySqe copyKind=%u,Opcode=0x%x, streamId=%u, len=%u, src:0x%llx, dst:0x%llx",
100 : static_cast<u32>(rtReduceOp), static_cast<u32>(sqe->opcode), streamId, length, src, dst);
101 1 : sqe->length = length;
102 :
103 : // extract common function for this
104 1 : sqe->srcAddrLow = GetAddrLow(src);
105 1 : sqe->srcAddrHigh = GetAddrHigh(src);
106 1 : sqe->dstAddrLow = GetAddrLow(dst);
107 1 : sqe->dstAddrHigh = GetAddrHigh(dst);
108 :
109 1 : sqe->partid = partId;
110 1 : }
111 :
112 1 : u8 HcclSdmaSqe::GetSdmaOpCode(u32 copyKind, u8 copyDataType) const
113 : {
114 1 : const u8 memcpyDataType = ConvertToMemcpyDataType(copyDataType);
115 1 : const u8 opType = ConvertToMemcpyOpType(copyKind);
116 : // opcode: 高4bit为datatype，低4bit为optype
117 1 : return memcpyDataType | opType;
118 : }
119 :
120 : // change name: convert
121 1 : u8 HcclSdmaSqe::ConvertToMemcpyDataType(u8 copyDataType) const
122 : {
123 : u8 opcode;
124 1 : switch (static_cast<RtDataType>(copyDataType)) {
125 0 : case RtDataType::RT_DATA_TYPE_INT8: {
126 0 : opcode = static_cast<u8>(RtStarsMemcpyAsyncDataType::RT_STARS_MEMCPY_ASYNC_DATA_TYPE_INT8);
127 0 : break;
128 : }
129 0 : case RtDataType::RT_DATA_TYPE_INT16: {
130 0 : opcode = static_cast<u8>(RtStarsMemcpyAsyncDataType::RT_STARS_MEMCPY_ASYNC_DATA_TYPE_INT16);
131 0 : break;
132 : }
133 0 : case RtDataType::RT_DATA_TYPE_INT32: {
134 0 : opcode = static_cast<u8>(RtStarsMemcpyAsyncDataType::RT_STARS_MEMCPY_ASYNC_DATA_TYPE_INT32);
135 0 : break;
136 : }
137 0 : case RtDataType::RT_DATA_TYPE_FP16: {
138 0 : opcode = static_cast<u8>(RtStarsMemcpyAsyncDataType::RT_STARS_MEMCPY_ASYNC_DATA_TYPE_FP16);
139 0 : break;
140 : }
141 1 : case RtDataType::RT_DATA_TYPE_FP32: {
142 1 : opcode = static_cast<u8>(RtStarsMemcpyAsyncDataType::RT_STARS_MEMCPY_ASYNC_DATA_TYPE_FP32);
143 1 : break;
144 : }
145 0 : case RtDataType::RT_DATA_TYPE_BFP16: {
146 0 : opcode = static_cast<u8>(RtStarsMemcpyAsyncDataType::RT_STARS_MEMCPY_ASYNC_DATA_TYPE_BFP16);
147 0 : break;
148 : }
149 0 : default: {
150 : // Should not run here.
151 : // Only for code style, 0x80 is reserved value of STRAS opcode.
152 0 : MACRO_THROW(
153 : NotSupportException, StringFormat("DataType=%u do not support.", static_cast<u32>(copyDataType)));
154 : break;
155 : }
156 : }
157 1 : return opcode;
158 : }
159 :
160 1 : u8 HcclSdmaSqe::ConvertToMemcpyOpType(u32 copyKind) const
161 : {
162 : u8 opcode;
163 1 : switch (static_cast<RtReduceKind>(copyKind)) {
164 1 : case RtReduceKind::RT_MEMCPY_SDMA_AUTOMATIC_ADD: {
165 1 : opcode = static_cast<u8>(RtStarsMemcpyAsyncOperationKind::RT_STARS_MEMCPY_ASYNC_OP_KIND_ADD);
166 1 : break;
167 : }
168 0 : case RtReduceKind::RT_MEMCPY_SDMA_AUTOMATIC_MAX: {
169 0 : opcode = static_cast<u8>(RtStarsMemcpyAsyncOperationKind::RT_STARS_MEMCPY_ASYNC_OP_KIND_MAX);
170 0 : break;
171 : }
172 0 : case RtReduceKind::RT_MEMCPY_SDMA_AUTOMATIC_MIN: {
173 0 : opcode = static_cast<u8>(RtStarsMemcpyAsyncOperationKind::RT_STARS_MEMCPY_ASYNC_OP_KIND_MIN);
174 0 : break;
175 : }
176 0 : case RtReduceKind::RT_MEMCPY_SDMA_AUTOMATIC_EQUAL: {
177 0 : opcode = static_cast<u8>(RtStarsMemcpyAsyncOperationKind::RT_STARS_MEMCPY_ASYNC_OP_KIND_EQUAL);
178 0 : break;
179 : }
180 0 : default: {
181 0 : MACRO_THROW(NotSupportException, StringFormat("Type out of range: copyKind=%u", copyKind));
182 : break;
183 : }
184 : }
185 1 : return opcode;
186 : }
187 :
188 2 : u64 HcclSdmaSqe::GetSqe() { return reinterpret_cast<u64>(sqe.get()); }
189 :
190 : } // namespace Hccl
|
