Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #ifndef __AICPU_HCCL_SQCQ_H__
12 : #define __AICPU_HCCL_SQCQ_H__
13 :
14 : #include <memory>
15 : #include <vector>
16 : #include "ascend_hal.h"
17 : #include "log.h"
18 : #include "rt_external.h"
19 : #include "acl/acl_rt.h"
20 : #include "driver/ascend_hal_define.h"
21 :
22 : namespace dfx {
23 : enum class CqeStatus : int64_t {
24 : kDefault = 0,
25 : kCqeException,
26 : kCqeTimeOut,
27 : kCqeInnerError,
28 : kCqeUnknown,
29 : };
30 :
31 : const uint64_t kCreditTimeInvalid = 255U;
32 : const uint64_t kCreditTimeDefault = 240U; // 240对应到rts的credit字段就代表960s, 硬件同步时间
33 : const uint64_t kTimeOutTimeInvalid = 0U;
34 : const uint64_t kKfcTimeOut = 960U; // 960s
35 : const uint64_t kSqFullWaitTimeOut = 60U; // 60s, rtsq full等待时间要大于超时代答时间
36 : const uint64_t kPrintSqInterval = 30U; // 算子执行阶段打印sqe状态的间隔，单位s
37 :
38 : struct DfxTimeOutConfig {
39 : uint64_t sqeTimeOutTimeOut; // 软件同步
40 : uint64_t sqeCreditTimeOut; // 硬件同步
41 : bool useCredit; // 是否使用硬件同步
42 : uint64_t sqeWaitTimeOut; // kfc自己的超时
43 : uint64_t sqFullWaitTimeOut; // sq满的时候等待的超时
44 542 : std::string ToString() const
45 : {
46 542 : std::stringstream ss;
47 542 : ss << "sqeTimeOutTimeOut: " << sqeTimeOutTimeOut << " s";
48 542 : ss << " sqeCreditTimeOut: " << sqeCreditTimeOut << " s";
49 542 : ss << " useCredit: " << useCredit;
50 542 : ss << " sqeWaitTimeOut: " << sqeWaitTimeOut << " s";
51 542 : ss << " sqFullWaitTimeOut: " << sqFullWaitTimeOut << " s";
52 1084 : return ss.str();
53 542 : }
54 : };
55 :
56 : struct CqeQueryInput {
57 : uint32_t devId;
58 : uint16_t streamId;
59 : uint32_t sqId;
60 : uint32_t cqId;
61 : uint32_t type;
62 : uint8_t* cqeAddr;
63 12 : std::string ToString() const
64 : {
65 12 : std::stringstream ss;
66 12 : ss << "devId: " << devId;
67 12 : ss << " streamId: " << streamId;
68 12 : ss << " sqId: " << sqId;
69 12 : ss << " cqId: " << cqId;
70 12 : ss << " type: " << type;
71 24 : return ss.str();
72 12 : }
73 : };
74 : } // namespace dfx
75 : namespace ts {
76 : enum ts_app_abort_status {
77 : APP_ABORT_TERMINATE_FAIL = 0x0U,
78 : APP_ABORT_INIT,
79 : APP_ABORT_KILL_FINISH,
80 : APP_ABORT_TERMINATE_FINISH,
81 : APP_ABORT_STATUS_INVALID,
82 : };
83 : }
84 : using rtError_t = int32_t;
85 : constexpr int32_t RT_ERROR_FEATURE_NOT_SUPPORT = 0x0711000d;
86 :
87 : constexpr uint32_t UINT32_BIT_NUM = 32U;
88 : constexpr uint32_t MASK_32_BIT = 0xFFFFFFFFU;
89 : constexpr uint32_t MASK_17_BIT = 0x0001FFFFU;
90 :
91 : constexpr uint16_t RT_TASK_TYPE_FLIP = 98;
92 : constexpr uint8_t RT_STARS_NEVER_TIMEOUT_KERNEL_CREDIT = dfx::kCreditTimeInvalid;
93 : constexpr uint8_t RT_STARS_DEFAULT_KERNEL_CREDIT = RT_STARS_NEVER_TIMEOUT_KERNEL_CREDIT - 1U;
94 : constexpr uint8_t RT_STARS_EXIST_ERROR = 0x3FU;
95 : constexpr uint8_t RT_STARS_EXIST_WARNING = 0x40U;
96 : constexpr uint16_t RT_SYNC_TASK_FLAG = 0x8000U;
97 : constexpr u64 RT_GET_HEAD_CYCLE_NUM = 100000000U;
98 : constexpr uint8_t RT_STARS_AICPU_MODEL_KERNEL_CREDIT = 18U;
99 : constexpr uint8_t RT_STARS_AICPU_DEFAULT_TIMEOUT = 28U;
100 : constexpr uint8_t RT_STARS_TASK_KERNEL_CREDIT_SCALE_UINT8 = 4U;
101 :
102 : constexpr u64 RT_CHIP_ADDR_OFFSET = 0x80000000000ULL;
103 : constexpr u64 RT_DIE_ADDR_OFFSET = 0x10000000000ULL;
104 :
105 : constexpr u64 RT_STARS_PCIE_BASE_ADDR = 0x400004008000ULL;
106 : constexpr u64 RT_PCIE_LOCAL_DEV_OFFSET = 36ULL;
107 : constexpr u64 RT_PCIE_REMOTE_DEV_OFFSET = 32ULL;
108 :
109 : constexpr u64 STARS_NOTIFY_BASE_ADDR = 0x100000ULL;
110 : constexpr u64 STARS_NOTIFY_OFFSET = 0x4ULL;
111 : constexpr uint32_t STARS_NOTIFY_NUM_OF_SINGLE_TABLE = 512U;
112 : constexpr u64 STARS_NOTIFY_TABLE_OFFSET = 0x10000ULL;
113 :
114 : constexpr u64 RT_STARS_BASE_ADDR = 0x06a0000000ULL;
115 : constexpr u64 STARS_EVENT_BASE_ADDR = 0x200000ULL;
116 : constexpr u64 STARS_EVENT_OFFSET = 0x4ULL;
117 : constexpr uint32_t STARS_EVENT_NUM_OF_SINGLE_TABLE = 4096U;
118 : constexpr u64 STARS_EVENT_TABLE_OFFSET = 0x10000ULL;
119 :
120 : constexpr u32 RT_SDMA_COMPERR = 0x9; // A3 sdma error类型为0x9时，表示写拷贝发生超时代答，或者数据搬移时地址译码错误
121 : constexpr u32 RT_SDMA_COMPDATAERR = 0xa; // A3 sdma error类型为0xa时，表示读拷贝发生超时代答，或者读HBM返回ERROR
122 : constexpr u32 RT_SDMA_DATAERR = 0x8; // A3 sdma error类型为0x8时，表示读HBM返回ERROR
123 : constexpr uint16_t TS_ERROR_RETRY_CONSTRAINT = 1000; // 重执行失败，约束是算子不一致或者inplace算子不支持
124 : constexpr uint16_t TS_ERROR_AICPU_SDMA = 1001; // AICPU算子失败，sqe类型为SDMA是，上报给aicpu的框架错误码
125 :
126 : constexpr u32 AC_SQE_REV_MAX_CNT = 32U; // 910B平台下调用halCqReportRecv接口最大能够接收的个数
127 : constexpr uint32_t MAX_REPORT_CNT = 256U;
128 :
129 : enum SqeType : uint8_t {
130 : SQE_TYPE_DEFAULT = 0,
131 : NOTIFY_SQE,
132 : WRITE_VALUE_SQE,
133 : EVENT_SQE,
134 : MEMCPY_ASYNC_SQE,
135 : CCORE_WAIT_START_SQE,
136 : CCORE_WRITE_VALUE_SQE,
137 : NOTIFY_SQE_V2,
138 : WRITE_VALUE_SQE_V2,
139 : EVENT_SQE_V2,
140 : MEMCPY_ASYNC_SQE_V2,
141 : RDMA_DB_SEND_SQE,
142 : FLIP_PLACEHOLDER_SQE,
143 : MEMCPY_ASYNC_SQE_V3,
144 : CACHE_MEMCPY_PLACEHOLDER_SQE,
145 : CACHE_NOTIFY_PLACEHOLDER_SQE,
146 : CACHE_WRITE_VALUE_PLACEHOLDER_SQE,
147 : CACHE_MEMCPY_RECORD_PLACEHOLDER_SQE
148 : };
149 :
150 : constexpr aclDataType DT_MAP_TABLE[HCCL_DATA_TYPE_RESERVED + 1] = {
151 : ACL_INT8, /* HCCL_DATA_TYPE_INT8 = 0 */
152 : ACL_INT16, /* HCCL_DATA_TYPE_INT16 = 1 */
153 : ACL_INT32, /* HCCL_DATA_TYPE_INT32 = 2 */
154 : ACL_FLOAT16, /* HCCL_DATA_TYPE_FP16 = 3 */
155 : ACL_FLOAT, /* HCCL_DATA_TYPE_FP32 = 4 */
156 : ACL_DT_UNDEFINED, /* HCCL_DATA_TYPE_INT64 = 5 */
157 : ACL_DT_UNDEFINED, /* HCCL_DATA_TYPE_UINT64 = 6 */
158 : ACL_UINT8, /* HCCL_DATA_TYPE_UINT8 = 7 */
159 : ACL_UINT16, /* HCCL_DATA_TYPE_UINT16 = 8 */
160 : ACL_UINT32, /* HCCL_DATA_TYPE_UINT32 = 9 */
161 : ACL_DT_UNDEFINED, /* HCCL_DATA_TYPE_FP64 = 10 */
162 : ACL_BF16, /* HCCL_DATA_TYPE_BFP16 = 11 */
163 : ACL_DT_UNDEFINED /* HCCL_DATA_TYPE_RESERVED */
164 : };
165 :
166 : constexpr aclrtReduceKind ACL_RT_MEMCPY_INVALID = static_cast<aclrtReduceKind>(-1);
167 :
168 : constexpr aclrtReduceKind RK_MAP_TABLE[HCCL_REDUCE_RESERVED + 1] = {
169 : ACL_RT_MEMCPY_SDMA_AUTOMATIC_SUM, /* HCCL_REDUCE_SUM = 0 */
170 : ACL_RT_MEMCPY_INVALID, /* HCCL_REDUCE_PROD = 1 */
171 : ACL_RT_MEMCPY_SDMA_AUTOMATIC_MAX, /* HCCL_REDUCE_MAX = 2 */
172 : ACL_RT_MEMCPY_SDMA_AUTOMATIC_MIN, /* HCCL_REDUCE_MIN = 3 */
173 : ACL_RT_MEMCPY_INVALID, /* HCCL_REDUCE_RESERVED = 4 */
174 : };
175 :
176 : enum aicpuNotifySqeType { NOTIFY_RECORD = 0, NOTIFY_WAIT = 1 };
177 :
178 : enum rtStarsWriteValueSubType {
179 : RT_STARS_WRITE_VALUE_SUB_TYPE_DEFAULT = 0,
180 : RT_STARS_WRITE_VALUE_SUB_TYPE_EVENT_RESET = 1,
181 : RT_STARS_WRITE_VALUE_SUB_TYPE_RDMA_DB_SEND = 2,
182 : RT_STARS_WRITE_VALUE_SUB_TYPE_NOTIFY_RECORD_IPC_NO_PCIE = 3,
183 : RT_STARS_WRITE_VALUE_SUB_TYPE_NOTIFY_RECORD_IPC_PCIE = 4,
184 : RT_STARS_WRITE_VALUE_SUB_TYPE_MAX,
185 : };
186 :
187 : enum rtStarsSqeType {
188 : RT_STARS_SQE_TYPE_FFTS = 0, // FFTS
189 : RT_STARS_SQE_TYPE_AICPU = 1, // AICPU
190 : RT_STARS_SQE_TYPE_PLACE_HOLDER = 3, // PLACE_HOLDER
191 : RT_STARS_SQE_TYPE_EVENT_RECORD = 4, // EVENT_RECORD
192 : RT_STARS_SQE_TYPE_EVENT_WAIT = 5, // EVENT_WAIT
193 : RT_STARS_SQE_TYPE_NOTIFY_RECORD = 6, // NOTIFY_RECORD
194 : RT_STARS_SQE_TYPE_NOTIFY_WAIT = 7, // NOTIFY_WAIT
195 : RT_STARS_SQE_TYPE_WRITE_VALUE = 8, // for EVENT_RESET task
196 : RT_STARS_SQE_TYPE_SDMA = 11, // SDMA
197 : RT_STARS_SQE_TYPE_VPC = 12, // VPC
198 : RT_STARS_SQE_TYPE_JPEGE = 13, // JPEGE
199 : RT_STARS_SQE_TYPE_JPEGD = 14, // JPEGD
200 : RT_STARS_SQE_TYPE_DSA = 15,
201 : RT_STARS_SQE_TYPE_ROCCE = 16, // RoCCE
202 : RT_STARS_SQE_TYPE_PCIE_DMA = 17, // PCIE_DMA
203 : RT_STARS_SQE_TYPE_RESV = 18, // reserve
204 : RT_STARS_SQE_TYPE_CDQM = 19, // CDQM
205 : RT_STARS_SQE_TYPE_COND = 20, // condition
206 : RT_STARS_SQE_TYPE_END = 21,
207 : RT_STARS_SQE_TYPE_INVALID = 63, // STARS_SQE_TYPE_INVALID
208 : RT_STARS_SQE_TYPE_VIR_TYPE = 0xFF // DVPP virtual SQE TYPE
209 : };
210 :
211 : /* stars send interrupt direction */
212 : enum rtStarsSqeIntDirType {
213 : RT_STARS_SQE_INT_DIR_NO = 0, // send no interrupt
214 : RT_STARS_SQE_INT_DIR_TO_TSCPU = 1, // to tscpu
215 : RT_STARS_SQE_INT_DIR_TO_CTRLCPU = 2, // to ctrlcpu
216 : RT_STARS_SQE_INT_DIR_TO_HOST = 3, // to host
217 : RT_STARS_SQE_INT_DIR_END = 4
218 : };
219 :
220 : enum rtStarsMemcpyAsyncOperationKind {
221 : RT_STARS_MEMCPY_ASYNC_OP_KIND_CPY = 0x00,
222 : RT_STARS_MEMCPY_ASYNC_OP_KIND_ADD = 0x01,
223 : RT_STARS_MEMCPY_ASYNC_OP_KIND_MAX = 0x02,
224 : RT_STARS_MEMCPY_ASYNC_OP_KIND_MIN = 0x03,
225 : RT_STARS_MEMCPY_ASYNC_OP_KIND_EQUAL = 0x04
226 : };
227 :
228 : enum rtStarsMemcpyAsyncDataType {
229 : RT_STARS_MEMCPY_ASYNC_DATA_TYPE_INT8 = 0x00,
230 : RT_STARS_MEMCPY_ASYNC_DATA_TYPE_INT16 = 0x10,
231 : RT_STARS_MEMCPY_ASYNC_DATA_TYPE_INT32 = 0x20,
232 : RT_STARS_MEMCPY_ASYNC_DATA_TYPE_UINT8 = 0x30,
233 : RT_STARS_MEMCPY_ASYNC_DATA_TYPE_UINT16 = 0x40,
234 : RT_STARS_MEMCPY_ASYNC_DATA_TYPE_UINT32 = 0x50,
235 : RT_STARS_MEMCPY_ASYNC_DATA_TYPE_FP16 = 0x60,
236 : RT_STARS_MEMCPY_ASYNC_DATA_TYPE_FP32 = 0x70,
237 : RT_STARS_MEMCPY_ASYNC_DATA_TYPE_BFP16 = 0x80,
238 : RT_STARS_MEMCPY_ASYNC_OP_RESERVED = 0xf0
239 : };
240 :
241 : enum rtStarsWriteValueSizeType {
242 : RT_STARS_WRITE_VALUE_SIZE_TYPE_8BIT = 0,
243 : RT_STARS_WRITE_VALUE_SIZE_TYPE_16BIT = 1,
244 : RT_STARS_WRITE_VALUE_SIZE_TYPE_32BIT = 2,
245 : RT_STARS_WRITE_VALUE_SIZE_TYPE_64BIT = 3,
246 : RT_STARS_WRITE_VALUE_SIZE_TYPE_128BIT = 4,
247 : RT_STARS_WRITE_VALUE_SIZE_TYPE_256BIT = 5
248 : };
249 :
250 : enum rtStarsCondIsaRegister_t {
251 : RT_STARS_COND_ISA_REGISTER_R0 = 0, // R0 is always zero, can't be destination register
252 : RT_STARS_COND_ISA_REGISTER_R1 = 1,
253 : RT_STARS_COND_ISA_REGISTER_R2 = 2,
254 : RT_STARS_COND_ISA_REGISTER_R3 = 3,
255 : RT_STARS_COND_ISA_REGISTER_R4 = 4,
256 : RT_STARS_COND_ISA_REGISTER_R5 = 5
257 : };
258 :
259 : // enum for isa op LOAD func3
260 : enum rtStarsCondIsaLoadFunc3_t { RT_STARS_COND_ISA_LOAD_FUNC3_LDR = 0B011 };
261 :
262 : // enum for isa op LWI func3
263 : enum rtStarsCondIsaLwiFunc3_t { RT_STARS_COND_ISA_LWI_FUNC3_LHWI = 0B000, RT_STARS_COND_ISA_LWI_FUNC3_LLWI = 0B001 };
264 :
265 : // enum for isa op Branch func3
266 : enum rtStarsCondIsaBranchFunc3_t {
267 : RT_STARS_COND_ISA_BRANCH_FUNC3_BEQ = 0B000,
268 : RT_STARS_COND_ISA_BRANCH_FUNC3_BNE = 0B001,
269 : RT_STARS_COND_ISA_BRANCH_FUNC3_BLT = 0B100,
270 : RT_STARS_COND_ISA_BRANCH_FUNC3_BGE = 0B101,
271 : RT_STARS_COND_ISA_BRANCH_FUNC3_BLTU = 0B110,
272 : RT_STARS_COND_ISA_BRANCH_FUNC3_BGEU = 0B111
273 : };
274 :
275 : enum rtStarsCondIsaOpCode_t {
276 : RT_STARS_COND_ISA_OP_CODE_OP_IMM = 0B0010011, // Integer Register-immd Instructions
277 : RT_STARS_COND_ISA_OP_CODE_NOP = RT_STARS_COND_ISA_OP_CODE_OP_IMM, // NOP is using OP_IMM ADDI R0,R0,0
278 : RT_STARS_COND_ISA_OP_CODE_OP = 0B0110011, // Integer Register-Register Operations
279 : RT_STARS_COND_ISA_OP_CODE_LWI = 0B1011011, // load immd
280 : RT_STARS_COND_ISA_OP_CODE_BRANCH = 0B1100011, // Conditional stream-jump
281 : RT_STARS_COND_ISA_OP_CODE_LOOP = 0B1111011, // LOOP
282 : RT_STARS_COND_ISA_OP_CODE_STREAM = 0B0101011, // STREAM
283 : RT_STARS_COND_ISA_OP_CODE_LOAD_IMM = 0B0000111, // LOAD immd
284 : RT_STARS_COND_ISA_OP_CODE_LOAD = 0B0000011, // Load
285 : RT_STARS_COND_ISA_OP_CODE_STORE = 0B0100111, // Store
286 : RT_STARS_COND_ISA_OP_CODE_FUNC_CALL = 0B1101011, // FUNC_CALL
287 : RT_STARS_COND_ISA_OP_CODE_SYSTEM = 0B1110011 // CSR
288 : };
289 :
290 : // enum for isa op Op Imm func3
291 : enum rtStarsCondIsaOpImmFunc3_t {
292 : RT_STARS_COND_ISA_OP_IMM_FUNC3_ADDI = 0B000,
293 : RT_STARS_COND_ISA_OP_IMM_FUNC3_NOP = RT_STARS_COND_ISA_OP_IMM_FUNC3_ADDI, // NOP is using OP_IMM ADDI R0,R0,0
294 : RT_STARS_COND_ISA_OP_IMM_FUNC3_SLLI = 0B001,
295 : RT_STARS_COND_ISA_OP_IMM_FUNC3_SLTI = 0B010,
296 : RT_STARS_COND_ISA_OP_IMM_FUNC3_SLTIU = 0B011,
297 : RT_STARS_COND_ISA_OP_IMM_FUNC3_XORI = 0B100,
298 : RT_STARS_COND_ISA_OP_IMM_FUNC3_SRLI = 0B101,
299 : RT_STARS_COND_ISA_OP_IMM_FUNC3_ORI = 0B110,
300 : RT_STARS_COND_ISA_OP_IMM_FUNC3_ANDI = 0B111,
301 : RT_STARS_COND_ISA_OP_IMM_FUNC3_SRAI = 0B101 // diff with SRLI by func7
302 : };
303 :
304 : // enum for isa op store func3
305 : enum rtStarsCondIsaStoreFunc3_t {
306 : RT_STARS_COND_ISA_STORE_FUNC3_SB = 0B000,
307 : RT_STARS_COND_ISA_STORE_FUNC3_SH = 0B001,
308 : RT_STARS_COND_ISA_STORE_FUNC3_SW = 0B010,
309 : RT_STARS_COND_ISA_STORE_FUNC3_SD = 0B011,
310 : };
311 :
312 : #define TOPOLOGY_HCCS 0
313 : #define TOPOLOGY_PIX 1
314 : #define TOPOLOGY_PIB 2
315 : #define TOPOLOGY_PHB 3
316 : #define TOPOLOGY_SYS 4
317 :
318 : #pragma pack(push)
319 : #pragma pack(1)
320 :
321 : struct rtStarsWriteValueSqe_t {
322 : rtStarsSqeHeader_t header;
323 :
324 : uint32_t res3;
325 :
326 : uint32_t res4 : 16;
327 : uint32_t kernel_credit : 8;
328 : uint32_t res5 : 8;
329 :
330 : uint32_t write_addr_low;
331 :
332 : uint32_t write_addr_high : 17;
333 : uint32_t res6 : 3;
334 : uint32_t awsize : 3;
335 : uint32_t snoop : 1;
336 : uint32_t awcache : 4;
337 : uint32_t awprot : 3;
338 : uint32_t va : 1; // 1 /* 1: virtual address; 0: phy addr */
339 :
340 : uint32_t res7; // eventId for event reset task
341 : uint32_t sub_type;
342 :
343 : uint32_t write_value_part0;
344 : uint32_t write_value_part1;
345 : uint32_t rdmaWrLenth; // rdmaWr len
346 : uint32_t rdmaType; // notify / payload
347 : uint32_t write_value_part4;
348 : uint32_t write_value_part5;
349 : uint32_t write_value_part6;
350 : uint32_t write_value_part7;
351 : };
352 :
353 : struct rtStarsEventSqe_t {
354 : rtStarsSqeHeader_t header;
355 : uint16_t eventId;
356 : uint16_t res2;
357 :
358 : uint16_t res3;
359 : uint8_t kernel_credit;
360 : uint8_t res4;
361 : uint32_t exe_result;
362 : uint32_t timeout;
363 : uint32_t res5[10];
364 : };
365 :
366 : struct rtStarsMemcpyAsyncSqe_t {
367 : rtStarsSqeHeader_t header;
368 :
369 : uint32_t res3;
370 : /* *******12 bytes********* */
371 :
372 : uint16_t res4;
373 : uint8_t kernel_credit;
374 : uint8_t ptr_mode : 1;
375 : uint8_t res5 : 7;
376 : /* *******16 bytes********* */
377 :
378 : uint32_t opcode : 8;
379 : uint32_t ie2 : 1;
380 : uint32_t sssv : 1;
381 : uint32_t dssv : 1;
382 : uint32_t sns : 1;
383 : uint32_t dns : 1;
384 : uint32_t qos : 4;
385 : uint32_t sro : 1;
386 : uint32_t dro : 1;
387 : uint32_t partid : 8;
388 : uint32_t mpam : 1;
389 : uint32_t res6 : 4;
390 : /* *******20 bytes********* */
391 :
392 : uint16_t src_streamid;
393 : uint16_t src_sub_streamid;
394 : uint16_t dst_streamid;
395 : uint16_t dst_sub_streamid;
396 : /* *******28 bytes********* */
397 :
398 : uint32_t length;
399 : uint32_t src_addr_low;
400 : uint32_t src_addr_high;
401 : uint32_t dst_addr_low;
402 : uint32_t dst_addr_high;
403 :
404 : uint8_t linkType;
405 : uint8_t resvered[3];
406 : uint32_t reslast[3];
407 : };
408 :
409 : struct rtFlipTaskTag_t {
410 : uint16_t flipNumReport;
411 : uint8_t reserved[46];
412 : };
413 :
414 : // 用于alltoallv算子小数据量下展开时cache中的cache-memcpy placeholder (for zero-len memcpy SQE)
415 : struct rtCacheMemcpyTaskTag_t {
416 : // 注意: 不需要维护length, 因为每次cache hit时, length会根据send/recv info动态计算得到
417 :
418 : // 第一次展开时保留src/dst addr, cache miss后处理时, 用于UpdateRefreshAddrInfoForAlltoallv
419 : // 后续cache hit时只需要保留hccl buffer内的src/dst addr
420 : // 如果是user memory内的src/dst addr, 则根据send/recv info动态计算得到, 不需要在placeholder中维护
421 : uint32_t src_addr_low;
422 : uint32_t src_addr_high;
423 : uint32_t dst_addr_low;
424 : uint32_t dst_addr_high;
425 :
426 : // 始终需要维护
427 : // LocalCopy/PrepareIntraData: linkType使用默认参数LINK_ONCHIP
428 : // RemoteCopy: linkType使用srcRank-localRank之间的LINK对应的linkType
429 : uint8_t kernel_credit;
430 : uint8_t linkType;
431 : uint32_t qos;
432 :
433 : uint8_t reserved[26];
434 : };
435 :
436 : struct rtCacheNotifyTaskTag_t {
437 : uint8_t is_wait; // NotifyWait or NotifyRecord
438 : // Only for NotifyWait
439 : uint8_t kernel_credit;
440 : uint32_t timeout;
441 : // Shared by NotifyWait and NotifyRecord
442 : uint32_t notify_id;
443 : uint8_t reserved[38];
444 : };
445 :
446 : struct rtCacheWriteValueTaskTag_t {
447 : // For WriteValueRecord
448 : uint32_t write_addr_low;
449 : uint32_t write_addr_high;
450 : uint8_t reserved[40];
451 : };
452 :
453 : struct rtCacheMemcpyRecordTaskTag_t {
454 : // 20 bytes
455 : uint32_t length;
456 : uint32_t src_addr_low;
457 : uint32_t src_addr_high;
458 : uint32_t dst_addr_low;
459 : uint32_t dst_addr_high;
460 :
461 : // 4 bytes
462 : uint32_t opcode : 8;
463 : uint32_t partid : 8;
464 : uint32_t res : 16;
465 :
466 : // 2 bytes
467 : uint8_t kernel_credit;
468 : uint8_t linkType;
469 :
470 : // 4 bytes
471 : uint32_t qos;
472 :
473 : uint8_t reserved[18];
474 : };
475 :
476 : struct rtStarsPlaceHolderSqe_t {
477 : rtStarsSqeHeader_t header;
478 :
479 : uint32_t res1;
480 : uint16_t res2;
481 : uint8_t kernel_credit;
482 : uint8_t res3;
483 :
484 : /* The struct in the union must be 48 bytes */
485 : union {
486 : rtFlipTaskTag_t flip_task_info;
487 : rtCacheMemcpyTaskTag_t cache_memcpy_task_info;
488 : rtCacheNotifyTaskTag_t cache_notify_task_info;
489 : rtCacheWriteValueTaskTag_t cache_write_value_task_info;
490 : rtCacheMemcpyRecordTaskTag_t cache_memcpy_record_task_info;
491 : uint32_t resv[12];
492 : } u;
493 : };
494 :
495 : struct rtStarsCondOpLHWI_t {
496 : uint32_t opCode : 7;
497 : uint32_t rd : 3;
498 : uint32_t reserved0 : 2; // reserved
499 : uint32_t func3 : 3;
500 : uint32_t reserved1 : 2; // reserved
501 : uint32_t immd : 15;
502 : };
503 :
504 : // LWI LLWI: RT_STARS_COND_ISA_OP_CODE_LWI
505 : struct rtStarsCondOpLLWI_t {
506 : uint32_t opCode : 7;
507 : uint32_t rd : 3;
508 : uint32_t reserved0 : 2; // reserved
509 : uint32_t func3 : 3;
510 : uint32_t immdHigh : 17;
511 : uint32_t immdLow : 32;
512 : };
513 :
514 : // Branch: RT_STARS_COND_ISA_OP_CODE_BRANCH
515 : struct rtStarsCondOpBranch_t {
516 : uint32_t opCode : 7;
517 : uint32_t jumpInstrOffset : 4;
518 : uint32_t rsvd : 1;
519 : uint32_t func3 : 3;
520 : uint32_t rs1 : 3;
521 : uint32_t rsvd1 : 2;
522 : uint32_t rs2 : 3;
523 : uint32_t rsvd2 : 2;
524 : uint32_t rsvd3 : 7;
525 : };
526 :
527 : struct rtStarsCondOpLoad_t {
528 : uint32_t opCode : 7;
529 : uint32_t rd : 3;
530 : uint32_t reserved0 : 2;
531 : uint32_t func3 : 3;
532 : uint32_t rs1 : 3;
533 : uint32_t reserved1 : 2;
534 : uint32_t immd : 12;
535 : };
536 :
537 : // enum for isa op load imm func3
538 : enum rtStarsCondIsaLoadImmFunc3_t {
539 : RT_STARS_COND_ISA_LOAD_IMM_FUNC3_LB = 0B000,
540 : RT_STARS_COND_ISA_LOAD_IMM_FUNC3_LH = 0B001,
541 : RT_STARS_COND_ISA_LOAD_IMM_FUNC3_LW = 0B010,
542 : RT_STARS_COND_ISA_LOAD_IMM_FUNC3_LD = 0B011,
543 : RT_STARS_COND_ISA_LOAD_IMM_FUNC3_LBU = 0B100,
544 : RT_STARS_COND_ISA_LOAD_IMM_FUNC3_LHU = 0B101,
545 : RT_STARS_COND_ISA_LOAD_IMM_FUNC3_LWU = 0B110
546 : };
547 :
548 : struct rtStarsCondOpLoadImm_t {
549 : uint32_t opCode : 7;
550 : uint32_t rd : 3;
551 : uint32_t reserved : 2;
552 : uint32_t func3 : 3;
553 : uint32_t immdAddrHigh : 17;
554 : uint32_t immdAddrLow;
555 : };
556 :
557 : struct rtStarsCondOpImm_t {
558 : uint32_t opCode : 7;
559 : uint32_t rd : 3;
560 : uint32_t reserved0 : 2; // reserved
561 : uint32_t func3 : 3;
562 : uint32_t rs1 : 3;
563 : uint32_t reserved1 : 2; // reserved
564 : uint32_t immd : 12;
565 : };
566 :
567 : // store register data to ddr, RT_STARS_COND_ISA_OP_CODE_STORE
568 : struct rtStarsCondOpStore_t {
569 : uint32_t opCode : 7;
570 : uint32_t immdLow : 5;
571 : uint32_t func3 : 3;
572 : uint32_t rs1 : 3;
573 : uint32_t reserved1 : 2;
574 : uint32_t rs2 : 3;
575 : uint32_t reserved2 : 2;
576 : uint32_t immdHigh : 7;
577 : };
578 :
579 : // nop is using op-imm ADDI
580 : using rtStarsCondOpNop_t = rtStarsCondOpImm_t;
581 : struct rtStarsCondOpClear_t {
582 : rtStarsCondOpLLWI_t llwi1;
583 : rtStarsCondOpLHWI_t lhwi1; // load wait address as the immediate to R1
584 : rtStarsCondOpStore_t sw; // the last turn clear write_value
585 : rtStarsCondOpNop_t nop[3];
586 : };
587 : struct rtStarsCcoreWaitStartSqe_t {
588 : rtStarsSqeHeader_t sqeHeader;
589 :
590 : uint32_t reserved0;
591 : uint16_t reserved1;
592 : uint8_t kernel_credit;
593 : uint8_t reserved2 : 7;
594 : uint8_t csc : 1;
595 :
596 : rtStarsCondOpLoadImm_t ldrImm1; // load current turn as the immediate to R3
597 : rtStarsCondOpLoadImm_t ldrImm2; // load wait value, to R2
598 : rtStarsCondOpBranch_t beq; // if waitvalue == 0, goto read R2
599 : union {
600 : rtStarsCondOpClear_t clear;
601 : rtStarsCondOpNop_t nop[7];
602 : };
603 : };
604 :
605 : struct rtStarsCcoreWriteValueSqe_t {
606 : rtStarsSqeHeader_t sqeHeader;
607 :
608 : uint32_t reserved0;
609 : uint16_t reserved1;
610 : uint8_t kernel_credit;
611 : uint8_t reserved2 : 7;
612 : uint8_t csc : 1;
613 :
614 : rtStarsCondOpLoadImm_t ldrImm;
615 : rtStarsCondOpLLWI_t llwi1;
616 : rtStarsCondOpLHWI_t lhwi1;
617 : rtStarsCondOpStore_t sw;
618 : rtStarsCondOpNop_t nop[6];
619 : };
620 : struct rtLogicCqReport_t {
621 : volatile uint16_t streamId;
622 : volatile uint16_t taskId;
623 : volatile uint32_t errorCode; // cqe acc_status/sq_sw_status
624 : volatile uint8_t errorType; // bit0 ~ bit5 cqe stars_defined_err_code, bit 6 cqe warning bit
625 : volatile uint8_t sqeType;
626 : volatile uint16_t sqId;
627 : volatile uint16_t sqHead;
628 : volatile uint16_t matchFlag : 1;
629 : volatile uint16_t dropFlag : 1;
630 : volatile uint16_t errorBit : 1;
631 : volatile uint16_t accError : 1;
632 : volatile uint16_t reserved0 : 12;
633 : union {
634 : volatile uint64_t timeStamp;
635 : volatile uint16_t sqeIndex;
636 : } u1;
637 : /* Union description:
638 : * Internal: enque_timestamp temporarily used as dfx
639 : * External: reserved1
640 : */
641 : union {
642 : volatile uint64_t enqueTimeStamp;
643 : volatile uint64_t reserved1;
644 : } u2;
645 : };
646 :
647 : const std::vector<std::string> StarsCqeErrorDesc
648 : = {"task exception", "task trap", "task timeout", "sqe error",
649 : "resource conflict error", "sq sw status error", "warning"};
650 : #pragma pack(pop)
651 :
652 : inline void
653 : AddOneWriteValueRecordSqe(uint16_t streamId, uint16_t taskId, u64 notifyWRAddr, rtStarsWriteValueSqe_t* const sqe)
654 : {
655 : sqe->header.type = RT_STARS_SQE_TYPE_WRITE_VALUE;
656 : sqe->header.rtStreamId = streamId;
657 : sqe->header.taskId = taskId;
658 : sqe->kernel_credit = RT_STARS_DEFAULT_KERNEL_CREDIT;
659 : sqe->awsize = RT_STARS_WRITE_VALUE_SIZE_TYPE_32BIT;
660 : sqe->write_value_part0 = 1U;
661 : sqe->sub_type = RT_STARS_WRITE_VALUE_SUB_TYPE_NOTIFY_RECORD_IPC_NO_PCIE;
662 : sqe->write_addr_low = static_cast<uint32_t>(notifyWRAddr & MASK_32_BIT);
663 : sqe->write_addr_high = static_cast<uint32_t>((notifyWRAddr >> UINT32_BIT_NUM) & MASK_17_BIT);
664 : HCCL_DEBUG("[SQE] write value: writePtr=0x%lx, streamId=%u, taskId=%u.", notifyWRAddr, streamId, taskId);
665 : }
666 :
667 : inline void AddOneEventRecordSqe(uint16_t streamId, int32_t eventId, uint16_t taskId, rtStarsEventSqe_t* const ev_sqe)
668 : {
669 : ev_sqe->header.type = RT_STARS_SQE_TYPE_EVENT_RECORD;
670 : ev_sqe->header.wrCqe = 1U; // 1: set wrCqe
671 : ev_sqe->kernel_credit = RT_STARS_DEFAULT_KERNEL_CREDIT;
672 :
673 : // eventRecordTaskInfo->waitCqflag 是否为同步task
674 : if (false) {
675 : streamId |= RT_SYNC_TASK_FLAG;
676 : }
677 :
678 : ev_sqe->header.rtStreamId = streamId;
679 : ev_sqe->eventId = static_cast<uint16_t>(eventId);
680 : ev_sqe->header.taskId = taskId;
681 : HCCL_DEBUG("[SQE] event record: eventId=%d, streamId=%u, taskId=%u.", eventId, streamId, taskId);
682 : }
683 :
684 : inline void AddOneEventWaitSqe(uint16_t streamId, int32_t eventId, uint16_t taskId, rtStarsEventSqe_t* const ev_sqe)
685 : {
686 : ev_sqe->header.type = RT_STARS_SQE_TYPE_EVENT_WAIT;
687 : ev_sqe->kernel_credit = RT_STARS_NEVER_TIMEOUT_KERNEL_CREDIT;
688 :
689 : ev_sqe->header.rtStreamId = streamId;
690 : ev_sqe->eventId = eventId;
691 : ev_sqe->header.taskId = taskId;
692 : HCCL_DEBUG("[SQE] event wait: eventId=%d, streamId=%u, taskId=%u", eventId, streamId, taskId);
693 : }
694 :
695 : extern void AddOneWaitStartSqe(
696 : uint16_t streamId, uint16_t taskId, u64 waitAddr, u64 curTurnCntAddr, bool last,
697 : rtStarsCcoreWaitStartSqe_t* const sqe, uint8_t* sqeType);
698 : extern void AddOneWriteValueStartSqe(
699 : uint16_t streamId, uint16_t taskId, u64 writeAddr, u64 valueAddr, rtStarsCcoreWriteValueSqe_t* const sqe,
700 : uint8_t* sqeType);
701 :
702 : extern HcclResult QuerySqStatus(uint32_t devId, uint32_t sqId, uint32_t& sqHead, uint32_t& sqTail);
703 : extern HcclResult QuerySqStatusByType(uint32_t devId, uint32_t sqId, drvSqCqPropType_t type, uint32_t& outVal);
704 : extern HcclResult ConfigSqStatusByType(uint32_t devId, uint32_t sqId, drvSqCqPropType_t type, uint32_t value);
705 : extern HcclResult QuerySqBaseAddr(uint32_t devId, uint32_t sqId, u64& outVal);
706 : using CqeStatus = dfx::CqeStatus;
707 : using CqeQueryInput = dfx::CqeQueryInput;
708 : extern CqeStatus CqReportRecv(const CqeQueryInput& cqeQueryInput, rtLogicCqReport_t& cqeException);
709 : extern void PrintTaskException(const rtLogicCqReport_t& reportOfOne);
710 : extern HcclResult StreamsKill(const uint32_t devId);
711 : extern HcclResult DeviceQuery(const uint32_t devId, const uint32_t step, const uint32_t timeout);
712 : namespace hccl_plf {
713 : HcclResult SendTaskExceptionByMBox(
714 : const u32 localDeviceId, const u32 notifyId, const u32 tsId, const s32 userStreamId, const u32 cqeErrCode);
715 : }
716 : #endif // __AICPU_HCCL_SQCQ_H__
|
