Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "aicpu/aicpu_hccl_sqcqv2.h"
12 : #include "hccl_common.h"
13 : #include "dispatcher_task_types.h"
14 : namespace {
15 : enum class SdmaReduceOpcode {
16 : SDMA_OPCODE_NOT_ATOMIC = 0,
17 : SDMA_OPCODE_INT_16 = 1,
18 : SDMA_OPCODE_FLOAT_32 = 2,
19 : SDMA_OPCODE_FLOAT_16 = 3
20 : };
21 :
22 2 : uint8_t ReduceOpcode(uint8_t copyDataType)
23 : {
24 : uint8_t opcode;
25 2 : switch (copyDataType) {
26 0 : case ACL_INT16: {
27 0 : opcode = static_cast<uint8_t>(SdmaReduceOpcode::SDMA_OPCODE_INT_16);
28 0 : break;
29 : }
30 2 : case ACL_FLOAT16: {
31 2 : opcode = static_cast<uint8_t>(SdmaReduceOpcode::SDMA_OPCODE_FLOAT_16);
32 2 : break;
33 : }
34 0 : case ACL_FLOAT: {
35 0 : opcode = static_cast<uint8_t>(SdmaReduceOpcode::SDMA_OPCODE_FLOAT_32);
36 0 : break;
37 : }
38 0 : default: {
39 0 : opcode = static_cast<uint8_t>(SdmaReduceOpcode::SDMA_OPCODE_FLOAT_32);
40 0 : break;
41 : }
42 : }
43 2 : return opcode;
44 : }
45 : } // namespace
46 :
47 1 : void AddOneNotifyWaitSqeV2(uint16_t streamId, uint16_t taskId, u64 notifyId, const uint8_t *sqeIn, uint8_t *sqeType,
48 : const dfx::DfxTimeOutConfig &dfxTimeOutConfig)
49 : {
50 : (void)dfxTimeOutConfig;
51 1 : *sqeType = SqeType::NOTIFY_SQE_V2;
52 1 : rtStarsNotifySqeV2_t * const sqe = (rtStarsNotifySqeV2_t * const)sqeIn;
53 1 : sqe->header.type = RT_HW_STARS_SQE_TYPE_NOTIFY_WAIT;
54 1 : sqe->kernel_credit = RT_STARS_NEVER_TIMEOUT_KERNEL_CREDIT;
55 1 : sqe->header.rt_stream_id = streamId;
56 1 : sqe->notify_id = notifyId;
57 1 : sqe->header.task_id = taskId;
58 1 : HCCL_INFO("[SQE] notify wait: notifyId=%lu, streamId=%u, taskId=%u.", notifyId, streamId, taskId);
59 1 : }
60 :
61 1 : void AddOneRecordSqeV2(uint16_t streamId, uint16_t taskId, u64 notifyId, const uint8_t *sqeIn, uint8_t *sqeType)
62 : {
63 1 : *sqeType = SqeType::NOTIFY_SQE_V2;
64 1 : rtStarsNotifySqeV2_t * const sqe = (rtStarsNotifySqeV2_t * const)sqeIn;
65 1 : sqe->header.type = RT_HW_STARS_SQE_TYPE_NOTIFY_RECORD;
66 1 : sqe->kernel_credit = RT_STARS_DEFAULT_KERNEL_CREDIT;
67 1 : sqe->header.rt_stream_id = streamId;
68 1 : sqe->notify_id = notifyId;
69 1 : sqe->header.task_id = taskId;
70 1 : HCCL_INFO("[SQE] notify record: notifyId=%lu, streamId=%u, taskId=%u.", notifyId, streamId, taskId);
71 1 : }
72 :
73 1 : void AddOneWriteValueRecordSqeV2(uint16_t streamId, uint16_t taskId, u64 notifyWRAddr, const uint8_t *sqeIn,
74 : uint8_t *sqeType)
75 : {
76 1 : *sqeType = SqeType::WRITE_VALUE_SQE_V2;
77 1 : rtStarsWriteValueSqeV2_t * const sqe = (rtStarsWriteValueSqeV2_t * const)sqeIn;
78 1 : sqe->header.type = RT_HW_STARS_SQE_TYPE_WRITE_VALUE;
79 1 : sqe->header.rt_stream_id = streamId;
80 1 : sqe->header.task_id = taskId;
81 :
82 1 : sqe->awsize = RT_STARS_WRITE_VALUE_SIZE_TYPE_64BIT;
83 1 : sqe->awprot = 2; /* 2: b:010 unprivileged access non-secure access instruction access */
84 :
85 1 : sqe->write_val[0] = 1U;
86 1 : sqe->reg_addr_low = static_cast<uint32_t>(notifyWRAddr & MASK_32_BIT);
87 1 : sqe->reg_addr_high = static_cast<uint16_t>(notifyWRAddr >> UINT32_BIT_NUM);
88 1 : HCCL_INFO("[SQE] write value: writePtr=0x%lx, streamId=%u, task_id=%u.", notifyWRAddr, streamId, taskId);
89 1 : }
90 :
91 3 : void AddOneMemcpySqeV2(uint16_t streamId, uint16_t taskId, const void *src, uint32_t length,
92 : const aclDataType runtimeDataType, aclrtReduceKind rtReduceOp, const void *dst, uint32_t partId, uint32_t ssid,
93 : uint32_t devId, u64 overflowAddr, uint8_t linkType, const uint8_t *sqeIn, uint8_t *sqeType, uint32_t hcclQos)
94 : {
95 : (void)partId;
96 : (void)linkType;
97 3 : *sqeType = SqeType::MEMCPY_ASYNC_SQE_V2;
98 3 : rtStarsMemcpyAsyncSqeV2_t * const sqe = (rtStarsMemcpyAsyncSqeV2_t * const)sqeIn;
99 : uint16_t smmuStreamId;
100 3 : if (devId == 0) {
101 2 : smmuStreamId = 0x7F45;
102 : } else {
103 1 : smmuStreamId = 0xBF45;
104 : }
105 :
106 3 : sqe->type = RT_HW_STARS_SQE_TYPE_SDMA;
107 :
108 3 : sqe->rt_stream_id = streamId;
109 3 : sqe->task_id = taskId;
110 :
111 3 : sqe->sro = 1U;
112 3 : sqe->dro = 1U;
113 3 : sqe->sns = 1U;
114 3 : sqe->dns = 1U;
115 3 : sqe->sssv = 1U;
116 3 : sqe->dssv = 1U;
117 :
118 3 : sqe->src_streamid = static_cast<uint16_t>(smmuStreamId);
119 3 : sqe->dst_streamid = static_cast<uint16_t>(smmuStreamId);
120 :
121 3 : sqe->src_substreamid = ssid;
122 3 : sqe->dst_substreamid = ssid;
123 :
124 3 : sqe->length = length;
125 3 : sqe->kernel_credit = RT_STARS_NEVER_TIMEOUT_KERNEL_CREDIT;
126 3 : sqe->src_addr_low = static_cast<uint32_t>(reinterpret_cast<u64>(src) & 0x00000000ffffffffU);
127 3 : sqe->src_addr_high =
128 3 : static_cast<uint32_t>((reinterpret_cast<u64>(src) & 0xffffffff00000000U) >> UINT32_BIT_NUM);
129 3 : sqe->dst_addr_low = static_cast<uint32_t>(reinterpret_cast<u64>(dst) & 0x00000000ffffffffU);
130 3 : sqe->dst_addr_high =
131 3 : static_cast<uint32_t>((reinterpret_cast<u64>(dst) & 0xffffffff00000000U) >> UINT32_BIT_NUM);
132 3 : sqe->overflow_en = 1U;
133 3 : sqe->overflow_addr_low = static_cast<uint32_t>(reinterpret_cast<u64>(overflowAddr) & 0x00000000ffffffffU);
134 3 : sqe->overflow_addr_high =
135 3 : static_cast<uint32_t>((reinterpret_cast<u64>(overflowAddr) & 0xffffffff00000000U) >> UINT32_BIT_NUM);
136 3 : if (linkType == static_cast<uint8_t>(hccl::LinkType::LINK_SIO) || linkType == static_cast<uint8_t>(hccl::LinkType::LINK_ONCHIP)) {
137 3 : hcclQos = SDMA_QOS_DEFAULT;
138 : }
139 3 : sqe->qos = hcclQos;
140 3 : const bool isReduce = (rtReduceOp == ACL_RT_MEMCPY_SDMA_AUTOMATIC_SUM);
141 3 : sqe->opcode = isReduce ? ReduceOpcode(runtimeDataType) : static_cast<uint8_t>(SdmaReduceOpcode::SDMA_OPCODE_NOT_ATOMIC);
142 3 : HCCL_INFO("[SQE]MemcpySqe copyKind=%u,Opcode=0x%x, streamId=%u, len=%u, src:%p, dst:%p src_substreamid:%u "
143 : "dst_substreamid:%u src_streamid:%x dst_streamid:%x overflowAddr:%llx sqe->linkType=%u hcclQos=%u",
144 : static_cast<uint32_t>(rtReduceOp), static_cast<uint32_t>(sqe->opcode), streamId, length, src, dst,
145 : sqe->src_substreamid, sqe->dst_substreamid, sqe->src_streamid, sqe->dst_streamid, overflowAddr, static_cast<unsigned int>(linkType), hcclQos);
146 3 : }
147 :
148 1 : void AddOneEventResetSqeV2(uint16_t streamId, int32_t eventId, uint16_t taskId, int64_t phyChipId, int64_t phyDieId,
149 : u64 addr, const uint8_t *sqeIn, uint8_t *sqeType)
150 : {
151 : (void)phyChipId;
152 : (void)phyDieId;
153 1 : *sqeType = SqeType::WRITE_VALUE_SQE_V2;
154 1 : rtStarsWriteValueSqeV2_t * const sqe = (rtStarsWriteValueSqeV2_t * const)sqeIn;
155 1 : sqe->header.type = RT_STARS_SQE_TYPE_WRITE_VALUE;
156 :
157 1 : sqe->header.rt_stream_id = streamId;
158 1 : sqe->header.task_id = taskId;
159 :
160 1 : sqe->awprot = 0x2; /* b:010 unprivileged access non-secure access instruction access */
161 1 : sqe->awsize = RT_STARS_WRITE_VALUE_SIZE_TYPE_64BIT; /* 64bit */
162 :
163 1 : sqe->reg_addr_low = static_cast<uint32_t>(addr & MASK_32_BIT);
164 1 : sqe->reg_addr_high = static_cast<uint32_t>((addr >> UINT32_BIT_NUM) & MASK_17_BIT);
165 1 : HCCL_INFO("[SQE] event_reset: eventId=%u, streamId=%u, taskId=%u, addr:%p", eventId, streamId, taskId, addr);
166 1 : }
167 :
168 1 : void AddOneEventRecordSqeV2(uint16_t streamId, int32_t eventId, uint16_t taskId, const uint8_t *sqeIn, uint8_t *sqeType)
169 : {
170 1 : *sqeType = SqeType::EVENT_SQE_V2;
171 1 : rtStarsEventSqeV2_t * const sqe = (rtStarsEventSqeV2_t * const)sqeIn;
172 1 : sqe->type = RT_STARS_SQE_TYPE_EVENT_RECORD;
173 :
174 1 : sqe->kernel_credit = RT_STARS_DEFAULT_KERNEL_CREDIT;
175 :
176 1 : sqe->rt_stream_id = streamId;
177 1 : sqe->event_id = static_cast<uint16_t>(eventId);
178 1 : sqe->task_id = taskId;
179 :
180 1 : HCCL_INFO("[SQE] event record: eventId=%d, streamId=%u, taskId=%u.", eventId, streamId, taskId);
181 1 : }
182 :
183 1 : void AddOneEventWaitSqeV2(uint16_t streamId, int32_t eventId, uint16_t taskId, const uint8_t *sqeIn, uint8_t *sqeType)
184 : {
185 1 : *sqeType = SqeType::EVENT_SQE_V2;
186 1 : rtStarsEventSqeV2_t * const sqe = (rtStarsEventSqeV2_t * const)sqeIn;
187 1 : sqe->type = RT_STARS_SQE_TYPE_EVENT_WAIT;
188 1 : sqe->kernel_credit = RT_STARS_NEVER_TIMEOUT_KERNEL_CREDIT;
189 :
190 1 : sqe->rt_stream_id = streamId;
191 1 : sqe->event_id = eventId;
192 1 : sqe->task_id = taskId;
193 1 : HCCL_INFO("[SQE] event wait: eventId=%d, streamId=%u, taskId=%u", eventId, streamId, taskId);
194 1 : }
|
