Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "aicpu/aicpu_hccl_sqcqv2.h"
12 : #include "hccl_common.h"
13 : #include "dispatcher_task_types.h"
14 : namespace {
15 : enum class SdmaReduceOpcode {
16 : SDMA_OPCODE_NOT_ATOMIC = 0,
17 : SDMA_OPCODE_INT_16 = 1,
18 : SDMA_OPCODE_FLOAT_32 = 2,
19 : SDMA_OPCODE_FLOAT_16 = 3
20 : };
21 :
22 2 : uint8_t ReduceOpcode(uint8_t copyDataType)
23 : {
24 : uint8_t opcode;
25 2 : switch (copyDataType) {
26 0 : case ACL_INT16: {
27 0 : opcode = static_cast<uint8_t>(SdmaReduceOpcode::SDMA_OPCODE_INT_16);
28 0 : break;
29 : }
30 2 : case ACL_FLOAT16: {
31 2 : opcode = static_cast<uint8_t>(SdmaReduceOpcode::SDMA_OPCODE_FLOAT_16);
32 2 : break;
33 : }
34 0 : case ACL_FLOAT: {
35 0 : opcode = static_cast<uint8_t>(SdmaReduceOpcode::SDMA_OPCODE_FLOAT_32);
36 0 : break;
37 : }
38 0 : default: {
39 0 : opcode = static_cast<uint8_t>(SdmaReduceOpcode::SDMA_OPCODE_FLOAT_32);
40 0 : break;
41 : }
42 : }
43 2 : return opcode;
44 : }
45 : } // namespace
46 :
47 1 : void AddOneNotifyWaitSqeV2(
48 : uint16_t streamId, uint16_t taskId, u64 notifyId, const uint8_t* sqeIn, uint8_t* sqeType,
49 : const dfx::DfxTimeOutConfig& dfxTimeOutConfig)
50 : {
51 : (void)dfxTimeOutConfig;
52 1 : *sqeType = SqeType::NOTIFY_SQE_V2;
53 1 : rtStarsNotifySqeV2_t* const sqe = (rtStarsNotifySqeV2_t* const)sqeIn;
54 1 : sqe->header.type = RT_HW_STARS_SQE_TYPE_NOTIFY_WAIT;
55 1 : sqe->kernel_credit = RT_STARS_NEVER_TIMEOUT_KERNEL_CREDIT;
56 1 : sqe->header.rt_stream_id = streamId;
57 1 : sqe->notify_id = notifyId;
58 1 : sqe->header.task_id = taskId;
59 1 : HCCL_INFO("[SQE] notify wait: notifyId=%lu, streamId=%u, taskId=%u.", notifyId, streamId, taskId);
60 1 : }
61 :
62 1 : void AddOneRecordSqeV2(uint16_t streamId, uint16_t taskId, u64 notifyId, const uint8_t* sqeIn, uint8_t* sqeType)
63 : {
64 1 : *sqeType = SqeType::NOTIFY_SQE_V2;
65 1 : rtStarsNotifySqeV2_t* const sqe = (rtStarsNotifySqeV2_t* const)sqeIn;
66 1 : sqe->header.type = RT_HW_STARS_SQE_TYPE_NOTIFY_RECORD;
67 1 : sqe->kernel_credit = RT_STARS_DEFAULT_KERNEL_CREDIT;
68 1 : sqe->header.rt_stream_id = streamId;
69 1 : sqe->notify_id = notifyId;
70 1 : sqe->header.task_id = taskId;
71 1 : HCCL_INFO("[SQE] notify record: notifyId=%lu, streamId=%u, taskId=%u.", notifyId, streamId, taskId);
72 1 : }
73 :
74 1 : void AddOneWriteValueRecordSqeV2(
75 : uint16_t streamId, uint16_t taskId, u64 notifyWRAddr, const uint8_t* sqeIn, uint8_t* sqeType)
76 : {
77 1 : *sqeType = SqeType::WRITE_VALUE_SQE_V2;
78 1 : rtStarsWriteValueSqeV2_t* const sqe = (rtStarsWriteValueSqeV2_t* const)sqeIn;
79 1 : sqe->header.type = RT_HW_STARS_SQE_TYPE_WRITE_VALUE;
80 1 : sqe->header.rt_stream_id = streamId;
81 1 : sqe->header.task_id = taskId;
82 :
83 1 : sqe->awsize = RT_STARS_WRITE_VALUE_SIZE_TYPE_64BIT;
84 1 : sqe->awprot = 2; /* 2: b:010 unprivileged access non-secure access instruction access */
85 :
86 1 : sqe->write_val[0] = 1U;
87 1 : sqe->reg_addr_low = static_cast<uint32_t>(notifyWRAddr & MASK_32_BIT);
88 1 : sqe->reg_addr_high = static_cast<uint16_t>(notifyWRAddr >> UINT32_BIT_NUM);
89 1 : HCCL_INFO("[SQE] write value: writePtr=0x%lx, streamId=%u, task_id=%u.", notifyWRAddr, streamId, taskId);
90 1 : }
91 :
92 3 : void AddOneMemcpySqeV2(
93 : uint16_t streamId, uint16_t taskId, const void* src, uint32_t length, const aclDataType runtimeDataType,
94 : aclrtReduceKind rtReduceOp, const void* dst, uint32_t partId, uint32_t ssid, uint32_t devId, u64 overflowAddr,
95 : uint8_t linkType, const uint8_t* sqeIn, uint8_t* sqeType, uint32_t hcclQos)
96 : {
97 : (void)partId;
98 : (void)linkType;
99 3 : *sqeType = SqeType::MEMCPY_ASYNC_SQE_V2;
100 3 : rtStarsMemcpyAsyncSqeV2_t* const sqe = (rtStarsMemcpyAsyncSqeV2_t* const)sqeIn;
101 : uint16_t smmuStreamId;
102 3 : if (devId == 0) {
103 2 : smmuStreamId = 0x7F45;
104 : } else {
105 1 : smmuStreamId = 0xBF45;
106 : }
107 :
108 3 : sqe->type = RT_HW_STARS_SQE_TYPE_SDMA;
109 :
110 3 : sqe->rt_stream_id = streamId;
111 3 : sqe->task_id = taskId;
112 :
113 3 : sqe->sro = 1U;
114 3 : sqe->dro = 1U;
115 3 : sqe->sns = 1U;
116 3 : sqe->dns = 1U;
117 3 : sqe->sssv = 1U;
118 3 : sqe->dssv = 1U;
119 :
120 3 : sqe->src_streamid = static_cast<uint16_t>(smmuStreamId);
121 3 : sqe->dst_streamid = static_cast<uint16_t>(smmuStreamId);
122 :
123 3 : sqe->src_substreamid = ssid;
124 3 : sqe->dst_substreamid = ssid;
125 :
126 3 : sqe->length = length;
127 3 : sqe->kernel_credit = RT_STARS_NEVER_TIMEOUT_KERNEL_CREDIT;
128 3 : sqe->src_addr_low = static_cast<uint32_t>(reinterpret_cast<u64>(src) & 0x00000000ffffffffU);
129 3 : sqe->src_addr_high = static_cast<uint32_t>((reinterpret_cast<u64>(src) & 0xffffffff00000000U) >> UINT32_BIT_NUM);
130 3 : sqe->dst_addr_low = static_cast<uint32_t>(reinterpret_cast<u64>(dst) & 0x00000000ffffffffU);
131 3 : sqe->dst_addr_high = static_cast<uint32_t>((reinterpret_cast<u64>(dst) & 0xffffffff00000000U) >> UINT32_BIT_NUM);
132 3 : sqe->overflow_en = 1U;
133 3 : sqe->overflow_addr_low = static_cast<uint32_t>(reinterpret_cast<u64>(overflowAddr) & 0x00000000ffffffffU);
134 : sqe->overflow_addr_high
135 3 : = static_cast<uint32_t>((reinterpret_cast<u64>(overflowAddr) & 0xffffffff00000000U) >> UINT32_BIT_NUM);
136 3 : if (linkType == static_cast<uint8_t>(hccl::LinkType::LINK_SIO)
137 2 : || linkType == static_cast<uint8_t>(hccl::LinkType::LINK_ONCHIP)) {
138 3 : hcclQos = SDMA_QOS_DEFAULT;
139 : }
140 3 : sqe->qos = hcclQos;
141 3 : const bool isReduce = (rtReduceOp == ACL_RT_MEMCPY_SDMA_AUTOMATIC_SUM);
142 : sqe->opcode
143 3 : = isReduce ? ReduceOpcode(runtimeDataType) : static_cast<uint8_t>(SdmaReduceOpcode::SDMA_OPCODE_NOT_ATOMIC);
144 3 : HCCL_INFO(
145 : "[SQE]MemcpySqe copyKind=%u,Opcode=0x%x, streamId=%u, len=%u, src:%p, dst:%p src_substreamid:%u "
146 : "dst_substreamid:%u src_streamid:%x dst_streamid:%x overflowAddr:%llx sqe->linkType=%u hcclQos=%u",
147 : static_cast<uint32_t>(rtReduceOp), static_cast<uint32_t>(sqe->opcode), streamId, length, src, dst,
148 : sqe->src_substreamid, sqe->dst_substreamid, sqe->src_streamid, sqe->dst_streamid, overflowAddr,
149 : static_cast<unsigned int>(linkType), hcclQos);
150 3 : }
151 :
152 1 : void AddOneEventResetSqeV2(
153 : uint16_t streamId, int32_t eventId, uint16_t taskId, int64_t phyChipId, int64_t phyDieId, u64 addr,
154 : const uint8_t* sqeIn, uint8_t* sqeType)
155 : {
156 : (void)phyChipId;
157 : (void)phyDieId;
158 1 : *sqeType = SqeType::WRITE_VALUE_SQE_V2;
159 1 : rtStarsWriteValueSqeV2_t* const sqe = (rtStarsWriteValueSqeV2_t* const)sqeIn;
160 1 : sqe->header.type = RT_STARS_SQE_TYPE_WRITE_VALUE;
161 :
162 1 : sqe->header.rt_stream_id = streamId;
163 1 : sqe->header.task_id = taskId;
164 :
165 1 : sqe->awprot = 0x2; /* b:010 unprivileged access non-secure access instruction access */
166 1 : sqe->awsize = RT_STARS_WRITE_VALUE_SIZE_TYPE_64BIT; /* 64bit */
167 :
168 1 : sqe->reg_addr_low = static_cast<uint32_t>(addr & MASK_32_BIT);
169 1 : sqe->reg_addr_high = static_cast<uint32_t>((addr >> UINT32_BIT_NUM) & MASK_17_BIT);
170 1 : HCCL_INFO("[SQE] event_reset: eventId=%u, streamId=%u, taskId=%u, addr:%p", eventId, streamId, taskId, addr);
171 1 : }
172 :
173 1 : void AddOneEventRecordSqeV2(uint16_t streamId, int32_t eventId, uint16_t taskId, const uint8_t* sqeIn, uint8_t* sqeType)
174 : {
175 1 : *sqeType = SqeType::EVENT_SQE_V2;
176 1 : rtStarsEventSqeV2_t* const sqe = (rtStarsEventSqeV2_t* const)sqeIn;
177 1 : sqe->type = RT_STARS_SQE_TYPE_EVENT_RECORD;
178 :
179 1 : sqe->kernel_credit = RT_STARS_DEFAULT_KERNEL_CREDIT;
180 :
181 1 : sqe->rt_stream_id = streamId;
182 1 : sqe->event_id = static_cast<uint16_t>(eventId);
183 1 : sqe->task_id = taskId;
184 :
185 1 : HCCL_INFO("[SQE] event record: eventId=%d, streamId=%u, taskId=%u.", eventId, streamId, taskId);
186 1 : }
187 :
188 1 : void AddOneEventWaitSqeV2(uint16_t streamId, int32_t eventId, uint16_t taskId, const uint8_t* sqeIn, uint8_t* sqeType)
189 : {
190 1 : *sqeType = SqeType::EVENT_SQE_V2;
191 1 : rtStarsEventSqeV2_t* const sqe = (rtStarsEventSqeV2_t* const)sqeIn;
192 1 : sqe->type = RT_STARS_SQE_TYPE_EVENT_WAIT;
193 1 : sqe->kernel_credit = RT_STARS_NEVER_TIMEOUT_KERNEL_CREDIT;
194 :
195 1 : sqe->rt_stream_id = streamId;
196 1 : sqe->event_id = eventId;
197 1 : sqe->task_id = taskId;
198 1 : HCCL_INFO("[SQE] event wait: eventId=%d, streamId=%u, taskId=%u", eventId, streamId, taskId);
199 1 : }
|
