Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 : #include "aicpu/aicpu_hccl_sqcqv1.h"
11 : #include "hccl_common.h"
12 : #include "dispatcher_task_types.h"
13 : #include <unordered_map>
14 :
15 : namespace {
16 14 : bool ChipIsHaveStars()
17 : {
18 14 : return true;
19 : }
20 :
21 165 : uint8_t ReduceOpcodeHigh(uint8_t copyDataType)
22 : {
23 : uint8_t opcode;
24 165 : switch (copyDataType) {
25 1 : case ACL_INT8: {
26 1 : opcode = static_cast<uint8_t>(RT_STARS_MEMCPY_ASYNC_DATA_TYPE_INT8);
27 1 : break;
28 : }
29 0 : case ACL_INT16: {
30 0 : opcode = static_cast<uint8_t>(RT_STARS_MEMCPY_ASYNC_DATA_TYPE_INT16);
31 0 : break;
32 : }
33 0 : case ACL_INT32: {
34 0 : opcode = static_cast<uint8_t>(RT_STARS_MEMCPY_ASYNC_DATA_TYPE_INT32);
35 0 : break;
36 : }
37 148 : case ACL_FLOAT16: {
38 148 : opcode = static_cast<uint8_t>(RT_STARS_MEMCPY_ASYNC_DATA_TYPE_FP16);
39 148 : break;
40 : }
41 1 : case ACL_FLOAT: {
42 1 : opcode = static_cast<uint8_t>(RT_STARS_MEMCPY_ASYNC_DATA_TYPE_FP32);
43 1 : break;
44 : }
45 14 : case ACL_BF16: {
46 14 : if (ChipIsHaveStars()) {
47 14 : opcode = static_cast<uint8_t>(RT_STARS_MEMCPY_ASYNC_DATA_TYPE_BFP16);
48 : } else {
49 0 : HCCL_ERROR("DataType=%u do not support.", static_cast<uint32_t>(copyDataType));
50 0 : opcode = static_cast<uint8_t>(RT_STARS_MEMCPY_ASYNC_OP_RESERVED);
51 : }
52 14 : break;
53 : }
54 1 : default: {
55 : // Should not run here.
56 : // if not support, it will return RT_ERROR_FEATURE_NOT_SUPPORT at context.cc's reduce ability check.
57 : // Only for code style, 0x80 is reserved value of STRAS opcode.
58 1 : HCCL_ERROR("DataType=%u do not support.", static_cast<uint32_t>(copyDataType));
59 1 : opcode = static_cast<uint8_t>(RT_STARS_MEMCPY_ASYNC_OP_RESERVED);
60 1 : break;
61 : }
62 : }
63 165 : return opcode;
64 : }
65 :
66 165 : uint8_t ReduceOpcodeLow(uint32_t copyKind)
67 : {
68 : uint8_t opcode;
69 165 : switch (copyKind) {
70 165 : case ACL_RT_MEMCPY_SDMA_AUTOMATIC_SUM: {
71 165 : opcode = static_cast<uint8_t>(RT_STARS_MEMCPY_ASYNC_OP_KIND_ADD);
72 165 : break;
73 : }
74 0 : case ACL_RT_MEMCPY_SDMA_AUTOMATIC_MAX: {
75 0 : opcode = static_cast<uint8_t>(RT_STARS_MEMCPY_ASYNC_OP_KIND_MAX);
76 0 : break;
77 : }
78 0 : case ACL_RT_MEMCPY_SDMA_AUTOMATIC_MIN: {
79 0 : opcode = static_cast<uint8_t>(RT_STARS_MEMCPY_ASYNC_OP_KIND_MIN);
80 0 : break;
81 : }
82 0 : case ACL_RT_MEMCPY_SDMA_AUTOMATIC_EQUAL: {
83 0 : opcode = static_cast<uint8_t>(RT_STARS_MEMCPY_ASYNC_OP_KIND_EQUAL);
84 0 : break;
85 : }
86 0 : default: {
87 0 : HCCL_ERROR("Type out of range: copyKind=%u", copyKind);
88 0 : opcode = static_cast<uint8_t>(RT_STARS_MEMCPY_ASYNC_OP_RESERVED);
89 0 : break;
90 : }
91 : }
92 165 : return opcode;
93 : }
94 :
95 165 : uint8_t GetOpcodeForReduce(uint32_t copyKind, uint8_t copyDataType)
96 : {
97 165 : const uint8_t opcodeHigh = ReduceOpcodeHigh(copyDataType);
98 165 : const uint8_t opcodeLow = ReduceOpcodeLow(copyKind);
99 165 : if ((static_cast<int32_t>(opcodeHigh) == RT_STARS_MEMCPY_ASYNC_OP_RESERVED) ||
100 : (static_cast<int32_t>(opcodeLow) == RT_STARS_MEMCPY_ASYNC_OP_RESERVED)) {
101 : // Should not run here. 0x80 is reserved value of STRAS opcode
102 1 : return static_cast<uint8_t>(RT_STARS_MEMCPY_ASYNC_OP_RESERVED);
103 : } else {
104 164 : return opcodeHigh | opcodeLow;
105 : }
106 : }
107 :
108 : std::unordered_map<uint8_t, uint8_t> RT2HCCL_REDUCE_OP_MAP = {
109 : {static_cast<uint8_t>(RT_STARS_MEMCPY_ASYNC_OP_KIND_CPY), static_cast<uint8_t>(HCCL_REDUCE_RESERVED)},
110 : {static_cast<uint8_t>(RT_STARS_MEMCPY_ASYNC_OP_KIND_ADD), static_cast<uint8_t>(HCCL_REDUCE_SUM)},
111 : {static_cast<uint8_t>(RT_STARS_MEMCPY_ASYNC_OP_KIND_MAX), static_cast<uint8_t>(HCCL_REDUCE_MAX)},
112 : {static_cast<uint8_t>(RT_STARS_MEMCPY_ASYNC_OP_KIND_MIN), static_cast<uint8_t>(HCCL_REDUCE_MIN)},
113 : {static_cast<uint8_t>(RT_STARS_MEMCPY_ASYNC_OP_KIND_EQUAL), static_cast<uint8_t>(HCCL_REDUCE_RESERVED)},
114 : };
115 :
116 : } // namespace
117 :
118 5 : void TranslateOpcode(uint8_t opCode, uint8_t &reduceType)
119 : {
120 5 : reduceType = RT2HCCL_REDUCE_OP_MAP[opCode & 0x0F]; // opCode的低4位表示reduce类型
121 5 : HCCL_DEBUG("[TranslateOpcode] opCode=%u, reduceType=%u.", opCode, reduceType);
122 5 : }
123 :
124 3618 : void AddOneNotifyWaitSqeV1(uint16_t streamId, uint16_t taskId, u64 notifyId, const uint8_t *sqeIn, uint8_t *sqeType,
125 : const dfx::DfxTimeOutConfig &dfxTimeOutConfig)
126 : {
127 3618 : *sqeType = SqeType::NOTIFY_SQE;
128 3618 : rtStarsNotifySqeV1_t * const sqe = (rtStarsNotifySqeV1_t * const)sqeIn;
129 3618 : sqe->header.type = RT_STARS_SQE_TYPE_NOTIFY_WAIT;
130 3618 : const auto &credit_and_time_out = GetTimeOutValue(dfxTimeOutConfig);
131 3618 : sqe->kernel_credit = static_cast<uint8_t>(credit_and_time_out.first);
132 3618 : sqe->timeout = credit_and_time_out.second;
133 3618 : sqe->header.rtStreamId = streamId;
134 3618 : sqe->notify_id = notifyId;
135 3618 : sqe->header.taskId = taskId;
136 3618 : HCCL_INFO("[SQE] notify wait: notifyId=%lu, streamId=%u, taskId=%u, "
137 : "kernel_credit %u, timeout %u s.", notifyId, streamId, taskId, sqe->kernel_credit,
138 : sqe->timeout);
139 3618 : }
140 :
141 2081 : void AddOneRecordSqeV1(uint16_t streamId, uint16_t taskId, u64 notifyId, const uint8_t *sqeIn, uint8_t *sqeType)
142 : {
143 2081 : *sqeType = SqeType::NOTIFY_SQE;
144 2081 : rtStarsNotifySqeV1_t * const sqe = (rtStarsNotifySqeV1_t * const)sqeIn;
145 2081 : sqe->header.type = RT_STARS_SQE_TYPE_NOTIFY_RECORD;
146 2081 : sqe->kernel_credit = RT_STARS_DEFAULT_KERNEL_CREDIT;
147 2081 : sqe->header.rtStreamId = streamId;
148 2081 : sqe->notify_id = notifyId;
149 2081 : sqe->header.taskId = taskId;
150 2081 : HCCL_INFO("[SQE] notify record: notifyId=%lu, streamId=%u, taskId=%u, "
151 : "kernel_credit %u, timeout %u s.", notifyId, streamId, taskId, sqe->kernel_credit,
152 : sqe->timeout);
153 2081 : }
154 :
155 1586 : void AddOneWriteValueRecordSqeV1(uint16_t streamId, uint16_t taskId, u64 notifyWRAddr, const uint8_t *sqeIn,
156 : uint8_t *sqeType)
157 : {
158 1586 : *sqeType = SqeType::WRITE_VALUE_SQE;
159 1586 : rtStarsWriteValueSqe_t * const sqe = (rtStarsWriteValueSqe_t * const)sqeIn;
160 1586 : sqe->header.type = RT_STARS_SQE_TYPE_WRITE_VALUE;
161 1586 : sqe->header.rtStreamId = streamId;
162 1586 : sqe->header.taskId = taskId;
163 1586 : sqe->kernel_credit = RT_STARS_DEFAULT_KERNEL_CREDIT;
164 1586 : sqe->awsize = RT_STARS_WRITE_VALUE_SIZE_TYPE_32BIT;
165 1586 : sqe->write_value_part0 = 1U;
166 1586 : sqe->sub_type = RT_STARS_WRITE_VALUE_SUB_TYPE_NOTIFY_RECORD_IPC_NO_PCIE;
167 1586 : sqe->write_addr_low = static_cast<uint32_t>(notifyWRAddr & MASK_32_BIT);
168 1586 : sqe->write_addr_high = static_cast<uint32_t>((notifyWRAddr >> UINT32_BIT_NUM) & MASK_17_BIT);
169 1586 : HCCL_INFO("[SQE] write value: writePtr=0x%lx, streamId=%u, taskId=%u.", notifyWRAddr, streamId, taskId);
170 1586 : }
171 :
172 860 : void AddOneMemcpySqeV1(uint16_t streamId, uint16_t taskId, const void *src, uint32_t length,
173 : const aclDataType runtimeDataType, aclrtReduceKind rtReduceOp, const void *dst, uint32_t partId, uint32_t ssid,
174 : uint32_t devId, u64 overflowAddr, uint8_t linkType, const uint8_t *sqeIn, uint8_t *sqeType, uint32_t hcclQos)
175 : {
176 : (void)ssid;
177 : (void)devId;
178 : (void)overflowAddr;
179 860 : *sqeType = SqeType::MEMCPY_ASYNC_SQE;
180 860 : rtStarsMemcpyAsyncSqe_t * const sqe = (rtStarsMemcpyAsyncSqe_t * const)sqeIn;
181 :
182 : u32 len, srcAddrLow, srcAddrHigh, dstAddrLow, dstAddrHigh;
183 860 : if (length != 0U || src != nullptr || dst != nullptr) {
184 859 : len = length;
185 859 : srcAddrLow = static_cast<uint32_t>(reinterpret_cast<u64>(src) & 0x00000000ffffffffU);
186 859 : srcAddrHigh = static_cast<uint32_t>((reinterpret_cast<u64>(src) & 0xffffffff00000000U) >> UINT32_BIT_NUM);
187 859 : dstAddrLow = static_cast<uint32_t>(reinterpret_cast<u64>(dst) & 0x00000000ffffffffU);
188 859 : dstAddrHigh = static_cast<uint32_t>((reinterpret_cast<u64>(dst) & 0xffffffff00000000U) >> UINT32_BIT_NUM);
189 : } else {
190 1 : len = sqe->length;
191 1 : srcAddrLow = sqe->src_addr_low;
192 1 : srcAddrHigh = sqe->src_addr_high;
193 1 : dstAddrLow = sqe->dst_addr_low;
194 1 : dstAddrHigh = sqe->dst_addr_high;
195 1 : (void)memset_s(sqe, sizeof(rtStarsMemcpyAsyncSqe_t), 0, sizeof(rtStarsMemcpyAsyncSqe_t));
196 : }
197 :
198 860 : sqe->header.type = RT_STARS_SQE_TYPE_SDMA;
199 860 : sqe->header.rtStreamId = streamId;
200 860 : sqe->header.taskId = taskId;
201 860 : sqe->kernel_credit = dfx::kCreditTimeDefault;
202 860 : const bool isReduce =
203 697 : ((rtReduceOp == ACL_RT_MEMCPY_SDMA_AUTOMATIC_SUM) || (rtReduceOp == ACL_RT_MEMCPY_SDMA_AUTOMATIC_MAX) ||
204 1557 : (rtReduceOp == ACL_RT_MEMCPY_SDMA_AUTOMATIC_MIN) || (rtReduceOp == ACL_RT_MEMCPY_SDMA_AUTOMATIC_EQUAL));
205 :
206 860 : sqe->opcode = isReduce ? GetOpcodeForReduce(rtReduceOp, runtimeDataType) : 0U;
207 860 : if (linkType == static_cast<uint8_t>(hccl::LinkType::LINK_SIO) || linkType == static_cast<uint8_t>(hccl::LinkType::LINK_ONCHIP)) {
208 4 : hcclQos = SDMA_QOS_DEFAULT;
209 : }
210 860 : HCCL_INFO("[SQE]MemcpySqe copyKind=%u,Opcode=0x%x, streamId=%u, len=%u, src:%p, dst:%p, sqe->linkType=%u, hcclQos=%u",
211 : static_cast<uint32_t>(rtReduceOp), static_cast<uint32_t>(sqe->opcode), streamId, length, src, dst, static_cast<unsigned int>(linkType), hcclQos);
212 :
213 860 : sqe->length = len;
214 860 : sqe->src_addr_low = srcAddrLow;
215 860 : sqe->src_addr_high = srcAddrHigh;
216 860 : sqe->dst_addr_low = dstAddrLow;
217 860 : sqe->dst_addr_high = dstAddrHigh;
218 860 : sqe->sssv = 1U;
219 860 : sqe->dssv = 1U;
220 860 : sqe->sns = 1U;
221 860 : sqe->dns = 1U;
222 860 : sqe->qos = hcclQos;
223 860 : sqe->partid = partId;
224 860 : sqe->linkType = linkType;
225 860 : }
226 :
227 9 : void AddOneRdmaDbSendSqeV1(uint16_t streamId, uint16_t taskId, uint64_t dbInfo, uint64_t dbAddr,
228 : uint32_t length, uint8_t rdmaType, const uint8_t *sqeIn, uint8_t *sqeType)
229 : {
230 9 : *sqeType = SqeType::RDMA_DB_SEND_SQE;
231 9 : rtStarsWriteValueSqe_t * const sqe = (rtStarsWriteValueSqe_t * const)sqeIn;
232 :
233 9 : sqe->header.type = RT_STARS_SQE_TYPE_WRITE_VALUE;
234 9 : sqe->header.ie = RT_STARS_SQE_INT_DIR_NO;
235 9 : sqe->header.preP = RT_STARS_SQE_INT_DIR_NO;
236 9 : sqe->header.postP = RT_STARS_SQE_INT_DIR_NO;
237 9 : sqe->header.wrCqe = 0U;
238 9 : sqe->header.rtStreamId = streamId;
239 9 : sqe->header.taskId = taskId;
240 :
241 9 : sqe->va = 0U;
242 9 : sqe->kernel_credit = RT_STARS_DEFAULT_KERNEL_CREDIT;
243 9 : sqe->awsize = RT_STARS_WRITE_VALUE_SIZE_TYPE_64BIT;
244 :
245 9 : sqe->sub_type = RT_STARS_WRITE_VALUE_SUB_TYPE_RDMA_DB_SEND;
246 :
247 9 : if (dbAddr == 0ULL) {
248 1 : sqe->header.type = RT_STARS_SQE_TYPE_INVALID;
249 1 : return;
250 : }
251 8 : sqe->write_value_part0 = static_cast<uint32_t>(dbInfo & MASK_32_BIT);
252 8 : sqe->write_value_part1 = static_cast<uint32_t>(dbInfo >> UINT32_BIT_NUM);
253 8 : sqe->write_addr_low = static_cast<uint32_t>(dbAddr & MASK_32_BIT);
254 8 : sqe->write_addr_high = static_cast<uint32_t>((dbAddr >> UINT32_BIT_NUM) & MASK_17_BIT);
255 8 : sqe->rdmaWrLenth = length; // wr len
256 8 : sqe->rdmaType = static_cast<uint32_t>(rdmaType);
257 8 : HCCL_INFO("[SQE]RdmaDbSend: length=%u, rdmaType=%u, dbAddr=0x%lx, streamId=%u, taskId=%u.",
258 : length, rdmaType, dbAddr, streamId, taskId);
259 : }
260 :
261 3 : void AddOneEventResetSqeV1(uint16_t streamId, int32_t eventId, uint16_t taskId, int64_t phyChipId, int64_t phyDieId,
262 : u64 eventAddr, const uint8_t *sqeIn, uint8_t *sqeType)
263 : {
264 : (void)eventAddr;
265 3 : *sqeType = SqeType::WRITE_VALUE_SQE;
266 3 : rtStarsWriteValueSqe_t * const sqe = (rtStarsWriteValueSqe_t * const)sqeIn;
267 3 : sqe->header.type = RT_STARS_SQE_TYPE_WRITE_VALUE;
268 :
269 3 : sqe->header.rtStreamId = streamId;
270 3 : sqe->header.taskId = taskId;
271 :
272 3 : sqe->kernel_credit = RT_STARS_DEFAULT_KERNEL_CREDIT;
273 3 : sqe->res7 = static_cast<uint32_t>(eventId);
274 3 : sqe->sub_type = RT_STARS_WRITE_VALUE_SUB_TYPE_EVENT_RESET;
275 :
276 3 : const u64 eventTableId = static_cast<u64>(eventId) / STARS_EVENT_NUM_OF_SINGLE_TABLE;
277 :
278 : /* same as eventid % STARS_EVENT_NUM_OF_SINGLE_TABLE */
279 3 : const u64 eventNum = (static_cast<u64>(eventId)) & 0xFFFUL;
280 : // 默认devType 为DevType::DEV_TYPE_COUNT stream->Device_()->GetPhyChipId() 默认0 stream->Device_()->GetPhyDieId() 默认0
281 3 : u64 base =
282 3 : static_cast<u64>(RT_STARS_BASE_ADDR + (RT_CHIP_ADDR_OFFSET * static_cast<u64>(phyChipId)) +
283 : (RT_DIE_ADDR_OFFSET * static_cast<u64>(phyDieId)) + STARS_EVENT_BASE_ADDR);
284 3 : const u64 addr = base + (eventTableId * STARS_EVENT_TABLE_OFFSET) + (eventNum * STARS_EVENT_OFFSET);
285 3 : sqe->write_addr_low = static_cast<uint32_t>(addr & MASK_32_BIT);
286 3 : sqe->write_addr_high = static_cast<uint32_t>((addr >> UINT32_BIT_NUM) & MASK_17_BIT);
287 3 : HCCL_INFO("[SQE] event_reset: eventId=%u, streamId=%u, taskId=%u, addr:%p", eventId, streamId, taskId, addr);
288 3 : }
289 :
290 1 : void AddOneEventRecordSqeV1(uint16_t streamId, int32_t eventId, uint16_t taskId, const uint8_t *sqeIn, uint8_t *sqeType)
291 : {
292 1 : *sqeType = SqeType::EVENT_SQE;
293 1 : rtStarsEventSqe_t * const sqe = (rtStarsEventSqe_t * const)sqeIn;
294 1 : sqe->header.type = RT_STARS_SQE_TYPE_EVENT_RECORD;
295 1 : sqe->header.wrCqe = 1U; // 1: set wrCqe
296 1 : sqe->kernel_credit = RT_STARS_DEFAULT_KERNEL_CREDIT;
297 :
298 : // eventRecordTaskInfo->waitCqflag 是否为同步task
299 : if (false) {
300 : streamId |= RT_SYNC_TASK_FLAG;
301 : }
302 :
303 1 : sqe->header.rtStreamId = streamId;
304 1 : sqe->eventId = static_cast<uint16_t>(eventId);
305 1 : sqe->header.taskId = taskId;
306 1 : HCCL_INFO("[SQE] event record: eventId=%d, streamId=%u, taskId=%u.", eventId, streamId, taskId);
307 1 : }
308 :
309 3 : void AddOneEventWaitSqeV1(uint16_t streamId, int32_t eventId, uint16_t taskId, const uint8_t *sqeIn, uint8_t *sqeType)
310 : {
311 3 : *sqeType = SqeType::EVENT_SQE;
312 3 : rtStarsEventSqe_t * const sqe = (rtStarsEventSqe_t * const)sqeIn;
313 3 : sqe->header.type = RT_STARS_SQE_TYPE_EVENT_WAIT;
314 3 : sqe->kernel_credit = RT_STARS_NEVER_TIMEOUT_KERNEL_CREDIT;
315 :
316 3 : sqe->header.rtStreamId = streamId;
317 3 : sqe->eventId = eventId;
318 3 : sqe->header.taskId = taskId;
319 3 : HCCL_INFO("[SQE] event wait: eventId=%d, streamId=%u, taskId=%u", eventId, streamId, taskId);
320 3 : }
321 :
322 2 : void AddOneFlipPlaceHolderSqeV1(uint16_t streamId, uint16_t flipNum, uint16_t taskId, const uint8_t *sqeIn, uint8_t *sqeType)
323 : {
324 2 : *sqeType = SqeType::FLIP_PLACEHOLDER_SQE;
325 2 : rtStarsPlaceHolderSqe_t * const sqe = (rtStarsPlaceHolderSqe_t * const)sqeIn;
326 2 : sqe->header.type = RT_STARS_SQE_TYPE_PLACE_HOLDER;
327 2 : sqe->header.ie = 0U;
328 2 : sqe->header.preP = 1U;
329 2 : sqe->header.postP = 0U;
330 2 : sqe->header.wrCqe = 0U;
331 2 : sqe->header.reserved = 0U;
332 2 : sqe->header.blockDim = RT_TASK_TYPE_FLIP; // task type
333 2 : sqe->header.rtStreamId = streamId;
334 2 : sqe->header.taskId = taskId;
335 2 : sqe->kernel_credit = RT_STARS_DEFAULT_KERNEL_CREDIT;
336 2 : sqe->u.flip_task_info.flipNumReport = flipNum;
337 :
338 2 : HCCL_INFO("[SQE] placeholder: flipNum=%d, streamId=%u, taskId=%u", flipNum, streamId, taskId);
339 2 : }
340 :
341 2 : void AddOneCacheMemcpyPlaceHolderSqeV1(uint16_t streamId, uint16_t taskId, const void *src, const void *dst,
342 : uint8_t linkType, const uint8_t *sqeIn, uint8_t *sqeType, uint32_t hcclQos)
343 : {
344 2 : *sqeType = SqeType::CACHE_MEMCPY_PLACEHOLDER_SQE;
345 2 : SetCachePlaceholderHeaderV1(streamId, taskId, sqeIn);
346 :
347 2 : rtStarsPlaceHolderSqe_t * const sqe = (rtStarsPlaceHolderSqe_t * const)sqeIn;
348 2 : constexpr uint64_t uintBitWidth = 32;
349 2 : uint64_t srcAddr = reinterpret_cast<uint64_t>(src);
350 2 : uint64_t dstAddr = reinterpret_cast<uint64_t>(dst);
351 2 : sqe->u.cache_memcpy_task_info.src_addr_high = static_cast<uint32_t>(srcAddr >> uintBitWidth);
352 2 : sqe->u.cache_memcpy_task_info.src_addr_low = static_cast<uint32_t>(srcAddr & 0xFFFFFFFFULL);
353 2 : sqe->u.cache_memcpy_task_info.dst_addr_high = static_cast<uint32_t>(dstAddr >> uintBitWidth);
354 2 : sqe->u.cache_memcpy_task_info.dst_addr_low = static_cast<uint32_t>(dstAddr & 0xFFFFFFFFULL);
355 2 : sqe->u.cache_memcpy_task_info.kernel_credit = dfx::kCreditTimeDefault;
356 2 : sqe->u.cache_memcpy_task_info.linkType = linkType;
357 :
358 2 : HCCL_INFO("[SQE] cache-memcpy placeholder: streamId=%u, taskId=%u, srcAddr=0x%016llx, dstAddr=0x%016llx, linkType=%u, hcclQos=%u",
359 : streamId, taskId, src, dst, linkType, hcclQos);
360 :
361 : // 适配QoS
362 2 : if (linkType == static_cast<uint8_t>(hccl::LinkType::LINK_SIO) || linkType == static_cast<uint8_t>(hccl::LinkType::LINK_ONCHIP)) {
363 2 : hcclQos = SDMA_QOS_DEFAULT;
364 : }
365 2 : HCCL_INFO("[AddOneCacheMemcpyPlaceHolderSqeV1] sqe->linkType=%u hcclQos=%u",
366 : static_cast<unsigned int>(linkType), static_cast<unsigned int>(hcclQos));
367 2 : sqe->u.cache_memcpy_task_info.qos = hcclQos;
368 2 : }
369 :
370 1 : void AddOneCacheNotifyWaitPlaceholderSqeV1(uint16_t streamId, uint16_t taskId, u64 notifyId, const uint8_t *sqeIn, uint8_t *sqeType,
371 : const dfx::DfxTimeOutConfig &dfxTimeOutConfig)
372 : {
373 1 : *sqeType = SqeType::CACHE_NOTIFY_PLACEHOLDER_SQE;
374 1 : SetCachePlaceholderHeaderV1(streamId, taskId, sqeIn);
375 :
376 1 : rtStarsPlaceHolderSqe_t * const sqe = (rtStarsPlaceHolderSqe_t * const)sqeIn;
377 1 : sqe->u.cache_notify_task_info.is_wait = 1; // NotifyWait
378 1 : const auto &credit_and_time_out = GetTimeOutValue(dfxTimeOutConfig);
379 1 : sqe->u.cache_notify_task_info.kernel_credit = static_cast<uint8_t>(credit_and_time_out.first);
380 1 : sqe->u.cache_notify_task_info.timeout = credit_and_time_out.second;
381 1 : sqe->u.cache_notify_task_info.notify_id = notifyId;
382 :
383 1 : HCCL_INFO("[SQE] cache-notify placeholder (wait): notifyId=%lu, streamId=%u, taskId=%u, u.kernel_credit %u, timeout %u s.",
384 : notifyId, streamId, taskId, sqe->u.cache_notify_task_info.kernel_credit, sqe->u.cache_notify_task_info.timeout);
385 1 : }
386 :
387 1 : void AddOneCacheNotifyRecordPlaceholderSqeV1(uint16_t streamId, uint16_t taskId, u64 notifyId, const uint8_t *sqeIn, uint8_t *sqeType)
388 : {
389 1 : *sqeType = SqeType::CACHE_NOTIFY_PLACEHOLDER_SQE;
390 1 : SetCachePlaceholderHeaderV1(streamId, taskId, sqeIn);
391 :
392 1 : rtStarsPlaceHolderSqe_t * const sqe = (rtStarsPlaceHolderSqe_t * const)sqeIn;
393 1 : sqe->u.cache_notify_task_info.is_wait = 0; // NotifyRecord
394 1 : sqe->u.cache_notify_task_info.notify_id = notifyId;
395 :
396 1 : HCCL_INFO("[SQE] cache-notify placeholder (record): notifyId=%lu, streamId=%u, taskId=%u, "
397 : "kernel_credit %u.", notifyId, streamId, taskId, sqe->kernel_credit);
398 1 : }
399 :
400 1 : void AddOneCacheWriteValuePlaceholderSqeV1(uint16_t streamId, uint16_t taskId, u64 notifyWRAddr, const uint8_t *sqeIn,
401 : uint8_t *sqeType)
402 : {
403 1 : *sqeType = SqeType::CACHE_WRITE_VALUE_PLACEHOLDER_SQE;
404 1 : SetCachePlaceholderHeaderV1(streamId, taskId, sqeIn);
405 :
406 1 : rtStarsPlaceHolderSqe_t * const sqe = (rtStarsPlaceHolderSqe_t * const)sqeIn;
407 1 : sqe->u.cache_write_value_task_info.write_addr_low = static_cast<uint32_t>(notifyWRAddr & MASK_32_BIT);
408 1 : sqe->u.cache_write_value_task_info.write_addr_high = static_cast<uint32_t>((notifyWRAddr >> UINT32_BIT_NUM) & MASK_17_BIT);
409 :
410 1 : HCCL_INFO("[SQE] cache-write placeholder: writePtr=0x%lx, streamId=%u, taskId=%u.", notifyWRAddr, streamId, taskId);
411 1 : }
412 :
413 2 : void AddOneCacheMemcpyRecordPlaceholderSqeV1(uint16_t streamId, uint16_t taskId, const void *src, uint32_t length,
414 : const aclDataType runtimeDataType, aclrtReduceKind rtReduceOp, const void *dst, uint32_t partId, uint32_t ssid,
415 : uint32_t devId, u64 overflowAddr, uint8_t linkType, const uint8_t *sqeIn, uint8_t *sqeType, uint32_t hcclQos)
416 : {
417 : (void)ssid;
418 : (void)devId;
419 : (void)overflowAddr;
420 2 : *sqeType = SqeType::CACHE_MEMCPY_RECORD_PLACEHOLDER_SQE;
421 2 : rtStarsPlaceHolderSqe_t * const sqe = (rtStarsPlaceHolderSqe_t * const)sqeIn;
422 :
423 : u32 len, srcAddrLow, srcAddrHigh, dstAddrLow, dstAddrHigh;
424 2 : if (length != 0U || src != nullptr || dst != nullptr) {
425 2 : len = length;
426 2 : srcAddrLow = static_cast<uint32_t>(reinterpret_cast<u64>(src) & 0x00000000ffffffffU);
427 2 : srcAddrHigh = static_cast<uint32_t>((reinterpret_cast<u64>(src) & 0xffffffff00000000U) >> UINT32_BIT_NUM);
428 2 : dstAddrLow = static_cast<uint32_t>(reinterpret_cast<u64>(dst) & 0x00000000ffffffffU);
429 2 : dstAddrHigh = static_cast<uint32_t>((reinterpret_cast<u64>(dst) & 0xffffffff00000000U) >> UINT32_BIT_NUM);
430 : } else {
431 0 : len = sqe->u.cache_memcpy_record_task_info.length;
432 0 : srcAddrLow = sqe->u.cache_memcpy_record_task_info.src_addr_low;
433 0 : srcAddrHigh = sqe->u.cache_memcpy_record_task_info.src_addr_high;
434 0 : dstAddrLow = sqe->u.cache_memcpy_record_task_info.dst_addr_low;
435 0 : dstAddrHigh = sqe->u.cache_memcpy_record_task_info.dst_addr_high;
436 0 : (void)memset_s(sqe, sizeof(rtStarsPlaceHolderSqe_t), 0, sizeof(rtStarsPlaceHolderSqe_t));
437 : }
438 :
439 : // 用于placeholder
440 2 : SetCachePlaceholderHeaderV1(streamId, taskId, sqeIn);
441 :
442 : // 保存memcpy-record SQE的相关信息
443 2 : sqe->u.cache_memcpy_record_task_info.kernel_credit = dfx::kCreditTimeDefault;
444 2 : const bool isReduce =
445 0 : ((rtReduceOp == ACL_RT_MEMCPY_SDMA_AUTOMATIC_SUM) || (rtReduceOp == ACL_RT_MEMCPY_SDMA_AUTOMATIC_MAX) ||
446 2 : (rtReduceOp == ACL_RT_MEMCPY_SDMA_AUTOMATIC_MIN) || (rtReduceOp == ACL_RT_MEMCPY_SDMA_AUTOMATIC_EQUAL));
447 2 : sqe->u.cache_memcpy_record_task_info.opcode = isReduce ? GetOpcodeForReduce(rtReduceOp, runtimeDataType) : 0U;
448 2 : HCCL_INFO("[SQE] cache-write-memcpy placeholder: copyKind=%u,Opcode=0x%x, streamId=%u, len=%u, src:%p, dst:%p, hcclQos: %u",
449 : static_cast<uint32_t>(rtReduceOp), static_cast<uint32_t>(sqe->u.cache_memcpy_record_task_info.opcode),
450 : streamId, length, src, dst, hcclQos);
451 2 : sqe->u.cache_memcpy_record_task_info.length = len;
452 2 : sqe->u.cache_memcpy_record_task_info.src_addr_low = srcAddrLow;
453 2 : sqe->u.cache_memcpy_record_task_info.src_addr_high = srcAddrHigh;
454 2 : sqe->u.cache_memcpy_record_task_info.dst_addr_low = dstAddrLow;
455 2 : sqe->u.cache_memcpy_record_task_info.dst_addr_high = dstAddrHigh;
456 2 : sqe->u.cache_memcpy_record_task_info.partid = partId;
457 2 : sqe->u.cache_memcpy_record_task_info.linkType = linkType;
458 :
459 : // 适配qos
460 2 : if (linkType == static_cast<uint8_t>(hccl::LinkType::LINK_SIO) || linkType == static_cast<uint8_t>(hccl::LinkType::LINK_ONCHIP)) {
461 2 : hcclQos = SDMA_QOS_DEFAULT;
462 : }
463 2 : HCCL_INFO("[AddOneCacheMemcpyRecordPlaceholderSqeV1] sqe->linkType=%u hcclQos=%u",
464 : static_cast<unsigned int>(linkType), static_cast<unsigned int>(hcclQos));
465 2 : sqe->u.cache_memcpy_record_task_info.qos = hcclQos;
466 2 : }
467 :
468 8 : void SetCachePlaceholderHeaderV1(uint16_t streamId, uint16_t taskId, const uint8_t *sqeIn) {
469 8 : rtStarsPlaceHolderSqe_t * const sqe = (rtStarsPlaceHolderSqe_t * const)sqeIn;
470 8 : sqe->header.type = RT_STARS_SQE_TYPE_PLACE_HOLDER;
471 8 : sqe->header.ie = 0U;
472 8 : sqe->header.preP = 0U; // 不需要STARS_FW参与任何预处理
473 8 : sqe->header.postP = 0U;
474 8 : sqe->header.wrCqe = 0U;
475 8 : sqe->header.reserved = 0U;
476 : // NOTE: task type在preP阶段被TASK_FW使用, 而此placeholder无preP阶段, 设置为RT_TASK_TYPE_FLIP不影响功能
477 8 : sqe->header.blockDim = RT_TASK_TYPE_FLIP;
478 8 : sqe->header.rtStreamId = streamId;
479 8 : sqe->header.taskId = taskId;
480 8 : sqe->kernel_credit = RT_STARS_DEFAULT_KERNEL_CREDIT;
481 8 : return;
482 : }
483 :
484 3621 : std::pair<uint64_t, uint64_t> GetTimeOutValue(const dfx::DfxTimeOutConfig &dfxTimeOutConfig)
485 : {
486 3621 : if (dfxTimeOutConfig.useCredit) {
487 3614 : HCCL_DEBUG("Use hard sync with %lu", dfxTimeOutConfig.sqeCreditTimeOut);
488 3614 : return {dfxTimeOutConfig.sqeCreditTimeOut, dfx::kTimeOutTimeInvalid};
489 : }
490 7 : HCCL_DEBUG("Use soft sync with %lu", dfxTimeOutConfig.sqeTimeOutTimeOut);
491 7 : return {dfx::kCreditTimeInvalid, dfxTimeOutConfig.sqeTimeOutTimeOut};
492 : }
|
