Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "aicpu/aicpu_hccl_sqcqv1.h"
12 : #include "hccl_common.h"
13 : #include "dispatcher_task_types.h"
14 : #include <unordered_map>
15 :
16 : namespace {
17 14 : bool ChipIsHaveStars() { return true; }
18 :
19 165 : uint8_t ReduceOpcodeHigh(uint8_t copyDataType)
20 : {
21 : uint8_t opcode;
22 165 : switch (copyDataType) {
23 1 : case ACL_INT8: {
24 1 : opcode = static_cast<uint8_t>(RT_STARS_MEMCPY_ASYNC_DATA_TYPE_INT8);
25 1 : break;
26 : }
27 0 : case ACL_INT16: {
28 0 : opcode = static_cast<uint8_t>(RT_STARS_MEMCPY_ASYNC_DATA_TYPE_INT16);
29 0 : break;
30 : }
31 0 : case ACL_INT32: {
32 0 : opcode = static_cast<uint8_t>(RT_STARS_MEMCPY_ASYNC_DATA_TYPE_INT32);
33 0 : break;
34 : }
35 148 : case ACL_FLOAT16: {
36 148 : opcode = static_cast<uint8_t>(RT_STARS_MEMCPY_ASYNC_DATA_TYPE_FP16);
37 148 : break;
38 : }
39 1 : case ACL_FLOAT: {
40 1 : opcode = static_cast<uint8_t>(RT_STARS_MEMCPY_ASYNC_DATA_TYPE_FP32);
41 1 : break;
42 : }
43 14 : case ACL_BF16: {
44 14 : if (ChipIsHaveStars()) {
45 14 : opcode = static_cast<uint8_t>(RT_STARS_MEMCPY_ASYNC_DATA_TYPE_BFP16);
46 : } else {
47 0 : HCCL_ERROR("DataType=%u do not support.", static_cast<uint32_t>(copyDataType));
48 0 : opcode = static_cast<uint8_t>(RT_STARS_MEMCPY_ASYNC_OP_RESERVED);
49 : }
50 14 : break;
51 : }
52 1 : default: {
53 : // Should not run here.
54 : // if not support, it will return RT_ERROR_FEATURE_NOT_SUPPORT at context.cc's reduce ability check.
55 : // Only for code style, 0x80 is reserved value of STRAS opcode.
56 1 : HCCL_ERROR("DataType=%u do not support.", static_cast<uint32_t>(copyDataType));
57 1 : opcode = static_cast<uint8_t>(RT_STARS_MEMCPY_ASYNC_OP_RESERVED);
58 1 : break;
59 : }
60 : }
61 165 : return opcode;
62 : }
63 :
64 165 : uint8_t ReduceOpcodeLow(uint32_t copyKind)
65 : {
66 : uint8_t opcode;
67 165 : switch (copyKind) {
68 165 : case ACL_RT_MEMCPY_SDMA_AUTOMATIC_SUM: {
69 165 : opcode = static_cast<uint8_t>(RT_STARS_MEMCPY_ASYNC_OP_KIND_ADD);
70 165 : break;
71 : }
72 0 : case ACL_RT_MEMCPY_SDMA_AUTOMATIC_MAX: {
73 0 : opcode = static_cast<uint8_t>(RT_STARS_MEMCPY_ASYNC_OP_KIND_MAX);
74 0 : break;
75 : }
76 0 : case ACL_RT_MEMCPY_SDMA_AUTOMATIC_MIN: {
77 0 : opcode = static_cast<uint8_t>(RT_STARS_MEMCPY_ASYNC_OP_KIND_MIN);
78 0 : break;
79 : }
80 0 : case ACL_RT_MEMCPY_SDMA_AUTOMATIC_EQUAL: {
81 0 : opcode = static_cast<uint8_t>(RT_STARS_MEMCPY_ASYNC_OP_KIND_EQUAL);
82 0 : break;
83 : }
84 0 : default: {
85 0 : HCCL_ERROR("Type out of range: copyKind=%u", copyKind);
86 0 : opcode = static_cast<uint8_t>(RT_STARS_MEMCPY_ASYNC_OP_RESERVED);
87 0 : break;
88 : }
89 : }
90 165 : return opcode;
91 : }
92 :
93 165 : uint8_t GetOpcodeForReduce(uint32_t copyKind, uint8_t copyDataType)
94 : {
95 165 : const uint8_t opcodeHigh = ReduceOpcodeHigh(copyDataType);
96 165 : const uint8_t opcodeLow = ReduceOpcodeLow(copyKind);
97 165 : if ((static_cast<int32_t>(opcodeHigh) == RT_STARS_MEMCPY_ASYNC_OP_RESERVED)
98 164 : || (static_cast<int32_t>(opcodeLow) == RT_STARS_MEMCPY_ASYNC_OP_RESERVED)) {
99 : // Should not run here. 0x80 is reserved value of STRAS opcode
100 1 : return static_cast<uint8_t>(RT_STARS_MEMCPY_ASYNC_OP_RESERVED);
101 : } else {
102 164 : return opcodeHigh | opcodeLow;
103 : }
104 : }
105 :
106 : std::unordered_map<uint8_t, uint8_t> RT2HCCL_REDUCE_OP_MAP = {
107 : {static_cast<uint8_t>(RT_STARS_MEMCPY_ASYNC_OP_KIND_CPY), static_cast<uint8_t>(HCCL_REDUCE_RESERVED)},
108 : {static_cast<uint8_t>(RT_STARS_MEMCPY_ASYNC_OP_KIND_ADD), static_cast<uint8_t>(HCCL_REDUCE_SUM)},
109 : {static_cast<uint8_t>(RT_STARS_MEMCPY_ASYNC_OP_KIND_MAX), static_cast<uint8_t>(HCCL_REDUCE_MAX)},
110 : {static_cast<uint8_t>(RT_STARS_MEMCPY_ASYNC_OP_KIND_MIN), static_cast<uint8_t>(HCCL_REDUCE_MIN)},
111 : {static_cast<uint8_t>(RT_STARS_MEMCPY_ASYNC_OP_KIND_EQUAL), static_cast<uint8_t>(HCCL_REDUCE_RESERVED)},
112 : };
113 :
114 : } // namespace
115 :
116 5 : void TranslateOpcode(uint8_t opCode, uint8_t& reduceType)
117 : {
118 5 : reduceType = RT2HCCL_REDUCE_OP_MAP[opCode & 0x0F]; // opCode的低4位表示reduce类型
119 5 : HCCL_DEBUG("[TranslateOpcode] opCode=%u, reduceType=%u.", opCode, reduceType);
120 5 : }
121 :
122 3618 : void AddOneNotifyWaitSqeV1(
123 : uint16_t streamId, uint16_t taskId, u64 notifyId, const uint8_t* sqeIn, uint8_t* sqeType,
124 : const dfx::DfxTimeOutConfig& dfxTimeOutConfig)
125 : {
126 3618 : *sqeType = SqeType::NOTIFY_SQE;
127 3618 : rtStarsNotifySqeV1_t* const sqe = (rtStarsNotifySqeV1_t* const)sqeIn;
128 3618 : sqe->header.type = RT_STARS_SQE_TYPE_NOTIFY_WAIT;
129 3618 : const auto& credit_and_time_out = GetTimeOutValue(dfxTimeOutConfig);
130 3618 : sqe->kernel_credit = static_cast<uint8_t>(credit_and_time_out.first);
131 3618 : sqe->timeout = credit_and_time_out.second;
132 3618 : sqe->header.rtStreamId = streamId;
133 3618 : sqe->notify_id = notifyId;
134 3618 : sqe->header.taskId = taskId;
135 3618 : HCCL_INFO(
136 : "[SQE] notify wait: notifyId=%lu, streamId=%u, taskId=%u, "
137 : "kernel_credit %u, timeout %u s.",
138 : notifyId, streamId, taskId, sqe->kernel_credit, sqe->timeout);
139 3618 : }
140 :
141 2081 : void AddOneRecordSqeV1(uint16_t streamId, uint16_t taskId, u64 notifyId, const uint8_t* sqeIn, uint8_t* sqeType)
142 : {
143 2081 : *sqeType = SqeType::NOTIFY_SQE;
144 2081 : rtStarsNotifySqeV1_t* const sqe = (rtStarsNotifySqeV1_t* const)sqeIn;
145 2081 : sqe->header.type = RT_STARS_SQE_TYPE_NOTIFY_RECORD;
146 2081 : sqe->kernel_credit = RT_STARS_DEFAULT_KERNEL_CREDIT;
147 2081 : sqe->header.rtStreamId = streamId;
148 2081 : sqe->notify_id = notifyId;
149 2081 : sqe->header.taskId = taskId;
150 2081 : HCCL_INFO(
151 : "[SQE] notify record: notifyId=%lu, streamId=%u, taskId=%u, "
152 : "kernel_credit %u, timeout %u s.",
153 : notifyId, streamId, taskId, sqe->kernel_credit, sqe->timeout);
154 2081 : }
155 :
156 1586 : void AddOneWriteValueRecordSqeV1(
157 : uint16_t streamId, uint16_t taskId, u64 notifyWRAddr, const uint8_t* sqeIn, uint8_t* sqeType)
158 : {
159 1586 : *sqeType = SqeType::WRITE_VALUE_SQE;
160 1586 : rtStarsWriteValueSqe_t* const sqe = (rtStarsWriteValueSqe_t* const)sqeIn;
161 1586 : sqe->header.type = RT_STARS_SQE_TYPE_WRITE_VALUE;
162 1586 : sqe->header.rtStreamId = streamId;
163 1586 : sqe->header.taskId = taskId;
164 1586 : sqe->kernel_credit = RT_STARS_DEFAULT_KERNEL_CREDIT;
165 1586 : sqe->awsize = RT_STARS_WRITE_VALUE_SIZE_TYPE_32BIT;
166 1586 : sqe->write_value_part0 = 1U;
167 1586 : sqe->sub_type = RT_STARS_WRITE_VALUE_SUB_TYPE_NOTIFY_RECORD_IPC_NO_PCIE;
168 1586 : sqe->write_addr_low = static_cast<uint32_t>(notifyWRAddr & MASK_32_BIT);
169 1586 : sqe->write_addr_high = static_cast<uint32_t>((notifyWRAddr >> UINT32_BIT_NUM) & MASK_17_BIT);
170 1586 : HCCL_INFO("[SQE] write value: writePtr=0x%lx, streamId=%u, taskId=%u.", notifyWRAddr, streamId, taskId);
171 1586 : }
172 :
173 860 : void AddOneMemcpySqeV1(
174 : uint16_t streamId, uint16_t taskId, const void* src, uint32_t length, const aclDataType runtimeDataType,
175 : aclrtReduceKind rtReduceOp, const void* dst, uint32_t partId, uint32_t ssid, uint32_t devId, u64 overflowAddr,
176 : uint8_t linkType, const uint8_t* sqeIn, uint8_t* sqeType, uint32_t hcclQos)
177 : {
178 : (void)ssid;
179 : (void)devId;
180 : (void)overflowAddr;
181 860 : *sqeType = SqeType::MEMCPY_ASYNC_SQE;
182 860 : rtStarsMemcpyAsyncSqe_t* const sqe = (rtStarsMemcpyAsyncSqe_t* const)sqeIn;
183 :
184 : u32 len, srcAddrLow, srcAddrHigh, dstAddrLow, dstAddrHigh;
185 860 : if (length != 0U || src != nullptr || dst != nullptr) {
186 859 : len = length;
187 859 : srcAddrLow = static_cast<uint32_t>(reinterpret_cast<u64>(src) & 0x00000000ffffffffU);
188 859 : srcAddrHigh = static_cast<uint32_t>((reinterpret_cast<u64>(src) & 0xffffffff00000000U) >> UINT32_BIT_NUM);
189 859 : dstAddrLow = static_cast<uint32_t>(reinterpret_cast<u64>(dst) & 0x00000000ffffffffU);
190 859 : dstAddrHigh = static_cast<uint32_t>((reinterpret_cast<u64>(dst) & 0xffffffff00000000U) >> UINT32_BIT_NUM);
191 : } else {
192 1 : len = sqe->length;
193 1 : srcAddrLow = sqe->src_addr_low;
194 1 : srcAddrHigh = sqe->src_addr_high;
195 1 : dstAddrLow = sqe->dst_addr_low;
196 1 : dstAddrHigh = sqe->dst_addr_high;
197 1 : (void)memset_s(sqe, sizeof(rtStarsMemcpyAsyncSqe_t), 0, sizeof(rtStarsMemcpyAsyncSqe_t));
198 : }
199 :
200 860 : sqe->header.type = RT_STARS_SQE_TYPE_SDMA;
201 860 : sqe->header.rtStreamId = streamId;
202 860 : sqe->header.taskId = taskId;
203 860 : sqe->kernel_credit = dfx::kCreditTimeDefault;
204 860 : const bool isReduce
205 697 : = ((rtReduceOp == ACL_RT_MEMCPY_SDMA_AUTOMATIC_SUM) || (rtReduceOp == ACL_RT_MEMCPY_SDMA_AUTOMATIC_MAX)
206 1557 : || (rtReduceOp == ACL_RT_MEMCPY_SDMA_AUTOMATIC_MIN) || (rtReduceOp == ACL_RT_MEMCPY_SDMA_AUTOMATIC_EQUAL));
207 :
208 860 : sqe->opcode = isReduce ? GetOpcodeForReduce(rtReduceOp, runtimeDataType) : 0U;
209 860 : if (linkType == static_cast<uint8_t>(hccl::LinkType::LINK_SIO)
210 859 : || linkType == static_cast<uint8_t>(hccl::LinkType::LINK_ONCHIP)) {
211 4 : hcclQos = SDMA_QOS_DEFAULT;
212 : }
213 860 : HCCL_INFO(
214 : "[SQE]MemcpySqe copyKind=%u,Opcode=0x%x, streamId=%u, len=%u, src:%p, dst:%p, sqe->linkType=%u, hcclQos=%u",
215 : static_cast<uint32_t>(rtReduceOp), static_cast<uint32_t>(sqe->opcode), streamId, length, src, dst,
216 : static_cast<unsigned int>(linkType), hcclQos);
217 :
218 860 : sqe->length = len;
219 860 : sqe->src_addr_low = srcAddrLow;
220 860 : sqe->src_addr_high = srcAddrHigh;
221 860 : sqe->dst_addr_low = dstAddrLow;
222 860 : sqe->dst_addr_high = dstAddrHigh;
223 860 : sqe->sssv = 1U;
224 860 : sqe->dssv = 1U;
225 860 : sqe->sns = 1U;
226 860 : sqe->dns = 1U;
227 860 : sqe->qos = hcclQos;
228 860 : sqe->partid = partId;
229 860 : sqe->linkType = linkType;
230 860 : }
231 :
232 9 : void AddOneRdmaDbSendSqeV1(
233 : uint16_t streamId, uint16_t taskId, uint64_t dbInfo, uint64_t dbAddr, uint32_t length, uint8_t rdmaType,
234 : const uint8_t* sqeIn, uint8_t* sqeType)
235 : {
236 9 : *sqeType = SqeType::RDMA_DB_SEND_SQE;
237 9 : rtStarsWriteValueSqe_t* const sqe = (rtStarsWriteValueSqe_t* const)sqeIn;
238 :
239 9 : sqe->header.type = RT_STARS_SQE_TYPE_WRITE_VALUE;
240 9 : sqe->header.ie = RT_STARS_SQE_INT_DIR_NO;
241 9 : sqe->header.preP = RT_STARS_SQE_INT_DIR_NO;
242 9 : sqe->header.postP = RT_STARS_SQE_INT_DIR_NO;
243 9 : sqe->header.wrCqe = 0U;
244 9 : sqe->header.rtStreamId = streamId;
245 9 : sqe->header.taskId = taskId;
246 :
247 9 : sqe->va = 0U;
248 9 : sqe->kernel_credit = RT_STARS_DEFAULT_KERNEL_CREDIT;
249 9 : sqe->awsize = RT_STARS_WRITE_VALUE_SIZE_TYPE_64BIT;
250 :
251 9 : sqe->sub_type = RT_STARS_WRITE_VALUE_SUB_TYPE_RDMA_DB_SEND;
252 :
253 9 : if (dbAddr == 0ULL) {
254 1 : sqe->header.type = RT_STARS_SQE_TYPE_INVALID;
255 1 : return;
256 : }
257 8 : sqe->write_value_part0 = static_cast<uint32_t>(dbInfo & MASK_32_BIT);
258 8 : sqe->write_value_part1 = static_cast<uint32_t>(dbInfo >> UINT32_BIT_NUM);
259 8 : sqe->write_addr_low = static_cast<uint32_t>(dbAddr & MASK_32_BIT);
260 8 : sqe->write_addr_high = static_cast<uint32_t>((dbAddr >> UINT32_BIT_NUM) & MASK_17_BIT);
261 8 : sqe->rdmaWrLenth = length; // wr len
262 8 : sqe->rdmaType = static_cast<uint32_t>(rdmaType);
263 8 : HCCL_INFO(
264 : "[SQE]RdmaDbSend: length=%u, rdmaType=%u, dbAddr=0x%lx, streamId=%u, taskId=%u.", length, rdmaType, dbAddr,
265 : streamId, taskId);
266 : }
267 :
268 3 : void AddOneEventResetSqeV1(
269 : uint16_t streamId, int32_t eventId, uint16_t taskId, int64_t phyChipId, int64_t phyDieId, u64 eventAddr,
270 : const uint8_t* sqeIn, uint8_t* sqeType)
271 : {
272 : (void)eventAddr;
273 3 : *sqeType = SqeType::WRITE_VALUE_SQE;
274 3 : rtStarsWriteValueSqe_t* const sqe = (rtStarsWriteValueSqe_t* const)sqeIn;
275 3 : sqe->header.type = RT_STARS_SQE_TYPE_WRITE_VALUE;
276 :
277 3 : sqe->header.rtStreamId = streamId;
278 3 : sqe->header.taskId = taskId;
279 :
280 3 : sqe->kernel_credit = RT_STARS_DEFAULT_KERNEL_CREDIT;
281 3 : sqe->res7 = static_cast<uint32_t>(eventId);
282 3 : sqe->sub_type = RT_STARS_WRITE_VALUE_SUB_TYPE_EVENT_RESET;
283 :
284 3 : const u64 eventTableId = static_cast<u64>(eventId) / STARS_EVENT_NUM_OF_SINGLE_TABLE;
285 :
286 : /* same as eventid % STARS_EVENT_NUM_OF_SINGLE_TABLE */
287 3 : const u64 eventNum = (static_cast<u64>(eventId)) & 0xFFFUL;
288 : // 默认devType 为DevType::DEV_TYPE_COUNT stream->Device_()->GetPhyChipId() 默认0 stream->Device_()->GetPhyDieId()
289 : // 默认0
290 3 : u64 base = static_cast<u64>(
291 3 : RT_STARS_BASE_ADDR + (RT_CHIP_ADDR_OFFSET * static_cast<u64>(phyChipId))
292 3 : + (RT_DIE_ADDR_OFFSET * static_cast<u64>(phyDieId)) + STARS_EVENT_BASE_ADDR);
293 3 : const u64 addr = base + (eventTableId * STARS_EVENT_TABLE_OFFSET) + (eventNum * STARS_EVENT_OFFSET);
294 3 : sqe->write_addr_low = static_cast<uint32_t>(addr & MASK_32_BIT);
295 3 : sqe->write_addr_high = static_cast<uint32_t>((addr >> UINT32_BIT_NUM) & MASK_17_BIT);
296 3 : HCCL_INFO("[SQE] event_reset: eventId=%u, streamId=%u, taskId=%u, addr:%p", eventId, streamId, taskId, addr);
297 3 : }
298 :
299 1 : void AddOneEventRecordSqeV1(uint16_t streamId, int32_t eventId, uint16_t taskId, const uint8_t* sqeIn, uint8_t* sqeType)
300 : {
301 1 : *sqeType = SqeType::EVENT_SQE;
302 1 : rtStarsEventSqe_t* const sqe = (rtStarsEventSqe_t* const)sqeIn;
303 1 : sqe->header.type = RT_STARS_SQE_TYPE_EVENT_RECORD;
304 1 : sqe->header.wrCqe = 1U; // 1: set wrCqe
305 1 : sqe->kernel_credit = RT_STARS_DEFAULT_KERNEL_CREDIT;
306 :
307 : // eventRecordTaskInfo->waitCqflag 是否为同步task
308 : if (false) {
309 : streamId |= RT_SYNC_TASK_FLAG;
310 : }
311 :
312 1 : sqe->header.rtStreamId = streamId;
313 1 : sqe->eventId = static_cast<uint16_t>(eventId);
314 1 : sqe->header.taskId = taskId;
315 1 : HCCL_INFO("[SQE] event record: eventId=%d, streamId=%u, taskId=%u.", eventId, streamId, taskId);
316 1 : }
317 :
318 3 : void AddOneEventWaitSqeV1(uint16_t streamId, int32_t eventId, uint16_t taskId, const uint8_t* sqeIn, uint8_t* sqeType)
319 : {
320 3 : *sqeType = SqeType::EVENT_SQE;
321 3 : rtStarsEventSqe_t* const sqe = (rtStarsEventSqe_t* const)sqeIn;
322 3 : sqe->header.type = RT_STARS_SQE_TYPE_EVENT_WAIT;
323 3 : sqe->kernel_credit = RT_STARS_NEVER_TIMEOUT_KERNEL_CREDIT;
324 :
325 3 : sqe->header.rtStreamId = streamId;
326 3 : sqe->eventId = eventId;
327 3 : sqe->header.taskId = taskId;
328 3 : HCCL_INFO("[SQE] event wait: eventId=%d, streamId=%u, taskId=%u", eventId, streamId, taskId);
329 3 : }
330 :
331 2 : void AddOneFlipPlaceHolderSqeV1(
332 : uint16_t streamId, uint16_t flipNum, uint16_t taskId, const uint8_t* sqeIn, uint8_t* sqeType)
333 : {
334 2 : *sqeType = SqeType::FLIP_PLACEHOLDER_SQE;
335 2 : rtStarsPlaceHolderSqe_t* const sqe = (rtStarsPlaceHolderSqe_t* const)sqeIn;
336 2 : sqe->header.type = RT_STARS_SQE_TYPE_PLACE_HOLDER;
337 2 : sqe->header.ie = 0U;
338 2 : sqe->header.preP = 1U;
339 2 : sqe->header.postP = 0U;
340 2 : sqe->header.wrCqe = 0U;
341 2 : sqe->header.reserved = 0U;
342 2 : sqe->header.blockDim = RT_TASK_TYPE_FLIP; // task type
343 2 : sqe->header.rtStreamId = streamId;
344 2 : sqe->header.taskId = taskId;
345 2 : sqe->kernel_credit = RT_STARS_DEFAULT_KERNEL_CREDIT;
346 2 : sqe->u.flip_task_info.flipNumReport = flipNum;
347 :
348 2 : HCCL_INFO("[SQE] placeholder: flipNum=%d, streamId=%u, taskId=%u", flipNum, streamId, taskId);
349 2 : }
350 :
351 2 : void AddOneCacheMemcpyPlaceHolderSqeV1(
352 : uint16_t streamId, uint16_t taskId, const void* src, const void* dst, uint8_t linkType, const uint8_t* sqeIn,
353 : uint8_t* sqeType, uint32_t hcclQos)
354 : {
355 2 : *sqeType = SqeType::CACHE_MEMCPY_PLACEHOLDER_SQE;
356 2 : SetCachePlaceholderHeaderV1(streamId, taskId, sqeIn);
357 :
358 2 : rtStarsPlaceHolderSqe_t* const sqe = (rtStarsPlaceHolderSqe_t* const)sqeIn;
359 2 : constexpr uint64_t uintBitWidth = 32;
360 2 : uint64_t srcAddr = reinterpret_cast<uint64_t>(src);
361 2 : uint64_t dstAddr = reinterpret_cast<uint64_t>(dst);
362 2 : sqe->u.cache_memcpy_task_info.src_addr_high = static_cast<uint32_t>(srcAddr >> uintBitWidth);
363 2 : sqe->u.cache_memcpy_task_info.src_addr_low = static_cast<uint32_t>(srcAddr & 0xFFFFFFFFULL);
364 2 : sqe->u.cache_memcpy_task_info.dst_addr_high = static_cast<uint32_t>(dstAddr >> uintBitWidth);
365 2 : sqe->u.cache_memcpy_task_info.dst_addr_low = static_cast<uint32_t>(dstAddr & 0xFFFFFFFFULL);
366 2 : sqe->u.cache_memcpy_task_info.kernel_credit = dfx::kCreditTimeDefault;
367 2 : sqe->u.cache_memcpy_task_info.linkType = linkType;
368 :
369 2 : HCCL_INFO(
370 : "[SQE] cache-memcpy placeholder: streamId=%u, taskId=%u, srcAddr=0x%016llx, dstAddr=0x%016llx, linkType=%u, "
371 : "hcclQos=%u",
372 : streamId, taskId, src, dst, linkType, hcclQos);
373 :
374 : // 适配QoS
375 2 : if (linkType == static_cast<uint8_t>(hccl::LinkType::LINK_SIO)
376 1 : || linkType == static_cast<uint8_t>(hccl::LinkType::LINK_ONCHIP)) {
377 2 : hcclQos = SDMA_QOS_DEFAULT;
378 : }
379 2 : HCCL_INFO(
380 : "[AddOneCacheMemcpyPlaceHolderSqeV1] sqe->linkType=%u hcclQos=%u", static_cast<unsigned int>(linkType),
381 : static_cast<unsigned int>(hcclQos));
382 2 : sqe->u.cache_memcpy_task_info.qos = hcclQos;
383 2 : }
384 :
385 1 : void AddOneCacheNotifyWaitPlaceholderSqeV1(
386 : uint16_t streamId, uint16_t taskId, u64 notifyId, const uint8_t* sqeIn, uint8_t* sqeType,
387 : const dfx::DfxTimeOutConfig& dfxTimeOutConfig)
388 : {
389 1 : *sqeType = SqeType::CACHE_NOTIFY_PLACEHOLDER_SQE;
390 1 : SetCachePlaceholderHeaderV1(streamId, taskId, sqeIn);
391 :
392 1 : rtStarsPlaceHolderSqe_t* const sqe = (rtStarsPlaceHolderSqe_t* const)sqeIn;
393 1 : sqe->u.cache_notify_task_info.is_wait = 1; // NotifyWait
394 1 : const auto& credit_and_time_out = GetTimeOutValue(dfxTimeOutConfig);
395 1 : sqe->u.cache_notify_task_info.kernel_credit = static_cast<uint8_t>(credit_and_time_out.first);
396 1 : sqe->u.cache_notify_task_info.timeout = credit_and_time_out.second;
397 1 : sqe->u.cache_notify_task_info.notify_id = notifyId;
398 :
399 1 : HCCL_INFO(
400 : "[SQE] cache-notify placeholder (wait): notifyId=%lu, streamId=%u, taskId=%u, u.kernel_credit %u, timeout %u "
401 : "s.",
402 : notifyId, streamId, taskId, sqe->u.cache_notify_task_info.kernel_credit, sqe->u.cache_notify_task_info.timeout);
403 1 : }
404 :
405 1 : void AddOneCacheNotifyRecordPlaceholderSqeV1(
406 : uint16_t streamId, uint16_t taskId, u64 notifyId, const uint8_t* sqeIn, uint8_t* sqeType)
407 : {
408 1 : *sqeType = SqeType::CACHE_NOTIFY_PLACEHOLDER_SQE;
409 1 : SetCachePlaceholderHeaderV1(streamId, taskId, sqeIn);
410 :
411 1 : rtStarsPlaceHolderSqe_t* const sqe = (rtStarsPlaceHolderSqe_t* const)sqeIn;
412 1 : sqe->u.cache_notify_task_info.is_wait = 0; // NotifyRecord
413 1 : sqe->u.cache_notify_task_info.notify_id = notifyId;
414 :
415 1 : HCCL_INFO(
416 : "[SQE] cache-notify placeholder (record): notifyId=%lu, streamId=%u, taskId=%u, "
417 : "kernel_credit %u.",
418 : notifyId, streamId, taskId, sqe->kernel_credit);
419 1 : }
420 :
421 1 : void AddOneCacheWriteValuePlaceholderSqeV1(
422 : uint16_t streamId, uint16_t taskId, u64 notifyWRAddr, const uint8_t* sqeIn, uint8_t* sqeType)
423 : {
424 1 : *sqeType = SqeType::CACHE_WRITE_VALUE_PLACEHOLDER_SQE;
425 1 : SetCachePlaceholderHeaderV1(streamId, taskId, sqeIn);
426 :
427 1 : rtStarsPlaceHolderSqe_t* const sqe = (rtStarsPlaceHolderSqe_t* const)sqeIn;
428 1 : sqe->u.cache_write_value_task_info.write_addr_low = static_cast<uint32_t>(notifyWRAddr & MASK_32_BIT);
429 : sqe->u.cache_write_value_task_info.write_addr_high
430 1 : = static_cast<uint32_t>((notifyWRAddr >> UINT32_BIT_NUM) & MASK_17_BIT);
431 :
432 1 : HCCL_INFO("[SQE] cache-write placeholder: writePtr=0x%lx, streamId=%u, taskId=%u.", notifyWRAddr, streamId, taskId);
433 1 : }
434 :
435 2 : void AddOneCacheMemcpyRecordPlaceholderSqeV1(
436 : uint16_t streamId, uint16_t taskId, const void* src, uint32_t length, const aclDataType runtimeDataType,
437 : aclrtReduceKind rtReduceOp, const void* dst, uint32_t partId, uint32_t ssid, uint32_t devId, u64 overflowAddr,
438 : uint8_t linkType, const uint8_t* sqeIn, uint8_t* sqeType, uint32_t hcclQos)
439 : {
440 : (void)ssid;
441 : (void)devId;
442 : (void)overflowAddr;
443 2 : *sqeType = SqeType::CACHE_MEMCPY_RECORD_PLACEHOLDER_SQE;
444 2 : rtStarsPlaceHolderSqe_t* const sqe = (rtStarsPlaceHolderSqe_t* const)sqeIn;
445 :
446 : u32 len, srcAddrLow, srcAddrHigh, dstAddrLow, dstAddrHigh;
447 2 : if (length != 0U || src != nullptr || dst != nullptr) {
448 2 : len = length;
449 2 : srcAddrLow = static_cast<uint32_t>(reinterpret_cast<u64>(src) & 0x00000000ffffffffU);
450 2 : srcAddrHigh = static_cast<uint32_t>((reinterpret_cast<u64>(src) & 0xffffffff00000000U) >> UINT32_BIT_NUM);
451 2 : dstAddrLow = static_cast<uint32_t>(reinterpret_cast<u64>(dst) & 0x00000000ffffffffU);
452 2 : dstAddrHigh = static_cast<uint32_t>((reinterpret_cast<u64>(dst) & 0xffffffff00000000U) >> UINT32_BIT_NUM);
453 : } else {
454 0 : len = sqe->u.cache_memcpy_record_task_info.length;
455 0 : srcAddrLow = sqe->u.cache_memcpy_record_task_info.src_addr_low;
456 0 : srcAddrHigh = sqe->u.cache_memcpy_record_task_info.src_addr_high;
457 0 : dstAddrLow = sqe->u.cache_memcpy_record_task_info.dst_addr_low;
458 0 : dstAddrHigh = sqe->u.cache_memcpy_record_task_info.dst_addr_high;
459 0 : (void)memset_s(sqe, sizeof(rtStarsPlaceHolderSqe_t), 0, sizeof(rtStarsPlaceHolderSqe_t));
460 : }
461 :
462 : // 用于placeholder
463 2 : SetCachePlaceholderHeaderV1(streamId, taskId, sqeIn);
464 :
465 : // 保存memcpy-record SQE的相关信息
466 2 : sqe->u.cache_memcpy_record_task_info.kernel_credit = dfx::kCreditTimeDefault;
467 2 : const bool isReduce
468 0 : = ((rtReduceOp == ACL_RT_MEMCPY_SDMA_AUTOMATIC_SUM) || (rtReduceOp == ACL_RT_MEMCPY_SDMA_AUTOMATIC_MAX)
469 2 : || (rtReduceOp == ACL_RT_MEMCPY_SDMA_AUTOMATIC_MIN) || (rtReduceOp == ACL_RT_MEMCPY_SDMA_AUTOMATIC_EQUAL));
470 2 : sqe->u.cache_memcpy_record_task_info.opcode = isReduce ? GetOpcodeForReduce(rtReduceOp, runtimeDataType) : 0U;
471 2 : HCCL_INFO(
472 : "[SQE] cache-write-memcpy placeholder: copyKind=%u,Opcode=0x%x, streamId=%u, len=%u, src:%p, dst:%p, hcclQos: "
473 : "%u",
474 : static_cast<uint32_t>(rtReduceOp), static_cast<uint32_t>(sqe->u.cache_memcpy_record_task_info.opcode), streamId,
475 : length, src, dst, hcclQos);
476 2 : sqe->u.cache_memcpy_record_task_info.length = len;
477 2 : sqe->u.cache_memcpy_record_task_info.src_addr_low = srcAddrLow;
478 2 : sqe->u.cache_memcpy_record_task_info.src_addr_high = srcAddrHigh;
479 2 : sqe->u.cache_memcpy_record_task_info.dst_addr_low = dstAddrLow;
480 2 : sqe->u.cache_memcpy_record_task_info.dst_addr_high = dstAddrHigh;
481 2 : sqe->u.cache_memcpy_record_task_info.partid = partId;
482 2 : sqe->u.cache_memcpy_record_task_info.linkType = linkType;
483 :
484 : // 适配qos
485 2 : if (linkType == static_cast<uint8_t>(hccl::LinkType::LINK_SIO)
486 1 : || linkType == static_cast<uint8_t>(hccl::LinkType::LINK_ONCHIP)) {
487 2 : hcclQos = SDMA_QOS_DEFAULT;
488 : }
489 2 : HCCL_INFO(
490 : "[AddOneCacheMemcpyRecordPlaceholderSqeV1] sqe->linkType=%u hcclQos=%u", static_cast<unsigned int>(linkType),
491 : static_cast<unsigned int>(hcclQos));
492 2 : sqe->u.cache_memcpy_record_task_info.qos = hcclQos;
493 2 : }
494 :
495 8 : void SetCachePlaceholderHeaderV1(uint16_t streamId, uint16_t taskId, const uint8_t* sqeIn)
496 : {
497 8 : rtStarsPlaceHolderSqe_t* const sqe = (rtStarsPlaceHolderSqe_t* const)sqeIn;
498 8 : sqe->header.type = RT_STARS_SQE_TYPE_PLACE_HOLDER;
499 8 : sqe->header.ie = 0U;
500 8 : sqe->header.preP = 0U; // 不需要STARS_FW参与任何预处理
501 8 : sqe->header.postP = 0U;
502 8 : sqe->header.wrCqe = 0U;
503 8 : sqe->header.reserved = 0U;
504 : // NOTE: task type在preP阶段被TASK_FW使用, 而此placeholder无preP阶段, 设置为RT_TASK_TYPE_FLIP不影响功能
505 8 : sqe->header.blockDim = RT_TASK_TYPE_FLIP;
506 8 : sqe->header.rtStreamId = streamId;
507 8 : sqe->header.taskId = taskId;
508 8 : sqe->kernel_credit = RT_STARS_DEFAULT_KERNEL_CREDIT;
509 8 : return;
510 : }
511 :
512 3621 : std::pair<uint64_t, uint64_t> GetTimeOutValue(const dfx::DfxTimeOutConfig& dfxTimeOutConfig)
513 : {
514 3621 : if (dfxTimeOutConfig.useCredit) {
515 3614 : HCCL_DEBUG("Use hard sync with [%lu s]", dfxTimeOutConfig.sqeCreditTimeOut);
516 3614 : return {dfxTimeOutConfig.sqeCreditTimeOut, dfx::kTimeOutTimeInvalid};
517 : }
518 7 : HCCL_DEBUG("Use soft sync with [%lu s]", dfxTimeOutConfig.sqeTimeOutTimeOut);
519 7 : return {dfx::kCreditTimeInvalid, dfxTimeOutConfig.sqeTimeOutTimeOut};
520 : }
|
