Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 : #include "aicpu/aicpu_hccl_sqcqv1.h"
11 : #include "hccl_common.h"
12 : #include "dispatcher_task_types.h"
13 : #include <unordered_map>
14 :
15 : namespace {
16 14 : bool ChipIsHaveStars() { return true; }
17 :
18 165 : uint8_t ReduceOpcodeHigh(uint8_t copyDataType)
19 : {
20 : uint8_t opcode;
21 165 : switch (copyDataType) {
22 1 : case ACL_INT8: {
23 1 : opcode = static_cast<uint8_t>(RT_STARS_MEMCPY_ASYNC_DATA_TYPE_INT8);
24 1 : break;
25 : }
26 0 : case ACL_INT16: {
27 0 : opcode = static_cast<uint8_t>(RT_STARS_MEMCPY_ASYNC_DATA_TYPE_INT16);
28 0 : break;
29 : }
30 0 : case ACL_INT32: {
31 0 : opcode = static_cast<uint8_t>(RT_STARS_MEMCPY_ASYNC_DATA_TYPE_INT32);
32 0 : break;
33 : }
34 148 : case ACL_FLOAT16: {
35 148 : opcode = static_cast<uint8_t>(RT_STARS_MEMCPY_ASYNC_DATA_TYPE_FP16);
36 148 : break;
37 : }
38 1 : case ACL_FLOAT: {
39 1 : opcode = static_cast<uint8_t>(RT_STARS_MEMCPY_ASYNC_DATA_TYPE_FP32);
40 1 : break;
41 : }
42 14 : case ACL_BF16: {
43 14 : if (ChipIsHaveStars()) {
44 14 : opcode = static_cast<uint8_t>(RT_STARS_MEMCPY_ASYNC_DATA_TYPE_BFP16);
45 : } else {
46 0 : HCCL_ERROR("DataType=%u do not support.", static_cast<uint32_t>(copyDataType));
47 0 : opcode = static_cast<uint8_t>(RT_STARS_MEMCPY_ASYNC_OP_RESERVED);
48 : }
49 14 : break;
50 : }
51 1 : default: {
52 : // Should not run here.
53 : // if not support, it will return RT_ERROR_FEATURE_NOT_SUPPORT at context.cc's reduce ability check.
54 : // Only for code style, 0x80 is reserved value of STRAS opcode.
55 1 : HCCL_ERROR("DataType=%u do not support.", static_cast<uint32_t>(copyDataType));
56 1 : opcode = static_cast<uint8_t>(RT_STARS_MEMCPY_ASYNC_OP_RESERVED);
57 1 : break;
58 : }
59 : }
60 165 : return opcode;
61 : }
62 :
63 165 : uint8_t ReduceOpcodeLow(uint32_t copyKind)
64 : {
65 : uint8_t opcode;
66 165 : switch (copyKind) {
67 165 : case ACL_RT_MEMCPY_SDMA_AUTOMATIC_SUM: {
68 165 : opcode = static_cast<uint8_t>(RT_STARS_MEMCPY_ASYNC_OP_KIND_ADD);
69 165 : break;
70 : }
71 0 : case ACL_RT_MEMCPY_SDMA_AUTOMATIC_MAX: {
72 0 : opcode = static_cast<uint8_t>(RT_STARS_MEMCPY_ASYNC_OP_KIND_MAX);
73 0 : break;
74 : }
75 0 : case ACL_RT_MEMCPY_SDMA_AUTOMATIC_MIN: {
76 0 : opcode = static_cast<uint8_t>(RT_STARS_MEMCPY_ASYNC_OP_KIND_MIN);
77 0 : break;
78 : }
79 0 : case ACL_RT_MEMCPY_SDMA_AUTOMATIC_EQUAL: {
80 0 : opcode = static_cast<uint8_t>(RT_STARS_MEMCPY_ASYNC_OP_KIND_EQUAL);
81 0 : break;
82 : }
83 0 : default: {
84 0 : HCCL_ERROR("Type out of range: copyKind=%u", copyKind);
85 0 : opcode = static_cast<uint8_t>(RT_STARS_MEMCPY_ASYNC_OP_RESERVED);
86 0 : break;
87 : }
88 : }
89 165 : return opcode;
90 : }
91 :
92 165 : uint8_t GetOpcodeForReduce(uint32_t copyKind, uint8_t copyDataType)
93 : {
94 165 : const uint8_t opcodeHigh = ReduceOpcodeHigh(copyDataType);
95 165 : const uint8_t opcodeLow = ReduceOpcodeLow(copyKind);
96 165 : if ((static_cast<int32_t>(opcodeHigh) == RT_STARS_MEMCPY_ASYNC_OP_RESERVED)
97 164 : || (static_cast<int32_t>(opcodeLow) == RT_STARS_MEMCPY_ASYNC_OP_RESERVED)) {
98 : // Should not run here. 0x80 is reserved value of STRAS opcode
99 1 : return static_cast<uint8_t>(RT_STARS_MEMCPY_ASYNC_OP_RESERVED);
100 : } else {
101 164 : return opcodeHigh | opcodeLow;
102 : }
103 : }
104 :
105 : std::unordered_map<uint8_t, uint8_t> RT2HCCL_REDUCE_OP_MAP = {
106 : {static_cast<uint8_t>(RT_STARS_MEMCPY_ASYNC_OP_KIND_CPY), static_cast<uint8_t>(HCCL_REDUCE_RESERVED)},
107 : {static_cast<uint8_t>(RT_STARS_MEMCPY_ASYNC_OP_KIND_ADD), static_cast<uint8_t>(HCCL_REDUCE_SUM)},
108 : {static_cast<uint8_t>(RT_STARS_MEMCPY_ASYNC_OP_KIND_MAX), static_cast<uint8_t>(HCCL_REDUCE_MAX)},
109 : {static_cast<uint8_t>(RT_STARS_MEMCPY_ASYNC_OP_KIND_MIN), static_cast<uint8_t>(HCCL_REDUCE_MIN)},
110 : {static_cast<uint8_t>(RT_STARS_MEMCPY_ASYNC_OP_KIND_EQUAL), static_cast<uint8_t>(HCCL_REDUCE_RESERVED)},
111 : };
112 :
113 : } // namespace
114 :
115 5 : void TranslateOpcode(uint8_t opCode, uint8_t& reduceType)
116 : {
117 5 : reduceType = RT2HCCL_REDUCE_OP_MAP[opCode & 0x0F]; // opCode的低4位表示reduce类型
118 5 : HCCL_DEBUG("[TranslateOpcode] opCode=%u, reduceType=%u.", opCode, reduceType);
119 5 : }
120 :
121 3618 : void AddOneNotifyWaitSqeV1(
122 : uint16_t streamId, uint16_t taskId, u64 notifyId, const uint8_t* sqeIn, uint8_t* sqeType,
123 : const dfx::DfxTimeOutConfig& dfxTimeOutConfig)
124 : {
125 3618 : *sqeType = SqeType::NOTIFY_SQE;
126 3618 : rtStarsNotifySqeV1_t* const sqe = (rtStarsNotifySqeV1_t* const)sqeIn;
127 3618 : sqe->header.type = RT_STARS_SQE_TYPE_NOTIFY_WAIT;
128 3618 : const auto& credit_and_time_out = GetTimeOutValue(dfxTimeOutConfig);
129 3618 : sqe->kernel_credit = static_cast<uint8_t>(credit_and_time_out.first);
130 3618 : sqe->timeout = credit_and_time_out.second;
131 3618 : sqe->header.rtStreamId = streamId;
132 3618 : sqe->notify_id = notifyId;
133 3618 : sqe->header.taskId = taskId;
134 3618 : HCCL_INFO(
135 : "[SQE] notify wait: notifyId=%lu, streamId=%u, taskId=%u, "
136 : "kernel_credit %u, timeout %u s.",
137 : notifyId, streamId, taskId, sqe->kernel_credit, sqe->timeout);
138 3618 : }
139 :
140 2081 : void AddOneRecordSqeV1(uint16_t streamId, uint16_t taskId, u64 notifyId, const uint8_t* sqeIn, uint8_t* sqeType)
141 : {
142 2081 : *sqeType = SqeType::NOTIFY_SQE;
143 2081 : rtStarsNotifySqeV1_t* const sqe = (rtStarsNotifySqeV1_t* const)sqeIn;
144 2081 : sqe->header.type = RT_STARS_SQE_TYPE_NOTIFY_RECORD;
145 2081 : sqe->kernel_credit = RT_STARS_DEFAULT_KERNEL_CREDIT;
146 2081 : sqe->header.rtStreamId = streamId;
147 2081 : sqe->notify_id = notifyId;
148 2081 : sqe->header.taskId = taskId;
149 2081 : HCCL_INFO(
150 : "[SQE] notify record: notifyId=%lu, streamId=%u, taskId=%u, "
151 : "kernel_credit %u, timeout %u s.",
152 : notifyId, streamId, taskId, sqe->kernel_credit, sqe->timeout);
153 2081 : }
154 :
155 1586 : void AddOneWriteValueRecordSqeV1(
156 : uint16_t streamId, uint16_t taskId, u64 notifyWRAddr, const uint8_t* sqeIn, uint8_t* sqeType)
157 : {
158 1586 : *sqeType = SqeType::WRITE_VALUE_SQE;
159 1586 : rtStarsWriteValueSqe_t* const sqe = (rtStarsWriteValueSqe_t* const)sqeIn;
160 1586 : sqe->header.type = RT_STARS_SQE_TYPE_WRITE_VALUE;
161 1586 : sqe->header.rtStreamId = streamId;
162 1586 : sqe->header.taskId = taskId;
163 1586 : sqe->kernel_credit = RT_STARS_DEFAULT_KERNEL_CREDIT;
164 1586 : sqe->awsize = RT_STARS_WRITE_VALUE_SIZE_TYPE_32BIT;
165 1586 : sqe->write_value_part0 = 1U;
166 1586 : sqe->sub_type = RT_STARS_WRITE_VALUE_SUB_TYPE_NOTIFY_RECORD_IPC_NO_PCIE;
167 1586 : sqe->write_addr_low = static_cast<uint32_t>(notifyWRAddr & MASK_32_BIT);
168 1586 : sqe->write_addr_high = static_cast<uint32_t>((notifyWRAddr >> UINT32_BIT_NUM) & MASK_17_BIT);
169 1586 : HCCL_INFO("[SQE] write value: writePtr=0x%lx, streamId=%u, taskId=%u.", notifyWRAddr, streamId, taskId);
170 1586 : }
171 :
172 860 : void AddOneMemcpySqeV1(
173 : uint16_t streamId, uint16_t taskId, const void* src, uint32_t length, const aclDataType runtimeDataType,
174 : aclrtReduceKind rtReduceOp, const void* dst, uint32_t partId, uint32_t ssid, uint32_t devId, u64 overflowAddr,
175 : uint8_t linkType, const uint8_t* sqeIn, uint8_t* sqeType, uint32_t hcclQos)
176 : {
177 : (void)ssid;
178 : (void)devId;
179 : (void)overflowAddr;
180 860 : *sqeType = SqeType::MEMCPY_ASYNC_SQE;
181 860 : rtStarsMemcpyAsyncSqe_t* const sqe = (rtStarsMemcpyAsyncSqe_t* const)sqeIn;
182 :
183 : u32 len, srcAddrLow, srcAddrHigh, dstAddrLow, dstAddrHigh;
184 860 : if (length != 0U || src != nullptr || dst != nullptr) {
185 859 : len = length;
186 859 : srcAddrLow = static_cast<uint32_t>(reinterpret_cast<u64>(src) & 0x00000000ffffffffU);
187 859 : srcAddrHigh = static_cast<uint32_t>((reinterpret_cast<u64>(src) & 0xffffffff00000000U) >> UINT32_BIT_NUM);
188 859 : dstAddrLow = static_cast<uint32_t>(reinterpret_cast<u64>(dst) & 0x00000000ffffffffU);
189 859 : dstAddrHigh = static_cast<uint32_t>((reinterpret_cast<u64>(dst) & 0xffffffff00000000U) >> UINT32_BIT_NUM);
190 : } else {
191 1 : len = sqe->length;
192 1 : srcAddrLow = sqe->src_addr_low;
193 1 : srcAddrHigh = sqe->src_addr_high;
194 1 : dstAddrLow = sqe->dst_addr_low;
195 1 : dstAddrHigh = sqe->dst_addr_high;
196 1 : (void)memset_s(sqe, sizeof(rtStarsMemcpyAsyncSqe_t), 0, sizeof(rtStarsMemcpyAsyncSqe_t));
197 : }
198 :
199 860 : sqe->header.type = RT_STARS_SQE_TYPE_SDMA;
200 860 : sqe->header.rtStreamId = streamId;
201 860 : sqe->header.taskId = taskId;
202 860 : sqe->kernel_credit = dfx::kCreditTimeDefault;
203 860 : const bool isReduce
204 697 : = ((rtReduceOp == ACL_RT_MEMCPY_SDMA_AUTOMATIC_SUM) || (rtReduceOp == ACL_RT_MEMCPY_SDMA_AUTOMATIC_MAX)
205 1557 : || (rtReduceOp == ACL_RT_MEMCPY_SDMA_AUTOMATIC_MIN) || (rtReduceOp == ACL_RT_MEMCPY_SDMA_AUTOMATIC_EQUAL));
206 :
207 860 : sqe->opcode = isReduce ? GetOpcodeForReduce(rtReduceOp, runtimeDataType) : 0U;
208 860 : if (linkType == static_cast<uint8_t>(hccl::LinkType::LINK_SIO)
209 859 : || linkType == static_cast<uint8_t>(hccl::LinkType::LINK_ONCHIP)) {
210 4 : hcclQos = SDMA_QOS_DEFAULT;
211 : }
212 860 : HCCL_INFO(
213 : "[SQE]MemcpySqe copyKind=%u,Opcode=0x%x, streamId=%u, len=%u, src:%p, dst:%p, sqe->linkType=%u, hcclQos=%u",
214 : static_cast<uint32_t>(rtReduceOp), static_cast<uint32_t>(sqe->opcode), streamId, length, src, dst,
215 : static_cast<unsigned int>(linkType), hcclQos);
216 :
217 860 : sqe->length = len;
218 860 : sqe->src_addr_low = srcAddrLow;
219 860 : sqe->src_addr_high = srcAddrHigh;
220 860 : sqe->dst_addr_low = dstAddrLow;
221 860 : sqe->dst_addr_high = dstAddrHigh;
222 860 : sqe->sssv = 1U;
223 860 : sqe->dssv = 1U;
224 860 : sqe->sns = 1U;
225 860 : sqe->dns = 1U;
226 860 : sqe->qos = hcclQos;
227 860 : sqe->partid = partId;
228 860 : sqe->linkType = linkType;
229 860 : }
230 :
231 9 : void AddOneRdmaDbSendSqeV1(
232 : uint16_t streamId, uint16_t taskId, uint64_t dbInfo, uint64_t dbAddr, uint32_t length, uint8_t rdmaType,
233 : const uint8_t* sqeIn, uint8_t* sqeType)
234 : {
235 9 : *sqeType = SqeType::RDMA_DB_SEND_SQE;
236 9 : rtStarsWriteValueSqe_t* const sqe = (rtStarsWriteValueSqe_t* const)sqeIn;
237 :
238 9 : sqe->header.type = RT_STARS_SQE_TYPE_WRITE_VALUE;
239 9 : sqe->header.ie = RT_STARS_SQE_INT_DIR_NO;
240 9 : sqe->header.preP = RT_STARS_SQE_INT_DIR_NO;
241 9 : sqe->header.postP = RT_STARS_SQE_INT_DIR_NO;
242 9 : sqe->header.wrCqe = 0U;
243 9 : sqe->header.rtStreamId = streamId;
244 9 : sqe->header.taskId = taskId;
245 :
246 9 : sqe->va = 0U;
247 9 : sqe->kernel_credit = RT_STARS_DEFAULT_KERNEL_CREDIT;
248 9 : sqe->awsize = RT_STARS_WRITE_VALUE_SIZE_TYPE_64BIT;
249 :
250 9 : sqe->sub_type = RT_STARS_WRITE_VALUE_SUB_TYPE_RDMA_DB_SEND;
251 :
252 9 : if (dbAddr == 0ULL) {
253 1 : sqe->header.type = RT_STARS_SQE_TYPE_INVALID;
254 1 : return;
255 : }
256 8 : sqe->write_value_part0 = static_cast<uint32_t>(dbInfo & MASK_32_BIT);
257 8 : sqe->write_value_part1 = static_cast<uint32_t>(dbInfo >> UINT32_BIT_NUM);
258 8 : sqe->write_addr_low = static_cast<uint32_t>(dbAddr & MASK_32_BIT);
259 8 : sqe->write_addr_high = static_cast<uint32_t>((dbAddr >> UINT32_BIT_NUM) & MASK_17_BIT);
260 8 : sqe->rdmaWrLenth = length; // wr len
261 8 : sqe->rdmaType = static_cast<uint32_t>(rdmaType);
262 8 : HCCL_INFO(
263 : "[SQE]RdmaDbSend: length=%u, rdmaType=%u, dbAddr=0x%lx, streamId=%u, taskId=%u.", length, rdmaType, dbAddr,
264 : streamId, taskId);
265 : }
266 :
267 3 : void AddOneEventResetSqeV1(
268 : uint16_t streamId, int32_t eventId, uint16_t taskId, int64_t phyChipId, int64_t phyDieId, u64 eventAddr,
269 : const uint8_t* sqeIn, uint8_t* sqeType)
270 : {
271 : (void)eventAddr;
272 3 : *sqeType = SqeType::WRITE_VALUE_SQE;
273 3 : rtStarsWriteValueSqe_t* const sqe = (rtStarsWriteValueSqe_t* const)sqeIn;
274 3 : sqe->header.type = RT_STARS_SQE_TYPE_WRITE_VALUE;
275 :
276 3 : sqe->header.rtStreamId = streamId;
277 3 : sqe->header.taskId = taskId;
278 :
279 3 : sqe->kernel_credit = RT_STARS_DEFAULT_KERNEL_CREDIT;
280 3 : sqe->res7 = static_cast<uint32_t>(eventId);
281 3 : sqe->sub_type = RT_STARS_WRITE_VALUE_SUB_TYPE_EVENT_RESET;
282 :
283 3 : const u64 eventTableId = static_cast<u64>(eventId) / STARS_EVENT_NUM_OF_SINGLE_TABLE;
284 :
285 : /* same as eventid % STARS_EVENT_NUM_OF_SINGLE_TABLE */
286 3 : const u64 eventNum = (static_cast<u64>(eventId)) & 0xFFFUL;
287 : // 默认devType 为DevType::DEV_TYPE_COUNT stream->Device_()->GetPhyChipId() 默认0 stream->Device_()->GetPhyDieId()
288 : // 默认0
289 3 : u64 base = static_cast<u64>(
290 3 : RT_STARS_BASE_ADDR + (RT_CHIP_ADDR_OFFSET * static_cast<u64>(phyChipId))
291 3 : + (RT_DIE_ADDR_OFFSET * static_cast<u64>(phyDieId)) + STARS_EVENT_BASE_ADDR);
292 3 : const u64 addr = base + (eventTableId * STARS_EVENT_TABLE_OFFSET) + (eventNum * STARS_EVENT_OFFSET);
293 3 : sqe->write_addr_low = static_cast<uint32_t>(addr & MASK_32_BIT);
294 3 : sqe->write_addr_high = static_cast<uint32_t>((addr >> UINT32_BIT_NUM) & MASK_17_BIT);
295 3 : HCCL_INFO("[SQE] event_reset: eventId=%u, streamId=%u, taskId=%u, addr:%p", eventId, streamId, taskId, addr);
296 3 : }
297 :
298 1 : void AddOneEventRecordSqeV1(uint16_t streamId, int32_t eventId, uint16_t taskId, const uint8_t* sqeIn, uint8_t* sqeType)
299 : {
300 1 : *sqeType = SqeType::EVENT_SQE;
301 1 : rtStarsEventSqe_t* const sqe = (rtStarsEventSqe_t* const)sqeIn;
302 1 : sqe->header.type = RT_STARS_SQE_TYPE_EVENT_RECORD;
303 1 : sqe->header.wrCqe = 1U; // 1: set wrCqe
304 1 : sqe->kernel_credit = RT_STARS_DEFAULT_KERNEL_CREDIT;
305 :
306 : // eventRecordTaskInfo->waitCqflag 是否为同步task
307 : if (false) {
308 : streamId |= RT_SYNC_TASK_FLAG;
309 : }
310 :
311 1 : sqe->header.rtStreamId = streamId;
312 1 : sqe->eventId = static_cast<uint16_t>(eventId);
313 1 : sqe->header.taskId = taskId;
314 1 : HCCL_INFO("[SQE] event record: eventId=%d, streamId=%u, taskId=%u.", eventId, streamId, taskId);
315 1 : }
316 :
317 3 : void AddOneEventWaitSqeV1(uint16_t streamId, int32_t eventId, uint16_t taskId, const uint8_t* sqeIn, uint8_t* sqeType)
318 : {
319 3 : *sqeType = SqeType::EVENT_SQE;
320 3 : rtStarsEventSqe_t* const sqe = (rtStarsEventSqe_t* const)sqeIn;
321 3 : sqe->header.type = RT_STARS_SQE_TYPE_EVENT_WAIT;
322 3 : sqe->kernel_credit = RT_STARS_NEVER_TIMEOUT_KERNEL_CREDIT;
323 :
324 3 : sqe->header.rtStreamId = streamId;
325 3 : sqe->eventId = eventId;
326 3 : sqe->header.taskId = taskId;
327 3 : HCCL_INFO("[SQE] event wait: eventId=%d, streamId=%u, taskId=%u", eventId, streamId, taskId);
328 3 : }
329 :
330 2 : void AddOneFlipPlaceHolderSqeV1(
331 : uint16_t streamId, uint16_t flipNum, uint16_t taskId, const uint8_t* sqeIn, uint8_t* sqeType)
332 : {
333 2 : *sqeType = SqeType::FLIP_PLACEHOLDER_SQE;
334 2 : rtStarsPlaceHolderSqe_t* const sqe = (rtStarsPlaceHolderSqe_t* const)sqeIn;
335 2 : sqe->header.type = RT_STARS_SQE_TYPE_PLACE_HOLDER;
336 2 : sqe->header.ie = 0U;
337 2 : sqe->header.preP = 1U;
338 2 : sqe->header.postP = 0U;
339 2 : sqe->header.wrCqe = 0U;
340 2 : sqe->header.reserved = 0U;
341 2 : sqe->header.blockDim = RT_TASK_TYPE_FLIP; // task type
342 2 : sqe->header.rtStreamId = streamId;
343 2 : sqe->header.taskId = taskId;
344 2 : sqe->kernel_credit = RT_STARS_DEFAULT_KERNEL_CREDIT;
345 2 : sqe->u.flip_task_info.flipNumReport = flipNum;
346 :
347 2 : HCCL_INFO("[SQE] placeholder: flipNum=%d, streamId=%u, taskId=%u", flipNum, streamId, taskId);
348 2 : }
349 :
350 2 : void AddOneCacheMemcpyPlaceHolderSqeV1(
351 : uint16_t streamId, uint16_t taskId, const void* src, const void* dst, uint8_t linkType, const uint8_t* sqeIn,
352 : uint8_t* sqeType, uint32_t hcclQos)
353 : {
354 2 : *sqeType = SqeType::CACHE_MEMCPY_PLACEHOLDER_SQE;
355 2 : SetCachePlaceholderHeaderV1(streamId, taskId, sqeIn);
356 :
357 2 : rtStarsPlaceHolderSqe_t* const sqe = (rtStarsPlaceHolderSqe_t* const)sqeIn;
358 2 : constexpr uint64_t uintBitWidth = 32;
359 2 : uint64_t srcAddr = reinterpret_cast<uint64_t>(src);
360 2 : uint64_t dstAddr = reinterpret_cast<uint64_t>(dst);
361 2 : sqe->u.cache_memcpy_task_info.src_addr_high = static_cast<uint32_t>(srcAddr >> uintBitWidth);
362 2 : sqe->u.cache_memcpy_task_info.src_addr_low = static_cast<uint32_t>(srcAddr & 0xFFFFFFFFULL);
363 2 : sqe->u.cache_memcpy_task_info.dst_addr_high = static_cast<uint32_t>(dstAddr >> uintBitWidth);
364 2 : sqe->u.cache_memcpy_task_info.dst_addr_low = static_cast<uint32_t>(dstAddr & 0xFFFFFFFFULL);
365 2 : sqe->u.cache_memcpy_task_info.kernel_credit = dfx::kCreditTimeDefault;
366 2 : sqe->u.cache_memcpy_task_info.linkType = linkType;
367 :
368 2 : HCCL_INFO(
369 : "[SQE] cache-memcpy placeholder: streamId=%u, taskId=%u, srcAddr=0x%016llx, dstAddr=0x%016llx, linkType=%u, "
370 : "hcclQos=%u",
371 : streamId, taskId, src, dst, linkType, hcclQos);
372 :
373 : // 适配QoS
374 2 : if (linkType == static_cast<uint8_t>(hccl::LinkType::LINK_SIO)
375 1 : || linkType == static_cast<uint8_t>(hccl::LinkType::LINK_ONCHIP)) {
376 2 : hcclQos = SDMA_QOS_DEFAULT;
377 : }
378 2 : HCCL_INFO(
379 : "[AddOneCacheMemcpyPlaceHolderSqeV1] sqe->linkType=%u hcclQos=%u", static_cast<unsigned int>(linkType),
380 : static_cast<unsigned int>(hcclQos));
381 2 : sqe->u.cache_memcpy_task_info.qos = hcclQos;
382 2 : }
383 :
384 1 : void AddOneCacheNotifyWaitPlaceholderSqeV1(
385 : uint16_t streamId, uint16_t taskId, u64 notifyId, const uint8_t* sqeIn, uint8_t* sqeType,
386 : const dfx::DfxTimeOutConfig& dfxTimeOutConfig)
387 : {
388 1 : *sqeType = SqeType::CACHE_NOTIFY_PLACEHOLDER_SQE;
389 1 : SetCachePlaceholderHeaderV1(streamId, taskId, sqeIn);
390 :
391 1 : rtStarsPlaceHolderSqe_t* const sqe = (rtStarsPlaceHolderSqe_t* const)sqeIn;
392 1 : sqe->u.cache_notify_task_info.is_wait = 1; // NotifyWait
393 1 : const auto& credit_and_time_out = GetTimeOutValue(dfxTimeOutConfig);
394 1 : sqe->u.cache_notify_task_info.kernel_credit = static_cast<uint8_t>(credit_and_time_out.first);
395 1 : sqe->u.cache_notify_task_info.timeout = credit_and_time_out.second;
396 1 : sqe->u.cache_notify_task_info.notify_id = notifyId;
397 :
398 1 : HCCL_INFO(
399 : "[SQE] cache-notify placeholder (wait): notifyId=%lu, streamId=%u, taskId=%u, u.kernel_credit %u, timeout %u "
400 : "s.",
401 : notifyId, streamId, taskId, sqe->u.cache_notify_task_info.kernel_credit, sqe->u.cache_notify_task_info.timeout);
402 1 : }
403 :
404 1 : void AddOneCacheNotifyRecordPlaceholderSqeV1(
405 : uint16_t streamId, uint16_t taskId, u64 notifyId, const uint8_t* sqeIn, uint8_t* sqeType)
406 : {
407 1 : *sqeType = SqeType::CACHE_NOTIFY_PLACEHOLDER_SQE;
408 1 : SetCachePlaceholderHeaderV1(streamId, taskId, sqeIn);
409 :
410 1 : rtStarsPlaceHolderSqe_t* const sqe = (rtStarsPlaceHolderSqe_t* const)sqeIn;
411 1 : sqe->u.cache_notify_task_info.is_wait = 0; // NotifyRecord
412 1 : sqe->u.cache_notify_task_info.notify_id = notifyId;
413 :
414 1 : HCCL_INFO(
415 : "[SQE] cache-notify placeholder (record): notifyId=%lu, streamId=%u, taskId=%u, "
416 : "kernel_credit %u.",
417 : notifyId, streamId, taskId, sqe->kernel_credit);
418 1 : }
419 :
420 1 : void AddOneCacheWriteValuePlaceholderSqeV1(
421 : uint16_t streamId, uint16_t taskId, u64 notifyWRAddr, const uint8_t* sqeIn, uint8_t* sqeType)
422 : {
423 1 : *sqeType = SqeType::CACHE_WRITE_VALUE_PLACEHOLDER_SQE;
424 1 : SetCachePlaceholderHeaderV1(streamId, taskId, sqeIn);
425 :
426 1 : rtStarsPlaceHolderSqe_t* const sqe = (rtStarsPlaceHolderSqe_t* const)sqeIn;
427 1 : sqe->u.cache_write_value_task_info.write_addr_low = static_cast<uint32_t>(notifyWRAddr & MASK_32_BIT);
428 : sqe->u.cache_write_value_task_info.write_addr_high
429 1 : = static_cast<uint32_t>((notifyWRAddr >> UINT32_BIT_NUM) & MASK_17_BIT);
430 :
431 1 : HCCL_INFO("[SQE] cache-write placeholder: writePtr=0x%lx, streamId=%u, taskId=%u.", notifyWRAddr, streamId, taskId);
432 1 : }
433 :
434 2 : void AddOneCacheMemcpyRecordPlaceholderSqeV1(
435 : uint16_t streamId, uint16_t taskId, const void* src, uint32_t length, const aclDataType runtimeDataType,
436 : aclrtReduceKind rtReduceOp, const void* dst, uint32_t partId, uint32_t ssid, uint32_t devId, u64 overflowAddr,
437 : uint8_t linkType, const uint8_t* sqeIn, uint8_t* sqeType, uint32_t hcclQos)
438 : {
439 : (void)ssid;
440 : (void)devId;
441 : (void)overflowAddr;
442 2 : *sqeType = SqeType::CACHE_MEMCPY_RECORD_PLACEHOLDER_SQE;
443 2 : rtStarsPlaceHolderSqe_t* const sqe = (rtStarsPlaceHolderSqe_t* const)sqeIn;
444 :
445 : u32 len, srcAddrLow, srcAddrHigh, dstAddrLow, dstAddrHigh;
446 2 : if (length != 0U || src != nullptr || dst != nullptr) {
447 2 : len = length;
448 2 : srcAddrLow = static_cast<uint32_t>(reinterpret_cast<u64>(src) & 0x00000000ffffffffU);
449 2 : srcAddrHigh = static_cast<uint32_t>((reinterpret_cast<u64>(src) & 0xffffffff00000000U) >> UINT32_BIT_NUM);
450 2 : dstAddrLow = static_cast<uint32_t>(reinterpret_cast<u64>(dst) & 0x00000000ffffffffU);
451 2 : dstAddrHigh = static_cast<uint32_t>((reinterpret_cast<u64>(dst) & 0xffffffff00000000U) >> UINT32_BIT_NUM);
452 : } else {
453 0 : len = sqe->u.cache_memcpy_record_task_info.length;
454 0 : srcAddrLow = sqe->u.cache_memcpy_record_task_info.src_addr_low;
455 0 : srcAddrHigh = sqe->u.cache_memcpy_record_task_info.src_addr_high;
456 0 : dstAddrLow = sqe->u.cache_memcpy_record_task_info.dst_addr_low;
457 0 : dstAddrHigh = sqe->u.cache_memcpy_record_task_info.dst_addr_high;
458 0 : (void)memset_s(sqe, sizeof(rtStarsPlaceHolderSqe_t), 0, sizeof(rtStarsPlaceHolderSqe_t));
459 : }
460 :
461 : // 用于placeholder
462 2 : SetCachePlaceholderHeaderV1(streamId, taskId, sqeIn);
463 :
464 : // 保存memcpy-record SQE的相关信息
465 2 : sqe->u.cache_memcpy_record_task_info.kernel_credit = dfx::kCreditTimeDefault;
466 2 : const bool isReduce
467 0 : = ((rtReduceOp == ACL_RT_MEMCPY_SDMA_AUTOMATIC_SUM) || (rtReduceOp == ACL_RT_MEMCPY_SDMA_AUTOMATIC_MAX)
468 2 : || (rtReduceOp == ACL_RT_MEMCPY_SDMA_AUTOMATIC_MIN) || (rtReduceOp == ACL_RT_MEMCPY_SDMA_AUTOMATIC_EQUAL));
469 2 : sqe->u.cache_memcpy_record_task_info.opcode = isReduce ? GetOpcodeForReduce(rtReduceOp, runtimeDataType) : 0U;
470 2 : HCCL_INFO(
471 : "[SQE] cache-write-memcpy placeholder: copyKind=%u,Opcode=0x%x, streamId=%u, len=%u, src:%p, dst:%p, hcclQos: "
472 : "%u",
473 : static_cast<uint32_t>(rtReduceOp), static_cast<uint32_t>(sqe->u.cache_memcpy_record_task_info.opcode), streamId,
474 : length, src, dst, hcclQos);
475 2 : sqe->u.cache_memcpy_record_task_info.length = len;
476 2 : sqe->u.cache_memcpy_record_task_info.src_addr_low = srcAddrLow;
477 2 : sqe->u.cache_memcpy_record_task_info.src_addr_high = srcAddrHigh;
478 2 : sqe->u.cache_memcpy_record_task_info.dst_addr_low = dstAddrLow;
479 2 : sqe->u.cache_memcpy_record_task_info.dst_addr_high = dstAddrHigh;
480 2 : sqe->u.cache_memcpy_record_task_info.partid = partId;
481 2 : sqe->u.cache_memcpy_record_task_info.linkType = linkType;
482 :
483 : // 适配qos
484 2 : if (linkType == static_cast<uint8_t>(hccl::LinkType::LINK_SIO)
485 1 : || linkType == static_cast<uint8_t>(hccl::LinkType::LINK_ONCHIP)) {
486 2 : hcclQos = SDMA_QOS_DEFAULT;
487 : }
488 2 : HCCL_INFO(
489 : "[AddOneCacheMemcpyRecordPlaceholderSqeV1] sqe->linkType=%u hcclQos=%u", static_cast<unsigned int>(linkType),
490 : static_cast<unsigned int>(hcclQos));
491 2 : sqe->u.cache_memcpy_record_task_info.qos = hcclQos;
492 2 : }
493 :
494 8 : void SetCachePlaceholderHeaderV1(uint16_t streamId, uint16_t taskId, const uint8_t* sqeIn)
495 : {
496 8 : rtStarsPlaceHolderSqe_t* const sqe = (rtStarsPlaceHolderSqe_t* const)sqeIn;
497 8 : sqe->header.type = RT_STARS_SQE_TYPE_PLACE_HOLDER;
498 8 : sqe->header.ie = 0U;
499 8 : sqe->header.preP = 0U; // 不需要STARS_FW参与任何预处理
500 8 : sqe->header.postP = 0U;
501 8 : sqe->header.wrCqe = 0U;
502 8 : sqe->header.reserved = 0U;
503 : // NOTE: task type在preP阶段被TASK_FW使用, 而此placeholder无preP阶段, 设置为RT_TASK_TYPE_FLIP不影响功能
504 8 : sqe->header.blockDim = RT_TASK_TYPE_FLIP;
505 8 : sqe->header.rtStreamId = streamId;
506 8 : sqe->header.taskId = taskId;
507 8 : sqe->kernel_credit = RT_STARS_DEFAULT_KERNEL_CREDIT;
508 8 : return;
509 : }
510 :
511 3621 : std::pair<uint64_t, uint64_t> GetTimeOutValue(const dfx::DfxTimeOutConfig& dfxTimeOutConfig)
512 : {
513 3621 : if (dfxTimeOutConfig.useCredit) {
514 3614 : HCCL_DEBUG("Use hard sync with [%lu s]", dfxTimeOutConfig.sqeCreditTimeOut);
515 3614 : return {dfxTimeOutConfig.sqeCreditTimeOut, dfx::kTimeOutTimeInvalid};
516 : }
517 7 : HCCL_DEBUG("Use soft sync with [%lu s]", dfxTimeOutConfig.sqeTimeOutTimeOut);
518 7 : return {dfx::kCreditTimeInvalid, dfxTimeOutConfig.sqeTimeOutTimeOut};
519 : }
|
