Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "aicpu_sqe_context.h"
12 :
13 : #include <sstream>
14 : #include <unordered_map>
15 : #include "common/aicpu_hccl_common.h"
16 : #include "utils/hccl_aicpu_utils.h"
17 : #include "common/aicpu_kfc_utils.h"
18 :
19 : struct SqeContextVariable {
20 : int32_t lastClusterId = -1;
21 : SqeLocalRingBuffer* variablePtr = nullptr;
22 : };
23 :
24 : static SqeLocalRingBuffer g_ringBuffer[CLUSTER_CNT][AC_MAX_RANK_NUM];
25 : static SqeContext g_sqeContext[CLUSTER_CNT];
26 : static SqeContextVariable g_sqeVariable;
27 :
28 22523 : SqeContext* GetSqeContext() { return &g_sqeContext[HcclAicpuUtils::GetCurClusterId()]; }
29 :
30 298 : void AicpuSqeContext::InitSqeContext()
31 : {
32 894 : for (uint32_t i = 0U; i < CLUSTER_CNT; i++) {
33 596 : SqeContext* context = &g_sqeContext[i];
34 596 : context->buffPtr = g_ringBuffer[i];
35 596 : (void)memset_s(
36 596 : context->buffPtr, sizeof(SqeLocalRingBuffer[AC_MAX_RANK_NUM]), 0,
37 : sizeof(SqeLocalRingBuffer[AC_MAX_RANK_NUM]));
38 596 : context->clusterId = i;
39 : }
40 298 : }
41 :
42 66 : void AicpuSqeContext::SyncVariable()
43 : {
44 66 : SqeContext* context = GetSqeContext();
45 66 : HCCL_DEBUG(
46 : "SyncCtxVariable, cur clusterId %d, last ClusterId %d, buffPtr %p", context->clusterId,
47 : g_sqeVariable.lastClusterId, g_sqeVariable.variablePtr);
48 66 : if (context->clusterId == g_sqeVariable.lastClusterId) {
49 64 : return;
50 : }
51 2 : if (g_sqeVariable.lastClusterId < 0 || g_sqeVariable.lastClusterId >= CLUSTER_CNT) {
52 2 : HCCL_DEBUG("SyncCtxVariable, invalid lastClusterId = %d", g_sqeVariable.lastClusterId);
53 2 : return;
54 : }
55 0 : context->buffPtr = g_sqeVariable.variablePtr;
56 : }
57 :
58 45 : void AicpuSqeContext::SaveVariable()
59 : {
60 45 : SqeContext* context = GetSqeContext();
61 45 : HCCL_DEBUG("Save sqe context variable, cur clusterId=%d, buffPtr=%p", context->clusterId, context->buffPtr);
62 45 : g_sqeVariable.lastClusterId = context->clusterId;
63 45 : g_sqeVariable.variablePtr = context->buffPtr;
64 45 : }
65 :
66 8481 : HcclResult AicpuSqeContext::GetNextSqeBufferAddr(
67 : uint32_t streamId, uint8_t*& sqeBufferAddr, uint8_t*& sqeTypeAddr, uint16_t& taskId)
68 : {
69 8481 : CHK_PRT_RET(
70 : (streamId >= AC_MAX_RANK_NUM),
71 : HCCL_ERROR("[AicpuSqeContext][GetNextSqeBufferAddr]Invalid streamId[%u] >= %u", streamId, AC_MAX_RANK_NUM),
72 : HCCL_E_PARA);
73 8481 : SqeContext* context = GetSqeContext();
74 8481 : CHK_PTR_NULL(context->buffPtr);
75 8481 : auto& buff = context->buffPtr[streamId];
76 8481 : if (buff.tailSqeIdx >= AC_SQE_MAX_CNT) {
77 1 : HCCL_WARNING("Sqe cnt is overflow, need revise buff content, current streamid: %u", streamId);
78 1 : HCCL_INFO(
79 : "buffer modify before ==> sqTail: %u, sqHead: %u, sqeCnt: %u, tailSqeTaskId: %u, tailSqeIdx: %u",
80 : buff.sqTail, buff.sqHead, buff.sqeCnt, buff.tailSqeTaskId, buff.tailSqeIdx);
81 1 : CHK_RET(AicpuKfcUtils::TraceProfSubmit());
82 1 : CHK_RET(AicpuSqeContext::ModifyBuffer(streamId));
83 1 : HCCL_INFO(
84 : "buffer modify after ==> sqTail: %u, sqHead: %u, sqeCnt: %u, tailSqeTaskId: %u, tailSqeIdx: %u",
85 : buff.sqTail, buff.sqHead, buff.sqeCnt, buff.tailSqeTaskId, buff.tailSqeIdx);
86 : }
87 : // nextTaskId=0的时候下发PlaceHolder
88 8481 : if (UNLIKELY(buff.tailSqeTaskId == 0 && buff.filpNum != 0)) {
89 1 : CHK_RET(AddFlipTask(streamId));
90 : }
91 :
92 8481 : buff.profTimestap[buff.tailSqeIdx] = GetCurCpuTimestamp(true);
93 8481 : sqeBufferAddr = buff.localBuff + buff.tailSqeIdx * AC_SQE_SIZE;
94 8481 : sqeTypeAddr = &buff.sqeType[buff.tailSqeIdx];
95 8481 : taskId = buff.tailSqeTaskId;
96 8481 : HCCL_DEBUG(
97 : "Get stream:%u next idx:%u, taskId:%u, clusterId:%u", streamId, buff.tailSqeIdx, taskId, context->clusterId);
98 8481 : if (buff.tailSqeTaskId == UINT16_MAX) {
99 1 : buff.filpNum++;
100 1 : HCCL_WARNING("Sqe context cur taskId is uint16_max");
101 : }
102 8481 : buff.tailSqeTaskId++;
103 8481 : buff.tailSqeIdx++;
104 8481 : buff.sqeCnt++;
105 8481 : return HCCL_SUCCESS;
106 : }
107 :
108 1 : HcclResult AicpuSqeContext::AddFlipTask(uint32_t streamId)
109 : {
110 1 : if (!dfx::ProfilingManager::GetProfL0State()) {
111 0 : return HCCL_SUCCESS;
112 : }
113 1 : SqeContext* context = GetSqeContext();
114 1 : CHK_PTR_NULL(context->buffPtr);
115 1 : auto& buff = context->buffPtr[streamId];
116 1 : uint16_t filpNum = buff.filpNum;
117 1 : uint16_t taskId = buff.tailSqeTaskId;
118 1 : auto ctx = AicpuGetComContext();
119 1 : HcclComStreamInfo* streamInfo = &ctx->streamInfo[streamId];
120 1 : CHK_RET(dfx::ProfilingManager::ReportFilpTask(streamInfo->actualStreamId, taskId, filpNum));
121 :
122 1 : buff.profTimestap[buff.tailSqeIdx] = GetCurCpuTimestamp(true);
123 1 : uint8_t* sqeBufferAddr = buff.localBuff + buff.tailSqeIdx * AC_SQE_SIZE;
124 1 : uint8_t* sqeTypeAddr = &buff.sqeType[buff.tailSqeIdx];
125 1 : AicpuAddOneFlipPlaceHolderSqe addOneFlipPlaceHolderSqe = AicpuGetAddOneFlipPlaceHolderSqe();
126 1 : if (addOneFlipPlaceHolderSqe == nullptr) {
127 0 : HCCL_WARNING("AicpuAddOneFlipPlaceHolderSqe is null");
128 0 : return HCCL_SUCCESS;
129 : }
130 1 : addOneFlipPlaceHolderSqe(streamInfo->actualStreamId, filpNum, taskId, sqeBufferAddr, sqeTypeAddr);
131 1 : buff.tailSqeTaskId++;
132 1 : buff.tailSqeIdx++;
133 1 : buff.sqeCnt++;
134 :
135 1 : HCCL_INFO(
136 : "[AicpuSqeContext][AddFlipTask] Call AddFlipTask. para: taskId[%u], streamId[%u], filpNum[%u]]", taskId,
137 : streamInfo->actualStreamId, filpNum);
138 :
139 1 : return HCCL_SUCCESS;
140 : }
141 :
142 8471 : HcclResult AicpuSqeContext::RecordAddInfo(uint32_t streamId, uint32_t addInfo)
143 : {
144 8471 : CHK_PRT_RET(
145 : (streamId >= AC_MAX_RANK_NUM),
146 : HCCL_ERROR("[AicpuSqeContext][RecordAddInfo]Invalid streamId[%u] >= %u", streamId, AC_MAX_RANK_NUM),
147 : HCCL_E_PARA);
148 8471 : SqeContext* context = GetSqeContext();
149 8471 : CHK_PTR_NULL(context->buffPtr);
150 8471 : auto& buff = context->buffPtr[streamId];
151 8471 : CHK_PRT_RET(
152 : ((buff.tailSqeIdx == 0) || (buff.tailSqeIdx > AC_SQE_MAX_CNT)),
153 : HCCL_ERROR("[AicpuSqeContext][RecordAddInfo]Invalid tailSqeIdx[%u]", buff.tailSqeIdx), HCCL_E_PARA);
154 8471 : buff.addInfo[buff.tailSqeIdx - 1] = addInfo;
155 8471 : return HCCL_SUCCESS;
156 : }
157 :
158 69 : HcclResult AicpuSqeContext::QuerySqeInfoByHead(uint32_t streamId, uint32_t sqHead, SqeInfo* info)
159 : {
160 69 : CHK_PRT_RET(
161 : (streamId >= AC_MAX_RANK_NUM),
162 : HCCL_ERROR("[AicpuSqeContext][QuerySqeInfoByHead]Invalid streamId[%u] >= %u", streamId, AC_MAX_RANK_NUM),
163 : HCCL_E_PARA);
164 69 : CHK_PTR_NULL(info);
165 69 : SqeContext* context = GetSqeContext();
166 69 : CHK_PTR_NULL(context->buffPtr);
167 69 : auto& buff = context->buffPtr[streamId];
168 69 : const uint32_t sqDepth = AicpuGetComContext()->streamInfo[streamId].sqDepth;
169 69 : uint32_t sqUnexecuted = (buff.sqTail + sqDepth - sqHead) % sqDepth;
170 69 : if (buff.tailSqeIdx < sqUnexecuted) {
171 3 : HCCL_WARNING("tail sqe idx %u is less then sq unexecuted num %u", buff.tailSqeIdx, sqUnexecuted);
172 3 : return HCCL_E_INTERNAL;
173 : }
174 66 : uint16_t idx = buff.tailSqeIdx - sqUnexecuted;
175 66 : HCCL_INFO("Query streamId:%u, sqeIdx:%u, actual idx:%u, type:%u", streamId, sqHead, idx, buff.sqeType[idx]);
176 66 : info->sqeHeadIdx = sqHead;
177 132 : return SqeContextUtils::QuerySqeInfo(
178 66 : buff.localBuff + idx * AC_SQE_SIZE, buff.sqeType[idx], buff.addInfo[idx], info);
179 : }
180 :
181 2 : HcclResult AicpuSqeContext::QuerySqeInfoByTaskId(uint32_t streamId, uint16_t taskId, SqeInfo* info)
182 : {
183 2 : CHK_PRT_RET(
184 : (streamId >= AC_MAX_RANK_NUM),
185 : HCCL_ERROR("[AicpuSqeContext][QuerySqeInfoByTaskId]Invalid streamId[%u] >= %u", streamId, AC_MAX_RANK_NUM),
186 : HCCL_E_PARA);
187 2 : CHK_PTR_NULL(info);
188 2 : SqeContext* context = GetSqeContext();
189 2 : CHK_PTR_NULL(context->buffPtr);
190 2 : auto& buff = context->buffPtr[streamId];
191 2 : uint16_t tailRemain = buff.tailSqeTaskId - taskId;
192 2 : const uint32_t sqDepth = AicpuGetComContext()->streamInfo[streamId].sqDepth;
193 2 : uint32_t sqHeadIdx = (buff.sqTail + sqDepth - tailRemain) % sqDepth;
194 2 : if (buff.tailSqeIdx < tailRemain) {
195 1 : HCCL_WARNING("tail sqe idx %u is less then tail remain num %u", buff.tailSqeIdx, tailRemain);
196 1 : return HCCL_E_INTERNAL;
197 : }
198 1 : uint16_t idx = buff.tailSqeIdx - tailRemain;
199 1 : HCCL_INFO("Query streamId:%u, sqeIdx:%u, actual idx:%u, type:%u", streamId, sqHeadIdx, idx, buff.sqeType[idx]);
200 1 : info->sqeHeadIdx = sqHeadIdx;
201 2 : return SqeContextUtils::QuerySqeInfo(
202 1 : buff.localBuff + idx * AC_SQE_SIZE, buff.sqeType[idx], buff.addInfo[idx], info);
203 : }
204 :
205 4065 : HcclResult AicpuSqeContext::ClearCurBuff(uint32_t streamid, uint32_t leftBound)
206 : {
207 4065 : CHK_PRT_RET(
208 : (streamid >= AC_MAX_RANK_NUM),
209 : HCCL_ERROR("[AicpuSqeContext][ClearCurBuff]Invalid streamId[%u] >= %u", streamid, AC_MAX_RANK_NUM),
210 : HCCL_E_PARA);
211 4065 : SqeContext* context = GetSqeContext();
212 4065 : auto& buff = context->buffPtr[streamid];
213 4065 : HCCL_INFO(
214 : "leftBound:%u, buff.sqeCnt:%u, buff.sqHead:%u, buff.sqTail:%u, buff.tailSqeIdx:%u, buff.tailSqeTaskId:%u",
215 : leftBound, buff.sqeCnt, buff.sqHead, buff.sqTail, buff.tailSqeIdx, buff.tailSqeTaskId);
216 8130 : if (memset_s(
217 4065 : buff.localBuff + leftBound * AC_SQE_SIZE, sizeof(buff.localBuff) - leftBound * AC_SQE_SIZE, 0,
218 4065 : (buff.tailSqeIdx - leftBound) * AC_SQE_SIZE)
219 4065 : != EOK) {
220 0 : return HCCL_E_MEMORY;
221 : }
222 4065 : if (memset_s(buff.sqeType + leftBound, sizeof(buff.sqeType) - leftBound, 0, buff.tailSqeIdx - leftBound) != EOK) {
223 0 : return HCCL_E_MEMORY;
224 : }
225 4065 : if (memset_s(buff.addInfo + leftBound, sizeof(buff.addInfo) - leftBound, 0, buff.tailSqeIdx - leftBound) != EOK) {
226 0 : return HCCL_E_MEMORY;
227 : }
228 4065 : buff.sqeCnt = 0;
229 4065 : buff.tailSqeIdx = 0;
230 4065 : AicpuGetComContext()->profilingExtendInfo.lastSqeIdxs[streamid] = 0;
231 4065 : return HCCL_SUCCESS;
232 : }
233 :
234 1 : HcclResult AicpuSqeContext::ModifyBuffer(uint32_t streamid)
235 : {
236 1 : CHK_PRT_RET(
237 : (streamid >= AC_MAX_RANK_NUM),
238 : HCCL_ERROR("[AicpuSqeContext][ModifyBuffer]Invalid streamId[%u] >= %u", streamid, AC_MAX_RANK_NUM),
239 : HCCL_E_PARA);
240 : // 未下发的sqe移到前面
241 1 : SqeContext* context = GetSqeContext();
242 1 : auto& buff = context->buffPtr[streamid];
243 1 : uint32_t cnt = buff.sqeCnt;
244 1 : uint32_t leftSrc = buff.tailSqeIdx - buff.sqeCnt;
245 1 : HCCL_DEBUG(
246 : "buff.sqeCnt:%d, buff.sqHead:%u, buff.sqTail:%u, buff.tailSqeIdx:%u, buff.tailSqeTaskId:%u", buff.sqeCnt,
247 : buff.sqHead, buff.sqTail, buff.tailSqeIdx, buff.tailSqeTaskId);
248 1 : if (memmove_s(buff.localBuff, sizeof(buff.localBuff), buff.localBuff + leftSrc * AC_SQE_SIZE, cnt * AC_SQE_SIZE)
249 1 : != EOK) {
250 0 : return HCCL_E_MEMORY;
251 : }
252 1 : if (memmove_s(buff.sqeType, sizeof(buff.sqeType), buff.sqeType + leftSrc, cnt) != EOK) {
253 0 : return HCCL_E_MEMORY;
254 : }
255 1 : if (memmove_s(buff.addInfo, sizeof(buff.addInfo), buff.addInfo + leftSrc, cnt) != EOK) {
256 0 : return HCCL_E_MEMORY;
257 : }
258 : // 队列后面已经拷贝到rtsq上的sqe清除掉
259 1 : CHK_RET(ClearCurBuff(streamid, cnt));
260 : // 更新index和sqeCnt
261 1 : buff.tailSqeIdx = cnt;
262 1 : buff.sqeCnt = cnt;
263 1 : AicpuGetComContext()->profilingExtendInfo.lastSqeIdxs[streamid] = cnt;
264 1 : return HCCL_SUCCESS;
265 : }
266 :
267 127 : HcclResult AicpuSqeContext::ClearLocalBuff()
268 : {
269 4191 : for (uint32_t i = 0; i < AC_MAX_RANK_NUM; i++) {
270 4064 : CHK_RET(ClearCurBuff(i));
271 : }
272 127 : return HCCL_SUCCESS;
273 : }
274 :
275 25 : std::string AicpuSqeContext::GetString(const SqeInfo& sqeInfo)
276 : {
277 25 : std::stringstream ss;
278 25 : ss << "SqeInfo ";
279 25 : ss << "sqeIdx:" << sqeInfo.sqeHeadIdx << ",";
280 25 : ss << "type:" << SqeContextUtils::RtsqTaskTypeToStr(sqeInfo.type) << ",";
281 25 : ss << "subType:" << static_cast<uint16_t>(sqeInfo.subType) << ",";
282 25 : ss << "streamId:" << sqeInfo.streamId << ",";
283 25 : ss << "taskId:" << sqeInfo.taskId << ",";
284 25 : ss << "notifyId:" << sqeInfo.notifyId << ",";
285 25 : ss << "eventId:" << sqeInfo.eventId << ",";
286 25 : ss << "partId:" << sqeInfo.partId << ",";
287 25 : ss << "length:" << sqeInfo.length << ",";
288 25 : ss << "condValue:" << sqeInfo.condValue << ",";
289 25 : ss << "isLast:" << static_cast<uint16_t>(sqeInfo.isLast) << ",";
290 25 : ss << "opCode:" << static_cast<uint16_t>(sqeInfo.opCode) << ",";
291 25 : ss << "sqeNum:" << static_cast<uint16_t>(sqeInfo.sqeNum) << ",";
292 25 : ss << "valid:" << static_cast<uint16_t>(sqeInfo.valid) << ",";
293 25 : ss << "addr1High:0x" << std::hex << sqeInfo.addr1High << ",";
294 25 : ss << "addr1Low:0x" << std::hex << sqeInfo.addr1Low << ",";
295 25 : ss << "addr2High:0x" << std::hex << sqeInfo.addr2High << ",";
296 25 : ss << "addr2Low:0x" << std::hex << sqeInfo.addr2Low << ".";
297 50 : return ss.str();
298 25 : }
|
