Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "aicpu_sqe_context.h"
12 :
13 : #include <sstream>
14 : #include <unordered_map>
15 : #include "common/aicpu_hccl_common.h"
16 : #include "utils/hccl_aicpu_utils.h"
17 : #include "common/aicpu_kfc_utils.h"
18 :
19 : struct SqeContextVariable {
20 : int32_t lastClusterId = -1;
21 : SqeLocalRingBuffer *variablePtr = nullptr;
22 : };
23 :
24 : static SqeLocalRingBuffer g_ringBuffer[CLUSTER_CNT][AC_MAX_RANK_NUM];
25 : static SqeContext g_sqeContext[CLUSTER_CNT];
26 : static SqeContextVariable g_sqeVariable;
27 :
28 22523 : SqeContext *GetSqeContext()
29 : {
30 22523 : return &g_sqeContext[HcclAicpuUtils::GetCurClusterId()];
31 : }
32 :
33 298 : void AicpuSqeContext::InitSqeContext()
34 : {
35 894 : for (uint32_t i = 0U; i < CLUSTER_CNT; i++) {
36 596 : SqeContext *context = &g_sqeContext[i];
37 596 : context->buffPtr = g_ringBuffer[i];
38 596 : (void)memset_s(context->buffPtr, sizeof(SqeLocalRingBuffer[AC_MAX_RANK_NUM]), 0,
39 : sizeof(SqeLocalRingBuffer[AC_MAX_RANK_NUM]));
40 596 : context->clusterId = i;
41 : }
42 298 : }
43 :
44 66 : void AicpuSqeContext::SyncVariable()
45 : {
46 66 : SqeContext *context = GetSqeContext();
47 66 : HCCL_DEBUG("SyncCtxVariable, cur clusterId %d, last ClusterId %d, buffPtr %p", context->clusterId,
48 : g_sqeVariable.lastClusterId, g_sqeVariable.variablePtr);
49 66 : if (context->clusterId == g_sqeVariable.lastClusterId) {
50 62 : return;
51 : }
52 4 : if (g_sqeVariable.lastClusterId < 0 || g_sqeVariable.lastClusterId >= CLUSTER_CNT) {
53 2 : HCCL_DEBUG("SyncCtxVariable, invalid lastClusterId = %d", g_sqeVariable.lastClusterId);
54 2 : return;
55 : }
56 2 : context->buffPtr = g_sqeVariable.variablePtr;
57 : }
58 :
59 45 : void AicpuSqeContext::SaveVariable()
60 : {
61 45 : SqeContext *context = GetSqeContext();
62 45 : HCCL_DEBUG("Save sqe context variable, cur clusterId=%d, buffPtr=%p", context->clusterId, context->buffPtr);
63 45 : g_sqeVariable.lastClusterId = context->clusterId;
64 45 : g_sqeVariable.variablePtr = context->buffPtr;
65 45 : }
66 :
67 8481 : HcclResult AicpuSqeContext::GetNextSqeBufferAddr(uint32_t streamId, uint8_t *&sqeBufferAddr, uint8_t *&sqeTypeAddr,
68 : uint16_t &taskId)
69 : {
70 8481 : CHK_PRT_RET((streamId >= AC_MAX_RANK_NUM),
71 : HCCL_ERROR("[AicpuSqeContext][GetNextSqeBufferAddr]Invalid streamId[%u] >= %u", streamId, AC_MAX_RANK_NUM),
72 : HCCL_E_PARA);
73 8481 : SqeContext *context = GetSqeContext();
74 8481 : CHK_PTR_NULL(context->buffPtr);
75 8481 : auto &buff = context->buffPtr[streamId];
76 8481 : if (buff.tailSqeIdx >= AC_SQE_MAX_CNT) {
77 1 : HCCL_WARNING("Sqe cnt is overflow, need revise buff content, current streamid: %u", streamId);
78 1 : HCCL_INFO("buffer modify before ==> sqTail: %u, sqHead: %u, sqeCnt: %u, tailSqeTaskId: %u, tailSqeIdx: %u",
79 : buff.sqTail, buff.sqHead, buff.sqeCnt, buff.tailSqeTaskId, buff.tailSqeIdx);
80 1 : CHK_RET(AicpuKfcUtils::TraceProfSubmit());
81 1 : CHK_RET(AicpuSqeContext::ModifyBuffer(streamId));
82 1 : HCCL_INFO("buffer modify after ==> sqTail: %u, sqHead: %u, sqeCnt: %u, tailSqeTaskId: %u, tailSqeIdx: %u",
83 : buff.sqTail, buff.sqHead, buff.sqeCnt, buff.tailSqeTaskId, buff.tailSqeIdx);
84 : }
85 : // nextTaskId=0的时候下发PlaceHolder
86 8481 : if (UNLIKELY(buff.tailSqeTaskId == 0 && buff.filpNum != 0)) {
87 1 : CHK_RET(AddFlipTask(streamId));
88 : }
89 :
90 8481 : buff.profTimestap[buff.tailSqeIdx] = GetCurCpuTimestamp(true);
91 8481 : sqeBufferAddr = buff.localBuff + buff.tailSqeIdx * AC_SQE_SIZE;
92 8481 : sqeTypeAddr = &buff.sqeType[buff.tailSqeIdx];
93 8481 : taskId = buff.tailSqeTaskId;
94 8481 : HCCL_DEBUG("Get stream:%u next idx:%u, taskId:%u, clusterId:%u", streamId, buff.tailSqeIdx, taskId,
95 : context->clusterId);
96 8481 : if (buff.tailSqeTaskId == UINT16_MAX) {
97 1 : buff.filpNum++;
98 1 : HCCL_WARNING("Sqe context cur taskId is uint16_max");
99 : }
100 8481 : buff.tailSqeTaskId++;
101 8481 : buff.tailSqeIdx++;
102 8481 : buff.sqeCnt++;
103 8481 : return HCCL_SUCCESS;
104 : }
105 :
106 1 : HcclResult AicpuSqeContext::AddFlipTask(uint32_t streamId)
107 : {
108 1 : if (!dfx::ProfilingManager::GetProfL0State()) {
109 0 : return HCCL_SUCCESS;
110 : }
111 1 : SqeContext *context = GetSqeContext();
112 1 : CHK_PTR_NULL(context->buffPtr);
113 1 : auto &buff = context->buffPtr[streamId];
114 1 : uint16_t filpNum = buff.filpNum;
115 1 : uint16_t taskId = buff.tailSqeTaskId;
116 1 : auto ctx = AicpuGetComContext();
117 1 : HcclComStreamInfo *streamInfo = &ctx->streamInfo[streamId];
118 1 : CHK_RET(dfx::ProfilingManager::ReportFilpTask(streamInfo->actualStreamId, taskId, filpNum));
119 :
120 1 : buff.profTimestap[buff.tailSqeIdx] = GetCurCpuTimestamp(true);
121 1 : uint8_t *sqeBufferAddr = buff.localBuff + buff.tailSqeIdx * AC_SQE_SIZE;
122 1 : uint8_t *sqeTypeAddr = &buff.sqeType[buff.tailSqeIdx];
123 1 : AicpuAddOneFlipPlaceHolderSqe addOneFlipPlaceHolderSqe = AicpuGetAddOneFlipPlaceHolderSqe();
124 1 : if (addOneFlipPlaceHolderSqe == nullptr) {
125 0 : HCCL_WARNING("AicpuAddOneFlipPlaceHolderSqe is null");
126 0 : return HCCL_SUCCESS;
127 : }
128 1 : addOneFlipPlaceHolderSqe(streamInfo->actualStreamId, filpNum, taskId, sqeBufferAddr, sqeTypeAddr);
129 1 : buff.tailSqeTaskId++;
130 1 : buff.tailSqeIdx++;
131 1 : buff.sqeCnt++;
132 :
133 1 : HCCL_INFO("[AicpuSqeContext][AddFlipTask] Call AddFlipTask. para: taskId[%u], streamId[%u], filpNum[%u]]", taskId,
134 : streamInfo->actualStreamId, filpNum);
135 :
136 1 : return HCCL_SUCCESS;
137 : }
138 :
139 8471 : HcclResult AicpuSqeContext::RecordAddInfo(uint32_t streamId, uint32_t addInfo)
140 : {
141 8471 : CHK_PRT_RET((streamId >= AC_MAX_RANK_NUM),
142 : HCCL_ERROR("[AicpuSqeContext][RecordAddInfo]Invalid streamId[%u] >= %u", streamId, AC_MAX_RANK_NUM),
143 : HCCL_E_PARA);
144 8471 : SqeContext *context = GetSqeContext();
145 8471 : CHK_PTR_NULL(context->buffPtr);
146 8471 : auto &buff = context->buffPtr[streamId];
147 8471 : CHK_PRT_RET(((buff.tailSqeIdx == 0) || (buff.tailSqeIdx > AC_SQE_MAX_CNT)),
148 : HCCL_ERROR("[AicpuSqeContext][RecordAddInfo]Invalid tailSqeIdx[%u]", buff.tailSqeIdx),
149 : HCCL_E_PARA);
150 8471 : buff.addInfo[buff.tailSqeIdx - 1] = addInfo;
151 8471 : return HCCL_SUCCESS;
152 : }
153 :
154 69 : HcclResult AicpuSqeContext::QuerySqeInfoByHead(uint32_t streamId, uint32_t sqHead, SqeInfo *info)
155 : {
156 69 : CHK_PRT_RET((streamId >= AC_MAX_RANK_NUM),
157 : HCCL_ERROR("[AicpuSqeContext][QuerySqeInfoByHead]Invalid streamId[%u] >= %u", streamId, AC_MAX_RANK_NUM),
158 : HCCL_E_PARA);
159 69 : CHK_PTR_NULL(info);
160 69 : SqeContext *context = GetSqeContext();
161 69 : CHK_PTR_NULL(context->buffPtr);
162 69 : auto &buff = context->buffPtr[streamId];
163 69 : const uint32_t sqDepth = AicpuGetComContext()->streamInfo[streamId].sqDepth;
164 69 : uint32_t sqUnexecuted = (buff.sqTail + sqDepth - sqHead) % sqDepth;
165 69 : if (buff.tailSqeIdx < sqUnexecuted) {
166 3 : HCCL_WARNING("tail sqe idx %u is less then sq unexecuted num %u", buff.tailSqeIdx, sqUnexecuted);
167 3 : return HCCL_E_INTERNAL;
168 : }
169 66 : uint16_t idx = buff.tailSqeIdx - sqUnexecuted;
170 66 : HCCL_INFO("Query streamId:%u, sqeIdx:%u, actual idx:%u, type:%u", streamId, sqHead, idx, buff.sqeType[idx]);
171 66 : info->sqeHeadIdx = sqHead;
172 66 : return SqeContextUtils::QuerySqeInfo(buff.localBuff + idx * AC_SQE_SIZE, buff.sqeType[idx], buff.addInfo[idx], info);
173 : }
174 :
175 2 : HcclResult AicpuSqeContext::QuerySqeInfoByTaskId(uint32_t streamId, uint16_t taskId, SqeInfo *info)
176 : {
177 2 : CHK_PRT_RET((streamId >= AC_MAX_RANK_NUM),
178 : HCCL_ERROR("[AicpuSqeContext][QuerySqeInfoByTaskId]Invalid streamId[%u] >= %u", streamId, AC_MAX_RANK_NUM),
179 : HCCL_E_PARA);
180 2 : CHK_PTR_NULL(info);
181 2 : SqeContext *context = GetSqeContext();
182 2 : CHK_PTR_NULL(context->buffPtr);
183 2 : auto &buff = context->buffPtr[streamId];
184 2 : uint16_t tailRemain = buff.tailSqeTaskId - taskId;
185 2 : const uint32_t sqDepth = AicpuGetComContext()->streamInfo[streamId].sqDepth;
186 2 : uint32_t sqHeadIdx = (buff.sqTail + sqDepth - tailRemain) % sqDepth;
187 2 : if (buff.tailSqeIdx < tailRemain) {
188 1 : HCCL_WARNING("tail sqe idx %u is less then tail remain num %u", buff.tailSqeIdx, tailRemain);
189 1 : return HCCL_E_INTERNAL;
190 : }
191 1 : uint16_t idx = buff.tailSqeIdx - tailRemain;
192 1 : HCCL_INFO("Query streamId:%u, sqeIdx:%u, actual idx:%u, type:%u", streamId, sqHeadIdx, idx, buff.sqeType[idx]);
193 1 : info->sqeHeadIdx = sqHeadIdx;
194 1 : return SqeContextUtils::QuerySqeInfo(buff.localBuff + idx * AC_SQE_SIZE, buff.sqeType[idx], buff.addInfo[idx], info);
195 : }
196 :
197 4065 : HcclResult AicpuSqeContext::ClearCurBuff(uint32_t streamid, uint32_t leftBound)
198 : {
199 4065 : CHK_PRT_RET((streamid >= AC_MAX_RANK_NUM),
200 : HCCL_ERROR("[AicpuSqeContext][ClearCurBuff]Invalid streamId[%u] >= %u", streamid, AC_MAX_RANK_NUM),
201 : HCCL_E_PARA);
202 4065 : SqeContext *context = GetSqeContext();
203 4065 : auto &buff = context->buffPtr[streamid];
204 4065 : HCCL_INFO(
205 : "leftBound:%u, buff.sqeCnt:%u, buff.sqHead:%u, buff.sqTail:%u, buff.tailSqeIdx:%u, buff.tailSqeTaskId:%u",
206 : leftBound, buff.sqeCnt, buff.sqHead, buff.sqTail, buff.tailSqeIdx, buff.tailSqeTaskId);
207 8130 : if (memset_s(buff.localBuff + leftBound * AC_SQE_SIZE, sizeof(buff.localBuff) - leftBound * AC_SQE_SIZE, 0,
208 4065 : (buff.tailSqeIdx - leftBound) * AC_SQE_SIZE) != EOK) {
209 0 : return HCCL_E_MEMORY;
210 : }
211 4065 : if (memset_s(buff.sqeType + leftBound, sizeof(buff.sqeType) - leftBound, 0, buff.tailSqeIdx - leftBound) != EOK) {
212 0 : return HCCL_E_MEMORY;
213 : }
214 4065 : if (memset_s(buff.addInfo + leftBound, sizeof(buff.addInfo) - leftBound, 0, buff.tailSqeIdx - leftBound) != EOK) {
215 0 : return HCCL_E_MEMORY;
216 : }
217 4065 : buff.sqeCnt = 0;
218 4065 : buff.tailSqeIdx = 0;
219 4065 : AicpuGetComContext()->profilingExtendInfo.lastSqeIdxs[streamid] = 0;
220 4065 : return HCCL_SUCCESS;
221 : }
222 :
223 1 : HcclResult AicpuSqeContext::ModifyBuffer(uint32_t streamid)
224 : {
225 1 : CHK_PRT_RET((streamid >= AC_MAX_RANK_NUM),
226 : HCCL_ERROR("[AicpuSqeContext][ModifyBuffer]Invalid streamId[%u] >= %u", streamid, AC_MAX_RANK_NUM),
227 : HCCL_E_PARA);
228 : // 未下发的sqe移到前面
229 1 : SqeContext *context = GetSqeContext();
230 1 : auto &buff = context->buffPtr[streamid];
231 1 : uint32_t cnt = buff.sqeCnt;
232 1 : uint32_t leftSrc = buff.tailSqeIdx - buff.sqeCnt;
233 1 : HCCL_DEBUG("buff.sqeCnt:%d, buff.sqHead:%u, buff.sqTail:%u, buff.tailSqeIdx:%u, buff.tailSqeTaskId:%u", buff.sqeCnt,
234 : buff.sqHead, buff.sqTail, buff.tailSqeIdx, buff.tailSqeTaskId);
235 1 : if (memmove_s(buff.localBuff, sizeof(buff.localBuff), buff.localBuff + leftSrc * AC_SQE_SIZE, cnt * AC_SQE_SIZE) !=
236 : EOK) {
237 0 : return HCCL_E_MEMORY;
238 : }
239 1 : if (memmove_s(buff.sqeType, sizeof(buff.sqeType), buff.sqeType + leftSrc, cnt) != EOK) {
240 0 : return HCCL_E_MEMORY;
241 : }
242 1 : if (memmove_s(buff.addInfo, sizeof(buff.addInfo), buff.addInfo + leftSrc, cnt) != EOK) {
243 0 : return HCCL_E_MEMORY;
244 : }
245 : // 队列后面已经拷贝到rtsq上的sqe清除掉
246 1 : CHK_RET(ClearCurBuff(streamid, cnt));
247 : // 更新index和sqeCnt
248 1 : buff.tailSqeIdx = cnt;
249 1 : buff.sqeCnt = cnt;
250 1 : AicpuGetComContext()->profilingExtendInfo.lastSqeIdxs[streamid] = cnt;
251 1 : return HCCL_SUCCESS;
252 : }
253 :
254 127 : HcclResult AicpuSqeContext::ClearLocalBuff()
255 : {
256 4191 : for (uint32_t i = 0; i < AC_MAX_RANK_NUM; i++) {
257 4064 : CHK_RET(ClearCurBuff(i));
258 : }
259 127 : return HCCL_SUCCESS;
260 : }
261 :
262 25 : std::string AicpuSqeContext::GetString(const SqeInfo &sqeInfo)
263 : {
264 25 : std::stringstream ss;
265 25 : ss << "SqeInfo ";
266 25 : ss << "sqeIdx:" << sqeInfo.sqeHeadIdx << ",";
267 25 : ss << "type:" << SqeContextUtils::RtsqTaskTypeToStr(sqeInfo.type) << ",";
268 25 : ss << "subType:" << static_cast<uint16_t>(sqeInfo.subType) << ",";
269 25 : ss << "streamId:" << sqeInfo.streamId << ",";
270 25 : ss << "taskId:" << sqeInfo.taskId << ",";
271 25 : ss << "notifyId:" << sqeInfo.notifyId << ",";
272 25 : ss << "eventId:" << sqeInfo.eventId << ",";
273 25 : ss << "partId:" << sqeInfo.partId << ",";
274 25 : ss << "length:" << sqeInfo.length << ",";
275 25 : ss << "condValue:" << sqeInfo.condValue << ",";
276 25 : ss << "isLast:" << static_cast<uint16_t>(sqeInfo.isLast) << ",";
277 25 : ss << "opCode:" << static_cast<uint16_t>(sqeInfo.opCode) << ",";
278 25 : ss << "sqeNum:" << static_cast<uint16_t>(sqeInfo.sqeNum) << ",";
279 25 : ss << "valid:" << static_cast<uint16_t>(sqeInfo.valid) << ",";
280 25 : ss << "addr1High:0x" << std::hex << sqeInfo.addr1High << ",";
281 25 : ss << "addr1Low:0x" << std::hex << sqeInfo.addr1Low << ",";
282 25 : ss << "addr2High:0x" << std::hex << sqeInfo.addr2High << ",";
283 25 : ss << "addr2Low:0x" << std::hex << sqeInfo.addr2Low << ".";
284 50 : return ss.str();
285 25 : }
|
