Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "aicpusd_message_queue.h"
12 :
13 : #include "ascend_hal.h"
14 : #include "aicpusd_hal_interface_ref.h"
15 : #include "type_def.h"
16 : #include "aicpusd_status.h"
17 : #include "aicpusd_drv_manager.h"
18 : #include "aicpusd_feature_ctrl.h"
19 : #include "aicpusd_msq_operator_manager.h"
20 :
21 : namespace AicpuSchedule {
22 : namespace {
23 : enum class MsqDataSize : uint32_t {
24 : MSQ_DATA_SIZE_0 = 0b000, // No message
25 : MSQ_DATA_SIZE_128 = 0b100, // Message is 128bit, MSQ_DATA[0:1]_EL0 are valid for read
26 : MSQ_DATA_SIZE_256 = 0b101, // Message is 256bit, MSQ_DATA[0:3]_EL0 are valid for read
27 : MSQ_DATA_SIZE_512 = 0b110, // Message is 512bit, MSQ_DATA[0:7]_EL0 are valid for read
28 : };
29 :
30 : enum class CqeStatus : uint16_t {
31 : CQE_STATUS_OK = 0b00000,
32 : CQE_STATUS_DEBUG = 0b00010,
33 : CQE_STATUS_EXCEPTION = 0b00100,
34 : CQE_STATUS_WARNING = 0b10000
35 : };
36 :
37 : constexpr uint32_t HARD_THREAD_NUM_PER_CPU = 2U;
38 : constexpr uint32_t CQE_SIZE = 4U;
39 : } // namespace
40 :
41 : thread_local MessageQueue::MsqStatusFunc MessageQueue::readMsqStatusFunc_ = nullptr;
42 : thread_local MessageQueue::MsqDataFunc MessageQueue::readMsqDataFunc_ = nullptr;
43 : thread_local MessageQueue::MsqRspFunc MessageQueue::sendMsqRspFunc_ = nullptr;
44 : thread_local uint32_t* MessageQueue::cqeAddr_ = nullptr;
45 : std::shared_ptr<MsqImpl> MessageQueue::impl_ = nullptr;
46 :
47 15 : MessageQueue& MessageQueue::GetInstance()
48 : {
49 15 : static MessageQueue instance;
50 15 : return instance;
51 : }
52 :
53 25 : int32_t MessageQueue::InitMessageQueue(const uint32_t deviceId, const std::vector<uint32_t>& aicpuPhyIds)
54 : {
55 25 : int32_t ret = AICPU_SCHEDULE_OK;
56 25 : deviceId_ = deviceId;
57 25 : aicpuPhyIds_ = aicpuPhyIds;
58 :
59 25 : ret = InitMsqImpl();
60 25 : if (ret != AICPU_SCHEDULE_OK) {
61 0 : aicpusd_err("Init message queue impl instance failed");
62 0 : return ret;
63 : }
64 :
65 25 : ret = InitCqeBaseAddr();
66 25 : if (ret != AICPU_SCHEDULE_OK) {
67 2 : aicpusd_err("Init cqe addr failed, ret=%d, deviceId=%u", ret, deviceId);
68 2 : return ret;
69 : }
70 :
71 23 : aicpusd_run_info(
72 : "Init message queue success, deviceId=%u, mode=%d", deviceId, static_cast<int32_t>(FeatureCtrl::IsUseMsqV2()));
73 :
74 23 : return AICPU_SCHEDULE_OK;
75 : }
76 :
77 25 : int32_t MessageQueue::InitMsqImpl() const
78 : {
79 25 : int32_t ret = MsqOperatorManager::Init();
80 25 : if (ret != AICPU_SCHEDULE_OK) {
81 0 : aicpusd_err("Init msq operator manager failed");
82 0 : return ret;
83 : }
84 :
85 25 : impl_ = FeatureCtrl::IsUseMsqV2() ? std::make_shared<MsqImplV2>() : std::make_shared<MsqImplV1>();
86 25 : if (impl_ == nullptr) {
87 0 : aicpusd_err("Create msq impl failed by nullptr");
88 0 : return AICPU_SCHEDULE_ERROR_INNER_ERROR;
89 : }
90 :
91 25 : return AICPU_SCHEDULE_OK;
92 : }
93 :
94 25 : int32_t MessageQueue::InitCqeBaseAddr()
95 : {
96 25 : cqeBaseAddr_ = MapResAddr(RES_ADDR_TYPE_STARS_TOPIC_CQE);
97 25 : if (cqeBaseAddr_ == nullptr) {
98 2 : aicpusd_err("Failed to get CQE base address: nullptr");
99 2 : return AICPU_SCHEDULE_ERROR_DRV_ERR;
100 : }
101 23 : aicpusd_info("Init cqe base addr success, va=0x%x", cqeBaseAddr_);
102 23 : return AICPU_SCHEDULE_OK;
103 : }
104 :
105 25 : uint32_t* MessageQueue::MapResAddr(const res_addr_type resType) const
106 : {
107 25 : if (&halResAddrMap == nullptr) {
108 0 : aicpusd_err("Get resource address failed by nullptr");
109 0 : return nullptr;
110 : }
111 :
112 25 : res_addr_info resInfo = {};
113 25 : resInfo.id = 0U;
114 25 : resInfo.target_proc_type = PROCESS_CP1;
115 25 : resInfo.res_type = resType;
116 25 : resInfo.res_id = 0U;
117 25 : uint64_t va = 0UL;
118 25 : uint32_t len = 0U;
119 25 : const int32_t ret = halResAddrMap(deviceId_, &resInfo, &va, &len);
120 25 : if (ret != DRV_ERROR_NONE) {
121 2 : aicpusd_err("Get resource address failed, ret=%d, drvType=%u, deviceId=%u", ret, resType, deviceId_);
122 2 : return nullptr;
123 : }
124 :
125 23 : aicpusd_info("Get resource address success, type=%u, deviceId=%u, va=0x%x, len=%u", resType, deviceId_, va, len);
126 :
127 23 : return PtrToPtr<void, uint32_t>(ValueToPtr(va));
128 : }
129 :
130 11 : int32_t MessageQueue::InitMessageQueueForThread(const size_t threadIndex) const
131 : {
132 11 : aicpusd_info("Start initializing message queue for thread");
133 :
134 11 : int32_t ret = ResetMessageQueueStatus(threadIndex);
135 11 : if (ret != AICPU_SCHEDULE_OK) {
136 1 : aicpusd_err("Reset message queue status failed, threadIndex=%u", threadIndex);
137 1 : return ret;
138 : }
139 :
140 10 : ret = InitMessageQueueStatusReadFunc(threadIndex);
141 10 : if (ret != AICPU_SCHEDULE_OK) {
142 1 : aicpusd_err("Init message queue status read func failed, threadIndex=%u", threadIndex);
143 1 : return ret;
144 : }
145 :
146 9 : ret = InitMessageQueueDataReadFunc(threadIndex);
147 9 : if (ret != AICPU_SCHEDULE_OK) {
148 1 : aicpusd_err("Init message queue data read func failed, threadIndex=%u", threadIndex);
149 1 : return ret;
150 : }
151 :
152 8 : ret = InitMessageQueueRspFunc(threadIndex);
153 8 : if (ret != AICPU_SCHEDULE_OK) {
154 1 : aicpusd_err("Init message queue rsp func failed, threadIndex=%u", threadIndex);
155 1 : return ret;
156 : }
157 :
158 7 : ret = InitCqeAddr(threadIndex);
159 7 : if (ret != AICPU_SCHEDULE_OK) {
160 1 : aicpusd_err("Init cqe addr failed, threadIndex=%u", threadIndex);
161 1 : return ret;
162 : }
163 :
164 6 : aicpusd_info("Init message queue for thread success, threadIndex=%u", threadIndex);
165 :
166 6 : return AICPU_SCHEDULE_OK;
167 : }
168 :
169 10 : int32_t MessageQueue::ResetMessageQueueStatus(const size_t threadIndex) const
170 : {
171 : // MSQ{0-7}_STATUS_EL0
172 10 : aicpusd_info("Start reset message queue status");
173 10 : int32_t ret = AICPU_SCHEDULE_OK;
174 10 : MsqStatus msqStatus = {};
175 10 : if (isEnableHardThread_) {
176 1 : ret = IsUseMsqT0(threadIndex) ? ResetMsqT0Status() : ResetMsqT1Status();
177 1 : msqStatus = IsUseMsqT0(threadIndex) ? ReadMsqT0Status() : ReadMsqT1Status();
178 : } else {
179 9 : (void)ResetMsqT0Status();
180 9 : ret = ResetMsqT1Status();
181 9 : msqStatus = ReadMsqT0Status();
182 : }
183 :
184 10 : aicpusd_info("End reset message queue status, ret=%d, valid=%u, comp=%u", ret, msqStatus.valid, msqStatus.comp);
185 :
186 10 : return ret;
187 : }
188 :
189 10 : int32_t MessageQueue::ResetMsqT0Status() const { return impl_->ResetMsqT0Status(); }
190 :
191 9 : int32_t MessageQueue::ResetMsqT1Status() const { return impl_->ResetMsqT1Status(); }
192 :
193 9 : int32_t MessageQueue::InitMessageQueueStatusReadFunc(const size_t threadIndex) const
194 : {
195 9 : if (isEnableHardThread_) {
196 1 : readMsqStatusFunc_ = IsUseMsqT0(threadIndex) ? &MessageQueue::ReadMsqT0Status : &MessageQueue::ReadMsqT1Status;
197 : } else {
198 8 : readMsqStatusFunc_ = &MessageQueue::ReadMsqT0Status;
199 : }
200 :
201 9 : return AICPU_SCHEDULE_OK;
202 : }
203 :
204 11 : MsqStatus MessageQueue::ReadMsqT0Status() { return impl_->ReadMsqT0Status(); }
205 :
206 0 : MsqStatus MessageQueue::ReadMsqT1Status() { return impl_->ReadMsqT1Status(); }
207 :
208 8 : int32_t MessageQueue::InitMessageQueueDataReadFunc(const size_t threadIndex) const
209 : {
210 8 : if (isEnableHardThread_) {
211 1 : readMsqDataFunc_ = IsUseMsqT0(threadIndex) ? &MessageQueue::ReadMsqT0Data : &MessageQueue::ReadMsqT1Data;
212 : } else {
213 7 : readMsqDataFunc_ = &MessageQueue::ReadMsqT0Data;
214 : }
215 :
216 8 : return AICPU_SCHEDULE_OK;
217 : }
218 :
219 2 : void MessageQueue::ReadMsqT0Data(const uint32_t msgSize, MsqDatas& datas) { impl_->ReadMsqT0Data(msgSize, datas); }
220 :
221 2 : void MessageQueue::ReadMsqT1Data(const uint32_t msgSize, MsqDatas& datas) { impl_->ReadMsqT1Data(msgSize, datas); }
222 :
223 7 : int32_t MessageQueue::InitMessageQueueRspFunc(const size_t threadIndex) const
224 : {
225 7 : if (isEnableHardThread_) {
226 1 : sendMsqRspFunc_ = IsUseMsqT0(threadIndex) ? &MessageQueue::SendMsqT0Response : &MessageQueue::SendMsqT1Response;
227 : } else {
228 6 : sendMsqRspFunc_ = &MessageQueue::SendMsqT0Response;
229 : }
230 :
231 7 : return AICPU_SCHEDULE_OK;
232 : }
233 :
234 1 : void MessageQueue::SendMsqT0Response() { impl_->SendMsqT0Response(); }
235 :
236 1 : void MessageQueue::SendMsqT1Response() { impl_->SendMsqT1Response(); }
237 :
238 9 : int32_t MessageQueue::InitCqeAddr(const size_t threadIndex) const
239 : {
240 9 : if (threadIndex >= aicpuPhyIds_.size()) {
241 2 : aicpusd_err(
242 : "Init cqe addr failed, threadIdx larger than aicpuPhyIds size, threadIndex=%zu, size=%zu", threadIndex,
243 : aicpuPhyIds_.size());
244 2 : return AICPU_SCHEDULE_ERROR_PARAMETER_NOT_VALID;
245 : }
246 :
247 7 : cqeAddr_ = &(cqeBaseAddr_[aicpuPhyIds_[threadIndex] * CQE_SIZE]);
248 7 : aicpusd_info("Init cqe addr success, threadIndex=%zu, base=0x%x, va=0x%x", threadIndex, cqeBaseAddr_, cqeAddr_);
249 7 : return AICPU_SCHEDULE_OK;
250 : }
251 :
252 100 : void MessageQueue::SendResponse(const uint32_t errCode, const uint32_t status)
253 : {
254 100 : if (sendMsqRspFunc_ == nullptr) {
255 99 : return;
256 : }
257 1 : MessageQueue::sendMsqRspFunc_();
258 1 : SetCQE(errCode, status);
259 : }
260 :
261 7 : bool MessageQueue::IsMsqRspComplete()
262 : {
263 7 : if (readMsqStatusFunc_ == nullptr) {
264 7 : return false;
265 : }
266 :
267 0 : const MsqStatus status = readMsqStatusFunc_();
268 0 : return (status.valid == 0U);
269 : }
270 :
271 1 : void MessageQueue::SetCQE(const uint32_t errCode, const uint32_t status)
272 : {
273 : /**
274 : * Topic CQE
275 : * 上报执行错误码
276 : *
277 : * total 32bit
278 : * [31:16](RW): error code
279 : * [15:0](RW): status
280 : */
281 :
282 1 : const uint32_t cqeStatus = (status == 0U) ? static_cast<uint32_t>(CqeStatus::CQE_STATUS_OK) :
283 : static_cast<uint32_t>(CqeStatus::CQE_STATUS_EXCEPTION);
284 :
285 1 : aicpusd_info(
286 : "Begin to set cqe, va=0x%x, errCode=%u, status=%u, cqeStatus=%u", cqeAddr_, errCode, status, cqeStatus);
287 1 : *cqeAddr_ = ((errCode << 16U) | (cqeStatus));
288 1 : aicpusd_info("End to set cqe, va=0x%x, errCode=%u, status=%u, cqeStatus=%u", cqeAddr_, errCode, status, cqeStatus);
289 1 : }
290 :
291 1 : bool MessageQueue::WaitMsqInfoOnce(MsqDatas& datas)
292 : {
293 1 : const MsqStatus status = readMsqStatusFunc_();
294 1 : if (status.valid == 0U) {
295 1 : WaitForEvent();
296 1 : return false;
297 : }
298 :
299 0 : aicpusd_info("Begin to read message queue datas, size=%u", status.size);
300 0 : readMsqDataFunc_(status.size, datas);
301 0 : aicpusd_info("End to read message queue datas, size=%u", status.size);
302 :
303 0 : return true;
304 : }
305 :
306 1 : void MessageQueue::WaitForEvent()
307 : {
308 1 : MsqOperatorManager::CallWait();
309 1 : return;
310 : }
311 :
312 5 : bool MessageQueue::IsUseMsqT0(const size_t threadIndex) { return ((threadIndex % HARD_THREAD_NUM_PER_CPU) == 0U); }
313 :
314 10 : int32_t MsqImplV1::ResetMsqT0Status() const
315 : {
316 10 : MsqOperatorManager::CallV1ResetT0Status();
317 10 : return AICPU_SCHEDULE_OK;
318 : }
319 :
320 9 : int32_t MsqImplV1::ResetMsqT1Status() const
321 : {
322 9 : MsqOperatorManager::CallV1ResetT1Status();
323 9 : return AICPU_SCHEDULE_OK;
324 : }
325 :
326 11 : MsqStatus MsqImplV1::ReadMsqT0Status() const { return MsqOperatorManager::CallV1ReadT0Status(); }
327 :
328 0 : MsqStatus MsqImplV1::ReadMsqT1Status() const { return MsqOperatorManager::CallV1ReadT1Status(); }
329 :
330 2 : void MsqImplV1::ReadMsqT0Data(const uint32_t msgSize, MsqDatas& datas) const
331 : {
332 2 : if (msgSize == static_cast<uint32_t>(MsqDataSize::MSQ_DATA_SIZE_0)) {
333 1 : aicpusd_err("Message size is 0");
334 1 : return;
335 : }
336 :
337 1 : MsqOperatorManager::CallV1ReadT0Data(msgSize, &datas);
338 : }
339 :
340 2 : void MsqImplV1::ReadMsqT1Data(const uint32_t msgSize, MsqDatas& datas) const
341 : {
342 2 : if (msgSize == static_cast<uint32_t>(MsqDataSize::MSQ_DATA_SIZE_0)) {
343 1 : aicpusd_err("Message size is 0");
344 1 : return;
345 : }
346 :
347 1 : MsqOperatorManager::CallV1ReadT1Data(msgSize, &datas);
348 : }
349 :
350 1 : void __attribute__((optimize("O0"))) MsqImplV1::SendMsqT0Response() const
351 : {
352 : // O2 compilation optimization will optimize away the msq write operation, so O0 optimization must be used.
353 1 : MsqOperatorManager::CallV1SendT0Response();
354 1 : }
355 :
356 1 : void __attribute__((optimize("O0"))) MsqImplV1::SendMsqT1Response() const
357 : {
358 1 : MsqOperatorManager::CallV1SendT1Response();
359 1 : }
360 :
361 0 : int32_t MsqImplV2::ResetMsqT0Status() const
362 : {
363 0 : MsqOperatorManager::CallV2ResetT0Status();
364 0 : return AICPU_SCHEDULE_OK;
365 : }
366 :
367 0 : int32_t MsqImplV2::ResetMsqT1Status() const
368 : {
369 0 : MsqOperatorManager::CallV2ResetT1Status();
370 0 : return AICPU_SCHEDULE_OK;
371 : }
372 :
373 0 : MsqStatus MsqImplV2::ReadMsqT1Status() const { return MsqOperatorManager::CallV2ReadT1Status(); }
374 :
375 0 : void MsqImplV2::ReadMsqT1Data(const uint32_t msgSize, MsqDatas& datas) const
376 : {
377 0 : if (msgSize == static_cast<uint32_t>(MsqDataSize::MSQ_DATA_SIZE_0)) {
378 0 : aicpusd_err("Message size is 0");
379 0 : return;
380 : }
381 :
382 0 : MsqOperatorManager::CallV2ReadT1Data(msgSize, &datas);
383 : }
384 :
385 0 : void __attribute__((optimize("O0"))) MsqImplV2::SendMsqT1Response() const
386 : {
387 0 : MsqOperatorManager::CallV2SendT1Response();
388 0 : }
389 : } // namespace AicpuSchedule
|
