Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "aicpusd_message_queue.h"
12 :
13 : #include "ascend_hal.h"
14 : #include "aicpusd_hal_interface_ref.h"
15 : #include "type_def.h"
16 : #include "aicpusd_status.h"
17 : #include "aicpusd_drv_manager.h"
18 : #include "aicpusd_feature_ctrl.h"
19 : #include "aicpusd_msq_operator_manager.h"
20 :
21 :
22 : namespace AicpuSchedule {
23 : namespace {
24 : enum class MsqDataSize : uint32_t {
25 : MSQ_DATA_SIZE_0 = 0b000, // No message
26 : MSQ_DATA_SIZE_128 = 0b100, // Message is 128bit, MSQ_DATA[0:1]_EL0 are valid for read
27 : MSQ_DATA_SIZE_256 = 0b101, // Message is 256bit, MSQ_DATA[0:3]_EL0 are valid for read
28 : MSQ_DATA_SIZE_512 = 0b110, // Message is 512bit, MSQ_DATA[0:7]_EL0 are valid for read
29 : };
30 :
31 : enum class CqeStatus : uint16_t {
32 : CQE_STATUS_OK = 0b00000,
33 : CQE_STATUS_DEBUG = 0b00010,
34 : CQE_STATUS_EXCEPTION = 0b00100,
35 : CQE_STATUS_WARNING = 0b10000
36 : };
37 :
38 : constexpr uint32_t HARD_THREAD_NUM_PER_CPU = 2U;
39 : constexpr uint32_t CQE_SIZE = 4U;
40 : }
41 :
42 : thread_local MessageQueue::MsqStatusFunc MessageQueue::readMsqStatusFunc_ = nullptr;
43 : thread_local MessageQueue::MsqDataFunc MessageQueue::readMsqDataFunc_ = nullptr;
44 : thread_local MessageQueue::MsqRspFunc MessageQueue::sendMsqRspFunc_ = nullptr;
45 : thread_local uint32_t *MessageQueue::cqeAddr_ = nullptr;
46 : std::shared_ptr<MsqImpl> MessageQueue::impl_ = nullptr;
47 :
48 15 : MessageQueue &MessageQueue::GetInstance()
49 : {
50 15 : static MessageQueue instance;
51 15 : return instance;
52 : }
53 :
54 25 : int32_t MessageQueue::InitMessageQueue(const uint32_t deviceId, const std::vector<uint32_t> &aicpuPhyIds)
55 : {
56 25 : int32_t ret = AICPU_SCHEDULE_OK;
57 25 : deviceId_ = deviceId;
58 25 : aicpuPhyIds_ = aicpuPhyIds;
59 :
60 25 : ret = InitMsqImpl();
61 25 : if (ret != AICPU_SCHEDULE_OK) {
62 0 : aicpusd_err("Init message queue impl instance failed");
63 0 : return ret;
64 : }
65 :
66 25 : ret = InitCqeBaseAddr();
67 25 : if (ret != AICPU_SCHEDULE_OK) {
68 2 : aicpusd_err("Init cqe addr failed, ret=%d, deviceId=%u", ret, deviceId);
69 2 : return ret;
70 : }
71 :
72 23 : aicpusd_run_info("Init message queue success, deviceId=%u, mode=%d",
73 : deviceId, static_cast<int32_t>(FeatureCtrl::IsUseMsqV2()));
74 :
75 23 : return AICPU_SCHEDULE_OK;
76 : }
77 :
78 25 : int32_t MessageQueue::InitMsqImpl() const
79 : {
80 25 : int32_t ret = MsqOperatorManager::Init();
81 25 : if (ret != AICPU_SCHEDULE_OK) {
82 0 : aicpusd_err("Init msq operator manager failed");
83 0 : return ret;
84 : }
85 :
86 25 : impl_ = FeatureCtrl::IsUseMsqV2() ? std::make_shared<MsqImplV2>() : std::make_shared<MsqImplV1>();
87 25 : if (impl_ == nullptr) {
88 0 : aicpusd_err("Create msq impl failed by nullptr");
89 0 : return AICPU_SCHEDULE_ERROR_INNER_ERROR;
90 : }
91 :
92 25 : return AICPU_SCHEDULE_OK;
93 : }
94 :
95 25 : int32_t MessageQueue::InitCqeBaseAddr()
96 : {
97 25 : cqeBaseAddr_ = MapResAddr(RES_ADDR_TYPE_STARS_TOPIC_CQE);
98 25 : if (cqeBaseAddr_ == nullptr) {
99 2 : aicpusd_err("Failed to get CQE base address: nullptr");
100 2 : return AICPU_SCHEDULE_ERROR_DRV_ERR;
101 : }
102 23 : aicpusd_info("Init cqe base addr success, va=0x%x", cqeBaseAddr_);
103 23 : return AICPU_SCHEDULE_OK;
104 : }
105 :
106 25 : uint32_t *MessageQueue::MapResAddr(const res_addr_type resType) const
107 : {
108 25 : if (&halResAddrMap == nullptr) {
109 0 : aicpusd_err("Get resource address failed by nullptr");
110 0 : return nullptr;
111 : }
112 :
113 25 : res_addr_info resInfo = {};
114 25 : resInfo.id = 0U;
115 25 : resInfo.target_proc_type = PROCESS_CP1;
116 25 : resInfo.res_type = resType;
117 25 : resInfo.res_id = 0U;
118 25 : uint64_t va = 0UL;
119 25 : uint32_t len = 0U;
120 25 : const int32_t ret = halResAddrMap(deviceId_, &resInfo, &va, &len);
121 25 : if (ret != DRV_ERROR_NONE) {
122 2 : aicpusd_err("Get resource address failed, ret=%d, drvType=%u, deviceId=%u", ret, resType, deviceId_);
123 2 : return nullptr;
124 : }
125 :
126 23 : aicpusd_info("Get resource address success, type=%u, deviceId=%u, va=0x%x, len=%u",
127 : resType, deviceId_, va, len);
128 :
129 23 : return PtrToPtr<void, uint32_t>(ValueToPtr(va));
130 : }
131 :
132 11 : int32_t MessageQueue::InitMessageQueueForThread(const size_t threadIndex) const
133 : {
134 11 : aicpusd_info("Start initializing message queue for thread");
135 :
136 11 : int32_t ret = ResetMessageQueueStatus(threadIndex);
137 11 : if (ret != AICPU_SCHEDULE_OK) {
138 1 : aicpusd_err("Reset message queue status failed, threadIndex=%u", threadIndex);
139 1 : return ret;
140 : }
141 :
142 10 : ret = InitMessageQueueStatusReadFunc(threadIndex);
143 10 : if (ret != AICPU_SCHEDULE_OK) {
144 1 : aicpusd_err("Init message queue status read func failed, threadIndex=%u", threadIndex);
145 1 : return ret;
146 : }
147 :
148 9 : ret = InitMessageQueueDataReadFunc(threadIndex);
149 9 : if (ret != AICPU_SCHEDULE_OK) {
150 1 : aicpusd_err("Init message queue data read func failed, threadIndex=%u", threadIndex);
151 1 : return ret;
152 : }
153 :
154 8 : ret = InitMessageQueueRspFunc(threadIndex);
155 8 : if (ret != AICPU_SCHEDULE_OK) {
156 1 : aicpusd_err("Init message queue rsp func failed, threadIndex=%u", threadIndex);
157 1 : return ret;
158 : }
159 :
160 7 : ret = InitCqeAddr(threadIndex);
161 7 : if (ret != AICPU_SCHEDULE_OK) {
162 1 : aicpusd_err("Init cqe addr failed, threadIndex=%u", threadIndex);
163 1 : return ret;
164 : }
165 :
166 6 : aicpusd_info("Init message queue for thread success, threadIndex=%u", threadIndex);
167 :
168 6 : return AICPU_SCHEDULE_OK;
169 : }
170 :
171 10 : int32_t MessageQueue::ResetMessageQueueStatus(const size_t threadIndex) const
172 : {
173 : // MSQ{0-7}_STATUS_EL0
174 10 : aicpusd_info("Start reset message queue status");
175 10 : int32_t ret = AICPU_SCHEDULE_OK;
176 10 : MsqStatus msqStatus = {};
177 10 : if (isEnableHardThread_) {
178 1 : ret = IsUseMsqT0(threadIndex) ? ResetMsqT0Status() : ResetMsqT1Status();
179 1 : msqStatus = IsUseMsqT0(threadIndex) ? ReadMsqT0Status() : ReadMsqT1Status();
180 : } else {
181 9 : (void)ResetMsqT0Status();
182 9 : ret = ResetMsqT1Status();
183 9 : msqStatus = ReadMsqT0Status();
184 : }
185 :
186 10 : aicpusd_info("End reset message queue status, ret=%d, valid=%u, comp=%u", ret, msqStatus.valid, msqStatus.comp);
187 :
188 10 : return ret;
189 : }
190 :
191 10 : int32_t MessageQueue::ResetMsqT0Status() const
192 : {
193 10 : return impl_->ResetMsqT0Status();
194 : }
195 :
196 9 : int32_t MessageQueue::ResetMsqT1Status() const
197 : {
198 9 : return impl_->ResetMsqT1Status();
199 : }
200 :
201 9 : int32_t MessageQueue::InitMessageQueueStatusReadFunc(const size_t threadIndex) const
202 : {
203 9 : if (isEnableHardThread_) {
204 1 : readMsqStatusFunc_ = IsUseMsqT0(threadIndex) ? &MessageQueue::ReadMsqT0Status : &MessageQueue::ReadMsqT1Status;
205 : } else {
206 8 : readMsqStatusFunc_ = &MessageQueue::ReadMsqT0Status;
207 : }
208 :
209 9 : return AICPU_SCHEDULE_OK;
210 : }
211 :
212 11 : MsqStatus MessageQueue::ReadMsqT0Status()
213 : {
214 11 : return impl_->ReadMsqT0Status();
215 : }
216 :
217 0 : MsqStatus MessageQueue::ReadMsqT1Status()
218 : {
219 0 : return impl_->ReadMsqT1Status();
220 : }
221 :
222 8 : int32_t MessageQueue::InitMessageQueueDataReadFunc(const size_t threadIndex) const
223 : {
224 8 : if (isEnableHardThread_) {
225 1 : readMsqDataFunc_ = IsUseMsqT0(threadIndex) ? &MessageQueue::ReadMsqT0Data : &MessageQueue::ReadMsqT1Data;
226 : } else {
227 7 : readMsqDataFunc_ = &MessageQueue::ReadMsqT0Data;
228 : }
229 :
230 8 : return AICPU_SCHEDULE_OK;
231 : }
232 :
233 2 : void MessageQueue::ReadMsqT0Data(const uint32_t msgSize, MsqDatas &datas)
234 : {
235 2 : impl_->ReadMsqT0Data(msgSize, datas);
236 2 : }
237 :
238 2 : void MessageQueue::ReadMsqT1Data(const uint32_t msgSize, MsqDatas &datas)
239 : {
240 2 : impl_->ReadMsqT1Data(msgSize, datas);
241 2 : }
242 :
243 7 : int32_t MessageQueue::InitMessageQueueRspFunc(const size_t threadIndex) const
244 : {
245 7 : if (isEnableHardThread_) {
246 1 : sendMsqRspFunc_ = IsUseMsqT0(threadIndex) ? &MessageQueue::SendMsqT0Response : &MessageQueue::SendMsqT1Response;
247 : } else {
248 6 : sendMsqRspFunc_ = &MessageQueue::SendMsqT0Response;
249 : }
250 :
251 7 : return AICPU_SCHEDULE_OK;
252 : }
253 :
254 1 : void MessageQueue::SendMsqT0Response()
255 : {
256 1 : impl_->SendMsqT0Response();
257 1 : }
258 :
259 1 : void MessageQueue::SendMsqT1Response()
260 : {
261 1 : impl_->SendMsqT1Response();
262 1 : }
263 :
264 9 : int32_t MessageQueue::InitCqeAddr(const size_t threadIndex) const
265 : {
266 9 : if (threadIndex >= aicpuPhyIds_.size()) {
267 2 : aicpusd_err("Init cqe addr failed, threadIdx larger than aicpuPhyIds size, threadIndex=%zu, size=%zu",
268 : threadIndex, aicpuPhyIds_.size());
269 2 : return AICPU_SCHEDULE_ERROR_PARAMETER_NOT_VALID;
270 : }
271 :
272 7 : cqeAddr_ = &(cqeBaseAddr_[aicpuPhyIds_[threadIndex] * CQE_SIZE]);
273 7 : aicpusd_info("Init cqe addr success, threadIndex=%zu, base=0x%x, va=0x%x", threadIndex, cqeBaseAddr_, cqeAddr_);
274 7 : return AICPU_SCHEDULE_OK;
275 : }
276 :
277 100 : void MessageQueue::SendResponse(const uint32_t errCode, const uint32_t status)
278 : {
279 100 : if (sendMsqRspFunc_ == nullptr) {
280 99 : return;
281 : }
282 1 : MessageQueue::sendMsqRspFunc_();
283 1 : SetCQE(errCode, status);
284 : }
285 :
286 7 : bool MessageQueue::IsMsqRspComplete()
287 : {
288 7 : if (readMsqStatusFunc_ == nullptr) {
289 7 : return false;
290 : }
291 :
292 0 : const MsqStatus status = readMsqStatusFunc_();
293 0 : return (status.valid == 0U);
294 : }
295 :
296 1 : void MessageQueue::SetCQE(const uint32_t errCode, const uint32_t status)
297 : {
298 : /**
299 : * Topic CQE
300 : * 上报执行错误码
301 : *
302 : * total 32bit
303 : * [31:16](RW): error code
304 : * [15:0](RW): status
305 : */
306 :
307 1 : const uint32_t cqeStatus = (status == 0U) ? static_cast<uint32_t>(CqeStatus::CQE_STATUS_OK) :
308 : static_cast<uint32_t>(CqeStatus::CQE_STATUS_EXCEPTION);
309 :
310 1 : aicpusd_info("Begin to set cqe, va=0x%x, errCode=%u, status=%u, cqeStatus=%u",
311 : cqeAddr_, errCode, status, cqeStatus);
312 1 : *cqeAddr_ = ((errCode << 16U) | (cqeStatus));
313 1 : aicpusd_info("End to set cqe, va=0x%x, errCode=%u, status=%u, cqeStatus=%u",
314 : cqeAddr_, errCode, status, cqeStatus);
315 1 : }
316 :
317 1 : bool MessageQueue::WaitMsqInfoOnce(MsqDatas &datas)
318 : {
319 1 : const MsqStatus status = readMsqStatusFunc_();
320 1 : if (status.valid == 0U) {
321 1 : WaitForEvent();
322 1 : return false;
323 : }
324 :
325 0 : aicpusd_info("Begin to read message queue datas, size=%u", status.size);
326 0 : readMsqDataFunc_(status.size, datas);
327 0 : aicpusd_info("End to read message queue datas, size=%u", status.size);
328 :
329 0 : return true;
330 : }
331 :
332 1 : void MessageQueue::WaitForEvent()
333 : {
334 1 : MsqOperatorManager::CallWait();
335 1 : return;
336 : }
337 :
338 5 : bool MessageQueue::IsUseMsqT0(const size_t threadIndex)
339 : {
340 5 : return ((threadIndex % HARD_THREAD_NUM_PER_CPU) == 0U);
341 : }
342 :
343 10 : int32_t MsqImplV1::ResetMsqT0Status() const
344 : {
345 10 : MsqOperatorManager::CallV1ResetT0Status();
346 10 : return AICPU_SCHEDULE_OK;
347 : }
348 :
349 9 : int32_t MsqImplV1::ResetMsqT1Status() const
350 : {
351 9 : MsqOperatorManager::CallV1ResetT1Status();
352 9 : return AICPU_SCHEDULE_OK;
353 : }
354 :
355 11 : MsqStatus MsqImplV1::ReadMsqT0Status() const
356 : {
357 11 : return MsqOperatorManager::CallV1ReadT0Status();
358 : }
359 :
360 0 : MsqStatus MsqImplV1::ReadMsqT1Status() const
361 : {
362 0 : return MsqOperatorManager::CallV1ReadT1Status();
363 : }
364 :
365 2 : void MsqImplV1::ReadMsqT0Data(const uint32_t msgSize, MsqDatas &datas) const
366 : {
367 2 : if (msgSize == static_cast<uint32_t>(MsqDataSize::MSQ_DATA_SIZE_0)) {
368 1 : aicpusd_err("Message size is 0");
369 1 : return;
370 : }
371 :
372 1 : MsqOperatorManager::CallV1ReadT0Data(msgSize, &datas);
373 : }
374 :
375 2 : void MsqImplV1::ReadMsqT1Data(const uint32_t msgSize, MsqDatas &datas) const
376 : {
377 2 : if (msgSize == static_cast<uint32_t>(MsqDataSize::MSQ_DATA_SIZE_0)) {
378 1 : aicpusd_err("Message size is 0");
379 1 : return;
380 : }
381 :
382 1 : MsqOperatorManager::CallV1ReadT1Data(msgSize, &datas);
383 : }
384 :
385 1 : void __attribute__((optimize("O0"))) MsqImplV1::SendMsqT0Response() const
386 : {
387 : // O2 compilation optimization will optimize away the msq write operation, so O0 optimization must be used.
388 1 : MsqOperatorManager::CallV1SendT0Response();
389 1 : }
390 :
391 1 : void __attribute__((optimize("O0"))) MsqImplV1::SendMsqT1Response() const
392 : {
393 1 : MsqOperatorManager::CallV1SendT1Response();
394 1 : }
395 :
396 0 : int32_t MsqImplV2::ResetMsqT0Status() const
397 : {
398 0 : MsqOperatorManager::CallV2ResetT0Status();
399 0 : return AICPU_SCHEDULE_OK;
400 : }
401 :
402 0 : int32_t MsqImplV2::ResetMsqT1Status() const
403 : {
404 0 : MsqOperatorManager::CallV2ResetT1Status();
405 0 : return AICPU_SCHEDULE_OK;
406 : }
407 :
408 0 : MsqStatus MsqImplV2::ReadMsqT1Status() const
409 : {
410 0 : return MsqOperatorManager::CallV2ReadT1Status();
411 : }
412 :
413 0 : void MsqImplV2::ReadMsqT1Data(const uint32_t msgSize, MsqDatas &datas) const
414 : {
415 0 : if (msgSize == static_cast<uint32_t>(MsqDataSize::MSQ_DATA_SIZE_0)) {
416 0 : aicpusd_err("Message size is 0");
417 0 : return;
418 : }
419 :
420 0 : MsqOperatorManager::CallV2ReadT1Data(msgSize, &datas);
421 : }
422 :
423 0 : void __attribute__((optimize("O0"))) MsqImplV2::SendMsqT1Response() const
424 : {
425 0 : MsqOperatorManager::CallV2SendT1Response();
426 0 : }
427 : } // namespace AicpuSchedule
|
