Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "adapter_rts.h"
12 : #include "adapter_error_manager.h"
13 : #include "sal.h"
14 : #include "stream_pub.h"
15 :
16 : namespace hccl {
17 : // 默认构造函数只产生无效的Stream对象
18 12039 : Stream::Stream() : stream_(nullptr), device_id_(HCCL_DEVICE_NOT_SET), stream_owner_(false), streamId_(0),
19 6016 : isMainStream_(true), modeGotFlag_(false), streamMode_(0), sqId_(0), ctx_(nullptr), cqId_(0), logicCqid_(0) {}
20 :
21 2323 : Stream::Stream(const Stream &that)
22 6967 : : stream_(that.ptr()), device_id_(that.device_id_), stream_owner_(false), streamId_(that.streamId_),
23 2321 : isMainStream_(that.isMainStream_), modeGotFlag_(that.modeGotFlag_), streamMode_(that.streamMode_),
24 2321 : sqId_(that.sqId_), ctx_(that.ctx_), cqId_(that.cqId_), logicCqid_(that.logicCqid_),
25 2323 : sqeContext_(that.sqeContext_), cqeContext_(that.cqeContext_), streamInfo_(that.streamInfo_) {}
26 :
27 593 : Stream::Stream(Stream &&that)
28 1779 : : stream_(that.ptr()), device_id_(that.device_id_), stream_owner_(that.stream_owner_), streamId_(that.streamId_),
29 593 : isMainStream_(that.isMainStream_), modeGotFlag_(that.modeGotFlag_), streamMode_(that.streamMode_),
30 593 : sqId_(that.sqId_), ctx_(that.ctx_), cqId_(that.cqId_), logicCqid_(that.logicCqid_),
31 593 : sqeContext_(that.sqeContext_), cqeContext_(that.cqeContext_), streamInfo_(that.streamInfo_)
32 : {
33 593 : that.stream_ = nullptr;
34 593 : that.device_id_ = HCCL_DEVICE_NOT_SET;
35 593 : that.stream_owner_ = false;
36 593 : that.streamId_ = 0;
37 593 : that.isMainStream_ = true;
38 593 : that.modeGotFlag_ = false;
39 593 : that.streamMode_ = 0;
40 593 : that.sqId_ = 0;
41 593 : that.cqId_ = 0;
42 593 : that.logicCqid_ = 0;
43 593 : that.sqeContext_ = nullptr;
44 593 : that.cqeContext_ = nullptr;
45 593 : that.streamInfo_.actualStreamId = 0;
46 593 : that.streamInfo_.logicCqId = 0;
47 593 : that.streamInfo_.sqBaseAddr = nullptr;
48 593 : that.streamInfo_.sqDepth = 0;
49 593 : that.streamInfo_.sqId = 0;
50 593 : }
51 :
52 681 : Stream::Stream(const StreamType streamType, bool isMainStream)
53 1359 : : stream_(nullptr), device_id_(HCCL_DEVICE_NOT_SET), stream_owner_(true), streamId_(0),
54 681 : isMainStream_(isMainStream), modeGotFlag_(false), streamMode_(0), sqId_(0), ctx_(nullptr), cqId_(0), logicCqid_(0)
55 : {
56 : HcclResult ret;
57 678 : aclrtStream rtStream = nullptr;
58 :
59 : // 申请rtStream
60 678 : if (streamType == StreamType::STREAM_TYPE_ONLINE) {
61 281 : ret = hrtStreamCreateWithFlags(&rtStream, HCCL_STREAM_PRIORITY_HIGH,
62 : ACL_STREAM_FAST_LAUNCH | ACL_STREAM_FAST_SYNC);
63 397 : } else if (streamType == StreamType::STREAM_TYPE_DEVICE) {
64 274 : ret = hrtStreamCreateWithFlags(&rtStream, HCCL_STREAM_PRIORITY_HIGH,
65 : ACL_STREAM_DEVICE_USE_ONLY);
66 : } else {
67 123 : ret = hrtStreamCreateWithFlags(&rtStream, HCCL_STREAM_PRIORITY_LOW,
68 : ACL_STREAM_PERSISTENT);
69 : }
70 :
71 681 : if (ret == HCCL_SUCCESS) {
72 676 : HCCL_DEBUG("rtStreamCreate ok, streamType[%d]", streamType);
73 676 : stream_ = const_cast<void *>(rtStream);
74 676 : InitStream();
75 676 : HCCL_INFO("Construct stream by stream type success, ptr[%p] ctx[%p], stream id[%d], cqId[%d], logicCqid[%d]",
76 : rtStream, ctx_, streamId_, cqId_, logicCqid_);
77 : } else {
78 55 : RPT_ENV_ERR(true, "EI0007", std::vector<std::string>({"resource_type", "resource_info"}), \
79 : std::vector<std::string>({"stream", std::string("StreamCreateWithFlags, streamType:") + std::to_string(uint32_t(streamType))}));
80 5 : HCCL_ERROR("[%s][%s]Construct stream by stream type failed, errNo[0x%016llx] rtStreamCreate error, ret[%d]",
81 : LOG_KEYWORDS_INIT_GROUP.c_str(), LOG_KEYWORDS_RESOURCE.c_str(), HCCL_ERROR_CODE(HCCL_E_RUNTIME), ret);
82 : }
83 681 : bool isSupportV2 = false;
84 681 : CHK_PRT_CONT(hrtGetHcclV2Support(&isSupportV2), HCCL_WARNING("[Stream] Can not check hccl version"));
85 681 : if (isSupportV2 && streamType == StreamType::STREAM_TYPE_ONLINE) {
86 66 : HcclResult setModeRet = hrtStreamSetMode(stream_, STREAM_MODE_STOP_ON_FAILURE);
87 66 : if (setModeRet != HCCL_SUCCESS) {
88 0 : HCCL_ERROR("[Stream][SetMode]Failed to set stream mode, errNo[0x%016llx], ret[%d], stream id[%d]",
89 : HCCL_ERROR_CODE(setModeRet), setModeRet, streamId_);
90 : }
91 : }
92 696 : }
93 :
94 469 : Stream::Stream(const rtStream_t rtStream, bool isMainStream)
95 902 : : stream_(const_cast<void *>(rtStream)), device_id_(HCCL_DEVICE_NOT_SET), stream_owner_(false), streamId_(0),
96 469 : isMainStream_(isMainStream), modeGotFlag_(false), streamMode_(0), sqId_(0), ctx_(nullptr), cqId_(0), logicCqid_(0)
97 : {
98 433 : InitStream();
99 469 : }
100 :
101 500 : Stream::Stream(const HcclComStreamInfo &streamInfo, bool isMainStream)
102 500 : : stream_(static_cast<void *>(streamInfo.sqBaseAddr)), device_id_(HCCL_DEVICE_NOT_SET), stream_owner_(false),
103 500 : streamId_(streamInfo.actualStreamId), isMainStream_(isMainStream), modeGotFlag_(false), streamMode_(0),
104 500 : sqId_(streamInfo.sqId), ctx_(nullptr)
105 : {
106 500 : SetStreamInfo(streamInfo);
107 500 : }
108 :
109 10788 : Stream::~Stream()
110 : {
111 10586 : DestroyStream();
112 10784 : }
113 :
114 432 : Stream &Stream::operator=(const Stream &that)
115 : {
116 432 : if (&that != this) {
117 432 : stream_ = that.ptr();
118 431 : device_id_ = that.device_id_;
119 431 : stream_owner_ = false;
120 431 : taskLogicInfo_ = that.taskLogicInfo_;
121 400 : streamId_ = that.streamId_;
122 400 : isMainStream_ = that.isMainStream_;
123 400 : modeGotFlag_ = that.modeGotFlag_;
124 400 : streamMode_ = that.streamMode_;
125 400 : sqId_ = that.sqId_;
126 400 : ctx_ = that.ctx_;
127 400 : cqId_ = that.cqId_;
128 400 : logicCqid_ = that.logicCqid_;
129 400 : sqeContext_ = that.sqeContext_;
130 400 : cqeContext_ = that.cqeContext_;
131 400 : streamInfo_.actualStreamId = that.streamInfo_.actualStreamId;
132 400 : streamInfo_.logicCqId = that.streamInfo_.logicCqId;
133 400 : streamInfo_.sqBaseAddr = that.streamInfo_.sqBaseAddr;
134 400 : streamInfo_.sqDepth = that.streamInfo_.sqDepth;
135 400 : streamInfo_.sqId = that.streamInfo_.sqId;
136 : }
137 400 : return *this;
138 : }
139 :
140 582 : Stream Stream::operator=(Stream &&that)
141 : {
142 582 : if (&that != this) {
143 582 : stream_ = that.stream_;
144 582 : device_id_ = that.device_id_;
145 582 : stream_owner_ = that.stream_owner_;
146 582 : taskLogicInfo_ = that.taskLogicInfo_;
147 582 : streamId_ = that.streamId_;
148 582 : isMainStream_ = that.isMainStream_;
149 582 : modeGotFlag_ = that.modeGotFlag_;
150 582 : streamMode_ = that.streamMode_;
151 582 : sqId_ = that.sqId_;
152 582 : ctx_ = that.ctx_;
153 582 : cqId_ = that.cqId_;
154 582 : logicCqid_ = that.logicCqid_;
155 582 : sqeContext_ = that.sqeContext_;
156 582 : cqeContext_ = that.cqeContext_;
157 582 : streamInfo_.actualStreamId = that.streamInfo_.actualStreamId;
158 582 : streamInfo_.logicCqId = that.streamInfo_.logicCqId;
159 582 : streamInfo_.sqBaseAddr = that.streamInfo_.sqBaseAddr;
160 582 : streamInfo_.sqDepth = that.streamInfo_.sqDepth;
161 582 : streamInfo_.sqId = that.streamInfo_.sqId;
162 : }
163 :
164 582 : that.stream_ = nullptr;
165 582 : that.device_id_ = HCCL_DEVICE_NOT_SET;
166 582 : that.stream_owner_ = false;
167 582 : that.taskLogicInfo_ = taskLogicInfo_;
168 582 : that.streamId_ = 0;
169 582 : that.isMainStream_ = isMainStream_;
170 582 : that.modeGotFlag_ = modeGotFlag_;
171 582 : that.streamMode_ = streamMode_;
172 582 : that.sqId_ = sqId_;
173 582 : that.ctx_ = ctx_;
174 582 : that.cqId_ = cqId_;
175 582 : that.logicCqid_ = logicCqid_;
176 582 : that.sqeContext_ = nullptr;
177 582 : that.cqeContext_ = nullptr;
178 582 : that.streamInfo_.actualStreamId = streamInfo_.actualStreamId;
179 582 : that.streamInfo_.logicCqId = streamInfo_.logicCqId;
180 582 : that.streamInfo_.sqBaseAddr = nullptr;
181 582 : that.streamInfo_.sqDepth = streamInfo_.sqDepth;
182 582 : that.streamInfo_.sqId = streamInfo_.sqId;
183 582 : return *this;
184 : }
185 :
186 10585 : void Stream::DestroyStream()
187 : {
188 : // 销毁stream
189 10585 : if (stream_owner_ && stream_ != nullptr) {
190 : // stream需要在原ctx上销毁
191 666 : aclrtContext ctxTmp = nullptr;
192 666 : HcclResult ret = hrtCtxGetCurrent(&ctxTmp);
193 666 : bool needChangeCtx = (ret == HCCL_SUCCESS && ctx_ != nullptr);
194 666 : if (needChangeCtx) {
195 663 : ret = hrtCtxSetCurrent(ctx_);
196 663 : HCCL_INFO("Switch Ctx ret[%d], curCtx[%p], setCtx[%p], stream id[%d]",
197 : ret, ctxTmp, ctx_, streamId_);
198 : }
199 666 : ret = hrtStreamDestroy(stream_);
200 666 : HCCL_RUN_INFO("[HCCL_TRACE]StreamDestroy, streamPtr[%p], stream id[%d]", stream_, streamId_);
201 666 : if (ret != HCCL_SUCCESS) {
202 0 : HCCL_WARNING("errNo[0x%016llx] hrtStreamDestroy error, ret[%d]",
203 : HCCL_ERROR_CODE(HCCL_E_RUNTIME), ret);
204 : }
205 666 : if (needChangeCtx) {
206 663 : ret = hrtCtxSetCurrent(ctxTmp);
207 663 : HCCL_INFO("Restore Ctx ret[%d], setCtx[%p], stream id[%d]",
208 : ret, ctxTmp, streamId_);
209 : }
210 : }
211 10585 : }
212 :
213 2 : void Stream::SetEmpty()
214 : {
215 2 : DestroyStream();
216 2 : stream_ = nullptr;
217 2 : device_id_ = HCCL_DEVICE_NOT_SET;
218 2 : stream_owner_ = false;
219 2 : streamId_ = 0;
220 2 : isMainStream_ = true;
221 2 : sqId_ = 0;
222 2 : ctx_ = nullptr;
223 2 : cqId_ = 0;
224 2 : logicCqid_ = 0;
225 2 : }
226 :
227 1107 : HcclResult Stream::InitStream()
228 : {
229 1107 : if (stream_ != nullptr) {
230 1107 : HcclResult ret = hrtGetStreamId(stream_, streamId_);
231 1146 : if (ret != HCCL_SUCCESS) {
232 0 : SetEmpty();
233 0 : HCCL_ERROR("[InitStream]Failed to get the streamId through the rtstream, errNo[0x%016llx]" \
234 : "hrtGetStreamId error, ret[%d]", HCCL_ERROR_CODE(HCCL_E_RUNTIME), ret);
235 0 : return HCCL_E_INTERNAL;
236 : }
237 :
238 1146 : ret = hrtStreamGetSqid(stream_, &(sqId_));
239 1147 : if (ret != HCCL_SUCCESS) {
240 2 : SetEmpty();
241 2 : HCCL_ERROR("[InitStream]Failed to get the sqId through the rtstream, errNo[0x%016llx]" \
242 : "hrtStreamGetSqid error, ret[%d]", HCCL_ERROR_CODE(HCCL_E_RUNTIME), ret);
243 2 : return HCCL_E_INTERNAL;
244 : }
245 1145 : (void)hrtCtxGetCurrent(&ctx_);
246 :
247 1139 : ret = hrtStreamGetCqid(stream_, &(cqId_), &(logicCqid_));
248 1144 : if (ret != HCCL_SUCCESS) {
249 0 : SetEmpty();
250 0 : HCCL_ERROR("[InitStream]Failed to get the cqId through the rtstream, errNo[0x%016llx]" \
251 : "hrtStreamGetCqid error, ret[%d]", HCCL_ERROR_CODE(HCCL_E_RUNTIME), ret);
252 0 : return HCCL_E_INTERNAL;
253 : }
254 : }
255 1144 : return HCCL_SUCCESS;
256 : }
257 :
258 106 : HcclResult Stream::SetMode(const uint64_t stmMode)
259 : {
260 106 : HcclResult ret = hrtStreamSetMode(stream_, stmMode);
261 106 : if (ret != HCCL_SUCCESS) {
262 1 : HCCL_ERROR("[Stream][SetMode]errNo[0x%016llx] hrtStreamSetMode error, ret[%d]",
263 : HCCL_ERROR_CODE(HCCL_E_RUNTIME), ret);
264 1 : return HCCL_E_INTERNAL;
265 : }
266 105 : return HCCL_SUCCESS;
267 : }
268 :
269 99 : HcclResult Stream::GetMode(uint64_t *const stmMode)
270 : {
271 99 : if (modeGotFlag_ == false) {
272 99 : HcclResult ret = hrtStreamGetMode(stream_, &streamMode_);
273 99 : if (ret != HCCL_SUCCESS) {
274 0 : HCCL_ERROR("[Stream][GetMode]errNo[0x%016llx] hrtStreamGetMode error, ret[%d]",
275 : HCCL_ERROR_CODE(HCCL_E_RUNTIME), ret);
276 0 : return HCCL_E_INTERNAL;
277 : }
278 : }
279 99 : *stmMode = streamMode_;
280 99 : return HCCL_SUCCESS;
281 : }
282 :
283 0 : void Stream::PushTaskLogicInfo(TaskLogicInfo &taskLogicInfo)
284 : {
285 0 : taskLogicInfo_.push(taskLogicInfo);
286 0 : HCCL_INFO("[PushTaskLogicInfo] stream[%p], taskLogicType[%d], taskLogicFuncType[%d], taskLogicInfo size[%d]",
287 : stream_, taskLogicInfo.taskLogicCmd.taskLogicType, taskLogicInfo.taskFuncType, taskLogicInfo_.size());
288 0 : }
289 :
290 0 : HcclResult Stream::PopTaskLogicInfo(TaskLogicInfo &taskLogicInfo)
291 : {
292 0 : if (taskLogicInfo_.size() > 0) {
293 0 : taskLogicInfo = taskLogicInfo_.front();
294 0 : HCCL_INFO("[PopTaskLogicInfo] stream[%p], taskLogicType[%d], taskLogicFuncType[%d], taskLogicInfo size[%d]",
295 : stream_, taskLogicInfo.taskLogicCmd.taskLogicType, taskLogicInfo.taskFuncType, taskLogicInfo_.size());
296 0 : taskLogicInfo_.pop();
297 0 : return HCCL_SUCCESS;
298 : }
299 0 : return HCCL_E_NOT_FOUND;
300 : }
301 :
302 20 : HcclResult Stream::GetNextSqeBufferAddr(uint8_t *&sqeBufferAddr, uint8_t *&sqeTypeAddr, uint8_t *&sqeDfxInfoAddr,
303 : uint16_t &taskId)
304 : {
305 20 : if (UNLIKELY(sqeContext_ == nullptr)) {
306 0 : HCCL_ERROR("[Stream][GetNextSqeBufferAddr] Sqe context is null");
307 0 : return HCCL_E_INTERNAL;
308 : }
309 20 : auto &buff = sqeContext_->buffer;
310 20 : if (UNLIKELY(buff.tailSqeIdx >= HCCL_SQE_MAX_CNT)) {
311 0 : HCCL_INFO("[Stream][GetNextSqeBufferAddr] Sqe index to 2048, need clear");
312 0 : if (buff.sqeCnt != 0) {
313 0 : HCCL_ERROR("[Stream][GetNextSqeBufferAddr] Sqe index to 2048, but sqeCnt is not 0");
314 0 : return HCCL_E_INTERNAL;
315 : }
316 0 : CHK_RET(ClearLocalBuff());
317 : }
318 20 : sqeBufferAddr = buff.localBuff + buff.tailSqeIdx * HCCL_SQE_SIZE;
319 20 : sqeTypeAddr = &buff.sqeType[buff.tailSqeIdx];
320 20 : sqeDfxInfoAddr = reinterpret_cast<uint8_t*>(&buff.dfxInfo[buff.tailSqeIdx]);
321 :
322 20 : buff.profTimestap[buff.tailSqeIdx] = ProfGetCurCpuTimestamp();
323 20 : taskId = buff.tailSqeTaskId;
324 :
325 20 : HCCL_DEBUG("[Stream][GetNextSqeBufferAddr] streamId: %u Get next idx:%u, taskId:%u, flipNum:%u",
326 : streamInfo_.actualStreamId, buff.tailSqeIdx, taskId, buff.filpNum);
327 20 : if (UNLIKELY(buff.tailSqeTaskId == UINT16_MAX)) {
328 0 : buff.filpNum++;
329 0 : HCCL_WARNING("[Stream][GetNextSqeBufferAddr] Sqe context cur taskId is uint16_max");
330 : }
331 20 : buff.tailSqeTaskId++;
332 20 : buff.sqeCnt++;
333 20 : buff.tailSqeIdx++;
334 20 : return HCCL_SUCCESS;
335 : }
336 :
337 496 : HcclResult Stream::InitSqAndCqeContext(uint32_t sqHead, uint32_t sqTail, SqCqeContext* context)
338 : {
339 496 : CHK_PTR_NULL(context);
340 496 : sqeContext_ = &context->sqContext;
341 496 : CHK_PTR_NULL(sqeContext_);
342 496 : cqeContext_ = &context->cqeContext;
343 496 : CHK_PTR_NULL(cqeContext_);
344 :
345 496 : auto &buff = sqeContext_->buffer;
346 496 : buff.sqHead = sqHead;
347 496 : buff.sqTail = sqTail;
348 496 : cqeContext_->cqeStatus = 0;
349 496 : HCCL_INFO("%s success, streamId:%u, sqHead:%u, sqTail:%u, context:%p", __func__, streamId_, sqHead, sqTail, context);
350 496 : return HCCL_SUCCESS;
351 : }
352 :
353 6 : HcclResult Stream::ClearLocalBuff()
354 : {
355 6 : CHK_PTR_NULL(sqeContext_);
356 1 : auto &buff = sqeContext_->buffer;
357 1 : if (memset_s(buff.localBuff, sizeof(buff.localBuff), 0, buff.tailSqeIdx * HCCL_SQE_SIZE) != EOK) {
358 0 : HCCL_ERROR("[Stream][ClearLocalBuff] clear local buff failed");
359 0 : return HCCL_E_MEMORY;
360 : }
361 1 : if (memset_s(buff.sqeType, sizeof(buff.sqeType), 0, buff.tailSqeIdx) != EOK) {
362 0 : HCCL_ERROR("[Stream][ClearLocalBuff] clear sqe type failed");
363 0 : return HCCL_E_MEMORY;
364 : }
365 1 : if (memset_s(buff.addInfo, sizeof(buff.addInfo), 0, buff.tailSqeIdx) != EOK) {
366 0 : HCCL_ERROR("[Stream][ClearLocalBuff] clear add info failed");
367 0 : return HCCL_E_MEMORY;
368 : }
369 1 : buff.sqeCnt = 0;
370 1 : buff.tailSqeIdx = 0;
371 :
372 1 : if (cqeContext_ != nullptr && memset_s(cqeContext_, sizeof(ErrCqeContext), 0, sizeof(ErrCqeContext)) != EOK) {
373 0 : HCCL_ERROR("[Stream][ClearLocalBuff] clear cqe context failed");
374 0 : return HCCL_E_MEMORY;
375 : }
376 1 : return HCCL_SUCCESS;
377 : }
378 :
379 0 : HcclResult Stream::SetCqeContext(const ErrCqeContext &cqeContext)
380 : {
381 0 : CHK_PTR_NULL(cqeContext_);
382 0 : *cqeContext_ = cqeContext;
383 0 : return HCCL_SUCCESS;
384 : }
385 :
386 0 : HcclResult Stream::GetCqeContext(ErrCqeContext &cqeContext)
387 : {
388 0 : CHK_PTR_NULL(cqeContext_);
389 0 : cqeContext = *cqeContext_;
390 0 : return HCCL_SUCCESS;
391 : }
392 :
393 5 : HcclResult Stream::GetStreamInfo(const HcclComStreamInfo *&streamInfo)
394 : {
395 5 : streamInfo = &streamInfo_;
396 5 : return HCCL_SUCCESS;
397 : }
398 : } // namespace hccl
|
