Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "ccl_buffer_manager.h"
12 : #include "log.h"
13 : #include "externalinput_pub.h"
14 : #include "adapter_rts_common.h"
15 : #include "share_ccl_buffer_manager.h"
16 :
17 : namespace hccl {
18 817 : CCLBufferManager::CCLBufferManager()
19 817 : : inCCLbuffer_(DeviceMem()),
20 817 : outCCLbuffer_(DeviceMem()),
21 817 : winExpBuffer_(DeviceMem()),
22 817 : inCCLbufferSize_(0),
23 817 : outCCLbufferSize_(0),
24 817 : winExpBufferSize_(0),
25 817 : inAlltoAllvParaBuffer_(DeviceMem()),
26 1633 : outAlltoAllvParaBuffer_(DeviceMem())
27 816 : {}
28 :
29 817 : CCLBufferManager::~CCLBufferManager()
30 : {
31 817 : if (!static_cast<bool>(isShareCCLbuffer_)) {
32 817 : ReleaseCommCCLbuffer();
33 : }
34 817 : ReleaseAlltoAllvParaBuffer();
35 816 : ReleaseCommAIVbuffer();
36 817 : }
37 :
38 76 : HcclResult CCLBufferManager::CreateCCLbuffer(u64 size, DeviceMem& buffer)
39 : {
40 76 : CHK_PRT_RET(
41 : !size, HCCL_INFO("[CCLBufferManager][CreateCCLbuffer]buffer size is zero. not need to malloc memory"),
42 : HCCL_SUCCESS);
43 :
44 : CHK_PRT_RET(
45 : (size > ULONG_MAX),
46 : HCCL_ERROR("[CCLBufferManager][CreateCCLbuffer]buffer size is greater than %llu", ULONG_MAX), HCCL_E_PARA);
47 :
48 76 : CHK_RET(DeviceMem::alloc(buffer, size));
49 81 : HCCL_INFO("[CreateCCLbuffer] buffer ptr[%p], size[%llu]", buffer.ptr(), buffer.size());
50 81 : CHK_PRT_RET(
51 : size && !buffer,
52 : HCCL_ERROR(
53 : "[CCLBufferManager][CreateCCLbuffer]Create ccl buffer size[%llu] fail,"
54 : "please check environmental variable HCCL_BUFFSIZE.",
55 : size),
56 : HCCL_E_PTR);
57 81 : HCCL_RUN_INFO(
58 : "[HCCL_TRACE][CreateCCLbuffer]Create ccl buffer success. buffer ptr[%p], size[%llu]", buffer.ptr(),
59 : buffer.size());
60 81 : return HCCL_SUCCESS;
61 : }
62 :
63 81 : HcclResult CCLBufferManager::CreateCommCCLbuffer(const std::string& bufferName)
64 : {
65 81 : if (inCCLbufferSize_ == 0) {
66 14 : inCCLbufferSize_ = GetExternalInputCCLBuffSize();
67 : }
68 81 : if (outCCLbufferSize_ == 0) {
69 14 : outCCLbufferSize_ = GetExternalInputCCLBuffSize();
70 : }
71 81 : if (winExpBufferSize_ == 0) {
72 59 : winExpBufferSize_ = EXP_BUFFER_SIZE;
73 : }
74 :
75 81 : if (cclBuffer_.ptr() == nullptr) {
76 61 : u64 totalSize = inCCLbufferSize_ + outCCLbufferSize_ + winExpBufferSize_;
77 : // buffername非空则申请共享cclbuffer
78 61 : if (!bufferName.empty()) {
79 0 : CHK_RET(ShareCCLbufferMgr::GetInstance().CreateShareCCLbuffer(bufferName, totalSize, cclBuffer_));
80 0 : isShareCCLbuffer_ = true;
81 : } else {
82 64 : CHK_RET(CreateCCLbuffer(totalSize, cclBuffer_));
83 73 : CHK_RET(hrtMemSet(cclBuffer_.ptr(), totalSize, totalSize));
84 : }
85 : }
86 :
87 90 : if (inCCLbuffer_.ptr() == nullptr) {
88 73 : inCCLbuffer_ = DeviceMem::create(cclBuffer_.ptr(), inCCLbufferSize_);
89 : }
90 :
91 89 : if (outCCLbuffer_.ptr() == nullptr) {
92 73 : outCCLbuffer_ = DeviceMem::create(static_cast<u8*>(cclBuffer_.ptr()) + inCCLbufferSize_, outCCLbufferSize_);
93 : }
94 :
95 89 : if (winExpBuffer_.ptr() == nullptr) {
96 73 : winExpBuffer_ = DeviceMem::create(
97 73 : static_cast<u8*>(cclBuffer_.ptr()) + inCCLbufferSize_ + outCCLbufferSize_, winExpBufferSize_);
98 : }
99 89 : HCCL_INFO(
100 : "[CreateCommCCLbuffer] create cclbuffer, inPtr[%p], outPtr[%p], winExpPtr[%p], isSharebuffer[%d]",
101 : inCCLbuffer_.ptr(), outCCLbuffer_.ptr(), winExpBuffer_.ptr(), isShareCCLbuffer_);
102 91 : return HCCL_SUCCESS;
103 : }
104 :
105 0 : HcclResult CCLBufferManager::CleanCCLbuffer()
106 : {
107 0 : if (inCCLbuffer_.ptr() != nullptr) {
108 0 : CHK_RET(hrtMemSet(inCCLbuffer_.ptr(), inCCLbuffer_.size(), inCCLbuffer_.size()));
109 0 : HCCL_INFO("[CleanCCLbuffer] clean input buffer, ptr[%p], size[%llu]", inCCLbuffer_.ptr(), inCCLbuffer_.size());
110 : }
111 :
112 0 : if (outCCLbuffer_.ptr() != nullptr) {
113 0 : CHK_RET(hrtMemSet(outCCLbuffer_.ptr(), outCCLbuffer_.size(), outCCLbuffer_.size()));
114 0 : HCCL_INFO(
115 : "[CleanCCLbuffer] clean output buffer, ptr[%p], size[%llu]", outCCLbuffer_.ptr(), outCCLbuffer_.size());
116 : }
117 0 : return HCCL_SUCCESS;
118 : }
119 :
120 8 : HcclResult CCLBufferManager::CleanAIVbuffer(void* bufferPtr)
121 : {
122 : // 将aiv的bufferPtr空间置于0
123 8 : if (bufferPtr != nullptr) {
124 8 : CHK_RET(hrtMemSet(bufferPtr, AIV_FLAG_SIZE, AIV_FLAG_SIZE));
125 8 : HCCL_INFO("[CleanAIVbuffer] clean aiv buffer, ptr[%p], size[%llu]", bufferPtr, AIV_FLAG_SIZE);
126 : }
127 8 : return HCCL_SUCCESS;
128 : }
129 :
130 4 : HcclResult CCLBufferManager::CreateCommAIVbuffer(bool useOpbaseFlag)
131 : {
132 4 : if (useOpbaseFlag) {
133 2 : if (inAivOpbaseBuffer_.ptr() == nullptr) {
134 2 : CHK_RET(CreateCCLbuffer(AIV_DATA_SIZE, inAivOpbaseBuffer_));
135 2 : CHK_RET(CleanAIVbuffer(static_cast<u8*>(inAivOpbaseBuffer_.ptr()) + (AIV_DATA_SIZE - AIV_FLAG_SIZE)));
136 : }
137 2 : if (outAivOpbaseBuffer_.ptr() == nullptr) {
138 2 : CHK_RET(CreateCCLbuffer(AIV_FLAG_SIZE, outAivOpbaseBuffer_));
139 2 : CHK_RET(CleanAIVbuffer(outAivOpbaseBuffer_.ptr()));
140 2 : HCCL_RUN_INFO("[HCCL_TRACE][CreateCommAIVbuffer] OpbaseMode");
141 : }
142 : } else {
143 2 : if (inAivOffloadbuffer_.ptr() == nullptr) {
144 2 : CHK_RET(CreateCCLbuffer(AIV_DATA_SIZE, inAivOffloadbuffer_));
145 2 : CHK_RET(CleanAIVbuffer(static_cast<u8*>(inAivOffloadbuffer_.ptr()) + (AIV_DATA_SIZE - AIV_FLAG_SIZE)));
146 : }
147 2 : if (outAivOffloadbuffer_.ptr() == nullptr) {
148 2 : CHK_RET(CreateCCLbuffer(AIV_FLAG_SIZE, outAivOffloadbuffer_));
149 2 : CHK_RET(CleanAIVbuffer(outAivOffloadbuffer_.ptr()));
150 2 : HCCL_RUN_INFO("[HCCL_TRACE][CreateCommAIVbuffer] OffloadMode");
151 : }
152 : }
153 4 : return HCCL_SUCCESS;
154 : }
155 :
156 0 : HcclResult CCLBufferManager::CreateCommInfoAIVbuffer()
157 : {
158 0 : if (aivCommInfoBuffer_.ptr() == nullptr) {
159 0 : CHK_RET(CreateCCLbuffer(AIV_COMM_INFO_SIZE, aivCommInfoBuffer_));
160 : }
161 0 : return HCCL_SUCCESS;
162 : }
163 :
164 817 : HcclResult CCLBufferManager::ReleaseCommCCLbuffer()
165 : {
166 1561 : if ((cclBuffer_.ptr() == nullptr) && (inCCLbuffer_.ptr() == nullptr) && (outCCLbuffer_.ptr() == nullptr)
167 1558 : && (winExpBuffer_.ptr() == nullptr)) {
168 741 : HCCL_RUN_INFO("[HCCL_TRACE][ReleaseCCLbuffer]CCLBuffer is null, no need to release.");
169 744 : return HCCL_SUCCESS;
170 : }
171 :
172 73 : if (cclBuffer_.ptr() != nullptr) {
173 73 : HCCL_RUN_INFO(
174 : "[HCCL_TRACE][ReleaseCCLbuffer]Release cclBuffer. buffer ptr[%p], size[%llu]", cclBuffer_.ptr(),
175 : cclBuffer_.size());
176 73 : cclBuffer_.free();
177 : }
178 :
179 73 : if (inCCLbuffer_.ptr() != nullptr) {
180 73 : HCCL_RUN_INFO(
181 : "[HCCL_TRACE][ReleaseCCLbuffer]Release incclBuffer. buffer ptr[%p], size[%llu]", inCCLbuffer_.ptr(),
182 : inCCLbuffer_.size());
183 73 : inCCLbuffer_.free();
184 : }
185 :
186 73 : if (outCCLbuffer_.ptr() != nullptr) {
187 73 : HCCL_RUN_INFO(
188 : "[HCCL_TRACE][ReleaseCCLbuffer]Release outcclBuffer. buffer ptr[%p], size[%llu]", outCCLbuffer_.ptr(),
189 : outCCLbuffer_.size());
190 73 : outCCLbuffer_.free();
191 : }
192 :
193 73 : if (winExpBuffer_.ptr() != nullptr) {
194 73 : HCCL_RUN_INFO(
195 : "[HCCL_TRACE][ReleaseCCLbuffer]Release expcclBuffer. buffer ptr[%p], size[%llu]", winExpBuffer_.ptr(),
196 : winExpBuffer_.size());
197 73 : winExpBuffer_.free();
198 : }
199 :
200 146 : if ((cclBuffer_.ptr() == nullptr) && (inCCLbuffer_.ptr() == nullptr) && (outCCLbuffer_.ptr() == nullptr)
201 146 : && (winExpBuffer_.ptr() == nullptr)) {
202 73 : HCCL_RUN_INFO("[HCCL_TRACE][ReleaseCCLbuffer]Release CCLbuffer success.");
203 : }
204 73 : return HCCL_SUCCESS;
205 : }
206 :
207 817 : HcclResult CCLBufferManager::ReleaseCommAIVbuffer()
208 : {
209 817 : HCCL_RUN_INFO(
210 : "[HCCL_TRACE][ReleaseAIVbuffer]Release inAivOpbaseBuffer. buffer ptr[%p], size[%llu]", inAivOpbaseBuffer_.ptr(),
211 : inAivOpbaseBuffer_.size());
212 816 : inAivOpbaseBuffer_.free();
213 814 : HCCL_RUN_INFO(
214 : "[HCCL_TRACE][ReleaseAIVbuffer]Release outAivOpbaseBuffer. buffer ptr[%p], size[%llu]",
215 : outAivOpbaseBuffer_.ptr(), outAivOpbaseBuffer_.size());
216 817 : outAivOpbaseBuffer_.free();
217 817 : HCCL_RUN_INFO(
218 : "[HCCL_TRACE][ReleaseAIVbuffer]Release inAivOffloadbuffer. buffer ptr[%p], size[%llu]",
219 : inAivOffloadbuffer_.ptr(), inAivOffloadbuffer_.size());
220 817 : inAivOffloadbuffer_.free();
221 817 : HCCL_RUN_INFO(
222 : "[HCCL_TRACE][ReleaseAIVbuffer]Release outAivOffloadbuffer. buffer ptr[%p], size[%llu]",
223 : outAivOffloadbuffer_.ptr(), outAivOffloadbuffer_.size());
224 817 : outAivOffloadbuffer_.free();
225 817 : HCCL_RUN_INFO(
226 : "[HCCL_TRACE][ReleaseAIVbuffer]Release aivCommInfoBuffer. buffer ptr[%p], size[%llu]", aivCommInfoBuffer_.ptr(),
227 : aivCommInfoBuffer_.size());
228 817 : aivCommInfoBuffer_.free();
229 1634 : if (inAivOpbaseBuffer_.ptr() == nullptr && outAivOpbaseBuffer_.ptr() == nullptr
230 817 : && inAivOffloadbuffer_.ptr() == nullptr && outAivOffloadbuffer_.ptr() == nullptr
231 1634 : && aivCommInfoBuffer_.ptr() == nullptr) {
232 817 : HCCL_RUN_INFO("[HCCL_TRACE][ReleaseAIVbuffer]Release AIV buffer success.");
233 : }
234 817 : return HCCL_SUCCESS;
235 : }
236 :
237 0 : HcclResult CCLBufferManager::ClearCommAIVbuffer()
238 : {
239 0 : if (inAivOpbaseBuffer_.ptr() != nullptr) {
240 0 : CHK_RET(CleanAIVbuffer(static_cast<u8*>(inAivOpbaseBuffer_.ptr()) + (AIV_DATA_SIZE - AIV_FLAG_SIZE)));
241 : }
242 0 : if (outAivOpbaseBuffer_.ptr() != nullptr) {
243 0 : CHK_RET(CleanAIVbuffer(outAivOpbaseBuffer_.ptr()));
244 : }
245 0 : if (inAivOffloadbuffer_.ptr() != nullptr) {
246 0 : CHK_RET(CleanAIVbuffer(static_cast<u8*>(inAivOffloadbuffer_.ptr()) + (AIV_DATA_SIZE - AIV_FLAG_SIZE)));
247 : }
248 0 : if (outAivOffloadbuffer_.ptr() != nullptr) {
249 0 : CHK_RET(CleanAIVbuffer(outAivOffloadbuffer_.ptr()));
250 : }
251 0 : return HCCL_SUCCESS;
252 : }
253 :
254 2 : DeviceMem& CCLBufferManager::GetInAivOpbaseBuffer() { return inAivOpbaseBuffer_; }
255 :
256 2 : DeviceMem& CCLBufferManager::GetOutAivOpbaseBuffer() { return outAivOpbaseBuffer_; }
257 :
258 2 : DeviceMem& CCLBufferManager::GetInAivOffloadbuffer() { return inAivOffloadbuffer_; }
259 :
260 2 : DeviceMem& CCLBufferManager::GetOutAivOffloadbuffer() { return outAivOffloadbuffer_; }
261 :
262 113 : DeviceMem& CCLBufferManager::GetCommCCLBuffer() { return cclBuffer_; }
263 :
264 242 : HcclResult CCLBufferManager::InitCCLbuffer(u64 inCCLbufferSize, u64 outCCLbufferSize)
265 : {
266 242 : inCCLbufferSize_ = inCCLbufferSize;
267 242 : outCCLbufferSize_ = outCCLbufferSize;
268 242 : return HCCL_SUCCESS;
269 : }
270 :
271 125 : void* CCLBufferManager::GetCCLbufferAddr(const DeviceMem& buffer)
272 : {
273 125 : if (buffer.ptr() == nullptr) {
274 60 : return nullptr;
275 : } else {
276 63 : return static_cast<void*>(reinterpret_cast<u8*>(buffer.ptr()));
277 : }
278 : }
279 :
280 218 : DeviceMem& CCLBufferManager::GetInCCLbuffer() { return inCCLbuffer_; }
281 :
282 2 : DeviceMem& CCLBufferManager::GetCommExpBuffer() { return winExpBuffer_; }
283 :
284 0 : DeviceMem& CCLBufferManager::GetAivCommInfoBuffer() { return aivCommInfoBuffer_; }
285 :
286 78 : HcclResult CCLBufferManager::GetInCCLbuffer(void*& buffer, u64& size)
287 : {
288 78 : buffer = GetCCLbufferAddr(inCCLbuffer_);
289 76 : size = inCCLbufferSize_;
290 76 : return HCCL_SUCCESS;
291 : }
292 :
293 267 : u64 CCLBufferManager::GetInCCLbufferSize() { return inCCLbufferSize_; }
294 :
295 222 : DeviceMem& CCLBufferManager::GetOutCCLbuffer() { return outCCLbuffer_; }
296 :
297 37 : HcclResult CCLBufferManager::GetOutCCLbuffer(void*& buffer, u64& size)
298 : {
299 37 : buffer = GetCCLbufferAddr(outCCLbuffer_);
300 37 : size = outCCLbufferSize_;
301 37 : return HCCL_SUCCESS;
302 : }
303 :
304 154 : u64 CCLBufferManager::GetOutCCLbufferSize() { return outCCLbufferSize_; }
305 :
306 2 : u64 CCLBufferManager::GetExpBufferSize() { return winExpBufferSize_; }
307 :
308 2 : DeviceMem CCLBufferManager::GetCommRegMem(const DeviceMem& mem, MemAttr memAttr, bool aivMode)
309 : {
310 2 : u64 commMemSize = 0;
311 2 : if ((GetWorkflowMode() == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE) && (!aivMode)) {
312 : // 单算子模式时，仅在第一次集合通信时创建子通信域，注册通信内存。需要将整个CCLbuffer注册进通信域。
313 2 : if (memAttr == MemAttr::IN_CCL_BUFFER) {
314 1 : commMemSize = inCCLbufferSize_;
315 1 : } else if (memAttr == MemAttr::OUT_CCL_BUFFER) {
316 1 : commMemSize = outCCLbufferSize_;
317 : }
318 : } else {
319 0 : commMemSize = mem.size();
320 : }
321 2 : DeviceMem commMem = DeviceMem::create(mem.ptr(), commMemSize);
322 2 : return commMem;
323 : }
324 :
325 0 : HcclResult CCLBufferManager::InitAlltoAllvParaBuffer(u64 inBufferSize, u64 outBufferSize)
326 : {
327 0 : CHK_RET(CreateCCLbuffer(inBufferSize, inAlltoAllvParaBuffer_));
328 0 : CHK_RET(CreateCCLbuffer(outBufferSize, outAlltoAllvParaBuffer_));
329 0 : return HCCL_SUCCESS;
330 : }
331 :
332 0 : DeviceMem& CCLBufferManager::GetInAlltoAllvParaBuffer() { return inAlltoAllvParaBuffer_; }
333 :
334 0 : DeviceMem& CCLBufferManager::GetOutAlltoAllvParaBuffer() { return outAlltoAllvParaBuffer_; }
335 :
336 1339 : void CCLBufferManager::ReleaseAlltoAllvParaBuffer()
337 : {
338 1339 : inAlltoAllvParaBuffer_.free();
339 1339 : outAlltoAllvParaBuffer_.free();
340 1339 : }
341 :
342 7 : HcclResult CCLBufferManager::GetIndependentOpCCLbuffer(void*& buffer, uint64_t& size)
343 : {
344 7 : HCCL_INFO("[GetIndependentOpCCLbuffer] cclBuffer_[%p]", cclBuffer_.ptr());
345 7 : buffer = GetCCLbufferAddr(cclBuffer_);
346 7 : if (buffer == nullptr) {
347 8 : CHK_RET(CreateCommCCLbuffer());
348 4 : buffer = GetCCLbufferAddr(cclBuffer_);
349 : }
350 : // 大小在通信域初始化时调取InitCCLbuffer设置，MC1MB内存不对外暴露
351 7 : size = inCCLbufferSize_ + outCCLbufferSize_;
352 7 : return HCCL_SUCCESS;
353 : }
354 : } // namespace hccl
|
