Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "ccl_buffer_manager.h"
12 : #include "log.h"
13 : #include "externalinput_pub.h"
14 : #include "adapter_rts_common.h"
15 : #include "share_ccl_buffer_manager.h"
16 :
17 : namespace hccl {
18 814 : CCLBufferManager::CCLBufferManager()
19 814 : :inCCLbuffer_(DeviceMem()), outCCLbuffer_(DeviceMem()), winExpBuffer_(DeviceMem()),
20 814 : inCCLbufferSize_(0), outCCLbufferSize_(0), winExpBufferSize_(0),
21 1628 : inAlltoAllvParaBuffer_(DeviceMem()), outAlltoAllvParaBuffer_(DeviceMem())
22 : {
23 814 : }
24 :
25 814 : CCLBufferManager::~CCLBufferManager()
26 : {
27 814 : if (!static_cast<bool>(isShareCCLbuffer_)) {
28 814 : ReleaseCommCCLbuffer();
29 : }
30 814 : ReleaseAlltoAllvParaBuffer();
31 814 : ReleaseCommAIVbuffer();
32 814 : }
33 :
34 76 : HcclResult CCLBufferManager::CreateCCLbuffer(u64 size, DeviceMem &buffer)
35 : {
36 76 : CHK_PRT_RET(!size, HCCL_INFO("[CCLBufferManager][CreateCCLbuffer]buffer size is zero. not need to malloc memory"),
37 : HCCL_SUCCESS);
38 :
39 : CHK_PRT_RET((size > ULONG_MAX),
40 : HCCL_ERROR("[CCLBufferManager][CreateCCLbuffer]buffer size is greater than %llu", ULONG_MAX), HCCL_E_PARA);
41 :
42 76 : CHK_RET(DeviceMem::alloc(buffer, size));
43 81 : HCCL_INFO("[CreateCCLbuffer] buffer ptr[%p], size[%llu]", buffer.ptr(), buffer.size());
44 81 : CHK_PRT_RET(size && !buffer, HCCL_ERROR("[CCLBufferManager][CreateCCLbuffer]Create ccl buffer size[%llu] fail,"
45 : "please check environmental variable HCCL_BUFFSIZE.", size), HCCL_E_PTR);
46 81 : HCCL_RUN_INFO("[HCCL_TRACE][CreateCCLbuffer]Create ccl buffer success. buffer ptr[%p], size[%llu]",
47 : buffer.ptr(), buffer.size());
48 81 : return HCCL_SUCCESS;
49 : }
50 :
51 83 : HcclResult CCLBufferManager::CreateCommCCLbuffer(const std::string &bufferName)
52 : {
53 83 : if (inCCLbufferSize_ == 0) {
54 14 : inCCLbufferSize_ = GetExternalInputCCLBuffSize();
55 : }
56 83 : if (outCCLbufferSize_ == 0) {
57 14 : outCCLbufferSize_ = GetExternalInputCCLBuffSize();
58 : }
59 83 : if (winExpBufferSize_ == 0) {
60 62 : winExpBufferSize_ = EXP_BUFFER_SIZE;
61 : }
62 :
63 83 : if (cclBuffer_.ptr() == nullptr) {
64 65 : u64 totalSize = inCCLbufferSize_ + outCCLbufferSize_ + winExpBufferSize_;
65 : // buffername非空则申请共享cclbuffer
66 65 : if (!bufferName.empty()) {
67 0 : CHK_RET(ShareCCLbufferMgr::GetInstance().CreateShareCCLbuffer(bufferName, totalSize, cclBuffer_));
68 0 : isShareCCLbuffer_ = true;
69 : } else {
70 67 : CHK_RET(CreateCCLbuffer(totalSize, cclBuffer_));
71 73 : CHK_RET(hrtMemSet(cclBuffer_.ptr(), totalSize, totalSize));
72 : }
73 : }
74 :
75 91 : if (inCCLbuffer_.ptr() == nullptr) {
76 73 : inCCLbuffer_ = DeviceMem::create(cclBuffer_.ptr(), inCCLbufferSize_);
77 : }
78 :
79 91 : if (outCCLbuffer_.ptr() == nullptr) {
80 73 : outCCLbuffer_ = DeviceMem::create(static_cast<u8 *>(cclBuffer_.ptr()) + inCCLbufferSize_, outCCLbufferSize_);
81 : }
82 :
83 90 : if (winExpBuffer_.ptr() == nullptr) {
84 146 : winExpBuffer_ = DeviceMem::create(static_cast<u8 *>(cclBuffer_.ptr()) + inCCLbufferSize_ + outCCLbufferSize_,
85 73 : winExpBufferSize_);
86 : }
87 90 : HCCL_INFO("[CreateCommCCLbuffer] create cclbuffer, inPtr[%p], outPtr[%p], winExpPtr[%p], isSharebuffer[%d]",
88 : inCCLbuffer_.ptr(), outCCLbuffer_.ptr(), winExpBuffer_.ptr(), isShareCCLbuffer_);
89 91 : return HCCL_SUCCESS;
90 : }
91 :
92 0 : HcclResult CCLBufferManager::CleanCCLbuffer()
93 : {
94 0 : if (inCCLbuffer_.ptr() != nullptr) {
95 0 : CHK_RET(hrtMemSet(inCCLbuffer_.ptr(), inCCLbuffer_.size(), inCCLbuffer_.size()));
96 0 : HCCL_INFO("[CleanCCLbuffer] clean input buffer, ptr[%p], size[%llu]", inCCLbuffer_.ptr(), inCCLbuffer_.size());
97 : }
98 :
99 0 : if (outCCLbuffer_.ptr() != nullptr) {
100 0 : CHK_RET(hrtMemSet(outCCLbuffer_.ptr(), outCCLbuffer_.size(), outCCLbuffer_.size()));
101 0 : HCCL_INFO("[CleanCCLbuffer] clean output buffer, ptr[%p], size[%llu]",
102 : outCCLbuffer_.ptr(), outCCLbuffer_.size());
103 : }
104 0 : return HCCL_SUCCESS;
105 : }
106 :
107 8 : HcclResult CCLBufferManager::CleanAIVbuffer(void *bufferPtr)
108 : {
109 : // 将aiv的bufferPtr空间置于0
110 8 : if (bufferPtr != nullptr) {
111 8 : CHK_RET(hrtMemSet(bufferPtr, AIV_FLAG_SIZE, AIV_FLAG_SIZE));
112 8 : HCCL_INFO("[CleanAIVbuffer] clean aiv buffer, ptr[%p], size[%llu]", bufferPtr, AIV_FLAG_SIZE);
113 : }
114 8 : return HCCL_SUCCESS;
115 : }
116 :
117 4 : HcclResult CCLBufferManager::CreateCommAIVbuffer(bool useOpbaseFlag)
118 : {
119 4 : if (useOpbaseFlag) {
120 2 : if (inAivOpbaseBuffer_.ptr() == nullptr) {
121 2 : CHK_RET(CreateCCLbuffer(AIV_DATA_SIZE, inAivOpbaseBuffer_));
122 2 : CHK_RET(CleanAIVbuffer(static_cast<u8 *>(inAivOpbaseBuffer_.ptr()) + (AIV_DATA_SIZE - AIV_FLAG_SIZE)));
123 : }
124 2 : if (outAivOpbaseBuffer_.ptr() == nullptr) {
125 2 : CHK_RET(CreateCCLbuffer(AIV_FLAG_SIZE, outAivOpbaseBuffer_));
126 2 : CHK_RET(CleanAIVbuffer(outAivOpbaseBuffer_.ptr()));
127 2 : HCCL_RUN_INFO("[HCCL_TRACE][CreateCommAIVbuffer] OpbaseMode");
128 : }
129 : } else {
130 2 : if (inAivOffloadbuffer_.ptr() == nullptr) {
131 2 : CHK_RET(CreateCCLbuffer(AIV_DATA_SIZE, inAivOffloadbuffer_));
132 2 : CHK_RET(CleanAIVbuffer(static_cast<u8 *>(inAivOffloadbuffer_.ptr()) + (AIV_DATA_SIZE - AIV_FLAG_SIZE)));
133 : }
134 2 : if (outAivOffloadbuffer_.ptr() == nullptr) {
135 2 : CHK_RET(CreateCCLbuffer(AIV_FLAG_SIZE, outAivOffloadbuffer_));
136 2 : CHK_RET(CleanAIVbuffer(outAivOffloadbuffer_.ptr()));
137 2 : HCCL_RUN_INFO("[HCCL_TRACE][CreateCommAIVbuffer] OffloadMode");
138 : }
139 : }
140 4 : return HCCL_SUCCESS;
141 : }
142 :
143 0 : HcclResult CCLBufferManager::CreateCommInfoAIVbuffer()
144 : {
145 0 : if (aivCommInfoBuffer_.ptr() == nullptr) {
146 0 : CHK_RET(CreateCCLbuffer(AIV_COMM_INFO_SIZE, aivCommInfoBuffer_));
147 : }
148 0 : return HCCL_SUCCESS;
149 : }
150 :
151 811 : HcclResult CCLBufferManager::ReleaseCommCCLbuffer()
152 : {
153 1552 : if ((cclBuffer_.ptr() == nullptr) && (inCCLbuffer_.ptr() == nullptr) && (outCCLbuffer_.ptr() == nullptr) &&
154 741 : (winExpBuffer_.ptr() == nullptr)) {
155 741 : HCCL_RUN_INFO("[HCCL_TRACE][ReleaseCCLbuffer]CCLBuffer is null, no need to release.");
156 741 : return HCCL_SUCCESS;
157 : }
158 :
159 73 : if (cclBuffer_.ptr() != nullptr){
160 73 : HCCL_RUN_INFO("[HCCL_TRACE][ReleaseCCLbuffer]Release cclBuffer. buffer ptr[%p], size[%llu]",
161 : cclBuffer_.ptr(), cclBuffer_.size());
162 73 : cclBuffer_.free();
163 : }
164 :
165 73 : if (inCCLbuffer_.ptr() != nullptr){
166 73 : HCCL_RUN_INFO("[HCCL_TRACE][ReleaseCCLbuffer]Release incclBuffer. buffer ptr[%p], size[%llu]",
167 : inCCLbuffer_.ptr(), inCCLbuffer_.size());
168 73 : inCCLbuffer_.free();
169 : }
170 :
171 73 : if (outCCLbuffer_.ptr() != nullptr ){
172 73 : HCCL_RUN_INFO("[HCCL_TRACE][ReleaseCCLbuffer]Release outcclBuffer. buffer ptr[%p], size[%llu]",
173 : outCCLbuffer_.ptr(), outCCLbuffer_.size());
174 73 : outCCLbuffer_.free();
175 : }
176 :
177 73 : if (winExpBuffer_.ptr() != nullptr ){
178 73 : HCCL_RUN_INFO("[HCCL_TRACE][ReleaseCCLbuffer]Release expcclBuffer. buffer ptr[%p], size[%llu]",
179 : winExpBuffer_.ptr(), winExpBuffer_.size());
180 73 : winExpBuffer_.free();
181 : }
182 :
183 146 : if ((cclBuffer_.ptr() == nullptr) && (inCCLbuffer_.ptr() == nullptr) && (outCCLbuffer_.ptr() == nullptr) &&
184 73 : (winExpBuffer_.ptr() == nullptr)) {
185 73 : HCCL_RUN_INFO("[HCCL_TRACE][ReleaseCCLbuffer]Release CCLbuffer success.");
186 : }
187 73 : return HCCL_SUCCESS;
188 : }
189 :
190 814 : HcclResult CCLBufferManager::ReleaseCommAIVbuffer()
191 : {
192 814 : HCCL_RUN_INFO("[HCCL_TRACE][ReleaseAIVbuffer]Release inAivOpbaseBuffer. buffer ptr[%p], size[%llu]",
193 : inAivOpbaseBuffer_.ptr(), inAivOpbaseBuffer_.size());
194 814 : inAivOpbaseBuffer_.free();
195 812 : HCCL_RUN_INFO("[HCCL_TRACE][ReleaseAIVbuffer]Release outAivOpbaseBuffer. buffer ptr[%p], size[%llu]",
196 : outAivOpbaseBuffer_.ptr(), outAivOpbaseBuffer_.size());
197 814 : outAivOpbaseBuffer_.free();
198 813 : HCCL_RUN_INFO("[HCCL_TRACE][ReleaseAIVbuffer]Release inAivOffloadbuffer. buffer ptr[%p], size[%llu]",
199 : inAivOffloadbuffer_.ptr(), inAivOffloadbuffer_.size());
200 814 : inAivOffloadbuffer_.free();
201 814 : HCCL_RUN_INFO("[HCCL_TRACE][ReleaseAIVbuffer]Release outAivOffloadbuffer. buffer ptr[%p], size[%llu]",
202 : outAivOffloadbuffer_.ptr(), outAivOffloadbuffer_.size());
203 814 : outAivOffloadbuffer_.free();
204 814 : HCCL_RUN_INFO("[HCCL_TRACE][ReleaseAIVbuffer]Release aivCommInfoBuffer. buffer ptr[%p], size[%llu]",
205 : aivCommInfoBuffer_.ptr(), aivCommInfoBuffer_.size());
206 814 : aivCommInfoBuffer_.free();
207 2439 : if (inAivOpbaseBuffer_.ptr() == nullptr && outAivOpbaseBuffer_.ptr() == nullptr &&
208 3252 : inAivOffloadbuffer_.ptr() == nullptr && outAivOffloadbuffer_.ptr() == nullptr &&
209 813 : aivCommInfoBuffer_.ptr() == nullptr) {
210 813 : HCCL_RUN_INFO("[HCCL_TRACE][ReleaseAIVbuffer]Release AIV buffer success.");
211 : }
212 814 : return HCCL_SUCCESS;
213 : }
214 :
215 0 : HcclResult CCLBufferManager::ClearCommAIVbuffer()
216 : {
217 0 : if (inAivOpbaseBuffer_.ptr() != nullptr) {
218 0 : CHK_RET(CleanAIVbuffer(static_cast<u8 *>(inAivOpbaseBuffer_.ptr()) + (AIV_DATA_SIZE - AIV_FLAG_SIZE)));
219 : }
220 0 : if (outAivOpbaseBuffer_.ptr() != nullptr) {
221 0 : CHK_RET(CleanAIVbuffer(outAivOpbaseBuffer_.ptr()));
222 : }
223 0 : if (inAivOffloadbuffer_.ptr() != nullptr) {
224 0 : CHK_RET(CleanAIVbuffer(static_cast<u8 *>(inAivOffloadbuffer_.ptr()) + (AIV_DATA_SIZE - AIV_FLAG_SIZE)));
225 : }
226 0 : if (outAivOffloadbuffer_.ptr() != nullptr) {
227 0 : CHK_RET(CleanAIVbuffer(outAivOffloadbuffer_.ptr()));
228 : }
229 0 : return HCCL_SUCCESS;
230 : }
231 :
232 2 : DeviceMem& CCLBufferManager::GetInAivOpbaseBuffer()
233 : {
234 2 : return inAivOpbaseBuffer_;
235 : }
236 :
237 2 : DeviceMem& CCLBufferManager::GetOutAivOpbaseBuffer()
238 : {
239 2 : return outAivOpbaseBuffer_;
240 : }
241 :
242 2 : DeviceMem& CCLBufferManager::GetInAivOffloadbuffer()
243 : {
244 2 : return inAivOffloadbuffer_;
245 : }
246 :
247 2 : DeviceMem& CCLBufferManager::GetOutAivOffloadbuffer()
248 : {
249 2 : return outAivOffloadbuffer_;
250 : }
251 :
252 112 : DeviceMem& CCLBufferManager::GetCommCCLBuffer()
253 : {
254 112 : return cclBuffer_;
255 : }
256 :
257 242 : HcclResult CCLBufferManager::InitCCLbuffer(u64 inCCLbufferSize, u64 outCCLbufferSize)
258 : {
259 242 : inCCLbufferSize_ = inCCLbufferSize;
260 242 : outCCLbufferSize_ = outCCLbufferSize;
261 242 : return HCCL_SUCCESS;
262 : }
263 :
264 125 : void* CCLBufferManager::GetCCLbufferAddr(const DeviceMem &buffer)
265 : {
266 125 : if (buffer.ptr() == nullptr) {
267 60 : return nullptr;
268 : } else {
269 65 : return static_cast<void *>(reinterpret_cast<u8 *>(buffer.ptr()));
270 : }
271 : }
272 :
273 218 : DeviceMem& CCLBufferManager::GetInCCLbuffer()
274 : {
275 218 : return inCCLbuffer_;
276 : }
277 :
278 2 : DeviceMem& CCLBufferManager::GetCommExpBuffer()
279 : {
280 2 : return winExpBuffer_;
281 : }
282 :
283 0 : DeviceMem& CCLBufferManager::GetAivCommInfoBuffer()
284 : {
285 0 : return aivCommInfoBuffer_;
286 : }
287 :
288 77 : HcclResult CCLBufferManager::GetInCCLbuffer(void* &buffer, u64 &size)
289 : {
290 77 : buffer = GetCCLbufferAddr(inCCLbuffer_);
291 75 : size = inCCLbufferSize_;
292 75 : return HCCL_SUCCESS;
293 : }
294 :
295 266 : u64 CCLBufferManager::GetInCCLbufferSize()
296 : {
297 266 : return inCCLbufferSize_;
298 : }
299 :
300 218 : DeviceMem& CCLBufferManager::GetOutCCLbuffer()
301 : {
302 218 : return outCCLbuffer_;
303 : }
304 :
305 38 : HcclResult CCLBufferManager::GetOutCCLbuffer(void* &buffer, u64 &size)
306 : {
307 38 : buffer = GetCCLbufferAddr(outCCLbuffer_);
308 37 : size = outCCLbufferSize_;
309 37 : return HCCL_SUCCESS;
310 : }
311 :
312 151 : u64 CCLBufferManager::GetOutCCLbufferSize()
313 : {
314 151 : return outCCLbufferSize_;
315 : }
316 :
317 2 : u64 CCLBufferManager::GetExpBufferSize()
318 : {
319 2 : return winExpBufferSize_;
320 : }
321 :
322 2 : DeviceMem CCLBufferManager::GetCommRegMem(const DeviceMem &mem, MemAttr memAttr, bool aivMode)
323 : {
324 2 : u64 commMemSize = 0;
325 2 : if ((GetWorkflowMode() == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE) && (!aivMode)) {
326 : // 单算子模式时，仅在第一次集合通信时创建子通信域，注册通信内存。需要将整个CCLbuffer注册进通信域。
327 2 : if (memAttr == MemAttr::IN_CCL_BUFFER) {
328 1 : commMemSize = inCCLbufferSize_;
329 1 : } else if (memAttr == MemAttr::OUT_CCL_BUFFER) {
330 1 : commMemSize = outCCLbufferSize_;
331 : }
332 : } else {
333 0 : commMemSize = mem.size();
334 : }
335 2 : DeviceMem commMem = DeviceMem::create(mem.ptr(), commMemSize);
336 2 : return commMem;
337 : }
338 :
339 0 : HcclResult CCLBufferManager::InitAlltoAllvParaBuffer(u64 inBufferSize, u64 outBufferSize)
340 : {
341 0 : CHK_RET(CreateCCLbuffer(inBufferSize, inAlltoAllvParaBuffer_));
342 0 : CHK_RET(CreateCCLbuffer(outBufferSize, outAlltoAllvParaBuffer_));
343 0 : return HCCL_SUCCESS;
344 : }
345 :
346 0 : DeviceMem& CCLBufferManager::GetInAlltoAllvParaBuffer()
347 : {
348 0 : return inAlltoAllvParaBuffer_;
349 : }
350 :
351 0 : DeviceMem& CCLBufferManager::GetOutAlltoAllvParaBuffer()
352 : {
353 0 : return outAlltoAllvParaBuffer_;
354 : }
355 :
356 1331 : void CCLBufferManager::ReleaseAlltoAllvParaBuffer()
357 : {
358 1331 : inAlltoAllvParaBuffer_.free();
359 1329 : outAlltoAllvParaBuffer_.free();
360 1329 : }
361 :
362 7 : HcclResult CCLBufferManager::GetIndependentOpCCLbuffer(void* &buffer, uint64_t &size)
363 : {
364 7 : HCCL_INFO("[GetIndependentOpCCLbuffer] cclBuffer_[%p]", cclBuffer_.ptr());
365 7 : buffer = GetCCLbufferAddr(cclBuffer_);
366 7 : if (buffer == nullptr) {
367 8 : CHK_RET(CreateCommCCLbuffer());
368 4 : buffer = GetCCLbufferAddr(cclBuffer_);
369 : }
370 : // 大小在通信域初始化时调取InitCCLbuffer设置，MC1MB内存不对外暴露
371 7 : size = inCCLbufferSize_ + outCCLbufferSize_;
372 7 : return HCCL_SUCCESS;
373 : }
374 : } // namespace hccl
|
