Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "aicpu_launch_manager.h"
12 : #include "adapter_rts_common.h"
13 : #include "mem_device_pub.h"
14 : #include "notify_manager.h"
15 : #include "launch_aicpu.h"
16 : #include "comm_configer.h"
17 : #include <iomanip>
18 : #include "hcom_host_profiling.h"
19 : #include "adapter_prof.h"
20 :
21 : namespace hccl {
22 : template <typename OpParam, typename ApiParam>
23 : HcclResult AicpuLaunchMgr::KernelLaunch(OpParam &opParam, ApiParam &apiParam, rtStream_t aicpuInitStream)
24 : {
25 : return HCCL_SUCCESS;
26 : }
27 :
28 2 : HcclResult AicpuLaunchMgr::KernelLaunchAicpuCustom(uint64_t context, std::string kernelName, rtStream_t aicpuInitStream,
29 : aclrtBinHandle binCustomHandle)
30 : {
31 2 : u16 timeOut = NOTIFY_DEFAULT_WAIT_TIME > std::numeric_limits<uint16_t>::max() ?
32 : std::numeric_limits<uint16_t>::max() : NOTIFY_DEFAULT_WAIT_TIME;
33 2 : CHK_RET(AicpuAclKernelLaunch(aicpuInitStream, &context,
34 : sizeof(context), binCustomHandle, kernelName, true, timeOut));
35 2 : return HCCL_SUCCESS;
36 : }
37 :
38 0 : HcclResult AicpuLaunchMgr::AiCpuStreamAllocAndGet(rtStream_t &aiCpuStream)
39 : {
40 0 : if (opStream_.ptr() != nullptr) {
41 0 : HCCL_INFO("%s already alloc, stream id:%u", __func__, opStream_.id());
42 0 : aiCpuStream = opStream_.ptr();
43 0 : return HCCL_SUCCESS;
44 : }
45 :
46 0 : constexpr u32 aicpuStreamMode = 1; // 单独申请的kernel流，使能遇错即停，避免出错后流卡住不退
47 0 : opStream_ = Stream(StreamType::STREAM_TYPE_ONLINE);
48 0 : CHK_RET(hrtStreamSetMode(opStream_.ptr(), aicpuStreamMode));
49 0 : aiCpuStream = opStream_.ptr();
50 0 : HCCL_RUN_INFO("[AicpuLaunchMgr][%s] alloc success, stream id:%u, aicpuStreamMode:%u",
51 : __func__, opStream_.id(), aicpuStreamMode);
52 0 : return HCCL_SUCCESS;
53 : }
54 :
55 2 : static HcclResult CreateLocalStream(Stream &localStream)
56 : {
57 2 : HCCL_INFO("[%s] create local stream", __func__);
58 2 : localStream = Stream(StreamType::STREAM_TYPE_ONLINE);
59 2 : constexpr u32 aicpuStreamMode = 1;
60 2 : CHK_RET(hrtStreamSetMode(localStream.ptr(), aicpuStreamMode));
61 2 : return HCCL_SUCCESS;
62 : }
63 :
64 1 : static HcclResult PrepareThreadMgrParam(const std::vector<std::shared_ptr<Thread>> &newThreads,
65 : const ThreadKernelLaunchConfig &config,
66 : ThreadMgrAicpuParam &opParam,
67 : DeviceMem &deviceHandle)
68 : {
69 1 : HCCL_INFO("[%s] fill opParam", __func__);
70 1 : (void)memset_s(&opParam, sizeof(opParam), 0, sizeof(opParam));
71 1 : opParam.threadNum = newThreads.size();
72 :
73 : // 拷贝 commId
74 1 : errno_t sRet = strncpy_s(opParam.hcomId, HCOMID_MAX_SIZE, config.commId.c_str(), config.commId.length());
75 1 : CHK_PRT_RET(sRet != EOK,
76 : HCCL_ERROR("[%s] strncpy_s failed, return [%d].", __func__, sRet),
77 : HCCL_E_MEMORY);
78 1 : opParam.hcomId[HCOMID_MAX_SIZE - 1] = '\0';
79 :
80 : // 拷贝每个线程的 unique id
81 2 : for (u32 i = 0; i < opParam.threadNum; ++i) {
82 1 : const std::string &uid = newThreads[i]->GetUniqueId();
83 1 : size_t copyLen = std::min(uid.size(), static_cast<size_t>(THREAD_UNIQUE_ID_MAX_SIZE));
84 1 : sRet = memcpy_s(opParam.threadParam[i], THREAD_UNIQUE_ID_MAX_SIZE, uid.c_str(), copyLen);
85 1 : CHK_PRT_RET(sRet != EOK,
86 : HCCL_ERROR("[%s] memcpy_s failed, return [%d].", __func__, sRet),
87 : HCCL_E_MEMORY);
88 1 : opParam.threadParam[i][THREAD_UNIQUE_ID_MAX_SIZE - 1] = '\0';
89 :
90 1 : if (UNLIKELY(HcclCheckLogLevel(HCCL_LOG_INFO))) {
91 1 : std::ostringstream oss;
92 1 : oss << "threadParam[" << i << "] raw bytes: ";
93 1 : constexpr u32 HEX_WIDTH = 2;
94 6001 : for (u32 j = 0; j < THREAD_UNIQUE_ID_MAX_SIZE; ++j) {
95 6000 : oss << std::hex << std::setw(HEX_WIDTH) << std::setfill('0')
96 6000 : << static_cast<unsigned int>(static_cast<unsigned char>(opParam.threadParam[i][j])) << " ";
97 : }
98 1 : HCCL_INFO("[%s] %s", __func__, oss.str().c_str());
99 1 : }
100 : }
101 :
102 : // 分配设备内存
103 1 : size_t handleLen = sizeof(ThreadHandle) * newThreads.size();
104 1 : deviceHandle = DeviceMem::alloc(handleLen);
105 1 : CHK_SMART_PTR_NULL(deviceHandle);
106 1 : opParam.deviceHandle = deviceHandle.ptr();
107 :
108 : // 基础通信需要设备信息
109 1 : if (config.needDeviceInfo) {
110 0 : CHK_RET(hrtGetDevice(&opParam.deviceLogicId));
111 : DevType devType;
112 0 : CHK_RET(hrtGetDeviceType(devType));
113 0 : opParam.deviceType = static_cast<u32>(devType);
114 : }
115 1 : return HCCL_SUCCESS;
116 : }
117 :
118 1 : HcclResult AicpuLaunchMgr::ThreadKernelLaunchImpl(
119 : std::vector<std::shared_ptr<Thread>> &newThreads,
120 : std::unique_ptr<ThreadHandle[]> &aicpuHandle,
121 : const ThreadKernelLaunchConfig &config)
122 : {
123 : // 参数检查
124 1 : CHK_PRT_RET(newThreads.size() > SIGNAL_DEV_STREAM_MAX_NUM,
125 : HCCL_ERROR("[AicpuLaunchMgr][%s] streamNum[%zu] > SIGNAL_DEV_STREAM_MAX_NUM[%u]", __func__,
126 : newThreads.size(), SIGNAL_DEV_STREAM_MAX_NUM), HCCL_E_PARA);
127 :
128 1 : uint64_t beginTime = (config.needProfiling ? hrtMsprofSysCycleTime() : 0);
129 :
130 : // Step 1. 创建局部 stream
131 1 : Stream localStream;
132 1 : CHK_RET(CreateLocalStream(localStream));
133 :
134 : // Step 2. 填写 opParam 并分配设备内存
135 1 : ThreadMgrAicpuParam opParam{};
136 1 : DeviceMem deviceHandle;
137 1 : CHK_RET(PrepareThreadMgrParam(newThreads, config, opParam, deviceHandle));
138 :
139 1 : size_t handleLen = sizeof(ThreadHandle) * newThreads.size();
140 : // Step 3. 补充notify，将threadHanle拷到device侧
141 1 : if (config.isSupplementNotify) {
142 0 : CHK_RET(hrtMemSyncCopy(opParam.deviceHandle, handleLen, aicpuHandle.get(), handleLen,
143 : HcclRtMemcpyKind::HCCL_RT_MEMCPY_KIND_HOST_TO_DEVICE));
144 : }
145 :
146 : // Step 4. 调用 KernelLaunch
147 1 : DeviceMem addr = DeviceMem::alloc(sizeof(opParam));
148 1 : CHK_RET(hrtMemSyncCopy(addr.ptr(), sizeof(opParam), &opParam, sizeof(opParam),
149 : HcclRtMemcpyKind::HCCL_RT_MEMCPY_KIND_HOST_TO_DEVICE));
150 1 : uint64_t context = reinterpret_cast<uint64_t>(addr.ptr());
151 1 : HCCL_INFO("AicpuLaunchMgr::%s, call KernelLaunch", __func__);
152 1 : HcclResult ret = KernelLaunchAicpuCustom(context, config.kernelName.c_str(),
153 1 : localStream.ptr(), config.binHandle);
154 1 : CHK_PRT_RET(ret != HCCL_SUCCESS,
155 : HCCL_ERROR("[AicpuLaunchMgr][%s] KernelLaunch failed, return [%d].", __func__, ret), ret);
156 :
157 : // Step 5. 等待流完成
158 1 : CHK_RET(hcclStreamSynchronize(localStream.ptr(), config.timeoutSec));
159 :
160 : // Step 6. 非补充notify,返回 device 侧句柄
161 1 : if (!config.isSupplementNotify) {
162 1 : CHK_RET(hrtMemSyncCopy(aicpuHandle.get(), handleLen, opParam.deviceHandle, handleLen,
163 : HcclRtMemcpyKind::HCCL_RT_MEMCPY_KIND_DEVICE_TO_HOST));
164 : }
165 :
166 : // 性能分析上报
167 1 : if (config.needProfiling) {
168 1 : HcommProfilingReportKernel(beginTime, config.kernelName.c_str());
169 : }
170 1 : return HCCL_SUCCESS;
171 1 : }
172 :
173 : // 集合通信使用，待归一到基础通信接口
174 1 : HcclResult AicpuLaunchMgr::ThreadKernelLaunchForComm(std::vector<std::shared_ptr<Thread>> &newThreads,
175 : const std::string &commId, std::unique_ptr<ThreadHandle[]> &aicpuHandle, aclrtBinHandle binHandle)
176 : {
177 : ThreadKernelLaunchConfig config(
178 : commId,
179 : binHandle,
180 : "RunAicpuIndOpThreadInit",
181 : false,
182 1 : CommConfiger::GetInstance().GetCommConfigExecTimeOut(commId),
183 : true,
184 : false
185 2 : );
186 2 : return ThreadKernelLaunchImpl(newThreads, aicpuHandle, config);
187 1 : }
188 :
189 : // 基础通信使用
190 0 : HcclResult AicpuLaunchMgr::ThreadKernelLaunchForBase(std::vector<std::shared_ptr<Thread>> &newThreads,
191 : std::unique_ptr<ThreadHandle[]> &aicpuHandle, aclrtBinHandle binHandle)
192 : {
193 0 : constexpr uint32_t defaultTimeOutSec = 120;
194 : ThreadKernelLaunchConfig config(
195 : "",
196 : binHandle,
197 : "RunAicpuThreadInit",
198 : true,
199 : defaultTimeOutSec,
200 : false,
201 : false
202 0 : );
203 0 : return ThreadKernelLaunchImpl(newThreads, aicpuHandle, config);
204 0 : }
205 :
206 : // 补充notify使用
207 0 : HcclResult AicpuLaunchMgr::SupplementNotifyKernelLaunch(std::vector<std::shared_ptr<Thread>> &newThreads,
208 : const std::string &commId, std::unique_ptr<ThreadHandle[]> &aicpuHandle, aclrtBinHandle binHandle)
209 : {
210 0 : constexpr uint32_t defaultTimeOutSec = 120;
211 : ThreadKernelLaunchConfig config(
212 : commId,
213 : binHandle,
214 : "RunAicpuThreadSupplementNotify",
215 : true,
216 : defaultTimeOutSec,
217 : false,
218 : true
219 0 : );
220 0 : return ThreadKernelLaunchImpl(newThreads, aicpuHandle, config);
221 0 : }
222 :
223 1 : HcclResult AicpuLaunchMgr::ThreadKernelLaunchDestroy(ThreadHandle *threadHandles, uint32_t listNum,
224 : aclrtBinHandle binHandle)
225 : {
226 1 : HCCL_INFO("[AicpuLaunchMgr][%s] Start. listNum=%u, threadHandles=%p, binHandle=%p",
227 : __func__, listNum, threadHandles, binHandle);
228 : // Step 1. 创建局部 stream
229 1 : Stream localStream;
230 1 : CHK_RET(CreateLocalStream(localStream));
231 :
232 : // Step 2. 填写 opParam
233 1 : ThreadMgrAicpuParam opParam{};
234 1 : (void)memset_s(&opParam, sizeof(opParam), 0, sizeof(opParam));
235 1 : opParam.threadNum = listNum;
236 1 : size_t handleLen = sizeof(ThreadHandle) * listNum;
237 1 : DeviceMem deviceHandle = DeviceMem::alloc(handleLen);
238 1 : CHK_SMART_PTR_NULL(deviceHandle);
239 1 : opParam.deviceHandle = deviceHandle.ptr();
240 1 : CHK_RET(hrtMemSyncCopy(deviceHandle.ptr(), handleLen,
241 : threadHandles, handleLen,
242 : HcclRtMemcpyKind::HCCL_RT_MEMCPY_KIND_HOST_TO_DEVICE));
243 :
244 : // Step 3. 调用 KernelLaunch
245 1 : std::string kernelName = "RunAicpuThreadDestroy";
246 1 : HCCL_INFO("[AicpuLaunchMgr][%s] call KernelLaunch, kernelName=%s, stream=%p, binHandle=%p",
247 : __func__, kernelName.c_str(), localStream.ptr(), binHandle);
248 1 : DeviceMem addr = DeviceMem::alloc(sizeof(opParam));
249 1 : CHK_RET(hrtMemSyncCopy(addr.ptr(), sizeof(opParam), &opParam, sizeof(opParam),
250 : HcclRtMemcpyKind::HCCL_RT_MEMCPY_KIND_HOST_TO_DEVICE));
251 1 : uint64_t context = reinterpret_cast<uint64_t>(addr.ptr());
252 2 : HcclResult ret = KernelLaunchAicpuCustom(context, kernelName.c_str(), localStream.ptr(), binHandle);
253 1 : CHK_PRT_RET(ret != HCCL_SUCCESS,
254 : HCCL_ERROR("[AicpuLaunchMgr][%s] KernelLaunch failed, return [%d].", __func__, ret), ret);
255 :
256 : // Step 4. 等待流完成
257 1 : constexpr uint32_t defaultTimeOutSec = 120;
258 1 : CHK_RET(hcclStreamSynchronize(localStream.ptr(), defaultTimeOutSec));
259 1 : return HCCL_SUCCESS;
260 1 : }
261 :
262 : // 准备 opParam
263 0 : HcclResult AicpuLaunchMgr::PrepareAicpuNotifyParam(NotifyMgrAicpuParam &opParam,
264 : const std::string &commId, size_t notifyNum, bool freeFlag, void *deviceHandle)
265 : {
266 0 : (void)memset_s(&opParam, sizeof(opParam), 0, sizeof(opParam));
267 :
268 0 : opParam.notifyNum = notifyNum;
269 0 : opParam.freeFlag = freeFlag;
270 0 : opParam.deviceHandle = deviceHandle;
271 :
272 0 : errno_t sRet = strncpy_s(opParam.hcomId, HCOMID_MAX_SIZE, commId.c_str(), commId.length());
273 0 : CHK_PRT_RET(sRet != EOK,
274 : HCCL_ERROR("[AicpuLaunchMgr][PrepareAicpuNotifyParam] strncpy_s failed, ret[%d]", sRet),
275 : HCCL_E_MEMORY);
276 0 : opParam.hcomId[HCOMID_MAX_SIZE - 1] = '\0';
277 0 : return HCCL_SUCCESS;
278 : }
279 :
280 0 : HcclResult AicpuLaunchMgr::LaunchNotifyKernel(NotifyMgrAicpuParam &opParam, aclrtBinHandle binCustomHandle)
281 : {
282 0 : Stream localStream(StreamType::STREAM_TYPE_ONLINE);
283 0 : constexpr u32 aicpuStreamMode = 1;
284 0 : CHK_RET(hrtStreamSetMode(localStream.ptr(), aicpuStreamMode));
285 :
286 0 : DeviceMem addr = DeviceMem::alloc(sizeof(opParam));
287 0 : CHK_RET(hrtMemSyncCopy(addr.ptr(), sizeof(opParam), &opParam, sizeof(opParam),
288 : HcclRtMemcpyKind::HCCL_RT_MEMCPY_KIND_HOST_TO_DEVICE));
289 0 : uint64_t context = reinterpret_cast<uint64_t>(addr.ptr());
290 0 : HcclResult ret = KernelLaunchAicpuCustom(context, "RunAicpuIndOpNotify", localStream.ptr(), binCustomHandle);
291 0 : CHK_PRT_RET(ret != HCCL_SUCCESS,
292 : HCCL_ERROR("[AicpuLaunchMgr][LaunchNotifyKernel] KernelLaunch failed, ret[%d]", ret), ret);
293 0 : CHK_RET(hcclStreamSynchronize(localStream.ptr(), CommConfiger::GetInstance().GetCommConfigExecTimeOut(opParam.hcomId)));
294 0 : return HCCL_SUCCESS;
295 0 : }
296 :
297 0 : HcclResult AicpuLaunchMgr::NotifyKernelLaunchAlloc(std::vector<std::unique_ptr<LocalNotify>> &newNotifys,
298 : const std::string &commId, std::unique_ptr<NotifyHandle[]> &hostHandle, aclrtBinHandle binCustomHandle)
299 : {
300 0 : size_t handleLen = sizeof(NotifyHandle) * newNotifys.size();
301 0 : DeviceMem deviceHandle = DeviceMem::alloc(handleLen);
302 0 : CHK_SMART_PTR_NULL(deviceHandle);
303 :
304 : NotifyMgrAicpuParam opParam;
305 0 : CHK_RET(PrepareAicpuNotifyParam(opParam, commId, newNotifys.size(), false, deviceHandle.ptr()));
306 0 : std::string uid = NotifyManager::GetBinNotifys(newNotifys, NotifyLoadType::DEVICE_NOTIFY);
307 0 : if (UNLIKELY(uid.empty())) {
308 0 : HCCL_ERROR("[AicpuLaunchMgr][%s] uid is empty.", __func__, HCCL_E_MEMORY);
309 0 : return HCCL_E_MEMORY;
310 : }
311 0 : size_t copyLen = std::min(uid.size(), static_cast<size_t>(NOTIFY_UNIQUE_ID_MAX_SIZE));
312 0 : errno_t sRet = memcpy_s(opParam.notifyParam, NOTIFY_UNIQUE_ID_MAX_SIZE, uid.c_str(), copyLen);
313 0 : CHK_PRT_RET(sRet != EOK, HCCL_ERROR("[AicpuLaunchMgr][%s] call memcpy_s failed, return [%d].", __func__, sRet),
314 : HCCL_E_MEMORY);
315 : // 打印每个字节
316 0 : if (UNLIKELY(HcclCheckLogLevel(HCCL_LOG_INFO))) {
317 0 : std::ostringstream oss;
318 0 : oss << "notifyParam" << " raw bytes: ";
319 0 : for (u32 i = 0; i < NOTIFY_UNIQUE_ID_MAX_SIZE; ++i) {
320 0 : oss << std::hex << std::setw(2) << std::setfill('0')
321 0 : << static_cast<unsigned int>(static_cast<unsigned char>(opParam.notifyParam[i])) << " ";
322 : }
323 0 : HCCL_INFO("[AicpuLaunchMgr][%s] %s", __func__, oss.str().c_str());
324 0 : }
325 :
326 0 : CHK_RET(LaunchNotifyKernel(opParam, binCustomHandle));
327 :
328 0 : CHK_RET(hrtMemSyncCopy(hostHandle.get(), handleLen, opParam.deviceHandle,
329 : handleLen, HcclRtMemcpyKind::HCCL_RT_MEMCPY_KIND_DEVICE_TO_HOST));
330 0 : HCCL_RUN_INFO("[AicpuLaunchMgr][%s] notify alloc success, commid[%s], notifyNum[%u]",
331 : __func__, commId.c_str(), newNotifys.size());
332 0 : return HCCL_SUCCESS;
333 0 : }
334 :
335 0 : HcclResult AicpuLaunchMgr::NotifyKernelLaunchFree(std::vector<NotifyHandle> &aicpuNotifys, uint32_t notifyNum,
336 : const std::string &commId, aclrtBinHandle binCustomHandle)
337 : {
338 0 : size_t handleLen = sizeof(NotifyHandle) * notifyNum;
339 0 : DeviceMem deviceHandle = DeviceMem::alloc(handleLen);
340 0 : CHK_SMART_PTR_NULL(deviceHandle);
341 :
342 0 : CHK_RET(hrtMemSyncCopy(deviceHandle.ptr(), handleLen, aicpuNotifys.data(),
343 : handleLen, HcclRtMemcpyKind::HCCL_RT_MEMCPY_KIND_HOST_TO_DEVICE));
344 :
345 : NotifyMgrAicpuParam opParam;
346 0 : CHK_RET(PrepareAicpuNotifyParam(opParam, commId, notifyNum, true, deviceHandle.ptr()));
347 :
348 0 : CHK_RET(LaunchNotifyKernel(opParam, binCustomHandle));
349 0 : HCCL_RUN_INFO("[AicpuLaunchMgr][%s] notify free kernalLaunch success, commid[%s], notifyNum[%u]",
350 : __func__, commId.c_str(), aicpuNotifys.size());
351 0 : return HCCL_SUCCESS;
352 0 : }
353 : }
|
