Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "aicpu_launch_manager.h"
12 : #include "adapter_rts_common.h"
13 : #include "mem_device_pub.h"
14 : #include "notify_manager.h"
15 : #include "launch_aicpu.h"
16 : #include "comm_configer.h"
17 : #include <iomanip>
18 : #include "hcom_host_profiling.h"
19 : #include "adapter_prof.h"
20 : #include "env_config/env_config_v2.h"
21 :
22 : namespace hccl {
23 : template <typename OpParam, typename ApiParam>
24 : HcclResult AicpuLaunchMgr::KernelLaunch(OpParam& opParam, ApiParam& apiParam, rtStream_t aicpuInitStream)
25 : {
26 : return HCCL_SUCCESS;
27 : }
28 :
29 2 : HcclResult AicpuLaunchMgr::KernelLaunchAicpuCustom(
30 : uint64_t context, std::string kernelName, rtStream_t aicpuInitStream, aclrtBinHandle binCustomHandle)
31 : {
32 2 : uint32_t envTimeout = 0;
33 : DevType devType;
34 2 : CHK_RET(hrtGetDeviceType(devType));
35 2 : if (devType == DevType::DEV_TYPE_950 || devType == DevType::DEV_TYPE_960) {
36 0 : envTimeout = Hccl::EnvConfig::GetInstance().GetRtsConfig().GetExecTimeOut();
37 : } else {
38 6 : envTimeout = CommConfiger::GetInstance().GetCommConfigExecTimeOut("");
39 : }
40 2 : u16 timeOut = envTimeout + 25 > std::numeric_limits<u16>::max() ? std::numeric_limits<u16>::max() :
41 2 : envTimeout + 25; // 多25s，避免超时
42 2 : CHK_RET(
43 : AicpuAclKernelLaunch(aicpuInitStream, &context, sizeof(context), binCustomHandle, kernelName, true, timeOut));
44 2 : return HCCL_SUCCESS;
45 : }
46 :
47 0 : HcclResult AicpuLaunchMgr::AiCpuStreamAllocAndGet(rtStream_t& aiCpuStream)
48 : {
49 0 : if (opStream_.ptr() != nullptr) {
50 0 : HCCL_INFO("%s already alloc, stream id:%u", __func__, opStream_.id());
51 0 : aiCpuStream = opStream_.ptr();
52 0 : return HCCL_SUCCESS;
53 : }
54 :
55 0 : constexpr u32 aicpuStreamMode = 1; // 单独申请的kernel流，使能遇错即停，避免出错后流卡住不退
56 0 : opStream_ = Stream(StreamType::STREAM_TYPE_ONLINE);
57 0 : CHK_RET(hrtStreamSetMode(opStream_.ptr(), aicpuStreamMode));
58 0 : aiCpuStream = opStream_.ptr();
59 0 : HCCL_RUN_INFO(
60 : "[AicpuLaunchMgr][%s] alloc success, stream id:%u, aicpuStreamMode:%u", __func__, opStream_.id(),
61 : aicpuStreamMode);
62 0 : return HCCL_SUCCESS;
63 : }
64 :
65 2 : static HcclResult CreateLocalStream(Stream& localStream)
66 : {
67 2 : HCCL_INFO("[%s] create local stream", __func__);
68 2 : localStream = Stream(StreamType::STREAM_TYPE_ONLINE);
69 2 : constexpr u32 aicpuStreamMode = 1;
70 2 : CHK_RET(hrtStreamSetMode(localStream.ptr(), aicpuStreamMode));
71 2 : return HCCL_SUCCESS;
72 : }
73 :
74 1 : static HcclResult PrepareThreadMgrParam(
75 : const std::vector<std::shared_ptr<Thread>>& newThreads, const ThreadKernelLaunchConfig& config,
76 : ThreadMgrAicpuParam& opParam, DeviceMem& deviceHandle)
77 : {
78 1 : HCCL_INFO("[%s] fill opParam", __func__);
79 1 : CHK_PRT_RET(newThreads.empty(), HCCL_ERROR("[%s] newThreads is empty", __func__), HCCL_E_PARA);
80 1 : (void)memset_s(&opParam, sizeof(opParam), 0, sizeof(opParam));
81 1 : opParam.threadNum = newThreads.size();
82 :
83 : // 拷贝 commId
84 1 : errno_t sRet = strncpy_s(opParam.hcomId, HCOMID_MAX_SIZE, config.commId.c_str(), config.commId.length());
85 1 : CHK_PRT_RET(sRet != EOK, HCCL_ERROR("[%s] strncpy_s failed, return [%d].", __func__, sRet), HCCL_E_MEMORY);
86 1 : opParam.hcomId[HCOMID_MAX_SIZE - 1] = '\0';
87 :
88 : // 从 Thread 对象获取 engine（同一批 threads 属于同一 engine）
89 1 : opParam.engine = newThreads[0]->GetCommEngine();
90 :
91 : // 拷贝每个线程的 unique id
92 2 : for (u32 i = 0; i < opParam.threadNum; ++i) {
93 1 : const std::string& uid = newThreads[i]->GetUniqueId();
94 : // 不同于notify路径中uid边界条件，此处采用 >=， 最后一位留给'\0'
95 1 : CHK_PRT_RET(
96 : uid.size() >= THREAD_UNIQUE_ID_MAX_SIZE,
97 : HCCL_ERROR(
98 : "[%s] uid.size()[%zu] exceeds limitation[%u], return [%d].", __func__, uid.size(),
99 : THREAD_UNIQUE_ID_MAX_SIZE - 1, HCCL_E_MEMORY),
100 : HCCL_E_MEMORY);
101 : sRet
102 1 : = memcpy_s(opParam.threadParam[i], static_cast<size_t>(THREAD_UNIQUE_ID_MAX_SIZE), uid.c_str(), uid.size());
103 1 : CHK_PRT_RET(sRet != EOK, HCCL_ERROR("[%s] memcpy_s failed, return [%d].", __func__, sRet), HCCL_E_MEMORY);
104 1 : opParam.threadParam[i][THREAD_UNIQUE_ID_MAX_SIZE - 1] = '\0';
105 :
106 1 : if (UNLIKELY(HcclCheckLogLevel(HCCL_LOG_INFO))) {
107 1 : std::ostringstream oss;
108 1 : oss << "threadParam[" << i << "] raw bytes: ";
109 1 : constexpr u32 HEX_WIDTH = 2;
110 6001 : for (u32 j = 0; j < THREAD_UNIQUE_ID_MAX_SIZE; ++j) {
111 6000 : oss << std::hex << std::setw(HEX_WIDTH) << std::setfill('0')
112 6000 : << static_cast<unsigned int>(static_cast<unsigned char>(opParam.threadParam[i][j])) << " ";
113 : }
114 1 : HCCL_INFO("[%s] %s", __func__, oss.str().c_str());
115 1 : }
116 : }
117 :
118 : // 分配设备内存
119 1 : size_t handleLen = sizeof(ThreadHandle) * newThreads.size();
120 1 : deviceHandle = DeviceMem::alloc(handleLen);
121 1 : CHK_SMART_PTR_NULL(deviceHandle);
122 1 : opParam.deviceHandle = deviceHandle.ptr();
123 :
124 : // 基础通信需要设备信息
125 1 : if (config.needDeviceInfo) {
126 0 : CHK_RET(hrtGetDevice(&opParam.deviceLogicId));
127 : DevType devType;
128 0 : CHK_RET(hrtGetDeviceType(devType));
129 0 : opParam.deviceType = static_cast<u32>(devType);
130 : }
131 1 : return HCCL_SUCCESS;
132 : }
133 :
134 1 : HcclResult AicpuLaunchMgr::ThreadKernelLaunchImpl(
135 : std::vector<std::shared_ptr<Thread>>& newThreads, std::unique_ptr<ThreadHandle[]>& aicpuHandle,
136 : const ThreadKernelLaunchConfig& config)
137 : {
138 : // 参数检查
139 1 : CHK_PRT_RET(
140 : newThreads.size() > SIGNAL_DEV_STREAM_MAX_NUM,
141 : HCCL_ERROR(
142 : "[AicpuLaunchMgr][%s] streamNum[%zu] > SIGNAL_DEV_STREAM_MAX_NUM[%u]", __func__, newThreads.size(),
143 : SIGNAL_DEV_STREAM_MAX_NUM),
144 : HCCL_E_PARA);
145 :
146 1 : uint64_t beginTime = (config.needProfiling ? hrtMsprofSysCycleTime() : 0);
147 :
148 : // Step 1. 创建局部 stream
149 1 : Stream localStream;
150 1 : CHK_RET(CreateLocalStream(localStream));
151 :
152 : // Step 2. 填写 opParam 并分配设备内存
153 1 : ThreadMgrAicpuParam opParam{};
154 1 : DeviceMem deviceHandle;
155 1 : CHK_RET(PrepareThreadMgrParam(newThreads, config, opParam, deviceHandle));
156 :
157 1 : size_t handleLen = sizeof(ThreadHandle) * newThreads.size();
158 : // Step 3. 补充notify，将threadHanle拷到device侧
159 1 : if (config.isSupplementNotify) {
160 0 : CHK_RET(hrtMemSyncCopy(
161 : opParam.deviceHandle, handleLen, aicpuHandle.get(), handleLen,
162 : HcclRtMemcpyKind::HCCL_RT_MEMCPY_KIND_HOST_TO_DEVICE));
163 : }
164 :
165 : // Step 4. 调用 KernelLaunch
166 1 : DeviceMem addr = DeviceMem::alloc(sizeof(opParam));
167 1 : CHK_RET(hrtMemSyncCopy(
168 : addr.ptr(), sizeof(opParam), &opParam, sizeof(opParam), HcclRtMemcpyKind::HCCL_RT_MEMCPY_KIND_HOST_TO_DEVICE));
169 1 : uint64_t context = reinterpret_cast<uint64_t>(addr.ptr());
170 1 : HCCL_INFO("AicpuLaunchMgr::%s, call KernelLaunch", __func__);
171 2 : HcclResult ret = KernelLaunchAicpuCustom(context, config.kernelName.c_str(), localStream.ptr(), config.binHandle);
172 1 : CHK_PRT_RET(
173 : ret != HCCL_SUCCESS, HCCL_ERROR("[AicpuLaunchMgr][%s] KernelLaunch failed, return [%d].", __func__, ret), ret);
174 :
175 : // Step 5. 等待流完成
176 1 : CHK_RET(hcclStreamSynchronize(localStream.ptr(), config.timeoutSec));
177 :
178 : // Step 6. 非补充notify,返回 device 侧句柄
179 1 : if (!config.isSupplementNotify) {
180 1 : CHK_RET(hrtMemSyncCopy(
181 : aicpuHandle.get(), handleLen, opParam.deviceHandle, handleLen,
182 : HcclRtMemcpyKind::HCCL_RT_MEMCPY_KIND_DEVICE_TO_HOST));
183 : }
184 :
185 : // 性能分析上报
186 1 : if (config.needProfiling) {
187 1 : HcommProfilingReportKernel(beginTime, config.kernelName.c_str());
188 : }
189 1 : return HCCL_SUCCESS;
190 1 : }
191 :
192 : // 集合通信使用，待归一到基础通信接口
193 1 : HcclResult AicpuLaunchMgr::ThreadKernelLaunchForComm(
194 : std::vector<std::shared_ptr<Thread>>& newThreads, const std::string& commId,
195 : std::unique_ptr<ThreadHandle[]>& aicpuHandle, aclrtBinHandle binHandle)
196 : {
197 1 : uint32_t envTimeout = 0;
198 : DevType devType;
199 1 : CHK_RET(hrtGetDeviceType(devType));
200 1 : if (devType == DevType::DEV_TYPE_950 || devType == DevType::DEV_TYPE_960) {
201 0 : envTimeout = Hccl::EnvConfig::GetInstance().GetRtsConfig().GetExecTimeOut();
202 : } else {
203 1 : envTimeout = CommConfiger::GetInstance().GetCommConfigExecTimeOut(commId);
204 : }
205 1 : uint32_t timeOut = envTimeout + 25 > std::numeric_limits<uint32_t>::max() ? std::numeric_limits<uint32_t>::max() :
206 1 : envTimeout + 25; // 多25s，避免超时
207 1 : ThreadKernelLaunchConfig config(commId, binHandle, "RunAicpuIndOpThreadInit", false, timeOut, true, false);
208 1 : return ThreadKernelLaunchImpl(newThreads, aicpuHandle, config);
209 1 : }
210 :
211 : // 基础通信使用
212 0 : HcclResult AicpuLaunchMgr::ThreadKernelLaunchForBase(
213 : std::vector<std::shared_ptr<Thread>>& newThreads, std::unique_ptr<ThreadHandle[]>& aicpuHandle,
214 : aclrtBinHandle binHandle)
215 : {
216 0 : constexpr uint32_t defaultTimeOutSec = 120;
217 0 : ThreadKernelLaunchConfig config("", binHandle, "RunAicpuThreadInit", true, defaultTimeOutSec, false, false);
218 0 : return ThreadKernelLaunchImpl(newThreads, aicpuHandle, config);
219 0 : }
220 :
221 : // 补充notify使用
222 0 : HcclResult AicpuLaunchMgr::SupplementNotifyKernelLaunch(
223 : std::vector<std::shared_ptr<Thread>>& newThreads, const std::string& commId,
224 : std::unique_ptr<ThreadHandle[]>& aicpuHandle, aclrtBinHandle binHandle)
225 : {
226 0 : constexpr uint32_t defaultTimeOutSec = 120;
227 : ThreadKernelLaunchConfig config(
228 0 : commId, binHandle, "RunAicpuThreadSupplementNotify", true, defaultTimeOutSec, false, true);
229 0 : return ThreadKernelLaunchImpl(newThreads, aicpuHandle, config);
230 0 : }
231 :
232 : HcclResult
233 1 : AicpuLaunchMgr::ThreadKernelLaunchDestroy(ThreadHandle* threadHandles, uint32_t listNum, aclrtBinHandle binHandle)
234 : {
235 1 : HCCL_INFO(
236 : "[AicpuLaunchMgr][%s] Start. listNum=%u, threadHandles=%p, binHandle=%p", __func__, listNum, threadHandles,
237 : binHandle);
238 : // Step 1. 创建局部 stream
239 1 : Stream localStream;
240 1 : CHK_RET(CreateLocalStream(localStream));
241 :
242 : // Step 2. 填写 opParam
243 1 : ThreadMgrAicpuParam opParam{};
244 1 : (void)memset_s(&opParam, sizeof(opParam), 0, sizeof(opParam));
245 1 : opParam.threadNum = listNum;
246 1 : size_t handleLen = sizeof(ThreadHandle) * listNum;
247 1 : DeviceMem deviceHandle = DeviceMem::alloc(handleLen);
248 1 : CHK_SMART_PTR_NULL(deviceHandle);
249 1 : opParam.deviceHandle = deviceHandle.ptr();
250 1 : CHK_RET(hrtMemSyncCopy(
251 : deviceHandle.ptr(), handleLen, threadHandles, handleLen, HcclRtMemcpyKind::HCCL_RT_MEMCPY_KIND_HOST_TO_DEVICE));
252 :
253 : // Step 3. 调用 KernelLaunch
254 1 : std::string kernelName = "RunAicpuThreadDestroy";
255 1 : HCCL_INFO(
256 : "[AicpuLaunchMgr][%s] call KernelLaunch, kernelName=%s, stream=%p, binHandle=%p", __func__, kernelName.c_str(),
257 : localStream.ptr(), binHandle);
258 1 : DeviceMem addr = DeviceMem::alloc(sizeof(opParam));
259 1 : CHK_RET(hrtMemSyncCopy(
260 : addr.ptr(), sizeof(opParam), &opParam, sizeof(opParam), HcclRtMemcpyKind::HCCL_RT_MEMCPY_KIND_HOST_TO_DEVICE));
261 1 : uint64_t context = reinterpret_cast<uint64_t>(addr.ptr());
262 2 : HcclResult ret = KernelLaunchAicpuCustom(context, kernelName.c_str(), localStream.ptr(), binHandle);
263 1 : CHK_PRT_RET(
264 : ret != HCCL_SUCCESS, HCCL_ERROR("[AicpuLaunchMgr][%s] KernelLaunch failed, return [%d].", __func__, ret), ret);
265 :
266 : // Step 4. 等待流完成
267 1 : constexpr uint32_t defaultTimeOutSec = 120;
268 1 : CHK_RET(hcclStreamSynchronize(localStream.ptr(), defaultTimeOutSec));
269 1 : return HCCL_SUCCESS;
270 1 : }
271 :
272 : // 准备 opParam
273 0 : HcclResult AicpuLaunchMgr::PrepareAicpuNotifyParam(
274 : NotifyMgrAicpuParam& opParam, const std::string& commId, size_t notifyNum, bool freeFlag, void* deviceHandle)
275 : {
276 0 : (void)memset_s(&opParam, sizeof(opParam), 0, sizeof(opParam));
277 :
278 0 : opParam.notifyNum = notifyNum;
279 0 : opParam.freeFlag = freeFlag;
280 0 : opParam.deviceHandle = deviceHandle;
281 :
282 0 : errno_t sRet = strncpy_s(opParam.hcomId, HCOMID_MAX_SIZE, commId.c_str(), commId.length());
283 0 : CHK_PRT_RET(
284 : sRet != EOK, HCCL_ERROR("[AicpuLaunchMgr][PrepareAicpuNotifyParam] strncpy_s failed, ret[%d]", sRet),
285 : HCCL_E_MEMORY);
286 0 : opParam.hcomId[HCOMID_MAX_SIZE - 1] = '\0';
287 0 : return HCCL_SUCCESS;
288 : }
289 :
290 0 : HcclResult AicpuLaunchMgr::LaunchNotifyKernel(NotifyMgrAicpuParam& opParam, aclrtBinHandle binCustomHandle)
291 : {
292 0 : Stream localStream(StreamType::STREAM_TYPE_ONLINE);
293 0 : constexpr u32 aicpuStreamMode = 1;
294 0 : CHK_RET(hrtStreamSetMode(localStream.ptr(), aicpuStreamMode));
295 :
296 0 : DeviceMem addr = DeviceMem::alloc(sizeof(opParam));
297 0 : CHK_RET(hrtMemSyncCopy(
298 : addr.ptr(), sizeof(opParam), &opParam, sizeof(opParam), HcclRtMemcpyKind::HCCL_RT_MEMCPY_KIND_HOST_TO_DEVICE));
299 0 : uint64_t context = reinterpret_cast<uint64_t>(addr.ptr());
300 0 : HcclResult ret = KernelLaunchAicpuCustom(context, "RunAicpuIndOpNotify", localStream.ptr(), binCustomHandle);
301 0 : CHK_PRT_RET(
302 : ret != HCCL_SUCCESS, HCCL_ERROR("[AicpuLaunchMgr][LaunchNotifyKernel] KernelLaunch failed, ret[%d]", ret), ret);
303 0 : CHK_RET(
304 : hcclStreamSynchronize(localStream.ptr(), CommConfiger::GetInstance().GetCommConfigExecTimeOut(opParam.hcomId)));
305 0 : return HCCL_SUCCESS;
306 0 : }
307 :
308 0 : HcclResult AicpuLaunchMgr::NotifyKernelLaunchAlloc(
309 : std::vector<std::unique_ptr<LocalNotify>>& newNotifys, const std::string& commId,
310 : std::unique_ptr<NotifyHandle[]>& hostHandle, aclrtBinHandle binCustomHandle)
311 : {
312 0 : size_t handleLen = sizeof(NotifyHandle) * newNotifys.size();
313 0 : DeviceMem deviceHandle = DeviceMem::alloc(handleLen);
314 0 : CHK_SMART_PTR_NULL(deviceHandle);
315 :
316 : NotifyMgrAicpuParam opParam;
317 0 : CHK_RET(PrepareAicpuNotifyParam(opParam, commId, newNotifys.size(), false, deviceHandle.ptr()));
318 0 : std::string uid = NotifyManager::GetBinNotifys(newNotifys, NotifyLoadType::DEVICE_NOTIFY);
319 0 : if (UNLIKELY(uid.empty())) {
320 0 : HCCL_ERROR("[AicpuLaunchMgr][%s] uid is empty.", __func__, HCCL_E_MEMORY);
321 0 : return HCCL_E_MEMORY;
322 : }
323 : errno_t sRet
324 0 : = memcpy_s(opParam.notifyParam, static_cast<size_t>(NOTIFY_UNIQUE_ID_MAX_SIZE), uid.c_str(), uid.size());
325 0 : CHK_PRT_RET(
326 : sRet != EOK,
327 : HCCL_ERROR(
328 : "[%s] uid.size()[%zu] exceeds limitation[%u], return [%d].", __func__, uid.size(),
329 : NOTIFY_UNIQUE_ID_MAX_SIZE, HCCL_E_MEMORY),
330 : HCCL_E_MEMORY);
331 : // 打印每个字节
332 0 : if (UNLIKELY(HcclCheckLogLevel(HCCL_LOG_INFO))) {
333 0 : std::ostringstream oss;
334 0 : oss << "notifyParam" << " raw bytes: ";
335 0 : for (u32 i = 0; i < NOTIFY_UNIQUE_ID_MAX_SIZE; ++i) {
336 0 : oss << std::hex << std::setw(2) << std::setfill('0')
337 0 : << static_cast<unsigned int>(static_cast<unsigned char>(opParam.notifyParam[i])) << " ";
338 : }
339 0 : HCCL_INFO("[AicpuLaunchMgr][%s] %s", __func__, oss.str().c_str());
340 0 : }
341 :
342 0 : CHK_RET(LaunchNotifyKernel(opParam, binCustomHandle));
343 :
344 0 : CHK_RET(hrtMemSyncCopy(
345 : hostHandle.get(), handleLen, opParam.deviceHandle, handleLen,
346 : HcclRtMemcpyKind::HCCL_RT_MEMCPY_KIND_DEVICE_TO_HOST));
347 0 : HCCL_RUN_INFO(
348 : "[AicpuLaunchMgr][%s] notify alloc success, commid[%s], notifyNum[%u]", __func__, commId.c_str(),
349 : newNotifys.size());
350 0 : return HCCL_SUCCESS;
351 0 : }
352 :
353 0 : HcclResult AicpuLaunchMgr::NotifyKernelLaunchFree(
354 : std::vector<NotifyHandle>& aicpuNotifys, uint32_t notifyNum, const std::string& commId,
355 : aclrtBinHandle binCustomHandle)
356 : {
357 0 : size_t handleLen = sizeof(NotifyHandle) * notifyNum;
358 0 : DeviceMem deviceHandle = DeviceMem::alloc(handleLen);
359 0 : CHK_SMART_PTR_NULL(deviceHandle);
360 :
361 0 : CHK_RET(hrtMemSyncCopy(
362 : deviceHandle.ptr(), handleLen, aicpuNotifys.data(), handleLen,
363 : HcclRtMemcpyKind::HCCL_RT_MEMCPY_KIND_HOST_TO_DEVICE));
364 :
365 : NotifyMgrAicpuParam opParam;
366 0 : CHK_RET(PrepareAicpuNotifyParam(opParam, commId, notifyNum, true, deviceHandle.ptr()));
367 :
368 0 : CHK_RET(LaunchNotifyKernel(opParam, binCustomHandle));
369 0 : HCCL_RUN_INFO(
370 : "[AicpuLaunchMgr][%s] notify free kernalLaunch success, commid[%s], notifyNum[%u]", __func__, commId.c_str(),
371 : aicpuNotifys.size());
372 0 : return HCCL_SUCCESS;
373 0 : }
374 : } // namespace hccl
|
