Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "aicpu_launch_manager.h"
12 : #include "adapter_rts_common.h"
13 : #include "mem_device_pub.h"
14 : #include "notify_manager.h"
15 : #include "launch_aicpu.h"
16 : #include "comm_configer.h"
17 : #include <iomanip>
18 : #include "hcom_host_profiling.h"
19 : #include "adapter_prof.h"
20 : #include "env_config/env_config_v2.h"
21 :
22 : namespace hccl {
23 : template <typename OpParam, typename ApiParam>
24 : HcclResult AicpuLaunchMgr::KernelLaunch(OpParam& opParam, ApiParam& apiParam, rtStream_t aicpuInitStream)
25 : {
26 : return HCCL_SUCCESS;
27 : }
28 :
29 2 : HcclResult AicpuLaunchMgr::KernelLaunchAicpuCustom(
30 : uint64_t context, std::string kernelName, rtStream_t aicpuInitStream, aclrtBinHandle binCustomHandle)
31 : {
32 2 : uint32_t envTimeout = 0;
33 : DevType devType;
34 2 : CHK_RET(hrtGetDeviceType(devType));
35 2 : if (devType == DevType::DEV_TYPE_950 || devType == DevType::DEV_TYPE_960) {
36 0 : envTimeout = Hccl::EnvConfig::GetInstance().GetRtsConfig().GetExecTimeOut();
37 : } else {
38 6 : envTimeout = CommConfiger::GetInstance().GetCommConfigExecTimeOut("");
39 : }
40 2 : u16 timeOut = envTimeout + 25 > std::numeric_limits<u16>::max() ? std::numeric_limits<u16>::max() :
41 2 : envTimeout + 25; // 多25s，避免超时
42 2 : CHK_RET(
43 : AicpuAclKernelLaunch(aicpuInitStream, &context, sizeof(context), binCustomHandle, kernelName, true, timeOut));
44 2 : return HCCL_SUCCESS;
45 : }
46 :
47 0 : HcclResult AicpuLaunchMgr::AiCpuStreamAllocAndGet(rtStream_t& aiCpuStream)
48 : {
49 0 : if (opStream_.ptr() != nullptr) {
50 0 : HCCL_INFO("%s already alloc, stream id:%u", __func__, opStream_.id());
51 0 : aiCpuStream = opStream_.ptr();
52 0 : return HCCL_SUCCESS;
53 : }
54 :
55 0 : constexpr u32 aicpuStreamMode = 1; // 单独申请的kernel流，使能遇错即停，避免出错后流卡住不退
56 0 : opStream_ = Stream(StreamType::STREAM_TYPE_ONLINE);
57 0 : CHK_RET(hrtStreamSetMode(opStream_.ptr(), aicpuStreamMode));
58 0 : aiCpuStream = opStream_.ptr();
59 0 : HCCL_RUN_INFO(
60 : "[AicpuLaunchMgr][%s] alloc success, stream id:%u, aicpuStreamMode:%u", __func__, opStream_.id(),
61 : aicpuStreamMode);
62 0 : return HCCL_SUCCESS;
63 : }
64 :
65 2 : static HcclResult CreateLocalStream(Stream& localStream)
66 : {
67 2 : HCCL_INFO("[%s] create local stream", __func__);
68 2 : localStream = Stream(StreamType::STREAM_TYPE_ONLINE);
69 2 : constexpr u32 aicpuStreamMode = 1;
70 2 : CHK_RET(hrtStreamSetMode(localStream.ptr(), aicpuStreamMode));
71 2 : return HCCL_SUCCESS;
72 : }
73 :
74 1 : static HcclResult PrepareThreadMgrParam(
75 : const std::vector<std::shared_ptr<Thread>>& newThreads, const ThreadKernelLaunchConfig& config,
76 : ThreadMgrAicpuParam& opParam, DeviceMem& deviceHandle)
77 : {
78 1 : HCCL_INFO("[%s] fill opParam", __func__);
79 1 : (void)memset_s(&opParam, sizeof(opParam), 0, sizeof(opParam));
80 1 : opParam.threadNum = newThreads.size();
81 :
82 : // 拷贝 commId
83 1 : errno_t sRet = strncpy_s(opParam.hcomId, HCOMID_MAX_SIZE, config.commId.c_str(), config.commId.length());
84 1 : CHK_PRT_RET(sRet != EOK, HCCL_ERROR("[%s] strncpy_s failed, return [%d].", __func__, sRet), HCCL_E_MEMORY);
85 1 : opParam.hcomId[HCOMID_MAX_SIZE - 1] = '\0';
86 :
87 : // 拷贝每个线程的 unique id
88 2 : for (u32 i = 0; i < opParam.threadNum; ++i) {
89 1 : const std::string& uid = newThreads[i]->GetUniqueId();
90 : // 不同于notify路径中uid边界条件，此处采用 >=， 最后一位留给'\0'
91 1 : CHK_PRT_RET(
92 : uid.size() >= THREAD_UNIQUE_ID_MAX_SIZE,
93 : HCCL_ERROR(
94 : "[%s] uid.size()[%zu] exceeds limitation[%u], return [%d].", __func__, uid.size(),
95 : THREAD_UNIQUE_ID_MAX_SIZE - 1, HCCL_E_MEMORY),
96 : HCCL_E_MEMORY);
97 : sRet
98 1 : = memcpy_s(opParam.threadParam[i], static_cast<size_t>(THREAD_UNIQUE_ID_MAX_SIZE), uid.c_str(), uid.size());
99 1 : CHK_PRT_RET(sRet != EOK, HCCL_ERROR("[%s] memcpy_s failed, return [%d].", __func__, sRet), HCCL_E_MEMORY);
100 1 : opParam.threadParam[i][THREAD_UNIQUE_ID_MAX_SIZE - 1] = '\0';
101 :
102 1 : if (UNLIKELY(HcclCheckLogLevel(HCCL_LOG_INFO))) {
103 1 : std::ostringstream oss;
104 1 : oss << "threadParam[" << i << "] raw bytes: ";
105 1 : constexpr u32 HEX_WIDTH = 2;
106 6001 : for (u32 j = 0; j < THREAD_UNIQUE_ID_MAX_SIZE; ++j) {
107 6000 : oss << std::hex << std::setw(HEX_WIDTH) << std::setfill('0')
108 6000 : << static_cast<unsigned int>(static_cast<unsigned char>(opParam.threadParam[i][j])) << " ";
109 : }
110 1 : HCCL_INFO("[%s] %s", __func__, oss.str().c_str());
111 1 : }
112 : }
113 :
114 : // 分配设备内存
115 1 : size_t handleLen = sizeof(ThreadHandle) * newThreads.size();
116 1 : deviceHandle = DeviceMem::alloc(handleLen);
117 1 : CHK_SMART_PTR_NULL(deviceHandle);
118 1 : opParam.deviceHandle = deviceHandle.ptr();
119 :
120 : // 基础通信需要设备信息
121 1 : if (config.needDeviceInfo) {
122 0 : CHK_RET(hrtGetDevice(&opParam.deviceLogicId));
123 : DevType devType;
124 0 : CHK_RET(hrtGetDeviceType(devType));
125 0 : opParam.deviceType = static_cast<u32>(devType);
126 : }
127 1 : return HCCL_SUCCESS;
128 : }
129 :
130 1 : HcclResult AicpuLaunchMgr::ThreadKernelLaunchImpl(
131 : std::vector<std::shared_ptr<Thread>>& newThreads, std::unique_ptr<ThreadHandle[]>& aicpuHandle,
132 : const ThreadKernelLaunchConfig& config)
133 : {
134 : // 参数检查
135 1 : CHK_PRT_RET(
136 : newThreads.size() > SIGNAL_DEV_STREAM_MAX_NUM,
137 : HCCL_ERROR(
138 : "[AicpuLaunchMgr][%s] streamNum[%zu] > SIGNAL_DEV_STREAM_MAX_NUM[%u]", __func__, newThreads.size(),
139 : SIGNAL_DEV_STREAM_MAX_NUM),
140 : HCCL_E_PARA);
141 :
142 1 : uint64_t beginTime = (config.needProfiling ? hrtMsprofSysCycleTime() : 0);
143 :
144 : // Step 1. 创建局部 stream
145 1 : Stream localStream;
146 1 : CHK_RET(CreateLocalStream(localStream));
147 :
148 : // Step 2. 填写 opParam 并分配设备内存
149 1 : ThreadMgrAicpuParam opParam{};
150 1 : DeviceMem deviceHandle;
151 1 : CHK_RET(PrepareThreadMgrParam(newThreads, config, opParam, deviceHandle));
152 :
153 1 : size_t handleLen = sizeof(ThreadHandle) * newThreads.size();
154 : // Step 3. 补充notify，将threadHanle拷到device侧
155 1 : if (config.isSupplementNotify) {
156 0 : CHK_RET(hrtMemSyncCopy(
157 : opParam.deviceHandle, handleLen, aicpuHandle.get(), handleLen,
158 : HcclRtMemcpyKind::HCCL_RT_MEMCPY_KIND_HOST_TO_DEVICE));
159 : }
160 :
161 : // Step 4. 调用 KernelLaunch
162 1 : DeviceMem addr = DeviceMem::alloc(sizeof(opParam));
163 1 : CHK_RET(hrtMemSyncCopy(
164 : addr.ptr(), sizeof(opParam), &opParam, sizeof(opParam), HcclRtMemcpyKind::HCCL_RT_MEMCPY_KIND_HOST_TO_DEVICE));
165 1 : uint64_t context = reinterpret_cast<uint64_t>(addr.ptr());
166 1 : HCCL_INFO("AicpuLaunchMgr::%s, call KernelLaunch", __func__);
167 2 : HcclResult ret = KernelLaunchAicpuCustom(context, config.kernelName.c_str(), localStream.ptr(), config.binHandle);
168 1 : CHK_PRT_RET(
169 : ret != HCCL_SUCCESS, HCCL_ERROR("[AicpuLaunchMgr][%s] KernelLaunch failed, return [%d].", __func__, ret), ret);
170 :
171 : // Step 5. 等待流完成
172 1 : CHK_RET(hcclStreamSynchronize(localStream.ptr(), config.timeoutSec));
173 :
174 : // Step 6. 非补充notify,返回 device 侧句柄
175 1 : if (!config.isSupplementNotify) {
176 1 : CHK_RET(hrtMemSyncCopy(
177 : aicpuHandle.get(), handleLen, opParam.deviceHandle, handleLen,
178 : HcclRtMemcpyKind::HCCL_RT_MEMCPY_KIND_DEVICE_TO_HOST));
179 : }
180 :
181 : // 性能分析上报
182 1 : if (config.needProfiling) {
183 1 : HcommProfilingReportKernel(beginTime, config.kernelName.c_str());
184 : }
185 1 : return HCCL_SUCCESS;
186 1 : }
187 :
188 : // 集合通信使用，待归一到基础通信接口
189 1 : HcclResult AicpuLaunchMgr::ThreadKernelLaunchForComm(
190 : std::vector<std::shared_ptr<Thread>>& newThreads, const std::string& commId,
191 : std::unique_ptr<ThreadHandle[]>& aicpuHandle, aclrtBinHandle binHandle)
192 : {
193 1 : uint32_t envTimeout = 0;
194 : DevType devType;
195 1 : CHK_RET(hrtGetDeviceType(devType));
196 1 : if (devType == DevType::DEV_TYPE_950 || devType == DevType::DEV_TYPE_960) {
197 0 : envTimeout = Hccl::EnvConfig::GetInstance().GetRtsConfig().GetExecTimeOut();
198 : } else {
199 1 : envTimeout = CommConfiger::GetInstance().GetCommConfigExecTimeOut(commId);
200 : }
201 1 : uint32_t timeOut = envTimeout + 25 > std::numeric_limits<uint32_t>::max() ? std::numeric_limits<uint32_t>::max() :
202 1 : envTimeout + 25; // 多25s，避免超时
203 1 : ThreadKernelLaunchConfig config(commId, binHandle, "RunAicpuIndOpThreadInit", false, timeOut, true, false);
204 1 : return ThreadKernelLaunchImpl(newThreads, aicpuHandle, config);
205 1 : }
206 :
207 : // 基础通信使用
208 0 : HcclResult AicpuLaunchMgr::ThreadKernelLaunchForBase(
209 : std::vector<std::shared_ptr<Thread>>& newThreads, std::unique_ptr<ThreadHandle[]>& aicpuHandle,
210 : aclrtBinHandle binHandle)
211 : {
212 0 : constexpr uint32_t defaultTimeOutSec = 120;
213 0 : ThreadKernelLaunchConfig config("", binHandle, "RunAicpuThreadInit", true, defaultTimeOutSec, false, false);
214 0 : return ThreadKernelLaunchImpl(newThreads, aicpuHandle, config);
215 0 : }
216 :
217 : // 补充notify使用
218 0 : HcclResult AicpuLaunchMgr::SupplementNotifyKernelLaunch(
219 : std::vector<std::shared_ptr<Thread>>& newThreads, const std::string& commId,
220 : std::unique_ptr<ThreadHandle[]>& aicpuHandle, aclrtBinHandle binHandle)
221 : {
222 0 : constexpr uint32_t defaultTimeOutSec = 120;
223 : ThreadKernelLaunchConfig config(
224 0 : commId, binHandle, "RunAicpuThreadSupplementNotify", true, defaultTimeOutSec, false, true);
225 0 : return ThreadKernelLaunchImpl(newThreads, aicpuHandle, config);
226 0 : }
227 :
228 : HcclResult
229 1 : AicpuLaunchMgr::ThreadKernelLaunchDestroy(ThreadHandle* threadHandles, uint32_t listNum, aclrtBinHandle binHandle)
230 : {
231 1 : HCCL_INFO(
232 : "[AicpuLaunchMgr][%s] Start. listNum=%u, threadHandles=%p, binHandle=%p", __func__, listNum, threadHandles,
233 : binHandle);
234 : // Step 1. 创建局部 stream
235 1 : Stream localStream;
236 1 : CHK_RET(CreateLocalStream(localStream));
237 :
238 : // Step 2. 填写 opParam
239 1 : ThreadMgrAicpuParam opParam{};
240 1 : (void)memset_s(&opParam, sizeof(opParam), 0, sizeof(opParam));
241 1 : opParam.threadNum = listNum;
242 1 : size_t handleLen = sizeof(ThreadHandle) * listNum;
243 1 : DeviceMem deviceHandle = DeviceMem::alloc(handleLen);
244 1 : CHK_SMART_PTR_NULL(deviceHandle);
245 1 : opParam.deviceHandle = deviceHandle.ptr();
246 1 : CHK_RET(hrtMemSyncCopy(
247 : deviceHandle.ptr(), handleLen, threadHandles, handleLen, HcclRtMemcpyKind::HCCL_RT_MEMCPY_KIND_HOST_TO_DEVICE));
248 :
249 : // Step 3. 调用 KernelLaunch
250 1 : std::string kernelName = "RunAicpuThreadDestroy";
251 1 : HCCL_INFO(
252 : "[AicpuLaunchMgr][%s] call KernelLaunch, kernelName=%s, stream=%p, binHandle=%p", __func__, kernelName.c_str(),
253 : localStream.ptr(), binHandle);
254 1 : DeviceMem addr = DeviceMem::alloc(sizeof(opParam));
255 1 : CHK_RET(hrtMemSyncCopy(
256 : addr.ptr(), sizeof(opParam), &opParam, sizeof(opParam), HcclRtMemcpyKind::HCCL_RT_MEMCPY_KIND_HOST_TO_DEVICE));
257 1 : uint64_t context = reinterpret_cast<uint64_t>(addr.ptr());
258 2 : HcclResult ret = KernelLaunchAicpuCustom(context, kernelName.c_str(), localStream.ptr(), binHandle);
259 1 : CHK_PRT_RET(
260 : ret != HCCL_SUCCESS, HCCL_ERROR("[AicpuLaunchMgr][%s] KernelLaunch failed, return [%d].", __func__, ret), ret);
261 :
262 : // Step 4. 等待流完成
263 1 : constexpr uint32_t defaultTimeOutSec = 120;
264 1 : CHK_RET(hcclStreamSynchronize(localStream.ptr(), defaultTimeOutSec));
265 1 : return HCCL_SUCCESS;
266 1 : }
267 :
268 : // 准备 opParam
269 0 : HcclResult AicpuLaunchMgr::PrepareAicpuNotifyParam(
270 : NotifyMgrAicpuParam& opParam, const std::string& commId, size_t notifyNum, bool freeFlag, void* deviceHandle)
271 : {
272 0 : (void)memset_s(&opParam, sizeof(opParam), 0, sizeof(opParam));
273 :
274 0 : opParam.notifyNum = notifyNum;
275 0 : opParam.freeFlag = freeFlag;
276 0 : opParam.deviceHandle = deviceHandle;
277 :
278 0 : errno_t sRet = strncpy_s(opParam.hcomId, HCOMID_MAX_SIZE, commId.c_str(), commId.length());
279 0 : CHK_PRT_RET(
280 : sRet != EOK, HCCL_ERROR("[AicpuLaunchMgr][PrepareAicpuNotifyParam] strncpy_s failed, ret[%d]", sRet),
281 : HCCL_E_MEMORY);
282 0 : opParam.hcomId[HCOMID_MAX_SIZE - 1] = '\0';
283 0 : return HCCL_SUCCESS;
284 : }
285 :
286 0 : HcclResult AicpuLaunchMgr::LaunchNotifyKernel(NotifyMgrAicpuParam& opParam, aclrtBinHandle binCustomHandle)
287 : {
288 0 : Stream localStream(StreamType::STREAM_TYPE_ONLINE);
289 0 : constexpr u32 aicpuStreamMode = 1;
290 0 : CHK_RET(hrtStreamSetMode(localStream.ptr(), aicpuStreamMode));
291 :
292 0 : DeviceMem addr = DeviceMem::alloc(sizeof(opParam));
293 0 : CHK_RET(hrtMemSyncCopy(
294 : addr.ptr(), sizeof(opParam), &opParam, sizeof(opParam), HcclRtMemcpyKind::HCCL_RT_MEMCPY_KIND_HOST_TO_DEVICE));
295 0 : uint64_t context = reinterpret_cast<uint64_t>(addr.ptr());
296 0 : HcclResult ret = KernelLaunchAicpuCustom(context, "RunAicpuIndOpNotify", localStream.ptr(), binCustomHandle);
297 0 : CHK_PRT_RET(
298 : ret != HCCL_SUCCESS, HCCL_ERROR("[AicpuLaunchMgr][LaunchNotifyKernel] KernelLaunch failed, ret[%d]", ret), ret);
299 0 : CHK_RET(
300 : hcclStreamSynchronize(localStream.ptr(), CommConfiger::GetInstance().GetCommConfigExecTimeOut(opParam.hcomId)));
301 0 : return HCCL_SUCCESS;
302 0 : }
303 :
304 0 : HcclResult AicpuLaunchMgr::NotifyKernelLaunchAlloc(
305 : std::vector<std::unique_ptr<LocalNotify>>& newNotifys, const std::string& commId,
306 : std::unique_ptr<NotifyHandle[]>& hostHandle, aclrtBinHandle binCustomHandle)
307 : {
308 0 : size_t handleLen = sizeof(NotifyHandle) * newNotifys.size();
309 0 : DeviceMem deviceHandle = DeviceMem::alloc(handleLen);
310 0 : CHK_SMART_PTR_NULL(deviceHandle);
311 :
312 : NotifyMgrAicpuParam opParam;
313 0 : CHK_RET(PrepareAicpuNotifyParam(opParam, commId, newNotifys.size(), false, deviceHandle.ptr()));
314 0 : std::string uid = NotifyManager::GetBinNotifys(newNotifys, NotifyLoadType::DEVICE_NOTIFY);
315 0 : if (UNLIKELY(uid.empty())) {
316 0 : HCCL_ERROR("[AicpuLaunchMgr][%s] uid is empty.", __func__, HCCL_E_MEMORY);
317 0 : return HCCL_E_MEMORY;
318 : }
319 : errno_t sRet
320 0 : = memcpy_s(opParam.notifyParam, static_cast<size_t>(NOTIFY_UNIQUE_ID_MAX_SIZE), uid.c_str(), uid.size());
321 0 : CHK_PRT_RET(
322 : sRet != EOK,
323 : HCCL_ERROR(
324 : "[%s] uid.size()[%zu] exceeds limitation[%u], return [%d].", __func__, uid.size(),
325 : NOTIFY_UNIQUE_ID_MAX_SIZE, HCCL_E_MEMORY),
326 : HCCL_E_MEMORY);
327 : // 打印每个字节
328 0 : if (UNLIKELY(HcclCheckLogLevel(HCCL_LOG_INFO))) {
329 0 : std::ostringstream oss;
330 0 : oss << "notifyParam" << " raw bytes: ";
331 0 : for (u32 i = 0; i < NOTIFY_UNIQUE_ID_MAX_SIZE; ++i) {
332 0 : oss << std::hex << std::setw(2) << std::setfill('0')
333 0 : << static_cast<unsigned int>(static_cast<unsigned char>(opParam.notifyParam[i])) << " ";
334 : }
335 0 : HCCL_INFO("[AicpuLaunchMgr][%s] %s", __func__, oss.str().c_str());
336 0 : }
337 :
338 0 : CHK_RET(LaunchNotifyKernel(opParam, binCustomHandle));
339 :
340 0 : CHK_RET(hrtMemSyncCopy(
341 : hostHandle.get(), handleLen, opParam.deviceHandle, handleLen,
342 : HcclRtMemcpyKind::HCCL_RT_MEMCPY_KIND_DEVICE_TO_HOST));
343 0 : HCCL_RUN_INFO(
344 : "[AicpuLaunchMgr][%s] notify alloc success, commid[%s], notifyNum[%u]", __func__, commId.c_str(),
345 : newNotifys.size());
346 0 : return HCCL_SUCCESS;
347 0 : }
348 :
349 0 : HcclResult AicpuLaunchMgr::NotifyKernelLaunchFree(
350 : std::vector<NotifyHandle>& aicpuNotifys, uint32_t notifyNum, const std::string& commId,
351 : aclrtBinHandle binCustomHandle)
352 : {
353 0 : size_t handleLen = sizeof(NotifyHandle) * notifyNum;
354 0 : DeviceMem deviceHandle = DeviceMem::alloc(handleLen);
355 0 : CHK_SMART_PTR_NULL(deviceHandle);
356 :
357 0 : CHK_RET(hrtMemSyncCopy(
358 : deviceHandle.ptr(), handleLen, aicpuNotifys.data(), handleLen,
359 : HcclRtMemcpyKind::HCCL_RT_MEMCPY_KIND_HOST_TO_DEVICE));
360 :
361 : NotifyMgrAicpuParam opParam;
362 0 : CHK_RET(PrepareAicpuNotifyParam(opParam, commId, notifyNum, true, deviceHandle.ptr()));
363 :
364 0 : CHK_RET(LaunchNotifyKernel(opParam, binCustomHandle));
365 0 : HCCL_RUN_INFO(
366 : "[AicpuLaunchMgr][%s] notify free kernalLaunch success, commid[%s], notifyNum[%u]", __func__, commId.c_str(),
367 : aicpuNotifys.size());
368 0 : return HCCL_SUCCESS;
369 0 : }
370 : } // namespace hccl
|
