Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include <mutex>
12 : #include <vector>
13 : #include <iostream>
14 : #include <fstream>
15 : #include "mmpa_api.h"
16 : #include "acl/acl_rt.h"
17 : #include "env_config_v2.h"
18 : #include "hccl_aiv_utils.h"
19 : #include "aicpu/launch_device_v2.h"
20 :
21 : using namespace std;
22 :
23 : namespace Hccl {
24 : constexpr u32 SIG_MOVE_LEFT_BITS = 20;
25 : constexpr u32 MAX_BIN_FILE_SIZE = 100 * 1024 * 1024;
26 : constexpr s32 RESET_TAIL_SYNC_TAG = 2;
27 : constexpr uint64_t MIN_NPU_TIMEOUT = 1;
28 : constexpr uint64_t MAX_NPU_TIMEOUT = 254;
29 :
30 : static bool g_init = false;
31 : static mutex g_mut;
32 : static aclrtBinHandle g_binHandle;
33 : static std::unordered_map<const s8*, aclrtFuncHandle> g_aivFuncMap;
34 : static std::unordered_map<const s8*, std::string> g_aivNameMap;
35 :
36 : using AivKernelInfo = struct AivKernelInfoDef {
37 : const char* kernelName;
38 : HcclCMDType cmdType;
39 : DataType dataType;
40 : KernelArgsType argsType;
41 :
42 156 : AivKernelInfoDef(
43 : const char* kernelName, HcclCMDType cmdType, DataType dataType,
44 : KernelArgsType argsType = KernelArgsType::ARGS_TYPE_SERVER)
45 156 : : kernelName(kernelName),
46 156 : cmdType(cmdType),
47 156 : dataType(dataType),
48 156 : argsType(argsType)
49 156 : {}
50 : };
51 :
52 : static std::vector<AivKernelInfo> g_aivKernelInfoList = {
53 : // scatter
54 : {"aiv_scatter_half", HcclCMDType::HCCL_CMD_SCATTER, DataType::FP16},
55 : {"aiv_scatter_int16_t", HcclCMDType::HCCL_CMD_SCATTER, DataType::INT16},
56 : {"aiv_scatter_uint16_t", HcclCMDType::HCCL_CMD_SCATTER, DataType::UINT16},
57 : {"aiv_scatter_float", HcclCMDType::HCCL_CMD_SCATTER, DataType::FP32},
58 : {"aiv_scatter_uint64_t", HcclCMDType::HCCL_CMD_SCATTER, DataType::FP64},
59 : {"aiv_scatter_int32_t", HcclCMDType::HCCL_CMD_SCATTER, DataType::INT32},
60 : {"aiv_scatter_uint32_t", HcclCMDType::HCCL_CMD_SCATTER, DataType::UINT32},
61 : {"aiv_scatter_int8_t", HcclCMDType::HCCL_CMD_SCATTER, DataType::INT8},
62 : {"aiv_scatter_uint8_t", HcclCMDType::HCCL_CMD_SCATTER, DataType::UINT8},
63 : {"aiv_scatter_bfloat16_t", HcclCMDType::HCCL_CMD_SCATTER, DataType::BFP16},
64 : {"aiv_scatter_uint64_t", HcclCMDType::HCCL_CMD_SCATTER, DataType::INT64},
65 : {"aiv_scatter_int64_t", HcclCMDType::HCCL_CMD_SCATTER, DataType::UINT64},
66 : {"aiv_scatter_hifloat8_t", HcclCMDType::HCCL_CMD_SCATTER, DataType::HIF8},
67 : {"aiv_scatter_fp8_e4m3fn_t", HcclCMDType::HCCL_CMD_SCATTER, DataType::FP8E4M3},
68 : {"aiv_scatter_fp8_e5m2_t", HcclCMDType::HCCL_CMD_SCATTER, DataType::FP8E5M2},
69 : {"aiv_scatter_fp8_e8m0_t", HcclCMDType::HCCL_CMD_SCATTER, DataType::FP8E8M0},
70 :
71 : {"aiv_all_gather_half", HcclCMDType::HCCL_CMD_ALLGATHER, DataType::FP16},
72 : {"aiv_all_gather_int16_t", HcclCMDType::HCCL_CMD_ALLGATHER, DataType::INT16},
73 : {"aiv_all_gather_uint16_t", HcclCMDType::HCCL_CMD_ALLGATHER, DataType::UINT16},
74 : {"aiv_all_gather_float", HcclCMDType::HCCL_CMD_ALLGATHER, DataType::FP32},
75 : {"aiv_all_gather_uint64_t", HcclCMDType::HCCL_CMD_ALLGATHER, DataType::FP64},
76 : {"aiv_all_gather_int32_t", HcclCMDType::HCCL_CMD_ALLGATHER, DataType::INT32},
77 : {"aiv_all_gather_uint32_t", HcclCMDType::HCCL_CMD_ALLGATHER, DataType::UINT32},
78 : {"aiv_all_gather_int8_t", HcclCMDType::HCCL_CMD_ALLGATHER, DataType::INT8},
79 : {"aiv_all_gather_uint8_t", HcclCMDType::HCCL_CMD_ALLGATHER, DataType::UINT8},
80 : {"aiv_all_gather_bfloat16_t", HcclCMDType::HCCL_CMD_ALLGATHER, DataType::BFP16},
81 : {"aiv_all_gather_uint64_t", HcclCMDType::HCCL_CMD_ALLGATHER, DataType::INT64},
82 : {"aiv_all_gather_int64_t", HcclCMDType::HCCL_CMD_ALLGATHER, DataType::UINT64},
83 : {"aiv_all_gather_hifloat8_t", HcclCMDType::HCCL_CMD_ALLGATHER, DataType::HIF8},
84 : {"aiv_all_gather_fp8_e4m3fn_t", HcclCMDType::HCCL_CMD_ALLGATHER, DataType::FP8E4M3},
85 : {"aiv_all_gather_fp8_e5m2_t", HcclCMDType::HCCL_CMD_ALLGATHER, DataType::FP8E5M2},
86 : {"aiv_all_gather_fp8_e8m0_t", HcclCMDType::HCCL_CMD_ALLGATHER, DataType::FP8E8M0},
87 : // allreduce
88 : {"aiv_allreduce_half", HcclCMDType::HCCL_CMD_ALLREDUCE, DataType::FP16},
89 : {"aiv_allreduce_int16_t", HcclCMDType::HCCL_CMD_ALLREDUCE, DataType::INT16},
90 : {"aiv_allreduce_float", HcclCMDType::HCCL_CMD_ALLREDUCE, DataType::FP32},
91 : {"aiv_allreduce_int32_t", HcclCMDType::HCCL_CMD_ALLREDUCE, DataType::INT32},
92 : {"aiv_allreduce_int8_t", HcclCMDType::HCCL_CMD_ALLREDUCE, DataType::INT8},
93 : {"aiv_allreduce_bfloat16_t", HcclCMDType::HCCL_CMD_ALLREDUCE, DataType::BFP16},
94 : {"aiv_allreduce_int64_t", HcclCMDType::HCCL_CMD_ALLREDUCE, DataType::INT64},
95 : // broadcast
96 : {"aiv_broadcast_half", HcclCMDType::HCCL_CMD_BROADCAST, DataType::FP16},
97 : {"aiv_broadcast_int16_t", HcclCMDType::HCCL_CMD_BROADCAST, DataType::INT16},
98 : {"aiv_broadcast_uint16_t", HcclCMDType::HCCL_CMD_BROADCAST, DataType::UINT16},
99 : {"aiv_broadcast_float", HcclCMDType::HCCL_CMD_BROADCAST, DataType::FP32},
100 : {"aiv_broadcast_uint64_t", HcclCMDType::HCCL_CMD_BROADCAST, DataType::FP64},
101 : {"aiv_broadcast_int32_t", HcclCMDType::HCCL_CMD_BROADCAST, DataType::INT32},
102 : {"aiv_broadcast_uint32_t", HcclCMDType::HCCL_CMD_BROADCAST, DataType::UINT32},
103 : {"aiv_broadcast_int8_t", HcclCMDType::HCCL_CMD_BROADCAST, DataType::INT8},
104 : {"aiv_broadcast_uint8_t", HcclCMDType::HCCL_CMD_BROADCAST, DataType::UINT8},
105 : {"aiv_broadcast_bfloat16_t", HcclCMDType::HCCL_CMD_BROADCAST, DataType::BFP16},
106 : {"aiv_broadcast_uint64_t", HcclCMDType::HCCL_CMD_BROADCAST, DataType::INT64},
107 : {"aiv_broadcast_int64_t", HcclCMDType::HCCL_CMD_BROADCAST, DataType::UINT64},
108 : {"aiv_broadcast_hifloat8_t", HcclCMDType::HCCL_CMD_BROADCAST, DataType::HIF8},
109 : {"aiv_broadcast_fp8_e4m3fn_t", HcclCMDType::HCCL_CMD_BROADCAST, DataType::FP8E4M3},
110 : {"aiv_broadcast_fp8_e5m2_t", HcclCMDType::HCCL_CMD_BROADCAST, DataType::FP8E5M2},
111 : {"aiv_broadcast_fp8_e8m0_t", HcclCMDType::HCCL_CMD_BROADCAST, DataType::FP8E8M0},
112 : // allreduce two shot
113 : {"aiv_allreduce_mesh1d_twoshot_half", HcclCMDType::HCCL_CMD_ALLREDUCE, DataType::FP16,
114 : KernelArgsType::ARGS_TYPE_TWO_SHOT},
115 : {"aiv_allreduce_mesh1d_twoshot_int16_t", HcclCMDType::HCCL_CMD_ALLREDUCE, DataType::INT16,
116 : KernelArgsType::ARGS_TYPE_TWO_SHOT},
117 : {"aiv_allreduce_mesh1d_twoshot_float", HcclCMDType::HCCL_CMD_ALLREDUCE, DataType::FP32,
118 : KernelArgsType::ARGS_TYPE_TWO_SHOT},
119 : {"aiv_allreduce_mesh1d_twoshot_int32_t", HcclCMDType::HCCL_CMD_ALLREDUCE, DataType::INT32,
120 : KernelArgsType::ARGS_TYPE_TWO_SHOT},
121 : {"aiv_allreduce_mesh1d_twoshot_int8_t", HcclCMDType::HCCL_CMD_ALLREDUCE, DataType::INT8,
122 : KernelArgsType::ARGS_TYPE_TWO_SHOT},
123 : {"aiv_allreduce_mesh1d_twoshot_bfloat16_t", HcclCMDType::HCCL_CMD_ALLREDUCE, DataType::BFP16,
124 : KernelArgsType::ARGS_TYPE_TWO_SHOT},
125 : {"aiv_allreduce_mesh1d_twoshot_int64_t", HcclCMDType::HCCL_CMD_ALLREDUCE, DataType::INT64,
126 : KernelArgsType::ARGS_TYPE_TWO_SHOT},
127 : // alltoall
128 : {"aiv_alltoall_half", HcclCMDType::HCCL_CMD_ALLTOALL, DataType::FP16},
129 : {"aiv_alltoall_int16_t", HcclCMDType::HCCL_CMD_ALLTOALL, DataType::INT16},
130 : {"aiv_alltoall_uint16_t", HcclCMDType::HCCL_CMD_ALLTOALL, DataType::UINT16},
131 : {"aiv_alltoall_float", HcclCMDType::HCCL_CMD_ALLTOALL, DataType::FP32},
132 : {"aiv_alltoall_uint64_t", HcclCMDType::HCCL_CMD_ALLTOALL, DataType::FP64},
133 : {"aiv_alltoall_int32_t", HcclCMDType::HCCL_CMD_ALLTOALL, DataType::INT32},
134 : {"aiv_alltoall_uint32_t", HcclCMDType::HCCL_CMD_ALLTOALL, DataType::UINT32},
135 : {"aiv_alltoall_int8_t", HcclCMDType::HCCL_CMD_ALLTOALL, DataType::INT8},
136 : {"aiv_alltoall_uint8_t", HcclCMDType::HCCL_CMD_ALLTOALL, DataType::UINT8},
137 : {"aiv_alltoall_bfloat16_t", HcclCMDType::HCCL_CMD_ALLTOALL, DataType::BFP16},
138 : {"aiv_alltoall_uint64_t", HcclCMDType::HCCL_CMD_ALLTOALL, DataType::INT64},
139 : {"aiv_alltoall_int64_t", HcclCMDType::HCCL_CMD_ALLTOALL, DataType::UINT64},
140 : {"aiv_alltoall_hifloat8_t", HcclCMDType::HCCL_CMD_ALLTOALL, DataType::HIF8},
141 : {"aiv_alltoall_fp8_e4m3fn_t", HcclCMDType::HCCL_CMD_ALLTOALL, DataType::FP8E4M3},
142 : {"aiv_alltoall_fp8_e5m2_t", HcclCMDType::HCCL_CMD_ALLTOALL, DataType::FP8E5M2},
143 : {"aiv_alltoall_fp8_e8m0_t", HcclCMDType::HCCL_CMD_ALLTOALL, DataType::FP8E8M0},
144 : // alltoallv
145 : {"aiv_alltoallv_half", HcclCMDType::HCCL_CMD_ALLTOALLV, DataType::FP16},
146 : {"aiv_alltoallv_int16_t", HcclCMDType::HCCL_CMD_ALLTOALLV, DataType::INT16},
147 : {"aiv_alltoallv_uint16_t", HcclCMDType::HCCL_CMD_ALLTOALLV, DataType::UINT16},
148 : {"aiv_alltoallv_float", HcclCMDType::HCCL_CMD_ALLTOALLV, DataType::FP32},
149 : {"aiv_alltoallv_uint64_t", HcclCMDType::HCCL_CMD_ALLTOALLV, DataType::FP64},
150 : {"aiv_alltoallv_int32_t", HcclCMDType::HCCL_CMD_ALLTOALLV, DataType::INT32},
151 : {"aiv_alltoallv_uint32_t", HcclCMDType::HCCL_CMD_ALLTOALLV, DataType::UINT32},
152 : {"aiv_alltoallv_int8_t", HcclCMDType::HCCL_CMD_ALLTOALLV, DataType::INT8},
153 : {"aiv_alltoallv_uint8_t", HcclCMDType::HCCL_CMD_ALLTOALLV, DataType::UINT8},
154 : {"aiv_alltoallv_bfloat16_t", HcclCMDType::HCCL_CMD_ALLTOALLV, DataType::BFP16},
155 : {"aiv_alltoallv_uint64_t", HcclCMDType::HCCL_CMD_ALLTOALLV, DataType::INT64},
156 : {"aiv_alltoallv_int64_t", HcclCMDType::HCCL_CMD_ALLTOALLV, DataType::UINT64},
157 : {"aiv_alltoallv_hifloat8_t", HcclCMDType::HCCL_CMD_ALLTOALLV, DataType::HIF8},
158 : {"aiv_alltoallv_fp8_e4m3fn_t", HcclCMDType::HCCL_CMD_ALLTOALLV, DataType::FP8E4M3},
159 : {"aiv_alltoallv_fp8_e5m2_t", HcclCMDType::HCCL_CMD_ALLTOALLV, DataType::FP8E5M2},
160 : {"aiv_alltoallv_fp8_e8m0_t", HcclCMDType::HCCL_CMD_ALLTOALLV, DataType::FP8E8M0},
161 : // reduce
162 : {"aiv_reduce_half", HcclCMDType::HCCL_CMD_REDUCE, DataType::FP16},
163 : {"aiv_reduce_int16_t", HcclCMDType::HCCL_CMD_REDUCE, DataType::INT16},
164 : {"aiv_reduce_float", HcclCMDType::HCCL_CMD_REDUCE, DataType::FP32},
165 : {"aiv_reduce_int32_t", HcclCMDType::HCCL_CMD_REDUCE, DataType::INT32},
166 : {"aiv_reduce_int8_t", HcclCMDType::HCCL_CMD_REDUCE, DataType::INT8},
167 : {"aiv_reduce_bfloat16_t", HcclCMDType::HCCL_CMD_REDUCE, DataType::BFP16},
168 : {"aiv_reduce_int64_t", HcclCMDType::HCCL_CMD_REDUCE, DataType::INT64},
169 : // reducescatter
170 : {"aiv_reduce_scatter_half", HcclCMDType::HCCL_CMD_REDUCE_SCATTER, DataType::FP16},
171 : {"aiv_reduce_scatter_int16_t", HcclCMDType::HCCL_CMD_REDUCE_SCATTER, DataType::INT16},
172 : {"aiv_reduce_scatter_float", HcclCMDType::HCCL_CMD_REDUCE_SCATTER, DataType::FP32},
173 : {"aiv_reduce_scatter_int32_t", HcclCMDType::HCCL_CMD_REDUCE_SCATTER, DataType::INT32},
174 : {"aiv_reduce_scatter_int8_t", HcclCMDType::HCCL_CMD_REDUCE_SCATTER, DataType::INT8},
175 : {"aiv_reduce_scatter_bfloat16_t", HcclCMDType::HCCL_CMD_REDUCE_SCATTER, DataType::BFP16},
176 : {"aiv_reduce_scatter_int64_t", HcclCMDType::HCCL_CMD_REDUCE_SCATTER, DataType::INT64},
177 : // send
178 : {"aiv_send_int8_t", HcclCMDType::HCCL_CMD_SEND, DataType::INT8}, // hccl_types.h
179 : {"aiv_send_int16_t", HcclCMDType::HCCL_CMD_SEND, DataType::INT16},
180 : {"aiv_send_int32_t", HcclCMDType::HCCL_CMD_SEND, DataType::INT32},
181 : {"aiv_send_int64_t", HcclCMDType::HCCL_CMD_SEND, DataType::INT64},
182 : {"aiv_send_uint8_t", HcclCMDType::HCCL_CMD_SEND, DataType::UINT8},
183 : {"aiv_send_uint16_t", HcclCMDType::HCCL_CMD_SEND, DataType::UINT16},
184 : {"aiv_send_uint32_t", HcclCMDType::HCCL_CMD_SEND, DataType::UINT32},
185 : {"aiv_send_uint64_t", HcclCMDType::HCCL_CMD_SEND, DataType::UINT64},
186 : {"aiv_send_half", HcclCMDType::HCCL_CMD_SEND, DataType::FP16},
187 : {"aiv_send_float", HcclCMDType::HCCL_CMD_SEND, DataType::FP32},
188 : {"aiv_send_uint64_t", HcclCMDType::HCCL_CMD_SEND, DataType::FP64},
189 : {"aiv_send_bfloat16_t", HcclCMDType::HCCL_CMD_SEND, DataType::BFP16},
190 : {"aiv_send_hifloat8_t", HcclCMDType::HCCL_CMD_SEND, DataType::HIF8},
191 : {"aiv_send_fp8_e4m3fn_t", HcclCMDType::HCCL_CMD_SEND, DataType::FP8E4M3},
192 : {"aiv_send_fp8_e5m2_t", HcclCMDType::HCCL_CMD_SEND, DataType::FP8E5M2},
193 : {"aiv_send_fp8_e8m0_t", HcclCMDType::HCCL_CMD_SEND, DataType::FP8E8M0},
194 : // recv
195 : {"aiv_recv_int8_t", HcclCMDType::HCCL_CMD_RECEIVE, DataType::INT8},
196 : {"aiv_recv_int16_t", HcclCMDType::HCCL_CMD_RECEIVE, DataType::INT16},
197 : {"aiv_recv_int32_t", HcclCMDType::HCCL_CMD_RECEIVE, DataType::INT32},
198 : {"aiv_recv_int64_t", HcclCMDType::HCCL_CMD_RECEIVE, DataType::INT64},
199 : {"aiv_recv_uint8_t", HcclCMDType::HCCL_CMD_RECEIVE, DataType::UINT8},
200 : {"aiv_recv_uint16_t", HcclCMDType::HCCL_CMD_RECEIVE, DataType::UINT16},
201 : {"aiv_recv_uint32_t", HcclCMDType::HCCL_CMD_RECEIVE, DataType::UINT32},
202 : {"aiv_recv_uint64_t", HcclCMDType::HCCL_CMD_RECEIVE, DataType::UINT64},
203 : {"aiv_recv_half", HcclCMDType::HCCL_CMD_RECEIVE, DataType::FP16},
204 : {"aiv_recv_float", HcclCMDType::HCCL_CMD_RECEIVE, DataType::FP32},
205 : {"aiv_recv_uint64_t", HcclCMDType::HCCL_CMD_RECEIVE, DataType::FP64},
206 : {"aiv_recv_bfloat16_t", HcclCMDType::HCCL_CMD_RECEIVE, DataType::BFP16},
207 : {"aiv_recv_hifloat8_t", HcclCMDType::HCCL_CMD_RECEIVE, DataType::HIF8},
208 : {"aiv_recv_fp8_e4m3fn_t", HcclCMDType::HCCL_CMD_RECEIVE, DataType::FP8E4M3},
209 : {"aiv_recv_fp8_e5m2_t", HcclCMDType::HCCL_CMD_RECEIVE, DataType::FP8E5M2},
210 : {"aiv_recv_fp8_e8m0_t", HcclCMDType::HCCL_CMD_RECEIVE, DataType::FP8E8M0},
211 : // batchSendRecv
212 : {"aiv_batchSendRecv_int8_t", HcclCMDType::HCCL_CMD_BATCH_SEND_RECV, DataType::INT8},
213 : {"aiv_batchSendRecv_int16_t", HcclCMDType::HCCL_CMD_BATCH_SEND_RECV, DataType::INT16},
214 : {"aiv_batchSendRecv_int32_t", HcclCMDType::HCCL_CMD_BATCH_SEND_RECV, DataType::INT32},
215 : {"aiv_batchSendRecv_int64_t", HcclCMDType::HCCL_CMD_BATCH_SEND_RECV, DataType::INT64},
216 : {"aiv_batchSendRecv_uint8_t", HcclCMDType::HCCL_CMD_BATCH_SEND_RECV, DataType::UINT8},
217 : {"aiv_batchSendRecv_uint16_t", HcclCMDType::HCCL_CMD_BATCH_SEND_RECV, DataType::UINT16},
218 : {"aiv_batchSendRecv_uint32_t", HcclCMDType::HCCL_CMD_BATCH_SEND_RECV, DataType::UINT32},
219 : {"aiv_batchSendRecv_uint64_t", HcclCMDType::HCCL_CMD_BATCH_SEND_RECV, DataType::UINT64},
220 : {"aiv_batchSendRecv_half", HcclCMDType::HCCL_CMD_BATCH_SEND_RECV, DataType::FP16},
221 : {"aiv_batchSendRecv_float", HcclCMDType::HCCL_CMD_BATCH_SEND_RECV, DataType::FP32},
222 : {"aiv_batchSendRecv_uint64_t", HcclCMDType::HCCL_CMD_BATCH_SEND_RECV, DataType::FP64},
223 : {"aiv_batchSendRecv_bfloat16_t", HcclCMDType::HCCL_CMD_BATCH_SEND_RECV, DataType::BFP16},
224 : {"aiv_batchSendRecv_hifloat8_t", HcclCMDType::HCCL_CMD_BATCH_SEND_RECV, DataType::HIF8},
225 : {"aiv_batchSendRecv_fp8_e4m3fn_t", HcclCMDType::HCCL_CMD_BATCH_SEND_RECV, DataType::FP8E4M3},
226 : {"aiv_batchSendRecv_fp8_e5m2_t", HcclCMDType::HCCL_CMD_BATCH_SEND_RECV, DataType::FP8E5M2},
227 : {"aiv_batchSendRecv_fp8_e8m0_t", HcclCMDType::HCCL_CMD_BATCH_SEND_RECV, DataType::FP8E8M0},
228 : };
229 : using AivExtraKernelArgs = struct AivExtraKernelArgsDef {
230 : const void* buffersIn; // 注册的CCLIN地址，所有卡可访问
231 : u64 input;
232 : u64 output;
233 : u32 rank;
234 : u32 sendRecvRemoteRank;
235 : u32 rankSize;
236 : u64 xRankSize;
237 : u64 yRankSize;
238 : u64 zRankSize;
239 : u64 len;
240 : u32 dataType;
241 : u32 reduceOp;
242 : u32 root;
243 : u32 tag; // 第几次调用，定时重置成1
244 : u64 inputSliceStride;
245 : u64 outputSliceStride;
246 : u64 repeatNum;
247 : u64 inputRepeatStride;
248 : u64 outputRepeatStride;
249 : bool isOpBase;
250 : const void* headCountMem;
251 : const void* tailCountMem;
252 : const void* addOneMem;
253 : u32 counterMemSize;
254 : bool isEnableCounter;
255 : ExtraArgsA2A extraArgs;
256 :
257 1 : AivExtraKernelArgsDef(
258 : const void* buffIn, u64 input, u64 output, u32 rank, u32 sendRecvRemoteRank, u32 rankSize, u64 xRankSize,
259 : u64 yRankSize, u64 zRankSize, u64 len, u32 dataType, u32 reduceOp, u32 root, u32 tag, u64 inputSliceStride,
260 : u64 outputSliceStride, u64 repeatNum, u64 inputRepeatStride, u64 outputRepeatStride, bool isOpBase = true,
261 : const void* headCountMem = nullptr, const void* tailCountMem = nullptr, const void* addOneMem = nullptr,
262 : u32 counterMemSize = 0, const ExtraArgsA2A* extraArgsPtr = nullptr)
263 1 : : buffersIn(buffIn),
264 1 : input(input),
265 1 : output(output),
266 1 : rank(rank),
267 1 : sendRecvRemoteRank(sendRecvRemoteRank),
268 1 : rankSize(rankSize),
269 1 : xRankSize(xRankSize),
270 1 : yRankSize(yRankSize),
271 1 : zRankSize(zRankSize),
272 1 : len(len),
273 1 : dataType(dataType),
274 1 : reduceOp(reduceOp),
275 1 : root(root),
276 1 : tag(tag),
277 1 : inputSliceStride(inputSliceStride),
278 1 : outputSliceStride(outputSliceStride),
279 1 : repeatNum(repeatNum),
280 1 : inputRepeatStride(inputRepeatStride),
281 1 : outputRepeatStride(outputRepeatStride),
282 1 : isOpBase(isOpBase),
283 1 : headCountMem(headCountMem),
284 1 : tailCountMem(tailCountMem),
285 1 : addOneMem(addOneMem),
286 1 : counterMemSize(counterMemSize)
287 : {
288 1 : if (extraArgsPtr != nullptr) {
289 1 : extraArgs = *extraArgsPtr;
290 : }
291 1 : }
292 : };
293 :
294 1 : HcclResult GetAivOpBinaryPath(std::string& binaryPath)
295 : {
296 1 : char* envValue = nullptr;
297 1 : MM_SYS_GET_ENV(MM_ENV_ASCEND_HOME_PATH, envValue);
298 :
299 1 : std::string libPath;
300 1 : if (envValue != nullptr) {
301 1 : libPath = envValue;
302 : } else {
303 0 : libPath = "/usr/local/Ascend/cann";
304 0 : HCCL_WARNING("[AIV][GetAivOpBinaryPath]ENV:ASCEND_HOME_PATH is not set, use default path[%s]", libPath.c_str());
305 : }
306 :
307 1 : binaryPath = libPath + "/lib64/hccl_aiv_op_910_95.o";
308 :
309 3 : HCCL_INFO("[AIV][GetAivOpBinaryPath]binaryPath: %s", binaryPath.c_str());
310 :
311 1 : return HCCL_SUCCESS;
312 1 : }
313 :
314 157 : s8* GetStubFunc(HcclCMDType cmdType, DataType dataType, KernelArgsType argsType = KernelArgsType::ARGS_TYPE_SERVER)
315 : {
316 : return reinterpret_cast<s8*>(
317 157 : (((static_cast<s64>(cmdType) << SIG_MOVE_LEFT_BITS) + static_cast<s64>(dataType)) << SIG_MOVE_LEFT_BITS)
318 157 : + static_cast<s64>(argsType));
319 : }
320 :
321 156 : HcclResult RegisterBinaryKernel(const char* funcName, const aclrtBinHandle binHandle, const s8* stubFunc)
322 : {
323 156 : if (stubFunc == nullptr) {
324 0 : return HCCL_E_PARA;
325 : }
326 :
327 : aclrtFuncHandle funcHandle;
328 156 : aclError aclRet = aclrtBinaryGetFunction(binHandle, funcName, &funcHandle);
329 156 : CHK_PRT_RET(
330 : aclRet != ACL_SUCCESS,
331 : HCCL_ERROR("[RegisterBinaryKernel]errNo[0x%016llx] get function from binary error.", aclRet), HCCL_E_NOT_FOUND);
332 :
333 156 : g_aivFuncMap[stubFunc] = funcHandle;
334 156 : g_aivNameMap[stubFunc] = funcName;
335 :
336 156 : return HCCL_SUCCESS;
337 : }
338 :
339 42 : HcclResult RegisterKernel()
340 : {
341 42 : lock_guard<mutex> guard(g_mut);
342 42 : if (g_init) {
343 41 : return HCCL_SUCCESS;
344 : }
345 :
346 : HcclResult ret;
347 1 : string binFilePath;
348 1 : ret = GetAivOpBinaryPath(binFilePath);
349 3 : HCCL_INFO("[RegisterKernel] binFilePath: %s", binFilePath.c_str());
350 1 : CHK_PRT_RET(ret != HCCL_SUCCESS, HCCL_ERROR("[AIV][RegisterKernel] get aiv op binary path failed"), HCCL_E_RUNTIME);
351 :
352 1 : LoadBinaryFromFile(binFilePath.c_str(), ACL_RT_BINARY_LOAD_OPT_LAZY_LOAD, 1, g_binHandle);
353 157 : for (auto& aivKernelInfo : g_aivKernelInfoList) {
354 312 : ret = RegisterBinaryKernel(
355 : aivKernelInfo.kernelName, g_binHandle,
356 156 : GetStubFunc(aivKernelInfo.cmdType, aivKernelInfo.dataType, aivKernelInfo.argsType));
357 :
358 156 : CHK_PRT_RET(
359 : ret != HCCL_SUCCESS,
360 : HCCL_ERROR(
361 : "[AIV][RegisterKernel] register binary kernel for kernelName[%s] "
362 : "cmdType[%d] dataType[%d] argsType[%d] failed",
363 : aivKernelInfo.kernelName, aivKernelInfo.cmdType, aivKernelInfo.dataType, aivKernelInfo.argsType),
364 : HCCL_E_RUNTIME);
365 : }
366 :
367 1 : g_init = true;
368 :
369 1 : return HCCL_SUCCESS;
370 42 : }
371 :
372 0 : HcclResult UnRegisterAivKernel()
373 : {
374 0 : lock_guard<mutex> guard(g_mut);
375 0 : if (g_init) {
376 0 : aclError aclRet = aclrtBinaryUnLoad(g_binHandle);
377 0 : CHK_PRT_RET(
378 : aclRet != ACL_SUCCESS, HCCL_ERROR("[UnRegisterAivKernel] aclrtBinaryUnLoad failed, ret[%d]", aclRet),
379 : HCCL_E_RUNTIME);
380 0 : g_aivFuncMap.clear();
381 0 : g_init = false;
382 : }
383 0 : return HCCL_SUCCESS;
384 0 : }
385 :
386 1 : HcclResult GetMinAndMaxNpuSchedTimeOut(u64& minNpuSchedTimeout, u64& maxNpuSchedTimeout)
387 : {
388 1 : uint64_t interval = 0;
389 1 : aclError aclRet = aclrtGetOpTimeOutInterval(&interval);
390 1 : CHK_PRT_RET(
391 : aclRet != ACL_SUCCESS, HCCL_ERROR("aclrtGetOpTimeOutInterval get timeout interval failed, ret[%d]", aclRet),
392 : HCCL_E_RUNTIME);
393 :
394 : // NPU超时范围(1, 254) * interval
395 1 : minNpuSchedTimeout = MIN_NPU_TIMEOUT * interval;
396 1 : maxNpuSchedTimeout = MAX_NPU_TIMEOUT * interval;
397 3 : HCCL_INFO(
398 : "GetMinAndMaxNpuSchedTimeOut minNpuSchedTimeout[%u]us, maxNpuSchedTimeout[%u]us.", minNpuSchedTimeout,
399 : maxNpuSchedTimeout);
400 1 : return HCCL_SUCCESS;
401 : }
402 :
403 1 : u32 GetAivTimeout()
404 : {
405 1 : constexpr u32 TIME_S_TO_US = 1000000;
406 1 : constexpr u32 AIV_TIMEOUT_DEFAULT_US = 1091 * TIME_S_TO_US;
407 1 : constexpr u32 AIV_TIMEOUT_MAX_US = 1091 * TIME_S_TO_US;
408 1 : u32 timeout = AIV_TIMEOUT_DEFAULT_US;
409 1 : double execTimeOut = EnvConfig::GetInstance().GetRtsConfig().GetAivExecTimeOut();
410 :
411 1 : double timeoutUs = execTimeOut * TIME_S_TO_US;
412 1 : if (timeoutUs > static_cast<double>(std::numeric_limits<u32>::max())) {
413 0 : HCCL_INFO("[GetAivTimeout]Get input timeout[%.2f us] is out of valid range.", timeoutUs);
414 0 : return AIV_TIMEOUT_MAX_US;
415 : }
416 1 : u32 timeoutUsInt = static_cast<u32>(timeoutUs);
417 1 : if (timeoutUsInt == 0) {
418 0 : timeoutUsInt = AIV_TIMEOUT_MAX_US;
419 : }
420 1 : u64 minNpuSchedTimeout = 0;
421 1 : u64 maxNpuSchedTimeout = 0;
422 1 : CHK_RET(GetMinAndMaxNpuSchedTimeOut(minNpuSchedTimeout, maxNpuSchedTimeout));
423 1 : timeout = (timeoutUsInt < minNpuSchedTimeout) ? minNpuSchedTimeout :
424 1 : (timeoutUsInt > maxNpuSchedTimeout) ? maxNpuSchedTimeout :
425 : timeoutUsInt;
426 3 : HCCL_INFO(
427 : "[GetAivTimeout]timeout[%u]us, execTimeOut[%.2f]s, minNpuSchedTimeout[%u]us, maxNpuSchedTimeout[%u]us.",
428 : timeout, execTimeOut, minNpuSchedTimeout, maxNpuSchedTimeout);
429 :
430 1 : return timeout;
431 : }
432 :
433 1 : HcclResult GetKernelFunc(aclrtFuncHandle& funcHandle, const s8* stubFunc)
434 : {
435 1 : if (stubFunc == nullptr || g_aivFuncMap.find(stubFunc) == g_aivFuncMap.end()) {
436 3 : HCCL_ERROR("[GetKernelFunc] stubFunc not found in g_aivFuncMap");
437 1 : return HCCL_E_PARA;
438 : }
439 0 : funcHandle = g_aivFuncMap[stubFunc];
440 0 : return HCCL_SUCCESS;
441 : }
442 :
443 : // KernelLaunch内部接口
444 1 : HcclResult ExecuteKernelLaunchInner(const AivOpArgs& opArgs, void* args, u32 argsSize)
445 : {
446 1 : constexpr u32 AIV_ATTRNUM_THREE = 3;
447 3 : HCCL_INFO(
448 : "[AIV][ExecuteKernelLaunch] sendbuff [%llu] recvbuff [%llu] rank [%u] sendRecvRemoteRank [%u] rankSize [%u] "
449 : "count [%llu] "
450 : "dataType [%d] reduceOp [%d] root [%u] tag [%u] isOpBase [%d] "
451 : "extraArgsPtr [%p] argsSize [%u] numBlocks [%u]",
452 : opArgs.input, opArgs.output, opArgs.rank, opArgs.sendRecvRemoteRank, opArgs.rankSize, opArgs.count,
453 : opArgs.dataType, opArgs.op, opArgs.root, opArgs.aivTag, opArgs.isOpBase, args, argsSize, opArgs.numBlocks);
454 :
455 : aclrtLaunchKernelCfg cfg;
456 : aclrtLaunchKernelAttr attr[AIV_ATTRNUM_THREE];
457 :
458 1 : u32 timeoutUs = GetAivTimeout();
459 1 : attr[0].id = ACL_RT_LAUNCH_KERNEL_ATTR_SCHEM_MODE;
460 1 : attr[0].value.schemMode = 1;
461 1 : attr[1].id = ACL_RT_LAUNCH_KERNEL_ATTR_TIMEOUT_US;
462 1 : attr[1].value.timeoutUs.timeoutLow = timeoutUs;
463 1 : attr[1].value.timeoutUs.timeoutHigh = 0;
464 1 : attr[2].id = ACL_RT_LAUNCH_KERNEL_ATTR_ENGINE_TYPE;
465 1 : attr[2].value.engineType = ACL_RT_ENGINE_TYPE_AIV;
466 1 : cfg.numAttrs = AIV_ATTRNUM_THREE;
467 1 : cfg.attrs = attr;
468 :
469 3 : HCCL_INFO(
470 : "[AIV][ExecuteKernelLaunch] KernelAttr attr[0]: id=[%u], schemMode=[%u]; attr[1]: id=[%u], "
471 : "timeoutLow=[%u us], timeoutHigh=[%u us]; attr[2]: id=[%u], engineType=[%u]; cfg: numAttrs=[%u]",
472 : attr[0].id, attr[0].value.schemMode, attr[1].id, attr[1].value.timeoutUs.timeoutLow,
473 : attr[1].value.timeoutUs.timeoutHigh, attr[2].id, attr[2].value.engineType, cfg.numAttrs);
474 :
475 : aclrtFuncHandle funcHandle;
476 1 : const s8* stubFunc = GetStubFunc(opArgs.cmdType, opArgs.dataType, opArgs.argsType);
477 1 : HcclResult ret = GetKernelFunc(funcHandle, stubFunc);
478 4 : CHK_PRT_RET(
479 : ret != HCCL_SUCCESS,
480 : HCCL_ERROR(
481 : "[ExecuteKernelLaunchInner] errNo[0x%016llx] GetKernelFunc failed, "
482 : "return[%d]",
483 : HCCL_ERROR_CODE(HCCL_E_RUNTIME), ret),
484 : HCCL_E_RUNTIME);
485 :
486 : aclError aclRet
487 0 : = aclrtLaunchKernelWithHostArgs(funcHandle, opArgs.numBlocks, opArgs.stream, &cfg, args, argsSize, nullptr, 0);
488 0 : if (aclRet == ACL_ERROR_RT_INVALID_HANDLE) {
489 0 : aclError aclGetRet = aclrtBinaryGetFunction(g_binHandle, g_aivNameMap[stubFunc].c_str(), &funcHandle);
490 0 : CHK_PRT_RET(
491 : aclGetRet != ACL_SUCCESS,
492 : HCCL_ERROR("[RegisterBinaryKernel]errNo[0x%016llx] get function from binary error.", aclRet),
493 : HCCL_E_NOT_FOUND);
494 0 : aclRet = aclrtLaunchKernelWithHostArgs(
495 0 : funcHandle, opArgs.numBlocks, opArgs.stream, &cfg, args, argsSize, nullptr, 0);
496 : }
497 0 : CHK_PRT_RET(
498 : aclRet != ACL_SUCCESS,
499 : HCCL_ERROR(
500 : "[ExecuteKernelLaunchInner]errNo[0x%016llx] aclrtLaunchKernelWithHostArgs error[%d].",
501 : HCCL_ERROR_CODE(HCCL_E_RUNTIME), aclRet),
502 : HCCL_E_RUNTIME);
503 0 : return HCCL_SUCCESS;
504 : }
505 :
506 : // Kernel单次调用Launch外部接口
507 1 : HcclResult ExecuteKernelLaunch(const AivOpArgs& opArgs)
508 : {
509 : AivExtraKernelArgs aivExtraKernelArgs{
510 1 : opArgs.buffersIn,
511 1 : opArgs.input,
512 1 : opArgs.output,
513 1 : opArgs.rank,
514 1 : opArgs.sendRecvRemoteRank,
515 1 : opArgs.rankSize,
516 1 : opArgs.xRankSize,
517 1 : opArgs.yRankSize,
518 1 : opArgs.zRankSize,
519 1 : opArgs.count,
520 : opArgs.dataType,
521 : opArgs.op,
522 1 : opArgs.root,
523 1 : opArgs.aivTag,
524 1 : opArgs.inputSliceStride,
525 1 : opArgs.outputSliceStride,
526 1 : opArgs.repeatNum,
527 1 : opArgs.inputRepeatStride,
528 1 : opArgs.outputRepeatStride,
529 1 : opArgs.isOpBase,
530 1 : reinterpret_cast<void*>(opArgs.counter.headCountMem),
531 1 : reinterpret_cast<void*>(opArgs.counter.tailCountMem),
532 1 : reinterpret_cast<void*>(opArgs.counter.addOneMem),
533 1 : opArgs.counter.memSize,
534 : &opArgs.extraArgs // 这里是真正发送的地方，device侧单个kernel不能超过32K，先写死1024个结构体
535 1 : };
536 4 : CHK_RET(ExecuteKernelLaunchInner(opArgs, &aivExtraKernelArgs, sizeof(aivExtraKernelArgs)));
537 :
538 0 : return HCCL_SUCCESS;
539 : }
540 :
541 : } // namespace Hccl
|
