Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include <mutex>
12 : #include <vector>
13 : #include <iostream>
14 : #include <fstream>
15 : #include "mmpa_api.h"
16 : #include "acl/acl_rt.h"
17 : #include "env_config.h"
18 : #include "hccl_aiv_utils.h"
19 : #include "aicpu/launch_device.h"
20 :
21 : using namespace std;
22 :
23 : namespace Hccl {
24 : constexpr u32 SIG_MOVE_LEFT_BITS = 20;
25 : constexpr u32 MAX_BIN_FILE_SIZE = 100 * 1024 * 1024;
26 : constexpr s32 RESET_TAIL_SYNC_TAG = 2;
27 : constexpr uint64_t MIN_NPU_TIMEOUT = 1;
28 : constexpr uint64_t MAX_NPU_TIMEOUT = 254;
29 :
30 : static bool g_init = false;
31 : static mutex g_mut;
32 : static aclrtBinHandle g_binHandle;
33 : static std::unordered_map<const s8*, aclrtFuncHandle> g_aivFuncMap;
34 : static std::unordered_map<const s8*, std::string> g_aivNameMap;
35 :
36 : using AivKernelInfo = struct AivKernelInfoDef {
37 : const char* kernelName;
38 : HcclCMDType cmdType;
39 : DataType dataType;
40 : KernelArgsType argsType;
41 :
42 156 : AivKernelInfoDef(const char* kernelName, HcclCMDType cmdType, DataType dataType,
43 : KernelArgsType argsType = KernelArgsType::ARGS_TYPE_SERVER)
44 156 : : kernelName(kernelName), cmdType(cmdType), dataType(dataType), argsType(argsType)
45 : {
46 156 : }
47 : };
48 :
49 : static std::vector<AivKernelInfo> g_aivKernelInfoList = {
50 : // scatter
51 : {"aiv_scatter_half", HcclCMDType::HCCL_CMD_SCATTER, DataType::FP16},
52 : {"aiv_scatter_int16_t", HcclCMDType::HCCL_CMD_SCATTER, DataType::INT16},
53 : {"aiv_scatter_uint16_t", HcclCMDType::HCCL_CMD_SCATTER, DataType::UINT16},
54 : {"aiv_scatter_float", HcclCMDType::HCCL_CMD_SCATTER, DataType::FP32},
55 : {"aiv_scatter_uint64_t", HcclCMDType::HCCL_CMD_SCATTER, DataType::FP64},
56 : {"aiv_scatter_int32_t", HcclCMDType::HCCL_CMD_SCATTER, DataType::INT32},
57 : {"aiv_scatter_uint32_t", HcclCMDType::HCCL_CMD_SCATTER, DataType::UINT32},
58 : {"aiv_scatter_int8_t", HcclCMDType::HCCL_CMD_SCATTER, DataType::INT8},
59 : {"aiv_scatter_uint8_t", HcclCMDType::HCCL_CMD_SCATTER, DataType::UINT8},
60 : {"aiv_scatter_bfloat16_t", HcclCMDType::HCCL_CMD_SCATTER, DataType::BFP16},
61 : {"aiv_scatter_uint64_t", HcclCMDType::HCCL_CMD_SCATTER, DataType::INT64},
62 : {"aiv_scatter_int64_t", HcclCMDType::HCCL_CMD_SCATTER, DataType::UINT64},
63 : {"aiv_scatter_hifloat8_t", HcclCMDType::HCCL_CMD_SCATTER, DataType::HIF8},
64 : {"aiv_scatter_fp8_e4m3fn_t", HcclCMDType::HCCL_CMD_SCATTER, DataType::FP8E4M3},
65 : {"aiv_scatter_fp8_e5m2_t", HcclCMDType::HCCL_CMD_SCATTER, DataType::FP8E5M2},
66 : {"aiv_scatter_fp8_e8m0_t", HcclCMDType::HCCL_CMD_SCATTER, DataType::FP8E8M0},
67 :
68 : {"aiv_all_gather_half", HcclCMDType::HCCL_CMD_ALLGATHER, DataType::FP16},
69 : {"aiv_all_gather_int16_t", HcclCMDType::HCCL_CMD_ALLGATHER, DataType::INT16},
70 : {"aiv_all_gather_uint16_t", HcclCMDType::HCCL_CMD_ALLGATHER, DataType::UINT16},
71 : {"aiv_all_gather_float", HcclCMDType::HCCL_CMD_ALLGATHER, DataType::FP32},
72 : {"aiv_all_gather_uint64_t", HcclCMDType::HCCL_CMD_ALLGATHER, DataType::FP64},
73 : {"aiv_all_gather_int32_t", HcclCMDType::HCCL_CMD_ALLGATHER, DataType::INT32},
74 : {"aiv_all_gather_uint32_t", HcclCMDType::HCCL_CMD_ALLGATHER, DataType::UINT32},
75 : {"aiv_all_gather_int8_t", HcclCMDType::HCCL_CMD_ALLGATHER, DataType::INT8},
76 : {"aiv_all_gather_uint8_t", HcclCMDType::HCCL_CMD_ALLGATHER, DataType::UINT8},
77 : {"aiv_all_gather_bfloat16_t", HcclCMDType::HCCL_CMD_ALLGATHER, DataType::BFP16},
78 : {"aiv_all_gather_uint64_t", HcclCMDType::HCCL_CMD_ALLGATHER, DataType::INT64},
79 : {"aiv_all_gather_int64_t", HcclCMDType::HCCL_CMD_ALLGATHER, DataType::UINT64},
80 : {"aiv_all_gather_hifloat8_t", HcclCMDType::HCCL_CMD_ALLGATHER, DataType::HIF8},
81 : {"aiv_all_gather_fp8_e4m3fn_t", HcclCMDType::HCCL_CMD_ALLGATHER, DataType::FP8E4M3},
82 : {"aiv_all_gather_fp8_e5m2_t", HcclCMDType::HCCL_CMD_ALLGATHER, DataType::FP8E5M2},
83 : {"aiv_all_gather_fp8_e8m0_t", HcclCMDType::HCCL_CMD_ALLGATHER, DataType::FP8E8M0},
84 : //allreduce
85 : {"aiv_allreduce_half", HcclCMDType::HCCL_CMD_ALLREDUCE, DataType::FP16},
86 : {"aiv_allreduce_int16_t", HcclCMDType::HCCL_CMD_ALLREDUCE, DataType::INT16},
87 : {"aiv_allreduce_float", HcclCMDType::HCCL_CMD_ALLREDUCE, DataType::FP32},
88 : {"aiv_allreduce_int32_t", HcclCMDType::HCCL_CMD_ALLREDUCE, DataType::INT32},
89 : {"aiv_allreduce_int8_t", HcclCMDType::HCCL_CMD_ALLREDUCE, DataType::INT8},
90 : {"aiv_allreduce_bfloat16_t", HcclCMDType::HCCL_CMD_ALLREDUCE, DataType::BFP16},
91 : {"aiv_allreduce_int64_t", HcclCMDType::HCCL_CMD_ALLREDUCE, DataType::INT64},
92 : //broadcast
93 : {"aiv_broadcast_half", HcclCMDType::HCCL_CMD_BROADCAST, DataType::FP16},
94 : {"aiv_broadcast_int16_t", HcclCMDType::HCCL_CMD_BROADCAST, DataType::INT16},
95 : {"aiv_broadcast_uint16_t", HcclCMDType::HCCL_CMD_BROADCAST, DataType::UINT16},
96 : {"aiv_broadcast_float", HcclCMDType::HCCL_CMD_BROADCAST, DataType::FP32},
97 : {"aiv_broadcast_uint64_t", HcclCMDType::HCCL_CMD_BROADCAST, DataType::FP64},
98 : {"aiv_broadcast_int32_t", HcclCMDType::HCCL_CMD_BROADCAST, DataType::INT32},
99 : {"aiv_broadcast_uint32_t", HcclCMDType::HCCL_CMD_BROADCAST, DataType::UINT32},
100 : {"aiv_broadcast_int8_t", HcclCMDType::HCCL_CMD_BROADCAST, DataType::INT8},
101 : {"aiv_broadcast_uint8_t", HcclCMDType::HCCL_CMD_BROADCAST, DataType::UINT8},
102 : {"aiv_broadcast_bfloat16_t", HcclCMDType::HCCL_CMD_BROADCAST, DataType::BFP16},
103 : {"aiv_broadcast_uint64_t", HcclCMDType::HCCL_CMD_BROADCAST, DataType::INT64},
104 : {"aiv_broadcast_int64_t", HcclCMDType::HCCL_CMD_BROADCAST, DataType::UINT64},
105 : {"aiv_broadcast_hifloat8_t", HcclCMDType::HCCL_CMD_BROADCAST, DataType::HIF8},
106 : {"aiv_broadcast_fp8_e4m3fn_t", HcclCMDType::HCCL_CMD_BROADCAST, DataType::FP8E4M3},
107 : {"aiv_broadcast_fp8_e5m2_t", HcclCMDType::HCCL_CMD_BROADCAST, DataType::FP8E5M2},
108 : {"aiv_broadcast_fp8_e8m0_t", HcclCMDType::HCCL_CMD_BROADCAST, DataType::FP8E8M0},
109 : // allreduce two shot
110 : {"aiv_allreduce_mesh1d_twoshot_half", HcclCMDType::HCCL_CMD_ALLREDUCE, DataType::FP16, KernelArgsType::ARGS_TYPE_TWO_SHOT},
111 : {"aiv_allreduce_mesh1d_twoshot_int16_t", HcclCMDType::HCCL_CMD_ALLREDUCE, DataType::INT16,KernelArgsType::ARGS_TYPE_TWO_SHOT},
112 : {"aiv_allreduce_mesh1d_twoshot_float", HcclCMDType::HCCL_CMD_ALLREDUCE, DataType::FP32,KernelArgsType::ARGS_TYPE_TWO_SHOT},
113 : {"aiv_allreduce_mesh1d_twoshot_int32_t", HcclCMDType::HCCL_CMD_ALLREDUCE, DataType::INT32,KernelArgsType::ARGS_TYPE_TWO_SHOT},
114 : {"aiv_allreduce_mesh1d_twoshot_int8_t", HcclCMDType::HCCL_CMD_ALLREDUCE, DataType::INT8,KernelArgsType::ARGS_TYPE_TWO_SHOT},
115 : {"aiv_allreduce_mesh1d_twoshot_bfloat16_t", HcclCMDType::HCCL_CMD_ALLREDUCE, DataType::BFP16,KernelArgsType::ARGS_TYPE_TWO_SHOT},
116 : {"aiv_allreduce_mesh1d_twoshot_int64_t", HcclCMDType::HCCL_CMD_ALLREDUCE, DataType::INT64,KernelArgsType::ARGS_TYPE_TWO_SHOT},
117 : // alltoall
118 : {"aiv_alltoall_half", HcclCMDType::HCCL_CMD_ALLTOALL, DataType::FP16},
119 : {"aiv_alltoall_int16_t", HcclCMDType::HCCL_CMD_ALLTOALL, DataType::INT16},
120 : {"aiv_alltoall_uint16_t", HcclCMDType::HCCL_CMD_ALLTOALL, DataType::UINT16},
121 : {"aiv_alltoall_float", HcclCMDType::HCCL_CMD_ALLTOALL, DataType::FP32},
122 : {"aiv_alltoall_uint64_t", HcclCMDType::HCCL_CMD_ALLTOALL, DataType::FP64},
123 : {"aiv_alltoall_int32_t", HcclCMDType::HCCL_CMD_ALLTOALL, DataType::INT32},
124 : {"aiv_alltoall_uint32_t", HcclCMDType::HCCL_CMD_ALLTOALL, DataType::UINT32},
125 : {"aiv_alltoall_int8_t", HcclCMDType::HCCL_CMD_ALLTOALL, DataType::INT8},
126 : {"aiv_alltoall_uint8_t", HcclCMDType::HCCL_CMD_ALLTOALL, DataType::UINT8},
127 : {"aiv_alltoall_bfloat16_t", HcclCMDType::HCCL_CMD_ALLTOALL, DataType::BFP16},
128 : {"aiv_alltoall_uint64_t", HcclCMDType::HCCL_CMD_ALLTOALL, DataType::INT64},
129 : {"aiv_alltoall_int64_t", HcclCMDType::HCCL_CMD_ALLTOALL, DataType::UINT64},
130 : {"aiv_alltoall_hifloat8_t", HcclCMDType::HCCL_CMD_ALLTOALL, DataType::HIF8},
131 : {"aiv_alltoall_fp8_e4m3fn_t", HcclCMDType::HCCL_CMD_ALLTOALL, DataType::FP8E4M3},
132 : {"aiv_alltoall_fp8_e5m2_t", HcclCMDType::HCCL_CMD_ALLTOALL, DataType::FP8E5M2},
133 : {"aiv_alltoall_fp8_e8m0_t", HcclCMDType::HCCL_CMD_ALLTOALL, DataType::FP8E8M0},
134 : // alltoallv
135 : {"aiv_alltoallv_half", HcclCMDType::HCCL_CMD_ALLTOALLV, DataType::FP16},
136 : {"aiv_alltoallv_int16_t", HcclCMDType::HCCL_CMD_ALLTOALLV, DataType::INT16},
137 : {"aiv_alltoallv_uint16_t", HcclCMDType::HCCL_CMD_ALLTOALLV, DataType::UINT16},
138 : {"aiv_alltoallv_float", HcclCMDType::HCCL_CMD_ALLTOALLV, DataType::FP32},
139 : {"aiv_alltoallv_uint64_t", HcclCMDType::HCCL_CMD_ALLTOALLV, DataType::FP64},
140 : {"aiv_alltoallv_int32_t", HcclCMDType::HCCL_CMD_ALLTOALLV, DataType::INT32},
141 : {"aiv_alltoallv_uint32_t", HcclCMDType::HCCL_CMD_ALLTOALLV, DataType::UINT32},
142 : {"aiv_alltoallv_int8_t", HcclCMDType::HCCL_CMD_ALLTOALLV, DataType::INT8},
143 : {"aiv_alltoallv_uint8_t", HcclCMDType::HCCL_CMD_ALLTOALLV, DataType::UINT8},
144 : {"aiv_alltoallv_bfloat16_t", HcclCMDType::HCCL_CMD_ALLTOALLV, DataType::BFP16},
145 : {"aiv_alltoallv_uint64_t", HcclCMDType::HCCL_CMD_ALLTOALLV, DataType::INT64},
146 : {"aiv_alltoallv_int64_t", HcclCMDType::HCCL_CMD_ALLTOALLV, DataType::UINT64},
147 : {"aiv_alltoallv_hifloat8_t", HcclCMDType::HCCL_CMD_ALLTOALLV, DataType::HIF8},
148 : {"aiv_alltoallv_fp8_e4m3fn_t", HcclCMDType::HCCL_CMD_ALLTOALLV, DataType::FP8E4M3},
149 : {"aiv_alltoallv_fp8_e5m2_t", HcclCMDType::HCCL_CMD_ALLTOALLV, DataType::FP8E5M2},
150 : {"aiv_alltoallv_fp8_e8m0_t", HcclCMDType::HCCL_CMD_ALLTOALLV, DataType::FP8E8M0},
151 : // reduce
152 : {"aiv_reduce_half", HcclCMDType::HCCL_CMD_REDUCE, DataType::FP16},
153 : {"aiv_reduce_int16_t", HcclCMDType::HCCL_CMD_REDUCE, DataType::INT16},
154 : {"aiv_reduce_float", HcclCMDType::HCCL_CMD_REDUCE, DataType::FP32},
155 : {"aiv_reduce_int32_t", HcclCMDType::HCCL_CMD_REDUCE, DataType::INT32},
156 : {"aiv_reduce_int8_t", HcclCMDType::HCCL_CMD_REDUCE, DataType::INT8},
157 : {"aiv_reduce_bfloat16_t", HcclCMDType::HCCL_CMD_REDUCE, DataType::BFP16},
158 : {"aiv_reduce_int64_t", HcclCMDType::HCCL_CMD_REDUCE, DataType::INT64},
159 : //reducescatter
160 : {"aiv_reduce_scatter_half", HcclCMDType::HCCL_CMD_REDUCE_SCATTER, DataType::FP16},
161 : {"aiv_reduce_scatter_int16_t", HcclCMDType::HCCL_CMD_REDUCE_SCATTER, DataType::INT16},
162 : {"aiv_reduce_scatter_float", HcclCMDType::HCCL_CMD_REDUCE_SCATTER, DataType::FP32},
163 : {"aiv_reduce_scatter_int32_t", HcclCMDType::HCCL_CMD_REDUCE_SCATTER, DataType::INT32},
164 : {"aiv_reduce_scatter_int8_t", HcclCMDType::HCCL_CMD_REDUCE_SCATTER, DataType::INT8},
165 : {"aiv_reduce_scatter_bfloat16_t", HcclCMDType::HCCL_CMD_REDUCE_SCATTER, DataType::BFP16},
166 : {"aiv_reduce_scatter_int64_t", HcclCMDType::HCCL_CMD_REDUCE_SCATTER, DataType::INT64},
167 : // send
168 : {"aiv_send_int8_t", HcclCMDType::HCCL_CMD_SEND, DataType::INT8}, // hccl_types.h
169 : {"aiv_send_int16_t", HcclCMDType::HCCL_CMD_SEND, DataType::INT16},
170 : {"aiv_send_int32_t", HcclCMDType::HCCL_CMD_SEND, DataType::INT32},
171 : {"aiv_send_int64_t", HcclCMDType::HCCL_CMD_SEND, DataType::INT64},
172 : {"aiv_send_uint8_t", HcclCMDType::HCCL_CMD_SEND, DataType::UINT8},
173 : {"aiv_send_uint16_t", HcclCMDType::HCCL_CMD_SEND, DataType::UINT16},
174 : {"aiv_send_uint32_t", HcclCMDType::HCCL_CMD_SEND, DataType::UINT32},
175 : {"aiv_send_uint64_t", HcclCMDType::HCCL_CMD_SEND, DataType::UINT64},
176 : {"aiv_send_half", HcclCMDType::HCCL_CMD_SEND, DataType::FP16},
177 : {"aiv_send_float", HcclCMDType::HCCL_CMD_SEND, DataType::FP32},
178 : {"aiv_send_uint64_t", HcclCMDType::HCCL_CMD_SEND, DataType::FP64},
179 : {"aiv_send_bfloat16_t", HcclCMDType::HCCL_CMD_SEND, DataType::BFP16},
180 : {"aiv_send_hifloat8_t", HcclCMDType::HCCL_CMD_SEND, DataType::HIF8},
181 : {"aiv_send_fp8_e4m3fn_t", HcclCMDType::HCCL_CMD_SEND, DataType::FP8E4M3},
182 : {"aiv_send_fp8_e5m2_t", HcclCMDType::HCCL_CMD_SEND, DataType::FP8E5M2},
183 : {"aiv_send_fp8_e8m0_t", HcclCMDType::HCCL_CMD_SEND, DataType::FP8E8M0},
184 : // recv
185 : {"aiv_recv_int8_t", HcclCMDType::HCCL_CMD_RECEIVE, DataType::INT8},
186 : {"aiv_recv_int16_t", HcclCMDType::HCCL_CMD_RECEIVE, DataType::INT16},
187 : {"aiv_recv_int32_t", HcclCMDType::HCCL_CMD_RECEIVE, DataType::INT32},
188 : {"aiv_recv_int64_t", HcclCMDType::HCCL_CMD_RECEIVE, DataType::INT64},
189 : {"aiv_recv_uint8_t", HcclCMDType::HCCL_CMD_RECEIVE, DataType::UINT8},
190 : {"aiv_recv_uint16_t", HcclCMDType::HCCL_CMD_RECEIVE, DataType::UINT16},
191 : {"aiv_recv_uint32_t", HcclCMDType::HCCL_CMD_RECEIVE, DataType::UINT32},
192 : {"aiv_recv_uint64_t", HcclCMDType::HCCL_CMD_RECEIVE, DataType::UINT64},
193 : {"aiv_recv_half", HcclCMDType::HCCL_CMD_RECEIVE, DataType::FP16},
194 : {"aiv_recv_float", HcclCMDType::HCCL_CMD_RECEIVE, DataType::FP32},
195 : {"aiv_recv_uint64_t", HcclCMDType::HCCL_CMD_RECEIVE, DataType::FP64},
196 : {"aiv_recv_bfloat16_t", HcclCMDType::HCCL_CMD_RECEIVE, DataType::BFP16},
197 : {"aiv_recv_hifloat8_t", HcclCMDType::HCCL_CMD_RECEIVE, DataType::HIF8},
198 : {"aiv_recv_fp8_e4m3fn_t", HcclCMDType::HCCL_CMD_RECEIVE, DataType::FP8E4M3},
199 : {"aiv_recv_fp8_e5m2_t", HcclCMDType::HCCL_CMD_RECEIVE, DataType::FP8E5M2},
200 : {"aiv_recv_fp8_e8m0_t", HcclCMDType::HCCL_CMD_RECEIVE, DataType::FP8E8M0},
201 : // batchSendRecv
202 : {"aiv_batchSendRecv_int8_t", HcclCMDType::HCCL_CMD_BATCH_SEND_RECV, DataType::INT8},
203 : {"aiv_batchSendRecv_int16_t", HcclCMDType::HCCL_CMD_BATCH_SEND_RECV, DataType::INT16},
204 : {"aiv_batchSendRecv_int32_t", HcclCMDType::HCCL_CMD_BATCH_SEND_RECV, DataType::INT32},
205 : {"aiv_batchSendRecv_int64_t", HcclCMDType::HCCL_CMD_BATCH_SEND_RECV, DataType::INT64},
206 : {"aiv_batchSendRecv_uint8_t", HcclCMDType::HCCL_CMD_BATCH_SEND_RECV, DataType::UINT8},
207 : {"aiv_batchSendRecv_uint16_t", HcclCMDType::HCCL_CMD_BATCH_SEND_RECV, DataType::UINT16},
208 : {"aiv_batchSendRecv_uint32_t", HcclCMDType::HCCL_CMD_BATCH_SEND_RECV, DataType::UINT32},
209 : {"aiv_batchSendRecv_uint64_t", HcclCMDType::HCCL_CMD_BATCH_SEND_RECV, DataType::UINT64},
210 : {"aiv_batchSendRecv_half", HcclCMDType::HCCL_CMD_BATCH_SEND_RECV, DataType::FP16},
211 : {"aiv_batchSendRecv_float", HcclCMDType::HCCL_CMD_BATCH_SEND_RECV, DataType::FP32},
212 : {"aiv_batchSendRecv_uint64_t", HcclCMDType::HCCL_CMD_BATCH_SEND_RECV, DataType::FP64},
213 : {"aiv_batchSendRecv_bfloat16_t", HcclCMDType::HCCL_CMD_BATCH_SEND_RECV, DataType::BFP16},
214 : {"aiv_batchSendRecv_hifloat8_t", HcclCMDType::HCCL_CMD_BATCH_SEND_RECV, DataType::HIF8},
215 : {"aiv_batchSendRecv_fp8_e4m3fn_t", HcclCMDType::HCCL_CMD_BATCH_SEND_RECV, DataType::FP8E4M3},
216 : {"aiv_batchSendRecv_fp8_e5m2_t", HcclCMDType::HCCL_CMD_BATCH_SEND_RECV, DataType::FP8E5M2},
217 : {"aiv_batchSendRecv_fp8_e8m0_t", HcclCMDType::HCCL_CMD_BATCH_SEND_RECV, DataType::FP8E8M0},
218 : };
219 : using AivExtraKernelArgs = struct AivExtraKernelArgsDef {
220 : const void* buffersIn; // 注册的CCLIN地址，所有卡可访问
221 : u64 input;
222 : u64 output;
223 : u32 rank;
224 : u32 sendRecvRemoteRank;
225 : u32 rankSize;
226 : u64 xRankSize;
227 : u64 yRankSize;
228 : u64 zRankSize;
229 : u64 len;
230 : u32 dataType;
231 : u32 reduceOp;
232 : u32 root;
233 : u32 tag; // 第几次调用，定时重置成1
234 : u64 inputSliceStride;
235 : u64 outputSliceStride;
236 : u64 repeatNum;
237 : u64 inputRepeatStride;
238 : u64 outputRepeatStride;
239 : bool isOpBase;
240 : const void* headCountMem;
241 : const void* tailCountMem;
242 : const void* addOneMem;
243 : u32 counterMemSize;
244 : bool isEnableCounter;
245 : ExtraArgsA2A extraArgs;
246 :
247 1 : AivExtraKernelArgsDef(const void* buffIn, u64 input, u64 output, u32 rank, u32 sendRecvRemoteRank,
248 : u32 rankSize, u64 xRankSize, u64 yRankSize, u64 zRankSize,
249 : u64 len, u32 dataType, u32 reduceOp, u32 root, u32 tag,
250 : u64 inputSliceStride, u64 outputSliceStride, u64 repeatNum, u64 inputRepeatStride, u64 outputRepeatStride,
251 : bool isOpBase = true,
252 : const void* headCountMem = nullptr, const void* tailCountMem = nullptr, const void* addOneMem = nullptr,
253 : u32 counterMemSize = 0, const ExtraArgsA2A* extraArgsPtr = nullptr)
254 1 : : buffersIn(buffIn),input(input), output(output), rank(rank), sendRecvRemoteRank(sendRecvRemoteRank), rankSize(rankSize), xRankSize(xRankSize), yRankSize(yRankSize), zRankSize(zRankSize),
255 1 : len(len) ,dataType(dataType),
256 1 : reduceOp(reduceOp), root(root), tag(tag),
257 1 : inputSliceStride(inputSliceStride), outputSliceStride(outputSliceStride), repeatNum(repeatNum), inputRepeatStride(inputRepeatStride), outputRepeatStride(outputRepeatStride),
258 1 : isOpBase(isOpBase),
259 1 : headCountMem(headCountMem), tailCountMem(tailCountMem), addOneMem(addOneMem),
260 1 : counterMemSize(counterMemSize)
261 : {
262 1 : if (extraArgsPtr != nullptr) {
263 1 : extraArgs = *extraArgsPtr;
264 : }
265 1 : }
266 : };
267 :
268 1 : HcclResult GetAivOpBinaryPath(std::string &binaryPath)
269 : {
270 1 : char *envValue = nullptr;
271 1 : MM_SYS_GET_ENV(MM_ENV_ASCEND_HOME_PATH, envValue);
272 :
273 1 : std::string libPath;
274 1 : if (envValue != nullptr) {
275 1 : libPath = envValue;
276 : } else {
277 0 : libPath = "/usr/local/Ascend/cann";
278 0 : HCCL_WARNING("[AIV][GetAivOpBinaryPath]ENV:ASCEND_HOME_PATH is not set, use default path[%s]",
279 : libPath.c_str());
280 : }
281 :
282 1 : binaryPath = libPath + "/lib64/hccl_aiv_op_910_95.o";
283 :
284 3 : HCCL_INFO("[AIV][GetAivOpBinaryPath]binaryPath: %s", binaryPath.c_str());
285 :
286 1 : return HCCL_SUCCESS;
287 1 : }
288 :
289 157 : s8* GetStubFunc(HcclCMDType cmdType, DataType dataType, KernelArgsType argsType = KernelArgsType::ARGS_TYPE_SERVER)
290 : {
291 : return reinterpret_cast<s8*>(
292 157 : (((static_cast<s64>(cmdType) << SIG_MOVE_LEFT_BITS) + static_cast<s64>(dataType)) << SIG_MOVE_LEFT_BITS) +
293 157 : static_cast<s64>(argsType));
294 : }
295 :
296 156 : HcclResult RegisterBinaryKernel(const char* funcName, const aclrtBinHandle binHandle, const s8* stubFunc)
297 : {
298 156 : if (stubFunc == nullptr) {
299 0 : return HCCL_E_PARA;
300 : }
301 :
302 : aclrtFuncHandle funcHandle;
303 156 : aclError aclRet = aclrtBinaryGetFunction(binHandle, funcName, &funcHandle);
304 156 : CHK_PRT_RET(aclRet != ACL_SUCCESS,
305 : HCCL_ERROR("[RegisterBinaryKernel]errNo[0x%016llx] get function from binary error.", aclRet),
306 : HCCL_E_NOT_FOUND);
307 :
308 156 : g_aivFuncMap[stubFunc] = funcHandle;
309 156 : g_aivNameMap[stubFunc] = funcName;
310 :
311 156 : return HCCL_SUCCESS;
312 : }
313 :
314 42 : HcclResult RegisterKernel()
315 : {
316 42 : lock_guard<mutex> guard(g_mut);
317 42 : if (g_init) {
318 41 : return HCCL_SUCCESS;
319 : }
320 :
321 : HcclResult ret;
322 1 : string binFilePath;
323 1 : ret = GetAivOpBinaryPath(binFilePath);
324 3 : HCCL_INFO("[RegisterKernel] binFilePath: %s", binFilePath.c_str());
325 1 : CHK_PRT_RET(ret != HCCL_SUCCESS, HCCL_ERROR("[AIV][RegisterKernel] get aiv op binary path failed"), HCCL_E_RUNTIME);
326 :
327 1 : LoadBinaryFromFile(binFilePath.c_str(), ACL_RT_BINARY_LOAD_OPT_LAZY_LOAD, 1, g_binHandle);
328 157 : for (auto &aivKernelInfo: g_aivKernelInfoList) {
329 312 : ret = RegisterBinaryKernel(aivKernelInfo.kernelName, g_binHandle,
330 156 : GetStubFunc(aivKernelInfo.cmdType, aivKernelInfo.dataType, aivKernelInfo.argsType));
331 :
332 156 : CHK_PRT_RET(ret != HCCL_SUCCESS, HCCL_ERROR("[AIV][RegisterKernel] register binary kernel for kernelName[%s] "
333 : "cmdType[%d] dataType[%d] argsType[%d] failed", aivKernelInfo.kernelName, aivKernelInfo.cmdType,
334 : aivKernelInfo.dataType, aivKernelInfo.argsType), HCCL_E_RUNTIME);
335 : }
336 :
337 1 : g_init = true;
338 :
339 1 : return HCCL_SUCCESS;
340 42 : }
341 :
342 0 : HcclResult UnRegisterAivKernel()
343 : {
344 0 : lock_guard<mutex> guard(g_mut);
345 0 : if (g_init) {
346 0 : aclError aclRet = aclrtBinaryUnLoad(g_binHandle);
347 0 : CHK_PRT_RET(aclRet != ACL_SUCCESS,
348 : HCCL_ERROR("[UnRegisterAivKernel] aclrtBinaryUnLoad failed, ret[%d]", aclRet), HCCL_E_RUNTIME);
349 0 : g_aivFuncMap.clear();
350 0 : g_init = false;
351 : }
352 0 : return HCCL_SUCCESS;
353 0 : }
354 :
355 1 : HcclResult GetMinAndMaxNpuSchedTimeOut(u64 &minNpuSchedTimeout, u64 &maxNpuSchedTimeout)
356 : {
357 1 : uint64_t interval = 0;
358 1 : aclError aclRet = aclrtGetOpTimeOutInterval(&interval);
359 1 : CHK_PRT_RET(aclRet != ACL_SUCCESS, HCCL_ERROR("aclrtGetOpTimeOutInterval get timeout interval failed, ret[%d]",
360 : aclRet), HCCL_E_RUNTIME);
361 :
362 : // NPU超时范围(1, 254) * interval
363 1 : minNpuSchedTimeout = MIN_NPU_TIMEOUT * interval;
364 1 : maxNpuSchedTimeout = MAX_NPU_TIMEOUT * interval;
365 3 : HCCL_INFO("GetMinAndMaxNpuSchedTimeOut minNpuSchedTimeout[%u]us, maxNpuSchedTimeout[%u]us.",
366 : minNpuSchedTimeout, maxNpuSchedTimeout);
367 1 : return HCCL_SUCCESS;
368 : }
369 :
370 1 : u32 GetAivTimeout() {
371 1 : constexpr u32 TIME_S_TO_US = 1000000;
372 1 : constexpr u32 AIV_TIMEOUT_DEFAULT_US = 1091 * TIME_S_TO_US;
373 1 : constexpr u32 AIV_TIMEOUT_MAX_US = 1091 * TIME_S_TO_US;
374 1 : u32 timeout = AIV_TIMEOUT_DEFAULT_US;
375 1 : double execTimeOut = EnvConfig::GetInstance().GetRtsConfig().GetAivExecTimeOut();
376 :
377 1 : double timeoutUs = execTimeOut * TIME_S_TO_US;
378 1 : if (timeoutUs > static_cast<double>(std::numeric_limits<u32>::max())) {
379 0 : HCCL_INFO("[GetAivTimeout]Get input timeout[%.2f] is out of valid range.", timeoutUs);
380 0 : return AIV_TIMEOUT_MAX_US;
381 : }
382 1 : u32 timeoutUsInt = static_cast<u32>(timeoutUs);
383 1 : if (timeoutUsInt == 0) {
384 0 : timeoutUsInt = AIV_TIMEOUT_MAX_US;
385 : }
386 1 : u64 minNpuSchedTimeout = 0;
387 1 : u64 maxNpuSchedTimeout = 0;
388 1 : CHK_RET(GetMinAndMaxNpuSchedTimeOut(minNpuSchedTimeout, maxNpuSchedTimeout));
389 1 : timeout = (timeoutUsInt < minNpuSchedTimeout) ? minNpuSchedTimeout
390 1 : : (timeoutUsInt > maxNpuSchedTimeout) ? maxNpuSchedTimeout
391 : : timeoutUsInt;
392 3 : HCCL_INFO("[GetAivTimeout]timeout[%u]us, execTimeOut[%.2f]s, minNpuSchedTimeout[%u]us, maxNpuSchedTimeout[%u]us.",
393 : timeout, execTimeOut, minNpuSchedTimeout, maxNpuSchedTimeout);
394 :
395 1 : return timeout;
396 : }
397 :
398 1 : HcclResult GetKernelFunc(aclrtFuncHandle& funcHandle, const s8* stubFunc)
399 : {
400 1 : if (stubFunc == nullptr || g_aivFuncMap.find(stubFunc) == g_aivFuncMap.end()) {
401 3 : HCCL_ERROR("[GetKernelFunc] stubFunc not found in g_aivFuncMap");
402 1 : return HCCL_E_PARA;
403 : }
404 0 : funcHandle = g_aivFuncMap[stubFunc];
405 0 : return HCCL_SUCCESS;
406 : }
407 :
408 : // KernelLaunch内部接口
409 1 : HcclResult ExecuteKernelLaunchInner(const AivOpArgs &opArgs, void* args, u32 argsSize)
410 : {
411 1 : constexpr u32 AIV_ATTRNUM_THREE = 3;
412 3 : HCCL_INFO("[AIV][ExecuteKernelLaunch] sendbuff [%llu] recvbuff [%llu] rank [%u] sendRecvRemoteRank [%u] rankSize [%u] count [%llu] "
413 : "dataType [%d] reduceOp [%d] root [%u] tag [%u] isOpBase [%d] "
414 : "extraArgsPtr [%p] argsSize [%u] numBlocks [%u]", opArgs.input,
415 : opArgs.output, opArgs.rank, opArgs.sendRecvRemoteRank, opArgs.rankSize, opArgs.count,
416 : opArgs.dataType, opArgs.op, opArgs.root,
417 : opArgs.aivTag, opArgs.isOpBase, args, argsSize, opArgs.numBlocks);
418 :
419 : aclrtLaunchKernelCfg cfg;
420 : aclrtLaunchKernelAttr attr[AIV_ATTRNUM_THREE];
421 :
422 1 : u32 timeoutUs = GetAivTimeout();
423 1 : attr[0].id = ACL_RT_LAUNCH_KERNEL_ATTR_SCHEM_MODE;
424 1 : attr[0].value.schemMode = 1;
425 1 : attr[1].id = ACL_RT_LAUNCH_KERNEL_ATTR_TIMEOUT_US;
426 1 : attr[1].value.timeoutUs.timeoutLow = timeoutUs;
427 1 : attr[1].value.timeoutUs.timeoutHigh = 0;
428 1 : attr[2].id = ACL_RT_LAUNCH_KERNEL_ATTR_ENGINE_TYPE;
429 1 : attr[2].value.engineType = ACL_RT_ENGINE_TYPE_AIV;
430 1 : cfg.numAttrs = AIV_ATTRNUM_THREE;
431 1 : cfg.attrs = attr;
432 :
433 3 : HCCL_INFO("[AIV][ExecuteKernelLaunch] KernelAttr attr[0]: id=%u, schemMode=%u; attr[1]: id=%u, timeoutLow=%u, "
434 : "timeoutHigh=%u; attr[2]: id=%u, engineType=%u; cfg: numAttrs=%u",
435 : attr[0].id, attr[0].value.schemMode, attr[1].id, attr[1].value.timeoutUs.timeoutLow,
436 : attr[1].value.timeoutUs.timeoutHigh, attr[2].id, attr[2].value.engineType, cfg.numAttrs);
437 :
438 : aclrtFuncHandle funcHandle;
439 1 : const s8* stubFunc = GetStubFunc(opArgs.cmdType, opArgs.dataType, opArgs.argsType);
440 1 : HcclResult ret = GetKernelFunc(funcHandle, stubFunc);
441 4 : CHK_PRT_RET(ret != HCCL_SUCCESS, HCCL_ERROR("[ExecuteKernelLaunchInner] errNo[0x%016llx] GetKernelFunc failed, "
442 : "return[%d]", HCCL_ERROR_CODE(HCCL_E_RUNTIME), ret), HCCL_E_RUNTIME);
443 :
444 0 : aclError aclRet = aclrtLaunchKernelWithHostArgs(funcHandle, opArgs.numBlocks, opArgs.stream,
445 : &cfg, args, argsSize, nullptr, 0);
446 0 : if (aclRet == ACL_ERROR_RT_INVALID_HANDLE) {
447 0 : aclError aclGetRet = aclrtBinaryGetFunction(g_binHandle, g_aivNameMap[stubFunc].c_str(), &funcHandle);
448 0 : CHK_PRT_RET(aclGetRet != ACL_SUCCESS, HCCL_ERROR("[RegisterBinaryKernel]errNo[0x%016llx] get function from binary error.", aclRet),
449 : HCCL_E_NOT_FOUND);
450 0 : aclRet = aclrtLaunchKernelWithHostArgs(funcHandle, opArgs.numBlocks, opArgs.stream,
451 : &cfg, args, argsSize, nullptr, 0);
452 : }
453 0 : CHK_PRT_RET(aclRet != ACL_SUCCESS, HCCL_ERROR("[ExecuteKernelLaunchInner]errNo[0x%016llx] aclrtLaunchKernelWithHostArgs error[%d].",
454 : HCCL_ERROR_CODE(HCCL_E_RUNTIME), aclRet), HCCL_E_RUNTIME);
455 0 : return HCCL_SUCCESS;
456 : }
457 :
458 : // Kernel单次调用Launch外部接口
459 1 : HcclResult ExecuteKernelLaunch(const AivOpArgs &opArgs)
460 : {
461 : AivExtraKernelArgs aivExtraKernelArgs {
462 1 : opArgs.buffersIn, opArgs.input, opArgs.output,
463 2 : opArgs.rank, opArgs.sendRecvRemoteRank, opArgs.rankSize, opArgs.xRankSize, opArgs.yRankSize, opArgs.zRankSize, opArgs.count, opArgs.dataType, opArgs.op, opArgs.root, opArgs.aivTag,
464 1 : opArgs.inputSliceStride, opArgs.outputSliceStride, opArgs.repeatNum, opArgs.inputRepeatStride, opArgs.outputRepeatStride,
465 1 : opArgs.isOpBase,
466 1 : reinterpret_cast<void*>(opArgs.counter.headCountMem),
467 1 : reinterpret_cast<void*>(opArgs.counter.tailCountMem), reinterpret_cast<void*>(opArgs.counter.addOneMem),
468 1 : opArgs.counter.memSize, &opArgs.extraArgs // 这里是真正发送的地方，device侧单个kernel不能超过32K，先写死1024个结构体
469 1 : };
470 4 : CHK_RET(ExecuteKernelLaunchInner(opArgs, &aivExtraKernelArgs, sizeof(aivExtraKernelArgs)));
471 :
472 0 : return HCCL_SUCCESS;
473 : }
474 :
475 : } // ~~ namespace hccl
|
