Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include <mutex>
12 : #include <vector>
13 : #include <iostream>
14 : #include <fstream>
15 : #include <unordered_map>
16 : #include <cmath>
17 : #include <limits>
18 : #include "mmpa_api.h"
19 : #include "adapter_rts_common.h"
20 : #include "acl/acl_rt.h"
21 : #include "launch_device.h"
22 : #include "hccl_aiv.h"
23 :
24 : using namespace std;
25 :
26 : namespace hccl {
27 : constexpr u32 SIG_MOVE_LEFT_BITS = 20;
28 : constexpr u32 RANK_ZERO = 0;
29 : constexpr u32 RANK_ONE = 1;
30 : constexpr u32 RANK_TWO = 2;
31 : constexpr u32 RANK_THREE = 3;
32 : constexpr u32 RANK_FOUR = 4;
33 : constexpr u32 RANK_FIVE = 5;
34 : constexpr u32 RANK_SIX = 6;
35 : constexpr u32 RANK_SEVEN = 7;
36 : constexpr u32 MAX_ARGS_SIZE_A3_STRUCT = 9;
37 :
38 : constexpr u32 AIV_BUFFER_PING_PONG_FACTOR = 2;
39 :
40 : constexpr u32 MAX_BIN_FILE_SIZE = 100 * 1024 * 1024; // 最大读取100m的bin file到string中
41 :
42 : constexpr s32 RESET_TAIL_SYNC_TAG = 2;
43 : constexpr u32 AIV_FLAG_AREA_SIZE = 1024 * 1024;
44 :
45 : constexpr u32 AIV_ATTRNUM_THREE = 3;
46 :
47 : using AivKernelInfo = struct AivKernelInfoDef {
48 : const char* kernelName;
49 : HcclDataType dataType;
50 : KernelArgsType argsType;
51 :
52 5658 : AivKernelInfoDef(const char* kernelName, HcclDataType dataType,
53 : KernelArgsType argsType = KernelArgsType::ARGS_TYPE_SERVER)
54 5658 : : kernelName(kernelName), dataType(dataType), argsType(argsType)
55 : {
56 5658 : }
57 : };
58 :
59 : static bool g_init = false;
60 : static mutex g_mut;
61 : static std::unordered_map<HcclCMDType, aclrtBinHandle> g_binHandleMap;
62 : static std::unordered_map<s8*, aclrtFuncHandle> g_aivFuncMap;
63 : static std::unordered_map<s8*, std::string> g_aivNameMap;
64 :
65 : static std::vector<AivKernelInfo> g_allreduceAivKernelInfoList = {
66 : {"aiv_all_reduce_float", HcclDataType::HCCL_DATA_TYPE_FP32},
67 : {"aiv_all_reduce_half", HcclDataType::HCCL_DATA_TYPE_FP16},
68 : {"aiv_all_reduce_int16_t", HcclDataType::HCCL_DATA_TYPE_INT16},
69 : {"aiv_all_reduce_int32_t", HcclDataType::HCCL_DATA_TYPE_INT32},
70 : {"aiv_all_reduce_int8_t", HcclDataType::HCCL_DATA_TYPE_INT8},
71 : {"aiv_all_reduce_bfloat16_t", HcclDataType::HCCL_DATA_TYPE_BFP16},
72 : {"aiv_all_reduce_cn_float", HcclDataType::HCCL_DATA_TYPE_FP32, KernelArgsType::ARGS_TYPE_SIMPLE},
73 : {"aiv_all_reduce_cn_half", HcclDataType::HCCL_DATA_TYPE_FP16, KernelArgsType::ARGS_TYPE_SIMPLE},
74 : {"aiv_all_reduce_cn_int16_t", HcclDataType::HCCL_DATA_TYPE_INT16, KernelArgsType::ARGS_TYPE_SIMPLE},
75 : {"aiv_all_reduce_cn_int32_t", HcclDataType::HCCL_DATA_TYPE_INT32, KernelArgsType::ARGS_TYPE_SIMPLE},
76 : {"aiv_all_reduce_cn_int8_t", HcclDataType::HCCL_DATA_TYPE_INT8, KernelArgsType::ARGS_TYPE_SIMPLE},
77 : {"aiv_all_reduce_cn_bfloat16_t", HcclDataType::HCCL_DATA_TYPE_BFP16, KernelArgsType::ARGS_TYPE_SIMPLE},
78 : };
79 : static std::string g_allreduceAivBinaryName = "hccl_aiv_all_reduce_op.o";
80 :
81 : static std::vector<AivKernelInfo> g_reducescatterAivKernelInfoList = {
82 : {"aiv_reduce_scatter_float", HcclDataType::HCCL_DATA_TYPE_FP32},
83 : {"aiv_reduce_scatter_half", HcclDataType::HCCL_DATA_TYPE_FP16},
84 : {"aiv_reduce_scatter_int16_t", HcclDataType::HCCL_DATA_TYPE_INT16},
85 : {"aiv_reduce_scatter_int32_t", HcclDataType::HCCL_DATA_TYPE_INT32},
86 : {"aiv_reduce_scatter_int8_t", HcclDataType::HCCL_DATA_TYPE_INT8},
87 : {"aiv_reduce_scatter_bfloat16_t", HcclDataType::HCCL_DATA_TYPE_BFP16},
88 : {"aiv_reduce_scatter_cn_float", HcclDataType::HCCL_DATA_TYPE_FP32, KernelArgsType::ARGS_TYPE_SIMPLE},
89 : {"aiv_reduce_scatter_cn_half", HcclDataType::HCCL_DATA_TYPE_FP16, KernelArgsType::ARGS_TYPE_SIMPLE},
90 : {"aiv_reduce_scatter_cn_int16_t", HcclDataType::HCCL_DATA_TYPE_INT16, KernelArgsType::ARGS_TYPE_SIMPLE},
91 : {"aiv_reduce_scatter_cn_int32_t", HcclDataType::HCCL_DATA_TYPE_INT32, KernelArgsType::ARGS_TYPE_SIMPLE},
92 : {"aiv_reduce_scatter_cn_int8_t", HcclDataType::HCCL_DATA_TYPE_INT8, KernelArgsType::ARGS_TYPE_SIMPLE},
93 : {"aiv_reduce_scatter_cn_bfloat16_t", HcclDataType::HCCL_DATA_TYPE_BFP16, KernelArgsType::ARGS_TYPE_SIMPLE},
94 : };
95 : static std::string g_reducescatterAivBinaryName = "hccl_aiv_reduce_scatter_op.o";
96 :
97 : static std::vector<AivKernelInfo> g_allgatherAivKernelInfoList = {
98 : {"aiv_all_gather_half", HcclDataType::HCCL_DATA_TYPE_FP16},
99 : {"aiv_all_gather_int16_t", HcclDataType::HCCL_DATA_TYPE_INT16},
100 : {"aiv_all_gather_uint16_t", HcclDataType::HCCL_DATA_TYPE_UINT16},
101 : {"aiv_all_gather_float", HcclDataType::HCCL_DATA_TYPE_FP32},
102 : {"aiv_all_gather_int32_t", HcclDataType::HCCL_DATA_TYPE_INT32},
103 : {"aiv_all_gather_uint32_t", HcclDataType::HCCL_DATA_TYPE_UINT32},
104 : {"aiv_all_gather_int8_t", HcclDataType::HCCL_DATA_TYPE_INT8},
105 : {"aiv_all_gather_uint8_t", HcclDataType::HCCL_DATA_TYPE_UINT8},
106 : {"aiv_all_gather_bfloat16_t", HcclDataType::HCCL_DATA_TYPE_BFP16},
107 : {"aiv_all_gather_int64_t", HcclDataType::HCCL_DATA_TYPE_INT64},
108 : {"aiv_all_gather_uint64_t", HcclDataType::HCCL_DATA_TYPE_UINT64},
109 : {"aiv_all_gather_double", HcclDataType::HCCL_DATA_TYPE_FP64},
110 : {"aiv_all_gather_cn_half", HcclDataType::HCCL_DATA_TYPE_FP16, KernelArgsType::ARGS_TYPE_SIMPLE},
111 : {"aiv_all_gather_cn_int16_t", HcclDataType::HCCL_DATA_TYPE_INT16, KernelArgsType::ARGS_TYPE_SIMPLE},
112 : {"aiv_all_gather_cn_uint16_t", HcclDataType::HCCL_DATA_TYPE_UINT16, KernelArgsType::ARGS_TYPE_SIMPLE},
113 : {"aiv_all_gather_cn_float", HcclDataType::HCCL_DATA_TYPE_FP32, KernelArgsType::ARGS_TYPE_SIMPLE},
114 : {"aiv_all_gather_cn_int32_t", HcclDataType::HCCL_DATA_TYPE_INT32, KernelArgsType::ARGS_TYPE_SIMPLE},
115 : {"aiv_all_gather_cn_uint32_t", HcclDataType::HCCL_DATA_TYPE_UINT32, KernelArgsType::ARGS_TYPE_SIMPLE},
116 : {"aiv_all_gather_cn_int8_t", HcclDataType::HCCL_DATA_TYPE_INT8, KernelArgsType::ARGS_TYPE_SIMPLE},
117 : {"aiv_all_gather_cn_uint8_t", HcclDataType::HCCL_DATA_TYPE_UINT8, KernelArgsType::ARGS_TYPE_SIMPLE},
118 : {"aiv_all_gather_cn_bfloat16_t", HcclDataType::HCCL_DATA_TYPE_BFP16, KernelArgsType::ARGS_TYPE_SIMPLE},
119 : {"aiv_all_gather_cn_int64_t", HcclDataType::HCCL_DATA_TYPE_INT64, KernelArgsType::ARGS_TYPE_SIMPLE},
120 : {"aiv_all_gather_cn_uint64_t", HcclDataType::HCCL_DATA_TYPE_UINT64, KernelArgsType::ARGS_TYPE_SIMPLE},
121 : {"aiv_all_gather_cn_double", HcclDataType::HCCL_DATA_TYPE_FP64, KernelArgsType::ARGS_TYPE_SIMPLE},
122 : };
123 : static std::string g_allgatherAivBinaryName = "hccl_aiv_all_gather_op.o";
124 :
125 : static std::vector<AivKernelInfo> g_broadcastAivKernelInfoList = {
126 : {"aiv_broadcast_half", HcclDataType::HCCL_DATA_TYPE_FP16},
127 : {"aiv_broadcast_int16_t", HcclDataType::HCCL_DATA_TYPE_INT16},
128 : {"aiv_broadcast_uint16_t", HcclDataType::HCCL_DATA_TYPE_UINT16},
129 : {"aiv_broadcast_float", HcclDataType::HCCL_DATA_TYPE_FP32},
130 : {"aiv_broadcast_int32_t", HcclDataType::HCCL_DATA_TYPE_INT32},
131 : {"aiv_broadcast_uint32_t", HcclDataType::HCCL_DATA_TYPE_UINT32},
132 : {"aiv_broadcast_int8_t", HcclDataType::HCCL_DATA_TYPE_INT8},
133 : {"aiv_broadcast_uint8_t", HcclDataType::HCCL_DATA_TYPE_UINT8},
134 : {"aiv_broadcast_bfloat16_t", HcclDataType::HCCL_DATA_TYPE_BFP16},
135 : {"aiv_broadcast_int64_t", HcclDataType::HCCL_DATA_TYPE_INT64},
136 : {"aiv_broadcast_uint64_t", HcclDataType::HCCL_DATA_TYPE_UINT64},
137 : {"aiv_broadcast_double", HcclDataType::HCCL_DATA_TYPE_FP64},
138 : {"aiv_broadcast_cn_half", HcclDataType::HCCL_DATA_TYPE_FP16, KernelArgsType::ARGS_TYPE_SIMPLE},
139 : {"aiv_broadcast_cn_int16_t", HcclDataType::HCCL_DATA_TYPE_INT16, KernelArgsType::ARGS_TYPE_SIMPLE},
140 : {"aiv_broadcast_cn_uint16_t", HcclDataType::HCCL_DATA_TYPE_UINT16, KernelArgsType::ARGS_TYPE_SIMPLE},
141 : {"aiv_broadcast_cn_float", HcclDataType::HCCL_DATA_TYPE_FP32, KernelArgsType::ARGS_TYPE_SIMPLE},
142 : {"aiv_broadcast_cn_int32_t", HcclDataType::HCCL_DATA_TYPE_INT32, KernelArgsType::ARGS_TYPE_SIMPLE},
143 : {"aiv_broadcast_cn_uint32_t", HcclDataType::HCCL_DATA_TYPE_UINT32, KernelArgsType::ARGS_TYPE_SIMPLE},
144 : {"aiv_broadcast_cn_int8_t", HcclDataType::HCCL_DATA_TYPE_INT8, KernelArgsType::ARGS_TYPE_SIMPLE},
145 : {"aiv_broadcast_cn_uint8_t", HcclDataType::HCCL_DATA_TYPE_UINT8, KernelArgsType::ARGS_TYPE_SIMPLE},
146 : {"aiv_broadcast_cn_bfloat16_t", HcclDataType::HCCL_DATA_TYPE_BFP16, KernelArgsType::ARGS_TYPE_SIMPLE},
147 : {"aiv_broadcast_cn_int64_t", HcclDataType::HCCL_DATA_TYPE_INT64, KernelArgsType::ARGS_TYPE_SIMPLE},
148 : {"aiv_broadcast_cn_uint64_t", HcclDataType::HCCL_DATA_TYPE_UINT64, KernelArgsType::ARGS_TYPE_SIMPLE},
149 : {"aiv_broadcast_cn_double", HcclDataType::HCCL_DATA_TYPE_FP64, KernelArgsType::ARGS_TYPE_SIMPLE},
150 : };
151 : static std::string g_broadcastAivBinaryName = "hccl_aiv_broadcast_op.o";
152 :
153 : static std::vector<AivKernelInfo> g_alltoallAivKernelInfoList = {
154 : {"aiv_all_to_all_half", HcclDataType::HCCL_DATA_TYPE_FP16},
155 : {"aiv_all_to_all_int16_t", HcclDataType::HCCL_DATA_TYPE_INT16},
156 : {"aiv_all_to_all_uint16_t", HcclDataType::HCCL_DATA_TYPE_UINT16},
157 : {"aiv_all_to_all_float", HcclDataType::HCCL_DATA_TYPE_FP32},
158 : {"aiv_all_to_all_int32_t", HcclDataType::HCCL_DATA_TYPE_INT32},
159 : {"aiv_all_to_all_uint32_t", HcclDataType::HCCL_DATA_TYPE_UINT32},
160 : {"aiv_all_to_all_int8_t", HcclDataType::HCCL_DATA_TYPE_INT8},
161 : {"aiv_all_to_all_uint8_t", HcclDataType::HCCL_DATA_TYPE_UINT8},
162 : {"aiv_all_to_all_bfloat16_t", HcclDataType::HCCL_DATA_TYPE_BFP16},
163 : {"aiv_all_to_all_int64_t", HcclDataType::HCCL_DATA_TYPE_INT64},
164 : {"aiv_all_to_all_uint64_t", HcclDataType::HCCL_DATA_TYPE_UINT64},
165 : {"aiv_all_to_all_double", HcclDataType::HCCL_DATA_TYPE_FP64},
166 : };
167 : static std::string g_alltoallAivBinaryName = "hccl_aiv_all_to_all_op.o";
168 :
169 : static std::vector<AivKernelInfo> g_alltoallvAivKernelInfoList = {
170 : {"aiv_all_to_all_v_half", HcclDataType::HCCL_DATA_TYPE_FP16},
171 : {"aiv_all_to_all_v_int16_t", HcclDataType::HCCL_DATA_TYPE_INT16},
172 : {"aiv_all_to_all_v_uint16_t", HcclDataType::HCCL_DATA_TYPE_UINT16},
173 : {"aiv_all_to_all_v_float", HcclDataType::HCCL_DATA_TYPE_FP32},
174 : {"aiv_all_to_all_v_int32_t", HcclDataType::HCCL_DATA_TYPE_INT32},
175 : {"aiv_all_to_all_v_uint32_t", HcclDataType::HCCL_DATA_TYPE_UINT32},
176 : {"aiv_all_to_all_v_int8_t", HcclDataType::HCCL_DATA_TYPE_INT8},
177 : {"aiv_all_to_all_v_uint8_t", HcclDataType::HCCL_DATA_TYPE_UINT8},
178 : {"aiv_all_to_all_v_bfloat16_t", HcclDataType::HCCL_DATA_TYPE_BFP16},
179 : {"aiv_all_to_all_v_int64_t", HcclDataType::HCCL_DATA_TYPE_INT64},
180 : {"aiv_all_to_all_v_uint64_t", HcclDataType::HCCL_DATA_TYPE_UINT64},
181 : {"aiv_all_to_all_v_double", HcclDataType::HCCL_DATA_TYPE_FP64},
182 : {"aiv_all_to_all_v_sp_half", HcclDataType::HCCL_DATA_TYPE_FP16, KernelArgsType::ARGS_TYPE_SUPERPOD},
183 : {"aiv_all_to_all_v_sp_int16_t", HcclDataType::HCCL_DATA_TYPE_INT16, KernelArgsType::ARGS_TYPE_SUPERPOD},
184 : {"aiv_all_to_all_v_sp_uint16_t", HcclDataType::HCCL_DATA_TYPE_UINT16, KernelArgsType::ARGS_TYPE_SUPERPOD},
185 : {"aiv_all_to_all_v_sp_float", HcclDataType::HCCL_DATA_TYPE_FP32, KernelArgsType::ARGS_TYPE_SUPERPOD},
186 : {"aiv_all_to_all_v_sp_int32_t", HcclDataType::HCCL_DATA_TYPE_INT32, KernelArgsType::ARGS_TYPE_SUPERPOD},
187 : {"aiv_all_to_all_v_sp_uint32_t", HcclDataType::HCCL_DATA_TYPE_UINT32, KernelArgsType::ARGS_TYPE_SUPERPOD},
188 : {"aiv_all_to_all_v_sp_int8_t", HcclDataType::HCCL_DATA_TYPE_INT8, KernelArgsType::ARGS_TYPE_SUPERPOD},
189 : {"aiv_all_to_all_v_sp_uint8_t", HcclDataType::HCCL_DATA_TYPE_UINT8, KernelArgsType::ARGS_TYPE_SUPERPOD},
190 : {"aiv_all_to_all_v_sp_bfloat16_t", HcclDataType::HCCL_DATA_TYPE_BFP16, KernelArgsType::ARGS_TYPE_SUPERPOD},
191 : {"aiv_all_to_all_v_sp_int64_t", HcclDataType::HCCL_DATA_TYPE_INT64, KernelArgsType::ARGS_TYPE_SUPERPOD},
192 : {"aiv_all_to_all_v_sp_uint64_t", HcclDataType::HCCL_DATA_TYPE_UINT64, KernelArgsType::ARGS_TYPE_SUPERPOD},
193 : {"aiv_all_to_all_v_sp_double", HcclDataType::HCCL_DATA_TYPE_FP64, KernelArgsType::ARGS_TYPE_SUPERPOD},
194 : };
195 : static std::string g_alltoallvAivBinaryName = "hccl_aiv_all_to_all_v_op.o";
196 :
197 : static std::vector<AivKernelInfo> g_alltoallvcAivKernelInfoList = {
198 : {"aiv_all_to_all_vc_half", HcclDataType::HCCL_DATA_TYPE_FP16},
199 : {"aiv_all_to_all_vc_int16_t", HcclDataType::HCCL_DATA_TYPE_INT16},
200 : {"aiv_all_to_all_vc_uint16_t", HcclDataType::HCCL_DATA_TYPE_UINT16},
201 : {"aiv_all_to_all_vc_float", HcclDataType::HCCL_DATA_TYPE_FP32},
202 : {"aiv_all_to_all_vc_int32_t", HcclDataType::HCCL_DATA_TYPE_INT32},
203 : {"aiv_all_to_all_vc_uint32_t", HcclDataType::HCCL_DATA_TYPE_UINT32},
204 : {"aiv_all_to_all_vc_int8_t", HcclDataType::HCCL_DATA_TYPE_INT8},
205 : {"aiv_all_to_all_vc_uint8_t", HcclDataType::HCCL_DATA_TYPE_UINT8},
206 : {"aiv_all_to_all_vc_bfloat16_t", HcclDataType::HCCL_DATA_TYPE_BFP16},
207 : {"aiv_all_to_all_vc_int64_t", HcclDataType::HCCL_DATA_TYPE_INT64},
208 : {"aiv_all_to_all_vc_uint64_t", HcclDataType::HCCL_DATA_TYPE_UINT64},
209 : {"aiv_all_to_all_vc_double", HcclDataType::HCCL_DATA_TYPE_FP64},
210 : };
211 : static std::string g_alltoallvcAivBinaryName = "hccl_aiv_all_to_all_vc_op.o";
212 :
213 : static std::vector<AivKernelInfo> g_reducescattervAivKernelInfoList = {
214 : {"aiv_reduce_scatter_v_float", HcclDataType::HCCL_DATA_TYPE_FP32},
215 : {"aiv_reduce_scatter_v_half", HcclDataType::HCCL_DATA_TYPE_FP16},
216 : {"aiv_reduce_scatter_v_int16_t", HcclDataType::HCCL_DATA_TYPE_INT16},
217 : {"aiv_reduce_scatter_v_int32_t", HcclDataType::HCCL_DATA_TYPE_INT32},
218 : {"aiv_reduce_scatter_v_int8_t", HcclDataType::HCCL_DATA_TYPE_INT8},
219 : {"aiv_reduce_scatter_v_bfloat16_t", HcclDataType::HCCL_DATA_TYPE_BFP16},
220 : };
221 : static std::string g_reducescattervAivBinaryName = "hccl_aiv_reduce_scatter_v_op.o";
222 :
223 : static std::vector<AivKernelInfo> g_allgathervAivKernelInfoList = {
224 : {"aiv_all_gather_v_half", HcclDataType::HCCL_DATA_TYPE_FP16},
225 : {"aiv_all_gather_v_int16_t", HcclDataType::HCCL_DATA_TYPE_INT16},
226 : {"aiv_all_gather_v_uint16_t", HcclDataType::HCCL_DATA_TYPE_UINT16},
227 : {"aiv_all_gather_v_float", HcclDataType::HCCL_DATA_TYPE_FP32},
228 : {"aiv_all_gather_v_int32_t", HcclDataType::HCCL_DATA_TYPE_INT32},
229 : {"aiv_all_gather_v_uint32_t", HcclDataType::HCCL_DATA_TYPE_UINT32},
230 : {"aiv_all_gather_v_int8_t", HcclDataType::HCCL_DATA_TYPE_INT8},
231 : {"aiv_all_gather_v_uint8_t", HcclDataType::HCCL_DATA_TYPE_UINT8},
232 : {"aiv_all_gather_v_bfloat16_t", HcclDataType::HCCL_DATA_TYPE_BFP16},
233 : {"aiv_all_gather_v_int64_t", HcclDataType::HCCL_DATA_TYPE_INT64},
234 : {"aiv_all_gather_v_uint64_t", HcclDataType::HCCL_DATA_TYPE_UINT64},
235 : {"aiv_all_gather_v_double", HcclDataType::HCCL_DATA_TYPE_FP64},
236 : };
237 : static std::string g_allgathervAivBinaryName = "hccl_aiv_all_gather_v_op.o";
238 :
239 : static std::map<HcclCMDType, std::pair<std::string, std::vector<AivKernelInfo>>> g_aivKernelInfoMap = {
240 : {HcclCMDType::HCCL_CMD_ALLREDUCE, {g_allreduceAivBinaryName, g_allreduceAivKernelInfoList}},
241 : {HcclCMDType::HCCL_CMD_REDUCE_SCATTER, {g_reducescatterAivBinaryName, g_reducescatterAivKernelInfoList}},
242 : {HcclCMDType::HCCL_CMD_ALLGATHER, {g_allgatherAivBinaryName, g_allgatherAivKernelInfoList}},
243 : {HcclCMDType::HCCL_CMD_BROADCAST, {g_broadcastAivBinaryName, g_broadcastAivKernelInfoList}},
244 : {HcclCMDType::HCCL_CMD_ALLTOALL, {g_alltoallAivBinaryName, g_alltoallAivKernelInfoList}},
245 : {HcclCMDType::HCCL_CMD_ALLTOALLV, {g_alltoallvAivBinaryName, g_alltoallvAivKernelInfoList}},
246 : {HcclCMDType::HCCL_CMD_ALLTOALLVC, {g_alltoallvcAivBinaryName, g_alltoallvcAivKernelInfoList}},
247 : {HcclCMDType::HCCL_CMD_REDUCE_SCATTER_V, {g_reducescattervAivBinaryName, g_reducescattervAivKernelInfoList}},
248 : {HcclCMDType::HCCL_CMD_ALLGATHER_V, {g_allgathervAivBinaryName, g_allgathervAivKernelInfoList}},
249 : };
250 :
251 : using AivKernelArgs = struct AivKernelArgsDef {
252 : const void* buffersIn[MAX_RANK_SIZE] = {}; // 注册的CCLIN地址，所有卡可访问
253 : const void* buffersOut[MAX_RANK_SIZE] = {}; // 注册的CCLOUT地址，所有卡可访问
254 : const void* input;
255 : const void* output;
256 : u32 rank;
257 : u32 rankSize;
258 : u64 len;
259 : u32 dataType;
260 : u32 reduceOp;
261 : u32 root;
262 : s32 tag; // 第几次调用，定时重置成1
263 : u32 numBlocks;
264 : bool isOpBase;
265 : u64 bufferSize;
266 : s32 aivRdmaStep; // 用于AIV与rdma组合的场景，在不同step中kernel完成多机通信的不同部分
267 : bool useAivRdmaSmall; // 使用aivRdma小数据量kernel，否则使用中数据量kernel
268 : u32 serverNum;
269 : u32 devType;
270 : const void* headCounterAddr;
271 : const void* tailCounterAddr;
272 : const void* addOneAddr;
273 : u32 counterMemSize;
274 : bool isEnableCounter;
275 : u32 deterministic;
276 : u64 rmaInfo;
277 :
278 0 : AivKernelArgsDef(void** buffIn, void** buffOut, const void* input, const void* output, u32 rank,
279 : u32 rankSize, u64 len, u32 dataType, u32 reduceOp, u32 root, s32 tag, u32 numBlocks, bool isOpBase = true,
280 : u64 bufferSize = 200 * 1024 * 1024, s32 aivRdmaStep = -1, bool useAivRdmaSmall = false, u32 serverNum = 1,
281 : u32 devType = 2, const void* headCounterAddr = nullptr, const void* tailCounterAddr = nullptr, const void* addOneAddr = nullptr,
282 : u32 counterMemSize = 0, bool isEnableCounter = false, u32 deterministic = 0, u64 rmaInfo = 0)
283 0 : : input(input), output(output), rank(rank), rankSize(rankSize), len(len), dataType(dataType),
284 0 : reduceOp(reduceOp), root(root), tag(tag), numBlocks(numBlocks), isOpBase(isOpBase), bufferSize(bufferSize), aivRdmaStep(aivRdmaStep),
285 0 : useAivRdmaSmall(useAivRdmaSmall), serverNum(serverNum), devType(devType), headCounterAddr(headCounterAddr),
286 0 : tailCounterAddr(tailCounterAddr), addOneAddr(addOneAddr), counterMemSize(counterMemSize),
287 0 : isEnableCounter(isEnableCounter), deterministic(deterministic), rmaInfo(rmaInfo)
288 : {
289 0 : for (u32 i = 0; i < MAX_RANK_SIZE; i++) {
290 0 : buffersIn[i] = (u8 *) buffIn[i];
291 0 : buffersOut[i] = (u8 *) buffOut[i];
292 : }
293 0 : }
294 : };
295 :
296 : using AivExtraKernelArgs = struct AivExtraKernelArgsDef {
297 : const void* buffersIn[MAX_RANK_SIZE] = {}; // 注册的CCLIN地址，所有卡可访问
298 : const void* buffersOut[MAX_RANK_SIZE] = {}; // 注册的CCLOUT地址，所有卡可访问
299 : const void* input;
300 : const void* output;
301 : u32 rank;
302 : u32 rankSize;
303 : u64 len;
304 : u32 dataType;
305 : u32 reduceOp;
306 : u32 root;
307 : s32 tag; // 第几次调用，定时重置成1
308 : u32 numBlocks;
309 : bool isOpBase;
310 : u64 bufferSize;
311 : s32 aivRdmaStep; // 用于AIV与rdma组合的场景，在不同step中kernel完成多机通信的不同部分
312 : bool useAivRdmaSmall; // 使用aivRdma小数据量kernel，否则使用中数据量kernel
313 : u32 serverNum;
314 : u32 devType;
315 : const void* headCounterAddr;
316 : const void* tailCounterAddr;
317 : const void* addOneAddr;
318 : u32 counterMemSize;
319 : bool isEnableCounter;
320 : u32 deterministic;
321 : u64 rmaInfo;
322 : ExtraArgs extraArgs; // A2/A3单机
323 :
324 5 : AivExtraKernelArgsDef(void** buffIn, void** buffOut, const void* input, const void* output, u32 rank,
325 : u32 rankSize, u64 len, u32 dataType, u32 reduceOp, u32 root, s32 tag, u32 numBlocks, bool isOpBase = true,
326 : u64 bufferSize = 200 * 1024 * 1024, s32 aivRdmaStep = -1, bool useAivRdmaSmall = false, u32 serverNum = 1,
327 : u32 devType = 2, const void* headCounterAddr = nullptr, const void* tailCounterAddr = nullptr, const void* addOneAddr = nullptr,
328 : u32 counterMemSize = 0, bool isEnableCounter = false, u32 deterministic = 0, u64 rmaInfo = 0,
329 : const ExtraArgs* extraArgsPtr = nullptr)
330 5 : : input(input), output(output), rank(rank), rankSize(rankSize), len(len), dataType(dataType),
331 5 : reduceOp(reduceOp), root(root), tag(tag), numBlocks(numBlocks), isOpBase(isOpBase), bufferSize(bufferSize), aivRdmaStep(aivRdmaStep),
332 5 : useAivRdmaSmall(useAivRdmaSmall), serverNum(serverNum), devType(devType), headCounterAddr(headCounterAddr),
333 5 : tailCounterAddr(tailCounterAddr), addOneAddr(addOneAddr), counterMemSize(counterMemSize),
334 5 : isEnableCounter(isEnableCounter), deterministic(deterministic), rmaInfo(rmaInfo)
335 : {
336 85 : for (u32 i = 0; i < MAX_RANK_SIZE; i++) {
337 80 : buffersOut[i] = (u8 *) buffOut[i];
338 80 : buffersIn[i] = (u8 *) buffIn[i];
339 : }
340 5 : if (extraArgsPtr != nullptr) {
341 5 : extraArgs = *extraArgsPtr;
342 : }
343 5 : }
344 : };
345 :
346 : using AivExtraKernelArgsV2 = struct AivExtraKernelArgsV2Def {
347 : const void* buffersIn[MAX_RANK_SIZE] = {}; // 注册的CCLIN地址，所有卡可访问
348 : const void* buffersOut[MAX_RANK_SIZE] = {}; // 注册的CCLOUT地址，所有卡可访问
349 : const void* input;
350 : const void* output;
351 : u32 rank;
352 : u32 rankSize;
353 : u64 len;
354 : u32 dataType;
355 : u32 reduceOp;
356 : u32 root;
357 : s32 tag; // 第几次调用，定时重置成1
358 : u32 numBlocks;
359 : bool isOpBase;
360 : u64 bufferSize;
361 : s32 aivRdmaStep; // 用于AIV与rdma组合的场景，在不同step中kernel完成多机通信的不同部分
362 : bool useAivRdmaSmall; // 使用aivRdma小数据量kernel，否则使用中数据量kernel
363 : u32 serverNum;
364 : u32 devType;
365 : const void* headCounterAddr;
366 : const void* tailCounterAddr;
367 : const void* addOneAddr;
368 : u32 counterMemSize;
369 : bool isEnableCounter;
370 : u32 deterministic;
371 : u64 rmaInfo;
372 : ExtraArgsV2 extraArgs; // A3超节点内多机
373 :
374 0 : AivExtraKernelArgsV2Def(void** buffIn, void** buffOut, const void* input, const void* output, u32 rank,
375 : u32 rankSize, u64 len, u32 dataType, u32 reduceOp, u32 root, s32 tag, u32 numBlocks, bool isOpBase = true,
376 : u64 bufferSize = 200 * 1024 * 1024, s32 aivRdmaStep = -1, bool useAivRdmaSmall = false, u32 serverNum = 1,
377 : u32 devType = 2, const void* headCounterAddr = nullptr, const void* tailCounterAddr = nullptr, const void* addOneAddr = nullptr,
378 : u32 counterMemSize = 0, bool isEnableCounter = false, u32 deterministic = 0, u64 rmaInfo = 0,
379 : const ExtraArgsV2* extraArgsPtr = nullptr)
380 0 : : input(input), output(output), rank(rank), rankSize(rankSize), len(len), dataType(dataType),
381 0 : reduceOp(reduceOp), root(root), tag(tag), numBlocks(numBlocks), isOpBase(isOpBase), bufferSize(bufferSize), aivRdmaStep(aivRdmaStep),
382 0 : useAivRdmaSmall(useAivRdmaSmall), serverNum(serverNum), devType(devType), headCounterAddr(headCounterAddr),
383 0 : tailCounterAddr(tailCounterAddr), addOneAddr(addOneAddr), counterMemSize(counterMemSize), isEnableCounter(isEnableCounter),
384 0 : deterministic(deterministic), rmaInfo(rmaInfo)
385 : {
386 0 : for (u32 i = 0; i < MAX_RANK_SIZE; i++) {
387 0 : buffersIn[i] = (u8 *) buffIn[i];
388 0 : buffersOut[i] = (u8 *) buffOut[i];
389 : }
390 0 : if (extraArgsPtr != nullptr) {
391 0 : extraArgs = *extraArgsPtr;
392 : }
393 0 : }
394 : };
395 :
396 : using AivKernelArgsV3 = struct AivKernelArgsV3Def {
397 : u64 massArgs[MAX_ARGS_SIZE_A3_STRUCT] = {};
398 : const void* input;
399 : const void* output;
400 : u32 rank;
401 : u32 rankSize;
402 : u64 len;
403 : u32 dataType;
404 : u32 reduceOp;
405 : u32 root;
406 : s32 tag;
407 : u32 numBlocks;
408 : bool isOpBase;
409 : s32 step;
410 : u32 deterministic;
411 :
412 0 : AivKernelArgsV3Def(void** buffIn, void** buffOut, const void* input, const void* output, u32 rank,
413 : u32 rankSize, u64 len, u32 dataType, u32 reduceOp, u32 root, s32 tag, u32 numBlocks, bool isOpBase = true,
414 : u64 bufferSize = 200 * 1024 * 1024, s32 step = 0, void* headCounterAddr = nullptr,
415 : void* tailCounterAddr = nullptr, void* addOneAddr = nullptr,
416 : bool isEnableCounter = false, u32 deterministic = 0)
417 0 : : input(input), output(output), rank(rank), rankSize(rankSize), len(len), dataType(dataType),
418 0 : reduceOp(reduceOp), root(root), tag(tag), numBlocks(numBlocks), isOpBase(isOpBase),
419 0 : step(step), deterministic(deterministic)
420 : {
421 0 : massArgs[0] = reinterpret_cast<u64>(buffIn[0]);
422 0 : massArgs[1] = reinterpret_cast<u64>(buffIn[1]);
423 0 : massArgs[2] = reinterpret_cast<u64>(buffOut[0]);
424 0 : massArgs[3] = reinterpret_cast<u64>(buffOut[1]);
425 0 : massArgs[4] = reinterpret_cast<u64>(bufferSize);
426 0 : massArgs[5] = reinterpret_cast<u64>(headCounterAddr);
427 0 : massArgs[6] = reinterpret_cast<u64>(tailCounterAddr);
428 0 : massArgs[7] = reinterpret_cast<u64>(addOneAddr);
429 0 : massArgs[8] = isEnableCounter ? 1 : 0;
430 0 : }
431 : };
432 :
433 45 : HcclResult GetAivOpBinaryPath(const std::string &aivBinaryName, std::string &binaryPath)
434 : {
435 : // 获取二进制文件路径
436 45 : std::string libPath;
437 45 : char *getPath = nullptr;
438 45 : MM_SYS_GET_ENV(MM_ENV_ASCEND_HOME_PATH, getPath);
439 45 : if (getPath != nullptr) {
440 45 : libPath = getPath;
441 : } else {
442 0 : libPath = "/usr/local/Ascend/cann";
443 0 : HCCL_WARNING("[GetAivOpBinaryPath]ENV:ASCEND_HOME_PATH is not set");
444 : }
445 45 : binaryPath = libPath + "/lib64";
446 :
447 : // 拼接应该加载的文件
448 45 : binaryPath += "/" + aivBinaryName;
449 45 : HCCL_RUN_INFO("[GetAivOpBinaryPath]op binary file path[%s]", binaryPath.c_str());
450 45 : return HCCL_SUCCESS;
451 45 : }
452 :
453 0 : HcclResult ReadBinFile(const string& fileName, string& buffer)
454 : {
455 0 : char realFile[PATH_MAX] = { 0 };
456 0 : if (realpath(fileName.c_str(), realFile) == nullptr) {
457 0 : HCCL_INFO("[AIV][ReadBinFile] Binfile path %s is not a valid real path.", realFile);
458 0 : return HCCL_E_NOT_FOUND;
459 : }
460 0 : std::ifstream filestr;
461 0 : filestr.open(realFile, std::ios::binary);
462 0 : if (!filestr) {
463 0 : HCCL_ERROR("[AIV][ReadBinFile]open file [%s] failed!", fileName.c_str());
464 0 : return HCCL_E_OPEN_FILE_FAILURE;
465 : }
466 :
467 0 : filestr.seekg(0, std::ios::end);
468 0 : std::streampos fileSize = filestr.tellg();
469 0 : filestr.seekg(0, std::ios::beg);
470 :
471 0 : if (fileSize == 0 || fileSize >= MAX_BIN_FILE_SIZE) {
472 0 : HCCL_ERROR("[AIV][ReadBinFile] file [%s] size is invalid, is [%d]!", fileName.c_str(), fileSize);
473 0 : filestr.close();
474 0 : return HCCL_E_OPEN_FILE_FAILURE;
475 : }
476 0 : buffer.resize(fileSize);
477 0 : filestr.read(&buffer[0], fileSize);
478 :
479 0 : filestr.close();
480 0 : return HCCL_SUCCESS;
481 0 : }
482 :
483 695 : s8* GetStubFunc(HcclCMDType cmdType, HcclDataType dataType, KernelArgsType argsType = KernelArgsType::ARGS_TYPE_SERVER)
484 : {
485 : return reinterpret_cast<s8*>(
486 695 : (((static_cast<s64>(cmdType) << SIG_MOVE_LEFT_BITS) + static_cast<s64>(dataType)) << SIG_MOVE_LEFT_BITS) +
487 695 : static_cast<s64>(argsType));
488 : }
489 :
490 690 : HcclResult RegisterBinaryKernel(const char* funcName, const aclrtBinHandle binHandle, s8* stubFunc)
491 : {
492 690 : if (stubFunc == nullptr) {
493 0 : return HCCL_E_PARA;
494 : }
495 :
496 : aclrtFuncHandle funcHandle;
497 690 : aclError aclRet = aclrtBinaryGetFunction(binHandle, funcName, &funcHandle);
498 690 : CHK_PRT_RET(aclRet != ACL_SUCCESS, HCCL_ERROR("[RegisterBinaryKernel]errNo[0x%016llx] get function from binary error.", aclRet),
499 : HCCL_E_NOT_FOUND);
500 :
501 690 : g_aivFuncMap[stubFunc] = funcHandle;
502 690 : g_aivNameMap[stubFunc] = funcName;
503 :
504 690 : return HCCL_SUCCESS;
505 : }
506 :
507 5 : HcclResult GetKernelFunc(aclrtFuncHandle& funcHandle, s8* stubFunc)
508 : {
509 5 : if (stubFunc == nullptr || g_aivFuncMap.find(stubFunc) == g_aivFuncMap.end()) {
510 0 : return HCCL_E_PARA;
511 : }
512 5 : funcHandle = g_aivFuncMap[stubFunc];
513 5 : return HCCL_SUCCESS;
514 : }
515 :
516 : // Kernel注册入口，全局只需要初始化一次
517 68 : HcclResult RegisterKernel(DevType deviceType)
518 : {
519 68 : lock_guard<mutex> guard(g_mut);
520 68 : if (g_init) {
521 63 : return HCCL_SUCCESS;
522 : }
523 : HcclResult ret;
524 :
525 50 : for (auto& aivOpKernelInfo : g_aivKernelInfoMap) {
526 45 : HcclCMDType cmdType = aivOpKernelInfo.first;
527 45 : const std::string &aivBinaryName = aivOpKernelInfo.second.first;
528 45 : std::vector<AivKernelInfo> &aivKernelInfoList = aivOpKernelInfo.second.second;
529 :
530 45 : string binFilePath;
531 45 : ret = GetAivOpBinaryPath(aivBinaryName, binFilePath);
532 45 : CHK_PRT_RET(ret != HCCL_SUCCESS, HCCL_ERROR("[AIV][RegisterKernel] get aiv op binary path failed"), HCCL_E_RUNTIME);
533 :
534 45 : ret = LoadBinaryFromFile(binFilePath.c_str(), ACL_RT_BINARY_LOAD_OPT_LAZY_LOAD, 1, g_binHandleMap[cmdType]);
535 45 : CHK_PRT_RET(ret != HCCL_SUCCESS, HCCL_ERROR("[AIV][RegisterKernel] read aiv kernel bin file failed"),
536 : HCCL_E_RUNTIME);
537 :
538 735 : for (auto &aivKernelInfo: aivKernelInfoList) {
539 690 : ret = RegisterBinaryKernel(aivKernelInfo.kernelName, g_binHandleMap[cmdType],
540 : GetStubFunc(cmdType, aivKernelInfo.dataType, aivKernelInfo.argsType));
541 690 : CHK_PRT_RET(ret != HCCL_SUCCESS, HCCL_ERROR("[AIV][RegisterKernel] register binary kernel for kernelName[%s] "
542 : "cmdType[%d] dataType[%s] argsType[%d] failed", aivKernelInfo.kernelName, cmdType,
543 : GetDataTypeEnumStr(aivKernelInfo.dataType).c_str(), aivKernelInfo.argsType), HCCL_E_RUNTIME);
544 : }
545 45 : }
546 :
547 5 : g_init = true;
548 :
549 5 : return HCCL_SUCCESS;
550 68 : }
551 :
552 0 : HcclResult UnRegisterAivKernel()
553 : {
554 0 : lock_guard<mutex> guard(g_mut);
555 0 : if (g_init) {
556 0 : for (auto& binHandleItem : g_binHandleMap) {
557 0 : aclrtBinaryUnLoad(binHandleItem.second);
558 : }
559 0 : g_aivFuncMap.clear();
560 :
561 0 : g_init = false;
562 : }
563 :
564 0 : return HCCL_SUCCESS;
565 0 : }
566 :
567 5 : HcclResult GetMinAndMaxNpuSchedTimeOut(u64 &minNpuSchedTimeout, u64 &maxNpuSchedTimeout)
568 : {
569 5 : uint64_t interval = 0;
570 5 : aclError aclRet = aclrtGetOpTimeOutInterval(&interval);
571 5 : CHK_PRT_RET(aclRet != ACL_SUCCESS, HCCL_ERROR("aclrtGetOpTimeOutInterval get timeout interval failed, ret[%d]",
572 : aclRet), HCCL_E_RUNTIME);
573 :
574 5 : constexpr u64 MAX_INTERVAL = 254;
575 : // NPU超时范围(1, 254) * interval
576 5 : minNpuSchedTimeout = 1 * interval;
577 5 : maxNpuSchedTimeout = MAX_INTERVAL * interval;
578 5 : HCCL_INFO("GetMinAndMaxNpuSchedTimeOut minNpuSchedTimeout[%llu]us, maxNpuSchedTimeout[%llu]us.",
579 : minNpuSchedTimeout, maxNpuSchedTimeout);
580 5 : return HCCL_SUCCESS;
581 : }
582 :
583 5 : u32 GetAivTimeout(s32 execTimeOut, bool isSetByConfig) {
584 5 : u32 timeout = AIV_TIMEOUT_DEFAULT_US;
585 5 : if (GetExternalInputHcclExecTimeoutSet() != HcclExecTimeoutSet::HCCL_EXEC_TIMEOUT_NOT_SET || isSetByConfig) {
586 : // 配0时，使用最大超时时间
587 5 : if (execTimeOut == 0) {
588 0 : return AIV_TIMEOUT_MAX_US;
589 : }
590 5 : double timeoutUs = execTimeOut * TIME_S_TO_US;
591 5 : if (timeoutUs > static_cast<double>(std::numeric_limits<u32>::max())) {
592 0 : HCCL_INFO("[GetAivTimeout]Get input timeout[%.2f] is out of valid range.", timeoutUs);
593 0 : return AIV_TIMEOUT_MAX_US;
594 : }
595 5 : u32 timeoutUsInt = static_cast<u32>(timeoutUs);
596 5 : u64 minNpuSchedTimeout = 0;
597 5 : u64 maxNpuSchedTimeout = 0;
598 5 : CHK_RET(GetMinAndMaxNpuSchedTimeOut(minNpuSchedTimeout, maxNpuSchedTimeout));
599 5 : timeout = (timeoutUsInt < minNpuSchedTimeout) ? minNpuSchedTimeout
600 5 : : (timeoutUsInt > maxNpuSchedTimeout) ? maxNpuSchedTimeout
601 : : timeoutUsInt;
602 5 : HCCL_INFO("[GetAivTimeout]timeout[%u]us, minNpuSchedTimeout[%llu]us, maxNpuSchedTimeout[%llu]us.",
603 : timeout, minNpuSchedTimeout, maxNpuSchedTimeout);
604 : }
605 :
606 5 : return timeout < AIV_TIMEOUT_MAX_US ? timeout : AIV_TIMEOUT_MAX_US;
607 : }
608 :
609 5 : void TaskAivProfilerWrap(const AivOpArgs& opArgs, const AivTopoArgs& topoArgs,
610 : const AivResourceArgs& resourceArgs, const AivAlgArgs& algArgs, const AivProfilingInfo& aivProfilingInfo,
611 : void* flagMem)
612 : {
613 5 : struct TaskParaGeneral taskParaGeneral;
614 :
615 5 : TaskParaAiv taskParaAiv(opArgs.cmdType, resourceArgs.aivTag, opArgs.count*SIZE_TABLE[opArgs.dataType],
616 5 : resourceArgs.numBlocks, topoArgs.rankSize, algArgs.step, flagMem, topoArgs.rank, opArgs.isOpBase);
617 :
618 5 : if(taskParaAiv.flagMem == nullptr){
619 0 : taskParaAiv.flagMem = resourceArgs.buffersOut[topoArgs.rank];
620 : }
621 :
622 5 : taskParaGeneral.isMainStream = true;
623 5 : taskParaGeneral.stream = resourceArgs.stream;
624 5 : taskParaGeneral.beginTime = aivProfilingInfo.beginTime;
625 5 : taskParaGeneral.aiv = taskParaAiv;
626 :
627 5 : AlgWrap::GetInstance().TaskAivProfiler(topoArgs.identify, taskParaGeneral);
628 5 : }
629 :
630 0 : HcclResult CacheTaskOpInfo(rtStream_t stream, const std::string &identify)
631 : {
632 0 : aclmdlRI rtModel = nullptr;
633 0 : aclmdlRICaptureStatus captureStatus = aclmdlRICaptureStatus::ACL_MODEL_RI_CAPTURE_STATUS_NONE;
634 0 : aclError aclRet = aclmdlRICaptureGetInfo(stream, &captureStatus, &rtModel);
635 0 : if (aclRet == ACL_ERROR_RT_FEATURE_NOT_SUPPORT) {
636 0 : HCCL_WARNING("[%s]Stream capture does not support!", __func__);
637 0 : return HCCL_SUCCESS;
638 : }
639 0 : CHK_PRT_RET(aclRet != ACL_SUCCESS,
640 : HCCL_ERROR("[%s]rtGet stream get capture status fail. return[%d]", __func__, aclRet), HCCL_E_RUNTIME);
641 :
642 : aclrtStreamAttrValue value;
643 0 : aclRet = aclrtGetStreamAttribute(stream, ACL_STREAM_ATTR_CACHE_OP_INFO, &value);
644 0 : CHK_PRT_RET(aclRet != ACL_SUCCESS, HCCL_ERROR("[%s]stream get attribute fail. return[%d]", __func__, aclRet),
645 : HCCL_E_RUNTIME);
646 :
647 0 : HCCL_INFO("[CacheTaskOpInfo] cacheOpInfoSwitch[%u] captureStatus[%d] identify[%s]", value.cacheOpInfoSwitch, captureStatus, identify.c_str());
648 0 : if (value.cacheOpInfoSwitch == 1 && captureStatus == ACL_MODEL_RI_CAPTURE_STATUS_ACTIVE) {
649 0 : HcclResult cacheRet = hrtCacheLastTaskExtendInfo(identify.c_str(), strlen(identify.c_str()));
650 0 : if (cacheRet == HCCL_E_NOT_SUPPORT) {
651 0 : HCCL_INFO("[%s] aclrtCacheLastTaskExtendInfo not supported", __func__);
652 : } else {
653 0 : CHK_PRT_RET(cacheRet != HCCL_SUCCESS,
654 : HCCL_ERROR("[%s] stream cache task op info fail. return[%d]", __func__, cacheRet),
655 : HCCL_E_RUNTIME);
656 : }
657 : }
658 0 : return HCCL_SUCCESS;
659 : }
660 :
661 : // KernelLaunch内部接口
662 5 : HcclResult ExecuteKernelLaunchInner(const AivOpArgs &opArgs, const AivTopoArgs &topoArgs,
663 : const AivResourceArgs &resourceArgs, const AivAlgArgs &algArgs, void* args, u32 argsSize,
664 : AivProfilingInfo& aivProfilingInfo)
665 : {
666 5 : HCCL_INFO("[AIV][ExecuteKernelLaunchInner] sendbuff [%p] recvbuff [%p] rank [%d] rankSize [%d] count [%llu] "
667 : "dataType [%s] reduceOp [%s] root [%d] tag [%d] isOpBase [%d] bufferSize [%llu] step [%d] "
668 : "isSmallCount [%d] serverNum [%d] devType[%d] extraArgsPtr [%p] argsSize [%d], deterministic [%d].", opArgs.input,
669 : opArgs.output, topoArgs.rank, topoArgs.rankSize, opArgs.count,
670 : GetDataTypeEnumStr(opArgs.dataType).c_str(), GetReduceOpEnumStr(opArgs.op).c_str(), opArgs.root,
671 : resourceArgs.aivTag, opArgs.isOpBase, resourceArgs.bufferSize, algArgs.step, algArgs.isSmallCount,
672 : topoArgs.serverNum, topoArgs.devType, args, argsSize, algArgs.deterministic);
673 :
674 5 : HCCL_DEBUG("[AIV][ExecuteKernelLaunchInner] buffersIn [%p] [%p] [%p] [%p] [%p] [%p] [%p] [%p] "\
675 : "buffersOut [%p] [%p] [%p] [%p] [%p] [%p] [%p] [%p].", resourceArgs.buffersIn[RANK_ZERO],
676 : resourceArgs.buffersIn[RANK_ONE], resourceArgs.buffersIn[RANK_TWO], resourceArgs.buffersIn[RANK_THREE],
677 : resourceArgs.buffersIn[RANK_FOUR], resourceArgs.buffersIn[RANK_FIVE], resourceArgs.buffersIn[RANK_SIX],
678 : resourceArgs.buffersIn[RANK_SEVEN], resourceArgs.buffersOut[RANK_ZERO], resourceArgs.buffersOut[RANK_ONE],
679 : resourceArgs.buffersOut[RANK_TWO], resourceArgs.buffersOut[RANK_THREE], resourceArgs.buffersOut[RANK_FOUR],
680 : resourceArgs.buffersOut[RANK_FIVE], resourceArgs.buffersOut[RANK_SIX], resourceArgs.buffersOut[RANK_SEVEN]);
681 :
682 5 : KernelArgsType argsType = algArgs.argsType;
683 :
684 5 : HcclResult ret = HcclResult::HCCL_E_PARA;
685 : aclrtLaunchKernelCfg cfg;
686 : aclrtLaunchKernelAttr attr[AIV_ATTRNUM_THREE];
687 5 : attr[0].id = ACL_RT_LAUNCH_KERNEL_ATTR_SCHEM_MODE;
688 5 : attr[0].value.schemMode = 1;
689 5 : attr[1].id = ACL_RT_LAUNCH_KERNEL_ATTR_TIMEOUT_US;
690 5 : attr[1].value.timeoutUs.timeoutLow = GetAivTimeout(algArgs.execTimeOut, algArgs.execTimeOutSet);
691 5 : attr[1].value.timeoutUs.timeoutHigh = 0;
692 5 : attr[2].id = ACL_RT_LAUNCH_KERNEL_ATTR_ENGINE_TYPE;
693 5 : attr[2].value.engineType = ACL_RT_ENGINE_TYPE_AIV;
694 5 : cfg.numAttrs = AIV_ATTRNUM_THREE;
695 5 : cfg.attrs = attr;
696 :
697 : aclrtFuncHandle funcHandle;
698 5 : s8* stubFunc = GetStubFunc(opArgs.cmdType, opArgs.dataType, argsType);
699 5 : ret = GetKernelFunc(funcHandle, stubFunc);
700 5 : CHK_PRT_RET(ret != HCCL_SUCCESS, HCCL_ERROR("[ExecuteKernelLaunchInner] errNo[0x%016llx] GetKernelFunc failed, "
701 : "return[%d]", HCCL_ERROR_CODE(HCCL_E_RUNTIME), ret), HCCL_E_RUNTIME);
702 :
703 5 : aclError aclRet = aclrtLaunchKernelWithHostArgs(funcHandle, resourceArgs.numBlocks, resourceArgs.stream,
704 : &cfg, args, argsSize, nullptr, 0);
705 5 : if (aclRet == ACL_ERROR_RT_INVALID_HANDLE) {
706 0 : aclError aclGetRet = aclrtBinaryGetFunction(g_binHandleMap[opArgs.cmdType], g_aivNameMap[stubFunc].c_str(), &funcHandle);
707 0 : CHK_PRT_RET(aclGetRet != ACL_SUCCESS, HCCL_ERROR("[RegisterBinaryKernel]errNo[0x%016llx] get function from binary error.", aclRet),
708 : HCCL_E_NOT_FOUND);
709 0 : aclRet = aclrtLaunchKernelWithHostArgs(funcHandle, resourceArgs.numBlocks, resourceArgs.stream,
710 : &cfg, args, argsSize, nullptr, 0);
711 : }
712 5 : CHK_PRT_RET(aclRet != ACL_SUCCESS, HCCL_ERROR("[ExecuteKernelLaunchInner]errNo[0x%016llx] aclrtLaunchKernelWithHostArgs error[%d].",
713 : HCCL_ERROR_CODE(HCCL_E_RUNTIME), aclRet), HCCL_E_RUNTIME);
714 :
715 5 : CHK_RET(CacheTaskOpInfo(resourceArgs.stream, topoArgs.identify));
716 :
717 5 : TaskAivProfilerWrap(opArgs, topoArgs, resourceArgs, algArgs, aivProfilingInfo,
718 5 : (algArgs.argsType != KernelArgsType::ARGS_TYPE_SERVER) ? resourceArgs.buffersOut[0]: resourceArgs.buffersOut[topoArgs.rank]);
719 :
720 5 : CHK_PRT_RET(ret != HCCL_SUCCESS, HCCL_ERROR("[AIV][ExecuteKernelLaunchInner] errNo[0x%016llx] rtKernelLaunch aiv fail, "
721 : "return[%d]", HCCL_ERROR_CODE(HCCL_E_RUNTIME), ret), HCCL_E_RUNTIME);
722 5 : return HCCL_SUCCESS;
723 : }
724 :
725 : // Kernel单次调用Launch外部接口
726 0 : HcclResult ExecuteKernelLaunch(const AivOpArgs &opArgs, const AivTopoArgs &topoArgs,
727 : const AivResourceArgs &resourceArgs, const AivAlgArgs &algArgs, AivProfilingInfo& aivProfilingInfo)
728 : {
729 0 : SetAivProfilingInfoBeginTime(aivProfilingInfo);
730 0 : CHK_PTR_NULL(resourceArgs.buffersIn);
731 0 : CHK_PTR_NULL(resourceArgs.buffersOut);
732 :
733 0 : if (algArgs.argsType == KernelArgsType::ARGS_TYPE_SIMPLE) {
734 : AivKernelArgsV3 aivKernelArgs {
735 0 : resourceArgs.buffersIn, resourceArgs.buffersOut, opArgs.input, opArgs.output,
736 0 : topoArgs.rank, topoArgs.rankSize, opArgs.count, opArgs.dataType, opArgs.op, opArgs.root, resourceArgs.aivTag, resourceArgs.numBlocks,
737 0 : opArgs.isOpBase, resourceArgs.bufferSize, algArgs.step, reinterpret_cast<void*>(aivProfilingInfo.counter.headCountMem),
738 0 : reinterpret_cast<void*>(aivProfilingInfo.counter.tailCountMem), reinterpret_cast<void*>(aivProfilingInfo.counter.addOneMem),
739 0 : aivProfilingInfo.counter.isEnableCounter, algArgs.deterministic
740 0 : };
741 0 : CHK_RET(ExecuteKernelLaunchInner(opArgs, topoArgs, resourceArgs, algArgs, &aivKernelArgs, sizeof(aivKernelArgs), aivProfilingInfo));
742 : } else {
743 : AivKernelArgs aivKernelArgs {
744 0 : resourceArgs.buffersIn, resourceArgs.buffersOut, opArgs.input, opArgs.output,
745 0 : topoArgs.rank, topoArgs.rankSize, opArgs.count, opArgs.dataType, opArgs.op, opArgs.root, resourceArgs.aivTag, resourceArgs.numBlocks,
746 0 : opArgs.isOpBase, resourceArgs.bufferSize, algArgs.step, algArgs.isSmallCount, topoArgs.serverNum,
747 0 : static_cast<u32>(topoArgs.devType), reinterpret_cast<void*>(aivProfilingInfo.counter.headCountMem),
748 0 : reinterpret_cast<void*>(aivProfilingInfo.counter.tailCountMem), reinterpret_cast<void*>(aivProfilingInfo.counter.addOneMem),
749 0 : aivProfilingInfo.counter.memSize, aivProfilingInfo.counter.isEnableCounter, algArgs.deterministic, algArgs.rmaInfo
750 0 : };
751 0 : CHK_RET(ExecuteKernelLaunchInner(opArgs, topoArgs, resourceArgs, algArgs, &aivKernelArgs, sizeof(aivKernelArgs), aivProfilingInfo));
752 : }
753 :
754 0 : return HCCL_SUCCESS;
755 : }
756 :
757 : // Kernel单次调用Launch外部接口
758 5 : HcclResult ExecuteKernelLaunch(const AivOpArgs &opArgs, const AivTopoArgs &topoArgs,
759 : const AivResourceArgs &resourceArgs, const AivAlgArgs &algArgs, const ExtraArgs &extraArgs,
760 : AivProfilingInfo& aivProfilingInfo)
761 : {
762 5 : SetAivProfilingInfoBeginTime(aivProfilingInfo);
763 5 : CHK_PTR_NULL(resourceArgs.buffersIn);
764 5 : CHK_PTR_NULL(resourceArgs.buffersOut);
765 :
766 : AivExtraKernelArgs aivExtraKernelArgs {
767 5 : resourceArgs.buffersIn, resourceArgs.buffersOut, opArgs.input, opArgs.output,
768 5 : topoArgs.rank, topoArgs.rankSize, opArgs.count, opArgs.dataType, opArgs.op, opArgs.root, resourceArgs.aivTag,
769 5 : resourceArgs.numBlocks, opArgs.isOpBase, resourceArgs.bufferSize, algArgs.step, algArgs.isSmallCount, topoArgs.serverNum,
770 5 : static_cast<u32>(topoArgs.devType), reinterpret_cast<void*>(aivProfilingInfo.counter.headCountMem),
771 5 : reinterpret_cast<void*>(aivProfilingInfo.counter.tailCountMem), reinterpret_cast<void*>(aivProfilingInfo.counter.addOneMem),
772 5 : aivProfilingInfo.counter.memSize, aivProfilingInfo.counter.isEnableCounter, algArgs.deterministic, algArgs.rmaInfo, &extraArgs
773 5 : };
774 5 : CHK_RET(ExecuteKernelLaunchInner(opArgs, topoArgs, resourceArgs, algArgs,
775 : &aivExtraKernelArgs, sizeof(aivExtraKernelArgs), aivProfilingInfo));
776 :
777 5 : return HCCL_SUCCESS;
778 : }
779 :
780 : // Kernel单次调用Launch外部接口
781 0 : HcclResult ExecuteKernelLaunch(const AivOpArgs &opArgs, const AivTopoArgs &topoArgs,
782 : const AivResourceArgs &resourceArgs, const AivAlgArgs &algArgs, const ExtraArgsV2 &extraArgs,
783 : AivProfilingInfo& aivProfilingInfo)
784 : {
785 0 : SetAivProfilingInfoBeginTime(aivProfilingInfo);
786 0 : CHK_PTR_NULL(resourceArgs.buffersIn);
787 0 : CHK_PTR_NULL(resourceArgs.buffersOut);
788 :
789 : AivExtraKernelArgsV2 aivExtraKernelArgs {
790 0 : resourceArgs.buffersIn, resourceArgs.buffersOut, opArgs.input, opArgs.output,
791 0 : topoArgs.rank, topoArgs.rankSize, opArgs.count, opArgs.dataType, opArgs.op, opArgs.root, resourceArgs.aivTag,
792 0 : resourceArgs.numBlocks, opArgs.isOpBase, resourceArgs.bufferSize, algArgs.step, algArgs.isSmallCount, topoArgs.serverNum,
793 0 : static_cast<u32>(topoArgs.devType), reinterpret_cast<void*>(aivProfilingInfo.counter.headCountMem),
794 0 : reinterpret_cast<void*>(aivProfilingInfo.counter.tailCountMem), reinterpret_cast<void*>(aivProfilingInfo.counter.addOneMem),
795 0 : aivProfilingInfo.counter.memSize, aivProfilingInfo.counter.isEnableCounter, algArgs.deterministic, algArgs.rmaInfo, &extraArgs
796 0 : };
797 0 : CHK_RET(ExecuteKernelLaunchInner(opArgs, topoArgs, resourceArgs, algArgs, &aivExtraKernelArgs,
798 : sizeof(aivExtraKernelArgs), aivProfilingInfo));
799 :
800 0 : return HCCL_SUCCESS;
801 : }
802 :
803 : } // ~~ namespace hccl
|
