Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include <mutex>
12 : #include <vector>
13 : #include <iostream>
14 : #include <fstream>
15 : #include <unordered_map>
16 : #include <cmath>
17 : #include <limits>
18 : #include "mmpa_api.h"
19 : #include "adapter_rts_common.h"
20 : #include "acl/acl_rt.h"
21 : #include "launch_device.h"
22 : #include "hccl_aiv.h"
23 :
24 : using namespace std;
25 :
26 : namespace hccl {
27 : constexpr u32 SIG_MOVE_LEFT_BITS = 20;
28 : constexpr u32 RANK_ZERO = 0;
29 : constexpr u32 RANK_ONE = 1;
30 : constexpr u32 RANK_TWO = 2;
31 : constexpr u32 RANK_THREE = 3;
32 : constexpr u32 RANK_FOUR = 4;
33 : constexpr u32 RANK_FIVE = 5;
34 : constexpr u32 RANK_SIX = 6;
35 : constexpr u32 RANK_SEVEN = 7;
36 : constexpr u32 MAX_ARGS_SIZE_A3_STRUCT = 9;
37 :
38 : constexpr u32 AIV_BUFFER_PING_PONG_FACTOR = 2;
39 :
40 : constexpr u32 MAX_BIN_FILE_SIZE = 100 * 1024 * 1024; // 最大读取100m的bin file到string中
41 :
42 : constexpr s32 RESET_TAIL_SYNC_TAG = 2;
43 : constexpr u32 AIV_FLAG_AREA_SIZE = 1024 * 1024;
44 :
45 : constexpr u32 AIV_ATTRNUM_THREE = 3;
46 :
47 : using AivKernelInfo = struct AivKernelInfoDef {
48 : const char* kernelName;
49 : HcclDataType dataType;
50 : KernelArgsType argsType;
51 :
52 6486 : AivKernelInfoDef(
53 : const char* kernelName, HcclDataType dataType, KernelArgsType argsType = KernelArgsType::ARGS_TYPE_SERVER)
54 6486 : : kernelName(kernelName),
55 6486 : dataType(dataType),
56 6486 : argsType(argsType)
57 6486 : {}
58 : };
59 :
60 : static bool g_init = false;
61 : static mutex g_mut;
62 : static std::unordered_map<HcclCMDType, aclrtBinHandle> g_binHandleMap;
63 : static std::unordered_map<s8*, aclrtFuncHandle> g_aivFuncMap;
64 : static std::unordered_map<s8*, std::string> g_aivNameMap;
65 :
66 : static std::vector<AivKernelInfo> g_allreduceAivKernelInfoList = {
67 : {"aiv_all_reduce_float", HcclDataType::HCCL_DATA_TYPE_FP32},
68 : {"aiv_all_reduce_half", HcclDataType::HCCL_DATA_TYPE_FP16},
69 : {"aiv_all_reduce_int16_t", HcclDataType::HCCL_DATA_TYPE_INT16},
70 : {"aiv_all_reduce_int32_t", HcclDataType::HCCL_DATA_TYPE_INT32},
71 : {"aiv_all_reduce_int8_t", HcclDataType::HCCL_DATA_TYPE_INT8},
72 : {"aiv_all_reduce_bfloat16_t", HcclDataType::HCCL_DATA_TYPE_BFP16},
73 : {"aiv_all_reduce_cn_float", HcclDataType::HCCL_DATA_TYPE_FP32, KernelArgsType::ARGS_TYPE_SIMPLE},
74 : {"aiv_all_reduce_cn_half", HcclDataType::HCCL_DATA_TYPE_FP16, KernelArgsType::ARGS_TYPE_SIMPLE},
75 : {"aiv_all_reduce_cn_int16_t", HcclDataType::HCCL_DATA_TYPE_INT16, KernelArgsType::ARGS_TYPE_SIMPLE},
76 : {"aiv_all_reduce_cn_int32_t", HcclDataType::HCCL_DATA_TYPE_INT32, KernelArgsType::ARGS_TYPE_SIMPLE},
77 : {"aiv_all_reduce_cn_int8_t", HcclDataType::HCCL_DATA_TYPE_INT8, KernelArgsType::ARGS_TYPE_SIMPLE},
78 : {"aiv_all_reduce_cn_bfloat16_t", HcclDataType::HCCL_DATA_TYPE_BFP16, KernelArgsType::ARGS_TYPE_SIMPLE},
79 : };
80 : static std::string g_allreduceAivBinaryName = "hccl_aiv_all_reduce_op.o";
81 :
82 : static std::vector<AivKernelInfo> g_reducescatterAivKernelInfoList = {
83 : {"aiv_reduce_scatter_float", HcclDataType::HCCL_DATA_TYPE_FP32},
84 : {"aiv_reduce_scatter_half", HcclDataType::HCCL_DATA_TYPE_FP16},
85 : {"aiv_reduce_scatter_int16_t", HcclDataType::HCCL_DATA_TYPE_INT16},
86 : {"aiv_reduce_scatter_int32_t", HcclDataType::HCCL_DATA_TYPE_INT32},
87 : {"aiv_reduce_scatter_int8_t", HcclDataType::HCCL_DATA_TYPE_INT8},
88 : {"aiv_reduce_scatter_bfloat16_t", HcclDataType::HCCL_DATA_TYPE_BFP16},
89 : {"aiv_reduce_scatter_cn_float", HcclDataType::HCCL_DATA_TYPE_FP32, KernelArgsType::ARGS_TYPE_SIMPLE},
90 : {"aiv_reduce_scatter_cn_half", HcclDataType::HCCL_DATA_TYPE_FP16, KernelArgsType::ARGS_TYPE_SIMPLE},
91 : {"aiv_reduce_scatter_cn_int16_t", HcclDataType::HCCL_DATA_TYPE_INT16, KernelArgsType::ARGS_TYPE_SIMPLE},
92 : {"aiv_reduce_scatter_cn_int32_t", HcclDataType::HCCL_DATA_TYPE_INT32, KernelArgsType::ARGS_TYPE_SIMPLE},
93 : {"aiv_reduce_scatter_cn_int8_t", HcclDataType::HCCL_DATA_TYPE_INT8, KernelArgsType::ARGS_TYPE_SIMPLE},
94 : {"aiv_reduce_scatter_cn_bfloat16_t", HcclDataType::HCCL_DATA_TYPE_BFP16, KernelArgsType::ARGS_TYPE_SIMPLE},
95 : };
96 : static std::string g_reducescatterAivBinaryName = "hccl_aiv_reduce_scatter_op.o";
97 :
98 : static std::vector<AivKernelInfo> g_allgatherAivKernelInfoList = {
99 : {"aiv_all_gather_half", HcclDataType::HCCL_DATA_TYPE_FP16},
100 : {"aiv_all_gather_int16_t", HcclDataType::HCCL_DATA_TYPE_INT16},
101 : {"aiv_all_gather_uint16_t", HcclDataType::HCCL_DATA_TYPE_UINT16},
102 : {"aiv_all_gather_float", HcclDataType::HCCL_DATA_TYPE_FP32},
103 : {"aiv_all_gather_int32_t", HcclDataType::HCCL_DATA_TYPE_INT32},
104 : {"aiv_all_gather_uint32_t", HcclDataType::HCCL_DATA_TYPE_UINT32},
105 : {"aiv_all_gather_int8_t", HcclDataType::HCCL_DATA_TYPE_INT8},
106 : {"aiv_all_gather_uint8_t", HcclDataType::HCCL_DATA_TYPE_UINT8},
107 : {"aiv_all_gather_bfloat16_t", HcclDataType::HCCL_DATA_TYPE_BFP16},
108 : {"aiv_all_gather_int64_t", HcclDataType::HCCL_DATA_TYPE_INT64},
109 : {"aiv_all_gather_uint64_t", HcclDataType::HCCL_DATA_TYPE_UINT64},
110 : {"aiv_all_gather_double", HcclDataType::HCCL_DATA_TYPE_FP64},
111 : {"aiv_all_gather_cn_half", HcclDataType::HCCL_DATA_TYPE_FP16, KernelArgsType::ARGS_TYPE_SIMPLE},
112 : {"aiv_all_gather_cn_int16_t", HcclDataType::HCCL_DATA_TYPE_INT16, KernelArgsType::ARGS_TYPE_SIMPLE},
113 : {"aiv_all_gather_cn_uint16_t", HcclDataType::HCCL_DATA_TYPE_UINT16, KernelArgsType::ARGS_TYPE_SIMPLE},
114 : {"aiv_all_gather_cn_float", HcclDataType::HCCL_DATA_TYPE_FP32, KernelArgsType::ARGS_TYPE_SIMPLE},
115 : {"aiv_all_gather_cn_int32_t", HcclDataType::HCCL_DATA_TYPE_INT32, KernelArgsType::ARGS_TYPE_SIMPLE},
116 : {"aiv_all_gather_cn_uint32_t", HcclDataType::HCCL_DATA_TYPE_UINT32, KernelArgsType::ARGS_TYPE_SIMPLE},
117 : {"aiv_all_gather_cn_int8_t", HcclDataType::HCCL_DATA_TYPE_INT8, KernelArgsType::ARGS_TYPE_SIMPLE},
118 : {"aiv_all_gather_cn_uint8_t", HcclDataType::HCCL_DATA_TYPE_UINT8, KernelArgsType::ARGS_TYPE_SIMPLE},
119 : {"aiv_all_gather_cn_bfloat16_t", HcclDataType::HCCL_DATA_TYPE_BFP16, KernelArgsType::ARGS_TYPE_SIMPLE},
120 : {"aiv_all_gather_cn_int64_t", HcclDataType::HCCL_DATA_TYPE_INT64, KernelArgsType::ARGS_TYPE_SIMPLE},
121 : {"aiv_all_gather_cn_uint64_t", HcclDataType::HCCL_DATA_TYPE_UINT64, KernelArgsType::ARGS_TYPE_SIMPLE},
122 : {"aiv_all_gather_cn_double", HcclDataType::HCCL_DATA_TYPE_FP64, KernelArgsType::ARGS_TYPE_SIMPLE},
123 : };
124 : static std::string g_allgatherAivBinaryName = "hccl_aiv_all_gather_op.o";
125 :
126 : static std::vector<AivKernelInfo> g_broadcastAivKernelInfoList = {
127 : {"aiv_broadcast_half", HcclDataType::HCCL_DATA_TYPE_FP16},
128 : {"aiv_broadcast_int16_t", HcclDataType::HCCL_DATA_TYPE_INT16},
129 : {"aiv_broadcast_uint16_t", HcclDataType::HCCL_DATA_TYPE_UINT16},
130 : {"aiv_broadcast_float", HcclDataType::HCCL_DATA_TYPE_FP32},
131 : {"aiv_broadcast_int32_t", HcclDataType::HCCL_DATA_TYPE_INT32},
132 : {"aiv_broadcast_uint32_t", HcclDataType::HCCL_DATA_TYPE_UINT32},
133 : {"aiv_broadcast_int8_t", HcclDataType::HCCL_DATA_TYPE_INT8},
134 : {"aiv_broadcast_uint8_t", HcclDataType::HCCL_DATA_TYPE_UINT8},
135 : {"aiv_broadcast_bfloat16_t", HcclDataType::HCCL_DATA_TYPE_BFP16},
136 : {"aiv_broadcast_int64_t", HcclDataType::HCCL_DATA_TYPE_INT64},
137 : {"aiv_broadcast_uint64_t", HcclDataType::HCCL_DATA_TYPE_UINT64},
138 : {"aiv_broadcast_double", HcclDataType::HCCL_DATA_TYPE_FP64},
139 : {"aiv_broadcast_cn_half", HcclDataType::HCCL_DATA_TYPE_FP16, KernelArgsType::ARGS_TYPE_SIMPLE},
140 : {"aiv_broadcast_cn_int16_t", HcclDataType::HCCL_DATA_TYPE_INT16, KernelArgsType::ARGS_TYPE_SIMPLE},
141 : {"aiv_broadcast_cn_uint16_t", HcclDataType::HCCL_DATA_TYPE_UINT16, KernelArgsType::ARGS_TYPE_SIMPLE},
142 : {"aiv_broadcast_cn_float", HcclDataType::HCCL_DATA_TYPE_FP32, KernelArgsType::ARGS_TYPE_SIMPLE},
143 : {"aiv_broadcast_cn_int32_t", HcclDataType::HCCL_DATA_TYPE_INT32, KernelArgsType::ARGS_TYPE_SIMPLE},
144 : {"aiv_broadcast_cn_uint32_t", HcclDataType::HCCL_DATA_TYPE_UINT32, KernelArgsType::ARGS_TYPE_SIMPLE},
145 : {"aiv_broadcast_cn_int8_t", HcclDataType::HCCL_DATA_TYPE_INT8, KernelArgsType::ARGS_TYPE_SIMPLE},
146 : {"aiv_broadcast_cn_uint8_t", HcclDataType::HCCL_DATA_TYPE_UINT8, KernelArgsType::ARGS_TYPE_SIMPLE},
147 : {"aiv_broadcast_cn_bfloat16_t", HcclDataType::HCCL_DATA_TYPE_BFP16, KernelArgsType::ARGS_TYPE_SIMPLE},
148 : {"aiv_broadcast_cn_int64_t", HcclDataType::HCCL_DATA_TYPE_INT64, KernelArgsType::ARGS_TYPE_SIMPLE},
149 : {"aiv_broadcast_cn_uint64_t", HcclDataType::HCCL_DATA_TYPE_UINT64, KernelArgsType::ARGS_TYPE_SIMPLE},
150 : {"aiv_broadcast_cn_double", HcclDataType::HCCL_DATA_TYPE_FP64, KernelArgsType::ARGS_TYPE_SIMPLE},
151 : };
152 : static std::string g_broadcastAivBinaryName = "hccl_aiv_broadcast_op.o";
153 :
154 : static std::vector<AivKernelInfo> g_alltoallAivKernelInfoList = {
155 : {"aiv_all_to_all_half", HcclDataType::HCCL_DATA_TYPE_FP16},
156 : {"aiv_all_to_all_int16_t", HcclDataType::HCCL_DATA_TYPE_INT16},
157 : {"aiv_all_to_all_uint16_t", HcclDataType::HCCL_DATA_TYPE_UINT16},
158 : {"aiv_all_to_all_float", HcclDataType::HCCL_DATA_TYPE_FP32},
159 : {"aiv_all_to_all_int32_t", HcclDataType::HCCL_DATA_TYPE_INT32},
160 : {"aiv_all_to_all_uint32_t", HcclDataType::HCCL_DATA_TYPE_UINT32},
161 : {"aiv_all_to_all_int8_t", HcclDataType::HCCL_DATA_TYPE_INT8},
162 : {"aiv_all_to_all_uint8_t", HcclDataType::HCCL_DATA_TYPE_UINT8},
163 : {"aiv_all_to_all_bfloat16_t", HcclDataType::HCCL_DATA_TYPE_BFP16},
164 : {"aiv_all_to_all_int64_t", HcclDataType::HCCL_DATA_TYPE_INT64},
165 : {"aiv_all_to_all_uint64_t", HcclDataType::HCCL_DATA_TYPE_UINT64},
166 : {"aiv_all_to_all_double", HcclDataType::HCCL_DATA_TYPE_FP64},
167 : };
168 : static std::string g_alltoallAivBinaryName = "hccl_aiv_all_to_all_op.o";
169 :
170 : static std::vector<AivKernelInfo> g_alltoallvAivKernelInfoList = {
171 : {"aiv_all_to_all_v_half", HcclDataType::HCCL_DATA_TYPE_FP16},
172 : {"aiv_all_to_all_v_int16_t", HcclDataType::HCCL_DATA_TYPE_INT16},
173 : {"aiv_all_to_all_v_uint16_t", HcclDataType::HCCL_DATA_TYPE_UINT16},
174 : {"aiv_all_to_all_v_float", HcclDataType::HCCL_DATA_TYPE_FP32},
175 : {"aiv_all_to_all_v_int32_t", HcclDataType::HCCL_DATA_TYPE_INT32},
176 : {"aiv_all_to_all_v_uint32_t", HcclDataType::HCCL_DATA_TYPE_UINT32},
177 : {"aiv_all_to_all_v_int8_t", HcclDataType::HCCL_DATA_TYPE_INT8},
178 : {"aiv_all_to_all_v_uint8_t", HcclDataType::HCCL_DATA_TYPE_UINT8},
179 : {"aiv_all_to_all_v_bfloat16_t", HcclDataType::HCCL_DATA_TYPE_BFP16},
180 : {"aiv_all_to_all_v_int64_t", HcclDataType::HCCL_DATA_TYPE_INT64},
181 : {"aiv_all_to_all_v_uint64_t", HcclDataType::HCCL_DATA_TYPE_UINT64},
182 : {"aiv_all_to_all_v_double", HcclDataType::HCCL_DATA_TYPE_FP64},
183 : {"aiv_all_to_all_v_sp_half", HcclDataType::HCCL_DATA_TYPE_FP16, KernelArgsType::ARGS_TYPE_SUPERPOD},
184 : {"aiv_all_to_all_v_sp_int16_t", HcclDataType::HCCL_DATA_TYPE_INT16, KernelArgsType::ARGS_TYPE_SUPERPOD},
185 : {"aiv_all_to_all_v_sp_uint16_t", HcclDataType::HCCL_DATA_TYPE_UINT16, KernelArgsType::ARGS_TYPE_SUPERPOD},
186 : {"aiv_all_to_all_v_sp_float", HcclDataType::HCCL_DATA_TYPE_FP32, KernelArgsType::ARGS_TYPE_SUPERPOD},
187 : {"aiv_all_to_all_v_sp_int32_t", HcclDataType::HCCL_DATA_TYPE_INT32, KernelArgsType::ARGS_TYPE_SUPERPOD},
188 : {"aiv_all_to_all_v_sp_uint32_t", HcclDataType::HCCL_DATA_TYPE_UINT32, KernelArgsType::ARGS_TYPE_SUPERPOD},
189 : {"aiv_all_to_all_v_sp_int8_t", HcclDataType::HCCL_DATA_TYPE_INT8, KernelArgsType::ARGS_TYPE_SUPERPOD},
190 : {"aiv_all_to_all_v_sp_uint8_t", HcclDataType::HCCL_DATA_TYPE_UINT8, KernelArgsType::ARGS_TYPE_SUPERPOD},
191 : {"aiv_all_to_all_v_sp_bfloat16_t", HcclDataType::HCCL_DATA_TYPE_BFP16, KernelArgsType::ARGS_TYPE_SUPERPOD},
192 : {"aiv_all_to_all_v_sp_int64_t", HcclDataType::HCCL_DATA_TYPE_INT64, KernelArgsType::ARGS_TYPE_SUPERPOD},
193 : {"aiv_all_to_all_v_sp_uint64_t", HcclDataType::HCCL_DATA_TYPE_UINT64, KernelArgsType::ARGS_TYPE_SUPERPOD},
194 : {"aiv_all_to_all_v_sp_double", HcclDataType::HCCL_DATA_TYPE_FP64, KernelArgsType::ARGS_TYPE_SUPERPOD},
195 : };
196 : static std::string g_alltoallvAivBinaryName = "hccl_aiv_all_to_all_v_op.o";
197 :
198 : static std::vector<AivKernelInfo> g_alltoallvcAivKernelInfoList = {
199 : {"aiv_all_to_all_vc_half", HcclDataType::HCCL_DATA_TYPE_FP16},
200 : {"aiv_all_to_all_vc_int16_t", HcclDataType::HCCL_DATA_TYPE_INT16},
201 : {"aiv_all_to_all_vc_uint16_t", HcclDataType::HCCL_DATA_TYPE_UINT16},
202 : {"aiv_all_to_all_vc_float", HcclDataType::HCCL_DATA_TYPE_FP32},
203 : {"aiv_all_to_all_vc_int32_t", HcclDataType::HCCL_DATA_TYPE_INT32},
204 : {"aiv_all_to_all_vc_uint32_t", HcclDataType::HCCL_DATA_TYPE_UINT32},
205 : {"aiv_all_to_all_vc_int8_t", HcclDataType::HCCL_DATA_TYPE_INT8},
206 : {"aiv_all_to_all_vc_uint8_t", HcclDataType::HCCL_DATA_TYPE_UINT8},
207 : {"aiv_all_to_all_vc_bfloat16_t", HcclDataType::HCCL_DATA_TYPE_BFP16},
208 : {"aiv_all_to_all_vc_int64_t", HcclDataType::HCCL_DATA_TYPE_INT64},
209 : {"aiv_all_to_all_vc_uint64_t", HcclDataType::HCCL_DATA_TYPE_UINT64},
210 : {"aiv_all_to_all_vc_double", HcclDataType::HCCL_DATA_TYPE_FP64},
211 : };
212 : static std::string g_alltoallvcAivBinaryName = "hccl_aiv_all_to_all_vc_op.o";
213 :
214 : static std::vector<AivKernelInfo> g_reducescattervAivKernelInfoList = {
215 : {"aiv_reduce_scatter_v_float", HcclDataType::HCCL_DATA_TYPE_FP32},
216 : {"aiv_reduce_scatter_v_half", HcclDataType::HCCL_DATA_TYPE_FP16},
217 : {"aiv_reduce_scatter_v_int16_t", HcclDataType::HCCL_DATA_TYPE_INT16},
218 : {"aiv_reduce_scatter_v_int32_t", HcclDataType::HCCL_DATA_TYPE_INT32},
219 : {"aiv_reduce_scatter_v_int8_t", HcclDataType::HCCL_DATA_TYPE_INT8},
220 : {"aiv_reduce_scatter_v_bfloat16_t", HcclDataType::HCCL_DATA_TYPE_BFP16},
221 : };
222 : static std::string g_reducescattervAivBinaryName = "hccl_aiv_reduce_scatter_v_op.o";
223 :
224 : static std::vector<AivKernelInfo> g_allgathervAivKernelInfoList = {
225 : {"aiv_all_gather_v_half", HcclDataType::HCCL_DATA_TYPE_FP16},
226 : {"aiv_all_gather_v_int16_t", HcclDataType::HCCL_DATA_TYPE_INT16},
227 : {"aiv_all_gather_v_uint16_t", HcclDataType::HCCL_DATA_TYPE_UINT16},
228 : {"aiv_all_gather_v_float", HcclDataType::HCCL_DATA_TYPE_FP32},
229 : {"aiv_all_gather_v_int32_t", HcclDataType::HCCL_DATA_TYPE_INT32},
230 : {"aiv_all_gather_v_uint32_t", HcclDataType::HCCL_DATA_TYPE_UINT32},
231 : {"aiv_all_gather_v_int8_t", HcclDataType::HCCL_DATA_TYPE_INT8},
232 : {"aiv_all_gather_v_uint8_t", HcclDataType::HCCL_DATA_TYPE_UINT8},
233 : {"aiv_all_gather_v_bfloat16_t", HcclDataType::HCCL_DATA_TYPE_BFP16},
234 : {"aiv_all_gather_v_int64_t", HcclDataType::HCCL_DATA_TYPE_INT64},
235 : {"aiv_all_gather_v_uint64_t", HcclDataType::HCCL_DATA_TYPE_UINT64},
236 : {"aiv_all_gather_v_double", HcclDataType::HCCL_DATA_TYPE_FP64},
237 : };
238 : static std::string g_allgathervAivBinaryName = "hccl_aiv_all_gather_v_op.o";
239 :
240 : static std::map<HcclCMDType, std::pair<std::string, std::vector<AivKernelInfo>>> g_aivKernelInfoMap = {
241 : {HcclCMDType::HCCL_CMD_ALLREDUCE, {g_allreduceAivBinaryName, g_allreduceAivKernelInfoList}},
242 : {HcclCMDType::HCCL_CMD_REDUCE_SCATTER, {g_reducescatterAivBinaryName, g_reducescatterAivKernelInfoList}},
243 : {HcclCMDType::HCCL_CMD_ALLGATHER, {g_allgatherAivBinaryName, g_allgatherAivKernelInfoList}},
244 : {HcclCMDType::HCCL_CMD_BROADCAST, {g_broadcastAivBinaryName, g_broadcastAivKernelInfoList}},
245 : {HcclCMDType::HCCL_CMD_ALLTOALL, {g_alltoallAivBinaryName, g_alltoallAivKernelInfoList}},
246 : {HcclCMDType::HCCL_CMD_ALLTOALLV, {g_alltoallvAivBinaryName, g_alltoallvAivKernelInfoList}},
247 : {HcclCMDType::HCCL_CMD_ALLTOALLVC, {g_alltoallvcAivBinaryName, g_alltoallvcAivKernelInfoList}},
248 : {HcclCMDType::HCCL_CMD_REDUCE_SCATTER_V, {g_reducescattervAivBinaryName, g_reducescattervAivKernelInfoList}},
249 : {HcclCMDType::HCCL_CMD_ALLGATHER_V, {g_allgathervAivBinaryName, g_allgathervAivKernelInfoList}},
250 : };
251 :
252 : using AivKernelArgs = struct AivKernelArgsDef {
253 : const void* buffersIn[MAX_RANK_SIZE] = {}; // 注册的CCLIN地址，所有卡可访问
254 : const void* buffersOut[MAX_RANK_SIZE] = {}; // 注册的CCLOUT地址，所有卡可访问
255 : const void* input;
256 : const void* output;
257 : u32 rank;
258 : u32 rankSize;
259 : u64 len;
260 : u32 dataType;
261 : u32 reduceOp;
262 : u32 root;
263 : s32 tag; // 第几次调用，定时重置成1
264 : u32 numBlocks;
265 : bool isOpBase;
266 : u64 bufferSize;
267 : s32 aivRdmaStep; // 用于AIV与rdma组合的场景，在不同step中kernel完成多机通信的不同部分
268 : bool useAivRdmaSmall; // 使用aivRdma小数据量kernel，否则使用中数据量kernel
269 : u32 serverNum;
270 : u32 devType;
271 : const void* headCounterAddr;
272 : const void* tailCounterAddr;
273 : const void* addOneAddr;
274 : u32 counterMemSize;
275 : bool isEnableCounter;
276 : u32 deterministic;
277 : u64 rmaInfo;
278 :
279 0 : AivKernelArgsDef(
280 : void** buffIn, void** buffOut, const void* input, const void* output, u32 rank, u32 rankSize, u64 len,
281 : u32 dataType, u32 reduceOp, u32 root, s32 tag, u32 numBlocks, bool isOpBase = true,
282 : u64 bufferSize = 200 * 1024 * 1024, s32 aivRdmaStep = -1, bool useAivRdmaSmall = false, u32 serverNum = 1,
283 : u32 devType = 2, const void* headCounterAddr = nullptr, const void* tailCounterAddr = nullptr,
284 : const void* addOneAddr = nullptr, u32 counterMemSize = 0, bool isEnableCounter = false, u32 deterministic = 0,
285 : u64 rmaInfo = 0)
286 0 : : input(input),
287 0 : output(output),
288 0 : rank(rank),
289 0 : rankSize(rankSize),
290 0 : len(len),
291 0 : dataType(dataType),
292 0 : reduceOp(reduceOp),
293 0 : root(root),
294 0 : tag(tag),
295 0 : numBlocks(numBlocks),
296 0 : isOpBase(isOpBase),
297 0 : bufferSize(bufferSize),
298 0 : aivRdmaStep(aivRdmaStep),
299 0 : useAivRdmaSmall(useAivRdmaSmall),
300 0 : serverNum(serverNum),
301 0 : devType(devType),
302 0 : headCounterAddr(headCounterAddr),
303 0 : tailCounterAddr(tailCounterAddr),
304 0 : addOneAddr(addOneAddr),
305 0 : counterMemSize(counterMemSize),
306 0 : isEnableCounter(isEnableCounter),
307 0 : deterministic(deterministic),
308 0 : rmaInfo(rmaInfo)
309 : {
310 0 : for (u32 i = 0; i < MAX_RANK_SIZE; i++) {
311 0 : buffersIn[i] = (u8*)buffIn[i];
312 0 : buffersOut[i] = (u8*)buffOut[i];
313 : }
314 0 : }
315 : };
316 :
317 : using AivExtraKernelArgs = struct AivExtraKernelArgsDef {
318 : const void* buffersIn[MAX_RANK_SIZE] = {}; // 注册的CCLIN地址，所有卡可访问
319 : const void* buffersOut[MAX_RANK_SIZE] = {}; // 注册的CCLOUT地址，所有卡可访问
320 : const void* input;
321 : const void* output;
322 : u32 rank;
323 : u32 rankSize;
324 : u64 len;
325 : u32 dataType;
326 : u32 reduceOp;
327 : u32 root;
328 : s32 tag; // 第几次调用，定时重置成1
329 : u32 numBlocks;
330 : bool isOpBase;
331 : u64 bufferSize;
332 : s32 aivRdmaStep; // 用于AIV与rdma组合的场景，在不同step中kernel完成多机通信的不同部分
333 : bool useAivRdmaSmall; // 使用aivRdma小数据量kernel，否则使用中数据量kernel
334 : u32 serverNum;
335 : u32 devType;
336 : const void* headCounterAddr;
337 : const void* tailCounterAddr;
338 : const void* addOneAddr;
339 : u32 counterMemSize;
340 : bool isEnableCounter;
341 : u32 deterministic;
342 : u64 rmaInfo;
343 : ExtraArgs extraArgs; // A2/A3单机
344 :
345 5 : AivExtraKernelArgsDef(
346 : void** buffIn, void** buffOut, const void* input, const void* output, u32 rank, u32 rankSize, u64 len,
347 : u32 dataType, u32 reduceOp, u32 root, s32 tag, u32 numBlocks, bool isOpBase = true,
348 : u64 bufferSize = 200 * 1024 * 1024, s32 aivRdmaStep = -1, bool useAivRdmaSmall = false, u32 serverNum = 1,
349 : u32 devType = 2, const void* headCounterAddr = nullptr, const void* tailCounterAddr = nullptr,
350 : const void* addOneAddr = nullptr, u32 counterMemSize = 0, bool isEnableCounter = false, u32 deterministic = 0,
351 : u64 rmaInfo = 0, const ExtraArgs* extraArgsPtr = nullptr)
352 5 : : input(input),
353 5 : output(output),
354 5 : rank(rank),
355 5 : rankSize(rankSize),
356 5 : len(len),
357 5 : dataType(dataType),
358 5 : reduceOp(reduceOp),
359 5 : root(root),
360 5 : tag(tag),
361 5 : numBlocks(numBlocks),
362 5 : isOpBase(isOpBase),
363 5 : bufferSize(bufferSize),
364 5 : aivRdmaStep(aivRdmaStep),
365 5 : useAivRdmaSmall(useAivRdmaSmall),
366 5 : serverNum(serverNum),
367 5 : devType(devType),
368 5 : headCounterAddr(headCounterAddr),
369 5 : tailCounterAddr(tailCounterAddr),
370 5 : addOneAddr(addOneAddr),
371 5 : counterMemSize(counterMemSize),
372 5 : isEnableCounter(isEnableCounter),
373 5 : deterministic(deterministic),
374 5 : rmaInfo(rmaInfo)
375 : {
376 85 : for (u32 i = 0; i < MAX_RANK_SIZE; i++) {
377 80 : buffersOut[i] = (u8*)buffOut[i];
378 80 : buffersIn[i] = (u8*)buffIn[i];
379 : }
380 5 : if (extraArgsPtr != nullptr) {
381 5 : extraArgs = *extraArgsPtr;
382 : }
383 5 : }
384 : };
385 :
386 : using AivExtraKernelArgsV2 = struct AivExtraKernelArgsV2Def {
387 : const void* buffersIn[MAX_RANK_SIZE] = {}; // 注册的CCLIN地址，所有卡可访问
388 : const void* buffersOut[MAX_RANK_SIZE] = {}; // 注册的CCLOUT地址，所有卡可访问
389 : const void* input;
390 : const void* output;
391 : u32 rank;
392 : u32 rankSize;
393 : u64 len;
394 : u32 dataType;
395 : u32 reduceOp;
396 : u32 root;
397 : s32 tag; // 第几次调用，定时重置成1
398 : u32 numBlocks;
399 : bool isOpBase;
400 : u64 bufferSize;
401 : s32 aivRdmaStep; // 用于AIV与rdma组合的场景，在不同step中kernel完成多机通信的不同部分
402 : bool useAivRdmaSmall; // 使用aivRdma小数据量kernel，否则使用中数据量kernel
403 : u32 serverNum;
404 : u32 devType;
405 : const void* headCounterAddr;
406 : const void* tailCounterAddr;
407 : const void* addOneAddr;
408 : u32 counterMemSize;
409 : bool isEnableCounter;
410 : u32 deterministic;
411 : u64 rmaInfo;
412 : ExtraArgsV2 extraArgs; // A3超节点内多机
413 :
414 0 : AivExtraKernelArgsV2Def(
415 : void** buffIn, void** buffOut, const void* input, const void* output, u32 rank, u32 rankSize, u64 len,
416 : u32 dataType, u32 reduceOp, u32 root, s32 tag, u32 numBlocks, bool isOpBase = true,
417 : u64 bufferSize = 200 * 1024 * 1024, s32 aivRdmaStep = -1, bool useAivRdmaSmall = false, u32 serverNum = 1,
418 : u32 devType = 2, const void* headCounterAddr = nullptr, const void* tailCounterAddr = nullptr,
419 : const void* addOneAddr = nullptr, u32 counterMemSize = 0, bool isEnableCounter = false, u32 deterministic = 0,
420 : u64 rmaInfo = 0, const ExtraArgsV2* extraArgsPtr = nullptr)
421 0 : : input(input),
422 0 : output(output),
423 0 : rank(rank),
424 0 : rankSize(rankSize),
425 0 : len(len),
426 0 : dataType(dataType),
427 0 : reduceOp(reduceOp),
428 0 : root(root),
429 0 : tag(tag),
430 0 : numBlocks(numBlocks),
431 0 : isOpBase(isOpBase),
432 0 : bufferSize(bufferSize),
433 0 : aivRdmaStep(aivRdmaStep),
434 0 : useAivRdmaSmall(useAivRdmaSmall),
435 0 : serverNum(serverNum),
436 0 : devType(devType),
437 0 : headCounterAddr(headCounterAddr),
438 0 : tailCounterAddr(tailCounterAddr),
439 0 : addOneAddr(addOneAddr),
440 0 : counterMemSize(counterMemSize),
441 0 : isEnableCounter(isEnableCounter),
442 0 : deterministic(deterministic),
443 0 : rmaInfo(rmaInfo)
444 : {
445 0 : for (u32 i = 0; i < MAX_RANK_SIZE; i++) {
446 0 : buffersIn[i] = (u8*)buffIn[i];
447 0 : buffersOut[i] = (u8*)buffOut[i];
448 : }
449 0 : if (extraArgsPtr != nullptr) {
450 0 : extraArgs = *extraArgsPtr;
451 : }
452 0 : }
453 : };
454 :
455 : using AivKernelArgsV3 = struct AivKernelArgsV3Def {
456 : u64 massArgs[MAX_ARGS_SIZE_A3_STRUCT] = {};
457 : const void* input;
458 : const void* output;
459 : u32 rank;
460 : u32 rankSize;
461 : u64 len;
462 : u32 dataType;
463 : u32 reduceOp;
464 : u32 root;
465 : s32 tag;
466 : u32 numBlocks;
467 : bool isOpBase;
468 : s32 step;
469 : u32 deterministic;
470 :
471 0 : AivKernelArgsV3Def(
472 : void** buffIn, void** buffOut, const void* input, const void* output, u32 rank, u32 rankSize, u64 len,
473 : u32 dataType, u32 reduceOp, u32 root, s32 tag, u32 numBlocks, bool isOpBase = true,
474 : u64 bufferSize = 200 * 1024 * 1024, s32 step = 0, void* headCounterAddr = nullptr,
475 : void* tailCounterAddr = nullptr, void* addOneAddr = nullptr, bool isEnableCounter = false,
476 : u32 deterministic = 0)
477 0 : : input(input),
478 0 : output(output),
479 0 : rank(rank),
480 0 : rankSize(rankSize),
481 0 : len(len),
482 0 : dataType(dataType),
483 0 : reduceOp(reduceOp),
484 0 : root(root),
485 0 : tag(tag),
486 0 : numBlocks(numBlocks),
487 0 : isOpBase(isOpBase),
488 0 : step(step),
489 0 : deterministic(deterministic)
490 : {
491 0 : massArgs[0] = reinterpret_cast<u64>(buffIn[0]);
492 0 : massArgs[1] = reinterpret_cast<u64>(buffIn[1]);
493 0 : massArgs[2] = reinterpret_cast<u64>(buffOut[0]);
494 0 : massArgs[3] = reinterpret_cast<u64>(buffOut[1]);
495 0 : massArgs[4] = reinterpret_cast<u64>(bufferSize);
496 0 : massArgs[5] = reinterpret_cast<u64>(headCounterAddr);
497 0 : massArgs[6] = reinterpret_cast<u64>(tailCounterAddr);
498 0 : massArgs[7] = reinterpret_cast<u64>(addOneAddr);
499 0 : massArgs[8] = isEnableCounter ? 1 : 0;
500 0 : }
501 : };
502 :
503 45 : HcclResult GetAivOpBinaryPath(const std::string& aivBinaryName, std::string& binaryPath)
504 : {
505 : // 获取二进制文件路径
506 45 : std::string libPath;
507 45 : char* getPath = nullptr;
508 45 : MM_SYS_GET_ENV(MM_ENV_ASCEND_HOME_PATH, getPath);
509 45 : if (getPath != nullptr) {
510 45 : libPath = getPath;
511 : } else {
512 0 : libPath = "/usr/local/Ascend/cann";
513 0 : HCCL_WARNING("[GetAivOpBinaryPath]ENV:ASCEND_HOME_PATH is not set");
514 : }
515 45 : binaryPath = libPath + "/lib64";
516 :
517 : // 拼接应该加载的文件
518 45 : binaryPath += "/" + aivBinaryName;
519 45 : HCCL_RUN_INFO("[GetAivOpBinaryPath]op binary file path[%s]", binaryPath.c_str());
520 45 : return HCCL_SUCCESS;
521 45 : }
522 :
523 0 : HcclResult ReadBinFile(const string& fileName, string& buffer)
524 : {
525 0 : char realFile[PATH_MAX] = {0};
526 0 : if (realpath(fileName.c_str(), realFile) == nullptr) {
527 0 : HCCL_INFO("[AIV][ReadBinFile] Binfile path %s is not a valid real path.", realFile);
528 0 : return HCCL_E_NOT_FOUND;
529 : }
530 0 : std::ifstream filestr;
531 0 : filestr.open(realFile, std::ios::binary);
532 0 : if (!filestr) {
533 0 : HCCL_ERROR("[AIV][ReadBinFile]open file [%s] failed!", fileName.c_str());
534 0 : return HCCL_E_OPEN_FILE_FAILURE;
535 : }
536 :
537 0 : filestr.seekg(0, std::ios::end);
538 0 : std::streampos fileSize = filestr.tellg();
539 0 : filestr.seekg(0, std::ios::beg);
540 :
541 0 : if (fileSize == 0 || fileSize >= MAX_BIN_FILE_SIZE) {
542 0 : HCCL_ERROR("[AIV][ReadBinFile] file [%s] size is invalid, is [%d]!", fileName.c_str(), fileSize);
543 0 : filestr.close();
544 0 : return HCCL_E_OPEN_FILE_FAILURE;
545 : }
546 0 : buffer.resize(fileSize);
547 0 : filestr.read(&buffer[0], fileSize);
548 :
549 0 : filestr.close();
550 0 : return HCCL_SUCCESS;
551 0 : }
552 :
553 695 : s8* GetStubFunc(HcclCMDType cmdType, HcclDataType dataType, KernelArgsType argsType = KernelArgsType::ARGS_TYPE_SERVER)
554 : {
555 : return reinterpret_cast<s8*>(
556 695 : (((static_cast<s64>(cmdType) << SIG_MOVE_LEFT_BITS) + static_cast<s64>(dataType)) << SIG_MOVE_LEFT_BITS)
557 695 : + static_cast<s64>(argsType));
558 : }
559 :
560 690 : HcclResult RegisterBinaryKernel(const char* funcName, const aclrtBinHandle binHandle, s8* stubFunc)
561 : {
562 690 : if (stubFunc == nullptr) {
563 0 : return HCCL_E_PARA;
564 : }
565 :
566 : aclrtFuncHandle funcHandle;
567 690 : aclError aclRet = aclrtBinaryGetFunction(binHandle, funcName, &funcHandle);
568 690 : CHK_PRT_RET(
569 : aclRet != ACL_SUCCESS,
570 : HCCL_ERROR("[RegisterBinaryKernel]errNo[0x%016llx] get function from binary error.", aclRet), HCCL_E_NOT_FOUND);
571 :
572 690 : g_aivFuncMap[stubFunc] = funcHandle;
573 690 : g_aivNameMap[stubFunc] = funcName;
574 :
575 690 : return HCCL_SUCCESS;
576 : }
577 :
578 5 : HcclResult GetKernelFunc(aclrtFuncHandle& funcHandle, s8* stubFunc)
579 : {
580 5 : if (stubFunc == nullptr || g_aivFuncMap.find(stubFunc) == g_aivFuncMap.end()) {
581 0 : return HCCL_E_PARA;
582 : }
583 5 : funcHandle = g_aivFuncMap[stubFunc];
584 5 : return HCCL_SUCCESS;
585 : }
586 :
587 : // Kernel注册入口，全局只需要初始化一次
588 70 : HcclResult RegisterKernel([[maybe_unused]] DevType deviceType)
589 : {
590 70 : lock_guard<mutex> guard(g_mut);
591 70 : if (g_init) {
592 65 : return HCCL_SUCCESS;
593 : }
594 : HcclResult ret;
595 :
596 50 : for (auto& aivOpKernelInfo : g_aivKernelInfoMap) {
597 45 : HcclCMDType cmdType = aivOpKernelInfo.first;
598 45 : const std::string& aivBinaryName = aivOpKernelInfo.second.first;
599 45 : std::vector<AivKernelInfo>& aivKernelInfoList = aivOpKernelInfo.second.second;
600 :
601 45 : string binFilePath;
602 45 : ret = GetAivOpBinaryPath(aivBinaryName, binFilePath);
603 45 : CHK_PRT_RET(
604 : ret != HCCL_SUCCESS, HCCL_ERROR("[AIV][RegisterKernel] get aiv op binary path failed"), HCCL_E_RUNTIME);
605 :
606 45 : ret = LoadBinaryFromFile(binFilePath.c_str(), ACL_RT_BINARY_LOAD_OPT_LAZY_LOAD, 1, g_binHandleMap[cmdType]);
607 45 : CHK_PRT_RET(
608 : ret != HCCL_SUCCESS, HCCL_ERROR("[AIV][RegisterKernel] read aiv kernel bin file failed"), HCCL_E_RUNTIME);
609 :
610 735 : for (auto& aivKernelInfo : aivKernelInfoList) {
611 690 : ret = RegisterBinaryKernel(
612 690 : aivKernelInfo.kernelName, g_binHandleMap[cmdType],
613 : GetStubFunc(cmdType, aivKernelInfo.dataType, aivKernelInfo.argsType));
614 690 : CHK_PRT_RET(
615 : ret != HCCL_SUCCESS,
616 : HCCL_ERROR(
617 : "[AIV][RegisterKernel] register binary kernel for kernelName[%s] "
618 : "cmdType[%d] dataType[%s] argsType[%d] failed",
619 : aivKernelInfo.kernelName, cmdType, GetDataTypeEnumStr(aivKernelInfo.dataType).c_str(),
620 : aivKernelInfo.argsType),
621 : HCCL_E_RUNTIME);
622 : }
623 45 : }
624 :
625 5 : g_init = true;
626 :
627 5 : return HCCL_SUCCESS;
628 70 : }
629 :
630 0 : HcclResult UnRegisterAivKernel()
631 : {
632 0 : lock_guard<mutex> guard(g_mut);
633 0 : if (g_init) {
634 0 : for (auto& binHandleItem : g_binHandleMap) {
635 0 : aclrtBinaryUnLoad(binHandleItem.second);
636 : }
637 0 : g_aivFuncMap.clear();
638 :
639 0 : g_init = false;
640 : }
641 :
642 0 : return HCCL_SUCCESS;
643 0 : }
644 :
645 5 : HcclResult GetMinAndMaxNpuSchedTimeOut(u64& minNpuSchedTimeout, u64& maxNpuSchedTimeout)
646 : {
647 5 : uint64_t interval = 0;
648 5 : aclError aclRet = aclrtGetOpTimeOutInterval(&interval);
649 5 : CHK_PRT_RET(
650 : aclRet != ACL_SUCCESS, HCCL_ERROR("aclrtGetOpTimeOutInterval get timeout interval failed, ret[%d]", aclRet),
651 : HCCL_E_RUNTIME);
652 :
653 5 : constexpr u64 MAX_INTERVAL = 254;
654 : // NPU超时范围(1, 254) * interval
655 5 : minNpuSchedTimeout = 1 * interval;
656 5 : maxNpuSchedTimeout = MAX_INTERVAL * interval;
657 5 : HCCL_INFO(
658 : "GetMinAndMaxNpuSchedTimeOut minNpuSchedTimeout[%llu]us, maxNpuSchedTimeout[%llu]us.", minNpuSchedTimeout,
659 : maxNpuSchedTimeout);
660 5 : return HCCL_SUCCESS;
661 : }
662 :
663 5 : u32 GetAivTimeout(s32 execTimeOut, bool isSetByConfig)
664 : {
665 5 : u32 timeout = AIV_TIMEOUT_DEFAULT_US;
666 5 : if (GetExternalInputHcclExecTimeoutSet() != HcclExecTimeoutSet::HCCL_EXEC_TIMEOUT_NOT_SET || isSetByConfig) {
667 : // 配0时，使用最大超时时间
668 5 : if (execTimeOut == 0) {
669 0 : return AIV_TIMEOUT_MAX_US;
670 : }
671 5 : double timeoutUs = execTimeOut * TIME_S_TO_US;
672 5 : if (timeoutUs > static_cast<double>(std::numeric_limits<u32>::max())) {
673 0 : HCCL_INFO("[GetAivTimeout]Get input timeout[%.2f us] is out of valid range.", timeoutUs);
674 0 : return AIV_TIMEOUT_MAX_US;
675 : }
676 5 : u32 timeoutUsInt = static_cast<u32>(timeoutUs);
677 5 : u64 minNpuSchedTimeout = 0;
678 5 : u64 maxNpuSchedTimeout = 0;
679 5 : CHK_RET(GetMinAndMaxNpuSchedTimeOut(minNpuSchedTimeout, maxNpuSchedTimeout));
680 5 : timeout = (timeoutUsInt < minNpuSchedTimeout) ? minNpuSchedTimeout :
681 5 : (timeoutUsInt > maxNpuSchedTimeout) ? maxNpuSchedTimeout :
682 : timeoutUsInt;
683 5 : HCCL_INFO(
684 : "[GetAivTimeout]timeout[%u]us, minNpuSchedTimeout[%llu]us, maxNpuSchedTimeout[%llu]us.", timeout,
685 : minNpuSchedTimeout, maxNpuSchedTimeout);
686 : }
687 :
688 5 : return timeout < AIV_TIMEOUT_MAX_US ? timeout : AIV_TIMEOUT_MAX_US;
689 : }
690 :
691 5 : void TaskAivProfilerWrap(
692 : const AivOpArgs& opArgs, const AivTopoArgs& topoArgs, const AivResourceArgs& resourceArgs,
693 : const AivAlgArgs& algArgs, const AivProfilingInfo& aivProfilingInfo, void* flagMem)
694 : {
695 5 : struct TaskParaGeneral taskParaGeneral;
696 :
697 : TaskParaAiv taskParaAiv(
698 5 : opArgs.cmdType, resourceArgs.aivTag, opArgs.count * SIZE_TABLE[opArgs.dataType], resourceArgs.numBlocks,
699 5 : topoArgs.rankSize, algArgs.step, flagMem, topoArgs.rank, opArgs.isOpBase);
700 :
701 5 : if (taskParaAiv.flagMem == nullptr) {
702 0 : taskParaAiv.flagMem = resourceArgs.buffersOut[topoArgs.rank];
703 : }
704 :
705 5 : taskParaGeneral.isMainStream = true;
706 5 : taskParaGeneral.stream = resourceArgs.stream;
707 5 : taskParaGeneral.beginTime = aivProfilingInfo.beginTime;
708 5 : taskParaGeneral.aiv = taskParaAiv;
709 :
710 5 : AlgWrap::GetInstance().TaskAivProfiler(topoArgs.identify, taskParaGeneral);
711 5 : }
712 :
713 0 : HcclResult CacheTaskOpInfo(rtStream_t stream, const std::string& identify)
714 : {
715 0 : aclmdlRI rtModel = nullptr;
716 0 : aclmdlRICaptureStatus captureStatus = aclmdlRICaptureStatus::ACL_MODEL_RI_CAPTURE_STATUS_NONE;
717 0 : aclError aclRet = aclmdlRICaptureGetInfo(stream, &captureStatus, &rtModel);
718 0 : if (aclRet == ACL_ERROR_RT_FEATURE_NOT_SUPPORT) {
719 0 : HCCL_WARNING("[%s]Stream capture does not support!", __func__);
720 0 : return HCCL_SUCCESS;
721 : }
722 0 : CHK_PRT_RET(
723 : aclRet != ACL_SUCCESS, HCCL_ERROR("[%s]rtGet stream get capture status fail. return[%d]", __func__, aclRet),
724 : HCCL_E_RUNTIME);
725 :
726 : aclrtStreamAttrValue value;
727 0 : aclRet = aclrtGetStreamAttribute(stream, ACL_STREAM_ATTR_CACHE_OP_INFO, &value);
728 0 : CHK_PRT_RET(
729 : aclRet != ACL_SUCCESS, HCCL_ERROR("[%s]stream get attribute fail. return[%d]", __func__, aclRet),
730 : HCCL_E_RUNTIME);
731 :
732 0 : HCCL_INFO(
733 : "[CacheTaskOpInfo] cacheOpInfoSwitch[%u] captureStatus[%d] identify[%s]", value.cacheOpInfoSwitch,
734 : captureStatus, identify.c_str());
735 0 : if (value.cacheOpInfoSwitch == 1 && captureStatus == ACL_MODEL_RI_CAPTURE_STATUS_ACTIVE) {
736 0 : HcclResult cacheRet = hrtCacheLastTaskExtendInfo(identify.c_str(), strlen(identify.c_str()));
737 0 : if (cacheRet == HCCL_E_NOT_SUPPORT) {
738 0 : HCCL_INFO("[%s] aclrtCacheLastTaskExtendInfo not supported", __func__);
739 : } else {
740 0 : CHK_PRT_RET(
741 : cacheRet != HCCL_SUCCESS,
742 : HCCL_ERROR("[%s] stream cache task op info fail. return[%d]", __func__, cacheRet), HCCL_E_RUNTIME);
743 : }
744 : }
745 0 : return HCCL_SUCCESS;
746 : }
747 :
748 : // KernelLaunch内部接口
749 5 : HcclResult ExecuteKernelLaunchInner(
750 : const AivOpArgs& opArgs, const AivTopoArgs& topoArgs, const AivResourceArgs& resourceArgs,
751 : const AivAlgArgs& algArgs, void* args, u32 argsSize, AivProfilingInfo& aivProfilingInfo)
752 : {
753 5 : HCCL_INFO(
754 : "[AIV][ExecuteKernelLaunchInner] sendbuff [%p] recvbuff [%p] rank [%d] rankSize [%d] count [%llu] "
755 : "dataType [%s] reduceOp [%s] root [%d] tag [%d] isOpBase [%d] bufferSize [%llu] step [%d] "
756 : "isSmallCount [%d] serverNum [%d] devType[%d] extraArgsPtr [%p] argsSize [%d], deterministic [%d].",
757 : opArgs.input, opArgs.output, topoArgs.rank, topoArgs.rankSize, opArgs.count,
758 : GetDataTypeEnumStr(opArgs.dataType).c_str(), GetReduceOpEnumStr(opArgs.op).c_str(), opArgs.root,
759 : resourceArgs.aivTag, opArgs.isOpBase, resourceArgs.bufferSize, algArgs.step, algArgs.isSmallCount,
760 : topoArgs.serverNum, topoArgs.devType, args, argsSize, algArgs.deterministic);
761 :
762 5 : HCCL_DEBUG(
763 : "[AIV][ExecuteKernelLaunchInner] buffersIn [%p] [%p] [%p] [%p] [%p] [%p] [%p] [%p] "
764 : "buffersOut [%p] [%p] [%p] [%p] [%p] [%p] [%p] [%p].",
765 : resourceArgs.buffersIn[RANK_ZERO], resourceArgs.buffersIn[RANK_ONE], resourceArgs.buffersIn[RANK_TWO],
766 : resourceArgs.buffersIn[RANK_THREE], resourceArgs.buffersIn[RANK_FOUR], resourceArgs.buffersIn[RANK_FIVE],
767 : resourceArgs.buffersIn[RANK_SIX], resourceArgs.buffersIn[RANK_SEVEN], resourceArgs.buffersOut[RANK_ZERO],
768 : resourceArgs.buffersOut[RANK_ONE], resourceArgs.buffersOut[RANK_TWO], resourceArgs.buffersOut[RANK_THREE],
769 : resourceArgs.buffersOut[RANK_FOUR], resourceArgs.buffersOut[RANK_FIVE], resourceArgs.buffersOut[RANK_SIX],
770 : resourceArgs.buffersOut[RANK_SEVEN]);
771 :
772 5 : KernelArgsType argsType = algArgs.argsType;
773 :
774 5 : HcclResult ret = HcclResult::HCCL_E_PARA;
775 : aclrtLaunchKernelCfg cfg;
776 : aclrtLaunchKernelAttr attr[AIV_ATTRNUM_THREE];
777 5 : attr[0].id = ACL_RT_LAUNCH_KERNEL_ATTR_SCHEM_MODE;
778 5 : attr[0].value.schemMode = 1;
779 5 : attr[1].id = ACL_RT_LAUNCH_KERNEL_ATTR_TIMEOUT_US;
780 5 : attr[1].value.timeoutUs.timeoutLow = GetAivTimeout(algArgs.execTimeOut, algArgs.execTimeOutSet);
781 5 : attr[1].value.timeoutUs.timeoutHigh = 0;
782 5 : attr[2].id = ACL_RT_LAUNCH_KERNEL_ATTR_ENGINE_TYPE;
783 5 : attr[2].value.engineType = ACL_RT_ENGINE_TYPE_AIV;
784 5 : cfg.numAttrs = AIV_ATTRNUM_THREE;
785 5 : cfg.attrs = attr;
786 :
787 : aclrtFuncHandle funcHandle;
788 5 : s8* stubFunc = GetStubFunc(opArgs.cmdType, opArgs.dataType, argsType);
789 5 : ret = GetKernelFunc(funcHandle, stubFunc);
790 5 : CHK_PRT_RET(
791 : ret != HCCL_SUCCESS,
792 : HCCL_ERROR(
793 : "[ExecuteKernelLaunchInner] errNo[0x%016llx] GetKernelFunc failed, "
794 : "return[%d]",
795 : HCCL_ERROR_CODE(HCCL_E_RUNTIME), ret),
796 : HCCL_E_RUNTIME);
797 :
798 10 : aclError aclRet = aclrtLaunchKernelWithHostArgs(
799 5 : funcHandle, resourceArgs.numBlocks, resourceArgs.stream, &cfg, args, argsSize, nullptr, 0);
800 5 : if (aclRet == ACL_ERROR_RT_INVALID_HANDLE) {
801 : aclError aclGetRet
802 0 : = aclrtBinaryGetFunction(g_binHandleMap[opArgs.cmdType], g_aivNameMap[stubFunc].c_str(), &funcHandle);
803 0 : CHK_PRT_RET(
804 : aclGetRet != ACL_SUCCESS,
805 : HCCL_ERROR("[RegisterBinaryKernel]errNo[0x%016llx] get function from binary error.", aclRet),
806 : HCCL_E_NOT_FOUND);
807 0 : aclRet = aclrtLaunchKernelWithHostArgs(
808 0 : funcHandle, resourceArgs.numBlocks, resourceArgs.stream, &cfg, args, argsSize, nullptr, 0);
809 : }
810 5 : CHK_PRT_RET(
811 : aclRet != ACL_SUCCESS,
812 : HCCL_ERROR(
813 : "[ExecuteKernelLaunchInner]errNo[0x%016llx] aclrtLaunchKernelWithHostArgs error[%d].",
814 : HCCL_ERROR_CODE(HCCL_E_RUNTIME), aclRet),
815 : HCCL_E_RUNTIME);
816 :
817 5 : CHK_RET(CacheTaskOpInfo(resourceArgs.stream, topoArgs.identify));
818 :
819 5 : TaskAivProfilerWrap(
820 : opArgs, topoArgs, resourceArgs, algArgs, aivProfilingInfo,
821 5 : (algArgs.argsType != KernelArgsType::ARGS_TYPE_SERVER) ? resourceArgs.buffersOut[0] :
822 5 : resourceArgs.buffersOut[topoArgs.rank]);
823 :
824 5 : CHK_PRT_RET(
825 : ret != HCCL_SUCCESS,
826 : HCCL_ERROR(
827 : "[AIV][ExecuteKernelLaunchInner] errNo[0x%016llx] rtKernelLaunch aiv fail, "
828 : "return[%d]",
829 : HCCL_ERROR_CODE(HCCL_E_RUNTIME), ret),
830 : HCCL_E_RUNTIME);
831 5 : return HCCL_SUCCESS;
832 : }
833 :
834 : // Kernel单次调用Launch外部接口
835 0 : HcclResult ExecuteKernelLaunch(
836 : const AivOpArgs& opArgs, const AivTopoArgs& topoArgs, const AivResourceArgs& resourceArgs,
837 : const AivAlgArgs& algArgs, AivProfilingInfo& aivProfilingInfo)
838 : {
839 0 : SetAivProfilingInfoBeginTime(aivProfilingInfo);
840 0 : CHK_PTR_NULL(resourceArgs.buffersIn);
841 0 : CHK_PTR_NULL(resourceArgs.buffersOut);
842 :
843 0 : if (algArgs.argsType == KernelArgsType::ARGS_TYPE_SIMPLE) {
844 : AivKernelArgsV3 aivKernelArgs{
845 0 : resourceArgs.buffersIn,
846 0 : resourceArgs.buffersOut,
847 0 : opArgs.input,
848 0 : opArgs.output,
849 0 : topoArgs.rank,
850 0 : topoArgs.rankSize,
851 0 : opArgs.count,
852 0 : opArgs.dataType,
853 0 : opArgs.op,
854 0 : opArgs.root,
855 0 : resourceArgs.aivTag,
856 0 : resourceArgs.numBlocks,
857 0 : opArgs.isOpBase,
858 0 : resourceArgs.bufferSize,
859 0 : algArgs.step,
860 0 : reinterpret_cast<void*>(aivProfilingInfo.counter.headCountMem),
861 0 : reinterpret_cast<void*>(aivProfilingInfo.counter.tailCountMem),
862 0 : reinterpret_cast<void*>(aivProfilingInfo.counter.addOneMem),
863 0 : aivProfilingInfo.counter.isEnableCounter,
864 0 : algArgs.deterministic};
865 0 : CHK_RET(ExecuteKernelLaunchInner(
866 : opArgs, topoArgs, resourceArgs, algArgs, &aivKernelArgs, sizeof(aivKernelArgs), aivProfilingInfo));
867 : } else {
868 : AivKernelArgs aivKernelArgs{
869 0 : resourceArgs.buffersIn,
870 0 : resourceArgs.buffersOut,
871 0 : opArgs.input,
872 0 : opArgs.output,
873 0 : topoArgs.rank,
874 0 : topoArgs.rankSize,
875 0 : opArgs.count,
876 0 : opArgs.dataType,
877 0 : opArgs.op,
878 0 : opArgs.root,
879 0 : resourceArgs.aivTag,
880 0 : resourceArgs.numBlocks,
881 0 : opArgs.isOpBase,
882 0 : resourceArgs.bufferSize,
883 0 : algArgs.step,
884 0 : algArgs.isSmallCount,
885 0 : topoArgs.serverNum,
886 0 : static_cast<u32>(topoArgs.devType),
887 0 : reinterpret_cast<void*>(aivProfilingInfo.counter.headCountMem),
888 0 : reinterpret_cast<void*>(aivProfilingInfo.counter.tailCountMem),
889 0 : reinterpret_cast<void*>(aivProfilingInfo.counter.addOneMem),
890 : aivProfilingInfo.counter.memSize,
891 0 : aivProfilingInfo.counter.isEnableCounter,
892 0 : algArgs.deterministic,
893 0 : algArgs.rmaInfo};
894 0 : CHK_RET(ExecuteKernelLaunchInner(
895 : opArgs, topoArgs, resourceArgs, algArgs, &aivKernelArgs, sizeof(aivKernelArgs), aivProfilingInfo));
896 : }
897 :
898 0 : return HCCL_SUCCESS;
899 : }
900 :
901 : // Kernel单次调用Launch外部接口
902 5 : HcclResult ExecuteKernelLaunch(
903 : const AivOpArgs& opArgs, const AivTopoArgs& topoArgs, const AivResourceArgs& resourceArgs,
904 : const AivAlgArgs& algArgs, const ExtraArgs& extraArgs, AivProfilingInfo& aivProfilingInfo)
905 : {
906 5 : SetAivProfilingInfoBeginTime(aivProfilingInfo);
907 5 : CHK_PTR_NULL(resourceArgs.buffersIn);
908 5 : CHK_PTR_NULL(resourceArgs.buffersOut);
909 :
910 : AivExtraKernelArgs aivExtraKernelArgs{
911 5 : resourceArgs.buffersIn,
912 5 : resourceArgs.buffersOut,
913 5 : opArgs.input,
914 5 : opArgs.output,
915 5 : topoArgs.rank,
916 5 : topoArgs.rankSize,
917 5 : opArgs.count,
918 5 : opArgs.dataType,
919 5 : opArgs.op,
920 5 : opArgs.root,
921 5 : resourceArgs.aivTag,
922 5 : resourceArgs.numBlocks,
923 5 : opArgs.isOpBase,
924 5 : resourceArgs.bufferSize,
925 5 : algArgs.step,
926 5 : algArgs.isSmallCount,
927 5 : topoArgs.serverNum,
928 5 : static_cast<u32>(topoArgs.devType),
929 5 : reinterpret_cast<void*>(aivProfilingInfo.counter.headCountMem),
930 5 : reinterpret_cast<void*>(aivProfilingInfo.counter.tailCountMem),
931 5 : reinterpret_cast<void*>(aivProfilingInfo.counter.addOneMem),
932 : aivProfilingInfo.counter.memSize,
933 5 : aivProfilingInfo.counter.isEnableCounter,
934 5 : algArgs.deterministic,
935 5 : algArgs.rmaInfo,
936 5 : &extraArgs};
937 5 : CHK_RET(ExecuteKernelLaunchInner(
938 : opArgs, topoArgs, resourceArgs, algArgs, &aivExtraKernelArgs, sizeof(aivExtraKernelArgs), aivProfilingInfo));
939 :
940 5 : return HCCL_SUCCESS;
941 : }
942 :
943 : // Kernel单次调用Launch外部接口
944 0 : HcclResult ExecuteKernelLaunch(
945 : const AivOpArgs& opArgs, const AivTopoArgs& topoArgs, const AivResourceArgs& resourceArgs,
946 : const AivAlgArgs& algArgs, const ExtraArgsV2& extraArgs, AivProfilingInfo& aivProfilingInfo)
947 : {
948 0 : SetAivProfilingInfoBeginTime(aivProfilingInfo);
949 0 : CHK_PTR_NULL(resourceArgs.buffersIn);
950 0 : CHK_PTR_NULL(resourceArgs.buffersOut);
951 :
952 : AivExtraKernelArgsV2 aivExtraKernelArgs{
953 0 : resourceArgs.buffersIn,
954 0 : resourceArgs.buffersOut,
955 0 : opArgs.input,
956 0 : opArgs.output,
957 0 : topoArgs.rank,
958 0 : topoArgs.rankSize,
959 0 : opArgs.count,
960 0 : opArgs.dataType,
961 0 : opArgs.op,
962 0 : opArgs.root,
963 0 : resourceArgs.aivTag,
964 0 : resourceArgs.numBlocks,
965 0 : opArgs.isOpBase,
966 0 : resourceArgs.bufferSize,
967 0 : algArgs.step,
968 0 : algArgs.isSmallCount,
969 0 : topoArgs.serverNum,
970 0 : static_cast<u32>(topoArgs.devType),
971 0 : reinterpret_cast<void*>(aivProfilingInfo.counter.headCountMem),
972 0 : reinterpret_cast<void*>(aivProfilingInfo.counter.tailCountMem),
973 0 : reinterpret_cast<void*>(aivProfilingInfo.counter.addOneMem),
974 : aivProfilingInfo.counter.memSize,
975 0 : aivProfilingInfo.counter.isEnableCounter,
976 0 : algArgs.deterministic,
977 0 : algArgs.rmaInfo,
978 0 : &extraArgs};
979 0 : CHK_RET(ExecuteKernelLaunchInner(
980 : opArgs, topoArgs, resourceArgs, algArgs, &aivExtraKernelArgs, sizeof(aivExtraKernelArgs), aivProfilingInfo));
981 :
982 0 : return HCCL_SUCCESS;
983 : }
984 :
985 : } // namespace hccl
|
