Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #ifndef HCCLV2_ADAPTER_RTS_H
12 : #define HCCLV2_ADAPTER_RTS_H
13 :
14 : #include <string>
15 : #include <unordered_map>
16 : #include "acl/acl_rt.h"
17 : #include "types.h"
18 : #include "const_val.h"
19 : #include "dev_type.h"
20 : #include "rt_external.h"
21 : #include "rt_external_kernel.h"
22 :
23 : namespace Hccl {
24 : #ifdef CCL_FWK_LLT
25 : typedef void *aclrtCntNotify;
26 : #define ACL_NOTIFY_DEFAULT 0x00000000U
27 : #define ACL_ERROR_RT_FEATURE_NOT_SUPPORT 207000 // feature not support
28 : #define ACL_NOTIFY_DEVICE_USE_ONLY 0x00000001U
29 : #endif
30 :
31 : using HcclRtStream = void*;
32 : using RtNotify_t = void*;
33 : using RtEvent_t = void*;
34 : using RtCntNotify_t = void*;
35 :
36 : constexpr u32 RTS_IPC_MEM_NAME_LEN = 65;
37 : constexpr u32 CHIP_VERSION_MAX_LEN = 32;
38 : #ifdef __cplusplus
39 : extern "C" {
40 : #endif
41 :
42 : typedef enum tagRtMemcpyKind {
43 : RT_MEMCPY_HOST_TO_HOST = 0, // host to host
44 : RT_MEMCPY_HOST_TO_DEVICE, // host to device
45 : RT_MEMCPY_DEVICE_TO_HOST, // device to host
46 : RT_MEMCPY_DEVICE_TO_DEVICE, // device to device, 1P && P2P
47 : RT_MEMCPY_MANAGED, // managed memory
48 : RT_MEMCPY_ADDR_DEVICE_TO_DEVICE,
49 : RT_MEMCPY_HOST_TO_DEVICE_EX, // host to device ex (only used for 8 bytes)
50 : RT_MEMCPY_DEVICE_TO_HOST_EX, // device to host ex
51 : RT_MEMCPY_DEFAULT, // auto infer copy dir
52 : RT_MEMCPY_RESERVED,
53 : } rtMemcpyKind_t;
54 : typedef enum rtKernelType {
55 : KERNEL_TYPE_CCE = 0,
56 : KERNEL_TYPE_FWK = 1,
57 : KERNEL_TYPE_AICPU = 2,
58 : KERNEL_TYPE_AICPU_CUSTOM = 4,
59 : KERNEL_TYPE_AICPU_KFC = 5,
60 : KERNEL_TYPE_CUSTOM_KFC = 6,
61 : KERNEL_TYPE_HWTS = 10,
62 : KERNEL_TYPE_RESERVED = 99,
63 : } rtKernelType_t;
64 :
65 : typedef struct tagRtCcuTaskGroup {
66 : uint32_t taskNum;
67 : rtCcuTaskInfo_t ccuTaskInfo[FUSION_SUB_TASK_MAX_CCU_NUM];
68 : } rtCcuTaskGroup_t;
69 :
70 : typedef struct tagRtDevBinary {
71 : uint32_t magic; // magic number
72 : uint32_t version; // version of binary
73 : const void *data; // binary data
74 : uint64_t length; // binary length
75 : } rtDevBinary_t;
76 : /* 3-8包不支持的接口
77 : * aclrtCntNotifyWaitWithTimeout —— rtsCntNotifyWaitWithTimeout
78 : * aclrtCntNotifyRecord —— rtsCntNotifyRecord
79 : * aclrtCntNotifyDestroy —— rtCntNotifyDestroy
80 : * aclrtCntNotifyCreate —— rtCntNotifyCreateServer
81 : * aclrtGetPhyDevIdByLogicDevId —— rtsGetPhyDevIdByLogicDevId
82 : * aclrtSetDeviceTaskAbortCallback —— rtsSetDeviceTaskAbortCallback
83 : * aclrtCntNotifyGetId —— rtsCntNotifyGetId
84 : * aclrtMallocWithCfg —— rtsMalloc
85 : */
86 : using aclrtMemType_t = int;
87 : __attribute__((weak)) ACL_FUNC_VISIBILITY aclError aclrtMemP2PMap(void *devPtr, size_t size, int32_t dstDevId, uint64_t flags);
88 : HcclResult HrtResetXpuDevice(uint32_t devType, const uint32_t devId);
89 : HcclResult HrtSetXpuDevice(uint32_t devType, const uint32_t devId);
90 : extern rtError_t rtCCULaunch(rtCcuTaskInfo_t *taskInfo, rtStream_t const stm);
91 : extern rtError_t rtReleaseDevResAddress(rtDevResInfo * const resInfo);
92 : extern rtError_t rtGetDevResAddress(rtDevResInfo * const resInfo, rtDevResAddrInfo * const addrInfo);
93 : extern rtError_t rtUbDevQueryInfo(rtUbDevQueryCmd cmd, void *devInfo);
94 : #ifdef __cplusplus
95 : }
96 : #endif
97 : struct MsprofHcclInfo {
98 : uint64_t itemId;
99 : uint64_t cclTag;
100 : uint64_t groupName;
101 : uint32_t localRank;
102 : uint32_t remoteRank;
103 : uint32_t rankSize;
104 : uint32_t workFlowMode;
105 : uint32_t planeID;
106 : uint32_t ctxId;
107 : uint64_t notifyID;
108 : uint32_t stage;
109 : uint32_t role; // role {0: dst, 1:src}
110 : double durationEstimated;
111 : uint64_t srcAddr;
112 : uint64_t dstAddr;
113 : uint64_t dataSize; // bytes
114 : uint32_t opType; // {0: sum, 1: mul, 2: max, 3: min}
115 : uint32_t dataType; // data type {0: INT8, 1: INT16, 2: INT32, 3: FP16, 4:FP32, 5:INT64, 6:UINT64}
116 : uint32_t linkType; // link type {0: 'OnChip', 1: 'HCCS', 2: 'PCIe', 3: 'RoCE'}
117 : uint32_t transportType; // transport type {0: SDMA, 1: RDMA, 2:LOCAL}
118 : uint32_t rdmaType; // RDMA type {0: RDMASendNotify, 1:RDMASendPayload}
119 : uint32_t reserve2;
120 : #ifdef __cplusplus
121 16 : MsprofHcclInfo() : role(0xFFFFFFFF), srcAddr(0xFFFFFFFF), dstAddr(0xFFFFFFFF),
122 16 : dataSize(0), opType(0xFFFFFFFF),
123 16 : dataType(0xFFFFFFFF), linkType(0xFFFFFFFF),
124 16 : transportType(0xFFFFFFFF), rdmaType(0xFFFFFFFF)
125 : {
126 16 : }
127 : #endif
128 : };
129 :
130 : struct MsprofDpuHcclTrack {
131 : uint64_t itemId;
132 : uint64_t cclTag;
133 : uint64_t groupName;
134 : uint32_t localRank;
135 : uint32_t remoteRank;
136 : uint32_t rankSize;
137 : uint32_t stage;
138 : uint64_t notifyID;
139 : uint64_t timeStamp;
140 : double durationEstimated;
141 : uint64_t srcAddr;
142 : uint64_t dstAddr;
143 : uint64_t dataSize; // bytes
144 : uint32_t taskId;
145 : uint32_t aicpu_task_id;
146 : uint16_t streamId;
147 : uint16_t planeID;
148 : uint16_t npuDevId;
149 : uint16_t dpuDevId;
150 : uint8_t opType; // {0: sum, 1: mul, 2: max, 3: min}
151 : uint8_t dataType; // data type {0: INT8, 1: INT16, 2: INT32, 3: FP16, 4:FP32, 5:INT64, 6:UINT64}
152 : uint8_t linkType; // link type {0: 'OnChip', 1: 'HCCS', 2: 'PCIe', 3: 'RoCE'}
153 : uint8_t transportType; // transport type {0: SDMA, 1: RDMA, 2:LOCAL}
154 : uint8_t rdmaType; // RDMA type {0: RDMASendNotify, 1:RDMASendPayload}
155 : uint8_t role; // role {0: dst, 1:src}
156 : uint8_t workFlowMode;
157 : uint8_t reserves[1];
158 :
159 : #ifdef __cplusplus
160 9 : MsprofDpuHcclTrack() :
161 9 : itemId(0),
162 9 : cclTag(0),
163 9 : groupName(0),
164 9 : localRank(0),
165 9 : remoteRank(0),
166 9 : rankSize(0),
167 9 : stage(0),
168 9 : notifyID(0),
169 9 : timeStamp(0),
170 9 : durationEstimated(0),
171 9 : srcAddr(0xFFFFFFFF),
172 9 : dstAddr(0xFFFFFFFF),
173 9 : dataSize(0),
174 9 : taskId(0),
175 9 : aicpu_task_id(0xFFFFFFFF),
176 9 : streamId(0),
177 9 : planeID(0),
178 9 : npuDevId(0xFFFF),
179 9 : dpuDevId(0xFFFF),
180 9 : opType(0xFF),
181 9 : dataType(0xFF),
182 9 : linkType(0xFF),
183 9 : transportType(0xFF),
184 9 : rdmaType(0xFF),
185 9 : role(0xFF),
186 9 : workFlowMode(0),
187 9 : reserves{0}
188 : {
189 9 : }
190 : #endif
191 : };
192 : struct ProfilingDeviceCommResInfo {
193 : uint64_t groupName; // 通信域
194 : uint32_t rankSize; // 通信域内rank总数
195 : uint32_t rankId; // 当前device rankId，通信域内编号
196 : uint32_t usrRankId; // 当前device rankId，全局编号
197 : uint32_t aicpuKfcStreamId; // MC2中launch aicpu kfc算子的stream
198 : uint32_t commStreamSize; // 当前device侧使用的通信stream数量
199 : uint32_t commStreamIds[8]; // 具体streamId
200 : uint32_t reserve;
201 : };
202 : struct rtMemUbTokenInfo{
203 : uint64_t va;
204 : uint64_t size;
205 : uint32_t tokenId;
206 : uint32_t tokenValue;
207 : };
208 : constexpr uint32_t RT_NOTIFY_FLAG_DOWNLOAD_TO_DEV = 0x02U; // RT_NOTIFY_FLAG_DOWNLOAD_TO_DEV does not support OR with other flags
209 : constexpr uint64_t RT_NOTIFY_FLAG_DEFAULT = 0x00U;
210 : enum class HcclRtMemcpyKind {
211 : HCCL_RT_MEMCPY_KIND_HOST_TO_HOST = 0, /**< host to host */
212 : HCCL_RT_MEMCPY_KIND_HOST_TO_DEVICE, /**< host to device */
213 : HCCL_RT_MEMCPY_KIND_DEVICE_TO_HOST, /**< device to host */
214 : HCCL_RT_MEMCPY_KIND_DEVICE_TO_DEVICE, /**< device to device */
215 : HCCL_RT_MEMCPY_ADDR_DEVICE_TO_DEVICE, /**< Level-2 address copy, device to device */
216 : HCCL_RT_MEMCPY_KIND_RESERVED,
217 : };
218 : DevId HrtGetDevicePhyIdByIndex(s32 deviceLogicId);
219 : DevType HrtGetDeviceType();
220 : s32 HrtDeviceGetBareTgid();
221 : void HrtGetSocVer(std::string &socName);
222 : s32 HrtGetDevice();
223 : // 非主线程使用rts添加task情况下，需要先使用该函数通知RTS，将线程和 device logic id绑定
224 : void HrtSetDevice(s32 deviceLogicId);
225 : void HrtResetDevice(s32 deviceLogicId);
226 : u32 HrtGetDeviceCount();
227 : HcclResult HrtGetDeviceInfo(uint32_t deviceLogicId, int32_t moduleType, aclrtDevAttr infoType, int64_t &val);
228 : HcclResult HrtGetMainboardId(uint32_t deviceLogicId, HcclMainboardId &hcclMainboardId);
229 : aclrtStream HrtStreamCreateWithFlags(uint32_t priority, uint32_t flag);
230 : void HrtStreamDestroy(aclrtStream ptr);
231 : void HrtStreamSetMode(HcclRtStream streamPtr, const uint64_t stmMode);
232 : u64 HrtStreamGetMode(HcclRtStream const ptr);
233 : void HcclStreamSynchronize(HcclRtStream ptr);
234 : s32 HrtGetStreamId(aclrtStream ptr);
235 : void HrtStreamActive(aclrtStream activeStream, aclrtStream stream);
236 :
237 : void *HrtMalloc(u64 size, aclrtMemType_t memType);
238 : void HrtFree(void *devPtr);
239 : void HrtMemcpy(void *dst, uint64_t destMax, const void *src, uint64_t count, rtMemcpyKind_t kind);
240 : void HrtMemset(void *dst, uint64_t destMax, uint64_t count);
241 : void HrtMemsetV2(void *dst, size_t destMax, int32_t value, size_t count);
242 : void HrtIpcSetMemoryName(void *ptr, char_t *name, u64 ptrMaxLen, u32 nameMaxLen);
243 : void HrtIpcDestroyMemoryName(const char_t *name);
244 : void *HrtIpcOpenMemory(const char_t *name);
245 : void HrtIpcCloseMemory(const void *ptr);
246 : void HrtIpcSetMemoryPid(const char_t *name, int pid);
247 : aclrtPtrAttributes HrtPointerGetAttributes(const void *ptr);
248 : void PrintMemoryAttr(const void *memAddr);
249 : void HrtDevMemAlignWithPage(void *ptr, u64 size, void *&ipcPtr, u64 &ipcSize, u64 &ipcOff);
250 : HcclResult HrtMemPrefetchToDevice(void *devPtr, uint64_t len);
251 :
252 : void *HrtMallocHost(u64 size);
253 : void HrtFreeHost(void *hostPtr);
254 :
255 : // rts notify manager api
256 : aclrtNotify HrtNotifyCreate(s32 deviceLogicId);
257 : aclrtNotify HrtNotifyCreateWithFlag(u32 devId, u32 flag);
258 : void HrtNotifyDestroy(RtNotify_t ptr);
259 : void HrtIpcSetNotifyName(RtNotify_t ptr, char_t *name, uint32_t len);
260 :
261 : u32 HrtGetNotifyID(RtNotify_t notifyHandle);
262 : u64 HrtNotifyGetAddr(RtNotify_t notifyHandle);
263 : void HrtSetIpcNotifyPid(aclrtNotify notify, int32_t pid);
264 : RtNotify_t HrtIpcOpenNotify(const char_t *name);
265 : RtNotify_t HrtIpcOpenNotifyWithFlag(const char_t *name, uint32_t flags);
266 : u32 HrtNotifyGetOffset(RtNotify_t ptr);
267 :
268 : // rts notify task api
269 : void HrtNotifyWaitWithTimeOut(RtNotify_t notifyPtr, aclrtStream streamPtr, uint32_t timeOut);
270 : void HrtNotifyRecord(RtNotify_t notifyPtr, aclrtStream streamPtr);
271 :
272 : // rts memcpy task api
273 : void HrtMemAsyncCopy(void *dst, uint64_t destMax, const void *src, uint64_t count, aclrtMemcpyKind kind,
274 : aclrtStream streamPtr);
275 :
276 : // rts reduce task api
277 : void HrtReduceAsync(void *dst, uint64_t destMax, const void *src, uint64_t count, aclrtReduceKind kind,
278 : aclDataType type, aclrtStream streamPtr);
279 :
280 : // rts rdma task
281 : void HrtRDMASend(u32 qpn, u32 wqeIndex, aclrtStream streamPtr); // 910A offload
282 : void HrtRDMADBSend(uint32_t dbindex, uint64_t dbinfo,
283 : aclrtStream streamPtr); // 910A opbase and 910A2/910A3
284 : void HrtAicpuLaunchKernelWithHostArgs(aclrtFuncHandle funcHandle, uint32_t numBlocks, aclrtStream stream,
285 : aclrtLaunchKernelCfg *cfg, void *hostArgs, size_t argsSize,
286 : aclrtPlaceHolderInfo *placeHolderArray = nullptr, size_t placeHolderNum = 0);
287 :
288 : // rts task exception api
289 : void HrtRegTaskFailCallbackByModule(aclrtExceptionInfoCallback callback);
290 :
291 : // 添加任一task后可获取得到 taskId, streamId
292 : void HrtGetTaskIdAndStreamID(u32 &taskId, u32 &streamId);
293 : u64 HrtGetRdmaDoorbellAddr(s32 deviceLogicId, u32 dbIndex);
294 : u32 HrtStreamGetSqId(const aclrtStream ptr);
295 : u32 HrtStreamGetCqId(const aclrtStream ptr);
296 :
297 : // 对rts结构体打桩，联调用，待RTS接口上线后，删除掉
298 : struct HrtUbDbDetailInfo {
299 : u16 functionId;
300 : u16 dieId;
301 : u16 rsv;
302 : u16 jettyId;
303 : u16 piValue;
304 : };
305 :
306 : struct HrtUbDbInfo {
307 : u8 dbNum;
308 : u8 wrCqe;
309 : HrtUbDbDetailInfo info[2];
310 : };
311 :
312 : struct HrtUbWqeInfo {
313 : u16 wrCqe;
314 : u16 functionId;
315 : u16 dieId;
316 : u16 wqeSize;
317 : u16 jettyId;
318 : u8 *wqe;
319 : u16 wqePtrLen;
320 : };
321 :
322 : constexpr u32 DWQE_SIZE_64 = 64;
323 : constexpr u32 DWQE_SIZE_128 = 128;
324 :
325 : void HrtUbDbSend(const HrtUbDbInfo &info, aclrtStream streamPtr);
326 :
327 : void HrtUbDirectSend(const HrtUbWqeInfo &info, aclrtStream streamPtr);
328 :
329 : aclrtCntNotify HrtCntNotifyCreate(u32 deviceId);
330 :
331 : u32 HrtGetCntNotifyId(const aclrtCntNotify inCntNotify);
332 :
333 : void HrtCntNotifyDestroy(const aclrtCntNotify inCntNotify);
334 :
335 66 : MAKE_ENUM(HrtCntNotifyRecordMode, WRITE_BIT, STORE)
336 : void HrtCntNotifyRecord(const aclrtCntNotify inCntNotify, const aclrtStream streamPtr, HrtCntNotifyRecordMode mode, u32 value);
337 71 : MAKE_ENUM(HrtCntNotifyWaitMode, EQUAL, BITMAP)
338 : void HrtCntNotifyWaitWithTimeOut(const aclrtCntNotify inCntNotify, const aclrtStream streamPtr, HrtCntNotifyWaitMode mode, u32 value,
339 : u32 timeout, bool isClear = true);
340 :
341 : void HrtCcuLaunch(rtCcuTaskInfo_t &taskInfo, aclrtStream const streamPtr);
342 : void HrtUbDevQueryInfo(rtUbDevQueryCmd cmd, void *devInfo);
343 : // pair<tokendId, tokenValue>
344 : std::pair<u32, u32> HrtUbDevQueryToken(u64 addr, u64 size);
345 922 : MAKE_ENUM(HrtDevResProcType, PROCESS_CP1, PROCESS_HCCP)
346 1058 : MAKE_ENUM(HrtDevResType, RES_TYPE_STARS_NOTIFY_RECORD, RES_TYPE_CCU_CKE, RES_TYPE_CCU_XN,
347 : RES_TYPE_STARS_CNT_NOTIFY_BIT_WR)
348 : #define HRT_DEV_RES_FLAG_USE_UNIQUE_VA (1U << 7U) // bit0, map to unified va, for RT_RES_TYPE_STARS_NOTIFY_RECORD
349 : struct HrtDevResInfo {
350 : u32 dieId{0}; // for ccu res need set devId, for others set 0
351 : HrtDevResProcType procType{HrtDevResProcType::PROCESS_CP1};
352 : HrtDevResType resType{HrtDevResType::RES_TYPE_STARS_NOTIFY_RECORD};
353 : u32 resId{0};
354 : u32 flag{0};
355 : };
356 :
357 : struct HrtDevResAddrInfo {
358 : u64 address{0};
359 : u32 len{0};
360 : };
361 :
362 : HrtDevResAddrInfo HrtGetDevResAddress(const HrtDevResInfo &devResInfo);
363 : void HrtReleaseDevResAddress(const HrtDevResInfo &devResInfo);
364 :
365 6 : MAKE_ENUM(HrtEventStatus, EVENT_INIT, EVENT_RECORDED)
366 : aclrtEvent HrtEventCreateWithFlag(u32 flag);
367 : void HrtEventDestroy(RtEvent_t eventPtr);
368 : void HrtEventRecord(RtEvent_t eventPtr, aclrtStream streamPtr);
369 : HrtEventStatus HrtEventQueryStatus(RtEvent_t eventPtr);
370 :
371 : void HrtWriteValue(u64 addr, u32 piVal, const aclrtStream streamPtr);
372 : void HrtDeviceAbortRegCallBack(aclrtDeviceTaskAbortCallback callback, void *args, const std::string& name);
373 : HcclResult HrtEnableP2P(u32 deviceLogicId, u32 devicePhyId);
374 : HcclResult HrtDisableP2P(u32 deviceLogicId, u32 devicePhyId);
375 : HcclResult HrtGetP2PStatus(u32 deviceLogicId, u32 devicePhyId, uint32_t *status);
376 : } // namespace Hccl
377 :
378 : #endif // HCCL_ADAPTER_RTS_H
|
