Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #ifndef HCCLV2_ADAPTER_RTS_H
12 : #define HCCLV2_ADAPTER_RTS_H
13 :
14 : #include <string>
15 : #include <unordered_map>
16 : #include "acl/acl_rt.h"
17 : #include "types.h"
18 : #include "const_val.h"
19 : #include "dev_type.h"
20 : #include "rt_external.h"
21 : #include "rt_external_kernel.h"
22 :
23 : namespace Hccl {
24 : #ifdef CCL_FWK_LLT
25 : typedef void *aclrtCntNotify;
26 : #define ACL_NOTIFY_DEFAULT 0x00000000U
27 : #define ACL_ERROR_RT_FEATURE_NOT_SUPPORT 207000 // feature not support
28 : #define ACL_NOTIFY_DEVICE_USE_ONLY 0x00000001U
29 : #endif
30 :
31 : using HcclRtStream = void*;
32 : using RtNotify_t = void*;
33 : using RtEvent_t = void*;
34 : using RtCntNotify_t = void*;
35 :
36 : constexpr u32 RTS_IPC_MEM_NAME_LEN = 65;
37 : constexpr u32 CHIP_VERSION_MAX_LEN = 32;
38 : #ifdef __cplusplus
39 : extern "C" {
40 : #endif
41 :
42 : typedef enum tagRtMemcpyKind {
43 : RT_MEMCPY_HOST_TO_HOST = 0, // host to host
44 : RT_MEMCPY_HOST_TO_DEVICE, // host to device
45 : RT_MEMCPY_DEVICE_TO_HOST, // device to host
46 : RT_MEMCPY_DEVICE_TO_DEVICE, // device to device, 1P && P2P
47 : RT_MEMCPY_MANAGED, // managed memory
48 : RT_MEMCPY_ADDR_DEVICE_TO_DEVICE,
49 : RT_MEMCPY_HOST_TO_DEVICE_EX, // host to device ex (only used for 8 bytes)
50 : RT_MEMCPY_DEVICE_TO_HOST_EX, // device to host ex
51 : RT_MEMCPY_DEFAULT, // auto infer copy dir
52 : RT_MEMCPY_RESERVED,
53 : } rtMemcpyKind_t;
54 : typedef enum rtKernelType {
55 : KERNEL_TYPE_CCE = 0,
56 : KERNEL_TYPE_FWK = 1,
57 : KERNEL_TYPE_AICPU = 2,
58 : KERNEL_TYPE_AICPU_CUSTOM = 4,
59 : KERNEL_TYPE_AICPU_KFC = 5,
60 : KERNEL_TYPE_CUSTOM_KFC = 6,
61 : KERNEL_TYPE_HWTS = 10,
62 : KERNEL_TYPE_RESERVED = 99,
63 : } rtKernelType_t;
64 :
65 : typedef struct tagRtCcuTaskGroup {
66 : uint32_t taskNum;
67 : rtCcuTaskInfo_t ccuTaskInfo[FUSION_SUB_TASK_MAX_CCU_NUM];
68 : } rtCcuTaskGroup_t;
69 :
70 : typedef struct tagRtDevBinary {
71 : uint32_t magic; // magic number
72 : uint32_t version; // version of binary
73 : const void *data; // binary data
74 : uint64_t length; // binary length
75 : } rtDevBinary_t;
76 : /* 3-8包不支持的接口
77 : * aclrtCntNotifyWaitWithTimeout —— rtsCntNotifyWaitWithTimeout
78 : * aclrtCntNotifyRecord —— rtsCntNotifyRecord
79 : * aclrtCntNotifyDestroy —— rtCntNotifyDestroy
80 : * aclrtCntNotifyCreate —— rtCntNotifyCreateServer
81 : * aclrtGetPhyDevIdByLogicDevId —— rtsGetPhyDevIdByLogicDevId
82 : * aclrtSetDeviceTaskAbortCallback —— rtsSetDeviceTaskAbortCallback
83 : * aclrtCntNotifyGetId —— rtsCntNotifyGetId
84 : * aclrtMallocWithCfg —— rtsMalloc
85 : */
86 : using aclrtMemType_t = int;
87 : __attribute__((weak)) ACL_FUNC_VISIBILITY aclError aclrtMemP2PMap(void *devPtr, size_t size, int32_t dstDevId, uint64_t flags);
88 : HcclResult HrtResetXpuDevice(uint32_t devType, const uint32_t devId);
89 : HcclResult HrtSetXpuDevice(uint32_t devType, const uint32_t devId);
90 : extern rtError_t rtCCULaunch(rtCcuTaskInfo_t *taskInfo, rtStream_t const stm);
91 : extern rtError_t rtReleaseDevResAddress(rtDevResInfo * const resInfo);
92 : extern rtError_t rtGetDevResAddress(rtDevResInfo * const resInfo, rtDevResAddrInfo * const addrInfo);
93 : extern rtError_t rtUbDevQueryInfo(rtUbDevQueryCmd cmd, void *devInfo);
94 : #ifdef __cplusplus
95 : }
96 : #endif
97 : struct MsprofHcclInfo {
98 : uint64_t itemId;
99 : uint64_t cclTag;
100 : uint64_t groupName;
101 : uint32_t localRank;
102 : uint32_t remoteRank;
103 : uint32_t rankSize;
104 : uint32_t workFlowMode;
105 : uint32_t planeID;
106 : uint32_t ctxId;
107 : uint64_t notifyID;
108 : uint32_t stage;
109 : uint32_t role; // role {0: dst, 1:src}
110 : double durationEstimated;
111 : uint64_t srcAddr;
112 : uint64_t dstAddr;
113 : uint64_t dataSize; // bytes
114 : uint32_t opType; // {0: sum, 1: mul, 2: max, 3: min}
115 : uint32_t dataType; // data type {0: INT8, 1: INT16, 2: INT32, 3: FP16, 4:FP32, 5:INT64, 6:UINT64}
116 : uint32_t linkType; // link type {0: 'OnChip', 1: 'HCCS', 2: 'PCIe', 3: 'RoCE'}
117 : uint32_t transportType; // transport type {0: SDMA, 1: RDMA, 2:LOCAL}
118 : uint32_t rdmaType; // RDMA type {0: RDMASendNotify, 1:RDMASendPayload}
119 : uint32_t reserve2;
120 : #ifdef __cplusplus
121 16 : MsprofHcclInfo() : role(0xFFFFFFFF), srcAddr(0xFFFFFFFF), dstAddr(0xFFFFFFFF),
122 16 : dataSize(0), opType(0xFFFFFFFF),
123 16 : dataType(0xFFFFFFFF), linkType(0xFFFFFFFF),
124 16 : transportType(0xFFFFFFFF), rdmaType(0xFFFFFFFF)
125 : {
126 16 : }
127 : #endif
128 : };
129 :
130 : struct MsprofDpuHcclTrack {
131 : uint64_t itemId;
132 : uint64_t cclTag;
133 : uint64_t groupName;
134 : uint32_t localRank;
135 : uint32_t remoteRank;
136 : uint32_t rankSize;
137 : uint32_t stage;
138 : uint64_t notifyID;
139 : uint64_t timeStamp;
140 : double durationEstimated;
141 : uint64_t srcAddr;
142 : uint64_t dstAddr;
143 : uint64_t dataSize; // bytes
144 : uint32_t taskId;
145 : uint32_t aicpu_task_id;
146 : uint16_t streamId;
147 : uint16_t planeID;
148 : uint16_t npuDevId;
149 : uint16_t dpuDevId;
150 : uint8_t opType; // {0: sum, 1: mul, 2: max, 3: min}
151 : uint8_t dataType; // data type {0: INT8, 1: INT16, 2: INT32, 3: FP16, 4:FP32, 5:INT64, 6:UINT64}
152 : uint8_t linkType; // link type {0: 'OnChip', 1: 'HCCS', 2: 'PCIe', 3: 'RoCE'}
153 : uint8_t transportType; // transport type {0: SDMA, 1: RDMA, 2:LOCAL}
154 : uint8_t rdmaType; // RDMA type {0: RDMASendNotify, 1:RDMASendPayload}
155 : uint8_t role; // role {0: dst, 1:src}
156 : uint8_t workFlowMode;
157 : uint8_t reserves[1];
158 :
159 : #ifdef __cplusplus
160 9 : MsprofDpuHcclTrack() :
161 9 : itemId(0),
162 9 : cclTag(0),
163 9 : groupName(0),
164 9 : localRank(0),
165 9 : remoteRank(0),
166 9 : rankSize(0),
167 9 : stage(0),
168 9 : notifyID(0),
169 9 : timeStamp(0),
170 9 : durationEstimated(0),
171 9 : srcAddr(0xFFFFFFFF),
172 9 : dstAddr(0xFFFFFFFF),
173 9 : dataSize(0),
174 9 : taskId(0),
175 9 : aicpu_task_id(0xFFFFFFFF),
176 9 : streamId(0),
177 9 : planeID(0),
178 9 : npuDevId(0xFFFF),
179 9 : dpuDevId(0xFFFF),
180 9 : opType(0xFF),
181 9 : dataType(0xFF),
182 9 : linkType(0xFF),
183 9 : transportType(0xFF),
184 9 : rdmaType(0xFF),
185 9 : role(0xFF),
186 9 : workFlowMode(0),
187 9 : reserves{0}
188 : {
189 9 : }
190 : #endif
191 : };
192 : struct ProfilingDeviceCommResInfo {
193 : uint64_t groupName; // 通信域
194 : uint32_t rankSize; // 通信域内rank总数
195 : uint32_t rankId; // 当前device rankId，通信域内编号
196 : uint32_t usrRankId; // 当前device rankId，全局编号
197 : uint32_t aicpuKfcStreamId; // MC2中launch aicpu kfc算子的stream
198 : uint32_t commStreamSize; // 当前device侧使用的通信stream数量
199 : uint32_t commStreamIds[8]; // 具体streamId
200 : uint32_t reserve;
201 : };
202 : struct rtMemUbTokenInfo{
203 : uint64_t va;
204 : uint64_t size;
205 : uint32_t tokenId;
206 : uint32_t tokenValue;
207 : };
208 : constexpr uint32_t RT_NOTIFY_FLAG_DOWNLOAD_TO_DEV = 0x02U; // RT_NOTIFY_FLAG_DOWNLOAD_TO_DEV does not support OR with other flags
209 : constexpr uint64_t RT_NOTIFY_FLAG_DEFAULT = 0x00U;
210 : enum class HcclRtMemcpyKind {
211 : HCCL_RT_MEMCPY_KIND_HOST_TO_HOST = 0, /**< host to host */
212 : HCCL_RT_MEMCPY_KIND_HOST_TO_DEVICE, /**< host to device */
213 : HCCL_RT_MEMCPY_KIND_DEVICE_TO_HOST, /**< device to host */
214 : HCCL_RT_MEMCPY_KIND_DEVICE_TO_DEVICE, /**< device to device */
215 : HCCL_RT_MEMCPY_ADDR_DEVICE_TO_DEVICE, /**< Level-2 address copy, device to device */
216 : HCCL_RT_MEMCPY_KIND_RESERVED,
217 : };
218 : DevId HrtGetDevicePhyIdByIndex(s32 deviceLogicId);
219 : DevType HrtGetDeviceType();
220 : s32 HrtDeviceGetBareTgid();
221 : void HrtGetSocVer(std::string &socName);
222 : s32 HrtGetDevice();
223 : HcclResult HrtGetLogicDevIdByUserDevId(s32 userDevId, s32 &logicDevId);
224 : // 非主线程使用rts添加task情况下，需要先使用该函数通知RTS，将线程和 device logic id绑定
225 : void HrtSetDevice(s32 deviceLogicId);
226 : void HrtResetDevice(s32 deviceLogicId);
227 : u32 HrtGetDeviceCount();
228 : HcclResult HrtGetDeviceInfo(uint32_t deviceLogicId, int32_t moduleType, aclrtDevAttr infoType, int64_t &val);
229 : HcclResult HrtGetMainboardId(uint32_t deviceLogicId, HcclMainboardId &hcclMainboardId);
230 : aclrtStream HrtStreamCreateWithFlags(uint32_t priority, uint32_t flag);
231 : void HrtStreamDestroy(aclrtStream ptr);
232 : void HrtStreamSetMode(HcclRtStream streamPtr, const uint64_t stmMode);
233 : u64 HrtStreamGetMode(HcclRtStream const ptr);
234 : void HcclStreamSynchronize(HcclRtStream ptr);
235 : s32 HrtGetStreamId(aclrtStream ptr);
236 : void HrtStreamActive(aclrtStream activeStream, aclrtStream stream);
237 :
238 : void *HrtMalloc(u64 size, aclrtMemType_t memType);
239 : void HrtFree(void *devPtr);
240 : void HrtMemcpy(void *dst, uint64_t destMax, const void *src, uint64_t count, rtMemcpyKind_t kind);
241 : void HrtMemset(void *dst, uint64_t destMax, uint64_t count);
242 : void HrtMemsetV2(void *dst, size_t destMax, int32_t value, size_t count);
243 : void HrtIpcSetMemoryName(void *ptr, char_t *name, u64 ptrMaxLen, u32 nameMaxLen);
244 : void HrtIpcDestroyMemoryName(const char_t *name);
245 : void *HrtIpcOpenMemory(const char_t *name);
246 : void HrtIpcCloseMemory(const void *ptr);
247 : void HrtIpcSetMemoryPid(const char_t *name, int pid);
248 : aclrtPtrAttributes HrtPointerGetAttributes(const void *ptr);
249 : void PrintMemoryAttr(const void *memAddr);
250 : void HrtDevMemAlignWithPage(void *ptr, u64 size, void *&ipcPtr, u64 &ipcSize, u64 &ipcOff);
251 : HcclResult HrtMemPrefetchToDevice(void *devPtr, uint64_t len);
252 :
253 : void *HrtMallocHost(u64 size);
254 : void HrtFreeHost(void *hostPtr);
255 :
256 : // rts notify manager api
257 : aclrtNotify HrtNotifyCreate(s32 deviceLogicId);
258 : aclrtNotify HrtNotifyCreateWithFlag(u32 devId, u32 flag);
259 : void HrtNotifyDestroy(RtNotify_t ptr);
260 : void HrtIpcSetNotifyName(RtNotify_t ptr, char_t *name, uint32_t len);
261 :
262 : u32 HrtGetNotifyID(RtNotify_t notifyHandle);
263 : u64 HrtNotifyGetAddr(RtNotify_t notifyHandle);
264 : void HrtSetIpcNotifyPid(aclrtNotify notify, int32_t pid);
265 : RtNotify_t HrtIpcOpenNotify(const char_t *name);
266 : RtNotify_t HrtIpcOpenNotifyWithFlag(const char_t *name, uint32_t flags);
267 : u32 HrtNotifyGetOffset(RtNotify_t ptr);
268 :
269 : // rts notify task api
270 : void HrtNotifyWaitWithTimeOut(RtNotify_t notifyPtr, aclrtStream streamPtr, uint32_t timeOut);
271 : void HrtNotifyRecord(RtNotify_t notifyPtr, aclrtStream streamPtr);
272 :
273 : // rts memcpy task api
274 : void HrtMemAsyncCopy(void *dst, uint64_t destMax, const void *src, uint64_t count, aclrtMemcpyKind kind,
275 : aclrtStream streamPtr);
276 :
277 : // rts reduce task api
278 : void HrtReduceAsync(void *dst, uint64_t destMax, const void *src, uint64_t count, aclrtReduceKind kind,
279 : aclDataType type, aclrtStream streamPtr);
280 :
281 : // rts rdma task
282 : void HrtRDMASend(u32 qpn, u32 wqeIndex, aclrtStream streamPtr); // 910A offload
283 : void HrtRDMADBSend(uint32_t dbindex, uint64_t dbinfo,
284 : aclrtStream streamPtr); // 910A opbase and 910A2/910A3
285 : void HrtAicpuLaunchKernelWithHostArgs(aclrtFuncHandle funcHandle, uint32_t numBlocks, aclrtStream stream,
286 : aclrtLaunchKernelCfg *cfg, void *hostArgs, size_t argsSize,
287 : aclrtPlaceHolderInfo *placeHolderArray = nullptr, size_t placeHolderNum = 0);
288 :
289 : // rts task exception api
290 : void HrtRegTaskFailCallbackByModule(aclrtExceptionInfoCallback callback);
291 :
292 : // 添加任一task后可获取得到 taskId, streamId
293 : void HrtGetTaskIdAndStreamID(u32 &taskId, u32 &streamId);
294 : u64 HrtGetRdmaDoorbellAddr(s32 deviceLogicId, u32 dbIndex);
295 : u32 HrtStreamGetSqId(const aclrtStream ptr);
296 : u32 HrtStreamGetCqId(const aclrtStream ptr);
297 :
298 : // 对rts结构体打桩，联调用，待RTS接口上线后，删除掉
299 : struct HrtUbDbDetailInfo {
300 : u16 functionId;
301 : u16 dieId;
302 : u16 rsv;
303 : u16 jettyId;
304 : u16 piValue;
305 : };
306 :
307 : struct HrtUbDbInfo {
308 : u8 dbNum;
309 : u8 wrCqe;
310 : HrtUbDbDetailInfo info[2];
311 : };
312 :
313 : struct HrtUbWqeInfo {
314 : u16 wrCqe;
315 : u16 functionId;
316 : u16 dieId;
317 : u16 wqeSize;
318 : u16 jettyId;
319 : u8 *wqe;
320 : u16 wqePtrLen;
321 : };
322 :
323 : constexpr u32 DWQE_SIZE_64 = 64;
324 : constexpr u32 DWQE_SIZE_128 = 128;
325 :
326 : void HrtUbDbSend(const HrtUbDbInfo &info, aclrtStream streamPtr);
327 :
328 : void HrtUbDirectSend(const HrtUbWqeInfo &info, aclrtStream streamPtr);
329 :
330 : aclrtCntNotify HrtCntNotifyCreate(u32 deviceId);
331 :
332 : u32 HrtGetCntNotifyId(const aclrtCntNotify inCntNotify);
333 :
334 : void HrtCntNotifyDestroy(const aclrtCntNotify inCntNotify);
335 :
336 66 : MAKE_ENUM(HrtCntNotifyRecordMode, WRITE_BIT, STORE)
337 : void HrtCntNotifyRecord(const aclrtCntNotify inCntNotify, const aclrtStream streamPtr, HrtCntNotifyRecordMode mode, u32 value);
338 71 : MAKE_ENUM(HrtCntNotifyWaitMode, EQUAL, BITMAP)
339 : void HrtCntNotifyWaitWithTimeOut(const aclrtCntNotify inCntNotify, const aclrtStream streamPtr, HrtCntNotifyWaitMode mode, u32 value,
340 : u32 timeout, bool isClear = true);
341 :
342 : void HrtCcuLaunch(rtCcuTaskInfo_t &taskInfo, aclrtStream const streamPtr);
343 : void HrtUbDevQueryInfo(rtUbDevQueryCmd cmd, void *devInfo);
344 : // pair<tokendId, tokenValue>
345 : std::pair<u32, u32> HrtUbDevQueryToken(u64 addr, u64 size);
346 922 : MAKE_ENUM(HrtDevResProcType, PROCESS_CP1, PROCESS_HCCP)
347 1058 : MAKE_ENUM(HrtDevResType, RES_TYPE_STARS_NOTIFY_RECORD, RES_TYPE_CCU_CKE, RES_TYPE_CCU_XN,
348 : RES_TYPE_STARS_CNT_NOTIFY_BIT_WR)
349 : #define HRT_DEV_RES_FLAG_USE_UNIQUE_VA (1U << 7U) // bit0, map to unified va, for RT_RES_TYPE_STARS_NOTIFY_RECORD
350 : struct HrtDevResInfo {
351 : u32 dieId{0}; // for ccu res need set devId, for others set 0
352 : HrtDevResProcType procType{HrtDevResProcType::PROCESS_CP1};
353 : HrtDevResType resType{HrtDevResType::RES_TYPE_STARS_NOTIFY_RECORD};
354 : u32 resId{0};
355 : u32 flag{0};
356 : };
357 :
358 : struct HrtDevResAddrInfo {
359 : u64 address{0};
360 : u32 len{0};
361 : };
362 :
363 : HrtDevResAddrInfo HrtGetDevResAddress(const HrtDevResInfo &devResInfo);
364 : void HrtReleaseDevResAddress(const HrtDevResInfo &devResInfo);
365 :
366 6 : MAKE_ENUM(HrtEventStatus, EVENT_INIT, EVENT_RECORDED)
367 : aclrtEvent HrtEventCreateWithFlag(u32 flag);
368 : void HrtEventDestroy(RtEvent_t eventPtr);
369 : void HrtEventRecord(RtEvent_t eventPtr, aclrtStream streamPtr);
370 : HrtEventStatus HrtEventQueryStatus(RtEvent_t eventPtr);
371 :
372 : void HrtWriteValue(u64 addr, u32 piVal, const aclrtStream streamPtr);
373 : void HrtDeviceAbortRegCallBack(aclrtDeviceTaskAbortCallback callback, void *args, const std::string& name);
374 : HcclResult HrtEnableP2P(u32 deviceLogicId, u32 devicePhyId);
375 : HcclResult HrtDisableP2P(u32 deviceLogicId, u32 devicePhyId);
376 : HcclResult HrtGetP2PStatus(u32 deviceLogicId, u32 devicePhyId, uint32_t *status);
377 : } // namespace Hccl
378 :
379 : #endif // HCCL_ADAPTER_RTS_H
|
