Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #ifndef HCCLV2_ADAPTER_RTS_H
12 : #define HCCLV2_ADAPTER_RTS_H
13 :
14 : #include <string>
15 : #include <unordered_map>
16 : #include "acl/acl_rt.h"
17 : #include "types.h"
18 : #include "const_val.h"
19 : #include "dev_type.h"
20 : #include "rt_external.h"
21 : #include "rt_external_kernel.h"
22 :
23 : namespace Hccl {
24 : #ifdef CCL_FWK_LLT
25 : typedef void* aclrtCntNotify;
26 : #define ACL_NOTIFY_DEFAULT 0x00000000U
27 : #define ACL_ERROR_RT_FEATURE_NOT_SUPPORT 207000 // feature not support
28 : #define ACL_NOTIFY_DEVICE_USE_ONLY 0x00000001U
29 : #endif
30 :
31 : using HcclRtStream = void*;
32 : using RtNotify_t = void*;
33 : using RtEvent_t = void*;
34 : using RtCntNotify_t = void*;
35 :
36 : constexpr u32 RTS_IPC_MEM_NAME_LEN = 65;
37 : constexpr u32 CHIP_VERSION_MAX_LEN = 32;
38 : #ifdef __cplusplus
39 : extern "C" {
40 : #endif
41 :
42 : typedef enum tagRtMemcpyKind {
43 : RT_MEMCPY_HOST_TO_HOST = 0, // host to host
44 : RT_MEMCPY_HOST_TO_DEVICE, // host to device
45 : RT_MEMCPY_DEVICE_TO_HOST, // device to host
46 : RT_MEMCPY_DEVICE_TO_DEVICE, // device to device, 1P && P2P
47 : RT_MEMCPY_MANAGED, // managed memory
48 : RT_MEMCPY_ADDR_DEVICE_TO_DEVICE,
49 : RT_MEMCPY_HOST_TO_DEVICE_EX, // host to device ex (only used for 8 bytes)
50 : RT_MEMCPY_DEVICE_TO_HOST_EX, // device to host ex
51 : RT_MEMCPY_DEFAULT, // auto infer copy dir
52 : RT_MEMCPY_RESERVED,
53 : } rtMemcpyKind_t;
54 : typedef enum rtKernelType {
55 : KERNEL_TYPE_CCE = 0,
56 : KERNEL_TYPE_FWK = 1,
57 : KERNEL_TYPE_AICPU = 2,
58 : KERNEL_TYPE_AICPU_CUSTOM = 4,
59 : KERNEL_TYPE_AICPU_KFC = 5,
60 : KERNEL_TYPE_CUSTOM_KFC = 6,
61 : KERNEL_TYPE_HWTS = 10,
62 : KERNEL_TYPE_RESERVED = 99,
63 : } rtKernelType_t;
64 :
65 : typedef struct tagRtCcuTaskGroup {
66 : uint32_t taskNum;
67 : rtCcuTaskInfo_t ccuTaskInfo[FUSION_SUB_TASK_MAX_CCU_NUM];
68 : } rtCcuTaskGroup_t;
69 :
70 : typedef struct tagRtDevBinary {
71 : uint32_t magic; // magic number
72 : uint32_t version; // version of binary
73 : const void* data; // binary data
74 : uint64_t length; // binary length
75 : } rtDevBinary_t;
76 : /* 3-8包不支持的接口
77 : * aclrtCntNotifyWaitWithTimeout —— rtsCntNotifyWaitWithTimeout
78 : * aclrtCntNotifyRecord —— rtsCntNotifyRecord
79 : * aclrtCntNotifyDestroy —— rtCntNotifyDestroy
80 : * aclrtCntNotifyCreate —— rtCntNotifyCreateServer
81 : * aclrtGetPhyDevIdByLogicDevId —— rtsGetPhyDevIdByLogicDevId
82 : * aclrtSetDeviceTaskAbortCallback —— rtsSetDeviceTaskAbortCallback
83 : * aclrtCntNotifyGetId —— rtsCntNotifyGetId
84 : * aclrtMallocWithCfg —— rtsMalloc
85 : */
86 : using aclrtMemType_t = int;
87 : __attribute__((weak)) ACL_FUNC_VISIBILITY aclError
88 : aclrtMemP2PMap(void* devPtr, size_t size, int32_t dstDevId, uint64_t flags);
89 : HcclResult HrtResetXpuDevice(uint32_t devType, const uint32_t devId);
90 : HcclResult HrtSetXpuDevice(uint32_t devType, const uint32_t devId);
91 : extern rtError_t rtCCULaunch(rtCcuTaskInfo_t* taskInfo, rtStream_t const stm);
92 : extern rtError_t rtReleaseDevResAddress(rtDevResInfo* const resInfo);
93 : extern rtError_t rtGetDevResAddress(rtDevResInfo* const resInfo, rtDevResAddrInfo* const addrInfo);
94 : extern rtError_t rtUbDevQueryInfo(rtUbDevQueryCmd cmd, void* devInfo);
95 : #ifdef __cplusplus
96 : }
97 : #endif
98 : struct MsprofHcclInfo {
99 : uint64_t itemId;
100 : uint64_t cclTag;
101 : uint64_t groupName;
102 : uint32_t localRank;
103 : uint32_t remoteRank;
104 : uint32_t rankSize;
105 : uint32_t workFlowMode;
106 : uint32_t planeID;
107 : uint32_t ctxId;
108 : uint64_t notifyID;
109 : uint32_t stage;
110 : uint32_t role; // role {0: dst, 1:src}
111 : double durationEstimated;
112 : uint64_t srcAddr;
113 : uint64_t dstAddr;
114 : uint64_t dataSize; // bytes
115 : uint32_t opType; // {0: sum, 1: mul, 2: max, 3: min}
116 : uint32_t dataType; // data type {0: INT8, 1: INT16, 2: INT32, 3: FP16, 4:FP32, 5:INT64, 6:UINT64}
117 : uint32_t linkType; // link type {0: 'OnChip', 1: 'HCCS', 2: 'PCIe', 3: 'RoCE'}
118 : uint32_t transportType; // transport type {0: SDMA, 1: RDMA, 2:LOCAL}
119 : uint32_t rdmaType; // RDMA type {0: RDMASendNotify, 1:RDMASendPayload}
120 : uint32_t reserve2;
121 : #ifdef __cplusplus
122 16 : MsprofHcclInfo()
123 16 : : role(0xFFFFFFFF),
124 16 : srcAddr(0xFFFFFFFF),
125 16 : dstAddr(0xFFFFFFFF),
126 16 : dataSize(0),
127 16 : opType(0xFFFFFFFF),
128 16 : dataType(0xFFFFFFFF),
129 16 : linkType(0xFFFFFFFF),
130 16 : transportType(0xFFFFFFFF),
131 16 : rdmaType(0xFFFFFFFF)
132 16 : {}
133 : #endif
134 : };
135 :
136 : struct MsprofDpuHcclTrack {
137 : uint64_t itemId;
138 : uint64_t cclTag;
139 : uint64_t groupName;
140 : uint32_t localRank;
141 : uint32_t remoteRank;
142 : uint32_t rankSize;
143 : uint32_t stage;
144 : uint64_t notifyID;
145 : uint64_t timeStamp;
146 : double durationEstimated;
147 : uint64_t srcAddr;
148 : uint64_t dstAddr;
149 : uint64_t dataSize; // bytes
150 : uint32_t taskId;
151 : uint32_t aicpu_task_id;
152 : uint16_t streamId;
153 : uint16_t planeID;
154 : uint16_t npuDevId;
155 : uint16_t dpuDevId;
156 : uint8_t opType; // {0: sum, 1: mul, 2: max, 3: min}
157 : uint8_t dataType; // data type {0: INT8, 1: INT16, 2: INT32, 3: FP16, 4:FP32, 5:INT64, 6:UINT64}
158 : uint8_t linkType; // link type {0: 'OnChip', 1: 'HCCS', 2: 'PCIe', 3: 'RoCE'}
159 : uint8_t transportType; // transport type {0: SDMA, 1: RDMA, 2:LOCAL}
160 : uint8_t rdmaType; // RDMA type {0: RDMASendNotify, 1:RDMASendPayload}
161 : uint8_t role; // role {0: dst, 1:src}
162 : uint8_t workFlowMode;
163 : uint8_t reserves[1];
164 :
165 : #ifdef __cplusplus
166 9 : MsprofDpuHcclTrack()
167 9 : : itemId(0),
168 9 : cclTag(0),
169 9 : groupName(0),
170 9 : localRank(0),
171 9 : remoteRank(0),
172 9 : rankSize(0),
173 9 : stage(0),
174 9 : notifyID(0),
175 9 : timeStamp(0),
176 9 : durationEstimated(0),
177 9 : srcAddr(0xFFFFFFFF),
178 9 : dstAddr(0xFFFFFFFF),
179 9 : dataSize(0),
180 9 : taskId(0),
181 9 : aicpu_task_id(0xFFFFFFFF),
182 9 : streamId(0),
183 9 : planeID(0),
184 9 : npuDevId(0xFFFF),
185 9 : dpuDevId(0xFFFF),
186 9 : opType(0xFF),
187 9 : dataType(0xFF),
188 9 : linkType(0xFF),
189 9 : transportType(0xFF),
190 9 : rdmaType(0xFF),
191 9 : role(0xFF),
192 9 : workFlowMode(0),
193 9 : reserves{0}
194 9 : {}
195 : #endif
196 : };
197 : struct ProfilingDeviceCommResInfo {
198 : uint64_t groupName; // 通信域
199 : uint32_t rankSize; // 通信域内rank总数
200 : uint32_t rankId; // 当前device rankId，通信域内编号
201 : uint32_t usrRankId; // 当前device rankId，全局编号
202 : uint32_t aicpuKfcStreamId; // MC2中launch aicpu kfc算子的stream
203 : uint32_t commStreamSize; // 当前device侧使用的通信stream数量
204 : uint32_t commStreamIds[8]; // 具体streamId
205 : uint32_t reserve;
206 : };
207 : struct rtMemUbTokenInfo {
208 : uint64_t va;
209 : uint64_t size;
210 : uint32_t tokenId;
211 : uint32_t tokenValue;
212 : };
213 : constexpr uint32_t RT_NOTIFY_FLAG_DOWNLOAD_TO_DEV
214 : = 0x02U; // RT_NOTIFY_FLAG_DOWNLOAD_TO_DEV does not support OR with other flags
215 : constexpr uint64_t RT_NOTIFY_FLAG_DEFAULT = 0x00U;
216 : enum class HcclRtMemcpyKind {
217 : HCCL_RT_MEMCPY_KIND_HOST_TO_HOST = 0, /**< host to host */
218 : HCCL_RT_MEMCPY_KIND_HOST_TO_DEVICE, /**< host to device */
219 : HCCL_RT_MEMCPY_KIND_DEVICE_TO_HOST, /**< device to host */
220 : HCCL_RT_MEMCPY_KIND_DEVICE_TO_DEVICE, /**< device to device */
221 : HCCL_RT_MEMCPY_ADDR_DEVICE_TO_DEVICE, /**< Level-2 address copy, device to device */
222 : HCCL_RT_MEMCPY_KIND_RESERVED,
223 : };
224 : DevId HrtGetDevicePhyIdByIndex(s32 deviceLogicId);
225 : DevType HrtGetDeviceType();
226 : s32 HrtDeviceGetBareTgid();
227 : void HrtGetSocVer(std::string& socName);
228 : s32 HrtGetDevice();
229 : HcclResult HrtGetLogicDevIdByUserDevId(s32 userDevId, s32& logicDevId);
230 : // 非主线程使用rts添加task情况下，需要先使用该函数通知RTS，将线程和 device logic id绑定
231 : void HrtSetDevice(s32 deviceLogicId);
232 : void HrtResetDevice(s32 deviceLogicId);
233 : u32 HrtGetDeviceCount();
234 : HcclResult HrtGetDeviceInfo(uint32_t deviceLogicId, int32_t moduleType, aclrtDevAttr infoType, int64_t& val);
235 : HcclResult HrtGetMainboardId(uint32_t deviceLogicId, HcclMainboardId& hcclMainboardId);
236 : aclrtStream HrtStreamCreateWithFlags(uint32_t priority, uint32_t flag);
237 : void HrtStreamDestroy(aclrtStream ptr);
238 : void HrtStreamSetMode(HcclRtStream streamPtr, const uint64_t stmMode);
239 : u64 HrtStreamGetMode(HcclRtStream const ptr);
240 : void HcclStreamSynchronize(HcclRtStream ptr);
241 : s32 HrtGetStreamId(aclrtStream ptr);
242 : void HrtStreamActive(aclrtStream activeStream, aclrtStream stream);
243 :
244 : void* HrtMalloc(u64 size, aclrtMemType_t memType);
245 : void HrtFree(void* devPtr);
246 : void HrtMemcpy(void* dst, uint64_t destMax, const void* src, uint64_t count, rtMemcpyKind_t kind);
247 : void HrtMemset(void* dst, uint64_t destMax, uint64_t count);
248 : void HrtIpcSetMemoryName(void* ptr, char_t* name, u64 ptrMaxLen, u32 nameMaxLen);
249 : void HrtIpcDestroyMemoryName(const char_t* name);
250 : void* HrtIpcOpenMemory(const char_t* name);
251 : void HrtIpcCloseMemory(const void* ptr);
252 : void HrtIpcSetMemoryPid(const char_t* name, int pid);
253 : aclrtPtrAttributes HrtPointerGetAttributes(const void* ptr);
254 : void PrintMemoryAttr(const void* memAddr);
255 : void HrtDevMemAlignWithPage(void* ptr, u64 size, void*& ipcPtr, u64& ipcSize, u64& ipcOff);
256 : HcclResult HrtMemPrefetchToDevice(void* devPtr, uint64_t len);
257 :
258 : void* HrtMallocHost(u64 size);
259 : void HrtFreeHost(void* hostPtr);
260 :
261 : // rts notify manager api
262 : aclrtNotify HrtNotifyCreate(s32 deviceLogicId);
263 : aclrtNotify HrtNotifyCreateWithFlag(u32 devId, u32 flag);
264 : void HrtNotifyDestroy(RtNotify_t ptr);
265 : void HrtIpcSetNotifyName(RtNotify_t ptr, char_t* name, uint32_t len);
266 :
267 : u32 HrtGetNotifyID(RtNotify_t notifyHandle);
268 : u64 HrtNotifyGetAddr(RtNotify_t notifyHandle);
269 : void HrtSetIpcNotifyPid(aclrtNotify notify, int32_t pid);
270 : RtNotify_t HrtIpcOpenNotify(const char_t* name);
271 : RtNotify_t HrtIpcOpenNotifyWithFlag(const char_t* name, uint32_t flags);
272 : u32 HrtNotifyGetOffset(RtNotify_t ptr);
273 :
274 : // rts notify task api
275 : void HrtNotifyWaitWithTimeOut(RtNotify_t notifyPtr, aclrtStream streamPtr, uint32_t timeOut);
276 : void HrtNotifyRecord(RtNotify_t notifyPtr, aclrtStream streamPtr);
277 :
278 : // rts memcpy task api
279 : void HrtMemAsyncCopy(
280 : void* dst, uint64_t destMax, const void* src, uint64_t count, aclrtMemcpyKind kind, aclrtStream streamPtr);
281 :
282 : // rts reduce task api
283 : void HrtReduceAsync(
284 : void* dst, uint64_t destMax, const void* src, uint64_t count, aclrtReduceKind kind, aclDataType type,
285 : aclrtStream streamPtr);
286 :
287 : // rts rdma task
288 : void HrtRDMASend(u32 qpn, u32 wqeIndex, aclrtStream streamPtr); // 910A offload
289 : void HrtRDMADBSend(uint32_t dbindex, uint64_t dbinfo,
290 : aclrtStream streamPtr); // 910A opbase and 910A2/910A3
291 : void HrtAicpuLaunchKernelWithHostArgs(
292 : aclrtFuncHandle funcHandle, uint32_t numBlocks, aclrtStream stream, aclrtLaunchKernelCfg* cfg, void* hostArgs,
293 : size_t argsSize, aclrtPlaceHolderInfo* placeHolderArray = nullptr, size_t placeHolderNum = 0);
294 :
295 : // rts task exception api
296 : void HrtRegTaskFailCallbackByModule(aclrtExceptionInfoCallback callback);
297 : void HrtUnregTaskFailCallbackByModule(aclrtExceptionInfoCallback callback);
298 :
299 : // 添加任一task后可获取得到 taskId, streamId
300 : void HrtGetTaskIdAndStreamID(u32& taskId, u32& streamId);
301 : u64 HrtGetRdmaDoorbellAddr(s32 deviceLogicId, u32 dbIndex);
302 : u32 HrtStreamGetSqId(const aclrtStream ptr);
303 : u32 HrtStreamGetCqId(const aclrtStream ptr);
304 :
305 : // 对rts结构体打桩，联调用，待RTS接口上线后，删除掉
306 : struct HrtUbDbDetailInfo {
307 : u16 functionId;
308 : u16 dieId;
309 : u16 rsv;
310 : u16 jettyId;
311 : u16 piValue;
312 : };
313 :
314 : struct HrtUbDbInfo {
315 : u8 dbNum;
316 : u8 wrCqe;
317 : HrtUbDbDetailInfo info[2];
318 : };
319 :
320 : struct HrtUbWqeInfo {
321 : u16 wrCqe;
322 : u16 functionId;
323 : u16 dieId;
324 : u16 wqeSize;
325 : u16 jettyId;
326 : u8* wqe;
327 : u16 wqePtrLen;
328 : };
329 :
330 : constexpr u32 DWQE_SIZE_64 = 64;
331 : constexpr u32 DWQE_SIZE_128 = 128;
332 :
333 : void HrtUbDbSend(const HrtUbDbInfo& info, aclrtStream streamPtr);
334 :
335 : void HrtUbDirectSend(const HrtUbWqeInfo& info, aclrtStream streamPtr);
336 :
337 : aclrtCntNotify HrtCntNotifyCreate(u32 deviceId);
338 :
339 : u32 HrtGetCntNotifyId(const aclrtCntNotify inCntNotify);
340 :
341 : void HrtCntNotifyDestroy(const aclrtCntNotify inCntNotify);
342 :
343 66 : MAKE_ENUM(HrtCntNotifyRecordMode, WRITE_BIT, STORE)
344 : void HrtCntNotifyRecord(
345 : const aclrtCntNotify inCntNotify, const aclrtStream streamPtr, HrtCntNotifyRecordMode mode, u32 value);
346 77 : MAKE_ENUM(HrtCntNotifyWaitMode, EQUAL, BITMAP)
347 : void HrtCntNotifyWaitWithTimeOut(
348 : const aclrtCntNotify inCntNotify, const aclrtStream streamPtr, HrtCntNotifyWaitMode mode, u32 value, u32 timeout,
349 : bool isClear = true);
350 :
351 : void HrtCcuLaunch(rtCcuTaskInfo_t& taskInfo, aclrtStream const streamPtr);
352 : void HrtUbDevQueryInfo(rtUbDevQueryCmd cmd, void* devInfo);
353 : // pair<tokendId, tokenValue>
354 : std::pair<u32, u32> HrtUbDevQueryToken(u64 addr, u64 size);
355 1010 : MAKE_ENUM(HrtDevResProcType, PROCESS_CP1, PROCESS_HCCP)
356 1158 : MAKE_ENUM(
357 : HrtDevResType, RES_TYPE_STARS_NOTIFY_RECORD, RES_TYPE_CCU_CKE, RES_TYPE_CCU_XN, RES_TYPE_STARS_CNT_NOTIFY_BIT_WR)
358 : #define HRT_DEV_RES_FLAG_USE_UNIQUE_VA (1U << 7U) // bit0, map to unified va, for RT_RES_TYPE_STARS_NOTIFY_RECORD
359 : struct HrtDevResInfo {
360 : u32 dieId{0}; // for ccu res need set devId, for others set 0
361 : HrtDevResProcType procType{HrtDevResProcType::PROCESS_CP1};
362 : HrtDevResType resType{HrtDevResType::RES_TYPE_STARS_NOTIFY_RECORD};
363 : u32 resId{0};
364 : u32 flag{0};
365 : };
366 :
367 : struct HrtDevResAddrInfo {
368 : u64 address{0};
369 : u32 len{0};
370 : };
371 :
372 : HrtDevResAddrInfo HrtGetDevResAddress(const HrtDevResInfo& devResInfo);
373 : void HrtReleaseDevResAddress(const HrtDevResInfo& devResInfo);
374 :
375 6 : MAKE_ENUM(HrtEventStatus, EVENT_INIT, EVENT_RECORDED)
376 : aclrtEvent HrtEventCreateWithFlag(u32 flag);
377 : void HrtEventDestroy(RtEvent_t eventPtr);
378 : void HrtEventRecord(RtEvent_t eventPtr, aclrtStream streamPtr);
379 : HrtEventStatus HrtEventQueryStatus(RtEvent_t eventPtr);
380 :
381 : void HrtWriteValue(u64 addr, u32 piVal, const aclrtStream streamPtr);
382 : void HrtDeviceAbortRegCallBack(aclrtDeviceTaskAbortCallback callback, void* args, const std::string& name);
383 : HcclResult HrtEnableP2P(u32 deviceLogicId, u32 devicePhyId);
384 : HcclResult HrtDisableP2P(u32 deviceLogicId, u32 devicePhyId);
385 : HcclResult HrtGetP2PStatus(u32 deviceLogicId, u32 devicePhyId, uint32_t* status);
386 : } // namespace Hccl
387 :
388 : #endif // HCCL_ADAPTER_RTS_H
|
