Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #ifndef HCOMM_CCU_KERNEL_H
12 : #define HCOMM_CCU_KERNEL_H
13 :
14 : #include <cstdint>
15 : #include <functional>
16 : #include <array>
17 : #include <memory>
18 : #include <string>
19 : #include <vector>
20 : #include <unordered_map>
21 : #include <unordered_set>
22 :
23 : #include "ccu_task_arg_v1.h"
24 : #include "ccu_task_param_v1.h"
25 :
26 : #include "ccu_kernel_resource.h"
27 : #include "ccu_instr_info_v1.h"
28 : #include "ccu_rep_context_v1.h"
29 :
30 : #include "ccu_funccall_v1.h"
31 : #include "ccu_loopcall_v1.h"
32 :
33 : #include "hccl_types.h"
34 : #include "hcomm_primitives.h"
35 : #include "ccu_datatype_v1.h"
36 : #include "ccu_interface_assist_v1.h"
37 :
38 : #include "ccu_res_repo.h"
39 : #include "ccu_dev_mgr_imp.h" // CcuVersion
40 :
41 : // 暂时引用方便算法开发
42 : #include "ccu_repeat_v1.h"
43 : #include "ccu_condition_v1.h"
44 : #include "ccu_rep_funcblock_v1.h"
45 : #include "ccu_loopblock_v1.h"
46 : #include "ccu_loopcall_v1.h"
47 : #include "ccu_assist_pub.h"
48 :
49 : #ifndef CCU_PROFILING // 和hccl仓兼容性使用
50 : #define CCU_PROFILING
51 : #endif
52 : #include "ccu_types.h"
53 :
54 : namespace hcomm {
55 :
56 : struct GroupInfo {
57 : uint16_t loopParamId;
58 : uint16_t parallelParamId;
59 : uint16_t residualId;
60 : };
61 :
62 : struct GroupOpConfig {
63 : uint32_t msInterleave;
64 : uint32_t loopCount;
65 : uint64_t memSlice;
66 : };
67 :
68 : // Kernel 查询信息结构体，由 CcuKernel::GetCcuKernelInfo 在锁内填充。
69 : // 后续新增查询字段时只需在此结构体中添加，无需新增接口。
70 : struct CcuKernelInfo {
71 : uint32_t maxTaskArgsNum{0}; // taskArgs数组所需的元素个数，注册期间由LoadArg的argId推算: max(argId)+1，空则为0
72 : };
73 :
74 : class CcuKernel : public CcuRep::CcuRepContext {
75 : public:
76 4410 : CcuKernel() = default;
77 : ~CcuKernel() override;
78 :
79 : HcclResult SetupProfilingInfo(const char* kernelFuncName);
80 : HcclResult ApplyDieFromChannels();
81 : HcclResult ValidateAndApplyDie(uint32_t targetDieId);
82 :
83 : CcuResReq GetResourceRequest();
84 : CcuResRepository& GetResRepository();
85 : CcuRepResource& GetResource();
86 : CcuSharedResource& GetExportedRes();
87 : CcuSharedResource& GetImportedRes();
88 :
89 : void SetResRepository(const CcuResRepository& resRepo);
90 : void SetInstrId(uint32_t instrId);
91 : uint32_t GetInstrId() const;
92 : uint32_t GetInstrCount();
93 : // 统计会翻译出 CKE 写者微码 (profiling -> setcke / 非 profiling -> clearcke) 的三种 wait 类 rep
94 : // 个数 (含 block 子 rep), 用于按 CCU_CKE_RAW_LATENCY 为每个此类 rep 预留指令空间.
95 : // 每个此类 rep 只发射 1 条 setcke / clearcke.
96 : uint32_t GetRepNeedToAddLatency() const;
97 : void SetCcuInstrInfo(const CcuRep::CcuInstrInfo& instrInfo);
98 :
99 : CcuResult GeneTaskParams(const uint64_t* taskArgs, uint32_t argsNum, std::vector<CcuTaskParam>& taskParams);
100 :
101 : void SetInsGenerater(CcuRep::CcuInsGeneratorBase* insGeneratorBase);
102 72 : void SetCcuVersion(CcuVersion version) { ccuVersion_ = version; }
103 : // 该友元函数用于在context类外创建Variable并被context内的资源管理器管理
104 : friend CcuRep::Variable CcuRep::CreateVariable(CcuRep::CcuRepContext* context);
105 :
106 : HcclResult AddProfilingInfo(
107 : const ChannelHandle* channels, uint32_t channelNum, HcclDataType dataType, HcclDataType outputDataType,
108 : HcclReduceOp opType, const std::string& opName);
109 :
110 : HcclResult AddCcuProfiling(
111 : GroupInfo groupInfo, const std::vector<ChannelHandle> channelHandle, HcclDataType dataType,
112 : HcclDataType outputDataType, HcclReduceOp opType, const std::string& opName);
113 : HcclResult AddCcuProfiling(
114 : const ChannelHandle* channels, uint32_t channelNum, HcclDataType dataType, HcclDataType outputDataType,
115 : HcclReduceOp opType, const std::string& opName);
116 : HcclResult
117 : GetCcuProfilingInfo(const uint64_t* taskArgs, uint32_t argSize, std::vector<CcuProfilingInfo>& allCcuProfilingInfo);
118 :
119 : const std::vector<CcuProfilingInfo>& GetAllCcuProfilingInfo() { return allCcuProfilingInfos_; };
120 :
121 : // process const values
122 208 : std::unordered_map<uint64_t, CcuRep::Variable>& GetConstValue2VarMap() { return constValue2VarMap; }
123 : HcclResult Add2ConstValue2VarMap(std::vector<uint64_t>& values);
124 :
125 8 : const std::unordered_set<ChannelHandle>& GetChannels() { return channels_; }
126 :
127 6 : const std::unordered_set<uint32_t>& GetDeclaredLocXns() const { return declaredLocXns_; }
128 :
129 : struct PinnedGroupEntry {
130 : CcuRep::Variable baseVar;
131 : uint16_t count;
132 : };
133 : const std::vector<PinnedGroupEntry>& GetPinnedRegGroups() const { return pinnedRegGroups_; }
134 : void AddPinnedRegGroup(const CcuRep::Variable& baseVar, uint16_t count);
135 :
136 : public:
137 : // Alloc 相关接口
138 : CcuResult VariableAlloc(CcuVariableHandle* varHandle);
139 : CcuResult AddressAlloc(CcuAddressHandle* addrHandle);
140 : CcuResult EventAlloc(CcuEventHandle* eventHandle);
141 : CcuResult BufferAlloc(CcuBufferHandle* bufHandle);
142 : CcuResult
143 : LocalAddrAlloc(CcuLocalAddrHandle* localAddrHandle, CcuAddressHandle* addrHandle, CcuVariableHandle* tokenHandle);
144 : CcuResult RemoteAddrAlloc(
145 : CcuRemoteAddrHandle* remoteAddrHandle, CcuAddressHandle* addrHandle, CcuVariableHandle* tokenHandle);
146 : CcuResult BlockVariableAlloc(CcuVariableHandle* varHandles, uint32_t count);
147 : CcuResult BlockEventAlloc(CcuEventHandle* eventHandles, uint32_t count);
148 : CcuResult BlockBufferAlloc(CcuBufferHandle* bufHandles, uint32_t count);
149 : CcuResult VariableCreateByChannel(ChannelHandle channel, uint32_t varIndex, CcuVariableHandle* varHandle);
150 : CcuResult VariableCreateByAcquire(CcuVariableHandle acqHandle, uint32_t index, CcuVariableHandle* varHandle);
151 : CcuResult EventCreateByAcquire(CcuEventHandle acqHandle, uint32_t index, CcuEventHandle* eventHandle);
152 :
153 : // 参数加载类 相关接口
154 : CcuResult LoadArg(CcuVariableHandle varHandle, uint32_t argId);
155 : // 在锁内填充 Kernel 查询信息；当前包含 maxTaskArgsNum。
156 : CcuResult GetCcuKernelInfo(CcuKernelInfo& info) const;
157 : CcuResult LoadVar(uint64_t addr, CcuVariableHandle varHandle, uint32_t num);
158 : CcuResult CcuLoadVarFromVarAddr(CcuVariableHandle addrHandle, CcuVariableHandle varHandle, uint32_t num);
159 : CcuResult StoreVar(uint64_t addr, CcuVariableHandle varHandle, uint32_t num);
160 : CcuResult CcuStoreVarToVarAddr(CcuVariableHandle addrHandle, CcuVariableHandle varHandle, uint32_t num);
161 :
162 : // Event信号同步类 相关接口
163 : // mask 由调用方独立传入（与 Event 句柄解耦），不再设独立的 SetEventMask 接口。
164 : CcuResult EventRecord(CcuEventHandle eventHandle, uint32_t mask);
165 : CcuResult EventWait(CcuEventHandle eventHandle, uint32_t mask);
166 : CcuResult NotifyRecord(const ChannelHandle channel, uint32_t remoteNotifyIdx, uint32_t mask);
167 : CcuResult NotifyWait(const ChannelHandle channel, uint32_t localNotifyIdx, uint32_t mask);
168 : CcuResult WriteVariableWithNotify(
169 : const ChannelHandle channel, CcuVariableHandle varHandle, uint32_t remoteVarIdx, uint32_t remoteNotifyIdx,
170 : uint32_t mask);
171 : // 本地（同 device 内跨 core）通知同步：用 notifyTag 字符串作为对端标识，
172 : // 由调用方约定生产者/消费者使用相同的 tag 字符串完成配对。
173 : // 与 NotifyRecord/Wait（用 ChannelHandle 标识跨 rank 通道）的对偶。
174 : // 必须 public：C API ccu_primitives_impl.cc 直接调用。
175 : CcuResult LocalNotifyRecord(const char* notifyTag, const uint32_t mask);
176 : CcuResult LocalNotifyWait(const char* notifyTag, const uint32_t mask);
177 : // 本地数据拷贝 相关接口
178 : CcuResult LocalCopyMemToBuffer(
179 : CcuBufferHandle dstHandle, CcuLocalAddrHandle srcHandle, CcuVariableHandle lenHandle,
180 : CcuEventHandle eventHandle, uint32_t mask);
181 : CcuResult LocalCopyBufferToMem(
182 : CcuLocalAddrHandle dstHandle, CcuBufferHandle srcHandle, CcuVariableHandle lenHandle,
183 : CcuEventHandle eventHandle, uint32_t mask);
184 : CcuResult LocalCopyMemToMem(
185 : CcuLocalAddrHandle dstHandle, CcuLocalAddrHandle srcHandle, CcuVariableHandle lenHandle,
186 : CcuEventHandle eventHandle, uint32_t mask);
187 :
188 : // 本地reduce 相关接口
189 : CcuResult LocalMemReduce(
190 : CcuLocalAddrHandle dstHandle, CcuLocalAddrHandle srcHandle, CcuVariableHandle lenHandle, HcclDataType dataType,
191 : HcclReduceOp opType, CcuEventHandle eventHandle, uint32_t mask);
192 : CcuResult LocalBufferReduce(
193 : CcuBufferHandle* bufHandles, uint32_t count, HcclDataType dataType, HcclDataType outputDataType,
194 : HcclReduceOp opType, CcuVariableHandle lenHandle, CcuEventHandle eventHandle, uint32_t mask);
195 :
196 : // 运算重载 相关接口
197 : CcuResult VariableAssignImm(CcuVariableHandle var, uint64_t immediate);
198 : CcuResult VariableAssignVar(CcuVariableHandle var, CcuVariableHandle varA);
199 : CcuResult
200 : VariableAddVarToVar(CcuVariableHandle varHandle, CcuVariableHandle varAHandle, CcuVariableHandle varBHandle);
201 : CcuResult
202 : VariableSubVarToVar(CcuVariableHandle varHandle, CcuVariableHandle varAHandle, CcuVariableHandle varBHandle);
203 : CcuResult
204 : VariableMulVarToVar(CcuVariableHandle varHandle, CcuVariableHandle varAHandle, CcuVariableHandle varBHandle);
205 : CcuResult VariableAddImmToVar(CcuVariableHandle varHandle, CcuVariableHandle varAHandle, uint16_t immediate);
206 : CcuResult VariableSubImmToVar(CcuVariableHandle varHandle, CcuVariableHandle varAHandle, uint16_t immediate);
207 : CcuResult VariableMulImmToVar(CcuVariableHandle varHandle, CcuVariableHandle varAHandle, uint16_t immediate);
208 : CcuResult
209 : VariableAndVarToVar(CcuVariableHandle varHandle, CcuVariableHandle varAHandle, CcuVariableHandle varBHandle);
210 : CcuResult
211 : VariableOrVarToVar(CcuVariableHandle varHandle, CcuVariableHandle varAHandle, CcuVariableHandle varBHandle);
212 : CcuResult
213 : VariableXorVarToVar(CcuVariableHandle varHandle, CcuVariableHandle varAHandle, CcuVariableHandle varBHandle);
214 : CcuResult VariableNotVar(CcuVariableHandle varHandle, CcuVariableHandle varAHandle);
215 : CcuResult
216 : VariableShlVarToVar(CcuVariableHandle varHandle, CcuVariableHandle varAHandle, CcuVariableHandle varBHandle);
217 : CcuResult
218 : VariableShrVarToVar(CcuVariableHandle varHandle, CcuVariableHandle varAHandle, CcuVariableHandle varBHandle);
219 : CcuResult AddressAssignImm(CcuAddressHandle addr, uint64_t immediate);
220 : CcuResult AddressAssignVar(CcuAddressHandle addrHandle, CcuVariableHandle varHandle);
221 : CcuResult AddressAssignAddr(CcuAddressHandle dstAddrHandle, CcuAddressHandle srcAddrHandle);
222 : CcuResult AddressAddVarToAddr(CcuAddressHandle resAddr, CcuAddressHandle lhsAddr, CcuVariableHandle rhsVar);
223 : CcuResult
224 : AddressAddAddrToAddr(CcuAddressHandle resAddrHandle, CcuAddressHandle addrAHandle, CcuAddressHandle addrBHandle);
225 : CcuResult AddressAddAssignVar(CcuAddressHandle addr, CcuVariableHandle var);
226 : CcuResult AddressAddAssignAddr(CcuAddressHandle addrHandle, CcuAddressHandle otherHandle);
227 : CcuResult AddressAddImmToAddr(CcuAddressHandle resAddr, CcuAddressHandle addrA, uint16_t imm);
228 :
229 : // 远端数据传输操作
230 :
231 : CcuResult ReadMemToMem(
232 : ChannelHandle channel, CcuLocalAddrHandle localHandle, CcuRemoteAddrHandle remoteHandle,
233 : CcuVariableHandle lenHandle, CcuEventHandle eventHandle, uint32_t mask);
234 : CcuResult ReadMemToBuffer(
235 : ChannelHandle channel, CcuBufferHandle localHandle, CcuRemoteAddrHandle remoteHandle,
236 : CcuVariableHandle lenHandle, CcuEventHandle eventHandle, uint32_t mask);
237 : CcuResult ReadMemToMemReduce(
238 : ChannelHandle channel, CcuLocalAddrHandle localHandle, CcuRemoteAddrHandle remoteHandle,
239 : CcuVariableHandle lenHandle, HcclDataType dataType, HcclReduceOp opType, CcuEventHandle eventHandle,
240 : uint32_t mask);
241 : CcuResult WriteMemToMem(
242 : ChannelHandle channel, CcuRemoteAddrHandle remoteHandle, CcuLocalAddrHandle localHandle,
243 : CcuVariableHandle lenHandle, CcuEventHandle eventHandle, uint32_t mask);
244 : CcuResult WriteBufferToMem(
245 : ChannelHandle channel, CcuRemoteAddrHandle remoteHandle, CcuBufferHandle localHandle,
246 : CcuVariableHandle lenHandle, CcuEventHandle eventHandle, uint32_t mask);
247 : CcuResult WriteMemToMemReduce(
248 : ChannelHandle channel, CcuRemoteAddrHandle remoteHandle, CcuLocalAddrHandle localHandle,
249 : CcuVariableHandle lenHandle, HcclDataType dataType, HcclReduceOp opType, CcuEventHandle eventHandle,
250 : uint32_t mask);
251 :
252 : CcuResult IfBegin(CcuVariableHandle varHandle, uint64_t immediate, CcuConditionType condType, const char* label);
253 : CcuResult
254 : IfBeginVar(CcuVariableHandle lhsHandle, CcuVariableHandle rhsHandle, CcuConditionType condType, const char* label);
255 : CcuResult IfElse(const char* label);
256 : CcuResult IfEnd(const char* label);
257 :
258 : void IfLabelStackPush(const char* label);
259 : void IfLabelStackMarkBodyDone();
260 : const char* IfLabelStackPopForElse();
261 : bool IfLabelStackTopIsClosable();
262 : const char* IfLabelStackPop();
263 :
264 : void DoWhileLabelStackPush(const char* label);
265 : const char* DoWhileLabelStackPopForWhile();
266 :
267 : void FlushClosablePendingIfs();
268 : void Append(std::shared_ptr<CcuRep::CcuRepBase> rep) override;
269 : CcuResult WhileBegin(CcuVariableHandle varHandle, uint64_t immediate, CcuConditionType condType, const char* label);
270 : CcuResult WhileBeginVar(
271 : CcuVariableHandle lhsHandle, CcuVariableHandle rhsHandle, CcuConditionType condType, const char* label);
272 : CcuResult WhileEnd(const char* label);
273 : CcuResult DoWhileBegin(const char* label);
274 : CcuResult DoWhileEnd(CcuVariableHandle varHandle, uint64_t immediate, CcuConditionType condType, const char* label);
275 : CcuResult DoWhileEndVar(
276 : CcuVariableHandle lhsHandle, CcuVariableHandle rhsHandle, CcuConditionType condType, const char* label);
277 :
278 : CcuResult LoopCreate(CcuLoop* loop);
279 : CcuResult LoopBodyEnter(CcuLoop loop);
280 : CcuResult LoopBodyExit(CcuLoop loop);
281 : // 建组/加环统一以版本化 cfg 为准;旧 config 的转换在 C ABI 适配层完成。
282 : CcuResult LoopGroupCreate(CcuLoopGroup* group, uint32_t maxLoopNum, const CcuLoopGroupCfg* cfg);
283 : CcuResult LoopGroupCreateFromVar(
284 : CcuLoopGroup* group, uint32_t maxLoopNum, CcuVariableHandle parallelVarHandle,
285 : CcuVariableHandle offsetVarHandle);
286 : CcuResult LoopGroupCreateFromVarV2(
287 : CcuLoopGroup* group, uint32_t maxLoopNum, CcuVariableHandle parallelVarV2, CcuVariableHandle offsetVarV2,
288 : CcuVariableHandle varOffsetVar);
289 : CcuResult LoopGroupAddLoop(CcuLoopGroup group, CcuLoop loop, const CcuLoopCfg* cfg);
290 : CcuResult LoopGroupAddLoopFromVar(CcuLoopGroup group, CcuLoop loop, CcuVariableHandle loopParamVar);
291 : CcuResult LoopGroupAddLoopFromVarV2(
292 : CcuLoopGroup group, CcuLoop loop, CcuVariableHandle iterNumVar, CcuVariableHandle addrOffsetVar,
293 : CcuVariableHandle ctxIdVar);
294 :
295 : CcuResult FuncBlockLookup(const void* funcPtr, uint64_t* outHandle);
296 : CcuResult FuncBlockBegin(const void* funcPtr, uint64_t* outHandle);
297 : CcuResult FuncBlockEnd(uint64_t handle);
298 : CcuResult FuncDefineInArg(uint64_t handle, CcuVariableHandle formal);
299 : CcuResult FuncCall(uint64_t handle, const CcuVariableHandle* inArgs, uint32_t numIn);
300 :
301 : private:
302 : CcuResult GetVariableByHandle(CcuVariableHandle varHandle, CcuRep::Variable** variable);
303 : CcuResult GetEventByHandle(CcuEventHandle eventHandle, CcuRep::CompletedEvent** event);
304 : CcuResult LatchBodyError(CcuResult err);
305 : // 按需扩容 res_.blockExecutor[0]：不足 maxLoopNum 时补足，足够则不动；
306 : // 由 LoopGroupCreate / LoopGroupCreateFromVar 在 LoopGroup 创建时调用。
307 : CcuResult EnsureLoopEnginePool(uint32_t maxLoopNum);
308 :
309 : CcuResult ValidateTaskArgs(const uint64_t* taskArgs, uint32_t argsNum) const;
310 : void FillTaskParam(
311 : CcuTaskParam& param, uint32_t index, uint32_t seqNum, const uint64_t* taskArgs, uint32_t argsNum) const;
312 :
313 : CcuResult ResolveBufRemoteLenEvent(
314 : CcuBufferHandle bufHandle, CcuRemoteAddrHandle remoteHandle, CcuVariableHandle lenHandle,
315 : CcuEventHandle eventHandle, CcuRep::CcuBuf** buf, CcuRep::RemoteAddr** remote, CcuRep::Variable** len,
316 : CcuRep::CompletedEvent** event);
317 :
318 : CcuResult ResolveLocalRemoteLenEvent(
319 : CcuLocalAddrHandle localHandle, CcuRemoteAddrHandle remoteHandle, CcuVariableHandle lenHandle,
320 : CcuEventHandle eventHandle, CcuRep::LocalAddr** local, CcuRep::RemoteAddr** remote, CcuRep::Variable** len,
321 : CcuRep::CompletedEvent** event);
322 :
323 : CcuResult ResolveRemoteLocalLenEvent(
324 : CcuRemoteAddrHandle remoteHandle, CcuLocalAddrHandle localHandle, CcuVariableHandle lenHandle,
325 : CcuEventHandle eventHandle, CcuRep::RemoteAddr** remote, CcuRep::LocalAddr** local, CcuRep::Variable** len,
326 : CcuRep::CompletedEvent** event);
327 :
328 : // 校验从 varHandle 起的 num 个 Variable 句柄对应的内部变量 Id 连续递增，
329 : // 用于 LoadVar/StoreVar 等接口对“连续变量块”的前置校验。
330 : CcuResult CheckContinuousVariables(
331 : CcuVariableHandle varHandle, uint32_t num, const CcuRep::Variable& baseVar, const char* tag);
332 :
333 : // GetCcuProfilingInfo 的子步骤：处理 sqe & waitcke 类型的 profiling 信息，结果直接 push 到 allCcuProfilingInfos_
334 : // 中。
335 : HcclResult CollectSqeAndWaitCkeProfilingInfo();
336 : // GetCcuProfilingInfo 的子步骤：根据 LoopGroup 的 profiling 缓存构建
337 : // varId -> argIndex 与 varId -> varId 两个查找表，供 LoopGroup 段查询入参使用。
338 : HcclResult BuildLoopGroupVarIdMaps(
339 : std::unordered_map<uint16_t, uint32_t>& varId2ArgIndexMap,
340 : std::unordered_map<uint16_t, uint16_t>& varId2VarIdMap);
341 : // GetCcuProfilingInfo 的子步骤：处理 LoopGroup 的 profiling 信息，结果 push 到 allCcuProfilingInfos_ 中。
342 : HcclResult CollectLoopGroupProfilingInfo(
343 : const uint64_t* taskArgs, uint32_t argSize, const std::unordered_map<uint16_t, uint32_t>& varId2ArgIndexMap,
344 : const std::unordered_map<uint16_t, uint16_t>& varId2VarIdMap);
345 : // CollectLoopGroupProfilingInfo 的单次循环体：处理下标 i 对应的 loopParam 与 parallelParam 分支。
346 : HcclResult CollectSingleLoopGroupProfiling(
347 : uint32_t i, size_t repSize, const uint64_t* taskArgs, uint32_t argSize,
348 : const std::unordered_map<uint16_t, uint32_t>& varId2ArgIndexMap,
349 : const std::unordered_map<uint16_t, uint16_t>& varId2VarIdMap);
350 :
351 : CcuRep::Variable CreateJumpTargetVar();
352 :
353 : struct IfLabelEntry {
354 : const char* label{nullptr};
355 : bool bodyDone{false};
356 : };
357 : struct DoWhileLabelEntry {
358 : const char* label{nullptr};
359 : std::shared_ptr<CcuRep::CcuRepBlock> snapshotBlock{nullptr};
360 : size_t snapshotRepCount{0};
361 : };
362 : std::vector<IfLabelEntry> iflabelStack_;
363 : std::vector<DoWhileLabelEntry> doWhileLabelStack_;
364 : bool isFlushing_ = false;
365 :
366 : struct PendingIfContext {
367 : std::shared_ptr<CcuRep::CcuRepJumpLabel> elseLabel;
368 : std::shared_ptr<CcuRep::CcuRepJumpLabel> endLabel;
369 : bool hasElse{false};
370 : };
371 :
372 : struct PendingWhileContext {
373 : std::shared_ptr<CcuRep::CcuRepJumpLabel> beginLabel;
374 : std::shared_ptr<CcuRep::CcuRepJumpLabel> endLabel;
375 : CcuVariableHandle varHandle;
376 : uint64_t immediate;
377 : CcuConditionType condType;
378 : };
379 :
380 : struct PendingDoWhileContext {
381 : std::shared_ptr<CcuRep::CcuRepJumpLabel> beginLabel;
382 : };
383 :
384 : std::unordered_map<CcuVariableHandle, CcuRep::Variable> ccuVarMap_{};
385 :
386 : std::unordered_map<std::string, PendingIfContext> pendingIfCtx_{};
387 : std::unordered_map<std::string, PendingWhileContext> pendingWhileCtx_{};
388 : std::unordered_map<std::string, PendingDoWhileContext> pendingDoWhileCtx_{};
389 :
390 : std::unordered_map<CcuEventHandle, CcuRep::CompletedEvent> ccuEventMap_{};
391 :
392 : CcuResult GetBufferByHandle(CcuBufferHandle bufferHandle, CcuRep::CcuBuf** buffer);
393 : std::unordered_map<CcuBufferHandle, CcuRep::CcuBuf> ccuBufferMap_{};
394 :
395 : CcuResult GetAddressByHandle(CcuAddressHandle addrHandle, CcuRep::Address** address);
396 : std::unordered_map<CcuAddressHandle, CcuRep::Address> ccuAddrMap_{};
397 :
398 : CcuResult GetLocalAddrByHandle(CcuLocalAddrHandle handle, CcuRep::LocalAddr** localAddr);
399 : std::unordered_map<CcuLocalAddrHandle, CcuRep::LocalAddr> ccuLocalAddrMap_{};
400 :
401 : CcuResult GetRemoteAddrByHandle(CcuRemoteAddrHandle handle, CcuRep::RemoteAddr** remoteAddr);
402 : std::unordered_map<CcuRemoteAddrHandle, CcuRep::RemoteAddr> ccuRemoteAddrMap_{};
403 :
404 : std::unordered_set<uint32_t> loadArgUsedSet_{};
405 :
406 : protected:
407 : // 使用channel中的Variable
408 : HcclResult CreateVariable(const ChannelHandle channel, uint32_t varIndex, CcuRep::Variable* var);
409 : CcuRep::Variable CreateVariable();
410 : CcuRep::Variable CreateExpectVar();
411 : CcuRep::Variable CreateContinuousVariable();
412 : CcuRep::LocalAddr CreateLocalAddr();
413 : CcuRep::RemoteAddr CreateRemoteAddr();
414 : CcuRep::RemoteAddr GetRemoteAddr(const ChannelHandle channel, const uint32_t index);
415 : CcuRep::LocalNotify CreateLocalNotify();
416 : CcuRep::CompletedEvent CreateCompletedEvent();
417 : CcuRep::CcuBuf CreateCcuBuf();
418 : CcuRep::Executor CreateExecutor();
419 :
420 : HcclResult CreateBlockCcuBuf(const uint32_t count, CcuRep::CcuBuf* ccuBufs);
421 : HcclResult CreateBlockExecutor(const uint32_t count, CcuRep::Executor* ccuExes);
422 : HcclResult CreateBlockCompletedEvent(const uint32_t count, CcuRep::CompletedEvent* ccuEvents);
423 :
424 : // 内部 *Nb / RecordEvent / WaitEvent 系列：mask 由调用方独立传入，
425 : // 不再从 CompletedEvent 上读取。
426 : HcclResult RecordEvent(CcuRep::CompletedEvent event, uint32_t mask);
427 : HcclResult WaitEvent(CcuRep::CompletedEvent event, uint32_t mask);
428 :
429 : // 数据操作
430 : HcclResult WriteNb(
431 : const ChannelHandle channel, const CcuRep::RemoteAddr& rem, const CcuRep::LocalAddr& loc,
432 : const CcuRep::Variable& len, CcuRep::CompletedEvent event, uint32_t mask);
433 : HcclResult WriteNb(
434 : const ChannelHandle channel, const CcuRep::RemoteAddr& rem, const CcuRep::CcuBuf& loc,
435 : const CcuRep::Variable& len, CcuRep::CompletedEvent event, uint32_t mask);
436 :
437 : HcclResult ReadNb(
438 : const ChannelHandle channel, const CcuRep::LocalAddr& loc, const CcuRep::RemoteAddr& rem,
439 : const CcuRep::Variable& len, CcuRep::CompletedEvent event, uint32_t mask);
440 : HcclResult ReadNb(
441 : const ChannelHandle channel, const CcuRep::CcuBuf& loc, const CcuRep::RemoteAddr& rem,
442 : const CcuRep::Variable& len, CcuRep::CompletedEvent event, uint32_t mask);
443 :
444 : HcclResult WriteReduceNb(
445 : const ChannelHandle channel, const CcuRep::RemoteAddr& rem, const CcuRep::LocalAddr& loc,
446 : const CcuRep::Variable& len, HcclDataType dataType, HcclReduceOp opType, CcuRep::CompletedEvent event,
447 : uint32_t mask);
448 : HcclResult ReadReduceNb(
449 : const ChannelHandle channel, const CcuRep::LocalAddr& loc, const CcuRep::RemoteAddr& rem,
450 : const CcuRep::Variable& len, HcclDataType dataType, HcclReduceOp opType, CcuRep::CompletedEvent event,
451 : uint32_t mask);
452 :
453 : HcclResult LocalCopyNb(
454 : const CcuRep::LocalAddr& dst, const CcuRep::LocalAddr& src, const CcuRep::Variable& len,
455 : CcuRep::CompletedEvent event, uint32_t mask); // dst和src是否都是local
456 : HcclResult LocalCopyNb(
457 : const CcuRep::CcuBuf& dst, const CcuRep::LocalAddr& src, const CcuRep::Variable& len,
458 : CcuRep::CompletedEvent event, uint32_t mask);
459 : HcclResult LocalCopyNb(
460 : const CcuRep::LocalAddr& dst, const CcuRep::CcuBuf& src, const CcuRep::Variable& len,
461 : CcuRep::CompletedEvent event, uint32_t mask);
462 :
463 : HcclResult LocalReduceNb(
464 : const CcuRep::LocalAddr& dst, const CcuRep::LocalAddr& src, const CcuRep::Variable& len, HcclDataType dataType,
465 : HcclReduceOp opType, CcuRep::CompletedEvent event, uint32_t mask);
466 : HcclResult LocalReduceNb(
467 : const CcuRep::CcuBuf* bufs, uint32_t count, HcclDataType dataType, HcclDataType outputDataType,
468 : HcclReduceOp opType, const CcuRep::Variable& len, CcuRep::CompletedEvent event, uint32_t mask);
469 :
470 : // 参数操作
471 : void Load(const CcuRep::Variable& var);
472 :
473 : // Variable src中存放内存地址，从地址中加载数据到Variable var中
474 : void LoadVariable(const CcuRep::Variable& src, const CcuRep::Variable& var);
475 :
476 : void StoreVariable(const CcuRep::Variable& var, uint64_t addr);
477 : // 控制逻辑
478 : // 宏定义IF、WHILE
479 : CcuRep::FuncCall Func(const std::string& label);
480 : CcuRep::FuncCall Func(const CcuRep::Variable& funcAddr);
481 : CcuRep::LoopCall Loop(const std::string& label);
482 :
483 : private:
484 : CcuRep::Address CreateAddress();
485 : CcuRep::LocalAddr CreateLocalAddr(const CcuRep::Variable& token);
486 :
487 : protected:
488 : GroupOpConfig moConfig_{0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFFFFFFFFFF};
489 :
490 : private:
491 : template <typename T>
492 : T CreateResAssist(std::array<std::vector<T>, CCU_MAX_IODIE_NUM>& resRecord);
493 : template <typename T>
494 : std::vector<T> CreateBlockResAssist(const uint32_t count, std::array<std::vector<T>, CCU_MAX_IODIE_NUM>& resRecord);
495 :
496 : private:
497 : CcuRepResource res_{};
498 : CcuResRepository resRepo_{};
499 :
500 : std::unordered_set<ChannelHandle> channels_{};
501 :
502 : std::unordered_set<uint32_t> declaredLocXns_{};
503 :
504 : std::vector<PinnedGroupEntry> pinnedRegGroups_{};
505 :
506 : CcuRep::CcuInstrInfo instrInfo_{};
507 :
508 : uint32_t loadArgIndex_{0};
509 :
510 : CcuVersion ccuVersion_{CcuVersion::CCU_INVALID};
511 :
512 : // A5(CCU_V1) 专用：CCU_IF,CCU_ELSE 跳转共用一个 Xn；当前仅CCU_IF和CCU_ELSE复用，While和DoWhile暂未复用
513 : // A5 上 CCU_IF和ELSE的targetVar 仅用于LoadImdToXn + 紧接 Jump，生命周期不跨越其他指令，可安全复用。
514 : // A6不复用，保持每次独立分配的语义。
515 : std::unique_ptr<CcuRep::Variable> sharedJumpTargetVar_;
516 :
517 : CcuSharedResource exportedRes_{};
518 : CcuSharedResource importedRes_{};
519 : std::vector<GroupInfo> groupOpSizeInfo_;
520 : std::vector<CcuProfilingInfo> allCcuProfilingInfos_;
521 :
522 : // 记录每个kernel所需常量，适用于A6场景
523 : std::unordered_map<uint64_t, CcuRep::Variable> constValue2VarMap;
524 :
525 : struct LoopDescriptor {
526 : std::string label;
527 : std::shared_ptr<CcuRep::CcuRepLoopBlock> repLoopBlock;
528 : std::shared_ptr<CcuRep::CcuRepBlock> prevActiveBlock;
529 : bool bodyDefined{false};
530 : };
531 :
532 : struct VersionV2LoopRecord {
533 : CcuRep::Variable iterNumVar;
534 : CcuRep::Variable addrOffsetVar;
535 : CcuRep::Variable ctxIdVar;
536 : };
537 :
538 : struct LoopGroupDescriptor {
539 : CcuLoopGroupCfg config;
540 : uint64_t totalLoopNum{0};
541 : uint32_t loopCount{0};
542 : CcuRep::Variable parallelVar;
543 : CcuRep::Variable offsetVar;
544 : CcuRep::Variable xnOffsetVar;
545 : std::shared_ptr<CcuRep::CcuRepBase> bundleRep;
546 : bool isVarBased{false};
547 : bool isVersionV2{false};
548 : std::vector<VersionV2LoopRecord> versionV2Loops;
549 : };
550 :
551 : struct FuncDescriptor {
552 : const void* funcPtr{nullptr};
553 : std::string label;
554 : std::shared_ptr<CcuRep::CcuRepFuncBlock> repFuncBlock;
555 : std::shared_ptr<CcuRep::CcuRepBlock> prevActiveBlock;
556 : bool bodyDefined{false};
557 : };
558 :
559 : std::unordered_map<CcuLoop, LoopDescriptor> loopMap_;
560 : std::unordered_map<CcuLoopGroup, LoopGroupDescriptor> loopGroupMap_;
561 : CcuResult LookupLoopGroupAndLoop(
562 : CcuLoopGroup group, CcuLoop loop, const char* fnName, const char* createFnName, LoopGroupDescriptor*& grpDesc,
563 : LoopDescriptor*& loopDesc, uint32_t& loopIdx);
564 : uint32_t loopHandleCounter_{0};
565 : uint32_t loopGroupHandleCounter_{0};
566 : uint32_t loopBodyDepth_{0};
567 :
568 : std::unordered_map<uint64_t, FuncDescriptor> funcMap_;
569 : std::unordered_map<const void*, uint64_t> funcInstanceMap_;
570 : uint64_t funcHandleCounter_{0};
571 : bool inFuncBody_{false};
572 : // loop/func body 内首个非法错误的粘性闩，退出 body 时上抛（void body 无法直接回传）。
573 : CcuResult bodyError_{CcuResult::CCU_SUCCESS};
574 :
575 : std::unordered_map<CcuLoopExecutors, std::vector<CcuRep::Executor>> loopEnginePools_;
576 : uint32_t loopEnginePoolCounter_{0};
577 :
578 : std::string name_{};
579 : };
580 :
581 : } // namespace hcomm
582 :
583 : #endif // HCOMM_CCU_KERNEL_H
|
