Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #ifndef HCOMM_CCU_KERNEL_H
12 : #define HCOMM_CCU_KERNEL_H
13 :
14 : #include <cstdint>
15 : #include <functional>
16 : #include <array>
17 : #include <memory>
18 : #include <string>
19 : #include <vector>
20 : #include <unordered_map>
21 : #include <unordered_set>
22 :
23 : #include "ccu_task_param_v1.h"
24 :
25 : #include "ccu_kernel_resource.h"
26 : #include "ccu_instr_info_v1.h"
27 : #include "ccu_rep_context_v1.h"
28 :
29 : #include "ccu_funccall_v1.h"
30 : #include "ccu_loopcall_v1.h"
31 :
32 : #include "hccl_types.h"
33 : #include "hcomm_primitives.h"
34 : #include "ccu_datatype_v1.h"
35 : #include "ccu_interface_assist_v1.h"
36 :
37 : #include "ccu_res_repo.h"
38 : #include "ccu_dev_mgr_imp.h" // CcuVersion
39 :
40 : // 暂时引用方便算法开发
41 : #include "ccu_repeat_v1.h"
42 : #include "ccu_condition_v1.h"
43 : #include "ccu_rep_funcblock_v1.h"
44 : #include "ccu_loopblock_v1.h"
45 : #include "ccu_loopcall_v1.h"
46 : #include "ccu_assist_pub.h"
47 :
48 : #ifndef CCU_PROFILING // 和hccl仓兼容性使用
49 : #define CCU_PROFILING
50 : #endif
51 : #include "ccu_types.h"
52 :
53 : namespace hcomm {
54 :
55 : struct GroupInfo {
56 : uint16_t loopParamId;
57 : uint16_t parallelParamId;
58 : uint16_t residualId;
59 : };
60 :
61 : struct GroupOpConfig {
62 : uint32_t msInterleave;
63 : uint32_t loopCount;
64 : uint64_t memSlice;
65 : };
66 :
67 : // Kernel 查询信息结构体，由 CcuKernel::GetCcuKernelInfo 在锁内填充。
68 : // 后续新增查询字段时只需在此结构体中添加，无需新增接口。
69 : struct CcuKernelInfo {
70 : uint32_t maxTaskArgsNum{0}; // taskArgs数组所需的元素个数，注册期间由LoadArg的argId推算: max(argId)+1，空则为0
71 : };
72 :
73 : class CcuKernel : public CcuRep::CcuRepContext {
74 : public:
75 4635 : CcuKernel() = default;
76 : ~CcuKernel() override;
77 :
78 : HcclResult SetupProfilingInfo(const char* kernelFuncName);
79 : HcclResult ApplyDieFromChannels();
80 : HcclResult ValidateAndApplyDie(uint32_t targetDieId);
81 :
82 : CcuResReq GetResourceRequest();
83 : CcuResRepository& GetResRepository();
84 : CcuRepResource& GetResource();
85 : CcuSharedResource& GetExportedRes();
86 : CcuSharedResource& GetImportedRes();
87 :
88 : void SetResRepository(const CcuResRepository& resRepo);
89 : void SetInstrId(uint32_t instrId);
90 : uint32_t GetInstrId() const;
91 : uint32_t GetInstrCount();
92 : // 统计会翻译出 CKE 写者微码 (profiling -> setcke / 非 profiling -> clearcke) 的三种 wait 类 rep
93 : // 个数 (含 block 子 rep), 用于按 CCU_CKE_RAW_LATENCY 为每个此类 rep 预留指令空间.
94 : // 每个此类 rep 只发射 1 条 setcke / clearcke.
95 : uint32_t GetRepNeedToAddLatency() const;
96 : void SetCcuInstrInfo(const CcuRep::CcuInstrInfo& instrInfo);
97 :
98 : CcuResult GeneTaskParams(const uint64_t* taskArgs, uint32_t argsNum, std::vector<CcuTaskParam>& taskParams);
99 :
100 : void SetInsGenerater(CcuRep::CcuInsGeneratorBase* insGeneratorBase);
101 77 : void SetCcuVersion(CcuVersion version) { ccuVersion_ = version; }
102 : // 该友元函数用于在context类外创建Variable并被context内的资源管理器管理
103 : friend CcuRep::Variable CcuRep::CreateVariable(CcuRep::CcuRepContext* context);
104 :
105 : HcclResult AddProfilingInfo(
106 : const ChannelHandle* channels, uint32_t channelNum, HcclDataType dataType, HcclDataType outputDataType,
107 : HcclReduceOp opType, const std::string& opName);
108 :
109 : HcclResult AddCcuProfiling(
110 : GroupInfo groupInfo, const std::vector<ChannelHandle> channelHandle, HcclDataType dataType,
111 : HcclDataType outputDataType, HcclReduceOp opType, const std::string& opName);
112 : HcclResult AddCcuProfiling(
113 : const ChannelHandle* channels, uint32_t channelNum, HcclDataType dataType, HcclDataType outputDataType,
114 : HcclReduceOp opType, const std::string& opName);
115 : HcclResult
116 : GetCcuProfilingInfo(const uint64_t* taskArgs, uint32_t argSize, std::vector<CcuProfilingInfo>& allCcuProfilingInfo);
117 :
118 : const std::vector<CcuProfilingInfo>& GetAllCcuProfilingInfo() { return allCcuProfilingInfos_; };
119 :
120 : // process const values
121 212 : std::unordered_map<uint64_t, CcuRep::Variable>& GetConstValue2VarMap() { return constValue2VarMap; }
122 : HcclResult Add2ConstValue2VarMap(std::vector<uint64_t>& values);
123 :
124 8 : const std::unordered_set<ChannelHandle>& GetChannels() { return channels_; }
125 :
126 6 : const std::unordered_set<uint32_t>& GetDeclaredLocXns() const { return declaredLocXns_; }
127 :
128 : struct PinnedGroupEntry {
129 : CcuRep::Variable baseVar;
130 : uint16_t count;
131 : };
132 : const std::vector<PinnedGroupEntry>& GetPinnedRegGroups() const { return pinnedRegGroups_; }
133 : void AddPinnedRegGroup(const CcuRep::Variable& baseVar, uint16_t count);
134 :
135 : public:
136 : // Alloc 相关接口
137 : CcuResult VariableAlloc(CcuVariableHandle* varHandle);
138 : CcuResult AddressAlloc(CcuAddressHandle* addrHandle);
139 : CcuResult EventAlloc(CcuEventHandle* eventHandle);
140 : CcuResult BufferAlloc(CcuBufferHandle* bufHandle);
141 : CcuResult
142 : LocalAddrAlloc(CcuLocalAddrHandle* localAddrHandle, CcuAddressHandle* addrHandle, CcuVariableHandle* tokenHandle);
143 : CcuResult RemoteAddrAlloc(
144 : CcuRemoteAddrHandle* remoteAddrHandle, CcuAddressHandle* addrHandle, CcuVariableHandle* tokenHandle);
145 : CcuResult BlockVariableAlloc(CcuVariableHandle* varHandles, uint32_t count);
146 : CcuResult BlockEventAlloc(CcuEventHandle* eventHandles, uint32_t count);
147 : CcuResult BlockBufferAlloc(CcuBufferHandle* bufHandles, uint32_t count);
148 : CcuResult VariableCreateByChannel(ChannelHandle channel, uint32_t varIndex, CcuVariableHandle* varHandle);
149 : CcuResult VariableCreateByAcquire(CcuVariableHandle acqHandle, uint32_t index, CcuVariableHandle* varHandle);
150 : CcuResult EventCreateByAcquire(CcuEventHandle acqHandle, uint32_t index, CcuEventHandle* eventHandle);
151 :
152 : // 参数加载类 相关接口
153 : CcuResult LoadArg(CcuVariableHandle varHandle, uint32_t argId);
154 : // 在锁内填充 Kernel 查询信息；当前包含 maxTaskArgsNum。
155 : CcuResult GetCcuKernelInfo(CcuKernelInfo& info) const;
156 : CcuResult LoadVar(uint64_t addr, CcuVariableHandle varHandle, uint32_t num);
157 : CcuResult CcuLoadVarFromVarAddr(CcuVariableHandle addrHandle, CcuVariableHandle varHandle, uint32_t num);
158 : CcuResult StoreVar(uint64_t addr, CcuVariableHandle varHandle, uint32_t num);
159 : CcuResult CcuStoreVarToVarAddr(CcuVariableHandle addrHandle, CcuVariableHandle varHandle, uint32_t num);
160 :
161 : // Event信号同步类 相关接口
162 : // mask 由调用方独立传入（与 Event 句柄解耦），不再设独立的 SetEventMask 接口。
163 : CcuResult EventRecord(CcuEventHandle eventHandle, uint32_t mask);
164 : CcuResult EventWait(CcuEventHandle eventHandle, uint32_t mask);
165 : CcuResult NotifyRecord(const ChannelHandle channel, uint32_t remoteNotifyIdx, uint32_t mask);
166 : CcuResult NotifyWait(const ChannelHandle channel, uint32_t localNotifyIdx, uint32_t mask);
167 : CcuResult WriteVariableWithNotify(
168 : const ChannelHandle channel, CcuVariableHandle varHandle, uint32_t remoteVarIdx, uint32_t remoteNotifyIdx,
169 : uint32_t mask);
170 : // 本地（同 device 内跨 core）通知同步：用 notifyTag 字符串作为对端标识，
171 : // 由调用方约定生产者/消费者使用相同的 tag 字符串完成配对。
172 : // 与 NotifyRecord/Wait（用 ChannelHandle 标识跨 rank 通道）的对偶。
173 : // 必须 public：C API ccu_primitives_impl.cc 直接调用。
174 : CcuResult LocalNotifyRecord(const char* notifyTag, const uint32_t mask);
175 : CcuResult LocalNotifyWait(const char* notifyTag, const uint32_t mask);
176 : // 本地数据拷贝 相关接口
177 : CcuResult LocalCopyMemToBuffer(
178 : CcuBufferHandle dstHandle, CcuLocalAddrHandle srcHandle, CcuVariableHandle lenHandle,
179 : CcuEventHandle eventHandle, uint32_t mask);
180 : CcuResult LocalCopyBufferToMem(
181 : CcuLocalAddrHandle dstHandle, CcuBufferHandle srcHandle, CcuVariableHandle lenHandle,
182 : CcuEventHandle eventHandle, uint32_t mask);
183 : CcuResult LocalCopyMemToMem(
184 : CcuLocalAddrHandle dstHandle, CcuLocalAddrHandle srcHandle, CcuVariableHandle lenHandle,
185 : CcuEventHandle eventHandle, uint32_t mask);
186 :
187 : // 本地reduce 相关接口
188 : CcuResult LocalMemReduce(
189 : CcuLocalAddrHandle dstHandle, CcuLocalAddrHandle srcHandle, CcuVariableHandle lenHandle, HcclDataType dataType,
190 : HcclReduceOp opType, CcuEventHandle eventHandle, uint32_t mask);
191 : CcuResult LocalBufferReduce(
192 : CcuBufferHandle* bufHandles, uint32_t count, HcclDataType dataType, HcclDataType outputDataType,
193 : HcclReduceOp opType, CcuVariableHandle lenHandle, CcuEventHandle eventHandle, uint32_t mask);
194 :
195 : // 运算重载 相关接口
196 : CcuResult VariableAssignImm(CcuVariableHandle varHandle, uint64_t immediate);
197 : CcuResult VariableAssignVar(CcuVariableHandle varHandle, CcuVariableHandle varA);
198 : CcuResult
199 : VariableAddVarToVar(CcuVariableHandle varHandle, CcuVariableHandle varAHandle, CcuVariableHandle varBHandle);
200 : CcuResult
201 : VariableSubVarToVar(CcuVariableHandle varHandle, CcuVariableHandle varAHandle, CcuVariableHandle varBHandle);
202 : CcuResult
203 : VariableMulVarToVar(CcuVariableHandle varHandle, CcuVariableHandle varAHandle, CcuVariableHandle varBHandle);
204 : CcuResult VariableAddImmToVar(CcuVariableHandle varHandle, CcuVariableHandle varAHandle, uint16_t immediate);
205 : CcuResult VariableSubImmToVar(CcuVariableHandle varHandle, CcuVariableHandle varAHandle, uint16_t immediate);
206 : CcuResult VariableMulImmToVar(CcuVariableHandle varHandle, CcuVariableHandle varAHandle, uint16_t immediate);
207 : CcuResult
208 : VariableAndVarToVar(CcuVariableHandle varHandle, CcuVariableHandle varAHandle, CcuVariableHandle varBHandle);
209 : CcuResult
210 : VariableOrVarToVar(CcuVariableHandle varHandle, CcuVariableHandle varAHandle, CcuVariableHandle varBHandle);
211 : CcuResult
212 : VariableXorVarToVar(CcuVariableHandle varHandle, CcuVariableHandle varAHandle, CcuVariableHandle varBHandle);
213 : CcuResult VariableNotVar(CcuVariableHandle varHandle, CcuVariableHandle varAHandle);
214 : CcuResult
215 : VariableShlVarToVar(CcuVariableHandle varHandle, CcuVariableHandle varAHandle, CcuVariableHandle varBHandle);
216 : CcuResult
217 : VariableShrVarToVar(CcuVariableHandle varHandle, CcuVariableHandle varAHandle, CcuVariableHandle varBHandle);
218 : CcuResult AddressAssignImm(CcuAddressHandle addrHandle, uint64_t immediate);
219 : CcuResult AddressAssignVar(CcuAddressHandle addrHandle, CcuVariableHandle varHandle);
220 : CcuResult AddressAssignAddr(CcuAddressHandle dstAddrHandle, CcuAddressHandle srcAddrHandle);
221 : CcuResult
222 : AddressAddVarToAddr(CcuAddressHandle resAddrHandle, CcuAddressHandle lhsAddrHandle, CcuVariableHandle rhsVarHandle);
223 : CcuResult
224 : AddressAddAddrToAddr(CcuAddressHandle resAddrHandle, CcuAddressHandle addrAHandle, CcuAddressHandle addrBHandle);
225 : CcuResult AddressAddAssignVar(CcuAddressHandle addrHandle, CcuVariableHandle varHandle);
226 : CcuResult AddressAddAssignAddr(CcuAddressHandle addrHandle, CcuAddressHandle otherHandle);
227 : CcuResult AddressAddImmToAddr(CcuAddressHandle resAddrHandle, CcuAddressHandle addrAHandle, uint16_t imm);
228 :
229 : // 远端数据传输操作
230 :
231 : CcuResult ReadMemToMem(
232 : ChannelHandle channel, CcuLocalAddrHandle localHandle, CcuRemoteAddrHandle remoteHandle,
233 : CcuVariableHandle lenHandle, CcuEventHandle eventHandle, uint32_t mask);
234 : CcuResult ReadMemToBuffer(
235 : ChannelHandle channel, CcuBufferHandle localHandle, CcuRemoteAddrHandle remoteHandle,
236 : CcuVariableHandle lenHandle, CcuEventHandle eventHandle, uint32_t mask);
237 : CcuResult ReadMemToMemReduce(
238 : ChannelHandle channel, CcuLocalAddrHandle localHandle, CcuRemoteAddrHandle remoteHandle,
239 : CcuVariableHandle lenHandle, HcclDataType dataType, HcclReduceOp opType, CcuEventHandle eventHandle,
240 : uint32_t mask);
241 : CcuResult WriteMemToMem(
242 : ChannelHandle channel, CcuRemoteAddrHandle remoteHandle, CcuLocalAddrHandle localHandle,
243 : CcuVariableHandle lenHandle, CcuEventHandle eventHandle, uint32_t mask);
244 : CcuResult WriteBufferToMem(
245 : ChannelHandle channel, CcuRemoteAddrHandle remoteHandle, CcuBufferHandle localHandle,
246 : CcuVariableHandle lenHandle, CcuEventHandle eventHandle, uint32_t mask);
247 : CcuResult WriteMemToMemReduce(
248 : ChannelHandle channel, CcuRemoteAddrHandle remoteHandle, CcuLocalAddrHandle localHandle,
249 : CcuVariableHandle lenHandle, HcclDataType dataType, HcclReduceOp opType, CcuEventHandle eventHandle,
250 : uint32_t mask);
251 :
252 : CcuResult IfBegin(CcuVariableHandle varHandle, uint64_t immediate, CcuConditionType condType, const char* label);
253 : CcuResult
254 : IfBeginVar(CcuVariableHandle lhsHandle, CcuVariableHandle rhsHandle, CcuConditionType condType, const char* label);
255 : CcuResult IfElse(const char* label);
256 : CcuResult IfEnd(const char* label);
257 :
258 : void IfLabelStackPush(const char* label);
259 : void IfLabelStackMarkBodyDone();
260 : const char* IfLabelStackPopForElse();
261 : bool IfLabelStackTopIsClosable();
262 : const char* IfLabelStackPop();
263 :
264 : void DoWhileLabelStackPush(const char* label);
265 : const char* DoWhileLabelStackPopForWhile();
266 :
267 : void FlushClosablePendingIfs();
268 : void Append(std::shared_ptr<CcuRep::CcuRepBase> rep) override;
269 : CcuResult WhileBegin(CcuVariableHandle varHandle, uint64_t immediate, CcuConditionType condType, const char* label);
270 : CcuResult WhileBeginVar(
271 : CcuVariableHandle lhsHandle, CcuVariableHandle rhsHandle, CcuConditionType condType, const char* label);
272 : CcuResult WhileEnd(const char* label);
273 : CcuResult DoWhileBegin(const char* label);
274 : CcuResult DoWhileEnd(CcuVariableHandle varHandle, uint64_t immediate, CcuConditionType condType, const char* label);
275 : CcuResult DoWhileEndVar(
276 : CcuVariableHandle lhsHandle, CcuVariableHandle rhsHandle, CcuConditionType condType, const char* label);
277 :
278 : CcuResult LoopCreate(CcuLoop* loop);
279 : CcuResult LoopBodyEnter(CcuLoop loop);
280 : CcuResult LoopBodyExit(CcuLoop loop);
281 : // 建组/加环统一以版本化 cfg 为准;旧 config 的转换在 C ABI 适配层完成。
282 : CcuResult LoopGroupCreate(CcuLoopGroup* group, uint32_t maxLoopNum, const CcuLoopGroupCfg* cfg);
283 : CcuResult LoopGroupCreateFromVar(
284 : CcuLoopGroup* group, uint32_t maxLoopNum, CcuVariableHandle parallelVarHandle,
285 : CcuVariableHandle offsetVarHandle);
286 : CcuResult LoopGroupCreateFromVarV2(
287 : CcuLoopGroup* group, uint32_t maxLoopNum, CcuVariableHandle parallelVarV2Handle,
288 : CcuVariableHandle offsetVarV2Handle, CcuVariableHandle varOffsetVarHandle);
289 : CcuResult LoopGroupAddLoop(CcuLoopGroup group, CcuLoop loop, const CcuLoopCfg* cfg);
290 : CcuResult LoopGroupAddLoopFromVar(CcuLoopGroup group, CcuLoop loop, CcuVariableHandle loopParamVarHandle);
291 : CcuResult LoopGroupAddLoopFromVarV2(
292 : CcuLoopGroup group, CcuLoop loop, CcuVariableHandle iterNumVarHandle, CcuVariableHandle addrOffsetVarHandle,
293 : CcuVariableHandle ctxIdVarHandle);
294 :
295 : CcuResult FuncBlockLookup(const void* funcPtr, uint64_t* outHandle);
296 : CcuResult FuncBlockBegin(const void* funcPtr, uint64_t* outHandle);
297 : CcuResult FuncBlockEnd(uint64_t handle);
298 : CcuResult FuncDefineInArg(uint64_t handle, CcuVariableHandle formal);
299 : CcuResult FuncCall(uint64_t handle, const CcuVariableHandle* inArgs, uint32_t numIn);
300 :
301 : private:
302 : CcuResult GetVariableByHandle(CcuVariableHandle varHandle, CcuRep::Variable** variable);
303 : CcuResult GetEventByHandle(CcuEventHandle eventHandle, CcuRep::CompletedEvent** event);
304 : CcuResult LatchBodyError(CcuResult err);
305 : CcuResult EnsureLoopEnginePool(uint32_t maxLoopNum);
306 :
307 : CcuResult ValidateTaskArgs(const uint64_t* taskArgs, uint32_t argsNum) const;
308 : void FillTaskParam(
309 : CcuTaskParam& param, uint32_t index, uint32_t seqNum, const uint64_t* taskArgs, uint32_t argsNum) const;
310 :
311 : CcuResult ResolveBufRemoteLenEvent(
312 : CcuBufferHandle bufHandle, CcuRemoteAddrHandle remoteHandle, CcuVariableHandle lenHandle,
313 : CcuEventHandle eventHandle, CcuRep::CcuBuf** buf, CcuRep::RemoteAddr** remote, CcuRep::Variable** len,
314 : CcuRep::CompletedEvent** event);
315 :
316 : CcuResult ResolveLocalRemoteLenEvent(
317 : CcuLocalAddrHandle localHandle, CcuRemoteAddrHandle remoteHandle, CcuVariableHandle lenHandle,
318 : CcuEventHandle eventHandle, CcuRep::LocalAddr** local, CcuRep::RemoteAddr** remote, CcuRep::Variable** len,
319 : CcuRep::CompletedEvent** event);
320 :
321 : CcuResult ResolveRemoteLocalLenEvent(
322 : CcuRemoteAddrHandle remoteHandle, CcuLocalAddrHandle localHandle, CcuVariableHandle lenHandle,
323 : CcuEventHandle eventHandle, CcuRep::RemoteAddr** remote, CcuRep::LocalAddr** local, CcuRep::Variable** len,
324 : CcuRep::CompletedEvent** event);
325 :
326 : // 校验从 varHandle 起的 num 个 Variable 句柄对应的内部变量 Id 连续递增，
327 : // 用于 LoadVar/StoreVar 等接口对“连续变量块”的前置校验。
328 : CcuResult CheckContinuousVariables(
329 : CcuVariableHandle varHandle, uint32_t num, const CcuRep::Variable& baseVar, const char* tag);
330 :
331 : // GetCcuProfilingInfo 的子步骤：处理 sqe & waitcke 类型的 profiling 信息，结果直接 push 到 allCcuProfilingInfos_
332 : // 中。
333 : HcclResult CollectSqeAndWaitCkeProfilingInfo();
334 : // GetCcuProfilingInfo 的子步骤：根据 LoopGroup 的 profiling 缓存构建
335 : // varId -> argIndex 与 varId -> varId 两个查找表，供 LoopGroup 段查询入参使用。
336 : HcclResult BuildLoopGroupVarIdMaps(
337 : std::unordered_map<uint16_t, uint32_t>& varId2ArgIndexMap,
338 : std::unordered_map<uint16_t, uint16_t>& varId2VarIdMap);
339 : // GetCcuProfilingInfo 的子步骤：处理 LoopGroup 的 profiling 信息，结果 push 到 allCcuProfilingInfos_ 中。
340 : HcclResult CollectLoopGroupProfilingInfo(
341 : const uint64_t* taskArgs, uint32_t argSize, const std::unordered_map<uint16_t, uint32_t>& varId2ArgIndexMap,
342 : const std::unordered_map<uint16_t, uint16_t>& varId2VarIdMap);
343 : // CollectLoopGroupProfilingInfo 的单次循环体：处理下标 i 对应的 loopParam 与 parallelParam 分支。
344 : HcclResult CollectSingleLoopGroupProfiling(
345 : uint32_t i, size_t repSize, const uint64_t* taskArgs, uint32_t argSize,
346 : const std::unordered_map<uint16_t, uint32_t>& varId2ArgIndexMap,
347 : const std::unordered_map<uint16_t, uint16_t>& varId2VarIdMap);
348 :
349 : CcuRep::Variable CreateJumpTargetVar();
350 :
351 : struct IfLabelEntry {
352 : const char* label{nullptr};
353 : bool bodyDone{false};
354 : };
355 : struct DoWhileLabelEntry {
356 : const char* label{nullptr};
357 : std::shared_ptr<CcuRep::CcuRepBlock> snapshotBlock{nullptr};
358 : size_t snapshotRepCount{0};
359 : };
360 : std::vector<IfLabelEntry> iflabelStack_;
361 : std::vector<DoWhileLabelEntry> doWhileLabelStack_;
362 : bool isFlushing_ = false;
363 :
364 : struct PendingIfContext {
365 : std::shared_ptr<CcuRep::CcuRepJumpLabel> elseLabel;
366 : std::shared_ptr<CcuRep::CcuRepJumpLabel> endLabel;
367 : bool hasElse{false};
368 : };
369 :
370 : struct PendingWhileContext {
371 : std::shared_ptr<CcuRep::CcuRepJumpLabel> beginLabel;
372 : std::shared_ptr<CcuRep::CcuRepJumpLabel> endLabel;
373 : CcuVariableHandle varHandle;
374 : uint64_t immediate;
375 : CcuConditionType condType;
376 : };
377 :
378 : struct PendingDoWhileContext {
379 : std::shared_ptr<CcuRep::CcuRepJumpLabel> beginLabel;
380 : };
381 :
382 : std::unordered_map<CcuVariableHandle, CcuRep::Variable> ccuVarMap_{};
383 :
384 : std::unordered_map<std::string, PendingIfContext> pendingIfCtx_{};
385 : std::unordered_map<std::string, PendingWhileContext> pendingWhileCtx_{};
386 : std::unordered_map<std::string, PendingDoWhileContext> pendingDoWhileCtx_{};
387 :
388 : std::unordered_map<CcuEventHandle, CcuRep::CompletedEvent> ccuEventMap_{};
389 :
390 : CcuResult GetBufferByHandle(CcuBufferHandle bufferHandle, CcuRep::CcuBuf** buffer);
391 : std::unordered_map<CcuBufferHandle, CcuRep::CcuBuf> ccuBufferMap_{};
392 :
393 : CcuResult GetAddressByHandle(CcuAddressHandle addrHandle, CcuRep::Address** address);
394 : std::unordered_map<CcuAddressHandle, CcuRep::Address> ccuAddrMap_{};
395 :
396 : CcuResult GetLocalAddrByHandle(CcuLocalAddrHandle handle, CcuRep::LocalAddr** localAddr);
397 : std::unordered_map<CcuLocalAddrHandle, CcuRep::LocalAddr> ccuLocalAddrMap_{};
398 :
399 : CcuResult GetRemoteAddrByHandle(CcuRemoteAddrHandle handle, CcuRep::RemoteAddr** remoteAddr);
400 : std::unordered_map<CcuRemoteAddrHandle, CcuRep::RemoteAddr> ccuRemoteAddrMap_{};
401 :
402 : std::unordered_set<uint32_t> loadArgUsedSet_{};
403 :
404 : protected:
405 : // 使用channel中的Variable
406 : HcclResult CreateVariable(const ChannelHandle channel, uint32_t varIndex, CcuRep::Variable* var);
407 : CcuRep::Variable CreateVariable();
408 : CcuRep::Variable CreateExpectVar();
409 : CcuRep::Variable CreateContinuousVariable();
410 : CcuRep::LocalAddr CreateLocalAddr();
411 : CcuRep::RemoteAddr CreateRemoteAddr();
412 : CcuRep::RemoteAddr GetRemoteAddr(const ChannelHandle channel, const uint32_t index);
413 : CcuRep::LocalNotify CreateLocalNotify();
414 : CcuRep::CompletedEvent CreateCompletedEvent();
415 : CcuRep::CcuBuf CreateCcuBuf();
416 : CcuRep::Executor CreateExecutor();
417 :
418 : HcclResult CreateBlockCcuBuf(const uint32_t count, CcuRep::CcuBuf* ccuBufs);
419 : HcclResult CreateBlockExecutor(const uint32_t count, CcuRep::Executor* ccuExes);
420 : HcclResult CreateBlockCompletedEvent(const uint32_t count, CcuRep::CompletedEvent* ccuEvents);
421 :
422 : // 内部 *Nb / RecordEvent / WaitEvent 系列：mask 由调用方独立传入，
423 : // 不再从 CompletedEvent 上读取。
424 : HcclResult RecordEvent(CcuRep::CompletedEvent event, uint32_t mask);
425 : HcclResult WaitEvent(CcuRep::CompletedEvent event, uint32_t mask);
426 :
427 : // 数据操作
428 : HcclResult WriteNb(
429 : const ChannelHandle channel, const CcuRep::RemoteAddr& rem, const CcuRep::LocalAddr& loc,
430 : const CcuRep::Variable& len, CcuRep::CompletedEvent event, uint32_t mask);
431 : HcclResult WriteNb(
432 : const ChannelHandle channel, const CcuRep::RemoteAddr& rem, const CcuRep::CcuBuf& loc,
433 : const CcuRep::Variable& len, CcuRep::CompletedEvent event, uint32_t mask);
434 :
435 : HcclResult ReadNb(
436 : const ChannelHandle channel, const CcuRep::LocalAddr& loc, const CcuRep::RemoteAddr& rem,
437 : const CcuRep::Variable& len, CcuRep::CompletedEvent event, uint32_t mask);
438 : HcclResult ReadNb(
439 : const ChannelHandle channel, const CcuRep::CcuBuf& loc, const CcuRep::RemoteAddr& rem,
440 : const CcuRep::Variable& len, CcuRep::CompletedEvent event, uint32_t mask);
441 :
442 : HcclResult WriteReduceNb(
443 : const ChannelHandle channel, const CcuRep::RemoteAddr& rem, const CcuRep::LocalAddr& loc,
444 : const CcuRep::Variable& len, HcclDataType dataType, HcclReduceOp opType, CcuRep::CompletedEvent event,
445 : uint32_t mask);
446 : HcclResult ReadReduceNb(
447 : const ChannelHandle channel, const CcuRep::LocalAddr& loc, const CcuRep::RemoteAddr& rem,
448 : const CcuRep::Variable& len, HcclDataType dataType, HcclReduceOp opType, CcuRep::CompletedEvent event,
449 : uint32_t mask);
450 :
451 : HcclResult LocalCopyNb(
452 : const CcuRep::LocalAddr& dst, const CcuRep::LocalAddr& src, const CcuRep::Variable& len,
453 : CcuRep::CompletedEvent event, uint32_t mask); // dst和src是否都是local
454 : HcclResult LocalCopyNb(
455 : const CcuRep::CcuBuf& dst, const CcuRep::LocalAddr& src, const CcuRep::Variable& len,
456 : CcuRep::CompletedEvent event, uint32_t mask);
457 : HcclResult LocalCopyNb(
458 : const CcuRep::LocalAddr& dst, const CcuRep::CcuBuf& src, const CcuRep::Variable& len,
459 : CcuRep::CompletedEvent event, uint32_t mask);
460 :
461 : HcclResult LocalReduceNb(
462 : const CcuRep::LocalAddr& dst, const CcuRep::LocalAddr& src, const CcuRep::Variable& len, HcclDataType dataType,
463 : HcclReduceOp opType, CcuRep::CompletedEvent event, uint32_t mask);
464 : HcclResult LocalReduceNb(
465 : const CcuRep::CcuBuf* bufs, uint32_t count, HcclDataType dataType, HcclDataType outputDataType,
466 : HcclReduceOp opType, const CcuRep::Variable& len, CcuRep::CompletedEvent event, uint32_t mask);
467 :
468 : // 参数操作
469 : void Load(const CcuRep::Variable& var);
470 :
471 : // Variable src中存放内存地址，从地址中加载数据到Variable var中
472 : void LoadVariable(const CcuRep::Variable& src, const CcuRep::Variable& var);
473 :
474 : void StoreVariable(const CcuRep::Variable& var, uint64_t addr);
475 : // 控制逻辑
476 : // 宏定义IF、WHILE
477 : CcuRep::FuncCall Func(const std::string& label);
478 : CcuRep::FuncCall Func(const CcuRep::Variable& funcAddr);
479 : CcuRep::LoopCall Loop(const std::string& label);
480 :
481 : private:
482 : CcuRep::Address CreateAddress();
483 : CcuRep::LocalAddr CreateLocalAddr(const CcuRep::Variable& token);
484 :
485 : protected:
486 : GroupOpConfig moConfig_{0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFFFFFFFFFF};
487 :
488 : private:
489 : template <typename T>
490 : T CreateResAssist(std::array<std::vector<T>, CCU_MAX_IODIE_NUM>& resRecord);
491 : template <typename T>
492 : std::vector<T> CreateBlockResAssist(const uint32_t count, std::array<std::vector<T>, CCU_MAX_IODIE_NUM>& resRecord);
493 :
494 : private:
495 : CcuRepResource res_{};
496 : CcuResRepository resRepo_{};
497 :
498 : std::unordered_set<ChannelHandle> channels_{};
499 :
500 : std::unordered_set<uint32_t> declaredLocXns_{};
501 :
502 : std::vector<PinnedGroupEntry> pinnedRegGroups_{};
503 :
504 : CcuRep::CcuInstrInfo instrInfo_{};
505 :
506 : uint32_t loadArgIndex_{0};
507 :
508 : CcuVersion ccuVersion_{CcuVersion::CCU_INVALID};
509 :
510 : // A5(CCU_V1) 专用：CCU_IF,CCU_ELSE 跳转共用一个 Xn；当前仅CCU_IF和CCU_ELSE复用，While和DoWhile暂未复用
511 : // A5 上 CCU_IF和ELSE的targetVar 仅用于LoadImdToXn + 紧接 Jump，生命周期不跨越其他指令，可安全复用。
512 : // A6不复用，保持每次独立分配的语义。
513 : std::unique_ptr<CcuRep::Variable> sharedJumpTargetVar_;
514 :
515 : CcuSharedResource exportedRes_{};
516 : CcuSharedResource importedRes_{};
517 : std::vector<GroupInfo> groupOpSizeInfo_;
518 : std::vector<CcuProfilingInfo> allCcuProfilingInfos_;
519 :
520 : // 记录每个kernel所需常量，适用于A6场景
521 : std::unordered_map<uint64_t, CcuRep::Variable> constValue2VarMap;
522 :
523 : struct LoopDescriptor {
524 : std::string label;
525 : std::shared_ptr<CcuRep::CcuRepLoopBlock> repLoopBlock;
526 : std::shared_ptr<CcuRep::CcuRepBlock> prevActiveBlock;
527 : bool bodyDefined{false};
528 : };
529 :
530 : struct VersionV2LoopRecord {
531 : CcuRep::Variable iterNumVar;
532 : CcuRep::Variable addrOffsetVar;
533 : CcuRep::Variable ctxIdVar;
534 : };
535 :
536 : struct LoopGroupDescriptor {
537 : CcuLoopGroupCfg config;
538 : uint64_t totalLoopNum{0};
539 : uint32_t loopCount{0};
540 : CcuRep::Variable parallelVar;
541 : CcuRep::Variable offsetVar;
542 : CcuRep::Variable xnOffsetVar;
543 : std::shared_ptr<CcuRep::CcuRepBase> bundleRep;
544 : bool isVarBased{false};
545 : bool isVersionV2{false};
546 : std::vector<VersionV2LoopRecord> versionV2Loops;
547 : };
548 :
549 : struct FuncDescriptor {
550 : const void* funcPtr{nullptr};
551 : std::string label;
552 : std::shared_ptr<CcuRep::CcuRepFuncBlock> repFuncBlock;
553 : std::shared_ptr<CcuRep::CcuRepBlock> prevActiveBlock;
554 : bool bodyDefined{false};
555 : };
556 :
557 : std::unordered_map<CcuLoop, LoopDescriptor> loopMap_;
558 : std::unordered_map<CcuLoopGroup, LoopGroupDescriptor> loopGroupMap_;
559 : CcuResult LookupLoopGroupAndLoop(
560 : CcuLoopGroup group, CcuLoop loop, const char* fnName, const char* createFnName, LoopGroupDescriptor*& grpDesc,
561 : LoopDescriptor*& loopDesc, uint32_t& loopIdx);
562 : uint32_t loopHandleCounter_{0};
563 : uint32_t loopGroupHandleCounter_{0};
564 : uint32_t loopBodyDepth_{0};
565 :
566 : std::unordered_map<uint64_t, FuncDescriptor> funcMap_;
567 : std::unordered_map<const void*, uint64_t> funcInstanceMap_;
568 : uint64_t funcHandleCounter_{0};
569 : bool inFuncBody_{false};
570 : // loop/func body 内首个非法错误的粘性闩，退出 body 时上抛（void body 无法直接回传）。
571 : CcuResult bodyError_{CcuResult::CCU_SUCCESS};
572 :
573 : std::unordered_map<CcuLoopExecutors, std::vector<CcuRep::Executor>> loopEnginePools_;
574 : uint32_t loopEnginePoolCounter_{0};
575 :
576 : std::string name_{};
577 : };
578 :
579 : } // namespace hcomm
580 :
581 : #endif // HCOMM_CCU_KERNEL_H
|
