Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "ccu_kernel.h"
12 : #include "ccu_rep_v1.h"
13 : #include "ccu_kernel_resource.h"
14 : #include "ccu_microcode_v1.h"
15 :
16 : #include "ccu_types.h"
17 : #include "exception_util.h"
18 : #include "ccu_api_exception.h"
19 : #include "ccu_dev_mgr_imp.h"
20 : #include "env_config.h"
21 : #include "ccu_rep_type_v1.h"
22 :
23 : #include "hcomm_c_adpt.h"
24 :
25 : #include "ccu_rep_context_v1.h"
26 : #include "ccu_rep_funccall_v1.h"
27 : #include "../../endpoint_pairs/channels/ccu/ccu_urma_channel.h"
28 : #include "../ccu_instance/ccu_var_event_res_mgr.h"
29 :
30 : #include "ccu_log.h"
31 :
32 : #include "hcom_common.h"
33 :
34 : // todo: 引入头文件需要检查
35 : #include "ccu_assist_v1.h"
36 : #include "hccl_comm_pub.h"
37 : #include "hcclCommDfx.h"
38 : #include "task_info.h"
39 : #include "task_param.h"
40 :
41 : #include "ccu_ins_generator_base.h"
42 : #include "ccu_ins_generator_v1.h"
43 : #include "unified_platform/pub_inc/config_plf_log.h"
44 :
45 : namespace hcomm {
46 :
47 : using Hccl::PLF_DATA_OP;
48 :
49 : constexpr uint32_t TOKEN_VALUE_INDEX = 2;
50 : constexpr uint16_t INVALID_U16 = 65535;
51 :
52 : using CcuRep::CcuInsGeneratorBase;
53 : using CcuRep::CcuInsGeneratorV1;
54 :
55 : template <typename T>
56 213 : T CcuKernel::CreateResAssist(std::array<std::vector<T>, CCU_MAX_IODIE_NUM>& resRecord)
57 : {
58 : // kernel确认die之前默认为0，需要刷新资源
59 : // 确认die之后按实际使用die分配资源
60 213 : const uint32_t dieId = GetDieId();
61 213 : resRecord[dieId].emplace_back(this);
62 213 : auto& item = resRecord[dieId].back();
63 213 : item.Reset(resRecord[dieId].size(), dieId);
64 213 : return item;
65 : }
66 :
67 : template <typename T>
68 : std::vector<T>
69 893 : CcuKernel::CreateBlockResAssist(const uint32_t count, std::array<std::vector<T>, CCU_MAX_IODIE_NUM>& resRecord)
70 : {
71 893 : constexpr uint16_t CCU_BLOCK_RES_ID_BASE = 0x1000; // block 批量分配资源 id 基址，与单资源 id 区间隔离便于 DFX 定位
72 893 : std::vector<T> block;
73 893 : block.reserve(count);
74 893 : const uint32_t dieId = GetDieId();
75 2162 : for (size_t i = 0; i < count; i++) {
76 1269 : block.emplace_back(this);
77 1269 : block.back().Reset(static_cast<uint16_t>(CCU_BLOCK_RES_ID_BASE + resRecord[dieId].size() + i), dieId);
78 : }
79 893 : resRecord[dieId].insert(resRecord[dieId].end(), block.begin(), block.end());
80 893 : return block;
81 0 : }
82 :
83 155 : CcuKernel::~CcuKernel() {}
84 :
85 11 : static HcclResult GetDieIdByChannel(const ChannelHandle channel, uint32_t& dieId)
86 : {
87 11 : void* channelPtr{nullptr};
88 11 : CHK_RET(static_cast<HcclResult>(HcommChannelGet(channel, &channelPtr)));
89 11 : auto* channelImpl = dynamic_cast<CcuUrmaChannel*>(static_cast<Channel*>(channelPtr));
90 11 : if (channelImpl == nullptr) {
91 0 : HCCL_ERROR("[%s] failed to cast channel[0x%llx] to CcuUrmaChannel", __func__, channel);
92 0 : return HcclResult::HCCL_E_PTR;
93 : }
94 11 : dieId = channelImpl->GetDieId();
95 11 : HCCL_INFO("[%s], channelHandle[0x%llx], dieId[%u]", __func__, channel, dieId);
96 11 : return HcclResult::HCCL_SUCCESS;
97 : }
98 :
99 0 : static HcclResult GetDieIdByChannels(const std::unordered_set<ChannelHandle>& channels, uint32_t& dieId)
100 : {
101 0 : if (channels.empty()) {
102 0 : int32_t devLogicId = HcclGetThreadDeviceId();
103 0 : for (uint32_t die = 0; die < CCU_MAX_IODIE_NUM; die++) {
104 0 : bool enableFlag = false;
105 0 : CHK_RET(static_cast<HcclResult>(CcuGetDieEnableInfo(devLogicId, die, enableFlag)));
106 0 : if (enableFlag) {
107 0 : dieId = die;
108 0 : return HcclResult::HCCL_SUCCESS;
109 : }
110 : }
111 :
112 0 : HCCL_ERROR("[CcuKernel][%s] failed, all dies are disable, devLogicId[%d].", __func__, devLogicId);
113 0 : return HcclResult::HCCL_E_INTERNAL;
114 : }
115 :
116 0 : uint32_t firstDieId = 0;
117 0 : CHK_RET(GetDieIdByChannel(*channels.begin(), firstDieId));
118 0 : for (const auto channel : channels) {
119 0 : uint32_t nextDieId = 0;
120 0 : CHK_RET(GetDieIdByChannel(channel, nextDieId));
121 0 : if (firstDieId != nextDieId) {
122 0 : HCCL_ERROR("[%s] failed, the dies of channels are not same.", __func__);
123 0 : return HcclResult::HCCL_E_PARA;
124 : }
125 : }
126 :
127 0 : dieId = firstDieId;
128 0 : return HcclResult::HCCL_SUCCESS;
129 : }
130 :
131 57 : static HcclResult CheckChannelsDie(const std::unordered_set<ChannelHandle>& channels, const uint32_t targetDieId)
132 : {
133 65 : for (const auto channel : channels) {
134 11 : uint32_t channelDieId = 0;
135 14 : CHK_RET(GetDieIdByChannel(channel, channelDieId));
136 11 : if (channelDieId != targetDieId) {
137 3 : HCCL_ERROR(
138 : "[%s] failed, channel[0x%llx] dieId[%u] differs from target dieId[%u].", __func__, channel,
139 : channelDieId, targetDieId);
140 3 : return HcclResult::HCCL_E_PARA;
141 : }
142 : }
143 54 : return HcclResult::HCCL_SUCCESS;
144 : }
145 :
146 54 : static void MoveResourcesToDie(CcuRepResource& res, uint32_t targetDieId)
147 : {
148 54 : if (targetDieId == 0)
149 53 : return; // 初始资源位于die0，不用设置
150 :
151 11 : auto moveAndSet = [&](auto& arr) {
152 11 : arr[targetDieId] = std::move(arr[0]);
153 11 : for (auto& item : arr[targetDieId])
154 0 : item.SetDieId(targetDieId);
155 12 : };
156 :
157 1 : moveAndSet(res.ccubufs);
158 1 : moveAndSet(res.blockCcubufs);
159 1 : moveAndSet(res.executor);
160 1 : moveAndSet(res.blockExecutor);
161 1 : moveAndSet(res.completedEvent);
162 1 : moveAndSet(res.blockCompletedEvent);
163 1 : moveAndSet(res.address);
164 1 : moveAndSet(res.blockAddress);
165 1 : moveAndSet(res.continuousVariable);
166 1 : moveAndSet(res.variable);
167 1 : moveAndSet(res.localNotify);
168 : }
169 :
170 70 : HcclResult CcuKernel::SetupProfilingInfo(const char* kernelFuncName)
171 : {
172 70 : if (kernelFuncName == nullptr || strlen(kernelFuncName) == 0) {
173 0 : name_ = std::string("CCU_KERNEL"); // 默认名称
174 0 : AddSqeProfiling(name_);
175 0 : return HcclResult::HCCL_SUCCESS;
176 : }
177 :
178 70 : constexpr size_t MAX_KERNEL_FUNC_NAME_LEN = 128;
179 70 : const auto nameLen = strlen(kernelFuncName);
180 70 : if (nameLen > MAX_KERNEL_FUNC_NAME_LEN) {
181 0 : name_ = std::string(kernelFuncName, MAX_KERNEL_FUNC_NAME_LEN);
182 0 : HCCL_WARNING("[CcuKernel][%s] kernelFuncName is too long, reset to %s.", __func__, name_.c_str());
183 : }
184 :
185 : // 生成SQE粒度profiling信息，此时未选择die，默认die 0
186 70 : AddSqeProfiling(name_);
187 70 : return HcclResult::HCCL_SUCCESS;
188 : }
189 :
190 : static HcclResult
191 54 : UpdateProfilingInfo(std::vector<CcuProfilingInfo>& profilingInfos, uint32_t dieId, const std::string& kernelName)
192 : {
193 54 : if (dieId == 0) {
194 : // 与默认dieId相同，不需要修改
195 53 : return HcclResult::HCCL_SUCCESS;
196 : }
197 :
198 : // 正常情况仅首个info包含die信息，仅应为CCU_TASK_PROFILING类型
199 1 : if (UNLIKELY(profilingInfos.empty())) {
200 : // profiling不属于主流程，不打断算子业务
201 0 : HCCL_INFO("[%s] passed, profiling infos are empty, ccu kernel func[%s].", __func__, kernelName.c_str());
202 0 : return HcclResult::HCCL_SUCCESS;
203 : }
204 :
205 : // 根据选择的die跟新profiling信息
206 2 : for (auto& info : profilingInfos) {
207 1 : info.dieId = dieId;
208 : }
209 :
210 1 : HCCL_INFO("[%s] reset profiling info dieId to [%u], ccu kernel func[%s].", __func__, dieId, kernelName.c_str());
211 1 : return HcclResult::HCCL_SUCCESS;
212 : }
213 :
214 0 : HcclResult CcuKernel::ApplyDieFromChannels()
215 : {
216 0 : uint32_t dieId{0};
217 0 : CHK_RET(GetDieIdByChannels(channels_, dieId));
218 0 : CHK_PRT_RET(
219 : dieId >= CCU_MAX_IODIE_NUM,
220 : HCCL_ERROR("[CcuKernel][%s] failed, dieId[%u] should be less than [%u].", __func__, dieId, CCU_MAX_IODIE_NUM),
221 : HcclResult::HCCL_E_PARA);
222 0 : SetDieId(dieId);
223 0 : MoveResourcesToDie(res_, dieId);
224 0 : (void)UpdateProfilingInfo(profilingInfo, dieId, name_);
225 :
226 0 : return HcclResult::HCCL_SUCCESS;
227 : }
228 :
229 57 : HcclResult CcuKernel::ValidateAndApplyDie(const uint32_t targetDieId)
230 : {
231 57 : CHK_PRT_RET(
232 : targetDieId >= CCU_MAX_IODIE_NUM,
233 : HCCL_ERROR(
234 : "[CcuKernel][%s] failed, dieId[%u] should be less than [%u].", __func__, targetDieId, CCU_MAX_IODIE_NUM),
235 : HcclResult::HCCL_E_PARA);
236 :
237 57 : const int32_t devLogicId = HcclGetThreadDeviceId();
238 57 : bool enableFlag = false;
239 57 : CHK_RET(static_cast<HcclResult>(CcuGetDieEnableInfo(devLogicId, static_cast<uint8_t>(targetDieId), enableFlag)));
240 57 : CHK_PRT_RET(
241 : !enableFlag,
242 : HCCL_ERROR(
243 : "[CcuKernel][%s] failed, target dieId[%u] is disabled, devLogicId[%d].", __func__, targetDieId, devLogicId),
244 : HcclResult::HCCL_E_PARA);
245 57 : CHK_RET(CheckChannelsDie(channels_, targetDieId));
246 :
247 54 : SetDieId(targetDieId);
248 54 : MoveResourcesToDie(res_, targetDieId);
249 54 : (void)UpdateProfilingInfo(profilingInfo, targetDieId, name_);
250 :
251 54 : return HcclResult::HCCL_SUCCESS;
252 : }
253 :
254 73 : void CcuKernel::SetInsGenerater(CcuInsGeneratorBase* insGeneratorBase) { insGenerator = insGeneratorBase; }
255 :
256 5 : CcuResult CcuKernel::ValidateTaskArgs(const uint64_t* taskArgs, uint32_t argsNum) const
257 : {
258 5 : if (loadArgUsedSet_.size() != argsNum) {
259 0 : HCCL_ERROR(
260 : "[CcuKernel][%s] failed, args number does not match the Load instruction, "
261 : "argsNum = %u, loaded = %zu",
262 : __func__, argsNum, loadArgUsedSet_.size());
263 0 : return CcuResult::CCU_E_INTERNAL;
264 : }
265 50 : for (uint32_t i = 0; i < argsNum; ++i) {
266 45 : if (loadArgUsedSet_.count(i) == 0) {
267 0 : HCCL_ERROR("[CcuKernel][%s] failed, argId %u not loaded (argsNum=%u)", __func__, i, argsNum);
268 0 : return CcuResult::CCU_E_INTERNAL;
269 : }
270 : }
271 5 : if (argsNum != 0) {
272 3 : CCU_CHK_PTR_NULL(taskArgs);
273 : }
274 5 : if (instrInfo_.missionInstrCount == 0 || instrInfo_.instrVec.empty()) {
275 0 : HCCL_ERROR(
276 : "[CcuKernel][%s] failed, mission instructions are empty, "
277 : "the kernel is not been translated yet.",
278 : __func__);
279 0 : return CcuResult::CCU_E_INTERNAL;
280 : }
281 5 : return CcuResult::CCU_SUCCESS;
282 : }
283 :
284 8 : void CcuKernel::FillTaskParam(
285 : CcuTaskParam& param, uint32_t index, uint32_t seqNum, const uint64_t* taskArgs, uint32_t argsNum) const
286 : {
287 8 : param.dieId = GetDieId();
288 8 : param.missionId = GetMissionId();
289 8 : param.instStartId = instrInfo_.missionStartInstrId + index * CCU_SQE_ARGS_LEN;
290 8 : param.key = GetMissionKey();
291 8 : param.argSize = CCU_SQE_ARGS_LEN;
292 :
293 8 : const uint32_t preMissionInsCnt = index * CCU_SQE_ARGS_LEN;
294 8 : const bool isLast = (index == seqNum - 1);
295 8 : param.instCnt = isLast ? (instrInfo_.missionInstrCount - preMissionInsCnt) : CCU_SQE_ARGS_LEN;
296 :
297 8 : if (argsNum > preMissionInsCnt) {
298 6 : const uint32_t argsToCopy = isLast ? std::min(argsNum - preMissionInsCnt, CCU_SQE_ARGS_LEN) : CCU_SQE_ARGS_LEN;
299 12 : std::copy(taskArgs + preMissionInsCnt, taskArgs + preMissionInsCnt + argsToCopy, std::begin(param.args));
300 : }
301 :
302 8 : HCCL_INFO(
303 : "[GeneTaskParam]task Param, dieId[%u] missionId[%u] instStartId[%u] instCnt[%u], argSize[%u]", param.dieId,
304 : param.missionId, param.instStartId, param.instCnt, param.argSize);
305 8 : }
306 :
307 5 : CcuResult CcuKernel::GeneTaskParams(const uint64_t* taskArgs, uint32_t argsNum, std::vector<CcuTaskParam>& taskParams)
308 : {
309 5 : CCU_CHK_RET(ValidateTaskArgs(taskArgs, argsNum));
310 :
311 : // 如果agrs数量超过sqe arg的最大数量，则返回多个TaskParam，前面几个只从sqe中加载args;
312 : // args数量大于等于0、小于等于最大值时，返回1个TaskParam
313 5 : const uint32_t seqNum
314 5 : = (argsNum / CCU_SQE_ARGS_LEN) + ((argsNum % CCU_SQE_ARGS_LEN) == 0 ? 0 : 1) + (argsNum == 0 ? 1 : 0);
315 :
316 5 : const uint32_t preMissonSqeInsCnt = (seqNum - 1) * CCU_SQE_ARGS_LEN;
317 5 : if (instrInfo_.missionInstrCount < preMissonSqeInsCnt) {
318 0 : HCCL_ERROR(
319 : "[CcuKernel][%s] failed, missionInstrCount[%u] should be greater "
320 : "than preMissonSqeInsCnt[%u].",
321 : __func__, instrInfo_.missionInstrCount, preMissonSqeInsCnt);
322 0 : return CcuResult::CCU_E_INTERNAL;
323 : }
324 :
325 5 : taskParams.resize(seqNum);
326 13 : for (uint32_t index = 0; index < seqNum; index++) {
327 8 : FillTaskParam(taskParams[index], index, seqNum, taskArgs, argsNum);
328 : }
329 :
330 5 : return CcuResult::CCU_SUCCESS;
331 : }
332 :
333 14 : HcclResult CcuKernel::CreateVariable(const ChannelHandle channel, uint32_t varIndex, CcuRep::Variable* var)
334 : {
335 14 : channels_.insert(channel);
336 :
337 14 : void* channelPtr{nullptr};
338 14 : CHK_RET(static_cast<HcclResult>(HcommChannelGet(channel, &channelPtr)));
339 14 : auto* channelImpl = dynamic_cast<CcuUrmaChannel*>(static_cast<Channel*>(channelPtr));
340 14 : if (channelImpl == nullptr) {
341 0 : HCCL_ERROR("[%s] failed to cast channel[0x%llx] to CcuUrmaChannel", __func__, channel);
342 0 : return HcclResult::HCCL_E_PTR;
343 : }
344 14 : uint32_t locXnId{0};
345 14 : CHK_RET(channelImpl->GetLocXnByIndex(varIndex, locXnId));
346 14 : var->Reset(locXnId, channelImpl->GetDieId());
347 14 : return HcclResult::HCCL_SUCCESS;
348 : }
349 :
350 44 : CcuRepResource& CcuKernel::GetResource() { return res_; }
351 :
352 91 : CcuResReq CcuKernel::GetResourceRequest()
353 : {
354 91 : CcuResReq req;
355 91 : uint32_t dieId = GetDieId();
356 91 : req.msReq[dieId] = res_.ccubufs[dieId].size();
357 91 : req.blockMsReq[dieId] = res_.blockCcubufs[dieId].size();
358 91 : req.ckeReq[dieId] = res_.completedEvent[dieId].size();
359 91 : req.blockCkeReq[dieId] = res_.blockCompletedEvent[dieId].size() + res_.localNotify[dieId].size();
360 91 : req.loopEngineReq[dieId] = res_.executor[dieId].size();
361 91 : req.blockLoopEngineReq[dieId] = res_.blockExecutor[dieId].size();
362 91 : req.gsaReq[dieId] = res_.address[dieId].size();
363 91 : req.blockGsaReq[dieId] = res_.blockAddress[dieId].size();
364 91 : req.xnReq[dieId] = res_.variable[dieId].size();
365 91 : req.blockXnReq[dieId] = res_.continuousVariable[dieId].size();
366 :
367 91 : req.missionReq.reqType = MissionReqType::FUSION_MULTIPLE_DIE;
368 91 : req.missionReq.req[dieId] = 1;
369 :
370 : auto info = Hccl::StringFormat(
371 : "resource request: dieId[%u], ms[%u], blockMs[%u], cke[%u], blockCke[%u], "
372 : "loopEngine[%u], blockLoopEngine[%u], gsa[%u], blockGsa[%u], xn[%u], blockXn[%u], "
373 : "missionId[%u]",
374 364 : dieId, req.msReq[dieId], req.blockMsReq[dieId], req.ckeReq[dieId], req.blockCkeReq[dieId],
375 364 : req.loopEngineReq[dieId], req.blockLoopEngineReq[dieId], req.gsaReq[dieId], req.blockGsaReq[dieId],
376 91 : req.xnReq[dieId], req.blockXnReq[dieId], req.missionReq.req[dieId]);
377 :
378 91 : HCCL_INFO("%s", info.c_str());
379 :
380 182 : return req;
381 91 : }
382 :
383 : template <typename HandleType, typename ResourceType>
384 1701 : static CcuResult GetResourceByHandle(
385 : std::unordered_map<HandleType, ResourceType>& resourceMap, HandleType handle, ResourceType** resource,
386 : const char* resourceType)
387 : {
388 1701 : auto iter = resourceMap.find(handle);
389 1701 : if (iter == resourceMap.end()) {
390 0 : HCCL_ERROR("[%s] failed to find %s by handle: 0x%llx", __func__, resourceType, handle);
391 0 : return CcuResult::CCU_E_NOT_FOUND;
392 : }
393 :
394 : // ccu资源本身可能重载=，对象赋值会被转换成指令，导致流程失败
395 1701 : *resource = &(iter->second);
396 1701 : return CcuResult::CCU_SUCCESS;
397 : }
398 :
399 1373 : CcuResult CcuKernel::GetVariableByHandle(CcuVariableHandle varHandle, CcuRep::Variable** variable)
400 : {
401 1373 : return GetResourceByHandle(ccuVarMap_, varHandle, variable, "variable");
402 : }
403 : // Alloc 相关接口
404 496 : CcuResult CcuKernel::VariableAlloc(CcuVariableHandle* varHandle)
405 : {
406 496 : PLF_CONFIG_INFO(PLF_DATA_OP, "[VariableAlloc]");
407 496 : const auto& var = CreateBlockResAssist(1, res_.continuousVariable);
408 496 : CcuVariableHandle handle = ccuVarMap_.size();
409 496 : ccuVarMap_.emplace(handle, var[0]);
410 :
411 496 : *varHandle = handle;
412 496 : return CcuResult::CCU_SUCCESS;
413 496 : }
414 10 : CcuResult CcuKernel::AddressAlloc(CcuAddressHandle* addrHandle)
415 : {
416 10 : PLF_CONFIG_INFO(PLF_DATA_OP, "[AddressAlloc]");
417 10 : const auto addr = CreateAddress();
418 10 : CcuAddressHandle handle = ccuAddrMap_.size();
419 10 : ccuAddrMap_.emplace(handle, addr);
420 10 : *addrHandle = handle;
421 10 : return CcuResult::CCU_SUCCESS;
422 10 : }
423 10 : CcuResult CcuKernel::EventAlloc(CcuEventHandle* eventHandle)
424 : {
425 10 : PLF_CONFIG_INFO(PLF_DATA_OP, "[EventAlloc]");
426 10 : const auto& event = CreateResAssist(res_.blockCompletedEvent);
427 10 : CcuEventHandle handle = ccuEventMap_.size();
428 10 : ccuEventMap_.emplace(handle, event);
429 10 : *eventHandle = handle;
430 10 : return CcuResult::CCU_SUCCESS;
431 10 : }
432 2 : CcuResult CcuKernel::BufferAlloc(CcuBufferHandle* bufHandle)
433 : {
434 2 : PLF_CONFIG_INFO(PLF_DATA_OP, "[BufferAlloc]");
435 2 : const auto& buffer = CreateBlockResAssist(1, res_.blockCcubufs);
436 2 : CcuBufferHandle handle = ccuBufferMap_.size();
437 2 : ccuBufferMap_.emplace(handle, buffer[0]);
438 2 : *bufHandle = handle;
439 2 : return CcuResult::CCU_SUCCESS;
440 2 : }
441 156 : CcuResult CcuKernel::LocalAddrAlloc(
442 : CcuLocalAddrHandle* localAddrHandle, CcuAddressHandle* addrHandle, CcuVariableHandle* tokenHandle)
443 : {
444 156 : PLF_CONFIG_INFO(PLF_DATA_OP, "[LocalAddrAlloc]");
445 156 : auto localAddr = CreateLocalAddr();
446 :
447 156 : CcuAddressHandle aHandle = ccuAddrMap_.size();
448 156 : ccuAddrMap_.emplace(aHandle, localAddr.addr);
449 :
450 156 : CcuVariableHandle tHandle = ccuVarMap_.size();
451 156 : ccuVarMap_.emplace(tHandle, localAddr.token);
452 :
453 156 : CcuLocalAddrHandle laHandle = ccuLocalAddrMap_.size();
454 156 : ccuLocalAddrMap_.emplace(laHandle, localAddr);
455 :
456 156 : *localAddrHandle = laHandle;
457 156 : *addrHandle = aHandle;
458 156 : *tokenHandle = tHandle;
459 156 : return CcuResult::CCU_SUCCESS;
460 156 : }
461 37 : CcuResult CcuKernel::RemoteAddrAlloc(
462 : CcuRemoteAddrHandle* remoteAddrHandle, CcuAddressHandle* addrHandle, CcuVariableHandle* tokenHandle)
463 : {
464 37 : PLF_CONFIG_INFO(PLF_DATA_OP, "[RemoteAddrAlloc]");
465 37 : auto remoteAddr = CreateRemoteAddr();
466 :
467 37 : CcuAddressHandle aHandle = ccuAddrMap_.size();
468 37 : ccuAddrMap_.emplace(aHandle, remoteAddr.addr);
469 :
470 37 : CcuVariableHandle tHandle = ccuVarMap_.size();
471 37 : ccuVarMap_.emplace(tHandle, remoteAddr.token);
472 :
473 37 : CcuRemoteAddrHandle raHandle = ccuRemoteAddrMap_.size();
474 37 : ccuRemoteAddrMap_.emplace(raHandle, remoteAddr);
475 :
476 37 : *remoteAddrHandle = raHandle;
477 37 : *addrHandle = aHandle;
478 37 : *tokenHandle = tHandle;
479 37 : return CcuResult::CCU_SUCCESS;
480 37 : }
481 :
482 6 : CcuResult CcuKernel::BlockVariableAlloc(CcuVariableHandle* varHandles, uint32_t count)
483 : {
484 6 : PLF_CONFIG_INFO(PLF_DATA_OP, "[BlockVariableAlloc] count=%u", count);
485 6 : const auto& var = CreateBlockResAssist(count, res_.continuousVariable);
486 18 : for (uint32_t i = 0; i < count; i++) {
487 12 : CcuVariableHandle handle = ccuVarMap_.size();
488 12 : ccuVarMap_.emplace(handle, var[i]);
489 12 : varHandles[i] = handle;
490 : }
491 6 : return CcuResult::CCU_SUCCESS;
492 6 : }
493 :
494 6 : CcuResult CcuKernel::BlockEventAlloc(CcuEventHandle* eventHandles, uint32_t count)
495 : {
496 6 : PLF_CONFIG_INFO(PLF_DATA_OP, "[BlockEventAlloc] count=%u", count);
497 6 : const auto& event = CreateBlockResAssist(count, res_.blockCompletedEvent);
498 52 : for (uint32_t i = 0; i < count; i++) {
499 46 : CcuEventHandle handle = ccuEventMap_.size();
500 46 : ccuEventMap_.emplace(handle, event[i]);
501 46 : eventHandles[i] = handle;
502 : }
503 6 : return CcuResult::CCU_SUCCESS;
504 6 : }
505 :
506 7 : CcuResult CcuKernel::BlockBufferAlloc(CcuBufferHandle* bufHandles, uint32_t count)
507 : {
508 7 : PLF_CONFIG_INFO(PLF_DATA_OP, "[BlockBufferAlloc] count=%u", count);
509 7 : const auto& buffer = CreateBlockResAssist(count, res_.blockCcubufs);
510 332 : for (uint32_t i = 0; i < count; i++) {
511 325 : CcuBufferHandle handle = ccuBufferMap_.size();
512 325 : ccuBufferMap_.emplace(handle, buffer[i]);
513 325 : bufHandles[i] = handle;
514 : }
515 7 : return CcuResult::CCU_SUCCESS;
516 7 : }
517 :
518 14 : CcuResult CcuKernel::VariableCreateByChannel(ChannelHandle channel, uint32_t varIndex, CcuVariableHandle* varHandle)
519 : {
520 14 : PLF_CONFIG_INFO(PLF_DATA_OP, "[VariableCreateByChannel] channel=%llu, varIndex=%u", channel, varIndex);
521 14 : channels_.insert(channel);
522 14 : CcuRep::Variable var(this);
523 14 : CCU_CHK_RET(CreateVariable(channel, varIndex, &var));
524 14 : CcuVariableHandle handle = ccuVarMap_.size();
525 14 : ccuVarMap_.emplace(handle, var);
526 14 : *varHandle = handle;
527 14 : return CcuResult::CCU_SUCCESS;
528 14 : }
529 :
530 10 : CcuResult CcuKernel::VariableCreateByAcquire(CcuVariableHandle acqHandle, uint32_t index, CcuVariableHandle* varHandle)
531 : {
532 10 : uint8_t dieId = 0;
533 10 : uint32_t xnId = 0;
534 10 : CCU_CHK_RET(CcuVarEventResMgr::GetInstance(HcclGetThreadDeviceId()).GetVariableXnId(acqHandle, index, dieId, xnId));
535 :
536 10 : CcuRep::Variable var(this);
537 10 : var.Reset(static_cast<uint16_t>(xnId), static_cast<uint16_t>(dieId));
538 10 : CcuVariableHandle handle = ccuVarMap_.size();
539 10 : ccuVarMap_.emplace(handle, var);
540 10 : *varHandle = handle;
541 10 : return CcuResult::CCU_SUCCESS;
542 10 : }
543 :
544 4 : CcuResult CcuKernel::EventCreateByAcquire(CcuEventHandle acqHandle, uint32_t index, CcuEventHandle* eventHandle)
545 : {
546 4 : uint8_t dieId = 0;
547 4 : uint32_t ckeId = 0;
548 4 : CCU_CHK_RET(CcuVarEventResMgr::GetInstance(HcclGetThreadDeviceId()).GetEventCkeId(acqHandle, index, dieId, ckeId));
549 :
550 4 : CcuRep::CompletedEvent event(this);
551 4 : event.Reset(static_cast<uint16_t>(ckeId), static_cast<uint16_t>(dieId));
552 4 : CcuEventHandle handle = ccuEventMap_.size();
553 4 : ccuEventMap_.emplace(handle, event);
554 4 : *eventHandle = handle;
555 4 : return CcuResult::CCU_SUCCESS;
556 4 : }
557 :
558 283 : CcuResult CcuKernel::VariableAssignImm(CcuVariableHandle varHandle, uint64_t immediate)
559 : {
560 283 : PLF_CONFIG_INFO(PLF_DATA_OP, "[VariableAssignImm] varHandle=%llu, immediate=%llu", varHandle, immediate);
561 283 : CcuRep::Variable* variable{nullptr};
562 283 : CCU_CHK_RET(GetVariableByHandle(varHandle, &variable));
563 : // 通过符号重载实现，内部记录rep；异常由入口 HcommCcuKernelRegister 的
564 : // CCU_EXCEPTION_HANDLE_BEGIN/END 统一接住，无需在此局部 try/catch。
565 283 : (*variable) = immediate;
566 283 : return CcuResult::CCU_SUCCESS;
567 : }
568 :
569 158 : CcuResult CcuKernel::VariableAssignVar(CcuVariableHandle varHandle, CcuVariableHandle varA)
570 : {
571 158 : PLF_CONFIG_INFO(PLF_DATA_OP, "[VariableAssignVar] varHandle=%llu, varA=%llu", varHandle, varA);
572 158 : CcuRep::Variable* variable{nullptr};
573 158 : CCU_CHK_RET(GetVariableByHandle(varHandle, &variable));
574 158 : CcuRep::Variable* variableA{nullptr};
575 158 : CCU_CHK_RET(GetVariableByHandle(varA, &variableA));
576 : // 通过符号重载实现，内部记录rep；异常由入口统一 catch。
577 158 : (*variable) = (*variableA);
578 158 : return CcuResult::CCU_SUCCESS;
579 : }
580 :
581 : CcuResult
582 109 : CcuKernel::VariableAddVarToVar(CcuVariableHandle varHandle, CcuVariableHandle varAHandle, CcuVariableHandle varBHandle)
583 : {
584 109 : PLF_CONFIG_INFO(
585 : PLF_DATA_OP, "[VariableAddVarToVar] varHandle=%llu, varAHandle=%llu, varBHandle=%llu", varHandle, varAHandle,
586 : varBHandle);
587 109 : CcuRep::Variable *resVar{nullptr}, *leftVar{nullptr}, *rightVar{nullptr};
588 109 : CCU_CHK_RET(GetVariableByHandle(varHandle, &resVar));
589 109 : CCU_CHK_RET(GetVariableByHandle(varAHandle, &leftVar));
590 109 : CCU_CHK_RET(GetVariableByHandle(varBHandle, &rightVar));
591 :
592 : // 通过符号重载实现，内部记录rep；异常由入口统一 catch。
593 109 : *resVar = *leftVar + *rightVar;
594 109 : return CcuResult::CCU_SUCCESS;
595 : }
596 :
597 : CcuResult
598 2 : CcuKernel::VariableSubVarToVar(CcuVariableHandle varHandle, CcuVariableHandle varAHandle, CcuVariableHandle varBHandle)
599 : {
600 2 : CcuRep::Variable *resVar{nullptr}, *leftVar{nullptr}, *rightVar{nullptr};
601 2 : CCU_CHK_RET(GetVariableByHandle(varHandle, &resVar));
602 2 : CCU_CHK_RET(GetVariableByHandle(varAHandle, &leftVar));
603 2 : CCU_CHK_RET(GetVariableByHandle(varBHandle, &rightVar));
604 :
605 2 : *resVar = *leftVar - *rightVar;
606 2 : return CcuResult::CCU_SUCCESS;
607 : }
608 :
609 : CcuResult
610 2 : CcuKernel::VariableMulVarToVar(CcuVariableHandle varHandle, CcuVariableHandle varAHandle, CcuVariableHandle varBHandle)
611 : {
612 2 : CcuRep::Variable *resVar{nullptr}, *leftVar{nullptr}, *rightVar{nullptr};
613 2 : CCU_CHK_RET(GetVariableByHandle(varHandle, &resVar));
614 2 : CCU_CHK_RET(GetVariableByHandle(varAHandle, &leftVar));
615 2 : CCU_CHK_RET(GetVariableByHandle(varBHandle, &rightVar));
616 :
617 2 : *resVar = *leftVar * *rightVar;
618 2 : return CcuResult::CCU_SUCCESS;
619 : }
620 :
621 1 : CcuResult CcuKernel::VariableAddImmToVar(CcuVariableHandle varHandle, CcuVariableHandle varAHandle, uint16_t immediate)
622 : {
623 1 : CcuRep::Variable *resVar{nullptr}, *leftVar{nullptr};
624 1 : CCU_CHK_RET(GetVariableByHandle(varHandle, &resVar));
625 1 : CCU_CHK_RET(GetVariableByHandle(varAHandle, &leftVar));
626 :
627 1 : *resVar = *leftVar + immediate;
628 1 : return CcuResult::CCU_SUCCESS;
629 : }
630 :
631 2 : CcuResult CcuKernel::VariableSubImmToVar(CcuVariableHandle varHandle, CcuVariableHandle varAHandle, uint16_t immediate)
632 : {
633 2 : CcuRep::Variable *resVar{nullptr}, *leftVar{nullptr};
634 2 : CCU_CHK_RET(GetVariableByHandle(varHandle, &resVar));
635 2 : CCU_CHK_RET(GetVariableByHandle(varAHandle, &leftVar));
636 :
637 2 : *resVar = *leftVar - immediate;
638 2 : return CcuResult::CCU_SUCCESS;
639 : }
640 :
641 2 : CcuResult CcuKernel::VariableMulImmToVar(CcuVariableHandle varHandle, CcuVariableHandle varAHandle, uint16_t immediate)
642 : {
643 2 : CcuRep::Variable *resVar{nullptr}, *leftVar{nullptr};
644 2 : CCU_CHK_RET(GetVariableByHandle(varHandle, &resVar));
645 2 : CCU_CHK_RET(GetVariableByHandle(varAHandle, &leftVar));
646 :
647 2 : *resVar = *leftVar * immediate;
648 2 : return CcuResult::CCU_SUCCESS;
649 : }
650 :
651 : CcuResult
652 2 : CcuKernel::VariableAndVarToVar(CcuVariableHandle varHandle, CcuVariableHandle varAHandle, CcuVariableHandle varBHandle)
653 : {
654 2 : CcuRep::Variable *resVar{nullptr}, *leftVar{nullptr}, *rightVar{nullptr};
655 2 : CCU_CHK_RET(GetVariableByHandle(varHandle, &resVar));
656 2 : CCU_CHK_RET(GetVariableByHandle(varAHandle, &leftVar));
657 2 : CCU_CHK_RET(GetVariableByHandle(varBHandle, &rightVar));
658 :
659 2 : *resVar = *leftVar & *rightVar;
660 2 : return CcuResult::CCU_SUCCESS;
661 : }
662 :
663 : CcuResult
664 2 : CcuKernel::VariableOrVarToVar(CcuVariableHandle varHandle, CcuVariableHandle varAHandle, CcuVariableHandle varBHandle)
665 : {
666 2 : CcuRep::Variable *resVar{nullptr}, *leftVar{nullptr}, *rightVar{nullptr};
667 2 : CCU_CHK_RET(GetVariableByHandle(varHandle, &resVar));
668 2 : CCU_CHK_RET(GetVariableByHandle(varAHandle, &leftVar));
669 2 : CCU_CHK_RET(GetVariableByHandle(varBHandle, &rightVar));
670 :
671 2 : *resVar = *leftVar | *rightVar;
672 2 : return CcuResult::CCU_SUCCESS;
673 : }
674 :
675 : CcuResult
676 2 : CcuKernel::VariableXorVarToVar(CcuVariableHandle varHandle, CcuVariableHandle varAHandle, CcuVariableHandle varBHandle)
677 : {
678 2 : CcuRep::Variable *resVar{nullptr}, *leftVar{nullptr}, *rightVar{nullptr};
679 2 : CCU_CHK_RET(GetVariableByHandle(varHandle, &resVar));
680 2 : CCU_CHK_RET(GetVariableByHandle(varAHandle, &leftVar));
681 2 : CCU_CHK_RET(GetVariableByHandle(varBHandle, &rightVar));
682 :
683 2 : *resVar = *leftVar ^ *rightVar;
684 2 : return CcuResult::CCU_SUCCESS;
685 : }
686 :
687 1 : CcuResult CcuKernel::VariableNotVar(CcuVariableHandle varHandle, CcuVariableHandle varAHandle)
688 : {
689 1 : CcuRep::Variable *resVar{nullptr}, *leftVar{nullptr};
690 1 : CCU_CHK_RET(GetVariableByHandle(varHandle, &resVar));
691 1 : CCU_CHK_RET(GetVariableByHandle(varAHandle, &leftVar));
692 :
693 1 : *resVar = ~(*leftVar);
694 1 : return CcuResult::CCU_SUCCESS;
695 : }
696 :
697 : CcuResult
698 2 : CcuKernel::VariableShlVarToVar(CcuVariableHandle varHandle, CcuVariableHandle varAHandle, CcuVariableHandle varBHandle)
699 : {
700 2 : CcuRep::Variable *resVar{nullptr}, *leftVar{nullptr}, *rightVar{nullptr};
701 2 : CCU_CHK_RET(GetVariableByHandle(varHandle, &resVar));
702 2 : CCU_CHK_RET(GetVariableByHandle(varAHandle, &leftVar));
703 2 : CCU_CHK_RET(GetVariableByHandle(varBHandle, &rightVar));
704 :
705 2 : *resVar = *leftVar << *rightVar;
706 2 : return CcuResult::CCU_SUCCESS;
707 : }
708 :
709 : CcuResult
710 2 : CcuKernel::VariableShrVarToVar(CcuVariableHandle varHandle, CcuVariableHandle varAHandle, CcuVariableHandle varBHandle)
711 : {
712 2 : CcuRep::Variable *resVar{nullptr}, *leftVar{nullptr}, *rightVar{nullptr};
713 2 : CCU_CHK_RET(GetVariableByHandle(varHandle, &resVar));
714 2 : CCU_CHK_RET(GetVariableByHandle(varAHandle, &leftVar));
715 2 : CCU_CHK_RET(GetVariableByHandle(varBHandle, &rightVar));
716 :
717 2 : *resVar = *leftVar >> *rightVar;
718 2 : return CcuResult::CCU_SUCCESS;
719 : }
720 :
721 : /*========== Event信号同步类 相关接口 ==========*/
722 15 : CcuResult CcuKernel::EventRecord(CcuEventHandle eventHandle, uint32_t mask)
723 : {
724 15 : PLF_CONFIG_INFO(PLF_DATA_OP, "[EventRecord] eventHandle=%llu, mask=%u", eventHandle, mask);
725 15 : CcuRep::CompletedEvent* event{nullptr};
726 15 : CCU_CHK_RET(GetEventByHandle(eventHandle, &event));
727 : // 复用已有的 RecordEvent 实现（内部 Append CcuRepLocRecordEvent）
728 15 : CCU_CHK_RET(RecordEvent(*event, mask));
729 14 : return CcuResult::CCU_SUCCESS;
730 : }
731 :
732 41 : CcuResult CcuKernel::EventWait(CcuEventHandle eventHandle, uint32_t mask)
733 : {
734 41 : PLF_CONFIG_INFO(PLF_DATA_OP, "[EventWait] eventHandle=%llu, mask=%u", eventHandle, mask);
735 41 : CcuRep::CompletedEvent* event{nullptr};
736 41 : CCU_CHK_RET(GetEventByHandle(eventHandle, &event));
737 : // 复用已有的 WaitEvent 实现（内部 Append CcuRepLocWaitEvent）
738 41 : CCU_CHK_RET(WaitEvent(*event, mask));
739 41 : return CcuResult::CCU_SUCCESS;
740 : }
741 :
742 1 : CcuResult CcuKernel::LocalNotifyRecord(const char* notifyTag, const uint32_t mask)
743 : {
744 1 : PLF_CONFIG_INFO(PLF_DATA_OP, "[LocalNotifyRecord] tag=%s, mask=%u", (notifyTag ? notifyTag : "null"), mask);
745 1 : if (notifyTag == nullptr) {
746 0 : HCCL_ERROR("[CcuKernel][%s] notifyTag is nullptr, please check.", __func__);
747 0 : return CcuResult::CCU_E_PTR;
748 : }
749 1 : if (CurrentBlock()->Type() == CcuRep::CcuRepType::LOOP_BLOCK) {
750 1 : HCCL_ERROR("[CcuKernel][%s] is not supported in loop block, please check.", __func__);
751 1 : return LatchBodyError(CcuResult::CCU_E_NOT_SUPPORT);
752 : }
753 :
754 0 : const std::string tagKey(notifyTag);
755 :
756 0 : auto& sharedNotifies = importedRes_.sharedNotifies;
757 0 : if (sharedNotifies.find(tagKey) == sharedNotifies.end()) {
758 0 : CcuRep::LocalNotify localNotify;
759 0 : sharedNotifies.insert({tagKey, localNotify});
760 0 : }
761 :
762 0 : Append(std::make_shared<CcuRep::CcuRepRecordSharedNotify>(insGenerator, sharedNotifies.at(tagKey), mask));
763 :
764 0 : return CcuResult::CCU_SUCCESS;
765 0 : }
766 :
767 0 : CcuResult CcuKernel::LocalNotifyWait(const char* notifyTag, const uint32_t mask)
768 : {
769 0 : PLF_CONFIG_INFO(PLF_DATA_OP, "[LocalNotifyWait] tag=%s, mask=%u", (notifyTag ? notifyTag : "null"), mask);
770 0 : if (notifyTag == nullptr) {
771 0 : HCCL_ERROR("[CcuKernel][%s] notifyTag is nullptr, please check.", __func__);
772 0 : return CcuResult::CCU_E_PTR;
773 : }
774 :
775 0 : const std::string tagKey(notifyTag);
776 :
777 0 : auto& sharedNotifies = exportedRes_.sharedNotifies;
778 0 : if (sharedNotifies.find(tagKey) == sharedNotifies.end()) {
779 0 : CcuRep::LocalNotify notify = CreateLocalNotify();
780 0 : exportedRes_.sharedNotifies.insert({tagKey, notify});
781 0 : }
782 :
783 0 : bool isProfiling = CurrentBlock()->Type() != CcuRep::CcuRepType::LOOP_BLOCK;
784 0 : Append(std::make_shared<CcuRep::CcuRepLocWaitNotify>(
785 0 : insGenerator, exportedRes_.sharedNotifies.at(tagKey), mask, isProfiling));
786 0 : return CcuResult::CCU_SUCCESS;
787 0 : }
788 :
789 6 : CcuResult CcuKernel::NotifyRecord(const ChannelHandle channel, uint32_t remoteNotifyIdx, uint32_t mask)
790 : {
791 6 : PLF_CONFIG_INFO(
792 : PLF_DATA_OP, "[NotifyRecord] channel=%llu, remoteNotifyIdx=%u, mask=%u", channel, remoteNotifyIdx, mask);
793 6 : if (CurrentBlock()->Type() == CcuRep::CcuRepType::LOOP_BLOCK) {
794 1 : HCCL_ERROR("[%s] NotifyRecord is not allowed inside a ccu::Loop body", __func__);
795 1 : return LatchBodyError(CcuResult::CCU_E_NOT_SUPPORT);
796 : }
797 5 : channels_.insert(channel);
798 5 : Append(std::make_shared<CcuRep::CcuRepRemPostSem>(insGenerator, channel, remoteNotifyIdx, mask));
799 5 : return CCU_SUCCESS;
800 : }
801 :
802 11 : CcuResult CcuKernel::NotifyWait(const ChannelHandle channel, uint32_t localNotifyIdx, uint32_t mask)
803 : {
804 11 : PLF_CONFIG_INFO(
805 : PLF_DATA_OP, "[NotifyWait] channel=%llu, localNotifyIdx=%u, mask=%u", channel, localNotifyIdx, mask);
806 11 : channels_.insert(channel);
807 11 : bool isProfiling = CurrentBlock()->Type() != CcuRep::CcuRepType::LOOP_BLOCK;
808 11 : if (isProfiling) {
809 33 : CCU_CHK_RET(static_cast<HcclResult>(AddProfiling(channel, "NotifyWait", localNotifyIdx, mask)));
810 : }
811 10 : Append(std::make_shared<CcuRep::CcuRepRemWaitSem>(insGenerator, channel, localNotifyIdx, mask, isProfiling));
812 10 : return CcuResult::CCU_SUCCESS;
813 : }
814 :
815 10 : CcuResult CcuKernel::WriteVariableWithNotify(
816 : const ChannelHandle channel, CcuVariableHandle varHandle, uint32_t remoteVarIdx, uint32_t remoteNotifyIdx,
817 : uint32_t mask)
818 : {
819 10 : PLF_CONFIG_INFO(
820 : PLF_DATA_OP,
821 : "[WriteVariableWithNotify] channel=%llu, varHandle=%llu, remoteVarIdx=%u,"
822 : " remoteNotifyIdx=%u, mask=%u",
823 : channel, varHandle, remoteVarIdx, remoteNotifyIdx, mask);
824 10 : if (CurrentBlock()->Type() == CcuRep::CcuRepType::LOOP_BLOCK) {
825 1 : HCCL_ERROR("[%s] WriteVariableWithNotify is not allowed inside a ccu::Loop body", __func__);
826 1 : return LatchBodyError(CcuResult::CCU_E_NOT_SUPPORT);
827 : }
828 9 : channels_.insert(channel);
829 9 : CcuRep::Variable* var{nullptr};
830 9 : CCU_CHK_RET(GetVariableByHandle(varHandle, &var));
831 9 : Append(
832 18 : std::make_shared<CcuRep::CcuRepRemPostVar>(insGenerator, *var, channel, remoteVarIdx, remoteNotifyIdx, mask));
833 9 : return CcuResult::CCU_SUCCESS;
834 : }
835 :
836 : // 加载类 相关接口
837 49 : CcuResult CcuKernel::LoadArg(CcuVariableHandle varHandle, uint32_t argId)
838 : {
839 49 : PLF_CONFIG_INFO(PLF_DATA_OP, "[LoadArg] varHandle=%llu, argId=%u", varHandle, argId);
840 49 : loadArgUsedSet_.insert(argId);
841 49 : CcuRep::Variable* var{nullptr};
842 49 : CCU_CHK_RET(GetVariableByHandle(varHandle, &var));
843 : auto loadArgRep = std::make_shared<CcuRep::CcuRepLoadArg>(
844 49 : insGenerator, *var, argId % CCU_SQE_ARGS_LEN, static_cast<uint16_t>(argId));
845 49 : Append(loadArgRep);
846 49 : return CcuResult::CCU_SUCCESS;
847 49 : }
848 :
849 3 : CcuResult CcuKernel::GetCcuKernelInfo(CcuKernelInfo& info) const
850 : {
851 3 : uint32_t maxTaskArgsNum = 0;
852 : // 如果没有LoadArg指令，则说明不需要传入参数，直接返回0
853 3 : if (loadArgUsedSet_.empty()) {
854 1 : info.maxTaskArgsNum = 0;
855 1 : return CcuResult::CCU_SUCCESS;
856 : }
857 : // 遍历loadArgUsedSet_，获取最大的argId，加1得到最大任务参数数量（argId从0连续）
858 7 : for (const auto& argId : loadArgUsedSet_) {
859 5 : maxTaskArgsNum = std::max(maxTaskArgsNum, argId);
860 : }
861 2 : maxTaskArgsNum += 1;
862 2 : info.maxTaskArgsNum = maxTaskArgsNum;
863 2 : return CcuResult::CCU_SUCCESS;
864 : }
865 :
866 13 : CcuResult CcuKernel::CheckContinuousVariables(
867 : CcuVariableHandle varHandle, uint32_t num, const CcuRep::Variable& baseVar, const char* tag)
868 : {
869 13 : if (num <= 1) {
870 4 : return CcuResult::CCU_SUCCESS;
871 : }
872 18 : for (uint32_t i = 1; i < num; i++) {
873 9 : CcuRep::Variable* nextVar{nullptr};
874 9 : CCU_CHK_RET(GetVariableByHandle(varHandle + i, &nextVar));
875 9 : if (nextVar->Id() != baseVar.Id() + i) {
876 0 : HCCL_ERROR(
877 : "[CcuKernel][%s] variables not continuous at index %u, "
878 : "expected Id %u but got %u",
879 : tag, i, baseVar.Id() + i, nextVar->Id());
880 0 : return HCCL_TO_CCU_RET(HCCL_E_PARA);
881 : }
882 : }
883 9 : return CcuResult::CCU_SUCCESS;
884 : }
885 :
886 6 : CcuResult CcuKernel::LoadVar(uint64_t addr, CcuVariableHandle varHandle, uint32_t num)
887 : {
888 6 : PLF_CONFIG_INFO(PLF_DATA_OP, "[LoadVar] addr=0x%llx, varHandle=%llu, num=%u", addr, varHandle, num);
889 6 : CcuRep::Variable* var{nullptr};
890 6 : CCU_CHK_RET(GetVariableByHandle(varHandle, &var));
891 6 : CCU_CHK_RET(CheckContinuousVariables(varHandle, num, *var, "LoadVariable"));
892 6 : Append(std::make_shared<CcuRep::CcuRepLoad>(insGenerator, addr, *var, num));
893 6 : return CcuResult::CCU_SUCCESS;
894 : }
895 :
896 2 : CcuResult CcuKernel::CcuLoadVarFromVarAddr(CcuVariableHandle addrHandle, CcuVariableHandle varHandle, uint32_t num)
897 : {
898 2 : PLF_CONFIG_INFO(
899 : PLF_DATA_OP,
900 : "[CcuLoadVarFromVarAddr] addrHandle=%llu, varHandle=%llu,"
901 : " num=%u",
902 : addrHandle, varHandle, num);
903 2 : CcuRep::Variable* addrVar{nullptr};
904 2 : CCU_CHK_RET(GetVariableByHandle(addrHandle, &addrVar));
905 2 : CcuRep::Variable* var{nullptr};
906 2 : CCU_CHK_RET(GetVariableByHandle(varHandle, &var));
907 2 : CCU_CHK_RET(CheckContinuousVariables(varHandle, num, *var, "LoadVar dst"));
908 2 : Append(std::make_shared<CcuRep::CcuRepLoadVar>(insGenerator, *addrVar, *var, num));
909 2 : return CcuResult::CCU_SUCCESS;
910 : }
911 :
912 3 : CcuResult CcuKernel::StoreVar(uint64_t addr, CcuVariableHandle varHandle, uint32_t num)
913 : {
914 3 : PLF_CONFIG_INFO(PLF_DATA_OP, "[StoreVar] addr=0x%llx, varHandle=%llu, num=%u", addr, varHandle, num);
915 3 : CcuRep::Variable* var{nullptr};
916 3 : CCU_CHK_RET(GetVariableByHandle(varHandle, &var));
917 3 : CCU_CHK_RET(CheckContinuousVariables(varHandle, num, *var, "StoreVariable"));
918 3 : Append(std::make_shared<CcuRep::CcuRepStore>(insGenerator, *var, addr, num));
919 3 : return CcuResult::CCU_SUCCESS;
920 : }
921 :
922 2 : CcuResult CcuKernel::CcuStoreVarToVarAddr(CcuVariableHandle addrHandle, CcuVariableHandle varHandle, uint32_t num)
923 : {
924 2 : PLF_CONFIG_INFO(
925 : PLF_DATA_OP,
926 : "[CcuStoreVarToVarAddr] addrHandle=%llu, varHandle=%llu,"
927 : " num=%u",
928 : addrHandle, varHandle, num);
929 2 : CcuRep::Variable* addrVar{nullptr};
930 2 : CCU_CHK_RET(GetVariableByHandle(addrHandle, &addrVar));
931 2 : CcuRep::Variable* var{nullptr};
932 2 : CCU_CHK_RET(GetVariableByHandle(varHandle, &var));
933 2 : CCU_CHK_RET(CheckContinuousVariables(varHandle, num, *var, "StoreVar src"));
934 2 : Append(std::make_shared<CcuRep::CcuRepStoreVar>(insGenerator, *var, *addrVar, num));
935 2 : return CcuResult::CCU_SUCCESS;
936 : }
937 :
938 : // 本地数据拷贝 相关实现
939 12 : CcuResult CcuKernel::LocalCopyMemToBuffer(
940 : CcuBufferHandle dstHandle, CcuLocalAddrHandle srcHandle, CcuVariableHandle lenHandle, CcuEventHandle eventHandle,
941 : uint32_t mask)
942 : {
943 12 : PLF_CONFIG_INFO(
944 : PLF_DATA_OP,
945 : "[LocalCopyMemToBuffer] dstHandle=%llu, srcHandle=%llu, lenHandle=%llu,"
946 : " eventHandle=%llu, mask=%u",
947 : dstHandle, srcHandle, lenHandle, eventHandle, mask);
948 12 : CcuRep::CcuBuf* dst{nullptr};
949 12 : CCU_CHK_RET(GetBufferByHandle(dstHandle, &dst));
950 12 : CcuRep::LocalAddr* src{nullptr};
951 12 : CCU_CHK_RET(GetLocalAddrByHandle(srcHandle, &src));
952 12 : CcuRep::Variable* len{nullptr};
953 12 : CCU_CHK_RET(GetVariableByHandle(lenHandle, &len));
954 12 : CcuRep::CompletedEvent* event{nullptr};
955 12 : CCU_CHK_RET(GetEventByHandle(eventHandle, &event));
956 12 : auto ret = LocalCopyNb(*dst, *src, *len, *event, mask); // 复用 protected
957 12 : return HCCL_TO_CCU_RET(ret);
958 : }
959 :
960 8 : CcuResult CcuKernel::LocalCopyBufferToMem(
961 : CcuLocalAddrHandle dstHandle, CcuBufferHandle srcHandle, CcuVariableHandle lenHandle, CcuEventHandle eventHandle,
962 : uint32_t mask)
963 : {
964 8 : PLF_CONFIG_INFO(
965 : PLF_DATA_OP,
966 : "[LocalCopyBufferToMem] dstHandle=%llu, srcHandle=%llu, lenHandle=%llu,"
967 : " eventHandle=%llu, mask=%u",
968 : dstHandle, srcHandle, lenHandle, eventHandle, mask);
969 8 : CcuRep::LocalAddr* dst{nullptr};
970 8 : CCU_CHK_RET(GetLocalAddrByHandle(dstHandle, &dst));
971 8 : CcuRep::CcuBuf* src{nullptr};
972 8 : CCU_CHK_RET(GetBufferByHandle(srcHandle, &src));
973 8 : CcuRep::Variable* len{nullptr};
974 8 : CCU_CHK_RET(GetVariableByHandle(lenHandle, &len));
975 8 : CcuRep::CompletedEvent* event{nullptr};
976 8 : CCU_CHK_RET(GetEventByHandle(eventHandle, &event));
977 8 : auto ret = LocalCopyNb(*dst, *src, *len, *event, mask);
978 8 : return HCCL_TO_CCU_RET(ret);
979 : }
980 :
981 1 : CcuResult CcuKernel::LocalCopyMemToMem(
982 : CcuLocalAddrHandle dstHandle, CcuLocalAddrHandle srcHandle, CcuVariableHandle lenHandle, CcuEventHandle eventHandle,
983 : uint32_t mask)
984 : {
985 1 : PLF_CONFIG_INFO(
986 : PLF_DATA_OP,
987 : "[LocalCopyMemToMem] dstHandle=%llu, srcHandle=%llu, lenHandle=%llu,"
988 : " eventHandle=%llu, mask=%u",
989 : dstHandle, srcHandle, lenHandle, eventHandle, mask);
990 1 : CcuRep::LocalAddr* dst{nullptr};
991 1 : CCU_CHK_RET(GetLocalAddrByHandle(dstHandle, &dst));
992 1 : CcuRep::LocalAddr* src{nullptr};
993 1 : CCU_CHK_RET(GetLocalAddrByHandle(srcHandle, &src));
994 1 : CcuRep::Variable* len{nullptr};
995 1 : CCU_CHK_RET(GetVariableByHandle(lenHandle, &len));
996 1 : CcuRep::CompletedEvent* event{nullptr};
997 1 : CCU_CHK_RET(GetEventByHandle(eventHandle, &event));
998 1 : auto ret = LocalCopyNb(*dst, *src, *len, *event, mask);
999 1 : return HCCL_TO_CCU_RET(ret);
1000 : }
1001 :
1002 : // 本地reduce 相关实现
1003 0 : CcuResult CcuKernel::LocalMemReduce(
1004 : CcuLocalAddrHandle dstHandle, CcuLocalAddrHandle srcHandle, CcuVariableHandle lenHandle, HcclDataType dataType,
1005 : HcclReduceOp opType, CcuEventHandle eventHandle, uint32_t mask)
1006 : {
1007 0 : PLF_CONFIG_INFO(
1008 : PLF_DATA_OP,
1009 : "[LocalMemReduce] dstHandle=%llu, srcHandle=%llu, lenHandle=%llu, dataType=%d,"
1010 : " op=%d, eventHandle=%llu, mask=%u",
1011 : dstHandle, srcHandle, lenHandle, dataType, opType, eventHandle, mask);
1012 0 : CcuRep::LocalAddr* dst{nullptr};
1013 0 : CCU_CHK_RET(GetLocalAddrByHandle(dstHandle, &dst));
1014 0 : CcuRep::LocalAddr* src{nullptr};
1015 0 : CCU_CHK_RET(GetLocalAddrByHandle(srcHandle, &src));
1016 0 : CcuRep::Variable* len{nullptr};
1017 0 : CCU_CHK_RET(GetVariableByHandle(lenHandle, &len));
1018 0 : CcuRep::CompletedEvent* event{nullptr};
1019 0 : CCU_CHK_RET(GetEventByHandle(eventHandle, &event));
1020 0 : auto ret = LocalReduceNb(*dst, *src, *len, dataType, opType, *event, mask);
1021 0 : return HCCL_TO_CCU_RET(ret);
1022 : }
1023 :
1024 4 : CcuResult CcuKernel::LocalBufferReduce(
1025 : CcuBufferHandle* bufHandles, uint32_t count, HcclDataType dataType, HcclDataType outputDataType,
1026 : HcclReduceOp opType, CcuVariableHandle lenHandle, CcuEventHandle eventHandle, uint32_t mask)
1027 : {
1028 4 : PLF_CONFIG_INFO(
1029 : PLF_DATA_OP,
1030 : "[LocalBufferReduce] count=%u, dataType=%d, outDataType=%d,"
1031 : " op=%d, lenHandle=%llu, eventHandle=%llu, mask=%u",
1032 : count, dataType, outputDataType, opType, lenHandle, eventHandle, mask);
1033 4 : CcuRep::Variable* len{nullptr};
1034 4 : CCU_CHK_RET(GetVariableByHandle(lenHandle, &len));
1035 4 : CcuRep::CompletedEvent* event{nullptr};
1036 4 : CCU_CHK_RET(GetEventByHandle(eventHandle, &event));
1037 4 : std::vector<CcuRep::CcuBuf> bufs(count);
1038 12 : for (uint32_t i = 0; i < count; i++) {
1039 8 : CcuRep::CcuBuf* buf{nullptr};
1040 8 : CCU_CHK_RET(GetBufferByHandle(bufHandles[i], &buf));
1041 8 : bufs[i] = *buf;
1042 : }
1043 4 : auto ret = LocalReduceNb(bufs.data(), count, dataType, outputDataType, opType, *len, *event, mask);
1044 4 : return HCCL_TO_CCU_RET(ret);
1045 4 : }
1046 :
1047 : /*========== 远端数据传输操作 ==========*/
1048 :
1049 4 : CcuResult CcuKernel::ResolveBufRemoteLenEvent(
1050 : CcuBufferHandle bufHandle, CcuRemoteAddrHandle remoteHandle, CcuVariableHandle lenHandle,
1051 : CcuEventHandle eventHandle, CcuRep::CcuBuf** buf, CcuRep::RemoteAddr** remote, CcuRep::Variable** len,
1052 : CcuRep::CompletedEvent** event)
1053 : {
1054 4 : CCU_CHK_RET(GetBufferByHandle(bufHandle, buf));
1055 4 : CCU_CHK_RET(GetRemoteAddrByHandle(remoteHandle, remote));
1056 4 : CCU_CHK_RET(GetVariableByHandle(lenHandle, len));
1057 4 : CCU_CHK_RET(GetEventByHandle(eventHandle, event));
1058 4 : return CcuResult::CCU_SUCCESS;
1059 : }
1060 :
1061 6 : CcuResult CcuKernel::ResolveLocalRemoteLenEvent(
1062 : CcuLocalAddrHandle localHandle, CcuRemoteAddrHandle remoteHandle, CcuVariableHandle lenHandle,
1063 : CcuEventHandle eventHandle, CcuRep::LocalAddr** local, CcuRep::RemoteAddr** remote, CcuRep::Variable** len,
1064 : CcuRep::CompletedEvent** event)
1065 : {
1066 6 : CCU_CHK_RET(GetLocalAddrByHandle(localHandle, local));
1067 6 : CCU_CHK_RET(GetRemoteAddrByHandle(remoteHandle, remote));
1068 6 : CCU_CHK_RET(GetVariableByHandle(lenHandle, len));
1069 6 : CCU_CHK_RET(GetEventByHandle(eventHandle, event));
1070 6 : return CcuResult::CCU_SUCCESS;
1071 : }
1072 :
1073 5 : CcuResult CcuKernel::ResolveRemoteLocalLenEvent(
1074 : CcuRemoteAddrHandle remoteHandle, CcuLocalAddrHandle localHandle, CcuVariableHandle lenHandle,
1075 : CcuEventHandle eventHandle, CcuRep::RemoteAddr** remote, CcuRep::LocalAddr** local, CcuRep::Variable** len,
1076 : CcuRep::CompletedEvent** event)
1077 : {
1078 5 : CCU_CHK_RET(GetRemoteAddrByHandle(remoteHandle, remote));
1079 5 : CCU_CHK_RET(GetLocalAddrByHandle(localHandle, local));
1080 5 : CCU_CHK_RET(GetVariableByHandle(lenHandle, len));
1081 5 : CCU_CHK_RET(GetEventByHandle(eventHandle, event));
1082 5 : return CcuResult::CCU_SUCCESS;
1083 : }
1084 :
1085 4 : CcuResult CcuKernel::ReadMemToMem(
1086 : ChannelHandle channel, CcuLocalAddrHandle localHandle, CcuRemoteAddrHandle remoteHandle,
1087 : CcuVariableHandle lenHandle, CcuEventHandle eventHandle, uint32_t mask)
1088 : {
1089 4 : PLF_CONFIG_INFO(
1090 : PLF_DATA_OP,
1091 : "[ReadMemToMem] channel=%llu, localHandle=%llu, remoteHandle=%llu,"
1092 : " lenHandle=%llu, eventHandle=%llu, mask=%u",
1093 : channel, localHandle, remoteHandle, lenHandle, eventHandle, mask);
1094 4 : channels_.insert(channel);
1095 4 : CcuRep::LocalAddr* local{nullptr};
1096 4 : CcuRep::RemoteAddr* remote{nullptr};
1097 4 : CcuRep::Variable* len{nullptr};
1098 4 : CcuRep::CompletedEvent* event{nullptr};
1099 4 : CCU_CHK_RET(
1100 : ResolveLocalRemoteLenEvent(localHandle, remoteHandle, lenHandle, eventHandle, &local, &remote, &len, &event));
1101 4 : auto ret = ReadNb(channel, *local, *remote, *len, *event, mask);
1102 4 : return HCCL_TO_CCU_RET(ret);
1103 : }
1104 :
1105 2 : CcuResult CcuKernel::ReadMemToBuffer(
1106 : ChannelHandle channel, CcuBufferHandle localHandle, CcuRemoteAddrHandle remoteHandle, CcuVariableHandle lenHandle,
1107 : CcuEventHandle eventHandle, uint32_t mask)
1108 : {
1109 2 : PLF_CONFIG_INFO(
1110 : PLF_DATA_OP,
1111 : "[ReadMemToBuffer] channel=%llu, bufHandle=%llu, remoteHandle=%llu,"
1112 : " lenHandle=%llu, eventHandle=%llu, mask=%u",
1113 : channel, localHandle, remoteHandle, lenHandle, eventHandle, mask);
1114 2 : channels_.insert(channel);
1115 2 : CcuRep::CcuBuf* local{nullptr};
1116 2 : CcuRep::RemoteAddr* remote{nullptr};
1117 2 : CcuRep::Variable* len{nullptr};
1118 2 : CcuRep::CompletedEvent* event{nullptr};
1119 2 : CCU_CHK_RET(
1120 : ResolveBufRemoteLenEvent(localHandle, remoteHandle, lenHandle, eventHandle, &local, &remote, &len, &event));
1121 2 : auto ret = ReadNb(channel, *local, *remote, *len, *event, mask);
1122 2 : return HCCL_TO_CCU_RET(ret);
1123 : }
1124 :
1125 2 : CcuResult CcuKernel::ReadMemToMemReduce(
1126 : ChannelHandle channel, CcuLocalAddrHandle localHandle, CcuRemoteAddrHandle remoteHandle,
1127 : CcuVariableHandle lenHandle, HcclDataType dataType, HcclReduceOp opType, CcuEventHandle eventHandle, uint32_t mask)
1128 : {
1129 2 : PLF_CONFIG_INFO(
1130 : PLF_DATA_OP,
1131 : "[ReadMemToMemReduce] channel=%llu, lenHandle=%llu, dataType=%d, op=%d,"
1132 : " eventHandle=%llu, mask=%u",
1133 : channel, lenHandle, dataType, opType, eventHandle, mask);
1134 2 : channels_.insert(channel);
1135 2 : CcuRep::LocalAddr* local{nullptr};
1136 2 : CcuRep::RemoteAddr* remote{nullptr};
1137 2 : CcuRep::Variable* len{nullptr};
1138 2 : CcuRep::CompletedEvent* event{nullptr};
1139 2 : CCU_CHK_RET(
1140 : ResolveLocalRemoteLenEvent(localHandle, remoteHandle, lenHandle, eventHandle, &local, &remote, &len, &event));
1141 2 : auto ret = ReadReduceNb(channel, *local, *remote, *len, dataType, opType, *event, mask);
1142 2 : return HCCL_TO_CCU_RET(ret);
1143 : }
1144 :
1145 3 : CcuResult CcuKernel::WriteMemToMem(
1146 : ChannelHandle channel, CcuRemoteAddrHandle remoteHandle, CcuLocalAddrHandle localHandle,
1147 : CcuVariableHandle lenHandle, CcuEventHandle eventHandle, uint32_t mask)
1148 : {
1149 3 : PLF_CONFIG_INFO(
1150 : PLF_DATA_OP,
1151 : "[WriteMemToMem] channel=%llu, remoteHandle=%llu, localHandle=%llu,"
1152 : " lenHandle=%llu, eventHandle=%llu, mask=%u",
1153 : channel, remoteHandle, localHandle, lenHandle, eventHandle, mask);
1154 3 : channels_.insert(channel);
1155 3 : CcuRep::RemoteAddr* remote{nullptr};
1156 3 : CcuRep::LocalAddr* local{nullptr};
1157 3 : CcuRep::Variable* len{nullptr};
1158 3 : CcuRep::CompletedEvent* event{nullptr};
1159 3 : CCU_CHK_RET(
1160 : ResolveRemoteLocalLenEvent(remoteHandle, localHandle, lenHandle, eventHandle, &remote, &local, &len, &event));
1161 3 : auto ret = WriteNb(channel, *remote, *local, *len, *event, mask);
1162 3 : return HCCL_TO_CCU_RET(ret);
1163 : }
1164 :
1165 2 : CcuResult CcuKernel::WriteBufferToMem(
1166 : ChannelHandle channel, CcuRemoteAddrHandle remoteHandle, CcuBufferHandle localHandle, CcuVariableHandle lenHandle,
1167 : CcuEventHandle eventHandle, uint32_t mask)
1168 : {
1169 2 : PLF_CONFIG_INFO(
1170 : PLF_DATA_OP,
1171 : "[WriteBufferToMem] channel=%llu, remoteHandle=%llu, bufHandle=%llu,"
1172 : " lenHandle=%llu, eventHandle=%llu, mask=%u",
1173 : channel, remoteHandle, localHandle, lenHandle, eventHandle, mask);
1174 2 : channels_.insert(channel);
1175 2 : CcuRep::CcuBuf* local{nullptr};
1176 2 : CcuRep::RemoteAddr* remote{nullptr};
1177 2 : CcuRep::Variable* len{nullptr};
1178 2 : CcuRep::CompletedEvent* event{nullptr};
1179 2 : CCU_CHK_RET(
1180 : ResolveBufRemoteLenEvent(localHandle, remoteHandle, lenHandle, eventHandle, &local, &remote, &len, &event));
1181 2 : auto ret = WriteNb(channel, *remote, *local, *len, *event, mask);
1182 2 : return HCCL_TO_CCU_RET(ret);
1183 : }
1184 :
1185 2 : CcuResult CcuKernel::WriteMemToMemReduce(
1186 : ChannelHandle channel, CcuRemoteAddrHandle remoteHandle, CcuLocalAddrHandle localHandle,
1187 : CcuVariableHandle lenHandle, HcclDataType dataType, HcclReduceOp opType, CcuEventHandle eventHandle, uint32_t mask)
1188 : {
1189 2 : PLF_CONFIG_INFO(
1190 : PLF_DATA_OP,
1191 : "[WriteMemToMemReduce] channel=%llu, lenHandle=%llu, dataType=%d, op=%d,"
1192 : " eventHandle=%llu, mask=%u",
1193 : channel, lenHandle, dataType, opType, eventHandle, mask);
1194 2 : channels_.insert(channel);
1195 2 : CcuRep::RemoteAddr* remote{nullptr};
1196 2 : CcuRep::LocalAddr* local{nullptr};
1197 2 : CcuRep::Variable* len{nullptr};
1198 2 : CcuRep::CompletedEvent* event{nullptr};
1199 2 : CCU_CHK_RET(
1200 : ResolveRemoteLocalLenEvent(remoteHandle, localHandle, lenHandle, eventHandle, &remote, &local, &len, &event));
1201 2 : auto ret = WriteReduceNb(channel, *remote, *local, *len, dataType, opType, *event, mask);
1202 2 : return HCCL_TO_CCU_RET(ret);
1203 : }
1204 :
1205 1497 : void CcuKernel::FlushClosablePendingIfs()
1206 : {
1207 1497 : if (isFlushing_) {
1208 82 : return;
1209 : }
1210 1415 : isFlushing_ = true;
1211 2912 : while (IfLabelStackTopIsClosable()) {
1212 82 : const char* lbl = IfLabelStackPop();
1213 82 : if (lbl != nullptr) {
1214 82 : IfEnd(lbl);
1215 : }
1216 : }
1217 1415 : isFlushing_ = false;
1218 : }
1219 :
1220 1232 : void CcuKernel::Append(std::shared_ptr<CcuRep::CcuRepBase> rep)
1221 : {
1222 1232 : FlushClosablePendingIfs();
1223 1232 : CcuRep::CcuRepContext::Append(rep);
1224 1232 : }
1225 :
1226 : namespace {
1227 104 : std::shared_ptr<CcuRep::CcuRepJumpBase> MakeInvertedCondJumpImm(
1228 : CcuInsGeneratorBase* insGenerator, const std::string& destLabelStr, const CcuRep::Variable& targetVar,
1229 : const CcuRep::Variable& expectVar, const CcuRep::Variable& variable, uint64_t immediate,
1230 : CcuConditionType condType, const char* funcName)
1231 : {
1232 104 : switch (condType) {
1233 82 : case CCU_CONDITION_EQ:
1234 164 : return std::make_shared<CcuRep::CcuRepJumpNE>(
1235 82 : insGenerator, destLabelStr, targetVar, expectVar, variable, immediate);
1236 12 : case CCU_CONDITION_NE:
1237 24 : return std::make_shared<CcuRep::CcuRepJumpEQ>(
1238 12 : insGenerator, destLabelStr, targetVar, expectVar, variable, immediate);
1239 3 : case CCU_CONDITION_LT:
1240 6 : return std::make_shared<CcuRep::CcuRepJumpGE>(
1241 3 : insGenerator, destLabelStr, targetVar, expectVar, variable, immediate);
1242 2 : case CCU_CONDITION_LE:
1243 4 : return std::make_shared<CcuRep::CcuRepJumpGT>(
1244 2 : insGenerator, destLabelStr, targetVar, expectVar, variable, immediate);
1245 2 : case CCU_CONDITION_GT:
1246 4 : return std::make_shared<CcuRep::CcuRepJumpLE>(
1247 2 : insGenerator, destLabelStr, targetVar, expectVar, variable, immediate);
1248 3 : case CCU_CONDITION_GE:
1249 6 : return std::make_shared<CcuRep::CcuRepJumpLT>(
1250 3 : insGenerator, destLabelStr, targetVar, expectVar, variable, immediate);
1251 0 : default:
1252 0 : HCCL_ERROR("[%s] unsupported condition type: %d", funcName, condType);
1253 0 : return nullptr;
1254 : }
1255 : }
1256 :
1257 : // 双变量版本：当 (lhsVar OP rhsVar) 为假时跳转到 destLabelStr。
1258 4 : std::shared_ptr<CcuRep::CcuRepJumpBase> MakeInvertedCondJumpVar(
1259 : CcuInsGeneratorBase* insGenerator, const std::string& destLabelStr, const CcuRep::Variable& targetVar,
1260 : const CcuRep::Variable& lhsVar, const CcuRep::Variable& rhsVar, CcuConditionType condType, const char* funcName)
1261 : {
1262 4 : switch (condType) {
1263 1 : case CCU_CONDITION_EQ:
1264 1 : return std::make_shared<CcuRep::CcuRepJumpNE>(insGenerator, destLabelStr, targetVar, lhsVar, rhsVar);
1265 0 : case CCU_CONDITION_NE:
1266 0 : return std::make_shared<CcuRep::CcuRepJumpEQ>(insGenerator, destLabelStr, targetVar, lhsVar, rhsVar);
1267 2 : case CCU_CONDITION_LT:
1268 2 : return std::make_shared<CcuRep::CcuRepJumpGE>(insGenerator, destLabelStr, targetVar, lhsVar, rhsVar);
1269 0 : case CCU_CONDITION_LE:
1270 0 : return std::make_shared<CcuRep::CcuRepJumpGT>(insGenerator, destLabelStr, targetVar, lhsVar, rhsVar);
1271 0 : case CCU_CONDITION_GT:
1272 0 : return std::make_shared<CcuRep::CcuRepJumpLE>(insGenerator, destLabelStr, targetVar, lhsVar, rhsVar);
1273 1 : case CCU_CONDITION_GE:
1274 1 : return std::make_shared<CcuRep::CcuRepJumpLT>(insGenerator, destLabelStr, targetVar, lhsVar, rhsVar);
1275 0 : default:
1276 0 : HCCL_ERROR("[%s] unsupported condition type: %d", funcName, condType);
1277 0 : return nullptr;
1278 : }
1279 : }
1280 : } // namespace
1281 :
1282 : CcuResult
1283 102 : CcuKernel::IfBegin(CcuVariableHandle varHandle, uint64_t immediate, CcuConditionType condType, const char* label)
1284 : {
1285 102 : PLF_CONFIG_INFO(
1286 : PLF_DATA_OP, "[IfBegin] varHandle=%llu, immediate=%llu, condType=%d, label=%s", varHandle, immediate, condType,
1287 : (label ? label : "null"));
1288 102 : if (CurrentBlock()->Type() == CcuRep::CcuRepType::LOOP_BLOCK) {
1289 1 : HCCL_ERROR(
1290 : "[%s] CCU_IF is not allowed inside a ccu::Loop body (label='%s')", __func__,
1291 : label != nullptr ? label : "(null)");
1292 1 : return LatchBodyError(CcuResult::CCU_E_INTERNAL);
1293 : }
1294 101 : CcuRep::Variable* variable{nullptr};
1295 101 : CCU_CHK_RET(GetVariableByHandle(varHandle, &variable));
1296 :
1297 101 : FlushClosablePendingIfs();
1298 :
1299 101 : std::string labelStr(label);
1300 101 : if (pendingIfCtx_.find(labelStr) != pendingIfCtx_.end()) {
1301 0 : HCCL_ERROR("[%s] label '%s' already has a pending IfBegin without IfEnd", __func__, label);
1302 0 : return CcuResult::CCU_E_PARA;
1303 : }
1304 :
1305 101 : std::string elseLabelStr = labelStr + "_else";
1306 101 : std::string endLabelStr = labelStr + "_end";
1307 101 : auto elseLabel = std::make_shared<CcuRep::CcuRepJumpLabel>(insGenerator, elseLabelStr);
1308 101 : auto endLabel = std::make_shared<CcuRep::CcuRepJumpLabel>(insGenerator, endLabelStr);
1309 101 : auto targetVar = CreateJumpTargetVar();
1310 101 : auto expectVar = CreateExpectVar();
1311 :
1312 : // 反转条件："if <cond>, 执行块" 等价于 "!<cond> 时跳过块"。
1313 : auto jump = MakeInvertedCondJumpImm(
1314 101 : insGenerator, elseLabelStr, targetVar, expectVar, *variable, immediate, condType, __func__);
1315 101 : if (jump == nullptr) {
1316 0 : return CcuResult::CCU_E_PARA;
1317 : }
1318 101 : jump->Reference(elseLabel);
1319 101 : Append(jump);
1320 :
1321 101 : PendingIfContext ctx;
1322 101 : ctx.elseLabel = elseLabel;
1323 101 : ctx.endLabel = endLabel;
1324 101 : ctx.hasElse = false;
1325 101 : pendingIfCtx_.emplace(labelStr, std::move(ctx));
1326 :
1327 101 : return CcuResult::CCU_SUCCESS;
1328 101 : }
1329 :
1330 3 : CcuResult CcuKernel::IfBeginVar(
1331 : CcuVariableHandle lhsHandle, CcuVariableHandle rhsHandle, CcuConditionType condType, const char* label)
1332 : {
1333 3 : if (CurrentBlock()->Type() == CcuRep::CcuRepType::LOOP_BLOCK) {
1334 0 : HCCL_ERROR(
1335 : "[%s] CCU_IF is not allowed inside a ccu::Loop body (label='%s')", __func__,
1336 : label != nullptr ? label : "(null)");
1337 0 : return LatchBodyError(CcuResult::CCU_E_INTERNAL);
1338 : }
1339 3 : CcuRep::Variable* lhsVar{nullptr};
1340 3 : CcuRep::Variable* rhsVar{nullptr};
1341 3 : CCU_CHK_RET(GetVariableByHandle(lhsHandle, &lhsVar));
1342 3 : CCU_CHK_RET(GetVariableByHandle(rhsHandle, &rhsVar));
1343 :
1344 3 : FlushClosablePendingIfs();
1345 :
1346 3 : std::string labelStr(label);
1347 3 : if (pendingIfCtx_.find(labelStr) != pendingIfCtx_.end()) {
1348 0 : HCCL_ERROR("[%s] label '%s' already has a pending IfBegin without IfEnd", __func__, label);
1349 0 : return CcuResult::CCU_E_PARA;
1350 : }
1351 :
1352 3 : std::string endLabelStr = labelStr + "_end";
1353 3 : std::string elseLabelStr = labelStr + "_else";
1354 3 : auto endLabel = std::make_shared<CcuRep::CcuRepJumpLabel>(insGenerator, endLabelStr);
1355 3 : auto elseLabel = std::make_shared<CcuRep::CcuRepJumpLabel>(insGenerator, elseLabelStr);
1356 3 : auto targetVar = CreateJumpTargetVar();
1357 :
1358 3 : auto jump = MakeInvertedCondJumpVar(insGenerator, elseLabelStr, targetVar, *lhsVar, *rhsVar, condType, __func__);
1359 3 : if (jump == nullptr) {
1360 0 : return CcuResult::CCU_E_PARA;
1361 : }
1362 3 : jump->Reference(elseLabel);
1363 3 : Append(jump);
1364 :
1365 3 : PendingIfContext ctx;
1366 3 : ctx.hasElse = false;
1367 3 : ctx.endLabel = endLabel;
1368 3 : ctx.elseLabel = elseLabel;
1369 3 : pendingIfCtx_.emplace(labelStr, std::move(ctx));
1370 :
1371 3 : return CcuResult::CCU_SUCCESS;
1372 3 : }
1373 :
1374 22 : CcuResult CcuKernel::IfElse(const char* label)
1375 : {
1376 22 : PLF_CONFIG_INFO(PLF_DATA_OP, "[IfElse] label=%s", (label ? label : "null"));
1377 22 : if (CurrentBlock()->Type() == CcuRep::CcuRepType::LOOP_BLOCK) {
1378 0 : HCCL_ERROR(
1379 : "[%s] CCU_ELSE is not allowed inside a ccu::Loop body (label='%s')", __func__,
1380 : label != nullptr ? label : "(null)");
1381 0 : return LatchBodyError(CcuResult::CCU_E_INTERNAL);
1382 : }
1383 :
1384 22 : std::string labelStr(label);
1385 22 : auto iter = pendingIfCtx_.find(labelStr);
1386 22 : if (iter == pendingIfCtx_.end()) {
1387 0 : HCCL_ERROR("[%s] no matching IfBegin for label '%s'", __func__, label);
1388 0 : return CcuResult::CCU_E_NOT_FOUND;
1389 : }
1390 :
1391 22 : if (iter->second.hasElse) {
1392 0 : HCCL_ERROR("[%s] label '%s' already has an IfElse", __func__, label);
1393 0 : return CcuResult::CCU_E_PARA;
1394 : }
1395 :
1396 : // At end of then-block: unconditional jump past else-block to endLabel
1397 22 : std::string endLabelStr = labelStr + "_end";
1398 22 : auto skipElseVar = CreateJumpTargetVar();
1399 22 : auto skipElseJump = std::make_shared<CcuRep::CcuRepJump>(insGenerator, endLabelStr, skipElseVar);
1400 22 : skipElseJump->Reference(iter->second.endLabel);
1401 22 : Append(skipElseJump);
1402 :
1403 : // Place the else label (entry point of else-block)
1404 22 : Append(iter->second.elseLabel);
1405 :
1406 22 : iter->second.hasElse = true;
1407 :
1408 22 : return CcuResult::CCU_SUCCESS;
1409 22 : }
1410 :
1411 104 : CcuResult CcuKernel::IfEnd(const char* label)
1412 : {
1413 104 : PLF_CONFIG_INFO(PLF_DATA_OP, "[IfEnd] label=%s", (label ? label : "null"));
1414 104 : std::string labelStr(label);
1415 104 : auto iter = pendingIfCtx_.find(labelStr);
1416 104 : if (iter == pendingIfCtx_.end()) {
1417 0 : HCCL_ERROR("[%s] no matching IfBegin for label '%s'", __func__, label);
1418 0 : return CcuResult::CCU_E_NOT_FOUND;
1419 : }
1420 :
1421 104 : if (iter->second.hasElse) {
1422 : // Had else-block: place endLabel after else-block
1423 22 : Append(iter->second.endLabel);
1424 : } else {
1425 : // No else-block: place elseLabel as the skip target
1426 82 : Append(iter->second.elseLabel);
1427 : }
1428 :
1429 104 : pendingIfCtx_.erase(iter);
1430 :
1431 104 : return CcuResult::CCU_SUCCESS;
1432 104 : }
1433 :
1434 : CcuResult
1435 3 : CcuKernel::WhileBegin(CcuVariableHandle varHandle, uint64_t immediate, CcuConditionType condType, const char* label)
1436 : {
1437 3 : PLF_CONFIG_INFO(
1438 : PLF_DATA_OP, "[WhileBegin] varHandle=%llu, immediate=%llu, condType=%d, label=%s", varHandle, immediate,
1439 : condType, (label ? label : "null"));
1440 3 : if (CurrentBlock()->Type() == CcuRep::CcuRepType::LOOP_BLOCK) {
1441 0 : HCCL_ERROR(
1442 : "[%s] CCU_WHILE is not allowed inside a ccu::Loop body (label='%s')", __func__,
1443 : label != nullptr ? label : "(null)");
1444 0 : return LatchBodyError(CcuResult::CCU_E_INTERNAL);
1445 : }
1446 :
1447 3 : CcuRep::Variable* variable{nullptr};
1448 3 : CCU_CHK_RET(GetVariableByHandle(varHandle, &variable));
1449 :
1450 3 : std::string labelStr(label);
1451 3 : if (pendingWhileCtx_.find(labelStr) != pendingWhileCtx_.end()) {
1452 0 : HCCL_ERROR("[%s] label '%s' already has a pending WhileBegin without WhileEnd", __func__, label);
1453 0 : return CcuResult::CCU_E_PARA;
1454 : }
1455 :
1456 3 : std::string beginLabelStr = labelStr + "_begin";
1457 3 : std::string endLabelStr = labelStr + "_end";
1458 3 : auto beginLabel = std::make_shared<CcuRep::CcuRepJumpLabel>(insGenerator, beginLabelStr);
1459 3 : auto endLabel = std::make_shared<CcuRep::CcuRepJumpLabel>(insGenerator, endLabelStr);
1460 :
1461 3 : Append(beginLabel);
1462 :
1463 3 : auto targetVar = CcuRep::CreateVariable(this);
1464 3 : auto expectVar = CreateExpectVar();
1465 : auto jump = MakeInvertedCondJumpImm(
1466 3 : insGenerator, endLabelStr, targetVar, expectVar, *variable, immediate, condType, __func__);
1467 3 : if (jump == nullptr) {
1468 0 : return CcuResult::CCU_E_PARA;
1469 : }
1470 3 : jump->Reference(endLabel);
1471 3 : Append(jump);
1472 :
1473 3 : PendingWhileContext ctx;
1474 3 : ctx.beginLabel = beginLabel;
1475 3 : ctx.endLabel = endLabel;
1476 3 : ctx.varHandle = varHandle;
1477 3 : ctx.immediate = immediate;
1478 3 : ctx.condType = condType;
1479 3 : pendingWhileCtx_.emplace(labelStr, std::move(ctx));
1480 :
1481 3 : return CcuResult::CCU_SUCCESS;
1482 3 : }
1483 :
1484 1 : CcuResult CcuKernel::WhileBeginVar(
1485 : CcuVariableHandle lhsHandle, CcuVariableHandle rhsHandle, CcuConditionType condType, const char* label)
1486 : {
1487 1 : if (CurrentBlock()->Type() == CcuRep::CcuRepType::LOOP_BLOCK) {
1488 0 : HCCL_ERROR(
1489 : "[%s] CCU_WHILE is not allowed inside a ccu::Loop body (label='%s')", __func__,
1490 : label != nullptr ? label : "(null)");
1491 0 : return LatchBodyError(CcuResult::CCU_E_INTERNAL);
1492 : }
1493 :
1494 1 : CcuRep::Variable* lhsVar{nullptr};
1495 1 : CcuRep::Variable* rhsVar{nullptr};
1496 1 : CCU_CHK_RET(GetVariableByHandle(lhsHandle, &lhsVar));
1497 1 : CCU_CHK_RET(GetVariableByHandle(rhsHandle, &rhsVar));
1498 :
1499 1 : std::string labelStr(label);
1500 1 : if (pendingWhileCtx_.find(labelStr) != pendingWhileCtx_.end()) {
1501 0 : HCCL_ERROR("[%s] label '%s' already has a pending WhileBegin without WhileEnd", __func__, label);
1502 0 : return CcuResult::CCU_E_PARA;
1503 : }
1504 :
1505 1 : std::string endLabelStr = labelStr + "_end";
1506 1 : std::string beginLabelStr = labelStr + "_begin";
1507 1 : auto endLabel = std::make_shared<CcuRep::CcuRepJumpLabel>(insGenerator, endLabelStr);
1508 1 : auto beginLabel = std::make_shared<CcuRep::CcuRepJumpLabel>(insGenerator, beginLabelStr);
1509 :
1510 1 : Append(beginLabel);
1511 :
1512 1 : auto targetVar = CcuRep::CreateVariable(this);
1513 1 : auto jump = MakeInvertedCondJumpVar(insGenerator, endLabelStr, targetVar, *lhsVar, *rhsVar, condType, __func__);
1514 1 : if (jump == nullptr) {
1515 0 : return CcuResult::CCU_E_PARA;
1516 : }
1517 1 : jump->Reference(endLabel);
1518 1 : Append(jump);
1519 :
1520 1 : PendingWhileContext ctx;
1521 1 : ctx.beginLabel = beginLabel;
1522 1 : ctx.endLabel = endLabel;
1523 1 : ctx.varHandle = lhsHandle;
1524 1 : ctx.immediate = 0;
1525 1 : ctx.condType = condType;
1526 1 : pendingWhileCtx_.emplace(labelStr, std::move(ctx));
1527 :
1528 1 : return CcuResult::CCU_SUCCESS;
1529 1 : }
1530 :
1531 4 : CcuResult CcuKernel::WhileEnd(const char* label)
1532 : {
1533 4 : PLF_CONFIG_INFO(PLF_DATA_OP, "[WhileEnd] label=%s", (label ? label : "null"));
1534 4 : std::string labelStr(label);
1535 4 : auto iter = pendingWhileCtx_.find(labelStr);
1536 4 : if (iter == pendingWhileCtx_.end()) {
1537 0 : HCCL_ERROR("[%s] no matching WhileBegin for label '%s'", __func__, label);
1538 0 : return CcuResult::CCU_E_NOT_FOUND;
1539 : }
1540 :
1541 4 : std::string beginLabelStr = labelStr + "_begin";
1542 4 : auto loopBackVar = CcuRep::CreateVariable(this);
1543 4 : auto loopBackJump = std::make_shared<CcuRep::CcuRepJump>(insGenerator, beginLabelStr, loopBackVar);
1544 4 : loopBackJump->Reference(iter->second.beginLabel);
1545 4 : Append(loopBackJump);
1546 :
1547 4 : Append(iter->second.endLabel);
1548 :
1549 4 : pendingWhileCtx_.erase(iter);
1550 :
1551 4 : return CcuResult::CCU_SUCCESS;
1552 4 : }
1553 :
1554 9 : CcuResult CcuKernel::DoWhileBegin(const char* label)
1555 : {
1556 9 : PLF_CONFIG_INFO(PLF_DATA_OP, "[DoWhileBegin] label=%s", (label ? label : "null"));
1557 9 : if (CurrentBlock()->Type() == CcuRep::CcuRepType::LOOP_BLOCK) {
1558 0 : HCCL_ERROR(
1559 : "[%s] CCU_DO is not allowed inside a ccu::Loop body (label='%s')", __func__,
1560 : label != nullptr ? label : "(null)");
1561 0 : return LatchBodyError(CcuResult::CCU_E_INTERNAL);
1562 : }
1563 :
1564 9 : std::string labelStr(label);
1565 9 : if (pendingDoWhileCtx_.find(labelStr) != pendingDoWhileCtx_.end()) {
1566 0 : HCCL_ERROR("[%s] label '%s' already has a pending DoWhileBegin without DoWhileEnd", __func__, label);
1567 0 : return CcuResult::CCU_E_PARA;
1568 : }
1569 :
1570 9 : std::string beginLabelStr = labelStr + "_begin";
1571 9 : auto beginLabel = std::make_shared<CcuRep::CcuRepJumpLabel>(insGenerator, beginLabelStr);
1572 9 : Append(beginLabel);
1573 :
1574 9 : PendingDoWhileContext ctx;
1575 9 : ctx.beginLabel = beginLabel;
1576 9 : pendingDoWhileCtx_.emplace(labelStr, std::move(ctx));
1577 :
1578 9 : return CcuResult::CCU_SUCCESS;
1579 9 : }
1580 :
1581 : CcuResult
1582 5 : CcuKernel::DoWhileEnd(CcuVariableHandle varHandle, uint64_t immediate, CcuConditionType condType, const char* label)
1583 : {
1584 5 : PLF_CONFIG_INFO(
1585 : PLF_DATA_OP, "[DoWhileEnd] varHandle=%llu, immediate=%llu, condType=%d, label=%s", varHandle, immediate,
1586 : condType, (label ? label : "null"));
1587 5 : CcuRep::Variable* variable{nullptr};
1588 5 : CCU_CHK_RET(GetVariableByHandle(varHandle, &variable));
1589 :
1590 5 : std::string labelStr(label);
1591 5 : auto iter = pendingDoWhileCtx_.find(labelStr);
1592 5 : if (iter == pendingDoWhileCtx_.end()) {
1593 0 : HCCL_ERROR("[%s] no matching DoWhileBegin for label '%s'", __func__, label);
1594 0 : return CcuResult::CCU_E_NOT_FOUND;
1595 : }
1596 :
1597 5 : std::string beginLabelStr = labelStr + "_begin";
1598 5 : auto targetVar = CcuRep::CreateVariable(this);
1599 5 : auto expectVar = CreateExpectVar();
1600 5 : std::shared_ptr<CcuRep::CcuRepJumpBase> jump{nullptr};
1601 :
1602 : // "condition true => continue looping" means jump back to begin when condition holds
1603 5 : if (condType == CCU_CONDITION_EQ) {
1604 0 : jump = std::make_shared<CcuRep::CcuRepJumpEQ>(
1605 0 : insGenerator, beginLabelStr, targetVar, expectVar, *variable, immediate);
1606 5 : } else if (condType == CCU_CONDITION_NE) {
1607 4 : jump = std::make_shared<CcuRep::CcuRepJumpNE>(
1608 4 : insGenerator, beginLabelStr, targetVar, expectVar, *variable, immediate);
1609 1 : } else if (condType == CCU_CONDITION_LT) {
1610 0 : jump = std::make_shared<CcuRep::CcuRepJumpLT>(
1611 0 : insGenerator, beginLabelStr, targetVar, expectVar, *variable, immediate);
1612 1 : } else if (condType == CCU_CONDITION_LE) {
1613 1 : jump = std::make_shared<CcuRep::CcuRepJumpLE>(
1614 1 : insGenerator, beginLabelStr, targetVar, expectVar, *variable, immediate);
1615 0 : } else if (condType == CCU_CONDITION_GT) {
1616 0 : jump = std::make_shared<CcuRep::CcuRepJumpGT>(
1617 0 : insGenerator, beginLabelStr, targetVar, expectVar, *variable, immediate);
1618 0 : } else if (condType == CCU_CONDITION_GE) {
1619 0 : jump = std::make_shared<CcuRep::CcuRepJumpGE>(
1620 0 : insGenerator, beginLabelStr, targetVar, expectVar, *variable, immediate);
1621 : } else {
1622 0 : HCCL_ERROR("[%s] unsupported condition type: %d", __func__, condType);
1623 0 : return CcuResult::CCU_E_PARA;
1624 : }
1625 :
1626 5 : jump->Reference(iter->second.beginLabel);
1627 5 : Append(jump);
1628 :
1629 5 : pendingDoWhileCtx_.erase(iter);
1630 :
1631 5 : return CcuResult::CCU_SUCCESS;
1632 5 : }
1633 :
1634 1 : CcuResult CcuKernel::DoWhileEndVar(
1635 : CcuVariableHandle lhsHandle, CcuVariableHandle rhsHandle, CcuConditionType condType, const char* label)
1636 : {
1637 1 : CcuRep::Variable* lhsVar{nullptr};
1638 1 : CcuRep::Variable* rhsVar{nullptr};
1639 1 : CCU_CHK_RET(GetVariableByHandle(lhsHandle, &lhsVar));
1640 1 : CCU_CHK_RET(GetVariableByHandle(rhsHandle, &rhsVar));
1641 :
1642 1 : std::string labelStr(label);
1643 1 : auto iter = pendingDoWhileCtx_.find(labelStr);
1644 1 : if (iter == pendingDoWhileCtx_.end()) {
1645 0 : HCCL_ERROR("[%s] no matching DoWhileBegin for label '%s'", __func__, label);
1646 0 : return CcuResult::CCU_E_NOT_FOUND;
1647 : }
1648 :
1649 1 : std::string beginLabelStr = labelStr + "_begin";
1650 1 : auto targetVar = CcuRep::CreateVariable(this);
1651 1 : std::shared_ptr<CcuRep::CcuRepJumpBase> jump{nullptr};
1652 :
1653 1 : if (condType == CCU_CONDITION_EQ) {
1654 0 : jump = std::make_shared<CcuRep::CcuRepJumpEQ>(insGenerator, beginLabelStr, targetVar, *lhsVar, *rhsVar);
1655 1 : } else if (condType == CCU_CONDITION_NE) {
1656 1 : jump = std::make_shared<CcuRep::CcuRepJumpNE>(insGenerator, beginLabelStr, targetVar, *lhsVar, *rhsVar);
1657 0 : } else if (condType == CCU_CONDITION_LT) {
1658 0 : jump = std::make_shared<CcuRep::CcuRepJumpLT>(insGenerator, beginLabelStr, targetVar, *lhsVar, *rhsVar);
1659 0 : } else if (condType == CCU_CONDITION_LE) {
1660 0 : jump = std::make_shared<CcuRep::CcuRepJumpLE>(insGenerator, beginLabelStr, targetVar, *lhsVar, *rhsVar);
1661 0 : } else if (condType == CCU_CONDITION_GT) {
1662 0 : jump = std::make_shared<CcuRep::CcuRepJumpGT>(insGenerator, beginLabelStr, targetVar, *lhsVar, *rhsVar);
1663 0 : } else if (condType == CCU_CONDITION_GE) {
1664 0 : jump = std::make_shared<CcuRep::CcuRepJumpGE>(insGenerator, beginLabelStr, targetVar, *lhsVar, *rhsVar);
1665 : } else {
1666 0 : HCCL_ERROR("[%s] unsupported condition type: %d", __func__, condType);
1667 0 : return CcuResult::CCU_E_PARA;
1668 : }
1669 :
1670 1 : jump->Reference(iter->second.beginLabel);
1671 1 : Append(jump);
1672 :
1673 1 : pendingDoWhileCtx_.erase(iter);
1674 :
1675 1 : return CcuResult::CCU_SUCCESS;
1676 1 : }
1677 :
1678 : // 控制流标签栈实体
1679 104 : void CcuKernel::IfLabelStackPush(const char* label) { iflabelStack_.push_back({label, false}); }
1680 104 : void CcuKernel::IfLabelStackMarkBodyDone()
1681 : {
1682 104 : if (iflabelStack_.empty()) {
1683 0 : HCCL_ERROR("[CcuKernel::IfLabelStack][MarkBodyDone] stack is empty");
1684 0 : return;
1685 : }
1686 104 : iflabelStack_.back().bodyDone = true;
1687 : }
1688 22 : const char* CcuKernel::IfLabelStackPopForElse()
1689 : {
1690 22 : if (iflabelStack_.empty()) {
1691 0 : HCCL_ERROR("[CcuKernel::IfLabelStack][PopForElse] orphan CCU_ELSE: "
1692 : "no matching CCU_IF on the stack");
1693 0 : return nullptr;
1694 : }
1695 22 : if (!iflabelStack_.back().bodyDone) {
1696 0 : HCCL_ERROR(
1697 : "[CcuKernel::IfLabelStack][PopForElse] CCU_ELSE called while "
1698 : "top if-body is still InBody (label='%s')",
1699 : iflabelStack_.back().label != nullptr ? iflabelStack_.back().label : "(null)");
1700 0 : return nullptr;
1701 : }
1702 22 : const char* label = iflabelStack_.back().label;
1703 22 : iflabelStack_.pop_back();
1704 22 : return label;
1705 : }
1706 1497 : bool CcuKernel::IfLabelStackTopIsClosable() { return !iflabelStack_.empty() && iflabelStack_.back().bodyDone; }
1707 :
1708 82 : const char* CcuKernel::IfLabelStackPop()
1709 : {
1710 82 : if (iflabelStack_.empty()) {
1711 0 : return nullptr;
1712 : }
1713 82 : const char* label = iflabelStack_.back().label;
1714 82 : iflabelStack_.pop_back();
1715 82 : return label;
1716 : }
1717 :
1718 9 : void CcuKernel::DoWhileLabelStackPush(const char* label)
1719 : {
1720 9 : DoWhileLabelEntry entry;
1721 9 : entry.label = label;
1722 9 : entry.snapshotBlock = CurrentBlock();
1723 9 : entry.snapshotRepCount = (entry.snapshotBlock != nullptr) ? entry.snapshotBlock->GetReps().size() : 0;
1724 9 : doWhileLabelStack_.push_back(std::move(entry));
1725 9 : }
1726 :
1727 13 : const char* CcuKernel::DoWhileLabelStackPopForWhile()
1728 : {
1729 13 : if (CurrentBlock()->Type() == CcuRep::CcuRepType::LOOP_BLOCK) {
1730 1 : return nullptr;
1731 : }
1732 12 : if (doWhileLabelStack_.empty()) {
1733 4 : return nullptr;
1734 : }
1735 8 : DoWhileLabelEntry entry = doWhileLabelStack_.back();
1736 8 : doWhileLabelStack_.pop_back();
1737 :
1738 8 : auto currentBlock = CurrentBlock();
1739 8 : size_t currentRepCount = (currentBlock != nullptr) ? currentBlock->GetReps().size() : 0;
1740 8 : if (currentBlock != entry.snapshotBlock || currentRepCount != entry.snapshotRepCount) {
1741 1 : HCCL_ERROR(
1742 : "[CcuKernel::DoWhileLabelStackPopForWhile] dangling CCU calls between CCU_DO end "
1743 : "and CCU_WHILE (label='%s', snapRep=%zu, curRep=%zu, blockChanged=%d); they must "
1744 : "be syntactically adjacent, otherwise the code in between is pulled into the body.",
1745 : entry.label != nullptr ? entry.label : "(null)", entry.snapshotRepCount, currentRepCount,
1746 : currentBlock != entry.snapshotBlock ? 1 : 0);
1747 1 : return nullptr;
1748 : }
1749 7 : return entry.label;
1750 8 : }
1751 :
1752 152 : CcuResult CcuKernel::GetAddressByHandle(CcuAddressHandle addrHandle, CcuRep::Address** address)
1753 : {
1754 152 : return GetResourceByHandle(ccuAddrMap_, addrHandle, address, "address");
1755 : }
1756 :
1757 : // addr = 立即数 → CcuRepAssign(Address, uint64_t)
1758 13 : CcuResult CcuKernel::AddressAssignImm(CcuAddressHandle addrHandle, uint64_t immediate)
1759 : {
1760 13 : PLF_CONFIG_INFO(PLF_DATA_OP, "[AddressAssignImm] addrHandle=%llu, immediate=%llu", addrHandle, immediate);
1761 13 : CcuRep::Address* address{nullptr};
1762 13 : CCU_CHK_RET(GetAddressByHandle(addrHandle, &address));
1763 13 : (*address) = immediate;
1764 13 : return CcuResult::CCU_SUCCESS;
1765 : }
1766 :
1767 : // addr = variable → CcuRepAssign(Address, Variable)
1768 14 : CcuResult CcuKernel::AddressAssignVar(CcuAddressHandle addrHandle, CcuVariableHandle varHandle)
1769 : {
1770 14 : PLF_CONFIG_INFO(PLF_DATA_OP, "[AddressAssignVar] addrHandle=%llu, varHandle=%llu", addrHandle, varHandle);
1771 14 : CcuRep::Address* address{nullptr};
1772 14 : CCU_CHK_RET(GetAddressByHandle(addrHandle, &address));
1773 :
1774 14 : CcuRep::Variable* variable{nullptr};
1775 14 : CCU_CHK_RET(GetVariableByHandle(varHandle, &variable));
1776 :
1777 14 : (*address) = (*variable);
1778 14 : return CcuResult::CCU_SUCCESS;
1779 : }
1780 : // addr = addr → CcuRepAssign(Address, Address)
1781 38 : CcuResult CcuKernel::AddressAssignAddr(CcuAddressHandle dstAddrHandle, CcuAddressHandle srcAddrHandle)
1782 : {
1783 38 : PLF_CONFIG_INFO(
1784 : PLF_DATA_OP, "[AddressAssignAddr] dstAddrHandle=%llu, srcAddrHandle=%llu", dstAddrHandle, srcAddrHandle);
1785 38 : CcuRep::Address* dstAddress{nullptr};
1786 38 : CCU_CHK_RET(GetAddressByHandle(dstAddrHandle, &dstAddress));
1787 :
1788 38 : CcuRep::Address* srcAddress{nullptr};
1789 38 : CCU_CHK_RET(GetAddressByHandle(srcAddrHandle, &srcAddress));
1790 :
1791 38 : (*dstAddress) = (*srcAddress);
1792 38 : return CcuResult::CCU_SUCCESS;
1793 : }
1794 :
1795 : // resAddr = lhsAddr + rhsVar → CcuRepAdd(Address, Address, Variable)
1796 0 : CcuResult CcuKernel::AddressAddVarToAddr(
1797 : CcuAddressHandle resAddrHandle, CcuAddressHandle lhsAddrHandle, CcuVariableHandle rhsVarHandle)
1798 : {
1799 0 : PLF_CONFIG_INFO(
1800 : PLF_DATA_OP, "[AddressAddVarToAddr] resAddrHandle=%llu, lhsAddrHandle=%llu, rhsVarHandle=%llu", resAddrHandle,
1801 : lhsAddrHandle, rhsVarHandle);
1802 0 : CcuRep::Address *resAddr{nullptr}, *lhsAddr{nullptr};
1803 0 : CCU_CHK_RET(GetAddressByHandle(resAddrHandle, &resAddr));
1804 0 : CCU_CHK_RET(GetAddressByHandle(lhsAddrHandle, &lhsAddr));
1805 :
1806 0 : CcuRep::Variable* rhsVar{nullptr};
1807 0 : CCU_CHK_RET(GetVariableByHandle(rhsVarHandle, &rhsVar));
1808 :
1809 0 : *resAddr = *lhsAddr + *rhsVar;
1810 0 : return CcuResult::CCU_SUCCESS;
1811 : }
1812 :
1813 : // resAddr = addrA + addrB → CcuRepAdd(Address, Address, Address)
1814 2 : CcuResult CcuKernel::AddressAddAddrToAddr(
1815 : CcuAddressHandle resAddrHandle, CcuAddressHandle addrAHandle, CcuAddressHandle addrBHandle)
1816 : {
1817 2 : PLF_CONFIG_INFO(
1818 : PLF_DATA_OP, "[AddressAddAddrToAddr] resAddrHandle=%llu, addrAHandle=%llu, addrBHandle=%llu", resAddrHandle,
1819 : addrAHandle, addrBHandle);
1820 2 : CcuRep::Address *resAddr{nullptr}, *addrA{nullptr}, *addrB{nullptr};
1821 2 : CCU_CHK_RET(GetAddressByHandle(resAddrHandle, &resAddr));
1822 2 : CCU_CHK_RET(GetAddressByHandle(addrAHandle, &addrA));
1823 2 : CCU_CHK_RET(GetAddressByHandle(addrBHandle, &addrB));
1824 :
1825 2 : *resAddr = *addrA + *addrB;
1826 2 : return CcuResult::CCU_SUCCESS;
1827 : }
1828 :
1829 : // addr += variable → CcuRepAdd(Address, Variable) 就地加
1830 41 : CcuResult CcuKernel::AddressAddAssignVar(CcuAddressHandle addrHandle, CcuVariableHandle varHandle)
1831 : {
1832 41 : PLF_CONFIG_INFO(PLF_DATA_OP, "[AddressAddAssignVar] addrHandle=%llu, varHandle=%llu", addrHandle, varHandle);
1833 41 : CcuRep::Address* address{nullptr};
1834 41 : CCU_CHK_RET(GetAddressByHandle(addrHandle, &address));
1835 :
1836 41 : CcuRep::Variable* variable{nullptr};
1837 41 : CCU_CHK_RET(GetVariableByHandle(varHandle, &variable));
1838 :
1839 41 : (*address) += (*variable);
1840 41 : return CcuResult::CCU_SUCCESS;
1841 : }
1842 :
1843 : // addr += addr → 等价于 addr = addr + otherAddr
1844 0 : CcuResult CcuKernel::AddressAddAssignAddr(CcuAddressHandle addrHandle, CcuAddressHandle otherHandle)
1845 : {
1846 0 : PLF_CONFIG_INFO(PLF_DATA_OP, "[AddressAddAssignAddr] addrHandle=%llu, otherHandle=%llu", addrHandle, otherHandle);
1847 0 : CcuRep::Address* address{nullptr};
1848 0 : CCU_CHK_RET(GetAddressByHandle(addrHandle, &address));
1849 :
1850 0 : CcuRep::Address* other{nullptr};
1851 0 : CCU_CHK_RET(GetAddressByHandle(otherHandle, &other));
1852 :
1853 0 : (*address) = (*address) + (*other);
1854 0 : return CcuResult::CCU_SUCCESS;
1855 : }
1856 :
1857 1 : CcuResult CcuKernel::AddressAddImmToAddr(CcuAddressHandle resAddrHandle, CcuAddressHandle addrAHandle, uint16_t imm)
1858 : {
1859 1 : CcuRep::Address *resAddr{nullptr}, *addrA{nullptr};
1860 1 : CCU_CHK_RET(GetAddressByHandle(resAddrHandle, &resAddr));
1861 1 : CCU_CHK_RET(GetAddressByHandle(addrAHandle, &addrA));
1862 :
1863 1 : *resAddr = *addrA + imm;
1864 1 : return CcuResult::CCU_SUCCESS;
1865 : }
1866 :
1867 0 : void CcuKernel::Load(const CcuRep::Variable& var)
1868 : {
1869 : auto loadArgRep = std::make_shared<CcuRep::CcuRepLoadArg>(
1870 0 : insGenerator, var, loadArgIndex_ % CCU_SQE_ARGS_LEN, static_cast<uint16_t>(loadArgIndex_));
1871 0 : GetLGProfilingInfo().loadRep2ArgIdxMap[loadArgRep] = loadArgIndex_;
1872 0 : Append(loadArgRep);
1873 0 : loadArgIndex_++;
1874 0 : }
1875 :
1876 0 : void CcuKernel::StoreVariable(const CcuRep::Variable& var, uint64_t addr)
1877 : {
1878 0 : Append(std::make_shared<CcuRep::CcuRepStore>(insGenerator, var, addr));
1879 0 : }
1880 :
1881 0 : void CcuKernel::LoadVariable(const CcuRep::Variable& src, const CcuRep::Variable& var)
1882 : {
1883 0 : Append(std::make_shared<CcuRep::CcuRepLoadVar>(insGenerator, src, var));
1884 0 : }
1885 :
1886 15 : HcclResult CcuKernel::RecordEvent(CcuRep::CompletedEvent event, uint32_t mask)
1887 : {
1888 15 : if (CurrentBlock()->Type() == CcuRep::CcuRepType::LOOP_BLOCK) {
1889 1 : HCCL_ERROR("[CcuKernel][%s] is not supported in loop block, please check.", __func__);
1890 1 : LatchBodyError(HCCL_TO_CCU_RET(HcclResult::HCCL_E_NOT_SUPPORT));
1891 1 : return HcclResult::HCCL_E_NOT_SUPPORT;
1892 : }
1893 :
1894 14 : auto rep = std::make_shared<CcuRep::CcuRepLocRecordEvent>(insGenerator, event, mask);
1895 14 : Append(rep);
1896 14 : SetDependencyInfo(event.Id(), mask, rep);
1897 14 : return HCCL_SUCCESS;
1898 14 : }
1899 :
1900 41 : HcclResult CcuKernel::WaitEvent(CcuRep::CompletedEvent event, uint32_t mask)
1901 : {
1902 41 : bool isProfiling = CurrentBlock()->Type() != CcuRep::CcuRepType::LOOP_BLOCK;
1903 41 : auto rep = std::make_shared<CcuRep::CcuRepLocWaitEvent>(insGenerator, event, mask, isProfiling);
1904 41 : if (isProfiling) {
1905 75 : CHK_RET(static_cast<HcclResult>(AddProfiling("WaitEvent", rep->GetMask())));
1906 : }
1907 41 : rep->SetDependencyInfo(GetDependencyInfo(event.Id()));
1908 41 : EraseDependencyInfo(event.Id());
1909 41 : Append(rep);
1910 41 : return HCCL_SUCCESS;
1911 41 : }
1912 :
1913 96 : CcuResult CcuKernel::GetEventByHandle(CcuEventHandle eventHandle, CcuRep::CompletedEvent** event)
1914 : {
1915 96 : return GetResourceByHandle(ccuEventMap_, eventHandle, event, "completedEvent");
1916 : }
1917 :
1918 : /*
1919 : LocalAddr / RemoteAddr 相关接口
1920 : */
1921 33 : CcuResult CcuKernel::GetLocalAddrByHandle(CcuLocalAddrHandle handle, CcuRep::LocalAddr** localAddr)
1922 : {
1923 33 : return GetResourceByHandle(ccuLocalAddrMap_, handle, localAddr, "localAddr");
1924 : }
1925 :
1926 15 : CcuResult CcuKernel::GetRemoteAddrByHandle(CcuRemoteAddrHandle handle, CcuRep::RemoteAddr** remoteAddr)
1927 : {
1928 15 : return GetResourceByHandle(ccuRemoteAddrMap_, handle, remoteAddr, "remoteAddr");
1929 : }
1930 :
1931 : /*Read新接口*/
1932 2 : HcclResult CcuKernel::ReadNb(
1933 : const ChannelHandle channel, const CcuRep::CcuBuf& loc, const CcuRep::RemoteAddr& rem, const CcuRep::Variable& len,
1934 : CcuRep::CompletedEvent event, uint32_t mask)
1935 : {
1936 2 : channels_.insert(channel);
1937 2 : auto rep = std::make_shared<CcuRep::CcuRepBufRead>(insGenerator, channel, rem, loc, len, event, mask);
1938 2 : Append(rep);
1939 2 : SetDependencyInfo(event.Id(), mask, rep);
1940 2 : return HCCL_SUCCESS;
1941 2 : }
1942 :
1943 : /*Write新接口*/
1944 2 : HcclResult CcuKernel::WriteNb(
1945 : const ChannelHandle channel, const CcuRep::RemoteAddr& rem, const CcuRep::CcuBuf& loc, const CcuRep::Variable& len,
1946 : CcuRep::CompletedEvent event, uint32_t mask)
1947 : {
1948 2 : channels_.insert(channel);
1949 2 : auto rep = std::make_shared<CcuRep::CcuRepBufWrite>(insGenerator, channel, loc, rem, len, event, mask);
1950 2 : Append(rep);
1951 2 : SetDependencyInfo(event.Id(), mask, rep);
1952 2 : return HCCL_SUCCESS;
1953 2 : }
1954 :
1955 8 : static bool isLowPrecisionIn(Hccl::DataType dataType)
1956 : {
1957 16 : return dataType == Hccl::DataType::INT8 || dataType == Hccl::DataType::HIF8 || dataType == Hccl::DataType::FP8E4M3
1958 16 : || dataType == Hccl::DataType::FP8E5M2;
1959 : }
1960 :
1961 0 : static bool isLowPrecisionOut(Hccl::DataType dataType)
1962 : {
1963 0 : return dataType == Hccl::DataType::FP16 || dataType == Hccl::DataType::BFP16 || dataType == Hccl::DataType::FP32;
1964 : }
1965 :
1966 : constexpr uint32_t MAX_DATA_TYPE = 17;
1967 :
1968 : const Hccl::DataType orionDataTypes[]
1969 : = {Hccl::DataType::INT8, Hccl::DataType::INT16, Hccl::DataType::INT32,
1970 : Hccl::DataType::FP16, Hccl::DataType::FP32, Hccl::DataType::INT64,
1971 : Hccl::DataType::UINT64, Hccl::DataType::UINT8, Hccl::DataType::UINT16,
1972 : Hccl::DataType::UINT32, Hccl::DataType::FP64, Hccl::DataType::BFP16,
1973 : Hccl::DataType::INT128,
1974 : #if !defined(OPEN_BUILD_PROJECT) || defined(ORION_MODE)
1975 : Hccl::DataType::HIF8, Hccl::DataType::FP8E4M3, Hccl::DataType::FP8E5M2,
1976 : Hccl::DataType::FP8E8M0
1977 : #endif
1978 : };
1979 :
1980 12 : static Hccl::DataType HcommDataTypeToHcclDataType(const HcclDataType dataType)
1981 : {
1982 12 : const auto dataTypeNum = static_cast<uint32_t>(dataType);
1983 12 : if (dataTypeNum > MAX_DATA_TYPE) {
1984 0 : return Hccl::DataType::INVALID;
1985 : }
1986 :
1987 12 : return orionDataTypes[dataTypeNum];
1988 : }
1989 :
1990 : constexpr uint32_t MAX_REDUCE_TYPE = 4;
1991 : const Hccl::ReduceOp orionReduceOps[] = {
1992 : Hccl::ReduceOp::SUM,
1993 : Hccl::ReduceOp::PROD,
1994 : Hccl::ReduceOp::MAX,
1995 : Hccl::ReduceOp::MIN,
1996 : };
1997 :
1998 8 : static Hccl::ReduceOp HcommReduceOpToHcclReduceOp(const HcclReduceOp reduceOp)
1999 : {
2000 8 : const auto reduceOpNum = static_cast<uint32_t>(reduceOp);
2001 8 : if (reduceOpNum > MAX_REDUCE_TYPE) {
2002 0 : return Hccl::ReduceOp::INVALID;
2003 : }
2004 :
2005 8 : return orionReduceOps[reduceOpNum];
2006 : }
2007 :
2008 4 : HcclResult CcuKernel::LocalReduceNb(
2009 : const CcuRep::CcuBuf* bufs, uint32_t count, HcclDataType dataType, HcclDataType outputDataType, HcclReduceOp opType,
2010 : const CcuRep::Variable& len, CcuRep::CompletedEvent event, uint32_t mask)
2011 : {
2012 4 : auto opType_ = HcommReduceOpToHcclReduceOp(opType);
2013 4 : auto dataType_ = HcommDataTypeToHcclDataType(dataType);
2014 4 : auto outputDataType_ = HcommDataTypeToHcclDataType(outputDataType);
2015 :
2016 8 : if ((opType_ == Hccl::ReduceOp::SUM && isLowPrecisionIn(dataType_) && !isLowPrecisionOut(outputDataType_))
2017 4 : || (opType_ == Hccl::ReduceOp::SUM && !isLowPrecisionIn(dataType_) && dataType_ != outputDataType_)
2018 8 : || (opType_ != Hccl::ReduceOp::SUM && dataType_ != outputDataType_)) {
2019 0 : return HCCL_E_NOT_SUPPORT;
2020 : }
2021 :
2022 4 : std::vector<CcuRep::CcuBuf> ccuBufs(count);
2023 12 : for (uint32_t i = 0; i < count; i++) {
2024 8 : ccuBufs[i] = bufs[i];
2025 : }
2026 :
2027 : auto rep = std::make_shared<CcuRep::CcuRepBufReduce>(
2028 0 : insGenerator, ccuBufs, count, CcuRep::GetCcuDataType(dataType_, opType_),
2029 4 : CcuRep::GetCcuDataType(outputDataType_, opType_), CcuRep::GetCcuReduceType(opType_), event, len, mask);
2030 4 : Append(rep);
2031 4 : SetDependencyInfo(event.Id(), mask, rep);
2032 4 : return HCCL_SUCCESS;
2033 4 : }
2034 :
2035 : /*Read新接口*/
2036 4 : HcclResult CcuKernel::ReadNb(
2037 : const ChannelHandle channel, const CcuRep::LocalAddr& loc, const CcuRep::RemoteAddr& rem,
2038 : const CcuRep::Variable& len, CcuRep::CompletedEvent event, uint32_t mask)
2039 : {
2040 4 : channels_.insert(channel);
2041 4 : auto rep = std::make_shared<CcuRep::CcuRepRead>(insGenerator, channel, loc, rem, len, event, mask);
2042 4 : Append(rep);
2043 4 : SetDependencyInfo(event.Id(), mask, rep);
2044 4 : return HCCL_SUCCESS;
2045 4 : }
2046 :
2047 : /*ReadReduce新接口*/
2048 2 : HcclResult CcuKernel::ReadReduceNb(
2049 : const ChannelHandle channel, const CcuRep::LocalAddr& loc, const CcuRep::RemoteAddr& rem,
2050 : const CcuRep::Variable& len, HcclDataType dataType, HcclReduceOp opType, CcuRep::CompletedEvent event,
2051 : uint32_t mask)
2052 : {
2053 2 : channels_.insert(channel);
2054 2 : auto opType_ = HcommReduceOpToHcclReduceOp(opType);
2055 2 : auto dataType_ = HcommDataTypeToHcclDataType(dataType);
2056 :
2057 : auto rep = std::make_shared<CcuRep::CcuRepRead>(
2058 2 : insGenerator, channel, loc, rem, len, CcuRep::GetUBDataType(dataType_), CcuRep::GetUBReduceType(opType_), event,
2059 2 : mask);
2060 2 : Append(rep);
2061 2 : SetDependencyInfo(event.Id(), mask, rep);
2062 2 : return HCCL_SUCCESS;
2063 2 : }
2064 :
2065 : /*Write新接口*/
2066 3 : HcclResult CcuKernel::WriteNb(
2067 : const ChannelHandle channel, const CcuRep::RemoteAddr& rem, const CcuRep::LocalAddr& loc,
2068 : const CcuRep::Variable& len, CcuRep::CompletedEvent event, uint32_t mask)
2069 : {
2070 3 : channels_.insert(channel);
2071 3 : auto rep = std::make_shared<CcuRep::CcuRepWrite>(insGenerator, channel, rem, loc, len, event, mask);
2072 3 : Append(rep);
2073 3 : SetDependencyInfo(event.Id(), mask, rep);
2074 3 : return HCCL_SUCCESS;
2075 3 : }
2076 :
2077 2 : HcclResult CcuKernel::WriteReduceNb(
2078 : const ChannelHandle channel, const CcuRep::RemoteAddr& rem, const CcuRep::LocalAddr& loc,
2079 : const CcuRep::Variable& len, HcclDataType dataType, HcclReduceOp opType, CcuRep::CompletedEvent event,
2080 : uint32_t mask)
2081 : {
2082 2 : channels_.insert(channel);
2083 2 : auto opType_ = HcommReduceOpToHcclReduceOp(opType);
2084 2 : auto dataType_ = HcommDataTypeToHcclDataType(dataType);
2085 :
2086 : auto rep = std::make_shared<CcuRep::CcuRepWrite>(
2087 2 : insGenerator, channel, rem, loc, len, CcuRep::GetUBDataType(dataType_), CcuRep::GetUBReduceType(opType_), event,
2088 2 : mask);
2089 2 : Append(rep);
2090 2 : SetDependencyInfo(event.Id(), mask, rep);
2091 2 : return HCCL_SUCCESS;
2092 2 : }
2093 :
2094 32 : CcuResult CcuKernel::GetBufferByHandle(CcuBufferHandle bufferHandle, CcuRep::CcuBuf** buffer)
2095 : {
2096 32 : return GetResourceByHandle(ccuBufferMap_, bufferHandle, buffer, "buffer");
2097 : }
2098 :
2099 1 : HcclResult CcuKernel::LocalCopyNb(
2100 : const CcuRep::LocalAddr& dst, const CcuRep::LocalAddr& src, const CcuRep::Variable& len,
2101 : CcuRep::CompletedEvent event, uint32_t mask)
2102 : {
2103 1 : auto rep = std::make_shared<CcuRep::CcuRepLocCpy>(insGenerator, dst, src, len, event, mask);
2104 1 : Append(rep);
2105 1 : SetDependencyInfo(event.Id(), mask, rep);
2106 1 : return HCCL_SUCCESS;
2107 1 : }
2108 :
2109 12 : HcclResult CcuKernel::LocalCopyNb(
2110 : const CcuRep::CcuBuf& dst, const CcuRep::LocalAddr& src, const CcuRep::Variable& len, CcuRep::CompletedEvent event,
2111 : uint32_t mask)
2112 : {
2113 12 : auto rep = std::make_shared<CcuRep::CcuRepBufLocRead>(insGenerator, src, dst, len, event, mask);
2114 12 : Append(rep);
2115 12 : SetDependencyInfo(event.Id(), mask, rep);
2116 12 : return HCCL_SUCCESS;
2117 12 : }
2118 :
2119 8 : HcclResult CcuKernel::LocalCopyNb(
2120 : const CcuRep::LocalAddr& dst, const CcuRep::CcuBuf& src, const CcuRep::Variable& len, CcuRep::CompletedEvent event,
2121 : uint32_t mask)
2122 : {
2123 8 : auto rep = std::make_shared<CcuRep::CcuRepBufLocWrite>(insGenerator, src, dst, len, event, mask);
2124 8 : Append(rep);
2125 8 : SetDependencyInfo(event.Id(), mask, rep);
2126 8 : return HCCL_SUCCESS;
2127 8 : }
2128 :
2129 0 : HcclResult CcuKernel::LocalReduceNb(
2130 : const CcuRep::LocalAddr& dst, const CcuRep::LocalAddr& src, const CcuRep::Variable& len, HcclDataType dataType,
2131 : HcclReduceOp opType, CcuRep::CompletedEvent event, uint32_t mask)
2132 : {
2133 0 : auto opType_ = HcommReduceOpToHcclReduceOp(opType);
2134 0 : auto dataType_ = HcommDataTypeToHcclDataType(dataType);
2135 :
2136 : auto rep = std::make_shared<CcuRep::CcuRepLocCpy>(
2137 0 : insGenerator, dst, src, len, CcuRep::GetUBDataType(dataType_), CcuRep::GetUBReduceType(opType_), event, mask);
2138 0 : Append(rep);
2139 0 : SetDependencyInfo(event.Id(), mask, rep);
2140 0 : return HCCL_SUCCESS;
2141 0 : }
2142 :
2143 0 : CcuRep::FuncCall CcuKernel::Func(const std::string& label) { return CcuRep::FuncCall(this, label); }
2144 :
2145 0 : CcuRep::FuncCall CcuKernel::Func(const CcuRep::Variable& funcAddr) { return CcuRep::FuncCall(this, funcAddr); }
2146 :
2147 0 : CcuRep::LoopCall CcuKernel::Loop(const std::string& label) { return CcuRep::LoopCall(this, label); }
2148 :
2149 52 : CcuResult CcuKernel::LoopCreate(CcuLoop* loop)
2150 : {
2151 52 : PLF_CONFIG_INFO(PLF_DATA_OP, "[LoopCreate]");
2152 52 : if (loop == nullptr) {
2153 0 : HCCL_ERROR("[CcuKernel::LoopCreate] null pointer");
2154 0 : return CcuResult::CCU_E_PTR;
2155 : }
2156 52 : if (CurrentBlock()->Type() == CcuRep::CcuRepType::LOOP_BLOCK) {
2157 0 : HCCL_ERROR("[CcuKernel::LoopCreate] cannot create loop inside a loop body");
2158 0 : return LatchBodyError(CcuResult::CCU_E_INTERNAL);
2159 : }
2160 52 : if (inFuncBody_) {
2161 0 : HCCL_ERROR("[CcuKernel::LoopCreate] cannot create loop inside a func body");
2162 0 : return LatchBodyError(CcuResult::CCU_E_INTERNAL);
2163 : }
2164 :
2165 52 : CcuLoop handle = ++loopHandleCounter_;
2166 52 : std::string label = "loop_" + std::to_string(handle);
2167 :
2168 52 : LoopDescriptor desc;
2169 52 : desc.label = label;
2170 52 : desc.repLoopBlock = std::make_shared<CcuRep::CcuRepLoopBlock>(insGenerator, label);
2171 :
2172 52 : loopMap_[handle] = std::move(desc);
2173 52 : *loop = handle;
2174 52 : return CcuResult::CCU_SUCCESS;
2175 52 : }
2176 :
2177 7 : CcuResult CcuKernel::LatchBodyError(CcuResult err)
2178 : {
2179 15 : if ((CurrentBlock()->Type() == CcuRep::CcuRepType::LOOP_BLOCK || inFuncBody_)
2180 15 : && bodyError_ == CcuResult::CCU_SUCCESS) {
2181 7 : bodyError_ = err;
2182 : }
2183 7 : return err;
2184 : }
2185 :
2186 52 : CcuResult CcuKernel::LoopBodyEnter(CcuLoop loop)
2187 : {
2188 52 : PLF_CONFIG_INFO(PLF_DATA_OP, "[LoopBodyEnter] loop=%llu", loop);
2189 52 : auto it = loopMap_.find(loop);
2190 52 : if (it == loopMap_.end()) {
2191 0 : HCCL_ERROR("[CcuKernel::LoopBodyEnter] invalid loop handle %lu", loop);
2192 0 : return CcuResult::CCU_E_PARA;
2193 : }
2194 52 : auto& desc = it->second;
2195 52 : if (desc.bodyDefined) {
2196 0 : HCCL_ERROR("[CcuKernel::LoopBodyEnter] loop %lu body already defined", loop);
2197 0 : return CcuResult::CCU_E_INTERNAL;
2198 : }
2199 :
2200 52 : Append(desc.repLoopBlock);
2201 52 : desc.prevActiveBlock = CurrentBlock();
2202 52 : SetCurrentBlock(desc.repLoopBlock);
2203 52 : ++loopBodyDepth_;
2204 52 : bodyError_ = CcuResult::CCU_SUCCESS;
2205 :
2206 52 : return CcuResult::CCU_SUCCESS;
2207 : }
2208 :
2209 52 : CcuResult CcuKernel::LoopBodyExit(CcuLoop loop)
2210 : {
2211 52 : PLF_CONFIG_INFO(PLF_DATA_OP, "[LoopBodyExit] loop=%llu", loop);
2212 52 : auto it = loopMap_.find(loop);
2213 52 : if (it == loopMap_.end()) {
2214 0 : HCCL_ERROR("[CcuKernel::LoopBodyExit] invalid loop handle %lu", loop);
2215 0 : return CcuResult::CCU_E_PARA;
2216 : }
2217 52 : auto& desc = it->second;
2218 :
2219 52 : SetCurrentBlock(desc.prevActiveBlock);
2220 52 : desc.bodyDefined = true;
2221 52 : --loopBodyDepth_;
2222 :
2223 52 : if (bodyError_ != CcuResult::CCU_SUCCESS) {
2224 6 : const CcuResult err = bodyError_;
2225 6 : HCCL_ERROR("[CcuKernel::LoopBodyExit] illegal operation inside loop body, err=%d", err);
2226 6 : bodyError_ = CcuResult::CCU_SUCCESS;
2227 6 : return err;
2228 : }
2229 :
2230 46 : return CcuResult::CCU_SUCCESS;
2231 : }
2232 :
2233 8 : CcuResult CcuKernel::FuncBlockLookup(const void* funcPtr, uint64_t* outHandle)
2234 : {
2235 8 : PLF_CONFIG_INFO(PLF_DATA_OP, "[FuncBlockLookup] funcPtr=%p", funcPtr);
2236 8 : if (funcPtr == nullptr || outHandle == nullptr) {
2237 0 : HCCL_ERROR("[CcuKernel::FuncBlockLookup] null pointer");
2238 0 : return CcuResult::CCU_E_PTR;
2239 : }
2240 8 : if (CurrentBlock()->Type() == CcuRep::CcuRepType::LOOP_BLOCK || inFuncBody_) {
2241 2 : HCCL_ERROR("[CcuKernel::FuncBlockLookup] ccu::CallFunc only allowed at top level");
2242 2 : return LatchBodyError(CcuResult::CCU_E_INTERNAL);
2243 : }
2244 :
2245 6 : auto it = funcInstanceMap_.find(funcPtr);
2246 6 : *outHandle = (it == funcInstanceMap_.end()) ? 0 : it->second;
2247 6 : return CcuResult::CCU_SUCCESS;
2248 : }
2249 :
2250 5 : CcuResult CcuKernel::FuncBlockBegin(const void* funcPtr, uint64_t* outHandle)
2251 : {
2252 5 : PLF_CONFIG_INFO(PLF_DATA_OP, "[FuncBlockBegin] funcPtr=%p", funcPtr);
2253 5 : if (funcPtr == nullptr || outHandle == nullptr) {
2254 0 : HCCL_ERROR("[CcuKernel::FuncBlockBegin] null pointer");
2255 0 : return CcuResult::CCU_E_PTR;
2256 : }
2257 5 : if (CurrentBlock()->Type() == CcuRep::CcuRepType::LOOP_BLOCK || inFuncBody_) {
2258 0 : HCCL_ERROR("[CcuKernel::FuncBlockBegin] ccu::CallFunc only allowed at top level");
2259 0 : return LatchBodyError(CcuResult::CCU_E_INTERNAL);
2260 : }
2261 :
2262 5 : auto exist = funcInstanceMap_.find(funcPtr);
2263 5 : if (exist != funcInstanceMap_.end()) {
2264 0 : *outHandle = exist->second;
2265 0 : return CcuResult::CCU_SUCCESS;
2266 : }
2267 :
2268 5 : const uint64_t handle = ++funcHandleCounter_;
2269 5 : std::string label = "func_" + std::to_string(handle);
2270 :
2271 5 : FuncDescriptor desc;
2272 5 : desc.funcPtr = funcPtr;
2273 5 : desc.label = label;
2274 5 : desc.repFuncBlock = std::make_shared<CcuRep::CcuRepFuncBlock>(insGenerator, label);
2275 5 : desc.prevActiveBlock = CurrentBlock();
2276 :
2277 5 : funcMap_[handle] = desc;
2278 5 : SetCurrentBlock(desc.repFuncBlock);
2279 5 : inFuncBody_ = true;
2280 5 : bodyError_ = CcuResult::CCU_SUCCESS;
2281 5 : *outHandle = handle;
2282 5 : return CcuResult::CCU_SUCCESS;
2283 5 : }
2284 :
2285 4 : CcuResult CcuKernel::FuncBlockEnd(uint64_t handle)
2286 : {
2287 4 : PLF_CONFIG_INFO(PLF_DATA_OP, "[FuncBlockEnd] handle=%llu", handle);
2288 4 : auto it = funcMap_.find(handle);
2289 4 : if (it == funcMap_.end()) {
2290 0 : HCCL_ERROR("[CcuKernel::FuncBlockEnd] invalid func handle %lu", handle);
2291 0 : return CcuResult::CCU_E_PARA;
2292 : }
2293 4 : auto& desc = it->second;
2294 :
2295 4 : SetCurrentBlock(desc.prevActiveBlock);
2296 4 : inFuncBody_ = false;
2297 :
2298 4 : if (bodyError_ != CcuResult::CCU_SUCCESS) {
2299 0 : const CcuResult err = bodyError_;
2300 0 : HCCL_ERROR("[CcuKernel::FuncBlockEnd] illegal operation inside func body, err=%d", err);
2301 0 : bodyError_ = CcuResult::CCU_SUCCESS;
2302 0 : funcMap_.erase(it);
2303 0 : return err;
2304 : }
2305 :
2306 4 : Append(desc.repFuncBlock);
2307 4 : desc.bodyDefined = true;
2308 4 : funcInstanceMap_[desc.funcPtr] = handle;
2309 4 : return CcuResult::CCU_SUCCESS;
2310 : }
2311 :
2312 7 : CcuResult CcuKernel::FuncDefineInArg(uint64_t handle, CcuVariableHandle formal)
2313 : {
2314 7 : PLF_CONFIG_INFO(PLF_DATA_OP, "[FuncDefineInArg] handle=%llu, formal=%llu", handle, formal);
2315 7 : auto it = funcMap_.find(handle);
2316 7 : if (it == funcMap_.end()) {
2317 0 : HCCL_ERROR("[CcuKernel::FuncDefineInArg] invalid func handle %lu", handle);
2318 0 : return CcuResult::CCU_E_PARA;
2319 : }
2320 :
2321 7 : CcuRep::Variable* formalVar = nullptr;
2322 7 : CCU_CHK_RET(GetVariableByHandle(formal, &formalVar));
2323 7 : it->second.repFuncBlock->DefineInArg(*formalVar);
2324 7 : return CcuResult::CCU_SUCCESS;
2325 : }
2326 :
2327 5 : CcuResult CcuKernel::FuncCall(uint64_t handle, const CcuVariableHandle* inArgs, uint32_t numIn)
2328 : {
2329 5 : PLF_CONFIG_INFO(PLF_DATA_OP, "[FuncCall] handle=%llu, numIn=%u", handle, numIn);
2330 5 : if (CurrentBlock()->Type() == CcuRep::CcuRepType::LOOP_BLOCK || inFuncBody_) {
2331 0 : HCCL_ERROR("[CcuKernel::FuncCall] ccu::CallFunc only allowed at top level");
2332 0 : return LatchBodyError(CcuResult::CCU_E_INTERNAL);
2333 : }
2334 5 : if (numIn > 0 && inArgs == nullptr) {
2335 0 : HCCL_ERROR("[CcuKernel::FuncCall] null input args");
2336 0 : return CcuResult::CCU_E_PTR;
2337 : }
2338 :
2339 5 : auto it = funcMap_.find(handle);
2340 5 : if (it == funcMap_.end() || !it->second.bodyDefined) {
2341 0 : HCCL_ERROR("[CcuKernel::FuncCall] invalid func handle %lu", handle);
2342 0 : return CcuResult::CCU_E_PARA;
2343 : }
2344 :
2345 5 : auto repFuncCall = std::make_shared<CcuRep::CcuRepFuncCall>(insGenerator, it->second.label);
2346 12 : for (uint32_t i = 0; i < numIn; i++) {
2347 7 : CcuRep::Variable* actual = nullptr;
2348 7 : CCU_CHK_RET(GetVariableByHandle(inArgs[i], &actual));
2349 7 : repFuncCall->SetInArg(*actual);
2350 : }
2351 5 : Append(repFuncCall);
2352 5 : return CcuResult::CCU_SUCCESS;
2353 5 : }
2354 :
2355 : // 按 maxLoopNum 把 res_.blockExecutor[0] 扩容到至少 maxLoopNum 个 LoopEngine。
2356 : // 与 CreateBlockResAssist 对齐：所有 LoopEngine 资源先落在 die0 池，待实际 die 确定后
2357 : // 再由 MoveResourcesToDie 迁移到目标 die。
2358 : // 不同 LoopGroup 通过 local loopIdx 复用同一池低位 executorId，所以这里只
2359 : // "补足"而不是"累加"。
2360 28 : CcuResult CcuKernel::EnsureLoopEnginePool(uint32_t maxLoopNum)
2361 : {
2362 28 : if (maxLoopNum == 0) {
2363 0 : HCCL_ERROR("[CcuKernel::EnsureLoopEnginePool] maxLoopNum must be > 0");
2364 0 : return CcuResult::CCU_E_PARA;
2365 : }
2366 28 : constexpr uint32_t poolDieId = 0;
2367 28 : auto& loopEnginePool = res_.blockExecutor[poolDieId];
2368 28 : if (maxLoopNum <= loopEnginePool.size()) {
2369 10 : return CcuResult::CCU_SUCCESS;
2370 : }
2371 18 : const uint32_t deficit = maxLoopNum - static_cast<uint32_t>(loopEnginePool.size());
2372 18 : std::vector<CcuRep::Executor> tmp(deficit, CcuRep::Executor(this));
2373 18 : (void)CreateBlockExecutor(deficit, tmp.data());
2374 18 : return CcuResult::CCU_SUCCESS;
2375 18 : }
2376 :
2377 13 : CcuResult CcuKernel::LoopGroupCreate(CcuLoopGroup* group, uint32_t maxLoopNum, const CcuLoopGroupCfg* cfg)
2378 : {
2379 13 : PLF_CONFIG_INFO(PLF_DATA_OP, "[LoopGroupCreate] maxLoopNum=%u", maxLoopNum);
2380 13 : if (group == nullptr || cfg == nullptr) {
2381 0 : HCCL_ERROR("[CcuKernel::LoopGroupCreate] null pointer");
2382 0 : return CcuResult::CCU_E_PTR;
2383 : }
2384 13 : if (CurrentBlock()->Type() == CcuRep::CcuRepType::LOOP_BLOCK) {
2385 0 : HCCL_ERROR("[CcuKernel::LoopGroupCreate] cannot create loop group inside a loop body");
2386 0 : return LatchBodyError(CcuResult::CCU_E_INTERNAL);
2387 : }
2388 13 : if (inFuncBody_) {
2389 0 : HCCL_ERROR("[CcuKernel::LoopGroupCreate] cannot create loop group inside a func body");
2390 0 : return LatchBodyError(CcuResult::CCU_E_INTERNAL);
2391 : }
2392 :
2393 : // 按需扩 LoopEngine 池；池足够则复用低位 executorId，跨组共享。
2394 13 : CCU_CHK_RET(EnsureLoopEnginePool(maxLoopNum));
2395 :
2396 13 : CcuLoopGroup handle = ++loopGroupHandleCounter_;
2397 :
2398 13 : LoopGroupDescriptor desc;
2399 13 : desc.config = *cfg;
2400 13 : desc.parallelVar = CreateVariable();
2401 13 : desc.offsetVar = CreateVariable();
2402 13 : desc.isVarBased = false;
2403 :
2404 13 : auto bundle = std::make_shared<CcuRep::CcuRepLoopGroupBundle>(insGenerator, *cfg, desc.parallelVar, desc.offsetVar);
2405 13 : if (ccuVersion_ == CcuVersion::CCU_V2) {
2406 4 : bundle->SetXnOffsetVar(CreateVariable());
2407 : }
2408 13 : desc.bundleRep = bundle;
2409 13 : Append(bundle);
2410 :
2411 13 : loopGroupMap_[handle] = std::move(desc);
2412 13 : *group = handle;
2413 13 : return CcuResult::CCU_SUCCESS;
2414 13 : }
2415 :
2416 11 : CcuResult CcuKernel::LoopGroupCreateFromVar(
2417 : CcuLoopGroup* group, uint32_t maxLoopNum, CcuVariableHandle parallelVarHandle, CcuVariableHandle offsetVarHandle)
2418 : {
2419 11 : PLF_CONFIG_INFO(
2420 : PLF_DATA_OP, "[LoopGroupCreateFromVar] maxLoopNum=%u, parallelVarHandle=%llu, offsetVarHandle=%llu", maxLoopNum,
2421 : parallelVarHandle, offsetVarHandle);
2422 11 : if (group == nullptr) {
2423 0 : HCCL_ERROR("[CcuKernel::LoopGroupCreateFromVar] null pointer for group");
2424 0 : return CcuResult::CCU_E_PTR;
2425 : }
2426 11 : if (CurrentBlock()->Type() == CcuRep::CcuRepType::LOOP_BLOCK) {
2427 0 : HCCL_ERROR("[CcuKernel::LoopGroupCreateFromVar] cannot create loop group inside a loop body");
2428 0 : return LatchBodyError(CcuResult::CCU_E_INTERNAL);
2429 : }
2430 11 : if (inFuncBody_) {
2431 0 : HCCL_ERROR("[CcuKernel::LoopGroupCreateFromVar] cannot create loop group inside a func body");
2432 0 : return LatchBodyError(CcuResult::CCU_E_INTERNAL);
2433 : }
2434 :
2435 11 : CCU_CHK_RET(EnsureLoopEnginePool(maxLoopNum));
2436 :
2437 11 : CcuRep::Variable* parallelVarPtr = nullptr;
2438 11 : CcuRep::Variable* offsetVarPtr = nullptr;
2439 11 : CCU_CHK_RET(GetVariableByHandle(parallelVarHandle, ¶llelVarPtr));
2440 11 : CCU_CHK_RET(GetVariableByHandle(offsetVarHandle, &offsetVarPtr));
2441 :
2442 11 : CcuLoopGroup handle = ++loopGroupHandleCounter_;
2443 :
2444 11 : LoopGroupDescriptor desc;
2445 11 : desc.parallelVar = CcuRep::Variable(*parallelVarPtr);
2446 11 : desc.offsetVar = CcuRep::Variable(*offsetVarPtr);
2447 11 : desc.isVarBased = true;
2448 :
2449 11 : auto bundle = std::make_shared<CcuRep::CcuRepLoopGroupBundle>(insGenerator, desc.parallelVar, desc.offsetVar);
2450 11 : if (ccuVersion_ == CcuVersion::CCU_V2) {
2451 1 : bundle->SetCompatRemapVars(CreateVariable(), CreateVariable());
2452 : }
2453 11 : desc.bundleRep = bundle;
2454 11 : Append(bundle);
2455 :
2456 11 : loopGroupMap_[handle] = std::move(desc);
2457 11 : *group = handle;
2458 11 : return CcuResult::CCU_SUCCESS;
2459 11 : }
2460 :
2461 5 : CcuResult CcuKernel::LoopGroupCreateFromVarV2(
2462 : CcuLoopGroup* group, uint32_t maxLoopNum, CcuVariableHandle parallelVarV2Handle,
2463 : CcuVariableHandle offsetVarV2Handle, CcuVariableHandle varOffsetVarHandle)
2464 : {
2465 5 : if (ccuVersion_ != CcuVersion::CCU_V2) {
2466 1 : HCCL_ERROR("[CcuKernel::LoopGroupCreateFromVarV2] only supported on V2");
2467 1 : return CcuResult::CCU_E_NOT_SUPPORT;
2468 : }
2469 4 : if (group == nullptr) {
2470 0 : HCCL_ERROR("[CcuKernel::LoopGroupCreateFromVarV2] null pointer for group");
2471 0 : return CcuResult::CCU_E_PTR;
2472 : }
2473 4 : if (CurrentBlock()->Type() == CcuRep::CcuRepType::LOOP_BLOCK) {
2474 0 : HCCL_ERROR("[CcuKernel::LoopGroupCreateFromVarV2] cannot create loop group inside a loop body");
2475 0 : return LatchBodyError(CcuResult::CCU_E_INTERNAL);
2476 : }
2477 4 : if (inFuncBody_) {
2478 0 : HCCL_ERROR("[CcuKernel::LoopGroupCreateFromVarV2] cannot create loop group inside a func body");
2479 0 : return LatchBodyError(CcuResult::CCU_E_INTERNAL);
2480 : }
2481 :
2482 4 : CCU_CHK_RET(EnsureLoopEnginePool(maxLoopNum));
2483 :
2484 4 : CcuRep::Variable* parallelVarPtr = nullptr;
2485 4 : CcuRep::Variable* offsetVarPtr = nullptr;
2486 4 : CcuRep::Variable* xnOffsetVarPtr = nullptr;
2487 4 : CCU_CHK_RET(GetVariableByHandle(parallelVarV2Handle, ¶llelVarPtr));
2488 4 : CCU_CHK_RET(GetVariableByHandle(offsetVarV2Handle, &offsetVarPtr));
2489 4 : CCU_CHK_RET(GetVariableByHandle(varOffsetVarHandle, &xnOffsetVarPtr));
2490 :
2491 4 : CcuLoopGroup handle = ++loopGroupHandleCounter_;
2492 :
2493 4 : LoopGroupDescriptor desc;
2494 4 : desc.parallelVar = CcuRep::Variable(*parallelVarPtr);
2495 4 : desc.offsetVar = CcuRep::Variable(*offsetVarPtr);
2496 4 : desc.xnOffsetVar = CcuRep::Variable(*xnOffsetVarPtr);
2497 4 : desc.isVarBased = true;
2498 4 : desc.isVersionV2 = true;
2499 :
2500 4 : auto bundle = std::make_shared<CcuRep::CcuRepLoopGroupBundle>(insGenerator, desc.parallelVar, desc.offsetVar);
2501 4 : bundle->SetLayout(CcuRep::CcuRepLoopGroupBundle::Layout::VersionV2);
2502 4 : bundle->SetXnOffsetVar(desc.xnOffsetVar);
2503 4 : desc.bundleRep = bundle;
2504 4 : Append(bundle);
2505 :
2506 4 : loopGroupMap_[handle] = std::move(desc);
2507 4 : *group = handle;
2508 4 : return CcuResult::CCU_SUCCESS;
2509 4 : }
2510 :
2511 56 : CcuResult CcuKernel::LookupLoopGroupAndLoop(
2512 : CcuLoopGroup group, CcuLoop loop, const char* fnName, const char* createFnName, LoopGroupDescriptor*& grpDesc,
2513 : LoopDescriptor*& loopDesc, uint32_t& loopIdx)
2514 : {
2515 56 : auto grpIt = loopGroupMap_.find(group);
2516 56 : if (grpIt == loopGroupMap_.end()) {
2517 0 : HCCL_ERROR("[CcuKernel::%s] invalid group handle %lu", fnName, group);
2518 0 : return CcuResult::CCU_E_PARA;
2519 : }
2520 56 : grpDesc = &grpIt->second;
2521 :
2522 56 : auto loopIt = loopMap_.find(loop);
2523 56 : if (loopIt == loopMap_.end()) {
2524 0 : HCCL_ERROR("[CcuKernel::%s] invalid loop handle %lu", fnName, loop);
2525 0 : return CcuResult::CCU_E_PARA;
2526 : }
2527 56 : loopDesc = &loopIt->second;
2528 :
2529 56 : if (!loopDesc->bodyDefined) {
2530 0 : HCCL_ERROR("[CcuKernel::%s] loop %lu body not defined", fnName, loop);
2531 0 : return CcuResult::CCU_E_INTERNAL; // CCU_E_LOOP_BODY_UNDEFINED
2532 : }
2533 :
2534 56 : auto& loopEnginePool = res_.blockExecutor[0];
2535 56 : loopIdx = grpDesc->loopCount;
2536 56 : if (loopIdx >= loopEnginePool.size()) {
2537 0 : HCCL_ERROR(
2538 : "[CcuKernel::%s] loopEngine pool exhausted (pool size %zu, loopIdx %u). "
2539 : "Pass a larger maxLoopNum to %s so the pool can be extended at create time.",
2540 : fnName, loopEnginePool.size(), loopIdx, createFnName);
2541 0 : return CcuResult::CCU_E_PARA;
2542 : }
2543 56 : return CcuResult::CCU_SUCCESS;
2544 : }
2545 :
2546 25 : CcuResult CcuKernel::LoopGroupAddLoop(CcuLoopGroup group, CcuLoop loop, const CcuLoopCfg* cfg)
2547 : {
2548 25 : PLF_CONFIG_INFO(PLF_DATA_OP, "[LoopGroupAddLoop] group=%llu, loop=%llu", group, loop);
2549 25 : if (cfg == nullptr) {
2550 0 : HCCL_ERROR("[CcuKernel::LoopGroupAddLoop] null pointer for cfg");
2551 0 : return CcuResult::CCU_E_PTR;
2552 : }
2553 25 : LoopGroupDescriptor* grpDesc = nullptr;
2554 25 : LoopDescriptor* loopDesc = nullptr;
2555 25 : uint32_t loopIdx = 0;
2556 25 : CCU_CHK_RET(
2557 : LookupLoopGroupAndLoop(group, loop, "LoopGroupAddLoop", "CcuLoopGroupCreate", grpDesc, loopDesc, loopIdx));
2558 25 : auto& loopEnginePool = res_.blockExecutor[0];
2559 :
2560 25 : grpDesc->loopCount++;
2561 25 : grpDesc->totalLoopNum = grpDesc->loopCount;
2562 :
2563 25 : CcuRep::CcuRepLoopGroupBundle::LoopEntry entry;
2564 25 : entry.config = *cfg;
2565 25 : entry.executor = loopEnginePool[loopIdx];
2566 25 : entry.repLoopBlock = loopDesc->repLoopBlock;
2567 25 : entry.loopParamVar = CreateVariable();
2568 25 : entry.layout = CcuRep::CcuRepLoopGroupBundle::Layout::Config;
2569 25 : if (ccuVersion_ == CcuVersion::CCU_V2) {
2570 7 : entry.iterNumVar = CreateVariable();
2571 7 : entry.addrOffsetVar = CreateVariable();
2572 7 : entry.ctxIdVar = CreateVariable();
2573 : }
2574 :
2575 25 : auto bundle = std::static_pointer_cast<CcuRep::CcuRepLoopGroupBundle>(grpDesc->bundleRep);
2576 25 : bundle->AddLoop(entry);
2577 :
2578 25 : if (!grpDesc->isVarBased) {
2579 19 : bundle->SetRepeatLoopIdx(grpDesc->config.cloneLoopOffset);
2580 19 : bundle->SetTotalLoopNum(grpDesc->totalLoopNum);
2581 : }
2582 :
2583 25 : return CcuResult::CCU_SUCCESS;
2584 25 : }
2585 :
2586 23 : CcuResult CcuKernel::LoopGroupAddLoopFromVar(CcuLoopGroup group, CcuLoop loop, CcuVariableHandle loopParamVarHandle)
2587 : {
2588 23 : PLF_CONFIG_INFO(
2589 : PLF_DATA_OP, "[LoopGroupAddLoopFromVar] group=%llu, loop=%llu, loopParamVarHandle=%llu", group, loop,
2590 : loopParamVarHandle);
2591 23 : LoopGroupDescriptor* grpDesc = nullptr;
2592 23 : LoopDescriptor* loopDesc = nullptr;
2593 23 : uint32_t loopIdx = 0;
2594 23 : CCU_CHK_RET(LookupLoopGroupAndLoop(
2595 : group, loop, "LoopGroupAddLoopFromVar", "CcuLoopGroupCreateFromVar", grpDesc, loopDesc, loopIdx));
2596 23 : auto& loopEnginePool = res_.blockExecutor[0];
2597 :
2598 23 : CcuRep::Variable* loopParamVarPtr = nullptr;
2599 23 : CCU_CHK_RET(GetVariableByHandle(loopParamVarHandle, &loopParamVarPtr));
2600 :
2601 23 : CcuRep::CcuRepLoopGroupBundle::LoopEntry entry;
2602 23 : entry.executor = loopEnginePool[loopIdx];
2603 23 : entry.repLoopBlock = loopDesc->repLoopBlock;
2604 23 : entry.loopParamVar = CcuRep::Variable(*loopParamVarPtr);
2605 23 : entry.layout = CcuRep::CcuRepLoopGroupBundle::Layout::PackedVar;
2606 23 : if (ccuVersion_ == CcuVersion::CCU_V2) {
2607 4 : entry.iterNumVar = CreateVariable();
2608 4 : entry.addrOffsetVar = CreateVariable();
2609 4 : entry.ctxIdVar = CreateVariable();
2610 : }
2611 :
2612 23 : auto bundle = std::static_pointer_cast<CcuRep::CcuRepLoopGroupBundle>(grpDesc->bundleRep);
2613 23 : bundle->AddLoop(entry);
2614 :
2615 : // 计数放在入 bundle 之后统一更新：totalLoopNum 仅供 config 组同步编码使用，与 AddLoop 无先后依赖。
2616 23 : grpDesc->loopCount++;
2617 23 : grpDesc->totalLoopNum = grpDesc->loopCount;
2618 23 : if (!grpDesc->isVarBased) {
2619 7 : bundle->SetRepeatLoopIdx(grpDesc->config.cloneLoopOffset);
2620 7 : bundle->SetTotalLoopNum(grpDesc->totalLoopNum);
2621 : }
2622 :
2623 23 : return CcuResult::CCU_SUCCESS;
2624 23 : }
2625 :
2626 8 : CcuResult CcuKernel::LoopGroupAddLoopFromVarV2(
2627 : CcuLoopGroup group, CcuLoop loop, CcuVariableHandle iterNumVarHandle, CcuVariableHandle addrOffsetVarHandle,
2628 : CcuVariableHandle ctxIdVarHandle)
2629 : {
2630 8 : if (ccuVersion_ != CcuVersion::CCU_V2) {
2631 0 : HCCL_ERROR("[CcuKernel::LoopGroupAddLoopFromVarV2] only supported on V2");
2632 0 : return CcuResult::CCU_E_NOT_SUPPORT;
2633 : }
2634 :
2635 8 : LoopGroupDescriptor* grpDesc = nullptr;
2636 8 : LoopDescriptor* loopDesc = nullptr;
2637 8 : uint32_t loopIdx = 0;
2638 8 : CCU_CHK_RET(LookupLoopGroupAndLoop(
2639 : group, loop, "LoopGroupAddLoopFromVarV2", "CcuLoopGroupCreateFromVarV2", grpDesc, loopDesc, loopIdx));
2640 8 : auto& loopEnginePool = res_.blockExecutor[0];
2641 :
2642 8 : CcuRep::Variable* iterNumVarPtr = nullptr;
2643 8 : CcuRep::Variable* addrOffsetVarPtr = nullptr;
2644 8 : CcuRep::Variable* ctxIdVarPtr = nullptr;
2645 8 : CCU_CHK_RET(GetVariableByHandle(iterNumVarHandle, &iterNumVarPtr));
2646 8 : CCU_CHK_RET(GetVariableByHandle(addrOffsetVarHandle, &addrOffsetVarPtr));
2647 8 : CCU_CHK_RET(GetVariableByHandle(ctxIdVarHandle, &ctxIdVarPtr));
2648 :
2649 8 : grpDesc->loopCount++;
2650 8 : grpDesc->totalLoopNum = grpDesc->loopCount;
2651 :
2652 8 : VersionV2LoopRecord record;
2653 8 : record.iterNumVar = CcuRep::Variable(*iterNumVarPtr);
2654 8 : record.addrOffsetVar = CcuRep::Variable(*addrOffsetVarPtr);
2655 8 : record.ctxIdVar = CcuRep::Variable(*ctxIdVarPtr);
2656 8 : grpDesc->versionV2Loops.push_back(record);
2657 :
2658 8 : CcuRep::CcuRepLoopGroupBundle::LoopEntry entry;
2659 8 : entry.executor = loopEnginePool[loopIdx];
2660 8 : entry.repLoopBlock = loopDesc->repLoopBlock;
2661 8 : entry.iterNumVar = CcuRep::Variable(*iterNumVarPtr);
2662 8 : entry.addrOffsetVar = CcuRep::Variable(*addrOffsetVarPtr);
2663 8 : entry.ctxIdVar = CcuRep::Variable(*ctxIdVarPtr);
2664 8 : entry.layout = CcuRep::CcuRepLoopGroupBundle::Layout::VersionV2;
2665 :
2666 8 : auto bundle = std::static_pointer_cast<CcuRep::CcuRepLoopGroupBundle>(grpDesc->bundleRep);
2667 8 : bundle->AddLoop(entry);
2668 :
2669 8 : if (!grpDesc->isVarBased) {
2670 2 : bundle->SetRepeatLoopIdx(grpDesc->config.cloneLoopOffset);
2671 2 : bundle->SetTotalLoopNum(grpDesc->totalLoopNum);
2672 : }
2673 :
2674 8 : return CcuResult::CCU_SUCCESS;
2675 8 : }
2676 :
2677 44 : void CcuKernel::SetInstrId(uint32_t instrId) { instrInfo_.startInstrId = instrId; }
2678 :
2679 88 : uint32_t CcuKernel::GetInstrId() const { return instrInfo_.startInstrId; }
2680 :
2681 91 : uint32_t CcuKernel::GetInstrCount()
2682 : {
2683 91 : uint32_t instrCount = 0;
2684 2285 : for (const auto& rep : GetRepSequence()) {
2685 2194 : instrCount += rep->InstrCount();
2686 : }
2687 91 : instrInfo_.instrCount = instrCount;
2688 91 : HCCL_INFO("Kernel inst %u", instrCount);
2689 91 : return instrCount;
2690 : }
2691 :
2692 43 : void CcuKernel::SetCcuInstrInfo(const CcuRep::CcuInstrInfo& instrInfo) { this->instrInfo_ = instrInfo; }
2693 :
2694 358 : CcuRep::Variable CcuKernel::CreateVariable()
2695 : {
2696 358 : const auto& var = CreateBlockResAssist(1, res_.continuousVariable);
2697 716 : return var[0];
2698 358 : }
2699 :
2700 109 : CcuRep::Variable CcuKernel::CreateExpectVar()
2701 : {
2702 : // v2(A6) 的 jump 只支持 var-var 比较，需要真实 XN 承载立即数；
2703 : // v1(A5) 的 jump 直接支持 var-imm，expectVar 不参与翻译，
2704 : // 用不进 res_ 账本的壳 Variable 即可，避免 v1 资源虚高。
2705 109 : if (ccuVersion_ == CcuVersion::CCU_V2) {
2706 20 : return CreateVariable();
2707 : }
2708 89 : return CcuRep::Variable(this);
2709 : }
2710 :
2711 126 : CcuRep::Variable CcuKernel::CreateJumpTargetVar()
2712 : {
2713 126 : if (ccuVersion_ == CcuVersion::CCU_V1) {
2714 : // A5(CCU_V1): 没有指令重排，可复用同一个 Xn 存放跳转目标地址。targetVar 仅用于 LoadImdToXn + 紧接 Jump。
2715 102 : if (!sharedJumpTargetVar_) {
2716 14 : sharedJumpTargetVar_ = std::make_unique<CcuRep::Variable>(CreateVariable());
2717 : }
2718 102 : return *sharedJumpTargetVar_;
2719 : }
2720 24 : return CcuRep::CreateVariable(this);
2721 : }
2722 :
2723 203 : CcuRep::Address CcuKernel::CreateAddress()
2724 : {
2725 203 : if (ccuVersion_ == CcuVersion::CCU_V2) {
2726 : // A6创建Address时，需要添加到Variable的列表中，但是仍以Address返回
2727 91 : return CcuRep::Address(CreateResAssist(res_.continuousVariable));
2728 : }
2729 112 : return CreateResAssist(res_.blockAddress);
2730 : }
2731 :
2732 0 : CcuRep::LocalNotify CcuKernel::CreateLocalNotify() { return CreateResAssist(res_.localNotify); }
2733 :
2734 0 : CcuRep::CompletedEvent CcuKernel::CreateCompletedEvent() { return CreateResAssist(res_.blockCompletedEvent); }
2735 :
2736 0 : CcuRep::CcuBuf CcuKernel::CreateCcuBuf()
2737 : {
2738 0 : const auto& buffer = CreateBlockResAssist(1, res_.blockCcubufs);
2739 0 : return buffer[0];
2740 0 : }
2741 :
2742 0 : CcuRep::Executor CcuKernel::CreateExecutor() { return CreateResAssist(res_.blockExecutor); }
2743 :
2744 156 : CcuRep::LocalAddr CcuKernel::CreateLocalAddr() { return CcuRep::LocalAddr(CreateAddress(), CreateVariable()); }
2745 :
2746 37 : CcuRep::RemoteAddr CcuKernel::CreateRemoteAddr() { return CcuRep::RemoteAddr(CreateAddress(), CreateVariable()); }
2747 :
2748 0 : CcuRep::RemoteAddr CcuKernel::GetRemoteAddr(const ChannelHandle channel, uint32_t index)
2749 : {
2750 : (void)index;
2751 0 : channels_.insert(channel);
2752 0 : auto mem = CcuRep::RemoteAddr(CreateAddress(), CreateVariable());
2753 0 : Append(std::make_shared<CcuRep::CcuRepRemMem>(insGenerator, channel, mem));
2754 0 : return mem;
2755 0 : }
2756 :
2757 0 : CcuRep::LocalAddr CcuKernel::CreateLocalAddr(const CcuRep::Variable& token)
2758 : {
2759 0 : return CcuRep::LocalAddr(CreateAddress(), token);
2760 : }
2761 :
2762 0 : HcclResult CcuKernel::CreateBlockCcuBuf(const uint32_t count, CcuRep::CcuBuf* ccuBufs)
2763 : {
2764 0 : CHK_PTR_NULL(ccuBufs);
2765 0 : auto resources = CreateBlockResAssist(count, res_.blockCcubufs);
2766 :
2767 0 : for (uint32_t i = 0; i < count; i++) {
2768 0 : ccuBufs[i] = resources[i]; // 拷贝虚拟资源，通过shared_ptr链接到物理资源
2769 : }
2770 :
2771 0 : return HcclResult::HCCL_SUCCESS;
2772 0 : }
2773 :
2774 18 : HcclResult CcuKernel::CreateBlockExecutor(const uint32_t count, CcuRep::Executor* ccuExes)
2775 : {
2776 18 : CHK_PTR_NULL(ccuExes);
2777 18 : auto resources = CreateBlockResAssist(count, res_.blockExecutor);
2778 :
2779 48 : for (uint32_t i = 0; i < count; i++) {
2780 30 : ccuExes[i] = resources[i]; // 拷贝虚拟资源，通过shared_ptr链接到物理资源
2781 : }
2782 :
2783 18 : return HcclResult::HCCL_SUCCESS;
2784 18 : }
2785 :
2786 0 : HcclResult CcuKernel::CreateBlockCompletedEvent(const uint32_t count, CcuRep::CompletedEvent* ccuEvents)
2787 : {
2788 0 : CHK_PTR_NULL(ccuEvents);
2789 0 : auto resources = CreateBlockResAssist(count, res_.blockCompletedEvent);
2790 :
2791 0 : for (uint32_t i = 0; i < count; i++) {
2792 0 : ccuEvents[i] = resources[i]; // 拷贝虚拟资源，通过shared_ptr链接到物理资源
2793 : }
2794 :
2795 0 : return HcclResult::HCCL_SUCCESS;
2796 0 : }
2797 :
2798 44 : void CcuKernel::SetResRepository(const CcuResRepository& resRepo) { resRepo_ = resRepo; }
2799 :
2800 44 : CcuResRepository& CcuKernel::GetResRepository() { return resRepo_; }
2801 :
2802 44 : CcuSharedResource& CcuKernel::GetExportedRes() { return exportedRes_; }
2803 :
2804 44 : CcuSharedResource& CcuKernel::GetImportedRes() { return importedRes_; }
2805 :
2806 0 : static HcclResult GetArgIndex(
2807 : const std::unordered_map<uint16_t, uint16_t>& varId2VarIdMap,
2808 : const std::unordered_map<uint16_t, uint32_t>& varId2ArgIndexMap, const uint64_t* taskArgs, uint32_t argSize,
2809 : uint16_t varId, uint64_t& argIndex)
2810 : {
2811 0 : HCCL_INFO("[GetArgIndex] Enter varId(%u)", varId);
2812 0 : auto item = varId2ArgIndexMap.find(varId);
2813 0 : if (item == varId2ArgIndexMap.end()) {
2814 0 : uint16_t oriVarId = varId;
2815 0 : auto iter = varId2VarIdMap.find(varId);
2816 0 : while (iter != varId2VarIdMap.end()) { // 循环查找中间assign Rep，找到起始varId
2817 0 : oriVarId = iter->second;
2818 0 : iter = varId2VarIdMap.find(oriVarId);
2819 : }
2820 0 : if (oriVarId != varId) { // 起始varId预期通过LoadArg赋值
2821 0 : item = varId2ArgIndexMap.find(oriVarId);
2822 0 : if (item == varId2ArgIndexMap.end()) {
2823 0 : HCCL_ERROR("[%s]fail, Invalid goSize variable id(%u), oriVarId = %u", __func__, varId, oriVarId);
2824 0 : return HCCL_E_PARA;
2825 : }
2826 : } else {
2827 0 : HCCL_ERROR("[%s]fail, Invalid goSize variable id(%u)", __func__, varId);
2828 0 : return HCCL_E_PARA;
2829 : }
2830 : }
2831 0 : HCCL_INFO("[GetArgIndex] find end");
2832 0 : if (item->second >= argSize) {
2833 0 : HCCL_ERROR("Invalid goSize variable index(%u).", item->second);
2834 0 : return HCCL_E_PARA;
2835 : }
2836 0 : HCCL_INFO(
2837 : "GetArgIndex success: varId(%u) varId2VarIdMapSize(%u) varId2ArgIndexMapSize(%u) taskArgsSize(%u)", varId,
2838 : varId2VarIdMap.size(), varId2ArgIndexMap.size(), argSize);
2839 0 : argIndex = taskArgs[item->second];
2840 0 : return HCCL_SUCCESS;
2841 : }
2842 :
2843 0 : void DumpCcuProfilingInfo(const std::vector<CcuProfilingInfo>& ccuProfilingInfo)
2844 : {
2845 0 : auto dumpLinkInfo = [](const CcuProfilingInfo& info) -> void {
2846 0 : for (int i = 0; i < CCU_MAX_CHANNEL_NUM; i++) {
2847 0 : if (info.channelId[i] == INVALID_VALUE_CHANNELID) {
2848 0 : continue;
2849 : }
2850 0 : HCCL_INFO("channelId(%u), remoteRankId(%u).", info.channelId[i], info.remoteRankId[i]);
2851 : }
2852 0 : };
2853 :
2854 0 : for (const auto& profInfo : ccuProfilingInfo) {
2855 0 : if (profInfo.type == static_cast<uint8_t>(CcuProfilinType::CCU_TASK_PROFILING)) {
2856 0 : HCCL_INFO(
2857 : "Dump CCU Profiling Info:SQE Profiling Info: ctxSignautre(%s), "
2858 : "dieId(%d), missionId(%d), instrId(%d).",
2859 : profInfo.name.c_str(), static_cast<int>(profInfo.dieId), static_cast<int>(profInfo.missionId),
2860 : static_cast<int>(profInfo.instrId));
2861 0 : } else if (profInfo.type == static_cast<uint8_t>(CcuProfilinType::CCU_WAITCKE_PROFILING)) {
2862 0 : HCCL_INFO(
2863 : "Microcode WaitCKE Profiling Info: name(%s), "
2864 : "dieId(%d), missionId(%d), instrId(%d), ckeId(%u), mask(%u).",
2865 : profInfo.name.c_str(), static_cast<int>(profInfo.dieId), static_cast<int>(profInfo.missionId),
2866 : static_cast<int>(profInfo.instrId), profInfo.ckeId, profInfo.mask);
2867 0 : dumpLinkInfo(profInfo);
2868 0 : } else if (profInfo.type == static_cast<uint8_t>(CcuProfilinType::CCU_LOOPGROUP_PROFILING)) {
2869 0 : HCCL_INFO(
2870 : "Microcode LoopGroup Profiling Info: name(%s), "
2871 : "dieId(%d), missionId(%d), instrId(%d), reduceOpType(%d), inputDataType(%d), "
2872 : "outputDataType(%d), dataSize(%llu).",
2873 : profInfo.name.c_str(), static_cast<int>(profInfo.dieId), static_cast<int>(profInfo.missionId),
2874 : static_cast<int>(profInfo.instrId), static_cast<int>(profInfo.reduceOpType),
2875 : static_cast<int>(profInfo.inputDataType), static_cast<int>(profInfo.outputDataType), profInfo.dataSize);
2876 0 : dumpLinkInfo(profInfo);
2877 : }
2878 : }
2879 0 : }
2880 :
2881 0 : constexpr uint64_t SetBits(uint16_t end) { return ((uint64_t(1) << (end + 1)) - uint64_t(1)); }
2882 :
2883 0 : static uint16_t ParseRepeatNumFromParallelParam(uint64_t parallelParam)
2884 : {
2885 0 : constexpr uint16_t repeatBitNum = 7; // 7： repeat num 占 7 bits
2886 0 : constexpr uint16_t repeatNumShiftBit = 55; // 55： repeat num占[61:55]位置
2887 0 : return (parallelParam >> repeatNumShiftBit) & SetBits(repeatBitNum);
2888 : }
2889 :
2890 0 : HcclResult CcuKernel::CollectSqeAndWaitCkeProfilingInfo()
2891 : {
2892 0 : auto& ccuProfilingCache = GetProfilingInfo();
2893 0 : uint32_t count{0};
2894 0 : HCCL_INFO("[GetCcuProfilingInfo] Process sqe&waitcke profiling info start.");
2895 0 : for (auto& profInfo : ccuProfilingCache) {
2896 0 : profInfo.missionId = GetMissionId();
2897 0 : if (profInfo.type == static_cast<uint8_t>(hcomm::CcuProfilinType::CCU_TASK_PROFILING)) {
2898 0 : profInfo.instrId = GetInstrId();
2899 0 : allCcuProfilingInfos_.push_back(profInfo);
2900 0 : continue;
2901 : }
2902 0 : if (count >= GetWaiteCkeProfilingReps().size()) {
2903 0 : HCCL_ERROR(
2904 : "count[%u] out of range[0, %u], cache size(%u).", count, GetWaiteCkeProfilingReps().size(),
2905 : ccuProfilingCache.size());
2906 0 : return HCCL_E_INTERNAL;
2907 : }
2908 0 : auto waitCkeRep = GetWaiteCkeProfilingReps()[count];
2909 0 : profInfo.instrId = waitCkeRep->StartInstrId();
2910 0 : if (profInfo.ckeId == INVALID_CKE_ID) { // localWait Rep
2911 0 : if (waitCkeRep.get() == nullptr) {
2912 0 : HCCL_ERROR("[GetCcuProfilingInfo] localWaitRep is nullptr.");
2913 0 : return HCCL_E_PTR;
2914 : }
2915 0 : profInfo.ckeId = waitCkeRep->GetId();
2916 0 : HCCL_INFO("[CcuKernel][GetCcuProfilingInfo] waitcke[%u]", profInfo.ckeId);
2917 : }
2918 0 : allCcuProfilingInfos_.push_back(profInfo);
2919 0 : count++;
2920 0 : }
2921 0 : return HCCL_SUCCESS;
2922 : }
2923 :
2924 0 : HcclResult CcuKernel::BuildLoopGroupVarIdMaps(
2925 : std::unordered_map<uint16_t, uint32_t>& varId2ArgIndexMap, std::unordered_map<uint16_t, uint16_t>& varId2VarIdMap)
2926 : {
2927 0 : auto& lgProfInfo = GetLGProfilingInfo();
2928 0 : HCCL_INFO("[GetCcuProfilingInfo] create varId2ArgIndexMap start. size=%lu", lgProfInfo.loadRep2ArgIdxMap.size());
2929 0 : for (auto& iter : lgProfInfo.loadRep2ArgIdxMap) {
2930 0 : if (iter.first.get() == nullptr) {
2931 0 : HCCL_ERROR("[GetCcuProfilingInfo] loadRep is nullptr.");
2932 0 : return HCCL_E_PTR;
2933 : }
2934 0 : auto loadRep = dynamic_cast<CcuRep::CcuRepLoadArg*>(iter.first.get());
2935 0 : varId2ArgIndexMap[loadRep->GetVarId()] = iter.second;
2936 : }
2937 :
2938 0 : HCCL_INFO("[GetCcuProfilingInfo] create varId2VarIdMap start. size=%lu", lgProfInfo.assignProfilingReps.size());
2939 0 : for (auto& iter : lgProfInfo.assignProfilingReps) {
2940 0 : if (iter.get() == nullptr) {
2941 0 : HCCL_ERROR("[GetCcuProfilingInfo] assignRep is nullptr.");
2942 0 : return HCCL_E_PTR;
2943 : }
2944 0 : auto assignRep = dynamic_cast<CcuRep::CcuRepAssign*>(iter.get());
2945 0 : varId2VarIdMap[assignRep->GetVarB().Id()] = assignRep->GetVarA().Id();
2946 : }
2947 0 : return HCCL_SUCCESS;
2948 : }
2949 :
2950 0 : HcclResult CcuKernel::CollectLoopGroupProfilingInfo(
2951 : const uint64_t* taskArgs, uint32_t argSize, const std::unordered_map<uint16_t, uint32_t>& varId2ArgIndexMap,
2952 : const std::unordered_map<uint16_t, uint16_t>& varId2VarIdMap)
2953 : {
2954 0 : auto& lgProfInfo = GetLGProfilingInfo();
2955 0 : HCCL_INFO(
2956 : "[GetCcuProfilingInfo] process loop group profiling start: "
2957 : "lgsize(%lu), goSize(%lu)",
2958 : lgProfInfo.lgProfilingReps.size(), groupOpSizeInfo_.size());
2959 0 : for (uint32_t i = 0; i < lgProfInfo.lgProfilingReps.size();
2960 0 : i += 2) { // 2: 一个goSize对应一个CcuProfilingInfo，对应1个loopGroup Rep
2961 0 : if (argSize == 0 || varId2ArgIndexMap.empty()) {
2962 0 : continue;
2963 : }
2964 0 : uint64_t loopParam{0};
2965 0 : CHK_RET(GetArgIndex(
2966 : varId2VarIdMap, varId2ArgIndexMap, taskArgs, argSize, groupOpSizeInfo_[i].loopParamId, loopParam));
2967 0 : uint64_t parallelParam{0};
2968 0 : CHK_RET(GetArgIndex(
2969 : varId2VarIdMap, varId2ArgIndexMap, taskArgs, argSize, groupOpSizeInfo_[i].parallelParamId, parallelParam));
2970 0 : HCCL_INFO(
2971 : "Collect loopgroup profiling info: repSize[%u], index[%u],"
2972 : "loopParam[%llu], parallelParam[%llu].",
2973 : lgProfInfo.lgProfilingReps.size(), i, loopParam, parallelParam);
2974 :
2975 0 : if (loopParam != 0) {
2976 0 : lgProfInfo.ccuProfilingInfos[i].dataSize = loopParam * moConfig_.loopCount * moConfig_.memSlice;
2977 0 : lgProfInfo.ccuProfilingInfos[i].instrId
2978 0 : = dynamic_cast<CcuRep::CcuRepLoopGroupBundle*>(lgProfInfo.lgProfilingReps[i].get())->StartInstrId();
2979 0 : allCcuProfilingInfos_.push_back(lgProfInfo.ccuProfilingInfos[i]);
2980 : }
2981 :
2982 0 : if (parallelParam != 0) {
2983 0 : HCCL_INFO("[GetCcuProfilingInfo] collect lg, residual start i=%lu", i);
2984 0 : uint64_t residual{0};
2985 0 : CHK_RET(GetArgIndex(
2986 : varId2VarIdMap, varId2ArgIndexMap, taskArgs, argSize, groupOpSizeInfo_[i].residualId, residual));
2987 0 : uint64_t repeatNum = ParseRepeatNumFromParallelParam(parallelParam);
2988 0 : lgProfInfo.ccuProfilingInfos[i].dataSize = repeatNum * moConfig_.memSlice + residual;
2989 0 : lgProfInfo.ccuProfilingInfos[i].instrId
2990 0 : = dynamic_cast<CcuRep::CcuRepLoopGroupBundle*>(lgProfInfo.lgProfilingReps[i + 1].get())->StartInstrId();
2991 0 : allCcuProfilingInfos_.push_back(lgProfInfo.ccuProfilingInfos[i]);
2992 : }
2993 : }
2994 0 : return HCCL_SUCCESS;
2995 : }
2996 :
2997 : /*
2998 : * variable/maskSignal等资源变量Id，一定要在获取ccu profiling时才获取；
2999 : * 原因：在创建context Rep时，其资源Id属于虚拟资源；翻译时，才会绑定固定的物理资源。
3000 : */
3001 0 : HcclResult CcuKernel::GetCcuProfilingInfo(
3002 : const uint64_t* taskArgs, uint32_t argSize, std::vector<CcuProfilingInfo>& allCcuProfilingInfo)
3003 : {
3004 0 : HCCL_INFO("[GetCcuProfilingInfo] Enter.");
3005 0 : allCcuProfilingInfos_.clear();
3006 :
3007 0 : CHK_RET(CollectSqeAndWaitCkeProfilingInfo());
3008 :
3009 0 : std::unordered_map<uint16_t, uint32_t> varId2ArgIndexMap;
3010 0 : std::unordered_map<uint16_t, uint16_t> varId2VarIdMap;
3011 0 : CHK_RET(BuildLoopGroupVarIdMaps(varId2ArgIndexMap, varId2VarIdMap));
3012 :
3013 0 : CHK_RET(CollectLoopGroupProfilingInfo(taskArgs, argSize, varId2ArgIndexMap, varId2VarIdMap));
3014 :
3015 0 : DumpCcuProfilingInfo(allCcuProfilingInfos_);
3016 0 : allCcuProfilingInfo = allCcuProfilingInfos_;
3017 0 : return HCCL_SUCCESS;
3018 0 : }
3019 :
3020 0 : HcclResult CcuKernel::AddProfilingInfo(
3021 : const ChannelHandle* channels, uint32_t channelNum, HcclDataType dataType, HcclDataType outputDataType,
3022 : HcclReduceOp opType, const std::string& opName)
3023 : {
3024 0 : CHK_PTR_NULL(channels);
3025 0 : ccuProfilingInfoCache.type = (uint8_t)CcuProfilinType::CCU_LOOPGROUP_PROFILING;
3026 0 : ccuProfilingInfoCache.name = opName;
3027 0 : ccuProfilingInfoCache.reduceOpType = opType;
3028 0 : ccuProfilingInfoCache.inputDataType = dataType;
3029 0 : ccuProfilingInfoCache.outputDataType = outputDataType;
3030 0 : ccuProfilingInfoCache.missionId = GetMissionId();
3031 :
3032 0 : CHK_SAFETY_FUNC_RET(memset_s(
3033 : ccuProfilingInfoCache.channelId, sizeof(ccuProfilingInfoCache.channelId), INVALID_VALUE_CHANNELID,
3034 : sizeof(ccuProfilingInfoCache.channelId)));
3035 0 : for (uint32_t i = 0; i < channelNum; i++) {
3036 0 : void* channelPtr{nullptr};
3037 0 : CHK_RET(static_cast<HcclResult>(HcommChannelGet(channels[i], &channelPtr)));
3038 0 : auto* channelImpl = dynamic_cast<CcuUrmaChannel*>(static_cast<Channel*>(channelPtr));
3039 0 : CHK_PTR_NULL(channelImpl);
3040 0 : ccuProfilingInfoCache.channelId[i] = channelImpl->GetChannelId();
3041 0 : ccuProfilingInfoCache.channelHandle[i] = channels[i];
3042 0 : HCCL_INFO(
3043 : "[%s]type[%d], name[%s], opType[%d], dataType[%d], outputDataType[%d], missionId[%u], "
3044 : "channelHandle[0x%llx], channelId[%u]",
3045 : __func__, ccuProfilingInfoCache.type, ccuProfilingInfoCache.name.c_str(), opType, dataType, outputDataType,
3046 : ccuProfilingInfoCache.missionId, ccuProfilingInfoCache.channelHandle[i],
3047 : ccuProfilingInfoCache.channelId[i]);
3048 : }
3049 0 : lgProfilingInfo.ccuProfilingInfos.push_back(ccuProfilingInfoCache);
3050 0 : lgProfilingInfo.lgProfilingReps.push_back(allLgProfilingReps.back());
3051 0 : return HCCL_SUCCESS;
3052 : }
3053 :
3054 0 : HcclResult CcuKernel::AddCcuProfiling(
3055 : GroupInfo groupInfo, const std::vector<ChannelHandle> channelHandle, HcclDataType dataType,
3056 : HcclDataType outputDataType, HcclReduceOp opType, const std::string& opName)
3057 : {
3058 0 : CHK_RET(AddCcuProfiling(channelHandle.data(), channelHandle.size(), dataType, outputDataType, opType, opName));
3059 0 : groupOpSizeInfo_.push_back(groupInfo);
3060 0 : return HCCL_SUCCESS;
3061 : }
3062 :
3063 0 : HcclResult CcuKernel::AddCcuProfiling(
3064 : const ChannelHandle* channels, uint32_t channelNum, HcclDataType dataType, HcclDataType outputDataType,
3065 : HcclReduceOp opType, const std::string& opName)
3066 : {
3067 0 : CHK_PTR_NULL(channels);
3068 0 : CHK_RET(AddProfilingInfo(channels, channelNum, dataType, outputDataType, opType, opName));
3069 0 : return HCCL_SUCCESS;
3070 : }
3071 :
3072 5 : HcclResult CcuKernel::Add2ConstValue2VarMap(std::vector<uint64_t>& values)
3073 : {
3074 : // 记录当前context所需的常量，仅A6场景适用
3075 21 : for (uint64_t value : values) {
3076 16 : if (constValue2VarMap.find(value) == constValue2VarMap.end()) {
3077 3 : constValue2VarMap[value] = CreateVariable();
3078 : }
3079 : }
3080 5 : return HCCL_SUCCESS;
3081 : }
3082 :
3083 : }; // namespace hcomm
|
