Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "ccu_kernel.h"
12 : #include "ccu_rep_v1.h"
13 : #include "ccu_kernel_resource.h"
14 : #include "ccu_microcode_v1.h"
15 :
16 : #include "ccu_types.h"
17 : #include "exception_util.h"
18 : #include "ccu_api_exception.h"
19 : #include "ccu_dev_mgr_imp.h"
20 : #include "env_config.h"
21 : #include "ccu_rep_type_v1.h"
22 :
23 : #include "hcomm_c_adpt.h"
24 :
25 : #include "ccu_rep_context_v1.h"
26 : #include "ccu_rep_funccall_v1.h"
27 : #include "../../endpoint_pairs/channels/ccu/ccu_urma_channel.h"
28 :
29 : #include "ccu_log.h"
30 :
31 : #include "hcom_common.h"
32 :
33 : // todo: 引入头文件需要检查
34 : #include "ccu_assist_v1.h"
35 : #include "hccl_comm_pub.h"
36 : #include "hcclCommDfx.h"
37 : #include "task_info.h"
38 : #include "task_param.h"
39 :
40 : #include "ccu_ins_generator_base.h"
41 : #include "ccu_ins_generator_v1.h"
42 : #include "unified_platform/pub_inc/config_plf_log.h"
43 :
44 : namespace hcomm {
45 :
46 : using Hccl::PLF_DATA_OP;
47 :
48 : constexpr uint32_t TOKEN_VALUE_INDEX = 2;
49 : constexpr uint16_t INVALID_U16 = 65535;
50 :
51 : using CcuRep::CcuInsGeneratorBase;
52 : using CcuRep::CcuInsGeneratorV1;
53 :
54 : template <typename T>
55 988 : T CcuKernel::CreateResAssist(
56 : std::array<std::vector<T>, CCU_MAX_IODIE_NUM> &resRecord)
57 : {
58 : // kernel确认die之前默认为0，需要刷新资源
59 : // 确认die之后按实际使用die分配资源
60 988 : const uint32_t dieId = GetDieId();
61 988 : resRecord[dieId].emplace_back(this);
62 988 : auto& item = resRecord[dieId].back();
63 988 : item.Reset(resRecord[dieId].size(), dieId);
64 988 : return item;
65 : }
66 :
67 : template <typename T>
68 36 : std::vector<T> CcuKernel::CreateBlockResAssist(
69 : const uint32_t count,
70 : std::array<std::vector<T>, CCU_MAX_IODIE_NUM> &resRecord)
71 : {
72 36 : constexpr uint16_t CCU_BLOCK_RES_ID_BASE = 0x1000; // block 批量分配资源 id 基址，与单资源 id 区间隔离便于 DFX 定位
73 36 : std::vector<T> block;
74 36 : block.reserve(count);
75 36 : const uint32_t dieId = GetDieId();
76 448 : for (size_t i = 0; i < count; i++) {
77 412 : block.emplace_back(this);
78 412 : block.back().Reset(static_cast<uint16_t>(CCU_BLOCK_RES_ID_BASE + resRecord[dieId].size() + i), dieId);
79 : }
80 36 : resRecord[dieId].insert(resRecord[dieId].end(), block.begin(), block.end());
81 36 : return block;
82 0 : }
83 :
84 129 : CcuKernel::~CcuKernel()
85 : {
86 129 : }
87 :
88 11 : static HcclResult GetDieIdByChannel(const ChannelHandle channel, uint32_t &dieId)
89 : {
90 11 : void *channelPtr{nullptr};
91 11 : CHK_RET(static_cast<HcclResult>(HcommChannelGet(channel, &channelPtr)));
92 11 : auto *channelImpl = dynamic_cast<CcuUrmaChannel *>(static_cast<Channel *>(channelPtr));
93 11 : if (channelImpl == nullptr) {
94 0 : HCCL_ERROR("[%s] failed to cast channel[0x%llx] to CcuUrmaChannel", __func__, channel);
95 0 : return HcclResult::HCCL_E_PTR;
96 : }
97 11 : dieId = channelImpl->GetDieId();
98 11 : HCCL_INFO("[%s], channelHandle[0x%llx], dieId[%u]", __func__, channel, dieId);
99 11 : return HcclResult::HCCL_SUCCESS;
100 : }
101 :
102 0 : static HcclResult GetDieIdByChannels(const std::unordered_set<ChannelHandle> &channels, uint32_t &dieId)
103 : {
104 0 : if (channels.empty()) {
105 0 : int32_t devLogicId = HcclGetThreadDeviceId();
106 0 : for (uint32_t die = 0; die < CCU_MAX_IODIE_NUM; die++) {
107 0 : bool enableFlag = false;
108 0 : CHK_RET(static_cast<HcclResult>(CcuGetDieEnableInfo(devLogicId, die, enableFlag)));
109 0 : if (enableFlag) {
110 0 : dieId = die;
111 0 : return HcclResult::HCCL_SUCCESS;
112 : }
113 : }
114 :
115 0 : HCCL_ERROR("[CcuKernel][%s] failed, all dies are disable, devLogicId[%d].", __func__, devLogicId);
116 0 : return HcclResult::HCCL_E_INTERNAL;
117 : }
118 :
119 0 : uint32_t firstDieId = 0;
120 0 : CHK_RET(GetDieIdByChannel(*channels.begin(), firstDieId));
121 0 : for (const auto channel : channels) {
122 0 : uint32_t nextDieId = 0;
123 0 : CHK_RET(GetDieIdByChannel(channel, nextDieId));
124 0 : if (firstDieId != nextDieId) {
125 0 : HCCL_ERROR("[%s] failed, the dies of channels are not same.", __func__);
126 0 : return HcclResult::HCCL_E_PARA;
127 : }
128 : }
129 :
130 0 : dieId = firstDieId;
131 0 : return HcclResult::HCCL_SUCCESS;
132 : }
133 :
134 46 : static HcclResult CheckChannelsDie(
135 : const std::unordered_set<ChannelHandle> &channels, const uint32_t targetDieId)
136 : {
137 54 : for (const auto channel : channels) {
138 11 : uint32_t channelDieId = 0;
139 14 : CHK_RET(GetDieIdByChannel(channel, channelDieId));
140 11 : if (channelDieId != targetDieId) {
141 3 : HCCL_ERROR("[%s] failed, channel[0x%llx] dieId[%u] differs from target dieId[%u].",
142 : __func__, channel, channelDieId, targetDieId);
143 3 : return HcclResult::HCCL_E_PARA;
144 : }
145 : }
146 43 : return HcclResult::HCCL_SUCCESS;
147 : }
148 :
149 43 : static void MoveResourcesToDie(CcuRepResource &res, uint32_t targetDieId)
150 : {
151 43 : if (targetDieId == 0) return; // 初始资源位于die0，不用设置
152 :
153 11 : auto moveAndSet = [&](auto &arr) {
154 11 : arr[targetDieId] = std::move(arr[0]);
155 11 : for (auto &item : arr[targetDieId]) item.SetDieId(targetDieId);
156 12 : };
157 :
158 1 : moveAndSet(res.ccubufs);
159 1 : moveAndSet(res.blockCcubufs);
160 1 : moveAndSet(res.executor);
161 1 : moveAndSet(res.blockExecutor);
162 1 : moveAndSet(res.completedEvent);
163 1 : moveAndSet(res.blockCompletedEvent);
164 1 : moveAndSet(res.address);
165 1 : moveAndSet(res.blockAddress);
166 1 : moveAndSet(res.continuousVariable);
167 1 : moveAndSet(res.variable);
168 1 : moveAndSet(res.localNotify);
169 : }
170 :
171 59 : HcclResult CcuKernel::SetupProfilingInfo(const char *kernelFuncName)
172 : {
173 59 : if (kernelFuncName == nullptr || strlen(kernelFuncName) == 0) {
174 0 : name_ = std::string("CCU_KERNEL"); // 默认名称
175 0 : AddSqeProfiling(name_);
176 0 : return HcclResult::HCCL_SUCCESS;
177 : }
178 :
179 59 : constexpr size_t MAX_KERNEL_FUNC_NAME_LEN = 128;
180 59 : const auto nameLen = strlen(kernelFuncName);
181 59 : if (nameLen > MAX_KERNEL_FUNC_NAME_LEN) {
182 0 : name_ = std::string(kernelFuncName, MAX_KERNEL_FUNC_NAME_LEN);
183 0 : HCCL_WARNING("[CcuKernel][%s] kernelFuncName is too long, reset to %s.",
184 : __func__, name_.c_str());
185 : }
186 :
187 : // 生成SQE粒度profiling信息，此时未选择die，默认die 0
188 59 : AddSqeProfiling(name_);
189 59 : return HcclResult::HCCL_SUCCESS;
190 : }
191 :
192 43 : static HcclResult UpdateProfilingInfo(
193 : std::vector<CcuProfilingInfo> &profilingInfos, uint32_t dieId,
194 : const std::string &kernelName)
195 : {
196 43 : if (dieId == 0) {
197 : // 与默认dieId相同，不需要修改
198 42 : return HcclResult::HCCL_SUCCESS;
199 : }
200 :
201 : // 正常情况仅首个info包含die信息，仅应为CCU_TASK_PROFILING类型
202 1 : if (UNLIKELY(profilingInfos.empty())) {
203 : // profiling不属于主流程，不打断算子业务
204 0 : HCCL_INFO("[%s] passed, profiling infos are empty, ccu kernel func[%s].",
205 : __func__, kernelName.c_str());
206 0 : return HcclResult::HCCL_SUCCESS;
207 : }
208 :
209 : // 根据选择的die跟新profiling信息
210 2 : for (auto &info : profilingInfos) {
211 1 : info.dieId = dieId;
212 : }
213 :
214 1 : HCCL_INFO("[%s] reset profiling info dieId to [%u], ccu kernel func[%s].",
215 : __func__, dieId, kernelName.c_str());
216 1 : return HcclResult::HCCL_SUCCESS;
217 : }
218 :
219 0 : HcclResult CcuKernel::ApplyDieFromChannels()
220 : {
221 0 : uint32_t dieId{0};
222 0 : CHK_RET(GetDieIdByChannels(channels_, dieId));
223 0 : CHK_PRT_RET(
224 : dieId >= CCU_MAX_IODIE_NUM,
225 : HCCL_ERROR("[CcuKernel][%s] failed, dieId[%u] should be less than [%u].", __func__, dieId, CCU_MAX_IODIE_NUM),
226 : HcclResult::HCCL_E_PARA);
227 0 : SetDieId(dieId);
228 0 : MoveResourcesToDie(res_, dieId);
229 0 : (void)UpdateProfilingInfo(profilingInfo, dieId, name_);
230 :
231 0 : return HcclResult::HCCL_SUCCESS;
232 : }
233 :
234 46 : HcclResult CcuKernel::ValidateAndApplyDie(const uint32_t targetDieId)
235 : {
236 46 : CHK_PRT_RET(targetDieId >= CCU_MAX_IODIE_NUM,
237 : HCCL_ERROR("[CcuKernel][%s] failed, dieId[%u] should be less than [%u].",
238 : __func__, targetDieId, CCU_MAX_IODIE_NUM),
239 : HcclResult::HCCL_E_PARA);
240 :
241 46 : const int32_t devLogicId = HcclGetThreadDeviceId();
242 46 : bool enableFlag = false;
243 46 : CHK_RET(static_cast<HcclResult>(
244 : CcuGetDieEnableInfo(devLogicId, static_cast<uint8_t>(targetDieId), enableFlag)));
245 46 : CHK_PRT_RET(!enableFlag,
246 : HCCL_ERROR("[CcuKernel][%s] failed, target dieId[%u] is disabled, devLogicId[%d].",
247 : __func__, targetDieId, devLogicId),
248 : HcclResult::HCCL_E_PARA);
249 46 : CHK_RET(CheckChannelsDie(channels_, targetDieId));
250 :
251 43 : SetDieId(targetDieId);
252 43 : MoveResourcesToDie(res_, targetDieId);
253 43 : (void)UpdateProfilingInfo(profilingInfo, targetDieId, name_);
254 :
255 43 : return HcclResult::HCCL_SUCCESS;
256 : }
257 :
258 62 : void CcuKernel::SetInsGenerater(CcuInsGeneratorBase* insGeneratorBase)
259 : {
260 62 : insGenerator = insGeneratorBase;
261 62 : }
262 :
263 5 : CcuResult CcuKernel::ValidateTaskArgs(const uint64_t *taskArgs, uint32_t argsNum) const
264 : {
265 5 : if (loadArgUsedSet_.size() != argsNum) {
266 0 : HCCL_ERROR("[CcuKernel][%s] failed, args number does not match the Load instruction, "
267 : "argsNum = %u, loaded = %zu", __func__, argsNum, loadArgUsedSet_.size());
268 0 : return CcuResult::CCU_E_INTERNAL;
269 : }
270 50 : for (uint32_t i = 0; i < argsNum; ++i) {
271 45 : if (loadArgUsedSet_.count(i) == 0) {
272 0 : HCCL_ERROR("[CcuKernel][%s] failed, argId %u not loaded (argsNum=%u)",
273 : __func__, i, argsNum);
274 0 : return CcuResult::CCU_E_INTERNAL;
275 : }
276 : }
277 5 : if (argsNum != 0) {
278 3 : CCU_CHK_PTR_NULL(taskArgs);
279 : }
280 5 : if (instrInfo_.missionInstrCount == 0 || instrInfo_.instrVec.empty()) {
281 0 : HCCL_ERROR("[CcuKernel][%s] failed, mission instructions are empty, "
282 : "the kernel is not been translated yet.", __func__);
283 0 : return CcuResult::CCU_E_INTERNAL;
284 : }
285 5 : return CcuResult::CCU_SUCCESS;
286 : }
287 :
288 8 : void CcuKernel::FillTaskParam(CcuTaskParam ¶m, uint32_t index, uint32_t seqNum,
289 : const uint64_t *taskArgs, uint32_t argsNum) const
290 : {
291 8 : param.dieId = GetDieId();
292 8 : param.missionId = GetMissionId();
293 8 : param.instStartId = instrInfo_.missionStartInstrId + index * CCU_SQE_ARGS_LEN;
294 8 : param.key = GetMissionKey();
295 8 : param.argSize = CCU_SQE_ARGS_LEN;
296 :
297 8 : const uint32_t preMissionInsCnt = index * CCU_SQE_ARGS_LEN;
298 8 : const bool isLast = (index == seqNum - 1);
299 8 : param.instCnt = isLast ? (instrInfo_.missionInstrCount - preMissionInsCnt) : CCU_SQE_ARGS_LEN;
300 :
301 8 : if (argsNum > preMissionInsCnt) {
302 : const uint32_t argsToCopy = isLast
303 6 : ? std::min(argsNum - preMissionInsCnt, CCU_SQE_ARGS_LEN)
304 6 : : CCU_SQE_ARGS_LEN;
305 6 : std::copy(taskArgs + preMissionInsCnt, taskArgs + preMissionInsCnt + argsToCopy,
306 6 : std::begin(param.args));
307 : }
308 :
309 8 : HCCL_INFO("[GeneTaskParam]task Param, dieId[%u] missionId[%u] instStartId[%u] instCnt[%u], argSize[%u]",
310 : param.dieId, param.missionId, param.instStartId, param.instCnt, param.argSize);
311 8 : }
312 :
313 5 : CcuResult CcuKernel::GeneTaskParams(const uint64_t *taskArgs, uint32_t argsNum,
314 : std::vector<CcuTaskParam> &taskParams)
315 : {
316 5 : CCU_CHK_RET(ValidateTaskArgs(taskArgs, argsNum));
317 :
318 : // 如果agrs数量超过sqe arg的最大数量，则返回多个TaskParam，前面几个只从sqe中加载args;
319 : // args数量大于等于0、小于等于最大值时，返回1个TaskParam
320 5 : const uint32_t seqNum
321 5 : = (argsNum / CCU_SQE_ARGS_LEN) + ((argsNum % CCU_SQE_ARGS_LEN) == 0 ? 0 : 1) + (argsNum == 0 ? 1 : 0);
322 :
323 5 : const uint32_t preMissonSqeInsCnt = (seqNum - 1) * CCU_SQE_ARGS_LEN;
324 5 : if (instrInfo_.missionInstrCount < preMissonSqeInsCnt) {
325 0 : HCCL_ERROR("[CcuKernel][%s] failed, missionInstrCount[%u] should be greater "
326 : "than preMissonSqeInsCnt[%u].", __func__, instrInfo_.missionInstrCount,
327 : preMissonSqeInsCnt);
328 0 : return CcuResult::CCU_E_INTERNAL;
329 : }
330 :
331 5 : taskParams.resize(seqNum);
332 13 : for (uint32_t index = 0; index < seqNum; index++) {
333 8 : FillTaskParam(taskParams[index], index, seqNum, taskArgs, argsNum);
334 : }
335 :
336 5 : return CcuResult::CCU_SUCCESS;
337 : }
338 :
339 14 : HcclResult CcuKernel::CreateVariable(const ChannelHandle channel, uint32_t varIndex, CcuRep::Variable *var)
340 : {
341 14 : channels_.insert(channel);
342 :
343 14 : void *channelPtr{nullptr};
344 14 : CHK_RET(static_cast<HcclResult>(HcommChannelGet(channel, &channelPtr)));
345 14 : auto *channelImpl = dynamic_cast<CcuUrmaChannel *>(static_cast<Channel *>(channelPtr));
346 14 : if (channelImpl == nullptr) {
347 0 : HCCL_ERROR("[%s] failed to cast channel[0x%llx] to CcuUrmaChannel", __func__, channel);
348 0 : return HcclResult::HCCL_E_PTR;
349 : }
350 14 : uint32_t locXnId{0};
351 14 : CHK_RET(channelImpl->GetLocXnByIndex(varIndex, locXnId));
352 14 : var->Reset(locXnId, channelImpl->GetDieId());
353 14 : return HcclResult::HCCL_SUCCESS;
354 : }
355 :
356 33 : CcuRepResource &CcuKernel::GetResource()
357 : {
358 33 : return res_;
359 : }
360 :
361 69 : CcuResReq CcuKernel::GetResourceRequest()
362 : {
363 69 : CcuResReq req;
364 69 : uint32_t dieId = GetDieId();
365 69 : req.msReq[dieId] = res_.ccubufs[dieId].size();
366 69 : req.blockMsReq[dieId] = res_.blockCcubufs[dieId].size();
367 69 : req.ckeReq[dieId] = res_.completedEvent[dieId].size();
368 69 : req.blockCkeReq[dieId] = res_.blockCompletedEvent[dieId].size() + res_.localNotify[dieId].size();
369 69 : req.loopEngineReq[dieId] = res_.executor[dieId].size();
370 69 : req.blockLoopEngineReq[dieId] = res_.blockExecutor[dieId].size();
371 69 : req.gsaReq[dieId] = res_.address[dieId].size();
372 69 : req.blockGsaReq[dieId] = res_.blockAddress[dieId].size();
373 69 : req.xnReq[dieId] = res_.variable[dieId].size();
374 69 : req.blockXnReq[dieId] = res_.continuousVariable[dieId].size();
375 :
376 69 : req.missionReq.reqType = MissionReqType::FUSION_MULTIPLE_DIE;
377 69 : req.missionReq.req[dieId] = 1;
378 :
379 : auto info
380 : = Hccl::StringFormat("resource request: dieId[%u], ms[%u], blockMs[%u], cke[%u], blockCke[%u], "
381 : "loopEngine[%u], blockLoopEngine[%u], gsa[%u], blockGsa[%u], xn[%u], blockXn[%u], "
382 : "missionId[%u]",
383 276 : dieId, req.msReq[dieId], req.blockMsReq[dieId], req.ckeReq[dieId], req.blockCkeReq[dieId],
384 276 : req.loopEngineReq[dieId], req.blockLoopEngineReq[dieId], req.gsaReq[dieId], req.blockGsaReq[dieId],
385 69 : req.xnReq[dieId], req.blockXnReq[dieId], req.missionReq.req[dieId]);
386 :
387 69 : HCCL_INFO("%s", info.c_str());
388 :
389 138 : return req;
390 69 : }
391 :
392 : template<typename HandleType, typename ResourceType>
393 1313 : static CcuResult GetResourceByHandle(
394 : std::unordered_map<HandleType, ResourceType> &resourceMap,
395 : HandleType handle, ResourceType **resource, const char *resourceType)
396 : {
397 1313 : auto iter = resourceMap.find(handle);
398 1313 : if (iter == resourceMap.end()) {
399 0 : HCCL_ERROR("[%s] failed to find %s by handle: 0x%llx", __func__, resourceType, handle);
400 0 : return CcuResult::CCU_E_NOT_FOUND;
401 : }
402 :
403 : // ccu资源本身可能重载=，对象赋值会被转换成指令，导致流程失败
404 1313 : *resource = &(iter->second);
405 1313 : return CcuResult::CCU_SUCCESS;
406 : }
407 :
408 1015 : CcuResult CcuKernel::GetVariableByHandle(CcuVariableHandle varHandle, CcuRep::Variable **variable)
409 : {
410 1015 : return GetResourceByHandle(ccuVarMap_, varHandle, variable, "variable");
411 : }
412 : //Alloc 相关接口
413 431 : CcuResult CcuKernel::VariableAlloc(CcuVariableHandle *varHandle)
414 : {
415 431 : PLF_CONFIG_INFO(PLF_DATA_OP, "[VariableAlloc]");
416 431 : const auto &var = CreateResAssist(res_.continuousVariable);
417 431 : CcuVariableHandle handle = ccuVarMap_.size();
418 431 : ccuVarMap_.emplace(handle, var);
419 :
420 431 : *varHandle = handle;
421 431 : return CcuResult::CCU_SUCCESS;
422 431 : }
423 10 : CcuResult CcuKernel::AddressAlloc(CcuAddressHandle *addrHandle)
424 : {
425 10 : PLF_CONFIG_INFO(PLF_DATA_OP, "[AddressAlloc]");
426 10 : const auto addr = CreateAddress();
427 10 : CcuAddressHandle handle = ccuAddrMap_.size();
428 10 : ccuAddrMap_.emplace(handle, addr);
429 10 : *addrHandle = handle;
430 10 : return CcuResult::CCU_SUCCESS;
431 10 : }
432 9 : CcuResult CcuKernel::EventAlloc(CcuEventHandle *eventHandle)
433 : {
434 9 : PLF_CONFIG_INFO(PLF_DATA_OP, "[EventAlloc]");
435 9 : const auto &event = CreateResAssist(res_.blockCompletedEvent);
436 9 : CcuEventHandle handle = ccuEventMap_.size();
437 9 : ccuEventMap_.emplace(handle, event);
438 9 : *eventHandle = handle;
439 9 : return CcuResult::CCU_SUCCESS;
440 9 : }
441 1 : CcuResult CcuKernel::BufferAlloc(CcuBufferHandle *bufHandle)
442 : {
443 1 : PLF_CONFIG_INFO(PLF_DATA_OP, "[BufferAlloc]");
444 1 : const auto &buf = CreateResAssist(res_.blockCcubufs);
445 1 : CcuBufferHandle handle = ccuBufferMap_.size();
446 1 : ccuBufferMap_.emplace(handle, buf);
447 1 : *bufHandle = handle;
448 1 : return CcuResult::CCU_SUCCESS;
449 1 : }
450 154 : CcuResult CcuKernel::LocalAddrAlloc(CcuLocalAddrHandle *localAddrHandle, CcuAddressHandle *addrHandle, CcuVariableHandle *tokenHandle)
451 : {
452 154 : PLF_CONFIG_INFO(PLF_DATA_OP, "[LocalAddrAlloc]");
453 154 : auto localAddr = CreateLocalAddr();
454 :
455 154 : CcuAddressHandle aHandle = ccuAddrMap_.size();
456 154 : ccuAddrMap_.emplace(aHandle, localAddr.addr);
457 :
458 154 : CcuVariableHandle tHandle = ccuVarMap_.size();
459 154 : ccuVarMap_.emplace(tHandle, localAddr.token);
460 :
461 154 : CcuLocalAddrHandle laHandle = ccuLocalAddrMap_.size();
462 154 : ccuLocalAddrMap_.emplace(laHandle, localAddr);
463 :
464 154 : *localAddrHandle = laHandle;
465 154 : *addrHandle = aHandle;
466 154 : *tokenHandle = tHandle;
467 154 : return CcuResult::CCU_SUCCESS;
468 :
469 154 : }
470 37 : CcuResult CcuKernel::RemoteAddrAlloc(CcuRemoteAddrHandle *remoteAddrHandle, CcuAddressHandle *addrHandle, CcuVariableHandle *tokenHandle)
471 : {
472 37 : PLF_CONFIG_INFO(PLF_DATA_OP, "[RemoteAddrAlloc]");
473 37 : auto remoteAddr = CreateRemoteAddr();
474 :
475 37 : CcuAddressHandle aHandle = ccuAddrMap_.size();
476 37 : ccuAddrMap_.emplace(aHandle, remoteAddr.addr);
477 :
478 37 : CcuVariableHandle tHandle = ccuVarMap_.size();
479 37 : ccuVarMap_.emplace(tHandle, remoteAddr.token);
480 :
481 37 : CcuRemoteAddrHandle raHandle = ccuRemoteAddrMap_.size();
482 37 : ccuRemoteAddrMap_.emplace(raHandle, remoteAddr);
483 :
484 37 : *remoteAddrHandle = raHandle;
485 37 : *addrHandle = aHandle;
486 37 : *tokenHandle = tHandle;
487 37 : return CcuResult::CCU_SUCCESS;
488 37 : }
489 :
490 6 : CcuResult CcuKernel::BlockVariableAlloc(CcuVariableHandle *varHandles, uint32_t count)
491 : {
492 6 : PLF_CONFIG_INFO(PLF_DATA_OP, "[BlockVariableAlloc] count=%u", count);
493 6 : const auto& var = CreateBlockResAssist(count, res_.continuousVariable);
494 18 : for (uint32_t i = 0; i < count; i++) {
495 12 : CcuVariableHandle handle = ccuVarMap_.size();
496 12 : ccuVarMap_.emplace(handle, var[i]);
497 12 : varHandles[i] = handle;
498 : }
499 6 : return CcuResult::CCU_SUCCESS;
500 6 : }
501 :
502 6 : CcuResult CcuKernel::BlockEventAlloc(CcuEventHandle *eventHandles, uint32_t count)
503 : {
504 6 : PLF_CONFIG_INFO(PLF_DATA_OP, "[BlockEventAlloc] count=%u", count);
505 6 : const auto& event = CreateBlockResAssist(count, res_.blockCompletedEvent);
506 52 : for (uint32_t i = 0; i < count; i++) {
507 46 : CcuEventHandle handle = ccuEventMap_.size();
508 46 : ccuEventMap_.emplace(handle, event[i]);
509 46 : eventHandles[i] = handle;
510 : }
511 6 : return CcuResult::CCU_SUCCESS;
512 6 : }
513 :
514 6 : CcuResult CcuKernel::BlockBufferAlloc(CcuBufferHandle *bufHandles, uint32_t count)
515 : {
516 6 : PLF_CONFIG_INFO(PLF_DATA_OP, "[BlockBufferAlloc] count=%u", count);
517 6 : const auto& buffer = CreateBlockResAssist(count, res_.blockCcubufs);
518 330 : for (uint32_t i = 0; i < count; i++) {
519 324 : CcuBufferHandle handle = ccuBufferMap_.size();
520 324 : ccuBufferMap_.emplace(handle, buffer[i]);
521 324 : bufHandles[i] = handle;
522 : }
523 6 : return CcuResult::CCU_SUCCESS;
524 6 : }
525 :
526 14 : CcuResult CcuKernel::VariableCreateByChannel(ChannelHandle channel, uint32_t varIndex, CcuVariableHandle *varHandle)
527 : {
528 14 : PLF_CONFIG_INFO(PLF_DATA_OP, "[VariableCreateByChannel] channel=%llu, varIndex=%u", channel, varIndex);
529 14 : channels_.insert(channel);
530 14 : CcuRep::Variable var(this);
531 14 : CCU_CHK_RET(CreateVariable(channel, varIndex, &var));
532 14 : CcuVariableHandle handle = ccuVarMap_.size();
533 14 : ccuVarMap_.emplace(handle, var);
534 14 : *varHandle = handle;
535 14 : return CcuResult::CCU_SUCCESS;
536 14 : }
537 :
538 206 : CcuResult CcuKernel::VariableAssignImm(CcuVariableHandle varHandle, uint64_t immediate)
539 : {
540 206 : PLF_CONFIG_INFO(PLF_DATA_OP, "[VariableAssignImm] varHandle=%llu, immediate=%llu", varHandle, immediate);
541 206 : CcuRep::Variable *variable{nullptr};
542 206 : CCU_CHK_RET(GetVariableByHandle(varHandle, &variable));
543 : // 通过符号重载实现，内部记录rep；异常由入口 HcommCcuKernelRegister 的
544 : // CCU_EXCEPTION_HANDLE_BEGIN/END 统一接住，无需在此局部 try/catch。
545 206 : (*variable) = immediate;
546 206 : return CcuResult::CCU_SUCCESS;
547 : }
548 :
549 77 : CcuResult CcuKernel::VariableAssignVar(CcuVariableHandle varHandle, CcuVariableHandle varA)
550 : {
551 77 : PLF_CONFIG_INFO(PLF_DATA_OP, "[VariableAssignVar] varHandle=%llu, varA=%llu", varHandle, varA);
552 77 : CcuRep::Variable *variable{nullptr};
553 77 : CCU_CHK_RET(GetVariableByHandle(varHandle, &variable));
554 77 : CcuRep::Variable *variableA{nullptr};
555 77 : CCU_CHK_RET(GetVariableByHandle(varA, &variableA));
556 : // 通过符号重载实现，内部记录rep；异常由入口统一 catch。
557 77 : (*variable) = (*variableA);
558 77 : return CcuResult::CCU_SUCCESS;
559 : }
560 :
561 99 : CcuResult CcuKernel::VariableAddVarToVar(CcuVariableHandle varHandle, CcuVariableHandle varAHandle, CcuVariableHandle varBHandle)
562 : {
563 99 : PLF_CONFIG_INFO(PLF_DATA_OP, "[VariableAddVarToVar] varHandle=%llu, varAHandle=%llu, varBHandle=%llu",
564 : varHandle, varAHandle, varBHandle);
565 99 : CcuRep::Variable *resVar{nullptr}, *leftVar{nullptr}, *rightVar{nullptr};
566 99 : CCU_CHK_RET(GetVariableByHandle(varHandle, &resVar));
567 99 : CCU_CHK_RET(GetVariableByHandle(varAHandle, &leftVar));
568 99 : CCU_CHK_RET(GetVariableByHandle(varBHandle, &rightVar));
569 :
570 : // 通过符号重载实现，内部记录rep；异常由入口统一 catch。
571 99 : *resVar = *leftVar + *rightVar;
572 99 : return CcuResult::CCU_SUCCESS;
573 : }
574 :
575 2 : CcuResult CcuKernel::VariableSubVarToVar(CcuVariableHandle varHandle, CcuVariableHandle varAHandle, CcuVariableHandle varBHandle)
576 : {
577 2 : CcuRep::Variable *resVar{nullptr}, *leftVar{nullptr}, *rightVar{nullptr};
578 2 : CCU_CHK_RET(GetVariableByHandle(varHandle, &resVar));
579 2 : CCU_CHK_RET(GetVariableByHandle(varAHandle, &leftVar));
580 2 : CCU_CHK_RET(GetVariableByHandle(varBHandle, &rightVar));
581 :
582 2 : *resVar = *leftVar - *rightVar;
583 2 : return CcuResult::CCU_SUCCESS;
584 : }
585 :
586 2 : CcuResult CcuKernel::VariableMulVarToVar(CcuVariableHandle varHandle, CcuVariableHandle varAHandle, CcuVariableHandle varBHandle)
587 : {
588 2 : CcuRep::Variable *resVar{nullptr}, *leftVar{nullptr}, *rightVar{nullptr};
589 2 : CCU_CHK_RET(GetVariableByHandle(varHandle, &resVar));
590 2 : CCU_CHK_RET(GetVariableByHandle(varAHandle, &leftVar));
591 2 : CCU_CHK_RET(GetVariableByHandle(varBHandle, &rightVar));
592 :
593 2 : *resVar = *leftVar * *rightVar;
594 2 : return CcuResult::CCU_SUCCESS;
595 : }
596 :
597 1 : CcuResult CcuKernel::VariableAddImmToVar(CcuVariableHandle varHandle, CcuVariableHandle varAHandle, uint16_t immediate)
598 : {
599 1 : CcuRep::Variable *resVar{nullptr}, *leftVar{nullptr};
600 1 : CCU_CHK_RET(GetVariableByHandle(varHandle, &resVar));
601 1 : CCU_CHK_RET(GetVariableByHandle(varAHandle, &leftVar));
602 :
603 1 : *resVar = *leftVar + immediate;
604 1 : return CcuResult::CCU_SUCCESS;
605 : }
606 :
607 2 : CcuResult CcuKernel::VariableSubImmToVar(CcuVariableHandle varHandle, CcuVariableHandle varAHandle, uint16_t immediate)
608 : {
609 2 : CcuRep::Variable *resVar{nullptr}, *leftVar{nullptr};
610 2 : CCU_CHK_RET(GetVariableByHandle(varHandle, &resVar));
611 2 : CCU_CHK_RET(GetVariableByHandle(varAHandle, &leftVar));
612 :
613 2 : *resVar = *leftVar - immediate;
614 2 : return CcuResult::CCU_SUCCESS;
615 : }
616 :
617 2 : CcuResult CcuKernel::VariableMulImmToVar(CcuVariableHandle varHandle, CcuVariableHandle varAHandle, uint16_t immediate)
618 : {
619 2 : CcuRep::Variable *resVar{nullptr}, *leftVar{nullptr};
620 2 : CCU_CHK_RET(GetVariableByHandle(varHandle, &resVar));
621 2 : CCU_CHK_RET(GetVariableByHandle(varAHandle, &leftVar));
622 :
623 2 : *resVar = *leftVar * immediate;
624 2 : return CcuResult::CCU_SUCCESS;
625 : }
626 :
627 2 : CcuResult CcuKernel::VariableAndVarToVar(CcuVariableHandle varHandle, CcuVariableHandle varAHandle, CcuVariableHandle varBHandle)
628 : {
629 2 : CcuRep::Variable *resVar{nullptr}, *leftVar{nullptr}, *rightVar{nullptr};
630 2 : CCU_CHK_RET(GetVariableByHandle(varHandle, &resVar));
631 2 : CCU_CHK_RET(GetVariableByHandle(varAHandle, &leftVar));
632 2 : CCU_CHK_RET(GetVariableByHandle(varBHandle, &rightVar));
633 :
634 2 : *resVar = *leftVar & *rightVar;
635 2 : return CcuResult::CCU_SUCCESS;
636 : }
637 :
638 2 : CcuResult CcuKernel::VariableOrVarToVar(CcuVariableHandle varHandle, CcuVariableHandle varAHandle, CcuVariableHandle varBHandle)
639 : {
640 2 : CcuRep::Variable *resVar{nullptr}, *leftVar{nullptr}, *rightVar{nullptr};
641 2 : CCU_CHK_RET(GetVariableByHandle(varHandle, &resVar));
642 2 : CCU_CHK_RET(GetVariableByHandle(varAHandle, &leftVar));
643 2 : CCU_CHK_RET(GetVariableByHandle(varBHandle, &rightVar));
644 :
645 2 : *resVar = *leftVar | *rightVar;
646 2 : return CcuResult::CCU_SUCCESS;
647 : }
648 :
649 2 : CcuResult CcuKernel::VariableXorVarToVar(CcuVariableHandle varHandle, CcuVariableHandle varAHandle, CcuVariableHandle varBHandle)
650 : {
651 2 : CcuRep::Variable *resVar{nullptr}, *leftVar{nullptr}, *rightVar{nullptr};
652 2 : CCU_CHK_RET(GetVariableByHandle(varHandle, &resVar));
653 2 : CCU_CHK_RET(GetVariableByHandle(varAHandle, &leftVar));
654 2 : CCU_CHK_RET(GetVariableByHandle(varBHandle, &rightVar));
655 :
656 2 : *resVar = *leftVar ^ *rightVar;
657 2 : return CcuResult::CCU_SUCCESS;
658 : }
659 :
660 1 : CcuResult CcuKernel::VariableNotVar(CcuVariableHandle varHandle, CcuVariableHandle varAHandle)
661 : {
662 1 : CcuRep::Variable *resVar{nullptr}, *leftVar{nullptr};
663 1 : CCU_CHK_RET(GetVariableByHandle(varHandle, &resVar));
664 1 : CCU_CHK_RET(GetVariableByHandle(varAHandle, &leftVar));
665 :
666 1 : *resVar = ~(*leftVar);
667 1 : return CcuResult::CCU_SUCCESS;
668 : }
669 :
670 : /*========== Event信号同步类 相关接口 ==========*/
671 11 : CcuResult CcuKernel::EventRecord(CcuEventHandle eventHandle, uint32_t mask)
672 : {
673 11 : PLF_CONFIG_INFO(PLF_DATA_OP, "[EventRecord] eventHandle=%llu, mask=%u", eventHandle, mask);
674 11 : CcuRep::CompletedEvent *event{nullptr};
675 11 : CCU_CHK_RET(GetEventByHandle(eventHandle, &event));
676 : // 复用已有的 RecordEvent 实现（内部 Append CcuRepLocRecordEvent）
677 11 : CCU_CHK_RET(RecordEvent(*event, mask));
678 10 : return CcuResult::CCU_SUCCESS;
679 : }
680 :
681 32 : CcuResult CcuKernel::EventWait(CcuEventHandle eventHandle, uint32_t mask)
682 : {
683 32 : PLF_CONFIG_INFO(PLF_DATA_OP, "[EventWait] eventHandle=%llu, mask=%u", eventHandle, mask);
684 32 : CcuRep::CompletedEvent *event{nullptr};
685 32 : CCU_CHK_RET(GetEventByHandle(eventHandle, &event));
686 : // 复用已有的 WaitEvent 实现（内部 Append CcuRepLocWaitEvent）
687 32 : CCU_CHK_RET(WaitEvent(*event, mask));
688 32 : return CcuResult::CCU_SUCCESS;
689 : }
690 :
691 1 : CcuResult CcuKernel::LocalNotifyRecord(const char *notifyTag, const uint32_t mask)
692 : {
693 1 : PLF_CONFIG_INFO(PLF_DATA_OP, "[LocalNotifyRecord] tag=%s, mask=%u", (notifyTag ? notifyTag : "null"), mask);
694 1 : if (notifyTag == nullptr) {
695 0 : HCCL_ERROR("[CcuKernel][%s] notifyTag is nullptr, please check.", __func__);
696 0 : return CcuResult::CCU_E_PTR;
697 : }
698 1 : if (CurrentBlock()->Type() == CcuRep::CcuRepType::LOOP_BLOCK) {
699 1 : HCCL_ERROR("[CcuKernel][%s] is not supported in loop block, please check.", __func__);
700 1 : return LatchBodyError(CcuResult::CCU_E_NOT_SUPPORT);
701 : }
702 :
703 0 : const std::string tagKey(notifyTag);
704 :
705 0 : auto &sharedNotifies = importedRes_.sharedNotifies;
706 0 : if (sharedNotifies.find(tagKey) == sharedNotifies.end()) {
707 0 : CcuRep::LocalNotify localNotify;
708 0 : sharedNotifies.insert({tagKey, localNotify});
709 0 : }
710 :
711 0 : Append(std::make_shared<CcuRep::CcuRepRecordSharedNotify>(insGenerator, sharedNotifies.at(tagKey), mask));
712 :
713 0 : return CcuResult::CCU_SUCCESS;
714 0 : }
715 :
716 0 : CcuResult CcuKernel::LocalNotifyWait(const char *notifyTag, const uint32_t mask)
717 : {
718 0 : PLF_CONFIG_INFO(PLF_DATA_OP, "[LocalNotifyWait] tag=%s, mask=%u", (notifyTag ? notifyTag : "null"), mask);
719 0 : if (notifyTag == nullptr) {
720 0 : HCCL_ERROR("[CcuKernel][%s] notifyTag is nullptr, please check.", __func__);
721 0 : return CcuResult::CCU_E_PTR;
722 : }
723 :
724 0 : const std::string tagKey(notifyTag);
725 :
726 0 : auto &sharedNotifies = exportedRes_.sharedNotifies;
727 0 : if (sharedNotifies.find(tagKey) == sharedNotifies.end()) {
728 0 : CcuRep::LocalNotify notify = CreateLocalNotify();
729 0 : exportedRes_.sharedNotifies.insert({tagKey, notify});
730 0 : }
731 :
732 0 : bool isProfiling = CurrentBlock()->Type() != CcuRep::CcuRepType::LOOP_BLOCK;
733 0 : Append(std::make_shared<CcuRep::CcuRepLocWaitNotify>(
734 0 : insGenerator, exportedRes_.sharedNotifies.at(tagKey), mask, isProfiling));
735 0 : return CcuResult::CCU_SUCCESS;
736 0 : }
737 :
738 6 : CcuResult CcuKernel::NotifyRecord(const ChannelHandle channel,
739 : uint32_t remoteNotifyIdx, uint32_t mask)
740 : {
741 6 : PLF_CONFIG_INFO(PLF_DATA_OP, "[NotifyRecord] channel=%llu, remoteNotifyIdx=%u, mask=%u",
742 : channel, remoteNotifyIdx, mask);
743 6 : if (CurrentBlock()->Type() == CcuRep::CcuRepType::LOOP_BLOCK) {
744 1 : HCCL_ERROR("[%s] NotifyRecord is not allowed inside a ccu::Loop body", __func__);
745 1 : return LatchBodyError(CcuResult::CCU_E_NOT_SUPPORT);
746 : }
747 5 : channels_.insert(channel);
748 5 : Append(std::make_shared<CcuRep::CcuRepRemPostSem>(insGenerator, channel, remoteNotifyIdx, mask));
749 5 : return CCU_SUCCESS;
750 : }
751 :
752 11 : CcuResult CcuKernel::NotifyWait(const ChannelHandle channel, uint32_t localNotifyIdx, uint32_t mask)
753 : {
754 11 : PLF_CONFIG_INFO(PLF_DATA_OP, "[NotifyWait] channel=%llu, localNotifyIdx=%u, mask=%u",
755 : channel, localNotifyIdx, mask);
756 11 : channels_.insert(channel);
757 11 : bool isProfiling = CurrentBlock()->Type() != CcuRep::CcuRepType::LOOP_BLOCK;
758 11 : if (isProfiling) {
759 33 : CCU_CHK_RET(static_cast<HcclResult>(AddProfiling(channel, "NotifyWait", localNotifyIdx, mask)));
760 : }
761 10 : Append(std::make_shared<CcuRep::CcuRepRemWaitSem>(insGenerator, channel, localNotifyIdx, mask, isProfiling));
762 10 : return CcuResult::CCU_SUCCESS;
763 : }
764 :
765 10 : CcuResult CcuKernel::WriteVariableWithNotify(const ChannelHandle channel, CcuVariableHandle varHandle,
766 : uint32_t remoteVarIdx, uint32_t remoteNotifyIdx, uint32_t mask)
767 : {
768 10 : PLF_CONFIG_INFO(PLF_DATA_OP, "[WriteVariableWithNotify] channel=%llu, varHandle=%llu, remoteVarIdx=%u,"
769 : " remoteNotifyIdx=%u, mask=%u", channel, varHandle, remoteVarIdx, remoteNotifyIdx, mask);
770 10 : if (CurrentBlock()->Type() == CcuRep::CcuRepType::LOOP_BLOCK) {
771 1 : HCCL_ERROR("[%s] WriteVariableWithNotify is not allowed inside a ccu::Loop body", __func__);
772 1 : return LatchBodyError(CcuResult::CCU_E_NOT_SUPPORT);
773 : }
774 9 : channels_.insert(channel);
775 9 : CcuRep::Variable *var{nullptr};
776 9 : CCU_CHK_RET(GetVariableByHandle(varHandle, &var));
777 9 : Append(std::make_shared<CcuRep::CcuRepRemPostVar>(insGenerator, *var, channel, remoteVarIdx, remoteNotifyIdx, mask));
778 9 : return CcuResult::CCU_SUCCESS;
779 : }
780 :
781 :
782 : //加载类 相关接口
783 49 : CcuResult CcuKernel::LoadArg(CcuVariableHandle varHandle, uint32_t argId)
784 : {
785 49 : PLF_CONFIG_INFO(PLF_DATA_OP, "[LoadArg] varHandle=%llu, argId=%u", varHandle, argId);
786 49 : loadArgUsedSet_.insert(argId);
787 49 : CcuRep::Variable *var{nullptr};
788 49 : CCU_CHK_RET(GetVariableByHandle(varHandle,&var));
789 : auto loadArgRep = std::make_shared<CcuRep::CcuRepLoadArg>(
790 49 : insGenerator, *var, argId % CCU_SQE_ARGS_LEN, static_cast<uint16_t>(argId));
791 49 : Append(loadArgRep);
792 49 : return CcuResult::CCU_SUCCESS;
793 49 : }
794 :
795 13 : CcuResult CcuKernel::CheckContinuousVariables(CcuVariableHandle varHandle, uint32_t num,
796 : const CcuRep::Variable &baseVar, const char *tag)
797 : {
798 13 : if (num <= 1) {
799 4 : return CcuResult::CCU_SUCCESS;
800 : }
801 18 : for (uint32_t i = 1; i < num; i++) {
802 9 : CcuRep::Variable *nextVar{nullptr};
803 9 : CCU_CHK_RET(GetVariableByHandle(varHandle + i, &nextVar));
804 9 : if (nextVar->Id() != baseVar.Id() + i) {
805 0 : HCCL_ERROR("[CcuKernel][%s] variables not continuous at index %u, "
806 : "expected Id %u but got %u", tag, i, baseVar.Id() + i, nextVar->Id());
807 0 : return HCCL_TO_CCU_RET(HCCL_E_PARA);
808 : }
809 : }
810 9 : return CcuResult::CCU_SUCCESS;
811 : }
812 :
813 6 : CcuResult CcuKernel::LoadVar(uint64_t addr, CcuVariableHandle varHandle, uint32_t num)
814 : {
815 6 : PLF_CONFIG_INFO(PLF_DATA_OP, "[LoadVar] addr=0x%llx, varHandle=%llu, num=%u", addr, varHandle, num);
816 6 : CcuRep::Variable *var{nullptr};
817 6 : CCU_CHK_RET(GetVariableByHandle(varHandle, &var));
818 6 : CCU_CHK_RET(CheckContinuousVariables(varHandle, num, *var, "LoadVariable"));
819 6 : Append(std::make_shared<CcuRep::CcuRepLoad>(insGenerator, addr, *var, num));
820 6 : return CcuResult::CCU_SUCCESS;
821 : }
822 :
823 2 : CcuResult CcuKernel::CcuLoadVarFromVarAddr(CcuVariableHandle addrHandle, CcuVariableHandle varHandle, uint32_t num)
824 : {
825 2 : PLF_CONFIG_INFO(PLF_DATA_OP, "[CcuLoadVarFromVarAddr] addrHandle=%llu, varHandle=%llu,"
826 : " num=%u", addrHandle, varHandle, num);
827 2 : CcuRep::Variable *addrVar{nullptr};
828 2 : CCU_CHK_RET(GetVariableByHandle(addrHandle, &addrVar));
829 2 : CcuRep::Variable *var{nullptr};
830 2 : CCU_CHK_RET(GetVariableByHandle(varHandle, &var));
831 2 : CCU_CHK_RET(CheckContinuousVariables(varHandle, num, *var, "LoadVar dst"));
832 2 : Append(std::make_shared<CcuRep::CcuRepLoadVar>(insGenerator, *addrVar, *var, num));
833 2 : return CcuResult::CCU_SUCCESS;
834 : }
835 :
836 3 : CcuResult CcuKernel::StoreVar(uint64_t addr, CcuVariableHandle varHandle, uint32_t num)
837 : {
838 3 : PLF_CONFIG_INFO(PLF_DATA_OP, "[StoreVar] addr=0x%llx, varHandle=%llu, num=%u", addr, varHandle, num);
839 3 : CcuRep::Variable *var{nullptr};
840 3 : CCU_CHK_RET(GetVariableByHandle(varHandle, &var));
841 3 : CCU_CHK_RET(CheckContinuousVariables(varHandle, num, *var, "StoreVariable"));
842 3 : Append(std::make_shared<CcuRep::CcuRepStore>(insGenerator, *var, addr, num));
843 3 : return CcuResult::CCU_SUCCESS;
844 : }
845 :
846 2 : CcuResult CcuKernel::CcuStoreVarToVarAddr(CcuVariableHandle addrHandle, CcuVariableHandle varHandle, uint32_t num)
847 : {
848 2 : PLF_CONFIG_INFO(PLF_DATA_OP, "[CcuStoreVarToVarAddr] addrHandle=%llu, varHandle=%llu,"
849 : " num=%u", addrHandle, varHandle, num);
850 2 : CcuRep::Variable *addrVar{nullptr};
851 2 : CCU_CHK_RET(GetVariableByHandle(addrHandle, &addrVar));
852 2 : CcuRep::Variable *var{nullptr};
853 2 : CCU_CHK_RET(GetVariableByHandle(varHandle, &var));
854 2 : CCU_CHK_RET(CheckContinuousVariables(varHandle, num, *var, "StoreVar src"));
855 2 : Append(std::make_shared<CcuRep::CcuRepStoreVar>(insGenerator, *var, *addrVar, num));
856 2 : return CcuResult::CCU_SUCCESS;
857 : }
858 :
859 : //本地数据拷贝 相关实现
860 10 : CcuResult CcuKernel::LocalCopyMemToBuffer(CcuBufferHandle dstHandle, CcuLocalAddrHandle srcHandle,
861 : CcuVariableHandle lenHandle, CcuEventHandle eventHandle, uint32_t mask)
862 : {
863 10 : PLF_CONFIG_INFO(PLF_DATA_OP, "[LocalCopyMemToBuffer] dstHandle=%llu, srcHandle=%llu, lenHandle=%llu,"
864 : " eventHandle=%llu, mask=%u", dstHandle, srcHandle, lenHandle, eventHandle, mask);
865 10 : CcuRep::CcuBuf *dst{nullptr};
866 10 : CCU_CHK_RET(GetBufferByHandle(dstHandle, &dst));
867 10 : CcuRep::LocalAddr *src{nullptr};
868 10 : CCU_CHK_RET(GetLocalAddrByHandle(srcHandle, &src));
869 10 : CcuRep::Variable *len{nullptr};
870 10 : CCU_CHK_RET(GetVariableByHandle(lenHandle, &len));
871 10 : CcuRep::CompletedEvent *event{nullptr};
872 10 : CCU_CHK_RET(GetEventByHandle(eventHandle, &event));
873 10 : auto ret = LocalCopyNb(*dst, *src, *len, *event, mask); // 复用 protected
874 10 : return HCCL_TO_CCU_RET(ret);
875 : }
876 :
877 6 : CcuResult CcuKernel::LocalCopyBufferToMem(CcuLocalAddrHandle dstHandle, CcuBufferHandle srcHandle,
878 : CcuVariableHandle lenHandle, CcuEventHandle eventHandle, uint32_t mask)
879 : {
880 6 : PLF_CONFIG_INFO(PLF_DATA_OP, "[LocalCopyBufferToMem] dstHandle=%llu, srcHandle=%llu, lenHandle=%llu,"
881 : " eventHandle=%llu, mask=%u", dstHandle, srcHandle, lenHandle, eventHandle, mask);
882 6 : CcuRep::LocalAddr *dst{nullptr};
883 6 : CCU_CHK_RET(GetLocalAddrByHandle(dstHandle, &dst));
884 6 : CcuRep::CcuBuf *src{nullptr};
885 6 : CCU_CHK_RET(GetBufferByHandle(srcHandle, &src));
886 6 : CcuRep::Variable *len{nullptr};
887 6 : CCU_CHK_RET(GetVariableByHandle(lenHandle, &len));
888 6 : CcuRep::CompletedEvent *event{nullptr};
889 6 : CCU_CHK_RET(GetEventByHandle(eventHandle, &event));
890 6 : auto ret = LocalCopyNb(*dst, *src, *len, *event, mask);
891 6 : return HCCL_TO_CCU_RET(ret);
892 : }
893 :
894 0 : CcuResult CcuKernel::LocalCopyMemToMem(CcuLocalAddrHandle dstHandle, CcuLocalAddrHandle srcHandle,
895 : CcuVariableHandle lenHandle, CcuEventHandle eventHandle, uint32_t mask)
896 : {
897 0 : PLF_CONFIG_INFO(PLF_DATA_OP, "[LocalCopyMemToMem] dstHandle=%llu, srcHandle=%llu, lenHandle=%llu,"
898 : " eventHandle=%llu, mask=%u", dstHandle, srcHandle, lenHandle, eventHandle, mask);
899 0 : CcuRep::LocalAddr *dst{nullptr};
900 0 : CCU_CHK_RET(GetLocalAddrByHandle(dstHandle, &dst));
901 0 : CcuRep::LocalAddr *src{nullptr};
902 0 : CCU_CHK_RET(GetLocalAddrByHandle(srcHandle, &src));
903 0 : CcuRep::Variable *len{nullptr};
904 0 : CCU_CHK_RET(GetVariableByHandle(lenHandle, &len));
905 0 : CcuRep::CompletedEvent *event{nullptr};
906 0 : CCU_CHK_RET(GetEventByHandle(eventHandle, &event));
907 0 : auto ret = LocalCopyNb(*dst, *src, *len, *event, mask);
908 0 : return HCCL_TO_CCU_RET(ret);
909 : }
910 :
911 : //本地reduce 相关实现
912 0 : CcuResult CcuKernel::LocalMemReduce(CcuLocalAddrHandle dstHandle, CcuLocalAddrHandle srcHandle,
913 : CcuVariableHandle lenHandle, HcclDataType dataType,
914 : HcclReduceOp opType, CcuEventHandle eventHandle, uint32_t mask)
915 : {
916 0 : PLF_CONFIG_INFO(PLF_DATA_OP, "[LocalMemReduce] dstHandle=%llu, srcHandle=%llu, lenHandle=%llu, dataType=%d,"
917 : " op=%d, eventHandle=%llu, mask=%u",
918 : dstHandle, srcHandle, lenHandle, dataType, opType, eventHandle, mask);
919 0 : CcuRep::LocalAddr *dst{nullptr};
920 0 : CCU_CHK_RET(GetLocalAddrByHandle(dstHandle, &dst));
921 0 : CcuRep::LocalAddr *src{nullptr};
922 0 : CCU_CHK_RET(GetLocalAddrByHandle(srcHandle, &src));
923 0 : CcuRep::Variable *len{nullptr};
924 0 : CCU_CHK_RET(GetVariableByHandle(lenHandle, &len));
925 0 : CcuRep::CompletedEvent *event{nullptr};
926 0 : CCU_CHK_RET(GetEventByHandle(eventHandle, &event));
927 0 : auto ret = LocalReduceNb(*dst, *src, *len, dataType, opType, *event, mask);
928 0 : return HCCL_TO_CCU_RET(ret);
929 : }
930 :
931 4 : CcuResult CcuKernel::LocalBufferReduce(CcuBufferHandle* bufHandles, uint32_t count,
932 : HcclDataType dataType, HcclDataType outputDataType,
933 : HcclReduceOp opType, CcuVariableHandle lenHandle, CcuEventHandle eventHandle, uint32_t mask)
934 : {
935 4 : PLF_CONFIG_INFO(PLF_DATA_OP, "[LocalBufferReduce] count=%u, dataType=%d, outDataType=%d,"
936 : " op=%d, lenHandle=%llu, eventHandle=%llu, mask=%u",
937 : count, dataType, outputDataType, opType, lenHandle, eventHandle, mask);
938 4 : CcuRep::Variable *len{nullptr};
939 4 : CCU_CHK_RET(GetVariableByHandle(lenHandle, &len));
940 4 : CcuRep::CompletedEvent *event{nullptr};
941 4 : CCU_CHK_RET(GetEventByHandle(eventHandle, &event));
942 4 : std::vector<CcuRep::CcuBuf> bufs(count);
943 12 : for (uint32_t i = 0; i < count; i++) {
944 8 : CcuRep::CcuBuf *buf{nullptr};
945 8 : CCU_CHK_RET(GetBufferByHandle(bufHandles[i], &buf));
946 8 : bufs[i] = *buf;
947 : }
948 4 : auto ret = LocalReduceNb(bufs.data(), count, dataType, outputDataType, opType, *len, *event, mask);
949 4 : return HCCL_TO_CCU_RET(ret);
950 4 : }
951 :
952 : /*========== 远端数据传输操作 ==========*/
953 :
954 4 : CcuResult CcuKernel::ResolveBufRemoteLenEvent(CcuBufferHandle bufHandle, CcuRemoteAddrHandle remoteHandle,
955 : CcuVariableHandle lenHandle, CcuEventHandle eventHandle,
956 : CcuRep::CcuBuf **buf, CcuRep::RemoteAddr **remote,
957 : CcuRep::Variable **len, CcuRep::CompletedEvent **event)
958 : {
959 4 : CCU_CHK_RET(GetBufferByHandle(bufHandle, buf));
960 4 : CCU_CHK_RET(GetRemoteAddrByHandle(remoteHandle, remote));
961 4 : CCU_CHK_RET(GetVariableByHandle(lenHandle, len));
962 4 : CCU_CHK_RET(GetEventByHandle(eventHandle, event));
963 4 : return CcuResult::CCU_SUCCESS;
964 : }
965 :
966 6 : CcuResult CcuKernel::ResolveLocalRemoteLenEvent(CcuLocalAddrHandle localHandle, CcuRemoteAddrHandle remoteHandle,
967 : CcuVariableHandle lenHandle, CcuEventHandle eventHandle,
968 : CcuRep::LocalAddr **local, CcuRep::RemoteAddr **remote,
969 : CcuRep::Variable **len, CcuRep::CompletedEvent **event)
970 : {
971 6 : CCU_CHK_RET(GetLocalAddrByHandle(localHandle, local));
972 6 : CCU_CHK_RET(GetRemoteAddrByHandle(remoteHandle, remote));
973 6 : CCU_CHK_RET(GetVariableByHandle(lenHandle, len));
974 6 : CCU_CHK_RET(GetEventByHandle(eventHandle, event));
975 6 : return CcuResult::CCU_SUCCESS;
976 : }
977 :
978 5 : CcuResult CcuKernel::ResolveRemoteLocalLenEvent(CcuRemoteAddrHandle remoteHandle, CcuLocalAddrHandle localHandle,
979 : CcuVariableHandle lenHandle, CcuEventHandle eventHandle,
980 : CcuRep::RemoteAddr **remote, CcuRep::LocalAddr **local,
981 : CcuRep::Variable **len, CcuRep::CompletedEvent **event)
982 : {
983 5 : CCU_CHK_RET(GetRemoteAddrByHandle(remoteHandle, remote));
984 5 : CCU_CHK_RET(GetLocalAddrByHandle(localHandle, local));
985 5 : CCU_CHK_RET(GetVariableByHandle(lenHandle, len));
986 5 : CCU_CHK_RET(GetEventByHandle(eventHandle, event));
987 5 : return CcuResult::CCU_SUCCESS;
988 : }
989 :
990 4 : CcuResult CcuKernel::ReadMemToMem(ChannelHandle channel, CcuLocalAddrHandle localHandle, CcuRemoteAddrHandle remoteHandle,
991 : CcuVariableHandle lenHandle, CcuEventHandle eventHandle, uint32_t mask)
992 : {
993 4 : PLF_CONFIG_INFO(PLF_DATA_OP, "[ReadMemToMem] channel=%llu, localHandle=%llu, remoteHandle=%llu,"
994 : " lenHandle=%llu, eventHandle=%llu, mask=%u",
995 : channel, localHandle, remoteHandle, lenHandle, eventHandle, mask);
996 4 : channels_.insert(channel);
997 4 : CcuRep::LocalAddr *local{nullptr};
998 4 : CcuRep::RemoteAddr *remote{nullptr};
999 4 : CcuRep::Variable *len{nullptr};
1000 4 : CcuRep::CompletedEvent *event{nullptr};
1001 4 : CCU_CHK_RET(ResolveLocalRemoteLenEvent(localHandle, remoteHandle, lenHandle, eventHandle,
1002 : &local, &remote, &len, &event));
1003 4 : auto ret = ReadNb(channel, *local, *remote, *len, *event, mask);
1004 4 : return HCCL_TO_CCU_RET(ret);
1005 : }
1006 :
1007 2 : CcuResult CcuKernel::ReadMemToBuffer(ChannelHandle channel, CcuBufferHandle localHandle, CcuRemoteAddrHandle remoteHandle,
1008 : CcuVariableHandle lenHandle, CcuEventHandle eventHandle, uint32_t mask)
1009 : {
1010 2 : PLF_CONFIG_INFO(PLF_DATA_OP, "[ReadMemToBuffer] channel=%llu, bufHandle=%llu, remoteHandle=%llu,"
1011 : " lenHandle=%llu, eventHandle=%llu, mask=%u",
1012 : channel, localHandle, remoteHandle, lenHandle, eventHandle, mask);
1013 2 : channels_.insert(channel);
1014 2 : CcuRep::CcuBuf *local{nullptr};
1015 2 : CcuRep::RemoteAddr *remote{nullptr};
1016 2 : CcuRep::Variable *len{nullptr};
1017 2 : CcuRep::CompletedEvent *event{nullptr};
1018 2 : CCU_CHK_RET(ResolveBufRemoteLenEvent(localHandle, remoteHandle, lenHandle, eventHandle,
1019 : &local, &remote, &len, &event));
1020 2 : auto ret = ReadNb(channel, *local, *remote, *len, *event, mask);
1021 2 : return HCCL_TO_CCU_RET(ret);
1022 : }
1023 :
1024 2 : CcuResult CcuKernel::ReadMemToMemReduce(ChannelHandle channel, CcuLocalAddrHandle localHandle, CcuRemoteAddrHandle remoteHandle,
1025 : CcuVariableHandle lenHandle, HcclDataType dataType,
1026 : HcclReduceOp opType, CcuEventHandle eventHandle, uint32_t mask)
1027 : {
1028 2 : PLF_CONFIG_INFO(PLF_DATA_OP, "[ReadMemToMemReduce] channel=%llu, lenHandle=%llu, dataType=%d, op=%d,"
1029 : " eventHandle=%llu, mask=%u", channel, lenHandle, dataType, opType, eventHandle, mask);
1030 2 : channels_.insert(channel);
1031 2 : CcuRep::LocalAddr *local{nullptr};
1032 2 : CcuRep::RemoteAddr *remote{nullptr};
1033 2 : CcuRep::Variable *len{nullptr};
1034 2 : CcuRep::CompletedEvent *event{nullptr};
1035 2 : CCU_CHK_RET(ResolveLocalRemoteLenEvent(localHandle, remoteHandle, lenHandle, eventHandle,
1036 : &local, &remote, &len, &event));
1037 2 : auto ret = ReadReduceNb(channel, *local, *remote, *len, dataType, opType, *event, mask);
1038 2 : return HCCL_TO_CCU_RET(ret);
1039 : }
1040 :
1041 3 : CcuResult CcuKernel::WriteMemToMem(ChannelHandle channel, CcuRemoteAddrHandle remoteHandle, CcuLocalAddrHandle localHandle,
1042 : CcuVariableHandle lenHandle, CcuEventHandle eventHandle, uint32_t mask)
1043 : {
1044 3 : PLF_CONFIG_INFO(PLF_DATA_OP, "[WriteMemToMem] channel=%llu, remoteHandle=%llu, localHandle=%llu,"
1045 : " lenHandle=%llu, eventHandle=%llu, mask=%u",
1046 : channel, remoteHandle, localHandle, lenHandle, eventHandle, mask);
1047 3 : channels_.insert(channel);
1048 3 : CcuRep::RemoteAddr *remote{nullptr};
1049 3 : CcuRep::LocalAddr *local{nullptr};
1050 3 : CcuRep::Variable *len{nullptr};
1051 3 : CcuRep::CompletedEvent *event{nullptr};
1052 3 : CCU_CHK_RET(ResolveRemoteLocalLenEvent(remoteHandle, localHandle, lenHandle, eventHandle,
1053 : &remote, &local, &len, &event));
1054 3 : auto ret = WriteNb(channel, *remote, *local, *len, *event, mask);
1055 3 : return HCCL_TO_CCU_RET(ret);
1056 : }
1057 :
1058 2 : CcuResult CcuKernel::WriteBufferToMem(ChannelHandle channel, CcuRemoteAddrHandle remoteHandle, CcuBufferHandle localHandle,
1059 : CcuVariableHandle lenHandle, CcuEventHandle eventHandle, uint32_t mask)
1060 : {
1061 2 : PLF_CONFIG_INFO(PLF_DATA_OP, "[WriteBufferToMem] channel=%llu, remoteHandle=%llu, bufHandle=%llu,"
1062 : " lenHandle=%llu, eventHandle=%llu, mask=%u",
1063 : channel, remoteHandle, localHandle, lenHandle, eventHandle, mask);
1064 2 : channels_.insert(channel);
1065 2 : CcuRep::CcuBuf *local{nullptr};
1066 2 : CcuRep::RemoteAddr *remote{nullptr};
1067 2 : CcuRep::Variable *len{nullptr};
1068 2 : CcuRep::CompletedEvent *event{nullptr};
1069 2 : CCU_CHK_RET(ResolveBufRemoteLenEvent(localHandle, remoteHandle, lenHandle, eventHandle,
1070 : &local, &remote, &len, &event));
1071 2 : auto ret = WriteNb(channel, *remote, *local, *len, *event, mask);
1072 2 : return HCCL_TO_CCU_RET(ret);
1073 : }
1074 :
1075 2 : CcuResult CcuKernel::WriteMemToMemReduce(ChannelHandle channel, CcuRemoteAddrHandle remoteHandle, CcuLocalAddrHandle localHandle,
1076 : CcuVariableHandle lenHandle, HcclDataType dataType,
1077 : HcclReduceOp opType, CcuEventHandle eventHandle, uint32_t mask)
1078 : {
1079 2 : PLF_CONFIG_INFO(PLF_DATA_OP, "[WriteMemToMemReduce] channel=%llu, lenHandle=%llu, dataType=%d, op=%d,"
1080 : " eventHandle=%llu, mask=%u", channel, lenHandle, dataType, opType, eventHandle, mask);
1081 2 : channels_.insert(channel);
1082 2 : CcuRep::RemoteAddr *remote{nullptr};
1083 2 : CcuRep::LocalAddr *local{nullptr};
1084 2 : CcuRep::Variable *len{nullptr};
1085 2 : CcuRep::CompletedEvent *event{nullptr};
1086 2 : CCU_CHK_RET(ResolveRemoteLocalLenEvent(remoteHandle, localHandle, lenHandle, eventHandle,
1087 : &remote, &local, &len, &event));
1088 2 : auto ret = WriteReduceNb(channel, *remote, *local, *len, dataType, opType, *event, mask);
1089 2 : return HCCL_TO_CCU_RET(ret);
1090 : }
1091 :
1092 970 : void CcuKernel::FlushClosablePendingIfs()
1093 : {
1094 970 : if (isFlushing_) {
1095 28 : return;
1096 : }
1097 942 : isFlushing_ = true;
1098 1912 : while (IfLabelStackTopIsClosable()) {
1099 28 : const char *lbl = IfLabelStackPop();
1100 28 : if (lbl != nullptr) {
1101 28 : IfEnd(lbl);
1102 : }
1103 : }
1104 942 : isFlushing_ = false;
1105 : }
1106 :
1107 860 : void CcuKernel::Append(std::shared_ptr<CcuRep::CcuRepBase> rep)
1108 : {
1109 860 : FlushClosablePendingIfs();
1110 860 : CcuRep::CcuRepContext::Append(rep);
1111 860 : }
1112 :
1113 : namespace {
1114 32 : std::shared_ptr<CcuRep::CcuRepJumpBase> MakeInvertedCondJumpImm(
1115 : CcuInsGeneratorBase *insGenerator, const std::string &destLabelStr,
1116 : const CcuRep::Variable &targetVar, const CcuRep::Variable &expectVar,
1117 : const CcuRep::Variable &variable, uint64_t immediate,
1118 : CcuConditionType condType, const char *funcName)
1119 : {
1120 32 : switch (condType) {
1121 10 : case CCU_CONDITION_EQ:
1122 20 : return std::make_shared<CcuRep::CcuRepJumpNE>(
1123 10 : insGenerator, destLabelStr, targetVar, expectVar, variable, immediate);
1124 12 : case CCU_CONDITION_NE:
1125 24 : return std::make_shared<CcuRep::CcuRepJumpEQ>(
1126 12 : insGenerator, destLabelStr, targetVar, expectVar, variable, immediate);
1127 3 : case CCU_CONDITION_LT:
1128 6 : return std::make_shared<CcuRep::CcuRepJumpGE>(
1129 3 : insGenerator, destLabelStr, targetVar, expectVar, variable, immediate);
1130 2 : case CCU_CONDITION_LE:
1131 4 : return std::make_shared<CcuRep::CcuRepJumpGT>(
1132 2 : insGenerator, destLabelStr, targetVar, expectVar, variable, immediate);
1133 2 : case CCU_CONDITION_GT:
1134 4 : return std::make_shared<CcuRep::CcuRepJumpLE>(
1135 2 : insGenerator, destLabelStr, targetVar, expectVar, variable, immediate);
1136 3 : case CCU_CONDITION_GE:
1137 6 : return std::make_shared<CcuRep::CcuRepJumpLT>(
1138 3 : insGenerator, destLabelStr, targetVar, expectVar, variable, immediate);
1139 0 : default:
1140 0 : HCCL_ERROR("[%s] unsupported condition type: %d", funcName, condType);
1141 0 : return nullptr;
1142 : }
1143 : }
1144 :
1145 : // 双变量版本：当 (lhsVar OP rhsVar) 为假时跳转到 destLabelStr。
1146 4 : std::shared_ptr<CcuRep::CcuRepJumpBase> MakeInvertedCondJumpVar(
1147 : CcuInsGeneratorBase *insGenerator, const std::string &destLabelStr,
1148 : const CcuRep::Variable &targetVar,
1149 : const CcuRep::Variable &lhsVar, const CcuRep::Variable &rhsVar,
1150 : CcuConditionType condType, const char *funcName)
1151 : {
1152 4 : switch (condType) {
1153 1 : case CCU_CONDITION_EQ:
1154 2 : return std::make_shared<CcuRep::CcuRepJumpNE>(
1155 1 : insGenerator, destLabelStr, targetVar, lhsVar, rhsVar);
1156 0 : case CCU_CONDITION_NE:
1157 0 : return std::make_shared<CcuRep::CcuRepJumpEQ>(
1158 0 : insGenerator, destLabelStr, targetVar, lhsVar, rhsVar);
1159 2 : case CCU_CONDITION_LT:
1160 4 : return std::make_shared<CcuRep::CcuRepJumpGE>(
1161 2 : insGenerator, destLabelStr, targetVar, lhsVar, rhsVar);
1162 0 : case CCU_CONDITION_LE:
1163 0 : return std::make_shared<CcuRep::CcuRepJumpGT>(
1164 0 : insGenerator, destLabelStr, targetVar, lhsVar, rhsVar);
1165 0 : case CCU_CONDITION_GT:
1166 0 : return std::make_shared<CcuRep::CcuRepJumpLE>(
1167 0 : insGenerator, destLabelStr, targetVar, lhsVar, rhsVar);
1168 1 : case CCU_CONDITION_GE:
1169 2 : return std::make_shared<CcuRep::CcuRepJumpLT>(
1170 1 : insGenerator, destLabelStr, targetVar, lhsVar, rhsVar);
1171 0 : default:
1172 0 : HCCL_ERROR("[%s] unsupported condition type: %d", funcName, condType);
1173 0 : return nullptr;
1174 : }
1175 : }
1176 : } // namespace
1177 :
1178 30 : CcuResult CcuKernel::IfBegin(CcuVariableHandle varHandle, uint64_t immediate,
1179 : CcuConditionType condType, const char *label)
1180 : {
1181 30 : PLF_CONFIG_INFO(PLF_DATA_OP, "[IfBegin] varHandle=%llu, immediate=%llu, condType=%d, label=%s",
1182 : varHandle, immediate, condType, (label ? label : "null"));
1183 30 : if (CurrentBlock()->Type() == CcuRep::CcuRepType::LOOP_BLOCK) {
1184 1 : HCCL_ERROR("[%s] CCU_IF is not allowed inside a ccu::Loop body (label='%s')",
1185 : __func__, label != nullptr ? label : "(null)");
1186 1 : return LatchBodyError(CcuResult::CCU_E_INTERNAL);
1187 : }
1188 29 : CcuRep::Variable *variable{nullptr};
1189 29 : CCU_CHK_RET(GetVariableByHandle(varHandle, &variable));
1190 :
1191 29 : FlushClosablePendingIfs();
1192 :
1193 29 : std::string labelStr(label);
1194 29 : if (pendingIfCtx_.find(labelStr) != pendingIfCtx_.end()) {
1195 0 : HCCL_ERROR("[%s] label '%s' already has a pending IfBegin without IfEnd", __func__, label);
1196 0 : return CcuResult::CCU_E_PARA;
1197 : }
1198 :
1199 29 : std::string elseLabelStr = labelStr + "_else";
1200 29 : std::string endLabelStr = labelStr + "_end";
1201 29 : auto elseLabel = std::make_shared<CcuRep::CcuRepJumpLabel>(insGenerator, elseLabelStr);
1202 29 : auto endLabel = std::make_shared<CcuRep::CcuRepJumpLabel>(insGenerator, endLabelStr);
1203 29 : auto targetVar = CcuRep::CreateVariable(this);
1204 29 : auto expectVar = CreateExpectVar();
1205 :
1206 : // 反转条件："if <cond>, 执行块" 等价于 "!<cond> 时跳过块"。
1207 : auto jump = MakeInvertedCondJumpImm(insGenerator, elseLabelStr,
1208 29 : targetVar, expectVar, *variable, immediate, condType, __func__);
1209 29 : if (jump == nullptr) {
1210 0 : return CcuResult::CCU_E_PARA;
1211 : }
1212 29 : jump->Reference(elseLabel);
1213 29 : Append(jump);
1214 :
1215 29 : PendingIfContext ctx;
1216 29 : ctx.elseLabel = elseLabel;
1217 29 : ctx.endLabel = endLabel;
1218 29 : ctx.hasElse = false;
1219 29 : pendingIfCtx_.emplace(labelStr, std::move(ctx));
1220 :
1221 29 : return CcuResult::CCU_SUCCESS;
1222 29 : }
1223 :
1224 3 : CcuResult CcuKernel::IfBeginVar(CcuVariableHandle lhsHandle, CcuVariableHandle rhsHandle,
1225 : CcuConditionType condType, const char *label)
1226 : {
1227 3 : if (CurrentBlock()->Type() == CcuRep::CcuRepType::LOOP_BLOCK) {
1228 0 : HCCL_ERROR("[%s] CCU_IF is not allowed inside a ccu::Loop body (label='%s')",
1229 : __func__, label != nullptr ? label : "(null)");
1230 0 : return LatchBodyError(CcuResult::CCU_E_INTERNAL);
1231 : }
1232 3 : CcuRep::Variable *lhsVar{nullptr};
1233 3 : CcuRep::Variable *rhsVar{nullptr};
1234 3 : CCU_CHK_RET(GetVariableByHandle(lhsHandle, &lhsVar));
1235 3 : CCU_CHK_RET(GetVariableByHandle(rhsHandle, &rhsVar));
1236 :
1237 3 : FlushClosablePendingIfs();
1238 :
1239 3 : std::string labelStr(label);
1240 3 : if (pendingIfCtx_.find(labelStr) != pendingIfCtx_.end()) {
1241 0 : HCCL_ERROR("[%s] label '%s' already has a pending IfBegin without IfEnd", __func__, label);
1242 0 : return CcuResult::CCU_E_PARA;
1243 : }
1244 :
1245 3 : std::string endLabelStr = labelStr + "_end";
1246 3 : std::string elseLabelStr = labelStr + "_else";
1247 3 : auto endLabel = std::make_shared<CcuRep::CcuRepJumpLabel>(insGenerator, endLabelStr);
1248 3 : auto elseLabel = std::make_shared<CcuRep::CcuRepJumpLabel>(insGenerator, elseLabelStr);
1249 3 : auto targetVar = CcuRep::CreateVariable(this);
1250 :
1251 : auto jump = MakeInvertedCondJumpVar(insGenerator, elseLabelStr,
1252 3 : targetVar, *lhsVar, *rhsVar, condType, __func__);
1253 3 : if (jump == nullptr) {
1254 0 : return CcuResult::CCU_E_PARA;
1255 : }
1256 3 : jump->Reference(elseLabel);
1257 3 : Append(jump);
1258 :
1259 3 : PendingIfContext ctx;
1260 3 : ctx.hasElse = false;
1261 3 : ctx.endLabel = endLabel;
1262 3 : ctx.elseLabel = elseLabel;
1263 3 : pendingIfCtx_.emplace(labelStr, std::move(ctx));
1264 :
1265 3 : return CcuResult::CCU_SUCCESS;
1266 3 : }
1267 :
1268 4 : CcuResult CcuKernel::IfElse(const char *label)
1269 : {
1270 4 : PLF_CONFIG_INFO(PLF_DATA_OP, "[IfElse] label=%s", (label ? label : "null"));
1271 4 : if (CurrentBlock()->Type() == CcuRep::CcuRepType::LOOP_BLOCK) {
1272 0 : HCCL_ERROR("[%s] CCU_ELSE is not allowed inside a ccu::Loop body (label='%s')",
1273 : __func__, label != nullptr ? label : "(null)");
1274 0 : return LatchBodyError(CcuResult::CCU_E_INTERNAL);
1275 : }
1276 :
1277 4 : std::string labelStr(label);
1278 4 : auto iter = pendingIfCtx_.find(labelStr);
1279 4 : if (iter == pendingIfCtx_.end()) {
1280 0 : HCCL_ERROR("[%s] no matching IfBegin for label '%s'", __func__, label);
1281 0 : return CcuResult::CCU_E_NOT_FOUND;
1282 : }
1283 :
1284 4 : if (iter->second.hasElse) {
1285 0 : HCCL_ERROR("[%s] label '%s' already has an IfElse", __func__, label);
1286 0 : return CcuResult::CCU_E_PARA;
1287 : }
1288 :
1289 : // At end of then-block: unconditional jump past else-block to endLabel
1290 4 : std::string endLabelStr = labelStr + "_end";
1291 4 : auto skipElseVar = CcuRep::CreateVariable(this);
1292 : auto skipElseJump = std::make_shared<CcuRep::CcuRepJump>(
1293 4 : insGenerator, endLabelStr, skipElseVar);
1294 4 : skipElseJump->Reference(iter->second.endLabel);
1295 4 : Append(skipElseJump);
1296 :
1297 : // Place the else label (entry point of else-block)
1298 4 : Append(iter->second.elseLabel);
1299 :
1300 4 : iter->second.hasElse = true;
1301 :
1302 4 : return CcuResult::CCU_SUCCESS;
1303 4 : }
1304 :
1305 32 : CcuResult CcuKernel::IfEnd(const char *label)
1306 : {
1307 32 : PLF_CONFIG_INFO(PLF_DATA_OP, "[IfEnd] label=%s", (label ? label : "null"));
1308 32 : std::string labelStr(label);
1309 32 : auto iter = pendingIfCtx_.find(labelStr);
1310 32 : if (iter == pendingIfCtx_.end()) {
1311 0 : HCCL_ERROR("[%s] no matching IfBegin for label '%s'", __func__, label);
1312 0 : return CcuResult::CCU_E_NOT_FOUND;
1313 : }
1314 :
1315 32 : if (iter->second.hasElse) {
1316 : // Had else-block: place endLabel after else-block
1317 4 : Append(iter->second.endLabel);
1318 : } else {
1319 : // No else-block: place elseLabel as the skip target
1320 28 : Append(iter->second.elseLabel);
1321 : }
1322 :
1323 32 : pendingIfCtx_.erase(iter);
1324 :
1325 32 : return CcuResult::CCU_SUCCESS;
1326 32 : }
1327 :
1328 3 : CcuResult CcuKernel::WhileBegin(CcuVariableHandle varHandle, uint64_t immediate,
1329 : CcuConditionType condType, const char *label)
1330 : {
1331 3 : PLF_CONFIG_INFO(PLF_DATA_OP, "[WhileBegin] varHandle=%llu, immediate=%llu, condType=%d, label=%s",
1332 : varHandle, immediate, condType, (label ? label : "null"));
1333 3 : if (CurrentBlock()->Type() == CcuRep::CcuRepType::LOOP_BLOCK) {
1334 0 : HCCL_ERROR("[%s] CCU_WHILE is not allowed inside a ccu::Loop body (label='%s')",
1335 : __func__, label != nullptr ? label : "(null)");
1336 0 : return LatchBodyError(CcuResult::CCU_E_INTERNAL);
1337 : }
1338 :
1339 3 : CcuRep::Variable *variable{nullptr};
1340 3 : CCU_CHK_RET(GetVariableByHandle(varHandle, &variable));
1341 :
1342 3 : std::string labelStr(label);
1343 3 : if (pendingWhileCtx_.find(labelStr) != pendingWhileCtx_.end()) {
1344 0 : HCCL_ERROR("[%s] label '%s' already has a pending WhileBegin without WhileEnd", __func__, label);
1345 0 : return CcuResult::CCU_E_PARA;
1346 : }
1347 :
1348 3 : std::string beginLabelStr = labelStr + "_begin";
1349 3 : std::string endLabelStr = labelStr + "_end";
1350 3 : auto beginLabel = std::make_shared<CcuRep::CcuRepJumpLabel>(insGenerator, beginLabelStr);
1351 3 : auto endLabel = std::make_shared<CcuRep::CcuRepJumpLabel>(insGenerator, endLabelStr);
1352 :
1353 3 : Append(beginLabel);
1354 :
1355 3 : auto targetVar = CcuRep::CreateVariable(this);
1356 3 : auto expectVar = CreateExpectVar();
1357 : auto jump = MakeInvertedCondJumpImm(insGenerator, endLabelStr,
1358 3 : targetVar, expectVar, *variable, immediate, condType, __func__);
1359 3 : if (jump == nullptr) {
1360 0 : return CcuResult::CCU_E_PARA;
1361 : }
1362 3 : jump->Reference(endLabel);
1363 3 : Append(jump);
1364 :
1365 3 : PendingWhileContext ctx;
1366 3 : ctx.beginLabel = beginLabel;
1367 3 : ctx.endLabel = endLabel;
1368 3 : ctx.varHandle = varHandle;
1369 3 : ctx.immediate = immediate;
1370 3 : ctx.condType = condType;
1371 3 : pendingWhileCtx_.emplace(labelStr, std::move(ctx));
1372 :
1373 3 : return CcuResult::CCU_SUCCESS;
1374 3 : }
1375 :
1376 1 : CcuResult CcuKernel::WhileBeginVar(CcuVariableHandle lhsHandle, CcuVariableHandle rhsHandle,
1377 : CcuConditionType condType, const char *label)
1378 : {
1379 1 : if (CurrentBlock()->Type() == CcuRep::CcuRepType::LOOP_BLOCK) {
1380 0 : HCCL_ERROR("[%s] CCU_WHILE is not allowed inside a ccu::Loop body (label='%s')",
1381 : __func__, label != nullptr ? label : "(null)");
1382 0 : return LatchBodyError(CcuResult::CCU_E_INTERNAL);
1383 : }
1384 :
1385 1 : CcuRep::Variable *lhsVar{nullptr};
1386 1 : CcuRep::Variable *rhsVar{nullptr};
1387 1 : CCU_CHK_RET(GetVariableByHandle(lhsHandle, &lhsVar));
1388 1 : CCU_CHK_RET(GetVariableByHandle(rhsHandle, &rhsVar));
1389 :
1390 1 : std::string labelStr(label);
1391 1 : if (pendingWhileCtx_.find(labelStr) != pendingWhileCtx_.end()) {
1392 0 : HCCL_ERROR("[%s] label '%s' already has a pending WhileBegin without WhileEnd", __func__, label);
1393 0 : return CcuResult::CCU_E_PARA;
1394 : }
1395 :
1396 1 : std::string endLabelStr = labelStr + "_end";
1397 1 : std::string beginLabelStr = labelStr + "_begin";
1398 1 : auto endLabel = std::make_shared<CcuRep::CcuRepJumpLabel>(insGenerator, endLabelStr);
1399 1 : auto beginLabel = std::make_shared<CcuRep::CcuRepJumpLabel>(insGenerator, beginLabelStr);
1400 :
1401 1 : Append(beginLabel);
1402 :
1403 1 : auto targetVar = CcuRep::CreateVariable(this);
1404 : auto jump = MakeInvertedCondJumpVar(insGenerator, endLabelStr,
1405 1 : targetVar, *lhsVar, *rhsVar, condType, __func__);
1406 1 : if (jump == nullptr) {
1407 0 : return CcuResult::CCU_E_PARA;
1408 : }
1409 1 : jump->Reference(endLabel);
1410 1 : Append(jump);
1411 :
1412 1 : PendingWhileContext ctx;
1413 1 : ctx.beginLabel = beginLabel;
1414 1 : ctx.endLabel = endLabel;
1415 1 : ctx.varHandle = lhsHandle;
1416 1 : ctx.immediate = 0;
1417 1 : ctx.condType = condType;
1418 1 : pendingWhileCtx_.emplace(labelStr, std::move(ctx));
1419 :
1420 1 : return CcuResult::CCU_SUCCESS;
1421 1 : }
1422 :
1423 4 : CcuResult CcuKernel::WhileEnd(const char *label)
1424 : {
1425 4 : PLF_CONFIG_INFO(PLF_DATA_OP, "[WhileEnd] label=%s", (label ? label : "null"));
1426 4 : std::string labelStr(label);
1427 4 : auto iter = pendingWhileCtx_.find(labelStr);
1428 4 : if (iter == pendingWhileCtx_.end()) {
1429 0 : HCCL_ERROR("[%s] no matching WhileBegin for label '%s'", __func__, label);
1430 0 : return CcuResult::CCU_E_NOT_FOUND;
1431 : }
1432 :
1433 4 : std::string beginLabelStr = labelStr + "_begin";
1434 4 : auto loopBackVar = CcuRep::CreateVariable(this);
1435 : auto loopBackJump = std::make_shared<CcuRep::CcuRepJump>(
1436 4 : insGenerator, beginLabelStr, loopBackVar);
1437 4 : loopBackJump->Reference(iter->second.beginLabel);
1438 4 : Append(loopBackJump);
1439 :
1440 4 : Append(iter->second.endLabel);
1441 :
1442 4 : pendingWhileCtx_.erase(iter);
1443 :
1444 4 : return CcuResult::CCU_SUCCESS;
1445 4 : }
1446 :
1447 9 : CcuResult CcuKernel::DoWhileBegin(const char *label)
1448 : {
1449 9 : PLF_CONFIG_INFO(PLF_DATA_OP, "[DoWhileBegin] label=%s", (label ? label : "null"));
1450 9 : if (CurrentBlock()->Type() == CcuRep::CcuRepType::LOOP_BLOCK) {
1451 0 : HCCL_ERROR("[%s] CCU_DO is not allowed inside a ccu::Loop body (label='%s')",
1452 : __func__, label != nullptr ? label : "(null)");
1453 0 : return LatchBodyError(CcuResult::CCU_E_INTERNAL);
1454 : }
1455 :
1456 9 : std::string labelStr(label);
1457 9 : if (pendingDoWhileCtx_.find(labelStr) != pendingDoWhileCtx_.end()) {
1458 0 : HCCL_ERROR("[%s] label '%s' already has a pending DoWhileBegin without DoWhileEnd", __func__, label);
1459 0 : return CcuResult::CCU_E_PARA;
1460 : }
1461 :
1462 9 : std::string beginLabelStr = labelStr + "_begin";
1463 9 : auto beginLabel = std::make_shared<CcuRep::CcuRepJumpLabel>(insGenerator, beginLabelStr);
1464 9 : Append(beginLabel);
1465 :
1466 9 : PendingDoWhileContext ctx;
1467 9 : ctx.beginLabel = beginLabel;
1468 9 : pendingDoWhileCtx_.emplace(labelStr, std::move(ctx));
1469 :
1470 9 : return CcuResult::CCU_SUCCESS;
1471 9 : }
1472 :
1473 5 : CcuResult CcuKernel::DoWhileEnd(CcuVariableHandle varHandle, uint64_t immediate,
1474 : CcuConditionType condType, const char *label)
1475 : {
1476 5 : PLF_CONFIG_INFO(PLF_DATA_OP, "[DoWhileEnd] varHandle=%llu, immediate=%llu, condType=%d, label=%s",
1477 : varHandle, immediate, condType, (label ? label : "null"));
1478 5 : CcuRep::Variable *variable{nullptr};
1479 5 : CCU_CHK_RET(GetVariableByHandle(varHandle, &variable));
1480 :
1481 5 : std::string labelStr(label);
1482 5 : auto iter = pendingDoWhileCtx_.find(labelStr);
1483 5 : if (iter == pendingDoWhileCtx_.end()) {
1484 0 : HCCL_ERROR("[%s] no matching DoWhileBegin for label '%s'", __func__, label);
1485 0 : return CcuResult::CCU_E_NOT_FOUND;
1486 : }
1487 :
1488 5 : std::string beginLabelStr = labelStr + "_begin";
1489 5 : auto targetVar = CcuRep::CreateVariable(this);
1490 5 : auto expectVar = CreateExpectVar();
1491 5 : std::shared_ptr<CcuRep::CcuRepJumpBase> jump{nullptr};
1492 :
1493 : // "condition true => continue looping" means jump back to begin when condition holds
1494 5 : if (condType == CCU_CONDITION_EQ) {
1495 0 : jump = std::make_shared<CcuRep::CcuRepJumpEQ>(
1496 0 : insGenerator, beginLabelStr, targetVar, expectVar, *variable, immediate);
1497 5 : } else if (condType == CCU_CONDITION_NE) {
1498 4 : jump = std::make_shared<CcuRep::CcuRepJumpNE>(
1499 4 : insGenerator, beginLabelStr, targetVar, expectVar, *variable, immediate);
1500 1 : } else if (condType == CCU_CONDITION_LT) {
1501 0 : jump = std::make_shared<CcuRep::CcuRepJumpLT>(
1502 0 : insGenerator, beginLabelStr, targetVar, expectVar, *variable, immediate);
1503 1 : } else if (condType == CCU_CONDITION_LE) {
1504 1 : jump = std::make_shared<CcuRep::CcuRepJumpLE>(
1505 1 : insGenerator, beginLabelStr, targetVar, expectVar, *variable, immediate);
1506 0 : } else if (condType == CCU_CONDITION_GT) {
1507 0 : jump = std::make_shared<CcuRep::CcuRepJumpGT>(
1508 0 : insGenerator, beginLabelStr, targetVar, expectVar, *variable, immediate);
1509 0 : } else if (condType == CCU_CONDITION_GE) {
1510 0 : jump = std::make_shared<CcuRep::CcuRepJumpGE>(
1511 0 : insGenerator, beginLabelStr, targetVar, expectVar, *variable, immediate);
1512 : } else {
1513 0 : HCCL_ERROR("[%s] unsupported condition type: %d", __func__, condType);
1514 0 : return CcuResult::CCU_E_PARA;
1515 : }
1516 :
1517 5 : jump->Reference(iter->second.beginLabel);
1518 5 : Append(jump);
1519 :
1520 5 : pendingDoWhileCtx_.erase(iter);
1521 :
1522 5 : return CcuResult::CCU_SUCCESS;
1523 5 : }
1524 :
1525 1 : CcuResult CcuKernel::DoWhileEndVar(CcuVariableHandle lhsHandle, CcuVariableHandle rhsHandle,
1526 : CcuConditionType condType, const char *label)
1527 : {
1528 1 : CcuRep::Variable *lhsVar{nullptr};
1529 1 : CcuRep::Variable *rhsVar{nullptr};
1530 1 : CCU_CHK_RET(GetVariableByHandle(lhsHandle, &lhsVar));
1531 1 : CCU_CHK_RET(GetVariableByHandle(rhsHandle, &rhsVar));
1532 :
1533 1 : std::string labelStr(label);
1534 1 : auto iter = pendingDoWhileCtx_.find(labelStr);
1535 1 : if (iter == pendingDoWhileCtx_.end()) {
1536 0 : HCCL_ERROR("[%s] no matching DoWhileBegin for label '%s'", __func__, label);
1537 0 : return CcuResult::CCU_E_NOT_FOUND;
1538 : }
1539 :
1540 1 : std::string beginLabelStr = labelStr + "_begin";
1541 1 : auto targetVar = CcuRep::CreateVariable(this);
1542 1 : std::shared_ptr<CcuRep::CcuRepJumpBase> jump{nullptr};
1543 :
1544 1 : if (condType == CCU_CONDITION_EQ) {
1545 0 : jump = std::make_shared<CcuRep::CcuRepJumpEQ>(
1546 0 : insGenerator, beginLabelStr, targetVar, *lhsVar, *rhsVar);
1547 1 : } else if (condType == CCU_CONDITION_NE) {
1548 1 : jump = std::make_shared<CcuRep::CcuRepJumpNE>(
1549 1 : insGenerator, beginLabelStr, targetVar, *lhsVar, *rhsVar);
1550 0 : } else if (condType == CCU_CONDITION_LT) {
1551 0 : jump = std::make_shared<CcuRep::CcuRepJumpLT>(
1552 0 : insGenerator, beginLabelStr, targetVar, *lhsVar, *rhsVar);
1553 0 : } else if (condType == CCU_CONDITION_LE) {
1554 0 : jump = std::make_shared<CcuRep::CcuRepJumpLE>(
1555 0 : insGenerator, beginLabelStr, targetVar, *lhsVar, *rhsVar);
1556 0 : } else if (condType == CCU_CONDITION_GT) {
1557 0 : jump = std::make_shared<CcuRep::CcuRepJumpGT>(
1558 0 : insGenerator, beginLabelStr, targetVar, *lhsVar, *rhsVar);
1559 0 : } else if (condType == CCU_CONDITION_GE) {
1560 0 : jump = std::make_shared<CcuRep::CcuRepJumpGE>(
1561 0 : insGenerator, beginLabelStr, targetVar, *lhsVar, *rhsVar);
1562 : } else {
1563 0 : HCCL_ERROR("[%s] unsupported condition type: %d", __func__, condType);
1564 0 : return CcuResult::CCU_E_PARA;
1565 : }
1566 :
1567 1 : jump->Reference(iter->second.beginLabel);
1568 1 : Append(jump);
1569 :
1570 1 : pendingDoWhileCtx_.erase(iter);
1571 :
1572 1 : return CcuResult::CCU_SUCCESS;
1573 1 : }
1574 :
1575 : // 控制流标签栈实体
1576 32 : void CcuKernel::IfLabelStackPush(const char *label)
1577 : {
1578 32 : iflabelStack_.push_back({label, false});
1579 32 : }
1580 32 : void CcuKernel::IfLabelStackMarkBodyDone()
1581 : {
1582 32 : if (iflabelStack_.empty()) {
1583 0 : HCCL_ERROR("[CcuKernel::IfLabelStack][MarkBodyDone] stack is empty");
1584 0 : return;
1585 : }
1586 32 : iflabelStack_.back().bodyDone = true;
1587 : }
1588 4 : const char *CcuKernel::IfLabelStackPopForElse()
1589 : {
1590 4 : if (iflabelStack_.empty()) {
1591 0 : HCCL_ERROR("[CcuKernel::IfLabelStack][PopForElse] orphan CCU_ELSE: "
1592 : "no matching CCU_IF on the stack");
1593 0 : return nullptr;
1594 : }
1595 4 : if (!iflabelStack_.back().bodyDone) {
1596 0 : HCCL_ERROR("[CcuKernel::IfLabelStack][PopForElse] CCU_ELSE called while "
1597 : "top if-body is still InBody (label='%s')",
1598 : iflabelStack_.back().label != nullptr
1599 : ? iflabelStack_.back().label : "(null)");
1600 0 : return nullptr;
1601 : }
1602 4 : const char *label = iflabelStack_.back().label;
1603 4 : iflabelStack_.pop_back();
1604 4 : return label;
1605 : }
1606 970 : bool CcuKernel::IfLabelStackTopIsClosable()
1607 : {
1608 970 : return !iflabelStack_.empty() && iflabelStack_.back().bodyDone;
1609 : }
1610 :
1611 28 : const char *CcuKernel::IfLabelStackPop()
1612 : {
1613 28 : if (iflabelStack_.empty()) {
1614 0 : return nullptr;
1615 : }
1616 28 : const char *label = iflabelStack_.back().label;
1617 28 : iflabelStack_.pop_back();
1618 28 : return label;
1619 : }
1620 :
1621 9 : void CcuKernel::DoWhileLabelStackPush(const char *label)
1622 : {
1623 9 : DoWhileLabelEntry entry;
1624 9 : entry.label = label;
1625 9 : entry.snapshotBlock = CurrentBlock();
1626 9 : entry.snapshotRepCount = (entry.snapshotBlock != nullptr)
1627 9 : ? entry.snapshotBlock->GetReps().size()
1628 : : 0;
1629 9 : doWhileLabelStack_.push_back(std::move(entry));
1630 9 : }
1631 :
1632 13 : const char *CcuKernel::DoWhileLabelStackPopForWhile()
1633 : {
1634 13 : if (CurrentBlock()->Type() == CcuRep::CcuRepType::LOOP_BLOCK) {
1635 1 : return nullptr;
1636 : }
1637 12 : if (doWhileLabelStack_.empty()) {
1638 4 : return nullptr;
1639 : }
1640 8 : DoWhileLabelEntry entry = doWhileLabelStack_.back();
1641 8 : doWhileLabelStack_.pop_back();
1642 :
1643 8 : auto currentBlock = CurrentBlock();
1644 8 : size_t currentRepCount = (currentBlock != nullptr) ? currentBlock->GetReps().size() : 0;
1645 8 : if (currentBlock != entry.snapshotBlock || currentRepCount != entry.snapshotRepCount) {
1646 1 : HCCL_ERROR("[CcuKernel::DoWhileLabelStackPopForWhile] dangling CCU calls between CCU_DO end "
1647 : "and CCU_WHILE (label='%s', snapRep=%zu, curRep=%zu, blockChanged=%d); they must "
1648 : "be syntactically adjacent, otherwise the code in between is pulled into the body.",
1649 : entry.label != nullptr ? entry.label : "(null)", entry.snapshotRepCount,
1650 : currentRepCount, currentBlock != entry.snapshotBlock ? 1 : 0);
1651 1 : return nullptr;
1652 : }
1653 7 : return entry.label;
1654 8 : }
1655 :
1656 150 : CcuResult CcuKernel::GetAddressByHandle(CcuAddressHandle addrHandle, CcuRep::Address **address)
1657 : {
1658 150 : return GetResourceByHandle(ccuAddrMap_, addrHandle, address, "address");
1659 : }
1660 :
1661 : // addr = 立即数 → CcuRepAssign(Address, uint64_t)
1662 11 : CcuResult CcuKernel::AddressAssignImm(CcuAddressHandle addrHandle, uint64_t immediate)
1663 : {
1664 11 : PLF_CONFIG_INFO(PLF_DATA_OP, "[AddressAssignImm] addrHandle=%llu, immediate=%llu", addrHandle, immediate);
1665 11 : CcuRep::Address *address{nullptr};
1666 11 : CCU_CHK_RET(GetAddressByHandle(addrHandle, &address));
1667 11 : (*address) = immediate;
1668 11 : return CcuResult::CCU_SUCCESS;
1669 : }
1670 :
1671 : // addr = variable → CcuRepAssign(Address, Variable)
1672 14 : CcuResult CcuKernel::AddressAssignVar(CcuAddressHandle addrHandle, CcuVariableHandle varHandle)
1673 : {
1674 14 : PLF_CONFIG_INFO(PLF_DATA_OP, "[AddressAssignVar] addrHandle=%llu, varHandle=%llu", addrHandle, varHandle);
1675 14 : CcuRep::Address *address{nullptr};
1676 14 : CCU_CHK_RET(GetAddressByHandle(addrHandle, &address));
1677 :
1678 14 : CcuRep::Variable *variable{nullptr};
1679 14 : CCU_CHK_RET(GetVariableByHandle(varHandle, &variable));
1680 :
1681 14 : (*address) = (*variable);
1682 14 : return CcuResult::CCU_SUCCESS;
1683 : }
1684 : // addr = addr → CcuRepAssign(Address, Address)
1685 38 : CcuResult CcuKernel::AddressAssignAddr(CcuAddressHandle dstAddrHandle, CcuAddressHandle srcAddrHandle)
1686 : {
1687 38 : PLF_CONFIG_INFO(PLF_DATA_OP, "[AddressAssignAddr] dstAddrHandle=%llu, srcAddrHandle=%llu",
1688 : dstAddrHandle, srcAddrHandle);
1689 38 : CcuRep::Address *dstAddress{nullptr};
1690 38 : CCU_CHK_RET(GetAddressByHandle(dstAddrHandle, &dstAddress));
1691 :
1692 38 : CcuRep::Address *srcAddress{nullptr};
1693 38 : CCU_CHK_RET(GetAddressByHandle(srcAddrHandle, &srcAddress));
1694 :
1695 38 : (*dstAddress) = (*srcAddress);
1696 38 : return CcuResult::CCU_SUCCESS;
1697 : }
1698 :
1699 : // resAddr = lhsAddr + rhsVar → CcuRepAdd(Address, Address, Variable)
1700 0 : CcuResult CcuKernel::AddressAddVarToAddr(
1701 : CcuAddressHandle resAddrHandle, CcuAddressHandle lhsAddrHandle, CcuVariableHandle rhsVarHandle)
1702 : {
1703 0 : PLF_CONFIG_INFO(PLF_DATA_OP, "[AddressAddVarToAddr] resAddrHandle=%llu, lhsAddrHandle=%llu, rhsVarHandle=%llu",
1704 : resAddrHandle, lhsAddrHandle, rhsVarHandle);
1705 0 : CcuRep::Address *resAddr{nullptr}, *lhsAddr{nullptr};
1706 0 : CCU_CHK_RET(GetAddressByHandle(resAddrHandle, &resAddr));
1707 0 : CCU_CHK_RET(GetAddressByHandle(lhsAddrHandle, &lhsAddr));
1708 :
1709 0 : CcuRep::Variable *rhsVar{nullptr};
1710 0 : CCU_CHK_RET(GetVariableByHandle(rhsVarHandle, &rhsVar));
1711 :
1712 0 : *resAddr = *lhsAddr + *rhsVar;
1713 0 : return CcuResult::CCU_SUCCESS;
1714 : }
1715 :
1716 : // resAddr = addrA + addrB → CcuRepAdd(Address, Address, Address)
1717 2 : CcuResult CcuKernel::AddressAddAddrToAddr(
1718 : CcuAddressHandle resAddrHandle, CcuAddressHandle addrAHandle, CcuAddressHandle addrBHandle)
1719 : {
1720 2 : PLF_CONFIG_INFO(PLF_DATA_OP, "[AddressAddAddrToAddr] resAddrHandle=%llu, addrAHandle=%llu, addrBHandle=%llu",
1721 : resAddrHandle, addrAHandle, addrBHandle);
1722 2 : CcuRep::Address *resAddr{nullptr}, *addrA{nullptr}, *addrB{nullptr};
1723 2 : CCU_CHK_RET(GetAddressByHandle(resAddrHandle, &resAddr));
1724 2 : CCU_CHK_RET(GetAddressByHandle(addrAHandle, &addrA));
1725 2 : CCU_CHK_RET(GetAddressByHandle(addrBHandle, &addrB));
1726 :
1727 2 : *resAddr = *addrA + *addrB;
1728 2 : return CcuResult::CCU_SUCCESS;
1729 : }
1730 :
1731 : // addr += variable → CcuRepAdd(Address, Variable) 就地加
1732 41 : CcuResult CcuKernel::AddressAddAssignVar(CcuAddressHandle addrHandle, CcuVariableHandle varHandle)
1733 : {
1734 41 : PLF_CONFIG_INFO(PLF_DATA_OP, "[AddressAddAssignVar] addrHandle=%llu, varHandle=%llu", addrHandle, varHandle);
1735 41 : CcuRep::Address *address{nullptr};
1736 41 : CCU_CHK_RET(GetAddressByHandle(addrHandle, &address));
1737 :
1738 41 : CcuRep::Variable *variable{nullptr};
1739 41 : CCU_CHK_RET(GetVariableByHandle(varHandle, &variable));
1740 :
1741 41 : (*address) += (*variable);
1742 41 : return CcuResult::CCU_SUCCESS;
1743 : }
1744 :
1745 : // addr += addr → 等价于 addr = addr + otherAddr
1746 0 : CcuResult CcuKernel::AddressAddAssignAddr(CcuAddressHandle addrHandle, CcuAddressHandle otherHandle)
1747 : {
1748 0 : PLF_CONFIG_INFO(PLF_DATA_OP, "[AddressAddAssignAddr] addrHandle=%llu, otherHandle=%llu", addrHandle, otherHandle);
1749 0 : CcuRep::Address *address{nullptr};
1750 0 : CCU_CHK_RET(GetAddressByHandle(addrHandle, &address));
1751 :
1752 0 : CcuRep::Address *other{nullptr};
1753 0 : CCU_CHK_RET(GetAddressByHandle(otherHandle, &other));
1754 :
1755 0 : (*address) = (*address) + (*other);
1756 0 : return CcuResult::CCU_SUCCESS;
1757 : }
1758 :
1759 1 : CcuResult CcuKernel::AddressAddImmToAddr(CcuAddressHandle resAddrHandle, CcuAddressHandle addrAHandle, uint16_t imm)
1760 : {
1761 1 : CcuRep::Address *resAddr{nullptr}, *addrA{nullptr};
1762 1 : CCU_CHK_RET(GetAddressByHandle(resAddrHandle, &resAddr));
1763 1 : CCU_CHK_RET(GetAddressByHandle(addrAHandle, &addrA));
1764 :
1765 1 : *resAddr = *addrA + imm;
1766 1 : return CcuResult::CCU_SUCCESS;
1767 : }
1768 :
1769 0 : void CcuKernel::Load(const CcuRep::Variable &var)
1770 : {
1771 : auto loadArgRep = std::make_shared<CcuRep::CcuRepLoadArg>(
1772 0 : insGenerator, var, loadArgIndex_ % CCU_SQE_ARGS_LEN, static_cast<uint16_t>(loadArgIndex_));
1773 0 : GetLGProfilingInfo().loadRep2ArgIdxMap[loadArgRep] = loadArgIndex_;
1774 0 : Append(loadArgRep);
1775 0 : loadArgIndex_++;
1776 0 : }
1777 :
1778 0 : void CcuKernel::StoreVariable(const CcuRep::Variable &var, uint64_t addr)
1779 : {
1780 0 : Append(std::make_shared<CcuRep::CcuRepStore>(insGenerator, var, addr));
1781 0 : }
1782 :
1783 0 : void CcuKernel::LoadVariable(const CcuRep::Variable &src, const CcuRep::Variable &var)
1784 : {
1785 0 : Append(std::make_shared<CcuRep::CcuRepLoadVar>(insGenerator, src, var));
1786 0 : }
1787 :
1788 11 : HcclResult CcuKernel::RecordEvent(CcuRep::CompletedEvent event, uint32_t mask)
1789 : {
1790 11 : if (CurrentBlock()->Type() == CcuRep::CcuRepType::LOOP_BLOCK) {
1791 1 : HCCL_ERROR("[CcuKernel][%s] is not supported in loop block, please check.", __func__);
1792 1 : LatchBodyError(HCCL_TO_CCU_RET(HcclResult::HCCL_E_NOT_SUPPORT));
1793 1 : return HcclResult::HCCL_E_NOT_SUPPORT;
1794 : }
1795 :
1796 10 : auto rep = std::make_shared<CcuRep::CcuRepLocRecordEvent>(insGenerator, event, mask);
1797 10 : Append(rep);
1798 10 : SetDependencyInfo(event.Id(), mask, rep);
1799 10 : return HCCL_SUCCESS;
1800 10 : }
1801 :
1802 32 : HcclResult CcuKernel::WaitEvent(CcuRep::CompletedEvent event, uint32_t mask)
1803 : {
1804 32 : bool isProfiling = CurrentBlock()->Type() != CcuRep::CcuRepType::LOOP_BLOCK;
1805 32 : auto rep = std::make_shared<CcuRep::CcuRepLocWaitEvent>(insGenerator, event, mask, isProfiling);
1806 32 : if (isProfiling) {
1807 48 : CHK_RET(static_cast<HcclResult>(AddProfiling("WaitEvent", rep->GetMask())));
1808 : }
1809 32 : rep->SetDependencyInfo(GetDependencyInfo(event.Id()));
1810 32 : EraseDependencyInfo(event.Id());
1811 32 : Append(rep);
1812 32 : return HCCL_SUCCESS;
1813 32 : }
1814 :
1815 78 : CcuResult CcuKernel::GetEventByHandle(CcuEventHandle eventHandle, CcuRep::CompletedEvent **event)
1816 : {
1817 78 : return GetResourceByHandle(ccuEventMap_, eventHandle, event, "completedEvent");
1818 : }
1819 :
1820 : /*
1821 : LocalAddr / RemoteAddr 相关接口
1822 : */
1823 27 : CcuResult CcuKernel::GetLocalAddrByHandle(CcuLocalAddrHandle handle, CcuRep::LocalAddr **localAddr)
1824 : {
1825 27 : return GetResourceByHandle(ccuLocalAddrMap_, handle, localAddr, "localAddr");
1826 : }
1827 :
1828 15 : CcuResult CcuKernel::GetRemoteAddrByHandle(CcuRemoteAddrHandle handle, CcuRep::RemoteAddr **remoteAddr)
1829 : {
1830 15 : return GetResourceByHandle(ccuRemoteAddrMap_, handle, remoteAddr, "remoteAddr");
1831 : }
1832 :
1833 :
1834 : /*Read新接口*/
1835 2 : HcclResult CcuKernel::ReadNb(const ChannelHandle channel, const CcuRep::CcuBuf &loc, const CcuRep::RemoteAddr &rem,
1836 : const CcuRep::Variable &len, CcuRep::CompletedEvent event, uint32_t mask)
1837 : {
1838 2 : channels_.insert(channel);
1839 2 : auto rep = std::make_shared<CcuRep::CcuRepBufRead>(insGenerator, channel, rem, loc, len, event, mask);
1840 2 : Append(rep);
1841 2 : SetDependencyInfo(event.Id(), mask, rep);
1842 2 : return HCCL_SUCCESS;
1843 2 : }
1844 :
1845 : /*Write新接口*/
1846 2 : HcclResult CcuKernel::WriteNb(const ChannelHandle channel, const CcuRep::RemoteAddr &rem, const CcuRep::CcuBuf &loc,
1847 : const CcuRep::Variable &len, CcuRep::CompletedEvent event, uint32_t mask)
1848 : {
1849 2 : channels_.insert(channel);
1850 2 : auto rep = std::make_shared<CcuRep::CcuRepBufWrite>(insGenerator, channel, loc, rem, len, event, mask);
1851 2 : Append(rep);
1852 2 : SetDependencyInfo(event.Id(), mask, rep);
1853 2 : return HCCL_SUCCESS;
1854 2 : }
1855 :
1856 8 : static bool isLowPrecisionIn(Hccl::DataType dataType)
1857 : {
1858 16 : return dataType == Hccl::DataType::INT8 || dataType == Hccl::DataType::HIF8 || dataType == Hccl::DataType::FP8E4M3
1859 16 : || dataType == Hccl::DataType::FP8E5M2;
1860 : }
1861 :
1862 0 : static bool isLowPrecisionOut(Hccl::DataType dataType)
1863 : {
1864 0 : return dataType == Hccl::DataType::FP16 || dataType == Hccl::DataType::BFP16 || dataType == Hccl::DataType::FP32;
1865 : }
1866 :
1867 : constexpr uint32_t MAX_DATA_TYPE = 17;
1868 :
1869 : const Hccl::DataType orionDataTypes[] = {
1870 : Hccl::DataType::INT8,
1871 : Hccl::DataType::INT16,
1872 : Hccl::DataType::INT32,
1873 : Hccl::DataType::FP16,
1874 : Hccl::DataType::FP32,
1875 : Hccl::DataType::INT64,
1876 : Hccl::DataType::UINT64,
1877 : Hccl::DataType::UINT8,
1878 : Hccl::DataType::UINT16,
1879 : Hccl::DataType::UINT32,
1880 : Hccl::DataType::FP64,
1881 : Hccl::DataType::BFP16,
1882 : Hccl::DataType::INT128,
1883 : #if !defined (OPEN_BUILD_PROJECT) || defined (ORION_MODE)
1884 : Hccl::DataType::HIF8,
1885 : Hccl::DataType::FP8E4M3,
1886 : Hccl::DataType::FP8E5M2,
1887 : Hccl::DataType::FP8E8M0
1888 : #endif
1889 : };
1890 :
1891 12 : static Hccl::DataType HcommDataTypeToHcclDataType(const HcclDataType dataType)
1892 : {
1893 12 : const auto dataTypeNum = static_cast<uint32_t>(dataType);
1894 12 : if (dataTypeNum > MAX_DATA_TYPE) {
1895 0 : return Hccl::DataType::INVALID;
1896 : }
1897 :
1898 12 : return orionDataTypes[dataTypeNum];
1899 : }
1900 :
1901 : constexpr uint32_t MAX_REDUCE_TYPE = 4;
1902 : const Hccl::ReduceOp orionReduceOps[] = {
1903 : Hccl::ReduceOp::SUM,
1904 : Hccl::ReduceOp::PROD,
1905 : Hccl::ReduceOp::MAX,
1906 : Hccl::ReduceOp::MIN,
1907 : };
1908 :
1909 8 : static Hccl::ReduceOp HcommReduceOpToHcclReduceOp(const HcclReduceOp reduceOp)
1910 : {
1911 8 : const auto reduceOpNum = static_cast<uint32_t>(reduceOp);
1912 8 : if (reduceOpNum > MAX_REDUCE_TYPE) {
1913 0 : return Hccl::ReduceOp::INVALID;
1914 : }
1915 :
1916 8 : return orionReduceOps[reduceOpNum];
1917 : }
1918 :
1919 4 : HcclResult CcuKernel::LocalReduceNb(const CcuRep::CcuBuf *bufs, uint32_t count, HcclDataType dataType,
1920 : HcclDataType outputDataType, HcclReduceOp opType,
1921 : const CcuRep::Variable &len, CcuRep::CompletedEvent event, uint32_t mask)
1922 : {
1923 4 : auto opType_ = HcommReduceOpToHcclReduceOp(opType);
1924 4 : auto dataType_ = HcommDataTypeToHcclDataType(dataType);
1925 4 : auto outputDataType_ = HcommDataTypeToHcclDataType(outputDataType);
1926 :
1927 8 : if ((opType_ == Hccl::ReduceOp::SUM && isLowPrecisionIn(dataType_) && !isLowPrecisionOut(outputDataType_))
1928 4 : || (opType_ == Hccl::ReduceOp::SUM && !isLowPrecisionIn(dataType_) && dataType_ != outputDataType_)
1929 8 : || (opType_ != Hccl::ReduceOp::SUM && dataType_ != outputDataType_)) {
1930 0 : return HCCL_E_NOT_SUPPORT;
1931 : }
1932 :
1933 4 : std::vector<CcuRep::CcuBuf> ccuBufs(count);
1934 12 : for (uint32_t i = 0; i < count; i++) {
1935 8 : ccuBufs[i] = bufs[i];
1936 : }
1937 :
1938 0 : auto rep = std::make_shared<CcuRep::CcuRepBufReduce>(insGenerator, ccuBufs, count, CcuRep::GetCcuDataType(dataType_, opType_),
1939 4 : CcuRep::GetCcuDataType(outputDataType_, opType_), CcuRep::GetCcuReduceType(opType_), event, len, mask);
1940 4 : Append(rep);
1941 4 : SetDependencyInfo(event.Id(), mask, rep);
1942 4 : return HCCL_SUCCESS;
1943 4 : }
1944 :
1945 : /*Read新接口*/
1946 4 : HcclResult CcuKernel::ReadNb(const ChannelHandle channel, const CcuRep::LocalAddr &loc, const CcuRep::RemoteAddr &rem,
1947 : const CcuRep::Variable &len, CcuRep::CompletedEvent event, uint32_t mask)
1948 : {
1949 4 : channels_.insert(channel);
1950 4 : auto rep = std::make_shared<CcuRep::CcuRepRead>(insGenerator, channel, loc, rem, len, event, mask);
1951 4 : Append(rep);
1952 4 : SetDependencyInfo(event.Id(), mask, rep);
1953 4 : return HCCL_SUCCESS;
1954 4 : }
1955 :
1956 : /*ReadReduce新接口*/
1957 2 : HcclResult CcuKernel::ReadReduceNb(const ChannelHandle channel, const CcuRep::LocalAddr &loc, const CcuRep::RemoteAddr &rem,
1958 : const CcuRep::Variable &len, HcclDataType dataType, HcclReduceOp opType,
1959 : CcuRep::CompletedEvent event, uint32_t mask)
1960 : {
1961 2 : channels_.insert(channel);
1962 2 : auto opType_ = HcommReduceOpToHcclReduceOp(opType);
1963 2 : auto dataType_ = HcommDataTypeToHcclDataType(dataType);
1964 :
1965 0 : auto rep = std::make_shared<CcuRep::CcuRepRead>(insGenerator, channel, loc, rem, len, CcuRep::GetUBDataType(dataType_),
1966 2 : CcuRep::GetUBReduceType(opType_), event, mask);
1967 2 : Append(rep);
1968 2 : SetDependencyInfo(event.Id(), mask, rep);
1969 2 : return HCCL_SUCCESS;
1970 2 : }
1971 :
1972 : /*Write新接口*/
1973 3 : HcclResult CcuKernel::WriteNb(const ChannelHandle channel, const CcuRep::RemoteAddr &rem, const CcuRep::LocalAddr &loc,
1974 : const CcuRep::Variable &len, CcuRep::CompletedEvent event, uint32_t mask)
1975 : {
1976 3 : channels_.insert(channel);
1977 3 : auto rep = std::make_shared<CcuRep::CcuRepWrite>(insGenerator, channel, rem, loc, len, event, mask);
1978 3 : Append(rep);
1979 3 : SetDependencyInfo(event.Id(), mask, rep);
1980 3 : return HCCL_SUCCESS;
1981 3 : }
1982 :
1983 2 : HcclResult CcuKernel::WriteReduceNb(const ChannelHandle channel, const CcuRep::RemoteAddr &rem, const CcuRep::LocalAddr &loc,
1984 : const CcuRep::Variable &len, HcclDataType dataType, HcclReduceOp opType,
1985 : CcuRep::CompletedEvent event, uint32_t mask)
1986 : {
1987 2 : channels_.insert(channel);
1988 2 : auto opType_ = HcommReduceOpToHcclReduceOp(opType);
1989 2 : auto dataType_ = HcommDataTypeToHcclDataType(dataType);
1990 :
1991 2 : auto rep = std::make_shared<CcuRep::CcuRepWrite>(insGenerator,
1992 2 : channel, rem, loc, len, CcuRep::GetUBDataType(dataType_), CcuRep::GetUBReduceType(opType_), event, mask);
1993 2 : Append(rep);
1994 2 : SetDependencyInfo(event.Id(), mask, rep);
1995 2 : return HCCL_SUCCESS;
1996 2 : }
1997 :
1998 28 : CcuResult CcuKernel::GetBufferByHandle(CcuBufferHandle bufferHandle, CcuRep::CcuBuf **buffer)
1999 : {
2000 28 : return GetResourceByHandle(ccuBufferMap_, bufferHandle, buffer, "buffer");
2001 : }
2002 :
2003 0 : HcclResult CcuKernel::LocalCopyNb(const CcuRep::LocalAddr &dst, const CcuRep::LocalAddr &src, const CcuRep::Variable &len,
2004 : CcuRep::CompletedEvent event, uint32_t mask)
2005 : {
2006 0 : auto rep = std::make_shared<CcuRep::CcuRepLocCpy>(insGenerator, dst, src, len, event, mask);
2007 0 : Append(rep);
2008 0 : SetDependencyInfo(event.Id(), mask, rep);
2009 0 : return HCCL_SUCCESS;
2010 0 : }
2011 :
2012 10 : HcclResult CcuKernel::LocalCopyNb(const CcuRep::CcuBuf &dst, const CcuRep::LocalAddr &src, const CcuRep::Variable &len,
2013 : CcuRep::CompletedEvent event, uint32_t mask)
2014 : {
2015 10 : auto rep = std::make_shared<CcuRep::CcuRepBufLocRead>(insGenerator, src, dst, len, event, mask);
2016 10 : Append(rep);
2017 10 : SetDependencyInfo(event.Id(), mask, rep);
2018 10 : return HCCL_SUCCESS;
2019 10 : }
2020 :
2021 6 : HcclResult CcuKernel::LocalCopyNb(const CcuRep::LocalAddr &dst, const CcuRep::CcuBuf &src, const CcuRep::Variable &len,
2022 : CcuRep::CompletedEvent event, uint32_t mask)
2023 : {
2024 6 : auto rep = std::make_shared<CcuRep::CcuRepBufLocWrite>(insGenerator, src, dst, len, event, mask);
2025 6 : Append(rep);
2026 6 : SetDependencyInfo(event.Id(), mask, rep);
2027 6 : return HCCL_SUCCESS;
2028 6 : }
2029 :
2030 0 : HcclResult CcuKernel::LocalReduceNb(const CcuRep::LocalAddr &dst, const CcuRep::LocalAddr &src, const CcuRep::Variable &len,
2031 : HcclDataType dataType, HcclReduceOp opType, CcuRep::CompletedEvent event, uint32_t mask)
2032 : {
2033 0 : auto opType_ = HcommReduceOpToHcclReduceOp(opType);
2034 0 : auto dataType_ = HcommDataTypeToHcclDataType(dataType);
2035 :
2036 0 : auto rep = std::make_shared<CcuRep::CcuRepLocCpy>(insGenerator, dst, src, len, CcuRep::GetUBDataType(dataType_),
2037 0 : CcuRep::GetUBReduceType(opType_), event, mask);
2038 0 : Append(rep);
2039 0 : SetDependencyInfo(event.Id(), mask, rep);
2040 0 : return HCCL_SUCCESS;
2041 0 : }
2042 :
2043 0 : CcuRep::FuncCall CcuKernel::Func(const std::string &label)
2044 : {
2045 0 : return CcuRep::FuncCall(this, label);
2046 : }
2047 :
2048 0 : CcuRep::FuncCall CcuKernel::Func(const CcuRep::Variable &funcAddr)
2049 : {
2050 0 : return CcuRep::FuncCall(this, funcAddr);
2051 : }
2052 :
2053 0 : CcuRep::LoopCall CcuKernel::Loop(const std::string &label)
2054 : {
2055 0 : return CcuRep::LoopCall(this, label);
2056 : }
2057 :
2058 52 : CcuResult CcuKernel::LoopCreate(CcuLoop *loop)
2059 : {
2060 52 : PLF_CONFIG_INFO(PLF_DATA_OP, "[LoopCreate]");
2061 52 : if (loop == nullptr) {
2062 0 : HCCL_ERROR("[CcuKernel::LoopCreate] null pointer");
2063 0 : return CcuResult::CCU_E_PTR;
2064 : }
2065 52 : if (CurrentBlock()->Type() == CcuRep::CcuRepType::LOOP_BLOCK) {
2066 0 : HCCL_ERROR("[CcuKernel::LoopCreate] cannot create loop inside a loop body");
2067 0 : return LatchBodyError(CcuResult::CCU_E_INTERNAL);
2068 : }
2069 52 : if (inFuncBody_) {
2070 0 : HCCL_ERROR("[CcuKernel::LoopCreate] cannot create loop inside a func body");
2071 0 : return LatchBodyError(CcuResult::CCU_E_INTERNAL);
2072 : }
2073 :
2074 52 : CcuLoop handle = ++loopHandleCounter_;
2075 52 : std::string label = "loop_" + std::to_string(handle);
2076 :
2077 52 : LoopDescriptor desc;
2078 52 : desc.label = label;
2079 52 : desc.repLoopBlock = std::make_shared<CcuRep::CcuRepLoopBlock>(insGenerator, label);
2080 :
2081 52 : loopMap_[handle] = std::move(desc);
2082 52 : *loop = handle;
2083 52 : return CcuResult::CCU_SUCCESS;
2084 52 : }
2085 :
2086 7 : CcuResult CcuKernel::LatchBodyError(CcuResult err)
2087 : {
2088 15 : if ((CurrentBlock()->Type() == CcuRep::CcuRepType::LOOP_BLOCK || inFuncBody_)
2089 15 : && bodyError_ == CcuResult::CCU_SUCCESS) {
2090 7 : bodyError_ = err;
2091 : }
2092 7 : return err;
2093 : }
2094 :
2095 52 : CcuResult CcuKernel::LoopBodyEnter(CcuLoop loop)
2096 : {
2097 52 : PLF_CONFIG_INFO(PLF_DATA_OP, "[LoopBodyEnter] loop=%llu", loop);
2098 52 : auto it = loopMap_.find(loop);
2099 52 : if (it == loopMap_.end()) {
2100 0 : HCCL_ERROR("[CcuKernel::LoopBodyEnter] invalid loop handle %lu", loop);
2101 0 : return CcuResult::CCU_E_PARA;
2102 : }
2103 52 : auto &desc = it->second;
2104 52 : if (desc.bodyDefined) {
2105 0 : HCCL_ERROR("[CcuKernel::LoopBodyEnter] loop %lu body already defined", loop);
2106 0 : return CcuResult::CCU_E_INTERNAL;
2107 : }
2108 :
2109 52 : Append(desc.repLoopBlock);
2110 52 : desc.prevActiveBlock = CurrentBlock();
2111 52 : SetCurrentBlock(desc.repLoopBlock);
2112 52 : ++loopBodyDepth_;
2113 52 : bodyError_ = CcuResult::CCU_SUCCESS;
2114 :
2115 52 : return CcuResult::CCU_SUCCESS;
2116 : }
2117 :
2118 52 : CcuResult CcuKernel::LoopBodyExit(CcuLoop loop)
2119 : {
2120 52 : PLF_CONFIG_INFO(PLF_DATA_OP, "[LoopBodyExit] loop=%llu", loop);
2121 52 : auto it = loopMap_.find(loop);
2122 52 : if (it == loopMap_.end()) {
2123 0 : HCCL_ERROR("[CcuKernel::LoopBodyExit] invalid loop handle %lu", loop);
2124 0 : return CcuResult::CCU_E_PARA;
2125 : }
2126 52 : auto &desc = it->second;
2127 :
2128 52 : SetCurrentBlock(desc.prevActiveBlock);
2129 52 : desc.bodyDefined = true;
2130 52 : --loopBodyDepth_;
2131 :
2132 52 : if (bodyError_ != CcuResult::CCU_SUCCESS) {
2133 6 : const CcuResult err = bodyError_;
2134 6 : HCCL_ERROR("[CcuKernel::LoopBodyExit] illegal operation inside loop body, err=%d", err);
2135 6 : bodyError_ = CcuResult::CCU_SUCCESS;
2136 6 : return err;
2137 : }
2138 :
2139 46 : return CcuResult::CCU_SUCCESS;
2140 : }
2141 :
2142 8 : CcuResult CcuKernel::FuncBlockLookup(const void *funcPtr, uint64_t *outHandle)
2143 : {
2144 8 : PLF_CONFIG_INFO(PLF_DATA_OP, "[FuncBlockLookup] funcPtr=%p", funcPtr);
2145 8 : if (funcPtr == nullptr || outHandle == nullptr) {
2146 0 : HCCL_ERROR("[CcuKernel::FuncBlockLookup] null pointer");
2147 0 : return CcuResult::CCU_E_PTR;
2148 : }
2149 8 : if (CurrentBlock()->Type() == CcuRep::CcuRepType::LOOP_BLOCK || inFuncBody_) {
2150 2 : HCCL_ERROR("[CcuKernel::FuncBlockLookup] ccu::CallFunc only allowed at top level");
2151 2 : return LatchBodyError(CcuResult::CCU_E_INTERNAL);
2152 : }
2153 :
2154 6 : auto it = funcInstanceMap_.find(funcPtr);
2155 6 : *outHandle = (it == funcInstanceMap_.end()) ? 0 : it->second;
2156 6 : return CcuResult::CCU_SUCCESS;
2157 : }
2158 :
2159 5 : CcuResult CcuKernel::FuncBlockBegin(const void *funcPtr, uint64_t *outHandle)
2160 : {
2161 5 : PLF_CONFIG_INFO(PLF_DATA_OP, "[FuncBlockBegin] funcPtr=%p", funcPtr);
2162 5 : if (funcPtr == nullptr || outHandle == nullptr) {
2163 0 : HCCL_ERROR("[CcuKernel::FuncBlockBegin] null pointer");
2164 0 : return CcuResult::CCU_E_PTR;
2165 : }
2166 5 : if (CurrentBlock()->Type() == CcuRep::CcuRepType::LOOP_BLOCK || inFuncBody_) {
2167 0 : HCCL_ERROR("[CcuKernel::FuncBlockBegin] ccu::CallFunc only allowed at top level");
2168 0 : return LatchBodyError(CcuResult::CCU_E_INTERNAL);
2169 : }
2170 :
2171 5 : auto exist = funcInstanceMap_.find(funcPtr);
2172 5 : if (exist != funcInstanceMap_.end()) {
2173 0 : *outHandle = exist->second;
2174 0 : return CcuResult::CCU_SUCCESS;
2175 : }
2176 :
2177 5 : const uint64_t handle = ++funcHandleCounter_;
2178 5 : std::string label = "func_" + std::to_string(handle);
2179 :
2180 5 : FuncDescriptor desc;
2181 5 : desc.funcPtr = funcPtr;
2182 5 : desc.label = label;
2183 5 : desc.repFuncBlock = std::make_shared<CcuRep::CcuRepFuncBlock>(insGenerator, label);
2184 5 : desc.prevActiveBlock = CurrentBlock();
2185 :
2186 5 : funcMap_[handle] = desc;
2187 5 : SetCurrentBlock(desc.repFuncBlock);
2188 5 : inFuncBody_ = true;
2189 5 : bodyError_ = CcuResult::CCU_SUCCESS;
2190 5 : *outHandle = handle;
2191 5 : return CcuResult::CCU_SUCCESS;
2192 5 : }
2193 :
2194 4 : CcuResult CcuKernel::FuncBlockEnd(uint64_t handle)
2195 : {
2196 4 : PLF_CONFIG_INFO(PLF_DATA_OP, "[FuncBlockEnd] handle=%llu", handle);
2197 4 : auto it = funcMap_.find(handle);
2198 4 : if (it == funcMap_.end()) {
2199 0 : HCCL_ERROR("[CcuKernel::FuncBlockEnd] invalid func handle %lu", handle);
2200 0 : return CcuResult::CCU_E_PARA;
2201 : }
2202 4 : auto &desc = it->second;
2203 :
2204 4 : SetCurrentBlock(desc.prevActiveBlock);
2205 4 : inFuncBody_ = false;
2206 :
2207 4 : if (bodyError_ != CcuResult::CCU_SUCCESS) {
2208 0 : const CcuResult err = bodyError_;
2209 0 : HCCL_ERROR("[CcuKernel::FuncBlockEnd] illegal operation inside func body, err=%d", err);
2210 0 : bodyError_ = CcuResult::CCU_SUCCESS;
2211 0 : funcMap_.erase(it);
2212 0 : return err;
2213 : }
2214 :
2215 4 : Append(desc.repFuncBlock);
2216 4 : desc.bodyDefined = true;
2217 4 : funcInstanceMap_[desc.funcPtr] = handle;
2218 4 : return CcuResult::CCU_SUCCESS;
2219 : }
2220 :
2221 7 : CcuResult CcuKernel::FuncDefineInArg(uint64_t handle, CcuVariableHandle formal)
2222 : {
2223 7 : PLF_CONFIG_INFO(PLF_DATA_OP, "[FuncDefineInArg] handle=%llu, formal=%llu", handle, formal);
2224 7 : auto it = funcMap_.find(handle);
2225 7 : if (it == funcMap_.end()) {
2226 0 : HCCL_ERROR("[CcuKernel::FuncDefineInArg] invalid func handle %lu", handle);
2227 0 : return CcuResult::CCU_E_PARA;
2228 : }
2229 :
2230 7 : CcuRep::Variable *formalVar = nullptr;
2231 7 : CCU_CHK_RET(GetVariableByHandle(formal, &formalVar));
2232 7 : it->second.repFuncBlock->DefineInArg(*formalVar);
2233 7 : return CcuResult::CCU_SUCCESS;
2234 : }
2235 :
2236 5 : CcuResult CcuKernel::FuncCall(uint64_t handle, const CcuVariableHandle *inArgs, uint32_t numIn)
2237 : {
2238 5 : PLF_CONFIG_INFO(PLF_DATA_OP, "[FuncCall] handle=%llu, numIn=%u", handle, numIn);
2239 5 : if (CurrentBlock()->Type() == CcuRep::CcuRepType::LOOP_BLOCK || inFuncBody_) {
2240 0 : HCCL_ERROR("[CcuKernel::FuncCall] ccu::CallFunc only allowed at top level");
2241 0 : return LatchBodyError(CcuResult::CCU_E_INTERNAL);
2242 : }
2243 5 : if (numIn > 0 && inArgs == nullptr) {
2244 0 : HCCL_ERROR("[CcuKernel::FuncCall] null input args");
2245 0 : return CcuResult::CCU_E_PTR;
2246 : }
2247 :
2248 5 : auto it = funcMap_.find(handle);
2249 5 : if (it == funcMap_.end() || !it->second.bodyDefined) {
2250 0 : HCCL_ERROR("[CcuKernel::FuncCall] invalid func handle %lu", handle);
2251 0 : return CcuResult::CCU_E_PARA;
2252 : }
2253 :
2254 5 : auto repFuncCall = std::make_shared<CcuRep::CcuRepFuncCall>(insGenerator, it->second.label);
2255 12 : for (uint32_t i = 0; i < numIn; i++) {
2256 7 : CcuRep::Variable *actual = nullptr;
2257 7 : CCU_CHK_RET(GetVariableByHandle(inArgs[i], &actual));
2258 7 : repFuncCall->SetInArg(*actual);
2259 : }
2260 5 : Append(repFuncCall);
2261 5 : return CcuResult::CCU_SUCCESS;
2262 5 : }
2263 :
2264 : // 按 maxLoopNum 把 res_.blockExecutor[0] 扩容到至少 maxLoopNum 个 LoopEngine。
2265 : // 与 CreateBlockResAssist 对齐：所有 LoopEngine 资源先落在 die0 池，待实际 die 确定后
2266 : // 再由 MoveResourcesToDie 迁移到目标 die。
2267 : // 不同 LoopGroup 通过 local loopIdx 复用同一池低位 executorId，所以这里只
2268 : // "补足"而不是"累加"。
2269 28 : CcuResult CcuKernel::EnsureLoopEnginePool(uint32_t maxLoopNum)
2270 : {
2271 28 : if (maxLoopNum == 0) {
2272 0 : HCCL_ERROR("[CcuKernel::EnsureLoopEnginePool] maxLoopNum must be > 0");
2273 0 : return CcuResult::CCU_E_PARA;
2274 : }
2275 28 : constexpr uint32_t poolDieId = 0;
2276 28 : auto &loopEnginePool = res_.blockExecutor[poolDieId];
2277 28 : if (maxLoopNum <= loopEnginePool.size()) {
2278 10 : return CcuResult::CCU_SUCCESS;
2279 : }
2280 18 : const uint32_t deficit = maxLoopNum - static_cast<uint32_t>(loopEnginePool.size());
2281 18 : std::vector<CcuRep::Executor> tmp(deficit, CcuRep::Executor(this));
2282 18 : (void)CreateBlockExecutor(deficit, tmp.data());
2283 18 : return CcuResult::CCU_SUCCESS;
2284 18 : }
2285 :
2286 13 : CcuResult CcuKernel::LoopGroupCreate(CcuLoopGroup *group, uint32_t maxLoopNum,
2287 : const CcuLoopGroupCfg *cfg)
2288 : {
2289 13 : PLF_CONFIG_INFO(PLF_DATA_OP, "[LoopGroupCreate] maxLoopNum=%u", maxLoopNum);
2290 13 : if (group == nullptr || cfg == nullptr) {
2291 0 : HCCL_ERROR("[CcuKernel::LoopGroupCreate] null pointer");
2292 0 : return CcuResult::CCU_E_PTR;
2293 : }
2294 13 : if (CurrentBlock()->Type() == CcuRep::CcuRepType::LOOP_BLOCK) {
2295 0 : HCCL_ERROR("[CcuKernel::LoopGroupCreate] cannot create loop group inside a loop body");
2296 0 : return LatchBodyError(CcuResult::CCU_E_INTERNAL);
2297 : }
2298 13 : if (inFuncBody_) {
2299 0 : HCCL_ERROR("[CcuKernel::LoopGroupCreate] cannot create loop group inside a func body");
2300 0 : return LatchBodyError(CcuResult::CCU_E_INTERNAL);
2301 : }
2302 :
2303 : // 按需扩 LoopEngine 池；池足够则复用低位 executorId，跨组共享。
2304 13 : CCU_CHK_RET(EnsureLoopEnginePool(maxLoopNum));
2305 :
2306 13 : CcuLoopGroup handle = ++loopGroupHandleCounter_;
2307 :
2308 13 : LoopGroupDescriptor desc;
2309 13 : desc.config = *cfg;
2310 13 : desc.parallelVar = CreateVariable();
2311 13 : desc.offsetVar = CreateVariable();
2312 13 : desc.isVarBased = false;
2313 :
2314 : auto bundle = std::make_shared<CcuRep::CcuRepLoopGroupBundle>(
2315 13 : insGenerator, *cfg, desc.parallelVar, desc.offsetVar);
2316 13 : if (ccuVersion_ == CcuVersion::CCU_V2) {
2317 4 : bundle->SetXnOffsetVar(CreateVariable());
2318 : }
2319 13 : desc.bundleRep = bundle;
2320 13 : Append(bundle);
2321 :
2322 13 : loopGroupMap_[handle] = std::move(desc);
2323 13 : *group = handle;
2324 13 : return CcuResult::CCU_SUCCESS;
2325 13 : }
2326 :
2327 11 : CcuResult CcuKernel::LoopGroupCreateFromVar(CcuLoopGroup *group, uint32_t maxLoopNum,
2328 : CcuVariableHandle parallelVarHandle, CcuVariableHandle offsetVarHandle)
2329 : {
2330 11 : PLF_CONFIG_INFO(PLF_DATA_OP, "[LoopGroupCreateFromVar] maxLoopNum=%u, parallelVarHandle=%llu, offsetVarHandle=%llu",
2331 : maxLoopNum, parallelVarHandle, offsetVarHandle);
2332 11 : if (group == nullptr) {
2333 0 : HCCL_ERROR("[CcuKernel::LoopGroupCreateFromVar] null pointer for group");
2334 0 : return CcuResult::CCU_E_PTR;
2335 : }
2336 11 : if (CurrentBlock()->Type() == CcuRep::CcuRepType::LOOP_BLOCK) {
2337 0 : HCCL_ERROR("[CcuKernel::LoopGroupCreateFromVar] cannot create loop group inside a loop body");
2338 0 : return LatchBodyError(CcuResult::CCU_E_INTERNAL);
2339 : }
2340 11 : if (inFuncBody_) {
2341 0 : HCCL_ERROR("[CcuKernel::LoopGroupCreateFromVar] cannot create loop group inside a func body");
2342 0 : return LatchBodyError(CcuResult::CCU_E_INTERNAL);
2343 : }
2344 :
2345 11 : CCU_CHK_RET(EnsureLoopEnginePool(maxLoopNum));
2346 :
2347 11 : CcuRep::Variable *parallelVarPtr = nullptr;
2348 11 : CcuRep::Variable *offsetVarPtr = nullptr;
2349 11 : CCU_CHK_RET(GetVariableByHandle(parallelVarHandle, ¶llelVarPtr));
2350 11 : CCU_CHK_RET(GetVariableByHandle(offsetVarHandle, &offsetVarPtr));
2351 :
2352 11 : CcuLoopGroup handle = ++loopGroupHandleCounter_;
2353 :
2354 11 : LoopGroupDescriptor desc;
2355 11 : desc.parallelVar = CcuRep::Variable(*parallelVarPtr);
2356 11 : desc.offsetVar = CcuRep::Variable(*offsetVarPtr);
2357 11 : desc.isVarBased = true;
2358 :
2359 : auto bundle = std::make_shared<CcuRep::CcuRepLoopGroupBundle>(
2360 11 : insGenerator, desc.parallelVar, desc.offsetVar);
2361 11 : if (ccuVersion_ == CcuVersion::CCU_V2) {
2362 1 : bundle->SetCompatRemapVars(CreateVariable(), CreateVariable());
2363 : }
2364 11 : desc.bundleRep = bundle;
2365 11 : Append(bundle);
2366 :
2367 11 : loopGroupMap_[handle] = std::move(desc);
2368 11 : *group = handle;
2369 11 : return CcuResult::CCU_SUCCESS;
2370 11 : }
2371 :
2372 5 : CcuResult CcuKernel::LoopGroupCreateFromVarV2(CcuLoopGroup *group, uint32_t maxLoopNum,
2373 : CcuVariableHandle parallelVarV2Handle, CcuVariableHandle offsetVarV2Handle, CcuVariableHandle varOffsetVarHandle)
2374 : {
2375 5 : if (ccuVersion_ != CcuVersion::CCU_V2) {
2376 1 : HCCL_ERROR("[CcuKernel::LoopGroupCreateFromVarV2] only supported on V2");
2377 1 : return CcuResult::CCU_E_NOT_SUPPORT;
2378 : }
2379 4 : if (group == nullptr) {
2380 0 : HCCL_ERROR("[CcuKernel::LoopGroupCreateFromVarV2] null pointer for group");
2381 0 : return CcuResult::CCU_E_PTR;
2382 : }
2383 4 : if (CurrentBlock()->Type() == CcuRep::CcuRepType::LOOP_BLOCK) {
2384 0 : HCCL_ERROR("[CcuKernel::LoopGroupCreateFromVarV2] cannot create loop group inside a loop body");
2385 0 : return LatchBodyError(CcuResult::CCU_E_INTERNAL);
2386 : }
2387 4 : if (inFuncBody_) {
2388 0 : HCCL_ERROR("[CcuKernel::LoopGroupCreateFromVarV2] cannot create loop group inside a func body");
2389 0 : return LatchBodyError(CcuResult::CCU_E_INTERNAL);
2390 : }
2391 :
2392 4 : CCU_CHK_RET(EnsureLoopEnginePool(maxLoopNum));
2393 :
2394 4 : CcuRep::Variable *parallelVarPtr = nullptr;
2395 4 : CcuRep::Variable *offsetVarPtr = nullptr;
2396 4 : CcuRep::Variable *xnOffsetVarPtr = nullptr;
2397 4 : CCU_CHK_RET(GetVariableByHandle(parallelVarV2Handle, ¶llelVarPtr));
2398 4 : CCU_CHK_RET(GetVariableByHandle(offsetVarV2Handle, &offsetVarPtr));
2399 4 : CCU_CHK_RET(GetVariableByHandle(varOffsetVarHandle, &xnOffsetVarPtr));
2400 :
2401 4 : CcuLoopGroup handle = ++loopGroupHandleCounter_;
2402 :
2403 4 : LoopGroupDescriptor desc;
2404 4 : desc.parallelVar = CcuRep::Variable(*parallelVarPtr);
2405 4 : desc.offsetVar = CcuRep::Variable(*offsetVarPtr);
2406 4 : desc.xnOffsetVar = CcuRep::Variable(*xnOffsetVarPtr);
2407 4 : desc.isVarBased = true;
2408 4 : desc.isVersionV2 = true;
2409 :
2410 : auto bundle = std::make_shared<CcuRep::CcuRepLoopGroupBundle>(
2411 4 : insGenerator, desc.parallelVar, desc.offsetVar);
2412 4 : bundle->SetLayout(CcuRep::CcuRepLoopGroupBundle::Layout::VersionV2);
2413 4 : bundle->SetXnOffsetVar(desc.xnOffsetVar);
2414 4 : desc.bundleRep = bundle;
2415 4 : Append(bundle);
2416 :
2417 4 : loopGroupMap_[handle] = std::move(desc);
2418 4 : *group = handle;
2419 4 : return CcuResult::CCU_SUCCESS;
2420 4 : }
2421 :
2422 56 : CcuResult CcuKernel::LookupLoopGroupAndLoop(CcuLoopGroup group, CcuLoop loop,
2423 : const char *fnName, const char *createFnName,
2424 : LoopGroupDescriptor *&grpDesc, LoopDescriptor *&loopDesc,
2425 : uint32_t &loopIdx)
2426 : {
2427 56 : auto grpIt = loopGroupMap_.find(group);
2428 56 : if (grpIt == loopGroupMap_.end()) {
2429 0 : HCCL_ERROR("[CcuKernel::%s] invalid group handle %lu", fnName, group);
2430 0 : return CcuResult::CCU_E_PARA;
2431 : }
2432 56 : grpDesc = &grpIt->second;
2433 :
2434 56 : auto loopIt = loopMap_.find(loop);
2435 56 : if (loopIt == loopMap_.end()) {
2436 0 : HCCL_ERROR("[CcuKernel::%s] invalid loop handle %lu", fnName, loop);
2437 0 : return CcuResult::CCU_E_PARA;
2438 : }
2439 56 : loopDesc = &loopIt->second;
2440 :
2441 56 : if (!loopDesc->bodyDefined) {
2442 0 : HCCL_ERROR("[CcuKernel::%s] loop %lu body not defined", fnName, loop);
2443 0 : return CcuResult::CCU_E_INTERNAL; // CCU_E_LOOP_BODY_UNDEFINED
2444 : }
2445 :
2446 56 : auto &loopEnginePool = res_.blockExecutor[0];
2447 56 : loopIdx = grpDesc->loopCount;
2448 56 : if (loopIdx >= loopEnginePool.size()) {
2449 0 : HCCL_ERROR("[CcuKernel::%s] loopEngine pool exhausted (pool size %zu, loopIdx %u). "
2450 : "Pass a larger maxLoopNum to %s so the pool can be extended at create time.",
2451 : fnName, loopEnginePool.size(), loopIdx, createFnName);
2452 0 : return CcuResult::CCU_E_PARA;
2453 : }
2454 56 : return CcuResult::CCU_SUCCESS;
2455 : }
2456 :
2457 25 : CcuResult CcuKernel::LoopGroupAddLoop(CcuLoopGroup group, CcuLoop loop, const CcuLoopCfg *cfg)
2458 : {
2459 25 : PLF_CONFIG_INFO(PLF_DATA_OP, "[LoopGroupAddLoop] group=%llu, loop=%llu", group, loop);
2460 25 : if (cfg == nullptr) {
2461 0 : HCCL_ERROR("[CcuKernel::LoopGroupAddLoop] null pointer for cfg");
2462 0 : return CcuResult::CCU_E_PTR;
2463 : }
2464 25 : LoopGroupDescriptor *grpDesc = nullptr;
2465 25 : LoopDescriptor *loopDesc = nullptr;
2466 25 : uint32_t loopIdx = 0;
2467 25 : CCU_CHK_RET(LookupLoopGroupAndLoop(group, loop, "LoopGroupAddLoop", "CcuLoopGroupCreate",
2468 : grpDesc, loopDesc, loopIdx));
2469 25 : auto &loopEnginePool = res_.blockExecutor[0];
2470 :
2471 25 : grpDesc->loopCount++;
2472 25 : grpDesc->totalLoopNum = grpDesc->loopCount;
2473 :
2474 25 : CcuRep::CcuRepLoopGroupBundle::LoopEntry entry;
2475 25 : entry.config = *cfg;
2476 25 : entry.executor = loopEnginePool[loopIdx];
2477 25 : entry.repLoopBlock = loopDesc->repLoopBlock;
2478 25 : entry.loopParamVar = CreateVariable();
2479 25 : entry.layout = CcuRep::CcuRepLoopGroupBundle::Layout::Config;
2480 25 : if (ccuVersion_ == CcuVersion::CCU_V2) {
2481 7 : entry.iterNumVar = CreateVariable();
2482 7 : entry.addrOffsetVar = CreateVariable();
2483 7 : entry.ctxIdVar = CreateVariable();
2484 : }
2485 :
2486 25 : auto bundle = std::static_pointer_cast<CcuRep::CcuRepLoopGroupBundle>(grpDesc->bundleRep);
2487 25 : bundle->AddLoop(entry);
2488 :
2489 25 : if (!grpDesc->isVarBased) {
2490 19 : bundle->SetRepeatLoopIdx(grpDesc->config.cloneLoopOffset);
2491 19 : bundle->SetTotalLoopNum(grpDesc->totalLoopNum);
2492 : }
2493 :
2494 25 : return CcuResult::CCU_SUCCESS;
2495 25 : }
2496 :
2497 23 : CcuResult CcuKernel::LoopGroupAddLoopFromVar(CcuLoopGroup group,
2498 : CcuLoop loop, CcuVariableHandle loopParamVarHandle)
2499 : {
2500 23 : PLF_CONFIG_INFO(PLF_DATA_OP, "[LoopGroupAddLoopFromVar] group=%llu, loop=%llu, loopParamVarHandle=%llu",
2501 : group, loop, loopParamVarHandle);
2502 23 : LoopGroupDescriptor *grpDesc = nullptr;
2503 23 : LoopDescriptor *loopDesc = nullptr;
2504 23 : uint32_t loopIdx = 0;
2505 23 : CCU_CHK_RET(LookupLoopGroupAndLoop(group, loop, "LoopGroupAddLoopFromVar", "CcuLoopGroupCreateFromVar",
2506 : grpDesc, loopDesc, loopIdx));
2507 23 : auto &loopEnginePool = res_.blockExecutor[0];
2508 :
2509 23 : CcuRep::Variable *loopParamVarPtr = nullptr;
2510 23 : CCU_CHK_RET(GetVariableByHandle(loopParamVarHandle, &loopParamVarPtr));
2511 :
2512 23 : CcuRep::CcuRepLoopGroupBundle::LoopEntry entry;
2513 23 : entry.executor = loopEnginePool[loopIdx];
2514 23 : entry.repLoopBlock = loopDesc->repLoopBlock;
2515 23 : entry.loopParamVar = CcuRep::Variable(*loopParamVarPtr);
2516 23 : entry.layout = CcuRep::CcuRepLoopGroupBundle::Layout::PackedVar;
2517 23 : if (ccuVersion_ == CcuVersion::CCU_V2) {
2518 4 : entry.iterNumVar = CreateVariable();
2519 4 : entry.addrOffsetVar = CreateVariable();
2520 4 : entry.ctxIdVar = CreateVariable();
2521 : }
2522 :
2523 23 : auto bundle = std::static_pointer_cast<CcuRep::CcuRepLoopGroupBundle>(grpDesc->bundleRep);
2524 23 : bundle->AddLoop(entry);
2525 :
2526 : // 计数放在入 bundle 之后统一更新：totalLoopNum 仅供 config 组同步编码使用，与 AddLoop 无先后依赖。
2527 23 : grpDesc->loopCount++;
2528 23 : grpDesc->totalLoopNum = grpDesc->loopCount;
2529 23 : if (!grpDesc->isVarBased) {
2530 7 : bundle->SetRepeatLoopIdx(grpDesc->config.cloneLoopOffset);
2531 7 : bundle->SetTotalLoopNum(grpDesc->totalLoopNum);
2532 : }
2533 :
2534 23 : return CcuResult::CCU_SUCCESS;
2535 23 : }
2536 :
2537 8 : CcuResult CcuKernel::LoopGroupAddLoopFromVarV2(CcuLoopGroup group,
2538 : CcuLoop loop, CcuVariableHandle iterNumVarHandle, CcuVariableHandle addrOffsetVarHandle,
2539 : CcuVariableHandle ctxIdVarHandle)
2540 : {
2541 8 : if (ccuVersion_ != CcuVersion::CCU_V2) {
2542 0 : HCCL_ERROR("[CcuKernel::LoopGroupAddLoopFromVarV2] only supported on V2");
2543 0 : return CcuResult::CCU_E_NOT_SUPPORT;
2544 : }
2545 :
2546 8 : LoopGroupDescriptor *grpDesc = nullptr;
2547 8 : LoopDescriptor *loopDesc = nullptr;
2548 8 : uint32_t loopIdx = 0;
2549 8 : CCU_CHK_RET(LookupLoopGroupAndLoop(group, loop, "LoopGroupAddLoopFromVarV2", "CcuLoopGroupCreateFromVarV2",
2550 : grpDesc, loopDesc, loopIdx));
2551 8 : auto &loopEnginePool = res_.blockExecutor[0];
2552 :
2553 8 : CcuRep::Variable *iterNumVarPtr = nullptr;
2554 8 : CcuRep::Variable *addrOffsetVarPtr = nullptr;
2555 8 : CcuRep::Variable *ctxIdVarPtr = nullptr;
2556 8 : CCU_CHK_RET(GetVariableByHandle(iterNumVarHandle, &iterNumVarPtr));
2557 8 : CCU_CHK_RET(GetVariableByHandle(addrOffsetVarHandle, &addrOffsetVarPtr));
2558 8 : CCU_CHK_RET(GetVariableByHandle(ctxIdVarHandle, &ctxIdVarPtr));
2559 :
2560 8 : grpDesc->loopCount++;
2561 8 : grpDesc->totalLoopNum = grpDesc->loopCount;
2562 :
2563 8 : VersionV2LoopRecord record;
2564 8 : record.iterNumVar = CcuRep::Variable(*iterNumVarPtr);
2565 8 : record.addrOffsetVar = CcuRep::Variable(*addrOffsetVarPtr);
2566 8 : record.ctxIdVar = CcuRep::Variable(*ctxIdVarPtr);
2567 8 : grpDesc->versionV2Loops.push_back(record);
2568 :
2569 8 : CcuRep::CcuRepLoopGroupBundle::LoopEntry entry;
2570 8 : entry.executor = loopEnginePool[loopIdx];
2571 8 : entry.repLoopBlock = loopDesc->repLoopBlock;
2572 8 : entry.iterNumVar = CcuRep::Variable(*iterNumVarPtr);
2573 8 : entry.addrOffsetVar = CcuRep::Variable(*addrOffsetVarPtr);
2574 8 : entry.ctxIdVar = CcuRep::Variable(*ctxIdVarPtr);
2575 8 : entry.layout = CcuRep::CcuRepLoopGroupBundle::Layout::VersionV2;
2576 :
2577 8 : auto bundle = std::static_pointer_cast<CcuRep::CcuRepLoopGroupBundle>(grpDesc->bundleRep);
2578 8 : bundle->AddLoop(entry);
2579 :
2580 8 : if (!grpDesc->isVarBased) {
2581 2 : bundle->SetRepeatLoopIdx(grpDesc->config.cloneLoopOffset);
2582 2 : bundle->SetTotalLoopNum(grpDesc->totalLoopNum);
2583 : }
2584 :
2585 8 : return CcuResult::CCU_SUCCESS;
2586 8 : }
2587 :
2588 33 : void CcuKernel::SetInstrId(uint32_t instrId)
2589 : {
2590 33 : instrInfo_.startInstrId = instrId;
2591 33 : }
2592 :
2593 71 : uint32_t CcuKernel::GetInstrId() const
2594 : {
2595 71 : return instrInfo_.startInstrId;
2596 : }
2597 :
2598 69 : uint32_t CcuKernel::GetInstrCount()
2599 : {
2600 69 : uint32_t instrCount = 0;
2601 1519 : for (const auto &rep : GetRepSequence()) {
2602 1450 : instrCount += rep->InstrCount();
2603 : }
2604 69 : instrInfo_.instrCount = instrCount;
2605 69 : HCCL_INFO("Kernel inst %u", instrCount);
2606 69 : return instrCount;
2607 : }
2608 :
2609 32 : void CcuKernel::SetCcuInstrInfo(const CcuRep::CcuInstrInfo &instrInfo)
2610 : {
2611 32 : this->instrInfo_ = instrInfo;
2612 32 : }
2613 :
2614 346 : CcuRep::Variable CcuKernel::CreateVariable()
2615 : {
2616 346 : return CreateResAssist(res_.continuousVariable);
2617 : }
2618 :
2619 37 : CcuRep::Variable CcuKernel::CreateExpectVar()
2620 : {
2621 : // v2(A6) 的 jump 只支持 var-var 比较，需要真实 XN 承载立即数；
2622 : // v1(A5) 的 jump 直接支持 var-imm，expectVar 不参与翻译，
2623 : // 用不进 res_ 账本的壳 Variable 即可，避免 v1 资源虚高。
2624 37 : if (ccuVersion_ == CcuVersion::CCU_V2) {
2625 12 : return CreateVariable();
2626 : }
2627 25 : return CcuRep::Variable(this);
2628 : }
2629 :
2630 201 : CcuRep::Address CcuKernel::CreateAddress()
2631 : {
2632 201 : if (ccuVersion_ == CcuVersion::CCU_V2) {
2633 : // A6创建Address时，需要添加到Variable的列表中，但是仍以Address返回
2634 91 : return CcuRep::Address(CreateResAssist(res_.continuousVariable));
2635 : }
2636 110 : return CreateResAssist(res_.blockAddress);
2637 : }
2638 :
2639 0 : CcuRep::LocalNotify CcuKernel::CreateLocalNotify()
2640 : {
2641 0 : return CreateResAssist(res_.localNotify);
2642 : }
2643 :
2644 0 : CcuRep::CompletedEvent CcuKernel::CreateCompletedEvent()
2645 : {
2646 0 : return CreateResAssist(res_.blockCompletedEvent);
2647 : }
2648 :
2649 0 : CcuRep::CcuBuf CcuKernel::CreateCcuBuf()
2650 : {
2651 0 : return CreateResAssist(res_.blockCcubufs);
2652 : }
2653 :
2654 0 : CcuRep::Executor CcuKernel::CreateExecutor()
2655 : {
2656 0 : return CreateResAssist(res_.blockExecutor);
2657 : }
2658 :
2659 154 : CcuRep::LocalAddr CcuKernel::CreateLocalAddr()
2660 : {
2661 154 : return CcuRep::LocalAddr(CreateAddress(), CreateVariable());
2662 : }
2663 :
2664 37 : CcuRep::RemoteAddr CcuKernel::CreateRemoteAddr()
2665 : {
2666 37 : return CcuRep::RemoteAddr(CreateAddress(), CreateVariable());
2667 : }
2668 :
2669 0 : CcuRep::RemoteAddr CcuKernel::GetRemoteAddr(const ChannelHandle channel, uint32_t index)
2670 : {
2671 : (void)index;
2672 0 : channels_.insert(channel);
2673 0 : auto mem = CcuRep::RemoteAddr(CreateAddress(), CreateVariable());
2674 0 : Append(std::make_shared<CcuRep::CcuRepRemMem>(insGenerator, channel, mem));
2675 0 : return mem;
2676 0 : }
2677 :
2678 0 : CcuRep::LocalAddr CcuKernel::CreateLocalAddr(const CcuRep::Variable &token)
2679 : {
2680 0 : return CcuRep::LocalAddr(CreateAddress(), token);
2681 : }
2682 :
2683 0 : HcclResult CcuKernel::CreateBlockCcuBuf(const uint32_t count, CcuRep::CcuBuf *ccuBufs)
2684 : {
2685 0 : CHK_PTR_NULL(ccuBufs);
2686 0 : auto resources = CreateBlockResAssist(count, res_.blockCcubufs);
2687 :
2688 0 : for (uint32_t i = 0; i < count; i++) {
2689 0 : ccuBufs[i] = resources[i]; // 拷贝虚拟资源，通过shared_ptr链接到物理资源
2690 : }
2691 :
2692 0 : return HcclResult::HCCL_SUCCESS;
2693 0 : }
2694 :
2695 18 : HcclResult CcuKernel::CreateBlockExecutor(const uint32_t count, CcuRep::Executor *ccuExes)
2696 : {
2697 18 : CHK_PTR_NULL(ccuExes);
2698 18 : auto resources = CreateBlockResAssist(count, res_.blockExecutor);
2699 :
2700 48 : for (uint32_t i = 0; i < count; i++) {
2701 30 : ccuExes[i] = resources[i]; // 拷贝虚拟资源，通过shared_ptr链接到物理资源
2702 : }
2703 :
2704 18 : return HcclResult::HCCL_SUCCESS;
2705 18 : }
2706 :
2707 0 : HcclResult CcuKernel::CreateBlockCompletedEvent(const uint32_t count, CcuRep::CompletedEvent *ccuEvents)
2708 : {
2709 0 : CHK_PTR_NULL(ccuEvents);
2710 0 : auto resources = CreateBlockResAssist(count, res_.blockCompletedEvent);
2711 :
2712 0 : for (uint32_t i = 0; i < count; i++) {
2713 0 : ccuEvents[i] = resources[i]; // 拷贝虚拟资源，通过shared_ptr链接到物理资源
2714 : }
2715 :
2716 0 : return HcclResult::HCCL_SUCCESS;
2717 0 : }
2718 :
2719 33 : void CcuKernel::SetResRepository(const CcuResRepository &resRepo)
2720 : {
2721 33 : resRepo_ = resRepo;
2722 33 : }
2723 :
2724 33 : CcuResRepository &CcuKernel::GetResRepository()
2725 : {
2726 33 : return resRepo_;
2727 : }
2728 :
2729 33 : CcuSharedResource &CcuKernel::GetExportedRes()
2730 : {
2731 33 : return exportedRes_;
2732 : }
2733 :
2734 33 : CcuSharedResource &CcuKernel::GetImportedRes()
2735 : {
2736 33 : return importedRes_;
2737 : }
2738 :
2739 0 : static HcclResult GetArgIndex(const std::unordered_map<uint16_t, uint16_t> &varId2VarIdMap,
2740 : const std::unordered_map<uint16_t, uint32_t> &varId2ArgIndexMap,
2741 : const uint64_t *taskArgs, uint32_t argSize,
2742 : uint16_t varId, uint64_t& argIndex)
2743 : {
2744 0 : HCCL_INFO("[GetArgIndex] Enter varId(%u)", varId);
2745 0 : auto item = varId2ArgIndexMap.find(varId);
2746 0 : if (item == varId2ArgIndexMap.end()) {
2747 0 : uint16_t oriVarId = varId;
2748 0 : auto iter = varId2VarIdMap.find(varId);
2749 0 : while (iter != varId2VarIdMap.end()) { // 循环查找中间assign Rep，找到起始varId
2750 0 : oriVarId = iter->second;
2751 0 : iter = varId2VarIdMap.find(oriVarId);
2752 : }
2753 0 : if (oriVarId != varId) { // 起始varId预期通过LoadArg赋值
2754 0 : item = varId2ArgIndexMap.find(oriVarId);
2755 0 : if (item == varId2ArgIndexMap.end()) {
2756 0 : HCCL_ERROR("[%s]fail, Invalid goSize variable id(%u), oriVarId = %u", __func__, varId, oriVarId);
2757 0 : return HCCL_E_PARA;
2758 : }
2759 : } else {
2760 0 : HCCL_ERROR("[%s]fail, Invalid goSize variable id(%u)", __func__, varId);
2761 0 : return HCCL_E_PARA;
2762 : }
2763 : }
2764 0 : HCCL_INFO("[GetArgIndex] find end");
2765 0 : if (item->second >= argSize) {
2766 0 : HCCL_ERROR("Invalid goSize variable index(%u).", item->second);
2767 0 : return HCCL_E_PARA;
2768 : }
2769 0 : HCCL_INFO(
2770 : "GetArgIndex success: varId(%u) varId2VarIdMapSize(%u) varId2ArgIndexMapSize(%u) taskArgsSize(%u)",
2771 : varId, varId2VarIdMap.size(), varId2ArgIndexMap.size(), argSize);
2772 0 : argIndex = taskArgs[item->second];
2773 0 : return HCCL_SUCCESS;
2774 : }
2775 :
2776 5 : void DumpCcuProfilingInfo(const std::vector<CcuProfilingInfo> &ccuProfilingInfo)
2777 : {
2778 9 : auto dumpLinkInfo = [] (const CcuProfilingInfo &info) -> void {
2779 153 : for (int i = 0; i < CCU_MAX_CHANNEL_NUM; i++) {
2780 144 : if (info.channelId[i] == INVALID_VALUE_CHANNELID) {
2781 138 : continue;
2782 : }
2783 6 : HCCL_INFO("channelId(%u), remoteRankId(%u).", info.channelId[i], info.remoteRankId[i]);
2784 : }
2785 9 : };
2786 :
2787 19 : for (const auto &profInfo : ccuProfilingInfo) {
2788 14 : if (profInfo.type == static_cast<uint8_t>(CcuProfilinType::CCU_TASK_PROFILING)) {
2789 5 : HCCL_INFO("Dump CCU Profiling Info:SQE Profiling Info: ctxSignautre(%s), "
2790 : "dieId(%d), missionId(%d), instrId(%d).",
2791 : profInfo.name.c_str(), static_cast<int>(profInfo.dieId), static_cast<int>(profInfo.missionId),
2792 : static_cast<int>(profInfo.instrId));
2793 9 : } else if (profInfo.type == static_cast<uint8_t>(CcuProfilinType::CCU_WAITCKE_PROFILING)) {
2794 9 : HCCL_INFO("Microcode WaitCKE Profiling Info: name(%s), "
2795 : "dieId(%d), missionId(%d), instrId(%d), ckeId(%u), mask(%u).",
2796 : profInfo.name.c_str(), static_cast<int>(profInfo.dieId), static_cast<int>(profInfo.missionId),
2797 : static_cast<int>(profInfo.instrId), profInfo.ckeId, profInfo.mask);
2798 9 : dumpLinkInfo(profInfo);
2799 0 : } else if (profInfo.type == static_cast<uint8_t>(CcuProfilinType::CCU_LOOPGROUP_PROFILING)) {
2800 0 : HCCL_INFO("Microcode LoopGroup Profiling Info: name(%s), "
2801 : "dieId(%d), missionId(%d), instrId(%d), reduceOpType(%d), inputDataType(%d), "
2802 : "outputDataType(%d), dataSize(%llu).",
2803 : profInfo.name.c_str(), static_cast<int>(profInfo.dieId), static_cast<int>(profInfo.missionId),
2804 : static_cast<int>(profInfo.instrId), static_cast<int>(profInfo.reduceOpType),
2805 : static_cast<int>(profInfo.inputDataType), static_cast<int>(profInfo.outputDataType),
2806 : profInfo.dataSize);
2807 0 : dumpLinkInfo(profInfo);
2808 : }
2809 : }
2810 5 : }
2811 :
2812 0 : constexpr uint64_t SetBits(uint16_t end)
2813 : {
2814 0 : return ((uint64_t(1) << (end + 1)) - uint64_t(1));
2815 : }
2816 :
2817 0 : static uint16_t ParseRepeatNumFromParallelParam(uint64_t parallelParam)
2818 : {
2819 0 : constexpr uint16_t repeatBitNum = 7; // 7： repeat num 占 7 bits
2820 0 : constexpr uint16_t repeatNumShiftBit = 55; // 55： repeat num占[61:55]位置
2821 0 : return ( parallelParam >> repeatNumShiftBit) & SetBits(repeatBitNum);
2822 : }
2823 :
2824 5 : HcclResult CcuKernel::CollectSqeAndWaitCkeProfilingInfo()
2825 : {
2826 5 : auto &ccuProfilingCache = GetProfilingInfo();
2827 5 : uint32_t count {0};
2828 5 : HCCL_INFO("[GetCcuProfilingInfo] Process sqe&waitcke profiling info start.");
2829 19 : for (auto &profInfo : ccuProfilingCache) {
2830 14 : profInfo.missionId = GetMissionId();
2831 14 : if (profInfo.type == static_cast<uint8_t>(hcomm::CcuProfilinType::CCU_TASK_PROFILING)) {
2832 5 : profInfo.instrId = GetInstrId();
2833 5 : allCcuProfilingInfos_.push_back(profInfo);
2834 5 : continue;
2835 : }
2836 9 : if (count >= GetWaiteCkeProfilingReps().size()) {
2837 0 : HCCL_ERROR("count[%u] out of range[0, %u], cache size(%u).", count,
2838 : GetWaiteCkeProfilingReps().size(), ccuProfilingCache.size());
2839 0 : return HCCL_E_INTERNAL;
2840 : }
2841 9 : auto waitCkeRep = GetWaiteCkeProfilingReps()[count];
2842 9 : profInfo.instrId = waitCkeRep->StartInstrId();
2843 9 : if (profInfo.ckeId == INVALID_CKE_ID) { // localWait Rep
2844 3 : if (waitCkeRep.get() == nullptr) {
2845 0 : HCCL_ERROR("[GetCcuProfilingInfo] localWaitRep is nullptr.");
2846 0 : return HCCL_E_PTR;
2847 : }
2848 3 : profInfo.ckeId = waitCkeRep->GetId();
2849 3 : HCCL_INFO("[CcuKernel][GetCcuProfilingInfo] waitcke[%u]", profInfo.ckeId);
2850 : }
2851 9 : allCcuProfilingInfos_.push_back(profInfo);
2852 9 : count++;
2853 9 : }
2854 5 : return HCCL_SUCCESS;
2855 : }
2856 :
2857 5 : HcclResult CcuKernel::BuildLoopGroupVarIdMaps(std::unordered_map<uint16_t, uint32_t> &varId2ArgIndexMap,
2858 : std::unordered_map<uint16_t, uint16_t> &varId2VarIdMap)
2859 : {
2860 5 : auto &lgProfInfo = GetLGProfilingInfo();
2861 5 : HCCL_INFO("[GetCcuProfilingInfo] create varId2ArgIndexMap start. size=%lu",
2862 : lgProfInfo.loadRep2ArgIdxMap.size());
2863 5 : for (auto &iter : lgProfInfo.loadRep2ArgIdxMap) {
2864 0 : if (iter.first.get() == nullptr) {
2865 0 : HCCL_ERROR("[GetCcuProfilingInfo] loadRep is nullptr.");
2866 0 : return HCCL_E_PTR;
2867 : }
2868 0 : auto loadRep = dynamic_cast<CcuRep::CcuRepLoadArg*>(iter.first.get());
2869 0 : varId2ArgIndexMap[loadRep->GetVarId()] = iter.second;
2870 : }
2871 :
2872 5 : HCCL_INFO("[GetCcuProfilingInfo] create varId2VarIdMap start. size=%lu",
2873 : lgProfInfo.assignProfilingReps.size());
2874 80 : for (auto &iter : lgProfInfo.assignProfilingReps) {
2875 75 : if (iter.get() == nullptr) {
2876 0 : HCCL_ERROR("[GetCcuProfilingInfo] assignRep is nullptr.");
2877 0 : return HCCL_E_PTR;
2878 : }
2879 75 : auto assignRep = dynamic_cast<CcuRep::CcuRepAssign*>(iter.get());
2880 75 : varId2VarIdMap[assignRep->GetVarB().Id()] = assignRep->GetVarA().Id();
2881 : }
2882 5 : return HCCL_SUCCESS;
2883 : }
2884 :
2885 5 : HcclResult CcuKernel::CollectLoopGroupProfilingInfo(const uint64_t *taskArgs, uint32_t argSize,
2886 : const std::unordered_map<uint16_t, uint32_t> &varId2ArgIndexMap,
2887 : const std::unordered_map<uint16_t, uint16_t> &varId2VarIdMap)
2888 : {
2889 5 : auto &lgProfInfo = GetLGProfilingInfo();
2890 5 : HCCL_INFO("[GetCcuProfilingInfo] process loop group profiling start: "
2891 : "lgsize(%lu), goSize(%lu)", lgProfInfo.lgProfilingReps.size(), groupOpSizeInfo_.size());
2892 5 : for (uint32_t i = 0; i < lgProfInfo.lgProfilingReps.size(); i += 2) { // 2: 一个goSize对应一个CcuProfilingInfo，对应1个loopGroup Rep
2893 0 : if (argSize == 0 || varId2ArgIndexMap.empty()) {
2894 0 : continue;
2895 : }
2896 0 : uint64_t loopParam {0};
2897 0 : CHK_RET(GetArgIndex(varId2VarIdMap, varId2ArgIndexMap, taskArgs, argSize,
2898 : groupOpSizeInfo_[i].loopParamId, loopParam));
2899 0 : uint64_t parallelParam {0};
2900 0 : CHK_RET(GetArgIndex(varId2VarIdMap, varId2ArgIndexMap, taskArgs, argSize,
2901 : groupOpSizeInfo_[i].parallelParamId, parallelParam));
2902 0 : HCCL_INFO("Collect loopgroup profiling info: repSize[%u], index[%u],"
2903 : "loopParam[%llu], parallelParam[%llu].",
2904 : lgProfInfo.lgProfilingReps.size(), i, loopParam, parallelParam);
2905 :
2906 0 : if (loopParam != 0) {
2907 0 : lgProfInfo.ccuProfilingInfos[i].dataSize = loopParam * moConfig_.loopCount * moConfig_.memSlice;
2908 0 : lgProfInfo.ccuProfilingInfos[i].instrId = dynamic_cast<CcuRep::CcuRepLoopGroupBundle*>(lgProfInfo.lgProfilingReps[i].get())->StartInstrId();
2909 0 : allCcuProfilingInfos_.push_back(lgProfInfo.ccuProfilingInfos[i]);
2910 : }
2911 :
2912 0 : if (parallelParam != 0) {
2913 0 : HCCL_INFO("[GetCcuProfilingInfo] collect lg, residual start i=%lu", i);
2914 0 : uint64_t residual {0};
2915 0 : CHK_RET(GetArgIndex(varId2VarIdMap, varId2ArgIndexMap, taskArgs, argSize,
2916 : groupOpSizeInfo_[i].residualId, residual));
2917 0 : uint64_t repeatNum = ParseRepeatNumFromParallelParam(parallelParam);
2918 0 : lgProfInfo.ccuProfilingInfos[i].dataSize = repeatNum * moConfig_.memSlice + residual;
2919 0 : lgProfInfo.ccuProfilingInfos[i].instrId = dynamic_cast<CcuRep::CcuRepLoopGroupBundle*>(lgProfInfo.lgProfilingReps[i + 1].get())->StartInstrId();
2920 0 : allCcuProfilingInfos_.push_back(lgProfInfo.ccuProfilingInfos[i]);
2921 : }
2922 : }
2923 5 : return HCCL_SUCCESS;
2924 : }
2925 :
2926 : /*
2927 : * variable/maskSignal等资源变量Id，一定要在获取ccu profiling时才获取；
2928 : * 原因：在创建context Rep时，其资源Id属于虚拟资源；翻译时，才会绑定固定的物理资源。
2929 : */
2930 5 : HcclResult CcuKernel::GetCcuProfilingInfo(const uint64_t *taskArgs, uint32_t argSize,
2931 : std::vector<CcuProfilingInfo> &allCcuProfilingInfo)
2932 : {
2933 5 : HCCL_INFO("[GetCcuProfilingInfo] Enter.");
2934 5 : allCcuProfilingInfos_.clear();
2935 :
2936 5 : CHK_RET(CollectSqeAndWaitCkeProfilingInfo());
2937 :
2938 5 : std::unordered_map<uint16_t, uint32_t> varId2ArgIndexMap;
2939 5 : std::unordered_map<uint16_t, uint16_t> varId2VarIdMap;
2940 5 : CHK_RET(BuildLoopGroupVarIdMaps(varId2ArgIndexMap, varId2VarIdMap));
2941 :
2942 5 : CHK_RET(CollectLoopGroupProfilingInfo(taskArgs, argSize, varId2ArgIndexMap, varId2VarIdMap));
2943 :
2944 5 : DumpCcuProfilingInfo(allCcuProfilingInfos_);
2945 5 : allCcuProfilingInfo = allCcuProfilingInfos_;
2946 5 : return HCCL_SUCCESS;
2947 5 : }
2948 :
2949 0 : HcclResult CcuKernel::AddProfilingInfo(const ChannelHandle *channels, uint32_t channelNum, HcclDataType dataType,
2950 : HcclDataType outputDataType, HcclReduceOp opType, const std::string& opName)
2951 : {
2952 0 : CHK_PTR_NULL(channels);
2953 0 : ccuProfilingInfoCache.type = (uint8_t)CcuProfilinType::CCU_LOOPGROUP_PROFILING;
2954 0 : ccuProfilingInfoCache.name = opName;
2955 0 : ccuProfilingInfoCache.reduceOpType = opType;
2956 0 : ccuProfilingInfoCache.inputDataType = dataType;
2957 0 : ccuProfilingInfoCache.outputDataType = outputDataType;
2958 0 : ccuProfilingInfoCache.missionId = GetMissionId();
2959 :
2960 0 : CHK_SAFETY_FUNC_RET(memset_s(ccuProfilingInfoCache.channelId, sizeof(ccuProfilingInfoCache.channelId),
2961 : INVALID_VALUE_CHANNELID, sizeof(ccuProfilingInfoCache.channelId)));
2962 0 : for (uint32_t i = 0; i < channelNum; i++) {
2963 0 : void *channelPtr{nullptr};
2964 0 : CHK_RET(static_cast<HcclResult>(HcommChannelGet(channels[i], &channelPtr)));
2965 0 : auto *channelImpl = dynamic_cast<CcuUrmaChannel *>(static_cast<Channel *>(channelPtr));
2966 0 : CHK_PTR_NULL(channelImpl);
2967 0 : ccuProfilingInfoCache.channelId[i] = channelImpl->GetChannelId();
2968 0 : ccuProfilingInfoCache.channelHandle[i] = channels[i];
2969 0 : HCCL_INFO("[%s]type[%d], name[%s], opType[%d], dataType[%d], outputDataType[%d], missionId[%u], "
2970 : "channelHandle[0x%llx], channelId[%u]", __func__, ccuProfilingInfoCache.type,
2971 : ccuProfilingInfoCache.name.c_str(), opType, dataType, outputDataType, ccuProfilingInfoCache.missionId,
2972 : ccuProfilingInfoCache.channelHandle[i], ccuProfilingInfoCache.channelId[i]);
2973 : }
2974 0 : lgProfilingInfo.ccuProfilingInfos.push_back(ccuProfilingInfoCache);
2975 0 : lgProfilingInfo.lgProfilingReps.push_back(allLgProfilingReps.back());
2976 0 : return HCCL_SUCCESS;
2977 : }
2978 :
2979 0 : HcclResult CcuKernel::AddCcuProfiling(GroupInfo groupInfo, const std::vector<ChannelHandle> channelHandle, HcclDataType dataType,
2980 : HcclDataType outputDataType, HcclReduceOp opType, const std::string& opName)
2981 : {
2982 0 : CHK_RET(AddCcuProfiling(channelHandle.data(), channelHandle.size(), dataType, outputDataType, opType, opName));
2983 0 : groupOpSizeInfo_.push_back(groupInfo);
2984 0 : return HCCL_SUCCESS;
2985 : }
2986 :
2987 0 : HcclResult CcuKernel::AddCcuProfiling(const ChannelHandle *channels, uint32_t channelNum, HcclDataType dataType,
2988 : HcclDataType outputDataType, HcclReduceOp opType, const std::string& opName)
2989 : {
2990 0 : CHK_PTR_NULL(channels);
2991 0 : CHK_RET(AddProfilingInfo(channels, channelNum, dataType, outputDataType, opType, opName));
2992 0 : return HCCL_SUCCESS;
2993 : }
2994 :
2995 5 : HcclResult CcuKernel::Add2ConstValue2VarMap(std::vector<uint64_t> &values)
2996 : {
2997 : // 记录当前context所需的常量，仅A6场景适用
2998 21 : for (uint64_t value : values) {
2999 16 : if (constValue2VarMap.find(value) == constValue2VarMap.end()) {
3000 3 : constValue2VarMap[value] = CreateVariable();
3001 : }
3002 : }
3003 5 : return HCCL_SUCCESS;
3004 : }
3005 :
3006 : }; // namespace hcomm
|
