Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "ccu_kernel.h"
12 :
13 : #include <algorithm>
14 :
15 : #include "ccu_rep_v1.h"
16 : #include "ccu_kernel_resource.h"
17 : #include "ccu_microcode_v1.h"
18 :
19 : #include "ccu_types.h"
20 : #include "exception_util.h"
21 : #include "ccu_api_exception.h"
22 : #include "ccu_dev_mgr_imp.h"
23 : #include "env_config.h"
24 : #include "ccu_rep_type_v1.h"
25 :
26 : #include "hcomm_c_adpt.h"
27 :
28 : #include "ccu_rep_context_v1.h"
29 : #include "ccu_rep_funccall_v1.h"
30 : #include "../../endpoint_pairs/channels/ccu/ccu_urma_channel.h"
31 : #include "../ccu_instance/ccu_var_event_res_mgr.h"
32 :
33 : #include "ccu_log.h"
34 :
35 : #include "hcom_common.h"
36 :
37 : // todo: 引入头文件需要检查
38 : #include "ccu_assist_v1.h"
39 : #include "hccl_comm_pub.h"
40 : #include "hcclCommDfx.h"
41 : #include "task_info.h"
42 : #include "task_param.h"
43 :
44 : #include "ccu_ins_generator_base.h"
45 : #include "ccu_ins_generator_v1.h"
46 : #include "unified_platform/pub_inc/config_plf_log.h"
47 :
48 : namespace hcomm {
49 :
50 : using Hccl::PLF_DATA_OP;
51 :
52 : constexpr uint32_t TOKEN_VALUE_INDEX = 2;
53 : constexpr uint16_t INVALID_U16 = 65535;
54 :
55 : using CcuRep::CcuInsGeneratorBase;
56 : using CcuRep::CcuInsGeneratorV1;
57 :
58 : template <typename T>
59 217 : T CcuKernel::CreateResAssist(std::array<std::vector<T>, CCU_MAX_IODIE_NUM>& resRecord)
60 : {
61 : // kernel确认die之前默认为0，需要刷新资源
62 : // 确认die之后按实际使用die分配资源
63 217 : const uint32_t dieId = GetDieId();
64 217 : resRecord[dieId].emplace_back(this);
65 217 : auto& item = resRecord[dieId].back();
66 217 : item.Reset(resRecord[dieId].size(), dieId);
67 217 : return item;
68 : }
69 :
70 : template <typename T>
71 : std::vector<T>
72 892 : CcuKernel::CreateBlockResAssist(const uint32_t count, std::array<std::vector<T>, CCU_MAX_IODIE_NUM>& resRecord)
73 : {
74 892 : constexpr uint16_t CCU_BLOCK_RES_ID_BASE = 0x1000; // block 批量分配资源 id 基址，与单资源 id 区间隔离便于 DFX 定位
75 892 : std::vector<T> block;
76 892 : block.reserve(count);
77 892 : const uint32_t dieId = GetDieId();
78 2160 : for (size_t i = 0; i < count; i++) {
79 1268 : block.emplace_back(this);
80 1268 : block.back().Reset(static_cast<uint16_t>(CCU_BLOCK_RES_ID_BASE + resRecord[dieId].size() + i), dieId);
81 : }
82 892 : resRecord[dieId].insert(resRecord[dieId].end(), block.begin(), block.end());
83 892 : return block;
84 0 : }
85 :
86 172 : CcuKernel::~CcuKernel() {}
87 :
88 11 : static HcclResult GetDieIdByChannel(const ChannelHandle channel, uint32_t& dieId)
89 : {
90 11 : void* channelPtr{nullptr};
91 11 : CHK_RET(static_cast<HcclResult>(HcommChannelGet(channel, &channelPtr)));
92 11 : auto* channelImpl = dynamic_cast<CcuUrmaChannel*>(static_cast<Channel*>(channelPtr));
93 11 : if (channelImpl == nullptr) {
94 0 : HCCL_ERROR("[%s] failed to cast channel[0x%llx] to CcuUrmaChannel", __func__, channel);
95 0 : return HcclResult::HCCL_E_PTR;
96 : }
97 11 : dieId = channelImpl->GetDieId();
98 11 : HCCL_INFO("[%s], channelHandle[0x%llx], dieId[%u]", __func__, channel, dieId);
99 11 : return HcclResult::HCCL_SUCCESS;
100 : }
101 :
102 1 : static HcclResult GetDieIdByChannels(const std::unordered_set<ChannelHandle>& channels, uint32_t& dieId)
103 : {
104 1 : if (channels.empty()) {
105 1 : int32_t devLogicId = HcclGetThreadDeviceId();
106 1 : for (uint32_t die = 0; die < CCU_MAX_IODIE_NUM; die++) {
107 1 : bool enableFlag = false;
108 2 : CHK_RET(static_cast<HcclResult>(CcuGetDieEnableInfo(devLogicId, die, enableFlag)));
109 1 : if (enableFlag) {
110 1 : dieId = die;
111 1 : return HcclResult::HCCL_SUCCESS;
112 : }
113 : }
114 :
115 0 : HCCL_ERROR("[CcuKernel][%s] failed, all dies are disable, devLogicId[%d].", __func__, devLogicId);
116 0 : return HcclResult::HCCL_E_INTERNAL;
117 : }
118 :
119 0 : uint32_t firstDieId = 0;
120 0 : CHK_RET(GetDieIdByChannel(*channels.begin(), firstDieId));
121 0 : for (const auto channel : channels) {
122 0 : uint32_t nextDieId = 0;
123 0 : CHK_RET(GetDieIdByChannel(channel, nextDieId));
124 0 : if (firstDieId != nextDieId) {
125 0 : HCCL_ERROR("[%s] failed, the dies of channels are not same.", __func__);
126 0 : return HcclResult::HCCL_E_PARA;
127 : }
128 : }
129 :
130 0 : dieId = firstDieId;
131 0 : return HcclResult::HCCL_SUCCESS;
132 : }
133 :
134 58 : static HcclResult CheckChannelsDie(const std::unordered_set<ChannelHandle>& channels, const uint32_t targetDieId)
135 : {
136 66 : for (const auto channel : channels) {
137 11 : uint32_t channelDieId = 0;
138 14 : CHK_RET(GetDieIdByChannel(channel, channelDieId));
139 11 : if (channelDieId != targetDieId) {
140 3 : HCCL_ERROR(
141 : "[%s] failed, channel[0x%llx] dieId[%u] differs from target dieId[%u].", __func__, channel,
142 : channelDieId, targetDieId);
143 3 : return HcclResult::HCCL_E_PARA;
144 : }
145 : }
146 55 : return HcclResult::HCCL_SUCCESS;
147 : }
148 :
149 56 : static void MoveResourcesToDie(CcuRepResource& res, uint32_t targetDieId)
150 : {
151 56 : if (targetDieId == 0)
152 55 : return; // 初始资源位于die0，不用设置
153 :
154 11 : auto moveAndSet = [&](auto& arr) {
155 11 : arr[targetDieId] = std::move(arr[0]);
156 11 : for (auto& item : arr[targetDieId])
157 0 : item.SetDieId(targetDieId);
158 12 : };
159 :
160 1 : moveAndSet(res.ccubufs);
161 1 : moveAndSet(res.blockCcubufs);
162 1 : moveAndSet(res.executor);
163 1 : moveAndSet(res.blockExecutor);
164 1 : moveAndSet(res.completedEvent);
165 1 : moveAndSet(res.blockCompletedEvent);
166 1 : moveAndSet(res.address);
167 1 : moveAndSet(res.blockAddress);
168 1 : moveAndSet(res.continuousVariable);
169 1 : moveAndSet(res.variable);
170 1 : moveAndSet(res.localNotify);
171 : }
172 :
173 72 : HcclResult CcuKernel::SetupProfilingInfo(const char* kernelFuncName)
174 : {
175 72 : if (kernelFuncName == nullptr || strlen(kernelFuncName) == 0) {
176 0 : name_ = std::string("CCU_KERNEL"); // 默认名称
177 0 : AddSqeProfiling(name_);
178 0 : return HcclResult::HCCL_SUCCESS;
179 : }
180 :
181 72 : constexpr size_t MAX_KERNEL_FUNC_NAME_LEN = 128;
182 72 : const auto nameLen = strlen(kernelFuncName);
183 72 : if (nameLen > MAX_KERNEL_FUNC_NAME_LEN) {
184 0 : name_ = std::string(kernelFuncName, MAX_KERNEL_FUNC_NAME_LEN);
185 0 : HCCL_WARNING("[CcuKernel][%s] kernelFuncName is too long, reset to %s.", __func__, name_.c_str());
186 : }
187 :
188 : // 生成SQE粒度profiling信息，此时未选择die，默认die 0
189 72 : AddSqeProfiling(name_);
190 72 : return HcclResult::HCCL_SUCCESS;
191 : }
192 :
193 : static HcclResult
194 56 : UpdateProfilingInfo(std::vector<CcuProfilingInfo>& profilingInfos, uint32_t dieId, const std::string& kernelName)
195 : {
196 56 : if (dieId == 0) {
197 : // 与默认dieId相同，不需要修改
198 55 : return HcclResult::HCCL_SUCCESS;
199 : }
200 :
201 : // 正常情况仅首个info包含die信息，仅应为CCU_TASK_PROFILING类型
202 1 : if (UNLIKELY(profilingInfos.empty())) {
203 : // profiling不属于主流程，不打断算子业务
204 0 : HCCL_INFO("[%s] passed, profiling infos are empty, ccu kernel func[%s].", __func__, kernelName.c_str());
205 0 : return HcclResult::HCCL_SUCCESS;
206 : }
207 :
208 : // 根据选择的die跟新profiling信息
209 2 : for (auto& info : profilingInfos) {
210 1 : info.dieId = dieId;
211 : }
212 :
213 1 : HCCL_INFO("[%s] reset profiling info dieId to [%u], ccu kernel func[%s].", __func__, dieId, kernelName.c_str());
214 1 : return HcclResult::HCCL_SUCCESS;
215 : }
216 :
217 1 : HcclResult CcuKernel::ApplyDieFromChannels()
218 : {
219 1 : uint32_t dieId{0};
220 1 : CHK_RET(GetDieIdByChannels(channels_, dieId));
221 1 : CHK_PRT_RET(
222 : dieId >= CCU_MAX_IODIE_NUM,
223 : HCCL_ERROR("[CcuKernel][%s] failed, dieId[%u] should be less than [%u].", __func__, dieId, CCU_MAX_IODIE_NUM),
224 : HcclResult::HCCL_E_PARA);
225 1 : SetDieId(dieId);
226 1 : MoveResourcesToDie(res_, dieId);
227 1 : (void)UpdateProfilingInfo(profilingInfo, dieId, name_);
228 :
229 1 : return HcclResult::HCCL_SUCCESS;
230 : }
231 :
232 58 : HcclResult CcuKernel::ValidateAndApplyDie(const uint32_t targetDieId)
233 : {
234 58 : CHK_PRT_RET(
235 : targetDieId >= CCU_MAX_IODIE_NUM,
236 : HCCL_ERROR(
237 : "[CcuKernel][%s] failed, dieId[%u] should be less than [%u].", __func__, targetDieId, CCU_MAX_IODIE_NUM),
238 : HcclResult::HCCL_E_PARA);
239 :
240 58 : const int32_t devLogicId = HcclGetThreadDeviceId();
241 58 : bool enableFlag = false;
242 58 : CHK_RET(static_cast<HcclResult>(CcuGetDieEnableInfo(devLogicId, static_cast<uint8_t>(targetDieId), enableFlag)));
243 58 : CHK_PRT_RET(
244 : !enableFlag,
245 : HCCL_ERROR(
246 : "[CcuKernel][%s] failed, target dieId[%u] is disabled, devLogicId[%d].", __func__, targetDieId, devLogicId),
247 : HcclResult::HCCL_E_PARA);
248 58 : CHK_RET(CheckChannelsDie(channels_, targetDieId));
249 :
250 55 : SetDieId(targetDieId);
251 55 : MoveResourcesToDie(res_, targetDieId);
252 55 : (void)UpdateProfilingInfo(profilingInfo, targetDieId, name_);
253 :
254 55 : return HcclResult::HCCL_SUCCESS;
255 : }
256 :
257 75 : void CcuKernel::SetInsGenerater(CcuInsGeneratorBase* insGeneratorBase) { insGenerator = insGeneratorBase; }
258 :
259 5 : CcuResult CcuKernel::ValidateTaskArgs(const uint64_t* taskArgs, uint32_t argsNum) const
260 : {
261 5 : if (loadArgUsedSet_.size() != argsNum) {
262 0 : HCCL_ERROR(
263 : "[CcuKernel][%s] failed, args number does not match the Load instruction, "
264 : "argsNum = %u, loaded = %zu",
265 : __func__, argsNum, loadArgUsedSet_.size());
266 0 : return CcuResult::CCU_E_INTERNAL;
267 : }
268 50 : for (uint32_t i = 0; i < argsNum; ++i) {
269 45 : if (loadArgUsedSet_.count(i) == 0) {
270 0 : HCCL_ERROR("[CcuKernel][%s] failed, argId %u not loaded (argsNum=%u)", __func__, i, argsNum);
271 0 : return CcuResult::CCU_E_INTERNAL;
272 : }
273 : }
274 5 : if (argsNum != 0) {
275 3 : CCU_CHK_PTR_NULL(taskArgs);
276 : }
277 5 : if (instrInfo_.missionInstrCount == 0 || instrInfo_.instrVec.empty()) {
278 0 : HCCL_ERROR(
279 : "[CcuKernel][%s] failed, mission instructions are empty, "
280 : "the kernel is not been translated yet.",
281 : __func__);
282 0 : return CcuResult::CCU_E_INTERNAL;
283 : }
284 5 : return CcuResult::CCU_SUCCESS;
285 : }
286 :
287 8 : void CcuKernel::FillTaskParam(
288 : CcuTaskParam& param, uint32_t index, uint32_t seqNum, const uint64_t* taskArgs, uint32_t argsNum) const
289 : {
290 8 : param.dieId = GetDieId();
291 8 : param.missionId = GetMissionId();
292 8 : param.instStartId = instrInfo_.missionStartInstrId + index * CCU_SQE_ARGS_LEN;
293 8 : param.key = GetMissionKey();
294 8 : param.argSize = CCU_SQE_ARGS_LEN;
295 :
296 8 : const uint32_t preMissionInsCnt = index * CCU_SQE_ARGS_LEN;
297 8 : const bool isLast = (index == seqNum - 1);
298 8 : param.instCnt = isLast ? (instrInfo_.missionInstrCount - preMissionInsCnt) : CCU_SQE_ARGS_LEN;
299 :
300 8 : if (argsNum > preMissionInsCnt) {
301 6 : const uint32_t argsToCopy = isLast ? std::min(argsNum - preMissionInsCnt, CCU_SQE_ARGS_LEN) : CCU_SQE_ARGS_LEN;
302 12 : std::copy(taskArgs + preMissionInsCnt, taskArgs + preMissionInsCnt + argsToCopy, std::begin(param.args));
303 : }
304 :
305 8 : HCCL_INFO(
306 : "[GeneTaskParam]task Param, dieId[%u] missionId[%u] instStartId[%u] instCnt[%u], argSize[%u]", param.dieId,
307 : param.missionId, param.instStartId, param.instCnt, param.argSize);
308 8 : }
309 :
310 5 : CcuResult CcuKernel::GeneTaskParams(const uint64_t* taskArgs, uint32_t argsNum, std::vector<CcuTaskParam>& taskParams)
311 : {
312 5 : CCU_CHK_RET(ValidateTaskArgs(taskArgs, argsNum));
313 :
314 : // 如果agrs数量超过sqe arg的最大数量，则返回多个TaskParam，前面几个只从sqe中加载args;
315 : // args数量大于等于0、小于等于最大值时，返回1个TaskParam
316 5 : const uint32_t seqNum
317 5 : = (argsNum / CCU_SQE_ARGS_LEN) + ((argsNum % CCU_SQE_ARGS_LEN) == 0 ? 0 : 1) + (argsNum == 0 ? 1 : 0);
318 :
319 5 : const uint32_t preMissonSqeInsCnt = (seqNum - 1) * CCU_SQE_ARGS_LEN;
320 5 : if (instrInfo_.missionInstrCount < preMissonSqeInsCnt) {
321 0 : HCCL_ERROR(
322 : "[CcuKernel][%s] failed, missionInstrCount[%u] should be greater "
323 : "than preMissonSqeInsCnt[%u].",
324 : __func__, instrInfo_.missionInstrCount, preMissonSqeInsCnt);
325 0 : return CcuResult::CCU_E_INTERNAL;
326 : }
327 :
328 5 : taskParams.resize(seqNum);
329 13 : for (uint32_t index = 0; index < seqNum; index++) {
330 8 : FillTaskParam(taskParams[index], index, seqNum, taskArgs, argsNum);
331 : }
332 :
333 5 : return CcuResult::CCU_SUCCESS;
334 : }
335 :
336 21 : HcclResult CcuKernel::CreateVariable(const ChannelHandle channel, uint32_t varIndex, CcuRep::Variable* var)
337 : {
338 21 : channels_.insert(channel);
339 :
340 21 : void* channelPtr{nullptr};
341 21 : CHK_RET(static_cast<HcclResult>(HcommChannelGet(channel, &channelPtr)));
342 21 : auto* channelImpl = dynamic_cast<CcuUrmaChannel*>(static_cast<Channel*>(channelPtr));
343 21 : if (channelImpl == nullptr) {
344 0 : HCCL_ERROR("[%s] failed to cast channel[0x%llx] to CcuUrmaChannel", __func__, channel);
345 0 : return HcclResult::HCCL_E_PTR;
346 : }
347 21 : uint32_t locXnId{0};
348 21 : CHK_RET(channelImpl->GetLocXnByIndex(varIndex, locXnId));
349 20 : var->Reset(locXnId, channelImpl->GetDieId());
350 20 : return HcclResult::HCCL_SUCCESS;
351 : }
352 :
353 9 : void CcuKernel::AddPinnedRegGroup(const CcuRep::Variable& baseVar, uint16_t count)
354 : {
355 9 : if (count == 0) {
356 0 : return;
357 : }
358 9 : const uint16_t baseId = baseVar.Id();
359 11 : for (const auto& g : pinnedRegGroups_) {
360 5 : if (g.baseVar.Id() == baseId && g.count == count) {
361 3 : return;
362 : }
363 : }
364 6 : pinnedRegGroups_.push_back({baseVar, count});
365 6 : }
366 :
367 46 : CcuRepResource& CcuKernel::GetResource() { return res_; }
368 :
369 95 : CcuResReq CcuKernel::GetResourceRequest()
370 : {
371 95 : CcuResReq req;
372 95 : uint32_t dieId = GetDieId();
373 95 : req.msReq[dieId] = res_.ccubufs[dieId].size();
374 95 : req.blockMsReq[dieId] = res_.blockCcubufs[dieId].size();
375 95 : req.ckeReq[dieId] = res_.completedEvent[dieId].size();
376 95 : req.blockCkeReq[dieId] = res_.blockCompletedEvent[dieId].size() + res_.localNotify[dieId].size();
377 95 : req.loopEngineReq[dieId] = res_.executor[dieId].size();
378 95 : req.blockLoopEngineReq[dieId] = res_.blockExecutor[dieId].size();
379 95 : req.gsaReq[dieId] = res_.address[dieId].size();
380 95 : req.blockGsaReq[dieId] = res_.blockAddress[dieId].size();
381 95 : req.xnReq[dieId] = res_.variable[dieId].size();
382 95 : req.blockXnReq[dieId] = res_.continuousVariable[dieId].size();
383 :
384 95 : req.missionReq.reqType = MissionReqType::FUSION_MULTIPLE_DIE;
385 95 : req.missionReq.req[dieId] = 1;
386 :
387 : auto info = Hccl::StringFormat(
388 : "resource request: dieId[%u], ms[%u], blockMs[%u], cke[%u], blockCke[%u], "
389 : "loopEngine[%u], blockLoopEngine[%u], gsa[%u], blockGsa[%u], xn[%u], blockXn[%u], "
390 : "missionId[%u]",
391 380 : dieId, req.msReq[dieId], req.blockMsReq[dieId], req.ckeReq[dieId], req.blockCkeReq[dieId],
392 380 : req.loopEngineReq[dieId], req.blockLoopEngineReq[dieId], req.gsaReq[dieId], req.blockGsaReq[dieId],
393 95 : req.xnReq[dieId], req.blockXnReq[dieId], req.missionReq.req[dieId]);
394 :
395 95 : HCCL_INFO("%s", info.c_str());
396 :
397 190 : return req;
398 95 : }
399 :
400 : template <typename HandleType, typename ResourceType>
401 1704 : static CcuResult GetResourceByHandle(
402 : std::unordered_map<HandleType, ResourceType>& resourceMap, HandleType handle, ResourceType** resource,
403 : const char* resourceType)
404 : {
405 1704 : auto iter = resourceMap.find(handle);
406 1704 : if (iter == resourceMap.end()) {
407 0 : HCCL_ERROR("[%s] failed to find %s by handle: 0x%llx", __func__, resourceType, handle);
408 0 : return CcuResult::CCU_E_NOT_FOUND;
409 : }
410 :
411 : // ccu资源本身可能重载=，对象赋值会被转换成指令，导致流程失败
412 1704 : *resource = &(iter->second);
413 1704 : return CcuResult::CCU_SUCCESS;
414 : }
415 :
416 1370 : CcuResult CcuKernel::GetVariableByHandle(CcuVariableHandle varHandle, CcuRep::Variable** variable)
417 : {
418 1370 : return GetResourceByHandle(ccuVarMap_, varHandle, variable, "variable");
419 : }
420 : // Alloc 相关接口
421 495 : CcuResult CcuKernel::VariableAlloc(CcuVariableHandle* varHandle)
422 : {
423 495 : PLF_CONFIG_INFO(PLF_DATA_OP, "[VariableAlloc]");
424 495 : const auto& var = CreateBlockResAssist(1, res_.continuousVariable);
425 495 : CcuVariableHandle handle = ccuVarMap_.size();
426 495 : ccuVarMap_.emplace(handle, var[0]);
427 :
428 495 : *varHandle = handle;
429 495 : return CcuResult::CCU_SUCCESS;
430 495 : }
431 10 : CcuResult CcuKernel::AddressAlloc(CcuAddressHandle* addrHandle)
432 : {
433 10 : PLF_CONFIG_INFO(PLF_DATA_OP, "[AddressAlloc]");
434 10 : const auto addr = CreateAddress();
435 10 : CcuAddressHandle handle = ccuAddrMap_.size();
436 10 : ccuAddrMap_.emplace(handle, addr);
437 10 : *addrHandle = handle;
438 10 : return CcuResult::CCU_SUCCESS;
439 10 : }
440 14 : CcuResult CcuKernel::EventAlloc(CcuEventHandle* eventHandle)
441 : {
442 14 : PLF_CONFIG_INFO(PLF_DATA_OP, "[EventAlloc]");
443 14 : const auto& event = CreateResAssist(res_.blockCompletedEvent);
444 14 : CcuEventHandle handle = ccuEventMap_.size();
445 14 : ccuEventMap_.emplace(handle, event);
446 14 : *eventHandle = handle;
447 14 : return CcuResult::CCU_SUCCESS;
448 14 : }
449 2 : CcuResult CcuKernel::BufferAlloc(CcuBufferHandle* bufHandle)
450 : {
451 2 : PLF_CONFIG_INFO(PLF_DATA_OP, "[BufferAlloc]");
452 2 : const auto& buffer = CreateBlockResAssist(1, res_.blockCcubufs);
453 2 : CcuBufferHandle handle = ccuBufferMap_.size();
454 2 : ccuBufferMap_.emplace(handle, buffer[0]);
455 2 : *bufHandle = handle;
456 2 : return CcuResult::CCU_SUCCESS;
457 2 : }
458 156 : CcuResult CcuKernel::LocalAddrAlloc(
459 : CcuLocalAddrHandle* localAddrHandle, CcuAddressHandle* addrHandle, CcuVariableHandle* tokenHandle)
460 : {
461 156 : PLF_CONFIG_INFO(PLF_DATA_OP, "[LocalAddrAlloc]");
462 156 : auto localAddr = CreateLocalAddr();
463 :
464 156 : CcuAddressHandle aHandle = ccuAddrMap_.size();
465 156 : ccuAddrMap_.emplace(aHandle, localAddr.addr);
466 :
467 156 : CcuVariableHandle tHandle = ccuVarMap_.size();
468 156 : ccuVarMap_.emplace(tHandle, localAddr.token);
469 :
470 156 : CcuLocalAddrHandle laHandle = ccuLocalAddrMap_.size();
471 156 : ccuLocalAddrMap_.emplace(laHandle, localAddr);
472 :
473 156 : *localAddrHandle = laHandle;
474 156 : *addrHandle = aHandle;
475 156 : *tokenHandle = tHandle;
476 156 : return CcuResult::CCU_SUCCESS;
477 156 : }
478 37 : CcuResult CcuKernel::RemoteAddrAlloc(
479 : CcuRemoteAddrHandle* remoteAddrHandle, CcuAddressHandle* addrHandle, CcuVariableHandle* tokenHandle)
480 : {
481 37 : PLF_CONFIG_INFO(PLF_DATA_OP, "[RemoteAddrAlloc]");
482 37 : auto remoteAddr = CreateRemoteAddr();
483 :
484 37 : CcuAddressHandle aHandle = ccuAddrMap_.size();
485 37 : ccuAddrMap_.emplace(aHandle, remoteAddr.addr);
486 :
487 37 : CcuVariableHandle tHandle = ccuVarMap_.size();
488 37 : ccuVarMap_.emplace(tHandle, remoteAddr.token);
489 :
490 37 : CcuRemoteAddrHandle raHandle = ccuRemoteAddrMap_.size();
491 37 : ccuRemoteAddrMap_.emplace(raHandle, remoteAddr);
492 :
493 37 : *remoteAddrHandle = raHandle;
494 37 : *addrHandle = aHandle;
495 37 : *tokenHandle = tHandle;
496 37 : return CcuResult::CCU_SUCCESS;
497 37 : }
498 :
499 6 : CcuResult CcuKernel::BlockVariableAlloc(CcuVariableHandle* varHandles, uint32_t count)
500 : {
501 6 : PLF_CONFIG_INFO(PLF_DATA_OP, "[BlockVariableAlloc] count=%u", count);
502 6 : const auto& var = CreateBlockResAssist(count, res_.continuousVariable);
503 18 : for (uint32_t i = 0; i < count; i++) {
504 12 : CcuVariableHandle handle = ccuVarMap_.size();
505 12 : ccuVarMap_.emplace(handle, var[i]);
506 12 : varHandles[i] = handle;
507 : }
508 6 : return CcuResult::CCU_SUCCESS;
509 6 : }
510 :
511 6 : CcuResult CcuKernel::BlockEventAlloc(CcuEventHandle* eventHandles, uint32_t count)
512 : {
513 6 : PLF_CONFIG_INFO(PLF_DATA_OP, "[BlockEventAlloc] count=%u", count);
514 6 : const auto& event = CreateBlockResAssist(count, res_.blockCompletedEvent);
515 52 : for (uint32_t i = 0; i < count; i++) {
516 46 : CcuEventHandle handle = ccuEventMap_.size();
517 46 : ccuEventMap_.emplace(handle, event[i]);
518 46 : eventHandles[i] = handle;
519 : }
520 6 : return CcuResult::CCU_SUCCESS;
521 6 : }
522 :
523 7 : CcuResult CcuKernel::BlockBufferAlloc(CcuBufferHandle* bufHandles, uint32_t count)
524 : {
525 7 : PLF_CONFIG_INFO(PLF_DATA_OP, "[BlockBufferAlloc] count=%u", count);
526 7 : const auto& buffer = CreateBlockResAssist(count, res_.blockCcubufs);
527 332 : for (uint32_t i = 0; i < count; i++) {
528 325 : CcuBufferHandle handle = ccuBufferMap_.size();
529 325 : ccuBufferMap_.emplace(handle, buffer[i]);
530 325 : bufHandles[i] = handle;
531 : }
532 7 : return CcuResult::CCU_SUCCESS;
533 7 : }
534 :
535 21 : CcuResult CcuKernel::VariableCreateByChannel(ChannelHandle channel, uint32_t varIndex, CcuVariableHandle* varHandle)
536 : {
537 21 : PLF_CONFIG_INFO(PLF_DATA_OP, "[VariableCreateByChannel] channel=%llu, varIndex=%u", channel, varIndex);
538 21 : channels_.insert(channel);
539 21 : CcuRep::Variable var(this);
540 21 : CCU_CHK_RET(CreateVariable(channel, varIndex, &var));
541 20 : CcuVariableHandle handle = ccuVarMap_.size();
542 20 : ccuVarMap_.emplace(handle, var);
543 20 : *varHandle = handle;
544 20 : declaredLocXns_.insert(var.Id());
545 20 : return CcuResult::CCU_SUCCESS;
546 21 : }
547 :
548 10 : CcuResult CcuKernel::VariableCreateByAcquire(CcuVariableHandle acqHandle, uint32_t index, CcuVariableHandle* varHandle)
549 : {
550 10 : uint8_t dieId = 0;
551 10 : uint32_t xnId = 0;
552 10 : CCU_CHK_RET(CcuVarEventResMgr::GetInstance(HcclGetThreadDeviceId()).GetVariableXnId(acqHandle, index, dieId, xnId));
553 :
554 10 : CcuRep::Variable var(this);
555 10 : var.Reset(static_cast<uint16_t>(xnId), static_cast<uint16_t>(dieId));
556 10 : CcuVariableHandle handle = ccuVarMap_.size();
557 10 : ccuVarMap_.emplace(handle, var);
558 10 : *varHandle = handle;
559 10 : return CcuResult::CCU_SUCCESS;
560 10 : }
561 :
562 4 : CcuResult CcuKernel::EventCreateByAcquire(CcuEventHandle acqHandle, uint32_t index, CcuEventHandle* eventHandle)
563 : {
564 4 : uint8_t dieId = 0;
565 4 : uint32_t ckeId = 0;
566 4 : CCU_CHK_RET(CcuVarEventResMgr::GetInstance(HcclGetThreadDeviceId()).GetEventCkeId(acqHandle, index, dieId, ckeId));
567 :
568 4 : CcuRep::CompletedEvent event(this);
569 4 : event.Reset(static_cast<uint16_t>(ckeId), static_cast<uint16_t>(dieId));
570 4 : CcuEventHandle handle = ccuEventMap_.size();
571 4 : ccuEventMap_.emplace(handle, event);
572 4 : *eventHandle = handle;
573 4 : return CcuResult::CCU_SUCCESS;
574 4 : }
575 :
576 280 : CcuResult CcuKernel::VariableAssignImm(CcuVariableHandle varHandle, uint64_t immediate)
577 : {
578 280 : PLF_CONFIG_INFO(PLF_DATA_OP, "[VariableAssignImm] varHandle=%llu, immediate=%llu", varHandle, immediate);
579 280 : CcuRep::Variable* variable{nullptr};
580 280 : CCU_CHK_RET(GetVariableByHandle(varHandle, &variable));
581 : // 通过符号重载实现，内部记录rep；异常由入口 HcommCcuKernelRegister 的
582 : // CCU_EXCEPTION_HANDLE_BEGIN/END 统一接住，无需在此局部 try/catch。
583 280 : (*variable) = immediate;
584 280 : return CcuResult::CCU_SUCCESS;
585 : }
586 :
587 158 : CcuResult CcuKernel::VariableAssignVar(CcuVariableHandle varHandle, CcuVariableHandle varA)
588 : {
589 158 : PLF_CONFIG_INFO(PLF_DATA_OP, "[VariableAssignVar] varHandle=%llu, varA=%llu", varHandle, varA);
590 158 : CcuRep::Variable* variable{nullptr};
591 158 : CCU_CHK_RET(GetVariableByHandle(varHandle, &variable));
592 158 : CcuRep::Variable* variableA{nullptr};
593 158 : CCU_CHK_RET(GetVariableByHandle(varA, &variableA));
594 : // 通过符号重载实现，内部记录rep；异常由入口统一 catch。
595 158 : (*variable) = (*variableA);
596 158 : return CcuResult::CCU_SUCCESS;
597 : }
598 :
599 : CcuResult
600 109 : CcuKernel::VariableAddVarToVar(CcuVariableHandle varHandle, CcuVariableHandle varAHandle, CcuVariableHandle varBHandle)
601 : {
602 109 : PLF_CONFIG_INFO(
603 : PLF_DATA_OP, "[VariableAddVarToVar] varHandle=%llu, varAHandle=%llu, varBHandle=%llu", varHandle, varAHandle,
604 : varBHandle);
605 109 : CcuRep::Variable *resVar{nullptr}, *leftVar{nullptr}, *rightVar{nullptr};
606 109 : CCU_CHK_RET(GetVariableByHandle(varHandle, &resVar));
607 109 : CCU_CHK_RET(GetVariableByHandle(varAHandle, &leftVar));
608 109 : CCU_CHK_RET(GetVariableByHandle(varBHandle, &rightVar));
609 :
610 : // 通过符号重载实现，内部记录rep；异常由入口统一 catch。
611 109 : *resVar = *leftVar + *rightVar;
612 109 : return CcuResult::CCU_SUCCESS;
613 : }
614 :
615 : CcuResult
616 2 : CcuKernel::VariableSubVarToVar(CcuVariableHandle varHandle, CcuVariableHandle varAHandle, CcuVariableHandle varBHandle)
617 : {
618 2 : CcuRep::Variable *resVar{nullptr}, *leftVar{nullptr}, *rightVar{nullptr};
619 2 : CCU_CHK_RET(GetVariableByHandle(varHandle, &resVar));
620 2 : CCU_CHK_RET(GetVariableByHandle(varAHandle, &leftVar));
621 2 : CCU_CHK_RET(GetVariableByHandle(varBHandle, &rightVar));
622 :
623 2 : *resVar = *leftVar - *rightVar;
624 2 : return CcuResult::CCU_SUCCESS;
625 : }
626 :
627 : CcuResult
628 2 : CcuKernel::VariableMulVarToVar(CcuVariableHandle varHandle, CcuVariableHandle varAHandle, CcuVariableHandle varBHandle)
629 : {
630 2 : CcuRep::Variable *resVar{nullptr}, *leftVar{nullptr}, *rightVar{nullptr};
631 2 : CCU_CHK_RET(GetVariableByHandle(varHandle, &resVar));
632 2 : CCU_CHK_RET(GetVariableByHandle(varAHandle, &leftVar));
633 2 : CCU_CHK_RET(GetVariableByHandle(varBHandle, &rightVar));
634 :
635 2 : *resVar = *leftVar * *rightVar;
636 2 : return CcuResult::CCU_SUCCESS;
637 : }
638 :
639 1 : CcuResult CcuKernel::VariableAddImmToVar(CcuVariableHandle varHandle, CcuVariableHandle varAHandle, uint16_t immediate)
640 : {
641 1 : CcuRep::Variable *resVar{nullptr}, *leftVar{nullptr};
642 1 : CCU_CHK_RET(GetVariableByHandle(varHandle, &resVar));
643 1 : CCU_CHK_RET(GetVariableByHandle(varAHandle, &leftVar));
644 :
645 1 : *resVar = *leftVar + immediate;
646 1 : return CcuResult::CCU_SUCCESS;
647 : }
648 :
649 2 : CcuResult CcuKernel::VariableSubImmToVar(CcuVariableHandle varHandle, CcuVariableHandle varAHandle, uint16_t immediate)
650 : {
651 2 : CcuRep::Variable *resVar{nullptr}, *leftVar{nullptr};
652 2 : CCU_CHK_RET(GetVariableByHandle(varHandle, &resVar));
653 2 : CCU_CHK_RET(GetVariableByHandle(varAHandle, &leftVar));
654 :
655 2 : *resVar = *leftVar - immediate;
656 2 : return CcuResult::CCU_SUCCESS;
657 : }
658 :
659 2 : CcuResult CcuKernel::VariableMulImmToVar(CcuVariableHandle varHandle, CcuVariableHandle varAHandle, uint16_t immediate)
660 : {
661 2 : CcuRep::Variable *resVar{nullptr}, *leftVar{nullptr};
662 2 : CCU_CHK_RET(GetVariableByHandle(varHandle, &resVar));
663 2 : CCU_CHK_RET(GetVariableByHandle(varAHandle, &leftVar));
664 :
665 2 : *resVar = *leftVar * immediate;
666 2 : return CcuResult::CCU_SUCCESS;
667 : }
668 :
669 : CcuResult
670 2 : CcuKernel::VariableAndVarToVar(CcuVariableHandle varHandle, CcuVariableHandle varAHandle, CcuVariableHandle varBHandle)
671 : {
672 2 : CcuRep::Variable *resVar{nullptr}, *leftVar{nullptr}, *rightVar{nullptr};
673 2 : CCU_CHK_RET(GetVariableByHandle(varHandle, &resVar));
674 2 : CCU_CHK_RET(GetVariableByHandle(varAHandle, &leftVar));
675 2 : CCU_CHK_RET(GetVariableByHandle(varBHandle, &rightVar));
676 :
677 2 : *resVar = *leftVar & *rightVar;
678 2 : return CcuResult::CCU_SUCCESS;
679 : }
680 :
681 : CcuResult
682 2 : CcuKernel::VariableOrVarToVar(CcuVariableHandle varHandle, CcuVariableHandle varAHandle, CcuVariableHandle varBHandle)
683 : {
684 2 : CcuRep::Variable *resVar{nullptr}, *leftVar{nullptr}, *rightVar{nullptr};
685 2 : CCU_CHK_RET(GetVariableByHandle(varHandle, &resVar));
686 2 : CCU_CHK_RET(GetVariableByHandle(varAHandle, &leftVar));
687 2 : CCU_CHK_RET(GetVariableByHandle(varBHandle, &rightVar));
688 :
689 2 : *resVar = *leftVar | *rightVar;
690 2 : return CcuResult::CCU_SUCCESS;
691 : }
692 :
693 : CcuResult
694 2 : CcuKernel::VariableXorVarToVar(CcuVariableHandle varHandle, CcuVariableHandle varAHandle, CcuVariableHandle varBHandle)
695 : {
696 2 : CcuRep::Variable *resVar{nullptr}, *leftVar{nullptr}, *rightVar{nullptr};
697 2 : CCU_CHK_RET(GetVariableByHandle(varHandle, &resVar));
698 2 : CCU_CHK_RET(GetVariableByHandle(varAHandle, &leftVar));
699 2 : CCU_CHK_RET(GetVariableByHandle(varBHandle, &rightVar));
700 :
701 2 : *resVar = *leftVar ^ *rightVar;
702 2 : return CcuResult::CCU_SUCCESS;
703 : }
704 :
705 1 : CcuResult CcuKernel::VariableNotVar(CcuVariableHandle varHandle, CcuVariableHandle varAHandle)
706 : {
707 1 : CcuRep::Variable *resVar{nullptr}, *leftVar{nullptr};
708 1 : CCU_CHK_RET(GetVariableByHandle(varHandle, &resVar));
709 1 : CCU_CHK_RET(GetVariableByHandle(varAHandle, &leftVar));
710 :
711 1 : *resVar = ~(*leftVar);
712 1 : return CcuResult::CCU_SUCCESS;
713 : }
714 :
715 : CcuResult
716 2 : CcuKernel::VariableShlVarToVar(CcuVariableHandle varHandle, CcuVariableHandle varAHandle, CcuVariableHandle varBHandle)
717 : {
718 2 : CcuRep::Variable *resVar{nullptr}, *leftVar{nullptr}, *rightVar{nullptr};
719 2 : CCU_CHK_RET(GetVariableByHandle(varHandle, &resVar));
720 2 : CCU_CHK_RET(GetVariableByHandle(varAHandle, &leftVar));
721 2 : CCU_CHK_RET(GetVariableByHandle(varBHandle, &rightVar));
722 :
723 2 : *resVar = *leftVar << *rightVar;
724 2 : return CcuResult::CCU_SUCCESS;
725 : }
726 :
727 : CcuResult
728 2 : CcuKernel::VariableShrVarToVar(CcuVariableHandle varHandle, CcuVariableHandle varAHandle, CcuVariableHandle varBHandle)
729 : {
730 2 : CcuRep::Variable *resVar{nullptr}, *leftVar{nullptr}, *rightVar{nullptr};
731 2 : CCU_CHK_RET(GetVariableByHandle(varHandle, &resVar));
732 2 : CCU_CHK_RET(GetVariableByHandle(varAHandle, &leftVar));
733 2 : CCU_CHK_RET(GetVariableByHandle(varBHandle, &rightVar));
734 :
735 2 : *resVar = *leftVar >> *rightVar;
736 2 : return CcuResult::CCU_SUCCESS;
737 : }
738 :
739 : /*========== Event信号同步类 相关接口 ==========*/
740 18 : CcuResult CcuKernel::EventRecord(CcuEventHandle eventHandle, uint32_t mask)
741 : {
742 18 : PLF_CONFIG_INFO(PLF_DATA_OP, "[EventRecord] eventHandle=%llu, mask=%u", eventHandle, mask);
743 18 : CcuRep::CompletedEvent* event{nullptr};
744 18 : CCU_CHK_RET(GetEventByHandle(eventHandle, &event));
745 : // 复用已有的 RecordEvent 实现（内部 Append CcuRepLocRecordEvent）
746 18 : CCU_CHK_RET(RecordEvent(*event, mask));
747 17 : return CcuResult::CCU_SUCCESS;
748 : }
749 :
750 44 : CcuResult CcuKernel::EventWait(CcuEventHandle eventHandle, uint32_t mask)
751 : {
752 44 : PLF_CONFIG_INFO(PLF_DATA_OP, "[EventWait] eventHandle=%llu, mask=%u", eventHandle, mask);
753 44 : CcuRep::CompletedEvent* event{nullptr};
754 44 : CCU_CHK_RET(GetEventByHandle(eventHandle, &event));
755 : // 复用已有的 WaitEvent 实现（内部 Append CcuRepLocWaitEvent）
756 44 : CCU_CHK_RET(WaitEvent(*event, mask));
757 44 : return CcuResult::CCU_SUCCESS;
758 : }
759 :
760 1 : CcuResult CcuKernel::LocalNotifyRecord(const char* notifyTag, const uint32_t mask)
761 : {
762 1 : PLF_CONFIG_INFO(PLF_DATA_OP, "[LocalNotifyRecord] tag=%s, mask=%u", (notifyTag ? notifyTag : "null"), mask);
763 1 : if (notifyTag == nullptr) {
764 0 : HCCL_ERROR("[CcuKernel][%s] notifyTag is nullptr, please check.", __func__);
765 0 : return CcuResult::CCU_E_PTR;
766 : }
767 1 : if (CurrentBlock()->Type() == CcuRep::CcuRepType::LOOP_BLOCK) {
768 1 : HCCL_ERROR("[CcuKernel][%s] is not supported in loop block, please check.", __func__);
769 1 : return LatchBodyError(CcuResult::CCU_E_NOT_SUPPORT);
770 : }
771 :
772 0 : const std::string tagKey(notifyTag);
773 :
774 0 : auto& sharedNotifies = importedRes_.sharedNotifies;
775 0 : if (sharedNotifies.find(tagKey) == sharedNotifies.end()) {
776 0 : CcuRep::LocalNotify localNotify;
777 0 : sharedNotifies.insert({tagKey, localNotify});
778 0 : }
779 :
780 0 : Append(std::make_shared<CcuRep::CcuRepRecordSharedNotify>(insGenerator, sharedNotifies.at(tagKey), mask));
781 :
782 0 : return CcuResult::CCU_SUCCESS;
783 0 : }
784 :
785 0 : CcuResult CcuKernel::LocalNotifyWait(const char* notifyTag, const uint32_t mask)
786 : {
787 0 : PLF_CONFIG_INFO(PLF_DATA_OP, "[LocalNotifyWait] tag=%s, mask=%u", (notifyTag ? notifyTag : "null"), mask);
788 0 : if (notifyTag == nullptr) {
789 0 : HCCL_ERROR("[CcuKernel][%s] notifyTag is nullptr, please check.", __func__);
790 0 : return CcuResult::CCU_E_PTR;
791 : }
792 :
793 0 : const std::string tagKey(notifyTag);
794 :
795 0 : auto& sharedNotifies = exportedRes_.sharedNotifies;
796 0 : if (sharedNotifies.find(tagKey) == sharedNotifies.end()) {
797 0 : CcuRep::LocalNotify notify = CreateLocalNotify();
798 0 : exportedRes_.sharedNotifies.insert({tagKey, notify});
799 0 : }
800 :
801 0 : bool isProfiling = CurrentBlock()->Type() != CcuRep::CcuRepType::LOOP_BLOCK;
802 0 : Append(std::make_shared<CcuRep::CcuRepLocWaitNotify>(
803 0 : insGenerator, exportedRes_.sharedNotifies.at(tagKey), mask, isProfiling));
804 0 : return CcuResult::CCU_SUCCESS;
805 0 : }
806 :
807 6 : CcuResult CcuKernel::NotifyRecord(const ChannelHandle channel, uint32_t remoteNotifyIdx, uint32_t mask)
808 : {
809 6 : PLF_CONFIG_INFO(
810 : PLF_DATA_OP, "[NotifyRecord] channel=%llu, remoteNotifyIdx=%u, mask=%u", channel, remoteNotifyIdx, mask);
811 6 : if (CurrentBlock()->Type() == CcuRep::CcuRepType::LOOP_BLOCK) {
812 1 : HCCL_ERROR("[%s] NotifyRecord is not allowed inside a ccu::Loop body", __func__);
813 1 : return LatchBodyError(CcuResult::CCU_E_NOT_SUPPORT);
814 : }
815 5 : channels_.insert(channel);
816 5 : Append(std::make_shared<CcuRep::CcuRepRemPostSem>(insGenerator, channel, remoteNotifyIdx, mask));
817 5 : return CCU_SUCCESS;
818 : }
819 :
820 11 : CcuResult CcuKernel::NotifyWait(const ChannelHandle channel, uint32_t localNotifyIdx, uint32_t mask)
821 : {
822 11 : PLF_CONFIG_INFO(
823 : PLF_DATA_OP, "[NotifyWait] channel=%llu, localNotifyIdx=%u, mask=%u", channel, localNotifyIdx, mask);
824 11 : channels_.insert(channel);
825 11 : bool isProfiling = CurrentBlock()->Type() != CcuRep::CcuRepType::LOOP_BLOCK;
826 11 : if (isProfiling) {
827 33 : CCU_CHK_RET(static_cast<HcclResult>(AddProfiling(channel, "NotifyWait", localNotifyIdx, mask)));
828 : }
829 10 : Append(std::make_shared<CcuRep::CcuRepRemWaitSem>(insGenerator, channel, localNotifyIdx, mask, isProfiling));
830 10 : return CcuResult::CCU_SUCCESS;
831 : }
832 :
833 10 : CcuResult CcuKernel::WriteVariableWithNotify(
834 : const ChannelHandle channel, CcuVariableHandle varHandle, uint32_t remoteVarIdx, uint32_t remoteNotifyIdx,
835 : uint32_t mask)
836 : {
837 10 : PLF_CONFIG_INFO(
838 : PLF_DATA_OP,
839 : "[WriteVariableWithNotify] channel=%llu, varHandle=%llu, remoteVarIdx=%u,"
840 : " remoteNotifyIdx=%u, mask=%u",
841 : channel, varHandle, remoteVarIdx, remoteNotifyIdx, mask);
842 10 : if (CurrentBlock()->Type() == CcuRep::CcuRepType::LOOP_BLOCK) {
843 1 : HCCL_ERROR("[%s] WriteVariableWithNotify is not allowed inside a ccu::Loop body", __func__);
844 1 : return LatchBodyError(CcuResult::CCU_E_NOT_SUPPORT);
845 : }
846 9 : channels_.insert(channel);
847 9 : CcuRep::Variable* var{nullptr};
848 9 : CCU_CHK_RET(GetVariableByHandle(varHandle, &var));
849 9 : Append(
850 18 : std::make_shared<CcuRep::CcuRepRemPostVar>(insGenerator, *var, channel, remoteVarIdx, remoteNotifyIdx, mask));
851 9 : return CcuResult::CCU_SUCCESS;
852 : }
853 :
854 : // 加载类 相关接口
855 49 : CcuResult CcuKernel::LoadArg(CcuVariableHandle varHandle, uint32_t argId)
856 : {
857 49 : PLF_CONFIG_INFO(PLF_DATA_OP, "[LoadArg] varHandle=%llu, argId=%u", varHandle, argId);
858 49 : loadArgUsedSet_.insert(argId);
859 49 : CcuRep::Variable* var{nullptr};
860 49 : CCU_CHK_RET(GetVariableByHandle(varHandle, &var));
861 : auto loadArgRep = std::make_shared<CcuRep::CcuRepLoadArg>(
862 49 : insGenerator, *var, argId % CCU_SQE_ARGS_LEN, static_cast<uint16_t>(argId));
863 49 : Append(loadArgRep);
864 49 : return CcuResult::CCU_SUCCESS;
865 49 : }
866 :
867 3 : CcuResult CcuKernel::GetCcuKernelInfo(CcuKernelInfo& info) const
868 : {
869 3 : uint32_t maxTaskArgsNum = 0;
870 : // 如果没有LoadArg指令，则说明不需要传入参数，直接返回0
871 3 : if (loadArgUsedSet_.empty()) {
872 1 : info.maxTaskArgsNum = 0;
873 1 : return CcuResult::CCU_SUCCESS;
874 : }
875 : // 遍历loadArgUsedSet_，获取最大的argId，加1得到最大任务参数数量（argId从0连续）
876 7 : for (const auto& argId : loadArgUsedSet_) {
877 5 : maxTaskArgsNum = std::max(maxTaskArgsNum, argId);
878 : }
879 2 : maxTaskArgsNum += 1;
880 2 : info.maxTaskArgsNum = maxTaskArgsNum;
881 2 : return CcuResult::CCU_SUCCESS;
882 : }
883 :
884 13 : CcuResult CcuKernel::CheckContinuousVariables(
885 : CcuVariableHandle varHandle, uint32_t num, const CcuRep::Variable& baseVar, const char* tag)
886 : {
887 13 : if (num <= 1) {
888 4 : return CcuResult::CCU_SUCCESS;
889 : }
890 18 : for (uint32_t i = 1; i < num; i++) {
891 9 : CcuRep::Variable* nextVar{nullptr};
892 9 : CCU_CHK_RET(GetVariableByHandle(varHandle + i, &nextVar));
893 9 : if (nextVar->Id() != baseVar.Id() + i) {
894 0 : HCCL_ERROR(
895 : "[CcuKernel][%s] variables not continuous at index %u, "
896 : "expected Id %u but got %u",
897 : tag, i, baseVar.Id() + i, nextVar->Id());
898 0 : return HCCL_TO_CCU_RET(HCCL_E_PARA);
899 : }
900 : }
901 9 : return CcuResult::CCU_SUCCESS;
902 : }
903 :
904 6 : CcuResult CcuKernel::LoadVar(uint64_t addr, CcuVariableHandle varHandle, uint32_t num)
905 : {
906 6 : PLF_CONFIG_INFO(PLF_DATA_OP, "[LoadVar] addr=0x%llx, varHandle=%llu, num=%u", addr, varHandle, num);
907 6 : CcuRep::Variable* var{nullptr};
908 6 : CCU_CHK_RET(GetVariableByHandle(varHandle, &var));
909 6 : CCU_CHK_RET(CheckContinuousVariables(varHandle, num, *var, "LoadVariable"));
910 6 : Append(std::make_shared<CcuRep::CcuRepLoad>(insGenerator, addr, *var, num));
911 : // 数据面 num>=2 走 Array<Variable>: microcode 会用到 [baseVReg, baseVReg+num)
912 : // 这段连续 Xn 且组内相对偏移固定, 交给 microcode 后端优化 Pass 2 作为 pinned 组.
913 6 : if (num >= 2) {
914 5 : AddPinnedRegGroup(*var, static_cast<uint16_t>(num));
915 : }
916 6 : return CcuResult::CCU_SUCCESS;
917 : }
918 :
919 2 : CcuResult CcuKernel::CcuLoadVarFromVarAddr(CcuVariableHandle addrHandle, CcuVariableHandle varHandle, uint32_t num)
920 : {
921 2 : PLF_CONFIG_INFO(
922 : PLF_DATA_OP,
923 : "[CcuLoadVarFromVarAddr] addrHandle=%llu, varHandle=%llu,"
924 : " num=%u",
925 : addrHandle, varHandle, num);
926 2 : CcuRep::Variable* addrVar{nullptr};
927 2 : CCU_CHK_RET(GetVariableByHandle(addrHandle, &addrVar));
928 2 : CcuRep::Variable* var{nullptr};
929 2 : CCU_CHK_RET(GetVariableByHandle(varHandle, &var));
930 2 : CCU_CHK_RET(CheckContinuousVariables(varHandle, num, *var, "LoadVar dst"));
931 2 : Append(std::make_shared<CcuRep::CcuRepLoadVar>(insGenerator, *addrVar, *var, num));
932 2 : if (num >= 2) {
933 1 : AddPinnedRegGroup(*var, static_cast<uint16_t>(num));
934 : }
935 2 : return CcuResult::CCU_SUCCESS;
936 : }
937 :
938 3 : CcuResult CcuKernel::StoreVar(uint64_t addr, CcuVariableHandle varHandle, uint32_t num)
939 : {
940 3 : PLF_CONFIG_INFO(PLF_DATA_OP, "[StoreVar] addr=0x%llx, varHandle=%llu, num=%u", addr, varHandle, num);
941 3 : CcuRep::Variable* var{nullptr};
942 3 : CCU_CHK_RET(GetVariableByHandle(varHandle, &var));
943 3 : CCU_CHK_RET(CheckContinuousVariables(varHandle, num, *var, "StoreVariable"));
944 3 : Append(std::make_shared<CcuRep::CcuRepStore>(insGenerator, *var, addr, num));
945 3 : if (num >= 2) {
946 2 : AddPinnedRegGroup(*var, static_cast<uint16_t>(num));
947 : }
948 3 : return CcuResult::CCU_SUCCESS;
949 : }
950 :
951 2 : CcuResult CcuKernel::CcuStoreVarToVarAddr(CcuVariableHandle addrHandle, CcuVariableHandle varHandle, uint32_t num)
952 : {
953 2 : PLF_CONFIG_INFO(
954 : PLF_DATA_OP,
955 : "[CcuStoreVarToVarAddr] addrHandle=%llu, varHandle=%llu,"
956 : " num=%u",
957 : addrHandle, varHandle, num);
958 2 : CcuRep::Variable* addrVar{nullptr};
959 2 : CCU_CHK_RET(GetVariableByHandle(addrHandle, &addrVar));
960 2 : CcuRep::Variable* var{nullptr};
961 2 : CCU_CHK_RET(GetVariableByHandle(varHandle, &var));
962 2 : CCU_CHK_RET(CheckContinuousVariables(varHandle, num, *var, "StoreVar src"));
963 2 : Append(std::make_shared<CcuRep::CcuRepStoreVar>(insGenerator, *var, *addrVar, num));
964 2 : if (num >= 2) {
965 1 : AddPinnedRegGroup(*var, static_cast<uint16_t>(num));
966 : }
967 2 : return CcuResult::CCU_SUCCESS;
968 : }
969 :
970 : // 本地数据拷贝 相关实现
971 12 : CcuResult CcuKernel::LocalCopyMemToBuffer(
972 : CcuBufferHandle dstHandle, CcuLocalAddrHandle srcHandle, CcuVariableHandle lenHandle, CcuEventHandle eventHandle,
973 : uint32_t mask)
974 : {
975 12 : PLF_CONFIG_INFO(
976 : PLF_DATA_OP,
977 : "[LocalCopyMemToBuffer] dstHandle=%llu, srcHandle=%llu, lenHandle=%llu,"
978 : " eventHandle=%llu, mask=%u",
979 : dstHandle, srcHandle, lenHandle, eventHandle, mask);
980 12 : CcuRep::CcuBuf* dst{nullptr};
981 12 : CCU_CHK_RET(GetBufferByHandle(dstHandle, &dst));
982 12 : CcuRep::LocalAddr* src{nullptr};
983 12 : CCU_CHK_RET(GetLocalAddrByHandle(srcHandle, &src));
984 12 : CcuRep::Variable* len{nullptr};
985 12 : CCU_CHK_RET(GetVariableByHandle(lenHandle, &len));
986 12 : CcuRep::CompletedEvent* event{nullptr};
987 12 : CCU_CHK_RET(GetEventByHandle(eventHandle, &event));
988 12 : auto ret = LocalCopyNb(*dst, *src, *len, *event, mask); // 复用 protected
989 12 : return HCCL_TO_CCU_RET(ret);
990 : }
991 :
992 8 : CcuResult CcuKernel::LocalCopyBufferToMem(
993 : CcuLocalAddrHandle dstHandle, CcuBufferHandle srcHandle, CcuVariableHandle lenHandle, CcuEventHandle eventHandle,
994 : uint32_t mask)
995 : {
996 8 : PLF_CONFIG_INFO(
997 : PLF_DATA_OP,
998 : "[LocalCopyBufferToMem] dstHandle=%llu, srcHandle=%llu, lenHandle=%llu,"
999 : " eventHandle=%llu, mask=%u",
1000 : dstHandle, srcHandle, lenHandle, eventHandle, mask);
1001 8 : CcuRep::LocalAddr* dst{nullptr};
1002 8 : CCU_CHK_RET(GetLocalAddrByHandle(dstHandle, &dst));
1003 8 : CcuRep::CcuBuf* src{nullptr};
1004 8 : CCU_CHK_RET(GetBufferByHandle(srcHandle, &src));
1005 8 : CcuRep::Variable* len{nullptr};
1006 8 : CCU_CHK_RET(GetVariableByHandle(lenHandle, &len));
1007 8 : CcuRep::CompletedEvent* event{nullptr};
1008 8 : CCU_CHK_RET(GetEventByHandle(eventHandle, &event));
1009 8 : auto ret = LocalCopyNb(*dst, *src, *len, *event, mask);
1010 8 : return HCCL_TO_CCU_RET(ret);
1011 : }
1012 :
1013 1 : CcuResult CcuKernel::LocalCopyMemToMem(
1014 : CcuLocalAddrHandle dstHandle, CcuLocalAddrHandle srcHandle, CcuVariableHandle lenHandle, CcuEventHandle eventHandle,
1015 : uint32_t mask)
1016 : {
1017 1 : PLF_CONFIG_INFO(
1018 : PLF_DATA_OP,
1019 : "[LocalCopyMemToMem] dstHandle=%llu, srcHandle=%llu, lenHandle=%llu,"
1020 : " eventHandle=%llu, mask=%u",
1021 : dstHandle, srcHandle, lenHandle, eventHandle, mask);
1022 1 : CcuRep::LocalAddr* dst{nullptr};
1023 1 : CCU_CHK_RET(GetLocalAddrByHandle(dstHandle, &dst));
1024 1 : CcuRep::LocalAddr* src{nullptr};
1025 1 : CCU_CHK_RET(GetLocalAddrByHandle(srcHandle, &src));
1026 1 : CcuRep::Variable* len{nullptr};
1027 1 : CCU_CHK_RET(GetVariableByHandle(lenHandle, &len));
1028 1 : CcuRep::CompletedEvent* event{nullptr};
1029 1 : CCU_CHK_RET(GetEventByHandle(eventHandle, &event));
1030 1 : auto ret = LocalCopyNb(*dst, *src, *len, *event, mask);
1031 1 : return HCCL_TO_CCU_RET(ret);
1032 : }
1033 :
1034 : // 本地reduce 相关实现
1035 0 : CcuResult CcuKernel::LocalMemReduce(
1036 : CcuLocalAddrHandle dstHandle, CcuLocalAddrHandle srcHandle, CcuVariableHandle lenHandle, HcclDataType dataType,
1037 : HcclReduceOp opType, CcuEventHandle eventHandle, uint32_t mask)
1038 : {
1039 0 : PLF_CONFIG_INFO(
1040 : PLF_DATA_OP,
1041 : "[LocalMemReduce] dstHandle=%llu, srcHandle=%llu, lenHandle=%llu, dataType=%d,"
1042 : " op=%d, eventHandle=%llu, mask=%u",
1043 : dstHandle, srcHandle, lenHandle, dataType, opType, eventHandle, mask);
1044 0 : CcuRep::LocalAddr* dst{nullptr};
1045 0 : CCU_CHK_RET(GetLocalAddrByHandle(dstHandle, &dst));
1046 0 : CcuRep::LocalAddr* src{nullptr};
1047 0 : CCU_CHK_RET(GetLocalAddrByHandle(srcHandle, &src));
1048 0 : CcuRep::Variable* len{nullptr};
1049 0 : CCU_CHK_RET(GetVariableByHandle(lenHandle, &len));
1050 0 : CcuRep::CompletedEvent* event{nullptr};
1051 0 : CCU_CHK_RET(GetEventByHandle(eventHandle, &event));
1052 0 : auto ret = LocalReduceNb(*dst, *src, *len, dataType, opType, *event, mask);
1053 0 : return HCCL_TO_CCU_RET(ret);
1054 : }
1055 :
1056 4 : CcuResult CcuKernel::LocalBufferReduce(
1057 : CcuBufferHandle* bufHandles, uint32_t count, HcclDataType dataType, HcclDataType outputDataType,
1058 : HcclReduceOp opType, CcuVariableHandle lenHandle, CcuEventHandle eventHandle, uint32_t mask)
1059 : {
1060 4 : PLF_CONFIG_INFO(
1061 : PLF_DATA_OP,
1062 : "[LocalBufferReduce] count=%u, dataType=%d, outDataType=%d,"
1063 : " op=%d, lenHandle=%llu, eventHandle=%llu, mask=%u",
1064 : count, dataType, outputDataType, opType, lenHandle, eventHandle, mask);
1065 4 : CcuRep::Variable* len{nullptr};
1066 4 : CCU_CHK_RET(GetVariableByHandle(lenHandle, &len));
1067 4 : CcuRep::CompletedEvent* event{nullptr};
1068 4 : CCU_CHK_RET(GetEventByHandle(eventHandle, &event));
1069 4 : std::vector<CcuRep::CcuBuf> bufs(count);
1070 12 : for (uint32_t i = 0; i < count; i++) {
1071 8 : CcuRep::CcuBuf* buf{nullptr};
1072 8 : CCU_CHK_RET(GetBufferByHandle(bufHandles[i], &buf));
1073 8 : bufs[i] = *buf;
1074 : }
1075 4 : auto ret = LocalReduceNb(bufs.data(), count, dataType, outputDataType, opType, *len, *event, mask);
1076 4 : return HCCL_TO_CCU_RET(ret);
1077 4 : }
1078 :
1079 : /*========== 远端数据传输操作 ==========*/
1080 :
1081 4 : CcuResult CcuKernel::ResolveBufRemoteLenEvent(
1082 : CcuBufferHandle bufHandle, CcuRemoteAddrHandle remoteHandle, CcuVariableHandle lenHandle,
1083 : CcuEventHandle eventHandle, CcuRep::CcuBuf** buf, CcuRep::RemoteAddr** remote, CcuRep::Variable** len,
1084 : CcuRep::CompletedEvent** event)
1085 : {
1086 4 : CCU_CHK_RET(GetBufferByHandle(bufHandle, buf));
1087 4 : CCU_CHK_RET(GetRemoteAddrByHandle(remoteHandle, remote));
1088 4 : CCU_CHK_RET(GetVariableByHandle(lenHandle, len));
1089 4 : CCU_CHK_RET(GetEventByHandle(eventHandle, event));
1090 4 : return CcuResult::CCU_SUCCESS;
1091 : }
1092 :
1093 6 : CcuResult CcuKernel::ResolveLocalRemoteLenEvent(
1094 : CcuLocalAddrHandle localHandle, CcuRemoteAddrHandle remoteHandle, CcuVariableHandle lenHandle,
1095 : CcuEventHandle eventHandle, CcuRep::LocalAddr** local, CcuRep::RemoteAddr** remote, CcuRep::Variable** len,
1096 : CcuRep::CompletedEvent** event)
1097 : {
1098 6 : CCU_CHK_RET(GetLocalAddrByHandle(localHandle, local));
1099 6 : CCU_CHK_RET(GetRemoteAddrByHandle(remoteHandle, remote));
1100 6 : CCU_CHK_RET(GetVariableByHandle(lenHandle, len));
1101 6 : CCU_CHK_RET(GetEventByHandle(eventHandle, event));
1102 6 : return CcuResult::CCU_SUCCESS;
1103 : }
1104 :
1105 5 : CcuResult CcuKernel::ResolveRemoteLocalLenEvent(
1106 : CcuRemoteAddrHandle remoteHandle, CcuLocalAddrHandle localHandle, CcuVariableHandle lenHandle,
1107 : CcuEventHandle eventHandle, CcuRep::RemoteAddr** remote, CcuRep::LocalAddr** local, CcuRep::Variable** len,
1108 : CcuRep::CompletedEvent** event)
1109 : {
1110 5 : CCU_CHK_RET(GetRemoteAddrByHandle(remoteHandle, remote));
1111 5 : CCU_CHK_RET(GetLocalAddrByHandle(localHandle, local));
1112 5 : CCU_CHK_RET(GetVariableByHandle(lenHandle, len));
1113 5 : CCU_CHK_RET(GetEventByHandle(eventHandle, event));
1114 5 : return CcuResult::CCU_SUCCESS;
1115 : }
1116 :
1117 4 : CcuResult CcuKernel::ReadMemToMem(
1118 : ChannelHandle channel, CcuLocalAddrHandle localHandle, CcuRemoteAddrHandle remoteHandle,
1119 : CcuVariableHandle lenHandle, CcuEventHandle eventHandle, uint32_t mask)
1120 : {
1121 4 : PLF_CONFIG_INFO(
1122 : PLF_DATA_OP,
1123 : "[ReadMemToMem] channel=%llu, localHandle=%llu, remoteHandle=%llu,"
1124 : " lenHandle=%llu, eventHandle=%llu, mask=%u",
1125 : channel, localHandle, remoteHandle, lenHandle, eventHandle, mask);
1126 4 : channels_.insert(channel);
1127 4 : CcuRep::LocalAddr* local{nullptr};
1128 4 : CcuRep::RemoteAddr* remote{nullptr};
1129 4 : CcuRep::Variable* len{nullptr};
1130 4 : CcuRep::CompletedEvent* event{nullptr};
1131 4 : CCU_CHK_RET(
1132 : ResolveLocalRemoteLenEvent(localHandle, remoteHandle, lenHandle, eventHandle, &local, &remote, &len, &event));
1133 4 : auto ret = ReadNb(channel, *local, *remote, *len, *event, mask);
1134 4 : return HCCL_TO_CCU_RET(ret);
1135 : }
1136 :
1137 2 : CcuResult CcuKernel::ReadMemToBuffer(
1138 : ChannelHandle channel, CcuBufferHandle localHandle, CcuRemoteAddrHandle remoteHandle, CcuVariableHandle lenHandle,
1139 : CcuEventHandle eventHandle, uint32_t mask)
1140 : {
1141 2 : PLF_CONFIG_INFO(
1142 : PLF_DATA_OP,
1143 : "[ReadMemToBuffer] channel=%llu, bufHandle=%llu, remoteHandle=%llu,"
1144 : " lenHandle=%llu, eventHandle=%llu, mask=%u",
1145 : channel, localHandle, remoteHandle, lenHandle, eventHandle, mask);
1146 2 : channels_.insert(channel);
1147 2 : CcuRep::CcuBuf* local{nullptr};
1148 2 : CcuRep::RemoteAddr* remote{nullptr};
1149 2 : CcuRep::Variable* len{nullptr};
1150 2 : CcuRep::CompletedEvent* event{nullptr};
1151 2 : CCU_CHK_RET(
1152 : ResolveBufRemoteLenEvent(localHandle, remoteHandle, lenHandle, eventHandle, &local, &remote, &len, &event));
1153 2 : auto ret = ReadNb(channel, *local, *remote, *len, *event, mask);
1154 2 : return HCCL_TO_CCU_RET(ret);
1155 : }
1156 :
1157 2 : CcuResult CcuKernel::ReadMemToMemReduce(
1158 : ChannelHandle channel, CcuLocalAddrHandle localHandle, CcuRemoteAddrHandle remoteHandle,
1159 : CcuVariableHandle lenHandle, HcclDataType dataType, HcclReduceOp opType, CcuEventHandle eventHandle, uint32_t mask)
1160 : {
1161 2 : PLF_CONFIG_INFO(
1162 : PLF_DATA_OP,
1163 : "[ReadMemToMemReduce] channel=%llu, lenHandle=%llu, dataType=%d, op=%d,"
1164 : " eventHandle=%llu, mask=%u",
1165 : channel, lenHandle, dataType, opType, eventHandle, mask);
1166 2 : channels_.insert(channel);
1167 2 : CcuRep::LocalAddr* local{nullptr};
1168 2 : CcuRep::RemoteAddr* remote{nullptr};
1169 2 : CcuRep::Variable* len{nullptr};
1170 2 : CcuRep::CompletedEvent* event{nullptr};
1171 2 : CCU_CHK_RET(
1172 : ResolveLocalRemoteLenEvent(localHandle, remoteHandle, lenHandle, eventHandle, &local, &remote, &len, &event));
1173 2 : auto ret = ReadReduceNb(channel, *local, *remote, *len, dataType, opType, *event, mask);
1174 2 : return HCCL_TO_CCU_RET(ret);
1175 : }
1176 :
1177 3 : CcuResult CcuKernel::WriteMemToMem(
1178 : ChannelHandle channel, CcuRemoteAddrHandle remoteHandle, CcuLocalAddrHandle localHandle,
1179 : CcuVariableHandle lenHandle, CcuEventHandle eventHandle, uint32_t mask)
1180 : {
1181 3 : PLF_CONFIG_INFO(
1182 : PLF_DATA_OP,
1183 : "[WriteMemToMem] channel=%llu, remoteHandle=%llu, localHandle=%llu,"
1184 : " lenHandle=%llu, eventHandle=%llu, mask=%u",
1185 : channel, remoteHandle, localHandle, lenHandle, eventHandle, mask);
1186 3 : channels_.insert(channel);
1187 3 : CcuRep::RemoteAddr* remote{nullptr};
1188 3 : CcuRep::LocalAddr* local{nullptr};
1189 3 : CcuRep::Variable* len{nullptr};
1190 3 : CcuRep::CompletedEvent* event{nullptr};
1191 3 : CCU_CHK_RET(
1192 : ResolveRemoteLocalLenEvent(remoteHandle, localHandle, lenHandle, eventHandle, &remote, &local, &len, &event));
1193 3 : auto ret = WriteNb(channel, *remote, *local, *len, *event, mask);
1194 3 : return HCCL_TO_CCU_RET(ret);
1195 : }
1196 :
1197 2 : CcuResult CcuKernel::WriteBufferToMem(
1198 : ChannelHandle channel, CcuRemoteAddrHandle remoteHandle, CcuBufferHandle localHandle, CcuVariableHandle lenHandle,
1199 : CcuEventHandle eventHandle, uint32_t mask)
1200 : {
1201 2 : PLF_CONFIG_INFO(
1202 : PLF_DATA_OP,
1203 : "[WriteBufferToMem] channel=%llu, remoteHandle=%llu, bufHandle=%llu,"
1204 : " lenHandle=%llu, eventHandle=%llu, mask=%u",
1205 : channel, remoteHandle, localHandle, lenHandle, eventHandle, mask);
1206 2 : channels_.insert(channel);
1207 2 : CcuRep::CcuBuf* local{nullptr};
1208 2 : CcuRep::RemoteAddr* remote{nullptr};
1209 2 : CcuRep::Variable* len{nullptr};
1210 2 : CcuRep::CompletedEvent* event{nullptr};
1211 2 : CCU_CHK_RET(
1212 : ResolveBufRemoteLenEvent(localHandle, remoteHandle, lenHandle, eventHandle, &local, &remote, &len, &event));
1213 2 : auto ret = WriteNb(channel, *remote, *local, *len, *event, mask);
1214 2 : return HCCL_TO_CCU_RET(ret);
1215 : }
1216 :
1217 2 : CcuResult CcuKernel::WriteMemToMemReduce(
1218 : ChannelHandle channel, CcuRemoteAddrHandle remoteHandle, CcuLocalAddrHandle localHandle,
1219 : CcuVariableHandle lenHandle, HcclDataType dataType, HcclReduceOp opType, CcuEventHandle eventHandle, uint32_t mask)
1220 : {
1221 2 : PLF_CONFIG_INFO(
1222 : PLF_DATA_OP,
1223 : "[WriteMemToMemReduce] channel=%llu, lenHandle=%llu, dataType=%d, op=%d,"
1224 : " eventHandle=%llu, mask=%u",
1225 : channel, lenHandle, dataType, opType, eventHandle, mask);
1226 2 : channels_.insert(channel);
1227 2 : CcuRep::RemoteAddr* remote{nullptr};
1228 2 : CcuRep::LocalAddr* local{nullptr};
1229 2 : CcuRep::Variable* len{nullptr};
1230 2 : CcuRep::CompletedEvent* event{nullptr};
1231 2 : CCU_CHK_RET(
1232 : ResolveRemoteLocalLenEvent(remoteHandle, localHandle, lenHandle, eventHandle, &remote, &local, &len, &event));
1233 2 : auto ret = WriteReduceNb(channel, *remote, *local, *len, dataType, opType, *event, mask);
1234 2 : return HCCL_TO_CCU_RET(ret);
1235 : }
1236 :
1237 1502 : void CcuKernel::FlushClosablePendingIfs()
1238 : {
1239 1502 : if (isFlushing_) {
1240 82 : return;
1241 : }
1242 1420 : isFlushing_ = true;
1243 2922 : while (IfLabelStackTopIsClosable()) {
1244 82 : const char* lbl = IfLabelStackPop();
1245 82 : if (lbl != nullptr) {
1246 82 : IfEnd(lbl);
1247 : }
1248 : }
1249 1420 : isFlushing_ = false;
1250 : }
1251 :
1252 1235 : void CcuKernel::Append(std::shared_ptr<CcuRep::CcuRepBase> rep)
1253 : {
1254 1235 : FlushClosablePendingIfs();
1255 1235 : CcuRep::CcuRepContext::Append(rep);
1256 1235 : }
1257 :
1258 : namespace {
1259 104 : std::shared_ptr<CcuRep::CcuRepJumpBase> MakeInvertedCondJumpImm(
1260 : CcuInsGeneratorBase* insGenerator, const std::string& destLabelStr, const CcuRep::Variable& targetVar,
1261 : const CcuRep::Variable& expectVar, const CcuRep::Variable& variable, uint64_t immediate,
1262 : CcuConditionType condType, const char* funcName)
1263 : {
1264 104 : switch (condType) {
1265 82 : case CCU_CONDITION_EQ:
1266 164 : return std::make_shared<CcuRep::CcuRepJumpNE>(
1267 82 : insGenerator, destLabelStr, targetVar, expectVar, variable, immediate);
1268 12 : case CCU_CONDITION_NE:
1269 24 : return std::make_shared<CcuRep::CcuRepJumpEQ>(
1270 12 : insGenerator, destLabelStr, targetVar, expectVar, variable, immediate);
1271 3 : case CCU_CONDITION_LT:
1272 6 : return std::make_shared<CcuRep::CcuRepJumpGE>(
1273 3 : insGenerator, destLabelStr, targetVar, expectVar, variable, immediate);
1274 2 : case CCU_CONDITION_LE:
1275 4 : return std::make_shared<CcuRep::CcuRepJumpGT>(
1276 2 : insGenerator, destLabelStr, targetVar, expectVar, variable, immediate);
1277 2 : case CCU_CONDITION_GT:
1278 4 : return std::make_shared<CcuRep::CcuRepJumpLE>(
1279 2 : insGenerator, destLabelStr, targetVar, expectVar, variable, immediate);
1280 3 : case CCU_CONDITION_GE:
1281 6 : return std::make_shared<CcuRep::CcuRepJumpLT>(
1282 3 : insGenerator, destLabelStr, targetVar, expectVar, variable, immediate);
1283 0 : default:
1284 0 : HCCL_ERROR("[%s] unsupported condition type: %d", funcName, condType);
1285 0 : return nullptr;
1286 : }
1287 : }
1288 :
1289 : // 双变量版本：当 (lhsVar OP rhsVar) 为假时跳转到 destLabelStr。
1290 4 : std::shared_ptr<CcuRep::CcuRepJumpBase> MakeInvertedCondJumpVar(
1291 : CcuInsGeneratorBase* insGenerator, const std::string& destLabelStr, const CcuRep::Variable& targetVar,
1292 : const CcuRep::Variable& lhsVar, const CcuRep::Variable& rhsVar, CcuConditionType condType, const char* funcName)
1293 : {
1294 4 : switch (condType) {
1295 1 : case CCU_CONDITION_EQ:
1296 1 : return std::make_shared<CcuRep::CcuRepJumpNE>(insGenerator, destLabelStr, targetVar, lhsVar, rhsVar);
1297 0 : case CCU_CONDITION_NE:
1298 0 : return std::make_shared<CcuRep::CcuRepJumpEQ>(insGenerator, destLabelStr, targetVar, lhsVar, rhsVar);
1299 2 : case CCU_CONDITION_LT:
1300 2 : return std::make_shared<CcuRep::CcuRepJumpGE>(insGenerator, destLabelStr, targetVar, lhsVar, rhsVar);
1301 0 : case CCU_CONDITION_LE:
1302 0 : return std::make_shared<CcuRep::CcuRepJumpGT>(insGenerator, destLabelStr, targetVar, lhsVar, rhsVar);
1303 0 : case CCU_CONDITION_GT:
1304 0 : return std::make_shared<CcuRep::CcuRepJumpLE>(insGenerator, destLabelStr, targetVar, lhsVar, rhsVar);
1305 1 : case CCU_CONDITION_GE:
1306 1 : return std::make_shared<CcuRep::CcuRepJumpLT>(insGenerator, destLabelStr, targetVar, lhsVar, rhsVar);
1307 0 : default:
1308 0 : HCCL_ERROR("[%s] unsupported condition type: %d", funcName, condType);
1309 0 : return nullptr;
1310 : }
1311 : }
1312 : } // namespace
1313 :
1314 : CcuResult
1315 102 : CcuKernel::IfBegin(CcuVariableHandle varHandle, uint64_t immediate, CcuConditionType condType, const char* label)
1316 : {
1317 102 : PLF_CONFIG_INFO(
1318 : PLF_DATA_OP, "[IfBegin] varHandle=%llu, immediate=%llu, condType=%d, label=%s", varHandle, immediate, condType,
1319 : (label ? label : "null"));
1320 102 : if (CurrentBlock()->Type() == CcuRep::CcuRepType::LOOP_BLOCK) {
1321 1 : HCCL_ERROR(
1322 : "[%s] CCU_IF is not allowed inside a ccu::Loop body (label='%s')", __func__,
1323 : label != nullptr ? label : "(null)");
1324 1 : return LatchBodyError(CcuResult::CCU_E_INTERNAL);
1325 : }
1326 101 : CcuRep::Variable* variable{nullptr};
1327 101 : CCU_CHK_RET(GetVariableByHandle(varHandle, &variable));
1328 :
1329 101 : FlushClosablePendingIfs();
1330 :
1331 101 : std::string labelStr(label);
1332 101 : if (pendingIfCtx_.find(labelStr) != pendingIfCtx_.end()) {
1333 0 : HCCL_ERROR("[%s] label '%s' already has a pending IfBegin without IfEnd", __func__, label);
1334 0 : return CcuResult::CCU_E_PARA;
1335 : }
1336 :
1337 101 : std::string elseLabelStr = labelStr + "_else";
1338 101 : std::string endLabelStr = labelStr + "_end";
1339 101 : auto elseLabel = std::make_shared<CcuRep::CcuRepJumpLabel>(insGenerator, elseLabelStr);
1340 101 : auto endLabel = std::make_shared<CcuRep::CcuRepJumpLabel>(insGenerator, endLabelStr);
1341 101 : auto targetVar = CreateJumpTargetVar();
1342 101 : auto expectVar = CreateExpectVar();
1343 :
1344 : // 反转条件："if <cond>, 执行块" 等价于 "!<cond> 时跳过块"。
1345 : auto jump = MakeInvertedCondJumpImm(
1346 101 : insGenerator, elseLabelStr, targetVar, expectVar, *variable, immediate, condType, __func__);
1347 101 : if (jump == nullptr) {
1348 0 : return CcuResult::CCU_E_PARA;
1349 : }
1350 101 : jump->Reference(elseLabel);
1351 101 : Append(jump);
1352 :
1353 101 : PendingIfContext ctx;
1354 101 : ctx.elseLabel = elseLabel;
1355 101 : ctx.endLabel = endLabel;
1356 101 : ctx.hasElse = false;
1357 101 : pendingIfCtx_.emplace(labelStr, std::move(ctx));
1358 :
1359 101 : return CcuResult::CCU_SUCCESS;
1360 101 : }
1361 :
1362 3 : CcuResult CcuKernel::IfBeginVar(
1363 : CcuVariableHandle lhsHandle, CcuVariableHandle rhsHandle, CcuConditionType condType, const char* label)
1364 : {
1365 3 : if (CurrentBlock()->Type() == CcuRep::CcuRepType::LOOP_BLOCK) {
1366 0 : HCCL_ERROR(
1367 : "[%s] CCU_IF is not allowed inside a ccu::Loop body (label='%s')", __func__,
1368 : label != nullptr ? label : "(null)");
1369 0 : return LatchBodyError(CcuResult::CCU_E_INTERNAL);
1370 : }
1371 3 : CcuRep::Variable* lhsVar{nullptr};
1372 3 : CcuRep::Variable* rhsVar{nullptr};
1373 3 : CCU_CHK_RET(GetVariableByHandle(lhsHandle, &lhsVar));
1374 3 : CCU_CHK_RET(GetVariableByHandle(rhsHandle, &rhsVar));
1375 :
1376 3 : FlushClosablePendingIfs();
1377 :
1378 3 : std::string labelStr(label);
1379 3 : if (pendingIfCtx_.find(labelStr) != pendingIfCtx_.end()) {
1380 0 : HCCL_ERROR("[%s] label '%s' already has a pending IfBegin without IfEnd", __func__, label);
1381 0 : return CcuResult::CCU_E_PARA;
1382 : }
1383 :
1384 3 : std::string endLabelStr = labelStr + "_end";
1385 3 : std::string elseLabelStr = labelStr + "_else";
1386 3 : auto endLabel = std::make_shared<CcuRep::CcuRepJumpLabel>(insGenerator, endLabelStr);
1387 3 : auto elseLabel = std::make_shared<CcuRep::CcuRepJumpLabel>(insGenerator, elseLabelStr);
1388 3 : auto targetVar = CreateJumpTargetVar();
1389 :
1390 3 : auto jump = MakeInvertedCondJumpVar(insGenerator, elseLabelStr, targetVar, *lhsVar, *rhsVar, condType, __func__);
1391 3 : if (jump == nullptr) {
1392 0 : return CcuResult::CCU_E_PARA;
1393 : }
1394 3 : jump->Reference(elseLabel);
1395 3 : Append(jump);
1396 :
1397 3 : PendingIfContext ctx;
1398 3 : ctx.hasElse = false;
1399 3 : ctx.endLabel = endLabel;
1400 3 : ctx.elseLabel = elseLabel;
1401 3 : pendingIfCtx_.emplace(labelStr, std::move(ctx));
1402 :
1403 3 : return CcuResult::CCU_SUCCESS;
1404 3 : }
1405 :
1406 22 : CcuResult CcuKernel::IfElse(const char* label)
1407 : {
1408 22 : PLF_CONFIG_INFO(PLF_DATA_OP, "[IfElse] label=%s", (label ? label : "null"));
1409 22 : if (CurrentBlock()->Type() == CcuRep::CcuRepType::LOOP_BLOCK) {
1410 0 : HCCL_ERROR(
1411 : "[%s] CCU_ELSE is not allowed inside a ccu::Loop body (label='%s')", __func__,
1412 : label != nullptr ? label : "(null)");
1413 0 : return LatchBodyError(CcuResult::CCU_E_INTERNAL);
1414 : }
1415 :
1416 22 : std::string labelStr(label);
1417 22 : auto iter = pendingIfCtx_.find(labelStr);
1418 22 : if (iter == pendingIfCtx_.end()) {
1419 0 : HCCL_ERROR("[%s] no matching IfBegin for label '%s'", __func__, label);
1420 0 : return CcuResult::CCU_E_NOT_FOUND;
1421 : }
1422 :
1423 22 : if (iter->second.hasElse) {
1424 0 : HCCL_ERROR("[%s] label '%s' already has an IfElse", __func__, label);
1425 0 : return CcuResult::CCU_E_PARA;
1426 : }
1427 :
1428 : // At end of then-block: unconditional jump past else-block to endLabel
1429 22 : std::string endLabelStr = labelStr + "_end";
1430 22 : auto skipElseVar = CreateJumpTargetVar();
1431 22 : auto skipElseJump = std::make_shared<CcuRep::CcuRepJump>(insGenerator, endLabelStr, skipElseVar);
1432 22 : skipElseJump->Reference(iter->second.endLabel);
1433 22 : Append(skipElseJump);
1434 :
1435 : // Place the else label (entry point of else-block)
1436 22 : Append(iter->second.elseLabel);
1437 :
1438 22 : iter->second.hasElse = true;
1439 :
1440 22 : return CcuResult::CCU_SUCCESS;
1441 22 : }
1442 :
1443 104 : CcuResult CcuKernel::IfEnd(const char* label)
1444 : {
1445 104 : PLF_CONFIG_INFO(PLF_DATA_OP, "[IfEnd] label=%s", (label ? label : "null"));
1446 104 : std::string labelStr(label);
1447 104 : auto iter = pendingIfCtx_.find(labelStr);
1448 104 : if (iter == pendingIfCtx_.end()) {
1449 0 : HCCL_ERROR("[%s] no matching IfBegin for label '%s'", __func__, label);
1450 0 : return CcuResult::CCU_E_NOT_FOUND;
1451 : }
1452 :
1453 104 : if (iter->second.hasElse) {
1454 : // Had else-block: place endLabel after else-block
1455 22 : Append(iter->second.endLabel);
1456 : } else {
1457 : // No else-block: place elseLabel as the skip target
1458 82 : Append(iter->second.elseLabel);
1459 : }
1460 :
1461 104 : pendingIfCtx_.erase(iter);
1462 :
1463 104 : return CcuResult::CCU_SUCCESS;
1464 104 : }
1465 :
1466 : CcuResult
1467 3 : CcuKernel::WhileBegin(CcuVariableHandle varHandle, uint64_t immediate, CcuConditionType condType, const char* label)
1468 : {
1469 3 : PLF_CONFIG_INFO(
1470 : PLF_DATA_OP, "[WhileBegin] varHandle=%llu, immediate=%llu, condType=%d, label=%s", varHandle, immediate,
1471 : condType, (label ? label : "null"));
1472 3 : if (CurrentBlock()->Type() == CcuRep::CcuRepType::LOOP_BLOCK) {
1473 0 : HCCL_ERROR(
1474 : "[%s] CCU_WHILE is not allowed inside a ccu::Loop body (label='%s')", __func__,
1475 : label != nullptr ? label : "(null)");
1476 0 : return LatchBodyError(CcuResult::CCU_E_INTERNAL);
1477 : }
1478 :
1479 3 : CcuRep::Variable* variable{nullptr};
1480 3 : CCU_CHK_RET(GetVariableByHandle(varHandle, &variable));
1481 :
1482 3 : std::string labelStr(label);
1483 3 : if (pendingWhileCtx_.find(labelStr) != pendingWhileCtx_.end()) {
1484 0 : HCCL_ERROR("[%s] label '%s' already has a pending WhileBegin without WhileEnd", __func__, label);
1485 0 : return CcuResult::CCU_E_PARA;
1486 : }
1487 :
1488 3 : std::string beginLabelStr = labelStr + "_begin";
1489 3 : std::string endLabelStr = labelStr + "_end";
1490 3 : auto beginLabel = std::make_shared<CcuRep::CcuRepJumpLabel>(insGenerator, beginLabelStr);
1491 3 : auto endLabel = std::make_shared<CcuRep::CcuRepJumpLabel>(insGenerator, endLabelStr);
1492 :
1493 3 : Append(beginLabel);
1494 :
1495 3 : auto targetVar = CcuRep::CreateVariable(this);
1496 3 : auto expectVar = CreateExpectVar();
1497 : auto jump = MakeInvertedCondJumpImm(
1498 3 : insGenerator, endLabelStr, targetVar, expectVar, *variable, immediate, condType, __func__);
1499 3 : if (jump == nullptr) {
1500 0 : return CcuResult::CCU_E_PARA;
1501 : }
1502 3 : jump->Reference(endLabel);
1503 3 : Append(jump);
1504 :
1505 3 : PendingWhileContext ctx;
1506 3 : ctx.beginLabel = beginLabel;
1507 3 : ctx.endLabel = endLabel;
1508 3 : ctx.varHandle = varHandle;
1509 3 : ctx.immediate = immediate;
1510 3 : ctx.condType = condType;
1511 3 : pendingWhileCtx_.emplace(labelStr, std::move(ctx));
1512 :
1513 3 : return CcuResult::CCU_SUCCESS;
1514 3 : }
1515 :
1516 1 : CcuResult CcuKernel::WhileBeginVar(
1517 : CcuVariableHandle lhsHandle, CcuVariableHandle rhsHandle, CcuConditionType condType, const char* label)
1518 : {
1519 1 : if (CurrentBlock()->Type() == CcuRep::CcuRepType::LOOP_BLOCK) {
1520 0 : HCCL_ERROR(
1521 : "[%s] CCU_WHILE is not allowed inside a ccu::Loop body (label='%s')", __func__,
1522 : label != nullptr ? label : "(null)");
1523 0 : return LatchBodyError(CcuResult::CCU_E_INTERNAL);
1524 : }
1525 :
1526 1 : CcuRep::Variable* lhsVar{nullptr};
1527 1 : CcuRep::Variable* rhsVar{nullptr};
1528 1 : CCU_CHK_RET(GetVariableByHandle(lhsHandle, &lhsVar));
1529 1 : CCU_CHK_RET(GetVariableByHandle(rhsHandle, &rhsVar));
1530 :
1531 1 : std::string labelStr(label);
1532 1 : if (pendingWhileCtx_.find(labelStr) != pendingWhileCtx_.end()) {
1533 0 : HCCL_ERROR("[%s] label '%s' already has a pending WhileBegin without WhileEnd", __func__, label);
1534 0 : return CcuResult::CCU_E_PARA;
1535 : }
1536 :
1537 1 : std::string endLabelStr = labelStr + "_end";
1538 1 : std::string beginLabelStr = labelStr + "_begin";
1539 1 : auto endLabel = std::make_shared<CcuRep::CcuRepJumpLabel>(insGenerator, endLabelStr);
1540 1 : auto beginLabel = std::make_shared<CcuRep::CcuRepJumpLabel>(insGenerator, beginLabelStr);
1541 :
1542 1 : Append(beginLabel);
1543 :
1544 1 : auto targetVar = CcuRep::CreateVariable(this);
1545 1 : auto jump = MakeInvertedCondJumpVar(insGenerator, endLabelStr, targetVar, *lhsVar, *rhsVar, condType, __func__);
1546 1 : if (jump == nullptr) {
1547 0 : return CcuResult::CCU_E_PARA;
1548 : }
1549 1 : jump->Reference(endLabel);
1550 1 : Append(jump);
1551 :
1552 1 : PendingWhileContext ctx;
1553 1 : ctx.beginLabel = beginLabel;
1554 1 : ctx.endLabel = endLabel;
1555 1 : ctx.varHandle = lhsHandle;
1556 1 : ctx.immediate = 0;
1557 1 : ctx.condType = condType;
1558 1 : pendingWhileCtx_.emplace(labelStr, std::move(ctx));
1559 :
1560 1 : return CcuResult::CCU_SUCCESS;
1561 1 : }
1562 :
1563 4 : CcuResult CcuKernel::WhileEnd(const char* label)
1564 : {
1565 4 : PLF_CONFIG_INFO(PLF_DATA_OP, "[WhileEnd] label=%s", (label ? label : "null"));
1566 4 : std::string labelStr(label);
1567 4 : auto iter = pendingWhileCtx_.find(labelStr);
1568 4 : if (iter == pendingWhileCtx_.end()) {
1569 0 : HCCL_ERROR("[%s] no matching WhileBegin for label '%s'", __func__, label);
1570 0 : return CcuResult::CCU_E_NOT_FOUND;
1571 : }
1572 :
1573 4 : std::string beginLabelStr = labelStr + "_begin";
1574 4 : auto loopBackVar = CcuRep::CreateVariable(this);
1575 4 : auto loopBackJump = std::make_shared<CcuRep::CcuRepJump>(insGenerator, beginLabelStr, loopBackVar);
1576 4 : loopBackJump->Reference(iter->second.beginLabel);
1577 4 : Append(loopBackJump);
1578 :
1579 4 : Append(iter->second.endLabel);
1580 :
1581 4 : pendingWhileCtx_.erase(iter);
1582 :
1583 4 : return CcuResult::CCU_SUCCESS;
1584 4 : }
1585 :
1586 9 : CcuResult CcuKernel::DoWhileBegin(const char* label)
1587 : {
1588 9 : PLF_CONFIG_INFO(PLF_DATA_OP, "[DoWhileBegin] label=%s", (label ? label : "null"));
1589 9 : if (CurrentBlock()->Type() == CcuRep::CcuRepType::LOOP_BLOCK) {
1590 0 : HCCL_ERROR(
1591 : "[%s] CCU_DO is not allowed inside a ccu::Loop body (label='%s')", __func__,
1592 : label != nullptr ? label : "(null)");
1593 0 : return LatchBodyError(CcuResult::CCU_E_INTERNAL);
1594 : }
1595 :
1596 9 : std::string labelStr(label);
1597 9 : if (pendingDoWhileCtx_.find(labelStr) != pendingDoWhileCtx_.end()) {
1598 0 : HCCL_ERROR("[%s] label '%s' already has a pending DoWhileBegin without DoWhileEnd", __func__, label);
1599 0 : return CcuResult::CCU_E_PARA;
1600 : }
1601 :
1602 9 : std::string beginLabelStr = labelStr + "_begin";
1603 9 : auto beginLabel = std::make_shared<CcuRep::CcuRepJumpLabel>(insGenerator, beginLabelStr);
1604 9 : Append(beginLabel);
1605 :
1606 9 : PendingDoWhileContext ctx;
1607 9 : ctx.beginLabel = beginLabel;
1608 9 : pendingDoWhileCtx_.emplace(labelStr, std::move(ctx));
1609 :
1610 9 : return CcuResult::CCU_SUCCESS;
1611 9 : }
1612 :
1613 : CcuResult
1614 5 : CcuKernel::DoWhileEnd(CcuVariableHandle varHandle, uint64_t immediate, CcuConditionType condType, const char* label)
1615 : {
1616 5 : PLF_CONFIG_INFO(
1617 : PLF_DATA_OP, "[DoWhileEnd] varHandle=%llu, immediate=%llu, condType=%d, label=%s", varHandle, immediate,
1618 : condType, (label ? label : "null"));
1619 5 : CcuRep::Variable* variable{nullptr};
1620 5 : CCU_CHK_RET(GetVariableByHandle(varHandle, &variable));
1621 :
1622 5 : std::string labelStr(label);
1623 5 : auto iter = pendingDoWhileCtx_.find(labelStr);
1624 5 : if (iter == pendingDoWhileCtx_.end()) {
1625 0 : HCCL_ERROR("[%s] no matching DoWhileBegin for label '%s'", __func__, label);
1626 0 : return CcuResult::CCU_E_NOT_FOUND;
1627 : }
1628 :
1629 5 : std::string beginLabelStr = labelStr + "_begin";
1630 5 : auto targetVar = CcuRep::CreateVariable(this);
1631 5 : auto expectVar = CreateExpectVar();
1632 5 : std::shared_ptr<CcuRep::CcuRepJumpBase> jump{nullptr};
1633 :
1634 : // "condition true => continue looping" means jump back to begin when condition holds
1635 5 : if (condType == CCU_CONDITION_EQ) {
1636 0 : jump = std::make_shared<CcuRep::CcuRepJumpEQ>(
1637 0 : insGenerator, beginLabelStr, targetVar, expectVar, *variable, immediate);
1638 5 : } else if (condType == CCU_CONDITION_NE) {
1639 4 : jump = std::make_shared<CcuRep::CcuRepJumpNE>(
1640 4 : insGenerator, beginLabelStr, targetVar, expectVar, *variable, immediate);
1641 1 : } else if (condType == CCU_CONDITION_LT) {
1642 0 : jump = std::make_shared<CcuRep::CcuRepJumpLT>(
1643 0 : insGenerator, beginLabelStr, targetVar, expectVar, *variable, immediate);
1644 1 : } else if (condType == CCU_CONDITION_LE) {
1645 1 : jump = std::make_shared<CcuRep::CcuRepJumpLE>(
1646 1 : insGenerator, beginLabelStr, targetVar, expectVar, *variable, immediate);
1647 0 : } else if (condType == CCU_CONDITION_GT) {
1648 0 : jump = std::make_shared<CcuRep::CcuRepJumpGT>(
1649 0 : insGenerator, beginLabelStr, targetVar, expectVar, *variable, immediate);
1650 0 : } else if (condType == CCU_CONDITION_GE) {
1651 0 : jump = std::make_shared<CcuRep::CcuRepJumpGE>(
1652 0 : insGenerator, beginLabelStr, targetVar, expectVar, *variable, immediate);
1653 : } else {
1654 0 : HCCL_ERROR("[%s] unsupported condition type: %d", __func__, condType);
1655 0 : return CcuResult::CCU_E_PARA;
1656 : }
1657 :
1658 5 : jump->Reference(iter->second.beginLabel);
1659 5 : Append(jump);
1660 :
1661 5 : pendingDoWhileCtx_.erase(iter);
1662 :
1663 5 : return CcuResult::CCU_SUCCESS;
1664 5 : }
1665 :
1666 1 : CcuResult CcuKernel::DoWhileEndVar(
1667 : CcuVariableHandle lhsHandle, CcuVariableHandle rhsHandle, CcuConditionType condType, const char* label)
1668 : {
1669 1 : CcuRep::Variable* lhsVar{nullptr};
1670 1 : CcuRep::Variable* rhsVar{nullptr};
1671 1 : CCU_CHK_RET(GetVariableByHandle(lhsHandle, &lhsVar));
1672 1 : CCU_CHK_RET(GetVariableByHandle(rhsHandle, &rhsVar));
1673 :
1674 1 : std::string labelStr(label);
1675 1 : auto iter = pendingDoWhileCtx_.find(labelStr);
1676 1 : if (iter == pendingDoWhileCtx_.end()) {
1677 0 : HCCL_ERROR("[%s] no matching DoWhileBegin for label '%s'", __func__, label);
1678 0 : return CcuResult::CCU_E_NOT_FOUND;
1679 : }
1680 :
1681 1 : std::string beginLabelStr = labelStr + "_begin";
1682 1 : auto targetVar = CcuRep::CreateVariable(this);
1683 1 : std::shared_ptr<CcuRep::CcuRepJumpBase> jump{nullptr};
1684 :
1685 1 : if (condType == CCU_CONDITION_EQ) {
1686 0 : jump = std::make_shared<CcuRep::CcuRepJumpEQ>(insGenerator, beginLabelStr, targetVar, *lhsVar, *rhsVar);
1687 1 : } else if (condType == CCU_CONDITION_NE) {
1688 1 : jump = std::make_shared<CcuRep::CcuRepJumpNE>(insGenerator, beginLabelStr, targetVar, *lhsVar, *rhsVar);
1689 0 : } else if (condType == CCU_CONDITION_LT) {
1690 0 : jump = std::make_shared<CcuRep::CcuRepJumpLT>(insGenerator, beginLabelStr, targetVar, *lhsVar, *rhsVar);
1691 0 : } else if (condType == CCU_CONDITION_LE) {
1692 0 : jump = std::make_shared<CcuRep::CcuRepJumpLE>(insGenerator, beginLabelStr, targetVar, *lhsVar, *rhsVar);
1693 0 : } else if (condType == CCU_CONDITION_GT) {
1694 0 : jump = std::make_shared<CcuRep::CcuRepJumpGT>(insGenerator, beginLabelStr, targetVar, *lhsVar, *rhsVar);
1695 0 : } else if (condType == CCU_CONDITION_GE) {
1696 0 : jump = std::make_shared<CcuRep::CcuRepJumpGE>(insGenerator, beginLabelStr, targetVar, *lhsVar, *rhsVar);
1697 : } else {
1698 0 : HCCL_ERROR("[%s] unsupported condition type: %d", __func__, condType);
1699 0 : return CcuResult::CCU_E_PARA;
1700 : }
1701 :
1702 1 : jump->Reference(iter->second.beginLabel);
1703 1 : Append(jump);
1704 :
1705 1 : pendingDoWhileCtx_.erase(iter);
1706 :
1707 1 : return CcuResult::CCU_SUCCESS;
1708 1 : }
1709 :
1710 : // 控制流标签栈实体
1711 104 : void CcuKernel::IfLabelStackPush(const char* label) { iflabelStack_.push_back({label, false}); }
1712 104 : void CcuKernel::IfLabelStackMarkBodyDone()
1713 : {
1714 104 : if (iflabelStack_.empty()) {
1715 0 : HCCL_ERROR("[CcuKernel::IfLabelStack][MarkBodyDone] stack is empty");
1716 0 : return;
1717 : }
1718 104 : iflabelStack_.back().bodyDone = true;
1719 : }
1720 22 : const char* CcuKernel::IfLabelStackPopForElse()
1721 : {
1722 22 : if (iflabelStack_.empty()) {
1723 0 : HCCL_ERROR("[CcuKernel::IfLabelStack][PopForElse] orphan CCU_ELSE: "
1724 : "no matching CCU_IF on the stack");
1725 0 : return nullptr;
1726 : }
1727 22 : if (!iflabelStack_.back().bodyDone) {
1728 0 : HCCL_ERROR(
1729 : "[CcuKernel::IfLabelStack][PopForElse] CCU_ELSE called while "
1730 : "top if-body is still InBody (label='%s')",
1731 : iflabelStack_.back().label != nullptr ? iflabelStack_.back().label : "(null)");
1732 0 : return nullptr;
1733 : }
1734 22 : const char* label = iflabelStack_.back().label;
1735 22 : iflabelStack_.pop_back();
1736 22 : return label;
1737 : }
1738 1502 : bool CcuKernel::IfLabelStackTopIsClosable() { return !iflabelStack_.empty() && iflabelStack_.back().bodyDone; }
1739 :
1740 82 : const char* CcuKernel::IfLabelStackPop()
1741 : {
1742 82 : if (iflabelStack_.empty()) {
1743 0 : return nullptr;
1744 : }
1745 82 : const char* label = iflabelStack_.back().label;
1746 82 : iflabelStack_.pop_back();
1747 82 : return label;
1748 : }
1749 :
1750 9 : void CcuKernel::DoWhileLabelStackPush(const char* label)
1751 : {
1752 9 : DoWhileLabelEntry entry;
1753 9 : entry.label = label;
1754 9 : entry.snapshotBlock = CurrentBlock();
1755 9 : entry.snapshotRepCount = (entry.snapshotBlock != nullptr) ? entry.snapshotBlock->GetReps().size() : 0;
1756 9 : doWhileLabelStack_.push_back(std::move(entry));
1757 9 : }
1758 :
1759 13 : const char* CcuKernel::DoWhileLabelStackPopForWhile()
1760 : {
1761 13 : if (CurrentBlock()->Type() == CcuRep::CcuRepType::LOOP_BLOCK) {
1762 1 : return nullptr;
1763 : }
1764 12 : if (doWhileLabelStack_.empty()) {
1765 4 : return nullptr;
1766 : }
1767 8 : DoWhileLabelEntry entry = doWhileLabelStack_.back();
1768 8 : doWhileLabelStack_.pop_back();
1769 :
1770 8 : auto currentBlock = CurrentBlock();
1771 8 : size_t currentRepCount = (currentBlock != nullptr) ? currentBlock->GetReps().size() : 0;
1772 8 : if (currentBlock != entry.snapshotBlock || currentRepCount != entry.snapshotRepCount) {
1773 1 : HCCL_ERROR(
1774 : "[CcuKernel::DoWhileLabelStackPopForWhile] dangling CCU calls between CCU_DO end "
1775 : "and CCU_WHILE (label='%s', snapRep=%zu, curRep=%zu, blockChanged=%d); they must "
1776 : "be syntactically adjacent, otherwise the code in between is pulled into the body.",
1777 : entry.label != nullptr ? entry.label : "(null)", entry.snapshotRepCount, currentRepCount,
1778 : currentBlock != entry.snapshotBlock ? 1 : 0);
1779 1 : return nullptr;
1780 : }
1781 7 : return entry.label;
1782 8 : }
1783 :
1784 152 : CcuResult CcuKernel::GetAddressByHandle(CcuAddressHandle addrHandle, CcuRep::Address** address)
1785 : {
1786 152 : return GetResourceByHandle(ccuAddrMap_, addrHandle, address, "address");
1787 : }
1788 :
1789 : // addr = 立即数 → CcuRepAssign(Address, uint64_t)
1790 13 : CcuResult CcuKernel::AddressAssignImm(CcuAddressHandle addrHandle, uint64_t immediate)
1791 : {
1792 13 : PLF_CONFIG_INFO(PLF_DATA_OP, "[AddressAssignImm] addrHandle=%llu, immediate=%llu", addrHandle, immediate);
1793 13 : CcuRep::Address* address{nullptr};
1794 13 : CCU_CHK_RET(GetAddressByHandle(addrHandle, &address));
1795 13 : (*address) = immediate;
1796 13 : return CcuResult::CCU_SUCCESS;
1797 : }
1798 :
1799 : // addr = variable → CcuRepAssign(Address, Variable)
1800 14 : CcuResult CcuKernel::AddressAssignVar(CcuAddressHandle addrHandle, CcuVariableHandle varHandle)
1801 : {
1802 14 : PLF_CONFIG_INFO(PLF_DATA_OP, "[AddressAssignVar] addrHandle=%llu, varHandle=%llu", addrHandle, varHandle);
1803 14 : CcuRep::Address* address{nullptr};
1804 14 : CCU_CHK_RET(GetAddressByHandle(addrHandle, &address));
1805 :
1806 14 : CcuRep::Variable* variable{nullptr};
1807 14 : CCU_CHK_RET(GetVariableByHandle(varHandle, &variable));
1808 :
1809 14 : (*address) = (*variable);
1810 14 : return CcuResult::CCU_SUCCESS;
1811 : }
1812 : // addr = addr → CcuRepAssign(Address, Address)
1813 38 : CcuResult CcuKernel::AddressAssignAddr(CcuAddressHandle dstAddrHandle, CcuAddressHandle srcAddrHandle)
1814 : {
1815 38 : PLF_CONFIG_INFO(
1816 : PLF_DATA_OP, "[AddressAssignAddr] dstAddrHandle=%llu, srcAddrHandle=%llu", dstAddrHandle, srcAddrHandle);
1817 38 : CcuRep::Address* dstAddress{nullptr};
1818 38 : CCU_CHK_RET(GetAddressByHandle(dstAddrHandle, &dstAddress));
1819 :
1820 38 : CcuRep::Address* srcAddress{nullptr};
1821 38 : CCU_CHK_RET(GetAddressByHandle(srcAddrHandle, &srcAddress));
1822 :
1823 38 : (*dstAddress) = (*srcAddress);
1824 38 : return CcuResult::CCU_SUCCESS;
1825 : }
1826 :
1827 : // resAddr = lhsAddr + rhsVar → CcuRepAdd(Address, Address, Variable)
1828 0 : CcuResult CcuKernel::AddressAddVarToAddr(
1829 : CcuAddressHandle resAddrHandle, CcuAddressHandle lhsAddrHandle, CcuVariableHandle rhsVarHandle)
1830 : {
1831 0 : PLF_CONFIG_INFO(
1832 : PLF_DATA_OP, "[AddressAddVarToAddr] resAddrHandle=%llu, lhsAddrHandle=%llu, rhsVarHandle=%llu", resAddrHandle,
1833 : lhsAddrHandle, rhsVarHandle);
1834 0 : CcuRep::Address *resAddr{nullptr}, *lhsAddr{nullptr};
1835 0 : CCU_CHK_RET(GetAddressByHandle(resAddrHandle, &resAddr));
1836 0 : CCU_CHK_RET(GetAddressByHandle(lhsAddrHandle, &lhsAddr));
1837 :
1838 0 : CcuRep::Variable* rhsVar{nullptr};
1839 0 : CCU_CHK_RET(GetVariableByHandle(rhsVarHandle, &rhsVar));
1840 :
1841 0 : *resAddr = *lhsAddr + *rhsVar;
1842 0 : return CcuResult::CCU_SUCCESS;
1843 : }
1844 :
1845 : // resAddr = addrA + addrB → CcuRepAdd(Address, Address, Address)
1846 2 : CcuResult CcuKernel::AddressAddAddrToAddr(
1847 : CcuAddressHandle resAddrHandle, CcuAddressHandle addrAHandle, CcuAddressHandle addrBHandle)
1848 : {
1849 2 : PLF_CONFIG_INFO(
1850 : PLF_DATA_OP, "[AddressAddAddrToAddr] resAddrHandle=%llu, addrAHandle=%llu, addrBHandle=%llu", resAddrHandle,
1851 : addrAHandle, addrBHandle);
1852 2 : CcuRep::Address *resAddr{nullptr}, *addrA{nullptr}, *addrB{nullptr};
1853 2 : CCU_CHK_RET(GetAddressByHandle(resAddrHandle, &resAddr));
1854 2 : CCU_CHK_RET(GetAddressByHandle(addrAHandle, &addrA));
1855 2 : CCU_CHK_RET(GetAddressByHandle(addrBHandle, &addrB));
1856 :
1857 2 : *resAddr = *addrA + *addrB;
1858 2 : return CcuResult::CCU_SUCCESS;
1859 : }
1860 :
1861 : // addr += variable → CcuRepAdd(Address, Variable) 就地加
1862 41 : CcuResult CcuKernel::AddressAddAssignVar(CcuAddressHandle addrHandle, CcuVariableHandle varHandle)
1863 : {
1864 41 : PLF_CONFIG_INFO(PLF_DATA_OP, "[AddressAddAssignVar] addrHandle=%llu, varHandle=%llu", addrHandle, varHandle);
1865 41 : CcuRep::Address* address{nullptr};
1866 41 : CCU_CHK_RET(GetAddressByHandle(addrHandle, &address));
1867 :
1868 41 : CcuRep::Variable* variable{nullptr};
1869 41 : CCU_CHK_RET(GetVariableByHandle(varHandle, &variable));
1870 :
1871 41 : (*address) += (*variable);
1872 41 : return CcuResult::CCU_SUCCESS;
1873 : }
1874 :
1875 : // addr += addr → 等价于 addr = addr + otherAddr
1876 0 : CcuResult CcuKernel::AddressAddAssignAddr(CcuAddressHandle addrHandle, CcuAddressHandle otherHandle)
1877 : {
1878 0 : PLF_CONFIG_INFO(PLF_DATA_OP, "[AddressAddAssignAddr] addrHandle=%llu, otherHandle=%llu", addrHandle, otherHandle);
1879 0 : CcuRep::Address* address{nullptr};
1880 0 : CCU_CHK_RET(GetAddressByHandle(addrHandle, &address));
1881 :
1882 0 : CcuRep::Address* other{nullptr};
1883 0 : CCU_CHK_RET(GetAddressByHandle(otherHandle, &other));
1884 :
1885 0 : (*address) = (*address) + (*other);
1886 0 : return CcuResult::CCU_SUCCESS;
1887 : }
1888 :
1889 1 : CcuResult CcuKernel::AddressAddImmToAddr(CcuAddressHandle resAddrHandle, CcuAddressHandle addrAHandle, uint16_t imm)
1890 : {
1891 1 : CcuRep::Address *resAddr{nullptr}, *addrA{nullptr};
1892 1 : CCU_CHK_RET(GetAddressByHandle(resAddrHandle, &resAddr));
1893 1 : CCU_CHK_RET(GetAddressByHandle(addrAHandle, &addrA));
1894 :
1895 1 : *resAddr = *addrA + imm;
1896 1 : return CcuResult::CCU_SUCCESS;
1897 : }
1898 :
1899 0 : void CcuKernel::Load(const CcuRep::Variable& var)
1900 : {
1901 : auto loadArgRep = std::make_shared<CcuRep::CcuRepLoadArg>(
1902 0 : insGenerator, var, loadArgIndex_ % CCU_SQE_ARGS_LEN, static_cast<uint16_t>(loadArgIndex_));
1903 0 : GetLGProfilingInfo().loadRep2ArgIdxMap[loadArgRep] = loadArgIndex_;
1904 0 : Append(loadArgRep);
1905 0 : loadArgIndex_++;
1906 0 : }
1907 :
1908 0 : void CcuKernel::StoreVariable(const CcuRep::Variable& var, uint64_t addr)
1909 : {
1910 0 : Append(std::make_shared<CcuRep::CcuRepStore>(insGenerator, var, addr));
1911 0 : }
1912 :
1913 0 : void CcuKernel::LoadVariable(const CcuRep::Variable& src, const CcuRep::Variable& var)
1914 : {
1915 0 : Append(std::make_shared<CcuRep::CcuRepLoadVar>(insGenerator, src, var));
1916 0 : }
1917 :
1918 18 : HcclResult CcuKernel::RecordEvent(CcuRep::CompletedEvent event, uint32_t mask)
1919 : {
1920 18 : if (CurrentBlock()->Type() == CcuRep::CcuRepType::LOOP_BLOCK) {
1921 1 : HCCL_ERROR("[CcuKernel][%s] is not supported in loop block, please check.", __func__);
1922 1 : LatchBodyError(HCCL_TO_CCU_RET(HcclResult::HCCL_E_NOT_SUPPORT));
1923 1 : return HcclResult::HCCL_E_NOT_SUPPORT;
1924 : }
1925 :
1926 17 : auto rep = std::make_shared<CcuRep::CcuRepLocRecordEvent>(insGenerator, event, mask);
1927 17 : Append(rep);
1928 17 : SetDependencyInfo(event.Id(), mask, rep);
1929 17 : return HCCL_SUCCESS;
1930 17 : }
1931 :
1932 44 : HcclResult CcuKernel::WaitEvent(CcuRep::CompletedEvent event, uint32_t mask)
1933 : {
1934 44 : bool isProfiling = CurrentBlock()->Type() != CcuRep::CcuRepType::LOOP_BLOCK;
1935 44 : auto rep = std::make_shared<CcuRep::CcuRepLocWaitEvent>(insGenerator, event, mask, isProfiling);
1936 44 : if (isProfiling) {
1937 84 : CHK_RET(static_cast<HcclResult>(AddProfiling("WaitEvent", rep->GetMask())));
1938 : }
1939 44 : rep->SetDependencyInfo(GetDependencyInfo(event.Id()));
1940 44 : EraseDependencyInfo(event.Id());
1941 44 : Append(rep);
1942 44 : return HCCL_SUCCESS;
1943 44 : }
1944 :
1945 102 : CcuResult CcuKernel::GetEventByHandle(CcuEventHandle eventHandle, CcuRep::CompletedEvent** event)
1946 : {
1947 102 : return GetResourceByHandle(ccuEventMap_, eventHandle, event, "completedEvent");
1948 : }
1949 :
1950 : /*
1951 : LocalAddr / RemoteAddr 相关接口
1952 : */
1953 33 : CcuResult CcuKernel::GetLocalAddrByHandle(CcuLocalAddrHandle handle, CcuRep::LocalAddr** localAddr)
1954 : {
1955 33 : return GetResourceByHandle(ccuLocalAddrMap_, handle, localAddr, "localAddr");
1956 : }
1957 :
1958 15 : CcuResult CcuKernel::GetRemoteAddrByHandle(CcuRemoteAddrHandle handle, CcuRep::RemoteAddr** remoteAddr)
1959 : {
1960 15 : return GetResourceByHandle(ccuRemoteAddrMap_, handle, remoteAddr, "remoteAddr");
1961 : }
1962 :
1963 : /*Read新接口*/
1964 2 : HcclResult CcuKernel::ReadNb(
1965 : const ChannelHandle channel, const CcuRep::CcuBuf& loc, const CcuRep::RemoteAddr& rem, const CcuRep::Variable& len,
1966 : CcuRep::CompletedEvent event, uint32_t mask)
1967 : {
1968 2 : channels_.insert(channel);
1969 2 : auto rep = std::make_shared<CcuRep::CcuRepBufRead>(insGenerator, channel, rem, loc, len, event, mask);
1970 2 : Append(rep);
1971 2 : SetDependencyInfo(event.Id(), mask, rep);
1972 2 : return HCCL_SUCCESS;
1973 2 : }
1974 :
1975 : /*Write新接口*/
1976 2 : HcclResult CcuKernel::WriteNb(
1977 : const ChannelHandle channel, const CcuRep::RemoteAddr& rem, const CcuRep::CcuBuf& loc, const CcuRep::Variable& len,
1978 : CcuRep::CompletedEvent event, uint32_t mask)
1979 : {
1980 2 : channels_.insert(channel);
1981 2 : auto rep = std::make_shared<CcuRep::CcuRepBufWrite>(insGenerator, channel, loc, rem, len, event, mask);
1982 2 : Append(rep);
1983 2 : SetDependencyInfo(event.Id(), mask, rep);
1984 2 : return HCCL_SUCCESS;
1985 2 : }
1986 :
1987 8 : static bool isLowPrecisionIn(Hccl::DataType dataType)
1988 : {
1989 16 : return dataType == Hccl::DataType::INT8 || dataType == Hccl::DataType::HIF8 || dataType == Hccl::DataType::FP8E4M3
1990 16 : || dataType == Hccl::DataType::FP8E5M2;
1991 : }
1992 :
1993 0 : static bool isLowPrecisionOut(Hccl::DataType dataType)
1994 : {
1995 0 : return dataType == Hccl::DataType::FP16 || dataType == Hccl::DataType::BFP16 || dataType == Hccl::DataType::FP32;
1996 : }
1997 :
1998 : constexpr uint32_t MAX_DATA_TYPE = 17;
1999 :
2000 : const Hccl::DataType orionDataTypes[]
2001 : = {Hccl::DataType::INT8, Hccl::DataType::INT16, Hccl::DataType::INT32,
2002 : Hccl::DataType::FP16, Hccl::DataType::FP32, Hccl::DataType::INT64,
2003 : Hccl::DataType::UINT64, Hccl::DataType::UINT8, Hccl::DataType::UINT16,
2004 : Hccl::DataType::UINT32, Hccl::DataType::FP64, Hccl::DataType::BFP16,
2005 : Hccl::DataType::INT128,
2006 : #if !defined(OPEN_BUILD_PROJECT) || defined(ORION_MODE)
2007 : Hccl::DataType::HIF8, Hccl::DataType::FP8E4M3, Hccl::DataType::FP8E5M2,
2008 : Hccl::DataType::FP8E8M0
2009 : #endif
2010 : };
2011 :
2012 12 : static Hccl::DataType HcommDataTypeToHcclDataType(const HcclDataType dataType)
2013 : {
2014 12 : const auto dataTypeNum = static_cast<uint32_t>(dataType);
2015 12 : if (dataTypeNum > MAX_DATA_TYPE) {
2016 0 : return Hccl::DataType::INVALID;
2017 : }
2018 :
2019 12 : return orionDataTypes[dataTypeNum];
2020 : }
2021 :
2022 : constexpr uint32_t MAX_REDUCE_TYPE = 4;
2023 : const Hccl::ReduceOp orionReduceOps[] = {
2024 : Hccl::ReduceOp::SUM,
2025 : Hccl::ReduceOp::PROD,
2026 : Hccl::ReduceOp::MAX,
2027 : Hccl::ReduceOp::MIN,
2028 : };
2029 :
2030 8 : static Hccl::ReduceOp HcommReduceOpToHcclReduceOp(const HcclReduceOp reduceOp)
2031 : {
2032 8 : const auto reduceOpNum = static_cast<uint32_t>(reduceOp);
2033 8 : if (reduceOpNum > MAX_REDUCE_TYPE) {
2034 0 : return Hccl::ReduceOp::INVALID;
2035 : }
2036 :
2037 8 : return orionReduceOps[reduceOpNum];
2038 : }
2039 :
2040 4 : HcclResult CcuKernel::LocalReduceNb(
2041 : const CcuRep::CcuBuf* bufs, uint32_t count, HcclDataType dataType, HcclDataType outputDataType, HcclReduceOp opType,
2042 : const CcuRep::Variable& len, CcuRep::CompletedEvent event, uint32_t mask)
2043 : {
2044 4 : auto opType_ = HcommReduceOpToHcclReduceOp(opType);
2045 4 : auto dataType_ = HcommDataTypeToHcclDataType(dataType);
2046 4 : auto outputDataType_ = HcommDataTypeToHcclDataType(outputDataType);
2047 :
2048 8 : if ((opType_ == Hccl::ReduceOp::SUM && isLowPrecisionIn(dataType_) && !isLowPrecisionOut(outputDataType_))
2049 4 : || (opType_ == Hccl::ReduceOp::SUM && !isLowPrecisionIn(dataType_) && dataType_ != outputDataType_)
2050 8 : || (opType_ != Hccl::ReduceOp::SUM && dataType_ != outputDataType_)) {
2051 0 : return HCCL_E_NOT_SUPPORT;
2052 : }
2053 :
2054 4 : std::vector<CcuRep::CcuBuf> ccuBufs(count);
2055 12 : for (uint32_t i = 0; i < count; i++) {
2056 8 : ccuBufs[i] = bufs[i];
2057 : }
2058 :
2059 : auto rep = std::make_shared<CcuRep::CcuRepBufReduce>(
2060 0 : insGenerator, ccuBufs, count, CcuRep::GetCcuDataType(dataType_, opType_),
2061 4 : CcuRep::GetCcuDataType(outputDataType_, opType_), CcuRep::GetCcuReduceType(opType_), event, len, mask);
2062 4 : Append(rep);
2063 4 : SetDependencyInfo(event.Id(), mask, rep);
2064 4 : return HCCL_SUCCESS;
2065 4 : }
2066 :
2067 : /*Read新接口*/
2068 4 : HcclResult CcuKernel::ReadNb(
2069 : const ChannelHandle channel, const CcuRep::LocalAddr& loc, const CcuRep::RemoteAddr& rem,
2070 : const CcuRep::Variable& len, CcuRep::CompletedEvent event, uint32_t mask)
2071 : {
2072 4 : channels_.insert(channel);
2073 4 : auto rep = std::make_shared<CcuRep::CcuRepRead>(insGenerator, channel, loc, rem, len, event, mask);
2074 4 : Append(rep);
2075 4 : SetDependencyInfo(event.Id(), mask, rep);
2076 4 : return HCCL_SUCCESS;
2077 4 : }
2078 :
2079 : /*ReadReduce新接口*/
2080 2 : HcclResult CcuKernel::ReadReduceNb(
2081 : const ChannelHandle channel, const CcuRep::LocalAddr& loc, const CcuRep::RemoteAddr& rem,
2082 : const CcuRep::Variable& len, HcclDataType dataType, HcclReduceOp opType, CcuRep::CompletedEvent event,
2083 : uint32_t mask)
2084 : {
2085 2 : channels_.insert(channel);
2086 2 : auto opType_ = HcommReduceOpToHcclReduceOp(opType);
2087 2 : auto dataType_ = HcommDataTypeToHcclDataType(dataType);
2088 :
2089 : auto rep = std::make_shared<CcuRep::CcuRepRead>(
2090 2 : insGenerator, channel, loc, rem, len, CcuRep::GetUBDataType(dataType_), CcuRep::GetUBReduceType(opType_), event,
2091 2 : mask);
2092 2 : Append(rep);
2093 2 : SetDependencyInfo(event.Id(), mask, rep);
2094 2 : return HCCL_SUCCESS;
2095 2 : }
2096 :
2097 : /*Write新接口*/
2098 3 : HcclResult CcuKernel::WriteNb(
2099 : const ChannelHandle channel, const CcuRep::RemoteAddr& rem, const CcuRep::LocalAddr& loc,
2100 : const CcuRep::Variable& len, CcuRep::CompletedEvent event, uint32_t mask)
2101 : {
2102 3 : channels_.insert(channel);
2103 3 : auto rep = std::make_shared<CcuRep::CcuRepWrite>(insGenerator, channel, rem, loc, len, event, mask);
2104 3 : Append(rep);
2105 3 : SetDependencyInfo(event.Id(), mask, rep);
2106 3 : return HCCL_SUCCESS;
2107 3 : }
2108 :
2109 2 : HcclResult CcuKernel::WriteReduceNb(
2110 : const ChannelHandle channel, const CcuRep::RemoteAddr& rem, const CcuRep::LocalAddr& loc,
2111 : const CcuRep::Variable& len, HcclDataType dataType, HcclReduceOp opType, CcuRep::CompletedEvent event,
2112 : uint32_t mask)
2113 : {
2114 2 : channels_.insert(channel);
2115 2 : auto opType_ = HcommReduceOpToHcclReduceOp(opType);
2116 2 : auto dataType_ = HcommDataTypeToHcclDataType(dataType);
2117 :
2118 : auto rep = std::make_shared<CcuRep::CcuRepWrite>(
2119 2 : insGenerator, channel, rem, loc, len, CcuRep::GetUBDataType(dataType_), CcuRep::GetUBReduceType(opType_), event,
2120 2 : mask);
2121 2 : Append(rep);
2122 2 : SetDependencyInfo(event.Id(), mask, rep);
2123 2 : return HCCL_SUCCESS;
2124 2 : }
2125 :
2126 32 : CcuResult CcuKernel::GetBufferByHandle(CcuBufferHandle bufferHandle, CcuRep::CcuBuf** buffer)
2127 : {
2128 32 : return GetResourceByHandle(ccuBufferMap_, bufferHandle, buffer, "buffer");
2129 : }
2130 :
2131 1 : HcclResult CcuKernel::LocalCopyNb(
2132 : const CcuRep::LocalAddr& dst, const CcuRep::LocalAddr& src, const CcuRep::Variable& len,
2133 : CcuRep::CompletedEvent event, uint32_t mask)
2134 : {
2135 1 : auto rep = std::make_shared<CcuRep::CcuRepLocCpy>(insGenerator, dst, src, len, event, mask);
2136 1 : Append(rep);
2137 1 : SetDependencyInfo(event.Id(), mask, rep);
2138 1 : return HCCL_SUCCESS;
2139 1 : }
2140 :
2141 12 : HcclResult CcuKernel::LocalCopyNb(
2142 : const CcuRep::CcuBuf& dst, const CcuRep::LocalAddr& src, const CcuRep::Variable& len, CcuRep::CompletedEvent event,
2143 : uint32_t mask)
2144 : {
2145 12 : auto rep = std::make_shared<CcuRep::CcuRepBufLocRead>(insGenerator, src, dst, len, event, mask);
2146 12 : Append(rep);
2147 12 : SetDependencyInfo(event.Id(), mask, rep);
2148 12 : return HCCL_SUCCESS;
2149 12 : }
2150 :
2151 8 : HcclResult CcuKernel::LocalCopyNb(
2152 : const CcuRep::LocalAddr& dst, const CcuRep::CcuBuf& src, const CcuRep::Variable& len, CcuRep::CompletedEvent event,
2153 : uint32_t mask)
2154 : {
2155 8 : auto rep = std::make_shared<CcuRep::CcuRepBufLocWrite>(insGenerator, src, dst, len, event, mask);
2156 8 : Append(rep);
2157 8 : SetDependencyInfo(event.Id(), mask, rep);
2158 8 : return HCCL_SUCCESS;
2159 8 : }
2160 :
2161 0 : HcclResult CcuKernel::LocalReduceNb(
2162 : const CcuRep::LocalAddr& dst, const CcuRep::LocalAddr& src, const CcuRep::Variable& len, HcclDataType dataType,
2163 : HcclReduceOp opType, CcuRep::CompletedEvent event, uint32_t mask)
2164 : {
2165 0 : auto opType_ = HcommReduceOpToHcclReduceOp(opType);
2166 0 : auto dataType_ = HcommDataTypeToHcclDataType(dataType);
2167 :
2168 : auto rep = std::make_shared<CcuRep::CcuRepLocCpy>(
2169 0 : insGenerator, dst, src, len, CcuRep::GetUBDataType(dataType_), CcuRep::GetUBReduceType(opType_), event, mask);
2170 0 : Append(rep);
2171 0 : SetDependencyInfo(event.Id(), mask, rep);
2172 0 : return HCCL_SUCCESS;
2173 0 : }
2174 :
2175 0 : CcuRep::FuncCall CcuKernel::Func(const std::string& label) { return CcuRep::FuncCall(this, label); }
2176 :
2177 0 : CcuRep::FuncCall CcuKernel::Func(const CcuRep::Variable& funcAddr) { return CcuRep::FuncCall(this, funcAddr); }
2178 :
2179 0 : CcuRep::LoopCall CcuKernel::Loop(const std::string& label) { return CcuRep::LoopCall(this, label); }
2180 :
2181 52 : CcuResult CcuKernel::LoopCreate(CcuLoop* loop)
2182 : {
2183 52 : PLF_CONFIG_INFO(PLF_DATA_OP, "[LoopCreate]");
2184 52 : if (loop == nullptr) {
2185 0 : HCCL_ERROR("[CcuKernel::LoopCreate] null pointer");
2186 0 : return CcuResult::CCU_E_PTR;
2187 : }
2188 52 : if (CurrentBlock()->Type() == CcuRep::CcuRepType::LOOP_BLOCK) {
2189 0 : HCCL_ERROR("[CcuKernel::LoopCreate] cannot create loop inside a loop body");
2190 0 : return LatchBodyError(CcuResult::CCU_E_INTERNAL);
2191 : }
2192 52 : if (inFuncBody_) {
2193 0 : HCCL_ERROR("[CcuKernel::LoopCreate] cannot create loop inside a func body");
2194 0 : return LatchBodyError(CcuResult::CCU_E_INTERNAL);
2195 : }
2196 :
2197 52 : CcuLoop handle = ++loopHandleCounter_;
2198 52 : std::string label = "loop_" + std::to_string(handle);
2199 :
2200 52 : LoopDescriptor desc;
2201 52 : desc.label = label;
2202 52 : desc.repLoopBlock = std::make_shared<CcuRep::CcuRepLoopBlock>(insGenerator, label);
2203 :
2204 52 : loopMap_[handle] = std::move(desc);
2205 52 : *loop = handle;
2206 52 : return CcuResult::CCU_SUCCESS;
2207 52 : }
2208 :
2209 7 : CcuResult CcuKernel::LatchBodyError(CcuResult err)
2210 : {
2211 15 : if ((CurrentBlock()->Type() == CcuRep::CcuRepType::LOOP_BLOCK || inFuncBody_)
2212 15 : && bodyError_ == CcuResult::CCU_SUCCESS) {
2213 7 : bodyError_ = err;
2214 : }
2215 7 : return err;
2216 : }
2217 :
2218 52 : CcuResult CcuKernel::LoopBodyEnter(CcuLoop loop)
2219 : {
2220 52 : PLF_CONFIG_INFO(PLF_DATA_OP, "[LoopBodyEnter] loop=%llu", loop);
2221 52 : auto it = loopMap_.find(loop);
2222 52 : if (it == loopMap_.end()) {
2223 0 : HCCL_ERROR("[CcuKernel::LoopBodyEnter] invalid loop handle %lu", loop);
2224 0 : return CcuResult::CCU_E_PARA;
2225 : }
2226 52 : auto& desc = it->second;
2227 52 : if (desc.bodyDefined) {
2228 0 : HCCL_ERROR("[CcuKernel::LoopBodyEnter] loop %lu body already defined", loop);
2229 0 : return CcuResult::CCU_E_INTERNAL;
2230 : }
2231 :
2232 52 : Append(desc.repLoopBlock);
2233 52 : desc.prevActiveBlock = CurrentBlock();
2234 52 : SetCurrentBlock(desc.repLoopBlock);
2235 52 : ++loopBodyDepth_;
2236 52 : bodyError_ = CcuResult::CCU_SUCCESS;
2237 :
2238 52 : return CcuResult::CCU_SUCCESS;
2239 : }
2240 :
2241 52 : CcuResult CcuKernel::LoopBodyExit(CcuLoop loop)
2242 : {
2243 52 : PLF_CONFIG_INFO(PLF_DATA_OP, "[LoopBodyExit] loop=%llu", loop);
2244 52 : auto it = loopMap_.find(loop);
2245 52 : if (it == loopMap_.end()) {
2246 0 : HCCL_ERROR("[CcuKernel::LoopBodyExit] invalid loop handle %lu", loop);
2247 0 : return CcuResult::CCU_E_PARA;
2248 : }
2249 52 : auto& desc = it->second;
2250 :
2251 52 : SetCurrentBlock(desc.prevActiveBlock);
2252 52 : desc.bodyDefined = true;
2253 52 : --loopBodyDepth_;
2254 :
2255 52 : if (bodyError_ != CcuResult::CCU_SUCCESS) {
2256 6 : const CcuResult err = bodyError_;
2257 6 : HCCL_ERROR("[CcuKernel::LoopBodyExit] illegal operation inside loop body, err=%d", err);
2258 6 : bodyError_ = CcuResult::CCU_SUCCESS;
2259 6 : return err;
2260 : }
2261 :
2262 46 : return CcuResult::CCU_SUCCESS;
2263 : }
2264 :
2265 8 : CcuResult CcuKernel::FuncBlockLookup(const void* funcPtr, uint64_t* outHandle)
2266 : {
2267 8 : PLF_CONFIG_INFO(PLF_DATA_OP, "[FuncBlockLookup] funcPtr=%p", funcPtr);
2268 8 : if (funcPtr == nullptr || outHandle == nullptr) {
2269 0 : HCCL_ERROR("[CcuKernel::FuncBlockLookup] null pointer");
2270 0 : return CcuResult::CCU_E_PTR;
2271 : }
2272 8 : if (CurrentBlock()->Type() == CcuRep::CcuRepType::LOOP_BLOCK || inFuncBody_) {
2273 2 : HCCL_ERROR("[CcuKernel::FuncBlockLookup] ccu::CallFunc only allowed at top level");
2274 2 : return LatchBodyError(CcuResult::CCU_E_INTERNAL);
2275 : }
2276 :
2277 6 : auto it = funcInstanceMap_.find(funcPtr);
2278 6 : *outHandle = (it == funcInstanceMap_.end()) ? 0 : it->second;
2279 6 : return CcuResult::CCU_SUCCESS;
2280 : }
2281 :
2282 5 : CcuResult CcuKernel::FuncBlockBegin(const void* funcPtr, uint64_t* outHandle)
2283 : {
2284 5 : PLF_CONFIG_INFO(PLF_DATA_OP, "[FuncBlockBegin] funcPtr=%p", funcPtr);
2285 5 : if (funcPtr == nullptr || outHandle == nullptr) {
2286 0 : HCCL_ERROR("[CcuKernel::FuncBlockBegin] null pointer");
2287 0 : return CcuResult::CCU_E_PTR;
2288 : }
2289 5 : if (CurrentBlock()->Type() == CcuRep::CcuRepType::LOOP_BLOCK || inFuncBody_) {
2290 0 : HCCL_ERROR("[CcuKernel::FuncBlockBegin] ccu::CallFunc only allowed at top level");
2291 0 : return LatchBodyError(CcuResult::CCU_E_INTERNAL);
2292 : }
2293 :
2294 5 : auto exist = funcInstanceMap_.find(funcPtr);
2295 5 : if (exist != funcInstanceMap_.end()) {
2296 0 : *outHandle = exist->second;
2297 0 : return CcuResult::CCU_SUCCESS;
2298 : }
2299 :
2300 5 : const uint64_t handle = ++funcHandleCounter_;
2301 5 : std::string label = "func_" + std::to_string(handle);
2302 :
2303 5 : FuncDescriptor desc;
2304 5 : desc.funcPtr = funcPtr;
2305 5 : desc.label = label;
2306 5 : desc.repFuncBlock = std::make_shared<CcuRep::CcuRepFuncBlock>(insGenerator, label);
2307 5 : desc.prevActiveBlock = CurrentBlock();
2308 :
2309 5 : funcMap_[handle] = desc;
2310 5 : SetCurrentBlock(desc.repFuncBlock);
2311 5 : inFuncBody_ = true;
2312 5 : bodyError_ = CcuResult::CCU_SUCCESS;
2313 5 : *outHandle = handle;
2314 5 : return CcuResult::CCU_SUCCESS;
2315 5 : }
2316 :
2317 4 : CcuResult CcuKernel::FuncBlockEnd(uint64_t handle)
2318 : {
2319 4 : PLF_CONFIG_INFO(PLF_DATA_OP, "[FuncBlockEnd] handle=%llu", handle);
2320 4 : auto it = funcMap_.find(handle);
2321 4 : if (it == funcMap_.end()) {
2322 0 : HCCL_ERROR("[CcuKernel::FuncBlockEnd] invalid func handle %lu", handle);
2323 0 : return CcuResult::CCU_E_PARA;
2324 : }
2325 4 : auto& desc = it->second;
2326 :
2327 4 : SetCurrentBlock(desc.prevActiveBlock);
2328 4 : inFuncBody_ = false;
2329 :
2330 4 : if (bodyError_ != CcuResult::CCU_SUCCESS) {
2331 0 : const CcuResult err = bodyError_;
2332 0 : HCCL_ERROR("[CcuKernel::FuncBlockEnd] illegal operation inside func body, err=%d", err);
2333 0 : bodyError_ = CcuResult::CCU_SUCCESS;
2334 0 : funcMap_.erase(it);
2335 0 : return err;
2336 : }
2337 :
2338 4 : Append(desc.repFuncBlock);
2339 4 : desc.bodyDefined = true;
2340 4 : funcInstanceMap_[desc.funcPtr] = handle;
2341 4 : return CcuResult::CCU_SUCCESS;
2342 : }
2343 :
2344 7 : CcuResult CcuKernel::FuncDefineInArg(uint64_t handle, CcuVariableHandle formal)
2345 : {
2346 7 : PLF_CONFIG_INFO(PLF_DATA_OP, "[FuncDefineInArg] handle=%llu, formal=%llu", handle, formal);
2347 7 : auto it = funcMap_.find(handle);
2348 7 : if (it == funcMap_.end()) {
2349 0 : HCCL_ERROR("[CcuKernel::FuncDefineInArg] invalid func handle %lu", handle);
2350 0 : return CcuResult::CCU_E_PARA;
2351 : }
2352 :
2353 7 : CcuRep::Variable* formalVar = nullptr;
2354 7 : CCU_CHK_RET(GetVariableByHandle(formal, &formalVar));
2355 7 : it->second.repFuncBlock->DefineInArg(*formalVar);
2356 7 : return CcuResult::CCU_SUCCESS;
2357 : }
2358 :
2359 5 : CcuResult CcuKernel::FuncCall(uint64_t handle, const CcuVariableHandle* inArgs, uint32_t numIn)
2360 : {
2361 5 : PLF_CONFIG_INFO(PLF_DATA_OP, "[FuncCall] handle=%llu, numIn=%u", handle, numIn);
2362 5 : if (CurrentBlock()->Type() == CcuRep::CcuRepType::LOOP_BLOCK || inFuncBody_) {
2363 0 : HCCL_ERROR("[CcuKernel::FuncCall] ccu::CallFunc only allowed at top level");
2364 0 : return LatchBodyError(CcuResult::CCU_E_INTERNAL);
2365 : }
2366 5 : if (numIn > 0 && inArgs == nullptr) {
2367 0 : HCCL_ERROR("[CcuKernel::FuncCall] null input args");
2368 0 : return CcuResult::CCU_E_PTR;
2369 : }
2370 :
2371 5 : auto it = funcMap_.find(handle);
2372 5 : if (it == funcMap_.end() || !it->second.bodyDefined) {
2373 0 : HCCL_ERROR("[CcuKernel::FuncCall] invalid func handle %lu", handle);
2374 0 : return CcuResult::CCU_E_PARA;
2375 : }
2376 :
2377 5 : auto repFuncCall = std::make_shared<CcuRep::CcuRepFuncCall>(insGenerator, it->second.label);
2378 12 : for (uint32_t i = 0; i < numIn; i++) {
2379 7 : CcuRep::Variable* actual = nullptr;
2380 7 : CCU_CHK_RET(GetVariableByHandle(inArgs[i], &actual));
2381 7 : repFuncCall->SetInArg(*actual);
2382 : }
2383 5 : Append(repFuncCall);
2384 5 : return CcuResult::CCU_SUCCESS;
2385 5 : }
2386 :
2387 : // 按 maxLoopNum 把 res_.blockExecutor[0] 扩容到至少 maxLoopNum 个 LoopEngine。
2388 : // 与 CreateBlockResAssist 对齐：所有 LoopEngine 资源先落在 die0 池，待实际 die 确定后
2389 : // 再由 MoveResourcesToDie 迁移到目标 die。
2390 : // 不同 LoopGroup 通过 local loopIdx 复用同一池低位 executorId，所以这里只
2391 : // "补足"而不是"累加"。
2392 28 : CcuResult CcuKernel::EnsureLoopEnginePool(uint32_t maxLoopNum)
2393 : {
2394 28 : if (maxLoopNum == 0) {
2395 0 : HCCL_ERROR("[CcuKernel::EnsureLoopEnginePool] maxLoopNum must be > 0");
2396 0 : return CcuResult::CCU_E_PARA;
2397 : }
2398 28 : constexpr uint32_t poolDieId = 0;
2399 28 : auto& loopEnginePool = res_.blockExecutor[poolDieId];
2400 28 : if (maxLoopNum <= loopEnginePool.size()) {
2401 10 : return CcuResult::CCU_SUCCESS;
2402 : }
2403 18 : const uint32_t deficit = maxLoopNum - static_cast<uint32_t>(loopEnginePool.size());
2404 18 : std::vector<CcuRep::Executor> tmp(deficit, CcuRep::Executor(this));
2405 18 : (void)CreateBlockExecutor(deficit, tmp.data());
2406 18 : return CcuResult::CCU_SUCCESS;
2407 18 : }
2408 :
2409 13 : CcuResult CcuKernel::LoopGroupCreate(CcuLoopGroup* group, uint32_t maxLoopNum, const CcuLoopGroupCfg* cfg)
2410 : {
2411 13 : PLF_CONFIG_INFO(PLF_DATA_OP, "[LoopGroupCreate] maxLoopNum=%u", maxLoopNum);
2412 13 : if (group == nullptr || cfg == nullptr) {
2413 0 : HCCL_ERROR("[CcuKernel::LoopGroupCreate] null pointer");
2414 0 : return CcuResult::CCU_E_PTR;
2415 : }
2416 13 : if (CurrentBlock()->Type() == CcuRep::CcuRepType::LOOP_BLOCK) {
2417 0 : HCCL_ERROR("[CcuKernel::LoopGroupCreate] cannot create loop group inside a loop body");
2418 0 : return LatchBodyError(CcuResult::CCU_E_INTERNAL);
2419 : }
2420 13 : if (inFuncBody_) {
2421 0 : HCCL_ERROR("[CcuKernel::LoopGroupCreate] cannot create loop group inside a func body");
2422 0 : return LatchBodyError(CcuResult::CCU_E_INTERNAL);
2423 : }
2424 :
2425 : // 按需扩 LoopEngine 池；池足够则复用低位 executorId，跨组共享。
2426 13 : CCU_CHK_RET(EnsureLoopEnginePool(maxLoopNum));
2427 :
2428 13 : CcuLoopGroup handle = ++loopGroupHandleCounter_;
2429 :
2430 13 : LoopGroupDescriptor desc;
2431 13 : desc.config = *cfg;
2432 13 : desc.parallelVar = CreateVariable();
2433 13 : desc.offsetVar = CreateVariable();
2434 13 : desc.isVarBased = false;
2435 :
2436 13 : auto bundle = std::make_shared<CcuRep::CcuRepLoopGroupBundle>(insGenerator, *cfg, desc.parallelVar, desc.offsetVar);
2437 13 : if (ccuVersion_ == CcuVersion::CCU_V2) {
2438 4 : bundle->SetXnOffsetVar(CreateVariable());
2439 : }
2440 13 : desc.bundleRep = bundle;
2441 13 : Append(bundle);
2442 :
2443 13 : loopGroupMap_[handle] = std::move(desc);
2444 13 : *group = handle;
2445 13 : return CcuResult::CCU_SUCCESS;
2446 13 : }
2447 :
2448 11 : CcuResult CcuKernel::LoopGroupCreateFromVar(
2449 : CcuLoopGroup* group, uint32_t maxLoopNum, CcuVariableHandle parallelVarHandle, CcuVariableHandle offsetVarHandle)
2450 : {
2451 11 : PLF_CONFIG_INFO(
2452 : PLF_DATA_OP, "[LoopGroupCreateFromVar] maxLoopNum=%u, parallelVarHandle=%llu, offsetVarHandle=%llu", maxLoopNum,
2453 : parallelVarHandle, offsetVarHandle);
2454 11 : if (group == nullptr) {
2455 0 : HCCL_ERROR("[CcuKernel::LoopGroupCreateFromVar] null pointer for group");
2456 0 : return CcuResult::CCU_E_PTR;
2457 : }
2458 11 : if (CurrentBlock()->Type() == CcuRep::CcuRepType::LOOP_BLOCK) {
2459 0 : HCCL_ERROR("[CcuKernel::LoopGroupCreateFromVar] cannot create loop group inside a loop body");
2460 0 : return LatchBodyError(CcuResult::CCU_E_INTERNAL);
2461 : }
2462 11 : if (inFuncBody_) {
2463 0 : HCCL_ERROR("[CcuKernel::LoopGroupCreateFromVar] cannot create loop group inside a func body");
2464 0 : return LatchBodyError(CcuResult::CCU_E_INTERNAL);
2465 : }
2466 :
2467 11 : CCU_CHK_RET(EnsureLoopEnginePool(maxLoopNum));
2468 :
2469 11 : CcuRep::Variable* parallelVarPtr = nullptr;
2470 11 : CcuRep::Variable* offsetVarPtr = nullptr;
2471 11 : CCU_CHK_RET(GetVariableByHandle(parallelVarHandle, ¶llelVarPtr));
2472 11 : CCU_CHK_RET(GetVariableByHandle(offsetVarHandle, &offsetVarPtr));
2473 :
2474 11 : CcuLoopGroup handle = ++loopGroupHandleCounter_;
2475 :
2476 11 : LoopGroupDescriptor desc;
2477 11 : desc.parallelVar = CcuRep::Variable(*parallelVarPtr);
2478 11 : desc.offsetVar = CcuRep::Variable(*offsetVarPtr);
2479 11 : desc.isVarBased = true;
2480 :
2481 11 : auto bundle = std::make_shared<CcuRep::CcuRepLoopGroupBundle>(insGenerator, desc.parallelVar, desc.offsetVar);
2482 11 : if (ccuVersion_ == CcuVersion::CCU_V2) {
2483 1 : bundle->SetCompatRemapVars(CreateVariable(), CreateVariable());
2484 : }
2485 11 : desc.bundleRep = bundle;
2486 11 : Append(bundle);
2487 :
2488 11 : loopGroupMap_[handle] = std::move(desc);
2489 11 : *group = handle;
2490 11 : return CcuResult::CCU_SUCCESS;
2491 11 : }
2492 :
2493 5 : CcuResult CcuKernel::LoopGroupCreateFromVarV2(
2494 : CcuLoopGroup* group, uint32_t maxLoopNum, CcuVariableHandle parallelVarV2Handle,
2495 : CcuVariableHandle offsetVarV2Handle, CcuVariableHandle varOffsetVarHandle)
2496 : {
2497 5 : if (ccuVersion_ != CcuVersion::CCU_V2) {
2498 1 : HCCL_ERROR("[CcuKernel::LoopGroupCreateFromVarV2] only supported on V2");
2499 1 : return CcuResult::CCU_E_NOT_SUPPORT;
2500 : }
2501 4 : if (group == nullptr) {
2502 0 : HCCL_ERROR("[CcuKernel::LoopGroupCreateFromVarV2] null pointer for group");
2503 0 : return CcuResult::CCU_E_PTR;
2504 : }
2505 4 : if (CurrentBlock()->Type() == CcuRep::CcuRepType::LOOP_BLOCK) {
2506 0 : HCCL_ERROR("[CcuKernel::LoopGroupCreateFromVarV2] cannot create loop group inside a loop body");
2507 0 : return LatchBodyError(CcuResult::CCU_E_INTERNAL);
2508 : }
2509 4 : if (inFuncBody_) {
2510 0 : HCCL_ERROR("[CcuKernel::LoopGroupCreateFromVarV2] cannot create loop group inside a func body");
2511 0 : return LatchBodyError(CcuResult::CCU_E_INTERNAL);
2512 : }
2513 :
2514 4 : CCU_CHK_RET(EnsureLoopEnginePool(maxLoopNum));
2515 :
2516 4 : CcuRep::Variable* parallelVarPtr = nullptr;
2517 4 : CcuRep::Variable* offsetVarPtr = nullptr;
2518 4 : CcuRep::Variable* xnOffsetVarPtr = nullptr;
2519 4 : CCU_CHK_RET(GetVariableByHandle(parallelVarV2Handle, ¶llelVarPtr));
2520 4 : CCU_CHK_RET(GetVariableByHandle(offsetVarV2Handle, &offsetVarPtr));
2521 4 : CCU_CHK_RET(GetVariableByHandle(varOffsetVarHandle, &xnOffsetVarPtr));
2522 :
2523 4 : CcuLoopGroup handle = ++loopGroupHandleCounter_;
2524 :
2525 4 : LoopGroupDescriptor desc;
2526 4 : desc.parallelVar = CcuRep::Variable(*parallelVarPtr);
2527 4 : desc.offsetVar = CcuRep::Variable(*offsetVarPtr);
2528 4 : desc.xnOffsetVar = CcuRep::Variable(*xnOffsetVarPtr);
2529 4 : desc.isVarBased = true;
2530 4 : desc.isVersionV2 = true;
2531 :
2532 4 : auto bundle = std::make_shared<CcuRep::CcuRepLoopGroupBundle>(insGenerator, desc.parallelVar, desc.offsetVar);
2533 4 : bundle->SetLayout(CcuRep::CcuRepLoopGroupBundle::Layout::VersionV2);
2534 4 : bundle->SetXnOffsetVar(desc.xnOffsetVar);
2535 4 : desc.bundleRep = bundle;
2536 4 : Append(bundle);
2537 :
2538 4 : loopGroupMap_[handle] = std::move(desc);
2539 4 : *group = handle;
2540 4 : return CcuResult::CCU_SUCCESS;
2541 4 : }
2542 :
2543 56 : CcuResult CcuKernel::LookupLoopGroupAndLoop(
2544 : CcuLoopGroup group, CcuLoop loop, const char* fnName, const char* createFnName, LoopGroupDescriptor*& grpDesc,
2545 : LoopDescriptor*& loopDesc, uint32_t& loopIdx)
2546 : {
2547 56 : auto grpIt = loopGroupMap_.find(group);
2548 56 : if (grpIt == loopGroupMap_.end()) {
2549 0 : HCCL_ERROR("[CcuKernel::%s] invalid group handle %lu", fnName, group);
2550 0 : return CcuResult::CCU_E_PARA;
2551 : }
2552 56 : grpDesc = &grpIt->second;
2553 :
2554 56 : auto loopIt = loopMap_.find(loop);
2555 56 : if (loopIt == loopMap_.end()) {
2556 0 : HCCL_ERROR("[CcuKernel::%s] invalid loop handle %lu", fnName, loop);
2557 0 : return CcuResult::CCU_E_PARA;
2558 : }
2559 56 : loopDesc = &loopIt->second;
2560 :
2561 56 : if (!loopDesc->bodyDefined) {
2562 0 : HCCL_ERROR("[CcuKernel::%s] loop %lu body not defined", fnName, loop);
2563 0 : return CcuResult::CCU_E_INTERNAL; // CCU_E_LOOP_BODY_UNDEFINED
2564 : }
2565 :
2566 56 : auto& loopEnginePool = res_.blockExecutor[0];
2567 56 : loopIdx = grpDesc->loopCount;
2568 56 : if (loopIdx >= loopEnginePool.size()) {
2569 0 : HCCL_ERROR(
2570 : "[CcuKernel::%s] loopEngine pool exhausted (pool size %zu, loopIdx %u). "
2571 : "Pass a larger maxLoopNum to %s so the pool can be extended at create time.",
2572 : fnName, loopEnginePool.size(), loopIdx, createFnName);
2573 0 : return CcuResult::CCU_E_PARA;
2574 : }
2575 56 : return CcuResult::CCU_SUCCESS;
2576 : }
2577 :
2578 25 : CcuResult CcuKernel::LoopGroupAddLoop(CcuLoopGroup group, CcuLoop loop, const CcuLoopCfg* cfg)
2579 : {
2580 25 : PLF_CONFIG_INFO(PLF_DATA_OP, "[LoopGroupAddLoop] group=%llu, loop=%llu", group, loop);
2581 25 : if (cfg == nullptr) {
2582 0 : HCCL_ERROR("[CcuKernel::LoopGroupAddLoop] null pointer for cfg");
2583 0 : return CcuResult::CCU_E_PTR;
2584 : }
2585 25 : LoopGroupDescriptor* grpDesc = nullptr;
2586 25 : LoopDescriptor* loopDesc = nullptr;
2587 25 : uint32_t loopIdx = 0;
2588 25 : CCU_CHK_RET(
2589 : LookupLoopGroupAndLoop(group, loop, "LoopGroupAddLoop", "CcuLoopGroupCreate", grpDesc, loopDesc, loopIdx));
2590 25 : auto& loopEnginePool = res_.blockExecutor[0];
2591 :
2592 25 : grpDesc->loopCount++;
2593 25 : grpDesc->totalLoopNum = grpDesc->loopCount;
2594 :
2595 25 : CcuRep::CcuRepLoopGroupBundle::LoopEntry entry;
2596 25 : entry.config = *cfg;
2597 25 : entry.executor = loopEnginePool[loopIdx];
2598 25 : entry.repLoopBlock = loopDesc->repLoopBlock;
2599 25 : entry.loopParamVar = CreateVariable();
2600 25 : entry.layout = CcuRep::CcuRepLoopGroupBundle::Layout::Config;
2601 25 : if (ccuVersion_ == CcuVersion::CCU_V2) {
2602 7 : entry.iterNumVar = CreateVariable();
2603 7 : entry.addrOffsetVar = CreateVariable();
2604 7 : entry.ctxIdVar = CreateVariable();
2605 : }
2606 :
2607 25 : auto bundle = std::static_pointer_cast<CcuRep::CcuRepLoopGroupBundle>(grpDesc->bundleRep);
2608 25 : bundle->AddLoop(entry);
2609 :
2610 25 : if (!grpDesc->isVarBased) {
2611 19 : bundle->SetRepeatLoopIdx(grpDesc->config.cloneLoopOffset);
2612 19 : bundle->SetTotalLoopNum(grpDesc->totalLoopNum);
2613 : }
2614 :
2615 25 : return CcuResult::CCU_SUCCESS;
2616 25 : }
2617 :
2618 23 : CcuResult CcuKernel::LoopGroupAddLoopFromVar(CcuLoopGroup group, CcuLoop loop, CcuVariableHandle loopParamVarHandle)
2619 : {
2620 23 : PLF_CONFIG_INFO(
2621 : PLF_DATA_OP, "[LoopGroupAddLoopFromVar] group=%llu, loop=%llu, loopParamVarHandle=%llu", group, loop,
2622 : loopParamVarHandle);
2623 23 : LoopGroupDescriptor* grpDesc = nullptr;
2624 23 : LoopDescriptor* loopDesc = nullptr;
2625 23 : uint32_t loopIdx = 0;
2626 23 : CCU_CHK_RET(LookupLoopGroupAndLoop(
2627 : group, loop, "LoopGroupAddLoopFromVar", "CcuLoopGroupCreateFromVar", grpDesc, loopDesc, loopIdx));
2628 23 : auto& loopEnginePool = res_.blockExecutor[0];
2629 :
2630 23 : CcuRep::Variable* loopParamVarPtr = nullptr;
2631 23 : CCU_CHK_RET(GetVariableByHandle(loopParamVarHandle, &loopParamVarPtr));
2632 :
2633 23 : CcuRep::CcuRepLoopGroupBundle::LoopEntry entry;
2634 23 : entry.executor = loopEnginePool[loopIdx];
2635 23 : entry.repLoopBlock = loopDesc->repLoopBlock;
2636 23 : entry.loopParamVar = CcuRep::Variable(*loopParamVarPtr);
2637 23 : entry.layout = CcuRep::CcuRepLoopGroupBundle::Layout::PackedVar;
2638 23 : if (ccuVersion_ == CcuVersion::CCU_V2) {
2639 4 : entry.iterNumVar = CreateVariable();
2640 4 : entry.addrOffsetVar = CreateVariable();
2641 4 : entry.ctxIdVar = CreateVariable();
2642 : }
2643 :
2644 23 : auto bundle = std::static_pointer_cast<CcuRep::CcuRepLoopGroupBundle>(grpDesc->bundleRep);
2645 23 : bundle->AddLoop(entry);
2646 :
2647 : // 计数放在入 bundle 之后统一更新：totalLoopNum 仅供 config 组同步编码使用，与 AddLoop 无先后依赖。
2648 23 : grpDesc->loopCount++;
2649 23 : grpDesc->totalLoopNum = grpDesc->loopCount;
2650 23 : if (!grpDesc->isVarBased) {
2651 7 : bundle->SetRepeatLoopIdx(grpDesc->config.cloneLoopOffset);
2652 7 : bundle->SetTotalLoopNum(grpDesc->totalLoopNum);
2653 : }
2654 :
2655 23 : return CcuResult::CCU_SUCCESS;
2656 23 : }
2657 :
2658 8 : CcuResult CcuKernel::LoopGroupAddLoopFromVarV2(
2659 : CcuLoopGroup group, CcuLoop loop, CcuVariableHandle iterNumVarHandle, CcuVariableHandle addrOffsetVarHandle,
2660 : CcuVariableHandle ctxIdVarHandle)
2661 : {
2662 8 : if (ccuVersion_ != CcuVersion::CCU_V2) {
2663 0 : HCCL_ERROR("[CcuKernel::LoopGroupAddLoopFromVarV2] only supported on V2");
2664 0 : return CcuResult::CCU_E_NOT_SUPPORT;
2665 : }
2666 :
2667 8 : LoopGroupDescriptor* grpDesc = nullptr;
2668 8 : LoopDescriptor* loopDesc = nullptr;
2669 8 : uint32_t loopIdx = 0;
2670 8 : CCU_CHK_RET(LookupLoopGroupAndLoop(
2671 : group, loop, "LoopGroupAddLoopFromVarV2", "CcuLoopGroupCreateFromVarV2", grpDesc, loopDesc, loopIdx));
2672 8 : auto& loopEnginePool = res_.blockExecutor[0];
2673 :
2674 8 : CcuRep::Variable* iterNumVarPtr = nullptr;
2675 8 : CcuRep::Variable* addrOffsetVarPtr = nullptr;
2676 8 : CcuRep::Variable* ctxIdVarPtr = nullptr;
2677 8 : CCU_CHK_RET(GetVariableByHandle(iterNumVarHandle, &iterNumVarPtr));
2678 8 : CCU_CHK_RET(GetVariableByHandle(addrOffsetVarHandle, &addrOffsetVarPtr));
2679 8 : CCU_CHK_RET(GetVariableByHandle(ctxIdVarHandle, &ctxIdVarPtr));
2680 :
2681 8 : grpDesc->loopCount++;
2682 8 : grpDesc->totalLoopNum = grpDesc->loopCount;
2683 :
2684 8 : VersionV2LoopRecord record;
2685 8 : record.iterNumVar = CcuRep::Variable(*iterNumVarPtr);
2686 8 : record.addrOffsetVar = CcuRep::Variable(*addrOffsetVarPtr);
2687 8 : record.ctxIdVar = CcuRep::Variable(*ctxIdVarPtr);
2688 8 : grpDesc->versionV2Loops.push_back(record);
2689 :
2690 8 : CcuRep::CcuRepLoopGroupBundle::LoopEntry entry;
2691 8 : entry.executor = loopEnginePool[loopIdx];
2692 8 : entry.repLoopBlock = loopDesc->repLoopBlock;
2693 8 : entry.iterNumVar = CcuRep::Variable(*iterNumVarPtr);
2694 8 : entry.addrOffsetVar = CcuRep::Variable(*addrOffsetVarPtr);
2695 8 : entry.ctxIdVar = CcuRep::Variable(*ctxIdVarPtr);
2696 8 : entry.layout = CcuRep::CcuRepLoopGroupBundle::Layout::VersionV2;
2697 :
2698 8 : auto bundle = std::static_pointer_cast<CcuRep::CcuRepLoopGroupBundle>(grpDesc->bundleRep);
2699 8 : bundle->AddLoop(entry);
2700 :
2701 8 : if (!grpDesc->isVarBased) {
2702 2 : bundle->SetRepeatLoopIdx(grpDesc->config.cloneLoopOffset);
2703 2 : bundle->SetTotalLoopNum(grpDesc->totalLoopNum);
2704 : }
2705 :
2706 8 : return CcuResult::CCU_SUCCESS;
2707 8 : }
2708 :
2709 46 : void CcuKernel::SetInstrId(uint32_t instrId) { instrInfo_.startInstrId = instrId; }
2710 :
2711 92 : uint32_t CcuKernel::GetInstrId() const { return instrInfo_.startInstrId; }
2712 :
2713 143 : uint32_t CcuKernel::GetInstrCount()
2714 : {
2715 143 : uint32_t instrCount = 0;
2716 3435 : for (const auto& rep : GetRepSequence()) {
2717 3292 : instrCount += rep->InstrCount();
2718 : }
2719 143 : instrInfo_.instrCount = instrCount;
2720 143 : HCCL_INFO("Kernel inst %u", instrCount);
2721 143 : return instrCount;
2722 : }
2723 :
2724 : namespace {
2725 : // 识别三种 wait 类 rep: 无论 profiling 与否都会翻译出一条"既 wait 又 set/clear 同一 CKE 位"的
2726 : // CTRL 指令 (profiling 分支 -> setcke, 非 profiling 分支 -> clearcke, 二者都是 CKE 写者),
2727 : // 后端优化可能为其后续读者补 NOP, 因此都要按 CCU_CKE_RAW_LATENCY 预留指令空间.
2728 1089 : bool IsCkeWaitRep(const std::shared_ptr<CcuRep::CcuRepBase>& rep)
2729 : {
2730 1089 : if (rep == nullptr) {
2731 0 : return false;
2732 : }
2733 1089 : switch (rep->Type()) {
2734 36 : case CcuRep::CcuRepType::LOC_WAIT_EVENT:
2735 : case CcuRep::CcuRepType::LOC_WAIT_NOTIFY:
2736 : case CcuRep::CcuRepType::REM_WAIT_SEM:
2737 36 : return true;
2738 1053 : default:
2739 1053 : return false;
2740 : }
2741 : }
2742 :
2743 : // 与 CcuKernelMgr::PrepareConstValueResources 一致地下钻 block 子 rep, 统计其中会翻译出
2744 : // setcke / clearcke (CKE 写者) 的 wait 类 rep 个数.
2745 : // 只下钻一层是充分的: 表示层禁止 block 嵌套 —— CcuKernel::LoopCreate / FuncBlockBegin /
2746 : // FuncBlockLookup 在 CurrentBlock() 为 LOOP_BLOCK 或 inFuncBody_ 时直接报错 (见 ccu_kernel.cc),
2747 : // 故 LOOP_BLOCK / FUNC_BLOCK 内不可能再出现 block 类型子 rep, 无需递归下钻.
2748 : // 若后续放开 block 嵌套, 此处 (与 PrepareConstValueResources) 需同步改为递归统计, 否则会漏计.
2749 999 : uint32_t CountCkeWaitRepInBlock(const std::shared_ptr<CcuRep::CcuRepBase>& rep)
2750 : {
2751 999 : const auto repType = rep->Type();
2752 999 : if (repType != CcuRep::CcuRepType::BLOCK && repType != CcuRep::CcuRepType::FUNC_BLOCK
2753 999 : && repType != CcuRep::CcuRepType::LOOP_BLOCK) {
2754 945 : return 0;
2755 : }
2756 54 : auto* blockPtr = static_cast<CcuRep::CcuRepBlock*>(rep.get());
2757 54 : if (blockPtr == nullptr) {
2758 0 : return 0;
2759 : }
2760 54 : uint32_t count = 0;
2761 144 : for (const auto& child : blockPtr->GetReps()) {
2762 90 : if (IsCkeWaitRep(child)) {
2763 18 : count++;
2764 : }
2765 : }
2766 54 : return count;
2767 : }
2768 : } // namespace
2769 :
2770 : // 统计当前 kernel 中"需要按 cke 写后读 latency 补 NOP"的 rep 个数 (含 block 子 rep).
2771 : // 三种 wait 类 rep 无论 profiling 与否都翻译出一条 CKE 写者微码 (profiling -> setcke, 非 profiling
2772 : // -> clearcke), 后端优化最多为其后续读者补 (CCU_CKE_RAW_LATENCY - 1) 条 NOP, 上层据此为每个此类 rep
2773 : // 预留同等指令空间, 保证优化后指令数不越界.
2774 157 : uint32_t CcuKernel::GetRepNeedToAddLatency() const
2775 : {
2776 : // cke 写后读补 NOP 与指令空间预留只属于 A6(CCU_V2) 后端优化; A5(CCU_V1) 不跑后端优化,
2777 : // 不做任何预留, 直接返回 0, 避免影响 A5 的申请/释放口径.
2778 157 : if (ccuVersion_ != CcuVersion::CCU_V2) {
2779 106 : return 0;
2780 : }
2781 51 : uint32_t count = 0;
2782 : // GetRepSequence() 语义纯读但尚未标 const, 此处仅遍历不修改, 去 const 调用以保持本方法 const 契约.
2783 1050 : for (const auto& rep : const_cast<CcuKernel*>(this)->GetRepSequence()) {
2784 999 : if (rep == nullptr) {
2785 0 : continue;
2786 : }
2787 999 : if (IsCkeWaitRep(rep)) {
2788 18 : count++;
2789 : }
2790 999 : count += CountCkeWaitRepInBlock(rep);
2791 : }
2792 51 : HCCL_INFO("[CcuKernel] cke wait rep count %u (reserve %u instrs)", count, count * CcuRep::CCU_CKE_RAW_LATENCY);
2793 51 : return count;
2794 : }
2795 :
2796 45 : void CcuKernel::SetCcuInstrInfo(const CcuRep::CcuInstrInfo& instrInfo) { this->instrInfo_ = instrInfo; }
2797 :
2798 358 : CcuRep::Variable CcuKernel::CreateVariable()
2799 : {
2800 358 : const auto& var = CreateBlockResAssist(1, res_.continuousVariable);
2801 716 : return var[0];
2802 358 : }
2803 :
2804 109 : CcuRep::Variable CcuKernel::CreateExpectVar()
2805 : {
2806 : // v2(A6) 的 jump 只支持 var-var 比较，需要真实 XN 承载立即数；
2807 : // v1(A5) 的 jump 直接支持 var-imm，expectVar 不参与翻译，
2808 : // 用不进 res_ 账本的壳 Variable 即可，避免 v1 资源虚高。
2809 109 : if (ccuVersion_ == CcuVersion::CCU_V2) {
2810 20 : return CreateVariable();
2811 : }
2812 89 : return CcuRep::Variable(this);
2813 : }
2814 :
2815 126 : CcuRep::Variable CcuKernel::CreateJumpTargetVar()
2816 : {
2817 126 : if (ccuVersion_ == CcuVersion::CCU_V1) {
2818 : // A5(CCU_V1): 没有指令重排，可复用同一个 Xn 存放跳转目标地址。targetVar 仅用于 LoadImdToXn + 紧接 Jump。
2819 102 : if (!sharedJumpTargetVar_) {
2820 14 : sharedJumpTargetVar_ = std::make_unique<CcuRep::Variable>(CreateVariable());
2821 : }
2822 102 : return *sharedJumpTargetVar_;
2823 : }
2824 24 : return CcuRep::CreateVariable(this);
2825 : }
2826 :
2827 203 : CcuRep::Address CcuKernel::CreateAddress()
2828 : {
2829 203 : if (ccuVersion_ == CcuVersion::CCU_V2) {
2830 : // A6创建Address时，需要添加到Variable的列表中，但是仍以Address返回
2831 91 : return CcuRep::Address(CreateResAssist(res_.continuousVariable));
2832 : }
2833 112 : return CreateResAssist(res_.blockAddress);
2834 : }
2835 :
2836 0 : CcuRep::LocalNotify CcuKernel::CreateLocalNotify() { return CreateResAssist(res_.localNotify); }
2837 :
2838 0 : CcuRep::CompletedEvent CcuKernel::CreateCompletedEvent() { return CreateResAssist(res_.blockCompletedEvent); }
2839 :
2840 0 : CcuRep::CcuBuf CcuKernel::CreateCcuBuf()
2841 : {
2842 0 : const auto& buffer = CreateBlockResAssist(1, res_.blockCcubufs);
2843 0 : return buffer[0];
2844 0 : }
2845 :
2846 0 : CcuRep::Executor CcuKernel::CreateExecutor() { return CreateResAssist(res_.blockExecutor); }
2847 :
2848 156 : CcuRep::LocalAddr CcuKernel::CreateLocalAddr() { return CcuRep::LocalAddr(CreateAddress(), CreateVariable()); }
2849 :
2850 37 : CcuRep::RemoteAddr CcuKernel::CreateRemoteAddr() { return CcuRep::RemoteAddr(CreateAddress(), CreateVariable()); }
2851 :
2852 0 : CcuRep::RemoteAddr CcuKernel::GetRemoteAddr(const ChannelHandle channel, uint32_t index)
2853 : {
2854 : (void)index;
2855 0 : channels_.insert(channel);
2856 0 : auto mem = CcuRep::RemoteAddr(CreateAddress(), CreateVariable());
2857 0 : Append(std::make_shared<CcuRep::CcuRepRemMem>(insGenerator, channel, mem));
2858 0 : return mem;
2859 0 : }
2860 :
2861 0 : CcuRep::LocalAddr CcuKernel::CreateLocalAddr(const CcuRep::Variable& token)
2862 : {
2863 0 : return CcuRep::LocalAddr(CreateAddress(), token);
2864 : }
2865 :
2866 0 : HcclResult CcuKernel::CreateBlockCcuBuf(const uint32_t count, CcuRep::CcuBuf* ccuBufs)
2867 : {
2868 0 : CHK_PTR_NULL(ccuBufs);
2869 0 : auto resources = CreateBlockResAssist(count, res_.blockCcubufs);
2870 :
2871 0 : for (uint32_t i = 0; i < count; i++) {
2872 0 : ccuBufs[i] = resources[i]; // 拷贝虚拟资源，通过shared_ptr链接到物理资源
2873 : }
2874 :
2875 0 : return HcclResult::HCCL_SUCCESS;
2876 0 : }
2877 :
2878 18 : HcclResult CcuKernel::CreateBlockExecutor(const uint32_t count, CcuRep::Executor* ccuExes)
2879 : {
2880 18 : CHK_PTR_NULL(ccuExes);
2881 18 : auto resources = CreateBlockResAssist(count, res_.blockExecutor);
2882 :
2883 48 : for (uint32_t i = 0; i < count; i++) {
2884 30 : ccuExes[i] = resources[i]; // 拷贝虚拟资源，通过shared_ptr链接到物理资源
2885 : }
2886 :
2887 18 : return HcclResult::HCCL_SUCCESS;
2888 18 : }
2889 :
2890 0 : HcclResult CcuKernel::CreateBlockCompletedEvent(const uint32_t count, CcuRep::CompletedEvent* ccuEvents)
2891 : {
2892 0 : CHK_PTR_NULL(ccuEvents);
2893 0 : auto resources = CreateBlockResAssist(count, res_.blockCompletedEvent);
2894 :
2895 0 : for (uint32_t i = 0; i < count; i++) {
2896 0 : ccuEvents[i] = resources[i]; // 拷贝虚拟资源，通过shared_ptr链接到物理资源
2897 : }
2898 :
2899 0 : return HcclResult::HCCL_SUCCESS;
2900 0 : }
2901 :
2902 46 : void CcuKernel::SetResRepository(const CcuResRepository& resRepo) { resRepo_ = resRepo; }
2903 :
2904 46 : CcuResRepository& CcuKernel::GetResRepository() { return resRepo_; }
2905 :
2906 46 : CcuSharedResource& CcuKernel::GetExportedRes() { return exportedRes_; }
2907 :
2908 46 : CcuSharedResource& CcuKernel::GetImportedRes() { return importedRes_; }
2909 :
2910 10 : static HcclResult GetArgIndex(
2911 : const std::unordered_map<uint16_t, uint16_t>& varId2VarIdMap,
2912 : const std::unordered_map<uint16_t, uint32_t>& varId2ArgIndexMap, const uint64_t* taskArgs, uint32_t argSize,
2913 : uint16_t varId, uint64_t& argIndex)
2914 : {
2915 10 : HCCL_INFO("[GetArgIndex] Enter varId(%u)", varId);
2916 10 : auto item = varId2ArgIndexMap.find(varId);
2917 10 : if (item == varId2ArgIndexMap.end()) {
2918 0 : uint16_t oriVarId = varId;
2919 0 : auto iter = varId2VarIdMap.find(varId);
2920 0 : while (iter != varId2VarIdMap.end()) { // 循环查找中间assign Rep，找到起始varId
2921 0 : oriVarId = iter->second;
2922 0 : iter = varId2VarIdMap.find(oriVarId);
2923 : }
2924 0 : if (oriVarId != varId) { // 起始varId预期通过LoadArg赋值
2925 0 : item = varId2ArgIndexMap.find(oriVarId);
2926 0 : if (item == varId2ArgIndexMap.end()) {
2927 0 : HCCL_ERROR("[%s]fail, Invalid goSize variable id(%u), oriVarId = %u", __func__, varId, oriVarId);
2928 0 : return HCCL_E_PARA;
2929 : }
2930 : } else {
2931 0 : HCCL_ERROR("[%s]fail, Invalid goSize variable id(%u)", __func__, varId);
2932 0 : return HCCL_E_PARA;
2933 : }
2934 : }
2935 10 : HCCL_INFO("[GetArgIndex] find end");
2936 10 : if (item->second >= argSize) {
2937 0 : HCCL_ERROR("Invalid goSize variable index(%u).", item->second);
2938 0 : return HCCL_E_PARA;
2939 : }
2940 10 : HCCL_INFO(
2941 : "GetArgIndex success: varId(%u) varId2VarIdMapSize(%u) varId2ArgIndexMapSize(%u) taskArgsSize(%u)", varId,
2942 : varId2VarIdMap.size(), varId2ArgIndexMap.size(), argSize);
2943 10 : argIndex = taskArgs[item->second];
2944 10 : return HCCL_SUCCESS;
2945 : }
2946 :
2947 0 : void DumpCcuProfilingInfo(const std::vector<CcuProfilingInfo>& ccuProfilingInfo)
2948 : {
2949 0 : auto dumpLinkInfo = [](const CcuProfilingInfo& info) -> void {
2950 0 : for (int i = 0; i < CCU_MAX_CHANNEL_NUM; i++) {
2951 0 : if (info.channelId[i] == INVALID_VALUE_CHANNELID) {
2952 0 : continue;
2953 : }
2954 0 : HCCL_INFO("channelId(%u), remoteRankId(%u).", info.channelId[i], info.remoteRankId[i]);
2955 : }
2956 0 : };
2957 :
2958 0 : for (const auto& profInfo : ccuProfilingInfo) {
2959 0 : if (profInfo.type == static_cast<uint8_t>(CcuProfilinType::CCU_TASK_PROFILING)) {
2960 0 : HCCL_INFO(
2961 : "Dump CCU Profiling Info:SQE Profiling Info: ctxSignautre(%s), "
2962 : "dieId(%d), missionId(%d), instrId(%d).",
2963 : profInfo.name.c_str(), static_cast<int>(profInfo.dieId), static_cast<int>(profInfo.missionId),
2964 : static_cast<int>(profInfo.instrId));
2965 0 : } else if (profInfo.type == static_cast<uint8_t>(CcuProfilinType::CCU_WAITCKE_PROFILING)) {
2966 0 : HCCL_INFO(
2967 : "Microcode WaitCKE Profiling Info: name(%s), "
2968 : "dieId(%d), missionId(%d), instrId(%d), ckeId(%u), mask(%u).",
2969 : profInfo.name.c_str(), static_cast<int>(profInfo.dieId), static_cast<int>(profInfo.missionId),
2970 : static_cast<int>(profInfo.instrId), profInfo.ckeId, profInfo.mask);
2971 0 : dumpLinkInfo(profInfo);
2972 0 : } else if (profInfo.type == static_cast<uint8_t>(CcuProfilinType::CCU_LOOPGROUP_PROFILING)) {
2973 0 : HCCL_INFO(
2974 : "Microcode LoopGroup Profiling Info: name(%s), "
2975 : "dieId(%d), missionId(%d), instrId(%d), reduceOpType(%d), inputDataType(%d), "
2976 : "outputDataType(%d), dataSize(%llu).",
2977 : profInfo.name.c_str(), static_cast<int>(profInfo.dieId), static_cast<int>(profInfo.missionId),
2978 : static_cast<int>(profInfo.instrId), static_cast<int>(profInfo.reduceOpType),
2979 : static_cast<int>(profInfo.inputDataType), static_cast<int>(profInfo.outputDataType), profInfo.dataSize);
2980 0 : dumpLinkInfo(profInfo);
2981 : }
2982 : }
2983 0 : }
2984 :
2985 2 : constexpr uint64_t SetBits(uint16_t end) { return ((uint64_t(1) << (end + 1)) - uint64_t(1)); }
2986 :
2987 2 : static uint16_t ParseRepeatNumFromParallelParam(uint64_t parallelParam)
2988 : {
2989 2 : constexpr uint16_t repeatBitNum = 7; // 7： repeat num 占 7 bits
2990 2 : constexpr uint16_t repeatNumShiftBit = 55; // 55： repeat num占[61:55]位置
2991 2 : return (parallelParam >> repeatNumShiftBit) & SetBits(repeatBitNum);
2992 : }
2993 :
2994 0 : HcclResult CcuKernel::CollectSqeAndWaitCkeProfilingInfo()
2995 : {
2996 0 : auto& ccuProfilingCache = GetProfilingInfo();
2997 0 : uint32_t count{0};
2998 0 : HCCL_INFO("[GetCcuProfilingInfo] Process sqe&waitcke profiling info start.");
2999 0 : for (auto& profInfo : ccuProfilingCache) {
3000 0 : profInfo.missionId = GetMissionId();
3001 0 : if (profInfo.type == static_cast<uint8_t>(hcomm::CcuProfilinType::CCU_TASK_PROFILING)) {
3002 0 : profInfo.instrId = GetInstrId();
3003 0 : allCcuProfilingInfos_.push_back(profInfo);
3004 0 : continue;
3005 : }
3006 0 : if (count >= GetWaiteCkeProfilingReps().size()) {
3007 0 : HCCL_ERROR(
3008 : "count[%u] out of range[0, %u], cache size(%u).", count, GetWaiteCkeProfilingReps().size(),
3009 : ccuProfilingCache.size());
3010 0 : return HCCL_E_INTERNAL;
3011 : }
3012 0 : auto waitCkeRep = GetWaiteCkeProfilingReps()[count];
3013 0 : profInfo.instrId = waitCkeRep->StartInstrId();
3014 0 : if (profInfo.ckeId == INVALID_CKE_ID) { // localWait Rep
3015 0 : if (waitCkeRep.get() == nullptr) {
3016 0 : HCCL_ERROR("[GetCcuProfilingInfo] localWaitRep is nullptr.");
3017 0 : return HCCL_E_PTR;
3018 : }
3019 0 : profInfo.ckeId = waitCkeRep->GetId();
3020 0 : HCCL_INFO("[CcuKernel][GetCcuProfilingInfo] waitcke[%u]", profInfo.ckeId);
3021 : }
3022 0 : allCcuProfilingInfos_.push_back(profInfo);
3023 0 : count++;
3024 0 : }
3025 0 : return HCCL_SUCCESS;
3026 : }
3027 :
3028 0 : HcclResult CcuKernel::BuildLoopGroupVarIdMaps(
3029 : std::unordered_map<uint16_t, uint32_t>& varId2ArgIndexMap, std::unordered_map<uint16_t, uint16_t>& varId2VarIdMap)
3030 : {
3031 0 : auto& lgProfInfo = GetLGProfilingInfo();
3032 0 : HCCL_INFO("[GetCcuProfilingInfo] create varId2ArgIndexMap start. size=%lu", lgProfInfo.loadRep2ArgIdxMap.size());
3033 0 : for (auto& iter : lgProfInfo.loadRep2ArgIdxMap) {
3034 0 : if (iter.first.get() == nullptr) {
3035 0 : HCCL_ERROR("[GetCcuProfilingInfo] loadRep is nullptr.");
3036 0 : return HCCL_E_PTR;
3037 : }
3038 0 : auto loadRep = dynamic_cast<CcuRep::CcuRepLoadArg*>(iter.first.get());
3039 0 : varId2ArgIndexMap[loadRep->GetVarId()] = iter.second;
3040 : }
3041 :
3042 0 : HCCL_INFO("[GetCcuProfilingInfo] create varId2VarIdMap start. size=%lu", lgProfInfo.assignProfilingReps.size());
3043 0 : for (auto& iter : lgProfInfo.assignProfilingReps) {
3044 0 : if (iter.get() == nullptr) {
3045 0 : HCCL_ERROR("[GetCcuProfilingInfo] assignRep is nullptr.");
3046 0 : return HCCL_E_PTR;
3047 : }
3048 0 : auto assignRep = dynamic_cast<CcuRep::CcuRepAssign*>(iter.get());
3049 0 : varId2VarIdMap[assignRep->GetVarB().Id()] = assignRep->GetVarA().Id();
3050 : }
3051 0 : return HCCL_SUCCESS;
3052 : }
3053 :
3054 8 : HcclResult CcuKernel::CollectLoopGroupProfilingInfo(
3055 : const uint64_t* taskArgs, uint32_t argSize, const std::unordered_map<uint16_t, uint32_t>& varId2ArgIndexMap,
3056 : const std::unordered_map<uint16_t, uint16_t>& varId2VarIdMap)
3057 : {
3058 8 : auto& lgProfInfo = GetLGProfilingInfo();
3059 8 : HCCL_INFO(
3060 : "[GetCcuProfilingInfo] process loop group profiling start: "
3061 : "lgsize(%lu), goSize(%lu)",
3062 : lgProfInfo.lgProfilingReps.size(), groupOpSizeInfo_.size());
3063 : // lgProfilingReps、groupOpSizeInfo_、ccuProfilingInfos 设计上同步 push；
3064 : // 但 CcuRepContext::AddProfiling 可能只 push ccuProfilingInfos 而不 push lgProfilingReps，
3065 : // 取三者最小值作为循环安全上界，防止任一容器较短时下标越界。
3066 8 : size_t repSize = lgProfInfo.lgProfilingReps.size();
3067 8 : size_t goSizeNum = groupOpSizeInfo_.size();
3068 8 : size_t profSize = lgProfInfo.ccuProfilingInfos.size();
3069 8 : size_t safeSize = (repSize < goSizeNum) ? repSize : goSizeNum;
3070 8 : safeSize = (profSize < safeSize) ? profSize : safeSize;
3071 8 : if (safeSize != repSize) {
3072 1 : HCCL_WARNING(
3073 : "[CollectLoopGroupProfilingInfo] size mismatch: lgReps[%zu], goSize[%zu], ccuProf[%zu],"
3074 : " use safeSize[%zu].",
3075 : repSize, goSizeNum, profSize, safeSize);
3076 : }
3077 15 : for (uint32_t i = 0; i < safeSize; i += 2) { // 2: 一个goSize对应一个CcuProfilingInfo，对应1个loopGroup Rep
3078 7 : CHK_RET(CollectSingleLoopGroupProfiling(i, repSize, taskArgs, argSize, varId2ArgIndexMap, varId2VarIdMap));
3079 : }
3080 8 : return HCCL_SUCCESS;
3081 : }
3082 :
3083 7 : HcclResult CcuKernel::CollectSingleLoopGroupProfiling(
3084 : uint32_t i, size_t repSize, const uint64_t* taskArgs, uint32_t argSize,
3085 : const std::unordered_map<uint16_t, uint32_t>& varId2ArgIndexMap,
3086 : const std::unordered_map<uint16_t, uint16_t>& varId2VarIdMap)
3087 : {
3088 7 : if (argSize == 0 || varId2ArgIndexMap.empty()) {
3089 3 : return HCCL_SUCCESS;
3090 : }
3091 4 : auto& lgProfInfo = GetLGProfilingInfo();
3092 4 : uint64_t loopParam{0};
3093 4 : CHK_RET(
3094 : GetArgIndex(varId2VarIdMap, varId2ArgIndexMap, taskArgs, argSize, groupOpSizeInfo_[i].loopParamId, loopParam));
3095 4 : uint64_t parallelParam{0};
3096 4 : CHK_RET(GetArgIndex(
3097 : varId2VarIdMap, varId2ArgIndexMap, taskArgs, argSize, groupOpSizeInfo_[i].parallelParamId, parallelParam));
3098 4 : HCCL_INFO(
3099 : "Collect loopgroup profiling info: repSize[%u], index[%u],"
3100 : "loopParam[%llu], parallelParam[%llu].",
3101 : lgProfInfo.lgProfilingReps.size(), i, loopParam, parallelParam);
3102 :
3103 4 : if (loopParam != 0) {
3104 1 : lgProfInfo.ccuProfilingInfos[i].dataSize = loopParam * moConfig_.loopCount * moConfig_.memSlice;
3105 1 : lgProfInfo.ccuProfilingInfos[i].instrId
3106 1 : = dynamic_cast<CcuRep::CcuRepLoopGroupBundle*>(lgProfInfo.lgProfilingReps[i].get())->StartInstrId();
3107 1 : allCcuProfilingInfos_.push_back(lgProfInfo.ccuProfilingInfos[i]);
3108 : }
3109 :
3110 4 : if (parallelParam != 0) {
3111 2 : HCCL_INFO("[GetCcuProfilingInfo] collect lg, residual start i=%lu", i);
3112 2 : uint64_t residual{0};
3113 3 : CHK_RET(GetArgIndex(
3114 : varId2VarIdMap, varId2ArgIndexMap, taskArgs, argSize, groupOpSizeInfo_[i].residualId, residual));
3115 2 : uint64_t repeatNum = ParseRepeatNumFromParallelParam(parallelParam);
3116 2 : lgProfInfo.ccuProfilingInfos[i].dataSize = repeatNum * moConfig_.memSlice + residual;
3117 : // rep[i] 对应 loopParam 分支，rep[i+1] 对应 parallelParam 分支；尾项无配对 rep 时跳过避免越界
3118 2 : if (i + 1 >= repSize) {
3119 1 : HCCL_WARNING(
3120 : "[CollectLoopGroupProfilingInfo] parallelParam != 0 but no paired rep,"
3121 : " index(%u), repSize(%zu), skip parallelParam profiling.",
3122 : i, repSize);
3123 1 : return HCCL_SUCCESS;
3124 : }
3125 1 : lgProfInfo.ccuProfilingInfos[i].instrId
3126 1 : = dynamic_cast<CcuRep::CcuRepLoopGroupBundle*>(lgProfInfo.lgProfilingReps[i + 1].get())->StartInstrId();
3127 1 : allCcuProfilingInfos_.push_back(lgProfInfo.ccuProfilingInfos[i]);
3128 : }
3129 3 : return HCCL_SUCCESS;
3130 : }
3131 :
3132 : /*
3133 : * variable/maskSignal等资源变量Id，一定要在获取ccu profiling时才获取；
3134 : * 原因：在创建context Rep时，其资源Id属于虚拟资源；翻译时，才会绑定固定的物理资源。
3135 : */
3136 0 : HcclResult CcuKernel::GetCcuProfilingInfo(
3137 : const uint64_t* taskArgs, uint32_t argSize, std::vector<CcuProfilingInfo>& allCcuProfilingInfo)
3138 : {
3139 0 : HCCL_INFO("[GetCcuProfilingInfo] Enter.");
3140 0 : allCcuProfilingInfos_.clear();
3141 :
3142 0 : CHK_RET(CollectSqeAndWaitCkeProfilingInfo());
3143 :
3144 0 : std::unordered_map<uint16_t, uint32_t> varId2ArgIndexMap;
3145 0 : std::unordered_map<uint16_t, uint16_t> varId2VarIdMap;
3146 0 : CHK_RET(BuildLoopGroupVarIdMaps(varId2ArgIndexMap, varId2VarIdMap));
3147 :
3148 0 : CHK_RET(CollectLoopGroupProfilingInfo(taskArgs, argSize, varId2ArgIndexMap, varId2VarIdMap));
3149 :
3150 0 : DumpCcuProfilingInfo(allCcuProfilingInfos_);
3151 0 : allCcuProfilingInfo = allCcuProfilingInfos_;
3152 0 : return HCCL_SUCCESS;
3153 0 : }
3154 :
3155 0 : HcclResult CcuKernel::AddProfilingInfo(
3156 : const ChannelHandle* channels, uint32_t channelNum, HcclDataType dataType, HcclDataType outputDataType,
3157 : HcclReduceOp opType, const std::string& opName)
3158 : {
3159 0 : CHK_PTR_NULL(channels);
3160 0 : ccuProfilingInfoCache.type = (uint8_t)CcuProfilinType::CCU_LOOPGROUP_PROFILING;
3161 0 : ccuProfilingInfoCache.name = opName;
3162 0 : ccuProfilingInfoCache.reduceOpType = opType;
3163 0 : ccuProfilingInfoCache.inputDataType = dataType;
3164 0 : ccuProfilingInfoCache.outputDataType = outputDataType;
3165 0 : ccuProfilingInfoCache.missionId = GetMissionId();
3166 :
3167 0 : CHK_SAFETY_FUNC_RET(memset_s(
3168 : ccuProfilingInfoCache.channelId, sizeof(ccuProfilingInfoCache.channelId), INVALID_VALUE_CHANNELID,
3169 : sizeof(ccuProfilingInfoCache.channelId)));
3170 0 : for (uint32_t i = 0; i < channelNum; i++) {
3171 0 : void* channelPtr{nullptr};
3172 0 : CHK_RET(static_cast<HcclResult>(HcommChannelGet(channels[i], &channelPtr)));
3173 0 : auto* channelImpl = dynamic_cast<CcuUrmaChannel*>(static_cast<Channel*>(channelPtr));
3174 0 : CHK_PTR_NULL(channelImpl);
3175 0 : ccuProfilingInfoCache.channelId[i] = channelImpl->GetChannelId();
3176 0 : ccuProfilingInfoCache.channelHandle[i] = channels[i];
3177 0 : HCCL_INFO(
3178 : "[%s]type[%d], name[%s], opType[%d], dataType[%d], outputDataType[%d], missionId[%u], "
3179 : "channelHandle[0x%llx], channelId[%u]",
3180 : __func__, ccuProfilingInfoCache.type, ccuProfilingInfoCache.name.c_str(), opType, dataType, outputDataType,
3181 : ccuProfilingInfoCache.missionId, ccuProfilingInfoCache.channelHandle[i],
3182 : ccuProfilingInfoCache.channelId[i]);
3183 : }
3184 0 : lgProfilingInfo.ccuProfilingInfos.push_back(ccuProfilingInfoCache);
3185 0 : lgProfilingInfo.lgProfilingReps.push_back(allLgProfilingReps.back());
3186 0 : return HCCL_SUCCESS;
3187 : }
3188 :
3189 0 : HcclResult CcuKernel::AddCcuProfiling(
3190 : GroupInfo groupInfo, const std::vector<ChannelHandle> channelHandle, HcclDataType dataType,
3191 : HcclDataType outputDataType, HcclReduceOp opType, const std::string& opName)
3192 : {
3193 0 : CHK_RET(AddCcuProfiling(channelHandle.data(), channelHandle.size(), dataType, outputDataType, opType, opName));
3194 0 : groupOpSizeInfo_.push_back(groupInfo);
3195 0 : return HCCL_SUCCESS;
3196 : }
3197 :
3198 0 : HcclResult CcuKernel::AddCcuProfiling(
3199 : const ChannelHandle* channels, uint32_t channelNum, HcclDataType dataType, HcclDataType outputDataType,
3200 : HcclReduceOp opType, const std::string& opName)
3201 : {
3202 0 : CHK_PTR_NULL(channels);
3203 0 : CHK_RET(AddProfilingInfo(channels, channelNum, dataType, outputDataType, opType, opName));
3204 0 : return HCCL_SUCCESS;
3205 : }
3206 :
3207 5 : HcclResult CcuKernel::Add2ConstValue2VarMap(std::vector<uint64_t>& values)
3208 : {
3209 : // 记录当前context所需的常量，仅A6场景适用
3210 21 : for (uint64_t value : values) {
3211 16 : if (constValue2VarMap.find(value) == constValue2VarMap.end()) {
3212 3 : constValue2VarMap[value] = CreateVariable();
3213 : }
3214 : }
3215 5 : return HCCL_SUCCESS;
3216 : }
3217 :
3218 : }; // namespace hcomm
|
