Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "ccu_kernel.h"
12 :
13 : #include <algorithm>
14 :
15 : #include "ccu_rep_v1.h"
16 : #include "ccu_kernel_resource.h"
17 : #include "ccu_microcode_v1.h"
18 :
19 : #include "ccu_types.h"
20 : #include "exception_util.h"
21 : #include "ccu_api_exception.h"
22 : #include "ccu_dev_mgr_imp.h"
23 : #include "env_config.h"
24 : #include "ccu_rep_type_v1.h"
25 :
26 : #include "hcomm_c_adpt.h"
27 :
28 : #include "ccu_rep_context_v1.h"
29 : #include "ccu_rep_funccall_v1.h"
30 : #include "../../endpoint_pairs/channels/ccu/ccu_urma_channel.h"
31 : #include "../ccu_instance/ccu_var_event_res_mgr.h"
32 :
33 : #include "ccu_log.h"
34 :
35 : #include "hcom_common.h"
36 :
37 : // todo: 引入头文件需要检查
38 : #include "ccu_assist_v1.h"
39 : #include "hccl_comm_pub.h"
40 : #include "hcclCommDfx.h"
41 : #include "task_info.h"
42 : #include "task_param.h"
43 :
44 : #include "ccu_ins_generator_base.h"
45 : #include "ccu_ins_generator_v1.h"
46 : #include "unified_platform/pub_inc/config_plf_log.h"
47 :
48 : namespace hcomm {
49 :
50 : using Hccl::PLF_DATA_OP;
51 :
52 : constexpr uint32_t TOKEN_VALUE_INDEX = 2;
53 : constexpr uint16_t INVALID_U16 = 65535;
54 : constexpr uint32_t MAX_LOOP_ENGINE_POOL_SIZE_V1 = 128;
55 : constexpr uint32_t MAX_LOOP_ENGINE_POOL_SIZE_V2 = 512;
56 :
57 : using CcuRep::CcuInsGeneratorBase;
58 : using CcuRep::CcuInsGeneratorV1;
59 :
60 : template <typename T>
61 217 : T CcuKernel::CreateResAssist(std::array<std::vector<T>, CCU_MAX_IODIE_NUM>& resRecord)
62 : {
63 : // kernel确认die之前默认为0，需要刷新资源
64 : // 确认die之后按实际使用die分配资源
65 217 : const uint32_t dieId = GetDieId();
66 217 : resRecord[dieId].emplace_back(this);
67 217 : auto& item = resRecord[dieId].back();
68 217 : item.Reset(resRecord[dieId].size(), dieId);
69 217 : return item;
70 : }
71 :
72 : template <typename T>
73 : std::vector<T>
74 896 : CcuKernel::CreateBlockResAssist(const uint32_t count, std::array<std::vector<T>, CCU_MAX_IODIE_NUM>& resRecord)
75 : {
76 896 : constexpr uint16_t CCU_BLOCK_RES_ID_BASE = 0x1000; // block 批量分配资源 id 基址，与单资源 id 区间隔离便于 DFX 定位
77 896 : std::vector<T> block;
78 896 : block.reserve(count);
79 896 : const uint32_t dieId = GetDieId();
80 2295 : for (size_t i = 0; i < count; i++) {
81 1399 : block.emplace_back(this);
82 1399 : block.back().Reset(static_cast<uint16_t>(CCU_BLOCK_RES_ID_BASE + resRecord[dieId].size() + i), dieId);
83 : }
84 896 : resRecord[dieId].insert(resRecord[dieId].end(), block.begin(), block.end());
85 896 : return block;
86 0 : }
87 :
88 182 : CcuKernel::~CcuKernel() {}
89 :
90 11 : static HcclResult GetDieIdByChannel(const ChannelHandle channel, uint32_t& dieId)
91 : {
92 11 : void* channelPtr{nullptr};
93 11 : CHK_RET(static_cast<HcclResult>(HcommChannelGet(channel, &channelPtr)));
94 11 : auto* channelImpl = dynamic_cast<CcuUrmaChannel*>(static_cast<Channel*>(channelPtr));
95 11 : if (channelImpl == nullptr) {
96 0 : HCCL_ERROR("[%s] failed to cast channel[0x%llx] to CcuUrmaChannel", __func__, channel);
97 0 : return HcclResult::HCCL_E_PTR;
98 : }
99 11 : dieId = channelImpl->GetDieId();
100 11 : HCCL_INFO("[%s], channelHandle[0x%llx], dieId[%u]", __func__, channel, dieId);
101 11 : return HcclResult::HCCL_SUCCESS;
102 : }
103 :
104 1 : static HcclResult GetDieIdByChannels(const std::unordered_set<ChannelHandle>& channels, uint32_t& dieId)
105 : {
106 1 : if (channels.empty()) {
107 1 : int32_t devLogicId = HcclGetThreadDeviceId();
108 1 : for (uint32_t die = 0; die < CCU_MAX_IODIE_NUM; die++) {
109 1 : bool enableFlag = false;
110 2 : CHK_RET(static_cast<HcclResult>(CcuGetDieEnableInfo(devLogicId, die, enableFlag)));
111 1 : if (enableFlag) {
112 1 : dieId = die;
113 1 : return HcclResult::HCCL_SUCCESS;
114 : }
115 : }
116 :
117 0 : HCCL_ERROR("[CcuKernel][%s] failed, all dies are disable, devLogicId[%d].", __func__, devLogicId);
118 0 : return HcclResult::HCCL_E_INTERNAL;
119 : }
120 :
121 0 : uint32_t firstDieId = 0;
122 0 : CHK_RET(GetDieIdByChannel(*channels.begin(), firstDieId));
123 0 : for (const auto channel : channels) {
124 0 : uint32_t nextDieId = 0;
125 0 : CHK_RET(GetDieIdByChannel(channel, nextDieId));
126 0 : if (firstDieId != nextDieId) {
127 0 : HCCL_ERROR("[%s] failed, the dies of channels are not same.", __func__);
128 0 : return HcclResult::HCCL_E_PARA;
129 : }
130 : }
131 :
132 0 : dieId = firstDieId;
133 0 : return HcclResult::HCCL_SUCCESS;
134 : }
135 :
136 59 : static HcclResult CheckChannelsDie(const std::unordered_set<ChannelHandle>& channels, const uint32_t targetDieId)
137 : {
138 67 : for (const auto channel : channels) {
139 11 : uint32_t channelDieId = 0;
140 14 : CHK_RET(GetDieIdByChannel(channel, channelDieId));
141 11 : if (channelDieId != targetDieId) {
142 3 : HCCL_ERROR(
143 : "[%s] failed, channel[0x%llx] dieId[%u] differs from target dieId[%u].", __func__, channel,
144 : channelDieId, targetDieId);
145 3 : return HcclResult::HCCL_E_PARA;
146 : }
147 : }
148 56 : return HcclResult::HCCL_SUCCESS;
149 : }
150 :
151 57 : static void MoveResourcesToDie(CcuRepResource& res, uint32_t targetDieId)
152 : {
153 57 : if (targetDieId == 0)
154 56 : return; // 初始资源位于die0，不用设置
155 :
156 11 : auto moveAndSet = [&targetDieId](auto& arr) {
157 11 : arr[targetDieId] = std::move(arr[0]);
158 11 : for (auto& item : arr[targetDieId])
159 0 : item.SetDieId(targetDieId);
160 12 : };
161 :
162 1 : moveAndSet(res.ccubufs);
163 1 : moveAndSet(res.blockCcubufs);
164 1 : moveAndSet(res.executor);
165 1 : moveAndSet(res.blockExecutor);
166 1 : moveAndSet(res.completedEvent);
167 1 : moveAndSet(res.blockCompletedEvent);
168 1 : moveAndSet(res.address);
169 1 : moveAndSet(res.blockAddress);
170 1 : moveAndSet(res.continuousVariable);
171 1 : moveAndSet(res.variable);
172 1 : moveAndSet(res.localNotify);
173 : }
174 :
175 77 : HcclResult CcuKernel::SetupProfilingInfo(const char* kernelFuncName)
176 : {
177 77 : if (kernelFuncName == nullptr || strlen(kernelFuncName) == 0) {
178 0 : name_ = std::string("CCU_KERNEL"); // 默认名称
179 0 : AddSqeProfiling(name_);
180 0 : return HcclResult::HCCL_SUCCESS;
181 : }
182 :
183 77 : constexpr size_t MAX_KERNEL_FUNC_NAME_LEN = 128;
184 77 : const auto nameLen = strlen(kernelFuncName);
185 77 : if (nameLen > MAX_KERNEL_FUNC_NAME_LEN) {
186 0 : name_ = std::string(kernelFuncName, MAX_KERNEL_FUNC_NAME_LEN);
187 0 : HCCL_WARNING("[CcuKernel][%s] kernelFuncName is too long, reset to %s.", __func__, name_.c_str());
188 : }
189 :
190 : // 生成SQE粒度profiling信息，此时未选择die，默认die 0
191 77 : AddSqeProfiling(name_);
192 77 : return HcclResult::HCCL_SUCCESS;
193 : }
194 :
195 : static HcclResult
196 57 : UpdateProfilingInfo(std::vector<CcuProfilingInfo>& profilingInfos, uint32_t dieId, const std::string& kernelName)
197 : {
198 57 : if (dieId == 0) {
199 : // 与默认dieId相同，不需要修改
200 56 : return HcclResult::HCCL_SUCCESS;
201 : }
202 :
203 : // 正常情况仅首个info包含die信息，仅应为CCU_TASK_PROFILING类型
204 1 : if (UNLIKELY(profilingInfos.empty())) {
205 : // profiling不属于主流程，不打断算子业务
206 0 : HCCL_INFO("[%s] passed, profiling infos are empty, ccu kernel func[%s].", __func__, kernelName.c_str());
207 0 : return HcclResult::HCCL_SUCCESS;
208 : }
209 :
210 : // 根据选择的die跟新profiling信息
211 2 : for (auto& info : profilingInfos) {
212 1 : info.dieId = dieId;
213 : }
214 :
215 1 : HCCL_INFO("[%s] reset profiling info dieId to [%u], ccu kernel func[%s].", __func__, dieId, kernelName.c_str());
216 1 : return HcclResult::HCCL_SUCCESS;
217 : }
218 :
219 1 : HcclResult CcuKernel::ApplyDieFromChannels()
220 : {
221 1 : uint32_t dieId{0};
222 1 : CHK_RET(GetDieIdByChannels(channels_, dieId));
223 1 : CHK_PRT_RET(
224 : dieId >= CCU_MAX_IODIE_NUM,
225 : HCCL_ERROR("[CcuKernel][%s] failed, dieId[%u] should be less than [%u].", __func__, dieId, CCU_MAX_IODIE_NUM),
226 : HcclResult::HCCL_E_PARA);
227 1 : SetDieId(dieId);
228 1 : MoveResourcesToDie(res_, dieId);
229 1 : (void)UpdateProfilingInfo(profilingInfo, dieId, name_);
230 :
231 1 : return HcclResult::HCCL_SUCCESS;
232 : }
233 :
234 59 : HcclResult CcuKernel::ValidateAndApplyDie(uint32_t targetDieId)
235 : {
236 59 : CHK_PRT_RET(
237 : targetDieId >= CCU_MAX_IODIE_NUM,
238 : HCCL_ERROR(
239 : "[CcuKernel][%s] failed, dieId[%u] should be less than [%u].", __func__, targetDieId, CCU_MAX_IODIE_NUM),
240 : HcclResult::HCCL_E_PARA);
241 :
242 59 : const int32_t devLogicId = HcclGetThreadDeviceId();
243 59 : bool enableFlag = false;
244 59 : CHK_RET(static_cast<HcclResult>(CcuGetDieEnableInfo(devLogicId, static_cast<uint8_t>(targetDieId), enableFlag)));
245 59 : CHK_PRT_RET(
246 : !enableFlag,
247 : HCCL_ERROR(
248 : "[CcuKernel][%s] failed, target dieId[%u] is disabled, devLogicId[%d].", __func__, targetDieId, devLogicId),
249 : HcclResult::HCCL_E_PARA);
250 59 : CHK_RET(CheckChannelsDie(channels_, targetDieId));
251 :
252 56 : SetDieId(targetDieId);
253 56 : MoveResourcesToDie(res_, targetDieId);
254 56 : (void)UpdateProfilingInfo(profilingInfo, targetDieId, name_);
255 :
256 56 : return HcclResult::HCCL_SUCCESS;
257 : }
258 :
259 80 : void CcuKernel::SetInsGenerater(CcuInsGeneratorBase* insGeneratorBase) { insGenerator = insGeneratorBase; }
260 :
261 5 : CcuResult CcuKernel::ValidateTaskArgs(const uint64_t* taskArgs, uint32_t argsNum) const
262 : {
263 5 : if (loadArgUsedSet_.size() != argsNum) {
264 0 : HCCL_ERROR(
265 : "[CcuKernel][%s] failed, args number does not match the Load instruction, "
266 : "argsNum = %u, loaded = %zu",
267 : __func__, argsNum, loadArgUsedSet_.size());
268 0 : return CcuResult::CCU_E_INTERNAL;
269 : }
270 50 : for (uint32_t i = 0; i < argsNum; ++i) {
271 45 : if (loadArgUsedSet_.count(i) == 0) {
272 0 : HCCL_ERROR("[CcuKernel][%s] failed, argId %u not loaded (argsNum=%u)", __func__, i, argsNum);
273 0 : return CcuResult::CCU_E_INTERNAL;
274 : }
275 : }
276 5 : if (argsNum != 0) {
277 3 : CCU_CHK_PTR_NULL(taskArgs);
278 : }
279 5 : if (instrInfo_.missionInstrCount == 0 || instrInfo_.instrVec.empty()) {
280 0 : HCCL_ERROR(
281 : "[CcuKernel][%s] failed, mission instructions are empty, "
282 : "the kernel is not been translated yet.",
283 : __func__);
284 0 : return CcuResult::CCU_E_INTERNAL;
285 : }
286 5 : return CcuResult::CCU_SUCCESS;
287 : }
288 :
289 8 : void CcuKernel::FillTaskParam(
290 : CcuTaskParam& param, uint32_t index, uint32_t seqNum, const uint64_t* taskArgs, uint32_t argsNum) const
291 : {
292 8 : param.dieId = GetDieId();
293 8 : param.missionId = GetMissionId();
294 8 : param.instStartId = instrInfo_.missionStartInstrId + index * CCU_SQE_ARGS_LEN;
295 8 : param.key = GetMissionKey();
296 8 : param.argSize = CCU_SQE_ARGS_LEN;
297 :
298 8 : const uint32_t preMissionInsCnt = index * CCU_SQE_ARGS_LEN;
299 8 : const bool isLast = (index == seqNum - 1);
300 8 : param.instCnt = isLast ? (instrInfo_.missionInstrCount - preMissionInsCnt) : CCU_SQE_ARGS_LEN;
301 :
302 8 : if (argsNum > preMissionInsCnt) {
303 6 : const uint32_t argsToCopy = isLast ? std::min(argsNum - preMissionInsCnt, CCU_SQE_ARGS_LEN) : CCU_SQE_ARGS_LEN;
304 12 : std::copy(taskArgs + preMissionInsCnt, taskArgs + preMissionInsCnt + argsToCopy, std::begin(param.args));
305 : }
306 :
307 8 : HCCL_INFO(
308 : "[GeneTaskParam]task Param, dieId[%u] missionId[%u] instStartId[%u] instCnt[%u], argSize[%u]", param.dieId,
309 : param.missionId, param.instStartId, param.instCnt, param.argSize);
310 8 : }
311 :
312 5 : CcuResult CcuKernel::GeneTaskParams(const uint64_t* taskArgs, uint32_t argsNum, std::vector<CcuTaskParam>& taskParams)
313 : {
314 5 : CCU_CHK_RET(ValidateTaskArgs(taskArgs, argsNum));
315 :
316 : // 如果agrs数量超过sqe arg的最大数量，则返回多个TaskParam，前面几个只从sqe中加载args;
317 : // args数量大于等于0、小于等于最大值时，返回1个TaskParam
318 5 : const uint32_t seqNum
319 5 : = (argsNum / CCU_SQE_ARGS_LEN) + ((argsNum % CCU_SQE_ARGS_LEN) == 0 ? 0 : 1) + (argsNum == 0 ? 1 : 0);
320 :
321 5 : const uint32_t preMissionSqeInsCnt = (seqNum - 1) * CCU_SQE_ARGS_LEN;
322 5 : if (instrInfo_.missionInstrCount < preMissionSqeInsCnt) {
323 0 : HCCL_ERROR(
324 : "[CcuKernel][%s] failed, missionInstrCount[%u] should be greater "
325 : "than preMissionSqeInsCnt[%u].",
326 : __func__, instrInfo_.missionInstrCount, preMissionSqeInsCnt);
327 0 : return CcuResult::CCU_E_INTERNAL;
328 : }
329 :
330 5 : taskParams.resize(seqNum);
331 13 : for (uint32_t index = 0; index < seqNum; index++) {
332 8 : FillTaskParam(taskParams[index], index, seqNum, taskArgs, argsNum);
333 : }
334 :
335 5 : return CcuResult::CCU_SUCCESS;
336 : }
337 :
338 21 : HcclResult CcuKernel::CreateVariable(const ChannelHandle channel, uint32_t varIndex, CcuRep::Variable* var)
339 : {
340 21 : channels_.insert(channel);
341 :
342 21 : void* channelPtr{nullptr};
343 21 : CHK_RET(static_cast<HcclResult>(HcommChannelGet(channel, &channelPtr)));
344 21 : auto* channelImpl = dynamic_cast<CcuUrmaChannel*>(static_cast<Channel*>(channelPtr));
345 21 : if (channelImpl == nullptr) {
346 0 : HCCL_ERROR("[%s] failed to cast channel[0x%llx] to CcuUrmaChannel", __func__, channel);
347 0 : return HcclResult::HCCL_E_PTR;
348 : }
349 21 : uint32_t locXnId{0};
350 21 : CHK_RET(channelImpl->GetLocXnByIndex(varIndex, locXnId));
351 20 : var->Reset(locXnId, channelImpl->GetDieId());
352 20 : return HcclResult::HCCL_SUCCESS;
353 : }
354 :
355 9 : void CcuKernel::AddPinnedRegGroup(const CcuRep::Variable& baseVar, uint16_t count)
356 : {
357 9 : if (count == 0) {
358 0 : return;
359 : }
360 9 : const uint16_t baseId = baseVar.Id();
361 11 : for (const auto& g : pinnedRegGroups_) {
362 5 : if (g.baseVar.Id() == baseId && g.count == count) {
363 3 : return;
364 : }
365 : }
366 6 : pinnedRegGroups_.push_back({baseVar, count});
367 6 : }
368 :
369 47 : CcuRepResource& CcuKernel::GetResource() { return res_; }
370 :
371 97 : CcuResReq CcuKernel::GetResourceRequest()
372 : {
373 97 : CcuResReq req;
374 97 : uint32_t dieId = GetDieId();
375 97 : req.msReq[dieId] = res_.ccubufs[dieId].size();
376 97 : req.blockMsReq[dieId] = res_.blockCcubufs[dieId].size();
377 97 : req.ckeReq[dieId] = res_.completedEvent[dieId].size();
378 97 : req.blockCkeReq[dieId] = res_.blockCompletedEvent[dieId].size() + res_.localNotify[dieId].size();
379 97 : req.loopEngineReq[dieId] = res_.executor[dieId].size();
380 97 : req.blockLoopEngineReq[dieId] = res_.blockExecutor[dieId].size();
381 97 : req.gsaReq[dieId] = res_.address[dieId].size();
382 97 : req.blockGsaReq[dieId] = res_.blockAddress[dieId].size();
383 97 : req.xnReq[dieId] = res_.variable[dieId].size();
384 97 : req.blockXnReq[dieId] = res_.continuousVariable[dieId].size();
385 :
386 97 : req.missionReq.reqType = MissionReqType::FUSION_MULTIPLE_DIE;
387 97 : req.missionReq.req[dieId] = 1;
388 :
389 : auto info = Hccl::StringFormat(
390 : "resource request: dieId[%u], ms[%u], blockMs[%u], cke[%u], blockCke[%u], "
391 : "loopEngine[%u], blockLoopEngine[%u], gsa[%u], blockGsa[%u], xn[%u], blockXn[%u], "
392 : "missionId[%u]",
393 388 : dieId, req.msReq[dieId], req.blockMsReq[dieId], req.ckeReq[dieId], req.blockCkeReq[dieId],
394 388 : req.loopEngineReq[dieId], req.blockLoopEngineReq[dieId], req.gsaReq[dieId], req.blockGsaReq[dieId],
395 97 : req.xnReq[dieId], req.blockXnReq[dieId], req.missionReq.req[dieId]);
396 :
397 97 : HCCL_INFO("%s", info.c_str());
398 :
399 194 : return req;
400 97 : }
401 :
402 : template <typename HandleType, typename ResourceType>
403 1704 : static CcuResult GetResourceByHandle(
404 : std::unordered_map<HandleType, ResourceType>& resourceMap, HandleType handle, ResourceType** resource,
405 : const char* resourceType)
406 : {
407 1704 : auto iter = resourceMap.find(handle);
408 1704 : if (iter == resourceMap.end()) {
409 0 : HCCL_ERROR("[%s] failed to find %s by handle: 0x%llx", __func__, resourceType, handle);
410 0 : return CcuResult::CCU_E_NOT_FOUND;
411 : }
412 :
413 : // ccu资源本身可能重载=，对象赋值会被转换成指令，导致流程失败
414 1704 : *resource = &(iter->second);
415 1704 : return CcuResult::CCU_SUCCESS;
416 : }
417 :
418 1370 : CcuResult CcuKernel::GetVariableByHandle(CcuVariableHandle varHandle, CcuRep::Variable** variable)
419 : {
420 1370 : return GetResourceByHandle(ccuVarMap_, varHandle, variable, "variable");
421 : }
422 : // Alloc 相关接口
423 495 : CcuResult CcuKernel::VariableAlloc(CcuVariableHandle* varHandle)
424 : {
425 495 : PLF_CONFIG_INFO(PLF_DATA_OP, "[VariableAlloc]");
426 495 : const auto& var = CreateBlockResAssist(1, res_.continuousVariable);
427 495 : CcuVariableHandle handle = ccuVarMap_.size();
428 495 : ccuVarMap_.emplace(handle, var[0]);
429 :
430 495 : *varHandle = handle;
431 495 : return CcuResult::CCU_SUCCESS;
432 495 : }
433 10 : CcuResult CcuKernel::AddressAlloc(CcuAddressHandle* addrHandle)
434 : {
435 10 : PLF_CONFIG_INFO(PLF_DATA_OP, "[AddressAlloc]");
436 10 : const auto addr = CreateAddress();
437 10 : CcuAddressHandle handle = ccuAddrMap_.size();
438 10 : ccuAddrMap_.emplace(handle, addr);
439 10 : *addrHandle = handle;
440 10 : return CcuResult::CCU_SUCCESS;
441 10 : }
442 14 : CcuResult CcuKernel::EventAlloc(CcuEventHandle* eventHandle)
443 : {
444 14 : PLF_CONFIG_INFO(PLF_DATA_OP, "[EventAlloc]");
445 14 : const auto& event = CreateResAssist(res_.blockCompletedEvent);
446 14 : CcuEventHandle handle = ccuEventMap_.size();
447 14 : ccuEventMap_.emplace(handle, event);
448 14 : *eventHandle = handle;
449 14 : return CcuResult::CCU_SUCCESS;
450 14 : }
451 2 : CcuResult CcuKernel::BufferAlloc(CcuBufferHandle* bufHandle)
452 : {
453 2 : PLF_CONFIG_INFO(PLF_DATA_OP, "[BufferAlloc]");
454 2 : const auto& buffer = CreateBlockResAssist(1, res_.blockCcubufs);
455 2 : CcuBufferHandle handle = ccuBufferMap_.size();
456 2 : ccuBufferMap_.emplace(handle, buffer[0]);
457 2 : *bufHandle = handle;
458 2 : return CcuResult::CCU_SUCCESS;
459 2 : }
460 156 : CcuResult CcuKernel::LocalAddrAlloc(
461 : CcuLocalAddrHandle* localAddrHandle, CcuAddressHandle* addrHandle, CcuVariableHandle* tokenHandle)
462 : {
463 156 : PLF_CONFIG_INFO(PLF_DATA_OP, "[LocalAddrAlloc]");
464 156 : auto localAddr = CreateLocalAddr();
465 :
466 156 : CcuAddressHandle aHandle = ccuAddrMap_.size();
467 156 : ccuAddrMap_.emplace(aHandle, localAddr.addr);
468 :
469 156 : CcuVariableHandle tHandle = ccuVarMap_.size();
470 156 : ccuVarMap_.emplace(tHandle, localAddr.token);
471 :
472 156 : CcuLocalAddrHandle laHandle = ccuLocalAddrMap_.size();
473 156 : ccuLocalAddrMap_.emplace(laHandle, localAddr);
474 :
475 156 : *localAddrHandle = laHandle;
476 156 : *addrHandle = aHandle;
477 156 : *tokenHandle = tHandle;
478 156 : return CcuResult::CCU_SUCCESS;
479 156 : }
480 37 : CcuResult CcuKernel::RemoteAddrAlloc(
481 : CcuRemoteAddrHandle* remoteAddrHandle, CcuAddressHandle* addrHandle, CcuVariableHandle* tokenHandle)
482 : {
483 37 : PLF_CONFIG_INFO(PLF_DATA_OP, "[RemoteAddrAlloc]");
484 37 : auto remoteAddr = CreateRemoteAddr();
485 :
486 37 : CcuAddressHandle aHandle = ccuAddrMap_.size();
487 37 : ccuAddrMap_.emplace(aHandle, remoteAddr.addr);
488 :
489 37 : CcuVariableHandle tHandle = ccuVarMap_.size();
490 37 : ccuVarMap_.emplace(tHandle, remoteAddr.token);
491 :
492 37 : CcuRemoteAddrHandle raHandle = ccuRemoteAddrMap_.size();
493 37 : ccuRemoteAddrMap_.emplace(raHandle, remoteAddr);
494 :
495 37 : *remoteAddrHandle = raHandle;
496 37 : *addrHandle = aHandle;
497 37 : *tokenHandle = tHandle;
498 37 : return CcuResult::CCU_SUCCESS;
499 37 : }
500 :
501 6 : CcuResult CcuKernel::BlockVariableAlloc(CcuVariableHandle* varHandles, uint32_t count)
502 : {
503 6 : PLF_CONFIG_INFO(PLF_DATA_OP, "[BlockVariableAlloc] count=%u", count);
504 6 : const auto& var = CreateBlockResAssist(count, res_.continuousVariable);
505 18 : for (uint32_t i = 0; i < count; i++) {
506 12 : CcuVariableHandle handle = ccuVarMap_.size();
507 12 : ccuVarMap_.emplace(handle, var[i]);
508 12 : varHandles[i] = handle;
509 : }
510 6 : return CcuResult::CCU_SUCCESS;
511 6 : }
512 :
513 6 : CcuResult CcuKernel::BlockEventAlloc(CcuEventHandle* eventHandles, uint32_t count)
514 : {
515 6 : PLF_CONFIG_INFO(PLF_DATA_OP, "[BlockEventAlloc] count=%u", count);
516 6 : const auto& event = CreateBlockResAssist(count, res_.blockCompletedEvent);
517 52 : for (uint32_t i = 0; i < count; i++) {
518 46 : CcuEventHandle handle = ccuEventMap_.size();
519 46 : ccuEventMap_.emplace(handle, event[i]);
520 46 : eventHandles[i] = handle;
521 : }
522 6 : return CcuResult::CCU_SUCCESS;
523 6 : }
524 :
525 7 : CcuResult CcuKernel::BlockBufferAlloc(CcuBufferHandle* bufHandles, uint32_t count)
526 : {
527 7 : PLF_CONFIG_INFO(PLF_DATA_OP, "[BlockBufferAlloc] count=%u", count);
528 7 : const auto& buffer = CreateBlockResAssist(count, res_.blockCcubufs);
529 332 : for (uint32_t i = 0; i < count; i++) {
530 325 : CcuBufferHandle handle = ccuBufferMap_.size();
531 325 : ccuBufferMap_.emplace(handle, buffer[i]);
532 325 : bufHandles[i] = handle;
533 : }
534 7 : return CcuResult::CCU_SUCCESS;
535 7 : }
536 :
537 21 : CcuResult CcuKernel::VariableCreateByChannel(ChannelHandle channel, uint32_t varIndex, CcuVariableHandle* varHandle)
538 : {
539 21 : PLF_CONFIG_INFO(PLF_DATA_OP, "[VariableCreateByChannel] channel=%llu, varIndex=%u", channel, varIndex);
540 21 : channels_.insert(channel);
541 21 : CcuRep::Variable var(this);
542 21 : CCU_CHK_RET(CreateVariable(channel, varIndex, &var));
543 20 : CcuVariableHandle handle = ccuVarMap_.size();
544 20 : ccuVarMap_.emplace(handle, var);
545 20 : *varHandle = handle;
546 20 : declaredLocXns_.insert(var.Id());
547 20 : return CcuResult::CCU_SUCCESS;
548 21 : }
549 :
550 10 : CcuResult CcuKernel::VariableCreateByAcquire(CcuVariableHandle acqHandle, uint32_t index, CcuVariableHandle* varHandle)
551 : {
552 10 : uint8_t dieId = 0;
553 10 : uint32_t xnId = 0;
554 10 : CCU_CHK_RET(CcuVarEventResMgr::GetInstance(HcclGetThreadDeviceId()).GetVariableXnId(acqHandle, index, dieId, xnId));
555 :
556 10 : CcuRep::Variable var(this);
557 10 : var.Reset(static_cast<uint16_t>(xnId), static_cast<uint16_t>(dieId));
558 10 : CcuVariableHandle handle = ccuVarMap_.size();
559 10 : ccuVarMap_.emplace(handle, var);
560 10 : *varHandle = handle;
561 10 : return CcuResult::CCU_SUCCESS;
562 10 : }
563 :
564 4 : CcuResult CcuKernel::EventCreateByAcquire(CcuEventHandle acqHandle, uint32_t index, CcuEventHandle* eventHandle)
565 : {
566 4 : uint8_t dieId = 0;
567 4 : uint32_t ckeId = 0;
568 4 : CCU_CHK_RET(CcuVarEventResMgr::GetInstance(HcclGetThreadDeviceId()).GetEventCkeId(acqHandle, index, dieId, ckeId));
569 :
570 4 : CcuRep::CompletedEvent event(this);
571 4 : event.Reset(static_cast<uint16_t>(ckeId), static_cast<uint16_t>(dieId));
572 4 : CcuEventHandle handle = ccuEventMap_.size();
573 4 : ccuEventMap_.emplace(handle, event);
574 4 : *eventHandle = handle;
575 4 : return CcuResult::CCU_SUCCESS;
576 4 : }
577 :
578 280 : CcuResult CcuKernel::VariableAssignImm(CcuVariableHandle varHandle, uint64_t immediate)
579 : {
580 280 : PLF_CONFIG_INFO(PLF_DATA_OP, "[VariableAssignImm] varHandle=%llu, immediate=%llu", varHandle, immediate);
581 280 : CcuRep::Variable* variable{nullptr};
582 280 : CCU_CHK_RET(GetVariableByHandle(varHandle, &variable));
583 : // 通过符号重载实现，内部记录rep；异常由入口 HcommCcuKernelRegister 的
584 : // CCU_EXCEPTION_HANDLE_BEGIN/END 统一接住，无需在此局部 try/catch。
585 280 : (*variable) = immediate;
586 280 : return CcuResult::CCU_SUCCESS;
587 : }
588 :
589 158 : CcuResult CcuKernel::VariableAssignVar(CcuVariableHandle varHandle, CcuVariableHandle varA)
590 : {
591 158 : PLF_CONFIG_INFO(PLF_DATA_OP, "[VariableAssignVar] varHandle=%llu, varA=%llu", varHandle, varA);
592 158 : CcuRep::Variable* variable{nullptr};
593 158 : CCU_CHK_RET(GetVariableByHandle(varHandle, &variable));
594 158 : CcuRep::Variable* variableA{nullptr};
595 158 : CCU_CHK_RET(GetVariableByHandle(varA, &variableA));
596 : // 通过符号重载实现，内部记录rep；异常由入口统一 catch。
597 158 : (*variable) = (*variableA);
598 158 : return CcuResult::CCU_SUCCESS;
599 : }
600 :
601 : CcuResult
602 109 : CcuKernel::VariableAddVarToVar(CcuVariableHandle varHandle, CcuVariableHandle varAHandle, CcuVariableHandle varBHandle)
603 : {
604 109 : PLF_CONFIG_INFO(
605 : PLF_DATA_OP, "[VariableAddVarToVar] varHandle=%llu, varAHandle=%llu, varBHandle=%llu", varHandle, varAHandle,
606 : varBHandle);
607 109 : CcuRep::Variable *resVar{nullptr}, *leftVar{nullptr}, *rightVar{nullptr};
608 109 : CCU_CHK_RET(GetVariableByHandle(varHandle, &resVar));
609 109 : CCU_CHK_RET(GetVariableByHandle(varAHandle, &leftVar));
610 109 : CCU_CHK_RET(GetVariableByHandle(varBHandle, &rightVar));
611 :
612 : // 通过符号重载实现，内部记录rep；异常由入口统一 catch。
613 109 : *resVar = *leftVar + *rightVar;
614 109 : return CcuResult::CCU_SUCCESS;
615 : }
616 :
617 : CcuResult
618 2 : CcuKernel::VariableSubVarToVar(CcuVariableHandle varHandle, CcuVariableHandle varAHandle, CcuVariableHandle varBHandle)
619 : {
620 2 : CcuRep::Variable *resVar{nullptr}, *leftVar{nullptr}, *rightVar{nullptr};
621 2 : CCU_CHK_RET(GetVariableByHandle(varHandle, &resVar));
622 2 : CCU_CHK_RET(GetVariableByHandle(varAHandle, &leftVar));
623 2 : CCU_CHK_RET(GetVariableByHandle(varBHandle, &rightVar));
624 :
625 2 : *resVar = *leftVar - *rightVar;
626 2 : return CcuResult::CCU_SUCCESS;
627 : }
628 :
629 : CcuResult
630 2 : CcuKernel::VariableMulVarToVar(CcuVariableHandle varHandle, CcuVariableHandle varAHandle, CcuVariableHandle varBHandle)
631 : {
632 2 : CcuRep::Variable *resVar{nullptr}, *leftVar{nullptr}, *rightVar{nullptr};
633 2 : CCU_CHK_RET(GetVariableByHandle(varHandle, &resVar));
634 2 : CCU_CHK_RET(GetVariableByHandle(varAHandle, &leftVar));
635 2 : CCU_CHK_RET(GetVariableByHandle(varBHandle, &rightVar));
636 :
637 2 : *resVar = *leftVar * *rightVar;
638 2 : return CcuResult::CCU_SUCCESS;
639 : }
640 :
641 1 : CcuResult CcuKernel::VariableAddImmToVar(CcuVariableHandle varHandle, CcuVariableHandle varAHandle, uint16_t immediate)
642 : {
643 1 : CcuRep::Variable *resVar{nullptr}, *leftVar{nullptr};
644 1 : CCU_CHK_RET(GetVariableByHandle(varHandle, &resVar));
645 1 : CCU_CHK_RET(GetVariableByHandle(varAHandle, &leftVar));
646 :
647 1 : *resVar = *leftVar + immediate;
648 1 : return CcuResult::CCU_SUCCESS;
649 : }
650 :
651 2 : CcuResult CcuKernel::VariableSubImmToVar(CcuVariableHandle varHandle, CcuVariableHandle varAHandle, uint16_t immediate)
652 : {
653 2 : CcuRep::Variable *resVar{nullptr}, *leftVar{nullptr};
654 2 : CCU_CHK_RET(GetVariableByHandle(varHandle, &resVar));
655 2 : CCU_CHK_RET(GetVariableByHandle(varAHandle, &leftVar));
656 :
657 2 : *resVar = *leftVar - immediate;
658 2 : return CcuResult::CCU_SUCCESS;
659 : }
660 :
661 2 : CcuResult CcuKernel::VariableMulImmToVar(CcuVariableHandle varHandle, CcuVariableHandle varAHandle, uint16_t immediate)
662 : {
663 2 : CcuRep::Variable *resVar{nullptr}, *leftVar{nullptr};
664 2 : CCU_CHK_RET(GetVariableByHandle(varHandle, &resVar));
665 2 : CCU_CHK_RET(GetVariableByHandle(varAHandle, &leftVar));
666 :
667 2 : *resVar = *leftVar * immediate;
668 2 : return CcuResult::CCU_SUCCESS;
669 : }
670 :
671 : CcuResult
672 2 : CcuKernel::VariableAndVarToVar(CcuVariableHandle varHandle, CcuVariableHandle varAHandle, CcuVariableHandle varBHandle)
673 : {
674 2 : CcuRep::Variable *resVar{nullptr}, *leftVar{nullptr}, *rightVar{nullptr};
675 2 : CCU_CHK_RET(GetVariableByHandle(varHandle, &resVar));
676 2 : CCU_CHK_RET(GetVariableByHandle(varAHandle, &leftVar));
677 2 : CCU_CHK_RET(GetVariableByHandle(varBHandle, &rightVar));
678 :
679 2 : *resVar = *leftVar & *rightVar;
680 2 : return CcuResult::CCU_SUCCESS;
681 : }
682 :
683 : CcuResult
684 2 : CcuKernel::VariableOrVarToVar(CcuVariableHandle varHandle, CcuVariableHandle varAHandle, CcuVariableHandle varBHandle)
685 : {
686 2 : CcuRep::Variable *resVar{nullptr}, *leftVar{nullptr}, *rightVar{nullptr};
687 2 : CCU_CHK_RET(GetVariableByHandle(varHandle, &resVar));
688 2 : CCU_CHK_RET(GetVariableByHandle(varAHandle, &leftVar));
689 2 : CCU_CHK_RET(GetVariableByHandle(varBHandle, &rightVar));
690 :
691 2 : *resVar = *leftVar | *rightVar;
692 2 : return CcuResult::CCU_SUCCESS;
693 : }
694 :
695 : CcuResult
696 2 : CcuKernel::VariableXorVarToVar(CcuVariableHandle varHandle, CcuVariableHandle varAHandle, CcuVariableHandle varBHandle)
697 : {
698 2 : CcuRep::Variable *resVar{nullptr}, *leftVar{nullptr}, *rightVar{nullptr};
699 2 : CCU_CHK_RET(GetVariableByHandle(varHandle, &resVar));
700 2 : CCU_CHK_RET(GetVariableByHandle(varAHandle, &leftVar));
701 2 : CCU_CHK_RET(GetVariableByHandle(varBHandle, &rightVar));
702 :
703 2 : *resVar = *leftVar ^ *rightVar;
704 2 : return CcuResult::CCU_SUCCESS;
705 : }
706 :
707 1 : CcuResult CcuKernel::VariableNotVar(CcuVariableHandle varHandle, CcuVariableHandle varAHandle)
708 : {
709 1 : CcuRep::Variable *resVar{nullptr}, *leftVar{nullptr};
710 1 : CCU_CHK_RET(GetVariableByHandle(varHandle, &resVar));
711 1 : CCU_CHK_RET(GetVariableByHandle(varAHandle, &leftVar));
712 :
713 1 : *resVar = ~(*leftVar);
714 1 : return CcuResult::CCU_SUCCESS;
715 : }
716 :
717 : CcuResult
718 2 : CcuKernel::VariableShlVarToVar(CcuVariableHandle varHandle, CcuVariableHandle varAHandle, CcuVariableHandle varBHandle)
719 : {
720 2 : CcuRep::Variable *resVar{nullptr}, *leftVar{nullptr}, *rightVar{nullptr};
721 2 : CCU_CHK_RET(GetVariableByHandle(varHandle, &resVar));
722 2 : CCU_CHK_RET(GetVariableByHandle(varAHandle, &leftVar));
723 2 : CCU_CHK_RET(GetVariableByHandle(varBHandle, &rightVar));
724 :
725 2 : *resVar = *leftVar << *rightVar;
726 2 : return CcuResult::CCU_SUCCESS;
727 : }
728 :
729 : CcuResult
730 2 : CcuKernel::VariableShrVarToVar(CcuVariableHandle varHandle, CcuVariableHandle varAHandle, CcuVariableHandle varBHandle)
731 : {
732 2 : CcuRep::Variable *resVar{nullptr}, *leftVar{nullptr}, *rightVar{nullptr};
733 2 : CCU_CHK_RET(GetVariableByHandle(varHandle, &resVar));
734 2 : CCU_CHK_RET(GetVariableByHandle(varAHandle, &leftVar));
735 2 : CCU_CHK_RET(GetVariableByHandle(varBHandle, &rightVar));
736 :
737 2 : *resVar = *leftVar >> *rightVar;
738 2 : return CcuResult::CCU_SUCCESS;
739 : }
740 :
741 : /*========== Event信号同步类 相关接口 ==========*/
742 18 : CcuResult CcuKernel::EventRecord(CcuEventHandle eventHandle, uint32_t mask)
743 : {
744 18 : PLF_CONFIG_INFO(PLF_DATA_OP, "[EventRecord] eventHandle=%llu, mask=%u", eventHandle, mask);
745 18 : CcuRep::CompletedEvent* event{nullptr};
746 18 : CCU_CHK_RET(GetEventByHandle(eventHandle, &event));
747 : // 复用已有的 RecordEvent 实现（内部 Append CcuRepLocRecordEvent）
748 18 : CCU_CHK_RET(RecordEvent(*event, mask));
749 17 : return CcuResult::CCU_SUCCESS;
750 : }
751 :
752 44 : CcuResult CcuKernel::EventWait(CcuEventHandle eventHandle, uint32_t mask)
753 : {
754 44 : PLF_CONFIG_INFO(PLF_DATA_OP, "[EventWait] eventHandle=%llu, mask=%u", eventHandle, mask);
755 44 : CcuRep::CompletedEvent* event{nullptr};
756 44 : CCU_CHK_RET(GetEventByHandle(eventHandle, &event));
757 : // 复用已有的 WaitEvent 实现（内部 Append CcuRepLocWaitEvent）
758 44 : CCU_CHK_RET(WaitEvent(*event, mask));
759 44 : return CcuResult::CCU_SUCCESS;
760 : }
761 :
762 1 : CcuResult CcuKernel::LocalNotifyRecord(const char* notifyTag, const uint32_t mask)
763 : {
764 1 : PLF_CONFIG_INFO(PLF_DATA_OP, "[LocalNotifyRecord] tag=%s, mask=%u", (notifyTag ? notifyTag : "null"), mask);
765 1 : if (notifyTag == nullptr) {
766 0 : HCCL_ERROR("[CcuKernel][%s] notifyTag is nullptr, please check.", __func__);
767 0 : return CcuResult::CCU_E_PTR;
768 : }
769 1 : if (CurrentBlock()->Type() == CcuRep::CcuRepType::LOOP_BLOCK) {
770 1 : HCCL_ERROR("[CcuKernel][%s] is not supported in loop block, please check.", __func__);
771 1 : return LatchBodyError(CcuResult::CCU_E_NOT_SUPPORT);
772 : }
773 :
774 0 : const std::string tagKey(notifyTag);
775 :
776 0 : auto& sharedNotifies = importedRes_.sharedNotifies;
777 0 : if (sharedNotifies.find(tagKey) == sharedNotifies.end()) {
778 0 : CcuRep::LocalNotify localNotify;
779 0 : sharedNotifies.insert({tagKey, localNotify});
780 0 : }
781 :
782 0 : Append(std::make_shared<CcuRep::CcuRepRecordSharedNotify>(insGenerator, sharedNotifies.at(tagKey), mask));
783 :
784 0 : return CcuResult::CCU_SUCCESS;
785 0 : }
786 :
787 0 : CcuResult CcuKernel::LocalNotifyWait(const char* notifyTag, const uint32_t mask)
788 : {
789 0 : PLF_CONFIG_INFO(PLF_DATA_OP, "[LocalNotifyWait] tag=%s, mask=%u", (notifyTag ? notifyTag : "null"), mask);
790 0 : if (notifyTag == nullptr) {
791 0 : HCCL_ERROR("[CcuKernel][%s] notifyTag is nullptr, please check.", __func__);
792 0 : return CcuResult::CCU_E_PTR;
793 : }
794 :
795 0 : const std::string tagKey(notifyTag);
796 :
797 0 : auto& sharedNotifies = exportedRes_.sharedNotifies;
798 0 : if (sharedNotifies.find(tagKey) == sharedNotifies.end()) {
799 0 : CcuRep::LocalNotify notify = CreateLocalNotify();
800 0 : exportedRes_.sharedNotifies.insert({tagKey, notify});
801 0 : }
802 :
803 0 : bool isProfiling = CurrentBlock()->Type() != CcuRep::CcuRepType::LOOP_BLOCK;
804 0 : Append(std::make_shared<CcuRep::CcuRepLocWaitNotify>(
805 0 : insGenerator, exportedRes_.sharedNotifies.at(tagKey), mask, isProfiling));
806 0 : return CcuResult::CCU_SUCCESS;
807 0 : }
808 :
809 6 : CcuResult CcuKernel::NotifyRecord(const ChannelHandle channel, uint32_t remoteNotifyIdx, uint32_t mask)
810 : {
811 6 : PLF_CONFIG_INFO(
812 : PLF_DATA_OP, "[NotifyRecord] channel=%llu, remoteNotifyIdx=%u, mask=%u", channel, remoteNotifyIdx, mask);
813 6 : if (CurrentBlock()->Type() == CcuRep::CcuRepType::LOOP_BLOCK) {
814 1 : HCCL_ERROR("[%s] NotifyRecord is not allowed inside a ccu::Loop body", __func__);
815 1 : return LatchBodyError(CcuResult::CCU_E_NOT_SUPPORT);
816 : }
817 5 : channels_.insert(channel);
818 5 : Append(std::make_shared<CcuRep::CcuRepRemPostSem>(insGenerator, channel, remoteNotifyIdx, mask));
819 5 : return CCU_SUCCESS;
820 : }
821 :
822 11 : CcuResult CcuKernel::NotifyWait(const ChannelHandle channel, uint32_t localNotifyIdx, uint32_t mask)
823 : {
824 11 : PLF_CONFIG_INFO(
825 : PLF_DATA_OP, "[NotifyWait] channel=%llu, localNotifyIdx=%u, mask=%u", channel, localNotifyIdx, mask);
826 11 : channels_.insert(channel);
827 11 : bool isProfiling = CurrentBlock()->Type() != CcuRep::CcuRepType::LOOP_BLOCK;
828 11 : if (isProfiling) {
829 33 : CCU_CHK_RET(static_cast<HcclResult>(AddProfiling(channel, "NotifyWait", localNotifyIdx, mask)));
830 : }
831 10 : Append(std::make_shared<CcuRep::CcuRepRemWaitSem>(insGenerator, channel, localNotifyIdx, mask, isProfiling));
832 10 : return CcuResult::CCU_SUCCESS;
833 : }
834 :
835 10 : CcuResult CcuKernel::WriteVariableWithNotify(
836 : const ChannelHandle channel, CcuVariableHandle varHandle, uint32_t remoteVarIdx, uint32_t remoteNotifyIdx,
837 : uint32_t mask)
838 : {
839 10 : PLF_CONFIG_INFO(
840 : PLF_DATA_OP,
841 : "[WriteVariableWithNotify] channel=%llu, varHandle=%llu, remoteVarIdx=%u,"
842 : " remoteNotifyIdx=%u, mask=%u",
843 : channel, varHandle, remoteVarIdx, remoteNotifyIdx, mask);
844 10 : if (CurrentBlock()->Type() == CcuRep::CcuRepType::LOOP_BLOCK) {
845 1 : HCCL_ERROR("[%s] WriteVariableWithNotify is not allowed inside a ccu::Loop body", __func__);
846 1 : return LatchBodyError(CcuResult::CCU_E_NOT_SUPPORT);
847 : }
848 9 : channels_.insert(channel);
849 9 : CcuRep::Variable* var{nullptr};
850 9 : CCU_CHK_RET(GetVariableByHandle(varHandle, &var));
851 9 : Append(
852 18 : std::make_shared<CcuRep::CcuRepRemPostVar>(insGenerator, *var, channel, remoteVarIdx, remoteNotifyIdx, mask));
853 9 : return CcuResult::CCU_SUCCESS;
854 : }
855 :
856 : // 加载类 相关接口
857 49 : CcuResult CcuKernel::LoadArg(CcuVariableHandle varHandle, uint32_t argId)
858 : {
859 49 : PLF_CONFIG_INFO(PLF_DATA_OP, "[LoadArg] varHandle=%llu, argId=%u", varHandle, argId);
860 49 : loadArgUsedSet_.insert(argId);
861 49 : CcuRep::Variable* var{nullptr};
862 49 : CCU_CHK_RET(GetVariableByHandle(varHandle, &var));
863 : auto loadArgRep = std::make_shared<CcuRep::CcuRepLoadArg>(
864 49 : insGenerator, *var, argId % CCU_SQE_ARGS_LEN, static_cast<uint16_t>(argId));
865 49 : Append(loadArgRep);
866 49 : return CcuResult::CCU_SUCCESS;
867 49 : }
868 :
869 3 : CcuResult CcuKernel::GetCcuKernelInfo(CcuKernelInfo& info) const
870 : {
871 3 : uint32_t maxTaskArgsNum = 0;
872 : // 如果没有LoadArg指令，则说明不需要传入参数，直接返回0
873 3 : if (loadArgUsedSet_.empty()) {
874 1 : info.maxTaskArgsNum = 0;
875 1 : return CcuResult::CCU_SUCCESS;
876 : }
877 : // 遍历loadArgUsedSet_，获取最大的argId，加1得到最大任务参数数量（argId从0连续）
878 7 : for (const auto& argId : loadArgUsedSet_) {
879 5 : maxTaskArgsNum = std::max(maxTaskArgsNum, argId);
880 : }
881 2 : maxTaskArgsNum += 1;
882 2 : info.maxTaskArgsNum = maxTaskArgsNum;
883 2 : return CcuResult::CCU_SUCCESS;
884 : }
885 :
886 13 : CcuResult CcuKernel::CheckContinuousVariables(
887 : CcuVariableHandle varHandle, uint32_t num, const CcuRep::Variable& baseVar, const char* tag)
888 : {
889 13 : if (num <= 1) {
890 4 : return CcuResult::CCU_SUCCESS;
891 : }
892 18 : for (uint32_t i = 1; i < num; i++) {
893 9 : CcuRep::Variable* nextVar{nullptr};
894 9 : CCU_CHK_RET(GetVariableByHandle(varHandle + i, &nextVar));
895 9 : if (nextVar->Id() != baseVar.Id() + i) {
896 0 : HCCL_ERROR(
897 : "[CcuKernel][%s] variables not continuous at index %u, "
898 : "expected Id %u but got %u",
899 : tag, i, baseVar.Id() + i, nextVar->Id());
900 0 : return HCCL_TO_CCU_RET(HCCL_E_PARA);
901 : }
902 : }
903 9 : return CcuResult::CCU_SUCCESS;
904 : }
905 :
906 6 : CcuResult CcuKernel::LoadVar(uint64_t addr, CcuVariableHandle varHandle, uint32_t num)
907 : {
908 6 : PLF_CONFIG_INFO(PLF_DATA_OP, "[LoadVar] addr=0x%llx, varHandle=%llu, num=%u", addr, varHandle, num);
909 6 : CcuRep::Variable* var{nullptr};
910 6 : CCU_CHK_RET(GetVariableByHandle(varHandle, &var));
911 6 : CCU_CHK_RET(CheckContinuousVariables(varHandle, num, *var, "LoadVariable"));
912 6 : Append(std::make_shared<CcuRep::CcuRepLoad>(insGenerator, addr, *var, num));
913 : // 数据面 num>=2 走 Array<Variable>: microcode 会用到 [baseVReg, baseVReg+num)
914 : // 这段连续 Xn 且组内相对偏移固定, 交给 microcode 后端优化 Pass 2 作为 pinned 组.
915 6 : if (num >= 2) {
916 5 : AddPinnedRegGroup(*var, static_cast<uint16_t>(num));
917 : }
918 6 : return CcuResult::CCU_SUCCESS;
919 : }
920 :
921 2 : CcuResult CcuKernel::CcuLoadVarFromVarAddr(CcuVariableHandle addrHandle, CcuVariableHandle varHandle, uint32_t num)
922 : {
923 2 : PLF_CONFIG_INFO(
924 : PLF_DATA_OP,
925 : "[CcuLoadVarFromVarAddr] addrHandle=%llu, varHandle=%llu,"
926 : " num=%u",
927 : addrHandle, varHandle, num);
928 2 : CcuRep::Variable* addrVar{nullptr};
929 2 : CCU_CHK_RET(GetVariableByHandle(addrHandle, &addrVar));
930 2 : CcuRep::Variable* var{nullptr};
931 2 : CCU_CHK_RET(GetVariableByHandle(varHandle, &var));
932 2 : CCU_CHK_RET(CheckContinuousVariables(varHandle, num, *var, "LoadVar dst"));
933 2 : Append(std::make_shared<CcuRep::CcuRepLoadVar>(insGenerator, *addrVar, *var, num));
934 2 : if (num >= 2) {
935 1 : AddPinnedRegGroup(*var, static_cast<uint16_t>(num));
936 : }
937 2 : return CcuResult::CCU_SUCCESS;
938 : }
939 :
940 3 : CcuResult CcuKernel::StoreVar(uint64_t addr, CcuVariableHandle varHandle, uint32_t num)
941 : {
942 3 : PLF_CONFIG_INFO(PLF_DATA_OP, "[StoreVar] addr=0x%llx, varHandle=%llu, num=%u", addr, varHandle, num);
943 3 : CcuRep::Variable* var{nullptr};
944 3 : CCU_CHK_RET(GetVariableByHandle(varHandle, &var));
945 3 : CCU_CHK_RET(CheckContinuousVariables(varHandle, num, *var, "StoreVariable"));
946 3 : Append(std::make_shared<CcuRep::CcuRepStore>(insGenerator, *var, addr, num));
947 3 : if (num >= 2) {
948 2 : AddPinnedRegGroup(*var, static_cast<uint16_t>(num));
949 : }
950 3 : return CcuResult::CCU_SUCCESS;
951 : }
952 :
953 2 : CcuResult CcuKernel::CcuStoreVarToVarAddr(CcuVariableHandle addrHandle, CcuVariableHandle varHandle, uint32_t num)
954 : {
955 2 : PLF_CONFIG_INFO(
956 : PLF_DATA_OP,
957 : "[CcuStoreVarToVarAddr] addrHandle=%llu, varHandle=%llu,"
958 : " num=%u",
959 : addrHandle, varHandle, num);
960 2 : CcuRep::Variable* addrVar{nullptr};
961 2 : CCU_CHK_RET(GetVariableByHandle(addrHandle, &addrVar));
962 2 : CcuRep::Variable* var{nullptr};
963 2 : CCU_CHK_RET(GetVariableByHandle(varHandle, &var));
964 2 : CCU_CHK_RET(CheckContinuousVariables(varHandle, num, *var, "StoreVar src"));
965 2 : Append(std::make_shared<CcuRep::CcuRepStoreVar>(insGenerator, *var, *addrVar, num));
966 2 : if (num >= 2) {
967 1 : AddPinnedRegGroup(*var, static_cast<uint16_t>(num));
968 : }
969 2 : return CcuResult::CCU_SUCCESS;
970 : }
971 :
972 : // 本地数据拷贝 相关实现
973 12 : CcuResult CcuKernel::LocalCopyMemToBuffer(
974 : CcuBufferHandle dstHandle, CcuLocalAddrHandle srcHandle, CcuVariableHandle lenHandle, CcuEventHandle eventHandle,
975 : uint32_t mask)
976 : {
977 12 : PLF_CONFIG_INFO(
978 : PLF_DATA_OP,
979 : "[LocalCopyMemToBuffer] dstHandle=%llu, srcHandle=%llu, lenHandle=%llu,"
980 : " eventHandle=%llu, mask=%u",
981 : dstHandle, srcHandle, lenHandle, eventHandle, mask);
982 12 : CcuRep::CcuBuf* dst{nullptr};
983 12 : CCU_CHK_RET(GetBufferByHandle(dstHandle, &dst));
984 12 : CcuRep::LocalAddr* src{nullptr};
985 12 : CCU_CHK_RET(GetLocalAddrByHandle(srcHandle, &src));
986 12 : CcuRep::Variable* len{nullptr};
987 12 : CCU_CHK_RET(GetVariableByHandle(lenHandle, &len));
988 12 : CcuRep::CompletedEvent* event{nullptr};
989 12 : CCU_CHK_RET(GetEventByHandle(eventHandle, &event));
990 12 : auto ret = LocalCopyNb(*dst, *src, *len, *event, mask); // 复用 protected
991 12 : return HCCL_TO_CCU_RET(ret);
992 : }
993 :
994 8 : CcuResult CcuKernel::LocalCopyBufferToMem(
995 : CcuLocalAddrHandle dstHandle, CcuBufferHandle srcHandle, CcuVariableHandle lenHandle, CcuEventHandle eventHandle,
996 : uint32_t mask)
997 : {
998 8 : PLF_CONFIG_INFO(
999 : PLF_DATA_OP,
1000 : "[LocalCopyBufferToMem] dstHandle=%llu, srcHandle=%llu, lenHandle=%llu,"
1001 : " eventHandle=%llu, mask=%u",
1002 : dstHandle, srcHandle, lenHandle, eventHandle, mask);
1003 8 : CcuRep::LocalAddr* dst{nullptr};
1004 8 : CCU_CHK_RET(GetLocalAddrByHandle(dstHandle, &dst));
1005 8 : CcuRep::CcuBuf* src{nullptr};
1006 8 : CCU_CHK_RET(GetBufferByHandle(srcHandle, &src));
1007 8 : CcuRep::Variable* len{nullptr};
1008 8 : CCU_CHK_RET(GetVariableByHandle(lenHandle, &len));
1009 8 : CcuRep::CompletedEvent* event{nullptr};
1010 8 : CCU_CHK_RET(GetEventByHandle(eventHandle, &event));
1011 8 : auto ret = LocalCopyNb(*dst, *src, *len, *event, mask);
1012 8 : return HCCL_TO_CCU_RET(ret);
1013 : }
1014 :
1015 1 : CcuResult CcuKernel::LocalCopyMemToMem(
1016 : CcuLocalAddrHandle dstHandle, CcuLocalAddrHandle srcHandle, CcuVariableHandle lenHandle, CcuEventHandle eventHandle,
1017 : uint32_t mask)
1018 : {
1019 1 : PLF_CONFIG_INFO(
1020 : PLF_DATA_OP,
1021 : "[LocalCopyMemToMem] dstHandle=%llu, srcHandle=%llu, lenHandle=%llu,"
1022 : " eventHandle=%llu, mask=%u",
1023 : dstHandle, srcHandle, lenHandle, eventHandle, mask);
1024 1 : CcuRep::LocalAddr* dst{nullptr};
1025 1 : CCU_CHK_RET(GetLocalAddrByHandle(dstHandle, &dst));
1026 1 : CcuRep::LocalAddr* src{nullptr};
1027 1 : CCU_CHK_RET(GetLocalAddrByHandle(srcHandle, &src));
1028 1 : CcuRep::Variable* len{nullptr};
1029 1 : CCU_CHK_RET(GetVariableByHandle(lenHandle, &len));
1030 1 : CcuRep::CompletedEvent* event{nullptr};
1031 1 : CCU_CHK_RET(GetEventByHandle(eventHandle, &event));
1032 1 : auto ret = LocalCopyNb(*dst, *src, *len, *event, mask);
1033 1 : return HCCL_TO_CCU_RET(ret);
1034 : }
1035 :
1036 : // 本地reduce 相关实现
1037 0 : CcuResult CcuKernel::LocalMemReduce(
1038 : CcuLocalAddrHandle dstHandle, CcuLocalAddrHandle srcHandle, CcuVariableHandle lenHandle, HcclDataType dataType,
1039 : HcclReduceOp opType, CcuEventHandle eventHandle, uint32_t mask)
1040 : {
1041 0 : PLF_CONFIG_INFO(
1042 : PLF_DATA_OP,
1043 : "[LocalMemReduce] dstHandle=%llu, srcHandle=%llu, lenHandle=%llu, dataType=%d,"
1044 : " op=%d, eventHandle=%llu, mask=%u",
1045 : dstHandle, srcHandle, lenHandle, dataType, opType, eventHandle, mask);
1046 0 : CcuRep::LocalAddr* dst{nullptr};
1047 0 : CCU_CHK_RET(GetLocalAddrByHandle(dstHandle, &dst));
1048 0 : CcuRep::LocalAddr* src{nullptr};
1049 0 : CCU_CHK_RET(GetLocalAddrByHandle(srcHandle, &src));
1050 0 : CcuRep::Variable* len{nullptr};
1051 0 : CCU_CHK_RET(GetVariableByHandle(lenHandle, &len));
1052 0 : CcuRep::CompletedEvent* event{nullptr};
1053 0 : CCU_CHK_RET(GetEventByHandle(eventHandle, &event));
1054 0 : auto ret = LocalReduceNb(*dst, *src, *len, dataType, opType, *event, mask);
1055 0 : return HCCL_TO_CCU_RET(ret);
1056 : }
1057 :
1058 4 : CcuResult CcuKernel::LocalBufferReduce(
1059 : CcuBufferHandle* bufHandles, uint32_t count, HcclDataType dataType, HcclDataType outputDataType,
1060 : HcclReduceOp opType, CcuVariableHandle lenHandle, CcuEventHandle eventHandle, uint32_t mask)
1061 : {
1062 4 : PLF_CONFIG_INFO(
1063 : PLF_DATA_OP,
1064 : "[LocalBufferReduce] count=%u, dataType=%d, outDataType=%d,"
1065 : " op=%d, lenHandle=%llu, eventHandle=%llu, mask=%u",
1066 : count, dataType, outputDataType, opType, lenHandle, eventHandle, mask);
1067 4 : CcuRep::Variable* len{nullptr};
1068 4 : CCU_CHK_RET(GetVariableByHandle(lenHandle, &len));
1069 4 : CcuRep::CompletedEvent* event{nullptr};
1070 4 : CCU_CHK_RET(GetEventByHandle(eventHandle, &event));
1071 4 : std::vector<CcuRep::CcuBuf> bufs(count);
1072 12 : for (uint32_t i = 0; i < count; i++) {
1073 8 : CcuRep::CcuBuf* buf{nullptr};
1074 8 : CCU_CHK_RET(GetBufferByHandle(bufHandles[i], &buf));
1075 8 : bufs[i] = *buf;
1076 : }
1077 4 : auto ret = LocalReduceNb(bufs.data(), count, dataType, outputDataType, opType, *len, *event, mask);
1078 4 : return HCCL_TO_CCU_RET(ret);
1079 4 : }
1080 :
1081 : /*========== 远端数据传输操作 ==========*/
1082 :
1083 4 : CcuResult CcuKernel::ResolveBufRemoteLenEvent(
1084 : CcuBufferHandle bufHandle, CcuRemoteAddrHandle remoteHandle, CcuVariableHandle lenHandle,
1085 : CcuEventHandle eventHandle, CcuRep::CcuBuf** buf, CcuRep::RemoteAddr** remote, CcuRep::Variable** len,
1086 : CcuRep::CompletedEvent** event)
1087 : {
1088 4 : CCU_CHK_RET(GetBufferByHandle(bufHandle, buf));
1089 4 : CCU_CHK_RET(GetRemoteAddrByHandle(remoteHandle, remote));
1090 4 : CCU_CHK_RET(GetVariableByHandle(lenHandle, len));
1091 4 : CCU_CHK_RET(GetEventByHandle(eventHandle, event));
1092 4 : return CcuResult::CCU_SUCCESS;
1093 : }
1094 :
1095 6 : CcuResult CcuKernel::ResolveLocalRemoteLenEvent(
1096 : CcuLocalAddrHandle localHandle, CcuRemoteAddrHandle remoteHandle, CcuVariableHandle lenHandle,
1097 : CcuEventHandle eventHandle, CcuRep::LocalAddr** local, CcuRep::RemoteAddr** remote, CcuRep::Variable** len,
1098 : CcuRep::CompletedEvent** event)
1099 : {
1100 6 : CCU_CHK_RET(GetLocalAddrByHandle(localHandle, local));
1101 6 : CCU_CHK_RET(GetRemoteAddrByHandle(remoteHandle, remote));
1102 6 : CCU_CHK_RET(GetVariableByHandle(lenHandle, len));
1103 6 : CCU_CHK_RET(GetEventByHandle(eventHandle, event));
1104 6 : return CcuResult::CCU_SUCCESS;
1105 : }
1106 :
1107 5 : CcuResult CcuKernel::ResolveRemoteLocalLenEvent(
1108 : CcuRemoteAddrHandle remoteHandle, CcuLocalAddrHandle localHandle, CcuVariableHandle lenHandle,
1109 : CcuEventHandle eventHandle, CcuRep::RemoteAddr** remote, CcuRep::LocalAddr** local, CcuRep::Variable** len,
1110 : CcuRep::CompletedEvent** event)
1111 : {
1112 5 : CCU_CHK_RET(GetRemoteAddrByHandle(remoteHandle, remote));
1113 5 : CCU_CHK_RET(GetLocalAddrByHandle(localHandle, local));
1114 5 : CCU_CHK_RET(GetVariableByHandle(lenHandle, len));
1115 5 : CCU_CHK_RET(GetEventByHandle(eventHandle, event));
1116 5 : return CcuResult::CCU_SUCCESS;
1117 : }
1118 :
1119 4 : CcuResult CcuKernel::ReadMemToMem(
1120 : ChannelHandle channel, CcuLocalAddrHandle localHandle, CcuRemoteAddrHandle remoteHandle,
1121 : CcuVariableHandle lenHandle, CcuEventHandle eventHandle, uint32_t mask)
1122 : {
1123 4 : PLF_CONFIG_INFO(
1124 : PLF_DATA_OP,
1125 : "[ReadMemToMem] channel=%llu, localHandle=%llu, remoteHandle=%llu,"
1126 : " lenHandle=%llu, eventHandle=%llu, mask=%u",
1127 : channel, localHandle, remoteHandle, lenHandle, eventHandle, mask);
1128 4 : channels_.insert(channel);
1129 4 : CcuRep::LocalAddr* local{nullptr};
1130 4 : CcuRep::RemoteAddr* remote{nullptr};
1131 4 : CcuRep::Variable* len{nullptr};
1132 4 : CcuRep::CompletedEvent* event{nullptr};
1133 4 : CCU_CHK_RET(
1134 : ResolveLocalRemoteLenEvent(localHandle, remoteHandle, lenHandle, eventHandle, &local, &remote, &len, &event));
1135 4 : auto ret = ReadNb(channel, *local, *remote, *len, *event, mask);
1136 4 : return HCCL_TO_CCU_RET(ret);
1137 : }
1138 :
1139 2 : CcuResult CcuKernel::ReadMemToBuffer(
1140 : ChannelHandle channel, CcuBufferHandle localHandle, CcuRemoteAddrHandle remoteHandle, CcuVariableHandle lenHandle,
1141 : CcuEventHandle eventHandle, uint32_t mask)
1142 : {
1143 2 : PLF_CONFIG_INFO(
1144 : PLF_DATA_OP,
1145 : "[ReadMemToBuffer] channel=%llu, bufHandle=%llu, remoteHandle=%llu,"
1146 : " lenHandle=%llu, eventHandle=%llu, mask=%u",
1147 : channel, localHandle, remoteHandle, lenHandle, eventHandle, mask);
1148 2 : channels_.insert(channel);
1149 2 : CcuRep::CcuBuf* local{nullptr};
1150 2 : CcuRep::RemoteAddr* remote{nullptr};
1151 2 : CcuRep::Variable* len{nullptr};
1152 2 : CcuRep::CompletedEvent* event{nullptr};
1153 2 : CCU_CHK_RET(
1154 : ResolveBufRemoteLenEvent(localHandle, remoteHandle, lenHandle, eventHandle, &local, &remote, &len, &event));
1155 2 : auto ret = ReadNb(channel, *local, *remote, *len, *event, mask);
1156 2 : return HCCL_TO_CCU_RET(ret);
1157 : }
1158 :
1159 2 : CcuResult CcuKernel::ReadMemToMemReduce(
1160 : ChannelHandle channel, CcuLocalAddrHandle localHandle, CcuRemoteAddrHandle remoteHandle,
1161 : CcuVariableHandle lenHandle, HcclDataType dataType, HcclReduceOp opType, CcuEventHandle eventHandle, uint32_t mask)
1162 : {
1163 2 : PLF_CONFIG_INFO(
1164 : PLF_DATA_OP,
1165 : "[ReadMemToMemReduce] channel=%llu, lenHandle=%llu, dataType=%d, op=%d,"
1166 : " eventHandle=%llu, mask=%u",
1167 : channel, lenHandle, dataType, opType, eventHandle, mask);
1168 2 : channels_.insert(channel);
1169 2 : CcuRep::LocalAddr* local{nullptr};
1170 2 : CcuRep::RemoteAddr* remote{nullptr};
1171 2 : CcuRep::Variable* len{nullptr};
1172 2 : CcuRep::CompletedEvent* event{nullptr};
1173 2 : CCU_CHK_RET(
1174 : ResolveLocalRemoteLenEvent(localHandle, remoteHandle, lenHandle, eventHandle, &local, &remote, &len, &event));
1175 2 : auto ret = ReadReduceNb(channel, *local, *remote, *len, dataType, opType, *event, mask);
1176 2 : return HCCL_TO_CCU_RET(ret);
1177 : }
1178 :
1179 3 : CcuResult CcuKernel::WriteMemToMem(
1180 : ChannelHandle channel, CcuRemoteAddrHandle remoteHandle, CcuLocalAddrHandle localHandle,
1181 : CcuVariableHandle lenHandle, CcuEventHandle eventHandle, uint32_t mask)
1182 : {
1183 3 : PLF_CONFIG_INFO(
1184 : PLF_DATA_OP,
1185 : "[WriteMemToMem] channel=%llu, remoteHandle=%llu, localHandle=%llu,"
1186 : " lenHandle=%llu, eventHandle=%llu, mask=%u",
1187 : channel, remoteHandle, localHandle, lenHandle, eventHandle, mask);
1188 3 : channels_.insert(channel);
1189 3 : CcuRep::RemoteAddr* remote{nullptr};
1190 3 : CcuRep::LocalAddr* local{nullptr};
1191 3 : CcuRep::Variable* len{nullptr};
1192 3 : CcuRep::CompletedEvent* event{nullptr};
1193 3 : CCU_CHK_RET(
1194 : ResolveRemoteLocalLenEvent(remoteHandle, localHandle, lenHandle, eventHandle, &remote, &local, &len, &event));
1195 3 : auto ret = WriteNb(channel, *remote, *local, *len, *event, mask);
1196 3 : return HCCL_TO_CCU_RET(ret);
1197 : }
1198 :
1199 2 : CcuResult CcuKernel::WriteBufferToMem(
1200 : ChannelHandle channel, CcuRemoteAddrHandle remoteHandle, CcuBufferHandle localHandle, CcuVariableHandle lenHandle,
1201 : CcuEventHandle eventHandle, uint32_t mask)
1202 : {
1203 2 : PLF_CONFIG_INFO(
1204 : PLF_DATA_OP,
1205 : "[WriteBufferToMem] channel=%llu, remoteHandle=%llu, bufHandle=%llu,"
1206 : " lenHandle=%llu, eventHandle=%llu, mask=%u",
1207 : channel, remoteHandle, localHandle, lenHandle, eventHandle, mask);
1208 2 : channels_.insert(channel);
1209 2 : CcuRep::CcuBuf* local{nullptr};
1210 2 : CcuRep::RemoteAddr* remote{nullptr};
1211 2 : CcuRep::Variable* len{nullptr};
1212 2 : CcuRep::CompletedEvent* event{nullptr};
1213 2 : CCU_CHK_RET(
1214 : ResolveBufRemoteLenEvent(localHandle, remoteHandle, lenHandle, eventHandle, &local, &remote, &len, &event));
1215 2 : auto ret = WriteNb(channel, *remote, *local, *len, *event, mask);
1216 2 : return HCCL_TO_CCU_RET(ret);
1217 : }
1218 :
1219 2 : CcuResult CcuKernel::WriteMemToMemReduce(
1220 : ChannelHandle channel, CcuRemoteAddrHandle remoteHandle, CcuLocalAddrHandle localHandle,
1221 : CcuVariableHandle lenHandle, HcclDataType dataType, HcclReduceOp opType, CcuEventHandle eventHandle, uint32_t mask)
1222 : {
1223 2 : PLF_CONFIG_INFO(
1224 : PLF_DATA_OP,
1225 : "[WriteMemToMemReduce] channel=%llu, lenHandle=%llu, dataType=%d, op=%d,"
1226 : " eventHandle=%llu, mask=%u",
1227 : channel, lenHandle, dataType, opType, eventHandle, mask);
1228 2 : channels_.insert(channel);
1229 2 : CcuRep::RemoteAddr* remote{nullptr};
1230 2 : CcuRep::LocalAddr* local{nullptr};
1231 2 : CcuRep::Variable* len{nullptr};
1232 2 : CcuRep::CompletedEvent* event{nullptr};
1233 2 : CCU_CHK_RET(
1234 : ResolveRemoteLocalLenEvent(remoteHandle, localHandle, lenHandle, eventHandle, &remote, &local, &len, &event));
1235 2 : auto ret = WriteReduceNb(channel, *remote, *local, *len, dataType, opType, *event, mask);
1236 2 : return HCCL_TO_CCU_RET(ret);
1237 : }
1238 :
1239 1509 : void CcuKernel::FlushClosablePendingIfs()
1240 : {
1241 1509 : if (isFlushing_) {
1242 82 : return;
1243 : }
1244 1427 : isFlushing_ = true;
1245 2936 : while (IfLabelStackTopIsClosable()) {
1246 82 : const char* lbl = IfLabelStackPop();
1247 82 : if (lbl != nullptr) {
1248 82 : IfEnd(lbl);
1249 : }
1250 : }
1251 1427 : isFlushing_ = false;
1252 : }
1253 :
1254 1241 : void CcuKernel::Append(std::shared_ptr<CcuRep::CcuRepBase> rep)
1255 : {
1256 1241 : FlushClosablePendingIfs();
1257 1241 : CcuRep::CcuRepContext::Append(rep);
1258 1241 : }
1259 :
1260 : namespace {
1261 104 : std::shared_ptr<CcuRep::CcuRepJumpBase> MakeInvertedCondJumpImm(
1262 : CcuInsGeneratorBase* insGenerator, const std::string& destLabelStr, const CcuRep::Variable& targetVar,
1263 : const CcuRep::Variable& expectVar, const CcuRep::Variable& variable, uint64_t immediate,
1264 : CcuConditionType condType, const char* funcName)
1265 : {
1266 104 : switch (condType) {
1267 82 : case CCU_CONDITION_EQ:
1268 164 : return std::make_shared<CcuRep::CcuRepJumpNE>(
1269 82 : insGenerator, destLabelStr, targetVar, expectVar, variable, immediate);
1270 12 : case CCU_CONDITION_NE:
1271 24 : return std::make_shared<CcuRep::CcuRepJumpEQ>(
1272 12 : insGenerator, destLabelStr, targetVar, expectVar, variable, immediate);
1273 3 : case CCU_CONDITION_LT:
1274 6 : return std::make_shared<CcuRep::CcuRepJumpGE>(
1275 3 : insGenerator, destLabelStr, targetVar, expectVar, variable, immediate);
1276 2 : case CCU_CONDITION_LE:
1277 4 : return std::make_shared<CcuRep::CcuRepJumpGT>(
1278 2 : insGenerator, destLabelStr, targetVar, expectVar, variable, immediate);
1279 2 : case CCU_CONDITION_GT:
1280 4 : return std::make_shared<CcuRep::CcuRepJumpLE>(
1281 2 : insGenerator, destLabelStr, targetVar, expectVar, variable, immediate);
1282 3 : case CCU_CONDITION_GE:
1283 6 : return std::make_shared<CcuRep::CcuRepJumpLT>(
1284 3 : insGenerator, destLabelStr, targetVar, expectVar, variable, immediate);
1285 0 : default:
1286 0 : HCCL_ERROR("[%s] unsupported condition type: %d", funcName, condType);
1287 0 : return nullptr;
1288 : }
1289 : }
1290 :
1291 : // 双变量版本：当 (lhsVar OP rhsVar) 为假时跳转到 destLabelStr。
1292 4 : std::shared_ptr<CcuRep::CcuRepJumpBase> MakeInvertedCondJumpVar(
1293 : CcuInsGeneratorBase* insGenerator, const std::string& destLabelStr, const CcuRep::Variable& targetVar,
1294 : const CcuRep::Variable& lhsVar, const CcuRep::Variable& rhsVar, CcuConditionType condType, const char* funcName)
1295 : {
1296 4 : switch (condType) {
1297 1 : case CCU_CONDITION_EQ:
1298 1 : return std::make_shared<CcuRep::CcuRepJumpNE>(insGenerator, destLabelStr, targetVar, lhsVar, rhsVar);
1299 0 : case CCU_CONDITION_NE:
1300 0 : return std::make_shared<CcuRep::CcuRepJumpEQ>(insGenerator, destLabelStr, targetVar, lhsVar, rhsVar);
1301 2 : case CCU_CONDITION_LT:
1302 2 : return std::make_shared<CcuRep::CcuRepJumpGE>(insGenerator, destLabelStr, targetVar, lhsVar, rhsVar);
1303 0 : case CCU_CONDITION_LE:
1304 0 : return std::make_shared<CcuRep::CcuRepJumpGT>(insGenerator, destLabelStr, targetVar, lhsVar, rhsVar);
1305 0 : case CCU_CONDITION_GT:
1306 0 : return std::make_shared<CcuRep::CcuRepJumpLE>(insGenerator, destLabelStr, targetVar, lhsVar, rhsVar);
1307 1 : case CCU_CONDITION_GE:
1308 1 : return std::make_shared<CcuRep::CcuRepJumpLT>(insGenerator, destLabelStr, targetVar, lhsVar, rhsVar);
1309 0 : default:
1310 0 : HCCL_ERROR("[%s] unsupported condition type: %d", funcName, condType);
1311 0 : return nullptr;
1312 : }
1313 : }
1314 : } // namespace
1315 :
1316 : CcuResult
1317 102 : CcuKernel::IfBegin(CcuVariableHandle varHandle, uint64_t immediate, CcuConditionType condType, const char* label)
1318 : {
1319 102 : PLF_CONFIG_INFO(
1320 : PLF_DATA_OP, "[IfBegin] varHandle=%llu, immediate=%llu, condType=%d, label=%s", varHandle, immediate, condType,
1321 : (label ? label : "null"));
1322 102 : if (CurrentBlock()->Type() == CcuRep::CcuRepType::LOOP_BLOCK) {
1323 1 : HCCL_ERROR(
1324 : "[%s] CCU_IF is not allowed inside a ccu::Loop body (label='%s')", __func__,
1325 : label != nullptr ? label : "(null)");
1326 1 : return LatchBodyError(CcuResult::CCU_E_INTERNAL);
1327 : }
1328 101 : CcuRep::Variable* variable{nullptr};
1329 101 : CCU_CHK_RET(GetVariableByHandle(varHandle, &variable));
1330 :
1331 101 : FlushClosablePendingIfs();
1332 :
1333 101 : std::string labelStr(label);
1334 101 : if (pendingIfCtx_.find(labelStr) != pendingIfCtx_.end()) {
1335 0 : HCCL_ERROR("[%s] label '%s' already has a pending IfBegin without IfEnd", __func__, label);
1336 0 : return CcuResult::CCU_E_PARA;
1337 : }
1338 :
1339 101 : std::string elseLabelStr = labelStr + "_else";
1340 101 : std::string endLabelStr = labelStr + "_end";
1341 101 : auto elseLabel = std::make_shared<CcuRep::CcuRepJumpLabel>(insGenerator, elseLabelStr);
1342 101 : auto endLabel = std::make_shared<CcuRep::CcuRepJumpLabel>(insGenerator, endLabelStr);
1343 101 : auto targetVar = CreateJumpTargetVar();
1344 101 : auto expectVar = CreateExpectVar();
1345 :
1346 : // 反转条件："if <cond>, 执行块" 等价于 "!<cond> 时跳过块"。
1347 : auto jump = MakeInvertedCondJumpImm(
1348 101 : insGenerator, elseLabelStr, targetVar, expectVar, *variable, immediate, condType, __func__);
1349 101 : if (jump == nullptr) {
1350 0 : return CcuResult::CCU_E_PARA;
1351 : }
1352 101 : jump->Reference(elseLabel);
1353 101 : Append(jump);
1354 :
1355 101 : PendingIfContext ctx;
1356 101 : ctx.elseLabel = elseLabel;
1357 101 : ctx.endLabel = endLabel;
1358 101 : ctx.hasElse = false;
1359 101 : pendingIfCtx_.emplace(labelStr, std::move(ctx));
1360 :
1361 101 : return CcuResult::CCU_SUCCESS;
1362 101 : }
1363 :
1364 3 : CcuResult CcuKernel::IfBeginVar(
1365 : CcuVariableHandle lhsHandle, CcuVariableHandle rhsHandle, CcuConditionType condType, const char* label)
1366 : {
1367 3 : if (CurrentBlock()->Type() == CcuRep::CcuRepType::LOOP_BLOCK) {
1368 0 : HCCL_ERROR(
1369 : "[%s] CCU_IF is not allowed inside a ccu::Loop body (label='%s')", __func__,
1370 : label != nullptr ? label : "(null)");
1371 0 : return LatchBodyError(CcuResult::CCU_E_INTERNAL);
1372 : }
1373 3 : CcuRep::Variable* lhsVar{nullptr};
1374 3 : CcuRep::Variable* rhsVar{nullptr};
1375 3 : CCU_CHK_RET(GetVariableByHandle(lhsHandle, &lhsVar));
1376 3 : CCU_CHK_RET(GetVariableByHandle(rhsHandle, &rhsVar));
1377 :
1378 3 : FlushClosablePendingIfs();
1379 :
1380 3 : std::string labelStr(label);
1381 3 : if (pendingIfCtx_.find(labelStr) != pendingIfCtx_.end()) {
1382 0 : HCCL_ERROR("[%s] label '%s' already has a pending IfBegin without IfEnd", __func__, label);
1383 0 : return CcuResult::CCU_E_PARA;
1384 : }
1385 :
1386 3 : std::string endLabelStr = labelStr + "_end";
1387 3 : std::string elseLabelStr = labelStr + "_else";
1388 3 : auto endLabel = std::make_shared<CcuRep::CcuRepJumpLabel>(insGenerator, endLabelStr);
1389 3 : auto elseLabel = std::make_shared<CcuRep::CcuRepJumpLabel>(insGenerator, elseLabelStr);
1390 3 : auto targetVar = CreateJumpTargetVar();
1391 :
1392 3 : auto jump = MakeInvertedCondJumpVar(insGenerator, elseLabelStr, targetVar, *lhsVar, *rhsVar, condType, __func__);
1393 3 : if (jump == nullptr) {
1394 0 : return CcuResult::CCU_E_PARA;
1395 : }
1396 3 : jump->Reference(elseLabel);
1397 3 : Append(jump);
1398 :
1399 3 : PendingIfContext ctx;
1400 3 : ctx.hasElse = false;
1401 3 : ctx.endLabel = endLabel;
1402 3 : ctx.elseLabel = elseLabel;
1403 3 : pendingIfCtx_.emplace(labelStr, std::move(ctx));
1404 :
1405 3 : return CcuResult::CCU_SUCCESS;
1406 3 : }
1407 :
1408 22 : CcuResult CcuKernel::IfElse(const char* label)
1409 : {
1410 22 : PLF_CONFIG_INFO(PLF_DATA_OP, "[IfElse] label=%s", (label ? label : "null"));
1411 22 : if (CurrentBlock()->Type() == CcuRep::CcuRepType::LOOP_BLOCK) {
1412 0 : HCCL_ERROR(
1413 : "[%s] CCU_ELSE is not allowed inside a ccu::Loop body (label='%s')", __func__,
1414 : label != nullptr ? label : "(null)");
1415 0 : return LatchBodyError(CcuResult::CCU_E_INTERNAL);
1416 : }
1417 :
1418 22 : std::string labelStr(label);
1419 22 : auto iter = pendingIfCtx_.find(labelStr);
1420 22 : if (iter == pendingIfCtx_.end()) {
1421 0 : HCCL_ERROR("[%s] no matching IfBegin for label '%s'", __func__, label);
1422 0 : return CcuResult::CCU_E_NOT_FOUND;
1423 : }
1424 :
1425 22 : if (iter->second.hasElse) {
1426 0 : HCCL_ERROR("[%s] label '%s' already has an IfElse", __func__, label);
1427 0 : return CcuResult::CCU_E_PARA;
1428 : }
1429 :
1430 : // At end of then-block: unconditional jump past else-block to endLabel
1431 22 : std::string endLabelStr = labelStr + "_end";
1432 22 : auto skipElseVar = CreateJumpTargetVar();
1433 22 : auto skipElseJump = std::make_shared<CcuRep::CcuRepJump>(insGenerator, endLabelStr, skipElseVar);
1434 22 : skipElseJump->Reference(iter->second.endLabel);
1435 22 : Append(skipElseJump);
1436 :
1437 : // Place the else label (entry point of else-block)
1438 22 : Append(iter->second.elseLabel);
1439 :
1440 22 : iter->second.hasElse = true;
1441 :
1442 22 : return CcuResult::CCU_SUCCESS;
1443 22 : }
1444 :
1445 104 : CcuResult CcuKernel::IfEnd(const char* label)
1446 : {
1447 104 : PLF_CONFIG_INFO(PLF_DATA_OP, "[IfEnd] label=%s", (label ? label : "null"));
1448 104 : std::string labelStr(label);
1449 104 : auto iter = pendingIfCtx_.find(labelStr);
1450 104 : if (iter == pendingIfCtx_.end()) {
1451 0 : HCCL_ERROR("[%s] no matching IfBegin for label '%s'", __func__, label);
1452 0 : return CcuResult::CCU_E_NOT_FOUND;
1453 : }
1454 :
1455 104 : if (iter->second.hasElse) {
1456 : // Had else-block: place endLabel after else-block
1457 22 : Append(iter->second.endLabel);
1458 : } else {
1459 : // No else-block: place elseLabel as the skip target
1460 82 : Append(iter->second.elseLabel);
1461 : }
1462 :
1463 104 : pendingIfCtx_.erase(iter);
1464 :
1465 104 : return CcuResult::CCU_SUCCESS;
1466 104 : }
1467 :
1468 : CcuResult
1469 3 : CcuKernel::WhileBegin(CcuVariableHandle varHandle, uint64_t immediate, CcuConditionType condType, const char* label)
1470 : {
1471 3 : PLF_CONFIG_INFO(
1472 : PLF_DATA_OP, "[WhileBegin] varHandle=%llu, immediate=%llu, condType=%d, label=%s", varHandle, immediate,
1473 : condType, (label ? label : "null"));
1474 3 : if (CurrentBlock()->Type() == CcuRep::CcuRepType::LOOP_BLOCK) {
1475 0 : HCCL_ERROR(
1476 : "[%s] CCU_WHILE is not allowed inside a ccu::Loop body (label='%s')", __func__,
1477 : label != nullptr ? label : "(null)");
1478 0 : return LatchBodyError(CcuResult::CCU_E_INTERNAL);
1479 : }
1480 :
1481 3 : CcuRep::Variable* variable{nullptr};
1482 3 : CCU_CHK_RET(GetVariableByHandle(varHandle, &variable));
1483 :
1484 3 : std::string labelStr(label);
1485 3 : if (pendingWhileCtx_.find(labelStr) != pendingWhileCtx_.end()) {
1486 0 : HCCL_ERROR("[%s] label '%s' already has a pending WhileBegin without WhileEnd", __func__, label);
1487 0 : return CcuResult::CCU_E_PARA;
1488 : }
1489 :
1490 3 : std::string beginLabelStr = labelStr + "_begin";
1491 3 : std::string endLabelStr = labelStr + "_end";
1492 3 : auto beginLabel = std::make_shared<CcuRep::CcuRepJumpLabel>(insGenerator, beginLabelStr);
1493 3 : auto endLabel = std::make_shared<CcuRep::CcuRepJumpLabel>(insGenerator, endLabelStr);
1494 :
1495 3 : Append(beginLabel);
1496 :
1497 3 : auto targetVar = CcuRep::CreateVariable(this);
1498 3 : auto expectVar = CreateExpectVar();
1499 : auto jump = MakeInvertedCondJumpImm(
1500 3 : insGenerator, endLabelStr, targetVar, expectVar, *variable, immediate, condType, __func__);
1501 3 : if (jump == nullptr) {
1502 0 : return CcuResult::CCU_E_PARA;
1503 : }
1504 3 : jump->Reference(endLabel);
1505 3 : Append(jump);
1506 :
1507 3 : PendingWhileContext ctx;
1508 3 : ctx.beginLabel = beginLabel;
1509 3 : ctx.endLabel = endLabel;
1510 3 : ctx.varHandle = varHandle;
1511 3 : ctx.immediate = immediate;
1512 3 : ctx.condType = condType;
1513 3 : pendingWhileCtx_.emplace(labelStr, std::move(ctx));
1514 :
1515 3 : return CcuResult::CCU_SUCCESS;
1516 3 : }
1517 :
1518 1 : CcuResult CcuKernel::WhileBeginVar(
1519 : CcuVariableHandle lhsHandle, CcuVariableHandle rhsHandle, CcuConditionType condType, const char* label)
1520 : {
1521 1 : if (CurrentBlock()->Type() == CcuRep::CcuRepType::LOOP_BLOCK) {
1522 0 : HCCL_ERROR(
1523 : "[%s] CCU_WHILE is not allowed inside a ccu::Loop body (label='%s')", __func__,
1524 : label != nullptr ? label : "(null)");
1525 0 : return LatchBodyError(CcuResult::CCU_E_INTERNAL);
1526 : }
1527 :
1528 1 : CcuRep::Variable* lhsVar{nullptr};
1529 1 : CcuRep::Variable* rhsVar{nullptr};
1530 1 : CCU_CHK_RET(GetVariableByHandle(lhsHandle, &lhsVar));
1531 1 : CCU_CHK_RET(GetVariableByHandle(rhsHandle, &rhsVar));
1532 :
1533 1 : std::string labelStr(label);
1534 1 : if (pendingWhileCtx_.find(labelStr) != pendingWhileCtx_.end()) {
1535 0 : HCCL_ERROR("[%s] label '%s' already has a pending WhileBegin without WhileEnd", __func__, label);
1536 0 : return CcuResult::CCU_E_PARA;
1537 : }
1538 :
1539 1 : std::string endLabelStr = labelStr + "_end";
1540 1 : std::string beginLabelStr = labelStr + "_begin";
1541 1 : auto endLabel = std::make_shared<CcuRep::CcuRepJumpLabel>(insGenerator, endLabelStr);
1542 1 : auto beginLabel = std::make_shared<CcuRep::CcuRepJumpLabel>(insGenerator, beginLabelStr);
1543 :
1544 1 : Append(beginLabel);
1545 :
1546 1 : auto targetVar = CcuRep::CreateVariable(this);
1547 1 : auto jump = MakeInvertedCondJumpVar(insGenerator, endLabelStr, targetVar, *lhsVar, *rhsVar, condType, __func__);
1548 1 : if (jump == nullptr) {
1549 0 : return CcuResult::CCU_E_PARA;
1550 : }
1551 1 : jump->Reference(endLabel);
1552 1 : Append(jump);
1553 :
1554 1 : PendingWhileContext ctx;
1555 1 : ctx.beginLabel = beginLabel;
1556 1 : ctx.endLabel = endLabel;
1557 1 : ctx.varHandle = lhsHandle;
1558 1 : ctx.immediate = 0;
1559 1 : ctx.condType = condType;
1560 1 : pendingWhileCtx_.emplace(labelStr, std::move(ctx));
1561 :
1562 1 : return CcuResult::CCU_SUCCESS;
1563 1 : }
1564 :
1565 4 : CcuResult CcuKernel::WhileEnd(const char* label)
1566 : {
1567 4 : PLF_CONFIG_INFO(PLF_DATA_OP, "[WhileEnd] label=%s", (label ? label : "null"));
1568 4 : std::string labelStr(label);
1569 4 : auto iter = pendingWhileCtx_.find(labelStr);
1570 4 : if (iter == pendingWhileCtx_.end()) {
1571 0 : HCCL_ERROR("[%s] no matching WhileBegin for label '%s'", __func__, label);
1572 0 : return CcuResult::CCU_E_NOT_FOUND;
1573 : }
1574 :
1575 4 : std::string beginLabelStr = labelStr + "_begin";
1576 4 : auto loopBackVar = CcuRep::CreateVariable(this);
1577 4 : auto loopBackJump = std::make_shared<CcuRep::CcuRepJump>(insGenerator, beginLabelStr, loopBackVar);
1578 4 : loopBackJump->Reference(iter->second.beginLabel);
1579 4 : Append(loopBackJump);
1580 :
1581 4 : Append(iter->second.endLabel);
1582 :
1583 4 : pendingWhileCtx_.erase(iter);
1584 :
1585 4 : return CcuResult::CCU_SUCCESS;
1586 4 : }
1587 :
1588 9 : CcuResult CcuKernel::DoWhileBegin(const char* label)
1589 : {
1590 9 : PLF_CONFIG_INFO(PLF_DATA_OP, "[DoWhileBegin] label=%s", (label ? label : "null"));
1591 9 : if (CurrentBlock()->Type() == CcuRep::CcuRepType::LOOP_BLOCK) {
1592 0 : HCCL_ERROR(
1593 : "[%s] CCU_DO is not allowed inside a ccu::Loop body (label='%s')", __func__,
1594 : label != nullptr ? label : "(null)");
1595 0 : return LatchBodyError(CcuResult::CCU_E_INTERNAL);
1596 : }
1597 :
1598 9 : std::string labelStr(label);
1599 9 : if (pendingDoWhileCtx_.find(labelStr) != pendingDoWhileCtx_.end()) {
1600 0 : HCCL_ERROR("[%s] label '%s' already has a pending DoWhileBegin without DoWhileEnd", __func__, label);
1601 0 : return CcuResult::CCU_E_PARA;
1602 : }
1603 :
1604 9 : std::string beginLabelStr = labelStr + "_begin";
1605 9 : auto beginLabel = std::make_shared<CcuRep::CcuRepJumpLabel>(insGenerator, beginLabelStr);
1606 9 : Append(beginLabel);
1607 :
1608 9 : PendingDoWhileContext ctx;
1609 9 : ctx.beginLabel = beginLabel;
1610 9 : pendingDoWhileCtx_.emplace(labelStr, std::move(ctx));
1611 :
1612 9 : return CcuResult::CCU_SUCCESS;
1613 9 : }
1614 :
1615 : CcuResult
1616 5 : CcuKernel::DoWhileEnd(CcuVariableHandle varHandle, uint64_t immediate, CcuConditionType condType, const char* label)
1617 : {
1618 5 : PLF_CONFIG_INFO(
1619 : PLF_DATA_OP, "[DoWhileEnd] varHandle=%llu, immediate=%llu, condType=%d, label=%s", varHandle, immediate,
1620 : condType, (label ? label : "null"));
1621 5 : CcuRep::Variable* variable{nullptr};
1622 5 : CCU_CHK_RET(GetVariableByHandle(varHandle, &variable));
1623 :
1624 5 : std::string labelStr(label);
1625 5 : auto iter = pendingDoWhileCtx_.find(labelStr);
1626 5 : if (iter == pendingDoWhileCtx_.end()) {
1627 0 : HCCL_ERROR("[%s] no matching DoWhileBegin for label '%s'", __func__, label);
1628 0 : return CcuResult::CCU_E_NOT_FOUND;
1629 : }
1630 :
1631 5 : std::string beginLabelStr = labelStr + "_begin";
1632 5 : auto targetVar = CcuRep::CreateVariable(this);
1633 5 : auto expectVar = CreateExpectVar();
1634 5 : std::shared_ptr<CcuRep::CcuRepJumpBase> jump{nullptr};
1635 :
1636 : // "condition true => continue looping" means jump back to begin when condition holds
1637 5 : if (condType == CCU_CONDITION_EQ) {
1638 0 : jump = std::make_shared<CcuRep::CcuRepJumpEQ>(
1639 0 : insGenerator, beginLabelStr, targetVar, expectVar, *variable, immediate);
1640 5 : } else if (condType == CCU_CONDITION_NE) {
1641 4 : jump = std::make_shared<CcuRep::CcuRepJumpNE>(
1642 4 : insGenerator, beginLabelStr, targetVar, expectVar, *variable, immediate);
1643 1 : } else if (condType == CCU_CONDITION_LT) {
1644 0 : jump = std::make_shared<CcuRep::CcuRepJumpLT>(
1645 0 : insGenerator, beginLabelStr, targetVar, expectVar, *variable, immediate);
1646 1 : } else if (condType == CCU_CONDITION_LE) {
1647 1 : jump = std::make_shared<CcuRep::CcuRepJumpLE>(
1648 1 : insGenerator, beginLabelStr, targetVar, expectVar, *variable, immediate);
1649 0 : } else if (condType == CCU_CONDITION_GT) {
1650 0 : jump = std::make_shared<CcuRep::CcuRepJumpGT>(
1651 0 : insGenerator, beginLabelStr, targetVar, expectVar, *variable, immediate);
1652 0 : } else if (condType == CCU_CONDITION_GE) {
1653 0 : jump = std::make_shared<CcuRep::CcuRepJumpGE>(
1654 0 : insGenerator, beginLabelStr, targetVar, expectVar, *variable, immediate);
1655 : } else {
1656 0 : HCCL_ERROR("[%s] unsupported condition type: %d", __func__, condType);
1657 0 : return CcuResult::CCU_E_PARA;
1658 : }
1659 :
1660 5 : jump->Reference(iter->second.beginLabel);
1661 5 : Append(jump);
1662 :
1663 5 : pendingDoWhileCtx_.erase(iter);
1664 :
1665 5 : return CcuResult::CCU_SUCCESS;
1666 5 : }
1667 :
1668 1 : CcuResult CcuKernel::DoWhileEndVar(
1669 : CcuVariableHandle lhsHandle, CcuVariableHandle rhsHandle, CcuConditionType condType, const char* label)
1670 : {
1671 1 : CcuRep::Variable* lhsVar{nullptr};
1672 1 : CcuRep::Variable* rhsVar{nullptr};
1673 1 : CCU_CHK_RET(GetVariableByHandle(lhsHandle, &lhsVar));
1674 1 : CCU_CHK_RET(GetVariableByHandle(rhsHandle, &rhsVar));
1675 :
1676 1 : std::string labelStr(label);
1677 1 : auto iter = pendingDoWhileCtx_.find(labelStr);
1678 1 : if (iter == pendingDoWhileCtx_.end()) {
1679 0 : HCCL_ERROR("[%s] no matching DoWhileBegin for label '%s'", __func__, label);
1680 0 : return CcuResult::CCU_E_NOT_FOUND;
1681 : }
1682 :
1683 1 : std::string beginLabelStr = labelStr + "_begin";
1684 1 : auto targetVar = CcuRep::CreateVariable(this);
1685 1 : std::shared_ptr<CcuRep::CcuRepJumpBase> jump{nullptr};
1686 :
1687 1 : if (condType == CCU_CONDITION_EQ) {
1688 0 : jump = std::make_shared<CcuRep::CcuRepJumpEQ>(insGenerator, beginLabelStr, targetVar, *lhsVar, *rhsVar);
1689 1 : } else if (condType == CCU_CONDITION_NE) {
1690 1 : jump = std::make_shared<CcuRep::CcuRepJumpNE>(insGenerator, beginLabelStr, targetVar, *lhsVar, *rhsVar);
1691 0 : } else if (condType == CCU_CONDITION_LT) {
1692 0 : jump = std::make_shared<CcuRep::CcuRepJumpLT>(insGenerator, beginLabelStr, targetVar, *lhsVar, *rhsVar);
1693 0 : } else if (condType == CCU_CONDITION_LE) {
1694 0 : jump = std::make_shared<CcuRep::CcuRepJumpLE>(insGenerator, beginLabelStr, targetVar, *lhsVar, *rhsVar);
1695 0 : } else if (condType == CCU_CONDITION_GT) {
1696 0 : jump = std::make_shared<CcuRep::CcuRepJumpGT>(insGenerator, beginLabelStr, targetVar, *lhsVar, *rhsVar);
1697 0 : } else if (condType == CCU_CONDITION_GE) {
1698 0 : jump = std::make_shared<CcuRep::CcuRepJumpGE>(insGenerator, beginLabelStr, targetVar, *lhsVar, *rhsVar);
1699 : } else {
1700 0 : HCCL_ERROR("[%s] unsupported condition type: %d", __func__, condType);
1701 0 : return CcuResult::CCU_E_PARA;
1702 : }
1703 :
1704 1 : jump->Reference(iter->second.beginLabel);
1705 1 : Append(jump);
1706 :
1707 1 : pendingDoWhileCtx_.erase(iter);
1708 :
1709 1 : return CcuResult::CCU_SUCCESS;
1710 1 : }
1711 :
1712 : // 控制流标签栈实体
1713 104 : void CcuKernel::IfLabelStackPush(const char* label) { iflabelStack_.push_back({label, false}); }
1714 104 : void CcuKernel::IfLabelStackMarkBodyDone()
1715 : {
1716 104 : if (iflabelStack_.empty()) {
1717 0 : HCCL_ERROR("[CcuKernel::IfLabelStack][MarkBodyDone] stack is empty");
1718 0 : return;
1719 : }
1720 104 : iflabelStack_.back().bodyDone = true;
1721 : }
1722 22 : const char* CcuKernel::IfLabelStackPopForElse()
1723 : {
1724 22 : if (iflabelStack_.empty()) {
1725 0 : HCCL_ERROR("[CcuKernel::IfLabelStack][PopForElse] orphan CCU_ELSE: "
1726 : "no matching CCU_IF on the stack");
1727 0 : return nullptr;
1728 : }
1729 22 : if (!iflabelStack_.back().bodyDone) {
1730 0 : HCCL_ERROR(
1731 : "[CcuKernel::IfLabelStack][PopForElse] CCU_ELSE called while "
1732 : "top if-body is still InBody (label='%s')",
1733 : iflabelStack_.back().label != nullptr ? iflabelStack_.back().label : "(null)");
1734 0 : return nullptr;
1735 : }
1736 22 : const char* label = iflabelStack_.back().label;
1737 22 : iflabelStack_.pop_back();
1738 22 : return label;
1739 : }
1740 1509 : bool CcuKernel::IfLabelStackTopIsClosable() { return !iflabelStack_.empty() && iflabelStack_.back().bodyDone; }
1741 :
1742 82 : const char* CcuKernel::IfLabelStackPop()
1743 : {
1744 82 : if (iflabelStack_.empty()) {
1745 0 : return nullptr;
1746 : }
1747 82 : const char* label = iflabelStack_.back().label;
1748 82 : iflabelStack_.pop_back();
1749 82 : return label;
1750 : }
1751 :
1752 9 : void CcuKernel::DoWhileLabelStackPush(const char* label)
1753 : {
1754 9 : DoWhileLabelEntry entry;
1755 9 : entry.label = label;
1756 9 : entry.snapshotBlock = CurrentBlock();
1757 9 : entry.snapshotRepCount = (entry.snapshotBlock != nullptr) ? entry.snapshotBlock->GetReps().size() : 0;
1758 9 : doWhileLabelStack_.push_back(std::move(entry));
1759 9 : }
1760 :
1761 13 : const char* CcuKernel::DoWhileLabelStackPopForWhile()
1762 : {
1763 13 : if (CurrentBlock()->Type() == CcuRep::CcuRepType::LOOP_BLOCK) {
1764 1 : return nullptr;
1765 : }
1766 12 : if (doWhileLabelStack_.empty()) {
1767 4 : return nullptr;
1768 : }
1769 8 : DoWhileLabelEntry entry = doWhileLabelStack_.back();
1770 8 : doWhileLabelStack_.pop_back();
1771 :
1772 8 : auto currentBlock = CurrentBlock();
1773 8 : size_t currentRepCount = (currentBlock != nullptr) ? currentBlock->GetReps().size() : 0;
1774 8 : if (currentBlock != entry.snapshotBlock || currentRepCount != entry.snapshotRepCount) {
1775 1 : HCCL_ERROR(
1776 : "[CcuKernel::DoWhileLabelStackPopForWhile] dangling CCU calls between CCU_DO end "
1777 : "and CCU_WHILE (label='%s', snapRep=%zu, curRep=%zu, blockChanged=%d); they must "
1778 : "be syntactically adjacent, otherwise the code in between is pulled into the body.",
1779 : entry.label != nullptr ? entry.label : "(null)", entry.snapshotRepCount, currentRepCount,
1780 : currentBlock != entry.snapshotBlock ? 1 : 0);
1781 1 : return nullptr;
1782 : }
1783 7 : return entry.label;
1784 8 : }
1785 :
1786 152 : CcuResult CcuKernel::GetAddressByHandle(CcuAddressHandle addrHandle, CcuRep::Address** address)
1787 : {
1788 152 : return GetResourceByHandle(ccuAddrMap_, addrHandle, address, "address");
1789 : }
1790 :
1791 : // addr = 立即数 → CcuRepAssign(Address, uint64_t)
1792 13 : CcuResult CcuKernel::AddressAssignImm(CcuAddressHandle addrHandle, uint64_t immediate)
1793 : {
1794 13 : PLF_CONFIG_INFO(PLF_DATA_OP, "[AddressAssignImm] addrHandle=%llu, immediate=%llu", addrHandle, immediate);
1795 13 : CcuRep::Address* address{nullptr};
1796 13 : CCU_CHK_RET(GetAddressByHandle(addrHandle, &address));
1797 13 : (*address) = immediate;
1798 13 : return CcuResult::CCU_SUCCESS;
1799 : }
1800 :
1801 : // addr = variable → CcuRepAssign(Address, Variable)
1802 14 : CcuResult CcuKernel::AddressAssignVar(CcuAddressHandle addrHandle, CcuVariableHandle varHandle)
1803 : {
1804 14 : PLF_CONFIG_INFO(PLF_DATA_OP, "[AddressAssignVar] addrHandle=%llu, varHandle=%llu", addrHandle, varHandle);
1805 14 : CcuRep::Address* address{nullptr};
1806 14 : CCU_CHK_RET(GetAddressByHandle(addrHandle, &address));
1807 :
1808 14 : CcuRep::Variable* variable{nullptr};
1809 14 : CCU_CHK_RET(GetVariableByHandle(varHandle, &variable));
1810 :
1811 14 : (*address) = (*variable);
1812 14 : return CcuResult::CCU_SUCCESS;
1813 : }
1814 : // addr = addr → CcuRepAssign(Address, Address)
1815 38 : CcuResult CcuKernel::AddressAssignAddr(CcuAddressHandle dstAddrHandle, CcuAddressHandle srcAddrHandle)
1816 : {
1817 38 : PLF_CONFIG_INFO(
1818 : PLF_DATA_OP, "[AddressAssignAddr] dstAddrHandle=%llu, srcAddrHandle=%llu", dstAddrHandle, srcAddrHandle);
1819 38 : CcuRep::Address* dstAddress{nullptr};
1820 38 : CCU_CHK_RET(GetAddressByHandle(dstAddrHandle, &dstAddress));
1821 :
1822 38 : CcuRep::Address* srcAddress{nullptr};
1823 38 : CCU_CHK_RET(GetAddressByHandle(srcAddrHandle, &srcAddress));
1824 :
1825 38 : (*dstAddress) = (*srcAddress);
1826 38 : return CcuResult::CCU_SUCCESS;
1827 : }
1828 :
1829 : // resAddr = lhsAddr + rhsVar → CcuRepAdd(Address, Address, Variable)
1830 0 : CcuResult CcuKernel::AddressAddVarToAddr(
1831 : CcuAddressHandle resAddrHandle, CcuAddressHandle lhsAddrHandle, CcuVariableHandle rhsVarHandle)
1832 : {
1833 0 : PLF_CONFIG_INFO(
1834 : PLF_DATA_OP, "[AddressAddVarToAddr] resAddrHandle=%llu, lhsAddrHandle=%llu, rhsVarHandle=%llu", resAddrHandle,
1835 : lhsAddrHandle, rhsVarHandle);
1836 0 : CcuRep::Address *resAddr{nullptr}, *lhsAddr{nullptr};
1837 0 : CCU_CHK_RET(GetAddressByHandle(resAddrHandle, &resAddr));
1838 0 : CCU_CHK_RET(GetAddressByHandle(lhsAddrHandle, &lhsAddr));
1839 :
1840 0 : CcuRep::Variable* rhsVar{nullptr};
1841 0 : CCU_CHK_RET(GetVariableByHandle(rhsVarHandle, &rhsVar));
1842 :
1843 0 : *resAddr = *lhsAddr + *rhsVar;
1844 0 : return CcuResult::CCU_SUCCESS;
1845 : }
1846 :
1847 : // resAddr = addrA + addrB → CcuRepAdd(Address, Address, Address)
1848 2 : CcuResult CcuKernel::AddressAddAddrToAddr(
1849 : CcuAddressHandle resAddrHandle, CcuAddressHandle addrAHandle, CcuAddressHandle addrBHandle)
1850 : {
1851 2 : PLF_CONFIG_INFO(
1852 : PLF_DATA_OP, "[AddressAddAddrToAddr] resAddrHandle=%llu, addrAHandle=%llu, addrBHandle=%llu", resAddrHandle,
1853 : addrAHandle, addrBHandle);
1854 2 : CcuRep::Address *resAddr{nullptr}, *addrA{nullptr}, *addrB{nullptr};
1855 2 : CCU_CHK_RET(GetAddressByHandle(resAddrHandle, &resAddr));
1856 2 : CCU_CHK_RET(GetAddressByHandle(addrAHandle, &addrA));
1857 2 : CCU_CHK_RET(GetAddressByHandle(addrBHandle, &addrB));
1858 :
1859 2 : *resAddr = *addrA + *addrB;
1860 2 : return CcuResult::CCU_SUCCESS;
1861 : }
1862 :
1863 : // addr += variable → CcuRepAdd(Address, Variable) 就地加
1864 41 : CcuResult CcuKernel::AddressAddAssignVar(CcuAddressHandle addrHandle, CcuVariableHandle varHandle)
1865 : {
1866 41 : PLF_CONFIG_INFO(PLF_DATA_OP, "[AddressAddAssignVar] addrHandle=%llu, varHandle=%llu", addrHandle, varHandle);
1867 41 : CcuRep::Address* address{nullptr};
1868 41 : CCU_CHK_RET(GetAddressByHandle(addrHandle, &address));
1869 :
1870 41 : CcuRep::Variable* variable{nullptr};
1871 41 : CCU_CHK_RET(GetVariableByHandle(varHandle, &variable));
1872 :
1873 41 : (*address) += (*variable);
1874 41 : return CcuResult::CCU_SUCCESS;
1875 : }
1876 :
1877 : // addr += addr → 等价于 addr = addr + otherAddr
1878 0 : CcuResult CcuKernel::AddressAddAssignAddr(CcuAddressHandle addrHandle, CcuAddressHandle otherHandle)
1879 : {
1880 0 : PLF_CONFIG_INFO(PLF_DATA_OP, "[AddressAddAssignAddr] addrHandle=%llu, otherHandle=%llu", addrHandle, otherHandle);
1881 0 : CcuRep::Address* address{nullptr};
1882 0 : CCU_CHK_RET(GetAddressByHandle(addrHandle, &address));
1883 :
1884 0 : CcuRep::Address* other{nullptr};
1885 0 : CCU_CHK_RET(GetAddressByHandle(otherHandle, &other));
1886 :
1887 0 : (*address) = (*address) + (*other);
1888 0 : return CcuResult::CCU_SUCCESS;
1889 : }
1890 :
1891 1 : CcuResult CcuKernel::AddressAddImmToAddr(CcuAddressHandle resAddrHandle, CcuAddressHandle addrAHandle, uint16_t imm)
1892 : {
1893 1 : CcuRep::Address *resAddr{nullptr}, *addrA{nullptr};
1894 1 : CCU_CHK_RET(GetAddressByHandle(resAddrHandle, &resAddr));
1895 1 : CCU_CHK_RET(GetAddressByHandle(addrAHandle, &addrA));
1896 :
1897 1 : *resAddr = *addrA + imm;
1898 1 : return CcuResult::CCU_SUCCESS;
1899 : }
1900 :
1901 0 : void CcuKernel::Load(const CcuRep::Variable& var)
1902 : {
1903 : auto loadArgRep = std::make_shared<CcuRep::CcuRepLoadArg>(
1904 0 : insGenerator, var, loadArgIndex_ % CCU_SQE_ARGS_LEN, static_cast<uint16_t>(loadArgIndex_));
1905 0 : GetLGProfilingInfo().loadRep2ArgIdxMap[loadArgRep] = loadArgIndex_;
1906 0 : Append(loadArgRep);
1907 0 : loadArgIndex_++;
1908 0 : }
1909 :
1910 0 : void CcuKernel::StoreVariable(const CcuRep::Variable& var, uint64_t addr)
1911 : {
1912 0 : Append(std::make_shared<CcuRep::CcuRepStore>(insGenerator, var, addr));
1913 0 : }
1914 :
1915 0 : void CcuKernel::LoadVariable(const CcuRep::Variable& src, const CcuRep::Variable& var)
1916 : {
1917 0 : Append(std::make_shared<CcuRep::CcuRepLoadVar>(insGenerator, src, var));
1918 0 : }
1919 :
1920 18 : HcclResult CcuKernel::RecordEvent(CcuRep::CompletedEvent event, uint32_t mask)
1921 : {
1922 18 : if (CurrentBlock()->Type() == CcuRep::CcuRepType::LOOP_BLOCK) {
1923 1 : HCCL_ERROR("[CcuKernel][%s] is not supported in loop block, please check.", __func__);
1924 1 : LatchBodyError(HCCL_TO_CCU_RET(HcclResult::HCCL_E_NOT_SUPPORT));
1925 1 : return HcclResult::HCCL_E_NOT_SUPPORT;
1926 : }
1927 :
1928 17 : auto rep = std::make_shared<CcuRep::CcuRepLocRecordEvent>(insGenerator, event, mask);
1929 17 : Append(rep);
1930 17 : SetDependencyInfo(event.Id(), mask, rep);
1931 17 : return HCCL_SUCCESS;
1932 17 : }
1933 :
1934 44 : HcclResult CcuKernel::WaitEvent(CcuRep::CompletedEvent event, uint32_t mask)
1935 : {
1936 44 : bool isProfiling = CurrentBlock()->Type() != CcuRep::CcuRepType::LOOP_BLOCK;
1937 44 : auto rep = std::make_shared<CcuRep::CcuRepLocWaitEvent>(insGenerator, event, mask, isProfiling);
1938 44 : if (isProfiling) {
1939 84 : CHK_RET(static_cast<HcclResult>(AddProfiling("WaitEvent", rep->GetMask())));
1940 : }
1941 44 : rep->SetDependencyInfo(GetDependencyInfo(event.Id()));
1942 44 : EraseDependencyInfo(event.Id());
1943 44 : Append(rep);
1944 44 : return HCCL_SUCCESS;
1945 44 : }
1946 :
1947 102 : CcuResult CcuKernel::GetEventByHandle(CcuEventHandle eventHandle, CcuRep::CompletedEvent** event)
1948 : {
1949 102 : return GetResourceByHandle(ccuEventMap_, eventHandle, event, "completedEvent");
1950 : }
1951 :
1952 : /*
1953 : LocalAddr / RemoteAddr 相关接口
1954 : */
1955 33 : CcuResult CcuKernel::GetLocalAddrByHandle(CcuLocalAddrHandle handle, CcuRep::LocalAddr** localAddr)
1956 : {
1957 33 : return GetResourceByHandle(ccuLocalAddrMap_, handle, localAddr, "localAddr");
1958 : }
1959 :
1960 15 : CcuResult CcuKernel::GetRemoteAddrByHandle(CcuRemoteAddrHandle handle, CcuRep::RemoteAddr** remoteAddr)
1961 : {
1962 15 : return GetResourceByHandle(ccuRemoteAddrMap_, handle, remoteAddr, "remoteAddr");
1963 : }
1964 :
1965 : /*Read新接口*/
1966 2 : HcclResult CcuKernel::ReadNb(
1967 : const ChannelHandle channel, const CcuRep::CcuBuf& loc, const CcuRep::RemoteAddr& rem, const CcuRep::Variable& len,
1968 : CcuRep::CompletedEvent event, uint32_t mask)
1969 : {
1970 2 : channels_.insert(channel);
1971 2 : auto rep = std::make_shared<CcuRep::CcuRepBufRead>(insGenerator, channel, rem, loc, len, event, mask);
1972 2 : Append(rep);
1973 2 : SetDependencyInfo(event.Id(), mask, rep);
1974 2 : return HCCL_SUCCESS;
1975 2 : }
1976 :
1977 : /*Write新接口*/
1978 2 : HcclResult CcuKernel::WriteNb(
1979 : const ChannelHandle channel, const CcuRep::RemoteAddr& rem, const CcuRep::CcuBuf& loc, const CcuRep::Variable& len,
1980 : CcuRep::CompletedEvent event, uint32_t mask)
1981 : {
1982 2 : channels_.insert(channel);
1983 2 : auto rep = std::make_shared<CcuRep::CcuRepBufWrite>(insGenerator, channel, loc, rem, len, event, mask);
1984 2 : Append(rep);
1985 2 : SetDependencyInfo(event.Id(), mask, rep);
1986 2 : return HCCL_SUCCESS;
1987 2 : }
1988 :
1989 8 : static bool isLowPrecisionIn(Hccl::DataType dataType)
1990 : {
1991 16 : return dataType == Hccl::DataType::INT8 || dataType == Hccl::DataType::HIF8 || dataType == Hccl::DataType::FP8E4M3
1992 16 : || dataType == Hccl::DataType::FP8E5M2;
1993 : }
1994 :
1995 0 : static bool isLowPrecisionOut(Hccl::DataType dataType)
1996 : {
1997 0 : return dataType == Hccl::DataType::FP16 || dataType == Hccl::DataType::BFP16 || dataType == Hccl::DataType::FP32;
1998 : }
1999 :
2000 : constexpr uint32_t MAX_DATA_TYPE = 17;
2001 :
2002 : const Hccl::DataType orionDataTypes[]
2003 : = {Hccl::DataType::INT8, Hccl::DataType::INT16, Hccl::DataType::INT32,
2004 : Hccl::DataType::FP16, Hccl::DataType::FP32, Hccl::DataType::INT64,
2005 : Hccl::DataType::UINT64, Hccl::DataType::UINT8, Hccl::DataType::UINT16,
2006 : Hccl::DataType::UINT32, Hccl::DataType::FP64, Hccl::DataType::BFP16,
2007 : Hccl::DataType::INT128,
2008 : #if !defined(OPEN_BUILD_PROJECT) || defined(ORION_MODE)
2009 : Hccl::DataType::HIF8, Hccl::DataType::FP8E4M3, Hccl::DataType::FP8E5M2,
2010 : Hccl::DataType::FP8E8M0
2011 : #endif
2012 : };
2013 :
2014 12 : static Hccl::DataType HcommDataTypeToHcclDataType(const HcclDataType dataType)
2015 : {
2016 12 : const auto dataTypeNum = static_cast<uint32_t>(dataType);
2017 12 : if (dataTypeNum > MAX_DATA_TYPE) {
2018 0 : return Hccl::DataType::INVALID;
2019 : }
2020 :
2021 12 : return orionDataTypes[dataTypeNum];
2022 : }
2023 :
2024 : constexpr uint32_t MAX_REDUCE_TYPE = 4;
2025 : const Hccl::ReduceOp orionReduceOps[] = {
2026 : Hccl::ReduceOp::SUM,
2027 : Hccl::ReduceOp::PROD,
2028 : Hccl::ReduceOp::MAX,
2029 : Hccl::ReduceOp::MIN,
2030 : };
2031 :
2032 8 : static Hccl::ReduceOp HcommReduceOpToHcclReduceOp(const HcclReduceOp reduceOp)
2033 : {
2034 8 : const auto reduceOpNum = static_cast<uint32_t>(reduceOp);
2035 8 : if (reduceOpNum > MAX_REDUCE_TYPE) {
2036 0 : return Hccl::ReduceOp::INVALID;
2037 : }
2038 :
2039 8 : return orionReduceOps[reduceOpNum];
2040 : }
2041 :
2042 4 : HcclResult CcuKernel::LocalReduceNb(
2043 : const CcuRep::CcuBuf* bufs, uint32_t count, HcclDataType dataType, HcclDataType outputDataType, HcclReduceOp opType,
2044 : const CcuRep::Variable& len, CcuRep::CompletedEvent event, uint32_t mask)
2045 : {
2046 4 : auto opType_ = HcommReduceOpToHcclReduceOp(opType);
2047 4 : auto dataType_ = HcommDataTypeToHcclDataType(dataType);
2048 4 : auto outputDataType_ = HcommDataTypeToHcclDataType(outputDataType);
2049 :
2050 8 : if ((opType_ == Hccl::ReduceOp::SUM && isLowPrecisionIn(dataType_) && !isLowPrecisionOut(outputDataType_))
2051 4 : || (opType_ == Hccl::ReduceOp::SUM && !isLowPrecisionIn(dataType_) && dataType_ != outputDataType_)
2052 8 : || (opType_ != Hccl::ReduceOp::SUM && dataType_ != outputDataType_)) {
2053 0 : return HCCL_E_NOT_SUPPORT;
2054 : }
2055 :
2056 4 : std::vector<CcuRep::CcuBuf> ccuBufs(count);
2057 12 : for (uint32_t i = 0; i < count; i++) {
2058 8 : ccuBufs[i] = bufs[i];
2059 : }
2060 :
2061 : auto rep = std::make_shared<CcuRep::CcuRepBufReduce>(
2062 0 : insGenerator, ccuBufs, count, CcuRep::GetCcuDataType(dataType_, opType_),
2063 4 : CcuRep::GetCcuDataType(outputDataType_, opType_), CcuRep::GetCcuReduceType(opType_), event, len, mask);
2064 4 : Append(rep);
2065 4 : SetDependencyInfo(event.Id(), mask, rep);
2066 4 : return HCCL_SUCCESS;
2067 4 : }
2068 :
2069 : /*Read新接口*/
2070 4 : HcclResult CcuKernel::ReadNb(
2071 : const ChannelHandle channel, const CcuRep::LocalAddr& loc, const CcuRep::RemoteAddr& rem,
2072 : const CcuRep::Variable& len, CcuRep::CompletedEvent event, uint32_t mask)
2073 : {
2074 4 : channels_.insert(channel);
2075 4 : auto rep = std::make_shared<CcuRep::CcuRepRead>(insGenerator, channel, loc, rem, len, event, mask);
2076 4 : Append(rep);
2077 4 : SetDependencyInfo(event.Id(), mask, rep);
2078 4 : return HCCL_SUCCESS;
2079 4 : }
2080 :
2081 : /*ReadReduce新接口*/
2082 2 : HcclResult CcuKernel::ReadReduceNb(
2083 : const ChannelHandle channel, const CcuRep::LocalAddr& loc, const CcuRep::RemoteAddr& rem,
2084 : const CcuRep::Variable& len, HcclDataType dataType, HcclReduceOp opType, CcuRep::CompletedEvent event,
2085 : uint32_t mask)
2086 : {
2087 2 : channels_.insert(channel);
2088 2 : auto opType_ = HcommReduceOpToHcclReduceOp(opType);
2089 2 : auto dataType_ = HcommDataTypeToHcclDataType(dataType);
2090 :
2091 : auto rep = std::make_shared<CcuRep::CcuRepRead>(
2092 2 : insGenerator, channel, loc, rem, len, CcuRep::GetUBDataType(dataType_), CcuRep::GetUBReduceType(opType_), event,
2093 2 : mask);
2094 2 : Append(rep);
2095 2 : SetDependencyInfo(event.Id(), mask, rep);
2096 2 : return HCCL_SUCCESS;
2097 2 : }
2098 :
2099 : /*Write新接口*/
2100 3 : HcclResult CcuKernel::WriteNb(
2101 : const ChannelHandle channel, const CcuRep::RemoteAddr& rem, const CcuRep::LocalAddr& loc,
2102 : const CcuRep::Variable& len, CcuRep::CompletedEvent event, uint32_t mask)
2103 : {
2104 3 : channels_.insert(channel);
2105 3 : auto rep = std::make_shared<CcuRep::CcuRepWrite>(insGenerator, channel, rem, loc, len, event, mask);
2106 3 : Append(rep);
2107 3 : SetDependencyInfo(event.Id(), mask, rep);
2108 3 : return HCCL_SUCCESS;
2109 3 : }
2110 :
2111 2 : HcclResult CcuKernel::WriteReduceNb(
2112 : const ChannelHandle channel, const CcuRep::RemoteAddr& rem, const CcuRep::LocalAddr& loc,
2113 : const CcuRep::Variable& len, HcclDataType dataType, HcclReduceOp opType, CcuRep::CompletedEvent event,
2114 : uint32_t mask)
2115 : {
2116 2 : channels_.insert(channel);
2117 2 : auto opType_ = HcommReduceOpToHcclReduceOp(opType);
2118 2 : auto dataType_ = HcommDataTypeToHcclDataType(dataType);
2119 :
2120 : auto rep = std::make_shared<CcuRep::CcuRepWrite>(
2121 2 : insGenerator, channel, rem, loc, len, CcuRep::GetUBDataType(dataType_), CcuRep::GetUBReduceType(opType_), event,
2122 2 : mask);
2123 2 : Append(rep);
2124 2 : SetDependencyInfo(event.Id(), mask, rep);
2125 2 : return HCCL_SUCCESS;
2126 2 : }
2127 :
2128 32 : CcuResult CcuKernel::GetBufferByHandle(CcuBufferHandle bufferHandle, CcuRep::CcuBuf** buffer)
2129 : {
2130 32 : return GetResourceByHandle(ccuBufferMap_, bufferHandle, buffer, "buffer");
2131 : }
2132 :
2133 1 : HcclResult CcuKernel::LocalCopyNb(
2134 : const CcuRep::LocalAddr& dst, const CcuRep::LocalAddr& src, const CcuRep::Variable& len,
2135 : CcuRep::CompletedEvent event, uint32_t mask)
2136 : {
2137 1 : auto rep = std::make_shared<CcuRep::CcuRepLocCpy>(insGenerator, dst, src, len, event, mask);
2138 1 : Append(rep);
2139 1 : SetDependencyInfo(event.Id(), mask, rep);
2140 1 : return HCCL_SUCCESS;
2141 1 : }
2142 :
2143 12 : HcclResult CcuKernel::LocalCopyNb(
2144 : const CcuRep::CcuBuf& dst, const CcuRep::LocalAddr& src, const CcuRep::Variable& len, CcuRep::CompletedEvent event,
2145 : uint32_t mask)
2146 : {
2147 12 : auto rep = std::make_shared<CcuRep::CcuRepBufLocRead>(insGenerator, src, dst, len, event, mask);
2148 12 : Append(rep);
2149 12 : SetDependencyInfo(event.Id(), mask, rep);
2150 12 : return HCCL_SUCCESS;
2151 12 : }
2152 :
2153 8 : HcclResult CcuKernel::LocalCopyNb(
2154 : const CcuRep::LocalAddr& dst, const CcuRep::CcuBuf& src, const CcuRep::Variable& len, CcuRep::CompletedEvent event,
2155 : uint32_t mask)
2156 : {
2157 8 : auto rep = std::make_shared<CcuRep::CcuRepBufLocWrite>(insGenerator, src, dst, len, event, mask);
2158 8 : Append(rep);
2159 8 : SetDependencyInfo(event.Id(), mask, rep);
2160 8 : return HCCL_SUCCESS;
2161 8 : }
2162 :
2163 0 : HcclResult CcuKernel::LocalReduceNb(
2164 : const CcuRep::LocalAddr& dst, const CcuRep::LocalAddr& src, const CcuRep::Variable& len, HcclDataType dataType,
2165 : HcclReduceOp opType, CcuRep::CompletedEvent event, uint32_t mask)
2166 : {
2167 0 : auto opType_ = HcommReduceOpToHcclReduceOp(opType);
2168 0 : auto dataType_ = HcommDataTypeToHcclDataType(dataType);
2169 :
2170 : auto rep = std::make_shared<CcuRep::CcuRepLocCpy>(
2171 0 : insGenerator, dst, src, len, CcuRep::GetUBDataType(dataType_), CcuRep::GetUBReduceType(opType_), event, mask);
2172 0 : Append(rep);
2173 0 : SetDependencyInfo(event.Id(), mask, rep);
2174 0 : return HCCL_SUCCESS;
2175 0 : }
2176 :
2177 0 : CcuRep::FuncCall CcuKernel::Func(const std::string& label) { return CcuRep::FuncCall(this, label); }
2178 :
2179 0 : CcuRep::FuncCall CcuKernel::Func(const CcuRep::Variable& funcAddr) { return CcuRep::FuncCall(this, funcAddr); }
2180 :
2181 0 : CcuRep::LoopCall CcuKernel::Loop(const std::string& label) { return CcuRep::LoopCall(this, label); }
2182 :
2183 57 : CcuResult CcuKernel::LoopCreate(CcuLoop* loop)
2184 : {
2185 57 : PLF_CONFIG_INFO(PLF_DATA_OP, "[LoopCreate]");
2186 57 : if (loop == nullptr) {
2187 0 : HCCL_ERROR("[CcuKernel::LoopCreate] null pointer");
2188 0 : return CcuResult::CCU_E_PTR;
2189 : }
2190 57 : if (CurrentBlock()->Type() == CcuRep::CcuRepType::LOOP_BLOCK) {
2191 0 : HCCL_ERROR("[CcuKernel::LoopCreate] cannot create loop inside a loop body");
2192 0 : return LatchBodyError(CcuResult::CCU_E_INTERNAL);
2193 : }
2194 57 : if (inFuncBody_) {
2195 0 : HCCL_ERROR("[CcuKernel::LoopCreate] cannot create loop inside a func body");
2196 0 : return LatchBodyError(CcuResult::CCU_E_INTERNAL);
2197 : }
2198 :
2199 57 : CcuLoop handle = ++loopHandleCounter_;
2200 57 : std::string label = "loop_" + std::to_string(handle);
2201 :
2202 57 : LoopDescriptor desc;
2203 57 : desc.label = label;
2204 57 : desc.repLoopBlock = std::make_shared<CcuRep::CcuRepLoopBlock>(insGenerator, label);
2205 :
2206 57 : loopMap_[handle] = std::move(desc);
2207 57 : *loop = handle;
2208 57 : return CcuResult::CCU_SUCCESS;
2209 57 : }
2210 :
2211 7 : CcuResult CcuKernel::LatchBodyError(CcuResult err)
2212 : {
2213 15 : if ((CurrentBlock()->Type() == CcuRep::CcuRepType::LOOP_BLOCK || inFuncBody_)
2214 15 : && bodyError_ == CcuResult::CCU_SUCCESS) {
2215 7 : bodyError_ = err;
2216 : }
2217 7 : return err;
2218 : }
2219 :
2220 57 : CcuResult CcuKernel::LoopBodyEnter(CcuLoop loop)
2221 : {
2222 57 : PLF_CONFIG_INFO(PLF_DATA_OP, "[LoopBodyEnter] loop=%llu", loop);
2223 57 : auto it = loopMap_.find(loop);
2224 57 : if (it == loopMap_.end()) {
2225 0 : HCCL_ERROR("[CcuKernel::LoopBodyEnter] invalid loop handle %lu", loop);
2226 0 : return CcuResult::CCU_E_PARA;
2227 : }
2228 57 : auto& desc = it->second;
2229 57 : if (desc.bodyDefined) {
2230 0 : HCCL_ERROR("[CcuKernel::LoopBodyEnter] loop %lu body already defined", loop);
2231 0 : return CcuResult::CCU_E_INTERNAL;
2232 : }
2233 :
2234 57 : Append(desc.repLoopBlock);
2235 57 : desc.prevActiveBlock = CurrentBlock();
2236 57 : SetCurrentBlock(desc.repLoopBlock);
2237 57 : ++loopBodyDepth_;
2238 57 : bodyError_ = CcuResult::CCU_SUCCESS;
2239 :
2240 57 : return CcuResult::CCU_SUCCESS;
2241 : }
2242 :
2243 57 : CcuResult CcuKernel::LoopBodyExit(CcuLoop loop)
2244 : {
2245 57 : PLF_CONFIG_INFO(PLF_DATA_OP, "[LoopBodyExit] loop=%llu", loop);
2246 57 : auto it = loopMap_.find(loop);
2247 57 : if (it == loopMap_.end()) {
2248 0 : HCCL_ERROR("[CcuKernel::LoopBodyExit] invalid loop handle %lu", loop);
2249 0 : return CcuResult::CCU_E_PARA;
2250 : }
2251 57 : auto& desc = it->second;
2252 :
2253 57 : SetCurrentBlock(desc.prevActiveBlock);
2254 57 : desc.bodyDefined = true;
2255 57 : --loopBodyDepth_;
2256 :
2257 57 : if (bodyError_ != CcuResult::CCU_SUCCESS) {
2258 6 : const CcuResult err = bodyError_;
2259 6 : HCCL_ERROR("[CcuKernel::LoopBodyExit] illegal operation inside loop body, err=%d", err);
2260 6 : bodyError_ = CcuResult::CCU_SUCCESS;
2261 6 : return err;
2262 : }
2263 :
2264 51 : return CcuResult::CCU_SUCCESS;
2265 : }
2266 :
2267 8 : CcuResult CcuKernel::FuncBlockLookup(const void* funcPtr, uint64_t* outHandle)
2268 : {
2269 8 : PLF_CONFIG_INFO(PLF_DATA_OP, "[FuncBlockLookup] funcPtr=%p", funcPtr);
2270 8 : if (funcPtr == nullptr || outHandle == nullptr) {
2271 0 : HCCL_ERROR("[CcuKernel::FuncBlockLookup] null pointer");
2272 0 : return CcuResult::CCU_E_PTR;
2273 : }
2274 8 : if (CurrentBlock()->Type() == CcuRep::CcuRepType::LOOP_BLOCK || inFuncBody_) {
2275 2 : HCCL_ERROR("[CcuKernel::FuncBlockLookup] ccu::CallFunc only allowed at top level");
2276 2 : return LatchBodyError(CcuResult::CCU_E_INTERNAL);
2277 : }
2278 :
2279 6 : auto it = funcInstanceMap_.find(funcPtr);
2280 6 : *outHandle = (it == funcInstanceMap_.end()) ? 0 : it->second;
2281 6 : return CcuResult::CCU_SUCCESS;
2282 : }
2283 :
2284 5 : CcuResult CcuKernel::FuncBlockBegin(const void* funcPtr, uint64_t* outHandle)
2285 : {
2286 5 : PLF_CONFIG_INFO(PLF_DATA_OP, "[FuncBlockBegin] funcPtr=%p", funcPtr);
2287 5 : if (funcPtr == nullptr || outHandle == nullptr) {
2288 0 : HCCL_ERROR("[CcuKernel::FuncBlockBegin] null pointer");
2289 0 : return CcuResult::CCU_E_PTR;
2290 : }
2291 5 : if (CurrentBlock()->Type() == CcuRep::CcuRepType::LOOP_BLOCK || inFuncBody_) {
2292 0 : HCCL_ERROR("[CcuKernel::FuncBlockBegin] ccu::CallFunc only allowed at top level");
2293 0 : return LatchBodyError(CcuResult::CCU_E_INTERNAL);
2294 : }
2295 :
2296 5 : auto exist = funcInstanceMap_.find(funcPtr);
2297 5 : if (exist != funcInstanceMap_.end()) {
2298 0 : *outHandle = exist->second;
2299 0 : return CcuResult::CCU_SUCCESS;
2300 : }
2301 :
2302 5 : const uint64_t handle = ++funcHandleCounter_;
2303 5 : std::string label = "func_" + std::to_string(handle);
2304 :
2305 5 : FuncDescriptor desc;
2306 5 : desc.funcPtr = funcPtr;
2307 5 : desc.label = label;
2308 5 : desc.repFuncBlock = std::make_shared<CcuRep::CcuRepFuncBlock>(insGenerator, label);
2309 5 : desc.prevActiveBlock = CurrentBlock();
2310 :
2311 5 : funcMap_[handle] = desc;
2312 5 : SetCurrentBlock(desc.repFuncBlock);
2313 5 : inFuncBody_ = true;
2314 5 : bodyError_ = CcuResult::CCU_SUCCESS;
2315 5 : *outHandle = handle;
2316 5 : return CcuResult::CCU_SUCCESS;
2317 5 : }
2318 :
2319 4 : CcuResult CcuKernel::FuncBlockEnd(uint64_t handle)
2320 : {
2321 4 : PLF_CONFIG_INFO(PLF_DATA_OP, "[FuncBlockEnd] handle=%llu", handle);
2322 4 : auto it = funcMap_.find(handle);
2323 4 : if (it == funcMap_.end()) {
2324 0 : HCCL_ERROR("[CcuKernel::FuncBlockEnd] invalid func handle %lu", handle);
2325 0 : return CcuResult::CCU_E_PARA;
2326 : }
2327 4 : auto& desc = it->second;
2328 :
2329 4 : SetCurrentBlock(desc.prevActiveBlock);
2330 4 : inFuncBody_ = false;
2331 :
2332 4 : if (bodyError_ != CcuResult::CCU_SUCCESS) {
2333 0 : const CcuResult err = bodyError_;
2334 0 : HCCL_ERROR("[CcuKernel::FuncBlockEnd] illegal operation inside func body, err=%d", err);
2335 0 : bodyError_ = CcuResult::CCU_SUCCESS;
2336 0 : funcMap_.erase(it);
2337 0 : return err;
2338 : }
2339 :
2340 4 : Append(desc.repFuncBlock);
2341 4 : desc.bodyDefined = true;
2342 4 : funcInstanceMap_[desc.funcPtr] = handle;
2343 4 : return CcuResult::CCU_SUCCESS;
2344 : }
2345 :
2346 7 : CcuResult CcuKernel::FuncDefineInArg(uint64_t handle, CcuVariableHandle formal)
2347 : {
2348 7 : PLF_CONFIG_INFO(PLF_DATA_OP, "[FuncDefineInArg] handle=%llu, formal=%llu", handle, formal);
2349 7 : auto it = funcMap_.find(handle);
2350 7 : if (it == funcMap_.end()) {
2351 0 : HCCL_ERROR("[CcuKernel::FuncDefineInArg] invalid func handle %lu", handle);
2352 0 : return CcuResult::CCU_E_PARA;
2353 : }
2354 :
2355 7 : CcuRep::Variable* formalVar = nullptr;
2356 7 : CCU_CHK_RET(GetVariableByHandle(formal, &formalVar));
2357 7 : it->second.repFuncBlock->DefineInArg(*formalVar);
2358 7 : return CcuResult::CCU_SUCCESS;
2359 : }
2360 :
2361 5 : CcuResult CcuKernel::FuncCall(uint64_t handle, const CcuVariableHandle* inArgs, uint32_t numIn)
2362 : {
2363 5 : PLF_CONFIG_INFO(PLF_DATA_OP, "[FuncCall] handle=%llu, numIn=%u", handle, numIn);
2364 5 : if (CurrentBlock()->Type() == CcuRep::CcuRepType::LOOP_BLOCK || inFuncBody_) {
2365 0 : HCCL_ERROR("[CcuKernel::FuncCall] ccu::CallFunc only allowed at top level");
2366 0 : return LatchBodyError(CcuResult::CCU_E_INTERNAL);
2367 : }
2368 5 : if (numIn > 0 && inArgs == nullptr) {
2369 0 : HCCL_ERROR("[CcuKernel::FuncCall] null input args");
2370 0 : return CcuResult::CCU_E_PTR;
2371 : }
2372 :
2373 5 : auto it = funcMap_.find(handle);
2374 5 : if (it == funcMap_.end() || !it->second.bodyDefined) {
2375 0 : HCCL_ERROR("[CcuKernel::FuncCall] invalid func handle %lu", handle);
2376 0 : return CcuResult::CCU_E_PARA;
2377 : }
2378 :
2379 5 : auto repFuncCall = std::make_shared<CcuRep::CcuRepFuncCall>(insGenerator, it->second.label);
2380 12 : for (uint32_t i = 0; i < numIn; i++) {
2381 7 : CcuRep::Variable* actual = nullptr;
2382 7 : CCU_CHK_RET(GetVariableByHandle(inArgs[i], &actual));
2383 7 : repFuncCall->SetInArg(*actual);
2384 : }
2385 5 : Append(repFuncCall);
2386 5 : return CcuResult::CCU_SUCCESS;
2387 5 : }
2388 :
2389 : // 按 maxLoopNum 把 res_.blockExecutor[0] 扩容到至少 maxLoopNum 个 LoopEngine。
2390 : // 与 CreateBlockResAssist 对齐：所有 LoopEngine 资源先落在 die0 池，待实际 die 确定后
2391 : // 再由 MoveResourcesToDie 迁移到目标 die。
2392 : // 不同 LoopGroup 通过 local loopIdx 复用同一池低位 executorId，所以这里只
2393 : // "补足"而不是"累加"。
2394 33 : CcuResult CcuKernel::EnsureLoopEnginePool(uint32_t maxLoopNum)
2395 : {
2396 33 : if (maxLoopNum == 0) {
2397 0 : HCCL_ERROR("[CcuKernel::EnsureLoopEnginePool] maxLoopNum must be > 0");
2398 0 : return CcuResult::CCU_E_PARA;
2399 : }
2400 : const uint32_t maxPoolSize
2401 33 : = (ccuVersion_ == CcuVersion::CCU_V2) ? MAX_LOOP_ENGINE_POOL_SIZE_V2 : MAX_LOOP_ENGINE_POOL_SIZE_V1;
2402 33 : if (maxLoopNum > maxPoolSize) {
2403 4 : HCCL_ERROR(
2404 : "[CcuKernel::EnsureLoopEnginePool] maxLoopNum(%u) exceeds max supported %u", maxLoopNum, maxPoolSize);
2405 4 : return CcuResult::CCU_E_PARA;
2406 : }
2407 29 : constexpr uint32_t poolDieId = 0;
2408 29 : auto& loopEnginePool = res_.blockExecutor[poolDieId];
2409 29 : if (maxLoopNum <= loopEnginePool.size()) {
2410 10 : return CcuResult::CCU_SUCCESS;
2411 : }
2412 19 : const uint32_t deficit = maxLoopNum - static_cast<uint32_t>(loopEnginePool.size());
2413 19 : std::vector<CcuRep::Executor> tmp(deficit, CcuRep::Executor(this));
2414 19 : (void)CreateBlockExecutor(deficit, tmp.data());
2415 19 : return CcuResult::CCU_SUCCESS;
2416 19 : }
2417 :
2418 18 : CcuResult CcuKernel::LoopGroupCreate(CcuLoopGroup* group, uint32_t maxLoopNum, const CcuLoopGroupCfg* cfg)
2419 : {
2420 18 : PLF_CONFIG_INFO(PLF_DATA_OP, "[LoopGroupCreate] maxLoopNum=%u", maxLoopNum);
2421 18 : if (group == nullptr || cfg == nullptr) {
2422 0 : HCCL_ERROR("[CcuKernel::LoopGroupCreate] null pointer");
2423 0 : return CcuResult::CCU_E_PTR;
2424 : }
2425 18 : if (CurrentBlock()->Type() == CcuRep::CcuRepType::LOOP_BLOCK) {
2426 0 : HCCL_ERROR("[CcuKernel::LoopGroupCreate] cannot create loop group inside a loop body");
2427 0 : return LatchBodyError(CcuResult::CCU_E_INTERNAL);
2428 : }
2429 18 : if (inFuncBody_) {
2430 0 : HCCL_ERROR("[CcuKernel::LoopGroupCreate] cannot create loop group inside a func body");
2431 0 : return LatchBodyError(CcuResult::CCU_E_INTERNAL);
2432 : }
2433 :
2434 : // 按需扩 LoopEngine 池；池足够则复用低位 executorId，跨组共享。
2435 18 : CCU_CHK_RET(EnsureLoopEnginePool(maxLoopNum));
2436 :
2437 14 : CcuLoopGroup handle = ++loopGroupHandleCounter_;
2438 :
2439 14 : LoopGroupDescriptor desc;
2440 14 : desc.config = *cfg;
2441 14 : desc.parallelVar = CreateVariable();
2442 14 : desc.offsetVar = CreateVariable();
2443 14 : desc.isVarBased = false;
2444 :
2445 14 : auto bundle = std::make_shared<CcuRep::CcuRepLoopGroupBundle>(insGenerator, *cfg, desc.parallelVar, desc.offsetVar);
2446 14 : if (ccuVersion_ == CcuVersion::CCU_V2) {
2447 4 : bundle->SetXnOffsetVar(CreateVariable());
2448 : }
2449 14 : desc.bundleRep = bundle;
2450 14 : Append(bundle);
2451 :
2452 14 : loopGroupMap_[handle] = std::move(desc);
2453 14 : *group = handle;
2454 14 : return CcuResult::CCU_SUCCESS;
2455 14 : }
2456 :
2457 11 : CcuResult CcuKernel::LoopGroupCreateFromVar(
2458 : CcuLoopGroup* group, uint32_t maxLoopNum, CcuVariableHandle parallelVarHandle, CcuVariableHandle offsetVarHandle)
2459 : {
2460 11 : PLF_CONFIG_INFO(
2461 : PLF_DATA_OP, "[LoopGroupCreateFromVar] maxLoopNum=%u, parallelVarHandle=%llu, offsetVarHandle=%llu", maxLoopNum,
2462 : parallelVarHandle, offsetVarHandle);
2463 11 : if (group == nullptr) {
2464 0 : HCCL_ERROR("[CcuKernel::LoopGroupCreateFromVar] null pointer for group");
2465 0 : return CcuResult::CCU_E_PTR;
2466 : }
2467 11 : if (CurrentBlock()->Type() == CcuRep::CcuRepType::LOOP_BLOCK) {
2468 0 : HCCL_ERROR("[CcuKernel::LoopGroupCreateFromVar] cannot create loop group inside a loop body");
2469 0 : return LatchBodyError(CcuResult::CCU_E_INTERNAL);
2470 : }
2471 11 : if (inFuncBody_) {
2472 0 : HCCL_ERROR("[CcuKernel::LoopGroupCreateFromVar] cannot create loop group inside a func body");
2473 0 : return LatchBodyError(CcuResult::CCU_E_INTERNAL);
2474 : }
2475 :
2476 11 : CCU_CHK_RET(EnsureLoopEnginePool(maxLoopNum));
2477 :
2478 11 : CcuRep::Variable* parallelVarPtr = nullptr;
2479 11 : CcuRep::Variable* offsetVarPtr = nullptr;
2480 11 : CCU_CHK_RET(GetVariableByHandle(parallelVarHandle, ¶llelVarPtr));
2481 11 : CCU_CHK_RET(GetVariableByHandle(offsetVarHandle, &offsetVarPtr));
2482 :
2483 11 : CcuLoopGroup handle = ++loopGroupHandleCounter_;
2484 :
2485 11 : LoopGroupDescriptor desc;
2486 11 : desc.parallelVar = CcuRep::Variable(*parallelVarPtr);
2487 11 : desc.offsetVar = CcuRep::Variable(*offsetVarPtr);
2488 11 : desc.isVarBased = true;
2489 :
2490 11 : auto bundle = std::make_shared<CcuRep::CcuRepLoopGroupBundle>(insGenerator, desc.parallelVar, desc.offsetVar);
2491 11 : if (ccuVersion_ == CcuVersion::CCU_V2) {
2492 1 : bundle->SetCompatRemapVars(CreateVariable(), CreateVariable());
2493 : }
2494 11 : desc.bundleRep = bundle;
2495 11 : Append(bundle);
2496 :
2497 11 : loopGroupMap_[handle] = std::move(desc);
2498 11 : *group = handle;
2499 11 : return CcuResult::CCU_SUCCESS;
2500 11 : }
2501 :
2502 5 : CcuResult CcuKernel::LoopGroupCreateFromVarV2(
2503 : CcuLoopGroup* group, uint32_t maxLoopNum, CcuVariableHandle parallelVarV2Handle,
2504 : CcuVariableHandle offsetVarV2Handle, CcuVariableHandle varOffsetVarHandle)
2505 : {
2506 5 : if (ccuVersion_ != CcuVersion::CCU_V2) {
2507 1 : HCCL_ERROR("[CcuKernel::LoopGroupCreateFromVarV2] only supported on V2");
2508 1 : return CcuResult::CCU_E_NOT_SUPPORT;
2509 : }
2510 4 : if (group == nullptr) {
2511 0 : HCCL_ERROR("[CcuKernel::LoopGroupCreateFromVarV2] null pointer for group");
2512 0 : return CcuResult::CCU_E_PTR;
2513 : }
2514 4 : if (CurrentBlock()->Type() == CcuRep::CcuRepType::LOOP_BLOCK) {
2515 0 : HCCL_ERROR("[CcuKernel::LoopGroupCreateFromVarV2] cannot create loop group inside a loop body");
2516 0 : return LatchBodyError(CcuResult::CCU_E_INTERNAL);
2517 : }
2518 4 : if (inFuncBody_) {
2519 0 : HCCL_ERROR("[CcuKernel::LoopGroupCreateFromVarV2] cannot create loop group inside a func body");
2520 0 : return LatchBodyError(CcuResult::CCU_E_INTERNAL);
2521 : }
2522 :
2523 4 : CCU_CHK_RET(EnsureLoopEnginePool(maxLoopNum));
2524 :
2525 4 : CcuRep::Variable* parallelVarPtr = nullptr;
2526 4 : CcuRep::Variable* offsetVarPtr = nullptr;
2527 4 : CcuRep::Variable* xnOffsetVarPtr = nullptr;
2528 4 : CCU_CHK_RET(GetVariableByHandle(parallelVarV2Handle, ¶llelVarPtr));
2529 4 : CCU_CHK_RET(GetVariableByHandle(offsetVarV2Handle, &offsetVarPtr));
2530 4 : CCU_CHK_RET(GetVariableByHandle(varOffsetVarHandle, &xnOffsetVarPtr));
2531 :
2532 4 : CcuLoopGroup handle = ++loopGroupHandleCounter_;
2533 :
2534 4 : LoopGroupDescriptor desc;
2535 4 : desc.parallelVar = CcuRep::Variable(*parallelVarPtr);
2536 4 : desc.offsetVar = CcuRep::Variable(*offsetVarPtr);
2537 4 : desc.xnOffsetVar = CcuRep::Variable(*xnOffsetVarPtr);
2538 4 : desc.isVarBased = true;
2539 4 : desc.isVersionV2 = true;
2540 :
2541 4 : auto bundle = std::make_shared<CcuRep::CcuRepLoopGroupBundle>(insGenerator, desc.parallelVar, desc.offsetVar);
2542 4 : bundle->SetLayout(CcuRep::CcuRepLoopGroupBundle::Layout::VersionV2);
2543 4 : bundle->SetXnOffsetVar(desc.xnOffsetVar);
2544 4 : desc.bundleRep = bundle;
2545 4 : Append(bundle);
2546 :
2547 4 : loopGroupMap_[handle] = std::move(desc);
2548 4 : *group = handle;
2549 4 : return CcuResult::CCU_SUCCESS;
2550 4 : }
2551 :
2552 57 : CcuResult CcuKernel::LookupLoopGroupAndLoop(
2553 : CcuLoopGroup group, CcuLoop loop, const char* fnName, const char* createFnName, LoopGroupDescriptor*& grpDesc,
2554 : LoopDescriptor*& loopDesc, uint32_t& loopIdx)
2555 : {
2556 57 : auto grpIt = loopGroupMap_.find(group);
2557 57 : if (grpIt == loopGroupMap_.end()) {
2558 0 : HCCL_ERROR("[CcuKernel::%s] invalid group handle %lu", fnName, group);
2559 0 : return CcuResult::CCU_E_PARA;
2560 : }
2561 57 : grpDesc = &grpIt->second;
2562 :
2563 57 : auto loopIt = loopMap_.find(loop);
2564 57 : if (loopIt == loopMap_.end()) {
2565 0 : HCCL_ERROR("[CcuKernel::%s] invalid loop handle %lu", fnName, loop);
2566 0 : return CcuResult::CCU_E_PARA;
2567 : }
2568 57 : loopDesc = &loopIt->second;
2569 :
2570 57 : if (!loopDesc->bodyDefined) {
2571 0 : HCCL_ERROR("[CcuKernel::%s] loop %lu body not defined", fnName, loop);
2572 0 : return CcuResult::CCU_E_INTERNAL; // CCU_E_LOOP_BODY_UNDEFINED
2573 : }
2574 :
2575 57 : auto& loopEnginePool = res_.blockExecutor[0];
2576 57 : loopIdx = grpDesc->loopCount;
2577 57 : if (loopIdx >= loopEnginePool.size()) {
2578 0 : HCCL_ERROR(
2579 : "[CcuKernel::%s] loopEngine pool exhausted (pool size %zu, loopIdx %u). "
2580 : "Pass a larger maxLoopNum to %s so the pool can be extended at create time.",
2581 : fnName, loopEnginePool.size(), loopIdx, createFnName);
2582 0 : return CcuResult::CCU_E_PARA;
2583 : }
2584 57 : return CcuResult::CCU_SUCCESS;
2585 : }
2586 :
2587 26 : CcuResult CcuKernel::LoopGroupAddLoop(CcuLoopGroup group, CcuLoop loop, const CcuLoopCfg* cfg)
2588 : {
2589 26 : PLF_CONFIG_INFO(PLF_DATA_OP, "[LoopGroupAddLoop] group=%llu, loop=%llu", group, loop);
2590 26 : if (cfg == nullptr) {
2591 0 : HCCL_ERROR("[CcuKernel::LoopGroupAddLoop] null pointer for cfg");
2592 0 : return CcuResult::CCU_E_PTR;
2593 : }
2594 26 : LoopGroupDescriptor* grpDesc = nullptr;
2595 26 : LoopDescriptor* loopDesc = nullptr;
2596 26 : uint32_t loopIdx = 0;
2597 26 : CCU_CHK_RET(
2598 : LookupLoopGroupAndLoop(group, loop, "LoopGroupAddLoop", "CcuLoopGroupCreate", grpDesc, loopDesc, loopIdx));
2599 26 : auto& loopEnginePool = res_.blockExecutor[0];
2600 :
2601 26 : grpDesc->loopCount++;
2602 26 : grpDesc->totalLoopNum = grpDesc->loopCount;
2603 :
2604 26 : CcuRep::CcuRepLoopGroupBundle::LoopEntry entry;
2605 26 : entry.config = *cfg;
2606 26 : entry.executor = loopEnginePool[loopIdx];
2607 26 : entry.repLoopBlock = loopDesc->repLoopBlock;
2608 26 : entry.loopParamVar = CreateVariable();
2609 26 : entry.layout = CcuRep::CcuRepLoopGroupBundle::Layout::Config;
2610 26 : if (ccuVersion_ == CcuVersion::CCU_V2) {
2611 7 : entry.iterNumVar = CreateVariable();
2612 7 : entry.addrOffsetVar = CreateVariable();
2613 7 : entry.ctxIdVar = CreateVariable();
2614 : }
2615 :
2616 26 : auto bundle = std::static_pointer_cast<CcuRep::CcuRepLoopGroupBundle>(grpDesc->bundleRep);
2617 26 : bundle->AddLoop(entry);
2618 :
2619 26 : if (!grpDesc->isVarBased) {
2620 20 : bundle->SetRepeatLoopIdx(grpDesc->config.cloneLoopOffset);
2621 20 : bundle->SetTotalLoopNum(grpDesc->totalLoopNum);
2622 : }
2623 :
2624 26 : return CcuResult::CCU_SUCCESS;
2625 26 : }
2626 :
2627 23 : CcuResult CcuKernel::LoopGroupAddLoopFromVar(CcuLoopGroup group, CcuLoop loop, CcuVariableHandle loopParamVarHandle)
2628 : {
2629 23 : PLF_CONFIG_INFO(
2630 : PLF_DATA_OP, "[LoopGroupAddLoopFromVar] group=%llu, loop=%llu, loopParamVarHandle=%llu", group, loop,
2631 : loopParamVarHandle);
2632 23 : LoopGroupDescriptor* grpDesc = nullptr;
2633 23 : LoopDescriptor* loopDesc = nullptr;
2634 23 : uint32_t loopIdx = 0;
2635 23 : CCU_CHK_RET(LookupLoopGroupAndLoop(
2636 : group, loop, "LoopGroupAddLoopFromVar", "CcuLoopGroupCreateFromVar", grpDesc, loopDesc, loopIdx));
2637 23 : auto& loopEnginePool = res_.blockExecutor[0];
2638 :
2639 23 : CcuRep::Variable* loopParamVarPtr = nullptr;
2640 23 : CCU_CHK_RET(GetVariableByHandle(loopParamVarHandle, &loopParamVarPtr));
2641 :
2642 23 : CcuRep::CcuRepLoopGroupBundle::LoopEntry entry;
2643 23 : entry.executor = loopEnginePool[loopIdx];
2644 23 : entry.repLoopBlock = loopDesc->repLoopBlock;
2645 23 : entry.loopParamVar = CcuRep::Variable(*loopParamVarPtr);
2646 23 : entry.layout = CcuRep::CcuRepLoopGroupBundle::Layout::PackedVar;
2647 23 : if (ccuVersion_ == CcuVersion::CCU_V2) {
2648 4 : entry.iterNumVar = CreateVariable();
2649 4 : entry.addrOffsetVar = CreateVariable();
2650 4 : entry.ctxIdVar = CreateVariable();
2651 : }
2652 :
2653 23 : auto bundle = std::static_pointer_cast<CcuRep::CcuRepLoopGroupBundle>(grpDesc->bundleRep);
2654 23 : bundle->AddLoop(entry);
2655 :
2656 : // 计数放在入 bundle 之后统一更新：totalLoopNum 仅供 config 组同步编码使用，与 AddLoop 无先后依赖。
2657 23 : grpDesc->loopCount++;
2658 23 : grpDesc->totalLoopNum = grpDesc->loopCount;
2659 23 : if (!grpDesc->isVarBased) {
2660 7 : bundle->SetRepeatLoopIdx(grpDesc->config.cloneLoopOffset);
2661 7 : bundle->SetTotalLoopNum(grpDesc->totalLoopNum);
2662 : }
2663 :
2664 23 : return CcuResult::CCU_SUCCESS;
2665 23 : }
2666 :
2667 8 : CcuResult CcuKernel::LoopGroupAddLoopFromVarV2(
2668 : CcuLoopGroup group, CcuLoop loop, CcuVariableHandle iterNumVarHandle, CcuVariableHandle addrOffsetVarHandle,
2669 : CcuVariableHandle ctxIdVarHandle)
2670 : {
2671 8 : if (ccuVersion_ != CcuVersion::CCU_V2) {
2672 0 : HCCL_ERROR("[CcuKernel::LoopGroupAddLoopFromVarV2] only supported on V2");
2673 0 : return CcuResult::CCU_E_NOT_SUPPORT;
2674 : }
2675 :
2676 8 : LoopGroupDescriptor* grpDesc = nullptr;
2677 8 : LoopDescriptor* loopDesc = nullptr;
2678 8 : uint32_t loopIdx = 0;
2679 8 : CCU_CHK_RET(LookupLoopGroupAndLoop(
2680 : group, loop, "LoopGroupAddLoopFromVarV2", "CcuLoopGroupCreateFromVarV2", grpDesc, loopDesc, loopIdx));
2681 8 : auto& loopEnginePool = res_.blockExecutor[0];
2682 :
2683 8 : CcuRep::Variable* iterNumVarPtr = nullptr;
2684 8 : CcuRep::Variable* addrOffsetVarPtr = nullptr;
2685 8 : CcuRep::Variable* ctxIdVarPtr = nullptr;
2686 8 : CCU_CHK_RET(GetVariableByHandle(iterNumVarHandle, &iterNumVarPtr));
2687 8 : CCU_CHK_RET(GetVariableByHandle(addrOffsetVarHandle, &addrOffsetVarPtr));
2688 8 : CCU_CHK_RET(GetVariableByHandle(ctxIdVarHandle, &ctxIdVarPtr));
2689 :
2690 8 : grpDesc->loopCount++;
2691 8 : grpDesc->totalLoopNum = grpDesc->loopCount;
2692 :
2693 8 : VersionV2LoopRecord record;
2694 8 : record.iterNumVar = CcuRep::Variable(*iterNumVarPtr);
2695 8 : record.addrOffsetVar = CcuRep::Variable(*addrOffsetVarPtr);
2696 8 : record.ctxIdVar = CcuRep::Variable(*ctxIdVarPtr);
2697 8 : grpDesc->versionV2Loops.push_back(record);
2698 :
2699 8 : CcuRep::CcuRepLoopGroupBundle::LoopEntry entry;
2700 8 : entry.executor = loopEnginePool[loopIdx];
2701 8 : entry.repLoopBlock = loopDesc->repLoopBlock;
2702 8 : entry.iterNumVar = CcuRep::Variable(*iterNumVarPtr);
2703 8 : entry.addrOffsetVar = CcuRep::Variable(*addrOffsetVarPtr);
2704 8 : entry.ctxIdVar = CcuRep::Variable(*ctxIdVarPtr);
2705 8 : entry.layout = CcuRep::CcuRepLoopGroupBundle::Layout::VersionV2;
2706 :
2707 8 : auto bundle = std::static_pointer_cast<CcuRep::CcuRepLoopGroupBundle>(grpDesc->bundleRep);
2708 8 : bundle->AddLoop(entry);
2709 :
2710 8 : if (!grpDesc->isVarBased) {
2711 2 : bundle->SetRepeatLoopIdx(grpDesc->config.cloneLoopOffset);
2712 2 : bundle->SetTotalLoopNum(grpDesc->totalLoopNum);
2713 : }
2714 :
2715 8 : return CcuResult::CCU_SUCCESS;
2716 8 : }
2717 :
2718 47 : void CcuKernel::SetInstrId(uint32_t instrId) { instrInfo_.startInstrId = instrId; }
2719 :
2720 94 : uint32_t CcuKernel::GetInstrId() const { return instrInfo_.startInstrId; }
2721 :
2722 146 : uint32_t CcuKernel::GetInstrCount()
2723 : {
2724 146 : uint32_t instrCount = 0;
2725 3444 : for (const auto& rep : GetRepSequence()) {
2726 3298 : instrCount += rep->InstrCount();
2727 : }
2728 146 : instrInfo_.instrCount = instrCount;
2729 146 : HCCL_INFO("Kernel inst %u", instrCount);
2730 146 : return instrCount;
2731 : }
2732 :
2733 : namespace {
2734 : // 识别"会翻译出 waitCKEId!=0 && clearType=1 的 set/clearCKE (即后端优化 ExtractOperandsV2 眼里
2735 : // 的 CKE 读者+写者)"的 rep. 后端优化 (CkeOnly) 会为每个此类 CKE 读者最多补 (CCU_CKE_RAW_LATENCY-1)
2736 : // 条 NOP, 故每个此类 rep 都要按 CCU_CKE_RAW_LATENCY 预留指令空间, 保证优化后指令数不越界.
2737 : // 下列每类 rep 恰好翻译出 1 条此形态指令 (RECORD_SHARED_NOTIFY 仅跨 die 分支出, 本 die 分支
2738 : // waitCKEId=0 不是读者; 因预留为静态统计无法预知 die, 保守全计, 顶多多留 L 条, 无害), 故计 1 次即可:
2739 : // - 三种 wait 类 rep: waitCKEId=事件/通知/信号 CKE, profiling->setcke, 非 profiling->clearcke;
2740 : // - LOAD / LOAD_VAR / STORE / STORE_VAR: 搬运后一条 SetCKE(waitCKEId=commSignal) 等搬运完成;
2741 : // - RECORD_SHARED_NOTIFY (跨 die): store 对端 cke 后一条 SetCKE(waitCKEId=commSignal) 等 store 完成.
2742 : // commSignal 在 kernel/die 级共享同一 CKE, 多个搬运类 rep 的 SetCKE 会构成写后读链, 触发补 NOP,
2743 : // 故必须与三种 wait 类 rep 同等预留. 详见 ccu_ins_generator_v2.cc 各 Translate 与 extract_operands.cc.
2744 1089 : bool IsCkeWaitRep(const std::shared_ptr<CcuRep::CcuRepBase>& rep)
2745 : {
2746 1089 : if (rep == nullptr) {
2747 0 : return false;
2748 : }
2749 1089 : switch (rep->Type()) {
2750 36 : case CcuRep::CcuRepType::LOC_WAIT_EVENT:
2751 : case CcuRep::CcuRepType::LOC_WAIT_NOTIFY:
2752 : case CcuRep::CcuRepType::REM_WAIT_SEM:
2753 : case CcuRep::CcuRepType::LOAD:
2754 : case CcuRep::CcuRepType::LOAD_VAR:
2755 : case CcuRep::CcuRepType::STORE:
2756 : case CcuRep::CcuRepType::STORE_VAR:
2757 : case CcuRep::CcuRepType::RECORD_SHARED_NOTIFY:
2758 36 : return true;
2759 1053 : default:
2760 1053 : return false;
2761 : }
2762 : }
2763 :
2764 : // 与 CcuKernelMgr::PrepareConstValueResources 一致地下钻 block 子 rep, 统计其中会翻译出
2765 : // waitCKEId!=0 && clearType=1 的 set/clearCKE (CKE 读者+写者) 的 rep 个数 (判定见 IsCkeWaitRep).
2766 : // 只下钻一层是充分的: 表示层禁止 block 嵌套 —— CcuKernel::LoopCreate / FuncBlockBegin /
2767 : // FuncBlockLookup 在 CurrentBlock() 为 LOOP_BLOCK 或 inFuncBody_ 时直接报错 (见 ccu_kernel.cc),
2768 : // 故 LOOP_BLOCK / FUNC_BLOCK 内不可能再出现 block 类型子 rep, 无需递归下钻.
2769 : // 若后续放开 block 嵌套, 此处 (与 PrepareConstValueResources) 需同步改为递归统计, 否则会漏计.
2770 999 : uint32_t CountCkeWaitRepInBlock(const std::shared_ptr<CcuRep::CcuRepBase>& rep)
2771 : {
2772 999 : const auto repType = rep->Type();
2773 999 : if (repType != CcuRep::CcuRepType::BLOCK && repType != CcuRep::CcuRepType::FUNC_BLOCK
2774 999 : && repType != CcuRep::CcuRepType::LOOP_BLOCK) {
2775 945 : return 0;
2776 : }
2777 54 : auto* blockPtr = static_cast<CcuRep::CcuRepBlock*>(rep.get());
2778 54 : if (blockPtr == nullptr) {
2779 0 : return 0;
2780 : }
2781 54 : uint32_t count = 0;
2782 144 : for (const auto& child : blockPtr->GetReps()) {
2783 90 : if (IsCkeWaitRep(child)) {
2784 18 : count++;
2785 : }
2786 : }
2787 54 : return count;
2788 : }
2789 : } // namespace
2790 :
2791 : // 统计当前 kernel 中"需要按 cke 写后读 latency 补 NOP"的 rep 个数 (含 block 子 rep).
2792 : // 凡会翻译出 waitCKEId!=0 && clearType=1 的 set/clearCKE (后端优化眼里的 CKE 读者) 的 rep 均计入
2793 : // (判定集合见 IsCkeWaitRep: 三种 wait 类 + LOAD/LOAD_VAR/STORE/STORE_VAR/RECORD_SHARED_NOTIFY),
2794 : // 每个此类 rep 恰出 1 条此形态指令, 后端优化最多为该读者补 (CCU_CKE_RAW_LATENCY - 1) 条 NOP,
2795 : // 上层据此为每个此类 rep 预留同等指令空间, 保证优化后指令数不越界.
2796 160 : uint32_t CcuKernel::GetRepNeedToAddLatency() const
2797 : {
2798 : // cke 写后读补 NOP 与指令空间预留只属于 A6(CCU_V2) 后端优化; A5(CCU_V1) 不跑后端优化,
2799 : // 不做任何预留, 直接返回 0, 避免影响 A5 的申请/释放口径.
2800 160 : if (ccuVersion_ != CcuVersion::CCU_V2) {
2801 109 : return 0;
2802 : }
2803 51 : uint32_t count = 0;
2804 : // GetRepSequence() 语义纯读但尚未标 const, 此处仅遍历不修改, 去 const 调用以保持本方法 const 契约.
2805 1050 : for (const auto& rep : const_cast<CcuKernel*>(this)->GetRepSequence()) {
2806 999 : if (rep == nullptr) {
2807 0 : continue;
2808 : }
2809 999 : if (IsCkeWaitRep(rep)) {
2810 18 : count++;
2811 : }
2812 999 : count += CountCkeWaitRepInBlock(rep);
2813 : }
2814 51 : HCCL_INFO("[CcuKernel] cke wait rep count %u (reserve %u instrs)", count, count * CcuRep::CCU_CKE_RAW_LATENCY);
2815 51 : return count;
2816 : }
2817 :
2818 46 : void CcuKernel::SetCcuInstrInfo(const CcuRep::CcuInstrInfo& instrInfo) { this->instrInfo_ = instrInfo; }
2819 :
2820 361 : CcuRep::Variable CcuKernel::CreateVariable()
2821 : {
2822 361 : const auto& var = CreateBlockResAssist(1, res_.continuousVariable);
2823 722 : return var[0];
2824 361 : }
2825 :
2826 109 : CcuRep::Variable CcuKernel::CreateExpectVar()
2827 : {
2828 : // v2(A6) 的 jump 只支持 var-var 比较，需要真实 XN 承载立即数；
2829 : // v1(A5) 的 jump 直接支持 var-imm，expectVar 不参与翻译，
2830 : // 用不进 res_ 账本的壳 Variable 即可，避免 v1 资源虚高。
2831 109 : if (ccuVersion_ == CcuVersion::CCU_V2) {
2832 20 : return CreateVariable();
2833 : }
2834 89 : return CcuRep::Variable(this);
2835 : }
2836 :
2837 126 : CcuRep::Variable CcuKernel::CreateJumpTargetVar()
2838 : {
2839 126 : if (ccuVersion_ == CcuVersion::CCU_V1) {
2840 : // A5(CCU_V1): 没有指令重排，可复用同一个 Xn 存放跳转目标地址。targetVar 仅用于 LoadImdToXn + 紧接 Jump。
2841 102 : if (!sharedJumpTargetVar_) {
2842 14 : sharedJumpTargetVar_ = std::make_unique<CcuRep::Variable>(CreateVariable());
2843 : }
2844 102 : return *sharedJumpTargetVar_;
2845 : }
2846 24 : return CcuRep::CreateVariable(this);
2847 : }
2848 :
2849 203 : CcuRep::Address CcuKernel::CreateAddress()
2850 : {
2851 203 : if (ccuVersion_ == CcuVersion::CCU_V2) {
2852 : // A6创建Address时，需要添加到Variable的列表中，但是仍以Address返回
2853 91 : return CcuRep::Address(CreateResAssist(res_.continuousVariable));
2854 : }
2855 112 : return CreateResAssist(res_.blockAddress);
2856 : }
2857 :
2858 0 : CcuRep::LocalNotify CcuKernel::CreateLocalNotify() { return CreateResAssist(res_.localNotify); }
2859 :
2860 0 : CcuRep::CompletedEvent CcuKernel::CreateCompletedEvent() { return CreateResAssist(res_.blockCompletedEvent); }
2861 :
2862 0 : CcuRep::CcuBuf CcuKernel::CreateCcuBuf()
2863 : {
2864 0 : const auto& buffer = CreateBlockResAssist(1, res_.blockCcubufs);
2865 0 : return buffer[0];
2866 0 : }
2867 :
2868 0 : CcuRep::Executor CcuKernel::CreateExecutor() { return CreateResAssist(res_.blockExecutor); }
2869 :
2870 156 : CcuRep::LocalAddr CcuKernel::CreateLocalAddr() { return CcuRep::LocalAddr(CreateAddress(), CreateVariable()); }
2871 :
2872 37 : CcuRep::RemoteAddr CcuKernel::CreateRemoteAddr() { return CcuRep::RemoteAddr(CreateAddress(), CreateVariable()); }
2873 :
2874 0 : CcuRep::RemoteAddr CcuKernel::GetRemoteAddr(const ChannelHandle channel, uint32_t index)
2875 : {
2876 : (void)index;
2877 0 : channels_.insert(channel);
2878 0 : auto mem = CcuRep::RemoteAddr(CreateAddress(), CreateVariable());
2879 0 : Append(std::make_shared<CcuRep::CcuRepRemMem>(insGenerator, channel, mem));
2880 0 : return mem;
2881 0 : }
2882 :
2883 0 : CcuRep::LocalAddr CcuKernel::CreateLocalAddr(const CcuRep::Variable& token)
2884 : {
2885 0 : return CcuRep::LocalAddr(CreateAddress(), token);
2886 : }
2887 :
2888 0 : HcclResult CcuKernel::CreateBlockCcuBuf(const uint32_t count, CcuRep::CcuBuf* ccuBufs)
2889 : {
2890 0 : CHK_PTR_NULL(ccuBufs);
2891 0 : auto resources = CreateBlockResAssist(count, res_.blockCcubufs);
2892 :
2893 0 : for (uint32_t i = 0; i < count; i++) {
2894 0 : ccuBufs[i] = resources[i]; // 拷贝虚拟资源，通过shared_ptr链接到物理资源
2895 : }
2896 :
2897 0 : return HcclResult::HCCL_SUCCESS;
2898 0 : }
2899 :
2900 19 : HcclResult CcuKernel::CreateBlockExecutor(const uint32_t count, CcuRep::Executor* ccuExes)
2901 : {
2902 19 : CHK_PTR_NULL(ccuExes);
2903 19 : auto resources = CreateBlockResAssist(count, res_.blockExecutor);
2904 :
2905 177 : for (uint32_t i = 0; i < count; i++) {
2906 158 : ccuExes[i] = resources[i]; // 拷贝虚拟资源，通过shared_ptr链接到物理资源
2907 : }
2908 :
2909 19 : return HcclResult::HCCL_SUCCESS;
2910 19 : }
2911 :
2912 0 : HcclResult CcuKernel::CreateBlockCompletedEvent(const uint32_t count, CcuRep::CompletedEvent* ccuEvents)
2913 : {
2914 0 : CHK_PTR_NULL(ccuEvents);
2915 0 : auto resources = CreateBlockResAssist(count, res_.blockCompletedEvent);
2916 :
2917 0 : for (uint32_t i = 0; i < count; i++) {
2918 0 : ccuEvents[i] = resources[i]; // 拷贝虚拟资源，通过shared_ptr链接到物理资源
2919 : }
2920 :
2921 0 : return HcclResult::HCCL_SUCCESS;
2922 0 : }
2923 :
2924 47 : void CcuKernel::SetResRepository(const CcuResRepository& resRepo) { resRepo_ = resRepo; }
2925 :
2926 47 : CcuResRepository& CcuKernel::GetResRepository() { return resRepo_; }
2927 :
2928 47 : CcuSharedResource& CcuKernel::GetExportedRes() { return exportedRes_; }
2929 :
2930 47 : CcuSharedResource& CcuKernel::GetImportedRes() { return importedRes_; }
2931 :
2932 10 : static HcclResult GetArgIndex(
2933 : const std::unordered_map<uint16_t, uint16_t>& varId2VarIdMap,
2934 : const std::unordered_map<uint16_t, uint32_t>& varId2ArgIndexMap, const uint64_t* taskArgs, uint32_t argSize,
2935 : uint16_t varId, uint64_t& argIndex)
2936 : {
2937 10 : HCCL_INFO("[GetArgIndex] Enter varId(%u)", varId);
2938 10 : auto item = varId2ArgIndexMap.find(varId);
2939 10 : if (item == varId2ArgIndexMap.end()) {
2940 0 : uint16_t oriVarId = varId;
2941 0 : auto iter = varId2VarIdMap.find(varId);
2942 0 : while (iter != varId2VarIdMap.end()) { // 循环查找中间assign Rep，找到起始varId
2943 0 : oriVarId = iter->second;
2944 0 : iter = varId2VarIdMap.find(oriVarId);
2945 : }
2946 0 : if (oriVarId != varId) { // 起始varId预期通过LoadArg赋值
2947 0 : item = varId2ArgIndexMap.find(oriVarId);
2948 0 : if (item == varId2ArgIndexMap.end()) {
2949 0 : HCCL_ERROR("[%s]fail, Invalid goSize variable id(%u), oriVarId = %u", __func__, varId, oriVarId);
2950 0 : return HCCL_E_PARA;
2951 : }
2952 : } else {
2953 0 : HCCL_ERROR("[%s]fail, Invalid goSize variable id(%u)", __func__, varId);
2954 0 : return HCCL_E_PARA;
2955 : }
2956 : }
2957 10 : HCCL_INFO("[GetArgIndex] find end");
2958 10 : if (item->second >= argSize) {
2959 0 : HCCL_ERROR("Invalid goSize variable index(%u).", item->second);
2960 0 : return HCCL_E_PARA;
2961 : }
2962 10 : HCCL_INFO(
2963 : "GetArgIndex success: varId(%u) varId2VarIdMapSize(%u) varId2ArgIndexMapSize(%u) taskArgsSize(%u)", varId,
2964 : varId2VarIdMap.size(), varId2ArgIndexMap.size(), argSize);
2965 10 : argIndex = taskArgs[item->second];
2966 10 : return HCCL_SUCCESS;
2967 : }
2968 :
2969 0 : void DumpCcuProfilingInfo(const std::vector<CcuProfilingInfo>& ccuProfilingInfo)
2970 : {
2971 0 : auto dumpLinkInfo = [](const CcuProfilingInfo& info) -> void {
2972 0 : for (int i = 0; i < CCU_MAX_CHANNEL_NUM; i++) {
2973 0 : if (info.channelId[i] == INVALID_VALUE_CHANNELID) {
2974 0 : continue;
2975 : }
2976 0 : HCCL_INFO("channelId(%u), remoteRankId(%u).", info.channelId[i], info.remoteRankId[i]);
2977 : }
2978 0 : };
2979 :
2980 0 : for (const auto& profInfo : ccuProfilingInfo) {
2981 0 : if (profInfo.type == static_cast<uint8_t>(CcuProfilinType::CCU_TASK_PROFILING)) {
2982 0 : HCCL_INFO(
2983 : "Dump CCU Profiling Info:SQE Profiling Info: ctxSignature(%s), "
2984 : "dieId(%d), missionId(%d), instrId(%d).",
2985 : profInfo.name.c_str(), static_cast<int>(profInfo.dieId), static_cast<int>(profInfo.missionId),
2986 : static_cast<int>(profInfo.instrId));
2987 0 : } else if (profInfo.type == static_cast<uint8_t>(CcuProfilinType::CCU_WAITCKE_PROFILING)) {
2988 0 : HCCL_INFO(
2989 : "Microcode WaitCKE Profiling Info: name(%s), "
2990 : "dieId(%d), missionId(%d), instrId(%d), ckeId(%u), mask(%u).",
2991 : profInfo.name.c_str(), static_cast<int>(profInfo.dieId), static_cast<int>(profInfo.missionId),
2992 : static_cast<int>(profInfo.instrId), profInfo.ckeId, profInfo.mask);
2993 0 : dumpLinkInfo(profInfo);
2994 0 : } else if (profInfo.type == static_cast<uint8_t>(CcuProfilinType::CCU_LOOPGROUP_PROFILING)) {
2995 0 : HCCL_INFO(
2996 : "Microcode LoopGroup Profiling Info: name(%s), "
2997 : "dieId(%d), missionId(%d), instrId(%d), reduceOpType(%d), inputDataType(%d), "
2998 : "outputDataType(%d), dataSize(%llu).",
2999 : profInfo.name.c_str(), static_cast<int>(profInfo.dieId), static_cast<int>(profInfo.missionId),
3000 : static_cast<int>(profInfo.instrId), static_cast<int>(profInfo.reduceOpType),
3001 : static_cast<int>(profInfo.inputDataType), static_cast<int>(profInfo.outputDataType), profInfo.dataSize);
3002 0 : dumpLinkInfo(profInfo);
3003 : }
3004 : }
3005 0 : }
3006 :
3007 2 : constexpr uint64_t SetBits(uint16_t end) { return ((uint64_t(1) << (end + 1)) - uint64_t(1)); }
3008 :
3009 2 : static uint16_t ParseRepeatNumFromParallelParam(uint64_t parallelParam)
3010 : {
3011 2 : constexpr uint16_t repeatBitNum = 7; // 7： repeat num 占 7 bits
3012 2 : constexpr uint16_t repeatNumShiftBit = 55; // 55： repeat num占[61:55]位置
3013 2 : return (parallelParam >> repeatNumShiftBit) & SetBits(repeatBitNum);
3014 : }
3015 :
3016 0 : HcclResult CcuKernel::CollectSqeAndWaitCkeProfilingInfo()
3017 : {
3018 0 : auto& ccuProfilingCache = GetProfilingInfo();
3019 0 : uint32_t count{0};
3020 0 : HCCL_INFO("[GetCcuProfilingInfo] Process sqe&waitcke profiling info start.");
3021 0 : for (auto& profInfo : ccuProfilingCache) {
3022 0 : profInfo.missionId = GetMissionId();
3023 0 : if (profInfo.type == static_cast<uint8_t>(hcomm::CcuProfilinType::CCU_TASK_PROFILING)) {
3024 0 : profInfo.instrId = GetInstrId();
3025 0 : allCcuProfilingInfos_.push_back(profInfo);
3026 0 : continue;
3027 : }
3028 0 : if (count >= GetWaiteCkeProfilingReps().size()) {
3029 0 : HCCL_ERROR(
3030 : "count[%u] out of range[0, %u], cache size(%u).", count, GetWaiteCkeProfilingReps().size(),
3031 : ccuProfilingCache.size());
3032 0 : return HCCL_E_INTERNAL;
3033 : }
3034 0 : auto waitCkeRep = GetWaiteCkeProfilingReps()[count];
3035 0 : profInfo.instrId = waitCkeRep->StartInstrId();
3036 0 : if (profInfo.ckeId == INVALID_CKE_ID) { // localWait Rep
3037 0 : if (waitCkeRep.get() == nullptr) {
3038 0 : HCCL_ERROR("[GetCcuProfilingInfo] localWaitRep is nullptr.");
3039 0 : return HCCL_E_PTR;
3040 : }
3041 0 : profInfo.ckeId = waitCkeRep->GetId();
3042 0 : HCCL_INFO("[CcuKernel][GetCcuProfilingInfo] waitcke[%u]", profInfo.ckeId);
3043 : }
3044 0 : allCcuProfilingInfos_.push_back(profInfo);
3045 0 : count++;
3046 0 : }
3047 0 : return HCCL_SUCCESS;
3048 : }
3049 :
3050 0 : HcclResult CcuKernel::BuildLoopGroupVarIdMaps(
3051 : std::unordered_map<uint16_t, uint32_t>& varId2ArgIndexMap, std::unordered_map<uint16_t, uint16_t>& varId2VarIdMap)
3052 : {
3053 0 : auto& lgProfInfo = GetLGProfilingInfo();
3054 0 : HCCL_INFO("[GetCcuProfilingInfo] create varId2ArgIndexMap start. size=%lu", lgProfInfo.loadRep2ArgIdxMap.size());
3055 0 : for (auto& iter : lgProfInfo.loadRep2ArgIdxMap) {
3056 0 : if (iter.first.get() == nullptr) {
3057 0 : HCCL_ERROR("[GetCcuProfilingInfo] loadRep is nullptr.");
3058 0 : return HCCL_E_PTR;
3059 : }
3060 0 : auto loadRep = dynamic_cast<CcuRep::CcuRepLoadArg*>(iter.first.get());
3061 0 : varId2ArgIndexMap[loadRep->GetVarId()] = iter.second;
3062 : }
3063 :
3064 0 : HCCL_INFO("[GetCcuProfilingInfo] create varId2VarIdMap start. size=%lu", lgProfInfo.assignProfilingReps.size());
3065 0 : for (auto& iter : lgProfInfo.assignProfilingReps) {
3066 0 : if (iter.get() == nullptr) {
3067 0 : HCCL_ERROR("[GetCcuProfilingInfo] assignRep is nullptr.");
3068 0 : return HCCL_E_PTR;
3069 : }
3070 0 : auto assignRep = dynamic_cast<CcuRep::CcuRepAssign*>(iter.get());
3071 0 : varId2VarIdMap[assignRep->GetVarB().Id()] = assignRep->GetVarA().Id();
3072 : }
3073 0 : return HCCL_SUCCESS;
3074 : }
3075 :
3076 8 : HcclResult CcuKernel::CollectLoopGroupProfilingInfo(
3077 : const uint64_t* taskArgs, uint32_t argSize, const std::unordered_map<uint16_t, uint32_t>& varId2ArgIndexMap,
3078 : const std::unordered_map<uint16_t, uint16_t>& varId2VarIdMap)
3079 : {
3080 8 : auto& lgProfInfo = GetLGProfilingInfo();
3081 8 : HCCL_INFO(
3082 : "[GetCcuProfilingInfo] process loop group profiling start: "
3083 : "lgsize(%lu), goSize(%lu)",
3084 : lgProfInfo.lgProfilingReps.size(), groupOpSizeInfo_.size());
3085 : // lgProfilingReps、groupOpSizeInfo_、ccuProfilingInfos 设计上同步 push；
3086 : // 但 CcuRepContext::AddProfiling 可能只 push ccuProfilingInfos 而不 push lgProfilingReps，
3087 : // 取三者最小值作为循环安全上界，防止任一容器较短时下标越界。
3088 8 : size_t repSize = lgProfInfo.lgProfilingReps.size();
3089 8 : size_t goSizeNum = groupOpSizeInfo_.size();
3090 8 : size_t profSize = lgProfInfo.ccuProfilingInfos.size();
3091 8 : size_t safeSize = (repSize < goSizeNum) ? repSize : goSizeNum;
3092 8 : safeSize = (profSize < safeSize) ? profSize : safeSize;
3093 8 : if (safeSize != repSize) {
3094 1 : HCCL_WARNING(
3095 : "[CollectLoopGroupProfilingInfo] size mismatch: lgReps[%zu], goSize[%zu], ccuProf[%zu],"
3096 : " use safeSize[%zu].",
3097 : repSize, goSizeNum, profSize, safeSize);
3098 : }
3099 15 : for (uint32_t i = 0; i < safeSize; i += 2) { // 2: 一个goSize对应一个CcuProfilingInfo，对应1个loopGroup Rep
3100 7 : CHK_RET(CollectSingleLoopGroupProfiling(i, repSize, taskArgs, argSize, varId2ArgIndexMap, varId2VarIdMap));
3101 : }
3102 8 : return HCCL_SUCCESS;
3103 : }
3104 :
3105 7 : HcclResult CcuKernel::CollectSingleLoopGroupProfiling(
3106 : uint32_t i, size_t repSize, const uint64_t* taskArgs, uint32_t argSize,
3107 : const std::unordered_map<uint16_t, uint32_t>& varId2ArgIndexMap,
3108 : const std::unordered_map<uint16_t, uint16_t>& varId2VarIdMap)
3109 : {
3110 7 : if (argSize == 0 || varId2ArgIndexMap.empty()) {
3111 3 : return HCCL_SUCCESS;
3112 : }
3113 4 : auto& lgProfInfo = GetLGProfilingInfo();
3114 4 : uint64_t loopParam{0};
3115 4 : CHK_RET(
3116 : GetArgIndex(varId2VarIdMap, varId2ArgIndexMap, taskArgs, argSize, groupOpSizeInfo_[i].loopParamId, loopParam));
3117 4 : uint64_t parallelParam{0};
3118 4 : CHK_RET(GetArgIndex(
3119 : varId2VarIdMap, varId2ArgIndexMap, taskArgs, argSize, groupOpSizeInfo_[i].parallelParamId, parallelParam));
3120 4 : HCCL_INFO(
3121 : "Collect loopgroup profiling info: repSize[%u], index[%u],"
3122 : "loopParam[%llu], parallelParam[%llu].",
3123 : lgProfInfo.lgProfilingReps.size(), i, loopParam, parallelParam);
3124 :
3125 4 : if (loopParam != 0) {
3126 1 : lgProfInfo.ccuProfilingInfos[i].dataSize = loopParam * moConfig_.loopCount * moConfig_.memSlice;
3127 1 : lgProfInfo.ccuProfilingInfos[i].instrId
3128 1 : = dynamic_cast<CcuRep::CcuRepLoopGroupBundle*>(lgProfInfo.lgProfilingReps[i].get())->StartInstrId();
3129 1 : allCcuProfilingInfos_.push_back(lgProfInfo.ccuProfilingInfos[i]);
3130 : }
3131 :
3132 4 : if (parallelParam != 0) {
3133 2 : HCCL_INFO("[GetCcuProfilingInfo] collect lg, residual start i=%lu", i);
3134 2 : uint64_t residual{0};
3135 3 : CHK_RET(GetArgIndex(
3136 : varId2VarIdMap, varId2ArgIndexMap, taskArgs, argSize, groupOpSizeInfo_[i].residualId, residual));
3137 2 : uint64_t repeatNum = ParseRepeatNumFromParallelParam(parallelParam);
3138 2 : lgProfInfo.ccuProfilingInfos[i].dataSize = repeatNum * moConfig_.memSlice + residual;
3139 : // rep[i] 对应 loopParam 分支，rep[i+1] 对应 parallelParam 分支；尾项无配对 rep 时跳过避免越界
3140 2 : if (i + 1 >= repSize) {
3141 1 : HCCL_WARNING(
3142 : "[CollectLoopGroupProfilingInfo] parallelParam != 0 but no paired rep,"
3143 : " index(%u), repSize(%zu), skip parallelParam profiling.",
3144 : i, repSize);
3145 1 : return HCCL_SUCCESS;
3146 : }
3147 1 : lgProfInfo.ccuProfilingInfos[i].instrId
3148 1 : = dynamic_cast<CcuRep::CcuRepLoopGroupBundle*>(lgProfInfo.lgProfilingReps[i + 1].get())->StartInstrId();
3149 1 : allCcuProfilingInfos_.push_back(lgProfInfo.ccuProfilingInfos[i]);
3150 : }
3151 3 : return HCCL_SUCCESS;
3152 : }
3153 :
3154 : /*
3155 : * variable/maskSignal等资源变量Id，一定要在获取ccu profiling时才获取；
3156 : * 原因：在创建context Rep时，其资源Id属于虚拟资源；翻译时，才会绑定固定的物理资源。
3157 : */
3158 0 : HcclResult CcuKernel::GetCcuProfilingInfo(
3159 : const uint64_t* taskArgs, uint32_t argSize, std::vector<CcuProfilingInfo>& allCcuProfilingInfo)
3160 : {
3161 0 : HCCL_INFO("[GetCcuProfilingInfo] Enter.");
3162 0 : allCcuProfilingInfos_.clear();
3163 :
3164 0 : CHK_RET(CollectSqeAndWaitCkeProfilingInfo());
3165 :
3166 0 : std::unordered_map<uint16_t, uint32_t> varId2ArgIndexMap;
3167 0 : std::unordered_map<uint16_t, uint16_t> varId2VarIdMap;
3168 0 : CHK_RET(BuildLoopGroupVarIdMaps(varId2ArgIndexMap, varId2VarIdMap));
3169 :
3170 0 : CHK_RET(CollectLoopGroupProfilingInfo(taskArgs, argSize, varId2ArgIndexMap, varId2VarIdMap));
3171 :
3172 0 : DumpCcuProfilingInfo(allCcuProfilingInfos_);
3173 0 : allCcuProfilingInfo = allCcuProfilingInfos_;
3174 0 : return HCCL_SUCCESS;
3175 0 : }
3176 :
3177 0 : HcclResult CcuKernel::AddProfilingInfo(
3178 : const ChannelHandle* channels, uint32_t channelNum, HcclDataType dataType, HcclDataType outputDataType,
3179 : HcclReduceOp opType, const std::string& opName)
3180 : {
3181 0 : CHK_PTR_NULL(channels);
3182 0 : ccuProfilingInfoCache.type = static_cast<uint8_t>(CcuProfilinType::CCU_LOOPGROUP_PROFILING);
3183 0 : ccuProfilingInfoCache.name = opName;
3184 0 : ccuProfilingInfoCache.reduceOpType = opType;
3185 0 : ccuProfilingInfoCache.inputDataType = dataType;
3186 0 : ccuProfilingInfoCache.outputDataType = outputDataType;
3187 0 : ccuProfilingInfoCache.missionId = GetMissionId();
3188 :
3189 0 : CHK_SAFETY_FUNC_RET(memset_s(
3190 : ccuProfilingInfoCache.channelId, sizeof(ccuProfilingInfoCache.channelId), INVALID_VALUE_CHANNELID,
3191 : sizeof(ccuProfilingInfoCache.channelId)));
3192 0 : for (uint32_t i = 0; i < channelNum; i++) {
3193 0 : void* channelPtr{nullptr};
3194 0 : CHK_RET(static_cast<HcclResult>(HcommChannelGet(channels[i], &channelPtr)));
3195 0 : auto* channelImpl = dynamic_cast<CcuUrmaChannel*>(static_cast<Channel*>(channelPtr));
3196 0 : CHK_PTR_NULL(channelImpl);
3197 0 : ccuProfilingInfoCache.channelId[i] = channelImpl->GetChannelId();
3198 0 : ccuProfilingInfoCache.channelHandle[i] = channels[i];
3199 0 : HCCL_INFO(
3200 : "[%s]type[%d], name[%s], opType[%d], dataType[%d], outputDataType[%d], missionId[%u], "
3201 : "channelHandle[0x%llx], channelId[%u]",
3202 : __func__, ccuProfilingInfoCache.type, ccuProfilingInfoCache.name.c_str(), opType, dataType, outputDataType,
3203 : ccuProfilingInfoCache.missionId, ccuProfilingInfoCache.channelHandle[i],
3204 : ccuProfilingInfoCache.channelId[i]);
3205 : }
3206 0 : lgProfilingInfo.ccuProfilingInfos.push_back(ccuProfilingInfoCache);
3207 0 : lgProfilingInfo.lgProfilingReps.push_back(allLgProfilingReps.back());
3208 0 : return HCCL_SUCCESS;
3209 : }
3210 :
3211 0 : HcclResult CcuKernel::AddCcuProfiling(
3212 : GroupInfo groupInfo, const std::vector<ChannelHandle> channelHandle, HcclDataType dataType,
3213 : HcclDataType outputDataType, HcclReduceOp opType, const std::string& opName)
3214 : {
3215 0 : CHK_RET(AddCcuProfiling(channelHandle.data(), channelHandle.size(), dataType, outputDataType, opType, opName));
3216 0 : groupOpSizeInfo_.push_back(groupInfo);
3217 0 : return HCCL_SUCCESS;
3218 : }
3219 :
3220 0 : HcclResult CcuKernel::AddCcuProfiling(
3221 : const ChannelHandle* channels, uint32_t channelNum, HcclDataType dataType, HcclDataType outputDataType,
3222 : HcclReduceOp opType, const std::string& opName)
3223 : {
3224 0 : CHK_PTR_NULL(channels);
3225 0 : CHK_RET(AddProfilingInfo(channels, channelNum, dataType, outputDataType, opType, opName));
3226 0 : return HCCL_SUCCESS;
3227 : }
3228 :
3229 5 : HcclResult CcuKernel::Add2ConstValue2VarMap(std::vector<uint64_t>& values)
3230 : {
3231 : // 记录当前context所需的常量，仅A6场景适用
3232 21 : for (uint64_t value : values) {
3233 16 : if (constValue2VarMap.find(value) == constValue2VarMap.end()) {
3234 3 : constValue2VarMap[value] = CreateVariable();
3235 : }
3236 : }
3237 5 : return HCCL_SUCCESS;
3238 : }
3239 :
3240 : }; // namespace hcomm
|
