Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include <algorithm>
12 : #include <unordered_set>
13 : #include "mc2_compont.h"
14 : #include "ccu_assist.h"
15 : #include "mc2_context.h"
16 : #include "ccu_task_arg_mc2.h"
17 : #include "coll_service_device_mode.h"
18 : #include "mc2_global_mirror_tasks.h"
19 : #include "op_params_checker.h"
20 : #include "runtime_api_exception.h"
21 :
22 : namespace Hccl {
23 :
24 257 : Mc2Compont::~Mc2Compont()
25 : {
26 257 : auto deviceLogicId = comm->GetDeviceLogicId();
27 264 : for (const auto& server : ccuServerMap) {
28 7 : auto ret = InsExeQue::DeregisterExtendInstruction(deviceLogicId, server.first);
29 21 : HCCL_INFO("[Mc2Compont:%s]Destroy ccuServer execId[%u]", __func__, server.first);
30 7 : if (ret != HcclResult::HCCL_SUCCESS) {
31 15 : HCCL_ERROR("DeregisterExtendInstruction execId[%u] failed, ret[%d]", server.first, ret);
32 : }
33 : }
34 257 : ccuServerMap.clear();
35 257 : }
36 :
37 2 : void Mc2Compont::AllocCommResource(void* mc2Tiling, void** commContext)
38 : {
39 2 : auto tilingVersion = *static_cast<uint32_t*>(mc2Tiling);
40 6 : HCCL_INFO("[Mc2Compont:%s] Tiling version [%u]", __func__, tilingVersion);
41 2 : if (tilingVersion != UNKNOWN_TILING_V1 && tilingVersion != UNKNOWN_TILING_V2) {
42 0 : THROW<NotSupportException>(StringFormat("Tiling version not support, version[%u]", tilingVersion));
43 : }
44 :
45 2 : if (comm->GetRankSize() == 1) {
46 3 : HCCL_WARNING("Comm[%s] rank size is 1, Mc2 not support", comm->GetId().c_str());
47 1 : return;
48 : }
49 :
50 1 : std::unordered_set<uint64_t> algoTemplateRequire;
51 1 : if (tilingVersion == UNKNOWN_TILING_V1) {
52 : // 申请deviceMem、通信域信息获取、commContext赋值
53 0 : Alloc();
54 : // 生成本次需要的算子模板
55 0 : GenerateAlgoTemplates(reinterpret_cast<Mc2Tiling*>(mc2Tiling), algoTemplateRequire);
56 : } else {
57 : // 申请deviceMem、通信域信息获取、commContext赋值
58 1 : AllocV2();
59 : // 生成本次需要的算子模板
60 1 : GenerateAlgoTemplatesV2(reinterpret_cast<Mc2InitTilingInner*>(mc2Tiling), algoTemplateRequire);
61 : }
62 :
63 3 : HCCL_RUN_INFO(
64 : "hcclCombinOpParam info: workSpace = [%llu], rankId = [%u], rankDim = [%u], xnAddr = [%llu], "
65 : "ckeAddr = [%llu], winSize = [%llu], windowsOut[0] = [%llu]",
66 : combinOpParam.workSpace, combinOpParam.rankId, combinOpParam.rankDim, combinOpParam.xnAddr,
67 : combinOpParam.ckeAddr, combinOpParam.winSize, combinOpParam.windowsOut[0]);
68 3 : HCCL_RUN_INFO(
69 : "opType[0] = [%u], opType[1] = [%u], opType[2] = [%u], opType[3] = [%u], opType[4] = [%u], "
70 : "opType[5] = [%u], opType[6] = [%u], opType[7] = [%u], ",
71 : combinOpParam.opType[0], combinOpParam.opType[1], combinOpParam.opType[2], combinOpParam.opType[3],
72 : combinOpParam.opType[4], combinOpParam.opType[5], combinOpParam.opType[6], combinOpParam.opType[7]);
73 3 : HCCL_RUN_INFO(
74 : "algorithmType[0] = [%u], algorithmType[1] = [%u], algorithmType[2] = [%u], algorithmType[3] = [%u], "
75 : "algorithmType[4] = [%u], algorithmType[5] = [%u], algorithmType[6] = [%u], algorithmType[7] = [%u]",
76 : combinOpParam.algorithmType[0], combinOpParam.algorithmType[1], combinOpParam.algorithmType[2],
77 : combinOpParam.algorithmType[3], combinOpParam.algorithmType[4], combinOpParam.algorithmType[5],
78 : combinOpParam.algorithmType[6], combinOpParam.algorithmType[7]);
79 1 : auto paramSize = sizeof(HcclCombinOpParam);
80 1 : if (combinOpParamBuffer == nullptr) {
81 1 : combinOpParamBuffer = std::make_shared<DevBuffer>(paramSize);
82 : }
83 2 : HrtMemcpy(
84 1 : reinterpret_cast<void*>(combinOpParamBuffer->GetAddr()), paramSize, static_cast<void*>(&combinOpParam),
85 : paramSize, RT_MEMCPY_HOST_TO_DEVICE);
86 1 : *commContext = reinterpret_cast<void*>(combinOpParamBuffer->GetAddr());
87 : // 生成ccuServer指令，将注册得到的execId保存在curExecId，GetCcuTaskInfo时通过curExecId获取TaskParam
88 1 : GenerateCcuServer(algoTemplateRequire);
89 1 : }
90 :
91 2 : static bool GetArgSizeFlag(std::vector<std::vector<CcuTaskParam>>& taskParams)
92 : {
93 : /*
94 : * 在MC2场景下，暂定只支持三种场景：1）单die且一个mission，2）双die且每die一个mission，3）单die且多个mission
95 : * 一次调用HCCL接口只能支持一种场景，其中2）包括所有双die算法，3）包括带尾块处理的算法与HalfAlltoAllV算子，1）包括其余算法
96 : * 只有场景3）时需要使argSize=1，1）和2）argSize均不变
97 : */
98 2 : std::unordered_set<uint8_t> dieIdSet;
99 2 : std::unordered_set<uint8_t> missionIdSet;
100 4 : for (auto& task : taskParams) {
101 2 : dieIdSet.emplace(task[0].dieId);
102 2 : missionIdSet.emplace(task[0].missionId);
103 6 : HCCL_INFO("TaskParam: dieId = [%u], missionId = [%u]", task[0].dieId, task[0].missionId);
104 : }
105 2 : bool oneDieOneMission = (dieIdSet.size() == 1) && (missionIdSet.size() == 1);
106 2 : bool twoDieOneMission = (dieIdSet.size() == 2) && (missionIdSet.size() == 1);
107 2 : bool oneDieMultiMission = (dieIdSet.size() == 1) && (missionIdSet.size() > 1);
108 2 : if (!oneDieOneMission && !twoDieOneMission && !oneDieMultiMission) {
109 0 : THROW<NotSupportException>(
110 0 : StringFormat("MC2 Scene cannot support: not OneDieOneMission, TwoDieOneMission or OneDieMultiMission !"));
111 : }
112 2 : return oneDieMultiMission;
113 2 : }
114 :
115 2 : std::vector<CcuTaskParam> Mc2Compont::GetCcuTaskInfo(void* tilingData)
116 : {
117 6 : HCCL_INFO("%s start.", __func__);
118 6 : HCCL_INFO("tilingData=%llu", tilingData);
119 2 : std::vector<std::vector<CcuTaskParam>> taskParams;
120 2 : std::vector<CcuTaskParam> ccuTaskParam;
121 2 : if (tilingData == nullptr) {
122 0 : return ccuTaskParam;
123 : }
124 2 : auto mc2Tiling = reinterpret_cast<Mc2Tiling*>(tilingData);
125 6 : HCCL_INFO("mc2Tiling=%s", mc2Tiling->ToString().c_str());
126 2 : auto version = mc2Tiling->version;
127 :
128 2 : if (version != UNKNOWN_TILING_V1 && version != UNKNOWN_TILING_V2) {
129 0 : THROW<NotSupportException>(StringFormat("Tiling version not support, version[%u]", version));
130 : }
131 :
132 : // 校验curExecId是否有效
133 2 : if (ccuServerMap.find(curExecId) == ccuServerMap.end()) {
134 0 : THROW<Hccl::InternalException>(
135 0 : StringFormat("CcuServer not find, curExecId[%llu], ccuServerSize[%d]", curExecId, ccuServerMap.size()));
136 : }
137 :
138 2 : CcuTaskArgMc2 ccuTaskArg(tokenInfo);
139 2 : HcclResult ret = CcuCtxMgr::GetTaskParam(comm->GetDeviceLogicId(), ccuTaskArg, curExecId, taskParams);
140 2 : if (ret != HcclResult::HCCL_SUCCESS) {
141 0 : THROW<CcuApiException>(StringFormat("GetTaskParam failed, ret[%u]", ret));
142 : }
143 2 : bool argSizeFlag = GetArgSizeFlag(taskParams);
144 3 : for (auto& task : taskParams) {
145 2 : if (task.size() != 1) {
146 1 : THROW<Hccl::InternalException>(
147 3 : StringFormat("Task Num In TaskParams Should Be 1, While It Is %d", task.size()));
148 : }
149 1 : if (argSizeFlag) {
150 0 : task[0].argSize = 1;
151 : }
152 1 : u32 notifyTimeout = comm->GetNotifyTimeoutCfg().GetNotifyTimeout();
153 1 : task[0].timeout
154 1 : = (notifyTimeout > UINT16_MAX) ? static_cast<uint16_t>(UINT16_MAX) : static_cast<uint16_t>(notifyTimeout);
155 1 : ccuTaskParam.push_back(task[0]);
156 1 : SaveMc2DfxTaskInfo(task[0], curExecId);
157 : }
158 1 : std::sort(ccuTaskParam.begin(), ccuTaskParam.end(), [](const CcuTaskParam& a, const CcuTaskParam& b) {
159 0 : return a.missionId < b.missionId;
160 : });
161 3 : HCCL_INFO("GetCcuTaskInfo success");
162 1 : return ccuTaskParam;
163 4 : }
164 :
165 1 : void Mc2Compont::Alloc()
166 : {
167 : // inputMem给算法编排使用，只需要申请一次，按照最大数据类型申请
168 1 : inputMem = std::make_shared<DevBuffer>(dataCount * DataTypeSizeGet(DataType::INT64) * comm->GetRankSize());
169 3 : HCCL_INFO("[Mc2Compont][Alloc]inputMem addr[%p] size = [%llu]", inputMem->GetAddr(), inputMem->GetSize());
170 9 : for (uint32_t i = 0; i < MAX_OP_NUM; i++) {
171 8 : combinOpParam.opType[i] = 0;
172 8 : combinOpParam.algorithmType[i] = 0;
173 : }
174 1 : if (ccuResourceAlloced) {
175 0 : return;
176 : }
177 :
178 1 : constexpr uint32_t comSyncNum = 2; // 每轮同步使用2个同步信号
179 1 : uint32_t comParamBufSize = CCU_TASK_NUM_MAX * CCU_PARAM_NUM_MAX * CCU_ONE_PARAM_SIZE;
180 1 : uint32_t comSyncBufSize = CCU_TASK_NUM_MAX * comSyncNum * CCU_ONE_PARAM_SIZE;
181 1 : workspaceBuffer = std::make_shared<DevBuffer>(MC2_WORKSPACE_SIZE);
182 1 : comParamBuffer = std::make_shared<DevBuffer>(comParamBufSize);
183 1 : comSyncBuffer = std::make_shared<DevBuffer>(comSyncBufSize);
184 :
185 1 : combinOpParam.workSpace = static_cast<uint64_t>(workspaceBuffer->GetAddr());
186 1 : combinOpParam.workSpaceSize = MC2_WORKSPACE_SIZE;
187 1 : combinOpParam.rankId = comm->GetMyRank();
188 1 : combinOpParam.rankDim = comm->GetRankSize();
189 1 : combinOpParam.xnAddr = static_cast<uint64_t>(comParamBuffer->GetAddr());
190 1 : combinOpParam.ckeAddr = static_cast<uint64_t>(comSyncBuffer->GetAddr());
191 : // add cclbuffer info
192 1 : if (comm->GetCclBuffer() == nullptr) {
193 0 : THROW<Hccl::InternalException>(StringFormat("Cannot get CCL Buffer to fill window!"));
194 : }
195 1 : combinOpParam.winSize = static_cast<uint64_t>(comm->GetCclBuffer()->GetSize());
196 1 : combinOpParam.windowsOut[0] = static_cast<uint64_t>(comm->GetCclBuffer()->GetAddr());
197 1 : ccuResourceAlloced = true;
198 :
199 1 : tokenInfo = CcuRep::GetTokenInfo(
200 1 : static_cast<uint64_t>(workspaceBuffer->GetAddr()), static_cast<uint64_t>(workspaceBuffer->GetSize()));
201 : }
202 :
203 1 : void Mc2Compont::AllocV2()
204 : {
205 1 : inputMem = std::make_shared<DevBuffer>(dataCount * DataTypeSizeGet(DataType::INT64) * comm->GetRankSize());
206 3 : HCCL_INFO("[Mc2Compont][AllocV2]inputMem addr[%p] size = [%llu]", inputMem->GetAddr(), inputMem->GetSize());
207 9 : for (uint32_t i = 0; i < MAX_OP_NUM; i++) {
208 8 : combinOpParam.opType[i] = 0;
209 8 : combinOpParam.algorithmType[i] = 0;
210 : }
211 1 : if (ccuResourceAlloced) {
212 0 : return;
213 : }
214 :
215 1 : constexpr uint32_t comSyncNum = 2; // 每轮同步使用2个同步信号
216 1 : uint32_t comParamBufSize = CCU_TASK_NUM_MAX * CCU_PARAM_NUM_MAX * CCU_ONE_PARAM_SIZE;
217 1 : uint32_t comSyncBufSize = CCU_TASK_NUM_MAX * comSyncNum * CCU_ONE_PARAM_SIZE;
218 1 : workspaceBuffer = std::make_shared<DevBuffer>(MC2_WORKSPACE_SIZE);
219 1 : comParamBuffer = std::make_shared<DevBuffer>(comParamBufSize);
220 1 : comSyncBuffer = std::make_shared<DevBuffer>(comSyncBufSize);
221 1 : HrtMemset(reinterpret_cast<void*>(workspaceBuffer->GetAddr()), MC2_WORKSPACE_SIZE, MC2_WORKSPACE_SIZE);
222 1 : HrtMemset(reinterpret_cast<void*>(comParamBuffer->GetAddr()), comParamBufSize, comParamBufSize);
223 1 : HrtMemset(reinterpret_cast<void*>(comSyncBuffer->GetAddr()), comSyncBufSize, comSyncBufSize);
224 1 : combinOpParam.workSpace = static_cast<uint64_t>(workspaceBuffer->GetAddr());
225 1 : combinOpParam.workSpaceSize = MC2_WORKSPACE_SIZE;
226 1 : combinOpParam.rankId = comm->GetMyRank();
227 1 : combinOpParam.rankDim = comm->GetRankSize();
228 1 : combinOpParam.xnAddr = static_cast<uint64_t>(comParamBuffer->GetAddr());
229 1 : combinOpParam.ckeAddr = static_cast<uint64_t>(comSyncBuffer->GetAddr());
230 : // add cclbuffer info
231 1 : if (comm->GetCclBuffer() == nullptr) {
232 0 : THROW<Hccl::InternalException>(StringFormat("Cannot get CCL Buffer to fill window!"));
233 : }
234 1 : combinOpParam.winSize = static_cast<uint64_t>(comm->GetCclBuffer()->GetSize());
235 1 : combinOpParam.windowsOut[0] = static_cast<uint64_t>(comm->GetCclBuffer()->GetAddr());
236 1 : ccuResourceAlloced = true;
237 :
238 1 : tokenInfo = CcuRep::GetTokenInfo(
239 1 : static_cast<uint64_t>(workspaceBuffer->GetAddr()), static_cast<uint64_t>(workspaceBuffer->GetSize()));
240 : }
241 :
242 2 : void Mc2Compont::MC2Orchestrate(
243 : const CollAlgParams& params, std::shared_ptr<InsQueue>& insQueue, uint8_t commEngine) const
244 : {
245 2 : auto op = comm->GetCurrentCollOperator();
246 :
247 2 : CollOpParams opParams;
248 2 : opParams.commEngine = static_cast<HcclAccelerator::Value>(commEngine);
249 2 : opParams.opType = op->opType;
250 2 : opParams.dataType = op->dataType;
251 2 : opParams.count = op->dataCount;
252 2 : opParams.reduceOp = op->reduceOp;
253 2 : opParams.isMc2 = params.isMc2;
254 2 : comm->ExecAlgSelect(opParams, op->opMode);
255 2 : if (!comm->GetOpCcuFeatureFlag()) { // 算子粒度
256 : auto msg = StringFormat(
257 : "[Mc2Compont:%s]AlgSelect not ccu, accState[%s]", __func__,
258 0 : comm->GetOpExecuteConfig().accState.Describe().c_str());
259 0 : THROW<InternalException>(msg);
260 0 : }
261 :
262 2 : std::string algName = comm->GetCurAlgName();
263 : // 算子编排获取InsQueue
264 2 : auto ret = comm->GetCollAlgComponent()->Orchestrate(*op, params, algName, insQueue);
265 2 : if (ret != HcclResult::HCCL_SUCCESS) {
266 1 : THROW<InternalException>(
267 3 : StringFormat("Error occurs when call collAlgComponent.orchestrate(), error code: %d", ret));
268 : }
269 3 : }
270 :
271 1 : void Mc2Compont::MC2AllocCommRes(
272 : const CollAlgParams& params, std::shared_ptr<InsQueue>& insQueue, uint8_t commEngine) const
273 : {
274 1 : MC2Orchestrate(params, insQueue, commEngine);
275 : // 获取LinkData
276 1 : auto collService = dynamic_cast<CollServiceDeviceMode*>(comm->GetCollService());
277 1 : auto ccuLinks = collService->GetUniqueLinks(insQueue);
278 : // Socket建链
279 1 : comm->GetSocketManager().BatchCreateSockets(ccuLinks);
280 : // 对insQueue中ccuIns进行预处理(创建transport、ccuCtx、分配资源、注册等)
281 1 : collService->GetCcuInsPreprocessor()->Preprocess(insQueue, true);
282 1 : if (collService->GetCcuInsPreprocessor()->IsRollback()) { // mc2暂不能回退到aicpu
283 1 : THROW<InternalException>("[Mc2Compont][%s]ResAlloc unsuccessful.", __func__);
284 : }
285 1 : }
286 :
287 0 : void Mc2Compont::SaveAlgoInfo(uint32_t index, uint64_t templateSign, uint32_t opType, uint8_t algorithmType)
288 : {
289 0 : combinOpParam.opType[index] = opType;
290 0 : combinOpParam.algorithmType[index] = algorithmType;
291 0 : HcclAlgoInfo hcclAlgoInfo{};
292 0 : hcclAlgoInfo.opType = opType;
293 0 : hcclAlgoInfo.algorithmType = algorithmType;
294 0 : algoInfoMap_[templateSign] = hcclAlgoInfo;
295 0 : return;
296 : }
297 :
298 0 : void Mc2Compont::GenerateAlgoTemplates(Mc2Tiling* mc2TilingPtr, std::unordered_set<uint64_t>& algoTemplateRequire)
299 : {
300 0 : HCCL_INFO("GenerateAlgoTemplates start v1");
301 :
302 0 : auto tmpMemSize = comm->GetBufferSize();
303 0 : CollAlgParams params;
304 0 : params.opMode = OpMode::OPBASE;
305 0 : params.maxTmpMemSize = tmpMemSize;
306 0 : params.isMc2 = true;
307 : // 从mc2Tiling中获取需要的算法信息
308 0 : Mc2CommConfig* commConfigPtr = reinterpret_cast<Mc2CommConfig*>(
309 : reinterpret_cast<uint8_t*>(mc2TilingPtr) + sizeof(uint32_t) + sizeof(uint32_t) + sizeof(Mc2ServerCfg));
310 0 : for (uint32_t index = 0; index < mc2TilingPtr->commConfigNum; index++) {
311 0 : const auto& commConfig = *(commConfigPtr + index);
312 0 : OpParamsChecker::CheckOpDataTypeMC2(commConfig);
313 0 : uint64_t templateSign = GetTemplateSignature(commConfig);
314 0 : algoTemplateRequire.insert(templateSign);
315 : // 已经生成过的算法模板不再生成
316 0 : if (algoTemplateMap.find(templateSign) != algoTemplateMap.end()) {
317 0 : HCCL_INFO(
318 : "A algoTemplate that meets the requirement already exists, index = [%u], templateSign = [%llu]", index,
319 : templateSign);
320 0 : if (algoInfoMap_.find(templateSign) != algoInfoMap_.end()) {
321 0 : combinOpParam.opType[index] = algoInfoMap_[templateSign].opType;
322 0 : combinOpParam.algorithmType[index] = algoInfoMap_[templateSign].algorithmType;
323 0 : continue;
324 : } else {
325 0 : THROW<Hccl::InternalException>(
326 0 : StringFormat("algoInfoMap_ do not has templateSign = [%llu]", templateSign));
327 : }
328 : }
329 :
330 0 : FillCollOperator(commConfig);
331 :
332 0 : auto insQueue = make_shared<InsQueue>();
333 0 : MC2AllocCommRes(params, insQueue, commConfig.communicationEngine);
334 :
335 0 : std::string algName = comm->GetCurAlgName();
336 0 : HCCL_INFO(
337 : "Orchestrate: index = [%u], algName = [%s], templateSign = [%llu]", index, algName.c_str(), templateSign);
338 0 : if (insQueue->Iter()->GetType() != InstructionType::CCU_INS) {
339 0 : THROW<Hccl::InternalException>(
340 0 : StringFormat("InstructionType is not ccu ins, algName = [%s]", algName.c_str()));
341 : }
342 :
343 : // 获取taskParam
344 0 : const CcuInstruction& ccuInstruction = static_cast<const CcuInstruction&>(*insQueue->Iter());
345 0 : std::vector<std::vector<CcuTaskParam>> taskParams;
346 0 : ccuInstruction.Translate(taskParams);
347 0 : if (taskParams.empty()) {
348 0 : THROW<Hccl::InternalException>(
349 0 : StringFormat("CcuInstruction translate faild, index = [%u], algName = [%s]", index, algName.c_str()));
350 : }
351 0 : algoTemplateMap[templateSign] = taskParams;
352 0 : SaveAlgoInfo(index, templateSign, commConfig.opType, comm->GetAlgorithmType());
353 0 : for (const auto& task : taskParams) {
354 0 : HCCL_INFO("taskParam: dieId = [%u], instStartId = [%u]", task[0].dieId, task[0].instStartId);
355 0 : SaveMc2DfxTaskInfo(task[0], ccuInstruction.GetExecId());
356 : }
357 0 : }
358 0 : HCCL_INFO("GenerateAlgoTemplates success");
359 0 : }
360 :
361 1 : void Mc2Compont::GenerateAlgoTemplatesV2(
362 : const Mc2InitTilingInner* mc2TilingPtr, std::unordered_set<uint64_t>& algoTemplateRequire)
363 : {
364 3 : HCCL_INFO("GenerateAlgoTemplates start v2");
365 :
366 1 : auto tmpMemSize = comm->GetBufferSize();
367 1 : CollAlgParams params;
368 1 : params.opMode = OpMode::OPBASE;
369 1 : params.maxTmpMemSize = tmpMemSize;
370 1 : params.isMc2 = true;
371 1 : if (mc2TilingPtr->mc2HcommCnt > MAX_OP_NUM) {
372 0 : THROW<Hccl::InternalException>(
373 0 : StringFormat("mc2HcommCnt is lager than MAX_OP_NUM, mc2HcommCnt = [%u]", mc2TilingPtr->mc2HcommCnt));
374 : }
375 :
376 2 : for (uint32_t index = 0; index < mc2TilingPtr->mc2HcommCnt; index++) {
377 1 : const auto offset = mc2TilingPtr->offset[index];
378 1 : const auto& commConfig
379 1 : = *(reinterpret_cast<const Mc2CcTilingInner*>(reinterpret_cast<const uint8_t*>(mc2TilingPtr) + offset));
380 1 : OpParamsChecker::CheckOpDataTypeMC2V2(commConfig);
381 1 : uint64_t templateSign = GetTemplateSignatureV2(commConfig);
382 1 : algoTemplateRequire.insert(templateSign);
383 : // 已经生成过的算法模板不再生成
384 1 : if (algoTemplateMap.find(templateSign) != algoTemplateMap.end()) {
385 3 : HCCL_INFO(
386 : "A algoTemplate that meets the requirement already exists, index = [%u], templateSign = [%llu]", index,
387 : templateSign);
388 1 : if (algoInfoMap_.find(templateSign) != algoInfoMap_.end()) {
389 1 : combinOpParam.opType[index] = algoInfoMap_[templateSign].opType;
390 1 : combinOpParam.algorithmType[index] = algoInfoMap_[templateSign].algorithmType;
391 1 : continue;
392 : } else {
393 0 : THROW<Hccl::InternalException>(
394 0 : StringFormat("algoInfoMap_ do not has templateSign = [%llu]", templateSign));
395 : }
396 : }
397 :
398 0 : FillCollOperatorV2(commConfig);
399 :
400 0 : auto insQueue = make_shared<InsQueue>();
401 0 : MC2AllocCommRes(params, insQueue, commConfig.communicationEngine);
402 :
403 0 : std::string algName = comm->GetCurAlgName();
404 0 : HCCL_INFO(
405 : "Orchestrate: index = [%u], algName = [%s], templateSign = [%llu]", index, algName.c_str(), templateSign);
406 0 : if (insQueue->Iter()->GetType() != InstructionType::CCU_INS) {
407 0 : THROW<Hccl::InternalException>(
408 0 : StringFormat("InstructionType is not ccu ins, algName = [%s]", algName.c_str()));
409 : }
410 :
411 : // 获取taskParam
412 0 : const CcuInstruction& ccuInstruction = static_cast<const CcuInstruction&>(*insQueue->Iter());
413 0 : std::vector<std::vector<CcuTaskParam>> taskParams;
414 0 : ccuInstruction.Translate(taskParams);
415 0 : if (taskParams.empty()) {
416 0 : THROW<Hccl::InternalException>(
417 0 : StringFormat("CcuInstruction translate faild, index = [%u], algName = [%s]", index, algName.c_str()));
418 : }
419 0 : algoTemplateMap[templateSign] = taskParams;
420 0 : SaveAlgoInfo(index, templateSign, commConfig.opType, comm->GetAlgorithmType());
421 0 : for (const auto& task : taskParams) {
422 0 : HCCL_INFO("taskParam: dieId = [%u], instStartId = [%u]", task[0].dieId, task[0].instStartId);
423 0 : SaveMc2DfxTaskInfo(task[0], ccuInstruction.GetExecId());
424 : }
425 0 : }
426 3 : HCCL_INFO("GenerateAlgoTemplates success");
427 1 : }
428 :
429 3 : static std::map<uint8_t, std::map<uint32_t, uint32_t>> TransToMap(const std::vector<std::vector<CcuTaskParam>>& params)
430 : {
431 3 : std::map<uint8_t, std::map<uint32_t, uint32_t>> dieIdToInstrIdMap;
432 :
433 12 : for (const auto& param : params) {
434 9 : uint8_t dieId = param[0].dieId;
435 9 : uint32_t instrId = param[0].instStartId;
436 :
437 : // 检查 dieId 是否已经存在于 map 中
438 9 : if (dieIdToInstrIdMap.find(dieId) == dieIdToInstrIdMap.end()) {
439 : // 如果不存在，创建一个新的 map
440 5 : std::map<uint32_t, uint32_t> indexIdToInstrIdMap;
441 5 : indexIdToInstrIdMap[0] = instrId;
442 5 : dieIdToInstrIdMap[dieId] = indexIdToInstrIdMap;
443 5 : } else {
444 : // 如果存在，获取对应的 map
445 4 : std::map<uint32_t, uint32_t>& indexIdToInstrIdMap = dieIdToInstrIdMap[dieId];
446 : // 插入新的 IndexId 和 InstrId
447 4 : int indexId = indexIdToInstrIdMap.size();
448 4 : indexIdToInstrIdMap[indexId] = instrId;
449 : }
450 : }
451 3 : return dieIdToInstrIdMap;
452 0 : }
453 :
454 : static std::map<uint8_t, std::vector<uint32_t>>
455 2 : TransToDieIdMissionIdMap(const std::vector<std::vector<CcuTaskParam>>& params)
456 : {
457 2 : std::map<uint8_t, std::vector<uint32_t>> dieIdMissionIdMap;
458 :
459 7 : for (const auto& param : params) {
460 5 : uint8_t dieId = param[0].dieId;
461 5 : uint32_t missionId = param[0].missionId;
462 :
463 5 : if (dieIdMissionIdMap.find(dieId) == dieIdMissionIdMap.end()) {
464 3 : dieIdMissionIdMap[dieId] = std::vector<uint32_t>();
465 : }
466 5 : dieIdMissionIdMap[dieId].push_back(missionId);
467 : }
468 2 : return dieIdMissionIdMap;
469 0 : }
470 :
471 4 : bool Mc2Compont::CompareMissionMap(
472 : const std::map<uint8_t, std::map<uint32_t, uint32_t>>& mapA,
473 : const std::map<uint8_t, std::map<uint32_t, uint32_t>>& mapB) const
474 : {
475 4 : if (mapA.size() != mapB.size()) {
476 1 : return false;
477 : }
478 5 : for (auto& entry : mapA) {
479 4 : uint8_t curDieId = entry.first;
480 4 : if (mapB.find(curDieId) == mapB.end()) {
481 2 : return false;
482 : }
483 4 : const std::map<uint32_t, uint32_t>& curSubMapA = entry.second;
484 4 : const std::map<uint32_t, uint32_t>& curSubMapB = mapB.at(curDieId);
485 4 : if (curSubMapA.size() != curSubMapB.size()) {
486 2 : return false;
487 : }
488 6 : for (auto& elem : curSubMapA) {
489 4 : if (curSubMapB.find(elem.first) == curSubMapB.end()) {
490 0 : return false;
491 : }
492 : }
493 : }
494 1 : return true;
495 : }
496 :
497 5 : u32 Mc2Compont::GetCcuMc2ServerNum() { return ccuServerMap.size(); }
498 :
499 3 : void Mc2Compont::GenerateCcuServer(const std::unordered_set<uint64_t>& algoTemplateRequire)
500 : {
501 9 : HCCL_INFO("GenerateCcuServer start");
502 3 : if (algoTemplateRequire.empty()) {
503 0 : THROW<InvalidParamsException>(StringFormat("AlgoTemplate require num is zero!"));
504 : }
505 :
506 : InsExeQue::ExtInsExeEntityId execId;
507 : // 查找当前是否存在符合条件的ccuServer
508 3 : if (FindCcuServer(algoTemplateRequire, execId)) {
509 1 : curExecId = execId;
510 3 : HCCL_INFO("A CcuServer that meets the requirement already exists, execId = [%llu]", execId);
511 1 : return;
512 : }
513 :
514 : // 没有符合条件的ccuServer, 生成一个新的ccuServer
515 2 : std::map<uint64_t, std::map<uint8_t, std::map<uint32_t, uint32_t>>> signatureMap;
516 2 : std::map<uint8_t, std::map<uint32_t, uint32_t>> compareMap;
517 2 : std::map<uint8_t, std::vector<uint32_t>> dieIdMissionIdMap;
518 2 : bool initFlag = false;
519 5 : for (uint64_t templateSignature : algoTemplateRequire) {
520 3 : auto tmpMap = TransToMap(algoTemplateMap[templateSignature]);
521 3 : if (!initFlag) {
522 2 : compareMap = tmpMap;
523 2 : dieIdMissionIdMap = TransToDieIdMissionIdMap(algoTemplateMap[templateSignature]);
524 2 : initFlag = true;
525 : } else {
526 1 : if (!CompareMissionMap(compareMap, tmpMap)) {
527 0 : THROW<InvalidParamsException>(StringFormat("AlgoTemplate require is not the same!"));
528 : }
529 : }
530 : // algoTemplateRequire为unordered_set，可以保证键值唯一
531 3 : signatureMap[templateSignature] = tmpMap;
532 3 : }
533 :
534 2 : std::map<uint8_t, std::map<uint32_t, std::map<uint64_t, uint32_t>>> algoTemplate;
535 5 : for (const auto& signature : signatureMap) {
536 8 : for (const auto& dieId : signature.second) {
537 14 : for (const auto& indexId : dieId.second) {
538 9 : algoTemplate[dieId.first][indexId.first][signature.first] = indexId.second;
539 : }
540 : }
541 : }
542 :
543 : // 实例化Mc2Context
544 2 : CcuCtxGroup ctxGroup;
545 2 : uint32_t dieNum = algoTemplate.size();
546 5 : for (const auto& item : algoTemplate) {
547 3 : uint8_t dieId = item.first;
548 8 : for (const auto& mission : item.second) {
549 5 : std::unique_ptr<Mc2ContextBase> mc2Context;
550 5 : if (mission.first == 0) {
551 3 : mc2Context = std::make_unique<Mc2Context>();
552 3 : mc2Context->SetDieId(dieId);
553 3 : static_cast<Mc2Context*>(mc2Context.get())->SetDieNum(dieNum);
554 3 : static_cast<Mc2Context*>(mc2Context.get())
555 3 : ->SetCommAddr(
556 3 : static_cast<uint64_t>(comSyncBuffer->GetAddr()),
557 3 : static_cast<uint64_t>(comParamBuffer->GetAddr()));
558 : } else {
559 2 : mc2Context = std::make_unique<Mc2SlaveContext>();
560 2 : mc2Context->SetDieId(dieId);
561 : }
562 5 : mc2Context->SetMissionNumAndId(item.second.size(), mission.first);
563 5 : mc2Context->SetAlgoTemplateInfo(mission.second);
564 5 : ctxGroup.ctxs.push_back(std::move(mc2Context));
565 5 : }
566 : }
567 :
568 : // 申请资源
569 2 : HcclResult ret = CcuCtxMgr::AllocRes(comm->GetDeviceLogicId(), ctxGroup, ccuResPack);
570 2 : if (ret != HcclResult::HCCL_SUCCESS) {
571 0 : THROW<CcuApiException>("GenerateCcuServer AllocRes failed, ret[%d]", ret);
572 : }
573 :
574 : // needtodo 检查各个templateSignature对应的missionId相同
575 2 : std::map<uint8_t, uint32_t> dieIndex;
576 7 : for (auto& ctx : ctxGroup.ctxs) {
577 5 : if (dieIndex.find(ctx->GetDieId()) == dieIndex.end()) {
578 3 : dieIndex[ctx->GetDieId()] = 0;
579 : }
580 5 : ctx->SetMissionId(dieIdMissionIdMap[ctx->GetDieId()][dieIndex[ctx->GetDieId()]++]);
581 : }
582 :
583 : // 指令注册
584 2 : InsExeQue::ExtInsExeEntity entity;
585 2 : entity.ctxGroup = std::move(ctxGroup);
586 2 : ret = InsExeQue::RegisterExtendInstruction(comm->GetDeviceLogicId(), entity, execId);
587 2 : if (ret != HcclResult::HCCL_SUCCESS) {
588 0 : THROW<CcuApiException>("GenerateCcuServer RegisterExtendInstruction failed, ret[%d]", ret);
589 : }
590 2 : ccuServerMap[execId] = algoTemplateRequire;
591 2 : curExecId = execId;
592 6 : HCCL_INFO("GenerateCcuServer success, execId[%llu]", execId);
593 2 : }
594 :
595 1 : void Mc2Compont::DisplayRPCMsg() const
596 : {
597 1 : void* xnAddr = nullptr;
598 : try {
599 1 : xnAddr = HrtMallocHost(comParamBuffer->GetSize() + comSyncBuffer->GetSize());
600 1 : void* ckeAddr = reinterpret_cast<uint8_t*>(xnAddr) + comParamBuffer->GetSize();
601 2 : HrtMemcpy(
602 1 : xnAddr, comParamBuffer->GetSize(), reinterpret_cast<uint8_t*>(comParamBuffer->GetAddr()),
603 1 : comParamBuffer->GetSize(), RT_MEMCPY_DEVICE_TO_HOST);
604 2 : HrtMemcpy(
605 1 : ckeAddr, comSyncBuffer->GetSize(), reinterpret_cast<uint8_t*>(comSyncBuffer->GetAddr()),
606 1 : comSyncBuffer->GetSize(), RT_MEMCPY_DEVICE_TO_HOST);
607 0 : } catch (const RuntimeApiException& e) {
608 0 : HCCL_ERROR("[Mc2Compont:%s] host memory operation fail: %s", __func__, e.what());
609 0 : if (xnAddr != nullptr) {
610 0 : HrtFreeHost(xnAddr);
611 : }
612 0 : return;
613 0 : }
614 1 : void* ckeAddr = reinterpret_cast<uint8_t*>(xnAddr) + comParamBuffer->GetSize();
615 1 : uint64_t* xnBaseAddr = reinterpret_cast<uint64_t*>(xnAddr);
616 1 : uint64_t* waitCkeAddr = reinterpret_cast<uint64_t*>(ckeAddr);
617 1 : uint64_t* setCkeAddr = reinterpret_cast<uint64_t*>(ckeAddr) + CCU_TASK_NUM_MAX;
618 1 : uint32_t xnNumPerMsg = CCU_PARAM_NUM_MAX;
619 65 : for (uint32_t msgId = 0; msgId < CCU_TASK_NUM_MAX; msgId++) {
620 64 : uint64_t* xnMsg = xnBaseAddr + msgId * xnNumPerMsg;
621 64 : uint64_t* waitCkeMsg = waitCkeAddr + msgId;
622 64 : uint64_t* setCkeMsg = setCkeAddr + msgId;
623 : // print cke value and xn value
624 192 : HCCL_ERROR(
625 : "msgId[%u]: waitCkeAddr=[%p], setCkeAddr=[%p], waitCke=[%llu], setCke=[%llu], xnAddr=[%p]", msgId,
626 : waitCkeMsg, setCkeMsg, *waitCkeMsg, *setCkeMsg, xnMsg);
627 64 : std::string tmpXnInfo = "xnMsg=[";
628 4160 : for (uint32_t xnId = 0; xnId < xnNumPerMsg; xnId++) {
629 4096 : tmpXnInfo += StringFormat("%llx", *(xnMsg + xnId));
630 4096 : if (xnId != xnNumPerMsg - 1) {
631 4032 : tmpXnInfo += ", ";
632 : }
633 : }
634 64 : tmpXnInfo += "]";
635 192 : HCCL_ERROR("[Mc2Compont:%s] RPC MsgId[%u], %s", __func__, msgId, tmpXnInfo.c_str());
636 64 : }
637 1 : HrtFreeHost(xnAddr);
638 : }
639 :
640 3 : bool Mc2Compont::FindCcuServer(
641 : const std::unordered_set<uint64_t>& algoTemplateRequire, InsExeQue::ExtInsExeEntityId& execId) const
642 : {
643 : // 查找是否有符合的ccuServer存在，本次算子需求是已有sever中包含算子的子集既符合要求
644 3 : for (const auto& server : ccuServerMap) {
645 1 : bool isMatch = true;
646 2 : for (auto& templateSign : algoTemplateRequire) {
647 1 : if (server.second.find(templateSign) == server.second.end()) {
648 0 : isMatch = false;
649 0 : break;
650 : }
651 : }
652 1 : if (isMatch) {
653 1 : execId = server.first;
654 1 : return true;
655 : }
656 : }
657 2 : return false;
658 : }
659 :
660 0 : uint64_t Mc2Compont::GetTemplateSignature(const Mc2CommConfig& config) const
661 : {
662 : // 根据Mc2CommConfig生成算子模板签名
663 0 : if (config.opType > UINT8_MAX || config.reduceType > UINT8_MAX || config.dataType > UINT8_MAX
664 0 : || config.outputDataType > UINT8_MAX) {
665 0 : THROW<InvalidParamsException>(
666 0 : StringFormat("MC2 High Level API GetTemplateSignature Failed, Mc2CommConfig value is bigger than 256!"));
667 : }
668 0 : constexpr uint16_t algoTypeShift = 32;
669 0 : constexpr uint16_t outputDataTypeShift = 24;
670 0 : constexpr uint16_t reduceTypeShift = 16;
671 0 : constexpr uint16_t dataTypeShift = 8;
672 0 : constexpr uint16_t opTypeShift = 0;
673 0 : uint64_t opType = config.opType;
674 0 : uint64_t dataType = config.dataType;
675 0 : uint64_t reduceType = config.reduceType;
676 0 : uint64_t outputDataType = config.outputDataType;
677 0 : uint64_t algoType = 0; // 用于算法选择，当前暂不支持，固定为0
678 :
679 0 : uint64_t templateSignature = ((opType & 0xff) << opTypeShift) | ((dataType & 0xff) << dataTypeShift)
680 0 : | ((reduceType & 0xff) << reduceTypeShift)
681 0 : | ((outputDataType & 0xff) << outputDataTypeShift)
682 0 : | ((algoType & 0xff) << algoTypeShift);
683 0 : HCCL_INFO(
684 : "[GetTemplateSignature]: opType[%s] dataType[%s] reduceType[%s] outputDataType[%s] algoType[%u] "
685 : "templateSignature[%llu]",
686 : MC2OpType(static_cast<AicpuComType>(config.opType)).Describe().c_str(),
687 : MC2DataType(static_cast<HcclDataType>(config.dataType)).Describe().c_str(),
688 : MC2ReduceType(static_cast<HcclReduceOp>(config.reduceType)).Describe().c_str(),
689 : MC2DataType(static_cast<HcclDataType>(config.outputDataType)).Describe().c_str(), algoType, templateSignature);
690 0 : return templateSignature;
691 : }
692 :
693 2 : uint64_t Mc2Compont::GetTemplateSignatureV2(const Mc2CcTilingInner& config) const
694 : {
695 : // 根据Mc2CommConfig生成算子模板签名
696 2 : if (config.opType > UINT8_MAX || config.reduceType > UINT8_MAX) {
697 0 : THROW<InvalidParamsException>(
698 0 : StringFormat("MC2 High Level API GetTemplateSignature Failed, Mc2CommConfig value is bigger than 256!"));
699 : }
700 2 : constexpr uint16_t algoTypeShift = 32;
701 2 : constexpr uint16_t outputDataTypeShift = 24;
702 2 : constexpr uint16_t reduceTypeShift = 16;
703 2 : constexpr uint16_t dataTypeShift = 8;
704 2 : constexpr uint16_t opTypeShift = 0;
705 2 : uint64_t opType = config.opType;
706 2 : uint64_t dataType = config.srcDataType;
707 2 : uint64_t reduceType = config.reduceType;
708 2 : uint64_t outputDataType = config.dstDataType;
709 2 : uint64_t algoType = 0; // 用于算法选择，当前暂不支持，固定为0
710 :
711 2 : uint64_t templateSignature = ((opType & 0xff) << opTypeShift) | ((dataType & 0xff) << dataTypeShift)
712 2 : | ((reduceType & 0xff) << reduceTypeShift)
713 2 : | ((outputDataType & 0xff) << outputDataTypeShift)
714 2 : | ((algoType & 0xff) << algoTypeShift);
715 6 : HCCL_INFO(
716 : "[GetTemplateSignature]: opType[%s] dataType[%s] reduceType[%s] outputDataType[%s] algoType[%u] "
717 : "templateSignature[%llu]",
718 : MC2OpType(static_cast<AicpuComType>(config.opType)).Describe().c_str(),
719 : MC2DataType(static_cast<HcclDataType>(config.srcDataType)).Describe().c_str(),
720 : MC2ReduceType(static_cast<HcclReduceOp>(config.reduceType)).Describe().c_str(),
721 : MC2DataType(static_cast<HcclDataType>(config.dstDataType)).Describe().c_str(), algoType, templateSignature);
722 2 : return templateSignature;
723 : }
724 :
725 1 : void Mc2Compont::FillCollOperatorV2(const Mc2CcTilingInner& config)
726 : {
727 1 : CollOpParams opParams;
728 1 : opParams.opType = MC2OpType(static_cast<AicpuComType>(config.opType));
729 1 : opParams.reduceOp = MC2ReduceType(static_cast<HcclReduceOp>(config.reduceType));
730 1 : opParams.dataType = MC2DataType(static_cast<HcclDataType>(config.srcDataType));
731 1 : opParams.outputDataType = MC2DataType(static_cast<HcclDataType>(config.dstDataType));
732 1 : opParams.count = dataCount;
733 1 : opParams.sendBuf = reinterpret_cast<void*>(inputMem->GetAddr());
734 1 : opParams.recvBuf = reinterpret_cast<void*>(inputMem->GetAddr());
735 1 : if (opParams.opType == OpType::ALLTOALL) {
736 0 : opParams.all2AllDataDes.sendType = opParams.dataType;
737 0 : opParams.all2AllDataDes.recvType = opParams.outputDataType;
738 0 : opParams.all2AllDataDes.sendCount = dataCount;
739 0 : opParams.all2AllDataDes.recvCount = dataCount;
740 : }
741 1 : std::string opTag = comm->GetId();
742 :
743 1 : if (opParams.opType == OpType::ALLTOALLV) {
744 1 : opParams.all2AllVDataDes.sendType = opParams.dataType;
745 1 : opParams.all2AllVDataDes.recvType = opParams.outputDataType;
746 1 : dataCounts.resize(comm->GetRankSize());
747 1 : displs.resize(comm->GetRankSize());
748 1 : u64 countSum = 0;
749 2 : for (u32 i = 0; i < comm->GetRankSize(); i++) {
750 1 : dataCounts.at(i) = 1;
751 1 : displs.at(i) = countSum++;
752 : }
753 1 : opParams.all2AllVDataDes.sendCounts = reinterpret_cast<void*>(&dataCounts[0]);
754 1 : opParams.all2AllVDataDes.recvCounts = reinterpret_cast<void*>(&dataCounts[0]);
755 1 : opParams.all2AllVDataDes.sdispls = reinterpret_cast<void*>(&displs[0]);
756 1 : opParams.all2AllVDataDes.rdispls = reinterpret_cast<void*>(&displs[0]);
757 : }
758 :
759 1 : comm->CovertToCurrentCollOperator(opTag, opParams, OpMode::OPBASE);
760 1 : }
761 :
762 1 : void Mc2Compont::FillCollOperator(const Mc2CommConfig& config)
763 : {
764 1 : CollOpParams opParams;
765 1 : opParams.opType = MC2OpType(static_cast<AicpuComType>(config.opType));
766 1 : opParams.reduceOp = MC2ReduceType(static_cast<HcclReduceOp>(config.reduceType));
767 1 : opParams.dataType = MC2DataType(static_cast<HcclDataType>(config.dataType));
768 1 : opParams.outputDataType = MC2DataType(static_cast<HcclDataType>(config.outputDataType));
769 1 : opParams.count = dataCount;
770 1 : opParams.sendBuf = reinterpret_cast<void*>(inputMem->GetAddr());
771 1 : opParams.recvBuf = reinterpret_cast<void*>(inputMem->GetAddr());
772 1 : if (opParams.opType == OpType::ALLTOALL) {
773 0 : opParams.all2AllDataDes.sendType = opParams.dataType;
774 0 : opParams.all2AllDataDes.recvType = opParams.outputDataType;
775 0 : opParams.all2AllDataDes.sendCount = dataCount;
776 0 : opParams.all2AllDataDes.recvCount = dataCount;
777 : }
778 1 : std::string opTag = comm->GetId();
779 :
780 1 : if (opParams.opType == OpType::ALLTOALLV) {
781 1 : opParams.all2AllVDataDes.sendType = opParams.dataType;
782 1 : opParams.all2AllVDataDes.recvType = opParams.outputDataType;
783 1 : dataCounts.resize(comm->GetRankSize());
784 1 : displs.resize(comm->GetRankSize());
785 1 : u64 countSum = 0;
786 2 : for (u32 i = 0; i < comm->GetRankSize(); i++) {
787 1 : dataCounts.at(i) = 1;
788 1 : displs.at(i) = countSum++;
789 : }
790 1 : opParams.all2AllVDataDes.sendCounts = reinterpret_cast<void*>(&dataCounts[0]);
791 1 : opParams.all2AllVDataDes.recvCounts = reinterpret_cast<void*>(&dataCounts[0]);
792 1 : opParams.all2AllVDataDes.sdispls = reinterpret_cast<void*>(&displs[0]);
793 1 : opParams.all2AllVDataDes.rdispls = reinterpret_cast<void*>(&displs[0]);
794 : }
795 :
796 1 : comm->CovertToCurrentCollOperator(opTag, opParams, OpMode::OPBASE);
797 1 : }
798 :
799 2 : void Mc2Compont::SaveMc2DfxTaskInfo(const CcuTaskParam& ccuTaskParam, uint64_t execId) const
800 : {
801 2 : shared_ptr<DfxOpInfo> dfxOpInfo = std::make_shared<DfxOpInfo>();
802 2 : dfxOpInfo->comm_ = comm;
803 :
804 2 : TaskParam taskParam{};
805 2 : taskParam.taskType = TaskParamType::TASK_CCU;
806 2 : taskParam.taskPara.Ccu.dieId = ccuTaskParam.dieId;
807 2 : taskParam.taskPara.Ccu.missionId = ccuTaskParam.missionId;
808 2 : taskParam.taskPara.Ccu.instrId = ccuTaskParam.instStartId;
809 2 : taskParam.taskPara.Ccu.executeId = execId;
810 :
811 2 : shared_ptr<TaskInfo> taskInfo = std::make_shared<TaskInfo>(0, 0, 0, taskParam, dfxOpInfo);
812 :
813 2 : MC2GlobalMirrorTasks::GetInstance().AddTaskInfo(comm->GetDeviceLogicId(), taskInfo);
814 2 : }
815 :
816 4 : std::vector<CcuTaskParam> Mc2Compont::GetAlgoCcuTaskInfo(InsExeQue::ExtInsExeEntityId execId) const
817 : {
818 4 : std::vector<CcuTaskParam> ccuTaskParam{};
819 4 : auto serverItor = ccuServerMap.find(execId);
820 4 : if (serverItor == ccuServerMap.end()) {
821 6 : HCCL_INFO("[Mc2Compont]Failed to find ccuServer by executeId[%llu]", execId);
822 2 : return ccuTaskParam;
823 : }
824 5 : for (uint64_t algoSign : serverItor->second) {
825 3 : auto algoTemplateItor = algoTemplateMap.find(algoSign);
826 3 : if (algoTemplateItor == algoTemplateMap.end()) {
827 3 : HCCL_INFO("[Mc2Compont]Failed to find ccuTaskParam by algoSign[%llu]", algoSign);
828 1 : continue;
829 1 : }
830 6 : for (const auto& taskParam : algoTemplateItor->second) {
831 4 : ccuTaskParam.push_back(taskParam[0]);
832 : }
833 : }
834 2 : return ccuTaskParam;
835 0 : }
836 : } // namespace Hccl
|
