Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "ccu_context_mgr_imp.h"
12 :
13 : #include <unordered_set>
14 :
15 : #include "ccu_ctx_mgr.h"
16 : #include "ccu_res_pack_legacy.h"
17 :
18 : #include "hccl_common_v2.h"
19 : #include "orion_adapter_rts.h"
20 : #include "exception_util.h"
21 : #include "ccu_api_exception.h"
22 : #include "orion_adapter_hccp.h"
23 : #include "ccu_assist.h"
24 : #include "dev_buffer.h"
25 : #include "hccp_tlv_hdc_manager.h"
26 :
27 : #ifdef HCCL_ALG_ANALYZER_DAVID
28 : #include "instruction.h"
29 : #endif
30 :
31 : namespace Hccl {
32 66 : CtxMgrImp::CtxMgrImp() {}
33 :
34 66 : CtxMgrImp::~CtxMgrImp()
35 : {
36 66 : if (initializedFlag_) {
37 0 : HCCL_INFO("[CtxMgrImp]~CtxMgrImp: deviceLogicId[%d], free addr[%p]", deviceLogicId_, instructionLoadDevMem_);
38 0 : if (instructionLoadDevMem_ != nullptr) {
39 0 : DECTOR_TRY_CATCH("CtxMgrImp", HrtFree(instructionLoadDevMem_));
40 0 : instructionLoadDevMem_ = nullptr;
41 : }
42 0 : ctxGroupMap_.clear();
43 0 : initializedFlag_ = false;
44 : }
45 66 : }
46 :
47 51 : CtxMgrImp& CtxMgrImp::GetInstance(s32 deviceLogicId)
48 : {
49 117 : static CtxMgrImp contextManager[MAX_MODULE_DEVICE_NUM + 1];
50 :
51 51 : if (deviceLogicId < 0 || static_cast<u32>(deviceLogicId) > MAX_MODULE_DEVICE_NUM) {
52 0 : THROW<CcuApiException>("ProcessSharedResources failed deviceLogicId[%d]", deviceLogicId);
53 : }
54 :
55 51 : contextManager[deviceLogicId].deviceLogicId_ = deviceLogicId;
56 51 : return contextManager[deviceLogicId];
57 : }
58 :
59 0 : void CtxMgrImp::Init()
60 : {
61 0 : std::unique_lock<std::mutex> lock(contextMapMutex_);
62 0 : if (initializedFlag_) {
63 0 : return;
64 : }
65 :
66 0 : initializedFlag_ = true;
67 0 : ctxGroupMap_.clear();
68 0 : }
69 :
70 2 : void CtxMgrImp::Deinit()
71 : {
72 2 : std::unique_lock<std::mutex> lock(contextMapMutex_);
73 2 : translatorResPack.handles.clear();
74 2 : translatorResPack.count = 0;
75 2 : initializedFlag_ = false;
76 2 : ctxGroupMap_.clear();
77 2 : translators.clear();
78 2 : referenceMgrs.clear();
79 2 : }
80 :
81 3 : HcclResult CtxMgrImp::AllocRes(CcuCtxGroup& ctxGroup, CcuResPack& resPack)
82 : {
83 3 : std::unique_lock<std::mutex> lock(contextMapMutex_);
84 : // 初始化ctx
85 3 : CtxInit(ctxGroup);
86 :
87 : // 获取ctxGroup使用到的所有dieId
88 3 : std::unordered_set<uint16_t> usedDieId;
89 9 : for (const auto& ctx : ctxGroup.ctxs) {
90 6 : usedDieId.insert(ctx->GetDieId());
91 : }
92 7 : for (auto dieId : usedDieId) {
93 : // 实例化translator，并且为RefManager和translator申请&绑定资源
94 4 : CHK_RET_UNAVAIL(InstantiationTranslator(dieId));
95 : }
96 :
97 : // 获取通信域当前所持有的资源
98 3 : CcuResReq totalRes;
99 3 : CHK_RET(GetResPackTotalResNum(resPack, totalRes));
100 :
101 : // 计算本次编排逻辑所需的资源
102 3 : CcuResReq resReq = GetCtxGroupResReq(ctxGroup);
103 :
104 : // 比较额外需要的资源
105 3 : CHK_RET_UNAVAIL(CompareResAndApplyAsNeeded(totalRes, resReq, resPack));
106 :
107 : // 申请指令空间资源
108 3 : CHK_RET_UNAVAIL(AllocInstrRes(ctxGroup));
109 :
110 : // 保存本次编排的资源信息到Ctx中
111 3 : resPack.count++;
112 3 : SaveResPackToCtx(ctxGroup, resPack);
113 9 : HCCL_INFO(
114 : "[CtxMgrImp:%s]cur resPack count[%u], resHandle[%u], handle size[%u]", __func__, resPack.count, resPack.GetId(),
115 : resPack.handles.size());
116 3 : return HcclResult::HCCL_SUCCESS;
117 3 : }
118 :
119 : // 在外侧调用的UnRegister函数中已加锁，所以当前函数不需要加锁
120 2 : HcclResult CtxMgrImp::ReleaseRes(CcuCtxGroup& ctxGroup) const
121 : {
122 : // 获取本次编排Ctx多对应的资源信息
123 2 : CcuResPack* resPack = ctxGroup.ctxs[0]->GetResPack();
124 2 : CHK_PTR_NULL(resPack);
125 6 : HCCL_INFO(
126 : "[CtxMgrImp:%s]cur resPack count[%u], resHandle[%u], handle size[%u]", __func__, resPack->count,
127 : resPack->GetId(), resPack->handles.size());
128 2 : if (resPack->count > 0) {
129 2 : resPack->count--;
130 : }
131 :
132 : // 释放资源
133 2 : if (resPack->count == 0) {
134 4 : for (CcuResHandle resHandle : resPack->handles) {
135 6 : HCCL_INFO("[CtxMgrImp]ReleaseRes: deviceLogicId[%d], resHandle[%p]", deviceLogicId_, resHandle);
136 2 : CHK_RET(CcuDeviceManager::ReleaseResHandle(deviceLogicId_, resHandle));
137 : }
138 : }
139 :
140 2 : return HcclResult::HCCL_SUCCESS;
141 : }
142 :
143 3 : uint64_t CtxMgrImp::Register(CcuCtxGroup& ctxGroup, bool isFuncBlock)
144 : {
145 : // 多通信域场景需要加锁
146 3 : std::unique_lock<std::mutex> lock(contextMapMutex_);
147 :
148 : // REP编排虚拟资源和实际物理资源映射关联
149 3 : TransRepResToPhyRes(ctxGroup);
150 :
151 : // 构造翻译器,翻译本次编排的ctxGroup
152 3 : TransRepSequenceToMicrocode(ctxGroup, isFuncBlock);
153 :
154 : // 保存本次编排的ctxGroup和exeutorId关联关系
155 3 : executorId_++; // executorId_ = executorId_ > UINT64_MAX ? 0 : executorId_++;
156 3 : ctxGroupMap_[executorId_] = std::move(ctxGroup);
157 : // 清除referenceMgr中保存的引用关系
158 7 : for (auto& referenceMgrMap : referenceMgrs) {
159 68 : for (auto& referenceMgr : referenceMgrMap.second) {
160 64 : referenceMgr.second->ClearRepReference();
161 : }
162 : }
163 3 : return executorId_;
164 3 : }
165 :
166 9 : HcclResult CtxMgrImp::UnRegister(const uint64_t executorId)
167 : {
168 9 : std::unique_lock<std::mutex> lock(contextMapMutex_);
169 :
170 : // 校验ctxGroupMap_中是否存在executorId对应的ctxGroup
171 30 : CHK_PRT_RET(
172 : ctxGroupMap_.find(executorId) == ctxGroupMap_.end(),
173 : HCCL_ERROR("[CtxMgrImp][UnRegister]executorId [%llu] is not exist", executorId), HcclResult::HCCL_E_NOT_FOUND);
174 :
175 : // 释放指令空间
176 2 : ReleaseInstrRes(ctxGroupMap_[executorId]);
177 :
178 : // 尝试释放掉ctx对应的handle
179 2 : CHK_RET(ReleaseRes(ctxGroupMap_[executorId]));
180 :
181 2 : ctxGroupMap_.erase(executorId);
182 :
183 2 : return HcclResult::HCCL_SUCCESS;
184 9 : }
185 :
186 8 : std::vector<std::vector<CcuTaskParam>> CtxMgrImp::GetTaskParam(CcuTaskArg& ccuTaskArg, const uint64_t executorId)
187 : {
188 : // 根据executorId获取ctxGroup
189 8 : std::unique_lock<std::mutex> lock(contextMapMutex_);
190 :
191 : // 校验ctxGroupMap_中是否存在executorId对应的ctxGroup
192 8 : CHK_PRT_RET(
193 : ctxGroupMap_.find(executorId) == ctxGroupMap_.end(),
194 : HCCL_ERROR("[CtxMgrImp][GetTaskParam]executorId [%llu] is not exist", executorId),
195 : std::vector<std::vector<CcuTaskParam>>());
196 :
197 : // 获取每个ctx的taskParam信息
198 8 : std::vector<std::vector<CcuTaskParam>> taskParam;
199 12 : for (auto& ctx : ctxGroupMap_[executorId].ctxs) {
200 10 : std::vector<CcuTaskParam> tmp;
201 10 : auto ret = ctx->GeneTaskParam(ccuTaskArg, tmp);
202 4 : if (ret != HcclResult::HCCL_SUCCESS) {
203 0 : THROW<CcuApiException>("GeneTaskParam is failed. ret[%d]", ret);
204 : }
205 4 : taskParam.push_back(tmp);
206 10 : }
207 :
208 2 : return taskParam;
209 14 : }
210 :
211 : // ctx初始化
212 3 : void CtxMgrImp::CtxInit(CcuCtxGroup& ctxGroup) const
213 : {
214 9 : for (auto& ctx : ctxGroup.ctxs) {
215 : // 初始化ctx
216 6 : CHK_PRT_RET_NULL(ctx->Init(), HCCL_ERROR("Init failed"));
217 : }
218 3 : return;
219 : }
220 :
221 : // 申请指令空间资源
222 3 : HcclResult CtxMgrImp::AllocInstrRes(CcuCtxGroup& ctxGroup) const
223 : {
224 9 : for (auto& ctx : ctxGroup.ctxs) {
225 6 : ResInfo insInfo(0, 0);
226 6 : uint32_t instrCount = ctx->GetInstrCount() + CcuRepTranslator::GetInstrNum();
227 6 : CHK_RET_UNAVAIL(CcuDeviceManager::AllocIns(deviceLogicId_, ctx->GetDieId(), instrCount, insInfo));
228 18 : HCCL_INFO(
229 : "[CtxMgrImp]AllocInstrRes: deviceLogicId[%d], dieId[%u], startId[%u], count[%u]", deviceLogicId_,
230 : ctx->GetDieId(), insInfo.startId, insInfo.num);
231 6 : ctx->SetInstrId(insInfo.startId);
232 : }
233 :
234 3 : return HcclResult::HCCL_SUCCESS;
235 : }
236 :
237 : // 释放指令空间资源
238 2 : HcclResult CtxMgrImp::ReleaseInstrRes(CcuCtxGroup& ctxGroup) const
239 : {
240 6 : for (auto& ctx : ctxGroup.ctxs) {
241 4 : ResInfo insInfo(ctx->GetInstrId(), (ctx->GetInstrCount() + CcuRepTranslator::GetInstrNum()));
242 12 : HCCL_INFO(
243 : "[CtxMgrImp]ReleaseInstrRes: deviceLogicId[%d], dieId[%u], startId[%u], count[%u]", deviceLogicId_,
244 : ctx->GetDieId(), insInfo.startId, insInfo.num);
245 4 : CHK_RET(CcuDeviceManager::ReleaseIns(deviceLogicId_, ctx->GetDieId(), insInfo));
246 : }
247 :
248 2 : return HcclResult::HCCL_SUCCESS;
249 : }
250 :
251 3 : HcclResult CtxMgrImp::GetResPackTotalResNum(const CcuResPack& resPack, CcuResReq& totalRes) const
252 : {
253 3 : CcuResRepository tmpResRepository;
254 :
255 : // 获取通信域当前所持有的资源
256 3 : for (CcuResHandle resHandle : resPack.handles) {
257 0 : CHK_RET(CcuDeviceManager::GetResource(deviceLogicId_, resHandle, tmpResRepository));
258 :
259 : // 合并获取的所持有的资源信息, 按照类型合并资源总和到totalRes的第0个vector中
260 0 : for (u32 i = 0; i < MAX_CCU_IODIE_NUM; i++) {
261 0 : totalRes.msReq[i] += GetResTotalNum(tmpResRepository.ms[i]);
262 0 : totalRes.blockMsReq[i] += GetResTotalNum(tmpResRepository.blockMs[i]);
263 0 : totalRes.ckeReq[i] += GetResTotalNum(tmpResRepository.cke[i]);
264 0 : totalRes.blockCkeReq[i] += GetResTotalNum(tmpResRepository.blockCke[i]);
265 0 : totalRes.loopEngineReq[i] += GetResTotalNum(tmpResRepository.loopEngine[i]);
266 0 : totalRes.blockLoopEngineReq[i] += GetResTotalNum(tmpResRepository.blockLoopEngine[i]);
267 0 : totalRes.gsaReq[i] += GetResTotalNum(tmpResRepository.gsa[i]);
268 0 : totalRes.blockGsaReq[i] += GetResTotalNum(tmpResRepository.blockGsa[i]);
269 0 : totalRes.xnReq[i] += GetResTotalNum(tmpResRepository.xn[i]);
270 0 : totalRes.blockXnReq[i] += GetResTotalNum(tmpResRepository.blockXn[i]);
271 0 : totalRes.missionReq.req[i] += GetResTotalNum(tmpResRepository.mission.mission[i]);
272 : }
273 : }
274 :
275 3 : DumpResReqInfo(totalRes);
276 9 : HCCL_INFO("GetResPackTotalResNum:dumpInfos success.");
277 3 : return HcclResult::HCCL_SUCCESS;
278 3 : }
279 :
280 3 : CcuResReq CtxMgrImp::GetCtxGroupResReq(CcuCtxGroup& ctxGroup) const
281 : {
282 3 : CcuResReq totalResReq;
283 :
284 3 : std::unordered_set<uint16_t> usedDieId; // CCUCtxGroup使用到的所有dieId
285 :
286 : // 获取CCUCtxGroup所有ctx资源诉求
287 9 : for (auto& ctx : ctxGroup.ctxs) {
288 6 : auto dieId = ctx->GetDieId();
289 6 : usedDieId.insert(dieId);
290 :
291 6 : CcuResReq tmpResReq = ctx->GetResourceRequest();
292 6 : MergeCcuResReq(totalResReq, tmpResReq);
293 : }
294 :
295 3 : DumpResReqInfo(totalResReq);
296 9 : HCCL_INFO("CtxGroupResReq:dumpInfos success.");
297 :
298 6 : return totalResReq;
299 3 : }
300 :
301 102 : void CtxMgrImp::MergeCcuResReq(CcuResReq& resReqA, const CcuResReq& resReqB) const
302 : {
303 : // 合并获取的所持有的资源信息, 按照类型合并资源总和到totalRes的第0个vector中
304 306 : for (u32 i = 0; i < MAX_CCU_IODIE_NUM; i++) {
305 204 : resReqA.msReq[i] += resReqB.msReq[i];
306 204 : resReqA.blockMsReq[i] += resReqB.blockMsReq[i];
307 204 : resReqA.ckeReq[i] += resReqB.ckeReq[i];
308 204 : resReqA.blockCkeReq[i] += resReqB.blockCkeReq[i];
309 204 : resReqA.loopEngineReq[i] += resReqB.loopEngineReq[i];
310 204 : resReqA.blockLoopEngineReq[i] += resReqB.blockLoopEngineReq[i];
311 204 : resReqA.gsaReq[i] += resReqB.gsaReq[i];
312 204 : resReqA.blockGsaReq[i] += resReqB.blockGsaReq[i];
313 204 : resReqA.xnReq[i] += resReqB.xnReq[i];
314 204 : resReqA.blockXnReq[i] += resReqB.blockXnReq[i];
315 204 : resReqA.missionReq.req[i] += resReqB.missionReq.req[i];
316 :
317 204 : if (resReqB.missionReq.req[i] > 0) {
318 6 : resReqA.missionReq.reqType = resReqB.missionReq.reqType;
319 : }
320 : }
321 102 : return;
322 : }
323 :
324 : HcclResult
325 3 : CtxMgrImp::CompareResAndApplyAsNeeded(const CcuResReq& totalRes, const CcuResReq& resReq, CcuResPack& resPack) const
326 : {
327 : // 比较额外需要的资源
328 3 : bool isNeedAlloc = false;
329 3 : CcuResReq needResReq;
330 :
331 9 : for (u32 i = 0; i < MAX_CCU_IODIE_NUM; i++) {
332 6 : needResReq.msReq[i] = GetReqResNum(resReq.msReq[i], totalRes.msReq[i]);
333 6 : needResReq.blockMsReq[i] = GetReqResNum(resReq.blockMsReq[i], totalRes.blockMsReq[i]);
334 6 : needResReq.ckeReq[i] = GetReqResNum(resReq.ckeReq[i], totalRes.ckeReq[i]);
335 6 : needResReq.blockCkeReq[i] = GetReqResNum(resReq.blockCkeReq[i], totalRes.blockCkeReq[i]);
336 6 : needResReq.loopEngineReq[i] = GetReqResNum(resReq.loopEngineReq[i], totalRes.loopEngineReq[i]);
337 6 : needResReq.blockLoopEngineReq[i] = GetReqResNum(resReq.blockLoopEngineReq[i], totalRes.blockLoopEngineReq[i]);
338 6 : needResReq.gsaReq[i] = GetReqResNum(resReq.gsaReq[i], totalRes.gsaReq[i]);
339 6 : needResReq.blockGsaReq[i] = GetReqResNum(resReq.blockGsaReq[i], totalRes.blockGsaReq[i]);
340 6 : needResReq.xnReq[i] = GetReqResNum(resReq.xnReq[i], totalRes.xnReq[i]);
341 6 : needResReq.blockXnReq[i] = GetReqResNum(resReq.blockXnReq[i], totalRes.blockXnReq[i]);
342 6 : needResReq.missionReq.req[i] = GetReqResNum(resReq.missionReq.req[i], totalRes.missionReq.req[i]);
343 :
344 6 : if (needResReq.missionReq.req[i] > 0) {
345 4 : needResReq.missionReq.reqType = resReq.missionReq.reqType;
346 : }
347 :
348 12 : if (needResReq.msReq[i] != 0 || needResReq.blockMsReq[i] != 0 || needResReq.ckeReq[i] != 0
349 3 : || needResReq.blockCkeReq[i] != 0 || needResReq.loopEngineReq[i] != 0
350 3 : || needResReq.blockLoopEngineReq[i] != 0 || needResReq.gsaReq[i] != 0 || needResReq.blockGsaReq[i] != 0
351 12 : || needResReq.xnReq[i] != 0 || needResReq.blockXnReq[i] != 0 || needResReq.missionReq.req[i] != 0) {
352 4 : isNeedAlloc = true;
353 : }
354 : }
355 :
356 3 : if (isNeedAlloc) {
357 : // 申请额外资源
358 : CcuResHandle handle;
359 3 : DumpResReqInfo(needResReq);
360 :
361 3 : CHK_RET_UNAVAIL(CcuDeviceManager::AllocResHandle(deviceLogicId_, needResReq, handle));
362 : // 将申请的资源保存到resPack中
363 3 : resPack.handles.push_back(handle);
364 :
365 9 : HCCL_INFO("ApplyAsNeeded:dumpInfos success deviceLogicId[%d] handle[%p].", deviceLogicId_, handle);
366 : }
367 :
368 3 : return HcclResult::HCCL_SUCCESS;
369 : }
370 :
371 3 : void CtxMgrImp::SaveResPackToCtx(CcuCtxGroup& ctxGroup, CcuResPack& resPack) const
372 : {
373 9 : for (auto& ctx : ctxGroup.ctxs) {
374 6 : ctx->SetResPack(resPack);
375 : }
376 3 : return;
377 : }
378 :
379 4 : HcclResult CtxMgrImp::InstantiationTranslator(uint16_t dieId)
380 : {
381 4 : if (translators.find(dieId) != translators.end()) {
382 1 : return HcclResult::HCCL_SUCCESS;
383 : }
384 :
385 3 : std::array<uint16_t, MAX_CCU_IODIE_NUM> tmpChannelId{};
386 3 : uint32_t chaneelId = 0;
387 : // 获取innerDieChannelId
388 3 : auto ret = CcuDeviceManager::GetLoopChannelId(deviceLogicId_, dieId, dieId, chaneelId);
389 3 : if (ret != HcclResult::HCCL_SUCCESS) {
390 0 : THROW<CcuApiException>(
391 : "Failed to get inner die channel id. deviceLogicId = %d, dieId = %u, ret = %d", deviceLogicId_, dieId, ret);
392 : }
393 3 : tmpChannelId[0] = chaneelId;
394 : // 获取interDieChannelId
395 3 : uint8_t dstDieId = ((dieId == 0) ? 1 : 0);
396 3 : ret = CcuDeviceManager::GetLoopChannelId(deviceLogicId_, dieId, dstDieId, chaneelId);
397 3 : if (ret != HcclResult::HCCL_SUCCESS) {
398 : // 当前验证环境为单die环境，获取die间ChannelId会失败，打印WARNING日志。
399 0 : HCCL_WARNING(
400 : "Failed to get inter die channel id. deviceLogicId = %d, srcDieId = %u, dstDieId = %u, ret = %d",
401 : deviceLogicId_, dieId, dstDieId, ret);
402 : }
403 3 : tmpChannelId[1] = chaneelId;
404 : // 获取ccu token信息
405 3 : uint64_t tokenId = 0;
406 3 : uint64_t tokenValue = 0;
407 3 : ret = CcuDeviceManager::GetCcuResourceSpaceTokenInfoForLocal(deviceLogicId_, dieId, tokenId, tokenValue);
408 3 : if (ret != HcclResult::HCCL_SUCCESS) {
409 0 : THROW<CcuApiException>(
410 : "Failed to get ccu resource space token info. deviceLogicId = %d, dieId = %u, ret = %d", deviceLogicId_,
411 : dieId, ret);
412 : }
413 3 : std::pair<uint64_t, uint64_t> ccuTokenInfo(tokenId, tokenValue);
414 3 : CcuResReq totalResReq;
415 :
416 : // 先获取hbm token，避免创建mission时循环获取
417 3 : DevBuffer tmpDevMem{1};
418 3 : auto hbmTokenInfo = GetTokenInfo(tmpDevMem.GetAddr(), 1);
419 :
420 : // 实例化CcuRepReferenceManager和CcuRepTranslator，并为CcuRepReferenceManager绑定物理资源
421 51 : for (uint32_t i = 0; i < 16; i++) { // mgr有16个
422 48 : referenceMgrs[dieId][i] = std::make_shared<CcuRepReferenceManager>(dieId);
423 96 : translators[dieId][i] = std::make_shared<CcuRepTranslator>(
424 96 : deviceLogicId_, dieId, referenceMgrs[dieId][i], tmpChannelId, ccuTokenInfo, hbmTokenInfo);
425 :
426 : // 统计&合并refManager和translaotr所有资源REQ
427 48 : auto refMangerResReq = CcuRep::CcuRepReferenceManager::GetResReq(dieId);
428 48 : auto transLatorResReq = CcuRep::CcuRepTranslator::GetResReq(dieId);
429 48 : MergeCcuResReq(totalResReq, refMangerResReq);
430 48 : MergeCcuResReq(totalResReq, transLatorResReq);
431 : }
432 :
433 3 : DumpResReqInfo(totalResReq);
434 :
435 : // 为refManager和translaotr申请物理资源
436 : CcuResHandle handle;
437 3 : CHK_RET_UNAVAIL(CcuDeviceManager::AllocResHandle(deviceLogicId_, totalResReq, handle));
438 3 : translatorResPack.handles.push_back(handle);
439 :
440 3 : CcuRepResource translatorRepRes;
441 51 : for (uint32_t i = 0; i < 16; i++) { // mgr有16个
442 48 : referenceMgrs[dieId][i]->GetRes(translatorRepRes);
443 48 : translators[dieId][i]->GetRes(translatorRepRes);
444 : }
445 :
446 3 : CcuResRepository totalResRepository;
447 3 : CHK_RET(GetResPackTotalResRepository(translatorResPack, totalResRepository));
448 : // 将ctx中的rep虚拟资源按类型进行和CCU物理资源映射
449 3 : ResetRepResourceToResRepository(translatorRepRes, totalResRepository);
450 3 : return HcclResult::HCCL_SUCCESS;
451 3 : }
452 :
453 3 : HcclResult CtxMgrImp::TransRepResToPhyRes(CcuCtxGroup& ctxGroup) const
454 : {
455 : // 获取ctxGroup中CCU物理资源句柄
456 3 : CcuResPack* resPack = ctxGroup.ctxs[0]->GetResPack();
457 3 : CHK_PTR_NULL(resPack);
458 :
459 : // 获取通信域当前所持有的资源
460 3 : CcuResRepository totalResRepository;
461 3 : CHK_RET(GetResPackTotalResRepository(*resPack, totalResRepository));
462 :
463 : // 遍历ctxGroup, 将每个ctx的虚拟资源进行合并
464 3 : CcuRepResource totalRepRes = GetTotalCcuRepResource(ctxGroup);
465 :
466 : // 将ctx中的rep虚拟资源按类型进行和CCU物理资源映射
467 3 : ResetRepResourceToResRepository(totalRepRes, totalResRepository);
468 :
469 : // 针对跨ctx的资源进行特殊映射处理
470 3 : ProcessInterCtxRes(ctxGroup);
471 :
472 : // missionId、mission key赋值给ctx
473 3 : CHK_RET(SaveCtxMissionInfo(ctxGroup, totalResRepository.mission.mission));
474 3 : return HcclResult::HCCL_SUCCESS;
475 3 : }
476 :
477 6 : HcclResult CtxMgrImp::GetResPackTotalResRepository(const CcuResPack& resPack, CcuResRepository& totalRes) const
478 : {
479 6 : CcuResRepository tmpResRepository;
480 :
481 : // 获取通信域当前所持有的资源
482 13 : for (CcuResHandle resHandle : resPack.handles) {
483 7 : CHK_RET(CcuDeviceManager::GetResource(deviceLogicId_, resHandle, tmpResRepository));
484 :
485 : // 合并获取的所持有的资源信息, 按照类型合并资源总和到totalRes中
486 21 : for (u32 i = 0; i < MAX_CCU_IODIE_NUM; i++) {
487 14 : ExpandResInfo(totalRes.ms[i], tmpResRepository.ms[i]);
488 14 : ExpandResInfo(totalRes.blockMs[i], tmpResRepository.blockMs[i]);
489 14 : ExpandResInfo(totalRes.loopEngine[i], tmpResRepository.loopEngine[i]);
490 14 : ExpandResInfo(totalRes.blockLoopEngine[i], tmpResRepository.blockLoopEngine[i]);
491 14 : ExpandResInfo(totalRes.cke[i], tmpResRepository.cke[i]);
492 14 : ExpandResInfo(totalRes.blockCke[i], tmpResRepository.blockCke[i]);
493 14 : ExpandResInfo(totalRes.gsa[i], tmpResRepository.gsa[i]);
494 14 : ExpandResInfo(totalRes.blockGsa[i], tmpResRepository.blockGsa[i]);
495 14 : ExpandResInfo(totalRes.xn[i], tmpResRepository.xn[i]);
496 14 : ExpandResInfo(totalRes.blockXn[i], tmpResRepository.blockXn[i]);
497 14 : ExpandResInfo(totalRes.mission.mission[i], tmpResRepository.mission.mission[i]);
498 : }
499 7 : DumpResRepositoryInfo(totalRes);
500 21 : HCCL_INFO(
501 : "GetResPackTotalResRepository:dumpInfos success deviceLogicId[%d] resHandle[%p].", deviceLogicId_,
502 : resHandle);
503 : }
504 6 : return HcclResult::HCCL_SUCCESS;
505 6 : }
506 :
507 3 : CcuRepResource CtxMgrImp::GetTotalCcuRepResource(CcuCtxGroup& ctxGroup) const
508 : {
509 3 : CcuRepResource totalRepRes;
510 :
511 : // 遍历ctxGroup, 将每个ctx的虚拟资源进行合并
512 9 : for (auto& ctx : ctxGroup.ctxs) {
513 : // 获取ctx的虚拟资源
514 6 : CcuRepResource tmpRepRes = ctx->GetResource();
515 6 : MergeCtxRepResource(totalRepRes, tmpRepRes);
516 6 : }
517 :
518 3 : return totalRepRes;
519 0 : }
520 :
521 6 : void CtxMgrImp::MergeCtxRepResource(CcuRepResource& repResourceA, CcuRepResource& repResourceB) const
522 : {
523 18 : for (u32 i = 0; i < MAX_CCU_IODIE_NUM; i++) {
524 : // 合并repRes
525 48 : repResourceA.ccubuffers[i].insert(
526 36 : repResourceA.ccubuffers[i].end(), repResourceB.ccubuffers[i].begin(), repResourceB.ccubuffers[i].end());
527 :
528 48 : repResourceA.blockCcubuffers[i].insert(
529 24 : repResourceA.blockCcubuffers[i].end(), repResourceB.blockCcubuffers[i].begin(),
530 12 : repResourceB.blockCcubuffers[i].end());
531 :
532 48 : repResourceA.executor[i].insert(
533 36 : repResourceA.executor[i].end(), repResourceB.executor[i].begin(), repResourceB.executor[i].end());
534 :
535 48 : repResourceA.blockExecutor[i].insert(
536 24 : repResourceA.blockExecutor[i].end(), repResourceB.blockExecutor[i].begin(),
537 12 : repResourceB.blockExecutor[i].end());
538 :
539 48 : repResourceA.maskSignal[i].insert(
540 36 : repResourceA.maskSignal[i].end(), repResourceB.maskSignal[i].begin(), repResourceB.maskSignal[i].end());
541 :
542 48 : repResourceA.blockMaskSignal[i].insert(
543 24 : repResourceA.blockMaskSignal[i].end(), repResourceB.blockMaskSignal[i].begin(),
544 12 : repResourceB.blockMaskSignal[i].end());
545 :
546 48 : repResourceA.address[i].insert(
547 36 : repResourceA.address[i].end(), repResourceB.address[i].begin(), repResourceB.address[i].end());
548 :
549 48 : repResourceA.blockAddress[i].insert(
550 24 : repResourceA.blockAddress[i].end(), repResourceB.blockAddress[i].begin(),
551 12 : repResourceB.blockAddress[i].end());
552 :
553 48 : repResourceA.variable[i].insert(
554 36 : repResourceA.variable[i].end(), repResourceB.variable[i].begin(), repResourceB.variable[i].end());
555 :
556 48 : repResourceA.continuousVariable[i].insert(
557 36 : repResourceA.continuousVariable[i].end(), repResourceB.continuousVariable[i].begin(),
558 12 : repResourceB.continuousVariable[i].end());
559 : }
560 6 : return;
561 : }
562 :
563 : template <typename T1, typename T2>
564 120 : void CtxMgrImp::ResetRepResourceTemplate(std::vector<T1>& resource, const std::vector<T2>& repository) const
565 : {
566 120 : if (resource.size() > repository.size()) {
567 0 : THROW<CcuApiException>(
568 : "[CtxMgrImp][ResetRepResourceTemplate]resource size[%u] bigger repository size[%u] typeid[%s]",
569 : resource.size(), repository.size(), typeid(T1).name());
570 : return;
571 : }
572 :
573 3956 : for (uint32_t j = 0; j < resource.size(); j++) {
574 3836 : resource[j].Reset(repository[j].startId);
575 : }
576 : }
577 :
578 6 : void CtxMgrImp::ResetRepResourceToResRepository(
579 : CcuRepResource& totalRepRes, const CcuResRepository& totalResRepository) const
580 : {
581 : // 遍历translatorRepRes, 将每个rep的虚拟资源翻译到实际物理资源上
582 18 : for (u32 i = 0; i < MAX_CCU_IODIE_NUM; i++) {
583 12 : ResetRepResourceTemplate(totalRepRes.ccubuffers[i], totalResRepository.ms[i]);
584 12 : ResetRepResourceTemplate(totalRepRes.blockCcubuffers[i], totalResRepository.blockMs[i]);
585 12 : ResetRepResourceTemplate(totalRepRes.executor[i], totalResRepository.loopEngine[i]);
586 12 : ResetRepResourceTemplate(totalRepRes.blockExecutor[i], totalResRepository.blockLoopEngine[i]);
587 12 : ResetRepResourceTemplate(totalRepRes.maskSignal[i], totalResRepository.cke[i]);
588 12 : ResetRepResourceTemplate(totalRepRes.blockMaskSignal[i], totalResRepository.blockCke[i]);
589 12 : ResetRepResourceTemplate(totalRepRes.address[i], totalResRepository.gsa[i]);
590 12 : ResetRepResourceTemplate(totalRepRes.blockAddress[i], totalResRepository.blockGsa[i]);
591 12 : ResetRepResourceTemplate(totalRepRes.variable[i], totalResRepository.xn[i]);
592 12 : ResetRepResourceTemplate(totalRepRes.continuousVariable[i], totalResRepository.blockXn[i]);
593 : }
594 6 : }
595 :
596 : template <typename T>
597 12 : void CtxMgrImp::ProcessSharedResources(
598 : std::unordered_map<std::string, T>& resources, std::vector<std::unordered_map<std::string, T>>& exportedResources,
599 : uint32_t i) const
600 : {
601 18 : for (auto& res : resources) {
602 : uint32_t j;
603 13 : for (j = 0; j < exportedResources.size(); j++) {
604 13 : if (i != j) {
605 8 : auto exportedRes = exportedResources[j].find(res.first);
606 8 : if (exportedRes != exportedResources[j].end()) {
607 6 : res.second.Reset(exportedRes->second.Id(), exportedRes->second.DieId());
608 6 : break;
609 : }
610 : }
611 : }
612 :
613 6 : if (j >= exportedResources.size()) {
614 0 : THROW<CcuApiException>("ProcessSharedResources failed tag[%s]", res.first.c_str());
615 : }
616 : }
617 12 : }
618 :
619 3 : void CtxMgrImp::ProcessInterCtxRes(CcuCtxGroup& ctxGroup) const
620 : {
621 : // 针对跨ctx的资源进行特殊映射处理
622 9 : for (uint32_t i = 0; i < ctxGroup.ctxs.size(); i++) {
623 : // 获取当前ctx中导入的ccuShrRes
624 6 : CcuSharedResource importRes = ctxGroup.ctxs[i]->GetImportRes();
625 :
626 : // 提前计算所有ctx的exportRes，以减少不必要的重复计算
627 12 : std::vector<std::unordered_map<std::string, CcuRep::Variable>> exportVarResList(ctxGroup.ctxs.size());
628 6 : std::vector<std::unordered_map<std::string, CcuRep::MaskSignal>> exportSigResList(ctxGroup.ctxs.size());
629 :
630 20 : for (uint32_t j = 0; j < ctxGroup.ctxs.size(); j++) {
631 14 : if (i != j) {
632 8 : auto exportRes = ctxGroup.ctxs[j]->GetExportRes();
633 8 : exportVarResList[j] = exportRes.sharedVar;
634 8 : exportSigResList[j] = exportRes.sharedSig;
635 8 : }
636 : }
637 :
638 : // 处理sharedVar和sharedSig
639 6 : ProcessSharedResources(importRes.sharedVar, exportVarResList, i);
640 6 : ProcessSharedResources(importRes.sharedSig, exportSigResList, i);
641 6 : }
642 3 : }
643 :
644 : HcclResult
645 3 : CtxMgrImp::SaveCtxMissionInfo(CcuCtxGroup& ctxGroup, array<vector<ResInfo>, MAX_CCU_IODIE_NUM>& missionId) const
646 : {
647 9 : for (auto& ctx : ctxGroup.ctxs) {
648 6 : auto dieId = ctx->GetDieId();
649 : uint32_t missionKey;
650 6 : CHK_RET(CcuDeviceManager::GetMissionKey(deviceLogicId_, dieId, missionKey));
651 :
652 18 : HCCL_INFO("[SaveCtxMissionInfo]GetMissionKey:deviceLogicId[%d] dieId[%u]", deviceLogicId_, dieId);
653 :
654 6 : ctx->SetMissionKey(missionKey);
655 : // 从missionId中获取一个元素并从missionId中删除
656 6 : ctx->SetMissionId(missionId[dieId].back().startId);
657 6 : missionId[dieId].pop_back();
658 : }
659 :
660 3 : return HcclResult::HCCL_SUCCESS;
661 : }
662 :
663 3 : void CtxMgrImp::TransRepSequenceToMicrocode(CcuCtxGroup& ctxGroup, bool isFuncBlock)
664 : {
665 9 : for (auto& ctx : ctxGroup.ctxs) {
666 6 : auto dieId = ctx->GetDieId();
667 6 : auto missionId = ctx->GetMissionId();
668 : // 翻译本ctx的REP序列
669 : auto instrInfo
670 6 : = translators[dieId][missionId]->Translate(ctx->GetRepSequence(), ctx->GetInstrId(), isFuncBlock);
671 :
672 : #ifdef HCCL_ALG_ANALYZER_DAVID
673 : // 建立CCU指令和微码的映射关系
674 : extern std::map<const Hccl::Instruction*, std::vector<Hccl::CcuRep::CcuInstrInfo>> g_ccuIns2MicroCode;
675 : extern Hccl::Instruction* g_ccuIns;
676 : g_ccuIns2MicroCode[g_ccuIns].push_back(instrInfo);
677 : #endif
678 :
679 : // 保存翻译后的指令信息
680 6 : ctx->SetCcuInstrInfo(instrInfo);
681 :
682 : // 加载指令到CCU指令空间
683 6 : LoadInstruction(instrInfo, dieId);
684 6 : }
685 3 : return;
686 : }
687 :
688 6 : void CtxMgrImp::LoadInstruction(CcuRep::CcuInstrInfo& instrInfo, uint32_t dieId)
689 : {
690 6 : uint64_t instrInfoSize = instrInfo.instrVec.size() * sizeof(CcuInstr);
691 :
692 6 : if (!instructionLoadDevMem_) {
693 6 : uint32_t instrNum = 0;
694 6 : if (CcuDeviceManager::GetInstructionNum(deviceLogicId_, 0, instrNum) != HcclResult::HCCL_SUCCESS) {
695 0 : THROW<CcuApiException>("CtxMgrImp::Init failed");
696 : }
697 18 : HCCL_INFO("[CtxMgrImp]LoadInstruction: deviceLogicId[%d], instrNum[%u]", deviceLogicId_, instrNum);
698 : // rt接口申请device内存
699 6 : instructionLoadDevMem_ = HrtMalloc(instrNum * sizeof(CcuInstr), static_cast<int>(ACL_MEM_TYPE_HIGH_BAND_WIDTH));
700 : }
701 :
702 : // rt接口memcpySync
703 6 : HrtMemcpy(
704 6 : instructionLoadDevMem_, instrInfoSize, instrInfo.instrVec.data(), instrInfoSize, RT_MEMCPY_HOST_TO_DEVICE);
705 :
706 6 : auto tlvHandle = HccpTlvHdcManager::GetInstance().GetTlvHandle(deviceLogicId_);
707 6 : CHECK_NULLPTR(
708 12 : tlvHandle, StringFormat("[CtxMgrImp][%s] tlvHandle is nullptr, deviceLogicId[%d]", __func__, deviceLogicId_));
709 :
710 6 : struct CustomChannelInfoIn inBuff;
711 6 : struct CustomChannelInfoOut outBuff;
712 :
713 : CcuDataTypeUnion tmp;
714 6 : tmp.insinfo.resourceAddr = reinterpret_cast<uint64_t>(instructionLoadDevMem_);
715 :
716 : // 设置操作码和通道数据
717 6 : inBuff.op = CcuOpcodeType::CCU_U_OP_SET_INSTRUCTION;
718 6 : inBuff.offsetStartIdx = instrInfo.startInstrId;
719 6 : inBuff.data.dataInfo.udieIdx = dieId;
720 6 : inBuff.data.dataInfo.dataArraySize = 1;
721 6 : inBuff.data.dataInfo.dataLen = instrInfoSize;
722 :
723 : // 复制通道数据
724 6 : (void)memcpy_s(inBuff.data.dataInfo.dataArray, sizeof(CcuDataTypeUnion), &tmp, sizeof(CcuDataTypeUnion));
725 :
726 6 : HrtRaTlvRequestForCustomChannel(
727 : tlvHandle, MSG_TYPE_CCU_DISPATCH_CMD, static_cast<void*>(&inBuff), static_cast<void*>(&outBuff));
728 :
729 18 : HCCL_RUN_INFO(
730 : "Entry-LoadInstruction: load instruction success startInstrId[%u] instrCount[%u]", instrInfo.startInstrId,
731 : instrInfo.instrCount);
732 :
733 12 : return;
734 : }
735 :
736 12 : void CtxMgrImp::DumpResReqInfo(const CcuResReq& totalRes) const
737 : {
738 36 : for (uint32_t i = 0; i < MAX_CCU_IODIE_NUM; i++) {
739 48 : if (totalRes.msReq[i] != 0 || totalRes.blockMsReq[i] != 0 || totalRes.ckeReq[i] != 0
740 15 : || totalRes.blockCkeReq[i] != 0 || totalRes.loopEngineReq[i] != 0 || totalRes.blockLoopEngineReq[i] != 0
741 15 : || totalRes.gsaReq[i] != 0 || totalRes.blockGsaReq[i] != 0 || totalRes.xnReq[i] != 0
742 48 : || totalRes.blockXnReq[i] != 0 || totalRes.missionReq.req[i] != 0) {
743 33 : HCCL_INFO(
744 : "DumpResReqInfo: dieId[%u], msReq[%u], blockMsReq[%u], ckeReq[%u], blockCkeReq[%u], "
745 : "loopEngineReq[%u], blockLoopEngineReq[%u], gsaReq[%u], blockGsaReq[%u], "
746 : "xnReq[%u], blockXnReq[%u], missionReq[%u]",
747 : i, totalRes.msReq[i], totalRes.blockMsReq[i], totalRes.ckeReq[i], totalRes.blockCkeReq[i],
748 : totalRes.loopEngineReq[i], totalRes.blockLoopEngineReq[i], totalRes.gsaReq[i], totalRes.blockGsaReq[i],
749 : totalRes.xnReq[i], totalRes.blockXnReq[i], totalRes.missionReq.req[i]);
750 : }
751 : }
752 12 : }
753 :
754 7 : void CtxMgrImp::DumpResRepositoryInfo(const CcuResRepository& resRepo) const
755 : {
756 21 : for (uint32_t i = 0; i < MAX_CCU_IODIE_NUM; i++) {
757 28 : if (resRepo.ms[i].size() != 0 || resRepo.blockMs[i].size() != 0 || resRepo.cke[i].size() != 0
758 3 : || resRepo.blockCke[i].size() != 0 || resRepo.loopEngine[i].size() != 0
759 3 : || resRepo.blockLoopEngine[i].size() != 0 || resRepo.gsa[i].size() != 0 || resRepo.blockGsa[i].size() != 0
760 28 : || resRepo.xn[i].size() != 0 || resRepo.blockXn[i].size() != 0 || resRepo.mission.mission[i].size() != 0) {
761 33 : HCCL_INFO(
762 : "DumpResRepository: dieId[%u], ms size[%u], blockMs size[%u], cke size[%u], blockCke size[%u], "
763 : "loopEngine size[%u], blockLoopEngine size[%u], gsa size[%u], blockGsa size[%u], xn size[%u], "
764 : "block xn size[%u], mission size[%u]",
765 : i, resRepo.ms[i].size(), resRepo.blockMs[i].size(), resRepo.cke[i].size(), resRepo.blockCke[i].size(),
766 : resRepo.loopEngine[i].size(), resRepo.blockLoopEngine[i].size(), resRepo.gsa[i].size(),
767 : resRepo.blockGsa[i].size(), resRepo.xn[i].size(), resRepo.blockXn[i].size(),
768 : resRepo.mission.mission[i].size());
769 : }
770 : }
771 7 : }
772 :
773 0 : std::vector<std::vector<CcuProfilingInfo>> CtxMgrImp::GetProfilingInfo(CcuTaskArg& ccuTaskArg, const uint64_t entityId)
774 : {
775 : // 根据entityId获取ctxGroup
776 0 : std::unique_lock<std::mutex> lock(contextMapMutex_);
777 :
778 : // 校验ctxGroupMap_中是否存在entityId对应的ctxGroup
779 0 : CHK_PRT_RET(
780 : ctxGroupMap_.find(entityId) == ctxGroupMap_.end(),
781 : HCCL_ERROR("[CtxMgrImp][GetProfilingInfo]entityId [%llu] is not exist", entityId),
782 : std::vector<std::vector<CcuProfilingInfo>>());
783 :
784 : // 获取每个ctx的profiling信息
785 0 : std::vector<std::vector<CcuProfilingInfo>> ccuProfilingInfo;
786 0 : for (auto& ctx : ctxGroupMap_[entityId].ctxs) {
787 0 : std::vector<CcuProfilingInfo> tmp;
788 0 : auto ret = ctx->GetCcuProfilingInfo(ccuTaskArg, tmp);
789 0 : if (ret != HcclResult::HCCL_SUCCESS) {
790 0 : THROW<CcuApiException>("GetCcuProfilingInfo is failed. ret[%d]", ret);
791 : }
792 0 : ccuProfilingInfo.push_back(tmp);
793 0 : }
794 :
795 0 : return ccuProfilingInfo;
796 0 : }
797 :
798 2 : CcuContext* CtxMgrImp::GetCtx(uint64_t executorId, uint32_t dieId, uint32_t missionId)
799 : {
800 2 : std::unique_lock<std::mutex> lock(contextMapMutex_);
801 :
802 2 : if (ctxGroupMap_.find(executorId) == ctxGroupMap_.end()) {
803 3 : HCCL_ERROR("[CtxMgrImp][GetCtx] executorId [%llu] is not exist", executorId);
804 1 : return nullptr;
805 : }
806 :
807 1 : for (auto& ctx : ctxGroupMap_[executorId].ctxs) {
808 0 : if (ctx->GetDieId() == dieId && ctx->GetMissionId() == missionId) {
809 0 : return ctx.get();
810 : }
811 : }
812 :
813 3 : HCCL_ERROR("[CtxMgrImp][GetCtx] CcuContext is not exist, dieId[%u], missionId[%u]", dieId, missionId);
814 1 : return nullptr;
815 2 : }
816 : }; // namespace Hccl
|
