Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include <iomanip>
12 : #include "aicpu_communicator.h"
13 : #include "dispatcher.h"
14 : #include "aicpu_hccl_process.h"
15 : #include "coll_alg_exec_registry.h"
16 : #include "coll_all_to_all_executor.h"
17 : #include "common/aicpu_hccl_common.h"
18 : #include "executor_tracer.h"
19 : #include "profiling_manager_device.h"
20 : #include "utils/aicpu_hdc_utils.h"
21 : #include "comm_engine_utils.h"
22 : #include "utils/hccl_aicpu_utils.h"
23 : #include "framework/aicpu_hdc.h"
24 : #include "common/aicpu_sqe_context.h"
25 : #include "coll_batch_send_recv_retry_executor.h"
26 : #include "log_control.h"
27 : #include "aicpu_hccl_sqcq.h"
28 : #include "aicpu_hccl_sqcqv1.h"
29 : #include "sal_pub.h"
30 : #include "externalinput_pub.h"
31 : #include "env_config.h"
32 : #include "config_log.h"
33 : #include "aicpu_one_side_service.h"
34 : #include "notify_manager.h"
35 : #include "dispatcher_aicpu.h"
36 : #include "dlprof_function.h"
37 : #include "profiling_command_handle.h"
38 : #include "dispatcher_ctx.h"
39 : #include "aicpu_res_package_helper.h"
40 : #include "aicpu_symmetric_memory.h"
41 :
42 : namespace hccl {
43 : constexpr u32 IPC_SIGNAL_MODULUS = 2;
44 : constexpr u32 RDMA_SIGNAL_MODULUS = 3;
45 : constexpr u32 KEY_VALUE_TO_VECTOR_MODULUS = 2;
46 :
47 : constexpr u32 BSR_RETRY_SEND_STREAM_INDEX = 0;
48 : constexpr u32 BSR_RETRY_RECV_STREAM_INDEX = 1;
49 : constexpr u32 BSR_RETRY_STREAM_NUM = 2;
50 : constexpr u32 MAX_REPORT_STATUS = 100U; // reportStatus的最大缓存数量
51 : constexpr u32 INPUT = 0;
52 : constexpr u32 OUTPUT = 1;
53 : constexpr u32 AICPU_RETRY_LINKROCE_DEFAULT = 0;
54 : constexpr u32 AICPU_RETRY_LINKROCE_BACKUP = 1;
55 :
56 : constexpr u32 BSR_RETRY_SENDRECV_PAIR_NUM_MAX = 2;
57 : constexpr u32 BSR_RETRY_SENDRECV_PAIR_INDEX_0 = 0;
58 : constexpr u32 BSR_RETRY_SENDRECV_PAIR_INDEX_1 = 1;
59 :
60 : constexpr u32 NOTIFY_SIZE_FOUR = 4;
61 : constexpr u32 NOTIFY_SIZE_EIGHT = 8;
62 :
63 : bool HcclCommAicpu::errMessageReport_ = true;
64 :
65 : #define HCCL_RETRY_CHK_RET_AND_TRANS_FSM(result__, exeLog__, error__, state__) \
66 : do { \
67 : if (UNLIKELY((result__) != HCCL_SUCCESS)) { \
68 : exeLog__; \
69 : errorCode = (error__); \
70 : fsmState = (state__); \
71 : return (result__); \
72 : } \
73 : } while (0)
74 :
75 352 : HcclCommAicpu::HcclCommAicpu()
76 : {
77 32 : HCCL_RUN_INFO("Construct HcclCommAicpu complete.");
78 32 : }
79 :
80 279 : HcclCommAicpu::~HcclCommAicpu()
81 : {
82 31 : if (UtraceInfo_ != nullptr) {
83 11 : UtraceInfo_->DeInit();
84 11 : UtraceInfo_ = nullptr;
85 : }
86 31 : if (dispatcher_ != nullptr) {
87 11 : HcclDispatcherDestroy(dispatcher_);
88 11 : dispatcher_ = nullptr;
89 : }
90 31 : if (dispatcherCtx_ != nullptr) {
91 0 : DestroyDispatcherCtx(dispatcherCtx_, identifier_.c_str());
92 0 : HCCL_DEBUG("[%s] destroy dispatcherCtx[%p] group[%s] success!", __func__, dispatcherCtx_, identifier_.c_str());
93 0 : dispatcherCtx_ = nullptr;
94 : }
95 :
96 31 : commPlaneVector_.clear();
97 31 : isBridgeVector_.clear();
98 31 : indOpCommInitialized_ = false;
99 31 : initialized_ = false;
100 :
101 31 : HCCL_RUN_INFO("Destruct HcclCommAicpu group[%s] success!", identifier_.c_str());
102 341 : }
103 :
104 13 : HcclResult HcclCommAicpu::Init(const HcclOpResParam *commParam, bool isCustom)
105 : {
106 13 : if (initialized_) {
107 2 : HCCL_RUN_INFO("[%s][Init]Group[%s] already initialized, skip reinit", __func__,
108 : identifier_.c_str());
109 2 : return HCCL_SUCCESS;
110 : }
111 :
112 11 : CHK_PTR_NULL(commParam);
113 11 : identifier_ = commParam->hcomId;
114 11 : isCustom_ = isCustom;
115 11 : HCCL_RUN_INFO("[HcclCommAicpu][Init]Entry-Init group[%s], rankSize[%u], isCustom[%d].",
116 : identifier_.c_str(), commParam->rankSize, isCustom_);
117 11 : CHK_RET(aicpuShareData_.Init(commParam->aicpuCustomParamAddr, commParam->aicpuCustomParamSize));
118 11 : CHK_RET(SetHrtWorkMode(commParam));
119 11 : CHK_RET(SetHrtDeviceSatMode(commParam));
120 11 : CHK_RET(InitConfigInfo(commParam));
121 11 : CHK_RET(InitCclbuffer(commParam));
122 11 : CHK_RET(InitTopoInfo(commParam));
123 11 : CHK_RET(InitOpNotifyObj(commParam));
124 11 : CHK_RET(HcclDispatcherAicpuInit(&dispatcher_, devId_, commParam->hcclSdmaQos, DispatcherType::DISPATCHER_AICPU));
125 11 : CHK_RET(RegisterProfilingCallback());
126 11 : CHK_RET(InitLocalNotifyObj(commParam));
127 11 : CHK_RET(InitMainStreamObj(commParam));
128 11 : CHK_RET(InitSlaveStreamObjs(commParam));
129 11 : CHK_RET(InitOrderStreamObj(commParam));
130 11 : CHK_RET(InitLocalTagRes(commParam->localRes.nextTagRes));
131 11 : CHK_RET(InitTimeOutConfig(commParam));
132 11 : CHK_RET(InitHostDeviceLock(commParam));
133 11 : CHK_RET(InitTopoMatcher());
134 11 : CHK_RET(InitOpRetry(commParam));
135 11 : CHK_RET(RegisterDispatcherCallback());
136 11 : CHK_RET(InitTinyMem(commParam));
137 11 : CHK_RET(InitProfResource());
138 11 : CHK_RET(InitZeroCopyExchanger(commParam));
139 11 : CHK_RET(InitOpCounter(commParam->opCounterInfo));
140 11 : CHK_RET(InitUtraceInfo(commParam));
141 11 : CHK_RET(aicpuCacheManager_.InitOpUnfoldCache());
142 11 : CHK_RET(RegisterProfCallBack());
143 11 : InitCommInfoStatus(true);
144 11 : SetCommInfoStreamStatus(true);
145 :
146 11 : initialized_ = true;
147 :
148 11 : HCCL_RUN_INFO("[HcclCommAicpu][Init] group[%s] success!", identifier_.c_str());
149 11 : return HCCL_SUCCESS;
150 : }
151 :
152 11 : HcclResult HcclCommAicpu::InitUtraceInfo(const HcclOpResParam *commParam)
153 : {
154 11 : u32 hostpid = 0;
155 11 : u32 cpType = DEVDRV_PROCESS_CPTYPE_MAX;
156 11 : CHK_RET(HrtHalDrvQueryProcessHostPid(getpid(), nullptr, nullptr, &hostpid, &cpType));
157 :
158 : HcclTraceInfo::UtraceAttr utraceAttr;
159 11 : utraceAttr.utraceStatusFlag = commParam->utraceStatusFlag;
160 11 : utraceAttr.deviceid = GetDevId();
161 11 : utraceAttr.pid = hostpid;
162 11 : UtraceInfo_.reset(new (std::nothrow) HcclTraceInfo(utraceAttr));
163 11 : CHK_PTR_NULL(UtraceInfo_);
164 :
165 : /* 申请trace资源信息 */
166 11 : std::string logInfo = "HCCL_";
167 11 : logInfo.append(std::to_string(SalGetTid()));
168 11 : logInfo.append("_");
169 11 : logInfo.append(std::to_string(GetDevId()));
170 11 : CHK_RET(UtraceInfo_->Init(logInfo));
171 11 : return HCCL_SUCCESS;
172 11 : }
173 :
174 11 : HcclResult HcclCommAicpu::InitProfResource()
175 : {
176 11 : groupHashId_ = dfx::ProfilingManager::GetProfHashId(identifier_.c_str(), identifier_.length());
177 11 : HCCL_RUN_INFO("[Init][ProfResource]group[%s], groupHashId_[%llu].", identifier_.c_str(), groupHashId_);
178 :
179 11 : dfx::ProfCommInfo profInfo{ groupHashId_, topoInfo_.userRankSize, topoInfo_.userRank };
180 11 : CHK_RET(dfx::ProfilingManager::AddProfInfoByStreamId(mainStream_.id(), identifier_, profInfo));
181 451 : for (auto &slaveStream : slaveStreams_) {
182 440 : CHK_RET(dfx::ProfilingManager::AddProfInfoByStreamId(slaveStream.id(), identifier_, profInfo));
183 : }
184 11 : dfx::ProfilingExtendInfoHelper::InitProfItemId();
185 11 : return HCCL_SUCCESS;
186 : }
187 :
188 462 : HcclResult HcclCommAicpu::StreamRestore(u32 streamId)
189 : {
190 462 : HcclResult ret = hrtHalResourceIdRestore(devId_, 0, DRV_STREAM_ID, streamId, 0);
191 : // custom进程需要恢复stream资源, custom进程调用失败直接报错，aicpu进程调用失败做兼容性处理
192 462 : if (ret == HCCL_E_NOT_SUPPORT) {
193 0 : CHK_PRT_RET(isCustom_, HCCL_ERROR("%s hrtHalResourceIdRestore fail, drv not support, custom[%d], ret[%d]",
194 : __func__, isCustom_, ret), HCCL_E_DRV);
195 462 : } else if (ret != HCCL_SUCCESS) {
196 0 : HCCL_ERROR("%s hrtHalResourceIdRestore fail, ret[%d]", __func__, ret);
197 0 : return HCCL_E_DRV;
198 : }
199 462 : return HCCL_SUCCESS;
200 : }
201 :
202 11 : HcclResult HcclCommAicpu::SetHrtWorkMode(const HcclOpResParam *commParam)
203 : {
204 11 : CHK_RET(hrtSetWorkModeAicpu(true));
205 11 : CHK_RET(hrtSetlocalDevice(commParam->topoInfo.deviceLogicId));
206 11 : CHK_RET(hrtSetlocalDeviceType(static_cast<DevType>(commParam->topoInfo.deviceType)));
207 11 : return HCCL_SUCCESS;
208 : }
209 :
210 11 : HcclResult HcclCommAicpu::SetHrtDeviceSatMode(const HcclOpResParam *commParam)
211 : {
212 11 : CHK_RET(hrtSetLocalDeviceSatMode(commParam->config.floatOverflowMode));
213 11 : HCCL_RUN_INFO("[HcclCommAicpu][Init]SetHrtDeviceSatMode[%d]", static_cast<u32>(commParam->config.floatOverflowMode));
214 11 : return HCCL_SUCCESS;
215 : }
216 :
217 11 : HcclResult HcclCommAicpu::InitTopoMatcher()
218 : {
219 11 : externalEnable_.enableFfts = 1; // FFTS+在算法模块不会使用多线程，固定使能
220 11 : externalEnable_.deterministic = deterministic_;
221 11 : externalEnable_.intraRoceSwitch = 0;
222 11 : externalEnable_.dumpDebug = dumpDebug_;
223 11 : externalEnable_.interHccsDisable = interHccsDisable_;
224 :
225 11 : topoMatcher_.reset((new (std::nothrow) TopoMatcher(
226 22 : commPlaneVector_, isBridgeVector_, topoInfo_, algoInfo_, externalEnable_, serverAndsuperPodToRank_)));
227 11 : CHK_SMART_PTR_NULL(topoMatcher_);
228 11 : HCCL_RUN_INFO("[HcclCommAicpu][InitTopoMatcher]topo matcher init success. group[%s] deterministic:%u, "
229 : "dumpDebug:%u, interHccsDisable:%u", identifier_.c_str(), deterministic_, dumpDebug_, interHccsDisable_);
230 11 : return HCCL_SUCCESS;
231 : }
232 :
233 11 : HcclResult HcclCommAicpu::InitOpRetry(const HcclOpResParam *commParam)
234 : {
235 11 : CHK_PTR_NULL(commParam);
236 11 : retryEnable_ = (commParam->config.retryEnable == 1) ? true : false;
237 11 : retryHoldTime_ = commParam->config.retryHoldTime;
238 11 : retryIntervalTime_ = commParam->config.retryIntervalTime;
239 11 : HCCL_RUN_INFO("[InitOpRetry]retryEnable[%d], retryHoldTime[%u ms], retryIntervalTime[%u ms]",
240 : retryEnable_,
241 : retryHoldTime_,
242 : retryIntervalTime_);
243 :
244 11 : if (commParam->kfcControlTransferH2DParams.buffLen != 0 && kfcControlTransferH2D_ == nullptr) {
245 0 : EXCEPTION_CATCH((kfcControlTransferH2D_ = std::make_shared<hccl::HDCommunicate>()), return HCCL_E_PTR);
246 0 : CHK_SMART_PTR_NULL(kfcControlTransferH2D_);
247 0 : CHK_RET(kfcControlTransferH2D_->InitDevice(commParam->kfcControlTransferH2DParams));
248 : }
249 11 : if (commParam->kfcStatusTransferD2HParams.buffLen != 0 && kfcStatusTransferD2H_ == nullptr) {
250 0 : EXCEPTION_CATCH((kfcStatusTransferD2H_ = std::make_shared<hccl::HDCommunicate>()), return HCCL_E_PTR);
251 0 : CHK_SMART_PTR_NULL(kfcStatusTransferD2H_);
252 0 : CHK_RET(kfcStatusTransferD2H_->InitDevice(commParam->kfcStatusTransferD2HParams));
253 : }
254 11 : return HCCL_SUCCESS;
255 : }
256 :
257 11 : HcclResult HcclCommAicpu::InitZeroCopyExchanger(const HcclOpResParam *commParam)
258 : {
259 11 : CHK_PTR_NULL(commParam);
260 0 : auto nSecStopFunc = [this] () -> bool {
261 : // 检查到OP状态不Ok则认为需要终止
262 0 : auto ret = this->CheckOpExecStatus();
263 0 : if (ret == HCCL_SUCCESS) {
264 0 : return false;
265 0 : } else if (ret == HCCL_E_SUSPENDING) {
266 : // NS快恢场景，需要提前终止
267 0 : HcclOpExecFSM fsmState = HcclOpExecFSM::HCCL_OP_EXEC_FSM_INIT;
268 0 : KfcError errorCode = KfcError::kNone;
269 0 : SetCommRecoveryFlag(true);
270 0 : UpdateOpExecStatus(fsmState, KfcStatus::kStoplaunch, errorCode, 0);
271 0 : HCCL_RUN_INFO("[HcclCommAicpu][nSecStopFunc] need stop launch");
272 0 : return true;
273 : } else {
274 0 : return true;
275 : }
276 11 : };
277 :
278 11 : u32 timeoutSec = commParam->config.notifyWaitTime;
279 11 : HCCL_INFO("[HcclCommAicpu][InitZeroCopyExchanger] set timeout is [%u s]", timeoutSec);
280 :
281 11 : EXCEPTION_CATCH((ZeroCopyExchanger_ =
282 : std::make_shared<hccl::AicpuZeroCopyExchanger>(commParam->localUsrRankId, commParam->rankSize,
283 : commParam, nSecStopFunc, timeoutSec, topoInfo_.deviceNumPerAggregation, taskMonitorInterval_)), return HCCL_E_PTR);
284 :
285 : // 通信域第一次初始化时，如果IPC内存有不为空的则认为是使能该特性
286 11 : isZeroCopy_ = false;
287 363 : for (u32 i = 0; i < AICPU_ZERO_COPY_MAX_DEVICE_NUM_A3; ++i) {
288 352 : if (commParam->zeroCopyIpcPtrs[i] != 0) {
289 0 : isZeroCopy_ = true;
290 0 : break;
291 : }
292 : }
293 :
294 11 : return HCCL_SUCCESS;
295 : }
296 :
297 11 : HcclResult HcclCommAicpu::InitOpCounter(const OpCounterInfo &opCounterInfo)
298 : {
299 11 : if (opCounterInfo.isEnableCounter && !retryEnable_ && (opCounterInfo.headCountMem == 0
300 0 : || opCounterInfo.tailCountMem == 0 || opCounterInfo.memSize == 0 )) {
301 0 : HCCL_ERROR("[HcclCommAicpu][InitOpCounter] headCountMem or tailCountMem or memSize is null");
302 0 : return HCCL_E_PARA;
303 : }
304 11 : opCounterInfo_ = opCounterInfo;
305 11 : return HCCL_SUCCESS;
306 : }
307 :
308 1 : void HcclCommAicpu::SetZeroCopyEnable(bool enable)
309 : {
310 1 : isZeroCopy_ = enable;
311 1 : }
312 :
313 1 : void HcclCommAicpu::SetSymmetricMemoryEnable(bool enable)
314 : {
315 1 : HCCL_INFO("[HcclCommAicpu::SetSymmetricMemoryEnable] enable[%d]", enable);
316 1 : isSymmetricMemory_ = enable;
317 1 : }
318 :
319 0 : HcclResult HcclCommAicpu::PrepareZeroCopyExchanger(const std::string &newTag, OpParam &opParam,
320 : AlgResourceResponse *algResResponse)
321 : {
322 0 : return ZeroCopyExchanger_->ExchangeAddress(newTag, opParam.inputPtr, opParam.outputPtr, algResResponse);
323 : }
324 :
325 11 : HcclResult HcclCommAicpu::RegisterProfilingCallback()
326 : {
327 11 : (void)RegisterLoadTaskCallBack(dispatcher_, nullptr, dfx::TaskProfilingCallBack);
328 :
329 11 : return HCCL_SUCCESS;
330 : }
331 :
332 11 : HcclResult HcclCommAicpu::RegisterDispatcherCallback()
333 : {
334 0 : auto checkOpExecStatusCallback = [this](){ return this->CheckOpExecStatusCallback(); };
335 :
336 11 : return HcclSetOpExecStatusCallback(dispatcher_, checkOpExecStatusCallback);
337 : }
338 :
339 11 : HcclResult HcclCommAicpu::RegisterProfCallBack() {
340 11 : if (MsprofRegisterCallback != nullptr) {
341 11 : HCCL_INFO("RegisterProfCallBack not null");
342 11 : int32_t ret = MsprofRegisterCallback(AICPU, &DeviceCommandHandle);
343 11 : CHK_PRT_RET((ret != 0), HCCL_ERROR("[%s] failed. ret = [%d]", __func__, ret), HCCL_E_PARA);
344 : } else {
345 0 : HCCL_INFO("RegisterProfCallBack is null");
346 : }
347 11 : return HCCL_SUCCESS;
348 : }
349 :
350 0 : HcclResult HcclCommAicpu::GetSuspendingFlag(HcclComSuspendingFlag &flag)
351 : {
352 0 : CHK_RET(AicpuHdcUtils::GetSuspendingStatus(kfcControlTransferH2D_, flag));
353 0 : return HCCL_SUCCESS;
354 : }
355 :
356 57 : HcclResult HcclCommAicpu::BackGroundGetCmd(KfcCommand &cmd)
357 : {
358 57 : CHK_RET(aicpuHdc_.GetOpExecCtrlCmd(kfcControlTransferH2D_, cmd));
359 54 : return HCCL_SUCCESS;
360 : }
361 :
362 2 : HcclResult HcclCommAicpu::BackGroundSetStatus(KfcStatus status)
363 : {
364 2 : CHK_RET(aicpuHdc_.SetOpExecStatus(kfcStatusTransferD2H_, status, KfcError::kNone, 0));
365 2 : return HCCL_SUCCESS;
366 : }
367 :
368 0 : HcclResult HcclCommAicpu::SaveTraceInfo(std::string &logInfo)
369 : {
370 0 : CHK_PTR_NULL(UtraceInfo_);
371 0 : CHK_RET(UtraceInfo_->SaveTraceInfo(logInfo, AtraceOption::Opbasekey));
372 0 : return HCCL_SUCCESS;
373 : }
374 :
375 0 : HcclResult HcclCommAicpu::FlushUtraceInfo()
376 : {
377 0 : if (GetCommInfoStatus()) {
378 0 : CHK_RET(UtraceInfo_->Flush());
379 : }
380 0 : return HCCL_SUCCESS;
381 : }
382 :
383 0 : std::string HcclCommAicpu::GetExcuteOp()
384 : {
385 0 : std::stringstream ss;
386 0 : ss << "tag: " << excuteOpId_.tag << ", ";
387 0 : ss << "newTag: " << excuteOpId_.newTag << ", ";
388 0 : ss << "index: " << excuteOpId_.index;
389 0 : return ss.str();
390 0 : }
391 :
392 11 : void HcclCommAicpu::InitCommInfoStatus(bool commInfo)
393 : {
394 11 : commOpenStatus = commInfo;
395 11 : }
396 :
397 11 : HcclResult HcclCommAicpu::InitTinyMem(const HcclOpResParam *commParam)
398 : {
399 11 : CHK_PTR_NULL(commParam);
400 11 : auto tinyMemPtr = reinterpret_cast<void *>(commParam->tinyMem);
401 11 : tinySendRecvMem_ = DeviceMem::create(tinyMemPtr, commParam->tinyMemSize);
402 :
403 11 : return HCCL_SUCCESS;
404 : }
405 :
406 11 : HcclResult HcclCommAicpu::InitTimeOutConfig(const HcclOpResParam *commParam)
407 : {
408 11 : CHK_PTR_NULL(commParam);
409 11 : CHK_RET(HcclSetSqeTimeOut(dispatcher_, commParam->config.notifyWaitTime));
410 11 : linkTimeOut_ = commParam->config.linkTimeOut;
411 11 : return HCCL_SUCCESS;
412 : }
413 :
414 11 : HcclResult HcclCommAicpu::InitHostDeviceLock(const HcclOpResParam *commParam)
415 : {
416 11 : CHK_PTR_NULL(commParam);
417 11 : hostDeviceLock_.reset(new (std::nothrow)
418 11 : PetersonLock(reinterpret_cast<void *>(commParam->lockAddr), PetersonLock::DEFAULT_LOCK_TIMEOUT_SEC));
419 11 : CHK_SMART_PTR_NULL(hostDeviceLock_);
420 11 : CHK_RET(hostDeviceLock_->Init());
421 :
422 11 : return HCCL_SUCCESS;
423 : }
424 :
425 0 : HcclResult HcclCommAicpu::UpdateNotifyWaitTimeOut(SyncMode syncMode, u64 notifyWaitTime)
426 : {
427 0 : sqeWaitTimeOut_ = (notifyWaitTime == 0) ?
428 : notifyWaitTime : (notifyWaitTime + AICPU_SQE_TIMEOUT_INC);
429 0 : if (syncMode == SyncMode::UNLIMITED_TIMEWAITSYNCMODE) {
430 0 : CHK_RET(HcclSetSqeTimeOut(dispatcher_, GetNotifyMaxWaitTime()));
431 : }
432 0 : return HcclSetSqFullWaitTimeOut(dispatcher_, notifyWaitTime);
433 : }
434 :
435 0 : void HcclCommAicpu::PrepareOpRetryHandler(u8 inplaceSupportRetry, u8 retryEnable, u8 inPlaceSupportRetryStatus,
436 : u8 isInplacePreSync, u8 isPostSync)
437 : {
438 0 : algOpContext_.opRetryHandler.inplaceSupportRetry = static_cast<bool>(inplaceSupportRetry);
439 0 : algOpContext_.opRetryHandler.retryEnable = static_cast<bool>(retryEnable);
440 0 : algOpContext_.opRetryHandler.inPlaceSupportRetryStatus =
441 0 : static_cast<InplaceSupportRetryStatus>(inPlaceSupportRetryStatus);
442 0 : algOpContext_.opRetryHandler.isInplacePreSync = static_cast<bool>(isInplacePreSync);
443 0 : algOpContext_.opRetryHandler.isPostSync = static_cast<bool>(isPostSync);
444 0 : HCCL_INFO("[HcclCommAicpu][PrepareOpRetryHandler] inplaceSupportRetry %d, retryEnable %d, "
445 : "inPlaceSupportRetryStatus %d, isInplacePreSync %d, isPostSync %d.",
446 : algOpContext_.opRetryHandler.inplaceSupportRetry,
447 : algOpContext_.opRetryHandler.retryEnable,
448 : algOpContext_.opRetryHandler.inPlaceSupportRetryStatus,
449 : algOpContext_.opRetryHandler.isInplacePreSync,
450 : algOpContext_.opRetryHandler.isPostSync);
451 0 : }
452 :
453 0 : HcclResult HcclCommAicpu::UpdateOpRingBufferIdx()
454 : {
455 0 : return HcclSetOpRingBufferIdx(dispatcher_, aicpuShareData_.GetOpRingBufferIdx());
456 : }
457 :
458 0 : HcclResult HcclCommAicpu::InvokeKfcHandler(AicpuKfcHandlerType type, const std::vector<u64> args)
459 : {
460 0 : CHK_PRT_RET(static_cast<size_t>(type) >= static_cast<size_t>(AicpuKfcHandlerType::kMax),
461 : HCCL_ERROR("Device mode %u, handler type %u.",
462 : static_cast<u32>(isDeviceMode_), static_cast<size_t>(type)),
463 : HCCL_E_INTERNAL);
464 0 : const auto handler = kfcHandlers_[static_cast<size_t>(type)];
465 0 : if (handler == nullptr) {
466 0 : return HCCL_SUCCESS;
467 : }
468 0 : return handler(args);
469 0 : }
470 :
471 0 : HcclResult HcclCommAicpu::NotifyPost(void)
472 : {
473 0 : if (isDeviceMode_) {
474 0 : return InvokeKfcHandler(AicpuKfcHandlerType::kNotifyRecord,
475 0 : {rpc_, reinterpret_cast<u64>(dispatcher_), reinterpret_cast<u64>(&mainStream_)});
476 : } else {
477 0 : CHK_RET(LocalNotify::Post(mainStream_, dispatcher_, opNotifies_[1]));
478 0 : HcclSqeContext *sqeContext = mainStream_.GetSqeContextPtr();
479 0 : SqeRingBuffer *sqeContextBuffer = &(sqeContext->buffer);
480 0 : return dfx::ProfilingManager::ReportMainStreamTask(mainStream_, sqeContextBuffer->tailSqeTaskId - 1, TAIL_TASK);
481 : }
482 : }
483 :
484 0 : HcclResult HcclCommAicpu::NotifyWait(void)
485 : {
486 0 : if (isDeviceMode_) {
487 0 : return InvokeKfcHandler(AicpuKfcHandlerType::kNotifyWait,
488 0 : {rpc_, reinterpret_cast<u64>(dispatcher_), reinterpret_cast<u64>(&mainStream_)});
489 : } else {
490 0 : HcclSqeContext *sqeContext = mainStream_.GetSqeContextPtr();
491 0 : SqeRingBuffer *sqeContextBuffer = &(sqeContext->buffer);
492 0 : CHK_RET(dfx::ProfilingManager::ReportMainStreamTask(mainStream_, sqeContextBuffer->tailSqeTaskId, HEAD_TASK));
493 0 : return LocalNotify::Wait(mainStream_, dispatcher_, opNotifies_[0]);
494 : }
495 : }
496 :
497 : // 按照算子模式来Post对应的Notify
498 0 : HcclResult HcclCommAicpu::RecordHostOrder(const HcclOpResParam *commParam, const std::string& tag, u8 orderLaunchMode)
499 : {
500 0 : const u8 orderLaunchInvalidInHcom = 255;
501 0 : if (orderLaunchMode == orderLaunchInvalidInHcom) {
502 0 : HCCL_INFO("[%s] attachedStreams_[%d] is invalid in graph mode", __func__, orderStream_.id());
503 0 : return HCCL_SUCCESS;
504 : }
505 0 : if (orderNotifies_[orderLaunchMode] == nullptr) {
506 0 : std::shared_ptr<LocalNotify> notify;
507 0 : HcclSignalInfo *aicpuOrderNotify = reinterpret_cast<HcclSignalInfo*>(static_cast<u64>(commParam->aicpuOrderNotifyAddr) +
508 0 : (sizeof(HcclSignalInfo) * orderLaunchMode));
509 0 : HCCL_INFO("[%s] attachedStreams_[%d] aicpuOrderNotify resId[%llu], addr[0x%llx], flag[%u], devId[%u], tsId[%u], rankId[%u]",
510 : __func__, orderStream_.id(), aicpuOrderNotify->resId, aicpuOrderNotify->addr,
511 : aicpuOrderNotify->flag, aicpuOrderNotify->devId, aicpuOrderNotify->tsId, aicpuOrderNotify->rankId);
512 0 : HcclResult ret = InitAndVerifySingleSignal(*aicpuOrderNotify, notify);
513 0 : CHK_PRT_RET(ret != HCCL_SUCCESS, HCCL_ERROR("[%s] check localRes noftify failed, resId[%u], group[%s]",
514 : __func__, aicpuOrderNotify->resId, identifier_.c_str()), ret);
515 0 : orderNotifies_[orderLaunchMode] = notify;
516 0 : HCCL_INFO("%s success, group[%s], resId[%llu]", __func__, identifier_.c_str(), aicpuOrderNotify->resId);
517 0 : }
518 :
519 0 : HCCL_INFO("%s group[%s] tag[%s] isDeviceMode[%d] orderLaunchMode[%u] mode[%d] streamId[%d] notifyId[%u]",
520 : __func__, identifier_.c_str(), tag.c_str(), isDeviceMode_, orderLaunchMode, GetWorkflowMode(), orderStream_.id(),
521 : orderNotifies_[orderLaunchMode]->notifyId_);
522 0 : CHK_RET(LocalNotify::Post(orderStream_, dispatcher_, orderNotifies_[orderLaunchMode]));
523 0 : CHK_RET(LaunchTask(dispatcher_, const_cast<Stream &>(orderStream_)));
524 0 : return HCCL_SUCCESS;
525 : }
526 :
527 462 : HcclResult HcclCommAicpu::GetStreamData(
528 : const HcclStreamInfo &streamInfo, HcclComStreamInfo &comStreamInfo, u32 &sqHead, u32 &sqTail)
529 : {
530 462 : comStreamInfo.sqId = streamInfo.sqIds;
531 462 : comStreamInfo.actualStreamId = streamInfo.streamIds;
532 462 : comStreamInfo.logicCqId = streamInfo.logicCqids;
533 462 : u64 sq_addr = 0;
534 462 : CHK_RET(QuerySqBaseAddr(devId_, streamInfo.sqIds, sq_addr));
535 462 : comStreamInfo.sqBaseAddr = reinterpret_cast<void *>(sq_addr);
536 462 : if (comStreamInfo.sqBaseAddr == nullptr) {
537 0 : HCCL_ERROR("[HcclCommAicpu][GetStreamData]sqe base addr ptr is null.");
538 0 : return HCCL_E_PARA;
539 : }
540 462 : CHK_RET(QuerySqStatusByType(devId_, streamInfo.sqIds, DRV_SQCQ_PROP_SQ_DEPTH, comStreamInfo.sqDepth));
541 462 : CHK_RET(QuerySqStatusByType(devId_, streamInfo.sqIds, DRV_SQCQ_PROP_SQ_TAIL, sqTail));
542 462 : CHK_RET(QuerySqStatusByType(devId_, streamInfo.sqIds, DRV_SQCQ_PROP_SQ_HEAD, sqHead));
543 462 : HCCL_DEBUG("[HcclCommAicpu][GetStreamData] get stream data success, group[%s], streamId[%d], sqId[%d], "
544 : "logicCqId[%u], sqDepth[%u], sqHead[%u], sqTail[%u]",
545 : identifier_.c_str(),
546 : comStreamInfo.actualStreamId,
547 : comStreamInfo.sqId,
548 : comStreamInfo.logicCqId,
549 : comStreamInfo.sqDepth,
550 : sqHead,
551 : sqTail);
552 462 : return HCCL_SUCCESS;
553 : }
554 :
555 11 : HcclResult HcclCommAicpu::InitMainStreamObj(const HcclOpResParam *commParam)
556 : {
557 11 : CHK_RET(InitStreamObj(commParam->localRes.mainStreamParam, mainStream_));
558 11 : HCCL_INFO("%s success, group[%s], streamId[%d]", __func__, identifier_.c_str(), mainStream_.id());
559 11 : return HCCL_SUCCESS;
560 : }
561 :
562 11 : HcclResult HcclCommAicpu::InitSlaveStreamObjs(const HcclOpResParam *commParam)
563 : {
564 11 : if (commParam->localRes.streamNum > LOCAL_STREAM_MAX_NUM) {
565 0 : HCCL_ERROR("[HcclCommAicpu][InitSlaveStreamObjs] local streams great max numbers,current numbers[%u], max "
566 : "numbers[%u], group[%s]",
567 : commParam->localRes.streamNum,
568 : LOCAL_STREAM_MAX_NUM,
569 : identifier_.c_str());
570 0 : return HCCL_E_PARA;
571 : }
572 451 : for (u32 i = 0; i < commParam->localRes.streamNum; i++) {
573 440 : if (streamToObj_.find(commParam->localRes.streamParam[i].streamInfo.sqIds) == streamToObj_.end()) {
574 440 : Stream stream;
575 440 : CHK_RET(InitStreamObj(commParam->localRes.streamParam[i], stream));
576 440 : slaveStreams_.emplace_back(stream);
577 440 : streamToObj_.insert(commParam->localRes.streamParam[i].streamInfo.sqIds);
578 440 : }
579 : }
580 11 : HCCL_DEBUG("[HcclCommAicpu][InitSlaveStreamObjs] success, group[%s], slave stream numbers[%u]",
581 : identifier_.c_str(),
582 : commParam->localRes.streamNum);
583 11 : return HCCL_SUCCESS;
584 : }
585 :
586 11 : HcclResult HcclCommAicpu::InitOrderStreamObj(const HcclOpResParam *commParam)
587 : {
588 11 : CHK_RET(InitStreamObj(commParam->aicpuOrderStreamParam, orderStream_));
589 11 : HCCL_INFO("%s success, group[%s], streamId[%d]", __func__, identifier_.c_str(), orderStream_.id());
590 11 : return HCCL_SUCCESS;
591 : }
592 :
593 462 : HcclResult HcclCommAicpu::InitStreamObj(const HcclStreamParam& streamParam, Stream& stream)
594 : {
595 : u32 sqTail;
596 : u32 sqHead;
597 462 : HcclResult ret = HCCL_SUCCESS;
598 462 : ret = StreamRestore(streamParam.streamInfo.streamIds);
599 462 : CHK_PRT_RET(ret != HCCL_SUCCESS, HCCL_ERROR("%s StreamId[%d] Restore error group[%s]",
600 : __func__, streamParam.streamInfo.streamIds, identifier_.c_str()), ret);
601 :
602 462 : HcclComStreamInfo comStreamInfo = {0};
603 462 : ret = GetStreamData(streamParam.streamInfo, comStreamInfo, sqHead, sqTail);
604 462 : CHK_PRT_RET(ret != HCCL_SUCCESS, HCCL_ERROR("%s error group[%s]", __func__, identifier_.c_str()), ret);
605 462 : stream = Stream(comStreamInfo);
606 :
607 : // 初始化stream的sqeContext
608 462 : SqCqeContext* sqCqeContext = reinterpret_cast<SqCqeContext*>(streamParam.sqCqContextAddr);
609 462 : u64 sqCqContextSize = streamParam.sqCqContextSize;
610 462 : if (sqCqeContext == nullptr || sqCqContextSize != sizeof(SqCqeContext)) {
611 0 : HCCL_ERROR("%s failed, sqCqeContext[%p] is null or size[%llu] does not match expected size[%llu], group[%s]",
612 : __func__, sqCqeContext, sqCqContextSize, sizeof(SqCqeContext), identifier_.c_str());
613 0 : return HCCL_E_PARA;
614 : }
615 462 : ret = stream.InitSqAndCqeContext(sqHead, sqTail, sqCqeContext);
616 462 : CHK_PRT_RET(ret != HCCL_SUCCESS,
617 : HCCL_ERROR("%s: InitSqAndCqeContext failed, group[%s]", __func__, identifier_.c_str()), ret);
618 462 : HCCL_INFO("%s success, group[%s], streamId[%d]", __func__, identifier_.c_str(), stream.id());
619 462 : return HCCL_SUCCESS;
620 : }
621 :
622 11 : HcclResult HcclCommAicpu::InitOpNotifyObj(const HcclOpResParam *commParam)
623 : {
624 11 : HcclResult ret = HCCL_SUCCESS;
625 33 : for (u32 i = 0; i < AICPU_OP_NOTIFY_MAX_NUM; i++) {
626 22 : ret = InitAndVerifySignal(commParam->localRes.aicpuOpNotify[i], opNotifies_);
627 22 : CHK_PRT_RET(ret != HCCL_SUCCESS,
628 : HCCL_ERROR("[HcclCommAicpu][InitOpNotifyObj] check localRes op noftify failed, resId[%u], group[%s]",
629 : commParam->localRes.localSignals[i].resId,
630 : identifier_.c_str()),
631 : ret);
632 : }
633 11 : CHK_RET(hrtDrvGetLocalDevIDByHostDevID(commParam->localRes.aicpuOpNotify[0].devId, &devId_));
634 11 : HCCL_INFO("[HcclCommAicpu][InitOpNotifyObj] success, group[%s]", identifier_.c_str());
635 11 : return HCCL_SUCCESS;
636 : }
637 :
638 11 : HcclResult HcclCommAicpu::InitLocalNotifyObj(const HcclOpResParam *commParam)
639 : {
640 11 : HcclResult ret = HCCL_SUCCESS;
641 :
642 11 : if (commParam->localRes.signalNum > LOCAL_NOTIFY_MAX_NUM) {
643 0 : HCCL_ERROR("[HcclCommAicpu][InitLocalNotifyObj] local notifys great max numbers, numbers[%u], max numbers[%u], "
644 : "group[%s]",
645 : commParam->localRes.signalNum,
646 : LOCAL_NOTIFY_MAX_NUM,
647 : identifier_.c_str());
648 0 : return HCCL_E_PARA;
649 : }
650 715 : for (u32 i = 0; i < commParam->localRes.signalNum; i++) {
651 704 : if (notifysToObj_.find(commParam->localRes.localSignals[i].resId) == notifysToObj_.end()) {
652 704 : ret = InitAndVerifySignal(commParam->localRes.localSignals[i], localNotifies_);
653 704 : CHK_PRT_RET(ret != HCCL_SUCCESS,
654 : HCCL_ERROR("[HcclCommAicpu][InitLocalNotifyObj] check localRes prenoftify failed, resId[%u]",
655 : commParam->localRes.localSignals[i].resId),
656 : ret);
657 704 : notifysToObj_.insert(commParam->localRes.localSignals[i].resId);
658 : }
659 : }
660 11 : HCCL_DEBUG("[HcclCommAicpu][InitLocalNotifyObj] success, group[%s], signal numbers[%u]",
661 : identifier_.c_str(),
662 : commParam->localRes.signalNum);
663 11 : return HCCL_SUCCESS;
664 : }
665 :
666 22 : HcclResult HcclCommAicpu::ParseTlvToVector(
667 : u64 srcTlv, u64 srcTlvTotalLength, std::vector<std::vector<std::vector<u32>>> &vectorInfo)
668 : {
669 22 : u64 parseLength = 0;
670 22 : u8 *srcTlvptr = reinterpret_cast<u8 *>(srcTlv);
671 22 : if (srcTlvptr == nullptr) {
672 0 : HCCL_ERROR("[HcclCommAicpu][ParseTlvToVector]tlv ptr is null.");
673 0 : return HCCL_E_PARA;
674 : }
675 22 : u64 unPareseLength = srcTlvTotalLength;
676 143 : while (unPareseLength > 0) {
677 121 : CommonTlv *commonTlv = reinterpret_cast<CommonTlv *>(srcTlvptr + parseLength);
678 121 : if (unPareseLength <= (sizeof(LENGTH_TYPE) + sizeof(TAG_TYPE))) {
679 0 : HCCL_ERROR("[HcclCommAicpu][ParseTlvToVector] Tlv length is error, tag[%s], totalLength[%lu], "
680 : "unParseLength[%lu], already parsed Length[%lu]", identifier_.c_str(), srcTlvTotalLength,
681 : unPareseLength, parseLength);
682 0 : return HCCL_E_PARA;
683 : }
684 121 : if (commonTlv->length > unPareseLength || commonTlv->length % sizeof(RANK_TYPE) != 0) {
685 0 : HCCL_ERROR(
686 : "[HcclCommAicpu][ParseTlvToVector] parse Tlv error, group[%s], total Length[%lu], tlvLength[%lu], "
687 : "unParsed Length[%lu], already parsed Length[%lu]", identifier_.c_str(), srcTlvTotalLength,
688 : commonTlv->length, unPareseLength, parseLength);
689 0 : return HCCL_E_PARA;
690 : }
691 121 : u16 level0 = (commonTlv->type & TOP_COMM_LEVEL0_LOCATION) >> TOP_COMM_LEVEL0_SHIFT;
692 121 : u16 level1 = (commonTlv->type & TOP_COMM_LEVEL1_LOCATION);
693 121 : u64 itemNum = (commonTlv->length - sizeof(TAG_TYPE) - sizeof(LENGTH_TYPE)) / sizeof(RANK_TYPE);
694 242 : std::vector<RANK_TYPE> values{&commonTlv->value, (&commonTlv->value) + itemNum};
695 121 : if (level0 >= vectorInfo.size()) {
696 99 : vectorInfo.resize(level0 + 1);
697 : }
698 121 : if (level1 >= vectorInfo[level0].size()) {
699 121 : vectorInfo[level0].resize(level1 + 1);
700 : }
701 121 : vectorInfo[level0][level1] = std::move(values);
702 121 : parseLength += commonTlv->length;
703 121 : unPareseLength -= commonTlv->length;
704 121 : HCCL_DEBUG("[HcclCommAicpu][ParseTlvToVector] parse Tlv group[%s], level0[%u], level1[%u], total Length[%lu], "
705 : "tlvLength[%lu], unParsed Length[%lu], already parsed Length[%lu]", identifier_.c_str(),
706 : level0, level1, srcTlvTotalLength, commonTlv->length, unPareseLength, parseLength);
707 121 : }
708 143 : for (u32 idx = 0; idx < vectorInfo.size(); idx++) {
709 242 : for (u32 ringidx = 0; ringidx < vectorInfo[idx].size(); ringidx++) {
710 121 : HCCL_DEBUG("[HcclCommAicpu][ParseTlvToVector] idx[%u] ringidx[%u] size[%u]", idx, ringidx,
711 : vectorInfo[idx][ringidx].size());
712 : }
713 : }
714 22 : HCCL_INFO("[HcclCommAicpu][ParseTlvToVector] success, group[%s]", identifier_.c_str());
715 22 : return HCCL_SUCCESS;
716 : }
717 :
718 11 : HcclResult HcclCommAicpu::ParseTlvToSubGroupVector(
719 : u64 srcTlv, u64 srcTlvTotalLength, std::vector<std::vector<std::vector<std::vector<u32>>>> &vectorInfo)
720 : {
721 11 : u64 parseLength = 0;
722 11 : u8 *srcTlvptr = reinterpret_cast<u8 *>(srcTlv);
723 11 : if (srcTlvptr == nullptr) {
724 0 : HCCL_ERROR("[HcclCommAicpu][ParseTlvToSubGroupVector]tlv ptr is null.");
725 0 : return HCCL_E_PARA;
726 : }
727 11 : u64 unPareseLength = srcTlvTotalLength;
728 33 : while (unPareseLength > 0) {
729 22 : CommonTlv *commonTlv = reinterpret_cast<CommonTlv *>(srcTlvptr + parseLength);
730 22 : if (unPareseLength <= (sizeof(LENGTH_TYPE) + sizeof(TAG_TYPE))) {
731 0 : HCCL_ERROR("[HcclCommAicpu][ParseTlvToSubGroupVector] Tlv length is error, tag[%s], totalLength[%lu], "
732 : "unParseLength[%lu], already parsed Length[%lu]", identifier_.c_str(), srcTlvTotalLength,
733 : unPareseLength, parseLength);
734 0 : return HCCL_E_PARA;
735 : }
736 22 : if (commonTlv->length > unPareseLength || commonTlv->length % sizeof(RANK_TYPE) != 0) {
737 0 : HCCL_ERROR(
738 : "[HcclCommAicpu][ParseTlvToSubGroupVector] parse Tlv error, group[%s], total Length[%lu], tlvLength[%lu], "
739 : "unParsed Length[%lu], already parsed Length[%lu]", identifier_.c_str(), srcTlvTotalLength,
740 : commonTlv->length, unPareseLength, parseLength);
741 0 : return HCCL_E_PARA;
742 : }
743 22 : u16 level0 = (commonTlv->type & TOP_HIERARCHICAL_COMM_LEVEL0_LOCATION) >> (TOP_HIERARCHICAL_COMM_LEVEL0_SHIFT + TOP_HIERARCHICAL_COMM_LEVEL1_SHIFT);
744 22 : u16 level1 = (commonTlv->type & TOP_HIERARCHICAL_COMM_LEVEL1_LOCATION) >> TOP_HIERARCHICAL_COMM_LEVEL1_SHIFT;
745 22 : u16 level2 = (commonTlv->type & TOP_HIERARCHICAL_COMM_LEVEL2_LOCATION);
746 22 : u64 itemNum = (commonTlv->length - sizeof(TAG_TYPE) - sizeof(LENGTH_TYPE)) / sizeof(RANK_TYPE);
747 44 : std::vector<RANK_TYPE> values{&commonTlv->value, (&commonTlv->value) + itemNum};
748 22 : if (level0 >= vectorInfo.size()) {
749 11 : vectorInfo.resize(level0 + 1);
750 : }
751 22 : if (level1 >= vectorInfo[level0].size()) {
752 11 : vectorInfo[level0].resize(level1 + 1);
753 : }
754 22 : if (level2 >= vectorInfo[level0][level1].size()) {
755 22 : vectorInfo[level0][level1].resize(level2 + 1);
756 : }
757 22 : vectorInfo[level0][level1][level2] = std::move(values);
758 22 : parseLength += commonTlv->length;
759 22 : unPareseLength -= commonTlv->length;
760 22 : HCCL_DEBUG("[HcclCommAicpu][ParseTlvToSubGroupVector] parse Tlv group[%s], level0[%u], level1[%u], total Length[%lu], "
761 : "tlvLength[%lu], unParsed Length[%lu], already parsed Length[%lu]", identifier_.c_str(),
762 : level0, level1, srcTlvTotalLength, commonTlv->length, unPareseLength, parseLength);
763 22 : }
764 22 : for (u32 level0Idx = 0; level0Idx < vectorInfo.size(); level0Idx++) {
765 22 : for (u32 level1Idx = 0; level1Idx < vectorInfo[level0Idx].size(); level1Idx++) {
766 33 : for (u32 level2Idx = 0; level2Idx < vectorInfo[level0Idx][level1Idx].size(); level2Idx++) {
767 22 : HCCL_DEBUG("[HcclCommAicpu][ParseTlvToSubGroupVector] level0Idx[%u] level1Idx[%u] level2Idx[%u] size[%u]", level0Idx, level1Idx,
768 : level2Idx, vectorInfo[level0Idx][level1Idx][level2Idx].size());
769 : }
770 : }
771 : }
772 11 : HCCL_INFO("[HcclCommAicpu][ParseTlvToSubGroupVector] success, group[%s]", identifier_.c_str());
773 11 : return HCCL_SUCCESS;
774 : }
775 :
776 11 : HcclResult HcclCommAicpu::InitTopoInfo(const HcclOpResParam *commParam)
777 : {
778 11 : topoInfo_.userRank = commParam->topoInfo.userRank;
779 11 : topoInfo_.userRankSize = commParam->topoInfo.userRankSize;
780 11 : topoInfo_.deviceLogicId = commParam->topoInfo.deviceLogicId;
781 11 : topoInfo_.isSingleMeshAggregation = commParam->topoInfo.isSingleMeshAggregation;
782 11 : topoInfo_.deviceNumPerAggregation = commParam->topoInfo.deviceNumPerAggregation;
783 11 : topoInfo_.superPodNum = commParam->topoInfo.superPodNum;
784 11 : topoInfo_.devicePhyId = commParam->topoInfo.devicePhyId;
785 11 : topoInfo_.deviceType = static_cast<DevType>(commParam->topoInfo.deviceType);
786 11 : topoInfo_.topoType = static_cast<TopoType>(commParam->topoInfo.topoType);
787 11 : topoInfo_.is310P3Common = (topoInfo_.deviceType == DevType::DEV_TYPE_310P3);
788 11 : topoInfo_.serverNum = commParam->topoInfo.serverNum;
789 11 : topoInfo_.meshAggregationRankSize = commParam->topoInfo.meshAggregationRankSize;
790 11 : topoInfo_.multiModuleDiffDeviceNumMode = commParam->topoInfo.multiModuleDiffDeviceNumMode;
791 11 : topoInfo_.multiSuperPodDiffServerNumMode = commParam->topoInfo.multiSuperPodDiffServerNumMode;
792 11 : topoInfo_.realUserRank = commParam->topoInfo.realUserRank;
793 11 : topoInfo_.isDiffDeviceModule = commParam->topoInfo.isDiffDeviceModule;
794 11 : topoInfo_.isDiffDeviceType = commParam->topoInfo.isDiffDeviceType;
795 11 : topoInfo_.gcdDeviceNumPerAggregation = commParam->topoInfo.gcdDeviceNumPerAggregation;
796 11 : topoInfo_.moduleNum = commParam->topoInfo.moduleNum;
797 11 : topoInfo_.useSuperPodMode = true;
798 11 : topoInfo_.isARSDoubleRing = commParam->isARSDoubleRing;
799 11 : topoInfo_.multiSuperPodDiffDeviceNumMode = commParam->multiSuperPodDiffDeviceNumMode;
800 11 : if (commParam->topoInfo.isUsedRdmaRankPairNum % KEY_VALUE_TO_VECTOR_MODULUS != 0) {
801 0 : HCCL_ERROR("[HcclCommAicpu][InitTopoInfo]rdma rank pair number[%lu] is error.",
802 : commParam->topoInfo.isUsedRdmaRankPairNum);
803 0 : return HCCL_E_PARA;
804 : }
805 11 : u32 *isUsedRdmaRankPairPtr = reinterpret_cast<u32 *>(commParam->topoInfo.isUsedRdmaRankPair);
806 11 : if (isUsedRdmaRankPairPtr == nullptr) {
807 0 : HCCL_ERROR("[HcclCommAicpu][InitTopoInfo]rdma rank pair ptr is null.");
808 0 : return HCCL_E_PARA;
809 : }
810 55 : for (u64 i = 0; i < commParam->topoInfo.isUsedRdmaRankPairNum; i += KEY_VALUE_TO_VECTOR_MODULUS) {
811 44 : topoInfo_.isUsedRdmaMap.insert({isUsedRdmaRankPairPtr[i], static_cast<bool>(isUsedRdmaRankPairPtr[i + 1])});
812 : }
813 :
814 11 : if (commParam->topoInfo.pairLinkCounterNum % KEY_VALUE_TO_VECTOR_MODULUS != 0) {
815 0 : HCCL_ERROR("[HcclCommAicpu][InitTopoInfo]pair link count number[%lu] is error.",
816 : commParam->topoInfo.pairLinkCounterNum);
817 0 : return HCCL_E_PARA;
818 : }
819 11 : u32 *pairLinkCounterPtr = reinterpret_cast<u32 *>(commParam->topoInfo.pairLinkCounter);
820 11 : if (pairLinkCounterPtr == nullptr) {
821 0 : HCCL_ERROR("[HcclCommAicpu][InitTopoInfo]rdma rank pair ptr is null.");
822 0 : return HCCL_E_PARA;
823 : }
824 :
825 55 : for (u64 i = 0; i < commParam->topoInfo.pairLinkCounterNum; i += KEY_VALUE_TO_VECTOR_MODULUS) {
826 44 : topoInfo_.pairLinkCounter.insert({pairLinkCounterPtr[i], pairLinkCounterPtr[i + 1]});
827 : }
828 :
829 11 : u32 *nicListPtr = reinterpret_cast<u32 *>(commParam->topoInfo.nicList);
830 11 : if (nicListPtr == nullptr) {
831 0 : HCCL_ERROR("[HcclCommAicpu][InitTopoInfo]nic list ptr is null.");
832 0 : return HCCL_E_PARA;
833 : }
834 22 : std::vector<u32> niclist{nicListPtr, nicListPtr + commParam->topoInfo.nicNum};
835 11 : topoInfo_.nicList = std::move(niclist);
836 :
837 11 : bool *bridgeRankPtr = reinterpret_cast<bool *>(commParam->topoInfo.bridgeRank);
838 11 : if (bridgeRankPtr != nullptr) {
839 11 : isBridgeVector_.resize(commParam->topoInfo.bridgeRankNum);
840 33 : for (u32 i = 0; i < commParam->topoInfo.bridgeRankNum; ++i) {
841 22 : isBridgeVector_[i] = bridgeRankPtr[i];
842 22 : HCCL_DEBUG("[HcclCommAicpu][InitTopoInfo] bridge rank info idx[%u] value[%u]", i, isBridgeVector_[i]);
843 : }
844 : } else {
845 0 : HCCL_RUN_INFO("[HcclCommAicpu][InitTopoInfo] bridge rank number is 0, group[%s]", identifier_.c_str());
846 : }
847 :
848 11 : HcclResult ret = HCCL_SUCCESS;
849 11 : ret = ParseTlvToVector(commParam->topoInfo.complanRank, commParam->topoInfo.complanRankLength, commPlaneVector_);
850 11 : CHK_PRT_RET(ret != HCCL_SUCCESS,
851 : HCCL_ERROR("[HcclCommAicpu][InitTopoInfo]Init CommPlane error group[%s]", identifier_.c_str()),
852 : ret);
853 :
854 22 : ret = ParseTlvToVector(commParam->topoInfo.serverAndsuperPodRank,
855 11 : commParam->topoInfo.serverAndsuperPodRankLength,
856 11 : serverAndsuperPodToRank_);
857 11 : CHK_PRT_RET(ret != HCCL_SUCCESS,
858 : HCCL_ERROR("[HcclCommAicpu][InitTopoInfo]Init server and superPod rank error group[%s]", identifier_.c_str()),
859 : ret);
860 :
861 11 : ret = ParseTlvToSubGroupVector(commParam->hierarchicalAlgInfo.commplaneSubGroupRank, commParam->hierarchicalAlgInfo.commplaneSubGroupRankLength, topoInfo_.CommPlaneSubGroupVector);
862 11 : CHK_PRT_RET(ret != HCCL_SUCCESS,
863 : HCCL_ERROR("[HcclCommAicpu][InitTopoInfo]Init CommPlaneSubGroup error group[%s]", identifier_.c_str()),
864 : ret);
865 :
866 11 : HCCL_INFO("[HcclCommAicpu][InitTopoInfo] success, group[%s], device Type[%u]",
867 : identifier_.c_str(),
868 : commParam->topoInfo.deviceType);
869 11 : return HCCL_SUCCESS;
870 11 : }
871 :
872 11 : HcclResult HcclCommAicpu::InitCclbuffer(const HcclOpResParam *commParam)
873 : {
874 22 : if (commParam->localWindowsIn == 0 || commParam->localWindowsOut == 0 ||
875 33 : cclInputBuffer_.ptr() != nullptr || cclOutputBuffer_.ptr() != nullptr) {
876 0 : HCCL_INFO("[HcclCommAicpu][InitCclBuffer] don't need init cclbuffer "
877 : "ccl winIn[0x%lx] winout[0x%lx] cclin ptr[%p] cclout ptr[%p]",
878 : commParam->localWindowsIn, commParam->localWindowsOut, cclInputBuffer_.ptr(), cclOutputBuffer_.ptr());
879 0 : return HCCL_SUCCESS;
880 : }
881 :
882 11 : auto cclInPtr = reinterpret_cast<void *>(commParam->localWindowsIn);
883 11 : auto cclOutPtr = reinterpret_cast<void *>(commParam->localWindowsOut);
884 11 : cclInputBuffer_ = DeviceMem::create(cclInPtr, commParam->winSize);
885 11 : cclOutputBuffer_ = DeviceMem::create(cclOutPtr, commParam->winSize);
886 11 : cclbufferSize_ = commParam->winSize;
887 11 : HCCL_INFO("[HcclCommAicpu][InitCclbuffer] success, group[%s], cclin[%llu], cclout[%llu], size[%lu]",
888 : identifier_.c_str(),
889 : commParam->localWindowsIn,
890 : commParam->localWindowsOut,
891 : commParam->winSize
892 : );
893 11 : return HCCL_SUCCESS;
894 : }
895 :
896 11 : HcclResult HcclCommAicpu::InitConfigInfo(const HcclOpResParam *commParam)
897 : {
898 11 : deterministic_ = commParam->config.deterministic;
899 11 : interHccsDisable_ = commParam->config.interHccsDisable;
900 11 : multiQpThreshold_ = commParam->config.multiQpThreshold;
901 11 : inlineReducEnable_ = true;
902 11 : fftsEnable_ = false;
903 11 : taskMonitorInterval_ = commParam->config.taskMonitorInterval;
904 11 : algoInfo_.inlineReduceSwitchOn = true;
905 11 : algoInfo_.identifier = commParam->hcomId;
906 11 : algoInfo_.isSupportAtomicWrite = static_cast<bool>(commParam->config.isSupportAtomicWrite);
907 11 : notifySize_ = commParam->notifysize;
908 11 : slaveStreams_.reserve(LOCAL_STREAM_MAX_NUM);
909 11 : localNotifies_.reserve(LOCAL_NOTIFY_MAX_NUM);
910 11 : SetExternalInputDebugConfig(commParam->debugConfig);
911 11 : InitDebugConfigByValue(commParam->debugConfig);
912 11 : HCCL_INFO("[HcclCommAicpu][Init]success, group[%s] reserve noipc notifys[%lu], slave streams[%lu].",
913 : identifier_.c_str(),
914 : LOCAL_NOTIFY_MAX_NUM,
915 : LOCAL_STREAM_MAX_NUM);
916 11 : return HCCL_SUCCESS;
917 : }
918 :
919 13 : void HcclCommAicpu::SetDumpDebug(bool dumpDebug)
920 : {
921 13 : dumpDebug_ = dumpDebug;
922 13 : }
923 :
924 : template <typename T>
925 726 : HcclResult HcclCommAicpu::InitAndVerifySignal(const HcclSignalInfo &signalInfo, std::vector<std::shared_ptr<T>> ¬ifyVec)
926 : {
927 726 : if (signalInfo.resId == INVALID_U64) {
928 : // 无效值不做校验
929 0 : HCCL_INFO("[HcclCommAicpu][InitAndVerifySignal] resId is invalid, need not check");
930 0 : return HCCL_SUCCESS;
931 : }
932 :
933 726 : std::shared_ptr<T> notify;
934 726 : EXCEPTION_CATCH((notify = std::make_shared<T>()), return HCCL_E_PTR);
935 726 : CHK_SMART_PTR_NULL(notify);
936 726 : CHK_RET(notify->Init(signalInfo, NotifyLoadType::DEVICE_NOTIFY));
937 726 : notifyVec.push_back(notify);
938 726 : HCCL_INFO("[HcclCommAicpu][InitAndVerifySignal] success group[%s], resId[%u], tsId:%d, devId[%u]",
939 : identifier_.c_str(),
940 : signalInfo.resId,
941 : signalInfo.tsId,
942 : signalInfo.devId);
943 :
944 726 : return HCCL_SUCCESS;
945 726 : }
946 :
947 33 : void HcclCommAicpu::HandleExistTagReAlloc(HccltagLocalResV2* tagRes, const std::string& tag, bool reAllocFlag,
948 : ListCommon*& curList, bool& needSkip)
949 : {
950 33 : needSkip = false;
951 : // 如果 tag 已经存在且需要重新申请内存, 删除旧的资源与记录
952 33 : if (localTagResToObj_.find(tag) != localTagResToObj_.end() && reAllocFlag) {
953 0 : u64 currSize = tagRes->ScratchmemSize;
954 0 : u64 existSize = tagScratchMem_[tag]->size();
955 : // 新内存更大，删除旧记录，保留新的
956 0 : if (currSize > existSize) {
957 0 : localTagResToObj_.erase(tag);
958 0 : tagScratchMem_.erase(tag);
959 0 : HCCL_DEBUG("[HcclCommAicpu][%s] tag exists, replace old small mem, tag[%s], old[%lu] new[%lu]",
960 : __func__, tag.c_str(), existSize, currSize);
961 : } else {
962 : // 旧内存更大，需要跳过
963 0 : HCCL_DEBUG("[HcclCommAicpu][%s] tag exists, keep larger old mem, skip new, tag[%s], old[%lu] new[%lu]",
964 : __func__, tag.c_str(), existSize, currSize);
965 0 : curList = reinterpret_cast<ListCommon *>(curList->nextDevice);
966 0 : needSkip = true;
967 : }
968 : }
969 33 : }
970 :
971 11 : HcclResult HcclCommAicpu::InitLocalTagRes(const ListCommon &head, bool reAllocFlag)
972 : {
973 11 : ListCommon *curList = reinterpret_cast<ListCommon *>(head.nextDevice);
974 11 : if (curList == nullptr) {
975 0 : HCCL_ERROR("[HcclCommAicpu][InitLocalTagRes]list ptr is null.");
976 0 : return HCCL_E_PARA;
977 : }
978 44 : while (curList != &head) {
979 33 : HccltagLocalResV2 *tagRes = list_entry(curList, HccltagLocalResV2, nextTagRes);
980 33 : std::string tag = tagRes->tag;
981 33 : bool needSkip = false;
982 33 : HandleExistTagReAlloc(tagRes, tag, reAllocFlag, curList, needSkip);
983 33 : if (needSkip) {
984 0 : continue;
985 : }
986 33 : if (localTagResToObj_.find(tag) == localTagResToObj_.end() ||
987 33 : localTagResToObj_[tag].find(tagRes->Scratchmem) == localTagResToObj_[tag].end()) {
988 33 : auto scratchMemPtr = reinterpret_cast<void *>(tagRes->Scratchmem);
989 33 : if (scratchMemPtr == nullptr) {
990 0 : HCCL_ERROR("[HcclCommAicpu][InitLocalTagRes]scratch mem ptr is null.");
991 0 : return HCCL_E_PARA;
992 : }
993 33 : DeviceMem loalScratchmem = DeviceMem::create(scratchMemPtr, tagRes->ScratchmemSize);
994 33 : std::shared_ptr<DeviceMem> loalScratchmemPtr;
995 33 : EXCEPTION_CATCH(
996 : (loalScratchmemPtr = std::make_shared<DeviceMem>(std::move(loalScratchmem))), return HCCL_E_PTR);
997 33 : CHK_SMART_PTR_NULL(loalScratchmemPtr);
998 33 : if (tagScratchMem_.find(tag) == tagScratchMem_.end()) {
999 33 : tagScratchMem_.insert({tag, loalScratchmemPtr});
1000 : }
1001 33 : std::unordered_set<u64> tmpTagRes;
1002 33 : tmpTagRes.insert(tagRes->Scratchmem);
1003 33 : localTagResToObj_[tag] = tmpTagRes;
1004 33 : HCCL_DEBUG("[HcclCommAicpu][InitLocalTagRes] parse remote resource, tag[%s], Scratchmem[%p], "
1005 : "ScratchmemSize[%lu]", tag.c_str(), tagRes->Scratchmem, tagRes->ScratchmemSize);
1006 33 : }
1007 33 : curList = reinterpret_cast<ListCommon *>(curList->nextDevice);
1008 33 : if (curList == nullptr) {
1009 0 : HCCL_ERROR("[HcclCommAicpu][InitLocalTagRes] next list ptr is null.");
1010 0 : return HCCL_E_PARA;
1011 : }
1012 33 : };
1013 11 : HCCL_INFO("[HcclCommAicpu][InitLocalTagRes] success, group[%s]", identifier_.c_str());
1014 11 : return HCCL_SUCCESS;
1015 : }
1016 :
1017 0 : HcclResult HcclCommAicpu::CheckNotifyOrQPMaxNum(u64 &existNum, const u64 &MaxNum, const bool &isNotifyRes)
1018 : {
1019 0 : std::string resType = isNotifyRes ? "Notify" : "QP";
1020 0 : if (existNum + 1 > MaxNum) {
1021 0 : HCCL_ERROR("[%s]%s resources are insufficient, existNum[%llu], MaxNum is [%llu]",
1022 : __func__, resType.c_str(), existNum, MaxNum);
1023 0 : return HCCL_E_INTERNAL;
1024 : }
1025 0 : HCCL_DEBUG("[%s]%s resources are sufficient, existNum[%llu], MaxNum is [%llu]",
1026 : __func__, resType.c_str(), existNum, MaxNum);
1027 0 : return HCCL_SUCCESS;
1028 0 : }
1029 :
1030 0 : HcclResult HcclCommAicpu::SetTransportMachinePara(MachinePara &machinePara, u32 &rankId,
1031 : const std::string &newTag, TransportLinkType linkType)
1032 : {
1033 0 : machinePara.linkAttribute = 0x03; /* 0x03同时支持目的端和源端发起 */
1034 0 : if (rankData_.find(rankId) == rankData_.end()) {
1035 0 : HCCL_ERROR("[%s]there is no link with rankId[%u]", __func__, rankId);
1036 0 : return HCCL_E_NOT_FOUND;
1037 : }
1038 :
1039 0 : machinePara.localUserrank = localUserRank_;
1040 0 : machinePara.remoteWorldRank = rankData_[rankId].remoteWorldRank;
1041 0 : machinePara.remoteUserrank = rankData_[rankId].remoteUsrRankId;
1042 0 : machinePara.deviceLogicId = topoInfo_.deviceLogicId;
1043 0 : machinePara.localDeviceId = topoInfo_.devicePhyId;
1044 0 : machinePara.deviceType = topoInfo_.deviceType;
1045 0 : machinePara.tag = newTag;
1046 0 : machinePara.isAicpuModeEn = true;
1047 0 : if (linkType == TransportLinkType::RESERVED) {
1048 : // 非910_93 2die sio与hccs并发场景，specifyLink设置为RESERVED_LINK_TYPE，平台层将按实际链路类型建链
1049 0 : machinePara.specifyLink = LinkTypeInServer::RESERVED_LINK_TYPE;
1050 : } else {
1051 : // 910_93 2die sio与hccs并发场景，
1052 : // 并发链路中的的hccs链路specifyLink设置为HCCS_SW_TYPE，平台层将使用hccs链路来建链；
1053 : // 并发链路中的的sio链路specifyLink设置为SIO_TYPE，平台层将使用sio链路来建链
1054 0 : machinePara.specifyLink =
1055 0 : (linkType == TransportLinkType::SIO) ? LinkTypeInServer::SIO_TYPE : LinkTypeInServer::HCCS_SW_TYPE;
1056 : }
1057 :
1058 0 : HCCL_INFO("%s success, group[%s], rankId[%u], linkAttribute[%x], localUserRank[%u], remoteWorldRank[%u], "
1059 : "remoteUserrank[%u], deviceLogicId[%d], localDeviceId[%d], deviceType[%d], newTag[%s], specifyLink[%d], "
1060 : "isAicpuModeEn[%d]",
1061 : __func__, identifier_.c_str(), rankId, machinePara.linkAttribute, machinePara.localUserrank,
1062 : machinePara.remoteWorldRank, machinePara.remoteUserrank, machinePara.deviceLogicId, machinePara.localDeviceId,
1063 : machinePara.deviceType, machinePara.tag.c_str(), machinePara.specifyLink, machinePara.isAicpuModeEn);
1064 0 : return HCCL_SUCCESS;
1065 : }
1066 :
1067 : template <typename T>
1068 0 : HcclResult HcclCommAicpu::InitAndVerifySingleSignal(const HcclSignalInfo &signalInfo, std::shared_ptr<T> ¬ify)
1069 : {
1070 0 : if (signalInfo.resId == INVALID_U64) {
1071 : // 无效值不做校验
1072 0 : HCCL_DEBUG("[%s]resId[%llu] is invalid, need not check", __func__, signalInfo.resId);
1073 0 : return HCCL_SUCCESS;
1074 : }
1075 : HcclSignalInfo tmpSignalInfo;
1076 :
1077 0 : EXCEPTION_CATCH((notify = std::make_shared<T>()), return HCCL_E_PTR);
1078 0 : CHK_SMART_PTR_NULL(notify);
1079 0 : CHK_RET(notify->Init(signalInfo, NotifyLoadType::DEVICE_NOTIFY));
1080 0 : CHK_RET(notify->GetNotifyData(tmpSignalInfo));
1081 0 : HCCL_DEBUG("[%s] success group[%s], resId[%llu], tsId:%d, devId[%u]", __func__, identifier_.c_str(),
1082 : tmpSignalInfo.resId, tmpSignalInfo.tsId, tmpSignalInfo.devId);
1083 :
1084 0 : return HCCL_SUCCESS;
1085 : }
1086 :
1087 0 : HcclResult HcclCommAicpu::SetTagRemoteRes(u32 &rankId, const std::string &tag, HccltagRemoteResV2 *tagRes)
1088 : {
1089 0 : if (rankTagRemoteRes_.find(rankId) == rankTagRemoteRes_.end() ||
1090 0 : rankTagRemoteRes_[rankId].find(tag) == rankTagRemoteRes_[rankId].end()) {
1091 0 : HccltagRemoteResV3 tempTagRemoteRes;
1092 0 : tempTagRemoteRes.tagRemoteResPtr = tagRes;
1093 0 : rankTagRemoteRes_[rankId][tag] = tempTagRemoteRes;
1094 : }
1095 0 : HCCL_DEBUG("[%s]get TagRemoteRes success, rankId[%u], tag[%s]", __func__, rankId, tag.c_str());
1096 0 : return HCCL_SUCCESS;
1097 : }
1098 :
1099 0 : HcclResult HcclCommAicpu::SetTransportPtpNotify(TransportDeviceP2pData &transDevP2pData,
1100 : u64 &p2pNotifyNum, HcclLinkP2pV2 &linkP2p, u32 notifyNum)
1101 : {
1102 0 : u64 actualNotifyNum = 0;
1103 : // 获取Ipc notify信息
1104 0 : CHK_RET(CheckNotifyOrQPMaxNum(actualNotifyNum, LINK_P2P_MAX_NUM, true));
1105 0 : std::shared_ptr<LocalNotify> ipcPreWaitNotify = std::make_shared<LocalNotify>();
1106 0 : CHK_RET(InitAndVerifySingleSignal(linkP2p.localIpcSignal[actualNotifyNum], ipcPreWaitNotify));
1107 0 : transDevP2pData.ipcPreWaitNotify = ipcPreWaitNotify;
1108 :
1109 0 : std::shared_ptr<RemoteNotify> ipcPreRecordNotify = std::make_shared<RemoteNotify>();
1110 0 : CHK_RET(InitAndVerifySingleSignal(linkP2p.remoteIpcSignal[actualNotifyNum], ipcPreRecordNotify));
1111 0 : transDevP2pData.ipcPreRecordNotify = ipcPreRecordNotify;
1112 0 : actualNotifyNum++;
1113 :
1114 0 : CHK_RET(CheckNotifyOrQPMaxNum(actualNotifyNum, LINK_P2P_MAX_NUM, true));
1115 0 : std::shared_ptr<LocalNotify> ipcPostWaitNotify = std::make_shared<LocalNotify>();
1116 0 : CHK_RET(InitAndVerifySingleSignal(linkP2p.localIpcSignal[actualNotifyNum], ipcPostWaitNotify));
1117 0 : transDevP2pData.ipcPostWaitNotify = ipcPostWaitNotify;
1118 :
1119 0 : std::shared_ptr<RemoteNotify> ipcPostRecordNotify = std::make_shared<RemoteNotify>();
1120 0 : CHK_RET(InitAndVerifySingleSignal(linkP2p.remoteIpcSignal[actualNotifyNum], ipcPostRecordNotify));
1121 0 : transDevP2pData.ipcPostRecordNotify = ipcPostRecordNotify;
1122 0 : actualNotifyNum++;
1123 :
1124 0 : transDevP2pData.userLocalNotify.resize(notifyNum, nullptr);
1125 0 : transDevP2pData.userRemoteNotify.resize(notifyNum, nullptr);
1126 :
1127 0 : for (u32 idx = 0; idx < notifyNum; idx++) {
1128 0 : CHK_RET(CheckNotifyOrQPMaxNum(actualNotifyNum, LINK_P2P_MAX_NUM, true));
1129 0 : std::shared_ptr<LocalNotify> ipcWaitNotify = std::make_shared<LocalNotify>();
1130 0 : CHK_RET(InitAndVerifySingleSignal(linkP2p.localIpcSignal[actualNotifyNum], ipcWaitNotify));
1131 0 : transDevP2pData.userLocalNotify[idx] = ipcWaitNotify;
1132 :
1133 0 : std::shared_ptr<RemoteNotify> ipcRecordNotify = std::make_shared<RemoteNotify>();
1134 0 : CHK_RET(InitAndVerifySingleSignal(linkP2p.remoteIpcSignal[actualNotifyNum], ipcRecordNotify));
1135 0 : transDevP2pData.userRemoteNotify[idx] = ipcRecordNotify;
1136 :
1137 0 : actualNotifyNum++;
1138 0 : }
1139 :
1140 0 : HCCL_DEBUG("[%s]get p2pNotify success, actualNotifyNum[%u]", __func__, actualNotifyNum);
1141 0 : p2pNotifyNum = actualNotifyNum;
1142 :
1143 0 : return HCCL_SUCCESS;
1144 0 : }
1145 :
1146 0 : HcclResult HcclCommAicpu::SetTransportRoceQP(TransportDeviceIbverbsData &transDevIbverbsData,
1147 : u64 &roceQpNum, HcclLinkRoceV2 *linkRoce)
1148 : {
1149 0 : roceQpNum = linkRoce->qpsPerConnection;
1150 0 : u32 roceQpNumSum = linkRoce->qpsPerConnection + static_cast<u32>(linkRoce->qpsPerConnection != 1);
1151 0 : transDevIbverbsData.qpInfo.resize(roceQpNumSum);
1152 0 : std::copy_n(linkRoce->QpInfo, roceQpNumSum, transDevIbverbsData.qpInfo.begin());
1153 0 : transDevIbverbsData.multiQpThreshold = multiQpThreshold_;
1154 0 : transDevIbverbsData.qpsPerConnection = linkRoce->qpsPerConnection;
1155 0 : HCCL_INFO("[%s]transDevIbverbsData.qpInfo.qpPtr[%llu], transDevIbverbsData.qpInfo.sqIndex[%u], "
1156 : "transDevIbverbsData.qpInfo.dbIndex[%u], roceQpNum[%llu], roceQpNumSum[%u], multiQpThreshold_[%u]",
1157 : __func__,
1158 : transDevIbverbsData.qpInfo[0].qpPtr,
1159 : transDevIbverbsData.qpInfo[0].sqIndex,
1160 : transDevIbverbsData.qpInfo[0].dbIndex,
1161 : roceQpNum,
1162 : roceQpNumSum,
1163 : multiQpThreshold_);
1164 0 : return HCCL_SUCCESS;
1165 : }
1166 :
1167 0 : HcclResult HcclCommAicpu::SetTransportRoceNotify(TransportDeviceIbverbsData &transDevIbverbsData,
1168 : u64 &roceNotifyNum, HcclLinkRoceV2 *linkRoce, u32 notifyNum)
1169 : {
1170 0 : u64 actualNotifyNum = 0;
1171 0 : if (linkRoce->localNotifyList == 0 || linkRoce->remoteNotifyList == 0)
1172 : {
1173 0 : HCCL_DEBUG("[%s] Empty local and remote notify lists, skipping notify resource creation.", __func__);
1174 0 : return HCCL_SUCCESS;
1175 : }
1176 0 : HcclSignalInfo *localNotifyList = reinterpret_cast<HcclSignalInfo *>(linkRoce->localNotifyList);
1177 0 : AddrKey *remoteNotifyList = reinterpret_cast<AddrKey *>(linkRoce->remoteNotifyList);
1178 0 : if (localNotifyList == nullptr || remoteNotifyList == nullptr) {
1179 0 : HCCL_ERROR("[%s]nullptr found in localNotifyList or remoteNotifyList from device mem, check.", __func__);
1180 0 : return HCCL_E_INTERNAL;
1181 : }
1182 : // 获取RDMA Notify信息
1183 0 : std::shared_ptr<LocalNotify> ackNotify = std::make_shared<LocalNotify>();
1184 0 : CHK_RET(InitAndVerifySingleSignal(localNotifyList[actualNotifyNum], ackNotify));
1185 0 : transDevIbverbsData.ackNotify = ackNotify;
1186 0 : transDevIbverbsData.remoteAckNotifyDetails = remoteNotifyList[actualNotifyNum];
1187 0 : actualNotifyNum++;
1188 :
1189 0 : std::shared_ptr<LocalNotify> dataNotify = std::make_shared<LocalNotify>();
1190 0 : CHK_RET(InitAndVerifySingleSignal(localNotifyList[actualNotifyNum], dataNotify));
1191 0 : transDevIbverbsData.dataNotify = dataNotify;
1192 0 : transDevIbverbsData.remoteDataNotifyDetails = remoteNotifyList[actualNotifyNum];
1193 0 : actualNotifyNum++;
1194 :
1195 0 : std::shared_ptr<LocalNotify> dataAckNotify = std::make_shared<LocalNotify>();
1196 0 : CHK_RET(InitAndVerifySingleSignal(localNotifyList[actualNotifyNum], dataAckNotify));
1197 0 : transDevIbverbsData.dataAckNotify = dataAckNotify;
1198 0 : transDevIbverbsData.remoteDataAckNotifyDetails = remoteNotifyList[actualNotifyNum];
1199 0 : transDevIbverbsData.notifySize = notifySize_;
1200 0 : actualNotifyNum++;
1201 :
1202 0 : transDevIbverbsData.userLocalNotify.resize(linkRoce->qpsPerConnection);
1203 0 : transDevIbverbsData.userRemoteNotifyDetails.resize(linkRoce->qpsPerConnection);
1204 : // 当前多QP下每个QP会多申请一个DataNotify
1205 0 : u64 singleQpNotifySize = linkRoce->singleQPNotifyNum + static_cast<u32>(linkRoce->qpsPerConnection > 1);
1206 0 : for (u32 qpIndex = 0; qpIndex < linkRoce->qpsPerConnection; qpIndex++) {
1207 0 : transDevIbverbsData.userLocalNotify[qpIndex].resize(singleQpNotifySize, nullptr);
1208 0 : transDevIbverbsData.userRemoteNotifyDetails[qpIndex].resize(singleQpNotifySize);
1209 0 : for (u32 i = 0, idx = actualNotifyNum + singleQpNotifySize * qpIndex; i < singleQpNotifySize; ++idx, ++i) {
1210 0 : std::shared_ptr<LocalNotify> locNotify = std::make_shared<LocalNotify>();
1211 0 : CHK_RET(InitAndVerifySingleSignal(localNotifyList[idx], locNotify));
1212 0 : transDevIbverbsData.userLocalNotify[qpIndex][i] = locNotify;
1213 0 : transDevIbverbsData.userRemoteNotifyDetails[qpIndex][i] = remoteNotifyList[idx];
1214 0 : }
1215 : }
1216 0 : roceNotifyNum = linkRoce->singleQPNotifyNum;
1217 0 : HCCL_DEBUG("[%s]get roceNotify success, roceNotifyNum[%u]", __func__, roceNotifyNum);
1218 0 : return HCCL_SUCCESS;
1219 0 : }
1220 :
1221 0 : HcclResult HcclCommAicpu::InitLinkP2p(HccltagRemoteResV2 *tagRes, u32 &rankId, const std::string &newTag, u32 notifyNum,
1222 : TransportLinkType linkType)
1223 : {
1224 0 : std::unordered_map<u32, std::unordered_map<std::string, std::shared_ptr<Transport>>> &linkRes =
1225 : (linkType == TransportLinkType::SIO) ? linkResSio_ : linkRes_;
1226 0 : HcclLinkP2pV2 &linkP2p = (linkType == TransportLinkType::SIO) ? tagRes->linkP2pSio : tagRes->linkP2p;
1227 :
1228 0 : if (linkRes.find(rankId) == linkRes.end() ||
1229 0 : linkRes[rankId].find(newTag) == linkRes[rankId].end()){
1230 : // 优先校验notify去判定link是否有效
1231 0 : if (linkP2p.localIpcSignal[0].resId == INVALID_U64) {
1232 0 : HCCL_INFO("[%s]the link is invalid, no need to create transport, rankId[%u], newTag[%s]",
1233 : __func__, rankId, newTag.c_str());
1234 0 : return HCCL_SUCCESS;
1235 : }
1236 : // 创建Transport对象
1237 0 : MachinePara machinePara;
1238 0 : CHK_RET(SetTransportMachinePara(machinePara, rankId, newTag, linkType));
1239 0 : machinePara.notifyNum = notifyNum;
1240 : // 获取localMem & remoteMem
1241 0 : TransportDeviceP2pData transDevP2pData;
1242 0 : transDevP2pData.inputBufferPtr = reinterpret_cast<void *>((linkP2p.remoteMem)[INPUT].addr);
1243 0 : transDevP2pData.outputBufferPtr = reinterpret_cast<void *>((linkP2p.remoteMem)[OUTPUT].addr);
1244 0 : if (transDevP2pData.inputBufferPtr == nullptr || transDevP2pData.outputBufferPtr == nullptr) {
1245 0 : HCCL_ERROR("[%s]input ptr[%p] or output ptr[%p] is null.", __func__,
1246 : transDevP2pData.inputBufferPtr, transDevP2pData.outputBufferPtr);
1247 0 : return HCCL_E_PARA;
1248 : }
1249 : // 获取Notify资源
1250 0 : CHK_RET(SetTagRemoteRes(rankId, newTag, tagRes));
1251 0 : HccltagRemoteResV3 *tagRemoteRes = &(rankTagRemoteRes_[rankId][newTag]);
1252 0 : CHK_RET(SetTransportPtpNotify(transDevP2pData, tagRemoteRes->p2pNotifyNum, linkP2p, notifyNum));
1253 : // 获取transportAttr信息
1254 0 : transDevP2pData.transportAttr = linkP2p.transportAttr;
1255 : // 创建Transport对象
1256 0 : std::shared_ptr<Transport> link;
1257 0 : TransportPara para{};
1258 0 : const std::unique_ptr<NotifyPool> notifyPool;
1259 0 : link.reset(new (std::nothrow) Transport(
1260 0 : TransportType::TRANS_TYPE_DEVICE_P2P, para, dispatcher_, notifyPool, machinePara, transDevP2pData));
1261 0 : CHK_SMART_PTR_NULL(link);
1262 0 : CHK_RET(link->Init());
1263 0 : linkRes[rankId][newTag] = link;
1264 0 : HCCL_INFO("[%s]linkRes_, rankId[%u], newTag[%s]", __func__, rankId, newTag.c_str());
1265 0 : }
1266 0 : return HCCL_SUCCESS;
1267 : }
1268 0 : HcclResult HcclCommAicpu::InitLinkRoce(HccltagRemoteResV2 *tagRes, HcclLinkRoceV2 *linkRoce, u32 &rankId,
1269 : const std::string &newTag, u32 notifyNum, const bool isBackup, const bool isSecond)
1270 : {
1271 : // 优先校验notify去判定link是否有效
1272 0 : if (linkRoce->localNotifyList == 0) {
1273 0 : HCCL_INFO("[%s]the link is invalid, no need to create transport, rankId[%u], newTag[%s], isBackup[%d]",
1274 : __func__,
1275 : rankId,
1276 : newTag.c_str(),
1277 : isBackup);
1278 0 : return HCCL_SUCCESS;
1279 : }
1280 0 : HcclSignalInfo *localNotifyList = reinterpret_cast<HcclSignalInfo *>(linkRoce->localNotifyList);
1281 0 : if (localNotifyList[0].resId == INVALID_U64) {
1282 0 : HCCL_INFO("[%s]the link notify resource is invalid, no need to create transport, rankId[%u], newTag[%s], resId[%llu], "
1283 : "isBackup[%d]",
1284 : __func__,
1285 : rankId,
1286 : newTag.c_str(),
1287 : localNotifyList[0].resId,
1288 : isBackup);
1289 0 : return HCCL_SUCCESS;
1290 : }
1291 :
1292 : // 创建Transport对象
1293 0 : MachinePara machinePara;
1294 0 : CHK_RET(SetTransportMachinePara(machinePara, rankId, newTag));
1295 0 : machinePara.notifyNum = notifyNum;
1296 : // 获取localMem & remoteMem
1297 0 : TransportDeviceIbverbsData transDevIbverbsData;
1298 0 : transDevIbverbsData.inputBufferPtr = reinterpret_cast<void *>((linkRoce->remoteMem)[INPUT].addr);
1299 0 : transDevIbverbsData.outputBufferPtr = reinterpret_cast<void *>((linkRoce->remoteMem)[OUTPUT].addr);
1300 0 : if (transDevIbverbsData.inputBufferPtr == nullptr || transDevIbverbsData.outputBufferPtr == nullptr) {
1301 0 : HCCL_ERROR("[%s]input ptr[%p] or output ptr[%p] is null.", __func__,
1302 : transDevIbverbsData.inputBufferPtr, transDevIbverbsData.outputBufferPtr);
1303 0 : return HCCL_E_PARA;
1304 : }
1305 0 : transDevIbverbsData.localInputMem = (linkRoce->localMem)[INPUT];
1306 0 : transDevIbverbsData.localOutputMem = (linkRoce->localMem)[OUTPUT];
1307 0 : transDevIbverbsData.localNotifyValueAddr = linkRoce->notifyValue;
1308 0 : transDevIbverbsData.notifyValueKey = linkRoce->notifyValueKey;
1309 0 : transDevIbverbsData.remoteInputKey = (linkRoce->remoteMem)[INPUT].key;
1310 0 : transDevIbverbsData.remoteOutputKey = (linkRoce->remoteMem)[OUTPUT].key;
1311 : // 获取QPinfo
1312 0 : CHK_RET(SetTagRemoteRes(rankId, newTag, tagRes));
1313 0 : HccltagRemoteResV3 *tagRemoteRes = &(rankTagRemoteRes_[rankId][newTag]);
1314 0 : u64 &roceQpNum = isBackup ? tagRemoteRes->qpNumBackup : tagRemoteRes->qpNum;
1315 0 : CHK_RET(SetTransportRoceQP(transDevIbverbsData, roceQpNum, linkRoce));
1316 : // 获取notify
1317 0 : u64 &roceNotifyNum = isBackup ? tagRemoteRes->roceNotifyNumBackup : tagRemoteRes->roceNotifyNum;
1318 0 : CHK_RET(SetTransportRoceNotify(transDevIbverbsData, roceNotifyNum, linkRoce, notifyNum));
1319 0 : HCCL_INFO("[%s]transDevIbverbsData isBackup[%d]", __func__, isBackup);
1320 : // 获取atomic write
1321 0 : transDevIbverbsData.useAtomicWrite = linkRoce->useAtomicWrite;
1322 : // 创建Transport对象
1323 0 : std::shared_ptr<Transport> link;
1324 0 : TransportPara para{};
1325 0 : para.timeout = linkTimeOut_;
1326 0 : const std::unique_ptr<NotifyPool> notifyPool;
1327 0 : link.reset(new (std::nothrow) Transport(
1328 : TransportType::TRANS_TYPE_DEVICE_IBVERBS, para, dispatcher_, notifyPool,
1329 0 : machinePara, TransportDeviceP2pData(), transDevIbverbsData));
1330 0 : CHK_SMART_PTR_NULL(link);
1331 0 : CHK_RET(link->Init());
1332 0 : if (isBackup) {
1333 0 : linkRdmaResBackUp_[rankId][newTag].push_back(link);
1334 0 : HCCL_INFO("[%s]linkRdmaResBackUp_, rankId[%u], newTag[%s], isBackup[%d], isSecond[%d], qpNum[%u], notifyNum[%u]",
1335 : __func__, rankId, newTag.c_str(), isBackup, isSecond, roceQpNum, roceNotifyNum);
1336 : } else {
1337 0 : linkRdmaRes_[rankId][newTag].push_back(link);
1338 0 : HCCL_INFO("[%s]linkRdmaRes_, rankId[%u], newTag[%s], isBackup[%d], isSecond[%d], qpNum[%u], notifyNum[%u]",
1339 : __func__, rankId, newTag.c_str(), isBackup, isSecond, roceQpNum, roceNotifyNum);
1340 : }
1341 0 : return HCCL_SUCCESS;
1342 0 : }
1343 :
1344 :
1345 0 : HcclResult HcclCommAicpu::InitLinkRoce(HccltagRemoteResV2 *tagRes, u32 &rankId, const std::string &newTag,
1346 : u32 notifyNum, const bool isBackup)
1347 : {
1348 0 : auto tempLinkRes = isBackup ? linkRdmaResBackUp_ : linkRdmaRes_;
1349 0 : if (tempLinkRes.find(rankId) == tempLinkRes.end() ||
1350 0 : tempLinkRes[rankId].find(newTag) == tempLinkRes[rankId].end()) {
1351 0 : bool isBatchSendRecv = newTag.find("BatchSendRecv") != std::string::npos;
1352 0 : if (isBatchSendRecv) {
1353 : //如果是batchsendrecv，相同rank，需要刷新两次transport，如果是主的话就刷新0,2 备就刷新1,3
1354 0 : if (isBackup){
1355 0 : CHK_RET(InitLinkRoce(tagRes, &(tagRes->linkRoce[AICPU_RETRY_LINKROCE_BACKUP]), rankId, newTag,
1356 : notifyNum, isBackup));
1357 0 : CHK_RET(InitLinkRoce(tagRes, &(tagRes->linkRoce[AICPU_RETRY_LINKROCE_BACKUP + 2]), rankId, newTag,
1358 : notifyNum, isBackup, true));
1359 : } else {
1360 0 : CHK_RET(InitLinkRoce(tagRes, &(tagRes->linkRoce[AICPU_RETRY_LINKROCE_DEFAULT]), rankId, newTag,
1361 : notifyNum, isBackup));
1362 0 : CHK_RET(InitLinkRoce(tagRes, &(tagRes->linkRoce[AICPU_RETRY_LINKROCE_DEFAULT + 2]), rankId, newTag,
1363 : notifyNum, isBackup, true));
1364 : }
1365 : } else {
1366 : //非batchsendrecv只刷新主0，备1
1367 0 : HcclLinkRoceV2 *linkRoce = isBackup ? &(tagRes->linkRoce[AICPU_RETRY_LINKROCE_BACKUP])
1368 : : &(tagRes->linkRoce[AICPU_RETRY_LINKROCE_DEFAULT]);
1369 0 : CHK_RET(InitLinkRoce(tagRes, linkRoce, rankId, newTag, notifyNum, isBackup));
1370 : }
1371 : }
1372 0 : return HCCL_SUCCESS;
1373 0 : }
1374 :
1375 0 : HcclResult HcclCommAicpu::InitRemoteTagRes(u32 &rankId, const ListCommon &head,
1376 : const std::string &newTag, u32 notifyNum, TransportLinkType linkType)
1377 : {
1378 0 : HCCL_RUN_INFO("[%s] Entry parse remote resource rankId[%u], group[%s], newTag[%s], head[%p], "
1379 : "linkType[%d]", __func__,
1380 : rankId, identifier_.c_str(), newTag.c_str(), &head, linkType);
1381 0 : ListCommon *curList = reinterpret_cast<ListCommon *>(head.nextDevice);
1382 0 : if (curList == nullptr) {
1383 0 : HCCL_ERROR("[%s]cur list ptr is null.", __func__);
1384 0 : return HCCL_E_PARA;
1385 : }
1386 :
1387 0 : HccltagRemoteResV2 *tagRes = nullptr;
1388 0 : while (curList != &head) {
1389 0 : HccltagRemoteResV2 *tagResTemp = list_entry(curList, HccltagRemoteResV2, nextTagRes);
1390 0 : if (strcmp(tagResTemp->tag, newTag.c_str()) == 0) {
1391 0 : tagRes = tagResTemp;
1392 0 : break;
1393 : }
1394 :
1395 0 : curList = reinterpret_cast<ListCommon *>(curList->nextDevice);
1396 0 : if (curList == nullptr) {
1397 0 : HCCL_ERROR("[%s]next list ptr is null.", __func__);
1398 0 : return HCCL_E_PARA;
1399 : }
1400 : }
1401 :
1402 0 : if (tagRes == nullptr) {
1403 0 : HCCL_ERROR("[%s]newTag[%s] not found, rankId[%u], head[%p], curList[%p], nextList[%llu], notifyNum[%u], "
1404 : "linkType[%d]",
1405 : __func__, newTag.c_str(), rankId, &head, curList, curList->nextDevice, notifyNum, linkType);
1406 0 : return HCCL_E_PARA;
1407 : }
1408 :
1409 0 : HCCL_INFO("[%s]newTag[%s], rankId[%u], head[%p], curList[%p], nextList[%llu], notifyNum[%u], linkType[%d]",
1410 : __func__, newTag.c_str(), rankId, &head, curList, curList->nextDevice, notifyNum, linkType);
1411 0 : if (linkType != TransportLinkType::RDMA) {
1412 : // 创建P2P链路
1413 0 : CHK_RET(InitLinkP2p(tagRes, rankId, newTag, notifyNum, linkType));
1414 : } else {
1415 : // 创建roce链路
1416 0 : CHK_RET(InitLinkRoce(tagRes, rankId, newTag, notifyNum));
1417 : // 创建roce链路（备用链路）
1418 0 : CHK_RET(InitLinkRoce(tagRes, rankId, newTag, notifyNum, true));
1419 : }
1420 0 : HCCL_INFO("[%s] End parse remote resource rankId[%u] tag[%s], newTag[%s]",
1421 : __func__, rankId, identifier_.c_str(), newTag.c_str());
1422 0 : return HCCL_SUCCESS;
1423 : }
1424 :
1425 0 : HcclResult HcclCommAicpu::RefreshTransportsResForRank(const HcclOpResParam *commParam, u32 rankId,
1426 : const std::string &newTag, u32 notifyNum, TransportLinkType linkType)
1427 : {
1428 0 : if (rankId >= AICPU_MAX_RANK_NUM) {
1429 0 : HCCL_ERROR("[%s] rankId[%u] overflow for group[%s], newTag[%s]", __func__,
1430 : rankId, identifier_.c_str(), newTag.c_str());
1431 0 : return HCCL_E_PARA;
1432 : }
1433 0 : if (commParam->remoteRes[rankId].nextDevicePtr == 0) {
1434 0 : return HCCL_SUCCESS;
1435 : }
1436 0 : HcclRankRelationResV2 *rankRelationResPtr =
1437 0 : reinterpret_cast<HcclRankRelationResV2 *>(commParam->remoteRes[rankId].nextDevicePtr);
1438 0 : if (rankRelationResPtr == nullptr) {
1439 0 : HCCL_ERROR("[%s]rank relation resource ptr is null, commParam->remoteRes[rankId].nextDevicePtr[%p],"
1440 : " rankId[%u]", __func__,
1441 : reinterpret_cast<HcclRankRelationResV2 *>(commParam->remoteRes[rankId].nextDevicePtr), rankId);
1442 0 : return HCCL_E_PARA;
1443 : }
1444 :
1445 : // 1. init公共参数（对应remoteWorldRank，remoteUsrRankId暂不处理：windowsIn，windowsOut）
1446 0 : rankData_[rankId].remoteWorldRank = rankRelationResPtr->remoteWorldRank;
1447 0 : rankData_[rankId].remoteUsrRankId = rankRelationResPtr->remoteUsrRankId;
1448 : // 2. 遍历链表，获取HccltagRemoteResV2创建Tranport对象
1449 0 : if (reinterpret_cast<ListCommon *>(rankRelationResPtr->nextTagRes.nextDevice) !=
1450 0 : &(rankRelationResPtr->nextTagRes)) {
1451 0 : HCCL_DEBUG("[%s] Start to parse rankId[%u] tag resources, head[%p], nextDevice[%p], pre Device[%p], group[%s]",
1452 : __func__, rankId, &rankRelationResPtr->nextTagRes, rankRelationResPtr->nextTagRes.nextDevice,
1453 : rankRelationResPtr->nextTagRes.preDevice, identifier_.c_str());
1454 0 : CHK_RET(InitRemoteTagRes(rankId, rankRelationResPtr->nextTagRes, newTag, notifyNum, linkType));
1455 : } else {
1456 0 : HCCL_ERROR("[%s]could not find member in rankRelationRes list, rankId[%u], head[%p], nextDevice[%p]", __func__,
1457 : rankId, &rankRelationResPtr->nextTagRes, rankRelationResPtr->nextTagRes.nextDevice);
1458 0 : return HCCL_E_PARA;
1459 : }
1460 0 : HCCL_INFO("[%s] process success rankId[%u], group[%s], newTag[%s]",
1461 : __func__, rankId, identifier_.c_str(), newTag.c_str());
1462 0 : return HCCL_SUCCESS;
1463 : }
1464 :
1465 0 : HcclResult HcclCommAicpu::GetRdmaLinksByRankAndTag(const HcclOpResParam *commParam, CommTransportsType type, u32 rankId,
1466 : const std::string &newTag, LINK &link, bool isBackup, u32 notifyNum, bool isSecond)
1467 : {
1468 0 : HCCL_INFO("[%s] Start to rdma get Link group[%s], rankId[%u], newTag[%s], isBackup[%d], deviceLogicId[%d],"
1469 : "notifyNum[%u], isSecond[%d]",
1470 : __func__, identifier_.c_str(), rankId, newTag.c_str(), isBackup, commParam->topoInfo.deviceLogicId,
1471 : notifyNum, isSecond);
1472 :
1473 0 : auto *linkRes = isBackup ? &linkRdmaResBackUp_ : &linkRdmaRes_;
1474 0 : auto iterRankLinks = linkRes->find(rankId);
1475 0 : if (iterRankLinks == linkRes->end() || iterRankLinks->second.find(newTag) == iterRankLinks->second.end()) {
1476 0 : HCCL_INFO("[%s] could not find link resource, rankId[%u], group[%s], newTag[%s]", __func__, rankId,
1477 : identifier_.c_str(), newTag.c_str());
1478 0 : CHK_RET(RefreshTransportsResForRank(commParam, rankId, newTag, notifyNum, TransportLinkType::RDMA));
1479 0 : iterRankLinks = linkRes->find(rankId);
1480 0 : if (iterRankLinks == linkRes->end() || iterRankLinks->second.find(newTag) == iterRankLinks->second.end()) {
1481 0 : HCCL_ERROR("[%s] refresh transport failed, newTag[%s], remoteUserRankId[%u]", __func__,
1482 : newTag.c_str(), rankId);
1483 0 : return HCCL_E_INTERNAL;
1484 : }
1485 : }
1486 0 : if (isSecond && iterRankLinks->second[newTag].size() <= 1) {
1487 0 : HCCL_ERROR("[%s] get rdma Link failed, newTag[%s], remoteUserRankId[%u]", __func__,
1488 : newTag.c_str(), rankId);
1489 0 : return HCCL_E_INTERNAL;
1490 : }
1491 :
1492 0 : link = isSecond ? iterRankLinks->second[newTag][1] : iterRankLinks->second[newTag][0];
1493 :
1494 0 : if (receivedAcks_.find(rankId) == receivedAcks_.end()) {
1495 0 : HCCL_ERROR("[%s]there is no link with rankId[%u]", __func__, rankId);
1496 0 : return HCCL_E_NOT_FOUND;
1497 : }
1498 0 : link->SetSupportDataReceivedAck(receivedAcks_[rankId]);
1499 0 : HCCL_DEBUG("[HcclCommAicpu][GetLinksByRankAndTag]rankid[%d] supportDataReceivedAck is %d",
1500 : rankId, receivedAcks_[rankId]);
1501 0 : HCCL_INFO("[%s] group[%s], newTag[%s], rankId[%u] type[%u] success!", __func__, identifier_.c_str(),
1502 : newTag.c_str(), rankId, type);
1503 0 : return HCCL_SUCCESS;
1504 : }
1505 :
1506 0 : HcclResult HcclCommAicpu::GetSdmaLinksByRankAndTag(const HcclOpResParam *commParam, CommTransportsType type, u32 rankId,
1507 : const std::string &newTag, LINK &link, bool isBackup, u32 notifyNum, TransportLinkType linkType)
1508 : {
1509 0 : HCCL_INFO("[%s] Start to get sdma Link group[%s], rankId[%u], newTag[%s], isBackup[%d], deviceLogicId[%d], "
1510 : "notifyNum[%u], linkType[%d]",
1511 : __func__, identifier_.c_str(), rankId, newTag.c_str(), isBackup, commParam->topoInfo.deviceLogicId, notifyNum,
1512 : linkType);
1513 :
1514 0 : auto *linkRes = &linkRes_;
1515 0 : if (linkType == TransportLinkType::SIO) { // HCCS SIO并发场景，HCCS链路与SIO链路分开管理
1516 0 : linkRes = &linkResSio_;
1517 : }
1518 0 : auto iterRankLinks = linkRes->find(rankId);
1519 0 : if (iterRankLinks == linkRes->end() || iterRankLinks->second.find(newTag) == iterRankLinks->second.end()) {
1520 0 : HCCL_INFO("[%s] could not find link resource, rankId[%u], group[%s], newTag[%s]", __func__, rankId,
1521 : identifier_.c_str(), newTag.c_str());
1522 0 : CHK_RET(RefreshTransportsResForRank(commParam, rankId, newTag, notifyNum, linkType));
1523 0 : iterRankLinks = linkRes->find(rankId);
1524 0 : if (iterRankLinks == linkRes->end() || iterRankLinks->second.find(newTag) == iterRankLinks->second.end()) {
1525 0 : HCCL_ERROR("[%s] refresh transport failed, newTag[%s], remoteUserRankId[%u], %p", __func__,
1526 : newTag.c_str(), rankId, linkRes);
1527 0 : return HCCL_E_INTERNAL;
1528 : }
1529 : }
1530 :
1531 0 : link = iterRankLinks->second[newTag];
1532 0 : if (receivedAcks_.find(rankId) == receivedAcks_.end()) {
1533 0 : HCCL_ERROR("[%s]there is no link with rankId[%u]", __func__, rankId);
1534 0 : return HCCL_E_NOT_FOUND;
1535 : }
1536 0 : link->SetSupportDataReceivedAck(receivedAcks_[rankId]);
1537 0 : HCCL_DEBUG("[HcclCommAicpu][GetLinksByRankAndTag]rankid[%d] supportDataReceivedAck is %d",
1538 : rankId, receivedAcks_[rankId]);
1539 0 : HCCL_INFO("[%s] group[%s], newTag[%s], rankId[%u] type[%u] success!", __func__, identifier_.c_str(),
1540 : newTag.c_str(), rankId, type);
1541 0 : return HCCL_SUCCESS;
1542 : }
1543 :
1544 0 : HcclResult HcclCommAicpu::CleanRoceResource(const std::string &newTag, AlgResourceResponse &algResResponse,
1545 : const std::map<u32, bool> &remoteRankPortMap, const OpParam ¶m)
1546 : {
1547 0 : HCCL_INFO("[%s] Entry alloc transport group[%s], tag[%s]", __func__, identifier_.c_str(), newTag.c_str());
1548 :
1549 0 : for (auto &levelNSubCommTransport : algResResponse.opTransportResponse) {
1550 0 : for (auto &singleSubCommTransport : levelNSubCommTransport) {
1551 0 : for (auto &transportRequest : singleSubCommTransport.transportRequests) {
1552 0 : if (transportRequest.isValid && transportRequest.isUsedRdma) {
1553 0 : u32 remoteUserRank = transportRequest.remoteUserRank;
1554 0 : linkRdmaRes_[remoteUserRank].erase(newTag);
1555 0 : linkRdmaResBackUp_[remoteUserRank].erase(newTag);
1556 : }
1557 : }
1558 : }
1559 : }
1560 :
1561 0 : return HCCL_SUCCESS;
1562 : }
1563 :
1564 1 : HcclResult HcclCommAicpu::CleanAllRoceResource(){
1565 1 : HCCL_INFO("Clean all link rdna resources");
1566 : // 清空主链路资源
1567 1 : linkRdmaRes_.clear();
1568 : // 清空备链路资源
1569 1 : linkRdmaResBackUp_.clear();
1570 1 : return HCCL_SUCCESS;
1571 : }
1572 :
1573 2 : HcclResult HcclCommAicpu::ClearOpResource(const std::string &tag)
1574 : {
1575 : // 与 GetAlgResponseRes/AllocAlgResource/ReAllocTransportResource 互斥，避免读写竞争 resMap_/linkRes_
1576 2 : std::lock_guard<std::mutex> lock(preemptMutexForResMap_);
1577 :
1578 6 : auto eraseTagFromAll = [this](const std::string &t) -> bool {
1579 6 : bool hit = false;
1580 : // 主资源map：vector<LINK>析构会让shared_ptr<Transport>引用-1，归零时触发driver资源释放
1581 6 : if (resMap_.erase(t) > 0) {
1582 1 : hit = true;
1583 : }
1584 : // SDMA HCCS链路缓存：报错日志中driver halResAddrMap ioctl失败的根本累积点
1585 9 : for (auto &rankIt : linkRes_) {
1586 3 : if (rankIt.second.erase(t) > 0) {
1587 1 : hit = true;
1588 : }
1589 : }
1590 : // SDMA SIO并发链路缓存
1591 9 : for (auto &rankIt : linkResSio_) {
1592 3 : if (rankIt.second.erase(t) > 0) {
1593 1 : hit = true;
1594 : }
1595 : }
1596 : // RDMA主备链路缓存（CleanAllRoceResource按通信域全清，本接口按tag细粒度清）
1597 9 : for (auto &rankIt : linkRdmaRes_) {
1598 3 : if (rankIt.second.erase(t) > 0) {
1599 1 : hit = true;
1600 : }
1601 : }
1602 9 : for (auto &rankIt : linkRdmaResBackUp_) {
1603 3 : if (rankIt.second.erase(t) > 0) {
1604 1 : hit = true;
1605 : }
1606 : }
1607 : // 本地scratchMem和localTagResToObj按tag索引
1608 6 : if (tagScratchMem_.erase(t) > 0) {
1609 1 : hit = true;
1610 : }
1611 6 : if (localTagResToObj_.erase(t) > 0) {
1612 1 : hit = true;
1613 : }
1614 9 : for (auto &rankIt : rankTagRemoteRes_) {
1615 3 : if (rankIt.second.erase(t) > 0) {
1616 1 : hit = true;
1617 : }
1618 : }
1619 6 : return hit;
1620 2 : };
1621 :
1622 : // 与 host 端 ClearOpResource 对齐：尝试 base/_host/_device 三种后缀
1623 2 : bool found = false;
1624 2 : found = eraseTagFromAll(tag) || found;
1625 2 : found = eraseTagFromAll(tag + "_host") || found;
1626 2 : found = eraseTagFromAll(tag + "_device") || found;
1627 2 : if (!found) {
1628 1 : HCCL_WARNING("[HcclCommAicpu][ClearOpResource] no entries matched tag[%s] "
1629 : "(also tried _host/_device variants), group[%s]",
1630 : tag.c_str(), identifier_.c_str());
1631 : } else {
1632 1 : HCCL_INFO("[HcclCommAicpu][ClearOpResource] cleared resources for tag[%s], group[%s]",
1633 : tag.c_str(), identifier_.c_str());
1634 : }
1635 2 : return HCCL_SUCCESS;
1636 2 : }
1637 :
1638 : // 借轨重新刷新资源
1639 0 : HcclResult HcclCommAicpu::ReAllocTransportResource(const std::string &newTag, AlgResourceResponse &algResResponse,
1640 : std::map<u32, bool> &remoteRankPortMap, const HcclOpResParam *commParam, const OpParam ¶m)
1641 : {
1642 0 : HCCL_INFO("[%s] Entry alloc transport group[%s], tag[%s]", __func__, identifier_.c_str(), newTag.c_str());
1643 0 : std::set<u32> bsrTansportRank;
1644 0 : for (auto &levelNSubCommTransport : algResResponse.opTransportResponse) {
1645 0 : for (auto &singleSubCommTransport : levelNSubCommTransport) {
1646 0 : singleSubCommTransport.links.clear();
1647 0 : singleSubCommTransport.links.reserve(singleSubCommTransport.transportRequests.size());
1648 0 : for (auto &transportRequest : singleSubCommTransport.transportRequests) {
1649 0 : singleSubCommTransport.links.push_back(nullptr);
1650 0 : if (transportRequest.isValid) {
1651 0 : HCCL_INFO("[%s] alloc transport, newTag[%s], rankId[%u], "
1652 : "input memory type[%u], output memory type[%u], ", __func__, newTag.c_str(),
1653 : transportRequest.remoteUserRank, transportRequest.inputMemType, transportRequest.outputMemType);
1654 0 : receivedAcks_[transportRequest.remoteUserRank] = singleSubCommTransport.supportDataReceivedAck;
1655 0 : bool isBackup = remoteRankPortMap.find(transportRequest.remoteUserRank) != remoteRankPortMap.end() &&
1656 0 : !remoteRankPortMap[transportRequest.remoteUserRank];
1657 0 : bool isSecondBuild = false;
1658 0 : bool isBatchSendRecv = newTag.find("BatchSendRecv") != std::string::npos;
1659 0 : if (transportRequest.isUsedRdma && isBatchSendRecv &&
1660 0 : bsrTansportRank.find(transportRequest.remoteUserRank) != bsrTansportRank.end()){
1661 : //仅在batchsendrecv rdma下发的时候需要第二次刷新，实际第一次下发都刷好了，第二次就是get一下
1662 0 : isSecondBuild = true;
1663 : }
1664 : // A3 bsr远端是DirectNpu 链路的话则跳过
1665 0 : if ((param.opType == HcclCMDType::HCCL_CMD_BATCH_SEND_RECV) &&
1666 0 : (param.BatchSendRecvDataDes.isDirectRemoteRank[transportRequest.remoteUserRank])) {
1667 0 : continue;
1668 : }
1669 0 : bsrTansportRank.insert(transportRequest.remoteUserRank);
1670 0 : CHK_RET(CreateLink(newTag, transportRequest, commParam, singleSubCommTransport.links.back(),
1671 : transportRequest.notifyNum, isBackup, isSecondBuild));
1672 : }
1673 : }
1674 : }
1675 : }
1676 :
1677 0 : return HCCL_SUCCESS;
1678 0 : }
1679 :
1680 0 : HcclResult HcclCommAicpu::AllocTransportResource(const std::string &newTag, const OpParam &opParam,
1681 : const HcclOpResParam *commParam, AlgResourceRequest &resRequest, AlgResourceResponse &algResResponse)
1682 : {
1683 0 : HCCL_INFO("[%s] Entry alloc transport group[%s]", __func__, identifier_.c_str());
1684 0 : algResResponse.opTransportResponse = resRequest.opTransport;
1685 :
1686 0 : std::set<u32> bsrTansportRank;
1687 0 : for (auto &levelNSubCommTransport : algResResponse.opTransportResponse) {
1688 0 : for (auto &singleSubCommTransport : levelNSubCommTransport) {
1689 0 : singleSubCommTransport.links.clear();
1690 0 : singleSubCommTransport.links.reserve(singleSubCommTransport.transportRequests.size());
1691 0 : for (auto &transportRequest : singleSubCommTransport.transportRequests) {
1692 0 : singleSubCommTransport.links.push_back(nullptr);
1693 0 : if (transportRequest.isValid) {
1694 0 : localUserRank_ = transportRequest.localUserRank;
1695 0 : receivedAcks_[transportRequest.remoteUserRank] = singleSubCommTransport.supportDataReceivedAck;
1696 0 : HCCL_DEBUG("[%s] alloc transport, newTag[%s], rankId[%u], input memory type[%u], "
1697 : "output memory type[%u], ", __func__, newTag.c_str(), transportRequest.remoteUserRank,
1698 : transportRequest.inputMemType, transportRequest.outputMemType);
1699 :
1700 0 : bool isSecondBuild = false;
1701 0 : if (transportRequest.isUsedRdma &&
1702 0 : opParam.opType == HcclCMDType::HCCL_CMD_BATCH_SEND_RECV &&
1703 0 : bsrTansportRank.find(transportRequest.remoteUserRank) != bsrTansportRank.end()){
1704 : //仅仅在batchsendrecv rdma下发的时候需要第二次刷新，实际第一次下发都刷好了，第二次就是get一下
1705 0 : isSecondBuild = true;
1706 : }
1707 : // A3 bsr远端是DirectNpu 链路的话则跳过
1708 0 : if ((opParam.opType == HcclCMDType::HCCL_CMD_BATCH_SEND_RECV) &&
1709 0 : (opParam.BatchSendRecvDataDes.isDirectRemoteRank[transportRequest.remoteUserRank])) {
1710 0 : continue;
1711 : }
1712 0 : bsrTansportRank.insert(transportRequest.remoteUserRank);
1713 0 : CHK_RET(CreateLink(newTag, transportRequest, commParam, singleSubCommTransport.links.back(),
1714 : transportRequest.notifyNum, false, isSecondBuild));
1715 : }
1716 : }
1717 : }
1718 : }
1719 :
1720 0 : return HCCL_SUCCESS;
1721 0 : }
1722 :
1723 : // 在resMap_[tag]对应原有通信资源的基础上继续增量建链，目前batchsendrecv会用到本接口
1724 0 : HcclResult HcclCommAicpu::IncreAllocTransportResource(const std::string &newTag, const OpParam &opParam,
1725 : const HcclOpResParam *commParam, AlgResourceRequest &resRequest, AlgResourceResponse &algResResponse)
1726 : {
1727 0 : HCCL_INFO("[HcclCommAicpu][IncreAllocTransportResource] Entry alloc transport group[%s]", identifier_.c_str());
1728 0 : std::set<u32> bsrTansportRank;
1729 0 : for (u32 levelIndex = 0; levelIndex < resRequest.opTransport.size(); levelIndex++) {
1730 0 : for (u32 ringIndex = 0; ringIndex < resRequest.opTransport[levelIndex].size(); ringIndex++) {
1731 0 : SingleSubCommTransport &reqSingleSubComm = resRequest.opTransport[levelIndex][ringIndex];
1732 0 : SingleSubCommTransport &respSingleSubComm = algResResponse.opTransportResponse[levelIndex][ringIndex];
1733 0 : for (u32 rankIndex = 0; rankIndex < reqSingleSubComm.transportRequests.size(); rankIndex++){
1734 0 : TransportRequest &transportRequest = reqSingleSubComm.transportRequests[rankIndex];
1735 0 : CHK_PRT_RET(rankIndex >= respSingleSubComm.links.size(),
1736 : HCCL_ERROR("[HcclCommAicpu][IncreAllocTransportResource] The remote rank_id[%u] is larger than "\
1737 : "the existent respSingleSubComm map size[%u]", rankIndex, respSingleSubComm.links.size()),
1738 : HCCL_E_PARA);
1739 0 : if (respSingleSubComm.links[rankIndex] != nullptr &&
1740 0 : respSingleSubComm.links[rankIndex]->GetLinkType() != hccl::LinkType::LINK_RESERVED) {
1741 0 : HCCL_INFO("[IncreAlloc] The link to remote userRank[%u] has existed",
1742 : transportRequest.remoteUserRank);
1743 0 : continue;
1744 : }
1745 0 : if (transportRequest.isValid) {
1746 0 : receivedAcks_[transportRequest.remoteUserRank] = reqSingleSubComm.supportDataReceivedAck;
1747 0 : respSingleSubComm.transportRequests[rankIndex] = transportRequest;
1748 0 : HCCL_DEBUG("[HcclCommAicpu][IncreAllocTransportResource] alloc transport, newTag[%s], rankId[%u], "
1749 : "input memory type[%u], output memory type[%u], ", newTag.c_str(),
1750 : transportRequest.remoteUserRank, transportRequest.inputMemType, transportRequest.outputMemType);
1751 0 : bool isSecondBuild = false;
1752 0 : if (transportRequest.isUsedRdma &&
1753 0 : opParam.opType == HcclCMDType::HCCL_CMD_BATCH_SEND_RECV &&
1754 0 : bsrTansportRank.find(transportRequest.remoteUserRank) != bsrTansportRank.end()){
1755 : //仅仅在batchsendrecv rdma下发的时候需要第二次刷新，实际第一次下发都刷好了，第二次就是get一下
1756 0 : isSecondBuild = true;
1757 : }
1758 : // A3 bsr远端是DirectNpu 链路的话则跳过
1759 0 : if ((opParam.opType == HcclCMDType::HCCL_CMD_BATCH_SEND_RECV) &&
1760 0 : (opParam.BatchSendRecvDataDes.isDirectRemoteRank[transportRequest.remoteUserRank])) {
1761 0 : continue;
1762 : }
1763 0 : bsrTansportRank.insert(transportRequest.remoteUserRank);
1764 0 : CHK_RET(CreateLink(newTag, transportRequest, commParam, respSingleSubComm.links[rankIndex],
1765 : transportRequest.notifyNum, false, isSecondBuild));
1766 : }
1767 : }
1768 : }
1769 : }
1770 0 : return HCCL_SUCCESS;
1771 0 : }
1772 :
1773 0 : HcclResult HcclCommAicpu::CreateLink(const std::string &newTag, TransportRequest& transportRequest,
1774 : const HcclOpResParam *commParam, LINK& link, u32 notifyNum, bool isBackup, bool isSecond) // 主备的选择
1775 : {
1776 0 : if (transportRequest.isUsedRdma){
1777 0 : CHK_RET(GetRdmaLinksByRankAndTag(commParam, CommTransportsType::SPECIAL, transportRequest.remoteUserRank,
1778 : newTag, link, isBackup, notifyNum, isSecond));
1779 : } else {
1780 0 : CHK_RET(GetSdmaLinksByRankAndTag(commParam, CommTransportsType::SPECIAL, transportRequest.remoteUserRank,
1781 : newTag, link, isBackup, notifyNum, transportRequest.linkType));
1782 : }
1783 :
1784 0 : HCCL_DEBUG("[%s] alloc special transport success!, tag[%s]", __func__, newTag.c_str());
1785 0 : return HCCL_SUCCESS;
1786 : }
1787 :
1788 0 : HcclResult HcclCommAicpu::AllocLocalNotifysResource(const std::string &newTag, const HcclOpResParam *commParam,
1789 : const u32 notifyNum, std::vector<std::shared_ptr<LocalNotify>> ¬ifiesMain,
1790 : std::vector<std::shared_ptr<LocalNotify>> ¬ifiesAux)
1791 : {
1792 0 : HCCL_INFO(
1793 : "[HcclCommAicpu][AllocLocalNotifysResource]requesting for [%u] notifys, tag[%s].", notifyNum, newTag.c_str());
1794 0 : if (localNotifies_.capacity() < notifyNum) {
1795 0 : HCCL_ERROR(
1796 : "[HcclCommAicpu][AllocLocalNotifysResource]request number exceed max notify numbers, alloc failed. Max "
1797 : "number is [%u],request num[%u], tag[%s]",
1798 : localNotifies_.capacity(),
1799 : notifyNum,
1800 : newTag.c_str());
1801 0 : return HCCL_E_PARA;
1802 : }
1803 :
1804 0 : if (localNotifies_.size() < notifyNum) {
1805 0 : if (InitLocalNotifyObj(commParam) != HCCL_SUCCESS || localNotifies_.size() < notifyNum) {
1806 0 : HCCL_ERROR(
1807 : "[HcclCommAicpu][AllocLocalNotifysResource] the need of notify is more than the available, group[%s], "
1808 : "need[%u], total[%u]",
1809 : newTag.c_str(),
1810 : notifyNum,
1811 : localNotifies_.size());
1812 0 : return HCCL_E_INTERNAL;
1813 : }
1814 : }
1815 :
1816 0 : u32 halfNotifyNum = notifyNum >> 1;
1817 0 : notifiesMain.resize(halfNotifyNum);
1818 0 : notifiesAux.resize(halfNotifyNum);
1819 0 : for (u32 i = 0; i < halfNotifyNum; i++) {
1820 0 : notifiesMain[i] = localNotifies_[i << 1];
1821 0 : notifiesAux[i] = localNotifies_[(i << 1) + 1];
1822 : }
1823 0 : HCCL_INFO("[HcclCommAicpu][AllocLocalNotifysResource]find enough notifys, numbers[%u], tag[%s].",
1824 : notifyNum,
1825 : newTag.c_str());
1826 0 : return HCCL_SUCCESS;
1827 : }
1828 :
1829 0 : HcclResult HcclCommAicpu::AllocStreamsResource(
1830 : const std::string &newTag, const HcclOpResParam *commParam, const u32 streamNum, std::vector<Stream> &streams)
1831 : {
1832 0 : HCCL_INFO(
1833 : "[HcclCommAicpu][AllocStreamsResource]requesting for [%u] slave streams, newTag[%s], group[%s].", streamNum, newTag.c_str(), identifier_.c_str());
1834 0 : if (streamNum == 0) {
1835 0 : return HCCL_SUCCESS;
1836 : }
1837 0 : if (slaveStreams_.capacity() < streamNum) {
1838 0 : HCCL_ERROR("[HcclCommAicpu][AllocStreamsResource]request number exceed max substream num, alloc failed. Max "
1839 : "number is [%u],request num[%u], tag[%s]",
1840 : slaveStreams_.capacity(),
1841 : streamNum,
1842 : newTag.c_str());
1843 0 : return HCCL_E_PARA;
1844 : }
1845 0 : if (slaveStreams_.size() < streamNum) {
1846 0 : if (InitSlaveStreamObjs(commParam) != HCCL_SUCCESS || slaveStreams_.size() < streamNum) {
1847 0 : HCCL_ERROR("[HcclCommAicpu][AllocStreamsResource] the need of streams is more than the "
1848 : "available, tag[%s], need[%u], total[%u]",
1849 : newTag.c_str(),
1850 : streamNum,
1851 : slaveStreams_.size());
1852 0 : return HCCL_E_INTERNAL;
1853 : }
1854 : }
1855 0 : streams = std::vector<Stream>(slaveStreams_.begin(), slaveStreams_.begin() + streamNum);
1856 0 : HCCL_INFO(
1857 : "[HcclCommAicpu][AllocStreamsResource]find enough slave streams [%u], tag[%s].", streamNum, newTag.c_str());
1858 0 : return HCCL_SUCCESS;
1859 : }
1860 :
1861 0 : HcclResult HcclCommAicpu::AllocScratchMemResource(const std::string &newTag, const HcclOpResParam *commParam,
1862 : const u64 &scratchMemSize, DeviceMem &scratchMem, bool reAllocFlag)
1863 : {
1864 0 : HCCL_INFO("[HcclCommAicpu][AllocScratchMemResource]requesting for [%llu] bytes scratch mem, tag[%s].",
1865 : scratchMemSize,
1866 : newTag.c_str());
1867 0 : if (scratchMemSize != 0) {
1868 0 : if (tagScratchMem_.find(newTag) == tagScratchMem_.end()) {
1869 0 : HcclResult ret = InitLocalTagRes(commParam->localRes.nextTagRes);
1870 0 : CHK_PRT_RET(ret != HCCL_SUCCESS,
1871 : HCCL_ERROR(
1872 : "[HcclCommAicpu][AllocScratchMemResource]InitLocalTagRes error group[%s]", identifier_.c_str()),
1873 : ret);
1874 : }
1875 :
1876 0 : if (tagScratchMem_.find(newTag) == tagScratchMem_.end()) {
1877 0 : HCCL_ERROR("[HcclCommAicpu][AllocScratchMemResource]alloc scratch memory failed."
1878 : "requesting for [%llu] bytes, tag[%s].",
1879 : scratchMemSize, newTag.c_str());
1880 0 : return HCCL_E_NOT_FOUND;
1881 : }
1882 :
1883 0 : if (reAllocFlag) {
1884 0 : HCCL_INFO("[HcclCommAicpu][AllocScratchMemResource]need to reAlloc mem [%llu] bytes scratch mem, tag[%s].",
1885 : scratchMemSize, newTag.c_str());
1886 0 : HcclResult ret = InitLocalTagRes(commParam->localRes.nextTagRes, reAllocFlag);
1887 0 : CHK_PRT_RET(ret != HCCL_SUCCESS,
1888 : HCCL_ERROR(
1889 : "[HcclCommAicpu][AllocScratchMemResource]InitLocalTagRes error group[%s]", identifier_.c_str()),
1890 : ret);
1891 : }
1892 :
1893 : // 因为aicpu_communicator中会对scratchMem做对齐，所以tagScratchMem_中的大小会偏小（被对齐截断一部分）
1894 : // 但是两者的差值应该不能超过2个CCE_REDUCE_ALIGN_SIZE，否则应该是不对的
1895 0 : if (scratchMemSize - tagScratchMem_[newTag]->size() > (CCE_REDUCE_ALIGN_SIZE + CCE_REDUCE_ALIGN_SIZE)) {
1896 0 : HCCL_ERROR(
1897 : "[HcclCommAicpu][AllocScratchMemResource]alloc tag[%s] scratch memory failed."
1898 : "requesting [%llu] bytes actual [%llu] bytes", newTag.c_str(),
1899 : scratchMemSize, tagScratchMem_[newTag]->size());
1900 0 : return HCCL_E_PARA;
1901 : }
1902 0 : scratchMem = DeviceMem::create(tagScratchMem_[newTag]->ptr(), tagScratchMem_[newTag]->size());
1903 : }
1904 0 : HCCL_INFO("[HcclCommAicpu][AllocScratchMemResource]find enough [%llu] bytes scratch mem, tag[%s].",
1905 : scratchMemSize,
1906 : newTag.c_str());
1907 0 : return HCCL_SUCCESS;
1908 : }
1909 :
1910 0 : HcclResult HcclCommAicpu::AllocAlgResource(const std::string &newTag, const OpParam &opParam,
1911 : const HcclOpResParam *commParam, AlgResourceRequest &resRequest, AlgResourceResponse &algResResponse)
1912 : {
1913 0 : algResResponse.cclInputMem = cclInputBuffer_;
1914 0 : algResResponse.cclOutputMem = cclOutputBuffer_;
1915 0 : algResResponse.paramInputMem = DeviceMem::create(opParam.inputPtr, opParam.inputSize);
1916 0 : algResResponse.paramOutputMem = DeviceMem::create(opParam.outputPtr, opParam.outputSize);
1917 :
1918 0 : PetersonLockGuard guard(hostDeviceLock_.get());
1919 0 : CHK_PRT_RET(guard.IsLockFailed(),
1920 : HCCL_ERROR("[HcclCommAicpu][AllocAlgResource] hostDeviceLock lock failed"), HCCL_E_INTERNAL);
1921 :
1922 0 : CHK_RET(AllocScratchMemResource(newTag, commParam, resRequest.scratchMemSize, algResResponse.scratchMem));
1923 0 : CHK_RET(AllocStreamsResource(newTag, commParam, resRequest.streamNum, algResResponse.slaveStreams));
1924 0 : CHK_RET(AllocLocalNotifysResource(newTag, commParam, resRequest.notifyNum,
1925 : algResResponse.notifiesMain, algResResponse.notifiesAux));
1926 0 : CHK_RET(AllocTransportResource(newTag, opParam, commParam, resRequest, algResResponse));
1927 0 : HCCL_INFO("[HcclCommAicpu][AllocAlgResource] alloc resource success tag[%s].", newTag.c_str());
1928 0 : return HCCL_SUCCESS;
1929 0 : }
1930 :
1931 0 : HcclResult HcclCommAicpu::CalcResRequest(const std::string &algName, const OpParam ¶m,
1932 : std::unique_ptr<CollExecutorBase> &executor, AlgResourceRequest &resourceRequest)
1933 : {
1934 0 : if (executor.get() == nullptr) {
1935 0 : executor = CollAlgExecRegistry::Instance().GetAlgExec(algName, dispatcher_, topoMatcher_);
1936 0 : CHK_PRT_RET(executor.get() == nullptr,
1937 : HCCL_ERROR("[HcclCommAicpu][CalcResRequest]Fail to find executor for algName[%s]", algName.c_str()),
1938 : HCCL_E_PARA);
1939 0 : executor->SetAlgType(algType_);
1940 0 : executor->SetCCLInBuffer(cclbufferSize_);
1941 :
1942 0 : if (param.opType == HcclCMDType::HCCL_CMD_REDUCE_SCATTER) {
1943 0 : bool isSupportSDMAReduce = false;
1944 0 : if (GetWorkflowMode() != HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE) {
1945 0 : isSupportSDMAReduce = IsSupportSDMAReduce(param.inputPtr, param.outputPtr, param.DataDes.dataType,
1946 0 : param.reduceType);
1947 : } else {
1948 0 : isSupportSDMAReduce = IsSupportSDMAReduce(cclInputBuffer_.ptr(), cclOutputBuffer_.ptr(),
1949 0 : param.DataDes.dataType, param.reduceType);
1950 : }
1951 0 : executor->SetIsSupportSDMAReduce(isSupportSDMAReduce);
1952 : }
1953 : }
1954 0 : CHK_RET(CalSendRecvInfoFor910B(algName, param, executor));
1955 0 : return executor->CalcResRequest(param, resourceRequest);
1956 : }
1957 :
1958 0 : u32 HcclCommAicpu::CalculateOpExecIndex(const OpParam &opParam, u32 userRank)
1959 : {
1960 0 : u32 opIndex = 0;
1961 0 : s32 commIndex = 0;
1962 : // 用于重执行和taskException打印的算子计数，bsr/sendrecv/其他算子分别计数
1963 0 : if (opParam.opType == HcclCMDType::HCCL_CMD_BATCH_SEND_RECV) {
1964 0 : commIndex = -1; // batchSendRecv使用 key = -1
1965 0 : } else if (opParam.opType == HcclCMDType::HCCL_CMD_SEND) {
1966 0 : commIndex = opParam.dstRank;
1967 0 : } else if (opParam.opType == HcclCMDType::HCCL_CMD_RECEIVE) {
1968 0 : commIndex = opParam.srcRank;
1969 : } else {
1970 0 : commIndex = userRank;
1971 : }
1972 :
1973 0 : auto it = opExecIndexMap_.find(commIndex);
1974 0 : if (it != opExecIndexMap_.end()) {
1975 0 : opIndex = ++(it->second);
1976 : } else {
1977 0 : opExecIndexMap_.insert({commIndex, 1});
1978 0 : opIndex = 1;
1979 : }
1980 :
1981 0 : HCCL_DEBUG("%s tag:%s opType:%u commIndex:%d opIndex:%u",
1982 : __func__, opParam.tag.c_str(), opParam.opType, commIndex, opIndex);
1983 0 : return opIndex;
1984 : }
1985 :
1986 2 : HcclResult HcclCommAicpu::PrepareSymmetricMemory(const OpParam ¶m, OpCommTransport &opTransportResponse)
1987 : {
1988 2 : CHK_PRT_RET(opTransportResponse.size() == 0,
1989 : HCCL_ERROR("[HcclCommAicpu][PrepareSymmetricMemory] opTransportResponse size is 0"),
1990 : HCCL_E_PARA);
1991 :
1992 2 : const std::unordered_set<LinkType> supportedLinkTypes = {LinkType::LINK_HCCS, LinkType::LINK_SIO, LinkType::LINK_HCCS_SW};
1993 2 : for (u32 levelIdx = 0; levelIdx < opTransportResponse.size(); levelIdx ++) {
1994 2 : for (auto &singleSubCommTransport : opTransportResponse[levelIdx]) {
1995 1 : if (singleSubCommTransport.isZeroCopy == false) {
1996 1 : continue;
1997 : }
1998 0 : for (u64 i = 0; i < singleSubCommTransport.links.size(); ++i) {
1999 0 : LINK &link = singleSubCommTransport.links[i];
2000 0 : if (link == nullptr || !singleSubCommTransport.transportRequests[i].isValid || supportedLinkTypes.count(link->GetLinkType()) == 0) {
2001 0 : continue; // 无效或者不支持的链路
2002 : }
2003 0 : u32 peerRank = link->GetRemoteRank();
2004 0 : void *remoteIn = nullptr;
2005 0 : CHK_RET(HcclSymWinGetPeerPointer(param.inputSymWindow, param.inputOffset, peerRank, &remoteIn));
2006 0 : void *remoteOut = nullptr;
2007 0 : CHK_RET(HcclSymWinGetPeerPointer(param.outputSymWindow, param.outputOffset, peerRank, &remoteOut));
2008 :
2009 0 : CHK_PRT_RET(remoteIn == nullptr || remoteOut == nullptr,
2010 : HCCL_ERROR("[HcclCommAicpu][PrepareSymmetricMemory] remoteRank[%d] in[%p] out[%p] is invalid", peerRank, remoteIn, remoteOut),
2011 : HCCL_E_INTERNAL);
2012 0 : HCCL_INFO("[HcclCommAicpu][PrepareSymmetricMemory] remoteRank[%d] in[%p] out[%p]", peerRank, remoteIn, remoteOut);
2013 0 : CHK_RET(link->UpdateRemoteAddr(remoteIn, remoteOut));
2014 : }
2015 : }
2016 : }
2017 1 : return HCCL_SUCCESS;
2018 1 : }
2019 :
2020 1 : HcclResult HcclCommAicpu::ExecOp(const std::string &newTag, const std::string &algName,
2021 : OpParam &opParam, const HcclOpResParam *commParam)
2022 : {
2023 1 : std::unique_ptr<CollExecutorBase> executor;
2024 : hccl::AlgResourceResponse *algResResponse;
2025 1 : CHK_RET(GetAlgResponseRes(newTag, algName, opParam, commParam, executor, algResResponse));
2026 :
2027 1 : if (isZeroCopy_ || isSymmetricMemory_) {
2028 1 : if (isSymmetricMemory_) {
2029 1 : CHK_RET(PrepareSymmetricMemory(opParam, algResResponse->opTransportResponse));
2030 : } else {
2031 0 : HcclResult ret = PrepareZeroCopyExchanger(newTag, opParam, algResResponse);
2032 0 : CHK_PRT_RET(ret != HCCL_SUCCESS,
2033 : HCCL_ERROR("[HcclCommAicpu][ExecOp] newTag[%s], localRankId[%u]", newTag.c_str(), commParam->localUsrRankId), ret);
2034 : }
2035 :
2036 : // 零拷贝场景scratchMem的大小会与用户的输入大小不同，会导致后续算法展开模块计算出错
2037 : // 但是该场景下不会直接访问scratchMem，因此直接使用输入作为scratchMem，使得后续计算正确
2038 0 : if (opParam.opType == HcclCMDType::HCCL_CMD_REDUCE_SCATTER) {
2039 0 : algResResponse->scratchMem = DeviceMem::create(opParam.inputPtr, opParam.inputSize);
2040 0 : HCCL_INFO("[HcclCommAicpu][ExecOp] ZeroCopy reduce-scatter use userInput as scratchMem, inputPtr[%p] intputSize[%lu]",
2041 : opParam.inputPtr, opParam.inputSize);
2042 : }
2043 :
2044 0 : algResResponse->paramInputMem = DeviceMem::create(opParam.inputPtr, opParam.inputSize);
2045 0 : algResResponse->paramOutputMem = DeviceMem::create(opParam.outputPtr, opParam.outputSize);
2046 0 : HCCL_INFO("[HcclCommAicpu][ExecOp] zero copy modify paramInput paramOutput to algResResp inputPtr[%p] inputSize[%lu] "
2047 : "outputPtr[%p] outputSize[%lu]", algResResponse->paramInputMem.ptr(), algResResponse->paramInputMem.size(),
2048 : algResResponse->paramOutputMem.ptr(), algResResponse->paramOutputMem.size());
2049 : }
2050 :
2051 0 : hcclOpExecIndex_ = CalculateOpExecIndex(opParam, localUserRank_);
2052 0 : HcclResult ret = Orchestrate(newTag, algName, opParam, executor, *algResResponse, commParam);
2053 0 : if (ret != HCCL_SUCCESS) {
2054 0 : HCCL_ERROR("[HcclCommAicpu][ExecOp] executor op fail, tag[%s], algName[%s], identifier[%s]",
2055 : newTag.c_str(), algName.c_str(), identifier_.c_str());
2056 0 : CHK_PRT_CONT(retryEnable_,
2057 : HCCL_ERROR("[HcclCommAicpu][ExecOp] executor op fail, some error logs may be recorded in the "\
2058 : "log/run/device directory, search keyword [ErrToWarn]"));
2059 0 : if (printTaskExceptionForErr_) {
2060 0 : PrintTaskExceptionAllComm();
2061 0 : PrintAicpuCommExecStatus();
2062 0 : printTaskExceptionForErr_ = false;
2063 : }
2064 0 : return ret;
2065 : }
2066 :
2067 0 : HCCL_ENTRY_INFO(commParam->opEntry, "[HcclCommAicpu][ExecOp] executor op success tag[%s], newTag[%s], algName[%s], identifier[%s].",
2068 : opParam.tag.c_str(), newTag.c_str(), algName.c_str(), identifier_.c_str());
2069 0 : return HCCL_SUCCESS;
2070 1 : }
2071 :
2072 0 : HcclResult HcclCommAicpu::RefreshAlgResponseTransportRes(const std::string &newTag, AlgResourceResponse& algResResponse,
2073 : std::map<u32, bool> &remoteRankPortMap, bool isChangeLinkFlag, const HcclOpResParam *commParam,
2074 : const OpParam ¶m)
2075 : {
2076 0 : CHK_PTR_NULL(commParam);
2077 0 : auto iter = resMap_.find(newTag);
2078 0 : CHK_PRT_RET(iter == resMap_.end(),
2079 : HCCL_ERROR("[%s]Fail to find algResResponse for tag[%s]", __func__, newTag.c_str()), HCCL_E_PARA);
2080 :
2081 0 : PetersonLockGuard guard(hostDeviceLock_.get());
2082 0 : CHK_PRT_RET(guard.IsLockFailed(), HCCL_ERROR("[%s] hostDeviceLock lock failed", __func__), HCCL_E_INTERNAL);
2083 0 : if (!isChangeLinkFlag) {
2084 0 : CleanRoceResource(newTag, algResResponse, remoteRankPortMap, param);
2085 0 : CHK_RET(ReAllocTransportResource(newTag, algResResponse, remoteRankPortMap, commParam, param));
2086 0 : HCCL_RUN_INFO("[%s] ChangeLinkFlag[%d], current tag[%s].", __func__, isChangeLinkFlag, newTag.c_str());
2087 : } else {
2088 : // 提前清理所有tag的链路，避免冲突
2089 0 : for (auto &resMapIt: resMap_) {
2090 0 : HCCL_RUN_INFO("[%s] clean roce resource of tag[%s].", __func__, resMapIt.first.c_str());
2091 0 : CleanRoceResource(resMapIt.first, resMapIt.second, remoteRankPortMap, param);
2092 : }
2093 : // 对resMap中所有tag的transport link根据主备进行刷新
2094 0 : for (auto &resMapIt: resMap_) {
2095 0 : HCCL_RUN_INFO("[%s] refresh algResResponse of tag[%s].", __func__, resMapIt.first.c_str());
2096 0 : CHK_RET(ReAllocTransportResource(resMapIt.first, resMapIt.second, remoteRankPortMap, commParam, param));
2097 0 : if (resMapIt.first == newTag) {
2098 0 : HCCL_RUN_INFO("[%s] current tag[%s].", __func__, newTag.c_str());
2099 0 : algResResponse = resMapIt.second;
2100 : }
2101 : }
2102 : }
2103 :
2104 0 : HCCL_RUN_INFO("[%s] alloc resource success tag[%s].", __func__, newTag.c_str());
2105 0 : return HCCL_SUCCESS;
2106 0 : }
2107 :
2108 0 : HcclResult HcclCommAicpu::CalSendRecvInfoForAlltoall(const OpParam ¶m)
2109 : {
2110 0 : if (param.opType == HcclCMDType::HCCL_CMD_ALLTOALL) {
2111 0 : std::vector<u64> sendCountMatrix(topoInfo_.userRankSize * topoInfo_.userRankSize,
2112 0 : param.All2AllDataDes.sendCount);
2113 0 : CHK_RET(GetAlltoAllvcSendRecvInfo(static_cast<void *>(sendCountMatrix.data()),
2114 : param.All2AllDataDes.sendType, param.All2AllDataDes.recvType));
2115 0 : } else if (param.opType == HcclCMDType::HCCL_CMD_ALLTOALLV) {
2116 0 : CHK_PTR_NULL(sendRecvInfoPtr_);
2117 0 : CHK_RET(GetAlltoAllvSendRecvInfo(sendRecvInfoPtr_, param.All2AllDataDes.sendType,
2118 : param.All2AllDataDes.recvType));
2119 : } else {
2120 0 : CHK_RET(GetAlltoAllvcSendRecvInfo(param.All2AllDataDes.sendCountMatrix, param.All2AllDataDes.sendType,
2121 : param.All2AllDataDes.recvType));
2122 : }
2123 0 : return HCCL_SUCCESS;
2124 : }
2125 :
2126 0 : HcclResult HcclCommAicpu::CalSendRecvInfoFor910B(const std::string &algName, const OpParam ¶m,
2127 : std::unique_ptr<CollExecutorBase> &executor)
2128 : {
2129 : // A2 AICPU才有机会走入RunAlltoAllVStaged
2130 0 : if (algName == "RunAlltoAllVStaged" || algName == "RunAlltoAllVFullMesh") {
2131 0 : CHK_PRT_RET(executor.get() == nullptr,
2132 : HCCL_ERROR("[HcclCommAicpu][%s]Fail to find executor for algName[%s]", __func__, algName.c_str()),
2133 : HCCL_E_PARA);
2134 0 : CHK_RET(CalSendRecvInfoForAlltoall(param));
2135 0 : CollAlltoAllExecutor* alltoAllExecutor = dynamic_cast<CollAlltoAllExecutor *>(executor.get());
2136 0 : CHK_PTR_NULL(alltoAllExecutor);
2137 0 : CHK_RET(alltoAllExecutor->SetExcutorExtraInfo(allMeshAggregationSendRecvInfo_, cclbufferSize_));
2138 0 : HCCL_DEBUG("[HcclCommAicpu][%s] running algName[%s], prepare SendRecvInfo.", __func__, algName.c_str());
2139 0 : return HCCL_SUCCESS;
2140 : }
2141 0 : return HCCL_SUCCESS;
2142 : }
2143 :
2144 0 : HcclResult HcclCommAicpu::GetAlgResponseRes(const std::string &newTag, const std::string &algName,
2145 : const OpParam &opParam, const HcclOpResParam *commParam,
2146 : std::unique_ptr<CollExecutorBase> &executor, AlgResourceResponse*& algResResponse)
2147 : {
2148 0 : HCCL_INFO("[%s] algName[%s]", __func__, algName.c_str());
2149 0 : CHK_PTR_NULL(commParam);
2150 : // 刷新CCLBuffer
2151 0 : CHK_RET(InitCclbuffer(commParam));
2152 0 : auto iter = resMap_.find(newTag);
2153 0 : if (iter == resMap_.end()) {
2154 0 : std::lock_guard<std::mutex> lock(preemptMutexForResMap_);
2155 0 : iter = resMap_.find(newTag);
2156 0 : if (iter == resMap_.end()) {
2157 0 : HCCL_RUN_INFO("[%s] Alloc resource for alg[%s], tag[%s]", __func__, algName.c_str(), newTag.c_str());
2158 0 : AlgResourceRequest resRequest;
2159 0 : CHK_RET(CalcResRequest(algName, opParam, executor, resRequest));
2160 0 : CHK_RET(AllocAlgResource(newTag, opParam, commParam, resRequest, resMap_[newTag]));
2161 0 : iter = resMap_.find(newTag);
2162 0 : } else {
2163 0 : HCCL_INFO("[%s] Repeatedly inited for alg [%s] is not allowed.", __func__, algName.c_str());
2164 : }
2165 0 : } else if (algName == "BatchSendRecv" || algName == "BatchSendRecvRetry" || algName == "BatchSendRecvGroup") {
2166 : // 如果是非aclgraph模式，而且不需要增量建链，则跳过CalcResRequest这个计算，节省时间。在非aclgraph模式下，跳过是安全的。
2167 0 : bool canSkipCalcResRequest = !opParam.isCapture && !opParam.needIncreLink;
2168 0 : if (!canSkipCalcResRequest) { // 如果不能跳过计算，则走一遍计算的流程，否则就跳过以下计算
2169 0 : AlgResourceRequest resRequest;
2170 0 : HCCL_INFO("[%s]IncreAlloc resource for alg[%s], tag[%s]", __func__, algName.c_str(), newTag.c_str());
2171 0 : CHK_RET(CalcResRequest(algName, opParam, executor, resRequest));
2172 0 : CHK_RET(IncreAllocTransportResource(newTag, opParam, commParam, resRequest, resMap_[newTag]));
2173 0 : }
2174 0 : } else if (algName == "RunAlltoAllVStaged" || algName == "RunAlltoAllVFullMesh") {
2175 0 : AlgResourceRequest resRequest;
2176 0 : CHK_RET(CalcResRequest(algName, opParam, executor, resRequest));
2177 0 : HCCL_INFO("[%s] check if need refresh resource for alg[%s], tag[%s], old[%lu], new[%lu]",
2178 : __func__, algName.c_str(), newTag.c_str(), resMap_[newTag].scratchMem.size(), resRequest.scratchMemSize);
2179 0 : bool reAllocFlag = !(resMap_[newTag].scratchMem.size() == resRequest.scratchMemSize);
2180 0 : if (reAllocFlag) {
2181 0 : PetersonLockGuard guard(hostDeviceLock_.get());
2182 0 : CHK_PRT_RET(guard.IsLockFailed(),
2183 : HCCL_ERROR("[HcclCommAicpu][AllocAlgResource] hostDeviceLock lock failed"), HCCL_E_INTERNAL);
2184 0 : CHK_RET(AllocScratchMemResource(newTag, commParam, resRequest.scratchMemSize,
2185 : resMap_[newTag].scratchMem, true));
2186 0 : HCCL_INFO("[%s] refresh resource success for alg[%s], tag[%s], scratchMemSize[%lu], ptr[%p]",
2187 : __func__, algName.c_str(), newTag.c_str(), resRequest.scratchMemSize, resMap_[newTag].scratchMem.ptr());
2188 0 : }
2189 0 : }
2190 0 : CHK_PRT_RET(iter == resMap_.end(),
2191 : HCCL_ERROR("[%s]Fail to find algResResponse for tag[%s]", __func__, newTag.c_str()), HCCL_E_PARA);
2192 0 : algResResponse = &iter->second;
2193 0 : HCCL_INFO("[HcclCommAicpu][GetAlgResponseRes] success!");
2194 0 : return HCCL_SUCCESS;
2195 : }
2196 :
2197 0 : HcclResult HcclCommAicpu::GetAlltoAllVCTotalCount(OpParam ¶m, u64 &sendCount, u64 &recvCount)
2198 : {
2199 0 : for (u32 i = 0; i < topoInfo_.userRankSize; i++) {
2200 0 : sendCount += *(static_cast<const u64 *>(param.All2AllDataDes.sendCountMatrix) +
2201 0 : topoInfo_.userRank * topoInfo_.userRankSize + i);
2202 0 : recvCount += *(static_cast<const u64 *>(param.All2AllDataDes.sendCountMatrix) +
2203 0 : topoInfo_.userRank + topoInfo_.userRankSize * i);
2204 : }
2205 0 : return HCCL_SUCCESS;
2206 : }
2207 :
2208 0 : HcclResult HcclCommAicpu::GetAlltoAllTotalCount(OpParam ¶m, u64 &sendCount, u64 &recvCount)
2209 : {
2210 0 : sendCount = param.All2AllDataDes.sendCount * topoInfo_.userRankSize;
2211 0 : recvCount = param.All2AllDataDes.sendCount * topoInfo_.userRankSize;
2212 0 : return HCCL_SUCCESS;
2213 : }
2214 :
2215 0 : HcclResult HcclCommAicpu::GetAlltoAllVTotalCount(OpParam ¶m, u64 &sendCount, u64 &recvCount)
2216 : {
2217 0 : for (u32 i = 0; i < topoInfo_.userRankSize; i++) {
2218 0 : u64 curSendCount = *(static_cast<const u64 *>(param.All2AllDataDes.sendCounts) + i) +
2219 0 : *(static_cast<const u64 *>(param.All2AllDataDes.sdispls) + i);
2220 0 : sendCount = std::max(sendCount, curSendCount);
2221 0 : u64 curRecvCount = *(static_cast<const u64 *>(param.All2AllDataDes.recvCounts) + i) +
2222 0 : *(static_cast<const u64 *>(param.All2AllDataDes.rdispls) + i);
2223 0 : recvCount = std::max(recvCount, curRecvCount);
2224 : }
2225 0 : return HCCL_SUCCESS;
2226 : }
2227 :
2228 0 : HcclResult HcclCommAicpu::SetAlltoAllInputAndOutPutMem(OpParam ¶m, AlgResourceResponse &algResource)
2229 : {
2230 0 : u32 sendTypeSize = 0, recvTypeSize = 0;
2231 0 : CHK_RET(SalGetDataTypeSize(param.All2AllDataDes.sendType, sendTypeSize));
2232 0 : CHK_RET(SalGetDataTypeSize(param.All2AllDataDes.recvType, recvTypeSize));
2233 0 : u64 sendCount = 0;
2234 0 : u64 recvCount = 0;
2235 0 : if (param.opType == HcclCMDType::HCCL_CMD_ALLTOALL) {
2236 0 : CHK_RET(GetAlltoAllTotalCount(param, sendCount, recvCount));
2237 0 : } else if (param.opType == HcclCMDType::HCCL_CMD_ALLTOALLV) {
2238 0 : CHK_RET(GetAlltoAllVTotalCount(param, sendCount, recvCount));
2239 0 : } else if (param.opType == HcclCMDType::HCCL_CMD_ALLTOALLVC) {
2240 0 : CHK_RET(GetAlltoAllVCTotalCount(param, sendCount, recvCount));
2241 : }
2242 0 : u64 inputSize = sendCount * sendTypeSize;
2243 0 : u64 outputSize = recvCount * recvTypeSize;
2244 0 : algResource.paramInputMem = inputSize == 0 ?
2245 0 : tinySendRecvMem_ : DeviceMem::create(param.inputPtr, inputSize);
2246 0 : algResource.paramOutputMem = outputSize == 0 ?
2247 0 : tinySendRecvMem_ : DeviceMem::create(param.outputPtr, outputSize);
2248 0 : HCCL_DEBUG("[HcclCommAicpu][SetAlltoAllInputAndOutPutMem] Set memory for AllToAll, inputSize[%llu], inputPtr[%p],"
2249 : "outputPtr[%p]!", inputSize, param.inputPtr, param.outputPtr);
2250 0 : return HCCL_SUCCESS;
2251 : }
2252 :
2253 0 : HcclResult HcclCommAicpu::CombineReportOpInfo(OpParam ¶m, bool isRetry, bool isRelay)
2254 : {
2255 0 : MsprofAicpuHCCLOPInfo hcclOpInfo{0};
2256 0 : hcclOpInfo.relay = (isRelay) ? 1 : 0;
2257 0 : hcclOpInfo.retry = (isRetry) ? 1 : 0;
2258 0 : hcclOpInfo.dataType = param.DataDes.dataType;
2259 0 : hcclOpInfo.count = param.DataDes.count;
2260 0 : hcclOpInfo.groupName = groupHashId_;
2261 0 : hcclOpInfo.ranksize = topoInfo_.userRankSize;
2262 0 : std::string algTypeStr = TransferAlgType(algType_);
2263 0 : CHK_RET(dfx::ProfilingManager::ReportHcclOpInfo(hcclOpInfo, algTypeStr));
2264 0 : return HCCL_SUCCESS;
2265 0 : }
2266 :
2267 0 : HcclResult HcclCommAicpu::UpdateProfReportStartSqeIdx()
2268 : {
2269 0 : if (dfx::ProfilingManager::IsL1fromOffToOn()) {
2270 0 : std::vector<Stream> streams;
2271 0 : CHK_RET(GetStreamAll(streams));
2272 0 : for (auto &tmpStream : streams) {
2273 0 : HcclSqeContext *sqeContext = tmpStream.GetSqeContextPtr();
2274 0 : SqeRingBuffer *sqeContextBuffer = &(sqeContext->buffer);
2275 0 : CHK_RET(dfx::ProfilingManager::UpdateStartReportSqeIdx(tmpStream.id(), sqeContextBuffer->tailSqeIdx));
2276 : }
2277 0 : }
2278 0 : return HCCL_SUCCESS;
2279 : }
2280 :
2281 0 : HcclResult HcclCommAicpu::Orchestrate(const std::string &newTag, const std::string &algName, OpParam ¶m,
2282 : std::unique_ptr<CollExecutorBase> &executor, AlgResourceResponse &algResource, const HcclOpResParam *commParam)
2283 : {
2284 0 : CHK_PTR_NULL(commParam);
2285 : // 算子下发信息记录在共享内存区
2286 0 : UpdateOpRingBufferIdx();
2287 0 : CHK_RET(aicpuShareData_.RecordOpInfo(newTag, param, (isDeviceMode_ ? mc2OpIndex_ : hcclOpExecIndex_),
2288 : localUserRank_, isCustom_));
2289 0 : CHK_RET(UpdateProfReportStartSqeIdx());
2290 :
2291 : // 每个算子都刷新一下profiling开关, 支持profiling从中间迭代采集
2292 0 : bool profL0Open = dfx::ProfilingManager::IsProfL0On();
2293 0 : bool profL1Open = dfx::ProfilingManager::IsProfL1On();
2294 0 : HCCL_DEBUG("profL0Open:%d, profL1Open:%d", profL0Open, profL1Open);
2295 :
2296 0 : LogControl logControl(false, false); // 重执行ERROR日志保底控制，析构时重置日志设置
2297 0 : HCCL_ENTRY_INFO(commParam->opEntry, "[HcclCommAicpu][Orchestrate]start tag[%s] newTag[%s] algName[%s] identifier[%s]",
2298 : param.tag.c_str(), newTag.c_str(), algName.c_str(), identifier_.c_str());
2299 0 : HCCL_INFO("opRetryHandler.isInplacePreSync[%d] opRetryHandler.isPostSync[%d]",
2300 : algOpContext_.opRetryHandler.isInplacePreSync, algOpContext_.opRetryHandler.isPostSync);
2301 0 : if (executor.get() == nullptr) {
2302 0 : executor = CollAlgExecRegistry::Instance().GetAlgExec(algName, dispatcher_, topoMatcher_);
2303 0 : CHK_PRT_RET(executor.get() == nullptr, HCCL_ERROR("[HcclCommAicpu][Orchestrate]Fail to find executor "
2304 : "for algName[%s]", algName.c_str()), HCCL_E_PARA);
2305 0 : executor->SetAlgType(algType_);
2306 0 : executor->SetCCLInBuffer(cclbufferSize_);
2307 :
2308 0 : if (param.opType == HcclCMDType::HCCL_CMD_REDUCE_SCATTER) {
2309 0 : bool isSupportSDMAReduce = IsSupportSDMAReduce(cclInputBuffer_.ptr(), cclOutputBuffer_.ptr(),
2310 : param.DataDes.dataType, param.reduceType);
2311 0 : executor->SetIsSupportSDMAReduce(isSupportSDMAReduce);
2312 : }
2313 : }
2314 0 : if (param.opType == HcclCMDType::HCCL_CMD_ALLTOALL || param.opType == HcclCMDType::HCCL_CMD_ALLTOALLV ||
2315 0 : param.opType == HcclCMDType::HCCL_CMD_ALLTOALLVC) {
2316 0 : CHK_RET(CalSendRecvInfoFor910B(algName, param, executor));
2317 0 : CHK_RET(SetAlltoAllInputAndOutPutMem(param, algResource));
2318 0 : if (algName == "RunAlltoAllVTwoLevelPipeline") {
2319 0 : CHK_RET(CalSendRecvInfoForAlltoall(param));
2320 0 : HCCL_DEBUG("[HcclCommAicpu][Orchestrate] running RunAlltoAllVTwoLevelPipeline, prepare SendRecvInfo.");
2321 0 : CollAlltoAllExecutor* alltoAllExecutor = dynamic_cast<CollAlltoAllExecutor *>(executor.get());
2322 0 : CHK_PTR_NULL(alltoAllExecutor);
2323 0 : CHK_RET(alltoAllExecutor->SetExcutorExtraInfo(allMeshAggregationSendRecvInfo_, cclbufferSize_));
2324 : }
2325 : }
2326 0 : auto waitStopExecCmdTimeoutMs = HcclGetWaitStopExecCmdTimeout();
2327 0 : auto waitStopExecCmdTimeout = std::chrono::milliseconds(waitStopExecCmdTimeoutMs);
2328 :
2329 0 : auto opStartTime = std::chrono::steady_clock::now(); // 记录重执行算子耗时
2330 0 : auto startTime = std::chrono::steady_clock::now();
2331 :
2332 0 : KfcError errorCode = KfcError::kNone;
2333 0 : uint32_t retryCnt = 0;
2334 0 : bool retryProcessing = false;
2335 0 : KfcCommand lastCmd = KfcCommand::kNone;
2336 0 : uint32_t beginSqePos = INVALID_UINT;
2337 0 : uint32_t endSqePos = INVALID_UINT;
2338 0 : HcclOpExecFSM state = HcclOpExecFSM::HCCL_OP_EXEC_FSM_INIT;
2339 0 : HcclResult ret = HCCL_SUCCESS;
2340 0 : dfxExtendInfo_.kfcStatus = DfxKfcStatus::kOneStart;
2341 0 : AicpuComContext *ctx = AicpuGetComContext();
2342 0 : AicpuHcclProcess::CallMC2MaintenanceThread(ctx);
2343 0 : u32 loopCnt = 0;
2344 0 : u32 loopNum = 1;
2345 0 : commParam_ = commParam;
2346 0 : CHK_RET(InitExecLoop(param, executor, loopNum));
2347 :
2348 : while (true) {
2349 0 : switch (state) {
2350 0 : case HcclOpExecFSM::HCCL_OP_EXEC_FSM_INIT:
2351 0 : HCCL_INFO("hccl aicpu execute loop %u", loopCnt);
2352 0 : ret = HcclOpExecFsmInitProcess(newTag, param, algResource, state, errorCode);
2353 0 : break;
2354 0 : case HcclOpExecFSM::HCCL_OP_EXEC_FSM_LAUNCH:
2355 0 : ret = HcclOpExecFsmLaunchProcess(
2356 : algName, param, executor, algResource, state, errorCode, beginSqePos, endSqePos, retryCnt);
2357 0 : if (ret == HCCL_E_SUSPENDING && isDeviceMode_ && retryEnable_) {
2358 0 : return HCCL_E_SUSPENDING;
2359 : }
2360 0 : break;
2361 0 : case HcclOpExecFSM::HCCL_OP_EXEC_FSM_WAIT_END:
2362 0 : ret = HcclOpExecFsmWaitEndProcess(param, algResource, state, errorCode, retryCnt, param.tag, beginSqePos);
2363 0 : if (state == HcclOpExecFSM::HCCL_OP_EXEC_FSM_STOPPING) {
2364 0 : startTime = std::chrono::steady_clock::now();
2365 : }
2366 0 : break;
2367 0 : case HcclOpExecFSM::HCCL_OP_EXEC_FSM_STOPPING:
2368 0 : retryProcessing = true;
2369 0 : if ((std::chrono::steady_clock::now() - startTime) >= waitStopExecCmdTimeout) {
2370 0 : HCCL_ERROR("[OpRetry][AICPU]hccl aicpu wait stop exec timeout[%u ms].", waitStopExecCmdTimeoutMs);
2371 0 : errorCode = KfcError::kTimeout;
2372 0 : state = HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR;
2373 : } else {
2374 0 : ret = HcclOpExecFsmStoppingProcess(param, state, errorCode, retryCnt);
2375 : }
2376 0 : break;
2377 0 : case HcclOpExecFSM::HCCL_OP_EXEC_FSM_STOPPED:
2378 0 : ret = HcclOpExecFsmStoppedProcess(state, errorCode, retryCnt, algName, param, beginSqePos, endSqePos);
2379 0 : if (state == HcclOpExecFSM::HCCL_OP_EXEC_FSM_WAIT_RETRY) {
2380 0 : startTime = std::chrono::steady_clock::now();
2381 : }
2382 0 : break;
2383 0 : case HcclOpExecFSM::HCCL_OP_EXEC_FSM_CHANGE_LINK:
2384 0 : ret = HcclOpExecChangeLinkProcess(newTag, state, errorCode, retryCnt, algResource, commParam, param);
2385 0 : HCCL_DEBUG("[OpRetry][AICPU]retry change link finish, retryCnt:%u, tag:%s, state:%d",
2386 : retryCnt, param.tag.c_str(), state);
2387 0 : break;
2388 0 : case HcclOpExecFSM::HCCL_OP_EXEC_FSM_WAIT_RETRY:
2389 : {
2390 0 : auto waitRetryCmdTimeoutMs = HcclGetWaitRetryCmdTimeout(retryCnt);
2391 0 : auto waitRetryCmdTimeout = std::chrono::milliseconds(waitRetryCmdTimeoutMs);
2392 0 : if ((std::chrono::steady_clock::now() - startTime) >= waitRetryCmdTimeout) {
2393 0 : HCCL_ERROR("[OpRetry][AICPU]hccl aicpu wait retry timeout[%u ms].", waitRetryCmdTimeoutMs);
2394 0 : errorCode = KfcError::kTimeout;
2395 0 : state = HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR;
2396 : } else {
2397 0 : ret = HcclOpExecFsmWaitRetryProcess(param, state, errorCode, lastCmd);
2398 : }
2399 : }
2400 0 : break;
2401 0 : case HcclOpExecFSM::HCCL_OP_EXEC_FSM_RETRY:
2402 : // 重执行前清理当前算子展开的SQE缓存 (if any), 防止命中非完整的cache
2403 0 : CHK_RET(aicpuCacheManager_.ClearOpUnfoldCacheEntry(algName, param, algResource, isDeviceMode_, topoInfo_,
2404 : topoMatcher_, algOpContext_, GetWorkflowMode()));
2405 0 : ret = HcclOpExecFsmRetryProcess(algName, param, executor, algResource, state, errorCode, retryCnt,
2406 : beginSqePos, endSqePos);
2407 0 : break;
2408 0 : case HcclOpExecFSM::HCCL_OP_EXEC_FSM_END:
2409 0 : loopCnt++;
2410 0 : if ((param.opType == HcclCMDType::HCCL_CMD_BATCH_SEND_RECV) && retryEnable_) {
2411 0 : param.BatchSendRecvDataDes.curIterNum = loopCnt;
2412 0 : ResetBSRRetryCnt();
2413 : }
2414 0 : if (loopCnt < loopNum) {
2415 0 : state = HcclOpExecFSM::HCCL_OP_EXEC_FSM_INIT;
2416 0 : break;
2417 : }
2418 0 : if (retryCnt > 0) {
2419 0 : RecordReportStatus(dfx::ReportStatus::kRetrySuccess);
2420 0 : retryProcessing = false;
2421 0 : auto opEndTime = std::chrono::steady_clock::now();
2422 0 : auto duration = std::chrono::duration_cast<std::chrono::seconds>(opEndTime - opStartTime).count();
2423 0 : HCCL_RUN_INFO("[OpRetry][AICPU]retry exec success, retryCnt [%u], tag [%s], take time [%ld]s",
2424 : retryCnt, param.tag.c_str(), duration);
2425 : }
2426 0 : CHK_RET(CombineReportOpInfo(param, (retryCnt > 0), false));
2427 0 : return HcclOpExecFsmEndProcess(retryCnt);
2428 0 : case HcclOpExecFSM::HCCL_OP_EXEC_STOP_LAUNCH:
2429 0 : HCCL_DEBUG("[NsRecovery][AICPU] stop the kernel");
2430 : // 停止前清理当前算子展开的SQE缓存 (if any), 防止host侧重新展开该算子并命中非完整的cache (例如step快恢)
2431 0 : CHK_RET(aicpuCacheManager_.ClearOpUnfoldCacheEntry(algName, param, algResource, isDeviceMode_, topoInfo_,
2432 : topoMatcher_, algOpContext_, GetWorkflowMode()));
2433 0 : if (!needsResponseStopLaunch_) {
2434 0 : return HCCL_E_SUSPENDING;
2435 : } else {
2436 0 : HCCL_RUN_INFO("[NsRecovery][AICPU] stop the kernel for stop cmd");
2437 0 : needsResponseStopLaunch_ = false;
2438 0 : SetCommRecoveryFlag(true);
2439 0 : if (UpdateOpExecStatus(state, KfcStatus::kStoplaunch, errorCode, 0) == HCCL_SUCCESS) {
2440 0 : return HCCL_E_SUSPENDING;
2441 : } else {
2442 0 : break;
2443 : }
2444 : }
2445 0 : case HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR:
2446 : default: {
2447 0 : if (retryProcessing) {
2448 0 : RecordReportStatus(dfx::ReportStatus::kRetryFail);
2449 0 : retryProcessing = false;
2450 : }
2451 0 : UpdateOpExecStatus(state, excuteOpId_, KfcStatus::kRetryError, errorCode, retryCnt);
2452 0 : dfxExtendInfo_.kfcStatus = DfxKfcStatus::kOneFinished;
2453 0 : if (!isDeviceMode_) {
2454 0 : isOpLaunch = false;
2455 : }
2456 0 : HCCL_INFO("hccl aicpu set kfcStatus[%d]", dfxExtendInfo_.kfcStatus);
2457 0 : if (errorCode == KfcError::kExecConstraint) {
2458 0 : return HCCL_E_OPRETRY_FAIL;
2459 : }
2460 0 : return (ret == HCCL_SUCCESS) ? HCCL_E_INTERNAL : ret;
2461 : }
2462 : }
2463 0 : }
2464 : return ret;
2465 0 : }
2466 :
2467 1 : HcclResult HcclCommAicpu::InitBsrSendRecvOpIdAndExcuteOpId(OpParam ¶m, AlgResourceResponse &algResource,
2468 : HcclOpExecFSM &fsmState, KfcError &errorCode)
2469 : {
2470 1 : auto hcclRet = InitBatchSendRecvOpId(param, algResource);
2471 1 : HCCL_RETRY_CHK_RET_AND_TRANS_FSM(hcclRet, HCCL_ERROR("InitBatchSendRecvOpId failed, ret:%u", hcclRet),
2472 : KfcError::kInner, HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR);
2473 1 : param.BatchSendRecvDataDes.curMode = BatchSendRecvCurMode::SEND_RECV;
2474 1 : if (param.BatchSendRecvDataDes.curIterNum == 0) {
2475 1 : bsrSendStream_ = algResource.slaveStreams[BSR_RETRY_SEND_STREAM_INDEX];
2476 1 : bsrRecvStream_ = algResource.slaveStreams[BSR_RETRY_RECV_STREAM_INDEX];
2477 : }
2478 1 : HCCL_INFO("BSR: iter %u, tag:%s index:%u", param.BatchSendRecvDataDes.curIterNum, excuteOpId_.tag,
2479 : excuteOpId_.index);
2480 1 : HCCL_INFO("BSR: iter %u, send op Tag:%s index:%u", param.BatchSendRecvDataDes.curIterNum, bsrSendOpId_.tag,
2481 : bsrSendOpId_.index);
2482 1 : HCCL_INFO("BSR: iter %u, recv op Tag:%s index:%u", param.BatchSendRecvDataDes.curIterNum, bsrRecvOpId_.tag,
2483 : bsrRecvOpId_.index);
2484 1 : excuteOpId_.bsrInfo[HCCL_SEND].index = bsrSendOpId_.index;
2485 1 : excuteOpId_.bsrInfo[HCCL_RECV].index = bsrRecvOpId_.index;
2486 1 : excuteOpId_.bsrInfo[HCCL_SEND].tpQpn = bsrSendOpId_.bsrInfo[HCCL_SEND].tpQpn;
2487 1 : excuteOpId_.bsrInfo[HCCL_RECV].tpQpn = bsrRecvOpId_.bsrInfo[HCCL_RECV].tpQpn;
2488 1 : excuteOpId_.bsrInfo[HCCL_SEND].streamId = bsrSendOpId_.streamId;
2489 1 : excuteOpId_.bsrInfo[HCCL_RECV].streamId = bsrRecvOpId_.streamId;
2490 1 : excuteOpId_.bsrInfo[HCCL_SEND].srcRank = bsrSendOpId_.srcRank;
2491 1 : excuteOpId_.bsrInfo[HCCL_SEND].detRank = bsrSendOpId_.detRank;
2492 1 : excuteOpId_.bsrInfo[HCCL_RECV].srcRank = bsrRecvOpId_.srcRank;
2493 1 : excuteOpId_.bsrInfo[HCCL_RECV].detRank = bsrRecvOpId_.detRank;
2494 1 : CHK_SAFETY_FUNC_RET(memcpy_s(excuteOpId_.bsrInfo[HCCL_SEND].bsrTag, sizeof(excuteOpId_.bsrInfo[HCCL_SEND].bsrTag),
2495 : bsrSendOpId_.tag, sizeof(bsrSendOpId_.tag)));
2496 1 : CHK_SAFETY_FUNC_RET(memcpy_s(excuteOpId_.bsrInfo[HCCL_RECV].bsrTag, sizeof(excuteOpId_.bsrInfo[HCCL_RECV].bsrTag),
2497 : bsrRecvOpId_.tag, sizeof(bsrRecvOpId_.tag)));
2498 1 : return HCCL_SUCCESS;
2499 : }
2500 :
2501 1 : HcclResult HcclCommAicpu::HcclOpExecFsmInitProcess(const std::string &newTag, OpParam ¶m,
2502 : AlgResourceResponse &algResource, HcclOpExecFSM &fsmState, KfcError &errorCode)
2503 : {
2504 1 : excuteOpId_.index = isDeviceMode_ ? (++mc2OpIndex_) : hcclOpExecIndex_;
2505 1 : CHK_SAFETY_FUNC_RET(memset_s(excuteOpId_.tag, sizeof(excuteOpId_.tag), 0, sizeof(excuteOpId_.tag)));
2506 1 : CHK_SAFETY_FUNC_RET(memcpy_s(excuteOpId_.tag, sizeof(excuteOpId_.tag), param.tag.c_str(), param.tag.size()));
2507 1 : CHK_SAFETY_FUNC_RET(memset_s(excuteOpId_.newTag, sizeof(excuteOpId_.newTag), 0, sizeof(excuteOpId_.newTag)));
2508 1 : CHK_SAFETY_FUNC_RET(memcpy_s(excuteOpId_.newTag, sizeof(excuteOpId_.newTag), newTag.c_str(), newTag.size()));
2509 1 : excuteOpId_.isSendRecv = false;
2510 1 : excuteOpId_.streamId = ~0u;
2511 1 : excuteOpId_.opType = param.opType;
2512 1 : excuteOpId_.isBsrTaskStart = false;
2513 1 : if (param.opType == HcclCMDType::HCCL_CMD_SEND || param.opType == HcclCMDType::HCCL_CMD_RECEIVE) {
2514 0 : InitSendRecvOpId(param, excuteOpId_);
2515 1 : } else if ((param.opType == HcclCMDType::HCCL_CMD_BATCH_SEND_RECV) && retryEnable_) {
2516 1 : CHK_RET(InitBsrSendRecvOpIdAndExcuteOpId(param, algResource, fsmState, errorCode));
2517 : }
2518 1 : if (GetNsStopLaunchStatus()) {
2519 0 : HCCL_WARNING("the op should not be launched in the suspending status");
2520 0 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_STOP_LAUNCH;
2521 0 : return HCCL_SUCCESS;
2522 : }
2523 1 : auto ret = aicpuHdc_.InitOpExecStatus(kfcStatusTransferD2H_, excuteOpId_);
2524 1 : isOpLaunch = true;
2525 1 : HCCL_INFO("%s tag:%s, isDeviceMode:%d, index:%u", __func__, excuteOpId_.tag, isDeviceMode_, excuteOpId_.index);
2526 1 : HCCL_RETRY_CHK_RET_AND_TRANS_FSM(ret, HCCL_ERROR("InitOpExecStatus failed, ret:%u", ret), KfcError::kInner,
2527 : HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR);
2528 1 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_FSM_LAUNCH;
2529 1 : return ret;
2530 : }
2531 :
2532 0 : bool HcclCommAicpu::HcclOpCheckSupportRetry(HcclCMDType opType)
2533 : {
2534 : const std::set<HcclCMDType> HcclSupportRetryOpSet = {
2535 : HcclCMDType::HCCL_CMD_BROADCAST, HcclCMDType::HCCL_CMD_ALLREDUCE, HcclCMDType::HCCL_CMD_REDUCE,
2536 : HcclCMDType::HCCL_CMD_ALLGATHER, HcclCMDType::HCCL_CMD_REDUCE_SCATTER,
2537 : HcclCMDType::HCCL_CMD_ALLTOALLV, HcclCMDType::HCCL_CMD_ALLTOALLVC, HcclCMDType::HCCL_CMD_ALLTOALL,
2538 : HcclCMDType::HCCL_CMD_GATHER, HcclCMDType::HCCL_CMD_SCATTER, HcclCMDType::HCCL_CMD_SEND,
2539 : HcclCMDType::HCCL_CMD_RECEIVE, HcclCMDType::HCCL_CMD_BATCH_SEND_RECV
2540 0 : };
2541 0 : return (HcclSupportRetryOpSet.find(opType) != HcclSupportRetryOpSet.end());
2542 0 : }
2543 :
2544 1 : HcclResult HcclCommAicpu::HcclOpExecFsmLaunchProcess(const std::string &algName, OpParam ¶m,
2545 : std::unique_ptr<CollExecutorBase> &executor, AlgResourceResponse &algResource, HcclOpExecFSM &fsmState,
2546 : KfcError &errorCode, uint32_t &beginSqePos, uint32_t &endSqePos, uint32_t retryCnt)
2547 : {
2548 1 : HCCL_DEBUG("hccl aicpu start launch task.");
2549 :
2550 1 : HcclResult ret = OrchestrateHcclOp(algName, param, executor, algResource, beginSqePos, endSqePos);
2551 1 : if (ret == HCCL_SUCCESS) { // 下发成功, 并且没有检测到异常cq或中断命令
2552 0 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_FSM_WAIT_END;
2553 1 : } else if (ret == HCCL_E_SUSPENDING) { // 检测到异常cq或中断命令
2554 1 : if (isDeviceMode_ && retryEnable_) {
2555 1 : HCCL_RUN_INFO("Orchestrate hccl op suspending, restart handle by mc2 process.");
2556 1 : return ret;
2557 : }
2558 0 : if ((param.opType == HcclCMDType::HCCL_CMD_BATCH_SEND_RECV) && retryEnable_) {
2559 : // batchsendrecv算子下发过程中出现异常，task下发未完成，send 和 recv 均需要重执行;
2560 : // 第一个g故障op重执行下发task完成后，需要主动上报故障，触发第二个op进行重执行
2561 0 : SetBSRSendOpExecException();
2562 0 : SetBSRRecvOpExecException();
2563 0 : HCCL_RUN_INFO("hccl aicpu abort launch batchsendrecv op, need retry");
2564 : }
2565 0 : CHK_RET(UpdateSuspendStatus(param, fsmState, errorCode, retryCnt));
2566 : } else {
2567 0 : HCCL_ERROR("OrchestrateHcclOp failed, ret:%u", ret);
2568 0 : errorCode = KfcError::kInner;
2569 0 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR;
2570 : }
2571 0 : return ret;
2572 : }
2573 :
2574 4 : HcclResult HcclCommAicpu::HcclOpExecFsmWaitEndProcess(OpParam ¶m, AlgResourceResponse &algResource,
2575 : HcclOpExecFSM &fsmState, KfcError &errorCode, uint32_t retryCnt, std::string &tag, const uint32_t &beginSqePos)
2576 : {
2577 4 : HCCL_DEBUG("hccl aicpu wait task finish.");
2578 4 : auto ret = WaitFinishWhileLoop(mainStream_, algResource.slaveStreams, tag, beginSqePos, param);
2579 4 : if (ret == HCCL_SUCCESS) {
2580 0 : HCCL_DEBUG("hccl aicpu exec complete.");
2581 0 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_FSM_END;
2582 4 : } else if (ret == HCCL_E_SUSPENDING) {
2583 4 : HCCL_RUN_INFO("hccl aicpu force stop in wait end, retryCnt[%u]", retryCnt);
2584 4 : CHK_RET(UpdateSuspendStatus(param, fsmState, errorCode, retryCnt));
2585 : } else {
2586 0 : HCCL_ERROR("WaitTaskFinish failed, ret:%u, identifier[%s]", ret, identifier_.c_str());
2587 0 : errorCode = KfcError::kExec;
2588 0 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR;
2589 : }
2590 4 : return ret;
2591 : }
2592 :
2593 0 : HcclResult HcclCommAicpu::HcclOpExecFsmStoppingProcess(const OpParam ¶m, HcclOpExecFSM &fsmState,
2594 : KfcError &errorCode, uint32_t retryCnt)
2595 : {
2596 0 : HCCL_DEBUG("hccl aicpu stopping.");
2597 0 : KfcCommand cmd = KfcCommand::kNone;
2598 0 : auto ret = aicpuHdc_.GetOpExecCtrlCmd(kfcControlTransferH2D_, cmd);
2599 0 : if (ret != HCCL_SUCCESS) {
2600 0 : HCCL_ERROR("[OpRetry][AICPU]GetOpExecCtrlCmd failed, ret:%u", ret);
2601 0 : errorCode = KfcError::kExec;
2602 0 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR;
2603 0 : return ret;
2604 : }
2605 0 : if (cmd == KfcCommand::kExit) {
2606 0 : HCCL_RUN_INFO("[OpRetry][AICPU]hccl aicpu exec fsm stop by exit cmd.");
2607 0 : errorCode = KfcError::kExit;
2608 0 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR;
2609 0 : } else if (cmd == KfcCommand::kStopExec) {
2610 0 : HCCL_RUN_INFO("[OpRetry][AICPU]hccl aicpu get stop exec cmd.");
2611 0 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_FSM_STOPPED;
2612 0 : } else if (cmd == KfcCommand::kStopLaunch) {
2613 0 : if ((param.opType == HcclCMDType::HCCL_CMD_BATCH_SEND_RECV) && retryEnable_) {
2614 0 : HcclOpIdentifier targetOp;
2615 0 : CHK_RET(aicpuHdc_.GetOpExecCtrlTargetOp(kfcControlTransferH2D_, targetOp));
2616 0 : std::string targetOpTag = std::string(reinterpret_cast<char*>(&targetOp.tag[0]));
2617 0 : if (targetOpTag != std::string(reinterpret_cast<char*>(&bsrTargetOpId_.tag[0]))) {
2618 0 : CHK_RET(UpdateSuspendStatus(param, fsmState, errorCode, retryCnt));
2619 : }
2620 0 : }
2621 0 : } else if ((cmd == KfcCommand::kNone) || (cmd == KfcCommand::kRetry)) {
2622 0 : HCCL_DEBUG("hccl aicpu wait for stop exec cmd.");
2623 : // do nothing
2624 0 : } else if (cmd == KfcCommand::kReportRetryErr) {
2625 0 : HCCL_ERROR("[OpRetry][AICPU][HcclOpExecFsmStoppingProcess]hccl aicpu can not retry err cmd[%d]", cmd);
2626 0 : uint16_t rsErrorCode = TS_ERROR_RETRY_CONSTRAINT;
2627 0 : CHK_PRT(SendTaskExceptionByMBox(rsErrorCode));
2628 0 : errorCode = KfcError::kExit;
2629 0 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR;
2630 0 : return HCCL_E_OPRETRY_FAIL;
2631 : } else {
2632 0 : HCCL_ERROR("[OpRetry][AICPU]GetOpExecCtrlCmd failed, invalid cmd[%u]", cmd);
2633 0 : errorCode = KfcError::kExec;
2634 0 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR;
2635 : }
2636 0 : return HCCL_SUCCESS;
2637 : }
2638 :
2639 0 : HcclResult HcclCommAicpu::LoadChangeLinkInfo(ChangeLinkInfo &changeLinkInfo)
2640 : {
2641 0 : HcclResult ret = aicpuHdc_.GetOpExecChangeLink(kfcControlTransferH2D_, changeLinkInfo);
2642 :
2643 : // DEBUG_INFO aicpu接收的changelinkinfo
2644 0 : std::string changeLinkInfoStr = "";
2645 0 : for (u32 i = 0; i < changeLinkInfo.remoteRankNum; i++) {
2646 0 : changeLinkInfoStr += (std::to_string(changeLinkInfo.remoteRankList[i]) + ":" +
2647 0 : std::to_string(changeLinkInfo.isUseDefaultPort[i]) + "; ");
2648 : }
2649 0 : HCCL_RUN_INFO("[%s]rank[%u], isChangeLinkFlag[%d], changeLinkInfoStr:%s", __func__, localUserRank_,
2650 : changeLinkInfo.isChangeLinkFlag, changeLinkInfoStr.c_str());
2651 :
2652 0 : return ret;
2653 0 : }
2654 :
2655 0 : HcclResult HcclCommAicpu::HcclOpExecChangeLinkProcess(const std::string &newTag, HcclOpExecFSM &state,
2656 : KfcError &errorCode, uint32_t &retryCnt, AlgResourceResponse &algResource, const HcclOpResParam *commParam,
2657 : const OpParam ¶m)
2658 : {
2659 0 : ChangeLinkInfo changeLinkInfo;
2660 0 : HcclResult ret = LoadChangeLinkInfo(changeLinkInfo);
2661 0 : if (ret != HCCL_SUCCESS) {
2662 0 : errorCode = KfcError::kExec;
2663 0 : state = HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR;
2664 0 : return ret;
2665 : }
2666 0 : bool useBackupLink = false;
2667 0 : std::map<u32, bool> remoteRankPortMap;
2668 0 : for (u32 i = 0; i < changeLinkInfo.remoteRankNum; i++) {
2669 0 : remoteRankPortMap.insert({changeLinkInfo.remoteRankList[i], changeLinkInfo.isUseDefaultPort[i]});
2670 0 : useBackupLink |= (!changeLinkInfo.isUseDefaultPort[i]);
2671 : }
2672 0 : ret = RefreshAlgResponseTransportRes(newTag, algResource, remoteRankPortMap,
2673 0 : changeLinkInfo.isChangeLinkFlag, commParam, param);
2674 0 : if (ret != HCCL_SUCCESS) {
2675 0 : errorCode = KfcError::kExec;
2676 0 : state = HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR;
2677 0 : return ret;
2678 : }
2679 0 : if (useBackupLink) {
2680 0 : RecordReportStatus(dfx::ReportStatus::kRetryWithBackupLink);
2681 : }
2682 0 : if ((param.opType == HcclCMDType::HCCL_CMD_BATCH_SEND_RECV) && retryEnable_) {
2683 0 : retryCnt = (bsrRetryOp_ == HCCL_SEND) ? bsrSendRetryCnt_ : bsrRecvRetryCnt_;
2684 : }
2685 0 : errorCode = KfcError::kNone;
2686 0 : CHK_RET(UpdateOpExecStatus(state, KfcStatus::kChanged, errorCode, retryCnt));
2687 0 : state = HcclOpExecFSM::HCCL_OP_EXEC_FSM_WAIT_RETRY;
2688 0 : return HCCL_SUCCESS;
2689 0 : }
2690 :
2691 0 : HcclResult HcclCommAicpu::BSRStopedProcess(HcclOpExecFSM &fsmState, KfcError &errorCode)
2692 : {
2693 : // 判断batchsendrecv算子的send 和recv 操作停止的位置是否满足重执行条件
2694 : // send / recv 的stream停止位置不能位于该算子的首个sqe 和末尾sqe
2695 : u32 bsrSendSqHead;
2696 : u32 bsrRecvSqHead;
2697 0 : auto ret = QuerySqStatusByType(devId_, bsrSendStream_.sqId(), DRV_SQCQ_PROP_SQ_HEAD, bsrSendSqHead);
2698 0 : HCCL_RETRY_CHK_RET_AND_TRANS_FSM(ret, HCCL_ERROR("[OpRetry][AICPU]quert send stream sq head failed, ret:%u", ret),
2699 : KfcError::kExec, HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR);
2700 :
2701 0 : ret = QuerySqStatusByType(devId_, bsrRecvStream_.sqId(), DRV_SQCQ_PROP_SQ_HEAD, bsrRecvSqHead);
2702 0 : HCCL_RETRY_CHK_RET_AND_TRANS_FSM(ret, HCCL_ERROR("[OpRetry][AICPU]quert recv stream sq head failed, ret:%u", ret),
2703 : KfcError::kExec, HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR);
2704 :
2705 0 : ret = ((bsrSendOpBeginSqePos_ == bsrSendSqHead) || (bsrRecvOpBeginSqePos_ == bsrRecvSqHead)) ? HCCL_E_OPRETRY_FAIL :
2706 : HCCL_SUCCESS;
2707 0 : if (ret != HCCL_SUCCESS) {
2708 0 : uint16_t rsErrorCode = TS_ERROR_RETRY_CONSTRAINT;
2709 0 : CHK_PRT(SendTaskExceptionByMBox(rsErrorCode));
2710 : }
2711 0 : HCCL_RETRY_CHK_RET_AND_TRANS_FSM(ret,
2712 : HCCL_ERROR("[OpRetry][AICPU]hccl aicpu wait start task is not complete, can not retry. params: send sq head "
2713 : "%u, recv sq head %u, send sq begin %u, recv sq begin %u",
2714 : bsrSendSqHead, bsrRecvSqHead, bsrSendOpBeginSqePos_, bsrRecvOpBeginSqePos_),
2715 : KfcError::kExecConstraint, HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR);
2716 :
2717 0 : ret = ((bsrRetryOp_ == HCCL_SEND) && (bsrSendSqHead == bsrSendOpEndSqePos_)) ? HCCL_E_OPRETRY_FAIL : HCCL_SUCCESS;
2718 0 : HCCL_RETRY_CHK_RET_AND_TRANS_FSM(ret,
2719 : HCCL_ERROR("[OpRetry][AICPU]hccl aicpu send record complete task is complete, can not retry. params: send sq "
2720 : "head %u, recv sq head %u, send sq begin %u, send sq end %u, recv sq begin %u, recv sq end %u",
2721 : bsrSendSqHead, bsrRecvSqHead, bsrSendOpBeginSqePos_, bsrSendOpEndSqePos_, bsrRecvOpBeginSqePos_,
2722 : bsrRecvOpEndSqePos_),
2723 : KfcError::kExecConstraint, HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR);
2724 :
2725 0 : ret = ((bsrRetryOp_ == HCCL_RECV) && (bsrRecvSqHead == bsrRecvOpEndSqePos_)) ? HCCL_E_OPRETRY_FAIL : HCCL_SUCCESS;
2726 0 : HCCL_RETRY_CHK_RET_AND_TRANS_FSM(ret,
2727 : HCCL_ERROR("[OpRetry][AICPU]hccl aicpu recv record complete task is complete, can not retry. params: send sq "
2728 : "head %u, recv sq head %u, send sq begin %u, send sq end %u, recv sq begin %u, recv sq end %u",
2729 : bsrSendSqHead, bsrRecvSqHead, bsrSendOpBeginSqePos_, bsrSendOpEndSqePos_, bsrRecvOpBeginSqePos_,
2730 : bsrRecvOpEndSqePos_),
2731 : KfcError::kExecConstraint, HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR);
2732 :
2733 0 : HCCL_RUN_INFO("[OpRetry][AICPU]hccl aicpu op is running, can retry. params: send sq head "
2734 : "%u, recv sq head %u, send sq begin %u, send sq end %u, recv sq begin %u, recv sq end %u",
2735 : bsrSendSqHead, bsrRecvSqHead, bsrSendOpBeginSqePos_, bsrSendOpEndSqePos_, bsrRecvOpBeginSqePos_,
2736 : bsrRecvOpEndSqePos_);
2737 0 : if (IsTaskExceptionForHccs()) {
2738 0 : HCCL_RUN_INFO("[OpRetry][AICPU]hccl aicpu stop by sdma/write task exception, can retry.");
2739 : }
2740 0 : errorCode = KfcError::kNone;
2741 0 : uint32_t retryCnt = (bsrRetryOp_ == HCCL_SEND) ? bsrSendRetryCnt_ : bsrRecvRetryCnt_;
2742 0 : CHK_RET(UpdateOpExecStatus(fsmState, KfcStatus::kStopExec, errorCode, retryCnt));
2743 0 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_FSM_WAIT_RETRY;
2744 0 : return HCCL_SUCCESS;
2745 : }
2746 :
2747 0 : HcclResult HcclCommAicpu::HcclOpExecFsmStoppedProcess(HcclOpExecFSM &fsmState, KfcError &errorCode, uint32_t retryCnt,
2748 : const std::string &algName, OpParam ¶m, uint32_t beginSqePos, uint32_t endSqePos)
2749 : {
2750 0 : HCCL_DEBUG("hccl aicpu stop exec.");
2751 0 : KfcCommand cmd = KfcCommand::kNone;
2752 0 : auto ret = aicpuHdc_.GetOpExecCtrlCmd(kfcControlTransferH2D_, cmd);
2753 0 : uint16_t rsErrorCode = 0;
2754 0 : if (ret != HCCL_SUCCESS) {
2755 0 : HCCL_ERROR("[OpRetry][AICPU]GetOpExecCtrlCmd failed, ret:%u", ret);
2756 0 : errorCode = KfcError::kExec;
2757 0 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR;
2758 0 : return ret;
2759 : }
2760 :
2761 0 : if (cmd == KfcCommand::kExit) {
2762 0 : HCCL_ERROR("[OpRetry][AICPU]hccl aicpu exec fsm stop by exit cmd.");
2763 0 : errorCode = KfcError::kExit;
2764 0 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR;
2765 0 : return HCCL_SUCCESS;
2766 : }
2767 :
2768 0 : if (cmd == KfcCommand::kReportRetryErr) {
2769 0 : HCCL_ERROR("[OpRetry][AICPU][HcclOpExecFsmStoppedProcess]hccl aicpu can not retry err cmd[%d]", cmd);
2770 0 : rsErrorCode = TS_ERROR_RETRY_CONSTRAINT;
2771 0 : CHK_PRT(SendTaskExceptionByMBox(rsErrorCode));
2772 0 : errorCode = KfcError::kExit;
2773 0 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR;
2774 0 : return HCCL_E_OPRETRY_FAIL;
2775 : }
2776 :
2777 0 : if (!HcclOpSupportRetry(algName, retryEnable_, param)) {
2778 0 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR;
2779 0 : if (param.isInplaceError) {
2780 0 : errorCode = KfcError::kExecConstraint;
2781 0 : rsErrorCode = TS_ERROR_RETRY_CONSTRAINT;
2782 0 : CHK_PRT(SendTaskExceptionByMBox(rsErrorCode));
2783 0 : HCCL_RETRY_CHK_RET_AND_TRANS_FSM(HCCL_E_OPRETRY_FAIL,
2784 : HCCL_ERROR("[Opretry][AICPU][HcclOpExecFsmStoppedProcess]can not retry for inpace error."),
2785 : KfcError::kExecConstraint, HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR);
2786 0 : } else if (isPollutedZeroCopyOp(param)) {
2787 0 : errorCode = KfcError::kExecConstraint;
2788 0 : rsErrorCode = TS_ERROR_RETRY_CONSTRAINT;
2789 0 : CHK_PRT(SendTaskExceptionByMBox(rsErrorCode));
2790 0 : HCCL_RETRY_CHK_RET_AND_TRANS_FSM(HCCL_E_OPRETRY_FAIL,
2791 : HCCL_ERROR("[Opretry][AICPU][HcclOpExecFsmStoppedProcess]can not retry for zero copy op."),
2792 : KfcError::kExecConstraint, HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR);
2793 : return HCCL_E_OPRETRY_FAIL;
2794 : } else {
2795 0 : errorCode = KfcError::kExec;
2796 : }
2797 0 : return HCCL_SUCCESS;
2798 : }
2799 :
2800 0 : uint32_t sqHead = 0xFFFFFFFF;
2801 0 : CHK_RET(QuerySqStatusByType(devId_, mainStream_.sqId(), DRV_SQCQ_PROP_SQ_HEAD, sqHead));
2802 0 : if (sqHead == endSqePos) {
2803 0 : rsErrorCode = TS_ERROR_RETRY_CONSTRAINT;
2804 0 : CHK_PRT(SendTaskExceptionByMBox(rsErrorCode));
2805 0 : HCCL_RETRY_CHK_RET_AND_TRANS_FSM(HCCL_E_OPRETRY_FAIL,
2806 : HCCL_ERROR("[OpRetry][AICPU]hccl aicpu record complete task is complete, can not retry. params: "
2807 : "sqHead %u, beginSqePos %u endSqePos %u", sqHead, beginSqePos, endSqePos),
2808 : KfcError::kExecConstraint, HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR);
2809 0 : } else if (sqHead == beginSqePos) {
2810 0 : rsErrorCode = TS_ERROR_RETRY_CONSTRAINT;
2811 0 : CHK_PRT(SendTaskExceptionByMBox(rsErrorCode));
2812 0 : HCCL_RETRY_CHK_RET_AND_TRANS_FSM(HCCL_E_OPRETRY_FAIL,
2813 : HCCL_ERROR("[OpRetry][AICPU]hccl aicpu wait start task is not complete, can not retry. "\
2814 : "params: sqHead %u, beginSqePos %u endSqePos %u", sqHead, beginSqePos, endSqePos),
2815 : KfcError::kExecConstraint, HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR);
2816 0 : } else if ((param.opType == HcclCMDType::HCCL_CMD_BATCH_SEND_RECV) && retryEnable_) {
2817 0 : CHK_RET(BSRStopedProcess(fsmState, errorCode));
2818 0 : } else {
2819 0 : HCCL_RUN_INFO("[OpRetry][AICPU]hccl aicpu op is running, can retry. params: sqHead %u, beginSqePos %u "
2820 : "endSqePos %u", sqHead, beginSqePos, endSqePos);
2821 0 : if (IsTaskExceptionForHccs()) {
2822 0 : HCCL_RUN_INFO("[OpRetry][AICPU]hccl aicpu stop by sdma/write task exception, can retry.");
2823 : }
2824 0 : errorCode = KfcError::kNone;
2825 0 : CHK_RET(UpdateOpExecStatus(fsmState, KfcStatus::kStopExec, errorCode, retryCnt));
2826 0 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_FSM_WAIT_RETRY;
2827 : }
2828 0 : return HCCL_SUCCESS;
2829 : }
2830 :
2831 0 : void HcclCommAicpu::NsCommStop()
2832 : {
2833 0 : if ((StreamsKill(devId_) != HCCL_SUCCESS) || (DeviceQuery(devId_, ts::APP_ABORT_KILL_FINISH, 0U) != HCCL_SUCCESS)) {
2834 0 : (void)aicpuHdc_.SetOpExecStatus(kfcStatusTransferD2H_, KfcStatus::kError, KfcError::kExec, 0);
2835 0 : HCCL_ERROR("[NsRecovery][AICPU]Stop failed");
2836 0 : return;
2837 : }
2838 : // 停止条件算子
2839 0 : if (isDeviceMode_) {
2840 0 : (void)InvokeKfcHandler(AicpuKfcHandlerType::kClearCommitTurn, {rpc_});
2841 : }
2842 0 : (void)aicpuHdc_.SetOpExecStatus(kfcStatusTransferD2H_, KfcStatus::kStopExec, KfcError::kNone, 0);
2843 0 : (void)HcclOneSideServiceAicpu::DisableAllStreamFunc();
2844 0 : HCCL_RUN_INFO("[NsRecovery][AICPU]stopFunc Finished");
2845 : }
2846 :
2847 0 : void HcclCommAicpu::NsCommClean()
2848 : {
2849 : // 等待drv任务停止
2850 0 : if ((DeviceQuery(devId_, ts::APP_ABORT_TERMINATE_FINISH, 0U) != HCCL_SUCCESS) ||
2851 0 : (CleanStreamFunc() != HCCL_SUCCESS) || (HcclOneSideServiceAicpu::CleanAllStreamFunc() != HCCL_SUCCESS) ||
2852 0 : (ResetSqBuff() != HCCL_SUCCESS)) {
2853 0 : (void)aicpuHdc_.SetOpExecStatus(kfcStatusTransferD2H_, KfcStatus::kError, KfcError::kExec, 0);
2854 0 : HCCL_ERROR("[NsRecovery][AICPU]stream terminate failed");
2855 0 : return;
2856 : } else {
2857 0 : if (isDeviceMode_) {
2858 0 : (void)InvokeKfcHandler(AicpuKfcHandlerType::kClearMsgArea, {rpc_});
2859 : }
2860 0 : HCCL_INFO("ClearFunc, after APP_ABORT_TERMINATE_FINISH");
2861 0 : dfxExtendInfo_.pollStatus = PollStatus::kDefault;
2862 0 : dfxExtendInfo_.cqeStatus = dfx::CqeStatus::kDefault;
2863 0 : (void)aicpuHdc_.SetOpExecStatus(kfcStatusTransferD2H_, KfcStatus::kClear, KfcError::kNone, 0);
2864 0 : endStopLaunch = false;
2865 0 : isOpLaunch = false;
2866 0 : needsResponseStopLaunch_ = false;
2867 0 : errMessageReport_ = true;
2868 0 : HCCL_RUN_INFO("[NsRecovery][AICPU] clean Finish");
2869 : }
2870 : }
2871 :
2872 0 : HcclResult HcclCommAicpu::GetBackGroundCommand(BackgroundCommand &bgCmd)
2873 : {
2874 0 : return AicpuHdcUtils::GetBackGroundCommand(kfcControlTransferH2D_, bgCmd);
2875 : }
2876 :
2877 0 : HcclResult HcclCommAicpu::ResponseBackGroundStatus(KfcExecStatus &status)
2878 : {
2879 0 : return AicpuHdcUtils::ResponseBackGroundStatus(kfcStatusTransferD2H_, status);
2880 : }
2881 :
2882 0 : HcclResult HcclCommAicpu::GetKfcCommand(KfcCommand &cmd)
2883 : {
2884 0 : return AicpuHdcUtils::GetKfcCommand(kfcControlTransferH2D_, cmd);
2885 : }
2886 :
2887 :
2888 0 : HcclResult HcclCommAicpu::SetStreamEnable(Stream &stream) {
2889 0 : const HcclComStreamInfo &streamInfo = stream.GetHcclStreamInfo();
2890 0 : HCCL_INFO("[SetStreamEnable] streamid[%d]", streamInfo.actualStreamId);
2891 0 : CHK_RET(ConfigSqStatusByType(GetDevId(), streamInfo.sqId, DRV_SQCQ_PROP_SQ_DISABLE_TO_ENABLE, 1));
2892 0 : HandleCqeException(stream, true);
2893 0 : return HCCL_SUCCESS;
2894 : }
2895 :
2896 0 : HcclResult HcclCommAicpu::CleanStreamFunc()
2897 : {
2898 0 : CHK_RET(SetStreamEnable(mainStream_));
2899 0 : for (auto &stream : slaveStreams_) {
2900 0 : CHK_RET(SetStreamEnable(stream));
2901 : }
2902 0 : CHK_RET(SetStreamEnable(orderStream_));
2903 0 : return HCCL_SUCCESS;
2904 : }
2905 :
2906 0 : std::string HcclCommAicpu::PrintInplaceStatus(u8 isInplaceStatus)
2907 : {
2908 0 : const u8 kNoOverlap = 0;
2909 0 : const u8 kAllToAllOverlap = 1;
2910 0 : const u8 kInplaceOverlap = 2;
2911 0 : switch (isInplaceStatus) {
2912 0 : case kNoOverlap:
2913 : // input和output不重叠
2914 0 : return "There is no overlap.";
2915 0 : case kAllToAllOverlap:
2916 : // alltoall类算子的input和output重叠
2917 0 : return "The param.inputPtr is equal to param.outputPtr, hence they overlap.";
2918 0 : case kInplaceOverlap:
2919 : // input和output重叠
2920 0 : return "It's inplace case. hence they overlap.";
2921 0 : default:
2922 0 : return "It's an unknown overlap case.";
2923 : }
2924 : return "";
2925 : }
2926 :
2927 0 : std::string HcclCommAicpu::PrintInplaceSupportRetryStatus(InplaceSupportRetryStatus inPlaceSupportRetryStatus)
2928 : {
2929 0 : switch (inPlaceSupportRetryStatus) {
2930 0 : case InplaceSupportRetryStatus::AG_BD_CASE: // 不需要去变成非DMA削减
2931 : // allgather or broadcast 算子
2932 0 : return "The Allgather or broadcast op supports inplace retry.";
2933 0 : case InplaceSupportRetryStatus::RETRY_1_ALLOW_NO_DMA_REDUCE_CASE1: // 需要去变成非DMA削减
2934 : // 使用AllReduceMeshSmallCountExecutor, ReduceScatterDeterExecutor
2935 : // 且环境变量配置RetryEnable:1
2936 0 : return "Since retryEnable:1, the executor without DMAReduce will be applied.";
2937 0 : case InplaceSupportRetryStatus::RETRY_0_NOT_ALLOW_NO_DMA_REDUCE_CASE1: // 不需要去变成非DMA削减
2938 : // 使用AllReduceMeshSmallCountExecutor, ReduceScatterDeterExecutor
2939 : // 且环境变量配置RetryEnable:0
2940 0 : return "Since retryEnable:0, ExecutorOnlySupportDMAReduce is not allowed for inplace case.";
2941 0 : case InplaceSupportRetryStatus::ALWAYS_NO_DMA_REDUCE: // 不需要去变成非DMA削减，本身就是
2942 : // 使用AllReduceComm/ReduceScatterComm
2943 0 : return "AllReduceComm or ReduceScatterComm is used for inplace case.";
2944 0 : case InplaceSupportRetryStatus::RETRY_1_ALLOW_NO_DMA_REDUCE_CASE2: // 需要去变成非DMA削减
2945 : // 使用其余在91093场景下使用的reduce scatter, allreduce executor
2946 : // 且环境变量配置RetryEnable:1
2947 0 : return "Since retryEnable:1, the executor will be applied without DMAReduce operation.";
2948 0 : case InplaceSupportRetryStatus::RETRY_0_NOT_ALLOW_NO_DMA_REDUCE_CASE2: // 不需要去变成非DMA削减
2949 : // 使用其余在91093场景下使用的reduce scatter, allreduce executor
2950 : // 且环境变量配置RetryEnable:0
2951 0 : return "Since retryEnable:0, the executor without DMAReduce operation can not be applied.";
2952 0 : case InplaceSupportRetryStatus::UNKONWN_EXECUTOR: // 不需要去变成非DMA削减
2953 : // 使用未知的executor
2954 0 : return "The unknown executor does not support for an inplace case yet.";
2955 0 : case InplaceSupportRetryStatus::USER_LARGER_THAN_CCL: // 不需要去变成非DMA削减
2956 : // UserInMem > CCLInMem 场景
2957 0 : return "UserInMem > CCLInMem case";
2958 0 : case InplaceSupportRetryStatus::NOT_BASIC_OP_CASE: // 不需要去变成非DMA削减
2959 : // 非 RS AR AG BD算子场景
2960 0 : return "Is not ReduceScatter, AllReduce, AllGather or Broadcast case";
2961 0 : default:
2962 0 : return "It's unknown case. They overlap.";
2963 : }
2964 : return "";
2965 : }
2966 :
2967 0 : bool HcclCommAicpu::IsNoNeedMonitor(void)
2968 : {
2969 0 : if (taskMonitorInterval_ == 0) return true;
2970 :
2971 0 : KfcCommand kfcCmd = KfcCommand::kNone;
2972 0 : (void)GetKfcCommand(kfcCmd);
2973 0 : if (kfcCmd != KfcCommand::kNone) return true;
2974 :
2975 0 : BackgroundCommand bgCmd = BackgroundCommand::kNone;
2976 0 : (void)GetBackGroundCommand(bgCmd);
2977 0 : if (bgCmd != BackgroundCommand::kNone) return true;
2978 :
2979 0 : HcclComSuspendingFlag suspendingFlag = HcclComSuspendingFlag::isNull;
2980 0 : (void)GetSuspendingFlag(suspendingFlag);
2981 0 : if (suspendingFlag != HcclComSuspendingFlag::isNull) return true;
2982 0 : return false;
2983 : }
2984 :
2985 0 : void HcclCommAicpu::InsertMonitorData(Stream &stream, HcclUs &curTime, u32 sqHead, uint16_t taskId, uint8_t type)
2986 : {
2987 0 : AicpuStreamMontior tmpTaskMonitor;
2988 0 : tmpTaskMonitor.historyTime = curTime;
2989 0 : tmpTaskMonitor.historyHead = sqHead;
2990 0 : tmpTaskMonitor.historyTaskId = taskId;
2991 0 : tmpTaskMonitor.historyType = type;
2992 0 : streamTaskMonitor_.insert(std::make_pair(stream.sqId(), tmpTaskMonitor));
2993 0 : return;
2994 : }
2995 :
2996 0 : bool HcclCommAicpu::IsNeedRefreshMonitorData(AicpuStreamMontior &streamMontior, HcclUs &curTime, uint32_t remoteRank,
2997 : uint16_t taskId, u32 sqHead, u32 sqTail, uint8_t type)
2998 : {
2999 0 : auto &historyTime = streamMontior.historyTime;
3000 0 : auto &historyHead = streamMontior.historyHead;
3001 0 : auto &historyTaskId = streamMontior.historyTaskId;
3002 0 : auto &historyType = streamMontior.historyType;
3003 0 : if((historyTaskId != taskId) || (sqHead != historyHead) || (sqHead == sqTail) || (historyType != type) ||
3004 0 : ((type == RT_STARS_SQE_TYPE_NOTIFY_WAIT) && (remoteRank == INVALID_VALUE_RANKID))) {
3005 0 : historyTime = curTime;
3006 0 : historyHead = sqHead;
3007 0 : historyTaskId = taskId;
3008 0 : historyType = type;
3009 0 : return true;
3010 : }
3011 0 : return false;
3012 : }
3013 :
3014 0 : HcclResult HcclCommAicpu::StreamTaskMonitor(void)
3015 : {
3016 : // 通信域资源已经释放
3017 0 : CHK_PRT_RET(!commOpenStatus,
3018 : HCCL_DEBUG("[StreamTaskMonitor]group[%s] has been destroyed", identifier_.c_str()), HCCL_SUCCESS);
3019 0 : if (IsNoNeedMonitor()) return HCCL_SUCCESS;
3020 0 : HCCL_DEBUG("StreamTaskMonitor print");
3021 0 : std::vector<Stream> totalStream = {mainStream_};
3022 0 : totalStream.insert(totalStream.end(), slaveStreams_.begin(), slaveStreams_.end());
3023 0 : HcclUs curTime = TIME_NOW();
3024 0 : for (auto &stream : totalStream) {
3025 0 : u32 sqHead = 0U, sqTail = 0U;
3026 0 : (void)QuerySqStatus(devId_, stream.sqId(), sqHead, sqTail);
3027 0 : HcclSqeContext *sqeContext = stream.GetSqeContextPtr();
3028 0 : CHK_PTR_NULL(sqeContext);
3029 0 : SqeRingBuffer *sqeContextBuffer = &(sqeContext->buffer);
3030 0 : CHK_PTR_NULL(sqeContextBuffer);
3031 :
3032 0 : uint8_t type = 0;
3033 0 : uint16_t taskId = 0;
3034 0 : uint32_t remoteRank = 0;
3035 0 : std::string tmp = GetTaskExceptionTaskInfo(sqHead, sqeContextBuffer, type, taskId, remoteRank);
3036 0 : HCCL_DEBUG("GetTaskExceptionTaskInfo type %u taskId %u", type, taskId);
3037 0 : auto mapIt = streamTaskMonitor_.find(stream.sqId());
3038 0 : if (mapIt == streamTaskMonitor_.end()) {
3039 0 : InsertMonitorData(stream, curTime, sqHead, taskId, type);
3040 0 : continue;
3041 : }
3042 :
3043 0 : auto &streamMontior = mapIt->second;
3044 0 : if (IsNeedRefreshMonitorData(streamMontior, curTime, remoteRank, taskId, sqHead, sqTail, type)) {
3045 0 : continue;
3046 : }
3047 :
3048 0 : auto timeVal = DURATION_US(curTime - streamMontior.historyTime).count();
3049 0 : const int TIME_CONVERSION = 1000;
3050 0 : if (timeVal >= taskMonitorInterval_ * TIME_CONVERSION) {
3051 0 : HCCL_RUN_INFO("[StreamTaskMonitor]prof monitor streamId:%d, sqid:%d, head:%u, tail:%u, time %s us, %s",
3052 : stream.id(), stream.sqId(), sqHead, sqTail, std::to_string(timeVal).c_str(), tmp.c_str());
3053 0 : HCCL_RUN_INFO("[StreamTaskMonitor]prof monitor %s", GetTaskExceptionOpInfo(sqHead,sqeContextBuffer).c_str());
3054 0 : PrintTaskExceptionTaskQue(sqHead, sqeContextBuffer, true);
3055 0 : streamMontior.historyTime = curTime;
3056 0 : streamMontior.historyHead = sqHead;
3057 0 : streamMontior.historyTaskId = taskId;
3058 0 : streamMontior.historyType = type;
3059 : }
3060 0 : }
3061 0 : return HCCL_SUCCESS;
3062 0 : }
3063 :
3064 0 : HcclResult HcclCommAicpu::SupportRetryWithInplaceCheck(const std::string &algName, OpParam ¶m)
3065 : {
3066 : // 不支持inplace的通信算子重执行
3067 0 : u8 isInplaceStatus = 0;
3068 0 : InplaceSupportRetryStatus inPlaceSupportRetryStatus = InplaceSupportRetryStatus::INPLACE_STATUS_END;
3069 0 : if (IsHcclOpInplace(param.opType, param, topoInfo_.userRank, topoInfo_.userRankSize, isInplaceStatus)) {
3070 0 : if(!FitRetryConditionforInPlaceOp(param.opType, param, algName, cclbufferSize_, topoInfo_.userRankSize,
3071 0 : algOpContext_.opRetryHandler.retryEnable,
3072 : inPlaceSupportRetryStatus)) {
3073 0 : HCCL_RUN_INFO("[OpRetry][AICPU]hccl supports inplace status: isInplaceStatus[%s], "
3074 : "opRetryHandler.inplaceSupportRetry[%d], opRetryHandler.inPlaceSupportRetryStatus[%s], "
3075 : "opRetryHandler.isInplacePreSync[%d], opRetryHandler.isPostSync[%d].",
3076 : PrintInplaceStatus(isInplaceStatus).c_str(), 0,
3077 : PrintInplaceSupportRetryStatus(inPlaceSupportRetryStatus).c_str(),
3078 : algOpContext_.opRetryHandler.isInplacePreSync, algOpContext_.opRetryHandler.isPostSync);
3079 : } else {
3080 0 : HCCL_RUN_INFO("[OpRetry][AICPU]hccl supports inplace status: isInplaceStatus[%s], "
3081 : "opRetryHandler.inplaceSupportRetry[%d], opRetryHandler.inPlaceSupportRetryStatus[%s], "
3082 : "opRetryHandler.isInplacePreSync[%d], opRetryHandler.isPostSync[%d].",
3083 : PrintInplaceStatus(isInplaceStatus).c_str(), 1,
3084 : PrintInplaceSupportRetryStatus(inPlaceSupportRetryStatus).c_str(),
3085 : algOpContext_.opRetryHandler.isInplacePreSync, algOpContext_.opRetryHandler.isPostSync);
3086 : }
3087 : } else {
3088 0 : HCCL_RUN_INFO("[OpRetry][AICPU]hccl supports inplace status: isInplaceStatus[%s], "
3089 : "opRetryHandler.isInplacePreSync[%d], opRetryHandler.isPostSync[%d].",
3090 : PrintInplaceStatus(isInplaceStatus).c_str(), algOpContext_.opRetryHandler.isInplacePreSync,
3091 : algOpContext_.opRetryHandler.isPostSync);
3092 : }
3093 0 : return HCCL_SUCCESS;
3094 : }
3095 :
3096 0 : bool HcclCommAicpu::HcclOpSupportRetry(const std::string &algName, bool retryEnable, OpParam ¶m)
3097 : {
3098 0 : HCCL_RUN_INFO("[OpRetry][AICPU]hccl supports retry status: enable[%u], param.tag[%s].",
3099 : retryEnable, param.tag.c_str());
3100 0 : if (!retryEnable) {
3101 0 : HCCL_ERROR("[OpRetry][AICPU]hccl aicpu can not retry, enable[%u].", retryEnable);
3102 0 : return false;
3103 : }
3104 0 : if (isPollutedZeroCopyOp(param)) {
3105 0 : HCCL_ERROR("[OpRetry][AICPU]hccl aicpu can not retry, isZeroCopy[%d], opType[%s].",
3106 : param.isZeroCopy, GetCMDTypeEnumStr(param.opType).c_str());
3107 0 : return false;
3108 : }
3109 :
3110 0 : CHK_RET(SupportRetryWithInplaceCheck(algName, param));
3111 : // 不支持inplace的通信算子重执行
3112 0 : if ((!algOpContext_.opRetryHandler.inplaceSupportRetry) && (!algOpContext_.opRetryHandler.isInplacePreSync) &&
3113 0 : (!algOpContext_.opRetryHandler.isPostSync)) {
3114 0 : HCCL_ERROR("[OpRetry][AICPU]hccl aicpu can not retry, not support inplace case, opType[%s], "
3115 : "inputPtr[0x%016lx], outputPtr[0x%016lx], opRetryHandler.inplaceSupportRetry[%d], "
3116 : "opRetryHandler.isInplacePreSync[%d], opRetryHandler.isPostSync[%d]",
3117 : GetCMDTypeEnumStr(param.opType).c_str(), param.inputPtr, param.outputPtr,
3118 : algOpContext_.opRetryHandler.inplaceSupportRetry,
3119 : algOpContext_.opRetryHandler.isInplacePreSync,
3120 : algOpContext_.opRetryHandler.isPostSync);
3121 0 : param.isInplaceError = true;
3122 0 : return false;
3123 : }
3124 :
3125 : // 不支持的通信算子重执行
3126 0 : if (HcclOpCheckSupportRetry(param.opType) == false) {
3127 0 : HCCL_ERROR("[OpRetry][AICPU]hccl aicpu can not retry, not support opType[%s].",
3128 : GetCMDTypeEnumStr(param.opType).c_str());
3129 0 : return false;
3130 : }
3131 0 : return true;
3132 : }
3133 :
3134 0 : bool HcclCommAicpu::isPollutedZeroCopyOp(OpParam ¶m)
3135 : {
3136 : // allreduce\reduce\reducescatter\reducescatterv with zerocopy can not support retry.
3137 0 : bool isPollutedOp = ((param.opType == HcclCMDType::HCCL_CMD_ALLREDUCE) ||
3138 0 : (param.opType == HcclCMDType::HCCL_CMD_REDUCE) ||
3139 0 : (param.opType == HcclCMDType::HCCL_CMD_REDUCE_SCATTER) ||
3140 0 : (param.opType == HcclCMDType::HCCL_CMD_REDUCE_SCATTER_V));
3141 0 : return param.isZeroCopy && isPollutedOp;
3142 : }
3143 :
3144 0 : HcclResult HcclCommAicpu::UpdateOpExecStatus(HcclOpExecFSM &fsmState, HcclOpIdentifier &opId, KfcStatus state,
3145 : KfcError &errorCode, uint32_t retryCnt)
3146 : {
3147 0 : HCCL_INFO("UpdateOpExecStatus fsmState %d, tag %s, index %u, state %d, errorCode %d, retryCnt %u.",
3148 : fsmState, opId.tag, opId.index, state, errorCode, retryCnt);
3149 0 : auto ret = aicpuHdc_.SetOpExecStatus(kfcStatusTransferD2H_, opId, state, errorCode, retryCnt);
3150 :
3151 0 : HCCL_RETRY_CHK_RET_AND_TRANS_FSM(ret, HCCL_ERROR("SetOpExecStatus failed, ret:%u", ret), KfcError::kExec,
3152 : HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR);
3153 :
3154 0 : return ret;
3155 : }
3156 :
3157 7 : HcclResult HcclCommAicpu::UpdateOpExecStatus(HcclOpExecFSM &fsmState, KfcStatus state, KfcError &errorCode,
3158 : uint32_t retryCnt)
3159 : {
3160 7 : HCCL_INFO("UpdateOpExecStatus fsmState %d, state %d, errorCode %d, retryCnt %u.",
3161 : fsmState, state, errorCode, retryCnt);
3162 7 : auto ret = aicpuHdc_.SetOpExecStatus(kfcStatusTransferD2H_, state, errorCode, retryCnt);
3163 7 : if (ret != HCCL_SUCCESS) {
3164 0 : HCCL_ERROR("SetOpExecStatus failed, ret:%u", ret);
3165 0 : errorCode = KfcError::kExec;
3166 0 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR;
3167 : }
3168 7 : return ret;
3169 : }
3170 :
3171 : static constexpr u32 HCCL_AICPU_WAIT_HOST_BASE_TIME_MS = 200 * 1000;
3172 : static constexpr u32 TIME_S_TO_MS = 1000;
3173 14 : u32 HcclCommAicpu::HcclGetWaitStopExecCmdTimeout()
3174 : {
3175 14 : return std::max(static_cast<u32>(linkTimeOut_.count()), HCCL_AICPU_WAIT_HOST_BASE_TIME_MS);
3176 : }
3177 :
3178 7 : u32 HcclCommAicpu::HcclGetWaitRetryCmdTimeout(uint32_t retryCnt)
3179 : {
3180 7 : if (retryCnt == 0) {
3181 7 : return HcclGetWaitStopExecCmdTimeout() + retryHoldTime_;
3182 : } else {
3183 0 : return HcclGetWaitStopExecCmdTimeout() + retryIntervalTime_;
3184 : }
3185 : }
3186 :
3187 0 : HcclResult HcclCommAicpu::HcclOpExecFsmWaitRetryProcess(const OpParam ¶m, HcclOpExecFSM &fsmState,
3188 : KfcError &errorCode, KfcCommand &lastCmd)
3189 : {
3190 0 : HCCL_DEBUG("hccl aicpu wait for retry cmd.");
3191 0 : KfcCommand cmd = KfcCommand::kNone;
3192 0 : auto ret = aicpuHdc_.GetOpExecCtrlCmd(kfcControlTransferH2D_, cmd);
3193 0 : if (ret != HCCL_SUCCESS) {
3194 0 : HCCL_ERROR("GetOpExecCtrlCmd failed, ret:%u", ret);
3195 0 : errorCode = KfcError::kExec;
3196 0 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR;
3197 0 : return ret;
3198 : }
3199 0 : if (cmd == KfcCommand::kRetry) {
3200 0 : HCCL_RUN_INFO("[OpRetry][AICPU]hccl aicpu recv retry cmd from host.");
3201 0 : dfxExtendInfo_.pollStatus = PollStatus::kDefault;
3202 0 : dfxExtendInfo_.cqeStatus = dfx::CqeStatus::kDefault;
3203 0 : ret = ResetOpRetryException(param.opType);
3204 0 : HCCL_RETRY_CHK_RET_AND_TRANS_FSM(ret, HCCL_ERROR("reset stream buff failed, ret:%u", ret), KfcError::kInner,
3205 : HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR);
3206 0 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_FSM_RETRY;
3207 0 : } else if (cmd == KfcCommand::kChangeLink && lastCmd != KfcCommand::kChangeLink) { // 防止重复执行
3208 0 : HCCL_RUN_INFO("[OpRetry][AICPU]hccl aicpu recv change link cmd, identify[%s]", identifier_.c_str());
3209 0 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_FSM_CHANGE_LINK;
3210 0 : } else if (cmd == KfcCommand::kExit) {
3211 0 : HCCL_RUN_INFO("[OpRetry][AICPU]hccl aicpu recv exit cmd from host.");
3212 0 : errorCode = KfcError::kExit;
3213 0 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR;
3214 0 : } else if (cmd == KfcCommand::kReportRetryErr) {
3215 0 : HCCL_RUN_INFO("[OpRetry][AICPU]hccl aicpu get report retry err cmd.");
3216 0 : CHK_PRT(SendTaskExceptionByMBox(TS_ERROR_RETRY_CONSTRAINT));
3217 0 : errorCode = KfcError::kExit;
3218 0 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR;
3219 0 : return HCCL_E_OPRETRY_FAIL;
3220 0 : } else if (cmd == KfcCommand::NsStopLaunch && endStopLaunch == false) {
3221 0 : HCCL_RUN_INFO("[NsRecovery][AICPU]hccl aicpu force stop in launch loop.");
3222 0 : endStopLaunch = true;
3223 0 : needsResponseStopLaunch_ = true;
3224 0 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_STOP_LAUNCH;
3225 : } else {
3226 : // do nothing
3227 : }
3228 0 : lastCmd = cmd;
3229 0 : return HCCL_SUCCESS;
3230 : }
3231 :
3232 2 : HcclResult HcclCommAicpu::ResetOpRetryException(HcclCMDType opType)
3233 : {
3234 2 : if (opType == HcclCMDType::HCCL_CMD_BATCH_SEND_RECV) {
3235 0 : CHK_RET(ResetBSRException());
3236 : } else {
3237 6 : std::vector<Stream> totalStream = {mainStream_};
3238 2 : totalStream.insert(totalStream.end(), slaveStreams_.begin(), slaveStreams_.end());
3239 4 : for (auto &stream : totalStream) {
3240 2 : CHK_RET(CleanStream(stream));
3241 2 : CHK_RET(ClearStreamCqeException(stream));
3242 : }
3243 2 : }
3244 2 : return HCCL_SUCCESS;
3245 2 : }
3246 :
3247 0 : HcclResult HcclCommAicpu::ResetSqBuff()
3248 : {
3249 0 : CHK_RET(CleanStream(mainStream_));
3250 0 : for (auto &stream : slaveStreams_) {
3251 0 : CHK_RET(CleanStream(stream));
3252 : }
3253 0 : CHK_RET(CleanStream(orderStream_));
3254 0 : HCCL_INFO("reset stream sq buffer success.");
3255 0 : return HCCL_SUCCESS;
3256 : }
3257 :
3258 0 : HcclResult HcclCommAicpu::UpdateSqStatus(Stream &stream)
3259 : {
3260 0 : HcclSqeContext *sqeContext = stream.GetSqeContextPtr();
3261 0 : CHK_PTR_NULL(sqeContext);
3262 0 : SqeRingBuffer *sqeContextBuffer = &(sqeContext->buffer);
3263 0 : CHK_PTR_NULL(sqeContextBuffer);
3264 0 : auto &head = sqeContextBuffer->sqHead;
3265 0 : auto &tail = sqeContextBuffer->sqTail;
3266 :
3267 0 : CHK_RET(QuerySqStatusByType(devId_, stream.sqId(), DRV_SQCQ_PROP_SQ_TAIL, head));
3268 0 : CHK_RET(QuerySqStatusByType(devId_, stream.sqId(), DRV_SQCQ_PROP_SQ_HEAD, tail));
3269 0 : HCCL_INFO("UpdateSqStatus, sqid:%u head:%u tail:%u.", stream.sqId(), head, tail);
3270 0 : return HCCL_SUCCESS;
3271 : }
3272 :
3273 0 : HcclResult HcclCommAicpu::HcclOpExecFsmRetryProcess(const std::string &algName, OpParam ¶m,
3274 : std::unique_ptr<CollExecutorBase> &executor, AlgResourceResponse &algResource, HcclOpExecFSM &fsmState,
3275 : KfcError &errorCode, uint32_t &retryCnt, uint32_t &beginSqePos, uint32_t &endSqePos)
3276 : {
3277 0 : retryCnt++;
3278 0 : if ((param.opType == HcclCMDType::HCCL_CMD_BATCH_SEND_RECV) && retryEnable_) {
3279 0 : UpdateBSRRetryCnt();
3280 : }
3281 0 : HCCL_RUN_INFO("[OpRetry][AICPU]retry launch start, retryCnt:%u, tag[%s].", retryCnt, param.tag.c_str());
3282 :
3283 0 : auto ret = RetryOrchestrateHcclOp(algName, param, executor, algResource, beginSqePos, endSqePos);
3284 0 : if (ret == HCCL_SUCCESS) {
3285 0 : errorCode = KfcError::kNone;
3286 0 : if ((param.opType == HcclCMDType::HCCL_CMD_BATCH_SEND_RECV) && retryEnable_) {
3287 : // 是否有前一个重执行阶段积累的故障未上报，需要再次触发重执行
3288 0 : CHK_RET(CommitBSRStoredException(fsmState, errorCode));
3289 0 : } else {
3290 0 : CHK_RET(UpdateOpExecStatus(fsmState, KfcStatus::kRuning, errorCode, retryCnt));
3291 0 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_FSM_WAIT_END;
3292 : }
3293 0 : } else if (ret == HCCL_E_SUSPENDING) {
3294 0 : HCCL_RUN_INFO("hccl aicpu force stop in retry launch process");
3295 0 : if ((param.opType == HcclCMDType::HCCL_CMD_BATCH_SEND_RECV) && retryEnable_) {
3296 : // batchsendrecv算子下发过程中出现异常，task下发未完成，send 和 recv 均需要重执行
3297 0 : if (bsrRetryOp_ == HCCL_SEND){
3298 0 : SetBSRSendOpExecException();
3299 : } else {
3300 0 : SetBSRRecvOpExecException();
3301 : }
3302 0 : HCCL_RUN_INFO("hccl aicpu abort launch batchsendrecv op, need retry.");
3303 : }
3304 0 : CHK_RET(UpdateSuspendStatus(param, fsmState, errorCode, retryCnt));
3305 : } else {
3306 0 : HCCL_ERROR("RetryLaunchHcclOp failed, ret:%u", ret);
3307 0 : errorCode = KfcError::kInner;
3308 0 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR;
3309 : }
3310 0 : return ret;
3311 : }
3312 :
3313 0 : HcclResult HcclCommAicpu::HcclOpExecFsmEndProcess(uint32_t retryCnt)
3314 : {
3315 : auto ret =
3316 0 : aicpuHdc_.SetOpExecStatus(kfcStatusTransferD2H_, excuteOpId_, KfcStatus::kEnd, KfcError::kNone, retryCnt);
3317 0 : if (!isDeviceMode_) {
3318 0 : isOpLaunch = false;
3319 : }
3320 0 : dfxExtendInfo_.kfcStatus = DfxKfcStatus::kOneFinished;
3321 0 : HCCL_DEBUG("---------- end AICPU_HcclOpExecFsmEndProcess ----------");
3322 0 : return ret;
3323 : }
3324 :
3325 0 : HcclResult HcclCommAicpu::PrintTaskExceptionAllThreads()
3326 : {
3327 : // 非独立算子场景，跳过
3328 0 : CHK_PRT_RET(!GetIsInitIndOp(),
3329 : HCCL_RUN_INFO("[%s] IndOp group[%s] not init, skip", __func__, identifier_.c_str()), HCCL_SUCCESS);
3330 :
3331 0 : for (auto &thread : threads_) {
3332 0 : CHK_RET(taskExecption_.PrintTaskException(*(thread->GetStream())));
3333 : }
3334 0 : return HCCL_SUCCESS;
3335 : }
3336 :
3337 0 : void HcclCommAicpu::PrintTaskExceptionAllComm()
3338 : {
3339 0 : std::shared_lock<std::shared_mutex> rwlock(AicpuHcclProcess::AicpuGetCommMutex());
3340 :
3341 : // 先打印本通信域的taskException
3342 0 : (void)PrintTaskExceptionAllStreams();
3343 0 : (void)PrintTaskExceptionAllThreads();
3344 :
3345 : // 再打印其他通信域的taskException
3346 0 : std::vector<std::pair<std::string, hccl::HcclCommAicpu *>> aicpuCommInfo;
3347 0 : (void)AicpuHcclProcess::AicpuGetCommAll(aicpuCommInfo);
3348 0 : for (auto &commInfo : aicpuCommInfo) {
3349 0 : hccl::HcclCommAicpu *hcclAicpu = commInfo.second;
3350 0 : if (hcclAicpu == nullptr || hcclAicpu->identifier_ == identifier_) {
3351 0 : continue;
3352 : }
3353 0 : (void)hcclAicpu->PrintTaskExceptionAllThreads();
3354 0 : (void)hcclAicpu->PrintTaskExceptionAllStreams();
3355 : }
3356 0 : }
3357 :
3358 0 : void HcclCommAicpu::PrintAicpuCommExecStatus()
3359 : {
3360 0 : std::shared_lock<std::shared_mutex> rwlock(AicpuHcclProcess::AicpuGetCommMutex());
3361 :
3362 : // 记录通信域占核情况
3363 0 : int64_t inExecGroupNum = 0;
3364 0 : int64_t aicpuCoreNum = 0;
3365 0 : (void)hrtHalGetDeviceInfo(devId_, MODULE_TYPE_AICPU, INFO_TYPE_CORE_NUM, &aicpuCoreNum);
3366 :
3367 0 : std::vector<std::pair<std::string, hccl::HcclCommAicpu *>> aicpuCommInfo;
3368 0 : (void)AicpuHcclProcess::AicpuGetCommAll(aicpuCommInfo);
3369 0 : for (auto &commInfo : aicpuCommInfo) {
3370 0 : hccl::HcclCommAicpu *hcclAicpu = commInfo.second;
3371 0 : if (hcclAicpu == nullptr || !hcclAicpu->GetCommInfoStatus()) {
3372 0 : continue;
3373 : }
3374 :
3375 : // 获取并打印通信域是否在执行中，以及最后一次下发的算子
3376 0 : bool isInExec = AicpuHcclProcess::GetCommExecStatus(hcclAicpu->identifier_);
3377 0 : inExecGroupNum += isInExec ? 1 : 0;
3378 0 : std::string execStatus = isInExec ? "inExec" : "unExec";
3379 0 : HCCL_RUN_INFO("AicpuComm: group[%s], status[%s], op[%s], aicpuCoreNum[%lld]",
3380 : hcclAicpu->identifier_.c_str(), execStatus.c_str(), hcclAicpu->GetExcuteOp().c_str(), aicpuCoreNum);
3381 0 : }
3382 :
3383 : // AICPU核被占满，部分通信域得不到调度，可能导致通信阻塞，打印维测信息
3384 0 : if (inExecGroupNum >= aicpuCoreNum && static_cast<int64_t>(aicpuCommInfo.size()) > aicpuCoreNum) {
3385 0 : HCCL_RUN_WARNING("In Execution group num[%lld], total group num[%u], bigger than Aicpu cores num[%lld]. "
3386 : "Aicpu core being fully utilized may cause tasks to get stuck, and it is necessary to reduce the num of comm.",
3387 : inExecGroupNum, aicpuCommInfo.size(), aicpuCoreNum);
3388 : }
3389 0 : }
3390 :
3391 0 : HcclResult HcclCommAicpu::PrintTaskExceptionAllStreams()
3392 : {
3393 : // 通信域资源已经释放
3394 0 : CHK_PRT_RET(!commOpenStatus,
3395 : HCCL_RUN_INFO("[PrintTaskExceptionAllStreams]group[%s] has been destroyed", identifier_.c_str()), HCCL_SUCCESS);
3396 0 : CHK_RET(UtraceInfo_->Flush());
3397 0 : std::vector<Stream> totalStream = {mainStream_};
3398 : HcclResult ret;
3399 0 : totalStream.insert(totalStream.end(), slaveStreams_.begin(), slaveStreams_.end());
3400 0 : for (auto &stream : totalStream) {
3401 0 : HCCL_RUN_INFO("[PrintTaskExceptionAllStreams]group[%s] streamid[%d] print", identifier_.c_str(), stream.id());
3402 0 : u32 sqHead = 0U;
3403 0 : u32 sqTail = 0U;
3404 0 : ret = QuerySqStatus(devId_, stream.sqId(), sqHead, sqTail);
3405 0 : if (ret != HCCL_SUCCESS || sqHead == sqTail) { // 此流为空时，不打印
3406 0 : HCCL_RUN_INFO("[PrintTaskExceptionAllStreams] group[%s] streamid[%d] is empty"
3407 : "or QuerySqStatus failed, ret[%d]", identifier_.c_str(), stream.id(), ret);
3408 0 : continue;
3409 : }
3410 0 : HcclSqeContext *sqeContext = stream.GetSqeContextPtr();
3411 0 : SqeRingBuffer *sqeContextBuffer = &(sqeContext->buffer);
3412 0 : CHK_PTR_NULL(sqeContextBuffer);
3413 0 : if (stream.id() == mainStream_.id()) {
3414 0 : SqeInfo sqeInfo;
3415 0 : ret = SqeContextUtils::QuerySqeInfo(sqeContextBuffer->rtsMirrorBuffer + sqHead * HCCL_SQE_SIZE,
3416 0 : sqeContextBuffer->rtsqSqeType[sqHead], sqeContextBuffer->addInfo[sqHead], &sqeInfo);
3417 0 : if (ret != HCCL_SUCCESS) {
3418 0 : HCCL_ERROR("[%s]QuerySqeInfo failed, ret[%d]", __func__, ret);
3419 : } else {
3420 : // 根据主流卡在host notify上，则说明未被执行到不打印
3421 0 : if (sqeInfo.type == RT_STARS_SQE_TYPE_NOTIFY_WAIT && sqeInfo.notifyId == opNotifies_[0]->notifyId_) {
3422 0 : HCCL_RUN_INFO("[PrintTaskExceptionAllStreams] group[%s] op is not activated, do nothing",
3423 : identifier_.c_str());
3424 0 : return HCCL_SUCCESS;
3425 : }
3426 : // 根据主流当前位置，判断该算子是否已经打印过taskException
3427 0 : if (IsRepeatedOpTaskException(sqHead, sqeContextBuffer)) {
3428 0 : HCCL_INFO("[PrintTaskExceptionAllStreams] group[%s] op has been printed, do nothing",
3429 : identifier_.c_str());
3430 0 : return HCCL_SUCCESS;
3431 : }
3432 : }
3433 : }
3434 :
3435 0 : uint8_t type = 0;
3436 0 : uint16_t taskId = 0;
3437 0 : uint32_t remoteRank = 0;
3438 0 : HCCL_ERROR("[TaskException]base information is streamId:%d, sqid:%d, head:%u, tail:%u, %s",
3439 : stream.id(), stream.sqId(), sqHead, sqTail,
3440 : GetTaskExceptionTaskInfo(sqHead, sqeContextBuffer, type, taskId, remoteRank).c_str());
3441 0 : PrintTaskExceptionTaskQue(sqHead, sqeContextBuffer);
3442 : }
3443 0 : return HCCL_SUCCESS;
3444 0 : }
3445 :
3446 0 : bool HcclCommAicpu::IsRepeatedOpTaskException(u32 idx, SqeRingBuffer *sqeContextBuffer)
3447 : {
3448 0 : const AicpuOpInfo *opInfo = aicpuShareData_.GetAicpuOpInfo(sqeContextBuffer->rtsDfxInfo[idx].opRingBufferIdx);
3449 0 : CHK_PRT_RET(opInfo == nullptr, HCCL_ERROR("%s fail, opInfo is nullptr", __func__), false);
3450 0 : std::string opTag = opInfo->tagBuff;
3451 0 : u32 opIndex = opInfo->opIndex;
3452 0 : bool opHasPrinted = opTaskException_.find(opTag) != opTaskException_.end() && opTaskException_[opTag] == opIndex;
3453 0 : opTaskException_[opTag] = opIndex;
3454 0 : CHK_PRT_CONT(opHasPrinted, HCCL_RUN_INFO("[IsRepeatedOpTaskException]group[%s], op[%s], opIndex[%u] "\
3455 : "has been printed", identifier_.c_str(), opInfo->tagBuff, opInfo->opIndex));
3456 0 : return opHasPrinted;
3457 0 : }
3458 :
3459 0 : void HcclCommAicpu::PrepareMc2Handler()
3460 : {
3461 0 : auto &handler = algOpContext_.mc2Handler;
3462 0 : handler.stepSize = 0U;
3463 0 : if (!isDeviceMode_) {
3464 0 : HCCL_INFO("Unset step size for non-MC2.");
3465 0 : return;
3466 : }
3467 0 : (void)InvokeKfcHandler(AicpuKfcHandlerType::kSetStepSize, {rpc_, reinterpret_cast<u64>(&handler), GetRankSize()});
3468 : }
3469 :
3470 0 : HcclResult HcclCommAicpu::OrchestrateHcclOp(const std::string &algName, OpParam ¶m,
3471 : std::unique_ptr<CollExecutorBase> &executor, AlgResourceResponse &algResource, uint32_t &beginSqePos,
3472 : uint32_t &endSqePos)
3473 : {
3474 0 : LogControl logControl(false, false); // 重执行ERROR日志控制，析构时重置日志设置
3475 0 : PrepareMc2Handler();
3476 0 : HcclResult ret = HCCL_SUCCESS;
3477 : // task的尾指针，已便重执行stop时判断是否已执行该task，如果该task已执行完成则可支持通信重执行
3478 0 : CHK_RET(QuerySqStatusByType(devId_, mainStream_.sqId(), DRV_SQCQ_PROP_SQ_TAIL, beginSqePos));
3479 :
3480 0 : const bool retryForBatchSndRcv = (param.opType == HcclCMDType::HCCL_CMD_BATCH_SEND_RECV && retryEnable_);
3481 0 : if (retryForBatchSndRcv) {
3482 0 : CHK_RET(QueryBatchSendRecvPairBeginPos());
3483 0 : if (param.BatchSendRecvDataDes.curIterNum == 0) {
3484 : // batchsendrecv算子拆分为多轮执行，只有第一个step和最后一个step需要和主stream交互
3485 0 : CHK_RET(NotifyWait());
3486 : }
3487 0 : HCCL_INFO("batch send recv op: step %u, mode:%u", param.BatchSendRecvDataDes.curIterNum,
3488 : param.BatchSendRecvDataDes.curMode);
3489 : } else {
3490 0 : CHK_RET(NotifyWait());
3491 : // 重执行场景, 算子计数在host侧; MC2场景也不开启卡住检测能力
3492 0 : if (opCounterInfo_.isEnableCounter && !retryEnable_ && !isDeviceMode_) {
3493 0 : CHK_RET(HcclReduceAsync(dispatcher_, reinterpret_cast<void *>(opCounterInfo_.addOneMem), opCounterInfo_.memSize / sizeof(int32_t),
3494 : HCCL_DATA_TYPE_INT32, HCCL_REDUCE_SUM, mainStream_, reinterpret_cast<void *>(opCounterInfo_.headCountMem), INVALID_VALUE_RANKID,
3495 : LinkType::LINK_ONCHIP, INLINE_REDUCE_BIT));
3496 : }
3497 : }
3498 :
3499 : // 打印当前展开的算子信息
3500 0 : HCCL_INFO("[HcclCommAicpu][OrchestrateHcclOp] opUnfoldIdx_[%u] opType[%d] curRank[%u] rankSize[%u] algName[%s]",
3501 : opUnfoldIdx_, param.opType, topoInfo_.userRank, GetRankSize(), algName.c_str());
3502 0 : HCCL_INFO("[HcclCommAicpu][OrchestrateHcclOp] inputPtr[0x%016llx] inputSize[%llu] outputPtr[0x%016llx] outputSize[%llu]",
3503 : param.inputPtr, param.inputSize, param.outputPtr, param.outputSize);
3504 0 : opUnfoldIdx_ += 1;
3505 :
3506 : // 检查算子展开的动态缓存, 确认是否可以跳过算子展开
3507 0 : bool needExecute = true;
3508 0 : bool isCacheMiss = false;
3509 0 : auto setProfStartCallback = [this](){
3510 0 : return this->InvokeKfcHandler(AicpuKfcHandlerType::kSetProfTimeStart, {});
3511 0 : };
3512 0 : CHK_RET(aicpuCacheManager_.LookupOpUnfoldCache(algName, param, algResource, needExecute, isCacheMiss,
3513 : mainStream_, slaveStreams_, dispatcher_, isDeviceMode_, topoInfo_, topoMatcher_, algOpContext_,
3514 : ZeroCopyExchanger_, GetWorkflowMode(), tinySendRecvMem_, setProfStartCallback));
3515 :
3516 : // 根据needExecute有条件的执行算子展开
3517 : // 需要算子执行的场景: (i) expansion mode为AI_CPU_NO_CACHE; (ii) uncacheable算子/场景; (iii) cache miss
3518 0 : if (needExecute) {
3519 : // Cache miss前执行cache相关的预处理
3520 0 : if (isCacheMiss) {
3521 0 : CHK_RET(aicpuCacheManager_.PreProcessForCacheMiss(param, executor));
3522 : }
3523 :
3524 : // executor设置AlgOpContext
3525 0 : CHK_RET(executor->SetAlgOpContext(algOpContext_));
3526 0 : (void)InvokeKfcHandler(AicpuKfcHandlerType::kSetProfTimeStart, {});
3527 0 : ret = executor->Orchestrate(param, algResource);
3528 0 : if (ret != HCCL_SUCCESS) {
3529 0 : HCCL_ERROR("[HcclCommAicpu][Orchestrate]executor process failed algName[%s], ret = %u", algName.c_str(), ret);
3530 0 : printTaskExceptionForErr_ |= (ret == HCCL_E_AGAIN);
3531 0 : return ret;
3532 : }
3533 :
3534 : // Cache miss后执行cache相关的后处理
3535 0 : if (isCacheMiss) {
3536 0 : CHK_RET(aicpuCacheManager_.PostProcessForCacheMiss(param, executor, mainStream_, slaveStreams_, dispatcher_,
3537 : topoInfo_, algOpContext_, GetWorkflowMode()));
3538 : }
3539 : } // 正常算子展开
3540 :
3541 : // batchsendrecv算子拆分为多轮执行，只有第一个step和最后一个step需要和主stream交互
3542 0 : if (!retryForBatchSndRcv || param.BatchSendRecvDataDes.curIterNum + 1 >= bsrSendRecvPairs_.size()) {
3543 : // 重执行场景, 算子计数在host侧 MC2场景也不开
3544 0 : if (opCounterInfo_.isEnableCounter && !retryEnable_ && !isDeviceMode_) {
3545 0 : CHK_RET(HcclReduceAsync(dispatcher_, reinterpret_cast<void *>(opCounterInfo_.addOneMem), opCounterInfo_.memSize / sizeof(int32_t),
3546 : HCCL_DATA_TYPE_INT32, HCCL_REDUCE_SUM, mainStream_, reinterpret_cast<void *>(opCounterInfo_.tailCountMem), INVALID_VALUE_RANKID,
3547 : LinkType::LINK_ONCHIP, INLINE_REDUCE_BIT));
3548 : }
3549 0 : CHK_RET(NotifyPost());
3550 : }
3551 0 : (void)InvokeKfcHandler(AicpuKfcHandlerType::kSetProfTimeOrch, {});
3552 0 : ret = LaunchTask(dispatcher_, const_cast<Stream &>(mainStream_));
3553 0 : if (ret != HCCL_SUCCESS) {
3554 0 : HCCL_ERROR("[HcclCommAicpu][LaunchTask]algName[%s] ret = %u", algName.c_str(), ret);
3555 0 : printTaskExceptionForErr_ |= (ret == HCCL_E_AGAIN);
3556 0 : return ret;
3557 : }
3558 0 : ret = LaunchSlaveStreamTask(algResource);
3559 0 : if (ret != HCCL_SUCCESS) {
3560 0 : HCCL_ERROR("[HcclCommAicpu][LaunchSlaveStreamTask]algName[%s] ret = %u", algName.c_str(), ret);
3561 0 : printTaskExceptionForErr_ |= (ret == HCCL_E_AGAIN);
3562 0 : return ret;
3563 : }
3564 0 : (void)InvokeKfcHandler(AicpuKfcHandlerType::kSetProfTimeEnd, {});
3565 0 : if (retryForBatchSndRcv) {
3566 0 : CHK_RET(QueryBatchSendRecvPairEndPos());
3567 : }
3568 :
3569 0 : CHK_RET(QuerySqStatusByType(devId_, mainStream_.sqId(), DRV_SQCQ_PROP_SQ_TAIL, endSqePos));
3570 :
3571 0 : HCCL_INFO("hccl aicpu launch hccl op task success. stream sqid:%u begin:%u end:%u",
3572 : mainStream_.sqId(), beginSqePos, endSqePos);
3573 0 : return HCCL_SUCCESS;
3574 0 : }
3575 :
3576 0 : HcclResult HcclCommAicpu::RetryOrchestrateHcclOp(const std::string &algName, OpParam ¶m,
3577 : std::unique_ptr<CollExecutorBase> &executor, AlgResourceResponse &algResource, uint32_t &beginSqePos,
3578 : uint32_t &endSqePos)
3579 : {
3580 0 : LogControl logControl(false, false); // 重执行ERROR日志控制，析构时重置日志设置
3581 0 : if ((param.opType == HcclCMDType::HCCL_CMD_BATCH_SEND_RECV) && retryEnable_) {
3582 0 : param.BatchSendRecvDataDes.curMode =
3583 0 : (bsrRetryOp_ == HCCL_SEND) ? BatchSendRecvCurMode::SEND : BatchSendRecvCurMode::RECV;
3584 0 : if (param.BatchSendRecvDataDes.curMode == BatchSendRecvCurMode::SEND) {
3585 0 : HCCL_INFO("BSR: iter %u, retry send op tag:%s index:%u", param.BatchSendRecvDataDes.curIterNum,
3586 : bsrSendOpId_.tag, bsrSendOpId_.index);
3587 : } else {
3588 0 : HCCL_INFO("BSR: iter %u, retry recv op tag:%s index:%u", param.BatchSendRecvDataDes.curIterNum,
3589 : bsrRecvOpId_.tag, bsrRecvOpId_.index);
3590 : }
3591 : }
3592 :
3593 0 : CHK_RET(AddRetryExecFlipTask(algResource));
3594 0 : HcclResult ret = executor->Orchestrate(param, algResource);
3595 0 : if (ret != HCCL_SUCCESS) {
3596 0 : HCCL_ERROR("[HcclCommAicpu][Orchestrate]executor process failed algName[%s]", algName.c_str());
3597 0 : return ret;
3598 : }
3599 :
3600 0 : if ((param.opType == HcclCMDType::HCCL_CMD_BATCH_SEND_RECV) && retryEnable_) {
3601 : // batchsendrecv算子重执行时，aicpu 主stream没有clean，不需要重新下发notify record
3602 : // do nothing
3603 : } else {
3604 0 : CHK_RET(NotifyPost());
3605 : }
3606 0 : CHK_RET(LaunchTask(dispatcher_, const_cast<Stream &>(mainStream_)));
3607 0 : CHK_RET(LaunchSlaveStreamTask(algResource));
3608 :
3609 0 : CHK_RET(QuerySqStatusByType(devId_, mainStream_.sqId(), DRV_SQCQ_PROP_SQ_TAIL, endSqePos));
3610 0 : if ((param.opType == HcclCMDType::HCCL_CMD_BATCH_SEND_RECV) && retryEnable_) {
3611 0 : CHK_RET(QueryBatchSendRecvPairEndPos());
3612 : }
3613 :
3614 0 : HCCL_RUN_INFO("[OpRetry][AICPU]hccl aicpu retry launch hccl op task success. stream sqid:%u begin:%u end:%u",
3615 : mainStream_.sqId(), beginSqePos, endSqePos);
3616 0 : return HCCL_SUCCESS;
3617 0 : }
3618 :
3619 4 : bool HcclCommAicpu::IsTaskExceptionForHccs()
3620 : {
3621 4 : if (dfxExtendInfo_.cqeStatus != dfx::CqeStatus::kCqeException) {
3622 4 : return false;
3623 : }
3624 :
3625 : // NOTE: 需要task exception补全dfx能力，定位故障task的remote rank; 目前暂不具备识别是否跨片的能力，默认失败的task均为跨片操作。
3626 0 : if (dfxExtendInfo_.cqeException.sqeType == RT_STARS_SQE_TYPE_SDMA &&
3627 0 : (dfxExtendInfo_.cqeException.errorCode == RT_SDMA_COMPDATAERR ||
3628 0 : dfxExtendInfo_.cqeException.errorCode == RT_SDMA_COMPERR)) {
3629 0 : return true;
3630 : }
3631 0 : return false;
3632 : }
3633 :
3634 3 : void HcclCommAicpu::SetAlgType(u64 algType)
3635 : {
3636 3 : algType_.algoLevel0 = static_cast<AlgTypeLevel0>(static_cast<u32>(algType) & ((1 << HCCL_LEVEL_ALGO_WIDTH) - 1));
3637 3 : algType_.algoLevel1 = static_cast<AlgTypeLevel1>((static_cast<u32>(algType) >>
3638 3 : HCCL_LEVEL_ALGO_WIDTH) & ((1 << HCCL_LEVEL_ALGO_WIDTH) - 1));
3639 3 : algType_.algoLevel2 = static_cast<AlgTypeLevel2>(static_cast<u32>(algType) >> (HCCL_LEVEL_ALGO_WIDTH + HCCL_LEVEL_ALGO_WIDTH));
3640 3 : HCCL_INFO("[HcclCommAicpu][SetAlgType]algType:%u", algType);
3641 3 : }
3642 :
3643 0 : void HcclCommAicpu::SetDebugMode(u8 debugMode)
3644 : {
3645 0 : debugMode_ = debugMode;
3646 0 : }
3647 :
3648 0 : void HcclCommAicpu::SetSendRecvInfoPtr(void* sendRecvInfoPtr)
3649 : {
3650 0 : sendRecvInfoPtr_ = sendRecvInfoPtr;
3651 0 : }
3652 :
3653 4 : bool HcclCommAicpu::IsNoNeedWait(void)
3654 : {
3655 4 : return isDeviceMode_ || (debugMode_ != MC2_DEBUG_WAIT_COMM && retryEnable_ == false);
3656 : }
3657 :
3658 45 : bool HcclCommAicpu::GetOpRetryEnable()
3659 : {
3660 45 : return retryEnable_;
3661 : }
3662 :
3663 4 : HcclResult HcclCommAicpu::ReportHcclTaskInfo(Stream &mainStream, std::vector<Stream> &subStreams)
3664 : {
3665 4 : if (dfx::ProfilingManager::GetProfL1State()) {
3666 0 : CHK_RET(dfx::ProfilingManager::ReportTaskInfo(mainStream.id(), mainStream.GetSqeContextPtr()));
3667 0 : for (auto& subStream : subStreams) {
3668 0 : CHK_RET(dfx::ProfilingManager::ReportTaskInfo(subStream.id(), subStream.GetSqeContextPtr()));
3669 : }
3670 : }
3671 4 : return HCCL_SUCCESS;
3672 : }
3673 :
3674 4 : HcclResult HcclCommAicpu::ClearLocalBuff(Stream &mainStream, std::vector<Stream> &subStreams)
3675 : {
3676 4 : CHK_RET(mainStream.ClearLocalBuff());
3677 4 : CHK_RET(dfx::ProfilingManager::UpdateStartReportSqeIdx(mainStream.id(), 0));
3678 4 : for (auto &subStream : subStreams) {
3679 0 : CHK_RET(subStream.ClearLocalBuff());
3680 0 : CHK_RET(dfx::ProfilingManager::UpdateStartReportSqeIdx(subStream.id(), 0));
3681 : }
3682 4 : return HCCL_SUCCESS;
3683 : }
3684 :
3685 4 : HcclResult HcclCommAicpu::WaitFinishWhileLoop(Stream &mainStream, std::vector<Stream> &subStreams,
3686 : std::string &tag, const uint32_t &beginSqePos, OpParam ¶m)
3687 : {
3688 : // 上报Profiling HCCL INFO信息
3689 4 : CHK_RET(ReportHcclTaskInfo(mainStream, subStreams));
3690 4 : CHK_RET(ClearLocalBuff(mainStream, subStreams));
3691 4 : if (IsNoNeedWait()) {
3692 0 : return HCCL_SUCCESS;
3693 : }
3694 4 : const uint64_t startUsec = GetCurCpuTimestamp();
3695 4 : uint64_t lastUsec = startUsec;
3696 4 : int32_t sqId = mainStream.sqId();
3697 4 : uint32_t sqHead = 0;
3698 4 : uint32_t sqTail = 0;
3699 4 : CHK_RET(QuerySqStatusByType(devId_, sqId, DRV_SQCQ_PROP_SQ_TAIL, sqTail));
3700 4 : CHK_RET(QuerySqStatusByType(devId_, sqId, DRV_SQCQ_PROP_SQ_HEAD, sqHead));
3701 4 : while (sqHead != sqTail) {
3702 4 : HcclResult ret = CheckOpExecStatus(); // 检查执行状态，判断是否有异常cq或中断命令
3703 4 : CHK_PRT_RET(ret != HCCL_SUCCESS,
3704 : HCCL_RUN_INFO("[HcclCommAicpu][WaitFinishWhileLoop]CheckOpExecStatus exception, ret[%u]", ret), ret);
3705 :
3706 0 : CHK_RET(QuerySqStatusByType(devId_, sqId, DRV_SQCQ_PROP_SQ_HEAD, sqHead));
3707 0 : if ((param.opType == HcclCMDType::HCCL_CMD_BATCH_SEND_RECV) &&
3708 0 : (retryEnable_) && (sqHead != beginSqePos) && (!excuteOpId_.isBsrTaskStart)) {
3709 : //更新D2H通道里的信息opid中isBsrTaskStart = true
3710 0 : excuteOpId_.isBsrTaskStart = true;
3711 0 : HcclResult ret1 = aicpuHdc_.SetOpExecStatus(kfcStatusTransferD2H_, excuteOpId_, KfcStatus::kRuning,
3712 : KfcError::kNone, 0);
3713 0 : CHK_PRT_RET(ret1 != HCCL_SUCCESS, HCCL_ERROR("update OpExecStatus failed, ret:%u", ret1), ret1);
3714 0 : HCCL_INFO("[HcclCommAicpu][WaitFinishWhileLoop]bsr start task is completed. devId:%d sqid:%d, head:%u,"
3715 : "beginSqePos[%u] group[%s] tag[%s]",
3716 : devId_, sqId, sqHead, beginSqePos, identifier_.c_str(), tag.c_str());
3717 : }
3718 0 : uint64_t curUsec = GetCurCpuTimestamp();
3719 0 : if (curUsec - lastUsec > static_cast<uint64_t>(NSEC_PER_SEC) * dfx::kPrintSqInterval) {
3720 0 : lastUsec = curUsec;
3721 0 : HCCL_RUN_INFO("[HcclCommAicpu][WaitFinishWhileLoop]Current state. devId:%d sqid:%d, head:%u, tail:%u, "
3722 : "group[%s] tag[%s]", devId_, sqId, sqHead, sqTail, identifier_.c_str(), tag.c_str());
3723 : }
3724 0 : CHK_RET(CheckTaskTimeout(mainStream, startUsec));
3725 : }
3726 0 : return HCCL_SUCCESS;
3727 : }
3728 :
3729 0 : HcclResult HcclCommAicpu::CheckTaskTimeout(const Stream &mainStream, const uint64_t startUsec)
3730 : {
3731 0 : if (sqeWaitTimeOut_ != 0 && (GetCurCpuTimestamp() -
3732 0 : startUsec > static_cast<uint64_t>(NSEC_PER_SEC) * sqeWaitTimeOut_)) {
3733 0 : uint32_t status = 0U;
3734 0 : int32_t sqId = mainStream.sqId();
3735 0 : auto ret = QuerySqStatusByType(devId_, sqId, DRV_SQCQ_PROP_SQ_CQE_STATUS, status);
3736 0 : if (ret != 0) {
3737 0 : HCCL_ERROR(
3738 : "[HcclCommAicpu]QuerySqStatusByType status failed. ret = %u sqid:%d", ret, sqId);
3739 : }
3740 :
3741 0 : HCCL_ERROR("[HcclCommAicpu]KFC timeout.. group[%s].", identifier_.c_str());
3742 0 : printTaskExceptionForErr_ = true;
3743 0 : return HCCL_E_TIMEOUT;
3744 : }
3745 0 : return HCCL_SUCCESS;
3746 : }
3747 :
3748 0 : HcclResult HcclCommAicpu::AddRetryExecFlipTask(AlgResourceResponse &algResource)
3749 : {
3750 0 : CHK_RET(AddRetryPreamble(dispatcher_, mainStream_));
3751 0 : for (u32 i = 0; i < algResource.slaveStreams.size(); ++i) {
3752 0 : HcclResult ret = AddRetryPreamble(dispatcher_, algResource.slaveStreams[i]);
3753 0 : if (ret != HCCL_SUCCESS) {
3754 0 : HCCL_ERROR("[HcclCommAicpu][RetryOrchestrateHcclOp] launch place holder failed, sqid:%u, ret:%u",
3755 : algResource.slaveStreams[i].sqId(), ret);
3756 0 : return ret;
3757 : }
3758 : }
3759 0 : return HCCL_SUCCESS;
3760 : }
3761 :
3762 2 : HcclResult HcclCommAicpu::LaunchSlaveStreamTask(AlgResourceResponse &algResource)
3763 : {
3764 : // 单算子模式中在算法编排中已经执行过LaunchTask，所以这里不需要再执行
3765 : // 只有图模式需要再额外执行一次对从流中的task下发
3766 2 : if (GetWorkflowMode() != HcclWorkflowMode::HCCL_WORKFLOW_MODE_OPS_KERNEL_INFO_LIB) {
3767 1 : HCCL_INFO("[HcclCommAicpu][LaunchSlaveStreamTask] op base mode don't need launch slave stream task");
3768 1 : return HCCL_SUCCESS;
3769 : }
3770 :
3771 1 : for (u32 i = 0; i < algResource.slaveStreams.size(); ++i) {
3772 0 : HcclResult ret = LaunchTask(dispatcher_, algResource.slaveStreams[i]);
3773 0 : if (ret != HCCL_SUCCESS) {
3774 0 : HCCL_ERROR("[HcclCommAicpu][LaunchSlaveStreamTask] launch task failed, sqid:%u, ret:%u",
3775 : algResource.slaveStreams[i].sqId(), ret);
3776 0 : return ret;
3777 : }
3778 : }
3779 :
3780 1 : return HCCL_SUCCESS;
3781 : }
3782 :
3783 0 : HcclResult HcclCommAicpu::GetAlltoAllvSendRecvInfo(const void* sendRecvInfoPtr, HcclDataType sendType,
3784 : HcclDataType recvType)
3785 : {
3786 0 : allMeshAggregationSendRecvInfo_.clear();
3787 0 : u64 stepSize = sizeof(u64) * topoInfo_.userRankSize;
3788 0 : const u32 addrItemNum = 4;
3789 0 : const u32 recvLengthStep = 2;
3790 0 : const u32 recvOffsetStep = 3;
3791 0 : for (u32 i = 0; i < topoInfo_.userRankSize; i++) {
3792 0 : SendRecvInfo sendRecvInfo;
3793 0 : sendRecvInfo.sendLength.resize(topoInfo_.userRankSize);
3794 0 : sendRecvInfo.sendOffset.resize(topoInfo_.userRankSize);
3795 0 : sendRecvInfo.recvLength.resize(topoInfo_.userRankSize);
3796 0 : sendRecvInfo.recvOffset.resize(topoInfo_.userRankSize);
3797 0 : CHK_SAFETY_FUNC_RET(memcpy_s(sendRecvInfo.sendLength.data(),
3798 : stepSize,
3799 : static_cast<const u8 *>(sendRecvInfoPtr) + i * stepSize * addrItemNum + 0 * stepSize,
3800 : stepSize));
3801 0 : CHK_SAFETY_FUNC_RET(memcpy_s(sendRecvInfo.sendOffset.data(),
3802 : stepSize,
3803 : static_cast<const u8 *>(sendRecvInfoPtr) + i * stepSize * addrItemNum + stepSize,
3804 : stepSize));
3805 0 : CHK_SAFETY_FUNC_RET(memcpy_s(sendRecvInfo.recvLength.data(),
3806 : stepSize,
3807 : static_cast<const u8 *>(sendRecvInfoPtr) + i * stepSize * addrItemNum + recvLengthStep * stepSize,
3808 : stepSize));
3809 0 : CHK_SAFETY_FUNC_RET(memcpy_s(sendRecvInfo.recvOffset.data(),
3810 : stepSize,
3811 : static_cast<const u8 *>(sendRecvInfoPtr) + i * stepSize * addrItemNum + recvOffsetStep * stepSize,
3812 : stepSize));
3813 0 : allMeshAggregationSendRecvInfo_.push_back(std::move(sendRecvInfo));
3814 0 : }
3815 :
3816 0 : for (auto &sendRecvInfo : allMeshAggregationSendRecvInfo_) {
3817 0 : for (u32 i = 0; i < topoInfo_.userRankSize; i++) {
3818 0 : sendRecvInfo.sendCounts.push_back(sendRecvInfo.sendLength[i] / SIZE_TABLE[sendType]);
3819 0 : sendRecvInfo.sendDispls.push_back(sendRecvInfo.sendOffset[i] / SIZE_TABLE[sendType]);
3820 0 : sendRecvInfo.recvCounts.push_back(sendRecvInfo.recvLength[i] / SIZE_TABLE[recvType]);
3821 0 : sendRecvInfo.recvDispls.push_back(sendRecvInfo.recvOffset[i] / SIZE_TABLE[recvType]);
3822 0 : HCCL_INFO("[GetAlltoAllvSendRecvInfo] rank[%u], sendCounts[%llu], sendDispls[%llu], "\
3823 : "recvCounts[%llu], recvDispls[%llu]", i, sendRecvInfo.sendCounts[i], sendRecvInfo.sendDispls[i],
3824 : sendRecvInfo.recvCounts[i], sendRecvInfo.recvDispls[i]);
3825 0 : HCCL_INFO("[GetAlltoAllvSendRecvInfo] rank[%u], sendLength[%llu], sendOffset[%llu], "\
3826 : "recvLength[%llu], recvOffset[%llu]", i, sendRecvInfo.sendLength[i], sendRecvInfo.sendOffset[i],
3827 : sendRecvInfo.recvLength[i], sendRecvInfo.recvOffset[i]);
3828 : }
3829 : }
3830 0 : CHK_RET(CheckSendRecvParams(allMeshAggregationSendRecvInfo_));
3831 0 : return HCCL_SUCCESS;
3832 : }
3833 :
3834 0 : HcclResult HcclCommAicpu::GetAlltoAllvcSendRecvInfo(const void *sendCountMatrix, HcclDataType sendType,
3835 : HcclDataType recvType)
3836 : {
3837 0 : allMeshAggregationSendRecvInfo_.clear();
3838 0 : for (u32 i = 0; i < topoInfo_.userRankSize; i++) {
3839 0 : SendRecvInfo sendRecvInfo;
3840 0 : sendRecvInfo.sendCounts.resize(topoInfo_.userRankSize);
3841 0 : sendRecvInfo.sendDispls.resize(topoInfo_.userRankSize);
3842 0 : sendRecvInfo.sendLength.resize(topoInfo_.userRankSize);
3843 0 : sendRecvInfo.sendOffset.resize(topoInfo_.userRankSize);
3844 0 : u64 curSendDispls = 0;
3845 0 : u64 curSendOffset = 0;
3846 0 : sendRecvInfo.recvCounts.resize(topoInfo_.userRankSize);
3847 0 : sendRecvInfo.recvDispls.resize(topoInfo_.userRankSize);
3848 0 : sendRecvInfo.recvLength.resize(topoInfo_.userRankSize);
3849 0 : sendRecvInfo.recvOffset.resize(topoInfo_.userRankSize);
3850 0 : u64 curRecvDispls = 0;
3851 0 : u64 curRecvOffset = 0;
3852 0 : for (u32 j = 0; j < topoInfo_.userRankSize; j++) {
3853 0 : u64 curSendCounts = *(static_cast<const u64 *>(sendCountMatrix) + i * topoInfo_.userRankSize + j);
3854 0 : u64 curSendLength = curSendCounts * SIZE_TABLE[sendType];
3855 0 : sendRecvInfo.sendCounts[j] = curSendCounts;
3856 0 : sendRecvInfo.sendDispls[j] = curSendDispls;
3857 0 : sendRecvInfo.sendLength[j] = curSendLength;
3858 0 : sendRecvInfo.sendOffset[j] = curSendOffset;
3859 0 : curSendDispls += curSendCounts;
3860 0 : curSendOffset += curSendLength;
3861 0 : u64 curRecvCounts = *(static_cast<const u64 *>(sendCountMatrix) + i + topoInfo_.userRankSize * j);
3862 0 : u64 curRecvLength = curRecvCounts * SIZE_TABLE[recvType];
3863 0 : sendRecvInfo.recvCounts[j] = curRecvCounts;
3864 0 : sendRecvInfo.recvDispls[j] = curRecvDispls;
3865 0 : sendRecvInfo.recvLength[j] = curRecvLength;
3866 0 : sendRecvInfo.recvOffset[j] = curRecvOffset;
3867 0 : curRecvDispls += curRecvCounts;
3868 0 : curRecvOffset += curRecvLength;
3869 0 : HCCL_DEBUG("GetAlltoAllvcSendRecvInfo rank[%u], sendCounts[%llu], sendDispls[%llu] "\
3870 : "recvCounts[%llu], recvDispls[%llu]", i, sendRecvInfo.sendCounts[j], sendRecvInfo.sendDispls[j],
3871 : sendRecvInfo.recvCounts[j], sendRecvInfo.recvDispls[j]);
3872 : }
3873 0 : allMeshAggregationSendRecvInfo_.push_back(sendRecvInfo);
3874 0 : }
3875 0 : CHK_RET(CheckSendRecvParams(allMeshAggregationSendRecvInfo_));
3876 0 : return HCCL_SUCCESS;
3877 : }
3878 :
3879 0 : HcclResult HcclCommAicpu::CheckSendRecvParams(const std::vector<SendRecvInfo> &allMeshAggregationSendRecvInfo)
3880 : {
3881 0 : u32 rankSize = allMeshAggregationSendRecvInfo.size();
3882 0 : for (u32 i = 0; i < rankSize; i++) {
3883 0 : u32 sendsSize = allMeshAggregationSendRecvInfo[i].sendLength.size();
3884 0 : u32 recvsSize = allMeshAggregationSendRecvInfo[i].recvLength.size();
3885 0 : if (rankSize != sendsSize || rankSize != recvsSize) {
3886 0 : HCCL_ERROR(
3887 : "[AlltoAllV][CheckSendRecvParam] rankSize[%u], sendsSize[%u], recvsSize[%u] are not match Index[%u]",
3888 : rankSize, sendsSize, recvsSize, i);
3889 0 : return HCCL_E_PARA;
3890 : }
3891 0 : for (u32 j = 0; j < sendsSize; j++) {
3892 0 : if (allMeshAggregationSendRecvInfo[i].sendLength[j] != allMeshAggregationSendRecvInfo[j].recvLength[i]) {
3893 0 : HCCL_ERROR("SendLength[%u][%u]: %llu and recvLength[%u][%u]: %llu are not match", i, j,
3894 : allMeshAggregationSendRecvInfo[i].sendLength[j], j, i,
3895 : allMeshAggregationSendRecvInfo[j].recvLength[i]);
3896 0 : return HCCL_E_PARA;
3897 : }
3898 : }
3899 : }
3900 0 : return HCCL_SUCCESS;
3901 : }
3902 0 : HcclResult HcclCommAicpu::GetStreamAll(std::vector<Stream> &streams)
3903 : {
3904 0 : streams.assign(slaveStreams_.begin(), slaveStreams_.end());
3905 0 : streams.push_back(mainStream_);
3906 0 : return HCCL_SUCCESS;
3907 : }
3908 :
3909 : // 校验是否有ERROR CQE和停止/退出命令，注册到dispatcher层调用
3910 0 : HcclResult HcclCommAicpu::CheckOpExecStatusCallback()
3911 : {
3912 0 : HcclResult ret = CheckOpExecStatus();
3913 0 : bool logLevel = (ret == HCCL_E_SUSPENDING);
3914 : // 返回HCCL_E_SUSPENDING时，需要跨作用域修改ERROR日志->RUN_WARNING
3915 0 : LogControl(logLevel, logLevel);
3916 0 : return ret;
3917 : }
3918 :
3919 4 : HcclResult HcclCommAicpu::CheckOpExecStatus()
3920 : {
3921 : // 检测是否有ERROR CQE
3922 4 : if (dfxExtendInfo_.pollStatus == PollStatus::kStopAsException) {
3923 0 : if (IsTaskExceptionForHccs() && retryEnable_) {
3924 0 : HCCL_RUN_INFO("[OpRetry][AICPU]hccl aicpu stop wait task exec finish, for task exception, identify[%s]",
3925 : identifier_.c_str());
3926 0 : return HCCL_E_SUSPENDING;
3927 : } else {
3928 0 : if (!printTaskExceptionForErr_) {
3929 0 : printTaskExceptionForErr_ = true;
3930 0 : HCCL_ERROR("hccl aicpu exec failed, for task exception, identify[%s], cqeStatus[%d], sqeType[%u], "
3931 : "errorCode[%u]", identifier_.c_str(), dfxExtendInfo_.cqeStatus, dfxExtendInfo_.cqeException.sqeType,
3932 : dfxExtendInfo_.cqeException.errorCode);
3933 : }
3934 0 : return HCCL_E_INTERNAL;
3935 : }
3936 : }
3937 :
3938 : // 检测是否有停止/退出命令
3939 4 : KfcCommand cmd = KfcCommand::kNone;
3940 4 : CHK_RET(aicpuHdc_.GetOpExecCtrlCmd(kfcControlTransferH2D_, cmd));
3941 4 : if (cmd == KfcCommand::kStopLaunch && retryEnable_) {
3942 4 : HCCL_RUN_INFO("[OpRetry][AICPU]hccl aicpu stop wait finish, for recv stop launch cmd, identify[%s]",
3943 : identifier_.c_str());
3944 4 : return HCCL_E_SUSPENDING;
3945 0 : } else if ((cmd == KfcCommand::NsStopLaunch) && (endStopLaunch == false)) {
3946 0 : needsResponseStopLaunch_ = true;
3947 0 : endStopLaunch = true;
3948 0 : HCCL_RUN_INFO("hccl aicpu stop wait finish, for recv stop launch cmd");
3949 0 : return HCCL_E_SUSPENDING;
3950 0 : } else if (cmd == KfcCommand::kDestroyComm) {
3951 0 : HCCL_ERROR("hccl aicpu stop wait finish, for recv destroy comm cmd");
3952 0 : return HCCL_E_INTERNAL;
3953 0 : } else if (cmd == KfcCommand::kExit) {
3954 0 : HCCL_ERROR("hccl aicpu stop wait finish, for recv exit cmd, identify[%s]", identifier_.c_str());
3955 0 : return HCCL_E_INTERNAL;
3956 : }
3957 0 : return HCCL_SUCCESS;
3958 : }
3959 :
3960 4 : HcclResult HcclCommAicpu::UpdateSuspendStatus(const OpParam ¶m, HcclOpExecFSM &fsmState, KfcError &errorCode,
3961 : uint32_t retryCnt)
3962 : {
3963 4 : if (needsResponseStopLaunch_ == true) {
3964 0 : HCCL_RUN_INFO("[NsRecovery][AICPU]hccl aicpu force stop in launch loop");
3965 0 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_STOP_LAUNCH;
3966 4 : } else if (retryEnable_) {
3967 4 : HCCL_RUN_INFO("[OpRetry][AICPU]hccl aicpu force stop for stop cmd or recoverable task exception, identify[%s]",
3968 : identifier_.c_str());
3969 4 : errorCode = IsTaskExceptionForHccs() ? KfcError::kSdma : errorCode;
3970 :
3971 4 : if ((param.opType == HcclCMDType::HCCL_CMD_BATCH_SEND_RECV)) {
3972 0 : HcclResult ret = GetBSRRetryOpId(param, bsrTargetOpId_);
3973 0 : HCCL_RETRY_CHK_RET_AND_TRANS_FSM(ret, HCCL_ERROR("get batchsendrecv target op failed, ret:%u", ret),
3974 : KfcError::kExec, HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR);
3975 0 : uint32_t bsrRetryCnt = (bsrRetryOp_ == HCCL_SEND) ? bsrSendRetryCnt_ : bsrRecvRetryCnt_;
3976 0 : bsrTargetOpId_.isBsrTaskStart = true;
3977 0 : ret = aicpuHdc_.SetOpExecStatus(kfcStatusTransferD2H_, bsrTargetOpId_, KfcStatus::kStoplaunch, errorCode,
3978 : bsrRetryCnt);
3979 0 : HCCL_RETRY_CHK_RET_AND_TRANS_FSM(ret, HCCL_ERROR("SetOpExecStatus failed, ret:%u", ret), KfcError::kExec,
3980 : HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR);
3981 : } else {
3982 4 : CHK_RET(UpdateOpExecStatus(fsmState, KfcStatus::kStoplaunch, errorCode, retryCnt));
3983 : }
3984 4 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_FSM_STOPPING;
3985 : } else {
3986 0 : HCCL_RUN_INFO("[HcclCommAicpu][UpdateSuspendStatus] aicpu force stop in launch loop; needsResponseStopLaunch_[%u] retryEnable_[%u]",
3987 : needsResponseStopLaunch_, retryEnable_);
3988 0 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_STOP_LAUNCH;
3989 : }
3990 4 : return HCCL_SUCCESS;
3991 : }
3992 :
3993 0 : HcclResult HcclCommAicpu::TasktypeTransferD2H(const uint8_t sqeType, TaskType &taskType)
3994 : {
3995 0 : switch (sqeType) {
3996 0 : case RT_STARS_SQE_TYPE_PLACE_HOLDER:
3997 : case RT_STARS_SQE_TYPE_NOTIFY_WAIT:
3998 0 : taskType = TaskType::TASK_NOTIFY_WAIT;
3999 0 : break;
4000 0 : case RT_STARS_SQE_TYPE_SDMA:
4001 0 : taskType = TaskType::TASK_SDMA;
4002 0 : break;
4003 0 : case RT_STARS_SQE_TYPE_NOTIFY_RECORD:
4004 0 : taskType = TaskType::TASK_NOTIFY_RECORD;
4005 0 : break;
4006 0 : case RT_STARS_SQE_TYPE_WRITE_VALUE:
4007 0 : taskType = TaskType::TASK_NOTIFY_WAIT;
4008 0 : break;
4009 0 : default:
4010 0 : HCCL_ERROR("TasktypeTransferD2H sqeType[%d] error.", sqeType);
4011 0 : return HCCL_E_PARA;
4012 : }
4013 0 : return HCCL_SUCCESS;
4014 : }
4015 :
4016 0 : HcclResult HcclCommAicpu::GenTaskExceptionInfo(u8 sqeType, hccl::Stream &stream, u32 head)
4017 : {
4018 0 : HcclSqeContext *sqeContext = stream.GetSqeContextPtr();
4019 0 : SqeRingBuffer *sqeContextBuffer = &(sqeContext->buffer);
4020 0 : CHK_PTR_NULL(sqeContextBuffer);
4021 :
4022 0 : const AicpuOpInfo *opInfo = aicpuShareData_.GetAicpuOpInfo(sqeContextBuffer->rtsDfxInfo[head].opRingBufferIdx);
4023 0 : std::string opTag = opInfo == nullptr ? "unKnown" : opInfo->tagBuff;
4024 :
4025 : // 获取需要上报的关键信息
4026 0 : ErrorMessageReport emrInfo{};
4027 0 : SqeInfo sqeInfo;
4028 0 : SqeContextUtils::QuerySqeInfo(sqeContextBuffer->rtsMirrorBuffer + head * HCCL_SQE_SIZE,
4029 0 : sqeContextBuffer->rtsqSqeType[head], sqeContextBuffer->addInfo[head], &sqeInfo);
4030 0 : emrInfo.remoteUserRank = sqeContextBuffer->rtsDfxInfo[head].remoteRank;
4031 0 : emrInfo.streamId = stream.id();
4032 0 : emrInfo.taskId = sqeInfo.taskId;
4033 0 : emrInfo.notifyId = sqeInfo.notifyId;
4034 0 : emrInfo.rankId = localUserRank_;
4035 0 : emrInfo.rankSize = topoInfo_.userRankSize;
4036 0 : emrInfo.algType = algType_;
4037 0 : emrInfo.opIndex = opInfo == nullptr ? 0 : opInfo->opIndex;
4038 0 : emrInfo.count = opInfo == nullptr ? 0 : opInfo->count;
4039 0 : emrInfo.dataType = opInfo == nullptr ? 0 : opInfo->dataType;
4040 0 : emrInfo.dstAddr = opInfo == nullptr ? 0 : opInfo->dstAddr;
4041 0 : emrInfo.srcAddr = opInfo == nullptr ? 0 : opInfo->srcAddr;
4042 0 : emrInfo.reduceType = opInfo == nullptr ? 255 : opInfo->reduceType; // 255 为 HcclReduceOp::HCCL_REDUCE_RESERVED
4043 0 : CHK_RET(TasktypeTransferD2H(sqeType, emrInfo.taskType));
4044 :
4045 0 : CHK_SAFETY_FUNC_RET(memcpy_s(emrInfo.tag, sizeof(emrInfo.tag), opTag.c_str(), opTag.size()));
4046 0 : CHK_SAFETY_FUNC_RET(memcpy_s(emrInfo.group, sizeof(emrInfo.group), identifier_.c_str(), identifier_.size()));
4047 0 : CHK_RET(aicpuHdc_.SetErrorMessage(kfcStatusTransferD2H_, emrInfo));
4048 0 : return HCCL_SUCCESS;
4049 0 : }
4050 :
4051 0 : HcclResult HcclCommAicpu::PrintTaskExceptionByTaskId(u8 sqeType, u16 taskId, hccl::Stream &stream, u32 tail)
4052 : {
4053 0 : HcclSqeContext *sqeContext = stream.GetSqeContextPtr();
4054 0 : CHK_PTR_NULL(sqeContext);
4055 0 : HCCL_ERROR("[HcclCommAicpu][PrintTaskExceptionByTaskId]streamId:%d tail:%u cqeType:%u", stream.id(), tail,
4056 : sqeType);
4057 0 : SqeRingBuffer *sqeContextBuffer = &(sqeContext->buffer);
4058 0 : CHK_PTR_NULL(sqeContextBuffer);
4059 0 : uint8_t *sqeMirrorBufferAddr = sqeContextBuffer->rtsMirrorBuffer + (tail - 1) * HCCL_SQE_SIZE;
4060 0 : rtStarsSqeHeader_t * const sqeHeader = (rtStarsSqeHeader_t * const)sqeMirrorBufferAddr;
4061 :
4062 0 : s32 taskNum = sqeHeader->taskId - taskId;
4063 0 : HCCL_DEBUG("[HcclCommAicpu]tail sqe taskId[%u] cqe taskId[%u] cqe type[%u]", sqeHeader->taskId,
4064 : taskId, sqeType);
4065 0 : s32 sqeIdx = tail - taskNum - 1;
4066 0 : u32 sqHead = (sqeIdx + HCCL_SQE_MAX_CNT) % HCCL_SQE_MAX_CNT;
4067 0 : uint8_t type = 0;
4068 0 : uint16_t taskIdTmp = 0;
4069 0 : uint32_t remoteRank = 0;
4070 0 : HCCL_ERROR("[TaskException][AICPU]base information is streamId:%d, sqid:%d, head:%u, tail:%u, %s",
4071 : stream.id(), stream.sqId(), sqHead, tail,
4072 : GetTaskExceptionTaskInfo(sqHead, sqeContextBuffer, type, taskIdTmp, remoteRank).c_str());
4073 0 : PrintTaskExceptionTaskQue(sqHead, sqeContextBuffer);
4074 0 : return HCCL_SUCCESS;
4075 : }
4076 :
4077 0 : std::string HcclCommAicpu::GetTaskExceptionOpInfo(u32 idx, SqeRingBuffer *sqeContextBuffer)
4078 : {
4079 0 : const AicpuOpInfo *opInfo = aicpuShareData_.GetAicpuOpInfo(sqeContextBuffer->rtsDfxInfo[idx].opRingBufferIdx);
4080 0 : CHK_PRT_RET(opInfo == nullptr, HCCL_ERROR("%s fail, opInfo is nullptr", __func__), "unKnown");
4081 :
4082 0 : std::stringstream ss;
4083 0 : ss << "tag:" << opInfo->tagBuff << ", ";
4084 0 : ss << "group:" << identifier_ << ", ";
4085 0 : ss << "isCustom:" << opInfo->isCustom << ", ";
4086 0 : ss << "opLaunchIdx:" << opInfo->opIndex << ", ";
4087 0 : ss << "opExecIdx:" << opInfo->opExecIndex << ", ";
4088 0 : ss << "count:" << opInfo->count << ", ";
4089 0 : ss << "dataType:" << static_cast<u16>(opInfo->dataType) << ", ";
4090 0 : ss << "opType:" << static_cast<u16>(opInfo->opType) << ", ";
4091 0 : ss << "rootId:" << opInfo->rootId << ", ";
4092 0 : ss << "dstAddr:0x" << std::hex << opInfo->dstAddr << ", ";
4093 0 : ss << "srcAddr:0x" << std::hex << opInfo->srcAddr << ".";
4094 0 : return ss.str();
4095 0 : }
4096 :
4097 0 : std::string HcclCommAicpu::GetTaskExceptionTaskInfo(u32 sqHead, SqeRingBuffer *sqeContextBuffer, uint8_t &type,
4098 : uint16_t &taskId, uint32_t &remoteRank)
4099 : {
4100 0 : SqeInfo sqeInfo;
4101 0 : SqeContextUtils::QuerySqeInfo(sqeContextBuffer->rtsMirrorBuffer + sqHead * HCCL_SQE_SIZE,
4102 0 : sqeContextBuffer->rtsqSqeType[sqHead], sqeContextBuffer->addInfo[sqHead], &sqeInfo);
4103 0 : type = sqeInfo.type;
4104 0 : taskId = sqeInfo.taskId;
4105 0 : remoteRank = sqeContextBuffer->rtsDfxInfo[sqHead].remoteRank;
4106 0 : std::stringstream ss;
4107 0 : ss << "type:" << SqeContextUtils::RtsqTaskTypeToStr(sqeInfo.type) << ", ";
4108 0 : ss << "localRank:" << localUserRank_ << ", ";
4109 0 : ss << "remoteRank:" << remoteRank << ", ";
4110 0 : ss << "taskId:" << sqeInfo.taskId << ", ";
4111 0 : ss << "notifyId:" << sqeInfo.notifyId << ", ";
4112 0 : ss << "length:" << sqeInfo.length << ", ";
4113 0 : ss << "addr1High:0x" << std::hex << sqeInfo.addr1High << ", ";
4114 0 : ss << "addr1Low:0x" << std::hex << sqeInfo.addr1Low << ", ";
4115 0 : ss << "addr2High:0x" << std::hex << sqeInfo.addr2High << ", ";
4116 0 : ss << "addr2Low:0x" << std::hex << sqeInfo.addr2Low << ".";
4117 0 : return ss.str();
4118 0 : }
4119 :
4120 : // 获取指定索引的算子信息
4121 0 : const AicpuOpInfo* HcclCommAicpu::GetOpInfoFromSqIdx(u32 sqIdx, SqeRingBuffer *sqeContextBuffer)
4122 : {
4123 0 : return aicpuShareData_.GetAicpuOpInfo(sqeContextBuffer->rtsDfxInfo[sqIdx].opRingBufferIdx);
4124 : }
4125 :
4126 : // 打印算子数据信息
4127 0 : void HcclCommAicpu::PrintOpDataInfo(u32 sqIdx, SqeRingBuffer *sqeContextBuffer, bool isMonitor)
4128 : {
4129 0 : std::string opInfo = GetTaskExceptionOpInfo(sqIdx, sqeContextBuffer);
4130 0 : if (isMonitor) {
4131 0 : HCCL_RUN_INFO("[StreamTaskMonitor]opData information is %s", opInfo.c_str());
4132 : } else {
4133 0 : HCCL_ERROR("[TaskException]opData information is %s", opInfo.c_str());
4134 : }
4135 0 : }
4136 :
4137 : // 打印task序列行
4138 0 : void HcclCommAicpu::PrintTaskLine(bool isMonitor, u32 lineNum, u32 totalPrinted,
4139 : const std::string& taskLine) const
4140 : {
4141 0 : if (isMonitor) {
4142 0 : HCCL_RUN_INFO("[StreamTaskMonitor]task sequence[%u/%u] is %s",
4143 : lineNum, totalPrinted, taskLine.c_str());
4144 : } else {
4145 0 : HCCL_ERROR("[TaskException]task sequence[%u/%u] is %s",
4146 : lineNum, totalPrinted, taskLine.c_str());
4147 : }
4148 0 : }
4149 :
4150 : // 更新算子上下文
4151 0 : void HcclCommAicpu::UpdateOpContext(u32& opIndex, std::string& opTag, u32& lineCount,
4152 : std::vector<std::string>& currentOpTasks,
4153 : u32 newOpIndex, const std::string& newOpTag) const
4154 : {
4155 0 : opIndex = newOpIndex;
4156 0 : opTag = newOpTag;
4157 0 : lineCount = 0;
4158 0 : currentOpTasks.clear();
4159 0 : currentOpTasks.push_back("OP(" + std::to_string(opIndex) + ")");
4160 0 : }
4161 :
4162 : // 准备下一行数据
4163 0 : void HcclCommAicpu::PrepareNextLine(u32 opIndex, u32& lineCount,
4164 : std::vector<std::string>& currentOpTasks) const
4165 : {
4166 0 : lineCount++;
4167 0 : currentOpTasks.clear();
4168 0 : currentOpTasks.push_back("OP(" + std::to_string(opIndex) + ") continued");
4169 0 : }
4170 :
4171 : // 拼接task列表为字符串
4172 0 : std::string HcclCommAicpu::ConcatTaskLine(const std::vector<std::string>& tasks) const
4173 : {
4174 0 : std::string taskLine;
4175 0 : for (const auto& task : tasks) {
4176 0 : taskLine += task;
4177 : }
4178 0 : return taskLine;
4179 0 : }
4180 :
4181 : // 打印剩余未满行的tasks
4182 0 : void HcclCommAicpu::PrintRemainingTasks(bool isMonitor, u32 lineCount, u32 printedCount,
4183 : const std::vector<std::string>& currentOpTasks) const
4184 : {
4185 : // size > 1 表示除了OP头还有task
4186 0 : if (currentOpTasks.size() > 1) {
4187 0 : std::string taskLine = ConcatTaskLine(currentOpTasks);
4188 0 : PrintTaskLine(isMonitor, lineCount + 1, printedCount, taskLine);
4189 0 : }
4190 0 : }
4191 :
4192 0 : void HcclCommAicpu::PrintTaskExceptionTaskQue(u32 sqIdx, SqeRingBuffer *sqeContextBuffer, bool isMonitor)
4193 : {
4194 0 : const u32 sqeNum = 200;
4195 0 : const u32 maxTasksPerLine = 50;
4196 :
4197 : // 获取初始算子信息
4198 0 : const AicpuOpInfo *lastOpInfo = GetOpInfoFromSqIdx(sqIdx, sqeContextBuffer);
4199 0 : CHK_PRT_RET(lastOpInfo == nullptr, HCCL_ERROR("%s fail, opInfo is nullptr", __func__),);
4200 :
4201 : // 初始化状态
4202 0 : u32 opIndex = lastOpInfo->opIndex;
4203 0 : std::string opTag = lastOpInfo->tagBuff;
4204 0 : u32 lineCount = 0;
4205 0 : u32 printedCount = 0;
4206 0 : u32 extraSlotPerLine = 2;
4207 0 : std::vector<std::string> currentOpTasks;
4208 0 : currentOpTasks.push_back("OP(" + std::to_string(opIndex) + ")");
4209 :
4210 : // 主循环：遍历并打印
4211 0 : for (u32 i = 0; i < sqeNum; i++) {
4212 : // 计算新索引并获取算子信息
4213 0 : u32 newSqIdx = (sqIdx - i + HCCL_SQE_MAX_CNT) % HCCL_SQE_MAX_CNT;
4214 0 : const AicpuOpInfo *newOpInfo = GetOpInfoFromSqIdx(newSqIdx, sqeContextBuffer);
4215 0 : CHK_PRT_RET(newOpInfo == nullptr, HCCL_ERROR("%s fail, opInfo is nullptr", __func__),);
4216 :
4217 0 : u32 sizeAfterAdd = currentOpTasks.size() + 1;
4218 0 : if (sizeAfterAdd >= (maxTasksPerLine + extraSlotPerLine) || (i > 0 && (newOpInfo->opIndex != opIndex || newOpInfo->tagBuff != opTag))) {
4219 : // 打印算子信息（仅第一次）
4220 0 : if (lineCount == 0) {
4221 0 : PrintOpDataInfo(sqIdx, sqeContextBuffer, isMonitor);
4222 : }
4223 :
4224 : // 拼接并打印task行
4225 0 : std::string taskLine = ConcatTaskLine(currentOpTasks);
4226 0 : PrintTaskLine(isMonitor, lineCount + 1, printedCount, taskLine);
4227 :
4228 : // 更新状态
4229 0 : if (newOpInfo->opIndex != opIndex || newOpInfo->tagBuff != opTag) {
4230 0 : UpdateOpContext(opIndex, opTag, lineCount, currentOpTasks,
4231 0 : newOpInfo->opIndex, newOpInfo->tagBuff);
4232 : } else {
4233 0 : PrepareNextLine(opIndex, lineCount, currentOpTasks);
4234 : }
4235 0 : }
4236 : // 添加task到序列
4237 0 : currentOpTasks.push_back("," + GetTaskBriefsInfo(newSqIdx, sqeContextBuffer));
4238 0 : printedCount++;
4239 : }
4240 : // 打印剩余tasks
4241 0 : PrintRemainingTasks(isMonitor, lineCount, printedCount, currentOpTasks);
4242 0 : }
4243 :
4244 0 : std::string HcclCommAicpu::GetTaskBriefsInfo(u32 idx, SqeRingBuffer *sqeContextBuffer)
4245 : {
4246 0 : uint8_t *sqeMirrorBufferAddr = sqeContextBuffer->rtsMirrorBuffer + idx * HCCL_SQE_SIZE;
4247 0 : rtStarsSqeHeader_t * const sqeHeader = (rtStarsSqeHeader_t * const)sqeMirrorBufferAddr;
4248 0 : uint8_t sqeType = sqeHeader->type;
4249 :
4250 0 : SqeInfo sqeInfo;
4251 0 : SqeContextUtils::QuerySqeInfo(sqeContextBuffer->rtsMirrorBuffer + idx * HCCL_SQE_SIZE,
4252 0 : sqeContextBuffer->rtsqSqeType[idx], sqeContextBuffer->addInfo[idx], &sqeInfo);
4253 0 : uint8_t subType = sqeInfo.subType;
4254 :
4255 0 : std::stringstream ss;
4256 0 : std::string taskName = "UN";
4257 0 : switch (sqeType) {
4258 0 : case RT_STARS_SQE_TYPE_NOTIFY_RECORD:
4259 0 : taskName = "NR"; // Notify Record
4260 0 : break;
4261 0 : case RT_STARS_SQE_TYPE_WRITE_VALUE:
4262 0 : if (subType == RT_STARS_WRITE_VALUE_SUB_TYPE_NOTIFY_RECORD_IPC_NO_PCIE) {
4263 0 : taskName = "NR";
4264 0 : } else if (subType == RT_STARS_WRITE_VALUE_SUB_TYPE_EVENT_RESET) {
4265 0 : taskName = "NW"; // Notify Wait
4266 0 : } else if (subType == RT_STARS_WRITE_VALUE_SUB_TYPE_RDMA_DB_SEND) {
4267 0 : taskName = "RS"; // Rdma Send
4268 : }
4269 0 : break;
4270 0 : case RT_STARS_SQE_TYPE_NOTIFY_WAIT:
4271 0 : taskName = "NW";
4272 0 : break;
4273 0 : case RT_STARS_SQE_TYPE_EVENT_WAIT:
4274 0 : taskName = "NW";
4275 0 : break;
4276 0 : case RT_STARS_SQE_TYPE_SDMA:
4277 0 : taskName = "SD"; // SDMA
4278 0 : break;
4279 0 : case RT_STARS_SQE_TYPE_COND:
4280 0 : taskName = "CO";
4281 0 : break;
4282 0 : case RT_STARS_SQE_TYPE_PLACE_HOLDER:
4283 0 : taskName = "PH";
4284 0 : break;
4285 0 : default:
4286 0 : taskName = std::to_string(sqeType);
4287 0 : break;
4288 : }
4289 :
4290 0 : ss << taskName << "(";
4291 0 : if (sqeContextBuffer->rtsDfxInfo[idx].remoteRank != INVALID_VALUE_RANKID) {
4292 0 : ss << sqeContextBuffer->rtsDfxInfo[idx].remoteRank;
4293 : } else {
4294 0 : ss << "/";
4295 : }
4296 0 : ss << ",";
4297 0 : if (sqeContextBuffer->rtsDfxInfo[idx].notifyId != INVALID_VALUE_RANKID) {
4298 0 : ss << sqeContextBuffer->rtsDfxInfo[idx].notifyId;
4299 : } else {
4300 0 : ss << "/";
4301 : }
4302 0 : ss << ")";
4303 0 : return ss.str();
4304 0 : }
4305 :
4306 1 : void HcclCommAicpu::RecordReportStatus(dfx::ReportStatus status)
4307 : {
4308 1 : std::unique_lock<std::mutex> lock(reportQueueMutex_);
4309 1 : while (reportStatusQueue_.size() >= MAX_REPORT_STATUS) {
4310 0 : HCCL_WARNING("[HcclCommAicpu][RecordReportStatus] retry status queue reach the limit[%u], " \
4311 : "the front status[%u] is dropped.", MAX_REPORT_STATUS, reportStatusQueue_.front());
4312 0 : reportStatusQueue_.pop();
4313 : }
4314 1 : reportStatusQueue_.push(status);
4315 1 : HCCL_INFO("[HcclCommAicpu][RecordReportStatus]push[%u], retry queue size()[%u]", status, reportStatusQueue_.size());
4316 1 : }
4317 :
4318 0 : void HcclCommAicpu::GetReportStatusQueue(std::queue<dfx::ReportStatus> &reportStatusQue)
4319 : {
4320 0 : std::unique_lock<std::mutex> lock(reportQueueMutex_);
4321 0 : std::swap(reportStatusQueue_, reportStatusQue);
4322 0 : }
4323 :
4324 0 : void HcclCommAicpu::SetStreamCqeExceptionStatus(const Stream &stream, CqeExceptionStatus cqeStatus)
4325 : {
4326 0 : HCCL_RUN_INFO("SetStreamCqeExceptionStatus: stream sq id %u, cqe exception %u", stream.sqId(), cqeStatus);
4327 0 : auto iter = streamCqeExceptionStatus_.find(stream.sqId());
4328 0 : if (iter != streamCqeExceptionStatus_.end()) {
4329 0 : iter->second = cqeStatus;
4330 : } else {
4331 0 : streamCqeExceptionStatus_.insert({ stream.sqId(), cqeStatus });
4332 : }
4333 0 : return;
4334 : }
4335 :
4336 0 : CqeExceptionStatus HcclCommAicpu::GetStreamCqeExceptionStatus(const Stream &stream)
4337 : {
4338 0 : auto iter = streamCqeExceptionStatus_.find(stream.sqId());
4339 0 : if (iter != streamCqeExceptionStatus_.end()) {
4340 0 : return iter->second;
4341 : } else {
4342 0 : return CqeExceptionStatus::kNone;
4343 : }
4344 : }
4345 :
4346 0 : void HcclCommAicpu::ResetStreamCqeExceptionStatus(const Stream &stream)
4347 : {
4348 0 : auto iter = streamCqeExceptionStatus_.find(stream.sqId());
4349 0 : if (iter != streamCqeExceptionStatus_.end()) {
4350 0 : iter->second = CqeExceptionStatus::kNone;
4351 : }
4352 0 : HCCL_INFO("ResetStreamCqeExceptionStatus: stream sq id %u", stream.sqId());
4353 0 : return;
4354 : }
4355 :
4356 0 : void HcclCommAicpu::SetBSRSendOpExecException()
4357 : {
4358 0 : HCCL_INFO("set send stream exec exception");
4359 0 : bsrSendOpExecException_ = true;
4360 0 : return;
4361 : }
4362 :
4363 0 : void HcclCommAicpu::SetBSRRecvOpExecException()
4364 : {
4365 0 : HCCL_INFO("set recv stream exec exception");
4366 0 : bsrRecvOpExecException_ = true;
4367 0 : return;
4368 : }
4369 :
4370 0 : bool HcclCommAicpu::GetBSRSendOpExecException()
4371 : {
4372 0 : bool ret = (GetStreamCqeExceptionStatus(bsrSendStream_) == CqeExceptionStatus::kSdmaErr) || bsrSendOpExecException_;
4373 0 : HCCL_INFO("GetBSRSendOpExecException: stream %u cqe status %u, send exec status %u", bsrSendStream_.sqId(),
4374 : GetStreamCqeExceptionStatus(bsrSendStream_), bsrSendOpExecException_);
4375 0 : return ret;
4376 : }
4377 :
4378 0 : bool HcclCommAicpu::GetBSRRecvOpExecException()
4379 : {
4380 0 : HCCL_INFO("GetBSRRecvOpExecException: stream %u cqe status %u, recv exec status %u", bsrRecvStream_.sqId(),
4381 : GetStreamCqeExceptionStatus(bsrRecvStream_), bsrRecvOpExecException_);
4382 0 : return (GetStreamCqeExceptionStatus(bsrRecvStream_) == CqeExceptionStatus::kSdmaErr) || bsrRecvOpExecException_;
4383 : }
4384 :
4385 0 : HcclResult HcclCommAicpu::CleanStream(Stream &stream)
4386 : {
4387 0 : CHK_RET(stream.ClearLocalBuff());
4388 0 : CHK_RET(UpdateSqStatus(stream));
4389 0 : ResetStreamCqeExceptionStatus(stream);
4390 0 : HCCL_INFO("CleanStream %u success.", stream.sqId());
4391 0 : return HCCL_SUCCESS;
4392 : }
4393 :
4394 0 : HcclResult HcclCommAicpu::ClearStreamCqeException(Stream &stream)
4395 : {
4396 0 : HandleCqeException(stream, true);
4397 0 : return HCCL_SUCCESS;
4398 : }
4399 :
4400 0 : HcclResult HcclCommAicpu::ResetBSRSendOpExecException()
4401 : {
4402 0 : bsrSendOpExecException_ = false;
4403 0 : CHK_RET(CleanStream(bsrSendStream_));
4404 0 : CHK_RET(ClearStreamCqeException(bsrSendStream_));
4405 0 : HCCL_INFO("ResetBSRSendOpExecException success.");
4406 0 : return HCCL_SUCCESS;
4407 : }
4408 :
4409 0 : HcclResult HcclCommAicpu::ResetBSRRecvOpExecException()
4410 : {
4411 0 : bsrRecvOpExecException_ = false;
4412 0 : CHK_RET(CleanStream(bsrRecvStream_));
4413 0 : CHK_RET(ClearStreamCqeException(bsrRecvStream_));
4414 0 : HCCL_INFO("ResetBSRRecvOpExecException success.");
4415 0 : return HCCL_SUCCESS;
4416 : }
4417 :
4418 0 : HcclResult HcclCommAicpu::ResetBSRException()
4419 : {
4420 0 : if (bsrRetryOp_ == HCCL_SEND) {
4421 0 : CHK_RET(ResetBSRSendOpExecException());
4422 0 : HCCL_INFO("reset batchsendrecv exception success, tag:%s, index:%u", bsrSendOpId_.tag,
4423 : bsrSendOpId_.index);
4424 0 : } else if (bsrRetryOp_ == HCCL_RECV) {
4425 0 : CHK_RET(ResetBSRRecvOpExecException());
4426 0 : HCCL_INFO("reset batchsendrecv exception success, tag:%s, index:%u", bsrRecvOpId_.tag,
4427 : bsrRecvOpId_.index);
4428 : } else {
4429 0 : HCCL_INFO("reset batchsendrecv exception success, tag:%s", bsrTargetOpId_.tag);
4430 : }
4431 0 : return HCCL_SUCCESS;
4432 : }
4433 :
4434 0 : void HcclCommAicpu::UpdateBSRRetryCnt()
4435 : {
4436 0 : if (bsrRetryOp_ == HCCL_SEND) {
4437 0 : bsrSendRetryCnt_++;
4438 : } else {
4439 0 : bsrRecvRetryCnt_++;
4440 : }
4441 0 : HCCL_INFO("UpdateBSRRetryCnt, SendCnt[%u], RecvCnt[%u]", bsrRecvRetryCnt_, bsrRecvRetryCnt_);
4442 0 : return;
4443 : }
4444 :
4445 0 : void HcclCommAicpu::ResetBSRRetryCnt()
4446 : {
4447 0 : bsrSendRetryCnt_ = 0;
4448 0 : bsrRecvRetryCnt_ = 0;
4449 0 : return ;
4450 : }
4451 :
4452 0 : void HcclCommAicpu::InitSendRecvOpId(const OpParam ¶m, HcclOpIdentifier &opId)
4453 : {
4454 : // send算子入参中只有dst对端rank号，而recv算子入参中只有src源端rank号
4455 0 : if (param.opType == HcclCMDType::HCCL_CMD_SEND) {
4456 0 : opId.detRank = param.dstRank;
4457 0 : opId.srcRank = topoInfo_.userRank;
4458 : } else {
4459 0 : opId.srcRank = param.srcRank;
4460 0 : opId.detRank = topoInfo_.userRank;
4461 : }
4462 0 : opId.isSendRecv = true;
4463 0 : HCCL_DEBUG("[HcclCommAicpu][InitSendRecvOpId]src=[%u] dst=[%u] isSendRecv=[%u]", opId.srcRank, opId.detRank,
4464 : opId.isSendRecv);
4465 0 : return;
4466 : }
4467 :
4468 0 : u32 HcclCommAicpu::HcclUpdateBatchSendRecvOpIndex(std::map<u32, u32> &bsrIndexMap, u32 peerRank)
4469 : {
4470 0 : u32 ret = 0;
4471 0 : auto opIndexMapIter = bsrIndexMap.find(peerRank);
4472 0 : if (opIndexMapIter != bsrIndexMap.end()) {
4473 0 : (opIndexMapIter->second)++;
4474 0 : ret = opIndexMapIter->second;
4475 : } else {
4476 0 : bsrIndexMap.insert({ peerRank, 1 });
4477 0 : ret = 1;
4478 : }
4479 0 : return ret;
4480 : }
4481 :
4482 0 : u32 HcclCommAicpu::HcclUpdateBatchSendRecvOpIndex(HcclSendRecvType opType, u32 srcRank, u32 dstRank)
4483 : {
4484 0 : u32 peerRank = (opType == HcclSendRecvType::HCCL_SEND) ? dstRank : srcRank;
4485 0 : auto &opIndexMap = (opType == HcclSendRecvType::HCCL_SEND) ? bsrSendIndexMap_ : bsrRecvIndexMap_;
4486 :
4487 0 : return HcclUpdateBatchSendRecvOpIndex(opIndexMap, peerRank);
4488 : }
4489 0 : HcclResult HcclCommAicpu::GetBsrTransportQpn( const HcclSendRecvItem *sendrecvPair, AlgResourceResponse &algResource,
4490 : u32 &qpn)
4491 : {
4492 0 : CHK_PTR_NULL(sendrecvPair);
4493 0 : LINK targetLink;
4494 0 : u32 commIndex = 0;
4495 0 : u32 remoteRank = sendrecvPair->remoteRank;
4496 0 : u32 localRank = topoInfo_.userRank;
4497 0 : HcclSendRecvType sendRecvType = sendrecvPair->sendRecvType;
4498 0 : HCCL_DEBUG("[GetBsrTransportQpn] bsrOptype =[%d], localRank=[%u] remoteRank=[%u]",
4499 : sendRecvType, localRank, remoteRank);
4500 :
4501 0 : if ((sendRecvType == HcclSendRecvType::HCCL_SEND && remoteRank < localRank) ||
4502 0 : (sendRecvType == HcclSendRecvType::HCCL_RECV && remoteRank > localRank)) {
4503 0 : commIndex = COMM_INDEX_0;
4504 : } else {
4505 0 : commIndex = COMM_INDEX_1;
4506 : }
4507 0 : CHK_PRT_RET(commIndex >= algResource.opTransportResponse[COMM_COMBINE_ORDER].size(),
4508 : HCCL_ERROR("[GetBsrTransportQpn] batchsendrecv op commIndex[%u] is larger than "\
4509 : "opTransportResponse size[%zu]",
4510 : commIndex, algResource.opTransportResponse[COMM_COMBINE_ORDER].size()), HCCL_E_PARA);
4511 : SingleSubCommTransport &commCombined =
4512 0 : static_cast<SingleSubCommTransport&>(algResource.opTransportResponse[COMM_COMBINE_ORDER][commIndex]);
4513 :
4514 0 : CHK_PRT_RET(sendrecvPair->remoteRank >= commCombined.userRank2subCommRank.size(),
4515 : HCCL_ERROR("[GetBsrTransportQpn]batchsendrecv op remoteUserRank[%u] is larger than "\
4516 : "userRank2subCommRank map size[%zu]",
4517 : sendrecvPair->remoteRank, commCombined.userRank2subCommRank.size()), HCCL_E_PARA);
4518 :
4519 0 : u32 rank = commCombined.userRank2subCommRank[sendrecvPair->remoteRank];
4520 0 : CHK_PRT_RET(rank >= commCombined.links.size(),
4521 : HCCL_ERROR("[GetBsrTransportQpn] batchsendrecv op remoteUserRank[%u], get rank[%u]," \
4522 : "the size of combinedComm links is [%zu]", sendrecvPair->remoteRank, rank, commCombined.links.size()),
4523 : HCCL_E_PARA);
4524 0 : targetLink = commCombined.links[rank];
4525 :
4526 0 : CHK_SMART_PTR_NULL(targetLink);
4527 0 : if (targetLink->GetLinkType() == LinkType::LINK_ROCE){
4528 0 : CHK_RET(targetLink->GetTransportId(qpn));
4529 : }
4530 0 : HCCL_DEBUG("[HcclCommAicpu][GetBsrTransportQpn] localrank=[%u] remoteuserRank=[%u] remoteRank=[%u],sendrecvType=[%d] qpn =[%u], comindex [%u]",
4531 : topoInfo_.userRank, sendrecvPair->remoteRank, remoteRank, sendrecvPair->sendRecvType, qpn, commIndex);
4532 0 : return HCCL_SUCCESS;
4533 0 : }
4534 0 : HcclResult HcclCommAicpu::InitBatchSendRecvOpId(const OpParam ¶m, const HcclSendRecvItem *sendrecvPair,
4535 : HcclOpIdentifier &opId, u32 streamId, AlgResourceResponse &algResource)
4536 : {
4537 0 : CHK_PTR_NULL(sendrecvPair);
4538 0 : if (sendrecvPair->sendRecvType == HcclSendRecvType::HCCL_RECV) {
4539 0 : opId.srcRank = sendrecvPair->remoteRank;
4540 0 : opId.detRank = topoInfo_.userRank;
4541 : } else {
4542 0 : opId.srcRank = topoInfo_.userRank;
4543 0 : opId.detRank = sendrecvPair->remoteRank;
4544 : }
4545 0 : opId.index = HcclUpdateBatchSendRecvOpIndex(sendrecvPair->sendRecvType, opId.srcRank, opId.detRank);
4546 0 : opId.isSendRecv = true;
4547 0 : opId.opType = HcclCMDType::HCCL_CMD_BATCH_SEND_RECV;
4548 0 : std::string sendrecvTag = param.tag + "_BSR_" + std::to_string(opId.srcRank) + "_" + std::to_string(opId.detRank);
4549 0 : CHK_SAFETY_FUNC_RET(memcpy_s(opId.tag, sizeof(opId.tag), sendrecvTag.c_str(), sendrecvTag.size()));
4550 0 : std::string sendrecvNewTag = param.tag + "_device";
4551 0 : CHK_SAFETY_FUNC_RET(memcpy_s(opId.newTag, sizeof(opId.newTag), sendrecvNewTag.c_str(), sendrecvNewTag.size()));
4552 :
4553 0 : u32 qpn = 0 ;
4554 0 : if ((opId.srcRank != opId.detRank) && (!param.BatchSendRecvDataDes.isDirectRemoteRank[sendrecvPair->remoteRank])){
4555 0 : CHK_RET(GetBsrTransportQpn(sendrecvPair, algResource, qpn));
4556 : }
4557 :
4558 0 : auto &bsrinfo = opId.bsrInfo[sendrecvPair->sendRecvType];
4559 0 : bsrinfo.detRank = opId.detRank;
4560 0 : bsrinfo.srcRank = opId.srcRank;
4561 0 : bsrinfo.index = opId.index;
4562 0 : bsrinfo.streamId = streamId;
4563 0 : bsrinfo.tpQpn = qpn;
4564 0 : CHK_SAFETY_FUNC_RET(memcpy_s(bsrinfo.bsrTag, sizeof(bsrinfo.bsrTag), sendrecvTag.c_str(), sendrecvTag.size()));
4565 :
4566 0 : HCCL_INFO("[HcclCommAicpu][InitBatchSendRecvOpId] tag=[%s] index=[%u] src=[%u] det=[%u] qpn =[%u]",
4567 : opId.tag, opId.index, opId.srcRank, opId.detRank, qpn);
4568 0 : return HCCL_SUCCESS;
4569 0 : }
4570 :
4571 0 : HcclResult HcclCommAicpu::InitBatchSendRecvOpId(const OpParam ¶m, AlgResourceResponse &algResource)
4572 : {
4573 0 : CHK_SAFETY_FUNC_RET(
4574 : memset_s(reinterpret_cast<void *>(&bsrSendOpId_), sizeof(bsrSendOpId_), 0, sizeof(bsrSendOpId_)));
4575 0 : CHK_SAFETY_FUNC_RET(
4576 : memset_s(reinterpret_cast<void *>(&bsrRecvOpId_), sizeof(bsrRecvOpId_), 0, sizeof(bsrRecvOpId_)));
4577 :
4578 0 : CHK_PRT_RET((algResource.slaveStreams.size() < BSR_RETRY_STREAM_NUM),
4579 : HCCL_ERROR("in batchsendrecv op, slave stream is not enough."), HCCL_E_INTERNAL);
4580 0 : bsrSendOpId_.streamId = algResource.slaveStreams[BSR_RETRY_SEND_STREAM_INDEX].id();
4581 0 : bsrRecvOpId_.streamId = algResource.slaveStreams[BSR_RETRY_RECV_STREAM_INDEX].id();
4582 :
4583 0 : u32 iter = param.BatchSendRecvDataDes.curIterNum;
4584 0 : std::vector<std::vector<HcclSendRecvItem *>> &pairs = bsrSendRecvPairs_;
4585 0 : CHK_PRT_RET((pairs.size() <= iter),
4586 : HCCL_ERROR("batchsendrecv sendrecv pairs size[%u] less than or equal to curiter[%u]", pairs.size(), iter),
4587 : HCCL_E_INTERNAL);
4588 :
4589 0 : for (auto &pair : pairs[iter]) {
4590 0 : HcclOpIdentifier &opId =
4591 0 : (pair->sendRecvType == HcclSendRecvType::HCCL_SEND) ? bsrSendOpId_ : bsrRecvOpId_;
4592 0 : auto streamId =
4593 0 : (pair->sendRecvType == HcclSendRecvType::HCCL_SEND) ? bsrSendOpId_.streamId : bsrRecvOpId_.streamId;
4594 0 : CHK_RET(InitBatchSendRecvOpId(param, pair, opId, streamId, algResource));
4595 : }
4596 :
4597 : //补全batchsendrecv中sendrecv的bsrInfo, 两边都发生故障的时候要用
4598 0 : bsrSendOpId_.bsrInfo[HCCL_RECV].index = bsrRecvOpId_.index;
4599 0 : bsrSendOpId_.bsrInfo[HCCL_RECV].streamId = bsrRecvOpId_.streamId;
4600 0 : bsrSendOpId_.bsrInfo[HCCL_RECV].srcRank = bsrRecvOpId_.srcRank;
4601 0 : bsrSendOpId_.bsrInfo[HCCL_RECV].detRank = bsrRecvOpId_.detRank;
4602 0 : bsrSendOpId_.bsrInfo[HCCL_RECV].tpQpn = bsrRecvOpId_.bsrInfo[HCCL_RECV].tpQpn;
4603 0 : CHK_SAFETY_FUNC_RET(memcpy_s(bsrSendOpId_.bsrInfo[HCCL_RECV].bsrTag, sizeof(bsrSendOpId_.bsrInfo[HCCL_RECV].bsrTag),
4604 : bsrRecvOpId_.tag, sizeof(bsrRecvOpId_.tag)));
4605 :
4606 0 : bsrRecvOpId_.bsrInfo[HCCL_SEND].index = bsrSendOpId_.index;
4607 0 : bsrRecvOpId_.bsrInfo[HCCL_SEND].streamId = bsrSendOpId_.streamId;
4608 0 : bsrRecvOpId_.bsrInfo[HCCL_SEND].srcRank = bsrSendOpId_.srcRank;
4609 0 : bsrRecvOpId_.bsrInfo[HCCL_SEND].detRank = bsrSendOpId_.detRank;
4610 0 : bsrRecvOpId_.bsrInfo[HCCL_SEND].tpQpn = bsrSendOpId_.bsrInfo[HCCL_SEND].tpQpn;
4611 0 : CHK_SAFETY_FUNC_RET(memcpy_s(bsrRecvOpId_.bsrInfo[HCCL_SEND].bsrTag, sizeof(bsrRecvOpId_.bsrInfo[HCCL_SEND].bsrTag),
4612 : bsrSendOpId_.tag, sizeof(bsrSendOpId_.tag)));
4613 0 : return HCCL_SUCCESS;
4614 : }
4615 :
4616 0 : HcclResult HcclCommAicpu::QueryBatchSendRecvPairBeginPos()
4617 : {
4618 : // 前面已经生成 batchsendrecv 的 send & recv opid 时已经校验过slave stream num数，此处不再重复校验
4619 0 : CHK_RET(QuerySqStatusByType(devId_, bsrSendStream_.sqId(), DRV_SQCQ_PROP_SQ_TAIL, bsrSendOpBeginSqePos_));
4620 0 : CHK_RET(QuerySqStatusByType(devId_, bsrRecvStream_.sqId(), DRV_SQCQ_PROP_SQ_TAIL, bsrRecvOpBeginSqePos_));
4621 :
4622 0 : HCCL_INFO("QueryBatchSendRecvPairBeginPos send sqePos[%u] recv sqePos[%u]", bsrSendOpBeginSqePos_,
4623 : bsrRecvOpBeginSqePos_);
4624 0 : return HCCL_SUCCESS;
4625 : }
4626 :
4627 0 : HcclResult HcclCommAicpu::QueryBatchSendRecvPairEndPos()
4628 : {
4629 : // 前面已经生成 batchsendrecv 的 send & recv opid 时已经校验过slave stream num数，此处不再重复校验
4630 0 : CHK_RET(QuerySqStatusByType(devId_, bsrSendStream_.sqId(), DRV_SQCQ_PROP_SQ_TAIL, bsrSendOpEndSqePos_));
4631 0 : CHK_RET(QuerySqStatusByType(devId_, bsrRecvStream_.sqId(), DRV_SQCQ_PROP_SQ_TAIL, bsrRecvOpEndSqePos_));
4632 :
4633 0 : HCCL_INFO("QueryBatchSendRecvPairEndPos send sqePos[%u] recv sqePos[%u]", bsrSendOpEndSqePos_, bsrRecvOpEndSqePos_);
4634 0 : return HCCL_SUCCESS;
4635 : }
4636 :
4637 0 : HcclResult HcclCommAicpu::CommitBSRStoredException(HcclOpExecFSM &fsmState, KfcError &errorCode)
4638 : {
4639 0 : if (GetBSRSendOpExecException()) {
4640 0 : bsrRetryOp_ = HCCL_SEND;
4641 0 : errorCode = KfcError::kSdma;
4642 0 : HCCL_INFO("CommitBSRStoredException: send stream remain retry error.");
4643 0 : } else if (GetBSRRecvOpExecException()) {
4644 0 : bsrRetryOp_ = HCCL_RECV;
4645 0 : errorCode = KfcError::kSdma;
4646 0 : HCCL_INFO("CommitBSRStoredException: recv stream remain retry error.");
4647 : }
4648 :
4649 0 : u32 retryCnt = (bsrRetryOp_ == HCCL_SEND) ? bsrSendRetryCnt_ : bsrRecvRetryCnt_;
4650 0 : if (errorCode != KfcError::kNone) {
4651 0 : bsrTargetOpId_ = (bsrRetryOp_ == HCCL_SEND) ? bsrSendOpId_ : bsrRecvOpId_;
4652 0 : HCCL_RUN_INFO("CommitBSRStoredException: stored op tag %s index %u , report retry error. curSendRetryCnt[%u],"
4653 : "curRecvRetryCnt[%u]",
4654 : bsrTargetOpId_.tag, bsrTargetOpId_.index, bsrSendRetryCnt_, bsrRecvRetryCnt_);
4655 0 : auto ret = aicpuHdc_.SetOpExecStatus(kfcStatusTransferD2H_, bsrTargetOpId_, KfcStatus::kStoplaunch, errorCode,
4656 : retryCnt);
4657 0 : HCCL_RETRY_CHK_RET_AND_TRANS_FSM(ret, HCCL_ERROR("SetOpExecStatus failed, ret:%u", ret), KfcError::kExec,
4658 : HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR);
4659 0 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_FSM_STOPPING;
4660 : } else {
4661 0 : CHK_RET(UpdateOpExecStatus(fsmState, KfcStatus::kRuning, errorCode, retryCnt));
4662 0 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_FSM_WAIT_END;
4663 : }
4664 0 : return HCCL_SUCCESS;
4665 : }
4666 :
4667 0 : HcclResult HcclCommAicpu::GetBSRRetryOpId(const OpParam ¶m, HcclOpIdentifier &targetOpId)
4668 : {
4669 0 : KfcCommand cmd = KfcCommand::kNone;
4670 0 : CHK_RET(aicpuHdc_.GetOpExecCtrlCmd(kfcControlTransferH2D_, cmd));
4671 0 : if (cmd == KfcCommand::kStopLaunch) {
4672 0 : HcclOpIdentifier targetOp;
4673 0 : CHK_RET(aicpuHdc_.GetOpExecCtrlTargetOp(kfcControlTransferH2D_, targetOp));
4674 0 : std::string targetOpTag = std::string(reinterpret_cast<char*>(&targetOp.tag[0]));
4675 0 : if (targetOpTag == std::string(reinterpret_cast<char*>(&bsrSendOpId_.tag[0]))) {
4676 0 : bsrRetryOp_ = HCCL_SEND;
4677 0 : } else if (targetOpTag == std::string(reinterpret_cast<char*>(&bsrRecvOpId_.tag[0]))) {
4678 0 : bsrRetryOp_ = HCCL_RECV;
4679 0 : } else if (targetOpTag == param.tag) {
4680 0 : if (targetOp.detRank == bsrSendOpId_.detRank) {
4681 0 : bsrRetryOp_ = HCCL_SEND;
4682 0 : } else if (targetOp.srcRank == bsrRecvOpId_.srcRank) {
4683 0 : bsrRetryOp_ = HCCL_RECV;
4684 : } else {
4685 0 : HCCL_ERROR("hccl aicpu can not retry, got stop launch command, but target op srcRank[%u] and"
4686 : "dstRank[%u] is not match with send (dst:%u) or recv (src:%u) op",
4687 : targetOp.srcRank, targetOp.detRank, bsrSendOpId_.detRank, bsrRecvOpId_.srcRank);
4688 0 : return HCCL_E_INTERNAL;
4689 : }
4690 : } else {
4691 : // tag 不匹配，报错退出
4692 0 : HCCL_ERROR("hccl aicpu can not retry, got stop launch command, but target op tag[%s] is not match with"
4693 : "send (tag:%s) or recv (tag:%s) or batchsendrecv (tag:%s)",
4694 : targetOpTag.c_str(), bsrSendOpId_.tag, bsrRecvOpId_.tag, param.tag.c_str());
4695 0 : return HCCL_E_INTERNAL;
4696 : }
4697 0 : HCCL_RUN_INFO("hccl aicpu got command %u at op[tag: %s, index: %u].",cmd, targetOpTag.c_str(), targetOp.index);
4698 0 : } else {
4699 0 : if (GetBSRSendOpExecException()) {
4700 0 : bsrRetryOp_ = HCCL_SEND;
4701 0 : } else if (GetBSRRecvOpExecException()) {
4702 0 : bsrRetryOp_ = HCCL_RECV;
4703 : } else {
4704 : // 其他场景，报错退出
4705 0 : HCCL_ERROR("hccl aicpu find task exception, but send and recv op has no exception");
4706 0 : return HCCL_E_INTERNAL;
4707 : }
4708 : }
4709 0 : if (bsrRetryOp_ == HCCL_SEND) {
4710 0 : targetOpId = bsrSendOpId_;
4711 : } else {
4712 0 : targetOpId = bsrRecvOpId_;
4713 : }
4714 0 : HCCL_RUN_INFO("GetBSRRetryOpId: bsrRetryOpType %u, targetOpId: tag %s, index %u", bsrRetryOp_, targetOpId.tag,
4715 : targetOpId.index);
4716 0 : return HCCL_SUCCESS;
4717 : }
4718 :
4719 0 : HcclResult HcclCommAicpu::InitExecLoop(OpParam ¶m, std::unique_ptr<CollExecutorBase> &executor, u32 &loopNum)
4720 : {
4721 0 : if ((param.opType == HcclCMDType::HCCL_CMD_BATCH_SEND_RECV) && retryEnable_) {
4722 0 : CHK_RET(executor->CreatePairWiseList(param.BatchSendRecvDataDes.sendRecvItemsPtr,
4723 : param.BatchSendRecvDataDes.itemNum));
4724 0 : CHK_RET(executor->GetPairWiseList(bsrSendRecvPairs_));
4725 0 : CHK_PRT_RET(bsrSendRecvPairs_.empty(),
4726 : HCCL_ERROR("[HcclCommAicpu][InitExecLoop]batchsendrecv pairs is empty"), HCCL_E_INTERNAL);
4727 :
4728 0 : for (size_t i = 0; i < bsrSendRecvPairs_.size(); i++) {
4729 0 : CHK_PRT_RET((bsrSendRecvPairs_[i].size() > BSR_RETRY_SENDRECV_PAIR_NUM_MAX) || bsrSendRecvPairs_[i].empty(),
4730 : HCCL_ERROR("batchsendrecv pairs[%u] size[%u] is out of range [1,2]", i,
4731 : bsrSendRecvPairs_.size()),
4732 : HCCL_E_INTERNAL);
4733 :
4734 0 : for (size_t j = 0; j < bsrSendRecvPairs_[i].size(); j++) {
4735 0 : CHK_PTR_NULL(bsrSendRecvPairs_[i][j]);
4736 : }
4737 :
4738 0 : CHK_PRT_RET(((bsrSendRecvPairs_[i].size() == BSR_RETRY_SENDRECV_PAIR_NUM_MAX) &&
4739 : (bsrSendRecvPairs_[i][BSR_RETRY_SENDRECV_PAIR_INDEX_0]->sendRecvType ==
4740 : bsrSendRecvPairs_[i][BSR_RETRY_SENDRECV_PAIR_INDEX_1]->sendRecvType)),
4741 : HCCL_ERROR("batchsendrecv pairs[%u] sendRecvType[%u] is same", i,
4742 : bsrSendRecvPairs_[i][BSR_RETRY_SENDRECV_PAIR_INDEX_0]->sendRecvType),
4743 : HCCL_E_INTERNAL);
4744 : }
4745 :
4746 0 : param.BatchSendRecvDataDes.curIterNum = 0;
4747 0 : loopNum = bsrSendRecvPairs_.size();
4748 0 : } else {
4749 0 : loopNum = 1;
4750 : }
4751 0 : HCCL_INFO("InitExecLoop: execute loop num %u", loopNum);
4752 0 : return HCCL_SUCCESS;
4753 : }
4754 :
4755 0 : HcclResult HcclCommAicpu::ParseHierarchicalAlgOption(u32 *ahcConfInfo)
4756 : {
4757 0 : u32 algOptionSize= ahcConfInfo[TOP_HIERARCHICAL_CONF_lENGTH_INDEX];
4758 0 : if (algOptionSize >= (TOP_HIERARCHICAL_CONF_SIZE-1)) {
4759 0 : HCCL_ERROR("[HcclCommAicpu][ParseHierarchicalAlgOption] hierarchicalAlgOption size[%lu] exceed maxsize[%lu]",
4760 : algOptionSize, (TOP_HIERARCHICAL_CONF_SIZE-1));
4761 0 : return HCCL_E_INTERNAL;
4762 : }
4763 :
4764 0 : std::map<AHCConcOpType, TemplateType> hierarchicalAlgOption;
4765 0 : for (u32 i = TOP_HIERARCHICAL_CONF_INFO_INDEX ; i < (TOP_HIERARCHICAL_CONF_INFO_INDEX + algOptionSize); i++) {
4766 0 : AHCConcOpType ahcConcOpType;
4767 0 : ahcConcOpType.ahcLevel = static_cast<AHCLevel>((ahcConfInfo[i] & TOP_HIERARCHICAL_CONF_LEVEL_LOCATION) >> TOP_HIERARCHICAL_CONF_LEVEL_SHIFT);
4768 0 : ahcConcOpType.concType = static_cast<ConcType>((ahcConfInfo[i] & TOP_HIERARCHICAL_CONF_CONC_TYPE_LOCATION) >> TOP_HIERARCHICAL_CONF_CONC_TYPE_SHIFT);
4769 0 : ahcConcOpType.ahcOpType = static_cast<AHCOpType>((ahcConfInfo[i] & TOP_HIERARCHICAL_CONF_OP_TYPE_LOCATION) >> TOP_HIERARCHICAL_CONF_OP_TYPE_SHIFT);
4770 0 : TemplateType templateType = static_cast<TemplateType>((ahcConfInfo[i] & TOP_HIERARCHICAL_CONF_TEMPLATE_TYPE_LOCATION) >> TOP_HIERARCHICAL_CONF_TEMPLATE_TYPE_SHIFT);
4771 :
4772 0 : hierarchicalAlgOption[ahcConcOpType] = templateType;
4773 0 : HCCL_DEBUG("[HcclCommAicpu][ParseHierarchicalAlgOption]: index[%u] ahcLevel[%u] concType[%u] ahcOpType[%u], templateType[%u]",
4774 : i, ahcConcOpType.ahcLevel, ahcConcOpType.concType, ahcConcOpType.ahcOpType, templateType);
4775 : }
4776 0 : topoMatcher_->SetAHCAlgOption(hierarchicalAlgOption);
4777 0 : return HCCL_SUCCESS;
4778 0 : }
4779 :
4780 0 : void HcclCommAicpu::PollCqeException(hccl::Stream &stream, bool isReadClear, rtLogicCqReport_t &cqeException,
4781 : CqeStatus &cqeStatus)
4782 : {
4783 0 : const HcclComStreamInfo &streamInfo = stream.GetHcclStreamInfo();
4784 : // 以下条件满足其中之一，进行Poll CQE：1、AICPU/Custom读清CQ 2、AICPU进程读取CQ信息
4785 0 : bool isPollCqe = true;
4786 0 : while (isPollCqe) {
4787 0 : LogControl logControl(false, retryEnable_); // 使能重执行场景，修改ERROR->RUN_WARNING，析构时自动恢复
4788 : CqeQueryInput cqeQueryInput;
4789 0 : dfx_tracer::ExecutorTracer::SetCqeQueryInput(GetDevId(), streamInfo, cqeQueryInput);
4790 0 : constexpr u32 reportSize = 256;
4791 : rtLogicCqReport_t streamReport[reportSize];
4792 0 : cqeQueryInput.cqeAddr = reinterpret_cast<uint8_t *>(streamReport); // 用于存放接收到的cq
4793 0 : cqeStatus = CqReportRecv(cqeQueryInput, cqeException);
4794 0 : isPollCqe = (cqeStatus == dfx::CqeStatus::kCqeException && isReadClear); // 读清CQ场景，继续查询
4795 0 : }
4796 0 : }
4797 :
4798 0 : void HcclCommAicpu::ExchangeCqeContext(hccl::Stream &stream, rtLogicCqReport_t &cqeException, CqeStatus &cqeStatus,
4799 : ErrCqeContext &cqeCtx)
4800 : {
4801 0 : HcclResult ret = HCCL_SUCCESS;
4802 : // aicpu进程把读取到的异常cq，写入共享内存
4803 0 : if (cqeStatus == dfx::CqeStatus::kCqeException) {
4804 0 : ret = stream.SetCqeContext(ErrCqeContext(static_cast<u32>(cqeStatus), cqeException.taskId,
4805 0 : cqeException.errorCode, cqeException.sqeType));
4806 : }
4807 :
4808 : // aicpu进程和custom进程，都从共享内存中读取cqe信息
4809 0 : if (ret != HCCL_SUCCESS || stream.GetCqeContext(cqeCtx) != HCCL_SUCCESS) {
4810 : // 写入/读取共享内存失败，标记流状态，避免背景线程继续轮询这条流，导致刷屏
4811 0 : SetStreamCqeExceptionStatus(stream, CqeExceptionStatus::kOther);
4812 0 : HCCL_ERROR("%s fail, set streamCqeExceptionStatus streamId[%u] exception status[%d]",
4813 : __func__, stream.id(), CqeExceptionStatus::kOther);
4814 0 : return;
4815 : }
4816 : }
4817 :
4818 0 : void HcclCommAicpu::HandleCqeException(hccl::Stream &stream, bool isReadClear)
4819 : {
4820 0 : std::unique_lock<std::mutex> lock(queryCqeMutex_);
4821 :
4822 : // poll cqe信息
4823 : rtLogicCqReport_t cqeException;
4824 0 : CqeStatus cqeStatus = CqeStatus::kDefault;
4825 0 : PollCqeException(stream, isReadClear, cqeException, cqeStatus);
4826 :
4827 : // 以下两种情况直接返回，不处理cq信息：1、读清CQ场景只读取不处理 2、本地记录流上已经有异常信息(避免刷屏)
4828 0 : if (isReadClear || GetStreamCqeExceptionStatus(stream) != CqeExceptionStatus::kNone) {
4829 0 : return;
4830 : }
4831 :
4832 : // aicpu和custom同步异常cqe信息，并记录在本地
4833 0 : ErrCqeContext cqeCtx;
4834 0 : ExchangeCqeContext(stream, cqeException, cqeStatus, cqeCtx);
4835 0 : cqeStatus = static_cast<dfx::CqeStatus>(cqeCtx.cqeStatus);
4836 :
4837 : // 处理流上的异常cq
4838 0 : if (cqeStatus != dfx::CqeStatus::kDefault) {
4839 0 : bool isSdmaTypeErr = cqeStatus == dfx::CqeStatus::kCqeException &&
4840 0 : cqeCtx.sqeType == RT_STARS_SQE_TYPE_SDMA;
4841 0 : bool isCompDataErr = cqeCtx.errorCode == RT_SDMA_COMPDATAERR ||
4842 0 : cqeCtx.errorCode == RT_SDMA_COMPERR ||
4843 0 : cqeCtx.errorCode == RT_SDMA_DATAERR;
4844 0 : bool isSdmaCompDataErr = isSdmaTypeErr && isCompDataErr;
4845 :
4846 0 : ReportErrCqe(stream, cqeCtx);
4847 :
4848 : // 标记发生ErrCqe的流
4849 0 : hccl::CqeExceptionStatus cqeExceptionStatus =
4850 0 : isSdmaCompDataErr ? hccl::CqeExceptionStatus::kSdmaErr : hccl::CqeExceptionStatus::kOther;
4851 0 : SetStreamCqeExceptionStatus(stream, cqeExceptionStatus);
4852 :
4853 : // 通知kfc线程
4854 0 : dfxExtendInfo_.cqeException.sqeType = cqeCtx.sqeType;
4855 0 : dfxExtendInfo_.cqeException.errorCode = cqeCtx.errorCode;
4856 0 : dfxExtendInfo_.cqeStatus = cqeStatus;
4857 0 : dfxExtendInfo_.pollStatus = PollStatus::kStopAsException;
4858 0 : HCCL_INFO("update dfxExtendInfo, group %s, streamId %u, cqeStatus %d, sqetype %u, errorCode %u",
4859 : identifier_.c_str(), stream.id(), cqeStatus, cqeCtx.sqeType, cqeCtx.errorCode);
4860 : }
4861 0 : }
4862 :
4863 0 : void HcclCommAicpu::ReportErrCqe(hccl::Stream &stream, ErrCqeContext &cqeCtx)
4864 : {
4865 0 : dfx::CqeStatus cqeStatus = static_cast<dfx::CqeStatus>(cqeCtx.cqeStatus);
4866 0 : const HcclComStreamInfo &streamInfo = stream.GetHcclStreamInfo();
4867 0 : u32 head = 0;
4868 0 : u32 tail = 0;
4869 0 : QuerySqStatusByType(devId_, streamInfo.sqId, DRV_SQCQ_PROP_SQ_HEAD, head);
4870 0 : QuerySqStatusByType(devId_, streamInfo.sqId, DRV_SQCQ_PROP_SQ_TAIL, tail);
4871 :
4872 0 : bool isSdmaTypeErr = cqeStatus == dfx::CqeStatus::kCqeException &&
4873 0 : cqeCtx.sqeType == RT_STARS_SQE_TYPE_SDMA;
4874 0 : bool isCompDataErr = cqeCtx.errorCode == RT_SDMA_COMPDATAERR ||
4875 0 : cqeCtx.errorCode == RT_SDMA_COMPERR ||
4876 0 : cqeCtx.errorCode == RT_SDMA_DATAERR;
4877 0 : bool isSdmaCompDataErr = isSdmaTypeErr && isCompDataErr;
4878 :
4879 : // 发生sdma、notify_wait、place_holder、write_value错误，上报error message
4880 0 : bool isComReportErrMesg = cqeStatus == dfx::CqeStatus::kCqeException &&
4881 0 : (cqeCtx.sqeType == RT_STARS_SQE_TYPE_SDMA ||
4882 0 : cqeCtx.sqeType == RT_STARS_SQE_TYPE_NOTIFY_WAIT ||
4883 0 : cqeCtx.sqeType == RT_STARS_SQE_TYPE_PLACE_HOLDER ||
4884 0 : cqeCtx.sqeType == RT_STARS_SQE_TYPE_WRITE_VALUE);
4885 :
4886 : // 使能重执行且触发SDMA ERROR的场景，修改ERROR->RUN_WARNING
4887 0 : LogControl retryLog(false, retryEnable_ && isSdmaCompDataErr);
4888 :
4889 : // 只在AICPU进程上报taskException，Custom进程不上报
4890 0 : if (isComReportErrMesg && errMessageReport_) {
4891 : // 记录关键信息，并通过D2H通信通道交给host内存
4892 0 : GenTaskExceptionInfo(cqeCtx.sqeType, stream, head);
4893 : // 通知ts错误信息，触发非SDMA错误、不使能重执行触发SDMA错误、使能重执行触发无法重执行的错误时，进行故障上报
4894 0 : if (!isSdmaTypeErr || (isSdmaTypeErr && (!retryEnable_ || !isCompDataErr))) {
4895 : HcclResult ret =
4896 0 : SendTaskExceptionByMBox(cqeCtx.errorCode);
4897 0 : HCCL_RUN_INFO("[HcclCommAicpu][SendTaskExceptionByMBox]group[%s]:"
4898 : "Try to send task exception by mailbox, errType[%u], errCode[%u], streamId[%d]",
4899 : identifier_.c_str(), cqeCtx.sqeType, cqeCtx.errorCode, stream.id());
4900 0 : CHK_PRT_CONT(ret != HCCL_SUCCESS,
4901 : HCCL_ERROR("[OpRetry][AICPU]group[%s]:Send task exception by mailBox failed, streamId[%d]",
4902 : identifier_.c_str(), stream.id()));
4903 : }
4904 : // 当前阶段， 每个通信域在plog打印一次，error message作为host上报，只打印首次
4905 0 : errMessageReport_ = false;
4906 : }
4907 :
4908 : // 发生ErrCqe之后，统一在AICPU进程上报taskException
4909 0 : if (cqeStatus == dfx::CqeStatus::kCqeException) {
4910 0 : if (IsNoNeedWait() && cqeCtx.sqeType == RT_STARS_SQE_TYPE_PLACE_HOLDER) {
4911 0 : PrintTaskExceptionAllComm(); // 超时场景打印所有通信域的taskException
4912 0 : PrintAicpuCommExecStatus();
4913 0 : } else if (IsNoNeedWait()) {
4914 0 : ErrCqeContext cqeCtx;
4915 0 : stream.GetCqeContext(cqeCtx);
4916 0 : PrintTaskExceptionByTaskId(cqeCtx.sqeType, cqeCtx.taskId, stream, tail); // 仅打印本条流的taskException
4917 : }
4918 : }
4919 :
4920 0 : CHK_PRT_CONT(!retryEnable_ && isSdmaCompDataErr,
4921 : HCCL_RUN_INFO("[OpRetry][AICPU]group[%s] hccl aicpu can not retry, retryEnable is false.", identifier_.c_str()));
4922 :
4923 0 : CHK_PRT_CONT(retryEnable_ && isSdmaTypeErr && !isCompDataErr,
4924 : HCCL_RUN_INFO("[OpRetry][AICPU]group[%s] hccl aicpu can not retry, errCode is [%u].", identifier_.c_str(), cqeCtx.errorCode));
4925 :
4926 0 : HCCL_ERROR("Exception happened, group %s, sqid %d, cqeStatus %d, sqetype %u, errorCode %u, head %u, tail %u",
4927 : identifier_.c_str(), streamInfo.sqId, cqeStatus, cqeCtx.sqeType, cqeCtx.errorCode, head, tail);
4928 0 : }
4929 :
4930 0 : HcclResult HcclCommAicpu::SendTaskExceptionByMBox(const uint16_t &rsErrorCode)
4931 : {
4932 0 : u32 localDeviceId = 0;
4933 : HcclSignalInfo notifyInfo;
4934 0 : opNotifies_[1]->GetNotifyData(notifyInfo);
4935 :
4936 0 : HCCL_INFO("[HcclCommAicpu][SendTaskExceptionByMBox] HostToDeviceLogicId[%u]", notifyInfo.devId);
4937 0 : CHK_RET(hrtDrvGetLocalDevIDByHostDevID(notifyInfo.devId, &localDeviceId));
4938 0 : CHK_RET(hccl_plf::SendTaskExceptionByMBox(localDeviceId, opNotifies_[1]->notifyId_, notifyInfo.tsId,
4939 : userStreamId_, rsErrorCode));
4940 0 : return HCCL_SUCCESS;
4941 : }
4942 :
4943 0 : void HcclCommAicpu::HandleIndOpCqe()
4944 : {
4945 0 : std::unique_lock<std::mutex> lock(queryCqeMutex_);
4946 0 : std::shared_lock<std::shared_mutex> rwLock(threadAicpuMutex_);
4947 0 : for (auto &thread : threads_) {
4948 0 : if (thread == nullptr) {
4949 0 : continue;
4950 : }
4951 0 : Stream stream = *thread->GetStream();
4952 : // 流上已有异常信息，不再重复读取
4953 0 : if (GetStreamCqeExceptionStatus(stream) != CqeExceptionStatus::kNone) {
4954 0 : continue;
4955 : }
4956 :
4957 : // poll cqe信息
4958 : rtLogicCqReport_t cqeException;
4959 : CqeQueryInput cqeQueryInput;
4960 0 : dfx_tracer::ExecutorTracer::SetCqeQueryInput(GetDevId(), stream.GetHcclStreamInfo(), cqeQueryInput);
4961 0 : constexpr u32 reportSize = 256;
4962 : rtLogicCqReport_t streamReport[reportSize];
4963 0 : cqeQueryInput.cqeAddr = reinterpret_cast<uint8_t *>(streamReport); // 用于存放接收到的cq
4964 0 : CqeStatus cqeStatus = CqReportRecv(cqeQueryInput, cqeException);
4965 : // 未读取到异常信息，返回
4966 0 : if (cqeStatus == dfx::CqeStatus::kDefault) {
4967 0 : continue;
4968 : }
4969 0 : ReportIndOpCqe(stream, cqeException, cqeStatus);
4970 0 : }
4971 0 : }
4972 :
4973 0 : void HcclCommAicpu::ReportIndOpCqe(hccl::Stream &stream, const rtLogicCqReport_t &cqeException, CqeStatus cqeStatus)
4974 : {
4975 0 : const HcclComStreamInfo &streamInfo = stream.GetHcclStreamInfo();
4976 0 : u32 head = 0;
4977 0 : u32 tail = 0;
4978 0 : QuerySqStatusByType(devId_, streamInfo.sqId, DRV_SQCQ_PROP_SQ_HEAD, head);
4979 0 : QuerySqStatusByType(devId_, streamInfo.sqId, DRV_SQCQ_PROP_SQ_TAIL, tail);
4980 :
4981 : // 打印taskException信息
4982 0 : if (cqeStatus == dfx::CqeStatus::kCqeException) {
4983 0 : if (cqeException.sqeType == RT_STARS_SQE_TYPE_PLACE_HOLDER) {
4984 0 : PrintTaskExceptionAllComm(); // 超时场景打印所有通信域的taskException
4985 0 : PrintAicpuCommExecStatus();
4986 : } else {
4987 0 : taskExecption_.PrintTaskExceptionByTaskId(cqeException.sqeType, cqeException.taskId, stream, tail);
4988 : }
4989 : }
4990 :
4991 : // 打印重执行提示信息
4992 0 : bool isSdmaTypeErr = cqeStatus == dfx::CqeStatus::kCqeException &&
4993 0 : cqeException.sqeType == RT_STARS_SQE_TYPE_SDMA;
4994 0 : bool isCompDataErr = cqeException.errorCode == RT_SDMA_COMPDATAERR ||
4995 0 : cqeException.errorCode == RT_SDMA_COMPERR;
4996 0 : bool isSdmaCompDataErr = isSdmaTypeErr && isCompDataErr;
4997 0 : if (retryEnable_ && isSdmaCompDataErr) {
4998 0 : uint16_t rsErrorCode = TS_ERROR_RETRY_CONSTRAINT;
4999 0 : CHK_PRT(SendTaskExceptionByMBox(rsErrorCode));
5000 0 : HCCL_RUN_INFO("[OpRetry][AICPU]group[%s] can not retry, IndOp does not support opRetry", identifier_.c_str());
5001 : }
5002 :
5003 : // 标记发生ErrCqe的流
5004 0 : hccl::CqeExceptionStatus cqeExceptionStatus =
5005 0 : isSdmaCompDataErr ? hccl::CqeExceptionStatus::kSdmaErr : hccl::CqeExceptionStatus::kOther;
5006 0 : SetStreamCqeExceptionStatus(stream, cqeExceptionStatus);
5007 :
5008 0 : HCCL_ERROR("Exception happened, group %s, streamId %u, sqid %d, cqeStatus %d, sqetype %u, errorCode %u, "
5009 : "head %u, tail %u", identifier_.c_str(), stream.id(), streamInfo.sqId, cqeStatus, cqeException.sqeType,
5010 : cqeException.errorCode, head, tail);
5011 0 : }
5012 :
5013 0 : HcclResult HcclCommAicpu::RefreshLinkForSwitchNic(const std::string &newTag, const TransportRequest &transportRequest,
5014 : const std::map<u32, bool> &remoteRankPortMap, bool isSecondBuild, LINK &switchLink)
5015 : {
5016 0 : u32 remoteRankId = transportRequest.remoteUserRank;
5017 0 : auto iterRemoteRank = remoteRankPortMap.find(remoteRankId);
5018 0 : bool needSwitch = iterRemoteRank != remoteRankPortMap.end();
5019 0 : bool isBackup = needSwitch && !(iterRemoteRank->second);
5020 0 : HCCL_INFO("[HcclCommAicpu][%s] newTag[%s], localRank[%u], remoteRank[%u], input memory type[%u], "
5021 : "output memory type[%u], isRdma[%u], isSecondBuild[%u], needSwitch[%u], isBackup[%u].",
5022 : __func__, newTag.c_str(), localUserRank_, remoteRankId, transportRequest.inputMemType,
5023 : transportRequest.outputMemType, transportRequest.isUsedRdma, isSecondBuild, needSwitch, isBackup);
5024 :
5025 0 : if (transportRequest.isUsedRdma && needSwitch) {
5026 0 : auto *linkRes = isBackup ? &linkRdmaResBackUp_ : &linkRdmaRes_;
5027 0 : auto iterRankLinks = linkRes->find(remoteRankId);
5028 0 : CHK_PRT_RET(iterRankLinks == linkRes->end(),
5029 : HCCL_ERROR("[HcclCommAicpu][%s] comm[%s], local rank[%u], fail to find relative link for remote rank[%u], "
5030 : "isBackup[%u]", __func__, identifier_.c_str(), localUserRank_, remoteRankId, isBackup),
5031 : HCCL_E_INTERNAL);
5032 0 : auto iterRankTagLinks = iterRankLinks->second.find(newTag);
5033 0 : CHK_PRT_RET(iterRankTagLinks == iterRankLinks->second.end(),
5034 : HCCL_ERROR("[HcclCommAicpu][%s] comm[%s], local rank[%u], "
5035 : "fail to find relative tag[%s] of links for remote rank[%u], isBackup[%u]",
5036 : __func__, identifier_.c_str(), localUserRank_, newTag.c_str(), remoteRankId, isBackup),
5037 : HCCL_E_INTERNAL);
5038 0 : CHK_PRT_RET(isSecondBuild && iterRankLinks->second[newTag].size() < 2U,
5039 : HCCL_ERROR("[HcclCommAicpu][%s] comm[%s], local rank[%u], isSecondBuild[%u], "
5040 : "fail to find second link for remmote rank[%u]",
5041 : __func__, identifier_.c_str(), localUserRank_, isSecondBuild, remoteRankId),
5042 : HCCL_E_INTERNAL);
5043 :
5044 : switchLink = isSecondBuild
5045 0 : ? iterRankLinks->second[newTag][1] : iterRankLinks->second[newTag][0];
5046 0 : CHK_SMART_PTR_NULL(switchLink);
5047 :
5048 0 : auto iterAck = receivedAcks_.find(remoteRankId);
5049 0 : CHK_PRT_RET(iterAck == receivedAcks_.end(),
5050 : HCCL_ERROR("[%s]there is no link with rankId[%u]", __func__, remoteRankId),
5051 : HCCL_E_NOT_FOUND);
5052 0 : switchLink->SetSupportDataReceivedAck(iterAck->second);
5053 : }
5054 0 : return HCCL_SUCCESS;
5055 : }
5056 :
5057 0 : HcclResult HcclCommAicpu::ReAllocTransportForSwitchNic(const std::string &newTag, AlgResourceResponse &algResResponse,
5058 : std::map<u32, bool> &remoteRankPortMap)
5059 : {
5060 0 : HCCL_INFO("[HcclCommAicpu][%s] Entry realloc transport for switch nic, comm identifier[%s], localRank[%u], tag[%s]",
5061 : __func__, identifier_.c_str(), localUserRank_, newTag.c_str());
5062 0 : std::set<u32> bsrTansportRank;
5063 0 : for (auto &levelNSubCommTransport : algResResponse.opTransportResponse) {
5064 0 : for (auto &singleSubCommTransport : levelNSubCommTransport) {
5065 0 : for (size_t i = 0; i < singleSubCommTransport.transportRequests.size(); i++) {
5066 0 : auto &transportRequest = singleSubCommTransport.transportRequests[i];
5067 0 : if (transportRequest.isValid) {
5068 0 : receivedAcks_[transportRequest.remoteUserRank] = singleSubCommTransport.supportDataReceivedAck;
5069 0 : bool isSecondBuild = false;
5070 0 : if (transportRequest.isUsedRdma && newTag.find("BatchSendRecv") != std::string::npos
5071 0 : && bsrTansportRank.find(transportRequest.remoteUserRank) != bsrTansportRank.end()) {
5072 : //仅在batchsendrecv rdma下发的时候需要第二次刷新，实际第一次下发都刷好了，第二次就是get一下
5073 0 : isSecondBuild = true;
5074 : }
5075 0 : bsrTansportRank.insert(transportRequest.remoteUserRank);
5076 0 : CHK_RET(RefreshLinkForSwitchNic(newTag, transportRequest, remoteRankPortMap, isSecondBuild,
5077 : singleSubCommTransport.links[i]));
5078 : }
5079 : }
5080 : }
5081 : }
5082 :
5083 0 : return HCCL_SUCCESS;
5084 0 : }
5085 :
5086 0 : HcclResult HcclCommAicpu::
5087 : RefreshRoceTransportsForSwitchNic(std::unordered_map<std::string, OpCommTransport> &reservedLinks)
5088 : {
5089 0 : ChangeLinkInfo changeLinkInfo;
5090 0 : HcclResult ret = LoadChangeLinkInfo(changeLinkInfo);
5091 :
5092 0 : CHK_PRT_RET(changeLinkInfo.isChangeLinkFlag == false,
5093 : HCCL_ERROR("[HcclCommAicpu][%s] some error happened on host, switch nic has failed on rank[%u]. "
5094 : "The error message will be broadcast to other ranks in comm[%s].",
5095 : __func__, localUserRank_,identifier_.c_str()), HCCL_E_PARA);
5096 :
5097 0 : CHK_PRT_RET(ret != HCCL_SUCCESS, HCCL_ERROR("[HcclCommAicpu][%s] comm identifier[%s], rank[%u], "
5098 : "load change link info failed.", __func__, identifier_.c_str(), localUserRank_), ret);
5099 :
5100 0 : std::map<u32, bool> remoteRankPortMap;
5101 0 : for (u32 i = 0; i < changeLinkInfo.remoteRankNum; i++) {
5102 0 : remoteRankPortMap.emplace(changeLinkInfo.remoteRankList[i], changeLinkInfo.isUseDefaultPort[i]);
5103 0 : HCCL_INFO("[HcclCommAicpu][%s] comm identifier[%s], rank[%u], remote rank[%u], isUseDefaultPort[%u].",
5104 : __func__, identifier_.c_str(), localUserRank_, changeLinkInfo.remoteRankList[i],
5105 : changeLinkInfo.isUseDefaultPort[i]);
5106 : }
5107 :
5108 : // 对resMap中所有tag的transport link根据主备进行刷新
5109 0 : for (auto &resMapIt: resMap_) {
5110 0 : HCCL_RUN_INFO("[HcclCommAicpu][%s] rank[%u] refresh algResResponse of tag[%s].", __func__,
5111 : localUserRank_, resMapIt.first.c_str());
5112 0 : reservedLinks.emplace(resMapIt.first, resMapIt.second.opTransportResponse);
5113 0 : CHK_RET(ReAllocTransportForSwitchNic(resMapIt.first, resMapIt.second, remoteRankPortMap));
5114 : }
5115 0 : return HCCL_SUCCESS;
5116 0 : }
5117 :
5118 0 : HcclResult HcclCommAicpu::RevertTransportsForSwitchNic(std::unordered_map<std::string, OpCommTransport> &reservedLinks)
5119 : {
5120 0 : if (reservedLinks.size() == 0) {
5121 0 : HCCL_INFO("[HcclCommAicpu][%s] comm identifier[%s], rank[%u], reserved link is empty, "
5122 : "no need to revert transport.", __func__, identifier_.c_str(), localUserRank_);
5123 0 : return HCCL_SUCCESS;
5124 : }
5125 0 : for (auto &resMapIt: resMap_) {
5126 0 : HCCL_RUN_INFO("[HcclCommAicpu][%s] comm identifier[%s], rank[%u], revert transport in algResResponse of tag[%s].",
5127 : __func__, identifier_.c_str(), localUserRank_, resMapIt.first.c_str());
5128 0 : auto linkIt = reservedLinks.find(resMapIt.first);
5129 0 : if (linkIt == reservedLinks.end()) {
5130 0 : HCCL_RUN_WARNING("[HcclCommAicpu][%s] comm identifier[%s], rank[%u], fail to find transport to revert.",
5131 : __func__, identifier_.c_str(), localUserRank_);
5132 0 : continue;
5133 : }
5134 0 : resMapIt.second.opTransportResponse = linkIt->second;
5135 : }
5136 0 : return HCCL_SUCCESS;
5137 : }
5138 :
5139 0 : HcclResult HcclCommAicpu::SwitchNicWaitHandleCommand(std::unordered_map<std::string, OpCommTransport> &reservedLinks)
5140 : {
5141 0 : auto waitSwitchCmdTimeoutMs = HcclGetWaitRetryCmdTimeout(0);
5142 0 : auto waitSwitchCmdTimeout = std::chrono::milliseconds(waitSwitchCmdTimeoutMs);
5143 :
5144 0 : HcclResult ret = HCCL_E_INTERNAL;
5145 0 : auto startTime = std::chrono::steady_clock::now();
5146 : while (true) {
5147 0 : KfcCommand switchHandleCmd = KfcCommand::kNone;
5148 0 : CHK_RET(aicpuHdc_.GetOpExecCtrlCmd(kfcControlTransferH2D_, switchHandleCmd));
5149 0 : if (switchHandleCmd == KfcCommand::kWaitSwitchNic) {
5150 0 : HCCL_INFO("[HcclCommAicpu][%s] comm identifier[%s], rank[%u], hccl aicpu recv switch nic handle command.",
5151 : __func__, identifier_.c_str(), localUserRank_);
5152 0 : ret = aicpuHdc_.SetOpExecStatus(kfcStatusTransferD2H_, KfcStatus::kWaitSwitchRes, KfcError::kNone, 0);
5153 0 : if (ret != HCCL_SUCCESS) {
5154 0 : (void) RevertTransportsForSwitchNic(reservedLinks);
5155 0 : return ret;
5156 : }
5157 0 : return HCCL_SUCCESS;
5158 0 : } else if ((std::chrono::steady_clock::now() - startTime) >= waitSwitchCmdTimeout) {
5159 0 : HCCL_ERROR("[HcclCommAicpu][%s] comm identifier[%s], rank[%u], "
5160 : "hccl aicpu wait switch nic handle command timeout[%u ms], local nic switch is reverted.",
5161 : __func__, identifier_.c_str(), localUserRank_, waitSwitchCmdTimeoutMs);
5162 0 : (void) RevertTransportsForSwitchNic(reservedLinks);
5163 0 : return HCCL_E_TIMEOUT;
5164 : }
5165 0 : }
5166 :
5167 : return ret;
5168 : }
5169 :
5170 0 : HcclResult HcclCommAicpu::SwitchNicWaitResult(std::unordered_map<std::string, OpCommTransport> &reservedLinks)
5171 : {
5172 0 : auto waitSwitchResultTimeoutSecond = GetExternalInputHcclLinkTimeOut() * 2U;
5173 0 : const auto waitSwitchResultTimeout = std::chrono::seconds(waitSwitchResultTimeoutSecond);
5174 :
5175 0 : auto startTime = std::chrono::steady_clock::now();
5176 : while (true) {
5177 0 : KfcCommand switchResultCmd = KfcCommand::kNone;
5178 0 : CHK_RET(aicpuHdc_.GetOpExecCtrlCmd(kfcControlTransferH2D_, switchResultCmd));
5179 0 : if (switchResultCmd == KfcCommand::kAllSwitched) {
5180 0 : HCCL_INFO("[HcclCommAicpu][%s] comm identifier[%s], rank[%u], "
5181 : "hccl aicpu switch nic success, all nic switch transport.",
5182 : __func__, identifier_.c_str(), localUserRank_);
5183 0 : return HCCL_SUCCESS;
5184 0 : } else if (switchResultCmd == KfcCommand::kSwitchFail) {
5185 0 : HCCL_ERROR("[HcclCommAicpu][%s] comm identifier[%s], rank[%u], "
5186 : "hccl aicpu comm switch nic error, local nic switch is reverted.",
5187 : __func__, identifier_.c_str(), localUserRank_);
5188 0 : (void) RevertTransportsForSwitchNic(reservedLinks);
5189 0 : return HCCL_E_INTERNAL;
5190 0 : } else if ((std::chrono::steady_clock::now() - startTime) >= waitSwitchResultTimeout) {
5191 0 : HCCL_ERROR("[HcclCommAicpu][%s] comm identifier[%s], rank[%u], "
5192 : "hccl aicpu wait switch nic result timeout[%u s].",
5193 : __func__, identifier_.c_str(), localUserRank_, waitSwitchResultTimeoutSecond);
5194 0 : (void) RevertTransportsForSwitchNic(reservedLinks);
5195 0 : return HCCL_E_TIMEOUT;
5196 : }
5197 0 : }
5198 :
5199 : return HCCL_E_INTERNAL;
5200 : }
5201 :
5202 0 : HcclResult HcclCommAicpu::SwitchNic()
5203 : {
5204 0 : KfcStatus state = KfcStatus::kSwitchError;
5205 0 : std::unordered_map<std::string, OpCommTransport> reservedLinks;
5206 0 : HcclResult ret = RefreshRoceTransportsForSwitchNic(reservedLinks);
5207 0 : if (ret != HCCL_SUCCESS) {
5208 0 : HCCL_ERROR("[HcclCommAicpu][%s] comm identifier[%s], rank[%u], refresh roce transports failed.",
5209 : __func__, identifier_.c_str(), localUserRank_);
5210 0 : state = KfcStatus::kSwitchError;
5211 0 : ret = aicpuHdc_.SetOpExecStatus(kfcStatusTransferD2H_, state, KfcError::kNone, 0);
5212 : } else {
5213 0 : HCCL_INFO("[HcclCommAicpu][%s] comm identifier[%s], rank[%u], refresh roce transports success.",
5214 : __func__, identifier_.c_str(), localUserRank_);
5215 0 : state = KfcStatus::kPlanSwitch;
5216 0 : ret = aicpuHdc_.SetOpExecStatus(kfcStatusTransferD2H_, state, KfcError::kNone, 0);
5217 : }
5218 :
5219 0 : if (ret != HCCL_SUCCESS) {
5220 0 : HCCL_ERROR("[HcclCommAicpu][SwitchNic] comm identifier[%s], rank[%u], "
5221 : "send switch status[%u] to host fail.", __func__, identifier_.c_str(), localUserRank_, state);
5222 : }
5223 :
5224 0 : CHK_RET(SwitchNicWaitHandleCommand(reservedLinks));
5225 :
5226 0 : return SwitchNicWaitResult(reservedLinks);
5227 0 : }
5228 :
5229 0 : HcclResult HcclCommAicpu::ResumeChangeLink()
5230 : {
5231 0 : ChangeLinkInfo changeLinkInfo;
5232 0 : HcclResult ret = LoadChangeLinkInfo(changeLinkInfo);
5233 0 : bool useBackupLink = false;
5234 0 : std::map<u32, bool> remoteRankPortMap;
5235 0 : if (ret != HCCL_SUCCESS) {
5236 0 : HCCL_ERROR("[HcclCommAicpu][%s] comm identifier[%s], rank[%u], load change link info failed.",
5237 : __func__, identifier_.c_str(), localUserRank_);
5238 0 : return ret;
5239 : }
5240 0 : for (u32 i = 0; i < changeLinkInfo.remoteRankNum; i++) {
5241 0 : remoteRankPortMap.insert({changeLinkInfo.remoteRankList[i], changeLinkInfo.isUseDefaultPort[i]});
5242 0 : useBackupLink |= (!changeLinkInfo.isUseDefaultPort[i]);
5243 : }
5244 0 : ret = RefreshCommResponseTransportRes(remoteRankPortMap);
5245 0 : if (ret != HCCL_SUCCESS) {
5246 0 : HCCL_ERROR("[HcclCommAicpu][%s] comm identifier[%s], rank[%u], refresh roce transports failed.",
5247 : __func__, identifier_.c_str(), localUserRank_);
5248 0 : return ret;
5249 : } else {
5250 0 : HCCL_INFO("[HcclCommAicpu][%s] comm identifier[%s], rank[%u], refresh roce transports success.",
5251 : __func__, identifier_.c_str(), localUserRank_);
5252 0 : return HCCL_SUCCESS;
5253 : }
5254 0 : }
5255 :
5256 0 : HcclResult HcclCommAicpu::RefreshCommResponseTransportRes(std::map<u32, bool> &remoteRankPortMap)
5257 : {
5258 0 : PetersonLockGuard guard(hostDeviceLock_.get());
5259 0 : CHK_PRT_RET(guard.IsLockFailed(), HCCL_ERROR("[%s] hostDeviceLock lock failed", __func__), HCCL_E_INTERNAL);
5260 0 : OpParam param;
5261 : // 提前清理所有tag的链路，避免冲突
5262 0 : for (auto &resMapIt: resMap_) {
5263 0 : HCCL_RUN_INFO("[%s] clean roce resource of tag[%s].", __func__, resMapIt.first.c_str());
5264 0 : CleanRoceResource(resMapIt.first, resMapIt.second, remoteRankPortMap, param);
5265 : }
5266 0 : for (auto &resMapIt: resMap_) {
5267 0 : HCCL_RUN_INFO("[%s] refresh algResResponse of tag[%s].", __func__, resMapIt.first.c_str());
5268 0 : CHK_RET(ReAllocTransportResource(resMapIt.first, resMapIt.second, remoteRankPortMap, commParam_, param));
5269 : }
5270 0 : return HCCL_SUCCESS;
5271 0 : }
5272 :
5273 0 : HcclResult HcclCommAicpu::InitAicpuIndOp(CommAicpuParam *commAicpuParam)
5274 : {
5275 0 : if (indOpCommInitialized_) {
5276 0 : HCCL_RUN_INFO("[%s][InitAicpuIndOp]Group[%s] already initialized, skip reinit", __func__,
5277 : identifier_.c_str());
5278 0 : return HCCL_SUCCESS;
5279 : }
5280 0 : CHK_PTR_NULL(commAicpuParam);
5281 0 : topoInfo_.deviceLogicId = commAicpuParam->deviceLogicId;
5282 0 : topoInfo_.devicePhyId = commAicpuParam->devicePhyId;
5283 0 : topoInfo_.deviceType = static_cast<DevType>(commAicpuParam->deviceType);
5284 0 : identifier_ = std::string(commAicpuParam->hcomId);
5285 0 : topoInfo_.userRankSize = commAicpuParam->userRankSize;
5286 0 : topoInfo_.userRank = commAicpuParam->userRank;
5287 0 : notifys_.reserve(hccl::HCCL_THREAD_NOTIFY_MAX_NUM);
5288 0 : if (topoInfo_.deviceType == DevType::DEV_TYPE_910_93 || topoInfo_.deviceType == DevType::DEV_TYPE_910B) {
5289 0 : notifySize_ = NOTIFY_SIZE_FOUR;
5290 : } else {
5291 0 : notifySize_ = NOTIFY_SIZE_EIGHT;
5292 : }
5293 :
5294 0 : CHK_RET(hrtSetWorkModeAicpu(true));
5295 0 : CHK_RET(hrtSetlocalDevice(topoInfo_.deviceLogicId));
5296 0 : CHK_RET(hrtSetlocalDeviceType(topoInfo_.deviceType));
5297 0 : CHK_RET(hrtDrvGetLocalDevIDByHostDevID(topoInfo_.devicePhyId, &devId_));
5298 0 : CHK_RET(taskExecption_.Init(devId_, topoInfo_.userRank, identifier_));
5299 0 : CHK_RET(RegisterProfCallBack());
5300 :
5301 0 : if (topoInfo_.deviceType == DevType::DEV_TYPE_950 || topoInfo_.deviceType == DevType::DEV_TYPE_960) {
5302 0 : HCCL_INFO("[HcclCommAicpu][InitAicpuIndOp] InitAicpuIndOpV2 start");
5303 0 : indOpCommInitialized_ = true;
5304 0 : return HCCL_SUCCESS;
5305 : }
5306 :
5307 0 : HCCL_INFO("[HcclCommAicpu][InitAicpuIndOp] InitAicpuIndOp start");
5308 0 : if (!FindDispatcherByCommId(&dispatcherCtx_, identifier_.c_str())) {
5309 0 : CHK_RET(CreateDispatcherCtx(&dispatcherCtx_, devId_, identifier_.c_str()));
5310 : }
5311 0 : CHK_PTR_NULL(dispatcherCtx_);
5312 0 : hccl::DispatcherCtx *Ctx_temp = static_cast<DispatcherCtx *>(dispatcherCtx_);
5313 0 : HCCL_INFO("[%s] Ctx_temp[%p]", __func__, (void*)Ctx_temp);
5314 0 : (void)RegisterLoadTaskCallBack(Ctx_temp->GetDispatcher(), nullptr, dfx::TaskProfilingCallBack); //注册dispatcher
5315 0 : if (commAicpuParam->kfcControlTransferH2DParams.buffLen != 0 && kfcControlTransferH2D_ == nullptr) {
5316 0 : EXCEPTION_CATCH((kfcControlTransferH2D_ = std::make_shared<hccl::HDCommunicate>()), return HCCL_E_PTR);
5317 0 : CHK_SMART_PTR_NULL(kfcControlTransferH2D_);
5318 0 : CHK_RET(kfcControlTransferH2D_->InitDevice(commAicpuParam->kfcControlTransferH2DParams));
5319 : }
5320 0 : if (commAicpuParam->kfcStatusTransferD2HParams.buffLen != 0 && kfcStatusTransferD2H_ == nullptr) {
5321 0 : EXCEPTION_CATCH((kfcStatusTransferD2H_ = std::make_shared<hccl::HDCommunicate>()), return HCCL_E_PTR);
5322 0 : CHK_SMART_PTR_NULL(kfcStatusTransferD2H_);
5323 0 : CHK_RET(kfcStatusTransferD2H_->InitDevice(commAicpuParam->kfcStatusTransferD2HParams));
5324 : }
5325 :
5326 0 : indOpCommInitialized_ = true;
5327 :
5328 : // 在indOpCommInitialized_变为true后拉起背景线程
5329 0 : AicpuComContext *ctx = AicpuGetComContext();
5330 0 : AicpuHcclProcess::CallMC2MaintenanceThread(ctx);
5331 :
5332 0 : HCCL_RUN_INFO("%s group[%s] success!, deviceLogicId[%u], devicePhyId[%u], deviceType[%u], notifySize[%u], "
5333 : "dispatcherCtx[%p]", __func__, identifier_.c_str(), topoInfo_.deviceLogicId, topoInfo_.devicePhyId,
5334 : topoInfo_.deviceType, notifySize_, dispatcherCtx_);
5335 :
5336 0 : return HCCL_SUCCESS;
5337 : }
5338 :
5339 0 : HcclResult HcclCommAicpu::InitThreads(ThreadMgrAicpuParam *param)
5340 : {
5341 0 : CHK_PTR_NULL(param);
5342 0 : u32 threadNum = param->threadNum;
5343 0 : std::vector<std::shared_ptr<Thread>> outThreads;
5344 0 : outThreads.reserve(threadNum);
5345 0 : std::string hcomId(param->hcomId);
5346 0 : for (u32 i = 0; i < threadNum; ++i) {
5347 0 : std::string thdUniqueId(param->threadParam[i], THREAD_UNIQUE_ID_MAX_SIZE);
5348 0 : if (UNLIKELY(HcclCheckLogLevel(HCCL_LOG_INFO))) {
5349 0 : std::ostringstream oss;
5350 0 : oss << "threadParam[" << i << "] raw bytes: ";
5351 0 : for (u32 j = 0; j < THREAD_UNIQUE_ID_MAX_SIZE; ++j) {
5352 0 : oss << std::hex << std::setw(2) << std::setfill('0')
5353 0 : << static_cast<unsigned int>(static_cast<unsigned char>(param->threadParam[i][j])) << " ";
5354 : }
5355 0 : HCCL_INFO("[HcclCommAicpu][%s] %s", __func__, oss.str().c_str());
5356 0 : }
5357 0 : std::shared_ptr<AicpuTsThread> thread;
5358 0 : EXCEPTION_CATCH((thread = std::make_shared<AicpuTsThread>(thdUniqueId)), return HCCL_E_PTR);
5359 0 : HcclResult ret = thread->Init();
5360 0 : if (ret != HCCL_SUCCESS) {
5361 0 : HCCL_ERROR("[HcclCommAicpu][%s] comm identifier[%s], init threads num[%u] failed at index %u",
5362 : __func__, hcomId.c_str(), param->threadNum, i);
5363 0 : return ret;
5364 : }
5365 0 : outThreads.emplace_back(thread);
5366 0 : }
5367 :
5368 0 : ThreadHandle *threadArray = static_cast<ThreadHandle*>(param->deviceHandle);
5369 : // 空指针校验
5370 0 : CHK_PTR_NULL(threadArray);
5371 0 : for (size_t i = 0; i < threadNum; ++i) {
5372 0 : threadArray[i] = reinterpret_cast<ThreadHandle>(outThreads[i].get()); // 拷贝裸指针
5373 0 : HCCL_INFO("[HcclCommAicpu][%s] threadArray[%u] = [%lu]", __func__, i, threadArray[i]);
5374 : }
5375 0 : std::unique_lock<std::shared_mutex> rwLock(threadAicpuMutex_);
5376 0 : threads_.insert(threads_.end(), std::make_move_iterator(outThreads.begin()),
5377 : std::make_move_iterator(outThreads.end()));
5378 0 : HCCL_INFO("[HcclCommAicpu][%s] comm identifier[%s], init threads num[%u] success",
5379 : __func__, hcomId.c_str(), threadNum);
5380 : // 为上报翻转初始化资源
5381 0 : if (topoInfo_.deviceType != DevType::DEV_TYPE_950 && topoInfo_.deviceType != DevType::DEV_TYPE_960) {
5382 0 : CHK_RET(InitProfthreadResource(threadNum));
5383 : }
5384 0 : return HCCL_SUCCESS;
5385 0 : }
5386 :
5387 0 : HcclResult HcclCommAicpu::InitProfthreadResource(u32 threadNum) {
5388 0 : groupHashId_ = dfx::ProfilingManager::GetProfHashId(identifier_.c_str(), identifier_.length());
5389 0 : HCCL_INFO("[%s], group[%s], groupHash[%llu], threadNum[%u] ", __func__, identifier_.c_str(), groupHashId_, threadNum);
5390 0 : dfx::ProfCommInfo profInfo{ groupHashId_, topoInfo_.userRankSize, topoInfo_.userRank };
5391 : // 添加检查确保 threadNum 不超过线程总数
5392 0 : if (threadNum > threads_.size()) {
5393 0 : HCCL_ERROR("[%s] threadNum Err", __func__);
5394 0 : return HCCL_E_PARA;
5395 : }
5396 : // 从后往前迭代指定数量
5397 0 : auto begin = threads_.rbegin();
5398 0 : auto end = begin + threadNum;
5399 0 : for (auto it = begin; it != end; ++it) {
5400 0 : CHK_RET(dfx::ProfilingManager::AddProfInfoByStreamId((*it)->GetStream()->id(), identifier_, profInfo));
5401 : }
5402 0 : dfx::ProfilingExtendInfoHelper::InitProfItemId();
5403 0 : return HCCL_SUCCESS;
5404 : }
5405 :
5406 0 : HcclResult HcclCommAicpu::AllocChannelResource(HcclIndOpChannelRemoteResV3 *commParam)
5407 : {
5408 0 : CHK_PTR_NULL(commParam);
5409 0 : if (commParam->engine != COMM_ENGINE_AICPU &&
5410 0 : commParam->engine != COMM_ENGINE_AICPU_TS) {
5411 0 : HCCL_ERROR("[HcclCommAicpu][%s] engine type[%s] is not supported", __func__, GetEnumToString(GetCommEngineStatusStrMap(), commParam->engine).c_str());
5412 0 : return HCCL_E_PARA;
5413 : }
5414 :
5415 0 : if (commParam->listNum != 0 && commParam->remoteResV2 == nullptr) {
5416 0 : HCCL_ERROR("[HcclCommAicpu][%s] listNum[%u], remoteResV2 is nullptr", __func__, commParam->listNum);
5417 0 : return HCCL_E_PARA;
5418 : }
5419 :
5420 0 : multiQpThreshold_ = commParam->multiQpThreshold;
5421 0 : localUserRank_ = commParam->localUserRank;
5422 0 : HCCL_INFO("%s multiQpThreshold[%u], localUserRank[%u], deviceLogicId[%d], devicePhyId[%u], deviceType[%d], "
5423 : "listNum[%u]", __func__, multiQpThreshold_, localUserRank_, topoInfo_.deviceLogicId, topoInfo_.devicePhyId,
5424 : topoInfo_.deviceType, commParam->listNum);
5425 0 : for (u32 idx = 0; idx < commParam->listNum; idx++) {
5426 0 : HCCL_INFO("%s listNum[%u], listIdx[%u], remoteWorldRank[%u], remoteRank[%u], isUsedRdma[%d]",
5427 : __func__, commParam->listNum, idx, commParam->remoteResV2[idx].remoteWorldRank,
5428 : commParam->remoteResV2[idx].remoteRank, commParam->remoteResV2[idx].isUsedRdma);
5429 :
5430 0 : rankData_[commParam->remoteResV2[idx].remoteRank].remoteWorldRank = commParam->remoteResV2[idx].remoteWorldRank;
5431 0 : rankData_[commParam->remoteResV2[idx].remoteRank].remoteUsrRankId = commParam->remoteResV2[idx].remoteRank;
5432 0 : if (commParam->remoteResV2[idx].isUsedRdma) {
5433 0 : CHK_RET(InitRoceChannel(commParam, idx));
5434 : } else {
5435 0 : CHK_RET(InitP2pChannel(commParam, idx));
5436 : }
5437 : }
5438 0 : return HCCL_SUCCESS;
5439 : }
5440 :
5441 0 : HcclResult HcclCommAicpu::InitP2pChannel(HcclIndOpChannelRemoteResV3 *commParam, uint32_t channelIndex)
5442 : {
5443 0 : CHK_PTR_NULL(commParam);
5444 0 : CHK_PTR_NULL(commParam->channelList);
5445 0 : HcclIndOpChannelRemoteResV2 &remoteResV2 = commParam->remoteResV2[channelIndex];
5446 0 : u32 linkType = static_cast<u32>(remoteResV2.channelP2p.transportAttr.linkType);
5447 0 : std::string channelKey = std::string(commParam->channelTag) + ":" + std::to_string(commParam->engine) + ":" +
5448 0 : std::to_string(remoteResV2.remoteRank) + ":" + std::to_string(linkType);
5449 0 : HCCL_INFO("%s channelKey[%s]", __func__, channelKey.c_str());
5450 0 : if (channelHandleMap_.find(channelKey) != channelHandleMap_.end()) {
5451 0 : HCCL_ERROR("[%s]the channel has existed.", __func__);
5452 0 : return HCCL_E_INTERNAL;
5453 : }
5454 :
5455 0 : HcclChannelP2p &channelP2p = remoteResV2.channelP2p;
5456 0 : if (channelP2p.localIpcSignal[0].resId == INVALID_U64) {
5457 0 : HCCL_ERROR("[%s]the Channel is invalid",__func__);
5458 0 : return HCCL_E_INTERNAL;
5459 : }
5460 :
5461 : // 创建Transport对象
5462 0 : MachinePara machinePara;
5463 0 : CHK_RET(SetTransportMachinePara(machinePara, remoteResV2.remoteRank, commParam->channelTag));
5464 0 : machinePara.notifyNum = remoteResV2.p2pNotifyNum;
5465 : // 获取localMem & remoteMem
5466 0 : TransportDeviceP2pData transDevP2pData;
5467 0 : transDevP2pData.inputBufferPtr = reinterpret_cast<void *>(channelP2p.remoteHcclbuffer.addr);
5468 0 : transDevP2pData.outputBufferPtr = reinterpret_cast<void *>(channelP2p.remoteHcclbuffer.addr);
5469 0 : if (transDevP2pData.inputBufferPtr == nullptr || transDevP2pData.outputBufferPtr == nullptr) {
5470 0 : HCCL_ERROR("[%s]input ptr[%p] or output ptr[%p] is null.", __func__,
5471 : transDevP2pData.inputBufferPtr, transDevP2pData.outputBufferPtr);
5472 0 : return HCCL_E_PARA;
5473 : }
5474 : // 获取Notify资源
5475 0 : CHK_RET(SetChannelP2pNotify(transDevP2pData, remoteResV2.p2pNotifyNum, channelP2p)); // 待确认notify是否
5476 : // 获取transportAttr信息
5477 0 : transDevP2pData.transportAttr = channelP2p.transportAttr;
5478 : // 创建Transport对象
5479 0 : std::shared_ptr<Transport> link;
5480 0 : TransportPara para{};
5481 0 : const std::unique_ptr<NotifyPool> notifyPool;
5482 0 : DispatcherCtx *ctx = static_cast<DispatcherCtx *>(dispatcherCtx_);
5483 0 : CHK_PRT(ctx->SetDispatcherHcclQos(remoteResV2.channelP2p.qos)); // 调度器添加hcclQos
5484 0 : CHK_PTR_NULL(ctx);
5485 0 : link.reset(new (std::nothrow) Transport(
5486 0 : TransportType::TRANS_TYPE_DEVICE_P2P, para, ctx->GetDispatcher(), notifyPool, machinePara, transDevP2pData));
5487 0 : CHK_SMART_PTR_NULL(link);
5488 0 : CHK_RET(link->Init()); // 初始化需要增加远端用户注册内存
5489 :
5490 0 : ChannelHandle channelHandle = reinterpret_cast<ChannelHandle>(link.get());
5491 0 : channelHandleMap_[channelKey] = channelHandle;
5492 0 : linkMap_[channelHandle] = link;
5493 :
5494 : // 恢复出的channelHandle回填到commParam中
5495 0 : CHK_PTR_NULL(commParam->channelList);
5496 0 : ChannelHandle* channelList = reinterpret_cast<ChannelHandle*>(commParam->channelList);
5497 0 : channelList[channelIndex] = channelHandle;
5498 :
5499 0 : return HCCL_SUCCESS;
5500 0 : }
5501 :
5502 0 : HcclResult HcclCommAicpu::SetChannelP2pNotify(TransportDeviceP2pData &transDevP2pData,
5503 : u64 &p2pNotifyNum, HcclChannelP2p &channelP2p)
5504 : {
5505 0 : u64 actualNotifyNum = 0;
5506 : // 获取Ipc notify信息
5507 0 : CHK_RET(CheckNotifyOrQPMaxNum(actualNotifyNum, LINK_P2P_MAX_NUM, true));
5508 0 : std::shared_ptr<LocalNotify> ipcPreWaitNotify = std::make_shared<LocalNotify>();
5509 0 : CHK_RET(InitAndVerifySingleSignal(channelP2p.localIpcSignal[actualNotifyNum], ipcPreWaitNotify));
5510 0 : transDevP2pData.ipcPreWaitNotify = ipcPreWaitNotify;
5511 :
5512 0 : std::shared_ptr<RemoteNotify> ipcPreRecordNotify = std::make_shared<RemoteNotify>();
5513 0 : CHK_RET(InitAndVerifySingleSignal(channelP2p.remoteIpcSignal[actualNotifyNum], ipcPreRecordNotify));
5514 0 : transDevP2pData.ipcPreRecordNotify = ipcPreRecordNotify;
5515 0 : actualNotifyNum++;
5516 :
5517 0 : CHK_RET(CheckNotifyOrQPMaxNum(actualNotifyNum, LINK_P2P_MAX_NUM, true));
5518 0 : std::shared_ptr<LocalNotify> ipcPostWaitNotify = std::make_shared<LocalNotify>();
5519 0 : CHK_RET(InitAndVerifySingleSignal(channelP2p.localIpcSignal[actualNotifyNum], ipcPostWaitNotify));
5520 0 : transDevP2pData.ipcPostWaitNotify = ipcPostWaitNotify;
5521 :
5522 0 : std::shared_ptr<RemoteNotify> ipcPostRecordNotify = std::make_shared<RemoteNotify>();
5523 0 : CHK_RET(InitAndVerifySingleSignal(channelP2p.remoteIpcSignal[actualNotifyNum], ipcPostRecordNotify));
5524 0 : transDevP2pData.ipcPostRecordNotify = ipcPostRecordNotify;
5525 0 : actualNotifyNum++;
5526 :
5527 0 : transDevP2pData.userLocalNotify.resize(p2pNotifyNum, nullptr);
5528 0 : transDevP2pData.userRemoteNotify.resize(p2pNotifyNum, nullptr);
5529 :
5530 0 : for (u32 idx = 0; idx < p2pNotifyNum; idx++) {
5531 0 : std::shared_ptr<LocalNotify> ipcWaitNotify = std::make_shared<LocalNotify>();
5532 0 : CHK_RET(InitAndVerifySingleSignal(channelP2p.localIpcSignal[actualNotifyNum], ipcWaitNotify));
5533 0 : transDevP2pData.userLocalNotify[idx] = ipcWaitNotify;
5534 :
5535 0 : std::shared_ptr<RemoteNotify> ipcRecordNotify = std::make_shared<RemoteNotify>();
5536 0 : CHK_RET(InitAndVerifySingleSignal(channelP2p.remoteIpcSignal[actualNotifyNum], ipcRecordNotify));
5537 0 : transDevP2pData.userRemoteNotify[idx] = ipcRecordNotify;
5538 :
5539 0 : actualNotifyNum++;
5540 0 : }
5541 :
5542 0 : HCCL_DEBUG("%s get p2pNotify success, p2pNotifyNum[%llu], actualNotifyNum[%llu]",
5543 : __func__, p2pNotifyNum, actualNotifyNum);
5544 0 : return HCCL_SUCCESS;
5545 0 : }
5546 :
5547 0 : HcclResult HcclCommAicpu::InitRoceChannel(HcclIndOpChannelRemoteResV3 *commParam, uint32_t channelIndex)
5548 : {
5549 0 : HcclIndOpChannelRemoteResV2 &remoteResV2 = commParam->remoteResV2[channelIndex];
5550 0 : std::string channelKey = std::string(commParam->channelTag) + ":" + std::to_string(commParam->engine) + ":" +
5551 0 : std::to_string(remoteResV2.remoteRank) + ":" + std::to_string(CommProtocol::COMM_PROTOCOL_ROCE);
5552 0 : if (channelHandleMap_.find(channelKey) != channelHandleMap_.end()) {
5553 0 : HCCL_ERROR("[%s]the channel has existed.", __func__);
5554 0 : return HCCL_E_INTERNAL;
5555 : }
5556 :
5557 0 : HcclChannelRoce &channelRoce = remoteResV2.channelRoce;
5558 0 : if (channelRoce.localNotifyList == 0) {
5559 0 : HCCL_ERROR("[%s]the Channel is invalid",__func__);
5560 0 : return HCCL_E_INTERNAL;
5561 : }
5562 0 : HcclSignalInfo *localNotifyList = reinterpret_cast<HcclSignalInfo *>(channelRoce.localNotifyList);
5563 0 : if (localNotifyList[0].resId == INVALID_U64) {
5564 0 : HCCL_INFO("[%s]the channel notify resource is invalid", __func__);
5565 0 : return HCCL_E_INTERNAL;
5566 : }
5567 :
5568 : // 创建Transport对象
5569 0 : MachinePara machinePara;
5570 0 : CHK_RET(SetTransportMachinePara(machinePara, remoteResV2.remoteRank, commParam->channelTag)); //待确认是否填充完毕
5571 0 : machinePara.notifyNum = remoteResV2.roceNotifyNum;
5572 : // 获取localMem & remoteMem
5573 0 : TransportDeviceIbverbsData transDevIbverbsData;
5574 0 : transDevIbverbsData.inputBufferPtr = reinterpret_cast<void *>(channelRoce.remoteHcclbuffer.addr);
5575 0 : transDevIbverbsData.outputBufferPtr = reinterpret_cast<void *>(channelRoce.remoteHcclbuffer.addr);
5576 0 : if (transDevIbverbsData.inputBufferPtr == nullptr || transDevIbverbsData.outputBufferPtr == nullptr) {
5577 0 : HCCL_ERROR("[%s]input ptr[%p] or output ptr[%p] is null.", __func__,
5578 : transDevIbverbsData.inputBufferPtr, transDevIbverbsData.outputBufferPtr);
5579 0 : return HCCL_E_PARA;
5580 : }
5581 0 : transDevIbverbsData.localInputMem = channelRoce.localHcclbuffer;
5582 0 : transDevIbverbsData.localOutputMem = channelRoce.localHcclbuffer;
5583 0 : transDevIbverbsData.localNotifyValueAddr = channelRoce.notifyValue;
5584 0 : transDevIbverbsData.notifyValueKey = channelRoce.notifyValueKey;
5585 0 : transDevIbverbsData.remoteInputKey = channelRoce.remoteHcclbuffer.key;
5586 0 : transDevIbverbsData.remoteOutputKey = channelRoce.remoteHcclbuffer.key;
5587 : // 需要添加远端用户注册内存
5588 :
5589 : // 获取QPinfo
5590 0 : u32 roceQpNumSum = channelRoce.qpsPerConnection + static_cast<u32>(channelRoce.qpsPerConnection != 1);
5591 0 : transDevIbverbsData.qpInfo.resize(roceQpNumSum);
5592 0 : std::copy_n(channelRoce.QpInfo, roceQpNumSum, transDevIbverbsData.qpInfo.begin());
5593 0 : transDevIbverbsData.multiQpThreshold = multiQpThreshold_;
5594 0 : transDevIbverbsData.qpsPerConnection = channelRoce.qpsPerConnection;
5595 :
5596 : // 获取notify
5597 0 : u64 &roceNotifyNum = remoteResV2.roceNotifyNum; // 是否需要待确认
5598 0 : CHK_RET(SetChannelRoceNotify(transDevIbverbsData, roceNotifyNum, channelRoce));
5599 :
5600 : // 创建Transport对象
5601 0 : std::shared_ptr<Transport> link;
5602 0 : TransportPara para{};
5603 0 : para.timeout = linkTimeOut_; // 暂无法设置
5604 0 : const std::unique_ptr<NotifyPool> notifyPool;
5605 0 : DispatcherCtx *ctx = static_cast<DispatcherCtx *>(dispatcherCtx_);
5606 0 : CHK_PTR_NULL(ctx);
5607 0 : link.reset(new (std::nothrow) Transport(
5608 : TransportType::TRANS_TYPE_DEVICE_IBVERBS, para, ctx->GetDispatcher(), notifyPool,
5609 0 : machinePara, TransportDeviceP2pData(), transDevIbverbsData));
5610 0 : CHK_SMART_PTR_NULL(link);
5611 0 : CHK_RET(link->Init()); // 初始化需要增加远端用户注册内存
5612 :
5613 0 : ChannelHandle channelHandle = reinterpret_cast<ChannelHandle>(link.get());
5614 0 : channelHandleMap_[channelKey] = channelHandle;
5615 0 : linkMap_[channelHandle] = link;
5616 :
5617 : // 恢复出的channelHandle回填到commParam中
5618 0 : ChannelHandle* channelList = reinterpret_cast<ChannelHandle*>(commParam->channelList);
5619 0 : channelList[channelIndex] = channelHandle;
5620 :
5621 0 : return HCCL_SUCCESS;
5622 0 : }
5623 :
5624 0 : HcclResult HcclCommAicpu::SetChannelRoceNotify(TransportDeviceIbverbsData &transDevIbverbsData,
5625 : u64 &roceNotifyNum, HcclChannelRoce &channelRoce)
5626 : {
5627 0 : u64 actualNotifyNum = 0;
5628 0 : if (channelRoce.localNotifyList == 0 || channelRoce.remoteNotifyList == 0)
5629 : {
5630 0 : HCCL_DEBUG("[%s] Empty local and remote notify lists, skipping notify resource creation.", __func__);
5631 0 : return HCCL_SUCCESS;
5632 : }
5633 0 : HcclSignalInfo *localNotifyList = reinterpret_cast<HcclSignalInfo *>(channelRoce.localNotifyList);
5634 0 : AddrKey *remoteNotifyList = reinterpret_cast<AddrKey *>(channelRoce.remoteNotifyList);
5635 0 : if (localNotifyList == nullptr || remoteNotifyList == nullptr) {
5636 0 : HCCL_ERROR("[%s]nullptr found in localNotifyList or remoteNotifyList from device mem, check.", __func__);
5637 0 : return HCCL_E_INTERNAL;
5638 : }
5639 : // 获取RDMA Notify信息
5640 0 : std::shared_ptr<LocalNotify> ackNotify = std::make_shared<LocalNotify>();
5641 0 : CHK_RET(InitAndVerifySingleSignal(localNotifyList[actualNotifyNum], ackNotify));
5642 0 : transDevIbverbsData.ackNotify = ackNotify;
5643 0 : transDevIbverbsData.remoteAckNotifyDetails = remoteNotifyList[actualNotifyNum];
5644 0 : actualNotifyNum++;
5645 :
5646 0 : std::shared_ptr<LocalNotify> dataNotify = std::make_shared<LocalNotify>();
5647 0 : CHK_RET(InitAndVerifySingleSignal(localNotifyList[actualNotifyNum], dataNotify));
5648 0 : transDevIbverbsData.dataNotify = dataNotify;
5649 0 : transDevIbverbsData.remoteDataNotifyDetails = remoteNotifyList[actualNotifyNum];
5650 0 : actualNotifyNum++;
5651 :
5652 0 : std::shared_ptr<LocalNotify> dataAckNotify = std::make_shared<LocalNotify>();
5653 0 : CHK_RET(InitAndVerifySingleSignal(localNotifyList[actualNotifyNum], dataAckNotify));
5654 0 : transDevIbverbsData.dataAckNotify = dataAckNotify;
5655 0 : transDevIbverbsData.remoteDataAckNotifyDetails = remoteNotifyList[actualNotifyNum];
5656 0 : transDevIbverbsData.notifySize = notifySize_;
5657 0 : actualNotifyNum++;
5658 :
5659 0 : transDevIbverbsData.userLocalNotify.resize(channelRoce.qpsPerConnection);
5660 0 : transDevIbverbsData.userRemoteNotifyDetails.resize(channelRoce.qpsPerConnection);
5661 : // 当前多QP下每个QP会多申请一个DataNotify
5662 0 : u64 singleQpNotifySize = channelRoce.singleQPNotifyNum + static_cast<u32>(channelRoce.qpsPerConnection > 1);
5663 0 : for (u32 qpIndex = 0; qpIndex < channelRoce.qpsPerConnection; qpIndex++) {
5664 0 : transDevIbverbsData.userLocalNotify[qpIndex].resize(singleQpNotifySize, nullptr);
5665 0 : transDevIbverbsData.userRemoteNotifyDetails[qpIndex].resize(singleQpNotifySize);
5666 0 : for (u32 i = 0, idx = actualNotifyNum + singleQpNotifySize * qpIndex; i < singleQpNotifySize; ++idx, ++i) {
5667 0 : std::shared_ptr<LocalNotify> locNotify = std::make_shared<LocalNotify>();
5668 0 : CHK_RET(InitAndVerifySingleSignal(localNotifyList[idx], locNotify));
5669 0 : transDevIbverbsData.userLocalNotify[qpIndex][i] = locNotify;
5670 0 : transDevIbverbsData.userRemoteNotifyDetails[qpIndex][i] = remoteNotifyList[idx];
5671 0 : }
5672 : }
5673 0 : roceNotifyNum = channelRoce.singleQPNotifyNum;
5674 0 : HCCL_DEBUG("[%s]get roceNotify success, roceNotifyNum[%u]", __func__, roceNotifyNum);
5675 0 : return HCCL_SUCCESS;
5676 0 : }
5677 :
5678 0 : HcclResult HcclCommAicpu::NotifyAlloc(NotifyMgrAicpuParam *param)
5679 : {
5680 0 : u32 notifyNum = param->notifyNum;
5681 0 : std::string notifysStr = std::string(param->notifyParam, NOTIFY_UNIQUE_ID_MAX_SIZE);
5682 0 : std::string hcomId(param->hcomId);
5683 0 : size_t notifySize = notifys_.size();
5684 0 : HCCL_INFO("[HcclCommAicpu][%s] comm identifier[%s], alloc notifys num[%u] begin, before notifySize[%u]",
5685 : __func__, hcomId.c_str(), notifyNum, notifySize);
5686 0 : if (UNLIKELY(HcclCheckLogLevel(HCCL_LOG_INFO))) {
5687 0 : std::ostringstream oss;
5688 0 : oss << "notifyParam" << " raw bytes: ";
5689 0 : for (u32 i = 0; i < NOTIFY_UNIQUE_ID_MAX_SIZE; ++i) {
5690 0 : oss << std::hex << std::setw(2) << std::setfill('0')
5691 0 : << static_cast<unsigned int>(static_cast<unsigned char>(param->notifyParam[i])) << " ";
5692 : }
5693 0 : HCCL_INFO("[HcclCommAicpu][%s] %s", __func__, oss.str().c_str());
5694 0 : }
5695 0 : HcclResult ret = NotifyManager::ParseBinNotifys(notifysStr, notifys_);
5696 0 : if (ret != HCCL_SUCCESS) {
5697 0 : HCCL_ERROR("[HcclCommAicpu][%s] comm identifier[%s], alloc notifys num[%u] failed %u",
5698 : __func__, hcomId.c_str(), notifyNum, ret);
5699 0 : return ret;
5700 : }
5701 0 : HCCL_INFO("[HcclCommAicpu][%s] comm identifier[%s], alloc notifys num[%u] end, after notifySize[%u]",
5702 : __func__, hcomId.c_str(), notifyNum, notifys_.size());
5703 0 : NotifyHandle *notifyArray = static_cast<NotifyHandle*>(param->deviceHandle);
5704 0 : CHK_PTR_NULL(notifyArray);
5705 : // 空指针校验
5706 0 : for (size_t i = 0; i < notifyNum; ++i) {
5707 0 : notifyArray[i] = reinterpret_cast<NotifyHandle>(notifys_[i + notifySize].get()); // 拷贝裸指针
5708 0 : HCCL_INFO("[HcclCommAicpu][%s] notifyArray[%u] = [%lu]", __func__, i + notifySize, notifyArray[i]);
5709 : }
5710 :
5711 0 : HCCL_INFO("[HcclCommAicpu][%s] comm identifier[%s], alloc notifys num[%u] success",
5712 : __func__, hcomId.c_str(), notifyNum);
5713 0 : return HCCL_SUCCESS;
5714 0 : }
5715 :
5716 0 : HcclResult HcclCommAicpu::NotifyFree(NotifyMgrAicpuParam *param)
5717 : {
5718 0 : u32 notifyNum = param->notifyNum;
5719 0 : NotifyHandle *notifyArray = static_cast<NotifyHandle*>(param->deviceHandle);
5720 0 : std::string hcomId(param->hcomId);
5721 : // 空指针校验
5722 0 : CHK_PTR_NULL(notifyArray);
5723 0 : for (size_t i = 0; i < notifyNum; ++i) {
5724 0 : LocalNotify* notify = reinterpret_cast<LocalNotify*>(notifyArray[i]);
5725 0 : HCCL_INFO("[HcclCommAicpu][%s] notifyArray[%u]=[%lu]", __func__, i, notifyArray[i]);
5726 0 : auto it = std::find_if(notifys_.begin(), notifys_.end(),
5727 0 : [notify](const std::unique_ptr<LocalNotify>& ptr) {
5728 0 : return ptr.get() == notify;
5729 : });
5730 0 : if (it != notifys_.end()) {
5731 0 : HCCL_INFO("[HcclCommAicpu][%s] comm identifier[%s], free notifys[%u] success",
5732 : __func__, hcomId.c_str(), notifyArray[i]);
5733 0 : notifys_.erase(it);
5734 : } else {
5735 0 : HCCL_RUN_WARNING("[HcclCommAicpu][%s] localNotify[%u] not found in notifys_", __func__, i);
5736 : }
5737 : }
5738 :
5739 0 : HCCL_INFO("[HcclCommAicpu][%s] comm identifier[%s], free notifys num[%u] success",
5740 : __func__, hcomId.c_str(), notifyNum);
5741 0 : return HCCL_SUCCESS;
5742 0 : }
5743 :
5744 0 : HcclResult HcclCommAicpu::RegisterOpInfo(void* opInfo, u32 size)
5745 : {
5746 0 : CHK_RET(taskExecption_.RegisterOpInfo(opInfo, size));
5747 0 : u32 opIdx = taskExecption_.GetOpRingBufferIdx();
5748 0 : CHK_RET(SetDispatcherCtxOpIdx(opIdx));
5749 0 : HCCL_INFO("%s success, group[%s], opRingBufferId[%u]", __func__, identifier_.c_str(), opIdx);
5750 0 : return HCCL_SUCCESS;
5751 : }
5752 :
5753 0 : HcclResult HcclCommAicpu::RegOpTaskException(HcommGetOpInfoCallback callback)
5754 : {
5755 0 : CHK_RET(taskExecption_.RegisterOpInfoCallback(callback));
5756 0 : return HCCL_SUCCESS;
5757 : }
5758 :
5759 0 : HcclResult HcclCommAicpu::SetDispatcherCtxOnThread()
5760 : {
5761 : // 设置 DispatcherCtx 到线程变量
5762 0 : CHK_RET(SetDispatcherCtx(dispatcherCtx_));
5763 0 : return HCCL_SUCCESS;
5764 : }
5765 : } // namespace hccl
|
