Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include <iomanip>
12 : #include "aicpu_communicator.h"
13 : #include "dispatcher.h"
14 : #include "aicpu_hccl_process.h"
15 : #include "coll_alg_exec_registry.h"
16 : #include "coll_all_to_all_executor.h"
17 : #include "common/aicpu_hccl_common.h"
18 : #include "executor_tracer.h"
19 : #include "profiling_manager_device.h"
20 : #include "utils/aicpu_hdc_utils.h"
21 : #include "comm_engine_utils.h"
22 : #include "utils/hccl_aicpu_utils.h"
23 : #include "framework/aicpu_hdc.h"
24 : #include "common/aicpu_sqe_context.h"
25 : #include "coll_batch_send_recv_retry_executor.h"
26 : #include "log_control.h"
27 : #include "aicpu_hccl_sqcq.h"
28 : #include "aicpu_hccl_sqcqv1.h"
29 : #include "sal_pub.h"
30 : #include "externalinput_pub.h"
31 : #include "env_config.h"
32 : #include "config_log.h"
33 : #include "aicpu_one_side_service.h"
34 : #include "notify_manager.h"
35 : #include "dispatcher_aicpu.h"
36 : #include "dlprof_function.h"
37 : #include "profiling_command_handle.h"
38 : #include "dispatcher_ctx.h"
39 : #include "aicpu_res_package_helper.h"
40 : #include "aicpu_symmetric_memory.h"
41 :
42 : namespace hccl {
43 : constexpr u32 IPC_SIGNAL_MODULUS = 2;
44 : constexpr u32 RDMA_SIGNAL_MODULUS = 3;
45 : constexpr u32 KEY_VALUE_TO_VECTOR_MODULUS = 2;
46 :
47 : constexpr u32 BSR_RETRY_SEND_STREAM_INDEX = 0;
48 : constexpr u32 BSR_RETRY_RECV_STREAM_INDEX = 1;
49 : constexpr u32 BSR_RETRY_STREAM_NUM = 2;
50 : constexpr u32 MAX_REPORT_STATUS = 100U; // reportStatus的最大缓存数量
51 : constexpr u32 INPUT = 0;
52 : constexpr u32 OUTPUT = 1;
53 : constexpr u32 AICPU_RETRY_LINKROCE_DEFAULT = 0;
54 : constexpr u32 AICPU_RETRY_LINKROCE_BACKUP = 1;
55 :
56 : constexpr u32 BSR_RETRY_SENDRECV_PAIR_NUM_MAX = 2;
57 : constexpr u32 BSR_RETRY_SENDRECV_PAIR_INDEX_0 = 0;
58 : constexpr u32 BSR_RETRY_SENDRECV_PAIR_INDEX_1 = 1;
59 :
60 : constexpr u32 NOTIFY_SIZE_FOUR = 4;
61 : constexpr u32 NOTIFY_SIZE_EIGHT = 8;
62 :
63 : bool HcclCommAicpu::errMessageReport_ = true;
64 :
65 : #define HCCL_RETRY_CHK_RET_AND_TRANS_FSM(result__, exeLog__, error__, state__) \
66 : do { \
67 : if (UNLIKELY((result__) != HCCL_SUCCESS)) { \
68 : exeLog__; \
69 : errorCode = (error__); \
70 : fsmState = (state__); \
71 : return (result__); \
72 : } \
73 : } while (0)
74 :
75 363 : HcclCommAicpu::HcclCommAicpu() { HCCL_RUN_INFO("Construct HcclCommAicpu complete."); }
76 :
77 288 : HcclCommAicpu::~HcclCommAicpu()
78 : {
79 32 : if (UtraceInfo_ != nullptr) {
80 11 : UtraceInfo_->DeInit();
81 11 : UtraceInfo_ = nullptr;
82 : }
83 32 : if (dispatcher_ != nullptr) {
84 11 : HcclDispatcherDestroy(dispatcher_);
85 11 : dispatcher_ = nullptr;
86 : }
87 32 : if (dispatcherCtx_ != nullptr) {
88 0 : DestroyDispatcherCtx(dispatcherCtx_, identifier_.c_str());
89 0 : HCCL_DEBUG("[%s] destroy dispatcherCtx[%p] group[%s] success!", __func__, dispatcherCtx_, identifier_.c_str());
90 0 : dispatcherCtx_ = nullptr;
91 : }
92 :
93 32 : commPlaneVector_.clear();
94 32 : isBridgeVector_.clear();
95 32 : indOpCommInitialized_ = false;
96 32 : initialized_ = false;
97 :
98 32 : HCCL_RUN_INFO("Destruct HcclCommAicpu group[%s] success!", identifier_.c_str());
99 352 : }
100 :
101 13 : HcclResult HcclCommAicpu::Init(const HcclOpResParam* commParam, bool isCustom)
102 : {
103 13 : if (initialized_) {
104 2 : HCCL_RUN_INFO("[%s][Init]Group[%s] already initialized, skip reinit", __func__, identifier_.c_str());
105 2 : return HCCL_SUCCESS;
106 : }
107 :
108 11 : CHK_PTR_NULL(commParam);
109 11 : identifier_ = commParam->hcomId;
110 11 : isCustom_ = isCustom;
111 11 : HCCL_RUN_INFO(
112 : "[HcclCommAicpu][Init]Entry-Init group[%s], rankSize[%u], isCustom[%d].", identifier_.c_str(),
113 : commParam->rankSize, isCustom_);
114 11 : CHK_RET(aicpuShareData_.Init(commParam->aicpuCustomParamAddr, commParam->aicpuCustomParamSize));
115 11 : CHK_RET(SetHrtWorkMode(commParam));
116 11 : CHK_RET(SetHrtDeviceSatMode(commParam));
117 11 : CHK_RET(InitConfigInfo(commParam));
118 11 : CHK_RET(InitCclbuffer(commParam));
119 11 : CHK_RET(InitTopoInfo(commParam));
120 11 : CHK_RET(InitOpNotifyObj(commParam));
121 11 : CHK_RET(HcclDispatcherAicpuInit(&dispatcher_, devId_, commParam->hcclSdmaQos, DispatcherType::DISPATCHER_AICPU));
122 11 : CHK_RET(RegisterProfilingCallback());
123 11 : CHK_RET(InitLocalNotifyObj(commParam));
124 11 : CHK_RET(InitMainStreamObj(commParam));
125 11 : CHK_RET(InitSlaveStreamObjs(commParam));
126 11 : CHK_RET(InitOrderStreamObj(commParam));
127 11 : CHK_RET(InitLocalTagRes(commParam->localRes.nextTagRes));
128 11 : CHK_RET(InitTimeOutConfig(commParam));
129 11 : CHK_RET(InitHostDeviceLock(commParam));
130 11 : CHK_RET(InitTopoMatcher());
131 11 : CHK_RET(InitOpRetry(commParam));
132 11 : CHK_RET(RegisterDispatcherCallback());
133 11 : CHK_RET(InitTinyMem(commParam));
134 11 : CHK_RET(InitProfResource());
135 11 : CHK_RET(InitZeroCopyExchanger(commParam));
136 11 : CHK_RET(InitOpCounter(commParam->opCounterInfo));
137 11 : CHK_RET(InitUtraceInfo(commParam));
138 11 : CHK_RET(aicpuCacheManager_.InitOpUnfoldCache());
139 11 : CHK_RET(RegisterProfCallBack());
140 11 : InitCommInfoStatus(true);
141 11 : SetCommInfoStreamStatus(true);
142 :
143 11 : initialized_ = true;
144 :
145 11 : HCCL_RUN_INFO("[HcclCommAicpu][Init] group[%s] success!", identifier_.c_str());
146 11 : return HCCL_SUCCESS;
147 : }
148 :
149 11 : HcclResult HcclCommAicpu::InitUtraceInfo(const HcclOpResParam* commParam)
150 : {
151 11 : u32 hostpid = 0;
152 11 : u32 cpType = DEVDRV_PROCESS_CPTYPE_MAX;
153 11 : CHK_RET(HrtHalDrvQueryProcessHostPid(getpid(), nullptr, nullptr, &hostpid, &cpType));
154 :
155 : HcclTraceInfo::UtraceAttr utraceAttr;
156 11 : utraceAttr.utraceStatusFlag = commParam->utraceStatusFlag;
157 11 : utraceAttr.deviceid = GetDevId();
158 11 : utraceAttr.pid = hostpid;
159 11 : UtraceInfo_.reset(new (std::nothrow) HcclTraceInfo(utraceAttr));
160 11 : CHK_PTR_NULL(UtraceInfo_);
161 :
162 : /* 申请trace资源信息 */
163 11 : std::string logInfo = "HCCL_";
164 11 : logInfo.append(std::to_string(SalGetTid()));
165 11 : logInfo.append("_");
166 11 : logInfo.append(std::to_string(GetDevId()));
167 11 : CHK_RET(UtraceInfo_->Init(logInfo));
168 11 : return HCCL_SUCCESS;
169 11 : }
170 :
171 11 : HcclResult HcclCommAicpu::InitProfResource()
172 : {
173 11 : groupHashId_ = dfx::ProfilingManager::GetProfHashId(identifier_.c_str(), identifier_.length());
174 11 : HCCL_RUN_INFO("[Init][ProfResource]group[%s], groupHashId_[%llu].", identifier_.c_str(), groupHashId_);
175 :
176 11 : dfx::ProfCommInfo profInfo{groupHashId_, topoInfo_.userRankSize, topoInfo_.userRank};
177 11 : CHK_RET(dfx::ProfilingManager::AddProfInfoByStreamId(mainStream_.id(), identifier_, profInfo));
178 451 : for (auto& slaveStream : slaveStreams_) {
179 440 : CHK_RET(dfx::ProfilingManager::AddProfInfoByStreamId(slaveStream.id(), identifier_, profInfo));
180 : }
181 11 : dfx::ProfilingExtendInfoHelper::InitProfItemId();
182 11 : return HCCL_SUCCESS;
183 : }
184 :
185 462 : HcclResult HcclCommAicpu::StreamRestore(u32 streamId)
186 : {
187 462 : HcclResult ret = hrtHalResourceIdRestore(devId_, 0, DRV_STREAM_ID, streamId, 0);
188 : // custom进程需要恢复stream资源, custom进程调用失败直接报错，aicpu进程调用失败做兼容性处理
189 462 : if (ret == HCCL_E_NOT_SUPPORT) {
190 0 : CHK_PRT_RET(
191 : isCustom_,
192 : HCCL_ERROR(
193 : "%s hrtHalResourceIdRestore fail, drv not support, custom[%d], ret[%d]", __func__, isCustom_, ret),
194 : HCCL_E_DRV);
195 462 : } else if (ret != HCCL_SUCCESS) {
196 0 : HCCL_ERROR("%s hrtHalResourceIdRestore fail, ret[%d]", __func__, ret);
197 0 : return HCCL_E_DRV;
198 : }
199 462 : return HCCL_SUCCESS;
200 : }
201 :
202 11 : HcclResult HcclCommAicpu::SetHrtWorkMode(const HcclOpResParam* commParam)
203 : {
204 11 : CHK_RET(hrtSetWorkModeAicpu(true));
205 11 : CHK_RET(hrtSetlocalDevice(commParam->topoInfo.deviceLogicId));
206 11 : CHK_RET(hrtSetlocalDeviceType(static_cast<DevType>(commParam->topoInfo.deviceType)));
207 11 : return HCCL_SUCCESS;
208 : }
209 :
210 11 : HcclResult HcclCommAicpu::SetHrtDeviceSatMode(const HcclOpResParam* commParam)
211 : {
212 11 : CHK_RET(hrtSetLocalDeviceSatMode(commParam->config.floatOverflowMode));
213 11 : HCCL_RUN_INFO(
214 : "[HcclCommAicpu][Init]SetHrtDeviceSatMode[%d]", static_cast<u32>(commParam->config.floatOverflowMode));
215 11 : return HCCL_SUCCESS;
216 : }
217 :
218 11 : HcclResult HcclCommAicpu::InitTopoMatcher()
219 : {
220 11 : externalEnable_.enableFfts = 1; // FFTS+在算法模块不会使用多线程，固定使能
221 11 : externalEnable_.deterministic = deterministic_;
222 11 : externalEnable_.intraRoceSwitch = 0;
223 11 : externalEnable_.dumpDebug = dumpDebug_;
224 11 : externalEnable_.interHccsDisable = interHccsDisable_;
225 :
226 11 : topoMatcher_.reset((new (std::nothrow) TopoMatcher(
227 22 : commPlaneVector_, isBridgeVector_, topoInfo_, algoInfo_, externalEnable_, serverAndsuperPodToRank_)));
228 11 : CHK_SMART_PTR_NULL(topoMatcher_);
229 11 : HCCL_RUN_INFO(
230 : "[HcclCommAicpu][InitTopoMatcher]topo matcher init success. group[%s] deterministic:%u, "
231 : "dumpDebug:%u, interHccsDisable:%u",
232 : identifier_.c_str(), deterministic_, dumpDebug_, interHccsDisable_);
233 11 : return HCCL_SUCCESS;
234 : }
235 :
236 11 : HcclResult HcclCommAicpu::InitOpRetry(const HcclOpResParam* commParam)
237 : {
238 11 : CHK_PTR_NULL(commParam);
239 11 : retryEnable_ = (commParam->config.retryEnable == 1) ? true : false;
240 11 : retryHoldTime_ = commParam->config.retryHoldTime;
241 11 : retryIntervalTime_ = commParam->config.retryIntervalTime;
242 11 : HCCL_RUN_INFO(
243 : "[InitOpRetry]retryEnable[%d], retryHoldTime[%u ms], retryIntervalTime[%u ms]", retryEnable_, retryHoldTime_,
244 : retryIntervalTime_);
245 :
246 11 : if (commParam->kfcControlTransferH2DParams.buffLen != 0 && kfcControlTransferH2D_ == nullptr) {
247 0 : EXCEPTION_CATCH((kfcControlTransferH2D_ = std::make_shared<hccl::HDCommunicate>()), return HCCL_E_PTR);
248 0 : CHK_SMART_PTR_NULL(kfcControlTransferH2D_);
249 0 : CHK_RET(kfcControlTransferH2D_->InitDevice(commParam->kfcControlTransferH2DParams));
250 : }
251 11 : if (commParam->kfcStatusTransferD2HParams.buffLen != 0 && kfcStatusTransferD2H_ == nullptr) {
252 0 : EXCEPTION_CATCH((kfcStatusTransferD2H_ = std::make_shared<hccl::HDCommunicate>()), return HCCL_E_PTR);
253 0 : CHK_SMART_PTR_NULL(kfcStatusTransferD2H_);
254 0 : CHK_RET(kfcStatusTransferD2H_->InitDevice(commParam->kfcStatusTransferD2HParams));
255 : }
256 11 : return HCCL_SUCCESS;
257 : }
258 :
259 11 : HcclResult HcclCommAicpu::InitZeroCopyExchanger(const HcclOpResParam* commParam)
260 : {
261 11 : CHK_PTR_NULL(commParam);
262 0 : auto nSecStopFunc = [this]() -> bool {
263 : // 检查到OP状态不Ok则认为需要终止
264 0 : auto ret = this->CheckOpExecStatus();
265 0 : if (ret == HCCL_SUCCESS) {
266 0 : return false;
267 0 : } else if (ret == HCCL_E_SUSPENDING) {
268 : // NS快恢场景，需要提前终止
269 0 : HcclOpExecFSM fsmState = HcclOpExecFSM::HCCL_OP_EXEC_FSM_INIT;
270 0 : KfcError errorCode = KfcError::kNone;
271 0 : SetCommRecoveryFlag(true);
272 0 : UpdateOpExecStatus(fsmState, KfcStatus::kStoplaunch, errorCode, 0);
273 0 : HCCL_RUN_INFO("[HcclCommAicpu][nSecStopFunc] need stop launch");
274 0 : return true;
275 : } else {
276 0 : return true;
277 : }
278 11 : };
279 :
280 11 : u32 timeoutSec = commParam->config.notifyWaitTime;
281 11 : HCCL_INFO("[HcclCommAicpu][InitZeroCopyExchanger] set timeout is [%u s]", timeoutSec);
282 :
283 11 : EXCEPTION_CATCH(
284 : (ZeroCopyExchanger_ = std::make_shared<hccl::AicpuZeroCopyExchanger>(
285 : commParam->localUsrRankId, commParam->rankSize, commParam, nSecStopFunc, timeoutSec,
286 : topoInfo_.deviceNumPerAggregation, taskMonitorInterval_)),
287 : return HCCL_E_PTR);
288 :
289 : // 通信域第一次初始化时，如果IPC内存有不为空的则认为是使能该特性
290 11 : isZeroCopy_ = false;
291 363 : for (u32 i = 0; i < AICPU_ZERO_COPY_MAX_DEVICE_NUM_A3; ++i) {
292 352 : if (commParam->zeroCopyIpcPtrs[i] != 0) {
293 0 : isZeroCopy_ = true;
294 0 : break;
295 : }
296 : }
297 :
298 11 : return HCCL_SUCCESS;
299 : }
300 :
301 11 : HcclResult HcclCommAicpu::InitOpCounter(const OpCounterInfo& opCounterInfo)
302 : {
303 11 : if (opCounterInfo.isEnableCounter && !retryEnable_
304 0 : && (opCounterInfo.headCountMem == 0 || opCounterInfo.tailCountMem == 0 || opCounterInfo.memSize == 0)) {
305 0 : HCCL_ERROR("[HcclCommAicpu][InitOpCounter] headCountMem or tailCountMem or memSize is null");
306 0 : return HCCL_E_PARA;
307 : }
308 11 : opCounterInfo_ = opCounterInfo;
309 11 : return HCCL_SUCCESS;
310 : }
311 :
312 1 : void HcclCommAicpu::SetZeroCopyEnable(bool enable) { isZeroCopy_ = enable; }
313 :
314 1 : void HcclCommAicpu::SetSymmetricMemoryEnable(bool enable)
315 : {
316 1 : HCCL_INFO("[HcclCommAicpu::SetSymmetricMemoryEnable] enable[%d]", enable);
317 1 : isSymmetricMemory_ = enable;
318 1 : aicpuCacheManager_.SetSymmetricMemoryEnable(enable);
319 1 : }
320 :
321 0 : HcclResult HcclCommAicpu::PrepareZeroCopyExchanger(
322 : const std::string& newTag, OpParam& opParam, AlgResourceResponse* algResResponse)
323 : {
324 0 : return ZeroCopyExchanger_->ExchangeAddress(newTag, opParam.inputPtr, opParam.outputPtr, algResResponse);
325 : }
326 :
327 11 : HcclResult HcclCommAicpu::RegisterProfilingCallback()
328 : {
329 11 : (void)RegisterLoadTaskCallBack(dispatcher_, nullptr, dfx::TaskProfilingCallBack);
330 :
331 11 : return HCCL_SUCCESS;
332 : }
333 :
334 11 : HcclResult HcclCommAicpu::RegisterDispatcherCallback()
335 : {
336 0 : auto checkOpExecStatusCallback = [this]() {
337 0 : return this->CheckOpExecStatusCallback();
338 11 : };
339 :
340 11 : return HcclSetOpExecStatusCallback(dispatcher_, checkOpExecStatusCallback);
341 : }
342 :
343 11 : HcclResult HcclCommAicpu::RegisterProfCallBack()
344 : {
345 11 : if (MsprofRegisterCallback != nullptr) {
346 11 : HCCL_INFO("RegisterProfCallBack not null");
347 11 : int32_t ret = MsprofRegisterCallback(AICPU, &DeviceCommandHandle);
348 11 : CHK_PRT_RET((ret != 0), HCCL_ERROR("[%s] failed. ret = [%d]", __func__, ret), HCCL_E_PARA);
349 : } else {
350 0 : HCCL_INFO("RegisterProfCallBack is null");
351 : }
352 11 : return HCCL_SUCCESS;
353 : }
354 :
355 0 : HcclResult HcclCommAicpu::GetSuspendingFlag(HcclComSuspendingFlag& flag)
356 : {
357 0 : CHK_RET(AicpuHdcUtils::GetSuspendingStatus(kfcControlTransferH2D_, flag));
358 0 : return HCCL_SUCCESS;
359 : }
360 :
361 57 : HcclResult HcclCommAicpu::BackGroundGetCmd(KfcCommand& cmd)
362 : {
363 57 : CHK_RET(aicpuHdc_.GetOpExecCtrlCmd(kfcControlTransferH2D_, cmd));
364 54 : return HCCL_SUCCESS;
365 : }
366 :
367 2 : HcclResult HcclCommAicpu::BackGroundSetStatus(KfcStatus status)
368 : {
369 2 : CHK_RET(aicpuHdc_.SetOpExecStatus(kfcStatusTransferD2H_, status, KfcError::kNone, 0));
370 2 : return HCCL_SUCCESS;
371 : }
372 :
373 0 : HcclResult HcclCommAicpu::SaveTraceInfo(std::string& logInfo)
374 : {
375 0 : CHK_PTR_NULL(UtraceInfo_);
376 0 : CHK_RET(UtraceInfo_->SaveTraceInfo(logInfo, AtraceOption::Opbasekey));
377 0 : return HCCL_SUCCESS;
378 : }
379 :
380 0 : HcclResult HcclCommAicpu::FlushUtraceInfo()
381 : {
382 0 : if (GetCommInfoStatus()) {
383 0 : CHK_RET(UtraceInfo_->Flush());
384 : }
385 0 : return HCCL_SUCCESS;
386 : }
387 :
388 0 : std::string HcclCommAicpu::GetExcuteOp()
389 : {
390 0 : std::stringstream ss;
391 0 : ss << "tag: " << excuteOpId_.tag << ", ";
392 0 : ss << "newTag: " << excuteOpId_.newTag << ", ";
393 0 : ss << "index: " << excuteOpId_.index;
394 0 : return ss.str();
395 0 : }
396 :
397 11 : void HcclCommAicpu::InitCommInfoStatus(bool commInfo) { commOpenStatus = commInfo; }
398 :
399 11 : HcclResult HcclCommAicpu::InitTinyMem(const HcclOpResParam* commParam)
400 : {
401 11 : CHK_PTR_NULL(commParam);
402 11 : auto tinyMemPtr = reinterpret_cast<void*>(commParam->tinyMem);
403 11 : tinySendRecvMem_ = DeviceMem::create(tinyMemPtr, commParam->tinyMemSize);
404 :
405 11 : return HCCL_SUCCESS;
406 : }
407 :
408 11 : HcclResult HcclCommAicpu::InitTimeOutConfig(const HcclOpResParam* commParam)
409 : {
410 11 : CHK_PTR_NULL(commParam);
411 11 : CHK_RET(HcclSetSqeTimeOut(dispatcher_, commParam->config.notifyWaitTime));
412 11 : linkTimeOut_ = commParam->config.linkTimeOut;
413 11 : return HCCL_SUCCESS;
414 : }
415 :
416 11 : HcclResult HcclCommAicpu::InitHostDeviceLock(const HcclOpResParam* commParam)
417 : {
418 11 : CHK_PTR_NULL(commParam);
419 11 : hostDeviceLock_.reset(new (std::nothrow) PetersonLock(
420 11 : reinterpret_cast<void*>(commParam->lockAddr), PetersonLock::DEFAULT_LOCK_TIMEOUT_SEC));
421 11 : CHK_SMART_PTR_NULL(hostDeviceLock_);
422 11 : CHK_RET(hostDeviceLock_->Init());
423 :
424 11 : return HCCL_SUCCESS;
425 : }
426 :
427 0 : HcclResult HcclCommAicpu::UpdateNotifyWaitTimeOut(SyncMode syncMode, u64 notifyWaitTime)
428 : {
429 0 : sqeWaitTimeOut_ = (notifyWaitTime == 0) ? notifyWaitTime : (notifyWaitTime + AICPU_SQE_TIMEOUT_INC);
430 0 : if (syncMode == SyncMode::UNLIMITED_TIMEWAITSYNCMODE) {
431 0 : CHK_RET(HcclSetSqeTimeOut(dispatcher_, GetNotifyMaxWaitTime()));
432 : }
433 0 : return HcclSetSqFullWaitTimeOut(dispatcher_, notifyWaitTime);
434 : }
435 :
436 0 : void HcclCommAicpu::PrepareOpRetryHandler(
437 : u8 inplaceSupportRetry, u8 retryEnable, u8 inPlaceSupportRetryStatus, u8 isInplacePreSync, u8 isPostSync)
438 : {
439 0 : algOpContext_.opRetryHandler.inplaceSupportRetry = static_cast<bool>(inplaceSupportRetry);
440 0 : algOpContext_.opRetryHandler.retryEnable = static_cast<bool>(retryEnable);
441 : algOpContext_.opRetryHandler.inPlaceSupportRetryStatus
442 0 : = static_cast<InplaceSupportRetryStatus>(inPlaceSupportRetryStatus);
443 0 : algOpContext_.opRetryHandler.isInplacePreSync = static_cast<bool>(isInplacePreSync);
444 0 : algOpContext_.opRetryHandler.isPostSync = static_cast<bool>(isPostSync);
445 0 : HCCL_INFO(
446 : "[HcclCommAicpu][PrepareOpRetryHandler] inplaceSupportRetry %d, retryEnable %d, "
447 : "inPlaceSupportRetryStatus %d, isInplacePreSync %d, isPostSync %d.",
448 : algOpContext_.opRetryHandler.inplaceSupportRetry, algOpContext_.opRetryHandler.retryEnable,
449 : algOpContext_.opRetryHandler.inPlaceSupportRetryStatus, algOpContext_.opRetryHandler.isInplacePreSync,
450 : algOpContext_.opRetryHandler.isPostSync);
451 0 : }
452 :
453 0 : HcclResult HcclCommAicpu::UpdateOpRingBufferIdx()
454 : {
455 0 : return HcclSetOpRingBufferIdx(dispatcher_, aicpuShareData_.GetOpRingBufferIdx());
456 : }
457 :
458 0 : HcclResult HcclCommAicpu::InvokeKfcHandler(AicpuKfcHandlerType type, const std::vector<u64> args)
459 : {
460 0 : CHK_PRT_RET(
461 : static_cast<size_t>(type) >= static_cast<size_t>(AicpuKfcHandlerType::kMax),
462 : HCCL_ERROR("Device mode %u, handler type %u.", static_cast<u32>(isDeviceMode_), static_cast<size_t>(type)),
463 : HCCL_E_INTERNAL);
464 0 : const auto handler = kfcHandlers_[static_cast<size_t>(type)];
465 0 : if (handler == nullptr) {
466 0 : return HCCL_SUCCESS;
467 : }
468 0 : return handler(args);
469 0 : }
470 :
471 0 : HcclResult HcclCommAicpu::NotifyPost(void)
472 : {
473 0 : if (isDeviceMode_) {
474 0 : return InvokeKfcHandler(
475 : AicpuKfcHandlerType::kNotifyRecord,
476 0 : {rpc_, reinterpret_cast<u64>(dispatcher_), reinterpret_cast<u64>(&mainStream_)});
477 : } else {
478 0 : CHK_RET(LocalNotify::Post(mainStream_, dispatcher_, opNotifies_[1]));
479 0 : HcclSqeContext* sqeContext = mainStream_.GetSqeContextPtr();
480 0 : SqeRingBuffer* sqeContextBuffer = &(sqeContext->buffer);
481 0 : return dfx::ProfilingManager::ReportMainStreamTask(mainStream_, sqeContextBuffer->tailSqeTaskId - 1, TAIL_TASK);
482 : }
483 : }
484 :
485 0 : HcclResult HcclCommAicpu::NotifyWait(void)
486 : {
487 0 : if (isDeviceMode_) {
488 0 : return InvokeKfcHandler(
489 : AicpuKfcHandlerType::kNotifyWait,
490 0 : {rpc_, reinterpret_cast<u64>(dispatcher_), reinterpret_cast<u64>(&mainStream_)});
491 : } else {
492 0 : HcclSqeContext* sqeContext = mainStream_.GetSqeContextPtr();
493 0 : SqeRingBuffer* sqeContextBuffer = &(sqeContext->buffer);
494 0 : CHK_RET(dfx::ProfilingManager::ReportMainStreamTask(mainStream_, sqeContextBuffer->tailSqeTaskId, HEAD_TASK));
495 0 : return LocalNotify::Wait(mainStream_, dispatcher_, opNotifies_[0]);
496 : }
497 : }
498 :
499 : // 按照算子模式来Post对应的Notify
500 0 : HcclResult HcclCommAicpu::RecordHostOrder(const HcclOpResParam* commParam, const std::string& tag, u8 orderLaunchMode)
501 : {
502 0 : const u8 orderLaunchInvalidInHcom = 255;
503 0 : if (orderLaunchMode == orderLaunchInvalidInHcom) {
504 0 : HCCL_INFO("[%s] attachedStreams_[%d] is invalid in graph mode", __func__, orderStream_.id());
505 0 : return HCCL_SUCCESS;
506 : }
507 0 : if (orderNotifies_[orderLaunchMode] == nullptr) {
508 0 : std::shared_ptr<LocalNotify> notify;
509 0 : HcclSignalInfo* aicpuOrderNotify = reinterpret_cast<HcclSignalInfo*>(
510 0 : static_cast<u64>(commParam->aicpuOrderNotifyAddr) + (sizeof(HcclSignalInfo) * orderLaunchMode));
511 0 : HCCL_INFO(
512 : "[%s] attachedStreams_[%d] aicpuOrderNotify resId[%llu], addr[0x%llx], flag[%u], devId[%u], tsId[%u], "
513 : "rankId[%u]",
514 : __func__, orderStream_.id(), aicpuOrderNotify->resId, aicpuOrderNotify->addr, aicpuOrderNotify->flag,
515 : aicpuOrderNotify->devId, aicpuOrderNotify->tsId, aicpuOrderNotify->rankId);
516 0 : HcclResult ret = InitAndVerifySingleSignal(*aicpuOrderNotify, notify);
517 0 : CHK_PRT_RET(
518 : ret != HCCL_SUCCESS,
519 : HCCL_ERROR(
520 : "[%s] check localRes noftify failed, resId[%u], group[%s]", __func__, aicpuOrderNotify->resId,
521 : identifier_.c_str()),
522 : ret);
523 0 : orderNotifies_[orderLaunchMode] = notify;
524 0 : HCCL_INFO("%s success, group[%s], resId[%llu]", __func__, identifier_.c_str(), aicpuOrderNotify->resId);
525 0 : }
526 :
527 0 : HCCL_INFO(
528 : "%s group[%s] tag[%s] isDeviceMode[%d] orderLaunchMode[%u] mode[%d] streamId[%d] notifyId[%u]", __func__,
529 : identifier_.c_str(), tag.c_str(), isDeviceMode_, orderLaunchMode, GetWorkflowMode(), orderStream_.id(),
530 : orderNotifies_[orderLaunchMode]->notifyId_);
531 0 : CHK_RET(LocalNotify::Post(orderStream_, dispatcher_, orderNotifies_[orderLaunchMode]));
532 0 : CHK_RET(LaunchTask(dispatcher_, const_cast<Stream&>(orderStream_)));
533 0 : return HCCL_SUCCESS;
534 : }
535 :
536 462 : HcclResult HcclCommAicpu::GetStreamData(
537 : const HcclStreamInfo& streamInfo, HcclComStreamInfo& comStreamInfo, u32& sqHead, u32& sqTail)
538 : {
539 462 : comStreamInfo.sqId = streamInfo.sqIds;
540 462 : comStreamInfo.actualStreamId = streamInfo.streamIds;
541 462 : comStreamInfo.logicCqId = streamInfo.logicCqids;
542 462 : u64 sq_addr = 0;
543 462 : CHK_RET(QuerySqBaseAddr(devId_, streamInfo.sqIds, sq_addr));
544 462 : comStreamInfo.sqBaseAddr = reinterpret_cast<void*>(sq_addr);
545 462 : if (comStreamInfo.sqBaseAddr == nullptr) {
546 0 : HCCL_ERROR("[HcclCommAicpu][GetStreamData]sqe base addr ptr is null.");
547 0 : return HCCL_E_PARA;
548 : }
549 462 : CHK_RET(QuerySqStatusByType(devId_, streamInfo.sqIds, DRV_SQCQ_PROP_SQ_DEPTH, comStreamInfo.sqDepth));
550 462 : CHK_RET(QuerySqStatusByType(devId_, streamInfo.sqIds, DRV_SQCQ_PROP_SQ_TAIL, sqTail));
551 462 : CHK_RET(QuerySqStatusByType(devId_, streamInfo.sqIds, DRV_SQCQ_PROP_SQ_HEAD, sqHead));
552 462 : HCCL_DEBUG(
553 : "[HcclCommAicpu][GetStreamData] get stream data success, group[%s], streamId[%d], sqId[%d], "
554 : "logicCqId[%u], sqDepth[%u], sqHead[%u], sqTail[%u]",
555 : identifier_.c_str(), comStreamInfo.actualStreamId, comStreamInfo.sqId, comStreamInfo.logicCqId,
556 : comStreamInfo.sqDepth, sqHead, sqTail);
557 462 : return HCCL_SUCCESS;
558 : }
559 :
560 11 : HcclResult HcclCommAicpu::InitMainStreamObj(const HcclOpResParam* commParam)
561 : {
562 11 : CHK_RET(InitStreamObj(commParam->localRes.mainStreamParam, mainStream_));
563 11 : HCCL_INFO("%s success, group[%s], streamId[%d]", __func__, identifier_.c_str(), mainStream_.id());
564 11 : return HCCL_SUCCESS;
565 : }
566 :
567 11 : HcclResult HcclCommAicpu::InitSlaveStreamObjs(const HcclOpResParam* commParam)
568 : {
569 11 : if (commParam->localRes.streamNum > LOCAL_STREAM_MAX_NUM) {
570 0 : HCCL_ERROR(
571 : "[HcclCommAicpu][InitSlaveStreamObjs] local streams great max numbers,current numbers[%u], max "
572 : "numbers[%u], group[%s]",
573 : commParam->localRes.streamNum, LOCAL_STREAM_MAX_NUM, identifier_.c_str());
574 0 : return HCCL_E_PARA;
575 : }
576 451 : for (u32 i = 0; i < commParam->localRes.streamNum; i++) {
577 440 : if (streamToObj_.find(commParam->localRes.streamParam[i].streamInfo.sqIds) == streamToObj_.end()) {
578 440 : Stream stream;
579 440 : CHK_RET(InitStreamObj(commParam->localRes.streamParam[i], stream));
580 440 : slaveStreams_.emplace_back(stream);
581 440 : streamToObj_.insert(commParam->localRes.streamParam[i].streamInfo.sqIds);
582 440 : }
583 : }
584 11 : HCCL_DEBUG(
585 : "[HcclCommAicpu][InitSlaveStreamObjs] success, group[%s], slave stream numbers[%u]", identifier_.c_str(),
586 : commParam->localRes.streamNum);
587 11 : return HCCL_SUCCESS;
588 : }
589 :
590 11 : HcclResult HcclCommAicpu::InitOrderStreamObj(const HcclOpResParam* commParam)
591 : {
592 11 : CHK_RET(InitStreamObj(commParam->aicpuOrderStreamParam, orderStream_));
593 11 : HCCL_INFO("%s success, group[%s], streamId[%d]", __func__, identifier_.c_str(), orderStream_.id());
594 11 : return HCCL_SUCCESS;
595 : }
596 :
597 462 : HcclResult HcclCommAicpu::InitStreamObj(const HcclStreamParam& streamParam, Stream& stream)
598 : {
599 : u32 sqTail;
600 : u32 sqHead;
601 462 : HcclResult ret = HCCL_SUCCESS;
602 462 : ret = StreamRestore(streamParam.streamInfo.streamIds);
603 462 : CHK_PRT_RET(
604 : ret != HCCL_SUCCESS,
605 : HCCL_ERROR(
606 : "%s StreamId[%d] Restore error group[%s]", __func__, streamParam.streamInfo.streamIds, identifier_.c_str()),
607 : ret);
608 :
609 462 : HcclComStreamInfo comStreamInfo = {};
610 462 : ret = GetStreamData(streamParam.streamInfo, comStreamInfo, sqHead, sqTail);
611 462 : CHK_PRT_RET(ret != HCCL_SUCCESS, HCCL_ERROR("%s error group[%s]", __func__, identifier_.c_str()), ret);
612 462 : stream = Stream(comStreamInfo);
613 :
614 : // 初始化stream的sqeContext
615 462 : SqCqeContext* sqCqeContext = reinterpret_cast<SqCqeContext*>(streamParam.sqCqContextAddr);
616 462 : u64 sqCqContextSize = streamParam.sqCqContextSize;
617 462 : if (sqCqeContext == nullptr || sqCqContextSize != sizeof(SqCqeContext)) {
618 0 : HCCL_ERROR(
619 : "%s failed, sqCqeContext[%p] is null or size[%llu] does not match expected size[%llu], group[%s]", __func__,
620 : sqCqeContext, sqCqContextSize, sizeof(SqCqeContext), identifier_.c_str());
621 0 : return HCCL_E_PARA;
622 : }
623 462 : ret = stream.InitSqAndCqeContext(sqHead, sqTail, sqCqeContext);
624 462 : CHK_PRT_RET(
625 : ret != HCCL_SUCCESS, HCCL_ERROR("%s: InitSqAndCqeContext failed, group[%s]", __func__, identifier_.c_str()),
626 : ret);
627 462 : HCCL_INFO("%s success, group[%s], streamId[%d]", __func__, identifier_.c_str(), stream.id());
628 462 : return HCCL_SUCCESS;
629 : }
630 :
631 11 : HcclResult HcclCommAicpu::InitOpNotifyObj(const HcclOpResParam* commParam)
632 : {
633 11 : HcclResult ret = HCCL_SUCCESS;
634 33 : for (u32 i = 0; i < AICPU_OP_NOTIFY_MAX_NUM; i++) {
635 22 : ret = InitAndVerifySignal(commParam->localRes.aicpuOpNotify[i], opNotifies_);
636 22 : CHK_PRT_RET(
637 : ret != HCCL_SUCCESS,
638 : HCCL_ERROR(
639 : "[HcclCommAicpu][InitOpNotifyObj] check localRes op noftify failed, resId[%u], group[%s]",
640 : commParam->localRes.localSignals[i].resId, identifier_.c_str()),
641 : ret);
642 : }
643 11 : CHK_RET(hrtDrvGetLocalDevIDByHostDevID(commParam->localRes.aicpuOpNotify[0].devId, &devId_));
644 11 : HCCL_INFO("[HcclCommAicpu][InitOpNotifyObj] success, group[%s]", identifier_.c_str());
645 11 : return HCCL_SUCCESS;
646 : }
647 :
648 11 : HcclResult HcclCommAicpu::InitLocalNotifyObj(const HcclOpResParam* commParam)
649 : {
650 11 : HcclResult ret = HCCL_SUCCESS;
651 :
652 11 : if (commParam->localRes.signalNum > LOCAL_NOTIFY_MAX_NUM) {
653 0 : HCCL_ERROR(
654 : "[HcclCommAicpu][InitLocalNotifyObj] local notifys great max numbers, numbers[%u], max numbers[%u], "
655 : "group[%s]",
656 : commParam->localRes.signalNum, LOCAL_NOTIFY_MAX_NUM, identifier_.c_str());
657 0 : return HCCL_E_PARA;
658 : }
659 715 : for (u32 i = 0; i < commParam->localRes.signalNum; i++) {
660 704 : if (notifysToObj_.find(commParam->localRes.localSignals[i].resId) == notifysToObj_.end()) {
661 704 : ret = InitAndVerifySignal(commParam->localRes.localSignals[i], localNotifies_);
662 704 : CHK_PRT_RET(
663 : ret != HCCL_SUCCESS,
664 : HCCL_ERROR(
665 : "[HcclCommAicpu][InitLocalNotifyObj] check localRes prenoftify failed, resId[%u]",
666 : commParam->localRes.localSignals[i].resId),
667 : ret);
668 704 : notifysToObj_.insert(commParam->localRes.localSignals[i].resId);
669 : }
670 : }
671 11 : HCCL_DEBUG(
672 : "[HcclCommAicpu][InitLocalNotifyObj] success, group[%s], signal numbers[%u]", identifier_.c_str(),
673 : commParam->localRes.signalNum);
674 11 : return HCCL_SUCCESS;
675 : }
676 :
677 22 : HcclResult HcclCommAicpu::ParseTlvToVector(
678 : u64 srcTlv, u64 srcTlvTotalLength, std::vector<std::vector<std::vector<u32>>>& vectorInfo)
679 : {
680 22 : u64 parseLength = 0;
681 22 : u8* srcTlvptr = reinterpret_cast<u8*>(srcTlv);
682 22 : if (srcTlvptr == nullptr) {
683 0 : HCCL_ERROR("[HcclCommAicpu][ParseTlvToVector]tlv ptr is null.");
684 0 : return HCCL_E_PARA;
685 : }
686 22 : u64 unPareseLength = srcTlvTotalLength;
687 143 : while (unPareseLength > 0) {
688 121 : CommonTlv* commonTlv = reinterpret_cast<CommonTlv*>(srcTlvptr + parseLength);
689 121 : if (unPareseLength <= (sizeof(LENGTH_TYPE) + sizeof(TAG_TYPE))) {
690 0 : HCCL_ERROR(
691 : "[HcclCommAicpu][ParseTlvToVector] Tlv length is error, tag[%s], totalLength[%lu], "
692 : "unParseLength[%lu], already parsed Length[%lu]",
693 : identifier_.c_str(), srcTlvTotalLength, unPareseLength, parseLength);
694 0 : return HCCL_E_PARA;
695 : }
696 121 : if (commonTlv->length > unPareseLength || commonTlv->length % sizeof(RANK_TYPE) != 0) {
697 0 : HCCL_ERROR(
698 : "[HcclCommAicpu][ParseTlvToVector] parse Tlv error, group[%s], total Length[%lu], tlvLength[%lu], "
699 : "unParsed Length[%lu], already parsed Length[%lu]",
700 : identifier_.c_str(), srcTlvTotalLength, commonTlv->length, unPareseLength, parseLength);
701 0 : return HCCL_E_PARA;
702 : }
703 121 : u16 level0 = (commonTlv->type & TOP_COMM_LEVEL0_LOCATION) >> TOP_COMM_LEVEL0_SHIFT;
704 121 : u16 level1 = (commonTlv->type & TOP_COMM_LEVEL1_LOCATION);
705 121 : u64 itemNum = (commonTlv->length - sizeof(TAG_TYPE) - sizeof(LENGTH_TYPE)) / sizeof(RANK_TYPE);
706 242 : std::vector<RANK_TYPE> values{&commonTlv->value, (&commonTlv->value) + itemNum};
707 121 : if (level0 >= vectorInfo.size()) {
708 99 : vectorInfo.resize(level0 + 1);
709 : }
710 121 : if (level1 >= vectorInfo[level0].size()) {
711 121 : vectorInfo[level0].resize(level1 + 1);
712 : }
713 121 : vectorInfo[level0][level1] = std::move(values);
714 121 : parseLength += commonTlv->length;
715 121 : unPareseLength -= commonTlv->length;
716 121 : HCCL_DEBUG(
717 : "[HcclCommAicpu][ParseTlvToVector] parse Tlv group[%s], level0[%u], level1[%u], total Length[%lu], "
718 : "tlvLength[%lu], unParsed Length[%lu], already parsed Length[%lu]",
719 : identifier_.c_str(), level0, level1, srcTlvTotalLength, commonTlv->length, unPareseLength, parseLength);
720 121 : }
721 143 : for (u32 idx = 0; idx < vectorInfo.size(); idx++) {
722 242 : for (u32 ringidx = 0; ringidx < vectorInfo[idx].size(); ringidx++) {
723 121 : HCCL_DEBUG(
724 : "[HcclCommAicpu][ParseTlvToVector] idx[%u] ringidx[%u] size[%u]", idx, ringidx,
725 : vectorInfo[idx][ringidx].size());
726 : }
727 : }
728 22 : HCCL_INFO("[HcclCommAicpu][ParseTlvToVector] success, group[%s]", identifier_.c_str());
729 22 : return HCCL_SUCCESS;
730 : }
731 :
732 11 : HcclResult HcclCommAicpu::ParseTlvToSubGroupVector(
733 : u64 srcTlv, u64 srcTlvTotalLength, std::vector<std::vector<std::vector<std::vector<u32>>>>& vectorInfo)
734 : {
735 11 : u64 parseLength = 0;
736 11 : u8* srcTlvptr = reinterpret_cast<u8*>(srcTlv);
737 11 : if (srcTlvptr == nullptr) {
738 0 : HCCL_ERROR("[HcclCommAicpu][ParseTlvToSubGroupVector]tlv ptr is null.");
739 0 : return HCCL_E_PARA;
740 : }
741 11 : u64 unPareseLength = srcTlvTotalLength;
742 33 : while (unPareseLength > 0) {
743 22 : CommonTlv* commonTlv = reinterpret_cast<CommonTlv*>(srcTlvptr + parseLength);
744 22 : if (unPareseLength <= (sizeof(LENGTH_TYPE) + sizeof(TAG_TYPE))) {
745 0 : HCCL_ERROR(
746 : "[HcclCommAicpu][ParseTlvToSubGroupVector] Tlv length is error, tag[%s], totalLength[%lu], "
747 : "unParseLength[%lu], already parsed Length[%lu]",
748 : identifier_.c_str(), srcTlvTotalLength, unPareseLength, parseLength);
749 0 : return HCCL_E_PARA;
750 : }
751 22 : if (commonTlv->length > unPareseLength || commonTlv->length % sizeof(RANK_TYPE) != 0) {
752 0 : HCCL_ERROR(
753 : "[HcclCommAicpu][ParseTlvToSubGroupVector] parse Tlv error, group[%s], total Length[%lu], "
754 : "tlvLength[%lu], "
755 : "unParsed Length[%lu], already parsed Length[%lu]",
756 : identifier_.c_str(), srcTlvTotalLength, commonTlv->length, unPareseLength, parseLength);
757 0 : return HCCL_E_PARA;
758 : }
759 22 : u16 level0 = (commonTlv->type & TOP_HIERARCHICAL_COMM_LEVEL0_LOCATION)
760 22 : >> (TOP_HIERARCHICAL_COMM_LEVEL0_SHIFT + TOP_HIERARCHICAL_COMM_LEVEL1_SHIFT);
761 22 : u16 level1 = (commonTlv->type & TOP_HIERARCHICAL_COMM_LEVEL1_LOCATION) >> TOP_HIERARCHICAL_COMM_LEVEL1_SHIFT;
762 22 : u16 level2 = (commonTlv->type & TOP_HIERARCHICAL_COMM_LEVEL2_LOCATION);
763 22 : u64 itemNum = (commonTlv->length - sizeof(TAG_TYPE) - sizeof(LENGTH_TYPE)) / sizeof(RANK_TYPE);
764 44 : std::vector<RANK_TYPE> values{&commonTlv->value, (&commonTlv->value) + itemNum};
765 22 : if (level0 >= vectorInfo.size()) {
766 11 : vectorInfo.resize(level0 + 1);
767 : }
768 22 : if (level1 >= vectorInfo[level0].size()) {
769 11 : vectorInfo[level0].resize(level1 + 1);
770 : }
771 22 : if (level2 >= vectorInfo[level0][level1].size()) {
772 22 : vectorInfo[level0][level1].resize(level2 + 1);
773 : }
774 22 : vectorInfo[level0][level1][level2] = std::move(values);
775 22 : parseLength += commonTlv->length;
776 22 : unPareseLength -= commonTlv->length;
777 22 : HCCL_DEBUG(
778 : "[HcclCommAicpu][ParseTlvToSubGroupVector] parse Tlv group[%s], level0[%u], level1[%u], total Length[%lu], "
779 : "tlvLength[%lu], unParsed Length[%lu], already parsed Length[%lu]",
780 : identifier_.c_str(), level0, level1, srcTlvTotalLength, commonTlv->length, unPareseLength, parseLength);
781 22 : }
782 22 : for (u32 level0Idx = 0; level0Idx < vectorInfo.size(); level0Idx++) {
783 22 : for (u32 level1Idx = 0; level1Idx < vectorInfo[level0Idx].size(); level1Idx++) {
784 33 : for (u32 level2Idx = 0; level2Idx < vectorInfo[level0Idx][level1Idx].size(); level2Idx++) {
785 22 : HCCL_DEBUG(
786 : "[HcclCommAicpu][ParseTlvToSubGroupVector] level0Idx[%u] level1Idx[%u] level2Idx[%u] size[%u]",
787 : level0Idx, level1Idx, level2Idx, vectorInfo[level0Idx][level1Idx][level2Idx].size());
788 : }
789 : }
790 : }
791 11 : HCCL_INFO("[HcclCommAicpu][ParseTlvToSubGroupVector] success, group[%s]", identifier_.c_str());
792 11 : return HCCL_SUCCESS;
793 : }
794 :
795 11 : HcclResult HcclCommAicpu::InitTopoInfo(const HcclOpResParam* commParam)
796 : {
797 11 : topoInfo_.userRank = commParam->topoInfo.userRank;
798 11 : topoInfo_.userRankSize = commParam->topoInfo.userRankSize;
799 11 : topoInfo_.deviceLogicId = commParam->topoInfo.deviceLogicId;
800 11 : topoInfo_.isSingleMeshAggregation = commParam->topoInfo.isSingleMeshAggregation;
801 11 : topoInfo_.deviceNumPerAggregation = commParam->topoInfo.deviceNumPerAggregation;
802 11 : topoInfo_.superPodNum = commParam->topoInfo.superPodNum;
803 11 : topoInfo_.devicePhyId = commParam->topoInfo.devicePhyId;
804 11 : topoInfo_.deviceType = static_cast<DevType>(commParam->topoInfo.deviceType);
805 11 : topoInfo_.topoType = static_cast<TopoType>(commParam->topoInfo.topoType);
806 11 : topoInfo_.is310P3Common = (topoInfo_.deviceType == DevType::DEV_TYPE_310P3);
807 11 : topoInfo_.serverNum = commParam->topoInfo.serverNum;
808 11 : topoInfo_.meshAggregationRankSize = commParam->topoInfo.meshAggregationRankSize;
809 11 : topoInfo_.multiModuleDiffDeviceNumMode = commParam->topoInfo.multiModuleDiffDeviceNumMode;
810 11 : topoInfo_.multiSuperPodDiffServerNumMode = commParam->topoInfo.multiSuperPodDiffServerNumMode;
811 11 : topoInfo_.realUserRank = commParam->topoInfo.realUserRank;
812 11 : topoInfo_.isDiffDeviceModule = commParam->topoInfo.isDiffDeviceModule;
813 11 : topoInfo_.isDiffDeviceType = commParam->topoInfo.isDiffDeviceType;
814 11 : topoInfo_.gcdDeviceNumPerAggregation = commParam->topoInfo.gcdDeviceNumPerAggregation;
815 11 : topoInfo_.moduleNum = commParam->topoInfo.moduleNum;
816 11 : topoInfo_.useSuperPodMode = true;
817 11 : topoInfo_.isARSDoubleRing = commParam->isARSDoubleRing;
818 11 : topoInfo_.multiSuperPodDiffDeviceNumMode = commParam->multiSuperPodDiffDeviceNumMode;
819 11 : if (commParam->topoInfo.isUsedRdmaRankPairNum % KEY_VALUE_TO_VECTOR_MODULUS != 0) {
820 0 : HCCL_ERROR(
821 : "[HcclCommAicpu][InitTopoInfo]rdma rank pair number[%lu] is error.",
822 : commParam->topoInfo.isUsedRdmaRankPairNum);
823 0 : return HCCL_E_PARA;
824 : }
825 11 : u32* isUsedRdmaRankPairPtr = reinterpret_cast<u32*>(commParam->topoInfo.isUsedRdmaRankPair);
826 11 : if (isUsedRdmaRankPairPtr == nullptr) {
827 0 : HCCL_ERROR("[HcclCommAicpu][InitTopoInfo]rdma rank pair ptr is null.");
828 0 : return HCCL_E_PARA;
829 : }
830 55 : for (u64 i = 0; i < commParam->topoInfo.isUsedRdmaRankPairNum; i += KEY_VALUE_TO_VECTOR_MODULUS) {
831 44 : topoInfo_.isUsedRdmaMap.insert({isUsedRdmaRankPairPtr[i], static_cast<bool>(isUsedRdmaRankPairPtr[i + 1])});
832 : }
833 :
834 11 : if (commParam->topoInfo.pairLinkCounterNum % KEY_VALUE_TO_VECTOR_MODULUS != 0) {
835 0 : HCCL_ERROR(
836 : "[HcclCommAicpu][InitTopoInfo]pair link count number[%lu] is error.",
837 : commParam->topoInfo.pairLinkCounterNum);
838 0 : return HCCL_E_PARA;
839 : }
840 11 : u32* pairLinkCounterPtr = reinterpret_cast<u32*>(commParam->topoInfo.pairLinkCounter);
841 11 : if (pairLinkCounterPtr == nullptr) {
842 0 : HCCL_ERROR("[HcclCommAicpu][InitTopoInfo]rdma rank pair ptr is null.");
843 0 : return HCCL_E_PARA;
844 : }
845 :
846 55 : for (u64 i = 0; i < commParam->topoInfo.pairLinkCounterNum; i += KEY_VALUE_TO_VECTOR_MODULUS) {
847 44 : topoInfo_.pairLinkCounter.insert({pairLinkCounterPtr[i], pairLinkCounterPtr[i + 1]});
848 : }
849 :
850 11 : u32* nicListPtr = reinterpret_cast<u32*>(commParam->topoInfo.nicList);
851 11 : if (nicListPtr == nullptr) {
852 0 : HCCL_ERROR("[HcclCommAicpu][InitTopoInfo]nic list ptr is null.");
853 0 : return HCCL_E_PARA;
854 : }
855 22 : std::vector<u32> niclist{nicListPtr, nicListPtr + commParam->topoInfo.nicNum};
856 11 : topoInfo_.nicList = std::move(niclist);
857 :
858 11 : bool* bridgeRankPtr = reinterpret_cast<bool*>(commParam->topoInfo.bridgeRank);
859 11 : if (bridgeRankPtr != nullptr) {
860 11 : isBridgeVector_.resize(commParam->topoInfo.bridgeRankNum);
861 33 : for (u32 i = 0; i < commParam->topoInfo.bridgeRankNum; ++i) {
862 22 : isBridgeVector_[i] = bridgeRankPtr[i];
863 22 : HCCL_DEBUG("[HcclCommAicpu][InitTopoInfo] bridge rank info idx[%u] value[%u]", i, isBridgeVector_[i]);
864 : }
865 : } else {
866 0 : HCCL_RUN_INFO("[HcclCommAicpu][InitTopoInfo] bridge rank number is 0, group[%s]", identifier_.c_str());
867 : }
868 :
869 11 : HcclResult ret = HCCL_SUCCESS;
870 11 : ret = ParseTlvToVector(commParam->topoInfo.complanRank, commParam->topoInfo.complanRankLength, commPlaneVector_);
871 11 : CHK_PRT_RET(
872 : ret != HCCL_SUCCESS,
873 : HCCL_ERROR("[HcclCommAicpu][InitTopoInfo]Init CommPlane error group[%s]", identifier_.c_str()), ret);
874 :
875 22 : ret = ParseTlvToVector(
876 11 : commParam->topoInfo.serverAndsuperPodRank, commParam->topoInfo.serverAndsuperPodRankLength,
877 11 : serverAndsuperPodToRank_);
878 11 : CHK_PRT_RET(
879 : ret != HCCL_SUCCESS,
880 : HCCL_ERROR("[HcclCommAicpu][InitTopoInfo]Init server and superPod rank error group[%s]", identifier_.c_str()),
881 : ret);
882 :
883 22 : ret = ParseTlvToSubGroupVector(
884 11 : commParam->hierarchicalAlgInfo.commplaneSubGroupRank,
885 11 : commParam->hierarchicalAlgInfo.commplaneSubGroupRankLength, topoInfo_.CommPlaneSubGroupVector);
886 11 : CHK_PRT_RET(
887 : ret != HCCL_SUCCESS,
888 : HCCL_ERROR("[HcclCommAicpu][InitTopoInfo]Init CommPlaneSubGroup error group[%s]", identifier_.c_str()), ret);
889 :
890 11 : HCCL_INFO(
891 : "[HcclCommAicpu][InitTopoInfo] success, group[%s], device Type[%u]", identifier_.c_str(),
892 : commParam->topoInfo.deviceType);
893 11 : return HCCL_SUCCESS;
894 11 : }
895 :
896 11 : HcclResult HcclCommAicpu::InitCclbuffer(const HcclOpResParam* commParam)
897 : {
898 11 : if (commParam->localWindowsIn == 0 || commParam->localWindowsOut == 0 || cclInputBuffer_.ptr() != nullptr
899 22 : || cclOutputBuffer_.ptr() != nullptr) {
900 0 : HCCL_INFO(
901 : "[HcclCommAicpu][InitCclBuffer] don't need init cclbuffer "
902 : "ccl winIn[0x%lx] winout[0x%lx] cclin ptr[%p] cclout ptr[%p]",
903 : commParam->localWindowsIn, commParam->localWindowsOut, cclInputBuffer_.ptr(), cclOutputBuffer_.ptr());
904 0 : return HCCL_SUCCESS;
905 : }
906 :
907 11 : auto cclInPtr = reinterpret_cast<void*>(commParam->localWindowsIn);
908 11 : auto cclOutPtr = reinterpret_cast<void*>(commParam->localWindowsOut);
909 11 : cclInputBuffer_ = DeviceMem::create(cclInPtr, commParam->winSize);
910 11 : cclOutputBuffer_ = DeviceMem::create(cclOutPtr, commParam->winSize);
911 11 : cclbufferSize_ = commParam->winSize;
912 11 : HCCL_INFO(
913 : "[HcclCommAicpu][InitCclbuffer] success, group[%s], cclin[%llu], cclout[%llu], size[%lu]", identifier_.c_str(),
914 : commParam->localWindowsIn, commParam->localWindowsOut, commParam->winSize);
915 11 : return HCCL_SUCCESS;
916 : }
917 :
918 11 : HcclResult HcclCommAicpu::InitConfigInfo(const HcclOpResParam* commParam)
919 : {
920 11 : deterministic_ = commParam->config.deterministic;
921 11 : interHccsDisable_ = commParam->config.interHccsDisable;
922 11 : multiQpThreshold_ = commParam->config.multiQpThreshold;
923 11 : inlineReducEnable_ = true;
924 11 : fftsEnable_ = false;
925 11 : taskMonitorInterval_ = commParam->config.taskMonitorInterval;
926 11 : algoInfo_.inlineReduceSwitchOn = true;
927 11 : algoInfo_.identifier = commParam->hcomId;
928 11 : algoInfo_.isSupportAtomicWrite = static_cast<bool>(commParam->config.isSupportAtomicWrite);
929 11 : notifySize_ = commParam->notifysize;
930 11 : slaveStreams_.reserve(LOCAL_STREAM_MAX_NUM);
931 11 : localNotifies_.reserve(LOCAL_NOTIFY_MAX_NUM);
932 11 : SetExternalInputDebugConfig(commParam->debugConfig);
933 11 : InitDebugConfigByValue(commParam->debugConfig);
934 11 : HCCL_INFO(
935 : "[HcclCommAicpu][Init]success, group[%s] reserve noipc notifys[%lu], slave streams[%lu].", identifier_.c_str(),
936 : LOCAL_NOTIFY_MAX_NUM, LOCAL_STREAM_MAX_NUM);
937 11 : return HCCL_SUCCESS;
938 : }
939 :
940 13 : void HcclCommAicpu::SetDumpDebug(bool dumpDebug) { dumpDebug_ = dumpDebug; }
941 :
942 : template <typename T>
943 : HcclResult
944 726 : HcclCommAicpu::InitAndVerifySignal(const HcclSignalInfo& signalInfo, std::vector<std::shared_ptr<T>>& notifyVec)
945 : {
946 726 : if (signalInfo.resId == INVALID_U64) {
947 : // 无效值不做校验
948 0 : HCCL_INFO("[HcclCommAicpu][InitAndVerifySignal] resId is invalid, need not check");
949 0 : return HCCL_SUCCESS;
950 : }
951 :
952 726 : std::shared_ptr<T> notify;
953 726 : EXCEPTION_CATCH((notify = std::make_shared<T>()), return HCCL_E_PTR);
954 726 : CHK_SMART_PTR_NULL(notify);
955 726 : CHK_RET(notify->Init(signalInfo, NotifyLoadType::DEVICE_NOTIFY));
956 726 : notifyVec.push_back(notify);
957 726 : HCCL_INFO(
958 : "[HcclCommAicpu][InitAndVerifySignal] success group[%s], resId[%u], tsId:%d, devId[%u]", identifier_.c_str(),
959 : signalInfo.resId, signalInfo.tsId, signalInfo.devId);
960 :
961 726 : return HCCL_SUCCESS;
962 726 : }
963 :
964 33 : void HcclCommAicpu::HandleExistTagReAlloc(
965 : HccltagLocalResV2* tagRes, const std::string& tag, bool reAllocFlag, ListCommon*& curList, bool& needSkip)
966 : {
967 33 : needSkip = false;
968 : // 如果 tag 已经存在且需要重新申请内存, 删除旧的资源与记录
969 33 : if (localTagResToObj_.find(tag) != localTagResToObj_.end() && reAllocFlag) {
970 0 : u64 currSize = tagRes->ScratchmemSize;
971 0 : u64 existSize = tagScratchMem_[tag]->size();
972 : // 新内存更大，删除旧记录，保留新的
973 0 : if (currSize > existSize) {
974 0 : localTagResToObj_.erase(tag);
975 0 : tagScratchMem_.erase(tag);
976 0 : HCCL_DEBUG(
977 : "[HcclCommAicpu][%s] tag exists, replace old small mem, tag[%s], old[%lu] new[%lu]", __func__,
978 : tag.c_str(), existSize, currSize);
979 : } else {
980 : // 旧内存更大，需要跳过
981 0 : HCCL_DEBUG(
982 : "[HcclCommAicpu][%s] tag exists, keep larger old mem, skip new, tag[%s], old[%lu] new[%lu]", __func__,
983 : tag.c_str(), existSize, currSize);
984 0 : curList = reinterpret_cast<ListCommon*>(curList->nextDevice);
985 0 : needSkip = true;
986 : }
987 : }
988 33 : }
989 :
990 11 : HcclResult HcclCommAicpu::InitLocalTagRes(const ListCommon& head, bool reAllocFlag)
991 : {
992 11 : ListCommon* curList = reinterpret_cast<ListCommon*>(head.nextDevice);
993 11 : if (curList == nullptr) {
994 0 : HCCL_ERROR("[HcclCommAicpu][InitLocalTagRes]list ptr is null.");
995 0 : return HCCL_E_PARA;
996 : }
997 44 : while (curList != &head) {
998 33 : HccltagLocalResV2* tagRes = list_entry(curList, HccltagLocalResV2, nextTagRes);
999 33 : std::string tag = tagRes->tag;
1000 33 : bool needSkip = false;
1001 33 : HandleExistTagReAlloc(tagRes, tag, reAllocFlag, curList, needSkip);
1002 33 : if (needSkip) {
1003 0 : continue;
1004 : }
1005 33 : if (localTagResToObj_.find(tag) == localTagResToObj_.end()
1006 33 : || localTagResToObj_[tag].find(tagRes->Scratchmem) == localTagResToObj_[tag].end()) {
1007 33 : auto scratchMemPtr = reinterpret_cast<void*>(tagRes->Scratchmem);
1008 33 : if (scratchMemPtr == nullptr) {
1009 0 : HCCL_ERROR("[HcclCommAicpu][InitLocalTagRes]scratch mem ptr is null.");
1010 0 : return HCCL_E_PARA;
1011 : }
1012 33 : DeviceMem loalScratchmem = DeviceMem::create(scratchMemPtr, tagRes->ScratchmemSize);
1013 33 : std::shared_ptr<DeviceMem> loalScratchmemPtr;
1014 33 : EXCEPTION_CATCH(
1015 : (loalScratchmemPtr = std::make_shared<DeviceMem>(std::move(loalScratchmem))), return HCCL_E_PTR);
1016 33 : CHK_SMART_PTR_NULL(loalScratchmemPtr);
1017 33 : if (tagScratchMem_.find(tag) == tagScratchMem_.end()) {
1018 33 : tagScratchMem_.insert({tag, loalScratchmemPtr});
1019 : }
1020 33 : std::unordered_set<u64> tmpTagRes;
1021 33 : tmpTagRes.insert(tagRes->Scratchmem);
1022 33 : localTagResToObj_[tag] = tmpTagRes;
1023 33 : HCCL_DEBUG(
1024 : "[HcclCommAicpu][InitLocalTagRes] parse remote resource, tag[%s], Scratchmem[%p], "
1025 : "ScratchmemSize[%lu]",
1026 : tag.c_str(), tagRes->Scratchmem, tagRes->ScratchmemSize);
1027 33 : }
1028 33 : curList = reinterpret_cast<ListCommon*>(curList->nextDevice);
1029 33 : if (curList == nullptr) {
1030 0 : HCCL_ERROR("[HcclCommAicpu][InitLocalTagRes] next list ptr is null.");
1031 0 : return HCCL_E_PARA;
1032 : }
1033 33 : };
1034 11 : HCCL_INFO("[HcclCommAicpu][InitLocalTagRes] success, group[%s]", identifier_.c_str());
1035 11 : return HCCL_SUCCESS;
1036 : }
1037 :
1038 0 : HcclResult HcclCommAicpu::CheckNotifyOrQPMaxNum(u64& existNum, const u64& MaxNum, const bool& isNotifyRes)
1039 : {
1040 0 : std::string resType = isNotifyRes ? "Notify" : "QP";
1041 0 : if (existNum + 1 > MaxNum) {
1042 0 : HCCL_ERROR(
1043 : "[%s]%s resources are insufficient, existNum[%llu], MaxNum is [%llu]", __func__, resType.c_str(), existNum,
1044 : MaxNum);
1045 0 : return HCCL_E_INTERNAL;
1046 : }
1047 0 : HCCL_DEBUG(
1048 : "[%s]%s resources are sufficient, existNum[%llu], MaxNum is [%llu]", __func__, resType.c_str(), existNum,
1049 : MaxNum);
1050 0 : return HCCL_SUCCESS;
1051 0 : }
1052 :
1053 0 : HcclResult HcclCommAicpu::SetTransportMachinePara(
1054 : MachinePara& machinePara, u32& rankId, const std::string& newTag, TransportLinkType linkType)
1055 : {
1056 0 : machinePara.linkAttribute = 0x03; /* 0x03同时支持目的端和源端发起 */
1057 0 : if (rankData_.find(rankId) == rankData_.end()) {
1058 0 : HCCL_ERROR("[%s]there is no link with rankId[%u]", __func__, rankId);
1059 0 : return HCCL_E_NOT_FOUND;
1060 : }
1061 :
1062 0 : machinePara.localUserrank = localUserRank_;
1063 0 : machinePara.remoteWorldRank = rankData_[rankId].remoteWorldRank;
1064 0 : machinePara.remoteUserrank = rankData_[rankId].remoteUsrRankId;
1065 0 : machinePara.deviceLogicId = topoInfo_.deviceLogicId;
1066 0 : machinePara.localDeviceId = topoInfo_.devicePhyId;
1067 0 : machinePara.deviceType = topoInfo_.deviceType;
1068 0 : machinePara.tag = newTag;
1069 0 : machinePara.isAicpuModeEn = true;
1070 0 : if (linkType == TransportLinkType::RESERVED) {
1071 : // 非910_93 2die sio与hccs并发场景，specifyLink设置为RESERVED_LINK_TYPE，平台层将按实际链路类型建链
1072 0 : machinePara.specifyLink = LinkTypeInServer::RESERVED_LINK_TYPE;
1073 : } else {
1074 : // 910_93 2die sio与hccs并发场景，
1075 : // 并发链路中的的hccs链路specifyLink设置为HCCS_SW_TYPE，平台层将使用hccs链路来建链；
1076 : // 并发链路中的的sio链路specifyLink设置为SIO_TYPE，平台层将使用sio链路来建链
1077 : machinePara.specifyLink
1078 0 : = (linkType == TransportLinkType::SIO) ? LinkTypeInServer::SIO_TYPE : LinkTypeInServer::HCCS_SW_TYPE;
1079 : }
1080 :
1081 0 : HCCL_INFO(
1082 : "%s success, group[%s], rankId[%u], linkAttribute[%x], localUserRank[%u], remoteWorldRank[%u], "
1083 : "remoteUserrank[%u], deviceLogicId[%d], localDeviceId[%d], deviceType[%d], newTag[%s], specifyLink[%d], "
1084 : "isAicpuModeEn[%d]",
1085 : __func__, identifier_.c_str(), rankId, machinePara.linkAttribute, machinePara.localUserrank,
1086 : machinePara.remoteWorldRank, machinePara.remoteUserrank, machinePara.deviceLogicId, machinePara.localDeviceId,
1087 : machinePara.deviceType, machinePara.tag.c_str(), machinePara.specifyLink, machinePara.isAicpuModeEn);
1088 0 : return HCCL_SUCCESS;
1089 : }
1090 :
1091 : template <typename T>
1092 0 : HcclResult HcclCommAicpu::InitAndVerifySingleSignal(const HcclSignalInfo& signalInfo, std::shared_ptr<T>& notify)
1093 : {
1094 0 : if (signalInfo.resId == INVALID_U64) {
1095 : // 无效值不做校验
1096 0 : HCCL_DEBUG("[%s]resId[%llu] is invalid, need not check", __func__, signalInfo.resId);
1097 0 : return HCCL_SUCCESS;
1098 : }
1099 : HcclSignalInfo tmpSignalInfo;
1100 :
1101 0 : EXCEPTION_CATCH((notify = std::make_shared<T>()), return HCCL_E_PTR);
1102 0 : CHK_SMART_PTR_NULL(notify);
1103 0 : CHK_RET(notify->Init(signalInfo, NotifyLoadType::DEVICE_NOTIFY));
1104 0 : CHK_RET(notify->GetNotifyData(tmpSignalInfo));
1105 0 : HCCL_DEBUG(
1106 : "[%s] success group[%s], resId[%llu], tsId:%d, devId[%u]", __func__, identifier_.c_str(), tmpSignalInfo.resId,
1107 : tmpSignalInfo.tsId, tmpSignalInfo.devId);
1108 :
1109 0 : return HCCL_SUCCESS;
1110 : }
1111 :
1112 0 : HcclResult HcclCommAicpu::SetTagRemoteRes(u32& rankId, const std::string& tag, HccltagRemoteResV2* tagRes)
1113 : {
1114 0 : if (rankTagRemoteRes_.find(rankId) == rankTagRemoteRes_.end()
1115 0 : || rankTagRemoteRes_[rankId].find(tag) == rankTagRemoteRes_[rankId].end()) {
1116 0 : HccltagRemoteResV3 tempTagRemoteRes;
1117 0 : tempTagRemoteRes.tagRemoteResPtr = tagRes;
1118 0 : rankTagRemoteRes_[rankId][tag] = tempTagRemoteRes;
1119 : }
1120 0 : HCCL_DEBUG("[%s]get TagRemoteRes success, rankId[%u], tag[%s]", __func__, rankId, tag.c_str());
1121 0 : return HCCL_SUCCESS;
1122 : }
1123 :
1124 0 : HcclResult HcclCommAicpu::SetTransportPtpNotify(
1125 : TransportDeviceP2pData& transDevP2pData, u64& p2pNotifyNum, HcclLinkP2pV2& linkP2p, u32 notifyNum)
1126 : {
1127 0 : u64 actualNotifyNum = 0;
1128 : // 获取Ipc notify信息
1129 0 : CHK_RET(CheckNotifyOrQPMaxNum(actualNotifyNum, LINK_P2P_MAX_NUM, true));
1130 0 : std::shared_ptr<LocalNotify> ipcPreWaitNotify = std::make_shared<LocalNotify>();
1131 0 : CHK_RET(InitAndVerifySingleSignal(linkP2p.localIpcSignal[actualNotifyNum], ipcPreWaitNotify));
1132 0 : transDevP2pData.ipcPreWaitNotify = ipcPreWaitNotify;
1133 :
1134 0 : std::shared_ptr<RemoteNotify> ipcPreRecordNotify = std::make_shared<RemoteNotify>();
1135 0 : CHK_RET(InitAndVerifySingleSignal(linkP2p.remoteIpcSignal[actualNotifyNum], ipcPreRecordNotify));
1136 0 : transDevP2pData.ipcPreRecordNotify = ipcPreRecordNotify;
1137 0 : actualNotifyNum++;
1138 :
1139 0 : CHK_RET(CheckNotifyOrQPMaxNum(actualNotifyNum, LINK_P2P_MAX_NUM, true));
1140 0 : std::shared_ptr<LocalNotify> ipcPostWaitNotify = std::make_shared<LocalNotify>();
1141 0 : CHK_RET(InitAndVerifySingleSignal(linkP2p.localIpcSignal[actualNotifyNum], ipcPostWaitNotify));
1142 0 : transDevP2pData.ipcPostWaitNotify = ipcPostWaitNotify;
1143 :
1144 0 : std::shared_ptr<RemoteNotify> ipcPostRecordNotify = std::make_shared<RemoteNotify>();
1145 0 : CHK_RET(InitAndVerifySingleSignal(linkP2p.remoteIpcSignal[actualNotifyNum], ipcPostRecordNotify));
1146 0 : transDevP2pData.ipcPostRecordNotify = ipcPostRecordNotify;
1147 0 : actualNotifyNum++;
1148 :
1149 0 : transDevP2pData.userLocalNotify.resize(notifyNum, nullptr);
1150 0 : transDevP2pData.userRemoteNotify.resize(notifyNum, nullptr);
1151 :
1152 0 : for (u32 idx = 0; idx < notifyNum; idx++) {
1153 0 : CHK_RET(CheckNotifyOrQPMaxNum(actualNotifyNum, LINK_P2P_MAX_NUM, true));
1154 0 : std::shared_ptr<LocalNotify> ipcWaitNotify = std::make_shared<LocalNotify>();
1155 0 : CHK_RET(InitAndVerifySingleSignal(linkP2p.localIpcSignal[actualNotifyNum], ipcWaitNotify));
1156 0 : transDevP2pData.userLocalNotify[idx] = ipcWaitNotify;
1157 :
1158 0 : std::shared_ptr<RemoteNotify> ipcRecordNotify = std::make_shared<RemoteNotify>();
1159 0 : CHK_RET(InitAndVerifySingleSignal(linkP2p.remoteIpcSignal[actualNotifyNum], ipcRecordNotify));
1160 0 : transDevP2pData.userRemoteNotify[idx] = ipcRecordNotify;
1161 :
1162 0 : actualNotifyNum++;
1163 0 : }
1164 :
1165 0 : HCCL_DEBUG("[%s]get p2pNotify success, actualNotifyNum[%u]", __func__, actualNotifyNum);
1166 0 : p2pNotifyNum = actualNotifyNum;
1167 :
1168 0 : return HCCL_SUCCESS;
1169 0 : }
1170 :
1171 0 : HcclResult HcclCommAicpu::SetTransportRoceQP(
1172 : TransportDeviceIbverbsData& transDevIbverbsData, u64& roceQpNum, HcclLinkRoceV2* linkRoce)
1173 : {
1174 0 : roceQpNum = linkRoce->qpsPerConnection;
1175 0 : u32 roceQpNumSum = linkRoce->qpsPerConnection + static_cast<u32>(linkRoce->qpsPerConnection != 1);
1176 0 : transDevIbverbsData.qpInfo.resize(roceQpNumSum);
1177 0 : std::copy_n(linkRoce->QpInfo, roceQpNumSum, transDevIbverbsData.qpInfo.begin());
1178 0 : transDevIbverbsData.multiQpThreshold = multiQpThreshold_;
1179 0 : transDevIbverbsData.qpsPerConnection = linkRoce->qpsPerConnection;
1180 0 : HCCL_INFO(
1181 : "[%s]transDevIbverbsData.qpInfo.qpPtr[%llu], transDevIbverbsData.qpInfo.sqIndex[%u], "
1182 : "transDevIbverbsData.qpInfo.dbIndex[%u], roceQpNum[%llu], roceQpNumSum[%u], multiQpThreshold_[%u]",
1183 : __func__, transDevIbverbsData.qpInfo[0].qpPtr, transDevIbverbsData.qpInfo[0].sqIndex,
1184 : transDevIbverbsData.qpInfo[0].dbIndex, roceQpNum, roceQpNumSum, multiQpThreshold_);
1185 0 : return HCCL_SUCCESS;
1186 : }
1187 :
1188 0 : HcclResult HcclCommAicpu::SetTransportRoceNotify(
1189 : TransportDeviceIbverbsData& transDevIbverbsData, u64& roceNotifyNum, HcclLinkRoceV2* linkRoce,
1190 : [[maybe_unused]] u32 notifyNum)
1191 : {
1192 0 : u64 actualNotifyNum = 0;
1193 0 : if (linkRoce->localNotifyList == 0 || linkRoce->remoteNotifyList == 0) {
1194 0 : HCCL_DEBUG("[%s] Empty local and remote notify lists, skipping notify resource creation.", __func__);
1195 0 : return HCCL_SUCCESS;
1196 : }
1197 0 : HcclSignalInfo* localNotifyList = reinterpret_cast<HcclSignalInfo*>(linkRoce->localNotifyList);
1198 0 : AddrKey* remoteNotifyList = reinterpret_cast<AddrKey*>(linkRoce->remoteNotifyList);
1199 0 : if (localNotifyList == nullptr || remoteNotifyList == nullptr) {
1200 0 : HCCL_ERROR("[%s]nullptr found in localNotifyList or remoteNotifyList from device mem, check.", __func__);
1201 0 : return HCCL_E_INTERNAL;
1202 : }
1203 : // 获取RDMA Notify信息
1204 0 : std::shared_ptr<LocalNotify> ackNotify = std::make_shared<LocalNotify>();
1205 0 : CHK_RET(InitAndVerifySingleSignal(localNotifyList[actualNotifyNum], ackNotify));
1206 0 : transDevIbverbsData.ackNotify = ackNotify;
1207 0 : transDevIbverbsData.remoteAckNotifyDetails = remoteNotifyList[actualNotifyNum];
1208 0 : actualNotifyNum++;
1209 :
1210 0 : std::shared_ptr<LocalNotify> dataNotify = std::make_shared<LocalNotify>();
1211 0 : CHK_RET(InitAndVerifySingleSignal(localNotifyList[actualNotifyNum], dataNotify));
1212 0 : transDevIbverbsData.dataNotify = dataNotify;
1213 0 : transDevIbverbsData.remoteDataNotifyDetails = remoteNotifyList[actualNotifyNum];
1214 0 : actualNotifyNum++;
1215 :
1216 0 : std::shared_ptr<LocalNotify> dataAckNotify = std::make_shared<LocalNotify>();
1217 0 : CHK_RET(InitAndVerifySingleSignal(localNotifyList[actualNotifyNum], dataAckNotify));
1218 0 : transDevIbverbsData.dataAckNotify = dataAckNotify;
1219 0 : transDevIbverbsData.remoteDataAckNotifyDetails = remoteNotifyList[actualNotifyNum];
1220 0 : transDevIbverbsData.notifySize = notifySize_;
1221 0 : actualNotifyNum++;
1222 :
1223 0 : transDevIbverbsData.userLocalNotify.resize(linkRoce->qpsPerConnection);
1224 0 : transDevIbverbsData.userRemoteNotifyDetails.resize(linkRoce->qpsPerConnection);
1225 : // 当前多QP下每个QP会多申请一个DataNotify
1226 0 : u64 singleQpNotifySize = linkRoce->singleQPNotifyNum + static_cast<u32>(linkRoce->qpsPerConnection > 1);
1227 0 : for (u32 qpIndex = 0; qpIndex < linkRoce->qpsPerConnection; qpIndex++) {
1228 0 : transDevIbverbsData.userLocalNotify[qpIndex].resize(singleQpNotifySize, nullptr);
1229 0 : transDevIbverbsData.userRemoteNotifyDetails[qpIndex].resize(singleQpNotifySize);
1230 0 : for (u32 i = 0, idx = actualNotifyNum + singleQpNotifySize * qpIndex; i < singleQpNotifySize; ++idx, ++i) {
1231 0 : std::shared_ptr<LocalNotify> locNotify = std::make_shared<LocalNotify>();
1232 0 : CHK_RET(InitAndVerifySingleSignal(localNotifyList[idx], locNotify));
1233 0 : transDevIbverbsData.userLocalNotify[qpIndex][i] = locNotify;
1234 0 : transDevIbverbsData.userRemoteNotifyDetails[qpIndex][i] = remoteNotifyList[idx];
1235 0 : }
1236 : }
1237 0 : roceNotifyNum = linkRoce->singleQPNotifyNum;
1238 0 : HCCL_DEBUG("[%s]get roceNotify success, roceNotifyNum[%u]", __func__, roceNotifyNum);
1239 0 : return HCCL_SUCCESS;
1240 0 : }
1241 :
1242 0 : HcclResult HcclCommAicpu::InitLinkP2p(
1243 : HccltagRemoteResV2* tagRes, u32& rankId, const std::string& newTag, u32 notifyNum, TransportLinkType linkType)
1244 : {
1245 0 : std::unordered_map<u32, std::unordered_map<std::string, std::shared_ptr<Transport>>>& linkRes
1246 : = (linkType == TransportLinkType::SIO) ? linkResSio_ : linkRes_;
1247 0 : HcclLinkP2pV2& linkP2p = (linkType == TransportLinkType::SIO) ? tagRes->linkP2pSio : tagRes->linkP2p;
1248 :
1249 0 : if (linkRes.find(rankId) == linkRes.end() || linkRes[rankId].find(newTag) == linkRes[rankId].end()) {
1250 : // 优先校验notify去判定link是否有效
1251 0 : if (linkP2p.localIpcSignal[0].resId == INVALID_U64) {
1252 0 : HCCL_INFO(
1253 : "[%s]the link is invalid, no need to create transport, rankId[%u], newTag[%s]", __func__, rankId,
1254 : newTag.c_str());
1255 0 : return HCCL_SUCCESS;
1256 : }
1257 : // 创建Transport对象
1258 0 : MachinePara machinePara;
1259 0 : CHK_RET(SetTransportMachinePara(machinePara, rankId, newTag, linkType));
1260 0 : machinePara.notifyNum = notifyNum;
1261 : // 获取localMem & remoteMem
1262 0 : TransportDeviceP2pData transDevP2pData;
1263 0 : transDevP2pData.inputBufferPtr = reinterpret_cast<void*>((linkP2p.remoteMem)[INPUT].addr);
1264 0 : transDevP2pData.outputBufferPtr = reinterpret_cast<void*>((linkP2p.remoteMem)[OUTPUT].addr);
1265 0 : if (transDevP2pData.inputBufferPtr == nullptr || transDevP2pData.outputBufferPtr == nullptr) {
1266 0 : HCCL_ERROR(
1267 : "[%s]input ptr[%p] or output ptr[%p] is null.", __func__, transDevP2pData.inputBufferPtr,
1268 : transDevP2pData.outputBufferPtr);
1269 0 : return HCCL_E_PARA;
1270 : }
1271 : // 获取Notify资源
1272 0 : CHK_RET(SetTagRemoteRes(rankId, newTag, tagRes));
1273 0 : HccltagRemoteResV3* tagRemoteRes = &(rankTagRemoteRes_[rankId][newTag]);
1274 0 : CHK_RET(SetTransportPtpNotify(transDevP2pData, tagRemoteRes->p2pNotifyNum, linkP2p, notifyNum));
1275 : // 获取transportAttr信息
1276 0 : transDevP2pData.transportAttr = linkP2p.transportAttr;
1277 : // 创建Transport对象
1278 0 : std::shared_ptr<Transport> link;
1279 0 : TransportPara para{};
1280 0 : const std::unique_ptr<NotifyPool> notifyPool;
1281 0 : link.reset(new (std::nothrow) Transport(
1282 0 : TransportType::TRANS_TYPE_DEVICE_P2P, para, dispatcher_, notifyPool, machinePara, transDevP2pData));
1283 0 : CHK_SMART_PTR_NULL(link);
1284 0 : CHK_RET(link->Init());
1285 0 : linkRes[rankId][newTag] = link;
1286 0 : HCCL_INFO("[%s]linkRes_, rankId[%u], newTag[%s]", __func__, rankId, newTag.c_str());
1287 0 : }
1288 0 : return HCCL_SUCCESS;
1289 : }
1290 0 : HcclResult HcclCommAicpu::InitLinkRoce(
1291 : HccltagRemoteResV2* tagRes, HcclLinkRoceV2* linkRoce, u32& rankId, const std::string& newTag, u32 notifyNum,
1292 : const bool isBackup, const bool isSecond)
1293 : {
1294 : // 优先校验notify去判定link是否有效
1295 0 : if (linkRoce->localNotifyList == 0) {
1296 0 : HCCL_INFO(
1297 : "[%s]the link is invalid, no need to create transport, rankId[%u], newTag[%s], isBackup[%d]", __func__,
1298 : rankId, newTag.c_str(), isBackup);
1299 0 : return HCCL_SUCCESS;
1300 : }
1301 0 : HcclSignalInfo* localNotifyList = reinterpret_cast<HcclSignalInfo*>(linkRoce->localNotifyList);
1302 0 : if (localNotifyList[0].resId == INVALID_U64) {
1303 0 : HCCL_INFO(
1304 : "[%s]the link notify resource is invalid, no need to create transport, rankId[%u], newTag[%s], "
1305 : "resId[%llu], "
1306 : "isBackup[%d]",
1307 : __func__, rankId, newTag.c_str(), localNotifyList[0].resId, isBackup);
1308 0 : return HCCL_SUCCESS;
1309 : }
1310 :
1311 : // 创建Transport对象
1312 0 : MachinePara machinePara;
1313 0 : CHK_RET(SetTransportMachinePara(machinePara, rankId, newTag));
1314 0 : machinePara.notifyNum = notifyNum;
1315 : // 获取localMem & remoteMem
1316 0 : TransportDeviceIbverbsData transDevIbverbsData;
1317 0 : transDevIbverbsData.inputBufferPtr = reinterpret_cast<void*>((linkRoce->remoteMem)[INPUT].addr);
1318 0 : transDevIbverbsData.outputBufferPtr = reinterpret_cast<void*>((linkRoce->remoteMem)[OUTPUT].addr);
1319 0 : if (transDevIbverbsData.inputBufferPtr == nullptr || transDevIbverbsData.outputBufferPtr == nullptr) {
1320 0 : HCCL_ERROR(
1321 : "[%s]input ptr[%p] or output ptr[%p] is null.", __func__, transDevIbverbsData.inputBufferPtr,
1322 : transDevIbverbsData.outputBufferPtr);
1323 0 : return HCCL_E_PARA;
1324 : }
1325 0 : transDevIbverbsData.localInputMem = (linkRoce->localMem)[INPUT];
1326 0 : transDevIbverbsData.localOutputMem = (linkRoce->localMem)[OUTPUT];
1327 0 : transDevIbverbsData.localNotifyValueAddr = linkRoce->notifyValue;
1328 0 : transDevIbverbsData.notifyValueKey = linkRoce->notifyValueKey;
1329 0 : transDevIbverbsData.remoteInputKey = (linkRoce->remoteMem)[INPUT].key;
1330 0 : transDevIbverbsData.remoteOutputKey = (linkRoce->remoteMem)[OUTPUT].key;
1331 : // 获取QPinfo
1332 0 : CHK_RET(SetTagRemoteRes(rankId, newTag, tagRes));
1333 0 : HccltagRemoteResV3* tagRemoteRes = &(rankTagRemoteRes_[rankId][newTag]);
1334 0 : u64& roceQpNum = isBackup ? tagRemoteRes->qpNumBackup : tagRemoteRes->qpNum;
1335 0 : CHK_RET(SetTransportRoceQP(transDevIbverbsData, roceQpNum, linkRoce));
1336 : // 获取notify
1337 0 : u64& roceNotifyNum = isBackup ? tagRemoteRes->roceNotifyNumBackup : tagRemoteRes->roceNotifyNum;
1338 0 : CHK_RET(SetTransportRoceNotify(transDevIbverbsData, roceNotifyNum, linkRoce, notifyNum));
1339 0 : HCCL_INFO("[%s]transDevIbverbsData isBackup[%d]", __func__, isBackup);
1340 : // 获取atomic write
1341 0 : transDevIbverbsData.useAtomicWrite = linkRoce->useAtomicWrite;
1342 : // 创建Transport对象
1343 0 : std::shared_ptr<Transport> link;
1344 0 : TransportPara para{};
1345 0 : para.timeout = linkTimeOut_;
1346 0 : const std::unique_ptr<NotifyPool> notifyPool;
1347 0 : link.reset(new (std::nothrow) Transport(
1348 0 : TransportType::TRANS_TYPE_DEVICE_IBVERBS, para, dispatcher_, notifyPool, machinePara, TransportDeviceP2pData(),
1349 0 : transDevIbverbsData));
1350 0 : CHK_SMART_PTR_NULL(link);
1351 0 : CHK_RET(link->Init());
1352 0 : if (isBackup) {
1353 0 : linkRdmaResBackUp_[rankId][newTag].push_back(link);
1354 0 : HCCL_INFO(
1355 : "[%s]linkRdmaResBackUp_, rankId[%u], newTag[%s], isBackup[%d], isSecond[%d], qpNum[%u], notifyNum[%u]",
1356 : __func__, rankId, newTag.c_str(), isBackup, isSecond, roceQpNum, roceNotifyNum);
1357 : } else {
1358 0 : linkRdmaRes_[rankId][newTag].push_back(link);
1359 0 : HCCL_INFO(
1360 : "[%s]linkRdmaRes_, rankId[%u], newTag[%s], isBackup[%d], isSecond[%d], qpNum[%u], notifyNum[%u]", __func__,
1361 : rankId, newTag.c_str(), isBackup, isSecond, roceQpNum, roceNotifyNum);
1362 : }
1363 0 : return HCCL_SUCCESS;
1364 0 : }
1365 :
1366 0 : HcclResult HcclCommAicpu::InitLinkRoce(
1367 : HccltagRemoteResV2* tagRes, u32& rankId, const std::string& newTag, u32 notifyNum, const bool isBackup)
1368 : {
1369 0 : auto tempLinkRes = isBackup ? linkRdmaResBackUp_ : linkRdmaRes_;
1370 0 : if (tempLinkRes.find(rankId) == tempLinkRes.end()
1371 0 : || tempLinkRes[rankId].find(newTag) == tempLinkRes[rankId].end()) {
1372 0 : bool isBatchSendRecv = newTag.find("BatchSendRecv") != std::string::npos;
1373 0 : if (isBatchSendRecv) {
1374 : // 如果是batchsendrecv，相同rank，需要刷新两次transport，如果是主的话就刷新0,2 备就刷新1,3
1375 0 : if (isBackup) {
1376 0 : CHK_RET(InitLinkRoce(
1377 : tagRes, &(tagRes->linkRoce[AICPU_RETRY_LINKROCE_BACKUP]), rankId, newTag, notifyNum, isBackup));
1378 0 : CHK_RET(InitLinkRoce(
1379 : tagRes, &(tagRes->linkRoce[AICPU_RETRY_LINKROCE_BACKUP + 2]), rankId, newTag, notifyNum, isBackup,
1380 : true));
1381 : } else {
1382 0 : CHK_RET(InitLinkRoce(
1383 : tagRes, &(tagRes->linkRoce[AICPU_RETRY_LINKROCE_DEFAULT]), rankId, newTag, notifyNum, isBackup));
1384 0 : CHK_RET(InitLinkRoce(
1385 : tagRes, &(tagRes->linkRoce[AICPU_RETRY_LINKROCE_DEFAULT + 2]), rankId, newTag, notifyNum, isBackup,
1386 : true));
1387 : }
1388 : } else {
1389 : // 非batchsendrecv只刷新主0，备1
1390 0 : HcclLinkRoceV2* linkRoce = isBackup ? &(tagRes->linkRoce[AICPU_RETRY_LINKROCE_BACKUP]) :
1391 : &(tagRes->linkRoce[AICPU_RETRY_LINKROCE_DEFAULT]);
1392 0 : CHK_RET(InitLinkRoce(tagRes, linkRoce, rankId, newTag, notifyNum, isBackup));
1393 : }
1394 : }
1395 0 : return HCCL_SUCCESS;
1396 0 : }
1397 :
1398 0 : HcclResult HcclCommAicpu::InitRemoteTagRes(
1399 : u32& rankId, const ListCommon& head, const std::string& newTag, u32 notifyNum, TransportLinkType linkType)
1400 : {
1401 0 : HCCL_RUN_INFO(
1402 : "[%s] Entry parse remote resource rankId[%u], group[%s], newTag[%s], head[%p], "
1403 : "linkType[%d]",
1404 : __func__, rankId, identifier_.c_str(), newTag.c_str(), &head, linkType);
1405 0 : ListCommon* curList = reinterpret_cast<ListCommon*>(head.nextDevice);
1406 0 : if (curList == nullptr) {
1407 0 : HCCL_ERROR("[%s]cur list ptr is null.", __func__);
1408 0 : return HCCL_E_PARA;
1409 : }
1410 :
1411 0 : HccltagRemoteResV2* tagRes = nullptr;
1412 0 : while (curList != &head) {
1413 0 : HccltagRemoteResV2* tagResTemp = list_entry(curList, HccltagRemoteResV2, nextTagRes);
1414 0 : if (strcmp(tagResTemp->tag, newTag.c_str()) == 0) {
1415 0 : tagRes = tagResTemp;
1416 0 : break;
1417 : }
1418 :
1419 0 : curList = reinterpret_cast<ListCommon*>(curList->nextDevice);
1420 0 : if (curList == nullptr) {
1421 0 : HCCL_ERROR("[%s]next list ptr is null.", __func__);
1422 0 : return HCCL_E_PARA;
1423 : }
1424 : }
1425 :
1426 0 : if (tagRes == nullptr) {
1427 0 : HCCL_ERROR(
1428 : "[%s]newTag[%s] not found, rankId[%u], head[%p], curList[%p], nextList[%llu], notifyNum[%u], "
1429 : "linkType[%d]",
1430 : __func__, newTag.c_str(), rankId, &head, curList, curList->nextDevice, notifyNum, linkType);
1431 0 : return HCCL_E_PARA;
1432 : }
1433 :
1434 0 : HCCL_INFO(
1435 : "[%s]newTag[%s], rankId[%u], head[%p], curList[%p], nextList[%llu], notifyNum[%u], linkType[%d]", __func__,
1436 : newTag.c_str(), rankId, &head, curList, curList->nextDevice, notifyNum, linkType);
1437 0 : if (linkType != TransportLinkType::RDMA) {
1438 : // 创建P2P链路
1439 0 : CHK_RET(InitLinkP2p(tagRes, rankId, newTag, notifyNum, linkType));
1440 : } else {
1441 : // 创建roce链路
1442 0 : CHK_RET(InitLinkRoce(tagRes, rankId, newTag, notifyNum));
1443 : // 创建roce链路（备用链路）
1444 0 : CHK_RET(InitLinkRoce(tagRes, rankId, newTag, notifyNum, true));
1445 : }
1446 0 : HCCL_INFO(
1447 : "[%s] End parse remote resource rankId[%u] tag[%s], newTag[%s]", __func__, rankId, identifier_.c_str(),
1448 : newTag.c_str());
1449 0 : return HCCL_SUCCESS;
1450 : }
1451 :
1452 0 : HcclResult HcclCommAicpu::RefreshTransportsResForRank(
1453 : const HcclOpResParam* commParam, u32 rankId, const std::string& newTag, u32 notifyNum, TransportLinkType linkType)
1454 : {
1455 0 : if (rankId >= AICPU_MAX_RANK_NUM) {
1456 0 : HCCL_ERROR(
1457 : "[%s] rankId[%u] overflow for group[%s], newTag[%s]", __func__, rankId, identifier_.c_str(),
1458 : newTag.c_str());
1459 0 : return HCCL_E_PARA;
1460 : }
1461 0 : if (commParam->remoteRes[rankId].nextDevicePtr == 0) {
1462 0 : return HCCL_SUCCESS;
1463 : }
1464 0 : HcclRankRelationResV2* rankRelationResPtr
1465 0 : = reinterpret_cast<HcclRankRelationResV2*>(commParam->remoteRes[rankId].nextDevicePtr);
1466 0 : if (rankRelationResPtr == nullptr) {
1467 0 : HCCL_ERROR(
1468 : "[%s]rank relation resource ptr is null, commParam->remoteRes[rankId].nextDevicePtr[%p],"
1469 : " rankId[%u]",
1470 : __func__, reinterpret_cast<HcclRankRelationResV2*>(commParam->remoteRes[rankId].nextDevicePtr), rankId);
1471 0 : return HCCL_E_PARA;
1472 : }
1473 :
1474 : // 1. init公共参数（对应remoteWorldRank，remoteUsrRankId暂不处理：windowsIn，windowsOut）
1475 0 : rankData_[rankId].remoteWorldRank = rankRelationResPtr->remoteWorldRank;
1476 0 : rankData_[rankId].remoteUsrRankId = rankRelationResPtr->remoteUsrRankId;
1477 : // 2. 遍历链表，获取HccltagRemoteResV2创建Tranport对象
1478 0 : if (reinterpret_cast<ListCommon*>(rankRelationResPtr->nextTagRes.nextDevice) != &(rankRelationResPtr->nextTagRes)) {
1479 0 : HCCL_DEBUG(
1480 : "[%s] Start to parse rankId[%u] tag resources, head[%p], nextDevice[%p], pre Device[%p], group[%s]",
1481 : __func__, rankId, &rankRelationResPtr->nextTagRes, rankRelationResPtr->nextTagRes.nextDevice,
1482 : rankRelationResPtr->nextTagRes.preDevice, identifier_.c_str());
1483 0 : CHK_RET(InitRemoteTagRes(rankId, rankRelationResPtr->nextTagRes, newTag, notifyNum, linkType));
1484 : } else {
1485 0 : HCCL_ERROR(
1486 : "[%s]could not find member in rankRelationRes list, rankId[%u], head[%p], nextDevice[%p]", __func__, rankId,
1487 : &rankRelationResPtr->nextTagRes, rankRelationResPtr->nextTagRes.nextDevice);
1488 0 : return HCCL_E_PARA;
1489 : }
1490 0 : HCCL_INFO(
1491 : "[%s] process success rankId[%u], group[%s], newTag[%s]", __func__, rankId, identifier_.c_str(),
1492 : newTag.c_str());
1493 0 : return HCCL_SUCCESS;
1494 : }
1495 :
1496 0 : HcclResult HcclCommAicpu::GetRdmaLinksByRankAndTag(
1497 : const HcclOpResParam* commParam, CommTransportsType type, u32 rankId, const std::string& newTag, LINK& link,
1498 : bool isBackup, u32 notifyNum, bool isSecond)
1499 : {
1500 0 : HCCL_INFO(
1501 : "[%s] Start to rdma get Link group[%s], rankId[%u], newTag[%s], isBackup[%d], deviceLogicId[%d],"
1502 : "notifyNum[%u], isSecond[%d]",
1503 : __func__, identifier_.c_str(), rankId, newTag.c_str(), isBackup, commParam->topoInfo.deviceLogicId, notifyNum,
1504 : isSecond);
1505 :
1506 0 : auto* linkRes = isBackup ? &linkRdmaResBackUp_ : &linkRdmaRes_;
1507 0 : auto iterRankLinks = linkRes->find(rankId);
1508 0 : if (iterRankLinks == linkRes->end() || iterRankLinks->second.find(newTag) == iterRankLinks->second.end()) {
1509 0 : HCCL_INFO(
1510 : "[%s] could not find link resource, rankId[%u], group[%s], newTag[%s]", __func__, rankId,
1511 : identifier_.c_str(), newTag.c_str());
1512 0 : CHK_RET(RefreshTransportsResForRank(commParam, rankId, newTag, notifyNum, TransportLinkType::RDMA));
1513 0 : iterRankLinks = linkRes->find(rankId);
1514 0 : if (iterRankLinks == linkRes->end() || iterRankLinks->second.find(newTag) == iterRankLinks->second.end()) {
1515 0 : HCCL_ERROR(
1516 : "[%s] refresh transport failed, newTag[%s], remoteUserRankId[%u]", __func__, newTag.c_str(), rankId);
1517 0 : return HCCL_E_INTERNAL;
1518 : }
1519 : }
1520 0 : if (isSecond && iterRankLinks->second[newTag].size() <= 1) {
1521 0 : HCCL_ERROR("[%s] get rdma Link failed, newTag[%s], remoteUserRankId[%u]", __func__, newTag.c_str(), rankId);
1522 0 : return HCCL_E_INTERNAL;
1523 : }
1524 :
1525 0 : link = isSecond ? iterRankLinks->second[newTag][1] : iterRankLinks->second[newTag][0];
1526 :
1527 0 : if (receivedAcks_.find(rankId) == receivedAcks_.end()) {
1528 0 : HCCL_ERROR("[%s]there is no link with rankId[%u]", __func__, rankId);
1529 0 : return HCCL_E_NOT_FOUND;
1530 : }
1531 0 : link->SetSupportDataReceivedAck(receivedAcks_[rankId]);
1532 0 : HCCL_DEBUG(
1533 : "[HcclCommAicpu][GetLinksByRankAndTag]rankid[%d] supportDataReceivedAck is %d", rankId, receivedAcks_[rankId]);
1534 0 : HCCL_INFO(
1535 : "[%s] group[%s], newTag[%s], rankId[%u] type[%u] success!", __func__, identifier_.c_str(), newTag.c_str(),
1536 : rankId, type);
1537 0 : return HCCL_SUCCESS;
1538 : }
1539 :
1540 0 : HcclResult HcclCommAicpu::GetSdmaLinksByRankAndTag(
1541 : const HcclOpResParam* commParam, CommTransportsType type, u32 rankId, const std::string& newTag, LINK& link,
1542 : bool isBackup, u32 notifyNum, TransportLinkType linkType)
1543 : {
1544 0 : HCCL_INFO(
1545 : "[%s] Start to get sdma Link group[%s], rankId[%u], newTag[%s], isBackup[%d], deviceLogicId[%d], "
1546 : "notifyNum[%u], linkType[%d]",
1547 : __func__, identifier_.c_str(), rankId, newTag.c_str(), isBackup, commParam->topoInfo.deviceLogicId, notifyNum,
1548 : linkType);
1549 :
1550 0 : auto* linkRes = &linkRes_;
1551 0 : if (linkType == TransportLinkType::SIO) { // HCCS SIO并发场景，HCCS链路与SIO链路分开管理
1552 0 : linkRes = &linkResSio_;
1553 : }
1554 0 : auto iterRankLinks = linkRes->find(rankId);
1555 0 : if (iterRankLinks == linkRes->end() || iterRankLinks->second.find(newTag) == iterRankLinks->second.end()) {
1556 0 : HCCL_INFO(
1557 : "[%s] could not find link resource, rankId[%u], group[%s], newTag[%s]", __func__, rankId,
1558 : identifier_.c_str(), newTag.c_str());
1559 0 : CHK_RET(RefreshTransportsResForRank(commParam, rankId, newTag, notifyNum, linkType));
1560 0 : iterRankLinks = linkRes->find(rankId);
1561 0 : if (iterRankLinks == linkRes->end() || iterRankLinks->second.find(newTag) == iterRankLinks->second.end()) {
1562 0 : HCCL_ERROR(
1563 : "[%s] refresh transport failed, newTag[%s], remoteUserRankId[%u], %p", __func__, newTag.c_str(), rankId,
1564 : linkRes);
1565 0 : return HCCL_E_INTERNAL;
1566 : }
1567 : }
1568 :
1569 0 : link = iterRankLinks->second[newTag];
1570 0 : if (receivedAcks_.find(rankId) == receivedAcks_.end()) {
1571 0 : HCCL_ERROR("[%s]there is no link with rankId[%u]", __func__, rankId);
1572 0 : return HCCL_E_NOT_FOUND;
1573 : }
1574 0 : link->SetSupportDataReceivedAck(receivedAcks_[rankId]);
1575 0 : HCCL_DEBUG(
1576 : "[HcclCommAicpu][GetLinksByRankAndTag]rankid[%d] supportDataReceivedAck is %d", rankId, receivedAcks_[rankId]);
1577 0 : HCCL_INFO(
1578 : "[%s] group[%s], newTag[%s], rankId[%u] type[%u] success!", __func__, identifier_.c_str(), newTag.c_str(),
1579 : rankId, type);
1580 0 : return HCCL_SUCCESS;
1581 : }
1582 :
1583 0 : HcclResult HcclCommAicpu::CleanRoceResource(
1584 : const std::string& newTag, AlgResourceResponse& algResResponse,
1585 : [[maybe_unused]] const std::map<u32, bool>& remoteRankPortMap, [[maybe_unused]] const OpParam& param)
1586 : {
1587 0 : HCCL_INFO("[%s] Entry alloc transport group[%s], tag[%s]", __func__, identifier_.c_str(), newTag.c_str());
1588 :
1589 0 : for (auto& levelNSubCommTransport : algResResponse.opTransportResponse) {
1590 0 : for (auto& singleSubCommTransport : levelNSubCommTransport) {
1591 0 : for (auto& transportRequest : singleSubCommTransport.transportRequests) {
1592 0 : if (transportRequest.isValid && transportRequest.isUsedRdma) {
1593 0 : u32 remoteUserRank = transportRequest.remoteUserRank;
1594 0 : linkRdmaRes_[remoteUserRank].erase(newTag);
1595 0 : linkRdmaResBackUp_[remoteUserRank].erase(newTag);
1596 : }
1597 : }
1598 : }
1599 : }
1600 :
1601 0 : return HCCL_SUCCESS;
1602 : }
1603 :
1604 1 : HcclResult HcclCommAicpu::CleanAllRoceResource()
1605 : {
1606 1 : HCCL_INFO("Clean all link rdna resources");
1607 : // 清空主链路资源
1608 1 : linkRdmaRes_.clear();
1609 : // 清空备链路资源
1610 1 : linkRdmaResBackUp_.clear();
1611 1 : return HCCL_SUCCESS;
1612 : }
1613 :
1614 2 : HcclResult HcclCommAicpu::ClearOpResource(const std::string& tag)
1615 : {
1616 : // 与 GetAlgResponseRes/AllocAlgResource/ReAllocTransportResource 互斥，避免读写竞争 resMap_/linkRes_
1617 2 : std::lock_guard<std::mutex> lock(preemptMutexForResMap_);
1618 :
1619 6 : auto eraseTagFromAll = [this](const std::string& t) -> bool {
1620 6 : bool hit = false;
1621 : // 主资源map：vector<LINK>析构会让shared_ptr<Transport>引用-1，归零时触发driver资源释放
1622 6 : if (resMap_.erase(t) > 0) {
1623 1 : hit = true;
1624 : }
1625 : // SDMA HCCS链路缓存：报错日志中driver halResAddrMap ioctl失败的根本累积点
1626 9 : for (auto& rankIt : linkRes_) {
1627 3 : if (rankIt.second.erase(t) > 0) {
1628 1 : hit = true;
1629 : }
1630 : }
1631 : // SDMA SIO并发链路缓存
1632 9 : for (auto& rankIt : linkResSio_) {
1633 3 : if (rankIt.second.erase(t) > 0) {
1634 1 : hit = true;
1635 : }
1636 : }
1637 : // RDMA主备链路缓存（CleanAllRoceResource按通信域全清，本接口按tag细粒度清）
1638 9 : for (auto& rankIt : linkRdmaRes_) {
1639 3 : if (rankIt.second.erase(t) > 0) {
1640 1 : hit = true;
1641 : }
1642 : }
1643 9 : for (auto& rankIt : linkRdmaResBackUp_) {
1644 3 : if (rankIt.second.erase(t) > 0) {
1645 1 : hit = true;
1646 : }
1647 : }
1648 : // 本地scratchMem和localTagResToObj按tag索引
1649 6 : if (tagScratchMem_.erase(t) > 0) {
1650 1 : hit = true;
1651 : }
1652 6 : if (localTagResToObj_.erase(t) > 0) {
1653 1 : hit = true;
1654 : }
1655 9 : for (auto& rankIt : rankTagRemoteRes_) {
1656 3 : if (rankIt.second.erase(t) > 0) {
1657 1 : hit = true;
1658 : }
1659 : }
1660 6 : return hit;
1661 2 : };
1662 :
1663 : // 与 host 端 ClearOpResource 对齐：尝试 base/_host/_device 三种后缀
1664 2 : bool found = false;
1665 2 : found = eraseTagFromAll(tag) || found;
1666 2 : found = eraseTagFromAll(tag + "_host") || found;
1667 2 : found = eraseTagFromAll(tag + "_device") || found;
1668 2 : if (!found) {
1669 1 : HCCL_WARNING(
1670 : "[HcclCommAicpu][ClearOpResource] no entries matched tag[%s] "
1671 : "(also tried _host/_device variants), group[%s]",
1672 : tag.c_str(), identifier_.c_str());
1673 : } else {
1674 1 : HCCL_INFO(
1675 : "[HcclCommAicpu][ClearOpResource] cleared resources for tag[%s], group[%s]", tag.c_str(),
1676 : identifier_.c_str());
1677 : }
1678 2 : return HCCL_SUCCESS;
1679 2 : }
1680 :
1681 : // 借轨重新刷新资源
1682 0 : HcclResult HcclCommAicpu::ReAllocTransportResource(
1683 : const std::string& newTag, AlgResourceResponse& algResResponse, std::map<u32, bool>& remoteRankPortMap,
1684 : const HcclOpResParam* commParam, const OpParam& param)
1685 : {
1686 0 : HCCL_INFO("[%s] Entry alloc transport group[%s], tag[%s]", __func__, identifier_.c_str(), newTag.c_str());
1687 0 : std::set<u32> bsrTansportRank;
1688 0 : for (auto& levelNSubCommTransport : algResResponse.opTransportResponse) {
1689 0 : for (auto& singleSubCommTransport : levelNSubCommTransport) {
1690 0 : singleSubCommTransport.links.clear();
1691 0 : singleSubCommTransport.links.reserve(singleSubCommTransport.transportRequests.size());
1692 0 : for (auto& transportRequest : singleSubCommTransport.transportRequests) {
1693 0 : singleSubCommTransport.links.push_back(nullptr);
1694 0 : if (transportRequest.isValid) {
1695 0 : HCCL_INFO(
1696 : "[%s] alloc transport, newTag[%s], rankId[%u], "
1697 : "input memory type[%u], output memory type[%u], ",
1698 : __func__, newTag.c_str(), transportRequest.remoteUserRank, transportRequest.inputMemType,
1699 : transportRequest.outputMemType);
1700 0 : receivedAcks_[transportRequest.remoteUserRank] = singleSubCommTransport.supportDataReceivedAck;
1701 0 : bool isBackup = remoteRankPortMap.find(transportRequest.remoteUserRank) != remoteRankPortMap.end()
1702 0 : && !remoteRankPortMap[transportRequest.remoteUserRank];
1703 0 : bool isSecondBuild = false;
1704 0 : bool isBatchSendRecv = newTag.find("BatchSendRecv") != std::string::npos;
1705 0 : if (transportRequest.isUsedRdma && isBatchSendRecv
1706 0 : && bsrTansportRank.find(transportRequest.remoteUserRank) != bsrTansportRank.end()) {
1707 : // 仅在batchsendrecv rdma下发的时候需要第二次刷新，实际第一次下发都刷好了，第二次就是get一下
1708 0 : isSecondBuild = true;
1709 : }
1710 : // A3 bsr远端是DirectNpu 链路的话则跳过
1711 0 : if ((param.opType == HcclCMDType::HCCL_CMD_BATCH_SEND_RECV)
1712 0 : && (param.BatchSendRecvDataDes.isDirectRemoteRank[transportRequest.remoteUserRank])) {
1713 0 : continue;
1714 : }
1715 0 : bsrTansportRank.insert(transportRequest.remoteUserRank);
1716 0 : CHK_RET(CreateLink(
1717 : newTag, transportRequest, commParam, singleSubCommTransport.links.back(),
1718 : transportRequest.notifyNum, isBackup, isSecondBuild));
1719 : }
1720 : }
1721 : }
1722 : }
1723 :
1724 0 : return HCCL_SUCCESS;
1725 0 : }
1726 :
1727 0 : HcclResult HcclCommAicpu::AllocTransportResource(
1728 : const std::string& newTag, const OpParam& opParam, const HcclOpResParam* commParam, AlgResourceRequest& resRequest,
1729 : AlgResourceResponse& algResResponse)
1730 : {
1731 0 : HCCL_INFO("[%s] Entry alloc transport group[%s]", __func__, identifier_.c_str());
1732 0 : algResResponse.opTransportResponse = resRequest.opTransport;
1733 :
1734 0 : std::set<u32> bsrTansportRank;
1735 0 : for (auto& levelNSubCommTransport : algResResponse.opTransportResponse) {
1736 0 : for (auto& singleSubCommTransport : levelNSubCommTransport) {
1737 0 : singleSubCommTransport.links.clear();
1738 0 : singleSubCommTransport.links.reserve(singleSubCommTransport.transportRequests.size());
1739 0 : for (auto& transportRequest : singleSubCommTransport.transportRequests) {
1740 0 : singleSubCommTransport.links.push_back(nullptr);
1741 0 : if (transportRequest.isValid) {
1742 0 : localUserRank_ = transportRequest.localUserRank;
1743 0 : receivedAcks_[transportRequest.remoteUserRank] = singleSubCommTransport.supportDataReceivedAck;
1744 0 : HCCL_DEBUG(
1745 : "[%s] alloc transport, newTag[%s], rankId[%u], input memory type[%u], "
1746 : "output memory type[%u], ",
1747 : __func__, newTag.c_str(), transportRequest.remoteUserRank, transportRequest.inputMemType,
1748 : transportRequest.outputMemType);
1749 :
1750 0 : bool isSecondBuild = false;
1751 0 : if (transportRequest.isUsedRdma && opParam.opType == HcclCMDType::HCCL_CMD_BATCH_SEND_RECV
1752 0 : && bsrTansportRank.find(transportRequest.remoteUserRank) != bsrTansportRank.end()) {
1753 : // 仅仅在batchsendrecv rdma下发的时候需要第二次刷新，实际第一次下发都刷好了，第二次就是get一下
1754 0 : isSecondBuild = true;
1755 : }
1756 : // A3 bsr远端是DirectNpu 链路的话则跳过
1757 0 : if ((opParam.opType == HcclCMDType::HCCL_CMD_BATCH_SEND_RECV)
1758 0 : && (opParam.BatchSendRecvDataDes.isDirectRemoteRank[transportRequest.remoteUserRank])) {
1759 0 : continue;
1760 : }
1761 0 : bsrTansportRank.insert(transportRequest.remoteUserRank);
1762 0 : CHK_RET(CreateLink(
1763 : newTag, transportRequest, commParam, singleSubCommTransport.links.back(),
1764 : transportRequest.notifyNum, false, isSecondBuild));
1765 : }
1766 : }
1767 : }
1768 : }
1769 :
1770 0 : return HCCL_SUCCESS;
1771 0 : }
1772 :
1773 : // 在resMap_[tag]对应原有通信资源的基础上继续增量建链，目前batchsendrecv会用到本接口
1774 0 : HcclResult HcclCommAicpu::IncreAllocTransportResource(
1775 : const std::string& newTag, const OpParam& opParam, const HcclOpResParam* commParam, AlgResourceRequest& resRequest,
1776 : AlgResourceResponse& algResResponse)
1777 : {
1778 0 : HCCL_INFO("[HcclCommAicpu][IncreAllocTransportResource] Entry alloc transport group[%s]", identifier_.c_str());
1779 0 : std::set<u32> bsrTansportRank;
1780 0 : for (u32 levelIndex = 0; levelIndex < resRequest.opTransport.size(); levelIndex++) {
1781 0 : for (u32 ringIndex = 0; ringIndex < resRequest.opTransport[levelIndex].size(); ringIndex++) {
1782 0 : SingleSubCommTransport& reqSingleSubComm = resRequest.opTransport[levelIndex][ringIndex];
1783 0 : SingleSubCommTransport& respSingleSubComm = algResResponse.opTransportResponse[levelIndex][ringIndex];
1784 0 : for (u32 rankIndex = 0; rankIndex < reqSingleSubComm.transportRequests.size(); rankIndex++) {
1785 0 : TransportRequest& transportRequest = reqSingleSubComm.transportRequests[rankIndex];
1786 0 : CHK_PRT_RET(
1787 : rankIndex >= respSingleSubComm.links.size(),
1788 : HCCL_ERROR(
1789 : "[HcclCommAicpu][IncreAllocTransportResource] The remote rank_id[%u] is larger than "
1790 : "the existent respSingleSubComm map size[%u]",
1791 : rankIndex, respSingleSubComm.links.size()),
1792 : HCCL_E_PARA);
1793 0 : if (respSingleSubComm.links[rankIndex] != nullptr
1794 0 : && respSingleSubComm.links[rankIndex]->GetLinkType() != hccl::LinkType::LINK_RESERVED) {
1795 0 : HCCL_INFO(
1796 : "[IncreAlloc] The link to remote userRank[%u] has existed", transportRequest.remoteUserRank);
1797 0 : continue;
1798 : }
1799 0 : if (transportRequest.isValid) {
1800 0 : receivedAcks_[transportRequest.remoteUserRank] = reqSingleSubComm.supportDataReceivedAck;
1801 0 : respSingleSubComm.transportRequests[rankIndex] = transportRequest;
1802 0 : HCCL_DEBUG(
1803 : "[HcclCommAicpu][IncreAllocTransportResource] alloc transport, newTag[%s], rankId[%u], "
1804 : "input memory type[%u], output memory type[%u], ",
1805 : newTag.c_str(), transportRequest.remoteUserRank, transportRequest.inputMemType,
1806 : transportRequest.outputMemType);
1807 0 : bool isSecondBuild = false;
1808 0 : if (transportRequest.isUsedRdma && opParam.opType == HcclCMDType::HCCL_CMD_BATCH_SEND_RECV
1809 0 : && bsrTansportRank.find(transportRequest.remoteUserRank) != bsrTansportRank.end()) {
1810 : // 仅仅在batchsendrecv rdma下发的时候需要第二次刷新，实际第一次下发都刷好了，第二次就是get一下
1811 0 : isSecondBuild = true;
1812 : }
1813 : // A3 bsr远端是DirectNpu 链路的话则跳过
1814 0 : if ((opParam.opType == HcclCMDType::HCCL_CMD_BATCH_SEND_RECV)
1815 0 : && (opParam.BatchSendRecvDataDes.isDirectRemoteRank[transportRequest.remoteUserRank])) {
1816 0 : continue;
1817 : }
1818 0 : bsrTansportRank.insert(transportRequest.remoteUserRank);
1819 0 : CHK_RET(CreateLink(
1820 : newTag, transportRequest, commParam, respSingleSubComm.links[rankIndex],
1821 : transportRequest.notifyNum, false, isSecondBuild));
1822 : }
1823 : }
1824 : }
1825 : }
1826 0 : return HCCL_SUCCESS;
1827 0 : }
1828 :
1829 0 : HcclResult HcclCommAicpu::CreateLink(
1830 : const std::string& newTag, TransportRequest& transportRequest, const HcclOpResParam* commParam, LINK& link,
1831 : u32 notifyNum, bool isBackup, bool isSecond) // 主备的选择
1832 : {
1833 0 : if (transportRequest.isUsedRdma) {
1834 0 : CHK_RET(GetRdmaLinksByRankAndTag(
1835 : commParam, CommTransportsType::SPECIAL, transportRequest.remoteUserRank, newTag, link, isBackup, notifyNum,
1836 : isSecond));
1837 : } else {
1838 0 : CHK_RET(GetSdmaLinksByRankAndTag(
1839 : commParam, CommTransportsType::SPECIAL, transportRequest.remoteUserRank, newTag, link, isBackup, notifyNum,
1840 : transportRequest.linkType));
1841 : }
1842 :
1843 0 : HCCL_DEBUG("[%s] alloc special transport success!, tag[%s]", __func__, newTag.c_str());
1844 0 : return HCCL_SUCCESS;
1845 : }
1846 :
1847 0 : HcclResult HcclCommAicpu::AllocLocalNotifysResource(
1848 : const std::string& newTag, const HcclOpResParam* commParam, const u32 notifyNum,
1849 : std::vector<std::shared_ptr<LocalNotify>>& notifiesMain, std::vector<std::shared_ptr<LocalNotify>>& notifiesAux)
1850 : {
1851 0 : HCCL_INFO(
1852 : "[HcclCommAicpu][AllocLocalNotifysResource]requesting for [%u] notifys, tag[%s].", notifyNum, newTag.c_str());
1853 0 : if (localNotifies_.capacity() < notifyNum) {
1854 0 : HCCL_ERROR(
1855 : "[HcclCommAicpu][AllocLocalNotifysResource]request number exceed max notify numbers, alloc failed. Max "
1856 : "number is [%u],request num[%u], tag[%s]",
1857 : localNotifies_.capacity(), notifyNum, newTag.c_str());
1858 0 : return HCCL_E_PARA;
1859 : }
1860 :
1861 0 : if (localNotifies_.size() < notifyNum) {
1862 0 : if (InitLocalNotifyObj(commParam) != HCCL_SUCCESS || localNotifies_.size() < notifyNum) {
1863 0 : HCCL_ERROR(
1864 : "[HcclCommAicpu][AllocLocalNotifysResource] the need of notify is more than the available, group[%s], "
1865 : "need[%u], total[%u]",
1866 : newTag.c_str(), notifyNum, localNotifies_.size());
1867 0 : return HCCL_E_INTERNAL;
1868 : }
1869 : }
1870 :
1871 0 : u32 halfNotifyNum = notifyNum >> 1;
1872 0 : notifiesMain.resize(halfNotifyNum);
1873 0 : notifiesAux.resize(halfNotifyNum);
1874 0 : for (u32 i = 0; i < halfNotifyNum; i++) {
1875 0 : notifiesMain[i] = localNotifies_[i << 1];
1876 0 : notifiesAux[i] = localNotifies_[(i << 1) + 1];
1877 : }
1878 0 : HCCL_INFO(
1879 : "[HcclCommAicpu][AllocLocalNotifysResource]find enough notifys, numbers[%u], tag[%s].", notifyNum,
1880 : newTag.c_str());
1881 0 : return HCCL_SUCCESS;
1882 : }
1883 :
1884 0 : HcclResult HcclCommAicpu::AllocStreamsResource(
1885 : const std::string& newTag, const HcclOpResParam* commParam, const u32 streamNum, std::vector<Stream>& streams)
1886 : {
1887 0 : HCCL_INFO(
1888 : "[HcclCommAicpu][AllocStreamsResource]requesting for [%u] slave streams, newTag[%s], group[%s].", streamNum,
1889 : newTag.c_str(), identifier_.c_str());
1890 0 : if (streamNum == 0) {
1891 0 : return HCCL_SUCCESS;
1892 : }
1893 0 : if (slaveStreams_.capacity() < streamNum) {
1894 0 : HCCL_ERROR(
1895 : "[HcclCommAicpu][AllocStreamsResource]request number exceed max substream num, alloc failed. Max "
1896 : "number is [%u],request num[%u], tag[%s]",
1897 : slaveStreams_.capacity(), streamNum, newTag.c_str());
1898 0 : return HCCL_E_PARA;
1899 : }
1900 0 : if (slaveStreams_.size() < streamNum) {
1901 0 : if (InitSlaveStreamObjs(commParam) != HCCL_SUCCESS || slaveStreams_.size() < streamNum) {
1902 0 : HCCL_ERROR(
1903 : "[HcclCommAicpu][AllocStreamsResource] the need of streams is more than the "
1904 : "available, tag[%s], need[%u], total[%u]",
1905 : newTag.c_str(), streamNum, slaveStreams_.size());
1906 0 : return HCCL_E_INTERNAL;
1907 : }
1908 : }
1909 0 : streams = std::vector<Stream>(slaveStreams_.begin(), slaveStreams_.begin() + streamNum);
1910 0 : HCCL_INFO(
1911 : "[HcclCommAicpu][AllocStreamsResource]find enough slave streams [%u], tag[%s].", streamNum, newTag.c_str());
1912 0 : return HCCL_SUCCESS;
1913 : }
1914 :
1915 0 : HcclResult HcclCommAicpu::AllocScratchMemResource(
1916 : const std::string& newTag, const HcclOpResParam* commParam, const u64& scratchMemSize, DeviceMem& scratchMem,
1917 : bool reAllocFlag)
1918 : {
1919 0 : HCCL_INFO(
1920 : "[HcclCommAicpu][AllocScratchMemResource]requesting for [%llu] bytes scratch mem, tag[%s].", scratchMemSize,
1921 : newTag.c_str());
1922 0 : if (scratchMemSize != 0) {
1923 0 : if (tagScratchMem_.find(newTag) == tagScratchMem_.end()) {
1924 0 : HcclResult ret = InitLocalTagRes(commParam->localRes.nextTagRes);
1925 0 : CHK_PRT_RET(
1926 : ret != HCCL_SUCCESS,
1927 : HCCL_ERROR(
1928 : "[HcclCommAicpu][AllocScratchMemResource]InitLocalTagRes error group[%s]", identifier_.c_str()),
1929 : ret);
1930 : }
1931 :
1932 0 : if (tagScratchMem_.find(newTag) == tagScratchMem_.end()) {
1933 0 : HCCL_ERROR(
1934 : "[HcclCommAicpu][AllocScratchMemResource]alloc scratch memory failed."
1935 : "requesting for [%llu] bytes, tag[%s].",
1936 : scratchMemSize, newTag.c_str());
1937 0 : return HCCL_E_NOT_FOUND;
1938 : }
1939 :
1940 0 : if (reAllocFlag) {
1941 0 : HCCL_INFO(
1942 : "[HcclCommAicpu][AllocScratchMemResource]need to reAlloc mem [%llu] bytes scratch mem, tag[%s].",
1943 : scratchMemSize, newTag.c_str());
1944 0 : HcclResult ret = InitLocalTagRes(commParam->localRes.nextTagRes, reAllocFlag);
1945 0 : CHK_PRT_RET(
1946 : ret != HCCL_SUCCESS,
1947 : HCCL_ERROR(
1948 : "[HcclCommAicpu][AllocScratchMemResource]InitLocalTagRes error group[%s]", identifier_.c_str()),
1949 : ret);
1950 : }
1951 :
1952 : // 因为aicpu_communicator中会对scratchMem做对齐，所以tagScratchMem_中的大小会偏小（被对齐截断一部分）
1953 : // 但是两者的差值应该不能超过2个CCE_REDUCE_ALIGN_SIZE，否则应该是不对的
1954 0 : if (scratchMemSize - tagScratchMem_[newTag]->size() > (CCE_REDUCE_ALIGN_SIZE + CCE_REDUCE_ALIGN_SIZE)) {
1955 0 : HCCL_ERROR(
1956 : "[HcclCommAicpu][AllocScratchMemResource]alloc tag[%s] scratch memory failed."
1957 : "requesting [%llu] bytes actual [%llu] bytes",
1958 : newTag.c_str(), scratchMemSize, tagScratchMem_[newTag]->size());
1959 0 : return HCCL_E_PARA;
1960 : }
1961 0 : scratchMem = DeviceMem::create(tagScratchMem_[newTag]->ptr(), tagScratchMem_[newTag]->size());
1962 : }
1963 0 : HCCL_INFO(
1964 : "[HcclCommAicpu][AllocScratchMemResource]find enough [%llu] bytes scratch mem, tag[%s].", scratchMemSize,
1965 : newTag.c_str());
1966 0 : return HCCL_SUCCESS;
1967 : }
1968 :
1969 0 : HcclResult HcclCommAicpu::AllocAlgResource(
1970 : const std::string& newTag, const OpParam& opParam, const HcclOpResParam* commParam, AlgResourceRequest& resRequest,
1971 : AlgResourceResponse& algResResponse)
1972 : {
1973 0 : algResResponse.cclInputMem = cclInputBuffer_;
1974 0 : algResResponse.cclOutputMem = cclOutputBuffer_;
1975 0 : algResResponse.paramInputMem = DeviceMem::create(opParam.inputPtr, opParam.inputSize);
1976 0 : algResResponse.paramOutputMem = DeviceMem::create(opParam.outputPtr, opParam.outputSize);
1977 :
1978 0 : PetersonLockGuard guard(hostDeviceLock_.get());
1979 0 : CHK_PRT_RET(
1980 : guard.IsLockFailed(), HCCL_ERROR("[HcclCommAicpu][AllocAlgResource] hostDeviceLock lock failed"),
1981 : HCCL_E_INTERNAL);
1982 :
1983 0 : CHK_RET(AllocScratchMemResource(newTag, commParam, resRequest.scratchMemSize, algResResponse.scratchMem));
1984 0 : CHK_RET(AllocStreamsResource(newTag, commParam, resRequest.streamNum, algResResponse.slaveStreams));
1985 0 : CHK_RET(AllocLocalNotifysResource(
1986 : newTag, commParam, resRequest.notifyNum, algResResponse.notifiesMain, algResResponse.notifiesAux));
1987 0 : CHK_RET(AllocTransportResource(newTag, opParam, commParam, resRequest, algResResponse));
1988 0 : HCCL_INFO("[HcclCommAicpu][AllocAlgResource] alloc resource success tag[%s].", newTag.c_str());
1989 0 : return HCCL_SUCCESS;
1990 0 : }
1991 :
1992 0 : HcclResult HcclCommAicpu::CalcResRequest(
1993 : const std::string& algName, const OpParam& param, std::unique_ptr<CollExecutorBase>& executor,
1994 : AlgResourceRequest& resourceRequest)
1995 : {
1996 0 : if (executor.get() == nullptr) {
1997 0 : executor = CollAlgExecRegistry::Instance().GetAlgExec(algName, dispatcher_, topoMatcher_);
1998 0 : CHK_PRT_RET(
1999 : executor.get() == nullptr,
2000 : HCCL_ERROR("[HcclCommAicpu][CalcResRequest]Fail to find executor for algName[%s]", algName.c_str()),
2001 : HCCL_E_PARA);
2002 0 : executor->SetAlgType(algType_);
2003 0 : executor->SetCCLInBuffer(cclbufferSize_);
2004 :
2005 0 : if (param.opType == HcclCMDType::HCCL_CMD_REDUCE_SCATTER) {
2006 0 : bool isSupportSDMAReduce = false;
2007 0 : if (GetWorkflowMode() != HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE) {
2008 : isSupportSDMAReduce
2009 0 : = IsSupportSDMAReduce(param.inputPtr, param.outputPtr, param.DataDes.dataType, param.reduceType);
2010 : } else {
2011 0 : isSupportSDMAReduce = IsSupportSDMAReduce(
2012 0 : cclInputBuffer_.ptr(), cclOutputBuffer_.ptr(), param.DataDes.dataType, param.reduceType);
2013 : }
2014 0 : executor->SetIsSupportSDMAReduce(isSupportSDMAReduce);
2015 : }
2016 : }
2017 0 : CHK_RET(CalSendRecvInfoFor910B(algName, param, executor));
2018 0 : return executor->CalcResRequest(param, resourceRequest);
2019 : }
2020 :
2021 0 : u32 HcclCommAicpu::CalculateOpExecIndex(const OpParam& opParam, u32 userRank)
2022 : {
2023 0 : u32 opIndex = 0;
2024 0 : s32 commIndex = 0;
2025 : // 用于重执行和taskException打印的算子计数，bsr/sendrecv/其他算子分别计数
2026 0 : if (opParam.opType == HcclCMDType::HCCL_CMD_BATCH_SEND_RECV) {
2027 0 : commIndex = -1; // batchSendRecv使用 key = -1
2028 0 : } else if (opParam.opType == HcclCMDType::HCCL_CMD_SEND) {
2029 0 : commIndex = opParam.dstRank;
2030 0 : } else if (opParam.opType == HcclCMDType::HCCL_CMD_RECEIVE) {
2031 0 : commIndex = opParam.srcRank;
2032 : } else {
2033 0 : commIndex = userRank;
2034 : }
2035 :
2036 0 : auto it = opExecIndexMap_.find(commIndex);
2037 0 : if (it != opExecIndexMap_.end()) {
2038 0 : opIndex = ++(it->second);
2039 : } else {
2040 0 : opExecIndexMap_.insert({commIndex, 1});
2041 0 : opIndex = 1;
2042 : }
2043 :
2044 0 : HCCL_DEBUG(
2045 : "%s tag:%s opType:%u commIndex:%d opIndex:%u", __func__, opParam.tag.c_str(), opParam.opType, commIndex,
2046 : opIndex);
2047 0 : return opIndex;
2048 : }
2049 :
2050 2 : HcclResult HcclCommAicpu::PrepareSymmetricMemory(const OpParam& param, OpCommTransport& opTransportResponse)
2051 : {
2052 2 : CHK_PRT_RET(
2053 : opTransportResponse.size() == 0,
2054 : HCCL_ERROR("[HcclCommAicpu][PrepareSymmetricMemory] opTransportResponse size is 0"), HCCL_E_PARA);
2055 :
2056 : const std::unordered_set<LinkType> supportedLinkTypes
2057 2 : = {LinkType::LINK_HCCS, LinkType::LINK_SIO, LinkType::LINK_HCCS_SW};
2058 2 : for (u32 levelIdx = 0; levelIdx < opTransportResponse.size(); levelIdx++) {
2059 2 : for (auto& singleSubCommTransport : opTransportResponse[levelIdx]) {
2060 1 : if (singleSubCommTransport.isZeroCopy == false) {
2061 1 : continue;
2062 : }
2063 0 : for (u64 i = 0; i < singleSubCommTransport.links.size(); ++i) {
2064 0 : LINK& link = singleSubCommTransport.links[i];
2065 0 : if (link == nullptr || !singleSubCommTransport.transportRequests[i].isValid
2066 0 : || supportedLinkTypes.count(link->GetLinkType()) == 0) {
2067 0 : continue; // 无效或者不支持的链路
2068 : }
2069 0 : u32 peerRank = link->GetRemoteRank();
2070 0 : void* remoteIn = nullptr;
2071 0 : CHK_RET(HcclSymWinGetPeerPointer(param.inputSymWindow, param.inputOffset, peerRank, &remoteIn));
2072 0 : void* remoteOut = nullptr;
2073 0 : CHK_RET(HcclSymWinGetPeerPointer(param.outputSymWindow, param.outputOffset, peerRank, &remoteOut));
2074 :
2075 0 : CHK_PRT_RET(
2076 : remoteIn == nullptr || remoteOut == nullptr,
2077 : HCCL_ERROR(
2078 : "[HcclCommAicpu][PrepareSymmetricMemory] remoteRank[%d] in[%p] out[%p] is invalid", peerRank,
2079 : remoteIn, remoteOut),
2080 : HCCL_E_INTERNAL);
2081 0 : HCCL_INFO(
2082 : "[HcclCommAicpu][PrepareSymmetricMemory] remoteRank[%d] in[%p] out[%p]", peerRank, remoteIn,
2083 : remoteOut);
2084 0 : CHK_RET(link->UpdateRemoteAddr(remoteIn, remoteOut));
2085 : }
2086 : }
2087 : }
2088 1 : return HCCL_SUCCESS;
2089 1 : }
2090 :
2091 1 : HcclResult HcclCommAicpu::ExecOp(
2092 : const std::string& newTag, const std::string& algName, OpParam& opParam, const HcclOpResParam* commParam)
2093 : {
2094 1 : std::unique_ptr<CollExecutorBase> executor;
2095 : hccl::AlgResourceResponse* algResResponse;
2096 1 : CHK_RET(GetAlgResponseRes(newTag, algName, opParam, commParam, executor, algResResponse));
2097 :
2098 1 : if (isZeroCopy_ || isSymmetricMemory_) {
2099 1 : if (isSymmetricMemory_) {
2100 1 : CHK_RET(PrepareSymmetricMemory(opParam, algResResponse->opTransportResponse));
2101 : } else {
2102 0 : HcclResult ret = PrepareZeroCopyExchanger(newTag, opParam, algResResponse);
2103 0 : CHK_PRT_RET(
2104 : ret != HCCL_SUCCESS,
2105 : HCCL_ERROR(
2106 : "[HcclCommAicpu][ExecOp] newTag[%s], localRankId[%u]", newTag.c_str(), commParam->localUsrRankId),
2107 : ret);
2108 : }
2109 :
2110 : // 零拷贝场景scratchMem的大小会与用户的输入大小不同，会导致后续算法展开模块计算出错
2111 : // 但是该场景下不会直接访问scratchMem，因此直接使用输入作为scratchMem，使得后续计算正确
2112 0 : if (opParam.opType == HcclCMDType::HCCL_CMD_REDUCE_SCATTER) {
2113 0 : algResResponse->scratchMem = DeviceMem::create(opParam.inputPtr, opParam.inputSize);
2114 0 : HCCL_INFO(
2115 : "[HcclCommAicpu][ExecOp] ZeroCopy reduce-scatter use userInput as scratchMem, inputPtr[%p] "
2116 : "intputSize[%lu]",
2117 : opParam.inputPtr, opParam.inputSize);
2118 : }
2119 :
2120 0 : algResResponse->paramInputMem = DeviceMem::create(opParam.inputPtr, opParam.inputSize);
2121 0 : algResResponse->paramOutputMem = DeviceMem::create(opParam.outputPtr, opParam.outputSize);
2122 0 : HCCL_INFO(
2123 : "[HcclCommAicpu][ExecOp] zero copy modify paramInput paramOutput to algResResp inputPtr[%p] inputSize[%lu] "
2124 : "outputPtr[%p] outputSize[%lu]",
2125 : algResResponse->paramInputMem.ptr(), algResResponse->paramInputMem.size(),
2126 : algResResponse->paramOutputMem.ptr(), algResResponse->paramOutputMem.size());
2127 : }
2128 :
2129 0 : hcclOpExecIndex_ = CalculateOpExecIndex(opParam, localUserRank_);
2130 0 : HcclResult ret = Orchestrate(newTag, algName, opParam, executor, *algResResponse, commParam);
2131 0 : if (ret != HCCL_SUCCESS) {
2132 0 : HCCL_ERROR(
2133 : "[HcclCommAicpu][ExecOp] executor op fail, tag[%s], algName[%s], identifier[%s]", newTag.c_str(),
2134 : algName.c_str(), identifier_.c_str());
2135 0 : CHK_PRT_CONT(
2136 : retryEnable_, HCCL_ERROR("[HcclCommAicpu][ExecOp] executor op fail, some error logs may be recorded in the "
2137 : "log/run/device directory, search keyword [ErrToWarn]"));
2138 0 : if (printTaskExceptionForErr_) {
2139 0 : PrintTaskExceptionAllComm();
2140 0 : PrintAicpuCommExecStatus();
2141 0 : printTaskExceptionForErr_ = false;
2142 : }
2143 0 : return ret;
2144 : }
2145 :
2146 0 : HCCL_ENTRY_INFO(
2147 : commParam->opEntry,
2148 : "[HcclCommAicpu][ExecOp] executor op success tag[%s], newTag[%s], algName[%s], identifier[%s].",
2149 : opParam.tag.c_str(), newTag.c_str(), algName.c_str(), identifier_.c_str());
2150 0 : return HCCL_SUCCESS;
2151 1 : }
2152 :
2153 0 : HcclResult HcclCommAicpu::RefreshAlgResponseTransportRes(
2154 : const std::string& newTag, AlgResourceResponse& algResResponse, std::map<u32, bool>& remoteRankPortMap,
2155 : bool isChangeLinkFlag, const HcclOpResParam* commParam, const OpParam& param)
2156 : {
2157 0 : CHK_PTR_NULL(commParam);
2158 0 : auto iter = resMap_.find(newTag);
2159 0 : CHK_PRT_RET(
2160 : iter == resMap_.end(), HCCL_ERROR("[%s]Fail to find algResResponse for tag[%s]", __func__, newTag.c_str()),
2161 : HCCL_E_PARA);
2162 :
2163 0 : PetersonLockGuard guard(hostDeviceLock_.get());
2164 0 : CHK_PRT_RET(guard.IsLockFailed(), HCCL_ERROR("[%s] hostDeviceLock lock failed", __func__), HCCL_E_INTERNAL);
2165 0 : if (!isChangeLinkFlag) {
2166 0 : CleanRoceResource(newTag, algResResponse, remoteRankPortMap, param);
2167 0 : CHK_RET(ReAllocTransportResource(newTag, algResResponse, remoteRankPortMap, commParam, param));
2168 0 : HCCL_RUN_INFO("[%s] ChangeLinkFlag[%d], current tag[%s].", __func__, isChangeLinkFlag, newTag.c_str());
2169 : } else {
2170 : // 提前清理所有tag的链路，避免冲突
2171 0 : for (auto& resMapIt : resMap_) {
2172 0 : HCCL_RUN_INFO("[%s] clean roce resource of tag[%s].", __func__, resMapIt.first.c_str());
2173 0 : CleanRoceResource(resMapIt.first, resMapIt.second, remoteRankPortMap, param);
2174 : }
2175 : // 对resMap中所有tag的transport link根据主备进行刷新
2176 0 : for (auto& resMapIt : resMap_) {
2177 0 : HCCL_RUN_INFO("[%s] refresh algResResponse of tag[%s].", __func__, resMapIt.first.c_str());
2178 0 : CHK_RET(ReAllocTransportResource(resMapIt.first, resMapIt.second, remoteRankPortMap, commParam, param));
2179 0 : if (resMapIt.first == newTag) {
2180 0 : HCCL_RUN_INFO("[%s] current tag[%s].", __func__, newTag.c_str());
2181 0 : algResResponse = resMapIt.second;
2182 : }
2183 : }
2184 : }
2185 :
2186 0 : HCCL_RUN_INFO("[%s] alloc resource success tag[%s].", __func__, newTag.c_str());
2187 0 : return HCCL_SUCCESS;
2188 0 : }
2189 :
2190 0 : HcclResult HcclCommAicpu::CalSendRecvInfoForAlltoall(const OpParam& param)
2191 : {
2192 0 : if (param.opType == HcclCMDType::HCCL_CMD_ALLTOALL) {
2193 : std::vector<u64> sendCountMatrix(
2194 0 : topoInfo_.userRankSize * topoInfo_.userRankSize, param.All2AllDataDes.sendCount);
2195 0 : CHK_RET(GetAlltoAllvcSendRecvInfo(
2196 : static_cast<void*>(sendCountMatrix.data()), param.All2AllDataDes.sendType, param.All2AllDataDes.recvType));
2197 0 : } else if (param.opType == HcclCMDType::HCCL_CMD_ALLTOALLV) {
2198 0 : CHK_PTR_NULL(sendRecvInfoPtr_);
2199 0 : CHK_RET(
2200 : GetAlltoAllvSendRecvInfo(sendRecvInfoPtr_, param.All2AllDataDes.sendType, param.All2AllDataDes.recvType));
2201 : } else {
2202 0 : CHK_RET(GetAlltoAllvcSendRecvInfo(
2203 : param.All2AllDataDes.sendCountMatrix, param.All2AllDataDes.sendType, param.All2AllDataDes.recvType));
2204 : }
2205 0 : return HCCL_SUCCESS;
2206 : }
2207 :
2208 0 : HcclResult HcclCommAicpu::CalSendRecvInfoFor910B(
2209 : const std::string& algName, const OpParam& param, std::unique_ptr<CollExecutorBase>& executor)
2210 : {
2211 : // A2 AICPU才有机会走入RunAlltoAllVStaged
2212 0 : if (algName == "RunAlltoAllVStaged" || algName == "RunAlltoAllVFullMesh") {
2213 0 : CHK_PRT_RET(
2214 : executor.get() == nullptr,
2215 : HCCL_ERROR("[HcclCommAicpu][%s]Fail to find executor for algName[%s]", __func__, algName.c_str()),
2216 : HCCL_E_PARA);
2217 0 : CHK_RET(CalSendRecvInfoForAlltoall(param));
2218 0 : CollAlltoAllExecutor* alltoAllExecutor = dynamic_cast<CollAlltoAllExecutor*>(executor.get());
2219 0 : CHK_PTR_NULL(alltoAllExecutor);
2220 0 : CHK_RET(alltoAllExecutor->SetExcutorExtraInfo(allMeshAggregationSendRecvInfo_, cclbufferSize_));
2221 0 : HCCL_DEBUG("[HcclCommAicpu][%s] running algName[%s], prepare SendRecvInfo.", __func__, algName.c_str());
2222 0 : return HCCL_SUCCESS;
2223 : }
2224 0 : return HCCL_SUCCESS;
2225 : }
2226 :
2227 0 : HcclResult HcclCommAicpu::GetAlgResponseRes(
2228 : const std::string& newTag, const std::string& algName, const OpParam& opParam, const HcclOpResParam* commParam,
2229 : std::unique_ptr<CollExecutorBase>& executor, AlgResourceResponse*& algResResponse)
2230 : {
2231 0 : HCCL_INFO("[%s] algName[%s]", __func__, algName.c_str());
2232 0 : CHK_PTR_NULL(commParam);
2233 : // 刷新CCLBuffer
2234 0 : CHK_RET(InitCclbuffer(commParam));
2235 0 : auto iter = resMap_.find(newTag);
2236 0 : if (iter == resMap_.end()) {
2237 0 : std::lock_guard<std::mutex> lock(preemptMutexForResMap_);
2238 0 : iter = resMap_.find(newTag);
2239 0 : if (iter == resMap_.end()) {
2240 0 : HCCL_RUN_INFO("[%s] Alloc resource for alg[%s], tag[%s]", __func__, algName.c_str(), newTag.c_str());
2241 0 : AlgResourceRequest resRequest;
2242 0 : CHK_RET(CalcResRequest(algName, opParam, executor, resRequest));
2243 0 : CHK_RET(AllocAlgResource(newTag, opParam, commParam, resRequest, resMap_[newTag]));
2244 0 : iter = resMap_.find(newTag);
2245 0 : } else {
2246 0 : HCCL_INFO("[%s] Repeatedly inited for alg [%s] is not allowed.", __func__, algName.c_str());
2247 : }
2248 0 : } else if (algName == "BatchSendRecv" || algName == "BatchSendRecvRetry" || algName == "BatchSendRecvGroup") {
2249 : // 如果是非aclgraph模式，而且不需要增量建链，则跳过CalcResRequest这个计算，节省时间。在非aclgraph模式下，跳过是安全的。
2250 0 : bool canSkipCalcResRequest = !opParam.isCapture && !opParam.needIncreLink;
2251 0 : if (!canSkipCalcResRequest) { // 如果不能跳过计算，则走一遍计算的流程，否则就跳过以下计算
2252 0 : AlgResourceRequest resRequest;
2253 0 : HCCL_INFO("[%s]IncreAlloc resource for alg[%s], tag[%s]", __func__, algName.c_str(), newTag.c_str());
2254 0 : CHK_RET(CalcResRequest(algName, opParam, executor, resRequest));
2255 0 : CHK_RET(IncreAllocTransportResource(newTag, opParam, commParam, resRequest, resMap_[newTag]));
2256 0 : }
2257 0 : } else if (algName == "RunAlltoAllVStaged" || algName == "RunAlltoAllVFullMesh") {
2258 0 : AlgResourceRequest resRequest;
2259 0 : CHK_RET(CalcResRequest(algName, opParam, executor, resRequest));
2260 0 : HCCL_INFO(
2261 : "[%s] check if need refresh resource for alg[%s], tag[%s], old[%lu], new[%lu]", __func__, algName.c_str(),
2262 : newTag.c_str(), resMap_[newTag].scratchMem.size(), resRequest.scratchMemSize);
2263 0 : bool reAllocFlag = !(resMap_[newTag].scratchMem.size() == resRequest.scratchMemSize);
2264 0 : if (reAllocFlag) {
2265 0 : PetersonLockGuard guard(hostDeviceLock_.get());
2266 0 : CHK_PRT_RET(
2267 : guard.IsLockFailed(), HCCL_ERROR("[HcclCommAicpu][AllocAlgResource] hostDeviceLock lock failed"),
2268 : HCCL_E_INTERNAL);
2269 0 : CHK_RET(AllocScratchMemResource(
2270 : newTag, commParam, resRequest.scratchMemSize, resMap_[newTag].scratchMem, true));
2271 0 : HCCL_INFO(
2272 : "[%s] refresh resource success for alg[%s], tag[%s], scratchMemSize[%lu], ptr[%p]", __func__,
2273 : algName.c_str(), newTag.c_str(), resRequest.scratchMemSize, resMap_[newTag].scratchMem.ptr());
2274 0 : }
2275 0 : }
2276 0 : CHK_PRT_RET(
2277 : iter == resMap_.end(), HCCL_ERROR("[%s]Fail to find algResResponse for tag[%s]", __func__, newTag.c_str()),
2278 : HCCL_E_PARA);
2279 0 : algResResponse = &iter->second;
2280 0 : HCCL_INFO("[HcclCommAicpu][GetAlgResponseRes] success!");
2281 0 : return HCCL_SUCCESS;
2282 : }
2283 :
2284 0 : HcclResult HcclCommAicpu::GetAlltoAllVCTotalCount(OpParam& param, u64& sendCount, u64& recvCount)
2285 : {
2286 0 : for (u32 i = 0; i < topoInfo_.userRankSize; i++) {
2287 0 : sendCount += *(
2288 0 : static_cast<const u64*>(param.All2AllDataDes.sendCountMatrix) + topoInfo_.userRank * topoInfo_.userRankSize
2289 0 : + i);
2290 0 : recvCount += *(
2291 0 : static_cast<const u64*>(param.All2AllDataDes.sendCountMatrix) + topoInfo_.userRank
2292 0 : + topoInfo_.userRankSize * i);
2293 : }
2294 0 : return HCCL_SUCCESS;
2295 : }
2296 :
2297 0 : HcclResult HcclCommAicpu::GetAlltoAllTotalCount(OpParam& param, u64& sendCount, u64& recvCount)
2298 : {
2299 0 : sendCount = param.All2AllDataDes.sendCount * topoInfo_.userRankSize;
2300 0 : recvCount = param.All2AllDataDes.sendCount * topoInfo_.userRankSize;
2301 0 : return HCCL_SUCCESS;
2302 : }
2303 :
2304 0 : HcclResult HcclCommAicpu::GetAlltoAllVTotalCount(OpParam& param, u64& sendCount, u64& recvCount)
2305 : {
2306 0 : for (u32 i = 0; i < topoInfo_.userRankSize; i++) {
2307 0 : u64 curSendCount = *(static_cast<const u64*>(param.All2AllDataDes.sendCounts) + i)
2308 0 : + *(static_cast<const u64*>(param.All2AllDataDes.sdispls) + i);
2309 0 : sendCount = std::max(sendCount, curSendCount);
2310 0 : u64 curRecvCount = *(static_cast<const u64*>(param.All2AllDataDes.recvCounts) + i)
2311 0 : + *(static_cast<const u64*>(param.All2AllDataDes.rdispls) + i);
2312 0 : recvCount = std::max(recvCount, curRecvCount);
2313 : }
2314 0 : return HCCL_SUCCESS;
2315 : }
2316 :
2317 0 : HcclResult HcclCommAicpu::SetAlltoAllInputAndOutPutMem(OpParam& param, AlgResourceResponse& algResource)
2318 : {
2319 0 : u32 sendTypeSize = 0, recvTypeSize = 0;
2320 0 : CHK_RET(SalGetDataTypeSize(param.All2AllDataDes.sendType, sendTypeSize));
2321 0 : CHK_RET(SalGetDataTypeSize(param.All2AllDataDes.recvType, recvTypeSize));
2322 0 : u64 sendCount = 0;
2323 0 : u64 recvCount = 0;
2324 0 : if (param.opType == HcclCMDType::HCCL_CMD_ALLTOALL) {
2325 0 : CHK_RET(GetAlltoAllTotalCount(param, sendCount, recvCount));
2326 0 : } else if (param.opType == HcclCMDType::HCCL_CMD_ALLTOALLV) {
2327 0 : CHK_RET(GetAlltoAllVTotalCount(param, sendCount, recvCount));
2328 0 : } else if (param.opType == HcclCMDType::HCCL_CMD_ALLTOALLVC) {
2329 0 : CHK_RET(GetAlltoAllVCTotalCount(param, sendCount, recvCount));
2330 : }
2331 0 : u64 inputSize = sendCount * sendTypeSize;
2332 0 : u64 outputSize = recvCount * recvTypeSize;
2333 0 : algResource.paramInputMem = inputSize == 0 ? tinySendRecvMem_ : DeviceMem::create(param.inputPtr, inputSize);
2334 0 : algResource.paramOutputMem = outputSize == 0 ? tinySendRecvMem_ : DeviceMem::create(param.outputPtr, outputSize);
2335 0 : HCCL_DEBUG(
2336 : "[HcclCommAicpu][SetAlltoAllInputAndOutPutMem] Set memory for AllToAll, inputSize[%llu], inputPtr[%p],"
2337 : "outputPtr[%p]!",
2338 : inputSize, param.inputPtr, param.outputPtr);
2339 0 : return HCCL_SUCCESS;
2340 : }
2341 :
2342 0 : HcclResult HcclCommAicpu::CombineReportOpInfo(OpParam& param, bool isRetry, bool isRelay)
2343 : {
2344 0 : MsprofAicpuHCCLOPInfo hcclOpInfo{};
2345 0 : hcclOpInfo.relay = (isRelay) ? 1 : 0;
2346 0 : hcclOpInfo.retry = (isRetry) ? 1 : 0;
2347 0 : hcclOpInfo.dataType = param.DataDes.dataType;
2348 0 : hcclOpInfo.count = param.DataDes.count;
2349 0 : hcclOpInfo.groupName = groupHashId_;
2350 0 : hcclOpInfo.ranksize = topoInfo_.userRankSize;
2351 0 : std::string algTypeStr = TransferAlgType(algType_);
2352 0 : CHK_RET(dfx::ProfilingManager::ReportHcclOpInfo(hcclOpInfo, algTypeStr));
2353 0 : return HCCL_SUCCESS;
2354 0 : }
2355 :
2356 0 : HcclResult HcclCommAicpu::UpdateProfReportStartSqeIdx()
2357 : {
2358 0 : if (dfx::ProfilingManager::IsL1fromOffToOn()) {
2359 0 : std::vector<Stream> streams;
2360 0 : CHK_RET(GetStreamAll(streams));
2361 0 : for (auto& tmpStream : streams) {
2362 0 : HcclSqeContext* sqeContext = tmpStream.GetSqeContextPtr();
2363 0 : SqeRingBuffer* sqeContextBuffer = &(sqeContext->buffer);
2364 0 : CHK_RET(dfx::ProfilingManager::UpdateStartReportSqeIdx(tmpStream.id(), sqeContextBuffer->tailSqeIdx));
2365 : }
2366 0 : }
2367 0 : return HCCL_SUCCESS;
2368 : }
2369 :
2370 0 : HcclResult HcclCommAicpu::Orchestrate(
2371 : const std::string& newTag, const std::string& algName, OpParam& param, std::unique_ptr<CollExecutorBase>& executor,
2372 : AlgResourceResponse& algResource, const HcclOpResParam* commParam)
2373 : {
2374 0 : CHK_PTR_NULL(commParam);
2375 : // 算子下发信息记录在共享内存区
2376 0 : UpdateOpRingBufferIdx();
2377 0 : CHK_RET(aicpuShareData_.RecordOpInfo(
2378 : newTag, param, (isDeviceMode_ ? mc2OpIndex_ : hcclOpExecIndex_), localUserRank_, isCustom_));
2379 0 : CHK_RET(UpdateProfReportStartSqeIdx());
2380 :
2381 : // 每个算子都刷新一下profiling开关, 支持profiling从中间迭代采集
2382 0 : bool profL0Open = dfx::ProfilingManager::IsProfL0On();
2383 0 : bool profL1Open = dfx::ProfilingManager::IsProfL1On();
2384 0 : HCCL_DEBUG("profL0Open:%d, profL1Open:%d", profL0Open, profL1Open);
2385 :
2386 0 : LogControl logControl(false, false); // 重执行ERROR日志保底控制，析构时重置日志设置
2387 0 : HCCL_ENTRY_INFO(
2388 : commParam->opEntry, "[HcclCommAicpu][Orchestrate]start tag[%s] newTag[%s] algName[%s] identifier[%s]",
2389 : param.tag.c_str(), newTag.c_str(), algName.c_str(), identifier_.c_str());
2390 0 : HCCL_INFO(
2391 : "opRetryHandler.isInplacePreSync[%d] opRetryHandler.isPostSync[%d]",
2392 : algOpContext_.opRetryHandler.isInplacePreSync, algOpContext_.opRetryHandler.isPostSync);
2393 0 : if (executor.get() == nullptr) {
2394 0 : executor = CollAlgExecRegistry::Instance().GetAlgExec(algName, dispatcher_, topoMatcher_);
2395 0 : CHK_PRT_RET(
2396 : executor.get() == nullptr,
2397 : HCCL_ERROR(
2398 : "[HcclCommAicpu][Orchestrate]Fail to find executor "
2399 : "for algName[%s]",
2400 : algName.c_str()),
2401 : HCCL_E_PARA);
2402 0 : executor->SetAlgType(algType_);
2403 0 : executor->SetCCLInBuffer(cclbufferSize_);
2404 :
2405 0 : if (param.opType == HcclCMDType::HCCL_CMD_REDUCE_SCATTER) {
2406 0 : bool isSupportSDMAReduce = IsSupportSDMAReduce(
2407 0 : cclInputBuffer_.ptr(), cclOutputBuffer_.ptr(), param.DataDes.dataType, param.reduceType);
2408 0 : executor->SetIsSupportSDMAReduce(isSupportSDMAReduce);
2409 : }
2410 : }
2411 0 : if (param.opType == HcclCMDType::HCCL_CMD_ALLTOALL || param.opType == HcclCMDType::HCCL_CMD_ALLTOALLV
2412 0 : || param.opType == HcclCMDType::HCCL_CMD_ALLTOALLVC) {
2413 0 : CHK_RET(CalSendRecvInfoFor910B(algName, param, executor));
2414 0 : CHK_RET(SetAlltoAllInputAndOutPutMem(param, algResource));
2415 0 : if (algName == "RunAlltoAllVTwoLevelPipeline") {
2416 0 : CHK_RET(CalSendRecvInfoForAlltoall(param));
2417 0 : HCCL_DEBUG("[HcclCommAicpu][Orchestrate] running RunAlltoAllVTwoLevelPipeline, prepare SendRecvInfo.");
2418 0 : CollAlltoAllExecutor* alltoAllExecutor = dynamic_cast<CollAlltoAllExecutor*>(executor.get());
2419 0 : CHK_PTR_NULL(alltoAllExecutor);
2420 0 : CHK_RET(alltoAllExecutor->SetExcutorExtraInfo(allMeshAggregationSendRecvInfo_, cclbufferSize_));
2421 : }
2422 : }
2423 0 : auto waitStopExecCmdTimeoutMs = HcclGetWaitStopExecCmdTimeout();
2424 0 : auto waitStopExecCmdTimeout = std::chrono::milliseconds(waitStopExecCmdTimeoutMs);
2425 :
2426 0 : auto opStartTime = std::chrono::steady_clock::now(); // 记录重执行算子耗时
2427 0 : auto startTime = std::chrono::steady_clock::now();
2428 :
2429 0 : KfcError errorCode = KfcError::kNone;
2430 0 : uint32_t retryCnt = 0;
2431 0 : bool retryProcessing = false;
2432 0 : KfcCommand lastCmd = KfcCommand::kNone;
2433 0 : uint32_t beginSqePos = INVALID_UINT;
2434 0 : uint32_t endSqePos = INVALID_UINT;
2435 0 : HcclOpExecFSM state = HcclOpExecFSM::HCCL_OP_EXEC_FSM_INIT;
2436 0 : HcclResult ret = HCCL_SUCCESS;
2437 0 : dfxExtendInfo_.kfcStatus = DfxKfcStatus::kOneStart;
2438 0 : AicpuComContext* ctx = AicpuGetComContext();
2439 0 : AicpuHcclProcess::CallMC2MaintenanceThread(ctx);
2440 0 : u32 loopCnt = 0;
2441 0 : u32 loopNum = 1;
2442 0 : commParam_ = commParam;
2443 0 : CHK_RET(InitExecLoop(param, executor, loopNum));
2444 :
2445 : while (true) {
2446 0 : switch (state) {
2447 0 : case HcclOpExecFSM::HCCL_OP_EXEC_FSM_INIT:
2448 0 : HCCL_INFO("hccl aicpu execute loop %u", loopCnt);
2449 0 : ret = HcclOpExecFsmInitProcess(newTag, param, algResource, state, errorCode);
2450 0 : break;
2451 0 : case HcclOpExecFSM::HCCL_OP_EXEC_FSM_LAUNCH:
2452 0 : ret = HcclOpExecFsmLaunchProcess(
2453 : algName, param, executor, algResource, state, errorCode, beginSqePos, endSqePos, retryCnt);
2454 0 : if (ret == HCCL_E_SUSPENDING && isDeviceMode_ && retryEnable_) {
2455 0 : return HCCL_E_SUSPENDING;
2456 : }
2457 0 : break;
2458 0 : case HcclOpExecFSM::HCCL_OP_EXEC_FSM_WAIT_END:
2459 0 : ret = HcclOpExecFsmWaitEndProcess(
2460 0 : param, algResource, state, errorCode, retryCnt, param.tag, beginSqePos);
2461 0 : if (state == HcclOpExecFSM::HCCL_OP_EXEC_FSM_STOPPING) {
2462 0 : startTime = std::chrono::steady_clock::now();
2463 : }
2464 0 : break;
2465 0 : case HcclOpExecFSM::HCCL_OP_EXEC_FSM_STOPPING:
2466 0 : retryProcessing = true;
2467 0 : if ((std::chrono::steady_clock::now() - startTime) >= waitStopExecCmdTimeout) {
2468 0 : HCCL_ERROR("[OpRetry][AICPU]hccl aicpu wait stop exec timeout[%u ms].", waitStopExecCmdTimeoutMs);
2469 0 : errorCode = KfcError::kTimeout;
2470 0 : state = HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR;
2471 : } else {
2472 0 : ret = HcclOpExecFsmStoppingProcess(param, state, errorCode, retryCnt);
2473 : }
2474 0 : break;
2475 0 : case HcclOpExecFSM::HCCL_OP_EXEC_FSM_STOPPED:
2476 0 : ret = HcclOpExecFsmStoppedProcess(state, errorCode, retryCnt, algName, param, beginSqePos, endSqePos);
2477 0 : if (state == HcclOpExecFSM::HCCL_OP_EXEC_FSM_WAIT_RETRY) {
2478 0 : startTime = std::chrono::steady_clock::now();
2479 : }
2480 0 : break;
2481 0 : case HcclOpExecFSM::HCCL_OP_EXEC_FSM_CHANGE_LINK:
2482 0 : ret = HcclOpExecChangeLinkProcess(newTag, state, errorCode, retryCnt, algResource, commParam, param);
2483 0 : HCCL_DEBUG(
2484 : "[OpRetry][AICPU]retry change link finish, retryCnt:%u, tag:%s, state:%d", retryCnt,
2485 : param.tag.c_str(), state);
2486 0 : break;
2487 0 : case HcclOpExecFSM::HCCL_OP_EXEC_FSM_WAIT_RETRY: {
2488 0 : auto waitRetryCmdTimeoutMs = HcclGetWaitRetryCmdTimeout(retryCnt);
2489 0 : auto waitRetryCmdTimeout = std::chrono::milliseconds(waitRetryCmdTimeoutMs);
2490 0 : if ((std::chrono::steady_clock::now() - startTime) >= waitRetryCmdTimeout) {
2491 0 : HCCL_ERROR("[OpRetry][AICPU]hccl aicpu wait retry timeout[%u ms].", waitRetryCmdTimeoutMs);
2492 0 : errorCode = KfcError::kTimeout;
2493 0 : state = HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR;
2494 : } else {
2495 0 : ret = HcclOpExecFsmWaitRetryProcess(param, state, errorCode, lastCmd);
2496 : }
2497 0 : } break;
2498 0 : case HcclOpExecFSM::HCCL_OP_EXEC_FSM_RETRY:
2499 : // 重执行前清理当前算子展开的SQE缓存 (if any), 防止命中非完整的cache
2500 0 : CHK_RET(aicpuCacheManager_.ClearOpUnfoldCacheEntry(
2501 : algName, param, algResource, isDeviceMode_, topoInfo_, topoMatcher_, algOpContext_,
2502 : GetWorkflowMode()));
2503 0 : ret = HcclOpExecFsmRetryProcess(
2504 : algName, param, executor, algResource, state, errorCode, retryCnt, beginSqePos, endSqePos);
2505 0 : break;
2506 0 : case HcclOpExecFSM::HCCL_OP_EXEC_FSM_END:
2507 0 : loopCnt++;
2508 0 : if ((param.opType == HcclCMDType::HCCL_CMD_BATCH_SEND_RECV) && retryEnable_) {
2509 0 : param.BatchSendRecvDataDes.curIterNum = loopCnt;
2510 0 : ResetBSRRetryCnt();
2511 : }
2512 0 : if (loopCnt < loopNum) {
2513 0 : state = HcclOpExecFSM::HCCL_OP_EXEC_FSM_INIT;
2514 0 : break;
2515 : }
2516 0 : if (retryCnt > 0) {
2517 0 : RecordReportStatus(dfx::ReportStatus::kRetrySuccess);
2518 0 : retryProcessing = false;
2519 0 : auto opEndTime = std::chrono::steady_clock::now();
2520 0 : auto duration = std::chrono::duration_cast<std::chrono::seconds>(opEndTime - opStartTime).count();
2521 0 : HCCL_RUN_INFO(
2522 : "[OpRetry][AICPU]retry exec success, retryCnt [%u], tag [%s], take time [%ld]s", retryCnt,
2523 : param.tag.c_str(), duration);
2524 : }
2525 0 : CHK_RET(CombineReportOpInfo(param, (retryCnt > 0), false));
2526 0 : return HcclOpExecFsmEndProcess(retryCnt);
2527 0 : case HcclOpExecFSM::HCCL_OP_EXEC_STOP_LAUNCH:
2528 0 : HCCL_DEBUG("[NsRecovery][AICPU] stop the kernel");
2529 : // 停止前清理当前算子展开的SQE缓存 (if any), 防止host侧重新展开该算子并命中非完整的cache (例如step快恢)
2530 0 : CHK_RET(aicpuCacheManager_.ClearOpUnfoldCacheEntry(
2531 : algName, param, algResource, isDeviceMode_, topoInfo_, topoMatcher_, algOpContext_,
2532 : GetWorkflowMode()));
2533 0 : if (!needsResponseStopLaunch_) {
2534 0 : return HCCL_E_SUSPENDING;
2535 : } else {
2536 0 : HCCL_RUN_INFO("[NsRecovery][AICPU] stop the kernel for stop cmd");
2537 0 : needsResponseStopLaunch_ = false;
2538 0 : SetCommRecoveryFlag(true);
2539 0 : if (UpdateOpExecStatus(state, KfcStatus::kStoplaunch, errorCode, 0) == HCCL_SUCCESS) {
2540 0 : return HCCL_E_SUSPENDING;
2541 : } else {
2542 0 : break;
2543 : }
2544 : }
2545 0 : case HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR:
2546 : default: {
2547 0 : if (retryProcessing) {
2548 0 : RecordReportStatus(dfx::ReportStatus::kRetryFail);
2549 0 : retryProcessing = false;
2550 : }
2551 0 : UpdateOpExecStatus(state, excuteOpId_, KfcStatus::kRetryError, errorCode, retryCnt);
2552 0 : dfxExtendInfo_.kfcStatus = DfxKfcStatus::kOneFinished;
2553 0 : if (!isDeviceMode_) {
2554 0 : isOpLaunch = false;
2555 : }
2556 0 : HCCL_INFO("hccl aicpu set kfcStatus[%d]", dfxExtendInfo_.kfcStatus);
2557 0 : if (errorCode == KfcError::kExecConstraint) {
2558 0 : return HCCL_E_OPRETRY_FAIL;
2559 : }
2560 0 : return (ret == HCCL_SUCCESS) ? HCCL_E_INTERNAL : ret;
2561 : }
2562 : }
2563 0 : }
2564 : return ret;
2565 0 : }
2566 :
2567 1 : HcclResult HcclCommAicpu::InitBsrSendRecvOpIdAndExcuteOpId(
2568 : OpParam& param, AlgResourceResponse& algResource, HcclOpExecFSM& fsmState, KfcError& errorCode)
2569 : {
2570 1 : auto hcclRet = InitBatchSendRecvOpId(param, algResource);
2571 1 : HCCL_RETRY_CHK_RET_AND_TRANS_FSM(
2572 : hcclRet, HCCL_ERROR("InitBatchSendRecvOpId failed, ret:%u", hcclRet), KfcError::kInner,
2573 : HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR);
2574 1 : param.BatchSendRecvDataDes.curMode = BatchSendRecvCurMode::SEND_RECV;
2575 1 : if (param.BatchSendRecvDataDes.curIterNum == 0) {
2576 1 : bsrSendStream_ = algResource.slaveStreams[BSR_RETRY_SEND_STREAM_INDEX];
2577 1 : bsrRecvStream_ = algResource.slaveStreams[BSR_RETRY_RECV_STREAM_INDEX];
2578 : }
2579 1 : HCCL_INFO(
2580 : "BSR: iter %u, tag:%s index:%u", param.BatchSendRecvDataDes.curIterNum, excuteOpId_.tag, excuteOpId_.index);
2581 1 : HCCL_INFO(
2582 : "BSR: iter %u, send op Tag:%s index:%u", param.BatchSendRecvDataDes.curIterNum, bsrSendOpId_.tag,
2583 : bsrSendOpId_.index);
2584 1 : HCCL_INFO(
2585 : "BSR: iter %u, recv op Tag:%s index:%u", param.BatchSendRecvDataDes.curIterNum, bsrRecvOpId_.tag,
2586 : bsrRecvOpId_.index);
2587 1 : excuteOpId_.bsrInfo[HCCL_SEND].index = bsrSendOpId_.index;
2588 1 : excuteOpId_.bsrInfo[HCCL_RECV].index = bsrRecvOpId_.index;
2589 1 : excuteOpId_.bsrInfo[HCCL_SEND].tpQpn = bsrSendOpId_.bsrInfo[HCCL_SEND].tpQpn;
2590 1 : excuteOpId_.bsrInfo[HCCL_RECV].tpQpn = bsrRecvOpId_.bsrInfo[HCCL_RECV].tpQpn;
2591 1 : excuteOpId_.bsrInfo[HCCL_SEND].streamId = bsrSendOpId_.streamId;
2592 1 : excuteOpId_.bsrInfo[HCCL_RECV].streamId = bsrRecvOpId_.streamId;
2593 1 : excuteOpId_.bsrInfo[HCCL_SEND].srcRank = bsrSendOpId_.srcRank;
2594 1 : excuteOpId_.bsrInfo[HCCL_SEND].detRank = bsrSendOpId_.detRank;
2595 1 : excuteOpId_.bsrInfo[HCCL_RECV].srcRank = bsrRecvOpId_.srcRank;
2596 1 : excuteOpId_.bsrInfo[HCCL_RECV].detRank = bsrRecvOpId_.detRank;
2597 1 : CHK_SAFETY_FUNC_RET(memcpy_s(
2598 : excuteOpId_.bsrInfo[HCCL_SEND].bsrTag, sizeof(excuteOpId_.bsrInfo[HCCL_SEND].bsrTag), bsrSendOpId_.tag,
2599 : sizeof(bsrSendOpId_.tag)));
2600 1 : CHK_SAFETY_FUNC_RET(memcpy_s(
2601 : excuteOpId_.bsrInfo[HCCL_RECV].bsrTag, sizeof(excuteOpId_.bsrInfo[HCCL_RECV].bsrTag), bsrRecvOpId_.tag,
2602 : sizeof(bsrRecvOpId_.tag)));
2603 1 : return HCCL_SUCCESS;
2604 : }
2605 :
2606 1 : HcclResult HcclCommAicpu::HcclOpExecFsmInitProcess(
2607 : const std::string& newTag, OpParam& param, AlgResourceResponse& algResource, HcclOpExecFSM& fsmState,
2608 : KfcError& errorCode)
2609 : {
2610 1 : excuteOpId_.index = isDeviceMode_ ? (++mc2OpIndex_) : hcclOpExecIndex_;
2611 1 : CHK_SAFETY_FUNC_RET(memset_s(excuteOpId_.tag, sizeof(excuteOpId_.tag), 0, sizeof(excuteOpId_.tag)));
2612 1 : CHK_SAFETY_FUNC_RET(memcpy_s(excuteOpId_.tag, sizeof(excuteOpId_.tag), param.tag.c_str(), param.tag.size()));
2613 1 : CHK_SAFETY_FUNC_RET(memset_s(excuteOpId_.newTag, sizeof(excuteOpId_.newTag), 0, sizeof(excuteOpId_.newTag)));
2614 1 : CHK_SAFETY_FUNC_RET(memcpy_s(excuteOpId_.newTag, sizeof(excuteOpId_.newTag), newTag.c_str(), newTag.size()));
2615 1 : excuteOpId_.isSendRecv = false;
2616 1 : excuteOpId_.streamId = ~0u;
2617 1 : excuteOpId_.opType = param.opType;
2618 1 : excuteOpId_.isBsrTaskStart = false;
2619 1 : if (param.opType == HcclCMDType::HCCL_CMD_SEND || param.opType == HcclCMDType::HCCL_CMD_RECEIVE) {
2620 0 : InitSendRecvOpId(param, excuteOpId_);
2621 1 : } else if ((param.opType == HcclCMDType::HCCL_CMD_BATCH_SEND_RECV) && retryEnable_) {
2622 1 : CHK_RET(InitBsrSendRecvOpIdAndExcuteOpId(param, algResource, fsmState, errorCode));
2623 : }
2624 1 : if (GetNsStopLaunchStatus()) {
2625 0 : HCCL_WARNING("the op should not be launched in the suspending status");
2626 0 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_STOP_LAUNCH;
2627 0 : return HCCL_SUCCESS;
2628 : }
2629 1 : auto ret = aicpuHdc_.InitOpExecStatus(kfcStatusTransferD2H_, excuteOpId_);
2630 1 : isOpLaunch = true;
2631 1 : HCCL_INFO("%s tag:%s, isDeviceMode:%d, index:%u", __func__, excuteOpId_.tag, isDeviceMode_, excuteOpId_.index);
2632 1 : HCCL_RETRY_CHK_RET_AND_TRANS_FSM(
2633 : ret, HCCL_ERROR("InitOpExecStatus failed, ret:%u", ret), KfcError::kInner,
2634 : HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR);
2635 1 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_FSM_LAUNCH;
2636 1 : return ret;
2637 : }
2638 :
2639 0 : bool HcclCommAicpu::HcclOpCheckSupportRetry(HcclCMDType opType)
2640 : {
2641 : const std::set<HcclCMDType> HcclSupportRetryOpSet
2642 : = {HcclCMDType::HCCL_CMD_BROADCAST, HcclCMDType::HCCL_CMD_ALLREDUCE, HcclCMDType::HCCL_CMD_REDUCE,
2643 : HcclCMDType::HCCL_CMD_ALLGATHER, HcclCMDType::HCCL_CMD_REDUCE_SCATTER, HcclCMDType::HCCL_CMD_ALLTOALLV,
2644 : HcclCMDType::HCCL_CMD_ALLTOALLVC, HcclCMDType::HCCL_CMD_ALLTOALL, HcclCMDType::HCCL_CMD_GATHER,
2645 : HcclCMDType::HCCL_CMD_SCATTER, HcclCMDType::HCCL_CMD_SEND, HcclCMDType::HCCL_CMD_RECEIVE,
2646 0 : HcclCMDType::HCCL_CMD_BATCH_SEND_RECV};
2647 0 : return (HcclSupportRetryOpSet.find(opType) != HcclSupportRetryOpSet.end());
2648 0 : }
2649 :
2650 1 : HcclResult HcclCommAicpu::HcclOpExecFsmLaunchProcess(
2651 : const std::string& algName, OpParam& param, std::unique_ptr<CollExecutorBase>& executor,
2652 : AlgResourceResponse& algResource, HcclOpExecFSM& fsmState, KfcError& errorCode, uint32_t& beginSqePos,
2653 : uint32_t& endSqePos, uint32_t retryCnt)
2654 : {
2655 1 : HCCL_DEBUG("hccl aicpu start launch task.");
2656 :
2657 1 : HcclResult ret = OrchestrateHcclOp(algName, param, executor, algResource, beginSqePos, endSqePos);
2658 1 : if (ret == HCCL_SUCCESS) { // 下发成功, 并且没有检测到异常cq或中断命令
2659 0 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_FSM_WAIT_END;
2660 1 : } else if (ret == HCCL_E_SUSPENDING) { // 检测到异常cq或中断命令
2661 1 : if (isDeviceMode_ && retryEnable_) {
2662 1 : HCCL_RUN_INFO("Orchestrate hccl op suspending, restart handle by mc2 process.");
2663 1 : return ret;
2664 : }
2665 0 : if ((param.opType == HcclCMDType::HCCL_CMD_BATCH_SEND_RECV) && retryEnable_) {
2666 : // batchsendrecv算子下发过程中出现异常，task下发未完成，send 和 recv 均需要重执行;
2667 : // 第一个g故障op重执行下发task完成后，需要主动上报故障，触发第二个op进行重执行
2668 0 : SetBSRSendOpExecException();
2669 0 : SetBSRRecvOpExecException();
2670 0 : HCCL_RUN_INFO("hccl aicpu abort launch batchsendrecv op, need retry");
2671 : }
2672 0 : CHK_RET(UpdateSuspendStatus(param, fsmState, errorCode, retryCnt));
2673 : } else {
2674 0 : HCCL_ERROR("OrchestrateHcclOp failed, ret:%u", ret);
2675 0 : errorCode = KfcError::kInner;
2676 0 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR;
2677 : }
2678 0 : return ret;
2679 : }
2680 :
2681 4 : HcclResult HcclCommAicpu::HcclOpExecFsmWaitEndProcess(
2682 : OpParam& param, AlgResourceResponse& algResource, HcclOpExecFSM& fsmState, KfcError& errorCode, uint32_t retryCnt,
2683 : std::string& tag, const uint32_t& beginSqePos)
2684 : {
2685 4 : HCCL_DEBUG("hccl aicpu wait task finish.");
2686 4 : auto ret = WaitFinishWhileLoop(mainStream_, algResource.slaveStreams, tag, beginSqePos, param);
2687 4 : if (ret == HCCL_SUCCESS) {
2688 0 : HCCL_DEBUG("hccl aicpu exec complete.");
2689 0 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_FSM_END;
2690 4 : } else if (ret == HCCL_E_SUSPENDING) {
2691 4 : HCCL_RUN_INFO("hccl aicpu force stop in wait end, retryCnt[%u]", retryCnt);
2692 4 : CHK_RET(UpdateSuspendStatus(param, fsmState, errorCode, retryCnt));
2693 : } else {
2694 0 : HCCL_ERROR("WaitTaskFinish failed, ret:%u, identifier[%s]", ret, identifier_.c_str());
2695 0 : errorCode = KfcError::kExec;
2696 0 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR;
2697 : }
2698 4 : return ret;
2699 : }
2700 :
2701 0 : HcclResult HcclCommAicpu::HcclOpExecFsmStoppingProcess(
2702 : const OpParam& param, HcclOpExecFSM& fsmState, KfcError& errorCode, uint32_t retryCnt)
2703 : {
2704 0 : HCCL_DEBUG("hccl aicpu stopping.");
2705 0 : KfcCommand cmd = KfcCommand::kNone;
2706 0 : auto ret = aicpuHdc_.GetOpExecCtrlCmd(kfcControlTransferH2D_, cmd);
2707 0 : if (ret != HCCL_SUCCESS) {
2708 0 : HCCL_ERROR("[OpRetry][AICPU]GetOpExecCtrlCmd failed, ret:%u", ret);
2709 0 : errorCode = KfcError::kExec;
2710 0 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR;
2711 0 : return ret;
2712 : }
2713 0 : if (cmd == KfcCommand::kExit) {
2714 0 : HCCL_RUN_INFO("[OpRetry][AICPU]hccl aicpu exec fsm stop by exit cmd.");
2715 0 : errorCode = KfcError::kExit;
2716 0 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR;
2717 0 : } else if (cmd == KfcCommand::kStopExec) {
2718 0 : HCCL_RUN_INFO("[OpRetry][AICPU]hccl aicpu get stop exec cmd.");
2719 0 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_FSM_STOPPED;
2720 0 : } else if (cmd == KfcCommand::kStopLaunch) {
2721 0 : if ((param.opType == HcclCMDType::HCCL_CMD_BATCH_SEND_RECV) && retryEnable_) {
2722 0 : HcclOpIdentifier targetOp;
2723 0 : CHK_RET(aicpuHdc_.GetOpExecCtrlTargetOp(kfcControlTransferH2D_, targetOp));
2724 0 : std::string targetOpTag = std::string(reinterpret_cast<char*>(&targetOp.tag[0]));
2725 0 : if (targetOpTag != std::string(reinterpret_cast<char*>(&bsrTargetOpId_.tag[0]))) {
2726 0 : CHK_RET(UpdateSuspendStatus(param, fsmState, errorCode, retryCnt));
2727 : }
2728 0 : }
2729 0 : } else if ((cmd == KfcCommand::kNone) || (cmd == KfcCommand::kRetry)) {
2730 0 : HCCL_DEBUG("hccl aicpu wait for stop exec cmd.");
2731 : // do nothing
2732 0 : } else if (cmd == KfcCommand::kReportRetryErr) {
2733 0 : HCCL_ERROR("[OpRetry][AICPU][HcclOpExecFsmStoppingProcess]hccl aicpu can not retry err cmd[%d]", cmd);
2734 0 : uint16_t rsErrorCode = TS_ERROR_RETRY_CONSTRAINT;
2735 0 : CHK_PRT(SendTaskExceptionByMBox(rsErrorCode));
2736 0 : errorCode = KfcError::kExit;
2737 0 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR;
2738 0 : return HCCL_E_OPRETRY_FAIL;
2739 : } else {
2740 0 : HCCL_ERROR("[OpRetry][AICPU]GetOpExecCtrlCmd failed, invalid cmd[%u]", cmd);
2741 0 : errorCode = KfcError::kExec;
2742 0 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR;
2743 : }
2744 0 : return HCCL_SUCCESS;
2745 : }
2746 :
2747 0 : HcclResult HcclCommAicpu::LoadChangeLinkInfo(ChangeLinkInfo& changeLinkInfo)
2748 : {
2749 0 : HcclResult ret = aicpuHdc_.GetOpExecChangeLink(kfcControlTransferH2D_, changeLinkInfo);
2750 :
2751 : // DEBUG_INFO aicpu接收的changelinkinfo
2752 0 : std::string changeLinkInfoStr = "";
2753 0 : for (u32 i = 0; i < changeLinkInfo.remoteRankNum; i++) {
2754 : changeLinkInfoStr
2755 0 : += (std::to_string(changeLinkInfo.remoteRankList[i]) + ":"
2756 0 : + std::to_string(changeLinkInfo.isUseDefaultPort[i]) + "; ");
2757 : }
2758 0 : HCCL_RUN_INFO(
2759 : "[%s]rank[%u], isChangeLinkFlag[%d], changeLinkInfoStr:%s", __func__, localUserRank_,
2760 : changeLinkInfo.isChangeLinkFlag, changeLinkInfoStr.c_str());
2761 :
2762 0 : return ret;
2763 0 : }
2764 :
2765 0 : HcclResult HcclCommAicpu::HcclOpExecChangeLinkProcess(
2766 : const std::string& newTag, HcclOpExecFSM& state, KfcError& errorCode, uint32_t& retryCnt,
2767 : AlgResourceResponse& algResource, const HcclOpResParam* commParam, const OpParam& param)
2768 : {
2769 0 : ChangeLinkInfo changeLinkInfo;
2770 0 : HcclResult ret = LoadChangeLinkInfo(changeLinkInfo);
2771 0 : if (ret != HCCL_SUCCESS) {
2772 0 : errorCode = KfcError::kExec;
2773 0 : state = HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR;
2774 0 : return ret;
2775 : }
2776 0 : bool useBackupLink = false;
2777 0 : std::map<u32, bool> remoteRankPortMap;
2778 0 : for (u32 i = 0; i < changeLinkInfo.remoteRankNum; i++) {
2779 0 : remoteRankPortMap.insert({changeLinkInfo.remoteRankList[i], changeLinkInfo.isUseDefaultPort[i]});
2780 0 : useBackupLink |= (!changeLinkInfo.isUseDefaultPort[i]);
2781 : }
2782 0 : ret = RefreshAlgResponseTransportRes(
2783 0 : newTag, algResource, remoteRankPortMap, changeLinkInfo.isChangeLinkFlag, commParam, param);
2784 0 : if (ret != HCCL_SUCCESS) {
2785 0 : errorCode = KfcError::kExec;
2786 0 : state = HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR;
2787 0 : return ret;
2788 : }
2789 0 : if (useBackupLink) {
2790 0 : RecordReportStatus(dfx::ReportStatus::kRetryWithBackupLink);
2791 : }
2792 0 : if ((param.opType == HcclCMDType::HCCL_CMD_BATCH_SEND_RECV) && retryEnable_) {
2793 0 : retryCnt = (bsrRetryOp_ == HCCL_SEND) ? bsrSendRetryCnt_ : bsrRecvRetryCnt_;
2794 : }
2795 0 : errorCode = KfcError::kNone;
2796 0 : CHK_RET(UpdateOpExecStatus(state, KfcStatus::kChanged, errorCode, retryCnt));
2797 0 : state = HcclOpExecFSM::HCCL_OP_EXEC_FSM_WAIT_RETRY;
2798 0 : return HCCL_SUCCESS;
2799 0 : }
2800 :
2801 0 : HcclResult HcclCommAicpu::BSRStopedProcess(HcclOpExecFSM& fsmState, KfcError& errorCode)
2802 : {
2803 : // 判断batchsendrecv算子的send 和recv 操作停止的位置是否满足重执行条件
2804 : // send / recv 的stream停止位置不能位于该算子的首个sqe 和末尾sqe
2805 : u32 bsrSendSqHead;
2806 : u32 bsrRecvSqHead;
2807 0 : auto ret = QuerySqStatusByType(devId_, bsrSendStream_.sqId(), DRV_SQCQ_PROP_SQ_HEAD, bsrSendSqHead);
2808 0 : HCCL_RETRY_CHK_RET_AND_TRANS_FSM(
2809 : ret, HCCL_ERROR("[OpRetry][AICPU]quert send stream sq head failed, ret:%u", ret), KfcError::kExec,
2810 : HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR);
2811 :
2812 0 : ret = QuerySqStatusByType(devId_, bsrRecvStream_.sqId(), DRV_SQCQ_PROP_SQ_HEAD, bsrRecvSqHead);
2813 0 : HCCL_RETRY_CHK_RET_AND_TRANS_FSM(
2814 : ret, HCCL_ERROR("[OpRetry][AICPU]quert recv stream sq head failed, ret:%u", ret), KfcError::kExec,
2815 : HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR);
2816 :
2817 0 : ret = ((bsrSendOpBeginSqePos_ == bsrSendSqHead) || (bsrRecvOpBeginSqePos_ == bsrRecvSqHead)) ? HCCL_E_OPRETRY_FAIL :
2818 : HCCL_SUCCESS;
2819 0 : if (ret != HCCL_SUCCESS) {
2820 0 : uint16_t rsErrorCode = TS_ERROR_RETRY_CONSTRAINT;
2821 0 : CHK_PRT(SendTaskExceptionByMBox(rsErrorCode));
2822 : }
2823 0 : HCCL_RETRY_CHK_RET_AND_TRANS_FSM(
2824 : ret,
2825 : HCCL_ERROR(
2826 : "[OpRetry][AICPU]hccl aicpu wait start task is not complete, can not retry. params: send sq head "
2827 : "%u, recv sq head %u, send sq begin %u, recv sq begin %u",
2828 : bsrSendSqHead, bsrRecvSqHead, bsrSendOpBeginSqePos_, bsrRecvOpBeginSqePos_),
2829 : KfcError::kExecConstraint, HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR);
2830 :
2831 0 : ret = ((bsrRetryOp_ == HCCL_SEND) && (bsrSendSqHead == bsrSendOpEndSqePos_)) ? HCCL_E_OPRETRY_FAIL : HCCL_SUCCESS;
2832 0 : HCCL_RETRY_CHK_RET_AND_TRANS_FSM(
2833 : ret,
2834 : HCCL_ERROR(
2835 : "[OpRetry][AICPU]hccl aicpu send record complete task is complete, can not retry. params: send sq "
2836 : "head %u, recv sq head %u, send sq begin %u, send sq end %u, recv sq begin %u, recv sq end %u",
2837 : bsrSendSqHead, bsrRecvSqHead, bsrSendOpBeginSqePos_, bsrSendOpEndSqePos_, bsrRecvOpBeginSqePos_,
2838 : bsrRecvOpEndSqePos_),
2839 : KfcError::kExecConstraint, HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR);
2840 :
2841 0 : ret = ((bsrRetryOp_ == HCCL_RECV) && (bsrRecvSqHead == bsrRecvOpEndSqePos_)) ? HCCL_E_OPRETRY_FAIL : HCCL_SUCCESS;
2842 0 : HCCL_RETRY_CHK_RET_AND_TRANS_FSM(
2843 : ret,
2844 : HCCL_ERROR(
2845 : "[OpRetry][AICPU]hccl aicpu recv record complete task is complete, can not retry. params: send sq "
2846 : "head %u, recv sq head %u, send sq begin %u, send sq end %u, recv sq begin %u, recv sq end %u",
2847 : bsrSendSqHead, bsrRecvSqHead, bsrSendOpBeginSqePos_, bsrSendOpEndSqePos_, bsrRecvOpBeginSqePos_,
2848 : bsrRecvOpEndSqePos_),
2849 : KfcError::kExecConstraint, HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR);
2850 :
2851 0 : HCCL_RUN_INFO(
2852 : "[OpRetry][AICPU]hccl aicpu op is running, can retry. params: send sq head "
2853 : "%u, recv sq head %u, send sq begin %u, send sq end %u, recv sq begin %u, recv sq end %u",
2854 : bsrSendSqHead, bsrRecvSqHead, bsrSendOpBeginSqePos_, bsrSendOpEndSqePos_, bsrRecvOpBeginSqePos_,
2855 : bsrRecvOpEndSqePos_);
2856 0 : if (IsTaskExceptionForHccs()) {
2857 0 : HCCL_RUN_INFO("[OpRetry][AICPU]hccl aicpu stop by sdma/write task exception, can retry.");
2858 : }
2859 0 : errorCode = KfcError::kNone;
2860 0 : uint32_t retryCnt = (bsrRetryOp_ == HCCL_SEND) ? bsrSendRetryCnt_ : bsrRecvRetryCnt_;
2861 0 : CHK_RET(UpdateOpExecStatus(fsmState, KfcStatus::kStopExec, errorCode, retryCnt));
2862 0 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_FSM_WAIT_RETRY;
2863 0 : return HCCL_SUCCESS;
2864 : }
2865 :
2866 0 : HcclResult HcclCommAicpu::HcclOpExecFsmStoppedProcess(
2867 : HcclOpExecFSM& fsmState, KfcError& errorCode, uint32_t retryCnt, const std::string& algName, OpParam& param,
2868 : uint32_t beginSqePos, uint32_t endSqePos)
2869 : {
2870 0 : HCCL_DEBUG("hccl aicpu stop exec.");
2871 0 : KfcCommand cmd = KfcCommand::kNone;
2872 0 : auto ret = aicpuHdc_.GetOpExecCtrlCmd(kfcControlTransferH2D_, cmd);
2873 0 : uint16_t rsErrorCode = 0;
2874 0 : if (ret != HCCL_SUCCESS) {
2875 0 : HCCL_ERROR("[OpRetry][AICPU]GetOpExecCtrlCmd failed, ret:%u", ret);
2876 0 : errorCode = KfcError::kExec;
2877 0 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR;
2878 0 : return ret;
2879 : }
2880 :
2881 0 : if (cmd == KfcCommand::kExit) {
2882 0 : HCCL_ERROR("[OpRetry][AICPU]hccl aicpu exec fsm stop by exit cmd.");
2883 0 : errorCode = KfcError::kExit;
2884 0 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR;
2885 0 : return HCCL_SUCCESS;
2886 : }
2887 :
2888 0 : if (cmd == KfcCommand::kReportRetryErr) {
2889 0 : HCCL_ERROR("[OpRetry][AICPU][HcclOpExecFsmStoppedProcess]hccl aicpu can not retry err cmd[%d]", cmd);
2890 0 : rsErrorCode = TS_ERROR_RETRY_CONSTRAINT;
2891 0 : CHK_PRT(SendTaskExceptionByMBox(rsErrorCode));
2892 0 : errorCode = KfcError::kExit;
2893 0 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR;
2894 0 : return HCCL_E_OPRETRY_FAIL;
2895 : }
2896 :
2897 0 : if (!HcclOpSupportRetry(algName, retryEnable_, param)) {
2898 0 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR;
2899 0 : if (param.isInplaceError) {
2900 0 : errorCode = KfcError::kExecConstraint;
2901 0 : rsErrorCode = TS_ERROR_RETRY_CONSTRAINT;
2902 0 : CHK_PRT(SendTaskExceptionByMBox(rsErrorCode));
2903 0 : HCCL_RETRY_CHK_RET_AND_TRANS_FSM(
2904 : HCCL_E_OPRETRY_FAIL,
2905 : HCCL_ERROR("[Opretry][AICPU][HcclOpExecFsmStoppedProcess]can not retry for inpace error."),
2906 : KfcError::kExecConstraint, HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR);
2907 0 : } else if (isPollutedZeroCopyOp(param)) {
2908 0 : errorCode = KfcError::kExecConstraint;
2909 0 : rsErrorCode = TS_ERROR_RETRY_CONSTRAINT;
2910 0 : CHK_PRT(SendTaskExceptionByMBox(rsErrorCode));
2911 0 : HCCL_RETRY_CHK_RET_AND_TRANS_FSM(
2912 : HCCL_E_OPRETRY_FAIL,
2913 : HCCL_ERROR("[Opretry][AICPU][HcclOpExecFsmStoppedProcess]can not retry for zero copy op."),
2914 : KfcError::kExecConstraint, HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR);
2915 : return HCCL_E_OPRETRY_FAIL;
2916 : } else {
2917 0 : errorCode = KfcError::kExec;
2918 : }
2919 0 : return HCCL_SUCCESS;
2920 : }
2921 :
2922 0 : uint32_t sqHead = 0xFFFFFFFF;
2923 0 : CHK_RET(QuerySqStatusByType(devId_, mainStream_.sqId(), DRV_SQCQ_PROP_SQ_HEAD, sqHead));
2924 0 : if (sqHead == endSqePos) {
2925 0 : rsErrorCode = TS_ERROR_RETRY_CONSTRAINT;
2926 0 : CHK_PRT(SendTaskExceptionByMBox(rsErrorCode));
2927 0 : HCCL_RETRY_CHK_RET_AND_TRANS_FSM(
2928 : HCCL_E_OPRETRY_FAIL,
2929 : HCCL_ERROR(
2930 : "[OpRetry][AICPU]hccl aicpu record complete task is complete, can not retry. params: "
2931 : "sqHead %u, beginSqePos %u endSqePos %u",
2932 : sqHead, beginSqePos, endSqePos),
2933 : KfcError::kExecConstraint, HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR);
2934 0 : } else if (sqHead == beginSqePos) {
2935 0 : rsErrorCode = TS_ERROR_RETRY_CONSTRAINT;
2936 0 : CHK_PRT(SendTaskExceptionByMBox(rsErrorCode));
2937 0 : HCCL_RETRY_CHK_RET_AND_TRANS_FSM(
2938 : HCCL_E_OPRETRY_FAIL,
2939 : HCCL_ERROR(
2940 : "[OpRetry][AICPU]hccl aicpu wait start task is not complete, can not retry. "
2941 : "params: sqHead %u, beginSqePos %u endSqePos %u",
2942 : sqHead, beginSqePos, endSqePos),
2943 : KfcError::kExecConstraint, HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR);
2944 0 : } else if ((param.opType == HcclCMDType::HCCL_CMD_BATCH_SEND_RECV) && retryEnable_) {
2945 0 : CHK_RET(BSRStopedProcess(fsmState, errorCode));
2946 0 : } else {
2947 0 : HCCL_RUN_INFO(
2948 : "[OpRetry][AICPU]hccl aicpu op is running, can retry. params: sqHead %u, beginSqePos %u "
2949 : "endSqePos %u",
2950 : sqHead, beginSqePos, endSqePos);
2951 0 : if (IsTaskExceptionForHccs()) {
2952 0 : HCCL_RUN_INFO("[OpRetry][AICPU]hccl aicpu stop by sdma/write task exception, can retry.");
2953 : }
2954 0 : errorCode = KfcError::kNone;
2955 0 : CHK_RET(UpdateOpExecStatus(fsmState, KfcStatus::kStopExec, errorCode, retryCnt));
2956 0 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_FSM_WAIT_RETRY;
2957 : }
2958 0 : return HCCL_SUCCESS;
2959 : }
2960 :
2961 0 : void HcclCommAicpu::NsCommStop()
2962 : {
2963 0 : if ((StreamsKill(devId_) != HCCL_SUCCESS) || (DeviceQuery(devId_, ts::APP_ABORT_KILL_FINISH, 0U) != HCCL_SUCCESS)) {
2964 0 : (void)aicpuHdc_.SetOpExecStatus(kfcStatusTransferD2H_, KfcStatus::kError, KfcError::kExec, 0);
2965 0 : HCCL_ERROR("[NsRecovery][AICPU]Stop failed");
2966 0 : return;
2967 : }
2968 : // 停止条件算子
2969 0 : if (isDeviceMode_) {
2970 0 : (void)InvokeKfcHandler(AicpuKfcHandlerType::kClearCommitTurn, {rpc_});
2971 : }
2972 0 : (void)aicpuHdc_.SetOpExecStatus(kfcStatusTransferD2H_, KfcStatus::kStopExec, KfcError::kNone, 0);
2973 0 : (void)HcclOneSideServiceAicpu::DisableAllStreamFunc();
2974 0 : HCCL_RUN_INFO("[NsRecovery][AICPU]stopFunc Finished");
2975 : }
2976 :
2977 0 : void HcclCommAicpu::NsCommClean()
2978 : {
2979 : // 等待drv任务停止
2980 0 : if ((DeviceQuery(devId_, ts::APP_ABORT_TERMINATE_FINISH, 0U) != HCCL_SUCCESS) || (CleanStreamFunc() != HCCL_SUCCESS)
2981 0 : || (HcclOneSideServiceAicpu::CleanAllStreamFunc() != HCCL_SUCCESS) || (ResetSqBuff() != HCCL_SUCCESS)) {
2982 0 : (void)aicpuHdc_.SetOpExecStatus(kfcStatusTransferD2H_, KfcStatus::kError, KfcError::kExec, 0);
2983 0 : HCCL_ERROR("[NsRecovery][AICPU]stream terminate failed");
2984 0 : return;
2985 : } else {
2986 0 : if (isDeviceMode_) {
2987 0 : (void)InvokeKfcHandler(AicpuKfcHandlerType::kClearMsgArea, {rpc_});
2988 : }
2989 0 : HCCL_INFO("ClearFunc, after APP_ABORT_TERMINATE_FINISH");
2990 0 : dfxExtendInfo_.pollStatus = PollStatus::kDefault;
2991 0 : dfxExtendInfo_.cqeStatus = dfx::CqeStatus::kDefault;
2992 0 : (void)aicpuHdc_.SetOpExecStatus(kfcStatusTransferD2H_, KfcStatus::kClear, KfcError::kNone, 0);
2993 0 : endStopLaunch = false;
2994 0 : isOpLaunch = false;
2995 0 : needsResponseStopLaunch_ = false;
2996 0 : errMessageReport_ = true;
2997 0 : HCCL_RUN_INFO("[NsRecovery][AICPU] clean Finish");
2998 : }
2999 : }
3000 :
3001 0 : HcclResult HcclCommAicpu::GetBackGroundCommand(BackgroundCommand& bgCmd)
3002 : {
3003 0 : return AicpuHdcUtils::GetBackGroundCommand(kfcControlTransferH2D_, bgCmd);
3004 : }
3005 :
3006 0 : HcclResult HcclCommAicpu::ResponseBackGroundStatus(KfcExecStatus& status)
3007 : {
3008 0 : return AicpuHdcUtils::ResponseBackGroundStatus(kfcStatusTransferD2H_, status);
3009 : }
3010 :
3011 0 : HcclResult HcclCommAicpu::GetKfcCommand(KfcCommand& cmd)
3012 : {
3013 0 : return AicpuHdcUtils::GetKfcCommand(kfcControlTransferH2D_, cmd);
3014 : }
3015 :
3016 0 : HcclResult HcclCommAicpu::SetStreamEnable(Stream& stream)
3017 : {
3018 0 : const HcclComStreamInfo& streamInfo = stream.GetHcclStreamInfo();
3019 0 : HCCL_INFO("[SetStreamEnable] streamid[%d]", streamInfo.actualStreamId);
3020 0 : CHK_RET(ConfigSqStatusByType(GetDevId(), streamInfo.sqId, DRV_SQCQ_PROP_SQ_DISABLE_TO_ENABLE, 1));
3021 0 : HandleCqeException(stream, true);
3022 0 : return HCCL_SUCCESS;
3023 : }
3024 :
3025 0 : HcclResult HcclCommAicpu::CleanStreamFunc()
3026 : {
3027 0 : CHK_RET(SetStreamEnable(mainStream_));
3028 0 : for (auto& stream : slaveStreams_) {
3029 0 : CHK_RET(SetStreamEnable(stream));
3030 : }
3031 0 : CHK_RET(SetStreamEnable(orderStream_));
3032 0 : return HCCL_SUCCESS;
3033 : }
3034 :
3035 0 : std::string HcclCommAicpu::PrintInplaceStatus(u8 isInplaceStatus)
3036 : {
3037 0 : const u8 kNoOverlap = 0;
3038 0 : const u8 kAllToAllOverlap = 1;
3039 0 : const u8 kInplaceOverlap = 2;
3040 0 : switch (isInplaceStatus) {
3041 0 : case kNoOverlap:
3042 : // input和output不重叠
3043 0 : return "There is no overlap.";
3044 0 : case kAllToAllOverlap:
3045 : // alltoall类算子的input和output重叠
3046 0 : return "The param.inputPtr is equal to param.outputPtr, hence they overlap.";
3047 0 : case kInplaceOverlap:
3048 : // input和output重叠
3049 0 : return "It's inplace case. hence they overlap.";
3050 0 : default:
3051 0 : return "It's an unknown overlap case.";
3052 : }
3053 : return "";
3054 : }
3055 :
3056 0 : std::string HcclCommAicpu::PrintInplaceSupportRetryStatus(InplaceSupportRetryStatus inPlaceSupportRetryStatus)
3057 : {
3058 0 : switch (inPlaceSupportRetryStatus) {
3059 0 : case InplaceSupportRetryStatus::AG_BD_CASE: // 不需要去变成非DMA削减
3060 : // allgather or broadcast 算子
3061 0 : return "The Allgather or broadcast op supports inplace retry.";
3062 0 : case InplaceSupportRetryStatus::RETRY_1_ALLOW_NO_DMA_REDUCE_CASE1: // 需要去变成非DMA削减
3063 : // 使用AllReduceMeshSmallCountExecutor, ReduceScatterDeterExecutor
3064 : // 且环境变量配置RetryEnable:1
3065 0 : return "Since retryEnable:1, the executor without DMAReduce will be applied.";
3066 0 : case InplaceSupportRetryStatus::RETRY_0_NOT_ALLOW_NO_DMA_REDUCE_CASE1: // 不需要去变成非DMA削减
3067 : // 使用AllReduceMeshSmallCountExecutor, ReduceScatterDeterExecutor
3068 : // 且环境变量配置RetryEnable:0
3069 0 : return "Since retryEnable:0, ExecutorOnlySupportDMAReduce is not allowed for inplace case.";
3070 0 : case InplaceSupportRetryStatus::ALWAYS_NO_DMA_REDUCE: // 不需要去变成非DMA削减，本身就是
3071 : // 使用AllReduceComm/ReduceScatterComm
3072 0 : return "AllReduceComm or ReduceScatterComm is used for inplace case.";
3073 0 : case InplaceSupportRetryStatus::RETRY_1_ALLOW_NO_DMA_REDUCE_CASE2: // 需要去变成非DMA削减
3074 : // 使用其余在91093场景下使用的reduce scatter, allreduce executor
3075 : // 且环境变量配置RetryEnable:1
3076 0 : return "Since retryEnable:1, the executor will be applied without DMAReduce operation.";
3077 0 : case InplaceSupportRetryStatus::RETRY_0_NOT_ALLOW_NO_DMA_REDUCE_CASE2: // 不需要去变成非DMA削减
3078 : // 使用其余在91093场景下使用的reduce scatter, allreduce executor
3079 : // 且环境变量配置RetryEnable:0
3080 0 : return "Since retryEnable:0, the executor without DMAReduce operation can not be applied.";
3081 0 : case InplaceSupportRetryStatus::UNKONWN_EXECUTOR: // 不需要去变成非DMA削减
3082 : // 使用未知的executor
3083 0 : return "The unknown executor does not support for an inplace case yet.";
3084 0 : case InplaceSupportRetryStatus::USER_LARGER_THAN_CCL: // 不需要去变成非DMA削减
3085 : // UserInMem > CCLInMem 场景
3086 0 : return "UserInMem > CCLInMem case";
3087 0 : case InplaceSupportRetryStatus::NOT_BASIC_OP_CASE: // 不需要去变成非DMA削减
3088 : // 非 RS AR AG BD算子场景
3089 0 : return "Is not ReduceScatter, AllReduce, AllGather or Broadcast case";
3090 0 : default:
3091 0 : return "It's unknown case. They overlap.";
3092 : }
3093 : return "";
3094 : }
3095 :
3096 0 : bool HcclCommAicpu::IsNoNeedMonitor(void)
3097 : {
3098 0 : if (taskMonitorInterval_ == 0)
3099 0 : return true;
3100 :
3101 0 : KfcCommand kfcCmd = KfcCommand::kNone;
3102 0 : (void)GetKfcCommand(kfcCmd);
3103 0 : if (kfcCmd != KfcCommand::kNone)
3104 0 : return true;
3105 :
3106 0 : BackgroundCommand bgCmd = BackgroundCommand::kNone;
3107 0 : (void)GetBackGroundCommand(bgCmd);
3108 0 : if (bgCmd != BackgroundCommand::kNone)
3109 0 : return true;
3110 :
3111 0 : HcclComSuspendingFlag suspendingFlag = HcclComSuspendingFlag::isNull;
3112 0 : (void)GetSuspendingFlag(suspendingFlag);
3113 0 : if (suspendingFlag != HcclComSuspendingFlag::isNull)
3114 0 : return true;
3115 0 : return false;
3116 : }
3117 :
3118 0 : void HcclCommAicpu::InsertMonitorData(Stream& stream, HcclUs& curTime, u32 sqHead, uint16_t taskId, uint8_t type)
3119 : {
3120 0 : AicpuStreamMontior tmpTaskMonitor;
3121 0 : tmpTaskMonitor.historyTime = curTime;
3122 0 : tmpTaskMonitor.historyHead = sqHead;
3123 0 : tmpTaskMonitor.historyTaskId = taskId;
3124 0 : tmpTaskMonitor.historyType = type;
3125 0 : streamTaskMonitor_.insert(std::make_pair(stream.sqId(), tmpTaskMonitor));
3126 0 : return;
3127 : }
3128 :
3129 0 : bool HcclCommAicpu::IsNeedRefreshMonitorData(
3130 : AicpuStreamMontior& streamMontior, HcclUs& curTime, uint32_t remoteRank, uint16_t taskId, u32 sqHead, u32 sqTail,
3131 : uint8_t type)
3132 : {
3133 0 : auto& historyTime = streamMontior.historyTime;
3134 0 : auto& historyHead = streamMontior.historyHead;
3135 0 : auto& historyTaskId = streamMontior.historyTaskId;
3136 0 : auto& historyType = streamMontior.historyType;
3137 0 : if ((historyTaskId != taskId) || (sqHead != historyHead) || (sqHead == sqTail) || (historyType != type)
3138 0 : || ((type == RT_STARS_SQE_TYPE_NOTIFY_WAIT) && (remoteRank == INVALID_VALUE_RANKID))) {
3139 0 : historyTime = curTime;
3140 0 : historyHead = sqHead;
3141 0 : historyTaskId = taskId;
3142 0 : historyType = type;
3143 0 : return true;
3144 : }
3145 0 : return false;
3146 : }
3147 :
3148 0 : HcclResult HcclCommAicpu::StreamTaskMonitor(void)
3149 : {
3150 : // 通信域资源已经释放
3151 0 : CHK_PRT_RET(
3152 : !commOpenStatus, HCCL_DEBUG("[StreamTaskMonitor]group[%s] has been destroyed", identifier_.c_str()),
3153 : HCCL_SUCCESS);
3154 0 : if (IsNoNeedMonitor())
3155 0 : return HCCL_SUCCESS;
3156 0 : HCCL_DEBUG("StreamTaskMonitor print");
3157 0 : std::vector<Stream> totalStream = {mainStream_};
3158 0 : totalStream.insert(totalStream.end(), slaveStreams_.begin(), slaveStreams_.end());
3159 0 : HcclUs curTime = TIME_NOW();
3160 0 : for (auto& stream : totalStream) {
3161 0 : u32 sqHead = 0U, sqTail = 0U;
3162 0 : (void)QuerySqStatus(devId_, stream.sqId(), sqHead, sqTail);
3163 0 : HcclSqeContext* sqeContext = stream.GetSqeContextPtr();
3164 0 : CHK_PTR_NULL(sqeContext);
3165 0 : SqeRingBuffer* sqeContextBuffer = &(sqeContext->buffer);
3166 0 : CHK_PTR_NULL(sqeContextBuffer);
3167 :
3168 0 : uint8_t type = 0;
3169 0 : uint16_t taskId = 0;
3170 0 : uint32_t remoteRank = 0;
3171 0 : std::string tmp = GetTaskExceptionTaskInfo(sqHead, sqeContextBuffer, type, taskId, remoteRank);
3172 0 : HCCL_DEBUG("GetTaskExceptionTaskInfo type %u taskId %u", type, taskId);
3173 0 : auto mapIt = streamTaskMonitor_.find(stream.sqId());
3174 0 : if (mapIt == streamTaskMonitor_.end()) {
3175 0 : InsertMonitorData(stream, curTime, sqHead, taskId, type);
3176 0 : continue;
3177 : }
3178 :
3179 0 : auto& streamMontior = mapIt->second;
3180 0 : if (IsNeedRefreshMonitorData(streamMontior, curTime, remoteRank, taskId, sqHead, sqTail, type)) {
3181 0 : continue;
3182 : }
3183 :
3184 0 : auto timeVal = DURATION_US(curTime - streamMontior.historyTime).count();
3185 0 : const int TIME_CONVERSION = 1000;
3186 0 : if (timeVal >= taskMonitorInterval_ * TIME_CONVERSION) {
3187 0 : HCCL_RUN_INFO(
3188 : "[StreamTaskMonitor]prof monitor streamId:%d, sqid:%d, head:%u, tail:%u, time %s us, %s", stream.id(),
3189 : stream.sqId(), sqHead, sqTail, std::to_string(timeVal).c_str(), tmp.c_str());
3190 0 : HCCL_RUN_INFO(
3191 : "[StreamTaskMonitor]prof monitor %s", GetTaskExceptionOpInfo(sqHead, sqeContextBuffer).c_str());
3192 0 : PrintTaskExceptionTaskQue(sqHead, sqeContextBuffer, true);
3193 0 : streamMontior.historyTime = curTime;
3194 0 : streamMontior.historyHead = sqHead;
3195 0 : streamMontior.historyTaskId = taskId;
3196 0 : streamMontior.historyType = type;
3197 : }
3198 0 : }
3199 0 : return HCCL_SUCCESS;
3200 0 : }
3201 :
3202 0 : HcclResult HcclCommAicpu::SupportRetryWithInplaceCheck(const std::string& algName, OpParam& param)
3203 : {
3204 : // 不支持inplace的通信算子重执行
3205 0 : u8 isInplaceStatus = 0;
3206 0 : InplaceSupportRetryStatus inPlaceSupportRetryStatus = InplaceSupportRetryStatus::INPLACE_STATUS_END;
3207 0 : if (IsHcclOpInplace(param.opType, param, topoInfo_.userRank, topoInfo_.userRankSize, isInplaceStatus)) {
3208 0 : if (!FitRetryConditionforInPlaceOp(
3209 0 : param.opType, param, algName, cclbufferSize_, topoInfo_.userRankSize,
3210 0 : algOpContext_.opRetryHandler.retryEnable, inPlaceSupportRetryStatus)) {
3211 0 : HCCL_RUN_INFO(
3212 : "[OpRetry][AICPU]hccl supports inplace status: isInplaceStatus[%s], "
3213 : "opRetryHandler.inplaceSupportRetry[%d], opRetryHandler.inPlaceSupportRetryStatus[%s], "
3214 : "opRetryHandler.isInplacePreSync[%d], opRetryHandler.isPostSync[%d].",
3215 : PrintInplaceStatus(isInplaceStatus).c_str(), 0,
3216 : PrintInplaceSupportRetryStatus(inPlaceSupportRetryStatus).c_str(),
3217 : algOpContext_.opRetryHandler.isInplacePreSync, algOpContext_.opRetryHandler.isPostSync);
3218 : } else {
3219 0 : HCCL_RUN_INFO(
3220 : "[OpRetry][AICPU]hccl supports inplace status: isInplaceStatus[%s], "
3221 : "opRetryHandler.inplaceSupportRetry[%d], opRetryHandler.inPlaceSupportRetryStatus[%s], "
3222 : "opRetryHandler.isInplacePreSync[%d], opRetryHandler.isPostSync[%d].",
3223 : PrintInplaceStatus(isInplaceStatus).c_str(), 1,
3224 : PrintInplaceSupportRetryStatus(inPlaceSupportRetryStatus).c_str(),
3225 : algOpContext_.opRetryHandler.isInplacePreSync, algOpContext_.opRetryHandler.isPostSync);
3226 : }
3227 : } else {
3228 0 : HCCL_RUN_INFO(
3229 : "[OpRetry][AICPU]hccl supports inplace status: isInplaceStatus[%s], "
3230 : "opRetryHandler.isInplacePreSync[%d], opRetryHandler.isPostSync[%d].",
3231 : PrintInplaceStatus(isInplaceStatus).c_str(), algOpContext_.opRetryHandler.isInplacePreSync,
3232 : algOpContext_.opRetryHandler.isPostSync);
3233 : }
3234 0 : return HCCL_SUCCESS;
3235 : }
3236 :
3237 0 : bool HcclCommAicpu::HcclOpSupportRetry(const std::string& algName, bool retryEnable, OpParam& param)
3238 : {
3239 0 : HCCL_RUN_INFO(
3240 : "[OpRetry][AICPU]hccl supports retry status: enable[%u], param.tag[%s].", retryEnable, param.tag.c_str());
3241 0 : if (!retryEnable) {
3242 0 : HCCL_ERROR("[OpRetry][AICPU]hccl aicpu can not retry, enable[%u].", retryEnable);
3243 0 : return false;
3244 : }
3245 0 : if (isPollutedZeroCopyOp(param)) {
3246 0 : HCCL_ERROR(
3247 : "[OpRetry][AICPU]hccl aicpu can not retry, isZeroCopy[%d], opType[%s].", param.isZeroCopy,
3248 : GetCMDTypeEnumStr(param.opType).c_str());
3249 0 : return false;
3250 : }
3251 :
3252 0 : CHK_RET(SupportRetryWithInplaceCheck(algName, param));
3253 : // 不支持inplace的通信算子重执行
3254 0 : if ((!algOpContext_.opRetryHandler.inplaceSupportRetry) && (!algOpContext_.opRetryHandler.isInplacePreSync)
3255 0 : && (!algOpContext_.opRetryHandler.isPostSync)) {
3256 0 : HCCL_ERROR(
3257 : "[OpRetry][AICPU]hccl aicpu can not retry, not support inplace case, opType[%s], "
3258 : "inputPtr[0x%016lx], outputPtr[0x%016lx], opRetryHandler.inplaceSupportRetry[%d], "
3259 : "opRetryHandler.isInplacePreSync[%d], opRetryHandler.isPostSync[%d]",
3260 : GetCMDTypeEnumStr(param.opType).c_str(), param.inputPtr, param.outputPtr,
3261 : algOpContext_.opRetryHandler.inplaceSupportRetry, algOpContext_.opRetryHandler.isInplacePreSync,
3262 : algOpContext_.opRetryHandler.isPostSync);
3263 0 : param.isInplaceError = true;
3264 0 : return false;
3265 : }
3266 :
3267 : // 不支持的通信算子重执行
3268 0 : if (HcclOpCheckSupportRetry(param.opType) == false) {
3269 0 : HCCL_ERROR(
3270 : "[OpRetry][AICPU]hccl aicpu can not retry, not support opType[%s].",
3271 : GetCMDTypeEnumStr(param.opType).c_str());
3272 0 : return false;
3273 : }
3274 0 : return true;
3275 : }
3276 :
3277 0 : bool HcclCommAicpu::isPollutedZeroCopyOp(OpParam& param)
3278 : {
3279 : // allreduce\reduce\reducescatter\reducescatterv with zerocopy can not support retry.
3280 0 : bool isPollutedOp
3281 0 : = ((param.opType == HcclCMDType::HCCL_CMD_ALLREDUCE) || (param.opType == HcclCMDType::HCCL_CMD_REDUCE)
3282 0 : || (param.opType == HcclCMDType::HCCL_CMD_REDUCE_SCATTER)
3283 0 : || (param.opType == HcclCMDType::HCCL_CMD_REDUCE_SCATTER_V));
3284 0 : return param.isZeroCopy && isPollutedOp;
3285 : }
3286 :
3287 0 : HcclResult HcclCommAicpu::UpdateOpExecStatus(
3288 : HcclOpExecFSM& fsmState, HcclOpIdentifier& opId, KfcStatus state, KfcError& errorCode, uint32_t retryCnt)
3289 : {
3290 0 : HCCL_INFO(
3291 : "UpdateOpExecStatus fsmState %d, tag %s, index %u, state %d, errorCode %d, retryCnt %u.", fsmState, opId.tag,
3292 : opId.index, state, errorCode, retryCnt);
3293 0 : auto ret = aicpuHdc_.SetOpExecStatus(kfcStatusTransferD2H_, opId, state, errorCode, retryCnt);
3294 :
3295 0 : HCCL_RETRY_CHK_RET_AND_TRANS_FSM(
3296 : ret, HCCL_ERROR("SetOpExecStatus failed, ret:%u", ret), KfcError::kExec, HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR);
3297 :
3298 0 : return ret;
3299 : }
3300 :
3301 : HcclResult
3302 7 : HcclCommAicpu::UpdateOpExecStatus(HcclOpExecFSM& fsmState, KfcStatus state, KfcError& errorCode, uint32_t retryCnt)
3303 : {
3304 7 : HCCL_INFO(
3305 : "UpdateOpExecStatus fsmState %d, state %d, errorCode %d, retryCnt %u.", fsmState, state, errorCode, retryCnt);
3306 7 : auto ret = aicpuHdc_.SetOpExecStatus(kfcStatusTransferD2H_, state, errorCode, retryCnt);
3307 7 : if (ret != HCCL_SUCCESS) {
3308 0 : HCCL_ERROR("SetOpExecStatus failed, ret:%u", ret);
3309 0 : errorCode = KfcError::kExec;
3310 0 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR;
3311 : }
3312 7 : return ret;
3313 : }
3314 :
3315 : static constexpr u32 HCCL_AICPU_WAIT_HOST_BASE_TIME_MS = 200 * 1000;
3316 : static constexpr u32 TIME_S_TO_MS = 1000;
3317 14 : u32 HcclCommAicpu::HcclGetWaitStopExecCmdTimeout()
3318 : {
3319 14 : return std::max(static_cast<u32>(linkTimeOut_.count()), HCCL_AICPU_WAIT_HOST_BASE_TIME_MS);
3320 : }
3321 :
3322 7 : u32 HcclCommAicpu::HcclGetWaitRetryCmdTimeout(uint32_t retryCnt)
3323 : {
3324 7 : if (retryCnt == 0) {
3325 7 : return HcclGetWaitStopExecCmdTimeout() + retryHoldTime_;
3326 : } else {
3327 0 : return HcclGetWaitStopExecCmdTimeout() + retryIntervalTime_;
3328 : }
3329 : }
3330 :
3331 0 : HcclResult HcclCommAicpu::HcclOpExecFsmWaitRetryProcess(
3332 : const OpParam& param, HcclOpExecFSM& fsmState, KfcError& errorCode, KfcCommand& lastCmd)
3333 : {
3334 0 : HCCL_DEBUG("hccl aicpu wait for retry cmd.");
3335 0 : KfcCommand cmd = KfcCommand::kNone;
3336 0 : auto ret = aicpuHdc_.GetOpExecCtrlCmd(kfcControlTransferH2D_, cmd);
3337 0 : if (ret != HCCL_SUCCESS) {
3338 0 : HCCL_ERROR("GetOpExecCtrlCmd failed, ret:%u", ret);
3339 0 : errorCode = KfcError::kExec;
3340 0 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR;
3341 0 : return ret;
3342 : }
3343 0 : if (cmd == KfcCommand::kRetry) {
3344 0 : HCCL_RUN_INFO("[OpRetry][AICPU]hccl aicpu recv retry cmd from host.");
3345 0 : dfxExtendInfo_.pollStatus = PollStatus::kDefault;
3346 0 : dfxExtendInfo_.cqeStatus = dfx::CqeStatus::kDefault;
3347 0 : ret = ResetOpRetryException(param.opType);
3348 0 : HCCL_RETRY_CHK_RET_AND_TRANS_FSM(
3349 : ret, HCCL_ERROR("reset stream buff failed, ret:%u", ret), KfcError::kInner,
3350 : HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR);
3351 0 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_FSM_RETRY;
3352 0 : } else if (cmd == KfcCommand::kChangeLink && lastCmd != KfcCommand::kChangeLink) { // 防止重复执行
3353 0 : HCCL_RUN_INFO("[OpRetry][AICPU]hccl aicpu recv change link cmd, identify[%s]", identifier_.c_str());
3354 0 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_FSM_CHANGE_LINK;
3355 0 : } else if (cmd == KfcCommand::kExit) {
3356 0 : HCCL_RUN_INFO("[OpRetry][AICPU]hccl aicpu recv exit cmd from host.");
3357 0 : errorCode = KfcError::kExit;
3358 0 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR;
3359 0 : } else if (cmd == KfcCommand::kReportRetryErr) {
3360 0 : HCCL_RUN_INFO("[OpRetry][AICPU]hccl aicpu get report retry err cmd.");
3361 0 : CHK_PRT(SendTaskExceptionByMBox(TS_ERROR_RETRY_CONSTRAINT));
3362 0 : errorCode = KfcError::kExit;
3363 0 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR;
3364 0 : return HCCL_E_OPRETRY_FAIL;
3365 0 : } else if (cmd == KfcCommand::NsStopLaunch && endStopLaunch == false) {
3366 0 : HCCL_RUN_INFO("[NsRecovery][AICPU]hccl aicpu force stop in launch loop.");
3367 0 : endStopLaunch = true;
3368 0 : needsResponseStopLaunch_ = true;
3369 0 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_STOP_LAUNCH;
3370 : } else {
3371 : // do nothing
3372 : }
3373 0 : lastCmd = cmd;
3374 0 : return HCCL_SUCCESS;
3375 : }
3376 :
3377 2 : HcclResult HcclCommAicpu::ResetOpRetryException(HcclCMDType opType)
3378 : {
3379 2 : if (opType == HcclCMDType::HCCL_CMD_BATCH_SEND_RECV) {
3380 0 : CHK_RET(ResetBSRException());
3381 : } else {
3382 6 : std::vector<Stream> totalStream = {mainStream_};
3383 2 : totalStream.insert(totalStream.end(), slaveStreams_.begin(), slaveStreams_.end());
3384 4 : for (auto& stream : totalStream) {
3385 2 : CHK_RET(CleanStream(stream));
3386 2 : CHK_RET(ClearStreamCqeException(stream));
3387 : }
3388 2 : }
3389 2 : return HCCL_SUCCESS;
3390 2 : }
3391 :
3392 0 : HcclResult HcclCommAicpu::ResetSqBuff()
3393 : {
3394 0 : CHK_RET(CleanStream(mainStream_));
3395 0 : for (auto& stream : slaveStreams_) {
3396 0 : CHK_RET(CleanStream(stream));
3397 : }
3398 0 : CHK_RET(CleanStream(orderStream_));
3399 0 : HCCL_INFO("reset stream sq buffer success.");
3400 0 : return HCCL_SUCCESS;
3401 : }
3402 :
3403 0 : HcclResult HcclCommAicpu::UpdateSqStatus(Stream& stream)
3404 : {
3405 0 : HcclSqeContext* sqeContext = stream.GetSqeContextPtr();
3406 0 : CHK_PTR_NULL(sqeContext);
3407 0 : SqeRingBuffer* sqeContextBuffer = &(sqeContext->buffer);
3408 0 : CHK_PTR_NULL(sqeContextBuffer);
3409 0 : auto& head = sqeContextBuffer->sqHead;
3410 0 : auto& tail = sqeContextBuffer->sqTail;
3411 :
3412 0 : CHK_RET(QuerySqStatusByType(devId_, stream.sqId(), DRV_SQCQ_PROP_SQ_TAIL, head));
3413 0 : CHK_RET(QuerySqStatusByType(devId_, stream.sqId(), DRV_SQCQ_PROP_SQ_HEAD, tail));
3414 0 : HCCL_INFO("UpdateSqStatus, sqid:%u head:%u tail:%u.", stream.sqId(), head, tail);
3415 0 : return HCCL_SUCCESS;
3416 : }
3417 :
3418 0 : HcclResult HcclCommAicpu::HcclOpExecFsmRetryProcess(
3419 : const std::string& algName, OpParam& param, std::unique_ptr<CollExecutorBase>& executor,
3420 : AlgResourceResponse& algResource, HcclOpExecFSM& fsmState, KfcError& errorCode, uint32_t& retryCnt,
3421 : uint32_t& beginSqePos, uint32_t& endSqePos)
3422 : {
3423 0 : retryCnt++;
3424 0 : if ((param.opType == HcclCMDType::HCCL_CMD_BATCH_SEND_RECV) && retryEnable_) {
3425 0 : UpdateBSRRetryCnt();
3426 : }
3427 0 : HCCL_RUN_INFO("[OpRetry][AICPU]retry launch start, retryCnt:%u, tag[%s].", retryCnt, param.tag.c_str());
3428 :
3429 0 : auto ret = RetryOrchestrateHcclOp(algName, param, executor, algResource, beginSqePos, endSqePos);
3430 0 : if (ret == HCCL_SUCCESS) {
3431 0 : errorCode = KfcError::kNone;
3432 0 : if ((param.opType == HcclCMDType::HCCL_CMD_BATCH_SEND_RECV) && retryEnable_) {
3433 : // 是否有前一个重执行阶段积累的故障未上报，需要再次触发重执行
3434 0 : CHK_RET(CommitBSRStoredException(fsmState, errorCode));
3435 0 : } else {
3436 0 : CHK_RET(UpdateOpExecStatus(fsmState, KfcStatus::kRuning, errorCode, retryCnt));
3437 0 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_FSM_WAIT_END;
3438 : }
3439 0 : } else if (ret == HCCL_E_SUSPENDING) {
3440 0 : HCCL_RUN_INFO("hccl aicpu force stop in retry launch process");
3441 0 : if ((param.opType == HcclCMDType::HCCL_CMD_BATCH_SEND_RECV) && retryEnable_) {
3442 : // batchsendrecv算子下发过程中出现异常，task下发未完成，send 和 recv 均需要重执行
3443 0 : if (bsrRetryOp_ == HCCL_SEND) {
3444 0 : SetBSRSendOpExecException();
3445 : } else {
3446 0 : SetBSRRecvOpExecException();
3447 : }
3448 0 : HCCL_RUN_INFO("hccl aicpu abort launch batchsendrecv op, need retry.");
3449 : }
3450 0 : CHK_RET(UpdateSuspendStatus(param, fsmState, errorCode, retryCnt));
3451 : } else {
3452 0 : HCCL_ERROR("RetryLaunchHcclOp failed, ret:%u", ret);
3453 0 : errorCode = KfcError::kInner;
3454 0 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR;
3455 : }
3456 0 : return ret;
3457 : }
3458 :
3459 0 : HcclResult HcclCommAicpu::HcclOpExecFsmEndProcess(uint32_t retryCnt)
3460 : {
3461 : auto ret
3462 0 : = aicpuHdc_.SetOpExecStatus(kfcStatusTransferD2H_, excuteOpId_, KfcStatus::kEnd, KfcError::kNone, retryCnt);
3463 0 : if (!isDeviceMode_) {
3464 0 : isOpLaunch = false;
3465 : }
3466 0 : dfxExtendInfo_.kfcStatus = DfxKfcStatus::kOneFinished;
3467 0 : HCCL_DEBUG("---------- end AICPU_HcclOpExecFsmEndProcess ----------");
3468 0 : return ret;
3469 : }
3470 :
3471 0 : HcclResult HcclCommAicpu::PrintTaskExceptionAllThreads()
3472 : {
3473 : // 非独立算子场景，跳过
3474 0 : CHK_PRT_RET(
3475 : !GetIsInitIndOp(), HCCL_RUN_INFO("[%s] IndOp group[%s] not init, skip", __func__, identifier_.c_str()),
3476 : HCCL_SUCCESS);
3477 :
3478 0 : for (auto& thread : threads_) {
3479 0 : CHK_RET(taskExecption_.PrintTaskException(*(thread->GetStream())));
3480 : }
3481 0 : return HCCL_SUCCESS;
3482 : }
3483 :
3484 0 : void HcclCommAicpu::PrintTaskExceptionAllComm()
3485 : {
3486 0 : std::shared_lock<std::shared_mutex> rwlock(AicpuHcclProcess::AicpuGetCommMutex());
3487 :
3488 : // 先打印本通信域的taskException
3489 0 : (void)PrintTaskExceptionAllStreams();
3490 0 : (void)PrintTaskExceptionAllThreads();
3491 :
3492 : // 再打印其他通信域的taskException
3493 0 : std::vector<std::pair<std::string, hccl::HcclCommAicpu*>> aicpuCommInfo;
3494 0 : (void)AicpuHcclProcess::AicpuGetCommAll(aicpuCommInfo);
3495 0 : for (auto& commInfo : aicpuCommInfo) {
3496 0 : hccl::HcclCommAicpu* hcclAicpu = commInfo.second;
3497 0 : if (hcclAicpu == nullptr || hcclAicpu->identifier_ == identifier_) {
3498 0 : continue;
3499 : }
3500 0 : (void)hcclAicpu->PrintTaskExceptionAllThreads();
3501 0 : (void)hcclAicpu->PrintTaskExceptionAllStreams();
3502 : }
3503 0 : }
3504 :
3505 0 : void HcclCommAicpu::PrintAicpuCommExecStatus()
3506 : {
3507 0 : std::shared_lock<std::shared_mutex> rwlock(AicpuHcclProcess::AicpuGetCommMutex());
3508 :
3509 : // 记录通信域占核情况
3510 0 : int64_t inExecGroupNum = 0;
3511 0 : int64_t aicpuCoreNum = 0;
3512 0 : (void)hrtHalGetDeviceInfo(devId_, MODULE_TYPE_AICPU, INFO_TYPE_CORE_NUM, &aicpuCoreNum);
3513 :
3514 0 : std::vector<std::pair<std::string, hccl::HcclCommAicpu*>> aicpuCommInfo;
3515 0 : (void)AicpuHcclProcess::AicpuGetCommAll(aicpuCommInfo);
3516 0 : for (auto& commInfo : aicpuCommInfo) {
3517 0 : hccl::HcclCommAicpu* hcclAicpu = commInfo.second;
3518 0 : if (hcclAicpu == nullptr || !hcclAicpu->GetCommInfoStatus()) {
3519 0 : continue;
3520 : }
3521 :
3522 : // 获取并打印通信域是否在执行中，以及最后一次下发的算子
3523 0 : bool isInExec = AicpuHcclProcess::GetCommExecStatus(hcclAicpu->identifier_);
3524 0 : inExecGroupNum += isInExec ? 1 : 0;
3525 0 : std::string execStatus = isInExec ? "inExec" : "unExec";
3526 0 : HCCL_RUN_INFO(
3527 : "AicpuComm: group[%s], status[%s], op[%s], aicpuCoreNum[%lld]", hcclAicpu->identifier_.c_str(),
3528 : execStatus.c_str(), hcclAicpu->GetExcuteOp().c_str(), aicpuCoreNum);
3529 0 : }
3530 :
3531 : // AICPU核被占满，部分通信域得不到调度，可能导致通信阻塞，打印维测信息
3532 0 : if (inExecGroupNum >= aicpuCoreNum && static_cast<int64_t>(aicpuCommInfo.size()) > aicpuCoreNum) {
3533 0 : HCCL_RUN_WARNING(
3534 : "In Execution group num[%lld], total group num[%u], bigger than Aicpu cores num[%lld]. "
3535 : "Aicpu core being fully utilized may cause tasks to get stuck, and it is necessary to reduce the num of "
3536 : "comm.",
3537 : inExecGroupNum, aicpuCommInfo.size(), aicpuCoreNum);
3538 : }
3539 0 : }
3540 :
3541 0 : HcclResult HcclCommAicpu::PrintTaskExceptionAllStreams()
3542 : {
3543 : // 通信域资源已经释放
3544 0 : CHK_PRT_RET(
3545 : !commOpenStatus,
3546 : HCCL_RUN_INFO("[PrintTaskExceptionAllStreams]group[%s] has been destroyed", identifier_.c_str()), HCCL_SUCCESS);
3547 0 : CHK_RET(UtraceInfo_->Flush());
3548 0 : std::vector<Stream> totalStream = {mainStream_};
3549 : HcclResult ret;
3550 0 : totalStream.insert(totalStream.end(), slaveStreams_.begin(), slaveStreams_.end());
3551 0 : for (auto& stream : totalStream) {
3552 0 : HCCL_RUN_INFO("[PrintTaskExceptionAllStreams]group[%s] streamid[%d] print", identifier_.c_str(), stream.id());
3553 0 : u32 sqHead = 0U;
3554 0 : u32 sqTail = 0U;
3555 0 : ret = QuerySqStatus(devId_, stream.sqId(), sqHead, sqTail);
3556 0 : if (ret != HCCL_SUCCESS || sqHead == sqTail) { // 此流为空时，不打印
3557 0 : HCCL_RUN_INFO(
3558 : "[PrintTaskExceptionAllStreams] group[%s] streamid[%d] is empty"
3559 : "or QuerySqStatus failed, ret[%d]",
3560 : identifier_.c_str(), stream.id(), ret);
3561 0 : continue;
3562 : }
3563 0 : HcclSqeContext* sqeContext = stream.GetSqeContextPtr();
3564 0 : SqeRingBuffer* sqeContextBuffer = &(sqeContext->buffer);
3565 0 : CHK_PTR_NULL(sqeContextBuffer);
3566 0 : if (stream.id() == mainStream_.id()) {
3567 0 : SqeInfo sqeInfo;
3568 0 : ret = SqeContextUtils::QuerySqeInfo(
3569 0 : sqeContextBuffer->rtsMirrorBuffer + sqHead * HCCL_SQE_SIZE, sqeContextBuffer->rtsqSqeType[sqHead],
3570 : sqeContextBuffer->addInfo[sqHead], &sqeInfo);
3571 0 : if (ret != HCCL_SUCCESS) {
3572 0 : HCCL_ERROR("[%s]QuerySqeInfo failed, ret[%d]", __func__, ret);
3573 : } else {
3574 : // 根据主流卡在host notify上，则说明未被执行到不打印
3575 0 : if (sqeInfo.type == RT_STARS_SQE_TYPE_NOTIFY_WAIT && sqeInfo.notifyId == opNotifies_[0]->notifyId_) {
3576 0 : HCCL_RUN_INFO(
3577 : "[PrintTaskExceptionAllStreams] group[%s] op is not activated, do nothing",
3578 : identifier_.c_str());
3579 0 : return HCCL_SUCCESS;
3580 : }
3581 : // 根据主流当前位置，判断该算子是否已经打印过taskException
3582 0 : if (IsRepeatedOpTaskException(sqHead, sqeContextBuffer)) {
3583 0 : HCCL_INFO(
3584 : "[PrintTaskExceptionAllStreams] group[%s] op has been printed, do nothing",
3585 : identifier_.c_str());
3586 0 : return HCCL_SUCCESS;
3587 : }
3588 : }
3589 : }
3590 :
3591 0 : uint8_t type = 0;
3592 0 : uint16_t taskId = 0;
3593 0 : uint32_t remoteRank = 0;
3594 0 : HCCL_ERROR(
3595 : "[TaskException]base information is streamId:%d, sqid:%d, head:%u, tail:%u, %s", stream.id(), stream.sqId(),
3596 : sqHead, sqTail, GetTaskExceptionTaskInfo(sqHead, sqeContextBuffer, type, taskId, remoteRank).c_str());
3597 0 : PrintTaskExceptionTaskQue(sqHead, sqeContextBuffer);
3598 : }
3599 0 : return HCCL_SUCCESS;
3600 0 : }
3601 :
3602 0 : bool HcclCommAicpu::IsRepeatedOpTaskException(u32 idx, SqeRingBuffer* sqeContextBuffer)
3603 : {
3604 0 : const AicpuOpInfo* opInfo = aicpuShareData_.GetAicpuOpInfo(sqeContextBuffer->rtsDfxInfo[idx].opRingBufferIdx);
3605 0 : CHK_PRT_RET(opInfo == nullptr, HCCL_ERROR("%s fail, opInfo is nullptr", __func__), false);
3606 0 : std::string opTag = opInfo->tagBuff;
3607 0 : u32 opIndex = opInfo->opIndex;
3608 0 : bool opHasPrinted = opTaskException_.find(opTag) != opTaskException_.end() && opTaskException_[opTag] == opIndex;
3609 0 : opTaskException_[opTag] = opIndex;
3610 0 : CHK_PRT_CONT(
3611 : opHasPrinted, HCCL_RUN_INFO(
3612 : "[IsRepeatedOpTaskException]group[%s], op[%s], opIndex[%u] "
3613 : "has been printed",
3614 : identifier_.c_str(), opInfo->tagBuff, opInfo->opIndex));
3615 0 : return opHasPrinted;
3616 0 : }
3617 :
3618 0 : void HcclCommAicpu::PrepareMc2Handler()
3619 : {
3620 0 : auto& handler = algOpContext_.mc2Handler;
3621 0 : handler.stepSize = 0U;
3622 0 : if (!isDeviceMode_) {
3623 0 : HCCL_INFO("Unset step size for non-MC2.");
3624 0 : return;
3625 : }
3626 0 : (void)InvokeKfcHandler(AicpuKfcHandlerType::kSetStepSize, {rpc_, reinterpret_cast<u64>(&handler), GetRankSize()});
3627 : }
3628 :
3629 0 : HcclResult HcclCommAicpu::OrchestrateHcclOp(
3630 : const std::string& algName, OpParam& param, std::unique_ptr<CollExecutorBase>& executor,
3631 : AlgResourceResponse& algResource, uint32_t& beginSqePos, uint32_t& endSqePos)
3632 : {
3633 0 : LogControl logControl(false, false); // 重执行ERROR日志控制，析构时重置日志设置
3634 0 : PrepareMc2Handler();
3635 0 : HcclResult ret = HCCL_SUCCESS;
3636 : // task的尾指针，已便重执行stop时判断是否已执行该task，如果该task已执行完成则可支持通信重执行
3637 0 : CHK_RET(QuerySqStatusByType(devId_, mainStream_.sqId(), DRV_SQCQ_PROP_SQ_TAIL, beginSqePos));
3638 :
3639 0 : const bool retryForBatchSndRcv = (param.opType == HcclCMDType::HCCL_CMD_BATCH_SEND_RECV && retryEnable_);
3640 0 : if (retryForBatchSndRcv) {
3641 0 : CHK_RET(QueryBatchSendRecvPairBeginPos());
3642 0 : if (param.BatchSendRecvDataDes.curIterNum == 0) {
3643 : // batchsendrecv算子拆分为多轮执行，只有第一个step和最后一个step需要和主stream交互
3644 0 : CHK_RET(NotifyWait());
3645 : }
3646 0 : HCCL_INFO(
3647 : "batch send recv op: step %u, mode:%u", param.BatchSendRecvDataDes.curIterNum,
3648 : param.BatchSendRecvDataDes.curMode);
3649 : } else {
3650 0 : CHK_RET(NotifyWait());
3651 : // 重执行场景, 算子计数在host侧; MC2场景也不开启卡住检测能力
3652 0 : if (opCounterInfo_.isEnableCounter && !retryEnable_ && !isDeviceMode_) {
3653 0 : CHK_RET(HcclReduceAsync(
3654 : dispatcher_, reinterpret_cast<void*>(opCounterInfo_.addOneMem),
3655 : opCounterInfo_.memSize / sizeof(int32_t), HCCL_DATA_TYPE_INT32, HCCL_REDUCE_SUM, mainStream_,
3656 : reinterpret_cast<void*>(opCounterInfo_.headCountMem), INVALID_VALUE_RANKID, LinkType::LINK_ONCHIP,
3657 : INLINE_REDUCE_BIT));
3658 : }
3659 : }
3660 :
3661 : // 打印当前展开的算子信息
3662 0 : HCCL_INFO(
3663 : "[HcclCommAicpu][OrchestrateHcclOp] opUnfoldIdx_[%u] opType[%d] curRank[%u] rankSize[%u] algName[%s]",
3664 : opUnfoldIdx_, param.opType, topoInfo_.userRank, GetRankSize(), algName.c_str());
3665 0 : HCCL_INFO(
3666 : "[HcclCommAicpu][OrchestrateHcclOp] inputPtr[0x%016llx] inputSize[%llu] outputPtr[0x%016llx] outputSize[%llu]",
3667 : param.inputPtr, param.inputSize, param.outputPtr, param.outputSize);
3668 0 : opUnfoldIdx_ += 1;
3669 :
3670 : // 检查算子展开的动态缓存, 确认是否可以跳过算子展开
3671 0 : bool needExecute = true;
3672 0 : bool isCacheMiss = false;
3673 0 : auto setProfStartCallback = [this]() {
3674 0 : return this->InvokeKfcHandler(AicpuKfcHandlerType::kSetProfTimeStart, {});
3675 0 : };
3676 0 : CHK_RET(aicpuCacheManager_.LookupOpUnfoldCache(
3677 : algName, param, algResource, needExecute, isCacheMiss, mainStream_, slaveStreams_, dispatcher_, isDeviceMode_,
3678 : topoInfo_, topoMatcher_, algOpContext_, ZeroCopyExchanger_, GetWorkflowMode(), tinySendRecvMem_,
3679 : setProfStartCallback));
3680 :
3681 : // 根据needExecute有条件的执行算子展开
3682 : // 需要算子执行的场景: (i) expansion mode为AI_CPU_NO_CACHE; (ii) uncacheable算子/场景; (iii) cache miss
3683 0 : if (needExecute) {
3684 : // Cache miss前执行cache相关的预处理
3685 0 : if (isCacheMiss) {
3686 0 : CHK_RET(aicpuCacheManager_.PreProcessForCacheMiss(param, executor));
3687 : }
3688 :
3689 : // executor设置AlgOpContext
3690 0 : CHK_RET(executor->SetAlgOpContext(algOpContext_));
3691 0 : (void)InvokeKfcHandler(AicpuKfcHandlerType::kSetProfTimeStart, {});
3692 0 : ret = executor->Orchestrate(param, algResource);
3693 0 : if (ret != HCCL_SUCCESS) {
3694 0 : HCCL_ERROR(
3695 : "[HcclCommAicpu][Orchestrate]executor process failed algName[%s], ret = %u", algName.c_str(), ret);
3696 0 : printTaskExceptionForErr_ |= (ret == HCCL_E_AGAIN);
3697 0 : return ret;
3698 : }
3699 :
3700 : // Cache miss后执行cache相关的后处理
3701 0 : if (isCacheMiss) {
3702 0 : CHK_RET(aicpuCacheManager_.PostProcessForCacheMiss(
3703 : param, executor, mainStream_, slaveStreams_, dispatcher_, topoInfo_, algOpContext_, GetWorkflowMode()));
3704 : }
3705 : } // 正常算子展开
3706 :
3707 : // batchsendrecv算子拆分为多轮执行，只有第一个step和最后一个step需要和主stream交互
3708 0 : if (!retryForBatchSndRcv || param.BatchSendRecvDataDes.curIterNum + 1 >= bsrSendRecvPairs_.size()) {
3709 : // 重执行场景, 算子计数在host侧 MC2场景也不开
3710 0 : if (opCounterInfo_.isEnableCounter && !retryEnable_ && !isDeviceMode_) {
3711 0 : CHK_RET(HcclReduceAsync(
3712 : dispatcher_, reinterpret_cast<void*>(opCounterInfo_.addOneMem),
3713 : opCounterInfo_.memSize / sizeof(int32_t), HCCL_DATA_TYPE_INT32, HCCL_REDUCE_SUM, mainStream_,
3714 : reinterpret_cast<void*>(opCounterInfo_.tailCountMem), INVALID_VALUE_RANKID, LinkType::LINK_ONCHIP,
3715 : INLINE_REDUCE_BIT));
3716 : }
3717 0 : CHK_RET(NotifyPost());
3718 : }
3719 0 : (void)InvokeKfcHandler(AicpuKfcHandlerType::kSetProfTimeOrch, {});
3720 0 : ret = LaunchTask(dispatcher_, const_cast<Stream&>(mainStream_));
3721 0 : if (ret != HCCL_SUCCESS) {
3722 0 : HCCL_ERROR("[HcclCommAicpu][LaunchTask]algName[%s] ret = %u", algName.c_str(), ret);
3723 0 : printTaskExceptionForErr_ |= (ret == HCCL_E_AGAIN);
3724 0 : return ret;
3725 : }
3726 0 : ret = LaunchSlaveStreamTask(algResource);
3727 0 : if (ret != HCCL_SUCCESS) {
3728 0 : HCCL_ERROR("[HcclCommAicpu][LaunchSlaveStreamTask]algName[%s] ret = %u", algName.c_str(), ret);
3729 0 : printTaskExceptionForErr_ |= (ret == HCCL_E_AGAIN);
3730 0 : return ret;
3731 : }
3732 0 : (void)InvokeKfcHandler(AicpuKfcHandlerType::kSetProfTimeEnd, {});
3733 0 : if (retryForBatchSndRcv) {
3734 0 : CHK_RET(QueryBatchSendRecvPairEndPos());
3735 : }
3736 :
3737 0 : CHK_RET(QuerySqStatusByType(devId_, mainStream_.sqId(), DRV_SQCQ_PROP_SQ_TAIL, endSqePos));
3738 :
3739 0 : HCCL_INFO(
3740 : "hccl aicpu launch hccl op task success. stream sqid:%u begin:%u end:%u", mainStream_.sqId(), beginSqePos,
3741 : endSqePos);
3742 0 : return HCCL_SUCCESS;
3743 0 : }
3744 :
3745 0 : HcclResult HcclCommAicpu::RetryOrchestrateHcclOp(
3746 : const std::string& algName, OpParam& param, std::unique_ptr<CollExecutorBase>& executor,
3747 : AlgResourceResponse& algResource, uint32_t& beginSqePos, uint32_t& endSqePos)
3748 : {
3749 0 : LogControl logControl(false, false); // 重执行ERROR日志控制，析构时重置日志设置
3750 0 : if ((param.opType == HcclCMDType::HCCL_CMD_BATCH_SEND_RECV) && retryEnable_) {
3751 : param.BatchSendRecvDataDes.curMode
3752 0 : = (bsrRetryOp_ == HCCL_SEND) ? BatchSendRecvCurMode::SEND : BatchSendRecvCurMode::RECV;
3753 0 : if (param.BatchSendRecvDataDes.curMode == BatchSendRecvCurMode::SEND) {
3754 0 : HCCL_INFO(
3755 : "BSR: iter %u, retry send op tag:%s index:%u", param.BatchSendRecvDataDes.curIterNum, bsrSendOpId_.tag,
3756 : bsrSendOpId_.index);
3757 : } else {
3758 0 : HCCL_INFO(
3759 : "BSR: iter %u, retry recv op tag:%s index:%u", param.BatchSendRecvDataDes.curIterNum, bsrRecvOpId_.tag,
3760 : bsrRecvOpId_.index);
3761 : }
3762 : }
3763 :
3764 0 : CHK_RET(AddRetryExecFlipTask(algResource));
3765 0 : HcclResult ret = executor->Orchestrate(param, algResource);
3766 0 : if (ret != HCCL_SUCCESS) {
3767 0 : HCCL_ERROR("[HcclCommAicpu][Orchestrate]executor process failed algName[%s]", algName.c_str());
3768 0 : return ret;
3769 : }
3770 :
3771 0 : if ((param.opType == HcclCMDType::HCCL_CMD_BATCH_SEND_RECV) && retryEnable_) {
3772 : // batchsendrecv算子重执行时，aicpu 主stream没有clean，不需要重新下发notify record
3773 : // do nothing
3774 : } else {
3775 0 : CHK_RET(NotifyPost());
3776 : }
3777 0 : CHK_RET(LaunchTask(dispatcher_, const_cast<Stream&>(mainStream_)));
3778 0 : CHK_RET(LaunchSlaveStreamTask(algResource));
3779 :
3780 0 : CHK_RET(QuerySqStatusByType(devId_, mainStream_.sqId(), DRV_SQCQ_PROP_SQ_TAIL, endSqePos));
3781 0 : if ((param.opType == HcclCMDType::HCCL_CMD_BATCH_SEND_RECV) && retryEnable_) {
3782 0 : CHK_RET(QueryBatchSendRecvPairEndPos());
3783 : }
3784 :
3785 0 : HCCL_RUN_INFO(
3786 : "[OpRetry][AICPU]hccl aicpu retry launch hccl op task success. stream sqid:%u begin:%u end:%u",
3787 : mainStream_.sqId(), beginSqePos, endSqePos);
3788 0 : return HCCL_SUCCESS;
3789 0 : }
3790 :
3791 4 : bool HcclCommAicpu::IsTaskExceptionForHccs()
3792 : {
3793 4 : if (dfxExtendInfo_.cqeStatus != dfx::CqeStatus::kCqeException) {
3794 4 : return false;
3795 : }
3796 :
3797 : // NOTE: 需要task exception补全dfx能力，定位故障task的remote rank;
3798 : // 目前暂不具备识别是否跨片的能力，默认失败的task均为跨片操作。
3799 0 : if (dfxExtendInfo_.cqeException.sqeType == RT_STARS_SQE_TYPE_SDMA
3800 0 : && (dfxExtendInfo_.cqeException.errorCode == RT_SDMA_COMPDATAERR
3801 0 : || dfxExtendInfo_.cqeException.errorCode == RT_SDMA_COMPERR)) {
3802 0 : return true;
3803 : }
3804 0 : return false;
3805 : }
3806 :
3807 3 : void HcclCommAicpu::SetAlgType(u64 algType)
3808 : {
3809 3 : algType_.algoLevel0 = static_cast<AlgTypeLevel0>(static_cast<u32>(algType) & ((1 << HCCL_LEVEL_ALGO_WIDTH) - 1));
3810 3 : algType_.algoLevel1 = static_cast<AlgTypeLevel1>(
3811 3 : (static_cast<u32>(algType) >> HCCL_LEVEL_ALGO_WIDTH) & ((1 << HCCL_LEVEL_ALGO_WIDTH) - 1));
3812 : algType_.algoLevel2
3813 3 : = static_cast<AlgTypeLevel2>(static_cast<u32>(algType) >> (HCCL_LEVEL_ALGO_WIDTH + HCCL_LEVEL_ALGO_WIDTH));
3814 3 : HCCL_INFO("[HcclCommAicpu][SetAlgType]algType:%u", algType);
3815 3 : }
3816 :
3817 0 : void HcclCommAicpu::SetDebugMode(u8 debugMode) { debugMode_ = debugMode; }
3818 :
3819 0 : void HcclCommAicpu::SetSendRecvInfoPtr(void* sendRecvInfoPtr) { sendRecvInfoPtr_ = sendRecvInfoPtr; }
3820 :
3821 4 : bool HcclCommAicpu::IsNoNeedWait(void)
3822 : {
3823 4 : return isDeviceMode_ || (debugMode_ != MC2_DEBUG_WAIT_COMM && retryEnable_ == false);
3824 : }
3825 :
3826 45 : bool HcclCommAicpu::GetOpRetryEnable() { return retryEnable_; }
3827 :
3828 4 : HcclResult HcclCommAicpu::ReportHcclTaskInfo(Stream& mainStream, std::vector<Stream>& subStreams)
3829 : {
3830 4 : if (dfx::ProfilingManager::GetProfL1State()) {
3831 0 : CHK_RET(dfx::ProfilingManager::ReportTaskInfo(mainStream.id(), mainStream.GetSqeContextPtr()));
3832 0 : for (auto& subStream : subStreams) {
3833 0 : CHK_RET(dfx::ProfilingManager::ReportTaskInfo(subStream.id(), subStream.GetSqeContextPtr()));
3834 : }
3835 : }
3836 4 : return HCCL_SUCCESS;
3837 : }
3838 :
3839 4 : HcclResult HcclCommAicpu::ClearLocalBuff(Stream& mainStream, std::vector<Stream>& subStreams)
3840 : {
3841 4 : CHK_RET(mainStream.ClearLocalBuff());
3842 4 : CHK_RET(dfx::ProfilingManager::UpdateStartReportSqeIdx(mainStream.id(), 0));
3843 4 : for (auto& subStream : subStreams) {
3844 0 : CHK_RET(subStream.ClearLocalBuff());
3845 0 : CHK_RET(dfx::ProfilingManager::UpdateStartReportSqeIdx(subStream.id(), 0));
3846 : }
3847 4 : return HCCL_SUCCESS;
3848 : }
3849 :
3850 4 : HcclResult HcclCommAicpu::WaitFinishWhileLoop(
3851 : Stream& mainStream, std::vector<Stream>& subStreams, std::string& tag, const uint32_t& beginSqePos, OpParam& param)
3852 : {
3853 : // 上报Profiling HCCL INFO信息
3854 4 : CHK_RET(ReportHcclTaskInfo(mainStream, subStreams));
3855 4 : CHK_RET(ClearLocalBuff(mainStream, subStreams));
3856 4 : if (IsNoNeedWait()) {
3857 0 : return HCCL_SUCCESS;
3858 : }
3859 4 : const uint64_t startUsec = GetCurCpuTimestamp();
3860 4 : uint64_t lastUsec = startUsec;
3861 4 : int32_t sqId = mainStream.sqId();
3862 4 : uint32_t sqHead = 0;
3863 4 : uint32_t sqTail = 0;
3864 4 : CHK_RET(QuerySqStatusByType(devId_, sqId, DRV_SQCQ_PROP_SQ_TAIL, sqTail));
3865 4 : CHK_RET(QuerySqStatusByType(devId_, sqId, DRV_SQCQ_PROP_SQ_HEAD, sqHead));
3866 4 : while (sqHead != sqTail) {
3867 4 : HcclResult ret = CheckOpExecStatus(); // 检查执行状态，判断是否有异常cq或中断命令
3868 4 : CHK_PRT_RET(
3869 : ret != HCCL_SUCCESS,
3870 : HCCL_RUN_INFO("[HcclCommAicpu][WaitFinishWhileLoop]CheckOpExecStatus exception, ret[%u]", ret), ret);
3871 :
3872 0 : CHK_RET(QuerySqStatusByType(devId_, sqId, DRV_SQCQ_PROP_SQ_HEAD, sqHead));
3873 0 : if ((param.opType == HcclCMDType::HCCL_CMD_BATCH_SEND_RECV) && (retryEnable_) && (sqHead != beginSqePos)
3874 0 : && (!excuteOpId_.isBsrTaskStart)) {
3875 : // 更新D2H通道里的信息opid中isBsrTaskStart = true
3876 0 : excuteOpId_.isBsrTaskStart = true;
3877 : HcclResult ret1
3878 0 : = aicpuHdc_.SetOpExecStatus(kfcStatusTransferD2H_, excuteOpId_, KfcStatus::kRuning, KfcError::kNone, 0);
3879 0 : CHK_PRT_RET(ret1 != HCCL_SUCCESS, HCCL_ERROR("update OpExecStatus failed, ret:%u", ret1), ret1);
3880 0 : HCCL_INFO(
3881 : "[HcclCommAicpu][WaitFinishWhileLoop]bsr start task is completed. devId:%d sqid:%d, head:%u,"
3882 : "beginSqePos[%u] group[%s] tag[%s]",
3883 : devId_, sqId, sqHead, beginSqePos, identifier_.c_str(), tag.c_str());
3884 : }
3885 0 : uint64_t curUsec = GetCurCpuTimestamp();
3886 0 : if (curUsec - lastUsec > static_cast<uint64_t>(NSEC_PER_SEC) * dfx::kPrintSqInterval) {
3887 0 : lastUsec = curUsec;
3888 0 : HCCL_RUN_INFO(
3889 : "[HcclCommAicpu][WaitFinishWhileLoop]Current state. devId:%d sqid:%d, head:%u, tail:%u, "
3890 : "group[%s] tag[%s]",
3891 : devId_, sqId, sqHead, sqTail, identifier_.c_str(), tag.c_str());
3892 : }
3893 0 : CHK_RET(CheckTaskTimeout(mainStream, startUsec));
3894 : }
3895 0 : return HCCL_SUCCESS;
3896 : }
3897 :
3898 0 : HcclResult HcclCommAicpu::CheckTaskTimeout(const Stream& mainStream, const uint64_t startUsec)
3899 : {
3900 0 : if (sqeWaitTimeOut_ != 0
3901 0 : && (GetCurCpuTimestamp() - startUsec > static_cast<uint64_t>(NSEC_PER_SEC) * sqeWaitTimeOut_)) {
3902 0 : uint32_t status = 0U;
3903 0 : int32_t sqId = mainStream.sqId();
3904 0 : auto ret = QuerySqStatusByType(devId_, sqId, DRV_SQCQ_PROP_SQ_CQE_STATUS, status);
3905 0 : if (ret != 0) {
3906 0 : HCCL_ERROR("[HcclCommAicpu]QuerySqStatusByType status failed. ret = %u sqid:%d", ret, sqId);
3907 : }
3908 :
3909 0 : HCCL_ERROR("[HcclCommAicpu]KFC timeout.. group[%s].", identifier_.c_str());
3910 0 : printTaskExceptionForErr_ = true;
3911 0 : return HCCL_E_TIMEOUT;
3912 : }
3913 0 : return HCCL_SUCCESS;
3914 : }
3915 :
3916 0 : HcclResult HcclCommAicpu::AddRetryExecFlipTask(AlgResourceResponse& algResource)
3917 : {
3918 0 : CHK_RET(AddRetryPreamble(dispatcher_, mainStream_));
3919 0 : for (u32 i = 0; i < algResource.slaveStreams.size(); ++i) {
3920 0 : HcclResult ret = AddRetryPreamble(dispatcher_, algResource.slaveStreams[i]);
3921 0 : if (ret != HCCL_SUCCESS) {
3922 0 : HCCL_ERROR(
3923 : "[HcclCommAicpu][RetryOrchestrateHcclOp] launch place holder failed, sqid:%u, ret:%u",
3924 : algResource.slaveStreams[i].sqId(), ret);
3925 0 : return ret;
3926 : }
3927 : }
3928 0 : return HCCL_SUCCESS;
3929 : }
3930 :
3931 2 : HcclResult HcclCommAicpu::LaunchSlaveStreamTask(AlgResourceResponse& algResource)
3932 : {
3933 : // 单算子模式中在算法编排中已经执行过LaunchTask，所以这里不需要再执行
3934 : // 只有图模式需要再额外执行一次对从流中的task下发
3935 2 : if (GetWorkflowMode() != HcclWorkflowMode::HCCL_WORKFLOW_MODE_OPS_KERNEL_INFO_LIB) {
3936 1 : HCCL_INFO("[HcclCommAicpu][LaunchSlaveStreamTask] op base mode don't need launch slave stream task");
3937 1 : return HCCL_SUCCESS;
3938 : }
3939 :
3940 1 : for (u32 i = 0; i < algResource.slaveStreams.size(); ++i) {
3941 0 : HcclResult ret = LaunchTask(dispatcher_, algResource.slaveStreams[i]);
3942 0 : if (ret != HCCL_SUCCESS) {
3943 0 : HCCL_ERROR(
3944 : "[HcclCommAicpu][LaunchSlaveStreamTask] launch task failed, sqid:%u, ret:%u",
3945 : algResource.slaveStreams[i].sqId(), ret);
3946 0 : return ret;
3947 : }
3948 : }
3949 :
3950 1 : return HCCL_SUCCESS;
3951 : }
3952 :
3953 : HcclResult
3954 0 : HcclCommAicpu::GetAlltoAllvSendRecvInfo(const void* sendRecvInfoPtr, HcclDataType sendType, HcclDataType recvType)
3955 : {
3956 0 : allMeshAggregationSendRecvInfo_.clear();
3957 0 : u64 stepSize = sizeof(u64) * topoInfo_.userRankSize;
3958 0 : const u32 addrItemNum = 4;
3959 0 : const u32 recvLengthStep = 2;
3960 0 : const u32 recvOffsetStep = 3;
3961 0 : for (u32 i = 0; i < topoInfo_.userRankSize; i++) {
3962 0 : SendRecvInfo sendRecvInfo;
3963 0 : sendRecvInfo.sendLength.resize(topoInfo_.userRankSize);
3964 0 : sendRecvInfo.sendOffset.resize(topoInfo_.userRankSize);
3965 0 : sendRecvInfo.recvLength.resize(topoInfo_.userRankSize);
3966 0 : sendRecvInfo.recvOffset.resize(topoInfo_.userRankSize);
3967 0 : CHK_SAFETY_FUNC_RET(memcpy_s(
3968 : sendRecvInfo.sendLength.data(), stepSize,
3969 : static_cast<const u8*>(sendRecvInfoPtr) + i * stepSize * addrItemNum + 0 * stepSize, stepSize));
3970 0 : CHK_SAFETY_FUNC_RET(memcpy_s(
3971 : sendRecvInfo.sendOffset.data(), stepSize,
3972 : static_cast<const u8*>(sendRecvInfoPtr) + i * stepSize * addrItemNum + stepSize, stepSize));
3973 0 : CHK_SAFETY_FUNC_RET(memcpy_s(
3974 : sendRecvInfo.recvLength.data(), stepSize,
3975 : static_cast<const u8*>(sendRecvInfoPtr) + i * stepSize * addrItemNum + recvLengthStep * stepSize,
3976 : stepSize));
3977 0 : CHK_SAFETY_FUNC_RET(memcpy_s(
3978 : sendRecvInfo.recvOffset.data(), stepSize,
3979 : static_cast<const u8*>(sendRecvInfoPtr) + i * stepSize * addrItemNum + recvOffsetStep * stepSize,
3980 : stepSize));
3981 0 : allMeshAggregationSendRecvInfo_.push_back(std::move(sendRecvInfo));
3982 0 : }
3983 :
3984 0 : for (auto& sendRecvInfo : allMeshAggregationSendRecvInfo_) {
3985 0 : for (u32 i = 0; i < topoInfo_.userRankSize; i++) {
3986 0 : sendRecvInfo.sendCounts.push_back(sendRecvInfo.sendLength[i] / SIZE_TABLE[sendType]);
3987 0 : sendRecvInfo.sendDispls.push_back(sendRecvInfo.sendOffset[i] / SIZE_TABLE[sendType]);
3988 0 : sendRecvInfo.recvCounts.push_back(sendRecvInfo.recvLength[i] / SIZE_TABLE[recvType]);
3989 0 : sendRecvInfo.recvDispls.push_back(sendRecvInfo.recvOffset[i] / SIZE_TABLE[recvType]);
3990 0 : HCCL_INFO(
3991 : "[GetAlltoAllvSendRecvInfo] rank[%u], sendCounts[%llu], sendDispls[%llu], "
3992 : "recvCounts[%llu], recvDispls[%llu]",
3993 : i, sendRecvInfo.sendCounts[i], sendRecvInfo.sendDispls[i], sendRecvInfo.recvCounts[i],
3994 : sendRecvInfo.recvDispls[i]);
3995 0 : HCCL_INFO(
3996 : "[GetAlltoAllvSendRecvInfo] rank[%u], sendLength[%llu], sendOffset[%llu], "
3997 : "recvLength[%llu], recvOffset[%llu]",
3998 : i, sendRecvInfo.sendLength[i], sendRecvInfo.sendOffset[i], sendRecvInfo.recvLength[i],
3999 : sendRecvInfo.recvOffset[i]);
4000 : }
4001 : }
4002 0 : CHK_RET(CheckSendRecvParams(allMeshAggregationSendRecvInfo_));
4003 0 : return HCCL_SUCCESS;
4004 : }
4005 :
4006 : HcclResult
4007 0 : HcclCommAicpu::GetAlltoAllvcSendRecvInfo(const void* sendCountMatrix, HcclDataType sendType, HcclDataType recvType)
4008 : {
4009 0 : allMeshAggregationSendRecvInfo_.clear();
4010 0 : for (u32 i = 0; i < topoInfo_.userRankSize; i++) {
4011 0 : SendRecvInfo sendRecvInfo;
4012 0 : sendRecvInfo.sendCounts.resize(topoInfo_.userRankSize);
4013 0 : sendRecvInfo.sendDispls.resize(topoInfo_.userRankSize);
4014 0 : sendRecvInfo.sendLength.resize(topoInfo_.userRankSize);
4015 0 : sendRecvInfo.sendOffset.resize(topoInfo_.userRankSize);
4016 0 : u64 curSendDispls = 0;
4017 0 : u64 curSendOffset = 0;
4018 0 : sendRecvInfo.recvCounts.resize(topoInfo_.userRankSize);
4019 0 : sendRecvInfo.recvDispls.resize(topoInfo_.userRankSize);
4020 0 : sendRecvInfo.recvLength.resize(topoInfo_.userRankSize);
4021 0 : sendRecvInfo.recvOffset.resize(topoInfo_.userRankSize);
4022 0 : u64 curRecvDispls = 0;
4023 0 : u64 curRecvOffset = 0;
4024 0 : for (u32 j = 0; j < topoInfo_.userRankSize; j++) {
4025 0 : u64 curSendCounts = *(static_cast<const u64*>(sendCountMatrix) + i * topoInfo_.userRankSize + j);
4026 0 : u64 curSendLength = curSendCounts * SIZE_TABLE[sendType];
4027 0 : sendRecvInfo.sendCounts[j] = curSendCounts;
4028 0 : sendRecvInfo.sendDispls[j] = curSendDispls;
4029 0 : sendRecvInfo.sendLength[j] = curSendLength;
4030 0 : sendRecvInfo.sendOffset[j] = curSendOffset;
4031 0 : curSendDispls += curSendCounts;
4032 0 : curSendOffset += curSendLength;
4033 0 : u64 curRecvCounts = *(static_cast<const u64*>(sendCountMatrix) + i + topoInfo_.userRankSize * j);
4034 0 : u64 curRecvLength = curRecvCounts * SIZE_TABLE[recvType];
4035 0 : sendRecvInfo.recvCounts[j] = curRecvCounts;
4036 0 : sendRecvInfo.recvDispls[j] = curRecvDispls;
4037 0 : sendRecvInfo.recvLength[j] = curRecvLength;
4038 0 : sendRecvInfo.recvOffset[j] = curRecvOffset;
4039 0 : curRecvDispls += curRecvCounts;
4040 0 : curRecvOffset += curRecvLength;
4041 0 : HCCL_DEBUG(
4042 : "GetAlltoAllvcSendRecvInfo rank[%u], sendCounts[%llu], sendDispls[%llu] "
4043 : "recvCounts[%llu], recvDispls[%llu]",
4044 : i, sendRecvInfo.sendCounts[j], sendRecvInfo.sendDispls[j], sendRecvInfo.recvCounts[j],
4045 : sendRecvInfo.recvDispls[j]);
4046 : }
4047 0 : allMeshAggregationSendRecvInfo_.push_back(sendRecvInfo);
4048 0 : }
4049 0 : CHK_RET(CheckSendRecvParams(allMeshAggregationSendRecvInfo_));
4050 0 : return HCCL_SUCCESS;
4051 : }
4052 :
4053 0 : HcclResult HcclCommAicpu::CheckSendRecvParams(const std::vector<SendRecvInfo>& allMeshAggregationSendRecvInfo)
4054 : {
4055 0 : u32 rankSize = allMeshAggregationSendRecvInfo.size();
4056 0 : for (u32 i = 0; i < rankSize; i++) {
4057 0 : u32 sendsSize = allMeshAggregationSendRecvInfo[i].sendLength.size();
4058 0 : u32 recvsSize = allMeshAggregationSendRecvInfo[i].recvLength.size();
4059 0 : if (rankSize != sendsSize || rankSize != recvsSize) {
4060 0 : HCCL_ERROR(
4061 : "[AlltoAllV][CheckSendRecvParam] rankSize[%u], sendsSize[%u], recvsSize[%u] are not match Index[%u]",
4062 : rankSize, sendsSize, recvsSize, i);
4063 0 : return HCCL_E_PARA;
4064 : }
4065 0 : for (u32 j = 0; j < sendsSize; j++) {
4066 0 : if (allMeshAggregationSendRecvInfo[i].sendLength[j] != allMeshAggregationSendRecvInfo[j].recvLength[i]) {
4067 0 : HCCL_ERROR(
4068 : "SendLength[%u][%u]: %llu and recvLength[%u][%u]: %llu are not match", i, j,
4069 : allMeshAggregationSendRecvInfo[i].sendLength[j], j, i,
4070 : allMeshAggregationSendRecvInfo[j].recvLength[i]);
4071 0 : return HCCL_E_PARA;
4072 : }
4073 : }
4074 : }
4075 0 : return HCCL_SUCCESS;
4076 : }
4077 0 : HcclResult HcclCommAicpu::GetStreamAll(std::vector<Stream>& streams)
4078 : {
4079 0 : streams.assign(slaveStreams_.begin(), slaveStreams_.end());
4080 0 : streams.push_back(mainStream_);
4081 0 : return HCCL_SUCCESS;
4082 : }
4083 :
4084 : // 校验是否有ERROR CQE和停止/退出命令，注册到dispatcher层调用
4085 0 : HcclResult HcclCommAicpu::CheckOpExecStatusCallback()
4086 : {
4087 0 : HcclResult ret = CheckOpExecStatus();
4088 0 : bool logLevel = (ret == HCCL_E_SUSPENDING);
4089 : // 返回HCCL_E_SUSPENDING时，需要跨作用域修改ERROR日志->RUN_WARNING
4090 0 : LogControl(logLevel, logLevel);
4091 0 : return ret;
4092 : }
4093 :
4094 4 : HcclResult HcclCommAicpu::CheckOpExecStatus()
4095 : {
4096 : // 检测是否有ERROR CQE
4097 4 : if (dfxExtendInfo_.pollStatus == PollStatus::kStopAsException) {
4098 0 : if (IsTaskExceptionForHccs() && retryEnable_) {
4099 0 : HCCL_RUN_INFO(
4100 : "[OpRetry][AICPU]hccl aicpu stop wait task exec finish, for task exception, identify[%s]",
4101 : identifier_.c_str());
4102 0 : return HCCL_E_SUSPENDING;
4103 : } else {
4104 0 : if (!printTaskExceptionForErr_) {
4105 0 : printTaskExceptionForErr_ = true;
4106 0 : HCCL_ERROR(
4107 : "hccl aicpu exec failed, for task exception, identify[%s], cqeStatus[%d], sqeType[%u], "
4108 : "errorCode[%u]",
4109 : identifier_.c_str(), dfxExtendInfo_.cqeStatus, dfxExtendInfo_.cqeException.sqeType,
4110 : dfxExtendInfo_.cqeException.errorCode);
4111 : }
4112 0 : return HCCL_E_INTERNAL;
4113 : }
4114 : }
4115 :
4116 : // 检测是否有停止/退出命令
4117 4 : KfcCommand cmd = KfcCommand::kNone;
4118 4 : CHK_RET(aicpuHdc_.GetOpExecCtrlCmd(kfcControlTransferH2D_, cmd));
4119 4 : if (cmd == KfcCommand::kStopLaunch && retryEnable_) {
4120 4 : HCCL_RUN_INFO(
4121 : "[OpRetry][AICPU]hccl aicpu stop wait finish, for recv stop launch cmd, identify[%s]", identifier_.c_str());
4122 4 : return HCCL_E_SUSPENDING;
4123 0 : } else if ((cmd == KfcCommand::NsStopLaunch) && (endStopLaunch == false)) {
4124 0 : needsResponseStopLaunch_ = true;
4125 0 : endStopLaunch = true;
4126 0 : HCCL_RUN_INFO("hccl aicpu stop wait finish, for recv stop launch cmd");
4127 0 : return HCCL_E_SUSPENDING;
4128 0 : } else if (cmd == KfcCommand::kDestroyComm) {
4129 0 : HCCL_ERROR("hccl aicpu stop wait finish, for recv destroy comm cmd");
4130 0 : return HCCL_E_INTERNAL;
4131 0 : } else if (cmd == KfcCommand::kExit) {
4132 0 : HCCL_ERROR("hccl aicpu stop wait finish, for recv exit cmd, identify[%s]", identifier_.c_str());
4133 0 : return HCCL_E_INTERNAL;
4134 : }
4135 0 : return HCCL_SUCCESS;
4136 : }
4137 :
4138 4 : HcclResult HcclCommAicpu::UpdateSuspendStatus(
4139 : const OpParam& param, HcclOpExecFSM& fsmState, KfcError& errorCode, uint32_t retryCnt)
4140 : {
4141 4 : if (needsResponseStopLaunch_ == true) {
4142 0 : HCCL_RUN_INFO("[NsRecovery][AICPU]hccl aicpu force stop in launch loop");
4143 0 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_STOP_LAUNCH;
4144 4 : } else if (retryEnable_) {
4145 4 : HCCL_RUN_INFO(
4146 : "[OpRetry][AICPU]hccl aicpu force stop for stop cmd or recoverable task exception, identify[%s]",
4147 : identifier_.c_str());
4148 4 : errorCode = IsTaskExceptionForHccs() ? KfcError::kSdma : errorCode;
4149 :
4150 4 : if ((param.opType == HcclCMDType::HCCL_CMD_BATCH_SEND_RECV)) {
4151 0 : HcclResult ret = GetBSRRetryOpId(param, bsrTargetOpId_);
4152 0 : HCCL_RETRY_CHK_RET_AND_TRANS_FSM(
4153 : ret, HCCL_ERROR("get batchsendrecv target op failed, ret:%u", ret), KfcError::kExec,
4154 : HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR);
4155 0 : uint32_t bsrRetryCnt = (bsrRetryOp_ == HCCL_SEND) ? bsrSendRetryCnt_ : bsrRecvRetryCnt_;
4156 0 : bsrTargetOpId_.isBsrTaskStart = true;
4157 0 : ret = aicpuHdc_.SetOpExecStatus(
4158 0 : kfcStatusTransferD2H_, bsrTargetOpId_, KfcStatus::kStoplaunch, errorCode, bsrRetryCnt);
4159 0 : HCCL_RETRY_CHK_RET_AND_TRANS_FSM(
4160 : ret, HCCL_ERROR("SetOpExecStatus failed, ret:%u", ret), KfcError::kExec,
4161 : HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR);
4162 : } else {
4163 4 : CHK_RET(UpdateOpExecStatus(fsmState, KfcStatus::kStoplaunch, errorCode, retryCnt));
4164 : }
4165 4 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_FSM_STOPPING;
4166 : } else {
4167 0 : HCCL_RUN_INFO(
4168 : "[HcclCommAicpu][UpdateSuspendStatus] aicpu force stop in launch loop; needsResponseStopLaunch_[%u] "
4169 : "retryEnable_[%u]",
4170 : needsResponseStopLaunch_, retryEnable_);
4171 0 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_STOP_LAUNCH;
4172 : }
4173 4 : return HCCL_SUCCESS;
4174 : }
4175 :
4176 0 : HcclResult HcclCommAicpu::TasktypeTransferD2H(const uint8_t sqeType, TaskType& taskType)
4177 : {
4178 0 : switch (sqeType) {
4179 0 : case RT_STARS_SQE_TYPE_PLACE_HOLDER:
4180 : case RT_STARS_SQE_TYPE_NOTIFY_WAIT:
4181 0 : taskType = TaskType::TASK_NOTIFY_WAIT;
4182 0 : break;
4183 0 : case RT_STARS_SQE_TYPE_SDMA:
4184 0 : taskType = TaskType::TASK_SDMA;
4185 0 : break;
4186 0 : case RT_STARS_SQE_TYPE_NOTIFY_RECORD:
4187 0 : taskType = TaskType::TASK_NOTIFY_RECORD;
4188 0 : break;
4189 0 : case RT_STARS_SQE_TYPE_WRITE_VALUE:
4190 0 : taskType = TaskType::TASK_NOTIFY_WAIT;
4191 0 : break;
4192 0 : default:
4193 0 : HCCL_ERROR("TasktypeTransferD2H sqeType[%d] error.", sqeType);
4194 0 : return HCCL_E_PARA;
4195 : }
4196 0 : return HCCL_SUCCESS;
4197 : }
4198 :
4199 0 : HcclResult HcclCommAicpu::GenTaskExceptionInfo(u8 sqeType, hccl::Stream& stream, u32 head)
4200 : {
4201 0 : HcclSqeContext* sqeContext = stream.GetSqeContextPtr();
4202 0 : SqeRingBuffer* sqeContextBuffer = &(sqeContext->buffer);
4203 0 : CHK_PTR_NULL(sqeContextBuffer);
4204 :
4205 0 : const AicpuOpInfo* opInfo = aicpuShareData_.GetAicpuOpInfo(sqeContextBuffer->rtsDfxInfo[head].opRingBufferIdx);
4206 0 : std::string opTag = opInfo == nullptr ? "unKnown" : opInfo->tagBuff;
4207 :
4208 : // 获取需要上报的关键信息
4209 0 : ErrorMessageReport emrInfo{};
4210 0 : SqeInfo sqeInfo;
4211 0 : SqeContextUtils::QuerySqeInfo(
4212 0 : sqeContextBuffer->rtsMirrorBuffer + head * HCCL_SQE_SIZE, sqeContextBuffer->rtsqSqeType[head],
4213 : sqeContextBuffer->addInfo[head], &sqeInfo);
4214 0 : emrInfo.remoteUserRank = sqeContextBuffer->rtsDfxInfo[head].remoteRank;
4215 0 : emrInfo.streamId = stream.id();
4216 0 : emrInfo.taskId = sqeInfo.taskId;
4217 0 : emrInfo.notifyId = sqeInfo.notifyId;
4218 0 : emrInfo.rankId = localUserRank_;
4219 0 : emrInfo.rankSize = topoInfo_.userRankSize;
4220 0 : emrInfo.algType = algType_;
4221 0 : emrInfo.opIndex = opInfo == nullptr ? 0 : opInfo->opIndex;
4222 0 : emrInfo.count = opInfo == nullptr ? 0 : opInfo->count;
4223 0 : emrInfo.dataType = opInfo == nullptr ? 0 : opInfo->dataType;
4224 0 : emrInfo.dstAddr = opInfo == nullptr ? 0 : opInfo->dstAddr;
4225 0 : emrInfo.srcAddr = opInfo == nullptr ? 0 : opInfo->srcAddr;
4226 0 : emrInfo.reduceType = opInfo == nullptr ? 255 : opInfo->reduceType; // 255 为 HcclReduceOp::HCCL_REDUCE_RESERVED
4227 0 : CHK_RET(TasktypeTransferD2H(sqeType, emrInfo.taskType));
4228 :
4229 0 : CHK_SAFETY_FUNC_RET(memcpy_s(emrInfo.tag, sizeof(emrInfo.tag), opTag.c_str(), opTag.size()));
4230 0 : CHK_SAFETY_FUNC_RET(memcpy_s(emrInfo.group, sizeof(emrInfo.group), identifier_.c_str(), identifier_.size()));
4231 0 : CHK_RET(aicpuHdc_.SetErrorMessage(kfcStatusTransferD2H_, emrInfo));
4232 0 : return HCCL_SUCCESS;
4233 0 : }
4234 :
4235 0 : HcclResult HcclCommAicpu::PrintTaskExceptionByTaskId(u8 sqeType, u16 taskId, hccl::Stream& stream, u32 tail)
4236 : {
4237 0 : HcclSqeContext* sqeContext = stream.GetSqeContextPtr();
4238 0 : CHK_PTR_NULL(sqeContext);
4239 0 : HCCL_ERROR("[HcclCommAicpu][PrintTaskExceptionByTaskId]streamId:%d tail:%u cqeType:%u", stream.id(), tail, sqeType);
4240 0 : SqeRingBuffer* sqeContextBuffer = &(sqeContext->buffer);
4241 0 : CHK_PTR_NULL(sqeContextBuffer);
4242 0 : uint8_t* sqeMirrorBufferAddr = sqeContextBuffer->rtsMirrorBuffer + (tail - 1) * HCCL_SQE_SIZE;
4243 0 : rtStarsSqeHeader_t* const sqeHeader = (rtStarsSqeHeader_t* const)sqeMirrorBufferAddr;
4244 :
4245 0 : s32 taskNum = sqeHeader->taskId - taskId;
4246 0 : HCCL_DEBUG("[HcclCommAicpu]tail sqe taskId[%u] cqe taskId[%u] cqe type[%u]", sqeHeader->taskId, taskId, sqeType);
4247 0 : s32 sqeIdx = tail - taskNum - 1;
4248 0 : u32 sqHead = (sqeIdx + HCCL_SQE_MAX_CNT) % HCCL_SQE_MAX_CNT;
4249 0 : uint8_t type = 0;
4250 0 : uint16_t taskIdTmp = 0;
4251 0 : uint32_t remoteRank = 0;
4252 0 : HCCL_ERROR(
4253 : "[TaskException][AICPU]base information is streamId:%d, sqid:%d, head:%u, tail:%u, %s", stream.id(),
4254 : stream.sqId(), sqHead, tail,
4255 : GetTaskExceptionTaskInfo(sqHead, sqeContextBuffer, type, taskIdTmp, remoteRank).c_str());
4256 0 : PrintTaskExceptionTaskQue(sqHead, sqeContextBuffer);
4257 0 : return HCCL_SUCCESS;
4258 : }
4259 :
4260 0 : std::string HcclCommAicpu::GetTaskExceptionOpInfo(u32 idx, SqeRingBuffer* sqeContextBuffer)
4261 : {
4262 0 : const AicpuOpInfo* opInfo = aicpuShareData_.GetAicpuOpInfo(sqeContextBuffer->rtsDfxInfo[idx].opRingBufferIdx);
4263 0 : CHK_PRT_RET(opInfo == nullptr, HCCL_ERROR("%s fail, opInfo is nullptr", __func__), "unKnown");
4264 :
4265 0 : std::stringstream ss;
4266 0 : ss << "tag:" << opInfo->tagBuff << ", ";
4267 0 : ss << "group:" << identifier_ << ", ";
4268 0 : ss << "isCustom:" << opInfo->isCustom << ", ";
4269 0 : ss << "opLaunchIdx:" << opInfo->opIndex << ", ";
4270 0 : ss << "opExecIdx:" << opInfo->opExecIndex << ", ";
4271 0 : ss << "count:" << opInfo->count << ", ";
4272 0 : ss << "dataType:" << static_cast<u16>(opInfo->dataType) << ", ";
4273 0 : ss << "opType:" << static_cast<u16>(opInfo->opType) << ", ";
4274 0 : ss << "rootId:" << opInfo->rootId << ", ";
4275 0 : ss << "dstAddr:0x" << std::hex << opInfo->dstAddr << ", ";
4276 0 : ss << "srcAddr:0x" << std::hex << opInfo->srcAddr << ".";
4277 0 : return ss.str();
4278 0 : }
4279 :
4280 0 : std::string HcclCommAicpu::GetTaskExceptionTaskInfo(
4281 : u32 sqHead, SqeRingBuffer* sqeContextBuffer, uint8_t& type, uint16_t& taskId, uint32_t& remoteRank)
4282 : {
4283 0 : SqeInfo sqeInfo;
4284 0 : SqeContextUtils::QuerySqeInfo(
4285 0 : sqeContextBuffer->rtsMirrorBuffer + sqHead * HCCL_SQE_SIZE, sqeContextBuffer->rtsqSqeType[sqHead],
4286 : sqeContextBuffer->addInfo[sqHead], &sqeInfo);
4287 0 : type = sqeInfo.type;
4288 0 : taskId = sqeInfo.taskId;
4289 0 : remoteRank = sqeContextBuffer->rtsDfxInfo[sqHead].remoteRank;
4290 0 : std::stringstream ss;
4291 0 : ss << "type:" << SqeContextUtils::RtsqTaskTypeToStr(sqeInfo.type) << ", ";
4292 0 : ss << "localRank:" << localUserRank_ << ", ";
4293 0 : ss << "remoteRank:" << remoteRank << ", ";
4294 0 : ss << "taskId:" << sqeInfo.taskId << ", ";
4295 0 : ss << "notifyId:" << sqeInfo.notifyId << ", ";
4296 0 : ss << "length:" << sqeInfo.length << ", ";
4297 0 : ss << "addr1High:0x" << std::hex << sqeInfo.addr1High << ", ";
4298 0 : ss << "addr1Low:0x" << std::hex << sqeInfo.addr1Low << ", ";
4299 0 : ss << "addr2High:0x" << std::hex << sqeInfo.addr2High << ", ";
4300 0 : ss << "addr2Low:0x" << std::hex << sqeInfo.addr2Low << ".";
4301 0 : return ss.str();
4302 0 : }
4303 :
4304 : // 获取指定索引的算子信息
4305 0 : const AicpuOpInfo* HcclCommAicpu::GetOpInfoFromSqIdx(u32 sqIdx, SqeRingBuffer* sqeContextBuffer)
4306 : {
4307 0 : return aicpuShareData_.GetAicpuOpInfo(sqeContextBuffer->rtsDfxInfo[sqIdx].opRingBufferIdx);
4308 : }
4309 :
4310 : // 打印算子数据信息
4311 0 : void HcclCommAicpu::PrintOpDataInfo(u32 sqIdx, SqeRingBuffer* sqeContextBuffer, bool isMonitor)
4312 : {
4313 0 : std::string opInfo = GetTaskExceptionOpInfo(sqIdx, sqeContextBuffer);
4314 0 : if (isMonitor) {
4315 0 : HCCL_RUN_INFO("[StreamTaskMonitor]opData information is %s", opInfo.c_str());
4316 : } else {
4317 0 : HCCL_ERROR("[TaskException]opData information is %s", opInfo.c_str());
4318 : }
4319 0 : }
4320 :
4321 : // 打印task序列行
4322 0 : void HcclCommAicpu::PrintTaskLine(bool isMonitor, u32 lineNum, u32 totalPrinted, const std::string& taskLine) const
4323 : {
4324 0 : if (isMonitor) {
4325 0 : HCCL_RUN_INFO("[StreamTaskMonitor]task sequence[%u/%u] is %s", lineNum, totalPrinted, taskLine.c_str());
4326 : } else {
4327 0 : HCCL_ERROR("[TaskException]task sequence[%u/%u] is %s", lineNum, totalPrinted, taskLine.c_str());
4328 : }
4329 0 : }
4330 :
4331 : // 更新算子上下文
4332 0 : void HcclCommAicpu::UpdateOpContext(
4333 : u32& opIndex, std::string& opTag, u32& lineCount, std::vector<std::string>& currentOpTasks, u32 newOpIndex,
4334 : const std::string& newOpTag) const
4335 : {
4336 0 : opIndex = newOpIndex;
4337 0 : opTag = newOpTag;
4338 0 : lineCount = 0;
4339 0 : currentOpTasks.clear();
4340 0 : currentOpTasks.push_back("OP(" + std::to_string(opIndex) + ")");
4341 0 : }
4342 :
4343 : // 准备下一行数据
4344 0 : void HcclCommAicpu::PrepareNextLine(u32 opIndex, u32& lineCount, std::vector<std::string>& currentOpTasks) const
4345 : {
4346 0 : lineCount++;
4347 0 : currentOpTasks.clear();
4348 0 : currentOpTasks.push_back("OP(" + std::to_string(opIndex) + ") continued");
4349 0 : }
4350 :
4351 : // 拼接task列表为字符串
4352 0 : std::string HcclCommAicpu::ConcatTaskLine(const std::vector<std::string>& tasks) const
4353 : {
4354 0 : std::string taskLine;
4355 0 : for (const auto& task : tasks) {
4356 0 : taskLine += task;
4357 : }
4358 0 : return taskLine;
4359 0 : }
4360 :
4361 : // 打印剩余未满行的tasks
4362 0 : void HcclCommAicpu::PrintRemainingTasks(
4363 : bool isMonitor, u32 lineCount, u32 printedCount, const std::vector<std::string>& currentOpTasks) const
4364 : {
4365 : // size > 1 表示除了OP头还有task
4366 0 : if (currentOpTasks.size() > 1) {
4367 0 : std::string taskLine = ConcatTaskLine(currentOpTasks);
4368 0 : PrintTaskLine(isMonitor, lineCount + 1, printedCount, taskLine);
4369 0 : }
4370 0 : }
4371 :
4372 0 : void HcclCommAicpu::PrintTaskExceptionTaskQue(u32 sqIdx, SqeRingBuffer* sqeContextBuffer, bool isMonitor)
4373 : {
4374 0 : const u32 sqeNum = 200;
4375 0 : const u32 maxTasksPerLine = 50;
4376 :
4377 : // 获取初始算子信息
4378 0 : const AicpuOpInfo* lastOpInfo = GetOpInfoFromSqIdx(sqIdx, sqeContextBuffer);
4379 0 : CHK_PRT_RET(lastOpInfo == nullptr, HCCL_ERROR("%s fail, opInfo is nullptr", __func__), );
4380 :
4381 : // 初始化状态
4382 0 : u32 opIndex = lastOpInfo->opIndex;
4383 0 : std::string opTag = lastOpInfo->tagBuff;
4384 0 : u32 lineCount = 0;
4385 0 : u32 printedCount = 0;
4386 0 : u32 extraSlotPerLine = 2;
4387 0 : std::vector<std::string> currentOpTasks;
4388 0 : currentOpTasks.push_back("OP(" + std::to_string(opIndex) + ")");
4389 :
4390 : // 主循环：遍历并打印
4391 0 : for (u32 i = 0; i < sqeNum; i++) {
4392 : // 计算新索引并获取算子信息
4393 0 : u32 newSqIdx = (sqIdx - i + HCCL_SQE_MAX_CNT) % HCCL_SQE_MAX_CNT;
4394 0 : const AicpuOpInfo* newOpInfo = GetOpInfoFromSqIdx(newSqIdx, sqeContextBuffer);
4395 0 : CHK_PRT_RET(newOpInfo == nullptr, HCCL_ERROR("%s fail, opInfo is nullptr", __func__), );
4396 :
4397 0 : u32 sizeAfterAdd = currentOpTasks.size() + 1;
4398 0 : if (sizeAfterAdd >= (maxTasksPerLine + extraSlotPerLine)
4399 0 : || (i > 0 && (newOpInfo->opIndex != opIndex || newOpInfo->tagBuff != opTag))) {
4400 : // 打印算子信息（仅第一次）
4401 0 : if (lineCount == 0) {
4402 0 : PrintOpDataInfo(sqIdx, sqeContextBuffer, isMonitor);
4403 : }
4404 :
4405 : // 拼接并打印task行
4406 0 : std::string taskLine = ConcatTaskLine(currentOpTasks);
4407 0 : PrintTaskLine(isMonitor, lineCount + 1, printedCount, taskLine);
4408 :
4409 : // 更新状态
4410 0 : if (newOpInfo->opIndex != opIndex || newOpInfo->tagBuff != opTag) {
4411 0 : UpdateOpContext(opIndex, opTag, lineCount, currentOpTasks, newOpInfo->opIndex, newOpInfo->tagBuff);
4412 : } else {
4413 0 : PrepareNextLine(opIndex, lineCount, currentOpTasks);
4414 : }
4415 0 : }
4416 : // 添加task到序列
4417 0 : currentOpTasks.push_back("," + GetTaskBriefsInfo(newSqIdx, sqeContextBuffer));
4418 0 : printedCount++;
4419 : }
4420 : // 打印剩余tasks
4421 0 : PrintRemainingTasks(isMonitor, lineCount, printedCount, currentOpTasks);
4422 0 : }
4423 :
4424 0 : std::string HcclCommAicpu::GetTaskBriefsInfo(u32 idx, SqeRingBuffer* sqeContextBuffer)
4425 : {
4426 0 : uint8_t* sqeMirrorBufferAddr = sqeContextBuffer->rtsMirrorBuffer + idx * HCCL_SQE_SIZE;
4427 0 : rtStarsSqeHeader_t* const sqeHeader = (rtStarsSqeHeader_t* const)sqeMirrorBufferAddr;
4428 0 : uint8_t sqeType = sqeHeader->type;
4429 :
4430 0 : SqeInfo sqeInfo;
4431 0 : SqeContextUtils::QuerySqeInfo(
4432 0 : sqeContextBuffer->rtsMirrorBuffer + idx * HCCL_SQE_SIZE, sqeContextBuffer->rtsqSqeType[idx],
4433 : sqeContextBuffer->addInfo[idx], &sqeInfo);
4434 0 : uint8_t subType = sqeInfo.subType;
4435 :
4436 0 : std::stringstream ss;
4437 0 : std::string taskName = "UN";
4438 0 : switch (sqeType) {
4439 0 : case RT_STARS_SQE_TYPE_NOTIFY_RECORD:
4440 0 : taskName = "NR"; // Notify Record
4441 0 : break;
4442 0 : case RT_STARS_SQE_TYPE_WRITE_VALUE:
4443 0 : if (subType == RT_STARS_WRITE_VALUE_SUB_TYPE_NOTIFY_RECORD_IPC_NO_PCIE) {
4444 0 : taskName = "NR";
4445 0 : } else if (subType == RT_STARS_WRITE_VALUE_SUB_TYPE_EVENT_RESET) {
4446 0 : taskName = "NW"; // Notify Wait
4447 0 : } else if (subType == RT_STARS_WRITE_VALUE_SUB_TYPE_RDMA_DB_SEND) {
4448 0 : taskName = "RS"; // Rdma Send
4449 : }
4450 0 : break;
4451 0 : case RT_STARS_SQE_TYPE_NOTIFY_WAIT:
4452 0 : taskName = "NW";
4453 0 : break;
4454 0 : case RT_STARS_SQE_TYPE_EVENT_WAIT:
4455 0 : taskName = "NW";
4456 0 : break;
4457 0 : case RT_STARS_SQE_TYPE_SDMA:
4458 0 : taskName = "SD"; // SDMA
4459 0 : break;
4460 0 : case RT_STARS_SQE_TYPE_COND:
4461 0 : taskName = "CO";
4462 0 : break;
4463 0 : case RT_STARS_SQE_TYPE_PLACE_HOLDER:
4464 0 : taskName = "PH";
4465 0 : break;
4466 0 : default:
4467 0 : taskName = std::to_string(sqeType);
4468 0 : break;
4469 : }
4470 :
4471 0 : ss << taskName << "(";
4472 0 : if (sqeContextBuffer->rtsDfxInfo[idx].remoteRank != INVALID_VALUE_RANKID) {
4473 0 : ss << sqeContextBuffer->rtsDfxInfo[idx].remoteRank;
4474 : } else {
4475 0 : ss << "/";
4476 : }
4477 0 : ss << ",";
4478 0 : if (sqeContextBuffer->rtsDfxInfo[idx].notifyId != INVALID_VALUE_RANKID) {
4479 0 : ss << sqeContextBuffer->rtsDfxInfo[idx].notifyId;
4480 : } else {
4481 0 : ss << "/";
4482 : }
4483 0 : ss << ")";
4484 0 : return ss.str();
4485 0 : }
4486 :
4487 1 : void HcclCommAicpu::RecordReportStatus(dfx::ReportStatus status)
4488 : {
4489 1 : std::unique_lock<std::mutex> lock(reportQueueMutex_);
4490 1 : while (reportStatusQueue_.size() >= MAX_REPORT_STATUS) {
4491 0 : HCCL_WARNING(
4492 : "[HcclCommAicpu][RecordReportStatus] retry status queue reach the limit[%u], "
4493 : "the front status[%u] is dropped.",
4494 : MAX_REPORT_STATUS, reportStatusQueue_.front());
4495 0 : reportStatusQueue_.pop();
4496 : }
4497 1 : reportStatusQueue_.push(status);
4498 1 : HCCL_INFO("[HcclCommAicpu][RecordReportStatus]push[%u], retry queue size()[%u]", status, reportStatusQueue_.size());
4499 1 : }
4500 :
4501 0 : void HcclCommAicpu::GetReportStatusQueue(std::queue<dfx::ReportStatus>& reportStatusQue)
4502 : {
4503 0 : std::unique_lock<std::mutex> lock(reportQueueMutex_);
4504 0 : std::swap(reportStatusQueue_, reportStatusQue);
4505 0 : }
4506 :
4507 0 : void HcclCommAicpu::SetStreamCqeExceptionStatus(const Stream& stream, CqeExceptionStatus cqeStatus)
4508 : {
4509 0 : HCCL_RUN_INFO("SetStreamCqeExceptionStatus: stream sq id %u, cqe exception %u", stream.sqId(), cqeStatus);
4510 0 : auto iter = streamCqeExceptionStatus_.find(stream.sqId());
4511 0 : if (iter != streamCqeExceptionStatus_.end()) {
4512 0 : iter->second = cqeStatus;
4513 : } else {
4514 0 : streamCqeExceptionStatus_.insert({stream.sqId(), cqeStatus});
4515 : }
4516 0 : return;
4517 : }
4518 :
4519 0 : CqeExceptionStatus HcclCommAicpu::GetStreamCqeExceptionStatus(const Stream& stream)
4520 : {
4521 0 : auto iter = streamCqeExceptionStatus_.find(stream.sqId());
4522 0 : if (iter != streamCqeExceptionStatus_.end()) {
4523 0 : return iter->second;
4524 : } else {
4525 0 : return CqeExceptionStatus::kNone;
4526 : }
4527 : }
4528 :
4529 0 : void HcclCommAicpu::ResetStreamCqeExceptionStatus(const Stream& stream)
4530 : {
4531 0 : auto iter = streamCqeExceptionStatus_.find(stream.sqId());
4532 0 : if (iter != streamCqeExceptionStatus_.end()) {
4533 0 : iter->second = CqeExceptionStatus::kNone;
4534 : }
4535 0 : HCCL_INFO("ResetStreamCqeExceptionStatus: stream sq id %u", stream.sqId());
4536 0 : return;
4537 : }
4538 :
4539 0 : void HcclCommAicpu::SetBSRSendOpExecException()
4540 : {
4541 0 : HCCL_INFO("set send stream exec exception");
4542 0 : bsrSendOpExecException_ = true;
4543 0 : return;
4544 : }
4545 :
4546 0 : void HcclCommAicpu::SetBSRRecvOpExecException()
4547 : {
4548 0 : HCCL_INFO("set recv stream exec exception");
4549 0 : bsrRecvOpExecException_ = true;
4550 0 : return;
4551 : }
4552 :
4553 0 : bool HcclCommAicpu::GetBSRSendOpExecException()
4554 : {
4555 0 : bool ret = (GetStreamCqeExceptionStatus(bsrSendStream_) == CqeExceptionStatus::kSdmaErr) || bsrSendOpExecException_;
4556 0 : HCCL_INFO(
4557 : "GetBSRSendOpExecException: stream %u cqe status %u, send exec status %u", bsrSendStream_.sqId(),
4558 : GetStreamCqeExceptionStatus(bsrSendStream_), bsrSendOpExecException_);
4559 0 : return ret;
4560 : }
4561 :
4562 0 : bool HcclCommAicpu::GetBSRRecvOpExecException()
4563 : {
4564 0 : HCCL_INFO(
4565 : "GetBSRRecvOpExecException: stream %u cqe status %u, recv exec status %u", bsrRecvStream_.sqId(),
4566 : GetStreamCqeExceptionStatus(bsrRecvStream_), bsrRecvOpExecException_);
4567 0 : return (GetStreamCqeExceptionStatus(bsrRecvStream_) == CqeExceptionStatus::kSdmaErr) || bsrRecvOpExecException_;
4568 : }
4569 :
4570 0 : HcclResult HcclCommAicpu::CleanStream(Stream& stream)
4571 : {
4572 0 : CHK_RET(stream.ClearLocalBuff());
4573 0 : CHK_RET(UpdateSqStatus(stream));
4574 0 : ResetStreamCqeExceptionStatus(stream);
4575 0 : HCCL_INFO("CleanStream %u success.", stream.sqId());
4576 0 : return HCCL_SUCCESS;
4577 : }
4578 :
4579 0 : HcclResult HcclCommAicpu::ClearStreamCqeException(Stream& stream)
4580 : {
4581 0 : HandleCqeException(stream, true);
4582 0 : return HCCL_SUCCESS;
4583 : }
4584 :
4585 0 : HcclResult HcclCommAicpu::ResetBSRSendOpExecException()
4586 : {
4587 0 : bsrSendOpExecException_ = false;
4588 0 : CHK_RET(CleanStream(bsrSendStream_));
4589 0 : CHK_RET(ClearStreamCqeException(bsrSendStream_));
4590 0 : HCCL_INFO("ResetBSRSendOpExecException success.");
4591 0 : return HCCL_SUCCESS;
4592 : }
4593 :
4594 0 : HcclResult HcclCommAicpu::ResetBSRRecvOpExecException()
4595 : {
4596 0 : bsrRecvOpExecException_ = false;
4597 0 : CHK_RET(CleanStream(bsrRecvStream_));
4598 0 : CHK_RET(ClearStreamCqeException(bsrRecvStream_));
4599 0 : HCCL_INFO("ResetBSRRecvOpExecException success.");
4600 0 : return HCCL_SUCCESS;
4601 : }
4602 :
4603 0 : HcclResult HcclCommAicpu::ResetBSRException()
4604 : {
4605 0 : if (bsrRetryOp_ == HCCL_SEND) {
4606 0 : CHK_RET(ResetBSRSendOpExecException());
4607 0 : HCCL_INFO("reset batchsendrecv exception success, tag:%s, index:%u", bsrSendOpId_.tag, bsrSendOpId_.index);
4608 0 : } else if (bsrRetryOp_ == HCCL_RECV) {
4609 0 : CHK_RET(ResetBSRRecvOpExecException());
4610 0 : HCCL_INFO("reset batchsendrecv exception success, tag:%s, index:%u", bsrRecvOpId_.tag, bsrRecvOpId_.index);
4611 : } else {
4612 0 : HCCL_INFO("reset batchsendrecv exception success, tag:%s", bsrTargetOpId_.tag);
4613 : }
4614 0 : return HCCL_SUCCESS;
4615 : }
4616 :
4617 0 : void HcclCommAicpu::UpdateBSRRetryCnt()
4618 : {
4619 0 : if (bsrRetryOp_ == HCCL_SEND) {
4620 0 : bsrSendRetryCnt_++;
4621 : } else {
4622 0 : bsrRecvRetryCnt_++;
4623 : }
4624 0 : HCCL_INFO("UpdateBSRRetryCnt, SendCnt[%u], RecvCnt[%u]", bsrRecvRetryCnt_, bsrRecvRetryCnt_);
4625 0 : return;
4626 : }
4627 :
4628 0 : void HcclCommAicpu::ResetBSRRetryCnt()
4629 : {
4630 0 : bsrSendRetryCnt_ = 0;
4631 0 : bsrRecvRetryCnt_ = 0;
4632 0 : return;
4633 : }
4634 :
4635 0 : void HcclCommAicpu::InitSendRecvOpId(const OpParam& param, HcclOpIdentifier& opId)
4636 : {
4637 : // send算子入参中只有dst对端rank号，而recv算子入参中只有src源端rank号
4638 0 : if (param.opType == HcclCMDType::HCCL_CMD_SEND) {
4639 0 : opId.detRank = param.dstRank;
4640 0 : opId.srcRank = topoInfo_.userRank;
4641 : } else {
4642 0 : opId.srcRank = param.srcRank;
4643 0 : opId.detRank = topoInfo_.userRank;
4644 : }
4645 0 : opId.isSendRecv = true;
4646 0 : HCCL_DEBUG(
4647 : "[HcclCommAicpu][InitSendRecvOpId]src=[%u] dst=[%u] isSendRecv=[%u]", opId.srcRank, opId.detRank,
4648 : opId.isSendRecv);
4649 0 : return;
4650 : }
4651 :
4652 0 : u32 HcclCommAicpu::HcclUpdateBatchSendRecvOpIndex(std::map<u32, u32>& bsrIndexMap, u32 peerRank)
4653 : {
4654 0 : u32 ret = 0;
4655 0 : auto opIndexMapIter = bsrIndexMap.find(peerRank);
4656 0 : if (opIndexMapIter != bsrIndexMap.end()) {
4657 0 : (opIndexMapIter->second)++;
4658 0 : ret = opIndexMapIter->second;
4659 : } else {
4660 0 : bsrIndexMap.insert({peerRank, 1});
4661 0 : ret = 1;
4662 : }
4663 0 : return ret;
4664 : }
4665 :
4666 0 : u32 HcclCommAicpu::HcclUpdateBatchSendRecvOpIndex(HcclSendRecvType opType, u32 srcRank, u32 dstRank)
4667 : {
4668 0 : u32 peerRank = (opType == HcclSendRecvType::HCCL_SEND) ? dstRank : srcRank;
4669 0 : auto& opIndexMap = (opType == HcclSendRecvType::HCCL_SEND) ? bsrSendIndexMap_ : bsrRecvIndexMap_;
4670 :
4671 0 : return HcclUpdateBatchSendRecvOpIndex(opIndexMap, peerRank);
4672 : }
4673 : HcclResult
4674 0 : HcclCommAicpu::GetBsrTransportQpn(const HcclSendRecvItem* sendrecvPair, AlgResourceResponse& algResource, u32& qpn)
4675 : {
4676 0 : CHK_PTR_NULL(sendrecvPair);
4677 0 : LINK targetLink;
4678 0 : u32 commIndex = 0;
4679 0 : u32 remoteRank = sendrecvPair->remoteRank;
4680 0 : u32 localRank = topoInfo_.userRank;
4681 0 : HcclSendRecvType sendRecvType = sendrecvPair->sendRecvType;
4682 0 : HCCL_DEBUG(
4683 : "[GetBsrTransportQpn] bsrOptype =[%d], localRank=[%u] remoteRank=[%u]", sendRecvType, localRank, remoteRank);
4684 :
4685 0 : if ((sendRecvType == HcclSendRecvType::HCCL_SEND && remoteRank < localRank)
4686 0 : || (sendRecvType == HcclSendRecvType::HCCL_RECV && remoteRank > localRank)) {
4687 0 : commIndex = COMM_INDEX_0;
4688 : } else {
4689 0 : commIndex = COMM_INDEX_1;
4690 : }
4691 0 : CHK_PRT_RET(
4692 : commIndex >= algResource.opTransportResponse[COMM_COMBINE_ORDER].size(),
4693 : HCCL_ERROR(
4694 : "[GetBsrTransportQpn] batchsendrecv op commIndex[%u] is larger than "
4695 : "opTransportResponse size[%zu]",
4696 : commIndex, algResource.opTransportResponse[COMM_COMBINE_ORDER].size()),
4697 : HCCL_E_PARA);
4698 : SingleSubCommTransport& commCombined
4699 0 : = static_cast<SingleSubCommTransport&>(algResource.opTransportResponse[COMM_COMBINE_ORDER][commIndex]);
4700 :
4701 0 : CHK_PRT_RET(
4702 : sendrecvPair->remoteRank >= commCombined.userRank2subCommRank.size(),
4703 : HCCL_ERROR(
4704 : "[GetBsrTransportQpn]batchsendrecv op remoteUserRank[%u] is larger than "
4705 : "userRank2subCommRank map size[%zu]",
4706 : sendrecvPair->remoteRank, commCombined.userRank2subCommRank.size()),
4707 : HCCL_E_PARA);
4708 :
4709 0 : u32 rank = commCombined.userRank2subCommRank[sendrecvPair->remoteRank];
4710 0 : CHK_PRT_RET(
4711 : rank >= commCombined.links.size(),
4712 : HCCL_ERROR(
4713 : "[GetBsrTransportQpn] batchsendrecv op remoteUserRank[%u], get rank[%u],"
4714 : "the size of combinedComm links is [%zu]",
4715 : sendrecvPair->remoteRank, rank, commCombined.links.size()),
4716 : HCCL_E_PARA);
4717 0 : targetLink = commCombined.links[rank];
4718 :
4719 0 : CHK_SMART_PTR_NULL(targetLink);
4720 0 : if (targetLink->GetLinkType() == LinkType::LINK_ROCE) {
4721 0 : CHK_RET(targetLink->GetTransportId(qpn));
4722 : }
4723 0 : HCCL_DEBUG(
4724 : "[HcclCommAicpu][GetBsrTransportQpn] localrank=[%u] remoteuserRank=[%u] remoteRank=[%u],sendrecvType=[%d] qpn "
4725 : "=[%u], comindex [%u]",
4726 : topoInfo_.userRank, sendrecvPair->remoteRank, remoteRank, sendrecvPair->sendRecvType, qpn, commIndex);
4727 0 : return HCCL_SUCCESS;
4728 0 : }
4729 0 : HcclResult HcclCommAicpu::InitBatchSendRecvOpId(
4730 : const OpParam& param, const HcclSendRecvItem* sendrecvPair, HcclOpIdentifier& opId, u32 streamId,
4731 : AlgResourceResponse& algResource)
4732 : {
4733 0 : CHK_PTR_NULL(sendrecvPair);
4734 0 : if (sendrecvPair->sendRecvType == HcclSendRecvType::HCCL_RECV) {
4735 0 : opId.srcRank = sendrecvPair->remoteRank;
4736 0 : opId.detRank = topoInfo_.userRank;
4737 : } else {
4738 0 : opId.srcRank = topoInfo_.userRank;
4739 0 : opId.detRank = sendrecvPair->remoteRank;
4740 : }
4741 0 : opId.index = HcclUpdateBatchSendRecvOpIndex(sendrecvPair->sendRecvType, opId.srcRank, opId.detRank);
4742 0 : opId.isSendRecv = true;
4743 0 : opId.opType = HcclCMDType::HCCL_CMD_BATCH_SEND_RECV;
4744 0 : std::string sendrecvTag = param.tag + "_BSR_" + std::to_string(opId.srcRank) + "_" + std::to_string(opId.detRank);
4745 0 : CHK_SAFETY_FUNC_RET(memcpy_s(opId.tag, sizeof(opId.tag), sendrecvTag.c_str(), sendrecvTag.size()));
4746 0 : std::string sendrecvNewTag = param.tag + "_device";
4747 0 : CHK_SAFETY_FUNC_RET(memcpy_s(opId.newTag, sizeof(opId.newTag), sendrecvNewTag.c_str(), sendrecvNewTag.size()));
4748 :
4749 0 : u32 qpn = 0;
4750 0 : if ((opId.srcRank != opId.detRank) && (!param.BatchSendRecvDataDes.isDirectRemoteRank[sendrecvPair->remoteRank])) {
4751 0 : CHK_RET(GetBsrTransportQpn(sendrecvPair, algResource, qpn));
4752 : }
4753 :
4754 0 : auto& bsrinfo = opId.bsrInfo[sendrecvPair->sendRecvType];
4755 0 : bsrinfo.detRank = opId.detRank;
4756 0 : bsrinfo.srcRank = opId.srcRank;
4757 0 : bsrinfo.index = opId.index;
4758 0 : bsrinfo.streamId = streamId;
4759 0 : bsrinfo.tpQpn = qpn;
4760 0 : CHK_SAFETY_FUNC_RET(memcpy_s(bsrinfo.bsrTag, sizeof(bsrinfo.bsrTag), sendrecvTag.c_str(), sendrecvTag.size()));
4761 :
4762 0 : HCCL_INFO(
4763 : "[HcclCommAicpu][InitBatchSendRecvOpId] tag=[%s] index=[%u] src=[%u] det=[%u] qpn =[%u]", opId.tag, opId.index,
4764 : opId.srcRank, opId.detRank, qpn);
4765 0 : return HCCL_SUCCESS;
4766 0 : }
4767 :
4768 0 : HcclResult HcclCommAicpu::InitBatchSendRecvOpId(const OpParam& param, AlgResourceResponse& algResource)
4769 : {
4770 0 : CHK_SAFETY_FUNC_RET(
4771 : memset_s(reinterpret_cast<void*>(&bsrSendOpId_), sizeof(bsrSendOpId_), 0, sizeof(bsrSendOpId_)));
4772 0 : CHK_SAFETY_FUNC_RET(
4773 : memset_s(reinterpret_cast<void*>(&bsrRecvOpId_), sizeof(bsrRecvOpId_), 0, sizeof(bsrRecvOpId_)));
4774 :
4775 0 : CHK_PRT_RET(
4776 : (algResource.slaveStreams.size() < BSR_RETRY_STREAM_NUM),
4777 : HCCL_ERROR("in batchsendrecv op, slave stream is not enough."), HCCL_E_INTERNAL);
4778 0 : bsrSendOpId_.streamId = algResource.slaveStreams[BSR_RETRY_SEND_STREAM_INDEX].id();
4779 0 : bsrRecvOpId_.streamId = algResource.slaveStreams[BSR_RETRY_RECV_STREAM_INDEX].id();
4780 :
4781 0 : u32 iter = param.BatchSendRecvDataDes.curIterNum;
4782 0 : std::vector<std::vector<HcclSendRecvItem*>>& pairs = bsrSendRecvPairs_;
4783 0 : CHK_PRT_RET(
4784 : (pairs.size() <= iter),
4785 : HCCL_ERROR("batchsendrecv sendrecv pairs size[%u] less than or equal to curiter[%u]", pairs.size(), iter),
4786 : HCCL_E_INTERNAL);
4787 :
4788 0 : for (auto& pair : pairs[iter]) {
4789 0 : HcclOpIdentifier& opId = (pair->sendRecvType == HcclSendRecvType::HCCL_SEND) ? bsrSendOpId_ : bsrRecvOpId_;
4790 0 : auto streamId
4791 0 : = (pair->sendRecvType == HcclSendRecvType::HCCL_SEND) ? bsrSendOpId_.streamId : bsrRecvOpId_.streamId;
4792 0 : CHK_RET(InitBatchSendRecvOpId(param, pair, opId, streamId, algResource));
4793 : }
4794 :
4795 : // 补全batchsendrecv中sendrecv的bsrInfo, 两边都发生故障的时候要用
4796 0 : bsrSendOpId_.bsrInfo[HCCL_RECV].index = bsrRecvOpId_.index;
4797 0 : bsrSendOpId_.bsrInfo[HCCL_RECV].streamId = bsrRecvOpId_.streamId;
4798 0 : bsrSendOpId_.bsrInfo[HCCL_RECV].srcRank = bsrRecvOpId_.srcRank;
4799 0 : bsrSendOpId_.bsrInfo[HCCL_RECV].detRank = bsrRecvOpId_.detRank;
4800 0 : bsrSendOpId_.bsrInfo[HCCL_RECV].tpQpn = bsrRecvOpId_.bsrInfo[HCCL_RECV].tpQpn;
4801 0 : CHK_SAFETY_FUNC_RET(memcpy_s(
4802 : bsrSendOpId_.bsrInfo[HCCL_RECV].bsrTag, sizeof(bsrSendOpId_.bsrInfo[HCCL_RECV].bsrTag), bsrRecvOpId_.tag,
4803 : sizeof(bsrRecvOpId_.tag)));
4804 :
4805 0 : bsrRecvOpId_.bsrInfo[HCCL_SEND].index = bsrSendOpId_.index;
4806 0 : bsrRecvOpId_.bsrInfo[HCCL_SEND].streamId = bsrSendOpId_.streamId;
4807 0 : bsrRecvOpId_.bsrInfo[HCCL_SEND].srcRank = bsrSendOpId_.srcRank;
4808 0 : bsrRecvOpId_.bsrInfo[HCCL_SEND].detRank = bsrSendOpId_.detRank;
4809 0 : bsrRecvOpId_.bsrInfo[HCCL_SEND].tpQpn = bsrSendOpId_.bsrInfo[HCCL_SEND].tpQpn;
4810 0 : CHK_SAFETY_FUNC_RET(memcpy_s(
4811 : bsrRecvOpId_.bsrInfo[HCCL_SEND].bsrTag, sizeof(bsrRecvOpId_.bsrInfo[HCCL_SEND].bsrTag), bsrSendOpId_.tag,
4812 : sizeof(bsrSendOpId_.tag)));
4813 0 : return HCCL_SUCCESS;
4814 : }
4815 :
4816 0 : HcclResult HcclCommAicpu::QueryBatchSendRecvPairBeginPos()
4817 : {
4818 : // 前面已经生成 batchsendrecv 的 send & recv opid 时已经校验过slave stream num数，此处不再重复校验
4819 0 : CHK_RET(QuerySqStatusByType(devId_, bsrSendStream_.sqId(), DRV_SQCQ_PROP_SQ_TAIL, bsrSendOpBeginSqePos_));
4820 0 : CHK_RET(QuerySqStatusByType(devId_, bsrRecvStream_.sqId(), DRV_SQCQ_PROP_SQ_TAIL, bsrRecvOpBeginSqePos_));
4821 :
4822 0 : HCCL_INFO(
4823 : "QueryBatchSendRecvPairBeginPos send sqePos[%u] recv sqePos[%u]", bsrSendOpBeginSqePos_, bsrRecvOpBeginSqePos_);
4824 0 : return HCCL_SUCCESS;
4825 : }
4826 :
4827 0 : HcclResult HcclCommAicpu::QueryBatchSendRecvPairEndPos()
4828 : {
4829 : // 前面已经生成 batchsendrecv 的 send & recv opid 时已经校验过slave stream num数，此处不再重复校验
4830 0 : CHK_RET(QuerySqStatusByType(devId_, bsrSendStream_.sqId(), DRV_SQCQ_PROP_SQ_TAIL, bsrSendOpEndSqePos_));
4831 0 : CHK_RET(QuerySqStatusByType(devId_, bsrRecvStream_.sqId(), DRV_SQCQ_PROP_SQ_TAIL, bsrRecvOpEndSqePos_));
4832 :
4833 0 : HCCL_INFO("QueryBatchSendRecvPairEndPos send sqePos[%u] recv sqePos[%u]", bsrSendOpEndSqePos_, bsrRecvOpEndSqePos_);
4834 0 : return HCCL_SUCCESS;
4835 : }
4836 :
4837 0 : HcclResult HcclCommAicpu::CommitBSRStoredException(HcclOpExecFSM& fsmState, KfcError& errorCode)
4838 : {
4839 0 : if (GetBSRSendOpExecException()) {
4840 0 : bsrRetryOp_ = HCCL_SEND;
4841 0 : errorCode = KfcError::kSdma;
4842 0 : HCCL_INFO("CommitBSRStoredException: send stream remain retry error.");
4843 0 : } else if (GetBSRRecvOpExecException()) {
4844 0 : bsrRetryOp_ = HCCL_RECV;
4845 0 : errorCode = KfcError::kSdma;
4846 0 : HCCL_INFO("CommitBSRStoredException: recv stream remain retry error.");
4847 : }
4848 :
4849 0 : u32 retryCnt = (bsrRetryOp_ == HCCL_SEND) ? bsrSendRetryCnt_ : bsrRecvRetryCnt_;
4850 0 : if (errorCode != KfcError::kNone) {
4851 0 : bsrTargetOpId_ = (bsrRetryOp_ == HCCL_SEND) ? bsrSendOpId_ : bsrRecvOpId_;
4852 0 : HCCL_RUN_INFO(
4853 : "CommitBSRStoredException: stored op tag %s index %u , report retry error. curSendRetryCnt[%u],"
4854 : "curRecvRetryCnt[%u]",
4855 : bsrTargetOpId_.tag, bsrTargetOpId_.index, bsrSendRetryCnt_, bsrRecvRetryCnt_);
4856 0 : auto ret = aicpuHdc_.SetOpExecStatus(
4857 0 : kfcStatusTransferD2H_, bsrTargetOpId_, KfcStatus::kStoplaunch, errorCode, retryCnt);
4858 0 : HCCL_RETRY_CHK_RET_AND_TRANS_FSM(
4859 : ret, HCCL_ERROR("SetOpExecStatus failed, ret:%u", ret), KfcError::kExec,
4860 : HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR);
4861 0 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_FSM_STOPPING;
4862 : } else {
4863 0 : CHK_RET(UpdateOpExecStatus(fsmState, KfcStatus::kRuning, errorCode, retryCnt));
4864 0 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_FSM_WAIT_END;
4865 : }
4866 0 : return HCCL_SUCCESS;
4867 : }
4868 :
4869 0 : HcclResult HcclCommAicpu::GetBSRRetryOpId(const OpParam& param, HcclOpIdentifier& targetOpId)
4870 : {
4871 0 : KfcCommand cmd = KfcCommand::kNone;
4872 0 : CHK_RET(aicpuHdc_.GetOpExecCtrlCmd(kfcControlTransferH2D_, cmd));
4873 0 : if (cmd == KfcCommand::kStopLaunch) {
4874 0 : HcclOpIdentifier targetOp;
4875 0 : CHK_RET(aicpuHdc_.GetOpExecCtrlTargetOp(kfcControlTransferH2D_, targetOp));
4876 0 : std::string targetOpTag = std::string(reinterpret_cast<char*>(&targetOp.tag[0]));
4877 0 : if (targetOpTag == std::string(reinterpret_cast<char*>(&bsrSendOpId_.tag[0]))) {
4878 0 : bsrRetryOp_ = HCCL_SEND;
4879 0 : } else if (targetOpTag == std::string(reinterpret_cast<char*>(&bsrRecvOpId_.tag[0]))) {
4880 0 : bsrRetryOp_ = HCCL_RECV;
4881 0 : } else if (targetOpTag == param.tag) {
4882 0 : if (targetOp.detRank == bsrSendOpId_.detRank) {
4883 0 : bsrRetryOp_ = HCCL_SEND;
4884 0 : } else if (targetOp.srcRank == bsrRecvOpId_.srcRank) {
4885 0 : bsrRetryOp_ = HCCL_RECV;
4886 : } else {
4887 0 : HCCL_ERROR(
4888 : "hccl aicpu can not retry, got stop launch command, but target op srcRank[%u] and"
4889 : "dstRank[%u] is not match with send (dst:%u) or recv (src:%u) op",
4890 : targetOp.srcRank, targetOp.detRank, bsrSendOpId_.detRank, bsrRecvOpId_.srcRank);
4891 0 : return HCCL_E_INTERNAL;
4892 : }
4893 : } else {
4894 : // tag 不匹配，报错退出
4895 0 : HCCL_ERROR(
4896 : "hccl aicpu can not retry, got stop launch command, but target op tag[%s] is not match with"
4897 : "send (tag:%s) or recv (tag:%s) or batchsendrecv (tag:%s)",
4898 : targetOpTag.c_str(), bsrSendOpId_.tag, bsrRecvOpId_.tag, param.tag.c_str());
4899 0 : return HCCL_E_INTERNAL;
4900 : }
4901 0 : HCCL_RUN_INFO("hccl aicpu got command %u at op[tag: %s, index: %u].", cmd, targetOpTag.c_str(), targetOp.index);
4902 0 : } else {
4903 0 : if (GetBSRSendOpExecException()) {
4904 0 : bsrRetryOp_ = HCCL_SEND;
4905 0 : } else if (GetBSRRecvOpExecException()) {
4906 0 : bsrRetryOp_ = HCCL_RECV;
4907 : } else {
4908 : // 其他场景，报错退出
4909 0 : HCCL_ERROR("hccl aicpu find task exception, but send and recv op has no exception");
4910 0 : return HCCL_E_INTERNAL;
4911 : }
4912 : }
4913 0 : if (bsrRetryOp_ == HCCL_SEND) {
4914 0 : targetOpId = bsrSendOpId_;
4915 : } else {
4916 0 : targetOpId = bsrRecvOpId_;
4917 : }
4918 0 : HCCL_RUN_INFO(
4919 : "GetBSRRetryOpId: bsrRetryOpType %u, targetOpId: tag %s, index %u", bsrRetryOp_, targetOpId.tag,
4920 : targetOpId.index);
4921 0 : return HCCL_SUCCESS;
4922 : }
4923 :
4924 0 : HcclResult HcclCommAicpu::InitExecLoop(OpParam& param, std::unique_ptr<CollExecutorBase>& executor, u32& loopNum)
4925 : {
4926 0 : if ((param.opType == HcclCMDType::HCCL_CMD_BATCH_SEND_RECV) && retryEnable_) {
4927 0 : CHK_RET(executor->CreatePairWiseList(
4928 : param.BatchSendRecvDataDes.sendRecvItemsPtr, param.BatchSendRecvDataDes.itemNum));
4929 0 : CHK_RET(executor->GetPairWiseList(bsrSendRecvPairs_));
4930 0 : CHK_PRT_RET(
4931 : bsrSendRecvPairs_.empty(), HCCL_ERROR("[HcclCommAicpu][InitExecLoop]batchsendrecv pairs is empty"),
4932 : HCCL_E_INTERNAL);
4933 :
4934 0 : for (size_t i = 0; i < bsrSendRecvPairs_.size(); i++) {
4935 0 : CHK_PRT_RET(
4936 : (bsrSendRecvPairs_[i].size() > BSR_RETRY_SENDRECV_PAIR_NUM_MAX) || bsrSendRecvPairs_[i].empty(),
4937 : HCCL_ERROR("batchsendrecv pairs[%u] size[%u] is out of range [1,2]", i, bsrSendRecvPairs_.size()),
4938 : HCCL_E_INTERNAL);
4939 :
4940 0 : for (size_t j = 0; j < bsrSendRecvPairs_[i].size(); j++) {
4941 0 : CHK_PTR_NULL(bsrSendRecvPairs_[i][j]);
4942 : }
4943 :
4944 0 : CHK_PRT_RET(
4945 : ((bsrSendRecvPairs_[i].size() == BSR_RETRY_SENDRECV_PAIR_NUM_MAX)
4946 : && (bsrSendRecvPairs_[i][BSR_RETRY_SENDRECV_PAIR_INDEX_0]->sendRecvType
4947 : == bsrSendRecvPairs_[i][BSR_RETRY_SENDRECV_PAIR_INDEX_1]->sendRecvType)),
4948 : HCCL_ERROR(
4949 : "batchsendrecv pairs[%u] sendRecvType[%u] is same", i,
4950 : bsrSendRecvPairs_[i][BSR_RETRY_SENDRECV_PAIR_INDEX_0]->sendRecvType),
4951 : HCCL_E_INTERNAL);
4952 : }
4953 :
4954 0 : param.BatchSendRecvDataDes.curIterNum = 0;
4955 0 : loopNum = bsrSendRecvPairs_.size();
4956 0 : } else {
4957 0 : loopNum = 1;
4958 : }
4959 0 : HCCL_INFO("InitExecLoop: execute loop num %u", loopNum);
4960 0 : return HCCL_SUCCESS;
4961 : }
4962 :
4963 1 : HcclResult HcclCommAicpu::ParseHierarchicalAlgOption(u32* ahcConfInfo)
4964 : {
4965 1 : u32 algOptionSize = ahcConfInfo[TOP_HIERARCHICAL_CONF_lENGTH_INDEX];
4966 1 : if (algOptionSize >= (TOP_HIERARCHICAL_CONF_SIZE - 1)) {
4967 1 : HCCL_ERROR(
4968 : "[HcclCommAicpu][ParseHierarchicalAlgOption] hierarchicalAlgOption size[%lu] exceed maxsize[%lu]",
4969 : algOptionSize, (TOP_HIERARCHICAL_CONF_SIZE - 1));
4970 1 : return HCCL_E_INTERNAL;
4971 : }
4972 :
4973 0 : std::map<AHCConcOpType, TemplateType> hierarchicalAlgOption;
4974 0 : for (u32 i = TOP_HIERARCHICAL_CONF_INFO_INDEX; i < (TOP_HIERARCHICAL_CONF_INFO_INDEX + algOptionSize); i++) {
4975 0 : AHCConcOpType ahcConcOpType;
4976 0 : ahcConcOpType.ahcLevel = static_cast<AHCLevel>(
4977 0 : (ahcConfInfo[i] & TOP_HIERARCHICAL_CONF_LEVEL_LOCATION) >> TOP_HIERARCHICAL_CONF_LEVEL_SHIFT);
4978 0 : ahcConcOpType.concType = static_cast<ConcType>(
4979 0 : (ahcConfInfo[i] & TOP_HIERARCHICAL_CONF_CONC_TYPE_LOCATION) >> TOP_HIERARCHICAL_CONF_CONC_TYPE_SHIFT);
4980 0 : ahcConcOpType.ahcOpType = static_cast<AHCOpType>(
4981 0 : (ahcConfInfo[i] & TOP_HIERARCHICAL_CONF_OP_TYPE_LOCATION) >> TOP_HIERARCHICAL_CONF_OP_TYPE_SHIFT);
4982 0 : TemplateType templateType = static_cast<TemplateType>(
4983 0 : (ahcConfInfo[i] & TOP_HIERARCHICAL_CONF_TEMPLATE_TYPE_LOCATION)
4984 : >> TOP_HIERARCHICAL_CONF_TEMPLATE_TYPE_SHIFT);
4985 :
4986 0 : hierarchicalAlgOption[ahcConcOpType] = templateType;
4987 0 : HCCL_DEBUG(
4988 : "[HcclCommAicpu][ParseHierarchicalAlgOption]: index[%u] ahcLevel[%u] concType[%u] ahcOpType[%u], "
4989 : "templateType[%u]",
4990 : i, ahcConcOpType.ahcLevel, ahcConcOpType.concType, ahcConcOpType.ahcOpType, templateType);
4991 : }
4992 0 : topoMatcher_->SetAHCAlgOption(hierarchicalAlgOption);
4993 0 : return HCCL_SUCCESS;
4994 0 : }
4995 :
4996 0 : void HcclCommAicpu::PollCqeException(
4997 : hccl::Stream& stream, bool isReadClear, rtLogicCqReport_t& cqeException, CqeStatus& cqeStatus)
4998 : {
4999 0 : const HcclComStreamInfo& streamInfo = stream.GetHcclStreamInfo();
5000 : // 以下条件满足其中之一，进行Poll CQE：1、AICPU/Custom读清CQ 2、AICPU进程读取CQ信息
5001 0 : bool isPollCqe = true;
5002 0 : while (isPollCqe) {
5003 0 : LogControl logControl(false, retryEnable_); // 使能重执行场景，修改ERROR->RUN_WARNING，析构时自动恢复
5004 : CqeQueryInput cqeQueryInput;
5005 0 : dfx_tracer::ExecutorTracer::SetCqeQueryInput(GetDevId(), streamInfo, cqeQueryInput);
5006 0 : constexpr u32 reportSize = 256;
5007 : rtLogicCqReport_t streamReport[reportSize];
5008 0 : cqeQueryInput.cqeAddr = reinterpret_cast<uint8_t*>(streamReport); // 用于存放接收到的cq
5009 0 : cqeStatus = CqReportRecv(cqeQueryInput, cqeException);
5010 0 : isPollCqe = (cqeStatus == dfx::CqeStatus::kCqeException && isReadClear); // 读清CQ场景，继续查询
5011 0 : }
5012 0 : }
5013 :
5014 0 : void HcclCommAicpu::ExchangeCqeContext(
5015 : hccl::Stream& stream, rtLogicCqReport_t& cqeException, CqeStatus& cqeStatus, ErrCqeContext& cqeCtx)
5016 : {
5017 0 : HcclResult ret = HCCL_SUCCESS;
5018 : // aicpu进程把读取到的异常cq，写入共享内存
5019 0 : if (cqeStatus == dfx::CqeStatus::kCqeException) {
5020 0 : ret = stream.SetCqeContext(ErrCqeContext(
5021 0 : static_cast<u32>(cqeStatus), cqeException.taskId, cqeException.errorCode, cqeException.sqeType));
5022 : }
5023 :
5024 : // aicpu进程和custom进程，都从共享内存中读取cqe信息
5025 0 : if (ret != HCCL_SUCCESS || stream.GetCqeContext(cqeCtx) != HCCL_SUCCESS) {
5026 : // 写入/读取共享内存失败，标记流状态，避免背景线程继续轮询这条流，导致刷屏
5027 0 : SetStreamCqeExceptionStatus(stream, CqeExceptionStatus::kOther);
5028 0 : HCCL_ERROR(
5029 : "%s fail, set streamCqeExceptionStatus streamId[%u] exception status[%d]", __func__, stream.id(),
5030 : CqeExceptionStatus::kOther);
5031 0 : return;
5032 : }
5033 : }
5034 :
5035 0 : void HcclCommAicpu::HandleCqeException(hccl::Stream& stream, bool isReadClear)
5036 : {
5037 0 : std::unique_lock<std::mutex> lock(queryCqeMutex_);
5038 :
5039 : // poll cqe信息
5040 : rtLogicCqReport_t cqeException;
5041 0 : CqeStatus cqeStatus = CqeStatus::kDefault;
5042 0 : PollCqeException(stream, isReadClear, cqeException, cqeStatus);
5043 :
5044 : // 以下两种情况直接返回，不处理cq信息：1、读清CQ场景只读取不处理 2、本地记录流上已经有异常信息(避免刷屏)
5045 0 : if (isReadClear || GetStreamCqeExceptionStatus(stream) != CqeExceptionStatus::kNone) {
5046 0 : return;
5047 : }
5048 :
5049 : // aicpu和custom同步异常cqe信息，并记录在本地
5050 0 : ErrCqeContext cqeCtx;
5051 0 : ExchangeCqeContext(stream, cqeException, cqeStatus, cqeCtx);
5052 0 : cqeStatus = static_cast<dfx::CqeStatus>(cqeCtx.cqeStatus);
5053 :
5054 : // 处理流上的异常cq
5055 0 : if (cqeStatus != dfx::CqeStatus::kDefault) {
5056 0 : bool isSdmaTypeErr = cqeStatus == dfx::CqeStatus::kCqeException && cqeCtx.sqeType == RT_STARS_SQE_TYPE_SDMA;
5057 0 : bool isCompDataErr = cqeCtx.errorCode == RT_SDMA_COMPDATAERR || cqeCtx.errorCode == RT_SDMA_COMPERR
5058 0 : || cqeCtx.errorCode == RT_SDMA_DATAERR;
5059 0 : bool isSdmaCompDataErr = isSdmaTypeErr && isCompDataErr;
5060 :
5061 0 : ReportErrCqe(stream, cqeCtx);
5062 :
5063 : // 标记发生ErrCqe的流
5064 0 : hccl::CqeExceptionStatus cqeExceptionStatus
5065 0 : = isSdmaCompDataErr ? hccl::CqeExceptionStatus::kSdmaErr : hccl::CqeExceptionStatus::kOther;
5066 0 : SetStreamCqeExceptionStatus(stream, cqeExceptionStatus);
5067 :
5068 : // 通知kfc线程
5069 0 : dfxExtendInfo_.cqeException.sqeType = cqeCtx.sqeType;
5070 0 : dfxExtendInfo_.cqeException.errorCode = cqeCtx.errorCode;
5071 0 : dfxExtendInfo_.cqeStatus = cqeStatus;
5072 0 : dfxExtendInfo_.pollStatus = PollStatus::kStopAsException;
5073 0 : HCCL_INFO(
5074 : "update dfxExtendInfo, group %s, streamId %u, cqeStatus %d, sqetype %u, errorCode %u", identifier_.c_str(),
5075 : stream.id(), cqeStatus, cqeCtx.sqeType, cqeCtx.errorCode);
5076 : }
5077 0 : }
5078 :
5079 0 : void HcclCommAicpu::ReportErrCqe(hccl::Stream& stream, ErrCqeContext& cqeCtx)
5080 : {
5081 0 : dfx::CqeStatus cqeStatus = static_cast<dfx::CqeStatus>(cqeCtx.cqeStatus);
5082 0 : const HcclComStreamInfo& streamInfo = stream.GetHcclStreamInfo();
5083 0 : u32 head = 0;
5084 0 : u32 tail = 0;
5085 0 : QuerySqStatusByType(devId_, streamInfo.sqId, DRV_SQCQ_PROP_SQ_HEAD, head);
5086 0 : QuerySqStatusByType(devId_, streamInfo.sqId, DRV_SQCQ_PROP_SQ_TAIL, tail);
5087 :
5088 0 : bool isSdmaTypeErr = cqeStatus == dfx::CqeStatus::kCqeException && cqeCtx.sqeType == RT_STARS_SQE_TYPE_SDMA;
5089 0 : bool isCompDataErr = cqeCtx.errorCode == RT_SDMA_COMPDATAERR || cqeCtx.errorCode == RT_SDMA_COMPERR
5090 0 : || cqeCtx.errorCode == RT_SDMA_DATAERR;
5091 0 : bool isSdmaCompDataErr = isSdmaTypeErr && isCompDataErr;
5092 :
5093 : // 发生sdma、notify_wait、place_holder、write_value错误，上报error message
5094 0 : bool isComReportErrMesg
5095 : = cqeStatus == dfx::CqeStatus::kCqeException
5096 0 : && (cqeCtx.sqeType == RT_STARS_SQE_TYPE_SDMA || cqeCtx.sqeType == RT_STARS_SQE_TYPE_NOTIFY_WAIT
5097 0 : || cqeCtx.sqeType == RT_STARS_SQE_TYPE_PLACE_HOLDER || cqeCtx.sqeType == RT_STARS_SQE_TYPE_WRITE_VALUE);
5098 :
5099 : // 使能重执行且触发SDMA ERROR的场景，修改ERROR->RUN_WARNING
5100 0 : LogControl retryLog(false, retryEnable_ && isSdmaCompDataErr);
5101 :
5102 : // 只在AICPU进程上报taskException，Custom进程不上报
5103 0 : if (isComReportErrMesg && errMessageReport_) {
5104 : // 记录关键信息，并通过D2H通信通道交给host内存
5105 0 : GenTaskExceptionInfo(cqeCtx.sqeType, stream, head);
5106 : // 通知ts错误信息，触发非SDMA错误、不使能重执行触发SDMA错误、使能重执行触发无法重执行的错误时，进行故障上报
5107 0 : if (!isSdmaTypeErr || (isSdmaTypeErr && (!retryEnable_ || !isCompDataErr))) {
5108 0 : HcclResult ret = SendTaskExceptionByMBox(cqeCtx.errorCode);
5109 0 : HCCL_RUN_INFO(
5110 : "[HcclCommAicpu][SendTaskExceptionByMBox]group[%s]:"
5111 : "Try to send task exception by mailbox, errType[%u], errCode[%u], streamId[%d]",
5112 : identifier_.c_str(), cqeCtx.sqeType, cqeCtx.errorCode, stream.id());
5113 0 : CHK_PRT_CONT(
5114 : ret != HCCL_SUCCESS,
5115 : HCCL_ERROR(
5116 : "[OpRetry][AICPU]group[%s]:Send task exception by mailBox failed, streamId[%d]",
5117 : identifier_.c_str(), stream.id()));
5118 : }
5119 : // 当前阶段， 每个通信域在plog打印一次，error message作为host上报，只打印首次
5120 0 : errMessageReport_ = false;
5121 : }
5122 :
5123 : // 发生ErrCqe之后，统一在AICPU进程上报taskException
5124 0 : if (cqeStatus == dfx::CqeStatus::kCqeException) {
5125 0 : if (IsNoNeedWait() && cqeCtx.sqeType == RT_STARS_SQE_TYPE_PLACE_HOLDER) {
5126 0 : PrintTaskExceptionAllComm(); // 超时场景打印所有通信域的taskException
5127 0 : PrintAicpuCommExecStatus();
5128 0 : } else if (IsNoNeedWait()) {
5129 0 : ErrCqeContext cqeCtx;
5130 0 : stream.GetCqeContext(cqeCtx);
5131 0 : PrintTaskExceptionByTaskId(cqeCtx.sqeType, cqeCtx.taskId, stream, tail); // 仅打印本条流的taskException
5132 : }
5133 : }
5134 :
5135 0 : CHK_PRT_CONT(
5136 : !retryEnable_ && isSdmaCompDataErr,
5137 : HCCL_RUN_INFO(
5138 : "[OpRetry][AICPU]group[%s] hccl aicpu can not retry, retryEnable is false.", identifier_.c_str()));
5139 :
5140 0 : CHK_PRT_CONT(
5141 : retryEnable_ && isSdmaTypeErr && !isCompDataErr,
5142 : HCCL_RUN_INFO(
5143 : "[OpRetry][AICPU]group[%s] hccl aicpu can not retry, errCode is [%u].", identifier_.c_str(),
5144 : cqeCtx.errorCode));
5145 :
5146 0 : HCCL_ERROR(
5147 : "Exception happened, group %s, sqid %d, cqeStatus %d, sqetype %u, errorCode %u, head %u, tail %u",
5148 : identifier_.c_str(), streamInfo.sqId, cqeStatus, cqeCtx.sqeType, cqeCtx.errorCode, head, tail);
5149 0 : }
5150 :
5151 0 : HcclResult HcclCommAicpu::SendTaskExceptionByMBox(const uint16_t& rsErrorCode)
5152 : {
5153 0 : u32 localDeviceId = 0;
5154 : HcclSignalInfo notifyInfo;
5155 0 : opNotifies_[1]->GetNotifyData(notifyInfo);
5156 :
5157 0 : HCCL_INFO("[HcclCommAicpu][SendTaskExceptionByMBox] HostToDeviceLogicId[%u]", notifyInfo.devId);
5158 0 : CHK_RET(hrtDrvGetLocalDevIDByHostDevID(notifyInfo.devId, &localDeviceId));
5159 0 : CHK_RET(hccl_plf::SendTaskExceptionByMBox(
5160 : localDeviceId, opNotifies_[1]->notifyId_, notifyInfo.tsId, userStreamId_, rsErrorCode));
5161 0 : return HCCL_SUCCESS;
5162 : }
5163 :
5164 0 : void HcclCommAicpu::HandleIndOpCqe()
5165 : {
5166 0 : std::unique_lock<std::mutex> lock(queryCqeMutex_);
5167 0 : std::shared_lock<std::shared_mutex> rwLock(threadAicpuMutex_);
5168 0 : for (auto& thread : threads_) {
5169 0 : if (thread == nullptr) {
5170 0 : continue;
5171 : }
5172 0 : Stream stream = *thread->GetStream();
5173 : // 流上已有异常信息，不再重复读取
5174 0 : if (GetStreamCqeExceptionStatus(stream) != CqeExceptionStatus::kNone) {
5175 0 : continue;
5176 : }
5177 :
5178 : // poll cqe信息
5179 : rtLogicCqReport_t cqeException;
5180 : CqeQueryInput cqeQueryInput;
5181 0 : dfx_tracer::ExecutorTracer::SetCqeQueryInput(GetDevId(), stream.GetHcclStreamInfo(), cqeQueryInput);
5182 0 : constexpr u32 reportSize = 256;
5183 : rtLogicCqReport_t streamReport[reportSize];
5184 0 : cqeQueryInput.cqeAddr = reinterpret_cast<uint8_t*>(streamReport); // 用于存放接收到的cq
5185 0 : CqeStatus cqeStatus = CqReportRecv(cqeQueryInput, cqeException);
5186 : // 未读取到异常信息，返回
5187 0 : if (cqeStatus == dfx::CqeStatus::kDefault) {
5188 0 : continue;
5189 : }
5190 0 : ReportIndOpCqe(stream, cqeException, cqeStatus);
5191 0 : }
5192 0 : }
5193 :
5194 0 : void HcclCommAicpu::ReportIndOpCqe(hccl::Stream& stream, const rtLogicCqReport_t& cqeException, CqeStatus cqeStatus)
5195 : {
5196 0 : const HcclComStreamInfo& streamInfo = stream.GetHcclStreamInfo();
5197 0 : u32 head = 0;
5198 0 : u32 tail = 0;
5199 0 : QuerySqStatusByType(devId_, streamInfo.sqId, DRV_SQCQ_PROP_SQ_HEAD, head);
5200 0 : QuerySqStatusByType(devId_, streamInfo.sqId, DRV_SQCQ_PROP_SQ_TAIL, tail);
5201 :
5202 : // 打印taskException信息
5203 0 : if (cqeStatus == dfx::CqeStatus::kCqeException) {
5204 0 : if (cqeException.sqeType == RT_STARS_SQE_TYPE_PLACE_HOLDER) {
5205 0 : PrintTaskExceptionAllComm(); // 超时场景打印所有通信域的taskException
5206 0 : PrintAicpuCommExecStatus();
5207 : } else {
5208 0 : taskExecption_.PrintTaskExceptionByTaskId(cqeException.sqeType, cqeException.taskId, stream, tail);
5209 : }
5210 : }
5211 :
5212 : // 打印重执行提示信息
5213 0 : bool isSdmaTypeErr = cqeStatus == dfx::CqeStatus::kCqeException && cqeException.sqeType == RT_STARS_SQE_TYPE_SDMA;
5214 0 : bool isCompDataErr = cqeException.errorCode == RT_SDMA_COMPDATAERR || cqeException.errorCode == RT_SDMA_COMPERR;
5215 0 : bool isSdmaCompDataErr = isSdmaTypeErr && isCompDataErr;
5216 0 : if (retryEnable_ && isSdmaCompDataErr) {
5217 0 : uint16_t rsErrorCode = TS_ERROR_RETRY_CONSTRAINT;
5218 0 : CHK_PRT(SendTaskExceptionByMBox(rsErrorCode));
5219 0 : HCCL_RUN_INFO("[OpRetry][AICPU]group[%s] can not retry, IndOp does not support opRetry", identifier_.c_str());
5220 : }
5221 :
5222 : // 标记发生ErrCqe的流
5223 0 : hccl::CqeExceptionStatus cqeExceptionStatus
5224 0 : = isSdmaCompDataErr ? hccl::CqeExceptionStatus::kSdmaErr : hccl::CqeExceptionStatus::kOther;
5225 0 : SetStreamCqeExceptionStatus(stream, cqeExceptionStatus);
5226 :
5227 0 : HCCL_ERROR(
5228 : "Exception happened, group %s, streamId %u, sqid %d, cqeStatus %d, sqetype %u, errorCode %u, "
5229 : "head %u, tail %u",
5230 : identifier_.c_str(), stream.id(), streamInfo.sqId, cqeStatus, cqeException.sqeType, cqeException.errorCode,
5231 : head, tail);
5232 0 : }
5233 :
5234 0 : HcclResult HcclCommAicpu::RefreshLinkForSwitchNic(
5235 : const std::string& newTag, const TransportRequest& transportRequest, const std::map<u32, bool>& remoteRankPortMap,
5236 : bool isSecondBuild, LINK& switchLink)
5237 : {
5238 0 : u32 remoteRankId = transportRequest.remoteUserRank;
5239 0 : auto iterRemoteRank = remoteRankPortMap.find(remoteRankId);
5240 0 : bool needSwitch = iterRemoteRank != remoteRankPortMap.end();
5241 0 : bool isBackup = needSwitch && !(iterRemoteRank->second);
5242 0 : HCCL_INFO(
5243 : "[HcclCommAicpu][%s] newTag[%s], localRank[%u], remoteRank[%u], input memory type[%u], "
5244 : "output memory type[%u], isRdma[%u], isSecondBuild[%u], needSwitch[%u], isBackup[%u].",
5245 : __func__, newTag.c_str(), localUserRank_, remoteRankId, transportRequest.inputMemType,
5246 : transportRequest.outputMemType, transportRequest.isUsedRdma, isSecondBuild, needSwitch, isBackup);
5247 :
5248 0 : if (transportRequest.isUsedRdma && needSwitch) {
5249 0 : auto* linkRes = isBackup ? &linkRdmaResBackUp_ : &linkRdmaRes_;
5250 0 : auto iterRankLinks = linkRes->find(remoteRankId);
5251 0 : CHK_PRT_RET(
5252 : iterRankLinks == linkRes->end(),
5253 : HCCL_ERROR(
5254 : "[HcclCommAicpu][%s] comm[%s], local rank[%u], fail to find relative link for remote rank[%u], "
5255 : "isBackup[%u]",
5256 : __func__, identifier_.c_str(), localUserRank_, remoteRankId, isBackup),
5257 : HCCL_E_INTERNAL);
5258 0 : auto iterRankTagLinks = iterRankLinks->second.find(newTag);
5259 0 : CHK_PRT_RET(
5260 : iterRankTagLinks == iterRankLinks->second.end(),
5261 : HCCL_ERROR(
5262 : "[HcclCommAicpu][%s] comm[%s], local rank[%u], "
5263 : "fail to find relative tag[%s] of links for remote rank[%u], isBackup[%u]",
5264 : __func__, identifier_.c_str(), localUserRank_, newTag.c_str(), remoteRankId, isBackup),
5265 : HCCL_E_INTERNAL);
5266 0 : CHK_PRT_RET(
5267 : isSecondBuild && iterRankLinks->second[newTag].size() < 2U,
5268 : HCCL_ERROR(
5269 : "[HcclCommAicpu][%s] comm[%s], local rank[%u], isSecondBuild[%u], "
5270 : "fail to find second link for remmote rank[%u]",
5271 : __func__, identifier_.c_str(), localUserRank_, isSecondBuild, remoteRankId),
5272 : HCCL_E_INTERNAL);
5273 :
5274 0 : switchLink = isSecondBuild ? iterRankLinks->second[newTag][1] : iterRankLinks->second[newTag][0];
5275 0 : CHK_SMART_PTR_NULL(switchLink);
5276 :
5277 0 : auto iterAck = receivedAcks_.find(remoteRankId);
5278 0 : CHK_PRT_RET(
5279 : iterAck == receivedAcks_.end(), HCCL_ERROR("[%s]there is no link with rankId[%u]", __func__, remoteRankId),
5280 : HCCL_E_NOT_FOUND);
5281 0 : switchLink->SetSupportDataReceivedAck(iterAck->second);
5282 : }
5283 0 : return HCCL_SUCCESS;
5284 : }
5285 :
5286 0 : HcclResult HcclCommAicpu::ReAllocTransportForSwitchNic(
5287 : const std::string& newTag, AlgResourceResponse& algResResponse, std::map<u32, bool>& remoteRankPortMap)
5288 : {
5289 0 : HCCL_INFO(
5290 : "[HcclCommAicpu][%s] Entry realloc transport for switch nic, comm identifier[%s], localRank[%u], tag[%s]",
5291 : __func__, identifier_.c_str(), localUserRank_, newTag.c_str());
5292 0 : std::set<u32> bsrTansportRank;
5293 0 : for (auto& levelNSubCommTransport : algResResponse.opTransportResponse) {
5294 0 : for (auto& singleSubCommTransport : levelNSubCommTransport) {
5295 0 : for (size_t i = 0; i < singleSubCommTransport.transportRequests.size(); i++) {
5296 0 : auto& transportRequest = singleSubCommTransport.transportRequests[i];
5297 0 : if (transportRequest.isValid) {
5298 0 : receivedAcks_[transportRequest.remoteUserRank] = singleSubCommTransport.supportDataReceivedAck;
5299 0 : bool isSecondBuild = false;
5300 0 : if (transportRequest.isUsedRdma && newTag.find("BatchSendRecv") != std::string::npos
5301 0 : && bsrTansportRank.find(transportRequest.remoteUserRank) != bsrTansportRank.end()) {
5302 : // 仅在batchsendrecv rdma下发的时候需要第二次刷新，实际第一次下发都刷好了，第二次就是get一下
5303 0 : isSecondBuild = true;
5304 : }
5305 0 : bsrTansportRank.insert(transportRequest.remoteUserRank);
5306 0 : CHK_RET(RefreshLinkForSwitchNic(
5307 : newTag, transportRequest, remoteRankPortMap, isSecondBuild, singleSubCommTransport.links[i]));
5308 : }
5309 : }
5310 : }
5311 : }
5312 :
5313 0 : return HCCL_SUCCESS;
5314 0 : }
5315 :
5316 : HcclResult
5317 0 : HcclCommAicpu::RefreshRoceTransportsForSwitchNic(std::unordered_map<std::string, OpCommTransport>& reservedLinks)
5318 : {
5319 0 : ChangeLinkInfo changeLinkInfo;
5320 0 : HcclResult ret = LoadChangeLinkInfo(changeLinkInfo);
5321 :
5322 0 : CHK_PRT_RET(
5323 : changeLinkInfo.isChangeLinkFlag == false,
5324 : HCCL_ERROR(
5325 : "[HcclCommAicpu][%s] some error happened on host, switch nic has failed on rank[%u]. "
5326 : "The error message will be broadcast to other ranks in comm[%s].",
5327 : __func__, localUserRank_, identifier_.c_str()),
5328 : HCCL_E_PARA);
5329 :
5330 0 : CHK_PRT_RET(
5331 : ret != HCCL_SUCCESS,
5332 : HCCL_ERROR(
5333 : "[HcclCommAicpu][%s] comm identifier[%s], rank[%u], "
5334 : "load change link info failed.",
5335 : __func__, identifier_.c_str(), localUserRank_),
5336 : ret);
5337 :
5338 0 : std::map<u32, bool> remoteRankPortMap;
5339 0 : for (u32 i = 0; i < changeLinkInfo.remoteRankNum; i++) {
5340 0 : remoteRankPortMap.emplace(changeLinkInfo.remoteRankList[i], changeLinkInfo.isUseDefaultPort[i]);
5341 0 : HCCL_INFO(
5342 : "[HcclCommAicpu][%s] comm identifier[%s], rank[%u], remote rank[%u], isUseDefaultPort[%u].", __func__,
5343 : identifier_.c_str(), localUserRank_, changeLinkInfo.remoteRankList[i], changeLinkInfo.isUseDefaultPort[i]);
5344 : }
5345 :
5346 : // 对resMap中所有tag的transport link根据主备进行刷新
5347 0 : for (auto& resMapIt : resMap_) {
5348 0 : HCCL_RUN_INFO(
5349 : "[HcclCommAicpu][%s] rank[%u] refresh algResResponse of tag[%s].", __func__, localUserRank_,
5350 : resMapIt.first.c_str());
5351 0 : reservedLinks.emplace(resMapIt.first, resMapIt.second.opTransportResponse);
5352 0 : CHK_RET(ReAllocTransportForSwitchNic(resMapIt.first, resMapIt.second, remoteRankPortMap));
5353 : }
5354 0 : return HCCL_SUCCESS;
5355 0 : }
5356 :
5357 0 : HcclResult HcclCommAicpu::RevertTransportsForSwitchNic(std::unordered_map<std::string, OpCommTransport>& reservedLinks)
5358 : {
5359 0 : if (reservedLinks.size() == 0) {
5360 0 : HCCL_INFO(
5361 : "[HcclCommAicpu][%s] comm identifier[%s], rank[%u], reserved link is empty, "
5362 : "no need to revert transport.",
5363 : __func__, identifier_.c_str(), localUserRank_);
5364 0 : return HCCL_SUCCESS;
5365 : }
5366 0 : for (auto& resMapIt : resMap_) {
5367 0 : HCCL_RUN_INFO(
5368 : "[HcclCommAicpu][%s] comm identifier[%s], rank[%u], revert transport in algResResponse of tag[%s].",
5369 : __func__, identifier_.c_str(), localUserRank_, resMapIt.first.c_str());
5370 0 : auto linkIt = reservedLinks.find(resMapIt.first);
5371 0 : if (linkIt == reservedLinks.end()) {
5372 0 : HCCL_RUN_WARNING(
5373 : "[HcclCommAicpu][%s] comm identifier[%s], rank[%u], fail to find transport to revert.", __func__,
5374 : identifier_.c_str(), localUserRank_);
5375 0 : continue;
5376 : }
5377 0 : resMapIt.second.opTransportResponse = linkIt->second;
5378 : }
5379 0 : return HCCL_SUCCESS;
5380 : }
5381 :
5382 0 : HcclResult HcclCommAicpu::SwitchNicWaitHandleCommand(std::unordered_map<std::string, OpCommTransport>& reservedLinks)
5383 : {
5384 0 : auto waitSwitchCmdTimeoutMs = HcclGetWaitRetryCmdTimeout(0);
5385 0 : auto waitSwitchCmdTimeout = std::chrono::milliseconds(waitSwitchCmdTimeoutMs);
5386 :
5387 0 : HcclResult ret = HCCL_E_INTERNAL;
5388 0 : auto startTime = std::chrono::steady_clock::now();
5389 : while (true) {
5390 0 : KfcCommand switchHandleCmd = KfcCommand::kNone;
5391 0 : CHK_RET(aicpuHdc_.GetOpExecCtrlCmd(kfcControlTransferH2D_, switchHandleCmd));
5392 0 : if (switchHandleCmd == KfcCommand::kWaitSwitchNic) {
5393 0 : HCCL_INFO(
5394 : "[HcclCommAicpu][%s] comm identifier[%s], rank[%u], hccl aicpu recv switch nic handle command.",
5395 : __func__, identifier_.c_str(), localUserRank_);
5396 0 : ret = aicpuHdc_.SetOpExecStatus(kfcStatusTransferD2H_, KfcStatus::kWaitSwitchRes, KfcError::kNone, 0);
5397 0 : if (ret != HCCL_SUCCESS) {
5398 0 : (void)RevertTransportsForSwitchNic(reservedLinks);
5399 0 : return ret;
5400 : }
5401 0 : return HCCL_SUCCESS;
5402 0 : } else if ((std::chrono::steady_clock::now() - startTime) >= waitSwitchCmdTimeout) {
5403 0 : HCCL_ERROR(
5404 : "[HcclCommAicpu][%s] comm identifier[%s], rank[%u], "
5405 : "hccl aicpu wait switch nic handle command timeout[%u ms], local nic switch is reverted.",
5406 : __func__, identifier_.c_str(), localUserRank_, waitSwitchCmdTimeoutMs);
5407 0 : (void)RevertTransportsForSwitchNic(reservedLinks);
5408 0 : return HCCL_E_TIMEOUT;
5409 : }
5410 0 : }
5411 :
5412 : return ret;
5413 : }
5414 :
5415 0 : HcclResult HcclCommAicpu::SwitchNicWaitResult(std::unordered_map<std::string, OpCommTransport>& reservedLinks)
5416 : {
5417 0 : auto waitSwitchResultTimeoutSecond = GetExternalInputHcclLinkTimeOut() * 2U;
5418 0 : const auto waitSwitchResultTimeout = std::chrono::seconds(waitSwitchResultTimeoutSecond);
5419 :
5420 0 : auto startTime = std::chrono::steady_clock::now();
5421 : while (true) {
5422 0 : KfcCommand switchResultCmd = KfcCommand::kNone;
5423 0 : CHK_RET(aicpuHdc_.GetOpExecCtrlCmd(kfcControlTransferH2D_, switchResultCmd));
5424 0 : if (switchResultCmd == KfcCommand::kAllSwitched) {
5425 0 : HCCL_INFO(
5426 : "[HcclCommAicpu][%s] comm identifier[%s], rank[%u], "
5427 : "hccl aicpu switch nic success, all nic switch transport.",
5428 : __func__, identifier_.c_str(), localUserRank_);
5429 0 : return HCCL_SUCCESS;
5430 0 : } else if (switchResultCmd == KfcCommand::kSwitchFail) {
5431 0 : HCCL_ERROR(
5432 : "[HcclCommAicpu][%s] comm identifier[%s], rank[%u], "
5433 : "hccl aicpu comm switch nic error, local nic switch is reverted.",
5434 : __func__, identifier_.c_str(), localUserRank_);
5435 0 : (void)RevertTransportsForSwitchNic(reservedLinks);
5436 0 : return HCCL_E_INTERNAL;
5437 0 : } else if ((std::chrono::steady_clock::now() - startTime) >= waitSwitchResultTimeout) {
5438 0 : HCCL_ERROR(
5439 : "[HcclCommAicpu][%s] comm identifier[%s], rank[%u], "
5440 : "hccl aicpu wait switch nic result timeout[%u s].",
5441 : __func__, identifier_.c_str(), localUserRank_, waitSwitchResultTimeoutSecond);
5442 0 : (void)RevertTransportsForSwitchNic(reservedLinks);
5443 0 : return HCCL_E_TIMEOUT;
5444 : }
5445 0 : }
5446 :
5447 : return HCCL_E_INTERNAL;
5448 : }
5449 :
5450 0 : HcclResult HcclCommAicpu::SwitchNic()
5451 : {
5452 0 : KfcStatus state = KfcStatus::kSwitchError;
5453 0 : std::unordered_map<std::string, OpCommTransport> reservedLinks;
5454 0 : HcclResult ret = RefreshRoceTransportsForSwitchNic(reservedLinks);
5455 0 : if (ret != HCCL_SUCCESS) {
5456 0 : HCCL_ERROR(
5457 : "[HcclCommAicpu][%s] comm identifier[%s], rank[%u], refresh roce transports failed.", __func__,
5458 : identifier_.c_str(), localUserRank_);
5459 0 : state = KfcStatus::kSwitchError;
5460 0 : ret = aicpuHdc_.SetOpExecStatus(kfcStatusTransferD2H_, state, KfcError::kNone, 0);
5461 : } else {
5462 0 : HCCL_INFO(
5463 : "[HcclCommAicpu][%s] comm identifier[%s], rank[%u], refresh roce transports success.", __func__,
5464 : identifier_.c_str(), localUserRank_);
5465 0 : state = KfcStatus::kPlanSwitch;
5466 0 : ret = aicpuHdc_.SetOpExecStatus(kfcStatusTransferD2H_, state, KfcError::kNone, 0);
5467 : }
5468 :
5469 0 : if (ret != HCCL_SUCCESS) {
5470 0 : HCCL_ERROR(
5471 : "[HcclCommAicpu][SwitchNic] comm identifier[%s], rank[%u], "
5472 : "send switch status[%u] to host fail.",
5473 : __func__, identifier_.c_str(), localUserRank_, state);
5474 : }
5475 :
5476 0 : CHK_RET(SwitchNicWaitHandleCommand(reservedLinks));
5477 :
5478 0 : return SwitchNicWaitResult(reservedLinks);
5479 0 : }
5480 :
5481 0 : HcclResult HcclCommAicpu::ResumeChangeLink()
5482 : {
5483 0 : ChangeLinkInfo changeLinkInfo;
5484 0 : HcclResult ret = LoadChangeLinkInfo(changeLinkInfo);
5485 0 : bool useBackupLink = false;
5486 0 : std::map<u32, bool> remoteRankPortMap;
5487 0 : if (ret != HCCL_SUCCESS) {
5488 0 : HCCL_ERROR(
5489 : "[HcclCommAicpu][%s] comm identifier[%s], rank[%u], load change link info failed.", __func__,
5490 : identifier_.c_str(), localUserRank_);
5491 0 : return ret;
5492 : }
5493 0 : for (u32 i = 0; i < changeLinkInfo.remoteRankNum; i++) {
5494 0 : remoteRankPortMap.insert({changeLinkInfo.remoteRankList[i], changeLinkInfo.isUseDefaultPort[i]});
5495 0 : useBackupLink |= (!changeLinkInfo.isUseDefaultPort[i]);
5496 : }
5497 0 : ret = RefreshCommResponseTransportRes(remoteRankPortMap);
5498 0 : if (ret != HCCL_SUCCESS) {
5499 0 : HCCL_ERROR(
5500 : "[HcclCommAicpu][%s] comm identifier[%s], rank[%u], refresh roce transports failed.", __func__,
5501 : identifier_.c_str(), localUserRank_);
5502 0 : return ret;
5503 : } else {
5504 0 : HCCL_INFO(
5505 : "[HcclCommAicpu][%s] comm identifier[%s], rank[%u], refresh roce transports success.", __func__,
5506 : identifier_.c_str(), localUserRank_);
5507 0 : return HCCL_SUCCESS;
5508 : }
5509 0 : }
5510 :
5511 0 : HcclResult HcclCommAicpu::RefreshCommResponseTransportRes(std::map<u32, bool>& remoteRankPortMap)
5512 : {
5513 0 : PetersonLockGuard guard(hostDeviceLock_.get());
5514 0 : CHK_PRT_RET(guard.IsLockFailed(), HCCL_ERROR("[%s] hostDeviceLock lock failed", __func__), HCCL_E_INTERNAL);
5515 0 : OpParam param;
5516 : // 提前清理所有tag的链路，避免冲突
5517 0 : for (auto& resMapIt : resMap_) {
5518 0 : HCCL_RUN_INFO("[%s] clean roce resource of tag[%s].", __func__, resMapIt.first.c_str());
5519 0 : CleanRoceResource(resMapIt.first, resMapIt.second, remoteRankPortMap, param);
5520 : }
5521 0 : for (auto& resMapIt : resMap_) {
5522 0 : HCCL_RUN_INFO("[%s] refresh algResResponse of tag[%s].", __func__, resMapIt.first.c_str());
5523 0 : CHK_RET(ReAllocTransportResource(resMapIt.first, resMapIt.second, remoteRankPortMap, commParam_, param));
5524 : }
5525 0 : return HCCL_SUCCESS;
5526 0 : }
5527 :
5528 0 : HcclResult HcclCommAicpu::InitAicpuIndOp(CommAicpuParam* commAicpuParam)
5529 : {
5530 0 : if (indOpCommInitialized_) {
5531 0 : HCCL_RUN_INFO("[%s][InitAicpuIndOp]Group[%s] already initialized, skip reinit", __func__, identifier_.c_str());
5532 0 : return HCCL_SUCCESS;
5533 : }
5534 0 : CHK_PTR_NULL(commAicpuParam);
5535 0 : topoInfo_.deviceLogicId = commAicpuParam->deviceLogicId;
5536 0 : topoInfo_.devicePhyId = commAicpuParam->devicePhyId;
5537 0 : topoInfo_.deviceType = static_cast<DevType>(commAicpuParam->deviceType);
5538 0 : identifier_ = std::string(commAicpuParam->hcomId);
5539 0 : topoInfo_.userRankSize = commAicpuParam->userRankSize;
5540 0 : topoInfo_.userRank = commAicpuParam->userRank;
5541 0 : notifys_.reserve(hccl::HCCL_THREAD_NOTIFY_MAX_NUM);
5542 0 : if (topoInfo_.deviceType == DevType::DEV_TYPE_910_93 || topoInfo_.deviceType == DevType::DEV_TYPE_910B) {
5543 0 : notifySize_ = NOTIFY_SIZE_FOUR;
5544 : } else {
5545 0 : notifySize_ = NOTIFY_SIZE_EIGHT;
5546 : }
5547 :
5548 0 : CHK_RET(hrtSetWorkModeAicpu(true));
5549 0 : CHK_RET(hrtSetlocalDevice(topoInfo_.deviceLogicId));
5550 0 : CHK_RET(hrtSetlocalDeviceType(topoInfo_.deviceType));
5551 0 : CHK_RET(hrtDrvGetLocalDevIDByHostDevID(topoInfo_.devicePhyId, &devId_));
5552 0 : CHK_RET(taskExecption_.Init(devId_, topoInfo_.userRank, identifier_));
5553 0 : CHK_RET(RegisterProfCallBack());
5554 :
5555 0 : if (topoInfo_.deviceType == DevType::DEV_TYPE_950 || topoInfo_.deviceType == DevType::DEV_TYPE_960) {
5556 0 : HCCL_INFO("[HcclCommAicpu][InitAicpuIndOp] InitAicpuIndOpV2 start");
5557 0 : indOpCommInitialized_ = true;
5558 0 : return HCCL_SUCCESS;
5559 : }
5560 :
5561 0 : HCCL_INFO("[HcclCommAicpu][InitAicpuIndOp] InitAicpuIndOp start");
5562 0 : if (!FindDispatcherByCommId(&dispatcherCtx_, identifier_.c_str())) {
5563 0 : CHK_RET(CreateDispatcherCtx(&dispatcherCtx_, devId_, identifier_.c_str()));
5564 : }
5565 0 : CHK_PTR_NULL(dispatcherCtx_);
5566 0 : hccl::DispatcherCtx* Ctx_temp = static_cast<DispatcherCtx*>(dispatcherCtx_);
5567 0 : HCCL_INFO("[%s] Ctx_temp[%p]", __func__, (void*)Ctx_temp);
5568 0 : (void)RegisterLoadTaskCallBack(Ctx_temp->GetDispatcher(), nullptr, dfx::TaskProfilingCallBack); // 注册dispatcher
5569 0 : if (commAicpuParam->kfcControlTransferH2DParams.buffLen != 0 && kfcControlTransferH2D_ == nullptr) {
5570 0 : EXCEPTION_CATCH((kfcControlTransferH2D_ = std::make_shared<hccl::HDCommunicate>()), return HCCL_E_PTR);
5571 0 : CHK_SMART_PTR_NULL(kfcControlTransferH2D_);
5572 0 : CHK_RET(kfcControlTransferH2D_->InitDevice(commAicpuParam->kfcControlTransferH2DParams));
5573 : }
5574 0 : if (commAicpuParam->kfcStatusTransferD2HParams.buffLen != 0 && kfcStatusTransferD2H_ == nullptr) {
5575 0 : EXCEPTION_CATCH((kfcStatusTransferD2H_ = std::make_shared<hccl::HDCommunicate>()), return HCCL_E_PTR);
5576 0 : CHK_SMART_PTR_NULL(kfcStatusTransferD2H_);
5577 0 : CHK_RET(kfcStatusTransferD2H_->InitDevice(commAicpuParam->kfcStatusTransferD2HParams));
5578 : }
5579 :
5580 0 : indOpCommInitialized_ = true;
5581 :
5582 : // 在indOpCommInitialized_变为true后拉起背景线程
5583 0 : AicpuComContext* ctx = AicpuGetComContext();
5584 0 : AicpuHcclProcess::CallMC2MaintenanceThread(ctx);
5585 :
5586 0 : HCCL_RUN_INFO(
5587 : "%s group[%s] success!, deviceLogicId[%u], devicePhyId[%u], deviceType[%u], notifySize[%u], "
5588 : "dispatcherCtx[%p]",
5589 : __func__, identifier_.c_str(), topoInfo_.deviceLogicId, topoInfo_.devicePhyId, topoInfo_.deviceType,
5590 : notifySize_, dispatcherCtx_);
5591 :
5592 0 : return HCCL_SUCCESS;
5593 : }
5594 :
5595 0 : HcclResult HcclCommAicpu::InitThreads(ThreadMgrAicpuParam* param)
5596 : {
5597 0 : CHK_PTR_NULL(param);
5598 0 : u32 threadNum = param->threadNum;
5599 0 : std::vector<std::shared_ptr<Thread>> outThreads;
5600 0 : outThreads.reserve(threadNum);
5601 0 : std::string hcomId(param->hcomId);
5602 0 : for (u32 i = 0; i < threadNum; ++i) {
5603 0 : std::string thdUniqueId(param->threadParam[i], THREAD_UNIQUE_ID_MAX_SIZE);
5604 0 : if (UNLIKELY(HcclCheckLogLevel(HCCL_LOG_INFO))) {
5605 0 : std::ostringstream oss;
5606 0 : oss << "threadParam[" << i << "] raw bytes: ";
5607 0 : for (u32 j = 0; j < THREAD_UNIQUE_ID_MAX_SIZE; ++j) {
5608 0 : oss << std::hex << std::setw(2) << std::setfill('0')
5609 0 : << static_cast<unsigned int>(static_cast<unsigned char>(param->threadParam[i][j])) << " ";
5610 : }
5611 0 : HCCL_INFO("[HcclCommAicpu][%s] %s", __func__, oss.str().c_str());
5612 0 : }
5613 0 : std::shared_ptr<AicpuTsThread> thread;
5614 0 : EXCEPTION_CATCH((thread = std::make_shared<AicpuTsThread>(thdUniqueId)), return HCCL_E_PTR);
5615 0 : thread->SetCommEngine(param->engine);
5616 0 : HcclResult ret = thread->Init();
5617 0 : if (ret != HCCL_SUCCESS) {
5618 0 : HCCL_ERROR(
5619 : "[HcclCommAicpu][%s] comm identifier[%s], init threads num[%u] failed at index %u", __func__,
5620 : hcomId.c_str(), param->threadNum, i);
5621 0 : return ret;
5622 : }
5623 0 : outThreads.emplace_back(thread);
5624 0 : }
5625 :
5626 0 : ThreadHandle* threadArray = static_cast<ThreadHandle*>(param->deviceHandle);
5627 : // 空指针校验
5628 0 : CHK_PTR_NULL(threadArray);
5629 0 : for (size_t i = 0; i < threadNum; ++i) {
5630 0 : threadArray[i] = reinterpret_cast<ThreadHandle>(outThreads[i].get()); // 拷贝裸指针
5631 0 : HCCL_INFO("[HcclCommAicpu][%s] threadArray[%u] = [%lu]", __func__, i, threadArray[i]);
5632 : }
5633 0 : std::unique_lock<std::shared_mutex> rwLock(threadAicpuMutex_);
5634 0 : threads_.insert(
5635 0 : threads_.end(), std::make_move_iterator(outThreads.begin()), std::make_move_iterator(outThreads.end()));
5636 0 : HCCL_INFO(
5637 : "[HcclCommAicpu][%s] comm identifier[%s], init threads num[%u] success", __func__, hcomId.c_str(), threadNum);
5638 : // 为上报翻转初始化资源
5639 0 : if (topoInfo_.deviceType != DevType::DEV_TYPE_950 && topoInfo_.deviceType != DevType::DEV_TYPE_960) {
5640 0 : CHK_RET(InitProfthreadResource(threadNum));
5641 : }
5642 0 : return HCCL_SUCCESS;
5643 0 : }
5644 :
5645 0 : HcclResult HcclCommAicpu::InitProfthreadResource(u32 threadNum)
5646 : {
5647 0 : groupHashId_ = dfx::ProfilingManager::GetProfHashId(identifier_.c_str(), identifier_.length());
5648 0 : HCCL_INFO(
5649 : "[%s], group[%s], groupHash[%llu], threadNum[%u] ", __func__, identifier_.c_str(), groupHashId_, threadNum);
5650 0 : dfx::ProfCommInfo profInfo{groupHashId_, topoInfo_.userRankSize, topoInfo_.userRank};
5651 : // 添加检查确保 threadNum 不超过线程总数
5652 0 : if (threadNum > threads_.size()) {
5653 0 : HCCL_ERROR("[%s] threadNum Err", __func__);
5654 0 : return HCCL_E_PARA;
5655 : }
5656 : // 从后往前迭代指定数量
5657 0 : auto begin = threads_.rbegin();
5658 0 : auto end = begin + threadNum;
5659 0 : for (auto it = begin; it != end; ++it) {
5660 0 : CHK_RET(dfx::ProfilingManager::AddProfInfoByStreamId((*it)->GetStream()->id(), identifier_, profInfo));
5661 : }
5662 0 : dfx::ProfilingExtendInfoHelper::InitProfItemId();
5663 0 : return HCCL_SUCCESS;
5664 : }
5665 :
5666 0 : HcclResult HcclCommAicpu::AllocChannelResource(HcclIndOpChannelRemoteResV3* commParam)
5667 : {
5668 0 : CHK_PTR_NULL(commParam);
5669 0 : if (commParam->engine != COMM_ENGINE_AICPU && commParam->engine != COMM_ENGINE_AICPU_TS) {
5670 0 : HCCL_ERROR(
5671 : "[HcclCommAicpu][%s] engine type[%s] is not supported", __func__,
5672 : GetEnumToString(GetCommEngineStatusStrMap(), commParam->engine).c_str());
5673 0 : return HCCL_E_PARA;
5674 : }
5675 :
5676 0 : if (commParam->listNum != 0 && commParam->remoteResV2 == nullptr) {
5677 0 : HCCL_ERROR("[HcclCommAicpu][%s] listNum[%u], remoteResV2 is nullptr", __func__, commParam->listNum);
5678 0 : return HCCL_E_PARA;
5679 : }
5680 :
5681 0 : multiQpThreshold_ = commParam->multiQpThreshold;
5682 0 : localUserRank_ = commParam->localUserRank;
5683 0 : HCCL_INFO(
5684 : "%s multiQpThreshold[%u], localUserRank[%u], deviceLogicId[%d], devicePhyId[%u], deviceType[%d], "
5685 : "listNum[%u]",
5686 : __func__, multiQpThreshold_, localUserRank_, topoInfo_.deviceLogicId, topoInfo_.devicePhyId,
5687 : topoInfo_.deviceType, commParam->listNum);
5688 0 : for (u32 idx = 0; idx < commParam->listNum; idx++) {
5689 0 : HCCL_INFO(
5690 : "%s listNum[%u], listIdx[%u], remoteWorldRank[%u], remoteRank[%u], isUsedRdma[%d]", __func__,
5691 : commParam->listNum, idx, commParam->remoteResV2[idx].remoteWorldRank,
5692 : commParam->remoteResV2[idx].remoteRank, commParam->remoteResV2[idx].isUsedRdma);
5693 :
5694 0 : rankData_[commParam->remoteResV2[idx].remoteRank].remoteWorldRank = commParam->remoteResV2[idx].remoteWorldRank;
5695 0 : rankData_[commParam->remoteResV2[idx].remoteRank].remoteUsrRankId = commParam->remoteResV2[idx].remoteRank;
5696 0 : if (commParam->remoteResV2[idx].isUsedRdma) {
5697 0 : CHK_RET(InitRoceChannel(commParam, idx));
5698 : } else {
5699 0 : CHK_RET(InitP2pChannel(commParam, idx));
5700 : }
5701 : }
5702 0 : return HCCL_SUCCESS;
5703 : }
5704 :
5705 0 : HcclResult HcclCommAicpu::InitP2pChannel(HcclIndOpChannelRemoteResV3* commParam, uint32_t channelIndex)
5706 : {
5707 0 : CHK_PTR_NULL(commParam);
5708 0 : CHK_PTR_NULL(commParam->channelList);
5709 0 : HcclIndOpChannelRemoteResV2& remoteResV2 = commParam->remoteResV2[channelIndex];
5710 0 : u32 linkType = static_cast<u32>(remoteResV2.channelP2p.transportAttr.linkType);
5711 0 : std::string channelKey = std::string(commParam->channelTag) + ":" + std::to_string(commParam->engine) + ":"
5712 0 : + std::to_string(remoteResV2.remoteRank) + ":" + std::to_string(linkType);
5713 0 : HCCL_INFO("%s channelKey[%s]", __func__, channelKey.c_str());
5714 0 : if (channelHandleMap_.find(channelKey) != channelHandleMap_.end()) {
5715 0 : HCCL_ERROR("[%s]the channel has existed.", __func__);
5716 0 : return HCCL_E_INTERNAL;
5717 : }
5718 :
5719 0 : HcclChannelP2p& channelP2p = remoteResV2.channelP2p;
5720 0 : if (channelP2p.localIpcSignal[0].resId == INVALID_U64) {
5721 0 : HCCL_ERROR("[%s]the Channel is invalid", __func__);
5722 0 : return HCCL_E_INTERNAL;
5723 : }
5724 :
5725 : // 创建Transport对象
5726 0 : MachinePara machinePara;
5727 0 : CHK_RET(SetTransportMachinePara(machinePara, remoteResV2.remoteRank, commParam->channelTag));
5728 0 : machinePara.notifyNum = remoteResV2.p2pNotifyNum;
5729 : // 获取localMem & remoteMem
5730 0 : TransportDeviceP2pData transDevP2pData;
5731 0 : transDevP2pData.inputBufferPtr = reinterpret_cast<void*>(channelP2p.remoteHcclbuffer.addr);
5732 0 : transDevP2pData.outputBufferPtr = reinterpret_cast<void*>(channelP2p.remoteHcclbuffer.addr);
5733 0 : if (transDevP2pData.inputBufferPtr == nullptr || transDevP2pData.outputBufferPtr == nullptr) {
5734 0 : HCCL_ERROR(
5735 : "[%s]input ptr[%p] or output ptr[%p] is null.", __func__, transDevP2pData.inputBufferPtr,
5736 : transDevP2pData.outputBufferPtr);
5737 0 : return HCCL_E_PARA;
5738 : }
5739 : // 获取Notify资源
5740 0 : CHK_RET(SetChannelP2pNotify(transDevP2pData, remoteResV2.p2pNotifyNum, channelP2p)); // 待确认notify是否
5741 : // 获取transportAttr信息
5742 0 : transDevP2pData.transportAttr = channelP2p.transportAttr;
5743 : // 创建Transport对象
5744 0 : std::shared_ptr<Transport> link;
5745 0 : TransportPara para{};
5746 0 : const std::unique_ptr<NotifyPool> notifyPool;
5747 0 : DispatcherCtx* ctx = static_cast<DispatcherCtx*>(dispatcherCtx_);
5748 0 : CHK_PRT(ctx->SetDispatcherHcclQos(remoteResV2.channelP2p.qos)); // 调度器添加hcclQos
5749 0 : CHK_PTR_NULL(ctx);
5750 0 : link.reset(new (std::nothrow) Transport(
5751 0 : TransportType::TRANS_TYPE_DEVICE_P2P, para, ctx->GetDispatcher(), notifyPool, machinePara, transDevP2pData));
5752 0 : CHK_SMART_PTR_NULL(link);
5753 0 : CHK_RET(link->Init()); // 初始化需要增加远端用户注册内存
5754 :
5755 0 : ChannelHandle channelHandle = reinterpret_cast<ChannelHandle>(link.get());
5756 0 : channelHandleMap_[channelKey] = channelHandle;
5757 0 : linkMap_[channelHandle] = link;
5758 :
5759 : // 恢复出的channelHandle回填到commParam中
5760 0 : CHK_PTR_NULL(commParam->channelList);
5761 0 : ChannelHandle* channelList = reinterpret_cast<ChannelHandle*>(commParam->channelList);
5762 0 : channelList[channelIndex] = channelHandle;
5763 :
5764 0 : return HCCL_SUCCESS;
5765 0 : }
5766 :
5767 0 : HcclResult HcclCommAicpu::SetChannelP2pNotify(
5768 : TransportDeviceP2pData& transDevP2pData, u64& p2pNotifyNum, HcclChannelP2p& channelP2p)
5769 : {
5770 0 : u64 actualNotifyNum = 0;
5771 : // 获取Ipc notify信息
5772 0 : CHK_RET(CheckNotifyOrQPMaxNum(actualNotifyNum, LINK_P2P_MAX_NUM, true));
5773 0 : std::shared_ptr<LocalNotify> ipcPreWaitNotify = std::make_shared<LocalNotify>();
5774 0 : CHK_RET(InitAndVerifySingleSignal(channelP2p.localIpcSignal[actualNotifyNum], ipcPreWaitNotify));
5775 0 : transDevP2pData.ipcPreWaitNotify = ipcPreWaitNotify;
5776 :
5777 0 : std::shared_ptr<RemoteNotify> ipcPreRecordNotify = std::make_shared<RemoteNotify>();
5778 0 : CHK_RET(InitAndVerifySingleSignal(channelP2p.remoteIpcSignal[actualNotifyNum], ipcPreRecordNotify));
5779 0 : transDevP2pData.ipcPreRecordNotify = ipcPreRecordNotify;
5780 0 : actualNotifyNum++;
5781 :
5782 0 : CHK_RET(CheckNotifyOrQPMaxNum(actualNotifyNum, LINK_P2P_MAX_NUM, true));
5783 0 : std::shared_ptr<LocalNotify> ipcPostWaitNotify = std::make_shared<LocalNotify>();
5784 0 : CHK_RET(InitAndVerifySingleSignal(channelP2p.localIpcSignal[actualNotifyNum], ipcPostWaitNotify));
5785 0 : transDevP2pData.ipcPostWaitNotify = ipcPostWaitNotify;
5786 :
5787 0 : std::shared_ptr<RemoteNotify> ipcPostRecordNotify = std::make_shared<RemoteNotify>();
5788 0 : CHK_RET(InitAndVerifySingleSignal(channelP2p.remoteIpcSignal[actualNotifyNum], ipcPostRecordNotify));
5789 0 : transDevP2pData.ipcPostRecordNotify = ipcPostRecordNotify;
5790 0 : actualNotifyNum++;
5791 :
5792 0 : transDevP2pData.userLocalNotify.resize(p2pNotifyNum, nullptr);
5793 0 : transDevP2pData.userRemoteNotify.resize(p2pNotifyNum, nullptr);
5794 :
5795 0 : for (u32 idx = 0; idx < p2pNotifyNum; idx++) {
5796 0 : std::shared_ptr<LocalNotify> ipcWaitNotify = std::make_shared<LocalNotify>();
5797 0 : CHK_RET(InitAndVerifySingleSignal(channelP2p.localIpcSignal[actualNotifyNum], ipcWaitNotify));
5798 0 : transDevP2pData.userLocalNotify[idx] = ipcWaitNotify;
5799 :
5800 0 : std::shared_ptr<RemoteNotify> ipcRecordNotify = std::make_shared<RemoteNotify>();
5801 0 : CHK_RET(InitAndVerifySingleSignal(channelP2p.remoteIpcSignal[actualNotifyNum], ipcRecordNotify));
5802 0 : transDevP2pData.userRemoteNotify[idx] = ipcRecordNotify;
5803 :
5804 0 : actualNotifyNum++;
5805 0 : }
5806 :
5807 0 : HCCL_DEBUG(
5808 : "%s get p2pNotify success, p2pNotifyNum[%llu], actualNotifyNum[%llu]", __func__, p2pNotifyNum, actualNotifyNum);
5809 0 : return HCCL_SUCCESS;
5810 0 : }
5811 :
5812 0 : HcclResult HcclCommAicpu::InitRoceChannel(HcclIndOpChannelRemoteResV3* commParam, uint32_t channelIndex)
5813 : {
5814 0 : HcclIndOpChannelRemoteResV2& remoteResV2 = commParam->remoteResV2[channelIndex];
5815 0 : std::string channelKey = std::string(commParam->channelTag) + ":" + std::to_string(commParam->engine) + ":"
5816 0 : + std::to_string(remoteResV2.remoteRank) + ":"
5817 0 : + std::to_string(CommProtocol::COMM_PROTOCOL_ROCE);
5818 0 : if (channelHandleMap_.find(channelKey) != channelHandleMap_.end()) {
5819 0 : HCCL_ERROR("[%s]the channel has existed.", __func__);
5820 0 : return HCCL_E_INTERNAL;
5821 : }
5822 :
5823 0 : HcclChannelRoce& channelRoce = remoteResV2.channelRoce;
5824 0 : if (channelRoce.localNotifyList == 0) {
5825 0 : HCCL_ERROR("[%s]the Channel is invalid", __func__);
5826 0 : return HCCL_E_INTERNAL;
5827 : }
5828 0 : HcclSignalInfo* localNotifyList = reinterpret_cast<HcclSignalInfo*>(channelRoce.localNotifyList);
5829 0 : if (localNotifyList[0].resId == INVALID_U64) {
5830 0 : HCCL_INFO("[%s]the channel notify resource is invalid", __func__);
5831 0 : return HCCL_E_INTERNAL;
5832 : }
5833 :
5834 : // 创建Transport对象
5835 0 : MachinePara machinePara;
5836 0 : CHK_RET(SetTransportMachinePara(machinePara, remoteResV2.remoteRank, commParam->channelTag)); // 待确认是否填充完毕
5837 0 : machinePara.notifyNum = remoteResV2.roceNotifyNum;
5838 : // 获取localMem & remoteMem
5839 0 : TransportDeviceIbverbsData transDevIbverbsData;
5840 0 : transDevIbverbsData.inputBufferPtr = reinterpret_cast<void*>(channelRoce.remoteHcclbuffer.addr);
5841 0 : transDevIbverbsData.outputBufferPtr = reinterpret_cast<void*>(channelRoce.remoteHcclbuffer.addr);
5842 0 : if (transDevIbverbsData.inputBufferPtr == nullptr || transDevIbverbsData.outputBufferPtr == nullptr) {
5843 0 : HCCL_ERROR(
5844 : "[%s]input ptr[%p] or output ptr[%p] is null.", __func__, transDevIbverbsData.inputBufferPtr,
5845 : transDevIbverbsData.outputBufferPtr);
5846 0 : return HCCL_E_PARA;
5847 : }
5848 0 : transDevIbverbsData.localInputMem = channelRoce.localHcclbuffer;
5849 0 : transDevIbverbsData.localOutputMem = channelRoce.localHcclbuffer;
5850 0 : transDevIbverbsData.localNotifyValueAddr = channelRoce.notifyValue;
5851 0 : transDevIbverbsData.notifyValueKey = channelRoce.notifyValueKey;
5852 0 : transDevIbverbsData.remoteInputKey = channelRoce.remoteHcclbuffer.key;
5853 0 : transDevIbverbsData.remoteOutputKey = channelRoce.remoteHcclbuffer.key;
5854 : // 需要添加远端用户注册内存
5855 :
5856 : // 获取QPinfo
5857 0 : u32 roceQpNumSum = channelRoce.qpsPerConnection + static_cast<u32>(channelRoce.qpsPerConnection != 1);
5858 0 : transDevIbverbsData.qpInfo.resize(roceQpNumSum);
5859 0 : std::copy_n(channelRoce.QpInfo, roceQpNumSum, transDevIbverbsData.qpInfo.begin());
5860 0 : transDevIbverbsData.multiQpThreshold = multiQpThreshold_;
5861 0 : transDevIbverbsData.qpsPerConnection = channelRoce.qpsPerConnection;
5862 :
5863 : // 获取notify
5864 0 : u64& roceNotifyNum = remoteResV2.roceNotifyNum; // 是否需要待确认
5865 0 : CHK_RET(SetChannelRoceNotify(transDevIbverbsData, roceNotifyNum, channelRoce));
5866 :
5867 : // 创建Transport对象
5868 0 : std::shared_ptr<Transport> link;
5869 0 : TransportPara para{};
5870 0 : para.timeout = linkTimeOut_; // 暂无法设置
5871 0 : const std::unique_ptr<NotifyPool> notifyPool;
5872 0 : DispatcherCtx* ctx = static_cast<DispatcherCtx*>(dispatcherCtx_);
5873 0 : CHK_PTR_NULL(ctx);
5874 0 : link.reset(new (std::nothrow) Transport(
5875 : TransportType::TRANS_TYPE_DEVICE_IBVERBS, para, ctx->GetDispatcher(), notifyPool, machinePara,
5876 0 : TransportDeviceP2pData(), transDevIbverbsData));
5877 0 : CHK_SMART_PTR_NULL(link);
5878 0 : CHK_RET(link->Init()); // 初始化需要增加远端用户注册内存
5879 :
5880 0 : ChannelHandle channelHandle = reinterpret_cast<ChannelHandle>(link.get());
5881 0 : channelHandleMap_[channelKey] = channelHandle;
5882 0 : linkMap_[channelHandle] = link;
5883 :
5884 : // 恢复出的channelHandle回填到commParam中
5885 0 : ChannelHandle* channelList = reinterpret_cast<ChannelHandle*>(commParam->channelList);
5886 0 : channelList[channelIndex] = channelHandle;
5887 :
5888 0 : return HCCL_SUCCESS;
5889 0 : }
5890 :
5891 0 : HcclResult HcclCommAicpu::SetChannelRoceNotify(
5892 : TransportDeviceIbverbsData& transDevIbverbsData, u64& roceNotifyNum, HcclChannelRoce& channelRoce)
5893 : {
5894 0 : u64 actualNotifyNum = 0;
5895 0 : if (channelRoce.localNotifyList == 0 || channelRoce.remoteNotifyList == 0) {
5896 0 : HCCL_DEBUG("[%s] Empty local and remote notify lists, skipping notify resource creation.", __func__);
5897 0 : return HCCL_SUCCESS;
5898 : }
5899 0 : HcclSignalInfo* localNotifyList = reinterpret_cast<HcclSignalInfo*>(channelRoce.localNotifyList);
5900 0 : AddrKey* remoteNotifyList = reinterpret_cast<AddrKey*>(channelRoce.remoteNotifyList);
5901 0 : if (localNotifyList == nullptr || remoteNotifyList == nullptr) {
5902 0 : HCCL_ERROR("[%s]nullptr found in localNotifyList or remoteNotifyList from device mem, check.", __func__);
5903 0 : return HCCL_E_INTERNAL;
5904 : }
5905 : // 获取RDMA Notify信息
5906 0 : std::shared_ptr<LocalNotify> ackNotify = std::make_shared<LocalNotify>();
5907 0 : CHK_RET(InitAndVerifySingleSignal(localNotifyList[actualNotifyNum], ackNotify));
5908 0 : transDevIbverbsData.ackNotify = ackNotify;
5909 0 : transDevIbverbsData.remoteAckNotifyDetails = remoteNotifyList[actualNotifyNum];
5910 0 : actualNotifyNum++;
5911 :
5912 0 : std::shared_ptr<LocalNotify> dataNotify = std::make_shared<LocalNotify>();
5913 0 : CHK_RET(InitAndVerifySingleSignal(localNotifyList[actualNotifyNum], dataNotify));
5914 0 : transDevIbverbsData.dataNotify = dataNotify;
5915 0 : transDevIbverbsData.remoteDataNotifyDetails = remoteNotifyList[actualNotifyNum];
5916 0 : actualNotifyNum++;
5917 :
5918 0 : std::shared_ptr<LocalNotify> dataAckNotify = std::make_shared<LocalNotify>();
5919 0 : CHK_RET(InitAndVerifySingleSignal(localNotifyList[actualNotifyNum], dataAckNotify));
5920 0 : transDevIbverbsData.dataAckNotify = dataAckNotify;
5921 0 : transDevIbverbsData.remoteDataAckNotifyDetails = remoteNotifyList[actualNotifyNum];
5922 0 : transDevIbverbsData.notifySize = notifySize_;
5923 0 : actualNotifyNum++;
5924 :
5925 0 : transDevIbverbsData.userLocalNotify.resize(channelRoce.qpsPerConnection);
5926 0 : transDevIbverbsData.userRemoteNotifyDetails.resize(channelRoce.qpsPerConnection);
5927 : // 当前多QP下每个QP会多申请一个DataNotify
5928 0 : u64 singleQpNotifySize = channelRoce.singleQPNotifyNum + static_cast<u32>(channelRoce.qpsPerConnection > 1);
5929 0 : for (u32 qpIndex = 0; qpIndex < channelRoce.qpsPerConnection; qpIndex++) {
5930 0 : transDevIbverbsData.userLocalNotify[qpIndex].resize(singleQpNotifySize, nullptr);
5931 0 : transDevIbverbsData.userRemoteNotifyDetails[qpIndex].resize(singleQpNotifySize);
5932 0 : for (u32 i = 0, idx = actualNotifyNum + singleQpNotifySize * qpIndex; i < singleQpNotifySize; ++idx, ++i) {
5933 0 : std::shared_ptr<LocalNotify> locNotify = std::make_shared<LocalNotify>();
5934 0 : CHK_RET(InitAndVerifySingleSignal(localNotifyList[idx], locNotify));
5935 0 : transDevIbverbsData.userLocalNotify[qpIndex][i] = locNotify;
5936 0 : transDevIbverbsData.userRemoteNotifyDetails[qpIndex][i] = remoteNotifyList[idx];
5937 0 : }
5938 : }
5939 0 : roceNotifyNum = channelRoce.singleQPNotifyNum;
5940 0 : HCCL_DEBUG("[%s]get roceNotify success, roceNotifyNum[%u]", __func__, roceNotifyNum);
5941 0 : return HCCL_SUCCESS;
5942 0 : }
5943 :
5944 0 : HcclResult HcclCommAicpu::NotifyAlloc(NotifyMgrAicpuParam* param)
5945 : {
5946 0 : u32 notifyNum = param->notifyNum;
5947 0 : std::string notifysStr = std::string(param->notifyParam, NOTIFY_UNIQUE_ID_MAX_SIZE);
5948 0 : std::string hcomId(param->hcomId);
5949 0 : size_t notifySize = notifys_.size();
5950 0 : HCCL_INFO(
5951 : "[HcclCommAicpu][%s] comm identifier[%s], alloc notifys num[%u] begin, before notifySize[%u]", __func__,
5952 : hcomId.c_str(), notifyNum, notifySize);
5953 0 : if (UNLIKELY(HcclCheckLogLevel(HCCL_LOG_INFO))) {
5954 0 : std::ostringstream oss;
5955 0 : oss << "notifyParam" << " raw bytes: ";
5956 0 : for (u32 i = 0; i < NOTIFY_UNIQUE_ID_MAX_SIZE; ++i) {
5957 0 : oss << std::hex << std::setw(2) << std::setfill('0')
5958 0 : << static_cast<unsigned int>(static_cast<unsigned char>(param->notifyParam[i])) << " ";
5959 : }
5960 0 : HCCL_INFO("[HcclCommAicpu][%s] %s", __func__, oss.str().c_str());
5961 0 : }
5962 0 : HcclResult ret = NotifyManager::ParseBinNotifys(notifysStr, notifys_);
5963 0 : if (ret != HCCL_SUCCESS) {
5964 0 : HCCL_ERROR(
5965 : "[HcclCommAicpu][%s] comm identifier[%s], alloc notifys num[%u] failed %u", __func__, hcomId.c_str(),
5966 : notifyNum, ret);
5967 0 : return ret;
5968 : }
5969 0 : HCCL_INFO(
5970 : "[HcclCommAicpu][%s] comm identifier[%s], alloc notifys num[%u] end, after notifySize[%u]", __func__,
5971 : hcomId.c_str(), notifyNum, notifys_.size());
5972 0 : NotifyHandle* notifyArray = static_cast<NotifyHandle*>(param->deviceHandle);
5973 0 : CHK_PTR_NULL(notifyArray);
5974 : // 空指针校验
5975 0 : for (size_t i = 0; i < notifyNum; ++i) {
5976 0 : notifyArray[i] = reinterpret_cast<NotifyHandle>(notifys_[i + notifySize].get()); // 拷贝裸指针
5977 0 : HCCL_INFO("[HcclCommAicpu][%s] notifyArray[%u] = [%lu]", __func__, i + notifySize, notifyArray[i]);
5978 : }
5979 :
5980 0 : HCCL_INFO(
5981 : "[HcclCommAicpu][%s] comm identifier[%s], alloc notifys num[%u] success", __func__, hcomId.c_str(), notifyNum);
5982 0 : return HCCL_SUCCESS;
5983 0 : }
5984 :
5985 0 : HcclResult HcclCommAicpu::NotifyFree(NotifyMgrAicpuParam* param)
5986 : {
5987 0 : u32 notifyNum = param->notifyNum;
5988 0 : NotifyHandle* notifyArray = static_cast<NotifyHandle*>(param->deviceHandle);
5989 0 : std::string hcomId(param->hcomId);
5990 : // 空指针校验
5991 0 : CHK_PTR_NULL(notifyArray);
5992 0 : for (size_t i = 0; i < notifyNum; ++i) {
5993 0 : LocalNotify* notify = reinterpret_cast<LocalNotify*>(notifyArray[i]);
5994 0 : HCCL_INFO("[HcclCommAicpu][%s] notifyArray[%u]=[%lu]", __func__, i, notifyArray[i]);
5995 0 : auto it = std::find_if(notifys_.begin(), notifys_.end(), [notify](const std::unique_ptr<LocalNotify>& ptr) {
5996 0 : return ptr.get() == notify;
5997 : });
5998 0 : if (it != notifys_.end()) {
5999 0 : HCCL_INFO(
6000 : "[HcclCommAicpu][%s] comm identifier[%s], free notifys[%u] success", __func__, hcomId.c_str(),
6001 : notifyArray[i]);
6002 0 : notifys_.erase(it);
6003 : } else {
6004 0 : HCCL_RUN_WARNING("[HcclCommAicpu][%s] localNotify[%u] not found in notifys_", __func__, i);
6005 : }
6006 : }
6007 :
6008 0 : HCCL_INFO(
6009 : "[HcclCommAicpu][%s] comm identifier[%s], free notifys num[%u] success", __func__, hcomId.c_str(), notifyNum);
6010 0 : return HCCL_SUCCESS;
6011 0 : }
6012 :
6013 0 : HcclResult HcclCommAicpu::RegisterOpInfo(void* opInfo, u32 size)
6014 : {
6015 0 : CHK_RET(taskExecption_.RegisterOpInfo(opInfo, size));
6016 0 : u32 opIdx = taskExecption_.GetOpRingBufferIdx();
6017 0 : CHK_RET(SetDispatcherCtxOpIdx(opIdx));
6018 0 : HCCL_INFO("%s success, group[%s], opRingBufferId[%u]", __func__, identifier_.c_str(), opIdx);
6019 0 : return HCCL_SUCCESS;
6020 : }
6021 :
6022 0 : HcclResult HcclCommAicpu::RegOpTaskException(HcommGetOpInfoCallback callback)
6023 : {
6024 0 : CHK_RET(taskExecption_.RegisterOpInfoCallback(callback));
6025 0 : return HCCL_SUCCESS;
6026 : }
6027 :
6028 0 : HcclResult HcclCommAicpu::SetDispatcherCtxOnThread()
6029 : {
6030 : // 设置 DispatcherCtx 到线程变量
6031 0 : CHK_RET(SetDispatcherCtx(dispatcherCtx_));
6032 0 : return HCCL_SUCCESS;
6033 : }
6034 : } // namespace hccl
|
