Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include <iomanip>
12 : #include "aicpu_communicator.h"
13 : #include "dispatcher.h"
14 : #include "aicpu_hccl_process.h"
15 : #include "coll_alg_exec_registry.h"
16 : #include "coll_all_to_all_executor.h"
17 : #include "common/aicpu_hccl_common.h"
18 : #include "executor_tracer.h"
19 : #include "profiling_manager_device.h"
20 : #include "utils/aicpu_hdc_utils.h"
21 : #include "comm_engine_utils.h"
22 : #include "utils/hccl_aicpu_utils.h"
23 : #include "framework/aicpu_hdc.h"
24 : #include "common/aicpu_sqe_context.h"
25 : #include "coll_batch_send_recv_retry_executor.h"
26 : #include "log_control.h"
27 : #include "aicpu_hccl_sqcq.h"
28 : #include "aicpu_hccl_sqcqv1.h"
29 : #include "sal_pub.h"
30 : #include "externalinput_pub.h"
31 : #include "env_config.h"
32 : #include "config_log.h"
33 : #include "aicpu_one_side_service.h"
34 : #include "notify_manager.h"
35 : #include "dispatcher_aicpu.h"
36 : #include "dlprof_function.h"
37 : #include "profiling_command_handle.h"
38 : #include "dispatcher_ctx.h"
39 : #include "aicpu_res_package_helper.h"
40 : #include "aicpu_symmetric_memory.h"
41 :
42 : namespace hccl {
43 : constexpr u32 IPC_SIGNAL_MODULUS = 2;
44 : constexpr u32 RDMA_SIGNAL_MODULUS = 3;
45 : constexpr u32 KEY_VALUE_TO_VECTOR_MODULUS = 2;
46 :
47 : constexpr u32 BSR_RETRY_SEND_STREAM_INDEX = 0;
48 : constexpr u32 BSR_RETRY_RECV_STREAM_INDEX = 1;
49 : constexpr u32 BSR_RETRY_STREAM_NUM = 2;
50 : constexpr u32 MAX_REPORT_STATUS = 100U; // reportStatus的最大缓存数量
51 : constexpr u32 INPUT = 0;
52 : constexpr u32 OUTPUT = 1;
53 : constexpr u32 AICPU_RETRY_LINKROCE_DEFAULT = 0;
54 : constexpr u32 AICPU_RETRY_LINKROCE_BACKUP = 1;
55 :
56 : constexpr u32 BSR_RETRY_SENDRECV_PAIR_NUM_MAX = 2;
57 : constexpr u32 BSR_RETRY_SENDRECV_PAIR_INDEX_0 = 0;
58 : constexpr u32 BSR_RETRY_SENDRECV_PAIR_INDEX_1 = 1;
59 :
60 : constexpr u32 NOTIFY_SIZE_FOUR = 4;
61 : constexpr u32 NOTIFY_SIZE_EIGHT = 8;
62 :
63 : bool HcclCommAicpu::errMessageReport_ = true;
64 :
65 : #define HCCL_RETRY_CHK_RET_AND_TRANS_FSM(result__, exeLog__, error__, state__) \
66 : do { \
67 : if (UNLIKELY((result__) != HCCL_SUCCESS)) { \
68 : exeLog__; \
69 : errorCode = (error__); \
70 : fsmState = (state__); \
71 : return (result__); \
72 : } \
73 : } while (0)
74 :
75 363 : HcclCommAicpu::HcclCommAicpu() { HCCL_RUN_INFO("Construct HcclCommAicpu complete."); }
76 :
77 288 : HcclCommAicpu::~HcclCommAicpu()
78 : {
79 32 : if (UtraceInfo_ != nullptr) {
80 11 : UtraceInfo_->DeInit();
81 11 : UtraceInfo_ = nullptr;
82 : }
83 32 : if (dispatcher_ != nullptr) {
84 11 : HcclDispatcherDestroy(dispatcher_);
85 11 : dispatcher_ = nullptr;
86 : }
87 32 : if (dispatcherCtx_ != nullptr) {
88 0 : DestroyDispatcherCtx(dispatcherCtx_, identifier_.c_str());
89 0 : HCCL_DEBUG("[%s] destroy dispatcherCtx[%p] group[%s] success!", __func__, dispatcherCtx_, identifier_.c_str());
90 0 : dispatcherCtx_ = nullptr;
91 : }
92 :
93 32 : commPlaneVector_.clear();
94 32 : isBridgeVector_.clear();
95 32 : indOpCommInitialized_ = false;
96 32 : initialized_ = false;
97 :
98 32 : HCCL_RUN_INFO("Destruct HcclCommAicpu group[%s] success!", identifier_.c_str());
99 352 : }
100 :
101 13 : HcclResult HcclCommAicpu::Init(const HcclOpResParam* commParam, bool isCustom)
102 : {
103 13 : if (initialized_) {
104 2 : HCCL_RUN_INFO("[%s][Init]Group[%s] already initialized, skip reinit", __func__, identifier_.c_str());
105 2 : return HCCL_SUCCESS;
106 : }
107 :
108 11 : CHK_PTR_NULL(commParam);
109 11 : identifier_ = commParam->hcomId;
110 11 : isCustom_ = isCustom;
111 11 : HCCL_RUN_INFO(
112 : "[HcclCommAicpu][Init]Entry-Init group[%s], rankSize[%u], isCustom[%d].", identifier_.c_str(),
113 : commParam->rankSize, isCustom_);
114 11 : CHK_RET(aicpuShareData_.Init(commParam->aicpuCustomParamAddr, commParam->aicpuCustomParamSize));
115 11 : CHK_RET(SetHrtWorkMode(commParam));
116 11 : CHK_RET(SetHrtDeviceSatMode(commParam));
117 11 : CHK_RET(InitConfigInfo(commParam));
118 11 : CHK_RET(InitCclbuffer(commParam));
119 11 : CHK_RET(InitTopoInfo(commParam));
120 11 : CHK_RET(InitOpNotifyObj(commParam));
121 11 : CHK_RET(HcclDispatcherAicpuInit(&dispatcher_, devId_, commParam->hcclSdmaQos, DispatcherType::DISPATCHER_AICPU));
122 11 : CHK_RET(RegisterProfilingCallback());
123 11 : CHK_RET(InitLocalNotifyObj(commParam));
124 11 : CHK_RET(InitMainStreamObj(commParam));
125 11 : CHK_RET(InitSlaveStreamObjs(commParam));
126 11 : CHK_RET(InitOrderStreamObj(commParam));
127 11 : CHK_RET(InitLocalTagRes(commParam->localRes.nextTagRes));
128 11 : CHK_RET(InitTimeOutConfig(commParam));
129 11 : CHK_RET(InitHostDeviceLock(commParam));
130 11 : CHK_RET(InitTopoMatcher());
131 11 : CHK_RET(InitOpRetry(commParam));
132 11 : CHK_RET(RegisterDispatcherCallback());
133 11 : CHK_RET(InitTinyMem(commParam));
134 11 : CHK_RET(InitProfResource());
135 11 : CHK_RET(InitZeroCopyExchanger(commParam));
136 11 : CHK_RET(InitOpCounter(commParam->opCounterInfo));
137 11 : CHK_RET(InitUtraceInfo(commParam));
138 11 : CHK_RET(aicpuCacheManager_.InitOpUnfoldCache());
139 11 : CHK_RET(RegisterProfCallBack());
140 11 : InitCommInfoStatus(true);
141 11 : SetCommInfoStreamStatus(true);
142 :
143 11 : initialized_ = true;
144 :
145 11 : HCCL_RUN_INFO("[HcclCommAicpu][Init] group[%s] success!", identifier_.c_str());
146 11 : return HCCL_SUCCESS;
147 : }
148 :
149 11 : HcclResult HcclCommAicpu::InitUtraceInfo(const HcclOpResParam* commParam)
150 : {
151 11 : u32 hostpid = 0;
152 11 : u32 cpType = DEVDRV_PROCESS_CPTYPE_MAX;
153 11 : CHK_RET(HrtHalDrvQueryProcessHostPid(getpid(), nullptr, nullptr, &hostpid, &cpType));
154 :
155 : HcclTraceInfo::UtraceAttr utraceAttr;
156 11 : utraceAttr.utraceStatusFlag = commParam->utraceStatusFlag;
157 11 : utraceAttr.deviceid = GetDevId();
158 11 : utraceAttr.pid = hostpid;
159 11 : UtraceInfo_.reset(new (std::nothrow) HcclTraceInfo(utraceAttr));
160 11 : CHK_PTR_NULL(UtraceInfo_);
161 :
162 : /* 申请trace资源信息 */
163 11 : std::string logInfo = "HCCL_";
164 11 : logInfo.append(std::to_string(SalGetTid()));
165 11 : logInfo.append("_");
166 11 : logInfo.append(std::to_string(GetDevId()));
167 11 : CHK_RET(UtraceInfo_->Init(logInfo));
168 11 : return HCCL_SUCCESS;
169 11 : }
170 :
171 11 : HcclResult HcclCommAicpu::InitProfResource()
172 : {
173 11 : groupHashId_ = dfx::ProfilingManager::GetProfHashId(identifier_.c_str(), identifier_.length());
174 11 : HCCL_RUN_INFO("[Init][ProfResource]group[%s], groupHashId_[%llu].", identifier_.c_str(), groupHashId_);
175 :
176 11 : dfx::ProfCommInfo profInfo{groupHashId_, topoInfo_.userRankSize, topoInfo_.userRank};
177 11 : CHK_RET(dfx::ProfilingManager::AddProfInfoByStreamId(mainStream_.id(), identifier_, profInfo));
178 451 : for (auto& slaveStream : slaveStreams_) {
179 440 : CHK_RET(dfx::ProfilingManager::AddProfInfoByStreamId(slaveStream.id(), identifier_, profInfo));
180 : }
181 11 : dfx::ProfilingExtendInfoHelper::InitProfItemId();
182 11 : return HCCL_SUCCESS;
183 : }
184 :
185 462 : HcclResult HcclCommAicpu::StreamRestore(u32 streamId)
186 : {
187 462 : HcclResult ret = hrtHalResourceIdRestore(devId_, 0, DRV_STREAM_ID, streamId, 0);
188 : // custom进程需要恢复stream资源, custom进程调用失败直接报错，aicpu进程调用失败做兼容性处理
189 462 : if (ret == HCCL_E_NOT_SUPPORT) {
190 0 : CHK_PRT_RET(
191 : isCustom_,
192 : HCCL_ERROR(
193 : "%s hrtHalResourceIdRestore fail, drv not support, custom[%d], ret[%d]", __func__, isCustom_, ret),
194 : HCCL_E_DRV);
195 462 : } else if (ret != HCCL_SUCCESS) {
196 0 : HCCL_ERROR("%s hrtHalResourceIdRestore fail, ret[%d]", __func__, ret);
197 0 : return HCCL_E_DRV;
198 : }
199 462 : return HCCL_SUCCESS;
200 : }
201 :
202 11 : HcclResult HcclCommAicpu::SetHrtWorkMode(const HcclOpResParam* commParam)
203 : {
204 11 : CHK_RET(hrtSetWorkModeAicpu(true));
205 11 : CHK_RET(hrtSetlocalDevice(commParam->topoInfo.deviceLogicId));
206 11 : CHK_RET(hrtSetlocalDeviceType(static_cast<DevType>(commParam->topoInfo.deviceType)));
207 11 : return HCCL_SUCCESS;
208 : }
209 :
210 11 : HcclResult HcclCommAicpu::SetHrtDeviceSatMode(const HcclOpResParam* commParam)
211 : {
212 11 : CHK_RET(hrtSetLocalDeviceSatMode(commParam->config.floatOverflowMode));
213 11 : HCCL_RUN_INFO(
214 : "[HcclCommAicpu][Init]SetHrtDeviceSatMode[%d]", static_cast<u32>(commParam->config.floatOverflowMode));
215 11 : return HCCL_SUCCESS;
216 : }
217 :
218 11 : HcclResult HcclCommAicpu::InitTopoMatcher()
219 : {
220 11 : externalEnable_.enableFfts = 1; // FFTS+在算法模块不会使用多线程，固定使能
221 11 : externalEnable_.deterministic = deterministic_;
222 11 : externalEnable_.intraRoceSwitch = 0;
223 11 : externalEnable_.dumpDebug = dumpDebug_;
224 11 : externalEnable_.interHccsDisable = interHccsDisable_;
225 :
226 11 : topoMatcher_.reset((new (std::nothrow) TopoMatcher(
227 22 : commPlaneVector_, isBridgeVector_, topoInfo_, algoInfo_, externalEnable_, serverAndsuperPodToRank_)));
228 11 : CHK_SMART_PTR_NULL(topoMatcher_);
229 11 : HCCL_RUN_INFO(
230 : "[HcclCommAicpu][InitTopoMatcher]topo matcher init success. group[%s] deterministic:%u, "
231 : "dumpDebug:%u, interHccsDisable:%u",
232 : identifier_.c_str(), deterministic_, dumpDebug_, interHccsDisable_);
233 11 : return HCCL_SUCCESS;
234 : }
235 :
236 11 : HcclResult HcclCommAicpu::InitOpRetry(const HcclOpResParam* commParam)
237 : {
238 11 : CHK_PTR_NULL(commParam);
239 11 : retryEnable_ = (commParam->config.retryEnable == 1) ? true : false;
240 11 : retryHoldTime_ = commParam->config.retryHoldTime;
241 11 : retryIntervalTime_ = commParam->config.retryIntervalTime;
242 11 : HCCL_RUN_INFO(
243 : "[InitOpRetry]retryEnable[%d], retryHoldTime[%u ms], retryIntervalTime[%u ms]", retryEnable_, retryHoldTime_,
244 : retryIntervalTime_);
245 :
246 11 : if (commParam->kfcControlTransferH2DParams.buffLen != 0 && kfcControlTransferH2D_ == nullptr) {
247 0 : EXCEPTION_CATCH((kfcControlTransferH2D_ = std::make_shared<hccl::HDCommunicate>()), return HCCL_E_PTR);
248 0 : CHK_SMART_PTR_NULL(kfcControlTransferH2D_);
249 0 : CHK_RET(kfcControlTransferH2D_->InitDevice(commParam->kfcControlTransferH2DParams));
250 : }
251 11 : if (commParam->kfcStatusTransferD2HParams.buffLen != 0 && kfcStatusTransferD2H_ == nullptr) {
252 0 : EXCEPTION_CATCH((kfcStatusTransferD2H_ = std::make_shared<hccl::HDCommunicate>()), return HCCL_E_PTR);
253 0 : CHK_SMART_PTR_NULL(kfcStatusTransferD2H_);
254 0 : CHK_RET(kfcStatusTransferD2H_->InitDevice(commParam->kfcStatusTransferD2HParams));
255 : }
256 11 : return HCCL_SUCCESS;
257 : }
258 :
259 11 : HcclResult HcclCommAicpu::InitZeroCopyExchanger(const HcclOpResParam* commParam)
260 : {
261 11 : CHK_PTR_NULL(commParam);
262 0 : auto nSecStopFunc = [this]() -> bool {
263 : // 检查到OP状态不Ok则认为需要终止
264 0 : auto ret = this->CheckOpExecStatus();
265 0 : if (ret == HCCL_SUCCESS) {
266 0 : return false;
267 0 : } else if (ret == HCCL_E_SUSPENDING) {
268 : // NS快恢场景，需要提前终止
269 0 : HcclOpExecFSM fsmState = HcclOpExecFSM::HCCL_OP_EXEC_FSM_INIT;
270 0 : KfcError errorCode = KfcError::kNone;
271 0 : SetCommRecoveryFlag(true);
272 0 : UpdateOpExecStatus(fsmState, KfcStatus::kStoplaunch, errorCode, 0);
273 0 : HCCL_RUN_INFO("[HcclCommAicpu][nSecStopFunc] need stop launch");
274 0 : return true;
275 : } else {
276 0 : return true;
277 : }
278 11 : };
279 :
280 11 : u32 timeoutSec = commParam->config.notifyWaitTime;
281 11 : HCCL_INFO("[HcclCommAicpu][InitZeroCopyExchanger] set timeout is [%u s]", timeoutSec);
282 :
283 11 : EXCEPTION_CATCH(
284 : (ZeroCopyExchanger_ = std::make_shared<hccl::AicpuZeroCopyExchanger>(
285 : commParam->localUsrRankId, commParam->rankSize, commParam, nSecStopFunc, timeoutSec,
286 : topoInfo_.deviceNumPerAggregation, taskMonitorInterval_)),
287 : return HCCL_E_PTR);
288 :
289 : // 通信域第一次初始化时，如果IPC内存有不为空的则认为是使能该特性
290 11 : isZeroCopy_ = false;
291 363 : for (u32 i = 0; i < AICPU_ZERO_COPY_MAX_DEVICE_NUM_A3; ++i) {
292 352 : if (commParam->zeroCopyIpcPtrs[i] != 0) {
293 0 : isZeroCopy_ = true;
294 0 : break;
295 : }
296 : }
297 :
298 11 : return HCCL_SUCCESS;
299 : }
300 :
301 11 : HcclResult HcclCommAicpu::InitOpCounter(const OpCounterInfo& opCounterInfo)
302 : {
303 11 : if (opCounterInfo.isEnableCounter && !retryEnable_
304 0 : && (opCounterInfo.headCountMem == 0 || opCounterInfo.tailCountMem == 0 || opCounterInfo.memSize == 0)) {
305 0 : HCCL_ERROR("[HcclCommAicpu][InitOpCounter] headCountMem or tailCountMem or memSize is null");
306 0 : return HCCL_E_PARA;
307 : }
308 11 : opCounterInfo_ = opCounterInfo;
309 11 : return HCCL_SUCCESS;
310 : }
311 :
312 1 : void HcclCommAicpu::SetZeroCopyEnable(bool enable) { isZeroCopy_ = enable; }
313 :
314 1 : void HcclCommAicpu::SetSymmetricMemoryEnable(bool enable)
315 : {
316 1 : HCCL_INFO("[HcclCommAicpu::SetSymmetricMemoryEnable] enable[%d]", enable);
317 1 : isSymmetricMemory_ = enable;
318 1 : aicpuCacheManager_.SetSymmetricMemoryEnable(enable);
319 1 : }
320 :
321 0 : HcclResult HcclCommAicpu::PrepareZeroCopyExchanger(
322 : const std::string& newTag, OpParam& opParam, AlgResourceResponse* algResResponse)
323 : {
324 0 : return ZeroCopyExchanger_->ExchangeAddress(newTag, opParam.inputPtr, opParam.outputPtr, algResResponse);
325 : }
326 :
327 11 : HcclResult HcclCommAicpu::RegisterProfilingCallback()
328 : {
329 11 : (void)RegisterLoadTaskCallBack(dispatcher_, nullptr, dfx::TaskProfilingCallBack);
330 :
331 11 : return HCCL_SUCCESS;
332 : }
333 :
334 11 : HcclResult HcclCommAicpu::RegisterDispatcherCallback()
335 : {
336 0 : auto checkOpExecStatusCallback = [this]() {
337 0 : return this->CheckOpExecStatusCallback();
338 11 : };
339 :
340 11 : return HcclSetOpExecStatusCallback(dispatcher_, checkOpExecStatusCallback);
341 : }
342 :
343 11 : HcclResult HcclCommAicpu::RegisterProfCallBack()
344 : {
345 11 : if (MsprofRegisterCallback != nullptr) {
346 11 : HCCL_INFO("RegisterProfCallBack not null");
347 11 : int32_t ret = MsprofRegisterCallback(AICPU, &DeviceCommandHandle);
348 11 : CHK_PRT_RET((ret != 0), HCCL_ERROR("[%s] failed. ret = [%d]", __func__, ret), HCCL_E_PARA);
349 : } else {
350 0 : HCCL_INFO("RegisterProfCallBack is null");
351 : }
352 11 : return HCCL_SUCCESS;
353 : }
354 :
355 0 : HcclResult HcclCommAicpu::GetSuspendingFlag(HcclComSuspendingFlag& flag)
356 : {
357 0 : CHK_RET(AicpuHdcUtils::GetSuspendingStatus(kfcControlTransferH2D_, flag));
358 0 : return HCCL_SUCCESS;
359 : }
360 :
361 57 : HcclResult HcclCommAicpu::BackGroundGetCmd(KfcCommand& cmd)
362 : {
363 57 : CHK_RET(aicpuHdc_.GetOpExecCtrlCmd(kfcControlTransferH2D_, cmd));
364 54 : return HCCL_SUCCESS;
365 : }
366 :
367 2 : HcclResult HcclCommAicpu::BackGroundSetStatus(KfcStatus status)
368 : {
369 2 : CHK_RET(aicpuHdc_.SetOpExecStatus(kfcStatusTransferD2H_, status, KfcError::kNone, 0));
370 2 : return HCCL_SUCCESS;
371 : }
372 :
373 0 : HcclResult HcclCommAicpu::SaveTraceInfo(std::string& logInfo)
374 : {
375 0 : CHK_PTR_NULL(UtraceInfo_);
376 0 : CHK_RET(UtraceInfo_->SaveTraceInfo(logInfo, AtraceOption::Opbasekey));
377 0 : return HCCL_SUCCESS;
378 : }
379 :
380 0 : HcclResult HcclCommAicpu::FlushUtraceInfo()
381 : {
382 0 : if (GetCommInfoStatus()) {
383 0 : CHK_RET(UtraceInfo_->Flush());
384 : }
385 0 : return HCCL_SUCCESS;
386 : }
387 :
388 0 : std::string HcclCommAicpu::GetExcuteOp()
389 : {
390 0 : std::stringstream ss;
391 0 : ss << "tag: " << excuteOpId_.tag << ", ";
392 0 : ss << "newTag: " << excuteOpId_.newTag << ", ";
393 0 : ss << "index: " << excuteOpId_.index;
394 0 : return ss.str();
395 0 : }
396 :
397 11 : void HcclCommAicpu::InitCommInfoStatus(bool commInfo) { commOpenStatus = commInfo; }
398 :
399 11 : HcclResult HcclCommAicpu::InitTinyMem(const HcclOpResParam* commParam)
400 : {
401 11 : CHK_PTR_NULL(commParam);
402 11 : auto tinyMemPtr = reinterpret_cast<void*>(commParam->tinyMem);
403 11 : tinySendRecvMem_ = DeviceMem::create(tinyMemPtr, commParam->tinyMemSize);
404 :
405 11 : return HCCL_SUCCESS;
406 : }
407 :
408 11 : HcclResult HcclCommAicpu::InitTimeOutConfig(const HcclOpResParam* commParam)
409 : {
410 11 : CHK_PTR_NULL(commParam);
411 11 : CHK_RET(HcclSetSqeTimeOut(dispatcher_, commParam->config.notifyWaitTime));
412 11 : linkTimeOut_ = commParam->config.linkTimeOut;
413 11 : return HCCL_SUCCESS;
414 : }
415 :
416 11 : HcclResult HcclCommAicpu::InitHostDeviceLock(const HcclOpResParam* commParam)
417 : {
418 11 : CHK_PTR_NULL(commParam);
419 11 : hostDeviceLock_.reset(new (std::nothrow) PetersonLock(
420 11 : reinterpret_cast<void*>(commParam->lockAddr), PetersonLock::DEFAULT_LOCK_TIMEOUT_SEC));
421 11 : CHK_SMART_PTR_NULL(hostDeviceLock_);
422 11 : CHK_RET(hostDeviceLock_->Init());
423 :
424 11 : return HCCL_SUCCESS;
425 : }
426 :
427 0 : HcclResult HcclCommAicpu::UpdateNotifyWaitTimeOut(SyncMode syncMode, u64 notifyWaitTime)
428 : {
429 0 : sqeWaitTimeOut_ = (notifyWaitTime == 0) ? notifyWaitTime : (notifyWaitTime + AICPU_SQE_TIMEOUT_INC);
430 0 : if (syncMode == SyncMode::UNLIMITED_TIMEWAITSYNCMODE) {
431 0 : CHK_RET(HcclSetSqeTimeOut(dispatcher_, GetNotifyMaxWaitTime()));
432 : }
433 0 : return HcclSetSqFullWaitTimeOut(dispatcher_, notifyWaitTime);
434 : }
435 :
436 0 : void HcclCommAicpu::PrepareOpRetryHandler(
437 : u8 inplaceSupportRetry, u8 retryEnable, u8 inPlaceSupportRetryStatus, u8 isInplacePreSync, u8 isPostSync)
438 : {
439 0 : algOpContext_.opRetryHandler.inplaceSupportRetry = static_cast<bool>(inplaceSupportRetry);
440 0 : algOpContext_.opRetryHandler.retryEnable = static_cast<bool>(retryEnable);
441 : algOpContext_.opRetryHandler.inPlaceSupportRetryStatus
442 0 : = static_cast<InplaceSupportRetryStatus>(inPlaceSupportRetryStatus);
443 0 : algOpContext_.opRetryHandler.isInplacePreSync = static_cast<bool>(isInplacePreSync);
444 0 : algOpContext_.opRetryHandler.isPostSync = static_cast<bool>(isPostSync);
445 0 : HCCL_INFO(
446 : "[HcclCommAicpu][PrepareOpRetryHandler] inplaceSupportRetry %d, retryEnable %d, "
447 : "inPlaceSupportRetryStatus %d, isInplacePreSync %d, isPostSync %d.",
448 : algOpContext_.opRetryHandler.inplaceSupportRetry, algOpContext_.opRetryHandler.retryEnable,
449 : algOpContext_.opRetryHandler.inPlaceSupportRetryStatus, algOpContext_.opRetryHandler.isInplacePreSync,
450 : algOpContext_.opRetryHandler.isPostSync);
451 0 : }
452 :
453 0 : HcclResult HcclCommAicpu::UpdateOpRingBufferIdx()
454 : {
455 0 : return HcclSetOpRingBufferIdx(dispatcher_, aicpuShareData_.GetOpRingBufferIdx());
456 : }
457 :
458 0 : HcclResult HcclCommAicpu::InvokeKfcHandler(AicpuKfcHandlerType type, const std::vector<u64> args)
459 : {
460 0 : CHK_PRT_RET(
461 : static_cast<size_t>(type) >= static_cast<size_t>(AicpuKfcHandlerType::kMax),
462 : HCCL_ERROR("Device mode %u, handler type %u.", static_cast<u32>(isDeviceMode_), static_cast<size_t>(type)),
463 : HCCL_E_INTERNAL);
464 0 : const auto handler = kfcHandlers_[static_cast<size_t>(type)];
465 0 : if (handler == nullptr) {
466 0 : return HCCL_SUCCESS;
467 : }
468 0 : return handler(args);
469 0 : }
470 :
471 0 : HcclResult HcclCommAicpu::NotifyPost(void)
472 : {
473 0 : if (isDeviceMode_) {
474 0 : return InvokeKfcHandler(
475 : AicpuKfcHandlerType::kNotifyRecord,
476 0 : {rpc_, reinterpret_cast<u64>(dispatcher_), reinterpret_cast<u64>(&mainStream_)});
477 : } else {
478 0 : CHK_RET(LocalNotify::Post(mainStream_, dispatcher_, opNotifies_[1]));
479 0 : HcclSqeContext* sqeContext = mainStream_.GetSqeContextPtr();
480 0 : SqeRingBuffer* sqeContextBuffer = &(sqeContext->buffer);
481 0 : return dfx::ProfilingManager::ReportMainStreamTask(mainStream_, sqeContextBuffer->tailSqeTaskId - 1, TAIL_TASK);
482 : }
483 : }
484 :
485 0 : HcclResult HcclCommAicpu::NotifyWait(void)
486 : {
487 0 : if (isDeviceMode_) {
488 0 : return InvokeKfcHandler(
489 : AicpuKfcHandlerType::kNotifyWait,
490 0 : {rpc_, reinterpret_cast<u64>(dispatcher_), reinterpret_cast<u64>(&mainStream_)});
491 : } else {
492 0 : HcclSqeContext* sqeContext = mainStream_.GetSqeContextPtr();
493 0 : SqeRingBuffer* sqeContextBuffer = &(sqeContext->buffer);
494 0 : CHK_RET(dfx::ProfilingManager::ReportMainStreamTask(mainStream_, sqeContextBuffer->tailSqeTaskId, HEAD_TASK));
495 0 : return LocalNotify::Wait(mainStream_, dispatcher_, opNotifies_[0]);
496 : }
497 : }
498 :
499 : // 按照算子模式来Post对应的Notify
500 0 : HcclResult HcclCommAicpu::RecordHostOrder(const HcclOpResParam* commParam, const std::string& tag, u8 orderLaunchMode)
501 : {
502 0 : const u8 orderLaunchInvalidInHcom = 255;
503 0 : if (orderLaunchMode == orderLaunchInvalidInHcom) {
504 0 : HCCL_INFO("[%s] attachedStreams_[%d] is invalid in graph mode", __func__, orderStream_.id());
505 0 : return HCCL_SUCCESS;
506 : }
507 0 : if (orderNotifies_[orderLaunchMode] == nullptr) {
508 0 : std::shared_ptr<LocalNotify> notify;
509 0 : HcclSignalInfo* aicpuOrderNotify = reinterpret_cast<HcclSignalInfo*>(
510 0 : static_cast<u64>(commParam->aicpuOrderNotifyAddr) + (sizeof(HcclSignalInfo) * orderLaunchMode));
511 0 : HCCL_INFO(
512 : "[%s] attachedStreams_[%d] aicpuOrderNotify resId[%llu], addr[0x%llx], flag[%u], devId[%u], tsId[%u], "
513 : "rankId[%u]",
514 : __func__, orderStream_.id(), aicpuOrderNotify->resId, aicpuOrderNotify->addr, aicpuOrderNotify->flag,
515 : aicpuOrderNotify->devId, aicpuOrderNotify->tsId, aicpuOrderNotify->rankId);
516 0 : HcclResult ret = InitAndVerifySingleSignal(*aicpuOrderNotify, notify);
517 0 : CHK_PRT_RET(
518 : ret != HCCL_SUCCESS,
519 : HCCL_ERROR(
520 : "[%s] check localRes noftify failed, resId[%u], group[%s]", __func__, aicpuOrderNotify->resId,
521 : identifier_.c_str()),
522 : ret);
523 0 : orderNotifies_[orderLaunchMode] = notify;
524 0 : HCCL_INFO("%s success, group[%s], resId[%llu]", __func__, identifier_.c_str(), aicpuOrderNotify->resId);
525 0 : }
526 :
527 0 : HCCL_INFO(
528 : "%s group[%s] tag[%s] isDeviceMode[%d] orderLaunchMode[%u] mode[%d] streamId[%d] notifyId[%u]", __func__,
529 : identifier_.c_str(), tag.c_str(), isDeviceMode_, orderLaunchMode, GetWorkflowMode(), orderStream_.id(),
530 : orderNotifies_[orderLaunchMode]->notifyId_);
531 0 : CHK_RET(LocalNotify::Post(orderStream_, dispatcher_, orderNotifies_[orderLaunchMode]));
532 0 : CHK_RET(LaunchTask(dispatcher_, const_cast<Stream&>(orderStream_)));
533 0 : return HCCL_SUCCESS;
534 : }
535 :
536 462 : HcclResult HcclCommAicpu::GetStreamData(
537 : const HcclStreamInfo& streamInfo, HcclComStreamInfo& comStreamInfo, u32& sqHead, u32& sqTail)
538 : {
539 462 : comStreamInfo.sqId = streamInfo.sqIds;
540 462 : comStreamInfo.actualStreamId = streamInfo.streamIds;
541 462 : comStreamInfo.logicCqId = streamInfo.logicCqids;
542 462 : u64 sq_addr = 0;
543 462 : CHK_RET(QuerySqBaseAddr(devId_, streamInfo.sqIds, sq_addr));
544 462 : comStreamInfo.sqBaseAddr = reinterpret_cast<void*>(sq_addr);
545 462 : if (comStreamInfo.sqBaseAddr == nullptr) {
546 0 : HCCL_ERROR("[HcclCommAicpu][GetStreamData]sqe base addr ptr is null.");
547 0 : return HCCL_E_PARA;
548 : }
549 462 : CHK_RET(QuerySqStatusByType(devId_, streamInfo.sqIds, DRV_SQCQ_PROP_SQ_DEPTH, comStreamInfo.sqDepth));
550 462 : CHK_RET(QuerySqStatusByType(devId_, streamInfo.sqIds, DRV_SQCQ_PROP_SQ_TAIL, sqTail));
551 462 : CHK_RET(QuerySqStatusByType(devId_, streamInfo.sqIds, DRV_SQCQ_PROP_SQ_HEAD, sqHead));
552 462 : HCCL_DEBUG(
553 : "[HcclCommAicpu][GetStreamData] get stream data success, group[%s], streamId[%d], sqId[%d], "
554 : "logicCqId[%u], sqDepth[%u], sqHead[%u], sqTail[%u]",
555 : identifier_.c_str(), comStreamInfo.actualStreamId, comStreamInfo.sqId, comStreamInfo.logicCqId,
556 : comStreamInfo.sqDepth, sqHead, sqTail);
557 462 : return HCCL_SUCCESS;
558 : }
559 :
560 11 : HcclResult HcclCommAicpu::InitMainStreamObj(const HcclOpResParam* commParam)
561 : {
562 11 : CHK_RET(InitStreamObj(commParam->localRes.mainStreamParam, mainStream_));
563 11 : HCCL_INFO("%s success, group[%s], streamId[%d]", __func__, identifier_.c_str(), mainStream_.id());
564 11 : return HCCL_SUCCESS;
565 : }
566 :
567 11 : HcclResult HcclCommAicpu::InitSlaveStreamObjs(const HcclOpResParam* commParam)
568 : {
569 11 : if (commParam->localRes.streamNum > LOCAL_STREAM_MAX_NUM) {
570 0 : HCCL_ERROR(
571 : "[HcclCommAicpu][InitSlaveStreamObjs] local streams great max numbers,current numbers[%u], max "
572 : "numbers[%u], group[%s]",
573 : commParam->localRes.streamNum, LOCAL_STREAM_MAX_NUM, identifier_.c_str());
574 0 : return HCCL_E_PARA;
575 : }
576 451 : for (u32 i = 0; i < commParam->localRes.streamNum; i++) {
577 440 : if (streamToObj_.find(commParam->localRes.streamParam[i].streamInfo.sqIds) == streamToObj_.end()) {
578 440 : Stream stream;
579 440 : CHK_RET(InitStreamObj(commParam->localRes.streamParam[i], stream));
580 440 : slaveStreams_.emplace_back(stream);
581 440 : streamToObj_.insert(commParam->localRes.streamParam[i].streamInfo.sqIds);
582 440 : }
583 : }
584 11 : HCCL_DEBUG(
585 : "[HcclCommAicpu][InitSlaveStreamObjs] success, group[%s], slave stream numbers[%u]", identifier_.c_str(),
586 : commParam->localRes.streamNum);
587 11 : return HCCL_SUCCESS;
588 : }
589 :
590 11 : HcclResult HcclCommAicpu::InitOrderStreamObj(const HcclOpResParam* commParam)
591 : {
592 11 : CHK_RET(InitStreamObj(commParam->aicpuOrderStreamParam, orderStream_));
593 11 : HCCL_INFO("%s success, group[%s], streamId[%d]", __func__, identifier_.c_str(), orderStream_.id());
594 11 : return HCCL_SUCCESS;
595 : }
596 :
597 462 : HcclResult HcclCommAicpu::InitStreamObj(const HcclStreamParam& streamParam, Stream& stream)
598 : {
599 : u32 sqTail;
600 : u32 sqHead;
601 462 : HcclResult ret = HCCL_SUCCESS;
602 462 : ret = StreamRestore(streamParam.streamInfo.streamIds);
603 462 : CHK_PRT_RET(
604 : ret != HCCL_SUCCESS,
605 : HCCL_ERROR(
606 : "%s StreamId[%d] Restore error group[%s]", __func__, streamParam.streamInfo.streamIds, identifier_.c_str()),
607 : ret);
608 :
609 462 : HcclComStreamInfo comStreamInfo = {0};
610 462 : ret = GetStreamData(streamParam.streamInfo, comStreamInfo, sqHead, sqTail);
611 462 : CHK_PRT_RET(ret != HCCL_SUCCESS, HCCL_ERROR("%s error group[%s]", __func__, identifier_.c_str()), ret);
612 462 : stream = Stream(comStreamInfo);
613 :
614 : // 初始化stream的sqeContext
615 462 : SqCqeContext* sqCqeContext = reinterpret_cast<SqCqeContext*>(streamParam.sqCqContextAddr);
616 462 : u64 sqCqContextSize = streamParam.sqCqContextSize;
617 462 : if (sqCqeContext == nullptr || sqCqContextSize != sizeof(SqCqeContext)) {
618 0 : HCCL_ERROR(
619 : "%s failed, sqCqeContext[%p] is null or size[%llu] does not match expected size[%llu], group[%s]", __func__,
620 : sqCqeContext, sqCqContextSize, sizeof(SqCqeContext), identifier_.c_str());
621 0 : return HCCL_E_PARA;
622 : }
623 462 : ret = stream.InitSqAndCqeContext(sqHead, sqTail, sqCqeContext);
624 462 : CHK_PRT_RET(
625 : ret != HCCL_SUCCESS, HCCL_ERROR("%s: InitSqAndCqeContext failed, group[%s]", __func__, identifier_.c_str()),
626 : ret);
627 462 : HCCL_INFO("%s success, group[%s], streamId[%d]", __func__, identifier_.c_str(), stream.id());
628 462 : return HCCL_SUCCESS;
629 : }
630 :
631 11 : HcclResult HcclCommAicpu::InitOpNotifyObj(const HcclOpResParam* commParam)
632 : {
633 11 : HcclResult ret = HCCL_SUCCESS;
634 33 : for (u32 i = 0; i < AICPU_OP_NOTIFY_MAX_NUM; i++) {
635 22 : ret = InitAndVerifySignal(commParam->localRes.aicpuOpNotify[i], opNotifies_);
636 22 : CHK_PRT_RET(
637 : ret != HCCL_SUCCESS,
638 : HCCL_ERROR(
639 : "[HcclCommAicpu][InitOpNotifyObj] check localRes op noftify failed, resId[%u], group[%s]",
640 : commParam->localRes.localSignals[i].resId, identifier_.c_str()),
641 : ret);
642 : }
643 11 : CHK_RET(hrtDrvGetLocalDevIDByHostDevID(commParam->localRes.aicpuOpNotify[0].devId, &devId_));
644 11 : HCCL_INFO("[HcclCommAicpu][InitOpNotifyObj] success, group[%s]", identifier_.c_str());
645 11 : return HCCL_SUCCESS;
646 : }
647 :
648 11 : HcclResult HcclCommAicpu::InitLocalNotifyObj(const HcclOpResParam* commParam)
649 : {
650 11 : HcclResult ret = HCCL_SUCCESS;
651 :
652 11 : if (commParam->localRes.signalNum > LOCAL_NOTIFY_MAX_NUM) {
653 0 : HCCL_ERROR(
654 : "[HcclCommAicpu][InitLocalNotifyObj] local notifys great max numbers, numbers[%u], max numbers[%u], "
655 : "group[%s]",
656 : commParam->localRes.signalNum, LOCAL_NOTIFY_MAX_NUM, identifier_.c_str());
657 0 : return HCCL_E_PARA;
658 : }
659 715 : for (u32 i = 0; i < commParam->localRes.signalNum; i++) {
660 704 : if (notifysToObj_.find(commParam->localRes.localSignals[i].resId) == notifysToObj_.end()) {
661 704 : ret = InitAndVerifySignal(commParam->localRes.localSignals[i], localNotifies_);
662 704 : CHK_PRT_RET(
663 : ret != HCCL_SUCCESS,
664 : HCCL_ERROR(
665 : "[HcclCommAicpu][InitLocalNotifyObj] check localRes prenoftify failed, resId[%u]",
666 : commParam->localRes.localSignals[i].resId),
667 : ret);
668 704 : notifysToObj_.insert(commParam->localRes.localSignals[i].resId);
669 : }
670 : }
671 11 : HCCL_DEBUG(
672 : "[HcclCommAicpu][InitLocalNotifyObj] success, group[%s], signal numbers[%u]", identifier_.c_str(),
673 : commParam->localRes.signalNum);
674 11 : return HCCL_SUCCESS;
675 : }
676 :
677 22 : HcclResult HcclCommAicpu::ParseTlvToVector(
678 : u64 srcTlv, u64 srcTlvTotalLength, std::vector<std::vector<std::vector<u32>>>& vectorInfo)
679 : {
680 22 : u64 parseLength = 0;
681 22 : u8* srcTlvptr = reinterpret_cast<u8*>(srcTlv);
682 22 : if (srcTlvptr == nullptr) {
683 0 : HCCL_ERROR("[HcclCommAicpu][ParseTlvToVector]tlv ptr is null.");
684 0 : return HCCL_E_PARA;
685 : }
686 22 : u64 unPareseLength = srcTlvTotalLength;
687 143 : while (unPareseLength > 0) {
688 121 : CommonTlv* commonTlv = reinterpret_cast<CommonTlv*>(srcTlvptr + parseLength);
689 121 : if (unPareseLength <= (sizeof(LENGTH_TYPE) + sizeof(TAG_TYPE))) {
690 0 : HCCL_ERROR(
691 : "[HcclCommAicpu][ParseTlvToVector] Tlv length is error, tag[%s], totalLength[%lu], "
692 : "unParseLength[%lu], already parsed Length[%lu]",
693 : identifier_.c_str(), srcTlvTotalLength, unPareseLength, parseLength);
694 0 : return HCCL_E_PARA;
695 : }
696 121 : if (commonTlv->length > unPareseLength || commonTlv->length % sizeof(RANK_TYPE) != 0) {
697 0 : HCCL_ERROR(
698 : "[HcclCommAicpu][ParseTlvToVector] parse Tlv error, group[%s], total Length[%lu], tlvLength[%lu], "
699 : "unParsed Length[%lu], already parsed Length[%lu]",
700 : identifier_.c_str(), srcTlvTotalLength, commonTlv->length, unPareseLength, parseLength);
701 0 : return HCCL_E_PARA;
702 : }
703 121 : u16 level0 = (commonTlv->type & TOP_COMM_LEVEL0_LOCATION) >> TOP_COMM_LEVEL0_SHIFT;
704 121 : u16 level1 = (commonTlv->type & TOP_COMM_LEVEL1_LOCATION);
705 121 : u64 itemNum = (commonTlv->length - sizeof(TAG_TYPE) - sizeof(LENGTH_TYPE)) / sizeof(RANK_TYPE);
706 242 : std::vector<RANK_TYPE> values{&commonTlv->value, (&commonTlv->value) + itemNum};
707 121 : if (level0 >= vectorInfo.size()) {
708 99 : vectorInfo.resize(level0 + 1);
709 : }
710 121 : if (level1 >= vectorInfo[level0].size()) {
711 121 : vectorInfo[level0].resize(level1 + 1);
712 : }
713 121 : vectorInfo[level0][level1] = std::move(values);
714 121 : parseLength += commonTlv->length;
715 121 : unPareseLength -= commonTlv->length;
716 121 : HCCL_DEBUG(
717 : "[HcclCommAicpu][ParseTlvToVector] parse Tlv group[%s], level0[%u], level1[%u], total Length[%lu], "
718 : "tlvLength[%lu], unParsed Length[%lu], already parsed Length[%lu]",
719 : identifier_.c_str(), level0, level1, srcTlvTotalLength, commonTlv->length, unPareseLength, parseLength);
720 121 : }
721 143 : for (u32 idx = 0; idx < vectorInfo.size(); idx++) {
722 242 : for (u32 ringidx = 0; ringidx < vectorInfo[idx].size(); ringidx++) {
723 121 : HCCL_DEBUG(
724 : "[HcclCommAicpu][ParseTlvToVector] idx[%u] ringidx[%u] size[%u]", idx, ringidx,
725 : vectorInfo[idx][ringidx].size());
726 : }
727 : }
728 22 : HCCL_INFO("[HcclCommAicpu][ParseTlvToVector] success, group[%s]", identifier_.c_str());
729 22 : return HCCL_SUCCESS;
730 : }
731 :
732 11 : HcclResult HcclCommAicpu::ParseTlvToSubGroupVector(
733 : u64 srcTlv, u64 srcTlvTotalLength, std::vector<std::vector<std::vector<std::vector<u32>>>>& vectorInfo)
734 : {
735 11 : u64 parseLength = 0;
736 11 : u8* srcTlvptr = reinterpret_cast<u8*>(srcTlv);
737 11 : if (srcTlvptr == nullptr) {
738 0 : HCCL_ERROR("[HcclCommAicpu][ParseTlvToSubGroupVector]tlv ptr is null.");
739 0 : return HCCL_E_PARA;
740 : }
741 11 : u64 unPareseLength = srcTlvTotalLength;
742 33 : while (unPareseLength > 0) {
743 22 : CommonTlv* commonTlv = reinterpret_cast<CommonTlv*>(srcTlvptr + parseLength);
744 22 : if (unPareseLength <= (sizeof(LENGTH_TYPE) + sizeof(TAG_TYPE))) {
745 0 : HCCL_ERROR(
746 : "[HcclCommAicpu][ParseTlvToSubGroupVector] Tlv length is error, tag[%s], totalLength[%lu], "
747 : "unParseLength[%lu], already parsed Length[%lu]",
748 : identifier_.c_str(), srcTlvTotalLength, unPareseLength, parseLength);
749 0 : return HCCL_E_PARA;
750 : }
751 22 : if (commonTlv->length > unPareseLength || commonTlv->length % sizeof(RANK_TYPE) != 0) {
752 0 : HCCL_ERROR(
753 : "[HcclCommAicpu][ParseTlvToSubGroupVector] parse Tlv error, group[%s], total Length[%lu], "
754 : "tlvLength[%lu], "
755 : "unParsed Length[%lu], already parsed Length[%lu]",
756 : identifier_.c_str(), srcTlvTotalLength, commonTlv->length, unPareseLength, parseLength);
757 0 : return HCCL_E_PARA;
758 : }
759 22 : u16 level0 = (commonTlv->type & TOP_HIERARCHICAL_COMM_LEVEL0_LOCATION)
760 22 : >> (TOP_HIERARCHICAL_COMM_LEVEL0_SHIFT + TOP_HIERARCHICAL_COMM_LEVEL1_SHIFT);
761 22 : u16 level1 = (commonTlv->type & TOP_HIERARCHICAL_COMM_LEVEL1_LOCATION) >> TOP_HIERARCHICAL_COMM_LEVEL1_SHIFT;
762 22 : u16 level2 = (commonTlv->type & TOP_HIERARCHICAL_COMM_LEVEL2_LOCATION);
763 22 : u64 itemNum = (commonTlv->length - sizeof(TAG_TYPE) - sizeof(LENGTH_TYPE)) / sizeof(RANK_TYPE);
764 44 : std::vector<RANK_TYPE> values{&commonTlv->value, (&commonTlv->value) + itemNum};
765 22 : if (level0 >= vectorInfo.size()) {
766 11 : vectorInfo.resize(level0 + 1);
767 : }
768 22 : if (level1 >= vectorInfo[level0].size()) {
769 11 : vectorInfo[level0].resize(level1 + 1);
770 : }
771 22 : if (level2 >= vectorInfo[level0][level1].size()) {
772 22 : vectorInfo[level0][level1].resize(level2 + 1);
773 : }
774 22 : vectorInfo[level0][level1][level2] = std::move(values);
775 22 : parseLength += commonTlv->length;
776 22 : unPareseLength -= commonTlv->length;
777 22 : HCCL_DEBUG(
778 : "[HcclCommAicpu][ParseTlvToSubGroupVector] parse Tlv group[%s], level0[%u], level1[%u], total Length[%lu], "
779 : "tlvLength[%lu], unParsed Length[%lu], already parsed Length[%lu]",
780 : identifier_.c_str(), level0, level1, srcTlvTotalLength, commonTlv->length, unPareseLength, parseLength);
781 22 : }
782 22 : for (u32 level0Idx = 0; level0Idx < vectorInfo.size(); level0Idx++) {
783 22 : for (u32 level1Idx = 0; level1Idx < vectorInfo[level0Idx].size(); level1Idx++) {
784 33 : for (u32 level2Idx = 0; level2Idx < vectorInfo[level0Idx][level1Idx].size(); level2Idx++) {
785 22 : HCCL_DEBUG(
786 : "[HcclCommAicpu][ParseTlvToSubGroupVector] level0Idx[%u] level1Idx[%u] level2Idx[%u] size[%u]",
787 : level0Idx, level1Idx, level2Idx, vectorInfo[level0Idx][level1Idx][level2Idx].size());
788 : }
789 : }
790 : }
791 11 : HCCL_INFO("[HcclCommAicpu][ParseTlvToSubGroupVector] success, group[%s]", identifier_.c_str());
792 11 : return HCCL_SUCCESS;
793 : }
794 :
795 11 : HcclResult HcclCommAicpu::InitTopoInfo(const HcclOpResParam* commParam)
796 : {
797 11 : topoInfo_.userRank = commParam->topoInfo.userRank;
798 11 : topoInfo_.userRankSize = commParam->topoInfo.userRankSize;
799 11 : topoInfo_.deviceLogicId = commParam->topoInfo.deviceLogicId;
800 11 : topoInfo_.isSingleMeshAggregation = commParam->topoInfo.isSingleMeshAggregation;
801 11 : topoInfo_.deviceNumPerAggregation = commParam->topoInfo.deviceNumPerAggregation;
802 11 : topoInfo_.superPodNum = commParam->topoInfo.superPodNum;
803 11 : topoInfo_.devicePhyId = commParam->topoInfo.devicePhyId;
804 11 : topoInfo_.deviceType = static_cast<DevType>(commParam->topoInfo.deviceType);
805 11 : topoInfo_.topoType = static_cast<TopoType>(commParam->topoInfo.topoType);
806 11 : topoInfo_.is310P3Common = (topoInfo_.deviceType == DevType::DEV_TYPE_310P3);
807 11 : topoInfo_.serverNum = commParam->topoInfo.serverNum;
808 11 : topoInfo_.meshAggregationRankSize = commParam->topoInfo.meshAggregationRankSize;
809 11 : topoInfo_.multiModuleDiffDeviceNumMode = commParam->topoInfo.multiModuleDiffDeviceNumMode;
810 11 : topoInfo_.multiSuperPodDiffServerNumMode = commParam->topoInfo.multiSuperPodDiffServerNumMode;
811 11 : topoInfo_.realUserRank = commParam->topoInfo.realUserRank;
812 11 : topoInfo_.isDiffDeviceModule = commParam->topoInfo.isDiffDeviceModule;
813 11 : topoInfo_.isDiffDeviceType = commParam->topoInfo.isDiffDeviceType;
814 11 : topoInfo_.gcdDeviceNumPerAggregation = commParam->topoInfo.gcdDeviceNumPerAggregation;
815 11 : topoInfo_.moduleNum = commParam->topoInfo.moduleNum;
816 11 : topoInfo_.useSuperPodMode = true;
817 11 : topoInfo_.isARSDoubleRing = commParam->isARSDoubleRing;
818 11 : topoInfo_.multiSuperPodDiffDeviceNumMode = commParam->multiSuperPodDiffDeviceNumMode;
819 11 : if (commParam->topoInfo.isUsedRdmaRankPairNum % KEY_VALUE_TO_VECTOR_MODULUS != 0) {
820 0 : HCCL_ERROR(
821 : "[HcclCommAicpu][InitTopoInfo]rdma rank pair number[%lu] is error.",
822 : commParam->topoInfo.isUsedRdmaRankPairNum);
823 0 : return HCCL_E_PARA;
824 : }
825 11 : u32* isUsedRdmaRankPairPtr = reinterpret_cast<u32*>(commParam->topoInfo.isUsedRdmaRankPair);
826 11 : if (isUsedRdmaRankPairPtr == nullptr) {
827 0 : HCCL_ERROR("[HcclCommAicpu][InitTopoInfo]rdma rank pair ptr is null.");
828 0 : return HCCL_E_PARA;
829 : }
830 55 : for (u64 i = 0; i < commParam->topoInfo.isUsedRdmaRankPairNum; i += KEY_VALUE_TO_VECTOR_MODULUS) {
831 44 : topoInfo_.isUsedRdmaMap.insert({isUsedRdmaRankPairPtr[i], static_cast<bool>(isUsedRdmaRankPairPtr[i + 1])});
832 : }
833 :
834 11 : if (commParam->topoInfo.pairLinkCounterNum % KEY_VALUE_TO_VECTOR_MODULUS != 0) {
835 0 : HCCL_ERROR(
836 : "[HcclCommAicpu][InitTopoInfo]pair link count number[%lu] is error.",
837 : commParam->topoInfo.pairLinkCounterNum);
838 0 : return HCCL_E_PARA;
839 : }
840 11 : u32* pairLinkCounterPtr = reinterpret_cast<u32*>(commParam->topoInfo.pairLinkCounter);
841 11 : if (pairLinkCounterPtr == nullptr) {
842 0 : HCCL_ERROR("[HcclCommAicpu][InitTopoInfo]rdma rank pair ptr is null.");
843 0 : return HCCL_E_PARA;
844 : }
845 :
846 55 : for (u64 i = 0; i < commParam->topoInfo.pairLinkCounterNum; i += KEY_VALUE_TO_VECTOR_MODULUS) {
847 44 : topoInfo_.pairLinkCounter.insert({pairLinkCounterPtr[i], pairLinkCounterPtr[i + 1]});
848 : }
849 :
850 11 : u32* nicListPtr = reinterpret_cast<u32*>(commParam->topoInfo.nicList);
851 11 : if (nicListPtr == nullptr) {
852 0 : HCCL_ERROR("[HcclCommAicpu][InitTopoInfo]nic list ptr is null.");
853 0 : return HCCL_E_PARA;
854 : }
855 22 : std::vector<u32> niclist{nicListPtr, nicListPtr + commParam->topoInfo.nicNum};
856 11 : topoInfo_.nicList = std::move(niclist);
857 :
858 11 : bool* bridgeRankPtr = reinterpret_cast<bool*>(commParam->topoInfo.bridgeRank);
859 11 : if (bridgeRankPtr != nullptr) {
860 11 : isBridgeVector_.resize(commParam->topoInfo.bridgeRankNum);
861 33 : for (u32 i = 0; i < commParam->topoInfo.bridgeRankNum; ++i) {
862 22 : isBridgeVector_[i] = bridgeRankPtr[i];
863 22 : HCCL_DEBUG("[HcclCommAicpu][InitTopoInfo] bridge rank info idx[%u] value[%u]", i, isBridgeVector_[i]);
864 : }
865 : } else {
866 0 : HCCL_RUN_INFO("[HcclCommAicpu][InitTopoInfo] bridge rank number is 0, group[%s]", identifier_.c_str());
867 : }
868 :
869 11 : HcclResult ret = HCCL_SUCCESS;
870 11 : ret = ParseTlvToVector(commParam->topoInfo.complanRank, commParam->topoInfo.complanRankLength, commPlaneVector_);
871 11 : CHK_PRT_RET(
872 : ret != HCCL_SUCCESS,
873 : HCCL_ERROR("[HcclCommAicpu][InitTopoInfo]Init CommPlane error group[%s]", identifier_.c_str()), ret);
874 :
875 22 : ret = ParseTlvToVector(
876 11 : commParam->topoInfo.serverAndsuperPodRank, commParam->topoInfo.serverAndsuperPodRankLength,
877 11 : serverAndsuperPodToRank_);
878 11 : CHK_PRT_RET(
879 : ret != HCCL_SUCCESS,
880 : HCCL_ERROR("[HcclCommAicpu][InitTopoInfo]Init server and superPod rank error group[%s]", identifier_.c_str()),
881 : ret);
882 :
883 22 : ret = ParseTlvToSubGroupVector(
884 11 : commParam->hierarchicalAlgInfo.commplaneSubGroupRank,
885 11 : commParam->hierarchicalAlgInfo.commplaneSubGroupRankLength, topoInfo_.CommPlaneSubGroupVector);
886 11 : CHK_PRT_RET(
887 : ret != HCCL_SUCCESS,
888 : HCCL_ERROR("[HcclCommAicpu][InitTopoInfo]Init CommPlaneSubGroup error group[%s]", identifier_.c_str()), ret);
889 :
890 11 : HCCL_INFO(
891 : "[HcclCommAicpu][InitTopoInfo] success, group[%s], device Type[%u]", identifier_.c_str(),
892 : commParam->topoInfo.deviceType);
893 11 : return HCCL_SUCCESS;
894 11 : }
895 :
896 11 : HcclResult HcclCommAicpu::InitCclbuffer(const HcclOpResParam* commParam)
897 : {
898 11 : if (commParam->localWindowsIn == 0 || commParam->localWindowsOut == 0 || cclInputBuffer_.ptr() != nullptr
899 22 : || cclOutputBuffer_.ptr() != nullptr) {
900 0 : HCCL_INFO(
901 : "[HcclCommAicpu][InitCclBuffer] don't need init cclbuffer "
902 : "ccl winIn[0x%lx] winout[0x%lx] cclin ptr[%p] cclout ptr[%p]",
903 : commParam->localWindowsIn, commParam->localWindowsOut, cclInputBuffer_.ptr(), cclOutputBuffer_.ptr());
904 0 : return HCCL_SUCCESS;
905 : }
906 :
907 11 : auto cclInPtr = reinterpret_cast<void*>(commParam->localWindowsIn);
908 11 : auto cclOutPtr = reinterpret_cast<void*>(commParam->localWindowsOut);
909 11 : cclInputBuffer_ = DeviceMem::create(cclInPtr, commParam->winSize);
910 11 : cclOutputBuffer_ = DeviceMem::create(cclOutPtr, commParam->winSize);
911 11 : cclbufferSize_ = commParam->winSize;
912 11 : HCCL_INFO(
913 : "[HcclCommAicpu][InitCclbuffer] success, group[%s], cclin[%llu], cclout[%llu], size[%lu]", identifier_.c_str(),
914 : commParam->localWindowsIn, commParam->localWindowsOut, commParam->winSize);
915 11 : return HCCL_SUCCESS;
916 : }
917 :
918 11 : HcclResult HcclCommAicpu::InitConfigInfo(const HcclOpResParam* commParam)
919 : {
920 11 : deterministic_ = commParam->config.deterministic;
921 11 : interHccsDisable_ = commParam->config.interHccsDisable;
922 11 : multiQpThreshold_ = commParam->config.multiQpThreshold;
923 11 : inlineReducEnable_ = true;
924 11 : fftsEnable_ = false;
925 11 : taskMonitorInterval_ = commParam->config.taskMonitorInterval;
926 11 : algoInfo_.inlineReduceSwitchOn = true;
927 11 : algoInfo_.identifier = commParam->hcomId;
928 11 : algoInfo_.isSupportAtomicWrite = static_cast<bool>(commParam->config.isSupportAtomicWrite);
929 11 : notifySize_ = commParam->notifysize;
930 11 : slaveStreams_.reserve(LOCAL_STREAM_MAX_NUM);
931 11 : localNotifies_.reserve(LOCAL_NOTIFY_MAX_NUM);
932 11 : SetExternalInputDebugConfig(commParam->debugConfig);
933 11 : InitDebugConfigByValue(commParam->debugConfig);
934 11 : HCCL_INFO(
935 : "[HcclCommAicpu][Init]success, group[%s] reserve noipc notifys[%lu], slave streams[%lu].", identifier_.c_str(),
936 : LOCAL_NOTIFY_MAX_NUM, LOCAL_STREAM_MAX_NUM);
937 11 : return HCCL_SUCCESS;
938 : }
939 :
940 13 : void HcclCommAicpu::SetDumpDebug(bool dumpDebug) { dumpDebug_ = dumpDebug; }
941 :
942 : template <typename T>
943 : HcclResult
944 726 : HcclCommAicpu::InitAndVerifySignal(const HcclSignalInfo& signalInfo, std::vector<std::shared_ptr<T>>& notifyVec)
945 : {
946 726 : if (signalInfo.resId == INVALID_U64) {
947 : // 无效值不做校验
948 0 : HCCL_INFO("[HcclCommAicpu][InitAndVerifySignal] resId is invalid, need not check");
949 0 : return HCCL_SUCCESS;
950 : }
951 :
952 726 : std::shared_ptr<T> notify;
953 726 : EXCEPTION_CATCH((notify = std::make_shared<T>()), return HCCL_E_PTR);
954 726 : CHK_SMART_PTR_NULL(notify);
955 726 : CHK_RET(notify->Init(signalInfo, NotifyLoadType::DEVICE_NOTIFY));
956 726 : notifyVec.push_back(notify);
957 726 : HCCL_INFO(
958 : "[HcclCommAicpu][InitAndVerifySignal] success group[%s], resId[%u], tsId:%d, devId[%u]", identifier_.c_str(),
959 : signalInfo.resId, signalInfo.tsId, signalInfo.devId);
960 :
961 726 : return HCCL_SUCCESS;
962 726 : }
963 :
964 33 : void HcclCommAicpu::HandleExistTagReAlloc(
965 : HccltagLocalResV2* tagRes, const std::string& tag, bool reAllocFlag, ListCommon*& curList, bool& needSkip)
966 : {
967 33 : needSkip = false;
968 : // 如果 tag 已经存在且需要重新申请内存, 删除旧的资源与记录
969 33 : if (localTagResToObj_.find(tag) != localTagResToObj_.end() && reAllocFlag) {
970 0 : u64 currSize = tagRes->ScratchmemSize;
971 0 : u64 existSize = tagScratchMem_[tag]->size();
972 : // 新内存更大，删除旧记录，保留新的
973 0 : if (currSize > existSize) {
974 0 : localTagResToObj_.erase(tag);
975 0 : tagScratchMem_.erase(tag);
976 0 : HCCL_DEBUG(
977 : "[HcclCommAicpu][%s] tag exists, replace old small mem, tag[%s], old[%lu] new[%lu]", __func__,
978 : tag.c_str(), existSize, currSize);
979 : } else {
980 : // 旧内存更大，需要跳过
981 0 : HCCL_DEBUG(
982 : "[HcclCommAicpu][%s] tag exists, keep larger old mem, skip new, tag[%s], old[%lu] new[%lu]", __func__,
983 : tag.c_str(), existSize, currSize);
984 0 : curList = reinterpret_cast<ListCommon*>(curList->nextDevice);
985 0 : needSkip = true;
986 : }
987 : }
988 33 : }
989 :
990 11 : HcclResult HcclCommAicpu::InitLocalTagRes(const ListCommon& head, bool reAllocFlag)
991 : {
992 11 : ListCommon* curList = reinterpret_cast<ListCommon*>(head.nextDevice);
993 11 : if (curList == nullptr) {
994 0 : HCCL_ERROR("[HcclCommAicpu][InitLocalTagRes]list ptr is null.");
995 0 : return HCCL_E_PARA;
996 : }
997 44 : while (curList != &head) {
998 33 : HccltagLocalResV2* tagRes = list_entry(curList, HccltagLocalResV2, nextTagRes);
999 33 : std::string tag = tagRes->tag;
1000 33 : bool needSkip = false;
1001 33 : HandleExistTagReAlloc(tagRes, tag, reAllocFlag, curList, needSkip);
1002 33 : if (needSkip) {
1003 0 : continue;
1004 : }
1005 33 : if (localTagResToObj_.find(tag) == localTagResToObj_.end()
1006 33 : || localTagResToObj_[tag].find(tagRes->Scratchmem) == localTagResToObj_[tag].end()) {
1007 33 : auto scratchMemPtr = reinterpret_cast<void*>(tagRes->Scratchmem);
1008 33 : if (scratchMemPtr == nullptr) {
1009 0 : HCCL_ERROR("[HcclCommAicpu][InitLocalTagRes]scratch mem ptr is null.");
1010 0 : return HCCL_E_PARA;
1011 : }
1012 33 : DeviceMem loalScratchmem = DeviceMem::create(scratchMemPtr, tagRes->ScratchmemSize);
1013 33 : std::shared_ptr<DeviceMem> loalScratchmemPtr;
1014 33 : EXCEPTION_CATCH(
1015 : (loalScratchmemPtr = std::make_shared<DeviceMem>(std::move(loalScratchmem))), return HCCL_E_PTR);
1016 33 : CHK_SMART_PTR_NULL(loalScratchmemPtr);
1017 33 : if (tagScratchMem_.find(tag) == tagScratchMem_.end()) {
1018 33 : tagScratchMem_.insert({tag, loalScratchmemPtr});
1019 : }
1020 33 : std::unordered_set<u64> tmpTagRes;
1021 33 : tmpTagRes.insert(tagRes->Scratchmem);
1022 33 : localTagResToObj_[tag] = tmpTagRes;
1023 33 : HCCL_DEBUG(
1024 : "[HcclCommAicpu][InitLocalTagRes] parse remote resource, tag[%s], Scratchmem[%p], "
1025 : "ScratchmemSize[%lu]",
1026 : tag.c_str(), tagRes->Scratchmem, tagRes->ScratchmemSize);
1027 33 : }
1028 33 : curList = reinterpret_cast<ListCommon*>(curList->nextDevice);
1029 33 : if (curList == nullptr) {
1030 0 : HCCL_ERROR("[HcclCommAicpu][InitLocalTagRes] next list ptr is null.");
1031 0 : return HCCL_E_PARA;
1032 : }
1033 33 : };
1034 11 : HCCL_INFO("[HcclCommAicpu][InitLocalTagRes] success, group[%s]", identifier_.c_str());
1035 11 : return HCCL_SUCCESS;
1036 : }
1037 :
1038 0 : HcclResult HcclCommAicpu::CheckNotifyOrQPMaxNum(u64& existNum, const u64& MaxNum, const bool& isNotifyRes)
1039 : {
1040 0 : std::string resType = isNotifyRes ? "Notify" : "QP";
1041 0 : if (existNum + 1 > MaxNum) {
1042 0 : HCCL_ERROR(
1043 : "[%s]%s resources are insufficient, existNum[%llu], MaxNum is [%llu]", __func__, resType.c_str(), existNum,
1044 : MaxNum);
1045 0 : return HCCL_E_INTERNAL;
1046 : }
1047 0 : HCCL_DEBUG(
1048 : "[%s]%s resources are sufficient, existNum[%llu], MaxNum is [%llu]", __func__, resType.c_str(), existNum,
1049 : MaxNum);
1050 0 : return HCCL_SUCCESS;
1051 0 : }
1052 :
1053 0 : HcclResult HcclCommAicpu::SetTransportMachinePara(
1054 : MachinePara& machinePara, u32& rankId, const std::string& newTag, TransportLinkType linkType)
1055 : {
1056 0 : machinePara.linkAttribute = 0x03; /* 0x03同时支持目的端和源端发起 */
1057 0 : if (rankData_.find(rankId) == rankData_.end()) {
1058 0 : HCCL_ERROR("[%s]there is no link with rankId[%u]", __func__, rankId);
1059 0 : return HCCL_E_NOT_FOUND;
1060 : }
1061 :
1062 0 : machinePara.localUserrank = localUserRank_;
1063 0 : machinePara.remoteWorldRank = rankData_[rankId].remoteWorldRank;
1064 0 : machinePara.remoteUserrank = rankData_[rankId].remoteUsrRankId;
1065 0 : machinePara.deviceLogicId = topoInfo_.deviceLogicId;
1066 0 : machinePara.localDeviceId = topoInfo_.devicePhyId;
1067 0 : machinePara.deviceType = topoInfo_.deviceType;
1068 0 : machinePara.tag = newTag;
1069 0 : machinePara.isAicpuModeEn = true;
1070 0 : if (linkType == TransportLinkType::RESERVED) {
1071 : // 非910_93 2die sio与hccs并发场景，specifyLink设置为RESERVED_LINK_TYPE，平台层将按实际链路类型建链
1072 0 : machinePara.specifyLink = LinkTypeInServer::RESERVED_LINK_TYPE;
1073 : } else {
1074 : // 910_93 2die sio与hccs并发场景，
1075 : // 并发链路中的的hccs链路specifyLink设置为HCCS_SW_TYPE，平台层将使用hccs链路来建链；
1076 : // 并发链路中的的sio链路specifyLink设置为SIO_TYPE，平台层将使用sio链路来建链
1077 : machinePara.specifyLink
1078 0 : = (linkType == TransportLinkType::SIO) ? LinkTypeInServer::SIO_TYPE : LinkTypeInServer::HCCS_SW_TYPE;
1079 : }
1080 :
1081 0 : HCCL_INFO(
1082 : "%s success, group[%s], rankId[%u], linkAttribute[%x], localUserRank[%u], remoteWorldRank[%u], "
1083 : "remoteUserrank[%u], deviceLogicId[%d], localDeviceId[%d], deviceType[%d], newTag[%s], specifyLink[%d], "
1084 : "isAicpuModeEn[%d]",
1085 : __func__, identifier_.c_str(), rankId, machinePara.linkAttribute, machinePara.localUserrank,
1086 : machinePara.remoteWorldRank, machinePara.remoteUserrank, machinePara.deviceLogicId, machinePara.localDeviceId,
1087 : machinePara.deviceType, machinePara.tag.c_str(), machinePara.specifyLink, machinePara.isAicpuModeEn);
1088 0 : return HCCL_SUCCESS;
1089 : }
1090 :
1091 : template <typename T>
1092 0 : HcclResult HcclCommAicpu::InitAndVerifySingleSignal(const HcclSignalInfo& signalInfo, std::shared_ptr<T>& notify)
1093 : {
1094 0 : if (signalInfo.resId == INVALID_U64) {
1095 : // 无效值不做校验
1096 0 : HCCL_DEBUG("[%s]resId[%llu] is invalid, need not check", __func__, signalInfo.resId);
1097 0 : return HCCL_SUCCESS;
1098 : }
1099 : HcclSignalInfo tmpSignalInfo;
1100 :
1101 0 : EXCEPTION_CATCH((notify = std::make_shared<T>()), return HCCL_E_PTR);
1102 0 : CHK_SMART_PTR_NULL(notify);
1103 0 : CHK_RET(notify->Init(signalInfo, NotifyLoadType::DEVICE_NOTIFY));
1104 0 : CHK_RET(notify->GetNotifyData(tmpSignalInfo));
1105 0 : HCCL_DEBUG(
1106 : "[%s] success group[%s], resId[%llu], tsId:%d, devId[%u]", __func__, identifier_.c_str(), tmpSignalInfo.resId,
1107 : tmpSignalInfo.tsId, tmpSignalInfo.devId);
1108 :
1109 0 : return HCCL_SUCCESS;
1110 : }
1111 :
1112 0 : HcclResult HcclCommAicpu::SetTagRemoteRes(u32& rankId, const std::string& tag, HccltagRemoteResV2* tagRes)
1113 : {
1114 0 : if (rankTagRemoteRes_.find(rankId) == rankTagRemoteRes_.end()
1115 0 : || rankTagRemoteRes_[rankId].find(tag) == rankTagRemoteRes_[rankId].end()) {
1116 0 : HccltagRemoteResV3 tempTagRemoteRes;
1117 0 : tempTagRemoteRes.tagRemoteResPtr = tagRes;
1118 0 : rankTagRemoteRes_[rankId][tag] = tempTagRemoteRes;
1119 : }
1120 0 : HCCL_DEBUG("[%s]get TagRemoteRes success, rankId[%u], tag[%s]", __func__, rankId, tag.c_str());
1121 0 : return HCCL_SUCCESS;
1122 : }
1123 :
1124 0 : HcclResult HcclCommAicpu::SetTransportPtpNotify(
1125 : TransportDeviceP2pData& transDevP2pData, u64& p2pNotifyNum, HcclLinkP2pV2& linkP2p, u32 notifyNum)
1126 : {
1127 0 : u64 actualNotifyNum = 0;
1128 : // 获取Ipc notify信息
1129 0 : CHK_RET(CheckNotifyOrQPMaxNum(actualNotifyNum, LINK_P2P_MAX_NUM, true));
1130 0 : std::shared_ptr<LocalNotify> ipcPreWaitNotify = std::make_shared<LocalNotify>();
1131 0 : CHK_RET(InitAndVerifySingleSignal(linkP2p.localIpcSignal[actualNotifyNum], ipcPreWaitNotify));
1132 0 : transDevP2pData.ipcPreWaitNotify = ipcPreWaitNotify;
1133 :
1134 0 : std::shared_ptr<RemoteNotify> ipcPreRecordNotify = std::make_shared<RemoteNotify>();
1135 0 : CHK_RET(InitAndVerifySingleSignal(linkP2p.remoteIpcSignal[actualNotifyNum], ipcPreRecordNotify));
1136 0 : transDevP2pData.ipcPreRecordNotify = ipcPreRecordNotify;
1137 0 : actualNotifyNum++;
1138 :
1139 0 : CHK_RET(CheckNotifyOrQPMaxNum(actualNotifyNum, LINK_P2P_MAX_NUM, true));
1140 0 : std::shared_ptr<LocalNotify> ipcPostWaitNotify = std::make_shared<LocalNotify>();
1141 0 : CHK_RET(InitAndVerifySingleSignal(linkP2p.localIpcSignal[actualNotifyNum], ipcPostWaitNotify));
1142 0 : transDevP2pData.ipcPostWaitNotify = ipcPostWaitNotify;
1143 :
1144 0 : std::shared_ptr<RemoteNotify> ipcPostRecordNotify = std::make_shared<RemoteNotify>();
1145 0 : CHK_RET(InitAndVerifySingleSignal(linkP2p.remoteIpcSignal[actualNotifyNum], ipcPostRecordNotify));
1146 0 : transDevP2pData.ipcPostRecordNotify = ipcPostRecordNotify;
1147 0 : actualNotifyNum++;
1148 :
1149 0 : transDevP2pData.userLocalNotify.resize(notifyNum, nullptr);
1150 0 : transDevP2pData.userRemoteNotify.resize(notifyNum, nullptr);
1151 :
1152 0 : for (u32 idx = 0; idx < notifyNum; idx++) {
1153 0 : CHK_RET(CheckNotifyOrQPMaxNum(actualNotifyNum, LINK_P2P_MAX_NUM, true));
1154 0 : std::shared_ptr<LocalNotify> ipcWaitNotify = std::make_shared<LocalNotify>();
1155 0 : CHK_RET(InitAndVerifySingleSignal(linkP2p.localIpcSignal[actualNotifyNum], ipcWaitNotify));
1156 0 : transDevP2pData.userLocalNotify[idx] = ipcWaitNotify;
1157 :
1158 0 : std::shared_ptr<RemoteNotify> ipcRecordNotify = std::make_shared<RemoteNotify>();
1159 0 : CHK_RET(InitAndVerifySingleSignal(linkP2p.remoteIpcSignal[actualNotifyNum], ipcRecordNotify));
1160 0 : transDevP2pData.userRemoteNotify[idx] = ipcRecordNotify;
1161 :
1162 0 : actualNotifyNum++;
1163 0 : }
1164 :
1165 0 : HCCL_DEBUG("[%s]get p2pNotify success, actualNotifyNum[%u]", __func__, actualNotifyNum);
1166 0 : p2pNotifyNum = actualNotifyNum;
1167 :
1168 0 : return HCCL_SUCCESS;
1169 0 : }
1170 :
1171 0 : HcclResult HcclCommAicpu::SetTransportRoceQP(
1172 : TransportDeviceIbverbsData& transDevIbverbsData, u64& roceQpNum, HcclLinkRoceV2* linkRoce)
1173 : {
1174 0 : roceQpNum = linkRoce->qpsPerConnection;
1175 0 : u32 roceQpNumSum = linkRoce->qpsPerConnection + static_cast<u32>(linkRoce->qpsPerConnection != 1);
1176 0 : transDevIbverbsData.qpInfo.resize(roceQpNumSum);
1177 0 : std::copy_n(linkRoce->QpInfo, roceQpNumSum, transDevIbverbsData.qpInfo.begin());
1178 0 : transDevIbverbsData.multiQpThreshold = multiQpThreshold_;
1179 0 : transDevIbverbsData.qpsPerConnection = linkRoce->qpsPerConnection;
1180 0 : HCCL_INFO(
1181 : "[%s]transDevIbverbsData.qpInfo.qpPtr[%llu], transDevIbverbsData.qpInfo.sqIndex[%u], "
1182 : "transDevIbverbsData.qpInfo.dbIndex[%u], roceQpNum[%llu], roceQpNumSum[%u], multiQpThreshold_[%u]",
1183 : __func__, transDevIbverbsData.qpInfo[0].qpPtr, transDevIbverbsData.qpInfo[0].sqIndex,
1184 : transDevIbverbsData.qpInfo[0].dbIndex, roceQpNum, roceQpNumSum, multiQpThreshold_);
1185 0 : return HCCL_SUCCESS;
1186 : }
1187 :
1188 0 : HcclResult HcclCommAicpu::SetTransportRoceNotify(
1189 : TransportDeviceIbverbsData& transDevIbverbsData, u64& roceNotifyNum, HcclLinkRoceV2* linkRoce, u32 notifyNum)
1190 : {
1191 0 : u64 actualNotifyNum = 0;
1192 0 : if (linkRoce->localNotifyList == 0 || linkRoce->remoteNotifyList == 0) {
1193 0 : HCCL_DEBUG("[%s] Empty local and remote notify lists, skipping notify resource creation.", __func__);
1194 0 : return HCCL_SUCCESS;
1195 : }
1196 0 : HcclSignalInfo* localNotifyList = reinterpret_cast<HcclSignalInfo*>(linkRoce->localNotifyList);
1197 0 : AddrKey* remoteNotifyList = reinterpret_cast<AddrKey*>(linkRoce->remoteNotifyList);
1198 0 : if (localNotifyList == nullptr || remoteNotifyList == nullptr) {
1199 0 : HCCL_ERROR("[%s]nullptr found in localNotifyList or remoteNotifyList from device mem, check.", __func__);
1200 0 : return HCCL_E_INTERNAL;
1201 : }
1202 : // 获取RDMA Notify信息
1203 0 : std::shared_ptr<LocalNotify> ackNotify = std::make_shared<LocalNotify>();
1204 0 : CHK_RET(InitAndVerifySingleSignal(localNotifyList[actualNotifyNum], ackNotify));
1205 0 : transDevIbverbsData.ackNotify = ackNotify;
1206 0 : transDevIbverbsData.remoteAckNotifyDetails = remoteNotifyList[actualNotifyNum];
1207 0 : actualNotifyNum++;
1208 :
1209 0 : std::shared_ptr<LocalNotify> dataNotify = std::make_shared<LocalNotify>();
1210 0 : CHK_RET(InitAndVerifySingleSignal(localNotifyList[actualNotifyNum], dataNotify));
1211 0 : transDevIbverbsData.dataNotify = dataNotify;
1212 0 : transDevIbverbsData.remoteDataNotifyDetails = remoteNotifyList[actualNotifyNum];
1213 0 : actualNotifyNum++;
1214 :
1215 0 : std::shared_ptr<LocalNotify> dataAckNotify = std::make_shared<LocalNotify>();
1216 0 : CHK_RET(InitAndVerifySingleSignal(localNotifyList[actualNotifyNum], dataAckNotify));
1217 0 : transDevIbverbsData.dataAckNotify = dataAckNotify;
1218 0 : transDevIbverbsData.remoteDataAckNotifyDetails = remoteNotifyList[actualNotifyNum];
1219 0 : transDevIbverbsData.notifySize = notifySize_;
1220 0 : actualNotifyNum++;
1221 :
1222 0 : transDevIbverbsData.userLocalNotify.resize(linkRoce->qpsPerConnection);
1223 0 : transDevIbverbsData.userRemoteNotifyDetails.resize(linkRoce->qpsPerConnection);
1224 : // 当前多QP下每个QP会多申请一个DataNotify
1225 0 : u64 singleQpNotifySize = linkRoce->singleQPNotifyNum + static_cast<u32>(linkRoce->qpsPerConnection > 1);
1226 0 : for (u32 qpIndex = 0; qpIndex < linkRoce->qpsPerConnection; qpIndex++) {
1227 0 : transDevIbverbsData.userLocalNotify[qpIndex].resize(singleQpNotifySize, nullptr);
1228 0 : transDevIbverbsData.userRemoteNotifyDetails[qpIndex].resize(singleQpNotifySize);
1229 0 : for (u32 i = 0, idx = actualNotifyNum + singleQpNotifySize * qpIndex; i < singleQpNotifySize; ++idx, ++i) {
1230 0 : std::shared_ptr<LocalNotify> locNotify = std::make_shared<LocalNotify>();
1231 0 : CHK_RET(InitAndVerifySingleSignal(localNotifyList[idx], locNotify));
1232 0 : transDevIbverbsData.userLocalNotify[qpIndex][i] = locNotify;
1233 0 : transDevIbverbsData.userRemoteNotifyDetails[qpIndex][i] = remoteNotifyList[idx];
1234 0 : }
1235 : }
1236 0 : roceNotifyNum = linkRoce->singleQPNotifyNum;
1237 0 : HCCL_DEBUG("[%s]get roceNotify success, roceNotifyNum[%u]", __func__, roceNotifyNum);
1238 0 : return HCCL_SUCCESS;
1239 0 : }
1240 :
1241 0 : HcclResult HcclCommAicpu::InitLinkP2p(
1242 : HccltagRemoteResV2* tagRes, u32& rankId, const std::string& newTag, u32 notifyNum, TransportLinkType linkType)
1243 : {
1244 0 : std::unordered_map<u32, std::unordered_map<std::string, std::shared_ptr<Transport>>>& linkRes
1245 : = (linkType == TransportLinkType::SIO) ? linkResSio_ : linkRes_;
1246 0 : HcclLinkP2pV2& linkP2p = (linkType == TransportLinkType::SIO) ? tagRes->linkP2pSio : tagRes->linkP2p;
1247 :
1248 0 : if (linkRes.find(rankId) == linkRes.end() || linkRes[rankId].find(newTag) == linkRes[rankId].end()) {
1249 : // 优先校验notify去判定link是否有效
1250 0 : if (linkP2p.localIpcSignal[0].resId == INVALID_U64) {
1251 0 : HCCL_INFO(
1252 : "[%s]the link is invalid, no need to create transport, rankId[%u], newTag[%s]", __func__, rankId,
1253 : newTag.c_str());
1254 0 : return HCCL_SUCCESS;
1255 : }
1256 : // 创建Transport对象
1257 0 : MachinePara machinePara;
1258 0 : CHK_RET(SetTransportMachinePara(machinePara, rankId, newTag, linkType));
1259 0 : machinePara.notifyNum = notifyNum;
1260 : // 获取localMem & remoteMem
1261 0 : TransportDeviceP2pData transDevP2pData;
1262 0 : transDevP2pData.inputBufferPtr = reinterpret_cast<void*>((linkP2p.remoteMem)[INPUT].addr);
1263 0 : transDevP2pData.outputBufferPtr = reinterpret_cast<void*>((linkP2p.remoteMem)[OUTPUT].addr);
1264 0 : if (transDevP2pData.inputBufferPtr == nullptr || transDevP2pData.outputBufferPtr == nullptr) {
1265 0 : HCCL_ERROR(
1266 : "[%s]input ptr[%p] or output ptr[%p] is null.", __func__, transDevP2pData.inputBufferPtr,
1267 : transDevP2pData.outputBufferPtr);
1268 0 : return HCCL_E_PARA;
1269 : }
1270 : // 获取Notify资源
1271 0 : CHK_RET(SetTagRemoteRes(rankId, newTag, tagRes));
1272 0 : HccltagRemoteResV3* tagRemoteRes = &(rankTagRemoteRes_[rankId][newTag]);
1273 0 : CHK_RET(SetTransportPtpNotify(transDevP2pData, tagRemoteRes->p2pNotifyNum, linkP2p, notifyNum));
1274 : // 获取transportAttr信息
1275 0 : transDevP2pData.transportAttr = linkP2p.transportAttr;
1276 : // 创建Transport对象
1277 0 : std::shared_ptr<Transport> link;
1278 0 : TransportPara para{};
1279 0 : const std::unique_ptr<NotifyPool> notifyPool;
1280 0 : link.reset(new (std::nothrow) Transport(
1281 0 : TransportType::TRANS_TYPE_DEVICE_P2P, para, dispatcher_, notifyPool, machinePara, transDevP2pData));
1282 0 : CHK_SMART_PTR_NULL(link);
1283 0 : CHK_RET(link->Init());
1284 0 : linkRes[rankId][newTag] = link;
1285 0 : HCCL_INFO("[%s]linkRes_, rankId[%u], newTag[%s]", __func__, rankId, newTag.c_str());
1286 0 : }
1287 0 : return HCCL_SUCCESS;
1288 : }
1289 0 : HcclResult HcclCommAicpu::InitLinkRoce(
1290 : HccltagRemoteResV2* tagRes, HcclLinkRoceV2* linkRoce, u32& rankId, const std::string& newTag, u32 notifyNum,
1291 : const bool isBackup, const bool isSecond)
1292 : {
1293 : // 优先校验notify去判定link是否有效
1294 0 : if (linkRoce->localNotifyList == 0) {
1295 0 : HCCL_INFO(
1296 : "[%s]the link is invalid, no need to create transport, rankId[%u], newTag[%s], isBackup[%d]", __func__,
1297 : rankId, newTag.c_str(), isBackup);
1298 0 : return HCCL_SUCCESS;
1299 : }
1300 0 : HcclSignalInfo* localNotifyList = reinterpret_cast<HcclSignalInfo*>(linkRoce->localNotifyList);
1301 0 : if (localNotifyList[0].resId == INVALID_U64) {
1302 0 : HCCL_INFO(
1303 : "[%s]the link notify resource is invalid, no need to create transport, rankId[%u], newTag[%s], "
1304 : "resId[%llu], "
1305 : "isBackup[%d]",
1306 : __func__, rankId, newTag.c_str(), localNotifyList[0].resId, isBackup);
1307 0 : return HCCL_SUCCESS;
1308 : }
1309 :
1310 : // 创建Transport对象
1311 0 : MachinePara machinePara;
1312 0 : CHK_RET(SetTransportMachinePara(machinePara, rankId, newTag));
1313 0 : machinePara.notifyNum = notifyNum;
1314 : // 获取localMem & remoteMem
1315 0 : TransportDeviceIbverbsData transDevIbverbsData;
1316 0 : transDevIbverbsData.inputBufferPtr = reinterpret_cast<void*>((linkRoce->remoteMem)[INPUT].addr);
1317 0 : transDevIbverbsData.outputBufferPtr = reinterpret_cast<void*>((linkRoce->remoteMem)[OUTPUT].addr);
1318 0 : if (transDevIbverbsData.inputBufferPtr == nullptr || transDevIbverbsData.outputBufferPtr == nullptr) {
1319 0 : HCCL_ERROR(
1320 : "[%s]input ptr[%p] or output ptr[%p] is null.", __func__, transDevIbverbsData.inputBufferPtr,
1321 : transDevIbverbsData.outputBufferPtr);
1322 0 : return HCCL_E_PARA;
1323 : }
1324 0 : transDevIbverbsData.localInputMem = (linkRoce->localMem)[INPUT];
1325 0 : transDevIbverbsData.localOutputMem = (linkRoce->localMem)[OUTPUT];
1326 0 : transDevIbverbsData.localNotifyValueAddr = linkRoce->notifyValue;
1327 0 : transDevIbverbsData.notifyValueKey = linkRoce->notifyValueKey;
1328 0 : transDevIbverbsData.remoteInputKey = (linkRoce->remoteMem)[INPUT].key;
1329 0 : transDevIbverbsData.remoteOutputKey = (linkRoce->remoteMem)[OUTPUT].key;
1330 : // 获取QPinfo
1331 0 : CHK_RET(SetTagRemoteRes(rankId, newTag, tagRes));
1332 0 : HccltagRemoteResV3* tagRemoteRes = &(rankTagRemoteRes_[rankId][newTag]);
1333 0 : u64& roceQpNum = isBackup ? tagRemoteRes->qpNumBackup : tagRemoteRes->qpNum;
1334 0 : CHK_RET(SetTransportRoceQP(transDevIbverbsData, roceQpNum, linkRoce));
1335 : // 获取notify
1336 0 : u64& roceNotifyNum = isBackup ? tagRemoteRes->roceNotifyNumBackup : tagRemoteRes->roceNotifyNum;
1337 0 : CHK_RET(SetTransportRoceNotify(transDevIbverbsData, roceNotifyNum, linkRoce, notifyNum));
1338 0 : HCCL_INFO("[%s]transDevIbverbsData isBackup[%d]", __func__, isBackup);
1339 : // 获取atomic write
1340 0 : transDevIbverbsData.useAtomicWrite = linkRoce->useAtomicWrite;
1341 : // 创建Transport对象
1342 0 : std::shared_ptr<Transport> link;
1343 0 : TransportPara para{};
1344 0 : para.timeout = linkTimeOut_;
1345 0 : const std::unique_ptr<NotifyPool> notifyPool;
1346 0 : link.reset(new (std::nothrow) Transport(
1347 0 : TransportType::TRANS_TYPE_DEVICE_IBVERBS, para, dispatcher_, notifyPool, machinePara, TransportDeviceP2pData(),
1348 0 : transDevIbverbsData));
1349 0 : CHK_SMART_PTR_NULL(link);
1350 0 : CHK_RET(link->Init());
1351 0 : if (isBackup) {
1352 0 : linkRdmaResBackUp_[rankId][newTag].push_back(link);
1353 0 : HCCL_INFO(
1354 : "[%s]linkRdmaResBackUp_, rankId[%u], newTag[%s], isBackup[%d], isSecond[%d], qpNum[%u], notifyNum[%u]",
1355 : __func__, rankId, newTag.c_str(), isBackup, isSecond, roceQpNum, roceNotifyNum);
1356 : } else {
1357 0 : linkRdmaRes_[rankId][newTag].push_back(link);
1358 0 : HCCL_INFO(
1359 : "[%s]linkRdmaRes_, rankId[%u], newTag[%s], isBackup[%d], isSecond[%d], qpNum[%u], notifyNum[%u]", __func__,
1360 : rankId, newTag.c_str(), isBackup, isSecond, roceQpNum, roceNotifyNum);
1361 : }
1362 0 : return HCCL_SUCCESS;
1363 0 : }
1364 :
1365 0 : HcclResult HcclCommAicpu::InitLinkRoce(
1366 : HccltagRemoteResV2* tagRes, u32& rankId, const std::string& newTag, u32 notifyNum, const bool isBackup)
1367 : {
1368 0 : auto tempLinkRes = isBackup ? linkRdmaResBackUp_ : linkRdmaRes_;
1369 0 : if (tempLinkRes.find(rankId) == tempLinkRes.end()
1370 0 : || tempLinkRes[rankId].find(newTag) == tempLinkRes[rankId].end()) {
1371 0 : bool isBatchSendRecv = newTag.find("BatchSendRecv") != std::string::npos;
1372 0 : if (isBatchSendRecv) {
1373 : // 如果是batchsendrecv，相同rank，需要刷新两次transport，如果是主的话就刷新0,2 备就刷新1,3
1374 0 : if (isBackup) {
1375 0 : CHK_RET(InitLinkRoce(
1376 : tagRes, &(tagRes->linkRoce[AICPU_RETRY_LINKROCE_BACKUP]), rankId, newTag, notifyNum, isBackup));
1377 0 : CHK_RET(InitLinkRoce(
1378 : tagRes, &(tagRes->linkRoce[AICPU_RETRY_LINKROCE_BACKUP + 2]), rankId, newTag, notifyNum, isBackup,
1379 : true));
1380 : } else {
1381 0 : CHK_RET(InitLinkRoce(
1382 : tagRes, &(tagRes->linkRoce[AICPU_RETRY_LINKROCE_DEFAULT]), rankId, newTag, notifyNum, isBackup));
1383 0 : CHK_RET(InitLinkRoce(
1384 : tagRes, &(tagRes->linkRoce[AICPU_RETRY_LINKROCE_DEFAULT + 2]), rankId, newTag, notifyNum, isBackup,
1385 : true));
1386 : }
1387 : } else {
1388 : // 非batchsendrecv只刷新主0，备1
1389 0 : HcclLinkRoceV2* linkRoce = isBackup ? &(tagRes->linkRoce[AICPU_RETRY_LINKROCE_BACKUP]) :
1390 : &(tagRes->linkRoce[AICPU_RETRY_LINKROCE_DEFAULT]);
1391 0 : CHK_RET(InitLinkRoce(tagRes, linkRoce, rankId, newTag, notifyNum, isBackup));
1392 : }
1393 : }
1394 0 : return HCCL_SUCCESS;
1395 0 : }
1396 :
1397 0 : HcclResult HcclCommAicpu::InitRemoteTagRes(
1398 : u32& rankId, const ListCommon& head, const std::string& newTag, u32 notifyNum, TransportLinkType linkType)
1399 : {
1400 0 : HCCL_RUN_INFO(
1401 : "[%s] Entry parse remote resource rankId[%u], group[%s], newTag[%s], head[%p], "
1402 : "linkType[%d]",
1403 : __func__, rankId, identifier_.c_str(), newTag.c_str(), &head, linkType);
1404 0 : ListCommon* curList = reinterpret_cast<ListCommon*>(head.nextDevice);
1405 0 : if (curList == nullptr) {
1406 0 : HCCL_ERROR("[%s]cur list ptr is null.", __func__);
1407 0 : return HCCL_E_PARA;
1408 : }
1409 :
1410 0 : HccltagRemoteResV2* tagRes = nullptr;
1411 0 : while (curList != &head) {
1412 0 : HccltagRemoteResV2* tagResTemp = list_entry(curList, HccltagRemoteResV2, nextTagRes);
1413 0 : if (strcmp(tagResTemp->tag, newTag.c_str()) == 0) {
1414 0 : tagRes = tagResTemp;
1415 0 : break;
1416 : }
1417 :
1418 0 : curList = reinterpret_cast<ListCommon*>(curList->nextDevice);
1419 0 : if (curList == nullptr) {
1420 0 : HCCL_ERROR("[%s]next list ptr is null.", __func__);
1421 0 : return HCCL_E_PARA;
1422 : }
1423 : }
1424 :
1425 0 : if (tagRes == nullptr) {
1426 0 : HCCL_ERROR(
1427 : "[%s]newTag[%s] not found, rankId[%u], head[%p], curList[%p], nextList[%llu], notifyNum[%u], "
1428 : "linkType[%d]",
1429 : __func__, newTag.c_str(), rankId, &head, curList, curList->nextDevice, notifyNum, linkType);
1430 0 : return HCCL_E_PARA;
1431 : }
1432 :
1433 0 : HCCL_INFO(
1434 : "[%s]newTag[%s], rankId[%u], head[%p], curList[%p], nextList[%llu], notifyNum[%u], linkType[%d]", __func__,
1435 : newTag.c_str(), rankId, &head, curList, curList->nextDevice, notifyNum, linkType);
1436 0 : if (linkType != TransportLinkType::RDMA) {
1437 : // 创建P2P链路
1438 0 : CHK_RET(InitLinkP2p(tagRes, rankId, newTag, notifyNum, linkType));
1439 : } else {
1440 : // 创建roce链路
1441 0 : CHK_RET(InitLinkRoce(tagRes, rankId, newTag, notifyNum));
1442 : // 创建roce链路（备用链路）
1443 0 : CHK_RET(InitLinkRoce(tagRes, rankId, newTag, notifyNum, true));
1444 : }
1445 0 : HCCL_INFO(
1446 : "[%s] End parse remote resource rankId[%u] tag[%s], newTag[%s]", __func__, rankId, identifier_.c_str(),
1447 : newTag.c_str());
1448 0 : return HCCL_SUCCESS;
1449 : }
1450 :
1451 0 : HcclResult HcclCommAicpu::RefreshTransportsResForRank(
1452 : const HcclOpResParam* commParam, u32 rankId, const std::string& newTag, u32 notifyNum, TransportLinkType linkType)
1453 : {
1454 0 : if (rankId >= AICPU_MAX_RANK_NUM) {
1455 0 : HCCL_ERROR(
1456 : "[%s] rankId[%u] overflow for group[%s], newTag[%s]", __func__, rankId, identifier_.c_str(),
1457 : newTag.c_str());
1458 0 : return HCCL_E_PARA;
1459 : }
1460 0 : if (commParam->remoteRes[rankId].nextDevicePtr == 0) {
1461 0 : return HCCL_SUCCESS;
1462 : }
1463 0 : HcclRankRelationResV2* rankRelationResPtr
1464 0 : = reinterpret_cast<HcclRankRelationResV2*>(commParam->remoteRes[rankId].nextDevicePtr);
1465 0 : if (rankRelationResPtr == nullptr) {
1466 0 : HCCL_ERROR(
1467 : "[%s]rank relation resource ptr is null, commParam->remoteRes[rankId].nextDevicePtr[%p],"
1468 : " rankId[%u]",
1469 : __func__, reinterpret_cast<HcclRankRelationResV2*>(commParam->remoteRes[rankId].nextDevicePtr), rankId);
1470 0 : return HCCL_E_PARA;
1471 : }
1472 :
1473 : // 1. init公共参数（对应remoteWorldRank，remoteUsrRankId暂不处理：windowsIn，windowsOut）
1474 0 : rankData_[rankId].remoteWorldRank = rankRelationResPtr->remoteWorldRank;
1475 0 : rankData_[rankId].remoteUsrRankId = rankRelationResPtr->remoteUsrRankId;
1476 : // 2. 遍历链表，获取HccltagRemoteResV2创建Tranport对象
1477 0 : if (reinterpret_cast<ListCommon*>(rankRelationResPtr->nextTagRes.nextDevice) != &(rankRelationResPtr->nextTagRes)) {
1478 0 : HCCL_DEBUG(
1479 : "[%s] Start to parse rankId[%u] tag resources, head[%p], nextDevice[%p], pre Device[%p], group[%s]",
1480 : __func__, rankId, &rankRelationResPtr->nextTagRes, rankRelationResPtr->nextTagRes.nextDevice,
1481 : rankRelationResPtr->nextTagRes.preDevice, identifier_.c_str());
1482 0 : CHK_RET(InitRemoteTagRes(rankId, rankRelationResPtr->nextTagRes, newTag, notifyNum, linkType));
1483 : } else {
1484 0 : HCCL_ERROR(
1485 : "[%s]could not find member in rankRelationRes list, rankId[%u], head[%p], nextDevice[%p]", __func__, rankId,
1486 : &rankRelationResPtr->nextTagRes, rankRelationResPtr->nextTagRes.nextDevice);
1487 0 : return HCCL_E_PARA;
1488 : }
1489 0 : HCCL_INFO(
1490 : "[%s] process success rankId[%u], group[%s], newTag[%s]", __func__, rankId, identifier_.c_str(),
1491 : newTag.c_str());
1492 0 : return HCCL_SUCCESS;
1493 : }
1494 :
1495 0 : HcclResult HcclCommAicpu::GetRdmaLinksByRankAndTag(
1496 : const HcclOpResParam* commParam, CommTransportsType type, u32 rankId, const std::string& newTag, LINK& link,
1497 : bool isBackup, u32 notifyNum, bool isSecond)
1498 : {
1499 0 : HCCL_INFO(
1500 : "[%s] Start to rdma get Link group[%s], rankId[%u], newTag[%s], isBackup[%d], deviceLogicId[%d],"
1501 : "notifyNum[%u], isSecond[%d]",
1502 : __func__, identifier_.c_str(), rankId, newTag.c_str(), isBackup, commParam->topoInfo.deviceLogicId, notifyNum,
1503 : isSecond);
1504 :
1505 0 : auto* linkRes = isBackup ? &linkRdmaResBackUp_ : &linkRdmaRes_;
1506 0 : auto iterRankLinks = linkRes->find(rankId);
1507 0 : if (iterRankLinks == linkRes->end() || iterRankLinks->second.find(newTag) == iterRankLinks->second.end()) {
1508 0 : HCCL_INFO(
1509 : "[%s] could not find link resource, rankId[%u], group[%s], newTag[%s]", __func__, rankId,
1510 : identifier_.c_str(), newTag.c_str());
1511 0 : CHK_RET(RefreshTransportsResForRank(commParam, rankId, newTag, notifyNum, TransportLinkType::RDMA));
1512 0 : iterRankLinks = linkRes->find(rankId);
1513 0 : if (iterRankLinks == linkRes->end() || iterRankLinks->second.find(newTag) == iterRankLinks->second.end()) {
1514 0 : HCCL_ERROR(
1515 : "[%s] refresh transport failed, newTag[%s], remoteUserRankId[%u]", __func__, newTag.c_str(), rankId);
1516 0 : return HCCL_E_INTERNAL;
1517 : }
1518 : }
1519 0 : if (isSecond && iterRankLinks->second[newTag].size() <= 1) {
1520 0 : HCCL_ERROR("[%s] get rdma Link failed, newTag[%s], remoteUserRankId[%u]", __func__, newTag.c_str(), rankId);
1521 0 : return HCCL_E_INTERNAL;
1522 : }
1523 :
1524 0 : link = isSecond ? iterRankLinks->second[newTag][1] : iterRankLinks->second[newTag][0];
1525 :
1526 0 : if (receivedAcks_.find(rankId) == receivedAcks_.end()) {
1527 0 : HCCL_ERROR("[%s]there is no link with rankId[%u]", __func__, rankId);
1528 0 : return HCCL_E_NOT_FOUND;
1529 : }
1530 0 : link->SetSupportDataReceivedAck(receivedAcks_[rankId]);
1531 0 : HCCL_DEBUG(
1532 : "[HcclCommAicpu][GetLinksByRankAndTag]rankid[%d] supportDataReceivedAck is %d", rankId, receivedAcks_[rankId]);
1533 0 : HCCL_INFO(
1534 : "[%s] group[%s], newTag[%s], rankId[%u] type[%u] success!", __func__, identifier_.c_str(), newTag.c_str(),
1535 : rankId, type);
1536 0 : return HCCL_SUCCESS;
1537 : }
1538 :
1539 0 : HcclResult HcclCommAicpu::GetSdmaLinksByRankAndTag(
1540 : const HcclOpResParam* commParam, CommTransportsType type, u32 rankId, const std::string& newTag, LINK& link,
1541 : bool isBackup, u32 notifyNum, TransportLinkType linkType)
1542 : {
1543 0 : HCCL_INFO(
1544 : "[%s] Start to get sdma Link group[%s], rankId[%u], newTag[%s], isBackup[%d], deviceLogicId[%d], "
1545 : "notifyNum[%u], linkType[%d]",
1546 : __func__, identifier_.c_str(), rankId, newTag.c_str(), isBackup, commParam->topoInfo.deviceLogicId, notifyNum,
1547 : linkType);
1548 :
1549 0 : auto* linkRes = &linkRes_;
1550 0 : if (linkType == TransportLinkType::SIO) { // HCCS SIO并发场景，HCCS链路与SIO链路分开管理
1551 0 : linkRes = &linkResSio_;
1552 : }
1553 0 : auto iterRankLinks = linkRes->find(rankId);
1554 0 : if (iterRankLinks == linkRes->end() || iterRankLinks->second.find(newTag) == iterRankLinks->second.end()) {
1555 0 : HCCL_INFO(
1556 : "[%s] could not find link resource, rankId[%u], group[%s], newTag[%s]", __func__, rankId,
1557 : identifier_.c_str(), newTag.c_str());
1558 0 : CHK_RET(RefreshTransportsResForRank(commParam, rankId, newTag, notifyNum, linkType));
1559 0 : iterRankLinks = linkRes->find(rankId);
1560 0 : if (iterRankLinks == linkRes->end() || iterRankLinks->second.find(newTag) == iterRankLinks->second.end()) {
1561 0 : HCCL_ERROR(
1562 : "[%s] refresh transport failed, newTag[%s], remoteUserRankId[%u], %p", __func__, newTag.c_str(), rankId,
1563 : linkRes);
1564 0 : return HCCL_E_INTERNAL;
1565 : }
1566 : }
1567 :
1568 0 : link = iterRankLinks->second[newTag];
1569 0 : if (receivedAcks_.find(rankId) == receivedAcks_.end()) {
1570 0 : HCCL_ERROR("[%s]there is no link with rankId[%u]", __func__, rankId);
1571 0 : return HCCL_E_NOT_FOUND;
1572 : }
1573 0 : link->SetSupportDataReceivedAck(receivedAcks_[rankId]);
1574 0 : HCCL_DEBUG(
1575 : "[HcclCommAicpu][GetLinksByRankAndTag]rankid[%d] supportDataReceivedAck is %d", rankId, receivedAcks_[rankId]);
1576 0 : HCCL_INFO(
1577 : "[%s] group[%s], newTag[%s], rankId[%u] type[%u] success!", __func__, identifier_.c_str(), newTag.c_str(),
1578 : rankId, type);
1579 0 : return HCCL_SUCCESS;
1580 : }
1581 :
1582 0 : HcclResult HcclCommAicpu::CleanRoceResource(
1583 : const std::string& newTag, AlgResourceResponse& algResResponse, const std::map<u32, bool>& remoteRankPortMap,
1584 : const OpParam& param)
1585 : {
1586 0 : HCCL_INFO("[%s] Entry alloc transport group[%s], tag[%s]", __func__, identifier_.c_str(), newTag.c_str());
1587 :
1588 0 : for (auto& levelNSubCommTransport : algResResponse.opTransportResponse) {
1589 0 : for (auto& singleSubCommTransport : levelNSubCommTransport) {
1590 0 : for (auto& transportRequest : singleSubCommTransport.transportRequests) {
1591 0 : if (transportRequest.isValid && transportRequest.isUsedRdma) {
1592 0 : u32 remoteUserRank = transportRequest.remoteUserRank;
1593 0 : linkRdmaRes_[remoteUserRank].erase(newTag);
1594 0 : linkRdmaResBackUp_[remoteUserRank].erase(newTag);
1595 : }
1596 : }
1597 : }
1598 : }
1599 :
1600 0 : return HCCL_SUCCESS;
1601 : }
1602 :
1603 1 : HcclResult HcclCommAicpu::CleanAllRoceResource()
1604 : {
1605 1 : HCCL_INFO("Clean all link rdna resources");
1606 : // 清空主链路资源
1607 1 : linkRdmaRes_.clear();
1608 : // 清空备链路资源
1609 1 : linkRdmaResBackUp_.clear();
1610 1 : return HCCL_SUCCESS;
1611 : }
1612 :
1613 2 : HcclResult HcclCommAicpu::ClearOpResource(const std::string& tag)
1614 : {
1615 : // 与 GetAlgResponseRes/AllocAlgResource/ReAllocTransportResource 互斥，避免读写竞争 resMap_/linkRes_
1616 2 : std::lock_guard<std::mutex> lock(preemptMutexForResMap_);
1617 :
1618 6 : auto eraseTagFromAll = [this](const std::string& t) -> bool {
1619 6 : bool hit = false;
1620 : // 主资源map：vector<LINK>析构会让shared_ptr<Transport>引用-1，归零时触发driver资源释放
1621 6 : if (resMap_.erase(t) > 0) {
1622 1 : hit = true;
1623 : }
1624 : // SDMA HCCS链路缓存：报错日志中driver halResAddrMap ioctl失败的根本累积点
1625 9 : for (auto& rankIt : linkRes_) {
1626 3 : if (rankIt.second.erase(t) > 0) {
1627 1 : hit = true;
1628 : }
1629 : }
1630 : // SDMA SIO并发链路缓存
1631 9 : for (auto& rankIt : linkResSio_) {
1632 3 : if (rankIt.second.erase(t) > 0) {
1633 1 : hit = true;
1634 : }
1635 : }
1636 : // RDMA主备链路缓存（CleanAllRoceResource按通信域全清，本接口按tag细粒度清）
1637 9 : for (auto& rankIt : linkRdmaRes_) {
1638 3 : if (rankIt.second.erase(t) > 0) {
1639 1 : hit = true;
1640 : }
1641 : }
1642 9 : for (auto& rankIt : linkRdmaResBackUp_) {
1643 3 : if (rankIt.second.erase(t) > 0) {
1644 1 : hit = true;
1645 : }
1646 : }
1647 : // 本地scratchMem和localTagResToObj按tag索引
1648 6 : if (tagScratchMem_.erase(t) > 0) {
1649 1 : hit = true;
1650 : }
1651 6 : if (localTagResToObj_.erase(t) > 0) {
1652 1 : hit = true;
1653 : }
1654 9 : for (auto& rankIt : rankTagRemoteRes_) {
1655 3 : if (rankIt.second.erase(t) > 0) {
1656 1 : hit = true;
1657 : }
1658 : }
1659 6 : return hit;
1660 2 : };
1661 :
1662 : // 与 host 端 ClearOpResource 对齐：尝试 base/_host/_device 三种后缀
1663 2 : bool found = false;
1664 2 : found = eraseTagFromAll(tag) || found;
1665 2 : found = eraseTagFromAll(tag + "_host") || found;
1666 2 : found = eraseTagFromAll(tag + "_device") || found;
1667 2 : if (!found) {
1668 1 : HCCL_WARNING(
1669 : "[HcclCommAicpu][ClearOpResource] no entries matched tag[%s] "
1670 : "(also tried _host/_device variants), group[%s]",
1671 : tag.c_str(), identifier_.c_str());
1672 : } else {
1673 1 : HCCL_INFO(
1674 : "[HcclCommAicpu][ClearOpResource] cleared resources for tag[%s], group[%s]", tag.c_str(),
1675 : identifier_.c_str());
1676 : }
1677 2 : return HCCL_SUCCESS;
1678 2 : }
1679 :
1680 : // 借轨重新刷新资源
1681 0 : HcclResult HcclCommAicpu::ReAllocTransportResource(
1682 : const std::string& newTag, AlgResourceResponse& algResResponse, std::map<u32, bool>& remoteRankPortMap,
1683 : const HcclOpResParam* commParam, const OpParam& param)
1684 : {
1685 0 : HCCL_INFO("[%s] Entry alloc transport group[%s], tag[%s]", __func__, identifier_.c_str(), newTag.c_str());
1686 0 : std::set<u32> bsrTansportRank;
1687 0 : for (auto& levelNSubCommTransport : algResResponse.opTransportResponse) {
1688 0 : for (auto& singleSubCommTransport : levelNSubCommTransport) {
1689 0 : singleSubCommTransport.links.clear();
1690 0 : singleSubCommTransport.links.reserve(singleSubCommTransport.transportRequests.size());
1691 0 : for (auto& transportRequest : singleSubCommTransport.transportRequests) {
1692 0 : singleSubCommTransport.links.push_back(nullptr);
1693 0 : if (transportRequest.isValid) {
1694 0 : HCCL_INFO(
1695 : "[%s] alloc transport, newTag[%s], rankId[%u], "
1696 : "input memory type[%u], output memory type[%u], ",
1697 : __func__, newTag.c_str(), transportRequest.remoteUserRank, transportRequest.inputMemType,
1698 : transportRequest.outputMemType);
1699 0 : receivedAcks_[transportRequest.remoteUserRank] = singleSubCommTransport.supportDataReceivedAck;
1700 0 : bool isBackup = remoteRankPortMap.find(transportRequest.remoteUserRank) != remoteRankPortMap.end()
1701 0 : && !remoteRankPortMap[transportRequest.remoteUserRank];
1702 0 : bool isSecondBuild = false;
1703 0 : bool isBatchSendRecv = newTag.find("BatchSendRecv") != std::string::npos;
1704 0 : if (transportRequest.isUsedRdma && isBatchSendRecv
1705 0 : && bsrTansportRank.find(transportRequest.remoteUserRank) != bsrTansportRank.end()) {
1706 : // 仅在batchsendrecv rdma下发的时候需要第二次刷新，实际第一次下发都刷好了，第二次就是get一下
1707 0 : isSecondBuild = true;
1708 : }
1709 : // A3 bsr远端是DirectNpu 链路的话则跳过
1710 0 : if ((param.opType == HcclCMDType::HCCL_CMD_BATCH_SEND_RECV)
1711 0 : && (param.BatchSendRecvDataDes.isDirectRemoteRank[transportRequest.remoteUserRank])) {
1712 0 : continue;
1713 : }
1714 0 : bsrTansportRank.insert(transportRequest.remoteUserRank);
1715 0 : CHK_RET(CreateLink(
1716 : newTag, transportRequest, commParam, singleSubCommTransport.links.back(),
1717 : transportRequest.notifyNum, isBackup, isSecondBuild));
1718 : }
1719 : }
1720 : }
1721 : }
1722 :
1723 0 : return HCCL_SUCCESS;
1724 0 : }
1725 :
1726 0 : HcclResult HcclCommAicpu::AllocTransportResource(
1727 : const std::string& newTag, const OpParam& opParam, const HcclOpResParam* commParam, AlgResourceRequest& resRequest,
1728 : AlgResourceResponse& algResResponse)
1729 : {
1730 0 : HCCL_INFO("[%s] Entry alloc transport group[%s]", __func__, identifier_.c_str());
1731 0 : algResResponse.opTransportResponse = resRequest.opTransport;
1732 :
1733 0 : std::set<u32> bsrTansportRank;
1734 0 : for (auto& levelNSubCommTransport : algResResponse.opTransportResponse) {
1735 0 : for (auto& singleSubCommTransport : levelNSubCommTransport) {
1736 0 : singleSubCommTransport.links.clear();
1737 0 : singleSubCommTransport.links.reserve(singleSubCommTransport.transportRequests.size());
1738 0 : for (auto& transportRequest : singleSubCommTransport.transportRequests) {
1739 0 : singleSubCommTransport.links.push_back(nullptr);
1740 0 : if (transportRequest.isValid) {
1741 0 : localUserRank_ = transportRequest.localUserRank;
1742 0 : receivedAcks_[transportRequest.remoteUserRank] = singleSubCommTransport.supportDataReceivedAck;
1743 0 : HCCL_DEBUG(
1744 : "[%s] alloc transport, newTag[%s], rankId[%u], input memory type[%u], "
1745 : "output memory type[%u], ",
1746 : __func__, newTag.c_str(), transportRequest.remoteUserRank, transportRequest.inputMemType,
1747 : transportRequest.outputMemType);
1748 :
1749 0 : bool isSecondBuild = false;
1750 0 : if (transportRequest.isUsedRdma && opParam.opType == HcclCMDType::HCCL_CMD_BATCH_SEND_RECV
1751 0 : && bsrTansportRank.find(transportRequest.remoteUserRank) != bsrTansportRank.end()) {
1752 : // 仅仅在batchsendrecv rdma下发的时候需要第二次刷新，实际第一次下发都刷好了，第二次就是get一下
1753 0 : isSecondBuild = true;
1754 : }
1755 : // A3 bsr远端是DirectNpu 链路的话则跳过
1756 0 : if ((opParam.opType == HcclCMDType::HCCL_CMD_BATCH_SEND_RECV)
1757 0 : && (opParam.BatchSendRecvDataDes.isDirectRemoteRank[transportRequest.remoteUserRank])) {
1758 0 : continue;
1759 : }
1760 0 : bsrTansportRank.insert(transportRequest.remoteUserRank);
1761 0 : CHK_RET(CreateLink(
1762 : newTag, transportRequest, commParam, singleSubCommTransport.links.back(),
1763 : transportRequest.notifyNum, false, isSecondBuild));
1764 : }
1765 : }
1766 : }
1767 : }
1768 :
1769 0 : return HCCL_SUCCESS;
1770 0 : }
1771 :
1772 : // 在resMap_[tag]对应原有通信资源的基础上继续增量建链，目前batchsendrecv会用到本接口
1773 0 : HcclResult HcclCommAicpu::IncreAllocTransportResource(
1774 : const std::string& newTag, const OpParam& opParam, const HcclOpResParam* commParam, AlgResourceRequest& resRequest,
1775 : AlgResourceResponse& algResResponse)
1776 : {
1777 0 : HCCL_INFO("[HcclCommAicpu][IncreAllocTransportResource] Entry alloc transport group[%s]", identifier_.c_str());
1778 0 : std::set<u32> bsrTansportRank;
1779 0 : for (u32 levelIndex = 0; levelIndex < resRequest.opTransport.size(); levelIndex++) {
1780 0 : for (u32 ringIndex = 0; ringIndex < resRequest.opTransport[levelIndex].size(); ringIndex++) {
1781 0 : SingleSubCommTransport& reqSingleSubComm = resRequest.opTransport[levelIndex][ringIndex];
1782 0 : SingleSubCommTransport& respSingleSubComm = algResResponse.opTransportResponse[levelIndex][ringIndex];
1783 0 : for (u32 rankIndex = 0; rankIndex < reqSingleSubComm.transportRequests.size(); rankIndex++) {
1784 0 : TransportRequest& transportRequest = reqSingleSubComm.transportRequests[rankIndex];
1785 0 : CHK_PRT_RET(
1786 : rankIndex >= respSingleSubComm.links.size(),
1787 : HCCL_ERROR(
1788 : "[HcclCommAicpu][IncreAllocTransportResource] The remote rank_id[%u] is larger than "
1789 : "the existent respSingleSubComm map size[%u]",
1790 : rankIndex, respSingleSubComm.links.size()),
1791 : HCCL_E_PARA);
1792 0 : if (respSingleSubComm.links[rankIndex] != nullptr
1793 0 : && respSingleSubComm.links[rankIndex]->GetLinkType() != hccl::LinkType::LINK_RESERVED) {
1794 0 : HCCL_INFO(
1795 : "[IncreAlloc] The link to remote userRank[%u] has existed", transportRequest.remoteUserRank);
1796 0 : continue;
1797 : }
1798 0 : if (transportRequest.isValid) {
1799 0 : receivedAcks_[transportRequest.remoteUserRank] = reqSingleSubComm.supportDataReceivedAck;
1800 0 : respSingleSubComm.transportRequests[rankIndex] = transportRequest;
1801 0 : HCCL_DEBUG(
1802 : "[HcclCommAicpu][IncreAllocTransportResource] alloc transport, newTag[%s], rankId[%u], "
1803 : "input memory type[%u], output memory type[%u], ",
1804 : newTag.c_str(), transportRequest.remoteUserRank, transportRequest.inputMemType,
1805 : transportRequest.outputMemType);
1806 0 : bool isSecondBuild = false;
1807 0 : if (transportRequest.isUsedRdma && opParam.opType == HcclCMDType::HCCL_CMD_BATCH_SEND_RECV
1808 0 : && bsrTansportRank.find(transportRequest.remoteUserRank) != bsrTansportRank.end()) {
1809 : // 仅仅在batchsendrecv rdma下发的时候需要第二次刷新，实际第一次下发都刷好了，第二次就是get一下
1810 0 : isSecondBuild = true;
1811 : }
1812 : // A3 bsr远端是DirectNpu 链路的话则跳过
1813 0 : if ((opParam.opType == HcclCMDType::HCCL_CMD_BATCH_SEND_RECV)
1814 0 : && (opParam.BatchSendRecvDataDes.isDirectRemoteRank[transportRequest.remoteUserRank])) {
1815 0 : continue;
1816 : }
1817 0 : bsrTansportRank.insert(transportRequest.remoteUserRank);
1818 0 : CHK_RET(CreateLink(
1819 : newTag, transportRequest, commParam, respSingleSubComm.links[rankIndex],
1820 : transportRequest.notifyNum, false, isSecondBuild));
1821 : }
1822 : }
1823 : }
1824 : }
1825 0 : return HCCL_SUCCESS;
1826 0 : }
1827 :
1828 0 : HcclResult HcclCommAicpu::CreateLink(
1829 : const std::string& newTag, TransportRequest& transportRequest, const HcclOpResParam* commParam, LINK& link,
1830 : u32 notifyNum, bool isBackup, bool isSecond) // 主备的选择
1831 : {
1832 0 : if (transportRequest.isUsedRdma) {
1833 0 : CHK_RET(GetRdmaLinksByRankAndTag(
1834 : commParam, CommTransportsType::SPECIAL, transportRequest.remoteUserRank, newTag, link, isBackup, notifyNum,
1835 : isSecond));
1836 : } else {
1837 0 : CHK_RET(GetSdmaLinksByRankAndTag(
1838 : commParam, CommTransportsType::SPECIAL, transportRequest.remoteUserRank, newTag, link, isBackup, notifyNum,
1839 : transportRequest.linkType));
1840 : }
1841 :
1842 0 : HCCL_DEBUG("[%s] alloc special transport success!, tag[%s]", __func__, newTag.c_str());
1843 0 : return HCCL_SUCCESS;
1844 : }
1845 :
1846 0 : HcclResult HcclCommAicpu::AllocLocalNotifysResource(
1847 : const std::string& newTag, const HcclOpResParam* commParam, const u32 notifyNum,
1848 : std::vector<std::shared_ptr<LocalNotify>>& notifiesMain, std::vector<std::shared_ptr<LocalNotify>>& notifiesAux)
1849 : {
1850 0 : HCCL_INFO(
1851 : "[HcclCommAicpu][AllocLocalNotifysResource]requesting for [%u] notifys, tag[%s].", notifyNum, newTag.c_str());
1852 0 : if (localNotifies_.capacity() < notifyNum) {
1853 0 : HCCL_ERROR(
1854 : "[HcclCommAicpu][AllocLocalNotifysResource]request number exceed max notify numbers, alloc failed. Max "
1855 : "number is [%u],request num[%u], tag[%s]",
1856 : localNotifies_.capacity(), notifyNum, newTag.c_str());
1857 0 : return HCCL_E_PARA;
1858 : }
1859 :
1860 0 : if (localNotifies_.size() < notifyNum) {
1861 0 : if (InitLocalNotifyObj(commParam) != HCCL_SUCCESS || localNotifies_.size() < notifyNum) {
1862 0 : HCCL_ERROR(
1863 : "[HcclCommAicpu][AllocLocalNotifysResource] the need of notify is more than the available, group[%s], "
1864 : "need[%u], total[%u]",
1865 : newTag.c_str(), notifyNum, localNotifies_.size());
1866 0 : return HCCL_E_INTERNAL;
1867 : }
1868 : }
1869 :
1870 0 : u32 halfNotifyNum = notifyNum >> 1;
1871 0 : notifiesMain.resize(halfNotifyNum);
1872 0 : notifiesAux.resize(halfNotifyNum);
1873 0 : for (u32 i = 0; i < halfNotifyNum; i++) {
1874 0 : notifiesMain[i] = localNotifies_[i << 1];
1875 0 : notifiesAux[i] = localNotifies_[(i << 1) + 1];
1876 : }
1877 0 : HCCL_INFO(
1878 : "[HcclCommAicpu][AllocLocalNotifysResource]find enough notifys, numbers[%u], tag[%s].", notifyNum,
1879 : newTag.c_str());
1880 0 : return HCCL_SUCCESS;
1881 : }
1882 :
1883 0 : HcclResult HcclCommAicpu::AllocStreamsResource(
1884 : const std::string& newTag, const HcclOpResParam* commParam, const u32 streamNum, std::vector<Stream>& streams)
1885 : {
1886 0 : HCCL_INFO(
1887 : "[HcclCommAicpu][AllocStreamsResource]requesting for [%u] slave streams, newTag[%s], group[%s].", streamNum,
1888 : newTag.c_str(), identifier_.c_str());
1889 0 : if (streamNum == 0) {
1890 0 : return HCCL_SUCCESS;
1891 : }
1892 0 : if (slaveStreams_.capacity() < streamNum) {
1893 0 : HCCL_ERROR(
1894 : "[HcclCommAicpu][AllocStreamsResource]request number exceed max substream num, alloc failed. Max "
1895 : "number is [%u],request num[%u], tag[%s]",
1896 : slaveStreams_.capacity(), streamNum, newTag.c_str());
1897 0 : return HCCL_E_PARA;
1898 : }
1899 0 : if (slaveStreams_.size() < streamNum) {
1900 0 : if (InitSlaveStreamObjs(commParam) != HCCL_SUCCESS || slaveStreams_.size() < streamNum) {
1901 0 : HCCL_ERROR(
1902 : "[HcclCommAicpu][AllocStreamsResource] the need of streams is more than the "
1903 : "available, tag[%s], need[%u], total[%u]",
1904 : newTag.c_str(), streamNum, slaveStreams_.size());
1905 0 : return HCCL_E_INTERNAL;
1906 : }
1907 : }
1908 0 : streams = std::vector<Stream>(slaveStreams_.begin(), slaveStreams_.begin() + streamNum);
1909 0 : HCCL_INFO(
1910 : "[HcclCommAicpu][AllocStreamsResource]find enough slave streams [%u], tag[%s].", streamNum, newTag.c_str());
1911 0 : return HCCL_SUCCESS;
1912 : }
1913 :
1914 0 : HcclResult HcclCommAicpu::AllocScratchMemResource(
1915 : const std::string& newTag, const HcclOpResParam* commParam, const u64& scratchMemSize, DeviceMem& scratchMem,
1916 : bool reAllocFlag)
1917 : {
1918 0 : HCCL_INFO(
1919 : "[HcclCommAicpu][AllocScratchMemResource]requesting for [%llu] bytes scratch mem, tag[%s].", scratchMemSize,
1920 : newTag.c_str());
1921 0 : if (scratchMemSize != 0) {
1922 0 : if (tagScratchMem_.find(newTag) == tagScratchMem_.end()) {
1923 0 : HcclResult ret = InitLocalTagRes(commParam->localRes.nextTagRes);
1924 0 : CHK_PRT_RET(
1925 : ret != HCCL_SUCCESS,
1926 : HCCL_ERROR(
1927 : "[HcclCommAicpu][AllocScratchMemResource]InitLocalTagRes error group[%s]", identifier_.c_str()),
1928 : ret);
1929 : }
1930 :
1931 0 : if (tagScratchMem_.find(newTag) == tagScratchMem_.end()) {
1932 0 : HCCL_ERROR(
1933 : "[HcclCommAicpu][AllocScratchMemResource]alloc scratch memory failed."
1934 : "requesting for [%llu] bytes, tag[%s].",
1935 : scratchMemSize, newTag.c_str());
1936 0 : return HCCL_E_NOT_FOUND;
1937 : }
1938 :
1939 0 : if (reAllocFlag) {
1940 0 : HCCL_INFO(
1941 : "[HcclCommAicpu][AllocScratchMemResource]need to reAlloc mem [%llu] bytes scratch mem, tag[%s].",
1942 : scratchMemSize, newTag.c_str());
1943 0 : HcclResult ret = InitLocalTagRes(commParam->localRes.nextTagRes, reAllocFlag);
1944 0 : CHK_PRT_RET(
1945 : ret != HCCL_SUCCESS,
1946 : HCCL_ERROR(
1947 : "[HcclCommAicpu][AllocScratchMemResource]InitLocalTagRes error group[%s]", identifier_.c_str()),
1948 : ret);
1949 : }
1950 :
1951 : // 因为aicpu_communicator中会对scratchMem做对齐，所以tagScratchMem_中的大小会偏小（被对齐截断一部分）
1952 : // 但是两者的差值应该不能超过2个CCE_REDUCE_ALIGN_SIZE，否则应该是不对的
1953 0 : if (scratchMemSize - tagScratchMem_[newTag]->size() > (CCE_REDUCE_ALIGN_SIZE + CCE_REDUCE_ALIGN_SIZE)) {
1954 0 : HCCL_ERROR(
1955 : "[HcclCommAicpu][AllocScratchMemResource]alloc tag[%s] scratch memory failed."
1956 : "requesting [%llu] bytes actual [%llu] bytes",
1957 : newTag.c_str(), scratchMemSize, tagScratchMem_[newTag]->size());
1958 0 : return HCCL_E_PARA;
1959 : }
1960 0 : scratchMem = DeviceMem::create(tagScratchMem_[newTag]->ptr(), tagScratchMem_[newTag]->size());
1961 : }
1962 0 : HCCL_INFO(
1963 : "[HcclCommAicpu][AllocScratchMemResource]find enough [%llu] bytes scratch mem, tag[%s].", scratchMemSize,
1964 : newTag.c_str());
1965 0 : return HCCL_SUCCESS;
1966 : }
1967 :
1968 0 : HcclResult HcclCommAicpu::AllocAlgResource(
1969 : const std::string& newTag, const OpParam& opParam, const HcclOpResParam* commParam, AlgResourceRequest& resRequest,
1970 : AlgResourceResponse& algResResponse)
1971 : {
1972 0 : algResResponse.cclInputMem = cclInputBuffer_;
1973 0 : algResResponse.cclOutputMem = cclOutputBuffer_;
1974 0 : algResResponse.paramInputMem = DeviceMem::create(opParam.inputPtr, opParam.inputSize);
1975 0 : algResResponse.paramOutputMem = DeviceMem::create(opParam.outputPtr, opParam.outputSize);
1976 :
1977 0 : PetersonLockGuard guard(hostDeviceLock_.get());
1978 0 : CHK_PRT_RET(
1979 : guard.IsLockFailed(), HCCL_ERROR("[HcclCommAicpu][AllocAlgResource] hostDeviceLock lock failed"),
1980 : HCCL_E_INTERNAL);
1981 :
1982 0 : CHK_RET(AllocScratchMemResource(newTag, commParam, resRequest.scratchMemSize, algResResponse.scratchMem));
1983 0 : CHK_RET(AllocStreamsResource(newTag, commParam, resRequest.streamNum, algResResponse.slaveStreams));
1984 0 : CHK_RET(AllocLocalNotifysResource(
1985 : newTag, commParam, resRequest.notifyNum, algResResponse.notifiesMain, algResResponse.notifiesAux));
1986 0 : CHK_RET(AllocTransportResource(newTag, opParam, commParam, resRequest, algResResponse));
1987 0 : HCCL_INFO("[HcclCommAicpu][AllocAlgResource] alloc resource success tag[%s].", newTag.c_str());
1988 0 : return HCCL_SUCCESS;
1989 0 : }
1990 :
1991 0 : HcclResult HcclCommAicpu::CalcResRequest(
1992 : const std::string& algName, const OpParam& param, std::unique_ptr<CollExecutorBase>& executor,
1993 : AlgResourceRequest& resourceRequest)
1994 : {
1995 0 : if (executor.get() == nullptr) {
1996 0 : executor = CollAlgExecRegistry::Instance().GetAlgExec(algName, dispatcher_, topoMatcher_);
1997 0 : CHK_PRT_RET(
1998 : executor.get() == nullptr,
1999 : HCCL_ERROR("[HcclCommAicpu][CalcResRequest]Fail to find executor for algName[%s]", algName.c_str()),
2000 : HCCL_E_PARA);
2001 0 : executor->SetAlgType(algType_);
2002 0 : executor->SetCCLInBuffer(cclbufferSize_);
2003 :
2004 0 : if (param.opType == HcclCMDType::HCCL_CMD_REDUCE_SCATTER) {
2005 0 : bool isSupportSDMAReduce = false;
2006 0 : if (GetWorkflowMode() != HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE) {
2007 : isSupportSDMAReduce
2008 0 : = IsSupportSDMAReduce(param.inputPtr, param.outputPtr, param.DataDes.dataType, param.reduceType);
2009 : } else {
2010 0 : isSupportSDMAReduce = IsSupportSDMAReduce(
2011 0 : cclInputBuffer_.ptr(), cclOutputBuffer_.ptr(), param.DataDes.dataType, param.reduceType);
2012 : }
2013 0 : executor->SetIsSupportSDMAReduce(isSupportSDMAReduce);
2014 : }
2015 : }
2016 0 : CHK_RET(CalSendRecvInfoFor910B(algName, param, executor));
2017 0 : return executor->CalcResRequest(param, resourceRequest);
2018 : }
2019 :
2020 0 : u32 HcclCommAicpu::CalculateOpExecIndex(const OpParam& opParam, u32 userRank)
2021 : {
2022 0 : u32 opIndex = 0;
2023 0 : s32 commIndex = 0;
2024 : // 用于重执行和taskException打印的算子计数，bsr/sendrecv/其他算子分别计数
2025 0 : if (opParam.opType == HcclCMDType::HCCL_CMD_BATCH_SEND_RECV) {
2026 0 : commIndex = -1; // batchSendRecv使用 key = -1
2027 0 : } else if (opParam.opType == HcclCMDType::HCCL_CMD_SEND) {
2028 0 : commIndex = opParam.dstRank;
2029 0 : } else if (opParam.opType == HcclCMDType::HCCL_CMD_RECEIVE) {
2030 0 : commIndex = opParam.srcRank;
2031 : } else {
2032 0 : commIndex = userRank;
2033 : }
2034 :
2035 0 : auto it = opExecIndexMap_.find(commIndex);
2036 0 : if (it != opExecIndexMap_.end()) {
2037 0 : opIndex = ++(it->second);
2038 : } else {
2039 0 : opExecIndexMap_.insert({commIndex, 1});
2040 0 : opIndex = 1;
2041 : }
2042 :
2043 0 : HCCL_DEBUG(
2044 : "%s tag:%s opType:%u commIndex:%d opIndex:%u", __func__, opParam.tag.c_str(), opParam.opType, commIndex,
2045 : opIndex);
2046 0 : return opIndex;
2047 : }
2048 :
2049 2 : HcclResult HcclCommAicpu::PrepareSymmetricMemory(const OpParam& param, OpCommTransport& opTransportResponse)
2050 : {
2051 2 : CHK_PRT_RET(
2052 : opTransportResponse.size() == 0,
2053 : HCCL_ERROR("[HcclCommAicpu][PrepareSymmetricMemory] opTransportResponse size is 0"), HCCL_E_PARA);
2054 :
2055 : const std::unordered_set<LinkType> supportedLinkTypes
2056 2 : = {LinkType::LINK_HCCS, LinkType::LINK_SIO, LinkType::LINK_HCCS_SW};
2057 2 : for (u32 levelIdx = 0; levelIdx < opTransportResponse.size(); levelIdx++) {
2058 2 : for (auto& singleSubCommTransport : opTransportResponse[levelIdx]) {
2059 1 : if (singleSubCommTransport.isZeroCopy == false) {
2060 1 : continue;
2061 : }
2062 0 : for (u64 i = 0; i < singleSubCommTransport.links.size(); ++i) {
2063 0 : LINK& link = singleSubCommTransport.links[i];
2064 0 : if (link == nullptr || !singleSubCommTransport.transportRequests[i].isValid
2065 0 : || supportedLinkTypes.count(link->GetLinkType()) == 0) {
2066 0 : continue; // 无效或者不支持的链路
2067 : }
2068 0 : u32 peerRank = link->GetRemoteRank();
2069 0 : void* remoteIn = nullptr;
2070 0 : CHK_RET(HcclSymWinGetPeerPointer(param.inputSymWindow, param.inputOffset, peerRank, &remoteIn));
2071 0 : void* remoteOut = nullptr;
2072 0 : CHK_RET(HcclSymWinGetPeerPointer(param.outputSymWindow, param.outputOffset, peerRank, &remoteOut));
2073 :
2074 0 : CHK_PRT_RET(
2075 : remoteIn == nullptr || remoteOut == nullptr,
2076 : HCCL_ERROR(
2077 : "[HcclCommAicpu][PrepareSymmetricMemory] remoteRank[%d] in[%p] out[%p] is invalid", peerRank,
2078 : remoteIn, remoteOut),
2079 : HCCL_E_INTERNAL);
2080 0 : HCCL_INFO(
2081 : "[HcclCommAicpu][PrepareSymmetricMemory] remoteRank[%d] in[%p] out[%p]", peerRank, remoteIn,
2082 : remoteOut);
2083 0 : CHK_RET(link->UpdateRemoteAddr(remoteIn, remoteOut));
2084 : }
2085 : }
2086 : }
2087 1 : return HCCL_SUCCESS;
2088 1 : }
2089 :
2090 1 : HcclResult HcclCommAicpu::ExecOp(
2091 : const std::string& newTag, const std::string& algName, OpParam& opParam, const HcclOpResParam* commParam)
2092 : {
2093 1 : std::unique_ptr<CollExecutorBase> executor;
2094 : hccl::AlgResourceResponse* algResResponse;
2095 1 : CHK_RET(GetAlgResponseRes(newTag, algName, opParam, commParam, executor, algResResponse));
2096 :
2097 1 : if (isZeroCopy_ || isSymmetricMemory_) {
2098 1 : if (isSymmetricMemory_) {
2099 1 : CHK_RET(PrepareSymmetricMemory(opParam, algResResponse->opTransportResponse));
2100 : } else {
2101 0 : HcclResult ret = PrepareZeroCopyExchanger(newTag, opParam, algResResponse);
2102 0 : CHK_PRT_RET(
2103 : ret != HCCL_SUCCESS,
2104 : HCCL_ERROR(
2105 : "[HcclCommAicpu][ExecOp] newTag[%s], localRankId[%u]", newTag.c_str(), commParam->localUsrRankId),
2106 : ret);
2107 : }
2108 :
2109 : // 零拷贝场景scratchMem的大小会与用户的输入大小不同，会导致后续算法展开模块计算出错
2110 : // 但是该场景下不会直接访问scratchMem，因此直接使用输入作为scratchMem，使得后续计算正确
2111 0 : if (opParam.opType == HcclCMDType::HCCL_CMD_REDUCE_SCATTER) {
2112 0 : algResResponse->scratchMem = DeviceMem::create(opParam.inputPtr, opParam.inputSize);
2113 0 : HCCL_INFO(
2114 : "[HcclCommAicpu][ExecOp] ZeroCopy reduce-scatter use userInput as scratchMem, inputPtr[%p] "
2115 : "intputSize[%lu]",
2116 : opParam.inputPtr, opParam.inputSize);
2117 : }
2118 :
2119 0 : algResResponse->paramInputMem = DeviceMem::create(opParam.inputPtr, opParam.inputSize);
2120 0 : algResResponse->paramOutputMem = DeviceMem::create(opParam.outputPtr, opParam.outputSize);
2121 0 : HCCL_INFO(
2122 : "[HcclCommAicpu][ExecOp] zero copy modify paramInput paramOutput to algResResp inputPtr[%p] inputSize[%lu] "
2123 : "outputPtr[%p] outputSize[%lu]",
2124 : algResResponse->paramInputMem.ptr(), algResResponse->paramInputMem.size(),
2125 : algResResponse->paramOutputMem.ptr(), algResResponse->paramOutputMem.size());
2126 : }
2127 :
2128 0 : hcclOpExecIndex_ = CalculateOpExecIndex(opParam, localUserRank_);
2129 0 : HcclResult ret = Orchestrate(newTag, algName, opParam, executor, *algResResponse, commParam);
2130 0 : if (ret != HCCL_SUCCESS) {
2131 0 : HCCL_ERROR(
2132 : "[HcclCommAicpu][ExecOp] executor op fail, tag[%s], algName[%s], identifier[%s]", newTag.c_str(),
2133 : algName.c_str(), identifier_.c_str());
2134 0 : CHK_PRT_CONT(
2135 : retryEnable_, HCCL_ERROR("[HcclCommAicpu][ExecOp] executor op fail, some error logs may be recorded in the "
2136 : "log/run/device directory, search keyword [ErrToWarn]"));
2137 0 : if (printTaskExceptionForErr_) {
2138 0 : PrintTaskExceptionAllComm();
2139 0 : PrintAicpuCommExecStatus();
2140 0 : printTaskExceptionForErr_ = false;
2141 : }
2142 0 : return ret;
2143 : }
2144 :
2145 0 : HCCL_ENTRY_INFO(
2146 : commParam->opEntry,
2147 : "[HcclCommAicpu][ExecOp] executor op success tag[%s], newTag[%s], algName[%s], identifier[%s].",
2148 : opParam.tag.c_str(), newTag.c_str(), algName.c_str(), identifier_.c_str());
2149 0 : return HCCL_SUCCESS;
2150 1 : }
2151 :
2152 0 : HcclResult HcclCommAicpu::RefreshAlgResponseTransportRes(
2153 : const std::string& newTag, AlgResourceResponse& algResResponse, std::map<u32, bool>& remoteRankPortMap,
2154 : bool isChangeLinkFlag, const HcclOpResParam* commParam, const OpParam& param)
2155 : {
2156 0 : CHK_PTR_NULL(commParam);
2157 0 : auto iter = resMap_.find(newTag);
2158 0 : CHK_PRT_RET(
2159 : iter == resMap_.end(), HCCL_ERROR("[%s]Fail to find algResResponse for tag[%s]", __func__, newTag.c_str()),
2160 : HCCL_E_PARA);
2161 :
2162 0 : PetersonLockGuard guard(hostDeviceLock_.get());
2163 0 : CHK_PRT_RET(guard.IsLockFailed(), HCCL_ERROR("[%s] hostDeviceLock lock failed", __func__), HCCL_E_INTERNAL);
2164 0 : if (!isChangeLinkFlag) {
2165 0 : CleanRoceResource(newTag, algResResponse, remoteRankPortMap, param);
2166 0 : CHK_RET(ReAllocTransportResource(newTag, algResResponse, remoteRankPortMap, commParam, param));
2167 0 : HCCL_RUN_INFO("[%s] ChangeLinkFlag[%d], current tag[%s].", __func__, isChangeLinkFlag, newTag.c_str());
2168 : } else {
2169 : // 提前清理所有tag的链路，避免冲突
2170 0 : for (auto& resMapIt : resMap_) {
2171 0 : HCCL_RUN_INFO("[%s] clean roce resource of tag[%s].", __func__, resMapIt.first.c_str());
2172 0 : CleanRoceResource(resMapIt.first, resMapIt.second, remoteRankPortMap, param);
2173 : }
2174 : // 对resMap中所有tag的transport link根据主备进行刷新
2175 0 : for (auto& resMapIt : resMap_) {
2176 0 : HCCL_RUN_INFO("[%s] refresh algResResponse of tag[%s].", __func__, resMapIt.first.c_str());
2177 0 : CHK_RET(ReAllocTransportResource(resMapIt.first, resMapIt.second, remoteRankPortMap, commParam, param));
2178 0 : if (resMapIt.first == newTag) {
2179 0 : HCCL_RUN_INFO("[%s] current tag[%s].", __func__, newTag.c_str());
2180 0 : algResResponse = resMapIt.second;
2181 : }
2182 : }
2183 : }
2184 :
2185 0 : HCCL_RUN_INFO("[%s] alloc resource success tag[%s].", __func__, newTag.c_str());
2186 0 : return HCCL_SUCCESS;
2187 0 : }
2188 :
2189 0 : HcclResult HcclCommAicpu::CalSendRecvInfoForAlltoall(const OpParam& param)
2190 : {
2191 0 : if (param.opType == HcclCMDType::HCCL_CMD_ALLTOALL) {
2192 : std::vector<u64> sendCountMatrix(
2193 0 : topoInfo_.userRankSize * topoInfo_.userRankSize, param.All2AllDataDes.sendCount);
2194 0 : CHK_RET(GetAlltoAllvcSendRecvInfo(
2195 : static_cast<void*>(sendCountMatrix.data()), param.All2AllDataDes.sendType, param.All2AllDataDes.recvType));
2196 0 : } else if (param.opType == HcclCMDType::HCCL_CMD_ALLTOALLV) {
2197 0 : CHK_PTR_NULL(sendRecvInfoPtr_);
2198 0 : CHK_RET(
2199 : GetAlltoAllvSendRecvInfo(sendRecvInfoPtr_, param.All2AllDataDes.sendType, param.All2AllDataDes.recvType));
2200 : } else {
2201 0 : CHK_RET(GetAlltoAllvcSendRecvInfo(
2202 : param.All2AllDataDes.sendCountMatrix, param.All2AllDataDes.sendType, param.All2AllDataDes.recvType));
2203 : }
2204 0 : return HCCL_SUCCESS;
2205 : }
2206 :
2207 0 : HcclResult HcclCommAicpu::CalSendRecvInfoFor910B(
2208 : const std::string& algName, const OpParam& param, std::unique_ptr<CollExecutorBase>& executor)
2209 : {
2210 : // A2 AICPU才有机会走入RunAlltoAllVStaged
2211 0 : if (algName == "RunAlltoAllVStaged" || algName == "RunAlltoAllVFullMesh") {
2212 0 : CHK_PRT_RET(
2213 : executor.get() == nullptr,
2214 : HCCL_ERROR("[HcclCommAicpu][%s]Fail to find executor for algName[%s]", __func__, algName.c_str()),
2215 : HCCL_E_PARA);
2216 0 : CHK_RET(CalSendRecvInfoForAlltoall(param));
2217 0 : CollAlltoAllExecutor* alltoAllExecutor = dynamic_cast<CollAlltoAllExecutor*>(executor.get());
2218 0 : CHK_PTR_NULL(alltoAllExecutor);
2219 0 : CHK_RET(alltoAllExecutor->SetExcutorExtraInfo(allMeshAggregationSendRecvInfo_, cclbufferSize_));
2220 0 : HCCL_DEBUG("[HcclCommAicpu][%s] running algName[%s], prepare SendRecvInfo.", __func__, algName.c_str());
2221 0 : return HCCL_SUCCESS;
2222 : }
2223 0 : return HCCL_SUCCESS;
2224 : }
2225 :
2226 0 : HcclResult HcclCommAicpu::GetAlgResponseRes(
2227 : const std::string& newTag, const std::string& algName, const OpParam& opParam, const HcclOpResParam* commParam,
2228 : std::unique_ptr<CollExecutorBase>& executor, AlgResourceResponse*& algResResponse)
2229 : {
2230 0 : HCCL_INFO("[%s] algName[%s]", __func__, algName.c_str());
2231 0 : CHK_PTR_NULL(commParam);
2232 : // 刷新CCLBuffer
2233 0 : CHK_RET(InitCclbuffer(commParam));
2234 0 : auto iter = resMap_.find(newTag);
2235 0 : if (iter == resMap_.end()) {
2236 0 : std::lock_guard<std::mutex> lock(preemptMutexForResMap_);
2237 0 : iter = resMap_.find(newTag);
2238 0 : if (iter == resMap_.end()) {
2239 0 : HCCL_RUN_INFO("[%s] Alloc resource for alg[%s], tag[%s]", __func__, algName.c_str(), newTag.c_str());
2240 0 : AlgResourceRequest resRequest;
2241 0 : CHK_RET(CalcResRequest(algName, opParam, executor, resRequest));
2242 0 : CHK_RET(AllocAlgResource(newTag, opParam, commParam, resRequest, resMap_[newTag]));
2243 0 : iter = resMap_.find(newTag);
2244 0 : } else {
2245 0 : HCCL_INFO("[%s] Repeatedly inited for alg [%s] is not allowed.", __func__, algName.c_str());
2246 : }
2247 0 : } else if (algName == "BatchSendRecv" || algName == "BatchSendRecvRetry" || algName == "BatchSendRecvGroup") {
2248 : // 如果是非aclgraph模式，而且不需要增量建链，则跳过CalcResRequest这个计算，节省时间。在非aclgraph模式下，跳过是安全的。
2249 0 : bool canSkipCalcResRequest = !opParam.isCapture && !opParam.needIncreLink;
2250 0 : if (!canSkipCalcResRequest) { // 如果不能跳过计算，则走一遍计算的流程，否则就跳过以下计算
2251 0 : AlgResourceRequest resRequest;
2252 0 : HCCL_INFO("[%s]IncreAlloc resource for alg[%s], tag[%s]", __func__, algName.c_str(), newTag.c_str());
2253 0 : CHK_RET(CalcResRequest(algName, opParam, executor, resRequest));
2254 0 : CHK_RET(IncreAllocTransportResource(newTag, opParam, commParam, resRequest, resMap_[newTag]));
2255 0 : }
2256 0 : } else if (algName == "RunAlltoAllVStaged" || algName == "RunAlltoAllVFullMesh") {
2257 0 : AlgResourceRequest resRequest;
2258 0 : CHK_RET(CalcResRequest(algName, opParam, executor, resRequest));
2259 0 : HCCL_INFO(
2260 : "[%s] check if need refresh resource for alg[%s], tag[%s], old[%lu], new[%lu]", __func__, algName.c_str(),
2261 : newTag.c_str(), resMap_[newTag].scratchMem.size(), resRequest.scratchMemSize);
2262 0 : bool reAllocFlag = !(resMap_[newTag].scratchMem.size() == resRequest.scratchMemSize);
2263 0 : if (reAllocFlag) {
2264 0 : PetersonLockGuard guard(hostDeviceLock_.get());
2265 0 : CHK_PRT_RET(
2266 : guard.IsLockFailed(), HCCL_ERROR("[HcclCommAicpu][AllocAlgResource] hostDeviceLock lock failed"),
2267 : HCCL_E_INTERNAL);
2268 0 : CHK_RET(AllocScratchMemResource(
2269 : newTag, commParam, resRequest.scratchMemSize, resMap_[newTag].scratchMem, true));
2270 0 : HCCL_INFO(
2271 : "[%s] refresh resource success for alg[%s], tag[%s], scratchMemSize[%lu], ptr[%p]", __func__,
2272 : algName.c_str(), newTag.c_str(), resRequest.scratchMemSize, resMap_[newTag].scratchMem.ptr());
2273 0 : }
2274 0 : }
2275 0 : CHK_PRT_RET(
2276 : iter == resMap_.end(), HCCL_ERROR("[%s]Fail to find algResResponse for tag[%s]", __func__, newTag.c_str()),
2277 : HCCL_E_PARA);
2278 0 : algResResponse = &iter->second;
2279 0 : HCCL_INFO("[HcclCommAicpu][GetAlgResponseRes] success!");
2280 0 : return HCCL_SUCCESS;
2281 : }
2282 :
2283 0 : HcclResult HcclCommAicpu::GetAlltoAllVCTotalCount(OpParam& param, u64& sendCount, u64& recvCount)
2284 : {
2285 0 : for (u32 i = 0; i < topoInfo_.userRankSize; i++) {
2286 0 : sendCount += *(
2287 0 : static_cast<const u64*>(param.All2AllDataDes.sendCountMatrix) + topoInfo_.userRank * topoInfo_.userRankSize
2288 0 : + i);
2289 0 : recvCount += *(
2290 0 : static_cast<const u64*>(param.All2AllDataDes.sendCountMatrix) + topoInfo_.userRank
2291 0 : + topoInfo_.userRankSize * i);
2292 : }
2293 0 : return HCCL_SUCCESS;
2294 : }
2295 :
2296 0 : HcclResult HcclCommAicpu::GetAlltoAllTotalCount(OpParam& param, u64& sendCount, u64& recvCount)
2297 : {
2298 0 : sendCount = param.All2AllDataDes.sendCount * topoInfo_.userRankSize;
2299 0 : recvCount = param.All2AllDataDes.sendCount * topoInfo_.userRankSize;
2300 0 : return HCCL_SUCCESS;
2301 : }
2302 :
2303 0 : HcclResult HcclCommAicpu::GetAlltoAllVTotalCount(OpParam& param, u64& sendCount, u64& recvCount)
2304 : {
2305 0 : for (u32 i = 0; i < topoInfo_.userRankSize; i++) {
2306 0 : u64 curSendCount = *(static_cast<const u64*>(param.All2AllDataDes.sendCounts) + i)
2307 0 : + *(static_cast<const u64*>(param.All2AllDataDes.sdispls) + i);
2308 0 : sendCount = std::max(sendCount, curSendCount);
2309 0 : u64 curRecvCount = *(static_cast<const u64*>(param.All2AllDataDes.recvCounts) + i)
2310 0 : + *(static_cast<const u64*>(param.All2AllDataDes.rdispls) + i);
2311 0 : recvCount = std::max(recvCount, curRecvCount);
2312 : }
2313 0 : return HCCL_SUCCESS;
2314 : }
2315 :
2316 0 : HcclResult HcclCommAicpu::SetAlltoAllInputAndOutPutMem(OpParam& param, AlgResourceResponse& algResource)
2317 : {
2318 0 : u32 sendTypeSize = 0, recvTypeSize = 0;
2319 0 : CHK_RET(SalGetDataTypeSize(param.All2AllDataDes.sendType, sendTypeSize));
2320 0 : CHK_RET(SalGetDataTypeSize(param.All2AllDataDes.recvType, recvTypeSize));
2321 0 : u64 sendCount = 0;
2322 0 : u64 recvCount = 0;
2323 0 : if (param.opType == HcclCMDType::HCCL_CMD_ALLTOALL) {
2324 0 : CHK_RET(GetAlltoAllTotalCount(param, sendCount, recvCount));
2325 0 : } else if (param.opType == HcclCMDType::HCCL_CMD_ALLTOALLV) {
2326 0 : CHK_RET(GetAlltoAllVTotalCount(param, sendCount, recvCount));
2327 0 : } else if (param.opType == HcclCMDType::HCCL_CMD_ALLTOALLVC) {
2328 0 : CHK_RET(GetAlltoAllVCTotalCount(param, sendCount, recvCount));
2329 : }
2330 0 : u64 inputSize = sendCount * sendTypeSize;
2331 0 : u64 outputSize = recvCount * recvTypeSize;
2332 0 : algResource.paramInputMem = inputSize == 0 ? tinySendRecvMem_ : DeviceMem::create(param.inputPtr, inputSize);
2333 0 : algResource.paramOutputMem = outputSize == 0 ? tinySendRecvMem_ : DeviceMem::create(param.outputPtr, outputSize);
2334 0 : HCCL_DEBUG(
2335 : "[HcclCommAicpu][SetAlltoAllInputAndOutPutMem] Set memory for AllToAll, inputSize[%llu], inputPtr[%p],"
2336 : "outputPtr[%p]!",
2337 : inputSize, param.inputPtr, param.outputPtr);
2338 0 : return HCCL_SUCCESS;
2339 : }
2340 :
2341 0 : HcclResult HcclCommAicpu::CombineReportOpInfo(OpParam& param, bool isRetry, bool isRelay)
2342 : {
2343 0 : MsprofAicpuHCCLOPInfo hcclOpInfo{0};
2344 0 : hcclOpInfo.relay = (isRelay) ? 1 : 0;
2345 0 : hcclOpInfo.retry = (isRetry) ? 1 : 0;
2346 0 : hcclOpInfo.dataType = param.DataDes.dataType;
2347 0 : hcclOpInfo.count = param.DataDes.count;
2348 0 : hcclOpInfo.groupName = groupHashId_;
2349 0 : hcclOpInfo.ranksize = topoInfo_.userRankSize;
2350 0 : std::string algTypeStr = TransferAlgType(algType_);
2351 0 : CHK_RET(dfx::ProfilingManager::ReportHcclOpInfo(hcclOpInfo, algTypeStr));
2352 0 : return HCCL_SUCCESS;
2353 0 : }
2354 :
2355 0 : HcclResult HcclCommAicpu::UpdateProfReportStartSqeIdx()
2356 : {
2357 0 : if (dfx::ProfilingManager::IsL1fromOffToOn()) {
2358 0 : std::vector<Stream> streams;
2359 0 : CHK_RET(GetStreamAll(streams));
2360 0 : for (auto& tmpStream : streams) {
2361 0 : HcclSqeContext* sqeContext = tmpStream.GetSqeContextPtr();
2362 0 : SqeRingBuffer* sqeContextBuffer = &(sqeContext->buffer);
2363 0 : CHK_RET(dfx::ProfilingManager::UpdateStartReportSqeIdx(tmpStream.id(), sqeContextBuffer->tailSqeIdx));
2364 : }
2365 0 : }
2366 0 : return HCCL_SUCCESS;
2367 : }
2368 :
2369 0 : HcclResult HcclCommAicpu::Orchestrate(
2370 : const std::string& newTag, const std::string& algName, OpParam& param, std::unique_ptr<CollExecutorBase>& executor,
2371 : AlgResourceResponse& algResource, const HcclOpResParam* commParam)
2372 : {
2373 0 : CHK_PTR_NULL(commParam);
2374 : // 算子下发信息记录在共享内存区
2375 0 : UpdateOpRingBufferIdx();
2376 0 : CHK_RET(aicpuShareData_.RecordOpInfo(
2377 : newTag, param, (isDeviceMode_ ? mc2OpIndex_ : hcclOpExecIndex_), localUserRank_, isCustom_));
2378 0 : CHK_RET(UpdateProfReportStartSqeIdx());
2379 :
2380 : // 每个算子都刷新一下profiling开关, 支持profiling从中间迭代采集
2381 0 : bool profL0Open = dfx::ProfilingManager::IsProfL0On();
2382 0 : bool profL1Open = dfx::ProfilingManager::IsProfL1On();
2383 0 : HCCL_DEBUG("profL0Open:%d, profL1Open:%d", profL0Open, profL1Open);
2384 :
2385 0 : LogControl logControl(false, false); // 重执行ERROR日志保底控制，析构时重置日志设置
2386 0 : HCCL_ENTRY_INFO(
2387 : commParam->opEntry, "[HcclCommAicpu][Orchestrate]start tag[%s] newTag[%s] algName[%s] identifier[%s]",
2388 : param.tag.c_str(), newTag.c_str(), algName.c_str(), identifier_.c_str());
2389 0 : HCCL_INFO(
2390 : "opRetryHandler.isInplacePreSync[%d] opRetryHandler.isPostSync[%d]",
2391 : algOpContext_.opRetryHandler.isInplacePreSync, algOpContext_.opRetryHandler.isPostSync);
2392 0 : if (executor.get() == nullptr) {
2393 0 : executor = CollAlgExecRegistry::Instance().GetAlgExec(algName, dispatcher_, topoMatcher_);
2394 0 : CHK_PRT_RET(
2395 : executor.get() == nullptr,
2396 : HCCL_ERROR(
2397 : "[HcclCommAicpu][Orchestrate]Fail to find executor "
2398 : "for algName[%s]",
2399 : algName.c_str()),
2400 : HCCL_E_PARA);
2401 0 : executor->SetAlgType(algType_);
2402 0 : executor->SetCCLInBuffer(cclbufferSize_);
2403 :
2404 0 : if (param.opType == HcclCMDType::HCCL_CMD_REDUCE_SCATTER) {
2405 0 : bool isSupportSDMAReduce = IsSupportSDMAReduce(
2406 0 : cclInputBuffer_.ptr(), cclOutputBuffer_.ptr(), param.DataDes.dataType, param.reduceType);
2407 0 : executor->SetIsSupportSDMAReduce(isSupportSDMAReduce);
2408 : }
2409 : }
2410 0 : if (param.opType == HcclCMDType::HCCL_CMD_ALLTOALL || param.opType == HcclCMDType::HCCL_CMD_ALLTOALLV
2411 0 : || param.opType == HcclCMDType::HCCL_CMD_ALLTOALLVC) {
2412 0 : CHK_RET(CalSendRecvInfoFor910B(algName, param, executor));
2413 0 : CHK_RET(SetAlltoAllInputAndOutPutMem(param, algResource));
2414 0 : if (algName == "RunAlltoAllVTwoLevelPipeline") {
2415 0 : CHK_RET(CalSendRecvInfoForAlltoall(param));
2416 0 : HCCL_DEBUG("[HcclCommAicpu][Orchestrate] running RunAlltoAllVTwoLevelPipeline, prepare SendRecvInfo.");
2417 0 : CollAlltoAllExecutor* alltoAllExecutor = dynamic_cast<CollAlltoAllExecutor*>(executor.get());
2418 0 : CHK_PTR_NULL(alltoAllExecutor);
2419 0 : CHK_RET(alltoAllExecutor->SetExcutorExtraInfo(allMeshAggregationSendRecvInfo_, cclbufferSize_));
2420 : }
2421 : }
2422 0 : auto waitStopExecCmdTimeoutMs = HcclGetWaitStopExecCmdTimeout();
2423 0 : auto waitStopExecCmdTimeout = std::chrono::milliseconds(waitStopExecCmdTimeoutMs);
2424 :
2425 0 : auto opStartTime = std::chrono::steady_clock::now(); // 记录重执行算子耗时
2426 0 : auto startTime = std::chrono::steady_clock::now();
2427 :
2428 0 : KfcError errorCode = KfcError::kNone;
2429 0 : uint32_t retryCnt = 0;
2430 0 : bool retryProcessing = false;
2431 0 : KfcCommand lastCmd = KfcCommand::kNone;
2432 0 : uint32_t beginSqePos = INVALID_UINT;
2433 0 : uint32_t endSqePos = INVALID_UINT;
2434 0 : HcclOpExecFSM state = HcclOpExecFSM::HCCL_OP_EXEC_FSM_INIT;
2435 0 : HcclResult ret = HCCL_SUCCESS;
2436 0 : dfxExtendInfo_.kfcStatus = DfxKfcStatus::kOneStart;
2437 0 : AicpuComContext* ctx = AicpuGetComContext();
2438 0 : AicpuHcclProcess::CallMC2MaintenanceThread(ctx);
2439 0 : u32 loopCnt = 0;
2440 0 : u32 loopNum = 1;
2441 0 : commParam_ = commParam;
2442 0 : CHK_RET(InitExecLoop(param, executor, loopNum));
2443 :
2444 : while (true) {
2445 0 : switch (state) {
2446 0 : case HcclOpExecFSM::HCCL_OP_EXEC_FSM_INIT:
2447 0 : HCCL_INFO("hccl aicpu execute loop %u", loopCnt);
2448 0 : ret = HcclOpExecFsmInitProcess(newTag, param, algResource, state, errorCode);
2449 0 : break;
2450 0 : case HcclOpExecFSM::HCCL_OP_EXEC_FSM_LAUNCH:
2451 0 : ret = HcclOpExecFsmLaunchProcess(
2452 : algName, param, executor, algResource, state, errorCode, beginSqePos, endSqePos, retryCnt);
2453 0 : if (ret == HCCL_E_SUSPENDING && isDeviceMode_ && retryEnable_) {
2454 0 : return HCCL_E_SUSPENDING;
2455 : }
2456 0 : break;
2457 0 : case HcclOpExecFSM::HCCL_OP_EXEC_FSM_WAIT_END:
2458 0 : ret = HcclOpExecFsmWaitEndProcess(
2459 0 : param, algResource, state, errorCode, retryCnt, param.tag, beginSqePos);
2460 0 : if (state == HcclOpExecFSM::HCCL_OP_EXEC_FSM_STOPPING) {
2461 0 : startTime = std::chrono::steady_clock::now();
2462 : }
2463 0 : break;
2464 0 : case HcclOpExecFSM::HCCL_OP_EXEC_FSM_STOPPING:
2465 0 : retryProcessing = true;
2466 0 : if ((std::chrono::steady_clock::now() - startTime) >= waitStopExecCmdTimeout) {
2467 0 : HCCL_ERROR("[OpRetry][AICPU]hccl aicpu wait stop exec timeout[%u ms].", waitStopExecCmdTimeoutMs);
2468 0 : errorCode = KfcError::kTimeout;
2469 0 : state = HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR;
2470 : } else {
2471 0 : ret = HcclOpExecFsmStoppingProcess(param, state, errorCode, retryCnt);
2472 : }
2473 0 : break;
2474 0 : case HcclOpExecFSM::HCCL_OP_EXEC_FSM_STOPPED:
2475 0 : ret = HcclOpExecFsmStoppedProcess(state, errorCode, retryCnt, algName, param, beginSqePos, endSqePos);
2476 0 : if (state == HcclOpExecFSM::HCCL_OP_EXEC_FSM_WAIT_RETRY) {
2477 0 : startTime = std::chrono::steady_clock::now();
2478 : }
2479 0 : break;
2480 0 : case HcclOpExecFSM::HCCL_OP_EXEC_FSM_CHANGE_LINK:
2481 0 : ret = HcclOpExecChangeLinkProcess(newTag, state, errorCode, retryCnt, algResource, commParam, param);
2482 0 : HCCL_DEBUG(
2483 : "[OpRetry][AICPU]retry change link finish, retryCnt:%u, tag:%s, state:%d", retryCnt,
2484 : param.tag.c_str(), state);
2485 0 : break;
2486 0 : case HcclOpExecFSM::HCCL_OP_EXEC_FSM_WAIT_RETRY: {
2487 0 : auto waitRetryCmdTimeoutMs = HcclGetWaitRetryCmdTimeout(retryCnt);
2488 0 : auto waitRetryCmdTimeout = std::chrono::milliseconds(waitRetryCmdTimeoutMs);
2489 0 : if ((std::chrono::steady_clock::now() - startTime) >= waitRetryCmdTimeout) {
2490 0 : HCCL_ERROR("[OpRetry][AICPU]hccl aicpu wait retry timeout[%u ms].", waitRetryCmdTimeoutMs);
2491 0 : errorCode = KfcError::kTimeout;
2492 0 : state = HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR;
2493 : } else {
2494 0 : ret = HcclOpExecFsmWaitRetryProcess(param, state, errorCode, lastCmd);
2495 : }
2496 0 : } break;
2497 0 : case HcclOpExecFSM::HCCL_OP_EXEC_FSM_RETRY:
2498 : // 重执行前清理当前算子展开的SQE缓存 (if any), 防止命中非完整的cache
2499 0 : CHK_RET(aicpuCacheManager_.ClearOpUnfoldCacheEntry(
2500 : algName, param, algResource, isDeviceMode_, topoInfo_, topoMatcher_, algOpContext_,
2501 : GetWorkflowMode()));
2502 0 : ret = HcclOpExecFsmRetryProcess(
2503 : algName, param, executor, algResource, state, errorCode, retryCnt, beginSqePos, endSqePos);
2504 0 : break;
2505 0 : case HcclOpExecFSM::HCCL_OP_EXEC_FSM_END:
2506 0 : loopCnt++;
2507 0 : if ((param.opType == HcclCMDType::HCCL_CMD_BATCH_SEND_RECV) && retryEnable_) {
2508 0 : param.BatchSendRecvDataDes.curIterNum = loopCnt;
2509 0 : ResetBSRRetryCnt();
2510 : }
2511 0 : if (loopCnt < loopNum) {
2512 0 : state = HcclOpExecFSM::HCCL_OP_EXEC_FSM_INIT;
2513 0 : break;
2514 : }
2515 0 : if (retryCnt > 0) {
2516 0 : RecordReportStatus(dfx::ReportStatus::kRetrySuccess);
2517 0 : retryProcessing = false;
2518 0 : auto opEndTime = std::chrono::steady_clock::now();
2519 0 : auto duration = std::chrono::duration_cast<std::chrono::seconds>(opEndTime - opStartTime).count();
2520 0 : HCCL_RUN_INFO(
2521 : "[OpRetry][AICPU]retry exec success, retryCnt [%u], tag [%s], take time [%ld]s", retryCnt,
2522 : param.tag.c_str(), duration);
2523 : }
2524 0 : CHK_RET(CombineReportOpInfo(param, (retryCnt > 0), false));
2525 0 : return HcclOpExecFsmEndProcess(retryCnt);
2526 0 : case HcclOpExecFSM::HCCL_OP_EXEC_STOP_LAUNCH:
2527 0 : HCCL_DEBUG("[NsRecovery][AICPU] stop the kernel");
2528 : // 停止前清理当前算子展开的SQE缓存 (if any), 防止host侧重新展开该算子并命中非完整的cache (例如step快恢)
2529 0 : CHK_RET(aicpuCacheManager_.ClearOpUnfoldCacheEntry(
2530 : algName, param, algResource, isDeviceMode_, topoInfo_, topoMatcher_, algOpContext_,
2531 : GetWorkflowMode()));
2532 0 : if (!needsResponseStopLaunch_) {
2533 0 : return HCCL_E_SUSPENDING;
2534 : } else {
2535 0 : HCCL_RUN_INFO("[NsRecovery][AICPU] stop the kernel for stop cmd");
2536 0 : needsResponseStopLaunch_ = false;
2537 0 : SetCommRecoveryFlag(true);
2538 0 : if (UpdateOpExecStatus(state, KfcStatus::kStoplaunch, errorCode, 0) == HCCL_SUCCESS) {
2539 0 : return HCCL_E_SUSPENDING;
2540 : } else {
2541 0 : break;
2542 : }
2543 : }
2544 0 : case HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR:
2545 : default: {
2546 0 : if (retryProcessing) {
2547 0 : RecordReportStatus(dfx::ReportStatus::kRetryFail);
2548 0 : retryProcessing = false;
2549 : }
2550 0 : UpdateOpExecStatus(state, excuteOpId_, KfcStatus::kRetryError, errorCode, retryCnt);
2551 0 : dfxExtendInfo_.kfcStatus = DfxKfcStatus::kOneFinished;
2552 0 : if (!isDeviceMode_) {
2553 0 : isOpLaunch = false;
2554 : }
2555 0 : HCCL_INFO("hccl aicpu set kfcStatus[%d]", dfxExtendInfo_.kfcStatus);
2556 0 : if (errorCode == KfcError::kExecConstraint) {
2557 0 : return HCCL_E_OPRETRY_FAIL;
2558 : }
2559 0 : return (ret == HCCL_SUCCESS) ? HCCL_E_INTERNAL : ret;
2560 : }
2561 : }
2562 0 : }
2563 : return ret;
2564 0 : }
2565 :
2566 1 : HcclResult HcclCommAicpu::InitBsrSendRecvOpIdAndExcuteOpId(
2567 : OpParam& param, AlgResourceResponse& algResource, HcclOpExecFSM& fsmState, KfcError& errorCode)
2568 : {
2569 1 : auto hcclRet = InitBatchSendRecvOpId(param, algResource);
2570 1 : HCCL_RETRY_CHK_RET_AND_TRANS_FSM(
2571 : hcclRet, HCCL_ERROR("InitBatchSendRecvOpId failed, ret:%u", hcclRet), KfcError::kInner,
2572 : HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR);
2573 1 : param.BatchSendRecvDataDes.curMode = BatchSendRecvCurMode::SEND_RECV;
2574 1 : if (param.BatchSendRecvDataDes.curIterNum == 0) {
2575 1 : bsrSendStream_ = algResource.slaveStreams[BSR_RETRY_SEND_STREAM_INDEX];
2576 1 : bsrRecvStream_ = algResource.slaveStreams[BSR_RETRY_RECV_STREAM_INDEX];
2577 : }
2578 1 : HCCL_INFO(
2579 : "BSR: iter %u, tag:%s index:%u", param.BatchSendRecvDataDes.curIterNum, excuteOpId_.tag, excuteOpId_.index);
2580 1 : HCCL_INFO(
2581 : "BSR: iter %u, send op Tag:%s index:%u", param.BatchSendRecvDataDes.curIterNum, bsrSendOpId_.tag,
2582 : bsrSendOpId_.index);
2583 1 : HCCL_INFO(
2584 : "BSR: iter %u, recv op Tag:%s index:%u", param.BatchSendRecvDataDes.curIterNum, bsrRecvOpId_.tag,
2585 : bsrRecvOpId_.index);
2586 1 : excuteOpId_.bsrInfo[HCCL_SEND].index = bsrSendOpId_.index;
2587 1 : excuteOpId_.bsrInfo[HCCL_RECV].index = bsrRecvOpId_.index;
2588 1 : excuteOpId_.bsrInfo[HCCL_SEND].tpQpn = bsrSendOpId_.bsrInfo[HCCL_SEND].tpQpn;
2589 1 : excuteOpId_.bsrInfo[HCCL_RECV].tpQpn = bsrRecvOpId_.bsrInfo[HCCL_RECV].tpQpn;
2590 1 : excuteOpId_.bsrInfo[HCCL_SEND].streamId = bsrSendOpId_.streamId;
2591 1 : excuteOpId_.bsrInfo[HCCL_RECV].streamId = bsrRecvOpId_.streamId;
2592 1 : excuteOpId_.bsrInfo[HCCL_SEND].srcRank = bsrSendOpId_.srcRank;
2593 1 : excuteOpId_.bsrInfo[HCCL_SEND].detRank = bsrSendOpId_.detRank;
2594 1 : excuteOpId_.bsrInfo[HCCL_RECV].srcRank = bsrRecvOpId_.srcRank;
2595 1 : excuteOpId_.bsrInfo[HCCL_RECV].detRank = bsrRecvOpId_.detRank;
2596 1 : CHK_SAFETY_FUNC_RET(memcpy_s(
2597 : excuteOpId_.bsrInfo[HCCL_SEND].bsrTag, sizeof(excuteOpId_.bsrInfo[HCCL_SEND].bsrTag), bsrSendOpId_.tag,
2598 : sizeof(bsrSendOpId_.tag)));
2599 1 : CHK_SAFETY_FUNC_RET(memcpy_s(
2600 : excuteOpId_.bsrInfo[HCCL_RECV].bsrTag, sizeof(excuteOpId_.bsrInfo[HCCL_RECV].bsrTag), bsrRecvOpId_.tag,
2601 : sizeof(bsrRecvOpId_.tag)));
2602 1 : return HCCL_SUCCESS;
2603 : }
2604 :
2605 1 : HcclResult HcclCommAicpu::HcclOpExecFsmInitProcess(
2606 : const std::string& newTag, OpParam& param, AlgResourceResponse& algResource, HcclOpExecFSM& fsmState,
2607 : KfcError& errorCode)
2608 : {
2609 1 : excuteOpId_.index = isDeviceMode_ ? (++mc2OpIndex_) : hcclOpExecIndex_;
2610 1 : CHK_SAFETY_FUNC_RET(memset_s(excuteOpId_.tag, sizeof(excuteOpId_.tag), 0, sizeof(excuteOpId_.tag)));
2611 1 : CHK_SAFETY_FUNC_RET(memcpy_s(excuteOpId_.tag, sizeof(excuteOpId_.tag), param.tag.c_str(), param.tag.size()));
2612 1 : CHK_SAFETY_FUNC_RET(memset_s(excuteOpId_.newTag, sizeof(excuteOpId_.newTag), 0, sizeof(excuteOpId_.newTag)));
2613 1 : CHK_SAFETY_FUNC_RET(memcpy_s(excuteOpId_.newTag, sizeof(excuteOpId_.newTag), newTag.c_str(), newTag.size()));
2614 1 : excuteOpId_.isSendRecv = false;
2615 1 : excuteOpId_.streamId = ~0u;
2616 1 : excuteOpId_.opType = param.opType;
2617 1 : excuteOpId_.isBsrTaskStart = false;
2618 1 : if (param.opType == HcclCMDType::HCCL_CMD_SEND || param.opType == HcclCMDType::HCCL_CMD_RECEIVE) {
2619 0 : InitSendRecvOpId(param, excuteOpId_);
2620 1 : } else if ((param.opType == HcclCMDType::HCCL_CMD_BATCH_SEND_RECV) && retryEnable_) {
2621 1 : CHK_RET(InitBsrSendRecvOpIdAndExcuteOpId(param, algResource, fsmState, errorCode));
2622 : }
2623 1 : if (GetNsStopLaunchStatus()) {
2624 0 : HCCL_WARNING("the op should not be launched in the suspending status");
2625 0 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_STOP_LAUNCH;
2626 0 : return HCCL_SUCCESS;
2627 : }
2628 1 : auto ret = aicpuHdc_.InitOpExecStatus(kfcStatusTransferD2H_, excuteOpId_);
2629 1 : isOpLaunch = true;
2630 1 : HCCL_INFO("%s tag:%s, isDeviceMode:%d, index:%u", __func__, excuteOpId_.tag, isDeviceMode_, excuteOpId_.index);
2631 1 : HCCL_RETRY_CHK_RET_AND_TRANS_FSM(
2632 : ret, HCCL_ERROR("InitOpExecStatus failed, ret:%u", ret), KfcError::kInner,
2633 : HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR);
2634 1 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_FSM_LAUNCH;
2635 1 : return ret;
2636 : }
2637 :
2638 0 : bool HcclCommAicpu::HcclOpCheckSupportRetry(HcclCMDType opType)
2639 : {
2640 : const std::set<HcclCMDType> HcclSupportRetryOpSet
2641 : = {HcclCMDType::HCCL_CMD_BROADCAST, HcclCMDType::HCCL_CMD_ALLREDUCE, HcclCMDType::HCCL_CMD_REDUCE,
2642 : HcclCMDType::HCCL_CMD_ALLGATHER, HcclCMDType::HCCL_CMD_REDUCE_SCATTER, HcclCMDType::HCCL_CMD_ALLTOALLV,
2643 : HcclCMDType::HCCL_CMD_ALLTOALLVC, HcclCMDType::HCCL_CMD_ALLTOALL, HcclCMDType::HCCL_CMD_GATHER,
2644 : HcclCMDType::HCCL_CMD_SCATTER, HcclCMDType::HCCL_CMD_SEND, HcclCMDType::HCCL_CMD_RECEIVE,
2645 0 : HcclCMDType::HCCL_CMD_BATCH_SEND_RECV};
2646 0 : return (HcclSupportRetryOpSet.find(opType) != HcclSupportRetryOpSet.end());
2647 0 : }
2648 :
2649 1 : HcclResult HcclCommAicpu::HcclOpExecFsmLaunchProcess(
2650 : const std::string& algName, OpParam& param, std::unique_ptr<CollExecutorBase>& executor,
2651 : AlgResourceResponse& algResource, HcclOpExecFSM& fsmState, KfcError& errorCode, uint32_t& beginSqePos,
2652 : uint32_t& endSqePos, uint32_t retryCnt)
2653 : {
2654 1 : HCCL_DEBUG("hccl aicpu start launch task.");
2655 :
2656 1 : HcclResult ret = OrchestrateHcclOp(algName, param, executor, algResource, beginSqePos, endSqePos);
2657 1 : if (ret == HCCL_SUCCESS) { // 下发成功, 并且没有检测到异常cq或中断命令
2658 0 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_FSM_WAIT_END;
2659 1 : } else if (ret == HCCL_E_SUSPENDING) { // 检测到异常cq或中断命令
2660 1 : if (isDeviceMode_ && retryEnable_) {
2661 1 : HCCL_RUN_INFO("Orchestrate hccl op suspending, restart handle by mc2 process.");
2662 1 : return ret;
2663 : }
2664 0 : if ((param.opType == HcclCMDType::HCCL_CMD_BATCH_SEND_RECV) && retryEnable_) {
2665 : // batchsendrecv算子下发过程中出现异常，task下发未完成，send 和 recv 均需要重执行;
2666 : // 第一个g故障op重执行下发task完成后，需要主动上报故障，触发第二个op进行重执行
2667 0 : SetBSRSendOpExecException();
2668 0 : SetBSRRecvOpExecException();
2669 0 : HCCL_RUN_INFO("hccl aicpu abort launch batchsendrecv op, need retry");
2670 : }
2671 0 : CHK_RET(UpdateSuspendStatus(param, fsmState, errorCode, retryCnt));
2672 : } else {
2673 0 : HCCL_ERROR("OrchestrateHcclOp failed, ret:%u", ret);
2674 0 : errorCode = KfcError::kInner;
2675 0 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR;
2676 : }
2677 0 : return ret;
2678 : }
2679 :
2680 4 : HcclResult HcclCommAicpu::HcclOpExecFsmWaitEndProcess(
2681 : OpParam& param, AlgResourceResponse& algResource, HcclOpExecFSM& fsmState, KfcError& errorCode, uint32_t retryCnt,
2682 : std::string& tag, const uint32_t& beginSqePos)
2683 : {
2684 4 : HCCL_DEBUG("hccl aicpu wait task finish.");
2685 4 : auto ret = WaitFinishWhileLoop(mainStream_, algResource.slaveStreams, tag, beginSqePos, param);
2686 4 : if (ret == HCCL_SUCCESS) {
2687 0 : HCCL_DEBUG("hccl aicpu exec complete.");
2688 0 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_FSM_END;
2689 4 : } else if (ret == HCCL_E_SUSPENDING) {
2690 4 : HCCL_RUN_INFO("hccl aicpu force stop in wait end, retryCnt[%u]", retryCnt);
2691 4 : CHK_RET(UpdateSuspendStatus(param, fsmState, errorCode, retryCnt));
2692 : } else {
2693 0 : HCCL_ERROR("WaitTaskFinish failed, ret:%u, identifier[%s]", ret, identifier_.c_str());
2694 0 : errorCode = KfcError::kExec;
2695 0 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR;
2696 : }
2697 4 : return ret;
2698 : }
2699 :
2700 0 : HcclResult HcclCommAicpu::HcclOpExecFsmStoppingProcess(
2701 : const OpParam& param, HcclOpExecFSM& fsmState, KfcError& errorCode, uint32_t retryCnt)
2702 : {
2703 0 : HCCL_DEBUG("hccl aicpu stopping.");
2704 0 : KfcCommand cmd = KfcCommand::kNone;
2705 0 : auto ret = aicpuHdc_.GetOpExecCtrlCmd(kfcControlTransferH2D_, cmd);
2706 0 : if (ret != HCCL_SUCCESS) {
2707 0 : HCCL_ERROR("[OpRetry][AICPU]GetOpExecCtrlCmd failed, ret:%u", ret);
2708 0 : errorCode = KfcError::kExec;
2709 0 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR;
2710 0 : return ret;
2711 : }
2712 0 : if (cmd == KfcCommand::kExit) {
2713 0 : HCCL_RUN_INFO("[OpRetry][AICPU]hccl aicpu exec fsm stop by exit cmd.");
2714 0 : errorCode = KfcError::kExit;
2715 0 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR;
2716 0 : } else if (cmd == KfcCommand::kStopExec) {
2717 0 : HCCL_RUN_INFO("[OpRetry][AICPU]hccl aicpu get stop exec cmd.");
2718 0 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_FSM_STOPPED;
2719 0 : } else if (cmd == KfcCommand::kStopLaunch) {
2720 0 : if ((param.opType == HcclCMDType::HCCL_CMD_BATCH_SEND_RECV) && retryEnable_) {
2721 0 : HcclOpIdentifier targetOp;
2722 0 : CHK_RET(aicpuHdc_.GetOpExecCtrlTargetOp(kfcControlTransferH2D_, targetOp));
2723 0 : std::string targetOpTag = std::string(reinterpret_cast<char*>(&targetOp.tag[0]));
2724 0 : if (targetOpTag != std::string(reinterpret_cast<char*>(&bsrTargetOpId_.tag[0]))) {
2725 0 : CHK_RET(UpdateSuspendStatus(param, fsmState, errorCode, retryCnt));
2726 : }
2727 0 : }
2728 0 : } else if ((cmd == KfcCommand::kNone) || (cmd == KfcCommand::kRetry)) {
2729 0 : HCCL_DEBUG("hccl aicpu wait for stop exec cmd.");
2730 : // do nothing
2731 0 : } else if (cmd == KfcCommand::kReportRetryErr) {
2732 0 : HCCL_ERROR("[OpRetry][AICPU][HcclOpExecFsmStoppingProcess]hccl aicpu can not retry err cmd[%d]", cmd);
2733 0 : uint16_t rsErrorCode = TS_ERROR_RETRY_CONSTRAINT;
2734 0 : CHK_PRT(SendTaskExceptionByMBox(rsErrorCode));
2735 0 : errorCode = KfcError::kExit;
2736 0 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR;
2737 0 : return HCCL_E_OPRETRY_FAIL;
2738 : } else {
2739 0 : HCCL_ERROR("[OpRetry][AICPU]GetOpExecCtrlCmd failed, invalid cmd[%u]", cmd);
2740 0 : errorCode = KfcError::kExec;
2741 0 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR;
2742 : }
2743 0 : return HCCL_SUCCESS;
2744 : }
2745 :
2746 0 : HcclResult HcclCommAicpu::LoadChangeLinkInfo(ChangeLinkInfo& changeLinkInfo)
2747 : {
2748 0 : HcclResult ret = aicpuHdc_.GetOpExecChangeLink(kfcControlTransferH2D_, changeLinkInfo);
2749 :
2750 : // DEBUG_INFO aicpu接收的changelinkinfo
2751 0 : std::string changeLinkInfoStr = "";
2752 0 : for (u32 i = 0; i < changeLinkInfo.remoteRankNum; i++) {
2753 : changeLinkInfoStr
2754 0 : += (std::to_string(changeLinkInfo.remoteRankList[i]) + ":"
2755 0 : + std::to_string(changeLinkInfo.isUseDefaultPort[i]) + "; ");
2756 : }
2757 0 : HCCL_RUN_INFO(
2758 : "[%s]rank[%u], isChangeLinkFlag[%d], changeLinkInfoStr:%s", __func__, localUserRank_,
2759 : changeLinkInfo.isChangeLinkFlag, changeLinkInfoStr.c_str());
2760 :
2761 0 : return ret;
2762 0 : }
2763 :
2764 0 : HcclResult HcclCommAicpu::HcclOpExecChangeLinkProcess(
2765 : const std::string& newTag, HcclOpExecFSM& state, KfcError& errorCode, uint32_t& retryCnt,
2766 : AlgResourceResponse& algResource, const HcclOpResParam* commParam, const OpParam& param)
2767 : {
2768 0 : ChangeLinkInfo changeLinkInfo;
2769 0 : HcclResult ret = LoadChangeLinkInfo(changeLinkInfo);
2770 0 : if (ret != HCCL_SUCCESS) {
2771 0 : errorCode = KfcError::kExec;
2772 0 : state = HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR;
2773 0 : return ret;
2774 : }
2775 0 : bool useBackupLink = false;
2776 0 : std::map<u32, bool> remoteRankPortMap;
2777 0 : for (u32 i = 0; i < changeLinkInfo.remoteRankNum; i++) {
2778 0 : remoteRankPortMap.insert({changeLinkInfo.remoteRankList[i], changeLinkInfo.isUseDefaultPort[i]});
2779 0 : useBackupLink |= (!changeLinkInfo.isUseDefaultPort[i]);
2780 : }
2781 0 : ret = RefreshAlgResponseTransportRes(
2782 0 : newTag, algResource, remoteRankPortMap, changeLinkInfo.isChangeLinkFlag, commParam, param);
2783 0 : if (ret != HCCL_SUCCESS) {
2784 0 : errorCode = KfcError::kExec;
2785 0 : state = HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR;
2786 0 : return ret;
2787 : }
2788 0 : if (useBackupLink) {
2789 0 : RecordReportStatus(dfx::ReportStatus::kRetryWithBackupLink);
2790 : }
2791 0 : if ((param.opType == HcclCMDType::HCCL_CMD_BATCH_SEND_RECV) && retryEnable_) {
2792 0 : retryCnt = (bsrRetryOp_ == HCCL_SEND) ? bsrSendRetryCnt_ : bsrRecvRetryCnt_;
2793 : }
2794 0 : errorCode = KfcError::kNone;
2795 0 : CHK_RET(UpdateOpExecStatus(state, KfcStatus::kChanged, errorCode, retryCnt));
2796 0 : state = HcclOpExecFSM::HCCL_OP_EXEC_FSM_WAIT_RETRY;
2797 0 : return HCCL_SUCCESS;
2798 0 : }
2799 :
2800 0 : HcclResult HcclCommAicpu::BSRStopedProcess(HcclOpExecFSM& fsmState, KfcError& errorCode)
2801 : {
2802 : // 判断batchsendrecv算子的send 和recv 操作停止的位置是否满足重执行条件
2803 : // send / recv 的stream停止位置不能位于该算子的首个sqe 和末尾sqe
2804 : u32 bsrSendSqHead;
2805 : u32 bsrRecvSqHead;
2806 0 : auto ret = QuerySqStatusByType(devId_, bsrSendStream_.sqId(), DRV_SQCQ_PROP_SQ_HEAD, bsrSendSqHead);
2807 0 : HCCL_RETRY_CHK_RET_AND_TRANS_FSM(
2808 : ret, HCCL_ERROR("[OpRetry][AICPU]quert send stream sq head failed, ret:%u", ret), KfcError::kExec,
2809 : HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR);
2810 :
2811 0 : ret = QuerySqStatusByType(devId_, bsrRecvStream_.sqId(), DRV_SQCQ_PROP_SQ_HEAD, bsrRecvSqHead);
2812 0 : HCCL_RETRY_CHK_RET_AND_TRANS_FSM(
2813 : ret, HCCL_ERROR("[OpRetry][AICPU]quert recv stream sq head failed, ret:%u", ret), KfcError::kExec,
2814 : HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR);
2815 :
2816 0 : ret = ((bsrSendOpBeginSqePos_ == bsrSendSqHead) || (bsrRecvOpBeginSqePos_ == bsrRecvSqHead)) ? HCCL_E_OPRETRY_FAIL :
2817 : HCCL_SUCCESS;
2818 0 : if (ret != HCCL_SUCCESS) {
2819 0 : uint16_t rsErrorCode = TS_ERROR_RETRY_CONSTRAINT;
2820 0 : CHK_PRT(SendTaskExceptionByMBox(rsErrorCode));
2821 : }
2822 0 : HCCL_RETRY_CHK_RET_AND_TRANS_FSM(
2823 : ret,
2824 : HCCL_ERROR(
2825 : "[OpRetry][AICPU]hccl aicpu wait start task is not complete, can not retry. params: send sq head "
2826 : "%u, recv sq head %u, send sq begin %u, recv sq begin %u",
2827 : bsrSendSqHead, bsrRecvSqHead, bsrSendOpBeginSqePos_, bsrRecvOpBeginSqePos_),
2828 : KfcError::kExecConstraint, HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR);
2829 :
2830 0 : ret = ((bsrRetryOp_ == HCCL_SEND) && (bsrSendSqHead == bsrSendOpEndSqePos_)) ? HCCL_E_OPRETRY_FAIL : HCCL_SUCCESS;
2831 0 : HCCL_RETRY_CHK_RET_AND_TRANS_FSM(
2832 : ret,
2833 : HCCL_ERROR(
2834 : "[OpRetry][AICPU]hccl aicpu send record complete task is complete, can not retry. params: send sq "
2835 : "head %u, recv sq head %u, send sq begin %u, send sq end %u, recv sq begin %u, recv sq end %u",
2836 : bsrSendSqHead, bsrRecvSqHead, bsrSendOpBeginSqePos_, bsrSendOpEndSqePos_, bsrRecvOpBeginSqePos_,
2837 : bsrRecvOpEndSqePos_),
2838 : KfcError::kExecConstraint, HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR);
2839 :
2840 0 : ret = ((bsrRetryOp_ == HCCL_RECV) && (bsrRecvSqHead == bsrRecvOpEndSqePos_)) ? HCCL_E_OPRETRY_FAIL : HCCL_SUCCESS;
2841 0 : HCCL_RETRY_CHK_RET_AND_TRANS_FSM(
2842 : ret,
2843 : HCCL_ERROR(
2844 : "[OpRetry][AICPU]hccl aicpu recv record complete task is complete, can not retry. params: send sq "
2845 : "head %u, recv sq head %u, send sq begin %u, send sq end %u, recv sq begin %u, recv sq end %u",
2846 : bsrSendSqHead, bsrRecvSqHead, bsrSendOpBeginSqePos_, bsrSendOpEndSqePos_, bsrRecvOpBeginSqePos_,
2847 : bsrRecvOpEndSqePos_),
2848 : KfcError::kExecConstraint, HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR);
2849 :
2850 0 : HCCL_RUN_INFO(
2851 : "[OpRetry][AICPU]hccl aicpu op is running, can retry. params: send sq head "
2852 : "%u, recv sq head %u, send sq begin %u, send sq end %u, recv sq begin %u, recv sq end %u",
2853 : bsrSendSqHead, bsrRecvSqHead, bsrSendOpBeginSqePos_, bsrSendOpEndSqePos_, bsrRecvOpBeginSqePos_,
2854 : bsrRecvOpEndSqePos_);
2855 0 : if (IsTaskExceptionForHccs()) {
2856 0 : HCCL_RUN_INFO("[OpRetry][AICPU]hccl aicpu stop by sdma/write task exception, can retry.");
2857 : }
2858 0 : errorCode = KfcError::kNone;
2859 0 : uint32_t retryCnt = (bsrRetryOp_ == HCCL_SEND) ? bsrSendRetryCnt_ : bsrRecvRetryCnt_;
2860 0 : CHK_RET(UpdateOpExecStatus(fsmState, KfcStatus::kStopExec, errorCode, retryCnt));
2861 0 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_FSM_WAIT_RETRY;
2862 0 : return HCCL_SUCCESS;
2863 : }
2864 :
2865 0 : HcclResult HcclCommAicpu::HcclOpExecFsmStoppedProcess(
2866 : HcclOpExecFSM& fsmState, KfcError& errorCode, uint32_t retryCnt, const std::string& algName, OpParam& param,
2867 : uint32_t beginSqePos, uint32_t endSqePos)
2868 : {
2869 0 : HCCL_DEBUG("hccl aicpu stop exec.");
2870 0 : KfcCommand cmd = KfcCommand::kNone;
2871 0 : auto ret = aicpuHdc_.GetOpExecCtrlCmd(kfcControlTransferH2D_, cmd);
2872 0 : uint16_t rsErrorCode = 0;
2873 0 : if (ret != HCCL_SUCCESS) {
2874 0 : HCCL_ERROR("[OpRetry][AICPU]GetOpExecCtrlCmd failed, ret:%u", ret);
2875 0 : errorCode = KfcError::kExec;
2876 0 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR;
2877 0 : return ret;
2878 : }
2879 :
2880 0 : if (cmd == KfcCommand::kExit) {
2881 0 : HCCL_ERROR("[OpRetry][AICPU]hccl aicpu exec fsm stop by exit cmd.");
2882 0 : errorCode = KfcError::kExit;
2883 0 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR;
2884 0 : return HCCL_SUCCESS;
2885 : }
2886 :
2887 0 : if (cmd == KfcCommand::kReportRetryErr) {
2888 0 : HCCL_ERROR("[OpRetry][AICPU][HcclOpExecFsmStoppedProcess]hccl aicpu can not retry err cmd[%d]", cmd);
2889 0 : rsErrorCode = TS_ERROR_RETRY_CONSTRAINT;
2890 0 : CHK_PRT(SendTaskExceptionByMBox(rsErrorCode));
2891 0 : errorCode = KfcError::kExit;
2892 0 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR;
2893 0 : return HCCL_E_OPRETRY_FAIL;
2894 : }
2895 :
2896 0 : if (!HcclOpSupportRetry(algName, retryEnable_, param)) {
2897 0 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR;
2898 0 : if (param.isInplaceError) {
2899 0 : errorCode = KfcError::kExecConstraint;
2900 0 : rsErrorCode = TS_ERROR_RETRY_CONSTRAINT;
2901 0 : CHK_PRT(SendTaskExceptionByMBox(rsErrorCode));
2902 0 : HCCL_RETRY_CHK_RET_AND_TRANS_FSM(
2903 : HCCL_E_OPRETRY_FAIL,
2904 : HCCL_ERROR("[Opretry][AICPU][HcclOpExecFsmStoppedProcess]can not retry for inpace error."),
2905 : KfcError::kExecConstraint, HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR);
2906 0 : } else if (isPollutedZeroCopyOp(param)) {
2907 0 : errorCode = KfcError::kExecConstraint;
2908 0 : rsErrorCode = TS_ERROR_RETRY_CONSTRAINT;
2909 0 : CHK_PRT(SendTaskExceptionByMBox(rsErrorCode));
2910 0 : HCCL_RETRY_CHK_RET_AND_TRANS_FSM(
2911 : HCCL_E_OPRETRY_FAIL,
2912 : HCCL_ERROR("[Opretry][AICPU][HcclOpExecFsmStoppedProcess]can not retry for zero copy op."),
2913 : KfcError::kExecConstraint, HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR);
2914 : return HCCL_E_OPRETRY_FAIL;
2915 : } else {
2916 0 : errorCode = KfcError::kExec;
2917 : }
2918 0 : return HCCL_SUCCESS;
2919 : }
2920 :
2921 0 : uint32_t sqHead = 0xFFFFFFFF;
2922 0 : CHK_RET(QuerySqStatusByType(devId_, mainStream_.sqId(), DRV_SQCQ_PROP_SQ_HEAD, sqHead));
2923 0 : if (sqHead == endSqePos) {
2924 0 : rsErrorCode = TS_ERROR_RETRY_CONSTRAINT;
2925 0 : CHK_PRT(SendTaskExceptionByMBox(rsErrorCode));
2926 0 : HCCL_RETRY_CHK_RET_AND_TRANS_FSM(
2927 : HCCL_E_OPRETRY_FAIL,
2928 : HCCL_ERROR(
2929 : "[OpRetry][AICPU]hccl aicpu record complete task is complete, can not retry. params: "
2930 : "sqHead %u, beginSqePos %u endSqePos %u",
2931 : sqHead, beginSqePos, endSqePos),
2932 : KfcError::kExecConstraint, HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR);
2933 0 : } else if (sqHead == beginSqePos) {
2934 0 : rsErrorCode = TS_ERROR_RETRY_CONSTRAINT;
2935 0 : CHK_PRT(SendTaskExceptionByMBox(rsErrorCode));
2936 0 : HCCL_RETRY_CHK_RET_AND_TRANS_FSM(
2937 : HCCL_E_OPRETRY_FAIL,
2938 : HCCL_ERROR(
2939 : "[OpRetry][AICPU]hccl aicpu wait start task is not complete, can not retry. "
2940 : "params: sqHead %u, beginSqePos %u endSqePos %u",
2941 : sqHead, beginSqePos, endSqePos),
2942 : KfcError::kExecConstraint, HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR);
2943 0 : } else if ((param.opType == HcclCMDType::HCCL_CMD_BATCH_SEND_RECV) && retryEnable_) {
2944 0 : CHK_RET(BSRStopedProcess(fsmState, errorCode));
2945 0 : } else {
2946 0 : HCCL_RUN_INFO(
2947 : "[OpRetry][AICPU]hccl aicpu op is running, can retry. params: sqHead %u, beginSqePos %u "
2948 : "endSqePos %u",
2949 : sqHead, beginSqePos, endSqePos);
2950 0 : if (IsTaskExceptionForHccs()) {
2951 0 : HCCL_RUN_INFO("[OpRetry][AICPU]hccl aicpu stop by sdma/write task exception, can retry.");
2952 : }
2953 0 : errorCode = KfcError::kNone;
2954 0 : CHK_RET(UpdateOpExecStatus(fsmState, KfcStatus::kStopExec, errorCode, retryCnt));
2955 0 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_FSM_WAIT_RETRY;
2956 : }
2957 0 : return HCCL_SUCCESS;
2958 : }
2959 :
2960 0 : void HcclCommAicpu::NsCommStop()
2961 : {
2962 0 : if ((StreamsKill(devId_) != HCCL_SUCCESS) || (DeviceQuery(devId_, ts::APP_ABORT_KILL_FINISH, 0U) != HCCL_SUCCESS)) {
2963 0 : (void)aicpuHdc_.SetOpExecStatus(kfcStatusTransferD2H_, KfcStatus::kError, KfcError::kExec, 0);
2964 0 : HCCL_ERROR("[NsRecovery][AICPU]Stop failed");
2965 0 : return;
2966 : }
2967 : // 停止条件算子
2968 0 : if (isDeviceMode_) {
2969 0 : (void)InvokeKfcHandler(AicpuKfcHandlerType::kClearCommitTurn, {rpc_});
2970 : }
2971 0 : (void)aicpuHdc_.SetOpExecStatus(kfcStatusTransferD2H_, KfcStatus::kStopExec, KfcError::kNone, 0);
2972 0 : (void)HcclOneSideServiceAicpu::DisableAllStreamFunc();
2973 0 : HCCL_RUN_INFO("[NsRecovery][AICPU]stopFunc Finished");
2974 : }
2975 :
2976 0 : void HcclCommAicpu::NsCommClean()
2977 : {
2978 : // 等待drv任务停止
2979 0 : if ((DeviceQuery(devId_, ts::APP_ABORT_TERMINATE_FINISH, 0U) != HCCL_SUCCESS) || (CleanStreamFunc() != HCCL_SUCCESS)
2980 0 : || (HcclOneSideServiceAicpu::CleanAllStreamFunc() != HCCL_SUCCESS) || (ResetSqBuff() != HCCL_SUCCESS)) {
2981 0 : (void)aicpuHdc_.SetOpExecStatus(kfcStatusTransferD2H_, KfcStatus::kError, KfcError::kExec, 0);
2982 0 : HCCL_ERROR("[NsRecovery][AICPU]stream terminate failed");
2983 0 : return;
2984 : } else {
2985 0 : if (isDeviceMode_) {
2986 0 : (void)InvokeKfcHandler(AicpuKfcHandlerType::kClearMsgArea, {rpc_});
2987 : }
2988 0 : HCCL_INFO("ClearFunc, after APP_ABORT_TERMINATE_FINISH");
2989 0 : dfxExtendInfo_.pollStatus = PollStatus::kDefault;
2990 0 : dfxExtendInfo_.cqeStatus = dfx::CqeStatus::kDefault;
2991 0 : (void)aicpuHdc_.SetOpExecStatus(kfcStatusTransferD2H_, KfcStatus::kClear, KfcError::kNone, 0);
2992 0 : endStopLaunch = false;
2993 0 : isOpLaunch = false;
2994 0 : needsResponseStopLaunch_ = false;
2995 0 : errMessageReport_ = true;
2996 0 : HCCL_RUN_INFO("[NsRecovery][AICPU] clean Finish");
2997 : }
2998 : }
2999 :
3000 0 : HcclResult HcclCommAicpu::GetBackGroundCommand(BackgroundCommand& bgCmd)
3001 : {
3002 0 : return AicpuHdcUtils::GetBackGroundCommand(kfcControlTransferH2D_, bgCmd);
3003 : }
3004 :
3005 0 : HcclResult HcclCommAicpu::ResponseBackGroundStatus(KfcExecStatus& status)
3006 : {
3007 0 : return AicpuHdcUtils::ResponseBackGroundStatus(kfcStatusTransferD2H_, status);
3008 : }
3009 :
3010 0 : HcclResult HcclCommAicpu::GetKfcCommand(KfcCommand& cmd)
3011 : {
3012 0 : return AicpuHdcUtils::GetKfcCommand(kfcControlTransferH2D_, cmd);
3013 : }
3014 :
3015 0 : HcclResult HcclCommAicpu::SetStreamEnable(Stream& stream)
3016 : {
3017 0 : const HcclComStreamInfo& streamInfo = stream.GetHcclStreamInfo();
3018 0 : HCCL_INFO("[SetStreamEnable] streamid[%d]", streamInfo.actualStreamId);
3019 0 : CHK_RET(ConfigSqStatusByType(GetDevId(), streamInfo.sqId, DRV_SQCQ_PROP_SQ_DISABLE_TO_ENABLE, 1));
3020 0 : HandleCqeException(stream, true);
3021 0 : return HCCL_SUCCESS;
3022 : }
3023 :
3024 0 : HcclResult HcclCommAicpu::CleanStreamFunc()
3025 : {
3026 0 : CHK_RET(SetStreamEnable(mainStream_));
3027 0 : for (auto& stream : slaveStreams_) {
3028 0 : CHK_RET(SetStreamEnable(stream));
3029 : }
3030 0 : CHK_RET(SetStreamEnable(orderStream_));
3031 0 : return HCCL_SUCCESS;
3032 : }
3033 :
3034 0 : std::string HcclCommAicpu::PrintInplaceStatus(u8 isInplaceStatus)
3035 : {
3036 0 : const u8 kNoOverlap = 0;
3037 0 : const u8 kAllToAllOverlap = 1;
3038 0 : const u8 kInplaceOverlap = 2;
3039 0 : switch (isInplaceStatus) {
3040 0 : case kNoOverlap:
3041 : // input和output不重叠
3042 0 : return "There is no overlap.";
3043 0 : case kAllToAllOverlap:
3044 : // alltoall类算子的input和output重叠
3045 0 : return "The param.inputPtr is equal to param.outputPtr, hence they overlap.";
3046 0 : case kInplaceOverlap:
3047 : // input和output重叠
3048 0 : return "It's inplace case. hence they overlap.";
3049 0 : default:
3050 0 : return "It's an unknown overlap case.";
3051 : }
3052 : return "";
3053 : }
3054 :
3055 0 : std::string HcclCommAicpu::PrintInplaceSupportRetryStatus(InplaceSupportRetryStatus inPlaceSupportRetryStatus)
3056 : {
3057 0 : switch (inPlaceSupportRetryStatus) {
3058 0 : case InplaceSupportRetryStatus::AG_BD_CASE: // 不需要去变成非DMA削减
3059 : // allgather or broadcast 算子
3060 0 : return "The Allgather or broadcast op supports inplace retry.";
3061 0 : case InplaceSupportRetryStatus::RETRY_1_ALLOW_NO_DMA_REDUCE_CASE1: // 需要去变成非DMA削减
3062 : // 使用AllReduceMeshSmallCountExecutor, ReduceScatterDeterExecutor
3063 : // 且环境变量配置RetryEnable:1
3064 0 : return "Since retryEnable:1, the executor without DMAReduce will be applied.";
3065 0 : case InplaceSupportRetryStatus::RETRY_0_NOT_ALLOW_NO_DMA_REDUCE_CASE1: // 不需要去变成非DMA削减
3066 : // 使用AllReduceMeshSmallCountExecutor, ReduceScatterDeterExecutor
3067 : // 且环境变量配置RetryEnable:0
3068 0 : return "Since retryEnable:0, ExecutorOnlySupportDMAReduce is not allowed for inplace case.";
3069 0 : case InplaceSupportRetryStatus::ALWAYS_NO_DMA_REDUCE: // 不需要去变成非DMA削减，本身就是
3070 : // 使用AllReduceComm/ReduceScatterComm
3071 0 : return "AllReduceComm or ReduceScatterComm is used for inplace case.";
3072 0 : case InplaceSupportRetryStatus::RETRY_1_ALLOW_NO_DMA_REDUCE_CASE2: // 需要去变成非DMA削减
3073 : // 使用其余在91093场景下使用的reduce scatter, allreduce executor
3074 : // 且环境变量配置RetryEnable:1
3075 0 : return "Since retryEnable:1, the executor will be applied without DMAReduce operation.";
3076 0 : case InplaceSupportRetryStatus::RETRY_0_NOT_ALLOW_NO_DMA_REDUCE_CASE2: // 不需要去变成非DMA削减
3077 : // 使用其余在91093场景下使用的reduce scatter, allreduce executor
3078 : // 且环境变量配置RetryEnable:0
3079 0 : return "Since retryEnable:0, the executor without DMAReduce operation can not be applied.";
3080 0 : case InplaceSupportRetryStatus::UNKONWN_EXECUTOR: // 不需要去变成非DMA削减
3081 : // 使用未知的executor
3082 0 : return "The unknown executor does not support for an inplace case yet.";
3083 0 : case InplaceSupportRetryStatus::USER_LARGER_THAN_CCL: // 不需要去变成非DMA削减
3084 : // UserInMem > CCLInMem 场景
3085 0 : return "UserInMem > CCLInMem case";
3086 0 : case InplaceSupportRetryStatus::NOT_BASIC_OP_CASE: // 不需要去变成非DMA削减
3087 : // 非 RS AR AG BD算子场景
3088 0 : return "Is not ReduceScatter, AllReduce, AllGather or Broadcast case";
3089 0 : default:
3090 0 : return "It's unknown case. They overlap.";
3091 : }
3092 : return "";
3093 : }
3094 :
3095 0 : bool HcclCommAicpu::IsNoNeedMonitor(void)
3096 : {
3097 0 : if (taskMonitorInterval_ == 0)
3098 0 : return true;
3099 :
3100 0 : KfcCommand kfcCmd = KfcCommand::kNone;
3101 0 : (void)GetKfcCommand(kfcCmd);
3102 0 : if (kfcCmd != KfcCommand::kNone)
3103 0 : return true;
3104 :
3105 0 : BackgroundCommand bgCmd = BackgroundCommand::kNone;
3106 0 : (void)GetBackGroundCommand(bgCmd);
3107 0 : if (bgCmd != BackgroundCommand::kNone)
3108 0 : return true;
3109 :
3110 0 : HcclComSuspendingFlag suspendingFlag = HcclComSuspendingFlag::isNull;
3111 0 : (void)GetSuspendingFlag(suspendingFlag);
3112 0 : if (suspendingFlag != HcclComSuspendingFlag::isNull)
3113 0 : return true;
3114 0 : return false;
3115 : }
3116 :
3117 0 : void HcclCommAicpu::InsertMonitorData(Stream& stream, HcclUs& curTime, u32 sqHead, uint16_t taskId, uint8_t type)
3118 : {
3119 0 : AicpuStreamMontior tmpTaskMonitor;
3120 0 : tmpTaskMonitor.historyTime = curTime;
3121 0 : tmpTaskMonitor.historyHead = sqHead;
3122 0 : tmpTaskMonitor.historyTaskId = taskId;
3123 0 : tmpTaskMonitor.historyType = type;
3124 0 : streamTaskMonitor_.insert(std::make_pair(stream.sqId(), tmpTaskMonitor));
3125 0 : return;
3126 : }
3127 :
3128 0 : bool HcclCommAicpu::IsNeedRefreshMonitorData(
3129 : AicpuStreamMontior& streamMontior, HcclUs& curTime, uint32_t remoteRank, uint16_t taskId, u32 sqHead, u32 sqTail,
3130 : uint8_t type)
3131 : {
3132 0 : auto& historyTime = streamMontior.historyTime;
3133 0 : auto& historyHead = streamMontior.historyHead;
3134 0 : auto& historyTaskId = streamMontior.historyTaskId;
3135 0 : auto& historyType = streamMontior.historyType;
3136 0 : if ((historyTaskId != taskId) || (sqHead != historyHead) || (sqHead == sqTail) || (historyType != type)
3137 0 : || ((type == RT_STARS_SQE_TYPE_NOTIFY_WAIT) && (remoteRank == INVALID_VALUE_RANKID))) {
3138 0 : historyTime = curTime;
3139 0 : historyHead = sqHead;
3140 0 : historyTaskId = taskId;
3141 0 : historyType = type;
3142 0 : return true;
3143 : }
3144 0 : return false;
3145 : }
3146 :
3147 0 : HcclResult HcclCommAicpu::StreamTaskMonitor(void)
3148 : {
3149 : // 通信域资源已经释放
3150 0 : CHK_PRT_RET(
3151 : !commOpenStatus, HCCL_DEBUG("[StreamTaskMonitor]group[%s] has been destroyed", identifier_.c_str()),
3152 : HCCL_SUCCESS);
3153 0 : if (IsNoNeedMonitor())
3154 0 : return HCCL_SUCCESS;
3155 0 : HCCL_DEBUG("StreamTaskMonitor print");
3156 0 : std::vector<Stream> totalStream = {mainStream_};
3157 0 : totalStream.insert(totalStream.end(), slaveStreams_.begin(), slaveStreams_.end());
3158 0 : HcclUs curTime = TIME_NOW();
3159 0 : for (auto& stream : totalStream) {
3160 0 : u32 sqHead = 0U, sqTail = 0U;
3161 0 : (void)QuerySqStatus(devId_, stream.sqId(), sqHead, sqTail);
3162 0 : HcclSqeContext* sqeContext = stream.GetSqeContextPtr();
3163 0 : CHK_PTR_NULL(sqeContext);
3164 0 : SqeRingBuffer* sqeContextBuffer = &(sqeContext->buffer);
3165 0 : CHK_PTR_NULL(sqeContextBuffer);
3166 :
3167 0 : uint8_t type = 0;
3168 0 : uint16_t taskId = 0;
3169 0 : uint32_t remoteRank = 0;
3170 0 : std::string tmp = GetTaskExceptionTaskInfo(sqHead, sqeContextBuffer, type, taskId, remoteRank);
3171 0 : HCCL_DEBUG("GetTaskExceptionTaskInfo type %u taskId %u", type, taskId);
3172 0 : auto mapIt = streamTaskMonitor_.find(stream.sqId());
3173 0 : if (mapIt == streamTaskMonitor_.end()) {
3174 0 : InsertMonitorData(stream, curTime, sqHead, taskId, type);
3175 0 : continue;
3176 : }
3177 :
3178 0 : auto& streamMontior = mapIt->second;
3179 0 : if (IsNeedRefreshMonitorData(streamMontior, curTime, remoteRank, taskId, sqHead, sqTail, type)) {
3180 0 : continue;
3181 : }
3182 :
3183 0 : auto timeVal = DURATION_US(curTime - streamMontior.historyTime).count();
3184 0 : const int TIME_CONVERSION = 1000;
3185 0 : if (timeVal >= taskMonitorInterval_ * TIME_CONVERSION) {
3186 0 : HCCL_RUN_INFO(
3187 : "[StreamTaskMonitor]prof monitor streamId:%d, sqid:%d, head:%u, tail:%u, time %s us, %s", stream.id(),
3188 : stream.sqId(), sqHead, sqTail, std::to_string(timeVal).c_str(), tmp.c_str());
3189 0 : HCCL_RUN_INFO(
3190 : "[StreamTaskMonitor]prof monitor %s", GetTaskExceptionOpInfo(sqHead, sqeContextBuffer).c_str());
3191 0 : PrintTaskExceptionTaskQue(sqHead, sqeContextBuffer, true);
3192 0 : streamMontior.historyTime = curTime;
3193 0 : streamMontior.historyHead = sqHead;
3194 0 : streamMontior.historyTaskId = taskId;
3195 0 : streamMontior.historyType = type;
3196 : }
3197 0 : }
3198 0 : return HCCL_SUCCESS;
3199 0 : }
3200 :
3201 0 : HcclResult HcclCommAicpu::SupportRetryWithInplaceCheck(const std::string& algName, OpParam& param)
3202 : {
3203 : // 不支持inplace的通信算子重执行
3204 0 : u8 isInplaceStatus = 0;
3205 0 : InplaceSupportRetryStatus inPlaceSupportRetryStatus = InplaceSupportRetryStatus::INPLACE_STATUS_END;
3206 0 : if (IsHcclOpInplace(param.opType, param, topoInfo_.userRank, topoInfo_.userRankSize, isInplaceStatus)) {
3207 0 : if (!FitRetryConditionforInPlaceOp(
3208 0 : param.opType, param, algName, cclbufferSize_, topoInfo_.userRankSize,
3209 0 : algOpContext_.opRetryHandler.retryEnable, inPlaceSupportRetryStatus)) {
3210 0 : HCCL_RUN_INFO(
3211 : "[OpRetry][AICPU]hccl supports inplace status: isInplaceStatus[%s], "
3212 : "opRetryHandler.inplaceSupportRetry[%d], opRetryHandler.inPlaceSupportRetryStatus[%s], "
3213 : "opRetryHandler.isInplacePreSync[%d], opRetryHandler.isPostSync[%d].",
3214 : PrintInplaceStatus(isInplaceStatus).c_str(), 0,
3215 : PrintInplaceSupportRetryStatus(inPlaceSupportRetryStatus).c_str(),
3216 : algOpContext_.opRetryHandler.isInplacePreSync, algOpContext_.opRetryHandler.isPostSync);
3217 : } else {
3218 0 : HCCL_RUN_INFO(
3219 : "[OpRetry][AICPU]hccl supports inplace status: isInplaceStatus[%s], "
3220 : "opRetryHandler.inplaceSupportRetry[%d], opRetryHandler.inPlaceSupportRetryStatus[%s], "
3221 : "opRetryHandler.isInplacePreSync[%d], opRetryHandler.isPostSync[%d].",
3222 : PrintInplaceStatus(isInplaceStatus).c_str(), 1,
3223 : PrintInplaceSupportRetryStatus(inPlaceSupportRetryStatus).c_str(),
3224 : algOpContext_.opRetryHandler.isInplacePreSync, algOpContext_.opRetryHandler.isPostSync);
3225 : }
3226 : } else {
3227 0 : HCCL_RUN_INFO(
3228 : "[OpRetry][AICPU]hccl supports inplace status: isInplaceStatus[%s], "
3229 : "opRetryHandler.isInplacePreSync[%d], opRetryHandler.isPostSync[%d].",
3230 : PrintInplaceStatus(isInplaceStatus).c_str(), algOpContext_.opRetryHandler.isInplacePreSync,
3231 : algOpContext_.opRetryHandler.isPostSync);
3232 : }
3233 0 : return HCCL_SUCCESS;
3234 : }
3235 :
3236 0 : bool HcclCommAicpu::HcclOpSupportRetry(const std::string& algName, bool retryEnable, OpParam& param)
3237 : {
3238 0 : HCCL_RUN_INFO(
3239 : "[OpRetry][AICPU]hccl supports retry status: enable[%u], param.tag[%s].", retryEnable, param.tag.c_str());
3240 0 : if (!retryEnable) {
3241 0 : HCCL_ERROR("[OpRetry][AICPU]hccl aicpu can not retry, enable[%u].", retryEnable);
3242 0 : return false;
3243 : }
3244 0 : if (isPollutedZeroCopyOp(param)) {
3245 0 : HCCL_ERROR(
3246 : "[OpRetry][AICPU]hccl aicpu can not retry, isZeroCopy[%d], opType[%s].", param.isZeroCopy,
3247 : GetCMDTypeEnumStr(param.opType).c_str());
3248 0 : return false;
3249 : }
3250 :
3251 0 : CHK_RET(SupportRetryWithInplaceCheck(algName, param));
3252 : // 不支持inplace的通信算子重执行
3253 0 : if ((!algOpContext_.opRetryHandler.inplaceSupportRetry) && (!algOpContext_.opRetryHandler.isInplacePreSync)
3254 0 : && (!algOpContext_.opRetryHandler.isPostSync)) {
3255 0 : HCCL_ERROR(
3256 : "[OpRetry][AICPU]hccl aicpu can not retry, not support inplace case, opType[%s], "
3257 : "inputPtr[0x%016lx], outputPtr[0x%016lx], opRetryHandler.inplaceSupportRetry[%d], "
3258 : "opRetryHandler.isInplacePreSync[%d], opRetryHandler.isPostSync[%d]",
3259 : GetCMDTypeEnumStr(param.opType).c_str(), param.inputPtr, param.outputPtr,
3260 : algOpContext_.opRetryHandler.inplaceSupportRetry, algOpContext_.opRetryHandler.isInplacePreSync,
3261 : algOpContext_.opRetryHandler.isPostSync);
3262 0 : param.isInplaceError = true;
3263 0 : return false;
3264 : }
3265 :
3266 : // 不支持的通信算子重执行
3267 0 : if (HcclOpCheckSupportRetry(param.opType) == false) {
3268 0 : HCCL_ERROR(
3269 : "[OpRetry][AICPU]hccl aicpu can not retry, not support opType[%s].",
3270 : GetCMDTypeEnumStr(param.opType).c_str());
3271 0 : return false;
3272 : }
3273 0 : return true;
3274 : }
3275 :
3276 0 : bool HcclCommAicpu::isPollutedZeroCopyOp(OpParam& param)
3277 : {
3278 : // allreduce\reduce\reducescatter\reducescatterv with zerocopy can not support retry.
3279 0 : bool isPollutedOp
3280 0 : = ((param.opType == HcclCMDType::HCCL_CMD_ALLREDUCE) || (param.opType == HcclCMDType::HCCL_CMD_REDUCE)
3281 0 : || (param.opType == HcclCMDType::HCCL_CMD_REDUCE_SCATTER)
3282 0 : || (param.opType == HcclCMDType::HCCL_CMD_REDUCE_SCATTER_V));
3283 0 : return param.isZeroCopy && isPollutedOp;
3284 : }
3285 :
3286 0 : HcclResult HcclCommAicpu::UpdateOpExecStatus(
3287 : HcclOpExecFSM& fsmState, HcclOpIdentifier& opId, KfcStatus state, KfcError& errorCode, uint32_t retryCnt)
3288 : {
3289 0 : HCCL_INFO(
3290 : "UpdateOpExecStatus fsmState %d, tag %s, index %u, state %d, errorCode %d, retryCnt %u.", fsmState, opId.tag,
3291 : opId.index, state, errorCode, retryCnt);
3292 0 : auto ret = aicpuHdc_.SetOpExecStatus(kfcStatusTransferD2H_, opId, state, errorCode, retryCnt);
3293 :
3294 0 : HCCL_RETRY_CHK_RET_AND_TRANS_FSM(
3295 : ret, HCCL_ERROR("SetOpExecStatus failed, ret:%u", ret), KfcError::kExec, HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR);
3296 :
3297 0 : return ret;
3298 : }
3299 :
3300 : HcclResult
3301 7 : HcclCommAicpu::UpdateOpExecStatus(HcclOpExecFSM& fsmState, KfcStatus state, KfcError& errorCode, uint32_t retryCnt)
3302 : {
3303 7 : HCCL_INFO(
3304 : "UpdateOpExecStatus fsmState %d, state %d, errorCode %d, retryCnt %u.", fsmState, state, errorCode, retryCnt);
3305 7 : auto ret = aicpuHdc_.SetOpExecStatus(kfcStatusTransferD2H_, state, errorCode, retryCnt);
3306 7 : if (ret != HCCL_SUCCESS) {
3307 0 : HCCL_ERROR("SetOpExecStatus failed, ret:%u", ret);
3308 0 : errorCode = KfcError::kExec;
3309 0 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR;
3310 : }
3311 7 : return ret;
3312 : }
3313 :
3314 : static constexpr u32 HCCL_AICPU_WAIT_HOST_BASE_TIME_MS = 200 * 1000;
3315 : static constexpr u32 TIME_S_TO_MS = 1000;
3316 14 : u32 HcclCommAicpu::HcclGetWaitStopExecCmdTimeout()
3317 : {
3318 14 : return std::max(static_cast<u32>(linkTimeOut_.count()), HCCL_AICPU_WAIT_HOST_BASE_TIME_MS);
3319 : }
3320 :
3321 7 : u32 HcclCommAicpu::HcclGetWaitRetryCmdTimeout(uint32_t retryCnt)
3322 : {
3323 7 : if (retryCnt == 0) {
3324 7 : return HcclGetWaitStopExecCmdTimeout() + retryHoldTime_;
3325 : } else {
3326 0 : return HcclGetWaitStopExecCmdTimeout() + retryIntervalTime_;
3327 : }
3328 : }
3329 :
3330 0 : HcclResult HcclCommAicpu::HcclOpExecFsmWaitRetryProcess(
3331 : const OpParam& param, HcclOpExecFSM& fsmState, KfcError& errorCode, KfcCommand& lastCmd)
3332 : {
3333 0 : HCCL_DEBUG("hccl aicpu wait for retry cmd.");
3334 0 : KfcCommand cmd = KfcCommand::kNone;
3335 0 : auto ret = aicpuHdc_.GetOpExecCtrlCmd(kfcControlTransferH2D_, cmd);
3336 0 : if (ret != HCCL_SUCCESS) {
3337 0 : HCCL_ERROR("GetOpExecCtrlCmd failed, ret:%u", ret);
3338 0 : errorCode = KfcError::kExec;
3339 0 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR;
3340 0 : return ret;
3341 : }
3342 0 : if (cmd == KfcCommand::kRetry) {
3343 0 : HCCL_RUN_INFO("[OpRetry][AICPU]hccl aicpu recv retry cmd from host.");
3344 0 : dfxExtendInfo_.pollStatus = PollStatus::kDefault;
3345 0 : dfxExtendInfo_.cqeStatus = dfx::CqeStatus::kDefault;
3346 0 : ret = ResetOpRetryException(param.opType);
3347 0 : HCCL_RETRY_CHK_RET_AND_TRANS_FSM(
3348 : ret, HCCL_ERROR("reset stream buff failed, ret:%u", ret), KfcError::kInner,
3349 : HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR);
3350 0 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_FSM_RETRY;
3351 0 : } else if (cmd == KfcCommand::kChangeLink && lastCmd != KfcCommand::kChangeLink) { // 防止重复执行
3352 0 : HCCL_RUN_INFO("[OpRetry][AICPU]hccl aicpu recv change link cmd, identify[%s]", identifier_.c_str());
3353 0 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_FSM_CHANGE_LINK;
3354 0 : } else if (cmd == KfcCommand::kExit) {
3355 0 : HCCL_RUN_INFO("[OpRetry][AICPU]hccl aicpu recv exit cmd from host.");
3356 0 : errorCode = KfcError::kExit;
3357 0 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR;
3358 0 : } else if (cmd == KfcCommand::kReportRetryErr) {
3359 0 : HCCL_RUN_INFO("[OpRetry][AICPU]hccl aicpu get report retry err cmd.");
3360 0 : CHK_PRT(SendTaskExceptionByMBox(TS_ERROR_RETRY_CONSTRAINT));
3361 0 : errorCode = KfcError::kExit;
3362 0 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR;
3363 0 : return HCCL_E_OPRETRY_FAIL;
3364 0 : } else if (cmd == KfcCommand::NsStopLaunch && endStopLaunch == false) {
3365 0 : HCCL_RUN_INFO("[NsRecovery][AICPU]hccl aicpu force stop in launch loop.");
3366 0 : endStopLaunch = true;
3367 0 : needsResponseStopLaunch_ = true;
3368 0 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_STOP_LAUNCH;
3369 : } else {
3370 : // do nothing
3371 : }
3372 0 : lastCmd = cmd;
3373 0 : return HCCL_SUCCESS;
3374 : }
3375 :
3376 2 : HcclResult HcclCommAicpu::ResetOpRetryException(HcclCMDType opType)
3377 : {
3378 2 : if (opType == HcclCMDType::HCCL_CMD_BATCH_SEND_RECV) {
3379 0 : CHK_RET(ResetBSRException());
3380 : } else {
3381 6 : std::vector<Stream> totalStream = {mainStream_};
3382 2 : totalStream.insert(totalStream.end(), slaveStreams_.begin(), slaveStreams_.end());
3383 4 : for (auto& stream : totalStream) {
3384 2 : CHK_RET(CleanStream(stream));
3385 2 : CHK_RET(ClearStreamCqeException(stream));
3386 : }
3387 2 : }
3388 2 : return HCCL_SUCCESS;
3389 2 : }
3390 :
3391 0 : HcclResult HcclCommAicpu::ResetSqBuff()
3392 : {
3393 0 : CHK_RET(CleanStream(mainStream_));
3394 0 : for (auto& stream : slaveStreams_) {
3395 0 : CHK_RET(CleanStream(stream));
3396 : }
3397 0 : CHK_RET(CleanStream(orderStream_));
3398 0 : HCCL_INFO("reset stream sq buffer success.");
3399 0 : return HCCL_SUCCESS;
3400 : }
3401 :
3402 0 : HcclResult HcclCommAicpu::UpdateSqStatus(Stream& stream)
3403 : {
3404 0 : HcclSqeContext* sqeContext = stream.GetSqeContextPtr();
3405 0 : CHK_PTR_NULL(sqeContext);
3406 0 : SqeRingBuffer* sqeContextBuffer = &(sqeContext->buffer);
3407 0 : CHK_PTR_NULL(sqeContextBuffer);
3408 0 : auto& head = sqeContextBuffer->sqHead;
3409 0 : auto& tail = sqeContextBuffer->sqTail;
3410 :
3411 0 : CHK_RET(QuerySqStatusByType(devId_, stream.sqId(), DRV_SQCQ_PROP_SQ_TAIL, head));
3412 0 : CHK_RET(QuerySqStatusByType(devId_, stream.sqId(), DRV_SQCQ_PROP_SQ_HEAD, tail));
3413 0 : HCCL_INFO("UpdateSqStatus, sqid:%u head:%u tail:%u.", stream.sqId(), head, tail);
3414 0 : return HCCL_SUCCESS;
3415 : }
3416 :
3417 0 : HcclResult HcclCommAicpu::HcclOpExecFsmRetryProcess(
3418 : const std::string& algName, OpParam& param, std::unique_ptr<CollExecutorBase>& executor,
3419 : AlgResourceResponse& algResource, HcclOpExecFSM& fsmState, KfcError& errorCode, uint32_t& retryCnt,
3420 : uint32_t& beginSqePos, uint32_t& endSqePos)
3421 : {
3422 0 : retryCnt++;
3423 0 : if ((param.opType == HcclCMDType::HCCL_CMD_BATCH_SEND_RECV) && retryEnable_) {
3424 0 : UpdateBSRRetryCnt();
3425 : }
3426 0 : HCCL_RUN_INFO("[OpRetry][AICPU]retry launch start, retryCnt:%u, tag[%s].", retryCnt, param.tag.c_str());
3427 :
3428 0 : auto ret = RetryOrchestrateHcclOp(algName, param, executor, algResource, beginSqePos, endSqePos);
3429 0 : if (ret == HCCL_SUCCESS) {
3430 0 : errorCode = KfcError::kNone;
3431 0 : if ((param.opType == HcclCMDType::HCCL_CMD_BATCH_SEND_RECV) && retryEnable_) {
3432 : // 是否有前一个重执行阶段积累的故障未上报，需要再次触发重执行
3433 0 : CHK_RET(CommitBSRStoredException(fsmState, errorCode));
3434 0 : } else {
3435 0 : CHK_RET(UpdateOpExecStatus(fsmState, KfcStatus::kRuning, errorCode, retryCnt));
3436 0 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_FSM_WAIT_END;
3437 : }
3438 0 : } else if (ret == HCCL_E_SUSPENDING) {
3439 0 : HCCL_RUN_INFO("hccl aicpu force stop in retry launch process");
3440 0 : if ((param.opType == HcclCMDType::HCCL_CMD_BATCH_SEND_RECV) && retryEnable_) {
3441 : // batchsendrecv算子下发过程中出现异常，task下发未完成，send 和 recv 均需要重执行
3442 0 : if (bsrRetryOp_ == HCCL_SEND) {
3443 0 : SetBSRSendOpExecException();
3444 : } else {
3445 0 : SetBSRRecvOpExecException();
3446 : }
3447 0 : HCCL_RUN_INFO("hccl aicpu abort launch batchsendrecv op, need retry.");
3448 : }
3449 0 : CHK_RET(UpdateSuspendStatus(param, fsmState, errorCode, retryCnt));
3450 : } else {
3451 0 : HCCL_ERROR("RetryLaunchHcclOp failed, ret:%u", ret);
3452 0 : errorCode = KfcError::kInner;
3453 0 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR;
3454 : }
3455 0 : return ret;
3456 : }
3457 :
3458 0 : HcclResult HcclCommAicpu::HcclOpExecFsmEndProcess(uint32_t retryCnt)
3459 : {
3460 : auto ret
3461 0 : = aicpuHdc_.SetOpExecStatus(kfcStatusTransferD2H_, excuteOpId_, KfcStatus::kEnd, KfcError::kNone, retryCnt);
3462 0 : if (!isDeviceMode_) {
3463 0 : isOpLaunch = false;
3464 : }
3465 0 : dfxExtendInfo_.kfcStatus = DfxKfcStatus::kOneFinished;
3466 0 : HCCL_DEBUG("---------- end AICPU_HcclOpExecFsmEndProcess ----------");
3467 0 : return ret;
3468 : }
3469 :
3470 0 : HcclResult HcclCommAicpu::PrintTaskExceptionAllThreads()
3471 : {
3472 : // 非独立算子场景，跳过
3473 0 : CHK_PRT_RET(
3474 : !GetIsInitIndOp(), HCCL_RUN_INFO("[%s] IndOp group[%s] not init, skip", __func__, identifier_.c_str()),
3475 : HCCL_SUCCESS);
3476 :
3477 0 : for (auto& thread : threads_) {
3478 0 : CHK_RET(taskExecption_.PrintTaskException(*(thread->GetStream())));
3479 : }
3480 0 : return HCCL_SUCCESS;
3481 : }
3482 :
3483 0 : void HcclCommAicpu::PrintTaskExceptionAllComm()
3484 : {
3485 0 : std::shared_lock<std::shared_mutex> rwlock(AicpuHcclProcess::AicpuGetCommMutex());
3486 :
3487 : // 先打印本通信域的taskException
3488 0 : (void)PrintTaskExceptionAllStreams();
3489 0 : (void)PrintTaskExceptionAllThreads();
3490 :
3491 : // 再打印其他通信域的taskException
3492 0 : std::vector<std::pair<std::string, hccl::HcclCommAicpu*>> aicpuCommInfo;
3493 0 : (void)AicpuHcclProcess::AicpuGetCommAll(aicpuCommInfo);
3494 0 : for (auto& commInfo : aicpuCommInfo) {
3495 0 : hccl::HcclCommAicpu* hcclAicpu = commInfo.second;
3496 0 : if (hcclAicpu == nullptr || hcclAicpu->identifier_ == identifier_) {
3497 0 : continue;
3498 : }
3499 0 : (void)hcclAicpu->PrintTaskExceptionAllThreads();
3500 0 : (void)hcclAicpu->PrintTaskExceptionAllStreams();
3501 : }
3502 0 : }
3503 :
3504 0 : void HcclCommAicpu::PrintAicpuCommExecStatus()
3505 : {
3506 0 : std::shared_lock<std::shared_mutex> rwlock(AicpuHcclProcess::AicpuGetCommMutex());
3507 :
3508 : // 记录通信域占核情况
3509 0 : int64_t inExecGroupNum = 0;
3510 0 : int64_t aicpuCoreNum = 0;
3511 0 : (void)hrtHalGetDeviceInfo(devId_, MODULE_TYPE_AICPU, INFO_TYPE_CORE_NUM, &aicpuCoreNum);
3512 :
3513 0 : std::vector<std::pair<std::string, hccl::HcclCommAicpu*>> aicpuCommInfo;
3514 0 : (void)AicpuHcclProcess::AicpuGetCommAll(aicpuCommInfo);
3515 0 : for (auto& commInfo : aicpuCommInfo) {
3516 0 : hccl::HcclCommAicpu* hcclAicpu = commInfo.second;
3517 0 : if (hcclAicpu == nullptr || !hcclAicpu->GetCommInfoStatus()) {
3518 0 : continue;
3519 : }
3520 :
3521 : // 获取并打印通信域是否在执行中，以及最后一次下发的算子
3522 0 : bool isInExec = AicpuHcclProcess::GetCommExecStatus(hcclAicpu->identifier_);
3523 0 : inExecGroupNum += isInExec ? 1 : 0;
3524 0 : std::string execStatus = isInExec ? "inExec" : "unExec";
3525 0 : HCCL_RUN_INFO(
3526 : "AicpuComm: group[%s], status[%s], op[%s], aicpuCoreNum[%lld]", hcclAicpu->identifier_.c_str(),
3527 : execStatus.c_str(), hcclAicpu->GetExcuteOp().c_str(), aicpuCoreNum);
3528 0 : }
3529 :
3530 : // AICPU核被占满，部分通信域得不到调度，可能导致通信阻塞，打印维测信息
3531 0 : if (inExecGroupNum >= aicpuCoreNum && static_cast<int64_t>(aicpuCommInfo.size()) > aicpuCoreNum) {
3532 0 : HCCL_RUN_WARNING(
3533 : "In Execution group num[%lld], total group num[%u], bigger than Aicpu cores num[%lld]. "
3534 : "Aicpu core being fully utilized may cause tasks to get stuck, and it is necessary to reduce the num of "
3535 : "comm.",
3536 : inExecGroupNum, aicpuCommInfo.size(), aicpuCoreNum);
3537 : }
3538 0 : }
3539 :
3540 0 : HcclResult HcclCommAicpu::PrintTaskExceptionAllStreams()
3541 : {
3542 : // 通信域资源已经释放
3543 0 : CHK_PRT_RET(
3544 : !commOpenStatus,
3545 : HCCL_RUN_INFO("[PrintTaskExceptionAllStreams]group[%s] has been destroyed", identifier_.c_str()), HCCL_SUCCESS);
3546 0 : CHK_RET(UtraceInfo_->Flush());
3547 0 : std::vector<Stream> totalStream = {mainStream_};
3548 : HcclResult ret;
3549 0 : totalStream.insert(totalStream.end(), slaveStreams_.begin(), slaveStreams_.end());
3550 0 : for (auto& stream : totalStream) {
3551 0 : HCCL_RUN_INFO("[PrintTaskExceptionAllStreams]group[%s] streamid[%d] print", identifier_.c_str(), stream.id());
3552 0 : u32 sqHead = 0U;
3553 0 : u32 sqTail = 0U;
3554 0 : ret = QuerySqStatus(devId_, stream.sqId(), sqHead, sqTail);
3555 0 : if (ret != HCCL_SUCCESS || sqHead == sqTail) { // 此流为空时，不打印
3556 0 : HCCL_RUN_INFO(
3557 : "[PrintTaskExceptionAllStreams] group[%s] streamid[%d] is empty"
3558 : "or QuerySqStatus failed, ret[%d]",
3559 : identifier_.c_str(), stream.id(), ret);
3560 0 : continue;
3561 : }
3562 0 : HcclSqeContext* sqeContext = stream.GetSqeContextPtr();
3563 0 : SqeRingBuffer* sqeContextBuffer = &(sqeContext->buffer);
3564 0 : CHK_PTR_NULL(sqeContextBuffer);
3565 0 : if (stream.id() == mainStream_.id()) {
3566 0 : SqeInfo sqeInfo;
3567 0 : ret = SqeContextUtils::QuerySqeInfo(
3568 0 : sqeContextBuffer->rtsMirrorBuffer + sqHead * HCCL_SQE_SIZE, sqeContextBuffer->rtsqSqeType[sqHead],
3569 : sqeContextBuffer->addInfo[sqHead], &sqeInfo);
3570 0 : if (ret != HCCL_SUCCESS) {
3571 0 : HCCL_ERROR("[%s]QuerySqeInfo failed, ret[%d]", __func__, ret);
3572 : } else {
3573 : // 根据主流卡在host notify上，则说明未被执行到不打印
3574 0 : if (sqeInfo.type == RT_STARS_SQE_TYPE_NOTIFY_WAIT && sqeInfo.notifyId == opNotifies_[0]->notifyId_) {
3575 0 : HCCL_RUN_INFO(
3576 : "[PrintTaskExceptionAllStreams] group[%s] op is not activated, do nothing",
3577 : identifier_.c_str());
3578 0 : return HCCL_SUCCESS;
3579 : }
3580 : // 根据主流当前位置，判断该算子是否已经打印过taskException
3581 0 : if (IsRepeatedOpTaskException(sqHead, sqeContextBuffer)) {
3582 0 : HCCL_INFO(
3583 : "[PrintTaskExceptionAllStreams] group[%s] op has been printed, do nothing",
3584 : identifier_.c_str());
3585 0 : return HCCL_SUCCESS;
3586 : }
3587 : }
3588 : }
3589 :
3590 0 : uint8_t type = 0;
3591 0 : uint16_t taskId = 0;
3592 0 : uint32_t remoteRank = 0;
3593 0 : HCCL_ERROR(
3594 : "[TaskException]base information is streamId:%d, sqid:%d, head:%u, tail:%u, %s", stream.id(), stream.sqId(),
3595 : sqHead, sqTail, GetTaskExceptionTaskInfo(sqHead, sqeContextBuffer, type, taskId, remoteRank).c_str());
3596 0 : PrintTaskExceptionTaskQue(sqHead, sqeContextBuffer);
3597 : }
3598 0 : return HCCL_SUCCESS;
3599 0 : }
3600 :
3601 0 : bool HcclCommAicpu::IsRepeatedOpTaskException(u32 idx, SqeRingBuffer* sqeContextBuffer)
3602 : {
3603 0 : const AicpuOpInfo* opInfo = aicpuShareData_.GetAicpuOpInfo(sqeContextBuffer->rtsDfxInfo[idx].opRingBufferIdx);
3604 0 : CHK_PRT_RET(opInfo == nullptr, HCCL_ERROR("%s fail, opInfo is nullptr", __func__), false);
3605 0 : std::string opTag = opInfo->tagBuff;
3606 0 : u32 opIndex = opInfo->opIndex;
3607 0 : bool opHasPrinted = opTaskException_.find(opTag) != opTaskException_.end() && opTaskException_[opTag] == opIndex;
3608 0 : opTaskException_[opTag] = opIndex;
3609 0 : CHK_PRT_CONT(
3610 : opHasPrinted, HCCL_RUN_INFO(
3611 : "[IsRepeatedOpTaskException]group[%s], op[%s], opIndex[%u] "
3612 : "has been printed",
3613 : identifier_.c_str(), opInfo->tagBuff, opInfo->opIndex));
3614 0 : return opHasPrinted;
3615 0 : }
3616 :
3617 0 : void HcclCommAicpu::PrepareMc2Handler()
3618 : {
3619 0 : auto& handler = algOpContext_.mc2Handler;
3620 0 : handler.stepSize = 0U;
3621 0 : if (!isDeviceMode_) {
3622 0 : HCCL_INFO("Unset step size for non-MC2.");
3623 0 : return;
3624 : }
3625 0 : (void)InvokeKfcHandler(AicpuKfcHandlerType::kSetStepSize, {rpc_, reinterpret_cast<u64>(&handler), GetRankSize()});
3626 : }
3627 :
3628 0 : HcclResult HcclCommAicpu::OrchestrateHcclOp(
3629 : const std::string& algName, OpParam& param, std::unique_ptr<CollExecutorBase>& executor,
3630 : AlgResourceResponse& algResource, uint32_t& beginSqePos, uint32_t& endSqePos)
3631 : {
3632 0 : LogControl logControl(false, false); // 重执行ERROR日志控制，析构时重置日志设置
3633 0 : PrepareMc2Handler();
3634 0 : HcclResult ret = HCCL_SUCCESS;
3635 : // task的尾指针，已便重执行stop时判断是否已执行该task，如果该task已执行完成则可支持通信重执行
3636 0 : CHK_RET(QuerySqStatusByType(devId_, mainStream_.sqId(), DRV_SQCQ_PROP_SQ_TAIL, beginSqePos));
3637 :
3638 0 : const bool retryForBatchSndRcv = (param.opType == HcclCMDType::HCCL_CMD_BATCH_SEND_RECV && retryEnable_);
3639 0 : if (retryForBatchSndRcv) {
3640 0 : CHK_RET(QueryBatchSendRecvPairBeginPos());
3641 0 : if (param.BatchSendRecvDataDes.curIterNum == 0) {
3642 : // batchsendrecv算子拆分为多轮执行，只有第一个step和最后一个step需要和主stream交互
3643 0 : CHK_RET(NotifyWait());
3644 : }
3645 0 : HCCL_INFO(
3646 : "batch send recv op: step %u, mode:%u", param.BatchSendRecvDataDes.curIterNum,
3647 : param.BatchSendRecvDataDes.curMode);
3648 : } else {
3649 0 : CHK_RET(NotifyWait());
3650 : // 重执行场景, 算子计数在host侧; MC2场景也不开启卡住检测能力
3651 0 : if (opCounterInfo_.isEnableCounter && !retryEnable_ && !isDeviceMode_) {
3652 0 : CHK_RET(HcclReduceAsync(
3653 : dispatcher_, reinterpret_cast<void*>(opCounterInfo_.addOneMem),
3654 : opCounterInfo_.memSize / sizeof(int32_t), HCCL_DATA_TYPE_INT32, HCCL_REDUCE_SUM, mainStream_,
3655 : reinterpret_cast<void*>(opCounterInfo_.headCountMem), INVALID_VALUE_RANKID, LinkType::LINK_ONCHIP,
3656 : INLINE_REDUCE_BIT));
3657 : }
3658 : }
3659 :
3660 : // 打印当前展开的算子信息
3661 0 : HCCL_INFO(
3662 : "[HcclCommAicpu][OrchestrateHcclOp] opUnfoldIdx_[%u] opType[%d] curRank[%u] rankSize[%u] algName[%s]",
3663 : opUnfoldIdx_, param.opType, topoInfo_.userRank, GetRankSize(), algName.c_str());
3664 0 : HCCL_INFO(
3665 : "[HcclCommAicpu][OrchestrateHcclOp] inputPtr[0x%016llx] inputSize[%llu] outputPtr[0x%016llx] outputSize[%llu]",
3666 : param.inputPtr, param.inputSize, param.outputPtr, param.outputSize);
3667 0 : opUnfoldIdx_ += 1;
3668 :
3669 : // 检查算子展开的动态缓存, 确认是否可以跳过算子展开
3670 0 : bool needExecute = true;
3671 0 : bool isCacheMiss = false;
3672 0 : auto setProfStartCallback = [this]() {
3673 0 : return this->InvokeKfcHandler(AicpuKfcHandlerType::kSetProfTimeStart, {});
3674 0 : };
3675 0 : CHK_RET(aicpuCacheManager_.LookupOpUnfoldCache(
3676 : algName, param, algResource, needExecute, isCacheMiss, mainStream_, slaveStreams_, dispatcher_, isDeviceMode_,
3677 : topoInfo_, topoMatcher_, algOpContext_, ZeroCopyExchanger_, GetWorkflowMode(), tinySendRecvMem_,
3678 : setProfStartCallback));
3679 :
3680 : // 根据needExecute有条件的执行算子展开
3681 : // 需要算子执行的场景: (i) expansion mode为AI_CPU_NO_CACHE; (ii) uncacheable算子/场景; (iii) cache miss
3682 0 : if (needExecute) {
3683 : // Cache miss前执行cache相关的预处理
3684 0 : if (isCacheMiss) {
3685 0 : CHK_RET(aicpuCacheManager_.PreProcessForCacheMiss(param, executor));
3686 : }
3687 :
3688 : // executor设置AlgOpContext
3689 0 : CHK_RET(executor->SetAlgOpContext(algOpContext_));
3690 0 : (void)InvokeKfcHandler(AicpuKfcHandlerType::kSetProfTimeStart, {});
3691 0 : ret = executor->Orchestrate(param, algResource);
3692 0 : if (ret != HCCL_SUCCESS) {
3693 0 : HCCL_ERROR(
3694 : "[HcclCommAicpu][Orchestrate]executor process failed algName[%s], ret = %u", algName.c_str(), ret);
3695 0 : printTaskExceptionForErr_ |= (ret == HCCL_E_AGAIN);
3696 0 : return ret;
3697 : }
3698 :
3699 : // Cache miss后执行cache相关的后处理
3700 0 : if (isCacheMiss) {
3701 0 : CHK_RET(aicpuCacheManager_.PostProcessForCacheMiss(
3702 : param, executor, mainStream_, slaveStreams_, dispatcher_, topoInfo_, algOpContext_, GetWorkflowMode()));
3703 : }
3704 : } // 正常算子展开
3705 :
3706 : // batchsendrecv算子拆分为多轮执行，只有第一个step和最后一个step需要和主stream交互
3707 0 : if (!retryForBatchSndRcv || param.BatchSendRecvDataDes.curIterNum + 1 >= bsrSendRecvPairs_.size()) {
3708 : // 重执行场景, 算子计数在host侧 MC2场景也不开
3709 0 : if (opCounterInfo_.isEnableCounter && !retryEnable_ && !isDeviceMode_) {
3710 0 : CHK_RET(HcclReduceAsync(
3711 : dispatcher_, reinterpret_cast<void*>(opCounterInfo_.addOneMem),
3712 : opCounterInfo_.memSize / sizeof(int32_t), HCCL_DATA_TYPE_INT32, HCCL_REDUCE_SUM, mainStream_,
3713 : reinterpret_cast<void*>(opCounterInfo_.tailCountMem), INVALID_VALUE_RANKID, LinkType::LINK_ONCHIP,
3714 : INLINE_REDUCE_BIT));
3715 : }
3716 0 : CHK_RET(NotifyPost());
3717 : }
3718 0 : (void)InvokeKfcHandler(AicpuKfcHandlerType::kSetProfTimeOrch, {});
3719 0 : ret = LaunchTask(dispatcher_, const_cast<Stream&>(mainStream_));
3720 0 : if (ret != HCCL_SUCCESS) {
3721 0 : HCCL_ERROR("[HcclCommAicpu][LaunchTask]algName[%s] ret = %u", algName.c_str(), ret);
3722 0 : printTaskExceptionForErr_ |= (ret == HCCL_E_AGAIN);
3723 0 : return ret;
3724 : }
3725 0 : ret = LaunchSlaveStreamTask(algResource);
3726 0 : if (ret != HCCL_SUCCESS) {
3727 0 : HCCL_ERROR("[HcclCommAicpu][LaunchSlaveStreamTask]algName[%s] ret = %u", algName.c_str(), ret);
3728 0 : printTaskExceptionForErr_ |= (ret == HCCL_E_AGAIN);
3729 0 : return ret;
3730 : }
3731 0 : (void)InvokeKfcHandler(AicpuKfcHandlerType::kSetProfTimeEnd, {});
3732 0 : if (retryForBatchSndRcv) {
3733 0 : CHK_RET(QueryBatchSendRecvPairEndPos());
3734 : }
3735 :
3736 0 : CHK_RET(QuerySqStatusByType(devId_, mainStream_.sqId(), DRV_SQCQ_PROP_SQ_TAIL, endSqePos));
3737 :
3738 0 : HCCL_INFO(
3739 : "hccl aicpu launch hccl op task success. stream sqid:%u begin:%u end:%u", mainStream_.sqId(), beginSqePos,
3740 : endSqePos);
3741 0 : return HCCL_SUCCESS;
3742 0 : }
3743 :
3744 0 : HcclResult HcclCommAicpu::RetryOrchestrateHcclOp(
3745 : const std::string& algName, OpParam& param, std::unique_ptr<CollExecutorBase>& executor,
3746 : AlgResourceResponse& algResource, uint32_t& beginSqePos, uint32_t& endSqePos)
3747 : {
3748 0 : LogControl logControl(false, false); // 重执行ERROR日志控制，析构时重置日志设置
3749 0 : if ((param.opType == HcclCMDType::HCCL_CMD_BATCH_SEND_RECV) && retryEnable_) {
3750 : param.BatchSendRecvDataDes.curMode
3751 0 : = (bsrRetryOp_ == HCCL_SEND) ? BatchSendRecvCurMode::SEND : BatchSendRecvCurMode::RECV;
3752 0 : if (param.BatchSendRecvDataDes.curMode == BatchSendRecvCurMode::SEND) {
3753 0 : HCCL_INFO(
3754 : "BSR: iter %u, retry send op tag:%s index:%u", param.BatchSendRecvDataDes.curIterNum, bsrSendOpId_.tag,
3755 : bsrSendOpId_.index);
3756 : } else {
3757 0 : HCCL_INFO(
3758 : "BSR: iter %u, retry recv op tag:%s index:%u", param.BatchSendRecvDataDes.curIterNum, bsrRecvOpId_.tag,
3759 : bsrRecvOpId_.index);
3760 : }
3761 : }
3762 :
3763 0 : CHK_RET(AddRetryExecFlipTask(algResource));
3764 0 : HcclResult ret = executor->Orchestrate(param, algResource);
3765 0 : if (ret != HCCL_SUCCESS) {
3766 0 : HCCL_ERROR("[HcclCommAicpu][Orchestrate]executor process failed algName[%s]", algName.c_str());
3767 0 : return ret;
3768 : }
3769 :
3770 0 : if ((param.opType == HcclCMDType::HCCL_CMD_BATCH_SEND_RECV) && retryEnable_) {
3771 : // batchsendrecv算子重执行时，aicpu 主stream没有clean，不需要重新下发notify record
3772 : // do nothing
3773 : } else {
3774 0 : CHK_RET(NotifyPost());
3775 : }
3776 0 : CHK_RET(LaunchTask(dispatcher_, const_cast<Stream&>(mainStream_)));
3777 0 : CHK_RET(LaunchSlaveStreamTask(algResource));
3778 :
3779 0 : CHK_RET(QuerySqStatusByType(devId_, mainStream_.sqId(), DRV_SQCQ_PROP_SQ_TAIL, endSqePos));
3780 0 : if ((param.opType == HcclCMDType::HCCL_CMD_BATCH_SEND_RECV) && retryEnable_) {
3781 0 : CHK_RET(QueryBatchSendRecvPairEndPos());
3782 : }
3783 :
3784 0 : HCCL_RUN_INFO(
3785 : "[OpRetry][AICPU]hccl aicpu retry launch hccl op task success. stream sqid:%u begin:%u end:%u",
3786 : mainStream_.sqId(), beginSqePos, endSqePos);
3787 0 : return HCCL_SUCCESS;
3788 0 : }
3789 :
3790 4 : bool HcclCommAicpu::IsTaskExceptionForHccs()
3791 : {
3792 4 : if (dfxExtendInfo_.cqeStatus != dfx::CqeStatus::kCqeException) {
3793 4 : return false;
3794 : }
3795 :
3796 : // NOTE: 需要task exception补全dfx能力，定位故障task的remote rank;
3797 : // 目前暂不具备识别是否跨片的能力，默认失败的task均为跨片操作。
3798 0 : if (dfxExtendInfo_.cqeException.sqeType == RT_STARS_SQE_TYPE_SDMA
3799 0 : && (dfxExtendInfo_.cqeException.errorCode == RT_SDMA_COMPDATAERR
3800 0 : || dfxExtendInfo_.cqeException.errorCode == RT_SDMA_COMPERR)) {
3801 0 : return true;
3802 : }
3803 0 : return false;
3804 : }
3805 :
3806 3 : void HcclCommAicpu::SetAlgType(u64 algType)
3807 : {
3808 3 : algType_.algoLevel0 = static_cast<AlgTypeLevel0>(static_cast<u32>(algType) & ((1 << HCCL_LEVEL_ALGO_WIDTH) - 1));
3809 3 : algType_.algoLevel1 = static_cast<AlgTypeLevel1>(
3810 3 : (static_cast<u32>(algType) >> HCCL_LEVEL_ALGO_WIDTH) & ((1 << HCCL_LEVEL_ALGO_WIDTH) - 1));
3811 : algType_.algoLevel2
3812 3 : = static_cast<AlgTypeLevel2>(static_cast<u32>(algType) >> (HCCL_LEVEL_ALGO_WIDTH + HCCL_LEVEL_ALGO_WIDTH));
3813 3 : HCCL_INFO("[HcclCommAicpu][SetAlgType]algType:%u", algType);
3814 3 : }
3815 :
3816 0 : void HcclCommAicpu::SetDebugMode(u8 debugMode) { debugMode_ = debugMode; }
3817 :
3818 0 : void HcclCommAicpu::SetSendRecvInfoPtr(void* sendRecvInfoPtr) { sendRecvInfoPtr_ = sendRecvInfoPtr; }
3819 :
3820 4 : bool HcclCommAicpu::IsNoNeedWait(void)
3821 : {
3822 4 : return isDeviceMode_ || (debugMode_ != MC2_DEBUG_WAIT_COMM && retryEnable_ == false);
3823 : }
3824 :
3825 45 : bool HcclCommAicpu::GetOpRetryEnable() { return retryEnable_; }
3826 :
3827 4 : HcclResult HcclCommAicpu::ReportHcclTaskInfo(Stream& mainStream, std::vector<Stream>& subStreams)
3828 : {
3829 4 : if (dfx::ProfilingManager::GetProfL1State()) {
3830 0 : CHK_RET(dfx::ProfilingManager::ReportTaskInfo(mainStream.id(), mainStream.GetSqeContextPtr()));
3831 0 : for (auto& subStream : subStreams) {
3832 0 : CHK_RET(dfx::ProfilingManager::ReportTaskInfo(subStream.id(), subStream.GetSqeContextPtr()));
3833 : }
3834 : }
3835 4 : return HCCL_SUCCESS;
3836 : }
3837 :
3838 4 : HcclResult HcclCommAicpu::ClearLocalBuff(Stream& mainStream, std::vector<Stream>& subStreams)
3839 : {
3840 4 : CHK_RET(mainStream.ClearLocalBuff());
3841 4 : CHK_RET(dfx::ProfilingManager::UpdateStartReportSqeIdx(mainStream.id(), 0));
3842 4 : for (auto& subStream : subStreams) {
3843 0 : CHK_RET(subStream.ClearLocalBuff());
3844 0 : CHK_RET(dfx::ProfilingManager::UpdateStartReportSqeIdx(subStream.id(), 0));
3845 : }
3846 4 : return HCCL_SUCCESS;
3847 : }
3848 :
3849 4 : HcclResult HcclCommAicpu::WaitFinishWhileLoop(
3850 : Stream& mainStream, std::vector<Stream>& subStreams, std::string& tag, const uint32_t& beginSqePos, OpParam& param)
3851 : {
3852 : // 上报Profiling HCCL INFO信息
3853 4 : CHK_RET(ReportHcclTaskInfo(mainStream, subStreams));
3854 4 : CHK_RET(ClearLocalBuff(mainStream, subStreams));
3855 4 : if (IsNoNeedWait()) {
3856 0 : return HCCL_SUCCESS;
3857 : }
3858 4 : const uint64_t startUsec = GetCurCpuTimestamp();
3859 4 : uint64_t lastUsec = startUsec;
3860 4 : int32_t sqId = mainStream.sqId();
3861 4 : uint32_t sqHead = 0;
3862 4 : uint32_t sqTail = 0;
3863 4 : CHK_RET(QuerySqStatusByType(devId_, sqId, DRV_SQCQ_PROP_SQ_TAIL, sqTail));
3864 4 : CHK_RET(QuerySqStatusByType(devId_, sqId, DRV_SQCQ_PROP_SQ_HEAD, sqHead));
3865 4 : while (sqHead != sqTail) {
3866 4 : HcclResult ret = CheckOpExecStatus(); // 检查执行状态，判断是否有异常cq或中断命令
3867 4 : CHK_PRT_RET(
3868 : ret != HCCL_SUCCESS,
3869 : HCCL_RUN_INFO("[HcclCommAicpu][WaitFinishWhileLoop]CheckOpExecStatus exception, ret[%u]", ret), ret);
3870 :
3871 0 : CHK_RET(QuerySqStatusByType(devId_, sqId, DRV_SQCQ_PROP_SQ_HEAD, sqHead));
3872 0 : if ((param.opType == HcclCMDType::HCCL_CMD_BATCH_SEND_RECV) && (retryEnable_) && (sqHead != beginSqePos)
3873 0 : && (!excuteOpId_.isBsrTaskStart)) {
3874 : // 更新D2H通道里的信息opid中isBsrTaskStart = true
3875 0 : excuteOpId_.isBsrTaskStart = true;
3876 : HcclResult ret1
3877 0 : = aicpuHdc_.SetOpExecStatus(kfcStatusTransferD2H_, excuteOpId_, KfcStatus::kRuning, KfcError::kNone, 0);
3878 0 : CHK_PRT_RET(ret1 != HCCL_SUCCESS, HCCL_ERROR("update OpExecStatus failed, ret:%u", ret1), ret1);
3879 0 : HCCL_INFO(
3880 : "[HcclCommAicpu][WaitFinishWhileLoop]bsr start task is completed. devId:%d sqid:%d, head:%u,"
3881 : "beginSqePos[%u] group[%s] tag[%s]",
3882 : devId_, sqId, sqHead, beginSqePos, identifier_.c_str(), tag.c_str());
3883 : }
3884 0 : uint64_t curUsec = GetCurCpuTimestamp();
3885 0 : if (curUsec - lastUsec > static_cast<uint64_t>(NSEC_PER_SEC) * dfx::kPrintSqInterval) {
3886 0 : lastUsec = curUsec;
3887 0 : HCCL_RUN_INFO(
3888 : "[HcclCommAicpu][WaitFinishWhileLoop]Current state. devId:%d sqid:%d, head:%u, tail:%u, "
3889 : "group[%s] tag[%s]",
3890 : devId_, sqId, sqHead, sqTail, identifier_.c_str(), tag.c_str());
3891 : }
3892 0 : CHK_RET(CheckTaskTimeout(mainStream, startUsec));
3893 : }
3894 0 : return HCCL_SUCCESS;
3895 : }
3896 :
3897 0 : HcclResult HcclCommAicpu::CheckTaskTimeout(const Stream& mainStream, const uint64_t startUsec)
3898 : {
3899 0 : if (sqeWaitTimeOut_ != 0
3900 0 : && (GetCurCpuTimestamp() - startUsec > static_cast<uint64_t>(NSEC_PER_SEC) * sqeWaitTimeOut_)) {
3901 0 : uint32_t status = 0U;
3902 0 : int32_t sqId = mainStream.sqId();
3903 0 : auto ret = QuerySqStatusByType(devId_, sqId, DRV_SQCQ_PROP_SQ_CQE_STATUS, status);
3904 0 : if (ret != 0) {
3905 0 : HCCL_ERROR("[HcclCommAicpu]QuerySqStatusByType status failed. ret = %u sqid:%d", ret, sqId);
3906 : }
3907 :
3908 0 : HCCL_ERROR("[HcclCommAicpu]KFC timeout.. group[%s].", identifier_.c_str());
3909 0 : printTaskExceptionForErr_ = true;
3910 0 : return HCCL_E_TIMEOUT;
3911 : }
3912 0 : return HCCL_SUCCESS;
3913 : }
3914 :
3915 0 : HcclResult HcclCommAicpu::AddRetryExecFlipTask(AlgResourceResponse& algResource)
3916 : {
3917 0 : CHK_RET(AddRetryPreamble(dispatcher_, mainStream_));
3918 0 : for (u32 i = 0; i < algResource.slaveStreams.size(); ++i) {
3919 0 : HcclResult ret = AddRetryPreamble(dispatcher_, algResource.slaveStreams[i]);
3920 0 : if (ret != HCCL_SUCCESS) {
3921 0 : HCCL_ERROR(
3922 : "[HcclCommAicpu][RetryOrchestrateHcclOp] launch place holder failed, sqid:%u, ret:%u",
3923 : algResource.slaveStreams[i].sqId(), ret);
3924 0 : return ret;
3925 : }
3926 : }
3927 0 : return HCCL_SUCCESS;
3928 : }
3929 :
3930 2 : HcclResult HcclCommAicpu::LaunchSlaveStreamTask(AlgResourceResponse& algResource)
3931 : {
3932 : // 单算子模式中在算法编排中已经执行过LaunchTask，所以这里不需要再执行
3933 : // 只有图模式需要再额外执行一次对从流中的task下发
3934 2 : if (GetWorkflowMode() != HcclWorkflowMode::HCCL_WORKFLOW_MODE_OPS_KERNEL_INFO_LIB) {
3935 1 : HCCL_INFO("[HcclCommAicpu][LaunchSlaveStreamTask] op base mode don't need launch slave stream task");
3936 1 : return HCCL_SUCCESS;
3937 : }
3938 :
3939 1 : for (u32 i = 0; i < algResource.slaveStreams.size(); ++i) {
3940 0 : HcclResult ret = LaunchTask(dispatcher_, algResource.slaveStreams[i]);
3941 0 : if (ret != HCCL_SUCCESS) {
3942 0 : HCCL_ERROR(
3943 : "[HcclCommAicpu][LaunchSlaveStreamTask] launch task failed, sqid:%u, ret:%u",
3944 : algResource.slaveStreams[i].sqId(), ret);
3945 0 : return ret;
3946 : }
3947 : }
3948 :
3949 1 : return HCCL_SUCCESS;
3950 : }
3951 :
3952 : HcclResult
3953 0 : HcclCommAicpu::GetAlltoAllvSendRecvInfo(const void* sendRecvInfoPtr, HcclDataType sendType, HcclDataType recvType)
3954 : {
3955 0 : allMeshAggregationSendRecvInfo_.clear();
3956 0 : u64 stepSize = sizeof(u64) * topoInfo_.userRankSize;
3957 0 : const u32 addrItemNum = 4;
3958 0 : const u32 recvLengthStep = 2;
3959 0 : const u32 recvOffsetStep = 3;
3960 0 : for (u32 i = 0; i < topoInfo_.userRankSize; i++) {
3961 0 : SendRecvInfo sendRecvInfo;
3962 0 : sendRecvInfo.sendLength.resize(topoInfo_.userRankSize);
3963 0 : sendRecvInfo.sendOffset.resize(topoInfo_.userRankSize);
3964 0 : sendRecvInfo.recvLength.resize(topoInfo_.userRankSize);
3965 0 : sendRecvInfo.recvOffset.resize(topoInfo_.userRankSize);
3966 0 : CHK_SAFETY_FUNC_RET(memcpy_s(
3967 : sendRecvInfo.sendLength.data(), stepSize,
3968 : static_cast<const u8*>(sendRecvInfoPtr) + i * stepSize * addrItemNum + 0 * stepSize, stepSize));
3969 0 : CHK_SAFETY_FUNC_RET(memcpy_s(
3970 : sendRecvInfo.sendOffset.data(), stepSize,
3971 : static_cast<const u8*>(sendRecvInfoPtr) + i * stepSize * addrItemNum + stepSize, stepSize));
3972 0 : CHK_SAFETY_FUNC_RET(memcpy_s(
3973 : sendRecvInfo.recvLength.data(), stepSize,
3974 : static_cast<const u8*>(sendRecvInfoPtr) + i * stepSize * addrItemNum + recvLengthStep * stepSize,
3975 : stepSize));
3976 0 : CHK_SAFETY_FUNC_RET(memcpy_s(
3977 : sendRecvInfo.recvOffset.data(), stepSize,
3978 : static_cast<const u8*>(sendRecvInfoPtr) + i * stepSize * addrItemNum + recvOffsetStep * stepSize,
3979 : stepSize));
3980 0 : allMeshAggregationSendRecvInfo_.push_back(std::move(sendRecvInfo));
3981 0 : }
3982 :
3983 0 : for (auto& sendRecvInfo : allMeshAggregationSendRecvInfo_) {
3984 0 : for (u32 i = 0; i < topoInfo_.userRankSize; i++) {
3985 0 : sendRecvInfo.sendCounts.push_back(sendRecvInfo.sendLength[i] / SIZE_TABLE[sendType]);
3986 0 : sendRecvInfo.sendDispls.push_back(sendRecvInfo.sendOffset[i] / SIZE_TABLE[sendType]);
3987 0 : sendRecvInfo.recvCounts.push_back(sendRecvInfo.recvLength[i] / SIZE_TABLE[recvType]);
3988 0 : sendRecvInfo.recvDispls.push_back(sendRecvInfo.recvOffset[i] / SIZE_TABLE[recvType]);
3989 0 : HCCL_INFO(
3990 : "[GetAlltoAllvSendRecvInfo] rank[%u], sendCounts[%llu], sendDispls[%llu], "
3991 : "recvCounts[%llu], recvDispls[%llu]",
3992 : i, sendRecvInfo.sendCounts[i], sendRecvInfo.sendDispls[i], sendRecvInfo.recvCounts[i],
3993 : sendRecvInfo.recvDispls[i]);
3994 0 : HCCL_INFO(
3995 : "[GetAlltoAllvSendRecvInfo] rank[%u], sendLength[%llu], sendOffset[%llu], "
3996 : "recvLength[%llu], recvOffset[%llu]",
3997 : i, sendRecvInfo.sendLength[i], sendRecvInfo.sendOffset[i], sendRecvInfo.recvLength[i],
3998 : sendRecvInfo.recvOffset[i]);
3999 : }
4000 : }
4001 0 : CHK_RET(CheckSendRecvParams(allMeshAggregationSendRecvInfo_));
4002 0 : return HCCL_SUCCESS;
4003 : }
4004 :
4005 : HcclResult
4006 0 : HcclCommAicpu::GetAlltoAllvcSendRecvInfo(const void* sendCountMatrix, HcclDataType sendType, HcclDataType recvType)
4007 : {
4008 0 : allMeshAggregationSendRecvInfo_.clear();
4009 0 : for (u32 i = 0; i < topoInfo_.userRankSize; i++) {
4010 0 : SendRecvInfo sendRecvInfo;
4011 0 : sendRecvInfo.sendCounts.resize(topoInfo_.userRankSize);
4012 0 : sendRecvInfo.sendDispls.resize(topoInfo_.userRankSize);
4013 0 : sendRecvInfo.sendLength.resize(topoInfo_.userRankSize);
4014 0 : sendRecvInfo.sendOffset.resize(topoInfo_.userRankSize);
4015 0 : u64 curSendDispls = 0;
4016 0 : u64 curSendOffset = 0;
4017 0 : sendRecvInfo.recvCounts.resize(topoInfo_.userRankSize);
4018 0 : sendRecvInfo.recvDispls.resize(topoInfo_.userRankSize);
4019 0 : sendRecvInfo.recvLength.resize(topoInfo_.userRankSize);
4020 0 : sendRecvInfo.recvOffset.resize(topoInfo_.userRankSize);
4021 0 : u64 curRecvDispls = 0;
4022 0 : u64 curRecvOffset = 0;
4023 0 : for (u32 j = 0; j < topoInfo_.userRankSize; j++) {
4024 0 : u64 curSendCounts = *(static_cast<const u64*>(sendCountMatrix) + i * topoInfo_.userRankSize + j);
4025 0 : u64 curSendLength = curSendCounts * SIZE_TABLE[sendType];
4026 0 : sendRecvInfo.sendCounts[j] = curSendCounts;
4027 0 : sendRecvInfo.sendDispls[j] = curSendDispls;
4028 0 : sendRecvInfo.sendLength[j] = curSendLength;
4029 0 : sendRecvInfo.sendOffset[j] = curSendOffset;
4030 0 : curSendDispls += curSendCounts;
4031 0 : curSendOffset += curSendLength;
4032 0 : u64 curRecvCounts = *(static_cast<const u64*>(sendCountMatrix) + i + topoInfo_.userRankSize * j);
4033 0 : u64 curRecvLength = curRecvCounts * SIZE_TABLE[recvType];
4034 0 : sendRecvInfo.recvCounts[j] = curRecvCounts;
4035 0 : sendRecvInfo.recvDispls[j] = curRecvDispls;
4036 0 : sendRecvInfo.recvLength[j] = curRecvLength;
4037 0 : sendRecvInfo.recvOffset[j] = curRecvOffset;
4038 0 : curRecvDispls += curRecvCounts;
4039 0 : curRecvOffset += curRecvLength;
4040 0 : HCCL_DEBUG(
4041 : "GetAlltoAllvcSendRecvInfo rank[%u], sendCounts[%llu], sendDispls[%llu] "
4042 : "recvCounts[%llu], recvDispls[%llu]",
4043 : i, sendRecvInfo.sendCounts[j], sendRecvInfo.sendDispls[j], sendRecvInfo.recvCounts[j],
4044 : sendRecvInfo.recvDispls[j]);
4045 : }
4046 0 : allMeshAggregationSendRecvInfo_.push_back(sendRecvInfo);
4047 0 : }
4048 0 : CHK_RET(CheckSendRecvParams(allMeshAggregationSendRecvInfo_));
4049 0 : return HCCL_SUCCESS;
4050 : }
4051 :
4052 0 : HcclResult HcclCommAicpu::CheckSendRecvParams(const std::vector<SendRecvInfo>& allMeshAggregationSendRecvInfo)
4053 : {
4054 0 : u32 rankSize = allMeshAggregationSendRecvInfo.size();
4055 0 : for (u32 i = 0; i < rankSize; i++) {
4056 0 : u32 sendsSize = allMeshAggregationSendRecvInfo[i].sendLength.size();
4057 0 : u32 recvsSize = allMeshAggregationSendRecvInfo[i].recvLength.size();
4058 0 : if (rankSize != sendsSize || rankSize != recvsSize) {
4059 0 : HCCL_ERROR(
4060 : "[AlltoAllV][CheckSendRecvParam] rankSize[%u], sendsSize[%u], recvsSize[%u] are not match Index[%u]",
4061 : rankSize, sendsSize, recvsSize, i);
4062 0 : return HCCL_E_PARA;
4063 : }
4064 0 : for (u32 j = 0; j < sendsSize; j++) {
4065 0 : if (allMeshAggregationSendRecvInfo[i].sendLength[j] != allMeshAggregationSendRecvInfo[j].recvLength[i]) {
4066 0 : HCCL_ERROR(
4067 : "SendLength[%u][%u]: %llu and recvLength[%u][%u]: %llu are not match", i, j,
4068 : allMeshAggregationSendRecvInfo[i].sendLength[j], j, i,
4069 : allMeshAggregationSendRecvInfo[j].recvLength[i]);
4070 0 : return HCCL_E_PARA;
4071 : }
4072 : }
4073 : }
4074 0 : return HCCL_SUCCESS;
4075 : }
4076 0 : HcclResult HcclCommAicpu::GetStreamAll(std::vector<Stream>& streams)
4077 : {
4078 0 : streams.assign(slaveStreams_.begin(), slaveStreams_.end());
4079 0 : streams.push_back(mainStream_);
4080 0 : return HCCL_SUCCESS;
4081 : }
4082 :
4083 : // 校验是否有ERROR CQE和停止/退出命令，注册到dispatcher层调用
4084 0 : HcclResult HcclCommAicpu::CheckOpExecStatusCallback()
4085 : {
4086 0 : HcclResult ret = CheckOpExecStatus();
4087 0 : bool logLevel = (ret == HCCL_E_SUSPENDING);
4088 : // 返回HCCL_E_SUSPENDING时，需要跨作用域修改ERROR日志->RUN_WARNING
4089 0 : LogControl(logLevel, logLevel);
4090 0 : return ret;
4091 : }
4092 :
4093 4 : HcclResult HcclCommAicpu::CheckOpExecStatus()
4094 : {
4095 : // 检测是否有ERROR CQE
4096 4 : if (dfxExtendInfo_.pollStatus == PollStatus::kStopAsException) {
4097 0 : if (IsTaskExceptionForHccs() && retryEnable_) {
4098 0 : HCCL_RUN_INFO(
4099 : "[OpRetry][AICPU]hccl aicpu stop wait task exec finish, for task exception, identify[%s]",
4100 : identifier_.c_str());
4101 0 : return HCCL_E_SUSPENDING;
4102 : } else {
4103 0 : if (!printTaskExceptionForErr_) {
4104 0 : printTaskExceptionForErr_ = true;
4105 0 : HCCL_ERROR(
4106 : "hccl aicpu exec failed, for task exception, identify[%s], cqeStatus[%d], sqeType[%u], "
4107 : "errorCode[%u]",
4108 : identifier_.c_str(), dfxExtendInfo_.cqeStatus, dfxExtendInfo_.cqeException.sqeType,
4109 : dfxExtendInfo_.cqeException.errorCode);
4110 : }
4111 0 : return HCCL_E_INTERNAL;
4112 : }
4113 : }
4114 :
4115 : // 检测是否有停止/退出命令
4116 4 : KfcCommand cmd = KfcCommand::kNone;
4117 4 : CHK_RET(aicpuHdc_.GetOpExecCtrlCmd(kfcControlTransferH2D_, cmd));
4118 4 : if (cmd == KfcCommand::kStopLaunch && retryEnable_) {
4119 4 : HCCL_RUN_INFO(
4120 : "[OpRetry][AICPU]hccl aicpu stop wait finish, for recv stop launch cmd, identify[%s]", identifier_.c_str());
4121 4 : return HCCL_E_SUSPENDING;
4122 0 : } else if ((cmd == KfcCommand::NsStopLaunch) && (endStopLaunch == false)) {
4123 0 : needsResponseStopLaunch_ = true;
4124 0 : endStopLaunch = true;
4125 0 : HCCL_RUN_INFO("hccl aicpu stop wait finish, for recv stop launch cmd");
4126 0 : return HCCL_E_SUSPENDING;
4127 0 : } else if (cmd == KfcCommand::kDestroyComm) {
4128 0 : HCCL_ERROR("hccl aicpu stop wait finish, for recv destroy comm cmd");
4129 0 : return HCCL_E_INTERNAL;
4130 0 : } else if (cmd == KfcCommand::kExit) {
4131 0 : HCCL_ERROR("hccl aicpu stop wait finish, for recv exit cmd, identify[%s]", identifier_.c_str());
4132 0 : return HCCL_E_INTERNAL;
4133 : }
4134 0 : return HCCL_SUCCESS;
4135 : }
4136 :
4137 4 : HcclResult HcclCommAicpu::UpdateSuspendStatus(
4138 : const OpParam& param, HcclOpExecFSM& fsmState, KfcError& errorCode, uint32_t retryCnt)
4139 : {
4140 4 : if (needsResponseStopLaunch_ == true) {
4141 0 : HCCL_RUN_INFO("[NsRecovery][AICPU]hccl aicpu force stop in launch loop");
4142 0 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_STOP_LAUNCH;
4143 4 : } else if (retryEnable_) {
4144 4 : HCCL_RUN_INFO(
4145 : "[OpRetry][AICPU]hccl aicpu force stop for stop cmd or recoverable task exception, identify[%s]",
4146 : identifier_.c_str());
4147 4 : errorCode = IsTaskExceptionForHccs() ? KfcError::kSdma : errorCode;
4148 :
4149 4 : if ((param.opType == HcclCMDType::HCCL_CMD_BATCH_SEND_RECV)) {
4150 0 : HcclResult ret = GetBSRRetryOpId(param, bsrTargetOpId_);
4151 0 : HCCL_RETRY_CHK_RET_AND_TRANS_FSM(
4152 : ret, HCCL_ERROR("get batchsendrecv target op failed, ret:%u", ret), KfcError::kExec,
4153 : HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR);
4154 0 : uint32_t bsrRetryCnt = (bsrRetryOp_ == HCCL_SEND) ? bsrSendRetryCnt_ : bsrRecvRetryCnt_;
4155 0 : bsrTargetOpId_.isBsrTaskStart = true;
4156 0 : ret = aicpuHdc_.SetOpExecStatus(
4157 0 : kfcStatusTransferD2H_, bsrTargetOpId_, KfcStatus::kStoplaunch, errorCode, bsrRetryCnt);
4158 0 : HCCL_RETRY_CHK_RET_AND_TRANS_FSM(
4159 : ret, HCCL_ERROR("SetOpExecStatus failed, ret:%u", ret), KfcError::kExec,
4160 : HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR);
4161 : } else {
4162 4 : CHK_RET(UpdateOpExecStatus(fsmState, KfcStatus::kStoplaunch, errorCode, retryCnt));
4163 : }
4164 4 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_FSM_STOPPING;
4165 : } else {
4166 0 : HCCL_RUN_INFO(
4167 : "[HcclCommAicpu][UpdateSuspendStatus] aicpu force stop in launch loop; needsResponseStopLaunch_[%u] "
4168 : "retryEnable_[%u]",
4169 : needsResponseStopLaunch_, retryEnable_);
4170 0 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_STOP_LAUNCH;
4171 : }
4172 4 : return HCCL_SUCCESS;
4173 : }
4174 :
4175 0 : HcclResult HcclCommAicpu::TasktypeTransferD2H(const uint8_t sqeType, TaskType& taskType)
4176 : {
4177 0 : switch (sqeType) {
4178 0 : case RT_STARS_SQE_TYPE_PLACE_HOLDER:
4179 : case RT_STARS_SQE_TYPE_NOTIFY_WAIT:
4180 0 : taskType = TaskType::TASK_NOTIFY_WAIT;
4181 0 : break;
4182 0 : case RT_STARS_SQE_TYPE_SDMA:
4183 0 : taskType = TaskType::TASK_SDMA;
4184 0 : break;
4185 0 : case RT_STARS_SQE_TYPE_NOTIFY_RECORD:
4186 0 : taskType = TaskType::TASK_NOTIFY_RECORD;
4187 0 : break;
4188 0 : case RT_STARS_SQE_TYPE_WRITE_VALUE:
4189 0 : taskType = TaskType::TASK_NOTIFY_WAIT;
4190 0 : break;
4191 0 : default:
4192 0 : HCCL_ERROR("TasktypeTransferD2H sqeType[%d] error.", sqeType);
4193 0 : return HCCL_E_PARA;
4194 : }
4195 0 : return HCCL_SUCCESS;
4196 : }
4197 :
4198 0 : HcclResult HcclCommAicpu::GenTaskExceptionInfo(u8 sqeType, hccl::Stream& stream, u32 head)
4199 : {
4200 0 : HcclSqeContext* sqeContext = stream.GetSqeContextPtr();
4201 0 : SqeRingBuffer* sqeContextBuffer = &(sqeContext->buffer);
4202 0 : CHK_PTR_NULL(sqeContextBuffer);
4203 :
4204 0 : const AicpuOpInfo* opInfo = aicpuShareData_.GetAicpuOpInfo(sqeContextBuffer->rtsDfxInfo[head].opRingBufferIdx);
4205 0 : std::string opTag = opInfo == nullptr ? "unKnown" : opInfo->tagBuff;
4206 :
4207 : // 获取需要上报的关键信息
4208 0 : ErrorMessageReport emrInfo{};
4209 0 : SqeInfo sqeInfo;
4210 0 : SqeContextUtils::QuerySqeInfo(
4211 0 : sqeContextBuffer->rtsMirrorBuffer + head * HCCL_SQE_SIZE, sqeContextBuffer->rtsqSqeType[head],
4212 : sqeContextBuffer->addInfo[head], &sqeInfo);
4213 0 : emrInfo.remoteUserRank = sqeContextBuffer->rtsDfxInfo[head].remoteRank;
4214 0 : emrInfo.streamId = stream.id();
4215 0 : emrInfo.taskId = sqeInfo.taskId;
4216 0 : emrInfo.notifyId = sqeInfo.notifyId;
4217 0 : emrInfo.rankId = localUserRank_;
4218 0 : emrInfo.rankSize = topoInfo_.userRankSize;
4219 0 : emrInfo.algType = algType_;
4220 0 : emrInfo.opIndex = opInfo == nullptr ? 0 : opInfo->opIndex;
4221 0 : emrInfo.count = opInfo == nullptr ? 0 : opInfo->count;
4222 0 : emrInfo.dataType = opInfo == nullptr ? 0 : opInfo->dataType;
4223 0 : emrInfo.dstAddr = opInfo == nullptr ? 0 : opInfo->dstAddr;
4224 0 : emrInfo.srcAddr = opInfo == nullptr ? 0 : opInfo->srcAddr;
4225 0 : emrInfo.reduceType = opInfo == nullptr ? 255 : opInfo->reduceType; // 255 为 HcclReduceOp::HCCL_REDUCE_RESERVED
4226 0 : CHK_RET(TasktypeTransferD2H(sqeType, emrInfo.taskType));
4227 :
4228 0 : CHK_SAFETY_FUNC_RET(memcpy_s(emrInfo.tag, sizeof(emrInfo.tag), opTag.c_str(), opTag.size()));
4229 0 : CHK_SAFETY_FUNC_RET(memcpy_s(emrInfo.group, sizeof(emrInfo.group), identifier_.c_str(), identifier_.size()));
4230 0 : CHK_RET(aicpuHdc_.SetErrorMessage(kfcStatusTransferD2H_, emrInfo));
4231 0 : return HCCL_SUCCESS;
4232 0 : }
4233 :
4234 0 : HcclResult HcclCommAicpu::PrintTaskExceptionByTaskId(u8 sqeType, u16 taskId, hccl::Stream& stream, u32 tail)
4235 : {
4236 0 : HcclSqeContext* sqeContext = stream.GetSqeContextPtr();
4237 0 : CHK_PTR_NULL(sqeContext);
4238 0 : HCCL_ERROR("[HcclCommAicpu][PrintTaskExceptionByTaskId]streamId:%d tail:%u cqeType:%u", stream.id(), tail, sqeType);
4239 0 : SqeRingBuffer* sqeContextBuffer = &(sqeContext->buffer);
4240 0 : CHK_PTR_NULL(sqeContextBuffer);
4241 0 : uint8_t* sqeMirrorBufferAddr = sqeContextBuffer->rtsMirrorBuffer + (tail - 1) * HCCL_SQE_SIZE;
4242 0 : rtStarsSqeHeader_t* const sqeHeader = (rtStarsSqeHeader_t* const)sqeMirrorBufferAddr;
4243 :
4244 0 : s32 taskNum = sqeHeader->taskId - taskId;
4245 0 : HCCL_DEBUG("[HcclCommAicpu]tail sqe taskId[%u] cqe taskId[%u] cqe type[%u]", sqeHeader->taskId, taskId, sqeType);
4246 0 : s32 sqeIdx = tail - taskNum - 1;
4247 0 : u32 sqHead = (sqeIdx + HCCL_SQE_MAX_CNT) % HCCL_SQE_MAX_CNT;
4248 0 : uint8_t type = 0;
4249 0 : uint16_t taskIdTmp = 0;
4250 0 : uint32_t remoteRank = 0;
4251 0 : HCCL_ERROR(
4252 : "[TaskException][AICPU]base information is streamId:%d, sqid:%d, head:%u, tail:%u, %s", stream.id(),
4253 : stream.sqId(), sqHead, tail,
4254 : GetTaskExceptionTaskInfo(sqHead, sqeContextBuffer, type, taskIdTmp, remoteRank).c_str());
4255 0 : PrintTaskExceptionTaskQue(sqHead, sqeContextBuffer);
4256 0 : return HCCL_SUCCESS;
4257 : }
4258 :
4259 0 : std::string HcclCommAicpu::GetTaskExceptionOpInfo(u32 idx, SqeRingBuffer* sqeContextBuffer)
4260 : {
4261 0 : const AicpuOpInfo* opInfo = aicpuShareData_.GetAicpuOpInfo(sqeContextBuffer->rtsDfxInfo[idx].opRingBufferIdx);
4262 0 : CHK_PRT_RET(opInfo == nullptr, HCCL_ERROR("%s fail, opInfo is nullptr", __func__), "unKnown");
4263 :
4264 0 : std::stringstream ss;
4265 0 : ss << "tag:" << opInfo->tagBuff << ", ";
4266 0 : ss << "group:" << identifier_ << ", ";
4267 0 : ss << "isCustom:" << opInfo->isCustom << ", ";
4268 0 : ss << "opLaunchIdx:" << opInfo->opIndex << ", ";
4269 0 : ss << "opExecIdx:" << opInfo->opExecIndex << ", ";
4270 0 : ss << "count:" << opInfo->count << ", ";
4271 0 : ss << "dataType:" << static_cast<u16>(opInfo->dataType) << ", ";
4272 0 : ss << "opType:" << static_cast<u16>(opInfo->opType) << ", ";
4273 0 : ss << "rootId:" << opInfo->rootId << ", ";
4274 0 : ss << "dstAddr:0x" << std::hex << opInfo->dstAddr << ", ";
4275 0 : ss << "srcAddr:0x" << std::hex << opInfo->srcAddr << ".";
4276 0 : return ss.str();
4277 0 : }
4278 :
4279 0 : std::string HcclCommAicpu::GetTaskExceptionTaskInfo(
4280 : u32 sqHead, SqeRingBuffer* sqeContextBuffer, uint8_t& type, uint16_t& taskId, uint32_t& remoteRank)
4281 : {
4282 0 : SqeInfo sqeInfo;
4283 0 : SqeContextUtils::QuerySqeInfo(
4284 0 : sqeContextBuffer->rtsMirrorBuffer + sqHead * HCCL_SQE_SIZE, sqeContextBuffer->rtsqSqeType[sqHead],
4285 : sqeContextBuffer->addInfo[sqHead], &sqeInfo);
4286 0 : type = sqeInfo.type;
4287 0 : taskId = sqeInfo.taskId;
4288 0 : remoteRank = sqeContextBuffer->rtsDfxInfo[sqHead].remoteRank;
4289 0 : std::stringstream ss;
4290 0 : ss << "type:" << SqeContextUtils::RtsqTaskTypeToStr(sqeInfo.type) << ", ";
4291 0 : ss << "localRank:" << localUserRank_ << ", ";
4292 0 : ss << "remoteRank:" << remoteRank << ", ";
4293 0 : ss << "taskId:" << sqeInfo.taskId << ", ";
4294 0 : ss << "notifyId:" << sqeInfo.notifyId << ", ";
4295 0 : ss << "length:" << sqeInfo.length << ", ";
4296 0 : ss << "addr1High:0x" << std::hex << sqeInfo.addr1High << ", ";
4297 0 : ss << "addr1Low:0x" << std::hex << sqeInfo.addr1Low << ", ";
4298 0 : ss << "addr2High:0x" << std::hex << sqeInfo.addr2High << ", ";
4299 0 : ss << "addr2Low:0x" << std::hex << sqeInfo.addr2Low << ".";
4300 0 : return ss.str();
4301 0 : }
4302 :
4303 : // 获取指定索引的算子信息
4304 0 : const AicpuOpInfo* HcclCommAicpu::GetOpInfoFromSqIdx(u32 sqIdx, SqeRingBuffer* sqeContextBuffer)
4305 : {
4306 0 : return aicpuShareData_.GetAicpuOpInfo(sqeContextBuffer->rtsDfxInfo[sqIdx].opRingBufferIdx);
4307 : }
4308 :
4309 : // 打印算子数据信息
4310 0 : void HcclCommAicpu::PrintOpDataInfo(u32 sqIdx, SqeRingBuffer* sqeContextBuffer, bool isMonitor)
4311 : {
4312 0 : std::string opInfo = GetTaskExceptionOpInfo(sqIdx, sqeContextBuffer);
4313 0 : if (isMonitor) {
4314 0 : HCCL_RUN_INFO("[StreamTaskMonitor]opData information is %s", opInfo.c_str());
4315 : } else {
4316 0 : HCCL_ERROR("[TaskException]opData information is %s", opInfo.c_str());
4317 : }
4318 0 : }
4319 :
4320 : // 打印task序列行
4321 0 : void HcclCommAicpu::PrintTaskLine(bool isMonitor, u32 lineNum, u32 totalPrinted, const std::string& taskLine) const
4322 : {
4323 0 : if (isMonitor) {
4324 0 : HCCL_RUN_INFO("[StreamTaskMonitor]task sequence[%u/%u] is %s", lineNum, totalPrinted, taskLine.c_str());
4325 : } else {
4326 0 : HCCL_ERROR("[TaskException]task sequence[%u/%u] is %s", lineNum, totalPrinted, taskLine.c_str());
4327 : }
4328 0 : }
4329 :
4330 : // 更新算子上下文
4331 0 : void HcclCommAicpu::UpdateOpContext(
4332 : u32& opIndex, std::string& opTag, u32& lineCount, std::vector<std::string>& currentOpTasks, u32 newOpIndex,
4333 : const std::string& newOpTag) const
4334 : {
4335 0 : opIndex = newOpIndex;
4336 0 : opTag = newOpTag;
4337 0 : lineCount = 0;
4338 0 : currentOpTasks.clear();
4339 0 : currentOpTasks.push_back("OP(" + std::to_string(opIndex) + ")");
4340 0 : }
4341 :
4342 : // 准备下一行数据
4343 0 : void HcclCommAicpu::PrepareNextLine(u32 opIndex, u32& lineCount, std::vector<std::string>& currentOpTasks) const
4344 : {
4345 0 : lineCount++;
4346 0 : currentOpTasks.clear();
4347 0 : currentOpTasks.push_back("OP(" + std::to_string(opIndex) + ") continued");
4348 0 : }
4349 :
4350 : // 拼接task列表为字符串
4351 0 : std::string HcclCommAicpu::ConcatTaskLine(const std::vector<std::string>& tasks) const
4352 : {
4353 0 : std::string taskLine;
4354 0 : for (const auto& task : tasks) {
4355 0 : taskLine += task;
4356 : }
4357 0 : return taskLine;
4358 0 : }
4359 :
4360 : // 打印剩余未满行的tasks
4361 0 : void HcclCommAicpu::PrintRemainingTasks(
4362 : bool isMonitor, u32 lineCount, u32 printedCount, const std::vector<std::string>& currentOpTasks) const
4363 : {
4364 : // size > 1 表示除了OP头还有task
4365 0 : if (currentOpTasks.size() > 1) {
4366 0 : std::string taskLine = ConcatTaskLine(currentOpTasks);
4367 0 : PrintTaskLine(isMonitor, lineCount + 1, printedCount, taskLine);
4368 0 : }
4369 0 : }
4370 :
4371 0 : void HcclCommAicpu::PrintTaskExceptionTaskQue(u32 sqIdx, SqeRingBuffer* sqeContextBuffer, bool isMonitor)
4372 : {
4373 0 : const u32 sqeNum = 200;
4374 0 : const u32 maxTasksPerLine = 50;
4375 :
4376 : // 获取初始算子信息
4377 0 : const AicpuOpInfo* lastOpInfo = GetOpInfoFromSqIdx(sqIdx, sqeContextBuffer);
4378 0 : CHK_PRT_RET(lastOpInfo == nullptr, HCCL_ERROR("%s fail, opInfo is nullptr", __func__), );
4379 :
4380 : // 初始化状态
4381 0 : u32 opIndex = lastOpInfo->opIndex;
4382 0 : std::string opTag = lastOpInfo->tagBuff;
4383 0 : u32 lineCount = 0;
4384 0 : u32 printedCount = 0;
4385 0 : u32 extraSlotPerLine = 2;
4386 0 : std::vector<std::string> currentOpTasks;
4387 0 : currentOpTasks.push_back("OP(" + std::to_string(opIndex) + ")");
4388 :
4389 : // 主循环：遍历并打印
4390 0 : for (u32 i = 0; i < sqeNum; i++) {
4391 : // 计算新索引并获取算子信息
4392 0 : u32 newSqIdx = (sqIdx - i + HCCL_SQE_MAX_CNT) % HCCL_SQE_MAX_CNT;
4393 0 : const AicpuOpInfo* newOpInfo = GetOpInfoFromSqIdx(newSqIdx, sqeContextBuffer);
4394 0 : CHK_PRT_RET(newOpInfo == nullptr, HCCL_ERROR("%s fail, opInfo is nullptr", __func__), );
4395 :
4396 0 : u32 sizeAfterAdd = currentOpTasks.size() + 1;
4397 0 : if (sizeAfterAdd >= (maxTasksPerLine + extraSlotPerLine)
4398 0 : || (i > 0 && (newOpInfo->opIndex != opIndex || newOpInfo->tagBuff != opTag))) {
4399 : // 打印算子信息（仅第一次）
4400 0 : if (lineCount == 0) {
4401 0 : PrintOpDataInfo(sqIdx, sqeContextBuffer, isMonitor);
4402 : }
4403 :
4404 : // 拼接并打印task行
4405 0 : std::string taskLine = ConcatTaskLine(currentOpTasks);
4406 0 : PrintTaskLine(isMonitor, lineCount + 1, printedCount, taskLine);
4407 :
4408 : // 更新状态
4409 0 : if (newOpInfo->opIndex != opIndex || newOpInfo->tagBuff != opTag) {
4410 0 : UpdateOpContext(opIndex, opTag, lineCount, currentOpTasks, newOpInfo->opIndex, newOpInfo->tagBuff);
4411 : } else {
4412 0 : PrepareNextLine(opIndex, lineCount, currentOpTasks);
4413 : }
4414 0 : }
4415 : // 添加task到序列
4416 0 : currentOpTasks.push_back("," + GetTaskBriefsInfo(newSqIdx, sqeContextBuffer));
4417 0 : printedCount++;
4418 : }
4419 : // 打印剩余tasks
4420 0 : PrintRemainingTasks(isMonitor, lineCount, printedCount, currentOpTasks);
4421 0 : }
4422 :
4423 0 : std::string HcclCommAicpu::GetTaskBriefsInfo(u32 idx, SqeRingBuffer* sqeContextBuffer)
4424 : {
4425 0 : uint8_t* sqeMirrorBufferAddr = sqeContextBuffer->rtsMirrorBuffer + idx * HCCL_SQE_SIZE;
4426 0 : rtStarsSqeHeader_t* const sqeHeader = (rtStarsSqeHeader_t* const)sqeMirrorBufferAddr;
4427 0 : uint8_t sqeType = sqeHeader->type;
4428 :
4429 0 : SqeInfo sqeInfo;
4430 0 : SqeContextUtils::QuerySqeInfo(
4431 0 : sqeContextBuffer->rtsMirrorBuffer + idx * HCCL_SQE_SIZE, sqeContextBuffer->rtsqSqeType[idx],
4432 : sqeContextBuffer->addInfo[idx], &sqeInfo);
4433 0 : uint8_t subType = sqeInfo.subType;
4434 :
4435 0 : std::stringstream ss;
4436 0 : std::string taskName = "UN";
4437 0 : switch (sqeType) {
4438 0 : case RT_STARS_SQE_TYPE_NOTIFY_RECORD:
4439 0 : taskName = "NR"; // Notify Record
4440 0 : break;
4441 0 : case RT_STARS_SQE_TYPE_WRITE_VALUE:
4442 0 : if (subType == RT_STARS_WRITE_VALUE_SUB_TYPE_NOTIFY_RECORD_IPC_NO_PCIE) {
4443 0 : taskName = "NR";
4444 0 : } else if (subType == RT_STARS_WRITE_VALUE_SUB_TYPE_EVENT_RESET) {
4445 0 : taskName = "NW"; // Notify Wait
4446 0 : } else if (subType == RT_STARS_WRITE_VALUE_SUB_TYPE_RDMA_DB_SEND) {
4447 0 : taskName = "RS"; // Rdma Send
4448 : }
4449 0 : break;
4450 0 : case RT_STARS_SQE_TYPE_NOTIFY_WAIT:
4451 0 : taskName = "NW";
4452 0 : break;
4453 0 : case RT_STARS_SQE_TYPE_EVENT_WAIT:
4454 0 : taskName = "NW";
4455 0 : break;
4456 0 : case RT_STARS_SQE_TYPE_SDMA:
4457 0 : taskName = "SD"; // SDMA
4458 0 : break;
4459 0 : case RT_STARS_SQE_TYPE_COND:
4460 0 : taskName = "CO";
4461 0 : break;
4462 0 : case RT_STARS_SQE_TYPE_PLACE_HOLDER:
4463 0 : taskName = "PH";
4464 0 : break;
4465 0 : default:
4466 0 : taskName = std::to_string(sqeType);
4467 0 : break;
4468 : }
4469 :
4470 0 : ss << taskName << "(";
4471 0 : if (sqeContextBuffer->rtsDfxInfo[idx].remoteRank != INVALID_VALUE_RANKID) {
4472 0 : ss << sqeContextBuffer->rtsDfxInfo[idx].remoteRank;
4473 : } else {
4474 0 : ss << "/";
4475 : }
4476 0 : ss << ",";
4477 0 : if (sqeContextBuffer->rtsDfxInfo[idx].notifyId != INVALID_VALUE_RANKID) {
4478 0 : ss << sqeContextBuffer->rtsDfxInfo[idx].notifyId;
4479 : } else {
4480 0 : ss << "/";
4481 : }
4482 0 : ss << ")";
4483 0 : return ss.str();
4484 0 : }
4485 :
4486 1 : void HcclCommAicpu::RecordReportStatus(dfx::ReportStatus status)
4487 : {
4488 1 : std::unique_lock<std::mutex> lock(reportQueueMutex_);
4489 1 : while (reportStatusQueue_.size() >= MAX_REPORT_STATUS) {
4490 0 : HCCL_WARNING(
4491 : "[HcclCommAicpu][RecordReportStatus] retry status queue reach the limit[%u], "
4492 : "the front status[%u] is dropped.",
4493 : MAX_REPORT_STATUS, reportStatusQueue_.front());
4494 0 : reportStatusQueue_.pop();
4495 : }
4496 1 : reportStatusQueue_.push(status);
4497 1 : HCCL_INFO("[HcclCommAicpu][RecordReportStatus]push[%u], retry queue size()[%u]", status, reportStatusQueue_.size());
4498 1 : }
4499 :
4500 0 : void HcclCommAicpu::GetReportStatusQueue(std::queue<dfx::ReportStatus>& reportStatusQue)
4501 : {
4502 0 : std::unique_lock<std::mutex> lock(reportQueueMutex_);
4503 0 : std::swap(reportStatusQueue_, reportStatusQue);
4504 0 : }
4505 :
4506 0 : void HcclCommAicpu::SetStreamCqeExceptionStatus(const Stream& stream, CqeExceptionStatus cqeStatus)
4507 : {
4508 0 : HCCL_RUN_INFO("SetStreamCqeExceptionStatus: stream sq id %u, cqe exception %u", stream.sqId(), cqeStatus);
4509 0 : auto iter = streamCqeExceptionStatus_.find(stream.sqId());
4510 0 : if (iter != streamCqeExceptionStatus_.end()) {
4511 0 : iter->second = cqeStatus;
4512 : } else {
4513 0 : streamCqeExceptionStatus_.insert({stream.sqId(), cqeStatus});
4514 : }
4515 0 : return;
4516 : }
4517 :
4518 0 : CqeExceptionStatus HcclCommAicpu::GetStreamCqeExceptionStatus(const Stream& stream)
4519 : {
4520 0 : auto iter = streamCqeExceptionStatus_.find(stream.sqId());
4521 0 : if (iter != streamCqeExceptionStatus_.end()) {
4522 0 : return iter->second;
4523 : } else {
4524 0 : return CqeExceptionStatus::kNone;
4525 : }
4526 : }
4527 :
4528 0 : void HcclCommAicpu::ResetStreamCqeExceptionStatus(const Stream& stream)
4529 : {
4530 0 : auto iter = streamCqeExceptionStatus_.find(stream.sqId());
4531 0 : if (iter != streamCqeExceptionStatus_.end()) {
4532 0 : iter->second = CqeExceptionStatus::kNone;
4533 : }
4534 0 : HCCL_INFO("ResetStreamCqeExceptionStatus: stream sq id %u", stream.sqId());
4535 0 : return;
4536 : }
4537 :
4538 0 : void HcclCommAicpu::SetBSRSendOpExecException()
4539 : {
4540 0 : HCCL_INFO("set send stream exec exception");
4541 0 : bsrSendOpExecException_ = true;
4542 0 : return;
4543 : }
4544 :
4545 0 : void HcclCommAicpu::SetBSRRecvOpExecException()
4546 : {
4547 0 : HCCL_INFO("set recv stream exec exception");
4548 0 : bsrRecvOpExecException_ = true;
4549 0 : return;
4550 : }
4551 :
4552 0 : bool HcclCommAicpu::GetBSRSendOpExecException()
4553 : {
4554 0 : bool ret = (GetStreamCqeExceptionStatus(bsrSendStream_) == CqeExceptionStatus::kSdmaErr) || bsrSendOpExecException_;
4555 0 : HCCL_INFO(
4556 : "GetBSRSendOpExecException: stream %u cqe status %u, send exec status %u", bsrSendStream_.sqId(),
4557 : GetStreamCqeExceptionStatus(bsrSendStream_), bsrSendOpExecException_);
4558 0 : return ret;
4559 : }
4560 :
4561 0 : bool HcclCommAicpu::GetBSRRecvOpExecException()
4562 : {
4563 0 : HCCL_INFO(
4564 : "GetBSRRecvOpExecException: stream %u cqe status %u, recv exec status %u", bsrRecvStream_.sqId(),
4565 : GetStreamCqeExceptionStatus(bsrRecvStream_), bsrRecvOpExecException_);
4566 0 : return (GetStreamCqeExceptionStatus(bsrRecvStream_) == CqeExceptionStatus::kSdmaErr) || bsrRecvOpExecException_;
4567 : }
4568 :
4569 0 : HcclResult HcclCommAicpu::CleanStream(Stream& stream)
4570 : {
4571 0 : CHK_RET(stream.ClearLocalBuff());
4572 0 : CHK_RET(UpdateSqStatus(stream));
4573 0 : ResetStreamCqeExceptionStatus(stream);
4574 0 : HCCL_INFO("CleanStream %u success.", stream.sqId());
4575 0 : return HCCL_SUCCESS;
4576 : }
4577 :
4578 0 : HcclResult HcclCommAicpu::ClearStreamCqeException(Stream& stream)
4579 : {
4580 0 : HandleCqeException(stream, true);
4581 0 : return HCCL_SUCCESS;
4582 : }
4583 :
4584 0 : HcclResult HcclCommAicpu::ResetBSRSendOpExecException()
4585 : {
4586 0 : bsrSendOpExecException_ = false;
4587 0 : CHK_RET(CleanStream(bsrSendStream_));
4588 0 : CHK_RET(ClearStreamCqeException(bsrSendStream_));
4589 0 : HCCL_INFO("ResetBSRSendOpExecException success.");
4590 0 : return HCCL_SUCCESS;
4591 : }
4592 :
4593 0 : HcclResult HcclCommAicpu::ResetBSRRecvOpExecException()
4594 : {
4595 0 : bsrRecvOpExecException_ = false;
4596 0 : CHK_RET(CleanStream(bsrRecvStream_));
4597 0 : CHK_RET(ClearStreamCqeException(bsrRecvStream_));
4598 0 : HCCL_INFO("ResetBSRRecvOpExecException success.");
4599 0 : return HCCL_SUCCESS;
4600 : }
4601 :
4602 0 : HcclResult HcclCommAicpu::ResetBSRException()
4603 : {
4604 0 : if (bsrRetryOp_ == HCCL_SEND) {
4605 0 : CHK_RET(ResetBSRSendOpExecException());
4606 0 : HCCL_INFO("reset batchsendrecv exception success, tag:%s, index:%u", bsrSendOpId_.tag, bsrSendOpId_.index);
4607 0 : } else if (bsrRetryOp_ == HCCL_RECV) {
4608 0 : CHK_RET(ResetBSRRecvOpExecException());
4609 0 : HCCL_INFO("reset batchsendrecv exception success, tag:%s, index:%u", bsrRecvOpId_.tag, bsrRecvOpId_.index);
4610 : } else {
4611 0 : HCCL_INFO("reset batchsendrecv exception success, tag:%s", bsrTargetOpId_.tag);
4612 : }
4613 0 : return HCCL_SUCCESS;
4614 : }
4615 :
4616 0 : void HcclCommAicpu::UpdateBSRRetryCnt()
4617 : {
4618 0 : if (bsrRetryOp_ == HCCL_SEND) {
4619 0 : bsrSendRetryCnt_++;
4620 : } else {
4621 0 : bsrRecvRetryCnt_++;
4622 : }
4623 0 : HCCL_INFO("UpdateBSRRetryCnt, SendCnt[%u], RecvCnt[%u]", bsrRecvRetryCnt_, bsrRecvRetryCnt_);
4624 0 : return;
4625 : }
4626 :
4627 0 : void HcclCommAicpu::ResetBSRRetryCnt()
4628 : {
4629 0 : bsrSendRetryCnt_ = 0;
4630 0 : bsrRecvRetryCnt_ = 0;
4631 0 : return;
4632 : }
4633 :
4634 0 : void HcclCommAicpu::InitSendRecvOpId(const OpParam& param, HcclOpIdentifier& opId)
4635 : {
4636 : // send算子入参中只有dst对端rank号，而recv算子入参中只有src源端rank号
4637 0 : if (param.opType == HcclCMDType::HCCL_CMD_SEND) {
4638 0 : opId.detRank = param.dstRank;
4639 0 : opId.srcRank = topoInfo_.userRank;
4640 : } else {
4641 0 : opId.srcRank = param.srcRank;
4642 0 : opId.detRank = topoInfo_.userRank;
4643 : }
4644 0 : opId.isSendRecv = true;
4645 0 : HCCL_DEBUG(
4646 : "[HcclCommAicpu][InitSendRecvOpId]src=[%u] dst=[%u] isSendRecv=[%u]", opId.srcRank, opId.detRank,
4647 : opId.isSendRecv);
4648 0 : return;
4649 : }
4650 :
4651 0 : u32 HcclCommAicpu::HcclUpdateBatchSendRecvOpIndex(std::map<u32, u32>& bsrIndexMap, u32 peerRank)
4652 : {
4653 0 : u32 ret = 0;
4654 0 : auto opIndexMapIter = bsrIndexMap.find(peerRank);
4655 0 : if (opIndexMapIter != bsrIndexMap.end()) {
4656 0 : (opIndexMapIter->second)++;
4657 0 : ret = opIndexMapIter->second;
4658 : } else {
4659 0 : bsrIndexMap.insert({peerRank, 1});
4660 0 : ret = 1;
4661 : }
4662 0 : return ret;
4663 : }
4664 :
4665 0 : u32 HcclCommAicpu::HcclUpdateBatchSendRecvOpIndex(HcclSendRecvType opType, u32 srcRank, u32 dstRank)
4666 : {
4667 0 : u32 peerRank = (opType == HcclSendRecvType::HCCL_SEND) ? dstRank : srcRank;
4668 0 : auto& opIndexMap = (opType == HcclSendRecvType::HCCL_SEND) ? bsrSendIndexMap_ : bsrRecvIndexMap_;
4669 :
4670 0 : return HcclUpdateBatchSendRecvOpIndex(opIndexMap, peerRank);
4671 : }
4672 : HcclResult
4673 0 : HcclCommAicpu::GetBsrTransportQpn(const HcclSendRecvItem* sendrecvPair, AlgResourceResponse& algResource, u32& qpn)
4674 : {
4675 0 : CHK_PTR_NULL(sendrecvPair);
4676 0 : LINK targetLink;
4677 0 : u32 commIndex = 0;
4678 0 : u32 remoteRank = sendrecvPair->remoteRank;
4679 0 : u32 localRank = topoInfo_.userRank;
4680 0 : HcclSendRecvType sendRecvType = sendrecvPair->sendRecvType;
4681 0 : HCCL_DEBUG(
4682 : "[GetBsrTransportQpn] bsrOptype =[%d], localRank=[%u] remoteRank=[%u]", sendRecvType, localRank, remoteRank);
4683 :
4684 0 : if ((sendRecvType == HcclSendRecvType::HCCL_SEND && remoteRank < localRank)
4685 0 : || (sendRecvType == HcclSendRecvType::HCCL_RECV && remoteRank > localRank)) {
4686 0 : commIndex = COMM_INDEX_0;
4687 : } else {
4688 0 : commIndex = COMM_INDEX_1;
4689 : }
4690 0 : CHK_PRT_RET(
4691 : commIndex >= algResource.opTransportResponse[COMM_COMBINE_ORDER].size(),
4692 : HCCL_ERROR(
4693 : "[GetBsrTransportQpn] batchsendrecv op commIndex[%u] is larger than "
4694 : "opTransportResponse size[%zu]",
4695 : commIndex, algResource.opTransportResponse[COMM_COMBINE_ORDER].size()),
4696 : HCCL_E_PARA);
4697 : SingleSubCommTransport& commCombined
4698 0 : = static_cast<SingleSubCommTransport&>(algResource.opTransportResponse[COMM_COMBINE_ORDER][commIndex]);
4699 :
4700 0 : CHK_PRT_RET(
4701 : sendrecvPair->remoteRank >= commCombined.userRank2subCommRank.size(),
4702 : HCCL_ERROR(
4703 : "[GetBsrTransportQpn]batchsendrecv op remoteUserRank[%u] is larger than "
4704 : "userRank2subCommRank map size[%zu]",
4705 : sendrecvPair->remoteRank, commCombined.userRank2subCommRank.size()),
4706 : HCCL_E_PARA);
4707 :
4708 0 : u32 rank = commCombined.userRank2subCommRank[sendrecvPair->remoteRank];
4709 0 : CHK_PRT_RET(
4710 : rank >= commCombined.links.size(),
4711 : HCCL_ERROR(
4712 : "[GetBsrTransportQpn] batchsendrecv op remoteUserRank[%u], get rank[%u],"
4713 : "the size of combinedComm links is [%zu]",
4714 : sendrecvPair->remoteRank, rank, commCombined.links.size()),
4715 : HCCL_E_PARA);
4716 0 : targetLink = commCombined.links[rank];
4717 :
4718 0 : CHK_SMART_PTR_NULL(targetLink);
4719 0 : if (targetLink->GetLinkType() == LinkType::LINK_ROCE) {
4720 0 : CHK_RET(targetLink->GetTransportId(qpn));
4721 : }
4722 0 : HCCL_DEBUG(
4723 : "[HcclCommAicpu][GetBsrTransportQpn] localrank=[%u] remoteuserRank=[%u] remoteRank=[%u],sendrecvType=[%d] qpn "
4724 : "=[%u], comindex [%u]",
4725 : topoInfo_.userRank, sendrecvPair->remoteRank, remoteRank, sendrecvPair->sendRecvType, qpn, commIndex);
4726 0 : return HCCL_SUCCESS;
4727 0 : }
4728 0 : HcclResult HcclCommAicpu::InitBatchSendRecvOpId(
4729 : const OpParam& param, const HcclSendRecvItem* sendrecvPair, HcclOpIdentifier& opId, u32 streamId,
4730 : AlgResourceResponse& algResource)
4731 : {
4732 0 : CHK_PTR_NULL(sendrecvPair);
4733 0 : if (sendrecvPair->sendRecvType == HcclSendRecvType::HCCL_RECV) {
4734 0 : opId.srcRank = sendrecvPair->remoteRank;
4735 0 : opId.detRank = topoInfo_.userRank;
4736 : } else {
4737 0 : opId.srcRank = topoInfo_.userRank;
4738 0 : opId.detRank = sendrecvPair->remoteRank;
4739 : }
4740 0 : opId.index = HcclUpdateBatchSendRecvOpIndex(sendrecvPair->sendRecvType, opId.srcRank, opId.detRank);
4741 0 : opId.isSendRecv = true;
4742 0 : opId.opType = HcclCMDType::HCCL_CMD_BATCH_SEND_RECV;
4743 0 : std::string sendrecvTag = param.tag + "_BSR_" + std::to_string(opId.srcRank) + "_" + std::to_string(opId.detRank);
4744 0 : CHK_SAFETY_FUNC_RET(memcpy_s(opId.tag, sizeof(opId.tag), sendrecvTag.c_str(), sendrecvTag.size()));
4745 0 : std::string sendrecvNewTag = param.tag + "_device";
4746 0 : CHK_SAFETY_FUNC_RET(memcpy_s(opId.newTag, sizeof(opId.newTag), sendrecvNewTag.c_str(), sendrecvNewTag.size()));
4747 :
4748 0 : u32 qpn = 0;
4749 0 : if ((opId.srcRank != opId.detRank) && (!param.BatchSendRecvDataDes.isDirectRemoteRank[sendrecvPair->remoteRank])) {
4750 0 : CHK_RET(GetBsrTransportQpn(sendrecvPair, algResource, qpn));
4751 : }
4752 :
4753 0 : auto& bsrinfo = opId.bsrInfo[sendrecvPair->sendRecvType];
4754 0 : bsrinfo.detRank = opId.detRank;
4755 0 : bsrinfo.srcRank = opId.srcRank;
4756 0 : bsrinfo.index = opId.index;
4757 0 : bsrinfo.streamId = streamId;
4758 0 : bsrinfo.tpQpn = qpn;
4759 0 : CHK_SAFETY_FUNC_RET(memcpy_s(bsrinfo.bsrTag, sizeof(bsrinfo.bsrTag), sendrecvTag.c_str(), sendrecvTag.size()));
4760 :
4761 0 : HCCL_INFO(
4762 : "[HcclCommAicpu][InitBatchSendRecvOpId] tag=[%s] index=[%u] src=[%u] det=[%u] qpn =[%u]", opId.tag, opId.index,
4763 : opId.srcRank, opId.detRank, qpn);
4764 0 : return HCCL_SUCCESS;
4765 0 : }
4766 :
4767 0 : HcclResult HcclCommAicpu::InitBatchSendRecvOpId(const OpParam& param, AlgResourceResponse& algResource)
4768 : {
4769 0 : CHK_SAFETY_FUNC_RET(
4770 : memset_s(reinterpret_cast<void*>(&bsrSendOpId_), sizeof(bsrSendOpId_), 0, sizeof(bsrSendOpId_)));
4771 0 : CHK_SAFETY_FUNC_RET(
4772 : memset_s(reinterpret_cast<void*>(&bsrRecvOpId_), sizeof(bsrRecvOpId_), 0, sizeof(bsrRecvOpId_)));
4773 :
4774 0 : CHK_PRT_RET(
4775 : (algResource.slaveStreams.size() < BSR_RETRY_STREAM_NUM),
4776 : HCCL_ERROR("in batchsendrecv op, slave stream is not enough."), HCCL_E_INTERNAL);
4777 0 : bsrSendOpId_.streamId = algResource.slaveStreams[BSR_RETRY_SEND_STREAM_INDEX].id();
4778 0 : bsrRecvOpId_.streamId = algResource.slaveStreams[BSR_RETRY_RECV_STREAM_INDEX].id();
4779 :
4780 0 : u32 iter = param.BatchSendRecvDataDes.curIterNum;
4781 0 : std::vector<std::vector<HcclSendRecvItem*>>& pairs = bsrSendRecvPairs_;
4782 0 : CHK_PRT_RET(
4783 : (pairs.size() <= iter),
4784 : HCCL_ERROR("batchsendrecv sendrecv pairs size[%u] less than or equal to curiter[%u]", pairs.size(), iter),
4785 : HCCL_E_INTERNAL);
4786 :
4787 0 : for (auto& pair : pairs[iter]) {
4788 0 : HcclOpIdentifier& opId = (pair->sendRecvType == HcclSendRecvType::HCCL_SEND) ? bsrSendOpId_ : bsrRecvOpId_;
4789 0 : auto streamId
4790 0 : = (pair->sendRecvType == HcclSendRecvType::HCCL_SEND) ? bsrSendOpId_.streamId : bsrRecvOpId_.streamId;
4791 0 : CHK_RET(InitBatchSendRecvOpId(param, pair, opId, streamId, algResource));
4792 : }
4793 :
4794 : // 补全batchsendrecv中sendrecv的bsrInfo, 两边都发生故障的时候要用
4795 0 : bsrSendOpId_.bsrInfo[HCCL_RECV].index = bsrRecvOpId_.index;
4796 0 : bsrSendOpId_.bsrInfo[HCCL_RECV].streamId = bsrRecvOpId_.streamId;
4797 0 : bsrSendOpId_.bsrInfo[HCCL_RECV].srcRank = bsrRecvOpId_.srcRank;
4798 0 : bsrSendOpId_.bsrInfo[HCCL_RECV].detRank = bsrRecvOpId_.detRank;
4799 0 : bsrSendOpId_.bsrInfo[HCCL_RECV].tpQpn = bsrRecvOpId_.bsrInfo[HCCL_RECV].tpQpn;
4800 0 : CHK_SAFETY_FUNC_RET(memcpy_s(
4801 : bsrSendOpId_.bsrInfo[HCCL_RECV].bsrTag, sizeof(bsrSendOpId_.bsrInfo[HCCL_RECV].bsrTag), bsrRecvOpId_.tag,
4802 : sizeof(bsrRecvOpId_.tag)));
4803 :
4804 0 : bsrRecvOpId_.bsrInfo[HCCL_SEND].index = bsrSendOpId_.index;
4805 0 : bsrRecvOpId_.bsrInfo[HCCL_SEND].streamId = bsrSendOpId_.streamId;
4806 0 : bsrRecvOpId_.bsrInfo[HCCL_SEND].srcRank = bsrSendOpId_.srcRank;
4807 0 : bsrRecvOpId_.bsrInfo[HCCL_SEND].detRank = bsrSendOpId_.detRank;
4808 0 : bsrRecvOpId_.bsrInfo[HCCL_SEND].tpQpn = bsrSendOpId_.bsrInfo[HCCL_SEND].tpQpn;
4809 0 : CHK_SAFETY_FUNC_RET(memcpy_s(
4810 : bsrRecvOpId_.bsrInfo[HCCL_SEND].bsrTag, sizeof(bsrRecvOpId_.bsrInfo[HCCL_SEND].bsrTag), bsrSendOpId_.tag,
4811 : sizeof(bsrSendOpId_.tag)));
4812 0 : return HCCL_SUCCESS;
4813 : }
4814 :
4815 0 : HcclResult HcclCommAicpu::QueryBatchSendRecvPairBeginPos()
4816 : {
4817 : // 前面已经生成 batchsendrecv 的 send & recv opid 时已经校验过slave stream num数，此处不再重复校验
4818 0 : CHK_RET(QuerySqStatusByType(devId_, bsrSendStream_.sqId(), DRV_SQCQ_PROP_SQ_TAIL, bsrSendOpBeginSqePos_));
4819 0 : CHK_RET(QuerySqStatusByType(devId_, bsrRecvStream_.sqId(), DRV_SQCQ_PROP_SQ_TAIL, bsrRecvOpBeginSqePos_));
4820 :
4821 0 : HCCL_INFO(
4822 : "QueryBatchSendRecvPairBeginPos send sqePos[%u] recv sqePos[%u]", bsrSendOpBeginSqePos_, bsrRecvOpBeginSqePos_);
4823 0 : return HCCL_SUCCESS;
4824 : }
4825 :
4826 0 : HcclResult HcclCommAicpu::QueryBatchSendRecvPairEndPos()
4827 : {
4828 : // 前面已经生成 batchsendrecv 的 send & recv opid 时已经校验过slave stream num数，此处不再重复校验
4829 0 : CHK_RET(QuerySqStatusByType(devId_, bsrSendStream_.sqId(), DRV_SQCQ_PROP_SQ_TAIL, bsrSendOpEndSqePos_));
4830 0 : CHK_RET(QuerySqStatusByType(devId_, bsrRecvStream_.sqId(), DRV_SQCQ_PROP_SQ_TAIL, bsrRecvOpEndSqePos_));
4831 :
4832 0 : HCCL_INFO("QueryBatchSendRecvPairEndPos send sqePos[%u] recv sqePos[%u]", bsrSendOpEndSqePos_, bsrRecvOpEndSqePos_);
4833 0 : return HCCL_SUCCESS;
4834 : }
4835 :
4836 0 : HcclResult HcclCommAicpu::CommitBSRStoredException(HcclOpExecFSM& fsmState, KfcError& errorCode)
4837 : {
4838 0 : if (GetBSRSendOpExecException()) {
4839 0 : bsrRetryOp_ = HCCL_SEND;
4840 0 : errorCode = KfcError::kSdma;
4841 0 : HCCL_INFO("CommitBSRStoredException: send stream remain retry error.");
4842 0 : } else if (GetBSRRecvOpExecException()) {
4843 0 : bsrRetryOp_ = HCCL_RECV;
4844 0 : errorCode = KfcError::kSdma;
4845 0 : HCCL_INFO("CommitBSRStoredException: recv stream remain retry error.");
4846 : }
4847 :
4848 0 : u32 retryCnt = (bsrRetryOp_ == HCCL_SEND) ? bsrSendRetryCnt_ : bsrRecvRetryCnt_;
4849 0 : if (errorCode != KfcError::kNone) {
4850 0 : bsrTargetOpId_ = (bsrRetryOp_ == HCCL_SEND) ? bsrSendOpId_ : bsrRecvOpId_;
4851 0 : HCCL_RUN_INFO(
4852 : "CommitBSRStoredException: stored op tag %s index %u , report retry error. curSendRetryCnt[%u],"
4853 : "curRecvRetryCnt[%u]",
4854 : bsrTargetOpId_.tag, bsrTargetOpId_.index, bsrSendRetryCnt_, bsrRecvRetryCnt_);
4855 0 : auto ret = aicpuHdc_.SetOpExecStatus(
4856 0 : kfcStatusTransferD2H_, bsrTargetOpId_, KfcStatus::kStoplaunch, errorCode, retryCnt);
4857 0 : HCCL_RETRY_CHK_RET_AND_TRANS_FSM(
4858 : ret, HCCL_ERROR("SetOpExecStatus failed, ret:%u", ret), KfcError::kExec,
4859 : HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR);
4860 0 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_FSM_STOPPING;
4861 : } else {
4862 0 : CHK_RET(UpdateOpExecStatus(fsmState, KfcStatus::kRuning, errorCode, retryCnt));
4863 0 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_FSM_WAIT_END;
4864 : }
4865 0 : return HCCL_SUCCESS;
4866 : }
4867 :
4868 0 : HcclResult HcclCommAicpu::GetBSRRetryOpId(const OpParam& param, HcclOpIdentifier& targetOpId)
4869 : {
4870 0 : KfcCommand cmd = KfcCommand::kNone;
4871 0 : CHK_RET(aicpuHdc_.GetOpExecCtrlCmd(kfcControlTransferH2D_, cmd));
4872 0 : if (cmd == KfcCommand::kStopLaunch) {
4873 0 : HcclOpIdentifier targetOp;
4874 0 : CHK_RET(aicpuHdc_.GetOpExecCtrlTargetOp(kfcControlTransferH2D_, targetOp));
4875 0 : std::string targetOpTag = std::string(reinterpret_cast<char*>(&targetOp.tag[0]));
4876 0 : if (targetOpTag == std::string(reinterpret_cast<char*>(&bsrSendOpId_.tag[0]))) {
4877 0 : bsrRetryOp_ = HCCL_SEND;
4878 0 : } else if (targetOpTag == std::string(reinterpret_cast<char*>(&bsrRecvOpId_.tag[0]))) {
4879 0 : bsrRetryOp_ = HCCL_RECV;
4880 0 : } else if (targetOpTag == param.tag) {
4881 0 : if (targetOp.detRank == bsrSendOpId_.detRank) {
4882 0 : bsrRetryOp_ = HCCL_SEND;
4883 0 : } else if (targetOp.srcRank == bsrRecvOpId_.srcRank) {
4884 0 : bsrRetryOp_ = HCCL_RECV;
4885 : } else {
4886 0 : HCCL_ERROR(
4887 : "hccl aicpu can not retry, got stop launch command, but target op srcRank[%u] and"
4888 : "dstRank[%u] is not match with send (dst:%u) or recv (src:%u) op",
4889 : targetOp.srcRank, targetOp.detRank, bsrSendOpId_.detRank, bsrRecvOpId_.srcRank);
4890 0 : return HCCL_E_INTERNAL;
4891 : }
4892 : } else {
4893 : // tag 不匹配，报错退出
4894 0 : HCCL_ERROR(
4895 : "hccl aicpu can not retry, got stop launch command, but target op tag[%s] is not match with"
4896 : "send (tag:%s) or recv (tag:%s) or batchsendrecv (tag:%s)",
4897 : targetOpTag.c_str(), bsrSendOpId_.tag, bsrRecvOpId_.tag, param.tag.c_str());
4898 0 : return HCCL_E_INTERNAL;
4899 : }
4900 0 : HCCL_RUN_INFO("hccl aicpu got command %u at op[tag: %s, index: %u].", cmd, targetOpTag.c_str(), targetOp.index);
4901 0 : } else {
4902 0 : if (GetBSRSendOpExecException()) {
4903 0 : bsrRetryOp_ = HCCL_SEND;
4904 0 : } else if (GetBSRRecvOpExecException()) {
4905 0 : bsrRetryOp_ = HCCL_RECV;
4906 : } else {
4907 : // 其他场景，报错退出
4908 0 : HCCL_ERROR("hccl aicpu find task exception, but send and recv op has no exception");
4909 0 : return HCCL_E_INTERNAL;
4910 : }
4911 : }
4912 0 : if (bsrRetryOp_ == HCCL_SEND) {
4913 0 : targetOpId = bsrSendOpId_;
4914 : } else {
4915 0 : targetOpId = bsrRecvOpId_;
4916 : }
4917 0 : HCCL_RUN_INFO(
4918 : "GetBSRRetryOpId: bsrRetryOpType %u, targetOpId: tag %s, index %u", bsrRetryOp_, targetOpId.tag,
4919 : targetOpId.index);
4920 0 : return HCCL_SUCCESS;
4921 : }
4922 :
4923 0 : HcclResult HcclCommAicpu::InitExecLoop(OpParam& param, std::unique_ptr<CollExecutorBase>& executor, u32& loopNum)
4924 : {
4925 0 : if ((param.opType == HcclCMDType::HCCL_CMD_BATCH_SEND_RECV) && retryEnable_) {
4926 0 : CHK_RET(executor->CreatePairWiseList(
4927 : param.BatchSendRecvDataDes.sendRecvItemsPtr, param.BatchSendRecvDataDes.itemNum));
4928 0 : CHK_RET(executor->GetPairWiseList(bsrSendRecvPairs_));
4929 0 : CHK_PRT_RET(
4930 : bsrSendRecvPairs_.empty(), HCCL_ERROR("[HcclCommAicpu][InitExecLoop]batchsendrecv pairs is empty"),
4931 : HCCL_E_INTERNAL);
4932 :
4933 0 : for (size_t i = 0; i < bsrSendRecvPairs_.size(); i++) {
4934 0 : CHK_PRT_RET(
4935 : (bsrSendRecvPairs_[i].size() > BSR_RETRY_SENDRECV_PAIR_NUM_MAX) || bsrSendRecvPairs_[i].empty(),
4936 : HCCL_ERROR("batchsendrecv pairs[%u] size[%u] is out of range [1,2]", i, bsrSendRecvPairs_.size()),
4937 : HCCL_E_INTERNAL);
4938 :
4939 0 : for (size_t j = 0; j < bsrSendRecvPairs_[i].size(); j++) {
4940 0 : CHK_PTR_NULL(bsrSendRecvPairs_[i][j]);
4941 : }
4942 :
4943 0 : CHK_PRT_RET(
4944 : ((bsrSendRecvPairs_[i].size() == BSR_RETRY_SENDRECV_PAIR_NUM_MAX)
4945 : && (bsrSendRecvPairs_[i][BSR_RETRY_SENDRECV_PAIR_INDEX_0]->sendRecvType
4946 : == bsrSendRecvPairs_[i][BSR_RETRY_SENDRECV_PAIR_INDEX_1]->sendRecvType)),
4947 : HCCL_ERROR(
4948 : "batchsendrecv pairs[%u] sendRecvType[%u] is same", i,
4949 : bsrSendRecvPairs_[i][BSR_RETRY_SENDRECV_PAIR_INDEX_0]->sendRecvType),
4950 : HCCL_E_INTERNAL);
4951 : }
4952 :
4953 0 : param.BatchSendRecvDataDes.curIterNum = 0;
4954 0 : loopNum = bsrSendRecvPairs_.size();
4955 0 : } else {
4956 0 : loopNum = 1;
4957 : }
4958 0 : HCCL_INFO("InitExecLoop: execute loop num %u", loopNum);
4959 0 : return HCCL_SUCCESS;
4960 : }
4961 :
4962 1 : HcclResult HcclCommAicpu::ParseHierarchicalAlgOption(u32* ahcConfInfo)
4963 : {
4964 1 : u32 algOptionSize = ahcConfInfo[TOP_HIERARCHICAL_CONF_lENGTH_INDEX];
4965 1 : if (algOptionSize >= (TOP_HIERARCHICAL_CONF_SIZE - 1)) {
4966 1 : HCCL_ERROR(
4967 : "[HcclCommAicpu][ParseHierarchicalAlgOption] hierarchicalAlgOption size[%lu] exceed maxsize[%lu]",
4968 : algOptionSize, (TOP_HIERARCHICAL_CONF_SIZE - 1));
4969 1 : return HCCL_E_INTERNAL;
4970 : }
4971 :
4972 0 : std::map<AHCConcOpType, TemplateType> hierarchicalAlgOption;
4973 0 : for (u32 i = TOP_HIERARCHICAL_CONF_INFO_INDEX; i < (TOP_HIERARCHICAL_CONF_INFO_INDEX + algOptionSize); i++) {
4974 0 : AHCConcOpType ahcConcOpType;
4975 0 : ahcConcOpType.ahcLevel = static_cast<AHCLevel>(
4976 0 : (ahcConfInfo[i] & TOP_HIERARCHICAL_CONF_LEVEL_LOCATION) >> TOP_HIERARCHICAL_CONF_LEVEL_SHIFT);
4977 0 : ahcConcOpType.concType = static_cast<ConcType>(
4978 0 : (ahcConfInfo[i] & TOP_HIERARCHICAL_CONF_CONC_TYPE_LOCATION) >> TOP_HIERARCHICAL_CONF_CONC_TYPE_SHIFT);
4979 0 : ahcConcOpType.ahcOpType = static_cast<AHCOpType>(
4980 0 : (ahcConfInfo[i] & TOP_HIERARCHICAL_CONF_OP_TYPE_LOCATION) >> TOP_HIERARCHICAL_CONF_OP_TYPE_SHIFT);
4981 0 : TemplateType templateType = static_cast<TemplateType>(
4982 0 : (ahcConfInfo[i] & TOP_HIERARCHICAL_CONF_TEMPLATE_TYPE_LOCATION)
4983 : >> TOP_HIERARCHICAL_CONF_TEMPLATE_TYPE_SHIFT);
4984 :
4985 0 : hierarchicalAlgOption[ahcConcOpType] = templateType;
4986 0 : HCCL_DEBUG(
4987 : "[HcclCommAicpu][ParseHierarchicalAlgOption]: index[%u] ahcLevel[%u] concType[%u] ahcOpType[%u], "
4988 : "templateType[%u]",
4989 : i, ahcConcOpType.ahcLevel, ahcConcOpType.concType, ahcConcOpType.ahcOpType, templateType);
4990 : }
4991 0 : topoMatcher_->SetAHCAlgOption(hierarchicalAlgOption);
4992 0 : return HCCL_SUCCESS;
4993 0 : }
4994 :
4995 0 : void HcclCommAicpu::PollCqeException(
4996 : hccl::Stream& stream, bool isReadClear, rtLogicCqReport_t& cqeException, CqeStatus& cqeStatus)
4997 : {
4998 0 : const HcclComStreamInfo& streamInfo = stream.GetHcclStreamInfo();
4999 : // 以下条件满足其中之一，进行Poll CQE：1、AICPU/Custom读清CQ 2、AICPU进程读取CQ信息
5000 0 : bool isPollCqe = true;
5001 0 : while (isPollCqe) {
5002 0 : LogControl logControl(false, retryEnable_); // 使能重执行场景，修改ERROR->RUN_WARNING，析构时自动恢复
5003 : CqeQueryInput cqeQueryInput;
5004 0 : dfx_tracer::ExecutorTracer::SetCqeQueryInput(GetDevId(), streamInfo, cqeQueryInput);
5005 0 : constexpr u32 reportSize = 256;
5006 : rtLogicCqReport_t streamReport[reportSize];
5007 0 : cqeQueryInput.cqeAddr = reinterpret_cast<uint8_t*>(streamReport); // 用于存放接收到的cq
5008 0 : cqeStatus = CqReportRecv(cqeQueryInput, cqeException);
5009 0 : isPollCqe = (cqeStatus == dfx::CqeStatus::kCqeException && isReadClear); // 读清CQ场景，继续查询
5010 0 : }
5011 0 : }
5012 :
5013 0 : void HcclCommAicpu::ExchangeCqeContext(
5014 : hccl::Stream& stream, rtLogicCqReport_t& cqeException, CqeStatus& cqeStatus, ErrCqeContext& cqeCtx)
5015 : {
5016 0 : HcclResult ret = HCCL_SUCCESS;
5017 : // aicpu进程把读取到的异常cq，写入共享内存
5018 0 : if (cqeStatus == dfx::CqeStatus::kCqeException) {
5019 0 : ret = stream.SetCqeContext(ErrCqeContext(
5020 0 : static_cast<u32>(cqeStatus), cqeException.taskId, cqeException.errorCode, cqeException.sqeType));
5021 : }
5022 :
5023 : // aicpu进程和custom进程，都从共享内存中读取cqe信息
5024 0 : if (ret != HCCL_SUCCESS || stream.GetCqeContext(cqeCtx) != HCCL_SUCCESS) {
5025 : // 写入/读取共享内存失败，标记流状态，避免背景线程继续轮询这条流，导致刷屏
5026 0 : SetStreamCqeExceptionStatus(stream, CqeExceptionStatus::kOther);
5027 0 : HCCL_ERROR(
5028 : "%s fail, set streamCqeExceptionStatus streamId[%u] exception status[%d]", __func__, stream.id(),
5029 : CqeExceptionStatus::kOther);
5030 0 : return;
5031 : }
5032 : }
5033 :
5034 0 : void HcclCommAicpu::HandleCqeException(hccl::Stream& stream, bool isReadClear)
5035 : {
5036 0 : std::unique_lock<std::mutex> lock(queryCqeMutex_);
5037 :
5038 : // poll cqe信息
5039 : rtLogicCqReport_t cqeException;
5040 0 : CqeStatus cqeStatus = CqeStatus::kDefault;
5041 0 : PollCqeException(stream, isReadClear, cqeException, cqeStatus);
5042 :
5043 : // 以下两种情况直接返回，不处理cq信息：1、读清CQ场景只读取不处理 2、本地记录流上已经有异常信息(避免刷屏)
5044 0 : if (isReadClear || GetStreamCqeExceptionStatus(stream) != CqeExceptionStatus::kNone) {
5045 0 : return;
5046 : }
5047 :
5048 : // aicpu和custom同步异常cqe信息，并记录在本地
5049 0 : ErrCqeContext cqeCtx;
5050 0 : ExchangeCqeContext(stream, cqeException, cqeStatus, cqeCtx);
5051 0 : cqeStatus = static_cast<dfx::CqeStatus>(cqeCtx.cqeStatus);
5052 :
5053 : // 处理流上的异常cq
5054 0 : if (cqeStatus != dfx::CqeStatus::kDefault) {
5055 0 : bool isSdmaTypeErr = cqeStatus == dfx::CqeStatus::kCqeException && cqeCtx.sqeType == RT_STARS_SQE_TYPE_SDMA;
5056 0 : bool isCompDataErr = cqeCtx.errorCode == RT_SDMA_COMPDATAERR || cqeCtx.errorCode == RT_SDMA_COMPERR
5057 0 : || cqeCtx.errorCode == RT_SDMA_DATAERR;
5058 0 : bool isSdmaCompDataErr = isSdmaTypeErr && isCompDataErr;
5059 :
5060 0 : ReportErrCqe(stream, cqeCtx);
5061 :
5062 : // 标记发生ErrCqe的流
5063 0 : hccl::CqeExceptionStatus cqeExceptionStatus
5064 0 : = isSdmaCompDataErr ? hccl::CqeExceptionStatus::kSdmaErr : hccl::CqeExceptionStatus::kOther;
5065 0 : SetStreamCqeExceptionStatus(stream, cqeExceptionStatus);
5066 :
5067 : // 通知kfc线程
5068 0 : dfxExtendInfo_.cqeException.sqeType = cqeCtx.sqeType;
5069 0 : dfxExtendInfo_.cqeException.errorCode = cqeCtx.errorCode;
5070 0 : dfxExtendInfo_.cqeStatus = cqeStatus;
5071 0 : dfxExtendInfo_.pollStatus = PollStatus::kStopAsException;
5072 0 : HCCL_INFO(
5073 : "update dfxExtendInfo, group %s, streamId %u, cqeStatus %d, sqetype %u, errorCode %u", identifier_.c_str(),
5074 : stream.id(), cqeStatus, cqeCtx.sqeType, cqeCtx.errorCode);
5075 : }
5076 0 : }
5077 :
5078 0 : void HcclCommAicpu::ReportErrCqe(hccl::Stream& stream, ErrCqeContext& cqeCtx)
5079 : {
5080 0 : dfx::CqeStatus cqeStatus = static_cast<dfx::CqeStatus>(cqeCtx.cqeStatus);
5081 0 : const HcclComStreamInfo& streamInfo = stream.GetHcclStreamInfo();
5082 0 : u32 head = 0;
5083 0 : u32 tail = 0;
5084 0 : QuerySqStatusByType(devId_, streamInfo.sqId, DRV_SQCQ_PROP_SQ_HEAD, head);
5085 0 : QuerySqStatusByType(devId_, streamInfo.sqId, DRV_SQCQ_PROP_SQ_TAIL, tail);
5086 :
5087 0 : bool isSdmaTypeErr = cqeStatus == dfx::CqeStatus::kCqeException && cqeCtx.sqeType == RT_STARS_SQE_TYPE_SDMA;
5088 0 : bool isCompDataErr = cqeCtx.errorCode == RT_SDMA_COMPDATAERR || cqeCtx.errorCode == RT_SDMA_COMPERR
5089 0 : || cqeCtx.errorCode == RT_SDMA_DATAERR;
5090 0 : bool isSdmaCompDataErr = isSdmaTypeErr && isCompDataErr;
5091 :
5092 : // 发生sdma、notify_wait、place_holder、write_value错误，上报error message
5093 0 : bool isComReportErrMesg
5094 : = cqeStatus == dfx::CqeStatus::kCqeException
5095 0 : && (cqeCtx.sqeType == RT_STARS_SQE_TYPE_SDMA || cqeCtx.sqeType == RT_STARS_SQE_TYPE_NOTIFY_WAIT
5096 0 : || cqeCtx.sqeType == RT_STARS_SQE_TYPE_PLACE_HOLDER || cqeCtx.sqeType == RT_STARS_SQE_TYPE_WRITE_VALUE);
5097 :
5098 : // 使能重执行且触发SDMA ERROR的场景，修改ERROR->RUN_WARNING
5099 0 : LogControl retryLog(false, retryEnable_ && isSdmaCompDataErr);
5100 :
5101 : // 只在AICPU进程上报taskException，Custom进程不上报
5102 0 : if (isComReportErrMesg && errMessageReport_) {
5103 : // 记录关键信息，并通过D2H通信通道交给host内存
5104 0 : GenTaskExceptionInfo(cqeCtx.sqeType, stream, head);
5105 : // 通知ts错误信息，触发非SDMA错误、不使能重执行触发SDMA错误、使能重执行触发无法重执行的错误时，进行故障上报
5106 0 : if (!isSdmaTypeErr || (isSdmaTypeErr && (!retryEnable_ || !isCompDataErr))) {
5107 0 : HcclResult ret = SendTaskExceptionByMBox(cqeCtx.errorCode);
5108 0 : HCCL_RUN_INFO(
5109 : "[HcclCommAicpu][SendTaskExceptionByMBox]group[%s]:"
5110 : "Try to send task exception by mailbox, errType[%u], errCode[%u], streamId[%d]",
5111 : identifier_.c_str(), cqeCtx.sqeType, cqeCtx.errorCode, stream.id());
5112 0 : CHK_PRT_CONT(
5113 : ret != HCCL_SUCCESS,
5114 : HCCL_ERROR(
5115 : "[OpRetry][AICPU]group[%s]:Send task exception by mailBox failed, streamId[%d]",
5116 : identifier_.c_str(), stream.id()));
5117 : }
5118 : // 当前阶段， 每个通信域在plog打印一次，error message作为host上报，只打印首次
5119 0 : errMessageReport_ = false;
5120 : }
5121 :
5122 : // 发生ErrCqe之后，统一在AICPU进程上报taskException
5123 0 : if (cqeStatus == dfx::CqeStatus::kCqeException) {
5124 0 : if (IsNoNeedWait() && cqeCtx.sqeType == RT_STARS_SQE_TYPE_PLACE_HOLDER) {
5125 0 : PrintTaskExceptionAllComm(); // 超时场景打印所有通信域的taskException
5126 0 : PrintAicpuCommExecStatus();
5127 0 : } else if (IsNoNeedWait()) {
5128 0 : ErrCqeContext cqeCtx;
5129 0 : stream.GetCqeContext(cqeCtx);
5130 0 : PrintTaskExceptionByTaskId(cqeCtx.sqeType, cqeCtx.taskId, stream, tail); // 仅打印本条流的taskException
5131 : }
5132 : }
5133 :
5134 0 : CHK_PRT_CONT(
5135 : !retryEnable_ && isSdmaCompDataErr,
5136 : HCCL_RUN_INFO(
5137 : "[OpRetry][AICPU]group[%s] hccl aicpu can not retry, retryEnable is false.", identifier_.c_str()));
5138 :
5139 0 : CHK_PRT_CONT(
5140 : retryEnable_ && isSdmaTypeErr && !isCompDataErr,
5141 : HCCL_RUN_INFO(
5142 : "[OpRetry][AICPU]group[%s] hccl aicpu can not retry, errCode is [%u].", identifier_.c_str(),
5143 : cqeCtx.errorCode));
5144 :
5145 0 : HCCL_ERROR(
5146 : "Exception happened, group %s, sqid %d, cqeStatus %d, sqetype %u, errorCode %u, head %u, tail %u",
5147 : identifier_.c_str(), streamInfo.sqId, cqeStatus, cqeCtx.sqeType, cqeCtx.errorCode, head, tail);
5148 0 : }
5149 :
5150 0 : HcclResult HcclCommAicpu::SendTaskExceptionByMBox(const uint16_t& rsErrorCode)
5151 : {
5152 0 : u32 localDeviceId = 0;
5153 : HcclSignalInfo notifyInfo;
5154 0 : opNotifies_[1]->GetNotifyData(notifyInfo);
5155 :
5156 0 : HCCL_INFO("[HcclCommAicpu][SendTaskExceptionByMBox] HostToDeviceLogicId[%u]", notifyInfo.devId);
5157 0 : CHK_RET(hrtDrvGetLocalDevIDByHostDevID(notifyInfo.devId, &localDeviceId));
5158 0 : CHK_RET(hccl_plf::SendTaskExceptionByMBox(
5159 : localDeviceId, opNotifies_[1]->notifyId_, notifyInfo.tsId, userStreamId_, rsErrorCode));
5160 0 : return HCCL_SUCCESS;
5161 : }
5162 :
5163 0 : void HcclCommAicpu::HandleIndOpCqe()
5164 : {
5165 0 : std::unique_lock<std::mutex> lock(queryCqeMutex_);
5166 0 : std::shared_lock<std::shared_mutex> rwLock(threadAicpuMutex_);
5167 0 : for (auto& thread : threads_) {
5168 0 : if (thread == nullptr) {
5169 0 : continue;
5170 : }
5171 0 : Stream stream = *thread->GetStream();
5172 : // 流上已有异常信息，不再重复读取
5173 0 : if (GetStreamCqeExceptionStatus(stream) != CqeExceptionStatus::kNone) {
5174 0 : continue;
5175 : }
5176 :
5177 : // poll cqe信息
5178 : rtLogicCqReport_t cqeException;
5179 : CqeQueryInput cqeQueryInput;
5180 0 : dfx_tracer::ExecutorTracer::SetCqeQueryInput(GetDevId(), stream.GetHcclStreamInfo(), cqeQueryInput);
5181 0 : constexpr u32 reportSize = 256;
5182 : rtLogicCqReport_t streamReport[reportSize];
5183 0 : cqeQueryInput.cqeAddr = reinterpret_cast<uint8_t*>(streamReport); // 用于存放接收到的cq
5184 0 : CqeStatus cqeStatus = CqReportRecv(cqeQueryInput, cqeException);
5185 : // 未读取到异常信息，返回
5186 0 : if (cqeStatus == dfx::CqeStatus::kDefault) {
5187 0 : continue;
5188 : }
5189 0 : ReportIndOpCqe(stream, cqeException, cqeStatus);
5190 0 : }
5191 0 : }
5192 :
5193 0 : void HcclCommAicpu::ReportIndOpCqe(hccl::Stream& stream, const rtLogicCqReport_t& cqeException, CqeStatus cqeStatus)
5194 : {
5195 0 : const HcclComStreamInfo& streamInfo = stream.GetHcclStreamInfo();
5196 0 : u32 head = 0;
5197 0 : u32 tail = 0;
5198 0 : QuerySqStatusByType(devId_, streamInfo.sqId, DRV_SQCQ_PROP_SQ_HEAD, head);
5199 0 : QuerySqStatusByType(devId_, streamInfo.sqId, DRV_SQCQ_PROP_SQ_TAIL, tail);
5200 :
5201 : // 打印taskException信息
5202 0 : if (cqeStatus == dfx::CqeStatus::kCqeException) {
5203 0 : if (cqeException.sqeType == RT_STARS_SQE_TYPE_PLACE_HOLDER) {
5204 0 : PrintTaskExceptionAllComm(); // 超时场景打印所有通信域的taskException
5205 0 : PrintAicpuCommExecStatus();
5206 : } else {
5207 0 : taskExecption_.PrintTaskExceptionByTaskId(cqeException.sqeType, cqeException.taskId, stream, tail);
5208 : }
5209 : }
5210 :
5211 : // 打印重执行提示信息
5212 0 : bool isSdmaTypeErr = cqeStatus == dfx::CqeStatus::kCqeException && cqeException.sqeType == RT_STARS_SQE_TYPE_SDMA;
5213 0 : bool isCompDataErr = cqeException.errorCode == RT_SDMA_COMPDATAERR || cqeException.errorCode == RT_SDMA_COMPERR;
5214 0 : bool isSdmaCompDataErr = isSdmaTypeErr && isCompDataErr;
5215 0 : if (retryEnable_ && isSdmaCompDataErr) {
5216 0 : uint16_t rsErrorCode = TS_ERROR_RETRY_CONSTRAINT;
5217 0 : CHK_PRT(SendTaskExceptionByMBox(rsErrorCode));
5218 0 : HCCL_RUN_INFO("[OpRetry][AICPU]group[%s] can not retry, IndOp does not support opRetry", identifier_.c_str());
5219 : }
5220 :
5221 : // 标记发生ErrCqe的流
5222 0 : hccl::CqeExceptionStatus cqeExceptionStatus
5223 0 : = isSdmaCompDataErr ? hccl::CqeExceptionStatus::kSdmaErr : hccl::CqeExceptionStatus::kOther;
5224 0 : SetStreamCqeExceptionStatus(stream, cqeExceptionStatus);
5225 :
5226 0 : HCCL_ERROR(
5227 : "Exception happened, group %s, streamId %u, sqid %d, cqeStatus %d, sqetype %u, errorCode %u, "
5228 : "head %u, tail %u",
5229 : identifier_.c_str(), stream.id(), streamInfo.sqId, cqeStatus, cqeException.sqeType, cqeException.errorCode,
5230 : head, tail);
5231 0 : }
5232 :
5233 0 : HcclResult HcclCommAicpu::RefreshLinkForSwitchNic(
5234 : const std::string& newTag, const TransportRequest& transportRequest, const std::map<u32, bool>& remoteRankPortMap,
5235 : bool isSecondBuild, LINK& switchLink)
5236 : {
5237 0 : u32 remoteRankId = transportRequest.remoteUserRank;
5238 0 : auto iterRemoteRank = remoteRankPortMap.find(remoteRankId);
5239 0 : bool needSwitch = iterRemoteRank != remoteRankPortMap.end();
5240 0 : bool isBackup = needSwitch && !(iterRemoteRank->second);
5241 0 : HCCL_INFO(
5242 : "[HcclCommAicpu][%s] newTag[%s], localRank[%u], remoteRank[%u], input memory type[%u], "
5243 : "output memory type[%u], isRdma[%u], isSecondBuild[%u], needSwitch[%u], isBackup[%u].",
5244 : __func__, newTag.c_str(), localUserRank_, remoteRankId, transportRequest.inputMemType,
5245 : transportRequest.outputMemType, transportRequest.isUsedRdma, isSecondBuild, needSwitch, isBackup);
5246 :
5247 0 : if (transportRequest.isUsedRdma && needSwitch) {
5248 0 : auto* linkRes = isBackup ? &linkRdmaResBackUp_ : &linkRdmaRes_;
5249 0 : auto iterRankLinks = linkRes->find(remoteRankId);
5250 0 : CHK_PRT_RET(
5251 : iterRankLinks == linkRes->end(),
5252 : HCCL_ERROR(
5253 : "[HcclCommAicpu][%s] comm[%s], local rank[%u], fail to find relative link for remote rank[%u], "
5254 : "isBackup[%u]",
5255 : __func__, identifier_.c_str(), localUserRank_, remoteRankId, isBackup),
5256 : HCCL_E_INTERNAL);
5257 0 : auto iterRankTagLinks = iterRankLinks->second.find(newTag);
5258 0 : CHK_PRT_RET(
5259 : iterRankTagLinks == iterRankLinks->second.end(),
5260 : HCCL_ERROR(
5261 : "[HcclCommAicpu][%s] comm[%s], local rank[%u], "
5262 : "fail to find relative tag[%s] of links for remote rank[%u], isBackup[%u]",
5263 : __func__, identifier_.c_str(), localUserRank_, newTag.c_str(), remoteRankId, isBackup),
5264 : HCCL_E_INTERNAL);
5265 0 : CHK_PRT_RET(
5266 : isSecondBuild && iterRankLinks->second[newTag].size() < 2U,
5267 : HCCL_ERROR(
5268 : "[HcclCommAicpu][%s] comm[%s], local rank[%u], isSecondBuild[%u], "
5269 : "fail to find second link for remmote rank[%u]",
5270 : __func__, identifier_.c_str(), localUserRank_, isSecondBuild, remoteRankId),
5271 : HCCL_E_INTERNAL);
5272 :
5273 0 : switchLink = isSecondBuild ? iterRankLinks->second[newTag][1] : iterRankLinks->second[newTag][0];
5274 0 : CHK_SMART_PTR_NULL(switchLink);
5275 :
5276 0 : auto iterAck = receivedAcks_.find(remoteRankId);
5277 0 : CHK_PRT_RET(
5278 : iterAck == receivedAcks_.end(), HCCL_ERROR("[%s]there is no link with rankId[%u]", __func__, remoteRankId),
5279 : HCCL_E_NOT_FOUND);
5280 0 : switchLink->SetSupportDataReceivedAck(iterAck->second);
5281 : }
5282 0 : return HCCL_SUCCESS;
5283 : }
5284 :
5285 0 : HcclResult HcclCommAicpu::ReAllocTransportForSwitchNic(
5286 : const std::string& newTag, AlgResourceResponse& algResResponse, std::map<u32, bool>& remoteRankPortMap)
5287 : {
5288 0 : HCCL_INFO(
5289 : "[HcclCommAicpu][%s] Entry realloc transport for switch nic, comm identifier[%s], localRank[%u], tag[%s]",
5290 : __func__, identifier_.c_str(), localUserRank_, newTag.c_str());
5291 0 : std::set<u32> bsrTansportRank;
5292 0 : for (auto& levelNSubCommTransport : algResResponse.opTransportResponse) {
5293 0 : for (auto& singleSubCommTransport : levelNSubCommTransport) {
5294 0 : for (size_t i = 0; i < singleSubCommTransport.transportRequests.size(); i++) {
5295 0 : auto& transportRequest = singleSubCommTransport.transportRequests[i];
5296 0 : if (transportRequest.isValid) {
5297 0 : receivedAcks_[transportRequest.remoteUserRank] = singleSubCommTransport.supportDataReceivedAck;
5298 0 : bool isSecondBuild = false;
5299 0 : if (transportRequest.isUsedRdma && newTag.find("BatchSendRecv") != std::string::npos
5300 0 : && bsrTansportRank.find(transportRequest.remoteUserRank) != bsrTansportRank.end()) {
5301 : // 仅在batchsendrecv rdma下发的时候需要第二次刷新，实际第一次下发都刷好了，第二次就是get一下
5302 0 : isSecondBuild = true;
5303 : }
5304 0 : bsrTansportRank.insert(transportRequest.remoteUserRank);
5305 0 : CHK_RET(RefreshLinkForSwitchNic(
5306 : newTag, transportRequest, remoteRankPortMap, isSecondBuild, singleSubCommTransport.links[i]));
5307 : }
5308 : }
5309 : }
5310 : }
5311 :
5312 0 : return HCCL_SUCCESS;
5313 0 : }
5314 :
5315 : HcclResult
5316 0 : HcclCommAicpu::RefreshRoceTransportsForSwitchNic(std::unordered_map<std::string, OpCommTransport>& reservedLinks)
5317 : {
5318 0 : ChangeLinkInfo changeLinkInfo;
5319 0 : HcclResult ret = LoadChangeLinkInfo(changeLinkInfo);
5320 :
5321 0 : CHK_PRT_RET(
5322 : changeLinkInfo.isChangeLinkFlag == false,
5323 : HCCL_ERROR(
5324 : "[HcclCommAicpu][%s] some error happened on host, switch nic has failed on rank[%u]. "
5325 : "The error message will be broadcast to other ranks in comm[%s].",
5326 : __func__, localUserRank_, identifier_.c_str()),
5327 : HCCL_E_PARA);
5328 :
5329 0 : CHK_PRT_RET(
5330 : ret != HCCL_SUCCESS,
5331 : HCCL_ERROR(
5332 : "[HcclCommAicpu][%s] comm identifier[%s], rank[%u], "
5333 : "load change link info failed.",
5334 : __func__, identifier_.c_str(), localUserRank_),
5335 : ret);
5336 :
5337 0 : std::map<u32, bool> remoteRankPortMap;
5338 0 : for (u32 i = 0; i < changeLinkInfo.remoteRankNum; i++) {
5339 0 : remoteRankPortMap.emplace(changeLinkInfo.remoteRankList[i], changeLinkInfo.isUseDefaultPort[i]);
5340 0 : HCCL_INFO(
5341 : "[HcclCommAicpu][%s] comm identifier[%s], rank[%u], remote rank[%u], isUseDefaultPort[%u].", __func__,
5342 : identifier_.c_str(), localUserRank_, changeLinkInfo.remoteRankList[i], changeLinkInfo.isUseDefaultPort[i]);
5343 : }
5344 :
5345 : // 对resMap中所有tag的transport link根据主备进行刷新
5346 0 : for (auto& resMapIt : resMap_) {
5347 0 : HCCL_RUN_INFO(
5348 : "[HcclCommAicpu][%s] rank[%u] refresh algResResponse of tag[%s].", __func__, localUserRank_,
5349 : resMapIt.first.c_str());
5350 0 : reservedLinks.emplace(resMapIt.first, resMapIt.second.opTransportResponse);
5351 0 : CHK_RET(ReAllocTransportForSwitchNic(resMapIt.first, resMapIt.second, remoteRankPortMap));
5352 : }
5353 0 : return HCCL_SUCCESS;
5354 0 : }
5355 :
5356 0 : HcclResult HcclCommAicpu::RevertTransportsForSwitchNic(std::unordered_map<std::string, OpCommTransport>& reservedLinks)
5357 : {
5358 0 : if (reservedLinks.size() == 0) {
5359 0 : HCCL_INFO(
5360 : "[HcclCommAicpu][%s] comm identifier[%s], rank[%u], reserved link is empty, "
5361 : "no need to revert transport.",
5362 : __func__, identifier_.c_str(), localUserRank_);
5363 0 : return HCCL_SUCCESS;
5364 : }
5365 0 : for (auto& resMapIt : resMap_) {
5366 0 : HCCL_RUN_INFO(
5367 : "[HcclCommAicpu][%s] comm identifier[%s], rank[%u], revert transport in algResResponse of tag[%s].",
5368 : __func__, identifier_.c_str(), localUserRank_, resMapIt.first.c_str());
5369 0 : auto linkIt = reservedLinks.find(resMapIt.first);
5370 0 : if (linkIt == reservedLinks.end()) {
5371 0 : HCCL_RUN_WARNING(
5372 : "[HcclCommAicpu][%s] comm identifier[%s], rank[%u], fail to find transport to revert.", __func__,
5373 : identifier_.c_str(), localUserRank_);
5374 0 : continue;
5375 : }
5376 0 : resMapIt.second.opTransportResponse = linkIt->second;
5377 : }
5378 0 : return HCCL_SUCCESS;
5379 : }
5380 :
5381 0 : HcclResult HcclCommAicpu::SwitchNicWaitHandleCommand(std::unordered_map<std::string, OpCommTransport>& reservedLinks)
5382 : {
5383 0 : auto waitSwitchCmdTimeoutMs = HcclGetWaitRetryCmdTimeout(0);
5384 0 : auto waitSwitchCmdTimeout = std::chrono::milliseconds(waitSwitchCmdTimeoutMs);
5385 :
5386 0 : HcclResult ret = HCCL_E_INTERNAL;
5387 0 : auto startTime = std::chrono::steady_clock::now();
5388 : while (true) {
5389 0 : KfcCommand switchHandleCmd = KfcCommand::kNone;
5390 0 : CHK_RET(aicpuHdc_.GetOpExecCtrlCmd(kfcControlTransferH2D_, switchHandleCmd));
5391 0 : if (switchHandleCmd == KfcCommand::kWaitSwitchNic) {
5392 0 : HCCL_INFO(
5393 : "[HcclCommAicpu][%s] comm identifier[%s], rank[%u], hccl aicpu recv switch nic handle command.",
5394 : __func__, identifier_.c_str(), localUserRank_);
5395 0 : ret = aicpuHdc_.SetOpExecStatus(kfcStatusTransferD2H_, KfcStatus::kWaitSwitchRes, KfcError::kNone, 0);
5396 0 : if (ret != HCCL_SUCCESS) {
5397 0 : (void)RevertTransportsForSwitchNic(reservedLinks);
5398 0 : return ret;
5399 : }
5400 0 : return HCCL_SUCCESS;
5401 0 : } else if ((std::chrono::steady_clock::now() - startTime) >= waitSwitchCmdTimeout) {
5402 0 : HCCL_ERROR(
5403 : "[HcclCommAicpu][%s] comm identifier[%s], rank[%u], "
5404 : "hccl aicpu wait switch nic handle command timeout[%u ms], local nic switch is reverted.",
5405 : __func__, identifier_.c_str(), localUserRank_, waitSwitchCmdTimeoutMs);
5406 0 : (void)RevertTransportsForSwitchNic(reservedLinks);
5407 0 : return HCCL_E_TIMEOUT;
5408 : }
5409 0 : }
5410 :
5411 : return ret;
5412 : }
5413 :
5414 0 : HcclResult HcclCommAicpu::SwitchNicWaitResult(std::unordered_map<std::string, OpCommTransport>& reservedLinks)
5415 : {
5416 0 : auto waitSwitchResultTimeoutSecond = GetExternalInputHcclLinkTimeOut() * 2U;
5417 0 : const auto waitSwitchResultTimeout = std::chrono::seconds(waitSwitchResultTimeoutSecond);
5418 :
5419 0 : auto startTime = std::chrono::steady_clock::now();
5420 : while (true) {
5421 0 : KfcCommand switchResultCmd = KfcCommand::kNone;
5422 0 : CHK_RET(aicpuHdc_.GetOpExecCtrlCmd(kfcControlTransferH2D_, switchResultCmd));
5423 0 : if (switchResultCmd == KfcCommand::kAllSwitched) {
5424 0 : HCCL_INFO(
5425 : "[HcclCommAicpu][%s] comm identifier[%s], rank[%u], "
5426 : "hccl aicpu switch nic success, all nic switch transport.",
5427 : __func__, identifier_.c_str(), localUserRank_);
5428 0 : return HCCL_SUCCESS;
5429 0 : } else if (switchResultCmd == KfcCommand::kSwitchFail) {
5430 0 : HCCL_ERROR(
5431 : "[HcclCommAicpu][%s] comm identifier[%s], rank[%u], "
5432 : "hccl aicpu comm switch nic error, local nic switch is reverted.",
5433 : __func__, identifier_.c_str(), localUserRank_);
5434 0 : (void)RevertTransportsForSwitchNic(reservedLinks);
5435 0 : return HCCL_E_INTERNAL;
5436 0 : } else if ((std::chrono::steady_clock::now() - startTime) >= waitSwitchResultTimeout) {
5437 0 : HCCL_ERROR(
5438 : "[HcclCommAicpu][%s] comm identifier[%s], rank[%u], "
5439 : "hccl aicpu wait switch nic result timeout[%u s].",
5440 : __func__, identifier_.c_str(), localUserRank_, waitSwitchResultTimeoutSecond);
5441 0 : (void)RevertTransportsForSwitchNic(reservedLinks);
5442 0 : return HCCL_E_TIMEOUT;
5443 : }
5444 0 : }
5445 :
5446 : return HCCL_E_INTERNAL;
5447 : }
5448 :
5449 0 : HcclResult HcclCommAicpu::SwitchNic()
5450 : {
5451 0 : KfcStatus state = KfcStatus::kSwitchError;
5452 0 : std::unordered_map<std::string, OpCommTransport> reservedLinks;
5453 0 : HcclResult ret = RefreshRoceTransportsForSwitchNic(reservedLinks);
5454 0 : if (ret != HCCL_SUCCESS) {
5455 0 : HCCL_ERROR(
5456 : "[HcclCommAicpu][%s] comm identifier[%s], rank[%u], refresh roce transports failed.", __func__,
5457 : identifier_.c_str(), localUserRank_);
5458 0 : state = KfcStatus::kSwitchError;
5459 0 : ret = aicpuHdc_.SetOpExecStatus(kfcStatusTransferD2H_, state, KfcError::kNone, 0);
5460 : } else {
5461 0 : HCCL_INFO(
5462 : "[HcclCommAicpu][%s] comm identifier[%s], rank[%u], refresh roce transports success.", __func__,
5463 : identifier_.c_str(), localUserRank_);
5464 0 : state = KfcStatus::kPlanSwitch;
5465 0 : ret = aicpuHdc_.SetOpExecStatus(kfcStatusTransferD2H_, state, KfcError::kNone, 0);
5466 : }
5467 :
5468 0 : if (ret != HCCL_SUCCESS) {
5469 0 : HCCL_ERROR(
5470 : "[HcclCommAicpu][SwitchNic] comm identifier[%s], rank[%u], "
5471 : "send switch status[%u] to host fail.",
5472 : __func__, identifier_.c_str(), localUserRank_, state);
5473 : }
5474 :
5475 0 : CHK_RET(SwitchNicWaitHandleCommand(reservedLinks));
5476 :
5477 0 : return SwitchNicWaitResult(reservedLinks);
5478 0 : }
5479 :
5480 0 : HcclResult HcclCommAicpu::ResumeChangeLink()
5481 : {
5482 0 : ChangeLinkInfo changeLinkInfo;
5483 0 : HcclResult ret = LoadChangeLinkInfo(changeLinkInfo);
5484 0 : bool useBackupLink = false;
5485 0 : std::map<u32, bool> remoteRankPortMap;
5486 0 : if (ret != HCCL_SUCCESS) {
5487 0 : HCCL_ERROR(
5488 : "[HcclCommAicpu][%s] comm identifier[%s], rank[%u], load change link info failed.", __func__,
5489 : identifier_.c_str(), localUserRank_);
5490 0 : return ret;
5491 : }
5492 0 : for (u32 i = 0; i < changeLinkInfo.remoteRankNum; i++) {
5493 0 : remoteRankPortMap.insert({changeLinkInfo.remoteRankList[i], changeLinkInfo.isUseDefaultPort[i]});
5494 0 : useBackupLink |= (!changeLinkInfo.isUseDefaultPort[i]);
5495 : }
5496 0 : ret = RefreshCommResponseTransportRes(remoteRankPortMap);
5497 0 : if (ret != HCCL_SUCCESS) {
5498 0 : HCCL_ERROR(
5499 : "[HcclCommAicpu][%s] comm identifier[%s], rank[%u], refresh roce transports failed.", __func__,
5500 : identifier_.c_str(), localUserRank_);
5501 0 : return ret;
5502 : } else {
5503 0 : HCCL_INFO(
5504 : "[HcclCommAicpu][%s] comm identifier[%s], rank[%u], refresh roce transports success.", __func__,
5505 : identifier_.c_str(), localUserRank_);
5506 0 : return HCCL_SUCCESS;
5507 : }
5508 0 : }
5509 :
5510 0 : HcclResult HcclCommAicpu::RefreshCommResponseTransportRes(std::map<u32, bool>& remoteRankPortMap)
5511 : {
5512 0 : PetersonLockGuard guard(hostDeviceLock_.get());
5513 0 : CHK_PRT_RET(guard.IsLockFailed(), HCCL_ERROR("[%s] hostDeviceLock lock failed", __func__), HCCL_E_INTERNAL);
5514 0 : OpParam param;
5515 : // 提前清理所有tag的链路，避免冲突
5516 0 : for (auto& resMapIt : resMap_) {
5517 0 : HCCL_RUN_INFO("[%s] clean roce resource of tag[%s].", __func__, resMapIt.first.c_str());
5518 0 : CleanRoceResource(resMapIt.first, resMapIt.second, remoteRankPortMap, param);
5519 : }
5520 0 : for (auto& resMapIt : resMap_) {
5521 0 : HCCL_RUN_INFO("[%s] refresh algResResponse of tag[%s].", __func__, resMapIt.first.c_str());
5522 0 : CHK_RET(ReAllocTransportResource(resMapIt.first, resMapIt.second, remoteRankPortMap, commParam_, param));
5523 : }
5524 0 : return HCCL_SUCCESS;
5525 0 : }
5526 :
5527 0 : HcclResult HcclCommAicpu::InitAicpuIndOp(CommAicpuParam* commAicpuParam)
5528 : {
5529 0 : if (indOpCommInitialized_) {
5530 0 : HCCL_RUN_INFO("[%s][InitAicpuIndOp]Group[%s] already initialized, skip reinit", __func__, identifier_.c_str());
5531 0 : return HCCL_SUCCESS;
5532 : }
5533 0 : CHK_PTR_NULL(commAicpuParam);
5534 0 : topoInfo_.deviceLogicId = commAicpuParam->deviceLogicId;
5535 0 : topoInfo_.devicePhyId = commAicpuParam->devicePhyId;
5536 0 : topoInfo_.deviceType = static_cast<DevType>(commAicpuParam->deviceType);
5537 0 : identifier_ = std::string(commAicpuParam->hcomId);
5538 0 : topoInfo_.userRankSize = commAicpuParam->userRankSize;
5539 0 : topoInfo_.userRank = commAicpuParam->userRank;
5540 0 : notifys_.reserve(hccl::HCCL_THREAD_NOTIFY_MAX_NUM);
5541 0 : if (topoInfo_.deviceType == DevType::DEV_TYPE_910_93 || topoInfo_.deviceType == DevType::DEV_TYPE_910B) {
5542 0 : notifySize_ = NOTIFY_SIZE_FOUR;
5543 : } else {
5544 0 : notifySize_ = NOTIFY_SIZE_EIGHT;
5545 : }
5546 :
5547 0 : CHK_RET(hrtSetWorkModeAicpu(true));
5548 0 : CHK_RET(hrtSetlocalDevice(topoInfo_.deviceLogicId));
5549 0 : CHK_RET(hrtSetlocalDeviceType(topoInfo_.deviceType));
5550 0 : CHK_RET(hrtDrvGetLocalDevIDByHostDevID(topoInfo_.devicePhyId, &devId_));
5551 0 : CHK_RET(taskExecption_.Init(devId_, topoInfo_.userRank, identifier_));
5552 0 : CHK_RET(RegisterProfCallBack());
5553 :
5554 0 : if (topoInfo_.deviceType == DevType::DEV_TYPE_950 || topoInfo_.deviceType == DevType::DEV_TYPE_960) {
5555 0 : HCCL_INFO("[HcclCommAicpu][InitAicpuIndOp] InitAicpuIndOpV2 start");
5556 0 : indOpCommInitialized_ = true;
5557 0 : return HCCL_SUCCESS;
5558 : }
5559 :
5560 0 : HCCL_INFO("[HcclCommAicpu][InitAicpuIndOp] InitAicpuIndOp start");
5561 0 : if (!FindDispatcherByCommId(&dispatcherCtx_, identifier_.c_str())) {
5562 0 : CHK_RET(CreateDispatcherCtx(&dispatcherCtx_, devId_, identifier_.c_str()));
5563 : }
5564 0 : CHK_PTR_NULL(dispatcherCtx_);
5565 0 : hccl::DispatcherCtx* Ctx_temp = static_cast<DispatcherCtx*>(dispatcherCtx_);
5566 0 : HCCL_INFO("[%s] Ctx_temp[%p]", __func__, (void*)Ctx_temp);
5567 0 : (void)RegisterLoadTaskCallBack(Ctx_temp->GetDispatcher(), nullptr, dfx::TaskProfilingCallBack); // 注册dispatcher
5568 0 : if (commAicpuParam->kfcControlTransferH2DParams.buffLen != 0 && kfcControlTransferH2D_ == nullptr) {
5569 0 : EXCEPTION_CATCH((kfcControlTransferH2D_ = std::make_shared<hccl::HDCommunicate>()), return HCCL_E_PTR);
5570 0 : CHK_SMART_PTR_NULL(kfcControlTransferH2D_);
5571 0 : CHK_RET(kfcControlTransferH2D_->InitDevice(commAicpuParam->kfcControlTransferH2DParams));
5572 : }
5573 0 : if (commAicpuParam->kfcStatusTransferD2HParams.buffLen != 0 && kfcStatusTransferD2H_ == nullptr) {
5574 0 : EXCEPTION_CATCH((kfcStatusTransferD2H_ = std::make_shared<hccl::HDCommunicate>()), return HCCL_E_PTR);
5575 0 : CHK_SMART_PTR_NULL(kfcStatusTransferD2H_);
5576 0 : CHK_RET(kfcStatusTransferD2H_->InitDevice(commAicpuParam->kfcStatusTransferD2HParams));
5577 : }
5578 :
5579 0 : indOpCommInitialized_ = true;
5580 :
5581 : // 在indOpCommInitialized_变为true后拉起背景线程
5582 0 : AicpuComContext* ctx = AicpuGetComContext();
5583 0 : AicpuHcclProcess::CallMC2MaintenanceThread(ctx);
5584 :
5585 0 : HCCL_RUN_INFO(
5586 : "%s group[%s] success!, deviceLogicId[%u], devicePhyId[%u], deviceType[%u], notifySize[%u], "
5587 : "dispatcherCtx[%p]",
5588 : __func__, identifier_.c_str(), topoInfo_.deviceLogicId, topoInfo_.devicePhyId, topoInfo_.deviceType,
5589 : notifySize_, dispatcherCtx_);
5590 :
5591 0 : return HCCL_SUCCESS;
5592 : }
5593 :
5594 0 : HcclResult HcclCommAicpu::InitThreads(ThreadMgrAicpuParam* param)
5595 : {
5596 0 : CHK_PTR_NULL(param);
5597 0 : u32 threadNum = param->threadNum;
5598 0 : std::vector<std::shared_ptr<Thread>> outThreads;
5599 0 : outThreads.reserve(threadNum);
5600 0 : std::string hcomId(param->hcomId);
5601 0 : for (u32 i = 0; i < threadNum; ++i) {
5602 0 : std::string thdUniqueId(param->threadParam[i], THREAD_UNIQUE_ID_MAX_SIZE);
5603 0 : if (UNLIKELY(HcclCheckLogLevel(HCCL_LOG_INFO))) {
5604 0 : std::ostringstream oss;
5605 0 : oss << "threadParam[" << i << "] raw bytes: ";
5606 0 : for (u32 j = 0; j < THREAD_UNIQUE_ID_MAX_SIZE; ++j) {
5607 0 : oss << std::hex << std::setw(2) << std::setfill('0')
5608 0 : << static_cast<unsigned int>(static_cast<unsigned char>(param->threadParam[i][j])) << " ";
5609 : }
5610 0 : HCCL_INFO("[HcclCommAicpu][%s] %s", __func__, oss.str().c_str());
5611 0 : }
5612 0 : std::shared_ptr<AicpuTsThread> thread;
5613 0 : EXCEPTION_CATCH((thread = std::make_shared<AicpuTsThread>(thdUniqueId)), return HCCL_E_PTR);
5614 0 : HcclResult ret = thread->Init();
5615 0 : if (ret != HCCL_SUCCESS) {
5616 0 : HCCL_ERROR(
5617 : "[HcclCommAicpu][%s] comm identifier[%s], init threads num[%u] failed at index %u", __func__,
5618 : hcomId.c_str(), param->threadNum, i);
5619 0 : return ret;
5620 : }
5621 0 : outThreads.emplace_back(thread);
5622 0 : }
5623 :
5624 0 : ThreadHandle* threadArray = static_cast<ThreadHandle*>(param->deviceHandle);
5625 : // 空指针校验
5626 0 : CHK_PTR_NULL(threadArray);
5627 0 : for (size_t i = 0; i < threadNum; ++i) {
5628 0 : threadArray[i] = reinterpret_cast<ThreadHandle>(outThreads[i].get()); // 拷贝裸指针
5629 0 : HCCL_INFO("[HcclCommAicpu][%s] threadArray[%u] = [%lu]", __func__, i, threadArray[i]);
5630 : }
5631 0 : std::unique_lock<std::shared_mutex> rwLock(threadAicpuMutex_);
5632 0 : threads_.insert(
5633 0 : threads_.end(), std::make_move_iterator(outThreads.begin()), std::make_move_iterator(outThreads.end()));
5634 0 : HCCL_INFO(
5635 : "[HcclCommAicpu][%s] comm identifier[%s], init threads num[%u] success", __func__, hcomId.c_str(), threadNum);
5636 : // 为上报翻转初始化资源
5637 0 : if (topoInfo_.deviceType != DevType::DEV_TYPE_950 && topoInfo_.deviceType != DevType::DEV_TYPE_960) {
5638 0 : CHK_RET(InitProfthreadResource(threadNum));
5639 : }
5640 0 : return HCCL_SUCCESS;
5641 0 : }
5642 :
5643 0 : HcclResult HcclCommAicpu::InitProfthreadResource(u32 threadNum)
5644 : {
5645 0 : groupHashId_ = dfx::ProfilingManager::GetProfHashId(identifier_.c_str(), identifier_.length());
5646 0 : HCCL_INFO(
5647 : "[%s], group[%s], groupHash[%llu], threadNum[%u] ", __func__, identifier_.c_str(), groupHashId_, threadNum);
5648 0 : dfx::ProfCommInfo profInfo{groupHashId_, topoInfo_.userRankSize, topoInfo_.userRank};
5649 : // 添加检查确保 threadNum 不超过线程总数
5650 0 : if (threadNum > threads_.size()) {
5651 0 : HCCL_ERROR("[%s] threadNum Err", __func__);
5652 0 : return HCCL_E_PARA;
5653 : }
5654 : // 从后往前迭代指定数量
5655 0 : auto begin = threads_.rbegin();
5656 0 : auto end = begin + threadNum;
5657 0 : for (auto it = begin; it != end; ++it) {
5658 0 : CHK_RET(dfx::ProfilingManager::AddProfInfoByStreamId((*it)->GetStream()->id(), identifier_, profInfo));
5659 : }
5660 0 : dfx::ProfilingExtendInfoHelper::InitProfItemId();
5661 0 : return HCCL_SUCCESS;
5662 : }
5663 :
5664 0 : HcclResult HcclCommAicpu::AllocChannelResource(HcclIndOpChannelRemoteResV3* commParam)
5665 : {
5666 0 : CHK_PTR_NULL(commParam);
5667 0 : if (commParam->engine != COMM_ENGINE_AICPU && commParam->engine != COMM_ENGINE_AICPU_TS) {
5668 0 : HCCL_ERROR(
5669 : "[HcclCommAicpu][%s] engine type[%s] is not supported", __func__,
5670 : GetEnumToString(GetCommEngineStatusStrMap(), commParam->engine).c_str());
5671 0 : return HCCL_E_PARA;
5672 : }
5673 :
5674 0 : if (commParam->listNum != 0 && commParam->remoteResV2 == nullptr) {
5675 0 : HCCL_ERROR("[HcclCommAicpu][%s] listNum[%u], remoteResV2 is nullptr", __func__, commParam->listNum);
5676 0 : return HCCL_E_PARA;
5677 : }
5678 :
5679 0 : multiQpThreshold_ = commParam->multiQpThreshold;
5680 0 : localUserRank_ = commParam->localUserRank;
5681 0 : HCCL_INFO(
5682 : "%s multiQpThreshold[%u], localUserRank[%u], deviceLogicId[%d], devicePhyId[%u], deviceType[%d], "
5683 : "listNum[%u]",
5684 : __func__, multiQpThreshold_, localUserRank_, topoInfo_.deviceLogicId, topoInfo_.devicePhyId,
5685 : topoInfo_.deviceType, commParam->listNum);
5686 0 : for (u32 idx = 0; idx < commParam->listNum; idx++) {
5687 0 : HCCL_INFO(
5688 : "%s listNum[%u], listIdx[%u], remoteWorldRank[%u], remoteRank[%u], isUsedRdma[%d]", __func__,
5689 : commParam->listNum, idx, commParam->remoteResV2[idx].remoteWorldRank,
5690 : commParam->remoteResV2[idx].remoteRank, commParam->remoteResV2[idx].isUsedRdma);
5691 :
5692 0 : rankData_[commParam->remoteResV2[idx].remoteRank].remoteWorldRank = commParam->remoteResV2[idx].remoteWorldRank;
5693 0 : rankData_[commParam->remoteResV2[idx].remoteRank].remoteUsrRankId = commParam->remoteResV2[idx].remoteRank;
5694 0 : if (commParam->remoteResV2[idx].isUsedRdma) {
5695 0 : CHK_RET(InitRoceChannel(commParam, idx));
5696 : } else {
5697 0 : CHK_RET(InitP2pChannel(commParam, idx));
5698 : }
5699 : }
5700 0 : return HCCL_SUCCESS;
5701 : }
5702 :
5703 0 : HcclResult HcclCommAicpu::InitP2pChannel(HcclIndOpChannelRemoteResV3* commParam, uint32_t channelIndex)
5704 : {
5705 0 : CHK_PTR_NULL(commParam);
5706 0 : CHK_PTR_NULL(commParam->channelList);
5707 0 : HcclIndOpChannelRemoteResV2& remoteResV2 = commParam->remoteResV2[channelIndex];
5708 0 : u32 linkType = static_cast<u32>(remoteResV2.channelP2p.transportAttr.linkType);
5709 0 : std::string channelKey = std::string(commParam->channelTag) + ":" + std::to_string(commParam->engine) + ":"
5710 0 : + std::to_string(remoteResV2.remoteRank) + ":" + std::to_string(linkType);
5711 0 : HCCL_INFO("%s channelKey[%s]", __func__, channelKey.c_str());
5712 0 : if (channelHandleMap_.find(channelKey) != channelHandleMap_.end()) {
5713 0 : HCCL_ERROR("[%s]the channel has existed.", __func__);
5714 0 : return HCCL_E_INTERNAL;
5715 : }
5716 :
5717 0 : HcclChannelP2p& channelP2p = remoteResV2.channelP2p;
5718 0 : if (channelP2p.localIpcSignal[0].resId == INVALID_U64) {
5719 0 : HCCL_ERROR("[%s]the Channel is invalid", __func__);
5720 0 : return HCCL_E_INTERNAL;
5721 : }
5722 :
5723 : // 创建Transport对象
5724 0 : MachinePara machinePara;
5725 0 : CHK_RET(SetTransportMachinePara(machinePara, remoteResV2.remoteRank, commParam->channelTag));
5726 0 : machinePara.notifyNum = remoteResV2.p2pNotifyNum;
5727 : // 获取localMem & remoteMem
5728 0 : TransportDeviceP2pData transDevP2pData;
5729 0 : transDevP2pData.inputBufferPtr = reinterpret_cast<void*>(channelP2p.remoteHcclbuffer.addr);
5730 0 : transDevP2pData.outputBufferPtr = reinterpret_cast<void*>(channelP2p.remoteHcclbuffer.addr);
5731 0 : if (transDevP2pData.inputBufferPtr == nullptr || transDevP2pData.outputBufferPtr == nullptr) {
5732 0 : HCCL_ERROR(
5733 : "[%s]input ptr[%p] or output ptr[%p] is null.", __func__, transDevP2pData.inputBufferPtr,
5734 : transDevP2pData.outputBufferPtr);
5735 0 : return HCCL_E_PARA;
5736 : }
5737 : // 获取Notify资源
5738 0 : CHK_RET(SetChannelP2pNotify(transDevP2pData, remoteResV2.p2pNotifyNum, channelP2p)); // 待确认notify是否
5739 : // 获取transportAttr信息
5740 0 : transDevP2pData.transportAttr = channelP2p.transportAttr;
5741 : // 创建Transport对象
5742 0 : std::shared_ptr<Transport> link;
5743 0 : TransportPara para{};
5744 0 : const std::unique_ptr<NotifyPool> notifyPool;
5745 0 : DispatcherCtx* ctx = static_cast<DispatcherCtx*>(dispatcherCtx_);
5746 0 : CHK_PRT(ctx->SetDispatcherHcclQos(remoteResV2.channelP2p.qos)); // 调度器添加hcclQos
5747 0 : CHK_PTR_NULL(ctx);
5748 0 : link.reset(new (std::nothrow) Transport(
5749 0 : TransportType::TRANS_TYPE_DEVICE_P2P, para, ctx->GetDispatcher(), notifyPool, machinePara, transDevP2pData));
5750 0 : CHK_SMART_PTR_NULL(link);
5751 0 : CHK_RET(link->Init()); // 初始化需要增加远端用户注册内存
5752 :
5753 0 : ChannelHandle channelHandle = reinterpret_cast<ChannelHandle>(link.get());
5754 0 : channelHandleMap_[channelKey] = channelHandle;
5755 0 : linkMap_[channelHandle] = link;
5756 :
5757 : // 恢复出的channelHandle回填到commParam中
5758 0 : CHK_PTR_NULL(commParam->channelList);
5759 0 : ChannelHandle* channelList = reinterpret_cast<ChannelHandle*>(commParam->channelList);
5760 0 : channelList[channelIndex] = channelHandle;
5761 :
5762 0 : return HCCL_SUCCESS;
5763 0 : }
5764 :
5765 0 : HcclResult HcclCommAicpu::SetChannelP2pNotify(
5766 : TransportDeviceP2pData& transDevP2pData, u64& p2pNotifyNum, HcclChannelP2p& channelP2p)
5767 : {
5768 0 : u64 actualNotifyNum = 0;
5769 : // 获取Ipc notify信息
5770 0 : CHK_RET(CheckNotifyOrQPMaxNum(actualNotifyNum, LINK_P2P_MAX_NUM, true));
5771 0 : std::shared_ptr<LocalNotify> ipcPreWaitNotify = std::make_shared<LocalNotify>();
5772 0 : CHK_RET(InitAndVerifySingleSignal(channelP2p.localIpcSignal[actualNotifyNum], ipcPreWaitNotify));
5773 0 : transDevP2pData.ipcPreWaitNotify = ipcPreWaitNotify;
5774 :
5775 0 : std::shared_ptr<RemoteNotify> ipcPreRecordNotify = std::make_shared<RemoteNotify>();
5776 0 : CHK_RET(InitAndVerifySingleSignal(channelP2p.remoteIpcSignal[actualNotifyNum], ipcPreRecordNotify));
5777 0 : transDevP2pData.ipcPreRecordNotify = ipcPreRecordNotify;
5778 0 : actualNotifyNum++;
5779 :
5780 0 : CHK_RET(CheckNotifyOrQPMaxNum(actualNotifyNum, LINK_P2P_MAX_NUM, true));
5781 0 : std::shared_ptr<LocalNotify> ipcPostWaitNotify = std::make_shared<LocalNotify>();
5782 0 : CHK_RET(InitAndVerifySingleSignal(channelP2p.localIpcSignal[actualNotifyNum], ipcPostWaitNotify));
5783 0 : transDevP2pData.ipcPostWaitNotify = ipcPostWaitNotify;
5784 :
5785 0 : std::shared_ptr<RemoteNotify> ipcPostRecordNotify = std::make_shared<RemoteNotify>();
5786 0 : CHK_RET(InitAndVerifySingleSignal(channelP2p.remoteIpcSignal[actualNotifyNum], ipcPostRecordNotify));
5787 0 : transDevP2pData.ipcPostRecordNotify = ipcPostRecordNotify;
5788 0 : actualNotifyNum++;
5789 :
5790 0 : transDevP2pData.userLocalNotify.resize(p2pNotifyNum, nullptr);
5791 0 : transDevP2pData.userRemoteNotify.resize(p2pNotifyNum, nullptr);
5792 :
5793 0 : for (u32 idx = 0; idx < p2pNotifyNum; idx++) {
5794 0 : std::shared_ptr<LocalNotify> ipcWaitNotify = std::make_shared<LocalNotify>();
5795 0 : CHK_RET(InitAndVerifySingleSignal(channelP2p.localIpcSignal[actualNotifyNum], ipcWaitNotify));
5796 0 : transDevP2pData.userLocalNotify[idx] = ipcWaitNotify;
5797 :
5798 0 : std::shared_ptr<RemoteNotify> ipcRecordNotify = std::make_shared<RemoteNotify>();
5799 0 : CHK_RET(InitAndVerifySingleSignal(channelP2p.remoteIpcSignal[actualNotifyNum], ipcRecordNotify));
5800 0 : transDevP2pData.userRemoteNotify[idx] = ipcRecordNotify;
5801 :
5802 0 : actualNotifyNum++;
5803 0 : }
5804 :
5805 0 : HCCL_DEBUG(
5806 : "%s get p2pNotify success, p2pNotifyNum[%llu], actualNotifyNum[%llu]", __func__, p2pNotifyNum, actualNotifyNum);
5807 0 : return HCCL_SUCCESS;
5808 0 : }
5809 :
5810 0 : HcclResult HcclCommAicpu::InitRoceChannel(HcclIndOpChannelRemoteResV3* commParam, uint32_t channelIndex)
5811 : {
5812 0 : HcclIndOpChannelRemoteResV2& remoteResV2 = commParam->remoteResV2[channelIndex];
5813 0 : std::string channelKey = std::string(commParam->channelTag) + ":" + std::to_string(commParam->engine) + ":"
5814 0 : + std::to_string(remoteResV2.remoteRank) + ":"
5815 0 : + std::to_string(CommProtocol::COMM_PROTOCOL_ROCE);
5816 0 : if (channelHandleMap_.find(channelKey) != channelHandleMap_.end()) {
5817 0 : HCCL_ERROR("[%s]the channel has existed.", __func__);
5818 0 : return HCCL_E_INTERNAL;
5819 : }
5820 :
5821 0 : HcclChannelRoce& channelRoce = remoteResV2.channelRoce;
5822 0 : if (channelRoce.localNotifyList == 0) {
5823 0 : HCCL_ERROR("[%s]the Channel is invalid", __func__);
5824 0 : return HCCL_E_INTERNAL;
5825 : }
5826 0 : HcclSignalInfo* localNotifyList = reinterpret_cast<HcclSignalInfo*>(channelRoce.localNotifyList);
5827 0 : if (localNotifyList[0].resId == INVALID_U64) {
5828 0 : HCCL_INFO("[%s]the channel notify resource is invalid", __func__);
5829 0 : return HCCL_E_INTERNAL;
5830 : }
5831 :
5832 : // 创建Transport对象
5833 0 : MachinePara machinePara;
5834 0 : CHK_RET(SetTransportMachinePara(machinePara, remoteResV2.remoteRank, commParam->channelTag)); // 待确认是否填充完毕
5835 0 : machinePara.notifyNum = remoteResV2.roceNotifyNum;
5836 : // 获取localMem & remoteMem
5837 0 : TransportDeviceIbverbsData transDevIbverbsData;
5838 0 : transDevIbverbsData.inputBufferPtr = reinterpret_cast<void*>(channelRoce.remoteHcclbuffer.addr);
5839 0 : transDevIbverbsData.outputBufferPtr = reinterpret_cast<void*>(channelRoce.remoteHcclbuffer.addr);
5840 0 : if (transDevIbverbsData.inputBufferPtr == nullptr || transDevIbverbsData.outputBufferPtr == nullptr) {
5841 0 : HCCL_ERROR(
5842 : "[%s]input ptr[%p] or output ptr[%p] is null.", __func__, transDevIbverbsData.inputBufferPtr,
5843 : transDevIbverbsData.outputBufferPtr);
5844 0 : return HCCL_E_PARA;
5845 : }
5846 0 : transDevIbverbsData.localInputMem = channelRoce.localHcclbuffer;
5847 0 : transDevIbverbsData.localOutputMem = channelRoce.localHcclbuffer;
5848 0 : transDevIbverbsData.localNotifyValueAddr = channelRoce.notifyValue;
5849 0 : transDevIbverbsData.notifyValueKey = channelRoce.notifyValueKey;
5850 0 : transDevIbverbsData.remoteInputKey = channelRoce.remoteHcclbuffer.key;
5851 0 : transDevIbverbsData.remoteOutputKey = channelRoce.remoteHcclbuffer.key;
5852 : // 需要添加远端用户注册内存
5853 :
5854 : // 获取QPinfo
5855 0 : u32 roceQpNumSum = channelRoce.qpsPerConnection + static_cast<u32>(channelRoce.qpsPerConnection != 1);
5856 0 : transDevIbverbsData.qpInfo.resize(roceQpNumSum);
5857 0 : std::copy_n(channelRoce.QpInfo, roceQpNumSum, transDevIbverbsData.qpInfo.begin());
5858 0 : transDevIbverbsData.multiQpThreshold = multiQpThreshold_;
5859 0 : transDevIbverbsData.qpsPerConnection = channelRoce.qpsPerConnection;
5860 :
5861 : // 获取notify
5862 0 : u64& roceNotifyNum = remoteResV2.roceNotifyNum; // 是否需要待确认
5863 0 : CHK_RET(SetChannelRoceNotify(transDevIbverbsData, roceNotifyNum, channelRoce));
5864 :
5865 : // 创建Transport对象
5866 0 : std::shared_ptr<Transport> link;
5867 0 : TransportPara para{};
5868 0 : para.timeout = linkTimeOut_; // 暂无法设置
5869 0 : const std::unique_ptr<NotifyPool> notifyPool;
5870 0 : DispatcherCtx* ctx = static_cast<DispatcherCtx*>(dispatcherCtx_);
5871 0 : CHK_PTR_NULL(ctx);
5872 0 : link.reset(new (std::nothrow) Transport(
5873 : TransportType::TRANS_TYPE_DEVICE_IBVERBS, para, ctx->GetDispatcher(), notifyPool, machinePara,
5874 0 : TransportDeviceP2pData(), transDevIbverbsData));
5875 0 : CHK_SMART_PTR_NULL(link);
5876 0 : CHK_RET(link->Init()); // 初始化需要增加远端用户注册内存
5877 :
5878 0 : ChannelHandle channelHandle = reinterpret_cast<ChannelHandle>(link.get());
5879 0 : channelHandleMap_[channelKey] = channelHandle;
5880 0 : linkMap_[channelHandle] = link;
5881 :
5882 : // 恢复出的channelHandle回填到commParam中
5883 0 : ChannelHandle* channelList = reinterpret_cast<ChannelHandle*>(commParam->channelList);
5884 0 : channelList[channelIndex] = channelHandle;
5885 :
5886 0 : return HCCL_SUCCESS;
5887 0 : }
5888 :
5889 0 : HcclResult HcclCommAicpu::SetChannelRoceNotify(
5890 : TransportDeviceIbverbsData& transDevIbverbsData, u64& roceNotifyNum, HcclChannelRoce& channelRoce)
5891 : {
5892 0 : u64 actualNotifyNum = 0;
5893 0 : if (channelRoce.localNotifyList == 0 || channelRoce.remoteNotifyList == 0) {
5894 0 : HCCL_DEBUG("[%s] Empty local and remote notify lists, skipping notify resource creation.", __func__);
5895 0 : return HCCL_SUCCESS;
5896 : }
5897 0 : HcclSignalInfo* localNotifyList = reinterpret_cast<HcclSignalInfo*>(channelRoce.localNotifyList);
5898 0 : AddrKey* remoteNotifyList = reinterpret_cast<AddrKey*>(channelRoce.remoteNotifyList);
5899 0 : if (localNotifyList == nullptr || remoteNotifyList == nullptr) {
5900 0 : HCCL_ERROR("[%s]nullptr found in localNotifyList or remoteNotifyList from device mem, check.", __func__);
5901 0 : return HCCL_E_INTERNAL;
5902 : }
5903 : // 获取RDMA Notify信息
5904 0 : std::shared_ptr<LocalNotify> ackNotify = std::make_shared<LocalNotify>();
5905 0 : CHK_RET(InitAndVerifySingleSignal(localNotifyList[actualNotifyNum], ackNotify));
5906 0 : transDevIbverbsData.ackNotify = ackNotify;
5907 0 : transDevIbverbsData.remoteAckNotifyDetails = remoteNotifyList[actualNotifyNum];
5908 0 : actualNotifyNum++;
5909 :
5910 0 : std::shared_ptr<LocalNotify> dataNotify = std::make_shared<LocalNotify>();
5911 0 : CHK_RET(InitAndVerifySingleSignal(localNotifyList[actualNotifyNum], dataNotify));
5912 0 : transDevIbverbsData.dataNotify = dataNotify;
5913 0 : transDevIbverbsData.remoteDataNotifyDetails = remoteNotifyList[actualNotifyNum];
5914 0 : actualNotifyNum++;
5915 :
5916 0 : std::shared_ptr<LocalNotify> dataAckNotify = std::make_shared<LocalNotify>();
5917 0 : CHK_RET(InitAndVerifySingleSignal(localNotifyList[actualNotifyNum], dataAckNotify));
5918 0 : transDevIbverbsData.dataAckNotify = dataAckNotify;
5919 0 : transDevIbverbsData.remoteDataAckNotifyDetails = remoteNotifyList[actualNotifyNum];
5920 0 : transDevIbverbsData.notifySize = notifySize_;
5921 0 : actualNotifyNum++;
5922 :
5923 0 : transDevIbverbsData.userLocalNotify.resize(channelRoce.qpsPerConnection);
5924 0 : transDevIbverbsData.userRemoteNotifyDetails.resize(channelRoce.qpsPerConnection);
5925 : // 当前多QP下每个QP会多申请一个DataNotify
5926 0 : u64 singleQpNotifySize = channelRoce.singleQPNotifyNum + static_cast<u32>(channelRoce.qpsPerConnection > 1);
5927 0 : for (u32 qpIndex = 0; qpIndex < channelRoce.qpsPerConnection; qpIndex++) {
5928 0 : transDevIbverbsData.userLocalNotify[qpIndex].resize(singleQpNotifySize, nullptr);
5929 0 : transDevIbverbsData.userRemoteNotifyDetails[qpIndex].resize(singleQpNotifySize);
5930 0 : for (u32 i = 0, idx = actualNotifyNum + singleQpNotifySize * qpIndex; i < singleQpNotifySize; ++idx, ++i) {
5931 0 : std::shared_ptr<LocalNotify> locNotify = std::make_shared<LocalNotify>();
5932 0 : CHK_RET(InitAndVerifySingleSignal(localNotifyList[idx], locNotify));
5933 0 : transDevIbverbsData.userLocalNotify[qpIndex][i] = locNotify;
5934 0 : transDevIbverbsData.userRemoteNotifyDetails[qpIndex][i] = remoteNotifyList[idx];
5935 0 : }
5936 : }
5937 0 : roceNotifyNum = channelRoce.singleQPNotifyNum;
5938 0 : HCCL_DEBUG("[%s]get roceNotify success, roceNotifyNum[%u]", __func__, roceNotifyNum);
5939 0 : return HCCL_SUCCESS;
5940 0 : }
5941 :
5942 0 : HcclResult HcclCommAicpu::NotifyAlloc(NotifyMgrAicpuParam* param)
5943 : {
5944 0 : u32 notifyNum = param->notifyNum;
5945 0 : std::string notifysStr = std::string(param->notifyParam, NOTIFY_UNIQUE_ID_MAX_SIZE);
5946 0 : std::string hcomId(param->hcomId);
5947 0 : size_t notifySize = notifys_.size();
5948 0 : HCCL_INFO(
5949 : "[HcclCommAicpu][%s] comm identifier[%s], alloc notifys num[%u] begin, before notifySize[%u]", __func__,
5950 : hcomId.c_str(), notifyNum, notifySize);
5951 0 : if (UNLIKELY(HcclCheckLogLevel(HCCL_LOG_INFO))) {
5952 0 : std::ostringstream oss;
5953 0 : oss << "notifyParam" << " raw bytes: ";
5954 0 : for (u32 i = 0; i < NOTIFY_UNIQUE_ID_MAX_SIZE; ++i) {
5955 0 : oss << std::hex << std::setw(2) << std::setfill('0')
5956 0 : << static_cast<unsigned int>(static_cast<unsigned char>(param->notifyParam[i])) << " ";
5957 : }
5958 0 : HCCL_INFO("[HcclCommAicpu][%s] %s", __func__, oss.str().c_str());
5959 0 : }
5960 0 : HcclResult ret = NotifyManager::ParseBinNotifys(notifysStr, notifys_);
5961 0 : if (ret != HCCL_SUCCESS) {
5962 0 : HCCL_ERROR(
5963 : "[HcclCommAicpu][%s] comm identifier[%s], alloc notifys num[%u] failed %u", __func__, hcomId.c_str(),
5964 : notifyNum, ret);
5965 0 : return ret;
5966 : }
5967 0 : HCCL_INFO(
5968 : "[HcclCommAicpu][%s] comm identifier[%s], alloc notifys num[%u] end, after notifySize[%u]", __func__,
5969 : hcomId.c_str(), notifyNum, notifys_.size());
5970 0 : NotifyHandle* notifyArray = static_cast<NotifyHandle*>(param->deviceHandle);
5971 0 : CHK_PTR_NULL(notifyArray);
5972 : // 空指针校验
5973 0 : for (size_t i = 0; i < notifyNum; ++i) {
5974 0 : notifyArray[i] = reinterpret_cast<NotifyHandle>(notifys_[i + notifySize].get()); // 拷贝裸指针
5975 0 : HCCL_INFO("[HcclCommAicpu][%s] notifyArray[%u] = [%lu]", __func__, i + notifySize, notifyArray[i]);
5976 : }
5977 :
5978 0 : HCCL_INFO(
5979 : "[HcclCommAicpu][%s] comm identifier[%s], alloc notifys num[%u] success", __func__, hcomId.c_str(), notifyNum);
5980 0 : return HCCL_SUCCESS;
5981 0 : }
5982 :
5983 0 : HcclResult HcclCommAicpu::NotifyFree(NotifyMgrAicpuParam* param)
5984 : {
5985 0 : u32 notifyNum = param->notifyNum;
5986 0 : NotifyHandle* notifyArray = static_cast<NotifyHandle*>(param->deviceHandle);
5987 0 : std::string hcomId(param->hcomId);
5988 : // 空指针校验
5989 0 : CHK_PTR_NULL(notifyArray);
5990 0 : for (size_t i = 0; i < notifyNum; ++i) {
5991 0 : LocalNotify* notify = reinterpret_cast<LocalNotify*>(notifyArray[i]);
5992 0 : HCCL_INFO("[HcclCommAicpu][%s] notifyArray[%u]=[%lu]", __func__, i, notifyArray[i]);
5993 0 : auto it = std::find_if(notifys_.begin(), notifys_.end(), [notify](const std::unique_ptr<LocalNotify>& ptr) {
5994 0 : return ptr.get() == notify;
5995 : });
5996 0 : if (it != notifys_.end()) {
5997 0 : HCCL_INFO(
5998 : "[HcclCommAicpu][%s] comm identifier[%s], free notifys[%u] success", __func__, hcomId.c_str(),
5999 : notifyArray[i]);
6000 0 : notifys_.erase(it);
6001 : } else {
6002 0 : HCCL_RUN_WARNING("[HcclCommAicpu][%s] localNotify[%u] not found in notifys_", __func__, i);
6003 : }
6004 : }
6005 :
6006 0 : HCCL_INFO(
6007 : "[HcclCommAicpu][%s] comm identifier[%s], free notifys num[%u] success", __func__, hcomId.c_str(), notifyNum);
6008 0 : return HCCL_SUCCESS;
6009 0 : }
6010 :
6011 0 : HcclResult HcclCommAicpu::RegisterOpInfo(void* opInfo, u32 size)
6012 : {
6013 0 : CHK_RET(taskExecption_.RegisterOpInfo(opInfo, size));
6014 0 : u32 opIdx = taskExecption_.GetOpRingBufferIdx();
6015 0 : CHK_RET(SetDispatcherCtxOpIdx(opIdx));
6016 0 : HCCL_INFO("%s success, group[%s], opRingBufferId[%u]", __func__, identifier_.c_str(), opIdx);
6017 0 : return HCCL_SUCCESS;
6018 : }
6019 :
6020 0 : HcclResult HcclCommAicpu::RegOpTaskException(HcommGetOpInfoCallback callback)
6021 : {
6022 0 : CHK_RET(taskExecption_.RegisterOpInfoCallback(callback));
6023 0 : return HCCL_SUCCESS;
6024 : }
6025 :
6026 0 : HcclResult HcclCommAicpu::SetDispatcherCtxOnThread()
6027 : {
6028 : // 设置 DispatcherCtx 到线程变量
6029 0 : CHK_RET(SetDispatcherCtx(dispatcherCtx_));
6030 0 : return HCCL_SUCCESS;
6031 : }
6032 : } // namespace hccl
|
