Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include <iomanip>
12 : #include "aicpu_communicator.h"
13 : #include "dispatcher.h"
14 : #include "aicpu_hccl_process.h"
15 : #include "coll_alg_exec_registry.h"
16 : #include "coll_all_to_all_executor.h"
17 : #include "common/aicpu_hccl_common.h"
18 : #include "executor_tracer.h"
19 : #include "profiling_manager_device.h"
20 : #include "utils/aicpu_hdc_utils.h"
21 : #include "comm_engine_utils.h"
22 : #include "utils/hccl_aicpu_utils.h"
23 : #include "framework/aicpu_hdc.h"
24 : #include "common/aicpu_sqe_context.h"
25 : #include "coll_batch_send_recv_retry_executor.h"
26 : #include "log_control.h"
27 : #include "aicpu_hccl_sqcq.h"
28 : #include "aicpu_hccl_sqcqv1.h"
29 : #include "sal_pub.h"
30 : #include "externalinput_pub.h"
31 : #include "env_config.h"
32 : #include "config_log.h"
33 : #include "aicpu_one_side_service.h"
34 : #include "notify_manager.h"
35 : #include "dispatcher_aicpu.h"
36 : #include "dlprof_function.h"
37 : #include "profiling_command_handle.h"
38 : #include "dispatcher_ctx.h"
39 : #include "aicpu_res_package_helper.h"
40 : #include "aicpu_symmetric_memory.h"
41 :
42 : namespace hccl {
43 : constexpr u32 IPC_SIGNAL_MODULUS = 2;
44 : constexpr u32 RDMA_SIGNAL_MODULUS = 3;
45 : constexpr u32 KEY_VALUE_TO_VECTOR_MODULUS = 2;
46 :
47 : constexpr u32 BSR_RETRY_SEND_STREAM_INDEX = 0;
48 : constexpr u32 BSR_RETRY_RECV_STREAM_INDEX = 1;
49 : constexpr u32 BSR_RETRY_STREAM_NUM = 2;
50 : constexpr u32 MAX_REPORT_STATUS = 100U; // reportStatus的最大缓存数量
51 : constexpr u32 INPUT = 0;
52 : constexpr u32 OUTPUT = 1;
53 : constexpr u32 AICPU_RETRY_LINKROCE_DEFAULT = 0;
54 : constexpr u32 AICPU_RETRY_LINKROCE_BACKUP = 1;
55 :
56 : constexpr u32 BSR_RETRY_SENDRECV_PAIR_NUM_MAX = 2;
57 : constexpr u32 BSR_RETRY_SENDRECV_PAIR_INDEX_0 = 0;
58 : constexpr u32 BSR_RETRY_SENDRECV_PAIR_INDEX_1 = 1;
59 :
60 : constexpr u32 NOTIFY_SIZE_FOUR = 4;
61 : constexpr u32 NOTIFY_SIZE_EIGHT = 8;
62 :
63 : bool HcclCommAicpu::errMessageReport_ = true;
64 :
65 : #define HCCL_RETRY_CHK_RET_AND_TRANS_FSM(result__, exeLog__, error__, state__) \
66 : do { \
67 : if (UNLIKELY((result__) != HCCL_SUCCESS)) { \
68 : exeLog__; \
69 : errorCode = (error__); \
70 : fsmState = (state__); \
71 : return (result__); \
72 : } \
73 : } while (0)
74 :
75 385 : HcclCommAicpu::HcclCommAicpu() { HCCL_RUN_INFO("Construct HcclCommAicpu complete."); }
76 :
77 306 : HcclCommAicpu::~HcclCommAicpu()
78 : {
79 34 : if (UtraceInfo_ != nullptr) {
80 11 : UtraceInfo_->DeInit();
81 11 : UtraceInfo_ = nullptr;
82 : }
83 34 : if (dispatcher_ != nullptr) {
84 11 : HcclDispatcherDestroy(dispatcher_);
85 11 : dispatcher_ = nullptr;
86 : }
87 34 : if (dispatcherCtx_ != nullptr) {
88 0 : DestroyDispatcherCtx(dispatcherCtx_, identifier_.c_str());
89 0 : HCCL_DEBUG("[%s] destroy dispatcherCtx[%p] group[%s] success!", __func__, dispatcherCtx_, identifier_.c_str());
90 0 : dispatcherCtx_ = nullptr;
91 : }
92 :
93 34 : commPlaneVector_.clear();
94 34 : isBridgeVector_.clear();
95 34 : indOpCommInitialized_ = false;
96 34 : initialized_ = false;
97 :
98 34 : HCCL_RUN_INFO("Destruct HcclCommAicpu group[%s] success!", identifier_.c_str());
99 374 : }
100 :
101 13 : HcclResult HcclCommAicpu::Init(const HcclOpResParam* commParam, bool isCustom)
102 : {
103 13 : if (initialized_) {
104 2 : HCCL_RUN_INFO("[%s][Init]Group[%s] already initialized, skip reinit", __func__, identifier_.c_str());
105 2 : return HCCL_SUCCESS;
106 : }
107 :
108 11 : CHK_PTR_NULL(commParam);
109 11 : identifier_ = commParam->hcomId;
110 11 : isCustom_ = isCustom;
111 11 : HCCL_RUN_INFO(
112 : "[HcclCommAicpu][Init]Entry-Init group[%s], rankSize[%u], isCustom[%d].", identifier_.c_str(),
113 : commParam->rankSize, isCustom_);
114 11 : CHK_RET(aicpuShareData_.Init(commParam->aicpuCustomParamAddr, commParam->aicpuCustomParamSize));
115 11 : CHK_RET(SetHrtWorkMode(commParam));
116 11 : CHK_RET(SetHrtDeviceSatMode(commParam));
117 11 : CHK_RET(InitConfigInfo(commParam));
118 11 : CHK_RET(InitCclbuffer(commParam));
119 11 : CHK_RET(InitTopoInfo(commParam));
120 11 : CHK_RET(InitOpNotifyObj(commParam));
121 11 : CHK_RET(HcclDispatcherAicpuInit(&dispatcher_, devId_, commParam->hcclSdmaQos, DispatcherType::DISPATCHER_AICPU));
122 11 : CHK_RET(RegisterProfilingCallback());
123 11 : CHK_RET(InitLocalNotifyObj(commParam));
124 11 : CHK_RET(InitMainStreamObj(commParam));
125 11 : CHK_RET(InitSlaveStreamObjs(commParam));
126 11 : CHK_RET(InitOrderStreamObj(commParam));
127 11 : CHK_RET(InitLocalTagRes(commParam->localRes.nextTagRes));
128 11 : CHK_RET(InitTimeOutConfig(commParam));
129 11 : CHK_RET(InitHostDeviceLock(commParam));
130 11 : CHK_RET(InitTopoMatcher());
131 11 : CHK_RET(InitOpRetry(commParam));
132 11 : CHK_RET(RegisterDispatcherCallback());
133 11 : CHK_RET(InitTinyMem(commParam));
134 11 : CHK_RET(InitProfResource());
135 11 : CHK_RET(InitZeroCopyExchanger(commParam));
136 11 : CHK_RET(InitOpCounter(commParam->opCounterInfo));
137 11 : CHK_RET(InitUtraceInfo(commParam));
138 11 : CHK_RET(aicpuCacheManager_.InitOpUnfoldCache());
139 11 : CHK_RET(RegisterProfCallBack());
140 11 : InitCommInfoStatus(true);
141 11 : SetCommInfoStreamStatus(true);
142 :
143 11 : initialized_ = true;
144 :
145 11 : HCCL_RUN_INFO("[HcclCommAicpu][Init] group[%s] success!", identifier_.c_str());
146 11 : return HCCL_SUCCESS;
147 : }
148 :
149 11 : HcclResult HcclCommAicpu::InitUtraceInfo(const HcclOpResParam* commParam)
150 : {
151 11 : u32 hostpid = 0;
152 11 : u32 cpType = DEVDRV_PROCESS_CPTYPE_MAX;
153 11 : CHK_RET(HrtHalDrvQueryProcessHostPid(getpid(), nullptr, nullptr, &hostpid, &cpType));
154 :
155 : HcclTraceInfo::UtraceAttr utraceAttr;
156 11 : utraceAttr.utraceStatusFlag = commParam->utraceStatusFlag;
157 11 : utraceAttr.deviceid = GetDevId();
158 11 : utraceAttr.pid = hostpid;
159 11 : UtraceInfo_.reset(new (std::nothrow) HcclTraceInfo(utraceAttr));
160 11 : CHK_PTR_NULL(UtraceInfo_);
161 :
162 : /* 申请trace资源信息 */
163 11 : std::string logInfo = "HCCL_";
164 11 : logInfo.append(std::to_string(SalGetTid()));
165 11 : logInfo.append("_");
166 11 : logInfo.append(std::to_string(GetDevId()));
167 11 : CHK_RET(UtraceInfo_->Init(logInfo));
168 11 : return HCCL_SUCCESS;
169 11 : }
170 :
171 11 : HcclResult HcclCommAicpu::InitProfResource()
172 : {
173 11 : groupHashId_ = dfx::ProfilingManager::GetProfHashId(identifier_.c_str(), identifier_.length());
174 11 : HCCL_RUN_INFO("[Init][ProfResource]group[%s], groupHashId_[%llu].", identifier_.c_str(), groupHashId_);
175 :
176 11 : dfx::ProfCommInfo profInfo{groupHashId_, topoInfo_.userRankSize, topoInfo_.userRank};
177 11 : CHK_RET(dfx::ProfilingManager::AddProfInfoByStreamId(mainStream_.id(), identifier_, profInfo));
178 451 : for (auto& slaveStream : slaveStreams_) {
179 440 : CHK_RET(dfx::ProfilingManager::AddProfInfoByStreamId(slaveStream.id(), identifier_, profInfo));
180 : }
181 11 : dfx::ProfilingExtendInfoHelper::InitProfItemId();
182 11 : return HCCL_SUCCESS;
183 : }
184 :
185 462 : HcclResult HcclCommAicpu::StreamRestore(u32 streamId)
186 : {
187 462 : HcclResult ret = hrtHalResourceIdRestore(devId_, 0, DRV_STREAM_ID, streamId, 0);
188 : // custom进程需要恢复stream资源, custom进程调用失败直接报错，aicpu进程调用失败做兼容性处理
189 462 : if (ret == HCCL_E_NOT_SUPPORT) {
190 0 : CHK_PRT_RET(
191 : isCustom_,
192 : HCCL_ERROR(
193 : "%s hrtHalResourceIdRestore fail, drv not support, custom[%d], ret[%d]", __func__, isCustom_, ret),
194 : HCCL_E_DRV);
195 462 : } else if (ret != HCCL_SUCCESS) {
196 0 : HCCL_ERROR("%s hrtHalResourceIdRestore fail, ret[%d]", __func__, ret);
197 0 : return HCCL_E_DRV;
198 : }
199 462 : return HCCL_SUCCESS;
200 : }
201 :
202 11 : HcclResult HcclCommAicpu::SetHrtWorkMode(const HcclOpResParam* commParam)
203 : {
204 11 : CHK_RET(hrtSetWorkModeAicpu(true));
205 11 : CHK_RET(hrtSetlocalDevice(commParam->topoInfo.deviceLogicId));
206 11 : CHK_RET(hrtSetlocalDeviceType(static_cast<DevType>(commParam->topoInfo.deviceType)));
207 11 : return HCCL_SUCCESS;
208 : }
209 :
210 11 : HcclResult HcclCommAicpu::SetHrtDeviceSatMode(const HcclOpResParam* commParam)
211 : {
212 11 : CHK_RET(hrtSetLocalDeviceSatMode(commParam->config.floatOverflowMode));
213 11 : HCCL_RUN_INFO(
214 : "[HcclCommAicpu][Init]SetHrtDeviceSatMode[%d]", static_cast<u32>(commParam->config.floatOverflowMode));
215 11 : return HCCL_SUCCESS;
216 : }
217 :
218 11 : HcclResult HcclCommAicpu::InitTopoMatcher()
219 : {
220 11 : externalEnable_.enableFfts = 1; // FFTS+在算法模块不会使用多线程，固定使能
221 11 : externalEnable_.deterministic = deterministic_;
222 11 : externalEnable_.intraRoceSwitch = 0;
223 11 : externalEnable_.dumpDebug = dumpDebug_;
224 11 : externalEnable_.interHccsDisable = interHccsDisable_;
225 :
226 11 : topoMatcher_.reset((new (std::nothrow) TopoMatcher(
227 22 : commPlaneVector_, isBridgeVector_, topoInfo_, algoInfo_, externalEnable_, serverAndsuperPodToRank_)));
228 11 : CHK_SMART_PTR_NULL(topoMatcher_);
229 11 : HCCL_RUN_INFO(
230 : "[HcclCommAicpu][InitTopoMatcher]topo matcher init success. group[%s] deterministic:%u, "
231 : "dumpDebug:%u, interHccsDisable:%u",
232 : identifier_.c_str(), deterministic_, dumpDebug_, interHccsDisable_);
233 11 : return HCCL_SUCCESS;
234 : }
235 :
236 11 : HcclResult HcclCommAicpu::InitOpRetry(const HcclOpResParam* commParam)
237 : {
238 11 : CHK_PTR_NULL(commParam);
239 11 : retryEnable_ = (commParam->config.retryEnable == 1) ? true : false;
240 11 : retryHoldTime_ = commParam->config.retryHoldTime;
241 11 : retryIntervalTime_ = commParam->config.retryIntervalTime;
242 11 : HCCL_RUN_INFO(
243 : "[InitOpRetry]retryEnable[%d], retryHoldTime[%u ms], retryIntervalTime[%u ms]", retryEnable_, retryHoldTime_,
244 : retryIntervalTime_);
245 :
246 11 : if (commParam->kfcControlTransferH2DParams.buffLen != 0 && kfcControlTransferH2D_ == nullptr) {
247 0 : EXCEPTION_CATCH((kfcControlTransferH2D_ = std::make_shared<hccl::HDCommunicate>()), return HCCL_E_PTR);
248 0 : CHK_SMART_PTR_NULL(kfcControlTransferH2D_);
249 0 : CHK_RET(kfcControlTransferH2D_->InitDevice(commParam->kfcControlTransferH2DParams));
250 : }
251 11 : if (commParam->kfcStatusTransferD2HParams.buffLen != 0 && kfcStatusTransferD2H_ == nullptr) {
252 0 : EXCEPTION_CATCH((kfcStatusTransferD2H_ = std::make_shared<hccl::HDCommunicate>()), return HCCL_E_PTR);
253 0 : CHK_SMART_PTR_NULL(kfcStatusTransferD2H_);
254 0 : CHK_RET(kfcStatusTransferD2H_->InitDevice(commParam->kfcStatusTransferD2HParams));
255 : }
256 11 : return HCCL_SUCCESS;
257 : }
258 :
259 11 : HcclResult HcclCommAicpu::InitZeroCopyExchanger(const HcclOpResParam* commParam)
260 : {
261 11 : CHK_PTR_NULL(commParam);
262 0 : auto nSecStopFunc = [this]() -> bool {
263 : // 检查到OP状态不Ok则认为需要终止
264 0 : auto ret = this->CheckOpExecStatus();
265 0 : if (ret == HCCL_SUCCESS) {
266 0 : return false;
267 0 : } else if (ret == HCCL_E_SUSPENDING) {
268 : // NS快恢场景，需要提前终止
269 0 : HcclOpExecFSM fsmState = HcclOpExecFSM::HCCL_OP_EXEC_FSM_INIT;
270 0 : KfcError errorCode = KfcError::kNone;
271 0 : SetCommRecoveryFlag(true);
272 0 : UpdateOpExecStatus(fsmState, KfcStatus::kStoplaunch, errorCode, 0);
273 0 : HCCL_RUN_INFO("[HcclCommAicpu][nSecStopFunc] need stop launch");
274 0 : return true;
275 : } else {
276 0 : return true;
277 : }
278 11 : };
279 :
280 11 : u32 timeoutSec = commParam->config.notifyWaitTime;
281 11 : HCCL_INFO("[HcclCommAicpu][InitZeroCopyExchanger] set timeout is [%u s]", timeoutSec);
282 :
283 11 : EXCEPTION_CATCH(
284 : (ZeroCopyExchanger_ = std::make_shared<hccl::AicpuZeroCopyExchanger>(
285 : commParam->localUsrRankId, commParam->rankSize, commParam, nSecStopFunc, timeoutSec,
286 : topoInfo_.deviceNumPerAggregation, taskMonitorInterval_)),
287 : return HCCL_E_PTR);
288 :
289 : // 通信域第一次初始化时，如果IPC内存有不为空的则认为是使能该特性
290 11 : isZeroCopy_ = false;
291 363 : for (u32 i = 0; i < AICPU_ZERO_COPY_MAX_DEVICE_NUM_A3; ++i) {
292 352 : if (commParam->zeroCopyIpcPtrs[i] != 0) {
293 0 : isZeroCopy_ = true;
294 0 : break;
295 : }
296 : }
297 :
298 11 : return HCCL_SUCCESS;
299 : }
300 :
301 11 : HcclResult HcclCommAicpu::InitOpCounter(const OpCounterInfo& opCounterInfo)
302 : {
303 11 : if (opCounterInfo.isEnableCounter && !retryEnable_
304 0 : && (opCounterInfo.headCountMem == 0 || opCounterInfo.tailCountMem == 0 || opCounterInfo.memSize == 0)) {
305 0 : HCCL_ERROR("[HcclCommAicpu][InitOpCounter] headCountMem or tailCountMem or memSize is null");
306 0 : return HCCL_E_PARA;
307 : }
308 11 : opCounterInfo_ = opCounterInfo;
309 11 : return HCCL_SUCCESS;
310 : }
311 :
312 1 : void HcclCommAicpu::SetZeroCopyEnable(bool enable) { isZeroCopy_ = enable; }
313 :
314 1 : void HcclCommAicpu::SetSymmetricMemoryEnable(bool enable)
315 : {
316 1 : HCCL_INFO("[HcclCommAicpu::SetSymmetricMemoryEnable] enable[%d]", enable);
317 1 : isSymmetricMemory_ = enable;
318 1 : aicpuCacheManager_.SetSymmetricMemoryEnable(enable);
319 1 : }
320 :
321 0 : HcclResult HcclCommAicpu::PrepareZeroCopyExchanger(
322 : const std::string& newTag, OpParam& opParam, AlgResourceResponse* algResResponse)
323 : {
324 0 : return ZeroCopyExchanger_->ExchangeAddress(newTag, opParam.inputPtr, opParam.outputPtr, algResResponse);
325 : }
326 :
327 11 : HcclResult HcclCommAicpu::RegisterProfilingCallback()
328 : {
329 11 : (void)RegisterLoadTaskCallBack(dispatcher_, nullptr, dfx::TaskProfilingCallBack);
330 :
331 11 : return HCCL_SUCCESS;
332 : }
333 :
334 11 : HcclResult HcclCommAicpu::RegisterDispatcherCallback()
335 : {
336 0 : auto checkOpExecStatusCallback = [this]() {
337 0 : return this->CheckOpExecStatusCallback();
338 11 : };
339 :
340 11 : return HcclSetOpExecStatusCallback(dispatcher_, checkOpExecStatusCallback);
341 : }
342 :
343 11 : HcclResult HcclCommAicpu::RegisterProfCallBack()
344 : {
345 11 : if (MsprofRegisterCallback != nullptr) {
346 11 : HCCL_INFO("RegisterProfCallBack not null");
347 11 : int32_t ret = MsprofRegisterCallback(AICPU, &DeviceCommandHandle);
348 11 : CHK_PRT_RET((ret != 0), HCCL_ERROR("[%s] failed. ret = [%d]", __func__, ret), HCCL_E_PARA);
349 : } else {
350 0 : HCCL_INFO("RegisterProfCallBack is null");
351 : }
352 11 : return HCCL_SUCCESS;
353 : }
354 :
355 0 : HcclResult HcclCommAicpu::GetSuspendingFlag(HcclComSuspendingFlag& flag)
356 : {
357 0 : CHK_RET(AicpuHdcUtils::GetSuspendingStatus(kfcControlTransferH2D_, flag));
358 0 : return HCCL_SUCCESS;
359 : }
360 :
361 57 : HcclResult HcclCommAicpu::BackGroundGetCmd(KfcCommand& cmd)
362 : {
363 57 : CHK_RET(aicpuHdc_.GetOpExecCtrlCmd(kfcControlTransferH2D_, cmd));
364 54 : return HCCL_SUCCESS;
365 : }
366 :
367 2 : HcclResult HcclCommAicpu::BackGroundSetStatus(KfcStatus status)
368 : {
369 2 : CHK_RET(aicpuHdc_.SetOpExecStatus(kfcStatusTransferD2H_, status, KfcError::kNone, 0));
370 2 : return HCCL_SUCCESS;
371 : }
372 :
373 0 : HcclResult HcclCommAicpu::SaveTraceInfo(std::string& logInfo)
374 : {
375 0 : CHK_PTR_NULL(UtraceInfo_);
376 0 : CHK_RET(UtraceInfo_->SaveTraceInfo(logInfo, AtraceOption::Opbasekey));
377 0 : return HCCL_SUCCESS;
378 : }
379 :
380 0 : HcclResult HcclCommAicpu::FlushUtraceInfo()
381 : {
382 0 : if (GetCommInfoStatus()) {
383 0 : CHK_RET(UtraceInfo_->Flush());
384 : }
385 0 : return HCCL_SUCCESS;
386 : }
387 :
388 0 : std::string HcclCommAicpu::GetExcuteOp()
389 : {
390 0 : std::stringstream ss;
391 0 : ss << "tag: " << excuteOpId_.tag << ", ";
392 0 : ss << "newTag: " << excuteOpId_.newTag << ", ";
393 0 : ss << "index: " << excuteOpId_.index;
394 0 : return ss.str();
395 0 : }
396 :
397 11 : void HcclCommAicpu::InitCommInfoStatus(bool commInfo) { commOpenStatus = commInfo; }
398 :
399 11 : HcclResult HcclCommAicpu::InitTinyMem(const HcclOpResParam* commParam)
400 : {
401 11 : CHK_PTR_NULL(commParam);
402 11 : auto tinyMemPtr = reinterpret_cast<void*>(commParam->tinyMem);
403 11 : tinySendRecvMem_ = DeviceMem::create(tinyMemPtr, commParam->tinyMemSize);
404 :
405 11 : return HCCL_SUCCESS;
406 : }
407 :
408 11 : HcclResult HcclCommAicpu::InitTimeOutConfig(const HcclOpResParam* commParam)
409 : {
410 11 : CHK_PTR_NULL(commParam);
411 11 : CHK_RET(HcclSetSqeTimeOut(dispatcher_, commParam->config.notifyWaitTime));
412 11 : linkTimeOut_ = commParam->config.linkTimeOut;
413 11 : return HCCL_SUCCESS;
414 : }
415 :
416 11 : HcclResult HcclCommAicpu::InitHostDeviceLock(const HcclOpResParam* commParam)
417 : {
418 11 : CHK_PTR_NULL(commParam);
419 11 : hostDeviceLock_.reset(new (std::nothrow) PetersonLock(
420 11 : reinterpret_cast<void*>(commParam->lockAddr), PetersonLock::DEFAULT_LOCK_TIMEOUT_SEC));
421 11 : CHK_SMART_PTR_NULL(hostDeviceLock_);
422 11 : CHK_RET(hostDeviceLock_->Init());
423 :
424 11 : return HCCL_SUCCESS;
425 : }
426 :
427 0 : HcclResult HcclCommAicpu::UpdateNotifyWaitTimeOut(SyncMode syncMode, u64 notifyWaitTime)
428 : {
429 0 : sqeWaitTimeOut_ = (notifyWaitTime == 0) ? notifyWaitTime : (notifyWaitTime + AICPU_SQE_TIMEOUT_INC);
430 0 : if (syncMode == SyncMode::UNLIMITED_TIMEWAITSYNCMODE) {
431 0 : CHK_RET(HcclSetSqeTimeOut(dispatcher_, GetNotifyMaxWaitTime()));
432 : }
433 0 : return HcclSetSqFullWaitTimeOut(dispatcher_, notifyWaitTime);
434 : }
435 :
436 2 : void HcclCommAicpu::PrepareOpRetryHandler(
437 : u8 inplaceSupportRetry, u8 retryEnable, u8 inPlaceSupportRetryStatus, u8 isInplacePreSync, u8 isPostSync)
438 : {
439 2 : algOpContext_.opRetryHandler.inplaceSupportRetry = static_cast<bool>(inplaceSupportRetry);
440 2 : algOpContext_.opRetryHandler.retryEnable = static_cast<bool>(retryEnable);
441 : algOpContext_.opRetryHandler.inPlaceSupportRetryStatus
442 2 : = static_cast<InplaceSupportRetryStatus>(inPlaceSupportRetryStatus);
443 2 : algOpContext_.opRetryHandler.isInplacePreSync = static_cast<bool>(isInplacePreSync);
444 2 : algOpContext_.opRetryHandler.isPostSync = static_cast<bool>(isPostSync);
445 2 : HCCL_INFO(
446 : "[HcclCommAicpu][PrepareOpRetryHandler] inplaceSupportRetry %d, retryEnable %d, "
447 : "inPlaceSupportRetryStatus %d, isInplacePreSync %d, isPostSync %d.",
448 : algOpContext_.opRetryHandler.inplaceSupportRetry, algOpContext_.opRetryHandler.retryEnable,
449 : algOpContext_.opRetryHandler.inPlaceSupportRetryStatus, algOpContext_.opRetryHandler.isInplacePreSync,
450 : algOpContext_.opRetryHandler.isPostSync);
451 2 : }
452 :
453 0 : HcclResult HcclCommAicpu::UpdateOpRingBufferIdx()
454 : {
455 0 : return HcclSetOpRingBufferIdx(dispatcher_, aicpuShareData_.GetOpRingBufferIdx());
456 : }
457 :
458 0 : HcclResult HcclCommAicpu::InvokeKfcHandler(AicpuKfcHandlerType type, const std::vector<u64> args)
459 : {
460 0 : CHK_PRT_RET(
461 : static_cast<size_t>(type) >= static_cast<size_t>(AicpuKfcHandlerType::kMax),
462 : HCCL_ERROR("Device mode %u, handler type %u.", static_cast<u32>(isDeviceMode_), static_cast<size_t>(type)),
463 : HCCL_E_INTERNAL);
464 0 : const auto handler = kfcHandlers_[static_cast<size_t>(type)];
465 0 : if (handler == nullptr) {
466 0 : return HCCL_SUCCESS;
467 : }
468 0 : return handler(args);
469 0 : }
470 :
471 0 : HcclResult HcclCommAicpu::NotifyPost(void)
472 : {
473 0 : if (isDeviceMode_) {
474 0 : return InvokeKfcHandler(
475 : AicpuKfcHandlerType::kNotifyRecord,
476 0 : {rpc_, reinterpret_cast<u64>(dispatcher_), reinterpret_cast<u64>(&mainStream_)});
477 : } else {
478 0 : CHK_RET(LocalNotify::Post(mainStream_, dispatcher_, opNotifies_[1]));
479 0 : HcclSqeContext* sqeContext = mainStream_.GetSqeContextPtr();
480 0 : SqeRingBuffer* sqeContextBuffer = &(sqeContext->buffer);
481 0 : return dfx::ProfilingManager::ReportMainStreamTask(mainStream_, sqeContextBuffer->tailSqeTaskId - 1, TAIL_TASK);
482 : }
483 : }
484 :
485 0 : HcclResult HcclCommAicpu::NotifyWait(void)
486 : {
487 0 : if (isDeviceMode_) {
488 0 : return InvokeKfcHandler(
489 : AicpuKfcHandlerType::kNotifyWait,
490 0 : {rpc_, reinterpret_cast<u64>(dispatcher_), reinterpret_cast<u64>(&mainStream_)});
491 : } else {
492 0 : HcclSqeContext* sqeContext = mainStream_.GetSqeContextPtr();
493 0 : SqeRingBuffer* sqeContextBuffer = &(sqeContext->buffer);
494 0 : CHK_RET(dfx::ProfilingManager::ReportMainStreamTask(mainStream_, sqeContextBuffer->tailSqeTaskId, HEAD_TASK));
495 0 : return LocalNotify::Wait(mainStream_, dispatcher_, opNotifies_[0]);
496 : }
497 : }
498 :
499 : // 按照算子模式来Post对应的Notify
500 0 : HcclResult HcclCommAicpu::RecordHostOrder(const HcclOpResParam* commParam, const std::string& tag, u8 orderLaunchMode)
501 : {
502 0 : const u8 orderLaunchInvalidInHcom = 255;
503 0 : if (orderLaunchMode == orderLaunchInvalidInHcom) {
504 0 : HCCL_INFO("[%s] attachedStreams_[%d] is invalid in graph mode", __func__, orderStream_.id());
505 0 : return HCCL_SUCCESS;
506 : }
507 0 : if (orderNotifies_[orderLaunchMode] == nullptr) {
508 0 : std::shared_ptr<LocalNotify> notify;
509 0 : HcclSignalInfo* aicpuOrderNotify = reinterpret_cast<HcclSignalInfo*>(
510 0 : static_cast<u64>(commParam->aicpuOrderNotifyAddr) + (sizeof(HcclSignalInfo) * orderLaunchMode));
511 0 : HCCL_INFO(
512 : "[%s] attachedStreams_[%d] aicpuOrderNotify resId[%llu], addr[0x%llx], flag[%u], devId[%u], tsId[%u], "
513 : "rankId[%u]",
514 : __func__, orderStream_.id(), aicpuOrderNotify->resId, aicpuOrderNotify->addr, aicpuOrderNotify->flag,
515 : aicpuOrderNotify->devId, aicpuOrderNotify->tsId, aicpuOrderNotify->rankId);
516 0 : HcclResult ret = InitAndVerifySingleSignal(*aicpuOrderNotify, notify);
517 0 : CHK_PRT_RET(
518 : ret != HCCL_SUCCESS,
519 : HCCL_ERROR(
520 : "[%s] check localRes notify failed, resId[%u], group[%s]", __func__, aicpuOrderNotify->resId,
521 : identifier_.c_str()),
522 : ret);
523 0 : orderNotifies_[orderLaunchMode] = notify;
524 0 : HCCL_INFO("%s success, group[%s], resId[%llu]", __func__, identifier_.c_str(), aicpuOrderNotify->resId);
525 0 : }
526 :
527 0 : HCCL_INFO(
528 : "%s group[%s] tag[%s] isDeviceMode[%d] orderLaunchMode[%u] mode[%d] streamId[%d] notifyId[%u]", __func__,
529 : identifier_.c_str(), tag.c_str(), isDeviceMode_, orderLaunchMode, GetWorkflowMode(), orderStream_.id(),
530 : orderNotifies_[orderLaunchMode]->notifyId_);
531 0 : CHK_RET(LocalNotify::Post(orderStream_, dispatcher_, orderNotifies_[orderLaunchMode]));
532 0 : CHK_RET(LaunchTask(dispatcher_, const_cast<Stream&>(orderStream_)));
533 0 : return HCCL_SUCCESS;
534 : }
535 :
536 462 : HcclResult HcclCommAicpu::GetStreamData(
537 : const HcclStreamInfo& streamInfo, HcclComStreamInfo& comStreamInfo, u32& sqHead, u32& sqTail)
538 : {
539 462 : comStreamInfo.sqId = streamInfo.sqIds;
540 462 : comStreamInfo.actualStreamId = streamInfo.streamIds;
541 462 : comStreamInfo.logicCqId = streamInfo.logicCqids;
542 462 : u64 sq_addr = 0;
543 462 : CHK_RET(QuerySqBaseAddr(devId_, streamInfo.sqIds, sq_addr));
544 462 : comStreamInfo.sqBaseAddr = reinterpret_cast<void*>(sq_addr);
545 462 : if (comStreamInfo.sqBaseAddr == nullptr) {
546 0 : HCCL_ERROR("[HcclCommAicpu][GetStreamData]sqe base addr ptr is null.");
547 0 : return HCCL_E_PARA;
548 : }
549 462 : CHK_RET(QuerySqStatusByType(devId_, streamInfo.sqIds, DRV_SQCQ_PROP_SQ_DEPTH, comStreamInfo.sqDepth));
550 462 : CHK_RET(QuerySqStatusByType(devId_, streamInfo.sqIds, DRV_SQCQ_PROP_SQ_TAIL, sqTail));
551 462 : CHK_RET(QuerySqStatusByType(devId_, streamInfo.sqIds, DRV_SQCQ_PROP_SQ_HEAD, sqHead));
552 462 : HCCL_DEBUG(
553 : "[HcclCommAicpu][GetStreamData] get stream data success, group[%s], streamId[%d], sqId[%d], "
554 : "logicCqId[%u], sqDepth[%u], sqHead[%u], sqTail[%u]",
555 : identifier_.c_str(), comStreamInfo.actualStreamId, comStreamInfo.sqId, comStreamInfo.logicCqId,
556 : comStreamInfo.sqDepth, sqHead, sqTail);
557 462 : return HCCL_SUCCESS;
558 : }
559 :
560 11 : HcclResult HcclCommAicpu::InitMainStreamObj(const HcclOpResParam* commParam)
561 : {
562 11 : CHK_RET(InitStreamObj(commParam->localRes.mainStreamParam, mainStream_));
563 11 : HCCL_INFO("%s success, group[%s], streamId[%d]", __func__, identifier_.c_str(), mainStream_.id());
564 11 : return HCCL_SUCCESS;
565 : }
566 :
567 11 : HcclResult HcclCommAicpu::InitSlaveStreamObjs(const HcclOpResParam* commParam)
568 : {
569 11 : if (commParam->localRes.streamNum > LOCAL_STREAM_MAX_NUM) {
570 0 : HCCL_ERROR(
571 : "[HcclCommAicpu][InitSlaveStreamObjs] local streams exceed max number, current numbers[%u], max "
572 : "numbers[%u], group[%s]",
573 : commParam->localRes.streamNum, LOCAL_STREAM_MAX_NUM, identifier_.c_str());
574 0 : return HCCL_E_PARA;
575 : }
576 451 : for (u32 i = 0; i < commParam->localRes.streamNum; i++) {
577 440 : if (streamToObj_.find(commParam->localRes.streamParam[i].streamInfo.sqIds) == streamToObj_.end()) {
578 440 : Stream stream;
579 440 : CHK_RET(InitStreamObj(commParam->localRes.streamParam[i], stream));
580 440 : slaveStreams_.emplace_back(stream);
581 440 : streamToObj_.insert(commParam->localRes.streamParam[i].streamInfo.sqIds);
582 440 : }
583 : }
584 11 : HCCL_DEBUG(
585 : "[HcclCommAicpu][InitSlaveStreamObjs] success, group[%s], slave stream numbers[%u]", identifier_.c_str(),
586 : commParam->localRes.streamNum);
587 11 : return HCCL_SUCCESS;
588 : }
589 :
590 11 : HcclResult HcclCommAicpu::InitOrderStreamObj(const HcclOpResParam* commParam)
591 : {
592 11 : CHK_RET(InitStreamObj(commParam->aicpuOrderStreamParam, orderStream_));
593 11 : HCCL_INFO("%s success, group[%s], streamId[%d]", __func__, identifier_.c_str(), orderStream_.id());
594 11 : return HCCL_SUCCESS;
595 : }
596 :
597 462 : HcclResult HcclCommAicpu::InitStreamObj(const HcclStreamParam& streamParam, Stream& stream)
598 : {
599 : u32 sqTail;
600 : u32 sqHead;
601 462 : HcclResult ret = HCCL_SUCCESS;
602 462 : ret = StreamRestore(streamParam.streamInfo.streamIds);
603 462 : CHK_PRT_RET(
604 : ret != HCCL_SUCCESS,
605 : HCCL_ERROR(
606 : "%s StreamId[%d] Restore error group[%s]", __func__, streamParam.streamInfo.streamIds, identifier_.c_str()),
607 : ret);
608 :
609 462 : HcclComStreamInfo comStreamInfo = {};
610 462 : ret = GetStreamData(streamParam.streamInfo, comStreamInfo, sqHead, sqTail);
611 462 : CHK_PRT_RET(ret != HCCL_SUCCESS, HCCL_ERROR("%s error group[%s]", __func__, identifier_.c_str()), ret);
612 462 : stream = Stream(comStreamInfo);
613 :
614 : // 初始化stream的sqeContext
615 462 : SqCqeContext* sqCqeContext = reinterpret_cast<SqCqeContext*>(streamParam.sqCqContextAddr);
616 462 : u64 sqCqContextSize = streamParam.sqCqContextSize;
617 462 : if (sqCqeContext == nullptr || sqCqContextSize != sizeof(SqCqeContext)) {
618 0 : HCCL_ERROR(
619 : "%s failed, sqCqeContext[%p] is null or size[%llu] does not match expected size[%llu], group[%s]", __func__,
620 : sqCqeContext, sqCqContextSize, sizeof(SqCqeContext), identifier_.c_str());
621 0 : return HCCL_E_PARA;
622 : }
623 462 : ret = stream.InitSqAndCqeContext(sqHead, sqTail, sqCqeContext);
624 462 : CHK_PRT_RET(
625 : ret != HCCL_SUCCESS, HCCL_ERROR("%s: InitSqAndCqeContext failed, group[%s]", __func__, identifier_.c_str()),
626 : ret);
627 462 : HCCL_INFO("%s success, group[%s], streamId[%d]", __func__, identifier_.c_str(), stream.id());
628 462 : return HCCL_SUCCESS;
629 : }
630 :
631 11 : HcclResult HcclCommAicpu::InitOpNotifyObj(const HcclOpResParam* commParam)
632 : {
633 11 : HcclResult ret = HCCL_SUCCESS;
634 33 : for (u32 i = 0; i < AICPU_OP_NOTIFY_MAX_NUM; i++) {
635 22 : ret = InitAndVerifySignal(commParam->localRes.aicpuOpNotify[i], opNotifies_);
636 22 : CHK_PRT_RET(
637 : ret != HCCL_SUCCESS,
638 : HCCL_ERROR(
639 : "[HcclCommAicpu][InitOpNotifyObj] check localRes op notify failed, resId[%u], group[%s]",
640 : commParam->localRes.localSignals[i].resId, identifier_.c_str()),
641 : ret);
642 : }
643 11 : u32 hostPid = 0;
644 11 : CHK_RET(HrtHalDrvQueryProcessHostPid(getpid(), &devId_, nullptr, &hostPid, nullptr));
645 11 : HCCL_INFO("[HcclCommAicpu][InitOpNotifyObj] success, group[%s]", identifier_.c_str());
646 11 : return HCCL_SUCCESS;
647 : }
648 :
649 11 : HcclResult HcclCommAicpu::InitLocalNotifyObj(const HcclOpResParam* commParam)
650 : {
651 11 : HcclResult ret = HCCL_SUCCESS;
652 :
653 11 : if (commParam->localRes.signalNum > LOCAL_NOTIFY_MAX_NUM) {
654 0 : HCCL_ERROR(
655 : "[HcclCommAicpu][InitLocalNotifyObj] local notify count exceeds max number, numbers[%u], max "
656 : "numbers[%u], group[%s]",
657 : commParam->localRes.signalNum, LOCAL_NOTIFY_MAX_NUM, identifier_.c_str());
658 0 : return HCCL_E_PARA;
659 : }
660 715 : for (u32 i = 0; i < commParam->localRes.signalNum; i++) {
661 704 : if (notifysToObj_.find(commParam->localRes.localSignals[i].resId) == notifysToObj_.end()) {
662 704 : ret = InitAndVerifySignal(commParam->localRes.localSignals[i], localNotifies_);
663 704 : CHK_PRT_RET(
664 : ret != HCCL_SUCCESS,
665 : HCCL_ERROR(
666 : "[HcclCommAicpu][InitLocalNotifyObj] check localRes prenotify failed, resId[%u]",
667 : commParam->localRes.localSignals[i].resId),
668 : ret);
669 704 : notifysToObj_.insert(commParam->localRes.localSignals[i].resId);
670 : }
671 : }
672 11 : HCCL_DEBUG(
673 : "[HcclCommAicpu][InitLocalNotifyObj] success, group[%s], signal numbers[%u]", identifier_.c_str(),
674 : commParam->localRes.signalNum);
675 11 : return HCCL_SUCCESS;
676 : }
677 :
678 22 : HcclResult HcclCommAicpu::ParseTlvToVector(
679 : u64 srcTlv, u64 srcTlvTotalLength, std::vector<std::vector<std::vector<u32>>>& vectorInfo)
680 : {
681 22 : u64 parseLength = 0;
682 22 : u8* srcTlvptr = reinterpret_cast<u8*>(srcTlv);
683 22 : if (srcTlvptr == nullptr) {
684 0 : HCCL_ERROR("[HcclCommAicpu][ParseTlvToVector]tlv ptr is null.");
685 0 : return HCCL_E_PARA;
686 : }
687 22 : u64 unPareseLength = srcTlvTotalLength;
688 143 : while (unPareseLength > 0) {
689 121 : CommonTlv* commonTlv = reinterpret_cast<CommonTlv*>(srcTlvptr + parseLength);
690 121 : if (unPareseLength <= (sizeof(LENGTH_TYPE) + sizeof(TAG_TYPE))) {
691 0 : HCCL_ERROR(
692 : "[HcclCommAicpu][ParseTlvToVector] Tlv length is error, tag[%s], totalLength[%lu], "
693 : "unParseLength[%lu], already parsed Length[%lu]",
694 : identifier_.c_str(), srcTlvTotalLength, unPareseLength, parseLength);
695 0 : return HCCL_E_PARA;
696 : }
697 121 : if (commonTlv->length > unPareseLength || commonTlv->length % sizeof(RANK_TYPE) != 0) {
698 0 : HCCL_ERROR(
699 : "[HcclCommAicpu][ParseTlvToVector] parse Tlv error, group[%s], total Length[%lu], tlvLength[%lu], "
700 : "unParsed Length[%lu], already parsed Length[%lu]",
701 : identifier_.c_str(), srcTlvTotalLength, commonTlv->length, unPareseLength, parseLength);
702 0 : return HCCL_E_PARA;
703 : }
704 121 : u16 level0 = (commonTlv->type & TOP_COMM_LEVEL0_LOCATION) >> TOP_COMM_LEVEL0_SHIFT;
705 121 : u16 level1 = (commonTlv->type & TOP_COMM_LEVEL1_LOCATION);
706 121 : u64 itemNum = (commonTlv->length - sizeof(TAG_TYPE) - sizeof(LENGTH_TYPE)) / sizeof(RANK_TYPE);
707 242 : std::vector<RANK_TYPE> values{&commonTlv->value, (&commonTlv->value) + itemNum};
708 121 : if (level0 >= vectorInfo.size()) {
709 99 : vectorInfo.resize(level0 + 1);
710 : }
711 121 : if (level1 >= vectorInfo[level0].size()) {
712 121 : vectorInfo[level0].resize(level1 + 1);
713 : }
714 121 : vectorInfo[level0][level1] = std::move(values);
715 121 : parseLength += commonTlv->length;
716 121 : unPareseLength -= commonTlv->length;
717 121 : HCCL_DEBUG(
718 : "[HcclCommAicpu][ParseTlvToVector] parse Tlv group[%s], level0[%u], level1[%u], total Length[%lu], "
719 : "tlvLength[%lu], unParsed Length[%lu], already parsed Length[%lu]",
720 : identifier_.c_str(), level0, level1, srcTlvTotalLength, commonTlv->length, unPareseLength, parseLength);
721 121 : }
722 143 : for (u32 idx = 0; idx < vectorInfo.size(); idx++) {
723 242 : for (u32 ringidx = 0; ringidx < vectorInfo[idx].size(); ringidx++) {
724 121 : HCCL_DEBUG(
725 : "[HcclCommAicpu][ParseTlvToVector] idx[%u] ringidx[%u] size[%u]", idx, ringidx,
726 : vectorInfo[idx][ringidx].size());
727 : }
728 : }
729 22 : HCCL_INFO("[HcclCommAicpu][ParseTlvToVector] success, group[%s]", identifier_.c_str());
730 22 : return HCCL_SUCCESS;
731 : }
732 :
733 11 : HcclResult HcclCommAicpu::ParseTlvToSubGroupVector(
734 : u64 srcTlv, u64 srcTlvTotalLength, std::vector<std::vector<std::vector<std::vector<u32>>>>& vectorInfo)
735 : {
736 11 : u64 parseLength = 0;
737 11 : u8* srcTlvptr = reinterpret_cast<u8*>(srcTlv);
738 11 : if (srcTlvptr == nullptr) {
739 0 : HCCL_ERROR("[HcclCommAicpu][ParseTlvToSubGroupVector]tlv ptr is null.");
740 0 : return HCCL_E_PARA;
741 : }
742 11 : u64 unPareseLength = srcTlvTotalLength;
743 33 : while (unPareseLength > 0) {
744 22 : CommonTlv* commonTlv = reinterpret_cast<CommonTlv*>(srcTlvptr + parseLength);
745 22 : if (unPareseLength <= (sizeof(LENGTH_TYPE) + sizeof(TAG_TYPE))) {
746 0 : HCCL_ERROR(
747 : "[HcclCommAicpu][ParseTlvToSubGroupVector] Tlv length is error, tag[%s], totalLength[%lu], "
748 : "unParseLength[%lu], already parsed Length[%lu]",
749 : identifier_.c_str(), srcTlvTotalLength, unPareseLength, parseLength);
750 0 : return HCCL_E_PARA;
751 : }
752 22 : if (commonTlv->length > unPareseLength || commonTlv->length % sizeof(RANK_TYPE) != 0) {
753 0 : HCCL_ERROR(
754 : "[HcclCommAicpu][ParseTlvToSubGroupVector] parse Tlv error, group[%s], total Length[%lu], "
755 : "tlvLength[%lu], "
756 : "unParsed Length[%lu], already parsed Length[%lu]",
757 : identifier_.c_str(), srcTlvTotalLength, commonTlv->length, unPareseLength, parseLength);
758 0 : return HCCL_E_PARA;
759 : }
760 22 : u16 level0 = (commonTlv->type & TOP_HIERARCHICAL_COMM_LEVEL0_LOCATION)
761 22 : >> (TOP_HIERARCHICAL_COMM_LEVEL0_SHIFT + TOP_HIERARCHICAL_COMM_LEVEL1_SHIFT);
762 22 : u16 level1 = (commonTlv->type & TOP_HIERARCHICAL_COMM_LEVEL1_LOCATION) >> TOP_HIERARCHICAL_COMM_LEVEL1_SHIFT;
763 22 : u16 level2 = (commonTlv->type & TOP_HIERARCHICAL_COMM_LEVEL2_LOCATION);
764 22 : u64 itemNum = (commonTlv->length - sizeof(TAG_TYPE) - sizeof(LENGTH_TYPE)) / sizeof(RANK_TYPE);
765 44 : std::vector<RANK_TYPE> values{&commonTlv->value, (&commonTlv->value) + itemNum};
766 22 : if (level0 >= vectorInfo.size()) {
767 11 : vectorInfo.resize(level0 + 1);
768 : }
769 22 : if (level1 >= vectorInfo[level0].size()) {
770 11 : vectorInfo[level0].resize(level1 + 1);
771 : }
772 22 : if (level2 >= vectorInfo[level0][level1].size()) {
773 22 : vectorInfo[level0][level1].resize(level2 + 1);
774 : }
775 22 : vectorInfo[level0][level1][level2] = std::move(values);
776 22 : parseLength += commonTlv->length;
777 22 : unPareseLength -= commonTlv->length;
778 22 : HCCL_DEBUG(
779 : "[HcclCommAicpu][ParseTlvToSubGroupVector] parse Tlv group[%s], level0[%u], level1[%u], total Length[%lu], "
780 : "tlvLength[%lu], unParsed Length[%lu], already parsed Length[%lu]",
781 : identifier_.c_str(), level0, level1, srcTlvTotalLength, commonTlv->length, unPareseLength, parseLength);
782 22 : }
783 22 : for (u32 level0Idx = 0; level0Idx < vectorInfo.size(); level0Idx++) {
784 22 : for (u32 level1Idx = 0; level1Idx < vectorInfo[level0Idx].size(); level1Idx++) {
785 33 : for (u32 level2Idx = 0; level2Idx < vectorInfo[level0Idx][level1Idx].size(); level2Idx++) {
786 22 : HCCL_DEBUG(
787 : "[HcclCommAicpu][ParseTlvToSubGroupVector] level0Idx[%u] level1Idx[%u] level2Idx[%u] size[%u]",
788 : level0Idx, level1Idx, level2Idx, vectorInfo[level0Idx][level1Idx][level2Idx].size());
789 : }
790 : }
791 : }
792 11 : HCCL_INFO("[HcclCommAicpu][ParseTlvToSubGroupVector] success, group[%s]", identifier_.c_str());
793 11 : return HCCL_SUCCESS;
794 : }
795 :
796 11 : HcclResult HcclCommAicpu::InitTopoInfo(const HcclOpResParam* commParam)
797 : {
798 11 : topoInfo_.userRank = commParam->topoInfo.userRank;
799 11 : topoInfo_.userRankSize = commParam->topoInfo.userRankSize;
800 11 : topoInfo_.deviceLogicId = commParam->topoInfo.deviceLogicId;
801 11 : topoInfo_.isSingleMeshAggregation = commParam->topoInfo.isSingleMeshAggregation;
802 11 : topoInfo_.deviceNumPerAggregation = commParam->topoInfo.deviceNumPerAggregation;
803 11 : topoInfo_.superPodNum = commParam->topoInfo.superPodNum;
804 11 : topoInfo_.devicePhyId = commParam->topoInfo.devicePhyId;
805 11 : topoInfo_.deviceType = static_cast<DevType>(commParam->topoInfo.deviceType);
806 11 : topoInfo_.topoType = static_cast<TopoType>(commParam->topoInfo.topoType);
807 11 : topoInfo_.is310P3Common = (topoInfo_.deviceType == DevType::DEV_TYPE_310P3);
808 11 : topoInfo_.serverNum = commParam->topoInfo.serverNum;
809 11 : topoInfo_.meshAggregationRankSize = commParam->topoInfo.meshAggregationRankSize;
810 11 : topoInfo_.multiModuleDiffDeviceNumMode = commParam->topoInfo.multiModuleDiffDeviceNumMode;
811 11 : topoInfo_.multiSuperPodDiffServerNumMode = commParam->topoInfo.multiSuperPodDiffServerNumMode;
812 11 : topoInfo_.realUserRank = commParam->topoInfo.realUserRank;
813 11 : topoInfo_.isDiffDeviceModule = commParam->topoInfo.isDiffDeviceModule;
814 11 : topoInfo_.isDiffDeviceType = commParam->topoInfo.isDiffDeviceType;
815 11 : topoInfo_.gcdDeviceNumPerAggregation = commParam->topoInfo.gcdDeviceNumPerAggregation;
816 11 : topoInfo_.moduleNum = commParam->topoInfo.moduleNum;
817 11 : topoInfo_.useSuperPodMode = true;
818 11 : topoInfo_.isARSDoubleRing = commParam->isARSDoubleRing;
819 11 : topoInfo_.multiSuperPodDiffDeviceNumMode = commParam->multiSuperPodDiffDeviceNumMode;
820 11 : if (commParam->topoInfo.isUsedRdmaRankPairNum % KEY_VALUE_TO_VECTOR_MODULUS != 0) {
821 0 : HCCL_ERROR(
822 : "[HcclCommAicpu][InitTopoInfo]rdma rank pair number[%lu] is error.",
823 : commParam->topoInfo.isUsedRdmaRankPairNum);
824 0 : return HCCL_E_PARA;
825 : }
826 11 : u32* isUsedRdmaRankPairPtr = reinterpret_cast<u32*>(commParam->topoInfo.isUsedRdmaRankPair);
827 11 : if (isUsedRdmaRankPairPtr == nullptr) {
828 0 : HCCL_ERROR("[HcclCommAicpu][InitTopoInfo]rdma rank pair ptr is null.");
829 0 : return HCCL_E_PARA;
830 : }
831 55 : for (u64 i = 0; i < commParam->topoInfo.isUsedRdmaRankPairNum; i += KEY_VALUE_TO_VECTOR_MODULUS) {
832 44 : topoInfo_.isUsedRdmaMap.insert({isUsedRdmaRankPairPtr[i], static_cast<bool>(isUsedRdmaRankPairPtr[i + 1])});
833 : }
834 :
835 11 : if (commParam->topoInfo.pairLinkCounterNum % KEY_VALUE_TO_VECTOR_MODULUS != 0) {
836 0 : HCCL_ERROR(
837 : "[HcclCommAicpu][InitTopoInfo]pair link count number[%lu] is error.",
838 : commParam->topoInfo.pairLinkCounterNum);
839 0 : return HCCL_E_PARA;
840 : }
841 11 : u32* pairLinkCounterPtr = reinterpret_cast<u32*>(commParam->topoInfo.pairLinkCounter);
842 11 : if (pairLinkCounterPtr == nullptr) {
843 0 : HCCL_ERROR("[HcclCommAicpu][InitTopoInfo]pair link counter ptr is null.");
844 0 : return HCCL_E_PARA;
845 : }
846 :
847 55 : for (u64 i = 0; i < commParam->topoInfo.pairLinkCounterNum; i += KEY_VALUE_TO_VECTOR_MODULUS) {
848 44 : topoInfo_.pairLinkCounter.insert({pairLinkCounterPtr[i], pairLinkCounterPtr[i + 1]});
849 : }
850 :
851 11 : u32* nicListPtr = reinterpret_cast<u32*>(commParam->topoInfo.nicList);
852 11 : if (nicListPtr == nullptr) {
853 0 : HCCL_ERROR("[HcclCommAicpu][InitTopoInfo]nic list ptr is null.");
854 0 : return HCCL_E_PARA;
855 : }
856 22 : std::vector<u32> niclist{nicListPtr, nicListPtr + commParam->topoInfo.nicNum};
857 11 : topoInfo_.nicList = std::move(niclist);
858 :
859 11 : bool* bridgeRankPtr = reinterpret_cast<bool*>(commParam->topoInfo.bridgeRank);
860 11 : if (bridgeRankPtr != nullptr) {
861 11 : isBridgeVector_.resize(commParam->topoInfo.bridgeRankNum);
862 33 : for (u32 i = 0; i < commParam->topoInfo.bridgeRankNum; ++i) {
863 22 : isBridgeVector_[i] = bridgeRankPtr[i];
864 22 : HCCL_DEBUG("[HcclCommAicpu][InitTopoInfo] bridge rank info idx[%u] value[%u]", i, isBridgeVector_[i]);
865 : }
866 : } else {
867 0 : HCCL_RUN_INFO("[HcclCommAicpu][InitTopoInfo] bridge rank number is 0, group[%s]", identifier_.c_str());
868 : }
869 :
870 11 : HcclResult ret = HCCL_SUCCESS;
871 11 : ret = ParseTlvToVector(commParam->topoInfo.complanRank, commParam->topoInfo.complanRankLength, commPlaneVector_);
872 11 : CHK_PRT_RET(
873 : ret != HCCL_SUCCESS,
874 : HCCL_ERROR("[HcclCommAicpu][InitTopoInfo]Init CommPlane error group[%s]", identifier_.c_str()), ret);
875 :
876 22 : ret = ParseTlvToVector(
877 11 : commParam->topoInfo.serverAndsuperPodRank, commParam->topoInfo.serverAndsuperPodRankLength,
878 11 : serverAndsuperPodToRank_);
879 11 : CHK_PRT_RET(
880 : ret != HCCL_SUCCESS,
881 : HCCL_ERROR("[HcclCommAicpu][InitTopoInfo]Init server and superPod rank error group[%s]", identifier_.c_str()),
882 : ret);
883 :
884 22 : ret = ParseTlvToSubGroupVector(
885 11 : commParam->hierarchicalAlgInfo.commplaneSubGroupRank,
886 11 : commParam->hierarchicalAlgInfo.commplaneSubGroupRankLength, topoInfo_.CommPlaneSubGroupVector);
887 11 : CHK_PRT_RET(
888 : ret != HCCL_SUCCESS,
889 : HCCL_ERROR("[HcclCommAicpu][InitTopoInfo]Init CommPlaneSubGroup error group[%s]", identifier_.c_str()), ret);
890 :
891 11 : HCCL_INFO(
892 : "[HcclCommAicpu][InitTopoInfo] success, group[%s], device Type[%u]", identifier_.c_str(),
893 : commParam->topoInfo.deviceType);
894 11 : return HCCL_SUCCESS;
895 11 : }
896 :
897 11 : HcclResult HcclCommAicpu::InitCclbuffer(const HcclOpResParam* commParam)
898 : {
899 11 : if (commParam->localWindowsIn == 0 || commParam->localWindowsOut == 0 || cclInputBuffer_.ptr() != nullptr
900 22 : || cclOutputBuffer_.ptr() != nullptr) {
901 0 : HCCL_INFO(
902 : "[HcclCommAicpu][InitCclBuffer] don't need init cclbuffer "
903 : "ccl winIn[0x%lx] winout[0x%lx] cclin ptr[%p] cclout ptr[%p]",
904 : commParam->localWindowsIn, commParam->localWindowsOut, cclInputBuffer_.ptr(), cclOutputBuffer_.ptr());
905 0 : return HCCL_SUCCESS;
906 : }
907 :
908 11 : auto cclInPtr = reinterpret_cast<void*>(commParam->localWindowsIn);
909 11 : auto cclOutPtr = reinterpret_cast<void*>(commParam->localWindowsOut);
910 11 : cclInputBuffer_ = DeviceMem::create(cclInPtr, commParam->winSize);
911 11 : cclOutputBuffer_ = DeviceMem::create(cclOutPtr, commParam->winSize);
912 11 : cclbufferSize_ = commParam->winSize;
913 11 : HCCL_INFO(
914 : "[HcclCommAicpu][InitCclbuffer] success, group[%s], cclin[%llu], cclout[%llu], size[%lu]", identifier_.c_str(),
915 : commParam->localWindowsIn, commParam->localWindowsOut, commParam->winSize);
916 11 : return HCCL_SUCCESS;
917 : }
918 :
919 11 : HcclResult HcclCommAicpu::InitConfigInfo(const HcclOpResParam* commParam)
920 : {
921 11 : deterministic_ = commParam->config.deterministic;
922 11 : interHccsDisable_ = commParam->config.interHccsDisable;
923 11 : multiQpThreshold_ = commParam->config.multiQpThreshold;
924 11 : inlineReducEnable_ = true;
925 11 : fftsEnable_ = false;
926 11 : taskMonitorInterval_ = commParam->config.taskMonitorInterval;
927 11 : algoInfo_.inlineReduceSwitchOn = true;
928 11 : algoInfo_.identifier = commParam->hcomId;
929 11 : algoInfo_.isSupportAtomicWrite = static_cast<bool>(commParam->config.isSupportAtomicWrite);
930 11 : notifySize_ = commParam->notifysize;
931 11 : slaveStreams_.reserve(LOCAL_STREAM_MAX_NUM);
932 11 : localNotifies_.reserve(LOCAL_NOTIFY_MAX_NUM);
933 11 : SetExternalInputDebugConfig(commParam->debugConfig);
934 11 : InitDebugConfigByValue(commParam->debugConfig);
935 11 : HCCL_INFO(
936 : "[HcclCommAicpu][Init]success, group[%s] reserve noipc notifys[%lu], slave streams[%lu].", identifier_.c_str(),
937 : LOCAL_NOTIFY_MAX_NUM, LOCAL_STREAM_MAX_NUM);
938 11 : return HCCL_SUCCESS;
939 : }
940 :
941 15 : void HcclCommAicpu::SetDumpDebug(bool dumpDebug) { dumpDebug_ = dumpDebug; }
942 :
943 : template <typename T>
944 : HcclResult
945 726 : HcclCommAicpu::InitAndVerifySignal(const HcclSignalInfo& signalInfo, std::vector<std::shared_ptr<T>>& notifyVec)
946 : {
947 726 : if (signalInfo.resId == INVALID_U64) {
948 : // 无效值不做校验
949 0 : HCCL_INFO("[HcclCommAicpu][InitAndVerifySignal] resId is invalid, need not check");
950 0 : return HCCL_SUCCESS;
951 : }
952 :
953 726 : std::shared_ptr<T> notify;
954 726 : EXCEPTION_CATCH((notify = std::make_shared<T>()), return HCCL_E_PTR);
955 726 : CHK_SMART_PTR_NULL(notify);
956 726 : HcclSignalInfo notifySignalInfo = signalInfo;
957 726 : CHK_RET(notify->Init(notifySignalInfo, NotifyLoadType::DEVICE_NOTIFY));
958 726 : notifyVec.push_back(notify);
959 726 : HCCL_INFO(
960 : "[HcclCommAicpu][InitAndVerifySignal] success group[%s], resId[%u], tsId:%d, devId[%u]", identifier_.c_str(),
961 : signalInfo.resId, signalInfo.tsId, signalInfo.devId);
962 :
963 726 : return HCCL_SUCCESS;
964 726 : }
965 :
966 33 : void HcclCommAicpu::HandleExistTagReAlloc(
967 : HccltagLocalResV2* tagRes, const std::string& tag, bool reAllocFlag, ListCommon*& curList, bool& needSkip)
968 : {
969 33 : needSkip = false;
970 : // 如果 tag 已经存在且需要重新申请内存, 删除旧的资源与记录
971 33 : if (localTagResToObj_.find(tag) != localTagResToObj_.end() && reAllocFlag) {
972 0 : u64 currSize = tagRes->ScratchmemSize;
973 0 : u64 existSize = tagScratchMem_[tag]->size();
974 : // 新内存更大，删除旧记录，保留新的
975 0 : if (currSize > existSize) {
976 0 : localTagResToObj_.erase(tag);
977 0 : tagScratchMem_.erase(tag);
978 0 : HCCL_DEBUG(
979 : "[HcclCommAicpu][%s] tag exists, replace old small mem, tag[%s], old[%lu] new[%lu]", __func__,
980 : tag.c_str(), existSize, currSize);
981 : } else {
982 : // 旧内存更大，需要跳过
983 0 : HCCL_DEBUG(
984 : "[HcclCommAicpu][%s] tag exists, keep larger old mem, skip new, tag[%s], old[%lu] new[%lu]", __func__,
985 : tag.c_str(), existSize, currSize);
986 0 : curList = reinterpret_cast<ListCommon*>(curList->nextDevice);
987 0 : needSkip = true;
988 : }
989 : }
990 33 : }
991 :
992 11 : HcclResult HcclCommAicpu::InitLocalTagRes(const ListCommon& head, bool reAllocFlag)
993 : {
994 11 : ListCommon* curList = reinterpret_cast<ListCommon*>(head.nextDevice);
995 11 : if (curList == nullptr) {
996 0 : HCCL_ERROR("[HcclCommAicpu][InitLocalTagRes]list ptr is null.");
997 0 : return HCCL_E_PARA;
998 : }
999 44 : while (curList != &head) {
1000 33 : HccltagLocalResV2* tagRes = list_entry(curList, HccltagLocalResV2, nextTagRes);
1001 33 : std::string tag = tagRes->tag;
1002 33 : bool needSkip = false;
1003 33 : HandleExistTagReAlloc(tagRes, tag, reAllocFlag, curList, needSkip);
1004 33 : if (needSkip) {
1005 0 : continue;
1006 : }
1007 33 : if (localTagResToObj_.find(tag) == localTagResToObj_.end()
1008 33 : || localTagResToObj_[tag].find(tagRes->Scratchmem) == localTagResToObj_[tag].end()) {
1009 33 : auto scratchMemPtr = reinterpret_cast<void*>(tagRes->Scratchmem);
1010 33 : if (scratchMemPtr == nullptr) {
1011 0 : HCCL_ERROR("[HcclCommAicpu][InitLocalTagRes]scratch mem ptr is null.");
1012 0 : return HCCL_E_PARA;
1013 : }
1014 33 : DeviceMem loalScratchmem = DeviceMem::create(scratchMemPtr, tagRes->ScratchmemSize);
1015 33 : std::shared_ptr<DeviceMem> loalScratchmemPtr;
1016 33 : EXCEPTION_CATCH(
1017 : (loalScratchmemPtr = std::make_shared<DeviceMem>(std::move(loalScratchmem))), return HCCL_E_PTR);
1018 33 : CHK_SMART_PTR_NULL(loalScratchmemPtr);
1019 33 : if (tagScratchMem_.find(tag) == tagScratchMem_.end()) {
1020 33 : tagScratchMem_.insert({tag, loalScratchmemPtr});
1021 : }
1022 33 : std::unordered_set<u64> tmpTagRes;
1023 33 : tmpTagRes.insert(tagRes->Scratchmem);
1024 33 : localTagResToObj_[tag] = tmpTagRes;
1025 33 : HCCL_DEBUG(
1026 : "[HcclCommAicpu][InitLocalTagRes] parse remote resource, tag[%s], Scratchmem[%p], "
1027 : "ScratchmemSize[%lu]",
1028 : tag.c_str(), tagRes->Scratchmem, tagRes->ScratchmemSize);
1029 33 : }
1030 33 : curList = reinterpret_cast<ListCommon*>(curList->nextDevice);
1031 33 : if (curList == nullptr) {
1032 0 : HCCL_ERROR("[HcclCommAicpu][InitLocalTagRes] next list ptr is null.");
1033 0 : return HCCL_E_PARA;
1034 : }
1035 33 : };
1036 11 : HCCL_INFO("[HcclCommAicpu][InitLocalTagRes] success, group[%s]", identifier_.c_str());
1037 11 : return HCCL_SUCCESS;
1038 : }
1039 :
1040 0 : HcclResult HcclCommAicpu::CheckNotifyOrQPMaxNum(u64& existNum, const u64& MaxNum, const bool& isNotifyRes)
1041 : {
1042 0 : std::string resType = isNotifyRes ? "Notify" : "QP";
1043 0 : if (existNum + 1 > MaxNum) {
1044 0 : HCCL_ERROR(
1045 : "[%s]%s resources are insufficient, existNum[%llu], MaxNum is [%llu]", __func__, resType.c_str(), existNum,
1046 : MaxNum);
1047 0 : return HCCL_E_INTERNAL;
1048 : }
1049 0 : HCCL_DEBUG(
1050 : "[%s]%s resources are sufficient, existNum[%llu], MaxNum is [%llu]", __func__, resType.c_str(), existNum,
1051 : MaxNum);
1052 0 : return HCCL_SUCCESS;
1053 0 : }
1054 :
1055 0 : HcclResult HcclCommAicpu::SetTransportMachinePara(
1056 : MachinePara& machinePara, u32& rankId, const std::string& newTag, TransportLinkType linkType)
1057 : {
1058 0 : machinePara.linkAttribute = 0x03; /* 0x03同时支持目的端和源端发起 */
1059 0 : if (rankData_.find(rankId) == rankData_.end()) {
1060 0 : HCCL_ERROR("[%s]there is no link with rankId[%u]", __func__, rankId);
1061 0 : return HCCL_E_NOT_FOUND;
1062 : }
1063 :
1064 0 : machinePara.localUserrank = localUserRank_;
1065 0 : machinePara.remoteWorldRank = rankData_[rankId].remoteWorldRank;
1066 0 : machinePara.remoteUserrank = rankData_[rankId].remoteUsrRankId;
1067 0 : machinePara.deviceLogicId = topoInfo_.deviceLogicId;
1068 0 : machinePara.localDeviceId = topoInfo_.devicePhyId;
1069 0 : machinePara.deviceType = topoInfo_.deviceType;
1070 0 : machinePara.tag = newTag;
1071 0 : machinePara.isAicpuModeEn = true;
1072 0 : if (linkType == TransportLinkType::RESERVED) {
1073 : // 非910_93 2die sio与hccs并发场景，specifyLink设置为RESERVED_LINK_TYPE，平台层将按实际链路类型建链
1074 0 : machinePara.specifyLink = LinkTypeInServer::RESERVED_LINK_TYPE;
1075 : } else {
1076 : // 910_93 2die sio与hccs并发场景，
1077 : // 并发链路中的的hccs链路specifyLink设置为HCCS_SW_TYPE，平台层将使用hccs链路来建链；
1078 : // 并发链路中的的sio链路specifyLink设置为SIO_TYPE，平台层将使用sio链路来建链
1079 : machinePara.specifyLink
1080 0 : = (linkType == TransportLinkType::SIO) ? LinkTypeInServer::SIO_TYPE : LinkTypeInServer::HCCS_SW_TYPE;
1081 : }
1082 :
1083 0 : HCCL_INFO(
1084 : "%s success, group[%s], rankId[%u], linkAttribute[%x], localUserRank[%u], remoteWorldRank[%u], "
1085 : "remoteUserrank[%u], deviceLogicId[%d], localDeviceId[%d], deviceType[%d], newTag[%s], specifyLink[%d], "
1086 : "isAicpuModeEn[%d]",
1087 : __func__, identifier_.c_str(), rankId, machinePara.linkAttribute, machinePara.localUserrank,
1088 : machinePara.remoteWorldRank, machinePara.remoteUserrank, machinePara.deviceLogicId, machinePara.localDeviceId,
1089 : machinePara.deviceType, machinePara.tag.c_str(), machinePara.specifyLink, machinePara.isAicpuModeEn);
1090 0 : return HCCL_SUCCESS;
1091 : }
1092 :
1093 : template <typename T>
1094 0 : HcclResult HcclCommAicpu::InitAndVerifySingleSignal(const HcclSignalInfo& signalInfo, std::shared_ptr<T>& notify)
1095 : {
1096 0 : if (signalInfo.resId == INVALID_U64) {
1097 : // 无效值不做校验
1098 0 : HCCL_DEBUG("[%s]resId[%llu] is invalid, need not check", __func__, signalInfo.resId);
1099 0 : return HCCL_SUCCESS;
1100 : }
1101 0 : HcclSignalInfo tmpSignalInfo = signalInfo;
1102 :
1103 0 : EXCEPTION_CATCH((notify = std::make_shared<T>()), return HCCL_E_PTR);
1104 0 : CHK_SMART_PTR_NULL(notify);
1105 0 : CHK_RET(notify->Init(tmpSignalInfo, NotifyLoadType::DEVICE_NOTIFY));
1106 0 : CHK_RET(notify->GetNotifyData(tmpSignalInfo));
1107 0 : HCCL_DEBUG(
1108 : "[%s] success group[%s], resId[%llu], tsId:%d, devId[%u]", __func__, identifier_.c_str(), tmpSignalInfo.resId,
1109 : tmpSignalInfo.tsId, tmpSignalInfo.devId);
1110 :
1111 0 : return HCCL_SUCCESS;
1112 : }
1113 :
1114 0 : HcclResult HcclCommAicpu::SetTagRemoteRes(u32& rankId, const std::string& tag, HccltagRemoteResV2* tagRes)
1115 : {
1116 0 : if (rankTagRemoteRes_.find(rankId) == rankTagRemoteRes_.end()
1117 0 : || rankTagRemoteRes_[rankId].find(tag) == rankTagRemoteRes_[rankId].end()) {
1118 0 : HccltagRemoteResV3 tempTagRemoteRes;
1119 0 : tempTagRemoteRes.tagRemoteResPtr = tagRes;
1120 0 : rankTagRemoteRes_[rankId][tag] = tempTagRemoteRes;
1121 : }
1122 0 : HCCL_DEBUG("[%s]get TagRemoteRes success, rankId[%u], tag[%s]", __func__, rankId, tag.c_str());
1123 0 : return HCCL_SUCCESS;
1124 : }
1125 :
1126 0 : HcclResult HcclCommAicpu::SetTransportPtpNotify(
1127 : TransportDeviceP2pData& transDevP2pData, u64& p2pNotifyNum, HcclLinkP2pV2& linkP2p, u32 notifyNum)
1128 : {
1129 0 : u64 actualNotifyNum = 0;
1130 : // 获取Ipc notify信息
1131 0 : CHK_RET(CheckNotifyOrQPMaxNum(actualNotifyNum, LINK_P2P_MAX_NUM, true));
1132 0 : std::shared_ptr<LocalNotify> ipcPreWaitNotify = std::make_shared<LocalNotify>();
1133 0 : CHK_RET(InitAndVerifySingleSignal(linkP2p.localIpcSignal[actualNotifyNum], ipcPreWaitNotify));
1134 0 : transDevP2pData.ipcPreWaitNotify = ipcPreWaitNotify;
1135 :
1136 0 : std::shared_ptr<RemoteNotify> ipcPreRecordNotify = std::make_shared<RemoteNotify>();
1137 0 : CHK_RET(InitAndVerifySingleSignal(linkP2p.remoteIpcSignal[actualNotifyNum], ipcPreRecordNotify));
1138 0 : transDevP2pData.ipcPreRecordNotify = ipcPreRecordNotify;
1139 0 : actualNotifyNum++;
1140 :
1141 0 : CHK_RET(CheckNotifyOrQPMaxNum(actualNotifyNum, LINK_P2P_MAX_NUM, true));
1142 0 : std::shared_ptr<LocalNotify> ipcPostWaitNotify = std::make_shared<LocalNotify>();
1143 0 : CHK_RET(InitAndVerifySingleSignal(linkP2p.localIpcSignal[actualNotifyNum], ipcPostWaitNotify));
1144 0 : transDevP2pData.ipcPostWaitNotify = ipcPostWaitNotify;
1145 :
1146 0 : std::shared_ptr<RemoteNotify> ipcPostRecordNotify = std::make_shared<RemoteNotify>();
1147 0 : CHK_RET(InitAndVerifySingleSignal(linkP2p.remoteIpcSignal[actualNotifyNum], ipcPostRecordNotify));
1148 0 : transDevP2pData.ipcPostRecordNotify = ipcPostRecordNotify;
1149 0 : actualNotifyNum++;
1150 :
1151 0 : transDevP2pData.userLocalNotify.resize(notifyNum, nullptr);
1152 0 : transDevP2pData.userRemoteNotify.resize(notifyNum, nullptr);
1153 :
1154 0 : for (u32 idx = 0; idx < notifyNum; idx++) {
1155 0 : CHK_RET(CheckNotifyOrQPMaxNum(actualNotifyNum, LINK_P2P_MAX_NUM, true));
1156 0 : std::shared_ptr<LocalNotify> ipcWaitNotify = std::make_shared<LocalNotify>();
1157 0 : CHK_RET(InitAndVerifySingleSignal(linkP2p.localIpcSignal[actualNotifyNum], ipcWaitNotify));
1158 0 : transDevP2pData.userLocalNotify[idx] = ipcWaitNotify;
1159 :
1160 0 : std::shared_ptr<RemoteNotify> ipcRecordNotify = std::make_shared<RemoteNotify>();
1161 0 : CHK_RET(InitAndVerifySingleSignal(linkP2p.remoteIpcSignal[actualNotifyNum], ipcRecordNotify));
1162 0 : transDevP2pData.userRemoteNotify[idx] = ipcRecordNotify;
1163 :
1164 0 : actualNotifyNum++;
1165 0 : }
1166 :
1167 0 : HCCL_DEBUG("[%s]get p2pNotify success, actualNotifyNum[%u]", __func__, actualNotifyNum);
1168 0 : p2pNotifyNum = actualNotifyNum;
1169 :
1170 0 : return HCCL_SUCCESS;
1171 0 : }
1172 :
1173 0 : HcclResult HcclCommAicpu::SetTransportRoceQP(
1174 : TransportDeviceIbverbsData& transDevIbverbsData, u64& roceQpNum, HcclLinkRoceV2* linkRoce)
1175 : {
1176 0 : roceQpNum = linkRoce->qpsPerConnection;
1177 0 : u32 roceQpNumSum = linkRoce->qpsPerConnection + static_cast<u32>(linkRoce->qpsPerConnection != 1);
1178 0 : transDevIbverbsData.qpInfo.resize(roceQpNumSum);
1179 0 : std::copy_n(linkRoce->QpInfo, roceQpNumSum, transDevIbverbsData.qpInfo.begin());
1180 0 : transDevIbverbsData.multiQpThreshold = multiQpThreshold_;
1181 0 : transDevIbverbsData.qpsPerConnection = linkRoce->qpsPerConnection;
1182 0 : HCCL_INFO(
1183 : "[%s]transDevIbverbsData.qpInfo.qpPtr[%llu], transDevIbverbsData.qpInfo.sqIndex[%u], "
1184 : "transDevIbverbsData.qpInfo.dbIndex[%u], roceQpNum[%llu], roceQpNumSum[%u], multiQpThreshold_[%u]",
1185 : __func__, transDevIbverbsData.qpInfo[0].qpPtr, transDevIbverbsData.qpInfo[0].sqIndex,
1186 : transDevIbverbsData.qpInfo[0].dbIndex, roceQpNum, roceQpNumSum, multiQpThreshold_);
1187 0 : return HCCL_SUCCESS;
1188 : }
1189 :
1190 0 : HcclResult HcclCommAicpu::SetTransportRoceNotify(
1191 : TransportDeviceIbverbsData& transDevIbverbsData, u64& roceNotifyNum, HcclLinkRoceV2* linkRoce,
1192 : [[maybe_unused]] u32 notifyNum)
1193 : {
1194 0 : u64 actualNotifyNum = 0;
1195 0 : if (linkRoce->localNotifyList == 0 || linkRoce->remoteNotifyList == 0) {
1196 0 : HCCL_DEBUG("[%s] Empty local and remote notify lists, skipping notify resource creation.", __func__);
1197 0 : return HCCL_SUCCESS;
1198 : }
1199 0 : HcclSignalInfo* localNotifyList = reinterpret_cast<HcclSignalInfo*>(linkRoce->localNotifyList);
1200 0 : AddrKey* remoteNotifyList = reinterpret_cast<AddrKey*>(linkRoce->remoteNotifyList);
1201 0 : if (localNotifyList == nullptr || remoteNotifyList == nullptr) {
1202 0 : HCCL_ERROR("[%s]nullptr found in localNotifyList or remoteNotifyList from device mem, check.", __func__);
1203 0 : return HCCL_E_INTERNAL;
1204 : }
1205 : // 获取RDMA Notify信息
1206 0 : std::shared_ptr<LocalNotify> ackNotify = std::make_shared<LocalNotify>();
1207 0 : CHK_RET(InitAndVerifySingleSignal(localNotifyList[actualNotifyNum], ackNotify));
1208 0 : transDevIbverbsData.ackNotify = ackNotify;
1209 0 : transDevIbverbsData.remoteAckNotifyDetails = remoteNotifyList[actualNotifyNum];
1210 0 : actualNotifyNum++;
1211 :
1212 0 : std::shared_ptr<LocalNotify> dataNotify = std::make_shared<LocalNotify>();
1213 0 : CHK_RET(InitAndVerifySingleSignal(localNotifyList[actualNotifyNum], dataNotify));
1214 0 : transDevIbverbsData.dataNotify = dataNotify;
1215 0 : transDevIbverbsData.remoteDataNotifyDetails = remoteNotifyList[actualNotifyNum];
1216 0 : actualNotifyNum++;
1217 :
1218 0 : std::shared_ptr<LocalNotify> dataAckNotify = std::make_shared<LocalNotify>();
1219 0 : CHK_RET(InitAndVerifySingleSignal(localNotifyList[actualNotifyNum], dataAckNotify));
1220 0 : transDevIbverbsData.dataAckNotify = dataAckNotify;
1221 0 : transDevIbverbsData.remoteDataAckNotifyDetails = remoteNotifyList[actualNotifyNum];
1222 0 : transDevIbverbsData.notifySize = notifySize_;
1223 0 : actualNotifyNum++;
1224 :
1225 0 : transDevIbverbsData.userLocalNotify.resize(linkRoce->qpsPerConnection);
1226 0 : transDevIbverbsData.userRemoteNotifyDetails.resize(linkRoce->qpsPerConnection);
1227 : // 当前多QP下每个QP会多申请一个DataNotify
1228 0 : u64 singleQpNotifySize = linkRoce->singleQPNotifyNum + static_cast<u32>(linkRoce->qpsPerConnection > 1);
1229 0 : for (u32 qpIndex = 0; qpIndex < linkRoce->qpsPerConnection; qpIndex++) {
1230 0 : transDevIbverbsData.userLocalNotify[qpIndex].resize(singleQpNotifySize, nullptr);
1231 0 : transDevIbverbsData.userRemoteNotifyDetails[qpIndex].resize(singleQpNotifySize);
1232 0 : for (u32 i = 0, idx = actualNotifyNum + singleQpNotifySize * qpIndex; i < singleQpNotifySize; ++idx, ++i) {
1233 0 : std::shared_ptr<LocalNotify> locNotify = std::make_shared<LocalNotify>();
1234 0 : CHK_RET(InitAndVerifySingleSignal(localNotifyList[idx], locNotify));
1235 0 : transDevIbverbsData.userLocalNotify[qpIndex][i] = locNotify;
1236 0 : transDevIbverbsData.userRemoteNotifyDetails[qpIndex][i] = remoteNotifyList[idx];
1237 0 : }
1238 : }
1239 0 : roceNotifyNum = linkRoce->singleQPNotifyNum;
1240 0 : HCCL_DEBUG("[%s]get roceNotify success, roceNotifyNum[%u]", __func__, roceNotifyNum);
1241 0 : return HCCL_SUCCESS;
1242 0 : }
1243 :
1244 0 : HcclResult HcclCommAicpu::InitLinkP2p(
1245 : HccltagRemoteResV2* tagRes, u32& rankId, const std::string& newTag, u32 notifyNum, TransportLinkType linkType)
1246 : {
1247 0 : std::unordered_map<u32, std::unordered_map<std::string, std::shared_ptr<Transport>>>& linkRes
1248 : = (linkType == TransportLinkType::SIO) ? linkResSio_ : linkRes_;
1249 0 : HcclLinkP2pV2& linkP2p = (linkType == TransportLinkType::SIO) ? tagRes->linkP2pSio : tagRes->linkP2p;
1250 :
1251 0 : if (linkRes.find(rankId) == linkRes.end() || linkRes[rankId].find(newTag) == linkRes[rankId].end()) {
1252 : // 优先校验notify去判定link是否有效
1253 0 : if (linkP2p.localIpcSignal[0].resId == INVALID_U64) {
1254 0 : HCCL_INFO(
1255 : "[%s]the link is invalid, no need to create transport, rankId[%u], newTag[%s]", __func__, rankId,
1256 : newTag.c_str());
1257 0 : return HCCL_SUCCESS;
1258 : }
1259 : // 创建Transport对象
1260 0 : MachinePara machinePara;
1261 0 : CHK_RET(SetTransportMachinePara(machinePara, rankId, newTag, linkType));
1262 0 : machinePara.notifyNum = notifyNum;
1263 : // 获取localMem & remoteMem
1264 0 : TransportDeviceP2pData transDevP2pData;
1265 0 : transDevP2pData.inputBufferPtr = reinterpret_cast<void*>((linkP2p.remoteMem)[INPUT].addr);
1266 0 : transDevP2pData.outputBufferPtr = reinterpret_cast<void*>((linkP2p.remoteMem)[OUTPUT].addr);
1267 0 : if (transDevP2pData.inputBufferPtr == nullptr || transDevP2pData.outputBufferPtr == nullptr) {
1268 0 : HCCL_ERROR(
1269 : "[%s]input ptr[%p] or output ptr[%p] is null.", __func__, transDevP2pData.inputBufferPtr,
1270 : transDevP2pData.outputBufferPtr);
1271 0 : return HCCL_E_PARA;
1272 : }
1273 : // 获取Notify资源
1274 0 : CHK_RET(SetTagRemoteRes(rankId, newTag, tagRes));
1275 0 : HccltagRemoteResV3* tagRemoteRes = &(rankTagRemoteRes_[rankId][newTag]);
1276 0 : CHK_RET(SetTransportPtpNotify(transDevP2pData, tagRemoteRes->p2pNotifyNum, linkP2p, notifyNum));
1277 : // 获取transportAttr信息
1278 0 : transDevP2pData.transportAttr = linkP2p.transportAttr;
1279 : // 创建Transport对象
1280 0 : std::shared_ptr<Transport> link;
1281 0 : TransportPara para{};
1282 0 : const std::unique_ptr<NotifyPool> notifyPool;
1283 0 : link.reset(new (std::nothrow) Transport(
1284 0 : TransportType::TRANS_TYPE_DEVICE_P2P, para, dispatcher_, notifyPool, machinePara, transDevP2pData));
1285 0 : CHK_SMART_PTR_NULL(link);
1286 0 : CHK_RET(link->Init());
1287 0 : linkRes[rankId][newTag] = link;
1288 0 : HCCL_INFO("[%s]linkRes_, rankId[%u], newTag[%s]", __func__, rankId, newTag.c_str());
1289 0 : }
1290 0 : return HCCL_SUCCESS;
1291 : }
1292 0 : HcclResult HcclCommAicpu::InitLinkRoce(
1293 : HccltagRemoteResV2* tagRes, HcclLinkRoceV2* linkRoce, u32& rankId, const std::string& newTag, u32 notifyNum,
1294 : const bool isBackup, const bool isSecond)
1295 : {
1296 : // 优先校验notify去判定link是否有效
1297 0 : if (linkRoce->localNotifyList == 0) {
1298 0 : HCCL_INFO(
1299 : "[%s]the link is invalid, no need to create transport, rankId[%u], newTag[%s], isBackup[%d]", __func__,
1300 : rankId, newTag.c_str(), isBackup);
1301 0 : return HCCL_SUCCESS;
1302 : }
1303 0 : HcclSignalInfo* localNotifyList = reinterpret_cast<HcclSignalInfo*>(linkRoce->localNotifyList);
1304 0 : if (localNotifyList[0].resId == INVALID_U64) {
1305 0 : HCCL_INFO(
1306 : "[%s]the link notify resource is invalid, no need to create transport, rankId[%u], newTag[%s], "
1307 : "resId[%llu], "
1308 : "isBackup[%d]",
1309 : __func__, rankId, newTag.c_str(), localNotifyList[0].resId, isBackup);
1310 0 : return HCCL_SUCCESS;
1311 : }
1312 :
1313 : // 创建Transport对象
1314 0 : MachinePara machinePara;
1315 0 : CHK_RET(SetTransportMachinePara(machinePara, rankId, newTag));
1316 0 : machinePara.notifyNum = notifyNum;
1317 : // 获取localMem & remoteMem
1318 0 : TransportDeviceIbverbsData transDevIbverbsData;
1319 0 : transDevIbverbsData.inputBufferPtr = reinterpret_cast<void*>((linkRoce->remoteMem)[INPUT].addr);
1320 0 : transDevIbverbsData.outputBufferPtr = reinterpret_cast<void*>((linkRoce->remoteMem)[OUTPUT].addr);
1321 0 : if (transDevIbverbsData.inputBufferPtr == nullptr || transDevIbverbsData.outputBufferPtr == nullptr) {
1322 0 : HCCL_ERROR(
1323 : "[%s]input ptr[%p] or output ptr[%p] is null.", __func__, transDevIbverbsData.inputBufferPtr,
1324 : transDevIbverbsData.outputBufferPtr);
1325 0 : return HCCL_E_PARA;
1326 : }
1327 0 : transDevIbverbsData.localInputMem = (linkRoce->localMem)[INPUT];
1328 0 : transDevIbverbsData.localOutputMem = (linkRoce->localMem)[OUTPUT];
1329 0 : transDevIbverbsData.localNotifyValueAddr = linkRoce->notifyValue;
1330 0 : transDevIbverbsData.notifyValueKey = linkRoce->notifyValueKey;
1331 0 : transDevIbverbsData.remoteInputKey = (linkRoce->remoteMem)[INPUT].key;
1332 0 : transDevIbverbsData.remoteOutputKey = (linkRoce->remoteMem)[OUTPUT].key;
1333 : // 获取QPinfo
1334 0 : CHK_RET(SetTagRemoteRes(rankId, newTag, tagRes));
1335 0 : HccltagRemoteResV3* tagRemoteRes = &(rankTagRemoteRes_[rankId][newTag]);
1336 0 : u64& roceQpNum = isBackup ? tagRemoteRes->qpNumBackup : tagRemoteRes->qpNum;
1337 0 : CHK_RET(SetTransportRoceQP(transDevIbverbsData, roceQpNum, linkRoce));
1338 : // 获取notify
1339 0 : u64& roceNotifyNum = isBackup ? tagRemoteRes->roceNotifyNumBackup : tagRemoteRes->roceNotifyNum;
1340 0 : CHK_RET(SetTransportRoceNotify(transDevIbverbsData, roceNotifyNum, linkRoce, notifyNum));
1341 0 : HCCL_INFO("[%s]transDevIbverbsData isBackup[%d]", __func__, isBackup);
1342 : // 获取atomic write
1343 0 : transDevIbverbsData.useAtomicWrite = linkRoce->useAtomicWrite;
1344 : // 创建Transport对象
1345 0 : std::shared_ptr<Transport> link;
1346 0 : TransportPara para{};
1347 0 : para.timeout = linkTimeOut_;
1348 0 : const std::unique_ptr<NotifyPool> notifyPool;
1349 0 : link.reset(new (std::nothrow) Transport(
1350 0 : TransportType::TRANS_TYPE_DEVICE_IBVERBS, para, dispatcher_, notifyPool, machinePara, TransportDeviceP2pData(),
1351 0 : transDevIbverbsData));
1352 0 : CHK_SMART_PTR_NULL(link);
1353 0 : CHK_RET(link->Init());
1354 0 : if (isBackup) {
1355 0 : linkRdmaResBackUp_[rankId][newTag].push_back(link);
1356 0 : HCCL_INFO(
1357 : "[%s]linkRdmaResBackUp_, rankId[%u], newTag[%s], isBackup[%d], isSecond[%d], qpNum[%u], notifyNum[%u]",
1358 : __func__, rankId, newTag.c_str(), isBackup, isSecond, roceQpNum, roceNotifyNum);
1359 : } else {
1360 0 : linkRdmaRes_[rankId][newTag].push_back(link);
1361 0 : HCCL_INFO(
1362 : "[%s]linkRdmaRes_, rankId[%u], newTag[%s], isBackup[%d], isSecond[%d], qpNum[%u], notifyNum[%u]", __func__,
1363 : rankId, newTag.c_str(), isBackup, isSecond, roceQpNum, roceNotifyNum);
1364 : }
1365 0 : return HCCL_SUCCESS;
1366 0 : }
1367 :
1368 0 : HcclResult HcclCommAicpu::InitLinkRoce(
1369 : HccltagRemoteResV2* tagRes, u32& rankId, const std::string& newTag, u32 notifyNum, const bool isBackup)
1370 : {
1371 0 : auto tempLinkRes = isBackup ? linkRdmaResBackUp_ : linkRdmaRes_;
1372 0 : if (tempLinkRes.find(rankId) == tempLinkRes.end()
1373 0 : || tempLinkRes[rankId].find(newTag) == tempLinkRes[rankId].end()) {
1374 0 : bool isBatchSendRecv = newTag.find("BatchSendRecv") != std::string::npos;
1375 0 : if (isBatchSendRecv) {
1376 : // 如果是batchsendrecv，相同rank，需要刷新两次transport，如果是主的话就刷新0,2 备就刷新1,3
1377 0 : if (isBackup) {
1378 0 : CHK_RET(InitLinkRoce(
1379 : tagRes, &(tagRes->linkRoce[AICPU_RETRY_LINKROCE_BACKUP]), rankId, newTag, notifyNum, isBackup));
1380 0 : CHK_RET(InitLinkRoce(
1381 : tagRes, &(tagRes->linkRoce[AICPU_RETRY_LINKROCE_BACKUP + 2]), rankId, newTag, notifyNum, isBackup,
1382 : true));
1383 : } else {
1384 0 : CHK_RET(InitLinkRoce(
1385 : tagRes, &(tagRes->linkRoce[AICPU_RETRY_LINKROCE_DEFAULT]), rankId, newTag, notifyNum, isBackup));
1386 0 : CHK_RET(InitLinkRoce(
1387 : tagRes, &(tagRes->linkRoce[AICPU_RETRY_LINKROCE_DEFAULT + 2]), rankId, newTag, notifyNum, isBackup,
1388 : true));
1389 : }
1390 : } else {
1391 : // 非batchsendrecv只刷新主0，备1
1392 0 : HcclLinkRoceV2* linkRoce = isBackup ? &(tagRes->linkRoce[AICPU_RETRY_LINKROCE_BACKUP]) :
1393 : &(tagRes->linkRoce[AICPU_RETRY_LINKROCE_DEFAULT]);
1394 0 : CHK_RET(InitLinkRoce(tagRes, linkRoce, rankId, newTag, notifyNum, isBackup));
1395 : }
1396 : }
1397 0 : return HCCL_SUCCESS;
1398 0 : }
1399 :
1400 0 : HcclResult HcclCommAicpu::InitRemoteTagRes(
1401 : u32& rankId, const ListCommon& head, const std::string& newTag, u32 notifyNum, TransportLinkType linkType)
1402 : {
1403 0 : HCCL_RUN_INFO(
1404 : "[%s] Entry parse remote resource rankId[%u], group[%s], newTag[%s], head[%p], "
1405 : "linkType[%d]",
1406 : __func__, rankId, identifier_.c_str(), newTag.c_str(), &head, linkType);
1407 0 : ListCommon* curList = reinterpret_cast<ListCommon*>(head.nextDevice);
1408 0 : if (curList == nullptr) {
1409 0 : HCCL_ERROR("[%s]cur list ptr is null.", __func__);
1410 0 : return HCCL_E_PARA;
1411 : }
1412 :
1413 0 : HccltagRemoteResV2* tagRes = nullptr;
1414 0 : while (curList != &head) {
1415 0 : HccltagRemoteResV2* tagResTemp = list_entry(curList, HccltagRemoteResV2, nextTagRes);
1416 0 : if (strcmp(tagResTemp->tag, newTag.c_str()) == 0) {
1417 0 : tagRes = tagResTemp;
1418 0 : break;
1419 : }
1420 :
1421 0 : curList = reinterpret_cast<ListCommon*>(curList->nextDevice);
1422 0 : if (curList == nullptr) {
1423 0 : HCCL_ERROR("[%s]next list ptr is null.", __func__);
1424 0 : return HCCL_E_PARA;
1425 : }
1426 : }
1427 :
1428 0 : if (tagRes == nullptr) {
1429 0 : HCCL_ERROR(
1430 : "[%s]newTag[%s] not found, rankId[%u], head[%p], curList[%p], nextList[%llu], notifyNum[%u], "
1431 : "linkType[%d]",
1432 : __func__, newTag.c_str(), rankId, &head, curList, curList->nextDevice, notifyNum, linkType);
1433 0 : return HCCL_E_PARA;
1434 : }
1435 :
1436 0 : HCCL_INFO(
1437 : "[%s]newTag[%s], rankId[%u], head[%p], curList[%p], nextList[%llu], notifyNum[%u], linkType[%d]", __func__,
1438 : newTag.c_str(), rankId, &head, curList, curList->nextDevice, notifyNum, linkType);
1439 0 : if (linkType != TransportLinkType::RDMA) {
1440 : // 创建P2P链路
1441 0 : CHK_RET(InitLinkP2p(tagRes, rankId, newTag, notifyNum, linkType));
1442 : } else {
1443 : // 创建roce链路
1444 0 : CHK_RET(InitLinkRoce(tagRes, rankId, newTag, notifyNum));
1445 : // 创建roce链路（备用链路）
1446 0 : CHK_RET(InitLinkRoce(tagRes, rankId, newTag, notifyNum, true));
1447 : }
1448 0 : HCCL_INFO(
1449 : "[%s] End parse remote resource rankId[%u] tag[%s], newTag[%s]", __func__, rankId, identifier_.c_str(),
1450 : newTag.c_str());
1451 0 : return HCCL_SUCCESS;
1452 : }
1453 :
1454 0 : HcclResult HcclCommAicpu::RefreshTransportsResForRank(
1455 : const HcclOpResParam* commParam, u32 rankId, const std::string& newTag, u32 notifyNum, TransportLinkType linkType)
1456 : {
1457 0 : if (rankId >= AICPU_MAX_RANK_NUM) {
1458 0 : HCCL_ERROR(
1459 : "[%s] rankId[%u] overflow for group[%s], newTag[%s]", __func__, rankId, identifier_.c_str(),
1460 : newTag.c_str());
1461 0 : return HCCL_E_PARA;
1462 : }
1463 0 : if (commParam->remoteRes[rankId].nextDevicePtr == 0) {
1464 0 : return HCCL_SUCCESS;
1465 : }
1466 0 : HcclRankRelationResV2* rankRelationResPtr
1467 0 : = reinterpret_cast<HcclRankRelationResV2*>(commParam->remoteRes[rankId].nextDevicePtr);
1468 0 : if (rankRelationResPtr == nullptr) {
1469 0 : HCCL_ERROR(
1470 : "[%s]rank relation resource ptr is null, commParam->remoteRes[rankId].nextDevicePtr[%p],"
1471 : " rankId[%u]",
1472 : __func__, reinterpret_cast<HcclRankRelationResV2*>(commParam->remoteRes[rankId].nextDevicePtr), rankId);
1473 0 : return HCCL_E_PARA;
1474 : }
1475 :
1476 : // 1. init公共参数（对应remoteWorldRank，remoteUsrRankId暂不处理：windowsIn，windowsOut）
1477 0 : rankData_[rankId].remoteWorldRank = rankRelationResPtr->remoteWorldRank;
1478 0 : rankData_[rankId].remoteUsrRankId = rankRelationResPtr->remoteUsrRankId;
1479 : // 2. 遍历链表，获取HccltagRemoteResV2创建Tranport对象
1480 0 : if (reinterpret_cast<ListCommon*>(rankRelationResPtr->nextTagRes.nextDevice) != &(rankRelationResPtr->nextTagRes)) {
1481 0 : HCCL_DEBUG(
1482 : "[%s] Start to parse rankId[%u] tag resources, head[%p], nextDevice[%p], pre Device[%p], group[%s]",
1483 : __func__, rankId, &rankRelationResPtr->nextTagRes, rankRelationResPtr->nextTagRes.nextDevice,
1484 : rankRelationResPtr->nextTagRes.preDevice, identifier_.c_str());
1485 0 : CHK_RET(InitRemoteTagRes(rankId, rankRelationResPtr->nextTagRes, newTag, notifyNum, linkType));
1486 : } else {
1487 0 : HCCL_ERROR(
1488 : "[%s]could not find member in rankRelationRes list, rankId[%u], head[%p], nextDevice[%p]", __func__, rankId,
1489 : &rankRelationResPtr->nextTagRes, rankRelationResPtr->nextTagRes.nextDevice);
1490 0 : return HCCL_E_PARA;
1491 : }
1492 0 : HCCL_INFO(
1493 : "[%s] process success rankId[%u], group[%s], newTag[%s]", __func__, rankId, identifier_.c_str(),
1494 : newTag.c_str());
1495 0 : return HCCL_SUCCESS;
1496 : }
1497 :
1498 0 : HcclResult HcclCommAicpu::GetRdmaLinksByRankAndTag(
1499 : const HcclOpResParam* commParam, CommTransportsType type, u32 rankId, const std::string& newTag, LINK& link,
1500 : bool isBackup, u32 notifyNum, bool isSecond)
1501 : {
1502 0 : HCCL_INFO(
1503 : "[%s] Start to rdma get Link group[%s], rankId[%u], newTag[%s], isBackup[%d], deviceLogicId[%d],"
1504 : "notifyNum[%u], isSecond[%d]",
1505 : __func__, identifier_.c_str(), rankId, newTag.c_str(), isBackup, commParam->topoInfo.deviceLogicId, notifyNum,
1506 : isSecond);
1507 :
1508 0 : auto* linkRes = isBackup ? &linkRdmaResBackUp_ : &linkRdmaRes_;
1509 0 : auto iterRankLinks = linkRes->find(rankId);
1510 0 : if (iterRankLinks == linkRes->end() || iterRankLinks->second.find(newTag) == iterRankLinks->second.end()) {
1511 0 : HCCL_INFO(
1512 : "[%s] could not find link resource, rankId[%u], group[%s], newTag[%s]", __func__, rankId,
1513 : identifier_.c_str(), newTag.c_str());
1514 0 : CHK_RET(RefreshTransportsResForRank(commParam, rankId, newTag, notifyNum, TransportLinkType::RDMA));
1515 0 : iterRankLinks = linkRes->find(rankId);
1516 0 : if (iterRankLinks == linkRes->end() || iterRankLinks->second.find(newTag) == iterRankLinks->second.end()) {
1517 0 : HCCL_ERROR(
1518 : "[%s] refresh transport failed, newTag[%s], remoteUserRankId[%u]", __func__, newTag.c_str(), rankId);
1519 0 : return HCCL_E_INTERNAL;
1520 : }
1521 : }
1522 0 : if (isSecond && iterRankLinks->second[newTag].size() <= 1) {
1523 0 : HCCL_ERROR("[%s] get rdma Link failed, newTag[%s], remoteUserRankId[%u]", __func__, newTag.c_str(), rankId);
1524 0 : return HCCL_E_INTERNAL;
1525 : }
1526 :
1527 0 : link = isSecond ? iterRankLinks->second[newTag][1] : iterRankLinks->second[newTag][0];
1528 :
1529 0 : if (receivedAcks_.find(rankId) == receivedAcks_.end()) {
1530 0 : HCCL_ERROR("[%s]there is no link with rankId[%u]", __func__, rankId);
1531 0 : return HCCL_E_NOT_FOUND;
1532 : }
1533 0 : link->SetSupportDataReceivedAck(receivedAcks_[rankId]);
1534 0 : HCCL_DEBUG(
1535 : "[HcclCommAicpu][GetLinksByRankAndTag]rankid[%d] supportDataReceivedAck is %d", rankId, receivedAcks_[rankId]);
1536 0 : HCCL_INFO(
1537 : "[%s] group[%s], newTag[%s], rankId[%u] type[%u] success!", __func__, identifier_.c_str(), newTag.c_str(),
1538 : rankId, type);
1539 0 : return HCCL_SUCCESS;
1540 : }
1541 :
1542 0 : HcclResult HcclCommAicpu::GetSdmaLinksByRankAndTag(
1543 : const HcclOpResParam* commParam, CommTransportsType type, u32 rankId, const std::string& newTag, LINK& link,
1544 : bool isBackup, u32 notifyNum, TransportLinkType linkType)
1545 : {
1546 0 : HCCL_INFO(
1547 : "[%s] Start to get sdma Link group[%s], rankId[%u], newTag[%s], isBackup[%d], deviceLogicId[%d], "
1548 : "notifyNum[%u], linkType[%d]",
1549 : __func__, identifier_.c_str(), rankId, newTag.c_str(), isBackup, commParam->topoInfo.deviceLogicId, notifyNum,
1550 : linkType);
1551 :
1552 0 : auto* linkRes = &linkRes_;
1553 0 : if (linkType == TransportLinkType::SIO) { // HCCS SIO并发场景，HCCS链路与SIO链路分开管理
1554 0 : linkRes = &linkResSio_;
1555 : }
1556 0 : auto iterRankLinks = linkRes->find(rankId);
1557 0 : if (iterRankLinks == linkRes->end() || iterRankLinks->second.find(newTag) == iterRankLinks->second.end()) {
1558 0 : HCCL_INFO(
1559 : "[%s] could not find link resource, rankId[%u], group[%s], newTag[%s]", __func__, rankId,
1560 : identifier_.c_str(), newTag.c_str());
1561 0 : CHK_RET(RefreshTransportsResForRank(commParam, rankId, newTag, notifyNum, linkType));
1562 0 : iterRankLinks = linkRes->find(rankId);
1563 0 : if (iterRankLinks == linkRes->end() || iterRankLinks->second.find(newTag) == iterRankLinks->second.end()) {
1564 0 : HCCL_ERROR(
1565 : "[%s] refresh transport failed, newTag[%s], remoteUserRankId[%u], %p", __func__, newTag.c_str(), rankId,
1566 : linkRes);
1567 0 : return HCCL_E_INTERNAL;
1568 : }
1569 : }
1570 :
1571 0 : link = iterRankLinks->second[newTag];
1572 0 : if (receivedAcks_.find(rankId) == receivedAcks_.end()) {
1573 0 : HCCL_ERROR("[%s]there is no link with rankId[%u]", __func__, rankId);
1574 0 : return HCCL_E_NOT_FOUND;
1575 : }
1576 0 : link->SetSupportDataReceivedAck(receivedAcks_[rankId]);
1577 0 : HCCL_DEBUG(
1578 : "[HcclCommAicpu][GetLinksByRankAndTag]rankid[%d] supportDataReceivedAck is %d", rankId, receivedAcks_[rankId]);
1579 0 : HCCL_INFO(
1580 : "[%s] group[%s], newTag[%s], rankId[%u] type[%u] success!", __func__, identifier_.c_str(), newTag.c_str(),
1581 : rankId, type);
1582 0 : return HCCL_SUCCESS;
1583 : }
1584 :
1585 0 : HcclResult HcclCommAicpu::CleanRoceResource(
1586 : const std::string& newTag, AlgResourceResponse& algResResponse,
1587 : [[maybe_unused]] const std::map<u32, bool>& remoteRankPortMap, [[maybe_unused]] const OpParam& param)
1588 : {
1589 0 : HCCL_INFO("[%s] Entry alloc transport group[%s], tag[%s]", __func__, identifier_.c_str(), newTag.c_str());
1590 :
1591 0 : for (auto& levelNSubCommTransport : algResResponse.opTransportResponse) {
1592 0 : for (auto& singleSubCommTransport : levelNSubCommTransport) {
1593 0 : for (auto& transportRequest : singleSubCommTransport.transportRequests) {
1594 0 : if (transportRequest.isValid && transportRequest.isUsedRdma) {
1595 0 : u32 remoteUserRank = transportRequest.remoteUserRank;
1596 0 : linkRdmaRes_[remoteUserRank].erase(newTag);
1597 0 : linkRdmaResBackUp_[remoteUserRank].erase(newTag);
1598 : }
1599 : }
1600 : }
1601 : }
1602 :
1603 0 : return HCCL_SUCCESS;
1604 : }
1605 :
1606 1 : HcclResult HcclCommAicpu::CleanAllRoceResource()
1607 : {
1608 1 : HCCL_INFO("Clean all link rdna resources");
1609 : // 清空主链路资源
1610 1 : linkRdmaRes_.clear();
1611 : // 清空备链路资源
1612 1 : linkRdmaResBackUp_.clear();
1613 1 : return HCCL_SUCCESS;
1614 : }
1615 :
1616 2 : HcclResult HcclCommAicpu::ClearOpResource(const std::string& tag)
1617 : {
1618 : // 与 GetAlgResponseRes/AllocAlgResource/ReAllocTransportResource 互斥，避免读写竞争 resMap_/linkRes_
1619 2 : std::lock_guard<std::mutex> lock(preemptMutexForResMap_);
1620 :
1621 6 : auto eraseTagFromAll = [this](const std::string& t) -> bool {
1622 6 : bool hit = false;
1623 : // 主资源map：vector<LINK>析构会让shared_ptr<Transport>引用-1，归零时触发driver资源释放
1624 6 : if (resMap_.erase(t) > 0) {
1625 1 : hit = true;
1626 : }
1627 : // SDMA HCCS链路缓存：报错日志中driver halResAddrMap ioctl失败的根本累积点
1628 9 : for (auto& rankIt : linkRes_) {
1629 3 : if (rankIt.second.erase(t) > 0) {
1630 1 : hit = true;
1631 : }
1632 : }
1633 : // SDMA SIO并发链路缓存
1634 9 : for (auto& rankIt : linkResSio_) {
1635 3 : if (rankIt.second.erase(t) > 0) {
1636 1 : hit = true;
1637 : }
1638 : }
1639 : // RDMA主备链路缓存（CleanAllRoceResource按通信域全清，本接口按tag细粒度清）
1640 9 : for (auto& rankIt : linkRdmaRes_) {
1641 3 : if (rankIt.second.erase(t) > 0) {
1642 1 : hit = true;
1643 : }
1644 : }
1645 9 : for (auto& rankIt : linkRdmaResBackUp_) {
1646 3 : if (rankIt.second.erase(t) > 0) {
1647 1 : hit = true;
1648 : }
1649 : }
1650 : // 本地scratchMem和localTagResToObj按tag索引
1651 6 : if (tagScratchMem_.erase(t) > 0) {
1652 1 : hit = true;
1653 : }
1654 6 : if (localTagResToObj_.erase(t) > 0) {
1655 1 : hit = true;
1656 : }
1657 9 : for (auto& rankIt : rankTagRemoteRes_) {
1658 3 : if (rankIt.second.erase(t) > 0) {
1659 1 : hit = true;
1660 : }
1661 : }
1662 6 : return hit;
1663 2 : };
1664 :
1665 : // 与 host 端 ClearOpResource 对齐：尝试 base/_host/_device 三种后缀
1666 2 : bool found = false;
1667 2 : found = eraseTagFromAll(tag) || found;
1668 2 : found = eraseTagFromAll(tag + "_host") || found;
1669 2 : found = eraseTagFromAll(tag + "_device") || found;
1670 2 : if (!found) {
1671 1 : HCCL_WARNING(
1672 : "[HcclCommAicpu][ClearOpResource] no entries matched tag[%s] "
1673 : "(also tried _host/_device variants), group[%s]",
1674 : tag.c_str(), identifier_.c_str());
1675 : } else {
1676 1 : HCCL_INFO(
1677 : "[HcclCommAicpu][ClearOpResource] cleared resources for tag[%s], group[%s]", tag.c_str(),
1678 : identifier_.c_str());
1679 : }
1680 2 : return HCCL_SUCCESS;
1681 2 : }
1682 :
1683 : // 借轨重新刷新资源
1684 0 : HcclResult HcclCommAicpu::ReAllocTransportResource(
1685 : const std::string& newTag, AlgResourceResponse& algResResponse, std::map<u32, bool>& remoteRankPortMap,
1686 : const HcclOpResParam* commParam, const OpParam& param)
1687 : {
1688 0 : HCCL_INFO("[%s] Entry alloc transport group[%s], tag[%s]", __func__, identifier_.c_str(), newTag.c_str());
1689 0 : std::set<u32> bsrTansportRank;
1690 0 : for (auto& levelNSubCommTransport : algResResponse.opTransportResponse) {
1691 0 : for (auto& singleSubCommTransport : levelNSubCommTransport) {
1692 0 : singleSubCommTransport.links.clear();
1693 0 : singleSubCommTransport.links.reserve(singleSubCommTransport.transportRequests.size());
1694 0 : for (auto& transportRequest : singleSubCommTransport.transportRequests) {
1695 0 : singleSubCommTransport.links.push_back(nullptr);
1696 0 : if (transportRequest.isValid) {
1697 0 : HCCL_INFO(
1698 : "[%s] alloc transport, newTag[%s], rankId[%u], "
1699 : "input memory type[%u], output memory type[%u], ",
1700 : __func__, newTag.c_str(), transportRequest.remoteUserRank, transportRequest.inputMemType,
1701 : transportRequest.outputMemType);
1702 0 : receivedAcks_[transportRequest.remoteUserRank] = singleSubCommTransport.supportDataReceivedAck;
1703 0 : bool isBackup = remoteRankPortMap.find(transportRequest.remoteUserRank) != remoteRankPortMap.end()
1704 0 : && !remoteRankPortMap[transportRequest.remoteUserRank];
1705 0 : bool isSecondBuild = false;
1706 0 : bool isBatchSendRecv = newTag.find("BatchSendRecv") != std::string::npos;
1707 0 : if (transportRequest.isUsedRdma && isBatchSendRecv
1708 0 : && bsrTansportRank.find(transportRequest.remoteUserRank) != bsrTansportRank.end()) {
1709 : // 仅在batchsendrecv rdma下发的时候需要第二次刷新，实际第一次下发都刷好了，第二次就是get一下
1710 0 : isSecondBuild = true;
1711 : }
1712 : // A3 bsr远端是DirectNpu 链路的话则跳过
1713 0 : if ((param.opType == HcclCMDType::HCCL_CMD_BATCH_SEND_RECV)
1714 0 : && (param.BatchSendRecvDataDes.isDirectRemoteRank[transportRequest.remoteUserRank])) {
1715 0 : continue;
1716 : }
1717 0 : bsrTansportRank.insert(transportRequest.remoteUserRank);
1718 0 : CHK_RET(CreateLink(
1719 : newTag, transportRequest, commParam, singleSubCommTransport.links.back(),
1720 : transportRequest.notifyNum, isBackup, isSecondBuild));
1721 : }
1722 : }
1723 : }
1724 : }
1725 :
1726 0 : return HCCL_SUCCESS;
1727 0 : }
1728 :
1729 0 : HcclResult HcclCommAicpu::AllocTransportResource(
1730 : const std::string& newTag, const OpParam& opParam, const HcclOpResParam* commParam, AlgResourceRequest& resRequest,
1731 : AlgResourceResponse& algResResponse)
1732 : {
1733 0 : HCCL_INFO("[%s] Entry alloc transport group[%s]", __func__, identifier_.c_str());
1734 0 : algResResponse.opTransportResponse = resRequest.opTransport;
1735 :
1736 0 : std::set<u32> bsrTansportRank;
1737 0 : for (auto& levelNSubCommTransport : algResResponse.opTransportResponse) {
1738 0 : for (auto& singleSubCommTransport : levelNSubCommTransport) {
1739 0 : singleSubCommTransport.links.clear();
1740 0 : singleSubCommTransport.links.reserve(singleSubCommTransport.transportRequests.size());
1741 0 : for (auto& transportRequest : singleSubCommTransport.transportRequests) {
1742 0 : singleSubCommTransport.links.push_back(nullptr);
1743 0 : if (transportRequest.isValid) {
1744 0 : localUserRank_ = transportRequest.localUserRank;
1745 0 : receivedAcks_[transportRequest.remoteUserRank] = singleSubCommTransport.supportDataReceivedAck;
1746 0 : HCCL_DEBUG(
1747 : "[%s] alloc transport, newTag[%s], rankId[%u], input memory type[%u], "
1748 : "output memory type[%u], ",
1749 : __func__, newTag.c_str(), transportRequest.remoteUserRank, transportRequest.inputMemType,
1750 : transportRequest.outputMemType);
1751 :
1752 0 : bool isSecondBuild = false;
1753 0 : if (transportRequest.isUsedRdma && opParam.opType == HcclCMDType::HCCL_CMD_BATCH_SEND_RECV
1754 0 : && bsrTansportRank.find(transportRequest.remoteUserRank) != bsrTansportRank.end()) {
1755 : // 仅仅在batchsendrecv rdma下发的时候需要第二次刷新，实际第一次下发都刷好了，第二次就是get一下
1756 0 : isSecondBuild = true;
1757 : }
1758 : // A3 bsr远端是DirectNpu 链路的话则跳过
1759 0 : if ((opParam.opType == HcclCMDType::HCCL_CMD_BATCH_SEND_RECV)
1760 0 : && (opParam.BatchSendRecvDataDes.isDirectRemoteRank[transportRequest.remoteUserRank])) {
1761 0 : continue;
1762 : }
1763 0 : bsrTansportRank.insert(transportRequest.remoteUserRank);
1764 0 : CHK_RET(CreateLink(
1765 : newTag, transportRequest, commParam, singleSubCommTransport.links.back(),
1766 : transportRequest.notifyNum, false, isSecondBuild));
1767 : }
1768 : }
1769 : }
1770 : }
1771 :
1772 0 : return HCCL_SUCCESS;
1773 0 : }
1774 :
1775 : // 在resMap_[tag]对应原有通信资源的基础上继续增量建链，目前batchsendrecv会用到本接口
1776 0 : HcclResult HcclCommAicpu::IncreAllocTransportResource(
1777 : const std::string& newTag, const OpParam& opParam, const HcclOpResParam* commParam, AlgResourceRequest& resRequest,
1778 : AlgResourceResponse& algResResponse)
1779 : {
1780 0 : HCCL_INFO("[HcclCommAicpu][IncreAllocTransportResource] Entry alloc transport group[%s]", identifier_.c_str());
1781 0 : std::set<u32> bsrTansportRank;
1782 0 : for (u32 levelIndex = 0; levelIndex < resRequest.opTransport.size(); levelIndex++) {
1783 0 : for (u32 ringIndex = 0; ringIndex < resRequest.opTransport[levelIndex].size(); ringIndex++) {
1784 0 : SingleSubCommTransport& reqSingleSubComm = resRequest.opTransport[levelIndex][ringIndex];
1785 0 : SingleSubCommTransport& respSingleSubComm = algResResponse.opTransportResponse[levelIndex][ringIndex];
1786 0 : for (u32 rankIndex = 0; rankIndex < reqSingleSubComm.transportRequests.size(); rankIndex++) {
1787 0 : TransportRequest& transportRequest = reqSingleSubComm.transportRequests[rankIndex];
1788 0 : CHK_PRT_RET(
1789 : rankIndex >= respSingleSubComm.links.size(),
1790 : HCCL_ERROR(
1791 : "[HcclCommAicpu][IncreAllocTransportResource] The remote rank_id[%u] is larger than "
1792 : "the existent respSingleSubComm map size[%u]",
1793 : rankIndex, respSingleSubComm.links.size()),
1794 : HCCL_E_PARA);
1795 0 : if (respSingleSubComm.links[rankIndex] != nullptr
1796 0 : && respSingleSubComm.links[rankIndex]->GetLinkType() != hccl::LinkType::LINK_RESERVED) {
1797 0 : HCCL_INFO(
1798 : "[IncreAlloc] The link to remote userRank[%u] has existed", transportRequest.remoteUserRank);
1799 0 : continue;
1800 : }
1801 0 : if (transportRequest.isValid) {
1802 0 : receivedAcks_[transportRequest.remoteUserRank] = reqSingleSubComm.supportDataReceivedAck;
1803 0 : respSingleSubComm.transportRequests[rankIndex] = transportRequest;
1804 0 : HCCL_DEBUG(
1805 : "[HcclCommAicpu][IncreAllocTransportResource] alloc transport, newTag[%s], rankId[%u], "
1806 : "input memory type[%u], output memory type[%u], ",
1807 : newTag.c_str(), transportRequest.remoteUserRank, transportRequest.inputMemType,
1808 : transportRequest.outputMemType);
1809 0 : bool isSecondBuild = false;
1810 0 : if (transportRequest.isUsedRdma && opParam.opType == HcclCMDType::HCCL_CMD_BATCH_SEND_RECV
1811 0 : && bsrTansportRank.find(transportRequest.remoteUserRank) != bsrTansportRank.end()) {
1812 : // 仅仅在batchsendrecv rdma下发的时候需要第二次刷新，实际第一次下发都刷好了，第二次就是get一下
1813 0 : isSecondBuild = true;
1814 : }
1815 : // A3 bsr远端是DirectNpu 链路的话则跳过
1816 0 : if ((opParam.opType == HcclCMDType::HCCL_CMD_BATCH_SEND_RECV)
1817 0 : && (opParam.BatchSendRecvDataDes.isDirectRemoteRank[transportRequest.remoteUserRank])) {
1818 0 : continue;
1819 : }
1820 0 : bsrTansportRank.insert(transportRequest.remoteUserRank);
1821 0 : CHK_RET(CreateLink(
1822 : newTag, transportRequest, commParam, respSingleSubComm.links[rankIndex],
1823 : transportRequest.notifyNum, false, isSecondBuild));
1824 : }
1825 : }
1826 : }
1827 : }
1828 0 : return HCCL_SUCCESS;
1829 0 : }
1830 :
1831 0 : HcclResult HcclCommAicpu::CreateLink(
1832 : const std::string& newTag, TransportRequest& transportRequest, const HcclOpResParam* commParam, LINK& link,
1833 : u32 notifyNum, bool isBackup, bool isSecond) // 主备的选择
1834 : {
1835 0 : if (transportRequest.isUsedRdma) {
1836 0 : CHK_RET(GetRdmaLinksByRankAndTag(
1837 : commParam, CommTransportsType::SPECIAL, transportRequest.remoteUserRank, newTag, link, isBackup, notifyNum,
1838 : isSecond));
1839 : } else {
1840 0 : CHK_RET(GetSdmaLinksByRankAndTag(
1841 : commParam, CommTransportsType::SPECIAL, transportRequest.remoteUserRank, newTag, link, isBackup, notifyNum,
1842 : transportRequest.linkType));
1843 : }
1844 :
1845 0 : HCCL_DEBUG("[%s] alloc special transport success!, tag[%s]", __func__, newTag.c_str());
1846 0 : return HCCL_SUCCESS;
1847 : }
1848 :
1849 0 : HcclResult HcclCommAicpu::AllocLocalNotifysResource(
1850 : const std::string& newTag, const HcclOpResParam* commParam, const u32 notifyNum,
1851 : std::vector<std::shared_ptr<LocalNotify>>& notifiesMain, std::vector<std::shared_ptr<LocalNotify>>& notifiesAux)
1852 : {
1853 0 : HCCL_INFO(
1854 : "[HcclCommAicpu][AllocLocalNotifysResource]requesting for [%u] notifies, tag[%s].", notifyNum, newTag.c_str());
1855 0 : if (localNotifies_.capacity() < notifyNum) {
1856 0 : HCCL_ERROR(
1857 : "[HcclCommAicpu][AllocLocalNotifysResource]request number exceeds max notify number, alloc failed. Max "
1858 : "number is [%u],request num[%u], tag[%s]",
1859 : localNotifies_.capacity(), notifyNum, newTag.c_str());
1860 0 : return HCCL_E_PARA;
1861 : }
1862 :
1863 0 : if (localNotifies_.size() < notifyNum) {
1864 0 : if (InitLocalNotifyObj(commParam) != HCCL_SUCCESS || localNotifies_.size() < notifyNum) {
1865 0 : HCCL_ERROR(
1866 : "[HcclCommAicpu][AllocLocalNotifysResource] the need of notify is more than the available, group[%s], "
1867 : "need[%u], total[%u]",
1868 : newTag.c_str(), notifyNum, localNotifies_.size());
1869 0 : return HCCL_E_INTERNAL;
1870 : }
1871 : }
1872 :
1873 0 : u32 halfNotifyNum = notifyNum >> 1;
1874 0 : notifiesMain.resize(halfNotifyNum);
1875 0 : notifiesAux.resize(halfNotifyNum);
1876 0 : for (u32 i = 0; i < halfNotifyNum; i++) {
1877 0 : notifiesMain[i] = localNotifies_[i << 1];
1878 0 : notifiesAux[i] = localNotifies_[(i << 1) + 1];
1879 : }
1880 0 : HCCL_INFO(
1881 : "[HcclCommAicpu][AllocLocalNotifysResource]find enough notifies, numbers[%u], tag[%s].", notifyNum,
1882 : newTag.c_str());
1883 0 : return HCCL_SUCCESS;
1884 : }
1885 :
1886 0 : HcclResult HcclCommAicpu::AllocStreamsResource(
1887 : const std::string& newTag, const HcclOpResParam* commParam, const u32 streamNum, std::vector<Stream>& streams)
1888 : {
1889 0 : HCCL_INFO(
1890 : "[HcclCommAicpu][AllocStreamsResource]requesting for [%u] slave streams, newTag[%s], group[%s].", streamNum,
1891 : newTag.c_str(), identifier_.c_str());
1892 0 : if (streamNum == 0) {
1893 0 : return HCCL_SUCCESS;
1894 : }
1895 0 : if (slaveStreams_.capacity() < streamNum) {
1896 0 : HCCL_ERROR(
1897 : "[HcclCommAicpu][AllocStreamsResource]request number exceeds max substream number, alloc failed. Max "
1898 : "number is [%u],request num[%u], tag[%s]",
1899 : slaveStreams_.capacity(), streamNum, newTag.c_str());
1900 0 : return HCCL_E_PARA;
1901 : }
1902 0 : if (slaveStreams_.size() < streamNum) {
1903 0 : if (InitSlaveStreamObjs(commParam) != HCCL_SUCCESS || slaveStreams_.size() < streamNum) {
1904 0 : HCCL_ERROR(
1905 : "[HcclCommAicpu][AllocStreamsResource] the need of streams is more than the "
1906 : "available, tag[%s], need[%u], total[%u]",
1907 : newTag.c_str(), streamNum, slaveStreams_.size());
1908 0 : return HCCL_E_INTERNAL;
1909 : }
1910 : }
1911 0 : streams = std::vector<Stream>(slaveStreams_.begin(), slaveStreams_.begin() + streamNum);
1912 0 : HCCL_INFO(
1913 : "[HcclCommAicpu][AllocStreamsResource]find enough slave streams [%u], tag[%s].", streamNum, newTag.c_str());
1914 0 : return HCCL_SUCCESS;
1915 : }
1916 :
1917 0 : HcclResult HcclCommAicpu::AllocScratchMemResource(
1918 : const std::string& newTag, const HcclOpResParam* commParam, const u64& scratchMemSize, DeviceMem& scratchMem,
1919 : bool reAllocFlag)
1920 : {
1921 0 : HCCL_INFO(
1922 : "[HcclCommAicpu][AllocScratchMemResource]requesting for [%llu] bytes scratch mem, tag[%s].", scratchMemSize,
1923 : newTag.c_str());
1924 0 : if (scratchMemSize != 0) {
1925 0 : if (tagScratchMem_.find(newTag) == tagScratchMem_.end()) {
1926 0 : HcclResult ret = InitLocalTagRes(commParam->localRes.nextTagRes);
1927 0 : CHK_PRT_RET(
1928 : ret != HCCL_SUCCESS,
1929 : HCCL_ERROR(
1930 : "[HcclCommAicpu][AllocScratchMemResource]InitLocalTagRes error group[%s]", identifier_.c_str()),
1931 : ret);
1932 : }
1933 :
1934 0 : if (tagScratchMem_.find(newTag) == tagScratchMem_.end()) {
1935 0 : HCCL_ERROR(
1936 : "[HcclCommAicpu][AllocScratchMemResource]alloc scratch memory failed."
1937 : "requesting for [%llu] bytes, tag[%s].",
1938 : scratchMemSize, newTag.c_str());
1939 0 : return HCCL_E_NOT_FOUND;
1940 : }
1941 :
1942 0 : if (reAllocFlag) {
1943 0 : HCCL_INFO(
1944 : "[HcclCommAicpu][AllocScratchMemResource]need to reAlloc mem [%llu] bytes scratch mem, tag[%s].",
1945 : scratchMemSize, newTag.c_str());
1946 0 : HcclResult ret = InitLocalTagRes(commParam->localRes.nextTagRes, reAllocFlag);
1947 0 : CHK_PRT_RET(
1948 : ret != HCCL_SUCCESS,
1949 : HCCL_ERROR(
1950 : "[HcclCommAicpu][AllocScratchMemResource]InitLocalTagRes error group[%s]", identifier_.c_str()),
1951 : ret);
1952 : }
1953 :
1954 : // 因为aicpu_communicator中会对scratchMem做对齐，所以tagScratchMem_中的大小会偏小（被对齐截断一部分）
1955 : // 但是两者的差值应该不能超过2个CCE_REDUCE_ALIGN_SIZE，否则应该是不对的
1956 0 : if (scratchMemSize - tagScratchMem_[newTag]->size() > (CCE_REDUCE_ALIGN_SIZE + CCE_REDUCE_ALIGN_SIZE)) {
1957 0 : HCCL_ERROR(
1958 : "[HcclCommAicpu][AllocScratchMemResource]alloc tag[%s] scratch memory failed."
1959 : "requesting [%llu] bytes actual [%llu] bytes",
1960 : newTag.c_str(), scratchMemSize, tagScratchMem_[newTag]->size());
1961 0 : return HCCL_E_PARA;
1962 : }
1963 0 : scratchMem = DeviceMem::create(tagScratchMem_[newTag]->ptr(), tagScratchMem_[newTag]->size());
1964 : }
1965 0 : HCCL_INFO(
1966 : "[HcclCommAicpu][AllocScratchMemResource]find enough [%llu] bytes scratch mem, tag[%s].", scratchMemSize,
1967 : newTag.c_str());
1968 0 : return HCCL_SUCCESS;
1969 : }
1970 :
1971 0 : HcclResult HcclCommAicpu::AllocAlgResource(
1972 : const std::string& newTag, const OpParam& opParam, const HcclOpResParam* commParam, AlgResourceRequest& resRequest,
1973 : AlgResourceResponse& algResResponse)
1974 : {
1975 0 : algResResponse.cclInputMem = cclInputBuffer_;
1976 0 : algResResponse.cclOutputMem = cclOutputBuffer_;
1977 0 : algResResponse.paramInputMem = DeviceMem::create(opParam.inputPtr, opParam.inputSize);
1978 0 : algResResponse.paramOutputMem = DeviceMem::create(opParam.outputPtr, opParam.outputSize);
1979 :
1980 0 : PetersonLockGuard guard(hostDeviceLock_.get());
1981 0 : CHK_PRT_RET(
1982 : guard.IsLockFailed(), HCCL_ERROR("[HcclCommAicpu][AllocAlgResource] hostDeviceLock lock failed"),
1983 : HCCL_E_INTERNAL);
1984 :
1985 0 : CHK_RET(AllocScratchMemResource(newTag, commParam, resRequest.scratchMemSize, algResResponse.scratchMem));
1986 0 : CHK_RET(AllocStreamsResource(newTag, commParam, resRequest.streamNum, algResResponse.slaveStreams));
1987 0 : CHK_RET(AllocLocalNotifysResource(
1988 : newTag, commParam, resRequest.notifyNum, algResResponse.notifiesMain, algResResponse.notifiesAux));
1989 0 : CHK_RET(AllocTransportResource(newTag, opParam, commParam, resRequest, algResResponse));
1990 0 : HCCL_INFO("[HcclCommAicpu][AllocAlgResource] alloc resource success tag[%s].", newTag.c_str());
1991 0 : return HCCL_SUCCESS;
1992 0 : }
1993 :
1994 0 : HcclResult HcclCommAicpu::CalcResRequest(
1995 : const std::string& algName, const OpParam& param, std::unique_ptr<CollExecutorBase>& executor,
1996 : AlgResourceRequest& resourceRequest)
1997 : {
1998 0 : if (executor.get() == nullptr) {
1999 0 : executor = CollAlgExecRegistry::Instance().GetAlgExec(algName, dispatcher_, topoMatcher_);
2000 0 : CHK_PRT_RET(
2001 : executor.get() == nullptr,
2002 : HCCL_ERROR("[HcclCommAicpu][CalcResRequest]Fail to find executor for algName[%s]", algName.c_str()),
2003 : HCCL_E_PARA);
2004 0 : executor->SetAlgType(algType_);
2005 0 : executor->SetCCLInBuffer(cclbufferSize_);
2006 :
2007 0 : if (param.opType == HcclCMDType::HCCL_CMD_REDUCE_SCATTER) {
2008 0 : bool isSupportSDMAReduce = false;
2009 0 : if (GetWorkflowMode() != HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE) {
2010 : isSupportSDMAReduce
2011 0 : = IsSupportSDMAReduce(param.inputPtr, param.outputPtr, param.DataDes.dataType, param.reduceType);
2012 : } else {
2013 0 : isSupportSDMAReduce = IsSupportSDMAReduce(
2014 0 : cclInputBuffer_.ptr(), cclOutputBuffer_.ptr(), param.DataDes.dataType, param.reduceType);
2015 : }
2016 0 : executor->SetIsSupportSDMAReduce(isSupportSDMAReduce);
2017 : }
2018 : }
2019 0 : CHK_RET(CalSendRecvInfoFor910B(algName, param, executor));
2020 0 : return executor->CalcResRequest(param, resourceRequest);
2021 : }
2022 :
2023 0 : u32 HcclCommAicpu::CalculateOpExecIndex(const OpParam& opParam, u32 userRank)
2024 : {
2025 0 : u32 opIndex = 0;
2026 0 : s32 commIndex = 0;
2027 : // 用于重执行和taskException打印的算子计数，bsr/sendrecv/其他算子分别计数
2028 0 : if (opParam.opType == HcclCMDType::HCCL_CMD_BATCH_SEND_RECV) {
2029 0 : commIndex = -1; // batchSendRecv使用 key = -1
2030 0 : } else if (opParam.opType == HcclCMDType::HCCL_CMD_SEND) {
2031 0 : commIndex = opParam.dstRank;
2032 0 : } else if (opParam.opType == HcclCMDType::HCCL_CMD_RECEIVE) {
2033 0 : commIndex = opParam.srcRank;
2034 : } else {
2035 0 : commIndex = userRank;
2036 : }
2037 :
2038 0 : auto it = opExecIndexMap_.find(commIndex);
2039 0 : if (it != opExecIndexMap_.end()) {
2040 0 : opIndex = ++(it->second);
2041 : } else {
2042 0 : opExecIndexMap_.insert({commIndex, 1});
2043 0 : opIndex = 1;
2044 : }
2045 :
2046 0 : HCCL_DEBUG(
2047 : "%s tag:%s opType:%u commIndex:%d opIndex:%u", __func__, opParam.tag.c_str(), opParam.opType, commIndex,
2048 : opIndex);
2049 0 : return opIndex;
2050 : }
2051 :
2052 2 : HcclResult HcclCommAicpu::PrepareSymmetricMemory(const OpParam& param, OpCommTransport& opTransportResponse)
2053 : {
2054 2 : CHK_PRT_RET(
2055 : opTransportResponse.size() == 0,
2056 : HCCL_ERROR("[HcclCommAicpu][PrepareSymmetricMemory] opTransportResponse size is 0"), HCCL_E_PARA);
2057 :
2058 : const std::unordered_set<LinkType> supportedLinkTypes
2059 2 : = {LinkType::LINK_HCCS, LinkType::LINK_SIO, LinkType::LINK_HCCS_SW};
2060 2 : for (u32 levelIdx = 0; levelIdx < opTransportResponse.size(); levelIdx++) {
2061 2 : for (auto& singleSubCommTransport : opTransportResponse[levelIdx]) {
2062 1 : if (singleSubCommTransport.isZeroCopy == false) {
2063 1 : continue;
2064 : }
2065 0 : for (u64 i = 0; i < singleSubCommTransport.links.size(); ++i) {
2066 0 : LINK& link = singleSubCommTransport.links[i];
2067 0 : if (link == nullptr || !singleSubCommTransport.transportRequests[i].isValid
2068 0 : || supportedLinkTypes.count(link->GetLinkType()) == 0) {
2069 0 : continue; // 无效或者不支持的链路
2070 : }
2071 0 : u32 peerRank = link->GetRemoteRank();
2072 0 : void* remoteIn = nullptr;
2073 0 : CHK_RET(HcclSymWinGetPeerPointer(param.inputSymWindow, param.inputOffset, peerRank, &remoteIn));
2074 0 : void* remoteOut = nullptr;
2075 0 : CHK_RET(HcclSymWinGetPeerPointer(param.outputSymWindow, param.outputOffset, peerRank, &remoteOut));
2076 :
2077 0 : CHK_PRT_RET(
2078 : remoteIn == nullptr || remoteOut == nullptr,
2079 : HCCL_ERROR(
2080 : "[HcclCommAicpu][PrepareSymmetricMemory] remoteRank[%d] in[%p] out[%p] is invalid", peerRank,
2081 : remoteIn, remoteOut),
2082 : HCCL_E_INTERNAL);
2083 0 : HCCL_INFO(
2084 : "[HcclCommAicpu][PrepareSymmetricMemory] remoteRank[%d] in[%p] out[%p]", peerRank, remoteIn,
2085 : remoteOut);
2086 0 : CHK_RET(link->UpdateRemoteAddr(remoteIn, remoteOut));
2087 : }
2088 : }
2089 : }
2090 1 : return HCCL_SUCCESS;
2091 1 : }
2092 :
2093 1 : HcclResult HcclCommAicpu::ExecOp(
2094 : const std::string& newTag, const std::string& algName, OpParam& opParam, const HcclOpResParam* commParam)
2095 : {
2096 1 : std::unique_ptr<CollExecutorBase> executor;
2097 : hccl::AlgResourceResponse* algResResponse;
2098 1 : CHK_RET(GetAlgResponseRes(newTag, algName, opParam, commParam, executor, algResResponse));
2099 :
2100 1 : if (isZeroCopy_ || isSymmetricMemory_) {
2101 1 : if (isSymmetricMemory_) {
2102 1 : CHK_RET(PrepareSymmetricMemory(opParam, algResResponse->opTransportResponse));
2103 : } else {
2104 0 : HcclResult ret = PrepareZeroCopyExchanger(newTag, opParam, algResResponse);
2105 0 : CHK_PRT_RET(
2106 : ret != HCCL_SUCCESS,
2107 : HCCL_ERROR(
2108 : "[HcclCommAicpu][ExecOp] newTag[%s], localRankId[%u]", newTag.c_str(), commParam->localUsrRankId),
2109 : ret);
2110 : }
2111 :
2112 : // 零拷贝场景scratchMem的大小会与用户的输入大小不同，会导致后续算法展开模块计算出错
2113 : // 但是该场景下不会直接访问scratchMem，因此直接使用输入作为scratchMem，使得后续计算正确
2114 0 : if (opParam.opType == HcclCMDType::HCCL_CMD_REDUCE_SCATTER) {
2115 0 : algResResponse->scratchMem = DeviceMem::create(opParam.inputPtr, opParam.inputSize);
2116 0 : HCCL_INFO(
2117 : "[HcclCommAicpu][ExecOp] ZeroCopy reduce-scatter use userInput as scratchMem, inputPtr[%p] "
2118 : "inputSize[%lu]",
2119 : opParam.inputPtr, opParam.inputSize);
2120 : }
2121 :
2122 0 : algResResponse->paramInputMem = DeviceMem::create(opParam.inputPtr, opParam.inputSize);
2123 0 : algResResponse->paramOutputMem = DeviceMem::create(opParam.outputPtr, opParam.outputSize);
2124 0 : HCCL_INFO(
2125 : "[HcclCommAicpu][ExecOp] zero copy modify paramInput paramOutput to algResResp inputPtr[%p] inputSize[%lu] "
2126 : "outputPtr[%p] outputSize[%lu]",
2127 : algResResponse->paramInputMem.ptr(), algResResponse->paramInputMem.size(),
2128 : algResResponse->paramOutputMem.ptr(), algResResponse->paramOutputMem.size());
2129 : }
2130 :
2131 0 : hcclOpExecIndex_ = CalculateOpExecIndex(opParam, localUserRank_);
2132 0 : HcclResult ret = Orchestrate(newTag, algName, opParam, executor, *algResResponse, commParam);
2133 0 : if (ret != HCCL_SUCCESS) {
2134 0 : HCCL_ERROR(
2135 : "[HcclCommAicpu][ExecOp] executor op fail, tag[%s], algName[%s], identifier[%s]", newTag.c_str(),
2136 : algName.c_str(), identifier_.c_str());
2137 0 : CHK_PRT_CONT(
2138 : retryEnable_, HCCL_ERROR("[HcclCommAicpu][ExecOp] executor op fail, some error logs may be recorded in the "
2139 : "log/run/device directory, search keyword [ErrToWarn]"));
2140 0 : if (printTaskExceptionForErr_) {
2141 0 : PrintTaskExceptionAllComm();
2142 0 : PrintAicpuCommExecStatus();
2143 0 : printTaskExceptionForErr_ = false;
2144 : }
2145 0 : return ret;
2146 : }
2147 :
2148 0 : HCCL_ENTRY_INFO(
2149 : commParam->opEntry,
2150 : "[HcclCommAicpu][ExecOp] executor op success tag[%s], newTag[%s], algName[%s], identifier[%s].",
2151 : opParam.tag.c_str(), newTag.c_str(), algName.c_str(), identifier_.c_str());
2152 0 : return HCCL_SUCCESS;
2153 1 : }
2154 :
2155 0 : HcclResult HcclCommAicpu::RefreshAlgResponseTransportRes(
2156 : const std::string& newTag, AlgResourceResponse& algResResponse, std::map<u32, bool>& remoteRankPortMap,
2157 : bool isChangeLinkFlag, const HcclOpResParam* commParam, const OpParam& param)
2158 : {
2159 0 : CHK_PTR_NULL(commParam);
2160 0 : auto iter = resMap_.find(newTag);
2161 0 : CHK_PRT_RET(
2162 : iter == resMap_.end(), HCCL_ERROR("[%s]Fail to find algResResponse for tag[%s]", __func__, newTag.c_str()),
2163 : HCCL_E_PARA);
2164 :
2165 0 : PetersonLockGuard guard(hostDeviceLock_.get());
2166 0 : CHK_PRT_RET(guard.IsLockFailed(), HCCL_ERROR("[%s] hostDeviceLock lock failed", __func__), HCCL_E_INTERNAL);
2167 0 : if (!isChangeLinkFlag) {
2168 0 : CleanRoceResource(newTag, algResResponse, remoteRankPortMap, param);
2169 0 : CHK_RET(ReAllocTransportResource(newTag, algResResponse, remoteRankPortMap, commParam, param));
2170 0 : HCCL_RUN_INFO("[%s] ChangeLinkFlag[%d], current tag[%s].", __func__, isChangeLinkFlag, newTag.c_str());
2171 : } else {
2172 : // 提前清理所有tag的链路，避免冲突
2173 0 : for (auto& resMapIt : resMap_) {
2174 0 : HCCL_RUN_INFO("[%s] clean roce resource of tag[%s].", __func__, resMapIt.first.c_str());
2175 0 : CleanRoceResource(resMapIt.first, resMapIt.second, remoteRankPortMap, param);
2176 : }
2177 : // 对resMap中所有tag的transport link根据主备进行刷新
2178 0 : for (auto& resMapIt : resMap_) {
2179 0 : HCCL_RUN_INFO("[%s] refresh algResResponse of tag[%s].", __func__, resMapIt.first.c_str());
2180 0 : CHK_RET(ReAllocTransportResource(resMapIt.first, resMapIt.second, remoteRankPortMap, commParam, param));
2181 0 : if (resMapIt.first == newTag) {
2182 0 : HCCL_RUN_INFO("[%s] current tag[%s].", __func__, newTag.c_str());
2183 0 : algResResponse = resMapIt.second;
2184 : }
2185 : }
2186 : }
2187 :
2188 0 : HCCL_RUN_INFO("[%s] alloc resource success tag[%s].", __func__, newTag.c_str());
2189 0 : return HCCL_SUCCESS;
2190 0 : }
2191 :
2192 0 : HcclResult HcclCommAicpu::CalSendRecvInfoForAlltoall(const OpParam& param)
2193 : {
2194 0 : if (param.opType == HcclCMDType::HCCL_CMD_ALLTOALL) {
2195 : std::vector<u64> sendCountMatrix(
2196 0 : topoInfo_.userRankSize * topoInfo_.userRankSize, param.All2AllDataDes.sendCount);
2197 0 : CHK_RET(GetAlltoAllvcSendRecvInfo(
2198 : static_cast<void*>(sendCountMatrix.data()), param.All2AllDataDes.sendType, param.All2AllDataDes.recvType));
2199 0 : } else if (param.opType == HcclCMDType::HCCL_CMD_ALLTOALLV) {
2200 0 : CHK_PTR_NULL(sendRecvInfoPtr_);
2201 0 : CHK_RET(
2202 : GetAlltoAllvSendRecvInfo(sendRecvInfoPtr_, param.All2AllDataDes.sendType, param.All2AllDataDes.recvType));
2203 : } else {
2204 0 : CHK_RET(GetAlltoAllvcSendRecvInfo(
2205 : param.All2AllDataDes.sendCountMatrix, param.All2AllDataDes.sendType, param.All2AllDataDes.recvType));
2206 : }
2207 0 : return HCCL_SUCCESS;
2208 : }
2209 :
2210 0 : HcclResult HcclCommAicpu::CalSendRecvInfoFor910B(
2211 : const std::string& algName, const OpParam& param, std::unique_ptr<CollExecutorBase>& executor)
2212 : {
2213 : // A2 AICPU才有机会走入RunAlltoAllVStaged
2214 0 : if (algName == "RunAlltoAllVStaged" || algName == "RunAlltoAllVFullMesh") {
2215 0 : CHK_PRT_RET(
2216 : executor.get() == nullptr,
2217 : HCCL_ERROR("[HcclCommAicpu][%s]Fail to find executor for algName[%s]", __func__, algName.c_str()),
2218 : HCCL_E_PARA);
2219 0 : CHK_RET(CalSendRecvInfoForAlltoall(param));
2220 0 : CollAlltoAllExecutor* alltoAllExecutor = dynamic_cast<CollAlltoAllExecutor*>(executor.get());
2221 0 : CHK_PTR_NULL(alltoAllExecutor);
2222 0 : CHK_RET(alltoAllExecutor->SetExcutorExtraInfo(allMeshAggregationSendRecvInfo_, cclbufferSize_));
2223 0 : HCCL_DEBUG("[HcclCommAicpu][%s] running algName[%s], prepare SendRecvInfo.", __func__, algName.c_str());
2224 0 : return HCCL_SUCCESS;
2225 : }
2226 0 : return HCCL_SUCCESS;
2227 : }
2228 :
2229 0 : HcclResult HcclCommAicpu::GetAlgResponseRes(
2230 : const std::string& newTag, const std::string& algName, const OpParam& opParam, const HcclOpResParam* commParam,
2231 : std::unique_ptr<CollExecutorBase>& executor, AlgResourceResponse*& algResResponse)
2232 : {
2233 0 : HCCL_INFO("[%s] algName[%s]", __func__, algName.c_str());
2234 0 : CHK_PTR_NULL(commParam);
2235 : // 刷新CCLBuffer
2236 0 : CHK_RET(InitCclbuffer(commParam));
2237 0 : auto iter = resMap_.find(newTag);
2238 0 : if (iter == resMap_.end()) {
2239 0 : std::lock_guard<std::mutex> lock(preemptMutexForResMap_);
2240 0 : iter = resMap_.find(newTag);
2241 0 : if (iter == resMap_.end()) {
2242 0 : HCCL_RUN_INFO("[%s] Alloc resource for alg[%s], tag[%s]", __func__, algName.c_str(), newTag.c_str());
2243 0 : AlgResourceRequest resRequest;
2244 0 : CHK_RET(CalcResRequest(algName, opParam, executor, resRequest));
2245 0 : CHK_RET(AllocAlgResource(newTag, opParam, commParam, resRequest, resMap_[newTag]));
2246 0 : iter = resMap_.find(newTag);
2247 0 : } else {
2248 0 : HCCL_INFO("[%s] Repeatedly inited for alg [%s] is not allowed.", __func__, algName.c_str());
2249 : }
2250 0 : } else if (algName == "BatchSendRecv" || algName == "BatchSendRecvRetry" || algName == "BatchSendRecvGroup") {
2251 : // 如果是非aclgraph模式，而且不需要增量建链，则跳过CalcResRequest这个计算，节省时间。在非aclgraph模式下，跳过是安全的。
2252 0 : bool canSkipCalcResRequest = !opParam.isCapture && !opParam.needIncreLink;
2253 0 : if (!canSkipCalcResRequest) { // 如果不能跳过计算，则走一遍计算的流程，否则就跳过以下计算
2254 0 : AlgResourceRequest resRequest;
2255 0 : HCCL_INFO("[%s]IncreAlloc resource for alg[%s], tag[%s]", __func__, algName.c_str(), newTag.c_str());
2256 0 : CHK_RET(CalcResRequest(algName, opParam, executor, resRequest));
2257 0 : CHK_RET(IncreAllocTransportResource(newTag, opParam, commParam, resRequest, resMap_[newTag]));
2258 0 : }
2259 0 : } else if (algName == "RunAlltoAllVStaged" || algName == "RunAlltoAllVFullMesh") {
2260 0 : AlgResourceRequest resRequest;
2261 0 : CHK_RET(CalcResRequest(algName, opParam, executor, resRequest));
2262 0 : HCCL_INFO(
2263 : "[%s] check if need refresh resource for alg[%s], tag[%s], old[%lu], new[%lu]", __func__, algName.c_str(),
2264 : newTag.c_str(), resMap_[newTag].scratchMem.size(), resRequest.scratchMemSize);
2265 0 : bool reAllocFlag = !(resMap_[newTag].scratchMem.size() == resRequest.scratchMemSize);
2266 0 : if (reAllocFlag) {
2267 0 : PetersonLockGuard guard(hostDeviceLock_.get());
2268 0 : CHK_PRT_RET(
2269 : guard.IsLockFailed(), HCCL_ERROR("[HcclCommAicpu][AllocAlgResource] hostDeviceLock lock failed"),
2270 : HCCL_E_INTERNAL);
2271 0 : CHK_RET(AllocScratchMemResource(
2272 : newTag, commParam, resRequest.scratchMemSize, resMap_[newTag].scratchMem, true));
2273 0 : HCCL_INFO(
2274 : "[%s] refresh resource success for alg[%s], tag[%s], scratchMemSize[%lu], ptr[%p]", __func__,
2275 : algName.c_str(), newTag.c_str(), resRequest.scratchMemSize, resMap_[newTag].scratchMem.ptr());
2276 0 : }
2277 0 : }
2278 0 : CHK_PRT_RET(
2279 : iter == resMap_.end(), HCCL_ERROR("[%s]Fail to find algResResponse for tag[%s]", __func__, newTag.c_str()),
2280 : HCCL_E_PARA);
2281 0 : algResResponse = &iter->second;
2282 0 : HCCL_INFO("[HcclCommAicpu][GetAlgResponseRes] success!");
2283 0 : return HCCL_SUCCESS;
2284 : }
2285 :
2286 0 : HcclResult HcclCommAicpu::GetAlltoAllVCTotalCount(OpParam& param, u64& sendCount, u64& recvCount)
2287 : {
2288 0 : for (u32 i = 0; i < topoInfo_.userRankSize; i++) {
2289 0 : sendCount += *(
2290 0 : static_cast<const u64*>(param.All2AllDataDes.sendCountMatrix) + topoInfo_.userRank * topoInfo_.userRankSize
2291 0 : + i);
2292 0 : recvCount += *(
2293 0 : static_cast<const u64*>(param.All2AllDataDes.sendCountMatrix) + topoInfo_.userRank
2294 0 : + topoInfo_.userRankSize * i);
2295 : }
2296 0 : return HCCL_SUCCESS;
2297 : }
2298 :
2299 0 : HcclResult HcclCommAicpu::GetAlltoAllTotalCount(OpParam& param, u64& sendCount, u64& recvCount)
2300 : {
2301 0 : sendCount = param.All2AllDataDes.sendCount * topoInfo_.userRankSize;
2302 0 : recvCount = param.All2AllDataDes.sendCount * topoInfo_.userRankSize;
2303 0 : return HCCL_SUCCESS;
2304 : }
2305 :
2306 0 : HcclResult HcclCommAicpu::GetAlltoAllVTotalCount(OpParam& param, u64& sendCount, u64& recvCount)
2307 : {
2308 0 : for (u32 i = 0; i < topoInfo_.userRankSize; i++) {
2309 0 : u64 curSendCount = *(static_cast<const u64*>(param.All2AllDataDes.sendCounts) + i)
2310 0 : + *(static_cast<const u64*>(param.All2AllDataDes.sdispls) + i);
2311 0 : sendCount = std::max(sendCount, curSendCount);
2312 0 : u64 curRecvCount = *(static_cast<const u64*>(param.All2AllDataDes.recvCounts) + i)
2313 0 : + *(static_cast<const u64*>(param.All2AllDataDes.rdispls) + i);
2314 0 : recvCount = std::max(recvCount, curRecvCount);
2315 : }
2316 0 : return HCCL_SUCCESS;
2317 : }
2318 :
2319 0 : HcclResult HcclCommAicpu::SetAlltoAllInputAndOutPutMem(OpParam& param, AlgResourceResponse& algResource)
2320 : {
2321 0 : u32 sendTypeSize = 0, recvTypeSize = 0;
2322 0 : CHK_RET(SalGetDataTypeSize(param.All2AllDataDes.sendType, sendTypeSize));
2323 0 : CHK_RET(SalGetDataTypeSize(param.All2AllDataDes.recvType, recvTypeSize));
2324 0 : u64 sendCount = 0;
2325 0 : u64 recvCount = 0;
2326 0 : if (param.opType == HcclCMDType::HCCL_CMD_ALLTOALL) {
2327 0 : CHK_RET(GetAlltoAllTotalCount(param, sendCount, recvCount));
2328 0 : } else if (param.opType == HcclCMDType::HCCL_CMD_ALLTOALLV) {
2329 0 : CHK_RET(GetAlltoAllVTotalCount(param, sendCount, recvCount));
2330 0 : } else if (param.opType == HcclCMDType::HCCL_CMD_ALLTOALLVC) {
2331 0 : CHK_RET(GetAlltoAllVCTotalCount(param, sendCount, recvCount));
2332 : }
2333 0 : u64 inputSize = sendCount * sendTypeSize;
2334 0 : u64 outputSize = recvCount * recvTypeSize;
2335 0 : algResource.paramInputMem = inputSize == 0 ? tinySendRecvMem_ : DeviceMem::create(param.inputPtr, inputSize);
2336 0 : algResource.paramOutputMem = outputSize == 0 ? tinySendRecvMem_ : DeviceMem::create(param.outputPtr, outputSize);
2337 0 : HCCL_DEBUG(
2338 : "[HcclCommAicpu][SetAlltoAllInputAndOutPutMem] Set memory for AllToAll, inputSize[%llu], inputPtr[%p],"
2339 : "outputPtr[%p]!",
2340 : inputSize, param.inputPtr, param.outputPtr);
2341 0 : return HCCL_SUCCESS;
2342 : }
2343 :
2344 0 : HcclResult HcclCommAicpu::CombineReportOpInfo(OpParam& param, bool isRetry, bool isRelay)
2345 : {
2346 0 : MsprofAicpuHCCLOPInfo hcclOpInfo{};
2347 0 : hcclOpInfo.relay = (isRelay) ? 1 : 0;
2348 0 : hcclOpInfo.retry = (isRetry) ? 1 : 0;
2349 0 : hcclOpInfo.dataType = param.DataDes.dataType;
2350 0 : hcclOpInfo.count = param.DataDes.count;
2351 0 : hcclOpInfo.groupName = groupHashId_;
2352 0 : hcclOpInfo.ranksize = topoInfo_.userRankSize;
2353 0 : std::string algTypeStr = TransferAlgType(algType_);
2354 0 : CHK_RET(dfx::ProfilingManager::ReportHcclOpInfo(hcclOpInfo, algTypeStr));
2355 0 : return HCCL_SUCCESS;
2356 0 : }
2357 :
2358 0 : HcclResult HcclCommAicpu::UpdateProfReportStartSqeIdx()
2359 : {
2360 0 : if (dfx::ProfilingManager::IsL1fromOffToOn()) {
2361 0 : std::vector<Stream> streams;
2362 0 : CHK_RET(GetStreamAll(streams));
2363 0 : for (auto& tmpStream : streams) {
2364 0 : HcclSqeContext* sqeContext = tmpStream.GetSqeContextPtr();
2365 0 : SqeRingBuffer* sqeContextBuffer = &(sqeContext->buffer);
2366 0 : CHK_RET(dfx::ProfilingManager::UpdateStartReportSqeIdx(tmpStream.id(), sqeContextBuffer->tailSqeIdx));
2367 : }
2368 0 : }
2369 0 : return HCCL_SUCCESS;
2370 : }
2371 :
2372 0 : HcclResult HcclCommAicpu::Orchestrate(
2373 : const std::string& newTag, const std::string& algName, OpParam& param, std::unique_ptr<CollExecutorBase>& executor,
2374 : AlgResourceResponse& algResource, const HcclOpResParam* commParam)
2375 : {
2376 0 : CHK_PTR_NULL(commParam);
2377 : // 算子下发信息记录在共享内存区
2378 0 : UpdateOpRingBufferIdx();
2379 0 : CHK_RET(aicpuShareData_.RecordOpInfo(
2380 : newTag, param, (isDeviceMode_ ? mc2OpIndex_ : hcclOpExecIndex_), localUserRank_, isCustom_));
2381 0 : CHK_RET(UpdateProfReportStartSqeIdx());
2382 :
2383 : // 每个算子都刷新一下profiling开关, 支持profiling从中间迭代采集
2384 0 : bool profL0Open = dfx::ProfilingManager::IsProfL0On();
2385 0 : bool profL1Open = dfx::ProfilingManager::IsProfL1On();
2386 0 : HCCL_DEBUG("profL0Open:%d, profL1Open:%d", profL0Open, profL1Open);
2387 :
2388 0 : LogControl logControl(false, false); // 重执行ERROR日志保底控制，析构时重置日志设置
2389 0 : HCCL_ENTRY_INFO(
2390 : commParam->opEntry, "[HcclCommAicpu][Orchestrate]start tag[%s] newTag[%s] algName[%s] identifier[%s]",
2391 : param.tag.c_str(), newTag.c_str(), algName.c_str(), identifier_.c_str());
2392 0 : HCCL_INFO(
2393 : "opRetryHandler.isInplacePreSync[%d] opRetryHandler.isPostSync[%d]",
2394 : algOpContext_.opRetryHandler.isInplacePreSync, algOpContext_.opRetryHandler.isPostSync);
2395 0 : if (executor.get() == nullptr) {
2396 0 : executor = CollAlgExecRegistry::Instance().GetAlgExec(algName, dispatcher_, topoMatcher_);
2397 0 : CHK_PRT_RET(
2398 : executor.get() == nullptr,
2399 : HCCL_ERROR(
2400 : "[HcclCommAicpu][Orchestrate]Fail to find executor "
2401 : "for algName[%s]",
2402 : algName.c_str()),
2403 : HCCL_E_PARA);
2404 0 : executor->SetAlgType(algType_);
2405 0 : executor->SetCCLInBuffer(cclbufferSize_);
2406 :
2407 0 : if (param.opType == HcclCMDType::HCCL_CMD_REDUCE_SCATTER) {
2408 0 : bool isSupportSDMAReduce = IsSupportSDMAReduce(
2409 0 : cclInputBuffer_.ptr(), cclOutputBuffer_.ptr(), param.DataDes.dataType, param.reduceType);
2410 0 : executor->SetIsSupportSDMAReduce(isSupportSDMAReduce);
2411 : }
2412 : }
2413 0 : if (param.opType == HcclCMDType::HCCL_CMD_ALLTOALL || param.opType == HcclCMDType::HCCL_CMD_ALLTOALLV
2414 0 : || param.opType == HcclCMDType::HCCL_CMD_ALLTOALLVC) {
2415 0 : CHK_RET(CalSendRecvInfoFor910B(algName, param, executor));
2416 0 : CHK_RET(SetAlltoAllInputAndOutPutMem(param, algResource));
2417 0 : if (algName == "RunAlltoAllVTwoLevelPipeline") {
2418 0 : CHK_RET(CalSendRecvInfoForAlltoall(param));
2419 0 : HCCL_DEBUG("[HcclCommAicpu][Orchestrate] running RunAlltoAllVTwoLevelPipeline, prepare SendRecvInfo.");
2420 0 : CollAlltoAllExecutor* alltoAllExecutor = dynamic_cast<CollAlltoAllExecutor*>(executor.get());
2421 0 : CHK_PTR_NULL(alltoAllExecutor);
2422 0 : CHK_RET(alltoAllExecutor->SetExcutorExtraInfo(allMeshAggregationSendRecvInfo_, cclbufferSize_));
2423 : }
2424 : }
2425 0 : auto waitStopExecCmdTimeoutMs = HcclGetWaitStopExecCmdTimeout();
2426 0 : auto waitStopExecCmdTimeout = std::chrono::milliseconds(waitStopExecCmdTimeoutMs);
2427 :
2428 0 : auto opStartTime = std::chrono::steady_clock::now(); // 记录重执行算子耗时
2429 0 : auto startTime = std::chrono::steady_clock::now();
2430 :
2431 0 : KfcError errorCode = KfcError::kNone;
2432 0 : uint32_t retryCnt = 0;
2433 0 : bool retryProcessing = false;
2434 0 : KfcCommand lastCmd = KfcCommand::kNone;
2435 0 : uint32_t beginSqePos = INVALID_UINT;
2436 0 : uint32_t endSqePos = INVALID_UINT;
2437 0 : HcclOpExecFSM state = HcclOpExecFSM::HCCL_OP_EXEC_FSM_INIT;
2438 0 : HcclResult ret = HCCL_SUCCESS;
2439 0 : dfxExtendInfo_.kfcStatus = DfxKfcStatus::kOneStart;
2440 0 : AicpuComContext* ctx = AicpuGetComContext();
2441 0 : AicpuHcclProcess::CallMC2MaintenanceThread(ctx);
2442 0 : u32 loopCnt = 0;
2443 0 : u32 loopNum = 1;
2444 0 : commParam_ = commParam;
2445 0 : CHK_RET(InitExecLoop(param, executor, loopNum));
2446 :
2447 : while (true) {
2448 0 : switch (state) {
2449 0 : case HcclOpExecFSM::HCCL_OP_EXEC_FSM_INIT:
2450 0 : HCCL_INFO("hccl aicpu execute loop %u", loopCnt);
2451 0 : ret = HcclOpExecFsmInitProcess(newTag, param, algResource, state, errorCode);
2452 0 : break;
2453 0 : case HcclOpExecFSM::HCCL_OP_EXEC_FSM_LAUNCH:
2454 0 : ret = HcclOpExecFsmLaunchProcess(
2455 : algName, param, executor, algResource, state, errorCode, beginSqePos, endSqePos, retryCnt);
2456 0 : if (ret == HCCL_E_SUSPENDING && isDeviceMode_ && retryEnable_) {
2457 0 : return HCCL_E_SUSPENDING;
2458 : }
2459 0 : break;
2460 0 : case HcclOpExecFSM::HCCL_OP_EXEC_FSM_WAIT_END:
2461 0 : ret = HcclOpExecFsmWaitEndProcess(
2462 0 : param, algResource, state, errorCode, retryCnt, param.tag, beginSqePos);
2463 0 : if (state == HcclOpExecFSM::HCCL_OP_EXEC_FSM_STOPPING) {
2464 0 : startTime = std::chrono::steady_clock::now();
2465 : }
2466 0 : break;
2467 0 : case HcclOpExecFSM::HCCL_OP_EXEC_FSM_STOPPING:
2468 0 : retryProcessing = true;
2469 0 : if ((std::chrono::steady_clock::now() - startTime) >= waitStopExecCmdTimeout) {
2470 0 : HCCL_ERROR("[OpRetry][AICPU]hccl aicpu wait stop exec timeout[%u ms].", waitStopExecCmdTimeoutMs);
2471 0 : errorCode = KfcError::kTimeout;
2472 0 : state = HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR;
2473 : } else {
2474 0 : ret = HcclOpExecFsmStoppingProcess(param, state, errorCode, retryCnt);
2475 : }
2476 0 : break;
2477 0 : case HcclOpExecFSM::HCCL_OP_EXEC_FSM_STOPPED:
2478 0 : ret = HcclOpExecFsmStoppedProcess(state, errorCode, retryCnt, algName, param, beginSqePos, endSqePos);
2479 0 : if (state == HcclOpExecFSM::HCCL_OP_EXEC_FSM_WAIT_RETRY) {
2480 0 : startTime = std::chrono::steady_clock::now();
2481 : }
2482 0 : break;
2483 0 : case HcclOpExecFSM::HCCL_OP_EXEC_FSM_CHANGE_LINK:
2484 0 : ret = HcclOpExecChangeLinkProcess(newTag, state, errorCode, retryCnt, algResource, commParam, param);
2485 0 : HCCL_DEBUG(
2486 : "[OpRetry][AICPU]retry change link finish, retryCnt:%u, tag:%s, state:%d", retryCnt,
2487 : param.tag.c_str(), state);
2488 0 : break;
2489 0 : case HcclOpExecFSM::HCCL_OP_EXEC_FSM_WAIT_RETRY: {
2490 0 : auto waitRetryCmdTimeoutMs = HcclGetWaitRetryCmdTimeout(retryCnt);
2491 0 : auto waitRetryCmdTimeout = std::chrono::milliseconds(waitRetryCmdTimeoutMs);
2492 0 : if ((std::chrono::steady_clock::now() - startTime) >= waitRetryCmdTimeout) {
2493 0 : HCCL_ERROR("[OpRetry][AICPU]hccl aicpu wait retry timeout[%u ms].", waitRetryCmdTimeoutMs);
2494 0 : errorCode = KfcError::kTimeout;
2495 0 : state = HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR;
2496 : } else {
2497 0 : ret = HcclOpExecFsmWaitRetryProcess(param, state, errorCode, lastCmd);
2498 : }
2499 0 : } break;
2500 0 : case HcclOpExecFSM::HCCL_OP_EXEC_FSM_RETRY:
2501 : // 重执行前清理当前算子展开的SQE缓存 (if any), 防止命中非完整的cache
2502 0 : CHK_RET(aicpuCacheManager_.ClearOpUnfoldCacheEntry(
2503 : algName, param, algResource, isDeviceMode_, topoInfo_, topoMatcher_, algOpContext_,
2504 : GetWorkflowMode()));
2505 0 : ret = HcclOpExecFsmRetryProcess(
2506 : algName, param, executor, algResource, state, errorCode, retryCnt, beginSqePos, endSqePos);
2507 0 : break;
2508 0 : case HcclOpExecFSM::HCCL_OP_EXEC_FSM_END:
2509 0 : loopCnt++;
2510 0 : if ((param.opType == HcclCMDType::HCCL_CMD_BATCH_SEND_RECV) && retryEnable_) {
2511 0 : param.BatchSendRecvDataDes.curIterNum = loopCnt;
2512 0 : ResetBSRRetryCnt();
2513 : }
2514 0 : if (loopCnt < loopNum) {
2515 0 : state = HcclOpExecFSM::HCCL_OP_EXEC_FSM_INIT;
2516 0 : break;
2517 : }
2518 0 : if (retryCnt > 0) {
2519 0 : RecordReportStatus(dfx::ReportStatus::kRetrySuccess);
2520 0 : retryProcessing = false;
2521 0 : auto opEndTime = std::chrono::steady_clock::now();
2522 0 : auto duration = std::chrono::duration_cast<std::chrono::seconds>(opEndTime - opStartTime).count();
2523 0 : HCCL_RUN_INFO(
2524 : "[OpRetry][AICPU]retry exec success, retryCnt [%u], tag [%s], take time [%ld]s", retryCnt,
2525 : param.tag.c_str(), duration);
2526 : }
2527 0 : CHK_RET(CombineReportOpInfo(param, (retryCnt > 0), false));
2528 0 : return HcclOpExecFsmEndProcess(retryCnt);
2529 0 : case HcclOpExecFSM::HCCL_OP_EXEC_STOP_LAUNCH:
2530 0 : HCCL_DEBUG("[NsRecovery][AICPU] stop the kernel");
2531 : // 停止前清理当前算子展开的SQE缓存 (if any), 防止host侧重新展开该算子并命中非完整的cache (例如step快恢)
2532 0 : CHK_RET(aicpuCacheManager_.ClearOpUnfoldCacheEntry(
2533 : algName, param, algResource, isDeviceMode_, topoInfo_, topoMatcher_, algOpContext_,
2534 : GetWorkflowMode()));
2535 0 : if (!needsResponseStopLaunch_) {
2536 0 : return HCCL_E_SUSPENDING;
2537 : } else {
2538 0 : HCCL_RUN_INFO("[NsRecovery][AICPU] stop the kernel for stop cmd");
2539 0 : needsResponseStopLaunch_ = false;
2540 0 : SetCommRecoveryFlag(true);
2541 0 : if (UpdateOpExecStatus(state, KfcStatus::kStoplaunch, errorCode, 0) == HCCL_SUCCESS) {
2542 0 : return HCCL_E_SUSPENDING;
2543 : } else {
2544 0 : break;
2545 : }
2546 : }
2547 0 : case HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR:
2548 : default: {
2549 0 : if (retryProcessing) {
2550 0 : RecordReportStatus(dfx::ReportStatus::kRetryFail);
2551 0 : retryProcessing = false;
2552 : }
2553 0 : UpdateOpExecStatus(state, excuteOpId_, KfcStatus::kRetryError, errorCode, retryCnt);
2554 0 : dfxExtendInfo_.kfcStatus = DfxKfcStatus::kOneFinished;
2555 0 : if (!isDeviceMode_) {
2556 0 : isOpLaunch = false;
2557 : }
2558 0 : HCCL_INFO("hccl aicpu set kfcStatus[%d]", dfxExtendInfo_.kfcStatus);
2559 0 : if (errorCode == KfcError::kExecConstraint) {
2560 0 : return HCCL_E_OPRETRY_FAIL;
2561 : }
2562 0 : return (ret == HCCL_SUCCESS) ? HCCL_E_INTERNAL : ret;
2563 : }
2564 : }
2565 0 : }
2566 : return ret;
2567 0 : }
2568 :
2569 1 : HcclResult HcclCommAicpu::InitBsrSendRecvOpIdAndExcuteOpId(
2570 : OpParam& param, AlgResourceResponse& algResource, HcclOpExecFSM& fsmState, KfcError& errorCode)
2571 : {
2572 1 : auto hcclRet = InitBatchSendRecvOpId(param, algResource);
2573 1 : HCCL_RETRY_CHK_RET_AND_TRANS_FSM(
2574 : hcclRet, HCCL_ERROR("InitBatchSendRecvOpId failed, ret:%u", hcclRet), KfcError::kInner,
2575 : HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR);
2576 1 : param.BatchSendRecvDataDes.curMode = BatchSendRecvCurMode::SEND_RECV;
2577 1 : if (param.BatchSendRecvDataDes.curIterNum == 0) {
2578 1 : bsrSendStream_ = algResource.slaveStreams[BSR_RETRY_SEND_STREAM_INDEX];
2579 1 : bsrRecvStream_ = algResource.slaveStreams[BSR_RETRY_RECV_STREAM_INDEX];
2580 : }
2581 1 : HCCL_INFO(
2582 : "BSR: iter %u, tag:%s index:%u", param.BatchSendRecvDataDes.curIterNum, excuteOpId_.tag, excuteOpId_.index);
2583 1 : HCCL_INFO(
2584 : "BSR: iter %u, send op Tag:%s index:%u", param.BatchSendRecvDataDes.curIterNum, bsrSendOpId_.tag,
2585 : bsrSendOpId_.index);
2586 1 : HCCL_INFO(
2587 : "BSR: iter %u, recv op Tag:%s index:%u", param.BatchSendRecvDataDes.curIterNum, bsrRecvOpId_.tag,
2588 : bsrRecvOpId_.index);
2589 1 : excuteOpId_.bsrInfo[HCCL_SEND].index = bsrSendOpId_.index;
2590 1 : excuteOpId_.bsrInfo[HCCL_RECV].index = bsrRecvOpId_.index;
2591 1 : excuteOpId_.bsrInfo[HCCL_SEND].tpQpn = bsrSendOpId_.bsrInfo[HCCL_SEND].tpQpn;
2592 1 : excuteOpId_.bsrInfo[HCCL_RECV].tpQpn = bsrRecvOpId_.bsrInfo[HCCL_RECV].tpQpn;
2593 1 : excuteOpId_.bsrInfo[HCCL_SEND].streamId = bsrSendOpId_.streamId;
2594 1 : excuteOpId_.bsrInfo[HCCL_RECV].streamId = bsrRecvOpId_.streamId;
2595 1 : excuteOpId_.bsrInfo[HCCL_SEND].srcRank = bsrSendOpId_.srcRank;
2596 1 : excuteOpId_.bsrInfo[HCCL_SEND].detRank = bsrSendOpId_.detRank;
2597 1 : excuteOpId_.bsrInfo[HCCL_RECV].srcRank = bsrRecvOpId_.srcRank;
2598 1 : excuteOpId_.bsrInfo[HCCL_RECV].detRank = bsrRecvOpId_.detRank;
2599 1 : CHK_SAFETY_FUNC_RET(memcpy_s(
2600 : excuteOpId_.bsrInfo[HCCL_SEND].bsrTag, sizeof(excuteOpId_.bsrInfo[HCCL_SEND].bsrTag), bsrSendOpId_.tag,
2601 : sizeof(bsrSendOpId_.tag)));
2602 1 : CHK_SAFETY_FUNC_RET(memcpy_s(
2603 : excuteOpId_.bsrInfo[HCCL_RECV].bsrTag, sizeof(excuteOpId_.bsrInfo[HCCL_RECV].bsrTag), bsrRecvOpId_.tag,
2604 : sizeof(bsrRecvOpId_.tag)));
2605 1 : return HCCL_SUCCESS;
2606 : }
2607 :
2608 1 : HcclResult HcclCommAicpu::HcclOpExecFsmInitProcess(
2609 : const std::string& newTag, OpParam& param, AlgResourceResponse& algResource, HcclOpExecFSM& fsmState,
2610 : KfcError& errorCode)
2611 : {
2612 1 : excuteOpId_.index = isDeviceMode_ ? (++mc2OpIndex_) : hcclOpExecIndex_;
2613 1 : CHK_SAFETY_FUNC_RET(memset_s(excuteOpId_.tag, sizeof(excuteOpId_.tag), 0, sizeof(excuteOpId_.tag)));
2614 1 : CHK_SAFETY_FUNC_RET(memcpy_s(excuteOpId_.tag, sizeof(excuteOpId_.tag), param.tag.c_str(), param.tag.size()));
2615 1 : CHK_SAFETY_FUNC_RET(memset_s(excuteOpId_.newTag, sizeof(excuteOpId_.newTag), 0, sizeof(excuteOpId_.newTag)));
2616 1 : CHK_SAFETY_FUNC_RET(memcpy_s(excuteOpId_.newTag, sizeof(excuteOpId_.newTag), newTag.c_str(), newTag.size()));
2617 1 : excuteOpId_.isSendRecv = false;
2618 1 : excuteOpId_.streamId = ~0u;
2619 1 : excuteOpId_.opType = param.opType;
2620 1 : excuteOpId_.isBsrTaskStart = false;
2621 1 : if (param.opType == HcclCMDType::HCCL_CMD_SEND || param.opType == HcclCMDType::HCCL_CMD_RECEIVE) {
2622 0 : InitSendRecvOpId(param, excuteOpId_);
2623 1 : } else if ((param.opType == HcclCMDType::HCCL_CMD_BATCH_SEND_RECV) && retryEnable_) {
2624 1 : CHK_RET(InitBsrSendRecvOpIdAndExcuteOpId(param, algResource, fsmState, errorCode));
2625 : }
2626 1 : if (GetNsStopLaunchStatus()) {
2627 0 : HCCL_WARNING("the op should not be launched in the suspending status");
2628 0 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_STOP_LAUNCH;
2629 0 : return HCCL_SUCCESS;
2630 : }
2631 1 : auto ret = aicpuHdc_.InitOpExecStatus(kfcStatusTransferD2H_, excuteOpId_);
2632 1 : isOpLaunch = true;
2633 1 : HCCL_INFO("%s tag:%s, isDeviceMode:%d, index:%u", __func__, excuteOpId_.tag, isDeviceMode_, excuteOpId_.index);
2634 1 : HCCL_RETRY_CHK_RET_AND_TRANS_FSM(
2635 : ret, HCCL_ERROR("InitOpExecStatus failed, ret:%u", ret), KfcError::kInner,
2636 : HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR);
2637 1 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_FSM_LAUNCH;
2638 1 : return ret;
2639 : }
2640 :
2641 0 : bool HcclCommAicpu::HcclOpCheckSupportRetry(HcclCMDType opType)
2642 : {
2643 : const std::set<HcclCMDType> HcclSupportRetryOpSet
2644 : = {HcclCMDType::HCCL_CMD_BROADCAST, HcclCMDType::HCCL_CMD_ALLREDUCE, HcclCMDType::HCCL_CMD_REDUCE,
2645 : HcclCMDType::HCCL_CMD_ALLGATHER, HcclCMDType::HCCL_CMD_REDUCE_SCATTER, HcclCMDType::HCCL_CMD_ALLTOALLV,
2646 : HcclCMDType::HCCL_CMD_ALLTOALLVC, HcclCMDType::HCCL_CMD_ALLTOALL, HcclCMDType::HCCL_CMD_GATHER,
2647 : HcclCMDType::HCCL_CMD_SCATTER, HcclCMDType::HCCL_CMD_SEND, HcclCMDType::HCCL_CMD_RECEIVE,
2648 0 : HcclCMDType::HCCL_CMD_BATCH_SEND_RECV};
2649 0 : return (HcclSupportRetryOpSet.find(opType) != HcclSupportRetryOpSet.end());
2650 0 : }
2651 :
2652 1 : HcclResult HcclCommAicpu::HcclOpExecFsmLaunchProcess(
2653 : const std::string& algName, OpParam& param, std::unique_ptr<CollExecutorBase>& executor,
2654 : AlgResourceResponse& algResource, HcclOpExecFSM& fsmState, KfcError& errorCode, uint32_t& beginSqePos,
2655 : uint32_t& endSqePos, uint32_t retryCnt)
2656 : {
2657 1 : HCCL_DEBUG("hccl aicpu start launch task.");
2658 :
2659 1 : HcclResult ret = OrchestrateHcclOp(algName, param, executor, algResource, beginSqePos, endSqePos);
2660 1 : if (ret == HCCL_SUCCESS) { // 下发成功, 并且没有检测到异常cq或中断命令
2661 0 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_FSM_WAIT_END;
2662 1 : } else if (ret == HCCL_E_SUSPENDING) { // 检测到异常cq或中断命令
2663 1 : if (isDeviceMode_ && retryEnable_) {
2664 1 : HCCL_RUN_INFO("Orchestrate hccl op suspending, restart handle by mc2 process.");
2665 1 : return ret;
2666 : }
2667 0 : if ((param.opType == HcclCMDType::HCCL_CMD_BATCH_SEND_RECV) && retryEnable_) {
2668 : // batchsendrecv算子下发过程中出现异常，task下发未完成，send 和 recv 均需要重执行;
2669 : // 第一个g故障op重执行下发task完成后，需要主动上报故障，触发第二个op进行重执行
2670 0 : SetBSRSendOpExecException();
2671 0 : SetBSRRecvOpExecException();
2672 0 : HCCL_RUN_INFO("hccl aicpu abort launch batchsendrecv op, need retry");
2673 : }
2674 0 : CHK_RET(UpdateSuspendStatus(param, fsmState, errorCode, retryCnt));
2675 : } else {
2676 0 : HCCL_ERROR("OrchestrateHcclOp failed, ret:%u", ret);
2677 0 : errorCode = KfcError::kInner;
2678 0 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR;
2679 : }
2680 0 : return ret;
2681 : }
2682 :
2683 4 : HcclResult HcclCommAicpu::HcclOpExecFsmWaitEndProcess(
2684 : OpParam& param, AlgResourceResponse& algResource, HcclOpExecFSM& fsmState, KfcError& errorCode, uint32_t retryCnt,
2685 : std::string& tag, const uint32_t& beginSqePos)
2686 : {
2687 4 : HCCL_DEBUG("hccl aicpu wait task finish.");
2688 4 : auto ret = WaitFinishWhileLoop(mainStream_, algResource.slaveStreams, tag, beginSqePos, param);
2689 4 : if (ret == HCCL_SUCCESS) {
2690 0 : HCCL_DEBUG("hccl aicpu exec complete.");
2691 0 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_FSM_END;
2692 4 : } else if (ret == HCCL_E_SUSPENDING) {
2693 4 : HCCL_RUN_INFO("hccl aicpu force stop in wait end, retryCnt[%u]", retryCnt);
2694 4 : CHK_RET(UpdateSuspendStatus(param, fsmState, errorCode, retryCnt));
2695 : } else {
2696 0 : HCCL_ERROR("WaitTaskFinish failed, ret:%u, identifier[%s]", ret, identifier_.c_str());
2697 0 : errorCode = KfcError::kExec;
2698 0 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR;
2699 : }
2700 4 : return ret;
2701 : }
2702 :
2703 0 : HcclResult HcclCommAicpu::HcclOpExecFsmStoppingProcess(
2704 : const OpParam& param, HcclOpExecFSM& fsmState, KfcError& errorCode, uint32_t retryCnt)
2705 : {
2706 0 : HCCL_DEBUG("hccl aicpu stopping.");
2707 0 : KfcCommand cmd = KfcCommand::kNone;
2708 0 : auto ret = aicpuHdc_.GetOpExecCtrlCmd(kfcControlTransferH2D_, cmd);
2709 0 : if (ret != HCCL_SUCCESS) {
2710 0 : HCCL_ERROR("[OpRetry][AICPU]GetOpExecCtrlCmd failed, ret:%u", ret);
2711 0 : errorCode = KfcError::kExec;
2712 0 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR;
2713 0 : return ret;
2714 : }
2715 0 : if (cmd == KfcCommand::kExit) {
2716 0 : HCCL_RUN_INFO("[OpRetry][AICPU]hccl aicpu exec fsm stop by exit cmd.");
2717 0 : errorCode = KfcError::kExit;
2718 0 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR;
2719 0 : } else if (cmd == KfcCommand::kStopExec) {
2720 0 : HCCL_RUN_INFO("[OpRetry][AICPU]hccl aicpu get stop exec cmd.");
2721 0 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_FSM_STOPPED;
2722 0 : } else if (cmd == KfcCommand::kStopLaunch) {
2723 0 : if ((param.opType == HcclCMDType::HCCL_CMD_BATCH_SEND_RECV) && retryEnable_) {
2724 0 : HcclOpIdentifier targetOp;
2725 0 : CHK_RET(aicpuHdc_.GetOpExecCtrlTargetOp(kfcControlTransferH2D_, targetOp));
2726 0 : std::string targetOpTag = std::string(reinterpret_cast<char*>(&targetOp.tag[0]));
2727 0 : if (targetOpTag != std::string(reinterpret_cast<char*>(&bsrTargetOpId_.tag[0]))) {
2728 0 : CHK_RET(UpdateSuspendStatus(param, fsmState, errorCode, retryCnt));
2729 : }
2730 0 : }
2731 0 : } else if ((cmd == KfcCommand::kNone) || (cmd == KfcCommand::kRetry)) {
2732 0 : HCCL_DEBUG("hccl aicpu wait for stop exec cmd.");
2733 : // do nothing
2734 0 : } else if (cmd == KfcCommand::kReportRetryErr) {
2735 0 : HCCL_ERROR("[OpRetry][AICPU][HcclOpExecFsmStoppingProcess]hccl aicpu can not retry err cmd[%d]", cmd);
2736 0 : uint16_t rsErrorCode = TS_ERROR_RETRY_CONSTRAINT;
2737 0 : CHK_PRT(SendTaskExceptionByMBox(rsErrorCode));
2738 0 : errorCode = KfcError::kExit;
2739 0 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR;
2740 0 : return HCCL_E_OPRETRY_FAIL;
2741 : } else {
2742 0 : HCCL_ERROR("[OpRetry][AICPU]GetOpExecCtrlCmd failed, invalid cmd[%u]", cmd);
2743 0 : errorCode = KfcError::kExec;
2744 0 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR;
2745 : }
2746 0 : return HCCL_SUCCESS;
2747 : }
2748 :
2749 0 : HcclResult HcclCommAicpu::LoadChangeLinkInfo(ChangeLinkInfo& changeLinkInfo)
2750 : {
2751 0 : HcclResult ret = aicpuHdc_.GetOpExecChangeLink(kfcControlTransferH2D_, changeLinkInfo);
2752 :
2753 : // DEBUG_INFO aicpu接收的changelinkinfo
2754 0 : std::string changeLinkInfoStr = "";
2755 0 : for (u32 i = 0; i < changeLinkInfo.remoteRankNum; i++) {
2756 : changeLinkInfoStr
2757 0 : += (std::to_string(changeLinkInfo.remoteRankList[i]) + ":"
2758 0 : + std::to_string(changeLinkInfo.isUseDefaultPort[i]) + "; ");
2759 : }
2760 0 : HCCL_RUN_INFO(
2761 : "[%s]rank[%u], isChangeLinkFlag[%d], changeLinkInfoStr:%s", __func__, localUserRank_,
2762 : changeLinkInfo.isChangeLinkFlag, changeLinkInfoStr.c_str());
2763 :
2764 0 : return ret;
2765 0 : }
2766 :
2767 0 : HcclResult HcclCommAicpu::HcclOpExecChangeLinkProcess(
2768 : const std::string& newTag, HcclOpExecFSM& state, KfcError& errorCode, uint32_t& retryCnt,
2769 : AlgResourceResponse& algResource, const HcclOpResParam* commParam, const OpParam& param)
2770 : {
2771 0 : ChangeLinkInfo changeLinkInfo;
2772 0 : HcclResult ret = LoadChangeLinkInfo(changeLinkInfo);
2773 0 : if (ret != HCCL_SUCCESS) {
2774 0 : errorCode = KfcError::kExec;
2775 0 : state = HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR;
2776 0 : return ret;
2777 : }
2778 0 : bool useBackupLink = false;
2779 0 : std::map<u32, bool> remoteRankPortMap;
2780 0 : for (u32 i = 0; i < changeLinkInfo.remoteRankNum; i++) {
2781 0 : remoteRankPortMap.insert({changeLinkInfo.remoteRankList[i], changeLinkInfo.isUseDefaultPort[i]});
2782 0 : useBackupLink |= (!changeLinkInfo.isUseDefaultPort[i]);
2783 : }
2784 0 : ret = RefreshAlgResponseTransportRes(
2785 0 : newTag, algResource, remoteRankPortMap, changeLinkInfo.isChangeLinkFlag, commParam, param);
2786 0 : if (ret != HCCL_SUCCESS) {
2787 0 : errorCode = KfcError::kExec;
2788 0 : state = HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR;
2789 0 : return ret;
2790 : }
2791 0 : if (useBackupLink) {
2792 0 : RecordReportStatus(dfx::ReportStatus::kRetryWithBackupLink);
2793 : }
2794 0 : if ((param.opType == HcclCMDType::HCCL_CMD_BATCH_SEND_RECV) && retryEnable_) {
2795 0 : retryCnt = (bsrRetryOp_ == HCCL_SEND) ? bsrSendRetryCnt_ : bsrRecvRetryCnt_;
2796 : }
2797 0 : errorCode = KfcError::kNone;
2798 0 : CHK_RET(UpdateOpExecStatus(state, KfcStatus::kChanged, errorCode, retryCnt));
2799 0 : state = HcclOpExecFSM::HCCL_OP_EXEC_FSM_WAIT_RETRY;
2800 0 : return HCCL_SUCCESS;
2801 0 : }
2802 :
2803 0 : HcclResult HcclCommAicpu::BSRStopedProcess(HcclOpExecFSM& fsmState, KfcError& errorCode)
2804 : {
2805 : // 判断batchsendrecv算子的send 和recv 操作停止的位置是否满足重执行条件
2806 : // send / recv 的stream停止位置不能位于该算子的首个sqe 和末尾sqe
2807 : u32 bsrSendSqHead;
2808 : u32 bsrRecvSqHead;
2809 0 : auto ret = QuerySqStatusByType(devId_, bsrSendStream_.sqId(), DRV_SQCQ_PROP_SQ_HEAD, bsrSendSqHead);
2810 0 : HCCL_RETRY_CHK_RET_AND_TRANS_FSM(
2811 : ret, HCCL_ERROR("[OpRetry][AICPU]query send stream sq head failed, ret:%u", ret), KfcError::kExec,
2812 : HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR);
2813 :
2814 0 : ret = QuerySqStatusByType(devId_, bsrRecvStream_.sqId(), DRV_SQCQ_PROP_SQ_HEAD, bsrRecvSqHead);
2815 0 : HCCL_RETRY_CHK_RET_AND_TRANS_FSM(
2816 : ret, HCCL_ERROR("[OpRetry][AICPU]query recv stream sq head failed, ret:%u", ret), KfcError::kExec,
2817 : HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR);
2818 :
2819 0 : ret = ((bsrSendOpBeginSqePos_ == bsrSendSqHead) || (bsrRecvOpBeginSqePos_ == bsrRecvSqHead)) ? HCCL_E_OPRETRY_FAIL :
2820 : HCCL_SUCCESS;
2821 0 : if (ret != HCCL_SUCCESS) {
2822 0 : uint16_t rsErrorCode = TS_ERROR_RETRY_CONSTRAINT;
2823 0 : CHK_PRT(SendTaskExceptionByMBox(rsErrorCode));
2824 : }
2825 0 : HCCL_RETRY_CHK_RET_AND_TRANS_FSM(
2826 : ret,
2827 : HCCL_ERROR(
2828 : "[OpRetry][AICPU]hccl aicpu wait start task is not complete, can not retry. params: send sq head "
2829 : "%u, recv sq head %u, send sq begin %u, recv sq begin %u",
2830 : bsrSendSqHead, bsrRecvSqHead, bsrSendOpBeginSqePos_, bsrRecvOpBeginSqePos_),
2831 : KfcError::kExecConstraint, HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR);
2832 :
2833 0 : ret = ((bsrRetryOp_ == HCCL_SEND) && (bsrSendSqHead == bsrSendOpEndSqePos_)) ? HCCL_E_OPRETRY_FAIL : HCCL_SUCCESS;
2834 0 : HCCL_RETRY_CHK_RET_AND_TRANS_FSM(
2835 : ret,
2836 : HCCL_ERROR(
2837 : "[OpRetry][AICPU]hccl aicpu send record complete task is complete, can not retry. params: send sq "
2838 : "head %u, recv sq head %u, send sq begin %u, send sq end %u, recv sq begin %u, recv sq end %u",
2839 : bsrSendSqHead, bsrRecvSqHead, bsrSendOpBeginSqePos_, bsrSendOpEndSqePos_, bsrRecvOpBeginSqePos_,
2840 : bsrRecvOpEndSqePos_),
2841 : KfcError::kExecConstraint, HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR);
2842 :
2843 0 : ret = ((bsrRetryOp_ == HCCL_RECV) && (bsrRecvSqHead == bsrRecvOpEndSqePos_)) ? HCCL_E_OPRETRY_FAIL : HCCL_SUCCESS;
2844 0 : HCCL_RETRY_CHK_RET_AND_TRANS_FSM(
2845 : ret,
2846 : HCCL_ERROR(
2847 : "[OpRetry][AICPU]hccl aicpu recv record complete task is complete, can not retry. params: send sq "
2848 : "head %u, recv sq head %u, send sq begin %u, send sq end %u, recv sq begin %u, recv sq end %u",
2849 : bsrSendSqHead, bsrRecvSqHead, bsrSendOpBeginSqePos_, bsrSendOpEndSqePos_, bsrRecvOpBeginSqePos_,
2850 : bsrRecvOpEndSqePos_),
2851 : KfcError::kExecConstraint, HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR);
2852 :
2853 0 : HCCL_RUN_INFO(
2854 : "[OpRetry][AICPU]hccl aicpu op is running, can retry. params: send sq head "
2855 : "%u, recv sq head %u, send sq begin %u, send sq end %u, recv sq begin %u, recv sq end %u",
2856 : bsrSendSqHead, bsrRecvSqHead, bsrSendOpBeginSqePos_, bsrSendOpEndSqePos_, bsrRecvOpBeginSqePos_,
2857 : bsrRecvOpEndSqePos_);
2858 0 : if (IsTaskExceptionForHccs()) {
2859 0 : HCCL_RUN_INFO("[OpRetry][AICPU]hccl aicpu stop by sdma/write task exception, can retry.");
2860 : }
2861 0 : errorCode = KfcError::kNone;
2862 0 : uint32_t retryCnt = (bsrRetryOp_ == HCCL_SEND) ? bsrSendRetryCnt_ : bsrRecvRetryCnt_;
2863 0 : CHK_RET(UpdateOpExecStatus(fsmState, KfcStatus::kStopExec, errorCode, retryCnt));
2864 0 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_FSM_WAIT_RETRY;
2865 0 : return HCCL_SUCCESS;
2866 : }
2867 :
2868 0 : HcclResult HcclCommAicpu::HcclOpExecFsmStoppedProcess(
2869 : HcclOpExecFSM& fsmState, KfcError& errorCode, uint32_t retryCnt, const std::string& algName, OpParam& param,
2870 : uint32_t beginSqePos, uint32_t endSqePos)
2871 : {
2872 0 : HCCL_DEBUG("hccl aicpu stop exec.");
2873 0 : KfcCommand cmd = KfcCommand::kNone;
2874 0 : auto ret = aicpuHdc_.GetOpExecCtrlCmd(kfcControlTransferH2D_, cmd);
2875 0 : uint16_t rsErrorCode = 0;
2876 0 : if (ret != HCCL_SUCCESS) {
2877 0 : HCCL_ERROR("[OpRetry][AICPU]GetOpExecCtrlCmd failed, ret:%u", ret);
2878 0 : errorCode = KfcError::kExec;
2879 0 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR;
2880 0 : return ret;
2881 : }
2882 :
2883 0 : if (cmd == KfcCommand::kExit) {
2884 0 : HCCL_ERROR("[OpRetry][AICPU]hccl aicpu exec fsm stop by exit cmd.");
2885 0 : errorCode = KfcError::kExit;
2886 0 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR;
2887 0 : return HCCL_SUCCESS;
2888 : }
2889 :
2890 0 : if (cmd == KfcCommand::kReportRetryErr) {
2891 0 : HCCL_ERROR("[OpRetry][AICPU][HcclOpExecFsmStoppedProcess]hccl aicpu can not retry err cmd[%d]", cmd);
2892 0 : rsErrorCode = TS_ERROR_RETRY_CONSTRAINT;
2893 0 : CHK_PRT(SendTaskExceptionByMBox(rsErrorCode));
2894 0 : errorCode = KfcError::kExit;
2895 0 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR;
2896 0 : return HCCL_E_OPRETRY_FAIL;
2897 : }
2898 :
2899 0 : if (!HcclOpSupportRetry(algName, retryEnable_, param)) {
2900 0 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR;
2901 0 : if (param.isInplaceError) {
2902 0 : errorCode = KfcError::kExecConstraint;
2903 0 : rsErrorCode = TS_ERROR_RETRY_CONSTRAINT;
2904 0 : CHK_PRT(SendTaskExceptionByMBox(rsErrorCode));
2905 0 : HCCL_RETRY_CHK_RET_AND_TRANS_FSM(
2906 : HCCL_E_OPRETRY_FAIL,
2907 : HCCL_ERROR("[Opretry][AICPU][HcclOpExecFsmStoppedProcess]can not retry for inplace error."),
2908 : KfcError::kExecConstraint, HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR);
2909 0 : } else if (isPollutedZeroCopyOp(param)) {
2910 0 : errorCode = KfcError::kExecConstraint;
2911 0 : rsErrorCode = TS_ERROR_RETRY_CONSTRAINT;
2912 0 : CHK_PRT(SendTaskExceptionByMBox(rsErrorCode));
2913 0 : HCCL_RETRY_CHK_RET_AND_TRANS_FSM(
2914 : HCCL_E_OPRETRY_FAIL,
2915 : HCCL_ERROR("[Opretry][AICPU][HcclOpExecFsmStoppedProcess]can not retry for zero copy op."),
2916 : KfcError::kExecConstraint, HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR);
2917 : return HCCL_E_OPRETRY_FAIL;
2918 : } else {
2919 0 : errorCode = KfcError::kExec;
2920 : }
2921 0 : return HCCL_SUCCESS;
2922 : }
2923 :
2924 0 : uint32_t sqHead = 0xFFFFFFFF;
2925 0 : CHK_RET(QuerySqStatusByType(devId_, mainStream_.sqId(), DRV_SQCQ_PROP_SQ_HEAD, sqHead));
2926 0 : if (sqHead == endSqePos) {
2927 0 : rsErrorCode = TS_ERROR_RETRY_CONSTRAINT;
2928 0 : CHK_PRT(SendTaskExceptionByMBox(rsErrorCode));
2929 0 : HCCL_RETRY_CHK_RET_AND_TRANS_FSM(
2930 : HCCL_E_OPRETRY_FAIL,
2931 : HCCL_ERROR(
2932 : "[OpRetry][AICPU]hccl aicpu record complete task is complete, can not retry. params: "
2933 : "sqHead %u, beginSqePos %u endSqePos %u",
2934 : sqHead, beginSqePos, endSqePos),
2935 : KfcError::kExecConstraint, HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR);
2936 0 : } else if (sqHead == beginSqePos) {
2937 0 : rsErrorCode = TS_ERROR_RETRY_CONSTRAINT;
2938 0 : CHK_PRT(SendTaskExceptionByMBox(rsErrorCode));
2939 0 : HCCL_RETRY_CHK_RET_AND_TRANS_FSM(
2940 : HCCL_E_OPRETRY_FAIL,
2941 : HCCL_ERROR(
2942 : "[OpRetry][AICPU]hccl aicpu wait start task is not complete, can not retry. "
2943 : "params: sqHead %u, beginSqePos %u endSqePos %u",
2944 : sqHead, beginSqePos, endSqePos),
2945 : KfcError::kExecConstraint, HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR);
2946 0 : } else if ((param.opType == HcclCMDType::HCCL_CMD_BATCH_SEND_RECV) && retryEnable_) {
2947 0 : CHK_RET(BSRStopedProcess(fsmState, errorCode));
2948 0 : } else {
2949 0 : HCCL_RUN_INFO(
2950 : "[OpRetry][AICPU]hccl aicpu op is running, can retry. params: sqHead %u, beginSqePos %u "
2951 : "endSqePos %u",
2952 : sqHead, beginSqePos, endSqePos);
2953 0 : if (IsTaskExceptionForHccs()) {
2954 0 : HCCL_RUN_INFO("[OpRetry][AICPU]hccl aicpu stop by sdma/write task exception, can retry.");
2955 : }
2956 0 : errorCode = KfcError::kNone;
2957 0 : CHK_RET(UpdateOpExecStatus(fsmState, KfcStatus::kStopExec, errorCode, retryCnt));
2958 0 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_FSM_WAIT_RETRY;
2959 : }
2960 0 : return HCCL_SUCCESS;
2961 : }
2962 :
2963 0 : void HcclCommAicpu::NsCommStop()
2964 : {
2965 0 : if ((StreamsKill(devId_) != HCCL_SUCCESS) || (DeviceQuery(devId_, ts::APP_ABORT_KILL_FINISH, 0U) != HCCL_SUCCESS)) {
2966 0 : (void)aicpuHdc_.SetOpExecStatus(kfcStatusTransferD2H_, KfcStatus::kError, KfcError::kExec, 0);
2967 0 : HCCL_ERROR("[NsRecovery][AICPU]Stop failed");
2968 0 : return;
2969 : }
2970 : // 停止条件算子
2971 0 : if (isDeviceMode_) {
2972 0 : (void)InvokeKfcHandler(AicpuKfcHandlerType::kClearCommitTurn, {rpc_});
2973 : }
2974 0 : (void)aicpuHdc_.SetOpExecStatus(kfcStatusTransferD2H_, KfcStatus::kStopExec, KfcError::kNone, 0);
2975 0 : (void)HcclOneSideServiceAicpu::DisableAllStreamFunc();
2976 0 : HCCL_RUN_INFO("[NsRecovery][AICPU]stopFunc Finished");
2977 : }
2978 :
2979 0 : void HcclCommAicpu::NsCommClean()
2980 : {
2981 : // 等待drv任务停止
2982 0 : if ((DeviceQuery(devId_, ts::APP_ABORT_TERMINATE_FINISH, 0U) != HCCL_SUCCESS) || (CleanStreamFunc() != HCCL_SUCCESS)
2983 0 : || (HcclOneSideServiceAicpu::CleanAllStreamFunc() != HCCL_SUCCESS) || (ResetSqBuff() != HCCL_SUCCESS)) {
2984 0 : (void)aicpuHdc_.SetOpExecStatus(kfcStatusTransferD2H_, KfcStatus::kError, KfcError::kExec, 0);
2985 0 : HCCL_ERROR("[NsRecovery][AICPU]stream terminate failed");
2986 0 : return;
2987 : } else {
2988 0 : if (isDeviceMode_) {
2989 0 : (void)InvokeKfcHandler(AicpuKfcHandlerType::kClearMsgArea, {rpc_});
2990 : }
2991 0 : HCCL_INFO("ClearFunc, after APP_ABORT_TERMINATE_FINISH");
2992 0 : dfxExtendInfo_.pollStatus = PollStatus::kDefault;
2993 0 : dfxExtendInfo_.cqeStatus = dfx::CqeStatus::kDefault;
2994 0 : (void)aicpuHdc_.SetOpExecStatus(kfcStatusTransferD2H_, KfcStatus::kClear, KfcError::kNone, 0);
2995 0 : endStopLaunch = false;
2996 0 : isOpLaunch = false;
2997 0 : needsResponseStopLaunch_ = false;
2998 0 : errMessageReport_ = true;
2999 0 : HCCL_RUN_INFO("[NsRecovery][AICPU] clean Finish");
3000 : }
3001 : }
3002 :
3003 0 : HcclResult HcclCommAicpu::GetBackGroundCommand(BackgroundCommand& bgCmd)
3004 : {
3005 0 : return AicpuHdcUtils::GetBackGroundCommand(kfcControlTransferH2D_, bgCmd);
3006 : }
3007 :
3008 0 : HcclResult HcclCommAicpu::ResponseBackGroundStatus(KfcExecStatus& status)
3009 : {
3010 0 : return AicpuHdcUtils::ResponseBackGroundStatus(kfcStatusTransferD2H_, status);
3011 : }
3012 :
3013 0 : HcclResult HcclCommAicpu::GetKfcCommand(KfcCommand& cmd)
3014 : {
3015 0 : return AicpuHdcUtils::GetKfcCommand(kfcControlTransferH2D_, cmd);
3016 : }
3017 :
3018 0 : HcclResult HcclCommAicpu::SetStreamEnable(Stream& stream)
3019 : {
3020 0 : const HcclComStreamInfo& streamInfo = stream.GetHcclStreamInfo();
3021 0 : HCCL_INFO("[SetStreamEnable] streamid[%d]", streamInfo.actualStreamId);
3022 0 : CHK_RET(ConfigSqStatusByType(GetDevId(), streamInfo.sqId, DRV_SQCQ_PROP_SQ_DISABLE_TO_ENABLE, 1));
3023 0 : HandleCqeException(stream, true);
3024 0 : return HCCL_SUCCESS;
3025 : }
3026 :
3027 0 : HcclResult HcclCommAicpu::CleanStreamFunc()
3028 : {
3029 0 : CHK_RET(SetStreamEnable(mainStream_));
3030 0 : for (auto& stream : slaveStreams_) {
3031 0 : CHK_RET(SetStreamEnable(stream));
3032 : }
3033 0 : CHK_RET(SetStreamEnable(orderStream_));
3034 0 : return HCCL_SUCCESS;
3035 : }
3036 :
3037 0 : std::string HcclCommAicpu::PrintInplaceStatus(u8 isInplaceStatus)
3038 : {
3039 0 : const u8 kNoOverlap = 0;
3040 0 : const u8 kAllToAllOverlap = 1;
3041 0 : const u8 kInplaceOverlap = 2;
3042 0 : switch (isInplaceStatus) {
3043 0 : case kNoOverlap:
3044 : // input和output不重叠
3045 0 : return "There is no overlap.";
3046 0 : case kAllToAllOverlap:
3047 : // alltoall类算子的input和output重叠
3048 0 : return "The param.inputPtr is equal to param.outputPtr, hence they overlap.";
3049 0 : case kInplaceOverlap:
3050 : // input和output重叠
3051 0 : return "It's inplace case. hence they overlap.";
3052 0 : default:
3053 0 : return "It's an unknown overlap case.";
3054 : }
3055 : return "";
3056 : }
3057 :
3058 0 : std::string HcclCommAicpu::PrintInplaceSupportRetryStatus(InplaceSupportRetryStatus inPlaceSupportRetryStatus)
3059 : {
3060 0 : switch (inPlaceSupportRetryStatus) {
3061 0 : case InplaceSupportRetryStatus::AG_BD_CASE: // 不需要去变成非DMA削减
3062 : // allgather or broadcast 算子
3063 0 : return "The Allgather or broadcast op supports inplace retry.";
3064 0 : case InplaceSupportRetryStatus::RETRY_1_ALLOW_NO_DMA_REDUCE_CASE1: // 需要去变成非DMA削减
3065 : // 使用AllReduceMeshSmallCountExecutor, ReduceScatterDeterExecutor
3066 : // 且环境变量配置RetryEnable:1
3067 0 : return "Since retryEnable:1, the executor without DMAReduce will be applied.";
3068 0 : case InplaceSupportRetryStatus::RETRY_0_NOT_ALLOW_NO_DMA_REDUCE_CASE1: // 不需要去变成非DMA削减
3069 : // 使用AllReduceMeshSmallCountExecutor, ReduceScatterDeterExecutor
3070 : // 且环境变量配置RetryEnable:0
3071 0 : return "Since retryEnable:0, ExecutorOnlySupportDMAReduce is not allowed for inplace case.";
3072 0 : case InplaceSupportRetryStatus::ALWAYS_NO_DMA_REDUCE: // 不需要去变成非DMA削减，本身就是
3073 : // 使用AllReduceComm/ReduceScatterComm
3074 0 : return "AllReduceComm or ReduceScatterComm is used for inplace case.";
3075 0 : case InplaceSupportRetryStatus::RETRY_1_ALLOW_NO_DMA_REDUCE_CASE2: // 需要去变成非DMA削减
3076 : // 使用其余在91093场景下使用的reduce scatter, allreduce executor
3077 : // 且环境变量配置RetryEnable:1
3078 0 : return "Since retryEnable:1, the executor will be applied without DMAReduce operation.";
3079 0 : case InplaceSupportRetryStatus::RETRY_0_NOT_ALLOW_NO_DMA_REDUCE_CASE2: // 不需要去变成非DMA削减
3080 : // 使用其余在91093场景下使用的reduce scatter, allreduce executor
3081 : // 且环境变量配置RetryEnable:0
3082 0 : return "Since retryEnable:0, the executor without DMAReduce operation can not be applied.";
3083 0 : case InplaceSupportRetryStatus::UNKONWN_EXECUTOR: // 不需要去变成非DMA削减
3084 : // 使用未知的executor
3085 0 : return "The unknown executor does not support for an inplace case yet.";
3086 0 : case InplaceSupportRetryStatus::USER_LARGER_THAN_CCL: // 不需要去变成非DMA削减
3087 : // UserInMem > CCLInMem 场景
3088 0 : return "UserInMem > CCLInMem case";
3089 0 : case InplaceSupportRetryStatus::NOT_BASIC_OP_CASE: // 不需要去变成非DMA削减
3090 : // 非 RS AR AG BD算子场景
3091 0 : return "Is not ReduceScatter, AllReduce, AllGather or Broadcast case";
3092 0 : default:
3093 0 : return "It's unknown case. They overlap.";
3094 : }
3095 : return "";
3096 : }
3097 :
3098 0 : bool HcclCommAicpu::IsNoNeedMonitor(void)
3099 : {
3100 0 : if (taskMonitorInterval_ == 0)
3101 0 : return true;
3102 :
3103 0 : KfcCommand kfcCmd = KfcCommand::kNone;
3104 0 : (void)GetKfcCommand(kfcCmd);
3105 0 : if (kfcCmd != KfcCommand::kNone)
3106 0 : return true;
3107 :
3108 0 : BackgroundCommand bgCmd = BackgroundCommand::kNone;
3109 0 : (void)GetBackGroundCommand(bgCmd);
3110 0 : if (bgCmd != BackgroundCommand::kNone)
3111 0 : return true;
3112 :
3113 0 : HcclComSuspendingFlag suspendingFlag = HcclComSuspendingFlag::isNull;
3114 0 : (void)GetSuspendingFlag(suspendingFlag);
3115 0 : if (suspendingFlag != HcclComSuspendingFlag::isNull)
3116 0 : return true;
3117 0 : return false;
3118 : }
3119 :
3120 0 : void HcclCommAicpu::InsertMonitorData(Stream& stream, HcclUs& curTime, u32 sqHead, uint16_t taskId, uint8_t type)
3121 : {
3122 0 : AicpuStreamMontior tmpTaskMonitor;
3123 0 : tmpTaskMonitor.historyTime = curTime;
3124 0 : tmpTaskMonitor.historyHead = sqHead;
3125 0 : tmpTaskMonitor.historyTaskId = taskId;
3126 0 : tmpTaskMonitor.historyType = type;
3127 0 : streamTaskMonitor_.insert(std::make_pair(stream.sqId(), tmpTaskMonitor));
3128 0 : return;
3129 : }
3130 :
3131 0 : bool HcclCommAicpu::IsNeedRefreshMonitorData(
3132 : AicpuStreamMontior& streamMontior, HcclUs& curTime, uint32_t remoteRank, uint16_t taskId, u32 sqHead, u32 sqTail,
3133 : uint8_t type)
3134 : {
3135 0 : auto& historyTime = streamMontior.historyTime;
3136 0 : auto& historyHead = streamMontior.historyHead;
3137 0 : auto& historyTaskId = streamMontior.historyTaskId;
3138 0 : auto& historyType = streamMontior.historyType;
3139 0 : if ((historyTaskId != taskId) || (sqHead != historyHead) || (sqHead == sqTail) || (historyType != type)
3140 0 : || ((type == RT_STARS_SQE_TYPE_NOTIFY_WAIT) && (remoteRank == INVALID_VALUE_RANKID))) {
3141 0 : historyTime = curTime;
3142 0 : historyHead = sqHead;
3143 0 : historyTaskId = taskId;
3144 0 : historyType = type;
3145 0 : return true;
3146 : }
3147 0 : return false;
3148 : }
3149 :
3150 0 : HcclResult HcclCommAicpu::StreamTaskMonitor(void)
3151 : {
3152 : // 通信域资源已经释放
3153 0 : CHK_PRT_RET(
3154 : !commOpenStatus, HCCL_DEBUG("[StreamTaskMonitor]group[%s] has been destroyed", identifier_.c_str()),
3155 : HCCL_SUCCESS);
3156 0 : if (IsNoNeedMonitor())
3157 0 : return HCCL_SUCCESS;
3158 0 : HCCL_DEBUG("StreamTaskMonitor print");
3159 0 : std::vector<Stream> totalStream = {mainStream_};
3160 0 : totalStream.insert(totalStream.end(), slaveStreams_.begin(), slaveStreams_.end());
3161 0 : HcclUs curTime = TIME_NOW();
3162 0 : for (auto& stream : totalStream) {
3163 0 : u32 sqHead = 0U, sqTail = 0U;
3164 0 : (void)QuerySqStatus(devId_, stream.sqId(), sqHead, sqTail);
3165 0 : HcclSqeContext* sqeContext = stream.GetSqeContextPtr();
3166 0 : CHK_PTR_NULL(sqeContext);
3167 0 : SqeRingBuffer* sqeContextBuffer = &(sqeContext->buffer);
3168 0 : CHK_PTR_NULL(sqeContextBuffer);
3169 :
3170 0 : uint8_t type = 0;
3171 0 : uint16_t taskId = 0;
3172 0 : uint32_t remoteRank = 0;
3173 0 : std::string tmp = GetTaskExceptionTaskInfo(sqHead, sqeContextBuffer, type, taskId, remoteRank);
3174 0 : HCCL_DEBUG("GetTaskExceptionTaskInfo type %u taskId %u", type, taskId);
3175 0 : auto mapIt = streamTaskMonitor_.find(stream.sqId());
3176 0 : if (mapIt == streamTaskMonitor_.end()) {
3177 0 : InsertMonitorData(stream, curTime, sqHead, taskId, type);
3178 0 : continue;
3179 : }
3180 :
3181 0 : auto& streamMontior = mapIt->second;
3182 0 : if (IsNeedRefreshMonitorData(streamMontior, curTime, remoteRank, taskId, sqHead, sqTail, type)) {
3183 0 : continue;
3184 : }
3185 :
3186 0 : auto timeVal = DURATION_US(curTime - streamMontior.historyTime).count();
3187 0 : const int TIME_CONVERSION = 1000;
3188 0 : if (timeVal >= taskMonitorInterval_ * TIME_CONVERSION) {
3189 0 : HCCL_RUN_INFO(
3190 : "[StreamTaskMonitor]prof monitor streamId:%d, sqid:%d, head:%u, tail:%u, time %s us, %s", stream.id(),
3191 : stream.sqId(), sqHead, sqTail, std::to_string(timeVal).c_str(), tmp.c_str());
3192 0 : HCCL_RUN_INFO(
3193 : "[StreamTaskMonitor]prof monitor %s", GetTaskExceptionOpInfo(sqHead, sqeContextBuffer).c_str());
3194 0 : PrintTaskExceptionTaskQue(sqHead, sqeContextBuffer, true);
3195 0 : streamMontior.historyTime = curTime;
3196 0 : streamMontior.historyHead = sqHead;
3197 0 : streamMontior.historyTaskId = taskId;
3198 0 : streamMontior.historyType = type;
3199 : }
3200 0 : }
3201 0 : return HCCL_SUCCESS;
3202 0 : }
3203 :
3204 0 : HcclResult HcclCommAicpu::SupportRetryWithInplaceCheck(const std::string& algName, OpParam& param)
3205 : {
3206 : // 不支持inplace的通信算子重执行
3207 0 : u8 isInplaceStatus = 0;
3208 0 : InplaceSupportRetryStatus inPlaceSupportRetryStatus = InplaceSupportRetryStatus::INPLACE_STATUS_END;
3209 0 : if (IsHcclOpInplace(param.opType, param, topoInfo_.userRank, topoInfo_.userRankSize, isInplaceStatus)) {
3210 0 : if (!FitRetryConditionforInPlaceOp(
3211 0 : param.opType, param, algName, cclbufferSize_, topoInfo_.userRankSize,
3212 0 : algOpContext_.opRetryHandler.retryEnable, inPlaceSupportRetryStatus)) {
3213 0 : HCCL_RUN_INFO(
3214 : "[OpRetry][AICPU]hccl supports inplace status: isInplaceStatus[%s], "
3215 : "opRetryHandler.inplaceSupportRetry[%d], opRetryHandler.inPlaceSupportRetryStatus[%s], "
3216 : "opRetryHandler.isInplacePreSync[%d], opRetryHandler.isPostSync[%d].",
3217 : PrintInplaceStatus(isInplaceStatus).c_str(), 0,
3218 : PrintInplaceSupportRetryStatus(inPlaceSupportRetryStatus).c_str(),
3219 : algOpContext_.opRetryHandler.isInplacePreSync, algOpContext_.opRetryHandler.isPostSync);
3220 : } else {
3221 0 : HCCL_RUN_INFO(
3222 : "[OpRetry][AICPU]hccl supports inplace status: isInplaceStatus[%s], "
3223 : "opRetryHandler.inplaceSupportRetry[%d], opRetryHandler.inPlaceSupportRetryStatus[%s], "
3224 : "opRetryHandler.isInplacePreSync[%d], opRetryHandler.isPostSync[%d].",
3225 : PrintInplaceStatus(isInplaceStatus).c_str(), 1,
3226 : PrintInplaceSupportRetryStatus(inPlaceSupportRetryStatus).c_str(),
3227 : algOpContext_.opRetryHandler.isInplacePreSync, algOpContext_.opRetryHandler.isPostSync);
3228 : }
3229 : } else {
3230 0 : HCCL_RUN_INFO(
3231 : "[OpRetry][AICPU]hccl supports inplace status: isInplaceStatus[%s], "
3232 : "opRetryHandler.isInplacePreSync[%d], opRetryHandler.isPostSync[%d].",
3233 : PrintInplaceStatus(isInplaceStatus).c_str(), algOpContext_.opRetryHandler.isInplacePreSync,
3234 : algOpContext_.opRetryHandler.isPostSync);
3235 : }
3236 0 : return HCCL_SUCCESS;
3237 : }
3238 :
3239 0 : bool HcclCommAicpu::HcclOpSupportRetry(const std::string& algName, bool retryEnable, OpParam& param)
3240 : {
3241 0 : HCCL_RUN_INFO(
3242 : "[OpRetry][AICPU]hccl supports retry status: enable[%u], param.tag[%s].", retryEnable, param.tag.c_str());
3243 0 : if (!retryEnable) {
3244 0 : HCCL_ERROR("[OpRetry][AICPU]hccl aicpu can not retry, enable[%u].", retryEnable);
3245 0 : return false;
3246 : }
3247 0 : if (isPollutedZeroCopyOp(param)) {
3248 0 : HCCL_ERROR(
3249 : "[OpRetry][AICPU]hccl aicpu can not retry, isZeroCopy[%d], opType[%s].", param.isZeroCopy,
3250 : GetCMDTypeEnumStr(param.opType).c_str());
3251 0 : return false;
3252 : }
3253 :
3254 0 : CHK_RET(SupportRetryWithInplaceCheck(algName, param));
3255 : // 不支持inplace的通信算子重执行
3256 0 : if ((!algOpContext_.opRetryHandler.inplaceSupportRetry) && (!algOpContext_.opRetryHandler.isInplacePreSync)
3257 0 : && (!algOpContext_.opRetryHandler.isPostSync)) {
3258 0 : HCCL_ERROR(
3259 : "[OpRetry][AICPU]hccl aicpu can not retry, not support inplace case, opType[%s], "
3260 : "inputPtr[0x%016lx], outputPtr[0x%016lx], opRetryHandler.inplaceSupportRetry[%d], "
3261 : "opRetryHandler.isInplacePreSync[%d], opRetryHandler.isPostSync[%d]",
3262 : GetCMDTypeEnumStr(param.opType).c_str(), param.inputPtr, param.outputPtr,
3263 : algOpContext_.opRetryHandler.inplaceSupportRetry, algOpContext_.opRetryHandler.isInplacePreSync,
3264 : algOpContext_.opRetryHandler.isPostSync);
3265 0 : param.isInplaceError = true;
3266 0 : return false;
3267 : }
3268 :
3269 : // 不支持的通信算子重执行
3270 0 : if (HcclOpCheckSupportRetry(param.opType) == false) {
3271 0 : HCCL_ERROR(
3272 : "[OpRetry][AICPU]hccl aicpu can not retry, not support opType[%s].",
3273 : GetCMDTypeEnumStr(param.opType).c_str());
3274 0 : return false;
3275 : }
3276 0 : return true;
3277 : }
3278 :
3279 0 : bool HcclCommAicpu::isPollutedZeroCopyOp(OpParam& param)
3280 : {
3281 : // allreduce\reduce\reducescatter\reducescatterv with zerocopy can not support retry.
3282 0 : bool isPollutedOp
3283 0 : = ((param.opType == HcclCMDType::HCCL_CMD_ALLREDUCE) || (param.opType == HcclCMDType::HCCL_CMD_REDUCE)
3284 0 : || (param.opType == HcclCMDType::HCCL_CMD_REDUCE_SCATTER)
3285 0 : || (param.opType == HcclCMDType::HCCL_CMD_REDUCE_SCATTER_V));
3286 0 : return param.isZeroCopy && isPollutedOp;
3287 : }
3288 :
3289 0 : HcclResult HcclCommAicpu::UpdateOpExecStatus(
3290 : HcclOpExecFSM& fsmState, HcclOpIdentifier& opId, KfcStatus state, KfcError& errorCode, uint32_t retryCnt)
3291 : {
3292 0 : HCCL_INFO(
3293 : "UpdateOpExecStatus fsmState %d, tag %s, index %u, state %d, errorCode %d, retryCnt %u.", fsmState, opId.tag,
3294 : opId.index, state, errorCode, retryCnt);
3295 0 : auto ret = aicpuHdc_.SetOpExecStatus(kfcStatusTransferD2H_, opId, state, errorCode, retryCnt);
3296 :
3297 0 : HCCL_RETRY_CHK_RET_AND_TRANS_FSM(
3298 : ret, HCCL_ERROR("SetOpExecStatus failed, ret:%u", ret), KfcError::kExec, HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR);
3299 :
3300 0 : return ret;
3301 : }
3302 :
3303 : HcclResult
3304 7 : HcclCommAicpu::UpdateOpExecStatus(HcclOpExecFSM& fsmState, KfcStatus state, KfcError& errorCode, uint32_t retryCnt)
3305 : {
3306 7 : HCCL_INFO(
3307 : "UpdateOpExecStatus fsmState %d, state %d, errorCode %d, retryCnt %u.", fsmState, state, errorCode, retryCnt);
3308 7 : auto ret = aicpuHdc_.SetOpExecStatus(kfcStatusTransferD2H_, state, errorCode, retryCnt);
3309 7 : if (ret != HCCL_SUCCESS) {
3310 0 : HCCL_ERROR("SetOpExecStatus failed, ret:%u", ret);
3311 0 : errorCode = KfcError::kExec;
3312 0 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR;
3313 : }
3314 7 : return ret;
3315 : }
3316 :
3317 : static constexpr u32 HCCL_AICPU_WAIT_HOST_BASE_TIME_MS = 200 * 1000;
3318 : static constexpr u32 TIME_S_TO_MS = 1000;
3319 14 : u32 HcclCommAicpu::HcclGetWaitStopExecCmdTimeout()
3320 : {
3321 14 : return std::max(static_cast<u32>(linkTimeOut_.count()), HCCL_AICPU_WAIT_HOST_BASE_TIME_MS);
3322 : }
3323 :
3324 7 : u32 HcclCommAicpu::HcclGetWaitRetryCmdTimeout(uint32_t retryCnt)
3325 : {
3326 7 : if (retryCnt == 0) {
3327 7 : return HcclGetWaitStopExecCmdTimeout() + retryHoldTime_;
3328 : } else {
3329 0 : return HcclGetWaitStopExecCmdTimeout() + retryIntervalTime_;
3330 : }
3331 : }
3332 :
3333 0 : HcclResult HcclCommAicpu::HcclOpExecFsmWaitRetryProcess(
3334 : const OpParam& param, HcclOpExecFSM& fsmState, KfcError& errorCode, KfcCommand& lastCmd)
3335 : {
3336 0 : HCCL_DEBUG("hccl aicpu wait for retry cmd.");
3337 0 : KfcCommand cmd = KfcCommand::kNone;
3338 0 : auto ret = aicpuHdc_.GetOpExecCtrlCmd(kfcControlTransferH2D_, cmd);
3339 0 : if (ret != HCCL_SUCCESS) {
3340 0 : HCCL_ERROR("GetOpExecCtrlCmd failed, ret:%u", ret);
3341 0 : errorCode = KfcError::kExec;
3342 0 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR;
3343 0 : return ret;
3344 : }
3345 0 : if (cmd == KfcCommand::kRetry) {
3346 0 : HCCL_RUN_INFO("[OpRetry][AICPU]hccl aicpu recv retry cmd from host.");
3347 0 : dfxExtendInfo_.pollStatus = PollStatus::kDefault;
3348 0 : dfxExtendInfo_.cqeStatus = dfx::CqeStatus::kDefault;
3349 0 : ret = ResetOpRetryException(param.opType);
3350 0 : HCCL_RETRY_CHK_RET_AND_TRANS_FSM(
3351 : ret, HCCL_ERROR("reset stream buff failed, ret:%u", ret), KfcError::kInner,
3352 : HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR);
3353 0 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_FSM_RETRY;
3354 0 : } else if (cmd == KfcCommand::kChangeLink && lastCmd != KfcCommand::kChangeLink) { // 防止重复执行
3355 0 : HCCL_RUN_INFO("[OpRetry][AICPU]hccl aicpu recv change link cmd, identify[%s]", identifier_.c_str());
3356 0 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_FSM_CHANGE_LINK;
3357 0 : } else if (cmd == KfcCommand::kExit) {
3358 0 : HCCL_RUN_INFO("[OpRetry][AICPU]hccl aicpu recv exit cmd from host.");
3359 0 : errorCode = KfcError::kExit;
3360 0 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR;
3361 0 : } else if (cmd == KfcCommand::kReportRetryErr) {
3362 0 : HCCL_RUN_INFO("[OpRetry][AICPU]hccl aicpu get report retry err cmd.");
3363 0 : CHK_PRT(SendTaskExceptionByMBox(TS_ERROR_RETRY_CONSTRAINT));
3364 0 : errorCode = KfcError::kExit;
3365 0 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR;
3366 0 : return HCCL_E_OPRETRY_FAIL;
3367 0 : } else if (cmd == KfcCommand::NsStopLaunch && endStopLaunch == false) {
3368 0 : HCCL_RUN_INFO("[NsRecovery][AICPU]hccl aicpu force stop in launch loop.");
3369 0 : endStopLaunch = true;
3370 0 : needsResponseStopLaunch_ = true;
3371 0 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_STOP_LAUNCH;
3372 : } else {
3373 : // do nothing
3374 : }
3375 0 : lastCmd = cmd;
3376 0 : return HCCL_SUCCESS;
3377 : }
3378 :
3379 2 : HcclResult HcclCommAicpu::ResetOpRetryException(HcclCMDType opType)
3380 : {
3381 2 : if (opType == HcclCMDType::HCCL_CMD_BATCH_SEND_RECV) {
3382 0 : CHK_RET(ResetBSRException());
3383 : } else {
3384 6 : std::vector<Stream> totalStream = {mainStream_};
3385 2 : totalStream.insert(totalStream.end(), slaveStreams_.begin(), slaveStreams_.end());
3386 4 : for (auto& stream : totalStream) {
3387 2 : CHK_RET(CleanStream(stream));
3388 2 : CHK_RET(ClearStreamCqeException(stream));
3389 : }
3390 2 : }
3391 2 : return HCCL_SUCCESS;
3392 2 : }
3393 :
3394 0 : HcclResult HcclCommAicpu::ResetSqBuff()
3395 : {
3396 0 : CHK_RET(CleanStream(mainStream_));
3397 0 : for (auto& stream : slaveStreams_) {
3398 0 : CHK_RET(CleanStream(stream));
3399 : }
3400 0 : CHK_RET(CleanStream(orderStream_));
3401 0 : HCCL_INFO("reset stream sq buffer success.");
3402 0 : return HCCL_SUCCESS;
3403 : }
3404 :
3405 0 : HcclResult HcclCommAicpu::UpdateSqStatus(Stream& stream)
3406 : {
3407 0 : HcclSqeContext* sqeContext = stream.GetSqeContextPtr();
3408 0 : CHK_PTR_NULL(sqeContext);
3409 0 : SqeRingBuffer* sqeContextBuffer = &(sqeContext->buffer);
3410 0 : CHK_PTR_NULL(sqeContextBuffer);
3411 0 : auto& head = sqeContextBuffer->sqHead;
3412 0 : auto& tail = sqeContextBuffer->sqTail;
3413 :
3414 0 : CHK_RET(QuerySqStatusByType(devId_, stream.sqId(), DRV_SQCQ_PROP_SQ_TAIL, head));
3415 0 : CHK_RET(QuerySqStatusByType(devId_, stream.sqId(), DRV_SQCQ_PROP_SQ_HEAD, tail));
3416 0 : HCCL_INFO("UpdateSqStatus, sqid:%u head:%u tail:%u.", stream.sqId(), head, tail);
3417 0 : return HCCL_SUCCESS;
3418 : }
3419 :
3420 0 : HcclResult HcclCommAicpu::HcclOpExecFsmRetryProcess(
3421 : const std::string& algName, OpParam& param, std::unique_ptr<CollExecutorBase>& executor,
3422 : AlgResourceResponse& algResource, HcclOpExecFSM& fsmState, KfcError& errorCode, uint32_t& retryCnt,
3423 : uint32_t& beginSqePos, uint32_t& endSqePos)
3424 : {
3425 0 : retryCnt++;
3426 0 : if ((param.opType == HcclCMDType::HCCL_CMD_BATCH_SEND_RECV) && retryEnable_) {
3427 0 : UpdateBSRRetryCnt();
3428 : }
3429 0 : HCCL_RUN_INFO("[OpRetry][AICPU]retry launch start, retryCnt:%u, tag[%s].", retryCnt, param.tag.c_str());
3430 :
3431 0 : auto ret = RetryOrchestrateHcclOp(algName, param, executor, algResource, beginSqePos, endSqePos);
3432 0 : if (ret == HCCL_SUCCESS) {
3433 0 : errorCode = KfcError::kNone;
3434 0 : if ((param.opType == HcclCMDType::HCCL_CMD_BATCH_SEND_RECV) && retryEnable_) {
3435 : // 是否有前一个重执行阶段积累的故障未上报，需要再次触发重执行
3436 0 : CHK_RET(CommitBSRStoredException(fsmState, errorCode));
3437 0 : } else {
3438 0 : CHK_RET(UpdateOpExecStatus(fsmState, KfcStatus::kRuning, errorCode, retryCnt));
3439 0 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_FSM_WAIT_END;
3440 : }
3441 0 : } else if (ret == HCCL_E_SUSPENDING) {
3442 0 : HCCL_RUN_INFO("hccl aicpu force stop in retry launch process");
3443 0 : if ((param.opType == HcclCMDType::HCCL_CMD_BATCH_SEND_RECV) && retryEnable_) {
3444 : // batchsendrecv算子下发过程中出现异常，task下发未完成，send 和 recv 均需要重执行
3445 0 : if (bsrRetryOp_ == HCCL_SEND) {
3446 0 : SetBSRSendOpExecException();
3447 : } else {
3448 0 : SetBSRRecvOpExecException();
3449 : }
3450 0 : HCCL_RUN_INFO("hccl aicpu abort launch batchsendrecv op, need retry.");
3451 : }
3452 0 : CHK_RET(UpdateSuspendStatus(param, fsmState, errorCode, retryCnt));
3453 : } else {
3454 0 : HCCL_ERROR("RetryLaunchHcclOp failed, ret:%u", ret);
3455 0 : errorCode = KfcError::kInner;
3456 0 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR;
3457 : }
3458 0 : return ret;
3459 : }
3460 :
3461 0 : HcclResult HcclCommAicpu::HcclOpExecFsmEndProcess(uint32_t retryCnt)
3462 : {
3463 : auto ret
3464 0 : = aicpuHdc_.SetOpExecStatus(kfcStatusTransferD2H_, excuteOpId_, KfcStatus::kEnd, KfcError::kNone, retryCnt);
3465 0 : if (!isDeviceMode_) {
3466 0 : isOpLaunch = false;
3467 : }
3468 0 : dfxExtendInfo_.kfcStatus = DfxKfcStatus::kOneFinished;
3469 0 : HCCL_DEBUG("---------- end AICPU_HcclOpExecFsmEndProcess ----------");
3470 0 : return ret;
3471 : }
3472 :
3473 0 : HcclResult HcclCommAicpu::PrintTaskExceptionAllThreads()
3474 : {
3475 : // 非独立算子场景，跳过
3476 0 : CHK_PRT_RET(
3477 : !GetIsInitIndOp(), HCCL_RUN_INFO("[%s] IndOp group[%s] not init, skip", __func__, identifier_.c_str()),
3478 : HCCL_SUCCESS);
3479 :
3480 0 : for (auto& thread : threads_) {
3481 0 : CHK_RET(taskExecption_.PrintTaskException(*(thread->GetStream())));
3482 : }
3483 0 : return HCCL_SUCCESS;
3484 : }
3485 :
3486 0 : void HcclCommAicpu::PrintTaskExceptionAllComm()
3487 : {
3488 0 : std::shared_lock<std::shared_mutex> rwlock(AicpuHcclProcess::AicpuGetCommMutex());
3489 :
3490 : // 先打印本通信域的taskException
3491 0 : (void)PrintTaskExceptionAllStreams();
3492 0 : (void)PrintTaskExceptionAllThreads();
3493 :
3494 : // 再打印其他通信域的taskException
3495 0 : std::vector<std::pair<std::string, hccl::HcclCommAicpu*>> aicpuCommInfo;
3496 0 : (void)AicpuHcclProcess::AicpuGetCommAll(aicpuCommInfo);
3497 0 : for (auto& commInfo : aicpuCommInfo) {
3498 0 : hccl::HcclCommAicpu* hcclAicpu = commInfo.second;
3499 0 : if (hcclAicpu == nullptr || hcclAicpu->identifier_ == identifier_) {
3500 0 : continue;
3501 : }
3502 0 : (void)hcclAicpu->PrintTaskExceptionAllThreads();
3503 0 : (void)hcclAicpu->PrintTaskExceptionAllStreams();
3504 : }
3505 0 : }
3506 :
3507 0 : void HcclCommAicpu::PrintAicpuCommExecStatus()
3508 : {
3509 0 : std::shared_lock<std::shared_mutex> rwlock(AicpuHcclProcess::AicpuGetCommMutex());
3510 :
3511 : // 记录通信域占核情况
3512 0 : int64_t inExecGroupNum = 0;
3513 0 : int64_t aicpuCoreNum = 0;
3514 0 : (void)hrtHalGetDeviceInfo(devId_, MODULE_TYPE_AICPU, INFO_TYPE_CORE_NUM, &aicpuCoreNum);
3515 :
3516 0 : std::vector<std::pair<std::string, hccl::HcclCommAicpu*>> aicpuCommInfo;
3517 0 : (void)AicpuHcclProcess::AicpuGetCommAll(aicpuCommInfo);
3518 0 : for (auto& commInfo : aicpuCommInfo) {
3519 0 : hccl::HcclCommAicpu* hcclAicpu = commInfo.second;
3520 0 : if (hcclAicpu == nullptr || !hcclAicpu->GetCommInfoStatus()) {
3521 0 : continue;
3522 : }
3523 :
3524 : // 获取并打印通信域是否在执行中，以及最后一次下发的算子
3525 0 : bool isInExec = AicpuHcclProcess::GetCommExecStatus(hcclAicpu->identifier_);
3526 0 : inExecGroupNum += isInExec ? 1 : 0;
3527 0 : std::string execStatus = isInExec ? "inExec" : "unExec";
3528 0 : HCCL_RUN_INFO(
3529 : "AicpuComm: group[%s], status[%s], op[%s], aicpuCoreNum[%lld]", hcclAicpu->identifier_.c_str(),
3530 : execStatus.c_str(), hcclAicpu->GetExcuteOp().c_str(), aicpuCoreNum);
3531 0 : }
3532 :
3533 : // AICPU核被占满，部分通信域得不到调度，可能导致通信阻塞，打印维测信息
3534 0 : if (inExecGroupNum >= aicpuCoreNum && static_cast<int64_t>(aicpuCommInfo.size()) > aicpuCoreNum) {
3535 0 : HCCL_RUN_WARNING(
3536 : "In Execution group num[%lld], total group num[%u], bigger than Aicpu cores num[%lld]. "
3537 : "Aicpu core being fully utilized may cause tasks to get stuck, and it is necessary to reduce the num of "
3538 : "comm.",
3539 : inExecGroupNum, aicpuCommInfo.size(), aicpuCoreNum);
3540 : }
3541 0 : }
3542 :
3543 0 : HcclResult HcclCommAicpu::PrintTaskExceptionAllStreams()
3544 : {
3545 : // 通信域资源已经释放
3546 0 : CHK_PRT_RET(
3547 : !commOpenStatus,
3548 : HCCL_RUN_INFO("[PrintTaskExceptionAllStreams]group[%s] has been destroyed", identifier_.c_str()), HCCL_SUCCESS);
3549 0 : CHK_RET(UtraceInfo_->Flush());
3550 0 : std::vector<Stream> totalStream = {mainStream_};
3551 : HcclResult ret;
3552 0 : totalStream.insert(totalStream.end(), slaveStreams_.begin(), slaveStreams_.end());
3553 0 : for (auto& stream : totalStream) {
3554 0 : HCCL_RUN_INFO("[PrintTaskExceptionAllStreams]group[%s] streamid[%d] print", identifier_.c_str(), stream.id());
3555 0 : u32 sqHead = 0U;
3556 0 : u32 sqTail = 0U;
3557 0 : ret = QuerySqStatus(devId_, stream.sqId(), sqHead, sqTail);
3558 0 : if (ret != HCCL_SUCCESS || sqHead == sqTail) { // 此流为空时，不打印
3559 0 : HCCL_RUN_INFO(
3560 : "[PrintTaskExceptionAllStreams] group[%s] streamid[%d] is empty"
3561 : "or QuerySqStatus failed, ret[%d]",
3562 : identifier_.c_str(), stream.id(), ret);
3563 0 : continue;
3564 : }
3565 0 : HcclSqeContext* sqeContext = stream.GetSqeContextPtr();
3566 0 : SqeRingBuffer* sqeContextBuffer = &(sqeContext->buffer);
3567 0 : CHK_PTR_NULL(sqeContextBuffer);
3568 0 : if (stream.id() == mainStream_.id()) {
3569 0 : SqeInfo sqeInfo;
3570 0 : ret = SqeContextUtils::QuerySqeInfo(
3571 0 : sqeContextBuffer->rtsMirrorBuffer + sqHead * HCCL_SQE_SIZE, sqeContextBuffer->rtsqSqeType[sqHead],
3572 : sqeContextBuffer->addInfo[sqHead], &sqeInfo);
3573 0 : if (ret != HCCL_SUCCESS) {
3574 0 : HCCL_ERROR("[%s]QuerySqeInfo failed, ret[%d]", __func__, ret);
3575 : } else {
3576 : // 根据主流卡在host notify上，则说明未被执行到不打印
3577 0 : if (sqeInfo.type == RT_STARS_SQE_TYPE_NOTIFY_WAIT && sqeInfo.notifyId == opNotifies_[0]->notifyId_) {
3578 0 : HCCL_RUN_INFO(
3579 : "[PrintTaskExceptionAllStreams] group[%s] op is not activated, do nothing",
3580 : identifier_.c_str());
3581 0 : return HCCL_SUCCESS;
3582 : }
3583 : // 根据主流当前位置，判断该算子是否已经打印过taskException
3584 0 : if (IsRepeatedOpTaskException(sqHead, sqeContextBuffer)) {
3585 0 : HCCL_INFO(
3586 : "[PrintTaskExceptionAllStreams] group[%s] op has been printed, do nothing",
3587 : identifier_.c_str());
3588 0 : return HCCL_SUCCESS;
3589 : }
3590 : }
3591 : }
3592 :
3593 0 : uint8_t type = 0;
3594 0 : uint16_t taskId = 0;
3595 0 : uint32_t remoteRank = 0;
3596 0 : HCCL_ERROR(
3597 : "[TaskException]base information is streamId:%d, sqid:%d, head:%u, tail:%u, %s", stream.id(), stream.sqId(),
3598 : sqHead, sqTail, GetTaskExceptionTaskInfo(sqHead, sqeContextBuffer, type, taskId, remoteRank).c_str());
3599 0 : PrintTaskExceptionTaskQue(sqHead, sqeContextBuffer);
3600 : }
3601 0 : return HCCL_SUCCESS;
3602 0 : }
3603 :
3604 0 : bool HcclCommAicpu::IsRepeatedOpTaskException(u32 idx, SqeRingBuffer* sqeContextBuffer)
3605 : {
3606 0 : const AicpuOpInfo* opInfo = aicpuShareData_.GetAicpuOpInfo(sqeContextBuffer->rtsDfxInfo[idx].opRingBufferIdx);
3607 0 : CHK_PRT_RET(opInfo == nullptr, HCCL_ERROR("%s fail, opInfo is nullptr", __func__), false);
3608 0 : std::string opTag = opInfo->tagBuff;
3609 0 : u32 opIndex = opInfo->opIndex;
3610 0 : bool opHasPrinted = opTaskException_.find(opTag) != opTaskException_.end() && opTaskException_[opTag] == opIndex;
3611 0 : opTaskException_[opTag] = opIndex;
3612 0 : CHK_PRT_CONT(
3613 : opHasPrinted, HCCL_RUN_INFO(
3614 : "[IsRepeatedOpTaskException]group[%s], op[%s], opIndex[%u] "
3615 : "has been printed",
3616 : identifier_.c_str(), opInfo->tagBuff, opInfo->opIndex));
3617 0 : return opHasPrinted;
3618 0 : }
3619 :
3620 0 : void HcclCommAicpu::PrepareMc2Handler()
3621 : {
3622 0 : auto& handler = algOpContext_.mc2Handler;
3623 0 : handler.stepSize = 0U;
3624 0 : if (!isDeviceMode_) {
3625 0 : HCCL_INFO("Unset step size for non-MC2.");
3626 0 : return;
3627 : }
3628 0 : (void)InvokeKfcHandler(AicpuKfcHandlerType::kSetStepSize, {rpc_, reinterpret_cast<u64>(&handler), GetRankSize()});
3629 : }
3630 :
3631 0 : HcclResult HcclCommAicpu::OrchestrateHcclOp(
3632 : const std::string& algName, OpParam& param, std::unique_ptr<CollExecutorBase>& executor,
3633 : AlgResourceResponse& algResource, uint32_t& beginSqePos, uint32_t& endSqePos)
3634 : {
3635 0 : LogControl logControl(false, false); // 重执行ERROR日志控制，析构时重置日志设置
3636 0 : PrepareMc2Handler();
3637 0 : HcclResult ret = HCCL_SUCCESS;
3638 : // task的尾指针，已便重执行stop时判断是否已执行该task，如果该task已执行完成则可支持通信重执行
3639 0 : CHK_RET(QuerySqStatusByType(devId_, mainStream_.sqId(), DRV_SQCQ_PROP_SQ_TAIL, beginSqePos));
3640 :
3641 0 : const bool retryForBatchSndRcv = (param.opType == HcclCMDType::HCCL_CMD_BATCH_SEND_RECV && retryEnable_);
3642 0 : if (retryForBatchSndRcv) {
3643 0 : CHK_RET(QueryBatchSendRecvPairBeginPos());
3644 0 : if (param.BatchSendRecvDataDes.curIterNum == 0) {
3645 : // batchsendrecv算子拆分为多轮执行，只有第一个step和最后一个step需要和主stream交互
3646 0 : CHK_RET(NotifyWait());
3647 : }
3648 0 : HCCL_INFO(
3649 : "batch send recv op: step %u, mode:%u", param.BatchSendRecvDataDes.curIterNum,
3650 : param.BatchSendRecvDataDes.curMode);
3651 : } else {
3652 0 : CHK_RET(NotifyWait());
3653 : // 重执行场景, 算子计数在host侧; MC2场景也不开启卡住检测能力
3654 0 : if (opCounterInfo_.isEnableCounter && !retryEnable_ && !isDeviceMode_) {
3655 0 : CHK_RET(HcclReduceAsync(
3656 : dispatcher_, reinterpret_cast<void*>(opCounterInfo_.addOneMem),
3657 : opCounterInfo_.memSize / sizeof(int32_t), HCCL_DATA_TYPE_INT32, HCCL_REDUCE_SUM, mainStream_,
3658 : reinterpret_cast<void*>(opCounterInfo_.headCountMem), INVALID_VALUE_RANKID, LinkType::LINK_ONCHIP,
3659 : INLINE_REDUCE_BIT));
3660 : }
3661 : }
3662 :
3663 : // 打印当前展开的算子信息
3664 0 : HCCL_INFO(
3665 : "[HcclCommAicpu][OrchestrateHcclOp] opUnfoldIdx_[%u] opType[%d] curRank[%u] rankSize[%u] algName[%s]",
3666 : opUnfoldIdx_, param.opType, topoInfo_.userRank, GetRankSize(), algName.c_str());
3667 0 : HCCL_INFO(
3668 : "[HcclCommAicpu][OrchestrateHcclOp] inputPtr[0x%016llx] inputSize[%llu] outputPtr[0x%016llx] outputSize[%llu]",
3669 : param.inputPtr, param.inputSize, param.outputPtr, param.outputSize);
3670 0 : opUnfoldIdx_ += 1;
3671 :
3672 : // 检查算子展开的动态缓存, 确认是否可以跳过算子展开
3673 0 : bool needExecute = true;
3674 0 : bool isCacheMiss = false;
3675 0 : auto setProfStartCallback = [this]() {
3676 0 : return this->InvokeKfcHandler(AicpuKfcHandlerType::kSetProfTimeStart, {});
3677 0 : };
3678 0 : CHK_RET(aicpuCacheManager_.LookupOpUnfoldCache(
3679 : algName, param, algResource, needExecute, isCacheMiss, mainStream_, slaveStreams_, dispatcher_, isDeviceMode_,
3680 : topoInfo_, topoMatcher_, algOpContext_, ZeroCopyExchanger_, GetWorkflowMode(), tinySendRecvMem_,
3681 : setProfStartCallback));
3682 :
3683 : // 根据needExecute有条件的执行算子展开
3684 : // 需要算子执行的场景: (i) expansion mode为AI_CPU_NO_CACHE; (ii) uncacheable算子/场景; (iii) cache miss
3685 0 : if (needExecute) {
3686 : // Cache miss前执行cache相关的预处理
3687 0 : if (isCacheMiss) {
3688 0 : CHK_RET(aicpuCacheManager_.PreProcessForCacheMiss(param, executor));
3689 : }
3690 :
3691 : // executor设置AlgOpContext
3692 0 : CHK_RET(executor->SetAlgOpContext(algOpContext_));
3693 0 : (void)InvokeKfcHandler(AicpuKfcHandlerType::kSetProfTimeStart, {});
3694 0 : ret = executor->Orchestrate(param, algResource);
3695 0 : if (ret != HCCL_SUCCESS) {
3696 0 : HCCL_ERROR(
3697 : "[HcclCommAicpu][Orchestrate]executor process failed algName[%s], ret = %u", algName.c_str(), ret);
3698 0 : printTaskExceptionForErr_ |= (ret == HCCL_E_AGAIN);
3699 0 : return ret;
3700 : }
3701 :
3702 : // Cache miss后执行cache相关的后处理
3703 0 : if (isCacheMiss) {
3704 0 : CHK_RET(aicpuCacheManager_.PostProcessForCacheMiss(
3705 : param, executor, mainStream_, slaveStreams_, dispatcher_, topoInfo_, algOpContext_, GetWorkflowMode()));
3706 : }
3707 : } // 正常算子展开
3708 :
3709 : // batchsendrecv算子拆分为多轮执行，只有第一个step和最后一个step需要和主stream交互
3710 0 : if (!retryForBatchSndRcv || param.BatchSendRecvDataDes.curIterNum + 1 >= bsrSendRecvPairs_.size()) {
3711 : // 重执行场景, 算子计数在host侧 MC2场景也不开
3712 0 : if (opCounterInfo_.isEnableCounter && !retryEnable_ && !isDeviceMode_) {
3713 0 : CHK_RET(HcclReduceAsync(
3714 : dispatcher_, reinterpret_cast<void*>(opCounterInfo_.addOneMem),
3715 : opCounterInfo_.memSize / sizeof(int32_t), HCCL_DATA_TYPE_INT32, HCCL_REDUCE_SUM, mainStream_,
3716 : reinterpret_cast<void*>(opCounterInfo_.tailCountMem), INVALID_VALUE_RANKID, LinkType::LINK_ONCHIP,
3717 : INLINE_REDUCE_BIT));
3718 : }
3719 0 : CHK_RET(NotifyPost());
3720 : }
3721 0 : (void)InvokeKfcHandler(AicpuKfcHandlerType::kSetProfTimeOrch, {});
3722 0 : ret = LaunchTask(dispatcher_, const_cast<Stream&>(mainStream_));
3723 0 : if (ret != HCCL_SUCCESS) {
3724 0 : HCCL_ERROR("[HcclCommAicpu][LaunchTask]algName[%s] ret = %u", algName.c_str(), ret);
3725 0 : printTaskExceptionForErr_ |= (ret == HCCL_E_AGAIN);
3726 0 : return ret;
3727 : }
3728 0 : ret = LaunchSlaveStreamTask(algResource);
3729 0 : if (ret != HCCL_SUCCESS) {
3730 0 : HCCL_ERROR("[HcclCommAicpu][LaunchSlaveStreamTask]algName[%s] ret = %u", algName.c_str(), ret);
3731 0 : printTaskExceptionForErr_ |= (ret == HCCL_E_AGAIN);
3732 0 : return ret;
3733 : }
3734 0 : (void)InvokeKfcHandler(AicpuKfcHandlerType::kSetProfTimeEnd, {});
3735 0 : if (retryForBatchSndRcv) {
3736 0 : CHK_RET(QueryBatchSendRecvPairEndPos());
3737 : }
3738 :
3739 0 : CHK_RET(QuerySqStatusByType(devId_, mainStream_.sqId(), DRV_SQCQ_PROP_SQ_TAIL, endSqePos));
3740 :
3741 0 : HCCL_INFO(
3742 : "hccl aicpu launch hccl op task success. stream sqid:%u begin:%u end:%u", mainStream_.sqId(), beginSqePos,
3743 : endSqePos);
3744 0 : return HCCL_SUCCESS;
3745 0 : }
3746 :
3747 0 : HcclResult HcclCommAicpu::RetryOrchestrateHcclOp(
3748 : const std::string& algName, OpParam& param, std::unique_ptr<CollExecutorBase>& executor,
3749 : AlgResourceResponse& algResource, uint32_t& beginSqePos, uint32_t& endSqePos)
3750 : {
3751 0 : LogControl logControl(false, false); // 重执行ERROR日志控制，析构时重置日志设置
3752 0 : if ((param.opType == HcclCMDType::HCCL_CMD_BATCH_SEND_RECV) && retryEnable_) {
3753 : param.BatchSendRecvDataDes.curMode
3754 0 : = (bsrRetryOp_ == HCCL_SEND) ? BatchSendRecvCurMode::SEND : BatchSendRecvCurMode::RECV;
3755 0 : if (param.BatchSendRecvDataDes.curMode == BatchSendRecvCurMode::SEND) {
3756 0 : HCCL_INFO(
3757 : "BSR: iter %u, retry send op tag:%s index:%u", param.BatchSendRecvDataDes.curIterNum, bsrSendOpId_.tag,
3758 : bsrSendOpId_.index);
3759 : } else {
3760 0 : HCCL_INFO(
3761 : "BSR: iter %u, retry recv op tag:%s index:%u", param.BatchSendRecvDataDes.curIterNum, bsrRecvOpId_.tag,
3762 : bsrRecvOpId_.index);
3763 : }
3764 : }
3765 :
3766 0 : CHK_RET(AddRetryExecFlipTask(algResource));
3767 0 : HcclResult ret = executor->Orchestrate(param, algResource);
3768 0 : if (ret != HCCL_SUCCESS) {
3769 0 : HCCL_ERROR("[HcclCommAicpu][Orchestrate]executor process failed algName[%s]", algName.c_str());
3770 0 : return ret;
3771 : }
3772 :
3773 0 : if ((param.opType == HcclCMDType::HCCL_CMD_BATCH_SEND_RECV) && retryEnable_) {
3774 : // batchsendrecv算子重执行时，aicpu 主stream没有clean，不需要重新下发notify record
3775 : // do nothing
3776 : } else {
3777 0 : CHK_RET(NotifyPost());
3778 : }
3779 0 : CHK_RET(LaunchTask(dispatcher_, const_cast<Stream&>(mainStream_)));
3780 0 : CHK_RET(LaunchSlaveStreamTask(algResource));
3781 :
3782 0 : CHK_RET(QuerySqStatusByType(devId_, mainStream_.sqId(), DRV_SQCQ_PROP_SQ_TAIL, endSqePos));
3783 0 : if ((param.opType == HcclCMDType::HCCL_CMD_BATCH_SEND_RECV) && retryEnable_) {
3784 0 : CHK_RET(QueryBatchSendRecvPairEndPos());
3785 : }
3786 :
3787 0 : HCCL_RUN_INFO(
3788 : "[OpRetry][AICPU]hccl aicpu retry launch hccl op task success. stream sqid:%u begin:%u end:%u",
3789 : mainStream_.sqId(), beginSqePos, endSqePos);
3790 0 : return HCCL_SUCCESS;
3791 0 : }
3792 :
3793 4 : bool HcclCommAicpu::IsTaskExceptionForHccs()
3794 : {
3795 4 : if (dfxExtendInfo_.cqeStatus != dfx::CqeStatus::kCqeException) {
3796 4 : return false;
3797 : }
3798 :
3799 : // NOTE: 需要task exception补全dfx能力，定位故障task的remote rank;
3800 : // 目前暂不具备识别是否跨片的能力，默认失败的task均为跨片操作。
3801 0 : if (dfxExtendInfo_.cqeException.sqeType == RT_STARS_SQE_TYPE_SDMA
3802 0 : && (dfxExtendInfo_.cqeException.errorCode == RT_SDMA_COMPDATAERR
3803 0 : || dfxExtendInfo_.cqeException.errorCode == RT_SDMA_COMPERR)) {
3804 0 : return true;
3805 : }
3806 0 : return false;
3807 : }
3808 :
3809 5 : void HcclCommAicpu::SetAlgType(u64 algType)
3810 : {
3811 5 : algType_.algoLevel0 = static_cast<AlgTypeLevel0>(static_cast<u32>(algType) & ((1 << HCCL_LEVEL_ALGO_WIDTH) - 1));
3812 5 : algType_.algoLevel1 = static_cast<AlgTypeLevel1>(
3813 5 : (static_cast<u32>(algType) >> HCCL_LEVEL_ALGO_WIDTH) & ((1 << HCCL_LEVEL_ALGO_WIDTH) - 1));
3814 : algType_.algoLevel2
3815 5 : = static_cast<AlgTypeLevel2>(static_cast<u32>(algType) >> (HCCL_LEVEL_ALGO_WIDTH + HCCL_LEVEL_ALGO_WIDTH));
3816 5 : HCCL_INFO("[HcclCommAicpu][SetAlgType]algType:%u", algType);
3817 5 : }
3818 :
3819 2 : void HcclCommAicpu::SetDebugMode(u8 debugMode) { debugMode_ = debugMode; }
3820 :
3821 0 : void HcclCommAicpu::SetSendRecvInfoPtr(void* sendRecvInfoPtr) { sendRecvInfoPtr_ = sendRecvInfoPtr; }
3822 :
3823 4 : bool HcclCommAicpu::IsNoNeedWait(void)
3824 : {
3825 4 : return isDeviceMode_ || (debugMode_ != MC2_DEBUG_WAIT_COMM && retryEnable_ == false);
3826 : }
3827 :
3828 45 : bool HcclCommAicpu::GetOpRetryEnable() { return retryEnable_; }
3829 :
3830 4 : HcclResult HcclCommAicpu::ReportHcclTaskInfo(Stream& mainStream, std::vector<Stream>& subStreams)
3831 : {
3832 4 : if (dfx::ProfilingManager::GetProfL1State()) {
3833 0 : CHK_RET(dfx::ProfilingManager::ReportTaskInfo(mainStream.id(), mainStream.GetSqeContextPtr()));
3834 0 : for (auto& subStream : subStreams) {
3835 0 : CHK_RET(dfx::ProfilingManager::ReportTaskInfo(subStream.id(), subStream.GetSqeContextPtr()));
3836 : }
3837 : }
3838 4 : return HCCL_SUCCESS;
3839 : }
3840 :
3841 4 : HcclResult HcclCommAicpu::ClearLocalBuff(Stream& mainStream, std::vector<Stream>& subStreams)
3842 : {
3843 4 : CHK_RET(mainStream.ClearLocalBuff());
3844 4 : CHK_RET(dfx::ProfilingManager::UpdateStartReportSqeIdx(mainStream.id(), 0));
3845 4 : for (auto& subStream : subStreams) {
3846 0 : CHK_RET(subStream.ClearLocalBuff());
3847 0 : CHK_RET(dfx::ProfilingManager::UpdateStartReportSqeIdx(subStream.id(), 0));
3848 : }
3849 4 : return HCCL_SUCCESS;
3850 : }
3851 :
3852 4 : HcclResult HcclCommAicpu::WaitFinishWhileLoop(
3853 : Stream& mainStream, std::vector<Stream>& subStreams, std::string& tag, const uint32_t& beginSqePos, OpParam& param)
3854 : {
3855 : // 上报Profiling HCCL INFO信息
3856 4 : CHK_RET(ReportHcclTaskInfo(mainStream, subStreams));
3857 4 : CHK_RET(ClearLocalBuff(mainStream, subStreams));
3858 4 : if (IsNoNeedWait()) {
3859 0 : return HCCL_SUCCESS;
3860 : }
3861 4 : const uint64_t startUsec = GetCurCpuTimestamp();
3862 4 : uint64_t lastUsec = startUsec;
3863 4 : int32_t sqId = mainStream.sqId();
3864 4 : uint32_t sqHead = 0;
3865 4 : uint32_t sqTail = 0;
3866 4 : CHK_RET(QuerySqStatusByType(devId_, sqId, DRV_SQCQ_PROP_SQ_TAIL, sqTail));
3867 4 : CHK_RET(QuerySqStatusByType(devId_, sqId, DRV_SQCQ_PROP_SQ_HEAD, sqHead));
3868 4 : while (sqHead != sqTail) {
3869 4 : HcclResult ret = CheckOpExecStatus(); // 检查执行状态，判断是否有异常cq或中断命令
3870 4 : CHK_PRT_RET(
3871 : ret != HCCL_SUCCESS,
3872 : HCCL_RUN_INFO("[HcclCommAicpu][WaitFinishWhileLoop]CheckOpExecStatus exception, ret[%u]", ret), ret);
3873 :
3874 0 : CHK_RET(QuerySqStatusByType(devId_, sqId, DRV_SQCQ_PROP_SQ_HEAD, sqHead));
3875 0 : if ((param.opType == HcclCMDType::HCCL_CMD_BATCH_SEND_RECV) && (retryEnable_) && (sqHead != beginSqePos)
3876 0 : && (!excuteOpId_.isBsrTaskStart)) {
3877 : // 更新D2H通道里的信息opid中isBsrTaskStart = true
3878 0 : excuteOpId_.isBsrTaskStart = true;
3879 : HcclResult ret1
3880 0 : = aicpuHdc_.SetOpExecStatus(kfcStatusTransferD2H_, excuteOpId_, KfcStatus::kRuning, KfcError::kNone, 0);
3881 0 : CHK_PRT_RET(ret1 != HCCL_SUCCESS, HCCL_ERROR("update OpExecStatus failed, ret:%u", ret1), ret1);
3882 0 : HCCL_INFO(
3883 : "[HcclCommAicpu][WaitFinishWhileLoop]bsr start task is completed. devId:%d sqid:%d, head:%u,"
3884 : "beginSqePos[%u] group[%s] tag[%s]",
3885 : devId_, sqId, sqHead, beginSqePos, identifier_.c_str(), tag.c_str());
3886 : }
3887 0 : uint64_t curUsec = GetCurCpuTimestamp();
3888 0 : if (curUsec - lastUsec > static_cast<uint64_t>(NSEC_PER_SEC) * dfx::kPrintSqInterval) {
3889 0 : lastUsec = curUsec;
3890 0 : HCCL_RUN_INFO(
3891 : "[HcclCommAicpu][WaitFinishWhileLoop]Current state. devId:%d sqid:%d, head:%u, tail:%u, "
3892 : "group[%s] tag[%s]",
3893 : devId_, sqId, sqHead, sqTail, identifier_.c_str(), tag.c_str());
3894 : }
3895 0 : CHK_RET(CheckTaskTimeout(mainStream, startUsec));
3896 : }
3897 0 : return HCCL_SUCCESS;
3898 : }
3899 :
3900 0 : HcclResult HcclCommAicpu::CheckTaskTimeout(const Stream& mainStream, const uint64_t startUsec)
3901 : {
3902 0 : if (sqeWaitTimeOut_ != 0
3903 0 : && (GetCurCpuTimestamp() - startUsec > static_cast<uint64_t>(NSEC_PER_SEC) * sqeWaitTimeOut_)) {
3904 0 : uint32_t status = 0U;
3905 0 : int32_t sqId = mainStream.sqId();
3906 0 : auto ret = QuerySqStatusByType(devId_, sqId, DRV_SQCQ_PROP_SQ_CQE_STATUS, status);
3907 0 : if (ret != 0) {
3908 0 : HCCL_ERROR("[HcclCommAicpu]QuerySqStatusByType status failed. ret = %u sqid:%d", ret, sqId);
3909 : }
3910 :
3911 0 : HCCL_ERROR("[HcclCommAicpu]KFC timeout.. group[%s].", identifier_.c_str());
3912 0 : printTaskExceptionForErr_ = true;
3913 0 : return HCCL_E_TIMEOUT;
3914 : }
3915 0 : return HCCL_SUCCESS;
3916 : }
3917 :
3918 0 : HcclResult HcclCommAicpu::AddRetryExecFlipTask(AlgResourceResponse& algResource)
3919 : {
3920 0 : CHK_RET(AddRetryPreamble(dispatcher_, mainStream_));
3921 0 : for (u32 i = 0; i < algResource.slaveStreams.size(); ++i) {
3922 0 : HcclResult ret = AddRetryPreamble(dispatcher_, algResource.slaveStreams[i]);
3923 0 : if (ret != HCCL_SUCCESS) {
3924 0 : HCCL_ERROR(
3925 : "[HcclCommAicpu][RetryOrchestrateHcclOp] launch place holder failed, sqid:%u, ret:%u",
3926 : algResource.slaveStreams[i].sqId(), ret);
3927 0 : return ret;
3928 : }
3929 : }
3930 0 : return HCCL_SUCCESS;
3931 : }
3932 :
3933 2 : HcclResult HcclCommAicpu::LaunchSlaveStreamTask(AlgResourceResponse& algResource)
3934 : {
3935 : // 单算子模式中在算法编排中已经执行过LaunchTask，所以这里不需要再执行
3936 : // 只有图模式需要再额外执行一次对从流中的task下发
3937 2 : if (GetWorkflowMode() != HcclWorkflowMode::HCCL_WORKFLOW_MODE_OPS_KERNEL_INFO_LIB) {
3938 1 : HCCL_INFO("[HcclCommAicpu][LaunchSlaveStreamTask] op base mode don't need launch slave stream task");
3939 1 : return HCCL_SUCCESS;
3940 : }
3941 :
3942 1 : for (u32 i = 0; i < algResource.slaveStreams.size(); ++i) {
3943 0 : HcclResult ret = LaunchTask(dispatcher_, algResource.slaveStreams[i]);
3944 0 : if (ret != HCCL_SUCCESS) {
3945 0 : HCCL_ERROR(
3946 : "[HcclCommAicpu][LaunchSlaveStreamTask] launch task failed, sqid:%u, ret:%u",
3947 : algResource.slaveStreams[i].sqId(), ret);
3948 0 : return ret;
3949 : }
3950 : }
3951 :
3952 1 : return HCCL_SUCCESS;
3953 : }
3954 :
3955 : HcclResult
3956 0 : HcclCommAicpu::GetAlltoAllvSendRecvInfo(const void* sendRecvInfoPtr, HcclDataType sendType, HcclDataType recvType)
3957 : {
3958 0 : allMeshAggregationSendRecvInfo_.clear();
3959 0 : u64 stepSize = sizeof(u64) * topoInfo_.userRankSize;
3960 0 : const u32 addrItemNum = 4;
3961 0 : const u32 recvLengthStep = 2;
3962 0 : const u32 recvOffsetStep = 3;
3963 0 : for (u32 i = 0; i < topoInfo_.userRankSize; i++) {
3964 0 : SendRecvInfo sendRecvInfo;
3965 0 : sendRecvInfo.sendLength.resize(topoInfo_.userRankSize);
3966 0 : sendRecvInfo.sendOffset.resize(topoInfo_.userRankSize);
3967 0 : sendRecvInfo.recvLength.resize(topoInfo_.userRankSize);
3968 0 : sendRecvInfo.recvOffset.resize(topoInfo_.userRankSize);
3969 0 : CHK_SAFETY_FUNC_RET(memcpy_s(
3970 : sendRecvInfo.sendLength.data(), stepSize,
3971 : static_cast<const u8*>(sendRecvInfoPtr) + i * stepSize * addrItemNum + 0 * stepSize, stepSize));
3972 0 : CHK_SAFETY_FUNC_RET(memcpy_s(
3973 : sendRecvInfo.sendOffset.data(), stepSize,
3974 : static_cast<const u8*>(sendRecvInfoPtr) + i * stepSize * addrItemNum + stepSize, stepSize));
3975 0 : CHK_SAFETY_FUNC_RET(memcpy_s(
3976 : sendRecvInfo.recvLength.data(), stepSize,
3977 : static_cast<const u8*>(sendRecvInfoPtr) + i * stepSize * addrItemNum + recvLengthStep * stepSize,
3978 : stepSize));
3979 0 : CHK_SAFETY_FUNC_RET(memcpy_s(
3980 : sendRecvInfo.recvOffset.data(), stepSize,
3981 : static_cast<const u8*>(sendRecvInfoPtr) + i * stepSize * addrItemNum + recvOffsetStep * stepSize,
3982 : stepSize));
3983 0 : allMeshAggregationSendRecvInfo_.push_back(std::move(sendRecvInfo));
3984 0 : }
3985 :
3986 0 : for (auto& sendRecvInfo : allMeshAggregationSendRecvInfo_) {
3987 0 : for (u32 i = 0; i < topoInfo_.userRankSize; i++) {
3988 0 : sendRecvInfo.sendCounts.push_back(sendRecvInfo.sendLength[i] / SIZE_TABLE[sendType]);
3989 0 : sendRecvInfo.sendDispls.push_back(sendRecvInfo.sendOffset[i] / SIZE_TABLE[sendType]);
3990 0 : sendRecvInfo.recvCounts.push_back(sendRecvInfo.recvLength[i] / SIZE_TABLE[recvType]);
3991 0 : sendRecvInfo.recvDispls.push_back(sendRecvInfo.recvOffset[i] / SIZE_TABLE[recvType]);
3992 0 : HCCL_INFO(
3993 : "[GetAlltoAllvSendRecvInfo] rank[%u], sendCounts[%llu], sendDispls[%llu], "
3994 : "recvCounts[%llu], recvDispls[%llu]",
3995 : i, sendRecvInfo.sendCounts[i], sendRecvInfo.sendDispls[i], sendRecvInfo.recvCounts[i],
3996 : sendRecvInfo.recvDispls[i]);
3997 0 : HCCL_INFO(
3998 : "[GetAlltoAllvSendRecvInfo] rank[%u], sendLength[%llu], sendOffset[%llu], "
3999 : "recvLength[%llu], recvOffset[%llu]",
4000 : i, sendRecvInfo.sendLength[i], sendRecvInfo.sendOffset[i], sendRecvInfo.recvLength[i],
4001 : sendRecvInfo.recvOffset[i]);
4002 : }
4003 : }
4004 0 : CHK_RET(CheckSendRecvParams(allMeshAggregationSendRecvInfo_));
4005 0 : return HCCL_SUCCESS;
4006 : }
4007 :
4008 : HcclResult
4009 0 : HcclCommAicpu::GetAlltoAllvcSendRecvInfo(const void* sendCountMatrix, HcclDataType sendType, HcclDataType recvType)
4010 : {
4011 0 : allMeshAggregationSendRecvInfo_.clear();
4012 0 : for (u32 i = 0; i < topoInfo_.userRankSize; i++) {
4013 0 : SendRecvInfo sendRecvInfo;
4014 0 : sendRecvInfo.sendCounts.resize(topoInfo_.userRankSize);
4015 0 : sendRecvInfo.sendDispls.resize(topoInfo_.userRankSize);
4016 0 : sendRecvInfo.sendLength.resize(topoInfo_.userRankSize);
4017 0 : sendRecvInfo.sendOffset.resize(topoInfo_.userRankSize);
4018 0 : u64 curSendDispls = 0;
4019 0 : u64 curSendOffset = 0;
4020 0 : sendRecvInfo.recvCounts.resize(topoInfo_.userRankSize);
4021 0 : sendRecvInfo.recvDispls.resize(topoInfo_.userRankSize);
4022 0 : sendRecvInfo.recvLength.resize(topoInfo_.userRankSize);
4023 0 : sendRecvInfo.recvOffset.resize(topoInfo_.userRankSize);
4024 0 : u64 curRecvDispls = 0;
4025 0 : u64 curRecvOffset = 0;
4026 0 : for (u32 j = 0; j < topoInfo_.userRankSize; j++) {
4027 0 : u64 curSendCounts = *(static_cast<const u64*>(sendCountMatrix) + i * topoInfo_.userRankSize + j);
4028 0 : u64 curSendLength = curSendCounts * SIZE_TABLE[sendType];
4029 0 : sendRecvInfo.sendCounts[j] = curSendCounts;
4030 0 : sendRecvInfo.sendDispls[j] = curSendDispls;
4031 0 : sendRecvInfo.sendLength[j] = curSendLength;
4032 0 : sendRecvInfo.sendOffset[j] = curSendOffset;
4033 0 : curSendDispls += curSendCounts;
4034 0 : curSendOffset += curSendLength;
4035 0 : u64 curRecvCounts = *(static_cast<const u64*>(sendCountMatrix) + i + topoInfo_.userRankSize * j);
4036 0 : u64 curRecvLength = curRecvCounts * SIZE_TABLE[recvType];
4037 0 : sendRecvInfo.recvCounts[j] = curRecvCounts;
4038 0 : sendRecvInfo.recvDispls[j] = curRecvDispls;
4039 0 : sendRecvInfo.recvLength[j] = curRecvLength;
4040 0 : sendRecvInfo.recvOffset[j] = curRecvOffset;
4041 0 : curRecvDispls += curRecvCounts;
4042 0 : curRecvOffset += curRecvLength;
4043 0 : HCCL_DEBUG(
4044 : "GetAlltoAllvcSendRecvInfo rank[%u], sendCounts[%llu], sendDispls[%llu] "
4045 : "recvCounts[%llu], recvDispls[%llu]",
4046 : i, sendRecvInfo.sendCounts[j], sendRecvInfo.sendDispls[j], sendRecvInfo.recvCounts[j],
4047 : sendRecvInfo.recvDispls[j]);
4048 : }
4049 0 : allMeshAggregationSendRecvInfo_.push_back(sendRecvInfo);
4050 0 : }
4051 0 : CHK_RET(CheckSendRecvParams(allMeshAggregationSendRecvInfo_));
4052 0 : return HCCL_SUCCESS;
4053 : }
4054 :
4055 0 : HcclResult HcclCommAicpu::CheckSendRecvParams(const std::vector<SendRecvInfo>& allMeshAggregationSendRecvInfo)
4056 : {
4057 0 : u32 rankSize = allMeshAggregationSendRecvInfo.size();
4058 0 : for (u32 i = 0; i < rankSize; i++) {
4059 0 : u32 sendsSize = allMeshAggregationSendRecvInfo[i].sendLength.size();
4060 0 : u32 recvsSize = allMeshAggregationSendRecvInfo[i].recvLength.size();
4061 0 : if (rankSize != sendsSize || rankSize != recvsSize) {
4062 0 : HCCL_ERROR(
4063 : "[AlltoAllV][CheckSendRecvParam] rankSize[%u], sendsSize[%u], recvsSize[%u] are not match Index[%u]",
4064 : rankSize, sendsSize, recvsSize, i);
4065 0 : return HCCL_E_PARA;
4066 : }
4067 0 : for (u32 j = 0; j < sendsSize; j++) {
4068 0 : if (allMeshAggregationSendRecvInfo[i].sendLength[j] != allMeshAggregationSendRecvInfo[j].recvLength[i]) {
4069 0 : HCCL_ERROR(
4070 : "SendLength[%u][%u]: %llu and recvLength[%u][%u]: %llu are not match", i, j,
4071 : allMeshAggregationSendRecvInfo[i].sendLength[j], j, i,
4072 : allMeshAggregationSendRecvInfo[j].recvLength[i]);
4073 0 : return HCCL_E_PARA;
4074 : }
4075 : }
4076 : }
4077 0 : return HCCL_SUCCESS;
4078 : }
4079 0 : HcclResult HcclCommAicpu::GetStreamAll(std::vector<Stream>& streams)
4080 : {
4081 0 : streams.assign(slaveStreams_.begin(), slaveStreams_.end());
4082 0 : streams.push_back(mainStream_);
4083 0 : return HCCL_SUCCESS;
4084 : }
4085 :
4086 : // 校验是否有ERROR CQE和停止/退出命令，注册到dispatcher层调用
4087 0 : HcclResult HcclCommAicpu::CheckOpExecStatusCallback()
4088 : {
4089 0 : HcclResult ret = CheckOpExecStatus();
4090 0 : bool logLevel = (ret == HCCL_E_SUSPENDING);
4091 : // 返回HCCL_E_SUSPENDING时，需要跨作用域修改ERROR日志->RUN_WARNING
4092 0 : LogControl(logLevel, logLevel);
4093 0 : return ret;
4094 : }
4095 :
4096 4 : HcclResult HcclCommAicpu::CheckOpExecStatus()
4097 : {
4098 : // 检测是否有ERROR CQE
4099 4 : if (dfxExtendInfo_.pollStatus == PollStatus::kStopAsException) {
4100 0 : if (IsTaskExceptionForHccs() && retryEnable_) {
4101 0 : HCCL_RUN_INFO(
4102 : "[OpRetry][AICPU]hccl aicpu stop wait task exec finish, for task exception, identify[%s]",
4103 : identifier_.c_str());
4104 0 : return HCCL_E_SUSPENDING;
4105 : } else {
4106 0 : if (!printTaskExceptionForErr_) {
4107 0 : printTaskExceptionForErr_ = true;
4108 0 : HCCL_ERROR(
4109 : "hccl aicpu exec failed, for task exception, identify[%s], cqeStatus[%d], sqeType[%u], "
4110 : "errorCode[%u]",
4111 : identifier_.c_str(), dfxExtendInfo_.cqeStatus, dfxExtendInfo_.cqeException.sqeType,
4112 : dfxExtendInfo_.cqeException.errorCode);
4113 : }
4114 0 : return HCCL_E_INTERNAL;
4115 : }
4116 : }
4117 :
4118 : // 检测是否有停止/退出命令
4119 4 : KfcCommand cmd = KfcCommand::kNone;
4120 4 : CHK_RET(aicpuHdc_.GetOpExecCtrlCmd(kfcControlTransferH2D_, cmd));
4121 4 : if (cmd == KfcCommand::kStopLaunch && retryEnable_) {
4122 4 : HCCL_RUN_INFO(
4123 : "[OpRetry][AICPU]hccl aicpu stop wait finish, for recv stop launch cmd, identify[%s]", identifier_.c_str());
4124 4 : return HCCL_E_SUSPENDING;
4125 0 : } else if ((cmd == KfcCommand::NsStopLaunch) && (endStopLaunch == false)) {
4126 0 : needsResponseStopLaunch_ = true;
4127 0 : endStopLaunch = true;
4128 0 : HCCL_RUN_INFO("hccl aicpu stop wait finish, for recv stop launch cmd");
4129 0 : return HCCL_E_SUSPENDING;
4130 0 : } else if (cmd == KfcCommand::kDestroyComm) {
4131 0 : HCCL_ERROR("hccl aicpu stop wait finish, for recv destroy comm cmd");
4132 0 : return HCCL_E_INTERNAL;
4133 0 : } else if (cmd == KfcCommand::kExit) {
4134 0 : HCCL_ERROR("hccl aicpu stop wait finish, for recv exit cmd, identify[%s]", identifier_.c_str());
4135 0 : return HCCL_E_INTERNAL;
4136 : }
4137 0 : return HCCL_SUCCESS;
4138 : }
4139 :
4140 4 : HcclResult HcclCommAicpu::UpdateSuspendStatus(
4141 : const OpParam& param, HcclOpExecFSM& fsmState, KfcError& errorCode, uint32_t retryCnt)
4142 : {
4143 4 : if (needsResponseStopLaunch_ == true) {
4144 0 : HCCL_RUN_INFO("[NsRecovery][AICPU]hccl aicpu force stop in launch loop");
4145 0 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_STOP_LAUNCH;
4146 4 : } else if (retryEnable_) {
4147 4 : HCCL_RUN_INFO(
4148 : "[OpRetry][AICPU]hccl aicpu force stop for stop cmd or recoverable task exception, identify[%s]",
4149 : identifier_.c_str());
4150 4 : errorCode = IsTaskExceptionForHccs() ? KfcError::kSdma : errorCode;
4151 :
4152 4 : if ((param.opType == HcclCMDType::HCCL_CMD_BATCH_SEND_RECV)) {
4153 0 : HcclResult ret = GetBSRRetryOpId(param, bsrTargetOpId_);
4154 0 : HCCL_RETRY_CHK_RET_AND_TRANS_FSM(
4155 : ret, HCCL_ERROR("get batchsendrecv target op failed, ret:%u", ret), KfcError::kExec,
4156 : HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR);
4157 0 : uint32_t bsrRetryCnt = (bsrRetryOp_ == HCCL_SEND) ? bsrSendRetryCnt_ : bsrRecvRetryCnt_;
4158 0 : bsrTargetOpId_.isBsrTaskStart = true;
4159 0 : ret = aicpuHdc_.SetOpExecStatus(
4160 0 : kfcStatusTransferD2H_, bsrTargetOpId_, KfcStatus::kStoplaunch, errorCode, bsrRetryCnt);
4161 0 : HCCL_RETRY_CHK_RET_AND_TRANS_FSM(
4162 : ret, HCCL_ERROR("SetOpExecStatus failed, ret:%u", ret), KfcError::kExec,
4163 : HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR);
4164 : } else {
4165 4 : CHK_RET(UpdateOpExecStatus(fsmState, KfcStatus::kStoplaunch, errorCode, retryCnt));
4166 : }
4167 4 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_FSM_STOPPING;
4168 : } else {
4169 0 : HCCL_RUN_INFO(
4170 : "[HcclCommAicpu][UpdateSuspendStatus] aicpu force stop in launch loop; needsResponseStopLaunch_[%u] "
4171 : "retryEnable_[%u]",
4172 : needsResponseStopLaunch_, retryEnable_);
4173 0 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_STOP_LAUNCH;
4174 : }
4175 4 : return HCCL_SUCCESS;
4176 : }
4177 :
4178 0 : HcclResult HcclCommAicpu::TasktypeTransferD2H(const uint8_t sqeType, TaskType& taskType)
4179 : {
4180 0 : switch (sqeType) {
4181 0 : case RT_STARS_SQE_TYPE_PLACE_HOLDER:
4182 : case RT_STARS_SQE_TYPE_NOTIFY_WAIT:
4183 0 : taskType = TaskType::TASK_NOTIFY_WAIT;
4184 0 : break;
4185 0 : case RT_STARS_SQE_TYPE_SDMA:
4186 0 : taskType = TaskType::TASK_SDMA;
4187 0 : break;
4188 0 : case RT_STARS_SQE_TYPE_NOTIFY_RECORD:
4189 0 : taskType = TaskType::TASK_NOTIFY_RECORD;
4190 0 : break;
4191 0 : case RT_STARS_SQE_TYPE_WRITE_VALUE:
4192 0 : taskType = TaskType::TASK_NOTIFY_WAIT;
4193 0 : break;
4194 0 : default:
4195 0 : HCCL_ERROR("TasktypeTransferD2H sqeType[%d] error.", sqeType);
4196 0 : return HCCL_E_PARA;
4197 : }
4198 0 : return HCCL_SUCCESS;
4199 : }
4200 :
4201 0 : HcclResult HcclCommAicpu::GenTaskExceptionInfo(u8 sqeType, hccl::Stream& stream, u32 head)
4202 : {
4203 0 : HcclSqeContext* sqeContext = stream.GetSqeContextPtr();
4204 0 : SqeRingBuffer* sqeContextBuffer = &(sqeContext->buffer);
4205 0 : CHK_PTR_NULL(sqeContextBuffer);
4206 :
4207 0 : const AicpuOpInfo* opInfo = aicpuShareData_.GetAicpuOpInfo(sqeContextBuffer->rtsDfxInfo[head].opRingBufferIdx);
4208 0 : std::string opTag = opInfo == nullptr ? "unKnown" : opInfo->tagBuff;
4209 :
4210 : // 获取需要上报的关键信息
4211 0 : ErrorMessageReport emrInfo{};
4212 0 : SqeInfo sqeInfo;
4213 0 : SqeContextUtils::QuerySqeInfo(
4214 0 : sqeContextBuffer->rtsMirrorBuffer + head * HCCL_SQE_SIZE, sqeContextBuffer->rtsqSqeType[head],
4215 : sqeContextBuffer->addInfo[head], &sqeInfo);
4216 0 : emrInfo.remoteUserRank = sqeContextBuffer->rtsDfxInfo[head].remoteRank;
4217 0 : emrInfo.streamId = stream.id();
4218 0 : emrInfo.taskId = sqeInfo.taskId;
4219 0 : emrInfo.notifyId = sqeInfo.notifyId;
4220 0 : emrInfo.rankId = localUserRank_;
4221 0 : emrInfo.rankSize = topoInfo_.userRankSize;
4222 0 : emrInfo.algType = algType_;
4223 0 : emrInfo.opIndex = opInfo == nullptr ? 0 : opInfo->opIndex;
4224 0 : emrInfo.count = opInfo == nullptr ? 0 : opInfo->count;
4225 0 : emrInfo.dataType = opInfo == nullptr ? 0 : opInfo->dataType;
4226 0 : emrInfo.dstAddr = opInfo == nullptr ? 0 : opInfo->dstAddr;
4227 0 : emrInfo.srcAddr = opInfo == nullptr ? 0 : opInfo->srcAddr;
4228 0 : emrInfo.reduceType = opInfo == nullptr ? 255 : opInfo->reduceType; // 255 为 HcclReduceOp::HCCL_REDUCE_RESERVED
4229 0 : CHK_RET(TasktypeTransferD2H(sqeType, emrInfo.taskType));
4230 :
4231 0 : CHK_SAFETY_FUNC_RET(memcpy_s(emrInfo.tag, sizeof(emrInfo.tag), opTag.c_str(), opTag.size()));
4232 0 : CHK_SAFETY_FUNC_RET(memcpy_s(emrInfo.group, sizeof(emrInfo.group), identifier_.c_str(), identifier_.size()));
4233 0 : CHK_RET(aicpuHdc_.SetErrorMessage(kfcStatusTransferD2H_, emrInfo));
4234 0 : return HCCL_SUCCESS;
4235 0 : }
4236 :
4237 0 : HcclResult HcclCommAicpu::PrintTaskExceptionByTaskId(u8 sqeType, u16 taskId, hccl::Stream& stream, u32 tail)
4238 : {
4239 0 : HcclSqeContext* sqeContext = stream.GetSqeContextPtr();
4240 0 : CHK_PTR_NULL(sqeContext);
4241 0 : HCCL_ERROR("[HcclCommAicpu][PrintTaskExceptionByTaskId]streamId:%d tail:%u cqeType:%u", stream.id(), tail, sqeType);
4242 0 : SqeRingBuffer* sqeContextBuffer = &(sqeContext->buffer);
4243 0 : CHK_PTR_NULL(sqeContextBuffer);
4244 0 : uint8_t* sqeMirrorBufferAddr = sqeContextBuffer->rtsMirrorBuffer + (tail - 1) * HCCL_SQE_SIZE;
4245 0 : rtStarsSqeHeader_t* const sqeHeader = (rtStarsSqeHeader_t* const)sqeMirrorBufferAddr;
4246 :
4247 0 : s32 taskNum = sqeHeader->taskId - taskId;
4248 0 : HCCL_DEBUG("[HcclCommAicpu]tail sqe taskId[%u] cqe taskId[%u] cqe type[%u]", sqeHeader->taskId, taskId, sqeType);
4249 0 : s32 sqeIdx = tail - taskNum - 1;
4250 0 : u32 sqHead = (sqeIdx + HCCL_SQE_MAX_CNT) % HCCL_SQE_MAX_CNT;
4251 0 : uint8_t type = 0;
4252 0 : uint16_t taskIdTmp = 0;
4253 0 : uint32_t remoteRank = 0;
4254 0 : HCCL_ERROR(
4255 : "[TaskException][AICPU]base information is streamId:%d, sqid:%d, head:%u, tail:%u, %s", stream.id(),
4256 : stream.sqId(), sqHead, tail,
4257 : GetTaskExceptionTaskInfo(sqHead, sqeContextBuffer, type, taskIdTmp, remoteRank).c_str());
4258 0 : PrintTaskExceptionTaskQue(sqHead, sqeContextBuffer);
4259 0 : return HCCL_SUCCESS;
4260 : }
4261 :
4262 0 : std::string HcclCommAicpu::GetTaskExceptionOpInfo(u32 idx, SqeRingBuffer* sqeContextBuffer)
4263 : {
4264 0 : const AicpuOpInfo* opInfo = aicpuShareData_.GetAicpuOpInfo(sqeContextBuffer->rtsDfxInfo[idx].opRingBufferIdx);
4265 0 : CHK_PRT_RET(opInfo == nullptr, HCCL_ERROR("%s fail, opInfo is nullptr", __func__), "unKnown");
4266 :
4267 0 : std::stringstream ss;
4268 0 : ss << "tag:" << opInfo->tagBuff << ", ";
4269 0 : ss << "group:" << identifier_ << ", ";
4270 0 : ss << "isCustom:" << opInfo->isCustom << ", ";
4271 0 : ss << "opLaunchIdx:" << opInfo->opIndex << ", ";
4272 0 : ss << "opExecIdx:" << opInfo->opExecIndex << ", ";
4273 0 : ss << "count:" << opInfo->count << ", ";
4274 0 : ss << "dataType:" << static_cast<u16>(opInfo->dataType) << ", ";
4275 0 : ss << "opType:" << static_cast<u16>(opInfo->opType) << ", ";
4276 0 : ss << "rootId:" << opInfo->rootId << ", ";
4277 0 : ss << "dstAddr:0x" << std::hex << opInfo->dstAddr << ", ";
4278 0 : ss << "srcAddr:0x" << std::hex << opInfo->srcAddr << ".";
4279 0 : return ss.str();
4280 0 : }
4281 :
4282 0 : std::string HcclCommAicpu::GetTaskExceptionTaskInfo(
4283 : u32 sqHead, SqeRingBuffer* sqeContextBuffer, uint8_t& type, uint16_t& taskId, uint32_t& remoteRank)
4284 : {
4285 0 : SqeInfo sqeInfo;
4286 0 : SqeContextUtils::QuerySqeInfo(
4287 0 : sqeContextBuffer->rtsMirrorBuffer + sqHead * HCCL_SQE_SIZE, sqeContextBuffer->rtsqSqeType[sqHead],
4288 : sqeContextBuffer->addInfo[sqHead], &sqeInfo);
4289 0 : type = sqeInfo.type;
4290 0 : taskId = sqeInfo.taskId;
4291 0 : remoteRank = sqeContextBuffer->rtsDfxInfo[sqHead].remoteRank;
4292 0 : std::stringstream ss;
4293 0 : ss << "type:" << SqeContextUtils::RtsqTaskTypeToStr(sqeInfo.type) << ", ";
4294 0 : ss << "localRank:" << localUserRank_ << ", ";
4295 0 : ss << "remoteRank:" << remoteRank << ", ";
4296 0 : ss << "taskId:" << sqeInfo.taskId << ", ";
4297 0 : ss << "notifyId:" << sqeInfo.notifyId << ", ";
4298 0 : ss << "length:" << sqeInfo.length << ", ";
4299 0 : ss << "addr1High:0x" << std::hex << sqeInfo.addr1High << ", ";
4300 0 : ss << "addr1Low:0x" << std::hex << sqeInfo.addr1Low << ", ";
4301 0 : ss << "addr2High:0x" << std::hex << sqeInfo.addr2High << ", ";
4302 0 : ss << "addr2Low:0x" << std::hex << sqeInfo.addr2Low << ".";
4303 0 : return ss.str();
4304 0 : }
4305 :
4306 : // 获取指定索引的算子信息
4307 0 : const AicpuOpInfo* HcclCommAicpu::GetOpInfoFromSqIdx(u32 sqIdx, SqeRingBuffer* sqeContextBuffer)
4308 : {
4309 0 : return aicpuShareData_.GetAicpuOpInfo(sqeContextBuffer->rtsDfxInfo[sqIdx].opRingBufferIdx);
4310 : }
4311 :
4312 : // 打印算子数据信息
4313 0 : void HcclCommAicpu::PrintOpDataInfo(u32 sqIdx, SqeRingBuffer* sqeContextBuffer, bool isMonitor)
4314 : {
4315 0 : std::string opInfo = GetTaskExceptionOpInfo(sqIdx, sqeContextBuffer);
4316 0 : if (isMonitor) {
4317 0 : HCCL_RUN_INFO("[StreamTaskMonitor]opData information is %s", opInfo.c_str());
4318 : } else {
4319 0 : HCCL_ERROR("[TaskException]opData information is %s", opInfo.c_str());
4320 : }
4321 0 : }
4322 :
4323 : // 打印task序列行
4324 0 : void HcclCommAicpu::PrintTaskLine(bool isMonitor, u32 lineNum, u32 totalPrinted, const std::string& taskLine) const
4325 : {
4326 0 : if (isMonitor) {
4327 0 : HCCL_RUN_INFO("[StreamTaskMonitor]task sequence[%u/%u] is %s", lineNum, totalPrinted, taskLine.c_str());
4328 : } else {
4329 0 : HCCL_ERROR("[TaskException]task sequence[%u/%u] is %s", lineNum, totalPrinted, taskLine.c_str());
4330 : }
4331 0 : }
4332 :
4333 : // 更新算子上下文
4334 0 : void HcclCommAicpu::UpdateOpContext(
4335 : u32& opIndex, std::string& opTag, u32& lineCount, std::vector<std::string>& currentOpTasks, u32 newOpIndex,
4336 : const std::string& newOpTag) const
4337 : {
4338 0 : opIndex = newOpIndex;
4339 0 : opTag = newOpTag;
4340 0 : lineCount = 0;
4341 0 : currentOpTasks.clear();
4342 0 : currentOpTasks.push_back("OP(" + std::to_string(opIndex) + ")");
4343 0 : }
4344 :
4345 : // 准备下一行数据
4346 0 : void HcclCommAicpu::PrepareNextLine(u32 opIndex, u32& lineCount, std::vector<std::string>& currentOpTasks) const
4347 : {
4348 0 : lineCount++;
4349 0 : currentOpTasks.clear();
4350 0 : currentOpTasks.push_back("OP(" + std::to_string(opIndex) + ") continued");
4351 0 : }
4352 :
4353 : // 拼接task列表为字符串
4354 0 : std::string HcclCommAicpu::ConcatTaskLine(const std::vector<std::string>& tasks) const
4355 : {
4356 0 : std::string taskLine;
4357 0 : for (const auto& task : tasks) {
4358 0 : taskLine += task;
4359 : }
4360 0 : return taskLine;
4361 0 : }
4362 :
4363 : // 打印剩余未满行的tasks
4364 0 : void HcclCommAicpu::PrintRemainingTasks(
4365 : bool isMonitor, u32 lineCount, u32 printedCount, const std::vector<std::string>& currentOpTasks) const
4366 : {
4367 : // size > 1 表示除了OP头还有task
4368 0 : if (currentOpTasks.size() > 1) {
4369 0 : std::string taskLine = ConcatTaskLine(currentOpTasks);
4370 0 : PrintTaskLine(isMonitor, lineCount + 1, printedCount, taskLine);
4371 0 : }
4372 0 : }
4373 :
4374 0 : void HcclCommAicpu::PrintTaskExceptionTaskQue(u32 sqIdx, SqeRingBuffer* sqeContextBuffer, bool isMonitor)
4375 : {
4376 0 : const u32 sqeNum = 200;
4377 0 : const u32 maxTasksPerLine = 50;
4378 :
4379 : // 获取初始算子信息
4380 0 : const AicpuOpInfo* lastOpInfo = GetOpInfoFromSqIdx(sqIdx, sqeContextBuffer);
4381 0 : CHK_PRT_RET(lastOpInfo == nullptr, HCCL_ERROR("%s fail, opInfo is nullptr", __func__), );
4382 :
4383 : // 初始化状态
4384 0 : u32 opIndex = lastOpInfo->opIndex;
4385 0 : std::string opTag = lastOpInfo->tagBuff;
4386 0 : u32 lineCount = 0;
4387 0 : u32 printedCount = 0;
4388 0 : u32 extraSlotPerLine = 2;
4389 0 : std::vector<std::string> currentOpTasks;
4390 0 : currentOpTasks.push_back("OP(" + std::to_string(opIndex) + ")");
4391 :
4392 : // 主循环：遍历并打印
4393 0 : for (u32 i = 0; i < sqeNum; i++) {
4394 : // 计算新索引并获取算子信息
4395 0 : u32 newSqIdx = (sqIdx - i + HCCL_SQE_MAX_CNT) % HCCL_SQE_MAX_CNT;
4396 0 : const AicpuOpInfo* newOpInfo = GetOpInfoFromSqIdx(newSqIdx, sqeContextBuffer);
4397 0 : CHK_PRT_RET(newOpInfo == nullptr, HCCL_ERROR("%s fail, opInfo is nullptr", __func__), );
4398 :
4399 0 : u32 sizeAfterAdd = currentOpTasks.size() + 1;
4400 0 : if (sizeAfterAdd >= (maxTasksPerLine + extraSlotPerLine)
4401 0 : || (i > 0 && (newOpInfo->opIndex != opIndex || newOpInfo->tagBuff != opTag))) {
4402 : // 打印算子信息（仅第一次）
4403 0 : if (lineCount == 0) {
4404 0 : PrintOpDataInfo(sqIdx, sqeContextBuffer, isMonitor);
4405 : }
4406 :
4407 : // 拼接并打印task行
4408 0 : std::string taskLine = ConcatTaskLine(currentOpTasks);
4409 0 : PrintTaskLine(isMonitor, lineCount + 1, printedCount, taskLine);
4410 :
4411 : // 更新状态
4412 0 : if (newOpInfo->opIndex != opIndex || newOpInfo->tagBuff != opTag) {
4413 0 : UpdateOpContext(opIndex, opTag, lineCount, currentOpTasks, newOpInfo->opIndex, newOpInfo->tagBuff);
4414 : } else {
4415 0 : PrepareNextLine(opIndex, lineCount, currentOpTasks);
4416 : }
4417 0 : }
4418 : // 添加task到序列
4419 0 : currentOpTasks.push_back("," + GetTaskBriefsInfo(newSqIdx, sqeContextBuffer));
4420 0 : printedCount++;
4421 : }
4422 : // 打印剩余tasks
4423 0 : PrintRemainingTasks(isMonitor, lineCount, printedCount, currentOpTasks);
4424 0 : }
4425 :
4426 0 : std::string HcclCommAicpu::GetTaskBriefsInfo(u32 idx, SqeRingBuffer* sqeContextBuffer)
4427 : {
4428 0 : uint8_t* sqeMirrorBufferAddr = sqeContextBuffer->rtsMirrorBuffer + idx * HCCL_SQE_SIZE;
4429 0 : rtStarsSqeHeader_t* const sqeHeader = (rtStarsSqeHeader_t* const)sqeMirrorBufferAddr;
4430 0 : uint8_t sqeType = sqeHeader->type;
4431 :
4432 0 : SqeInfo sqeInfo;
4433 0 : SqeContextUtils::QuerySqeInfo(
4434 0 : sqeContextBuffer->rtsMirrorBuffer + idx * HCCL_SQE_SIZE, sqeContextBuffer->rtsqSqeType[idx],
4435 : sqeContextBuffer->addInfo[idx], &sqeInfo);
4436 0 : uint8_t subType = sqeInfo.subType;
4437 :
4438 0 : std::stringstream ss;
4439 0 : std::string taskName = "UN";
4440 0 : switch (sqeType) {
4441 0 : case RT_STARS_SQE_TYPE_NOTIFY_RECORD:
4442 0 : taskName = "NR"; // Notify Record
4443 0 : break;
4444 0 : case RT_STARS_SQE_TYPE_WRITE_VALUE:
4445 0 : if (subType == RT_STARS_WRITE_VALUE_SUB_TYPE_NOTIFY_RECORD_IPC_NO_PCIE) {
4446 0 : taskName = "NR";
4447 0 : } else if (subType == RT_STARS_WRITE_VALUE_SUB_TYPE_EVENT_RESET) {
4448 0 : taskName = "NW"; // Notify Wait
4449 0 : } else if (subType == RT_STARS_WRITE_VALUE_SUB_TYPE_RDMA_DB_SEND) {
4450 0 : taskName = "RS"; // Rdma Send
4451 : }
4452 0 : break;
4453 0 : case RT_STARS_SQE_TYPE_NOTIFY_WAIT:
4454 0 : taskName = "NW";
4455 0 : break;
4456 0 : case RT_STARS_SQE_TYPE_EVENT_WAIT:
4457 0 : taskName = "NW";
4458 0 : break;
4459 0 : case RT_STARS_SQE_TYPE_SDMA:
4460 0 : taskName = "SD"; // SDMA
4461 0 : break;
4462 0 : case RT_STARS_SQE_TYPE_COND:
4463 0 : taskName = "CO";
4464 0 : break;
4465 0 : case RT_STARS_SQE_TYPE_PLACE_HOLDER:
4466 0 : taskName = "PH";
4467 0 : break;
4468 0 : default:
4469 0 : taskName = std::to_string(sqeType);
4470 0 : break;
4471 : }
4472 :
4473 0 : ss << taskName << "(";
4474 0 : if (sqeContextBuffer->rtsDfxInfo[idx].remoteRank != INVALID_VALUE_RANKID) {
4475 0 : ss << sqeContextBuffer->rtsDfxInfo[idx].remoteRank;
4476 : } else {
4477 0 : ss << "/";
4478 : }
4479 0 : ss << ",";
4480 0 : if (sqeContextBuffer->rtsDfxInfo[idx].notifyId != INVALID_VALUE_RANKID) {
4481 0 : ss << sqeContextBuffer->rtsDfxInfo[idx].notifyId;
4482 : } else {
4483 0 : ss << "/";
4484 : }
4485 0 : ss << ")";
4486 0 : return ss.str();
4487 0 : }
4488 :
4489 1 : void HcclCommAicpu::RecordReportStatus(dfx::ReportStatus status)
4490 : {
4491 1 : std::unique_lock<std::mutex> lock(reportQueueMutex_);
4492 1 : while (reportStatusQueue_.size() >= MAX_REPORT_STATUS) {
4493 0 : HCCL_WARNING(
4494 : "[HcclCommAicpu][RecordReportStatus] retry status queue reach the limit[%u], "
4495 : "the front status[%u] is dropped.",
4496 : MAX_REPORT_STATUS, reportStatusQueue_.front());
4497 0 : reportStatusQueue_.pop();
4498 : }
4499 1 : reportStatusQueue_.push(status);
4500 1 : HCCL_INFO("[HcclCommAicpu][RecordReportStatus]push[%u], retry queue size()[%u]", status, reportStatusQueue_.size());
4501 1 : }
4502 :
4503 0 : void HcclCommAicpu::GetReportStatusQueue(std::queue<dfx::ReportStatus>& reportStatusQue)
4504 : {
4505 0 : std::unique_lock<std::mutex> lock(reportQueueMutex_);
4506 0 : std::swap(reportStatusQueue_, reportStatusQue);
4507 0 : }
4508 :
4509 0 : void HcclCommAicpu::SetStreamCqeExceptionStatus(const Stream& stream, CqeExceptionStatus cqeStatus)
4510 : {
4511 0 : HCCL_RUN_INFO("SetStreamCqeExceptionStatus: stream sq id %u, cqe exception %u", stream.sqId(), cqeStatus);
4512 0 : auto iter = streamCqeExceptionStatus_.find(stream.sqId());
4513 0 : if (iter != streamCqeExceptionStatus_.end()) {
4514 0 : iter->second = cqeStatus;
4515 : } else {
4516 0 : streamCqeExceptionStatus_.insert({stream.sqId(), cqeStatus});
4517 : }
4518 0 : return;
4519 : }
4520 :
4521 0 : CqeExceptionStatus HcclCommAicpu::GetStreamCqeExceptionStatus(const Stream& stream)
4522 : {
4523 0 : auto iter = streamCqeExceptionStatus_.find(stream.sqId());
4524 0 : if (iter != streamCqeExceptionStatus_.end()) {
4525 0 : return iter->second;
4526 : } else {
4527 0 : return CqeExceptionStatus::kNone;
4528 : }
4529 : }
4530 :
4531 0 : void HcclCommAicpu::ResetStreamCqeExceptionStatus(const Stream& stream)
4532 : {
4533 0 : auto iter = streamCqeExceptionStatus_.find(stream.sqId());
4534 0 : if (iter != streamCqeExceptionStatus_.end()) {
4535 0 : iter->second = CqeExceptionStatus::kNone;
4536 : }
4537 0 : HCCL_INFO("ResetStreamCqeExceptionStatus: stream sq id %u", stream.sqId());
4538 0 : return;
4539 : }
4540 :
4541 0 : void HcclCommAicpu::SetBSRSendOpExecException()
4542 : {
4543 0 : HCCL_INFO("set send stream exec exception");
4544 0 : bsrSendOpExecException_ = true;
4545 0 : return;
4546 : }
4547 :
4548 0 : void HcclCommAicpu::SetBSRRecvOpExecException()
4549 : {
4550 0 : HCCL_INFO("set recv stream exec exception");
4551 0 : bsrRecvOpExecException_ = true;
4552 0 : return;
4553 : }
4554 :
4555 0 : bool HcclCommAicpu::GetBSRSendOpExecException()
4556 : {
4557 0 : bool ret = (GetStreamCqeExceptionStatus(bsrSendStream_) == CqeExceptionStatus::kSdmaErr) || bsrSendOpExecException_;
4558 0 : HCCL_INFO(
4559 : "GetBSRSendOpExecException: stream %u cqe status %u, send exec status %u", bsrSendStream_.sqId(),
4560 : GetStreamCqeExceptionStatus(bsrSendStream_), bsrSendOpExecException_);
4561 0 : return ret;
4562 : }
4563 :
4564 0 : bool HcclCommAicpu::GetBSRRecvOpExecException()
4565 : {
4566 0 : HCCL_INFO(
4567 : "GetBSRRecvOpExecException: stream %u cqe status %u, recv exec status %u", bsrRecvStream_.sqId(),
4568 : GetStreamCqeExceptionStatus(bsrRecvStream_), bsrRecvOpExecException_);
4569 0 : return (GetStreamCqeExceptionStatus(bsrRecvStream_) == CqeExceptionStatus::kSdmaErr) || bsrRecvOpExecException_;
4570 : }
4571 :
4572 0 : HcclResult HcclCommAicpu::CleanStream(Stream& stream)
4573 : {
4574 0 : CHK_RET(stream.ClearLocalBuff());
4575 0 : CHK_RET(UpdateSqStatus(stream));
4576 0 : ResetStreamCqeExceptionStatus(stream);
4577 0 : HCCL_INFO("CleanStream %u success.", stream.sqId());
4578 0 : return HCCL_SUCCESS;
4579 : }
4580 :
4581 0 : HcclResult HcclCommAicpu::ClearStreamCqeException(Stream& stream)
4582 : {
4583 0 : HandleCqeException(stream, true);
4584 0 : return HCCL_SUCCESS;
4585 : }
4586 :
4587 0 : HcclResult HcclCommAicpu::ResetBSRSendOpExecException()
4588 : {
4589 0 : bsrSendOpExecException_ = false;
4590 0 : CHK_RET(CleanStream(bsrSendStream_));
4591 0 : CHK_RET(ClearStreamCqeException(bsrSendStream_));
4592 0 : HCCL_INFO("ResetBSRSendOpExecException success.");
4593 0 : return HCCL_SUCCESS;
4594 : }
4595 :
4596 0 : HcclResult HcclCommAicpu::ResetBSRRecvOpExecException()
4597 : {
4598 0 : bsrRecvOpExecException_ = false;
4599 0 : CHK_RET(CleanStream(bsrRecvStream_));
4600 0 : CHK_RET(ClearStreamCqeException(bsrRecvStream_));
4601 0 : HCCL_INFO("ResetBSRRecvOpExecException success.");
4602 0 : return HCCL_SUCCESS;
4603 : }
4604 :
4605 0 : HcclResult HcclCommAicpu::ResetBSRException()
4606 : {
4607 0 : if (bsrRetryOp_ == HCCL_SEND) {
4608 0 : CHK_RET(ResetBSRSendOpExecException());
4609 0 : HCCL_INFO("reset batchsendrecv exception success, tag:%s, index:%u", bsrSendOpId_.tag, bsrSendOpId_.index);
4610 0 : } else if (bsrRetryOp_ == HCCL_RECV) {
4611 0 : CHK_RET(ResetBSRRecvOpExecException());
4612 0 : HCCL_INFO("reset batchsendrecv exception success, tag:%s, index:%u", bsrRecvOpId_.tag, bsrRecvOpId_.index);
4613 : } else {
4614 0 : HCCL_INFO("reset batchsendrecv exception success, tag:%s", bsrTargetOpId_.tag);
4615 : }
4616 0 : return HCCL_SUCCESS;
4617 : }
4618 :
4619 0 : void HcclCommAicpu::UpdateBSRRetryCnt()
4620 : {
4621 0 : if (bsrRetryOp_ == HCCL_SEND) {
4622 0 : bsrSendRetryCnt_++;
4623 : } else {
4624 0 : bsrRecvRetryCnt_++;
4625 : }
4626 0 : HCCL_INFO("UpdateBSRRetryCnt, SendCnt[%u], RecvCnt[%u]", bsrSendRetryCnt_, bsrRecvRetryCnt_);
4627 0 : return;
4628 : }
4629 :
4630 0 : void HcclCommAicpu::ResetBSRRetryCnt()
4631 : {
4632 0 : bsrSendRetryCnt_ = 0;
4633 0 : bsrRecvRetryCnt_ = 0;
4634 0 : return;
4635 : }
4636 :
4637 0 : void HcclCommAicpu::InitSendRecvOpId(const OpParam& param, HcclOpIdentifier& opId)
4638 : {
4639 : // send算子入参中只有dst对端rank号，而recv算子入参中只有src源端rank号
4640 0 : if (param.opType == HcclCMDType::HCCL_CMD_SEND) {
4641 0 : opId.detRank = param.dstRank;
4642 0 : opId.srcRank = topoInfo_.userRank;
4643 : } else {
4644 0 : opId.srcRank = param.srcRank;
4645 0 : opId.detRank = topoInfo_.userRank;
4646 : }
4647 0 : opId.isSendRecv = true;
4648 0 : HCCL_DEBUG(
4649 : "[HcclCommAicpu][InitSendRecvOpId]src=[%u] dst=[%u] isSendRecv=[%u]", opId.srcRank, opId.detRank,
4650 : opId.isSendRecv);
4651 0 : return;
4652 : }
4653 :
4654 0 : u32 HcclCommAicpu::HcclUpdateBatchSendRecvOpIndex(std::map<u32, u32>& bsrIndexMap, u32 peerRank)
4655 : {
4656 0 : u32 ret = 0;
4657 0 : auto opIndexMapIter = bsrIndexMap.find(peerRank);
4658 0 : if (opIndexMapIter != bsrIndexMap.end()) {
4659 0 : (opIndexMapIter->second)++;
4660 0 : ret = opIndexMapIter->second;
4661 : } else {
4662 0 : bsrIndexMap.insert({peerRank, 1});
4663 0 : ret = 1;
4664 : }
4665 0 : return ret;
4666 : }
4667 :
4668 0 : u32 HcclCommAicpu::HcclUpdateBatchSendRecvOpIndex(HcclSendRecvType opType, u32 srcRank, u32 dstRank)
4669 : {
4670 0 : u32 peerRank = (opType == HcclSendRecvType::HCCL_SEND) ? dstRank : srcRank;
4671 0 : auto& opIndexMap = (opType == HcclSendRecvType::HCCL_SEND) ? bsrSendIndexMap_ : bsrRecvIndexMap_;
4672 :
4673 0 : return HcclUpdateBatchSendRecvOpIndex(opIndexMap, peerRank);
4674 : }
4675 : HcclResult
4676 0 : HcclCommAicpu::GetBsrTransportQpn(const HcclSendRecvItem* sendrecvPair, AlgResourceResponse& algResource, u32& qpn)
4677 : {
4678 0 : CHK_PTR_NULL(sendrecvPair);
4679 0 : LINK targetLink;
4680 0 : u32 commIndex = 0;
4681 0 : u32 remoteRank = sendrecvPair->remoteRank;
4682 0 : u32 localRank = topoInfo_.userRank;
4683 0 : HcclSendRecvType sendRecvType = sendrecvPair->sendRecvType;
4684 0 : HCCL_DEBUG(
4685 : "[GetBsrTransportQpn] bsrOptype =[%d], localRank=[%u] remoteRank=[%u]", sendRecvType, localRank, remoteRank);
4686 :
4687 0 : if ((sendRecvType == HcclSendRecvType::HCCL_SEND && remoteRank < localRank)
4688 0 : || (sendRecvType == HcclSendRecvType::HCCL_RECV && remoteRank > localRank)) {
4689 0 : commIndex = COMM_INDEX_0;
4690 : } else {
4691 0 : commIndex = COMM_INDEX_1;
4692 : }
4693 0 : CHK_PRT_RET(
4694 : commIndex >= algResource.opTransportResponse[COMM_COMBINE_ORDER].size(),
4695 : HCCL_ERROR(
4696 : "[GetBsrTransportQpn] batchsendrecv op commIndex[%u] is larger than "
4697 : "opTransportResponse size[%zu]",
4698 : commIndex, algResource.opTransportResponse[COMM_COMBINE_ORDER].size()),
4699 : HCCL_E_PARA);
4700 : SingleSubCommTransport& commCombined
4701 0 : = static_cast<SingleSubCommTransport&>(algResource.opTransportResponse[COMM_COMBINE_ORDER][commIndex]);
4702 :
4703 0 : CHK_PRT_RET(
4704 : sendrecvPair->remoteRank >= commCombined.userRank2subCommRank.size(),
4705 : HCCL_ERROR(
4706 : "[GetBsrTransportQpn]batchsendrecv op remoteUserRank[%u] is larger than "
4707 : "userRank2subCommRank map size[%zu]",
4708 : sendrecvPair->remoteRank, commCombined.userRank2subCommRank.size()),
4709 : HCCL_E_PARA);
4710 :
4711 0 : u32 rank = commCombined.userRank2subCommRank[sendrecvPair->remoteRank];
4712 0 : CHK_PRT_RET(
4713 : rank >= commCombined.links.size(),
4714 : HCCL_ERROR(
4715 : "[GetBsrTransportQpn] batchsendrecv op remoteUserRank[%u], get rank[%u],"
4716 : "the size of combinedComm links is [%zu]",
4717 : sendrecvPair->remoteRank, rank, commCombined.links.size()),
4718 : HCCL_E_PARA);
4719 0 : targetLink = commCombined.links[rank];
4720 :
4721 0 : CHK_SMART_PTR_NULL(targetLink);
4722 0 : if (targetLink->GetLinkType() == LinkType::LINK_ROCE) {
4723 0 : CHK_RET(targetLink->GetTransportId(qpn));
4724 : }
4725 0 : HCCL_DEBUG(
4726 : "[HcclCommAicpu][GetBsrTransportQpn] localrank=[%u] remoteuserRank=[%u] remoteRank=[%u],sendrecvType=[%d] qpn "
4727 : "=[%u], comindex [%u]",
4728 : topoInfo_.userRank, sendrecvPair->remoteRank, remoteRank, sendrecvPair->sendRecvType, qpn, commIndex);
4729 0 : return HCCL_SUCCESS;
4730 0 : }
4731 0 : HcclResult HcclCommAicpu::InitBatchSendRecvOpId(
4732 : const OpParam& param, const HcclSendRecvItem* sendrecvPair, HcclOpIdentifier& opId, u32 streamId,
4733 : AlgResourceResponse& algResource)
4734 : {
4735 0 : CHK_PTR_NULL(sendrecvPair);
4736 0 : if (sendrecvPair->sendRecvType == HcclSendRecvType::HCCL_RECV) {
4737 0 : opId.srcRank = sendrecvPair->remoteRank;
4738 0 : opId.detRank = topoInfo_.userRank;
4739 : } else {
4740 0 : opId.srcRank = topoInfo_.userRank;
4741 0 : opId.detRank = sendrecvPair->remoteRank;
4742 : }
4743 0 : opId.index = HcclUpdateBatchSendRecvOpIndex(sendrecvPair->sendRecvType, opId.srcRank, opId.detRank);
4744 0 : opId.isSendRecv = true;
4745 0 : opId.opType = HcclCMDType::HCCL_CMD_BATCH_SEND_RECV;
4746 0 : std::string sendrecvTag = param.tag + "_BSR_" + std::to_string(opId.srcRank) + "_" + std::to_string(opId.detRank);
4747 0 : CHK_SAFETY_FUNC_RET(memcpy_s(opId.tag, sizeof(opId.tag), sendrecvTag.c_str(), sendrecvTag.size()));
4748 0 : std::string sendrecvNewTag = param.tag + "_device";
4749 0 : CHK_SAFETY_FUNC_RET(memcpy_s(opId.newTag, sizeof(opId.newTag), sendrecvNewTag.c_str(), sendrecvNewTag.size()));
4750 :
4751 0 : u32 qpn = 0;
4752 0 : if ((opId.srcRank != opId.detRank) && (!param.BatchSendRecvDataDes.isDirectRemoteRank[sendrecvPair->remoteRank])) {
4753 0 : CHK_RET(GetBsrTransportQpn(sendrecvPair, algResource, qpn));
4754 : }
4755 :
4756 0 : auto& bsrinfo = opId.bsrInfo[sendrecvPair->sendRecvType];
4757 0 : bsrinfo.detRank = opId.detRank;
4758 0 : bsrinfo.srcRank = opId.srcRank;
4759 0 : bsrinfo.index = opId.index;
4760 0 : bsrinfo.streamId = streamId;
4761 0 : bsrinfo.tpQpn = qpn;
4762 0 : CHK_SAFETY_FUNC_RET(memcpy_s(bsrinfo.bsrTag, sizeof(bsrinfo.bsrTag), sendrecvTag.c_str(), sendrecvTag.size()));
4763 :
4764 0 : HCCL_INFO(
4765 : "[HcclCommAicpu][InitBatchSendRecvOpId] tag=[%s] index=[%u] src=[%u] det=[%u] qpn =[%u]", opId.tag, opId.index,
4766 : opId.srcRank, opId.detRank, qpn);
4767 0 : return HCCL_SUCCESS;
4768 0 : }
4769 :
4770 0 : HcclResult HcclCommAicpu::InitBatchSendRecvOpId(const OpParam& param, AlgResourceResponse& algResource)
4771 : {
4772 0 : CHK_SAFETY_FUNC_RET(
4773 : memset_s(reinterpret_cast<void*>(&bsrSendOpId_), sizeof(bsrSendOpId_), 0, sizeof(bsrSendOpId_)));
4774 0 : CHK_SAFETY_FUNC_RET(
4775 : memset_s(reinterpret_cast<void*>(&bsrRecvOpId_), sizeof(bsrRecvOpId_), 0, sizeof(bsrRecvOpId_)));
4776 :
4777 0 : CHK_PRT_RET(
4778 : (algResource.slaveStreams.size() < BSR_RETRY_STREAM_NUM),
4779 : HCCL_ERROR("in batchsendrecv op, slave stream is not enough."), HCCL_E_INTERNAL);
4780 0 : bsrSendOpId_.streamId = algResource.slaveStreams[BSR_RETRY_SEND_STREAM_INDEX].id();
4781 0 : bsrRecvOpId_.streamId = algResource.slaveStreams[BSR_RETRY_RECV_STREAM_INDEX].id();
4782 :
4783 0 : u32 iter = param.BatchSendRecvDataDes.curIterNum;
4784 0 : std::vector<std::vector<HcclSendRecvItem*>>& pairs = bsrSendRecvPairs_;
4785 0 : CHK_PRT_RET(
4786 : (pairs.size() <= iter),
4787 : HCCL_ERROR("batchsendrecv sendrecv pairs size[%u] less than or equal to curiter[%u]", pairs.size(), iter),
4788 : HCCL_E_INTERNAL);
4789 :
4790 0 : for (auto& pair : pairs[iter]) {
4791 0 : HcclOpIdentifier& opId = (pair->sendRecvType == HcclSendRecvType::HCCL_SEND) ? bsrSendOpId_ : bsrRecvOpId_;
4792 0 : auto streamId
4793 0 : = (pair->sendRecvType == HcclSendRecvType::HCCL_SEND) ? bsrSendOpId_.streamId : bsrRecvOpId_.streamId;
4794 0 : CHK_RET(InitBatchSendRecvOpId(param, pair, opId, streamId, algResource));
4795 : }
4796 :
4797 : // 补全batchsendrecv中sendrecv的bsrInfo, 两边都发生故障的时候要用
4798 0 : bsrSendOpId_.bsrInfo[HCCL_RECV].index = bsrRecvOpId_.index;
4799 0 : bsrSendOpId_.bsrInfo[HCCL_RECV].streamId = bsrRecvOpId_.streamId;
4800 0 : bsrSendOpId_.bsrInfo[HCCL_RECV].srcRank = bsrRecvOpId_.srcRank;
4801 0 : bsrSendOpId_.bsrInfo[HCCL_RECV].detRank = bsrRecvOpId_.detRank;
4802 0 : bsrSendOpId_.bsrInfo[HCCL_RECV].tpQpn = bsrRecvOpId_.bsrInfo[HCCL_RECV].tpQpn;
4803 0 : CHK_SAFETY_FUNC_RET(memcpy_s(
4804 : bsrSendOpId_.bsrInfo[HCCL_RECV].bsrTag, sizeof(bsrSendOpId_.bsrInfo[HCCL_RECV].bsrTag), bsrRecvOpId_.tag,
4805 : sizeof(bsrRecvOpId_.tag)));
4806 :
4807 0 : bsrRecvOpId_.bsrInfo[HCCL_SEND].index = bsrSendOpId_.index;
4808 0 : bsrRecvOpId_.bsrInfo[HCCL_SEND].streamId = bsrSendOpId_.streamId;
4809 0 : bsrRecvOpId_.bsrInfo[HCCL_SEND].srcRank = bsrSendOpId_.srcRank;
4810 0 : bsrRecvOpId_.bsrInfo[HCCL_SEND].detRank = bsrSendOpId_.detRank;
4811 0 : bsrRecvOpId_.bsrInfo[HCCL_SEND].tpQpn = bsrSendOpId_.bsrInfo[HCCL_SEND].tpQpn;
4812 0 : CHK_SAFETY_FUNC_RET(memcpy_s(
4813 : bsrRecvOpId_.bsrInfo[HCCL_SEND].bsrTag, sizeof(bsrRecvOpId_.bsrInfo[HCCL_SEND].bsrTag), bsrSendOpId_.tag,
4814 : sizeof(bsrSendOpId_.tag)));
4815 0 : return HCCL_SUCCESS;
4816 : }
4817 :
4818 0 : HcclResult HcclCommAicpu::QueryBatchSendRecvPairBeginPos()
4819 : {
4820 : // 前面已经生成 batchsendrecv 的 send & recv opid 时已经校验过slave stream num数，此处不再重复校验
4821 0 : CHK_RET(QuerySqStatusByType(devId_, bsrSendStream_.sqId(), DRV_SQCQ_PROP_SQ_TAIL, bsrSendOpBeginSqePos_));
4822 0 : CHK_RET(QuerySqStatusByType(devId_, bsrRecvStream_.sqId(), DRV_SQCQ_PROP_SQ_TAIL, bsrRecvOpBeginSqePos_));
4823 :
4824 0 : HCCL_INFO(
4825 : "QueryBatchSendRecvPairBeginPos send sqePos[%u] recv sqePos[%u]", bsrSendOpBeginSqePos_, bsrRecvOpBeginSqePos_);
4826 0 : return HCCL_SUCCESS;
4827 : }
4828 :
4829 0 : HcclResult HcclCommAicpu::QueryBatchSendRecvPairEndPos()
4830 : {
4831 : // 前面已经生成 batchsendrecv 的 send & recv opid 时已经校验过slave stream num数，此处不再重复校验
4832 0 : CHK_RET(QuerySqStatusByType(devId_, bsrSendStream_.sqId(), DRV_SQCQ_PROP_SQ_TAIL, bsrSendOpEndSqePos_));
4833 0 : CHK_RET(QuerySqStatusByType(devId_, bsrRecvStream_.sqId(), DRV_SQCQ_PROP_SQ_TAIL, bsrRecvOpEndSqePos_));
4834 :
4835 0 : HCCL_INFO("QueryBatchSendRecvPairEndPos send sqePos[%u] recv sqePos[%u]", bsrSendOpEndSqePos_, bsrRecvOpEndSqePos_);
4836 0 : return HCCL_SUCCESS;
4837 : }
4838 :
4839 0 : HcclResult HcclCommAicpu::CommitBSRStoredException(HcclOpExecFSM& fsmState, KfcError& errorCode)
4840 : {
4841 0 : if (GetBSRSendOpExecException()) {
4842 0 : bsrRetryOp_ = HCCL_SEND;
4843 0 : errorCode = KfcError::kSdma;
4844 0 : HCCL_INFO("CommitBSRStoredException: send stream remain retry error.");
4845 0 : } else if (GetBSRRecvOpExecException()) {
4846 0 : bsrRetryOp_ = HCCL_RECV;
4847 0 : errorCode = KfcError::kSdma;
4848 0 : HCCL_INFO("CommitBSRStoredException: recv stream remain retry error.");
4849 : }
4850 :
4851 0 : u32 retryCnt = (bsrRetryOp_ == HCCL_SEND) ? bsrSendRetryCnt_ : bsrRecvRetryCnt_;
4852 0 : if (errorCode != KfcError::kNone) {
4853 0 : bsrTargetOpId_ = (bsrRetryOp_ == HCCL_SEND) ? bsrSendOpId_ : bsrRecvOpId_;
4854 0 : HCCL_RUN_INFO(
4855 : "CommitBSRStoredException: stored op tag %s index %u , report retry error. curSendRetryCnt[%u],"
4856 : "curRecvRetryCnt[%u]",
4857 : bsrTargetOpId_.tag, bsrTargetOpId_.index, bsrSendRetryCnt_, bsrRecvRetryCnt_);
4858 0 : auto ret = aicpuHdc_.SetOpExecStatus(
4859 0 : kfcStatusTransferD2H_, bsrTargetOpId_, KfcStatus::kStoplaunch, errorCode, retryCnt);
4860 0 : HCCL_RETRY_CHK_RET_AND_TRANS_FSM(
4861 : ret, HCCL_ERROR("SetOpExecStatus failed, ret:%u", ret), KfcError::kExec,
4862 : HcclOpExecFSM::HCCL_OP_EXEC_FSM_ERROR);
4863 0 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_FSM_STOPPING;
4864 : } else {
4865 0 : CHK_RET(UpdateOpExecStatus(fsmState, KfcStatus::kRuning, errorCode, retryCnt));
4866 0 : fsmState = HcclOpExecFSM::HCCL_OP_EXEC_FSM_WAIT_END;
4867 : }
4868 0 : return HCCL_SUCCESS;
4869 : }
4870 :
4871 0 : HcclResult HcclCommAicpu::GetBSRRetryOpId(const OpParam& param, HcclOpIdentifier& targetOpId)
4872 : {
4873 0 : KfcCommand cmd = KfcCommand::kNone;
4874 0 : CHK_RET(aicpuHdc_.GetOpExecCtrlCmd(kfcControlTransferH2D_, cmd));
4875 0 : if (cmd == KfcCommand::kStopLaunch) {
4876 0 : HcclOpIdentifier targetOp;
4877 0 : CHK_RET(aicpuHdc_.GetOpExecCtrlTargetOp(kfcControlTransferH2D_, targetOp));
4878 0 : std::string targetOpTag = std::string(reinterpret_cast<char*>(&targetOp.tag[0]));
4879 0 : if (targetOpTag == std::string(reinterpret_cast<char*>(&bsrSendOpId_.tag[0]))) {
4880 0 : bsrRetryOp_ = HCCL_SEND;
4881 0 : } else if (targetOpTag == std::string(reinterpret_cast<char*>(&bsrRecvOpId_.tag[0]))) {
4882 0 : bsrRetryOp_ = HCCL_RECV;
4883 0 : } else if (targetOpTag == param.tag) {
4884 0 : if (targetOp.detRank == bsrSendOpId_.detRank) {
4885 0 : bsrRetryOp_ = HCCL_SEND;
4886 0 : } else if (targetOp.srcRank == bsrRecvOpId_.srcRank) {
4887 0 : bsrRetryOp_ = HCCL_RECV;
4888 : } else {
4889 0 : HCCL_ERROR(
4890 : "hccl aicpu can not retry, got stop launch command, but target op srcRank[%u] and"
4891 : "dstRank[%u] is not match with send (dst:%u) or recv (src:%u) op",
4892 : targetOp.srcRank, targetOp.detRank, bsrSendOpId_.detRank, bsrRecvOpId_.srcRank);
4893 0 : return HCCL_E_INTERNAL;
4894 : }
4895 : } else {
4896 : // tag 不匹配，报错退出
4897 0 : HCCL_ERROR(
4898 : "hccl aicpu can not retry, got stop launch command, but target op tag[%s] is not match with"
4899 : "send (tag:%s) or recv (tag:%s) or batchsendrecv (tag:%s)",
4900 : targetOpTag.c_str(), bsrSendOpId_.tag, bsrRecvOpId_.tag, param.tag.c_str());
4901 0 : return HCCL_E_INTERNAL;
4902 : }
4903 0 : HCCL_RUN_INFO("hccl aicpu got command %u at op[tag: %s, index: %u].", cmd, targetOpTag.c_str(), targetOp.index);
4904 0 : } else {
4905 0 : if (GetBSRSendOpExecException()) {
4906 0 : bsrRetryOp_ = HCCL_SEND;
4907 0 : } else if (GetBSRRecvOpExecException()) {
4908 0 : bsrRetryOp_ = HCCL_RECV;
4909 : } else {
4910 : // 其他场景，报错退出
4911 0 : HCCL_ERROR("hccl aicpu find task exception, but send and recv op has no exception");
4912 0 : return HCCL_E_INTERNAL;
4913 : }
4914 : }
4915 0 : if (bsrRetryOp_ == HCCL_SEND) {
4916 0 : targetOpId = bsrSendOpId_;
4917 : } else {
4918 0 : targetOpId = bsrRecvOpId_;
4919 : }
4920 0 : HCCL_RUN_INFO(
4921 : "GetBSRRetryOpId: bsrRetryOpType %u, targetOpId: tag %s, index %u", bsrRetryOp_, targetOpId.tag,
4922 : targetOpId.index);
4923 0 : return HCCL_SUCCESS;
4924 : }
4925 :
4926 0 : HcclResult HcclCommAicpu::InitExecLoop(OpParam& param, std::unique_ptr<CollExecutorBase>& executor, u32& loopNum)
4927 : {
4928 0 : if ((param.opType == HcclCMDType::HCCL_CMD_BATCH_SEND_RECV) && retryEnable_) {
4929 0 : CHK_RET(executor->CreatePairWiseList(
4930 : param.BatchSendRecvDataDes.sendRecvItemsPtr, param.BatchSendRecvDataDes.itemNum));
4931 0 : CHK_RET(executor->GetPairWiseList(bsrSendRecvPairs_));
4932 0 : CHK_PRT_RET(
4933 : bsrSendRecvPairs_.empty(), HCCL_ERROR("[HcclCommAicpu][InitExecLoop]batchsendrecv pairs is empty"),
4934 : HCCL_E_INTERNAL);
4935 :
4936 0 : for (size_t i = 0; i < bsrSendRecvPairs_.size(); i++) {
4937 0 : CHK_PRT_RET(
4938 : (bsrSendRecvPairs_[i].size() > BSR_RETRY_SENDRECV_PAIR_NUM_MAX) || bsrSendRecvPairs_[i].empty(),
4939 : HCCL_ERROR("batchsendrecv pairs[%u] size[%u] is out of range [1,2]", i, bsrSendRecvPairs_[i].size()),
4940 : HCCL_E_INTERNAL);
4941 :
4942 0 : for (size_t j = 0; j < bsrSendRecvPairs_[i].size(); j++) {
4943 0 : CHK_PTR_NULL(bsrSendRecvPairs_[i][j]);
4944 : }
4945 :
4946 0 : CHK_PRT_RET(
4947 : ((bsrSendRecvPairs_[i].size() == BSR_RETRY_SENDRECV_PAIR_NUM_MAX)
4948 : && (bsrSendRecvPairs_[i][BSR_RETRY_SENDRECV_PAIR_INDEX_0]->sendRecvType
4949 : == bsrSendRecvPairs_[i][BSR_RETRY_SENDRECV_PAIR_INDEX_1]->sendRecvType)),
4950 : HCCL_ERROR(
4951 : "batchsendrecv pairs[%u] sendRecvType[%u] is same", i,
4952 : bsrSendRecvPairs_[i][BSR_RETRY_SENDRECV_PAIR_INDEX_0]->sendRecvType),
4953 : HCCL_E_INTERNAL);
4954 : }
4955 :
4956 0 : param.BatchSendRecvDataDes.curIterNum = 0;
4957 0 : loopNum = bsrSendRecvPairs_.size();
4958 0 : } else {
4959 0 : loopNum = 1;
4960 : }
4961 0 : HCCL_INFO("InitExecLoop: execute loop num %u", loopNum);
4962 0 : return HCCL_SUCCESS;
4963 : }
4964 :
4965 1 : HcclResult HcclCommAicpu::ParseHierarchicalAlgOption(u32* ahcConfInfo)
4966 : {
4967 1 : u32 algOptionSize = ahcConfInfo[TOP_HIERARCHICAL_CONF_lENGTH_INDEX];
4968 1 : if (algOptionSize >= (TOP_HIERARCHICAL_CONF_SIZE - 1)) {
4969 1 : HCCL_ERROR(
4970 : "[HcclCommAicpu][ParseHierarchicalAlgOption] hierarchicalAlgOption size[%lu] exceed maxsize[%lu]",
4971 : algOptionSize, (TOP_HIERARCHICAL_CONF_SIZE - 1));
4972 1 : return HCCL_E_INTERNAL;
4973 : }
4974 :
4975 0 : std::map<AHCConcOpType, TemplateType> hierarchicalAlgOption;
4976 0 : for (u32 i = TOP_HIERARCHICAL_CONF_INFO_INDEX; i < (TOP_HIERARCHICAL_CONF_INFO_INDEX + algOptionSize); i++) {
4977 0 : AHCConcOpType ahcConcOpType;
4978 0 : ahcConcOpType.ahcLevel = static_cast<AHCLevel>(
4979 0 : (ahcConfInfo[i] & TOP_HIERARCHICAL_CONF_LEVEL_LOCATION) >> TOP_HIERARCHICAL_CONF_LEVEL_SHIFT);
4980 0 : ahcConcOpType.concType = static_cast<ConcType>(
4981 0 : (ahcConfInfo[i] & TOP_HIERARCHICAL_CONF_CONC_TYPE_LOCATION) >> TOP_HIERARCHICAL_CONF_CONC_TYPE_SHIFT);
4982 0 : ahcConcOpType.ahcOpType = static_cast<AHCOpType>(
4983 0 : (ahcConfInfo[i] & TOP_HIERARCHICAL_CONF_OP_TYPE_LOCATION) >> TOP_HIERARCHICAL_CONF_OP_TYPE_SHIFT);
4984 0 : TemplateType templateType = static_cast<TemplateType>(
4985 0 : (ahcConfInfo[i] & TOP_HIERARCHICAL_CONF_TEMPLATE_TYPE_LOCATION)
4986 : >> TOP_HIERARCHICAL_CONF_TEMPLATE_TYPE_SHIFT);
4987 :
4988 0 : hierarchicalAlgOption[ahcConcOpType] = templateType;
4989 0 : HCCL_DEBUG(
4990 : "[HcclCommAicpu][ParseHierarchicalAlgOption]: index[%u] ahcLevel[%u] concType[%u] ahcOpType[%u], "
4991 : "templateType[%u]",
4992 : i, ahcConcOpType.ahcLevel, ahcConcOpType.concType, ahcConcOpType.ahcOpType, templateType);
4993 : }
4994 0 : topoMatcher_->SetAHCAlgOption(hierarchicalAlgOption);
4995 0 : return HCCL_SUCCESS;
4996 0 : }
4997 :
4998 0 : void HcclCommAicpu::PollCqeException(
4999 : hccl::Stream& stream, bool isReadClear, rtLogicCqReport_t& cqeException, CqeStatus& cqeStatus)
5000 : {
5001 0 : const HcclComStreamInfo& streamInfo = stream.GetHcclStreamInfo();
5002 : // 以下条件满足其中之一，进行Poll CQE：1、AICPU/Custom读清CQ 2、AICPU进程读取CQ信息
5003 0 : bool isPollCqe = true;
5004 0 : while (isPollCqe) {
5005 0 : LogControl logControl(false, retryEnable_); // 使能重执行场景，修改ERROR->RUN_WARNING，析构时自动恢复
5006 : CqeQueryInput cqeQueryInput;
5007 0 : dfx_tracer::ExecutorTracer::SetCqeQueryInput(GetDevId(), streamInfo, cqeQueryInput);
5008 0 : constexpr u32 reportSize = 256;
5009 : rtLogicCqReport_t streamReport[reportSize];
5010 0 : cqeQueryInput.cqeAddr = reinterpret_cast<uint8_t*>(streamReport); // 用于存放接收到的cq
5011 0 : cqeStatus = CqReportRecv(cqeQueryInput, cqeException);
5012 0 : isPollCqe = (cqeStatus == dfx::CqeStatus::kCqeException && isReadClear); // 读清CQ场景，继续查询
5013 0 : }
5014 0 : }
5015 :
5016 0 : void HcclCommAicpu::ExchangeCqeContext(
5017 : hccl::Stream& stream, rtLogicCqReport_t& cqeException, CqeStatus& cqeStatus, ErrCqeContext& cqeCtx)
5018 : {
5019 0 : HcclResult ret = HCCL_SUCCESS;
5020 : // aicpu进程把读取到的异常cq，写入共享内存
5021 0 : if (cqeStatus == dfx::CqeStatus::kCqeException) {
5022 0 : ret = stream.SetCqeContext(ErrCqeContext(
5023 0 : static_cast<u32>(cqeStatus), cqeException.taskId, cqeException.errorCode, cqeException.sqeType));
5024 : }
5025 :
5026 : // aicpu进程和custom进程，都从共享内存中读取cqe信息
5027 0 : if (ret != HCCL_SUCCESS || stream.GetCqeContext(cqeCtx) != HCCL_SUCCESS) {
5028 : // 写入/读取共享内存失败，标记流状态，避免背景线程继续轮询这条流，导致刷屏
5029 0 : SetStreamCqeExceptionStatus(stream, CqeExceptionStatus::kOther);
5030 0 : HCCL_ERROR(
5031 : "%s fail, set streamCqeExceptionStatus streamId[%u] exception status[%d]", __func__, stream.id(),
5032 : CqeExceptionStatus::kOther);
5033 0 : return;
5034 : }
5035 : }
5036 :
5037 0 : void HcclCommAicpu::HandleCqeException(hccl::Stream& stream, bool isReadClear)
5038 : {
5039 0 : std::unique_lock<std::mutex> lock(queryCqeMutex_);
5040 :
5041 : // poll cqe信息
5042 : rtLogicCqReport_t cqeException;
5043 0 : CqeStatus cqeStatus = CqeStatus::kDefault;
5044 0 : PollCqeException(stream, isReadClear, cqeException, cqeStatus);
5045 :
5046 : // 以下两种情况直接返回，不处理cq信息：1、读清CQ场景只读取不处理 2、本地记录流上已经有异常信息(避免刷屏)
5047 0 : if (isReadClear || GetStreamCqeExceptionStatus(stream) != CqeExceptionStatus::kNone) {
5048 0 : return;
5049 : }
5050 :
5051 : // aicpu和custom同步异常cqe信息，并记录在本地
5052 0 : ErrCqeContext cqeCtx;
5053 0 : ExchangeCqeContext(stream, cqeException, cqeStatus, cqeCtx);
5054 0 : cqeStatus = static_cast<dfx::CqeStatus>(cqeCtx.cqeStatus);
5055 :
5056 : // 处理流上的异常cq
5057 0 : if (cqeStatus != dfx::CqeStatus::kDefault) {
5058 0 : bool isSdmaTypeErr = cqeStatus == dfx::CqeStatus::kCqeException && cqeCtx.sqeType == RT_STARS_SQE_TYPE_SDMA;
5059 0 : bool isCompDataErr = cqeCtx.errorCode == RT_SDMA_COMPDATAERR || cqeCtx.errorCode == RT_SDMA_COMPERR
5060 0 : || cqeCtx.errorCode == RT_SDMA_DATAERR;
5061 0 : bool isSdmaCompDataErr = isSdmaTypeErr && isCompDataErr;
5062 :
5063 0 : ReportErrCqe(stream, cqeCtx);
5064 :
5065 : // 标记发生ErrCqe的流
5066 0 : hccl::CqeExceptionStatus cqeExceptionStatus
5067 0 : = isSdmaCompDataErr ? hccl::CqeExceptionStatus::kSdmaErr : hccl::CqeExceptionStatus::kOther;
5068 0 : SetStreamCqeExceptionStatus(stream, cqeExceptionStatus);
5069 :
5070 : // 通知kfc线程
5071 0 : dfxExtendInfo_.cqeException.sqeType = cqeCtx.sqeType;
5072 0 : dfxExtendInfo_.cqeException.errorCode = cqeCtx.errorCode;
5073 0 : dfxExtendInfo_.cqeStatus = cqeStatus;
5074 0 : dfxExtendInfo_.pollStatus = PollStatus::kStopAsException;
5075 0 : HCCL_INFO(
5076 : "update dfxExtendInfo, group %s, streamId %u, cqeStatus %d, sqetype %u, errorCode %u", identifier_.c_str(),
5077 : stream.id(), cqeStatus, cqeCtx.sqeType, cqeCtx.errorCode);
5078 : }
5079 0 : }
5080 :
5081 0 : void HcclCommAicpu::ReportErrCqe(hccl::Stream& stream, ErrCqeContext& cqeCtx)
5082 : {
5083 0 : dfx::CqeStatus cqeStatus = static_cast<dfx::CqeStatus>(cqeCtx.cqeStatus);
5084 0 : const HcclComStreamInfo& streamInfo = stream.GetHcclStreamInfo();
5085 0 : u32 head = 0;
5086 0 : u32 tail = 0;
5087 0 : QuerySqStatusByType(devId_, streamInfo.sqId, DRV_SQCQ_PROP_SQ_HEAD, head);
5088 0 : QuerySqStatusByType(devId_, streamInfo.sqId, DRV_SQCQ_PROP_SQ_TAIL, tail);
5089 :
5090 0 : bool isSdmaTypeErr = cqeStatus == dfx::CqeStatus::kCqeException && cqeCtx.sqeType == RT_STARS_SQE_TYPE_SDMA;
5091 0 : bool isCompDataErr = cqeCtx.errorCode == RT_SDMA_COMPDATAERR || cqeCtx.errorCode == RT_SDMA_COMPERR
5092 0 : || cqeCtx.errorCode == RT_SDMA_DATAERR;
5093 0 : bool isSdmaCompDataErr = isSdmaTypeErr && isCompDataErr;
5094 :
5095 : // 发生sdma、notify_wait、place_holder、write_value错误，上报error message
5096 0 : bool isComReportErrMesg
5097 : = cqeStatus == dfx::CqeStatus::kCqeException
5098 0 : && (cqeCtx.sqeType == RT_STARS_SQE_TYPE_SDMA || cqeCtx.sqeType == RT_STARS_SQE_TYPE_NOTIFY_WAIT
5099 0 : || cqeCtx.sqeType == RT_STARS_SQE_TYPE_PLACE_HOLDER || cqeCtx.sqeType == RT_STARS_SQE_TYPE_WRITE_VALUE);
5100 :
5101 : // 使能重执行且触发SDMA ERROR的场景，修改ERROR->RUN_WARNING
5102 0 : LogControl retryLog(false, retryEnable_ && isSdmaCompDataErr);
5103 :
5104 : // 只在AICPU进程上报taskException，Custom进程不上报
5105 0 : if (isComReportErrMesg && errMessageReport_) {
5106 : // 记录关键信息，并通过D2H通信通道交给host内存
5107 0 : GenTaskExceptionInfo(cqeCtx.sqeType, stream, head);
5108 : // 通知ts错误信息，触发非SDMA错误、不使能重执行触发SDMA错误、使能重执行触发无法重执行的错误时，进行故障上报
5109 0 : if (!isSdmaTypeErr || (isSdmaTypeErr && (!retryEnable_ || !isCompDataErr))) {
5110 0 : HcclResult ret = SendTaskExceptionByMBox(cqeCtx.errorCode);
5111 0 : HCCL_RUN_INFO(
5112 : "[HcclCommAicpu][SendTaskExceptionByMBox]group[%s]:"
5113 : "Try to send task exception by mailbox, errType[%u], errCode[%u], streamId[%d]",
5114 : identifier_.c_str(), cqeCtx.sqeType, cqeCtx.errorCode, stream.id());
5115 0 : CHK_PRT_CONT(
5116 : ret != HCCL_SUCCESS,
5117 : HCCL_ERROR(
5118 : "[OpRetry][AICPU]group[%s]:Send task exception by mailBox failed, streamId[%d]",
5119 : identifier_.c_str(), stream.id()));
5120 : }
5121 : // 当前阶段， 每个通信域在plog打印一次，error message作为host上报，只打印首次
5122 0 : errMessageReport_ = false;
5123 : }
5124 :
5125 : // 发生ErrCqe之后，统一在AICPU进程上报taskException
5126 0 : if (cqeStatus == dfx::CqeStatus::kCqeException) {
5127 0 : if (IsNoNeedWait() && cqeCtx.sqeType == RT_STARS_SQE_TYPE_PLACE_HOLDER) {
5128 0 : PrintTaskExceptionAllComm(); // 超时场景打印所有通信域的taskException
5129 0 : PrintAicpuCommExecStatus();
5130 0 : } else if (IsNoNeedWait()) {
5131 0 : ErrCqeContext cqeCtx;
5132 0 : stream.GetCqeContext(cqeCtx);
5133 0 : PrintTaskExceptionByTaskId(cqeCtx.sqeType, cqeCtx.taskId, stream, tail); // 仅打印本条流的taskException
5134 : }
5135 : }
5136 :
5137 0 : CHK_PRT_CONT(
5138 : !retryEnable_ && isSdmaCompDataErr,
5139 : HCCL_RUN_INFO(
5140 : "[OpRetry][AICPU]group[%s] hccl aicpu can not retry, retryEnable is false.", identifier_.c_str()));
5141 :
5142 0 : CHK_PRT_CONT(
5143 : retryEnable_ && isSdmaTypeErr && !isCompDataErr,
5144 : HCCL_RUN_INFO(
5145 : "[OpRetry][AICPU]group[%s] hccl aicpu can not retry, errCode is [%u].", identifier_.c_str(),
5146 : cqeCtx.errorCode));
5147 :
5148 0 : HCCL_ERROR(
5149 : "Exception happened, group %s, sqid %d, cqeStatus %d, sqetype %u, errorCode %u, head %u, tail %u",
5150 : identifier_.c_str(), streamInfo.sqId, cqeStatus, cqeCtx.sqeType, cqeCtx.errorCode, head, tail);
5151 0 : }
5152 :
5153 0 : HcclResult HcclCommAicpu::SendTaskExceptionByMBox(const uint16_t& rsErrorCode)
5154 : {
5155 : HcclSignalInfo notifyInfo;
5156 0 : opNotifies_[1]->GetNotifyData(notifyInfo);
5157 :
5158 0 : HCCL_INFO("[HcclCommAicpu][SendTaskExceptionByMBox] HostToDeviceLogicId[%u]", devId_);
5159 0 : CHK_RET(hccl_plf::SendTaskExceptionByMBox(
5160 : devId_, opNotifies_[1]->notifyId_, notifyInfo.tsId, userStreamId_, rsErrorCode));
5161 0 : return HCCL_SUCCESS;
5162 : }
5163 :
5164 0 : void HcclCommAicpu::HandleIndOpCqe()
5165 : {
5166 0 : std::unique_lock<std::mutex> lock(queryCqeMutex_);
5167 0 : std::shared_lock<std::shared_mutex> rwLock(threadAicpuMutex_);
5168 0 : for (auto& thread : threads_) {
5169 0 : if (thread == nullptr) {
5170 0 : continue;
5171 : }
5172 0 : Stream stream = *thread->GetStream();
5173 : // 流上已有异常信息，不再重复读取
5174 0 : if (GetStreamCqeExceptionStatus(stream) != CqeExceptionStatus::kNone) {
5175 0 : continue;
5176 : }
5177 :
5178 : // poll cqe信息
5179 : rtLogicCqReport_t cqeException;
5180 : CqeQueryInput cqeQueryInput;
5181 0 : dfx_tracer::ExecutorTracer::SetCqeQueryInput(GetDevId(), stream.GetHcclStreamInfo(), cqeQueryInput);
5182 0 : constexpr u32 reportSize = 256;
5183 : rtLogicCqReport_t streamReport[reportSize];
5184 0 : cqeQueryInput.cqeAddr = reinterpret_cast<uint8_t*>(streamReport); // 用于存放接收到的cq
5185 0 : CqeStatus cqeStatus = CqReportRecv(cqeQueryInput, cqeException);
5186 : // 未读取到异常信息，返回
5187 0 : if (cqeStatus == dfx::CqeStatus::kDefault) {
5188 0 : continue;
5189 : }
5190 0 : ReportIndOpCqe(stream, cqeException, cqeStatus);
5191 0 : }
5192 0 : }
5193 :
5194 0 : void HcclCommAicpu::ReportIndOpCqe(hccl::Stream& stream, const rtLogicCqReport_t& cqeException, CqeStatus cqeStatus)
5195 : {
5196 0 : const HcclComStreamInfo& streamInfo = stream.GetHcclStreamInfo();
5197 0 : u32 head = 0;
5198 0 : u32 tail = 0;
5199 0 : QuerySqStatusByType(devId_, streamInfo.sqId, DRV_SQCQ_PROP_SQ_HEAD, head);
5200 0 : QuerySqStatusByType(devId_, streamInfo.sqId, DRV_SQCQ_PROP_SQ_TAIL, tail);
5201 :
5202 : // 打印taskException信息
5203 0 : if (cqeStatus == dfx::CqeStatus::kCqeException) {
5204 0 : if (cqeException.sqeType == RT_STARS_SQE_TYPE_PLACE_HOLDER) {
5205 0 : PrintTaskExceptionAllComm(); // 超时场景打印所有通信域的taskException
5206 0 : PrintAicpuCommExecStatus();
5207 : } else {
5208 0 : taskExecption_.PrintTaskExceptionByTaskId(cqeException.sqeType, cqeException.taskId, stream, tail);
5209 : }
5210 : }
5211 :
5212 : // 打印重执行提示信息
5213 0 : bool isSdmaTypeErr = cqeStatus == dfx::CqeStatus::kCqeException && cqeException.sqeType == RT_STARS_SQE_TYPE_SDMA;
5214 0 : bool isCompDataErr = cqeException.errorCode == RT_SDMA_COMPDATAERR || cqeException.errorCode == RT_SDMA_COMPERR;
5215 0 : bool isSdmaCompDataErr = isSdmaTypeErr && isCompDataErr;
5216 0 : if (retryEnable_ && isSdmaCompDataErr) {
5217 0 : uint16_t rsErrorCode = TS_ERROR_RETRY_CONSTRAINT;
5218 0 : CHK_PRT(SendTaskExceptionByMBox(rsErrorCode));
5219 0 : HCCL_RUN_INFO("[OpRetry][AICPU]group[%s] can not retry, IndOp does not support opRetry", identifier_.c_str());
5220 : }
5221 :
5222 : // 标记发生ErrCqe的流
5223 0 : hccl::CqeExceptionStatus cqeExceptionStatus
5224 0 : = isSdmaCompDataErr ? hccl::CqeExceptionStatus::kSdmaErr : hccl::CqeExceptionStatus::kOther;
5225 0 : SetStreamCqeExceptionStatus(stream, cqeExceptionStatus);
5226 :
5227 0 : HCCL_ERROR(
5228 : "Exception happened, group %s, streamId %u, sqid %d, cqeStatus %d, sqetype %u, errorCode %u, "
5229 : "head %u, tail %u",
5230 : identifier_.c_str(), stream.id(), streamInfo.sqId, cqeStatus, cqeException.sqeType, cqeException.errorCode,
5231 : head, tail);
5232 0 : }
5233 :
5234 0 : HcclResult HcclCommAicpu::RefreshLinkForSwitchNic(
5235 : const std::string& newTag, const TransportRequest& transportRequest, const std::map<u32, bool>& remoteRankPortMap,
5236 : bool isSecondBuild, LINK& switchLink)
5237 : {
5238 0 : u32 remoteRankId = transportRequest.remoteUserRank;
5239 0 : auto iterRemoteRank = remoteRankPortMap.find(remoteRankId);
5240 0 : bool needSwitch = iterRemoteRank != remoteRankPortMap.end();
5241 0 : bool isBackup = needSwitch && !(iterRemoteRank->second);
5242 0 : HCCL_INFO(
5243 : "[HcclCommAicpu][%s] newTag[%s], localRank[%u], remoteRank[%u], input memory type[%u], "
5244 : "output memory type[%u], isRdma[%u], isSecondBuild[%u], needSwitch[%u], isBackup[%u].",
5245 : __func__, newTag.c_str(), localUserRank_, remoteRankId, transportRequest.inputMemType,
5246 : transportRequest.outputMemType, transportRequest.isUsedRdma, isSecondBuild, needSwitch, isBackup);
5247 :
5248 0 : if (transportRequest.isUsedRdma && needSwitch) {
5249 0 : auto* linkRes = isBackup ? &linkRdmaResBackUp_ : &linkRdmaRes_;
5250 0 : auto iterRankLinks = linkRes->find(remoteRankId);
5251 0 : CHK_PRT_RET(
5252 : iterRankLinks == linkRes->end(),
5253 : HCCL_ERROR(
5254 : "[HcclCommAicpu][%s] comm[%s], local rank[%u], fail to find relative link for remote rank[%u], "
5255 : "isBackup[%u]",
5256 : __func__, identifier_.c_str(), localUserRank_, remoteRankId, isBackup),
5257 : HCCL_E_INTERNAL);
5258 0 : auto iterRankTagLinks = iterRankLinks->second.find(newTag);
5259 0 : CHK_PRT_RET(
5260 : iterRankTagLinks == iterRankLinks->second.end(),
5261 : HCCL_ERROR(
5262 : "[HcclCommAicpu][%s] comm[%s], local rank[%u], "
5263 : "fail to find relative tag[%s] of links for remote rank[%u], isBackup[%u]",
5264 : __func__, identifier_.c_str(), localUserRank_, newTag.c_str(), remoteRankId, isBackup),
5265 : HCCL_E_INTERNAL);
5266 0 : CHK_PRT_RET(
5267 : isSecondBuild && iterRankLinks->second[newTag].size() < 2U,
5268 : HCCL_ERROR(
5269 : "[HcclCommAicpu][%s] comm[%s], local rank[%u], isSecondBuild[%u], "
5270 : "fail to find second link for remote rank[%u]",
5271 : __func__, identifier_.c_str(), localUserRank_, isSecondBuild, remoteRankId),
5272 : HCCL_E_INTERNAL);
5273 :
5274 0 : switchLink = isSecondBuild ? iterRankLinks->second[newTag][1] : iterRankLinks->second[newTag][0];
5275 0 : CHK_SMART_PTR_NULL(switchLink);
5276 :
5277 0 : auto iterAck = receivedAcks_.find(remoteRankId);
5278 0 : CHK_PRT_RET(
5279 : iterAck == receivedAcks_.end(), HCCL_ERROR("[%s]there is no link with rankId[%u]", __func__, remoteRankId),
5280 : HCCL_E_NOT_FOUND);
5281 0 : switchLink->SetSupportDataReceivedAck(iterAck->second);
5282 : }
5283 0 : return HCCL_SUCCESS;
5284 : }
5285 :
5286 0 : HcclResult HcclCommAicpu::ReAllocTransportForSwitchNic(
5287 : const std::string& newTag, AlgResourceResponse& algResResponse, std::map<u32, bool>& remoteRankPortMap)
5288 : {
5289 0 : HCCL_INFO(
5290 : "[HcclCommAicpu][%s] Entry realloc transport for switch nic, comm identifier[%s], localRank[%u], tag[%s]",
5291 : __func__, identifier_.c_str(), localUserRank_, newTag.c_str());
5292 0 : std::set<u32> bsrTansportRank;
5293 0 : for (auto& levelNSubCommTransport : algResResponse.opTransportResponse) {
5294 0 : for (auto& singleSubCommTransport : levelNSubCommTransport) {
5295 0 : for (size_t i = 0; i < singleSubCommTransport.transportRequests.size(); i++) {
5296 0 : auto& transportRequest = singleSubCommTransport.transportRequests[i];
5297 0 : if (transportRequest.isValid) {
5298 0 : receivedAcks_[transportRequest.remoteUserRank] = singleSubCommTransport.supportDataReceivedAck;
5299 0 : bool isSecondBuild = false;
5300 0 : if (transportRequest.isUsedRdma && newTag.find("BatchSendRecv") != std::string::npos
5301 0 : && bsrTansportRank.find(transportRequest.remoteUserRank) != bsrTansportRank.end()) {
5302 : // 仅在batchsendrecv rdma下发的时候需要第二次刷新，实际第一次下发都刷好了，第二次就是get一下
5303 0 : isSecondBuild = true;
5304 : }
5305 0 : bsrTansportRank.insert(transportRequest.remoteUserRank);
5306 0 : CHK_RET(RefreshLinkForSwitchNic(
5307 : newTag, transportRequest, remoteRankPortMap, isSecondBuild, singleSubCommTransport.links[i]));
5308 : }
5309 : }
5310 : }
5311 : }
5312 :
5313 0 : return HCCL_SUCCESS;
5314 0 : }
5315 :
5316 : HcclResult
5317 0 : HcclCommAicpu::RefreshRoceTransportsForSwitchNic(std::unordered_map<std::string, OpCommTransport>& reservedLinks)
5318 : {
5319 0 : ChangeLinkInfo changeLinkInfo;
5320 0 : HcclResult ret = LoadChangeLinkInfo(changeLinkInfo);
5321 :
5322 0 : CHK_PRT_RET(
5323 : changeLinkInfo.isChangeLinkFlag == false,
5324 : HCCL_ERROR(
5325 : "[HcclCommAicpu][%s] some error happened on host, switch nic has failed on rank[%u]. "
5326 : "The error message will be broadcast to other ranks in comm[%s].",
5327 : __func__, localUserRank_, identifier_.c_str()),
5328 : HCCL_E_PARA);
5329 :
5330 0 : CHK_PRT_RET(
5331 : ret != HCCL_SUCCESS,
5332 : HCCL_ERROR(
5333 : "[HcclCommAicpu][%s] comm identifier[%s], rank[%u], "
5334 : "load change link info failed.",
5335 : __func__, identifier_.c_str(), localUserRank_),
5336 : ret);
5337 :
5338 0 : std::map<u32, bool> remoteRankPortMap;
5339 0 : for (u32 i = 0; i < changeLinkInfo.remoteRankNum; i++) {
5340 0 : remoteRankPortMap.emplace(changeLinkInfo.remoteRankList[i], changeLinkInfo.isUseDefaultPort[i]);
5341 0 : HCCL_INFO(
5342 : "[HcclCommAicpu][%s] comm identifier[%s], rank[%u], remote rank[%u], isUseDefaultPort[%u].", __func__,
5343 : identifier_.c_str(), localUserRank_, changeLinkInfo.remoteRankList[i], changeLinkInfo.isUseDefaultPort[i]);
5344 : }
5345 :
5346 : // 对resMap中所有tag的transport link根据主备进行刷新
5347 0 : for (auto& resMapIt : resMap_) {
5348 0 : HCCL_RUN_INFO(
5349 : "[HcclCommAicpu][%s] rank[%u] refresh algResResponse of tag[%s].", __func__, localUserRank_,
5350 : resMapIt.first.c_str());
5351 0 : reservedLinks.emplace(resMapIt.first, resMapIt.second.opTransportResponse);
5352 0 : CHK_RET(ReAllocTransportForSwitchNic(resMapIt.first, resMapIt.second, remoteRankPortMap));
5353 : }
5354 0 : return HCCL_SUCCESS;
5355 0 : }
5356 :
5357 0 : HcclResult HcclCommAicpu::RevertTransportsForSwitchNic(std::unordered_map<std::string, OpCommTransport>& reservedLinks)
5358 : {
5359 0 : if (reservedLinks.size() == 0) {
5360 0 : HCCL_INFO(
5361 : "[HcclCommAicpu][%s] comm identifier[%s], rank[%u], reserved link is empty, "
5362 : "no need to revert transport.",
5363 : __func__, identifier_.c_str(), localUserRank_);
5364 0 : return HCCL_SUCCESS;
5365 : }
5366 0 : for (auto& resMapIt : resMap_) {
5367 0 : HCCL_RUN_INFO(
5368 : "[HcclCommAicpu][%s] comm identifier[%s], rank[%u], revert transport in algResResponse of tag[%s].",
5369 : __func__, identifier_.c_str(), localUserRank_, resMapIt.first.c_str());
5370 0 : auto linkIt = reservedLinks.find(resMapIt.first);
5371 0 : if (linkIt == reservedLinks.end()) {
5372 0 : HCCL_RUN_WARNING(
5373 : "[HcclCommAicpu][%s] comm identifier[%s], rank[%u], fail to find transport to revert.", __func__,
5374 : identifier_.c_str(), localUserRank_);
5375 0 : continue;
5376 : }
5377 0 : resMapIt.second.opTransportResponse = linkIt->second;
5378 : }
5379 0 : return HCCL_SUCCESS;
5380 : }
5381 :
5382 0 : HcclResult HcclCommAicpu::SwitchNicWaitHandleCommand(std::unordered_map<std::string, OpCommTransport>& reservedLinks)
5383 : {
5384 0 : auto waitSwitchCmdTimeoutMs = HcclGetWaitRetryCmdTimeout(0);
5385 0 : auto waitSwitchCmdTimeout = std::chrono::milliseconds(waitSwitchCmdTimeoutMs);
5386 :
5387 0 : HcclResult ret = HCCL_E_INTERNAL;
5388 0 : auto startTime = std::chrono::steady_clock::now();
5389 : while (true) {
5390 0 : KfcCommand switchHandleCmd = KfcCommand::kNone;
5391 0 : CHK_RET(aicpuHdc_.GetOpExecCtrlCmd(kfcControlTransferH2D_, switchHandleCmd));
5392 0 : if (switchHandleCmd == KfcCommand::kWaitSwitchNic) {
5393 0 : HCCL_INFO(
5394 : "[HcclCommAicpu][%s] comm identifier[%s], rank[%u], hccl aicpu recv switch nic handle command.",
5395 : __func__, identifier_.c_str(), localUserRank_);
5396 0 : ret = aicpuHdc_.SetOpExecStatus(kfcStatusTransferD2H_, KfcStatus::kWaitSwitchRes, KfcError::kNone, 0);
5397 0 : if (ret != HCCL_SUCCESS) {
5398 0 : (void)RevertTransportsForSwitchNic(reservedLinks);
5399 0 : return ret;
5400 : }
5401 0 : return HCCL_SUCCESS;
5402 0 : } else if ((std::chrono::steady_clock::now() - startTime) >= waitSwitchCmdTimeout) {
5403 0 : HCCL_ERROR(
5404 : "[HcclCommAicpu][%s] comm identifier[%s], rank[%u], "
5405 : "hccl aicpu wait switch nic handle command timeout[%u ms], local nic switch is reverted.",
5406 : __func__, identifier_.c_str(), localUserRank_, waitSwitchCmdTimeoutMs);
5407 0 : (void)RevertTransportsForSwitchNic(reservedLinks);
5408 0 : return HCCL_E_TIMEOUT;
5409 : }
5410 0 : }
5411 :
5412 : return ret;
5413 : }
5414 :
5415 0 : HcclResult HcclCommAicpu::SwitchNicWaitResult(std::unordered_map<std::string, OpCommTransport>& reservedLinks)
5416 : {
5417 0 : auto waitSwitchResultTimeoutSecond = GetExternalInputHcclLinkTimeOut() * 2U;
5418 0 : const auto waitSwitchResultTimeout = std::chrono::seconds(waitSwitchResultTimeoutSecond);
5419 :
5420 0 : auto startTime = std::chrono::steady_clock::now();
5421 : while (true) {
5422 0 : KfcCommand switchResultCmd = KfcCommand::kNone;
5423 0 : CHK_RET(aicpuHdc_.GetOpExecCtrlCmd(kfcControlTransferH2D_, switchResultCmd));
5424 0 : if (switchResultCmd == KfcCommand::kAllSwitched) {
5425 0 : HCCL_INFO(
5426 : "[HcclCommAicpu][%s] comm identifier[%s], rank[%u], "
5427 : "hccl aicpu switch nic success, all nic switch transport.",
5428 : __func__, identifier_.c_str(), localUserRank_);
5429 0 : return HCCL_SUCCESS;
5430 0 : } else if (switchResultCmd == KfcCommand::kSwitchFail) {
5431 0 : HCCL_ERROR(
5432 : "[HcclCommAicpu][%s] comm identifier[%s], rank[%u], "
5433 : "hccl aicpu comm switch nic error, local nic switch is reverted.",
5434 : __func__, identifier_.c_str(), localUserRank_);
5435 0 : (void)RevertTransportsForSwitchNic(reservedLinks);
5436 0 : return HCCL_E_INTERNAL;
5437 0 : } else if ((std::chrono::steady_clock::now() - startTime) >= waitSwitchResultTimeout) {
5438 0 : HCCL_ERROR(
5439 : "[HcclCommAicpu][%s] comm identifier[%s], rank[%u], "
5440 : "hccl aicpu wait switch nic result timeout[%u s].",
5441 : __func__, identifier_.c_str(), localUserRank_, waitSwitchResultTimeoutSecond);
5442 0 : (void)RevertTransportsForSwitchNic(reservedLinks);
5443 0 : return HCCL_E_TIMEOUT;
5444 : }
5445 0 : }
5446 :
5447 : return HCCL_E_INTERNAL;
5448 : }
5449 :
5450 0 : HcclResult HcclCommAicpu::SwitchNic()
5451 : {
5452 0 : KfcStatus state = KfcStatus::kSwitchError;
5453 0 : std::unordered_map<std::string, OpCommTransport> reservedLinks;
5454 0 : HcclResult ret = RefreshRoceTransportsForSwitchNic(reservedLinks);
5455 0 : if (ret != HCCL_SUCCESS) {
5456 0 : HCCL_ERROR(
5457 : "[HcclCommAicpu][%s] comm identifier[%s], rank[%u], refresh roce transports failed.", __func__,
5458 : identifier_.c_str(), localUserRank_);
5459 0 : state = KfcStatus::kSwitchError;
5460 0 : ret = aicpuHdc_.SetOpExecStatus(kfcStatusTransferD2H_, state, KfcError::kNone, 0);
5461 : } else {
5462 0 : HCCL_INFO(
5463 : "[HcclCommAicpu][%s] comm identifier[%s], rank[%u], refresh roce transports success.", __func__,
5464 : identifier_.c_str(), localUserRank_);
5465 0 : state = KfcStatus::kPlanSwitch;
5466 0 : ret = aicpuHdc_.SetOpExecStatus(kfcStatusTransferD2H_, state, KfcError::kNone, 0);
5467 : }
5468 :
5469 0 : if (ret != HCCL_SUCCESS) {
5470 0 : HCCL_ERROR(
5471 : "[HcclCommAicpu][SwitchNic] comm identifier[%s], rank[%u], "
5472 : "send switch status[%u] to host fail.",
5473 : __func__, identifier_.c_str(), localUserRank_, state);
5474 : }
5475 :
5476 0 : CHK_RET(SwitchNicWaitHandleCommand(reservedLinks));
5477 :
5478 0 : return SwitchNicWaitResult(reservedLinks);
5479 0 : }
5480 :
5481 0 : HcclResult HcclCommAicpu::ResumeChangeLink()
5482 : {
5483 0 : ChangeLinkInfo changeLinkInfo;
5484 0 : HcclResult ret = LoadChangeLinkInfo(changeLinkInfo);
5485 0 : bool useBackupLink = false;
5486 0 : std::map<u32, bool> remoteRankPortMap;
5487 0 : if (ret != HCCL_SUCCESS) {
5488 0 : HCCL_ERROR(
5489 : "[HcclCommAicpu][%s] comm identifier[%s], rank[%u], load change link info failed.", __func__,
5490 : identifier_.c_str(), localUserRank_);
5491 0 : return ret;
5492 : }
5493 0 : for (u32 i = 0; i < changeLinkInfo.remoteRankNum; i++) {
5494 0 : remoteRankPortMap.insert({changeLinkInfo.remoteRankList[i], changeLinkInfo.isUseDefaultPort[i]});
5495 0 : useBackupLink |= (!changeLinkInfo.isUseDefaultPort[i]);
5496 : }
5497 0 : ret = RefreshCommResponseTransportRes(remoteRankPortMap);
5498 0 : if (ret != HCCL_SUCCESS) {
5499 0 : HCCL_ERROR(
5500 : "[HcclCommAicpu][%s] comm identifier[%s], rank[%u], refresh roce transports failed.", __func__,
5501 : identifier_.c_str(), localUserRank_);
5502 0 : return ret;
5503 : } else {
5504 0 : HCCL_INFO(
5505 : "[HcclCommAicpu][%s] comm identifier[%s], rank[%u], refresh roce transports success.", __func__,
5506 : identifier_.c_str(), localUserRank_);
5507 0 : return HCCL_SUCCESS;
5508 : }
5509 0 : }
5510 :
5511 0 : HcclResult HcclCommAicpu::RefreshCommResponseTransportRes(std::map<u32, bool>& remoteRankPortMap)
5512 : {
5513 0 : PetersonLockGuard guard(hostDeviceLock_.get());
5514 0 : CHK_PRT_RET(guard.IsLockFailed(), HCCL_ERROR("[%s] hostDeviceLock lock failed", __func__), HCCL_E_INTERNAL);
5515 0 : OpParam param;
5516 : // 提前清理所有tag的链路，避免冲突
5517 0 : for (auto& resMapIt : resMap_) {
5518 0 : HCCL_RUN_INFO("[%s] clean roce resource of tag[%s].", __func__, resMapIt.first.c_str());
5519 0 : CleanRoceResource(resMapIt.first, resMapIt.second, remoteRankPortMap, param);
5520 : }
5521 0 : for (auto& resMapIt : resMap_) {
5522 0 : HCCL_RUN_INFO("[%s] refresh algResResponse of tag[%s].", __func__, resMapIt.first.c_str());
5523 0 : CHK_RET(ReAllocTransportResource(resMapIt.first, resMapIt.second, remoteRankPortMap, commParam_, param));
5524 : }
5525 0 : return HCCL_SUCCESS;
5526 0 : }
5527 :
5528 0 : HcclResult HcclCommAicpu::InitAicpuIndOp(CommAicpuParam* commAicpuParam)
5529 : {
5530 0 : if (indOpCommInitialized_) {
5531 0 : HCCL_RUN_INFO("[%s][InitAicpuIndOp]Group[%s] already initialized, skip reinit", __func__, identifier_.c_str());
5532 0 : return HCCL_SUCCESS;
5533 : }
5534 0 : CHK_PTR_NULL(commAicpuParam);
5535 0 : topoInfo_.deviceLogicId = commAicpuParam->deviceLogicId;
5536 0 : topoInfo_.devicePhyId = commAicpuParam->devicePhyId;
5537 0 : topoInfo_.deviceType = static_cast<DevType>(commAicpuParam->deviceType);
5538 0 : identifier_ = std::string(commAicpuParam->hcomId);
5539 0 : topoInfo_.userRankSize = commAicpuParam->userRankSize;
5540 0 : topoInfo_.userRank = commAicpuParam->userRank;
5541 0 : notifys_.reserve(hccl::HCCL_THREAD_NOTIFY_MAX_NUM);
5542 0 : if (topoInfo_.deviceType == DevType::DEV_TYPE_910_93 || topoInfo_.deviceType == DevType::DEV_TYPE_910B) {
5543 0 : notifySize_ = NOTIFY_SIZE_FOUR;
5544 : } else {
5545 0 : notifySize_ = NOTIFY_SIZE_EIGHT;
5546 : }
5547 :
5548 0 : CHK_RET(hrtSetWorkModeAicpu(true));
5549 0 : CHK_RET(hrtSetlocalDevice(topoInfo_.deviceLogicId));
5550 0 : CHK_RET(hrtSetlocalDeviceType(topoInfo_.deviceType));
5551 0 : CHK_RET(hrtDrvGetLocalDevIDByHostDevID(topoInfo_.devicePhyId, &devId_));
5552 0 : CHK_RET(taskExecption_.Init(devId_, topoInfo_.userRank, identifier_));
5553 0 : CHK_RET(RegisterProfCallBack());
5554 :
5555 0 : if (topoInfo_.deviceType == DevType::DEV_TYPE_950 || topoInfo_.deviceType == DevType::DEV_TYPE_960) {
5556 0 : HCCL_INFO("[HcclCommAicpu][InitAicpuIndOp] InitAicpuIndOpV2 start");
5557 0 : indOpCommInitialized_ = true;
5558 0 : return HCCL_SUCCESS;
5559 : }
5560 :
5561 0 : HCCL_INFO("[HcclCommAicpu][InitAicpuIndOp] InitAicpuIndOp start");
5562 0 : if (!FindDispatcherByCommId(&dispatcherCtx_, identifier_.c_str())) {
5563 0 : CHK_RET(CreateDispatcherCtx(&dispatcherCtx_, devId_, identifier_.c_str()));
5564 : }
5565 0 : CHK_PTR_NULL(dispatcherCtx_);
5566 0 : hccl::DispatcherCtx* Ctx_temp = static_cast<DispatcherCtx*>(dispatcherCtx_);
5567 0 : HCCL_INFO("[%s] Ctx_temp[%p]", __func__, (void*)Ctx_temp);
5568 0 : (void)RegisterLoadTaskCallBack(Ctx_temp->GetDispatcher(), nullptr, dfx::TaskProfilingCallBack); // 注册dispatcher
5569 0 : if (commAicpuParam->kfcControlTransferH2DParams.buffLen != 0 && kfcControlTransferH2D_ == nullptr) {
5570 0 : EXCEPTION_CATCH((kfcControlTransferH2D_ = std::make_shared<hccl::HDCommunicate>()), return HCCL_E_PTR);
5571 0 : CHK_SMART_PTR_NULL(kfcControlTransferH2D_);
5572 0 : CHK_RET(kfcControlTransferH2D_->InitDevice(commAicpuParam->kfcControlTransferH2DParams));
5573 : }
5574 0 : if (commAicpuParam->kfcStatusTransferD2HParams.buffLen != 0 && kfcStatusTransferD2H_ == nullptr) {
5575 0 : EXCEPTION_CATCH((kfcStatusTransferD2H_ = std::make_shared<hccl::HDCommunicate>()), return HCCL_E_PTR);
5576 0 : CHK_SMART_PTR_NULL(kfcStatusTransferD2H_);
5577 0 : CHK_RET(kfcStatusTransferD2H_->InitDevice(commAicpuParam->kfcStatusTransferD2HParams));
5578 : }
5579 :
5580 0 : indOpCommInitialized_ = true;
5581 :
5582 : // 在indOpCommInitialized_变为true后拉起背景线程
5583 0 : AicpuComContext* ctx = AicpuGetComContext();
5584 0 : AicpuHcclProcess::CallMC2MaintenanceThread(ctx);
5585 :
5586 0 : HCCL_RUN_INFO(
5587 : "%s group[%s] success!, deviceLogicId[%u], devicePhyId[%u], deviceType[%u], notifySize[%u], "
5588 : "dispatcherCtx[%p]",
5589 : __func__, identifier_.c_str(), topoInfo_.deviceLogicId, topoInfo_.devicePhyId, topoInfo_.deviceType,
5590 : notifySize_, dispatcherCtx_);
5591 :
5592 0 : return HCCL_SUCCESS;
5593 : }
5594 :
5595 0 : HcclResult HcclCommAicpu::InitThreads(ThreadMgrAicpuParam* param)
5596 : {
5597 0 : CHK_PTR_NULL(param);
5598 0 : u32 threadNum = param->threadNum;
5599 0 : std::vector<std::shared_ptr<Thread>> outThreads;
5600 0 : outThreads.reserve(threadNum);
5601 0 : std::string hcomId(param->hcomId);
5602 0 : for (u32 i = 0; i < threadNum; ++i) {
5603 0 : std::string thdUniqueId(param->threadParam[i], THREAD_UNIQUE_ID_MAX_SIZE);
5604 0 : if (UNLIKELY(HcclCheckLogLevel(HCCL_LOG_INFO))) {
5605 0 : std::ostringstream oss;
5606 0 : oss << "threadParam[" << i << "] raw bytes: ";
5607 0 : for (u32 j = 0; j < THREAD_UNIQUE_ID_MAX_SIZE; ++j) {
5608 0 : oss << std::hex << std::setw(2) << std::setfill('0')
5609 0 : << static_cast<unsigned int>(static_cast<unsigned char>(param->threadParam[i][j])) << " ";
5610 : }
5611 0 : HCCL_INFO("[HcclCommAicpu][%s] %s", __func__, oss.str().c_str());
5612 0 : }
5613 0 : std::shared_ptr<AicpuTsThread> thread;
5614 0 : EXCEPTION_CATCH((thread = std::make_shared<AicpuTsThread>(thdUniqueId)), return HCCL_E_PTR);
5615 0 : thread->SetCommEngine(param->engine);
5616 0 : HcclResult ret = thread->Init();
5617 0 : if (ret != HCCL_SUCCESS) {
5618 0 : HCCL_ERROR(
5619 : "[HcclCommAicpu][%s] comm identifier[%s], init threads num[%u] failed at index %u", __func__,
5620 : hcomId.c_str(), param->threadNum, i);
5621 0 : return ret;
5622 : }
5623 0 : outThreads.emplace_back(thread);
5624 0 : }
5625 :
5626 0 : ThreadHandle* threadArray = static_cast<ThreadHandle*>(param->deviceHandle);
5627 : // 空指针校验
5628 0 : CHK_PTR_NULL(threadArray);
5629 0 : for (size_t i = 0; i < threadNum; ++i) {
5630 0 : threadArray[i] = reinterpret_cast<ThreadHandle>(outThreads[i].get()); // 拷贝裸指针
5631 0 : HCCL_INFO("[HcclCommAicpu][%s] threadArray[%u] = [%lu]", __func__, i, threadArray[i]);
5632 : }
5633 0 : std::unique_lock<std::shared_mutex> rwLock(threadAicpuMutex_);
5634 0 : threads_.insert(
5635 0 : threads_.end(), std::make_move_iterator(outThreads.begin()), std::make_move_iterator(outThreads.end()));
5636 0 : HCCL_INFO(
5637 : "[HcclCommAicpu][%s] comm identifier[%s], init threads num[%u] success", __func__, hcomId.c_str(), threadNum);
5638 : // 为上报翻转初始化资源
5639 0 : if (topoInfo_.deviceType != DevType::DEV_TYPE_950 && topoInfo_.deviceType != DevType::DEV_TYPE_960) {
5640 0 : CHK_RET(InitProfthreadResource(threadNum));
5641 : }
5642 0 : return HCCL_SUCCESS;
5643 0 : }
5644 :
5645 0 : HcclResult HcclCommAicpu::InitProfthreadResource(u32 threadNum)
5646 : {
5647 0 : groupHashId_ = dfx::ProfilingManager::GetProfHashId(identifier_.c_str(), identifier_.length());
5648 0 : HCCL_INFO(
5649 : "[%s], group[%s], groupHash[%llu], threadNum[%u] ", __func__, identifier_.c_str(), groupHashId_, threadNum);
5650 0 : dfx::ProfCommInfo profInfo{groupHashId_, topoInfo_.userRankSize, topoInfo_.userRank};
5651 : // 添加检查确保 threadNum 不超过线程总数
5652 0 : if (threadNum > threads_.size()) {
5653 0 : HCCL_ERROR("[%s] threadNum Err", __func__);
5654 0 : return HCCL_E_PARA;
5655 : }
5656 : // 从后往前迭代指定数量
5657 0 : auto begin = threads_.rbegin();
5658 0 : auto end = begin + threadNum;
5659 0 : for (auto it = begin; it != end; ++it) {
5660 0 : CHK_RET(dfx::ProfilingManager::AddProfInfoByStreamId((*it)->GetStream()->id(), identifier_, profInfo));
5661 : }
5662 0 : dfx::ProfilingExtendInfoHelper::InitProfItemId();
5663 0 : return HCCL_SUCCESS;
5664 : }
5665 :
5666 0 : HcclResult HcclCommAicpu::AllocChannelResource(HcclIndOpChannelRemoteResV3* commParam)
5667 : {
5668 0 : CHK_PTR_NULL(commParam);
5669 0 : if (commParam->engine != COMM_ENGINE_AICPU && commParam->engine != COMM_ENGINE_AICPU_TS) {
5670 0 : HCCL_ERROR(
5671 : "[HcclCommAicpu][%s] engine type[%s] is not supported", __func__,
5672 : GetEnumToString(GetCommEngineStatusStrMap(), commParam->engine).c_str());
5673 0 : return HCCL_E_PARA;
5674 : }
5675 :
5676 0 : if (commParam->listNum != 0 && commParam->remoteResV2 == nullptr) {
5677 0 : HCCL_ERROR("[HcclCommAicpu][%s] listNum[%u], remoteResV2 is nullptr", __func__, commParam->listNum);
5678 0 : return HCCL_E_PARA;
5679 : }
5680 :
5681 0 : multiQpThreshold_ = commParam->multiQpThreshold;
5682 0 : localUserRank_ = commParam->localUserRank;
5683 0 : HCCL_INFO(
5684 : "%s multiQpThreshold[%u], localUserRank[%u], deviceLogicId[%d], devicePhyId[%u], deviceType[%d], "
5685 : "listNum[%u]",
5686 : __func__, multiQpThreshold_, localUserRank_, topoInfo_.deviceLogicId, topoInfo_.devicePhyId,
5687 : topoInfo_.deviceType, commParam->listNum);
5688 0 : for (u32 idx = 0; idx < commParam->listNum; idx++) {
5689 0 : HCCL_INFO(
5690 : "%s listNum[%u], listIdx[%u], remoteWorldRank[%u], remoteRank[%u], isUsedRdma[%d]", __func__,
5691 : commParam->listNum, idx, commParam->remoteResV2[idx].remoteWorldRank,
5692 : commParam->remoteResV2[idx].remoteRank, commParam->remoteResV2[idx].isUsedRdma);
5693 :
5694 0 : rankData_[commParam->remoteResV2[idx].remoteRank].remoteWorldRank = commParam->remoteResV2[idx].remoteWorldRank;
5695 0 : rankData_[commParam->remoteResV2[idx].remoteRank].remoteUsrRankId = commParam->remoteResV2[idx].remoteRank;
5696 0 : if (commParam->remoteResV2[idx].isUsedRdma) {
5697 0 : CHK_RET(InitRoceChannel(commParam, idx));
5698 : } else {
5699 0 : CHK_RET(InitP2pChannel(commParam, idx));
5700 : }
5701 : }
5702 0 : return HCCL_SUCCESS;
5703 : }
5704 :
5705 0 : HcclResult HcclCommAicpu::InitP2pChannel(HcclIndOpChannelRemoteResV3* commParam, uint32_t channelIndex)
5706 : {
5707 0 : CHK_PTR_NULL(commParam);
5708 0 : CHK_PTR_NULL(commParam->channelList);
5709 0 : HcclIndOpChannelRemoteResV2& remoteResV2 = commParam->remoteResV2[channelIndex];
5710 0 : u32 linkType = static_cast<u32>(remoteResV2.channelP2p.transportAttr.linkType);
5711 0 : std::string channelKey = std::string(commParam->channelTag) + ":" + std::to_string(commParam->engine) + ":"
5712 0 : + std::to_string(remoteResV2.remoteRank) + ":" + std::to_string(linkType);
5713 0 : HCCL_INFO("%s channelKey[%s]", __func__, channelKey.c_str());
5714 0 : if (channelHandleMap_.find(channelKey) != channelHandleMap_.end()) {
5715 0 : HCCL_ERROR("[%s]the channel has existed.", __func__);
5716 0 : return HCCL_E_INTERNAL;
5717 : }
5718 :
5719 0 : HcclChannelP2p& channelP2p = remoteResV2.channelP2p;
5720 0 : if (channelP2p.localIpcSignal[0].resId == INVALID_U64) {
5721 0 : HCCL_ERROR("[%s]the Channel is invalid", __func__);
5722 0 : return HCCL_E_INTERNAL;
5723 : }
5724 :
5725 : // 创建Transport对象
5726 0 : MachinePara machinePara;
5727 0 : CHK_RET(SetTransportMachinePara(machinePara, remoteResV2.remoteRank, commParam->channelTag));
5728 0 : machinePara.notifyNum = remoteResV2.p2pNotifyNum;
5729 : // 获取localMem & remoteMem
5730 0 : TransportDeviceP2pData transDevP2pData;
5731 0 : transDevP2pData.inputBufferPtr = reinterpret_cast<void*>(channelP2p.remoteHcclbuffer.addr);
5732 0 : transDevP2pData.outputBufferPtr = reinterpret_cast<void*>(channelP2p.remoteHcclbuffer.addr);
5733 0 : if (transDevP2pData.inputBufferPtr == nullptr || transDevP2pData.outputBufferPtr == nullptr) {
5734 0 : HCCL_ERROR(
5735 : "[%s]input ptr[%p] or output ptr[%p] is null.", __func__, transDevP2pData.inputBufferPtr,
5736 : transDevP2pData.outputBufferPtr);
5737 0 : return HCCL_E_PARA;
5738 : }
5739 : // 获取Notify资源
5740 0 : CHK_RET(SetChannelP2pNotify(transDevP2pData, remoteResV2.p2pNotifyNum, channelP2p)); // 待确认notify是否
5741 : // 获取transportAttr信息
5742 0 : transDevP2pData.transportAttr = channelP2p.transportAttr;
5743 : // 创建Transport对象
5744 0 : std::shared_ptr<Transport> link;
5745 0 : TransportPara para{};
5746 0 : const std::unique_ptr<NotifyPool> notifyPool;
5747 0 : DispatcherCtx* ctx = static_cast<DispatcherCtx*>(dispatcherCtx_);
5748 0 : CHK_PRT(ctx->SetDispatcherHcclQos(remoteResV2.channelP2p.qos)); // 调度器添加hcclQos
5749 0 : CHK_PTR_NULL(ctx);
5750 0 : link.reset(new (std::nothrow) Transport(
5751 0 : TransportType::TRANS_TYPE_DEVICE_P2P, para, ctx->GetDispatcher(), notifyPool, machinePara, transDevP2pData));
5752 0 : CHK_SMART_PTR_NULL(link);
5753 0 : CHK_RET(link->Init()); // 初始化需要增加远端用户注册内存
5754 :
5755 0 : ChannelHandle channelHandle = reinterpret_cast<ChannelHandle>(link.get());
5756 0 : channelHandleMap_[channelKey] = channelHandle;
5757 0 : linkMap_[channelHandle] = link;
5758 :
5759 : // 恢复出的channelHandle回填到commParam中
5760 0 : CHK_PTR_NULL(commParam->channelList);
5761 0 : ChannelHandle* channelList = reinterpret_cast<ChannelHandle*>(commParam->channelList);
5762 0 : channelList[channelIndex] = channelHandle;
5763 :
5764 0 : return HCCL_SUCCESS;
5765 0 : }
5766 :
5767 0 : HcclResult HcclCommAicpu::SetChannelP2pNotify(
5768 : TransportDeviceP2pData& transDevP2pData, u64& p2pNotifyNum, HcclChannelP2p& channelP2p)
5769 : {
5770 0 : u64 actualNotifyNum = 0;
5771 : // 获取Ipc notify信息
5772 0 : CHK_RET(CheckNotifyOrQPMaxNum(actualNotifyNum, LINK_P2P_MAX_NUM, true));
5773 0 : std::shared_ptr<LocalNotify> ipcPreWaitNotify = std::make_shared<LocalNotify>();
5774 0 : CHK_RET(InitAndVerifySingleSignal(channelP2p.localIpcSignal[actualNotifyNum], ipcPreWaitNotify));
5775 0 : transDevP2pData.ipcPreWaitNotify = ipcPreWaitNotify;
5776 :
5777 0 : std::shared_ptr<RemoteNotify> ipcPreRecordNotify = std::make_shared<RemoteNotify>();
5778 0 : CHK_RET(InitAndVerifySingleSignal(channelP2p.remoteIpcSignal[actualNotifyNum], ipcPreRecordNotify));
5779 0 : transDevP2pData.ipcPreRecordNotify = ipcPreRecordNotify;
5780 0 : actualNotifyNum++;
5781 :
5782 0 : CHK_RET(CheckNotifyOrQPMaxNum(actualNotifyNum, LINK_P2P_MAX_NUM, true));
5783 0 : std::shared_ptr<LocalNotify> ipcPostWaitNotify = std::make_shared<LocalNotify>();
5784 0 : CHK_RET(InitAndVerifySingleSignal(channelP2p.localIpcSignal[actualNotifyNum], ipcPostWaitNotify));
5785 0 : transDevP2pData.ipcPostWaitNotify = ipcPostWaitNotify;
5786 :
5787 0 : std::shared_ptr<RemoteNotify> ipcPostRecordNotify = std::make_shared<RemoteNotify>();
5788 0 : CHK_RET(InitAndVerifySingleSignal(channelP2p.remoteIpcSignal[actualNotifyNum], ipcPostRecordNotify));
5789 0 : transDevP2pData.ipcPostRecordNotify = ipcPostRecordNotify;
5790 0 : actualNotifyNum++;
5791 :
5792 0 : transDevP2pData.userLocalNotify.resize(p2pNotifyNum, nullptr);
5793 0 : transDevP2pData.userRemoteNotify.resize(p2pNotifyNum, nullptr);
5794 :
5795 0 : for (u32 idx = 0; idx < p2pNotifyNum; idx++) {
5796 0 : std::shared_ptr<LocalNotify> ipcWaitNotify = std::make_shared<LocalNotify>();
5797 0 : CHK_RET(InitAndVerifySingleSignal(channelP2p.localIpcSignal[actualNotifyNum], ipcWaitNotify));
5798 0 : transDevP2pData.userLocalNotify[idx] = ipcWaitNotify;
5799 :
5800 0 : std::shared_ptr<RemoteNotify> ipcRecordNotify = std::make_shared<RemoteNotify>();
5801 0 : CHK_RET(InitAndVerifySingleSignal(channelP2p.remoteIpcSignal[actualNotifyNum], ipcRecordNotify));
5802 0 : transDevP2pData.userRemoteNotify[idx] = ipcRecordNotify;
5803 :
5804 0 : actualNotifyNum++;
5805 0 : }
5806 :
5807 0 : HCCL_DEBUG(
5808 : "%s get p2pNotify success, p2pNotifyNum[%llu], actualNotifyNum[%llu]", __func__, p2pNotifyNum, actualNotifyNum);
5809 0 : return HCCL_SUCCESS;
5810 0 : }
5811 :
5812 0 : HcclResult HcclCommAicpu::InitRoceChannel(HcclIndOpChannelRemoteResV3* commParam, uint32_t channelIndex)
5813 : {
5814 0 : HcclIndOpChannelRemoteResV2& remoteResV2 = commParam->remoteResV2[channelIndex];
5815 0 : std::string channelKey = std::string(commParam->channelTag) + ":" + std::to_string(commParam->engine) + ":"
5816 0 : + std::to_string(remoteResV2.remoteRank) + ":"
5817 0 : + std::to_string(CommProtocol::COMM_PROTOCOL_ROCE);
5818 0 : if (channelHandleMap_.find(channelKey) != channelHandleMap_.end()) {
5819 0 : HCCL_ERROR("[%s]the channel has existed.", __func__);
5820 0 : return HCCL_E_INTERNAL;
5821 : }
5822 :
5823 0 : HcclChannelRoce& channelRoce = remoteResV2.channelRoce;
5824 0 : if (channelRoce.localNotifyList == 0) {
5825 0 : HCCL_ERROR("[%s]the Channel is invalid", __func__);
5826 0 : return HCCL_E_INTERNAL;
5827 : }
5828 0 : HcclSignalInfo* localNotifyList = reinterpret_cast<HcclSignalInfo*>(channelRoce.localNotifyList);
5829 0 : if (localNotifyList[0].resId == INVALID_U64) {
5830 0 : HCCL_INFO("[%s]the channel notify resource is invalid", __func__);
5831 0 : return HCCL_E_INTERNAL;
5832 : }
5833 :
5834 : // 创建Transport对象
5835 0 : MachinePara machinePara;
5836 0 : CHK_RET(SetTransportMachinePara(machinePara, remoteResV2.remoteRank, commParam->channelTag)); // 待确认是否填充完毕
5837 0 : machinePara.notifyNum = remoteResV2.roceNotifyNum;
5838 : // 获取localMem & remoteMem
5839 0 : TransportDeviceIbverbsData transDevIbverbsData;
5840 0 : transDevIbverbsData.inputBufferPtr = reinterpret_cast<void*>(channelRoce.remoteHcclbuffer.addr);
5841 0 : transDevIbverbsData.outputBufferPtr = reinterpret_cast<void*>(channelRoce.remoteHcclbuffer.addr);
5842 0 : if (transDevIbverbsData.inputBufferPtr == nullptr || transDevIbverbsData.outputBufferPtr == nullptr) {
5843 0 : HCCL_ERROR(
5844 : "[%s]input ptr[%p] or output ptr[%p] is null.", __func__, transDevIbverbsData.inputBufferPtr,
5845 : transDevIbverbsData.outputBufferPtr);
5846 0 : return HCCL_E_PARA;
5847 : }
5848 0 : transDevIbverbsData.localInputMem = channelRoce.localHcclbuffer;
5849 0 : transDevIbverbsData.localOutputMem = channelRoce.localHcclbuffer;
5850 0 : transDevIbverbsData.localNotifyValueAddr = channelRoce.notifyValue;
5851 0 : transDevIbverbsData.notifyValueKey = channelRoce.notifyValueKey;
5852 0 : transDevIbverbsData.remoteInputKey = channelRoce.remoteHcclbuffer.key;
5853 0 : transDevIbverbsData.remoteOutputKey = channelRoce.remoteHcclbuffer.key;
5854 : // 需要添加远端用户注册内存
5855 :
5856 : // 获取QPinfo
5857 0 : u32 roceQpNumSum = channelRoce.qpsPerConnection + static_cast<u32>(channelRoce.qpsPerConnection != 1);
5858 0 : transDevIbverbsData.qpInfo.resize(roceQpNumSum);
5859 0 : std::copy_n(channelRoce.QpInfo, roceQpNumSum, transDevIbverbsData.qpInfo.begin());
5860 0 : transDevIbverbsData.multiQpThreshold = multiQpThreshold_;
5861 0 : transDevIbverbsData.qpsPerConnection = channelRoce.qpsPerConnection;
5862 :
5863 : // 获取notify
5864 0 : u64& roceNotifyNum = remoteResV2.roceNotifyNum; // 是否需要待确认
5865 0 : CHK_RET(SetChannelRoceNotify(transDevIbverbsData, roceNotifyNum, channelRoce));
5866 :
5867 : // 创建Transport对象
5868 0 : std::shared_ptr<Transport> link;
5869 0 : TransportPara para{};
5870 0 : para.timeout = linkTimeOut_; // 暂无法设置
5871 0 : const std::unique_ptr<NotifyPool> notifyPool;
5872 0 : DispatcherCtx* ctx = static_cast<DispatcherCtx*>(dispatcherCtx_);
5873 0 : CHK_PTR_NULL(ctx);
5874 0 : link.reset(new (std::nothrow) Transport(
5875 : TransportType::TRANS_TYPE_DEVICE_IBVERBS, para, ctx->GetDispatcher(), notifyPool, machinePara,
5876 0 : TransportDeviceP2pData(), transDevIbverbsData));
5877 0 : CHK_SMART_PTR_NULL(link);
5878 0 : CHK_RET(link->Init()); // 初始化需要增加远端用户注册内存
5879 :
5880 0 : ChannelHandle channelHandle = reinterpret_cast<ChannelHandle>(link.get());
5881 0 : channelHandleMap_[channelKey] = channelHandle;
5882 0 : linkMap_[channelHandle] = link;
5883 :
5884 : // 恢复出的channelHandle回填到commParam中
5885 0 : ChannelHandle* channelList = reinterpret_cast<ChannelHandle*>(commParam->channelList);
5886 0 : channelList[channelIndex] = channelHandle;
5887 :
5888 0 : return HCCL_SUCCESS;
5889 0 : }
5890 :
5891 0 : HcclResult HcclCommAicpu::SetChannelRoceNotify(
5892 : TransportDeviceIbverbsData& transDevIbverbsData, u64& roceNotifyNum, HcclChannelRoce& channelRoce)
5893 : {
5894 0 : u64 actualNotifyNum = 0;
5895 0 : if (channelRoce.localNotifyList == 0 || channelRoce.remoteNotifyList == 0) {
5896 0 : HCCL_DEBUG("[%s] Empty local and remote notify lists, skipping notify resource creation.", __func__);
5897 0 : return HCCL_SUCCESS;
5898 : }
5899 0 : HcclSignalInfo* localNotifyList = reinterpret_cast<HcclSignalInfo*>(channelRoce.localNotifyList);
5900 0 : AddrKey* remoteNotifyList = reinterpret_cast<AddrKey*>(channelRoce.remoteNotifyList);
5901 0 : if (localNotifyList == nullptr || remoteNotifyList == nullptr) {
5902 0 : HCCL_ERROR("[%s]nullptr found in localNotifyList or remoteNotifyList from device mem, check.", __func__);
5903 0 : return HCCL_E_INTERNAL;
5904 : }
5905 : // 获取RDMA Notify信息
5906 0 : std::shared_ptr<LocalNotify> ackNotify = std::make_shared<LocalNotify>();
5907 0 : CHK_RET(InitAndVerifySingleSignal(localNotifyList[actualNotifyNum], ackNotify));
5908 0 : transDevIbverbsData.ackNotify = ackNotify;
5909 0 : transDevIbverbsData.remoteAckNotifyDetails = remoteNotifyList[actualNotifyNum];
5910 0 : actualNotifyNum++;
5911 :
5912 0 : std::shared_ptr<LocalNotify> dataNotify = std::make_shared<LocalNotify>();
5913 0 : CHK_RET(InitAndVerifySingleSignal(localNotifyList[actualNotifyNum], dataNotify));
5914 0 : transDevIbverbsData.dataNotify = dataNotify;
5915 0 : transDevIbverbsData.remoteDataNotifyDetails = remoteNotifyList[actualNotifyNum];
5916 0 : actualNotifyNum++;
5917 :
5918 0 : std::shared_ptr<LocalNotify> dataAckNotify = std::make_shared<LocalNotify>();
5919 0 : CHK_RET(InitAndVerifySingleSignal(localNotifyList[actualNotifyNum], dataAckNotify));
5920 0 : transDevIbverbsData.dataAckNotify = dataAckNotify;
5921 0 : transDevIbverbsData.remoteDataAckNotifyDetails = remoteNotifyList[actualNotifyNum];
5922 0 : transDevIbverbsData.notifySize = notifySize_;
5923 0 : actualNotifyNum++;
5924 :
5925 0 : transDevIbverbsData.userLocalNotify.resize(channelRoce.qpsPerConnection);
5926 0 : transDevIbverbsData.userRemoteNotifyDetails.resize(channelRoce.qpsPerConnection);
5927 : // 当前多QP下每个QP会多申请一个DataNotify
5928 0 : u64 singleQpNotifySize = channelRoce.singleQPNotifyNum + static_cast<u32>(channelRoce.qpsPerConnection > 1);
5929 0 : for (u32 qpIndex = 0; qpIndex < channelRoce.qpsPerConnection; qpIndex++) {
5930 0 : transDevIbverbsData.userLocalNotify[qpIndex].resize(singleQpNotifySize, nullptr);
5931 0 : transDevIbverbsData.userRemoteNotifyDetails[qpIndex].resize(singleQpNotifySize);
5932 0 : for (u32 i = 0, idx = actualNotifyNum + singleQpNotifySize * qpIndex; i < singleQpNotifySize; ++idx, ++i) {
5933 0 : std::shared_ptr<LocalNotify> locNotify = std::make_shared<LocalNotify>();
5934 0 : CHK_RET(InitAndVerifySingleSignal(localNotifyList[idx], locNotify));
5935 0 : transDevIbverbsData.userLocalNotify[qpIndex][i] = locNotify;
5936 0 : transDevIbverbsData.userRemoteNotifyDetails[qpIndex][i] = remoteNotifyList[idx];
5937 0 : }
5938 : }
5939 0 : roceNotifyNum = channelRoce.singleQPNotifyNum;
5940 0 : HCCL_DEBUG("[%s]get roceNotify success, roceNotifyNum[%u]", __func__, roceNotifyNum);
5941 0 : return HCCL_SUCCESS;
5942 0 : }
5943 :
5944 0 : HcclResult HcclCommAicpu::NotifyAlloc(NotifyMgrAicpuParam* param)
5945 : {
5946 0 : u32 notifyNum = param->notifyNum;
5947 0 : std::string notifysStr = std::string(param->notifyParam, NOTIFY_UNIQUE_ID_MAX_SIZE);
5948 0 : std::string hcomId(param->hcomId);
5949 0 : size_t notifySize = notifys_.size();
5950 0 : HCCL_INFO(
5951 : "[HcclCommAicpu][%s] comm identifier[%s], alloc notifys num[%u] begin, before notifySize[%u]", __func__,
5952 : hcomId.c_str(), notifyNum, notifySize);
5953 0 : if (UNLIKELY(HcclCheckLogLevel(HCCL_LOG_INFO))) {
5954 0 : std::ostringstream oss;
5955 0 : oss << "notifyParam" << " raw bytes: ";
5956 0 : for (u32 i = 0; i < NOTIFY_UNIQUE_ID_MAX_SIZE; ++i) {
5957 0 : oss << std::hex << std::setw(2) << std::setfill('0')
5958 0 : << static_cast<unsigned int>(static_cast<unsigned char>(param->notifyParam[i])) << " ";
5959 : }
5960 0 : HCCL_INFO("[HcclCommAicpu][%s] %s", __func__, oss.str().c_str());
5961 0 : }
5962 0 : HcclResult ret = NotifyManager::ParseBinNotifys(notifysStr, notifys_);
5963 0 : if (ret != HCCL_SUCCESS) {
5964 0 : HCCL_ERROR(
5965 : "[HcclCommAicpu][%s] comm identifier[%s], alloc notifys num[%u] failed %u", __func__, hcomId.c_str(),
5966 : notifyNum, ret);
5967 0 : return ret;
5968 : }
5969 0 : HCCL_INFO(
5970 : "[HcclCommAicpu][%s] comm identifier[%s], alloc notifys num[%u] end, after notifySize[%u]", __func__,
5971 : hcomId.c_str(), notifyNum, notifys_.size());
5972 0 : NotifyHandle* notifyArray = static_cast<NotifyHandle*>(param->deviceHandle);
5973 0 : CHK_PTR_NULL(notifyArray);
5974 : // 空指针校验
5975 0 : for (size_t i = 0; i < notifyNum; ++i) {
5976 0 : notifyArray[i] = reinterpret_cast<NotifyHandle>(notifys_[i + notifySize].get()); // 拷贝裸指针
5977 0 : HCCL_INFO("[HcclCommAicpu][%s] notifyArray[%u] = [%lu]", __func__, i + notifySize, notifyArray[i]);
5978 : }
5979 :
5980 0 : HCCL_INFO(
5981 : "[HcclCommAicpu][%s] comm identifier[%s], alloc notifys num[%u] success", __func__, hcomId.c_str(), notifyNum);
5982 0 : return HCCL_SUCCESS;
5983 0 : }
5984 :
5985 0 : HcclResult HcclCommAicpu::NotifyFree(NotifyMgrAicpuParam* param)
5986 : {
5987 0 : u32 notifyNum = param->notifyNum;
5988 0 : NotifyHandle* notifyArray = static_cast<NotifyHandle*>(param->deviceHandle);
5989 0 : std::string hcomId(param->hcomId);
5990 : // 空指针校验
5991 0 : CHK_PTR_NULL(notifyArray);
5992 0 : for (size_t i = 0; i < notifyNum; ++i) {
5993 0 : LocalNotify* notify = reinterpret_cast<LocalNotify*>(notifyArray[i]);
5994 0 : HCCL_INFO("[HcclCommAicpu][%s] notifyArray[%u]=[%lu]", __func__, i, notifyArray[i]);
5995 0 : auto it = std::find_if(notifys_.begin(), notifys_.end(), [notify](const std::unique_ptr<LocalNotify>& ptr) {
5996 0 : return ptr.get() == notify;
5997 : });
5998 0 : if (it != notifys_.end()) {
5999 0 : HCCL_INFO(
6000 : "[HcclCommAicpu][%s] comm identifier[%s], free notifys[%u] success", __func__, hcomId.c_str(),
6001 : notifyArray[i]);
6002 0 : notifys_.erase(it);
6003 : } else {
6004 0 : HCCL_RUN_WARNING("[HcclCommAicpu][%s] localNotify[%u] not found in notifys_", __func__, i);
6005 : }
6006 : }
6007 :
6008 0 : HCCL_INFO(
6009 : "[HcclCommAicpu][%s] comm identifier[%s], free notifys num[%u] success", __func__, hcomId.c_str(), notifyNum);
6010 0 : return HCCL_SUCCESS;
6011 0 : }
6012 :
6013 0 : HcclResult HcclCommAicpu::RegisterOpInfo(void* opInfo, u32 size)
6014 : {
6015 0 : CHK_RET(taskExecption_.RegisterOpInfo(opInfo, size));
6016 0 : u32 opIdx = taskExecption_.GetOpRingBufferIdx();
6017 0 : CHK_RET(SetDispatcherCtxOpIdx(opIdx));
6018 0 : HCCL_INFO("%s success, group[%s], opRingBufferId[%u]", __func__, identifier_.c_str(), opIdx);
6019 0 : return HCCL_SUCCESS;
6020 : }
6021 :
6022 0 : HcclResult HcclCommAicpu::RegOpTaskException(HcommGetOpInfoCallback callback)
6023 : {
6024 0 : CHK_RET(taskExecption_.RegisterOpInfoCallback(callback));
6025 0 : return HCCL_SUCCESS;
6026 : }
6027 :
6028 0 : HcclResult HcclCommAicpu::SetDispatcherCtxOnThread()
6029 : {
6030 : // 设置 DispatcherCtx 到线程变量
6031 0 : CHK_RET(SetDispatcherCtx(dispatcherCtx_));
6032 0 : return HCCL_SUCCESS;
6033 : }
6034 : } // namespace hccl
|
