Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include <memory>
12 : #include <atomic>
13 : #include <chrono>
14 : #include <thread>
15 : #include <algorithm>
16 : #include <numeric>
17 : #include <unordered_set>
18 : #include <sys/time.h>
19 : #include "externalinput_pub.h"
20 : #include "env_config.h"
21 : #include "p2p_mgmt_pub.h"
22 : #include "opexecounter_pub.h"
23 : #include "config.h"
24 : #include "stream_active_manager.h"
25 : #include "device_capacity.h"
26 : #include "profiling_manager_pub.h"
27 : #include "task_exception_handler_pub.h"
28 : #include "rank_consistentcy_checker.h"
29 : #include "hccl_aiv.h"
30 : #include "adapter_rts_common.h"
31 : #include "coll_alg_utils.h"
32 : #include "../common/src/state_guard.h"
33 : #include "detect_connect_anomalies.h"
34 : #include "alg_profiling.h"
35 : #include "mmpa_api.h"
36 : #include "stream_utils.h"
37 : #include "config_log.h"
38 : #include "../nslbdp/hccl_nslbdp.h"
39 : #include "../common/src/h2d_tlv/hccl_h2dtlv.h"
40 : #include "hccl_one_sided_service.h"
41 : #include "launch_device.h"
42 : #include "hccl_communicator.h"
43 : #include "launch_aicpu.h"
44 : #include "order_launch/order_launch.h"
45 : #include "comm_configer.h"
46 : #include "snapshot_control.h"
47 : #include "comm_topo_desc.h"
48 : #include "hccl_net_dev_defs.h"
49 : #include "aclgraph_callback.h"
50 :
51 : using namespace std;
52 : constexpr u32 MODULE_NUM_FOUR = 4;
53 : constexpr u16 MAX_VALUE_U16 = 0xFFFF;
54 :
55 : namespace hccl {
56 : static std::mutex g_hcomInitMutex;
57 : static std::atomic<u32> g_enableBackupLinkCommCount{0}; // 开启借轨的通信域计数
58 : constexpr u32 MEMORY_CAPACITY = 256 * 1024;
59 : constexpr u32 WAIT_PREPARE_SLEEP_TIME = 5000;
60 : constexpr u32 SINGLE_SERVER_NUM = 1;
61 : constexpr u32 CONN_LIMIT = 4096;
62 : constexpr u32 COMM_DEV_TYPE_DIGIT_NUM = 8;
63 : constexpr u32 TILINGDATA_BUF_SIZE = 32 * 1024; // 单位：字节
64 : constexpr u32 ALLTOALL_INFO_MATRIX_SIZE = 4;
65 : constexpr u32 AICPU_RETRY_LINKROCE_DEFAULT = 0;
66 : constexpr u32 AICPU_RETRY_LINKROCE_BACKUP = 1;
67 : constexpr u32 SINGLE_PROCESS_MIN_PORT = 1024;
68 : constexpr u32 SINGLE_PROCESS_MAX_PORT = 65535;
69 : constexpr u32 TYPE_USER_MEM = 1;
70 : constexpr u32 NON_BATCH_WRITE_MAX_STREAM_NUM = 19U;
71 : constexpr u64 GIGABYTE_TO_BYTE = 1024ULL * 1024ULL * 1024ULL;
72 : constexpr u8 AICPU_ORDERLAUNCH_INVALID_HCOM_MODE = 255; // 图模式下无附属从流，不进行按序下发
73 : enum TransferMemInfoIdx {
74 : TRANSFER_MEM_INFO_KEY_IDX = 0,
75 : TRANSFER_MEM_INFO_VALUE_IDX = 1,
76 : TRANSFER_MEM_INFO_RDMA_ENVELOPE_IDX = 2,
77 : TRANSFER_MEM_INFO_IDX_NUM = 3
78 : };
79 :
80 : enum class AicpuLocalNotifyIdx : u32 {
81 : // host-aicpu同步
82 : HOST_TO_AICPU_0 = 0,
83 : HOST_TO_AICPU_1 = 1,
84 :
85 : // 用于控制单算子模式各通信域kernel按序占核的notify
86 : ORDER_INDEX_OPBASE_0 = 2, // host_order流 record, kernel流 wait
87 : ORDER_INDEX_OPBASE_1 = 3, // aicpu_order流 record, host_order流 wait
88 :
89 : // 用于控制Aclgraph模式各通信域kernel按序占核的notify
90 : ORDER_INDEX_ACLGRAPH_0 = 4, // host_order流 record, kernel流 wait
91 : ORDER_INDEX_ACLGRAPH_1 = 5, // aicpu_order流 record, host_order流 wait
92 :
93 : // 用于控制图模式各通信域kernel按序占核的notify
94 : ORDER_INDEX_HCOM_0 = 6, // host_order流 record, kernel流 wait
95 : ORDER_INDEX_HCOM_1 = 7 // aicpu_order流 record, host_order流 wait
96 : };
97 :
98 : enum class AicpuLocalEventIdx : u32 {
99 : /**
100 : *@brief 用于控制Aclgraph模式按序下发控制流入图的event
101 : *@note
102 : *通信域绑定Context，而Stream是Context管理的资源，因此对应下在Stream上的event与通信域强相关，需要communicator管理
103 : **/
104 : ORDER_INDEX_ACLGRAPH_EVENT_0 = 0, // kernel流 record, host_order流 wait
105 : ORDER_INDEX_ACLGRAPH_EVENT_1 = 1, // host_order流 record, kernel流 wait
106 : };
107 :
108 393 : HcclCommunicator::HcclCommunicator()
109 393 : : dispatcher_(nullptr),
110 393 : vDispatcher_(nullptr),
111 393 : notifyPool_(nullptr),
112 393 : initializedFlag_(ATOMIC_FLAG_INIT),
113 393 : userRank_(INVALID_VALUE_RANKID),
114 393 : realUserRank_(INVALID_VALUE_RANKID),
115 393 : userRankSize_(INVALID_VALUE_RANKSIZE),
116 393 : drvInit_(false),
117 393 : inlineReduceSwitchOn_(true),
118 393 : nicDeployment_(NICDeployment::NIC_DEPLOYMENT_DEVICE),
119 393 : devicePhyId_(INVALID_UINT),
120 393 : deviceLogicId_(-1),
121 393 : localRank_(INVALID_VALUE_RANKID),
122 393 : hostSocketHandle_(nullptr),
123 393 : isUsedRdmaLevel0_(false),
124 393 : nicInitialized_(0),
125 393 : hcomGroupNicInit_(false),
126 393 : profilingMode_(HcomProfilingMode::PROFILING_CLOSE),
127 393 : raResourceInit_(false),
128 393 : interServer_(false),
129 393 : isSingleMeshAggregation_(false),
130 393 : cclBufferManager_(CCLBufferManager()),
131 393 : isExecuteProfilingInit_(false),
132 393 : deviceType_(DevType::DEV_TYPE_COUNT),
133 393 : commHandle_(nullptr),
134 393 : commWorkMode_(WorkMode::HCCL_MODE_NORMAL),
135 393 : meshAggregationRankSize_(0),
136 393 : isHaveCpuRank_(false),
137 393 : ranktableCrc_(0),
138 393 : multiModuleDiffDeviceNumMode_(false),
139 393 : multiSuperPodDiffServerNumMode_(false),
140 393 : multiSuperPodDiffDeviceNumMode_(false),
141 393 : isStandardCard_(false),
142 393 : is310PDuoCard_(false),
143 393 : hccsPortNum_(-1),
144 393 : loopBackIp_(HcclIpAddress(COMM_LOOPBACK_IP)),
145 393 : profilingInitiated_(false),
146 393 : callbackThreadId_(INVALID_U64),
147 393 : role_(SERVER_ROLE_SOCKET),
148 393 : isHostUseDevNic_(false),
149 393 : isAllRankSamePlane_(false),
150 393 : serverNum_(0),
151 303003 : moduleNum_(0)
152 : {
153 393 : zeroCopyAclGraph_.reset(new (std::nothrow) ZeroCopyAclGraph());
154 393 : if (zeroCopyAclGraph_ == nullptr) {
155 0 : HCCL_ERROR("new ZeroCopyAclGraph failed!");
156 : }
157 393 : commConfig_ = CommConfig();
158 393 : dpuManager_.reset(new (std::nothrow) DpuManager());
159 393 : if (dpuManager_ == nullptr) {
160 0 : HCCL_ERROR("new DpuManager failed!");
161 : }
162 393 : }
163 :
164 414 : HcclCommunicator::HcclCommunicator(const CommConfig& commConfig)
165 414 : : dispatcher_(nullptr),
166 414 : vDispatcher_(nullptr),
167 414 : notifyPool_(nullptr),
168 414 : initializedFlag_(ATOMIC_FLAG_INIT),
169 414 : userRank_(INVALID_VALUE_RANKID),
170 414 : realUserRank_(INVALID_VALUE_RANKID),
171 414 : userRankSize_(INVALID_VALUE_RANKSIZE),
172 414 : drvInit_(false),
173 414 : inlineReduceSwitchOn_(true),
174 414 : nicDeployment_(NICDeployment::NIC_DEPLOYMENT_DEVICE),
175 414 : devicePhyId_(INVALID_UINT),
176 414 : deviceLogicId_(-1),
177 414 : localRank_(INVALID_VALUE_RANKID),
178 414 : hostSocketHandle_(nullptr),
179 414 : isUsedRdmaLevel0_(false),
180 414 : nicInitialized_(0),
181 414 : hcomGroupNicInit_(false),
182 414 : profilingMode_(HcomProfilingMode::PROFILING_CLOSE),
183 414 : raResourceInit_(false),
184 414 : interServer_(false),
185 414 : isSingleMeshAggregation_(false),
186 414 : cclBufferManager_(CCLBufferManager()),
187 414 : isExecuteProfilingInit_(false),
188 414 : deviceType_(DevType::DEV_TYPE_COUNT),
189 414 : commHandle_(nullptr),
190 414 : commWorkMode_(WorkMode::HCCL_MODE_NORMAL),
191 414 : meshAggregationRankSize_(0),
192 414 : isHaveCpuRank_(false),
193 414 : ranktableCrc_(0),
194 414 : multiModuleDiffDeviceNumMode_(false),
195 414 : multiSuperPodDiffServerNumMode_(false),
196 414 : isStandardCard_(false),
197 414 : is310PDuoCard_(false),
198 414 : hccsPortNum_(-1),
199 414 : loopBackIp_(HcclIpAddress(COMM_LOOPBACK_IP)),
200 414 : profilingInitiated_(false),
201 414 : callbackThreadId_(INVALID_U64),
202 414 : role_(SERVER_ROLE_SOCKET),
203 414 : isHostUseDevNic_(false),
204 414 : isAllRankSamePlane_(false),
205 414 : serverNum_(0),
206 319194 : moduleNum_(0)
207 : {
208 414 : zeroCopyAclGraph_.reset(new (std::nothrow) ZeroCopyAclGraph());
209 414 : if (zeroCopyAclGraph_ == nullptr) {
210 0 : HCCL_ERROR("new ZeroCopyAclGraph failed!");
211 : }
212 414 : commConfig_ = commConfig;
213 413 : dpuManager_.reset(new (std::nothrow) DpuManager());
214 414 : if (dpuManager_ == nullptr) {
215 0 : HCCL_ERROR("new DpuManager failed!");
216 : }
217 414 : }
218 :
219 7798 : HcclCommunicator::~HcclCommunicator()
220 : {
221 807 : HCCL_DEBUG("Enter ~HcclCommunicator.");
222 :
223 807 : DeinitZeroCopyMemoryAgent(true);
224 807 : if (!isInvalidComm_) {
225 807 : (void)DestroyAicpuComm();
226 807 : (void)UnRegisterBackGroundThread();
227 : } else {
228 0 : HCCL_WARNING(
229 : "The comm[%s] is invalid in snapshot, rank[%u]. deviceLogicId[%u]. "
230 : "There is no aicpu comm in device, skip aicpu comm destroy in destructor.",
231 : identifier_.c_str(), userRank_, deviceLogicId_);
232 : }
233 :
234 807 : if (dpuManager_ != nullptr) {
235 807 : (void)dpuManager_->DeInitDpuKernel();
236 807 : dpuManager_ = nullptr;
237 : }
238 :
239 806 : UnRegisterToHeartBeat();
240 806 : DeleteOpInfoToHeartBeat();
241 806 : AlgWrap::GetInstance().UnregisterAlgCallBack(identifier_);
242 807 : DetectConnectionAnomalies::GetInstance(deviceLogicId_).Deinit();
243 807 : UnRegisterToCommConfiger();
244 807 : AclgraphCallback::GetInstance().CleanCaptureRes(this);
245 :
246 807 : if (zeroCopyAclGraph_ != nullptr) {
247 807 : zeroCopyAclGraph_ = nullptr;
248 : }
249 :
250 806 : if (implAlg_ != nullptr) {
251 526 : implAlg_ = nullptr;
252 : }
253 :
254 881 : for (auto& res : resMap_) {
255 74 : DestroyAlgResource(res.second);
256 : }
257 :
258 807 : if (releaseChannel_ != nullptr) {
259 404 : releaseChannel_();
260 : }
261 :
262 806 : if (opRetryManager_ != nullptr) {
263 0 : OpRetryManager::DeleteLinkInfoByIdentifier(deviceLogicId_, identifier_);
264 0 : opRetryManager_->UnRegisterOpRetryManager(identifier_);
265 0 : opRetryManager_ = nullptr;
266 : }
267 :
268 807 : if (IsEnableBackupLink()) {
269 0 : if (g_enableBackupLinkCommCount.load() == 0) {
270 0 : HCCL_ERROR("[Destroy] g_enableBackupLinkCommCount is 0");
271 : } else {
272 0 : g_enableBackupLinkCommCount--;
273 : }
274 : }
275 :
276 807 : resMap_.clear();
277 806 : deviceResOrigMem_.clear();
278 807 : hostResMap_.clear();
279 807 : tagCommInfo_.clear();
280 807 : tagWorkSpaceMem_.clear();
281 807 : tagStreamInfo_.clear();
282 :
283 806 : if (opRetryStreamPtr_ != nullptr) {
284 523 : opRetryStreamPtr_->clear();
285 522 : opRetryStreamPtr_ = nullptr;
286 : }
287 :
288 807 : OrderLaunch::GetInstance(deviceLogicId_).UnRegisterOrderLaunch(identifier_);
289 2421 : for (u32 i = 0; i < AICPU_LOCAL_EVENT_SIZE; ++i) {
290 1614 : if (localAicpuOpEvent_[i] != nullptr) {
291 0 : (void)hrtEventDestroy(localAicpuOpEvent_[i]);
292 0 : localAicpuOpEvent_[i] = nullptr;
293 : }
294 : }
295 :
296 807 : (void)UnRegistTaskExceptionHandler();
297 806 : for (auto streamId : aicpuStreamIds_) {
298 0 : UnregisterGetAicpuTaskExceptionCallBack(streamId, deviceLogicId_);
299 : }
300 807 : aicpuStreamIds_.clear();
301 805 : kfcControlTransferH2D_ = nullptr;
302 807 : kfcStatusTransferD2H_ = nullptr;
303 807 : customControlTransferH2D_ = nullptr;
304 807 : customStatusTransferD2H_ = nullptr;
305 :
306 807 : oneSideService_ = nullptr;
307 807 : if (isOneSidedServiceNetDevCtxInited) {
308 0 : DeInitOneSidedServiceNetDevCtx();
309 : }
310 :
311 : /* 网络资源销毁 */
312 805 : DestroyNetworkResources();
313 807 : notifyPool_ = nullptr;
314 807 : queueNotifyManager_ = nullptr;
315 : /* driver关联资源释放 */
316 807 : if (drvInit_) {
317 307 : if (DisablePreResource() != HCCL_SUCCESS) {
318 0 : HCCL_WARNING("driver resource is not released successfully");
319 : }
320 : }
321 :
322 807 : if (isExecuteProfilingInit_) {
323 491 : (void)DeinitProfiling();
324 : }
325 :
326 807 : if (OpExeCounter::GetInstance(deviceLogicId_).DeInitCounter() != HCCL_SUCCESS) {
327 0 : HCCL_WARNING("op exec counter resource free failed");
328 : }
329 :
330 : /* 销毁当前trace句柄 */
331 807 : if (opBaseAtraceInfo_ != nullptr) {
332 524 : opBaseAtraceInfo_->DeInit();
333 524 : opBaseAtraceInfo_ = nullptr;
334 : }
335 :
336 807 : ReleaseWorkSpacebuffer();
337 807 : ReleaseCommContextbuffer();
338 :
339 7263 : for (u32 i = 0; i < AICPU_LOCAL_NOTIFY_SIZE; i++) {
340 6456 : if (localAiCpuOpNotify_[i]) {
341 0 : HcclResult ret = localAiCpuOpNotify_[i]->Destroy();
342 0 : localAiCpuOpNotify_[i] = nullptr;
343 0 : if (ret != RT_ERROR_NONE) {
344 0 : HCCL_ERROR(
345 : "[Destroy][AicpuNotify]errNo[0x%016llx] rt notify destroy fail, "
346 : "aicpuOpNotify[%u] return[%d].",
347 : HCCL_ERROR_CODE(HCCL_E_RUNTIME), i, ret);
348 : }
349 : }
350 : }
351 :
352 808 : while (!aiCpuNoIpcEvnet_.empty()) {
353 1 : rtEvent_t eventInfo = aiCpuNoIpcEvnet_.back();
354 1 : HcclResult ret = hrtEventDestroy(eventInfo);
355 1 : if (ret != HCCL_SUCCESS) {
356 0 : HCCL_ERROR(
357 : "[Destroy][AicpuNoIpcEvnet]errNo[0x%016llx] rt event destroy fail, "
358 : "return[%d].",
359 : HCCL_ERROR_CODE(HCCL_E_RUNTIME), ret);
360 : }
361 1 : aiCpuNoIpcEvnet_.pop_back();
362 : }
363 :
364 806 : UnloadAICPUKernel();
365 805 : UnloadCustomKernel();
366 807 : if (dispatcher_ != nullptr) {
367 524 : HcclDispatcherDestroy(dispatcher_);
368 522 : dispatcher_ = nullptr;
369 : }
370 805 : if (dispatcherCtx_ != nullptr) {
371 522 : DestroyDispatcherCtx(dispatcherCtx_, identifier_.c_str());
372 524 : dispatcherCtx_ = nullptr;
373 : }
374 807 : if (vDispatcher_ != nullptr) {
375 524 : HcclDispatcherDestroy(vDispatcher_);
376 524 : vDispatcher_ = nullptr;
377 : }
378 807 : if (deviceType_ == DevType::DEV_TYPE_910B || deviceType_ == DevType::DEV_TYPE_910_93) {
379 214 : UnRegisterFromSnapshot();
380 : }
381 807 : HCCL_DEBUG("~HcclCommunicator success.");
382 9408 : }
383 :
384 523 : HcclResult HcclCommunicator::SaveTopoDesc(std::string& identifier)
385 : {
386 523 : CommTopo topoType = CommTopo::COMM_TOPO_RESERVED;
387 523 : CHK_RET(GetInstTopoTypeByNetLayer(0, &topoType)); // layer 0
388 :
389 523 : CommTopoDesc::GetInstance().SaveRankSize(identifier, userRankSize_);
390 523 : CommTopoDesc::GetInstance().SaveL0TopoType(identifier, topoType);
391 523 : return HCCL_SUCCESS;
392 : }
393 :
394 501 : HcclResult HcclCommunicator::Init(HcclCommParams& params, const RankTable_t& rankTable)
395 : {
396 501 : CHK_RET(InitCommParams(params));
397 501 : CHK_RET(attrCollector_.Init(params, rankTable, commConfig_.GetConfigHcclAlgoMap()));
398 498 : CHK_RET(InitRankInfo(rankTable));
399 498 : CHK_RET(InitNetResource(rankTable));
400 491 : CHK_RET(InitDebug());
401 491 : CHK_RET(InitNotifyManager());
402 491 : CHK_RET(InitStreamManager());
403 491 : CHK_RET(InitProfiler());
404 491 : CHK_RET(InitDispatcher());
405 491 : CHK_RET(InitTransportManager());
406 491 : CHK_RET(InitCombinOpara());
407 491 : CHK_RET(RegisterRanksToDca());
408 : /*--------------加锁区--------------*/
409 491 : std::unique_lock<std::mutex> lock(g_hcomInitMutex);
410 491 : CHK_RET(RegistTaskExceptionHandler());
411 :
412 491 : attrCollector_.GenCollectiveId(params, rankTable);
413 491 : collectiveId_ = attrCollector_.GetCollectiveId();
414 :
415 : // 初始化参数(需要放置在ranktable解析之后)
416 491 : HcclResult ret = InitPara();
417 491 : CHK_PRT_RET(
418 : ret != HCCL_SUCCESS,
419 : HCCL_ERROR(
420 : "[HcclCommunicator][Init]errNo[0x%016llx] collectiveid[%s] parameter initialization failed",
421 : HCCL_ERROR_CODE(ret), params.id.internal),
422 : ret);
423 491 : lock.unlock();
424 : /*--------------加锁区--------------*/
425 491 : if (deviceType_ == DevType::DEV_TYPE_910B || deviceType_ == DevType::DEV_TYPE_910_93) {
426 198 : CHK_RET(RegisterKernel(deviceType_));
427 : }
428 491 : CHK_RET(LoadCustomKernel());
429 491 : CHK_RET(LoadAICPUKernel());
430 491 : CHK_RET(InitHDCommunicate());
431 491 : CHK_RET(InitOpRetry());
432 491 : CHK_RET(InitOpResPara());
433 :
434 491 : CHK_RET(InitOneSidedService(rankTable));
435 491 : CHK_RET(OrderLaunch::GetInstance(deviceLogicId_).RegisterOrderLaunch(identifier_));
436 491 : HcclTopoAttr topoAttr;
437 491 : attrCollector_.GetTopoAttr(topoAttr);
438 491 : CHK_RET(rankGraph_.Init(rankTable, topoAttr));
439 491 : CHK_RET(SaveTopoDesc(params.identifier));
440 491 : CHK_RET(RegisterToSnapshot());
441 491 : CHK_RET(InitSymmetricMemory());
442 :
443 491 : CHK_RET(InitMyRankConnectMode(params, rankTable));
444 491 : if (dpuManager_ != nullptr && myRankConnectMode_) { /* 当前只有host nic--device nic使用 */
445 0 : CHK_RET(dpuManager_->Init(identifier_, deviceLogicId_));
446 : }
447 :
448 491 : return HCCL_SUCCESS;
449 491 : }
450 :
451 : HcclResult
452 33 : HcclCommunicator::Init(HcclCommParams& params, const std::vector<RankInfo>& rankList, WorldGroupInfo& groupCommonData)
453 : {
454 33 : CHK_RET(InitCommParams(params));
455 33 : CHK_RET(attrCollector_.Init(params, rankList, groupCommonData, commConfig_.GetConfigHcclAlgoMap()));
456 33 : CHK_RET(InitRankInfoSubGroup(groupCommonData));
457 33 : CHK_RET(InitDebugSubGroup());
458 33 : CHK_RET(InitNotifyManager());
459 33 : CHK_RET(InitDispatcher());
460 33 : CHK_RET(InitStreamManager());
461 33 : CHK_RET(InitRaResource());
462 33 : CHK_RET(InitTransportManager());
463 33 : CHK_RET(InitHcclAlg());
464 32 : CHK_RET(LoadCustomKernel());
465 32 : CHK_RET(LoadAICPUKernel());
466 32 : CHK_RET(InitHDCommunicate());
467 32 : CHK_RET(InitOpRetry());
468 32 : CHK_RET(InitOpResPara());
469 32 : CHK_RET(RegisterRanksToDca());
470 32 : CHK_RET(OrderLaunch::GetInstance(deviceLogicId_).RegisterOrderLaunch(identifier_));
471 32 : HcclTopoAttr topoAttr;
472 32 : attrCollector_.GetTopoAttr(topoAttr);
473 32 : CHK_RET(rankGraph_.Init(topoAttr));
474 32 : CHK_RET(SaveTopoDesc(params.identifier));
475 32 : CHK_RET(RegisterToSnapshot());
476 32 : CHK_RET(InitSymmetricMemory());
477 32 : return HCCL_SUCCESS;
478 32 : }
479 :
480 523 : HcclResult HcclCommunicator::LoadAICPUKernel(void)
481 : {
482 523 : if (binHandle_ == nullptr) {
483 523 : std::string jsonPath;
484 523 : CHK_RET(GetKernelFilePath(jsonPath));
485 523 : jsonPath += "ccl_kernel.json";
486 523 : HcclResult ret = LoadBinaryFromFile(jsonPath.c_str(), ACL_RT_BINARY_LOAD_OPT_CPU_KERNEL_MODE, 0, binHandle_);
487 523 : CHK_PRT_RET(
488 : ret != HCCL_SUCCESS,
489 : HCCL_ERROR(
490 : "[LoadAICPUKernel]errNo[0x%016llx]load aicpu file fail, path[%s] optionType[%u]"
491 : "cpuKernelMode[%u].",
492 : ret, jsonPath.c_str(), ACL_RT_BINARY_LOAD_OPT_CPU_KERNEL_MODE, 0),
493 : ret);
494 523 : }
495 523 : return HCCL_SUCCESS;
496 : }
497 :
498 807 : void HcclCommunicator::UnloadAICPUKernel(void)
499 : {
500 807 : if (binHandle_ != nullptr) {
501 3 : aclError aclRet = aclrtBinaryUnLoad(binHandle_);
502 3 : if (aclRet != ACL_SUCCESS) {
503 0 : HCCL_ERROR(
504 : "[UnloadAICPUKernel]errNo[0x%016llx] unload binary from binHandel[%p] error.", aclRet, binHandle_);
505 : }
506 3 : binHandle_ = nullptr;
507 : }
508 807 : return;
509 : }
510 :
511 523 : HcclResult HcclCommunicator::LoadCustomKernel(void)
512 : {
513 : // 加载自定义算子
514 : // 请勿删除，该函数为用户自定义算子时使用，应加载句柄
515 : // 读取customEnable环境变量，开启了就执行
516 523 : std::string jsonPath;
517 523 : CHK_RET(GetCustomKernelFilePath(jsonPath));
518 523 : jsonPath += "libaicpu_custom.json";
519 523 : CHK_RET(LoadCustomFile(jsonPath.c_str(), ACL_RT_BINARY_LOAD_OPT_CPU_KERNEL_MODE, 1, binHandle_));
520 523 : return HCCL_SUCCESS;
521 523 : }
522 :
523 807 : void HcclCommunicator::UnloadCustomKernel(void)
524 : {
525 : // 卸载自定义算子
526 : // 请勿删除，该函数为用户自定义算子时使用，应释放句柄：UnloadBinary(binCustomHandle_);
527 807 : return;
528 : }
529 :
530 491 : HcclResult HcclCommunicator::InitOneSidedService(const RankTable_t& rankTable)
531 : {
532 491 : EXCEPTION_CATCH(
533 : (oneSideService_ = std::make_unique<HcclOneSidedService>(socketManager_, notifyPool_, commConfig_)),
534 : return HCCL_E_INTERNAL);
535 491 : hcclRankLinkInfo_.userRank = userRank_;
536 491 : hcclRankLinkInfo_.devicePhyId = devicePhyId_;
537 :
538 491 : if (devIpAddr_.empty()) {
539 0 : HCCL_ERROR("[%s] device ip is invalid, please set device ip first.", __func__);
540 0 : return HCCL_E_NOT_FOUND;
541 : }
542 491 : hcclRankLinkInfo_.ip = devIpAddr_[0];
543 491 : if (nicRanksPort_.size() <= userRank_) {
544 0 : HCCL_ERROR("[%s] userRank_[%u] port is invalid, please set port first", __func__, userRank_);
545 0 : return HCCL_E_NOT_FOUND;
546 : }
547 491 : hcclRankLinkInfo_.port = nicRanksPort_[userRank_];
548 491 : hcclRankLinkInfo_.socketsPerLink = 1;
549 491 : HCCL_DEBUG(
550 : "[%s]hcclRankLinkInfo_ userRank[%u], devicePhyId[%u], ip[%s], port[%u]", __func__, hcclRankLinkInfo_.userRank,
551 : hcclRankLinkInfo_.devicePhyId, hcclRankLinkInfo_.ip.GetReadableIP(), hcclRankLinkInfo_.port);
552 491 : CHK_RET(oneSideService_->Config(
553 : dispatcher_, hcclRankLinkInfo_, &rankTable, identifier_, isStandardCard_, enableP2PRankIds_));
554 491 : return HCCL_SUCCESS;
555 : }
556 :
557 0 : HcclResult HcclCommunicator::InitOneSidedServiceNetDevCtx(u32 remoteRankId)
558 : {
559 0 : if (nicDeployment_ != NICDeployment::NIC_DEPLOYMENT_DEVICE) {
560 : // 单边操作当前只支持Device网卡，不支持host
561 0 : HCCL_ERROR(
562 : "[%s]nicDeployment_[%d], userRankSize_[%u], do not support oneSidedService.", __func__, nicDeployment_,
563 : userRankSize_);
564 0 : return HCCL_E_INTERNAL;
565 : }
566 :
567 0 : std::string localServerId = serverId_;
568 0 : std::string localSuperPodId = superPodId_;
569 0 : std::string remoteServerId = rankInfoList_.at(remoteRankId).serverId;
570 0 : std::string remoteSuperPodId = rankInfoList_.at(remoteRankId).superPodId;
571 0 : u32 intraRoceSwitch = GetExternalInputIntraRoceSwitch();
572 0 : bool useRdma = false;
573 0 : if (intraRoceSwitch || (!useSuperPodMode_ && localServerId != remoteServerId)
574 0 : || (localSuperPodId != remoteSuperPodId)) {
575 : // 1. 初始化网口
576 0 : CHK_RET(InitNic());
577 0 : isOneSidedServiceNicInited = true;
578 :
579 : // 2. 单边操作SetNetDevCtx, RDMA
580 0 : if (netDevCtxMap_.find(devIpAddr_[0]) == netDevCtxMap_.end()) {
581 0 : HCCL_ERROR("[%s] nicDeployment_[%d], device nic init fail, please check", __func__, nicDeployment_);
582 0 : return HCCL_E_NOT_FOUND;
583 : }
584 0 : useRdma = true;
585 0 : oneSideService_->SetNetDevCtx(netDevCtxMap_[devIpAddr_[0]], useRdma);
586 0 : HCCL_INFO("[%s]init device Nic for oneSidedService success.", __func__);
587 : } else {
588 : // 单边操作SetNetDevCtx, IPC
589 0 : oneSideService_->SetNetDevCtx(netDevCtxMap_[localVnicIp_], useRdma);
590 0 : HCCL_INFO("[%s]init vNic for oneSidedService success.", __func__);
591 : }
592 0 : isOneSidedServiceNetDevCtxInited = true;
593 0 : HCCL_DEBUG("[%s]nicDeployment_[%d], intraRoceSwitch[%u]", __func__, nicDeployment_, intraRoceSwitch);
594 0 : return HCCL_SUCCESS;
595 0 : }
596 :
597 0 : HcclResult HcclCommunicator::DeInitOneSidedServiceNetDevCtx()
598 : {
599 0 : if (nicDeployment_ != NICDeployment::NIC_DEPLOYMENT_DEVICE) {
600 : // 单边操作当前只支持Device网卡，不支持host
601 0 : HCCL_ERROR(
602 : "[%s]nicDeployment_[%d], userRankSize_[%u], do not support oneSidedService.", __func__, nicDeployment_,
603 : userRankSize_);
604 0 : return HCCL_E_INTERNAL;
605 : }
606 0 : if (isOneSidedServiceNicStartListen_) {
607 0 : socketManager_->DestroySockets();
608 0 : u32 port = GetLocalNicPort(NicType::DEVICE_NIC_TYPE);
609 0 : CHK_RET(socketManager_->ServerDeInit(onesidedServiceNicIpAddr_, port));
610 0 : isOneSidedServiceNicStartListen_ = false;
611 0 : HCCL_INFO("[HcclCommunicator][%s] DeInit socket server success.tag[%s].", __func__, identifier_.c_str());
612 : }
613 0 : u32 intraRoceSwitch = GetExternalInputIntraRoceSwitch();
614 0 : if (isOneSidedServiceNicInited) {
615 : // 1. close sockets
616 0 : if (raResourceInit_) {
617 0 : socketManager_->DestroySockets();
618 : }
619 : // 2. 去初始化网口
620 0 : CHK_RET(DeinitNic());
621 0 : isOneSidedServiceNicInited = false;
622 0 : HCCL_INFO("[%s]Deinit device Nic for oneSidedService success.", __func__);
623 : }
624 0 : isOneSidedServiceNetDevCtxInited = false;
625 0 : HCCL_DEBUG("[%s]nicDeployment_[%d], intraRoceSwitch[%u]", __func__, nicDeployment_, intraRoceSwitch);
626 0 : return HCCL_SUCCESS;
627 : }
628 :
629 0 : HcclResult HcclCommunicator::GetOneSidedService(IHcclOneSidedService** service)
630 : {
631 0 : *service = oneSideService_.get();
632 0 : return HCCL_SUCCESS;
633 : }
634 :
635 0 : HcclResult HcclCommunicator::OneSidedServiceStartListen(NicType nicType, HcclNetDevCtx netDevCtx)
636 : {
637 0 : HCCL_INFO("[HcclCommunicator][%s] Start prepare netDevCtx.", __func__);
638 0 : u32 port = GetLocalNicPort(nicType);
639 0 : CHK_RET(socketManager_->ServerInit(netDevCtx, port));
640 0 : if (nicType == NicType::DEVICE_NIC_TYPE) {
641 0 : CHK_RET(HcclNetDevGetLocalIp(netDevCtx, onesidedServiceNicIpAddr_));
642 0 : isOneSidedServiceNicStartListen_ = true;
643 : }
644 0 : isOneSidedServiceNetDevCtxInited = true;
645 0 : HCCL_INFO("[HcclCommunicator][%s] netDevCtx[%p] port[%u] server init success.", __func__, netDevCtx, port);
646 0 : return HCCL_SUCCESS;
647 : }
648 :
649 0 : HcclResult HcclCommunicator::GetOneSidedServiceDevIpAndPort(NicType nicType, HcclIpAddress& ipAddress, u32& port)
650 : {
651 0 : if (nicDeployment_ != NICDeployment::NIC_DEPLOYMENT_DEVICE) {
652 : // 单边操作当前只支持Device网卡，不支持host
653 0 : HCCL_ERROR(
654 : "[%s]nicDeployment_[%d], userRankSize_[%u], do not support oneSidedService.", __func__, nicDeployment_,
655 : userRankSize_);
656 0 : return HCCL_E_INTERNAL;
657 : }
658 0 : port = GetLocalNicPort(nicType);
659 0 : if (nicType == NicType::VNIC_TYPE) {
660 0 : ipAddress = localVnicIp_;
661 0 : HCCL_INFO("[GetOneSidedServiceDevIpAddr] vnic ipAddress[%s] get success.", ipAddress.GetReadableAddress());
662 0 : return HCCL_SUCCESS;
663 0 : } else if (nicType == NicType::DEVICE_NIC_TYPE) {
664 0 : u32 nicNum = devIpAddr_.size();
665 0 : for (u32 i = 0; i < nicNum; i++) {
666 0 : if (devIpAddr_[i].IsInvalid()) {
667 0 : HCCL_INFO("[GetOneSidedServiceDevIpAddr]nic num[%u] deviceip is invalid, total nicNum[%u]", i, nicNum);
668 0 : continue;
669 : }
670 0 : ipAddress = devIpAddr_[i];
671 0 : HCCL_INFO("[GetOneSidedServiceDevIpAddr] nic ipAddress[%s] get success.", ipAddress.GetReadableAddress());
672 0 : return HCCL_SUCCESS;
673 : }
674 : }
675 0 : HCCL_ERROR("[HcclCommunicator][%s] ipAddress get fail. tag[%s]", __func__, identifier_.c_str());
676 0 : return HCCL_E_NOT_FOUND;
677 : }
678 :
679 0 : HcclResult HcclCommunicator::DeinitOneSidedService()
680 : {
681 0 : if (oneSideService_ != nullptr) {
682 0 : CHK_RET(oneSideService_->DeInit());
683 : }
684 0 : return HCCL_SUCCESS;
685 : }
686 :
687 82 : bool HcclCommunicator::IsSupportSymmetricMemory(HcclCMDType opType, OpParam& opParam)
688 : {
689 82 : CHK_PRT_RET(symmetricMemory_ == nullptr, HCCL_DEBUG("symmetricMemory_ is a nullptr"), false);
690 14 : HCCL_INFO(
691 : "[%s] aicpuUnfold[%d], workflowMode[%d], deviceType[%d], "
692 : "deviceNumPerAggregation_[%d], multiModuleDiffDeviceNumMode_[%d], tag[%s].",
693 : __func__, opParam.aicpuUnfoldMode, GetWorkflowMode(), deviceType_, deviceNumPerAggregation_,
694 : multiModuleDiffDeviceNumMode_, opParam.tag.c_str());
695 :
696 : // 目前只支持allgather, allreduce, reducescatter
697 14 : CHK_PRT_RET(
698 : opType != HcclCMDType::HCCL_CMD_ALLGATHER && opType != HcclCMDType::HCCL_CMD_ALLREDUCE
699 : && opType != HcclCMDType::HCCL_CMD_ALLTOALL && opType != HcclCMDType::HCCL_CMD_REDUCE_SCATTER,
700 : HCCL_INFO("[%s] opType[%d] not support symmetric memory", __func__, opType), false);
701 :
702 : // 只支持aicpu展开、单算子模式、910_93芯片
703 14 : CHK_PRT_RET(
704 : !opParam.aicpuUnfoldMode,
705 : HCCL_INFO("[%s] aicpuUnfold:%d not support symmetric memory", __func__, opParam.aicpuUnfoldMode), false);
706 4 : CHK_PRT_RET(
707 : GetWorkflowMode() != HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE,
708 : HCCL_INFO("[%s] workflowMode:%d not support symmetric memory", __func__, GetWorkflowMode()), false);
709 3 : CHK_PRT_RET(
710 : deviceType_ != DevType::DEV_TYPE_910_93,
711 : HCCL_INFO("[%s] deviceType:%d not support symmetric memory", __func__, deviceType_), false);
712 3 : CHK_PRT_RET(
713 : superPodNum_ == 1 && serverNum_ > 1 && GetExternalInputInterHccsDisable(),
714 : HCCL_INFO("[%s] mutilSever use roce not support symmetric memory", __func__), false);
715 :
716 : // 判断拓扑逻辑是否支持symmetric memory
717 : // 每个节点只有一张卡或节点间非对称场景不支持对称内存
718 3 : CHK_PRT_RET(
719 : deviceNumPerAggregation_ == 1 || multiModuleDiffDeviceNumMode_,
720 : HCCL_INFO(
721 : "[%s] deviceNumPerAggregation[%u], multiModuleDiffDeviceNumMode_[%d] not support symmetric memory",
722 : __func__, deviceNumPerAggregation_, multiModuleDiffDeviceNumMode_),
723 : false);
724 :
725 : // 判断输入输出地址是否都注册为对称内存
726 2 : HcclResult ret = symmetricMemory_->FindSymmetricWindow(
727 2 : opParam.inputPtr, opParam.inputSize, &opParam.inputSymWindow, &opParam.inputOffset);
728 2 : CHK_PRT_RET(
729 : ret != HCCL_SUCCESS || opParam.inputSymWindow == nullptr,
730 : HCCL_INFO(
731 : "[%s] input[%p] size[%llu] is not support symmetric memory", __func__, opParam.inputPtr, opParam.inputSize),
732 : false);
733 1 : ret = symmetricMemory_->FindSymmetricWindow(
734 1 : opParam.outputPtr, opParam.outputSize, &opParam.outputSymWindow, &opParam.outputOffset);
735 1 : CHK_PRT_RET(
736 : ret != HCCL_SUCCESS || opParam.outputSymWindow == nullptr,
737 : HCCL_INFO(
738 : "[%s] output[%p] size[%llu] is not support symmetric memory", __func__, opParam.outputPtr,
739 : opParam.outputSize),
740 : false);
741 :
742 1 : HCCL_INFO(
743 : "[HcclCommunicator][IsSupportSymmetricMemory] opParam.inputPtr[%p], inputOffset[%llu], inputSymWindow[%p]",
744 : opParam.inputPtr, opParam.inputOffset, opParam.inputSymWindow);
745 1 : HCCL_INFO(
746 : "[HcclCommunicator][IsSupportSymmetricMemory] opParam.outputPtr[%p], outputOffset[%llu], outputSymWindow[%p]",
747 : opParam.outputPtr, opParam.outputOffset, opParam.outputSymWindow);
748 :
749 1 : return true;
750 : }
751 :
752 78 : bool HcclCommunicator::IsSupportZeroCopy(const OpParam& opParam)
753 : {
754 78 : HCCL_INFO(
755 : "[%s] aicpuUnfold[%d], workflowMode[%d], deviceType[%d], "
756 : "deviceNumPerAggregation_[%d], multiModuleDiffDeviceNumMode_[%d], tag[%s].",
757 : __func__, opParam.aicpuUnfoldMode, GetWorkflowMode(), deviceType_, deviceNumPerAggregation_,
758 : multiModuleDiffDeviceNumMode_, opParam.tag.c_str());
759 :
760 : // 只支持aicpu展开、非重执行、单算子模式、910_93芯片
761 75 : CHK_PRT_RET(
762 : !opParam.aicpuUnfoldMode,
763 : HCCL_INFO("[%s] aicpuUnfold:%d not support zero copy", __func__, opParam.aicpuUnfoldMode), false);
764 4 : CHK_PRT_RET(
765 : GetWorkflowMode() != HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE,
766 : HCCL_INFO("[%s] workflowMode:%d not support zero copy", __func__, GetWorkflowMode()), false);
767 0 : CHK_PRT_RET(
768 : deviceType_ != DevType::DEV_TYPE_910_93,
769 : HCCL_INFO("[%s] deviceType:%d not support zero copy", __func__, deviceType_), false);
770 :
771 : // 判断拓扑逻辑是否支持zero copy
772 : // 每个节点只有一张卡或节点间非对称场景不支持零拷贝
773 0 : CHK_PRT_RET(
774 : deviceNumPerAggregation_ == 1 || multiModuleDiffDeviceNumMode_,
775 : HCCL_INFO(
776 : "[%s] deviceNumPerAggregation[%u], multiModuleDiffDeviceNumMode_[%d] not support zero copy", __func__,
777 : deviceNumPerAggregation_, multiModuleDiffDeviceNumMode_),
778 : false);
779 :
780 : // 判断输入输出地址是否都是支持零Copy特性的
781 0 : CHK_PRT_RET(
782 : !ZeroCopyMemoryAgent::IsActivateCommMemoryAddr(opParam.inputPtr, opParam.inputSize),
783 : HCCL_INFO("[%s] input[%p] size[%llu] is not support zero copy", __func__, opParam.inputPtr, opParam.inputSize),
784 : false);
785 0 : CHK_PRT_RET(
786 : !ZeroCopyMemoryAgent::IsActivateCommMemoryAddr(opParam.outputPtr, opParam.outputSize),
787 : HCCL_INFO(
788 : "[%s] output[%p] size[%llu] is not support zero copy", __func__, opParam.outputPtr, opParam.outputSize),
789 : false);
790 :
791 0 : return true;
792 : }
793 :
794 72 : HcclResult HcclCommunicator::PrepareZeroCopy(const std::string& algName, const AlgDesc& algDesc, OpParam& opParam)
795 : {
796 72 : if (!algDesc.isZeroCopy) {
797 : opParam.supportSymmetricMemory
798 72 : = false; // 当前对称内存与零拷贝算法绑定，对称内存使能关闭，确保aicpu侧不走对称内存分支
799 72 : HCCL_INFO("[HcclCommunicator][PrepareZeroCopy] algName[%s] not support zerocopy.", algName.c_str());
800 76 : return HCCL_SUCCESS;
801 : }
802 :
803 0 : if (opParam.supportSymmetricMemory) {
804 0 : HCCL_INFO(
805 : "[HcclCommunicator][PrepareZeroCopy] algName[%s] symmetric memory is enabled, not use zerocopy.",
806 : algName.c_str());
807 0 : return HCCL_SUCCESS;
808 : }
809 : // ARS特性不支持零拷贝
810 0 : if ((opParam.opType == HcclCMDType::HCCL_CMD_REDUCE_SCATTER || opParam.opType == HcclCMDType::HCCL_CMD_ALLGATHER
811 0 : || opParam.opType == HcclCMDType::HCCL_CMD_ALLREDUCE)
812 0 : && deviceType_ == DevType::DEV_TYPE_910_93 && multiModuleDiffDeviceNumMode_
813 0 : && !multiSuperPodDiffDeviceNumMode_) {
814 0 : return HCCL_SUCCESS;
815 : }
816 :
817 : // 如果自己侧的共享内存没有申请，那么进行申请，并设置给transportManager，后续p2p建链时进行交换
818 0 : if (zeroCopyLocalBuffer_.ptr() == nullptr) {
819 0 : CHK_RET(DeviceMem::alloc(zeroCopyLocalBuffer_, ZERO_COPY_IPC_BUFFER_LENGTH));
820 0 : CHK_RET(hrtMemSet(zeroCopyLocalBuffer_.ptr(), zeroCopyLocalBuffer_.size(), zeroCopyLocalBuffer_.size()));
821 0 : zeroCopyIpcPtrs_[userRank_ % deviceNumPerAggregation_] = zeroCopyLocalBuffer_.ptr();
822 :
823 0 : HCCL_RUN_INFO(
824 : "[HCCL_TRACE][PrepareZeroCopy]Create ZeroCopy buffer success. buffer ptr[%p] size[%llu]",
825 : zeroCopyLocalBuffer_.ptr(), zeroCopyLocalBuffer_.size());
826 : }
827 0 : opParam.isZeroCopy = true;
828 0 : HCCL_INFO("[HcclCommunicator][PrepareZeroCopy] success to use zero copy feature");
829 0 : return HCCL_SUCCESS;
830 : }
831 :
832 23 : HcclResult HcclCommunicator::UpdateZeroCopy(const OpParam& opParam, const AlgResourceResponse& algResource)
833 : {
834 23 : if (!opParam.isZeroCopy) {
835 23 : return HCCL_SUCCESS;
836 : }
837 :
838 : // 遍历所有transport，找出里面的p2p链路对应的对端地址
839 0 : for (auto& singleSubCommTransport : algResource.opTransportResponse[COMM_LEVEL0]) {
840 0 : for (u64 i = 0; i < singleSubCommTransport.links.size(); ++i) {
841 0 : LINK link = singleSubCommTransport.links[i];
842 0 : if (link == nullptr || !singleSubCommTransport.transportRequests[i].isValid) {
843 : // 无效或者不支持的链路
844 0 : continue;
845 : }
846 :
847 : // 在使能零拷贝场景，我们使用控制面内存做OpenIpc交换，因此这里取出input即可
848 0 : u32 remoteRank = link->GetRemoteRank();
849 :
850 0 : void* remotePtr = nullptr;
851 0 : CHK_RET(link->GetRemoteMem(UserMemType::INPUT_MEM, &remotePtr));
852 0 : CHK_PRT_RET(
853 : remotePtr == nullptr, HCCL_ERROR("[BuildZeroCopyParam] invalid remotePtr[%p]", remotePtr), HCCL_E_PARA);
854 0 : CHK_PRT_RET(
855 : zeroCopyIpcPtrs_[remoteRank % deviceNumPerAggregation_] != nullptr
856 : && zeroCopyIpcPtrs_[remoteRank % deviceNumPerAggregation_] != remotePtr,
857 : HCCL_ERROR(
858 : "[BuildZeroCopyParam] zeroCopyIpcPtrs_[%u] is [%p] not equal to %p", remoteRank,
859 : zeroCopyIpcPtrs_[remoteRank % deviceNumPerAggregation_], remotePtr),
860 : HCCL_E_PARA);
861 :
862 0 : zeroCopyIpcPtrs_[remoteRank % deviceNumPerAggregation_] = remotePtr;
863 0 : }
864 : }
865 0 : return HCCL_SUCCESS;
866 : }
867 :
868 3 : HcclResult HcclCommunicator::BuildZeroCopyParam()
869 : {
870 : // 不支持ZeroCopy
871 3 : if (zeroCopyLocalBuffer_.ptr() == nullptr) {
872 1 : return HCCL_SUCCESS;
873 : }
874 :
875 66 : for (u32 i = 0; i < AICPU_ZERO_COPY_MAX_DEVICE_NUM_A3; ++i) {
876 64 : opResPara_.zeroCopyIpcPtrs[i] = reinterpret_cast<u64>(zeroCopyIpcPtrs_[i]);
877 : }
878 :
879 2 : for (u32 i = 0; i < rankInfoList_.size(); ++i) {
880 0 : opResPara_.zeroCopyDevicePhyId[i % deviceNumPerAggregation_] = rankInfoList_[i].devicePhyId;
881 : }
882 :
883 2 : CHK_RET(ZeroCopyMemoryAgent::GetRingBufferAddr(
884 : opResPara_.zeroCopyRingBuffer, opResPara_.zeroCopyHeadPtr, opResPara_.zeroCopyTailPtr));
885 1 : return HCCL_SUCCESS;
886 : }
887 :
888 534 : HcclResult HcclCommunicator::InitCommParams(HcclCommParams& params)
889 : {
890 534 : commHandle_ = params.commHandle;
891 534 : userRank_ = params.rank;
892 534 : realUserRank_ = params.userRank;
893 534 : userRankSize_ = params.totalRanks;
894 534 : deviceLogicId_ = params.logicDevId;
895 534 : profilingOption_ = params.profilingOption;
896 534 : profilingInitiated_ = params.profilingInitiated;
897 534 : deviceType_ = params.deviceType;
898 534 : commWorkMode_ = params.commWorkMode;
899 534 : hcomGroupNicInit_ = params.hcomGroupNicInit;
900 534 : identifier_ = params.identifier;
901 534 : collectiveId_ = params.id.internal;
902 534 : ranktableCrc_ = params.ranktableCrc;
903 534 : commConnections_ = params.commConnections;
904 534 : commPortConfig_ = params.commPortConfig;
905 534 : cclBuffName_ = params.cclBuffName;
906 534 : isShareComm_ = !cclBuffName_.empty();
907 :
908 534 : HCCL_DEBUG(
909 : " userRank_: %u realUserRank_: %u userRankSize_: %u deviceLogicId_: %u deviceType_: %u commWorkMode_: %u.",
910 : userRank_, realUserRank_, userRankSize_, deviceLogicId_, deviceType_, commWorkMode_);
911 :
912 534 : return HCCL_SUCCESS;
913 : }
914 :
915 0 : bool HcclCommunicator::Is310PDuoCard()
916 : {
917 : return (
918 0 : Is310P3Common(isHaveCpuRank_, deviceType_)
919 0 : && (pairLinkInfo_[static_cast<u32>(LinkTypeInServer::HCCS_TYPE)].size() == userRankSize_));
920 : }
921 :
922 : // 910B A+X 在RDMA未启用情况下，两模块间的device数目需要一致且两模块中使用的卡都在同一平面上
923 498 : HcclResult HcclCommunicator::CheckSingleServerComm(const std::vector<RankInfo_t>& rankList) const
924 : {
925 396 : if (serverNum_ == 1 && moduleNum_ == HCCL_MODULE_NUM_TWO && GetExternalInputIntraRoceSwitch() == 0
926 894 : && !isStandardCard_) {
927 0 : std::vector<u32> devIdList0;
928 0 : std::vector<u32> devIdList1;
929 0 : for (RankInfo_t rankInfo : rankList) {
930 0 : if (rankInfo.deviceInfo.devicePhyId == HOST_DEVICE_ID) {
931 0 : HCCL_ERROR("[Check][SingleServerComm]not support cpu rank");
932 0 : return HCCL_E_NOT_SUPPORT;
933 : }
934 0 : if (rankInfo.deviceInfo.devicePhyId < DEVICE_PER_MODULE) {
935 0 : devIdList0.push_back(rankInfo.deviceInfo.devicePhyId);
936 : } else {
937 0 : devIdList1.push_back(rankInfo.deviceInfo.devicePhyId);
938 : }
939 0 : }
940 0 : std::sort(devIdList0.begin(), devIdList0.end());
941 0 : std::sort(devIdList1.begin(), devIdList1.end());
942 :
943 0 : auto buildDeviceListStr = [](const std::vector<u32>& list) -> std::string {
944 0 : std::string result;
945 0 : for (const auto& id : list) {
946 0 : if (!result.empty()) {
947 0 : result += " ";
948 : }
949 0 : result += std::to_string(id);
950 : }
951 0 : return result;
952 0 : };
953 :
954 0 : std::string devList0Str = buildDeviceListStr(devIdList0);
955 0 : std::string devList1Str = buildDeviceListStr(devIdList1);
956 :
957 0 : if (devIdList0.size() != devIdList1.size()) {
958 0 : std::string errormessage = "Device ID " + devList0Str + " in module 0 and device ID " + devList1Str
959 0 : + " in module 1 are not on the same plane.";
960 0 : RPT_INPUT_ERR(
961 : true, "EI0010", std::vector<std::string>({"reason"}), std::vector<std::string>({errormessage}));
962 0 : HCCL_ERROR(
963 : "[%s][%s]%s", LOG_KEYWORDS_INIT_CHANNEL.c_str(), LOG_KEYWORDS_TIMEOUT.c_str(), errormessage.c_str());
964 0 : return HCCL_E_NOT_SUPPORT;
965 0 : }
966 0 : for (size_t i = 0; i < devIdList0.size(); i++) {
967 0 : if (devIdList0[i] % DEVICE_PER_MODULE != devIdList1[i] % DEVICE_PER_MODULE) {
968 0 : std::string errormessage = "Device ID " + std::to_string(devIdList0[i]) + " in module 0 and device ID "
969 0 : + std::to_string(devIdList1[i]) + " in module 1 are not on the same plane.";
970 0 : RPT_INPUT_ERR(
971 : true, "EI0010", std::vector<std::string>({"reason"}), std::vector<std::string>({errormessage}));
972 0 : HCCL_ERROR(
973 : "[%s][%s]%s", LOG_KEYWORDS_INIT_CHANNEL.c_str(), LOG_KEYWORDS_TIMEOUT.c_str(),
974 : errormessage.c_str());
975 0 : return HCCL_E_NOT_SUPPORT;
976 0 : }
977 : }
978 0 : }
979 498 : return HCCL_SUCCESS;
980 0 : }
981 :
982 387 : HcclResult HcclCommunicator::CheckDataType(const HcclDataType dataType, bool needReduce)
983 : {
984 3392 : const vector<string> infoTitle({"ccl_op", "value", "parameter", "expect"});
985 404 : if (needReduce) {
986 183 : if (Is310P3Common(isHaveCpuRank_, deviceType_)) {
987 0 : if ((dataType == HCCL_DATA_TYPE_INT64) || (dataType == HCCL_DATA_TYPE_BFP16)) {
988 0 : RPT_INPUT_ERR(
989 : true, "EI0003", infoTitle,
990 : vector<string>(
991 : {"CheckDataType", GetDataTypeEnumStr(dataType), "dataType",
992 : "HCCL_DATA_TYPE_INT8, HCCL_DATA_TYPE_INT16, HCCL_DATA_TYPE_INT32, "
993 : "HCCL_DATA_TYPE_FP16, HCCL_DATA_TYPE_FP32"}));
994 0 : HCCL_ERROR(
995 : "[%s][%s]errNo[0x%016llx] data type[%s] not supported, support range=[%s]",
996 : LOG_KEYWORDS_TASK_EXEC.c_str(), LOG_KEYWORDS_INVALID_ARGUMENT.c_str(),
997 : HCCL_ERROR_CODE(HCCL_E_NOT_SUPPORT), GetDataTypeEnumStr(dataType).c_str(),
998 : GetSupportDataType(needReduce).c_str());
999 0 : return HCCL_E_NOT_SUPPORT;
1000 : }
1001 : }
1002 175 : if ((dataType == HCCL_DATA_TYPE_UINT64) || (dataType == HCCL_DATA_TYPE_UINT8)
1003 175 : || (dataType == HCCL_DATA_TYPE_UINT16) || (dataType == HCCL_DATA_TYPE_UINT32)
1004 175 : || (dataType == HCCL_DATA_TYPE_FP64) || (dataType == HCCL_DATA_TYPE_RESERVED)) {
1005 14 : RPT_INPUT_ERR(
1006 : true, "EI0003", infoTitle,
1007 : vector<string>(
1008 : {"CheckDataType", GetDataTypeEnumStr(dataType), "dataType",
1009 : "HCCL_DATA_TYPE_INT8, HCCL_DATA_TYPE_INT16, HCCL_DATA_TYPE_INT32, "
1010 : "HCCL_DATA_TYPE_FP16, HCCL_DATA_TYPE_FP32"}));
1011 6 : HCCL_ERROR(
1012 : "[%s][%s]errNo[0x%016llx] data type[%s] not supported, support range=[%s]",
1013 : LOG_KEYWORDS_TASK_EXEC.c_str(), LOG_KEYWORDS_INVALID_ARGUMENT.c_str(),
1014 : HCCL_ERROR_CODE(HCCL_E_NOT_SUPPORT), GetDataTypeEnumStr(dataType).c_str(),
1015 : GetSupportDataType(needReduce).c_str());
1016 1 : return HCCL_E_NOT_SUPPORT;
1017 : }
1018 : } else {
1019 221 : if ((dataType >= HCCL_DATA_TYPE_RESERVED) || (dataType < HCCL_DATA_TYPE_INT8)
1020 442 : || (Is310P3Common(isHaveCpuRank_, deviceType_) && dataType == HCCL_DATA_TYPE_BFP16)) {
1021 0 : RPT_INPUT_ERR(
1022 : true, "EI0003", infoTitle,
1023 : vector<string>(
1024 : {"CheckDataType", GetDataTypeEnumStr(dataType), "dataType",
1025 : "HCCL_DATA_TYPE_INT8, HCCL_DATA_TYPE_INT16, HCCL_DATA_TYPE_INT32, "
1026 : "HCCL_DATA_TYPE_FP16, HCCL_DATA_TYPE_FP32, HCCL_DATA_TYPE_UINT8, HCCL_DATA_TYPE_UINT16, "
1027 : "HCCL_DATA_TYPE_UINT32"}));
1028 0 : HCCL_ERROR(
1029 : "[%s][%s]errNo[0x%016llx] data type[%s] not supported, support range=[%s]",
1030 : LOG_KEYWORDS_TASK_EXEC.c_str(), LOG_KEYWORDS_INVALID_ARGUMENT.c_str(),
1031 : HCCL_ERROR_CODE(HCCL_E_NOT_SUPPORT), GetDataTypeEnumStr(dataType).c_str(),
1032 : GetSupportDataType(needReduce).c_str());
1033 0 : return HCCL_E_NOT_SUPPORT;
1034 : }
1035 : }
1036 390 : return HCCL_SUCCESS;
1037 393 : }
1038 :
1039 0 : HcclResult HcclCommunicator::InitZeroCopyMemoryAgent()
1040 : {
1041 0 : CHK_PRT_RET(
1042 : zeroCopyMemoryAgent_ != nullptr,
1043 : HCCL_ERROR("[HcclCommunicator][InitZeroCopyMemoryAgent] ipc memory agent has init"), HCCL_E_INTERNAL);
1044 :
1045 : // 获取节点内的ranktable
1046 0 : std::vector<std::vector<std::vector<RankInfo>>> commPlaneVector;
1047 0 : CHK_SMART_PTR_NULL(implAlg_);
1048 0 : implAlg_->GetCommPlaneVector(commPlaneVector);
1049 0 : rankInfoListIntraServer_ = commPlaneVector[COMM_LEVEL0][COMM_INDEX_0];
1050 0 : zeroCopyMemoryAgent_.reset(static_cast<ZeroCopyMemoryAgent*>(new (std::nothrow) ZeroCopyMemoryAgent(
1051 0 : socketManager_, devicePhyId_, deviceLogicId_, localVnicIp_, rankInfoListIntraServer_, userRank_,
1052 0 : useSuperPodMode_, identifier_)));
1053 0 : CHK_PTR_NULL(zeroCopyMemoryAgent_);
1054 0 : CHK_RET(zeroCopyMemoryAgent_->Init());
1055 0 : return HCCL_SUCCESS;
1056 0 : }
1057 :
1058 1042 : HcclResult HcclCommunicator::DeinitZeroCopyMemoryAgent(bool inDestructor)
1059 : {
1060 1042 : if (zeroCopyMemoryAgent_ != nullptr) {
1061 0 : if (!inDestructor && zeroCopyMemoryAgent_->IsResumed()) {
1062 : // 析构函数释放场景不做barrier close
1063 0 : CHK_RET(zeroCopyMemoryAgent_->BarrierClose());
1064 : }
1065 0 : CHK_RET(zeroCopyMemoryAgent_->DeInit());
1066 0 : zeroCopyMemoryAgent_ = nullptr;
1067 : }
1068 1042 : return HCCL_SUCCESS;
1069 : }
1070 :
1071 73 : u8 HcclCommunicator::GetConfigAclGraphZeroCopyEnable() { return commConfig_.GetConfigAclGraphZeroCopyEnable(); }
1072 :
1073 61 : HcclResult HcclCommunicator::ClearResMap(const std::string& tag, bool& findTag, bool aclGraphDestroyCbk)
1074 : {
1075 61 : auto resIter = resMap_.find(tag);
1076 60 : if (resIter != resMap_.end()) {
1077 12 : findTag = true;
1078 12 : DestroyAlgResource(resIter->second, aclGraphDestroyCbk);
1079 12 : CHK_RET(StreamActiveManager::GetInstance(deviceLogicId_).StreamsUnactive(resIter->second.slaveStreams));
1080 12 : resMap_.erase(resIter);
1081 12 : HCCL_INFO("[%s] clear resMap[%s]", __func__, tag.c_str());
1082 : }
1083 59 : return HCCL_SUCCESS;
1084 : }
1085 :
1086 4 : HcclResult HcclCommunicator::ClearAclgraphHostLinks(const std::unordered_set<std::string>& tags)
1087 : {
1088 4 : std::lock_guard<std::mutex> lock(commResMutex_);
1089 8 : for (const auto& tag : tags) {
1090 4 : auto it = tagsRequiringHostCleanup_.find(tag);
1091 4 : if (it == tagsRequiringHostCleanup_.end()) {
1092 1 : continue;
1093 : }
1094 4 : for (auto& rankIt : rankTagRemoteRes_) {
1095 1 : auto tagIt = rankIt.second.find(tag);
1096 1 : if (tagIt == rankIt.second.end()) {
1097 0 : continue;
1098 : }
1099 1 : HccltagRemoteResV2* hostPtr = tagIt->second.tagRemoteResPtr;
1100 1 : if (hostPtr != nullptr) {
1101 : // 顺序敏感: ListCommonRemove 必须先于 erase shared_ptr,
1102 : // 否则 hostPtr 成野指针, ListCommonRemove 访问 segfault。
1103 1 : ListCommonRemove(&hostPtr->nextTagRes);
1104 1 : for (auto vIt = hostMemVec_.begin(); vIt != hostMemVec_.end(); ++vIt) {
1105 1 : if (*vIt && (*vIt)->ptr() == hostPtr) {
1106 1 : size_t idx = static_cast<size_t>(vIt - hostMemVec_.begin());
1107 1 : if (idx < deviceMemVec_.size()) {
1108 1 : deviceMemVec_.erase(deviceMemVec_.begin() + idx);
1109 : }
1110 1 : hostMemVec_.erase(vIt);
1111 1 : break;
1112 : }
1113 : }
1114 : }
1115 1 : rankIt.second.erase(tagIt);
1116 : }
1117 3 : ibverbsLocalNotify_.erase(tag);
1118 3 : ibverbsRemoteNotify_.erase(tag);
1119 3 : tagsRequiringHostCleanup_.erase(it);
1120 : }
1121 4 : return HCCL_SUCCESS;
1122 4 : }
1123 :
1124 19 : HcclResult HcclCommunicator::ClearOpResource(const std::string& tag, bool aclGraphDestroyCbk)
1125 : {
1126 19 : std::lock_guard<std::mutex> lock(commResMutex_);
1127 19 : bool findTag = false;
1128 19 : CHK_RET(ClearResMap(tag, findTag, aclGraphDestroyCbk));
1129 19 : CHK_RET(ClearResMap(tag + "_host", findTag, aclGraphDestroyCbk));
1130 19 : CHK_RET(ClearResMap(tag + "_device", findTag, aclGraphDestroyCbk));
1131 19 : if (!findTag) {
1132 13 : HCCL_WARNING("[%s] not find tag[%s] in resMap", __func__, tag.c_str());
1133 : }
1134 :
1135 19 : tagCommInfo_.erase(tag);
1136 : // stream解绑定
1137 19 : auto iterStream = tagStreamInfo_.find(tag);
1138 17 : if (iterStream != tagStreamInfo_.end()) {
1139 0 : CHK_RET(StreamActiveManager::GetInstance(deviceLogicId_).StreamsUnactive(iterStream->second.ringStreams));
1140 : }
1141 17 : tagStreamInfo_.erase(tag);
1142 19 : if (opRetryStreamPtr_ != nullptr) {
1143 16 : opRetryStreamPtr_->erase(tag);
1144 : }
1145 18 : if (implAlg_ != nullptr) {
1146 15 : CHK_RET(implAlg_->ClearOpResource(tag));
1147 : }
1148 15 : DestroyWorkspaceResource(tag);
1149 19 : return HCCL_SUCCESS;
1150 19 : }
1151 :
1152 95 : HcclResult HcclCommunicator::CreateOpBasedResources(
1153 : const HcclCMDType& opType, const std::string& tag, const HcomCollOpInfo& opInfo)
1154 : {
1155 95 : return workSpaceRes_->CreateOpBasedResources(opType, tag, opInfo);
1156 : }
1157 :
1158 0 : HcclResult HcclCommunicator::CreateRemoteOpBasedResources(u64 memSize, const std::string& tag)
1159 : {
1160 0 : return workSpaceRes_->CreateRemoteOpBasedResources(memSize, tag);
1161 : }
1162 :
1163 0 : HcclResult HcclCommunicator::DestroyRemoteOpBasedMem(const std::string& tag)
1164 : {
1165 0 : return workSpaceRes_->DestroyRemoteOpBasedMem(tag);
1166 : }
1167 :
1168 145 : bool HcclCommunicator::IsAtomicInit()
1169 : {
1170 290 : if (!initializedFlag_.test_and_set()) {
1171 0 : initializedFlag_.clear();
1172 0 : return false;
1173 : }
1174 145 : return true;
1175 : }
1176 :
1177 5 : bool HcclCommunicator::IsNeedNicInit()
1178 : {
1179 : return (
1180 10 : (nicInitialized_ == 0) && (!hcomGroupNicInit_) && (userRankSize_ > 1) && !isSingleMeshAggregation_
1181 10 : && (superPodNum_ > 1 || !isUsedInterHccsMode_));
1182 : }
1183 :
1184 0 : HcclResult HcclCommunicator::GetBandWidthPerNPU(u32 level, float& bandWidth)
1185 : {
1186 0 : return hccl::GetBandWidthPerNPU(level, userRankSize_, deviceNumPerAggregation_, bandWidth);
1187 : }
1188 :
1189 0 : HcclResult HcclCommunicator::GetDeviceNumPerAggregation(u32& deviceNumPerAggregation)
1190 : {
1191 0 : deviceNumPerAggregation = deviceNumPerAggregation_;
1192 0 : return HCCL_SUCCESS;
1193 : }
1194 :
1195 235 : HcclResult HcclCommunicator::InitHccpChannel() { return hcclH2dTlv::GetInstance().InitHccpChannel(devicePhyId_); }
1196 :
1197 404 : std::vector<RankInfo> HcclCommunicator::GetRankLists() { return rankInfoList_; }
1198 :
1199 158 : HcclResult HcclCommunicator::CheckReduceDataType(const HcclDataType dataType, const HcclReduceOp op)
1200 : {
1201 158 : if ((deviceType_ == DevType::DEV_TYPE_910B) || (deviceType_ == DevType::DEV_TYPE_910_93)) {
1202 50 : if ((op == HCCL_REDUCE_PROD) && ((dataType == HCCL_DATA_TYPE_INT16) || (dataType == HCCL_DATA_TYPE_BFP16))) {
1203 1 : RPT_INPUT_ERR(
1204 : true, "EI0003", std::vector<std::string>({"ccl_op", "value", "parameter", "expect"}),
1205 : std::vector<std::string>(
1206 : {"CheckReduceDataType", GetDataTypeEnumStr(dataType), "dataType",
1207 : "HCCL_DATA_TYPE_INT8, HCCL_DATA_TYPE_INT32, HCCL_DATA_TYPE_FP16, HCCL_DATA_TYPE_FP32"}));
1208 1 : HCCL_ERROR(
1209 : "[%s][%s]errNo[0x%016llx] device type[%d] does not support the data type[%s] and data "
1210 : "type[%s] for Op[%s]",
1211 : LOG_KEYWORDS_TASK_EXEC.c_str(), LOG_KEYWORDS_INVALID_ARGUMENT.c_str(),
1212 : HCCL_ERROR_CODE(HCCL_E_NOT_SUPPORT), deviceType_, GetDataTypeEnumStr(HCCL_DATA_TYPE_BFP16).c_str(),
1213 : GetDataTypeEnumStr(HCCL_DATA_TYPE_INT16).c_str(), GetReduceOpEnumStr(op).c_str());
1214 0 : return HCCL_E_NOT_SUPPORT;
1215 : }
1216 108 : } else if (deviceType_ == DevType::DEV_TYPE_910) {
1217 108 : if (dataType == HCCL_DATA_TYPE_INT16) {
1218 0 : RPT_INPUT_ERR(
1219 : true, "EI0003", std::vector<std::string>({"ccl_op", "value", "parameter", "expect"}),
1220 : std::vector<std::string>(
1221 : {"CheckReduceDataType", GetDataTypeEnumStr(dataType), "dataType",
1222 : "HCCL_DATA_TYPE_INT8, HCCL_DATA_TYPE_INT32, HCCL_DATA_TYPE_FP16, HCCL_DATA_TYPE_FP32"}));
1223 0 : HCCL_ERROR(
1224 : "[%s][%s]errNo[0x%016llx] device type[%d] does not support the data type[%s]",
1225 : LOG_KEYWORDS_TASK_EXEC.c_str(), LOG_KEYWORDS_INVALID_ARGUMENT.c_str(),
1226 : HCCL_ERROR_CODE(HCCL_E_NOT_SUPPORT), deviceType_, GetDataTypeEnumStr(dataType).c_str());
1227 0 : return HCCL_E_NOT_SUPPORT;
1228 : }
1229 0 : } else if (deviceType_ == DevType::DEV_TYPE_310P3) {
1230 0 : if (dataType == HcclDataType::HCCL_DATA_TYPE_INT16 && op != HcclReduceOp::HCCL_REDUCE_SUM) {
1231 0 : RPT_INPUT_ERR(
1232 : true, "EI0003", std::vector<std::string>({"ccl_op", "value", "parameter", "expect"}),
1233 : std::vector<std::string>({"CheckReduceDataType", GetReduceOpEnumStr(op), "op", "sum"}));
1234 0 : HCCL_ERROR(
1235 : "[%s][%s]errNo[0x%016llx] device type[%d] does not support the data type[%s] for Op[%s]",
1236 : LOG_KEYWORDS_TASK_EXEC.c_str(), LOG_KEYWORDS_INVALID_ARGUMENT.c_str(),
1237 : HCCL_ERROR_CODE(HCCL_E_NOT_SUPPORT), deviceType_,
1238 : GetDataTypeEnumStr(HcclDataType::HCCL_DATA_TYPE_INT16).c_str(), GetReduceOpEnumStr(op).c_str());
1239 0 : return HCCL_E_NOT_SUPPORT;
1240 : }
1241 : }
1242 157 : return HCCL_SUCCESS;
1243 0 : }
1244 :
1245 234 : HcclResult HcclCommunicator::GetAlgType(AlgType& algType, HcclCMDType opType)
1246 : {
1247 234 : CHK_SMART_PTR_NULL(implAlg_);
1248 234 : return implAlg_->GetAlgType(algType, opType);
1249 : }
1250 :
1251 1 : HcclResult HcclCommunicator::GetCommParams(HcclCommParams& params)
1252 : {
1253 1 : params.commHandle = commHandle_;
1254 1 : params.rank = userRank_;
1255 1 : params.userRank = realUserRank_;
1256 1 : params.totalRanks = userRankSize_;
1257 1 : params.logicDevId = deviceLogicId_;
1258 1 : params.deviceType = deviceType_;
1259 1 : params.hcomGroupNicInit = hcomGroupNicInit_;
1260 1 : params.identifier = identifier_;
1261 1 : params.ranktableCrc = ranktableCrc_;
1262 1 : params.commConnections = commConnections_;
1263 1 : params.commPortConfig.devPortSwitchOn = commPortConfig_.devPortSwitchOn;
1264 1 : return HCCL_SUCCESS;
1265 : }
1266 :
1267 1 : HcclResult HcclCommunicator::GetCommRankTable(RankTable_t& rankTable)
1268 : {
1269 2 : for (auto& server : servRankInfo_) {
1270 2 : for (auto& rank : server.second) {
1271 1 : rankTable.rankList.emplace_back(rank);
1272 : }
1273 : }
1274 1 : rankTable.serverNum = serverNum_;
1275 1 : rankTable.superPodNum = superPodNum_;
1276 1 : rankTable.nicDeploy = nicDeployment_;
1277 1 : rankTable.version = attrCollector_.GetRankTableVersion();
1278 1 : return HCCL_SUCCESS;
1279 : }
1280 :
1281 491 : HcclResult HcclCommunicator::InitPara()
1282 : {
1283 : // 检查当前user_rank 对应的devid和rt查到的一致
1284 491 : CHK_RET(attrCollector_.CheckLocalRankInfo());
1285 491 : CHK_RET(attrCollector_.CalAndSetMeshAggRankSize());
1286 491 : meshAggregationRankSize_ = attrCollector_.GetMeshAggregationRankSize();
1287 :
1288 : // 初始化计数任务
1289 491 : CHK_RET(OpExeCounter::GetInstance(deviceLogicId_).InitCounter());
1290 :
1291 491 : notifyPool_.reset(new (std::nothrow) NotifyPool());
1292 491 : CHK_SMART_PTR_NULL(notifyPool_);
1293 491 : CHK_RET(notifyPool_->Init(devicePhyId_));
1294 :
1295 491 : callbackTask_.reset(new (std::nothrow) HcclCallbackTask(devicePhyId_, deviceLogicId_, dispatcher_, nicDeployment_));
1296 491 : CHK_SMART_PTR_NULL(callbackTask_);
1297 :
1298 491 : workSpaceRes_.reset(new (std::nothrow) WorkspaceResource(devicePhyId_, deviceLogicId_, &cclBufferManager_));
1299 491 : CHK_SMART_PTR_NULL(workSpaceRes_);
1300 :
1301 491 : CHK_RET(InitAlgResource());
1302 491 : return HCCL_SUCCESS;
1303 : }
1304 :
1305 0 : bool HcclCommunicator::IsStandardCard()
1306 : {
1307 0 : if (Is310P3Common(isHaveCpuRank_, deviceType_)) {
1308 0 : HCCL_INFO("The current device just support this StandardCard case.");
1309 0 : return true;
1310 : }
1311 :
1312 : return (
1313 0 : (pairLinkInfo_[static_cast<u32>(LinkTypeInServer::HCCS_TYPE)].size() == 0)
1314 0 : && (pairLinkInfo_[static_cast<u32>(LinkTypeInServer::HCCS_SW_TYPE)].size() == 0)
1315 0 : && (pairLinkInfo_[static_cast<u32>(LinkTypeInServer::SIO_TYPE)].size() == 0));
1316 : }
1317 :
1318 523 : HcclResult HcclCommunicator::InitOpRetry()
1319 : {
1320 523 : EXCEPTION_CATCH((opRetryStreamPtr_ = std::make_shared<HcclOpStreamRes>()), return HCCL_E_PTR);
1321 523 : if (retryEnable_) {
1322 0 : opRetryManager_.reset(new (std::nothrow) OpRetryManager());
1323 0 : CHK_SMART_PTR_NULL(opRetryManager_);
1324 0 : HcclIpAddress hostIp = !rankInfoList_.empty() ? rankInfoList_[0].hostIp : HcclIpAddress();
1325 0 : u32 hostPort = !rankInfoList_.empty() ? rankInfoList_[0].hostPort : HCCL_INVALID_PORT;
1326 0 : s32 hostDevId = !rankInfoList_.empty() ? rankInfoList_[0].devicePhyId : 0;
1327 0 : HcclIpAddress localIp = rankInfoList_.size() > userRank_ ? rankInfoList_[userRank_].hostIp : HcclIpAddress();
1328 0 : auto notifyResetCallback = [this](bool isSendRecv, s64 destRank) {
1329 0 : return isSendRecv ? this->ResetNotifyForDestRank(destRank) : this->ResetNotify();
1330 0 : };
1331 :
1332 0 : auto setTransportStatusCallback = [this](
1333 : const HcclOpIdentifier& opId, bool statusStop,
1334 : const std::map<u32, bool>& remoteRankPortMap,
1335 : const std::map<u32, bool>& isChangeLinkMap, bool isChangeLinkFlag) {
1336 0 : return this->SetTransportStatus(opId, statusStop, remoteRankPortMap, isChangeLinkMap, isChangeLinkFlag);
1337 0 : };
1338 : auto getSwitchRanksCallback
1339 0 : = [this](
1340 : u32* distSwitchRankList, bool* distSwitchUseBackup, u32& distSwitchRankNum,
1341 : u8* distRemoteRankNicStatus, u32& distRankSize, bool& needCheckDefaultNic, bool& needCheckBackupNic) {
1342 0 : return this->GetSwitchRanks(
1343 : distSwitchRankList, distSwitchUseBackup, distSwitchRankNum, distRemoteRankNicStatus, distRankSize,
1344 0 : needCheckDefaultNic, needCheckBackupNic);
1345 0 : };
1346 0 : auto setTransportResumeStatusCallback = [this](
1347 : const std::map<u32, bool>& remoteRankPortMap,
1348 : const std::map<u32, bool>& isChangeLinkMap, bool isChangeLinkFlag,
1349 : bool statusStop) {
1350 0 : return this->SetTransportResumeStatus(remoteRankPortMap, isChangeLinkMap, isChangeLinkFlag, statusStop);
1351 0 : };
1352 0 : HcclNetDevCtx netDevCtx = netDevCtxMap_[devIpAddr_[0]];
1353 0 : HcclNetDevCtx backUpNetDevCtx = {};
1354 0 : if (IsEnableBackupLink()) {
1355 0 : g_enableBackupLinkCommCount++;
1356 : }
1357 0 : if (IsEnableBackupLink() && netDevCtxMap_.find(devBackupIpAddr_[0]) != netDevCtxMap_.end()) {
1358 0 : backUpNetDevCtx = netDevCtxMap_[devBackupIpAddr_[0]];
1359 : }
1360 0 : OpRetryServerInfo serverInfo = {hostIp, hostPort, hostDevId};
1361 0 : OpRetryAgentInfo agentInfo = {userRank_, deviceLogicId_, localIp, devIpAddr_[0], netDevCtx, backUpNetDevCtx};
1362 :
1363 0 : OpRetryAgentParam agentParam;
1364 0 : agentParam.group = identifier_;
1365 0 : agentParam.agentConnection = commConnections_.agentConnection;
1366 0 : agentParam.h2dPtr = kfcControlTransferH2D_;
1367 0 : agentParam.d2hPtr = kfcStatusTransferD2H_;
1368 0 : agentParam.opStreamPtr = opRetryStreamPtr_;
1369 0 : agentParam.notifyResetCallback = notifyResetCallback;
1370 0 : agentParam.setTransportStatusCallback = setTransportStatusCallback;
1371 0 : agentParam.setTransportResumeStatusCallback = setTransportResumeStatusCallback;
1372 0 : agentParam.getSwitchRanksCallback = getSwitchRanksCallback;
1373 0 : agentParam.isEnableBackupLink = IsEnableBackupLink();
1374 0 : agentParam.isEnableSdmaRetry = commConfig_.GetConfigInterServerRetryEnable();
1375 0 : agentParam.agentInfo = agentInfo;
1376 :
1377 0 : CHK_RET(opRetryManager_->RegisterOpRetryMachine(
1378 : agentParam, userRankSize_, commConnections_.isRoot, commConnections_.serverConnections, serverInfo));
1379 0 : HCCL_RUN_INFO(
1380 : "[InitOpRetry] group[%s], isEnableBackupLink[%d], g_enableBackupLinkCommCount[%u]", identifier_.c_str(),
1381 : IsEnableBackupLink(), g_enableBackupLinkCommCount.load());
1382 0 : }
1383 523 : return HCCL_SUCCESS;
1384 : }
1385 :
1386 1 : bool HcclCommunicator::CompareWithServerId(const ServerInfo_t& left, const ServerInfo_t& right)
1387 : {
1388 1 : return (strcmp(left.serverId.c_str(), right.serverId.c_str()) < 0);
1389 : }
1390 :
1391 0 : bool HcclCommunicator::CompareWithNicName(const NetworkInfo_t& left, const NetworkInfo_t& right)
1392 : {
1393 0 : return (strcmp(left.ethName.c_str(), right.ethName.c_str()) < 0);
1394 : }
1395 :
1396 0 : bool HcclCommunicator::CompareWithUserRank(const RankInfo& left, const RankInfo& right)
1397 : {
1398 0 : return left.userRank < right.userRank;
1399 : }
1400 :
1401 306 : HcclResult HcclCommunicator::InitPreResource(const RankTable_t& rankTable)
1402 : {
1403 306 : if (static_cast<s32>(devicePhyId_) == HOST_DEVICE_ID) {
1404 0 : HCCL_ERROR("[Init][PreResource]not support cpu rank");
1405 0 : return HCCL_E_NOT_SUPPORT;
1406 : }
1407 : (void)rankTable;
1408 : // 判断是否为A3多docker场景，该场景需要使用sdid获取到的serverId判断是否属于同一server，若属于同一server则需要enablep2p
1409 306 : if (deviceType_ == DevType::DEV_TYPE_910_93) {
1410 16 : uint32_t localRankServerId = 0;
1411 16 : uint32_t remoteRankServerId = 0;
1412 16 : rtError_t ret = rtGetServerIDBySDID(rankInfoList_[userRank_].superDeviceId, &localRankServerId);
1413 16 : CHK_PRT_RET(
1414 : ret != RT_ERROR_NONE,
1415 : HCCL_ERROR(
1416 : "[Init][PreResource]rtGetServerIDBySDID failed sdid[0x%08x], serverID[%u], ret[%u]",
1417 : rankInfoList_[userRank_].superDeviceId, localRankServerId, ret),
1418 : HCCL_E_RUNTIME);
1419 65 : for (size_t index = 0; index < rankInfoList_.size(); ++index) {
1420 49 : const RankInfo& rankInfo = rankInfoList_[index];
1421 49 : ret = rtGetServerIDBySDID(rankInfo.superDeviceId, &remoteRankServerId);
1422 49 : CHK_PRT_RET(
1423 : ret != RT_ERROR_NONE,
1424 : HCCL_ERROR(
1425 : "[Init][PreResource]rtGetServerIDBySDID failed sdid[0x%08x], serverID[%u], ret[%u]",
1426 : rankInfo.superDeviceId, remoteRankServerId, ret),
1427 : HCCL_E_RUNTIME);
1428 49 : if (serverId_ != rankInfo.serverId && localRankServerId == remoteRankServerId) {
1429 8 : enableP2PDevices_.push_back(rankInfo.devicePhyId);
1430 8 : enableP2PRankIds_.insert(rankInfo.userRank);
1431 8 : HCCL_INFO(
1432 : "[Init][PreResource]localRankID[%u]-localDevicePhyId[%u] needs to enablep2p with "
1433 : "remoteRankId[%u]-remoteDevicePhyId[%u], "
1434 : "and localServerId[%s], localServerIdBySDID[%u], remoteServerId[%s], remoteServerIdBySDID[%u]",
1435 : userRank_, rankInfoList_[userRank_].devicePhyId, rankInfo.userRank, rankInfo.devicePhyId,
1436 : serverId_.c_str(), localRankServerId, rankInfo.serverId.c_str(), remoteRankServerId);
1437 : }
1438 : }
1439 : }
1440 : // 查询本rank所在服务器
1441 306 : auto iterServ = servRankInfo_.find(serverId_);
1442 :
1443 305 : bool check = (iterServ == servRankInfo_.end());
1444 305 : CHK_PRT_RET(
1445 : check, HCCL_ERROR("[Init][PreResource]can't find serverId[%s] in server map", serverId_.c_str()),
1446 : HCCL_E_NOT_FOUND);
1447 :
1448 1247 : for (u32 i = 0; i < iterServ->second.size(); i++) {
1449 937 : if (iterServ->second[i].deviceInfo.devicePhyId != HOST_DEVICE_ID) {
1450 936 : enableP2PDevices_.push_back(iterServ->second[i].deviceInfo.devicePhyId);
1451 934 : enableP2PRankIds_.insert(iterServ->second[i].rankId);
1452 935 : HCCL_INFO(
1453 : "[Init][PreResource]In the current server[%s], localRank[%u]-localDevicePhyId[%u] needs to enableP2P "
1454 : "with remoteRankId[%u]-remoteDevicePhyId[%u]",
1455 : serverId_.c_str(), userRank_, rankInfoList_[userRank_].devicePhyId, iterServ->second[i].rankId,
1456 : iterServ->second[i].deviceInfo.devicePhyId);
1457 : }
1458 : }
1459 306 : HCCL_INFO(
1460 : "[Init][PreResource]Current deviceType[%d], isStandardCard[%s]", deviceType_,
1461 : isStandardCard_ ? "true" : "false");
1462 306 : if (deviceType_ != DevType::DEV_TYPE_310P3 && !isStandardCard_) {
1463 154 : HcclResult ret = P2PMgmtPub::EnableP2P(enableP2PDevices_);
1464 155 : CHK_PRT_RET(
1465 : ret != HCCL_SUCCESS,
1466 : HCCL_ERROR("[Init][PreResource]Enable P2P Failed, deviceLogicId[%d], ret[%u]", deviceLogicId_, ret), ret);
1467 : }
1468 :
1469 307 : drvInit_ = true;
1470 307 : return HCCL_SUCCESS;
1471 : }
1472 :
1473 495 : HcclResult HcclCommunicator::InitTcpMode(const RankTable_t& rankTable) const
1474 : {
1475 495 : bool isTcpMode = false;
1476 495 : HCCL_INFO("[TcpMode][%u] [1:TCP, 2:RDMA, 3:RESERVED]", GetExternalInputProtocolType());
1477 498 : if (GetExternalInputProtocolType() == ProtocolType::TCP) {
1478 0 : isTcpMode = true;
1479 497 : } else if (GetExternalInputProtocolType() == ProtocolType::RDMA) {
1480 : // 通信协议选择RDMA
1481 : } else {
1482 296 : isTcpMode = (rankTable.nicDeploy == NICDeployment::NIC_DEPLOYMENT_HOST);
1483 296 : HCCL_INFO("[Init][TcpMode]isTcpMode[%d] nicDeploy[%d]", isTcpMode, rankTable.nicDeploy);
1484 : }
1485 498 : SetTcpMode(isTcpMode);
1486 :
1487 : // 异构场景解析外部输入,放在SetTcpMode前防止Tcp用例走错分支，放在RecordProtocolType确保hdc模式下建链通信协议校验正确
1488 498 : CHK_RET(InitExternalInputHeterog());
1489 496 : return HCCL_SUCCESS;
1490 : }
1491 :
1492 491 : HcclResult HcclCommunicator::InitMyRankConnectMode(HcclCommParams& params, const RankTable_t& rankTable)
1493 : {
1494 491 : if (deviceType_ != DevType::DEV_TYPE_910B) { /* 910B才支持host网卡特性 */
1495 326 : myRankConnectMode_ = 0;
1496 326 : return HCCL_SUCCESS;
1497 : }
1498 165 : uint32_t localRank = params.rank;
1499 165 : if (rankTable.nicDeploy != NICDeployment::NIC_DEPLOYMENT_HOST) {
1500 165 : myRankConnectMode_ = 0;
1501 165 : return HCCL_SUCCESS;
1502 : }
1503 0 : for (auto it : rankTable.rankList) {
1504 0 : if (it.rankId == localRank) {
1505 0 : continue;
1506 : }
1507 0 : if (it.deviceInfo.nicDeploy == NICDeployment::NIC_DEPLOYMENT_DEVICE) {
1508 0 : myRankConnectMode_ = 1;
1509 0 : return HCCL_SUCCESS;
1510 : }
1511 0 : }
1512 0 : myRankConnectMode_ = 0;
1513 0 : return HCCL_SUCCESS;
1514 : }
1515 :
1516 234 : uint32_t HcclCommunicator::GetConnectMode() { return myRankConnectMode_; }
1517 :
1518 1794 : bool HcclCommunicator::IsEnableBackupLink()
1519 : {
1520 224 : return deviceType_ == DevType::DEV_TYPE_910_93 && IsEnableRoce() && GetAicpuUnfoldConfig() && retryEnable_
1521 0 : && commConfig_.GetConfigInterSuperPodRetryEnable() && !devBackupIpAddr_[0].IsInvalid()
1522 2021 : && rtsSupportChangeLink_ && !isDiffDeviceType_;
1523 : }
1524 :
1525 162 : HcclResult HcclCommunicator::InitRaNetResource()
1526 : {
1527 162 : CHK_RET(IsHostUseDevNic(isHostUseDevNic_));
1528 :
1529 162 : if (static_cast<s32>(devicePhyId_) != HOST_DEVICE_ID || nicDeployment_ == NICDeployment::NIC_DEPLOYMENT_DEVICE) {
1530 162 : CHK_RET(HcclNetInit(NICDeployment::NIC_DEPLOYMENT_DEVICE, devicePhyId_, deviceLogicId_, false));
1531 161 : if (IsEnableBackupLink()) {
1532 : // 超节点 && level2支持重执行 && Aicpu -> 初始化主备hccp资源(Pid粒度)
1533 1 : CHK_RET(hrtGetPairDevicePhyId(devicePhyId_, deviceBackUpPhyId_));
1534 1 : if (hrtGetDeviceIndexByPhyId(deviceBackUpPhyId_, deviceBackUpLogicId_) != HCCL_SUCCESS) {
1535 1 : rtsSupportChangeLink_ = false;
1536 1 : HCCL_ERROR(
1537 : "[%s]Runtime does not support changelink, deviceLogicId_[%d], devicePhyId_[%u], "
1538 : "deviceBackUpPhyId_[%u], deviceBackUpLogicId_[%u], nicDeployment_[%d], IsEnableBackupLink[%d]"
1539 : "rtsSupportChangeLink_[%d]",
1540 : __func__, deviceLogicId_, devicePhyId_, deviceBackUpPhyId_, deviceBackUpLogicId_, nicDeployment_,
1541 : IsEnableBackupLink(), rtsSupportChangeLink_);
1542 1 : return HCCL_E_NOT_SUPPORT;
1543 : } else {
1544 0 : CHK_RET(HcclNetInit(
1545 : NICDeployment::NIC_DEPLOYMENT_DEVICE, deviceBackUpPhyId_, deviceBackUpLogicId_, false, true));
1546 0 : HCCL_DEBUG(
1547 : "[%s]Default & backup NetworkManager Init, deviceLogicId[%d], devicePhyId[%u], "
1548 : "deviceBackUpPhyId_[%u], deviceBackUpLogicId_[%u], nicDeployment_[%d], IsEnableBackupLink[%d]",
1549 : __func__, deviceLogicId_, devicePhyId_, deviceBackUpPhyId_, deviceBackUpLogicId_, nicDeployment_,
1550 : IsEnableBackupLink());
1551 : }
1552 : }
1553 : }
1554 :
1555 160 : if ((static_cast<s32>(devicePhyId_) != HOST_DEVICE_ID && isHaveCpuRank_)
1556 160 : || (IsEnableRoce() && nicDeployment_ == NICDeployment::NIC_DEPLOYMENT_HOST)
1557 320 : || (Is310PDevice() && nicDeployment_ == NICDeployment::NIC_DEPLOYMENT_HOST)) {
1558 0 : u32 devicePhyID = (static_cast<s32>(devicePhyId_) == HOST_DEVICE_ID) ? 0 : devicePhyId_;
1559 0 : u32 whiteListEn = (GetExternalInputHcclEnableWhitelist() == HCCL_WHITELIST_ON ? 1 : 0);
1560 0 : CHK_RET(HcclNetInit(NICDeployment::NIC_DEPLOYMENT_HOST, devicePhyID, deviceLogicId_, whiteListEn));
1561 : }
1562 160 : return HCCL_SUCCESS;
1563 : }
1564 :
1565 160 : HcclResult HcclCommunicator::InitRaNic()
1566 : {
1567 160 : CHK_RET(InitSocketManager());
1568 :
1569 160 : if (Is310PDevice()) {
1570 0 : CHK_RET(InitNic());
1571 160 : } else if (static_cast<s32>(devicePhyId_) != HOST_DEVICE_ID) {
1572 160 : std::shared_ptr<HcclSocket>& devVnicSocket = commPortConfig_.devVnicListen.first;
1573 160 : if (devVnicSocket) {
1574 1 : localVnicIp_ = devVnicSocket->GetLocalIp();
1575 1 : localVnicListenPort_ = devVnicSocket->GetLocalPort();
1576 1 : HcclNetDevCtx& devVnicCtx = commPortConfig_.devVnicListen.second;
1577 1 : CHK_PTR_NULL(devVnicCtx);
1578 1 : netDevCtxMap_.insert(std::make_pair(localVnicIp_, devVnicCtx));
1579 1 : CHK_RET(socketManager_->ServerInit(devVnicCtx, localVnicListenPort_));
1580 1 : commPortConfig_.devVnicListen.second = nullptr;
1581 1 : HCCL_INFO(
1582 : "[HcclCommunicator][InitRaResource] init vnic with listened socket success, "
1583 : "listened ip[%s] port[%u]",
1584 : localVnicIp_.GetReadableAddress(), localVnicListenPort_);
1585 : } else {
1586 159 : localVnicListenPort_ = GetLocalNicPort(NicType::VNIC_TYPE);
1587 159 : localVnicIp_ = HcclIpAddress(devicePhyId_);
1588 159 : if (useSuperPodMode_) {
1589 6 : CHK_RET(hrtRaGetSingleSocketVnicIpInfo(
1590 : devicePhyId_, DeviceIdType::DEVICE_ID_TYPE_SDID, superDeviceId_, localVnicIp_));
1591 : } else {
1592 159 : CHK_RET(hrtRaGetSingleSocketVnicIpInfo(
1593 : devicePhyId_, DeviceIdType::DEVICE_ID_TYPE_PHY_ID, devicePhyId_, localVnicIp_));
1594 : }
1595 :
1596 : HcclNetDevCtx vnicPortCtx;
1597 159 : CHK_RET(HcclNetOpenDev(&vnicPortCtx, NicType::VNIC_TYPE, devicePhyId_, deviceLogicId_, localVnicIp_));
1598 159 : CHK_PTR_NULL(vnicPortCtx);
1599 159 : netDevCtxMap_.insert(std::make_pair(localVnicIp_, vnicPortCtx));
1600 159 : CHK_RET(socketManager_->ServerInit(vnicPortCtx, localVnicListenPort_));
1601 153 : HCCL_INFO(
1602 : "[HcclCommunicator][InitRaResource] init vnic with ip[%s] port[%u] success",
1603 : localVnicIp_.GetReadableAddress(), localVnicListenPort_);
1604 : }
1605 :
1606 154 : if (IsEnableRoce()) {
1607 0 : CHK_RET(InitNic());
1608 : }
1609 : }
1610 154 : return HCCL_SUCCESS;
1611 : }
1612 :
1613 278 : HcclResult HcclCommunicator::InitRaResource()
1614 : {
1615 : /* 本通信域内只有1个device时，不需要初始化ra资源 */
1616 278 : if (userRankSize_ <= 1) {
1617 116 : HCCL_INFO("user rank size <= 1, ra is not needed for single device.");
1618 116 : return HCCL_SUCCESS;
1619 : }
1620 :
1621 162 : CHK_RET(InitRaNetResource());
1622 160 : CHK_RET(InitRaNic());
1623 :
1624 154 : HCCL_INFO(
1625 : "isUsedRdmaLevel0_[%u] nicNum[%u] hostIP[%s], nicDeployment[%d].", isUsedRdmaLevel0_, devIpAddr_.size(),
1626 : hostIp_.GetReadableAddress(), nicDeployment_);
1627 :
1628 154 : raResourceInit_ = true; // 全局通信域会初始化，子通信域不会初始化，但是析构均会进入此逻辑，需要标记
1629 154 : attrCollector_.GenSupportRdmaLite();
1630 154 : CHK_RET(attrCollector_.GenSupportHccsAndSio());
1631 152 : isSupportRdmaLite_ = attrCollector_.GetSupportRdmaLite(); // 是否支持Rdma Lite
1632 154 : isSupportHccsAndSio_ = attrCollector_.GetSupportHccsAndSio(); // 是否支持Hccs Sio并发
1633 154 : return HCCL_SUCCESS;
1634 : }
1635 :
1636 307 : HcclResult HcclCommunicator::DisablePreResource()
1637 : {
1638 : // 查询本rank所在服务器
1639 307 : auto iterServ = servRankInfo_.find(serverId_);
1640 307 : bool check = (iterServ == servRankInfo_.end());
1641 307 : CHK_PRT_RET(
1642 : check, HCCL_ERROR("[Disable][PreResource]can't find serverId[%s] in server map", serverId_.c_str()),
1643 : HCCL_E_NOT_FOUND);
1644 307 : HcclResult ret = P2PMgmtPub::DisableP2P(enableP2PDevices_);
1645 307 : CHK_PRT_RET(
1646 : ret != HCCL_SUCCESS,
1647 : HCCL_ERROR("[Disable][PreResource]Disable all P2P Failed, deviceLogicId[%d], ret[%u]", deviceLogicId_, ret),
1648 : ret);
1649 307 : enableP2PDevices_.clear();
1650 307 : enableP2PRankIds_.clear();
1651 307 : return HCCL_SUCCESS;
1652 : }
1653 :
1654 156 : HcclResult HcclCommunicator::GetWorkspaceSubStreamNum(
1655 : u64 count, HcclDataType dataType, HcclReduceOp op, const std::string& algName, u64& streamNum, u64 dataSize,
1656 : bool ifAiv, HcclCMDType opType)
1657 : {
1658 156 : AlgType algType;
1659 :
1660 156 : CHK_RET(GetAlgType(algType, opType));
1661 :
1662 : std::map<HcclCMDType, u64> gapMap
1663 : = {{HcclCMDType::HCCL_CMD_REDUCE_SCATTER, HCCL_SMALL_COUNT_512_KB + HCCL_SMALL_COUNT_512_KB},
1664 : {HcclCMDType::HCCL_CMD_ALLGATHER, HCCL_SMALL_COUNT_512_KB + HCCL_SMALL_COUNT_512_KB},
1665 310 : {HcclCMDType::HCCL_CMD_ALLREDUCE, (HCCL_SMALL_COUNT_512_KB + HCCL_SMALL_COUNT_512_KB) * userRankSize_}};
1666 :
1667 : // 图模式下AIV展开，需要重新计算streamNum
1668 157 : bool ifHcomWithAiv = ifAiv && (GetWorkflowMode() == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OPS_KERNEL_INFO_LIB);
1669 157 : HCCL_INFO(
1670 : "[GetWorkspaceSubStreamNum] ifAiv[%d], workflowMode[%d], ifHcomWithAiv[%d]", ifAiv, GetWorkflowMode(),
1671 : ifHcomWithAiv);
1672 158 : if (ifHcomWithAiv && (deviceType_ == DevType::DEV_TYPE_910_93 || deviceType_ == DevType::DEV_TYPE_910B)) {
1673 0 : HCCL_INFO("[GetWorkspaceSubStreamNum] Hcom AIV enabled, calculating the streamNum.");
1674 : // A3 和 A2 公用以下的参数
1675 0 : std::string newTag;
1676 0 : std::unique_ptr<CollAlgOperator> algOperator = implAlg_->GetAlgOperator(opType);
1677 0 : CHK_SMART_PTR_NULL(algOperator);
1678 0 : OpParam param;
1679 0 : param.reduceType = op;
1680 0 : param.opType = opType;
1681 :
1682 0 : if (opType == HcclCMDType::HCCL_CMD_ALLTOALL || opType == HcclCMDType::HCCL_CMD_ALLTOALLV) {
1683 0 : param.All2AllDataDes.sendType = dataType;
1684 0 : param.All2AllDataDes.recvType = dataType;
1685 0 : param.All2AllDataDes.sendCount = count;
1686 : } else { // 不论 A2 还是 A3，AIV场景下的AllReduce/ReduceScatter还是A2上单独支持AIV的算子都用以下参数
1687 0 : param.DataDes.count = count;
1688 0 : param.DataDes.dataType = dataType;
1689 : }
1690 0 : AlgResourceRequest resRequest;
1691 0 : CHK_RET(algOperator->CalcResRequest(algName, param, resRequest)); // 计算资源请求
1692 0 : streamNum = resRequest.streamNum;
1693 0 : HCCL_INFO(
1694 : "[GetWorkspaceSubStreamNum] Hcom AIV enabled on DeviceType[%d], the streamNum is [%llu]", deviceType_,
1695 : streamNum);
1696 0 : return HCCL_SUCCESS;
1697 0 : }
1698 :
1699 155 : if (serverNum_ == 1 && deviceType_ == DevType::DEV_TYPE_910_93 && opType == HcclCMDType::HCCL_CMD_ALLGATHER
1700 313 : && dataSize <= gapMap[opType] && deviceNumPerAggregation_ > HCCL_DEVICE_NUM_TWO) {
1701 0 : constexpr u64 streamForSmallCount = 3;
1702 0 : streamNum = streamForSmallCount;
1703 0 : HCCL_DEBUG("[GetWorkspaceSubStreamNum]DEV_TYPE_910_93 Single Server, the streamNum is %llu", streamNum);
1704 0 : return HCCL_SUCCESS;
1705 : }
1706 :
1707 155 : if (serverNum_ == 1 && deviceType_ == DevType::DEV_TYPE_910_93 && gapMap.find(opType) != gapMap.end()
1708 313 : && dataSize <= gapMap[opType] && deviceNumPerAggregation_ > HCCL_DEVICE_NUM_TWO) {
1709 0 : streamNum = deviceNumPerAggregation_ - HCCL_SUB_STREAM_NP_MESH;
1710 0 : HCCL_DEBUG("[GetWorkspaceSubStreamNum]DEV_TYPE_910_93 Single Server, the streamNum is %llu", streamNum);
1711 0 : return HCCL_SUCCESS;
1712 : }
1713 :
1714 158 : if (deviceType_ == DevType::DEV_TYPE_910_93) {
1715 8 : streamNum = HCCL_SUB_STREAM_NUM_DOUBLE_RING + RDMA_PLANE_NUM_IN_NPRING_DOUBLE;
1716 8 : if (algType.algoLevel0 == AlgTypeLevel0::ALG_LEVEL0_NP_DOUBLE_RING) {
1717 0 : streamNum += 1U; // semi_ring算法server内增加一条从流，需要2条从流
1718 : }
1719 8 : if (opType == HcclCMDType::HCCL_CMD_ALLTOALLV || opType == HcclCMDType::HCCL_CMD_ALLTOALL
1720 8 : || opType == HcclCMDType::HCCL_CMD_ALLTOALLVC) {
1721 0 : streamNum = MAX_RANK_SIZE;
1722 : }
1723 8 : HCCL_DEBUG("[GetWorkspaceSubStreamNum]DEV_TYPE_910_93, the streamNum is %llu", streamNum);
1724 8 : return HCCL_SUCCESS;
1725 : }
1726 :
1727 : // AR RS 在开启Strict && 静态图、RSv 在开启确定性 && 静态图时, 需要重新计算StreamNum
1728 300 : if (deviceType_ == DevType::DEV_TYPE_910B
1729 262 : && (((opType == HcclCMDType::HCCL_CMD_ALLREDUCE || opType == HcclCMDType::HCCL_CMD_REDUCE_SCATTER)
1730 0 : && GetExternalInputHcclDeterministicV2() == DETERMINISTIC_STRICT)
1731 112 : || (opType == HcclCMDType::HCCL_CMD_REDUCE_SCATTER_V
1732 0 : && GetExternalInputHcclDeterministicV2() != DETERMINISTIC_DISABLE)
1733 112 : || (opType == HcclCMDType::HCCL_CMD_REDUCE_SCATTER_V && !isSingleMeshAggregation_
1734 0 : && !multiModuleDiffDeviceNumMode_ // 多机&对称&图模式
1735 0 : && GetWorkflowMode() == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OPS_KERNEL_INFO_LIB))) {
1736 : // 图模式 A2规约保序场景，需要重新计算需要的streamNum
1737 0 : streamNum = CalcStreamNumForReduceOrderPreservation();
1738 0 : HCCL_DEBUG("[GetWorkspaceSubStreamNum]A2 reduce order preservation, the streamNum is %llu", streamNum);
1739 0 : return HCCL_SUCCESS;
1740 : }
1741 :
1742 150 : if (deviceType_ == DevType::DEV_TYPE_910B && opType == HcclCMDType::HCCL_CMD_ALLREDUCE
1743 0 : && algType.algoLevel1 == AlgTypeLevel1::ALG_LEVEL1_PIPELINE) {
1744 0 : streamNum = userRankSize_ / moduleNum_ - 1;
1745 0 : HCCL_DEBUG("[GetWorkspaceSubStreamNum]A2 pipeline AllReduce, the streamNum is %llu", streamNum);
1746 0 : return HCCL_SUCCESS;
1747 : }
1748 :
1749 : // 设置AG和RS的图模式pipeline算法能够申请的streamNum
1750 150 : if (deviceType_ == DevType::DEV_TYPE_910B && // 910B
1751 112 : algType.algoLevel0 == AlgTypeLevel0::ALG_LEVEL0_NP_MESH && // fullmesh
1752 88 : (opType == HcclCMDType::HCCL_CMD_ALLGATHER || opType == HcclCMDType::HCCL_CMD_REDUCE_SCATTER
1753 88 : || opType == HcclCMDType::HCCL_CMD_ALLGATHER_V)
1754 0 : && // AG或RS或AGV
1755 0 : moduleNum_ > 1 && deviceNumPerAggregation_ > 1 && // 多机且每机器出多卡
1756 0 : (moduleNum_ <= MODULE_NUM_FOUR || // "机器数量小于等于4"
1757 0 : dataSize > HCCL_SMALL_COUNT_1_MB || // "大数据量"
1758 0 : algType.algoLevel1 == AlgTypeLevel1::ALG_LEVEL1_PIPELINE)) { // "指定level1的算法为pipeline"
1759 0 : streamNum = userRankSize_ / moduleNum_;
1760 0 : HCCL_DEBUG("[GetWorkspaceSubStreamNum]DEV_TYPE_910B, the streamNum is %llu", streamNum);
1761 0 : return HCCL_SUCCESS;
1762 : }
1763 :
1764 : // 设置310P图模式 alltoall 的streamNum
1765 150 : if (deviceType_ == DevType::DEV_TYPE_310P3
1766 4 : && (opType == HcclCMDType::HCCL_CMD_ALLTOALL || opType == HcclCMDType::HCCL_CMD_ALLTOALLV
1767 4 : || opType == HcclCMDType::HCCL_CMD_ALLTOALLVC)) {
1768 0 : streamNum = userRankSize_ * RANK_SET_COMPUTE_CONST;
1769 0 : HCCL_DEBUG("[GetWorkspaceSubStreamNum]DEV_TYPE_310P3, the streamNum is %llu", streamNum);
1770 0 : return HCCL_SUCCESS;
1771 : }
1772 :
1773 : // 根据所用算法，选择所需的从stream数目
1774 150 : switch (algType.algoLevel0) {
1775 88 : case AlgTypeLevel0::ALG_LEVEL0_NP_MESH:
1776 88 : streamNum = userRankSize_ / moduleNum_ - HCCL_SUB_STREAM_NP_MESH;
1777 88 : break;
1778 16 : case AlgTypeLevel0::ALG_LEVEL0_8P_RING:
1779 16 : streamNum = HCCL_SUB_STREAM_NUM_8P_RING;
1780 16 : break;
1781 0 : case AlgTypeLevel0::ALG_LEVEL0_NP_DOUBLE_RING:
1782 0 : streamNum = HCCL_SUB_STREAM_NUM_DOUBLE_RING;
1783 0 : break;
1784 12 : case AlgTypeLevel0::ALG_LEVEL0_4P_MESH:
1785 12 : streamNum = HCCL_SUB_STREAM_NUM_4P_MESH;
1786 12 : break;
1787 34 : default:
1788 34 : streamNum = HCCL_SUB_STREAM_NUM_ZERO;
1789 34 : break;
1790 : }
1791 :
1792 150 : if (SatisfyIntraSuperPod(deviceType_, userRankSize_, useSuperPodMode_, superPodNum_)) {
1793 0 : streamNum = std::max(static_cast<u64>(userRankSize_ - 1u), streamNum);
1794 449 : } else if (FullmeshPairwiseSatisfyHighPerfAlltoallMeshCondition(
1795 149 : deviceType_, meshAggregationRankSize_, useSuperPodMode_,
1796 299 : commConfig_.GetConfigHcclAlgo(HcclCMDType::HCCL_CMD_ALLTOALL))) {
1797 0 : streamNum = std::max(static_cast<u64>(meshAggregationRankSize_ - 1u), streamNum);
1798 : }
1799 :
1800 150 : auto iter = HCCL_ALGO_LEVEL0_NAME_MAP.find(algType.algoLevel0);
1801 149 : CHK_PRT_RET(
1802 : iter == HCCL_ALGO_LEVEL0_NAME_MAP.end(),
1803 : HCCL_ERROR("[GetWorkspaceSubStreamNum]level0: algType[%u] is invalid.", algType.algoLevel0), HCCL_E_INTERNAL);
1804 149 : HCCL_DEBUG(
1805 : "[GetWorkspaceSubStreamNum]hccl algorithm: In level0, using %s algo, the streamNum is %llu",
1806 : iter->second.c_str(), streamNum);
1807 :
1808 150 : u64 sliceNum = CalculatePiplineSliceNum(opType, dataSize, algType, deviceType_, deviceNumPerServer_, serverNum_);
1809 : // 图模式下数据量固定, 按照当前数据量判断是否支持pipline切分并申请从流
1810 150 : if (implAlg_ != nullptr && sliceNum >= MIN_PIPLINE_SLICE_NUM) {
1811 0 : streamNum++;
1812 : }
1813 150 : return HCCL_SUCCESS;
1814 158 : }
1815 :
1816 806 : HcclResult HcclCommunicator::DestroyNetworkResources()
1817 : {
1818 806 : transportManager_ = nullptr;
1819 807 : if (raResourceInit_) {
1820 152 : socketManager_->DestroySockets();
1821 : }
1822 :
1823 : /* 本通信域内只有1个device时，不需要卸载ra资源 */
1824 806 : if (userRankSize_ <= 1) {
1825 209 : HCCL_INFO("user rank size <= 1, ra is not needed for single device");
1826 209 : return HCCL_SUCCESS;
1827 : }
1828 :
1829 : // nic的初始化独立调用，在此单独判断是否需要解初始化
1830 597 : if (nicInitialized_ > 0) {
1831 0 : CHK_RET(DeinitNic());
1832 : }
1833 :
1834 597 : if (raResourceInit_ && (static_cast<s32>(devicePhyId_) != HOST_DEVICE_ID) && !Is310PDevice()) {
1835 152 : CHK_RET(socketManager_->ServerDeInit(netDevCtxMap_[localVnicIp_], localVnicListenPort_));
1836 153 : HcclNetCloseDev(netDevCtxMap_[localVnicIp_]);
1837 153 : netDevCtxMap_.erase(localVnicIp_);
1838 : }
1839 :
1840 595 : CHK_RET(ReleasePreemptSocket());
1841 :
1842 597 : if (raResourceInit_) {
1843 153 : if (static_cast<s32>(devicePhyId_) != HOST_DEVICE_ID
1844 0 : || nicDeployment_ == NICDeployment::NIC_DEPLOYMENT_DEVICE) {
1845 153 : if (IsEnableBackupLink()) {
1846 : // 超节点 && level2支持重执行 && Aicpu -> 释放主备hccp资源
1847 0 : CHK_RET(HcclNetDeInit(NICDeployment::NIC_DEPLOYMENT_DEVICE, devicePhyId_, deviceLogicId_));
1848 0 : CHK_RET(HcclNetDeInit(
1849 : NICDeployment::NIC_DEPLOYMENT_DEVICE, deviceBackUpPhyId_, deviceBackUpLogicId_, true));
1850 0 : HCCL_DEBUG(
1851 : "[%s]Default & backup HcclNetDeInit, deviceLogicId[%d], devicePhyId[%u], "
1852 : "deviceBackUpPhyId_[%u], deviceBackUpLogicId_[%u], nicDeployment_[%d], IsEnableBackupLink[%d]",
1853 : __func__, deviceLogicId_, devicePhyId_, deviceBackUpPhyId_, deviceBackUpLogicId_, nicDeployment_,
1854 : IsEnableBackupLink());
1855 : } else {
1856 153 : CHK_RET(HcclNetDeInit(NICDeployment::NIC_DEPLOYMENT_DEVICE, devicePhyId_, deviceLogicId_));
1857 : }
1858 : }
1859 :
1860 153 : if ((static_cast<s32>(devicePhyId_) != HOST_DEVICE_ID && isHaveCpuRank_)
1861 153 : || (IsEnableRoce() && nicDeployment_ == NICDeployment::NIC_DEPLOYMENT_HOST)
1862 306 : || (Is310PDevice() && nicDeployment_ == NICDeployment::NIC_DEPLOYMENT_HOST)) {
1863 0 : u32 devicePhyID = (static_cast<s32>(devicePhyId_) == HOST_DEVICE_ID) ? 0 : devicePhyId_;
1864 0 : CHK_RET(HcclNetDeInit(NICDeployment::NIC_DEPLOYMENT_HOST, devicePhyID, deviceLogicId_));
1865 : }
1866 :
1867 153 : socketManager_ = nullptr;
1868 : }
1869 :
1870 597 : raResourceInit_ = false;
1871 597 : return HCCL_SUCCESS;
1872 : }
1873 :
1874 155 : HcclResult HcclCommunicator::SetWorkspaceResource(
1875 : const std::string& tag, void* memPtr, u64& maxSize, std::vector<rtStream_t>& stream)
1876 : {
1877 155 : return workSpaceRes_->SetWorkspaceResource(tag, memPtr, maxSize, stream);
1878 : }
1879 :
1880 18 : void HcclCommunicator::DestroyWorkspaceResource(const std::string& tag)
1881 : {
1882 18 : if (workSpaceRes_ == nullptr) {
1883 3 : return;
1884 : }
1885 16 : workSpaceRes_->DestroyWorkspaceResource(tag);
1886 : }
1887 :
1888 439 : HcclResult HcclCommunicator::AtomicInitSet()
1889 : {
1890 878 : CHK_PRT_RET(
1891 : initializedFlag_.test_and_set(),
1892 : HCCL_ERROR(
1893 : "[HcclCommunicator][AtomicInitSet]errNo[0x%016llx] instance "
1894 : "already been initialized",
1895 : HCCL_ERROR_CODE(HCCL_E_INTERNAL)),
1896 : HCCL_E_INTERNAL);
1897 438 : return HCCL_SUCCESS;
1898 : }
1899 :
1900 5 : void HcclCommunicator::AtomicInitClear() { initializedFlag_.clear(); }
1901 :
1902 347 : u32 HcclCommunicator::GetUserRank() { return realUserRank_; }
1903 :
1904 72 : u32 HcclCommunicator::GetGroupRank() { return userRank_; }
1905 :
1906 338 : u32 HcclCommunicator::GetRankSize() { return userRankSize_; }
1907 :
1908 0 : u32 HcclCommunicator::GetRankInParentComm() { return rankInParentComm_; }
1909 :
1910 0 : bool HcclCommunicator::GetNicInitialized() { return nicInitialized_ > 0; }
1911 :
1912 : /*
1913 : 1. 选择算法
1914 : 2. 计算resource，存到request内
1915 : 3. 创建和分配资源
1916 : */
1917 0 : HcclResult HcclCommunicator::HcclSelectAlg(
1918 : HcclCMDType opType, u64 count, void* counts, HcclDataType dataType, HcclReduceOp op, int32_t aivCoreLimit,
1919 : bool& ifAiv, std::string& algName)
1920 : {
1921 0 : HCCL_INFO(
1922 : "[HcclCommunicator][HcclSelectAlg] start to run with opType[%d], count[%llu], dataType[%d], reduceOp[%d], "
1923 : "aivCoreLimit[%d]",
1924 : opType, count, dataType, op, aivCoreLimit);
1925 0 : ifAiv = false;
1926 0 : if (opType == HcclCMDType::HCCL_CMD_REDUCE_SCATTER_V || opType == HcclCMDType::HCCL_CMD_ALLGATHER_V
1927 0 : || opType == HcclCMDType::HCCL_CMD_BATCH_SEND_RECV || opType == HcclCMDType::HCCL_CMD_BATCH_WRITE) {
1928 0 : HCCL_INFO("[HcclCommunicator][HcclSelectAlg] opType[%d] no need select AIV algorithm", opType);
1929 0 : return HCCL_SUCCESS;
1930 : }
1931 : /* 选择算法前，先更新成图模式 */
1932 0 : auto originWorkflowMode = GetWorkflowMode();
1933 0 : SetWorkflowMode(HcclWorkflowMode::HCCL_WORKFLOW_MODE_OPS_KERNEL_INFO_LIB);
1934 :
1935 0 : std::unique_ptr<CollAlgOperator> algOperator = implAlg_->GetAlgOperator(opType);
1936 0 : CHK_SMART_PTR_NULL(algOperator);
1937 :
1938 0 : OpParam param;
1939 0 : param.reduceType = op;
1940 0 : param.opType = opType;
1941 0 : if (opType == HcclCMDType::HCCL_CMD_ALLTOALL || opType == HcclCMDType::HCCL_CMD_ALLTOALLV
1942 0 : || opType == HcclCMDType::HCCL_CMD_ALLTOALLVC) {
1943 0 : param.All2AllDataDes.sendType = dataType;
1944 0 : param.All2AllDataDes.recvType = dataType;
1945 0 : param.All2AllDataDes.sendCount = count;
1946 0 : } else if (opType == HcclCMDType::HCCL_CMD_ALLGATHER_V || opType == HcclCMDType::HCCL_CMD_REDUCE_SCATTER_V) {
1947 0 : param.VDataDes.counts = counts;
1948 0 : param.VDataDes.dataType = dataType;
1949 : } else {
1950 0 : param.DataDes.count = count;
1951 0 : param.DataDes.dataType = dataType;
1952 : }
1953 :
1954 0 : AlgDesc algDesc;
1955 0 : std::string newTag;
1956 0 : ResourceLimit limit{true, true, 0};
1957 0 : limit.aivCoreLimit = aivCoreLimit;
1958 0 : CHK_RET(algOperator->SelectAlg("", param, limit, algName, algDesc, newTag));
1959 :
1960 : /* 非AIV算法直接返回 */
1961 0 : if (!algDesc.isAivMode) {
1962 0 : HCCL_INFO("[HcclCommunicator][HcclSelectAlg] select non-Aiv alg, early return");
1963 0 : return HCCL_SUCCESS;
1964 : }
1965 :
1966 : /* 完成算法选择和记录后，恢复成原来的模式 */
1967 0 : SetWorkflowMode(originWorkflowMode);
1968 0 : ifAiv = true;
1969 0 : HCCL_INFO("[HcclCommunicator][HcclSelectAlg] compile for aiv, select algName is [%s]", algName.c_str());
1970 0 : return HCCL_SUCCESS;
1971 0 : }
1972 :
1973 0 : HcclResult HcclCommunicator::HcclCalcNumBlocks(
1974 : HcclCMDType opType, u64 count, void* counts, HcclDataType dataType, int32_t aivCoreLimit, std::string& algName,
1975 : u32& numBlocks)
1976 : {
1977 0 : auto originWorkflowMode = GetWorkflowMode();
1978 0 : SetWorkflowMode(HcclWorkflowMode::HCCL_WORKFLOW_MODE_OPS_KERNEL_INFO_LIB);
1979 0 : std::unique_ptr<CollAlgOperator> algOperator = implAlg_->GetAlgOperator(opType);
1980 0 : CHK_SMART_PTR_NULL(algOperator);
1981 0 : OpParam param;
1982 :
1983 0 : param.opType = opType;
1984 0 : if (opType == HcclCMDType::HCCL_CMD_ALLTOALL || opType == HcclCMDType::HCCL_CMD_ALLTOALLV
1985 0 : || opType == HcclCMDType::HCCL_CMD_ALLTOALLVC) {
1986 0 : param.All2AllDataDes.sendType = dataType;
1987 0 : param.All2AllDataDes.recvType = dataType;
1988 0 : param.All2AllDataDes.sendCount = count;
1989 0 : } else if (opType == HcclCMDType::HCCL_CMD_ALLGATHER_V || opType == HcclCMDType::HCCL_CMD_REDUCE_SCATTER_V) {
1990 0 : param.VDataDes.counts = counts;
1991 0 : param.VDataDes.dataType = dataType;
1992 : } else {
1993 0 : param.DataDes.count = count;
1994 0 : param.DataDes.dataType = dataType;
1995 : }
1996 :
1997 0 : CHK_PRT_RET(
1998 : algOperator->CalNumBlocks(algName, param, numBlocks, aivCoreLimit) != HCCL_SUCCESS,
1999 : HCCL_ERROR("[%s] CalNumBlocks failed", __func__), HCCL_E_PARA);
2000 0 : SetWorkflowMode(originWorkflowMode);
2001 0 : return HCCL_SUCCESS;
2002 0 : }
2003 :
2004 1 : HcclResult HcclCommunicator::HcclGetAlgExecParam(
2005 : const std::string& tag, HcclCMDType opType, u64 count, void* inputPtr, void* outputPtr, bool clearEnable,
2006 : HcclDataType dataType, HcclReduceOp op, void*& commContext, u64& len, u32 aivCoreLimit)
2007 : {
2008 : /* 将Host申请和注册好的资源，传给AICPU */
2009 : // 1\ algName 从getstr里某一个名字里获取出来（要防止名字重复） commContext & len 从 response里拿
2010 : // 2\ rtmemcopy 先获取一下algoperator对象，用这个调用getalgxxx
2011 1 : AivSuperKernelArgs aivSuperKernelArgs;
2012 1 : SetWorkflowMode(HcclWorkflowMode::HCCL_WORKFLOW_MODE_OPS_KERNEL_INFO_LIB);
2013 :
2014 1 : OpParam param;
2015 1 : param.DataDes.count = count;
2016 1 : param.DataDes.dataType = dataType;
2017 1 : param.reduceType = op;
2018 1 : param.tag = tag;
2019 1 : param.inputPtr = inputPtr;
2020 1 : param.outputPtr = outputPtr;
2021 1 : param.opType = opType;
2022 : u64 totalSize;
2023 1 : std::vector<u64> sendCountMatrix(userRankSize_ * userRankSize_, count);
2024 1 : if (opType == HcclCMDType::HCCL_CMD_ALLTOALL) {
2025 0 : param.All2AllDataDes.sendType = dataType;
2026 0 : param.All2AllDataDes.recvType = dataType;
2027 0 : param.All2AllDataDes.sendCount = count;
2028 0 : param.All2AllDataDes.sendCountMatrix = static_cast<void*>(sendCountMatrix.data());
2029 : }
2030 :
2031 1 : if (opType == HcclCMDType::HCCL_CMD_REDUCE_SCATTER || opType == HcclCMDType::HCCL_CMD_ALLTOALL) {
2032 0 : totalSize = count * SIZE_TABLE[dataType] * userRankSize_;
2033 : } else {
2034 1 : totalSize = count * SIZE_TABLE[dataType]; // allreduce就是输入
2035 : }
2036 1 : param.inputSize = totalSize;
2037 1 : std::unique_ptr<CollAlgOperator> algOperator = implAlg_->GetAlgOperator(opType);
2038 1 : CHK_SMART_PTR_NULL(algOperator);
2039 1 : std::string algName;
2040 1 : AlgResourceResponse algResResponse;
2041 1 : std::string newTag;
2042 1 : ResourceLimit limit;
2043 1 : limit.ifLimit = true;
2044 1 : limit.aivCoreLimit = aivCoreLimit;
2045 1 : AlgDesc algDesc;
2046 1 : algDesc.isLastSelect = true;
2047 1 : CHK_RET(algOperator->SelectAlg(param.tag, param, limit, algName, algDesc, newTag));
2048 :
2049 : // 资源创建
2050 1 : InsertNewTagToTagMap(newTag, param.tag);
2051 1 : if (resMap_.find(newTag) == resMap_.end()) {
2052 1 : HCCL_INFO("[HcclCoommunicator][HcclAllocRes] algName[%s], alloc new res", algName.c_str());
2053 1 : AlgResourceRequest resRequest;
2054 1 : CHK_RET(algOperator->CalcResRequest(algName, param, resRequest)); // [重构建议] 计算和alloc可以拆开
2055 1 : CHK_RET(AllocAlgResource(newTag, opType, param, resRequest, resMap_[newTag]));
2056 1 : CHK_RET(algOperator->PrepareCommInfoToDevice(algName, resMap_[newTag]));
2057 : // 暂不作心跳注册
2058 1 : }
2059 :
2060 1 : CHK_RET(algOperator->GetAivExecParam(algName, param, resMap_[newTag], aivSuperKernelArgs));
2061 :
2062 : // gettag
2063 1 : HCCL_INFO("SPK, rank %llu.", userRank_);
2064 : u32 numBlocks;
2065 1 : CHK_PRT_RET(
2066 : algOperator->CalNumBlocks(algName, param, numBlocks, aivCoreLimit) != HCCL_SUCCESS,
2067 : HCCL_ERROR("[%s] CalNumBlocks failed", __func__), HCCL_E_PARA);
2068 1 : if (clearEnable) {
2069 1 : aivOffloadTag_ = 1;
2070 : }
2071 1 : GetAivTag(algDesc.aivTagNum, false, aivSuperKernelArgs.tag); // workflowmode为图模式
2072 1 : aivSuperKernelArgs.numBlocks = numBlocks;
2073 :
2074 1 : HCCL_INFO(
2075 : "SPK, Tag %llu aivCoreLimit %u, numBlocks %llu.", aivSuperKernelArgs.tag, aivCoreLimit,
2076 : aivSuperKernelArgs.numBlocks);
2077 : // clearenable
2078 : // 拷贝到Device
2079 1 : SetWorkflowMode(HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE);
2080 :
2081 1 : void* sendAlgParamMemPtr = nullptr;
2082 : // alloc device 地址
2083 1 : CHK_RET(hrtMalloc(&sendAlgParamMemPtr, sizeof(AivSuperKernelArgs)));
2084 1 : HCCL_INFO("SPK sendalgparam %p.", sendAlgParamMemPtr);
2085 :
2086 1 : HcclResult hcclRet = hrtMemSyncCopy(
2087 : sendAlgParamMemPtr, sizeof(AivSuperKernelArgs), &aivSuperKernelArgs, sizeof(AivSuperKernelArgs),
2088 : HcclRtMemcpyKind::HCCL_RT_MEMCPY_KIND_HOST_TO_DEVICE);
2089 1 : if (hcclRet != HCCL_SUCCESS) {
2090 0 : HCCL_ERROR("[HcclCommunicator][%s]hrtMemSyncCopy error, ret[%d]", __func__, hcclRet);
2091 0 : CHK_RET(hrtFree(sendAlgParamMemPtr));
2092 0 : return hcclRet;
2093 : }
2094 1 : commContext = sendAlgParamMemPtr;
2095 1 : len = sizeof(AivSuperKernelArgs);
2096 1 : return HCCL_SUCCESS;
2097 1 : }
2098 :
2099 6 : HcclResult HcclCommunicator::GetAivTag(s32 tagNum, bool isCapture, s32& aivTag)
2100 : {
2101 6 : bool useOpbaseFlag = (GetWorkflowMode() == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE && !isCapture);
2102 6 : if (useOpbaseFlag) {
2103 2 : aivTag = aivOpbaseTag_;
2104 2 : aivOpbaseTag_ = GetNextAivTag(aivOpbaseTag_, tagNum);
2105 : } else {
2106 4 : aivTag = aivOffloadTag_;
2107 4 : aivOffloadTag_ = GetNextAivTag(aivOffloadTag_, tagNum);
2108 : }
2109 6 : return HCCL_SUCCESS;
2110 : }
2111 :
2112 2 : HcclResult HcclCommunicator::CheckDeviceType(const DevType deviceType) const
2113 : {
2114 2 : if ((deviceType >= DevType::DEV_TYPE_COUNT) || (deviceType < DevType::DEV_TYPE_910)) {
2115 2 : HCCL_ERROR(
2116 : "[Check][DeviceType]errNo[0x%016llx] device Type[%d] out of range[%d, %d]", HCCL_ERROR_CODE(HCCL_E_PARA),
2117 : deviceType, DevType::DEV_TYPE_910, DevType::DEV_TYPE_NOSOC);
2118 2 : return HCCL_E_PARA;
2119 : }
2120 0 : return HCCL_SUCCESS;
2121 : }
2122 :
2123 44 : HcclResult HcclCommunicator::CheckReductionOp(const HcclReduceOp op) const
2124 : {
2125 44 : if ((op >= HCCL_REDUCE_RESERVED) || (op < HCCL_REDUCE_SUM)) {
2126 0 : HCCL_ERROR("[Check][ReductionOp]errNo[0x%016llx] op:[%d] not supported", HCCL_ERROR_CODE(HCCL_E_PARA), op);
2127 1 : return HCCL_E_PARA;
2128 : }
2129 46 : return HCCL_SUCCESS;
2130 : }
2131 :
2132 133 : HcclResult HcclCommunicator::CheckUserRank(const u32 userRank) const
2133 : {
2134 133 : if (userRankSize_ <= userRank) {
2135 3 : HCCL_ERROR(
2136 : "[Check][UserRank]errNo[0x%016llx] userRank:[%u] is out of range[0 ~ %u]", HCCL_ERROR_CODE(HCCL_E_PARA),
2137 : userRank, userRankSize_);
2138 3 : return HCCL_E_PARA;
2139 : }
2140 130 : return HCCL_SUCCESS;
2141 : }
2142 :
2143 92 : HcclResult HcclCommunicator::CheckCount(const u64 count) const
2144 : {
2145 92 : if (count > SYS_MAX_COUNT) {
2146 1 : HCCL_ERROR(
2147 : "[Check][Count]errNo[0x%016llx] count[%llu] is invalid(bigger than MAX count[%llu])",
2148 : HCCL_ERROR_CODE(HCCL_E_PARA), count, SYS_MAX_COUNT);
2149 1 : return HCCL_E_PARA;
2150 : }
2151 91 : return HCCL_SUCCESS;
2152 : }
2153 :
2154 0 : HcclResult HcclCommunicator::GetGroupRanksInfo(const std::vector<u32>& groupRanks, std::vector<RankInfo>& ranksInfo)
2155 : {
2156 0 : ranksInfo.clear();
2157 0 : std::vector<RankInfo> tmpRankInfoList;
2158 0 : tmpRankInfoList.assign(rankInfoList_.begin(), rankInfoList_.end());
2159 :
2160 0 : for (u32 index = 0; index < groupRanks.size(); index++) {
2161 0 : if (tmpRankInfoList.size() <= groupRanks[index]) {
2162 0 : HCCL_ERROR(
2163 : "[Get][GroupRanksInfo]errNo[0x%016llx] groupRanks[%u]=[%u], >= rankinfolist size[%zu]",
2164 : HCCL_ERROR_CODE(HCCL_E_PARA), index, groupRanks[index], tmpRankInfoList.size());
2165 0 : return HCCL_E_PARA;
2166 : }
2167 0 : tmpRankInfoList[groupRanks[index]].userRank = index;
2168 0 : ranksInfo.push_back(tmpRankInfoList[groupRanks[index]]);
2169 0 : HCCL_DEBUG(
2170 : "index: %d userRank: %dhost ip: %s host port: %u dev phy id: %d serverIdx:%d", index,
2171 : tmpRankInfoList[groupRanks[index]].userRank, tmpRankInfoList[groupRanks[index]].hostIp.GetReadableAddress(),
2172 : tmpRankInfoList[groupRanks[index]].hostPort, tmpRankInfoList[groupRanks[index]].devicePhyId,
2173 : tmpRankInfoList[groupRanks[index]].serverIdx);
2174 : }
2175 :
2176 : // 按rank id从小到大的顺序返回
2177 0 : std::sort(ranksInfo.begin(), ranksInfo.end(), CompareWithUserRank);
2178 :
2179 0 : for (u32 index = 0; index < ranksInfo.size(); ++index) {
2180 0 : if (index != ranksInfo[index].userRank) {
2181 0 : HCCL_ERROR(
2182 : "[Get][GroupRanksInfo]errNo[0x%016llx] index[%u] != user rank[%u]", HCCL_ERROR_CODE(HCCL_E_PARA), index,
2183 : ranksInfo[index].userRank);
2184 0 : return HCCL_E_PARA;
2185 : }
2186 : }
2187 0 : return HCCL_SUCCESS;
2188 0 : }
2189 :
2190 0 : HcclResult HcclCommunicator::GetGroupCommonData(WorldGroupInfo& groupCommonData) const
2191 : {
2192 0 : groupCommonData.inlineReduceSwitchOn = inlineReduceSwitchOn_;
2193 0 : groupCommonData.deviceType = deviceType_;
2194 0 : groupCommonData.deviceLogicId = deviceLogicId_;
2195 0 : groupCommonData.profilingInitiated = profilingInitiated_;
2196 0 : groupCommonData.serverId = serverId_;
2197 0 : groupCommonData.phyIdNicInfoMap = rankDevicePhyIdNicInfoMap_;
2198 0 : groupCommonData.worldRankInfoList = rankInfoList_;
2199 0 : groupCommonData.ranksPort = nicRanksPort_;
2200 0 : groupCommonData.vnicRanksPort = vnicRanksPort_;
2201 0 : groupCommonData.useSuperPodMode = useSuperPodMode_;
2202 0 : groupCommonData.devPortSwitchOn = commPortConfig_.devPortSwitchOn;
2203 0 : return HCCL_SUCCESS;
2204 : }
2205 :
2206 91 : HcclResult HcclCommunicator::GetWorkspaceMemSize(
2207 : const std::string& opType, u64 count, HcclDataType dataType, u32& rankSize, u64& memSize, DevType& deviceType) const
2208 : {
2209 91 : return workSpaceRes_->GetWorkspaceMemSize(opType, count, dataType, rankSize, memSize, deviceType);
2210 : }
2211 :
2212 24 : DeviceMem HcclCommunicator::GetWorkspaceScracthMem(const std::string& tag, u64 allocMemSize)
2213 : {
2214 24 : return workSpaceRes_->AllocDeviceMem(tag, allocMemSize);
2215 : }
2216 :
2217 15 : std::vector<Stream> HcclCommunicator::GetWorkspaceSubStreams(const std::string& tag, u32 num)
2218 : {
2219 15 : return workSpaceRes_->AllocSlaveStreams(tag, num);
2220 : }
2221 :
2222 491 : HcclResult HcclCommunicator::InitProfiling()
2223 : {
2224 491 : if (static_cast<s32>(devicePhyId_) == HOST_DEVICE_ID) {
2225 0 : HCCL_ERROR("[Init][Profiling]not support cpu rank");
2226 0 : return HCCL_E_NOT_SUPPORT;
2227 : }
2228 491 : CHK_PRT_RET(profilingInitiated_, HCCL_DEBUG("Profiling plugin has already been Initiated."), HCCL_SUCCESS);
2229 :
2230 491 : if (profilingMode_ != HcomProfilingMode::PROFILING_OPEN && GetExternalInputProfilingMode()) {
2231 0 : profilingMode_ = HcomProfilingMode::PROFILING_OPEN;
2232 0 : profilingOption_ = GetExternalInputProfilingOption();
2233 : }
2234 492 : HCCL_INFO("profiling config information:options[%s], mode[%d]", profilingOption_.c_str(), profilingMode_);
2235 :
2236 : // profilingInitiated_会广播给所有子通信域，用于避免taskInfoSaver的重复初始化
2237 491 : profilingInitiated_ = true;
2238 : // isExecuteProfilingInit_用于记录本impl是否执行了taskInfoSaver的初始化，用于进行对应的释放
2239 491 : isExecuteProfilingInit_ = true;
2240 491 : return HCCL_SUCCESS;
2241 : }
2242 :
2243 491 : HcclResult HcclCommunicator::DeinitProfiling()
2244 : {
2245 491 : CHK_PRT_RET(!profilingInitiated_, HCCL_DEBUG("Profiling plugin has not been Initiated"), HCCL_SUCCESS);
2246 491 : profilingInitiated_ = false;
2247 491 : HCCL_INFO("Profiling is deinitiated.");
2248 491 : return HCCL_SUCCESS;
2249 : }
2250 :
2251 491 : HcclResult HcclCommunicator::RegistTaskExceptionHandler() const
2252 : {
2253 491 : CHK_RET(TaskExceptionHandler::Init());
2254 491 : return HCCL_SUCCESS;
2255 : }
2256 :
2257 807 : HcclResult HcclCommunicator::UnRegistTaskExceptionHandler() const
2258 : {
2259 807 : CHK_RET(TaskExceptionHandler::DeInit());
2260 806 : return HCCL_SUCCESS;
2261 : }
2262 :
2263 0 : HcclResult HcclCommunicator::GetInCCLbuffer(void*& buffer, u64& size)
2264 : {
2265 0 : return cclBufferManager_.GetInCCLbuffer(buffer, size);
2266 : }
2267 :
2268 0 : HcclResult HcclCommunicator::GetOutCCLbuffer(void*& buffer, u64& size)
2269 : {
2270 0 : return cclBufferManager_.GetOutCCLbuffer(buffer, size);
2271 : }
2272 :
2273 0 : void HcclCommunicator::ReleaseCommCCLbuffer() { cclBufferManager_.ReleaseCommCCLbuffer(); }
2274 :
2275 0 : HcclResult HcclCommunicator::ReleaseCommInfos()
2276 : {
2277 0 : if (implAlg_ != nullptr) {
2278 0 : return implAlg_->ReleaseCommInfos();
2279 : }
2280 0 : return HCCL_SUCCESS;
2281 : }
2282 :
2283 312 : HcclResult HcclCommunicator::InitProfiler()
2284 : {
2285 312 : profilerManager_.reset(new (std::nothrow)
2286 312 : ProfilerManager(devicePhyId_, deviceLogicId_, realUserRank_, userRankSize_));
2287 312 : CHK_SMART_PTR_NULL(profilerManager_);
2288 312 : HcclResult ret = profilerManager_->InitProfiler();
2289 312 : CHK_PRT_RET(
2290 : (ret != HCCL_SUCCESS), HCCL_ERROR("[BASE][InitProfiler]profilerManager_ InitProfiler failed."), HCCL_E_PARA);
2291 :
2292 312 : HCCL_INFO("[BASE][InitProfiler]Register CtrlCallBack success.");
2293 312 : return HCCL_SUCCESS;
2294 : }
2295 :
2296 72 : HcclResult HcclCommunicator::CreateCommCCLbuffer()
2297 : {
2298 : // user mem和CCL buffer互斥，不支持同时使用
2299 72 : if (isUserMemRegisted_) {
2300 0 : HCCL_ERROR(
2301 : "[HcclCommunicator][%s]tag[%s]The user mem has been registered, "
2302 : "does not support create CCL Buffer.",
2303 : __func__, identifier_.c_str());
2304 0 : return HCCL_E_NOT_SUPPORT;
2305 : }
2306 72 : return cclBufferManager_.CreateCommCCLbuffer(cclBuffName_);
2307 : }
2308 :
2309 242 : HcclResult HcclCommunicator::InitCCLbuffer(u64 inCCLbufferSize, u64 outCCLbufferSize)
2310 : {
2311 242 : return cclBufferManager_.InitCCLbuffer(inCCLbufferSize, outCCLbufferSize);
2312 : }
2313 :
2314 180 : u32 HcclCommunicator::GetLocalNicPort(NicType nicType)
2315 : {
2316 180 : u32 port = HCCL_INVALID_PORT;
2317 180 : if (nicDeployment_ == NICDeployment::NIC_DEPLOYMENT_HOST) {
2318 0 : return GetHostPort(devicePhyId_);
2319 : }
2320 : // isUseRankPort_在ranksPort初始化时一同配置：1. 异构场景 2. 开启device侧端口配置
2321 : // groupRanksPort_为空说明此时处于全局通信域，要从ranksPort_取监听端口；否则取groupRanksPort_
2322 180 : bool devicePortSwitchOn = commPortConfig_.devPortSwitchOn;
2323 180 : if (nicType == NicType::HOST_NIC_TYPE) {
2324 0 : port = GetHostPort(devicePhyId_);
2325 180 : } else if (devicePortSwitchOn && nicType == NicType::VNIC_TYPE) {
2326 : // vnic ports仅在开启device侧端口配置时单独配置
2327 0 : std::vector<u32>& ranksPorts = groupVnicRanksPort_.empty() ? vnicRanksPort_ : groupVnicRanksPort_;
2328 0 : port = GetNicPort(devicePhyId_, ranksPorts, userRank_, isUseRankPort_);
2329 0 : } else {
2330 : // 1. 开启device侧端口配置时的nic port时使用ranksPorts
2331 : // 2. 异构场景使用ranksPorts
2332 : // 3. 其余场景场景isUseRankPort_应当为false，使用默认port
2333 180 : std::vector<u32>& ranksPorts = groupNicRanksPort_.empty() ? nicRanksPort_ : groupNicRanksPort_;
2334 180 : port = GetNicPort(devicePhyId_, ranksPorts, userRank_, isUseRankPort_);
2335 : }
2336 180 : HCCL_INFO(
2337 : "[HcclCommunicator][GetLocalNicPort] nicType[%u], devicePortSwitchOn[%u], isUseRankPort[%u], "
2338 : "get port[%u], devId[%u]",
2339 : nicType, devicePortSwitchOn, isUseRankPort_, port, devicePhyId_);
2340 180 : return port;
2341 : }
2342 :
2343 1 : HcclResult HcclCommunicator::CheckOneSidedBackupAndSetDevId(
2344 : u32& backupDevPhyId, u32& backupDevLogicId, std::vector<HcclIpAddress>& localIpList,
2345 : bool& isOneSidedTaskAndBackupInitA3)
2346 : {
2347 1 : if (!IsOneSidedIdentifier(identifier_)) {
2348 1 : isOneSidedTaskAndBackupInitA3 = false;
2349 1 : HCCL_INFO("[%s] comm[%s] is not one sided comm.", __func__, identifier_.c_str());
2350 1 : return HCCL_SUCCESS;
2351 : }
2352 0 : DevType deviceType = DevType::DEV_TYPE_COUNT;
2353 0 : CHK_RET(hrtGetDeviceType(deviceType));
2354 0 : if (deviceType != DevType::DEV_TYPE_910_93) {
2355 0 : isOneSidedTaskAndBackupInitA3 = false;
2356 0 : HCCL_INFO(
2357 : "[HcclCommunicator::CheckOneSidedBackupAndSetDevId] DeviceType[%d] is not 910_93, one sided backup not "
2358 : "support",
2359 : static_cast<u32>(deviceType));
2360 0 : return HCCL_SUCCESS;
2361 : }
2362 0 : CHK_RET(hrtGetPairDevicePhyId(devicePhyId_, backupDevPhyId));
2363 :
2364 0 : std::vector<HcclIpAddress> backupIpList;
2365 0 : std::vector<std::vector<HcclIpAddress>> chipDeviceIPs;
2366 0 : CHK_RET(hrtRaGetDeviceAllNicIP(chipDeviceIPs));
2367 0 : u32 ipIdex = 1U - (devicePhyId_ % 2U);
2368 0 : std::copy_if(
2369 0 : chipDeviceIPs[ipIdex].begin(), chipDeviceIPs[ipIdex].end(), std::back_inserter(backupIpList),
2370 0 : [](const HcclIpAddress& ip) {
2371 0 : return !ip.IsIPv6();
2372 : });
2373 0 : HCCL_INFO(
2374 : "devicePhysicID[%u], backupDeviceId[%d], backupDeviceIP[0]:[%s], devIpAddr_[%s], ", devicePhyId_,
2375 : backupDevPhyId, backupIpList[0].GetReadableAddress(), devIpAddr_[0].GetReadableAddress());
2376 0 : CHK_RET(hrtRaGetDeviceIP(devicePhyId_, localIpList));
2377 0 : auto equalToLocal = [this](const HcclIpAddress& entry) {
2378 0 : return entry == devIpAddr_[0];
2379 0 : };
2380 0 : isOneSidedTaskAndBackupInitA3 = any_of(backupIpList.begin(), backupIpList.end(), equalToLocal)
2381 0 : && !any_of(localIpList.begin(), localIpList.end(), equalToLocal);
2382 0 : if (isOneSidedTaskAndBackupInitA3) {
2383 0 : CHK_RET(hrtGetDeviceIndexByPhyId(backupDevPhyId, backupDevLogicId));
2384 : }
2385 :
2386 0 : HCCL_INFO(
2387 : "[HcclCommunicator::CheckOneSidedBackupAndSetDevId] isOneSidedTaskAndBackupInitA3[%s]",
2388 : isOneSidedTaskAndBackupInitA3 ? "true" : "false");
2389 0 : return HCCL_SUCCESS;
2390 0 : }
2391 :
2392 0 : HcclResult HcclCommunicator::OneSidedBackupInitNetResource(
2393 : HcclNetDevCtx& nicPortBackUpCtx, u32& backupDevPhyId, u32& backupDevLogicId,
2394 : std::vector<HcclIpAddress>& localIpList)
2395 : {
2396 0 : devBackupIpAddr_[0] = devIpAddr_[0];
2397 0 : deviceBackUpPhyId_ = backupDevPhyId;
2398 0 : deviceBackUpLogicId_ = backupDevLogicId;
2399 0 : CHK_RET(HcclNetInit(NICDeployment::NIC_DEPLOYMENT_DEVICE, backupDevPhyId, backupDevLogicId, false, true));
2400 0 : HCCL_INFO(
2401 : "[HcclCommunicator::OneSidedBackupInitNetResource] OpenDev with backupDevPhyId[%d], backupDevLogicId[%d], "
2402 : "localIpList[%s], backupIp[%s]",
2403 : backupDevPhyId, backupDevLogicId, localIpList[0].GetReadableAddress(), devIpAddr_[0].GetReadableAddress());
2404 0 : CHK_RET(HcclNetOpenDev(
2405 : &nicPortBackUpCtx, NicType::DEVICE_NIC_TYPE, backupDevPhyId, backupDevLogicId, devIpAddr_[0], localIpList[0]));
2406 0 : return HCCL_SUCCESS;
2407 : }
2408 :
2409 0 : HcclResult HcclCommunicator::OneSidedBackupServerInit(HcclNetDevCtx& nicPortBackUpCtx)
2410 : {
2411 0 : u32 backupPort = HCCL_INVALID_PORT;
2412 0 : for (const auto& rankInfo : rankInfoList_) {
2413 0 : if (rankInfo.userRank == userRank_) {
2414 0 : backupPort = rankInfo.deviceNicPort;
2415 : }
2416 : }
2417 0 : CHK_RET(socketManager_->ServerInit(nicPortBackUpCtx, backupPort));
2418 0 : return HCCL_SUCCESS;
2419 : }
2420 :
2421 0 : HcclResult HcclCommunicator::InitDevicePrimaryNic(bool isMC2ReInit, bool isOneSidedTaskAndBackupInitA3)
2422 : {
2423 0 : std::shared_ptr<HcclSocket>& devNicSocket = commPortConfig_.devNicListen.first;
2424 0 : if (devNicSocket && !isOneSidedTaskAndBackupInitA3) {
2425 0 : HcclNetDevCtx& devNicCtx = commPortConfig_.devNicListen.second;
2426 0 : CHK_PTR_NULL(devNicCtx);
2427 0 : netDevCtxMap_.insert(std::make_pair(devNicSocket->GetLocalIp(), devNicCtx));
2428 0 : CHK_RET(socketManager_->ServerInit(devNicCtx, devNicSocket->GetLocalPort()));
2429 0 : commPortConfig_.devNicListen.second = nullptr;
2430 0 : HCCL_INFO(
2431 : "[HcclCommunicator][InitNic] init nic with listened socket success, "
2432 : "listened ip[%s] port[%u]",
2433 : devNicSocket->GetLocalIp().GetReadableAddress(), devNicSocket->GetLocalPort());
2434 0 : } else if (!isOneSidedTaskAndBackupInitA3) {
2435 0 : u32 port = GetLocalNicPort(NicType::DEVICE_NIC_TYPE);
2436 0 : u32 nicNum = devIpAddr_.size();
2437 0 : for (u32 i = 0; i < nicNum; i++) {
2438 0 : if (devIpAddr_[i].IsInvalid()) {
2439 0 : HCCL_INFO("[Init][Nic]nic num[%u] deviceip is invalid, total nicNum[%u]", i, nicNum);
2440 0 : continue;
2441 : }
2442 : HcclNetDevCtx nicPortCtx;
2443 0 : CHK_RET(HcclNetOpenDev(&nicPortCtx, NicType::DEVICE_NIC_TYPE, devicePhyId_, deviceLogicId_, devIpAddr_[i]));
2444 0 : CHK_PTR_NULL(nicPortCtx);
2445 0 : netDevCtxMap_.insert(std::make_pair(devIpAddr_[i], nicPortCtx));
2446 0 : CHK_RET(socketManager_->ServerInit(nicPortCtx, port));
2447 0 : HCCL_INFO(
2448 : "[HcclCommunicator][InitNic] init nic with ip[%s] port[%u] success", devIpAddr_[i].GetReadableAddress(),
2449 : port);
2450 : }
2451 : }
2452 0 : return HCCL_SUCCESS;
2453 : }
2454 :
2455 0 : HcclResult HcclCommunicator::InitDeviceBackupNic(
2456 : u32 backupDevPhyId, u32 backupDevLogicId, std::vector<HcclIpAddress>& localIpList,
2457 : bool isOneSidedTaskAndBackupInitA3)
2458 : {
2459 0 : if (IsEnableBackupLink() || isOneSidedTaskAndBackupInitA3) {
2460 0 : std::shared_ptr<HcclSocket>& backupNicSocket = commPortConfig_.backupDevNicListen.first;
2461 0 : if (backupNicSocket) {
2462 0 : HcclNetDevCtx& backupNicCtx = commPortConfig_.backupDevNicListen.second;
2463 0 : CHK_PTR_NULL(backupNicCtx);
2464 0 : netDevCtxMap_.insert(std::make_pair(backupNicSocket->GetLocalIp(), backupNicCtx));
2465 0 : CHK_RET(socketManager_->ServerInit(backupNicCtx, backupNicSocket->GetLocalPort()));
2466 0 : commPortConfig_.backupDevNicListen.second = nullptr;
2467 0 : HCCL_INFO(
2468 : "[HcclCommunicator][InitNic] init backup nic with listened socket success, "
2469 : "listened ip[%s] port[%u]",
2470 : backupNicSocket->GetLocalIp().GetReadableAddress(), backupNicSocket->GetLocalPort());
2471 : } else {
2472 : HcclNetDevCtx nicPortBackUpCtx;
2473 0 : if (isOneSidedTaskAndBackupInitA3) {
2474 0 : CHK_RET(OneSidedBackupInitNetResource(nicPortBackUpCtx, backupDevPhyId, backupDevLogicId, localIpList));
2475 : } else {
2476 0 : CHK_RET(HcclNetOpenDev(
2477 : &nicPortBackUpCtx, NicType::DEVICE_NIC_TYPE, deviceBackUpPhyId_, deviceBackUpLogicId_,
2478 : devBackupIpAddr_[0], devIpAddr_[0]));
2479 : }
2480 0 : CHK_PTR_NULL(nicPortBackUpCtx);
2481 0 : netDevCtxMap_.insert(std::make_pair(devBackupIpAddr_[0], nicPortBackUpCtx));
2482 0 : if (isOneSidedTaskAndBackupInitA3) {
2483 0 : CHK_RET(OneSidedBackupServerInit(nicPortBackUpCtx));
2484 : } else {
2485 0 : CHK_RET(socketManager_->ServerInit(nicPortBackUpCtx, devBackupPort_));
2486 : }
2487 0 : HCCL_DEBUG(
2488 : "[%s]finish backup ServerInit, deviceBackUpPhyId_[%u], deviceBackUpLogicId_[%u], "
2489 : "devBackupIpAddr_[%s], devBackupPort_[%u], nicDeployment_[%d], IsEnableBackupLink[%d], "
2490 : "netDevCtxMap_.size[%d]",
2491 : __func__, deviceBackUpPhyId_, deviceBackUpLogicId_, devBackupIpAddr_[0].GetReadableAddress(),
2492 : devBackupPort_, nicDeployment_, IsEnableBackupLink(), netDevCtxMap_.size());
2493 0 : HCCL_INFO(
2494 : "[HcclCommunicator][InitNic] init backup nic with ip[%s] port[%u] success",
2495 : devBackupIpAddr_[0].GetReadableAddress(), devBackupPort_);
2496 : }
2497 : }
2498 0 : return HCCL_SUCCESS;
2499 : }
2500 :
2501 0 : HcclResult HcclCommunicator::InitNicDeviceDeploy(
2502 : bool isMC2ReInit, u32 backupDevPhyId, u32 backupDevLogicId, std::vector<HcclIpAddress>& localIpList,
2503 : bool isOneSidedTaskAndBackupInitA3)
2504 : {
2505 0 : CHK_RET(InitDevicePrimaryNic(isMC2ReInit, isOneSidedTaskAndBackupInitA3));
2506 0 : attrCollector_.GenUsedRdmaLevel0();
2507 0 : isUsedRdmaLevel0_ = attrCollector_.GetUsedRdmaLevel0();
2508 0 : CHK_RET(InitDeviceBackupNic(backupDevPhyId, backupDevLogicId, localIpList, isOneSidedTaskAndBackupInitA3));
2509 0 : return HCCL_SUCCESS;
2510 : }
2511 :
2512 0 : HcclResult HcclCommunicator::InitNicHostDeploy()
2513 : {
2514 0 : u32 port = GetLocalNicPort(NicType::HOST_NIC_TYPE);
2515 0 : CHK_PRT_RET(
2516 : (hostIp_.IsInvalid()),
2517 : HCCL_ERROR("[Init][Nic] host ip is invalid when NIC "
2518 : "deployment is host. "),
2519 : HCCL_E_PARA);
2520 0 : attrCollector_.GenUsedRdmaLevel0();
2521 0 : isUsedRdmaLevel0_ = attrCollector_.GetUsedRdmaLevel0();
2522 0 : u32 devicePhyID = (static_cast<s32>(devicePhyId_) == HOST_DEVICE_ID) ? 0 : devicePhyId_;
2523 :
2524 0 : u32 i = 0;
2525 : HcclNetDevCtx nicPortCtx;
2526 0 : for (i = 0; i < devIpAddr_.size(); i++) {
2527 0 : if (devIpAddr_[i].IsInvalid()) {
2528 0 : HCCL_INFO("[Init][Nic]nic num[%u] deviceip is invalid, total nicNum[%u]", i, devIpAddr_.size());
2529 0 : continue;
2530 : }
2531 0 : std::vector<u32>& ranksPorts = groupNicRanksPort_.empty() ? nicRanksPort_ : groupNicRanksPort_;
2532 0 : port = GetNicPort(devicePhyId_, ranksPorts, userRank_, isUseRankPort_);
2533 0 : CHK_RET(HcclNetOpenDev(&nicPortCtx, NicType::HOST_NIC_TYPE, devicePhyId_, deviceLogicId_, devIpAddr_[i]));
2534 0 : CHK_PTR_NULL(nicPortCtx);
2535 0 : netDevCtxMap_.insert(std::make_pair(devIpAddr_[i], nicPortCtx));
2536 0 : HcclNetDevSetProtoType(nicPortCtx, HCCL_PROTO_TYPE_ROCE);
2537 0 : break;
2538 : }
2539 :
2540 0 : if (i == devIpAddr_.size()) {
2541 0 : port = GetLocalNicPort(NicType::HOST_NIC_TYPE);
2542 0 : CHK_RET(HcclNetOpenDev(&nicPortCtx, NicType::HOST_NIC_TYPE, devicePhyId_, deviceLogicId_, hostIp_));
2543 0 : CHK_PTR_NULL(nicPortCtx);
2544 0 : netDevCtxMap_.insert(std::make_pair(hostIp_, nicPortCtx));
2545 : }
2546 0 : HCCL_INFO("[Init][Nic], hostPort[%u], devicePhyID[%u]", port, devicePhyID);
2547 0 : CHK_RET(socketManager_->ServerInit(nicPortCtx, port));
2548 0 : return HCCL_SUCCESS;
2549 : }
2550 :
2551 1 : HcclResult HcclCommunicator::InitNic(bool isMC2ReInit)
2552 : {
2553 1 : if (!GetExternalInputIntraRoceSwitch() && servRankInfo_.size() == 1 && isDiffDeviceModule_ && !isMC2ReInit) {
2554 0 : return HCCL_SUCCESS;
2555 : }
2556 1 : u32 backupDevPhyId = INVALID_INT;
2557 1 : u32 backupDevLogicId = INVALID_INT;
2558 1 : bool isOneSidedTaskAndBackupInitA3 = false;
2559 1 : vector<HcclIpAddress> localIpList;
2560 1 : CHK_RET(
2561 : CheckOneSidedBackupAndSetDevId(backupDevPhyId, backupDevLogicId, localIpList, isOneSidedTaskAndBackupInitA3));
2562 :
2563 1 : if (nicDeployment_ == NICDeployment::NIC_DEPLOYMENT_DEVICE) {
2564 0 : CHK_RET(InitNicDeviceDeploy(
2565 : isMC2ReInit, backupDevPhyId, backupDevLogicId, localIpList, isOneSidedTaskAndBackupInitA3));
2566 1 : } else if (nicDeployment_ == NICDeployment::NIC_DEPLOYMENT_HOST) {
2567 0 : CHK_RET(InitNicHostDeploy());
2568 : } else {
2569 1 : HCCL_ERROR("[Init][Nic]nic deployment[%d] is not supported", nicDeployment_);
2570 1 : return HCCL_E_PARA;
2571 : }
2572 0 : isNeedInitNic_ = true;
2573 0 : attrCollector_.SetNeedInitNicFlag(isNeedInitNic_);
2574 0 : nicInitialized_++;
2575 0 : return HCCL_SUCCESS;
2576 1 : }
2577 :
2578 0 : HcclResult HcclCommunicator::DeinitNicHostDeploy()
2579 : {
2580 0 : u32 port = GetLocalNicPort(NicType::HOST_NIC_TYPE);
2581 0 : CHK_PRT_RET(
2582 : (hostIp_.IsInvalid()),
2583 : HCCL_ERROR("[DeInit][Nic] host ip is invalid when NIC "
2584 : "deployment is host. "),
2585 : HCCL_E_PARA);
2586 :
2587 0 : u32 i = 0;
2588 0 : for (i = 0; i < devIpAddr_.size(); i++) {
2589 0 : if (devIpAddr_[i].IsInvalid()) {
2590 0 : HCCL_INFO("[Init][Nic]nic num[%u] deviceip is invalid, total nicNum[%u]", i, devIpAddr_.size());
2591 0 : continue;
2592 : }
2593 0 : std::vector<u32>& ranksPorts = groupNicRanksPort_.empty() ? nicRanksPort_ : groupNicRanksPort_;
2594 0 : port = GetNicPort(devicePhyId_, ranksPorts, userRank_, isUseRankPort_);
2595 :
2596 0 : CHK_RET(socketManager_->ServerDeInit(netDevCtxMap_[devIpAddr_[i]], port));
2597 0 : HcclNetCloseDev(netDevCtxMap_[devIpAddr_[i]]);
2598 0 : netDevCtxMap_.erase(devIpAddr_[i]);
2599 :
2600 0 : break;
2601 : }
2602 :
2603 0 : if (i == devIpAddr_.size()) {
2604 0 : CHK_RET(socketManager_->ServerDeInit(netDevCtxMap_[hostIp_], port));
2605 0 : HcclNetCloseDev(netDevCtxMap_[hostIp_]);
2606 0 : netDevCtxMap_.erase(hostIp_);
2607 : }
2608 0 : return HCCL_SUCCESS;
2609 : }
2610 :
2611 0 : HcclResult HcclCommunicator::DeinitNic()
2612 : {
2613 0 : if (nicDeployment_ == NICDeployment::NIC_DEPLOYMENT_DEVICE) {
2614 0 : u32 port = GetLocalNicPort(NicType::DEVICE_NIC_TYPE);
2615 0 : u32 nicNum = devIpAddr_.size();
2616 0 : for (u32 i = 0; i < nicNum; i++) {
2617 0 : if (devIpAddr_[i].IsInvalid()) {
2618 0 : HCCL_INFO("continue invalid devIp %s", devIpAddr_[i].GetReadableAddress());
2619 0 : continue;
2620 : }
2621 0 : if (netDevCtxMap_.find(devIpAddr_[i]) == netDevCtxMap_.end()) {
2622 0 : HCCL_INFO("devIp[%s] not found in netDevCtxMap_", devIpAddr_[i].GetReadableAddress());
2623 0 : continue;
2624 : }
2625 0 : CHK_RET(socketManager_->ServerDeInit(netDevCtxMap_[devIpAddr_[i]], port));
2626 : // 最后一次调用才删除netCtx
2627 0 : if (nicInitialized_ - 1 <= 0) {
2628 0 : HcclNetCloseDev(netDevCtxMap_[devIpAddr_[i]]);
2629 0 : netDevCtxMap_.erase(devIpAddr_[i]);
2630 : }
2631 : }
2632 0 : if (IsEnableBackupLink() && netDevCtxMap_.find(devBackupIpAddr_[0]) != netDevCtxMap_.end()) {
2633 : // 超节点 && level2支持重执行 && Aicpu -> 备用网卡 deinit
2634 0 : CHK_RET(socketManager_->ServerDeInit(netDevCtxMap_[devBackupIpAddr_[0]], devBackupPort_));
2635 0 : if (nicInitialized_ - 1 <= 0) {
2636 0 : HcclNetCloseDev(netDevCtxMap_[devBackupIpAddr_[0]]);
2637 0 : netDevCtxMap_.erase(devBackupIpAddr_[0]);
2638 0 : HCCL_DEBUG(
2639 : "[%s]finish backup ServerDeInit devBackupIpAddr_[%s], port[%u], IsEnableBackupLink[%d]", __func__,
2640 : devBackupIpAddr_[0].GetReadableAddress(), devBackupPort_, IsEnableBackupLink());
2641 : }
2642 : }
2643 0 : } else if (nicDeployment_ == NICDeployment::NIC_DEPLOYMENT_HOST) {
2644 0 : CHK_RET(DeinitNicHostDeploy());
2645 : } else {
2646 0 : HCCL_ERROR("[Deinit][Nic]nic deployment[%d] is not supported", nicDeployment_);
2647 0 : return HCCL_E_PARA;
2648 : }
2649 0 : nicInitialized_--;
2650 0 : return HCCL_SUCCESS;
2651 : }
2652 :
2653 520 : HcclResult HcclCommunicator::RegisterRanksToDca()
2654 : {
2655 520 : if (deviceType_ != DevType::DEV_TYPE_910_93 && deviceType_ != DevType::DEV_TYPE_910B) {
2656 325 : HCCL_WARNING("[RegisterRanksToDca] not support deviceType[%d]", deviceType_);
2657 325 : return HCCL_SUCCESS;
2658 : }
2659 195 : CHK_RET(setVnicIpToRankInfoList());
2660 198 : DetectConnectionAnomalies::GetInstance(deviceLogicId_).Init(rankInfoList_, isNeedInitNic_);
2661 198 : return HCCL_SUCCESS;
2662 : }
2663 :
2664 0 : HcclResult HcclCommunicator::AddOpInfoToHeartBeat(const OpInfoDesc& opInfo, const std::string& tag)
2665 : {
2666 0 : if (Is310PDevice() || deviceType_ == DevType::DEV_TYPE_310P3
2667 0 : || GetWorkflowMode() == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OPS_KERNEL_INFO_LIB) {
2668 0 : return HCCL_SUCCESS;
2669 : }
2670 0 : return Heartbeat::GetInstance(deviceLogicId_).AddOpInfoToHeartBeat(identifier_, opInfo, tag);
2671 : }
2672 :
2673 806 : void HcclCommunicator::DeleteOpInfoToHeartBeat()
2674 : {
2675 1613 : if (Is310PDevice() || deviceType_ == DevType::DEV_TYPE_310P3
2676 1614 : || GetWorkflowMode() == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OPS_KERNEL_INFO_LIB) {
2677 162 : return;
2678 : }
2679 645 : for (const auto& tag : hbSendRecvTags_) {
2680 0 : Heartbeat::GetInstance(deviceLogicId_).DeleteOpInfoToHeartBeat(identifier_, tag);
2681 : }
2682 1935 : Heartbeat::GetInstance(deviceLogicId_).DeleteOpInfoToHeartBeat(identifier_, "");
2683 : }
2684 :
2685 21 : HcclResult HcclCommunicator::RegisterToHeartBeat()
2686 : {
2687 21 : if (Is310PDevice() || deviceType_ == DevType::DEV_TYPE_310P3) {
2688 0 : return HCCL_SUCCESS;
2689 : }
2690 21 : u32 localPort = commPortConfig_.devPortSwitchOn ? HCCL_INVALID_PORT : GetLocalNicPort(NicType::DEVICE_NIC_TYPE);
2691 21 : return Heartbeat::GetInstance(deviceLogicId_)
2692 42 : .RegisterToHeartBeat(
2693 21 : userRank_, deviceType_, rankInfoList_, localPort, isNeedInitNic_, identifier_, useSuperPodMode_,
2694 21 : isUsedRdmaLevel0_, retryEnable_, IsEnableBackupLink());
2695 : }
2696 :
2697 0 : HcclResult HcclCommunicator::RegisterToHeartBeat(u32 peerRankId, string& tag)
2698 : {
2699 0 : u32 localPort = commPortConfig_.devPortSwitchOn ? HCCL_INVALID_PORT : GetLocalNicPort(NicType::DEVICE_NIC_TYPE);
2700 0 : return Heartbeat::GetInstance(deviceLogicId_)
2701 0 : .RegisterToHeartBeat(
2702 0 : userRank_, deviceType_, rankInfoList_, localPort, isNeedInitNic_, peerRankId, identifier_, tag,
2703 0 : useSuperPodMode_, isUsedRdmaLevel0_, retryEnable_, IsEnableBackupLink());
2704 : }
2705 :
2706 806 : void HcclCommunicator::UnRegisterToHeartBeat()
2707 : {
2708 806 : for (auto tag : hbSendRecvTags_) {
2709 0 : Heartbeat::GetInstance(deviceLogicId_).UnRegisterToHeartBeat(deviceType_, identifier_, tag);
2710 0 : }
2711 807 : Heartbeat::GetInstance(deviceLogicId_).UnRegisterToHeartBeat(deviceType_, identifier_);
2712 806 : }
2713 :
2714 807 : void HcclCommunicator::UnRegisterToCommConfiger() { CommConfiger::GetInstance().UnRegisterToCommConfiger(identifier_); }
2715 :
2716 92 : HcclResult HcclCommunicator::SetGlobalWorkSpace(std::vector<void*>& globalWorkSpaceAddr)
2717 : {
2718 92 : CHK_RET(HcclSetGlobalWorkSpace(dispatcher_, globalWorkSpaceAddr));
2719 92 : return HCCL_SUCCESS;
2720 : }
2721 :
2722 0 : HcclResult HcclCommunicator::GetandClearOverFlowTasks(std::vector<HcclDumpInfo>& hcclDumpInfo)
2723 : {
2724 0 : if (profilerManager_ != nullptr) {
2725 0 : CHK_RET(profilerManager_->GetandClearOverFlowTasks(hcclDumpInfo));
2726 : } else {
2727 0 : HCCL_WARNING("[impl][GetDumpTask] profilerManager_ not set");
2728 : }
2729 0 : return HCCL_SUCCESS;
2730 : }
2731 :
2732 236 : HcclResult HcclCommunicator::GetDeviceId(s32& deviceId) const
2733 : {
2734 236 : deviceId = deviceLogicId_;
2735 236 : return HCCL_SUCCESS;
2736 : }
2737 :
2738 1 : HcclResult HcclCommunicator::GetCqeError(HcclResult& result)
2739 : {
2740 1 : CHK_RET(Heartbeat::GetInstance(deviceLogicId_).CheckErrorCqe(identifier_, result));
2741 1 : return HCCL_SUCCESS;
2742 : }
2743 :
2744 0 : HcclResult HcclCommunicator::GetOpInconsistentError(HcclResult& result)
2745 : {
2746 0 : CHK_RET(Heartbeat::GetInstance(deviceLogicId_).CheckOpInconsistentError(identifier_, result));
2747 0 : return HCCL_SUCCESS;
2748 : }
2749 :
2750 0 : HcclResult HcclCommunicator::SupportDeterministicOptim(bool& isDeterministicOptim)
2751 : {
2752 0 : CHK_SMART_PTR_NULL(implAlg_);
2753 0 : CHK_RET(implAlg_->SupportDeterministicOptim(isDeterministicOptim));
2754 0 : return HCCL_SUCCESS;
2755 : }
2756 :
2757 0 : HcclResult HcclCommunicator::GetHccsLinkNum(u32& numHccsLink)
2758 : {
2759 0 : auto iter = pairLinkInfo_.find(static_cast<u32>(LinkTypeInServer::HCCS_TYPE));
2760 0 : if (iter == pairLinkInfo_.end()) {
2761 0 : HCCL_ERROR("[HcclCommunicator][GetHccsLinkNum]HCCS_TYPE is not found");
2762 0 : return HCCL_E_PARA;
2763 : }
2764 0 : numHccsLink = iter->second.size();
2765 0 : return HCCL_SUCCESS;
2766 : }
2767 :
2768 4 : HcclResult HcclCommunicator::AllGather(
2769 : const std::string& tag, void* inputPtr, void* outputPtr, u64 inputCount, HcclDataType dataType, HcclRtStream stream,
2770 : HcomCollOpInfo* opInfo)
2771 : {
2772 4 : bool isCapture = StreamIsCapture(stream);
2773 :
2774 11 : bool aicpuUnfoldMode = false;
2775 11 : if (EnableAicpuUnfold(isCapture) && (userRankSize_ != 1)) {
2776 0 : aicpuUnfoldMode = true;
2777 : }
2778 :
2779 6 : if (!IsAtomicInit()) {
2780 0 : HCCL_ERROR(
2781 : "[HcclCommunicator][AllGather]errNo[0x%016llx] hccl init must be called before call this function",
2782 : HCCL_ERROR_CODE(HCCL_E_UNAVAIL));
2783 0 : return HCCL_E_UNAVAIL;
2784 : }
2785 :
2786 11 : Stream streamObj(stream);
2787 11 : CHK_RET(callbackTask_->CallbackRegStream(stream));
2788 :
2789 11 : std::vector<u32>& ranksPorts = groupNicRanksPort_.empty() ? nicRanksPort_ : groupNicRanksPort_;
2790 10 : std::vector<u32>& vnicRanksPorts = groupVnicRanksPort_.empty() ? vnicRanksPort_ : groupVnicRanksPort_;
2791 8 : implAlg_->SetHDCModeInfo(rankDevicePhyIdNicInfoMap_, ranksPorts, vnicRanksPorts, isSetHDCModeInfo_, isUseRankPort_);
2792 :
2793 10 : u32 perDataSize = SIZE_TABLE[dataType];
2794 10 : u64 totalSize = inputCount * perDataSize;
2795 :
2796 10 : OpParam opParam;
2797 8 : opParam.tag = tag;
2798 10 : opParam.inputPtr = inputPtr;
2799 10 : opParam.inputSize = totalSize;
2800 10 : opParam.outputPtr = outputPtr;
2801 10 : opParam.outputSize = totalSize * userRankSize_;
2802 10 : opParam.DataDes.count = inputCount;
2803 10 : opParam.DataDes.dataType = dataType;
2804 10 : opParam.reduceType = HcclReduceOp::HCCL_REDUCE_RESERVED;
2805 10 : opParam.stream = streamObj;
2806 9 : opParam.syncMode = SyncMode::DEFAULT_TIMEWAITSYNCMODE;
2807 9 : opParam.isCapture = isCapture;
2808 9 : opParam.aicpuUnfoldMode = aicpuUnfoldMode;
2809 9 : opParam.aicpuCacheEnable = GetExternalInputAicpuCacheEnable();
2810 9 : opParam.opType = HcclCMDType::HCCL_CMD_ALLGATHER;
2811 :
2812 9 : CHK_RET(ExecOp(HcclCMDType::HCCL_CMD_ALLGATHER, opParam));
2813 :
2814 11 : return HCCL_SUCCESS;
2815 11 : }
2816 :
2817 1 : HcclResult HcclCommunicator::AllGatherV(
2818 : const std::string& tag, const void* sendBuf, u64 sendCount, const void* recvBuf, const void* recvCounts,
2819 : const void* rdispls, HcclDataType dataType, HcclRtStream stream)
2820 : {
2821 1 : bool aicpuUnfoldMode = false;
2822 :
2823 1 : if (GetAicpuUnfoldConfig() && (deviceType_ == DevType::DEV_TYPE_910_93) && (userRankSize_ != 1)) {
2824 0 : aicpuUnfoldMode = true;
2825 : }
2826 :
2827 1 : if (!IsAtomicInit()) {
2828 0 : HCCL_ERROR(
2829 : "[HcclCommunicator][AllGatherV]errNo[0x%016llx] hccl init must be called before call this function",
2830 : HCCL_ERROR_CODE(HCCL_E_UNAVAIL));
2831 0 : return HCCL_E_UNAVAIL;
2832 : }
2833 :
2834 1 : Stream streamObj(stream);
2835 1 : CHK_RET(callbackTask_->CallbackRegStream(stream));
2836 :
2837 1 : std::vector<u32>& ranksPorts = groupNicRanksPort_.empty() ? nicRanksPort_ : groupNicRanksPort_;
2838 1 : std::vector<u32>& vnicRanksPorts = groupVnicRanksPort_.empty() ? vnicRanksPort_ : groupVnicRanksPort_;
2839 1 : implAlg_->SetHDCModeInfo(rankDevicePhyIdNicInfoMap_, ranksPorts, vnicRanksPorts, isSetHDCModeInfo_, isUseRankPort_);
2840 :
2841 1 : u32 perDataSize = SIZE_TABLE[dataType];
2842 1 : u64 totalSize = sendCount * perDataSize;
2843 :
2844 1 : u64 outputSize = 0;
2845 1 : const u64* counts = static_cast<const u64*>(recvCounts);
2846 3 : for (u32 i = 0; i < userRankSize_; i++) {
2847 2 : outputSize += counts[i] * perDataSize;
2848 : }
2849 :
2850 1 : bool isCapture = StreamIsCapture(stream);
2851 :
2852 1 : OpParam opParam;
2853 1 : opParam.tag = tag;
2854 1 : opParam.inputPtr = const_cast<void*>(sendBuf);
2855 1 : opParam.inputSize = totalSize;
2856 1 : opParam.outputPtr = const_cast<void*>(recvBuf);
2857 1 : opParam.outputSize = outputSize;
2858 1 : opParam.VDataDes.dataType = dataType;
2859 1 : opParam.VDataDes.counts = const_cast<void*>(recvCounts);
2860 1 : opParam.VDataDes.displs = const_cast<void*>(rdispls);
2861 1 : opParam.reduceType = HcclReduceOp::HCCL_REDUCE_RESERVED;
2862 1 : opParam.stream = streamObj;
2863 1 : opParam.syncMode = SyncMode::DEFAULT_TIMEWAITSYNCMODE;
2864 1 : opParam.aicpuUnfoldMode = aicpuUnfoldMode;
2865 1 : opParam.isCapture = isCapture;
2866 1 : opParam.aicpuCacheEnable = GetExternalInputAicpuCacheEnable();
2867 1 : opParam.opType = HcclCMDType::HCCL_CMD_ALLGATHER_V;
2868 :
2869 1 : if (UNLIKELY(GetDebugConfig() & HCCL_ALG)) {
2870 0 : for (u32 i = 0; i < userRankSize_; i++) {
2871 0 : HCCL_CONFIG_DEBUG(
2872 : HCCL_ALG, "[HcclCommunicator][AllGatherV]userRank_[%u], rankIdx[%u], recvCounts[%llu], rdispls[%llu]",
2873 : userRank_, i, counts[i], static_cast<const u64*>(rdispls)[i]);
2874 : }
2875 : }
2876 :
2877 1 : CHK_RET(ExecOp(HcclCMDType::HCCL_CMD_ALLGATHER_V, opParam));
2878 :
2879 1 : return HCCL_SUCCESS;
2880 1 : }
2881 :
2882 0 : HcclResult HcclCommunicator::AicpuUnfold(
2883 : const std::string& tag, void* inputPtr, void* outputPtr, u64 count, HcclDataType dataType, HcclReduceOp op,
2884 : HcclRtStream stream, HcclCMDType cmdType)
2885 : {
2886 0 : Stream streamObj(stream);
2887 0 : u32 perDataSize = SIZE_TABLE[dataType];
2888 0 : u64 totalSize = count * perDataSize;
2889 0 : bool isCapture = StreamIsCapture(stream);
2890 0 : OpParam opParam;
2891 0 : opParam.tag = tag;
2892 0 : opParam.inputPtr = inputPtr;
2893 0 : opParam.inputSize = totalSize;
2894 0 : opParam.outputPtr = outputPtr;
2895 0 : opParam.outputSize = totalSize;
2896 0 : opParam.DataDes.count = count;
2897 0 : opParam.DataDes.dataType = dataType;
2898 0 : opParam.reduceType = op;
2899 0 : opParam.stream = streamObj;
2900 0 : opParam.isCapture = isCapture;
2901 0 : opParam.syncMode = SyncMode::DEFAULT_TIMEWAITSYNCMODE;
2902 0 : opParam.aicpuCacheEnable = GetExternalInputAicpuCacheEnable();
2903 0 : AlgType algType;
2904 0 : algType.algoLevel0 = AlgTypeLevel0::ALG_LEVEL0_NP_MESH;
2905 0 : algType.algoLevel1 = AlgTypeLevel1::ALG_LEVEL1_RING;
2906 :
2907 : // 构造空vector用于入参，无实际意义
2908 0 : const std::vector<Stream> slaveStreams;
2909 0 : CHK_RET(RegisterDfxInfo(opParam, algType, slaveStreams));
2910 0 : HcclResult ret = HCCL_SUCCESS;
2911 0 : if (!IsExistCommRes(identifier_)) {
2912 0 : HCCL_INFO(
2913 : "[AicpuUnfold] tag[%s] count[%llu] dataType[%s] op[%s].", identifier_.c_str(), count,
2914 : GetDataTypeEnumStr(dataType).c_str(), GetReduceOpEnumStr(op).c_str());
2915 0 : uint64_t streamMode = 0;
2916 0 : CHK_RET(hrtStreamGetMode(stream, &streamMode));
2917 :
2918 : rtStream_t aicpuStream;
2919 0 : ret = Mc2AiCpuStreamAllocAndGet(streamMode, aicpuStream);
2920 0 : void* commContext = nullptr;
2921 0 : ret = CreateCommResource(identifier_, stream, true, &commContext);
2922 0 : if (ret != HCCL_SUCCESS) {
2923 0 : HCCL_ERROR(
2924 : "[hcclImpl][CreateComm]create aicpu unfold comminfo by tag[%s] failed. return[%d]", identifier_.c_str(),
2925 : ret);
2926 0 : return ret;
2927 : }
2928 : }
2929 :
2930 0 : std::string kernelName = "RunAicpuRpcSrvLaunch";
2931 0 : AicpuOpTiling opTilingInfo;
2932 0 : ret = AicpuKfcTilingDataLaunch(opParam, cmdType, commContext_, kernelName, opTilingInfo);
2933 0 : if (ret != HCCL_SUCCESS) {
2934 0 : HCCL_ERROR(
2935 : "[hcclImpl][TilingData]aicpu unfold tiling data launch failed. return[%d] inputPtr[%p]"
2936 : "outputPtr[%p] count[%llu] dataType[%s] op[%s]",
2937 : ret, inputPtr, outputPtr, count, GetDataTypeEnumStr(dataType).c_str(), GetReduceOpEnumStr(op).c_str());
2938 0 : return ret;
2939 : }
2940 0 : CHK_RET(UnRegisterDfxInfo(opParam, slaveStreams));
2941 0 : return HCCL_SUCCESS;
2942 0 : }
2943 :
2944 3 : HcclResult HcclCommunicator::AllGatherOutPlace(
2945 : const std::string& tag, void* inputPtr, void* outputPtr, u64 inputCount, HcclDataType dataType, HcclRtStream stream)
2946 : {
2947 3 : CHK_RET(CheckSuspendingStatus());
2948 2 : if (!IsAtomicInit()) {
2949 0 : HCCL_ERROR(
2950 : "[HcclCommunicator][AllGatherOutPlace]errNo[0x%016llx] hccl init must be called before call this function",
2951 : HCCL_ERROR_CODE(HCCL_E_UNAVAIL));
2952 0 : return HCCL_E_UNAVAIL;
2953 : }
2954 :
2955 5 : bool isCapture = StreamIsCapture(stream);
2956 :
2957 8 : bool aicpuUnfoldMode = false;
2958 8 : if (EnableAicpuUnfold(isCapture) && (userRankSize_ != 1)) {
2959 4 : aicpuUnfoldMode = true;
2960 : }
2961 :
2962 8 : Stream streamObj(stream);
2963 8 : CHK_RET(callbackTask_->CallbackRegStream(stream));
2964 :
2965 7 : std::vector<u32>& ranksPorts = groupNicRanksPort_.empty() ? nicRanksPort_ : groupNicRanksPort_;
2966 7 : std::vector<u32>& vnicRanksPorts = groupVnicRanksPort_.empty() ? vnicRanksPort_ : groupVnicRanksPort_;
2967 6 : implAlg_->SetHDCModeInfo(rankDevicePhyIdNicInfoMap_, ranksPorts, vnicRanksPorts, isSetHDCModeInfo_, isUseRankPort_);
2968 :
2969 5 : u32 perDataSize = SIZE_TABLE[dataType];
2970 5 : u64 totalSize = inputCount * perDataSize * userRankSize_;
2971 :
2972 5 : OpParam opParam;
2973 7 : opParam.tag = tag;
2974 8 : opParam.inputPtr = inputPtr;
2975 8 : opParam.inputSize = inputCount * perDataSize;
2976 8 : opParam.outputPtr = outputPtr;
2977 8 : opParam.outputSize = totalSize;
2978 8 : opParam.DataDes.count = inputCount;
2979 8 : opParam.DataDes.dataType = dataType;
2980 8 : opParam.reduceType = HcclReduceOp::HCCL_REDUCE_RESERVED;
2981 8 : opParam.stream = streamObj;
2982 8 : opParam.syncMode = SyncMode::DEFAULT_TIMEWAITSYNCMODE;
2983 8 : opParam.opBaseAtraceInfo = opBaseAtraceInfo_.get();
2984 8 : opParam.aicpuUnfoldMode = aicpuUnfoldMode;
2985 8 : opParam.aicpuCacheEnable = GetExternalInputAicpuCacheEnable();
2986 8 : opParam.isCapture = isCapture;
2987 8 : opParam.opType = HcclCMDType::HCCL_CMD_ALLGATHER;
2988 :
2989 8 : CHK_RET(ExecOp(HcclCMDType::HCCL_CMD_ALLGATHER, opParam));
2990 :
2991 0 : return HCCL_SUCCESS;
2992 8 : }
2993 :
2994 1 : HcclResult HcclCommunicator::AllGatherVOutPlace(
2995 : const std::string& tag, void* inputPtr, void* outputPtr, u64 inputCount, const void* outputCounts,
2996 : const void* outputDispls, HcclDataType dataType, HcclRtStream stream)
2997 : {
2998 1 : CHK_RET(CheckSuspendingStatus());
2999 1 : if (userRankSize_ == 1) {
3000 : // rankSize为1时，退化为AllGather
3001 0 : return AllGatherOutPlace(tag, inputPtr, outputPtr, inputCount, dataType, stream);
3002 : }
3003 :
3004 1 : if (!IsAtomicInit()) {
3005 0 : HCCL_ERROR(
3006 : "[HcclCommunicator][AllGatherVOutPlace]errNo[0x%016llx] hccl init must be called before call this function",
3007 : HCCL_ERROR_CODE(HCCL_E_UNAVAIL));
3008 0 : return HCCL_E_UNAVAIL;
3009 : }
3010 :
3011 1 : bool aicpuUnfoldMode = false;
3012 1 : if (GetAicpuUnfoldConfig() && (deviceType_ == DevType::DEV_TYPE_910_93) && (userRankSize_ != 1)) {
3013 0 : aicpuUnfoldMode = true;
3014 : }
3015 :
3016 1 : bool isCapture = StreamIsCapture(stream);
3017 :
3018 1 : Stream streamObj(stream);
3019 1 : CHK_RET(callbackTask_->CallbackRegStream(stream));
3020 :
3021 1 : std::vector<u32>& ranksPorts = groupNicRanksPort_.empty() ? nicRanksPort_ : groupNicRanksPort_;
3022 1 : std::vector<u32>& vnicRanksPorts = groupVnicRanksPort_.empty() ? vnicRanksPort_ : groupVnicRanksPort_;
3023 1 : implAlg_->SetHDCModeInfo(rankDevicePhyIdNicInfoMap_, ranksPorts, vnicRanksPorts, isSetHDCModeInfo_, isUseRankPort_);
3024 :
3025 1 : u32 perDataSize = SIZE_TABLE[dataType];
3026 1 : u64 outputSize = 0;
3027 1 : const u64* counts = static_cast<const u64*>(outputCounts);
3028 3 : for (u32 i = 0; i < userRankSize_; i++) {
3029 2 : outputSize += counts[i] * perDataSize;
3030 : }
3031 :
3032 1 : OpParam opParam;
3033 1 : opParam.tag = tag;
3034 1 : opParam.inputPtr = inputPtr;
3035 1 : opParam.inputSize = inputCount * perDataSize;
3036 1 : opParam.outputPtr = outputPtr;
3037 1 : opParam.outputSize = outputSize;
3038 1 : opParam.VDataDes.counts = const_cast<void*>(outputCounts);
3039 1 : opParam.VDataDes.displs = const_cast<void*>(outputDispls);
3040 1 : opParam.VDataDes.dataType = dataType;
3041 1 : opParam.reduceType = HcclReduceOp::HCCL_REDUCE_RESERVED;
3042 1 : opParam.stream = streamObj;
3043 1 : opParam.syncMode = SyncMode::DEFAULT_TIMEWAITSYNCMODE;
3044 1 : opParam.opBaseAtraceInfo = opBaseAtraceInfo_.get();
3045 1 : opParam.aicpuUnfoldMode = aicpuUnfoldMode;
3046 1 : opParam.aicpuCacheEnable = GetExternalInputAicpuCacheEnable();
3047 1 : opParam.isCapture = isCapture;
3048 1 : opParam.rankSize = userRankSize_;
3049 1 : opParam.opType = HcclCMDType::HCCL_CMD_ALLGATHER_V;
3050 :
3051 1 : if (UNLIKELY(GetDebugConfig() & HCCL_ALG)) {
3052 0 : for (u32 i = 0; i < userRankSize_; i++) {
3053 0 : HCCL_CONFIG_DEBUG(
3054 : HCCL_ALG,
3055 : "[HcclCommunicator][AllGatherVOutPlace]userRank_[%u], rankIdx[%u],"
3056 : "outputCounts[%llu], outputDispls[%llu]",
3057 : userRank_, i, counts[i], static_cast<const u64*>(outputDispls)[i]);
3058 : }
3059 : }
3060 :
3061 1 : CHK_RET(ExecOp(HcclCMDType::HCCL_CMD_ALLGATHER_V, opParam));
3062 :
3063 1 : return HCCL_SUCCESS;
3064 1 : }
3065 :
3066 79 : void HcclCommunicator::GetAndSetSyncMode(SyncMode& preSyncMode, SyncMode newSyncMode)
3067 : {
3068 79 : if (newSyncMode == SyncMode::UNLIMITED_TIMEWAITSYNCMODE) {
3069 0 : if (Is310P3Common(isHaveCpuRank_, deviceType_)) {
3070 0 : HCCL_WARNING("310P don't support unlimited notify wait mode");
3071 : } else {
3072 0 : HcclGetNotifyWaitMode(dispatcher_, &preSyncMode);
3073 0 : HcclSetNotifyWaitMode(dispatcher_, newSyncMode);
3074 : }
3075 : }
3076 79 : }
3077 :
3078 60 : void HcclCommunicator::RestorePreSyncMode(SyncMode preSyncMode, SyncMode newSyncMode)
3079 : {
3080 60 : if (newSyncMode == SyncMode::UNLIMITED_TIMEWAITSYNCMODE && !Is310P3Common(isHaveCpuRank_, deviceType_)) {
3081 0 : HcclSetNotifyWaitMode(dispatcher_, preSyncMode);
3082 : }
3083 60 : }
3084 :
3085 40 : HcclResult HcclCommunicator::AllReduce(
3086 : const std::string& tag, void* inputPtr, void* outputPtr, u64 count, HcclDataType dataType, HcclReduceOp op,
3087 : HcclRtStream stream, SyncMode syncMode, const HcomCollOpInfo* opInfo)
3088 : {
3089 40 : CHK_RET(CheckSuspendingStatus());
3090 39 : bool aicpuUnfoldMode = false;
3091 49 : if (GetAicpuUnfoldConfig() == true && IsSupportSDMAReduce(inputPtr, outputPtr, dataType, op)
3092 50 : && deviceType_ == DevType::DEV_TYPE_910_93 && (userRankSize_ != 1)) {
3093 0 : aicpuUnfoldMode = true;
3094 : }
3095 :
3096 40 : if (!IsAtomicInit()) {
3097 0 : HCCL_ERROR(
3098 : "[HcclCommunicator][AllReduce]errNo[0x%016llx] hccl init must be called before call this function",
3099 : HCCL_ERROR_CODE(HCCL_E_UNAVAIL));
3100 0 : return HCCL_E_UNAVAIL;
3101 : }
3102 :
3103 : // 设置notify wait模式
3104 57 : SyncMode preSyncMode = SyncMode::DEFAULT_TIMEWAITSYNCMODE;
3105 57 : GetAndSetSyncMode(preSyncMode, syncMode);
3106 :
3107 57 : Stream streamObj(stream);
3108 59 : CHK_RET(callbackTask_->CallbackRegStream(stream));
3109 :
3110 56 : std::vector<u32>& ranksPorts = groupNicRanksPort_.empty() ? nicRanksPort_ : groupNicRanksPort_;
3111 52 : std::vector<u32>& vnicRanksPorts = groupVnicRanksPort_.empty() ? vnicRanksPort_ : groupVnicRanksPort_;
3112 46 : implAlg_->SetHDCModeInfo(rankDevicePhyIdNicInfoMap_, ranksPorts, vnicRanksPorts, isSetHDCModeInfo_, isUseRankPort_);
3113 :
3114 43 : u32 perDataSize = SIZE_TABLE[dataType];
3115 43 : u64 totalSize = count * perDataSize;
3116 :
3117 43 : OpParam opParam;
3118 40 : opParam.tag = tag;
3119 60 : opParam.inputPtr = inputPtr;
3120 60 : opParam.inputSize = totalSize;
3121 60 : opParam.outputPtr = outputPtr;
3122 60 : opParam.outputSize = totalSize;
3123 60 : opParam.DataDes.count = count;
3124 60 : opParam.DataDes.dataType = dataType;
3125 60 : opParam.reduceType = op;
3126 60 : opParam.stream = streamObj;
3127 46 : opParam.syncMode = syncMode;
3128 46 : opParam.aicpuUnfoldMode = aicpuUnfoldMode;
3129 46 : opParam.aicpuCacheEnable = GetExternalInputAicpuCacheEnable();
3130 47 : opParam.opType = HcclCMDType::HCCL_CMD_ALLREDUCE;
3131 : // 用于inplace支持重执行场景的图模式归一至单算子模式
3132 47 : retryOrigWorkflowMode_ = GetWorkflowMode();
3133 : bool isHcclOpInplace
3134 47 : = IsHcclOpInplace(HcclCMDType::HCCL_CMD_ALLREDUCE, opParam, userRank_, userRankSize_, isInplaceStatus_);
3135 61 : if (aicpuUnfoldMode && retryEnable_ && isHcclOpInplace) {
3136 0 : HCCL_DEBUG(
3137 : "The retry with inplace case is expected to be supported, "
3138 : "aicpuUnfoldMode[%d], retryEnable_[%d], isHcclOpInplace[%d], "
3139 : "therefore HcclWorkflowMode is converted from [%d] to HCCL_WORKFLOW_MODE_OP_BASE",
3140 : aicpuUnfoldMode, retryEnable_, isHcclOpInplace, static_cast<u8>(retryOrigWorkflowMode_));
3141 0 : CHK_RET(SetWorkflowMode(HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE));
3142 : }
3143 :
3144 61 : CHK_RET(ExecOp(HcclCMDType::HCCL_CMD_ALLREDUCE, opParam));
3145 :
3146 44 : RestorePreSyncMode(preSyncMode, syncMode);
3147 45 : CHK_RET(SetWorkflowMode(retryOrigWorkflowMode_));
3148 47 : return HCCL_SUCCESS;
3149 52 : }
3150 :
3151 0 : HcclResult HcclCommunicator::AllReduceAicpuUnfold(
3152 : const std::string& tag, void* inputPtr, void* outputPtr, u64 count, HcclDataType dataType, HcclReduceOp op,
3153 : HcclRtStream stream)
3154 : {
3155 0 : Stream streamObj(stream);
3156 0 : u32 perDataSize = SIZE_TABLE[dataType];
3157 0 : u64 totalSize = count * perDataSize;
3158 0 : OpParam opParam;
3159 0 : opParam.tag = tag;
3160 0 : opParam.inputPtr = inputPtr;
3161 0 : opParam.inputSize = totalSize;
3162 0 : opParam.outputPtr = outputPtr;
3163 0 : opParam.outputSize = totalSize;
3164 0 : opParam.DataDes.count = count;
3165 0 : opParam.DataDes.dataType = dataType;
3166 0 : opParam.reduceType = op;
3167 0 : opParam.stream = streamObj;
3168 0 : opParam.isCapture = StreamIsCapture(stream);
3169 0 : opParam.syncMode = SyncMode::DEFAULT_TIMEWAITSYNCMODE;
3170 0 : opParam.aicpuCacheEnable = GetExternalInputAicpuCacheEnable();
3171 0 : AlgType algType;
3172 0 : algType.algoLevel0 = AlgTypeLevel0::ALG_LEVEL0_NP_SINGLE_RING;
3173 0 : algType.algoLevel1 = AlgTypeLevel1::ALG_LEVEL1_RING;
3174 0 : auto slaveStreams = opParam.isCapture ? std::vector<Stream>{opStream_} : std::vector<Stream>{};
3175 0 : CaptureSlaveStreams(streamObj.ptr(), slaveStreams);
3176 0 : CHK_RET(RegisterDfxInfo(opParam, algType, slaveStreams));
3177 : HcclResult ret;
3178 0 : if (!IsExistCommRes(tag)) {
3179 0 : uint64_t streamMode = 0;
3180 0 : CHK_RET(hrtStreamGetMode(stream, &streamMode));
3181 : rtStream_t aicpuStream;
3182 0 : ret = Mc2AiCpuStreamAllocAndGet(streamMode, aicpuStream);
3183 0 : void* commContext = nullptr;
3184 0 : ret = CreateCommResource(tag, aicpuStream, true, &commContext);
3185 0 : if (ret != HCCL_SUCCESS) {
3186 0 : HCCL_ERROR(
3187 : "[hcclImpl][CreateComm]create aicpu unfold comminfo by tag[%s] failed. return[%d]", tag.c_str(), ret);
3188 0 : return ret;
3189 : }
3190 : }
3191 0 : AicpuOpTiling opTilingInfo;
3192 0 : std::string kernelName = "RunAicpuRpcSrvLaunch";
3193 0 : ret = AicpuKfcTilingDataLaunch(opParam, HcclCMDType::HCCL_CMD_ALLREDUCE, commContext_, kernelName, opTilingInfo);
3194 0 : if (ret != HCCL_SUCCESS) {
3195 0 : HCCL_ERROR(
3196 : "[hcclImpl][TilingData]aicpu unfold tiling data launch failed. return[%d] inputPtr[%p]"
3197 : "outputPtr[%p] count[%llu] dataType[%s] op[%s]",
3198 : ret, inputPtr, outputPtr, count, GetDataTypeEnumStr(dataType).c_str(), GetReduceOpEnumStr(op).c_str());
3199 0 : return ret;
3200 : }
3201 0 : CHK_RET(UnRegisterDfxInfo(opParam, slaveStreams));
3202 0 : return HCCL_SUCCESS;
3203 0 : }
3204 :
3205 11 : HcclResult HcclCommunicator::AllReduceOutPlace(
3206 : const std::string& tag, void* inputPtr, void* outputPtr, u64 count, HcclDataType dataType, HcclReduceOp op,
3207 : HcclRtStream stream, SyncMode syncMode)
3208 : {
3209 11 : CHK_RET(CheckSuspendingStatus());
3210 14 : const u32 RANK_SIZE_TWO = 2;
3211 14 : bool aicpuUnfoldMode = false;
3212 14 : if (GetAicpuUnfoldConfig() == true && IsSupportSDMAReduce(inputPtr, outputPtr, dataType, op)) {
3213 0 : if (userRankSize_ >= RANK_SIZE_TWO && Is310P3Common(isHaveCpuRank_, deviceType_)) {
3214 0 : HcclResult ret = AllReduceAicpuUnfold(tag, inputPtr, outputPtr, count, dataType, op, stream);
3215 0 : CHK_PRT_RET(
3216 : (ret != HCCL_SUCCESS),
3217 : HCCL_ERROR(
3218 : "[HcclCommunicator][AllReduce]errNo[0x%016llx] tag[%s], AllReduce aicpu unfold failed",
3219 : HCCL_ERROR_CODE(ret), tag.c_str()),
3220 : ret);
3221 :
3222 0 : return HCCL_SUCCESS;
3223 : }
3224 0 : if ((deviceType_ == DevType::DEV_TYPE_910_93) && (userRankSize_ != 1)) {
3225 0 : aicpuUnfoldMode = true;
3226 : }
3227 : }
3228 :
3229 15 : bool isCapture = StreamIsCapture(stream);
3230 :
3231 21 : if (!IsAtomicInit()) {
3232 0 : HCCL_ERROR(
3233 : "[HcclCommunicator][AllReduceOutPlace]errNo[0x%016llx] hccl init must be called before call this function",
3234 : HCCL_ERROR_CODE(HCCL_E_UNAVAIL));
3235 0 : return HCCL_E_UNAVAIL;
3236 : }
3237 :
3238 : // 设置notify wait模式
3239 21 : SyncMode preSyncMode = SyncMode::DEFAULT_TIMEWAITSYNCMODE;
3240 21 : GetAndSetSyncMode(preSyncMode, syncMode);
3241 :
3242 21 : Stream streamObj(stream);
3243 21 : CHK_RET(callbackTask_->CallbackRegStream(stream));
3244 :
3245 21 : std::vector<u32>& ranksPorts = groupNicRanksPort_.empty() ? nicRanksPort_ : groupNicRanksPort_;
3246 20 : std::vector<u32>& vnicRanksPorts = groupVnicRanksPort_.empty() ? vnicRanksPort_ : groupVnicRanksPort_;
3247 18 : implAlg_->SetHDCModeInfo(rankDevicePhyIdNicInfoMap_, ranksPorts, vnicRanksPorts, isSetHDCModeInfo_, isUseRankPort_);
3248 :
3249 16 : u32 perDataSize = SIZE_TABLE[dataType];
3250 16 : u64 totalSize = count * perDataSize;
3251 :
3252 16 : OpParam opParam;
3253 18 : opParam.tag = tag;
3254 20 : opParam.inputPtr = inputPtr;
3255 20 : opParam.inputSize = totalSize;
3256 20 : opParam.outputPtr = outputPtr;
3257 20 : opParam.outputSize = totalSize;
3258 20 : opParam.DataDes.count = count;
3259 20 : opParam.DataDes.dataType = dataType;
3260 20 : opParam.reduceType = op;
3261 20 : opParam.stream = streamObj;
3262 19 : opParam.syncMode = syncMode;
3263 19 : opParam.aicpuUnfoldMode = aicpuUnfoldMode;
3264 19 : opParam.aicpuCacheEnable = GetExternalInputAicpuCacheEnable();
3265 19 : opParam.isCapture = isCapture;
3266 19 : opParam.opBaseAtraceInfo = opBaseAtraceInfo_.get();
3267 19 : opParam.opType = HcclCMDType::HCCL_CMD_ALLREDUCE;
3268 :
3269 19 : CHK_RET(ExecOp(HcclCMDType::HCCL_CMD_ALLREDUCE, opParam));
3270 :
3271 15 : RestorePreSyncMode(preSyncMode, syncMode);
3272 15 : return HCCL_SUCCESS;
3273 16 : }
3274 :
3275 4 : HcclResult HcclCommunicator::AlltoAllV(
3276 : const void* sendBuf, const void* sendCounts, const void* sdispls, HcclDataType sendType, const void* recvBuf,
3277 : const void* recvCounts, const void* rdispls, HcclDataType recvType, rtStream_t stream, const std::string& tag)
3278 : {
3279 4 : CHK_RET(CheckSuspendingStatus());
3280 :
3281 4 : if (!IsAtomicInit()) {
3282 0 : HCCL_ERROR(
3283 : "[HcclCommunicator][AlltoAllV]errNo[0x%016llx] hccl init must be called before call this function",
3284 : HCCL_ERROR_CODE(HCCL_E_UNAVAIL));
3285 0 : return HCCL_E_UNAVAIL;
3286 : }
3287 :
3288 4 : if (IsNeedNicInit()) {
3289 4 : HCCL_INFO("InitNic.");
3290 4 : CHK_RET(InitNic());
3291 : }
3292 :
3293 4 : bool isCapture = StreamIsCapture(stream);
3294 :
3295 4 : Stream streamObj(stream);
3296 4 : CHK_RET(callbackTask_->CallbackRegStream(stream));
3297 :
3298 4 : std::vector<u32>& ranksPorts = groupNicRanksPort_.empty() ? nicRanksPort_ : groupNicRanksPort_;
3299 4 : std::vector<u32>& vnicRanksPorts = groupVnicRanksPort_.empty() ? vnicRanksPort_ : groupVnicRanksPort_;
3300 4 : implAlg_->SetHDCModeInfo(rankDevicePhyIdNicInfoMap_, ranksPorts, vnicRanksPorts, isSetHDCModeInfo_, isUseRankPort_);
3301 :
3302 4 : OpParam opParam;
3303 4 : opParam.tag = tag;
3304 4 : opParam.inputPtr = const_cast<void*>(sendBuf);
3305 4 : opParam.outputPtr = const_cast<void*>(recvBuf);
3306 4 : opParam.All2AllDataDes.sendType = sendType;
3307 4 : opParam.All2AllDataDes.recvType = recvType;
3308 4 : opParam.All2AllDataDes.sendCounts = const_cast<void*>(sendCounts);
3309 4 : opParam.All2AllDataDes.recvCounts = const_cast<void*>(recvCounts);
3310 4 : opParam.All2AllDataDes.sdispls = const_cast<void*>(sdispls);
3311 4 : opParam.All2AllDataDes.rdispls = const_cast<void*>(rdispls);
3312 4 : opParam.stream = streamObj;
3313 4 : opParam.opType = HcclCMDType::HCCL_CMD_ALLTOALLV;
3314 4 : opParam.aicpuUnfoldMode = EnableAicpuUnfold(isCapture);
3315 4 : opParam.aicpuCacheEnable = GetExternalInputAicpuCacheEnable();
3316 4 : opParam.isCapture = isCapture;
3317 :
3318 4 : if (UNLIKELY(GetDebugConfig() & HCCL_ALG)) {
3319 0 : for (u32 i = 0; i < userRankSize_; i++) {
3320 0 : HCCL_CONFIG_INFO(
3321 : HCCL_ALG,
3322 : "[HcclCommunicator][AlltoAllV] rank[%u], sendCounts[%llu], sendDispls[%llu] "
3323 : "recvCounts[%llu], recvDispls[%llu]",
3324 : userRank_, *(static_cast<const u64*>(opParam.All2AllDataDes.sendCounts) + i),
3325 : *(static_cast<const u64*>(opParam.All2AllDataDes.sdispls) + i),
3326 : *(static_cast<const u64*>(opParam.All2AllDataDes.recvCounts) + i),
3327 : *(static_cast<const u64*>(opParam.All2AllDataDes.rdispls) + i));
3328 : }
3329 : }
3330 :
3331 4 : CHK_RET(ExecOpAlltoAll(HcclCMDType::HCCL_CMD_ALLTOALLV, opParam));
3332 4 : return HCCL_SUCCESS;
3333 4 : }
3334 :
3335 0 : HcclResult HcclCommunicator::AlltoAllVOutPlace(
3336 : const void* sendBuf, const void* sendCounts, const void* sdispls, HcclDataType sendType, const void* recvBuf,
3337 : const void* recvCounts, const void* rdispls, HcclDataType recvType, rtStream_t stream, const std::string& tag)
3338 : {
3339 0 : CHK_RET(CheckSuspendingStatus());
3340 0 : CHK_PRT_RET(
3341 : Is310P3Common(isHaveCpuRank_, deviceType_),
3342 : HCCL_RUN_INFO("[AlltoAllVOutPlace]This method cannot be invoked in the current scenario."), HCCL_SUCCESS);
3343 0 : if (!IsAtomicInit()) {
3344 0 : HCCL_ERROR(
3345 : "[HcclCommunicator][AlltoAllVOutPlace]errNo[0x%016llx] hccl init must be called before call this function",
3346 : HCCL_ERROR_CODE(HCCL_E_UNAVAIL));
3347 0 : return HCCL_E_UNAVAIL;
3348 : }
3349 :
3350 0 : if (IsNeedNicInit()) {
3351 0 : HCCL_INFO("InitNic.");
3352 0 : CHK_RET(InitNic());
3353 : }
3354 :
3355 0 : bool isCapture = StreamIsCapture(stream);
3356 :
3357 0 : Stream streamObj(stream);
3358 0 : CHK_RET(callbackTask_->CallbackRegStream(stream));
3359 :
3360 0 : std::vector<u32>& ranksPorts = groupNicRanksPort_.empty() ? nicRanksPort_ : groupNicRanksPort_;
3361 0 : std::vector<u32>& vnicRanksPorts = groupVnicRanksPort_.empty() ? vnicRanksPort_ : groupVnicRanksPort_;
3362 0 : implAlg_->SetHDCModeInfo(rankDevicePhyIdNicInfoMap_, ranksPorts, vnicRanksPorts, isSetHDCModeInfo_, isUseRankPort_);
3363 :
3364 0 : OpParam opParam;
3365 0 : opParam.tag = tag;
3366 0 : opParam.inputPtr = const_cast<void*>(sendBuf);
3367 0 : opParam.outputPtr = const_cast<void*>(recvBuf);
3368 0 : opParam.All2AllDataDes.sendType = sendType;
3369 0 : opParam.All2AllDataDes.recvType = recvType;
3370 0 : opParam.All2AllDataDes.sendCounts = const_cast<void*>(sendCounts);
3371 0 : opParam.All2AllDataDes.recvCounts = const_cast<void*>(recvCounts);
3372 0 : opParam.All2AllDataDes.sdispls = const_cast<void*>(sdispls);
3373 0 : opParam.All2AllDataDes.rdispls = const_cast<void*>(rdispls);
3374 0 : opParam.stream = streamObj;
3375 0 : opParam.opType = HcclCMDType::HCCL_CMD_ALLTOALLV;
3376 0 : opParam.aicpuUnfoldMode = EnableAicpuUnfold(isCapture);
3377 0 : opParam.aicpuCacheEnable = GetExternalInputAicpuCacheEnable();
3378 0 : opParam.isCapture = isCapture;
3379 :
3380 0 : if (UNLIKELY(GetDebugConfig() & HCCL_ALG)) {
3381 0 : for (u32 i = 0; i < userRankSize_; i++) {
3382 0 : HCCL_CONFIG_INFO(
3383 : HCCL_ALG,
3384 : "[HcclCommunicator][AlltoAllVOutPlace] rank[%u], sendCounts[%llu],"
3385 : "sendDispls[%llu], recvCounts[%llu], recvDispls[%llu]",
3386 : userRank_, *(static_cast<const u64*>(opParam.All2AllDataDes.sendCounts) + i),
3387 : *(static_cast<const u64*>(opParam.All2AllDataDes.sdispls) + i),
3388 : *(static_cast<const u64*>(opParam.All2AllDataDes.recvCounts) + i),
3389 : *(static_cast<const u64*>(opParam.All2AllDataDes.rdispls) + i));
3390 : }
3391 : }
3392 :
3393 0 : CHK_RET(ExecOpAlltoAll(HcclCMDType::HCCL_CMD_ALLTOALLV, opParam));
3394 0 : return HCCL_SUCCESS;
3395 0 : }
3396 :
3397 1 : HcclResult HcclCommunicator::AlltoAllVC(
3398 : const void* sendBuf, const void* sendCountMatrix, HcclDataType sendType, const void* recvBuf, HcclDataType recvType,
3399 : rtStream_t stream, const std::string& tag)
3400 : {
3401 1 : CHK_RET(CheckSuspendingStatus());
3402 1 : if (Is310P3Common(isHaveCpuRank_, deviceType_)) {
3403 0 : HCCL_ERROR(
3404 : "[%s][%s]AlltoAllVC is not supported", LOG_KEYWORDS_TASK_EXEC.c_str(), LOG_KEYWORDS_NOT_SUPPORTED.c_str());
3405 0 : return HCCL_E_NOT_SUPPORT;
3406 : }
3407 1 : if (!IsAtomicInit()) {
3408 0 : HCCL_ERROR(
3409 : "[HcclCommunicator][AlltoAllVC]errNo[0x%016llx] hccl init must be called before call this function",
3410 : HCCL_ERROR_CODE(HCCL_E_UNAVAIL));
3411 0 : return HCCL_E_UNAVAIL;
3412 : }
3413 :
3414 1 : if (IsNeedNicInit()) {
3415 1 : HCCL_INFO("InitNic.");
3416 1 : CHK_RET(InitNic());
3417 : }
3418 :
3419 1 : bool isCapture = StreamIsCapture(stream);
3420 :
3421 1 : Stream streamObj(stream);
3422 1 : CHK_RET(callbackTask_->CallbackRegStream(stream));
3423 :
3424 1 : std::vector<u32>& ranksPorts = groupNicRanksPort_.empty() ? nicRanksPort_ : groupNicRanksPort_;
3425 1 : std::vector<u32>& vnicRanksPorts = groupVnicRanksPort_.empty() ? vnicRanksPort_ : groupVnicRanksPort_;
3426 1 : implAlg_->SetHDCModeInfo(rankDevicePhyIdNicInfoMap_, ranksPorts, vnicRanksPorts, isSetHDCModeInfo_, isUseRankPort_);
3427 :
3428 1 : OpParam opParam;
3429 1 : opParam.tag = tag;
3430 1 : opParam.inputPtr = const_cast<void*>(sendBuf);
3431 1 : opParam.outputPtr = const_cast<void*>(recvBuf);
3432 1 : opParam.All2AllDataDes.sendType = sendType;
3433 1 : opParam.All2AllDataDes.recvType = recvType;
3434 1 : opParam.All2AllDataDes.sendCountMatrix = const_cast<void*>(sendCountMatrix);
3435 1 : opParam.stream = streamObj;
3436 1 : opParam.opType = HcclCMDType::HCCL_CMD_ALLTOALLVC;
3437 1 : opParam.aicpuUnfoldMode = EnableAicpuUnfold(isCapture);
3438 1 : opParam.aicpuCacheEnable = GetExternalInputAicpuCacheEnable();
3439 1 : opParam.isCapture = isCapture;
3440 :
3441 1 : if (UNLIKELY(GetDebugConfig() & HCCL_ALG)) {
3442 0 : for (u32 i = 0; i < userRankSize_; i++) {
3443 0 : for (u32 j = 0; j < userRankSize_; j++) {
3444 0 : HCCL_CONFIG_DEBUG(
3445 : HCCL_ALG,
3446 : "[HcclCommunicator][AlltoAllVC] usrRank[%u] rank[%u] to remoteRank[%u], "
3447 : "sendCounts[%llu]",
3448 : userRank_, i, j,
3449 : *(static_cast<const u64*>(opParam.All2AllDataDes.sendCountMatrix) + i * userRankSize_ + j));
3450 : }
3451 : }
3452 : }
3453 :
3454 1 : CHK_RET(ExecOpAlltoAll(HcclCMDType::HCCL_CMD_ALLTOALLVC, opParam));
3455 1 : return HCCL_SUCCESS;
3456 1 : }
3457 :
3458 0 : HcclResult HcclCommunicator::AlltoAllVCOutPlace(
3459 : const void* sendBuf, const void* sendCountMatrix, HcclDataType sendType, const void* recvBuf, HcclDataType recvType,
3460 : rtStream_t stream, const std::string& tag)
3461 : {
3462 0 : CHK_RET(CheckSuspendingStatus());
3463 0 : CHK_PRT_RET(
3464 : Is310P3Common(isHaveCpuRank_, deviceType_),
3465 : HCCL_RUN_INFO("[AlltoAllVCOutPlace]This method cannot be invoked in the current scenario."), HCCL_SUCCESS);
3466 :
3467 0 : if (!IsAtomicInit()) {
3468 0 : HCCL_ERROR(
3469 : "[HcclCommunicator][AlltoAllVCOutPlace]errNo[0x%016llx] hccl init must be called before call this function",
3470 : HCCL_ERROR_CODE(HCCL_E_UNAVAIL));
3471 0 : return HCCL_E_UNAVAIL;
3472 : }
3473 :
3474 0 : if (IsNeedNicInit()) {
3475 0 : HCCL_INFO("InitNic");
3476 0 : CHK_RET(InitNic());
3477 : }
3478 :
3479 0 : bool isCapture = StreamIsCapture(stream);
3480 :
3481 0 : Stream streamObj(stream);
3482 0 : CHK_RET(callbackTask_->CallbackRegStream(stream));
3483 :
3484 0 : std::vector<u32>& ranksPorts = groupNicRanksPort_.empty() ? nicRanksPort_ : groupNicRanksPort_;
3485 0 : std::vector<u32>& vnicRanksPorts = groupVnicRanksPort_.empty() ? vnicRanksPort_ : groupVnicRanksPort_;
3486 0 : implAlg_->SetHDCModeInfo(rankDevicePhyIdNicInfoMap_, ranksPorts, vnicRanksPorts, isSetHDCModeInfo_, isUseRankPort_);
3487 :
3488 0 : OpParam opParam;
3489 0 : opParam.tag = tag;
3490 0 : opParam.inputPtr = const_cast<void*>(sendBuf);
3491 0 : opParam.outputPtr = const_cast<void*>(recvBuf);
3492 0 : opParam.All2AllDataDes.sendType = sendType;
3493 0 : opParam.All2AllDataDes.recvType = recvType;
3494 0 : opParam.All2AllDataDes.sendCountMatrix = const_cast<void*>(sendCountMatrix);
3495 0 : opParam.stream = streamObj;
3496 0 : opParam.opType = HcclCMDType::HCCL_CMD_ALLTOALLVC;
3497 0 : opParam.aicpuUnfoldMode = EnableAicpuUnfold(isCapture);
3498 0 : opParam.aicpuCacheEnable = GetExternalInputAicpuCacheEnable();
3499 0 : opParam.isCapture = isCapture;
3500 :
3501 0 : if (UNLIKELY(GetDebugConfig() & HCCL_ALG)) {
3502 0 : for (u32 i = 0; i < userRankSize_; i++) {
3503 0 : for (u32 j = 0; j < userRankSize_; j++) {
3504 0 : HCCL_CONFIG_DEBUG(
3505 : HCCL_ALG,
3506 : "[HcclCommunicator][AlltoAllVCOutPlace] usrRank[%u] rank[%u]"
3507 : "to remoteRank[%u], sendCounts[%llu]",
3508 : userRank_, i, j,
3509 : *(static_cast<const u64*>(opParam.All2AllDataDes.sendCountMatrix) + i * userRankSize_ + j));
3510 : }
3511 : }
3512 : }
3513 :
3514 0 : CHK_RET(ExecOpAlltoAll(HcclCMDType::HCCL_CMD_ALLTOALLVC, opParam));
3515 0 : return HCCL_SUCCESS;
3516 0 : }
3517 :
3518 0 : HcclResult HcclCommunicator::AlltoAll(
3519 : const void* sendBuf, u64 sendCount, HcclDataType sendType, const void* recvBuf, u64 recvCount,
3520 : HcclDataType recvType, rtStream_t stream, const std::string& tag)
3521 : {
3522 0 : CHK_RET(CheckSuspendingStatus());
3523 0 : if (!IsAtomicInit()) {
3524 0 : HCCL_ERROR(
3525 : "[HcclCommunicator][AlltoAll]errNo[0x%016llx] hccl init must be called before call this function",
3526 : HCCL_ERROR_CODE(HCCL_E_UNAVAIL));
3527 0 : return HCCL_E_UNAVAIL;
3528 : }
3529 :
3530 0 : if (IsNeedNicInit()) {
3531 0 : HCCL_INFO("InitNic.");
3532 0 : CHK_RET(InitNic());
3533 : }
3534 :
3535 0 : bool isCapture = StreamIsCapture(stream);
3536 :
3537 0 : CHK_RET(callbackTask_->CallbackRegStream(stream));
3538 :
3539 : // 生成sendCountMatrix矩阵，alltoall的底层实现走alltoallvc
3540 0 : std::vector<u64> sendCountMatrix(userRankSize_ * userRankSize_, sendCount);
3541 :
3542 0 : OpParam opParam;
3543 0 : opParam.tag = tag;
3544 0 : opParam.inputPtr = const_cast<void*>(sendBuf);
3545 0 : opParam.outputPtr = const_cast<void*>(recvBuf);
3546 0 : opParam.All2AllDataDes.sendType = sendType;
3547 0 : opParam.All2AllDataDes.recvType = recvType;
3548 0 : opParam.All2AllDataDes.sendCount = sendCount;
3549 0 : opParam.All2AllDataDes.recvCount = recvCount;
3550 0 : opParam.All2AllDataDes.sendCountMatrix = static_cast<void*>(sendCountMatrix.data());
3551 0 : opParam.stream = Stream(stream);
3552 0 : opParam.opType = HcclCMDType::HCCL_CMD_ALLTOALL;
3553 0 : opParam.aicpuUnfoldMode = false;
3554 0 : opParam.aicpuCacheEnable = 0;
3555 0 : opParam.isCapture = isCapture;
3556 0 : opParam.inputSize = sendCount * SIZE_TABLE[sendType] * userRankSize_;
3557 0 : opParam.outputSize = recvCount * SIZE_TABLE[recvType] * userRankSize_;
3558 0 : if (EnableAicpuUnfold(isCapture)) {
3559 0 : opParam.aicpuUnfoldMode = true;
3560 0 : opParam.aicpuCacheEnable = GetExternalInputAicpuCacheEnable();
3561 : }
3562 :
3563 0 : std::vector<u32>& ranksPorts = groupNicRanksPort_.empty() ? nicRanksPort_ : groupNicRanksPort_;
3564 0 : std::vector<u32>& vnicRanksPorts = groupVnicRanksPort_.empty() ? vnicRanksPort_ : groupVnicRanksPort_;
3565 0 : implAlg_->SetHDCModeInfo(rankDevicePhyIdNicInfoMap_, ranksPorts, vnicRanksPorts, isSetHDCModeInfo_, isUseRankPort_);
3566 0 : CHK_RET(ExecOpAlltoAll(HcclCMDType::HCCL_CMD_ALLTOALL, opParam));
3567 0 : return HCCL_SUCCESS;
3568 0 : }
3569 :
3570 2 : HcclResult HcclCommunicator::Broadcast(
3571 : const std::string& tag, void* ptr, u64 count, HcclDataType dataType, u32 root, HcclRtStream stream)
3572 : {
3573 2 : CHK_RET(CheckSuspendingStatus());
3574 2 : bool aicpuUnfoldMode = false;
3575 2 : if (GetAicpuUnfoldConfig() == true && deviceType_ == DevType::DEV_TYPE_910_93 && (userRankSize_ != 1)) {
3576 0 : aicpuUnfoldMode = true;
3577 : }
3578 :
3579 2 : if (!IsAtomicInit()) {
3580 0 : HCCL_ERROR(
3581 : "[HcclCommunicator][Broadcast]errNo[0x%016llx] hccl init must be called before call this function",
3582 : HCCL_ERROR_CODE(HCCL_E_UNAVAIL));
3583 0 : return HCCL_E_UNAVAIL;
3584 : }
3585 :
3586 2 : Stream streamObj(stream);
3587 2 : CHK_RET(callbackTask_->CallbackRegStream(stream));
3588 :
3589 2 : std::vector<u32>& ranksPorts = groupNicRanksPort_.empty() ? nicRanksPort_ : groupNicRanksPort_;
3590 2 : std::vector<u32>& vnicRanksPorts = groupVnicRanksPort_.empty() ? vnicRanksPort_ : groupVnicRanksPort_;
3591 2 : implAlg_->SetHDCModeInfo(rankDevicePhyIdNicInfoMap_, ranksPorts, vnicRanksPorts, isSetHDCModeInfo_, isUseRankPort_);
3592 2 : u32 perDataSize = SIZE_TABLE[dataType];
3593 2 : u64 totalSize = count * perDataSize;
3594 :
3595 2 : OpParam opParam;
3596 2 : opParam.tag = tag;
3597 2 : opParam.inputPtr = ptr;
3598 2 : opParam.outputPtr = ptr;
3599 2 : opParam.inputSize = totalSize;
3600 2 : opParam.outputSize = totalSize;
3601 2 : opParam.DataDes.count = count;
3602 2 : opParam.DataDes.dataType = dataType;
3603 2 : opParam.root = root;
3604 2 : opParam.stream = streamObj;
3605 2 : opParam.aicpuUnfoldMode = aicpuUnfoldMode;
3606 2 : opParam.aicpuCacheEnable = GetExternalInputAicpuCacheEnable();
3607 2 : opParam.opBaseAtraceInfo = opBaseAtraceInfo_.get();
3608 2 : opParam.opType = HcclCMDType::HCCL_CMD_BROADCAST;
3609 :
3610 2 : CHK_RET(ExecOp(HcclCMDType::HCCL_CMD_BROADCAST, opParam));
3611 :
3612 2 : return HCCL_SUCCESS;
3613 2 : }
3614 :
3615 0 : HcclResult HcclCommunicator::BroadcastOutPlace(
3616 : const std::string& tag, void* ptr, u64 count, HcclDataType dataType, u32 root, HcclRtStream stream)
3617 : {
3618 0 : CHK_RET(CheckSuspendingStatus());
3619 0 : bool aicpuUnfoldMode = false;
3620 0 : if (GetAicpuUnfoldConfig() == true && deviceType_ == DevType::DEV_TYPE_910_93 && (userRankSize_ != 1)) {
3621 0 : aicpuUnfoldMode = true;
3622 : }
3623 :
3624 0 : CHK_PRT_RET(
3625 : Is310P3Common(isHaveCpuRank_, deviceType_),
3626 : HCCL_RUN_INFO("[BroadcastOutPlace]This method cannot be invoked in the current scenario."), HCCL_SUCCESS);
3627 :
3628 0 : if (!IsAtomicInit()) {
3629 0 : HCCL_ERROR(
3630 : "[HcclCommunicator][BroadcastOutPlace]errNo[0x%016llx] hccl init must be called before"
3631 : " call this function",
3632 : HCCL_ERROR_CODE(HCCL_E_UNAVAIL));
3633 0 : return HCCL_E_UNAVAIL;
3634 : }
3635 :
3636 0 : bool isCapture = StreamIsCapture(stream);
3637 :
3638 0 : Stream streamObj(stream);
3639 0 : CHK_RET(callbackTask_->CallbackRegStream(stream));
3640 :
3641 0 : std::vector<u32>& ranksPorts = groupNicRanksPort_.empty() ? nicRanksPort_ : groupNicRanksPort_;
3642 0 : std::vector<u32>& vnicRanksPorts = groupVnicRanksPort_.empty() ? vnicRanksPort_ : groupVnicRanksPort_;
3643 0 : implAlg_->SetHDCModeInfo(rankDevicePhyIdNicInfoMap_, ranksPorts, vnicRanksPorts, isSetHDCModeInfo_, isUseRankPort_);
3644 :
3645 0 : u32 perDataSize = SIZE_TABLE[dataType];
3646 0 : u64 totalSize = count * perDataSize;
3647 :
3648 0 : OpParam opParam;
3649 0 : opParam.tag = tag;
3650 0 : opParam.inputPtr = ptr;
3651 0 : opParam.outputPtr = ptr;
3652 0 : opParam.inputSize = totalSize;
3653 0 : opParam.outputSize = totalSize;
3654 0 : opParam.DataDes.count = count;
3655 0 : opParam.DataDes.dataType = dataType;
3656 0 : opParam.root = root;
3657 0 : opParam.stream = streamObj;
3658 0 : opParam.aicpuUnfoldMode = aicpuUnfoldMode;
3659 0 : opParam.aicpuCacheEnable = GetExternalInputAicpuCacheEnable();
3660 0 : opParam.isCapture = isCapture;
3661 0 : opParam.opType = HcclCMDType::HCCL_CMD_BROADCAST;
3662 :
3663 0 : CHK_RET(ExecOp(HcclCMDType::HCCL_CMD_BROADCAST, opParam));
3664 :
3665 0 : return HCCL_SUCCESS;
3666 0 : }
3667 :
3668 0 : HcclResult HcclCommunicator::Scatter(
3669 : const std::string& tag, void* inputPtr, void* outputPtr, u64 recvCount, HcclDataType dataType, u32 root,
3670 : HcclRtStream stream)
3671 : {
3672 0 : CHK_RET(CheckSuspendingStatus());
3673 0 : if (Is310P3Common(isHaveCpuRank_, deviceType_)) {
3674 0 : HCCL_ERROR(
3675 : "[%s][%s]Scatter Not Supported Yet", LOG_KEYWORDS_TASK_EXEC.c_str(), LOG_KEYWORDS_NOT_SUPPORTED.c_str());
3676 0 : return HCCL_E_NOT_SUPPORT;
3677 : }
3678 0 : bool aicpuUnfoldMode = false;
3679 0 : if (GetAicpuUnfoldConfig() == true && deviceType_ == DevType::DEV_TYPE_910_93 && (userRankSize_ != 1)) {
3680 0 : aicpuUnfoldMode = true;
3681 : }
3682 :
3683 0 : if (!IsAtomicInit()) {
3684 0 : HCCL_ERROR(
3685 : "[HcclCommunicator][Scatter]errNo[0x%016llx] hccl init must be called before call this function",
3686 : HCCL_ERROR_CODE(HCCL_E_UNAVAIL));
3687 0 : return HCCL_E_UNAVAIL;
3688 : }
3689 :
3690 0 : Stream streamObj(stream);
3691 0 : CHK_RET(callbackTask_->CallbackRegStream(stream));
3692 :
3693 0 : std::vector<u32>& ranksPorts = groupNicRanksPort_.empty() ? nicRanksPort_ : groupNicRanksPort_;
3694 0 : std::vector<u32>& vnicRanksPorts = groupVnicRanksPort_.empty() ? vnicRanksPort_ : groupVnicRanksPort_;
3695 0 : implAlg_->SetHDCModeInfo(rankDevicePhyIdNicInfoMap_, ranksPorts, vnicRanksPorts, isSetHDCModeInfo_, isUseRankPort_);
3696 :
3697 0 : u32 perDataSize = SIZE_TABLE[dataType];
3698 0 : u64 outputSize = recvCount * perDataSize;
3699 0 : u64 totalSize = outputSize * userRankSize_;
3700 :
3701 0 : OpParam opParam;
3702 0 : opParam.tag = tag;
3703 0 : opParam.inputPtr = inputPtr;
3704 0 : opParam.inputSize = totalSize;
3705 0 : opParam.outputPtr = outputPtr;
3706 0 : opParam.outputSize = totalSize;
3707 0 : opParam.DataDes.count = recvCount;
3708 0 : opParam.DataDes.dataType = dataType;
3709 0 : opParam.stream = streamObj;
3710 0 : opParam.aicpuUnfoldMode = aicpuUnfoldMode;
3711 0 : opParam.aicpuCacheEnable = GetExternalInputAicpuCacheEnable();
3712 0 : opParam.root = root;
3713 0 : opParam.opType = HcclCMDType::HCCL_CMD_SCATTER;
3714 0 : CHK_RET(ExecOp(HcclCMDType::HCCL_CMD_SCATTER, opParam));
3715 0 : return HCCL_SUCCESS;
3716 0 : }
3717 :
3718 3 : HcclResult HcclCommunicator::ScatterOutPlace(
3719 : const std::string& tag, void* inputPtr, void* outputPtr, u64 recvCount, HcclDataType dataType, u32 root,
3720 : HcclRtStream stream)
3721 : {
3722 3 : CHK_RET(CheckSuspendingStatus());
3723 10 : if (Is310P3Common(isHaveCpuRank_, deviceType_)) {
3724 0 : HCCL_ERROR(
3725 : "[%s][%s]ScatterOutPlace Not Supported Yet", LOG_KEYWORDS_TASK_EXEC.c_str(),
3726 : LOG_KEYWORDS_NOT_SUPPORTED.c_str());
3727 0 : return HCCL_E_NOT_SUPPORT;
3728 : }
3729 :
3730 3 : bool aicpuUnfoldMode = false;
3731 3 : if (GetAicpuUnfoldConfig() == true && (deviceType_ == DevType::DEV_TYPE_910_93) && (userRankSize_ != 1)) {
3732 0 : aicpuUnfoldMode = true;
3733 : }
3734 :
3735 9 : bool isCapture = StreamIsCapture(stream);
3736 :
3737 10 : if (!IsAtomicInit()) {
3738 0 : HCCL_ERROR(
3739 : "[HcclCommunicator][ScatterOutPlace]errNo[0x%016llx] hccl init must be called before"
3740 : " call this function",
3741 : HCCL_ERROR_CODE(HCCL_E_UNAVAIL));
3742 0 : return HCCL_E_UNAVAIL;
3743 : }
3744 :
3745 12 : Stream streamObj(stream);
3746 12 : CHK_RET(callbackTask_->CallbackRegStream(stream));
3747 :
3748 12 : std::vector<u32>& ranksPorts = groupNicRanksPort_.empty() ? nicRanksPort_ : groupNicRanksPort_;
3749 11 : std::vector<u32>& vnicRanksPorts = groupVnicRanksPort_.empty() ? vnicRanksPort_ : groupVnicRanksPort_;
3750 9 : implAlg_->SetHDCModeInfo(rankDevicePhyIdNicInfoMap_, ranksPorts, vnicRanksPorts, isSetHDCModeInfo_, isUseRankPort_);
3751 :
3752 9 : u32 perDataSize = SIZE_TABLE[dataType];
3753 9 : u64 outputSize = recvCount * perDataSize;
3754 9 : u64 totalSize = outputSize * userRankSize_;
3755 :
3756 9 : OpParam opParam;
3757 11 : opParam.tag = tag;
3758 11 : opParam.inputPtr = inputPtr;
3759 11 : opParam.inputSize = totalSize;
3760 11 : opParam.outputPtr = outputPtr;
3761 11 : opParam.outputSize = totalSize;
3762 11 : opParam.DataDes.count = recvCount;
3763 11 : opParam.DataDes.dataType = dataType;
3764 11 : opParam.stream = streamObj;
3765 12 : opParam.aicpuUnfoldMode = aicpuUnfoldMode;
3766 12 : opParam.aicpuCacheEnable = GetExternalInputAicpuCacheEnable();
3767 12 : opParam.isCapture = isCapture;
3768 12 : opParam.root = root;
3769 12 : opParam.opBaseAtraceInfo = opBaseAtraceInfo_.get();
3770 12 : opParam.opType = HcclCMDType::HCCL_CMD_SCATTER;
3771 :
3772 12 : CHK_RET(ExecOp(HcclCMDType::HCCL_CMD_SCATTER, opParam));
3773 :
3774 8 : return HCCL_SUCCESS;
3775 12 : }
3776 :
3777 5 : HcclResult HcclCommunicator::Reduce(
3778 : const std::string& tag, void* inputPtr, void* outputPtr, u64 count, HcclDataType dataType, HcclReduceOp op,
3779 : u32 root, HcclRtStream stream)
3780 : {
3781 5 : CHK_RET(CheckSuspendingStatus());
3782 5 : if (Is310P3Common(isHaveCpuRank_, deviceType_)) {
3783 0 : HCCL_ERROR(
3784 : "[%s][%s]Reduce Not Supported Yet", LOG_KEYWORDS_TASK_EXEC.c_str(), LOG_KEYWORDS_NOT_SUPPORTED.c_str());
3785 0 : return HCCL_E_NOT_SUPPORT;
3786 : }
3787 5 : bool aicpuUnfoldMode = false;
3788 10 : if (GetAicpuUnfoldConfig() == true && IsSupportSDMAReduce(inputPtr, outputPtr, dataType, op)
3789 10 : && (deviceType_ == DevType::DEV_TYPE_910_93) && (userRankSize_ != 1)) {
3790 0 : aicpuUnfoldMode = true;
3791 : }
3792 :
3793 5 : if (!IsAtomicInit()) {
3794 0 : HCCL_ERROR(
3795 : "[HcclCommunicator][Reduce]errNo[0x%016llx] hccl init must be called before call this function",
3796 : HCCL_ERROR_CODE(HCCL_E_UNAVAIL));
3797 0 : return HCCL_E_UNAVAIL;
3798 : }
3799 :
3800 5 : Stream streamObj(stream);
3801 5 : CHK_RET(callbackTask_->CallbackRegStream(stream));
3802 :
3803 5 : u32 perDataSize = SIZE_TABLE[dataType];
3804 5 : u64 totalSize = count * perDataSize;
3805 5 : OpParam opParam;
3806 5 : opParam.tag = tag;
3807 5 : opParam.inputPtr = inputPtr;
3808 5 : opParam.inputSize = totalSize;
3809 5 : opParam.outputPtr = outputPtr;
3810 5 : opParam.outputSize = totalSize;
3811 5 : opParam.DataDes.count = count;
3812 5 : opParam.DataDes.dataType = dataType;
3813 5 : opParam.reduceType = op;
3814 5 : opParam.root = root;
3815 5 : opParam.stream = streamObj;
3816 5 : opParam.aicpuUnfoldMode = aicpuUnfoldMode;
3817 5 : opParam.aicpuCacheEnable = GetExternalInputAicpuCacheEnable();
3818 5 : opParam.opType = HcclCMDType::HCCL_CMD_REDUCE;
3819 :
3820 5 : CHK_RET(ExecOp(HcclCMDType::HCCL_CMD_REDUCE, opParam));
3821 :
3822 5 : return HCCL_SUCCESS;
3823 5 : }
3824 :
3825 4 : HcclResult HcclCommunicator::ReduceOutPlace(
3826 : const std::string& tag, void* inputPtr, void* outputPtr, u64 count, HcclDataType dataType, HcclReduceOp op,
3827 : u32 root, HcclRtStream stream)
3828 : {
3829 4 : CHK_RET(CheckSuspendingStatus());
3830 4 : CHK_PRT_RET(
3831 : Is310P3Common(isHaveCpuRank_, deviceType_),
3832 : HCCL_RUN_INFO("[ReduceOutPlace]This method cannot be invoked in the current scenario."), HCCL_SUCCESS);
3833 :
3834 3 : bool aicpuUnfoldMode = false;
3835 3 : if (GetAicpuUnfoldConfig() == true && IsSupportSDMAReduce(inputPtr, outputPtr, dataType, op)
3836 4 : && (deviceType_ == DevType::DEV_TYPE_910_93) && (userRankSize_ != 1)) {
3837 0 : aicpuUnfoldMode = true;
3838 : }
3839 :
3840 4 : bool isCapture = StreamIsCapture(stream);
3841 :
3842 3 : if (!IsAtomicInit()) {
3843 0 : HCCL_ERROR(
3844 : "[HcclCommunicator][ReduceOutPlace]errNo[0x%016llx] hccl init must be called before call this function",
3845 : HCCL_ERROR_CODE(HCCL_E_UNAVAIL));
3846 0 : return HCCL_E_UNAVAIL;
3847 : }
3848 :
3849 4 : Stream streamObj(stream);
3850 4 : CHK_RET(callbackTask_->CallbackRegStream(stream));
3851 :
3852 4 : std::vector<u32>& ranksPorts = groupNicRanksPort_.empty() ? nicRanksPort_ : groupNicRanksPort_;
3853 4 : std::vector<u32>& vnicRanksPorts = groupVnicRanksPort_.empty() ? vnicRanksPort_ : groupVnicRanksPort_;
3854 3 : implAlg_->SetHDCModeInfo(rankDevicePhyIdNicInfoMap_, ranksPorts, vnicRanksPorts, isSetHDCModeInfo_, isUseRankPort_);
3855 :
3856 4 : u32 perDataSize = SIZE_TABLE[dataType];
3857 4 : u64 totalSize = count * perDataSize;
3858 4 : OpParam opParam;
3859 4 : opParam.tag = tag;
3860 4 : opParam.inputPtr = inputPtr;
3861 4 : opParam.inputSize = totalSize;
3862 4 : opParam.outputPtr = outputPtr;
3863 4 : opParam.outputSize = totalSize;
3864 4 : opParam.DataDes.count = count;
3865 4 : opParam.DataDes.dataType = dataType;
3866 4 : opParam.reduceType = op;
3867 4 : opParam.root = root;
3868 4 : opParam.stream = streamObj;
3869 4 : opParam.opBaseAtraceInfo = opBaseAtraceInfo_.get();
3870 4 : opParam.aicpuUnfoldMode = aicpuUnfoldMode;
3871 4 : opParam.aicpuCacheEnable = GetExternalInputAicpuCacheEnable();
3872 4 : opParam.isCapture = isCapture;
3873 4 : opParam.opType = HcclCMDType::HCCL_CMD_REDUCE;
3874 :
3875 4 : CHK_RET(ExecOp(HcclCMDType::HCCL_CMD_REDUCE, opParam));
3876 :
3877 0 : return HCCL_SUCCESS;
3878 4 : }
3879 :
3880 11 : HcclResult HcclCommunicator::ReduceScatter(
3881 : const std::string& tag, void* inputPtr, void* outputPtr, u64 count, HcclDataType dataType, HcclReduceOp op,
3882 : HcclRtStream stream, HcomCollOpInfo* opInfo)
3883 : {
3884 11 : CHK_RET(CheckSuspendingStatus());
3885 11 : bool aicpuUnfoldMode = false;
3886 14 : if (GetAicpuUnfoldConfig() == true && IsSupportSDMAReduce(inputPtr, outputPtr, dataType, op)
3887 14 : && (deviceType_ == DevType::DEV_TYPE_910_93) && (userRankSize_ != 1)) {
3888 0 : aicpuUnfoldMode = true;
3889 : }
3890 :
3891 11 : if (!IsAtomicInit()) {
3892 0 : HCCL_ERROR(
3893 : "[HcclCommunicator][ReduceScatter]errNo[0x%016llx] hccl init must be called before call this function",
3894 : HCCL_ERROR_CODE(HCCL_E_UNAVAIL));
3895 0 : return HCCL_E_UNAVAIL;
3896 : }
3897 :
3898 21 : Stream streamObj(stream);
3899 21 : CHK_RET(callbackTask_->CallbackRegStream(stream));
3900 :
3901 21 : std::vector<u32>& ranksPorts = groupNicRanksPort_.empty() ? nicRanksPort_ : groupNicRanksPort_;
3902 19 : std::vector<u32>& vnicRanksPorts = groupVnicRanksPort_.empty() ? vnicRanksPort_ : groupVnicRanksPort_;
3903 17 : implAlg_->SetHDCModeInfo(rankDevicePhyIdNicInfoMap_, ranksPorts, vnicRanksPorts, isSetHDCModeInfo_, isUseRankPort_);
3904 :
3905 13 : u32 perDataSize = SIZE_TABLE[dataType];
3906 :
3907 13 : OpParam opParam;
3908 15 : opParam.tag = tag;
3909 21 : opParam.inputPtr = inputPtr;
3910 21 : opParam.inputSize = userRankSize_ * count * perDataSize;
3911 21 : opParam.outputPtr = outputPtr;
3912 21 : opParam.outputSize = count * perDataSize;
3913 21 : opParam.DataDes.count = count;
3914 21 : opParam.DataDes.dataType = dataType;
3915 21 : opParam.reduceType = op;
3916 21 : opParam.stream = streamObj;
3917 16 : opParam.opType = HcclCMDType::HCCL_CMD_REDUCE_SCATTER;
3918 16 : opParam.aicpuUnfoldMode = aicpuUnfoldMode;
3919 16 : opParam.aicpuCacheEnable = GetExternalInputAicpuCacheEnable();
3920 : // 用于inplace支持重执行场景的图模式归一至单算子模式
3921 15 : retryOrigWorkflowMode_ = GetWorkflowMode();
3922 : bool isHcclOpInplace
3923 15 : = IsHcclOpInplace(HcclCMDType::HCCL_CMD_REDUCE_SCATTER, opParam, userRank_, userRankSize_, isInplaceStatus_);
3924 21 : if (aicpuUnfoldMode && retryEnable_ && isHcclOpInplace) {
3925 0 : HCCL_DEBUG(
3926 : "The retry with inplace case is expected to be supported, "
3927 : "aicpuUnfoldMode[%d], retryEnable_[%d], isHcclOpInplace[%d], "
3928 : "therefore HcclWorkflowMode is converted from [%d] to HCCL_WORKFLOW_MODE_OP_BASE",
3929 : aicpuUnfoldMode, retryEnable_, isHcclOpInplace, static_cast<u8>(retryOrigWorkflowMode_));
3930 0 : CHK_RET(SetWorkflowMode(HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE));
3931 : }
3932 :
3933 21 : CHK_RET(ExecOp(HcclCMDType::HCCL_CMD_REDUCE_SCATTER, opParam));
3934 :
3935 8 : CHK_RET(SetWorkflowMode(retryOrigWorkflowMode_));
3936 8 : return HCCL_SUCCESS;
3937 19 : }
3938 :
3939 17 : HcclResult HcclCommunicator::ReduceScatterOutPlace(
3940 : const std::string& tag, void* inputPtr, void* outputPtr, u64 count, HcclDataType dataType, HcclReduceOp op,
3941 : HcclRtStream stream)
3942 : {
3943 17 : CHK_RET(CheckSuspendingStatus());
3944 19 : if (userRankSize_ > 1) {
3945 20 : CHK_RET(CreateCommCCLbuffer());
3946 : }
3947 :
3948 36 : bool aicpuUnfoldMode = false;
3949 36 : if (GetAicpuUnfoldConfig() == true
3950 16 : && IsSupportSDMAReduce(
3951 16 : cclBufferManager_.GetInCCLbuffer().ptr(), cclBufferManager_.GetOutCCLbuffer().ptr(), dataType, op)
3952 53 : && (deviceType_ == DevType::DEV_TYPE_910_93) && (userRankSize_ != 1)) {
3953 4 : aicpuUnfoldMode = true;
3954 : }
3955 :
3956 37 : bool isCapture = StreamIsCapture(stream);
3957 :
3958 37 : if (!IsAtomicInit()) {
3959 0 : HCCL_ERROR(
3960 : "[HcclCommunicator][ReduceScatterOutPlace]errNo[0x%016llx] hccl init must be called before"
3961 : " call this function",
3962 : HCCL_ERROR_CODE(HCCL_E_UNAVAIL));
3963 0 : return HCCL_E_UNAVAIL;
3964 : }
3965 :
3966 37 : Stream streamObj(stream);
3967 37 : CHK_RET(callbackTask_->CallbackRegStream(stream));
3968 :
3969 37 : std::vector<u32>& ranksPorts = groupNicRanksPort_.empty() ? nicRanksPort_ : groupNicRanksPort_;
3970 37 : std::vector<u32>& vnicRanksPorts = groupVnicRanksPort_.empty() ? vnicRanksPort_ : groupVnicRanksPort_;
3971 37 : implAlg_->SetHDCModeInfo(rankDevicePhyIdNicInfoMap_, ranksPorts, vnicRanksPorts, isSetHDCModeInfo_, isUseRankPort_);
3972 :
3973 35 : u32 perDataSize = SIZE_TABLE[dataType];
3974 :
3975 35 : OpParam opParam;
3976 33 : opParam.tag = tag;
3977 37 : opParam.inputPtr = inputPtr;
3978 37 : opParam.inputSize = userRankSize_ * count * perDataSize;
3979 37 : opParam.outputPtr = outputPtr;
3980 37 : opParam.outputSize = count * perDataSize;
3981 37 : opParam.DataDes.count = count;
3982 37 : opParam.DataDes.dataType = dataType;
3983 37 : opParam.reduceType = op;
3984 37 : opParam.stream = streamObj;
3985 34 : opParam.opType = HcclCMDType::HCCL_CMD_REDUCE_SCATTER;
3986 34 : opParam.aicpuUnfoldMode = aicpuUnfoldMode;
3987 34 : opParam.aicpuCacheEnable = GetExternalInputAicpuCacheEnable();
3988 34 : opParam.isCapture = isCapture;
3989 34 : opParam.opBaseAtraceInfo = opBaseAtraceInfo_.get();
3990 :
3991 34 : CHK_RET(ExecOp(HcclCMDType::HCCL_CMD_REDUCE_SCATTER, opParam));
3992 :
3993 11 : return HCCL_SUCCESS;
3994 35 : }
3995 :
3996 0 : HcclResult HcclCommunicator::ReduceScatterV(
3997 : const std::string& tag, void* inputPtr, const void* inputCounts, const void* inputDispls, void* outputPtr,
3998 : u64 outputCount, HcclDataType dataType, HcclReduceOp op, HcclRtStream stream, HcomCollOpInfo* opInfo)
3999 : {
4000 0 : CHK_RET(CheckSuspendingStatus());
4001 0 : if (userRankSize_ == 1) {
4002 : // rankSize为1时，退化为ReduceScatter
4003 0 : return ReduceScatter(tag, inputPtr, outputPtr, outputCount, dataType, op, stream);
4004 : }
4005 :
4006 0 : if (!IsAtomicInit()) {
4007 0 : HCCL_ERROR(
4008 : "[HcclCommunicator][ReduceScatterV]errNo[0x%016llx] hccl init must be called before call this function",
4009 : HCCL_ERROR_CODE(HCCL_E_UNAVAIL));
4010 0 : return HCCL_E_UNAVAIL;
4011 : }
4012 :
4013 0 : Stream streamObj(stream);
4014 0 : CHK_RET(callbackTask_->CallbackRegStream(stream));
4015 :
4016 0 : std::vector<u32>& ranksPorts = groupNicRanksPort_.empty() ? nicRanksPort_ : groupNicRanksPort_;
4017 0 : std::vector<u32>& vnicRanksPorts = groupVnicRanksPort_.empty() ? vnicRanksPort_ : groupVnicRanksPort_;
4018 0 : implAlg_->SetHDCModeInfo(rankDevicePhyIdNicInfoMap_, ranksPorts, vnicRanksPorts, isSetHDCModeInfo_, isUseRankPort_);
4019 :
4020 0 : const bool aicpuUnfoldMode = GetAicpuUnfoldConfig() && IsSupportSDMAReduce(inputPtr, outputPtr, dataType, op)
4021 0 : && (deviceType_ == DevType::DEV_TYPE_910_93);
4022 :
4023 0 : u32 perDataSize = SIZE_TABLE[dataType];
4024 0 : u64 inputSize = 0;
4025 0 : const u64* counts = static_cast<const u64*>(inputCounts);
4026 0 : for (u32 i = 0; i < userRankSize_; i++) {
4027 0 : inputSize += counts[i] * perDataSize;
4028 : }
4029 0 : CHK_PRT_RET(inputSize == 0, HCCL_WARNING("inputSize is 0, return ReduceScatterV success"), HCCL_SUCCESS);
4030 :
4031 0 : OpParam opParam;
4032 0 : opParam.tag = tag;
4033 0 : opParam.inputPtr = inputPtr;
4034 0 : opParam.inputSize = inputSize;
4035 0 : opParam.outputPtr = outputPtr;
4036 0 : opParam.outputSize = outputCount * perDataSize;
4037 0 : opParam.srcRank = userRank_; // rankId for access counts
4038 0 : opParam.VDataDes.counts = const_cast<void*>(inputCounts);
4039 0 : opParam.VDataDes.displs = const_cast<void*>(inputDispls);
4040 0 : opParam.VDataDes.dataType = dataType;
4041 0 : opParam.reduceType = op;
4042 0 : opParam.stream = streamObj;
4043 0 : opParam.opType = HcclCMDType::HCCL_CMD_REDUCE_SCATTER_V;
4044 0 : opParam.aicpuUnfoldMode = aicpuUnfoldMode;
4045 0 : opParam.aicpuCacheEnable = GetExternalInputAicpuCacheEnable();
4046 :
4047 0 : if (UNLIKELY(GetDebugConfig() & HCCL_ALG)) {
4048 0 : for (u32 i = 0; i < userRankSize_; i++) {
4049 0 : HCCL_CONFIG_DEBUG(
4050 : HCCL_ALG,
4051 : "[HcclCommunicator][ReduceScatterV]userRank_[%u], rankIdx[%u], inputCounts[%llu], inputDispls[%llu]",
4052 : userRank_, i, counts[i], static_cast<const u64*>(inputDispls)[i]);
4053 : }
4054 : }
4055 :
4056 0 : CHK_RET(ExecOp(HcclCMDType::HCCL_CMD_REDUCE_SCATTER_V, opParam));
4057 :
4058 0 : return HCCL_SUCCESS;
4059 0 : }
4060 :
4061 0 : HcclResult HcclCommunicator::ReduceScatterVOutPlace(
4062 : const std::string& tag, void* inputPtr, void* outputPtr, const void* inputCounts, const void* inputDispls,
4063 : u64 outputCount, HcclDataType dataType, HcclReduceOp op, HcclRtStream stream)
4064 : {
4065 0 : CHK_RET(CheckSuspendingStatus());
4066 0 : if (userRankSize_ == 1) {
4067 : // rankSize为1时，退化为ReduceScatter
4068 0 : return ReduceScatterOutPlace(tag, inputPtr, outputPtr, outputCount, dataType, op, stream);
4069 : }
4070 :
4071 0 : CHK_RET(CreateCommCCLbuffer());
4072 0 : if (!IsAtomicInit()) {
4073 0 : HCCL_ERROR(
4074 : "[HcclCommunicator][ReduceScatterVOutPlace]errNo[0x%016llx] hccl init must be called before"
4075 : " call this function",
4076 : HCCL_ERROR_CODE(HCCL_E_UNAVAIL));
4077 0 : return HCCL_E_UNAVAIL;
4078 : }
4079 :
4080 0 : bool isCapture = StreamIsCapture(stream);
4081 :
4082 0 : Stream streamObj(stream);
4083 0 : CHK_RET(callbackTask_->CallbackRegStream(stream));
4084 :
4085 0 : std::vector<u32>& ranksPorts = groupNicRanksPort_.empty() ? nicRanksPort_ : groupNicRanksPort_;
4086 0 : std::vector<u32>& vnicRanksPorts = groupVnicRanksPort_.empty() ? vnicRanksPort_ : groupVnicRanksPort_;
4087 0 : implAlg_->SetHDCModeInfo(rankDevicePhyIdNicInfoMap_, ranksPorts, vnicRanksPorts, isSetHDCModeInfo_, isUseRankPort_);
4088 :
4089 0 : const bool aicpuUnfoldMode = GetAicpuUnfoldConfig() && IsSupportSDMAReduce(inputPtr, outputPtr, dataType, op)
4090 0 : && (deviceType_ == DevType::DEV_TYPE_910_93);
4091 :
4092 0 : u32 perDataSize = SIZE_TABLE[dataType];
4093 0 : u64 inputSize = 0;
4094 0 : const u64* counts = static_cast<const u64*>(inputCounts);
4095 0 : for (u32 i = 0; i < userRankSize_; i++) {
4096 0 : inputSize += counts[i] * perDataSize;
4097 : }
4098 :
4099 0 : OpParam opParam;
4100 0 : opParam.tag = tag;
4101 0 : opParam.inputPtr = inputPtr;
4102 0 : opParam.inputSize = inputSize;
4103 0 : opParam.outputPtr = outputPtr;
4104 0 : opParam.outputSize = outputCount * perDataSize;
4105 0 : opParam.srcRank = userRank_; // rankId for access counts
4106 0 : opParam.VDataDes.counts = const_cast<void*>(inputCounts);
4107 0 : opParam.VDataDes.displs = const_cast<void*>(inputDispls);
4108 0 : opParam.VDataDes.dataType = dataType;
4109 0 : opParam.reduceType = op;
4110 0 : opParam.stream = streamObj;
4111 0 : opParam.opType = HcclCMDType::HCCL_CMD_REDUCE_SCATTER_V;
4112 0 : opParam.opBaseAtraceInfo = opBaseAtraceInfo_.get();
4113 0 : opParam.aicpuUnfoldMode = aicpuUnfoldMode;
4114 0 : opParam.aicpuCacheEnable = GetExternalInputAicpuCacheEnable();
4115 0 : opParam.isCapture = isCapture;
4116 0 : opParam.rankSize = userRankSize_;
4117 :
4118 0 : if (UNLIKELY(GetDebugConfig() & HCCL_ALG)) {
4119 0 : for (u32 i = 0; i < userRankSize_; i++) {
4120 0 : HCCL_CONFIG_DEBUG(
4121 : HCCL_ALG,
4122 : "[HcclCommunicator][ReduceScatterVOutPlace]userRank_[%u],"
4123 : "rankIdx[%u], inputCounts[%llu], inputDispls[%llu]",
4124 : userRank_, i, counts[i], static_cast<const u64*>(inputDispls)[i]);
4125 : }
4126 : }
4127 :
4128 0 : CHK_RET(ExecOp(HcclCMDType::HCCL_CMD_REDUCE_SCATTER_V, opParam));
4129 :
4130 0 : return HCCL_SUCCESS;
4131 0 : }
4132 :
4133 0 : HcclResult HcclCommunicator::BatchSendRecv(
4134 : const std::string& tag, HcclSendRecvItem* sendRecvItemsPtr, u32 itemNum, rtStream_t stream)
4135 : {
4136 0 : if (!IsAtomicInit()) {
4137 0 : HCCL_ERROR(
4138 : "[HcclCommunicator][BatchSendRecv]errNo[0x%016llx] hccl init must be called before call this function",
4139 : HCCL_ERROR_CODE(HCCL_E_UNAVAIL));
4140 0 : return HCCL_E_UNAVAIL;
4141 : }
4142 :
4143 0 : bool aicpuUnfoldMode = false;
4144 0 : if (GetAicpuUnfoldConfig() == true && (deviceType_ == DevType::DEV_TYPE_910_93) && (userRankSize_ != 1)) {
4145 0 : aicpuUnfoldMode = true;
4146 : }
4147 :
4148 0 : bool isCapture = StreamIsCapture(stream);
4149 :
4150 0 : if (!IsAtomicInit()) {
4151 0 : HCCL_ERROR(
4152 : "[HcclCommunicator][BatchSendRecv]errNo[0x%016llx] hccl init must be called before call this function",
4153 : HCCL_ERROR_CODE(HCCL_E_UNAVAIL));
4154 0 : return HCCL_E_UNAVAIL;
4155 : }
4156 0 : Stream streamObj(stream);
4157 0 : CHK_RET(callbackTask_->CallbackRegStream(stream));
4158 :
4159 0 : std::vector<u32>& ranksPorts = groupNicRanksPort_.empty() ? nicRanksPort_ : groupNicRanksPort_;
4160 0 : std::vector<u32>& vnicRanksPorts = groupVnicRanksPort_.empty() ? vnicRanksPort_ : groupVnicRanksPort_;
4161 0 : implAlg_->SetHDCModeInfo(rankDevicePhyIdNicInfoMap_, ranksPorts, vnicRanksPorts, isSetHDCModeInfo_, isUseRankPort_);
4162 0 : OpParam opParam;
4163 0 : opParam.tag = tag;
4164 0 : opParam.stream = streamObj;
4165 0 : opParam.aicpuUnfoldMode = aicpuUnfoldMode;
4166 0 : opParam.aicpuCacheEnable = GetExternalInputAicpuCacheEnable();
4167 0 : opParam.isCapture = isCapture;
4168 0 : opParam.BatchSendRecvDataDes.sendRecvItemsPtr = sendRecvItemsPtr;
4169 0 : opParam.BatchSendRecvDataDes.itemNum = itemNum;
4170 0 : opParam.opType = HcclCMDType::HCCL_CMD_BATCH_SEND_RECV;
4171 0 : opParam.isGroupMode = isGroupMode_;
4172 0 : if (isGroupMode_) {
4173 0 : opParam.aicpuUnfoldMode = true; // A2的GroupSendRecv也走aicpu模式
4174 : }
4175 :
4176 0 : CHK_RET(ExecOp(HcclCMDType::HCCL_CMD_BATCH_SEND_RECV, opParam));
4177 :
4178 0 : return HCCL_SUCCESS;
4179 0 : }
4180 :
4181 0 : HcclResult HcclCommunicator::Send(
4182 : const std::string& tag, void* inputPtr, u64 count, HcclDataType dataType, u32 destRank, rtStream_t stream,
4183 : u32 srTag, u32 localGroupRank)
4184 : {
4185 0 : CHK_RET(CheckSuspendingStatus());
4186 0 : bool aicpuUnfoldMode = false;
4187 0 : if (GetAicpuUnfoldConfig() == true && (deviceType_ == DevType::DEV_TYPE_910_93) && (userRankSize_ != 1)) {
4188 0 : aicpuUnfoldMode = true;
4189 : }
4190 :
4191 0 : if (!IsAtomicInit()) {
4192 0 : HCCL_ERROR(
4193 : "[HcclCommunicator][Send]errNo[0x%016llx] hccl init must be called before call this function",
4194 : HCCL_ERROR_CODE(HCCL_E_UNAVAIL));
4195 0 : return HCCL_E_UNAVAIL;
4196 : }
4197 :
4198 0 : Stream streamObj(stream);
4199 0 : CHK_RET(callbackTask_->CallbackRegStream(stream));
4200 :
4201 0 : u32 perDataSize = SIZE_TABLE[dataType];
4202 0 : u64 totalSize = count * perDataSize;
4203 :
4204 0 : OpParam opParam;
4205 0 : opParam.tag = tag;
4206 0 : opParam.inputPtr = inputPtr;
4207 0 : opParam.inputSize = totalSize;
4208 0 : opParam.outputPtr = inputPtr;
4209 0 : opParam.outputSize = totalSize;
4210 0 : opParam.DataDes.count = count;
4211 0 : opParam.DataDes.dataType = dataType;
4212 0 : opParam.stream = streamObj;
4213 0 : opParam.aicpuUnfoldMode = aicpuUnfoldMode;
4214 0 : opParam.aicpuCacheEnable = GetExternalInputAicpuCacheEnable();
4215 0 : opParam.opBaseAtraceInfo = opBaseAtraceInfo_.get();
4216 0 : opParam.dstRank = destRank;
4217 0 : opParam.opType = HcclCMDType::HCCL_CMD_SEND;
4218 0 : opParam.srTag = srTag;
4219 0 : opParam.localGroupRank = localGroupRank;
4220 0 : CHK_RET(ExecOp(HcclCMDType::HCCL_CMD_SEND, opParam));
4221 :
4222 0 : return HCCL_SUCCESS;
4223 0 : }
4224 :
4225 0 : HcclResult HcclCommunicator::SendOutPlace(
4226 : const std::string& tag, void* inputPtr, u64 count, HcclDataType dataType, u32 destRank, rtStream_t stream)
4227 : {
4228 0 : CHK_RET(CheckSuspendingStatus());
4229 0 : bool aicpuUnfoldMode = false;
4230 0 : if (GetAicpuUnfoldConfig() == true && (deviceType_ == DevType::DEV_TYPE_910_93) && (userRankSize_ != 1)) {
4231 0 : aicpuUnfoldMode = true;
4232 : }
4233 :
4234 0 : if (Is310P3Common(isHaveCpuRank_, deviceType_)) {
4235 0 : HCCL_ERROR(
4236 : "[%s][%s]SendOutPlace is not supported", LOG_KEYWORDS_TASK_EXEC.c_str(),
4237 : LOG_KEYWORDS_NOT_SUPPORTED.c_str());
4238 0 : return HCCL_E_NOT_SUPPORT;
4239 : }
4240 0 : if (!IsAtomicInit()) {
4241 0 : HCCL_ERROR(
4242 : "[HcclCommunicator][SendOutPlace]errNo[0x%016llx] hccl init must be called before call this function",
4243 : HCCL_ERROR_CODE(HCCL_E_UNAVAIL));
4244 0 : return HCCL_E_UNAVAIL;
4245 : }
4246 :
4247 0 : bool isCapture = StreamIsCapture(stream);
4248 :
4249 0 : Stream streamObj(stream);
4250 0 : CHK_RET(callbackTask_->CallbackRegStream(stream));
4251 :
4252 0 : std::vector<u32>& ranksPorts = groupNicRanksPort_.empty() ? nicRanksPort_ : groupNicRanksPort_;
4253 0 : std::vector<u32>& vnicRanksPorts = groupVnicRanksPort_.empty() ? vnicRanksPort_ : groupVnicRanksPort_;
4254 0 : implAlg_->SetHDCModeInfo(rankDevicePhyIdNicInfoMap_, ranksPorts, vnicRanksPorts, isSetHDCModeInfo_, isUseRankPort_);
4255 :
4256 0 : u32 perDataSize = SIZE_TABLE[dataType];
4257 0 : u64 totalSize = count * perDataSize;
4258 :
4259 0 : OpParam opParam;
4260 0 : opParam.tag = tag;
4261 0 : opParam.inputPtr = inputPtr;
4262 0 : opParam.inputSize = totalSize;
4263 0 : opParam.outputPtr = inputPtr;
4264 0 : opParam.outputSize = totalSize;
4265 0 : opParam.DataDes.count = count;
4266 0 : opParam.DataDes.dataType = dataType;
4267 0 : opParam.stream = streamObj;
4268 0 : opParam.aicpuUnfoldMode = aicpuUnfoldMode;
4269 0 : opParam.aicpuCacheEnable = GetExternalInputAicpuCacheEnable();
4270 0 : opParam.isCapture = isCapture;
4271 0 : opParam.opBaseAtraceInfo = opBaseAtraceInfo_.get();
4272 0 : opParam.dstRank = destRank;
4273 0 : opParam.opType = HcclCMDType::HCCL_CMD_SEND;
4274 0 : opParam.localGroupRank = userRank_;
4275 0 : CHK_RET(ExecOp(HcclCMDType::HCCL_CMD_SEND, opParam));
4276 :
4277 0 : return HCCL_SUCCESS;
4278 0 : }
4279 :
4280 0 : HcclResult HcclCommunicator::Receive(
4281 : const std::string& tag, void* outputPtr, u64 count, HcclDataType dataType, u32 srcRank, rtStream_t stream,
4282 : u32 srTag, u32 localGroupRank)
4283 : {
4284 0 : CHK_RET(CheckSuspendingStatus());
4285 0 : bool aicpuUnfoldMode = false;
4286 0 : if (GetAicpuUnfoldConfig() == true && (deviceType_ == DevType::DEV_TYPE_910_93) && (userRankSize_ != 1)) {
4287 0 : aicpuUnfoldMode = true;
4288 : }
4289 :
4290 0 : if (!IsAtomicInit()) {
4291 0 : HCCL_ERROR(
4292 : "[HcclCommunicator][Receive]errNo[0x%016llx] hccl init must be called before call this function",
4293 : HCCL_ERROR_CODE(HCCL_E_UNAVAIL));
4294 0 : return HCCL_E_UNAVAIL;
4295 : }
4296 :
4297 0 : Stream streamObj(stream);
4298 0 : CHK_RET(callbackTask_->CallbackRegStream(stream));
4299 :
4300 0 : u32 perDataSize = SIZE_TABLE[dataType];
4301 0 : u64 totalSize = count * perDataSize;
4302 :
4303 0 : OpParam opParam;
4304 0 : opParam.tag = tag;
4305 0 : opParam.inputPtr = outputPtr;
4306 0 : opParam.inputSize = totalSize;
4307 0 : opParam.outputPtr = outputPtr;
4308 0 : opParam.outputSize = totalSize;
4309 0 : opParam.DataDes.count = count;
4310 0 : opParam.DataDes.dataType = dataType;
4311 0 : opParam.stream = streamObj;
4312 0 : opParam.aicpuUnfoldMode = aicpuUnfoldMode;
4313 0 : opParam.aicpuCacheEnable = GetExternalInputAicpuCacheEnable();
4314 0 : opParam.opBaseAtraceInfo = opBaseAtraceInfo_.get();
4315 0 : opParam.srcRank = srcRank;
4316 0 : opParam.opType = HcclCMDType::HCCL_CMD_RECEIVE;
4317 0 : opParam.srTag = srTag;
4318 0 : opParam.localGroupRank = localGroupRank;
4319 0 : CHK_RET(ExecOp(HcclCMDType::HCCL_CMD_RECEIVE, opParam));
4320 :
4321 0 : return HCCL_SUCCESS;
4322 0 : }
4323 :
4324 0 : HcclResult HcclCommunicator::ReceiveOutPlace(
4325 : const std::string& tag, void* outputPtr, u64 count, HcclDataType dataType, u32 srcRank, rtStream_t stream)
4326 : {
4327 0 : CHK_RET(CheckSuspendingStatus());
4328 0 : bool aicpuUnfoldMode = false;
4329 0 : if (GetAicpuUnfoldConfig() == true && (deviceType_ == DevType::DEV_TYPE_910_93) && (userRankSize_ != 1)) {
4330 0 : aicpuUnfoldMode = true;
4331 : }
4332 :
4333 0 : if (Is310P3Common(isHaveCpuRank_, deviceType_)) {
4334 0 : HCCL_ERROR(
4335 : "[%s][%s]ReceiveOutPlace is not supported", LOG_KEYWORDS_TASK_EXEC.c_str(),
4336 : LOG_KEYWORDS_NOT_SUPPORTED.c_str());
4337 0 : return HCCL_E_NOT_SUPPORT;
4338 : }
4339 0 : if (!IsAtomicInit()) {
4340 0 : HCCL_ERROR(
4341 : "[HcclCommunicator][ReceiveOutPlace]errNo[0x%016llx] hccl init must be called before call this function",
4342 : HCCL_ERROR_CODE(HCCL_E_UNAVAIL));
4343 0 : return HCCL_E_UNAVAIL;
4344 : }
4345 :
4346 0 : bool isCapture = StreamIsCapture(stream);
4347 :
4348 0 : Stream streamObj(stream);
4349 0 : CHK_RET(callbackTask_->CallbackRegStream(stream));
4350 :
4351 0 : std::vector<u32>& ranksPorts = groupNicRanksPort_.empty() ? nicRanksPort_ : groupNicRanksPort_;
4352 0 : std::vector<u32>& vnicRanksPorts = groupVnicRanksPort_.empty() ? vnicRanksPort_ : groupVnicRanksPort_;
4353 0 : implAlg_->SetHDCModeInfo(rankDevicePhyIdNicInfoMap_, ranksPorts, vnicRanksPorts, isSetHDCModeInfo_, isUseRankPort_);
4354 :
4355 0 : u32 perDataSize = SIZE_TABLE[dataType];
4356 0 : u64 totalSize = count * perDataSize;
4357 :
4358 0 : OpParam opParam;
4359 0 : opParam.tag = tag;
4360 0 : opParam.inputPtr = outputPtr;
4361 0 : opParam.inputSize = totalSize;
4362 0 : opParam.outputPtr = outputPtr;
4363 0 : opParam.outputSize = totalSize;
4364 0 : opParam.DataDes.count = count;
4365 0 : opParam.DataDes.dataType = dataType;
4366 0 : opParam.stream = streamObj;
4367 0 : opParam.aicpuUnfoldMode = aicpuUnfoldMode;
4368 0 : opParam.aicpuCacheEnable = GetExternalInputAicpuCacheEnable();
4369 0 : opParam.isCapture = isCapture;
4370 0 : opParam.opBaseAtraceInfo = opBaseAtraceInfo_.get();
4371 0 : opParam.srcRank = srcRank;
4372 0 : opParam.opType = HcclCMDType::HCCL_CMD_RECEIVE;
4373 0 : opParam.localGroupRank = userRank_;
4374 0 : CHK_RET(ExecOp(HcclCMDType::HCCL_CMD_RECEIVE, opParam));
4375 :
4376 0 : return HCCL_SUCCESS;
4377 0 : }
4378 :
4379 0 : HcclResult HcclCommunicator::RegressCalPreOp(
4380 : AlltoAllOperator*& alltoAllOperator, const OpParam& opParam, std::unique_ptr<PreProcessMetaInfo>& preMetaInfo)
4381 : {
4382 0 : HCCL_INFO("Run with Graph, alloc new stream");
4383 0 : Stream stream(StreamType::STREAM_TYPE_ONLINE);
4384 0 : return RegressCalPreOp(alltoAllOperator, opParam, preMetaInfo, stream);
4385 0 : }
4386 :
4387 0 : HcclResult HcclCommunicator::RegressCalPreOp(
4388 : AlltoAllOperator*& alltoAllOperator, const OpParam& opParam, std::unique_ptr<PreProcessMetaInfo>& preMetaInfo,
4389 : Stream& preProcessStream)
4390 : {
4391 0 : OpParam preProcessOpParam;
4392 0 : HcclWorkflowMode mode = GetWorkflowMode();
4393 0 : CHK_PRT_RET(
4394 : mode == HcclWorkflowMode::HCCL_WORKFLOW_MODE_RESERVED, HCCL_ERROR("Invalid Workflow Mode[%d]", mode),
4395 : HCCL_E_INTERNAL);
4396 :
4397 : // h to d
4398 0 : CHK_RET(SetInfoToDevice(preMetaInfo, preProcessStream));
4399 : // opParam准备
4400 0 : CHK_RET(alltoAllOperator->PreparePreOpParam(preProcessOpParam, preMetaInfo, preProcessStream));
4401 :
4402 : // 回归调用其它算子
4403 0 : HCCL_INFO(
4404 : "[HcclCommunicator][RegressCalPreOp] Regression calls other operators and opType[%u]", preMetaInfo->opType);
4405 0 : CHK_RET(ExecOp(preMetaInfo->opType, preProcessOpParam));
4406 0 : CHK_RET(hcclStreamSynchronize(preProcessStream.ptr(), commConfig_.GetConfigExecTimeOut()));
4407 0 : HCCL_DEBUG("[HcclCommunicator][RegressCalPreOp] preProcess tag[%s].", preProcessOpParam.tag.c_str());
4408 0 : SetWorkflowMode(mode);
4409 :
4410 : // d to h
4411 0 : HostMem hostCollectBuffer = HostMem::alloc(preMetaInfo->outputSize);
4412 0 : CHK_PTR_NULL(hostCollectBuffer.ptr());
4413 0 : CHK_RET(GetInfoFromDevice(preMetaInfo, mode, hostCollectBuffer));
4414 :
4415 0 : hostCollectBuffer_ = hostCollectBuffer;
4416 0 : alltoAllOperator->SetPreProcessResult(std::move(hostCollectBuffer));
4417 0 : HCCL_INFO("[HcclCommunicator][RegressCalPreOp] run success!");
4418 0 : return HCCL_SUCCESS;
4419 0 : }
4420 :
4421 0 : HcclResult HcclCommunicator::SaveRankInfoHasLinked(const AlgResourceRequest& resRequest)
4422 : {
4423 0 : for (auto& levelNSubCommTransport : resRequest.opTransport) {
4424 0 : for (auto& singleSubCommTransport : levelNSubCommTransport) {
4425 0 : for (auto& transportRequest : singleSubCommTransport.transportRequests) {
4426 0 : if (transportRequest.isValid) {
4427 0 : ranksLinked_.insert(transportRequest.remoteUserRank);
4428 0 : HCCL_INFO(
4429 : "[HcclCommunicator][SaveRankInfoHasLinked]Insert remote Rank[%u] to ranksLinked Set.",
4430 : transportRequest.remoteUserRank);
4431 : }
4432 : }
4433 : }
4434 : }
4435 :
4436 0 : return HCCL_SUCCESS;
4437 : }
4438 :
4439 2 : HcclResult HcclCommunicator::GetCacheMap(
4440 : std::unique_ptr<CollAlgOperator>& algOperator, OpParam& opParam, AlgType& algType, bool selectAivAlg,
4441 : std::string& newTag)
4442 : {
4443 2 : HcclCacheInfo cacheInfo;
4444 2 : CHK_RET(algOperator->GetCache(cacheInfo));
4445 2 : if (cacheInfo.isUseCache == false) {
4446 2 : return HCCL_SUCCESS;
4447 : }
4448 0 : cacheInfo.algType = algType;
4449 0 : cacheInfo.selectAivAlg = selectAivAlg;
4450 0 : cacheInfo.newTag = newTag;
4451 :
4452 0 : if (hcclCacheMap_.size() > CACHEMAP_MAXSIZE) {
4453 0 : size_t clearCount = static_cast<size_t>(CACHEMAP_MAXSIZE * CACHEMAP_CLEARPERCENT);
4454 0 : for (auto it = hcclCacheMap_.begin(); clearCount > 0 && it != hcclCacheMap_.end(); clearCount--) {
4455 0 : it = hcclCacheMap_.erase(it);
4456 : }
4457 : }
4458 :
4459 0 : hcclCacheMap_.emplace(std::make_pair(opParam, std::move(cacheInfo)));
4460 :
4461 0 : HCCL_INFO(
4462 : "[HcclCommunicator][GetCacheMap] algType %s, selectAivAlg %d, newTag %s", AlgTypeToStr(algType).c_str(),
4463 : selectAivAlg, newTag.c_str());
4464 0 : return HCCL_SUCCESS;
4465 2 : }
4466 :
4467 0 : HcclResult HcclCommunicator::ExecOpCache(HcclCMDType opType, OpParam& opParam, HcclCacheInfo& cacheInfo)
4468 : {
4469 : // 可用核数也需要作为key的一部分，防止cache中拿出来的和计算出来的实际核数不一致
4470 : // cache目前仅支持executor的kernel为1的情况
4471 0 : cacheInfo.resourceArgs.buffersIn = cacheInfo.buffersIn;
4472 0 : cacheInfo.resourceArgs.buffersOut = cacheInfo.buffersOut;
4473 0 : cacheInfo.resourceArgs.stream = opParam.stream.ptr(); // 刷新cache下发的stream
4474 0 : cacheInfo.opArgs.input = opParam.inputPtr;
4475 0 : cacheInfo.opArgs.output = opParam.outputPtr;
4476 0 : AlgType& algType = cacheInfo.algType;
4477 0 : bool selectAivAlg = cacheInfo.selectAivAlg;
4478 0 : std::string newTag = cacheInfo.newTag;
4479 0 : HcclResult ret = HCCL_SUCCESS;
4480 : // 更新aivtag
4481 0 : GetAivTag(1, opParam.isCapture, cacheInfo.resourceArgs.aivTag);
4482 0 : HCCL_INFO(
4483 : "[HcclCommunicator][ExecOpCache]buffersIn[%p] buffersOut[%p] tag[%s] opType[%d] "
4484 : "deterministic [%u] count[%llu] op[%d] userRank[%u] aiv tag [%d] stream [%d]",
4485 : cacheInfo.buffersIn, cacheInfo.buffersOut, identifier_.c_str(), opType, opParam.deterministic,
4486 : cacheInfo.opArgs.count, cacheInfo.opArgs.op, userRank_, cacheInfo.resourceArgs.aivTag, opParam.stream.id());
4487 0 : CHK_RET(HandleAclGraphFirstOpAivBuff(opParam.stream.ptr()));
4488 : // 保留dfx
4489 0 : CHK_RET(RegisterDfxInfo(opParam, algType, resMap_[newTag].slaveStreams, selectAivAlg));
4490 : // 头计数
4491 0 : CHK_RET(StarsCounter(dispatcher_, opParam.stream, HEAD, opParam.aicpuUnfoldMode, retryEnable_, selectAivAlg));
4492 0 : u64 dataSize
4493 0 : = (opParam.opType == HcclCMDType::HCCL_CMD_ALLTOALL ?
4494 0 : opParam.All2AllDataDes.sendCount * SIZE_TABLE[opParam.All2AllDataDes.sendType] :
4495 : 0);
4496 0 : if (opType == HcclCMDType::HCCL_CMD_REDUCE_SCATTER_V || opType == HcclCMDType::HCCL_CMD_ALLGATHER_V
4497 0 : || (opType == HcclCMDType::HCCL_CMD_ALLTOALL && dataSize >= AIV_ALL_TO_ALL_BIG_SIZE)) {
4498 0 : ret = ExecuteKernelLaunch(
4499 0 : cacheInfo.opArgs, cacheInfo.topoArgs, cacheInfo.resourceArgs, cacheInfo.algArgs, cacheInfo.extraArgs,
4500 0 : cacheInfo.profilingInfo);
4501 : } else {
4502 0 : ret = ExecuteKernelLaunch(
4503 0 : cacheInfo.opArgs, cacheInfo.topoArgs, cacheInfo.resourceArgs, cacheInfo.algArgs, cacheInfo.profilingInfo);
4504 : }
4505 : // 刷新核数
4506 0 : numBlocks_ = cacheInfo.resourceArgs.numBlocks;
4507 0 : CHK_PRT_RET(ret != HCCL_SUCCESS, HCCL_ERROR("[ExecOpCache]launch aiv failed, return[%d]", ret), ret);
4508 0 : CHK_RET(StarsCounter(dispatcher_, opParam.stream, TAIL, opParam.aicpuUnfoldMode, retryEnable_, selectAivAlg));
4509 0 : CHK_RET(UnRegisterDfxInfo(opParam, resMap_[newTag].slaveStreams));
4510 0 : if (selectAivAlg) {
4511 0 : aivClearEnable_ = false;
4512 : }
4513 0 : return HCCL_SUCCESS;
4514 0 : }
4515 :
4516 1 : void HcclCommunicator::SplitBsrData(
4517 : OpParam& opParam, std::vector<u8>& isDirectRemoteRank, std::vector<HcclSendRecvItem>& hostSendRecvInfo,
4518 : std::vector<HcclSendRecvItem>& aicpuSendRecvInfo)
4519 : {
4520 1 : u32 itemNum = opParam.BatchSendRecvDataDes.itemNum;
4521 1 : isDirectRemoteRank.resize(userRankSize_);
4522 1 : HCCL_INFO("[HcclCommunicator][SplitBsrData] rankSize %u", userRankSize_);
4523 1 : HcclSendRecvItem* sendRecvInfo = opParam.BatchSendRecvDataDes.sendRecvItemsPtr;
4524 3 : for (u32 i = 0; i < itemNum; i++) {
4525 2 : if (sendRecvInfo->buf == nullptr) {
4526 2 : sendRecvInfo++;
4527 2 : continue;
4528 : }
4529 0 : if (remoteTransportMap_[sendRecvInfo->remoteRank] == TransportType::TRANS_TYPE_DEVICE_DIRECT) {
4530 : // host 侧需要下发的数据
4531 0 : HCCL_INFO(
4532 : "[HcclCommunicator][SplitBsrData]host localRank %u remoteRank %u type %d sendRecvType %d count %llu",
4533 : userRank_, sendRecvInfo->remoteRank, remoteTransportMap_[sendRecvInfo->remoteRank],
4534 : sendRecvInfo->sendRecvType, sendRecvInfo->count);
4535 0 : isDirectRemoteRank[sendRecvInfo->remoteRank] = true;
4536 0 : hostSendRecvInfo.push_back(*sendRecvInfo);
4537 : } else {
4538 : // aicpu侧需要下发的数据
4539 0 : HCCL_INFO(
4540 : "[HcclCommunicator][SplitBsrData]aicpu localRank %u remoteRank %u type %d sendRecvType %d count %llu",
4541 : userRank_, sendRecvInfo->remoteRank, remoteTransportMap_[sendRecvInfo->remoteRank],
4542 : sendRecvInfo->sendRecvType, sendRecvInfo->count);
4543 0 : isDirectRemoteRank[sendRecvInfo->remoteRank] = false;
4544 0 : aicpuSendRecvInfo.push_back(*sendRecvInfo);
4545 : }
4546 0 : sendRecvInfo++;
4547 : }
4548 1 : HCCL_INFO(
4549 : "[HcclCommunicator][SplitBsrData] itemNum %u hostItemNum %zu aicpuItemNum %zu", itemNum,
4550 : hostSendRecvInfo.size(), aicpuSendRecvInfo.size());
4551 1 : return;
4552 : }
4553 :
4554 0 : bool HcclCommunicator::IsReduceWithInt64OrProd(HcclCMDType opType, const OpParam& opParam) const
4555 : {
4556 0 : if (opType == HcclCMDType::HCCL_CMD_ALLREDUCE || opType == HcclCMDType::HCCL_CMD_REDUCE
4557 0 : || opType == HcclCMDType::HCCL_CMD_REDUCE_SCATTER) {
4558 0 : if (opParam.reduceType == HcclReduceOp::HCCL_REDUCE_PROD
4559 0 : || opParam.DataDes.dataType == HcclDataType::HCCL_DATA_TYPE_INT64) {
4560 0 : return true;
4561 : }
4562 : }
4563 :
4564 0 : if (opType == HcclCMDType::HCCL_CMD_REDUCE_SCATTER_V) {
4565 0 : if (opParam.reduceType == HcclReduceOp::HCCL_REDUCE_PROD
4566 0 : || opParam.VDataDes.dataType == HcclDataType::HCCL_DATA_TYPE_INT64) {
4567 0 : return true;
4568 : }
4569 : }
4570 0 : return false;
4571 : }
4572 :
4573 70 : HcclResult HcclCommunicator::ExecOp(HcclCMDType opType, OpParam& opParam, bool isCustom)
4574 : {
4575 70 : CHK_PRT_RET(
4576 : isInvalidComm_,
4577 : HCCL_ERROR(
4578 : "[HcclCommunicator][%s] comm[%s], rank[%u], devId[%d], snapshot recoverying, "
4579 : "this comm is invalid, no operator is allowed to execute.",
4580 : __func__, identifier_.c_str(), userRank_, deviceLogicId_),
4581 : HCCL_E_UNAVAIL);
4582 :
4583 70 : if (retryEnable_ && needWarnAboutReduceProdInt64_ && IsReduceWithInt64OrProd(opType, opParam)) {
4584 0 : HCCL_RUN_WARNING(
4585 : "[HcclCommunicator][%s]comm[%s], opType[%d], reduceType[%d]. Reduce operators with prod operation or int64 "
4586 : "data type. This operator type unsupportd for AICPU mode, retry disabled",
4587 : __func__, identifier_.c_str(), opType, opParam.reduceType);
4588 0 : needWarnAboutReduceProdInt64_ = false;
4589 : }
4590 70 : std::string tag = opParam.tag;
4591 73 : u32 aivCoreLimit = numBlocks_;
4592 : // 单机AIV场景下cache复用，提升下发性能
4593 73 : if (implAlg_->GetAivModeConfig() && GetWorkflowMode() == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE) {
4594 0 : if (aivCoreLimit == 0) {
4595 0 : aclError acl_ret = aclrtGetResInCurrentThread(ACL_RT_DEV_RES_VECTOR_CORE, &aivCoreLimit);
4596 0 : CHK_PRT_RET(
4597 : acl_ret != ACL_SUCCESS,
4598 : HCCL_ERROR("[HcclCommunicator][ExecOp] aclrtGetResInCurrentThread failed, ret=[%d]", acl_ret),
4599 : HCCL_E_PARA);
4600 : }
4601 0 : opParam.deterministic = implAlg_->GetDeterministicConfig();
4602 0 : opParam.aivCoreLimit = aivCoreLimit;
4603 0 : auto it = hcclCacheMap_.find(opParam);
4604 0 : if (it != hcclCacheMap_.end()) {
4605 0 : CHK_RET(ExecOpCache(opType, opParam, it->second));
4606 0 : return HCCL_SUCCESS;
4607 : }
4608 : }
4609 :
4610 70 : ForceProf(opParam.isCapture);
4611 69 : opParam.supportSymmetricMemory = IsSupportSymmetricMemory(opType, opParam);
4612 73 : opParam.supportZeroCopy = !opParam.supportSymmetricMemory && IsSupportZeroCopy(opParam);
4613 70 : opParam.aclGraphZeroCopyEnable = GetConfigAclGraphZeroCopyEnable();
4614 68 : bool isInGraphCaptureZeroCopy = false;
4615 68 : zeroCopyAclGraph_->SetRetryEnable(retryEnable_);
4616 68 : isInGraphCaptureZeroCopy = zeroCopyAclGraph_->SetAclGraphZeroCopyMode(
4617 : deviceType_, opType, opParam, implAlg_.get(), cclBufferManager_.GetOutCCLbufferSize());
4618 73 : if (isInGraphCaptureZeroCopy && userRankSize_ > 1) {
4619 0 : CHK_RET(CreateCommCCLbuffer());
4620 : }
4621 73 : if (isShareComm_) {
4622 0 : CHK_RET(ShareCCLbufferMgr::GetInstance().CheckCCLbuffConflict(cclBuffName_, opParam.stream.id()));
4623 : }
4624 73 : std::unique_ptr<CollAlgOperator> algOperator = implAlg_->GetAlgOperator(opType);
4625 55 : CHK_SMART_PTR_NULL(algOperator);
4626 : // 算法选择
4627 53 : std::string algName;
4628 53 : std::string newTag;
4629 53 : if (opParam.aicpuUnfoldMode) {
4630 : // 用于inplace支持重执行判断
4631 3 : CHK_RET(algOperator->SetRetryEnable(retryEnable_));
4632 : }
4633 54 : if (GetExternalInputHcclAivMode()) {
4634 : // 用于判断图模式是否清零
4635 0 : CHK_RET(algOperator->SetAivClearEnable(aivClearEnable_));
4636 : }
4637 :
4638 53 : std::unique_lock<std::mutex> lock(commResMutex_);
4639 69 : ResourceLimit limit;
4640 69 : limit.ifLimit = true;
4641 69 : limit.aivCoreLimit = aivCoreLimit;
4642 69 : AlgDesc algDesc;
4643 59 : algDesc.isLastSelect = true;
4644 59 : CHK_RET(algOperator->SelectAlg(opParam.tag, opParam, limit, algName, algDesc, newTag));
4645 67 : if (isOnlyAiv_ && !algDesc.isAivMode) {
4646 0 : std::string opTypeName = GetCMDTypeEnumStr(opType);
4647 0 : HCCL_ERROR(
4648 : "[HcclCommunicator][ExecOp] opType[%s] currently do not select aiv mode, aiv only not support.",
4649 : opTypeName.c_str());
4650 0 : return HCCL_E_NOT_SUPPORT;
4651 0 : }
4652 67 : CHK_RET(PrepareZeroCopy(algName, algDesc, opParam));
4653 :
4654 71 : if (opParam.isCapture) {
4655 : // aclgraph使用新的Tag，避免影响其他操作
4656 0 : newTag += "_Capture";
4657 : // aclgraph零拷贝场景下，每个算子都有单独的tag，需要记录，在graph销毁时清理相关资源
4658 0 : if (isInGraphCaptureZeroCopy) {
4659 0 : CHK_RET(AclgraphCallback::GetInstance().InsertNewTagToCaptureResMap(this, newTag, opParam));
4660 0 : tagsRequiringHostCleanup_.insert(newTag);
4661 : }
4662 : }
4663 :
4664 71 : if (GetWorkflowMode() == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE && userRankSize_ > 1) {
4665 41 : CHK_RET(CreateCommCCLbuffer());
4666 : }
4667 69 : if (hcclNslbDp::GetInstance().GetGlobalCommTaskId() != 0) {
4668 0 : NslbDp_CollectOperTable(opType, opParam, algOperator->GetAlgType(), algName);
4669 : }
4670 :
4671 : // 资源创建
4672 73 : if ((resMap_.find(newTag) != resMap_.end()) && opParam.isCapture) {
4673 0 : AlgResourceRequest resRequest;
4674 0 : CHK_RET(algOperator->CalcResRequest(algName, opParam, resRequest));
4675 0 : if (HasRoceTransportLinks(resRequest.opTransport)) {
4676 0 : auto resTmp = resMap_[newTag];
4677 0 : ++captureCnt_;
4678 0 : newTag += std::to_string(captureCnt_);
4679 0 : resMap_[newTag] = resTmp;
4680 0 : resRequest.isInGraphCaptureZeroCopy = isInGraphCaptureZeroCopy;
4681 0 : CHK_RET(CleanTransportLinks(resRequest.opTransport, resMap_[newTag].opTransportResponse));
4682 0 : if (IsEnableBackupLink()) {
4683 0 : CHK_RET(CleanTransportLinks(resRequest.opTransport, resMap_[newTag].opTransportResponseBackUp));
4684 : }
4685 : // 记录指令信息用于一致性校验
4686 0 : CHK_RET(RecordOpPara(opType, opParam));
4687 0 : CHK_RET(IncreAllocLink(newTag, opParam, resRequest, resMap_[newTag]));
4688 : // 移除tag对应的指令信息
4689 0 : CHK_RET(RankConsistentcyChecker::GetInstance().DelOpPara(opParam.tag));
4690 : // aclgraph零拷贝场景下，除第一个capture外，需要记录，在graph销毁时清理相关资源
4691 0 : CHK_RET(AclgraphCallback::GetInstance().InsertNewTagToCaptureResMap(this, newTag, opParam));
4692 0 : tagsRequiringHostCleanup_.insert(newTag);
4693 0 : }
4694 0 : }
4695 71 : InsertNewTagToTagMap(newTag, opParam.tag);
4696 70 : bool needIncreLink = false;
4697 : // aiv算法不需要申请host和device侧的从流
4698 70 : bool selectAivAlg = algDesc.isAivMode;
4699 70 : if (resMap_.find(newTag) == resMap_.end()) {
4700 67 : AlgResourceRequest resRequest;
4701 65 : CHK_RET(algOperator->CalcResRequest(algName, opParam, resRequest));
4702 69 : if (opType == HcclCMDType::HCCL_CMD_BATCH_SEND_RECV) {
4703 0 : CHK_RET(SaveRankInfoHasLinked(resRequest));
4704 : }
4705 69 : resRequest.isInGraphCaptureZeroCopy = isInGraphCaptureZeroCopy;
4706 69 : CHK_RET(RecordOpPara(opType, opParam));
4707 71 : HcclResult ret = AllocAlgResource(newTag, opType, opParam, resRequest, resMap_[newTag], selectAivAlg);
4708 73 : CHK_PRT_RET(
4709 : ret != HCCL_SUCCESS,
4710 : HCCL_ERROR("[HcclCommunicator][ExecOp] AllocAlgResource failed, algName=[%s]", algName.c_str()), ret);
4711 20 : CHK_RET(RankConsistentcyChecker::GetInstance().DelOpPara(opParam.tag));
4712 :
4713 : // 对于91093超节点内aiv跨机通信算子，将不同机的CCLbuffer地址存在约定好的aiv将读取的HBM位置
4714 20 : CHK_RET(algOperator->PrepareCommInfoToDevice(algName, resMap_[newTag]));
4715 :
4716 20 : if (!isHaveCpuRank_) {
4717 20 : if (isUseRankPort_) {
4718 20 : std::vector<u32>& nicPorts = groupNicRanksPort_.empty() ? nicRanksPort_ : groupNicRanksPort_;
4719 20 : std::vector<u32>& vnicPorts = groupVnicRanksPort_.empty() ? vnicRanksPort_ : groupVnicRanksPort_;
4720 20 : Heartbeat::GetInstance(deviceLogicId_)
4721 20 : .SetRankPortInfo(isUseRankPort_, nicPorts, vnicPorts, commPortConfig_.devPortSwitchOn);
4722 : }
4723 : // 开始注册心跳
4724 20 : if (opType == HcclCMDType::HCCL_CMD_SEND) {
4725 0 : CHK_RET(RegisterToHeartBeat(opParam.dstRank, tag));
4726 0 : hbSendRecvTags_.emplace(tag);
4727 20 : } else if (opType == HcclCMDType::HCCL_CMD_RECEIVE) {
4728 0 : CHK_RET(RegisterToHeartBeat(opParam.srcRank, tag));
4729 0 : hbSendRecvTags_.emplace(tag);
4730 : } else {
4731 20 : CHK_RET(RegisterToHeartBeat());
4732 : }
4733 : }
4734 19 : CHK_RET(UpdateZeroCopy(opParam, resMap_[newTag]));
4735 73 : } else if (opType == HcclCMDType::HCCL_CMD_BATCH_SEND_RECV) {
4736 : // batchsendrecv需要根据任务来确定和哪些卡建链，因此复用tag，并在此基础上实现增量建链
4737 0 : AlgResourceRequest resRequest;
4738 0 : CHK_RET(algOperator->CalcIncreLinkRequest(algName, opParam, ranksLinked_, resRequest, needIncreLink));
4739 0 : if (needIncreLink) {
4740 0 : CHK_RET(RecordOpPara(opType, opParam));
4741 0 : CHK_RET(IncreAllocLink(newTag, opParam, resRequest, resMap_[newTag]));
4742 0 : CHK_RET(RankConsistentcyChecker::GetInstance().DelOpPara(opParam.tag));
4743 0 : opParam.needIncreLink = true;
4744 : }
4745 0 : }
4746 :
4747 : // 算法执行
4748 19 : if (selectAivAlg) {
4749 0 : CHK_RET(HandleAclGraphFirstOpAivBuff(opParam.stream.ptr()));
4750 0 : if (aivClearEnable_) {
4751 : // 用于判断图模式是否清零
4752 0 : CHK_RET(algOperator->SetAivClearEnable(aivClearEnable_));
4753 0 : aivOffloadTag_ = 1;
4754 : }
4755 0 : GetAivTag(algDesc.aivTagNum, opParam.isCapture, opParam.aivTag);
4756 0 : HCCL_INFO(
4757 : "[HcclCommunicator][ExecOp] tag[%s] userRank[%u] cur aiv tag [%d]", identifier_.c_str(), userRank_,
4758 : opParam.aivTag);
4759 0 : opParam.aicpuUnfoldMode = false;
4760 0 : opParam.aicpuCacheEnable = 0;
4761 0 : CHK_RET(algOperator->SetNumBlocks(aivCoreLimit));
4762 : }
4763 19 : std::vector<HcclSendRecvItem> hostSendRecvInfo;
4764 19 : std::vector<HcclSendRecvItem> aicpuSendRecvInfo;
4765 19 : std::vector<u8> isDirectRemoteRank;
4766 19 : if (opType == HcclCMDType::HCCL_CMD_BATCH_SEND_RECV && deviceType_ == DevType::DEV_TYPE_910_93) {
4767 0 : SplitBsrData(opParam, isDirectRemoteRank, hostSendRecvInfo, aicpuSendRecvInfo);
4768 : // A3 bsr记录Direct下发方式数据
4769 0 : opParam.BatchSendRecvDataDes.isDirectRemoteRank = isDirectRemoteRank.data();
4770 0 : if (!retryEnable_) {
4771 0 : opParam.BatchSendRecvDataDes.sendRecvItemsPtr = aicpuSendRecvInfo.data();
4772 0 : opParam.BatchSendRecvDataDes.itemNum = aicpuSendRecvInfo.size();
4773 : }
4774 : }
4775 : // A2 Group SendRecv 将isDirectRemoteRank全部置为false
4776 19 : if (opType == HcclCMDType::HCCL_CMD_BATCH_SEND_RECV && deviceType_ == DevType::DEV_TYPE_910B && isGroupMode_) {
4777 0 : isDirectRemoteRank.resize(userRankSize_, 0);
4778 0 : opParam.BatchSendRecvDataDes.isDirectRemoteRank = isDirectRemoteRank.data();
4779 : }
4780 19 : auto algType = algOperator->GetAlgType();
4781 19 : CHK_RET(RegisterDfxInfo(opParam, algType, resMap_[newTag].slaveStreams, selectAivAlg, tag));
4782 : // 头计数
4783 19 : CHK_RET(StarsCounter(dispatcher_, opParam.stream, HEAD, opParam.aicpuUnfoldMode, retryEnable_, selectAivAlg));
4784 19 : if (opParam.aicpuUnfoldMode) {
4785 0 : isInplaceStatus_ = 0;
4786 0 : inPlaceSupportRetryStatus_ = InplaceSupportRetryStatus::INPLACE_STATUS_END;
4787 : // algOperator->SupportRetryWithInplaceCheck 依赖 algOperator->SetRetryEnable 才能正确返回是否支持inplace
4788 :
4789 0 : inplaceSupportRetry_ = algOperator->SupportRetryWithInplaceCheck(
4790 0 : opType, opParam, algName, isInplaceStatus_, inPlaceSupportRetryStatus_);
4791 0 : HCCL_INFO(
4792 : "[HcclCommunicator][ExecOp] aicpu Unfold mode algType[%s], inplaceSupportRetry_[%d], opType[%d], "
4793 : "isInplaceStatus_[%d], inPlaceSupportRetryStatus_[%d].",
4794 : AlgTypeToStr(algType).c_str(), inplaceSupportRetry_, opType, isInplaceStatus_, inPlaceSupportRetryStatus_);
4795 0 : CHK_RET(OrchestrateAicpu(opType, algName, opParam, resMap_[newTag], newTag, algType, isCustom, needIncreLink));
4796 : } else {
4797 : // HOST展开aclgraph场景，capture从流
4798 19 : if (!selectAivAlg) {
4799 22 : CHK_RET(CaptureSlaveStreams(opParam.stream.ptr(), resMap_[newTag].slaveStreams));
4800 : }
4801 19 : OpCounterInfo opCounter;
4802 19 : CHK_RET(GetOpCountInfo(opCounter));
4803 19 : CHK_RET(algOperator->SetOpCounter(opCounter));
4804 19 : CHK_RET(algOperator->Orchestrate(algName, opParam, resMap_[newTag]));
4805 16 : if (hostResMap_.find(newTag) == hostResMap_.end()) {
4806 16 : hostResMap_.insert(newTag);
4807 : }
4808 16 : CHK_RET(algOperator->GetNumBlocks(numBlocks_));
4809 16 : if (implAlg_->GetAivModeConfig() && GetWorkflowMode() == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE
4810 16 : && !opParam.isCapture) {
4811 0 : CHK_RET(GetCacheMap(algOperator, opParam, algType, selectAivAlg, newTag));
4812 : }
4813 : }
4814 : // A3 bsr 只有走NPU直驱的时候hostSendRecvInfo才有内容
4815 16 : if (!hostSendRecvInfo.empty()) {
4816 : // A3 bsr获取到host侧需要下发的数据
4817 0 : HCCL_INFO("[HcclCommunicator][ExecOp] hostSendRecvInfo size %zu", hostSendRecvInfo.size());
4818 0 : opParam.BatchSendRecvDataDes.sendRecvItemsPtr = hostSendRecvInfo.data();
4819 0 : opParam.BatchSendRecvDataDes.itemNum = hostSendRecvInfo.size();
4820 0 : opParam.aicpuUnfoldMode = false;
4821 0 : opParam.aicpuCacheEnable = 0;
4822 0 : std::string tempTag;
4823 0 : std::unique_ptr<CollAlgOperator> newalgOperator = implAlg_->GetAlgOperator(opType);
4824 0 : CHK_SMART_PTR_NULL(newalgOperator);
4825 0 : CHK_RET(newalgOperator->SelectAlg(opParam.tag, opParam, limit, algName, algDesc, tempTag));
4826 0 : CHK_RET(newalgOperator->Orchestrate(algName, opParam, resMap_[newTag]));
4827 0 : }
4828 16 : lock.unlock();
4829 : // 尾计数
4830 16 : CHK_RET(StarsCounter(dispatcher_, opParam.stream, TAIL, opParam.aicpuUnfoldMode, retryEnable_, selectAivAlg));
4831 16 : CHK_RET(UnRegisterDfxInfo(opParam, resMap_[newTag].slaveStreams));
4832 16 : if (selectAivAlg) {
4833 0 : CHK_RET(algOperator->SetAivClearEnable(false));
4834 0 : aivClearEnable_ = false;
4835 : }
4836 16 : if (hcclNslbDp::GetInstance().GetGlobalCommTaskId() != 0 && hcclNslbDp::GetInstance().GetInitNetCoFlag() == true) {
4837 0 : AdjInfo nslbAdjInfo = {};
4838 0 : CHK_RET(algOperator->GetAdjInfo(algName, opParam, resMap_[newTag], nslbAdjInfo));
4839 0 : NslbDp_CollectSendAdjTable(opType, opParam, algOperator->GetAlgType(), nslbAdjInfo);
4840 0 : }
4841 16 : if (isInGraphCaptureZeroCopy) {
4842 0 : SetWorkflowMode(HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE);
4843 : }
4844 16 : return HCCL_SUCCESS;
4845 73 : }
4846 :
4847 : HcclResult
4848 0 : HcclCommunicator::FreeScratchMemOnOpBaseMode(DeviceMem& scratchMem, const OpParam& opParam, const HcclCMDType& opType)
4849 : {
4850 : // 当前单算子模式下scratch内存为手动申请，需要手动进行释放
4851 0 : if (GetWorkflowMode() == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE || IsForceAicpuOpBaseMode(opParam, opType)) {
4852 0 : scratchMem.free();
4853 : }
4854 0 : return HCCL_SUCCESS;
4855 : }
4856 :
4857 0 : HcclResult HcclCommunicator::ReAllocScratchMemForAlltoall(
4858 : HcclCMDType opType, const OpParam& opParam, AlgResourceRequest& resRequest, AlgResourceResponse& algResResponse)
4859 : {
4860 0 : if (GetWorkflowMode() == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OPS_KERNEL_INFO_LIB
4861 0 : && !IsForceAicpuOpBaseMode(opParam, opType)) {
4862 0 : if (resRequest.scratchMemSize > 0) {
4863 0 : algResResponse.scratchMem = GetWorkspaceScracthMem(opParam.tag, resRequest.scratchMemSize);
4864 : }
4865 0 : HCCL_DEBUG("[%s] WorkflowMode set for workspace opType[%u] tag[%s]", __func__, opType, opParam.tag.c_str());
4866 0 : } else if (
4867 0 : GetWorkflowMode() == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE || IsForceAicpuOpBaseMode(opParam, opType)) {
4868 0 : CHK_RET(AllocOpBaseModeScratchMem(opType, opParam, resRequest, algResResponse));
4869 0 : HCCL_DEBUG("[%s] WorkflowMode set for opType[%u] tag[%s]", __func__, opType, opParam.tag.c_str());
4870 : } else {
4871 0 : HCCL_ERROR("[%s] WorkflowMode is not set for opType[%u] tag[%s]", __func__, opType, opParam.tag.c_str());
4872 0 : return HCCL_E_PARA;
4873 : }
4874 0 : return HCCL_SUCCESS;
4875 : }
4876 :
4877 1 : HcclResult HcclCommunicator::HandleExistAlgResource(
4878 : const std::string& newTag, const std::string& algName, HcclCMDType opType, const OpParam& opParam,
4879 : std::unique_ptr<CollAlgOperator>& algOperator, bool selectAivAlg, bool aicpuUnfoldModeFor910B,
4880 : bool needRecreateAlltoallComm)
4881 : {
4882 1 : if (needRecreateAlltoallComm) {
4883 0 : CHK_RET(hcclStreamSynchronize(opParam.stream.ptr(), commConfig_.GetConfigExecTimeOut()));
4884 :
4885 0 : AlgResourceRequest resRequest;
4886 0 : CHK_RET(algOperator->CalcResRequest(algName, opParam, resRequest));
4887 :
4888 : // 释放旧内存防止泄漏
4889 0 : CHK_RET(FreeScratchMemOnOpBaseMode(resMap_[newTag].scratchMem, opParam, opType));
4890 :
4891 0 : if (aicpuUnfoldModeFor910B) {
4892 0 : CHK_RET(ReAllocScratchMemForAlltoall(opType, opParam, resRequest, resMap_[newTag]));
4893 0 : isContextLaunched_ = true;
4894 : } else {
4895 0 : CHK_RET(RecordOpPara(opType, opParam));
4896 0 : CHK_RET(AllocAlgResource(newTag, opType, opParam, resRequest, resMap_[newTag], selectAivAlg));
4897 0 : CHK_RET(RankConsistentcyChecker::GetInstance().DelOpPara(opParam.tag));
4898 :
4899 0 : if (!isHaveCpuRank_) {
4900 0 : if (isUseRankPort_) {
4901 0 : std::vector<u32>& nicPorts = groupNicRanksPort_.empty() ? nicRanksPort_ : groupNicRanksPort_;
4902 0 : std::vector<u32>& vnicPorts = groupVnicRanksPort_.empty() ? vnicRanksPort_ : groupVnicRanksPort_;
4903 0 : Heartbeat::GetInstance(deviceLogicId_)
4904 0 : .SetRankPortInfo(isUseRankPort_, nicPorts, vnicPorts, commPortConfig_.devPortSwitchOn);
4905 : }
4906 0 : CHK_RET(RegisterToHeartBeat());
4907 : }
4908 : }
4909 0 : } else {
4910 1 : DeviceMem tinySendRecvMem;
4911 1 : CHK_RET(implAlg_->GetTinyMem(tinySendRecvMem));
4912 1 : CHK_RET(CalcTinySendRecvMem(opParam, resMap_[newTag], tinySendRecvMem));
4913 1 : }
4914 1 : return HCCL_SUCCESS;
4915 : }
4916 :
4917 5 : HcclResult HcclCommunicator::ExecOpAlltoAll(HcclCMDType opType, OpParam& opParam, bool isCustom)
4918 : {
4919 5 : CHK_PRT_RET(
4920 : isInvalidComm_,
4921 : HCCL_ERROR(
4922 : "[HcclCommunicator][%s] comm[%s], rank[%u], devId[%d], snapshot recoverying, "
4923 : "this comm is invalid, no operator is allowed to execute.",
4924 : __func__, identifier_.c_str(), userRank_, deviceLogicId_),
4925 : HCCL_E_UNAVAIL);
4926 :
4927 5 : std::string& tag = opParam.tag;
4928 5 : u32 aivCoreLimit = numBlocks_;
4929 : // 单机AIV场景下cache复用，提升下发性能
4930 5 : if (implAlg_->GetAivModeConfig() && GetWorkflowMode() == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE) {
4931 5 : if (aivCoreLimit == 0) {
4932 4 : aclError acl_ret = aclrtGetResInCurrentThread(ACL_RT_DEV_RES_VECTOR_CORE, &aivCoreLimit);
4933 4 : CHK_PRT_RET(
4934 : acl_ret != ACL_SUCCESS,
4935 : HCCL_ERROR("[HcclCommunicator][ExecOpAlltoAll] aclrtGetResInCurrentThread failed, ret=[%d]", acl_ret),
4936 : HCCL_E_PARA);
4937 : }
4938 5 : opParam.deterministic = implAlg_->GetDeterministicConfig();
4939 5 : opParam.aivCoreLimit = aivCoreLimit;
4940 5 : auto it = hcclCacheMap_.find(opParam);
4941 5 : if (it != hcclCacheMap_.end()) {
4942 0 : CHK_RET(ExecOpCache(opType, opParam, it->second));
4943 0 : return HCCL_SUCCESS;
4944 : }
4945 : }
4946 :
4947 5 : ForceProf(opParam.isCapture);
4948 5 : bool isInGraphCaptureZeroCopy = false;
4949 5 : zeroCopyAclGraph_->SetRetryEnable(retryEnable_);
4950 5 : opParam.supportSymmetricMemory = IsSupportSymmetricMemory(opType, opParam);
4951 5 : opParam.supportZeroCopy = !opParam.supportSymmetricMemory && IsSupportZeroCopy(opParam);
4952 5 : opParam.aclGraphZeroCopyEnable = GetConfigAclGraphZeroCopyEnable();
4953 5 : isInGraphCaptureZeroCopy = zeroCopyAclGraph_->SetAclGraphZeroCopyMode(
4954 : deviceType_, opType, opParam, implAlg_.get(), cclBufferManager_.GetOutCCLbufferSize());
4955 5 : if (isInGraphCaptureZeroCopy && userRankSize_ > 1) {
4956 0 : CHK_RET(CreateCommCCLbuffer());
4957 : }
4958 5 : if (isShareComm_) {
4959 0 : CHK_RET(ShareCCLbufferMgr::GetInstance().CheckCCLbuffConflict(cclBuffName_, opParam.stream.id()));
4960 : }
4961 5 : std::unique_ptr<CollAlgOperator> algOperator = implAlg_->GetAlgOperator(opType);
4962 5 : AlltoAllOperator* alltoAllOperator = dynamic_cast<AlltoAllOperator*>(algOperator.get());
4963 5 : CHK_PTR_NULL(alltoAllOperator);
4964 :
4965 5 : bool isSatisfyA2ACPForA3Condition = alltoAllOperator->IsSatisfyA2AContinuousPipelineFor91093Condition(opParam);
4966 5 : bool IsSatisfyA2ACPForA2Condition = alltoAllOperator->IsSatisfyAlltoallContinuousPipelineCondition(opParam);
4967 5 : if (IsSatisfyA2ACPForA2Condition || isSatisfyA2ACPForA3Condition) {
4968 0 : opParam.aicpuUnfoldMode = true;
4969 0 : opParam.aicpuCacheEnable = GetExternalInputAicpuCacheEnable();
4970 : }
4971 :
4972 : // 算法选择
4973 5 : std::string algName;
4974 5 : std::string newTag;
4975 5 : if (opParam.aicpuUnfoldMode) {
4976 : // 用于inplace支持重执行判断
4977 0 : CHK_RET(algOperator->SetRetryEnable(retryEnable_));
4978 : }
4979 5 : std::unique_ptr<PreProcessMetaInfo> preMetaInfo = std::make_unique<PreProcessMetaInfo>();
4980 5 : CHK_SMART_PTR_NULL(preMetaInfo);
4981 :
4982 5 : bool preProcessFlag = alltoAllOperator->JudgeIfNeedPreProcessAndGetParam(opParam, preMetaInfo);
4983 5 : if (preProcessFlag) {
4984 0 : if (GetWorkflowMode() == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE) {
4985 0 : CHK_RET(RegressCalPreOp(alltoAllOperator, opParam, preMetaInfo, const_cast<Stream&>(opParam.stream)));
4986 : } else {
4987 0 : CHK_RET(RegressCalPreOp(alltoAllOperator, opParam, preMetaInfo));
4988 : }
4989 : }
4990 :
4991 5 : if (deviceType_ == DevType::DEV_TYPE_910B && userRankSize_ > 1) {
4992 : // 用于AIV支持Roce直驱判断
4993 5 : CHK_RET(IsSupportAIVNormalQP(devicePhyId_, opParam.supportRoceDirect));
4994 : }
4995 :
4996 5 : std::unique_lock<std::mutex> lock(commResMutex_);
4997 5 : ResourceLimit limit;
4998 5 : limit.ifLimit = true;
4999 5 : limit.aivCoreLimit = aivCoreLimit;
5000 5 : AlgDesc algDesc;
5001 5 : algDesc.isLastSelect = true;
5002 5 : CHK_RET(algOperator->SelectAlg(opParam.tag, opParam, limit, algName, algDesc, newTag));
5003 : // 是否是AIV直驱Roce场景
5004 5 : opParam.isNpuDirectRoce = algName == "AlltoAllDirectFullmeshAIVExecutor";
5005 5 : if (isOnlyAiv_ && !algDesc.isAivMode) {
5006 0 : std::string opTypeName = GetCMDTypeEnumStr(opType);
5007 0 : HCCL_ERROR(
5008 : "[HcclCommunicator][ExecOp] opType[%s] currently do not select aiv mode, aiv only not support.",
5009 : opTypeName.c_str());
5010 0 : return HCCL_E_NOT_SUPPORT;
5011 0 : }
5012 5 : CHK_RET(PrepareZeroCopy(algName, algDesc, opParam));
5013 :
5014 5 : if (opParam.isCapture) {
5015 : // aclgraph使用新的Tag，避免影响其他操作
5016 3 : newTag += "_Capture";
5017 : // aclgraph零拷贝场景下，每个算子都有单独的tag，需要记录，在graph销毁时清理相关资源
5018 3 : if (isInGraphCaptureZeroCopy) {
5019 0 : CHK_RET(AclgraphCallback::GetInstance().InsertNewTagToCaptureResMap(this, newTag, opParam));
5020 0 : tagsRequiringHostCleanup_.insert(newTag);
5021 : }
5022 : }
5023 :
5024 0 : auto isSupportAlg = [](const std::string& algName, bool aicpuUnfoldMode) -> bool {
5025 0 : return ((algName == "RunAlltoAllVFullMesh" || algName == "RunAlltoAllVTwoLevelPipeline") && aicpuUnfoldMode)
5026 0 : || (algName == "RunAlltoAllDirectFullmesh" || algName == "RunAlltoAllFullMeshSymmetricMemory");
5027 : };
5028 5 : bool isOpbaseMode = GetWorkflowMode() == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE;
5029 5 : if ((isOpbaseMode && userRankSize_ > 1) || (isSupportAlg(algName, opParam.aicpuUnfoldMode))) {
5030 5 : CHK_RET(CreateCommCCLbuffer());
5031 : }
5032 : // 资源创建
5033 5 : bool selectAivAlg = algDesc.isAivMode;
5034 5 : if ((resMap_.find(newTag) != resMap_.end()) && opParam.isCapture) {
5035 1 : AlgResourceRequest resRequest;
5036 1 : CHK_RET(algOperator->CalcResRequest(algName, opParam, resRequest));
5037 1 : if (HasRoceTransportLinks(resRequest.opTransport)) {
5038 0 : auto resTmp = resMap_[newTag];
5039 0 : ++captureCnt_;
5040 0 : newTag += std::to_string(captureCnt_);
5041 0 : resMap_[newTag] = resTmp;
5042 0 : resRequest.isInGraphCaptureZeroCopy = isInGraphCaptureZeroCopy;
5043 0 : CHK_RET(CleanTransportLinks(resRequest.opTransport, resMap_[newTag].opTransportResponse));
5044 0 : if (IsEnableBackupLink()) {
5045 0 : CHK_RET(CleanTransportLinks(resRequest.opTransport, resMap_[newTag].opTransportResponseBackUp));
5046 : }
5047 : // 记录指令信息用于一致性校验
5048 0 : CHK_RET(RecordOpPara(opType, opParam));
5049 0 : CHK_RET(IncreAllocLink(newTag, opParam, resRequest, resMap_[newTag]));
5050 : // 移除tag对应的指令信息
5051 0 : CHK_RET(RankConsistentcyChecker::GetInstance().DelOpPara(opParam.tag));
5052 : // aclgraph零拷贝场景下，除第一个capture外，需要记录，在graph销毁时清理相关资源
5053 0 : CHK_RET(AclgraphCallback::GetInstance().InsertNewTagToCaptureResMap(this, newTag, opParam));
5054 0 : tagsRequiringHostCleanup_.insert(newTag);
5055 0 : }
5056 1 : }
5057 5 : InsertNewTagToTagMap(newTag, opParam.tag);
5058 5 : bool aicpuUnfoldModeFor910B = deviceType_ == DevType::DEV_TYPE_910B && opParam.aicpuUnfoldMode
5059 10 : && (algName == "RunAlltoAllVStaged" || algName == "RunAlltoAllVFullMesh");
5060 5 : bool needRecreateAlltoallComm = false;
5061 5 : if (resMap_.find(newTag) == resMap_.end()) {
5062 4 : AlgResourceRequest resRequest;
5063 4 : CHK_RET(algOperator->CalcResRequest(algName, opParam, resRequest));
5064 4 : resRequest.isInGraphCaptureZeroCopy = isInGraphCaptureZeroCopy;
5065 4 : CHK_RET(RecordOpPara(opType, opParam));
5066 4 : CHK_RET(AllocAlgResource(newTag, opType, opParam, resRequest, resMap_[newTag], selectAivAlg));
5067 4 : CHK_RET(RankConsistentcyChecker::GetInstance().DelOpPara(opParam.tag));
5068 4 : if (opParam.isNpuDirectRoce) {
5069 : // AIV直驱roce多机场景，需要生成RMAInfo并拷贝至Device
5070 0 : CHK_RET(GenAiRMAInfoV2(newTag));
5071 0 : CHK_RET(H2DAiRMAInfoV2(newTag, opParam.stream.ptr()));
5072 : }
5073 : // 对于91093超节点内aiv跨机通信算子，将不同机的CCLbuffer地址存在约定好的aiv将读取的HBM位置
5074 4 : CHK_RET(algOperator->PrepareCommInfoToDevice(algName, resMap_[newTag]));
5075 :
5076 4 : if (!isHaveCpuRank_) {
5077 4 : if (isUseRankPort_) {
5078 4 : std::vector<u32>& nicPorts = groupNicRanksPort_.empty() ? nicRanksPort_ : groupNicRanksPort_;
5079 4 : std::vector<u32>& vnicPorts = groupVnicRanksPort_.empty() ? vnicRanksPort_ : groupVnicRanksPort_;
5080 4 : Heartbeat::GetInstance(deviceLogicId_)
5081 4 : .SetRankPortInfo(isUseRankPort_, nicPorts, vnicPorts, commPortConfig_.devPortSwitchOn);
5082 : }
5083 4 : CHK_RET(RegisterToHeartBeat());
5084 : }
5085 4 : CHK_RET(UpdateZeroCopy(opParam, resMap_[newTag]));
5086 4 : } else {
5087 1 : CHK_RET(alltoAllOperator->CheckNeedRecreateComm(
5088 : algName, opParam, resMap_[newTag].scratchMem.size(), needRecreateAlltoallComm));
5089 1 : HCCL_INFO(
5090 : "resMap_ find this newTag[%s], and need to judge whether recreate comm [%d]", newTag.c_str(),
5091 : needRecreateAlltoallComm);
5092 1 : CHK_RET(HandleExistAlgResource(
5093 : newTag, algName, opType, opParam, algOperator, selectAivAlg, aicpuUnfoldModeFor910B,
5094 : needRecreateAlltoallComm));
5095 : }
5096 5 : auto& algRes = resMap_[newTag];
5097 :
5098 5 : if (hcclNslbDp::GetInstance().GetGlobalCommTaskId() != 0 && hcclNslbDp::GetInstance().GetInitNetCoFlag() == true) {
5099 : /* NSLB 填充 表 */
5100 0 : u32 srcLocalRankId = userRank_;
5101 0 : u32 rootRank = (opParam.root == INVALID_VALUE_RANKID) ? 0 : opParam.root;
5102 0 : AlgType nslbAlgType = algOperator->GetAlgType();
5103 0 : AlgTypeLevel1 algValue = nslbAlgType.algoLevel1;
5104 0 : uint8_t nslbAlg = hcclNslbDp::GetInstance().GetNslbLevel1AlgType(algValue);
5105 :
5106 0 : if (algName == "RunAlltoAllVFullMesh" || algName == "RunAlltoAllDirectFullmesh") {
5107 0 : nslbAlg = NSLBDP_PAIRWISE;
5108 0 : if (deviceType_ == DevType::DEV_TYPE_910_93) {
5109 0 : nslbAlg = NSLB_ALGO_TYPE_FULLMESH;
5110 : }
5111 : }
5112 :
5113 0 : std::string nslb_identifier = identifier_;
5114 0 : HCCL_INFO("NSLBDP-SWK NslbDp_CollectOperTable nslb_identifier[%s] .", nslb_identifier.c_str());
5115 0 : u32 rankSize = userRankSize_;
5116 0 : u64 count = opParam.All2AllDataDes.sendCount * SIZE_TABLE[opParam.All2AllDataDes.sendType];
5117 : // 填充表2
5118 0 : hcclNslbDp::GetInstance().GenerateOpAndAdjTable(
5119 : opType, rootRank, srcLocalRankId, nslbAlg, nslb_identifier, count, rankSize);
5120 0 : AdjInfo nslbAdjInfo = {};
5121 0 : CHK_RET(algOperator->GetAdjInfo(algName, opParam, algRes, nslbAdjInfo));
5122 0 : HCCL_INFO(
5123 : "[NSLBDP-WEN]-nslbAdjInfosize[%u]-algName[%s]-rankSize[%u]-commDesc[%s]..", nslbAdjInfo.dstRankNum,
5124 : algName.c_str(), userRankSize_, identifier_.c_str());
5125 : // 填充表3
5126 0 : hcclNslbDp::GetInstance().GetAlgAdjacencyTable(
5127 : opType, srcLocalRankId, rootRank, nslbAlg, nslb_identifier, nslbAdjInfo);
5128 : /*发送流程*/
5129 0 : hcclNslbDp::GetInstance().SendAlgorithmInfoTable();
5130 0 : }
5131 : // 算法执行
5132 5 : if (opParam.isNpuDirectRoce) {
5133 : // AIV直驱roce多机场景，需要生成RMAInfo并拷贝至Device
5134 0 : CHK_PTR_NULL(combinOparaMem_);
5135 0 : HcclCombinOpParam* combinOparaPtr = reinterpret_cast<HcclCombinOpParam*>(combinOparaMem_->ptr());
5136 0 : CHK_PTR_NULL(combinOparaPtr);
5137 0 : CHK_RET(algOperator->SetRmaInfo(combinOparaPtr->aiRMAInfo));
5138 : }
5139 5 : if (selectAivAlg) {
5140 5 : CHK_RET(HandleAclGraphFirstOpAivBuff(opParam.stream.ptr()));
5141 5 : if (aivClearEnable_) {
5142 : // 用于判断图模式是否清零
5143 2 : CHK_RET(algOperator->SetAivClearEnable(aivClearEnable_));
5144 2 : aivOffloadTag_ = 1;
5145 : }
5146 5 : GetAivTag(algDesc.aivTagNum, opParam.isCapture, opParam.aivTag);
5147 5 : HCCL_INFO(
5148 : "[HcclCommunicator][ExecOpAlltoAll] tag[%s] userRank[%u] cur aiv tag [%d].", identifier_.c_str(), userRank_,
5149 : opParam.aivTag);
5150 5 : opParam.aicpuUnfoldMode = false;
5151 5 : opParam.aicpuCacheEnable = 0;
5152 5 : CHK_RET(algOperator->SetNumBlocks(aivCoreLimit));
5153 : }
5154 :
5155 5 : auto algType = algOperator->GetAlgType();
5156 5 : CHK_RET(RegisterDfxInfo(opParam, algType, algRes.slaveStreams, selectAivAlg, tag));
5157 : // 头计数
5158 5 : CHK_RET(StarsCounter(dispatcher_, opParam.stream, HEAD, opParam.aicpuUnfoldMode, retryEnable_, selectAivAlg));
5159 : // 算法执行
5160 0 : auto isSupportAicpuAlg = [](const std::string& algName) {
5161 : static const std::set<std::string> aicpuAlgs
5162 : = {"RunAlltoAllVFullMesh", "RunAlltoAllDirectFullmesh",
5163 : "RunAlltoAllVTwoLevelPipeline", "RunAlltoAllFullMeshSymmetricMemory",
5164 0 : "RunAlltoAllVContinuousPipeline", "RunAlltoAllVPipelineFor91093"};
5165 0 : return aicpuAlgs.count(algName) > 0;
5166 : };
5167 5 : if (opParam.aicpuUnfoldMode && (isSupportAicpuAlg(algName) || aicpuUnfoldModeFor910B)) {
5168 0 : isInplaceStatus_ = 0;
5169 0 : inPlaceSupportRetryStatus_ = InplaceSupportRetryStatus::INPLACE_STATUS_END;
5170 : // algOperator->SupportRetryWithInplaceCheck 依赖 algOperator->SetRetryEnable 才能正确返回是否支持inplace
5171 :
5172 0 : inplaceSupportRetry_ = algOperator->SupportRetryWithInplaceCheck(
5173 0 : opType, opParam, algName, isInplaceStatus_, inPlaceSupportRetryStatus_);
5174 0 : HCCL_INFO(
5175 : "[HcclCommunicator][ExecOp] aicpu Unfold mode algType[%s], inplaceSupportRetry_[%d], opType[%d], "
5176 : "isInplaceStatus_[%d], inPlaceSupportRetryStatus_[%d].",
5177 : AlgTypeToStr(algType).c_str(), inplaceSupportRetry_, opType, isInplaceStatus_, inPlaceSupportRetryStatus_);
5178 0 : CHK_RET(OrchestrateAicpu(
5179 : opType, algName, opParam, algRes, newTag, algType, isCustom, false, needRecreateAlltoallComm));
5180 : } else {
5181 : // HOST展开aclgraph场景，capture从流
5182 5 : if (!selectAivAlg) {
5183 0 : CHK_RET(CaptureSlaveStreams(opParam.stream.ptr(), algRes.slaveStreams));
5184 : }
5185 5 : OpCounterInfo opCounter;
5186 5 : CHK_RET(GetOpCountInfo(opCounter));
5187 5 : CHK_RET(algOperator->SetOpCounter(opCounter));
5188 5 : CHK_RET(algOperator->Orchestrate(algName, opParam, algRes));
5189 : // for profiling, numBlocks upload
5190 5 : CHK_RET(algOperator->GetNumBlocks(numBlocks_));
5191 10 : if (implAlg_->GetAivModeConfig() && GetWorkflowMode() == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE
5192 10 : && !opParam.isCapture) {
5193 2 : CHK_RET(GetCacheMap(algOperator, opParam, algType, selectAivAlg, newTag));
5194 : }
5195 : }
5196 5 : lock.unlock();
5197 : // 尾计数
5198 5 : CHK_RET(StarsCounter(dispatcher_, opParam.stream, TAIL, opParam.aicpuUnfoldMode, retryEnable_, selectAivAlg));
5199 5 : CHK_RET(UnRegisterDfxInfo(opParam, algRes.slaveStreams));
5200 5 : if (selectAivAlg) {
5201 5 : CHK_RET(algOperator->SetAivClearEnable(false));
5202 5 : aivClearEnable_ = false;
5203 : }
5204 :
5205 5 : if (isInGraphCaptureZeroCopy) {
5206 0 : SetWorkflowMode(HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE);
5207 : }
5208 5 : return HCCL_SUCCESS;
5209 5 : }
5210 :
5211 73 : HcclResult HcclCommunicator::RecordOpPara(HcclCMDType opType, const OpParam& opParam)
5212 : {
5213 73 : u32 aivCoreLimit = (GetWorkflowMode() == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OPS_KERNEL_INFO_LIB) ? numBlocks_ : 0;
5214 73 : u8 deterministic = implAlg_->GetDeterministicConfig();
5215 71 : switch (opType) {
5216 59 : case HcclCMDType::HCCL_CMD_ALLGATHER:
5217 : case HcclCMDType::HCCL_CMD_ALLREDUCE:
5218 : case HcclCMDType::HCCL_CMD_REDUCE_SCATTER:
5219 : case HcclCMDType::HCCL_CMD_BROADCAST:
5220 59 : CHK_RET(RankConsistentcyChecker::GetInstance().RecordOpPara(
5221 : opType, opParam.tag, opParam.DataDes.count, opParam.DataDes.dataType, opParam.reduceType, opParam.root,
5222 : cclBufferManager_.GetInCCLbufferSize(), cclBufferManager_.GetOutCCLbufferSize(), identifier_.c_str(),
5223 : ranktableCrc_, deterministic, aivCoreLimit));
5224 64 : break;
5225 8 : case HcclCMDType::HCCL_CMD_SCATTER:
5226 : case HcclCMDType::HCCL_CMD_REDUCE:
5227 8 : CHK_RET(RankConsistentcyChecker::GetInstance().RecordOpPara(
5228 : opType, opParam.tag, opParam.DataDes.count, opParam.DataDes.dataType, opParam.reduceType, opParam.root,
5229 : cclBufferManager_.GetInCCLbufferSize(), cclBufferManager_.GetOutCCLbufferSize(), identifier_.c_str(),
5230 : ranktableCrc_, deterministic));
5231 8 : break;
5232 0 : case HcclCMDType::HCCL_CMD_REDUCE_SCATTER_V:
5233 : case HcclCMDType::HCCL_CMD_ALLGATHER_V:
5234 0 : CHK_RET(RankConsistentcyChecker::GetInstance().RecordOpPara(
5235 : opType, opParam.tag, opParam.VDataDes.counts, opParam.VDataDes.displs, userRankSize_,
5236 : opParam.VDataDes.dataType, opParam.reduceType, cclBufferManager_.GetInCCLbufferSize(),
5237 : cclBufferManager_.GetOutCCLbufferSize(), identifier_.c_str(), ranktableCrc_, deterministic,
5238 : aivCoreLimit));
5239 0 : break;
5240 0 : case HcclCMDType::HCCL_CMD_BATCH_SEND_RECV:
5241 0 : CHK_RET(RankConsistentcyChecker::GetInstance().RecordOpPara(
5242 : opType, opParam.tag, cclBufferManager_.GetInCCLbufferSize(), cclBufferManager_.GetOutCCLbufferSize(),
5243 : identifier_.c_str(), ranktableCrc_));
5244 0 : break;
5245 0 : case HcclCMDType::HCCL_CMD_SEND:
5246 0 : CHK_RET(RankConsistentcyChecker::GetInstance().RecordOpPara(
5247 : opType, opParam.tag, opParam.DataDes.count, opParam.DataDes.dataType, opParam.dstRank, opParam.srTag,
5248 : opParam.localGroupRank, cclBufferManager_.GetInCCLbufferSize(), cclBufferManager_.GetOutCCLbufferSize(),
5249 : identifier_.c_str(), ranktableCrc_));
5250 0 : break;
5251 0 : case HcclCMDType::HCCL_CMD_RECEIVE:
5252 0 : CHK_RET(RankConsistentcyChecker::GetInstance().RecordOpPara(
5253 : opType, opParam.tag, opParam.DataDes.count, opParam.DataDes.dataType, opParam.srcRank, opParam.srTag,
5254 : opParam.localGroupRank, cclBufferManager_.GetInCCLbufferSize(), cclBufferManager_.GetOutCCLbufferSize(),
5255 : identifier_.c_str(), ranktableCrc_));
5256 0 : break;
5257 0 : case HcclCMDType::HCCL_CMD_ALLTOALL:
5258 0 : CHK_RET(RankConsistentcyChecker::GetInstance().RecordOpPara(
5259 : opType, opParam.tag, opParam.All2AllDataDes.sendCount, opParam.All2AllDataDes.sendType,
5260 : opParam.reduceType, opParam.root, cclBufferManager_.GetInCCLbufferSize(),
5261 : cclBufferManager_.GetOutCCLbufferSize(), identifier_.c_str(), ranktableCrc_, aivCoreLimit));
5262 0 : break;
5263 4 : case HcclCMDType::HCCL_CMD_ALLTOALLV:
5264 : case HcclCMDType::HCCL_CMD_ALLTOALLVC:
5265 4 : CHK_RET(RankConsistentcyChecker::GetInstance().RecordOpPara(
5266 : opType, opParam.tag, 0, HCCL_DATA_TYPE_RESERVED, opParam.reduceType, opParam.root,
5267 : cclBufferManager_.GetInCCLbufferSize(), cclBufferManager_.GetOutCCLbufferSize(), identifier_.c_str(),
5268 : ranktableCrc_, aivCoreLimit));
5269 4 : break;
5270 0 : default:
5271 0 : break;
5272 : }
5273 76 : return HCCL_SUCCESS;
5274 : }
5275 5 : HcclResult HcclCommunicator::HandleAclGraphFirstOpAivBuff(rtStream_t mainStream)
5276 : {
5277 5 : aclmdlRI rtModel = nullptr;
5278 5 : bool isCapture = false;
5279 5 : u64 modelId = 0;
5280 5 : CHK_RET(GetStreamCaptureInfo(mainStream, rtModel, isCapture));
5281 5 : if (isCapture) {
5282 3 : CHK_PTR_NULL(rtModel);
5283 : // 获取不到modelId会报错
5284 3 : CHK_RET(GetModelId(rtModel, modelId));
5285 3 : if (captureModelIds_.find(modelId) == captureModelIds_.end()) {
5286 : // aclgraph场景，首算子清理AIV buff
5287 2 : aivClearEnable_ = true;
5288 2 : captureModelIds_.insert(modelId);
5289 2 : HCCL_INFO("[HcclCommunicator][%s] modelId[%u] is inserted to captureModelIds_", __func__, modelId);
5290 : }
5291 : }
5292 5 : return HCCL_SUCCESS;
5293 : }
5294 :
5295 89 : bool HcclCommunicator::StreamIsCapture(rtStream_t mainStream)
5296 : {
5297 89 : bool isCapture = false;
5298 89 : aclmdlRI rtModel = nullptr;
5299 89 : CHK_RET(GetStreamCaptureInfo(mainStream, rtModel, isCapture));
5300 95 : return isCapture;
5301 : }
5302 :
5303 19 : HcclResult HcclCommunicator::CaptureSlaveStreams(rtStream_t mainStream, vector<Stream>& slaveStreams)
5304 : {
5305 0 : if ((deviceType_ != DevType::DEV_TYPE_910_93) && (deviceType_ != DevType::DEV_TYPE_310P3)
5306 19 : && (deviceType_ != DevType::DEV_TYPE_910B || GetExternalInputHcclEnableFfts())) {
5307 0 : HCCL_INFO(
5308 : "[HcclCommunicator][%s]Only 310P3 or A2 or A3 device in host expand mode need to capture slave streams.",
5309 : __func__);
5310 0 : return HCCL_SUCCESS;
5311 : }
5312 19 : aclmdlRI rtModel = nullptr;
5313 19 : bool isCapture = false;
5314 19 : u64 modelId = 0;
5315 19 : CHK_RET(GetStreamCaptureInfo(mainStream, rtModel, isCapture));
5316 19 : if (isCapture) {
5317 0 : CHK_PTR_NULL(rtModel);
5318 0 : CHK_RET(GetModelId(rtModel, modelId));
5319 0 : for (auto slaveStream : slaveStreams) {
5320 0 : CHK_RET(AddStreamToModel(slaveStream.ptr(), rtModel));
5321 0 : HCCL_DEBUG(
5322 : "[HcclCommunicator][%s]Add stream[%d] to model[%u] success.", __func__, slaveStream.id(), modelId);
5323 0 : }
5324 : }
5325 19 : return HCCL_SUCCESS;
5326 : }
5327 :
5328 0 : HcclResult HcclCommunicator::BuildOpLocalScratchMemResParam(
5329 : const AlgResourceResponse& algResource, const std::string& newTag, LocalResInfoV2* localResHostPtr)
5330 : {
5331 0 : if (algResource.scratchMem.size() > 0) {
5332 0 : hostMemVec_.resize(hostMemVec_.size() + 1);
5333 0 : CHK_RET(AllocAndClearHostMem(sizeof(HccltagLocalResV2), hostMemVec_.back()));
5334 0 : HccltagLocalResV2* tagLocalResHostPtr = static_cast<HccltagLocalResV2*>(hostMemVec_.back().get()->ptr());
5335 :
5336 0 : deviceMemVec_.resize(deviceMemVec_.size() + 1);
5337 0 : CHK_RET(AllocAndClearDeviceMem(sizeof(HccltagLocalResV2), deviceMemVec_.back()));
5338 0 : HccltagLocalResV2* tagLocalResDevicePtr = static_cast<HccltagLocalResV2*>(deviceMemVec_.back().get()->ptr());
5339 :
5340 : // 初始化HcclRankRelationResV2中的tagRes链表
5341 0 : ListCommonInit(&tagLocalResDevicePtr->nextTagRes, &tagLocalResHostPtr->nextTagRes);
5342 : // 刷新host空间内容
5343 0 : CHK_SAFETY_FUNC_RET(
5344 : memcpy_s(tagLocalResHostPtr->tag, sizeof(tagLocalResHostPtr->tag), newTag.c_str(), newTag.length() + 1));
5345 0 : tagLocalResHostPtr->ScratchmemSize = algResource.scratchMem.size();
5346 0 : tagLocalResHostPtr->Scratchmem = reinterpret_cast<u64>(algResource.scratchMem.ptr());
5347 :
5348 : // 3、将节点插入链表头
5349 0 : ListCommonAddHead(
5350 : &tagLocalResDevicePtr->nextTagRes, &tagLocalResHostPtr->nextTagRes, &localResHostPtr->nextTagRes,
5351 0 : &opResDeviceParaPtr_->localRes.nextTagRes);
5352 0 : HCCL_RUN_INFO(
5353 : "[HcclCommunicator][BuildOpLocalScratchMemResParam] LocalResHostPtr head addr[%p], nextHost[%p], "
5354 : "preHost[%p], tag LocalResHostPtr head addr[%p], nextHost[%p],"
5355 : "preHost[%p], tag[%s]",
5356 : &localResHostPtr->nextTagRes, localResHostPtr->nextTagRes.nextHost, localResHostPtr->nextTagRes.preHost,
5357 : &tagLocalResHostPtr->nextTagRes, tagLocalResHostPtr->nextTagRes.nextHost,
5358 : tagLocalResHostPtr->nextTagRes.preHost, tagLocalResHostPtr->tag);
5359 : }
5360 0 : return HCCL_SUCCESS;
5361 : }
5362 :
5363 0 : HcclResult HcclCommunicator::CheckSetRetryStateToWaitResume()
5364 : {
5365 0 : if (retryEnable_ && opRetryManager_ != nullptr) {
5366 0 : HcclResult ret = opRetryManager_->SetRetryStateToWaitResume(identifier_, commConnections_.isRoot);
5367 0 : CHK_PRT_RET(
5368 : ret != HCCL_SUCCESS, HCCL_ERROR("[NsRecovery]set opretry state to wait resume timeout."), HCCL_E_INTERNAL);
5369 : }
5370 0 : return HCCL_SUCCESS;
5371 : }
5372 :
5373 0 : HcclResult HcclCommunicator::BuildOpLocalResParam(const AlgResourceResponse& algResource, const std::string& newTag)
5374 : {
5375 0 : LocalResInfoV2* localResHostPtr = &opResPara_.localRes;
5376 0 : ListCommonInit(&opResDeviceParaPtr_->localRes.nextTagRes, &opResPara_.localRes.nextTagRes);
5377 0 : if (algResource.slaveDevStreams.size() > LOCAL_STREAM_MAX_NUM) {
5378 0 : HCCL_ERROR("[HcclCommunicator][BuildOpLocalResParam]Fail to assign stream for tag[%s]", newTag.c_str());
5379 0 : return HCCL_E_PARA;
5380 : }
5381 0 : auto signalM2SNum = algResource.notifiesDevMain.size();
5382 0 : auto signalS2MNum = algResource.notifiesDevAux.size();
5383 0 : auto signalNum = signalM2SNum + signalS2MNum;
5384 0 : if (signalNum > LOCAL_NOTIFY_MAX_NUM) {
5385 0 : HCCL_ERROR("[HcclCommunicator][BuildOpLocalResParam]Fail to assign local notify for tag[%s]", newTag.c_str());
5386 0 : return HCCL_E_PARA;
5387 : }
5388 :
5389 0 : localResHostPtr->streamNum = algResource.slaveDevStreams.size();
5390 0 : for (u32 i = 0; i < algResource.slaveDevStreams.size(); i++) {
5391 0 : localResHostPtr->streamParam[i].streamInfo.streamIds = algResource.slaveDevStreams[i].id();
5392 0 : localResHostPtr->streamParam[i].streamInfo.sqIds = algResource.slaveDevStreams[i].sqId();
5393 0 : localResHostPtr->streamParam[i].streamInfo.cqIds = algResource.slaveDevStreams[i].cqId();
5394 0 : localResHostPtr->streamParam[i].streamInfo.logicCqids = algResource.slaveDevStreams[i].logicCqId();
5395 0 : CHK_RET(AllocAndGetStreamContextBuff(
5396 : algResource.slaveDevStreams[i].id(), localResHostPtr->streamParam[i].sqCqContextAddr,
5397 : localResHostPtr->streamParam[i].sqCqContextSize));
5398 : }
5399 :
5400 0 : localResHostPtr->signalNum = signalNum;
5401 :
5402 0 : for (u32 i = 0; i < signalM2SNum; i++) {
5403 0 : algResource.notifiesDevMain[i]->GetNotifyData(localResHostPtr->localSignals[i << 1]);
5404 0 : algResource.notifiesDevAux[i]->GetNotifyData(localResHostPtr->localSignals[(i << 1) + 1]);
5405 : }
5406 0 : HcclResult ret = HCCL_SUCCESS;
5407 0 : ret = CreateAndGetAiCpuNotify(
5408 0 : localAiCpuOpNotify_[static_cast<u32>(AicpuLocalNotifyIdx::HOST_TO_AICPU_0)],
5409 0 : localResHostPtr->aicpuOpNotify[static_cast<u32>(AicpuLocalNotifyIdx::HOST_TO_AICPU_0)]);
5410 0 : CHK_PRT_RET(
5411 : ret != HCCL_SUCCESS,
5412 : HCCL_ERROR(
5413 : "[HcclCommunicator][BuildOpLocalResParam]get aicpu notify 0 error,"
5414 : "errNo[0x%016llx]",
5415 : HCCL_ERROR_CODE(ret)),
5416 : ret);
5417 0 : ret = CreateAndGetAiCpuNotify(
5418 0 : localAiCpuOpNotify_[static_cast<u32>(AicpuLocalNotifyIdx::HOST_TO_AICPU_1)],
5419 0 : localResHostPtr->aicpuOpNotify[static_cast<u32>(AicpuLocalNotifyIdx::HOST_TO_AICPU_1)]);
5420 0 : CHK_PRT_RET(
5421 : ret != HCCL_SUCCESS,
5422 : HCCL_ERROR(
5423 : "[HcclCommunicator][BuildOpLocalResParam]get aicpu notify 1 error,errNo[0x%016llx]", HCCL_ERROR_CODE(ret)),
5424 : ret);
5425 :
5426 0 : if (opMainStream_.ptr() == nullptr) {
5427 0 : opMainStream_ = Stream(StreamType::STREAM_TYPE_DEVICE);
5428 : }
5429 0 : localResHostPtr->mainStreamParam.streamInfo.streamIds = opMainStream_.id();
5430 0 : localResHostPtr->mainStreamParam.streamInfo.sqIds = opMainStream_.sqId();
5431 0 : localResHostPtr->mainStreamParam.streamInfo.cqIds = opMainStream_.cqId();
5432 0 : localResHostPtr->mainStreamParam.streamInfo.logicCqids = opMainStream_.logicCqId();
5433 0 : CHK_RET(AllocAndGetStreamContextBuff(
5434 : opMainStream_.id(), localResHostPtr->mainStreamParam.sqCqContextAddr,
5435 : localResHostPtr->mainStreamParam.sqCqContextSize));
5436 :
5437 : // 按序下发的aicpu控制流
5438 0 : if (aicpuOrderStream_.ptr() == nullptr) {
5439 0 : aicpuOrderStream_ = Stream(StreamType::STREAM_TYPE_DEVICE);
5440 : }
5441 0 : opResPara_.aicpuOrderStreamParam.streamInfo.streamIds = aicpuOrderStream_.id();
5442 0 : opResPara_.aicpuOrderStreamParam.streamInfo.sqIds = aicpuOrderStream_.sqId();
5443 0 : opResPara_.aicpuOrderStreamParam.streamInfo.cqIds = aicpuOrderStream_.cqId();
5444 0 : opResPara_.aicpuOrderStreamParam.streamInfo.logicCqids = aicpuOrderStream_.logicCqId();
5445 0 : CHK_RET(AllocAndGetStreamContextBuff(
5446 : opResPara_.aicpuOrderStreamParam.streamInfo.streamIds, opResPara_.aicpuOrderStreamParam.sqCqContextAddr,
5447 : opResPara_.aicpuOrderStreamParam.sqCqContextSize));
5448 :
5449 : #ifndef CCL_KERNEL_AICPU
5450 0 : for (u32 i = 0; i < AICPU_LOCAL_EVENT_SIZE; ++i) {
5451 0 : aclError ret = aclrtCreateEventExWithFlag(&localAicpuOpEvent_[i], ACL_EVENT_SYNC);
5452 0 : CHK_PRT_RET(
5453 : ret != ACL_SUCCESS,
5454 : HCCL_ERROR(
5455 : "[%s]aclrtCreateEventExWithFlag failed, ret[%d] event[%p].", __func__, ret, localAicpuOpEvent_[i]),
5456 : HCCL_E_RUNTIME);
5457 : }
5458 : #endif
5459 :
5460 0 : CHK_RET(BuildOpLocalScratchMemResParam(algResource, newTag, localResHostPtr));
5461 0 : return HCCL_SUCCESS;
5462 : }
5463 :
5464 0 : HcclResult HcclCommunicator::AllocAndGetStreamContextBuff(u32 streamId, u64& addr, u64& size)
5465 : {
5466 0 : if (streamIdToStreamContext_.find(streamId) == streamIdToStreamContext_.end()) {
5467 0 : DeviceMem streamContext;
5468 0 : CHK_RET(CreateWorkSpace(sizeof(SqCqeContext), streamContext));
5469 0 : streamIdToStreamContext_.insert({streamId, std::move(streamContext)});
5470 0 : }
5471 0 : addr = reinterpret_cast<u64>(streamIdToStreamContext_.at(streamId).ptr());
5472 0 : size = streamIdToStreamContext_.at(streamId).size();
5473 0 : HCCL_INFO("%s success, streamId:%u, addr:0x%llx, size:%llu", __func__, streamId, addr, size);
5474 0 : return HCCL_SUCCESS;
5475 : }
5476 :
5477 0 : u32 HcclCommunicator::UpdateOpIndex(const OpParam& opParam)
5478 : {
5479 0 : u32 opIndex = 0;
5480 0 : u32 commIndex = 0;
5481 : // 用于重执行和taskException打印的算子计数，bsr/sendrecv/其他算子分别计数
5482 0 : if (opParam.opType == HcclCMDType::HCCL_CMD_BATCH_SEND_RECV) {
5483 0 : constexpr s32 batSendRecvIndex = -1; // batchSendRecv使用 key = -1
5484 0 : commIndex = batSendRecvIndex;
5485 0 : } else if (opParam.opType == HcclCMDType::HCCL_CMD_SEND) {
5486 0 : commIndex = opParam.dstRank;
5487 0 : } else if (opParam.opType == HcclCMDType::HCCL_CMD_RECEIVE) {
5488 0 : commIndex = opParam.srcRank;
5489 : } else {
5490 0 : commIndex = userRank_;
5491 : }
5492 :
5493 0 : auto it = opIndexMap_.find(commIndex);
5494 0 : if (it != opIndexMap_.end()) {
5495 0 : opIndex = ++(it->second);
5496 : } else {
5497 0 : opIndexMap_.insert({commIndex, 1});
5498 0 : opIndex = 1;
5499 : }
5500 :
5501 0 : HCCL_DEBUG(
5502 : "%s tag:%s opType:%u commIndex:%u opIndex:%u", __func__, opParam.tag.c_str(), opParam.opType, commIndex,
5503 : opIndex);
5504 0 : return opIndex;
5505 : }
5506 :
5507 0 : HcclResult HcclCommunicator::BuildAicpuCustomParam()
5508 : {
5509 0 : if (aicpuCustomDev_.ptr() == nullptr) {
5510 0 : CHK_RET(CreateWorkSpace(sizeof(AicpuCustomParam), aicpuCustomDev_));
5511 : }
5512 :
5513 0 : opResPara_.aicpuCustomParamAddr = reinterpret_cast<u64>(aicpuCustomDev_.ptr());
5514 0 : opResPara_.aicpuCustomParamSize = aicpuCustomDev_.size();
5515 0 : HCCL_INFO(
5516 : "%s success, aicpuCustomParamAddr:0x%llx, aicpuCustomParamSize:%llu", __func__, opResPara_.aicpuCustomParamAddr,
5517 : opResPara_.aicpuCustomParamSize);
5518 0 : return HCCL_SUCCESS;
5519 : }
5520 :
5521 0 : HcclResult HcclCommunicator::BuildAicpuOrderLaunchNotify()
5522 : {
5523 0 : if (aicpuOrderNotifyAddr_.ptr() == nullptr) {
5524 0 : CHK_RET(CreateWorkSpace(sizeof(HcclSignalInfo) * AICPU_ORDER_NOTIFY_MAX_NUM, aicpuOrderNotifyAddr_));
5525 : }
5526 :
5527 0 : opResPara_.aicpuOrderNotifyAddr = reinterpret_cast<u64>(aicpuOrderNotifyAddr_.ptr());
5528 0 : opResPara_.aicpuOrderNotifySize = aicpuOrderNotifyAddr_.size();
5529 0 : HCCL_INFO(
5530 : "%s success, aicpuOrderNotifyAddr:0x%llx, aicpuOrderNotifySize:%llu", __func__, opResPara_.aicpuOrderNotifyAddr,
5531 : opResPara_.aicpuOrderNotifySize);
5532 0 : return HCCL_SUCCESS;
5533 : }
5534 :
5535 : HcclResult
5536 0 : HcclCommunicator::BuildAiRmaInfoParam(const std::string& newTag, const std::string& algName, const HcclCMDType opType)
5537 : {
5538 0 : HCCL_DEBUG("[HcclCommunicator][%s] Start prepare.", __func__);
5539 0 : CHK_PTR_NULL(aiRMAInfoMem_);
5540 0 : HcclAiRMAInfo* aiRMAInfoPtr = reinterpret_cast<HcclAiRMAInfo*>(aiRMAInfoMem_->ptr());
5541 0 : CHK_PTR_NULL(aiRMAInfoPtr);
5542 0 : aiRMAInfoPtr->curRankId = userRank_;
5543 0 : aiRMAInfoPtr->rankNum = userRankSize_;
5544 0 : u32 localRankSize = meshAggregationRankSize_;
5545 0 : LevelNSubCommTransport& commTransport = resMap_[newTag].opTransportResponse[COMM_LEVEL0];
5546 0 : CHK_PRT_RET(
5547 : commTransport.size() <= 0,
5548 : HCCL_ERROR(
5549 : "[%s] no LevelComm resource, please create comm first. "
5550 : "tag[%s], curRankId[%u] rankNum[%u]",
5551 : __func__, newTag.c_str(), aiRMAInfoPtr->curRankId, aiRMAInfoPtr->rankNum),
5552 : HCCL_E_INTERNAL);
5553 0 : std::vector<LINK>& links = commTransport[0].links;
5554 0 : CHK_PRT_RET(
5555 : links.size() <= 0,
5556 : HCCL_ERROR(
5557 : "[%s] no transport resource, please create links first. "
5558 : "tag[%s], curRankId[%u] rankNum[%u]",
5559 : __func__, newTag.c_str(), aiRMAInfoPtr->curRankId, aiRMAInfoPtr->rankNum),
5560 : HCCL_E_INTERNAL);
5561 :
5562 0 : LevelNSubCommTransport& tmpCommTransport = resMap_[newTag].opTransportResponse[COMM_MESH_L1];
5563 0 : CHK_PRT_RET(
5564 : tmpCommTransport.size() <= 0,
5565 : HCCL_ERROR(
5566 : "[%s] no LevelComm resource, please create comm first. "
5567 : "tag[%s], curRankId[%u] rankNum[%u]",
5568 : __func__, newTag.c_str(), aiRMAInfoPtr->curRankId, aiRMAInfoPtr->rankNum),
5569 : HCCL_E_INTERNAL);
5570 0 : std::vector<LINK>& tmpLinks = tmpCommTransport[0].links;
5571 0 : CHK_PRT_RET(
5572 : tmpLinks.size() <= 0,
5573 : HCCL_ERROR(
5574 : "[%s] no transport resource, please create links first. "
5575 : "tag[%s], curRankId[%u] rankNum[%u]",
5576 : __func__, newTag.c_str(), aiRMAInfoPtr->curRankId, aiRMAInfoPtr->rankNum),
5577 : HCCL_E_INTERNAL);
5578 :
5579 0 : CHK_RET(GetAivQPInfoV2(tmpLinks, newTag));
5580 0 : u32 tmpQueueSize = aiRMAInfoPtr->rankNum * aiRMAInfoPtr->qpNum;
5581 0 : u32 tmpMemSize = aiRMAInfoPtr->rankNum;
5582 0 : u32 tmpMemDetailSize = aiRMAInfoPtr->rankNum * AiMemMaxNum;
5583 :
5584 0 : CHK_RET(AllocAndClearHostMem(sizeof(HcclAiRMAWQ) * tmpQueueSize, aiSqMem_));
5585 0 : CHK_RET(AllocAndClearHostMem(sizeof(HcclAiRMACQ) * tmpQueueSize, aiScqMem_));
5586 0 : CHK_RET(AllocAndClearHostMem(sizeof(HcclAiRMAWQ) * tmpQueueSize, aiRqMem_));
5587 0 : CHK_RET(AllocAndClearHostMem(sizeof(HcclAiRMACQ) * tmpQueueSize, aiRcqMem_));
5588 0 : CHK_RET(AllocAndClearHostMem(sizeof(HcclAiRMAMemInfo) * tmpMemSize, aiMemMem_));
5589 0 : HcclAiRMAMemInfo* aiMemHost = reinterpret_cast<HcclAiRMAMemInfo*>(aiMemMem_->ptr());
5590 :
5591 0 : CHK_RET(AllocAndClearHostMem(sizeof(MemDetails) * tmpMemDetailSize, aiMemDetailsMem_));
5592 0 : MemDetails* aiMemDetailsHost = reinterpret_cast<MemDetails*>(aiMemDetailsMem_->ptr());
5593 :
5594 0 : CHK_RET(DeviceMem::alloc(aiMemDetailsDev_, aiMemDetailsMem_->size()));
5595 0 : u64 memBase = reinterpret_cast<uint64_t>(aiMemDetailsDev_.ptr());
5596 :
5597 0 : for (u32 i = 0; i < aiRMAInfoPtr->rankNum; i++) {
5598 0 : MemDetails& remoteIn = aiMemDetailsHost[i * AiMemMaxNum + GetAiMemTypeVal(HcclAiRMAMemType::REMOTE_INPUT)];
5599 0 : MemDetails& remoteOut = aiMemDetailsHost[i * AiMemMaxNum + GetAiMemTypeVal(HcclAiRMAMemType::REMOTE_OUTPUT)];
5600 0 : MemDetails& localIn = aiMemDetailsHost[i * AiMemMaxNum + GetAiMemTypeVal(HcclAiRMAMemType::LOCAL_INPUT)];
5601 0 : MemDetails& localOut = aiMemDetailsHost[i * AiMemMaxNum + GetAiMemTypeVal(HcclAiRMAMemType::LOCAL_OUTPUT)];
5602 0 : if (i != aiRMAInfoPtr->curRankId
5603 0 : && ((i % localRankSize) == (aiRMAInfoPtr->curRankId % localRankSize)
5604 0 : || (i / localRankSize) == (aiRMAInfoPtr->curRankId / localRankSize))) {
5605 0 : auto transport = links[i % localRankSize]; // localranksize个
5606 0 : if ((i % localRankSize) == (aiRMAInfoPtr->curRankId % localRankSize)) {
5607 0 : transport = tmpLinks[i / localRankSize]; // servernum个
5608 : }
5609 : // link rank info
5610 0 : CHK_RET(GetTransportRemoteMem(transport, UserMemType::INPUT_MEM, remoteIn));
5611 0 : CHK_RET(GetTransportRemoteMem(transport, UserMemType::OUTPUT_MEM, remoteOut));
5612 0 : CHK_RET(GetTransportLocalMem(transport, UserMemType::INPUT_MEM, localIn));
5613 0 : CHK_RET(GetTransportLocalMem(transport, UserMemType::OUTPUT_MEM, localOut));
5614 :
5615 0 : if (transport->GetTransportType() == TransportType::TRANS_TYPE_IBV_EXP) {
5616 0 : CHK_RET(GenIbvAiRMAInfo(i, transport, newTag, aiRMAInfoPtr));
5617 : }
5618 0 : } else if (i == aiRMAInfoPtr->curRankId) {
5619 0 : void* commInPtr = nullptr;
5620 0 : void* commOutPtr = nullptr;
5621 : u64 commInSize;
5622 : u64 commOutSize;
5623 0 : CHK_RET(cclBufferManager_.GetInCCLbuffer(commInPtr, commInSize));
5624 0 : CHK_RET(cclBufferManager_.GetOutCCLbuffer(commOutPtr, commOutSize));
5625 0 : localIn.addr = reinterpret_cast<uint64_t>(commInPtr);
5626 0 : localIn.size = commInSize;
5627 0 : localOut.addr = reinterpret_cast<uint64_t>(commOutPtr);
5628 0 : localOut.size = commOutSize;
5629 : }
5630 :
5631 0 : aiMemHost[i].memMaxNum = AiMemMaxNum;
5632 0 : aiMemHost[i].sizeOfMemDetails = static_cast<u32>(sizeof(MemDetails));
5633 0 : aiMemHost[i].memDetailPtr = memBase + i * AiMemMaxNum * aiMemHost[i].sizeOfMemDetails;
5634 :
5635 0 : HCCL_DEBUG(
5636 : "[%s] tag[%s] curRankId[%u] dstRankId[%u] rankNum[%u] qpNum[%u] memMaxNum[%u] sizeOfMemDetails[%u] "
5637 : "memDetailPtr[%p] remoteInAddr[%p] remoteInSize[%llu] remoteOutAddr[%p] "
5638 : "remoteOutSize[%llu] localInAddr[%p] localInSize[%llu] "
5639 : "localOutAddr[%p] localOutSize[%llu] ",
5640 : __func__, newTag.c_str(), aiRMAInfoPtr->curRankId, i, aiRMAInfoPtr->rankNum, aiRMAInfoPtr->qpNum,
5641 : aiMemHost[i].memMaxNum, aiMemHost[i].sizeOfMemDetails, aiMemHost[i].memDetailPtr, remoteIn.addr,
5642 : remoteIn.size, remoteOut.addr, remoteOut.size, localIn.addr, localIn.size, localOut.addr, localOut.size);
5643 : }
5644 0 : return HCCL_SUCCESS;
5645 : }
5646 :
5647 : template <typename T>
5648 1 : HcclResult HcclCommunicator::CopyVectorToDeviceMem(const u64 len, DeviceMem& dstDeviceMem, const std::vector<T>& srcVec)
5649 : {
5650 1 : CHK_PRT_RET(
5651 : !len, HCCL_INFO("[HcclCommunicator][CopyVectorToDeviceMem] space size is zero. not need to malloc memory"),
5652 : HCCL_SUCCESS);
5653 :
5654 : CHK_PRT_RET(
5655 : (len > ULONG_MAX),
5656 : HCCL_ERROR("[HcclCommunicator][CopyVectorToDeviceMem] space size is greater than %llu", ULONG_MAX),
5657 : HCCL_E_PARA);
5658 :
5659 0 : CHK_RET(CreateWorkSpace(len, dstDeviceMem));
5660 0 : std::shared_ptr<HostMem> srcHostMem;
5661 0 : CHK_RET(AllocAndClearHostMem(len, srcHostMem));
5662 0 : std::copy(srcVec.begin(), srcVec.end(), static_cast<T*>(srcHostMem.get()->ptr()));
5663 0 : CHK_RET(hrtMemSyncCopy(
5664 : dstDeviceMem.ptr(), len, srcHostMem.get()->ptr(), len, HcclRtMemcpyKind::HCCL_RT_MEMCPY_KIND_HOST_TO_DEVICE));
5665 0 : return HCCL_SUCCESS;
5666 0 : }
5667 :
5668 0 : HcclResult HcclCommunicator::BuildOpTopoResTlvParam(
5669 : const std::string& algName, const std::vector<std::vector<std::vector<u32>>>& inputVectorInfo,
5670 : DeviceMem& dstTlvDeviceMem, u64& tlvLen)
5671 : {
5672 0 : vector<u32> tlv;
5673 : CommonTlv commonTlv;
5674 0 : HCCL_DEBUG(
5675 : "[HcclCommunicator][BuildOpTopoResTlvParam] input vector size[%lu], group[%s].", inputVectorInfo.size(),
5676 : identifier_.c_str());
5677 0 : for (u16 level0Idx = 0; level0Idx < inputVectorInfo.size(); level0Idx++) {
5678 0 : for (u16 level1Idx = 0; level1Idx < inputVectorInfo[level0Idx].size(); level1Idx++) {
5679 0 : commonTlv.type = ((level0Idx << TOP_COMM_LEVEL0_SHIFT) | level1Idx);
5680 0 : commonTlv.length = (sizeof(LENGTH_TYPE) + sizeof(TAG_TYPE))
5681 0 : + inputVectorInfo[level0Idx][level1Idx].size() * sizeof(RANK_TYPE);
5682 0 : tlv.push_back(commonTlv.type);
5683 0 : tlv.push_back(commonTlv.length);
5684 0 : tlv.insert(
5685 0 : tlv.end(), inputVectorInfo[level0Idx][level1Idx].begin(), inputVectorInfo[level0Idx][level1Idx].end());
5686 : }
5687 : }
5688 0 : for (u64 idx = 0; idx < tlv.size(); idx++) {
5689 0 : HCCL_DEBUG("[HcclCommunicator][BuildOpTopoResTlvParam] idx[%lu] tlv[%lu].", idx, tlv[idx]);
5690 : }
5691 0 : tlvLen = tlv.size() * sizeof(u32);
5692 0 : CHK_RET(CopyVectorToDeviceMem(tlvLen, dstTlvDeviceMem, tlv));
5693 0 : return HCCL_SUCCESS;
5694 0 : }
5695 :
5696 0 : HcclResult HcclCommunicator::BuildOpTopoResVectorTlvParam(
5697 : const std::string& algName, const std::vector<std::vector<std::vector<std::vector<u32>>>>& inputVectorInfo,
5698 : DeviceMem& dstTlvDeviceMem, u64& tlvLen)
5699 : {
5700 0 : vector<u32> tlv;
5701 : CommonTlv commonTlv;
5702 0 : HCCL_DEBUG(
5703 : "[HcclCommunicator][BuildOpTopoResVectorTlvParam] input vector size[%lu], group[%s]", inputVectorInfo.size(),
5704 : identifier_.c_str());
5705 0 : for (u16 level0Idx = 0; level0Idx < inputVectorInfo.size(); level0Idx++) {
5706 0 : for (u16 level1Idx = 0; level1Idx < inputVectorInfo[level0Idx].size(); level1Idx++) {
5707 0 : for (u16 level2Idx = 0; level2Idx < inputVectorInfo[level0Idx][level1Idx].size(); level2Idx++) {
5708 0 : commonTlv.type = (((level0Idx << TOP_HIERARCHICAL_COMM_LEVEL0_SHIFT) | level1Idx)
5709 0 : << TOP_HIERARCHICAL_COMM_LEVEL1_SHIFT)
5710 0 : | level2Idx;
5711 0 : commonTlv.length = (sizeof(LENGTH_TYPE) + sizeof(TAG_TYPE))
5712 0 : + inputVectorInfo[level0Idx][level1Idx][level2Idx].size() * sizeof(RANK_TYPE);
5713 0 : tlv.push_back(commonTlv.type);
5714 0 : tlv.push_back(commonTlv.length);
5715 0 : tlv.insert(
5716 0 : tlv.end(), inputVectorInfo[level0Idx][level1Idx][level2Idx].begin(),
5717 0 : inputVectorInfo[level0Idx][level1Idx][level2Idx].end());
5718 : }
5719 : }
5720 : }
5721 0 : for (u64 idx = 0; idx < tlv.size(); idx++) {
5722 0 : HCCL_DEBUG("[HcclCommunicator][BuildOpTopoResVectorTlvParam] idx[%lu] tlv[%lu]", idx, tlv[idx]);
5723 : }
5724 0 : tlvLen = tlv.size() * sizeof(u32);
5725 0 : CHK_RET(CopyVectorToDeviceMem(tlvLen, dstTlvDeviceMem, tlv));
5726 0 : return HCCL_SUCCESS;
5727 0 : }
5728 :
5729 0 : HcclResult HcclCommunicator::BuildPairLinkCounter(const std::string& algName)
5730 : {
5731 0 : constexpr u32 KEY_VALUE_TO_VECTOR_MODULUS = 2;
5732 0 : if (pairLinkCounterDevice_.ptr() == nullptr) {
5733 0 : u64 pairLinkCounterSize = pairLinkCounter_.size();
5734 0 : HCCL_DEBUG(
5735 : "[HcclCommunicator][BuildPairLinkCounter] pairLinkCounter size[%lu], group[%s]", pairLinkCounterSize,
5736 : identifier_.c_str());
5737 0 : std::vector<u32> pairLinkCounterVec(pairLinkCounterSize * KEY_VALUE_TO_VECTOR_MODULUS);
5738 0 : u64 index = 0;
5739 0 : for (auto& kt : pairLinkCounter_) {
5740 0 : pairLinkCounterVec[index] = kt.first;
5741 0 : pairLinkCounterVec[index + 1] = kt.second;
5742 0 : index += KEY_VALUE_TO_VECTOR_MODULUS; // 每次根据
5743 : }
5744 0 : u64 len = pairLinkCounterSize * sizeof(u32) * KEY_VALUE_TO_VECTOR_MODULUS; // key-value，都为u32
5745 0 : CHK_RET(CopyVectorToDeviceMem(len, pairLinkCounterDevice_, pairLinkCounterVec));
5746 0 : opResPara_.topoInfo.pairLinkCounter = reinterpret_cast<u64>(pairLinkCounterDevice_.ptr());
5747 0 : opResPara_.topoInfo.pairLinkCounterNum = pairLinkCounterSize * KEY_VALUE_TO_VECTOR_MODULUS;
5748 0 : }
5749 0 : return HCCL_SUCCESS;
5750 : }
5751 :
5752 0 : HcclResult HcclCommunicator::BuildIsUsedRdmaRank(const std::string& algName)
5753 : {
5754 0 : constexpr u32 KEY_VALUE_TO_VECTOR_MODULUS = 2;
5755 0 : if (isUsedRdmaRankPairDevice_.ptr() == nullptr) {
5756 0 : std::unordered_map<u32, bool> isUsedRdmaMap;
5757 0 : CHK_RET(implAlg_->GetIsUsedRdmaMap(isUsedRdmaMap));
5758 0 : u64 isUsedRdmaMapSize = isUsedRdmaMap.size();
5759 0 : HCCL_DEBUG(
5760 : "[HcclCommunicator][BuildIsUsedRdmaRank] is used Rdma rank size[%lu], group[%s]", isUsedRdmaMapSize,
5761 : identifier_.c_str());
5762 0 : std::vector<u32> isUsedRdmaPairVec(isUsedRdmaMapSize * KEY_VALUE_TO_VECTOR_MODULUS);
5763 0 : u64 index = 0;
5764 0 : for (auto& kt : isUsedRdmaMap) {
5765 0 : isUsedRdmaPairVec[index] = kt.first;
5766 0 : isUsedRdmaPairVec[index + 1] = static_cast<u32>(kt.second);
5767 0 : index += KEY_VALUE_TO_VECTOR_MODULUS;
5768 : }
5769 0 : u64 len = isUsedRdmaMapSize * sizeof(u32) * KEY_VALUE_TO_VECTOR_MODULUS; // key-value，都为u32
5770 0 : CHK_RET(CopyVectorToDeviceMem(len, isUsedRdmaRankPairDevice_, isUsedRdmaPairVec));
5771 0 : opResPara_.topoInfo.isUsedRdmaRankPair = reinterpret_cast<u64>(isUsedRdmaRankPairDevice_.ptr());
5772 0 : opResPara_.topoInfo.isUsedRdmaRankPairNum = isUsedRdmaMapSize * KEY_VALUE_TO_VECTOR_MODULUS;
5773 0 : }
5774 0 : return HCCL_SUCCESS;
5775 : }
5776 :
5777 0 : HcclResult HcclCommunicator::BuildNicList(const std::string& algName)
5778 : {
5779 0 : if (nicListDevice_.ptr() == nullptr) {
5780 0 : u64 len = nicList_.size() * sizeof(u32);
5781 0 : HCCL_DEBUG(
5782 : "[HcclCommunicator][BuildNicList] niclist size[%lu], group[%s]", nicList_.size(), identifier_.c_str());
5783 0 : CHK_RET(CopyVectorToDeviceMem(len, nicListDevice_, nicList_));
5784 0 : opResPara_.topoInfo.nicList = reinterpret_cast<u64>(nicListDevice_.ptr());
5785 0 : opResPara_.topoInfo.nicNum = nicList_.size();
5786 : }
5787 0 : return HCCL_SUCCESS;
5788 : }
5789 :
5790 0 : HcclResult HcclCommunicator::BuildBridgeRank(const std::string& algName)
5791 : {
5792 0 : if (bridgeRankDevice_.ptr() == nullptr) {
5793 0 : std::vector<bool> isBridgeVector;
5794 0 : CHK_RET(implAlg_->GetIsBridgeVector(isBridgeVector));
5795 0 : u64 len = isBridgeVector.size() * sizeof(bool);
5796 0 : HCCL_DEBUG(
5797 : "[HcclCommunicator][BuildBridgeRank] Bridge size[%lu], group[%s]", isBridgeVector.size(),
5798 : identifier_.c_str());
5799 0 : CHK_RET(CopyVectorToDeviceMem(len, bridgeRankDevice_, isBridgeVector));
5800 0 : opResPara_.topoInfo.bridgeRank = reinterpret_cast<u64>(bridgeRankDevice_.ptr());
5801 0 : opResPara_.topoInfo.bridgeRankNum = isBridgeVector.size();
5802 0 : }
5803 0 : return HCCL_SUCCESS;
5804 : }
5805 :
5806 0 : HcclResult HcclCommunicator::BuildCommPlanRank(const std::string& algName)
5807 : {
5808 0 : opResPara_.topoInfo.complanRank = 0;
5809 0 : opResPara_.topoInfo.complanRankLength = 0;
5810 0 : if (complanRankDevice_.ptr() == nullptr) {
5811 0 : std::vector<std::vector<std::vector<u32>>> commPlaneRanks;
5812 0 : CHK_RET(implAlg_->GetCommPlaneRanks(commPlaneRanks));
5813 0 : u64 tlvLen = 0;
5814 0 : CHK_RET(BuildOpTopoResTlvParam(algName, commPlaneRanks, complanRankDevice_, tlvLen));
5815 0 : opResPara_.topoInfo.complanRank = reinterpret_cast<u64>(complanRankDevice_.ptr());
5816 0 : opResPara_.topoInfo.complanRankLength = tlvLen;
5817 0 : HCCL_DEBUG(
5818 : "[HcclCommunicator][BuildCommPlanRank] comm plane ranks tlv length[%lu], ptr[%p], group[%s], "
5819 : "local user rankId[%u] ",
5820 : tlvLen, complanRankDevice_.ptr(), identifier_.c_str(), userRank_);
5821 0 : }
5822 0 : return HCCL_SUCCESS;
5823 : }
5824 :
5825 0 : HcclResult HcclCommunicator::BuildServerAndsuperPodRank(const std::string& algName)
5826 : {
5827 0 : opResPara_.topoInfo.serverAndsuperPodRank = 0;
5828 0 : opResPara_.topoInfo.serverAndsuperPodRankLength = 0;
5829 0 : if (serverAndsuperPodToRankDevice_.ptr() == nullptr) {
5830 0 : std::vector<std::vector<std::vector<u32>>> serverAndsuperPodToRank;
5831 0 : CHK_RET(implAlg_->GetRankVecInfo(serverAndsuperPodToRank));
5832 0 : u64 tlvLen = 0;
5833 0 : CHK_RET(BuildOpTopoResTlvParam(algName, serverAndsuperPodToRank, serverAndsuperPodToRankDevice_, tlvLen));
5834 0 : opResPara_.topoInfo.serverAndsuperPodRank = reinterpret_cast<u64>(serverAndsuperPodToRankDevice_.ptr());
5835 0 : opResPara_.topoInfo.serverAndsuperPodRankLength = tlvLen;
5836 0 : HCCL_DEBUG(
5837 : "[HcclCommunicator][BuildServerAndsuperPodRank] server and super pod ranks tlv length[%lu], ptr[%p], "
5838 : "group[%s], local user rankId[%u] ",
5839 : tlvLen, serverAndsuperPodToRankDevice_.ptr(), identifier_.c_str(), userRank_);
5840 0 : }
5841 0 : return HCCL_SUCCESS;
5842 : }
5843 :
5844 0 : HcclResult HcclCommunicator::BuildOpRetryParam(const AlgResourceResponse& algResource, const std::string& newTag)
5845 : {
5846 0 : opResPara_.config.retryEnable = static_cast<u8>(retryEnable_);
5847 0 : opResPara_.config.retryHoldTime = commConfig_.GetConfigRetryHoldTime();
5848 0 : opResPara_.config.retryIntervalTime = commConfig_.GetConfigRetryIntervalTime();
5849 : // aicpu和custom共用同一个opResPara_，aicpu初始化完成后，会修改h2d/d2h的指针，然后重新传给custom
5850 0 : opResPara_.kfcControlTransferH2DParams = kfcControlTransferH2D_->GetCommunicateParams();
5851 0 : opResPara_.kfcStatusTransferD2HParams = kfcStatusTransferD2H_->GetCommunicateParams();
5852 0 : opResPara_.debugConfig = GetDebugConfig();
5853 :
5854 0 : CHK_SMART_PTR_NULL(opRetryStreamPtr_);
5855 0 : if (opRetryStreamPtr_->find(newTag) == opRetryStreamPtr_->end()) {
5856 0 : std::vector<Stream> retryStreams(algResource.slaveDevStreams.begin(), algResource.slaveDevStreams.end());
5857 0 : retryStreams.push_back(opMainStream_);
5858 0 : opRetryStreamPtr_->insert(std::make_pair(newTag, retryStreams));
5859 0 : }
5860 0 : return HCCL_SUCCESS;
5861 : }
5862 :
5863 0 : HcclResult HcclCommunicator::BuildCommPlaneSubGroupRank(const std::string& algName)
5864 : {
5865 0 : opResPara_.hierarchicalAlgInfo.commplaneSubGroupRank = 0;
5866 0 : opResPara_.hierarchicalAlgInfo.commplaneSubGroupRankLength = 0;
5867 0 : if (commplaneSubGroupRankDevice_.ptr() == nullptr) {
5868 0 : std::vector<std::vector<std::vector<std::vector<u32>>>> commplaneSubGroupVector;
5869 0 : CHK_RET(implAlg_->GetCommPlaneSubGroupVector(commplaneSubGroupVector));
5870 0 : u64 tlvLen = 0;
5871 0 : CHK_RET(BuildOpTopoResVectorTlvParam(algName, commplaneSubGroupVector, commplaneSubGroupRankDevice_, tlvLen));
5872 : opResPara_.hierarchicalAlgInfo.commplaneSubGroupRank
5873 0 : = reinterpret_cast<u64>(commplaneSubGroupRankDevice_.ptr());
5874 0 : opResPara_.hierarchicalAlgInfo.commplaneSubGroupRankLength = tlvLen;
5875 0 : HCCL_DEBUG(
5876 : "[HcclCommunicator][BuildCommPlaneSubGroupRank] comm plane subGroups ranks tlv length[%lu], ptr[%p], "
5877 : "group[%s], local user rankId[%u] ",
5878 : tlvLen, commplaneSubGroupRankDevice_.ptr(), identifier_.c_str(), userRank_);
5879 0 : }
5880 0 : return HCCL_SUCCESS;
5881 : }
5882 :
5883 0 : HcclResult HcclCommunicator::BuildHierarchicalAlgOption(u32* ahcConfInfo)
5884 : {
5885 0 : std::map<AHCConcOpType, TemplateType> hierarchicalAlgOption;
5886 0 : CHK_RET(implAlg_->GetAHCAlgOption(hierarchicalAlgOption));
5887 0 : ahcConfInfo[TOP_HIERARCHICAL_CONF_lENGTH_INDEX] = hierarchicalAlgOption.size();
5888 :
5889 0 : if (hierarchicalAlgOption.size() >= (TOP_HIERARCHICAL_CONF_SIZE - 1)) {
5890 0 : HCCL_ERROR(
5891 : "[HcclCommunicator][BuildHierarchicalAlgOption] host hierarchicalAlgOption size[%u] exceed maxsize[%u]",
5892 : hierarchicalAlgOption.size(), (TOP_HIERARCHICAL_CONF_SIZE - 1));
5893 0 : return HCCL_E_INTERNAL;
5894 : }
5895 :
5896 0 : HCCL_DEBUG(
5897 : "[HcclCommunicator][BuildHierarchicalAlgOption] host hierarchicalAlgOption.size() [%u]",
5898 : hierarchicalAlgOption.size());
5899 :
5900 : // 默认清空内存
5901 0 : for (u32 i = TOP_HIERARCHICAL_CONF_INFO_INDEX; i < TOP_HIERARCHICAL_CONF_SIZE; i++) {
5902 0 : ahcConfInfo[i] = 0;
5903 : }
5904 :
5905 0 : u32 confDataStartIndex = TOP_HIERARCHICAL_CONF_INFO_INDEX;
5906 0 : for (auto it = hierarchicalAlgOption.begin(); it != hierarchicalAlgOption.end(); ++it) {
5907 0 : HCCL_DEBUG(
5908 : "[HcclCommunicator][BuildHierarchicalAlgOption] host Level [%u], ConcType[%u] AHCOpType[%u], TemplateType "
5909 : "[%u]",
5910 : it->first.ahcLevel, it->first.concType, it->first.ahcOpType, it->second);
5911 :
5912 0 : u32 confData = (static_cast<u32>(it->first.ahcLevel) << TOP_HIERARCHICAL_CONF_LEVEL_SHIFT)
5913 0 : | (static_cast<u32>(it->first.concType) << TOP_HIERARCHICAL_CONF_CONC_TYPE_SHIFT)
5914 0 : | (static_cast<u32>(it->first.ahcOpType) << TOP_HIERARCHICAL_CONF_OP_TYPE_SHIFT)
5915 0 : | (static_cast<u32>(it->second) << TOP_HIERARCHICAL_CONF_TEMPLATE_TYPE_SHIFT);
5916 0 : ahcConfInfo[confDataStartIndex] = confData;
5917 0 : confDataStartIndex = confDataStartIndex + 1;
5918 : }
5919 0 : return HCCL_SUCCESS;
5920 0 : }
5921 :
5922 0 : HcclResult HcclCommunicator::BuildOpTopoResParam(const std::string& algName, const AlgResourceResponse& algResource)
5923 : {
5924 0 : opResPara_.topoInfo.userRank = userRank_;
5925 0 : opResPara_.topoInfo.userRankSize = userRankSize_;
5926 0 : opResPara_.topoInfo.deviceLogicId = deviceLogicId_;
5927 0 : opResPara_.topoInfo.isSingleMeshAggregation = isSingleMeshAggregation_;
5928 0 : opResPara_.topoInfo.deviceNumPerAggregation = deviceNumPerAggregation_;
5929 0 : opResPara_.topoInfo.superPodNum = superPodNum_;
5930 0 : opResPara_.topoInfo.devicePhyId = devicePhyId_;
5931 0 : opResPara_.topoInfo.deviceType = static_cast<u32>(deviceType_);
5932 : TopoType topoType;
5933 0 : CHK_RET(implAlg_->GetTopoType(topoType));
5934 0 : opResPara_.topoInfo.topoType = static_cast<u32>(topoType);
5935 0 : opResPara_.topoInfo.serverNum = serverNum_;
5936 0 : opResPara_.topoInfo.meshAggregationRankSize = meshAggregationRankSize_;
5937 0 : opResPara_.topoInfo.multiModuleDiffDeviceNumMode = multiModuleDiffDeviceNumMode_;
5938 0 : opResPara_.topoInfo.multiSuperPodDiffServerNumMode = multiSuperPodDiffServerNumMode_;
5939 0 : opResPara_.topoInfo.realUserRank = realUserRank_;
5940 0 : opResPara_.topoInfo.isDiffDeviceModule = isDiffDeviceModule_;
5941 0 : opResPara_.topoInfo.isDiffDeviceType = isDiffDeviceType_;
5942 0 : opResPara_.topoInfo.gcdDeviceNumPerAggregation = gcdDeviceNumPerAggregation_;
5943 0 : opResPara_.topoInfo.moduleNum = moduleNum_;
5944 0 : opResPara_.isARSDoubleRing = isARSDoubleRing_;
5945 0 : opResPara_.multiSuperPodDiffDeviceNumMode = multiSuperPodDiffDeviceNumMode_;
5946 0 : CHK_RET(BuildPairLinkCounter(algName));
5947 0 : CHK_RET(BuildIsUsedRdmaRank(algName));
5948 0 : CHK_RET(BuildNicList(algName));
5949 0 : CHK_RET(BuildBridgeRank(algName));
5950 0 : CHK_RET(BuildCommPlanRank(algName));
5951 0 : CHK_RET(BuildServerAndsuperPodRank(algName));
5952 0 : CHK_RET(BuildCommPlaneSubGroupRank(algName));
5953 0 : return HCCL_SUCCESS;
5954 : }
5955 :
5956 0 : HcclResult HcclCommunicator::BuildOpRemoteLinkP2pResParam(
5957 : const LINK& link, HccltagRemoteResV3& tagRemoteRes, TransportLinkType linkType)
5958 : {
5959 : // hccs sio并发场景，sio链路（linkTyp为SIO）打包到linkP2pSio, hccs链路（linkTyp为HCCS）打包到linkP2p；
5960 : // 其他场景打包到linkP2p
5961 0 : HcclLinkP2pV2* linkp2p = &(tagRemoteRes.tagRemoteResPtr->linkP2p);
5962 0 : if (linkType == TransportLinkType::SIO) {
5963 0 : linkp2p = &(tagRemoteRes.tagRemoteResPtr->linkP2pSio);
5964 : }
5965 0 : if (linkp2p->localIpcSignal[0].resId != INVALID_U64) {
5966 0 : HCCL_INFO(
5967 : "[%s]the linkP2p is existed, no need to refresh transport resource, resId[%llu]", __func__,
5968 : linkp2p->localIpcSignal[0].resId);
5969 0 : return HCCL_SUCCESS;
5970 : }
5971 : // localMem & remoteMem
5972 0 : void* inbufferPtr = nullptr;
5973 0 : void* outbufferPtr = nullptr;
5974 0 : CHK_RET(link->GetRemoteMem(UserMemType::INPUT_MEM, &inbufferPtr));
5975 0 : CHK_RET(link->GetRemoteMem(UserMemType::OUTPUT_MEM, &outbufferPtr));
5976 0 : (linkp2p->remoteMem)[INPUT].addr = reinterpret_cast<u64>(inbufferPtr);
5977 0 : (linkp2p->remoteMem)[OUTPUT].addr = reinterpret_cast<u64>(outbufferPtr);
5978 0 : CHK_RET(link->GetRemoteMemSize(UserMemType::INPUT_MEM, (linkp2p->remoteMem)[INPUT].size));
5979 0 : CHK_RET(link->GetRemoteMemSize(UserMemType::OUTPUT_MEM, (linkp2p->remoteMem)[OUTPUT].size));
5980 0 : MemDetails localMem; // 暂时预留，赋值为空
5981 0 : (linkp2p->localMem)[0] = localMem;
5982 0 : (linkp2p->localMem)[1] = localMem;
5983 0 : HCCL_DEBUG("[%s] finish set localMem & remoteMem info", __func__);
5984 : // localnotify & remotenotify
5985 0 : u64 notifyNum = 0;
5986 0 : std::vector<HcclSignalInfo> locIpcSignals;
5987 0 : std::vector<HcclSignalInfo> rmtIpcSignals;
5988 0 : CHK_RET(link->GetLocalNotify(locIpcSignals));
5989 0 : CHK_RET(link->GetRemoteNotify(rmtIpcSignals));
5990 :
5991 0 : for (size_t i = 0; i < locIpcSignals.size(); i++) {
5992 0 : CHK_RET(CheckNotifyOrQPMaxNum(notifyNum, LINK_P2P_MAX_NUM, true));
5993 0 : linkp2p->localIpcSignal[notifyNum] = locIpcSignals[i];
5994 0 : linkp2p->remoteIpcSignal[notifyNum] = rmtIpcSignals[i];
5995 0 : notifyNum++;
5996 : }
5997 0 : tagRemoteRes.p2pNotifyNum = notifyNum;
5998 0 : HCCL_DEBUG("[%s] finish set localnotify & remotenotify info, notifyNum[%llu]", __func__, notifyNum);
5999 : // transportAttr
6000 0 : CHK_RET(link->GetTransportAttr(linkp2p->transportAttr));
6001 0 : HCCL_DEBUG("[%s] finish set RemoteLinkP2pResParam info", __func__);
6002 0 : return HCCL_SUCCESS;
6003 0 : }
6004 :
6005 0 : HcclResult HcclCommunicator::BuildOpRemoteLinkRoceResParam(
6006 : const LINK& link, HccltagRemoteResV3& tagRemoteRes, bool isBackup, bool isRetry, bool isSecondBuild)
6007 : {
6008 0 : u32 iter = isSecondBuild ? 2 : 0;
6009 0 : HcclLinkRoceV2* linkRoce = isBackup ?
6010 0 : &(tagRemoteRes.tagRemoteResPtr->linkRoce[AICPU_RETRY_LINKROCE_BACKUP + iter]) :
6011 0 : &(tagRemoteRes.tagRemoteResPtr->linkRoce[AICPU_RETRY_LINKROCE_DEFAULT + iter]);
6012 0 : if (!isRetry && linkRoce->localNotifyList != 0) {
6013 0 : HCCL_INFO(
6014 : "[%s]the linkRoce is existed, no need to refresh transport resource, localNotifyListPtr[%p], iter[%u]",
6015 : __func__, reinterpret_cast<void*>(linkRoce->localNotifyList), iter);
6016 0 : return HCCL_SUCCESS;
6017 : }
6018 : // localMem & remoteMem
6019 0 : CHK_RET(link->GetLocalMemDetails(UserMemType::INPUT_MEM, (linkRoce->localMem)[INPUT]));
6020 0 : CHK_RET(link->GetLocalMemDetails(UserMemType::OUTPUT_MEM, (linkRoce->localMem)[OUTPUT]));
6021 0 : void* inbufferPtr = nullptr;
6022 0 : void* outbufferPtr = nullptr;
6023 0 : CHK_RET(link->GetRemoteMem(UserMemType::INPUT_MEM, &inbufferPtr));
6024 0 : CHK_RET(link->GetRemoteMem(UserMemType::OUTPUT_MEM, &outbufferPtr));
6025 0 : HCCL_DEBUG("[%s]inbufferPtr[%p], outbufferPtr[%p]", __func__, inbufferPtr, outbufferPtr);
6026 0 : if (inbufferPtr == nullptr || outbufferPtr == nullptr) {
6027 0 : HCCL_ERROR("[%s]inbufferPtr[%p], outbufferPtr[%p]", __func__, inbufferPtr, outbufferPtr);
6028 0 : return HCCL_E_INTERNAL;
6029 : }
6030 0 : (linkRoce->remoteMem)[INPUT].addr = reinterpret_cast<u64>(inbufferPtr);
6031 0 : (linkRoce->remoteMem)[OUTPUT].addr = reinterpret_cast<u64>(outbufferPtr);
6032 0 : CHK_RET(link->GetRemoteMemKey(UserMemType::INPUT_MEM, &((linkRoce->remoteMem)[INPUT].key)));
6033 0 : CHK_RET(link->GetRemoteMemKey(UserMemType::OUTPUT_MEM, &((linkRoce->remoteMem)[OUTPUT].key)));
6034 0 : CHK_RET(link->GetRemoteMemSize(UserMemType::INPUT_MEM, (linkRoce->remoteMem)[INPUT].size));
6035 0 : CHK_RET(link->GetRemoteMemSize(UserMemType::OUTPUT_MEM, (linkRoce->remoteMem)[OUTPUT].size));
6036 0 : HCCL_DEBUG("[%s] finish set localMem & remoteMem info", __func__);
6037 : // notifyValue & Key
6038 0 : std::vector<AddrKey> notifyValueAddrKey;
6039 0 : CHK_RET(link->GetLocalNotifyValueAddrKey(notifyValueAddrKey));
6040 0 : linkRoce->notifyValue = notifyValueAddrKey[0].addr;
6041 0 : linkRoce->notifyValueKey = notifyValueAddrKey[0].key;
6042 : // QPInfo
6043 0 : std::vector<HcclQpInfoV2> aiQpInfos;
6044 0 : CHK_RET(link->GetAiQpInfo(aiQpInfos));
6045 0 : u32 qpNum = aiQpInfos.size();
6046 0 : if (qpNum > RDMA_QP_MAX_NUM || qpNum < 1) {
6047 0 : return HCCL_E_INTERNAL;
6048 : }
6049 0 : std::copy_n(aiQpInfos.begin(), qpNum, linkRoce->QpInfo);
6050 0 : linkRoce->qpsPerConnection = qpNum - static_cast<u32>(qpNum > 1); // 多QP数量或单QP模式
6051 :
6052 : // localnotify & remotenotify
6053 0 : std::vector<AddrKey> notifyAddrKey;
6054 0 : std::vector<HcclSignalInfo> signalInfos;
6055 0 : CHK_RET(link->GetLocalRdmaNotify(signalInfos));
6056 0 : CHK_RET(link->GetRemoteRdmaNotifyAddrKey(notifyAddrKey));
6057 0 : if ((signalInfos.size() != notifyAddrKey.size()) || (signalInfos.size() < RDMA_NOTIFY_MIN_NUM)
6058 0 : || (signalInfos.size() > RDMA_NOTIFY_MAX_NUM) || (notifyAddrKey.size() < RDMA_NOTIFY_MIN_NUM)
6059 0 : || (notifyAddrKey.size() > RDMA_NOTIFY_MAX_NUM)
6060 0 : || ((signalInfos.size() - RDMA_NOTIFY_MIN_NUM) % linkRoce->qpsPerConnection)
6061 0 : || ((notifyAddrKey.size() - RDMA_NOTIFY_MIN_NUM) % linkRoce->qpsPerConnection)) {
6062 0 : HCCL_ERROR(
6063 : "[HcclCommunicator][BuildOpRemoteLinkRoceResParam] signalInfos %zu notifyAddrKey %zu "
6064 : "qpsPerConnection %u",
6065 : signalInfos.size(), notifyAddrKey.size(), linkRoce->qpsPerConnection);
6066 0 : return HCCL_E_INTERNAL;
6067 : }
6068 0 : u64 notifyNum = (notifyAddrKey.size() - RDMA_NOTIFY_MIN_NUM) / linkRoce->qpsPerConnection
6069 0 : - static_cast<u32>(linkRoce->qpsPerConnection > 1);
6070 0 : linkRoce->singleQPNotifyNum = notifyNum;
6071 :
6072 0 : u64 len = signalInfos.size() * sizeof(HcclSignalInfo);
6073 0 : DeviceMem localNotifyListMem;
6074 0 : CHK_RET(CopyVectorToDeviceMem(len, localNotifyListMem, signalInfos));
6075 0 : linkRoce->localNotifyList = reinterpret_cast<u64>(localNotifyListMem.ptr());
6076 0 : ibverbsLocalNotify_[tagRemoteRes.tagRemoteResPtr->tag].emplace_back(std::move(localNotifyListMem));
6077 :
6078 0 : len = notifyAddrKey.size() * sizeof(AddrKey);
6079 0 : DeviceMem remoteNotifyListMem;
6080 0 : CHK_RET(CopyVectorToDeviceMem(len, remoteNotifyListMem, notifyAddrKey));
6081 0 : linkRoce->remoteNotifyList = reinterpret_cast<u64>(remoteNotifyListMem.ptr());
6082 0 : ibverbsRemoteNotify_[tagRemoteRes.tagRemoteResPtr->tag].emplace_back(std::move(remoteNotifyListMem));
6083 :
6084 0 : HCCL_DEBUG(
6085 : "[%s] finish set localnotify & remotenotify info, notifyNum[%llu], linkNotifyNum[%llu]", __func__, notifyNum,
6086 : signalInfos.size());
6087 :
6088 0 : if (isBackup) {
6089 0 : tagRemoteRes.roceNotifyNumBackup = linkRoce->singleQPNotifyNum;
6090 0 : tagRemoteRes.qpNumBackup = linkRoce->qpsPerConnection;
6091 : } else {
6092 0 : tagRemoteRes.roceNotifyNum = linkRoce->singleQPNotifyNum;
6093 0 : tagRemoteRes.qpNum = linkRoce->qpsPerConnection;
6094 : }
6095 :
6096 0 : linkRoce->useAtomicWrite = link->GetIsUseAtomicWrite();
6097 0 : HCCL_DEBUG(
6098 : "[%s] finish set Qp info qpNum[%u], linkRoce->localNotifyList[0].resId[%llu], "
6099 : "notifyNum[%u], isBackup[%d], isSecond[%d], qpPtr[%llu], useAtomicWrite[%d]",
6100 : __func__, linkRoce->qpsPerConnection, signalInfos[0].resId, linkRoce->singleQPNotifyNum, isBackup,
6101 : isSecondBuild, linkRoce->QpInfo[0].qpPtr, linkRoce->useAtomicWrite);
6102 0 : return HCCL_SUCCESS;
6103 0 : }
6104 :
6105 : template <typename T>
6106 0 : HcclResult HcclCommunicator::CreateListNode(T** resHostPtr, T** resDevicePtr)
6107 : {
6108 0 : hostMemVec_.resize(hostMemVec_.size() + 1);
6109 0 : CHK_RET(AllocAndClearHostMem(sizeof(T), hostMemVec_.back()));
6110 0 : *resHostPtr = static_cast<T*>(hostMemVec_.back().get()->ptr());
6111 :
6112 0 : deviceMemVec_.resize(deviceMemVec_.size() + 1);
6113 0 : CHK_RET(AllocAndClearDeviceMem(sizeof(T), deviceMemVec_.back()));
6114 :
6115 0 : *resDevicePtr = static_cast<T*>(deviceMemVec_.back().get()->ptr());
6116 : // 初始化HcclRankRelationResV2中的tagRes链表
6117 0 : ListCommonInit(&((*resDevicePtr)->nextTagRes), &((*resHostPtr)->nextTagRes));
6118 0 : return HCCL_SUCCESS;
6119 : }
6120 :
6121 0 : HcclResult HcclCommunicator::BuildRemoteResByTag(
6122 : const std::string& newTag, const u32& usrRankId, HcclRankRelationResV2*& rankRelationResHostPtr,
6123 : HcclRankRelationResV2*& rankRelationResDevicePtr, bool isBackup, bool isRetry)
6124 : {
6125 0 : HCCL_DEBUG(
6126 : "[%s]start to add RemoteRes with newtag[%s] and remoteRankId[%u] to list", __func__, newTag.c_str(), usrRankId);
6127 0 : if (rankTagRemoteRes_.find(usrRankId) == rankTagRemoteRes_.end()
6128 0 : || rankTagRemoteRes_[usrRankId].find(newTag) == rankTagRemoteRes_[usrRankId].end()) {
6129 0 : HccltagRemoteResV2* tagRemoteResHostPtr = nullptr;
6130 0 : HccltagRemoteResV2* tagRemoteResDevicePtr = nullptr;
6131 0 : CHK_RET(CreateListNode(&tagRemoteResHostPtr, &tagRemoteResDevicePtr));
6132 0 : CHK_SAFETY_FUNC_RET(
6133 : memcpy_s(tagRemoteResHostPtr->tag, sizeof(tagRemoteResHostPtr->tag), newTag.c_str(), newTag.length() + 1));
6134 0 : tagRemoteResHostPtr->linkP2p.localIpcSignal[0].resId = INVALID_U64;
6135 0 : tagRemoteResHostPtr->linkP2pSio.localIpcSignal[0].resId = INVALID_U64;
6136 0 : tagRemoteResHostPtr->linkRoce[0].localNotifyList = 0;
6137 0 : tagRemoteResHostPtr->linkRoce[1].localNotifyList = 0;
6138 0 : tagRemoteResHostPtr->linkRoce[2].localNotifyList = 0;
6139 0 : tagRemoteResHostPtr->linkRoce[3].localNotifyList = 0;
6140 0 : ListCommonAddHead(
6141 0 : &tagRemoteResDevicePtr->nextTagRes, &tagRemoteResHostPtr->nextTagRes, &rankRelationResHostPtr->nextTagRes,
6142 0 : &rankRelationResDevicePtr->nextTagRes);
6143 0 : HccltagRemoteResV3 tempTagRemoteRes;
6144 0 : tempTagRemoteRes.tagRemoteResPtr = tagRemoteResHostPtr;
6145 0 : rankTagRemoteRes_[usrRankId][newTag] = tempTagRemoteRes;
6146 0 : HCCL_RUN_INFO(
6147 : "[%s] successfully add RemoteRes to list with newtag[%s], remoteRankId[%u]"
6148 : "rankRelationResHostPtr head addr[%p], nextHost[%p], preHost[%p], nextDevice[%p], preDevice[%p], "
6149 : "tagRemoteResDevicePtr head addr[%p]",
6150 : __func__, newTag.c_str(), usrRankId, &rankRelationResHostPtr->nextTagRes,
6151 : rankRelationResHostPtr->nextTagRes.nextHost, rankRelationResHostPtr->nextTagRes.preHost,
6152 : rankRelationResHostPtr->nextTagRes.nextDevice, rankRelationResHostPtr->nextTagRes.preDevice,
6153 : &tagRemoteResDevicePtr->nextTagRes);
6154 : } else {
6155 0 : HCCL_DEBUG(
6156 : "[%s] the RemoteRes with usr rankid[%u] tag[%s] has been added list", __func__, usrRankId, newTag.c_str());
6157 : }
6158 0 : return HCCL_SUCCESS;
6159 : }
6160 :
6161 0 : HcclResult HcclCommunicator::BuildRelationResByRemoteRankId(
6162 : const TransportRequest& transportRequest, const LINK& link, HcclRankRelationResV2*& rankRelationResHostPtr,
6163 : HcclRankRelationResV2*& rankRelationResDevicePtr)
6164 : {
6165 0 : const u32 usrRankId = transportRequest.remoteUserRank;
6166 0 : HCCL_INFO("[%s]start to add RelationRes with remote usr rankid[%u] to list", __func__, usrRankId);
6167 0 : if (opResPara_.remoteRes[usrRankId].nextHostPtr != 0 && opResPara_.remoteRes[usrRankId].nextDevicePtr != 0) {
6168 0 : rankRelationResHostPtr = reinterpret_cast<HcclRankRelationResV2*>(opResPara_.remoteRes[usrRankId].nextHostPtr);
6169 : rankRelationResDevicePtr
6170 0 : = reinterpret_cast<HcclRankRelationResV2*>(opResPara_.remoteRes[usrRankId].nextDevicePtr);
6171 0 : HCCL_DEBUG(
6172 : "[%s] RelationRes with remote usr rankid[%u] has been added to list, "
6173 : "rankRelationResHostPtr[%p], rankRelationResDevicePtr[%p]",
6174 : __func__, usrRankId, rankRelationResHostPtr, rankRelationResDevicePtr);
6175 : } else {
6176 0 : CHK_RET(CreateListNode(&rankRelationResHostPtr, &rankRelationResDevicePtr));
6177 0 : opResPara_.remoteRes[usrRankId].nextHostPtr = reinterpret_cast<u64>(rankRelationResHostPtr);
6178 0 : opResPara_.remoteRes[usrRankId].nextDevicePtr = reinterpret_cast<u64>(rankRelationResDevicePtr);
6179 0 : rankRelationResHostPtr->remoteUsrRankId = usrRankId;
6180 0 : rankRelationResHostPtr->remoteWorldRank = rankInfoList_[usrRankId].worldRank;
6181 0 : HCCL_DEBUG(
6182 : "[%s]successfully add RelationRes with remote usr rankid[%u] to list, rankRelationResHostPtr[%p],"
6183 : "rankRelationResDevicePtr[%p]",
6184 : __func__, usrRankId, rankRelationResHostPtr, rankRelationResDevicePtr);
6185 : }
6186 : // 刷新远端对应的cclbuffer
6187 0 : std::vector<void*> extraMemVector;
6188 0 : if (transportRequest.inputMemType == TransportMemType::CCL_INPUT && rankRelationResHostPtr->windowsIn == 0) {
6189 0 : void* inbufferPtr = nullptr;
6190 0 : CHK_RET(link->GetRemoteMem(UserMemType::INPUT_MEM, &inbufferPtr));
6191 0 : rankRelationResHostPtr->windowsIn = reinterpret_cast<u64>(inbufferPtr);
6192 : }
6193 0 : if (transportRequest.outputMemType == TransportMemType::CCL_OUTPUT && rankRelationResHostPtr->windowsOut == 0) {
6194 0 : void* outbufferPtr = nullptr;
6195 0 : CHK_RET(link->GetRemoteMem(UserMemType::OUTPUT_MEM, &outbufferPtr));
6196 0 : rankRelationResHostPtr->windowsOut = reinterpret_cast<u64>(outbufferPtr);
6197 : }
6198 0 : if (rankRelationResHostPtr->windowsExp == 0) {
6199 0 : std::vector<void*> memPtrVec = {};
6200 0 : CHK_RET(link->GetRemoteMem(&memPtrVec));
6201 0 : if (memPtrVec.size() != 0) {
6202 0 : rankRelationResHostPtr->windowsExp = reinterpret_cast<u64>(memPtrVec[0]);
6203 0 : if (link->GetTransportType() == TransportType::TRANS_TYPE_P2P) {
6204 0 : p2pCclBuf_[usrRankId] = memPtrVec[0];
6205 : } else {
6206 0 : cclBuf_[usrRankId] = memPtrVec[0];
6207 : }
6208 0 : rankRelationResHostPtr->windowsExp
6209 0 : += cclBufferManager_.GetInCCLbufferSize() + cclBufferManager_.GetOutCCLbufferSize();
6210 : }
6211 0 : }
6212 0 : HCCL_INFO(
6213 : "group[%s] successfully set windowsIn & windowsOut & windowsExp info: userRank[%u], groupRank[%u], "
6214 : "remoteRank[%u], windowsIn[0x%llx], InSize[0x%llx], windowOut[0x%llx], OutSize[0x%llx], "
6215 : "windowExp[0x%llx], ExpSize[0x%llx]",
6216 : identifier_.c_str(), GetUserRank(), GetGroupRank(), transportRequest.remoteUserRank,
6217 : rankRelationResHostPtr->windowsIn, cclBufferManager_.GetInCCLbufferSize(), rankRelationResHostPtr->windowsOut,
6218 : cclBufferManager_.GetOutCCLbufferSize(), rankRelationResHostPtr->windowsExp,
6219 : cclBufferManager_.GetExpBufferSize());
6220 0 : return HCCL_SUCCESS;
6221 0 : }
6222 :
6223 0 : HcclResult HcclCommunicator::ParseRemoteDataToMem(
6224 : const OpCommTransport& opTransportResponse, const std::string& newTag, const HcclCMDType opType, bool isBackup,
6225 : bool isRetry)
6226 : {
6227 0 : HCCL_INFO("[%s] entry process newtag[%s], isBackup[%d]", __func__, newTag.c_str(), isBackup);
6228 0 : std::set<u32> bsrTansportRank;
6229 0 : for (auto& levelNSubCommTransport : opTransportResponse) {
6230 0 : for (auto& singleSubCommTransport : levelNSubCommTransport) {
6231 0 : u32 linkIdx = 0;
6232 0 : for (auto& transportRequest : singleSubCommTransport.transportRequests) {
6233 0 : if (transportRequest.isValid) {
6234 0 : auto tempLink = singleSubCommTransport.links[linkIdx];
6235 0 : HCCL_INFO(
6236 : "[%s]transportRequest.isUsedRdma[%d], isBackup[%d]", __func__, transportRequest.isUsedRdma,
6237 : isBackup);
6238 0 : if ((!transportRequest.isUsedRdma || tempLink->GetLinkType() == LinkType::LINK_SIO)
6239 0 : && (isBackup || isRetry)) {
6240 0 : HCCL_INFO(
6241 : "[%s]no need to add p2p backup Link resource, transportRequest.isUsedRdma[%d], "
6242 : "isBackup[%d]",
6243 : __func__, transportRequest.isUsedRdma, isBackup);
6244 0 : linkIdx++;
6245 0 : continue;
6246 : }
6247 0 : HcclRankRelationResV2* rankRelationResHostPtr = nullptr;
6248 0 : HcclRankRelationResV2* rankRelationResDevicePtr = nullptr;
6249 0 : CHK_RET(BuildRelationResByRemoteRankId(
6250 : transportRequest, tempLink, rankRelationResHostPtr, rankRelationResDevicePtr));
6251 0 : const u32 usrRankId = transportRequest.remoteUserRank;
6252 0 : HCCL_INFO(
6253 : "[%s]successfully BuildRelationResByRemoteRankId with remote usr rankid[%u], "
6254 : "rankRelationResHostPtr[%p], rankRelationResDevicePtr[%p], newTage[%s]",
6255 : __func__, usrRankId, rankRelationResHostPtr, rankRelationResDevicePtr, newTag.c_str());
6256 0 : CHK_RET(BuildRemoteResByTag(
6257 : newTag, usrRankId, rankRelationResHostPtr, rankRelationResDevicePtr, isBackup, isRetry));
6258 : // transport信息保存（notify、qp）
6259 0 : if (!transportRequest.isUsedRdma || tempLink->GetLinkType() == LinkType::LINK_SIO) {
6260 : // sdma -> P2P
6261 0 : CHK_RET(BuildOpRemoteLinkP2pResParam(
6262 : tempLink, rankTagRemoteRes_[usrRankId][newTag], transportRequest.linkType));
6263 : } else {
6264 : // rdma -> roce
6265 0 : bool isSecondBuild = false;
6266 0 : if (opType == HcclCMDType::HCCL_CMD_BATCH_SEND_RECV
6267 0 : && bsrTansportRank.find(transportRequest.remoteUserRank) != bsrTansportRank.end()) {
6268 0 : isSecondBuild = true;
6269 : }
6270 0 : bsrTansportRank.insert(transportRequest.remoteUserRank);
6271 0 : CHK_RET(BuildOpRemoteLinkRoceResParam(
6272 : tempLink, rankTagRemoteRes_[usrRankId][newTag], isBackup, isRetry, isSecondBuild));
6273 : }
6274 0 : HCCL_INFO(
6275 : "[%s] successfully add RemoteRes to list with newtag[%s] rankRelationResHostPtr "
6276 : "head addr[%p], nextHost[%p], preHost[%p], nextDevice[%p], preDevice[%p], "
6277 : "rankRelationResDevicePtr head addr[%p]",
6278 : __func__, newTag.c_str(), &rankRelationResHostPtr->nextTagRes,
6279 : rankRelationResHostPtr->nextTagRes.nextHost, rankRelationResHostPtr->nextTagRes.preHost,
6280 : rankRelationResHostPtr->nextTagRes.nextDevice, rankRelationResHostPtr->nextTagRes.preDevice,
6281 : &rankRelationResDevicePtr->nextTagRes);
6282 0 : HCCL_INFO(
6283 : "[%s] create link success with newtag[%s], linkIdx[%u], isBackup[%d], usrRankId[%u]", __func__,
6284 : newTag.c_str(), linkIdx, isBackup, usrRankId);
6285 0 : }
6286 0 : linkIdx++;
6287 : }
6288 : }
6289 : }
6290 0 : HCCL_DEBUG("[%s] process success newtag[%s]", __func__, newTag.c_str());
6291 0 : return HCCL_SUCCESS;
6292 0 : }
6293 :
6294 0 : HcclResult HcclCommunicator::BuildOpRemoteResParam(
6295 : const AlgResourceResponse& algResource, const std::string& newTag, const HcclCMDType opType, bool isRetry)
6296 : {
6297 0 : HCCL_DEBUG("[%s]start ParseRemoteDataToMem, IsEnableBackupLink[%d]", __func__, IsEnableBackupLink());
6298 0 : CHK_RET(ParseRemoteDataToMem(algResource.opTransportResponse, newTag, opType, false, isRetry));
6299 0 : if (IsEnableBackupLink()) {
6300 0 : HCCL_DEBUG("[%s]start Parse backupRemoteDataToMem, IsEnableBackupLink[%d]", __func__, IsEnableBackupLink());
6301 0 : CHK_RET(ParseRemoteDataToMem(algResource.opTransportResponseBackUp, newTag, opType, true, isRetry));
6302 : }
6303 0 : if (deviceType_ == DevType::DEV_TYPE_910_93 || deviceType_ == DevType::DEV_TYPE_910B) {
6304 0 : opResPara_.notifysize = 4; // 910B & 910_93 每个notify占4个字节
6305 : } else {
6306 0 : opResPara_.notifysize = 8; // 其他芯片类型每个notify占8个字节
6307 : }
6308 0 : return HCCL_SUCCESS;
6309 : }
6310 :
6311 5 : HcclResult HcclCommunicator::CopyHostListResToDeviceParam(
6312 : const std::string& newTag, const ListCommon* headHostList, const u64 size)
6313 : {
6314 5 : ListCommon* nextHostList = reinterpret_cast<ListCommon*>(headHostList->nextHost);
6315 5 : ListCommon* nextDeviceList = reinterpret_cast<ListCommon*>(headHostList->nextDevice);
6316 : // 该tag已分配过资源，只需刷新单节点（精确匹配）；否则为首次分配，批量拷贝前N个节点
6317 5 : bool isRefreshSingleNode = (newTagResAlloced_.find(newTag) != newTagResAlloced_.end());
6318 : // ListCommonAddHead把新节点头插，原头节点的preHost/preDevice被改写，两节点都需要刷新到device
6319 5 : constexpr uint32_t UPDATE_NODE_NUM = 2;
6320 5 : uint32_t updateNodeCnt = 0;
6321 :
6322 12 : while (nextHostList != headHostList && updateNodeCnt < UPDATE_NODE_NUM) {
6323 9 : HCCL_INFO(
6324 : "[HcclCommunicator][CopyHostListResToDeviceParam] remote resource, tag[%s], head Host List[%p], next "
6325 : "Host List[%p],next Device List[%p]",
6326 : newTag.c_str(), headHostList, nextHostList, nextDeviceList);
6327 9 : if (isRefreshSingleNode) {
6328 : // 刷新已有资源：遍历链表找到匹配的tag，只拷贝该节点，避免全量刷新
6329 : std::string curTag = (size == sizeof(HccltagLocalResV2)) ?
6330 7 : reinterpret_cast<HccltagLocalResV2*>(nextHostList)->tag :
6331 7 : reinterpret_cast<HccltagRemoteResV2*>(nextHostList)->tag;
6332 7 : if (curTag == newTag) {
6333 2 : CHK_RET(hrtMemSyncCopy(
6334 : reinterpret_cast<void*>(nextDeviceList), size, reinterpret_cast<void*>(nextHostList), size,
6335 : HcclRtMemcpyKind::HCCL_RT_MEMCPY_KIND_HOST_TO_DEVICE));
6336 2 : break;
6337 : }
6338 7 : } else {
6339 : // 首分配置：拷贝前UPDATE_NODE_NUM个节点到device，减少H2D次数
6340 2 : CHK_RET(hrtMemSyncCopy(
6341 : reinterpret_cast<void*>(nextDeviceList), size, reinterpret_cast<void*>(nextHostList), size,
6342 : HcclRtMemcpyKind::HCCL_RT_MEMCPY_KIND_HOST_TO_DEVICE));
6343 2 : updateNodeCnt++;
6344 : }
6345 7 : nextDeviceList = reinterpret_cast<ListCommon*>(nextHostList->nextDevice);
6346 7 : nextHostList = reinterpret_cast<ListCommon*>(nextHostList->nextHost);
6347 : }
6348 5 : return HCCL_SUCCESS;
6349 : }
6350 :
6351 0 : HcclResult HcclCommunicator::CopyHostOpResToDeviceParam(const std::string& newTag)
6352 : {
6353 : // 1、将opResPara_，H2D到device
6354 0 : CHK_RET(hrtMemSyncCopy(
6355 : opResDevicePara_.ptr(), sizeof(HcclOpResParam), reinterpret_cast<void*>(&opResPara_), sizeof(HcclOpResParam),
6356 : HcclRtMemcpyKind::HCCL_RT_MEMCPY_KIND_HOST_TO_DEVICE));
6357 0 : HCCL_DEBUG(
6358 : "[HcclCommunicator][CopyHostOpResToDeviceParam] tag[%s] local rankId[%u] workspace[%p] "
6359 : "workspacesize[%lu] ranksize[%u], cclbuffersize[%lu], cclinbuffer[%p], ccloutbuffer[%p], "
6360 : "remote winStart[%u], remote rWinOffset[%u], hostStateInfo[%p], aicpuStateInfo[%p], notifysize[%u], "
6361 : "sizeOfAiRMAInfo[%u],aiRMAInfo[%u]",
6362 : newTag.c_str(), userRank_, opResPara_.mc2WorkSpace.workSpace, opResPara_.mc2WorkSpace.workSpaceSize,
6363 : opResPara_.rankSize, opResPara_.winSize, opResPara_.localWindowsIn, opResPara_.localWindowsOut,
6364 : opResPara_.rWinStart, opResPara_.rWinOffset, opResPara_.hostStateInfo, opResPara_.aicpuStateInfo,
6365 : opResPara_.notifysize, opResPara_.sizeOfAiRMAInfo, opResPara_.aiRMAInfo);
6366 : // 2、将opResPara_中localres的tagRes，H2D到device
6367 0 : HCCL_DEBUG(
6368 : "[HcclCommunicator][CopyHostOpResToDeviceParam] local resource, tag[%s] streamNum[%u] signalNum[%u]",
6369 : newTag.c_str(), opResPara_.localRes.streamNum, opResPara_.localRes.signalNum);
6370 0 : CHK_RET(CopyHostListResToDeviceParam(
6371 : newTag, reinterpret_cast<ListCommon*>(&opResPara_.localRes.nextTagRes), sizeof(HccltagLocalResV2)));
6372 : // 3、遍历rank中tag资源，H2D到device
6373 0 : CHK_RET(CopyHostOpRemoteResToDeviceParam(newTag));
6374 0 : HCCL_DEBUG("[HcclCommunicator][CopyHostOpResToDeviceParam] copy host resource success!, tag[%s]", newTag.c_str());
6375 0 : return HCCL_SUCCESS;
6376 : }
6377 :
6378 0 : HcclResult HcclCommunicator::CopyHostAirmaInfoToDeviceParam(
6379 : const std::string& newTag, const HcclCMDType opType, const rtStream_t aiCpuStream)
6380 : {
6381 0 : HCCL_INFO("[HcclCommunicator][%s] Start prepare.", __func__);
6382 0 : CHK_PTR_NULL(aiRMAInfoMem_);
6383 0 : HcclAiRMAInfo* aiRMAInfoPtr = reinterpret_cast<HcclAiRMAInfo*>(aiRMAInfoMem_->ptr());
6384 0 : CHK_PTR_NULL(aiRMAInfoPtr);
6385 :
6386 0 : aiRMAInfoPtr->sizeOfAiRMAWQ = static_cast<u32>(sizeof(HcclAiRMAWQ));
6387 0 : aiRMAInfoPtr->sizeOfAiRMACQ = static_cast<u32>(sizeof(HcclAiRMACQ));
6388 0 : aiRMAInfoPtr->sizeOfAiRMAMem = static_cast<u32>(sizeof(HcclAiRMAMemInfo));
6389 :
6390 0 : CHK_RET(DeviceMem::alloc(aiSqDev_, aiSqMem_->size()));
6391 0 : aiRMAInfoPtr->sqPtr = aiSqDev_.ptr();
6392 0 : CHK_RET(hrtMemAsyncCopy(
6393 : aiSqDev_.ptr(), aiSqDev_.size(), aiSqMem_->ptr(), aiSqDev_.size(),
6394 : HcclRtMemcpyKind::HCCL_RT_MEMCPY_KIND_HOST_TO_DEVICE, aiCpuStream));
6395 :
6396 0 : CHK_RET(DeviceMem::alloc(aiScqDev_, aiScqMem_->size()));
6397 0 : aiRMAInfoPtr->scqPtr = aiScqDev_.ptr();
6398 0 : CHK_RET(hrtMemAsyncCopy(
6399 : aiScqDev_.ptr(), aiScqDev_.size(), aiScqMem_->ptr(), aiScqDev_.size(),
6400 : HcclRtMemcpyKind::HCCL_RT_MEMCPY_KIND_HOST_TO_DEVICE, aiCpuStream));
6401 :
6402 0 : CHK_RET(DeviceMem::alloc(aiRqDev_, aiRqMem_->size()));
6403 0 : aiRMAInfoPtr->rqPtr = aiRqDev_.ptr();
6404 0 : CHK_RET(hrtMemAsyncCopy(
6405 : aiRqDev_.ptr(), aiRqDev_.size(), aiRqMem_->ptr(), aiRqDev_.size(),
6406 : HcclRtMemcpyKind::HCCL_RT_MEMCPY_KIND_HOST_TO_DEVICE, aiCpuStream));
6407 :
6408 0 : CHK_RET(DeviceMem::alloc(aiRcqDev_, aiRcqMem_->size()));
6409 0 : aiRMAInfoPtr->rcqPtr = aiRcqDev_.ptr();
6410 0 : CHK_RET(hrtMemAsyncCopy(
6411 : aiRcqDev_.ptr(), aiRcqDev_.size(), aiRcqMem_->ptr(), aiRcqDev_.size(),
6412 : HcclRtMemcpyKind::HCCL_RT_MEMCPY_KIND_HOST_TO_DEVICE, aiCpuStream));
6413 :
6414 0 : CHK_RET(hrtMemAsyncCopy(
6415 : aiMemDetailsDev_.ptr(), aiMemDetailsDev_.size(), aiMemDetailsMem_->ptr(), aiMemDetailsDev_.size(),
6416 : HcclRtMemcpyKind::HCCL_RT_MEMCPY_KIND_HOST_TO_DEVICE, aiCpuStream));
6417 :
6418 0 : CHK_RET(DeviceMem::alloc(aiMemDev_, aiMemMem_->size()));
6419 0 : aiRMAInfoPtr->memPtr = aiMemDev_.ptr();
6420 0 : CHK_RET(hrtMemAsyncCopy(
6421 : aiMemDev_.ptr(), aiMemDev_.size(), aiMemMem_->ptr(), aiMemDev_.size(),
6422 : HcclRtMemcpyKind::HCCL_RT_MEMCPY_KIND_HOST_TO_DEVICE, aiCpuStream));
6423 :
6424 0 : opResPara_.sizeOfAiRMAInfo = static_cast<u64>(sizeof(HcclAiRMAInfo));
6425 0 : CHK_RET(DeviceMem::alloc(aiRMAInfoDev_, opResPara_.sizeOfAiRMAInfo));
6426 0 : opResPara_.aiRMAInfo = reinterpret_cast<u64>(aiRMAInfoDev_.ptr());
6427 :
6428 0 : CHK_RET(hrtMemAsyncCopy(
6429 : aiRMAInfoDev_.ptr(), aiRMAInfoDev_.size(), aiRMAInfoMem_->ptr(), aiRMAInfoDev_.size(),
6430 : HcclRtMemcpyKind::HCCL_RT_MEMCPY_KIND_HOST_TO_DEVICE, aiCpuStream));
6431 0 : HCCL_INFO(
6432 : "[%s] tag[%s] curRankId[%u] rankNum[%u] qpNum[%u] aiRMAInfo[%p] sizeOfAiRMAInfo[%llu] "
6433 : "sizeOfAiRMAWQ[%u] sizeOfAiRMACQ[%u] sizeOfAiRMAMem[%u] sqPtr[%p] sqSize[%llu] sqCount[%zu] "
6434 : "scqPtr[%p] scqSize[%llu] scqCount[%zu] rqPtr[%p] rqSize[%llu] rqCount[%zu] rcqPtr[%p] "
6435 : "rcqSize[%llu] rcqCount[%zu] memPtr[%p] memSize[%llu] memCount[%zu] memDetailCount[%zu],opResPara_.aiRMAInfo",
6436 : __func__, newTag.c_str(), aiRMAInfoPtr->curRankId, aiRMAInfoPtr->rankNum, aiRMAInfoPtr->qpNum,
6437 : opResPara_.aiRMAInfo, opResPara_.sizeOfAiRMAInfo, aiRMAInfoPtr->sizeOfAiRMAWQ, aiRMAInfoPtr->sizeOfAiRMACQ,
6438 : aiRMAInfoPtr->sizeOfAiRMAMem, aiRMAInfoPtr->sqPtr, aiSqDev_.size(), aiSqMem_->size(), aiRMAInfoPtr->scqPtr,
6439 : aiScqDev_.size(), aiScqMem_->size(), aiRMAInfoPtr->rqPtr, aiRqDev_.size(), aiRqMem_->size(),
6440 : aiRMAInfoPtr->rcqPtr, aiRcqDev_.size(), aiRcqMem_->size(), aiRMAInfoPtr->memPtr, aiMemDev_.size(),
6441 : aiMemMem_->size(), aiMemDetailsMem_->size());
6442 0 : return HCCL_SUCCESS;
6443 : }
6444 :
6445 0 : HcclResult HcclCommunicator::BuildOpResParam(
6446 : const std::string& algName, const AlgResourceResponse& algResource, const std::string& newTag,
6447 : const HcclCMDType opType, const rtStream_t aicpuStream)
6448 : {
6449 0 : opResPara_.localUsrRankId = userRank_;
6450 0 : opResPara_.rankSize = userRankSize_;
6451 :
6452 0 : bool isUseUserMem = isUserMemRegisted_ && !userMemMap_.empty();
6453 0 : if (!isUseUserMem) {
6454 0 : opResPara_.winSize = algResource.cclInputMem.size();
6455 0 : opResPara_.localWindowsIn = reinterpret_cast<u64>(algResource.cclInputMem.ptr());
6456 0 : opResPara_.localWindowsOut = reinterpret_cast<u64>(algResource.cclOutputMem.ptr());
6457 : } else {
6458 0 : opResPara_.winSize = userMemMap_.begin()->second->size();
6459 0 : opResPara_.localWindowsIn = reinterpret_cast<u64>(userMemMap_.begin()->second->ptr());
6460 0 : opResPara_.localWindowsOut = reinterpret_cast<u64>(userMemMap_.begin()->second->ptr());
6461 : }
6462 : // 填充Exp相关信息 当前该块内存大小恒为1M
6463 0 : opResPara_.winExpSize = EXP_BUFFER_SIZE;
6464 0 : opResPara_.localWindowsExp = reinterpret_cast<u64>(cclBufferManager_.GetCommExpBuffer().ptr());
6465 0 : HCCL_INFO(
6466 : "[HcclCommunicator][%s] isUseUserMem[%d], winSize[%llu], localWindowsIn[%llu],"
6467 : "localWindowsOut[%llu], localWindowsExp[%llu]",
6468 : __func__, isUseUserMem, opResPara_.winSize, opResPara_.localWindowsIn, opResPara_.localWindowsOut,
6469 : opResPara_.localWindowsExp);
6470 :
6471 0 : CHK_SAFETY_FUNC_RET(
6472 : memcpy_s(opResPara_.hcomId, sizeof(opResPara_.hcomId), identifier_.c_str(), identifier_.length() + 1));
6473 :
6474 0 : opResPara_.config.deterministic = GetDeterministicConfig();
6475 0 : opResPara_.config.highPerfEnable = 0;
6476 0 : aclrtFloatOverflowMode floatOverflowMode = ACL_RT_OVERFLOW_MODE_UNDEF;
6477 0 : CHK_RET(hrtGetDeviceSatMode(&floatOverflowMode));
6478 0 : opResPara_.config.floatOverflowMode = floatOverflowMode;
6479 0 : opResPara_.config.taskMonitorInterval = GetExternalInputDfsTaskMonitorInterval();
6480 0 : bool isSupportAtomicWrite = false; // 涉及到任务编排，当前不能只判断本机驱动版本是否支持
6481 0 : opResPara_.config.isSupportAtomicWrite = static_cast<u8>(isSupportAtomicWrite);
6482 : opResPara_.config.notifyWaitTime
6483 0 : = (GetExternalInputHcclExecTimeoutSet() != HcclExecTimeoutSet::HCCL_EXEC_TIMEOUT_NOT_SET
6484 0 : || commConfig_.GetConfigExecTimeOutSet()) ?
6485 0 : commConfig_.GetConfigExecTimeOut() :
6486 : NOTIFY_DEFAULT_WAIT_TIME;
6487 0 : opResPara_.config.linkTimeOut = std::chrono::seconds(GetExternalInputHcclLinkTimeOut());
6488 0 : opResPara_.config.retryEnable = static_cast<u8>(retryEnable_);
6489 0 : opResPara_.config.interHccsDisable = GetExternalInputInterHccsDisable();
6490 0 : opResPara_.config.multiQpThreshold = GetExternalInputMultiQpThreshold();
6491 0 : opResPara_.rWinStart = offsetof(HcclOpResParam, remoteRes);
6492 0 : opResPara_.rWinOffset = sizeof(RemoteResPtr);
6493 0 : opResPara_.notifysize = 0;
6494 0 : opResPara_.lockAddr = hostDeviceLock_->GetDevMemAddr();
6495 0 : opResPara_.utraceStatusFlag = GetExternalInputHcclEnableEntryLog();
6496 0 : DeviceMem tinySendRecvMem;
6497 0 : CHK_RET(implAlg_->GetTinyMem(tinySendRecvMem));
6498 0 : opResPara_.tinyMem = reinterpret_cast<u64>(tinySendRecvMem.ptr());
6499 0 : opResPara_.tinyMemSize = reinterpret_cast<u64>(tinySendRecvMem.size());
6500 0 : opResPara_.opEntry = GetExternalInputHcclEnableEntryLog();
6501 0 : opResPara_.hcclSdmaQos = GetHcclQos();
6502 :
6503 0 : CHK_RET(BuildOpLocalResParam(algResource, newTag));
6504 0 : CHK_RET(BuildOpRemoteResParam(algResource, newTag, opType));
6505 0 : CHK_RET(BuildOpTopoResParam(algName, algResource));
6506 0 : CHK_RET(BuildOpRetryParam(algResource, newTag));
6507 0 : CHK_RET(BuildZeroCopyParam());
6508 0 : CHK_RET(BuildAicpuCustomParam());
6509 0 : CHK_RET(BuildAicpuOrderLaunchNotify()); // 先申请device侧的关于按序下发的Notify内存
6510 0 : if (algName == "RunAlltoAllAivDirect") {
6511 : // AIV直驱ROCE
6512 0 : CHK_RET(BuildAiRmaInfoParam(newTag, algName, opType));
6513 0 : CHK_RET(CopyHostAirmaInfoToDeviceParam(newTag, opType, aicpuStream));
6514 : }
6515 0 : CHK_RET(CopyHostOpResToDeviceParam(newTag));
6516 0 : HCCL_RUN_INFO(
6517 : "[%s]build aicpu unfold resource success, tag[%s] rWinStart[%u] rWinOffset[%u] opEntry[%d]", __func__,
6518 : newTag.c_str(), opResPara_.rWinStart, opResPara_.rWinOffset, opResPara_.opEntry);
6519 0 : return HCCL_SUCCESS;
6520 0 : }
6521 :
6522 0 : HcclResult HcclCommunicator::BuildCustomOpResParam()
6523 : {
6524 : // custom进程需要刷新h2d/d2h内存
6525 0 : opResPara_.kfcControlTransferH2DParams = customControlTransferH2D_->GetCommunicateParams();
6526 0 : opResPara_.kfcStatusTransferD2HParams = customStatusTransferD2H_->GetCommunicateParams();
6527 0 : CHK_RET(hrtMemSyncCopy(
6528 : opResDevicePara_.ptr(), sizeof(HcclOpResParam), reinterpret_cast<void*>(&opResPara_), sizeof(HcclOpResParam),
6529 : HcclRtMemcpyKind::HCCL_RT_MEMCPY_KIND_HOST_TO_DEVICE));
6530 0 : return HCCL_SUCCESS;
6531 : }
6532 :
6533 24 : HcclResult HcclCommunicator::RegisterDfxInfo(
6534 : const OpParam& param, AlgType algType, const std::vector<Stream>& slaveStreams, bool isAiv, const std::string& tag)
6535 : {
6536 24 : u64 count = 0;
6537 24 : HcclDataType dataType = HcclDataType::HCCL_DATA_TYPE_RESERVED;
6538 24 : switch (param.opType) {
6539 0 : case HcclCMDType::HCCL_CMD_SEND:
6540 : case HcclCMDType::HCCL_CMD_RECEIVE:
6541 : case HcclCMDType::HCCL_CMD_BATCH_SEND_RECV:
6542 0 : count = param.GetDataCount(userRank_);
6543 0 : dataType = param.GetDataType();
6544 0 : HCCL_PROFILER_ADD_TAG_SENDRECV(param.tag, identifier_, GetWorkflowMode());
6545 0 : HCCL_PROFILER_ADD_GROUPRANK_SENDRECV(identifier_, userRankSize_, userRank_, param.dstRank);
6546 0 : break;
6547 5 : case HcclCMDType::HCCL_CMD_ALLTOALL:
6548 : case HcclCMDType::HCCL_CMD_ALLTOALLV:
6549 : case HcclCMDType::HCCL_CMD_ALLTOALLVC:
6550 5 : CHK_RET(AddGroupTagInfo(param.tag, isAiv));
6551 5 : count = param.All2AllDataDes.sendCount;
6552 5 : dataType = param.All2AllDataDes.sendType;
6553 5 : break;
6554 19 : default:
6555 19 : CHK_RET(AddGroupTagInfo(param.tag, isAiv));
6556 19 : count = param.GetDataCount(userRank_);
6557 19 : dataType = param.GetDataType();
6558 : }
6559 :
6560 24 : if (GetExternalInconsistentCheckSwitch() == InconsistentCheckMode::ON) {
6561 0 : if (param.opType != HcclCMDType::HCCL_CMD_BATCH_SEND_RECV) {
6562 0 : OpInfoDesc opInfo;
6563 0 : opInfo.opType = param.opType;
6564 0 : opInfo.dataType = dataType;
6565 0 : opInfo.reduceOp = param.reduceType;
6566 0 : opInfo.count = count;
6567 0 : opInfo.root = param.root;
6568 0 : opInfo.isValid = true;
6569 0 : AddOpInfoToHeartBeat(opInfo, tag);
6570 : }
6571 : }
6572 :
6573 : // task exception使用: 算子计数，算子入参信息(src/dst/datatype/reducetype)
6574 24 : HCCL_PROFILER_ADD_OPDATA_OP(
6575 : param.tag, count, param.inputPtr, param.outputPtr, dataType, param.root, identifier_, param.reduceType);
6576 : // 记录主流相关信息, 给profiling和task exception使用
6577 24 : HCCL_PROFILER_ADD_STREAM_BY_STREAMID(param.stream.id(), param.tag, 0, algType);
6578 24 : if (((GetWorkflowMode() == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE)
6579 14 : && !hccl::ProfilingManagerPub::GetAdditionInfoState() && !hccl::ProfilingManagerPub::GetTaskApiState())
6580 38 : && !param.isCapture) {
6581 11 : return HCCL_SUCCESS;
6582 : }
6583 : // 从流信息profiling开关打开的话再注册
6584 13 : for (u32 streamIndex = 0; streamIndex < slaveStreams.size(); streamIndex++) {
6585 0 : HCCL_PROFILER_ADD_STREAM_BY_STREAMID(slaveStreams[streamIndex].id(), param.tag, streamIndex + 1, algType);
6586 : }
6587 13 : return HCCL_SUCCESS;
6588 : }
6589 :
6590 0 : HcclResult HcclCommunicator::GetReportHcclMC2Info(const Stream& kfcStream, const std::vector<Stream>& aicpuStreams)
6591 : {
6592 0 : hcclMc2Info_.groupName = hrtMsprofGetHashId(identifier_.c_str(), identifier_.length());
6593 0 : hcclMc2Info_.rankSize = userRankSize_;
6594 0 : hcclMc2Info_.rankId = userRank_;
6595 0 : hcclMc2Info_.usrRankId = realUserRank_;
6596 0 : hcclMc2Info_.aicpuKfcStreamId = static_cast<uint32_t>(kfcStream.id());
6597 0 : hcclMc2Info_.reserve = 0;
6598 0 : const uint32_t ONCE_REPORT_STREAM_NUM_MAX = 8;
6599 0 : for (uint32_t streamIndex = 0, reportId = 0; streamIndex < aicpuStreams.size(); streamIndex++) {
6600 0 : HCCL_INFO("streamIndex:%u, reportId:%u, streamId:%d, ", streamIndex, reportId, aicpuStreams[streamIndex].id());
6601 0 : hcclMc2Info_.commStreamIds[reportId++] = aicpuStreams[streamIndex].id();
6602 0 : if (reportId == ONCE_REPORT_STREAM_NUM_MAX) {
6603 0 : hcclMc2Info_.commStreamSize = reportId;
6604 0 : CHK_RET(ProfilingManagerPub::CallMsprofReportMc2CommInfo(
6605 : hrtMsprofSysCycleTime(), &hcclMc2Info_, sizeof(hcclMc2Info_)));
6606 0 : reportId = 0;
6607 : }
6608 0 : if (streamIndex == (aicpuStreams.size() - 1)) {
6609 0 : HCCL_INFO("streamIndex:%u, reportId:%u, streamId:%d", streamIndex, reportId, opMainStream_.id());
6610 0 : hcclMc2Info_.commStreamIds[reportId++] = opMainStream_.id();
6611 0 : hcclMc2Info_.commStreamSize = reportId;
6612 0 : CHK_RET(ProfilingManagerPub::CallMsprofReportMc2CommInfo(
6613 : hrtMsprofSysCycleTime(), &hcclMc2Info_, sizeof(hcclMc2Info_)));
6614 0 : reportId = 0;
6615 : }
6616 : }
6617 0 : if (aicpuStreams.empty()) {
6618 0 : HCCL_INFO("only exist main stream, streamId:%d", opMainStream_.id());
6619 0 : hcclMc2Info_.commStreamIds[0] = opMainStream_.id();
6620 0 : hcclMc2Info_.commStreamSize = 1; // 只有主流1条
6621 0 : CHK_RET(ProfilingManagerPub::CallMsprofReportMc2CommInfo(
6622 : hrtMsprofSysCycleTime(), &hcclMc2Info_, sizeof(hcclMc2Info_)));
6623 : }
6624 0 : return HCCL_SUCCESS;
6625 : }
6626 :
6627 0 : HcclResult HcclCommunicator::OrchestrateAicpu(
6628 : const HcclCMDType& opType, const std::string& algName, const OpParam& param, const AlgResourceResponse& algResource,
6629 : const std::string& newTag, AlgType algType, bool isCustom, bool needIncreLink, bool needRecreateAlltoallComm)
6630 : {
6631 0 : uint64_t streamMode = 0;
6632 0 : CHK_RET(hrtStreamGetMode(param.stream.ptr(), &streamMode));
6633 : rtStream_t aicpuStream;
6634 0 : Mc2AiCpuStreamAllocAndGet(streamMode, aicpuStream); // aicpuStream需要在首次下发时申请
6635 0 : if (!isContextLaunched_) {
6636 : // 1、通信域内首次下发，从algResource中获取资源，H2D刷新资源，launch init
6637 : rtStream_t aicpuInitStream;
6638 0 : Mc2AiCpuInitStreamAllocAndGet(streamMode, aicpuInitStream); // 使用aicpuInitStream_下初始化kernel
6639 0 : Stream tmpStream(aicpuInitStream);
6640 0 : HCCL_DEBUG("%s ContextLaunched, aicpuInitStream:%p, aicpuStream:%p", __func__, aicpuInitStream, aicpuStream);
6641 0 : CHK_RET(AicpuResourceInit(algName, algResource, newTag, aicpuInitStream, opType, isCustom));
6642 0 : CHK_RET(GetReportHcclMC2Info(tmpStream, algResource.slaveDevStreams));
6643 0 : CHK_RET(SetAicpuUnfoldFlag());
6644 0 : } else if (
6645 0 : newTagResAlloced_.find(newTag) == newTagResAlloced_.end()
6646 0 : || (opType == HcclCMDType::HCCL_CMD_BATCH_SEND_RECV && needIncreLink) || needRecreateAlltoallComm) {
6647 : // 2、通信域内非首次，但是有新的newTag，查看是否需要补充资源。
6648 0 : PetersonLockGuard guard(hostDeviceLock_.get());
6649 0 : CHK_PRT_RET(
6650 : guard.IsLockFailed(), HCCL_ERROR("[HcclCommunicator][OrchestrateAicp] hostDeviceLock lock failed"),
6651 : HCCL_E_INTERNAL);
6652 0 : CHK_RET(AicpuResourceRefresh(algResource, newTag, opType));
6653 0 : }
6654 0 : HCCL_DEBUG(
6655 : "%s isContextLaunched[%u], needRecreateAlltoallComm[%u]", __func__, isContextLaunched_,
6656 : needRecreateAlltoallComm);
6657 0 : bool isUsedMainStream = (GetWorkflowMode() == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OPS_KERNEL_INFO_LIB);
6658 : // inplace支持重执行的stream资源处理逻辑
6659 0 : bool isHcclOpInplace = IsHcclOpInplace(opType, param, userRank_, userRankSize_, isInplaceStatus_);
6660 0 : if ((retryOrigWorkflowMode_ == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OPS_KERNEL_INFO_LIB) && retryEnable_
6661 0 : && isHcclOpInplace
6662 0 : && (opType == HcclCMDType::HCCL_CMD_ALLREDUCE || opType == HcclCMDType::HCCL_CMD_REDUCE_SCATTER)) {
6663 0 : isUsedMainStream = true;
6664 : }
6665 0 : AicpuOpTiling opTilingInfo;
6666 0 : opTilingInfo.algName = algName;
6667 0 : opTilingInfo.newTag = newTag;
6668 0 : opTilingInfo.algType = algType;
6669 0 : opTilingInfo.isUsedMainStream = isUsedMainStream;
6670 0 : opTilingInfo.dumpDebug = GetExternalInputHcclDumpDebug();
6671 0 : aclrtFloatOverflowMode floatOverflowMode = ACL_RT_OVERFLOW_MODE_UNDEF;
6672 0 : CHK_RET(hrtGetDeviceSatMode(&floatOverflowMode));
6673 0 : opTilingInfo.floatOverflowMode = floatOverflowMode;
6674 0 : HcclResult ret = HCCL_SUCCESS;
6675 : // 根据算子类型，获取 Aicpu Kernel 名称
6676 0 : auto iter = HCOM_CMD_TYPE_STR_MAP.find(opType);
6677 0 : CHK_PRT_RET(
6678 : (iter == HCOM_CMD_TYPE_STR_MAP.end()),
6679 : HCCL_ERROR("[%s] RunAicpuRpcSrvLaunchV2 kernel not found, opType=[%d]", __func__, static_cast<int>(opType)),
6680 : HCCL_E_INTERNAL);
6681 0 : std::string kernelName = std::string("RunAicpuRpcSrvLaunchV2") + "_" + iter->second;
6682 0 : ret = AicpuKfcTilingDataLaunchExt(param, opType, opResDevicePara_, kernelName, opTilingInfo, isCustom);
6683 0 : if (ret != HCCL_SUCCESS) {
6684 0 : HCCL_ERROR(
6685 : "[HcclCommunicator][OrchestrateAicpu]aicpu unfold launch kernel[%s] failed. ret[%d] inputPtr[%p]"
6686 : "outputPtr[%p] count[%llu] dataType[%s] op[%s]",
6687 : kernelName.c_str(), ret, param.inputPtr, param.outputPtr, param.DataDes.count,
6688 : GetDataTypeEnumStr(param.DataDes.dataType).c_str(), GetReduceOpEnumStr(param.reduceType).c_str());
6689 0 : return ret;
6690 : }
6691 0 : return HCCL_SUCCESS;
6692 0 : }
6693 :
6694 5 : HcclResult HcclCommunicator::CalcTinySendRecvMem(
6695 : const OpParam& opParam, AlgResourceResponse& algResResponse, DeviceMem& tinySendRecvMem)
6696 : {
6697 5 : u64 sendCount = 0;
6698 5 : u64 recvCount = 0;
6699 5 : if (opParam.opType == HcclCMDType::HCCL_CMD_ALLTOALLV) {
6700 12 : for (u32 i = 0; i < userRankSize_; i++) {
6701 8 : u64 curSendCount = *(static_cast<const u64*>(opParam.All2AllDataDes.sendCounts) + i)
6702 8 : + *(static_cast<const u64*>(opParam.All2AllDataDes.sdispls) + i);
6703 8 : sendCount = std::max(sendCount, curSendCount);
6704 8 : u64 curRecvCount = *(static_cast<const u64*>(opParam.All2AllDataDes.recvCounts) + i)
6705 8 : + *(static_cast<const u64*>(opParam.All2AllDataDes.rdispls) + i);
6706 8 : recvCount = std::max(recvCount, curRecvCount);
6707 : }
6708 : } else {
6709 3 : for (u32 i = 0; i < userRankSize_; i++) {
6710 : sendCount
6711 2 : += *(static_cast<const u64*>(opParam.All2AllDataDes.sendCountMatrix) + userRank_ * userRankSize_ + i);
6712 : recvCount
6713 2 : += *(static_cast<const u64*>(opParam.All2AllDataDes.sendCountMatrix) + userRank_ + userRankSize_ * i);
6714 : }
6715 : }
6716 :
6717 5 : u32 sendTypeSize = 0, recvTypeSize = 0;
6718 5 : CHK_RET(SalGetDataTypeSize(opParam.All2AllDataDes.sendType, sendTypeSize));
6719 5 : CHK_RET(SalGetDataTypeSize(opParam.All2AllDataDes.recvType, recvTypeSize));
6720 :
6721 : // 在sendCount/recvCount全0时, 使用tinySendRecvMem, 避免使用空deviceMem
6722 10 : algResResponse.paramInputMem = sendCount == 0 ? DeviceMem::create(tinySendRecvMem.ptr(), tinySendRecvMem.size()) :
6723 5 : DeviceMem::create(opParam.inputPtr, sendCount * sendTypeSize);
6724 10 : algResResponse.paramOutputMem = recvCount == 0 ? DeviceMem::create(tinySendRecvMem.ptr(), tinySendRecvMem.size()) :
6725 5 : DeviceMem::create(opParam.outputPtr, recvCount * recvTypeSize);
6726 :
6727 5 : HCCL_INFO(
6728 : "[HcclCommunicator][CalcTinySendRecvMem] senMem addr[%p], sendSize[%llu], "
6729 : "RecvMem addr[%p], RecvSize[%llu],",
6730 : algResResponse.paramInputMem.ptr(), algResResponse.paramInputMem.size(), algResResponse.paramOutputMem.ptr(),
6731 : algResResponse.paramOutputMem.size());
6732 5 : return HCCL_SUCCESS;
6733 : }
6734 :
6735 3 : bool HcclCommunicator::HasRoceTransportLinks(OpCommTransport& opTransportReq)
6736 : {
6737 20 : for (u32 levelIndex = 0; levelIndex < opTransportReq.size(); levelIndex++) {
6738 20 : for (u32 ringIndex = 0; ringIndex < opTransportReq[levelIndex].size(); ringIndex++) {
6739 3 : SingleSubCommTransport& reqSingleSubComm = opTransportReq[levelIndex][ringIndex];
6740 5 : for (u32 rankIndex = 0; rankIndex < reqSingleSubComm.transportRequests.size(); rankIndex++) {
6741 3 : TransportRequest& transportRequest = reqSingleSubComm.transportRequests[rankIndex];
6742 3 : if (transportRequest.isUsedRdma) {
6743 1 : return true;
6744 : }
6745 : }
6746 : }
6747 : }
6748 2 : return false;
6749 : }
6750 :
6751 0 : HcclResult HcclCommunicator::CleanTransportLinks(OpCommTransport& opTransportReq, OpCommTransport& opTransportResponse)
6752 : {
6753 0 : for (u32 levelIndex = 0; levelIndex < opTransportReq.size(); levelIndex++) {
6754 0 : for (u32 ringIndex = 0; ringIndex < opTransportReq[levelIndex].size(); ringIndex++) {
6755 0 : SingleSubCommTransport& reqSingleSubComm = opTransportReq[levelIndex][ringIndex];
6756 0 : SingleSubCommTransport& respSingleSubComm = opTransportResponse[levelIndex][ringIndex];
6757 0 : for (u32 rankIndex = 0; rankIndex < reqSingleSubComm.transportRequests.size(); rankIndex++) {
6758 0 : TransportRequest& transportRequest = reqSingleSubComm.transportRequests[rankIndex];
6759 0 : CHK_PRT_RET(
6760 : rankIndex >= respSingleSubComm.links.size(),
6761 : HCCL_ERROR(
6762 : "[CleanTransportLinks] The remote rank_id[%u] is larger than the existent respSingleSubComm "
6763 : "map "
6764 : "size[%u]",
6765 : rankIndex, respSingleSubComm.links.size()),
6766 : HCCL_E_PARA);
6767 0 : if (respSingleSubComm.links[rankIndex] != nullptr
6768 0 : && respSingleSubComm.links[rankIndex]->GetLinkType() != hccl::LinkType::LINK_RESERVED
6769 0 : && !transportRequest.isUsedRdma) {
6770 0 : HCCL_INFO(
6771 : "[CleanTransportLinks] The link to remote userRank[%u] has existed",
6772 : transportRequest.remoteUserRank);
6773 0 : continue;
6774 : }
6775 0 : respSingleSubComm.links[rankIndex] = nullptr;
6776 : }
6777 : }
6778 : }
6779 0 : return HCCL_SUCCESS;
6780 : }
6781 :
6782 116 : HcclResult HcclCommunicator::AllocAlgNotifys(
6783 : const std::string& tag, const NotifyLoadType notifyLoadType, const u32 notifyNum,
6784 : std::vector<std::shared_ptr<LocalNotify>>& notifiesMain, std::vector<std::shared_ptr<LocalNotify>>& notifiesAux)
6785 : {
6786 116 : std::vector<std::shared_ptr<LocalNotify>> notifys(notifyNum, nullptr);
6787 112 : CHK_RET(queueNotifyManagerRefac_->Alloc(tag, notifyNum, notifys, notifyLoadType));
6788 :
6789 116 : u32 signalNum = notifyNum >> 1;
6790 116 : notifiesMain.resize(signalNum);
6791 115 : notifiesAux.resize(signalNum);
6792 372 : for (u32 i = 0; i < signalNum; i++) {
6793 256 : notifiesMain[i] = notifys[i << 1];
6794 257 : notifiesAux[i] = notifys[(i << 1) + 1];
6795 : }
6796 116 : return HCCL_SUCCESS;
6797 116 : }
6798 :
6799 110 : HcclResult HcclCommunicator::AllocAlgResource(
6800 : const std::string& newTag, HcclCMDType opType, const OpParam& opParam, AlgResourceRequest& resRequest,
6801 : AlgResourceResponse& algResResponse, bool selectAivAlg)
6802 : {
6803 110 : HcclResult ret = HCCL_SUCCESS;
6804 110 : bool isGraphZeroCopyAlgAlloc = false;
6805 : // 只有aicpu模式下才需要申请从流和相关的notify资源，isNeedSlaveStream为true就代表算子下发是aicpu模式
6806 110 : bool isNeedSlaveStream = !selectAivAlg && opParam.aicpuUnfoldMode;
6807 110 : if (GetWorkflowMode() == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OPS_KERNEL_INFO_LIB
6808 110 : && !IsForceAicpuOpBaseMode(opParam, opType)) {
6809 59 : isGraphZeroCopyAlgAlloc = resRequest.isInGraphCaptureZeroCopy;
6810 59 : if (isGraphZeroCopyAlgAlloc) {
6811 0 : if (resRequest.scratchMemSize > 0) {
6812 : algResResponse.scratchMem
6813 0 : = DeviceMem::create(cclBufferManager_.GetOutCCLbuffer().ptr(), resRequest.scratchMemSize);
6814 : }
6815 59 : } else if (resRequest.scratchMemSize > 0) {
6816 24 : algResResponse.scratchMem = GetWorkspaceScracthMem(opParam.tag, resRequest.scratchMemSize);
6817 : }
6818 :
6819 59 : if (resRequest.streamNum > 0) {
6820 18 : if (isGraphZeroCopyAlgAlloc) {
6821 0 : CHK_RET(opStreamManager_->RegisterMaster(opParam.stream));
6822 : algResResponse.slaveStreams
6823 0 : = opStreamManager_->AllocSlaves(StreamType::STREAM_TYPE_ONLINE, resRequest.streamNum);
6824 0 : CHK_PRT_RET(
6825 : algResResponse.slaveStreams.empty(),
6826 : HCCL_ERROR(
6827 : "[AllocAlgResource]tag[%s] get slave stream failed, "
6828 : "expect to get size [%u], but only alloc 0.",
6829 : newTag.c_str(), resRequest.streamNum),
6830 : HCCL_E_INTERNAL);
6831 : } else {
6832 : // aicpu模式下不申请host从流
6833 18 : if (!opParam.aicpuUnfoldMode) {
6834 16 : algResResponse.slaveStreams = GetWorkspaceSubStreams(opParam.tag, resRequest.streamNum);
6835 : }
6836 : }
6837 : }
6838 51 : } else if (
6839 51 : GetWorkflowMode() == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE || IsForceAicpuOpBaseMode(opParam, opType)) {
6840 51 : CHK_RET(AllocOpBaseModeScratchMem(opType, opParam, resRequest, algResResponse));
6841 51 : if ((resRequest.streamNum > 0) && !selectAivAlg) {
6842 34 : CHK_RET(opStreamManager_->RegisterMaster(opParam.stream));
6843 : algResResponse.slaveStreams
6844 33 : = opStreamManager_->AllocSlaves(StreamType::STREAM_TYPE_ONLINE, resRequest.streamNum);
6845 36 : CHK_PRT_RET(
6846 : algResResponse.slaveStreams.empty(),
6847 : HCCL_ERROR(
6848 : "[AllocAlgResource]tag[%s] get slave stream failed, "
6849 : "expect to get size [%u], but only alloc 0.",
6850 : newTag.c_str(), resRequest.streamNum),
6851 : HCCL_E_INTERNAL);
6852 : }
6853 : } else {
6854 0 : HCCL_ERROR("[AllocAlgResource]WorkflowMode is not set.");
6855 0 : return HCCL_E_PARA;
6856 : }
6857 :
6858 108 : if (isNeedSlaveStream && ((userRankSize_ != 1) || IsForceAicpuOpBaseMode(opParam, opType))) {
6859 2 : CHK_RET(opStreamManager_->RegisterMaster(opParam.stream));
6860 : algResResponse.slaveDevStreams
6861 4 : = opStreamManager_->AllocSlaves(StreamType::STREAM_TYPE_DEVICE, LOCAL_STREAM_MAX_NUM);
6862 4 : CHK_PRT_RET(
6863 : algResResponse.slaveDevStreams.empty(),
6864 : HCCL_ERROR(
6865 : "[AllocAlgResource]tag[%s] get slave device stream failed, "
6866 : "expect to get size [%u], but only alloc 0.",
6867 : newTag.c_str(), LOCAL_STREAM_MAX_NUM),
6868 : HCCL_E_INTERNAL);
6869 4 : CHK_RET(AllocAlgNotifys(
6870 : opParam.tag, NotifyLoadType::DEVICE_NOTIFY, LOCAL_NOTIFY_MAX_NUM, algResResponse.notifiesDevMain,
6871 : algResResponse.notifiesDevAux));
6872 : }
6873 110 : uint8_t devNotifyNum = algResResponse.notifiesDevMain.size() + algResResponse.notifiesDevAux.size();
6874 110 : HCCL_INFO(
6875 : "[AllocAlgResource] tag[%s] alloc host slaveStreamNum[%u],"
6876 : "device slaveStreamNum[%u], devNotifyNum[%u], hostNotifyNum[%u]",
6877 : newTag.c_str(), algResResponse.slaveStreams.size(), algResResponse.slaveDevStreams.size(), devNotifyNum,
6878 : resRequest.notifyNum);
6879 112 : CHK_RET(AllocAlgNotifys(
6880 : opParam.tag, NotifyLoadType::HOST_NOTIFY, resRequest.notifyNum, algResResponse.notifiesMain,
6881 : algResResponse.notifiesAux));
6882 :
6883 112 : algResResponse.cclInputMem = cclBufferManager_.GetInCCLbuffer();
6884 112 : algResResponse.cclOutputMem = cclBufferManager_.GetOutCCLbuffer();
6885 111 : DeviceMem expMem = cclBufferManager_.GetCommCCLBuffer(); // 获取拓展内存
6886 111 : if (opParam.opType == HcclCMDType::HCCL_CMD_ALLTOALLV || opParam.opType == HcclCMDType::HCCL_CMD_ALLTOALLVC
6887 107 : || opParam.opType == HcclCMDType::HCCL_CMD_ALLTOALL) {
6888 4 : DeviceMem tinySendRecvMem;
6889 4 : CHK_RET(implAlg_->GetTinyMem(tinySendRecvMem));
6890 4 : CHK_RET(CalcTinySendRecvMem(opParam, algResResponse, tinySendRecvMem));
6891 8 : } else {
6892 107 : algResResponse.paramInputMem = DeviceMem::create(opParam.inputPtr, opParam.inputSize);
6893 106 : algResResponse.paramOutputMem = DeviceMem::create(opParam.outputPtr, opParam.outputSize);
6894 : }
6895 :
6896 109 : bool useOpbaseFlag = (GetWorkflowMode() == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE && !opParam.isCapture);
6897 109 : if (AIV_COMM_BUFFER_BITMASK & resRequest.aivBufferRequest) {
6898 4 : ret = cclBufferManager_.CreateCommAIVbuffer(useOpbaseFlag);
6899 4 : CHK_PRT_RET(ret != HCCL_SUCCESS, HCCL_ERROR("[Alloc][AlgResource]Create CommAIVbuffer failed"), ret);
6900 4 : if (useOpbaseFlag) { // 单算子非Capture模式，对应aivOpbaseTag_
6901 2 : algResResponse.aivInputMem = cclBufferManager_.GetInAivOpbaseBuffer();
6902 2 : algResResponse.aivOutputMem = cclBufferManager_.GetOutAivOpbaseBuffer();
6903 : } else { // 静态图或者Capture模式，对应aivOffloadTag_
6904 2 : algResResponse.aivInputMem = cclBufferManager_.GetInAivOffloadbuffer();
6905 2 : algResResponse.aivOutputMem = cclBufferManager_.GetOutAivOffloadbuffer();
6906 : }
6907 4 : HCCL_INFO("[AllocAlgResource] tag[%s] alloc aiv buffer", newTag.c_str());
6908 : }
6909 109 : if ((AIV_COMM_INFO_BUFFER_BITMASK & resRequest.aivBufferRequest) || opParam.isNpuDirectRoce) {
6910 0 : if (!useOpbaseFlag) {
6911 0 : DeviceMem aivCommInfoMem; // 图模式每个算子单独一块内存
6912 0 : CHK_RET(DeviceMem::alloc(aivCommInfoMem, AIV_COMM_INFO_SIZE));
6913 0 : algResResponse.aivCommInfoMem = aivCommInfoMem;
6914 0 : aivOffloadCommInfoMem_.emplace_back(std::move(aivCommInfoMem));
6915 0 : } else {
6916 0 : ret = cclBufferManager_.CreateCommInfoAIVbuffer();
6917 0 : CHK_PRT_RET(ret != HCCL_SUCCESS, HCCL_ERROR("[Alloc][AlgResource]Create CommInfoAIVbuffer failed"), ret);
6918 0 : algResResponse.aivCommInfoMem = cclBufferManager_.GetAivCommInfoBuffer(); // 单算子每个通信域只用一块内存
6919 : }
6920 0 : HCCL_INFO("[AllocAlgResource] tag[%s] alloc aiv comm info buffer", newTag.c_str());
6921 : }
6922 :
6923 : TransportIOMem transMem{
6924 109 : algResResponse.cclInputMem,
6925 109 : algResResponse.cclOutputMem,
6926 109 : algResResponse.paramInputMem,
6927 109 : algResResponse.paramOutputMem,
6928 109 : algResResponse.scratchMem,
6929 109 : algResResponse.aivInputMem,
6930 109 : algResResponse.aivOutputMem,
6931 : expMem,
6932 109 : DeviceMem()};
6933 109 : HCCL_DEBUG(
6934 : "algResResponse.cclInputMem[%p], size[%llu]; algResResponse.cclOutputMem[%p], "
6935 : "size[%llu]; algResResponse.paramInputMem[%p], size[%llu]; algResResponse.paramOutputMem[%p], size[%llu].",
6936 : algResResponse.cclInputMem.ptr(), algResResponse.cclInputMem.size(), algResResponse.cclOutputMem.ptr(),
6937 : algResResponse.cclOutputMem.size(), algResResponse.paramInputMem.ptr(), algResResponse.paramInputMem.size(),
6938 : algResResponse.paramOutputMem.ptr(), algResResponse.paramOutputMem.size());
6939 112 : algResResponse.opTransportResponse = resRequest.opTransport;
6940 :
6941 : // 零拷贝场景这里只借助P2p的openIpc能力交换控制面zeroCopyLocalBuffer_，不交换实际用户的输出输出
6942 112 : if (opParam.isZeroCopy) {
6943 0 : HCCL_INFO(
6944 : "[AllocAlgResource] zero copy change paramInput[%p] paramOutput[%p] scratchMem[%p] to localBuffer[%p]",
6945 : transMem.paramInputMem.ptr(), transMem.paramOutputMem.ptr(), transMem.scratchMem.ptr(),
6946 : zeroCopyLocalBuffer_.ptr());
6947 0 : transMem.scratchMem = zeroCopyLocalBuffer_;
6948 0 : transMem.paramInputMem = zeroCopyLocalBuffer_;
6949 0 : transMem.paramOutputMem = zeroCopyLocalBuffer_;
6950 : } else {
6951 112 : if (isGraphZeroCopyAlgAlloc) {
6952 : transMem.scratchMem
6953 0 : = DeviceMem::create(cclBufferManager_.GetOutCCLbuffer().ptr(), resRequest.scratchMemSize);
6954 0 : HCCL_INFO("[AllocAlgResource] acl graph set transMem.scratchMem =%ul", transMem.scratchMem.size());
6955 : }
6956 : }
6957 :
6958 112 : ClearOpTransportResponseLinks(algResResponse.opTransportResponse);
6959 110 : if (IsEnableBackupLink()) {
6960 0 : algResResponse.opTransportResponseBackUp = resRequest.opTransport;
6961 0 : ClearOpTransportResponseLinks(algResResponse.opTransportResponseBackUp);
6962 0 : HCCL_DEBUG(
6963 : "[%s]IsEnableBackupLink[%d] init backup & default opTransportResponse", __func__, IsEnableBackupLink());
6964 : }
6965 :
6966 110 : if (!GetExternalInputHcclEnableFfts() && GetWorkflowMode() == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE) {
6967 43 : u32 slaveNum = algResResponse.slaveStreams.size();
6968 43 : algResResponse.threadManage.resize(slaveNum);
6969 113 : for (u32 ringIndex = 0; ringIndex < slaveNum; ringIndex++) {
6970 140 : algResResponse.threadManage[ringIndex].reset(new (std::nothrow)
6971 70 : ThreadManage(deviceLogicId_, userRank_, dispatcher_));
6972 70 : CHK_SMART_PTR_NULL(algResResponse.threadManage[ringIndex]);
6973 70 : HcclResult ret = algResResponse.threadManage[ringIndex]->Init();
6974 70 : CHK_PRT_RET(
6975 : ret != HCCL_SUCCESS,
6976 : HCCL_ERROR("[Init][MultiRingResource]ringIndex[%u] ThreadManage failed,return[%d]", ringIndex, ret),
6977 : ret);
6978 70 : HCCL_INFO("ringThreadsManage Init success[%u]", ringIndex);
6979 : }
6980 : }
6981 110 : transportManager_->SetOpType(opParam.opType);
6982 112 : if (isUserMemRegisted_) {
6983 : // user win模式，用exchange接口建链的transport
6984 0 : algResResponse.opTransportResponse = userMemTransport_;
6985 0 : CHK_RET(GetRemoteUserMemResource());
6986 : } else {
6987 112 : StateGuard<HcclCommunicator, HcclCommState> guard(this, HcclCommState::BUILDING);
6988 112 : ret = transportManager_->Alloc(
6989 112 : opParam.tag, transMem, algResResponse.opTransportResponse, opParam.aicpuUnfoldMode, false,
6990 112 : opParam.isZeroCopy, opParam.opType, opParam.isCapture, false, opParam.isNpuDirectRoce, &opParam);
6991 112 : CHK_PRT_RET(
6992 : ret != HCCL_SUCCESS, HCCL_ERROR("[%s]Alloc transports failed, tag[%s]", __func__, newTag.c_str()), ret);
6993 112 : }
6994 59 : CHK_PRT_RET(ret != HCCL_SUCCESS, HCCL_ERROR("[%s]Alloc transports failed, tag[%s]", __func__, newTag.c_str()), ret);
6995 :
6996 59 : if (retryEnable_) {
6997 : // 获取当前rdma相连的所有对端rankList
6998 0 : std::vector<u32> rankList;
6999 0 : CHK_RET(transportManager_->GetRemoteRankList(
7000 : algResResponse.opTransportResponse, rankList, TransportType::TRANS_TYPE_IBV_EXP));
7001 0 : std::string rankListStr = "";
7002 0 : for (auto remoteRank : rankList) {
7003 0 : rankListStr += (std::to_string(remoteRank) + ";");
7004 : }
7005 0 : HCCL_DEBUG("identifier[%s] newTag[%s] rankList[%s]", identifier_.c_str(), newTag.c_str(), rankListStr.c_str());
7006 0 : CHK_RET(OpRetryManager::AddLinkInfoByIdentifier(deviceLogicId_, identifier_, newTag, rankList));
7007 0 : }
7008 :
7009 59 : if (IsEnableBackupLink()) {
7010 : // 超节点 && level2支持重执行 && Aicpu：创建备用Transport资源
7011 0 : StateGuard<HcclCommunicator, HcclCommState> guard(this, HcclCommState::BUILDING);
7012 0 : ret = transportManager_->Alloc(
7013 0 : opParam.tag, transMem, algResResponse.opTransportResponseBackUp, opParam.aicpuUnfoldMode, true,
7014 0 : opParam.isCapture);
7015 0 : CHK_PRT_RET(
7016 : ret != HCCL_SUCCESS, HCCL_ERROR("[%s]Alloc backup transports failed, tag[%s]", __func__, newTag.c_str()),
7017 : ret);
7018 0 : }
7019 59 : SaveLinkRes(algResResponse.opTransportResponse);
7020 59 : SaveLinkRes(algResResponse.opTransportResponseBackUp);
7021 59 : remoteTransportMap_ = transportManager_->GetRemoteTransportMap();
7022 59 : HCCL_DEBUG("[%s] process success newtag[%s]", __func__, newTag.c_str());
7023 59 : return HCCL_SUCCESS;
7024 112 : }
7025 :
7026 0 : HcclResult HcclCommunicator::GetRemoteUserMemResource()
7027 : {
7028 0 : for (auto& levelNSubCommTransport : userMemTransport_) {
7029 0 : for (auto& singleSubCommTransport : levelNSubCommTransport) {
7030 0 : u32 linkIdx = 0;
7031 0 : for (auto& transportRequest : singleSubCommTransport.transportRequests) {
7032 0 : if (!transportRequest.isValid) {
7033 0 : continue;
7034 : }
7035 0 : auto tempLink = singleSubCommTransport.links[linkIdx];
7036 0 : MemDetails remoteMem;
7037 0 : u32 remoteId = tempLink->GetRemoteRank();
7038 0 : CHK_PRT_RET(
7039 : (remoteId >= MAX_RANK_NUM_A3),
7040 : HCCL_ERROR(
7041 : "[%s]Invalid remoteId, valid range is [0, %u), remoteId[%u]", __func__, MAX_RANK_NUM_A3,
7042 : remoteId),
7043 : HCCL_E_PARA);
7044 0 : void* userMemPtr = nullptr;
7045 0 : CHK_RET(tempLink->GetRemoteMem(UserMemType::INPUT_MEM, &userMemPtr));
7046 0 : CHK_PTR_NULL(userMemPtr);
7047 0 : remoteMem.addr = reinterpret_cast<u64>(userMemPtr);
7048 0 : CHK_RET(tempLink->GetRemoteMemSize(UserMemType::INPUT_MEM, remoteMem.size));
7049 0 : opResPara_.userMemRes[remoteId] = remoteMem;
7050 0 : HCCL_INFO(
7051 : "[%s]add userMem res success, remoteId[%u], "
7052 : "remote addr[%llu], linkIdx[%u]",
7053 : __func__, remoteId, remoteMem.addr, linkIdx);
7054 0 : linkIdx++;
7055 0 : }
7056 : }
7057 : }
7058 0 : opResPara_.userMemType = TYPE_USER_MEM;
7059 0 : return HCCL_SUCCESS;
7060 : }
7061 :
7062 0 : HcclResult HcclCommunicator::IncreAllocLink(
7063 : const std::string& newTag, const OpParam& opParam, AlgResourceRequest& resRequest,
7064 : AlgResourceResponse& algResResponse)
7065 : {
7066 0 : algResResponse.cclInputMem = cclBufferManager_.GetInCCLbuffer();
7067 0 : algResResponse.cclOutputMem = cclBufferManager_.GetOutCCLbuffer();
7068 0 : DeviceMem expMem = cclBufferManager_.GetCommCCLBuffer();
7069 0 : transportManager_->SetOpType(opParam.opType);
7070 :
7071 : TransportIOMem transMem{
7072 0 : algResResponse.cclInputMem,
7073 0 : algResResponse.cclOutputMem,
7074 0 : algResResponse.paramInputMem,
7075 0 : algResResponse.paramOutputMem,
7076 0 : algResResponse.scratchMem,
7077 0 : algResResponse.aivInputMem,
7078 0 : algResResponse.aivOutputMem,
7079 : expMem,
7080 0 : DeviceMem()};
7081 : {
7082 0 : StateGuard<HcclCommunicator, HcclCommState> guard(this, HcclCommState::BUILDING);
7083 0 : CHK_RET(transportManager_->IncreAlloc(
7084 : opParam.tag, transMem, resRequest.opTransport, algResResponse.opTransportResponse, opParam.aicpuUnfoldMode,
7085 : false, opParam.isCapture, opParam.opType));
7086 0 : }
7087 0 : if (retryEnable_) {
7088 : // 获取当前rdma相连的所有对端rankList
7089 0 : std::vector<u32> rankList;
7090 0 : CHK_RET(transportManager_->GetIncreRemoteRankList(
7091 : resRequest.opTransport, rankList, TransportType::TRANS_TYPE_IBV_EXP));
7092 0 : std::string rankListStr = "";
7093 0 : for (auto remoteRank : rankList) {
7094 0 : rankListStr += (std::to_string(remoteRank) + ";");
7095 : }
7096 0 : HCCL_DEBUG("identifier[%s] newTag[%s] rankList[%s]", identifier_.c_str(), newTag.c_str(), rankListStr.c_str());
7097 0 : CHK_RET(OpRetryManager::AddLinkInfoByIdentifier(deviceLogicId_, identifier_, newTag, rankList, true));
7098 0 : }
7099 0 : if (IsEnableBackupLink()) {
7100 0 : StateGuard<HcclCommunicator, HcclCommState> guard(this, HcclCommState::BUILDING);
7101 0 : CHK_RET(transportManager_->IncreAlloc(
7102 : opParam.tag, transMem, resRequest.opTransport, algResResponse.opTransportResponseBackUp,
7103 : opParam.aicpuUnfoldMode, true, opParam.isCapture, opParam.opType));
7104 0 : }
7105 0 : remoteTransportMap_ = transportManager_->GetRemoteTransportMap();
7106 0 : SaveLinkRes(algResResponse.opTransportResponse);
7107 0 : SaveLinkRes(algResResponse.opTransportResponseBackUp);
7108 0 : return HCCL_SUCCESS;
7109 0 : }
7110 :
7111 0 : HcclResult HcclCommunicator::SetDevicePid(s32 devicePid)
7112 : {
7113 0 : devicePid_ = devicePid;
7114 0 : return HCCL_SUCCESS;
7115 : }
7116 :
7117 805 : void HcclCommunicator::ReleaseWorkSpacebuffer() { workSpace_.free(); }
7118 :
7119 0 : HcclResult HcclCommunicator::AllocAndClearDeviceMem(u64 size, std::shared_ptr<DeviceMem>& bufferPtr) const
7120 : {
7121 0 : CHK_PRT_RET(
7122 : !size,
7123 : HCCL_INFO("[HcclCommunicator][AllocAndClearDeviceMem]device memory size is zero. not need to malloc memory"),
7124 : HCCL_SUCCESS);
7125 :
7126 : CHK_PRT_RET(
7127 : (size > ULONG_MAX),
7128 : HCCL_ERROR("[HcclCommunicator][AllocAndClearDeviceMem]device memory size is greater than %llu", ULONG_MAX),
7129 : HCCL_E_PARA);
7130 :
7131 0 : DeviceMem tmpBuffer;
7132 0 : CHK_RET(DeviceMem::alloc(tmpBuffer, size));
7133 0 : EXCEPTION_CATCH((bufferPtr = std::make_shared<DeviceMem>(std::move(tmpBuffer))), return HCCL_E_PTR);
7134 :
7135 0 : CHK_PRT_RET(
7136 : size && !bufferPtr.get()->ptr(),
7137 : HCCL_ERROR(
7138 : "[HcclCommunicator][AllocAndClearDeviceMem]Create DeviceMem size[%llu] fail,"
7139 : "please check workspace size.",
7140 : size),
7141 : HCCL_E_PTR);
7142 0 : CHK_RET(hrtMemSet(bufferPtr.get()->ptr(), size, size));
7143 0 : return HCCL_SUCCESS;
7144 0 : }
7145 :
7146 1502 : HcclResult HcclCommunicator::AllocAndClearHostMem(u64 size, std::shared_ptr<HostMem>& bufferPtr) const
7147 : {
7148 1502 : CHK_PRT_RET(
7149 : !size,
7150 : HCCL_INFO("[HcclCommunicator][AllocAndClearHostMem] host memory size is zero. not need to malloc memory"),
7151 : HCCL_SUCCESS);
7152 :
7153 : CHK_PRT_RET(
7154 : (size > ULONG_MAX),
7155 : HCCL_ERROR("[HcclCommunicator][AllocAndClearHostMem] host memory size is greater than %llu", ULONG_MAX),
7156 : HCCL_E_PARA);
7157 :
7158 1502 : HostMem tmpBuffer = HostMem::alloc(size);
7159 1505 : EXCEPTION_CATCH((bufferPtr = std::make_shared<HostMem>(std::move(tmpBuffer))), return HCCL_E_PTR);
7160 :
7161 1499 : CHK_PRT_RET(
7162 : size && !bufferPtr.get()->ptr(),
7163 : HCCL_ERROR(
7164 : "[HcclCommunicator][AllocAndClearHostMem]host memory space size[%llu] fail,"
7165 : "please check workspace size.",
7166 : size),
7167 : HCCL_E_PTR);
7168 1501 : CHK_SAFETY_FUNC_RET(memset_s(bufferPtr.get()->ptr(), size, 0, size));
7169 1501 : return HCCL_SUCCESS;
7170 1501 : }
7171 :
7172 523 : HcclResult HcclCommunicator::CreateWorkSpace(u64 size, DeviceMem& buffer) const
7173 : {
7174 523 : CHK_PRT_RET(
7175 : !size, HCCL_INFO("[Create][WorkSpace]work space size is zero. not need to malloc memory"), HCCL_SUCCESS);
7176 :
7177 : CHK_PRT_RET(
7178 : (size > ULONG_MAX), HCCL_ERROR("[Create][WorkSpace]work space size is greater than %llu", ULONG_MAX),
7179 : HCCL_E_PARA);
7180 :
7181 523 : u64 memSize = size;
7182 523 : CHK_RET(DeviceMem::alloc(buffer, memSize));
7183 523 : CHK_RET(hrtMemSet(buffer.ptr(), size, size));
7184 523 : return HCCL_SUCCESS;
7185 : }
7186 :
7187 0 : HcclResult HcclCommunicator::GetWorkSpace(u64* workSpaceSize, u64* workSpace) const
7188 : {
7189 0 : *workSpaceSize = workSpaceSize_;
7190 0 : *workSpace = reinterpret_cast<u64>(workSpace_.ptr());
7191 0 : return HCCL_SUCCESS;
7192 : }
7193 :
7194 0 : HcclResult HcclCommunicator::InitWorkSpace()
7195 : {
7196 0 : if (workSpace_.ptr() == nullptr) {
7197 0 : workSpaceSize_ = COMM_MAX_WORK_SPACE_SIZE;
7198 0 : CHK_RET(CreateWorkSpace(workSpaceSize_, workSpace_));
7199 : }
7200 0 : return HCCL_SUCCESS;
7201 : }
7202 :
7203 0 : HcclResult HcclCommunicator::FillOpParam(
7204 : const HcclCMDType commType, OpParam& opParam, const uint64_t count, void* pCount, void* pDispls)
7205 : {
7206 0 : if (commType == HcclCMDType::HCCL_CMD_REDUCE_SCATTER || commType == HcclCMDType::HCCL_CMD_ALLGATHER
7207 0 : || commType == HcclCMDType::HCCL_CMD_ALLREDUCE) {
7208 0 : opParam.DataDes.count = count;
7209 0 : opParam.DataDes.dataType = HcclDataType::HCCL_DATA_TYPE_FP16; // 按照fp16配置
7210 0 : } else if (
7211 0 : commType == HcclCMDType::HCCL_CMD_ALLTOALLV || commType == HcclCMDType::HCCL_CMD_ALLTOALL
7212 0 : || commType == HcclCMDType::HCCL_CMD_ALLTOALLVC) {
7213 0 : opParam.All2AllDataDes.sendType = HcclDataType::HCCL_DATA_TYPE_FP16;
7214 0 : opParam.All2AllDataDes.recvType = HcclDataType::HCCL_DATA_TYPE_FP16;
7215 0 : opParam.All2AllDataDes.sendCounts = pCount;
7216 0 : opParam.All2AllDataDes.recvCounts = pCount;
7217 0 : opParam.All2AllDataDes.sdispls = pDispls;
7218 0 : opParam.All2AllDataDes.rdispls = pDispls;
7219 0 : opParam.All2AllDataDes.sendCountMatrix = pCount;
7220 0 : } else if (commType == HcclCMDType::HCCL_CMD_BATCH_WRITE) {
7221 : } else {
7222 0 : HCCL_ERROR("[%s] invalid commType=[%u]", __func__, static_cast<uint32_t>(commType));
7223 0 : return HCCL_E_PARA;
7224 : }
7225 0 : return HCCL_SUCCESS;
7226 : }
7227 :
7228 0 : HcclResult HcclCommunicator::AllocComResource(
7229 : const string& newTag, const string& algName, const HcclCMDType commType, const OpParam& opParam, rtStream_t stream,
7230 : bool isNeedHostSlaveStream)
7231 : {
7232 0 : if (resMap_.find(newTag) == resMap_.end()) { // 计算&申请通信资源
7233 0 : unique_ptr<CollAlgOperator> algOperator = implAlg_->GetAlgOperator(commType);
7234 0 : CHK_PRT_RET(algOperator == nullptr, HCCL_ERROR("[%s] algOperator is nullptr", __func__), HCCL_E_INTERNAL);
7235 0 : AlgResourceRequest resRequest;
7236 0 : CHK_RET(algOperator->CalcResRequest(algName, opParam, resRequest));
7237 0 : AlgResourceResponse algResResponse;
7238 0 : CHK_RET(AllocAlgResource(newTag, commType, opParam, resRequest, algResResponse, isNeedHostSlaveStream));
7239 0 : resMap_[newTag] = algResResponse;
7240 0 : CHK_RET(RegisterToHeartBeat());
7241 0 : }
7242 :
7243 0 : CHK_RET(InitWorkSpace());
7244 0 : HcclResult ret = GetWorkSpace(&(opResPara_.mc2WorkSpace.workSpaceSize), &(opResPara_.mc2WorkSpace.workSpace));
7245 0 : CHK_PRT_RET(
7246 : ret != HCCL_SUCCESS,
7247 : HCCL_ERROR(
7248 : "%s GetWorkSpace fail, size[%llu] space[%llu]", __func__, opResPara_.mc2WorkSpace.workSpaceSize,
7249 : opResPara_.mc2WorkSpace.workSpace),
7250 : ret);
7251 :
7252 0 : if (!isContextLaunched_) { // 通信域内首次下发
7253 0 : uint64_t streamMode = 0;
7254 0 : CHK_RET(hrtStreamGetMode(opParam.stream.ptr(), &streamMode));
7255 : rtStream_t aicpuStream;
7256 0 : Mc2AiCpuStreamAllocAndGet(streamMode, aicpuStream); // aicpuStream需要在首次下发时申请
7257 :
7258 : rtStream_t aicpuInitStream;
7259 0 : Mc2AiCpuInitStreamAllocAndGet(streamMode, aicpuInitStream);
7260 0 : Stream tmpStream(aicpuInitStream);
7261 0 : HCCL_DEBUG("%s ContextLaunched, aicpuInitStream:%p, aicpuStream:%p", __func__, aicpuInitStream, aicpuStream);
7262 0 : CHK_RET(AicpuResourceInit(algName, resMap_[newTag], newTag, stream, commType));
7263 0 : CHK_RET(GetReportHcclMC2Info(tmpStream, resMap_[newTag].slaveDevStreams));
7264 0 : } else if (newTagResAlloced_.find(newTag) == newTagResAlloced_.end()) {
7265 : // 通信域内非首次，但是有新的newTag
7266 0 : PetersonLockGuard guard(hostDeviceLock_.get());
7267 0 : CHK_PRT_RET(guard.IsLockFailed(), HCCL_ERROR("[%s] hostDeviceLock lock failed", __func__), HCCL_E_INTERNAL);
7268 0 : CHK_RET(AicpuResourceRefresh(resMap_[newTag], newTag, commType));
7269 0 : }
7270 0 : return HCCL_SUCCESS;
7271 : }
7272 :
7273 2 : HcclResult HcclCommunicator::AllocComResourceByTiling(const string& algConfig, void* param)
7274 : {
7275 2 : CHK_PTR_NULL(combinOparaMem_);
7276 2 : HcclCombinOpParam* combinOparaPtr = reinterpret_cast<HcclCombinOpParam*>(combinOparaMem_->ptr());
7277 2 : CHK_PTR_NULL(combinOparaPtr);
7278 :
7279 2 : string algName, newTag;
7280 2 : OpParam& opParam = *static_cast<OpParam*>(param);
7281 2 : CHK_RET(GetAlgInfo(algConfig, opParam.tag, opParam.opType, algName, newTag));
7282 2 : if (algName == "RunAlltoAllAivDirect") {
7283 1 : opParam.isNpuDirectRoce = true;
7284 : }
7285 2 : CHK_RET(CreateAndGetAiCpuNotifyWithNotifyRes(combinOparaPtr->signalInfo.aicpuNotify));
7286 0 : HCCL_INFO("Create aicpu notify %p.", localAiCpuNotifyRes_[0]->ptr());
7287 :
7288 : // 只有第一次创建，此处通过CCL Buffer地址有效来防止通信域内非首次重新申请内存
7289 : // 已注册user mem情况下，不创建ccl buffer，使用user mem通信
7290 0 : if (userMemMap_.empty()) {
7291 0 : CHK_RET(CreateCommCCLbuffer());
7292 0 : CHK_RET(cclBufferManager_.GetInCCLbuffer(opParam.inputPtr, opParam.inputSize));
7293 0 : CHK_RET(cclBufferManager_.GetOutCCLbuffer(opParam.outputPtr, opParam.outputSize));
7294 : } else {
7295 0 : auto it = userMemMap_.begin();
7296 0 : opParam.outputSize = it->second->size();
7297 0 : opParam.inputSize = it->second->size();
7298 : }
7299 :
7300 : // 按照 ccl buffer size 折算，不同算子折算方式不同, allreduce和cclbuffer size相同
7301 : // allgather、reducescatter、alltoall需除以rank size
7302 0 : uint64_t count = opParam.outputSize / SIZE_TABLE[HcclDataType::HCCL_DATA_TYPE_FP16];
7303 0 : if (opParam.opType != HcclCMDType::HCCL_CMD_ALLREDUCE) {
7304 0 : count = (count + userRankSize_ - 1) / userRankSize_;
7305 : }
7306 0 : HCCL_INFO("[%s] userRankSize=[%u], count=[%u]", __func__, userRankSize_, count);
7307 0 : vector<uint64_t> countList(userRankSize_ * userRankSize_, count);
7308 0 : vector<uint64_t> displsList(userRankSize_, 0);
7309 0 : void* pCount = reinterpret_cast<void*>(&countList[0]);
7310 0 : void* pDispls = reinterpret_cast<void*>(&displsList[0]);
7311 0 : CHK_RET(FillOpParam(opParam.opType, opParam, count, pCount, pDispls));
7312 : // MC2算子不需要申请host侧的从流
7313 0 : bool isNeedHostSlaveStream = false;
7314 0 : CHK_RET(AllocComResource(newTag, algName, opParam.opType, opParam, opParam.stream.ptr(), isNeedHostSlaveStream));
7315 0 : return HCCL_SUCCESS;
7316 2 : }
7317 :
7318 1 : HcclResult HcclCommunicator::CreateCommResource(
7319 : const std::string& tag, rtStream_t aiCpuStream, bool isOpbaseMode, void** commContext, const std::string& algConfig)
7320 : {
7321 1 : const std::string& suffix = HCCL_MC2_MULTISERVER_SUFFIX;
7322 1 : string algName = "";
7323 1 : string newTag = tag;
7324 1 : if (tag.size() > suffix.size() && tag.compare(tag.size() - suffix.size(), suffix.size(), suffix) == 0) {
7325 0 : HCCL_INFO("[HcclCommunicator][CreateCommResource] Set isA2MC2MultiServer_ to [true]");
7326 0 : isA2MC2MultiServer_ = true;
7327 0 : char* mmSysGetEnvValue = nullptr;
7328 0 : MM_SYS_GET_ENV(MM_ENV_HCCL_INTRA_PCIE_ENABLE, mmSysGetEnvValue);
7329 0 : std::string intraPcieEnableEnv = (mmSysGetEnvValue != nullptr) ? mmSysGetEnvValue : "EmptyString";
7330 0 : bool envA2MC2Hie = (intraPcieEnableEnv == "1") && (GetExternalInputIntraRoceSwitch() == 0);
7331 0 : if (!algConfig.empty()) {
7332 0 : CHK_RET(GetAlgInfo(algConfig, tag, algName));
7333 0 : if (algName == "DispatchCombineHierarchy" || (algName == "BatchWriteBySdma" && envA2MC2Hie)) {
7334 0 : isA2MC2IntraHie_ = true;
7335 0 : newTag.insert(newTag.size() - suffix.size(), "_HIE");
7336 : }
7337 : }
7338 0 : }
7339 1 : if (isA2MC2MultiServer_ && !isNeedInitNic_) {
7340 0 : InitNic(true);
7341 : }
7342 :
7343 1 : if ((deviceType_ != DevType::DEV_TYPE_910_93 && moduleNum_ > 1 && !isA2MC2MultiServer_)
7344 1 : || (deviceType_ == DevType::DEV_TYPE_910_93 && superPodNum_ > 1)) {
7345 0 : HCCL_ERROR(
7346 : "[HcclCommunicator][CommResource]MC2 does not support in the current scenario, "
7347 : "device type[%d] moduleNum[%d] serverNum[%d] superPodNum[%d], isMC2MultiServer[%d].",
7348 : deviceType_, moduleNum_, serverNum_, superPodNum_, isA2MC2MultiServer_);
7349 0 : return HCCL_E_NOT_SUPPORT;
7350 : }
7351 :
7352 1 : HCCL_INFO(
7353 : "[HcclCommunicator][CommResource]newTag[%s] aicpu stream[%p] isOpbaseMode[%u]", newTag.c_str(), aiCpuStream,
7354 : isOpbaseMode);
7355 :
7356 1 : Stream stream(aiCpuStream);
7357 1 : CHK_RET(CreateCommAndStreamRes(newTag, stream));
7358 :
7359 1 : CHK_RET(Mc2CreateAndLaunchContext(aiCpuStream, isOpbaseMode, commContext, newTag));
7360 1 : return HCCL_SUCCESS;
7361 1 : }
7362 :
7363 0 : HcclResult HcclCommunicator::Mc2CreateAndLaunchContext(
7364 : rtStream_t aiCpuStream, bool isOpbaseMode, void** commContext, const string& tag)
7365 : {
7366 0 : CHK_PTR_NULL(combinOparaMem_);
7367 0 : HcclCombinOpParam* combinOparaPtr = reinterpret_cast<HcclCombinOpParam*>(combinOparaMem_->ptr());
7368 0 : CHK_PTR_NULL(combinOparaPtr);
7369 0 : CHK_RET(InitWorkSpace());
7370 :
7371 : HcclResult result
7372 0 : = GetWorkSpace(&(combinOparaPtr->mc2WorkSpace.workSpaceSize), &(combinOparaPtr->mc2WorkSpace.workSpace));
7373 0 : CHK_PRT_RET(
7374 : result != HCCL_SUCCESS,
7375 : HCCL_ERROR(
7376 : "[HcclCommunicator][CommResource]errNo[0x%016llx] size[%llu] space[%llu]", HCCL_ERROR_CODE(result),
7377 : combinOparaPtr->mc2WorkSpace.workSpaceSize, combinOparaPtr->mc2WorkSpace.workSpace),
7378 : result);
7379 :
7380 0 : CHK_SAFETY_FUNC_RET(memcpy_s(
7381 : combinOparaPtr->hcomId, sizeof(combinOparaPtr->hcomId), identifier_.c_str(), identifier_.length() + 1));
7382 :
7383 0 : Stream tmpStream(aiCpuStream);
7384 0 : CHK_RET(CreateAndGetAiCpuNotifyWithNotifyRes(combinOparaPtr->signalInfo.aicpuNotify));
7385 0 : CHK_RET(CreateAndGetAiCpuNotify(
7386 : localAiCpuOpNotify_[static_cast<u32>(AicpuLocalNotifyIdx::HOST_TO_AICPU_0)],
7387 : combinOparaPtr->signalInfo.aicpuOpNotify[static_cast<u32>(AicpuLocalNotifyIdx::HOST_TO_AICPU_0)]));
7388 0 : CHK_RET(CreateAndGetAiCpuNotify(
7389 : localAiCpuOpNotify_[static_cast<u32>(AicpuLocalNotifyIdx::HOST_TO_AICPU_1)],
7390 : combinOparaPtr->signalInfo.aicpuOpNotify[static_cast<u32>(AicpuLocalNotifyIdx::HOST_TO_AICPU_1)]));
7391 : // 申请集合通信域存储context的device空间
7392 0 : CHK_RET(CreateDeviceCommContext(sizeof(HcclCombinOpParam), commContext_));
7393 0 : combinOparaPtr->config.deterministic = GetDeterministicConfig();
7394 : // retryEnable 写入aicpu_ctx
7395 0 : combinOparaPtr->config.retryEnable = static_cast<u8>(retryEnable_);
7396 0 : combinOparaPtr->config.retryHoldTime = commConfig_.GetConfigRetryHoldTime();
7397 0 : combinOparaPtr->config.retryIntervalTime = commConfig_.GetConfigRetryIntervalTime();
7398 : combinOparaPtr->config.notifyWaitTime
7399 0 : = (GetExternalInputHcclExecTimeoutSet() != HcclExecTimeoutSet::HCCL_EXEC_TIMEOUT_NOT_SET
7400 0 : || commConfig_.GetConfigExecTimeOutSet()) ?
7401 0 : commConfig_.GetConfigExecTimeOut() :
7402 : NOTIFY_DEFAULT_WAIT_TIME;
7403 0 : combinOparaPtr->config.linkTimeOut = std::chrono::seconds(GetExternalInputHcclLinkTimeOut());
7404 :
7405 0 : combinOparaPtr->kfcControlTransferH2DParams = kfcControlTransferH2D_->GetCommunicateParams();
7406 0 : combinOparaPtr->kfcStatusTransferD2HParams = kfcStatusTransferD2H_->GetCommunicateParams();
7407 :
7408 0 : void* overflowAddr = nullptr;
7409 0 : if (Is310P3Common(isHaveCpuRank_, deviceType_)) {
7410 0 : CHK_RET(hrtCtxGetOverflowAddr(&overflowAddr));
7411 0 : combinOparaPtr->overFlowAddr = reinterpret_cast<u64>(overflowAddr);
7412 0 : HCCL_INFO(
7413 : "[HcclImplBase][Mc2CreateAndLaunchContext]get combinOparaPtr->overFlowAddr %llx",
7414 : combinOparaPtr->overFlowAddr);
7415 : // 非整卡 (2DUO卡各取1芯的场景) 因为受到PCIE限制，不可以使用读操作进行数据拷贝
7416 0 : if (pairLinkInfo_[static_cast<u32>(LinkTypeInServer::HCCS_TYPE)].size() != userRankSize_) {
7417 0 : combinOparaPtr->onlyRead = 1;
7418 : }
7419 : }
7420 0 : HCCL_INFO("read only is set to %u", combinOparaPtr->onlyRead);
7421 :
7422 0 : if (isA2MC2MultiServer_) {
7423 : // 拷贝normal transport信息到device侧
7424 0 : bool isSupportAIVNormalQP = false;
7425 0 : CHK_RET(IsSupportAIVNormalQP(devicePhyId_, isSupportAIVNormalQP));
7426 0 : CHK_PTR_NULL(transDevIbverbsDataMem_);
7427 0 : const u64 ibverbsDataSize = transDevIbverbsDataMem_->size();
7428 0 : CHK_RET(DeviceMem::alloc(ibverbsDataBuffer_, ibverbsDataSize));
7429 0 : CHK_RET(hrtMemAsyncCopy(
7430 : ibverbsDataBuffer_.ptr(), ibverbsDataBuffer_.size(), transDevIbverbsDataMem_->ptr(), ibverbsDataSize,
7431 : HcclRtMemcpyKind::HCCL_RT_MEMCPY_KIND_HOST_TO_DEVICE, aiCpuStream));
7432 :
7433 0 : combinOparaPtr->ibverbsData = reinterpret_cast<u64>(ibverbsDataBuffer_.ptr());
7434 0 : combinOparaPtr->ibverbsDataSize = ibverbsDataSize;
7435 0 : combinOparaPtr->multiServerFlag = static_cast<u8>(true);
7436 :
7437 0 : CHK_PTR_NULL(combinedCapabilityMem_);
7438 0 : const u64 capabilitySize = sizeof(CombinedCapability);
7439 0 : CHK_RET(DeviceMem::alloc(combinedCapabilityBuffer_, capabilitySize));
7440 0 : CHK_RET(hrtMemAsyncCopy(
7441 : combinedCapabilityBuffer_.ptr(), combinedCapabilityBuffer_.size(), combinedCapabilityMem_->ptr(),
7442 : capabilitySize, HcclRtMemcpyKind::HCCL_RT_MEMCPY_KIND_HOST_TO_DEVICE, aiCpuStream));
7443 :
7444 0 : combinOparaPtr->capabilityPtr = reinterpret_cast<u64>(combinedCapabilityBuffer_.ptr());
7445 0 : combinOparaPtr->capabilitySize = capabilitySize;
7446 :
7447 0 : HCCL_INFO(
7448 : "[HcclImplBase][Mc2CreateAndLaunchContext] set ibverbsData to [%llu], "
7449 : "multiServerFlag to [%u]",
7450 : combinOparaPtr->ibverbsData, combinOparaPtr->multiServerFlag);
7451 0 : if (isSupportAIVNormalQP && isA2MC2IntraHie_) {
7452 0 : CHK_RET(H2DAiRMAInfo(tag, aiCpuStream));
7453 : }
7454 : }
7455 :
7456 : // 将通信数据拷贝到device侧，供AICPU算法编排使用
7457 0 : CHK_RET(hrtMemAsyncCopy(
7458 : commContext_.ptr(), commContext_.size(), combinOparaMem_->ptr(), combinOparaMem_->size(),
7459 : HcclRtMemcpyKind::HCCL_RT_MEMCPY_KIND_HOST_TO_DEVICE, aiCpuStream));
7460 :
7461 0 : std::string kernelName = "RunAicpuKfcResInit";
7462 0 : CHK_RET(AiCpuKernelLaunch(tmpStream.ptr(), reinterpret_cast<u64>(commContext_.ptr()), kernelName));
7463 0 : SetMC2EnvFlag();
7464 0 : if (isOpbaseMode == true) {
7465 0 : CHK_RET(hcclStreamSynchronize(tmpStream.ptr(), commConfig_.GetConfigExecTimeOut()));
7466 : }
7467 :
7468 0 : *commContext = commContext_.ptr();
7469 0 : return HCCL_SUCCESS;
7470 0 : }
7471 :
7472 : HcclResult
7473 0 : HcclCommunicator::GetAiCpuNotifyData(const std::shared_ptr<LocalNotify>& localNotify, HcclSignalInfo& notifyInfo)
7474 : {
7475 0 : if (localNotify == nullptr) {
7476 0 : HCCL_INFO("[HcclCommunicator][GetAiCpuNotifyData]notifyHandle is null");
7477 0 : notifyInfo.resId = INVALID_U64;
7478 0 : return HCCL_SUCCESS;
7479 : }
7480 :
7481 0 : CHK_RET(localNotify->GetNotifyData(notifyInfo));
7482 0 : HCCL_INFO(
7483 : "[HcclCommunicator][GetAiCpuNotifyData]resId[%lld], addr[%lld], devId[%u], tsId[%u].", notifyInfo.resId,
7484 : notifyInfo.addr, notifyInfo.devId, notifyInfo.tsId);
7485 0 : return HCCL_SUCCESS;
7486 : }
7487 :
7488 : HcclResult
7489 0 : HcclCommunicator::CreateAndGetAiCpuNotify(std::shared_ptr<LocalNotify>& localNotify, HcclSignalInfo& notifyInfo)
7490 : {
7491 0 : if (localNotify != nullptr) {
7492 0 : CHK_RET(GetAiCpuNotifyData(localNotify, notifyInfo));
7493 0 : HCCL_INFO("[HcclCommunicator][CreateAndGetAiCpuNotify]aicpu notify already create ptr[%p]", localNotify->ptr());
7494 0 : return HCCL_SUCCESS;
7495 : }
7496 :
7497 0 : EXCEPTION_CATCH((localNotify = std::make_shared<LocalNotify>()), return HCCL_E_PTR);
7498 0 : CHK_RET(localNotify->Init(NotifyLoadType::DEVICE_NOTIFY));
7499 0 : CHK_RET(localNotify->SetIpc());
7500 :
7501 0 : CHK_RET(GetAiCpuNotifyData(localNotify, notifyInfo));
7502 0 : return HCCL_SUCCESS;
7503 : }
7504 :
7505 2 : HcclResult HcclCommunicator::Mc2AiCpuStreamAllocAndGet(u32 streamMode, rtStream_t& aiCpuStream)
7506 : {
7507 2 : if (opStream_.ptr() != nullptr) {
7508 0 : HCCL_INFO("%s already alloc, group:%s, stream id:%u", __func__, identifier_.c_str(), opStream_.id());
7509 0 : aiCpuStream = opStream_.ptr();
7510 0 : return HCCL_SUCCESS;
7511 : }
7512 :
7513 2 : constexpr u32 aicpuStreamMode = 1; // 单独申请的kernel流，使能遇错即停，避免出错后流卡住不退
7514 2 : opStream_ = Stream(StreamType::STREAM_TYPE_ONLINE);
7515 2 : CHK_RET(hrtStreamSetMode(opStream_.ptr(), aicpuStreamMode));
7516 2 : aiCpuStream = opStream_.ptr();
7517 2 : HCCL_RUN_INFO(
7518 : "%s alloc success, group:%s, stream id:%u, mainStreamMode:%u, aicpuStreamMode:%u", __func__,
7519 : identifier_.c_str(), opStream_.id(), streamMode, aicpuStreamMode);
7520 2 : return HCCL_SUCCESS;
7521 : }
7522 :
7523 2 : HcclResult HcclCommunicator::Mc2AiCpuInitStreamAllocAndGet(u32 streamMode, rtStream_t& aiCpuStream)
7524 : {
7525 2 : if (aicpuInitStream_.ptr() != nullptr) {
7526 1 : HCCL_INFO("%s already alloc, group:%s, stream id:%u", __func__, identifier_.c_str(), aicpuInitStream_.id());
7527 1 : aiCpuStream = aicpuInitStream_.ptr();
7528 1 : return HCCL_SUCCESS;
7529 : }
7530 :
7531 1 : constexpr u32 aicpuStreamMode = 1; // 单独申请的kernel流，使能遇错即停，避免出错后流卡住不退
7532 1 : aicpuInitStream_ = Stream(StreamType::STREAM_TYPE_ONLINE);
7533 1 : CHK_RET(hrtStreamSetMode(aicpuInitStream_.ptr(), aicpuStreamMode));
7534 1 : aiCpuStream = aicpuInitStream_.ptr();
7535 1 : HCCL_RUN_INFO(
7536 : "%s alloc success, group:%s, stream id:%u, mainStreamMode:%u, aicpuStreamMode:%u", __func__,
7537 : identifier_.c_str(), aicpuInitStream_.id(), streamMode, aicpuStreamMode);
7538 1 : return HCCL_SUCCESS;
7539 : }
7540 :
7541 0 : HcclResult HcclCommunicator::AicpuResourceInit(
7542 : const std::string& algName, const AlgResourceResponse& algResource, const std::string& newTag,
7543 : const rtStream_t& aicpuStream, const HcclCMDType opType, bool isCustom)
7544 : {
7545 0 : HCCL_RUN_INFO(
7546 : "[%s] start to init group[%s] aicpu resources newTag[%s] local rankId[%u]", __func__, identifier_.c_str(),
7547 : newTag.c_str(), userRank_);
7548 0 : isContextLaunched_ = true;
7549 0 : CHK_RET(BuildOpResParam(algName, algResource, newTag, opType, aicpuStream)); // 构建context结构体
7550 0 : std::string kernelName = "RunAicpuKfcResInitV2";
7551 : // 在这里构建suspending状态码的HDC通道初始化，并且在host侧进行init
7552 : // （这个主要是针对hcomId；对算子通信域的复用；也就是多个算子复用（tag+Identifier）这个通信域的情况）
7553 0 : CHK_RET(AiCpuKernelLaunch(aicpuStream, reinterpret_cast<u64>(opResDevicePara_.ptr()), kernelName));
7554 0 : SetMC2EnvFlag();
7555 0 : newTagResAlloced_.insert(newTag);
7556 : // 图模多档位场景，需要保证执行序上优先下资源初始化的kernel
7557 0 : CHK_RET(hcclStreamSynchronize(aicpuStream, commConfig_.GetConfigExecTimeOut()));
7558 :
7559 0 : if (IsEnableCustom()) {
7560 : struct InitTask {
7561 : u64 context; // A矩阵地址，通信在前时为sendbuffer
7562 : bool isCustom;
7563 : };
7564 0 : InitTask customInitTask = {0};
7565 0 : customInitTask.context = reinterpret_cast<u64>(opResDevicePara_.ptr());
7566 0 : customInitTask.isCustom = true;
7567 0 : CHK_RET(BuildCustomOpResParam());
7568 0 : uint64_t customBeginTime = hrtMsprofSysCycleTime();
7569 0 : const std::string customProfName = "hcomAicpuCustomInit";
7570 :
7571 0 : u16 timeOut = 0;
7572 0 : if (opResPara_.config.notifyWaitTime == 0) {
7573 0 : timeOut = opResPara_.config.notifyWaitTime;
7574 0 : } else if (opResPara_.config.notifyWaitTime + AICPU_KERNEL_TIMEOUT_INC >= MAX_VALUE_U16) {
7575 0 : timeOut = MAX_VALUE_U16;
7576 : } else {
7577 0 : timeOut = opResPara_.config.notifyWaitTime + AICPU_KERNEL_TIMEOUT_INC;
7578 : }
7579 :
7580 0 : CHK_RET(AicpuAclKernelLaunch(
7581 : aicpuStream, reinterpret_cast<void*>(&customInitTask), sizeof(customInitTask), binCustomHandle_, kernelName,
7582 : true, timeOut));
7583 0 : uint64_t customEndTime = hrtMsprofSysCycleTime();
7584 0 : s32 customthreadId = SalGetTid();
7585 0 : CHK_RET(ProfilingManagerPub::CallMsprofReportNodeInfo(
7586 : customBeginTime, customEndTime, customProfName, customthreadId));
7587 0 : CHK_RET(hcclStreamSynchronize(aicpuStream, commConfig_.GetConfigExecTimeOut()));
7588 0 : }
7589 :
7590 0 : return HCCL_SUCCESS;
7591 0 : }
7592 :
7593 0 : HcclResult HcclCommunicator::AiCpuKernelLaunch(const rtStream_t stm, u64 addr, const std::string& kernelName)
7594 : {
7595 0 : uint64_t beginTime = hrtMsprofSysCycleTime();
7596 0 : const std::string profName = "hcomAicpuInit";
7597 : struct InitTask {
7598 : u64 context; // A矩阵地址，通信在前时为sendbuffer
7599 : bool isCustom;
7600 : };
7601 0 : InitTask initTask = {0};
7602 0 : initTask.context = addr;
7603 0 : initTask.isCustom = false;
7604 :
7605 0 : u16 timeOut = 0;
7606 0 : if (opResPara_.config.notifyWaitTime == 0) {
7607 0 : timeOut = opResPara_.config.notifyWaitTime;
7608 0 : } else if (opResPara_.config.notifyWaitTime + AICPU_KERNEL_TIMEOUT_INC >= MAX_VALUE_U16) {
7609 0 : timeOut = MAX_VALUE_U16;
7610 : } else {
7611 0 : timeOut = opResPara_.config.notifyWaitTime + AICPU_KERNEL_TIMEOUT_INC;
7612 : }
7613 0 : CHK_RET(AicpuAclKernelLaunch(
7614 : stm, reinterpret_cast<void*>(&initTask), sizeof(initTask), binHandle_, kernelName, true, timeOut));
7615 0 : uint64_t endTime = hrtMsprofSysCycleTime();
7616 0 : s32 threadId = SalGetTid();
7617 0 : CHK_RET(ProfilingManagerPub::CallMsprofReportNodeInfo(beginTime, endTime, profName, threadId));
7618 0 : return HCCL_SUCCESS;
7619 0 : }
7620 :
7621 0 : HcclResult HcclCommunicator::AicpuKfcTilingDataLaunch(
7622 : const OpParam& opParam, const HcclCMDType& opType, const DeviceMem& deviceContext, const std::string& kernelName,
7623 : const AicpuOpTiling opTilingInfo)
7624 : {
7625 0 : HCCL_DEBUG(
7626 : "AicpuKfcTilingDataLaunch count %llu dataType %s op %s opType %u", opParam.GetDataCount(userRank_),
7627 : GetDataTypeEnumStr(opParam.GetDataType()).c_str(), GetReduceOpEnumStr(opParam.reduceType).c_str(), opType);
7628 0 : struct HcclKFCTilingData tilingDate = {0};
7629 0 : tilingDate.sendCnt = opParam.DataDes.count;
7630 0 : tilingDate.dataType = opParam.DataDes.dataType;
7631 0 : tilingDate.commType = static_cast<uint8_t>(opType);
7632 0 : tilingDate.reduceOp = opParam.reduceType;
7633 0 : tilingDate.taskType = HCCL_KFC_TASK_HCCL_ONLY_EXE;
7634 0 : tilingDate.totalCnt = 1;
7635 0 : tilingDate.turnNum = 1;
7636 0 : tilingDate.hasCommOut = 1;
7637 0 : tilingDate.debugMode = 0;
7638 0 : CHK_RET(SetNormalMode(dispatcher_));
7639 0 : HcclWorkflowMode mode = GetWorkflowMode();
7640 0 : Stream mainStream(opParam.stream.ptr());
7641 0 : CHK_RET(LocalNotify::Post(
7642 : mainStream, dispatcher_, localAiCpuOpNotify_[static_cast<u32>(AicpuLocalNotifyIdx::HOST_TO_AICPU_0)],
7643 : INVALID_VALUE_STAGE));
7644 0 : rtStream_t kfcOpStream = opStream_.ptr();
7645 0 : if (opTilingInfo.isUsedMainStream) {
7646 0 : kfcOpStream = opParam.stream.ptr();
7647 : }
7648 0 : CHK_RET(AicpuUnfoldKernelLaunch(
7649 : opParam.inputPtr, opParam.outputPtr, kfcOpStream, reinterpret_cast<u64>(deviceContext.ptr()), &tilingDate,
7650 : sizeof(HcclKFCTilingData), kernelName, mode, opParam.tag));
7651 0 : CHK_RET(LocalNotify::Wait(
7652 : mainStream, dispatcher_, localAiCpuOpNotify_[static_cast<u32>(AicpuLocalNotifyIdx::HOST_TO_AICPU_1)],
7653 : INVALID_VALUE_STAGE));
7654 0 : return HCCL_SUCCESS;
7655 0 : }
7656 :
7657 6 : HcclResult HcclCommunicator::AicpuKfcClearOpResLaunch(const std::unordered_set<std::string>& tags)
7658 : {
7659 6 : if (tags.empty()) {
7660 1 : return HCCL_SUCCESS;
7661 : }
7662 : // 仅 aicpu unfold 模式有 aicpu 端 resMap_/linkRes_ 需要清理；host 模式下没有 binHandle_
7663 5 : if (binHandle_ == nullptr) {
7664 2 : HCCL_DEBUG(
7665 : "[AicpuKfcClearOpResLaunch] binHandle_ null (host-mode communicator), skip; tagCount[%zu]", tags.size());
7666 2 : return HCCL_SUCCESS;
7667 : }
7668 3 : if (opStream_.ptr() == nullptr) {
7669 1 : HCCL_WARNING("[AicpuKfcClearOpResLaunch] opStream_ null, skip aicpu cleanup; tagCount[%zu]", tags.size());
7670 1 : return HCCL_SUCCESS;
7671 : }
7672 : // host args 通道有 size 上限，大 payload 走 args/tiling 会被拒绝。沿用 RunAicpuKfcResInit 模式：HBM buffer 持载
7673 : // payload
7674 2 : if (!aicpuCleanupBuf_) {
7675 0 : CHK_RET(DeviceMem::alloc(aicpuCleanupBuf_, sizeof(HcclKfcClearOpResTilingData)));
7676 : }
7677 2 : if (!aicpuCleanupHostBuf_) {
7678 0 : aicpuCleanupHostBuf_.reset(new (std::nothrow) HcclKfcClearOpResTilingData());
7679 0 : CHK_SMART_PTR_NULL(aicpuCleanupHostBuf_);
7680 : }
7681 2 : HcclKfcClearOpResTilingData& payload = *aicpuCleanupHostBuf_;
7682 :
7683 : // 必须与 aicpu_kfc_def.h 中 KFCResInitTask 布局一致，aicpu 端按此解包
7684 : struct KFCResInitTask {
7685 : u64 context;
7686 : bool isCustom;
7687 : };
7688 2 : KFCResInitTask initTask = {reinterpret_cast<u64>(aicpuCleanupBuf_.ptr()), false};
7689 2 : const u16 timeOut = MAX_VALUE_U16;
7690 2 : const size_t groupCopyLen = std::min(identifier_.length() + 1, sizeof(payload.group));
7691 2 : size_t totalBatches = 0;
7692 :
7693 : // 分批 launch：同 buffer 复用，每批最多 MAX_BATCH 个 tag；launch 后 sync 保证 aicpu 完成才覆盖 buffer 下一批
7694 2 : auto it = tags.begin();
7695 5 : while (it != tags.end()) {
7696 3 : payload.magic = HCCL_KFC_CLEAR_OP_RES_MAGIC;
7697 3 : CHK_SAFETY_FUNC_RET(memcpy_s(payload.group, sizeof(payload.group), identifier_.c_str(), groupCopyLen));
7698 3 : payload.group[sizeof(payload.group) - 1] = '\0';
7699 :
7700 3 : u32 idx = 0;
7701 10255 : while (it != tags.end() && idx < HCCL_KFC_CLEAR_OP_RES_MAX_BATCH) {
7702 10252 : const std::string& t = *it;
7703 10252 : const size_t tagCopyLen = std::min(t.length() + 1, sizeof(payload.tags[idx]));
7704 10252 : CHK_SAFETY_FUNC_RET(memcpy_s(payload.tags[idx], sizeof(payload.tags[idx]), t.c_str(), tagCopyLen));
7705 10252 : payload.tags[idx][sizeof(payload.tags[idx]) - 1] = '\0';
7706 10252 : ++idx;
7707 10252 : ++it;
7708 : }
7709 3 : payload.tagCount = idx;
7710 :
7711 3 : CHK_RET(hrtMemSyncCopy(
7712 : aicpuCleanupBuf_.ptr(), sizeof(payload), reinterpret_cast<void*>(&payload), sizeof(payload),
7713 : HcclRtMemcpyKind::HCCL_RT_MEMCPY_KIND_HOST_TO_DEVICE));
7714 :
7715 3 : HcclResult ret = AicpuAclKernelLaunchV2(
7716 : opStream_.ptr(), reinterpret_cast<void*>(&initTask), sizeof(initTask), binHandle_, "RunAicpuKfcClearOpRes",
7717 3 : true, timeOut, nullptr, 0, identifier_);
7718 3 : if (ret != HCCL_SUCCESS) {
7719 0 : HCCL_ERROR(
7720 : "[AicpuKfcClearOpResLaunch] launch fail, group[%s] batch[%zu] tagCount[%u] ret[%d]",
7721 : identifier_.c_str(), totalBatches, idx, ret);
7722 0 : return ret;
7723 : }
7724 3 : CHK_RET(hcclStreamSynchronize(opStream_.ptr(), commConfig_.GetConfigExecTimeOut()));
7725 3 : ++totalBatches;
7726 : }
7727 :
7728 2 : HCCL_INFO(
7729 : "[AicpuKfcClearOpResLaunch] dispatched aicpu cleanup, group[%s] totalTags[%zu] batches[%zu]",
7730 : identifier_.c_str(), tags.size(), totalBatches);
7731 2 : return HCCL_SUCCESS;
7732 : }
7733 :
7734 8 : HcclResult HcclCommunicator::AicpuInitOpTilingDataAicpuCache(
7735 : const OpParam& opParam, const HcclCMDType& opType, struct OpTilingData* opTilingData)
7736 : {
7737 8 : opTilingData->aicpuCacheEnable = opParam.aicpuCacheEnable;
7738 : // 开启aicpu cache, 且原来是图模式建链但强制走单算子模式展开
7739 : // 开启aicpu cache，isCapture为true，且是图模式，证明选择了aclgraph零拷贝算法，需要强制刷新cache
7740 5 : if (opParam.aicpuCacheEnable != 0 && GetWorkflowMode() == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OPS_KERNEL_INFO_LIB
7741 13 : && ((IsForceAicpuOpBaseMode(opParam, opType) && !opParam.isZeroCopy) || opParam.isCapture)) {
7742 : // 环境变量传入的aicpuCacheEnable一定 < 10
7743 1 : constexpr uint8_t FORCE_OP_BASE_DELTA = 10;
7744 1 : CHK_PRT_RET(
7745 : opParam.aicpuCacheEnable >= FORCE_OP_BASE_DELTA,
7746 : HCCL_ERROR(
7747 : "[HcclCommunicator][AicpuInitOpTilingDataBuf] enforce opbase mode: opParam.aicpuCacheEnable >= %u",
7748 : opParam.aicpuCacheEnable, FORCE_OP_BASE_DELTA),
7749 : HCCL_E_INTERNAL);
7750 :
7751 : // 1 -> 11: 开启aicpu cache且存在强制单算子模式转换
7752 1 : opTilingData->aicpuCacheEnable += FORCE_OP_BASE_DELTA;
7753 1 : HCCL_WARNING(
7754 : "[HcclCommunicator][AicpuInitOpTilingDataBuf] enforce opbase mode: opParam.aicpuCacheEnable[%u]"
7755 : "opTilingData->aicpuCacheEnable[%u]",
7756 : opParam.aicpuCacheEnable, opTilingData->aicpuCacheEnable);
7757 :
7758 : // 注意: 开启aicpu cache且存在强制单算子模式转换, 传入device的aicpuCacheEnable一定 > 10
7759 1 : CHK_PRT_RET(
7760 : opTilingData->aicpuCacheEnable <= FORCE_OP_BASE_DELTA,
7761 : HCCL_ERROR(
7762 : "[HcclCommunicator][AicpuInitOpTilingDataBuf] enforce opbase mode: opTilingData->aicpuCacheEnable[%u] "
7763 : "<= %u",
7764 : opTilingData->aicpuCacheEnable, FORCE_OP_BASE_DELTA),
7765 : HCCL_E_INTERNAL);
7766 : }
7767 :
7768 8 : return HCCL_SUCCESS;
7769 : }
7770 :
7771 1 : HcclResult HcclCommunicator::AicpuInitOpTilingDataBuf(
7772 : const OpParam& opParam, const HcclCMDType& opType, const std::string& kernelName, const AicpuOpTiling opTilingInfo,
7773 : u64 dynamicDataSize)
7774 : {
7775 1 : u32 opTilingDataSize = sizeof(struct OpTilingData) + dynamicDataSize;
7776 :
7777 1 : if (opTilingDataBuf_.ptr() == nullptr) {
7778 1 : opTilingDataBuf_ = HostMem::alloc(TILINGDATA_BUF_SIZE);
7779 1 : CHK_PRT_RET(
7780 : opTilingDataBuf_.ptr() == nullptr,
7781 : HCCL_ERROR("[HcclCommunicator][AicpuInitOpTilingDataBuf] Alloc opTilingDataBuf failed!"), HCCL_E_INTERNAL);
7782 : }
7783 :
7784 1 : if (opTilingDataBuf_.ptr() != nullptr && opTilingDataSize > opTilingDataBuf_.size()) {
7785 0 : opTilingDataBuf_.free();
7786 0 : opTilingDataBuf_ = HostMem::alloc(opTilingDataSize);
7787 0 : CHK_PRT_RET(
7788 : opTilingDataBuf_.ptr() == nullptr,
7789 : HCCL_ERROR(
7790 : "[HcclCommunicator][AicpuInitOpTilingDataBuf] in create opTilingDataBuf len[%llu] failed!",
7791 : opTilingDataSize),
7792 : HCCL_E_INTERNAL);
7793 : }
7794 :
7795 : // 填充固定内容
7796 1 : HostMem opTilingDataMem = opTilingDataBuf_.range(0, opTilingDataSize);
7797 1 : struct OpTilingData* opTilingData = static_cast<struct OpTilingData*>(opTilingDataMem.ptr());
7798 1 : u32 algTypeTranfer
7799 1 : = (static_cast<u32>(opTilingInfo.algType.algoLevel2) << (HCCL_LEVEL_ALGO_WIDTH + HCCL_LEVEL_ALGO_WIDTH))
7800 1 : + (static_cast<u32>(opTilingInfo.algType.algoLevel1) << HCCL_LEVEL_ALGO_WIDTH)
7801 1 : + static_cast<u32>(opTilingInfo.algType.algoLevel0);
7802 1 : opTilingData->algType = static_cast<u64>(algTypeTranfer);
7803 1 : opTilingData->floatOverflowMode = opTilingInfo.floatOverflowMode;
7804 1 : opTilingData->dumpDebug = opTilingInfo.dumpDebug;
7805 1 : CHK_RET(AicpuInitOpTilingDataFromOpParam(opParam, opType, opTilingData));
7806 1 : opTilingData->length = dynamicDataSize;
7807 1 : opTilingData->customDataLength = 0;
7808 1 : opTilingData->index = UpdateOpIndex(opParam);
7809 1 : opTilingData->debugMode = 0;
7810 1 : opTilingData->isZeroCopy = opParam.isZeroCopy;
7811 1 : opTilingData->isCapture = opParam.isCapture;
7812 1 : opTilingData->orderLaunchMode = GetOrderLaunchMode(opParam.isCapture);
7813 1 : opTilingData->isSymmetricMemory = opParam.supportSymmetricMemory;
7814 1 : opTilingData->needIncreLink = opParam.needIncreLink;
7815 : // 有没有存在对应的Notify
7816 1 : CHK_RET(InitAndCheckAicpuOrderNotify(opTilingData->orderLaunchMode));
7817 1 : CHK_RET(BuildHierarchicalAlgOption(opTilingData->ahcConfInfo));
7818 1 : CHK_RET(AicpuInitOpTilingDataAicpuCache(opParam, opType, opTilingData));
7819 :
7820 : // 填充动态内容
7821 1 : HostMem dynamicDataMem = opTilingDataBuf_.range(sizeof(struct OpTilingData), dynamicDataSize);
7822 1 : CHK_PTR_NULL(dynamicDataMem.ptr());
7823 1 : if (opType == HcclCMDType::HCCL_CMD_BATCH_SEND_RECV) {
7824 : struct OpTilingBatchSendRecvDataDes* batchSendRecvDataPtr
7825 1 : = reinterpret_cast<struct OpTilingBatchSendRecvDataDes*>(dynamicDataMem.ptr());
7826 1 : batchSendRecvDataPtr->itemNum = opParam.BatchSendRecvDataDes.itemNum;
7827 2 : for (u32 i = 0; i < opParam.BatchSendRecvDataDes.itemNum; i++) {
7828 1 : CHK_PTR_NULL(opParam.BatchSendRecvDataDes.sendRecvItemsPtr + i);
7829 1 : batchSendRecvDataPtr->batchSendRecvItem[i] = *(opParam.BatchSendRecvDataDes.sendRecvItemsPtr + i);
7830 : }
7831 :
7832 1 : u8* isDirectRemoteRankPtr
7833 1 : = reinterpret_cast<u8*>(batchSendRecvDataPtr->batchSendRecvItem + opParam.BatchSendRecvDataDes.itemNum);
7834 3 : for (u32 i = 0; i < userRankSize_; i++) {
7835 2 : CHK_PTR_NULL(isDirectRemoteRankPtr + i);
7836 2 : isDirectRemoteRankPtr[i] = *(opParam.BatchSendRecvDataDes.isDirectRemoteRank + i);
7837 : }
7838 0 : } else if (opType == HcclCMDType::HCCL_CMD_ALLTOALL) {
7839 0 : CHK_RET(SetDynamicTilingDataAlltoall(opParam, dynamicDataMem));
7840 0 : } else if (opType == HcclCMDType::HCCL_CMD_ALLTOALLV) {
7841 0 : CHK_RET(SetDynamicTilingDataAlltoallv(opParam, dynamicDataMem, opTilingInfo.algName));
7842 0 : } else if (opType == HcclCMDType::HCCL_CMD_ALLTOALLVC) {
7843 0 : CHK_RET(SetDynamicTilingDataAlltoallvc(opParam, dynamicDataMem));
7844 0 : } else if (opType == HcclCMDType::HCCL_CMD_ALLGATHER_V || opType == HcclCMDType::HCCL_CMD_REDUCE_SCATTER_V) {
7845 0 : CHK_RET(SetDynamicTilingDataV(opParam, dynamicDataMem));
7846 0 : } else {
7847 0 : struct OpTilingDataDes* opDataDesPtr = reinterpret_cast<struct OpTilingDataDes*>(dynamicDataMem.ptr());
7848 0 : opDataDesPtr->count = opParam.DataDes.count;
7849 0 : opDataDesPtr->dataType = static_cast<u8>(opParam.DataDes.dataType);
7850 : }
7851 :
7852 1 : HCCL_INFO("[HcclCommunicator][AicpuInitOpTilingDataBuf]algType[%lu]", opTilingData->algType);
7853 1 : CHK_SAFETY_FUNC_RET(memcpy_s(
7854 : opTilingData->algName, sizeof(opTilingData->algName), opTilingInfo.algName.c_str(),
7855 : opTilingInfo.algName.length() + 1));
7856 1 : CHK_SAFETY_FUNC_RET(memcpy_s(
7857 : opTilingData->newTag, sizeof(opTilingData->newTag), opTilingInfo.newTag.c_str(),
7858 : opTilingInfo.newTag.length() + 1));
7859 1 : CHK_SAFETY_FUNC_RET(
7860 : memcpy_s(opTilingData->tag, sizeof(opTilingData->tag), opParam.tag.c_str(), opParam.tag.length() + 1));
7861 1 : return HCCL_SUCCESS;
7862 1 : }
7863 :
7864 0 : u8 HcclCommunicator::GetOrderLaunchMode(bool isCapture)
7865 : {
7866 : bool isSupportHcomAttachedStream
7867 0 : = !(attachedStreams_.empty() || attachedStreams_[0].ptr() == nullptr); // true 表示图模式下成功申请附属从流
7868 0 : u8 orderLaunchMode = 0;
7869 0 : HcclWorkflowMode mode = GetWorkflowMode();
7870 0 : if (isCapture) {
7871 0 : orderLaunchMode = static_cast<u8>(AicpuNotifyMode::ACLGRAPH_MODE);
7872 0 : } else if (mode == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE) {
7873 0 : orderLaunchMode = static_cast<u8>(AicpuNotifyMode::OPBASE_MODE);
7874 0 : } else if (mode == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OPS_KERNEL_INFO_LIB && isSupportHcomAttachedStream) {
7875 0 : orderLaunchMode = static_cast<u8>(AicpuNotifyMode::HCOM_MODE);
7876 : } else {
7877 0 : orderLaunchMode = AICPU_ORDERLAUNCH_INVALID_HCOM_MODE;
7878 : }
7879 :
7880 0 : return orderLaunchMode;
7881 : }
7882 :
7883 0 : HcclResult HcclCommunicator::InitAndCheckAicpuOrderNotify(u8& orderLaunchMode)
7884 : {
7885 0 : if (orderLaunchMode == AICPU_ORDERLAUNCH_INVALID_HCOM_MODE) {
7886 0 : HCCL_INFO("[HcclCommunicator][InitAndCheckAicpuOrderNotify] orderLaunchMode is invalid in hcom "
7887 : "for there is no attached stream included in this operator!");
7888 0 : return HCCL_SUCCESS;
7889 : }
7890 : u32 idx0;
7891 : u32 idx1;
7892 0 : if (orderLaunchMode == 0) {
7893 0 : idx0 = static_cast<u32>(AicpuLocalNotifyIdx::ORDER_INDEX_OPBASE_0);
7894 0 : idx1 = static_cast<u32>(AicpuLocalNotifyIdx::ORDER_INDEX_OPBASE_1);
7895 0 : } else if (orderLaunchMode == 1) {
7896 0 : idx0 = static_cast<u32>(AicpuLocalNotifyIdx::ORDER_INDEX_ACLGRAPH_0);
7897 0 : idx1 = static_cast<u32>(AicpuLocalNotifyIdx::ORDER_INDEX_ACLGRAPH_1);
7898 : } else {
7899 0 : idx0 = static_cast<u32>(AicpuLocalNotifyIdx::ORDER_INDEX_HCOM_0);
7900 0 : idx1 = static_cast<u32>(AicpuLocalNotifyIdx::ORDER_INDEX_HCOM_1);
7901 : }
7902 :
7903 0 : if (localAiCpuOpNotify_[idx0] != nullptr) {
7904 0 : HCCL_INFO("[%s], the orderNotify of orderLaunchMode [%u] is available", __func__, orderLaunchMode);
7905 0 : return HCCL_SUCCESS;
7906 : }
7907 : HcclSignalInfo orderSignalInfo0;
7908 0 : HcclResult ret = CreateAndGetAiCpuNotify(localAiCpuOpNotify_[idx0], orderSignalInfo0);
7909 0 : CHK_PRT_RET(
7910 : ret != HCCL_SUCCESS,
7911 : HCCL_ERROR(
7912 : "[HcclCommunicator][InitAndCheckAicpuOrderNotify]get aicpu notify [%u] errorCode[%u]", idx0,
7913 : HCCL_ERROR_CODE(ret)),
7914 : ret);
7915 :
7916 : // 按序下发(aicpu控制流 record host控制流) 使用的notify信息
7917 : HcclSignalInfo orderSignalInfo1;
7918 0 : ret = CreateAndGetAiCpuNotify(localAiCpuOpNotify_[idx1], orderSignalInfo1);
7919 0 : CHK_PRT_RET(
7920 : ret != HCCL_SUCCESS,
7921 : HCCL_ERROR(
7922 : "[HcclCommunicator][InitAndCheckAicpuOrderNotify]get aicpu notify [%u] errorCode[%u]", idx1,
7923 : HCCL_ERROR_CODE(ret)),
7924 : ret);
7925 0 : HCCL_INFO(
7926 : "[HcclCommunicator][InitAndCheckAicpuOrderNotify] ORDER INDEX 0: resId[%u], ORDER INDEX 1: resId[%u]",
7927 : orderSignalInfo0.resId, orderSignalInfo1.resId);
7928 :
7929 0 : CHK_RET(hrtMemSyncCopy(
7930 : static_cast<char*>(aicpuOrderNotifyAddr_.ptr()) + (sizeof(HcclSignalInfo) * orderLaunchMode),
7931 : sizeof(HcclSignalInfo), &orderSignalInfo1, sizeof(HcclSignalInfo),
7932 : HcclRtMemcpyKind::HCCL_RT_MEMCPY_KIND_HOST_TO_DEVICE));
7933 :
7934 0 : return HCCL_SUCCESS;
7935 : }
7936 :
7937 2 : HcclResult HcclCommunicator::AicpuKfcTilingDataLaunchIn(
7938 : const OpParam& opParam, const DeviceMem& deviceContext, const std::string& kernelName,
7939 : const AicpuOpTiling opTilingInfo, u64 opTilingDataSize, bool isCustom)
7940 : {
7941 2 : HostMem opTilingDataMem = opTilingDataBuf_.range(0, opTilingDataSize);
7942 2 : CHK_RET(SetNormalMode(dispatcher_));
7943 0 : Stream& mainStream = const_cast<Stream&>(opParam.stream);
7944 0 : CHK_RET(LocalNotify::Post(
7945 : mainStream, dispatcher_, localAiCpuOpNotify_[static_cast<u32>(AicpuLocalNotifyIdx::HOST_TO_AICPU_0)],
7946 : INVALID_VALUE_STAGE));
7947 :
7948 0 : Stream kfcOpStream;
7949 0 : HcclWorkflowMode mode = GetWorkflowMode();
7950 : bool isSupportHcomAttachedStream
7951 0 : = !(attachedStreams_.empty() || attachedStreams_[0].ptr() == nullptr); // true 表示图模式下成功申请附属从流
7952 0 : if (opParam.isCapture || mode == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE) {
7953 0 : kfcOpStream = opStream_;
7954 : } else {
7955 : // 如果是图模式，则尝试从附属从流中获取一下stream，如果能拿到则使用，否则退化
7956 0 : if (isSupportHcomAttachedStream) {
7957 0 : HCCL_INFO("[HcclCommunicator][AicpuKfcTilingDataLaunchIn] attachedStreams_ is valid in graph mode");
7958 0 : kfcOpStream = attachedStreams_[0];
7959 : } else {
7960 0 : HCCL_INFO("[HcclCommunicator][AicpuKfcTilingDataLaunchIn] attachedStreams_ is invalid in graph mode");
7961 0 : kfcOpStream = opParam.stream;
7962 : }
7963 : }
7964 0 : uint64_t beginTime = hrtMsprofSysCycleTime();
7965 0 : std::string profName = GetCMDTypeEnumStr(opParam.opType);
7966 0 : if (profName == "Invalid HcclCMDType" || profName == "invalid") {
7967 0 : profName = "HcclOpAicpuKernel";
7968 : } else {
7969 0 : profName += "AicpuKernel";
7970 : }
7971 0 : s32 streamId = kfcOpStream.id();
7972 0 : auto getAicpuTaskExceptionCallBack = [this]() {
7973 0 : return this->GetAicpuTaskException();
7974 0 : };
7975 0 : RegisterGetAicpuTaskExceptionCallBack(streamId, deviceLogicId_, getAicpuTaskExceptionCallBack);
7976 0 : aicpuStreamIds_.insert(streamId);
7977 0 : if (streamId != opParam.stream.id()) {
7978 0 : RegisterGetAicpuTaskExceptionCallBack(opParam.stream.id(), deviceLogicId_, getAicpuTaskExceptionCallBack);
7979 0 : aicpuStreamIds_.insert(opParam.stream.id());
7980 : }
7981 :
7982 0 : HCCL_INFO(
7983 : "%s profName[%s] tag[%s] kfcOpStreamId[%d] mainStreamId[%u] kfcStreamId[%d] isCapture[%d] mode[%d] ", __func__,
7984 : profName.c_str(), opParam.tag.c_str(), streamId, opParam.stream.id(), opStream_.id(), opParam.isCapture, mode);
7985 :
7986 0 : if (opParam.isCapture) { // 非主流下发时，acl graph场景，capture从流
7987 0 : u64 modelId = UINT64_MAX;
7988 0 : rtModel_t rtModel = nullptr;
7989 0 : bool isCapture = false;
7990 0 : CHK_RET(GetStreamCaptureInfo(opParam.stream.ptr(), rtModel, isCapture));
7991 0 : CHK_PTR_NULL(rtModel);
7992 0 : CHK_RET(AddStreamToModel(kfcOpStream.ptr(), rtModel));
7993 :
7994 0 : CHK_RET(GetModelId(rtModel, modelId));
7995 0 : HCCL_INFO(
7996 : "[HcclCommunicator][%s]tag[%s], add stream[%d] to modelId[%llu] success.", __func__, opParam.tag.c_str(),
7997 : streamId, modelId);
7998 : }
7999 :
8000 0 : u32 timeOut = (opResPara_.config.notifyWaitTime == 0) ? opResPara_.config.notifyWaitTime :
8001 0 : (opResPara_.config.notifyWaitTime + AICPU_H2D_TIMEOUT_INC);
8002 0 : OrderLaunch& orderLaunch = OrderLaunch::GetInstance(deviceLogicId_);
8003 0 : std::shared_ptr<LocalNotify> notify0;
8004 0 : std::shared_ptr<LocalNotify> notify1;
8005 0 : if (opParam.isCapture) {
8006 0 : notify0 = localAiCpuOpNotify_[static_cast<u32>(AicpuLocalNotifyIdx::ORDER_INDEX_ACLGRAPH_0)];
8007 0 : notify1 = localAiCpuOpNotify_[static_cast<u32>(AicpuLocalNotifyIdx::ORDER_INDEX_ACLGRAPH_1)];
8008 0 : HcclRtEvent event0 = localAicpuOpEvent_[static_cast<u32>(AicpuLocalEventIdx::ORDER_INDEX_ACLGRAPH_EVENT_0)];
8009 0 : CHK_RET(orderLaunch.AclgraphLaunchInOrderToOrderStream(
8010 : identifier_, kfcOpStream, notify0, notify1, timeOut, event0));
8011 0 : } else if (mode == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE) {
8012 0 : notify0 = localAiCpuOpNotify_[static_cast<u32>(AicpuLocalNotifyIdx::ORDER_INDEX_OPBASE_0)];
8013 0 : notify1 = localAiCpuOpNotify_[static_cast<u32>(AicpuLocalNotifyIdx::ORDER_INDEX_OPBASE_1)];
8014 0 : CHK_RET(orderLaunch.OpbaseLaunchInOrder(identifier_, kfcOpStream, notify0, notify1, timeOut));
8015 0 : } else if (mode == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OPS_KERNEL_INFO_LIB && isSupportHcomAttachedStream) {
8016 0 : notify0 = localAiCpuOpNotify_[static_cast<u32>(AicpuLocalNotifyIdx::ORDER_INDEX_HCOM_0)];
8017 0 : notify1 = localAiCpuOpNotify_[static_cast<u32>(AicpuLocalNotifyIdx::ORDER_INDEX_HCOM_1)];
8018 0 : CHK_RET(orderLaunch.HcomLaunchInOrder(identifier_, kfcOpStream, graphId_, notify0, notify1, timeOut));
8019 : }
8020 0 : CHK_RET(KernelLaunchChooseAicpuOrCustom(
8021 : opParam.inputPtr, opParam.outputPtr, kfcOpStream.ptr(), reinterpret_cast<u64>(deviceContext.ptr()),
8022 : opTilingDataMem.ptr(), opTilingDataSize, kernelName, mode, opParam.tag, isCustom));
8023 0 : if (opParam.isCapture) {
8024 0 : HcclRtEvent event1 = localAicpuOpEvent_[static_cast<u32>(AicpuLocalEventIdx::ORDER_INDEX_ACLGRAPH_EVENT_1)];
8025 0 : CHK_RET(orderLaunch.AclgraphLaunchInOrderToKernelStream(identifier_, kfcOpStream, event1));
8026 : }
8027 :
8028 0 : uint64_t endTime = hrtMsprofSysCycleTime();
8029 0 : s32 threadId = SalGetTid();
8030 0 : CHK_RET(ProfilingManagerPub::CallMsprofReportNodeInfo(beginTime, endTime, profName, threadId));
8031 0 : CHK_RET(LocalNotify::Wait(
8032 : mainStream, dispatcher_, localAiCpuOpNotify_[static_cast<u32>(AicpuLocalNotifyIdx::HOST_TO_AICPU_1)],
8033 : INVALID_VALUE_STAGE, timeOut));
8034 0 : return HCCL_SUCCESS;
8035 2 : }
8036 :
8037 0 : HcclResult HcclCommunicator::SetAttachedStream(u32 graphId, const std::vector<rtStream_t>& streams)
8038 : {
8039 0 : constexpr u32 GRAPH_ATTACHED_STREAM_INDEX = 0; // 图粒度的附属从流
8040 0 : constexpr u32 GROUP_ATTACHED_STREAM_INDEX = 1; // 通信域粒度的附属从流
8041 :
8042 : // 在图模式下，通信使用的附属从流可能不同，所以这里直接刷新所有
8043 0 : attachedStreams_.clear();
8044 :
8045 0 : bool isValid = !streams.empty() && (streams.size() > GROUP_ATTACHED_STREAM_INDEX)
8046 0 : && streams[GRAPH_ATTACHED_STREAM_INDEX] != nullptr
8047 0 : && streams[GROUP_ATTACHED_STREAM_INDEX] != nullptr;
8048 0 : if (!isValid) {
8049 0 : HCCL_ERROR("%s Invalid stream configuration, streams vector is null or invalid", __func__);
8050 0 : return HCCL_E_NOT_FOUND;
8051 : }
8052 :
8053 : // 向GE申请流的时候，图粒度的流排在第一个，所以在streams列表中，第一条流是图粒度的附属从流
8054 0 : s32 graphAttachedStreamId = 0;
8055 0 : OrderLaunch& orderLaunch = OrderLaunch::GetInstance(deviceLogicId_);
8056 0 : auto& graphStream = streams[GRAPH_ATTACHED_STREAM_INDEX];
8057 0 : CHK_RET(hrtGetStreamId(graphStream, graphAttachedStreamId));
8058 0 : orderLaunch.SetHcomStream(graphId, Stream(graphStream, false));
8059 0 : graphId_ = graphId;
8060 :
8061 : // 设置通信域粒度流
8062 0 : auto& groupStream = streams[GROUP_ATTACHED_STREAM_INDEX];
8063 0 : attachedStreams_.emplace_back(Stream(groupStream, false));
8064 :
8065 0 : HCCL_INFO(
8066 : "%s Streams configured graph[%u], graphAttachedStreamId[%d], group[%u],"
8067 : "groupStreamId[%u], graphId[%u], groupId[%s]",
8068 : __func__, GRAPH_ATTACHED_STREAM_INDEX, graphAttachedStreamId, GROUP_ATTACHED_STREAM_INDEX,
8069 : attachedStreams_.back().id(), graphId, identifier_.c_str());
8070 :
8071 0 : return HCCL_SUCCESS;
8072 : }
8073 :
8074 4 : HcclResult HcclCommunicator::AicpuKfcTilingDataLaunchExt(
8075 : const OpParam& opParam, const HcclCMDType& opType, const DeviceMem& deviceContext, const std::string& kernelName,
8076 : const AicpuOpTiling opTilingInfo, bool isCustom)
8077 : {
8078 4 : const u64 dataCount = opParam.GetDataCount(userRank_);
8079 4 : const HcclDataType dataType = opParam.GetDataType();
8080 4 : HCCL_DEBUG(
8081 : "AicpuKfcTilingDataLaunchExt count %llu dataType %s op %s opType %u retryEnable_ %d, "
8082 : "inPlaceSupportRetryStatus_ %d",
8083 : dataCount, GetDataTypeEnumStr(dataType).c_str(), GetReduceOpEnumStr(opParam.reduceType).c_str(), opType,
8084 : retryEnable_, inPlaceSupportRetryStatus_);
8085 :
8086 4 : bool postSyncEnable = false;
8087 4 : u32 severNum4PostSync = 4;
8088 1 : bool needPostSync = (superPodNum_ > 1 || serverNum_ >= severNum4PostSync)
8089 5 : && postSyncEnable; // reduce/reduce scatter算子是否需要PostSync
8090 4 : if (opType == HcclCMDType::HCCL_CMD_ALLREDUCE && retryEnable_
8091 1 : && (inPlaceSupportRetryStatus_ == InplaceSupportRetryStatus::USER_LARGER_THAN_CCL) && (!opParam.isZeroCopy)) {
8092 1 : u32 itemNum = 2;
8093 3 : for (u32 i = 0; i < itemNum; i++) {
8094 2 : if (i == 0) {
8095 1 : isInplacePreSync_ = true;
8096 : } else {
8097 1 : isInplacePreSync_ = false;
8098 : }
8099 2 : HCCL_DEBUG("[AicpuKfcTilingDataLaunchExt][PreSync]The op with isInplacePreSync_[%d].", isInplacePreSync_);
8100 2 : u64 dynamicDataSize = CalcOpTilingDynamicDataSize(opParam, opType, GetRankSize(), opTilingInfo.algName);
8101 2 : CHK_RET(AicpuInitOpTilingDataBuf(opParam, opType, kernelName, opTilingInfo, dynamicDataSize));
8102 2 : CHK_RET(AicpuKfcTilingDataLaunchIn(
8103 : opParam, deviceContext, kernelName, opTilingInfo, sizeof(struct OpTilingData) + dynamicDataSize,
8104 : isCustom));
8105 2 : isInplacePreSync_ = false;
8106 : }
8107 4 : } else if (opType == HcclCMDType::HCCL_CMD_REDUCE && retryEnable_ && needPostSync && (!opParam.isZeroCopy)) {
8108 0 : isPostSync_ = true;
8109 0 : HCCL_DEBUG("[AicpuKfcTilingDataLaunchExt][PreSync]The op with isPostSync_[%d].", isPostSync_);
8110 0 : u64 dynamicDataSize = CalcOpTilingDynamicDataSize(opParam, opType, GetRankSize(), opTilingInfo.algName);
8111 0 : CHK_RET(AicpuInitOpTilingDataBuf(opParam, opType, kernelName, opTilingInfo, dynamicDataSize));
8112 0 : CHK_RET(AicpuKfcTilingDataLaunchIn(
8113 : opParam, deviceContext, kernelName, opTilingInfo, sizeof(struct OpTilingData) + dynamicDataSize, isCustom));
8114 0 : isPostSync_ = false;
8115 3 : } else if (retryEnable_ && opType == HcclCMDType::HCCL_CMD_REDUCE_SCATTER && (!opParam.isZeroCopy)) {
8116 1 : if (inPlaceSupportRetryStatus_ == InplaceSupportRetryStatus::USER_LARGER_THAN_CCL) {
8117 1 : isInplacePreSync_ = true;
8118 1 : HCCL_DEBUG("[AicpuKfcTilingDataLaunchExt][PreSync]The op with isInplacePreSync_[%d].", isInplacePreSync_);
8119 1 : u64 dynamicDataSize = CalcOpTilingDynamicDataSize(opParam, opType, GetRankSize(), opTilingInfo.algName);
8120 1 : CHK_RET(AicpuInitOpTilingDataBuf(opParam, opType, kernelName, opTilingInfo, dynamicDataSize));
8121 1 : CHK_RET(AicpuKfcTilingDataLaunchIn(
8122 : opParam, deviceContext, kernelName, opTilingInfo, sizeof(struct OpTilingData) + dynamicDataSize,
8123 : isCustom));
8124 1 : isInplacePreSync_ = false;
8125 : }
8126 1 : isInplacePreSync_ = false;
8127 1 : if (needPostSync) {
8128 0 : isPostSync_ = true;
8129 : }
8130 1 : HCCL_DEBUG(
8131 : "[AicpuKfcTilingDataLaunchExt][PreSync]The op with "
8132 : "isInplacePreSync_[%d], isPostSync_[%d].",
8133 : isInplacePreSync_, isPostSync_);
8134 1 : u64 dynamicDataSize = CalcOpTilingDynamicDataSize(opParam, opType, GetRankSize(), opTilingInfo.algName);
8135 1 : CHK_RET(AicpuInitOpTilingDataBuf(opParam, opType, kernelName, opTilingInfo, dynamicDataSize));
8136 1 : CHK_RET(AicpuKfcTilingDataLaunchIn(
8137 : opParam, deviceContext, kernelName, opTilingInfo, sizeof(struct OpTilingData) + dynamicDataSize, isCustom));
8138 1 : isPostSync_ = false;
8139 1 : } else if (
8140 2 : retryEnable_
8141 2 : && (opType == HcclCMDType::HCCL_CMD_ALLTOALL || opType == HcclCMDType::HCCL_CMD_ALLTOALLV
8142 1 : || opType == HcclCMDType::HCCL_CMD_ALLTOALLVC)
8143 1 : && (!opParam.isZeroCopy)) {
8144 1 : isPostSync_ = postSyncEnable;
8145 1 : HCCL_DEBUG(
8146 : "[AicpuKfcTilingDataLaunchExt][PreSync]The op with "
8147 : "isInplacePreSync_[%d], isPostSync_[%d].",
8148 : isInplacePreSync_, isPostSync_);
8149 1 : u64 dynamicDataSize = CalcOpTilingDynamicDataSize(opParam, opType, GetRankSize(), opTilingInfo.algName);
8150 1 : CHK_RET(AicpuInitOpTilingDataBuf(opParam, opType, kernelName, opTilingInfo, dynamicDataSize));
8151 1 : CHK_RET(AicpuKfcTilingDataLaunchIn(
8152 : opParam, deviceContext, kernelName, opTilingInfo, sizeof(struct OpTilingData) + dynamicDataSize, isCustom));
8153 1 : isPostSync_ = false;
8154 1 : } else {
8155 1 : u64 dynamicDataSize = CalcOpTilingDynamicDataSize(opParam, opType, GetRankSize(), opTilingInfo.algName);
8156 1 : HCCL_DEBUG("[AicpuKfcTilingDataLaunchExt]dynamicDataSize[%u]", dynamicDataSize);
8157 1 : CHK_RET(AicpuInitOpTilingDataBuf(opParam, opType, kernelName, opTilingInfo, dynamicDataSize));
8158 1 : CHK_RET(AicpuKfcTilingDataLaunchIn(
8159 : opParam, deviceContext, kernelName, opTilingInfo, sizeof(struct OpTilingData) + dynamicDataSize, isCustom));
8160 : }
8161 :
8162 4 : return HCCL_SUCCESS;
8163 : }
8164 :
8165 0 : HcclResult HcclCommunicator::AicpuUnfoldKernelLaunch(
8166 : void* inputPtr, void* outputPtr, const rtStream_t stm, u64 addr, void* tilingDataPtr, u32 tilingDataSize,
8167 : const std::string& kernelName, HcclWorkflowMode mode, const std::string& tag)
8168 : {
8169 : struct ApiParamDef {
8170 : uint64_t x1; // 算子sendbuffer地址
8171 : uint64_t y = 0;
8172 : uint64_t gatherOut; // 算子recvbuffer地址
8173 : uint64_t context; // 通信资源准备的地址
8174 : uint64_t workspace; // 消息区地址
8175 : };
8176 :
8177 0 : struct ApiParamDef apiParam;
8178 0 : apiParam.x1 = reinterpret_cast<uint64_t>(inputPtr);
8179 0 : apiParam.gatherOut = reinterpret_cast<uint64_t>(outputPtr);
8180 0 : apiParam.context = addr;
8181 0 : apiParam.workspace = reinterpret_cast<uint64_t>(workSpace_.ptr());
8182 0 : u16 timeOut = 0;
8183 0 : if (opResPara_.config.notifyWaitTime == 0) {
8184 0 : timeOut = opResPara_.config.notifyWaitTime;
8185 0 : } else if (opResPara_.config.notifyWaitTime + AICPU_KERNEL_TIMEOUT_INC >= MAX_VALUE_U16) {
8186 0 : timeOut = MAX_VALUE_U16;
8187 : } else {
8188 0 : timeOut = opResPara_.config.notifyWaitTime + AICPU_KERNEL_TIMEOUT_INC;
8189 : }
8190 0 : CHK_PRT(AicpuAclKernelLaunch(
8191 : stm, reinterpret_cast<void*>(&apiParam), sizeof(apiParam), binHandle_, kernelName, false, timeOut,
8192 : tilingDataPtr, tilingDataSize));
8193 0 : HCCL_INFO("[HcclCommunicator][AicpuUnfoldKernelLaunch] exec succ.");
8194 0 : return HCCL_SUCCESS;
8195 : }
8196 :
8197 0 : HcclResult HcclCommunicator::AicpuUnfoldKernelLaunchV2(
8198 : void* inputPtr, void* outputPtr, const rtStream_t stm, u64 addr, void* tilingDataPtr, u32 tilingDataSize,
8199 : const std::string& kernelName, HcclWorkflowMode mode, const std::string& tag, bool isCustom)
8200 : {
8201 0 : u64 context = addr;
8202 0 : HCCL_INFO("[HcclCommunicator]context[%p] tilingDataPtr[%p] tilingData[%p]", context, tilingDataPtr, tilingDataSize);
8203 :
8204 0 : aclrtBinHandle binHandle = isCustom ? binCustomHandle_ : binHandle_;
8205 0 : if (binHandle == nullptr) {
8206 0 : HCCL_ERROR("[AicpuUnfoldKernelLaunchV2]isCustom[%d] binHandle is nullptr, please check.", isCustom);
8207 0 : return HCCL_E_NOT_SUPPORT;
8208 : }
8209 0 : u16 timeOut = 0;
8210 0 : if (opResPara_.config.notifyWaitTime == 0) {
8211 0 : timeOut = opResPara_.config.notifyWaitTime;
8212 0 : } else if (opResPara_.config.notifyWaitTime + AICPU_KERNEL_TIMEOUT_INC >= MAX_VALUE_U16) {
8213 0 : timeOut = MAX_VALUE_U16;
8214 : } else {
8215 0 : timeOut = opResPara_.config.notifyWaitTime + AICPU_KERNEL_TIMEOUT_INC;
8216 : }
8217 0 : HcclResult ret = AicpuAclKernelLaunchV2(
8218 : stm, reinterpret_cast<void*>(&context), sizeof(context), binHandle, kernelName, false, timeOut, tilingDataPtr,
8219 0 : tilingDataSize, identifier_);
8220 0 : CHK_PRT_RET(
8221 : ret != HCCL_SUCCESS,
8222 : HCCL_ERROR("[HcclCommunicator][AicpuUnfoldKernelLaunchV2]isCustom[%d] binHandle[%p]", isCustom, binHandle),
8223 : ret);
8224 0 : HCCL_INFO("[HcclCommunicator][AicpuUnfoldKernelLaunchV2] exec succ, isCustom[%d].", isCustom);
8225 0 : return HCCL_SUCCESS;
8226 : }
8227 :
8228 490 : HcclResult HcclCommunicator::InitCombinOpara()
8229 : {
8230 490 : if (combinOparaMem_ == nullptr) {
8231 490 : CHK_RET(AllocAndClearHostMem(sizeof(HcclCombinOpParam), combinOparaMem_));
8232 : }
8233 491 : CHK_PTR_NULL(combinOparaMem_);
8234 491 : HcclCombinOpParam* combinOparaPtr = reinterpret_cast<HcclCombinOpParam*>(combinOparaMem_->ptr());
8235 491 : CHK_PTR_NULL(combinOparaPtr);
8236 :
8237 491 : if (aiRMAInfoMem_ == nullptr) {
8238 490 : CHK_RET(AllocAndClearHostMem(sizeof(HcclAiRMAInfo), aiRMAInfoMem_));
8239 : }
8240 490 : if (rmaInfoMem_ == nullptr) {
8241 491 : CHK_RET(AllocAndClearHostMem(sizeof(HcclRMAInfo), rmaInfoMem_));
8242 : }
8243 489 : CHK_PTR_NULL(aiRMAInfoMem_);
8244 489 : CHK_PTR_NULL(aiRMAInfoMem_->ptr());
8245 489 : CHK_PTR_NULL(rmaInfoMem_);
8246 489 : CHK_PTR_NULL(rmaInfoMem_->ptr());
8247 :
8248 489 : CHK_SAFETY_FUNC_RET(memset_s(combinOparaPtr, sizeof(HcclCombinOpParam), 0, sizeof(HcclCombinOpParam)));
8249 :
8250 491 : combinOparaPtr->rankId = INVALID_UINT;
8251 491 : combinOparaPtr->signalInfo.aicpuNotify.rankId = INVALID_UINT;
8252 :
8253 491 : for (u32 i = 0;
8254 31851 : i < sizeof(combinOparaPtr->signalInfo.noIpcNotifys) / sizeof(combinOparaPtr->signalInfo.noIpcNotifys[0]);
8255 : i++) {
8256 31360 : combinOparaPtr->signalInfo.noIpcNotifys[i].rankId = INVALID_UINT;
8257 : }
8258 :
8259 491 : for (u32 i = 0;
8260 63212 : i < sizeof(combinOparaPtr->signalInfo.ipcNotifys) / sizeof(combinOparaPtr->signalInfo.ipcNotifys[0]); i++) {
8261 62721 : combinOparaPtr->signalInfo.ipcNotifys[i].rankId = INVALID_UINT;
8262 : }
8263 :
8264 491 : for (u32 i = 0;
8265 16203 : i < sizeof(combinOparaPtr->signalInfo.noIpcEvents) / sizeof(combinOparaPtr->signalInfo.noIpcEvents[0]); i++) {
8266 15712 : combinOparaPtr->signalInfo.noIpcEvents[i].rankId = INVALID_UINT;
8267 : }
8268 491 : return HCCL_SUCCESS;
8269 : }
8270 :
8271 7 : bool HcclCommunicator::GetCommResource(const std::string& tag, void** commContext)
8272 : {
8273 7 : if (LIKELY(IsExistCommRes(tag))) {
8274 0 : *commContext = commContext_.ptr();
8275 0 : return true;
8276 : }
8277 7 : return false;
8278 : }
8279 :
8280 7 : bool HcclCommunicator::GetCommResource(void*& commContext)
8281 : {
8282 7 : commContext = opResDevicePara_.ptr();
8283 7 : return true;
8284 : }
8285 :
8286 0 : HcclResult HcclCommunicator::GetAicpuOpStreamNotify(HcclRtStream* opStream, u8 aicpuNotifyNum, void** aicpuNotify)
8287 : {
8288 0 : CHK_RET(GetAicpuOpStreamAndNotify(opStream, aicpuNotifyNum, aicpuNotify));
8289 0 : HCCL_INFO("[HcclCommunicator][GetAicpuOpStreamNotify]opStream %p aicpuNotify %p.", *opStream, *aicpuNotify);
8290 0 : return HCCL_SUCCESS;
8291 : }
8292 :
8293 0 : HcclResult HcclCommunicator::GetAicpuOpStreamAndNotify(HcclRtStream* opStream, u8 aicpuNotifyNum, void** aicpuNotify)
8294 : {
8295 0 : *opStream = opStream_.ptr();
8296 0 : if (localAiCpuNotifyRes_.size() < aicpuNotifyNum) {
8297 0 : for (u16 i = localAiCpuNotifyRes_.size(); i < aicpuNotifyNum; i++) {
8298 0 : std::shared_ptr<LocalNotify> localNotify = {nullptr};
8299 : HcclSignalInfo aicpuNotify;
8300 0 : CHK_RET(CreateAndGetAiCpuNotify(localNotify, aicpuNotify));
8301 0 : localAiCpuNotifyRes_.push_back(localNotify);
8302 0 : }
8303 : }
8304 :
8305 0 : for (u16 i = 0; i < aicpuNotifyNum; i++) {
8306 0 : *(aicpuNotify + i) = localAiCpuNotifyRes_[i]->ptr();
8307 : }
8308 0 : return HCCL_SUCCESS;
8309 : }
8310 :
8311 0 : HcclResult HcclCommunicator::SetAicpuNotifyInvalid()
8312 : {
8313 0 : CHK_PTR_NULL(combinOparaMem_);
8314 0 : HcclCombinOpParam* combinOparaPtr = reinterpret_cast<HcclCombinOpParam*>(combinOparaMem_->ptr());
8315 0 : CHK_PTR_NULL(combinOparaPtr);
8316 0 : combinOparaPtr->signalInfo.aicpuNotify.resId = INVALID_U64;
8317 0 : return HCCL_SUCCESS;
8318 : }
8319 :
8320 1 : HcclResult HcclCommunicator::ReplaceCommInfoByTag(const std::string& tag, std::unique_ptr<CommInfo>& commInfo)
8321 : {
8322 1 : std::unique_lock<std::mutex> replLock(commLock_);
8323 1 : tagCommInfo_.erase(tag);
8324 1 : tagCommInfo_.insert(std::pair<std::string, CommInfo>(tag, std::move(*commInfo)));
8325 1 : return HCCL_SUCCESS;
8326 1 : }
8327 :
8328 0 : HcclResult HcclCommunicator::CreateMutiStreamResFor310P(const std::string& tag, level1StreamInfo_t& streamInfo)
8329 : {
8330 0 : u32 rankSize = GetRankSize();
8331 : s32 pid;
8332 0 : if (SalGetBareTgid(&pid) != HCCL_SUCCESS) {
8333 0 : HCCL_DEBUG("get pid fail");
8334 : }
8335 0 : HCCL_INFO(
8336 : "[HcclCommunicator][CreateMutiStreamRes]tag[%s] ranksize[%u] comminfo ranksize[%u] "
8337 : "auxRingCommStreamsDev_ size[%u] ringDeviceSignalAux size[%u] ringDeviceSignal size[%u] "
8338 : "ringDeviceStreams size[%u]",
8339 : tag.c_str(), rankSize, tagCommInfo_[tag].commIntraServer->RankSize(), auxRingCommStreamsDev_.size(),
8340 : streamInfo.ringDeviceSignalAux.size(), streamInfo.ringDeviceSignal.size(), streamInfo.ringDeviceStreams.size());
8341 0 : if (auxRingCommStreamsDev_.empty() || auxRingCommStreamsDev_.size() < rankSize) {
8342 0 : auxRingCommStreamsDev_.resize(rankSize);
8343 0 : u32 resNum = rankSize - 1;
8344 0 : streamInfo.ringDeviceSignalAux.resize(resNum);
8345 0 : streamInfo.ringDeviceSignal.resize(resNum);
8346 0 : for (u32 ringIndex = 0; ringIndex < rankSize; ringIndex++) {
8347 0 : auxRingCommStreamsDev_[ringIndex] = Stream(StreamType::STREAM_TYPE_DEVICE);
8348 : // 给device侧申请的流不需要setmode，否则rts会捕获流成员Flags为1024的异常
8349 : }
8350 0 : for (auto& signal : streamInfo.ringDeviceSignal) {
8351 0 : signal = nullptr;
8352 : }
8353 0 : for (auto& signal : streamInfo.ringDeviceSignalAux) {
8354 0 : signal = nullptr;
8355 : }
8356 :
8357 0 : u32 notifyNum = resNum * 2; // 2:Signal + SignalAux
8358 0 : std::vector<std::shared_ptr<LocalNotify>> notifys(notifyNum, nullptr);
8359 0 : CHK_RET(queueNotifyManager_->Alloc(tag, notifyNum, notifys, NotifyLoadType::DEVICE_NOTIFY));
8360 0 : for (u32 i = 0; i < resNum; i++) {
8361 0 : streamInfo.ringDeviceSignal[i] = notifys[2 * i];
8362 0 : streamInfo.ringDeviceSignalAux[i] = notifys[2 * i + 1];
8363 : }
8364 0 : }
8365 :
8366 0 : if (streamInfo.ringDeviceStreams.empty() || streamInfo.ringDeviceStreams.size() < rankSize) {
8367 0 : streamInfo.ringDeviceStreams.resize(rankSize);
8368 0 : for (u32 ringIndex = 0; ringIndex < rankSize; ringIndex++) {
8369 0 : streamInfo.ringDeviceStreams[ringIndex] = auxRingCommStreamsDev_[ringIndex];
8370 0 : CHK_SMART_PTR_NULL(streamInfo.ringDeviceStreams[ringIndex]);
8371 : }
8372 : }
8373 0 : return HCCL_SUCCESS;
8374 : }
8375 :
8376 1 : HcclResult HcclCommunicator::CreateCommAndStreamRes(const std::string& tag, Stream& stream)
8377 : {
8378 1 : CHK_SMART_PTR_NULL(implAlg_);
8379 1 : void* commInputPtr = nullptr;
8380 1 : void* commOutputPtr = nullptr;
8381 : u64 commInputSize, commOutputSize;
8382 :
8383 1 : HcclResult ret = CreateCommCCLbuffer();
8384 1 : CHK_PRT_RET(
8385 : ret != HCCL_SUCCESS,
8386 : HCCL_ERROR(
8387 : "[HcclImplBase][CreateCommAndStreamRes]errNo[0x%016llx],create cclbuff failed", HCCL_ERROR_CODE(ret)),
8388 : ret);
8389 :
8390 1 : if (isA2MC2MultiServer_) {
8391 : // 该场景下ccl buffer有一块区域在上层会被用作flag区，因此需要先清理一下
8392 0 : CHK_RET(cclBufferManager_.CleanCCLbuffer());
8393 : }
8394 :
8395 1 : CHK_RET(cclBufferManager_.GetInCCLbuffer(commInputPtr, commInputSize));
8396 1 : CHK_RET(cclBufferManager_.GetOutCCLbuffer(commOutputPtr, commOutputSize));
8397 1 : DeviceMem expMem = cclBufferManager_.GetCommExpBuffer();
8398 1 : DeviceMem inputMem = DeviceMem::create(commInputPtr, commInputSize);
8399 1 : DeviceMem outputMem = DeviceMem::create(commOutputPtr, commOutputSize);
8400 1 : AlgType algType;
8401 1 : AlgType algTypeTmp;
8402 :
8403 1 : CHK_RET(GetAlgType(algType, HcclCMDType::HCCL_CMD_ALL));
8404 1 : algTypeTmp = algType;
8405 :
8406 1 : CHK_RET(notifyPool_->RegisterOp(tag));
8407 :
8408 : // 根据tag创建comm和流资源
8409 1 : if (!(IsExistCommRes(tag))) {
8410 1 : std::unique_ptr<CommInfo> commInfo = nullptr;
8411 1 : std::vector<u32>& ranksPorts = groupNicRanksPort_.empty() ? nicRanksPort_ : groupNicRanksPort_;
8412 1 : std::vector<u32>& vnicRanksPorts = groupVnicRanksPort_.empty() ? vnicRanksPort_ : groupVnicRanksPort_;
8413 1 : implAlg_->SetHDCModeInfo(
8414 1 : rankDevicePhyIdNicInfoMap_, ranksPorts, vnicRanksPorts, isSetHDCModeInfo_, isUseRankPort_);
8415 : HcclResult ret
8416 1 : = implAlg_->CreateComm(tag, inputMem, outputMem, algType, commInfo, INVALID_VALUE_RANKID, false, true);
8417 :
8418 1 : CHK_PRT_RET(
8419 : ret != HCCL_SUCCESS,
8420 : HCCL_ERROR(
8421 : "[HcclCommunicator][CreateCommAndStreamRes]errNo[0x%016llx]tag[%s],comm resource create comm failed",
8422 : HCCL_ERROR_CODE(ret), tag.c_str()),
8423 : ret);
8424 :
8425 1 : CHK_RET(ReplaceCommInfoByTag(tag, commInfo));
8426 1 : if (isA2MC2MultiServer_ && isA2MC2IntraHie_) {
8427 0 : std::string hieSuffix = "_HIE";
8428 0 : size_t pos = tag.find(hieSuffix);
8429 0 : std::string oldtag = tag;
8430 0 : oldtag.erase(pos, hieSuffix.size());
8431 0 : CHK_RET(ReplaceCommInfoByTag(oldtag, commInfo));
8432 0 : }
8433 1 : }
8434 :
8435 1 : if (!(IsExistMutiStreamRes(tag))) {
8436 1 : level1StreamInfo_t streamInfo;
8437 1 : std::unique_lock<std::mutex> mutiStreamLock(tagStreamInfoLock_);
8438 : // 2p场景下，mc2当前algType为518，streamInfo.ringNum走默认流程值为1导致资源申请不足，910_93
8439 : // mc2固定在节点内默认用mesh
8440 1 : constexpr u32 RANK_SIZE_TWO = 2;
8441 1 : if ((GetRankSize() == RANK_SIZE_TWO && !isA2MC2MultiServer_) || (deviceType_ == DevType::DEV_TYPE_910_93)) {
8442 0 : algTypeTmp.algoLevel0 = AlgTypeLevel0::ALG_LEVEL0_NP_MESH;
8443 0 : algTypeTmp.algoLevel1 = AlgTypeLevel1::ALG_LEVEL1_RING;
8444 : }
8445 1 : HcclResult ret = HCCL_SUCCESS;
8446 1 : if (Is310P3Common(isHaveCpuRank_, deviceType_)) {
8447 0 : ret = CreateMutiStreamResFor310P(tag, streamInfo);
8448 : } else {
8449 1 : ret = implAlg_->CreateMutiStreamRes(tag, stream, streamInfo, algTypeTmp, true);
8450 : }
8451 1 : CHK_PRT_RET(
8452 : ret != HCCL_SUCCESS,
8453 : HCCL_ERROR(
8454 : "[HcclCommunicator][CreateCommAndStreamRes]errNo[0x%016llx]tag[%s],comm resource create stream "
8455 : "resource",
8456 : HCCL_ERROR_CODE(ret), tag.c_str()),
8457 : ret);
8458 1 : tagStreamInfo_.insert(std::pair<std::string, Level1StreamInfo>(tag, std::move(streamInfo)));
8459 1 : opRetryStreamPtr_->insert(std::make_pair(tag, tagStreamInfo_[tag].ringDeviceStreams));
8460 1 : mutiStreamLock.unlock();
8461 1 : }
8462 :
8463 1 : HCCL_INFO("resource creation (AllReduce) success, tag[%s]", tag.c_str());
8464 1 : CHK_RET(notifyPool_->UnregisterOp(tag));
8465 1 : if (commPortConfig_.devPortSwitchOn) {
8466 0 : std::vector<u32>& nicPorts = groupNicRanksPort_.empty() ? nicRanksPort_ : groupNicRanksPort_;
8467 0 : std::vector<u32>& vnicPorts = groupVnicRanksPort_.empty() ? vnicRanksPort_ : groupVnicRanksPort_;
8468 0 : Heartbeat::GetInstance(deviceLogicId_)
8469 0 : .SetRankPortInfo(isUseRankPort_, nicPorts, vnicPorts, commPortConfig_.devPortSwitchOn);
8470 : }
8471 1 : CHK_RET(RegisterToHeartBeat());
8472 :
8473 1 : CommBase* comm = nullptr;
8474 1 : CHK_RET(GetComm(tag, &comm));
8475 1 : if (comm == nullptr) {
8476 0 : HCCL_ERROR("comm get err, comm %p", comm);
8477 0 : return HCCL_E_PTR;
8478 : }
8479 1 : CHK_RET(
8480 : SetCommResource(commInputSize, commInputPtr, commOutputPtr, expMem.ptr(), comm, tagStreamInfo_[tag], stream));
8481 :
8482 1 : return HCCL_SUCCESS;
8483 1 : }
8484 :
8485 1 : HcclResult HcclCommunicator::GetComm(const std::string& tag, CommBase** comm)
8486 : {
8487 1 : if (Is310P3Common(isHaveCpuRank_, deviceType_)) {
8488 0 : *comm = tagCommInfo_[tag].commIntraServer.get();
8489 1 : } else if (isA2MC2MultiServer_) {
8490 : // 使用打平RDMA Mesh子通信域
8491 0 : *comm = tagCommInfo_[tag].commLevel1Rdma[0].get();
8492 : } else {
8493 1 : *comm = tagCommInfo_[tag].commLevel0[0].get();
8494 : }
8495 1 : return HCCL_SUCCESS;
8496 : }
8497 :
8498 1 : HcclResult HcclCommunicator::SetCommResource(
8499 : u64 commBufferSize, void* commInPtr, void* commOutPtr, void* commExpPtr, CommBase* comm,
8500 : level1StreamInfo_t& streamInfo, Stream& stream)
8501 : {
8502 1 : CHK_PTR_NULL(combinOparaMem_);
8503 1 : HcclCombinOpParam* combinOparaPtr = reinterpret_cast<HcclCombinOpParam*>(combinOparaMem_->ptr());
8504 1 : CHK_PTR_NULL(combinOparaPtr);
8505 :
8506 1 : u32 rankSize = comm->RankSize();
8507 1 : u32 curRankId = comm->Rank();
8508 1 : u32 usrRankId = comm->UserRank();
8509 1 : combinOparaPtr->rankId = curRankId;
8510 1 : combinOparaPtr->signalInfo.aicpuNotify.rankId = curRankId;
8511 1 : combinOparaPtr->rankNum = rankSize;
8512 1 : combinOparaPtr->winSize = commBufferSize;
8513 1 : combinOparaPtr->winExpSize = EXP_BUFFER_SIZE;
8514 1 : combinOparaPtr->config.deterministic = GetDeterministicConfig();
8515 : combinOparaPtr->config.notifyWaitTime
8516 1 : = (GetExternalInputHcclExecTimeoutSet() != HcclExecTimeoutSet::HCCL_EXEC_TIMEOUT_NOT_SET
8517 1 : || commConfig_.GetConfigExecTimeOutSet()) ?
8518 0 : commConfig_.GetConfigExecTimeOut() :
8519 : NOTIFY_DEFAULT_WAIT_TIME;
8520 1 : hcclMc2Info_.groupName = hrtMsprofGetHashId(identifier_.c_str(), identifier_.length());
8521 1 : combinOparaPtr->config.linkTimeOut = std::chrono::seconds(GetExternalInputHcclLinkTimeOut());
8522 1 : hcclMc2Info_.rankSize = rankSize;
8523 1 : hcclMc2Info_.rankId = curRankId;
8524 1 : hcclMc2Info_.usrRankId = usrRankId;
8525 1 : hcclMc2Info_.aicpuKfcStreamId = static_cast<uint32_t>(stream.id());
8526 1 : hcclMc2Info_.commStreamSize = rankSize;
8527 1 : hcclMc2Info_.reserve = 0;
8528 1 : rtEvent_t event = nullptr;
8529 1 : u32 eventId = 0;
8530 1 : u32 idx = 0;
8531 1 : u32 txSigleBase = 2;
8532 1 : u32 rxSigleBase = 3;
8533 :
8534 1 : if (isA2MC2MultiServer_) {
8535 : // MoE融合算子优化，MC2多机场景
8536 : // 判断是否支持NormalQP创建，若不支持，需要额外下发敲Doorbell任务
8537 0 : bool isSupportNormalQP = false;
8538 0 : CHK_RET(IsSupportAicpuNormalQP(devicePhyId_, isSupportNormalQP));
8539 0 : CHK_RET(SetDevIbverbsData(comm, isSupportNormalQP, commBufferSize, commInPtr, commOutPtr));
8540 :
8541 0 : bool isSupportAIVNormalQP = false;
8542 0 : CHK_RET(IsSupportAIVNormalQP(devicePhyId_, isSupportAIVNormalQP));
8543 0 : if (isSupportAIVNormalQP && isA2MC2IntraHie_) {
8544 0 : CHK_RET(GenAiRMAInfo(comm));
8545 0 : } else {
8546 0 : HCCL_WARNING(
8547 : "[%s] db transfer normal qp not support. tag[%s] curRankId[%u] rankNum[%u] isSupportAIVNormalQP[%u]",
8548 : __func__, comm->Tag().c_str(), curRankId, rankSize, isSupportAIVNormalQP);
8549 : }
8550 :
8551 0 : if (combinedCapabilityMem_ == nullptr) {
8552 0 : CHK_RET(AllocAndClearHostMem(sizeof(CombinedCapability), combinedCapabilityMem_));
8553 : }
8554 0 : CHK_PTR_NULL(combinedCapabilityMem_);
8555 : CombinedCapability* combinedCapabilityPtr
8556 0 : = reinterpret_cast<CombinedCapability*>(combinedCapabilityMem_->ptr());
8557 0 : CHK_PTR_NULL(combinedCapabilityPtr);
8558 0 : SalSetBitOne(combinedCapabilityPtr->dataplaneModeBitmap, POS_DATA_PLANE_MODE_HOST);
8559 0 : if (isSupportAIVNormalQP && isA2MC2IntraHie_) {
8560 0 : SalSetBitOne(combinedCapabilityPtr->dataplaneModeBitmap, POS_DATA_PLANE_MODE_AIV);
8561 : }
8562 0 : SalSetBitOne(combinedCapabilityPtr->dataplaneModeBitmap, POS_DATA_PLANE_MODE_AICPU);
8563 :
8564 0 : HCCL_INFO("[SetCommResource] Set dataplaneModeBitmap to [%llu]", combinedCapabilityPtr->dataplaneModeBitmap);
8565 :
8566 : // 非NormalQP场景需要传一条流，用于敲Doorbell
8567 0 : combinOparaPtr->streamInfo[0].streamIds = streamInfo.ringDeviceStreams[0].id();
8568 0 : combinOparaPtr->streamInfo[0].sqIds = streamInfo.ringDeviceStreams[0].sqId();
8569 0 : combinOparaPtr->streamInfo[0].cqIds = streamInfo.ringDeviceStreams[0].cqId();
8570 0 : combinOparaPtr->streamInfo[0].logicCqids = streamInfo.ringDeviceStreams[0].logicCqId();
8571 0 : HCCL_DEBUG(
8572 : "[SetCommResource] Set streamInfo[0].streamIds[%u].sqIds[%u].cqIds[%u].logicCqids[%u]",
8573 : combinOparaPtr->streamInfo[0].streamIds, combinOparaPtr->streamInfo[0].sqIds,
8574 : combinOparaPtr->streamInfo[0].cqIds, combinOparaPtr->streamInfo[0].logicCqids);
8575 : } else {
8576 2 : for (u32 i = 0; i < rankSize; i++) {
8577 1 : if (i != curRankId) {
8578 : void* bufferIn;
8579 : void* bufferOut;
8580 0 : std::vector<void*> remotePtrVec;
8581 0 : CHK_RET(comm->GetTransportByRank(i)->GetRemoteMem(UserMemType::INPUT_MEM, &bufferIn));
8582 0 : combinOparaPtr->windowsIn[i] = reinterpret_cast<u64>(bufferIn);
8583 :
8584 0 : CHK_RET(comm->GetTransportByRank(i)->GetRemoteMem(UserMemType::OUTPUT_MEM, &bufferOut));
8585 0 : combinOparaPtr->windowsOut[i] = reinterpret_cast<u64>(bufferOut);
8586 :
8587 0 : CHK_RET(comm->GetTransportByRank(i)->GetRemoteMem(&remotePtrVec));
8588 0 : if (remotePtrVec.size() != 0) {
8589 0 : combinOparaPtr->windowsExp[i] = reinterpret_cast<u64>(remotePtrVec[0]);
8590 0 : if (comm->GetTransportByRank(i)->GetTransportType() == TransportType::TRANS_TYPE_P2P) {
8591 0 : p2pCclBuf_[i] = remotePtrVec[0];
8592 : } else {
8593 0 : cclBuf_[i] = remotePtrVec[0];
8594 : }
8595 0 : combinOparaPtr->windowsExp[i]
8596 0 : += cclBufferManager_.GetInCCLbufferSize() + cclBufferManager_.GetOutCCLbufferSize();
8597 : }
8598 0 : CHK_RET(comm->GetTransportByRank(i)->GetTxAckDevNotifyInfo(combinOparaPtr->signalInfo.ipcNotifys[i]));
8599 0 : CHK_RET(comm->GetTransportByRank(i)->GetRxAckDevNotifyInfo(
8600 : combinOparaPtr->signalInfo.ipcNotifys[i + rankSize]));
8601 0 : CHK_RET(comm->GetTransportByRank(i)->GetTxDataSigleDevNotifyInfo(
8602 : combinOparaPtr->signalInfo.ipcNotifys[i + rankSize * txSigleBase]));
8603 0 : CHK_RET(comm->GetTransportByRank(i)->GetRxDataSigleDevNotifyInfo(
8604 : combinOparaPtr->signalInfo.ipcNotifys[i + rankSize * rxSigleBase]));
8605 0 : CHK_RET(GetAiCpuNotifyData(
8606 : streamInfo.ringDeviceSignalAux[idx], combinOparaPtr->signalInfo.noIpcNotifys[i]));
8607 :
8608 0 : CHK_RET(GetAiCpuNotifyData(
8609 : streamInfo.ringDeviceSignal[idx], combinOparaPtr->signalInfo.noIpcNotifys[i + rankSize]));
8610 0 : idx++;
8611 0 : } else {
8612 1 : combinOparaPtr->windowsIn[i] = reinterpret_cast<u64>(commInPtr);
8613 1 : combinOparaPtr->windowsOut[i] = reinterpret_cast<u64>(commOutPtr);
8614 1 : combinOparaPtr->windowsExp[i] = reinterpret_cast<u64>(commExpPtr);
8615 : // 在与aicpu商议后，本卡不再防止无效值。后续代码要删掉
8616 1 : combinOparaPtr->signalInfo.ipcNotifys[i].resId = INVALID_U64;
8617 1 : combinOparaPtr->signalInfo.ipcNotifys[i + rankSize].resId = INVALID_U64;
8618 1 : combinOparaPtr->signalInfo.ipcNotifys[i + rankSize * txSigleBase].resId = INVALID_U64;
8619 1 : combinOparaPtr->signalInfo.ipcNotifys[i + rankSize * rxSigleBase].resId = INVALID_U64;
8620 : }
8621 1 : HCCL_INFO(
8622 : "group[%s] successfully set windowsIn & windowsOut & windowsExp info: userRank[%u], groupRank[%u], "
8623 : "windowsIn[0x%llx], InSize[0x%llx], windowOut[0x%llx], OutSize[0x%llx], windowExp[0x%llx], "
8624 : "ExpSize[0x%llu]",
8625 : identifier_.c_str(), GetUserRank(), GetGroupRank(), combinOparaPtr->windowsIn[i],
8626 : cclBufferManager_.GetInCCLbufferSize(), combinOparaPtr->windowsOut[i],
8627 : cclBufferManager_.GetOutCCLbufferSize(), combinOparaPtr->windowsExp[i],
8628 : cclBufferManager_.GetExpBufferSize());
8629 :
8630 1 : combinOparaPtr->signalInfo.ipcNotifys[i].rankId = i;
8631 1 : combinOparaPtr->signalInfo.ipcNotifys[i + rankSize].rankId = i;
8632 1 : combinOparaPtr->signalInfo.ipcNotifys[i + rankSize * txSigleBase].rankId = i;
8633 1 : combinOparaPtr->signalInfo.ipcNotifys[i + rankSize * rxSigleBase].rankId = i;
8634 1 : combinOparaPtr->signalInfo.noIpcNotifys[i].rankId = i;
8635 :
8636 1 : hcclMc2Info_.commStreamIds[i] = streamInfo.ringDeviceStreams[i].id();
8637 1 : combinOparaPtr->streamInfo[i].streamIds = streamInfo.ringDeviceStreams[i].id();
8638 1 : combinOparaPtr->streamInfo[i].sqIds = streamInfo.ringDeviceStreams[i].sqId();
8639 1 : combinOparaPtr->streamInfo[i].cqIds = streamInfo.ringDeviceStreams[i].cqId();
8640 1 : combinOparaPtr->streamInfo[i].logicCqids = streamInfo.ringDeviceStreams[i].logicCqId();
8641 1 : HCCL_DEBUG("[hccl_Mc2_Info] commStreamIds[%u]:[%u]", i, streamInfo.ringDeviceStreams[i].id());
8642 :
8643 1 : CHK_RET(hrtEventCreateWithFlag(&event));
8644 :
8645 1 : CHK_RET(hrtGetEventID(event, &eventId));
8646 1 : aiCpuNoIpcEvnet_.push_back(event);
8647 1 : combinOparaPtr->signalInfo.noIpcEvents[i].resId = eventId;
8648 1 : HCCL_DEBUG(
8649 : "SetCommResource ipc notify info pre record local rankid: %u: remote rankid:%u, resId:%llu, "
8650 : "devId:%u, tsId:%u, addr:%llu.",
8651 : curRankId, combinOparaPtr->signalInfo.ipcNotifys[i].rankId,
8652 : combinOparaPtr->signalInfo.ipcNotifys[i].resId, combinOparaPtr->signalInfo.ipcNotifys[i].devId,
8653 : combinOparaPtr->signalInfo.ipcNotifys[i].tsId, combinOparaPtr->signalInfo.ipcNotifys[i].addr);
8654 1 : HCCL_DEBUG(
8655 : "SetCommResource ipc notify info pre wait local rankid: %u: remote rankid:%u, resId:%llu, "
8656 : "devId:%u, tsId:%u, addr:%llu.",
8657 : curRankId, combinOparaPtr->signalInfo.ipcNotifys[i + rankSize].rankId,
8658 : combinOparaPtr->signalInfo.ipcNotifys[i + rankSize].resId,
8659 : combinOparaPtr->signalInfo.ipcNotifys[i + rankSize].devId,
8660 : combinOparaPtr->signalInfo.ipcNotifys[i + rankSize].tsId,
8661 : combinOparaPtr->signalInfo.ipcNotifys[i + rankSize].addr);
8662 1 : HCCL_DEBUG(
8663 : "SetCommResource ipc notify info post record local rankid: %u: remote rankid:%u, resId:%llu, "
8664 : "devId:%u, tsId:%u, addr:%llu.",
8665 : curRankId, combinOparaPtr->signalInfo.ipcNotifys[i + rankSize * txSigleBase].rankId,
8666 : combinOparaPtr->signalInfo.ipcNotifys[i + rankSize * txSigleBase].resId,
8667 : combinOparaPtr->signalInfo.ipcNotifys[i + rankSize * txSigleBase].devId,
8668 : combinOparaPtr->signalInfo.ipcNotifys[i + rankSize * txSigleBase].tsId,
8669 : combinOparaPtr->signalInfo.ipcNotifys[i + rankSize * txSigleBase].addr);
8670 1 : HCCL_DEBUG(
8671 : "SetCommResource ipc notify info post wait local rankid: %u: remote rankid:%u, resId:%llu, "
8672 : "devId:%u, tsId:%u, addr:%llu.",
8673 : curRankId, combinOparaPtr->signalInfo.ipcNotifys[i + rankSize * rxSigleBase].rankId,
8674 : combinOparaPtr->signalInfo.ipcNotifys[i + rankSize * rxSigleBase].resId,
8675 : combinOparaPtr->signalInfo.ipcNotifys[i + rankSize * rxSigleBase].devId,
8676 : combinOparaPtr->signalInfo.ipcNotifys[i + rankSize * rxSigleBase].tsId,
8677 : combinOparaPtr->signalInfo.ipcNotifys[i + rankSize * rxSigleBase].addr);
8678 : }
8679 : }
8680 1 : HCCL_DEBUG(
8681 : "[hccl_Mc2_Info] groupname:[%s][%llu], rankSize[%u], rankId[%u], usrRankId[%u], aicpuKfcStreamId[%u], "
8682 : "commStreamSize[%u]",
8683 : identifier_.c_str(), hcclMc2Info_.groupName, rankSize, curRankId, usrRankId, static_cast<uint32_t>(stream.id()),
8684 : rankSize);
8685 1 : CHK_RET(
8686 : ProfilingManagerPub::CallMsprofReportMc2CommInfo(hrtMsprofSysCycleTime(), &hcclMc2Info_, sizeof(hcclMc2Info_)));
8687 1 : return HCCL_SUCCESS;
8688 : }
8689 :
8690 805 : void HcclCommunicator::ReleaseCommContextbuffer() { commContext_.free(); }
8691 :
8692 0 : HcclResult HcclCommunicator::CreateDeviceCommContext(u64 size, DeviceMem& buffer) const
8693 : {
8694 0 : CHK_PRT_RET(
8695 : !size,
8696 : HCCL_INFO("[Create][DeviceCommContext]device commContext size is zero. "
8697 : "not need to malloc memory"),
8698 : HCCL_SUCCESS);
8699 :
8700 : CHK_PRT_RET(
8701 : (size > ULONG_MAX),
8702 : HCCL_ERROR("[Create][DeviceCommContext]device commContext size %llu is large than ULONG_MAX", size),
8703 : HCCL_E_PARA);
8704 :
8705 0 : if (!buffer.ptr()) {
8706 0 : u64 memSize = size;
8707 0 : CHK_RET(DeviceMem::alloc(buffer, memSize));
8708 : }
8709 0 : return HCCL_SUCCESS;
8710 : }
8711 :
8712 0 : void HcclCommunicator::Break()
8713 : {
8714 0 : if (implAlg_ != nullptr) {
8715 0 : implAlg_->Break();
8716 : }
8717 0 : return;
8718 : }
8719 :
8720 3 : HcclResult HcclCommunicator::GetAlltoAllStagedWorkSpaceMemSize(
8721 : u64* sendCounts, u64* sdispls, HcclDataType sendType, u64* recvCounts, u64* rdispls, HcclDataType recvType,
8722 : u64& memSize)
8723 : {
8724 3 : if (Is310P3Common(isHaveCpuRank_, deviceType_)) {
8725 0 : HCCL_ERROR(
8726 : "[%s][%s]GetAlltoAllStagedWorkSpaceMemSize Not Supported!", LOG_KEYWORDS_TASK_EXEC.c_str(),
8727 : LOG_KEYWORDS_NOT_SUPPORTED.c_str());
8728 0 : return HCCL_E_NOT_SUPPORT;
8729 : }
8730 3 : CHK_SMART_PTR_NULL(implAlg_);
8731 3 : std::unique_ptr<CollAlgOperator> algOperator = implAlg_->GetAlgOperator(HcclCMDType::HCCL_CMD_ALLTOALLV);
8732 3 : AlltoAllOperator* alltoAllOperator = dynamic_cast<AlltoAllOperator*>(algOperator.get());
8733 3 : CHK_PTR_NULL(alltoAllOperator);
8734 :
8735 3 : OpParam opParam;
8736 3 : opParam.All2AllDataDes.sendType = sendType;
8737 3 : opParam.All2AllDataDes.recvType = recvType;
8738 3 : opParam.All2AllDataDes.sendCounts = static_cast<void*>(sendCounts);
8739 3 : opParam.All2AllDataDes.recvCounts = static_cast<void*>(recvCounts);
8740 3 : opParam.All2AllDataDes.sdispls = static_cast<void*>(sdispls);
8741 3 : opParam.All2AllDataDes.rdispls = static_cast<void*>(rdispls);
8742 3 : opParam.opType = HcclCMDType::HCCL_CMD_ALLTOALLV;
8743 3 : opParam.aicpuUnfoldMode = false;
8744 3 : opParam.aicpuCacheEnable = 0;
8745 :
8746 3 : if (alltoAllOperator->IsSatisfyAlltoAllAivCondition(opParam)
8747 3 : || alltoAllOperator->IsSatisfy91093OffloadCondition()) {
8748 3 : memSize = 0;
8749 3 : HCCL_INFO("Calculate workSpace MemSize for aiv AllToAll done, memSize[%llu]", memSize);
8750 3 : return HCCL_SUCCESS;
8751 : }
8752 :
8753 0 : std::unique_ptr<PreProcessMetaInfo> preMetaInfo = std::make_unique<PreProcessMetaInfo>();
8754 0 : CHK_SMART_PTR_NULL(preMetaInfo);
8755 :
8756 0 : CHK_RET(alltoAllOperator->PrepareAlltoAllAddrInfo(
8757 : opParam.All2AllDataDes.sendCounts, opParam.All2AllDataDes.sdispls, opParam.All2AllDataDes.sendType,
8758 : opParam.All2AllDataDes.recvCounts, opParam.All2AllDataDes.rdispls, opParam.All2AllDataDes.recvType,
8759 : preMetaInfo));
8760 :
8761 0 : preMetaInfo->opType = HcclCMDType::HCCL_CMD_ALLGATHER;
8762 :
8763 0 : CHK_RET(RegressCalPreOp(alltoAllOperator, opParam, preMetaInfo));
8764 :
8765 0 : return alltoAllOperator->GetAlltoAllStagedWorkSpaceMemSize(opParam, memSize);
8766 3 : }
8767 :
8768 1 : HcclResult HcclCommunicator::GetAlltoAllStagedWorkSpaceMemSize(
8769 : std::vector<SendRecvInfo>& allMeshAggregationSendRecvInfo, u64& memSize)
8770 : {
8771 1 : CHK_PRT_RET(
8772 : Is310P3Common(isHaveCpuRank_, deviceType_),
8773 : HCCL_ERROR("[HcclCommunicator][GetAlltoAllStagedWorkSpaceMemSize]Not Supported!"), HCCL_E_NOT_SUPPORT);
8774 :
8775 1 : CHK_SMART_PTR_NULL(implAlg_);
8776 0 : return implAlg_->GetAlltoAllStagedWorkSpaceMemSize(allMeshAggregationSendRecvInfo, memSize);
8777 : }
8778 :
8779 : HcclResult
8780 0 : HcclCommunicator::GetAllReduceScratchSize(const u64 count, const HcclDataType dataType, u64& scratchSize) const
8781 : {
8782 0 : CHK_SMART_PTR_NULL(implAlg_);
8783 0 : return implAlg_->GetAllReduceScratchSize(count, dataType, scratchSize);
8784 : }
8785 :
8786 33 : HcclResult HcclCommunicator::SetWorldGroupInfo(
8787 : std::unordered_map<std::string, std::map<u32, HcclIpAddress>> phyIdNicInfoMap, vector<RankInfo> worldRankInfoList,
8788 : vector<u32>& nicRanksPort, vector<u32>& vnicRanksPort)
8789 : {
8790 33 : for (auto& ipInfo : phyIdNicInfoMap) {
8791 0 : for (auto& devInfo : ipInfo.second) {
8792 0 : rankDevicePhyIdNicInfoMap_[ipInfo.first][devInfo.first] = devInfo.second;
8793 0 : HCCL_DEBUG(
8794 : "phyIdNicInfoMap print hostIp[%s] devId[%u] devIp[%s]", ipInfo.first.c_str(), devInfo.first,
8795 : devInfo.second.GetReadableAddress());
8796 : }
8797 : }
8798 :
8799 33 : for (auto& rankInfo : worldRankInfoList) {
8800 0 : worldRankInfoList_.push_back(rankInfo);
8801 : }
8802 :
8803 33 : for (auto& port : nicRanksPort) {
8804 0 : nicRanksPort_.push_back(port);
8805 0 : HCCL_DEBUG("nicRanksPort port[%u]", port);
8806 : }
8807 33 : for (auto& port : vnicRanksPort) {
8808 0 : vnicRanksPort_.push_back(port);
8809 0 : HCCL_DEBUG("vnicRanksPort port[%u]", port);
8810 : }
8811 33 : return HCCL_SUCCESS;
8812 : }
8813 :
8814 0 : HcclResult HcclCommunicator::GetTopoDesc(HcclTopoDescs* topoDescs, uint32_t topoSize)
8815 : {
8816 0 : if (topoSize < static_cast<uint32_t>(HcclTopoLevel::HCCL_TOPO_MAX)) {
8817 0 : HCCL_ERROR("topoDescs size is not enough, please check topoSize[%u]", topoSize);
8818 0 : return HCCL_E_PARA;
8819 : }
8820 :
8821 0 : if (deviceType_ == DevType::DEV_TYPE_910_93) {
8822 0 : topoDescs[static_cast<uint32_t>(HcclTopoLevel::HCCL_TOPO_L0)].algSets = HCCL_ALG_SWITCH | HCCL_ALG_RING;
8823 0 : topoDescs[static_cast<uint32_t>(HcclTopoLevel::HCCL_TOPO_L1)].algSets = HCCL_ALG_RING;
8824 0 : } else if (deviceType_ == DevType::DEV_TYPE_910B) {
8825 0 : topoDescs[static_cast<uint32_t>(HcclTopoLevel::HCCL_TOPO_L0)].algSets = HCCL_ALG_MESH;
8826 0 : topoDescs[static_cast<uint32_t>(HcclTopoLevel::HCCL_TOPO_L1)].algSets = 0;
8827 0 : } else if (deviceType_ == DevType::DEV_TYPE_310P3) {
8828 0 : topoDescs[static_cast<uint32_t>(HcclTopoLevel::HCCL_TOPO_L0)].algSets = HCCL_ALG_RING;
8829 0 : topoDescs[static_cast<uint32_t>(HcclTopoLevel::HCCL_TOPO_L1)].algSets = 0;
8830 : }
8831 :
8832 0 : topoDescs[static_cast<uint32_t>(HcclTopoLevel::HCCL_TOPO_L0)].rankSize = userRankSize_;
8833 0 : topoDescs[static_cast<uint32_t>(HcclTopoLevel::HCCL_TOPO_L1)].rankSize = 0;
8834 0 : return HCCL_SUCCESS;
8835 : }
8836 :
8837 235 : HcclResult HcclCommunicator::SetAivModeConfig(const bool aivMode)
8838 : {
8839 235 : CHK_SMART_PTR_NULL(implAlg_);
8840 235 : CHK_RET(implAlg_->SetAivModeConfig(aivMode));
8841 235 : return HCCL_SUCCESS;
8842 : }
8843 :
8844 235 : HcclResult HcclCommunicator::SetOnlyAivModeConfig(const bool isOnlyAiv)
8845 : {
8846 235 : CHK_SMART_PTR_NULL(implAlg_);
8847 235 : CHK_RET(implAlg_->SetOnlyAivModeConfig(isOnlyAiv));
8848 235 : isOnlyAiv_ = isOnlyAiv;
8849 235 : return HCCL_SUCCESS;
8850 : }
8851 :
8852 235 : HcclResult HcclCommunicator::SetAicpuUnfoldConfig(const bool aicpuUnfold)
8853 : {
8854 235 : CHK_SMART_PTR_NULL(implAlg_);
8855 235 : CHK_RET(implAlg_->SetAicpuUnfoldConfig(aicpuUnfold));
8856 235 : return HCCL_SUCCESS;
8857 : }
8858 :
8859 235 : HcclResult HcclCommunicator::SetExecTimeOutConfig(const s32 execTimeOut)
8860 : {
8861 235 : CHK_SMART_PTR_NULL(implAlg_);
8862 235 : CHK_RET(implAlg_->SetExecTimeOutConfig(execTimeOut));
8863 235 : return HCCL_SUCCESS;
8864 : }
8865 :
8866 235 : HcclResult HcclCommunicator::SetAlgoConfig(const std::map<HcclCMDType, std::vector<HcclAlgoType>>& algoMap)
8867 : {
8868 235 : CHK_SMART_PTR_NULL(implAlg_);
8869 235 : CHK_RET(implAlg_->SetAlgoConfig(algoMap));
8870 235 : return HCCL_SUCCESS;
8871 : }
8872 :
8873 530 : bool HcclCommunicator::GetAivModeConfig() { return commConfig_.GetConfigAivMode(); }
8874 :
8875 532 : bool HcclCommunicator::GetConfigIsOnlyAivMode() { return commConfig_.GetConfigIsOnlyAivMode(); }
8876 :
8877 1977 : bool HcclCommunicator::GetAicpuUnfoldConfig() { return commConfig_.GetConfigAicpuUnfold(); }
8878 :
8879 0 : void HcclCommunicator::SetQpQosAttr(u32 trafficClass, u32 serviceLevel)
8880 : {
8881 0 : if (oneSideService_) {
8882 0 : oneSideService_->SetTCAndSL(trafficClass, serviceLevel);
8883 0 : HCCL_INFO("[%s]Set TC[%u] and SL[%u] for oneSidedService success.", __func__, trafficClass, serviceLevel);
8884 : }
8885 0 : transportManager_->SetQpQosAttr(trafficClass, serviceLevel);
8886 0 : indptOpTransportManager_->SetQpQosAttr(trafficClass, serviceLevel);
8887 0 : }
8888 :
8889 0 : HcclResult HcclCommunicator::CheckExitWaitResumeState(bool& isChangedLink)
8890 : {
8891 0 : if (retryEnable_ && opRetryManager_ != nullptr) {
8892 0 : bool haveCommEnableBackupLink = false;
8893 0 : if (g_enableBackupLinkCommCount.load() > 0) {
8894 0 : haveCommEnableBackupLink = true;
8895 : }
8896 0 : HcclResult ret = opRetryManager_->ExitWaitResumeState(
8897 0 : identifier_, commConnections_.isRoot, haveCommEnableBackupLink, isChangedLink);
8898 0 : CHK_PRT_RET(
8899 : ret != HCCL_SUCCESS, HCCL_ERROR("[HcclCommunicator][Resume]opretry exit wait resume state failed."), ret);
8900 : }
8901 0 : return HCCL_SUCCESS;
8902 : }
8903 :
8904 0 : HcclResult HcclCommunicator::SetMemoryRange(void* baseVirPtr, size_t size, size_t alignment, uint64_t flags)
8905 : {
8906 0 : CHK_PRT_RET(
8907 : deviceType_ != DevType::DEV_TYPE_910_93,
8908 : HCCL_ERROR("[HcclCommunicator][SetMemoryRange] deviceType[%d] not support zero copy", deviceType_),
8909 : HCCL_E_NOT_SUPPORT);
8910 0 : if (zeroCopyMemoryAgent_ == nullptr) {
8911 0 : CHK_RET(InitZeroCopyMemoryAgent());
8912 : }
8913 0 : CHK_RET(zeroCopyMemoryAgent_->SetMemoryRange(baseVirPtr, size, alignment, flags));
8914 0 : return HCCL_SUCCESS;
8915 : }
8916 :
8917 0 : HcclResult HcclCommunicator::UnsetMemoryRange(void* baseVirPtr)
8918 : {
8919 0 : CHK_PRT_RET(
8920 : zeroCopyMemoryAgent_ == nullptr,
8921 : HCCL_ERROR("[HcclCommunicator][UnsetMemoryRange] not call HcclCommSetMemoryRange()"), HCCL_E_PARA);
8922 0 : CHK_RET(zeroCopyMemoryAgent_->UnsetMemoryRange(baseVirPtr));
8923 0 : return HCCL_SUCCESS;
8924 : }
8925 :
8926 0 : HcclResult HcclCommunicator::ActivateCommMemory(void* virPtr, size_t size, size_t offset, void* handle, uint64_t flags)
8927 : {
8928 0 : CHK_PRT_RET(
8929 : zeroCopyMemoryAgent_ == nullptr,
8930 : HCCL_ERROR("[HcclCommunicator][ActivateCommMemory] not call HcclCommSetMemoryRange()"), HCCL_E_PARA);
8931 0 : CHK_RET(zeroCopyMemoryAgent_->ActivateCommMemory(virPtr, size, offset, handle, flags));
8932 0 : return HCCL_SUCCESS;
8933 : }
8934 :
8935 0 : HcclResult HcclCommunicator::DeactivateCommMemory(void* virPtr)
8936 : {
8937 0 : CHK_PRT_RET(
8938 : zeroCopyMemoryAgent_ == nullptr,
8939 : HCCL_ERROR("[HcclCommunicator][DeactivateCommMemory] not call HcclCommSetMemoryRange()"), HCCL_E_PARA);
8940 0 : CHK_RET(zeroCopyMemoryAgent_->DeactivateCommMemory(virPtr));
8941 0 : return HCCL_SUCCESS;
8942 : }
8943 :
8944 0 : HcclResult HcclCommunicator::SetSingleLinkInfo(
8945 : std::unordered_map<u32, bool>& switchRanks, u32 remoteRankId, ChangeLinkInfo& changeLinkInfo)
8946 : {
8947 0 : auto iterLocal = switchRanks.find(userRank_);
8948 0 : auto iterRemote = switchRanks.find(remoteRankId);
8949 :
8950 0 : bool useBackupLink = false;
8951 0 : if (iterLocal != switchRanks.end() && iterRemote != switchRanks.end()) {
8952 : // 本端卡和对端卡都切，如果两者的目标网卡冲突，则切换失败；否则使用一致的目标网卡的的对应链路
8953 0 : CHK_PRT_RET(
8954 : iterLocal->second ^ iterRemote->second,
8955 : HCCL_ERROR(
8956 : "[HcclCommunicator][SetSingleLinkInfo] local rank[%u] plan to switch to nic[%u], "
8957 : "which is conflict with remote rank[%u] planning to switch to nic[%u].",
8958 : userRank_, iterLocal->second, remoteRankId, iterRemote->second),
8959 : HCCL_E_PARA);
8960 0 : useBackupLink = iterLocal->second;
8961 0 : } else if (iterLocal != switchRanks.end()) {
8962 : // 仅切换本端卡，根据本端卡的目标网卡，刷新对应链路
8963 0 : useBackupLink = iterLocal->second;
8964 0 : } else if (iterRemote != switchRanks.end()) {
8965 : // 仅切换对端卡，根据对端卡的目标网卡，刷新对应链路
8966 0 : useBackupLink = iterRemote->second;
8967 : } else {
8968 0 : HCCL_INFO(
8969 : "[HcclCommunicator][SetSingleLinkInfo] comm identifier[%s], local rank[%u], "
8970 : "remote rank[%u], neither the rank need switch, link will not be refreshed.",
8971 : identifier_.c_str(), userRank_, remoteRankId);
8972 0 : return HCCL_SUCCESS;
8973 : }
8974 :
8975 0 : changeLinkInfo.remoteRankList[changeLinkInfo.remoteRankNum] = remoteRankId;
8976 0 : changeLinkInfo.isUseDefaultPort[changeLinkInfo.remoteRankNum] = !(useBackupLink);
8977 0 : changeLinkInfo.remoteRankNum++;
8978 0 : remoteRankNicStatus_[remoteRankId] = useBackupLink ? CONNECT_REMOTE_BACKUP : CONNECT_REMOTE_DEFAULT;
8979 0 : needCheckBackupNic_ |= useBackupLink;
8980 0 : needCheckDefaultNic_ |= !useBackupLink;
8981 :
8982 0 : HCCL_RUN_INFO(
8983 : "[HcclCommunicator][SetSingleLinkInfo] comm identifier[%s], local rank[%u], "
8984 : "remote rank[%u], useBackupLink[%u], link info refreshed.",
8985 : identifier_.c_str(), userRank_, remoteRankId, useBackupLink);
8986 0 : return HCCL_SUCCESS;
8987 : }
8988 :
8989 : HcclResult
8990 0 : HcclCommunicator::SetRemoteRankLinkInfo(std::unordered_map<u32, bool>& switchRanks, ChangeLinkInfo& changeLinkInfo)
8991 : {
8992 : // 初始化重置changeLinkInfo
8993 0 : changeLinkInfo.remoteRankNum = 0;
8994 0 : needCheckBackupNic_ = false;
8995 0 : needCheckDefaultNic_ = false;
8996 : // 初始化重置remoteRankNicStatus_
8997 0 : (void)memset_s(remoteRankNicStatus_, sizeof(remoteRankNicStatus_), 0, sizeof(remoteRankNicStatus_));
8998 :
8999 0 : for (auto resIt : resMap_) {
9000 0 : for (auto& levelNSubCommTransport : resIt.second.opTransportResponse) {
9001 0 : for (auto& singleSubCommTransport : levelNSubCommTransport) {
9002 0 : for (auto& transportRequest : singleSubCommTransport.transportRequests) {
9003 0 : if (transportRequest.isValid && transportRequest.isUsedRdma) { // 仅RDMA链路需要刷新
9004 0 : CHK_RET(SetSingleLinkInfo(switchRanks, transportRequest.remoteUserRank, changeLinkInfo));
9005 : }
9006 : }
9007 : }
9008 : }
9009 0 : }
9010 0 : return HCCL_SUCCESS;
9011 : }
9012 :
9013 0 : HcclResult HcclCommunicator::ActiveStoppedLink(
9014 : std::map<u32, bool>& remoteRankPortMap, OpCommTransport& opTransportResponse, bool isBackup)
9015 : {
9016 0 : for (auto& levelNSubCommTransport : opTransportResponse) {
9017 0 : for (auto& singleSubCommTransport : levelNSubCommTransport) {
9018 0 : if (singleSubCommTransport.status.size() == 0) {
9019 0 : continue;
9020 : }
9021 0 : if (singleSubCommTransport.status.size() != singleSubCommTransport.transportRequests.size()
9022 0 : || singleSubCommTransport.links.size() != singleSubCommTransport.transportRequests.size()) {
9023 0 : HCCL_ERROR(
9024 : "[HcclCommunicator][ActiveStoppedLink] comm identifier[%s], local rank[%u], "
9025 : "status num[%u] or links num[%u] is inconsistent with transport request num[%u]. "
9026 : "Please check whether the resources are allocated correctly.",
9027 : identifier_.c_str(), userRank_, singleSubCommTransport.status.size(),
9028 : singleSubCommTransport.links.size(), singleSubCommTransport.transportRequests.size());
9029 0 : return HCCL_E_INTERNAL;
9030 : }
9031 :
9032 0 : for (size_t i = 0; i < singleSubCommTransport.transportRequests.size(); i++) {
9033 0 : auto& transportRequest = singleSubCommTransport.transportRequests[i];
9034 0 : auto remoteRankIter = remoteRankPortMap.find(transportRequest.remoteUserRank);
9035 0 : bool needLink = transportRequest.isValid && transportRequest.isUsedRdma
9036 0 : && remoteRankIter != remoteRankPortMap.end() && (remoteRankIter->second ^ isBackup);
9037 : // STOP状态的Transport需要唤醒，重置位到READY
9038 0 : if (needLink && singleSubCommTransport.status[i] == TransportStatus::STOP) {
9039 0 : HCCL_INFO(
9040 : "[HcclCommunicator][ActiveStoppedLink] comm identifier[%s], local rank[%u], "
9041 : "resuming link of remote rank[%u]",
9042 : identifier_.c_str(), userRank_, transportRequest.remoteUserRank);
9043 0 : CHK_RET(singleSubCommTransport.links[i]->Resume());
9044 0 : singleSubCommTransport.status[i] = TransportStatus::READY;
9045 : }
9046 : }
9047 : }
9048 : }
9049 0 : return HCCL_SUCCESS;
9050 : }
9051 :
9052 : HcclResult
9053 0 : HcclCommunicator::PrepareLinkForSwitchNic(std::unordered_map<u32, bool>& switchRanks, ChangeLinkInfo& changeLinkInfo)
9054 : {
9055 0 : CHK_RET(SetRemoteRankLinkInfo(switchRanks, changeLinkInfo));
9056 :
9057 0 : std::map<u32, bool> remoteRankPortMap;
9058 0 : for (u32 i = 0; i < changeLinkInfo.remoteRankNum; i++) {
9059 0 : remoteRankPortMap.emplace(changeLinkInfo.remoteRankList[i], changeLinkInfo.isUseDefaultPort[i]);
9060 : }
9061 0 : for (auto resIt : resMap_) {
9062 0 : CHK_RET(ActiveStoppedLink(remoteRankPortMap, resIt.second.opTransportResponse, false));
9063 0 : CHK_RET(ActiveStoppedLink(remoteRankPortMap, resIt.second.opTransportResponseBackUp, true));
9064 0 : }
9065 0 : return HCCL_SUCCESS;
9066 0 : }
9067 :
9068 0 : HcclResult HcclCommunicator::ParseSwitchRanks(
9069 : uint32_t nRanks, uint32_t* ranks, bool* useBackup, std::unordered_map<u32, bool>& switchRanks)
9070 : {
9071 0 : CHK_PTR_NULL(ranks);
9072 0 : CHK_PTR_NULL(useBackup);
9073 0 : switchRanksNum_ = nRanks;
9074 0 : (void)memset_s(switchRankList_, sizeof(switchRankList_), 0, sizeof(switchRankList_));
9075 0 : (void)memset_s(switchUseBackup_, sizeof(switchUseBackup_), 0, sizeof(switchUseBackup_));
9076 0 : s32 ret = memcpy_s(switchRankList_, sizeof(switchRankList_), ranks, sizeof(u32) * nRanks);
9077 0 : CHK_PRT_RET(
9078 : ret != EOK, HCCL_ERROR("[HcclCommunicator][ParseSwitchRanks] mem copy switch ranks fail."), HCCL_E_INTERNAL);
9079 0 : ret = memcpy_s(switchUseBackup_, sizeof(switchUseBackup_), useBackup, sizeof(bool) * nRanks);
9080 0 : CHK_PRT_RET(
9081 : ret != EOK, HCCL_ERROR("[HcclCommunicator][ParseSwitchRanks] mem copy switch use backup fail."),
9082 : HCCL_E_INTERNAL);
9083 :
9084 0 : std::string switchRankStr{};
9085 0 : for (uint32_t i = 0; i < nRanks; i++) {
9086 0 : CHK_PTR_NULL(ranks + i);
9087 0 : CHK_PTR_NULL(useBackup + i);
9088 0 : uint32_t switchRankId = ranks[i];
9089 0 : bool backup = useBackup[i];
9090 0 : CHK_PRT_RET(
9091 : switchRankId >= userRankSize_,
9092 : HCCL_ERROR(
9093 : "[HcclCommunicator][ParseSwitchRanks] invalid switchRankId[%u], "
9094 : "which should not be greater than rankSize[%u]",
9095 : switchRankId, userRankSize_),
9096 : HCCL_E_PARA);
9097 0 : CHK_PRT_RET(
9098 : switchRanks.find(switchRankId) != switchRanks.end(),
9099 : HCCL_ERROR("[HcclCommunicator][ParseSwitchRanks] duplicated switchRankId[%u]", switchRankId), HCCL_E_PARA);
9100 0 : switchRanks.emplace(switchRankId, backup);
9101 0 : switchRankStr += std::to_string(switchRankId) + ":" + std::to_string(backup) + ";";
9102 : }
9103 0 : HCCL_RUN_INFO(
9104 : "[HcclCommunicator][ParseSwitchRanks] comm identifier[%s], userRank[%u], load switchRanks:%s.",
9105 : identifier_.c_str(), userRank_, switchRankStr.c_str());
9106 0 : return HCCL_SUCCESS;
9107 0 : }
9108 :
9109 0 : HcclResult HcclCommunicator::SwitchNic(
9110 : uint32_t nRanks, uint32_t* ranks, bool* useBackup, std::shared_ptr<HDCommunicate>& controlH2D,
9111 : std::shared_ptr<HDCommunicate>& statusD2H)
9112 : {
9113 0 : HcclResult ret = HCCL_SUCCESS;
9114 0 : CHK_PRT_RET(
9115 : !IsEnableBackupLink(),
9116 : HCCL_RUN_WARNING(
9117 : "[HcclCommunicator][%s]Backup link is not enabled, "
9118 : "switch nic will not be prorocessed, comm identifier[%s], rank[%u], devType[%u], opretry enable[%u], "
9119 : "backup ip valid[%u], roce enable[%u].",
9120 : __func__, identifier_.c_str(), userRank_, deviceType_,
9121 : GetAicpuUnfoldConfig() && commConfig_.GetConfigInterSuperPodRetryEnable(), !devBackupIpAddr_[0].IsInvalid(),
9122 : IsEnableRoce()),
9123 : HCCL_SUCCESS);
9124 0 : CHK_PRT_RET(
9125 : resMap_.empty(),
9126 : HCCL_ERROR(
9127 : "[HcclCommunicator][%s] "
9128 : "no collective operation has been executed in this communication[%s] on rank[%u], "
9129 : "which does not support to set working device nic.",
9130 : __func__, identifier_.c_str(), userRank_),
9131 : HCCL_E_PARA);
9132 0 : std::unordered_map<u32, bool> switchRanks;
9133 0 : ChangeLinkInfo changeLinkInfo;
9134 0 : ret = ParseSwitchRanks(nRanks, ranks, useBackup, switchRanks);
9135 0 : if (ret == HCCL_SUCCESS) {
9136 0 : ret = PrepareLinkForSwitchNic(switchRanks, changeLinkInfo);
9137 : }
9138 : changeLinkInfo.isChangeLinkFlag
9139 0 : = ret == HCCL_SUCCESS; // 如果入参校验失败，则无需刷新链路；通知aicpu侧，防止其他卡超时等待
9140 :
9141 0 : switchNicWaitingResult_ = false;
9142 :
9143 0 : u32 changeLinkInfoStart
9144 : = sizeof(KfcCommand) + sizeof(BackgroundCommand) + sizeof(HcclComSuspendingFlag) + sizeof(HcclOpIdentifier);
9145 0 : CHK_RET(controlH2D->Put(changeLinkInfoStart, sizeof(ChangeLinkInfo), reinterpret_cast<uint8_t*>(&changeLinkInfo)));
9146 :
9147 0 : KfcCommand switchNicCommand = KfcCommand::kSwitchNic;
9148 0 : CHK_RET(controlH2D->Put(0, sizeof(KfcCommand), reinterpret_cast<uint8_t*>(&switchNicCommand)));
9149 :
9150 0 : KfcExecStatus switchStatus;
9151 0 : switchStatus.execStatus.kfcStatus = KfcStatus::kNull;
9152 0 : u32 waitSwitchExecCmdTimeout = static_cast<u32>(GetExternalInputHcclLinkTimeOut() * 1000 * 2.5f);
9153 : auto waitSwitchExecCmdTimeoutMs
9154 0 : = std::chrono::milliseconds(waitSwitchExecCmdTimeout); // 等待2.5倍的建链超时时间，给快慢卡场景提供冗余
9155 0 : auto startTime = std::chrono::steady_clock::now();
9156 : while (true) {
9157 0 : if (switchNicWaitingResult_) {
9158 0 : CHK_RET(statusD2H->Get(0, sizeof(KfcExecStatus), reinterpret_cast<uint8_t*>(&switchStatus)));
9159 : }
9160 0 : if (switchStatus.execStatus.kfcStatus == KfcStatus::kSwitchSuccess) {
9161 0 : HCCL_INFO(
9162 : "[HcclCommunicator][%s] comm identifier[%s], devicePhyId[%u], userRank[%u] switch nic success.",
9163 : __func__, identifier_.c_str(), devicePhyId_, userRank_);
9164 0 : ret = HCCL_SUCCESS;
9165 0 : break;
9166 0 : } else if (switchStatus.execStatus.kfcStatus == KfcStatus::kSwitchFail) {
9167 0 : HCCL_ERROR(
9168 : "[HcclCommunicator][%s] comm identifier[%s], devicePhyId[%u], userRank[%u] switch nic fail.", __func__,
9169 : identifier_.c_str(), devicePhyId_, userRank_);
9170 0 : ret = HCCL_E_INTERNAL;
9171 0 : break;
9172 0 : } else if ((std::chrono::steady_clock::now() - startTime) >= waitSwitchExecCmdTimeoutMs) {
9173 0 : HCCL_ERROR(
9174 : "[HcclCommunicator][%s] comm identifier[%s], devicePhyId[%u], "
9175 : "userRank[%u] switch nic timeout[%u ms], the transport status is undefined. "
9176 : "Please search log with keyword [ErrToWarn] for detail.",
9177 : __func__, identifier_.c_str(), devicePhyId_, userRank_, waitSwitchExecCmdTimeout);
9178 0 : ret = HCCL_E_TIMEOUT;
9179 0 : break;
9180 : } else {
9181 0 : SaluSleep(ONE_MILLISECOND_OF_USLEEP);
9182 : }
9183 0 : }
9184 0 : KfcExecControl clearCommand{};
9185 0 : CHK_RET(controlH2D->Put(0, sizeof(KfcExecControl), reinterpret_cast<uint8_t*>(&clearCommand)));
9186 0 : KfcExecStatus clearStatus{};
9187 0 : CHK_RET(controlH2D->Put(0, sizeof(KfcExecStatus), reinterpret_cast<uint8_t*>(&clearStatus)));
9188 0 : switchRanksNum_ = 0;
9189 0 : return ret;
9190 0 : }
9191 :
9192 0 : HcclResult HcclCommunicator::GetSwitchRanks(
9193 : u32* distSwitchRankList, bool* distSwitchUseBackup, u32& distSwitchRankNum, u8* distRemoteRankNicStatus,
9194 : u32& distNicStatusNum, bool& needCheckDefaultNic, bool& needCheckBackupNic)
9195 : {
9196 0 : s32 ret = memcpy_s(
9197 0 : distSwitchRankList, sizeof(u32) * AICPU_MAX_RANK_NUM, switchRankList_, sizeof(u32) * switchRanksNum_);
9198 0 : CHK_PRT_RET(
9199 : ret != EOK, HCCL_ERROR("[HcclCommunicator][GetSwitchRanks] mem copy switch rank list fail, ret[%u].", ret),
9200 : HCCL_E_INTERNAL);
9201 0 : ret = memcpy_s(
9202 0 : distSwitchUseBackup, sizeof(bool) * AICPU_MAX_RANK_NUM, switchUseBackup_, sizeof(bool) * switchRanksNum_);
9203 0 : CHK_PRT_RET(
9204 : ret != EOK, HCCL_ERROR("[HcclCommunicator][GetSwitchRanks] mem copy switch use backup fail, ret[%u].", ret),
9205 : HCCL_E_INTERNAL);
9206 0 : distSwitchRankNum = switchRanksNum_;
9207 0 : ret = memcpy_s(
9208 0 : distRemoteRankNicStatus, sizeof(u8) * AICPU_MAX_RANK_NUM, remoteRankNicStatus_, sizeof(u8) * userRankSize_);
9209 0 : CHK_PRT_RET(
9210 : ret != EOK,
9211 : HCCL_ERROR(
9212 : "[HcclCommunicator][GetSwitchRanks] mem copy remote rank nic status fail, "
9213 : "ret[%u].",
9214 : ret),
9215 : HCCL_E_INTERNAL);
9216 0 : distNicStatusNum = userRankSize_;
9217 0 : needCheckDefaultNic = needCheckDefaultNic_;
9218 0 : needCheckBackupNic = needCheckBackupNic_;
9219 0 : switchNicWaitingResult_ = true;
9220 0 : return HCCL_SUCCESS;
9221 : }
9222 :
9223 0 : HcclResult GetCannPath(const char* binPath, std::string& cannPath)
9224 : {
9225 0 : CHK_PRT_RET(binPath == nullptr, HCCL_ERROR("[HcclCommunicator][GetCannPath]binary path is nullptr"), HCCL_E_PTR);
9226 :
9227 0 : std::string tmpPath(binPath); // 存放cann安装路径
9228 0 : std::string libraryPath;
9229 0 : HcclResult ret = ParseLibraryPath(libraryPath);
9230 0 : CHK_PRT_RET(ret != HCCL_SUCCESS, HCCL_ERROR("[GetCannPath]errNo[0x%016llx]parse path fail.", ret), ret);
9231 :
9232 0 : ret = GetKeyWordPath(libraryPath, "/hccl", tmpPath);
9233 0 : CHK_PRT_RET(
9234 : ret != HCCL_SUCCESS, HCCL_ERROR("[GetCannPath]cannot found version file in %s.", libraryPath.c_str()),
9235 : HCCL_E_PARA);
9236 0 : tmpPath += binPath;
9237 0 : cannPath = tmpPath;
9238 :
9239 0 : return HCCL_SUCCESS;
9240 0 : }
9241 :
9242 523 : HcclResult HcclCommunicator::LoadCustomFile(
9243 : const char* binPath, aclrtBinaryLoadOptionType optionType, uint32_t cpuKernelMode, aclrtBinHandle& binHandle)
9244 : {
9245 523 : binHandle = nullptr;
9246 523 : HCCL_INFO("[LoadCustomFile]binPath[%s], optionType[%u], cpuKernelMode[%u]", binPath, optionType, cpuKernelMode);
9247 523 : return HCCL_SUCCESS;
9248 : }
9249 :
9250 0 : void HcclCommunicator::UnloadBinary(aclrtBinHandle& binHandle)
9251 : {
9252 0 : if (binHandle != nullptr) {
9253 0 : aclError ret = aclrtBinaryUnLoad(binHandle);
9254 0 : if (ret != ACL_SUCCESS) {
9255 0 : HCCL_ERROR("[UnloadBinary]errNo[0x%016llx] unload binary from file error.", ret);
9256 : }
9257 0 : binHandle = nullptr;
9258 : }
9259 0 : return;
9260 : }
9261 :
9262 0 : HcclResult HcclCommunicator::RegisterCommUserMem(void* addr, u64 size, void** handle)
9263 : {
9264 : // user mem和ccl buffer互斥，不支持同时创建
9265 0 : if (deviceType_ != DevType::DEV_TYPE_910_93 || superPodNum_ > 1 || isUserMemRegisted_
9266 0 : || cclBufferManager_.GetInCCLbuffer().ptr() != nullptr) {
9267 0 : HCCL_ERROR(
9268 : "[HcclCommunicator][%s]Registration user mem is not supported with the params. "
9269 : "Device type[%d], superPodNum[%u]; Or user mem/CCL buffer has already registered, addr[%p], "
9270 : "isUserMemRegisted[%d]",
9271 : __func__, deviceType_, superPodNum_, addr, isUserMemRegisted_);
9272 0 : return HCCL_E_NOT_SUPPORT;
9273 : }
9274 : // DeviceMem::create创建的DeviceMem对象为拷贝构造，析构时不释放内存，内存由上层管理
9275 0 : DeviceMem userMem = DeviceMem::create(addr, size);
9276 0 : std::shared_ptr<DeviceMem> userMemPtr = nullptr;
9277 0 : EXCEPTION_CATCH((userMemPtr = std::make_shared<DeviceMem>(std::move(userMem))), return HCCL_E_PTR);
9278 0 : *handle = static_cast<void*>(userMemPtr.get());
9279 0 : userMemMap_.insert(std::make_pair(*handle, userMemPtr));
9280 0 : HCCL_INFO(
9281 : "[HcclCommunicator][%s]Register user mem success, group[%s], handle[%p], addr[%llu], size[%llu]", __func__,
9282 : identifier_.c_str(), *handle, reinterpret_cast<uint64_t>(addr), size);
9283 0 : isUserMemRegisted_ = true;
9284 0 : return HCCL_SUCCESS;
9285 0 : }
9286 :
9287 0 : HcclResult HcclCommunicator::DeregisterCommUserMem(void* handle)
9288 : {
9289 0 : if (deviceType_ != DevType::DEV_TYPE_910_93 || superPodNum_ > 1) {
9290 0 : HCCL_ERROR(
9291 : "[HcclCommunicator][%s]Unsupported on the device type[%d] or superPodNum[%u]", __func__, deviceType_,
9292 : superPodNum_);
9293 0 : return HCCL_E_NOT_SUPPORT;
9294 : }
9295 :
9296 0 : CHK_PRT_RET(
9297 : !userMemMap_.erase(handle),
9298 : HCCL_RUN_WARNING("[HcclCommunicator][%s]Mem is not exist, handle[%p]", __func__, handle), HCCL_SUCCESS);
9299 :
9300 : // 重置user mem和userMemType
9301 0 : CHK_SAFETY_FUNC_RET(
9302 : memset_s(opResPara_.userMemRes, sizeof(opResPara_.userMemRes), 0, sizeof(opResPara_.userMemRes)));
9303 0 : opResPara_.userMemType = 0; // CCL Buffer
9304 0 : isUserMemRegisted_ = false;
9305 0 : HCCL_INFO(
9306 : "[HcclCommunicator][%s]Deregister mem success, group[%s], handle[%p]", __func__, identifier_.c_str(), handle);
9307 0 : return HCCL_SUCCESS;
9308 : }
9309 :
9310 0 : HcclResult HcclCommunicator::ExchangeCommUserMem(void* handle, std::vector<u32>& peerRanks)
9311 : {
9312 0 : if (deviceType_ != DevType::DEV_TYPE_910_93 || superPodNum_ > 1 || GetExternalInputInterHccsDisable()) {
9313 0 : HCCL_ERROR(
9314 : "[HcclCommunicator][%s]Unsupported configuration: device type[%d], superPodNum[%u], "
9315 : "or RDMA usage",
9316 : __func__, deviceType_, superPodNum_);
9317 0 : return HCCL_E_NOT_SUPPORT;
9318 : }
9319 :
9320 0 : if ((peerRanks.size() > rankInfoList_.size())) {
9321 0 : HCCL_ERROR(
9322 : "[HcclCommunicator][%s]Invalid peerRanksNum[%u], which should be less than communicator "
9323 : "rank nums[%u]",
9324 : __func__, peerRanks.size(), rankInfoList_.size());
9325 0 : return HCCL_E_PARA;
9326 : }
9327 : // 获取user mem，调exchange接口前需要先调注册接口注册user mem
9328 0 : if (userMemMap_.find(handle) == userMemMap_.end()) {
9329 0 : HCCL_ERROR("[HcclCommunicator][%s]Find user mem failed, handle[%p] is not registered", __func__, handle);
9330 0 : return HCCL_E_NOT_FOUND;
9331 : }
9332 0 : DeviceMem userMem = *userMemMap_[handle].get();
9333 0 : CHK_PTR_NULL(userMem.ptr());
9334 : // 构造建链param
9335 0 : TransportIOMem transMem;
9336 0 : transMem.userMem = userMem;
9337 0 : OpCommTransport opCommTransport;
9338 0 : LevelNSubCommTransport level0Transport;
9339 0 : SingleSubCommTransport commTransport;
9340 :
9341 0 : for (u32 rankIdx = 0; rankIdx < peerRanks.size(); rankIdx++) {
9342 0 : TransportRequest tmpTransport;
9343 0 : if (userRank_ != peerRanks[rankIdx]) {
9344 0 : tmpTransport.isValid = true;
9345 0 : tmpTransport.localUserRank = userRank_;
9346 0 : tmpTransport.remoteUserRank = peerRanks[rankIdx];
9347 0 : tmpTransport.inputMemType = TransportMemType::USER_MEM;
9348 0 : tmpTransport.outputMemType = TransportMemType::USER_MEM;
9349 : } else {
9350 : // 本rank不需要创建transport
9351 0 : tmpTransport.isValid = false;
9352 : }
9353 0 : commTransport.transportRequests.push_back(tmpTransport);
9354 : }
9355 0 : level0Transport.push_back(commTransport);
9356 0 : opCommTransport.push_back(level0Transport);
9357 0 : ClearOpTransportResponseLinks(opCommTransport);
9358 : // 建链
9359 0 : constexpr char EXCHANGE_USER_MEM_TAG_PREFIX[] = "ExchangeUserMem_";
9360 0 : string tag = EXCHANGE_USER_MEM_TAG_PREFIX + identifier_;
9361 0 : HcclResult ret = HCCL_SUCCESS;
9362 : {
9363 0 : StateGuard<HcclCommunicator, HcclCommState> guard(this, HcclCommState::BUILDING);
9364 0 : HCCL_RUN_INFO(
9365 : "[%s]Alloc transport, level size[%u], trans request size[%u], mem ptr[%p], mem size[%llu]", __func__,
9366 : opCommTransport.size(), commTransport.transportRequests.size(), userMem.ptr(), userMem.size());
9367 0 : CHK_PTR_NULL(transportManager_);
9368 0 : ret = transportManager_->Alloc(tag, transMem, opCommTransport, false);
9369 0 : }
9370 0 : CHK_PRT_RET(ret != HCCL_SUCCESS, HCCL_ERROR("[%s]Alloc transports failed, tag[%s]", __func__, tag.c_str()), ret);
9371 0 : userMemTransport_ = opCommTransport;
9372 0 : return HCCL_SUCCESS;
9373 0 : }
9374 0 : HcclResult HcclCommunicator::GetLocalCCLBuf(void** addr, uint64_t* size)
9375 : {
9376 0 : uint64_t cclbufSize = cclBufferManager_.GetInCCLbufferSize() + cclBufferManager_.GetOutCCLbufferSize()
9377 0 : + cclBufferManager_.GetExpBufferSize();
9378 0 : *addr = cclBufferManager_.GetCommCCLBuffer().ptr();
9379 0 : if (nullptr == cclBufferManager_.GetCommCCLBuffer().ptr()) {
9380 0 : cclbufSize = 0;
9381 : }
9382 0 : *size = cclbufSize;
9383 0 : HCCL_INFO(
9384 : "[%s] GetlocalCCLBuf success, addr[%p], size[%u]", identifier_.c_str(),
9385 : cclBufferManager_.GetCommCCLBuffer().ptr(), cclbufSize);
9386 0 : return HCCL_SUCCESS;
9387 : }
9388 :
9389 2 : HcclResult HcclCommunicator::GetRemoteCCLBuf(uint32_t remoteRank, void** addr, uint64_t* size)
9390 : {
9391 2 : CHK_PRT_RET(
9392 : (remoteRank >= AICPU_MAX_RANK_NUM), HCCL_ERROR("[%s] invalid remoteRank[%d]", __func__, remoteRank),
9393 : HCCL_E_PARA);
9394 : // 仅sdma场景
9395 1 : uint64_t cclbufSize = cclBufferManager_.GetInCCLbufferSize() + cclBufferManager_.GetOutCCLbufferSize()
9396 1 : + cclBufferManager_.GetExpBufferSize();
9397 1 : *addr = p2pCclBuf_[remoteRank];
9398 :
9399 1 : if (nullptr == p2pCclBuf_[remoteRank]) {
9400 1 : cclbufSize = 0;
9401 : }
9402 1 : *size = cclbufSize;
9403 1 : HCCL_INFO(
9404 : "[%s] GetRemoteCCLBuf success, remoteRank[%u], addr[%p], size[%u]", identifier_.c_str(), remoteRank,
9405 : p2pCclBuf_[remoteRank], cclbufSize);
9406 1 : return HCCL_SUCCESS;
9407 : }
9408 0 : HcclResult HcclCommunicator::GetKFCWorkSpace(void** addr, uint64_t* size)
9409 : {
9410 0 : *addr = workSpace_.ptr();
9411 0 : *size = workSpaceSize_;
9412 0 : HCCL_INFO(
9413 : "[%s] GetKFCWorkSpace success, addr[%p], size[%u]", identifier_.c_str(), workSpace_.ptr(), workSpaceSize_);
9414 0 : return HCCL_SUCCESS;
9415 : }
9416 4 : HcclResult HcclCommunicator::IndOpTransportAlloc(
9417 : const std::string& tag, OpCommTransport& opCommTransport, TransportIOMem& transMem, bool isAicpuModeEn)
9418 : {
9419 : // Aicpu侧不支持用户注册额外内存
9420 4 : if (isAicpuModeEn) {
9421 2 : if (transMem.indOpMem.userDeviceMem.size() > 0 || transMem.indOpMem.userHostMem.size() > 0) {
9422 2 : HCCL_ERROR("[%s] AICPU engine does not support user-registered memory", __func__);
9423 2 : return HCCL_E_NOT_SUPPORT;
9424 : }
9425 : }
9426 :
9427 2 : StateGuard<HcclCommunicator, HcclCommState> guard(this, HcclCommState::BUILDING);
9428 2 : CHK_PTR_NULL(indptOpTransportManager_);
9429 1 : bool isIndOp = true;
9430 1 : HcclResult ret = indptOpTransportManager_->Alloc(
9431 1 : tag, transMem, opCommTransport, isAicpuModeEn, false, false, HcclCMDType::HCCL_CMD_INVALID, false, isIndOp);
9432 1 : if (ret != HCCL_SUCCESS) {
9433 1 : HCCL_ERROR(
9434 : "[%s] Failed to alloc transport, tag[%s], isAicpuModeEn[%d], ret[%d]", __func__, tag.c_str(), isAicpuModeEn,
9435 : ret);
9436 1 : return ret;
9437 : }
9438 :
9439 0 : HCCL_RUN_INFO(
9440 : "[%s] Alloc transport success, tag[%s], isAicpuModeEn[%d], ret[%d]", __func__, tag.c_str(), isAicpuModeEn, ret);
9441 0 : return HCCL_SUCCESS;
9442 2 : }
9443 :
9444 235 : HcclTopoAttr HcclCommunicator::GetTopoAttr()
9445 : {
9446 235 : HcclTopoAttr topoAttr;
9447 235 : attrCollector_.GetTopoAttr(topoAttr);
9448 235 : return topoAttr;
9449 0 : }
9450 :
9451 102 : HcclResult HcclCommunicator::GetHDCommunicate(
9452 : HDCommunicateParams& kfcControlTransferH2DParams, HDCommunicateParams& kfcStatusTransferD2HParams)
9453 : {
9454 102 : if (GetSupportHDCommunicate() == false) {
9455 101 : HCCL_WARNING("%s not support HDCommunicate, skip", __func__);
9456 101 : return HCCL_SUCCESS;
9457 : }
9458 1 : CHK_SMART_PTR_NULL(kfcControlTransferH2D_);
9459 1 : CHK_SMART_PTR_NULL(kfcStatusTransferD2H_);
9460 1 : kfcControlTransferH2DParams = kfcControlTransferH2D_->GetCommunicateParams();
9461 1 : kfcStatusTransferD2HParams = kfcStatusTransferD2H_->GetCommunicateParams();
9462 1 : HCCL_INFO("%s success, group[%s]", __func__, identifier_.c_str());
9463 1 : return HCCL_SUCCESS;
9464 : }
9465 :
9466 235 : HcclResult HcclCommunicator::SetGetAicpuCommState(std::function<bool()> getAicpuCommState)
9467 : {
9468 235 : getAicpuCommState_ = getAicpuCommState;
9469 235 : HCCL_DEBUG("%s success, group[%s]", __func__, identifier_.c_str());
9470 235 : return HCCL_SUCCESS;
9471 : }
9472 :
9473 0 : HcclResult HcclCommunicator::CommGetNetLayers(uint32_t** netLayers, uint32_t* netLayerNum)
9474 : {
9475 0 : if (deviceType_ == DevType::DEV_TYPE_910_93) {
9476 0 : netLayer_[0] = static_cast<uint32_t>(HcclTopoLevel::HCCL_TOPO_L0);
9477 0 : netLayer_[1] = static_cast<uint32_t>(HcclTopoLevel::HCCL_TOPO_L1);
9478 0 : *netLayerNum = COMM_LAYER_NUM_MAX;
9479 0 : } else if (deviceType_ == DevType::DEV_TYPE_910B || deviceType_ == DevType::DEV_TYPE_310P3) {
9480 0 : netLayer_[0] = static_cast<uint32_t>(HcclTopoLevel::HCCL_TOPO_L0);
9481 0 : *netLayerNum = 1;
9482 : }
9483 0 : *netLayers = netLayer_;
9484 0 : return HCCL_SUCCESS;
9485 : }
9486 :
9487 0 : HcclResult HcclCommunicator::CommGetInstSizeByNetLayer(uint32_t netLayer, uint32_t* rankNum)
9488 : {
9489 0 : if ((netLayer == static_cast<uint32_t>(HcclTopoLevel::HCCL_TOPO_L0))
9490 0 : || (netLayer == static_cast<uint32_t>(HcclTopoLevel::HCCL_TOPO_L1))) {
9491 0 : *rankNum = userRankSize_;
9492 : }
9493 0 : return HCCL_SUCCESS;
9494 : }
9495 :
9496 0 : HcclResult HcclCommunicator::CommGetInstTopoTypeByNetLayer(uint32_t netLayer, u32* topoType)
9497 : {
9498 0 : if (deviceType_ == DevType::DEV_TYPE_910_93) {
9499 0 : if (netLayer == static_cast<uint32_t>(HcclTopoLevel::HCCL_TOPO_L0)) {
9500 0 : *topoType = HCCL_ALG_SWITCH | HCCL_ALG_RING;
9501 0 : } else if (netLayer == static_cast<uint32_t>(HcclTopoLevel::HCCL_TOPO_L1)) {
9502 0 : *topoType = HCCL_ALG_RING;
9503 : }
9504 0 : } else if (deviceType_ == DevType::DEV_TYPE_910B) {
9505 0 : if (netLayer == static_cast<uint32_t>(HcclTopoLevel::HCCL_TOPO_L0)) {
9506 0 : *topoType = HCCL_ALG_MESH;
9507 : }
9508 0 : } else if (deviceType_ == DevType::DEV_TYPE_310P3) {
9509 0 : if (netLayer == static_cast<uint32_t>(HcclTopoLevel::HCCL_TOPO_L0)) {
9510 0 : *topoType = HCCL_ALG_RING;
9511 : }
9512 : }
9513 0 : return HCCL_SUCCESS;
9514 : }
9515 :
9516 0 : HcclResult HcclCommunicator::GetNetLayers(uint32_t** netLayers, uint32_t* netLayerNum)
9517 : {
9518 0 : return rankGraph_.GetNetLayers(netLayers, netLayerNum);
9519 : }
9520 :
9521 0 : HcclResult HcclCommunicator::GetInstSizeByNetLayer(uint32_t netLayer, uint32_t* rankNum)
9522 : {
9523 0 : return rankGraph_.GetInstSizeByNetLayer(netLayer, rankNum);
9524 : }
9525 :
9526 523 : HcclResult HcclCommunicator::GetInstTopoTypeByNetLayer(uint32_t netLayer, CommTopo* topoType)
9527 : {
9528 523 : return rankGraph_.GetInstTopoTypeByNetLayer(netLayer, topoType);
9529 : }
9530 :
9531 0 : HcclResult HcclCommunicator::GetInstRanksByNetLayer(uint32_t netLayer, uint32_t** rankList, uint32_t* rankNum)
9532 : {
9533 0 : return rankGraph_.GetInstRanksByNetLayer(netLayer, rankList, rankNum);
9534 : }
9535 :
9536 0 : HcclResult HcclCommunicator::GetInstSizeListByNetLayer(uint32_t netLayer, uint32_t** instSizeList, uint32_t* listSize)
9537 : {
9538 0 : return rankGraph_.GetInstSizeListByNetLayer(netLayer, instSizeList, listSize);
9539 : }
9540 :
9541 0 : HcclResult HcclCommunicator::GetTopoInstsByLayer(uint32_t netLayer, uint32_t** topoInsts, uint32_t* topoInstNum)
9542 : {
9543 0 : return rankGraph_.GetTopoInstsByLayer(netLayer, topoInsts, topoInstNum);
9544 : }
9545 :
9546 0 : HcclResult HcclCommunicator::GetTopoType(uint32_t netLayer, uint32_t topoInstId, CommTopo* topoType)
9547 : {
9548 0 : return rankGraph_.GetTopoType(netLayer, topoType);
9549 : }
9550 :
9551 : HcclResult
9552 0 : HcclCommunicator::GetRanksByTopoInst(uint32_t netLayer, uint32_t topoInstId, uint32_t** ranks, uint32_t* rankNum)
9553 : {
9554 0 : return rankGraph_.GetRanksByTopoInst(netLayer, topoInstId, ranks, rankNum);
9555 : }
9556 :
9557 0 : HcclResult HcclCommunicator::GetEndpointNum(uint32_t netLayer, uint32_t topoInstId, uint32_t* num)
9558 : {
9559 0 : return rankGraph_.GetEndpointNum(netLayer, topoInstId, num);
9560 : }
9561 :
9562 : HcclResult
9563 0 : HcclCommunicator::GetEndpointDesc(uint32_t netLayer, uint32_t topoInstId, uint32_t* descNum, EndpointDesc* endpointDesc)
9564 : {
9565 0 : return rankGraph_.GetEndpointDesc(netLayer, topoInstId, descNum, endpointDesc);
9566 : }
9567 :
9568 0 : HcclResult HcclCommunicator::GetEndpointInfo(
9569 : uint32_t rankId, const EndpointDesc* endPointDesc, EndpointAttr endpointAttr, uint32_t infoLen, void* info)
9570 : {
9571 0 : return rankGraph_.GetEndpointInfo(rankId, endPointDesc, endpointAttr, infoLen, info);
9572 : }
9573 :
9574 0 : HcclResult HcclCommunicator::GetRankGraph(GraphType type, void** graph, uint32_t* len)
9575 : {
9576 0 : return rankGraph_.GetRankGraphInfo(type, graph, len);
9577 : }
9578 :
9579 0 : void* HcclCommunicator::GetRankGraphV1() { return static_cast<void*>(&rankGraph_); }
9580 :
9581 0 : HcclResult HcclCommunicator::GetLinks(
9582 : uint32_t netLayer, uint32_t srcRank, uint32_t dstRank, CommLink** linkList, uint32_t* listSize)
9583 : {
9584 0 : return rankGraph_.GetLinks(netLayer, srcRank, dstRank, linkList, listSize);
9585 : }
9586 :
9587 0 : HcclResult HcclCommunicator::GetHeterogMode(HcclHeterogMode* mode) { return rankGraph_.GetHeterogMode(mode); }
9588 :
9589 523 : HcclResult HcclCommunicator::RegisterToSnapshot()
9590 : {
9591 523 : if (deviceType_ != DevType::DEV_TYPE_910B && deviceType_ != DevType::DEV_TYPE_910_93) {
9592 325 : return HCCL_SUCCESS;
9593 : }
9594 198 : if (userRankSize_ <= 1) {
9595 2 : HCCL_RUN_INFO(
9596 : "[HcclCommunicator][RegisterToSnapshot]comm identifier[%s], deviceLogicId[%d], "
9597 : "rank size[%u] is no greater than 1, and then will not register to snapshot",
9598 : identifier_.c_str(), deviceLogicId_, userRankSize_);
9599 2 : return HCCL_SUCCESS;
9600 : }
9601 0 : auto setInvalidCommCallback = [this](bool isInvalid) {
9602 0 : return this->SetInvalidComm(isInvalid);
9603 196 : };
9604 0 : auto preProcessCallback = [this]() {
9605 0 : return this->SnapshotCheckPreProcess();
9606 196 : };
9607 0 : auto postProcessCallback = [this]() {
9608 0 : return this->SnapshotCheckPostProcess();
9609 196 : };
9610 196 : CHK_RET(SnapshotControl::GetInstance(deviceLogicId_)
9611 : .RegisterComm(identifier_, setInvalidCommCallback, preProcessCallback, postProcessCallback));
9612 196 : if (IsEnableBackupLink()) {
9613 0 : CHK_RET(SnapshotControl::GetInstance(deviceLogicId_).RegisterBackup(identifier_, deviceBackUpPhyId_));
9614 : }
9615 196 : return HCCL_SUCCESS;
9616 : }
9617 :
9618 214 : HcclResult HcclCommunicator::UnRegisterFromSnapshot()
9619 : {
9620 214 : if (deviceType_ != DevType::DEV_TYPE_910B && deviceType_ != DevType::DEV_TYPE_910_93) {
9621 0 : return HCCL_SUCCESS;
9622 : }
9623 214 : if (userRankSize_ <= 1) {
9624 3 : HCCL_RUN_INFO(
9625 : "[HcclCommunicator][UnRegisterFromSnapshot]comm identifier[%s], deviceLogicId[%d], "
9626 : "rank size[%u] is no greater than 1, and then will not unregister from snapshot",
9627 : identifier_.c_str(), deviceLogicId_, userRankSize_);
9628 3 : return HCCL_SUCCESS;
9629 : }
9630 211 : CHK_RET(SnapshotControl::GetInstance(deviceLogicId_).UnRegisterComm(identifier_));
9631 210 : if (IsEnableBackupLink()) {
9632 0 : CHK_RET(SnapshotControl::GetInstance(deviceLogicId_).UnRegisterBackup(identifier_, deviceBackUpPhyId_));
9633 : }
9634 211 : return HCCL_SUCCESS;
9635 : }
9636 :
9637 0 : HcclResult HcclCommunicator::SetInvalidComm(bool isInvalid)
9638 : {
9639 0 : isInvalidComm_ = isInvalid;
9640 0 : HCCL_INFO(
9641 : "[HcclCommunicator][SetInvalidComm] comm[%s] is set to invalid, rank[%u], deviceLogicId[%d]",
9642 : identifier_.c_str(), userRank_, deviceLogicId_);
9643 0 : return HCCL_SUCCESS;
9644 : }
9645 :
9646 0 : HcclResult HcclCommunicator::SnapshotCheckPreProcess()
9647 : {
9648 0 : bool errorFlag = false;
9649 0 : auto pauseTimeout = std::chrono::seconds(GetExternalInputHcclLinkTimeOut());
9650 0 : auto startTime = std::chrono::steady_clock::now();
9651 : while (true) {
9652 0 : CHK_PRT_BREAK(
9653 : Heartbeat::GetInstance(deviceLogicId_).IsPaused(),
9654 : HCCL_INFO(
9655 : "[HcclCommunicator][SnapshotCheckPreProcess] comm[%s], rank[%u], deviceLogicId[%d], "
9656 : "heartbeat thread has been paused.",
9657 : identifier_.c_str(), userRank_, deviceLogicId_), );
9658 0 : CHK_PRT_BREAK(
9659 : (std::chrono::steady_clock::now() - startTime) >= pauseTimeout,
9660 : HCCL_ERROR(
9661 : "[HcclCommunicator][SnapshotCheckPreProcess] comm[%s], rank[%u], deviceLogicId[%d], "
9662 : "pause heartbeat thread timeout[%u s].",
9663 : identifier_.c_str(), userRank_, deviceLogicId_, GetExternalInputHcclLinkTimeOut()),
9664 : errorFlag = true);
9665 : }
9666 0 : startTime = std::chrono::steady_clock::now();
9667 0 : while (retryEnable_ && opRetryManager_) {
9668 0 : CHK_PRT_BREAK(
9669 : opRetryManager_->IsPaused(identifier_),
9670 : HCCL_INFO(
9671 : "[HcclCommunicator][SnapshotCheckPreProcess] comm[%s], rank[%u], deviceLogicId[%d], "
9672 : "opretry threads have been paused.",
9673 : identifier_.c_str(), userRank_, deviceLogicId_), );
9674 0 : CHK_PRT_BREAK(
9675 : (std::chrono::steady_clock::now() - startTime) >= pauseTimeout,
9676 : HCCL_ERROR(
9677 : "[HcclCommunicator][SnapshotCheckPreProcess] comm[%s], rank[%u], deviceLogicId[%d], "
9678 : "pause opretry threads timeout[%u s].",
9679 : identifier_.c_str(), userRank_, deviceLogicId_, GetExternalInputHcclLinkTimeOut()),
9680 : errorFlag = true);
9681 : }
9682 0 : startTime = std::chrono::steady_clock::now();
9683 0 : while (zeroCopyMemoryAgent_) {
9684 0 : CHK_PRT_BREAK(
9685 : zeroCopyMemoryAgent_->IsPaused(),
9686 : HCCL_INFO(
9687 : "[HcclCommunicator][SnapshotCheckPreProcess] comm[%s], rank[%u], deviceLogicId[%d], "
9688 : "zero-copy memory agent thread has been paused.",
9689 : identifier_.c_str(), userRank_, deviceLogicId_), );
9690 0 : CHK_PRT_BREAK(
9691 : (std::chrono::steady_clock::now() - startTime) >= pauseTimeout,
9692 : HCCL_ERROR(
9693 : "[HcclCommunicator][SnapshotCheckPreProcess] comm[%s], rank[%u], deviceLogicId[%d], "
9694 : "pause zero-copy memory agent thread timeout[%u s].",
9695 : identifier_.c_str(), userRank_, deviceLogicId_, GetExternalInputHcclLinkTimeOut()),
9696 : errorFlag = true);
9697 : }
9698 0 : CHK_PRT_RET(
9699 : errorFlag,
9700 : HCCL_ERROR(
9701 : "[HcclCommunicator][SnapshotCheckPreProcess] comm[%s], rank[%u], "
9702 : "deviceLogicId[%d], snapshot pre-process fail due to some background threads pause timeout, please check.",
9703 : identifier_.c_str(), userRank_, deviceLogicId_),
9704 : HCCL_E_INTERNAL);
9705 0 : HCCL_INFO(
9706 : "[HcclCommunicator][SnapshotCheckPreProcess] comm[%s], rank[%u], deviceLogicId[%d], "
9707 : "snapshot pre-process success.",
9708 : identifier_.c_str(), userRank_, deviceLogicId_);
9709 0 : return HCCL_SUCCESS;
9710 : }
9711 :
9712 0 : HcclResult HcclCommunicator::SnapshotCheckPostProcess()
9713 : {
9714 0 : bool errorFlag = false;
9715 0 : auto resumeTimeout = std::chrono::seconds(GetExternalInputHcclLinkTimeOut());
9716 0 : auto startTime = std::chrono::steady_clock::now();
9717 : while (true) {
9718 0 : CHK_PRT_BREAK(
9719 : Heartbeat::GetInstance(deviceLogicId_).IsResumed(),
9720 : HCCL_INFO(
9721 : "[HcclCommunicator][SnapshotCheckPostProcess] comm[%s], rank[%u], deviceLogicId[%d], "
9722 : "heartbeat thread has been resumed.",
9723 : identifier_.c_str(), userRank_, deviceLogicId_), );
9724 0 : CHK_PRT_BREAK(
9725 : (std::chrono::steady_clock::now() - startTime) >= resumeTimeout,
9726 : HCCL_ERROR(
9727 : "[HcclCommunicator][SnapshotCheckPostProcess] comm[%s], rank[%u], deviceLogicId[%d], "
9728 : "resume heartbeat thread timeout[%u s].",
9729 : identifier_.c_str(), userRank_, deviceLogicId_, GetExternalInputHcclLinkTimeOut()),
9730 : errorFlag = true);
9731 : }
9732 0 : startTime = std::chrono::steady_clock::now();
9733 0 : while (retryEnable_ && opRetryManager_) {
9734 0 : CHK_PRT_BREAK(
9735 : opRetryManager_->IsResumed(identifier_),
9736 : HCCL_INFO(
9737 : "[HcclCommunicator][SnapshotCheckPostProcess] comm[%s], rank[%u], deviceLogicId[%d], "
9738 : "opretry threads have been resumed.",
9739 : identifier_.c_str(), userRank_, deviceLogicId_), );
9740 0 : CHK_PRT_BREAK(
9741 : (std::chrono::steady_clock::now() - startTime) >= resumeTimeout,
9742 : HCCL_ERROR(
9743 : "[HcclCommunicator][SnapshotCheckPostProcess] comm[%s], rank[%u], deviceLogicId[%d], "
9744 : "resume opretry threads timeout[%u s].",
9745 : identifier_.c_str(), userRank_, deviceLogicId_, GetExternalInputHcclLinkTimeOut()),
9746 : errorFlag = true);
9747 : }
9748 0 : startTime = std::chrono::steady_clock::now();
9749 0 : while (zeroCopyMemoryAgent_) {
9750 0 : CHK_PRT_BREAK(
9751 : zeroCopyMemoryAgent_->IsResumed(),
9752 : HCCL_INFO(
9753 : "[HcclCommunicator][SnapshotCheckPostProcess] comm[%s], rank[%u], deviceLogicId[%d], "
9754 : "zero-copy memory agent thread has been resumed.",
9755 : identifier_.c_str(), userRank_, deviceLogicId_), );
9756 0 : CHK_PRT_BREAK(
9757 : (std::chrono::steady_clock::now() - startTime) >= resumeTimeout,
9758 : HCCL_ERROR(
9759 : "[HcclCommunicator][SnapshotCheckPostProcess] comm[%s], rank[%u], deviceLogicId[%d], "
9760 : "resume zero-copy memory agent thread timeout[%u s].",
9761 : identifier_.c_str(), userRank_, deviceLogicId_, GetExternalInputHcclLinkTimeOut()),
9762 : errorFlag = true);
9763 : }
9764 0 : CHK_PRT_RET(
9765 : errorFlag,
9766 : HCCL_ERROR(
9767 : "[HcclCommunicator][SnapshotCheckPostProcess] comm[%s], rank[%u], "
9768 : "deviceLogicId[%d], snapshot post-process check fail due to some background threads resume timeout, "
9769 : "please check.",
9770 : identifier_.c_str(), userRank_, deviceLogicId_),
9771 : HCCL_E_INTERNAL);
9772 0 : HCCL_INFO(
9773 : "[HcclCommunicator][SnapshotCheckPostProcess] comm[%s], rank[%u], deviceLogicId[%d], "
9774 : "snapshot post-process check success.",
9775 : identifier_.c_str(), userRank_, deviceLogicId_);
9776 0 : return HCCL_SUCCESS;
9777 : }
9778 :
9779 404 : void HcclCommunicator::SetReleaseChannel(std::function<HcclResult()> releaseChannel)
9780 : {
9781 404 : releaseChannel_ = releaseChannel;
9782 404 : return;
9783 : }
9784 :
9785 235 : CCLBufferManager& HcclCommunicator::GetCCLbufferManager() { return cclBufferManager_; }
9786 :
9787 0 : void HcclCommunicator::SetHcclQos(u32 hcclQos)
9788 : {
9789 0 : HCCL_INFO("[HcclCommunicator][host][SetHcclQos] hcclQos[%u]", hcclQos);
9790 0 : hcclQos_ = hcclQos;
9791 0 : }
9792 :
9793 0 : u32 HcclCommunicator::GetHcclQos()
9794 : {
9795 0 : HCCL_INFO("[HcclCommunicator][host][GetHcclQos] hcclQos[%u]", hcclQos_);
9796 0 : return hcclQos_;
9797 : }
9798 :
9799 532 : HcclResult HcclCommunicator::InitSymmetricMemory()
9800 : {
9801 532 : if (superPodNum_ > 1) {
9802 13 : HCCL_DEBUG("[InitSymmetricMemory] Cross-SuperNode not support symmetric memory");
9803 13 : return HCCL_SUCCESS;
9804 : }
9805 519 : if (deviceType_ != DevType::DEV_TYPE_910_93) {
9806 491 : HCCL_DEBUG("[%s] deviceType:%d not support symmetric memory", __func__, deviceType_);
9807 491 : return HCCL_SUCCESS;
9808 : }
9809 :
9810 28 : u64 stride = commConfig_.GetConfigSymmetricMemoryStride() * GIGABYTE_TO_BYTE;
9811 28 : HCCL_RUN_INFO(
9812 : "InitSymmetricMemory, comm identifier[%s], userRank[%u], userRankSize[%u], stride[%llu], devicePhyId[%u].",
9813 : identifier_.c_str(), realUserRank_, userRankSize_, stride, devicePhyId_);
9814 :
9815 28 : symmetricMemoryAgent_ = std::make_shared<SymmetricMemoryAgent>(
9816 28 : socketManager_, devicePhyId_, deviceLogicId_, localVnicIp_, rankInfoList_, realUserRank_, useSuperPodMode_,
9817 28 : identifier_);
9818 28 : CHK_SMART_PTR_NULL(symmetricMemoryAgent_);
9819 :
9820 28 : symmetricMemory_ = std::make_unique<SymmetricMemory>(realUserRank_, userRankSize_, stride, symmetricMemoryAgent_);
9821 28 : CHK_SMART_PTR_NULL(symmetricMemory_);
9822 28 : return HCCL_SUCCESS;
9823 : }
9824 :
9825 0 : HcclResult HcclCommunicator::RegisterWindow(void* ptr, size_t size, HcclCommSymWindow* winHandle)
9826 : {
9827 0 : CHK_PRT_RET(
9828 : superPodNum_ > 1, HCCL_ERROR("[RegisterWindow] Cross-SuperNode not support symmetric memory"),
9829 : HCCL_E_NOT_SUPPORT);
9830 :
9831 0 : CHK_PRT_RET(
9832 : deviceType_ != DevType::DEV_TYPE_910_93,
9833 : HCCL_ERROR("[%s] deviceType:%d not support symmetric memory", __func__, deviceType_), HCCL_E_NOT_SUPPORT);
9834 :
9835 0 : CHK_SMART_PTR_NULL(symmetricMemory_);
9836 0 : return symmetricMemory_->RegisterSymmetricMem(ptr, size, winHandle);
9837 : }
9838 :
9839 0 : HcclResult HcclCommunicator::DeregisterWindow(HcclCommSymWindow winHandle)
9840 : {
9841 0 : CHK_SMART_PTR_NULL(symmetricMemory_);
9842 0 : return symmetricMemory_->DeregisterSymmetricMem(winHandle);
9843 : }
9844 :
9845 0 : HcclResult HcclCommunicator::GetCommSymWin(void* ptr, size_t size, HcclCommSymWindow* winHandle, size_t* offset)
9846 : {
9847 0 : CHK_SMART_PTR_NULL(symmetricMemory_);
9848 0 : return symmetricMemory_->FindSymmetricWindow(ptr, size, winHandle, reinterpret_cast<u64*>(offset));
9849 : }
9850 :
9851 22 : bool HcclCommunicator::EnableAicpuUnfold(bool isCapture)
9852 : {
9853 22 : if (deviceType_ != DevType::DEV_TYPE_910_93 && deviceType_ != DevType::DEV_TYPE_910B) {
9854 5 : return false;
9855 : }
9856 : // 910B在acl graph场景(isCapture)不启用aicpu展开
9857 17 : if (deviceType_ == DevType::DEV_TYPE_910B && isCapture) {
9858 3 : HCCL_INFO(
9859 : "[%s] deviceType[%d] isCapture[1], aicpuUnfoldConfig[%u] 910B does not support aicpuUnfold in acl graph "
9860 : "mode",
9861 : __func__, deviceType_, GetAicpuUnfoldConfig());
9862 3 : return false;
9863 : }
9864 14 : HCCL_INFO("[%s] aicpuUnfoldConfig[%u]", __func__, GetAicpuUnfoldConfig());
9865 15 : return GetAicpuUnfoldConfig();
9866 : }
9867 :
9868 235 : aclrtBinHandle HcclCommunicator::GetBinHandle()
9869 : {
9870 235 : if (binHandle_ == nullptr) {
9871 235 : HCCL_ERROR("[HcclCommunicator][GetBinHandle] GetBinHandle binHandle failed.binHandle is nullptr");
9872 235 : return nullptr;
9873 : }
9874 0 : return binHandle_;
9875 : }
9876 : HcclResult
9877 0 : HcclCommunicator::GetDevMemWorkSpace(const std::string& memTag, uint64_t* size, void** addr, bool* newCreated)
9878 : {
9879 0 : return dpuManager_->GetDevMemWorkSpace(memTag, size, addr, newCreated);
9880 : }
9881 : } // namespace hccl
|
