Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "coll_comm_aicpu.h"
12 : #include "coll_comm_aicpu_mgr.h"
13 : #include "aicpu_communicator.h"
14 : #include "adapter_hal_pub.h"
15 : #include "ns_recovery/aicpu/ns_recovery_func_lite.h"
16 : #include "dlhal_function_v2.h"
17 : #include "dfx_profiling_command_handle_lite.h"
18 : #include "hcclCommTaskExceptionLite.h"
19 : #include "hcclCommOp.h"
20 : #include "hcclCommDfxLite.h"
21 : #include "env_config/env_config_v2.h"
22 : #include "log.h"
23 :
24 53 : CollCommAicpu::~CollCommAicpu()
25 : {
26 53 : HCCL_RUN_INFO("[CollCommAicpu][%s]Group[%s] destroy success", __func__, identifier_.c_str());
27 53 : }
28 :
29 1 : HcclResult CollCommAicpu::InitAicpuIndOp(CommAicpuParam* commAicpuParam)
30 : {
31 1 : if (commStatus_ == HcclCommStatus::HCCL_COMM_STATUS_READY) {
32 0 : HCCL_RUN_INFO("[CollCommAicpu][%s]Group[%s] already initialized, skip reinit", __func__, identifier_.c_str());
33 0 : return HCCL_SUCCESS;
34 : }
35 1 : CHK_PTR_NULL(commAicpuParam);
36 1 : topoInfo_.deviceLogicId = commAicpuParam->deviceLogicId;
37 1 : topoInfo_.devicePhyId = commAicpuParam->devicePhyId;
38 1 : topoInfo_.deviceType = static_cast<DevType>(commAicpuParam->deviceType);
39 1 : identifier_ = std::string(commAicpuParam->hcomId);
40 1 : topoInfo_.userRankSize = commAicpuParam->userRankSize;
41 1 : topoInfo_.userRank = commAicpuParam->userRank;
42 :
43 1 : CHK_RET(hrtSetWorkModeAicpu(true));
44 1 : CHK_RET(hrtSetlocalDevice(topoInfo_.deviceLogicId));
45 1 : CHK_RET(hrtSetlocalDeviceType(topoInfo_.deviceType));
46 1 : CHK_RET(hrtDrvGetLocalDevIDByHostDevID(topoInfo_.devicePhyId, &devId_));
47 1 : CHK_RET(dfx_.Init(devId_, identifier_, topoInfo_.userRankSize, topoInfo_.userRank));
48 1 : CHK_RET(RegisterProfCallBack());
49 1 : CHK_RET(InitHDCommunicate(commAicpuParam));
50 :
51 1 : EXCEPTION_CATCH(nsRecoveryLitePtr_ = std::make_shared<NsRecoveryLite>(), return HCCL_E_PTR);
52 1 : nsRecoveryLitePtr_->Init(kfcControlTransferH2D_, kfcStatusTransferD2H_);
53 :
54 1 : CHK_RET(Hccl::DlHalFunctionV2::GetInstance().DlHalFunctionInit());
55 :
56 : // commEngineResMgr_/channelMgr_ 为 CollCommAicpu 成员（unique_ptr），生命周期被 this 严格包含，
57 : // 因此 lambda 捕获 this 安全，不会产生悬垂指针（析构顺序见 coll_comm_aicpu.h 成员声明）
58 1 : EXCEPTION_CATCH(
59 : commEngineResMgr_ = std::make_unique<CommEngineResAicpuMgr>(
60 : dfx_,
61 : [this](bool isTimeout) {
62 : return this->CheckIndOpExecStatus(isTimeout);
63 : }),
64 : return HCCL_E_PTR);
65 1 : CHK_SMART_PTR_NULL(commEngineResMgr_);
66 :
67 1 : EXCEPTION_CATCH(channelMgr_ = std::make_unique<ChannelAicpuMgr>(dfx_, topoInfo_), return HCCL_E_PTR);
68 1 : CHK_SMART_PTR_NULL(channelMgr_);
69 :
70 1 : commStatus_ = HcclCommStatus::HCCL_COMM_STATUS_READY;
71 :
72 1 : HCCL_RUN_INFO(
73 : "[%s]success, group[%s], deviceLogicId[%u], devicePhyId[%u], deviceType[%u], rankSize[%u] "
74 : "userRank[%u], devId[%u]",
75 : __func__, identifier_.c_str(), topoInfo_.deviceLogicId, topoInfo_.devicePhyId, topoInfo_.deviceType,
76 : topoInfo_.userRankSize, topoInfo_.userRank, devId_);
77 1 : return HCCL_SUCCESS;
78 : }
79 :
80 1 : HcclResult CollCommAicpu::InitHDCommunicate(CommAicpuParam* commAicpuParam)
81 : {
82 1 : if (commAicpuParam->kfcControlTransferH2DParams.buffLen != 0 && kfcControlTransferH2D_ == nullptr) {
83 0 : EXCEPTION_CATCH((kfcControlTransferH2D_ = std::make_shared<hccl::HDCommunicate>()), return HCCL_E_PTR);
84 0 : CHK_SMART_PTR_NULL(kfcControlTransferH2D_);
85 0 : CHK_RET(kfcControlTransferH2D_->InitDevice(commAicpuParam->kfcControlTransferH2DParams));
86 : }
87 1 : if (commAicpuParam->kfcStatusTransferD2HParams.buffLen != 0 && kfcStatusTransferD2H_ == nullptr) {
88 0 : EXCEPTION_CATCH((kfcStatusTransferD2H_ = std::make_shared<hccl::HDCommunicate>()), return HCCL_E_PTR);
89 0 : CHK_SMART_PTR_NULL(kfcStatusTransferD2H_);
90 0 : CHK_RET(kfcStatusTransferD2H_->InitDevice(commAicpuParam->kfcStatusTransferD2HParams));
91 : }
92 1 : return HCCL_SUCCESS;
93 : }
94 :
95 13 : void CollCommAicpu::SetCommmStatus(HcclCommStatus status)
96 : {
97 13 : HCCL_INFO("[%s]group[%s], commStatus[%d]", __func__, identifier_.c_str(), static_cast<int>(status));
98 13 : commStatus_ = status;
99 13 : }
100 :
101 1 : HcclResult CollCommAicpu::Clean()
102 : {
103 1 : CHK_SMART_PTR_NULL(channelMgr_);
104 0 : return channelMgr_->Clean();
105 : }
106 :
107 1 : HcclResult CollCommAicpu::Resume(HcclChannelUrmaRes* commParam)
108 : {
109 1 : CHK_PTR_NULL(commParam);
110 1 : CHK_SMART_PTR_NULL(channelMgr_);
111 1 : CHK_RET(channelMgr_->Resume(commParam));
112 1 : nsRecoveryLitePtr_->SetNeedClean(false);
113 :
114 1 : SetErrorReported(false);
115 1 : commStatus_ = HcclCommStatus::HCCL_COMM_STATUS_READY;
116 :
117 1 : return HCCL_SUCCESS;
118 : }
119 :
120 3 : hccl::NsRecoveryLitePtr CollCommAicpu::GetNsRecoveryLitePtr() { return nsRecoveryLitePtr_; }
121 :
122 2 : HcclResult CollCommAicpu::CheckIndOpExecStatus(bool timeout)
123 : {
124 2 : if (timeout) {
125 0 : HCCL_ERROR("[%s]comm[%s] op launch timeout, print taskException", __func__, identifier_.c_str());
126 0 : hcomm::HcclCommTaskExceptionLite::GetInstance().PrintCommTaskException(this);
127 0 : hcomm::HcclCommTaskExceptionLite::GetInstance().PrintAllCommTaskException();
128 0 : return HCCL_E_INTERNAL;
129 2 : } else if (commStatus_ == HCCL_COMM_STATUS_SUSPENDING) {
130 1 : HCCL_WARNING("[%s]comm[%s] commStatus[%d] is suspending", __func__, identifier_.c_str(), commStatus_);
131 1 : return HCCL_E_SUSPENDING;
132 1 : } else if (commStatus_ != HCCL_COMM_STATUS_READY) {
133 1 : HCCL_ERROR("[%s]comm[%s] commStatus[%d] is not ready, return fail", __func__, identifier_.c_str(), commStatus_);
134 1 : return HCCL_E_INTERNAL;
135 : }
136 0 : return HCCL_SUCCESS;
137 : }
138 :
139 0 : HcclResult CollCommAicpu::BackGroundGetCmd(Hccl::KfcCommand& cmd)
140 : {
141 0 : CHK_SMART_PTR_NULL(kfcControlTransferH2D_);
142 0 : HcclResult ret = kfcControlTransferH2D_->Get(0, sizeof(Hccl::KfcCommand), reinterpret_cast<uint8_t*>(&cmd));
143 0 : CHK_PRT_RET(ret != HCCL_SUCCESS, HCCL_ERROR("[%s]fail, group[%s]", __func__, identifier_.c_str()), ret);
144 0 : return HCCL_SUCCESS;
145 : }
146 :
147 0 : HcclResult CollCommAicpu::BackGroundSetStatus(Hccl::KfcStatus state)
148 : {
149 0 : Hccl::KfcExecStatus status;
150 0 : status.kfcStatus = state;
151 0 : HCCL_INFO("[%s]group[%s], state[%d]", __func__, identifier_.c_str(), static_cast<int>(state));
152 0 : HcclResult ret = kfcStatusTransferD2H_->Put(0, sizeof(status.kfcStatus), reinterpret_cast<uint8_t*>(&status));
153 0 : CHK_PRT_RET(ret != HCCL_SUCCESS, HCCL_ERROR("[%s]fail, group[%s]", __func__, identifier_.c_str()), ret);
154 0 : return HCCL_SUCCESS;
155 : }
156 :
157 0 : HcclResult CollCommAicpu::SendErrorMessageReportToHost(Hccl::ErrorMessageReport& errMsgInfo)
158 : {
159 0 : CHK_SMART_PTR_NULL(kfcStatusTransferD2H_);
160 0 : CHK_RET(kfcStatusTransferD2H_->Put(
161 : sizeof(Hccl::KfcStatus) + sizeof(Hccl::KfcErrType), sizeof(errMsgInfo),
162 : reinterpret_cast<uint8_t*>(&errMsgInfo)));
163 0 : return HCCL_SUCCESS;
164 : }
165 :
166 1 : HcclResult CollCommAicpu::RegisterProfCallBack() { return Hccl::DfxRegisterProfCallBack(); }
167 :
168 2 : u32 CollCommAicpu::UpdateIndex() { return index_ += 1; }
169 :
170 0 : HcclResult CollCommAicpu::InitDfxOpInfo(HcclDfxOpInfo* aicpuDfxInfo)
171 : {
172 0 : HCCL_INFO(
173 : "[%s]group[%s], algTag[%s], profiling L0[%d], L1[%d]", __func__, identifier_.c_str(), aicpuDfxInfo->algTag,
174 : Hccl::DfxProfilingHandlerLite::GetInstance().GetProfL0State(),
175 : Hccl::DfxProfilingHandlerLite::GetInstance().GetProfL1State());
176 :
177 0 : Hccl::DfxDfxOpInfo newDfxOpInfo{};
178 0 : newDfxOpInfo.opType = static_cast<u8>(aicpuDfxInfo->opType);
179 0 : newDfxOpInfo.dataType = static_cast<u8>(aicpuDfxInfo->dataType);
180 :
181 0 : newDfxOpInfo.commHandle = reinterpret_cast<void*>(this);
182 0 : newDfxOpInfo.count = aicpuDfxInfo->dataCount;
183 0 : newDfxOpInfo.srcAddr = aicpuDfxInfo->inputMemAddr;
184 0 : newDfxOpInfo.dstAddr = aicpuDfxInfo->outputMemAddr;
185 0 : newDfxOpInfo.srcSize = aicpuDfxInfo->inputMemSize;
186 0 : newDfxOpInfo.dstSize = aicpuDfxInfo->outputMemSize;
187 0 : newDfxOpInfo.opIndex = UpdateIndex();
188 0 : newDfxOpInfo.cpuWaitAicpuNotifyId = aicpuDfxInfo->cpuWaitAicpuNotifyId;
189 0 : newDfxOpInfo.algType = static_cast<u8>(Hccl::AlgTypeVal::ALG_TYPE_NOT_SPECIFIED);
190 0 : auto algTagLen = strnlen(aicpuDfxInfo->algTag, sizeof(newDfxOpInfo.algTag) - 1);
191 0 : CHK_SAFETY_FUNC_RET(
192 : memcpy_s(newDfxOpInfo.algTag, sizeof(newDfxOpInfo.algTag) - 1, aicpuDfxInfo->algTag, algTagLen));
193 :
194 0 : CHK_RET(dfx_.SetCurrDfxOpInfo(&newDfxOpInfo));
195 0 : return HCCL_SUCCESS;
196 : }
197 :
198 0 : HcclResult CollCommAicpu::ProfilingReportDeviceOp()
199 : {
200 0 : HcclCommDfxLite* hcclCommDfxLite = GetHcclCommDfxLite();
201 0 : CHK_PTR_NULL(hcclCommDfxLite);
202 0 : auto* currDfxOpInfo = static_cast<const Hccl::DfxDfxOpInfo*>(hcclCommDfxLite->GetLatestDfxOpInfo());
203 0 : if (currDfxOpInfo == nullptr) {
204 0 : HCCL_WARNING("[%s] no op info registered, skip ProfilingReportDeviceOp.", __func__);
205 0 : return HCCL_SUCCESS;
206 : }
207 :
208 0 : const auto& sharedThreads = commEngineResMgr_->GetAllThread();
209 0 : std::vector<hccl::Thread*> threads;
210 0 : threads.reserve(sharedThreads.size());
211 0 : for (const auto& t : sharedThreads) {
212 0 : threads.push_back(t.get());
213 : }
214 0 : hcclCommDfxLite->ReportAllTasks(threads);
215 0 : EXCEPTION_CATCH(
216 : Hccl::DfxProfilingHandlerLite::GetInstance().ReportHcclOpInfo(*currDfxOpInfo), return HCCL_E_INTERNAL);
217 0 : return HCCL_SUCCESS;
218 0 : }
219 :
220 0 : HcclResult CollCommAicpu::UpdateTask()
221 : {
222 0 : CHK_RET(dfx_.UpdateProfStat());
223 0 : return HCCL_SUCCESS;
224 : }
225 :
226 51 : hccl::HcclCommAicpu* CollCommAicpu::GetLegacy910CollComm() { return legacy910CollComm_.first.get(); }
227 :
228 11 : void CollCommAicpu::SetLegacy910CollComm(std::shared_ptr<hccl::HcclCommAicpu> comm)
229 : {
230 11 : legacy910CollComm_.first = std::move(comm);
231 11 : }
232 :
233 3 : bool CollCommAicpu::IsLegacy910CollCommBusy() { return legacy910CollComm_.second.load(); }
234 :
235 59 : void CollCommAicpu::SetLegacy910CollCommBusy(bool busy) { legacy910CollComm_.second.store(busy); }
|
