Line data Source code
1 : /**
2 : * Copyright (c) 2026 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "ns_recovery.h"
12 : #include "channel_process.h"
13 : #include "log.h"
14 : #include "comm_engine_utils.h"
15 :
16 : namespace hccl {
17 :
18 167 : void NsRecoveryProcessor::SetKfcControlTransfer(
19 : std::shared_ptr<HDCommunicate> kfcControlTransferH2D, std::shared_ptr<HDCommunicate> kfcStatusTransferD2H)
20 : {
21 167 : kfcControlTransferH2D_ = kfcControlTransferH2D;
22 167 : kfcStatusTransferD2H_ = kfcStatusTransferD2H;
23 167 : }
24 :
25 14 : void NsRecoveryProcessor::AddNsRecoveryData(
26 : const CommEngine& engine, const ChannelHandle* const channelHandles,
27 : const ChannelHandle* const hostChannelHandleList, uint32_t channelNum, const std::string& commTag)
28 : {
29 14 : HCCL_INFO(
30 : "[NsRecovery][AddData] AddNsRecoveryData for engine[%s], channelNum[%u], commTag[%s]",
31 : GetEnumToString(GetCommEngineStatusStrMap(), engine).c_str(), channelNum, commTag.c_str());
32 14 : std::vector<ChannelHandle> deviceList;
33 14 : std::vector<ChannelHandle> hostList;
34 29 : for (uint32_t index = 0; index < channelNum; ++index) {
35 15 : deviceList.push_back(channelHandles[index]);
36 15 : hostList.push_back(hostChannelHandleList[index]);
37 : }
38 14 : NsRecoveryData data{deviceList, hostList, channelNum, commTag};
39 14 : nsRecoveryDatas_[engine].emplace_back(std::move(data));
40 14 : }
41 :
42 : constexpr u32 WAIT_CMD_TIMEOUT = 10 * 1000; // 最大等待10秒
43 2 : HcclResult NsRecoveryProcessor::PollStopStatus()
44 : {
45 2 : Hccl::KfcExecStatus opInfo;
46 2 : auto timeout = std::chrono::milliseconds(WAIT_CMD_TIMEOUT);
47 2 : auto startTime = std::chrono::steady_clock::now();
48 : while (true) {
49 2 : CHK_RET(kfcStatusTransferD2H_->Get(0, sizeof(Hccl::KfcExecStatus), reinterpret_cast<uint8_t*>(&opInfo)));
50 2 : if (opInfo.kfcStatus == Hccl::KfcStatus::STOP_LAUNCH_DONE) {
51 2 : HCCL_INFO(
52 : "[NsRecovery][Suspend] received KfcStatus[%d], which is STOP_LAUNCH_DONE",
53 : static_cast<int>(opInfo.kfcStatus));
54 2 : return HcclResult::HCCL_E_SUSPENDING;
55 0 : } else if (opInfo.kfcStatus == Hccl::KfcStatus::ERROR) {
56 0 : HCCL_ERROR(
57 : "[NsRecovery][Suspend] received KfcStatus[%d], which is ERROR", static_cast<int>(opInfo.kfcStatus));
58 0 : return HcclResult::HCCL_E_INTERNAL;
59 : } else {
60 0 : if ((std::chrono::steady_clock::now() - startTime) >= timeout) {
61 0 : HCCL_ERROR(
62 : "[NsRecovery][Suspend] Wait suspend response status timeout[%u ms] and get the kfcStatus is [%d].",
63 : WAIT_CMD_TIMEOUT, static_cast<int>(opInfo.kfcStatus));
64 0 : return HcclResult::HCCL_E_TIMEOUT;
65 : }
66 0 : continue;
67 : }
68 0 : }
69 : return HcclResult::HCCL_E_INTERNAL;
70 : }
71 :
72 1 : HcclResult NsRecoveryProcessor::ListenBackGround(Hccl::KfcExecStatus& opInfo)
73 : {
74 1 : auto timeout = std::chrono::milliseconds(WAIT_CMD_TIMEOUT);
75 1 : auto startTime = std::chrono::steady_clock::now();
76 : while (true) {
77 1 : CHK_RET(kfcStatusTransferD2H_->Get(0, sizeof(Hccl::KfcExecStatus), reinterpret_cast<uint8_t*>(&opInfo)));
78 1 : if (opInfo.kfcStatus == Hccl::KfcStatus::CLEAN_DONE) {
79 1 : HCCL_INFO(
80 : "[NsRecovery][Clean] received KfcStatus[%d], which is CLEAN_DONE", static_cast<int>(opInfo.kfcStatus));
81 1 : return HcclResult::HCCL_E_SUSPENDING;
82 0 : } else if (opInfo.kfcStatus == Hccl::KfcStatus::ERROR) {
83 0 : HCCL_ERROR(
84 : "[NsRecovery][Clean] received KfcStatus[%d], which is ERROR", static_cast<int>(opInfo.kfcStatus));
85 0 : return HcclResult::HCCL_E_INTERNAL;
86 : } else {
87 0 : if ((std::chrono::steady_clock::now() - startTime) >= timeout) {
88 0 : HCCL_ERROR(
89 : "[NsRecovery][Clean] Wait clean response status timeout[%u ms] and get the kfcStatus is [%d].",
90 : WAIT_CMD_TIMEOUT, static_cast<int>(opInfo.kfcStatus));
91 0 : return HcclResult::HCCL_E_TIMEOUT;
92 : }
93 0 : continue;
94 : }
95 0 : }
96 : return HcclResult::HCCL_E_INTERNAL;
97 : }
98 :
99 3 : HcclResult NsRecoveryProcessor::StopLaunch()
100 : {
101 4 : for (const auto& recoveryData : nsRecoveryDatas_) {
102 3 : if (recoveryData.first == COMM_ENGINE_AICPU || recoveryData.first == COMM_ENGINE_AICPU_TS) {
103 : // Aicpu场景
104 2 : Hccl::KfcCommand opCmd = Hccl::KfcCommand::NS_STOP_LAUNCH;
105 2 : CHK_RET(kfcControlTransferH2D_->Put(0, sizeof(Hccl::KfcCommand), reinterpret_cast<uint8_t*>(&opCmd)));
106 2 : HCCL_INFO(
107 : "[NsRecovery][Suspend] send KfcCommand[%d] success, which is NS_STOP_LAUNCH.", static_cast<int>(opCmd));
108 :
109 2 : auto ret = PollStopStatus(); // todo：多CommEngine的管理存在问题
110 2 : if (ret != HcclResult::HCCL_E_SUSPENDING) {
111 0 : HCCL_ERROR("[NsRecovery][Suspend] PollStopStatus failed, ret[%d]", ret);
112 0 : return ret;
113 : }
114 2 : return HcclResult::HCCL_SUCCESS;
115 : } else {
116 1 : HCCL_INFO("[NsRecovery][Suspend] Aicpu kernel is not launched yet. Suspend host only.");
117 : }
118 : }
119 :
120 1 : return HcclResult::HCCL_SUCCESS;
121 : }
122 :
123 4 : HcclResult NsRecoveryProcessor::Clean()
124 : {
125 5 : for (const auto& recoveryData : nsRecoveryDatas_) {
126 4 : if (recoveryData.first == COMM_ENGINE_AICPU || recoveryData.first == COMM_ENGINE_AICPU_TS) {
127 : // 再清理device，后续优化全用host管理
128 3 : HCCL_INFO("[NsRecovery][Clean] start to clean device, waiting for device STOP_LAUNCH_DONE");
129 3 : Hccl::KfcExecStatus opInfo;
130 3 : CHK_RET(kfcStatusTransferD2H_->Get(0, sizeof(Hccl::KfcExecStatus), reinterpret_cast<uint8_t*>(&opInfo)));
131 3 : if (opInfo.kfcStatus == Hccl::KfcStatus::STOP_LAUNCH_DONE) {
132 1 : HCCL_INFO(
133 : "[NsRecovery][Clean] received KfcStatus[%d], which is STOP_LAUNCH_DONE",
134 : static_cast<int>(opInfo.kfcStatus));
135 : // 通知背景线程清理device侧资源
136 1 : Hccl::KfcCommand opCmd = Hccl::KfcCommand::NS_CLEAN;
137 1 : CHK_RET(kfcControlTransferH2D_->Put(0, sizeof(Hccl::KfcCommand), reinterpret_cast<uint8_t*>(&opCmd)));
138 1 : HCCL_INFO(
139 : "[NsRecovery][Clean] send KfcCommand [%d] success, which is NS_CLEAN", static_cast<int>(opCmd));
140 :
141 : // 监听背景线程状态
142 1 : auto ret = ListenBackGround(opInfo);
143 1 : if (ret != HcclResult::HCCL_E_SUSPENDING) {
144 0 : HCCL_ERROR("[NsRecovery][Clean] ListenBackGround failed, ret[%d]", ret);
145 0 : return ret;
146 : }
147 1 : return HcclResult::HCCL_SUCCESS;
148 : } else {
149 2 : HCCL_ERROR(
150 : "[NsRecovery][Clean] Aicpu kernel is not stopped yet. Cannot clean, kfcStatus is [%s]",
151 : opInfo.kfcStatus.Describe().c_str());
152 2 : return HcclResult::HCCL_E_INTERNAL;
153 : }
154 : return HcclResult::HCCL_SUCCESS;
155 : }
156 : }
157 :
158 1 : return HcclResult::HCCL_SUCCESS;
159 : }
160 :
161 3 : HcclResult NsRecoveryProcessor::Resume(aclrtBinHandle binHandle)
162 : {
163 5 : for (auto& recoveryData : nsRecoveryDatas_) {
164 3 : if (recoveryData.first == COMM_ENGINE_AICPU || recoveryData.first == COMM_ENGINE_AICPU_TS) {
165 3 : for (auto& handleData : recoveryData.second) {
166 2 : CHK_RET(hcomm::ChannelProcess::ChannelUpdateKernelLaunch(
167 : handleData.channelHandles_.data(), handleData.hostChannelHandleList_.data(), handleData.channelNum_,
168 : handleData.commTag_, binHandle));
169 : }
170 : }
171 : }
172 2 : return HCCL_SUCCESS;
173 : }
174 :
175 : } // namespace hccl
|
