Line data Source code
1 : /**
2 : * Copyright (c) 2026 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "ns_recovery.h"
12 : #include "channel_process.h"
13 : #include "log.h"
14 : #include "comm_engine_utils.h"
15 :
16 :
17 : namespace hccl
18 : {
19 :
20 119 : void NsRecoveryProcessor::SetKfcControlTransfer(std::shared_ptr<HDCommunicate> kfcControlTransferH2D,
21 : std::shared_ptr<HDCommunicate> kfcStatusTransferD2H)
22 : {
23 119 : kfcControlTransferH2D_ = kfcControlTransferH2D;
24 119 : kfcStatusTransferD2H_ = kfcStatusTransferD2H;
25 119 : }
26 :
27 14 : void NsRecoveryProcessor::AddNsRecoveryData(const CommEngine& engine, const ChannelHandle *const channelHandles,
28 : const ChannelHandle *const hostChannelHandleList, uint32_t channelNum, const std::string &commTag)
29 : {
30 14 : HCCL_INFO("[NsRecovery][AddData] AddNsRecoveryData for engine[%s], channelNum[%u], commTag[%s]",
31 : GetEnumToString(GetCommEngineStatusStrMap(), engine).c_str(), channelNum, commTag.c_str());
32 14 : std::vector<ChannelHandle> deviceList;
33 14 : std::vector<ChannelHandle> hostList;
34 29 : for (uint32_t index = 0; index < channelNum; ++index) {
35 15 : deviceList.push_back(channelHandles[index]);
36 15 : hostList.push_back(hostChannelHandleList[index]);
37 : }
38 14 : NsRecoveryData data{deviceList, hostList, channelNum, commTag};
39 14 : nsRecoveryDatas_[engine].emplace_back(std::move(data));
40 14 : }
41 :
42 : constexpr u32 WAIT_CMD_TIMEOUT = 10 * 1000; // 最大等待10秒
43 2 : HcclResult NsRecoveryProcessor::PollStopStatus()
44 : {
45 2 : Hccl::KfcExecStatus opInfo;
46 2 : auto timeout = std::chrono::milliseconds(WAIT_CMD_TIMEOUT);
47 2 : auto startTime = std::chrono::steady_clock::now();
48 : while (true) {
49 2 : CHK_RET(kfcStatusTransferD2H_->Get(0, sizeof(Hccl::KfcExecStatus), reinterpret_cast<uint8_t *>(&opInfo)));
50 2 : if (opInfo.kfcStatus == Hccl::KfcStatus::STOP_LAUNCH_DONE) {
51 2 : HCCL_INFO("[NsRecovery][Suspend] received KfcStatus[%d], which is STOP_LAUNCH_DONE", static_cast<int>(opInfo.kfcStatus));
52 2 : return HcclResult::HCCL_E_SUSPENDING;
53 0 : } else if (opInfo.kfcStatus == Hccl::KfcStatus::ERROR){
54 0 : HCCL_ERROR("[NsRecovery][Suspend] received KfcStatus[%d], which is ERROR", static_cast<int>(opInfo.kfcStatus));
55 0 : return HcclResult::HCCL_E_INTERNAL;
56 : } else {
57 0 : if((std::chrono::steady_clock::now() - startTime) >= timeout){
58 0 : HCCL_ERROR("[NsRecovery][Suspend] Wait suspend response status timeout[%u ms] and get the kfcStatus is [%d].", WAIT_CMD_TIMEOUT,
59 : static_cast<int>(opInfo.kfcStatus));
60 0 : return HcclResult::HCCL_E_TIMEOUT;
61 : }
62 0 : continue;
63 : }
64 0 : }
65 : return HcclResult::HCCL_E_INTERNAL;
66 : }
67 :
68 1 : HcclResult NsRecoveryProcessor::ListenBackGround(Hccl::KfcExecStatus& opInfo)
69 : {
70 1 : auto timeout = std::chrono::milliseconds(WAIT_CMD_TIMEOUT);
71 1 : auto startTime = std::chrono::steady_clock::now();
72 : while (true) {
73 1 : CHK_RET(kfcStatusTransferD2H_->Get(0, sizeof(Hccl::KfcExecStatus), reinterpret_cast<uint8_t *>(&opInfo)));
74 1 : if (opInfo.kfcStatus == Hccl::KfcStatus::CLEAN_DONE) {
75 1 : HCCL_INFO("[NsRecovery][Clean] received KfcStatus[%d], which is CLEAN_DONE", static_cast<int>(opInfo.kfcStatus));
76 1 : return HcclResult::HCCL_E_SUSPENDING;
77 0 : } else if (opInfo.kfcStatus == Hccl::KfcStatus::ERROR){
78 0 : HCCL_ERROR("[NsRecovery][Clean] received KfcStatus[%d], which is ERROR", static_cast<int>(opInfo.kfcStatus));
79 0 : return HcclResult::HCCL_E_INTERNAL;
80 : } else {
81 0 : if ((std::chrono::steady_clock::now() - startTime) >= timeout) {
82 0 : HCCL_ERROR("[NsRecovery][Clean] Wait clean response status timeout[%u ms] and get the kfcStatus is [%d].", WAIT_CMD_TIMEOUT,
83 : static_cast<int>(opInfo.kfcStatus));
84 0 : return HcclResult::HCCL_E_TIMEOUT;
85 : }
86 0 : continue;
87 : }
88 0 : }
89 : return HcclResult::HCCL_E_INTERNAL;
90 : }
91 :
92 3 : HcclResult NsRecoveryProcessor::StopLaunch()
93 : {
94 4 : for (const auto& recoveryData : nsRecoveryDatas_) {
95 3 : if (recoveryData.first == COMM_ENGINE_AICPU || recoveryData.first == COMM_ENGINE_AICPU_TS) {
96 : // Aicpu场景
97 2 : Hccl::KfcCommand opCmd = Hccl::KfcCommand::NS_STOP_LAUNCH;
98 2 : CHK_RET(kfcControlTransferH2D_->Put(0, sizeof(Hccl::KfcCommand), reinterpret_cast<uint8_t *>(&opCmd)));
99 2 : HCCL_INFO("[NsRecovery][Suspend] send KfcCommand[%d] success, which is NS_STOP_LAUNCH.", static_cast<int>(opCmd));
100 :
101 2 : auto ret = PollStopStatus(); // todo：多CommEngine的管理存在问题
102 2 : if (ret != HcclResult::HCCL_E_SUSPENDING) {
103 0 : HCCL_ERROR("[NsRecovery][Suspend] PollStopStatus failed, ret[%d]", ret);
104 0 : return ret;
105 : }
106 2 : return HcclResult::HCCL_SUCCESS;
107 : } else {
108 1 : HCCL_INFO("[NsRecovery][Suspend] Aicpu kernel is not launched yet. Suspend host only.");
109 : }
110 : }
111 :
112 1 : return HcclResult::HCCL_SUCCESS;
113 : }
114 :
115 4 : HcclResult NsRecoveryProcessor::Clean()
116 : {
117 5 : for (const auto& recoveryData : nsRecoveryDatas_) {
118 4 : if (recoveryData.first == COMM_ENGINE_AICPU || recoveryData.first == COMM_ENGINE_AICPU_TS) {
119 : // 再清理device，后续优化全用host管理
120 3 : HCCL_INFO("[NsRecovery][Clean] start to clean device, waiting for device STOP_LAUNCH_DONE");
121 3 : Hccl::KfcExecStatus opInfo;
122 3 : CHK_RET(kfcStatusTransferD2H_->Get(0, sizeof(Hccl::KfcExecStatus), reinterpret_cast<uint8_t *>(&opInfo)));
123 3 : if (opInfo.kfcStatus == Hccl::KfcStatus::STOP_LAUNCH_DONE) {
124 1 : HCCL_INFO("[NsRecovery][Clean] received KfcStatus[%d], which is STOP_LAUNCH_DONE", static_cast<int>(opInfo.kfcStatus));
125 : // 通知背景线程清理device侧资源
126 1 : Hccl::KfcCommand opCmd = Hccl::KfcCommand::NS_CLEAN;
127 1 : CHK_RET(kfcControlTransferH2D_->Put(0, sizeof(Hccl::KfcCommand), reinterpret_cast<uint8_t *>(&opCmd)));
128 1 : HCCL_INFO("[NsRecovery][Clean] send KfcCommand [%d] success, which is NS_CLEAN", static_cast<int>(opCmd));
129 :
130 : // 监听背景线程状态
131 1 : auto ret = ListenBackGround(opInfo);
132 1 : if (ret != HcclResult::HCCL_E_SUSPENDING) {
133 0 : HCCL_ERROR("[NsRecovery][Clean] ListenBackGround failed, ret[%d]", ret);
134 0 : return ret;
135 : }
136 1 : return HcclResult::HCCL_SUCCESS;
137 : } else {
138 2 : HCCL_ERROR("[NsRecovery][Clean] Aicpu kernel is not stopped yet. Cannot clean, kfcStatus is [%s]",
139 : opInfo.kfcStatus.Describe().c_str());
140 2 : return HcclResult::HCCL_E_INTERNAL;
141 : }
142 : return HcclResult::HCCL_SUCCESS;
143 : }
144 : }
145 :
146 1 : return HcclResult::HCCL_SUCCESS;
147 : }
148 :
149 3 : HcclResult NsRecoveryProcessor::Resume(aclrtBinHandle binHandle)
150 : {
151 5 : for (auto& recoveryData : nsRecoveryDatas_) {
152 3 : if (recoveryData.first == COMM_ENGINE_AICPU || recoveryData.first == COMM_ENGINE_AICPU_TS) {
153 3 : for (auto& handleData : recoveryData.second) {
154 2 : CHK_RET(hcomm::ChannelProcess::ChannelUpdateKernelLaunch(handleData.channelHandles_.data(), handleData.hostChannelHandleList_.data(),
155 : handleData.channelNum_, handleData.commTag_, binHandle));
156 : }
157 : }
158 : }
159 2 : return HCCL_SUCCESS;
160 : }
161 :
162 : }
|
