Line data Source code
1 : /**
2 : * Copyright (c) 2026 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "ns_recovery.h"
12 : #include "channel_process.h"
13 : #include "log.h"
14 : #include "comm_engine_utils.h"
15 :
16 :
17 : namespace hccl
18 : {
19 :
20 115 : void NsRecoveryProcessor::SetKfcControlTransfer(std::shared_ptr<HDCommunicate> kfcControlTransferH2D,
21 : std::shared_ptr<HDCommunicate> kfcStatusTransferD2H)
22 : {
23 115 : kfcControlTransferH2D_ = kfcControlTransferH2D;
24 115 : kfcStatusTransferD2H_ = kfcStatusTransferD2H;
25 115 : }
26 :
27 11 : void NsRecoveryProcessor::AddNsRecoveryData(const CommEngine& engine, const ChannelHandle *const channelHandles,
28 : const ChannelHandle *const hostChannelHandleList, uint32_t channelNum, const std::string &commTag)
29 : {
30 11 : HCCL_INFO("[NsRecovery][AddData] AddNsRecoveryData for engine[%s], channelNum[%u], commTag[%s]",
31 : GetEnumToString(GetCommEngineStatusStrMap(), engine).c_str(), channelNum, commTag.c_str());
32 11 : std::vector<ChannelHandle> deviceList;
33 11 : std::vector<ChannelHandle> hostList;
34 23 : for (uint32_t index = 0; index < channelNum; ++index) {
35 12 : deviceList.push_back(channelHandles[index]);
36 12 : hostList.push_back(hostChannelHandleList[index]);
37 : }
38 11 : NsRecoveryData data{deviceList, hostList, channelNum, commTag};
39 11 : nsRecoveryDatas_[engine].emplace_back(std::move(data));
40 11 : }
41 :
42 : constexpr u32 WAIT_CMD_TIMEOUT = 10 * 1000; // 最大等待10秒
43 0 : HcclResult NsRecoveryProcessor::PollStopStatus()
44 : {
45 0 : Hccl::KfcExecStatus opInfo;
46 0 : auto timeout = std::chrono::milliseconds(WAIT_CMD_TIMEOUT);
47 0 : auto startTime = std::chrono::steady_clock::now();
48 : while (true) {
49 0 : CHK_RET(kfcStatusTransferD2H_->Get(0, sizeof(Hccl::KfcExecStatus), reinterpret_cast<uint8_t *>(&opInfo)));
50 0 : if (opInfo.kfcStatus == Hccl::KfcStatus::STOP_LAUNCH_DONE) {
51 0 : HCCL_INFO("[NsRecovery][Suspend] received KfcStatus[%d], which is STOP_LAUNCH_DONE", opInfo.kfcStatus);
52 0 : return HcclResult::HCCL_E_SUSPENDING;
53 0 : } else if (opInfo.kfcStatus == Hccl::KfcStatus::ERROR){
54 0 : HCCL_ERROR("[NsRecovery][Suspend] received KfcStatus[%d], which is ERROR", opInfo.kfcStatus);
55 0 : return HcclResult::HCCL_E_INTERNAL;
56 : } else {
57 0 : if((std::chrono::steady_clock::now() - startTime) >= timeout){
58 0 : HCCL_ERROR("[NsRecovery][Suspend] Wait suspend response status timeout[%u ms] and get the opExecStatus is [%u].", WAIT_CMD_TIMEOUT,
59 : opInfo.kfcStatus);
60 0 : return HcclResult::HCCL_E_TIMEOUT;
61 : }
62 0 : continue;
63 : }
64 0 : }
65 : return HcclResult::HCCL_E_INTERNAL;
66 : }
67 :
68 0 : HcclResult NsRecoveryProcessor::ListenBackGround(Hccl::KfcExecStatus& opInfo)
69 : {
70 0 : auto timeout = std::chrono::milliseconds(WAIT_CMD_TIMEOUT);
71 0 : auto startTime = std::chrono::steady_clock::now();
72 : while (true) {
73 0 : CHK_RET(kfcStatusTransferD2H_->Get(0, sizeof(Hccl::KfcExecStatus), reinterpret_cast<uint8_t *>(&opInfo)));
74 0 : if (opInfo.kfcStatus == Hccl::KfcStatus::CLEAN_DONE) {
75 0 : HCCL_INFO("[NsRecovery][Clean] received KfcStatus[%d], which is CLEAN_DONE", opInfo.kfcStatus);
76 0 : return HcclResult::HCCL_E_SUSPENDING;
77 0 : } else if (opInfo.kfcStatus == Hccl::KfcStatus::ERROR){
78 0 : HCCL_ERROR("[NsRecovery][Clean] received KfcStatus[%d], which is ERROR", opInfo.kfcStatus);
79 0 : return HcclResult::HCCL_E_INTERNAL;
80 : } else {
81 0 : if ((std::chrono::steady_clock::now() - startTime) >= timeout) {
82 0 : HCCL_ERROR("[NsRecovery][Clean] Wait clean response status timeout[%u ms] and get the opExecStatus is [%u].", WAIT_CMD_TIMEOUT,
83 : opInfo.kfcStatus);
84 0 : return HcclResult::HCCL_E_TIMEOUT;
85 : }
86 0 : continue;
87 : }
88 0 : }
89 : return HcclResult::HCCL_E_INTERNAL;
90 : }
91 :
92 1 : HcclResult NsRecoveryProcessor::StopLaunch()
93 : {
94 2 : for (const auto& recoveryData : nsRecoveryDatas_) {
95 1 : if (recoveryData.first == COMM_ENGINE_AICPU || recoveryData.first == COMM_ENGINE_AICPU_TS) {
96 : // Aicpu场景
97 0 : Hccl::KfcCommand opCmd = Hccl::KfcCommand::NS_STOP_LAUNCH;
98 0 : CHK_RET(kfcControlTransferH2D_->Put(0, sizeof(Hccl::KfcCommand), reinterpret_cast<uint8_t *>(&opCmd)));
99 0 : HCCL_INFO("[NsRecovery][Suspend] send KfcCommand[%d] success, which is NS_STOP_LAUNCH.", opCmd);
100 :
101 0 : auto ret = PollStopStatus(); // todo：多CommEngine的管理存在问题
102 0 : if (ret != HcclResult::HCCL_E_SUSPENDING) {
103 0 : HCCL_ERROR("[NsRecovery][Suspend] PollStopStatus failed, ret[%d]", ret);
104 0 : return ret;
105 : }
106 0 : return HcclResult::HCCL_SUCCESS;
107 : } else {
108 1 : HCCL_INFO("[NsRecovery][Suspend] Aicpu kernel is not launched yet. Suspend host only.");
109 : }
110 : }
111 :
112 1 : return HcclResult::HCCL_SUCCESS;
113 : }
114 :
115 3 : HcclResult NsRecoveryProcessor::Clean()
116 : {
117 4 : for (const auto& recoveryData : nsRecoveryDatas_) {
118 3 : if (recoveryData.first == COMM_ENGINE_AICPU || recoveryData.first == COMM_ENGINE_AICPU_TS) {
119 : // 再清理device，后续优化全用host管理
120 2 : HCCL_INFO("[NsRecovery][Clean] start to clean device, waiting for device STOP_LAUNCH_DONE");
121 2 : Hccl::KfcExecStatus opInfo;
122 2 : CHK_RET(kfcStatusTransferD2H_->Get(0, sizeof(Hccl::KfcExecStatus), reinterpret_cast<uint8_t *>(&opInfo)));
123 2 : if (opInfo.kfcStatus == Hccl::KfcStatus::STOP_LAUNCH_DONE) {
124 0 : HCCL_INFO("[NsRecovery][Clean] received KfcStatus[%d], which is STOP_LAUNCH_DONE", opInfo.kfcStatus);
125 : // 通知背景线程清理device侧资源
126 0 : Hccl::KfcCommand opCmd = Hccl::KfcCommand::NS_CLEAN;
127 0 : CHK_RET(kfcControlTransferH2D_->Put(0, sizeof(Hccl::KfcCommand), reinterpret_cast<uint8_t *>(&opCmd)));
128 0 : HCCL_INFO("[NsRecovery][Clean] send KfcCommand [%d] success, which is NS_CLEAN", opCmd);
129 :
130 : // 监听背景线程状态
131 0 : auto ret = ListenBackGround(opInfo);
132 0 : if (ret != HcclResult::HCCL_E_SUSPENDING) {
133 0 : HCCL_ERROR("[NsRecovery][Clean] ListenBackGround failed, ret[%d]", ret);
134 0 : return ret;
135 : }
136 0 : return HcclResult::HCCL_SUCCESS;
137 : } else {
138 2 : HCCL_ERROR("[NsRecovery][Clean] Aicpu kernel is not stopped yet. Cannot clean, kfcStatus is [%s]",
139 : opInfo.kfcStatus.Describe().c_str());
140 2 : return HcclResult::HCCL_E_INTERNAL;
141 : }
142 : return HcclResult::HCCL_SUCCESS;
143 : }
144 : }
145 :
146 1 : return HcclResult::HCCL_SUCCESS;
147 : }
148 :
149 3 : HcclResult NsRecoveryProcessor::Resume(aclrtBinHandle binHandle)
150 : {
151 5 : for (auto& recoveryData : nsRecoveryDatas_) {
152 3 : if (recoveryData.first == COMM_ENGINE_AICPU || recoveryData.first == COMM_ENGINE_AICPU_TS) {
153 3 : for (auto& handleData : recoveryData.second) {
154 2 : CHK_RET(hcomm::ChannelProcess::ChannelUpdateKernelLaunch(handleData.channelHandles_.data(), handleData.hostChannelHandleList_.data(),
155 : handleData.channelNum_, handleData.commTag_, binHandle));
156 : }
157 : }
158 : }
159 2 : return HCCL_SUCCESS;
160 : }
161 :
162 : }
|
