Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "ns_recovery_handler_func.h"
12 : #include "kfc.h"
13 : #include "drv_api_exception.h"
14 : #include "exception_util.h"
15 : #include "internal_exception.h"
16 :
17 : namespace Hccl {
18 6 : NsRecoveryHandlerFunc &NsRecoveryHandlerFunc::GetInstance()
19 : {
20 6 : static NsRecoveryHandlerFunc func;
21 6 : return func;
22 : }
23 :
24 1 : void NsRecoveryHandlerFunc::Call()
25 : {
26 1 : std::vector<CommunicatorImplLite *> commLites = CommunicatorImplLiteMgr::GetInstance().GetAll();
27 3 : for (auto &comm : commLites) {
28 2 : if (!comm->IsCommReady()) {
29 2 : continue;
30 : }
31 0 : HandleStopLaunch(comm);
32 0 : HandleClean(comm);
33 : }
34 1 : }
35 :
36 1 : void NsRecoveryHandlerFunc::HandleStopLaunch(CommunicatorImplLite *comm) const
37 : {
38 1 : if (comm->IsSuspended()) {
39 0 : return;
40 : }
41 :
42 1 : KfcCommand cmd = comm->BackGroundGetCmd();
43 1 : if (cmd != KfcCommand::NS_STOP_LAUNCH) {
44 0 : return;
45 : }
46 3 : HCCL_INFO("[NsRecovery][BackGround] received KfcCommand[NS_STOP_LAUNCH]");
47 1 : comm->SetNeedClean(true);
48 1 : comm->SetIsSuspended(true);
49 1 : comm->BackGroundSetStatus(KfcStatus::STOP_LAUNCH_DONE);
50 3 : HCCL_INFO("[NsRecovery][BackGround] send KfcStatus[STOP_LAUNCH_DONE]");
51 : }
52 :
53 1 : void NsRecoveryHandlerFunc::HandleClean(CommunicatorImplLite *comm)
54 : {
55 1 : if (!comm->IsNeedClean()) {
56 0 : return;
57 : }
58 1 : KfcCommand cmd = comm->BackGroundGetCmd();
59 1 : if (cmd != KfcCommand::NS_CLEAN) {
60 0 : return;
61 : }
62 3 : HCCL_INFO("[NsRecovery][BackGround] received KfcCommand[NS_CLEAN]");
63 1 : comm->GetTransportLiteMgr()->Reset();
64 1 : StreamClean(comm);
65 1 : comm->SetNeedClean(false);
66 1 : comm->BackGroundSetStatus(KfcStatus::CLEAN_DONE);
67 1 : comm->ResetErrorReported();
68 3 : HCCL_INFO("[NsRecovery][BackGround] send KfcStatus[CLEAN_DONE]");
69 : }
70 :
71 1 : void NsRecoveryHandlerFunc::StreamClean(CommunicatorImplLite *comm)
72 : {
73 : // 查询停流是否完成
74 1 : u32 localDevId=0;
75 1 : auto ret = drvGetLocalDevIDByHostDevID(comm->GetDevPhyId(), &localDevId);
76 1 : if (ret != DRV_ERROR_NONE) {
77 : std::string formatStr = StringFormat(
78 0 : "NsRecoveryHandlerFunc::%s call drvGetLocalDevIDByHostDevID failed, devPhyId %u, ret %d", __func__, comm->GetDevPhyId(), ret);
79 0 : THROW<DrvApiException>(formatStr);
80 0 : }
81 1 : if (DeviceQuery(localDevId, APP_ABORT_STAUTS::APP_ABORT_KILL_FINISH, 0U) != HCCL_SUCCESS) {
82 0 : comm->BackGroundSetStatus(KfcStatus::ERROR, KfcErrType::EXEC);
83 0 : THROW<InternalException>("[NsRecovery][BackGround] Stream Stop failed");
84 : }
85 : // 清理资源
86 1 : auto streamLiteMgr = comm->GetStreamLiteMgr();
87 1 : CHECK_NULLPTR(streamLiteMgr->GetMaster(), "[StreamClean]master stream is nullptr!");
88 1 : streamLiteMgr->GetMaster()->GetRtsq()->Reset();
89 1 : for (u32 i = 0; i < streamLiteMgr->SizeOfSlaves(); ++i) {
90 0 : streamLiteMgr->GetSlave(i)->GetRtsq()->Reset();
91 : }
92 3 : HCCL_INFO("[NsRecovery][BackGround] StreamClean success.");
93 1 : }
94 :
95 : constexpr u64 NSEC_PER_SEC = 1000000000U;
96 :
97 2 : inline u64 GetCurCpuTimestamp()
98 : {
99 : struct timespec timestamp;
100 2 : (void)clock_gettime(CLOCK_MONOTONIC_RAW, ×tamp);
101 2 : return static_cast<u64>((timestamp.tv_sec * NSEC_PER_SEC) + (timestamp.tv_nsec));
102 : }
103 :
104 : constexpr u32 FIVE_MILLISECOND_OF_USLEEP = 5000U;
105 :
106 2 : HcclResult NsRecoveryHandlerFunc::DeviceQuery(const uint32_t devId, const uint32_t step, const uint64_t timeout)
107 : {
108 : uint32_t status;
109 : uint64_t endTime;
110 2 : const uint64_t startTime = GetCurCpuTimestamp();
111 2 : bool flag = true;
112 2 : while (flag) {
113 2 : ts_ctrl_msg_body_t queryIn = {};
114 2 : ts_ctrl_msg_body_t queryAck = {};
115 2 : size_t ackCount = sizeof(ts_ctrl_msg_body_t);
116 2 : queryIn.type = OPERATION_TYPE::OP_QUERY_ABORT_STATUS;
117 2 : queryIn.u.query_task_info.choice = APP_ABORT_STS_QUERY_CHOICE::APP_ABORT_STS_QUERY_BY_PID;
118 : struct tsdrv_ctrl_msg para;
119 2 : para.tsid = 0;
120 2 : para.msg_len = sizeof(ts_ctrl_msg_body_t);
121 2 : para.msg = static_cast<void*>(&queryIn);
122 2 : const drvError_t ret = halTsdrvCtl(devId, TSDRV_CTL_CMD_CTRL_MSG,
123 : static_cast<void*>(¶), sizeof(tsdrv_ctrl_msg), static_cast<void*>(&queryAck), &ackCount);
124 2 : if ((ret != DRV_ERROR_NONE) || (ackCount != sizeof(ts_ctrl_msg_body_t))) {
125 3 : HCCL_ERROR("halTsdrvCtl failed. ret = %d", ret);
126 1 : return HcclResult::HCCL_E_DRV;
127 : }
128 :
129 1 : status = queryAck.u.query_task_ack_info.status;
130 1 : if (status >= step) {
131 1 : flag = false;
132 1 : break;
133 : }
134 0 : endTime = GetCurCpuTimestamp();
135 0 : if ((timeout != 0U) && ((endTime - startTime) > timeout)) {
136 0 : HCCL_ERROR("[DeviceQuery]kill query timeout.");
137 0 : return HcclResult::HCCL_E_TIMEOUT;
138 : }
139 0 : SaluSleep(FIVE_MILLISECOND_OF_USLEEP);
140 : }
141 1 : return HcclResult::HCCL_SUCCESS;
142 : }
143 :
144 : }
|
