Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "ns_recovery_handler_func.h"
12 : #include "kfc.h"
13 : #include "drv_api_exception.h"
14 : #include "exception_util.h"
15 : #include "internal_exception.h"
16 :
17 : namespace Hccl {
18 6 : NsRecoveryHandlerFunc& NsRecoveryHandlerFunc::GetInstance()
19 : {
20 6 : static NsRecoveryHandlerFunc func;
21 6 : return func;
22 : }
23 :
24 1 : void NsRecoveryHandlerFunc::Call()
25 : {
26 1 : std::vector<CommunicatorImplLite*> commLites = CommunicatorImplLiteMgr::GetInstance().GetAll();
27 3 : for (auto& comm : commLites) {
28 2 : if (!comm->IsCommReady()) {
29 2 : continue;
30 : }
31 0 : HandleStopLaunch(comm);
32 0 : HandleClean(comm);
33 : }
34 1 : }
35 :
36 1 : void NsRecoveryHandlerFunc::HandleStopLaunch(CommunicatorImplLite* comm) const
37 : {
38 1 : if (comm->IsSuspended()) {
39 0 : return;
40 : }
41 :
42 1 : KfcCommand cmd = comm->BackGroundGetCmd();
43 1 : if (cmd != KfcCommand::NS_STOP_LAUNCH) {
44 0 : return;
45 : }
46 3 : HCCL_INFO("[NsRecovery][BackGround] received KfcCommand[NS_STOP_LAUNCH]");
47 1 : comm->SetNeedClean(true);
48 1 : comm->SetIsSuspended(true);
49 1 : comm->BackGroundSetStatus(KfcStatus::STOP_LAUNCH_DONE);
50 3 : HCCL_INFO("[NsRecovery][BackGround] send KfcStatus[STOP_LAUNCH_DONE]");
51 : }
52 :
53 1 : void NsRecoveryHandlerFunc::HandleClean(CommunicatorImplLite* comm)
54 : {
55 1 : if (!comm->IsNeedClean()) {
56 0 : return;
57 : }
58 1 : KfcCommand cmd = comm->BackGroundGetCmd();
59 1 : if (cmd != KfcCommand::NS_CLEAN) {
60 0 : return;
61 : }
62 3 : HCCL_INFO("[NsRecovery][BackGround] received KfcCommand[NS_CLEAN]");
63 1 : comm->GetTransportLiteMgr()->Reset();
64 1 : StreamClean(comm);
65 1 : comm->SetNeedClean(false);
66 1 : comm->BackGroundSetStatus(KfcStatus::CLEAN_DONE);
67 1 : comm->ResetErrorReported();
68 3 : HCCL_INFO("[NsRecovery][BackGround] send KfcStatus[CLEAN_DONE]");
69 : }
70 :
71 1 : void NsRecoveryHandlerFunc::StreamClean(CommunicatorImplLite* comm)
72 : {
73 : // 查询停流是否完成
74 1 : u32 localDevId = 0;
75 1 : auto ret = drvGetLocalDevIDByHostDevID(comm->GetDevPhyId(), &localDevId);
76 1 : if (ret != DRV_ERROR_NONE) {
77 : std::string formatStr = StringFormat(
78 : "NsRecoveryHandlerFunc::%s call drvGetLocalDevIDByHostDevID failed, devPhyId %u, ret %d", __func__,
79 0 : comm->GetDevPhyId(), ret);
80 0 : THROW<DrvApiException>(formatStr);
81 0 : }
82 1 : if (DeviceQuery(localDevId, APP_ABORT_STAUTS::APP_ABORT_KILL_FINISH, 0U) != HCCL_SUCCESS) {
83 0 : comm->BackGroundSetStatus(KfcStatus::ERROR, KfcErrType::EXEC);
84 0 : THROW<InternalException>("[NsRecovery][BackGround] Stream Stop failed");
85 : }
86 : // 清理资源
87 1 : auto streamLiteMgr = comm->GetStreamLiteMgr();
88 1 : CHECK_NULLPTR(streamLiteMgr->GetMaster(), "[StreamClean]master stream is nullptr!");
89 1 : streamLiteMgr->GetMaster()->GetRtsq()->Reset();
90 1 : for (u32 i = 0; i < streamLiteMgr->SizeOfSlaves(); ++i) {
91 0 : streamLiteMgr->GetSlave(i)->GetRtsq()->Reset();
92 : }
93 3 : HCCL_INFO("[NsRecovery][BackGround] StreamClean success.");
94 1 : }
95 :
96 : constexpr u64 NSEC_PER_SEC = 1000000000U;
97 :
98 2 : inline u64 GetCurCpuTimestamp()
99 : {
100 : struct timespec timestamp;
101 2 : (void)clock_gettime(CLOCK_MONOTONIC_RAW, ×tamp);
102 2 : return static_cast<u64>((timestamp.tv_sec * NSEC_PER_SEC) + (timestamp.tv_nsec));
103 : }
104 :
105 : constexpr u32 FIVE_MILLISECOND_OF_USLEEP = 5000U;
106 :
107 2 : HcclResult NsRecoveryHandlerFunc::DeviceQuery(const uint32_t devId, const uint32_t step, const uint64_t timeout)
108 : {
109 : uint32_t status;
110 : uint64_t endTime;
111 2 : const uint64_t startTime = GetCurCpuTimestamp();
112 2 : bool flag = true;
113 2 : while (flag) {
114 2 : ts_ctrl_msg_body_t queryIn = {};
115 2 : ts_ctrl_msg_body_t queryAck = {};
116 2 : size_t ackCount = sizeof(ts_ctrl_msg_body_t);
117 2 : queryIn.type = OPERATION_TYPE::OP_QUERY_ABORT_STATUS;
118 2 : queryIn.u.query_task_info.choice = APP_ABORT_STS_QUERY_CHOICE::APP_ABORT_STS_QUERY_BY_PID;
119 : struct tsdrv_ctrl_msg para;
120 2 : para.tsid = 0;
121 2 : para.msg_len = sizeof(ts_ctrl_msg_body_t);
122 2 : para.msg = static_cast<void*>(&queryIn);
123 2 : const drvError_t ret = halTsdrvCtl(
124 : devId, TSDRV_CTL_CMD_CTRL_MSG, static_cast<void*>(¶), sizeof(tsdrv_ctrl_msg),
125 : static_cast<void*>(&queryAck), &ackCount);
126 2 : if ((ret != DRV_ERROR_NONE) || (ackCount != sizeof(ts_ctrl_msg_body_t))) {
127 3 : HCCL_ERROR("halTsdrvCtl failed. ret = %d", ret);
128 1 : return HcclResult::HCCL_E_DRV;
129 : }
130 :
131 1 : status = queryAck.u.query_task_ack_info.status;
132 1 : if (status >= step) {
133 1 : flag = false;
134 1 : break;
135 : }
136 0 : endTime = GetCurCpuTimestamp();
137 0 : if ((timeout != 0U) && ((endTime - startTime) > timeout)) {
138 0 : HCCL_ERROR("[DeviceQuery]kill query timeout.");
139 0 : return HcclResult::HCCL_E_TIMEOUT;
140 : }
141 0 : SaluSleep(FIVE_MILLISECOND_OF_USLEEP);
142 : }
143 1 : return HcclResult::HCCL_SUCCESS;
144 : }
145 :
146 : } // namespace Hccl
|
