Line data Source code
1 : /**
2 : * Copyright (c) 2026 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 : #include "ns_recovery_func_lite.h"
11 : #include "kfc.h"
12 : #include "sal_pub.h"
13 : #include "ns_recovery_lite.h"
14 : #include "coll_comm_aicpu_mgr.h"
15 :
16 : namespace hccl {
17 11 : NsRecoveryFuncLite& NsRecoveryFuncLite::GetInstance()
18 : {
19 11 : static NsRecoveryFuncLite func;
20 11 : return func;
21 : }
22 :
23 2 : void NsRecoveryFuncLite::Call()
24 : {
25 2 : std::shared_lock<std::shared_mutex> rwlock(CollCommAicpuMgr::GetInstance().GetMutex());
26 :
27 2 : std::vector<std::pair<std::string, CollCommAicpu*>> aicpuCommInfo;
28 2 : auto ret = CollCommAicpuMgr::GetInstance().GetAllComms(aicpuCommInfo);
29 2 : if (ret != HCCL_SUCCESS) {
30 0 : HCCL_ERROR("[NsRecovery][BackGround] AicpuGetCommAll failed, errNo[0x%016llx]", ret);
31 0 : return;
32 : }
33 5 : for (auto& commInfo : aicpuCommInfo) {
34 3 : CollCommAicpu* deviceComm = commInfo.second;
35 3 : if (deviceComm->GetCommmStatus() == HcclCommStatus::HCCL_COMM_STATUS_INVALID) {
36 3 : continue;
37 : }
38 0 : HandleStopLaunch(deviceComm);
39 0 : HandleClean(deviceComm);
40 : }
41 2 : }
42 :
43 0 : void NsRecoveryFuncLite::HandleStopLaunch(CollCommAicpu* deviceComm) const
44 : {
45 0 : if (deviceComm->GetCommmStatus() == HcclCommStatus::HCCL_COMM_STATUS_SUSPENDING) {
46 0 : return;
47 : }
48 :
49 0 : Hccl::KfcCommand cmd = deviceComm->GetNsRecoveryLitePtr()->BackGroundGetCmd();
50 0 : if (cmd != Hccl::KfcCommand::NS_STOP_LAUNCH) {
51 0 : return;
52 : }
53 0 : HCCL_INFO("[NsRecovery][BackGround] received KfcCommand[NS_STOP_LAUNCH]");
54 0 : deviceComm->GetNsRecoveryLitePtr()->SetNeedClean(true);
55 0 : deviceComm->SetCommmStatus(HcclCommStatus::HCCL_COMM_STATUS_SUSPENDING);
56 0 : deviceComm->GetNsRecoveryLitePtr()->BackGroundSetStatus(Hccl::KfcStatus::STOP_LAUNCH_DONE);
57 0 : HCCL_INFO(
58 : "[NsRecovery][BackGround] commId[%s] send KfcStatus[STOP_LAUNCH_DONE]", deviceComm->GetIdentifier().c_str());
59 : }
60 :
61 0 : void NsRecoveryFuncLite::HandleClean(CollCommAicpu* deviceComm)
62 : {
63 0 : if (!deviceComm->GetNsRecoveryLitePtr()->IsNeedClean()) {
64 0 : return;
65 : }
66 0 : Hccl::KfcCommand cmd = deviceComm->GetNsRecoveryLitePtr()->BackGroundGetCmd();
67 0 : if (cmd != Hccl::KfcCommand::NS_CLEAN) {
68 0 : return;
69 : }
70 0 : HCCL_INFO("[NsRecovery][BackGround] received KfcCommand[NS_CLEAN]");
71 0 : StreamClean(deviceComm);
72 0 : deviceComm->Clean();
73 0 : deviceComm->GetNsRecoveryLitePtr()->SetNeedClean(false);
74 0 : deviceComm->GetNsRecoveryLitePtr()->BackGroundSetStatus(Hccl::KfcStatus::CLEAN_DONE);
75 0 : HCCL_INFO("[NsRecovery][BackGround] commId[%s] send KfcStatus[CLEAN_DONE]", deviceComm->GetIdentifier().c_str());
76 : }
77 :
78 0 : void NsRecoveryFuncLite::StreamClean(CollCommAicpu* deviceComm)
79 : {
80 : // 查询停流是否完成
81 0 : u32 localDevId{0};
82 0 : auto ret = drvGetLocalDevIDByHostDevID(deviceComm->GetTopoInfo().devicePhyId, &localDevId);
83 0 : if (ret != DRV_ERROR_NONE) {
84 0 : HCCL_ERROR(
85 : "NsRecoveryFuncLite::%s call drvGetLocalDevIDByHostDevID failed, devPhyId %u, ret %d", __func__,
86 : deviceComm->GetTopoInfo().devicePhyId, ret);
87 0 : return;
88 : }
89 0 : if (DeviceQuery(localDevId, APP_ABORT_STAUTS::APP_ABORT_KILL_FINISH, 0U) != HCCL_SUCCESS) {
90 0 : deviceComm->GetNsRecoveryLitePtr()->BackGroundSetStatus(Hccl::KfcStatus::ERROR, Hccl::KfcErrType::EXEC);
91 0 : HCCL_ERROR("[NsRecovery][BackGround] Stream Stop failed");
92 0 : return;
93 : }
94 :
95 : // 通过thread获得streamlite信息，清理资源
96 0 : std::vector<std::shared_ptr<hccl::Thread>> threads = deviceComm->GetCommEngineResMgr()->GetAllThread();
97 0 : for (auto& thread : threads) {
98 0 : Hccl::StreamLite* streamLitePtr = reinterpret_cast<Hccl::StreamLite*>(thread->GetStreamLitePtr());
99 0 : streamLitePtr->GetRtsq()->Reset();
100 : }
101 0 : HCCL_INFO("[NsRecovery][BackGround] commId[%s] streamClean success.", deviceComm->GetIdentifier().c_str());
102 0 : }
103 :
104 : constexpr u64 NSEC_PER_SEC = 1000000000U;
105 9 : inline u64 GetCurCpuTimestamp()
106 : {
107 9 : struct timespec timestamp {
108 : 0, 0
109 : };
110 9 : (void)clock_gettime(CLOCK_MONOTONIC_RAW, ×tamp);
111 9 : return static_cast<u64>((timestamp.tv_sec * NSEC_PER_SEC) + (timestamp.tv_nsec));
112 : }
113 :
114 : constexpr u32 FIVE_MILLISECOND_OF_USLEEP = 5000U;
115 :
116 6 : HcclResult NsRecoveryFuncLite::DeviceQuery(const uint32_t devId, const uint32_t step, const uint64_t timeout)
117 : {
118 : uint32_t status;
119 : uint64_t endTime;
120 6 : const uint64_t startTime = GetCurCpuTimestamp();
121 6 : bool flag = true;
122 8 : while (flag) {
123 8 : ts_ctrl_msg_body_t queryIn = {};
124 8 : ts_ctrl_msg_body_t queryAck = {};
125 8 : size_t ackCount = sizeof(ts_ctrl_msg_body_t);
126 8 : queryIn.type = OPERATION_TYPE::OP_QUERY_ABORT_STATUS;
127 8 : queryIn.u.query_task_info.choice = APP_ABORT_STS_QUERY_CHOICE::APP_ABORT_STS_QUERY_BY_PID;
128 : struct tsdrv_ctrl_msg para;
129 8 : para.tsid = 0;
130 8 : para.msg_len = sizeof(ts_ctrl_msg_body_t);
131 8 : para.msg = static_cast<void*>(&queryIn);
132 8 : const drvError_t ret = halTsdrvCtl(
133 : devId, TSDRV_CTL_CMD_CTRL_MSG, static_cast<void*>(¶), sizeof(tsdrv_ctrl_msg),
134 : static_cast<void*>(&queryAck), &ackCount);
135 8 : if ((ret != DRV_ERROR_NONE) || (ackCount != sizeof(ts_ctrl_msg_body_t))) {
136 2 : HCCL_ERROR("halTsdrvCtl failed. ret = %d", ret);
137 3 : return HcclResult::HCCL_E_DRV;
138 : }
139 :
140 6 : status = queryAck.u.query_task_ack_info.status;
141 6 : if (status >= step) {
142 3 : flag = false;
143 3 : break;
144 : }
145 3 : endTime = GetCurCpuTimestamp();
146 3 : if ((timeout != 0U) && ((endTime - startTime) > timeout)) {
147 1 : HCCL_ERROR("[DeviceQuery]kill query timeout.");
148 1 : return HcclResult::HCCL_E_TIMEOUT;
149 : }
150 2 : SaluSleep(FIVE_MILLISECOND_OF_USLEEP);
151 : }
152 3 : return HcclResult::HCCL_SUCCESS;
153 : }
154 :
155 : } // namespace hccl
|
