Line data Source code
1 : /**
2 : * Copyright (c) 2026 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "ns_recovery_func_lite.h"
12 : #include "kfc.h"
13 : #include "sal_pub.h"
14 : #include "ns_recovery_lite.h"
15 : #include "coll_comm_aicpu_mgr.h"
16 :
17 : namespace hccl {
18 11 : NsRecoveryFuncLite& NsRecoveryFuncLite::GetInstance()
19 : {
20 11 : static NsRecoveryFuncLite func;
21 11 : return func;
22 : }
23 :
24 2 : void NsRecoveryFuncLite::Call()
25 : {
26 2 : std::shared_lock<std::shared_mutex> rwlock(CollCommAicpuMgr::GetInstance().GetMutex());
27 :
28 2 : std::vector<std::pair<std::string, CollCommAicpu*>> aicpuCommInfo;
29 2 : auto ret = CollCommAicpuMgr::GetInstance().GetAllComms(aicpuCommInfo);
30 2 : if (ret != HCCL_SUCCESS) {
31 0 : HCCL_ERROR("[NsRecovery][BackGround] AicpuGetCommAll failed, errNo[0x%016llx]", ret);
32 0 : return;
33 : }
34 5 : for (auto& commInfo : aicpuCommInfo) {
35 3 : CollCommAicpu* deviceComm = commInfo.second;
36 3 : if (deviceComm->GetCommmStatus() == HcclCommStatus::HCCL_COMM_STATUS_INVALID) {
37 3 : continue;
38 : }
39 0 : HandleStopLaunch(deviceComm);
40 0 : HandleClean(deviceComm);
41 : }
42 2 : }
43 :
44 0 : void NsRecoveryFuncLite::HandleStopLaunch(CollCommAicpu* deviceComm) const
45 : {
46 0 : if (deviceComm->GetCommmStatus() == HcclCommStatus::HCCL_COMM_STATUS_SUSPENDING) {
47 0 : return;
48 : }
49 :
50 0 : Hccl::KfcCommand cmd = deviceComm->GetNsRecoveryLitePtr()->BackGroundGetCmd();
51 0 : if (cmd != Hccl::KfcCommand::NS_STOP_LAUNCH) {
52 0 : return;
53 : }
54 0 : HCCL_INFO("[NsRecovery][BackGround] received KfcCommand[NS_STOP_LAUNCH]");
55 0 : deviceComm->GetNsRecoveryLitePtr()->SetNeedClean(true);
56 0 : deviceComm->SetCommmStatus(HcclCommStatus::HCCL_COMM_STATUS_SUSPENDING);
57 0 : deviceComm->GetNsRecoveryLitePtr()->BackGroundSetStatus(Hccl::KfcStatus::STOP_LAUNCH_DONE);
58 0 : HCCL_INFO(
59 : "[NsRecovery][BackGround] commId[%s] send KfcStatus[STOP_LAUNCH_DONE]", deviceComm->GetIdentifier().c_str());
60 : }
61 :
62 0 : void NsRecoveryFuncLite::HandleClean(CollCommAicpu* deviceComm)
63 : {
64 0 : if (!deviceComm->GetNsRecoveryLitePtr()->IsNeedClean()) {
65 0 : return;
66 : }
67 0 : Hccl::KfcCommand cmd = deviceComm->GetNsRecoveryLitePtr()->BackGroundGetCmd();
68 0 : if (cmd != Hccl::KfcCommand::NS_CLEAN) {
69 0 : return;
70 : }
71 0 : HCCL_INFO("[NsRecovery][BackGround] received KfcCommand[NS_CLEAN]");
72 0 : StreamClean(deviceComm);
73 0 : deviceComm->Clean();
74 0 : deviceComm->GetNsRecoveryLitePtr()->SetNeedClean(false);
75 0 : deviceComm->GetNsRecoveryLitePtr()->BackGroundSetStatus(Hccl::KfcStatus::CLEAN_DONE);
76 0 : HCCL_INFO("[NsRecovery][BackGround] commId[%s] send KfcStatus[CLEAN_DONE]", deviceComm->GetIdentifier().c_str());
77 : }
78 :
79 0 : void NsRecoveryFuncLite::StreamClean(CollCommAicpu* deviceComm)
80 : {
81 : // 查询停流是否完成
82 0 : u32 localDevId{0};
83 0 : auto ret = drvGetLocalDevIDByHostDevID(deviceComm->GetTopoInfo().devicePhyId, &localDevId);
84 0 : if (ret != DRV_ERROR_NONE) {
85 0 : HCCL_ERROR(
86 : "NsRecoveryFuncLite::%s call drvGetLocalDevIDByHostDevID failed, devPhyId %u, ret %d", __func__,
87 : deviceComm->GetTopoInfo().devicePhyId, ret);
88 0 : return;
89 : }
90 0 : if (DeviceQuery(localDevId, APP_ABORT_STAUTS::APP_ABORT_KILL_FINISH, 0U) != HCCL_SUCCESS) {
91 0 : deviceComm->GetNsRecoveryLitePtr()->BackGroundSetStatus(Hccl::KfcStatus::ERROR, Hccl::KfcErrType::EXEC);
92 0 : HCCL_ERROR("[NsRecovery][BackGround] Stream Stop failed");
93 0 : return;
94 : }
95 :
96 : // 通过thread获得streamlite信息，清理资源
97 0 : std::vector<std::shared_ptr<hccl::Thread>> threads = deviceComm->GetCommEngineResMgr()->GetAllThread();
98 0 : for (auto& thread : threads) {
99 0 : Hccl::StreamLite* streamLitePtr = reinterpret_cast<Hccl::StreamLite*>(thread->GetStreamLitePtr());
100 0 : streamLitePtr->GetRtsq()->Reset();
101 : }
102 0 : HCCL_INFO("[NsRecovery][BackGround] commId[%s] streamClean success.", deviceComm->GetIdentifier().c_str());
103 0 : }
104 :
105 : constexpr u64 NSEC_PER_SEC = 1000000000U;
106 9 : inline u64 GetCurCpuTimestamp()
107 : {
108 9 : struct timespec timestamp {
109 : 0, 0
110 : };
111 9 : (void)clock_gettime(CLOCK_MONOTONIC_RAW, ×tamp);
112 9 : return static_cast<u64>((timestamp.tv_sec * NSEC_PER_SEC) + (timestamp.tv_nsec));
113 : }
114 :
115 : constexpr u32 FIVE_MILLISECOND_OF_USLEEP = 5000U;
116 :
117 6 : HcclResult NsRecoveryFuncLite::DeviceQuery(const uint32_t devId, const uint32_t step, const uint64_t timeout)
118 : {
119 : uint32_t status;
120 : uint64_t endTime;
121 6 : const uint64_t startTime = GetCurCpuTimestamp();
122 6 : bool flag = true;
123 8 : while (flag) {
124 8 : ts_ctrl_msg_body_t queryIn = {};
125 8 : ts_ctrl_msg_body_t queryAck = {};
126 8 : size_t ackCount = sizeof(ts_ctrl_msg_body_t);
127 8 : queryIn.type = OPERATION_TYPE::OP_QUERY_ABORT_STATUS;
128 8 : queryIn.u.query_task_info.choice = APP_ABORT_STS_QUERY_CHOICE::APP_ABORT_STS_QUERY_BY_PID;
129 : struct tsdrv_ctrl_msg para;
130 8 : para.tsid = 0;
131 8 : para.msg_len = sizeof(ts_ctrl_msg_body_t);
132 8 : para.msg = static_cast<void*>(&queryIn);
133 8 : const drvError_t ret = halTsdrvCtl(
134 : devId, TSDRV_CTL_CMD_CTRL_MSG, static_cast<void*>(¶), sizeof(tsdrv_ctrl_msg),
135 : static_cast<void*>(&queryAck), &ackCount);
136 8 : if ((ret != DRV_ERROR_NONE) || (ackCount != sizeof(ts_ctrl_msg_body_t))) {
137 2 : HCCL_ERROR("halTsdrvCtl failed. ret = %d", ret);
138 3 : return HcclResult::HCCL_E_DRV;
139 : }
140 :
141 6 : status = queryAck.u.query_task_ack_info.status;
142 6 : if (status >= step) {
143 3 : flag = false;
144 3 : break;
145 : }
146 3 : endTime = GetCurCpuTimestamp();
147 3 : if ((timeout != 0U) && ((endTime - startTime) > timeout)) {
148 1 : HCCL_ERROR("[DeviceQuery]kill query timeout.");
149 1 : return HcclResult::HCCL_E_TIMEOUT;
150 : }
151 2 : SaluSleep(FIVE_MILLISECOND_OF_USLEEP);
152 : }
153 3 : return HcclResult::HCCL_SUCCESS;
154 : }
155 :
156 : } // namespace hccl
|
