Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include <algorithm>
12 : #include <mutex>
13 : #include "task_abort_handler_v2.h"
14 : #include "ccu_dev_mgr.h"
15 : #include "log.h"
16 :
17 : namespace Hccl {
18 : using HcclUs = std::chrono::steady_clock::time_point;
19 4 : static int32_t TaskAbortPre(const std::vector<HcclCommunicator*>& commVector, const std::chrono::seconds& localtimeout)
20 : {
21 4 : HcclResult ret = HCCL_SUCCESS;
22 4 : bool isUseTimeOut = localtimeout != std::chrono::seconds(0);
23 4 : std::chrono::seconds elapsed{};
24 8 : for (const auto& comm : commVector) {
25 6 : if (isUseTimeOut) {
26 3 : std::chrono::steady_clock::time_point startTime = std::chrono::steady_clock::now();
27 3 : ret = comm->Suspend();
28 3 : elapsed = std::chrono::duration_cast<std::chrono::seconds>(std::chrono::steady_clock::now() - startTime);
29 : } else {
30 3 : ret = comm->Suspend();
31 : }
32 6 : if (ret != HCCL_SUCCESS && ret != HCCL_E_SUSPENDING) {
33 6 : HCCL_ERROR("[NsRecovery] finish suspend failed");
34 2 : return static_cast<int>(TaskAbortResult::TASK_ABORT_FAIL);
35 : }
36 12 : HCCL_DEBUG("[NsRecovery]finish suspend success");
37 4 : if (isUseTimeOut) {
38 2 : CHK_PRT_RET(
39 : elapsed > localtimeout, HCCL_ERROR("[NsRecovery][suspend] NsRecovery suspend timeOut"),
40 : static_cast<int>(TaskAbortResult::TASK_ABORT_TIMEOUT));
41 : }
42 : }
43 2 : return static_cast<int>(TaskAbortResult::TASK_ABORT_SUCCESS);
44 : }
45 :
46 4 : static int32_t TaskAbortPost(
47 : const std::vector<HcclCommunicator*>& commVector, int32_t deviceLogicId, const std::chrono::seconds& localtimeout)
48 : {
49 4 : HcclResult ret = HCCL_SUCCESS;
50 4 : bool isUseTimeOut = localtimeout != std::chrono::seconds(0);
51 4 : std::chrono::seconds elapsed{};
52 4 : if (CcuIsInited(deviceLogicId)) {
53 0 : CHK_RET(HcclCcuTaskKillPreProcess(deviceLogicId));
54 : } else {
55 12 : HCCL_INFO("[NsRecovery][Callback] CCU not inited, skip TaskKillPreProcess, deviceLogicId[%d]", deviceLogicId);
56 : }
57 8 : for (const auto& comm : commVector) {
58 6 : if (isUseTimeOut) {
59 3 : std::chrono::steady_clock::time_point startTime = std::chrono::steady_clock::now();
60 3 : ret = comm->Clean();
61 3 : elapsed = std::chrono::duration_cast<std::chrono::seconds>(std::chrono::steady_clock::now() - startTime);
62 : } else {
63 3 : ret = comm->Clean();
64 : }
65 6 : if (ret != HCCL_SUCCESS && ret != HCCL_E_SUSPENDING) {
66 6 : HCCL_ERROR("[NsRecovery][Callback] finish clean failed");
67 2 : return static_cast<int>(TaskAbortResult::TASK_ABORT_FAIL);
68 : }
69 12 : HCCL_INFO("[NsRecovery][Callback] finish clean success");
70 4 : if (isUseTimeOut) {
71 2 : CHK_PRT_RET(
72 : elapsed > localtimeout, HCCL_ERROR("[NsRecovery][Callback] NsRecovery Clean timeout"),
73 : static_cast<int>(TaskAbortResult::TASK_ABORT_TIMEOUT));
74 : }
75 : }
76 2 : if (CcuIsInited(deviceLogicId)) {
77 0 : CHK_RET(HcclCcuTaskKillPostProcess(deviceLogicId));
78 : } else {
79 6 : HCCL_INFO("[NsRecovery][Callback] CCU not inited, skip TaskKillPostProcess, deviceLogicId[%d]", deviceLogicId);
80 : }
81 2 : return static_cast<int>(TaskAbortResult::TASK_ABORT_SUCCESS);
82 : }
83 :
84 : int32_t
85 8 : ProcessTaskAbortHandleCallback(int32_t deviceLogicId, aclrtDeviceTaskAbortStage stage, uint32_t timeout, void* args)
86 : {
87 8 : HcclUs startut = std::chrono::steady_clock::now();
88 8 : CHK_PTR_NULL(args);
89 8 : auto& commVector = *(static_cast<std::vector<HcclCommunicator*>*>(args));
90 24 : HCCL_INFO(
91 : "[NsRecovery][Callback] ProcessTaskAbortHandleCallback begin, deviceLogicId [%d], stage [%d], commVector "
92 : "size [%lu]",
93 : deviceLogicId, stage, commVector.size());
94 8 : const std::chrono::seconds localtimeout = std::chrono::seconds(timeout);
95 :
96 8 : if (stage == aclrtDeviceTaskAbortStage::ACL_RT_DEVICE_TASK_ABORT_PRE) {
97 4 : auto result = TaskAbortPre(commVector, localtimeout);
98 4 : if (result != static_cast<int>(TaskAbortResult::TASK_ABORT_SUCCESS)) {
99 2 : return result;
100 : }
101 4 : } else if (stage == aclrtDeviceTaskAbortStage::ACL_RT_DEVICE_TASK_ABORT_POST) {
102 4 : auto result = TaskAbortPost(commVector, deviceLogicId, localtimeout);
103 4 : if (result != static_cast<int>(TaskAbortResult::TASK_ABORT_SUCCESS)) {
104 2 : return result;
105 : }
106 : }
107 4 : HcclUs endut = std::chrono::steady_clock::now();
108 12 : HCCL_INFO(
109 : "[NsRecovery][Callback] ProcessTaskAbortHandleCallback success, take time:[%lld]us",
110 : std::chrono::duration_cast<std::chrono::microseconds>(endut - startut).count());
111 4 : return static_cast<int>(TaskAbortResult::TASK_ABORT_SUCCESS);
112 : }
113 :
114 1 : TaskAbortHandler::TaskAbortHandler()
115 : {
116 1 : std::string name = "HCCL";
117 1 : HrtDeviceAbortRegCallBack(ProcessTaskAbortHandleCallback, static_cast<void*>(&commVector), name);
118 1 : }
119 :
120 0 : TaskAbortHandler::~TaskAbortHandler()
121 : {
122 0 : std::string name = "HCCL";
123 0 : DECTOR_TRY_CATCH("TaskAbortHandler", HrtDeviceAbortRegCallBack(nullptr, nullptr, name));
124 0 : }
125 :
126 719 : TaskAbortHandler& TaskAbortHandler::GetInstance()
127 : {
128 : // Leaky Singleton: 故意不释放，规避 Static Destruction Order Fiasco，
129 : // 确保析构回调访问 commVector 时单例仍有效
130 719 : static TaskAbortHandler& handler = *new TaskAbortHandler();
131 719 : return handler;
132 : }
133 :
134 276 : HcclResult TaskAbortHandler::Register(HcclCommunicator* communicator)
135 : {
136 276 : std::lock_guard<std::mutex> lock(vecMutex);
137 276 : commVector.push_back(communicator);
138 828 : HCCL_INFO("TaskAbortHandler::Register success, commVector size is [%lu]", commVector.size());
139 :
140 276 : return HCCL_SUCCESS;
141 276 : }
142 :
143 276 : HcclResult TaskAbortHandler::UnRegister(HcclCommunicator* communicator)
144 : {
145 276 : std::lock_guard<std::mutex> lock(vecMutex);
146 828 : HCCL_INFO("TaskAbortHandler::UnRegister Begin, commVector size is [%lu]", commVector.size());
147 276 : auto it = std::find(commVector.begin(), commVector.end(), communicator);
148 276 : if (it != commVector.end()) {
149 273 : commVector.erase(it);
150 : } else {
151 9 : HCCL_WARNING("TaskAbortHandler::UnRegister, comm not found.");
152 : }
153 828 : HCCL_INFO("TaskAbortHandler::UnRegister finish, commVector size is [%lu]", commVector.size());
154 276 : return HCCL_SUCCESS;
155 276 : }
156 : } // namespace Hccl
|
