Line data Source code
1 : /**
2 : * Copyright (c) 2026 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 : #include "task_abort_handler.h"
11 : #include <algorithm>
12 : #include "log.h"
13 : #include "coll_comm.h"
14 : #include "ccu_device_pub.h"
15 :
16 : namespace hccl {
17 : using HcclUs = std::chrono::steady_clock::time_point;
18 : static std::mutex vecMutex;
19 :
20 5 : int32_t ProcessTaskAbortPre(const std::vector<CollComm *> &commVector, const std::chrono::seconds &localtimeout)
21 : {
22 5 : HcclResult ret = HCCL_SUCCESS;
23 5 : bool isUseTimeOut = localtimeout != std::chrono::seconds(0);
24 5 : std::chrono::seconds elapsed{};
25 8 : for (auto& comm : commVector) {
26 5 : if (isUseTimeOut) {
27 2 : std::chrono::steady_clock::time_point startTime = std::chrono::steady_clock::now();
28 2 : ret = comm->Suspend();
29 2 : elapsed = std::chrono::duration_cast<std::chrono::seconds>(std::chrono::steady_clock::now() - startTime);
30 : } else {
31 3 : ret = comm->Suspend();
32 : }
33 5 : if (ret != HCCL_SUCCESS && ret != HCCL_E_SUSPENDING) {
34 2 : HCCL_ERROR("[NsRecovery] finish suspend failed, ret = 0x%016llx", HCCL_ERROR_CODE(ret));
35 2 : return static_cast<int>(TaskAbortResult::TASK_ABORT_FAIL);
36 : }
37 3 : HCCL_INFO("[NsRecovery]finish suspend success");
38 3 : if (isUseTimeOut) {
39 1 : CHK_PRT_RET(elapsed > localtimeout, HCCL_ERROR("[NsRecovery][suspend] NsRecovery suspend timeOut"),
40 : static_cast<int>(TaskAbortResult::TASK_ABORT_TIMEOUT));
41 : }
42 : }
43 3 : return static_cast<int>(TaskAbortResult::TASK_ABORT_SUCCESS);
44 : }
45 :
46 8 : int32_t ProcessTaskAbortPost(const std::vector<CollComm *> &commVector, int32_t deviceLogicId,
47 : const std::chrono::seconds &localtimeout)
48 : {
49 8 : HcclResult ret = HCCL_SUCCESS;
50 8 : bool isUseTimeOut = localtimeout != std::chrono::seconds(0);
51 8 : std::chrono::seconds elapsed{};
52 8 : if (hcomm::CcuIsInited(deviceLogicId)) {
53 6 : CHK_RET(hcomm::CcuSetTaskKill(deviceLogicId));
54 : } else {
55 2 : HCCL_INFO("[NsRecovery][Callback] CCU not inited, skip CcuSetTaskKill, deviceLogicId[%d]", deviceLogicId);
56 : }
57 12 : for (auto& comm : commVector) {
58 7 : if (isUseTimeOut) {
59 3 : std::chrono::steady_clock::time_point startTime = std::chrono::steady_clock::now();
60 3 : ret = comm->Clean();
61 3 : elapsed = std::chrono::duration_cast<std::chrono::seconds>(std::chrono::steady_clock::now() - startTime);
62 : } else {
63 4 : ret = comm->Clean();
64 : }
65 7 : if (ret != HCCL_SUCCESS && ret != HCCL_E_SUSPENDING) {
66 2 : HCCL_ERROR("[NsRecovery][Callback] finish clean failed, ret = 0x%016llx", HCCL_ERROR_CODE(ret));
67 2 : return static_cast<int>(TaskAbortResult::TASK_ABORT_FAIL);
68 : }
69 5 : HCCL_INFO("[NsRecovery][Callback] finish clean success");
70 5 : if (isUseTimeOut) {
71 2 : CHK_PRT_RET(elapsed > localtimeout, HCCL_ERROR("[NsRecovery][Callback] NsRecovery Clean timeout"),
72 : static_cast<int>(TaskAbortResult::TASK_ABORT_TIMEOUT));
73 : }
74 : }
75 5 : if (hcomm::CcuIsInited(deviceLogicId)) {
76 3 : CHK_RET(hcomm::CcuSetTaskKillDone(deviceLogicId));
77 : } else {
78 2 : HCCL_INFO("[NsRecovery][Callback] CCU not inited, skip CcuSetTaskKillDone, deviceLogicId[%d]", deviceLogicId);
79 : }
80 5 : return static_cast<int>(TaskAbortResult::TASK_ABORT_SUCCESS);
81 : }
82 :
83 14 : int32_t ProcessTaskAbortHandleCallback(int32_t deviceLogicId, aclrtDeviceTaskAbortStage stage,
84 : uint32_t timeout, void* args)
85 : {
86 14 : HcclUs startut = std::chrono::steady_clock::now();
87 14 : CHK_PTR_NULL(args);
88 13 : auto &commVector = *(static_cast<std::vector<CollComm *> *>(args));
89 13 : HCCL_INFO("[NsRecovery][Callback] ProcessTaskAbortHandleCallback start!");
90 13 : const std::chrono::seconds localtimeout = std::chrono::seconds(timeout);
91 :
92 13 : if (stage == aclrtDeviceTaskAbortStage::ACL_RT_DEVICE_TASK_ABORT_PRE) {
93 5 : auto result = ProcessTaskAbortPre(commVector, localtimeout);
94 5 : if (result != static_cast<int>(TaskAbortResult::TASK_ABORT_SUCCESS)) {
95 2 : return result;
96 : }
97 8 : } else if (stage == aclrtDeviceTaskAbortStage::ACL_RT_DEVICE_TASK_ABORT_POST) {
98 8 : auto result = ProcessTaskAbortPost(commVector, deviceLogicId, localtimeout);
99 8 : if (result != static_cast<int>(TaskAbortResult::TASK_ABORT_SUCCESS)) {
100 3 : return result;
101 : }
102 : }
103 8 : HcclUs endut = std::chrono::steady_clock::now();
104 8 : auto execTime = std::chrono::duration_cast<std::chrono::microseconds>(endut - startut).count();
105 8 : HCCL_RUN_INFO("[NsRecovery][Callback] ProcessTaskAbortHandleCallback success, take time:[%lld]us", execTime);
106 8 : return static_cast<int>(TaskAbortResult::TASK_ABORT_SUCCESS);
107 : }
108 :
109 6 : HcclTaskAbortHandler::HcclTaskAbortHandler()
110 : {
111 6 : std::string name = "HCOMM";
112 6 : Hccl::HrtDeviceAbortRegCallBack(ProcessTaskAbortHandleCallback, static_cast<void *>(&commVector_), name);
113 6 : }
114 :
115 6 : HcclTaskAbortHandler::~HcclTaskAbortHandler()
116 : {
117 6 : std::string name = "HCOMM";
118 6 : Hccl::HrtDeviceAbortRegCallBack(nullptr, nullptr, name);
119 6 : }
120 :
121 288 : HcclTaskAbortHandler &HcclTaskAbortHandler::GetInstance()
122 : {
123 288 : static HcclTaskAbortHandler handler;
124 288 : return handler;
125 : }
126 :
127 109 : HcclResult HcclTaskAbortHandler::Register(CollComm *communicator)
128 : {
129 109 : std::lock_guard<std::mutex> lock(vecMutex);
130 109 : commVector_.push_back(communicator);
131 109 : HCCL_INFO("HcclTaskAbortHandler::Register success, commVector_ size is [%zu]", commVector_.size());
132 :
133 109 : return HCCL_SUCCESS;
134 109 : }
135 :
136 146 : HcclResult HcclTaskAbortHandler::UnRegister(CollComm *communicator)
137 : {
138 146 : std::lock_guard<std::mutex> lock(vecMutex);
139 146 : HCCL_INFO("HcclTaskAbortHandler::UnRegister Begin, commVector_ size is [%zu]", commVector_.size());
140 146 : auto it = std::find(commVector_.begin(), commVector_.end(), communicator);
141 146 : if (it != commVector_.end()) {
142 109 : commVector_.erase(it);
143 : } else {
144 37 : HCCL_WARNING("HcclTaskAbortHandler::UnRegister, comm not found.");
145 : }
146 146 : HCCL_INFO("HcclTaskAbortHandler::UnRegister finish, commVector_ size is [%zu]", commVector_.size());
147 146 : return HCCL_SUCCESS;
148 146 : }
149 : }
|
