Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "snapshot_control.h"
12 : #include "adapter_rts_common.h"
13 : #include "adapter_hccp_common.h"
14 : #include "externalinput.h"
15 : #include "transport_pub.h"
16 : #include "rt_external.h"
17 :
18 : namespace hccl {
19 :
20 : bool SnapshotControl::registered = false;
21 :
22 0 : uint32_t PreProcessCallback(int32_t devId, void *args)
23 : {
24 0 : HCCL_RUN_INFO("[Snapshot] PreProcess callback, devId[%d]", devId);
25 0 : HcclResult ret = SnapshotControl::GetInstance(devId).PreProcess();
26 0 : CHK_PRT_RET(ret != HCCL_SUCCESS, HCCL_ERROR("[Snapshot] PreProcess fail, devId[%d].", devId), ret);
27 0 : HCCL_RUN_INFO("[Snapshot] PreProcess success, devId[%d]", devId);
28 0 : return 0;
29 : }
30 :
31 0 : uint32_t PostProcessCallback(int32_t devId, void *args)
32 : {
33 0 : HCCL_RUN_INFO("[Snapshot] PostProcess callback, devId[%d]", devId);
34 0 : HcclResult ret = SnapshotControl::GetInstance(devId).PostProcess();
35 0 : CHK_PRT_RET(ret != HCCL_SUCCESS, HCCL_ERROR("[Snapshot] PostProcess fail, devId[%d].", devId), ret);
36 0 : HCCL_RUN_INFO("[Snapshot] PostProcess success, devId[%d]", devId);
37 0 : return 0;
38 : }
39 :
40 0 : uint32_t RecoveryCallback(int32_t devId, void *args)
41 : {
42 0 : HCCL_RUN_INFO("[Snapshot] Recovery callback, devId[%d]", devId);
43 0 : HcclResult ret = SnapshotControl::GetInstance(devId).Recovery();
44 0 : CHK_PRT_RET(ret != HCCL_SUCCESS, HCCL_ERROR("[Snapshot] Recovery fail, devId[%d].", devId), ret);
45 0 : HCCL_RUN_INFO("[Snapshot] Recovery success, devId[%d]", devId);
46 0 : return 0;
47 : }
48 :
49 2 : HcclResult ResgisterSnapshotCallback()
50 : {
51 2 : rtError_t ret = aclrtSnapShotCallbackRegister(ACL_RT_SNAPSHOT_LOCK_PRE, PreProcessCallback, nullptr);
52 2 : CHK_PRT_RET(ret != ACL_SUCCESS,
53 : HCCL_ERROR("[SnapshotControl]errNo[0x%016llx] register preprocess callback fail, ret[%d]",
54 : HCCL_ERROR_CODE(HCCL_E_RUNTIME), ret), HCCL_E_RUNTIME);
55 2 : ret = aclrtSnapShotCallbackRegister(ACL_RT_SNAPSHOT_UNLOCK_POST, PostProcessCallback, nullptr);
56 2 : CHK_PRT_RET(ret != ACL_SUCCESS,
57 : HCCL_ERROR("[SnapshotControl]errNo[0x%016llx] register postprocess callback fail, ret[%d]",
58 : HCCL_ERROR_CODE(HCCL_E_RUNTIME), ret), HCCL_E_RUNTIME);
59 2 : ret = aclrtSnapShotCallbackRegister(ACL_RT_SNAPSHOT_RESTORE_POST, RecoveryCallback, nullptr);
60 2 : CHK_PRT_RET(ret != ACL_SUCCESS,
61 : HCCL_ERROR("[SnapshotControl]errNo[0x%016llx] register recovery callback fail, ret[%d]",
62 : HCCL_ERROR_CODE(HCCL_E_RUNTIME), ret), HCCL_E_RUNTIME);
63 2 : return HCCL_SUCCESS;
64 : }
65 :
66 2 : HcclResult UnResgisterSnapshotCallback()
67 : {
68 2 : rtError_t ret = aclrtSnapShotCallbackUnregister(ACL_RT_SNAPSHOT_LOCK_PRE, PreProcessCallback);
69 2 : CHK_PRT_RET(ret != ACL_SUCCESS,
70 : HCCL_ERROR("[SnapshotControl]errNo[0x%016llx] unregister preprocess callback fail, ret[%d]",
71 : HCCL_ERROR_CODE(HCCL_E_RUNTIME), ret), HCCL_E_RUNTIME);
72 2 : ret = aclrtSnapShotCallbackUnregister(ACL_RT_SNAPSHOT_UNLOCK_POST, PostProcessCallback);
73 2 : CHK_PRT_RET(ret != ACL_SUCCESS,
74 : HCCL_ERROR("[SnapshotControl]errNo[0x%016llx] unregister postprocess callback fail, ret[%d]",
75 : HCCL_ERROR_CODE(HCCL_E_RUNTIME), ret), HCCL_E_RUNTIME);
76 2 : ret = aclrtSnapShotCallbackUnregister(ACL_RT_SNAPSHOT_RESTORE_POST, RecoveryCallback);
77 2 : CHK_PRT_RET(ret != ACL_SUCCESS,
78 : HCCL_ERROR("[SnapshotControl]errNo[0x%016llx] unregister recovery callback fail, ret[%d]",
79 : HCCL_ERROR_CODE(HCCL_E_RUNTIME), ret), HCCL_E_RUNTIME);
80 2 : return HCCL_SUCCESS;
81 : }
82 :
83 415 : SnapshotControl &SnapshotControl::GetInstance(s32 deviceLogicId)
84 : {
85 870 : static SnapshotControl instances[MAX_MODULE_DEVICE_NUM];
86 415 : if (static_cast<u32>(deviceLogicId) >= MAX_MODULE_DEVICE_NUM) {
87 9 : return instances[0];
88 : }
89 406 : instances[deviceLogicId].deviceLogicId_ = deviceLogicId;
90 406 : return instances[deviceLogicId];
91 : }
92 :
93 455 : SnapshotControl::SnapshotControl()
94 : {
95 455 : if (!registered){
96 327 : DevType devType = DevType::DEV_TYPE_COUNT;
97 327 : HcclResult ret = hrtGetDeviceType(devType);
98 327 : CHK_PRT_CONT(ret != HCCL_SUCCESS, HCCL_ERROR("[SnapshotControl] Get device type fail, ret[%u]", ret));
99 327 : if (devType == DevType::DEV_TYPE_910B || devType == DevType::DEV_TYPE_910_93) {
100 2 : (void) ResgisterSnapshotCallback();
101 2 : registered = true;
102 : }
103 : }
104 455 : }
105 :
106 455 : SnapshotControl::~SnapshotControl()
107 : {
108 455 : if (registered) {
109 2 : (void) UnResgisterSnapshotCallback();
110 2 : registered = false;
111 : }
112 :
113 455 : std::lock_guard<std::mutex> lock(commMutex_);
114 455 : commCallbacks_.clear();
115 455 : }
116 :
117 4 : HcclResult SnapshotControl::SetStatus(SnapshotStatus status)
118 : {
119 4 : std::lock_guard<std::mutex> lock(statusMutex_);
120 4 : CHK_PRT_RET(status_ == status,
121 : HCCL_DEBUG("[SnapshotControl][SetStatus]status has already been set to [%u], deviceLogicId[%d]",
122 : status_, deviceLogicId_), HCCL_SUCCESS);
123 4 : status_ = status;
124 4 : HCCL_RUN_INFO("[SnapshotControl][SetStatus]set status to [%u], deviceLogicId[%d]", status_, deviceLogicId_);
125 4 : return HCCL_SUCCESS;
126 4 : }
127 :
128 4 : SnapshotStatus SnapshotControl::GetStatus()
129 : {
130 4 : std::lock_guard<std::mutex> lock(statusMutex_);
131 4 : return status_;
132 4 : }
133 :
134 197 : HcclResult SnapshotControl::RegisterComm(const std::string &identifier, SnapshotSetInvalidComm setInvalidCommCallback,
135 : SnapshotCheckPreProcess preProcessCallback, SnapshotCheckPostProcess postProcessCallback)
136 : {
137 197 : std::lock_guard<std::mutex> lock(commMutex_);
138 197 : if (commCallbacks_.find(identifier) != commCallbacks_.end()) {
139 2 : HCCL_WARNING("[SnapshotControl][RegisterComm] comm[%s] has already registered, devId[%d].",
140 : identifier.c_str(), deviceLogicId_);
141 2 : return HCCL_SUCCESS;
142 : }
143 195 : SnapshotCallbacks callbacks = {setInvalidCommCallback, preProcessCallback, postProcessCallback};
144 195 : commCallbacks_.emplace(identifier, callbacks);
145 195 : HCCL_RUN_INFO("[SnapshotControl][RegisterComm] comm[%s] register to snapshot control, devId[%d].",
146 : identifier.c_str(), deviceLogicId_);
147 195 : return HCCL_SUCCESS;
148 197 : }
149 :
150 3 : HcclResult SnapshotControl::RegisterBackup(const std::string &identifier, u32 backupDevicePhyId)
151 : {
152 3 : std::lock_guard<std::mutex> lock(commMutex_);
153 3 : backupDeviceCount_[backupDevicePhyId].Ref();
154 3 : HCCL_RUN_INFO("[SnapshotControl][RegisterBackup] comm[%s] register backup device to snapshot control, "
155 : "devId[%d], backupDevPhyId[%u].", identifier.c_str(), deviceLogicId_, backupDevicePhyId);
156 3 : return HCCL_SUCCESS;
157 3 : }
158 :
159 209 : HcclResult SnapshotControl::UnRegisterComm(const std::string &identifier)
160 : {
161 209 : std::lock_guard<std::mutex> lock(commMutex_);
162 209 : auto callbackIter = commCallbacks_.find(identifier);
163 209 : if (callbackIter == commCallbacks_.end()) {
164 15 : HCCL_RUN_WARNING("[SnapshotControl][UnRegisterComm] "
165 : "comm[%s] has not registered and cannot be unregistered, devId[%d].", identifier.c_str(), deviceLogicId_);
166 15 : return HCCL_SUCCESS;
167 : }
168 194 : commCallbacks_.erase(callbackIter);
169 194 : HCCL_RUN_INFO("[SnapshotControl][UnRegisterComm] comm[%s] unregister from snapshot control, devId[%d].",
170 : identifier.c_str(), deviceLogicId_);
171 194 : if (commCallbacks_.empty()) {
172 194 : HCCL_RUN_INFO("[SnapshotControl][UnRegisterComm] all comms have unregistered from snapshot control, devId[%d].",
173 : deviceLogicId_);
174 : }
175 194 : return HCCL_SUCCESS;
176 209 : }
177 :
178 0 : HcclResult SnapshotControl::UnRegisterBackup(const std::string &identifier, u32 backupDevicePhyId)
179 : {
180 0 : std::lock_guard<std::mutex> lock(commMutex_);
181 0 : auto backupIter = backupDeviceCount_.find(backupDevicePhyId);
182 0 : if (backupIter == backupDeviceCount_.end()) {
183 0 : HCCL_WARNING("[SnapshotControl][UnRegisterBackup] comm[%s] backupDevicePhyId[%u] has not been registered, "
184 : "devId[%d]", identifier.c_str(), backupDevicePhyId, deviceLogicId_);
185 0 : return HCCL_SUCCESS;
186 : }
187 0 : int count = backupDeviceCount_[backupDevicePhyId].Unref();
188 0 : if (count < 0) {
189 0 : HCCL_WARNING("[SnapshotControl][UnRegisterBackup] comm[%s] unregister backup device exceed, "
190 : "devId[%d], count[%d]", identifier.c_str(), deviceLogicId_, count);
191 0 : return HCCL_SUCCESS;
192 : }
193 0 : HCCL_RUN_INFO("[SnapshotControl][UnRegisterBackup] release backup device phydId[%u], comm[%s], "
194 : "devId[%d], count[%d]", backupDevicePhyId, identifier.c_str(), deviceLogicId_, count);
195 0 : if (count == 0) {
196 0 : backupDeviceCount_.erase(backupIter);
197 0 : HCCL_RUN_INFO("[SnapshotControl][UnRegisterBackup] backup device phydId[%u] is totally released, "
198 : "devId[%d], count[%d]", backupDevicePhyId, deviceLogicId_, count);
199 : }
200 0 : return HCCL_SUCCESS;
201 0 : }
202 :
203 1 : HcclResult SnapshotControl::CheckCommsPreProcess()
204 : {
205 1 : std::lock_guard<std::mutex> lock(commMutex_);
206 2 : for (auto callbackIter : commCallbacks_) {
207 1 : CHK_RET(callbackIter.second.preProcessCallback());
208 1 : HCCL_RUN_INFO("[SnapshotControl][CheckCommsPreProcess] comm[%s] check pre-process success, devId[%d].",
209 : callbackIter.first.c_str(), deviceLogicId_);
210 1 : }
211 1 : HCCL_INFO("[SnapshotControl][CheckCommsPreProcess] devId[%d], check pre-process success finish.", deviceLogicId_);
212 1 : return HCCL_SUCCESS;
213 1 : }
214 :
215 1 : HcclResult SnapshotControl::DevicePreProcess()
216 : {
217 1 : std::lock_guard<std::mutex> lock(commMutex_);
218 1 : CHK_RET(hrtGetDevicePhyIdByIndex(static_cast<u32>(deviceLogicId_), devicePhyId_, true));
219 :
220 1 : HcclResult ret = SnapShotSaveAction(static_cast<s32>(NICDeployment::NIC_DEPLOYMENT_DEVICE), devicePhyId_,
221 : HcclSaveSnapShotAction::HCCL_SAVE_SNAPSHOT_ACTION_PRE_PROCESSING);
222 1 : CHK_PRT_RET(ret != HCCL_SUCCESS,
223 : HCCL_ERROR("[SnapshotControl][DevicePreProcess] call SnapShotSaveAction fail, devicePhyId[%u], action[%u]",
224 : devicePhyId_, HcclSaveSnapShotAction::HCCL_SAVE_SNAPSHOT_ACTION_PRE_PROCESSING), ret);
225 1 : HCCL_INFO("[SnapshotControl][DevicePreProcess] device[%u] do device pre-process success, devId[%d].",
226 : devicePhyId_, deviceLogicId_);
227 :
228 2 : for (auto backupIter : backupDeviceCount_) {
229 1 : ret = SnapShotSaveAction(static_cast<s32>(NICDeployment::NIC_DEPLOYMENT_DEVICE), backupIter.first,
230 : HcclSaveSnapShotAction::HCCL_SAVE_SNAPSHOT_ACTION_PRE_PROCESSING);
231 1 : CHK_PRT_RET(ret != HCCL_SUCCESS,
232 : HCCL_ERROR("[SnapshotControl][DevicePreProcess] call SnapShotSaveAction fail, backup devicePhyId[%u], "
233 : "action[%u]", backupIter.first, HcclSaveSnapShotAction::HCCL_SAVE_SNAPSHOT_ACTION_PRE_PROCESSING), ret);
234 1 : HCCL_INFO("[SnapshotControl][DevicePreProcess] backup device[%u] do device pre-process success, devId[%d].",
235 : backupIter.first, deviceLogicId_);
236 1 : }
237 1 : return HCCL_SUCCESS;
238 1 : }
239 :
240 1 : HcclResult SnapshotControl::PreProcess()
241 : {
242 1 : CHK_RET(SetStatus(SnapshotStatus::PRE_SNAPSHOT));
243 1 : CHK_RET(CheckCommsPreProcess());
244 :
245 1 : if (commCallbacks_.size() != 0) {
246 1 : CHK_RET(DevicePreProcess());
247 : }
248 :
249 1 : HCCL_INFO("[SnapshotControl][PreProcess] snapshot pre-process success, devId[%d], devPhyId[%u].",
250 : deviceLogicId_, devicePhyId_);
251 1 : return HCCL_SUCCESS;
252 : }
253 :
254 1 : HcclResult SnapshotControl::CheckCommsPostProcess()
255 : {
256 1 : std::lock_guard<std::mutex> lock(commMutex_);
257 2 : for (auto callbackIter : commCallbacks_) {
258 1 : CHK_RET(callbackIter.second.postProcessCallback());
259 1 : HCCL_RUN_INFO("[SnapshotControl][CheckCommsPostProcess] comm[%s] check post-process success, devId[%d].",
260 : callbackIter.first.c_str(), deviceLogicId_);
261 1 : }
262 1 : HCCL_INFO("[SnapshotControl][CheckCommsPostProcess] devId[%d], check post-process finish.", deviceLogicId_);
263 1 : return HCCL_SUCCESS;
264 1 : }
265 :
266 1 : HcclResult SnapshotControl::DevicePostProcess()
267 : {
268 1 : std::lock_guard<std::mutex> lock(commMutex_);
269 1 : CHK_RET(hrtGetDevicePhyIdByIndex(static_cast<u32>(deviceLogicId_), devicePhyId_, true));
270 :
271 1 : HcclResult ret = SnapShotSaveAction(static_cast<s32>(NICDeployment::NIC_DEPLOYMENT_DEVICE), devicePhyId_,
272 : HcclSaveSnapShotAction::HCCL_SAVE_SNAPSHOT_ACTION_POST_PROCESSING);
273 1 : CHK_PRT_RET(ret != HCCL_SUCCESS,
274 : HCCL_ERROR("[SnapshotControl][DevicePostProcess] call SnapShotSaveAction fail, devicePhyId[%u], action[%u]",
275 : devicePhyId_, HcclSaveSnapShotAction::HCCL_SAVE_SNAPSHOT_ACTION_POST_PROCESSING), ret);
276 1 : HCCL_INFO("[SnapshotControl][DevicePostProcess] device[%u] do device post-process success, devId[%d].",
277 : devicePhyId_, deviceLogicId_);
278 :
279 2 : for (auto backupIter : backupDeviceCount_) {
280 1 : ret = SnapShotSaveAction(static_cast<s32>(NICDeployment::NIC_DEPLOYMENT_DEVICE), backupIter.first,
281 : HcclSaveSnapShotAction::HCCL_SAVE_SNAPSHOT_ACTION_POST_PROCESSING);
282 1 : CHK_PRT_RET(ret != HCCL_SUCCESS,
283 : HCCL_ERROR("[SnapshotControl][DevicePostProcess] call SnapShotSaveAction fail, backup devicePhyId[%u], "
284 : "action[%u]", backupIter.first, HcclSaveSnapShotAction::HCCL_SAVE_SNAPSHOT_ACTION_POST_PROCESSING), ret);
285 1 : HCCL_INFO("[SnapshotControl][DevicePostProcess] backup device[%u] do device post-process success, devId[%d].",
286 : backupIter.first, deviceLogicId_);
287 1 : }
288 1 : return HCCL_SUCCESS;
289 1 : }
290 :
291 1 : HcclResult SnapshotControl::PostProcess()
292 : {
293 1 : if (commCallbacks_.size() != 0) {
294 1 : CHK_RET(DevicePostProcess());
295 : }
296 :
297 1 : if (GetStatus() != SnapshotStatus::RESTORE_SNAPSHOT) {
298 1 : CHK_RET(SetStatus(SnapshotStatus::POST_SNAPSHOT));
299 1 : CHK_RET(CheckCommsPostProcess());
300 : }
301 1 : CHK_RET(SetStatus(SnapshotStatus::DEFAULT));
302 :
303 1 : HCCL_INFO("[SnapshotControl][PostProcess] snapshot post-process success, devId[%d], devPhyId[%u].",
304 : deviceLogicId_, devicePhyId_);
305 1 : return HCCL_SUCCESS;
306 : }
307 :
308 1 : HcclResult SnapshotControl::MarkInvalidComms()
309 : {
310 1 : std::lock_guard<std::mutex> lock(commMutex_);
311 2 : for (auto callbackIter : commCallbacks_) {
312 1 : CHK_RET(callbackIter.second.setInvalidCommCallback(true));
313 1 : HCCL_RUN_INFO("[SnapshotControl][MarkInvalidComms] comm[%s] has been marked as invalid comm, devId[%d].",
314 : callbackIter.first.c_str(), deviceLogicId_);
315 1 : }
316 1 : HCCL_INFO("[SnapshotControl][MarkInvalidComms] devId[%d], mark invalid comms finish.", deviceLogicId_);
317 1 : return HCCL_SUCCESS;
318 1 : }
319 :
320 1 : HcclResult SnapshotControl::DeviceRestore()
321 : {
322 1 : std::lock_guard<std::mutex> lock(commMutex_);
323 1 : CHK_RET(hrtGetDevicePhyIdByIndex(static_cast<u32>(deviceLogicId_), devicePhyId_, true));
324 :
325 1 : HcclResult ret = SnapShotRestoreAction(static_cast<s32>(NICDeployment::NIC_DEPLOYMENT_DEVICE), devicePhyId_);
326 1 : CHK_PRT_RET(ret != HCCL_SUCCESS, HCCL_ERROR("[SnapshotControl][DeviceRestore] call SnapShotRestoreAction fail, "
327 : "devicePhyId[%u]", devicePhyId_), ret);
328 1 : HCCL_INFO("[SnapshotControl][DeviceRestore] device[%u] do SnapShotRestoreAction success, devId[%d].",
329 : devicePhyId_, deviceLogicId_);
330 :
331 2 : for (auto backupIter : backupDeviceCount_) {
332 1 : ret = SnapShotRestoreAction(static_cast<s32>(NICDeployment::NIC_DEPLOYMENT_DEVICE), backupIter.first);
333 1 : CHK_PRT_RET(ret != HCCL_SUCCESS, HCCL_ERROR("[SnapshotControl][DeviceRestore] call SnapShotRestoreAction fail, "
334 : "backup devicePhyId[%u]", backupIter.first), ret);
335 1 : HCCL_INFO("[SnapshotControl][DeviceRestore] backup device[%u] do SnapShotRestoreAction success, devId[%d].",
336 : backupIter.first, deviceLogicId_);
337 1 : }
338 1 : return HCCL_SUCCESS;
339 1 : }
340 :
341 1 : HcclResult SnapshotControl::Recovery()
342 : {
343 1 : HCCL_ERROR("-------------------- THE ABOVE AND THIS ERROR LOG CAN BE IGNORED. --------------------");
344 :
345 1 : if (commCallbacks_.size() != 0) {
346 1 : CHK_RET(DeviceRestore());
347 : }
348 :
349 : // set device status to stopped, need to skip device operations
350 1 : CHK_RET(MarkInvalidComms());
351 1 : CHK_RET(SetStatus(SnapshotStatus::RESTORE_SNAPSHOT));
352 1 : CHK_RET(Transport::SetDeviceUnavailable(deviceLogicId_));
353 1 : CHK_RET(ResetInitState());
354 1 : HCCL_INFO("[SnapshotControl][PostProcess] snapshot recovery success, devId[%d], devPhyId[%u].",
355 : deviceLogicId_, devicePhyId_);
356 1 : return HCCL_SUCCESS;
357 : }
358 : }
|
