Line data Source code
1 : /**
2 : * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3 : * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 : * CANN Open Software License Agreement Version 2.0 (the "License").
5 : * Please refer to the License for details. You may not use this file except in compliance with the License.
6 : * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7 : * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8 : * See LICENSE in the root of the software repository for the full text of the License.
9 : */
10 :
11 : #include "snapshot_control.h"
12 : #include "adapter_rts_common.h"
13 : #include "adapter_hccp_common.h"
14 : #include "externalinput.h"
15 : #include "transport_pub.h"
16 : #include "rt_external.h"
17 :
18 : namespace hccl {
19 :
20 : bool SnapshotControl::registered = false;
21 :
22 0 : uint32_t PreProcessCallback(int32_t devId, [[maybe_unused]] void* args)
23 : {
24 0 : HCCL_RUN_INFO("[Snapshot] PreProcess callback, devId[%d]", devId);
25 0 : HcclResult ret = SnapshotControl::GetInstance(devId).PreProcess();
26 0 : CHK_PRT_RET(ret != HCCL_SUCCESS, HCCL_ERROR("[Snapshot] PreProcess fail, devId[%d].", devId), ret);
27 0 : HCCL_RUN_INFO("[Snapshot] PreProcess success, devId[%d]", devId);
28 0 : return 0;
29 : }
30 :
31 0 : uint32_t PostProcessCallback(int32_t devId, [[maybe_unused]] void* args)
32 : {
33 0 : HCCL_RUN_INFO("[Snapshot] PostProcess callback, devId[%d]", devId);
34 0 : HcclResult ret = SnapshotControl::GetInstance(devId).PostProcess();
35 0 : CHK_PRT_RET(ret != HCCL_SUCCESS, HCCL_ERROR("[Snapshot] PostProcess fail, devId[%d].", devId), ret);
36 0 : HCCL_RUN_INFO("[Snapshot] PostProcess success, devId[%d]", devId);
37 0 : return 0;
38 : }
39 :
40 0 : uint32_t RecoveryCallback(int32_t devId, [[maybe_unused]] void* args)
41 : {
42 0 : HCCL_RUN_INFO("[Snapshot] Recovery callback, devId[%d]", devId);
43 0 : HcclResult ret = SnapshotControl::GetInstance(devId).Recovery();
44 0 : CHK_PRT_RET(ret != HCCL_SUCCESS, HCCL_ERROR("[Snapshot] Recovery fail, devId[%d].", devId), ret);
45 0 : HCCL_RUN_INFO("[Snapshot] Recovery success, devId[%d]", devId);
46 0 : return 0;
47 : }
48 :
49 2 : HcclResult ResgisterSnapshotCallback()
50 : {
51 2 : rtError_t ret = aclrtSnapShotCallbackRegister(ACL_RT_SNAPSHOT_LOCK_PRE, PreProcessCallback, nullptr);
52 2 : CHK_PRT_RET(
53 : ret != ACL_SUCCESS,
54 : HCCL_ERROR(
55 : "[SnapshotControl]errNo[0x%016llx] register preprocess callback fail, ret[%d]",
56 : HCCL_ERROR_CODE(HCCL_E_RUNTIME), ret),
57 : HCCL_E_RUNTIME);
58 2 : ret = aclrtSnapShotCallbackRegister(ACL_RT_SNAPSHOT_UNLOCK_POST, PostProcessCallback, nullptr);
59 2 : CHK_PRT_RET(
60 : ret != ACL_SUCCESS,
61 : HCCL_ERROR(
62 : "[SnapshotControl]errNo[0x%016llx] register postprocess callback fail, ret[%d]",
63 : HCCL_ERROR_CODE(HCCL_E_RUNTIME), ret),
64 : HCCL_E_RUNTIME);
65 2 : ret = aclrtSnapShotCallbackRegister(ACL_RT_SNAPSHOT_RESTORE_POST, RecoveryCallback, nullptr);
66 2 : CHK_PRT_RET(
67 : ret != ACL_SUCCESS,
68 : HCCL_ERROR(
69 : "[SnapshotControl]errNo[0x%016llx] register recovery callback fail, ret[%d]",
70 : HCCL_ERROR_CODE(HCCL_E_RUNTIME), ret),
71 : HCCL_E_RUNTIME);
72 2 : return HCCL_SUCCESS;
73 : }
74 :
75 2 : HcclResult UnResgisterSnapshotCallback()
76 : {
77 2 : rtError_t ret = aclrtSnapShotCallbackUnregister(ACL_RT_SNAPSHOT_LOCK_PRE, PreProcessCallback);
78 2 : CHK_PRT_RET(
79 : ret != ACL_SUCCESS,
80 : HCCL_ERROR(
81 : "[SnapshotControl]errNo[0x%016llx] unregister preprocess callback fail, ret[%d]",
82 : HCCL_ERROR_CODE(HCCL_E_RUNTIME), ret),
83 : HCCL_E_RUNTIME);
84 2 : ret = aclrtSnapShotCallbackUnregister(ACL_RT_SNAPSHOT_UNLOCK_POST, PostProcessCallback);
85 2 : CHK_PRT_RET(
86 : ret != ACL_SUCCESS,
87 : HCCL_ERROR(
88 : "[SnapshotControl]errNo[0x%016llx] unregister postprocess callback fail, ret[%d]",
89 : HCCL_ERROR_CODE(HCCL_E_RUNTIME), ret),
90 : HCCL_E_RUNTIME);
91 2 : ret = aclrtSnapShotCallbackUnregister(ACL_RT_SNAPSHOT_RESTORE_POST, RecoveryCallback);
92 2 : CHK_PRT_RET(
93 : ret != ACL_SUCCESS,
94 : HCCL_ERROR(
95 : "[SnapshotControl]errNo[0x%016llx] unregister recovery callback fail, ret[%d]",
96 : HCCL_ERROR_CODE(HCCL_E_RUNTIME), ret),
97 : HCCL_E_RUNTIME);
98 2 : return HCCL_SUCCESS;
99 : }
100 :
101 419 : SnapshotControl& SnapshotControl::GetInstance(s32 deviceLogicId)
102 : {
103 874 : static SnapshotControl instances[MAX_MODULE_DEVICE_NUM];
104 419 : if (static_cast<u32>(deviceLogicId) >= MAX_MODULE_DEVICE_NUM) {
105 9 : return instances[0];
106 : }
107 410 : instances[deviceLogicId].deviceLogicId_ = deviceLogicId;
108 410 : return instances[deviceLogicId];
109 : }
110 :
111 455 : SnapshotControl::SnapshotControl()
112 : {
113 455 : if (!registered) {
114 327 : DevType devType = DevType::DEV_TYPE_COUNT;
115 327 : HcclResult ret = hrtGetDeviceType(devType);
116 327 : CHK_PRT_CONT(ret != HCCL_SUCCESS, HCCL_ERROR("[SnapshotControl] Get device type fail, ret[%u]", ret));
117 327 : if (devType == DevType::DEV_TYPE_910B || devType == DevType::DEV_TYPE_910_93) {
118 2 : (void)ResgisterSnapshotCallback();
119 2 : registered = true;
120 : }
121 : }
122 455 : }
123 :
124 455 : SnapshotControl::~SnapshotControl()
125 : {
126 455 : if (registered) {
127 2 : (void)UnResgisterSnapshotCallback();
128 2 : registered = false;
129 : }
130 :
131 455 : std::lock_guard<std::mutex> lock(commMutex_);
132 455 : commCallbacks_.clear();
133 455 : }
134 :
135 4 : HcclResult SnapshotControl::SetStatus(SnapshotStatus status)
136 : {
137 4 : std::lock_guard<std::mutex> lock(statusMutex_);
138 4 : CHK_PRT_RET(
139 : status_ == status,
140 : HCCL_DEBUG(
141 : "[SnapshotControl][SetStatus]status has already been set to [%u], deviceLogicId[%d]", status_,
142 : deviceLogicId_),
143 : HCCL_SUCCESS);
144 4 : status_ = status;
145 4 : HCCL_RUN_INFO("[SnapshotControl][SetStatus]set status to [%u], deviceLogicId[%d]", status_, deviceLogicId_);
146 4 : return HCCL_SUCCESS;
147 4 : }
148 :
149 4 : SnapshotStatus SnapshotControl::GetStatus()
150 : {
151 4 : std::lock_guard<std::mutex> lock(statusMutex_);
152 4 : return status_;
153 4 : }
154 :
155 199 : HcclResult SnapshotControl::RegisterComm(
156 : const std::string& identifier, SnapshotSetInvalidComm setInvalidCommCallback,
157 : SnapshotCheckPreProcess preProcessCallback, SnapshotCheckPostProcess postProcessCallback)
158 : {
159 199 : std::lock_guard<std::mutex> lock(commMutex_);
160 199 : if (commCallbacks_.find(identifier) != commCallbacks_.end()) {
161 2 : HCCL_WARNING(
162 : "[SnapshotControl][RegisterComm] comm[%s] has already registered, devId[%d].", identifier.c_str(),
163 : deviceLogicId_);
164 2 : return HCCL_SUCCESS;
165 : }
166 197 : SnapshotCallbacks callbacks = {setInvalidCommCallback, preProcessCallback, postProcessCallback};
167 197 : commCallbacks_.emplace(identifier, callbacks);
168 197 : HCCL_RUN_INFO(
169 : "[SnapshotControl][RegisterComm] comm[%s] register to snapshot control, devId[%d].", identifier.c_str(),
170 : deviceLogicId_);
171 197 : return HCCL_SUCCESS;
172 199 : }
173 :
174 3 : HcclResult SnapshotControl::RegisterBackup(const std::string& identifier, u32 backupDevicePhyId)
175 : {
176 3 : std::lock_guard<std::mutex> lock(commMutex_);
177 3 : backupDeviceCount_[backupDevicePhyId].Ref();
178 3 : HCCL_RUN_INFO(
179 : "[SnapshotControl][RegisterBackup] comm[%s] register backup device to snapshot control, "
180 : "devId[%d], backupDevPhyId[%u].",
181 : identifier.c_str(), deviceLogicId_, backupDevicePhyId);
182 3 : return HCCL_SUCCESS;
183 3 : }
184 :
185 211 : HcclResult SnapshotControl::UnRegisterComm(const std::string& identifier)
186 : {
187 211 : std::lock_guard<std::mutex> lock(commMutex_);
188 211 : auto callbackIter = commCallbacks_.find(identifier);
189 211 : if (callbackIter == commCallbacks_.end()) {
190 15 : HCCL_RUN_WARNING(
191 : "[SnapshotControl][UnRegisterComm] "
192 : "comm[%s] has not registered and cannot be unregistered, devId[%d].",
193 : identifier.c_str(), deviceLogicId_);
194 15 : return HCCL_SUCCESS;
195 : }
196 196 : commCallbacks_.erase(callbackIter);
197 196 : HCCL_RUN_INFO(
198 : "[SnapshotControl][UnRegisterComm] comm[%s] unregister from snapshot control, devId[%d].", identifier.c_str(),
199 : deviceLogicId_);
200 196 : if (commCallbacks_.empty()) {
201 196 : HCCL_RUN_INFO(
202 : "[SnapshotControl][UnRegisterComm] all comms have unregistered from snapshot control, devId[%d].",
203 : deviceLogicId_);
204 : }
205 196 : return HCCL_SUCCESS;
206 211 : }
207 :
208 0 : HcclResult SnapshotControl::UnRegisterBackup(const std::string& identifier, u32 backupDevicePhyId)
209 : {
210 0 : std::lock_guard<std::mutex> lock(commMutex_);
211 0 : auto backupIter = backupDeviceCount_.find(backupDevicePhyId);
212 0 : if (backupIter == backupDeviceCount_.end()) {
213 0 : HCCL_WARNING(
214 : "[SnapshotControl][UnRegisterBackup] comm[%s] backupDevicePhyId[%u] has not been registered, "
215 : "devId[%d]",
216 : identifier.c_str(), backupDevicePhyId, deviceLogicId_);
217 0 : return HCCL_SUCCESS;
218 : }
219 0 : int count = backupDeviceCount_[backupDevicePhyId].Unref();
220 0 : if (count < 0) {
221 0 : HCCL_WARNING(
222 : "[SnapshotControl][UnRegisterBackup] comm[%s] unregister backup device exceed, "
223 : "devId[%d], count[%d]",
224 : identifier.c_str(), deviceLogicId_, count);
225 0 : return HCCL_SUCCESS;
226 : }
227 0 : HCCL_RUN_INFO(
228 : "[SnapshotControl][UnRegisterBackup] release backup device phydId[%u], comm[%s], "
229 : "devId[%d], count[%d]",
230 : backupDevicePhyId, identifier.c_str(), deviceLogicId_, count);
231 0 : if (count == 0) {
232 0 : backupDeviceCount_.erase(backupIter);
233 0 : HCCL_RUN_INFO(
234 : "[SnapshotControl][UnRegisterBackup] backup device phydId[%u] is totally released, "
235 : "devId[%d], count[%d]",
236 : backupDevicePhyId, deviceLogicId_, count);
237 : }
238 0 : return HCCL_SUCCESS;
239 0 : }
240 :
241 1 : HcclResult SnapshotControl::CheckCommsPreProcess()
242 : {
243 1 : std::lock_guard<std::mutex> lock(commMutex_);
244 2 : for (auto callbackIter : commCallbacks_) {
245 1 : CHK_RET(callbackIter.second.preProcessCallback());
246 1 : HCCL_RUN_INFO(
247 : "[SnapshotControl][CheckCommsPreProcess] comm[%s] check pre-process success, devId[%d].",
248 : callbackIter.first.c_str(), deviceLogicId_);
249 1 : }
250 1 : HCCL_INFO("[SnapshotControl][CheckCommsPreProcess] devId[%d], check pre-process success finish.", deviceLogicId_);
251 1 : return HCCL_SUCCESS;
252 1 : }
253 :
254 1 : HcclResult SnapshotControl::DevicePreProcess()
255 : {
256 1 : std::lock_guard<std::mutex> lock(commMutex_);
257 1 : CHK_RET(hrtGetDevicePhyIdByIndex(static_cast<u32>(deviceLogicId_), devicePhyId_, true));
258 :
259 1 : HcclResult ret = SnapShotSaveAction(
260 : static_cast<s32>(NICDeployment::NIC_DEPLOYMENT_DEVICE), devicePhyId_,
261 : HcclSaveSnapShotAction::HCCL_SAVE_SNAPSHOT_ACTION_PRE_PROCESSING);
262 1 : CHK_PRT_RET(
263 : ret != HCCL_SUCCESS,
264 : HCCL_ERROR(
265 : "[SnapshotControl][DevicePreProcess] call SnapShotSaveAction fail, devicePhyId[%u], action[%u]",
266 : devicePhyId_, HcclSaveSnapShotAction::HCCL_SAVE_SNAPSHOT_ACTION_PRE_PROCESSING),
267 : ret);
268 1 : HCCL_INFO(
269 : "[SnapshotControl][DevicePreProcess] device[%u] do device pre-process success, devId[%d].", devicePhyId_,
270 : deviceLogicId_);
271 :
272 2 : for (auto backupIter : backupDeviceCount_) {
273 2 : ret = SnapShotSaveAction(
274 1 : static_cast<s32>(NICDeployment::NIC_DEPLOYMENT_DEVICE), backupIter.first,
275 : HcclSaveSnapShotAction::HCCL_SAVE_SNAPSHOT_ACTION_PRE_PROCESSING);
276 1 : CHK_PRT_RET(
277 : ret != HCCL_SUCCESS,
278 : HCCL_ERROR(
279 : "[SnapshotControl][DevicePreProcess] call SnapShotSaveAction fail, backup devicePhyId[%u], "
280 : "action[%u]",
281 : backupIter.first, HcclSaveSnapShotAction::HCCL_SAVE_SNAPSHOT_ACTION_PRE_PROCESSING),
282 : ret);
283 1 : HCCL_INFO(
284 : "[SnapshotControl][DevicePreProcess] backup device[%u] do device pre-process success, devId[%d].",
285 : backupIter.first, deviceLogicId_);
286 1 : }
287 1 : return HCCL_SUCCESS;
288 1 : }
289 :
290 1 : HcclResult SnapshotControl::PreProcess()
291 : {
292 1 : CHK_RET(SetStatus(SnapshotStatus::PRE_SNAPSHOT));
293 1 : CHK_RET(CheckCommsPreProcess());
294 :
295 1 : if (commCallbacks_.size() != 0) {
296 1 : CHK_RET(DevicePreProcess());
297 : }
298 :
299 1 : HCCL_INFO(
300 : "[SnapshotControl][PreProcess] snapshot pre-process success, devId[%d], devPhyId[%u].", deviceLogicId_,
301 : devicePhyId_);
302 1 : return HCCL_SUCCESS;
303 : }
304 :
305 1 : HcclResult SnapshotControl::CheckCommsPostProcess()
306 : {
307 1 : std::lock_guard<std::mutex> lock(commMutex_);
308 2 : for (auto callbackIter : commCallbacks_) {
309 1 : CHK_RET(callbackIter.second.postProcessCallback());
310 1 : HCCL_RUN_INFO(
311 : "[SnapshotControl][CheckCommsPostProcess] comm[%s] check post-process success, devId[%d].",
312 : callbackIter.first.c_str(), deviceLogicId_);
313 1 : }
314 1 : HCCL_INFO("[SnapshotControl][CheckCommsPostProcess] devId[%d], check post-process finish.", deviceLogicId_);
315 1 : return HCCL_SUCCESS;
316 1 : }
317 :
318 1 : HcclResult SnapshotControl::DevicePostProcess()
319 : {
320 1 : std::lock_guard<std::mutex> lock(commMutex_);
321 1 : CHK_RET(hrtGetDevicePhyIdByIndex(static_cast<u32>(deviceLogicId_), devicePhyId_, true));
322 :
323 1 : HcclResult ret = SnapShotSaveAction(
324 : static_cast<s32>(NICDeployment::NIC_DEPLOYMENT_DEVICE), devicePhyId_,
325 : HcclSaveSnapShotAction::HCCL_SAVE_SNAPSHOT_ACTION_POST_PROCESSING);
326 1 : CHK_PRT_RET(
327 : ret != HCCL_SUCCESS,
328 : HCCL_ERROR(
329 : "[SnapshotControl][DevicePostProcess] call SnapShotSaveAction fail, devicePhyId[%u], action[%u]",
330 : devicePhyId_, HcclSaveSnapShotAction::HCCL_SAVE_SNAPSHOT_ACTION_POST_PROCESSING),
331 : ret);
332 1 : HCCL_INFO(
333 : "[SnapshotControl][DevicePostProcess] device[%u] do device post-process success, devId[%d].", devicePhyId_,
334 : deviceLogicId_);
335 :
336 2 : for (auto backupIter : backupDeviceCount_) {
337 2 : ret = SnapShotSaveAction(
338 1 : static_cast<s32>(NICDeployment::NIC_DEPLOYMENT_DEVICE), backupIter.first,
339 : HcclSaveSnapShotAction::HCCL_SAVE_SNAPSHOT_ACTION_POST_PROCESSING);
340 1 : CHK_PRT_RET(
341 : ret != HCCL_SUCCESS,
342 : HCCL_ERROR(
343 : "[SnapshotControl][DevicePostProcess] call SnapShotSaveAction fail, backup devicePhyId[%u], "
344 : "action[%u]",
345 : backupIter.first, HcclSaveSnapShotAction::HCCL_SAVE_SNAPSHOT_ACTION_POST_PROCESSING),
346 : ret);
347 1 : HCCL_INFO(
348 : "[SnapshotControl][DevicePostProcess] backup device[%u] do device post-process success, devId[%d].",
349 : backupIter.first, deviceLogicId_);
350 1 : }
351 1 : return HCCL_SUCCESS;
352 1 : }
353 :
354 1 : HcclResult SnapshotControl::PostProcess()
355 : {
356 1 : if (commCallbacks_.size() != 0) {
357 1 : CHK_RET(DevicePostProcess());
358 : }
359 :
360 1 : if (GetStatus() != SnapshotStatus::RESTORE_SNAPSHOT) {
361 1 : CHK_RET(SetStatus(SnapshotStatus::POST_SNAPSHOT));
362 1 : CHK_RET(CheckCommsPostProcess());
363 : }
364 1 : CHK_RET(SetStatus(SnapshotStatus::DEFAULT));
365 :
366 1 : HCCL_INFO(
367 : "[SnapshotControl][PostProcess] snapshot post-process success, devId[%d], devPhyId[%u].", deviceLogicId_,
368 : devicePhyId_);
369 1 : return HCCL_SUCCESS;
370 : }
371 :
372 1 : HcclResult SnapshotControl::MarkInvalidComms()
373 : {
374 1 : std::lock_guard<std::mutex> lock(commMutex_);
375 2 : for (auto callbackIter : commCallbacks_) {
376 1 : CHK_RET(callbackIter.second.setInvalidCommCallback(true));
377 1 : HCCL_RUN_INFO(
378 : "[SnapshotControl][MarkInvalidComms] comm[%s] has been marked as invalid comm, devId[%d].",
379 : callbackIter.first.c_str(), deviceLogicId_);
380 1 : }
381 1 : HCCL_INFO("[SnapshotControl][MarkInvalidComms] devId[%d], mark invalid comms finish.", deviceLogicId_);
382 1 : return HCCL_SUCCESS;
383 1 : }
384 :
385 1 : HcclResult SnapshotControl::DeviceRestore()
386 : {
387 1 : std::lock_guard<std::mutex> lock(commMutex_);
388 1 : CHK_RET(hrtGetDevicePhyIdByIndex(static_cast<u32>(deviceLogicId_), devicePhyId_, true));
389 :
390 1 : HcclResult ret = SnapShotRestoreAction(static_cast<s32>(NICDeployment::NIC_DEPLOYMENT_DEVICE), devicePhyId_);
391 1 : CHK_PRT_RET(
392 : ret != HCCL_SUCCESS,
393 : HCCL_ERROR(
394 : "[SnapshotControl][DeviceRestore] call SnapShotRestoreAction fail, "
395 : "devicePhyId[%u]",
396 : devicePhyId_),
397 : ret);
398 1 : HCCL_INFO(
399 : "[SnapshotControl][DeviceRestore] device[%u] do SnapShotRestoreAction success, devId[%d].", devicePhyId_,
400 : deviceLogicId_);
401 :
402 2 : for (auto backupIter : backupDeviceCount_) {
403 1 : ret = SnapShotRestoreAction(static_cast<s32>(NICDeployment::NIC_DEPLOYMENT_DEVICE), backupIter.first);
404 1 : CHK_PRT_RET(
405 : ret != HCCL_SUCCESS,
406 : HCCL_ERROR(
407 : "[SnapshotControl][DeviceRestore] call SnapShotRestoreAction fail, "
408 : "backup devicePhyId[%u]",
409 : backupIter.first),
410 : ret);
411 1 : HCCL_INFO(
412 : "[SnapshotControl][DeviceRestore] backup device[%u] do SnapShotRestoreAction success, devId[%d].",
413 : backupIter.first, deviceLogicId_);
414 1 : }
415 1 : return HCCL_SUCCESS;
416 1 : }
417 :
418 1 : HcclResult SnapshotControl::Recovery()
419 : {
420 1 : HCCL_ERROR("-------------------- THE ABOVE AND THIS ERROR LOG CAN BE IGNORED. --------------------");
421 :
422 1 : if (commCallbacks_.size() != 0) {
423 1 : CHK_RET(DeviceRestore());
424 : }
425 :
426 : // set device status to stopped, need to skip device operations
427 1 : CHK_RET(MarkInvalidComms());
428 1 : CHK_RET(SetStatus(SnapshotStatus::RESTORE_SNAPSHOT));
429 1 : CHK_RET(Transport::SetDeviceUnavailable(deviceLogicId_));
430 1 : CHK_RET(ResetInitState());
431 1 : HCCL_INFO(
432 : "[SnapshotControl][PostProcess] snapshot recovery success, devId[%d], devPhyId[%u].", deviceLogicId_,
433 : devicePhyId_);
434 1 : return HCCL_SUCCESS;
435 : }
436 : } // namespace hccl
|
