Line data    Source code

       1              : /**
       2              :  * Copyright (c) 2025 Huawei Technologies Co., Ltd.
       3              :  * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
       4              :  * CANN Open Software License Agreement Version 2.0 (the "License").
       5              :  * Please refer to the License for details. You may not use this file except in compliance with the License.
       6              :  * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
       7              :  * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
       8              :  * See LICENSE in the root of the software repository for the full text of the License.
       9              :  */
      10              : #include <thread>
      11              : #include "log/adx_log.h"
      12              : #include "dump_manager.h"
      13              : #include "sys_utils.h"
      14              : #include "dump_args.h"
      15              : #include "dump_tensor_plugin.h"
      16              : #include "dump_core.h"
      17              : #include "dump_memory.h"
      18              : #include "exception_dumper.h"
      19              : 
      20              : namespace Adx {
      21              : namespace {
      22              : // Timeout Threshod For Fast Recovery
      23              : constexpr uint32_t TIMEOUT_THRESHOLD = 500U;
      24              : } // namespace
      25              : 
      26           49 : int32_t ExceptionDumper::LoadTensorPluginLib()
      27              : {
      28              :     // no tensor use the plugin feature
      29           49 :     return DumpTensorPlugin::Instance().InitPluginLib();
      30              : }
      31              : 
      32           59 : int32_t ExceptionDumper::DumpArgsException(const rtExceptionInfo& exception, const std::string& dumpPath)
      33              : {
      34              :     // L0 exception dump: support fast recovery
      35           59 :     uint32_t timeout = 0;
      36           59 :     rtError_t ret = rtGetOpExecuteTimeoutV2(&timeout);
      37           59 :     if (ret != ACL_RT_SUCCESS) {
      38            0 :         IDE_LOGE("Get operator timeout failed, ret: %d", ret);
      39              :     } else {
      40           59 :         IDE_LOGI("Get operator timeout %ums", timeout);
      41           59 :         if (timeout < TIMEOUT_THRESHOLD) {
      42            2 :             IDE_LOGE("Operator timeout %ums, enable fast recovery, not dump data to file.", timeout);
      43            2 :             return DumpArgsExceptionFastRecovery(exception);
      44              :         }
      45              :     }
      46              : 
      47           57 :     return DumpArgsExceptionInner(exception, dumpPath);
      48              : }
      49              : 
      50            2 : int32_t ExceptionDumper::DumpArgsExceptionFastRecovery(const rtExceptionInfo& exception) const
      51              : {
      52            2 :     IDE_CTRL_VALUE_WARN(
      53              :         ExceptionInfoCommon::IsSupportDefaultExceptionDump(exception), return ADUMP_FAILED,
      54              :         "Exception is not support default dump.");
      55              :     // copy exception and other data for thread because of RTS free item after exception callback
      56            1 :     void* exceptionCopy = DumpMemory::CopyHostToHost(&exception, sizeof(rtExceptionInfo));
      57            1 :     if (exceptionCopy == nullptr) {
      58            0 :         IDE_LOGE("Copy rtExceptionInfo failed.");
      59            0 :         return ADUMP_FAILED;
      60              :     }
      61            1 :     std::thread([exceptionCopy]() {
      62            1 :         DumpArgs args;
      63            1 :         rtExceptionInfo* exceptionPtr = static_cast<rtExceptionInfo*>(exceptionCopy);
      64            1 :         rtError_t ret = rtSetDevice(exceptionPtr->deviceid);
      65            1 :         if (ret != ADUMP_SUCCESS) {
      66            0 :             IDE_LOGE("Execute rtSetDevice on device %u failed with result %d", exceptionPtr->deviceid, ret);
      67              :         }
      68              :         // only print args information but not dump to file
      69            1 :         if (args.LoadArgsExceptionInfo(*exceptionPtr) != ADUMP_SUCCESS) {
      70            1 :             IDE_LOGE("Fast recovery LoadArgsExceptionInfo failed.");
      71              :         }
      72            1 :         void* exceptionFree = static_cast<void*>(exceptionPtr);
      73            1 :         DumpMemory::FreeHost(exceptionFree);
      74            2 :     }).detach();
      75            1 :     return ADUMP_SUCCESS;
      76              : }
      77              : 
      78            3 : int32_t ExceptionDumper::DumpDetailException(const rtExceptionInfo& exception, const std::string& dumpPath)
      79              : {
      80            3 :     IDE_CTRL_VALUE_WARN(
      81              :         ExceptionInfoCommon::IsSupportDefaultExceptionDump(exception), return ADUMP_FAILED,
      82              :         "Exception is not support default dump.");
      83            2 :     if (coredumpEnableComplete_) {
      84            1 :         std::lock_guard<std::mutex> lock(mutex_);
      85            1 :         DumpCore core(dumpPath, exception.deviceid);
      86            1 :         if (core.DumpCoreFile(exception) != ADUMP_SUCCESS) {
      87            1 :             return ADUMP_FAILED;
      88              :         }
      89              :         // exit the process after detail exception dump
      90            0 :         Exit();
      91            2 :     } else {
      92            1 :         IDE_CTRL_VALUE_WARN(
      93              :             LoadTensorPluginLib() == ADUMP_SUCCESS, return ADUMP_FAILED, "Load tensor custom plugin failed.");
      94              :         // detail exception dump downgrade to L0 exception dump
      95            1 :         return DumpArgsException(exception, dumpPath);
      96              :     }
      97            0 :     return ADUMP_SUCCESS;
      98              : }
      99              : } // namespace Adx