MooseApp.h

Go to the documentation of this file.

//* This file is part of the MOOSE framework
//* https://mooseframework.inl.gov
//*
//* All rights reserved, see COPYRIGHT for full restrictions
//* https://github.com/idaholab/moose/blob/master/COPYRIGHT
//*
//* Licensed under LGPL 2.1, please see LICENSE for details
//* https://www.gnu.org/licenses/lgpl-2.1.html
 
#pragma once
 
#ifdef MOOSE_LIBTORCH_ENABLED
// Libtorch includes
#include <torch/types.h>
#include <torch/mps.h>
#include <torch/cuda.h>
#include <c10/core/DeviceType.h>
#endif
 
// MOOSE includes
#include "Moose.h"
#include "Builder.h"
#include "ActionWarehouse.h"
#include "Factory.h"
#include "ActionFactory.h"
#include "OutputWarehouse.h"
#include "RestartableData.h"
#include "RestartableDataMap.h"
#include "ConsoleStreamInterface.h"
#include "PerfGraph.h"
#include "PerfGraphInterface.h"
#include "TheWarehouse.h"
#include "RankMap.h"
#include "MeshGeneratorSystem.h"
#include "ChainControlDataSystem.h"
#include "RestartableDataReader.h"
#include "Backup.h"
#include "MooseBase.h"
#include "Capability.h"
#include "MoosePassKey.h"
#include "SystemInfo.h"
#include "Syntax.h"
 
#include "libmesh/parallel_object.h"
#include "libmesh/mesh_base.h"
#include "libmesh/point.h"
 
// C++ includes
#include <list>
#include <map>
#include <set>
#include <unordered_set>
#include <typeindex>
#include <filesystem>
#include <variant>
 
// Forward declarations
class AppFactory;
class Executioner;
class Executor;
class NullExecutor;
class FEProblemBase;
class InputParameterWarehouse;
class CommandLine;
class RelationshipManager;
class ReporterData;
class SolutionInvalidity;
class MultiApp;
class MooseMesh;
#ifdef MOOSE_MFEM_ENABLED
class MFEMProblemSolve;
#endif
 
namespace libMesh
{
class ExodusII_IO;
}
namespace hit
{
class Node;
}
 
#ifdef MOOSE_KOKKOS_ENABLED
namespace Moose::Kokkos
{
class MemoryPool;
}
#endif
 
#ifdef MOOSE_UNIT_TEST
// forward declare unit tests
#include "gtest/gtest.h"
class GTEST_TEST_CLASS_NAME_(CapabilitiesTest, mooseAppAddBoolCapability);
class GTEST_TEST_CLASS_NAME_(CapabilitiesTest, mooseAppAddIntCapability);
class GTEST_TEST_CLASS_NAME_(CapabilitiesTest, mooseAppAddStringCapability);
class GTEST_TEST_CLASS_NAME_(CapabilitiesTest, mooseAppAddCapability);
#endif
 
class MooseApp : public PerfGraphInterface, public libMesh::ParallelObject, public MooseBase
{
public:
  std::optional<MooseEnum> getComputeDevice() const;
 
#ifdef MOOSE_LIBTORCH_ENABLED
  torch::DeviceType getLibtorchDevice() const { return _libtorch_device; }
#endif
 
  struct FixedPointConfig
  {
    FixedPointConfig() : sub_relaxation_factor(1.0) {}
    Real sub_relaxation_factor;
    std::vector<std::string> sub_transformed_vars;
    std::vector<PostprocessorName> sub_transformed_pps;
  };
 
  static const RestartableDataMapName MESH_META_DATA;
  static const std::string MESH_META_DATA_SUFFIX;
 
  static InputParameters validParams();
 
  virtual ~MooseApp();
 
  TheWarehouse & theWarehouse() { return *_the_warehouse; }
 
  virtual std::string getPrintableName() const { return "Application"; }
 
  virtual std::string appBinaryName() const
  {
    auto name = Moose::getExecutableName();
    name = name.substr(0, name.find_last_of("-"));
    if (name.find_first_of("/") != std::string::npos)
      name = name.substr(name.find_first_of("/") + 1, std::string::npos);
    return name;
  }
 
  int exitCode() const { return _exit_code; }
 
  void setExitCode(const int exit_code) { _exit_code = exit_code; }
 
  const RankMap & rankMap() { return _rank_map; }
 
  PerfGraph & perfGraph() { return _perf_graph; }
 
  SolutionInvalidity & solutionInvalidity() { return _solution_invalidity; }
  const SolutionInvalidity & solutionInvalidity() const { return _solution_invalidity; }
 
  virtual void run();
 
  std::string getFrameworkVersion() const;
 
  virtual std::string getVersion() const;
 
  std::string getPrintableVersion() const;
 
  virtual void setupOptions();
 
  ActionWarehouse & actionWarehouse() { return _action_warehouse; }
  const ActionWarehouse & actionWarehouse() const { return _action_warehouse; }
 
  Moose::Builder & builder() { return _builder; }
 
  Syntax & syntax() { return _syntax; }
 
  const std::vector<std::string> & getInputFileNames() const;
 
  const std::string & getLastInputFileName() const;
 
  void setOutputFileBase(const std::string & output_file_base);
 
  std::string getOutputFileBase(bool for_non_moose_build_output = false) const;
 
  void setOutputPosition(const Point & p);
 
  std::list<std::string> getCheckpointDirectories() const;
 
  std::list<std::string> getCheckpointFiles() const;
 
  bool hasOutputPosition() const { return _output_position_set; }
 
  Point getOutputPosition() const { return _output_position; }
 
  void setStartTime(Real time);
 
  bool hasStartTime() const { return _start_time_set; }
 
  Real getStartTime() const { return _start_time; }
 
  void setGlobalTimeOffset(Real offset) { _global_time_offset = offset; }
 
  Real getGlobalTimeOffset() const { return _global_time_offset; }
 
  std::string getFileName(bool stripLeadingPath = true) const;
 
  void setErrorOverridden();
 
  void disableCheckUnusedFlag();
 
  Executioner * getExecutioner() const;
  Executor * getExecutor() const { return _executor.get(); }
  NullExecutor * getNullExecutor() const { return _null_executor.get(); }
  bool useExecutor() const { return _use_executor; }
  FEProblemBase & feProblem() const;
 
  void setExecutioner(std::shared_ptr<Executioner> && executioner) { _executioner = executioner; }
  void setExecutor(std::shared_ptr<Executor> && executor) { _executor = executor; }
  void
  addExecutor(const std::string & type, const std::string & name, const InputParameters & params);
 
  void addExecutorParams(const std::string & type,
                         const std::string & name,
                         const InputParameters & params);
 
  const Parser & parser() const;
  Parser & parser();
 
  void createExecutors();
 
  Executor & getExecutor(const std::string & name, bool fail_if_not_found = true);
 
  FixedPointConfig & fixedPointConfig() { return _fixed_point_config; }
 
  bool & useNonlinear() { return _use_nonlinear; }
 
  bool & useEigenvalue() { return _use_eigen_value; }
 
  Factory & getFactory() { return _factory; }
 
  ActionFactory & getActionFactory() { return _action_factory; }
 
  processor_id_type processor_id() const { return _comm->rank(); }
 
  std::shared_ptr<CommandLine> commandLine() const { return _command_line; }
 
  void setExodusFileRestart(bool flag) { _initial_from_file = flag; }
 
  bool getExodusFileRestart() const { return _initial_from_file; }
 
  void setExReaderForRestart(std::shared_ptr<libMesh::ExodusII_IO> && exreader)
  {
    _ex_reader = exreader;
  }
 
  libMesh::ExodusII_IO * getExReaderForRestart() const { return _ex_reader.get(); }
 
  virtual void runInputFile();
 
  virtual void executeExecutioner();
 
  bool getDistributedMeshOnCommandLine() const { return _distributed_mesh_on_command_line; }
 
  bool isRecovering() const;
 
  bool isRestarting() const;
 
  bool isSplitMesh() const;
 
 
  bool hasRestartRecoverFileBase() const;
  bool hasRecoverFileBase() const;
 
 
  std::string getRestartRecoverFileBase() const { return _restart_recover_base; }
  std::string getRecoverFileBase() const
  {
    mooseDeprecated("MooseApp::getRecoverFileBase is deprecated, use "
                    "MooseApp::getRestartRecoverFileBase() instead.");
    return _restart_recover_base;
  }
 
  void setRestartRecoverFileBase(const std::string & file_base)
  {
    if (file_base.empty())
      _restart_recover_base = MooseUtils::getLatestCheckpointFilePrefix(getCheckpointFiles());
    else
      _restart_recover_base = file_base;
  }
 
  bool testCheckpointHalfTransient() const { return _test_checkpoint_half_transient; }
 
  bool testReStep() const { return _test_restep; }
 
  void setOutputFileNumbers(const std::map<std::string, unsigned int> & numbers)
  {
    _output_file_numbers = numbers;
  }
 
  const std::map<std::string, unsigned int> & getOutputFileNumbers() const
  {
    return _output_file_numbers;
  }
 
  OutputWarehouse & getOutputWarehouse();
  const OutputWarehouse & getOutputWarehouse() const;
 
  const SystemInfo & getSystemInfo() const { return _sys_info; }
 
 
  void dynamicAllRegistration(const std::string & app_name,
                              Factory * factory,
                              ActionFactory * action_factory,
                              Syntax * syntax,
                              std::string library_path,
                              const std::string & library_name);
  void dynamicAppRegistration(const std::string & app_name,
                              std::string library_path,
                              const std::string & library_name,
                              bool lib_load_deps);
 
  std::string appNameToLibName(const std::string & app_name) const;
 
  std::string libNameToAppName(const std::string & library_name) const;
 
  std::set<std::string> getLoadedLibraryPaths() const;
 
  std::set<std::string> getLibrarySearchPaths(const std::string & library_path_from_param) const;
 
  InputParameterWarehouse & getInputParameterWarehouse();
 
  /*
   * Register a piece of restartable data.  This is data that will get
   * written / read to / from a restart file.
   *
   * @param data The actual data object.
   * @param tid The thread id of the object.  Use 0 if the object is not threaded.
   * @param read_only Restrict the data for read-only
   * @param metaname (optional) register the data to the meta data storage (tid must be 0)
   */
  RestartableDataValue & registerRestartableData(std::unique_ptr<RestartableDataValue> data,
                                                 THREAD_ID tid,
                                                 bool read_only,
                                                 const RestartableDataMapName & metaname = "");
 
  /*
   * Deprecated method to register a piece of restartable data.
   *
   * Use the call without a data name instead.
   */
  RestartableDataValue & registerRestartableData(const std::string & name,
                                                 std::unique_ptr<RestartableDataValue> data,
                                                 THREAD_ID tid,
                                                 bool read_only,
                                                 const RestartableDataMapName & metaname = "");
 
  /*
   * Check if a restartable meta data exists or not.
   *
   * @param name The full (unique) name.
   * @param metaname The name to the meta data storage
   */
  bool hasRestartableMetaData(const std::string & name,
                              const RestartableDataMapName & metaname) const;
 
  /*
   * Retrieve restartable meta data from restartable data map
   *
   * @param name The full (unique) name.
   * @param metaname The name to the meta data storage
   * @return A reference to the restartable meta data value
   */
  RestartableDataValue & getRestartableMetaData(const std::string & name,
                                                const RestartableDataMapName & metaname,
                                                THREAD_ID tid);
 
  void possiblyLoadRestartableMetaData(const RestartableDataMapName & name,
                                       const std::filesystem::path & folder_base);
  void loadRestartableMetaData(const std::filesystem::path & folder_base);
 
  std::vector<std::filesystem::path>
  writeRestartableMetaData(const RestartableDataMapName & name,
                           const std::filesystem::path & folder_base);
  std::vector<std::filesystem::path>
  writeRestartableMetaData(const std::filesystem::path & folder_base);
 
  const std::vector<RestartableDataMap> & getRestartableData() const { return _restartable_data; }
  std::vector<RestartableDataMap> & getRestartableData() { return _restartable_data; }
 
  RestartableDataMap & getRestartableDataMap(const RestartableDataMapName & name);
 
  bool hasRestartableDataMap(const RestartableDataMapName & name) const;
 
  void registerRestartableDataMapName(const RestartableDataMapName & name, std::string suffix = "");
 
  const std::string & getRestartableDataMapName(const RestartableDataMapName & name) const;
 
  const DataNames & getRecoverableData() const { return _recoverable_data_names; }
 
  std::vector<std::filesystem::path> backup(const std::filesystem::path & folder_base);
  std::unique_ptr<Backup> backup();
 
  bool hasInitialBackupMesh() const;
 
  bool restoredInitialBackupMesh() const { return _restored_initial_backup_mesh; }
 
  void markMeshChangedForBackup() { _mesh_changed_for_backup = true; }
 
  bool meshChangedForBackup() const { return _mesh_changed_for_backup; }
 
  void restoreMeshFromInitialBackup(MooseMesh & mesh);
 
  virtual void preBackup() {}
 
  void restore(const std::filesystem::path & folder_base, const bool for_restart);
 
  void restore(std::unique_ptr<Backup> backup, const bool for_restart);
 
  virtual void postRestore(const bool /* for_restart */) {}
 
  void restoreFromInitialBackup(const bool for_restart);
 
  std::unique_ptr<Backup> finalizeRestore();
 
  bool restoreDataIfAvailable(RestartableDataValue & value,
                              const THREAD_ID tid,
                              Moose::PassKey<ReporterData>)
  {
    return _rd_reader.restoreDataIfAvailable(value, tid, {});
  }
 
  virtual std::string header() const;
 
  unsigned int multiAppLevel() const { return _multiapp_level; }
 
  unsigned int multiAppNumber() const { return _multiapp_number; }
 
  bool isUltimateMaster() const { return !_multiapp_level; }
 
  bool useMasterMesh() const { return _use_master_mesh; }
 
  const MooseMesh * masterMesh() const { return _master_mesh; }
 
  const MooseMesh * masterDisplacedMesh() const { return _master_displaced_mesh; }
 
  MeshGeneratorSystem & getMeshGeneratorSystem() { return _mesh_generator_system; }
 
  ChainControlDataSystem & getChainControlDataSystem() { return _chain_control_system; }
 
  void addMeshGenerator(const std::string & type,
                        const std::string & name,
                        const InputParameters & params)
  {
    _mesh_generator_system.addMeshGenerator(type, name, params);
  }
 
  bool hasMeshGenerator(const MeshGeneratorName & name) const
  {
    return _mesh_generator_system.hasMeshGenerator(name);
  }
 
  const MeshGenerator & getMeshGenerator(const std::string & name) const
  {
    return _mesh_generator_system.getMeshGenerator(name);
  }
 
  std::unique_ptr<MeshBase> getMeshGeneratorMesh()
  {
    return _mesh_generator_system.getSavedMesh(_mesh_generator_system.mainMeshGeneratorName());
  }
 
  std::vector<std::string> getMeshGeneratorNames() const
  {
    return _mesh_generator_system.getMeshGeneratorNames();
  }
 
  const MeshGenerator &
  appendMeshGenerator(const std::string & type, const std::string & name, InputParameters params)
  {
    return _mesh_generator_system.appendMeshGenerator(type, name, params);
  }
 
  virtual bool constructingMeshGenerators() const;
 
 
  void setRestart(bool value);
  void setRecover(bool value);
 
  bool checkInput() const { return _check_input; }
 
  bool getFPTrapFlag() const { return _trap_fpe; }
 
  bool hasRelationshipManager(const std::string & name) const;
 
  bool addRelationshipManager(std::shared_ptr<RelationshipManager> relationship_manager);
 
  static const std::string & checkpointSuffix();
  static std::filesystem::path metaDataFolderBase(const std::filesystem::path & folder_base,
                                                  const std::string & map_suffix);
  std::filesystem::path restartFolderBase(const std::filesystem::path & folder_base) const;
 
  const hit::Node * getCurrentActionHitNode() const;
 
  void attachRelationshipManagers(Moose::RelationshipManagerType rm_type,
                                  bool attach_geometric_rm_final = false);
 
  void attachRelationshipManagers(MeshBase & mesh, MooseMesh & moose_mesh);
 
  const std::vector<std::shared_ptr<RelationshipManager>> & getReleationshipManagers();
 
  std::vector<std::pair<std::string, std::string>> getRelationshipManagerInfo() const;
 
  const ExecFlagEnum & getExecuteOnEnum() const { return _execute_flags; }
 
  bool hasInitialBackup() const
  {
    return _initial_backup != nullptr && *_initial_backup != nullptr;
  }
 
  bool defaultAutomaticScaling() const { return _automatic_automatic_scaling; }
 
  // Return the communicator for this application
  const std::shared_ptr<libMesh::Parallel::Communicator> getCommunicator() const { return _comm; }
 
  const std::set<std::shared_ptr<RelationshipManager>> & relationshipManagers() const
  {
    return _relationship_managers;
  }
 
  void checkMetaDataIntegrity() const;
 
 
  auto getRestartableDataMapBegin() { return _restartable_meta_data.begin(); }
 
  auto getRestartableDataMapEnd() { return _restartable_meta_data.end(); }
 
  virtual bool errorOnJacobianNonzeroReallocation() const { return false; }
 
  template <class T>
  void registerInterfaceObject(T & interface);
 
  template <class T>
  const std::vector<T *> & getInterfaceObjects() const;
 
  static void addAppParam(InputParameters & params);
  static void addInputParam(InputParameters & params);
 
  bool forceRestart() const { return _force_restart; }
 
  bool unusedFlagIsWarning() const { return _enable_unused_check == WARN_UNUSED; }
 
  bool unusedFlagIsError() const { return _enable_unused_check == ERROR_UNUSED; }
 
#ifdef MOOSE_MFEM_ENABLED
  void setMFEMDevice(const std::string & device_string,
                     bool gpu_aware_mpi,
                     Moose::PassKey<MFEMProblemSolve>);
 
  std::shared_ptr<mfem::Device> getMFEMDevice(Moose::PassKey<MultiApp>) { return _mfem_device; }
 
  const std::set<std::string> & getMFEMDevices(Moose::PassKey<MultiApp>) const;
#endif
 
  bool isKokkosAvailable() const
  {
#ifdef MOOSE_KOKKOS_ENABLED
#ifdef MOOSE_ENABLE_KOKKOS_GPU
    return _has_kokkos_gpus;
#else
    return true;
#endif
#else
    return false;
#endif
  }
 
#ifdef MOOSE_KOKKOS_ENABLED
  void allocateKokkosMemoryPool(std::size_t size, unsigned int ways) const;
 
  const Moose::Kokkos::MemoryPool & getKokkosMemoryPool() const;
#endif
 
  static bool isRelocated();
 
  static bool isInTree();
 
protected:
#ifdef MOOSE_UNIT_TEST
  FRIEND_TEST(::CapabilitiesTest, mooseAppAddBoolCapability);
  FRIEND_TEST(::CapabilitiesTest, mooseAppAddIntCapability);
  FRIEND_TEST(::CapabilitiesTest, mooseAppAddStringCapability);
  FRIEND_TEST(::CapabilitiesTest, mooseAppAddCapability);
#endif
 
  void dynamicRegistration(const libMesh::Parameters & params);
 
  void loadLibraryAndDependencies(const std::string & library_filename,
                                  const libMesh::Parameters & params,
                                  bool load_dependencies = true);
 
  MooseApp(const InputParameters & parameters);
 
  void registerRestartableNameWithFilter(const std::string & name,
                                         Moose::RESTARTABLE_FILTER filter);
 
  void errorCheck();
 
  void outputMachineReadableData(const std::string & param,
                                 const std::string & start_marker,
                                 const std::string & end_marker,
                                 const std::string & data) const;
 
  static Moose::Capability & addBoolCapability(const std::string_view capability,
                                               const bool value,
                                               const std::string_view doc);
 
  static Moose::Capability &
  addIntCapability(const std::string_view capability, const int value, const std::string_view doc);
 
  static Moose::Capability & addStringCapability(const std::string_view capability,
                                                 const std::string_view value,
                                                 const std::string_view doc);
 
  static Moose::Capability & addCapability(const std::string_view capability,
                                           const Moose::Capability::Value & value,
                                           const std::string_view doc);
 
  const std::string _type;
 
  const std::shared_ptr<libMesh::Parallel::Communicator> _comm;
 
  std::string _output_file_base;
 
  bool _file_base_set_by_user;
 
  bool _output_position_set;
 
  Point _output_position;
 
  bool _start_time_set;
 
  Real _start_time;
 
  Real _global_time_offset;
 
  Syntax _syntax;
 
  std::unique_ptr<InputParameterWarehouse> _input_parameter_warehouse;
 
  ActionFactory _action_factory;
 
  ActionWarehouse _action_warehouse;
 
  OutputWarehouse _output_warehouse;
 
  const std::shared_ptr<Parser> _parser;
 
  const std::shared_ptr<CommandLine> _command_line;
 
  SystemInfo _sys_info;
 
  Moose::Builder _builder;
 
  std::vector<RestartableDataMap> _restartable_data;
 
  DataNames _recoverable_data_names;
 
  PerfGraph & _perf_graph;
 
  SolutionInvalidity & _solution_invalidity;
 
  const RankMap _rank_map;
 
  std::shared_ptr<Executioner> _executioner;
 
  std::shared_ptr<Executor> _executor;
 
  std::map<std::string, std::shared_ptr<Executor>> _executors;
 
  std::unordered_map<std::string, std::pair<std::string, std::unique_ptr<InputParameters>>>
      _executor_params;
 
  FixedPointConfig _fixed_point_config;
 
  const bool _use_executor;
 
  std::shared_ptr<NullExecutor> _null_executor;
 
  bool _use_nonlinear;
 
  bool _use_eigen_value;
 
  enum UNUSED_CHECK
  {
    OFF,
    WARN_UNUSED,
    ERROR_UNUSED
  } _enable_unused_check;
 
  Factory _factory;
 
  bool _error_overridden;
  std::string _early_exit_param;
  bool _ready_to_exit;
  int _exit_code;
 
  bool _initial_from_file;
 
  std::shared_ptr<libMesh::ExodusII_IO> _ex_reader;
 
  const bool _distributed_mesh_on_command_line;
 
  bool _recover;
 
  bool _restart;
 
  bool _split_mesh;
 
  const bool _use_split;
 
  const bool _force_restart;
 
  bool _trap_fpe;
 
  std::string _restart_recover_base;
 
  const bool _test_checkpoint_half_transient;
  const bool _test_restep;
 
  std::map<std::string, unsigned int> _output_file_numbers;
 
  const bool _check_input;
 
  std::set<std::shared_ptr<RelationshipManager>> _relationship_managers;
 
  std::map<Moose::RelationshipManagerType, std::set<const RelationshipManager *>>
      _attached_relationship_managers;
 
  std::unordered_map<RelationshipManager *, std::shared_ptr<libMesh::GhostingFunctor>>
      _undisp_to_disp_rms;
 
  struct DynamicLibraryInfo
  {
    void * library_handle;
    std::string full_path;
    std::unordered_set<std::string> entry_symbols;
  };
 
  std::unordered_map<std::string, DynamicLibraryInfo> _lib_handles;
 
private:
  void recursivelyCreateExecutors(const std::string & current_executor_name,
                                  std::list<std::string> & possible_roots,
                                  std::list<std::string> & current_branch);
 
  void removeRelationshipManager(std::shared_ptr<RelationshipManager> relationship_manager);
 
#ifdef MOOSE_LIBTORCH_ENABLED
  torch::DeviceType determineLibtorchDeviceType(const MooseEnum & device) const;
#endif
 
  struct InterfaceRegistryObjectsBase
  {
    virtual ~InterfaceRegistryObjectsBase() {}
  };
 
  template <class T>
  struct InterfaceRegistryObjects : public InterfaceRegistryObjectsBase
  {
    std::vector<T *> _objects;
  };
 
  void createMinimalApp();
 
  void setCheckUnusedFlag(bool warn_is_error = false);
 
  bool hasRMClone(const RelationshipManager & template_rm, const MeshBase & mesh) const;
 
  RelationshipManager & getRMClone(const RelationshipManager & template_rm,
                                   const MeshBase & mesh) const;
 
  RelationshipManager & createRMFromTemplateAndInit(const RelationshipManager & template_rm,
                                                    MooseMesh & moose_mesh,
                                                    MeshBase & mesh,
                                                    const libMesh::DofMap * dof_map = nullptr);
 
  PerfGraph & createRecoverablePerfGraph();
 
  SolutionInvalidity & createRecoverableSolutionInvalidity();
 
  bool showInputs() const;
 
  virtual std::string getInstallableInputs() const;
 
  bool copyInputs();
 
  bool runInputs();
 
  void requestCitations();
 
  void collectCitations(std::map<std::string, std::string> & citations) const;
 
  static Moose::Capability & addCapabilityInternal(const std::string_view capability,
                                                   const Moose::Capability::Value & value,
                                                   const std::string_view doc);
 
  std::unordered_map<RestartableDataMapName, std::pair<RestartableDataMap, std::string>>
      _restartable_meta_data;
 
  enum RegistrationType
  {
    APPLICATION,
    REGALL
  };
 
  std::unique_ptr<TheWarehouse> _the_warehouse;
 
  const unsigned int _multiapp_level;
 
  const unsigned int _multiapp_number;
 
  const bool _use_master_mesh;
 
  const MooseMesh * const _master_mesh;
 
  const MooseMesh * const _master_displaced_mesh;
 
  MeshGeneratorSystem _mesh_generator_system;
 
  ChainControlDataSystem _chain_control_system;
 
  RestartableDataReader _rd_reader;
 
  const ExecFlagEnum _execute_flags;
 
  std::streambuf * _output_buffer_cache;
 
  const bool _automatic_automatic_scaling;
 
  bool _cpu_profiling = false;
 
  bool _heap_profiling = false;
 
  std::map<const RelationshipManager *,
           std::map<const MeshBase *, std::unique_ptr<RelationshipManager>>>
      _template_to_clones;
 
  std::map<std::type_index, std::unique_ptr<InterfaceRegistryObjectsBase>> _interface_registry;
 
  std::unique_ptr<Backup> * const _initial_backup;
 
  bool _restored_initial_backup_mesh = false;
 
  bool _mesh_changed_for_backup = false;
 
#ifdef MOOSE_LIBTORCH_ENABLED
  const torch::DeviceType _libtorch_device;
#endif
 
#ifdef MOOSE_MFEM_ENABLED
  std::shared_ptr<mfem::Device> _mfem_device;
 
  std::set<std::string> _mfem_devices;
#endif
 
  // Allow FEProblemBase to set the recover/restart state, so make it a friend
  friend class FEProblemBase;
  friend class Restartable;
  friend class SubProblem;
 
#ifdef MOOSE_KOKKOS_ENABLED
  void queryKokkosGPUs();
 
  void deallocateKokkosMemoryPool();
 
  bool _has_kokkos_gpus = false;
#endif
};
 
template <class T>
void
MooseApp::registerInterfaceObject(T & interface)
{
  static_assert(!std::is_base_of<MooseObject, T>::value, "T is not an interface");
 
  InterfaceRegistryObjects<T> * registry = nullptr;
  auto it = _interface_registry.find(typeid(T));
  if (it == _interface_registry.end())
  {
    auto new_registry = std::make_unique<InterfaceRegistryObjects<T>>();
    registry = new_registry.get();
    _interface_registry.emplace(typeid(T), std::move(new_registry));
  }
  else
    registry = static_cast<InterfaceRegistryObjects<T> *>(it->second.get());
 
  mooseAssert(std::count(registry->_objects.begin(), registry->_objects.end(), &interface) == 0,
              "Interface already registered");
  registry->_objects.push_back(&interface);
}
 
template <class T>
const std::vector<T *> &
MooseApp::getInterfaceObjects() const
{
  static_assert(!std::is_base_of<MooseObject, T>::value, "T is not an interface");
 
  const auto it = _interface_registry.find(typeid(T));
  if (it != _interface_registry.end())
    return static_cast<InterfaceRegistryObjects<T> *>(it->second.get())->_objects;
  const static std::vector<T *> empty;
  return empty;
}
 
#ifdef MOOSE_MFEM_ENABLED
inline const std::set<std::string> &
MooseApp::getMFEMDevices(Moose::PassKey<MultiApp>) const
{
  return _mfem_devices;
}
#endif