MooseApp.h

Go to the documentation of this file.

//* This file is part of the MOOSE framework
//* https://mooseframework.inl.gov
//*
//* All rights reserved, see COPYRIGHT for full restrictions
//* https://github.com/idaholab/moose/blob/master/COPYRIGHT
//*
//* Licensed under LGPL 2.1, please see LICENSE for details
//* https://www.gnu.org/licenses/lgpl-2.1.html

#pragma once

#ifdef MOOSE_LIBTORCH_ENABLED
// Libtorch includes
#include <torch/types.h>
#include <torch/mps.h>
#include <torch/cuda.h>
#include <c10/core/DeviceType.h>
#endif

#ifdef MOOSE_MFEM_ENABLED
#include "libmesh/ignore_warnings.h"
#include <mfem.hpp>
#include "libmesh/restore_warnings.h"
#endif

// MOOSE includes
#include "Moose.h"
#include "Builder.h"
#include "ActionWarehouse.h"
#include "Factory.h"
#include "ActionFactory.h"
#include "OutputWarehouse.h"
#include "RestartableData.h"
#include "RestartableDataMap.h"
#include "ConsoleStreamInterface.h"
#include "PerfGraph.h"
#include "PerfGraphInterface.h"
#include "TheWarehouse.h"
#include "RankMap.h"
#include "MeshGeneratorSystem.h"
#include "ChainControlDataSystem.h"
#include "RestartableDataReader.h"
#include "Backup.h"
#include "MooseBase.h"
#include "Capabilities.h"
#include "MoosePassKey.h"
#include "SystemInfo.h"
#include "Syntax.h"

#include "libmesh/parallel_object.h"
#include "libmesh/mesh_base.h"
#include "libmesh/point.h"

// C++ includes
#include <list>
#include <map>
#include <set>
#include <unordered_set>
#include <typeindex>
#include <filesystem>
#include <variant>

// Forward declarations
class Executioner;
class Executor;
class NullExecutor;
class FEProblemBase;
class InputParameterWarehouse;
class CommandLine;
class RelationshipManager;
class SolutionInvalidity;
class MultiApp;
#ifdef MOOSE_MFEM_ENABLED
class MFEMProblemSolve;
#endif

namespace libMesh
{
class ExodusII_IO;
}
namespace hit
{
class Node;
}

#ifdef MOOSE_KOKKOS_ENABLED
namespace Moose::Kokkos
{
class MemoryPool;
}
#endif

class MooseApp : public PerfGraphInterface, public libMesh::ParallelObject, public MooseBase
{
public:
  std::optional<MooseEnum> getComputeDevice() const;

#ifdef MOOSE_LIBTORCH_ENABLED
  torch::DeviceType getLibtorchDevice() const { return _libtorch_device; }
#endif

  struct FixedPointConfig
  {
    FixedPointConfig() : sub_relaxation_factor(1.0) {}
    Real sub_relaxation_factor;
    std::vector<std::string> sub_transformed_vars;
    std::vector<PostprocessorName> sub_transformed_pps;
  };

  static const RestartableDataMapName MESH_META_DATA;
  static const std::string MESH_META_DATA_SUFFIX;

  static InputParameters validParams();

  virtual ~MooseApp();

  TheWarehouse & theWarehouse() { return *_the_warehouse; }

  virtual std::string getPrintableName() const { return "Application"; }

  virtual std::string appBinaryName() const
  {
    auto name = Moose::getExecutableName();
    name = name.substr(0, name.find_last_of("-"));
    if (name.find_first_of("/") != std::string::npos)
      name = name.substr(name.find_first_of("/") + 1, std::string::npos);
    return name;
  }

  int exitCode() const { return _exit_code; }

  void setExitCode(const int exit_code) { _exit_code = exit_code; }

  const RankMap & rankMap() { return _rank_map; }

  PerfGraph & perfGraph() { return _perf_graph; }

  SolutionInvalidity & solutionInvalidity() { return _solution_invalidity; }
  const SolutionInvalidity & solutionInvalidity() const { return _solution_invalidity; }

  virtual void run();

  std::string getFrameworkVersion() const;

  virtual std::string getVersion() const;

  std::string getPrintableVersion() const;

  virtual void setupOptions();

  ActionWarehouse & actionWarehouse() { return _action_warehouse; }
  const ActionWarehouse & actionWarehouse() const { return _action_warehouse; }

  Moose::Builder & builder() { return _builder; }

  Syntax & syntax() { return _syntax; }

  const std::vector<std::string> & getInputFileNames() const;

  const std::string & getLastInputFileName() const;

  void setOutputFileBase(const std::string & output_file_base);

  std::string getOutputFileBase(bool for_non_moose_build_output = false) const;

  void setOutputPosition(const Point & p);

  std::list<std::string> getCheckpointDirectories() const;

  std::list<std::string> getCheckpointFiles() const;

  bool hasOutputPosition() const { return _output_position_set; }

  Point getOutputPosition() const { return _output_position; }

  void setStartTime(Real time);

  bool hasStartTime() const { return _start_time_set; }

  Real getStartTime() const { return _start_time; }

  void setGlobalTimeOffset(Real offset) { _global_time_offset = offset; }

  Real getGlobalTimeOffset() const { return _global_time_offset; }

  std::string getFileName(bool stripLeadingPath = true) const;

  void setErrorOverridden();

  void disableCheckUnusedFlag();

  Executioner * getExecutioner() const;
  Executor * getExecutor() const { return _executor.get(); }
  NullExecutor * getNullExecutor() const { return _null_executor.get(); }
  bool useExecutor() const { return _use_executor; }
  FEProblemBase & feProblem() const;

  void setExecutioner(std::shared_ptr<Executioner> && executioner) { _executioner = executioner; }
  void setExecutor(std::shared_ptr<Executor> && executor) { _executor = executor; }
  void
  addExecutor(const std::string & type, const std::string & name, const InputParameters & params);

  void addExecutorParams(const std::string & type,
                         const std::string & name,
                         const InputParameters & params);

  const Parser & parser() const;
  Parser & parser();

  void createExecutors();

  Executor & getExecutor(const std::string & name, bool fail_if_not_found = true);

  FixedPointConfig & fixedPointConfig() { return _fixed_point_config; }

  bool & useNonlinear() { return _use_nonlinear; }

  bool & useEigenvalue() { return _use_eigen_value; }

  Factory & getFactory() { return _factory; }

  ActionFactory & getActionFactory() { return _action_factory; }

  processor_id_type processor_id() const { return _comm->rank(); }

  std::shared_ptr<CommandLine> commandLine() const { return _command_line; }

  void setExodusFileRestart(bool flag) { _initial_from_file = flag; }

  bool getExodusFileRestart() const { return _initial_from_file; }

  void setExReaderForRestart(std::shared_ptr<libMesh::ExodusII_IO> && exreader)
  {
    _ex_reader = exreader;
  }

  libMesh::ExodusII_IO * getExReaderForRestart() const { return _ex_reader.get(); }

  virtual void runInputFile();

  virtual void executeExecutioner();

  bool getDistributedMeshOnCommandLine() const { return _distributed_mesh_on_command_line; }

  bool isRecovering() const;

  bool isRestarting() const;

  bool isSplitMesh() const;


  bool hasRestartRecoverFileBase() const;
  bool hasRecoverFileBase() const;


  std::string getRestartRecoverFileBase() const { return _restart_recover_base; }
  std::string getRecoverFileBase() const
  {
    mooseDeprecated("MooseApp::getRecoverFileBase is deprecated, use "
                    "MooseApp::getRestartRecoverFileBase() instead.");
    return _restart_recover_base;
  }

  void setRestartRecoverFileBase(const std::string & file_base)
  {
    if (file_base.empty())
      _restart_recover_base = MooseUtils::getLatestCheckpointFilePrefix(getCheckpointFiles());
    else
      _restart_recover_base = file_base;
  }

  bool testCheckpointHalfTransient() const { return _test_checkpoint_half_transient; }

  bool testReStep() const { return _test_restep; }

  void setOutputFileNumbers(const std::map<std::string, unsigned int> & numbers)
  {
    _output_file_numbers = numbers;
  }

  const std::map<std::string, unsigned int> & getOutputFileNumbers() const
  {
    return _output_file_numbers;
  }

  OutputWarehouse & getOutputWarehouse();
  const OutputWarehouse & getOutputWarehouse() const;

  const SystemInfo & getSystemInfo() const { return _sys_info; }


  void dynamicAllRegistration(const std::string & app_name,
                              Factory * factory,
                              ActionFactory * action_factory,
                              Syntax * syntax,
                              std::string library_path,
                              const std::string & library_name);
  void dynamicAppRegistration(const std::string & app_name,
                              std::string library_path,
                              const std::string & library_name,
                              bool lib_load_deps);

  std::string appNameToLibName(const std::string & app_name) const;

  std::string libNameToAppName(const std::string & library_name) const;

  std::set<std::string> getLoadedLibraryPaths() const;

  std::set<std::string> getLibrarySearchPaths(const std::string & library_path_from_param) const;

  InputParameterWarehouse & getInputParameterWarehouse();

  /*
   * Register a piece of restartable data.  This is data that will get
   * written / read to / from a restart file.
   *
   * @param data The actual data object.
   * @param tid The thread id of the object.  Use 0 if the object is not threaded.
   * @param read_only Restrict the data for read-only
   * @param metaname (optional) register the data to the meta data storage (tid must be 0)
   */
  RestartableDataValue & registerRestartableData(std::unique_ptr<RestartableDataValue> data,
                                                 THREAD_ID tid,
                                                 bool read_only,
                                                 const RestartableDataMapName & metaname = "");

  /*
   * Deprecated method to register a piece of restartable data.
   *
   * Use the call without a data name instead.
   */
  RestartableDataValue & registerRestartableData(const std::string & name,
                                                 std::unique_ptr<RestartableDataValue> data,
                                                 THREAD_ID tid,
                                                 bool read_only,
                                                 const RestartableDataMapName & metaname = "");

  /*
   * Check if a restartable meta data exists or not.
   *
   * @param name The full (unique) name.
   * @param metaname The name to the meta data storage
   */
  bool hasRestartableMetaData(const std::string & name,
                              const RestartableDataMapName & metaname) const;

  /*
   * Retrieve restartable meta data from restartable data map
   *
   * @param name The full (unique) name.
   * @param metaname The name to the meta data storage
   * @return A reference to the restartable meta data value
   */
  RestartableDataValue & getRestartableMetaData(const std::string & name,
                                                const RestartableDataMapName & metaname,
                                                THREAD_ID tid);

  void possiblyLoadRestartableMetaData(const RestartableDataMapName & name,
                                       const std::filesystem::path & folder_base);
  void loadRestartableMetaData(const std::filesystem::path & folder_base);

  std::vector<std::filesystem::path>
  writeRestartableMetaData(const RestartableDataMapName & name,
                           const std::filesystem::path & folder_base);
  std::vector<std::filesystem::path>
  writeRestartableMetaData(const std::filesystem::path & folder_base);

  const std::vector<RestartableDataMap> & getRestartableData() const { return _restartable_data; }
  std::vector<RestartableDataMap> & getRestartableData() { return _restartable_data; }

  RestartableDataMap & getRestartableDataMap(const RestartableDataMapName & name);

  bool hasRestartableDataMap(const RestartableDataMapName & name) const;

  void registerRestartableDataMapName(const RestartableDataMapName & name, std::string suffix = "");

  const std::string & getRestartableDataMapName(const RestartableDataMapName & name) const;

  const DataNames & getRecoverableData() const { return _recoverable_data_names; }

  std::vector<std::filesystem::path> backup(const std::filesystem::path & folder_base);
  std::unique_ptr<Backup> backup();

  virtual void preBackup() {}

  void restore(const std::filesystem::path & folder_base, const bool for_restart);

  void restore(std::unique_ptr<Backup> backup, const bool for_restart);

  virtual void postRestore(const bool /* for_restart */) {}

  void restoreFromInitialBackup(const bool for_restart);

  std::unique_ptr<Backup> finalizeRestore();

  virtual std::string header() const;

  unsigned int multiAppLevel() const { return _multiapp_level; }

  unsigned int multiAppNumber() const { return _multiapp_number; }

  bool isUltimateMaster() const { return !_multiapp_level; }

  bool useMasterMesh() const { return _use_master_mesh; }

  const MooseMesh * masterMesh() const { return _master_mesh; }

  const MooseMesh * masterDisplacedMesh() const { return _master_displaced_mesh; }

  MeshGeneratorSystem & getMeshGeneratorSystem() { return _mesh_generator_system; }

  ChainControlDataSystem & getChainControlDataSystem() { return _chain_control_system; }

  void addMeshGenerator(const std::string & type,
                        const std::string & name,
                        const InputParameters & params)
  {
    _mesh_generator_system.addMeshGenerator(type, name, params);
  }

  bool hasMeshGenerator(const MeshGeneratorName & name) const
  {
    return _mesh_generator_system.hasMeshGenerator(name);
  }

  const MeshGenerator & getMeshGenerator(const std::string & name) const
  {
    return _mesh_generator_system.getMeshGenerator(name);
  }

  std::unique_ptr<MeshBase> getMeshGeneratorMesh()
  {
    return _mesh_generator_system.getSavedMesh(_mesh_generator_system.mainMeshGeneratorName());
  }

  std::vector<std::string> getMeshGeneratorNames() const
  {
    return _mesh_generator_system.getMeshGeneratorNames();
  }

  const MeshGenerator &
  appendMeshGenerator(const std::string & type, const std::string & name, InputParameters params)
  {
    return _mesh_generator_system.appendMeshGenerator(type, name, params);
  }

  virtual bool constructingMeshGenerators() const;


  void setRestart(bool value);
  void setRecover(bool value);

  bool checkInput() const { return _check_input; }

  bool getFPTrapFlag() const { return _trap_fpe; }

  bool hasRelationshipManager(const std::string & name) const;

  bool addRelationshipManager(std::shared_ptr<RelationshipManager> relationship_manager);

  static const std::string & checkpointSuffix();
  static std::filesystem::path metaDataFolderBase(const std::filesystem::path & folder_base,
                                                  const std::string & map_suffix);
  std::filesystem::path restartFolderBase(const std::filesystem::path & folder_base) const;

  const hit::Node * getCurrentActionHitNode() const;

  void attachRelationshipManagers(Moose::RelationshipManagerType rm_type,
                                  bool attach_geometric_rm_final = false);

  void attachRelationshipManagers(MeshBase & mesh, MooseMesh & moose_mesh);

  const std::vector<std::shared_ptr<RelationshipManager>> & getReleationshipManagers();

  std::vector<std::pair<std::string, std::string>> getRelationshipManagerInfo() const;

  const ExecFlagEnum & getExecuteOnEnum() const { return _execute_flags; }

  bool hasInitialBackup() const
  {
    return _initial_backup != nullptr && *_initial_backup != nullptr;
  }

  bool defaultAutomaticScaling() const { return _automatic_automatic_scaling; }

  // Return the communicator for this application
  const std::shared_ptr<libMesh::Parallel::Communicator> getCommunicator() const { return _comm; }

  const std::set<std::shared_ptr<RelationshipManager>> & relationshipManagers() const
  {
    return _relationship_managers;
  }

  void checkMetaDataIntegrity() const;


  auto getRestartableDataMapBegin() { return _restartable_meta_data.begin(); }

  auto getRestartableDataMapEnd() { return _restartable_meta_data.end(); }

  virtual bool errorOnJacobianNonzeroReallocation() const { return false; }

  template <class T>
  void registerInterfaceObject(T & interface);

  template <class T>
  const std::vector<T *> & getInterfaceObjects() const;

  static void addAppParam(InputParameters & params);
  static void addInputParam(InputParameters & params);

  bool forceRestart() const { return _force_restart; }

  bool unusedFlagIsWarning() const { return _enable_unused_check == WARN_UNUSED; }

  bool unusedFlagIsError() const { return _enable_unused_check == ERROR_UNUSED; }

#ifdef MOOSE_MFEM_ENABLED

  void setMFEMDevice(const std::string & device_string, Moose::PassKey<MFEMProblemSolve>);

  std::shared_ptr<mfem::Device> getMFEMDevice(Moose::PassKey<MultiApp>) { return _mfem_device; }

  const std::set<std::string> & getMFEMDevices(Moose::PassKey<MultiApp>) const;
#endif

  bool isKokkosAvailable() const
  {
#ifdef MOOSE_KOKKOS_ENABLED
#ifdef MOOSE_ENABLE_KOKKOS_GPU
    return _has_kokkos_gpus;
#else
    return true;
#endif
#else
    return false;
#endif
  }

#ifdef MOOSE_KOKKOS_ENABLED

  bool hasKokkosObjects() const { return _has_kokkos_objects; }

  void allocateKokkosMemoryPool(std::size_t size, unsigned int ways) const;

  const Moose::Kokkos::MemoryPool & getKokkosMemoryPool() const;
#endif

protected:
  void dynamicRegistration(const libMesh::Parameters & params);

  void loadLibraryAndDependencies(const std::string & library_filename,
                                  const libMesh::Parameters & params,
                                  bool load_dependencies = true);

  MooseApp(const InputParameters & parameters);

  void registerRestartableNameWithFilter(const std::string & name,
                                         Moose::RESTARTABLE_FILTER filter);

  void errorCheck();

  void outputMachineReadableData(const std::string & param,
                                 const std::string & start_marker,
                                 const std::string & end_marker,
                                 const std::string & data) const;
  static void addCapability(const std::string & capability,
                            CapabilityUtils::Type value,
                            const std::string & doc);
  static void
  addCapability(const std::string & capability, const char * value, const std::string & doc);

  const std::string _type;

  const std::shared_ptr<libMesh::Parallel::Communicator> _comm;

  std::string _output_file_base;

  bool _file_base_set_by_user;

  bool _output_position_set;

  Point _output_position;

  bool _start_time_set;

  Real _start_time;

  Real _global_time_offset;

  Syntax _syntax;

  std::unique_ptr<InputParameterWarehouse> _input_parameter_warehouse;

  ActionFactory _action_factory;

  ActionWarehouse _action_warehouse;

  OutputWarehouse _output_warehouse;

  const std::shared_ptr<Parser> _parser;

  const std::shared_ptr<CommandLine> _command_line;

  SystemInfo _sys_info;

  Moose::Builder _builder;

  std::vector<RestartableDataMap> _restartable_data;

  DataNames _recoverable_data_names;

  PerfGraph & _perf_graph;

  SolutionInvalidity & _solution_invalidity;

  const RankMap _rank_map;

  std::shared_ptr<Executioner> _executioner;

  std::shared_ptr<Executor> _executor;

  std::map<std::string, std::shared_ptr<Executor>> _executors;

  std::unordered_map<std::string, std::pair<std::string, std::unique_ptr<InputParameters>>>
      _executor_params;

  FixedPointConfig _fixed_point_config;

  const bool _use_executor;

  std::shared_ptr<NullExecutor> _null_executor;

  bool _use_nonlinear;

  bool _use_eigen_value;

  enum UNUSED_CHECK
  {
    OFF,
    WARN_UNUSED,
    ERROR_UNUSED
  } _enable_unused_check;

  Factory _factory;

  bool _error_overridden;
  std::string _early_exit_param;
  bool _ready_to_exit;
  int _exit_code;

  bool _initial_from_file;

  std::shared_ptr<libMesh::ExodusII_IO> _ex_reader;

  bool _distributed_mesh_on_command_line;

  bool _recover;

  bool _restart;

  bool _split_mesh;

  const bool _use_split;

  const bool _force_restart;

  bool _trap_fpe;

  std::string _restart_recover_base;

  const bool _test_checkpoint_half_transient;
  const bool _test_restep;

  std::map<std::string, unsigned int> _output_file_numbers;

  bool _check_input;

  std::set<std::shared_ptr<RelationshipManager>> _relationship_managers;

  std::map<Moose::RelationshipManagerType, std::set<const RelationshipManager *>>
      _attached_relationship_managers;

  std::unordered_map<RelationshipManager *, std::shared_ptr<libMesh::GhostingFunctor>>
      _undisp_to_disp_rms;

  struct DynamicLibraryInfo
  {
    void * library_handle;
    std::string full_path;
    std::unordered_set<std::string> entry_symbols;
  };

  std::unordered_map<std::string, DynamicLibraryInfo> _lib_handles;

private:
  void recursivelyCreateExecutors(const std::string & current_executor_name,
                                  std::list<std::string> & possible_roots,
                                  std::list<std::string> & current_branch);

  void registerCapabilities();

  void removeRelationshipManager(std::shared_ptr<RelationshipManager> relationship_manager);

#ifdef MOOSE_LIBTORCH_ENABLED

  torch::DeviceType determineLibtorchDeviceType(const MooseEnum & device) const;
#endif

  struct InterfaceRegistryObjectsBase
  {
    virtual ~InterfaceRegistryObjectsBase() {}
  };

  template <class T>
  struct InterfaceRegistryObjects : public InterfaceRegistryObjectsBase
  {
    std::vector<T *> _objects;
  };

  void createMinimalApp();

  void setCheckUnusedFlag(bool warn_is_error = false);

  bool hasRMClone(const RelationshipManager & template_rm, const MeshBase & mesh) const;

  RelationshipManager & getRMClone(const RelationshipManager & template_rm,
                                   const MeshBase & mesh) const;

  RelationshipManager & createRMFromTemplateAndInit(const RelationshipManager & template_rm,
                                                    MooseMesh & moose_mesh,
                                                    MeshBase & mesh,
                                                    const libMesh::DofMap * dof_map = nullptr);

  PerfGraph & createRecoverablePerfGraph();

  SolutionInvalidity & createRecoverableSolutionInvalidity();

  bool showInputs() const;

  virtual std::string getInstallableInputs() const;

  bool copyInputs();

  bool runInputs();

  static void checkReservedCapability(const std::string & capability);

  std::unordered_map<RestartableDataMapName, std::pair<RestartableDataMap, std::string>>
      _restartable_meta_data;

  enum RegistrationType
  {
    APPLICATION,
    REGALL
  };

  std::unique_ptr<TheWarehouse> _the_warehouse;

  unsigned int _multiapp_level;

  unsigned int _multiapp_number;

  const bool _use_master_mesh;

  const MooseMesh * const _master_mesh;

  const MooseMesh * const _master_displaced_mesh;

  MeshGeneratorSystem _mesh_generator_system;

  ChainControlDataSystem _chain_control_system;

  RestartableDataReader _rd_reader;

  const ExecFlagEnum _execute_flags;

  std::streambuf * _output_buffer_cache;

  const bool _automatic_automatic_scaling;

  bool _cpu_profiling = false;

  bool _heap_profiling = false;

  std::map<const RelationshipManager *,
           std::map<const MeshBase *, std::unique_ptr<RelationshipManager>>>
      _template_to_clones;

  std::map<std::type_index, std::unique_ptr<InterfaceRegistryObjectsBase>> _interface_registry;

  std::unique_ptr<Backup> * const _initial_backup;

#ifdef MOOSE_LIBTORCH_ENABLED
  const torch::DeviceType _libtorch_device;
#endif

#ifdef MOOSE_MFEM_ENABLED
  std::shared_ptr<mfem::Device> _mfem_device;

  std::set<std::string> _mfem_devices;
#endif

  // Allow FEProblemBase to set the recover/restart state, so make it a friend
  friend class FEProblemBase;
  friend class Restartable;
  friend class SubProblem;

#ifdef MOOSE_KOKKOS_ENABLED

  void queryKokkosGPUs();

  void deallocateKokkosMemoryPool();

  bool _has_kokkos_gpus = false;

  bool _has_kokkos_objects = false;
#endif
};

template <class T>
void
MooseApp::registerInterfaceObject(T & interface)
{
  static_assert(!std::is_base_of<MooseObject, T>::value, "T is not an interface");

  InterfaceRegistryObjects<T> * registry = nullptr;
  auto it = _interface_registry.find(typeid(T));
  if (it == _interface_registry.end())
  {
    auto new_registry = std::make_unique<InterfaceRegistryObjects<T>>();
    registry = new_registry.get();
    _interface_registry.emplace(typeid(T), std::move(new_registry));
  }
  else
    registry = static_cast<InterfaceRegistryObjects<T> *>(it->second.get());

  mooseAssert(std::count(registry->_objects.begin(), registry->_objects.end(), &interface) == 0,
              "Interface already registered");
  registry->_objects.push_back(&interface);
}

template <class T>
const std::vector<T *> &
MooseApp::getInterfaceObjects() const
{
  static_assert(!std::is_base_of<MooseObject, T>::value, "T is not an interface");

  const auto it = _interface_registry.find(typeid(T));
  if (it != _interface_registry.end())
    return static_cast<InterfaceRegistryObjects<T> *>(it->second.get())->_objects;
  const static std::vector<T *> empty;
  return empty;
}

#ifdef MOOSE_MFEM_ENABLED
inline const std::set<std::string> &
MooseApp::getMFEMDevices(Moose::PassKey<MultiApp>) const
{
  return _mfem_devices;
}
#endif