Line data    Source code

       1             : //* This file is part of the MOOSE framework
       2             : //* https://mooseframework.inl.gov
       3             : //*
       4             : //* All rights reserved, see COPYRIGHT for full restrictions
       5             : //* https://github.com/idaholab/moose/blob/master/COPYRIGHT
       6             : //*
       7             : //* Licensed under LGPL 2.1, please see LICENSE for details
       8             : //* https://www.gnu.org/licenses/lgpl-2.1.html
       9             : 
      10             : #pragma once
      11             : 
      12             : #include "MooseObject.h"
      13             : #include "SetupInterface.h"
      14             : #include "Restartable.h"
      15             : #include "PerfGraphInterface.h"
      16             : #include "Backup.h"
      17             : 
      18             : #include "libmesh/communicator.h"
      19             : #include "libmesh/point.h"
      20             : 
      21             : #include "nlohmann/json.h"
      22             : 
      23             : class UserObject;
      24             : class FEProblemBase;
      25             : class FEProblem;
      26             : class Executioner;
      27             : class MooseApp;
      28             : class MultiAppTransfer;
      29             : class MultiAppCoordTransform;
      30             : class Positions;
      31             : 
      32             : // libMesh forward declarations
      33             : namespace libMesh
      34             : {
      35             : class BoundingBox;
      36             : namespace MeshTools
      37             : {
      38             : class BoundingBox;
      39             : }
      40             : template <typename T>
      41             : class NumericVector;
      42             : } // namespace libMesh
      43             : 
      44             : /// Holds app partitioning information relevant to the a particular rank for a
      45             : /// multiapp scenario.
      46             : struct LocalRankConfig
      47             : {
      48             :   /// The number of simulations that should/will be run locally on this rank.
      49             :   dof_id_type num_local_sims;
      50             :   /// The (global) index of the first local simulation for this rank.  All
      51             :   /// ranks that are used to perform multi-proc parallel runs for a given
      52             :   /// simulation will have the same first_local_sim_index as each other.
      53             :   dof_id_type first_local_sim_index;
      54             :   /// The number of (sub)apps that should/will be run locally on this rank.
      55             :   /// This will generally be identical to num_local_sims unless operating in
      56             :   /// some sort of "batch" mode where a single subapp is reused for multiple
      57             :   /// simulations.
      58             :   dof_id_type num_local_apps;
      59             :   /// The (global) index of the first local app for this rank.  All ranks that
      60             :   /// are used to perform multi-proc parallel runs for a given app will have
      61             :   /// the same first_local_app_index as each other.  This will generally be
      62             :   /// identical to first_local_sim_index unless operating in some sort of "batch" mode
      63             :   /// where a single subapp is reused for multiple simulations.
      64             :   dof_id_type first_local_app_index;
      65             :   /// This is true if this rank is the primary/zero rank for a (sub)app slot.
      66             :   /// A slot is all ranks that are grouped together to run a single (sub)app
      67             :   /// together.  This field will be true for exactly one rank in each slot.
      68             :   /// This is important for things like multiapp transfers where you want to
      69             :   /// only transfer data to a given subapp once even though it may be running on
      70             :   /// multiple procs/ranks.
      71             :   bool is_first_local_rank;
      72             :   /// For every rank working on a subapp, we store the first rank on each
      73             :   /// process to make the communication to root simpler on the main app
      74             :   processor_id_type my_first_rank;
      75             : };
      76             : 
      77             : /// Returns app partitioning information relevant to the given rank for a
      78             : /// multiapp scenario with the given number of apps (napps) and parallel/mpi
      79             : /// procs (nprocs).  min_app_procs and max_app_procs define the min and max
      80             : /// number of procs that must/can be used in parallel to run a given (sub)app.
      81             : /// batch_mode affects whether 1 subapp is assigned per rank to be re-used to
      82             : /// run each of the (napps) simulations or whether 1 subapp is created for
      83             : /// each napps simulation (globally).
      84             : ///
      85             : /// Each proc calls this function in order to determine which (sub)apps among
      86             : /// the global list of all subapps for a multiapp should be run by the given
      87             : /// rank.
      88             : LocalRankConfig rankConfig(processor_id_type rank,
      89             :                            processor_id_type nprocs,
      90             :                            dof_id_type napps,
      91             :                            processor_id_type min_app_procs,
      92             :                            processor_id_type max_app_procs,
      93             :                            bool batch_mode = false);
      94             : 
      95             : /**
      96             :  * Helper class for holding Sub-app backups
      97             :  *
      98             :  * Stores the backups and also triggers a backup on store and restore on load
      99             :  */
     100             : class SubAppBackups : public std::vector<std::unique_ptr<Backup>>
     101             : {
     102             : };
     103             : 
     104             : /**
     105             :  * A MultiApp represents one or more MOOSE applications that are running simultaneously.
     106             :  * These other MOOSE apps generally represent some "sub-solve" or "embedded-solves"
     107             :  * of the overall nonlinear solve. If your system support dynamic libraries unregistered
     108             :  * Multiapps can be loaded on the fly by setting the exporting the appropriate library
     109             :  * path using "MOOSE_LIBRARY_PATH" or by specifying a single input file library path
     110             :  * in Multiapps InputParameters object.
     111             :  */
     112             : class MultiApp : public MooseObject,
     113             :                  public SetupInterface,
     114             :                  public Restartable,
     115             :                  public PerfGraphInterface
     116             : {
     117             : public:
     118             :   static InputParameters validParams();
     119             : 
     120             :   MultiApp(const InputParameters & parameters);
     121             : 
     122           0 :   virtual void preExecute() {}
     123             : 
     124             :   /**
     125             :    * Method called towards the end of the simulation to execute on final.
     126             :    */
     127             :   virtual void finalize();
     128             : 
     129             :   /**
     130             :    * Method called at the end of the simulation (after finalize).
     131             :    */
     132             :   virtual void postExecute();
     133             : 
     134             :   /**
     135             :    * Called just after construction to allow derived classes to set _positions and create
     136             :    * sub-apps accordingly.
     137             :    */
     138             :   void setupPositions();
     139             : 
     140             :   /**
     141             :    * Create the i-th local app
     142             :    * @param[in] i local app index
     143             :    */
     144             :   virtual void createLocalApp(const unsigned int i);
     145             : 
     146             :   /**
     147             :    * Method to be called in main-app initial setup for create sub-apps if using positions is false.
     148             :    */
     149             :   virtual void initialSetup() override;
     150             : 
     151             :   /**
     152             :    * Gets called just before transfers are done _to_ the MultiApp
     153             :    * (Which is just before the MultiApp is solved).
     154             :    */
     155             :   virtual void preTransfer(Real dt, Real target_time);
     156             : 
     157             :   /**
     158             :    * Re-solve all of the Apps.
     159             :    *
     160             :    * Can be called multiple times to resolve the same timestep
     161             :    * if auto_advance=false. Time is not actually advanced until
     162             :    * advanceStep() is called.
     163             :    *
     164             :    * Note that auto_advance=false might not be compatible with
     165             :    * the options for the MultiApp
     166             :    *
     167             :    * @return Whether or not all of the solves were successful (i.e. all solves made it to the
     168             :    * target_time)
     169             :    */
     170             :   virtual bool solveStep(Real dt, Real target_time, bool auto_advance = true) = 0;
     171             : 
     172             :   /**
     173             :    * Advances the multi-apps time step which is important for dt selection.
     174             :    * (Note this does not advance the *time*. That is done in Transient::endStep,
     175             :    * which is called either directly from solveStep() for loose coupling cases
     176             :    * or through finishStep() for Picard coupling cases)
     177             :    */
     178         181 :   virtual void incrementTStep(Real /*target_time*/) {}
     179             : 
     180             :   /**
     181             :    * Calls multi-apps executioners' endStep and postStep methods which creates output and advances
     182             :    * time (not the time step; see incrementTStep()) among other things. This method is only called
     183             :    * for Picard calculations because for loosely coupled calculations the executioners' endStep and
     184             :    * postStep methods are called from solveStep(). This may be called with the optional flag \p
     185             :    * recurse_through_multiapp_levels which may be useful if this method is being called for the
     186             :    * *final* time of program execution
     187             :    */
     188           0 :   virtual void finishStep(bool /*recurse_through_multiapp_levels*/ = false) {}
     189             : 
     190             :   /**
     191             :    * Save off the state of every Sub App
     192             :    *
     193             :    * This allows us to "Restore" this state later
     194             :    */
     195             :   virtual void backup();
     196             : 
     197             :   /**
     198             :    * Restore the state of every Sub App. This allows us to "Restore" this state later
     199             :    * If the app is not forced to restore, it is only done as needed (and also for the
     200             :    * apps underneath as needed)
     201             :    *
     202             :    * @param force Whether the restoration has to happen no matter what.
     203             :    */
     204             :   virtual void restore(bool force = true);
     205             : 
     206             :   /**
     207             :    * Whether or not this MultiApp should be restored at the beginning of
     208             :    * each Picard iteration.
     209             :    */
     210       41746 :   bool needsRestoration() { return !_no_restore; }
     211             : 
     212             :   /**
     213             :    * @param app The global app number to get the Executioner for
     214             :    * @return The Executioner associated with that App.
     215             :    */
     216             :   virtual Executioner * getExecutioner(unsigned int app);
     217             : 
     218             :   /**
     219             :    * Get the BoundingBox for the mesh associated with app
     220             :    * The bounding box will be shifted to be in the correct position
     221             :    * within the master domain.
     222             :    * If the MultiApp is in an RZ coordinate system the box will be
     223             :    * the size it would be if the geometry were 3D (ie if you were to revolve
     224             :    * the geometry around the axis to create the 3D geometry).
     225             :    * @param app The global app number you want to get the bounding box for
     226             :    * @param displaced_mesh True if the bounding box is retrieved for the displaced mesh, other false
     227             :    * @param coord_transform An optional coordinate transformation object
     228             :    */
     229             :   virtual libMesh::BoundingBox
     230             :   getBoundingBox(unsigned int app,
     231             :                  bool displaced_mesh,
     232             :                  const MultiAppCoordTransform * coord_transform = nullptr);
     233             : 
     234             :   /**
     235             :    * Get the FEProblemBase this MultiApp is part of.
     236             :    */
     237      167246 :   FEProblemBase & problemBase() { return _fe_problem; }
     238             : 
     239             :   /**
     240             :    * Get the FEProblemBase for the global app desired.
     241             :    * @param app The global app number
     242             :    */
     243             :   FEProblemBase & appProblemBase(unsigned int app);
     244             : 
     245             :   /**
     246             :    * Get the FEProblem for the global app is part of.
     247             :    * @param app The global app number
     248             :    */
     249             :   FEProblem & appProblem(unsigned int app);
     250             : 
     251             :   /**
     252             :    * Get a UserObject base for a specific global app
     253             :    * @param app The global app number you want to get a UserObject from.
     254             :    * @param name The name of the UserObject.
     255             :    */
     256             :   const UserObject & appUserObjectBase(unsigned int app, const std::string & name);
     257             : 
     258             :   /**
     259             :    * Get a Postprocessor value for a specified global app
     260             :    * @param app The global app number you want to get a Postprocessor from.
     261             :    * @param name The name of the Postprocessor.
     262             :    */
     263             :   Real appPostprocessorValue(unsigned int app, const std::string & name);
     264             : 
     265             :   /**
     266             :    * Get the vector to transfer to for this MultiApp.
     267             :    * In general this is the Auxiliary system solution vector.
     268             :    * @param app The global app number you want the transfer vector for.
     269             :    * @param var_name The name of the variable you are going to be transferring to.
     270             :    * @return The vector to fill.
     271             :    */
     272             :   virtual libMesh::NumericVector<libMesh::Number> & appTransferVector(unsigned int app,
     273             :                                                                       std::string var_name);
     274             : 
     275             :   /**
     276             :    * @return Number of Global Apps in this MultiApp
     277             :    */
     278     1214560 :   unsigned int numGlobalApps() const { return _total_num_apps; }
     279             : 
     280             :   /**
     281             :    * @return Number of Apps on local processor.
     282             :    */
     283       51204 :   unsigned int numLocalApps() { return _apps.size(); }
     284             : 
     285             :   /**
     286             :    * @return The global number of the first app on the local processor.
     287             :    */
     288       28415 :   unsigned int firstLocalApp() { return _first_local_app; }
     289             : 
     290             :   /**
     291             :    * @return Whether this rank is the first rank of the subapp(s) it's involved in
     292             :    */
     293             :   bool isFirstLocalRank() const;
     294             : 
     295             :   /**
     296             :    * Whether or not this MultiApp has an app on this processor.
     297             :    */
     298       28227 :   bool hasApp() { return _has_an_app; }
     299             : 
     300             :   /**
     301             :    * Whether or not the given global app number is on this processor.
     302             :    * @param global_app The global app number in question
     303             :    * @return True if the global app is on this processor
     304             :    */
     305             :   bool hasLocalApp(unsigned int global_app) const;
     306             : 
     307             :   /**
     308             :    * Get the local MooseApp object
     309             :    * @param local_app The local app number
     310             :    */
     311             :   MooseApp * localApp(unsigned int local_app);
     312             : 
     313             :   /**
     314             :    * The physical position of a global App number
     315             :    * @param app The global app number you want the position for.
     316             :    * @return the position
     317             :    */
     318             :   const Point & position(unsigned int app) const;
     319             : 
     320             :   /**
     321             :    * "Reset" the App corresponding to the global App number
     322             :    * passed in.  "Reset" means that the App will be deleted
     323             :    * and recreated.  The time for the new App will be set
     324             :    * to the current simulation time.  This might be handy
     325             :    * if some sub-app in your simulation needs to get replaced
     326             :    * by a "new" piece of material.
     327             :    *
     328             :    * @param global_app The global app number to reset.
     329             :    * @param time The time to set as the the time for the new app, this should really be the time the
     330             :    * old app was at.
     331             :    */
     332             :   virtual void resetApp(unsigned int global_app, Real time = 0.0);
     333             : 
     334             :   /**
     335             :    * Move the global_app to Point p.
     336             :    *
     337             :    * @param global_app The global app number in question
     338             :    * @param p The new position of the App.
     339             :    */
     340             :   virtual void moveApp(unsigned int global_app, Point p);
     341             : 
     342             :   /**
     343             :    * For apps outputting in position we need to change their output positions
     344             :    * if their parent app moves.
     345             :    */
     346             :   virtual void parentOutputPositionChanged();
     347             : 
     348             :   /**
     349             :    * Get the MPI communicator this MultiApp is operating on.
     350             :    * @return The MPI comm for this MultiApp
     351             :    */
     352      149803 :   MPI_Comm & comm() { return _my_comm; }
     353             : 
     354             :   /**
     355             :    * Whether or not this processor is the "root" processor for the sub communicator.
     356             :    * The "root" processor has rank 0 in the sub communicator
     357             :    */
     358       26362 :   bool isRootProcessor() { return _my_rank == 0; }
     359             : 
     360             :   /**
     361             :    * Whether or not this MultiApp is using positions or its own way for constructing sub-apps.
     362             :    */
     363      192387 :   bool usingPositions() const { return _use_positions; }
     364             : 
     365             :   /**
     366             :    * Whether or not this MultiApp is being run in position,
     367             :    * eg with the coordinate transform already applied
     368             :    */
     369      381502 :   bool runningInPosition() const { return _run_in_position; }
     370             : 
     371             :   /**
     372             :    * Add a transfer that is associated with this multiapp
     373             :    */
     374             :   void addAssociatedTransfer(MultiAppTransfer & transfer);
     375             : 
     376             :   /**
     377             :    * Transform a bounding box according to the transformations in the provided coordinate
     378             :    * transformation object
     379             :    */
     380             :   static void transformBoundingBox(libMesh::BoundingBox & box,
     381             :                                    const MultiAppCoordTransform & transform);
     382             : 
     383             :   /**
     384             :    * Sets all the app's output file bases. @see MooseApp::setOutputFileBase for usage
     385             :    */
     386             :   void setAppOutputFileBase();
     387             : 
     388             : protected:
     389             :   /// function that provides cli_args to subapps
     390             :   virtual std::vector<std::string> cliArgs() const;
     391             : 
     392             :   /**
     393             :    * _must_ fill in _positions with the positions of the sub-aps
     394             :    */
     395             :   virtual void fillPositions();
     396             : 
     397             :   /**
     398             :    * Fill command line arguments for sub apps
     399             :    */
     400             :   void readCommandLineArguments();
     401             : 
     402             :   /**
     403             :    * Helper function for creating an App instance.
     404             :    *
     405             :    * @param i The local app number to create.
     406             :    * @param start_time The initial time for the App
     407             :    */
     408             :   void createApp(unsigned int i, Real start_time);
     409             : 
     410             :   /**
     411             :    * Create an MPI communicator suitable for each app.
     412             :    *
     413             :    * Also find out which communicator we are using and what our first local app is.
     414             :    */
     415             :   void buildComm();
     416             : 
     417             :   /**
     418             :    * Map a global App number to the local number.
     419             :    * Note: This will error if given a global number that doesn't map to a local number.
     420             :    *
     421             :    * @param global_app The global app number.
     422             :    * @return The local app number.
     423             :    */
     424             :   unsigned int globalAppToLocal(unsigned int global_app);
     425             : 
     426             :   /// call back executed right before app->runInputFile()
     427             :   virtual void preRunInputFile();
     428             : 
     429             :   /**
     430             :    * @return The command line arguments to be applied to the subapp
     431             :    * with index \p local_app
     432             :    *
     433             :    * The method is virtual because it is needed to allow for batch runs
     434             :    * within the stochastic tools module; see SamplerFullSolveMultiApp for
     435             :    * an example.
     436             :    */
     437             :   virtual std::vector<std::string> getCommandLineArgs(const unsigned int local_app);
     438             : 
     439             :   /**
     440             :    * Build communicators and reserve backups.
     441             :    */
     442             :   void init(unsigned int num_apps, bool batch_mode = false);
     443             : 
     444             :   /**
     445             :    * Same as other init method, except defining a custom rank configuration
     446             :    */
     447             :   void init(unsigned int num_apps, const LocalRankConfig & config);
     448             : 
     449             :   /**
     450             :    * Create the provided number of apps.
     451             :    *
     452             :    * This is called in the setupPositions().
     453             :    */
     454             :   void createApps();
     455             : 
     456             :   /**
     457             :    * Preserve the solution from the previous simulation,
     458             :    * and it is used as an initial guess for the next run
     459             :    */
     460             :   void keepSolutionDuringRestore(bool keep_solution_during_restore);
     461             : 
     462             :   /**
     463             :    * Set the output file base of the application which corresponds to the index passed to the
     464             :    * function.
     465             :    *
     466             :    * @param index The sub-application index
     467             :    */
     468             :   void setAppOutputFileBase(unsigned int index);
     469             : 
     470             :   /**
     471             :    * Helper for constructing the name of the multiapp
     472             :    *
     473             :    * @param base_name The base name of the multiapp, usually name()
     474             :    * @param index The index of the app
     475             :    * @param total The total number of apps, which is used to pad the name with zeros
     476             :    * @return std::string The name of the multiapp
     477             :    */
     478             :   static std::string
     479             :   getMultiAppName(const std::string & base_name, dof_id_type index, dof_id_type total);
     480             : 
     481             :   /// The FEProblemBase this MultiApp is part of
     482             :   FEProblemBase & _fe_problem;
     483             : 
     484             :   /// The type of application to build
     485             :   std::string _app_type;
     486             : 
     487             :   /// The positions of all of the apps, using input constant vectors (to be deprecated)
     488             :   std::vector<Point> _positions;
     489             :   /// The positions of all of the apps, using the Positions system
     490             :   std::vector<const Positions *> _positions_objs;
     491             :   /// The offsets, in case multiple Positions objects are specified
     492             :   std::vector<unsigned int> _positions_index_offsets;
     493             : 
     494             :   /// Toggle use of "positions". Subapps are created at each different position.
     495             :   /// List of positions can be created using the Positions system
     496             :   const bool _use_positions;
     497             : 
     498             :   /// The input file for each app's simulation
     499             :   std::vector<FileName> _input_files;
     500             : 
     501             :   /// Whether to create the first app on rank 0 while all other MPI ranks are idle
     502             :   const bool & _wait_for_first_app_init;
     503             : 
     504             :   /// Number of positions for each input file
     505             :   std::vector<unsigned int> _npositions_inputfile;
     506             : 
     507             :   /// The output file basename for each multiapp
     508             :   std::string _output_base;
     509             : 
     510             :   /// The total number of apps to simulate
     511             :   unsigned int _total_num_apps;
     512             : 
     513             :   /// The number of apps this object is involved in simulating
     514             :   unsigned int _my_num_apps;
     515             : 
     516             :   /// The number of the first app on this processor
     517             :   unsigned int _first_local_app;
     518             : 
     519             :   /// The original comm handle
     520             :   const MPI_Comm & _orig_comm;
     521             : 
     522             :   /// The communicator object that holds the MPI_Comm that we're going to use
     523             :   libMesh::Parallel::Communicator _my_communicator;
     524             : 
     525             :   /// The MPI communicator this object is going to use.
     526             :   MPI_Comm & _my_comm;
     527             : 
     528             :   /// The number of processors in the original comm
     529             :   int _orig_num_procs;
     530             : 
     531             :   /// The mpi "rank" of this processor in the original communicator
     532             :   int _orig_rank;
     533             : 
     534             :   /// Node Name
     535             :   std::string _node_name;
     536             : 
     537             :   /// The mpi "rank" of this processor in the sub communicator
     538             :   int _my_rank;
     539             : 
     540             :   /// Pointers to each of the Apps
     541             :   std::vector<std::shared_ptr<MooseApp>> _apps;
     542             : 
     543             :   /// Flag if this multi-app computed its bounding box (valid only for non-displaced meshes)
     544             :   std::vector<bool> _has_bounding_box;
     545             : 
     546             :   /// This multi-app's bounding box
     547             :   std::vector<libMesh::BoundingBox> _bounding_box;
     548             : 
     549             :   /// Relative bounding box inflation
     550             :   Real _inflation;
     551             : 
     552             :   /// Additional padding added to the bounding box, useful for 1D meshes
     553             :   Point _bounding_box_padding;
     554             : 
     555             :   /// Maximum number of processors to give to each app
     556             :   processor_id_type _max_procs_per_app;
     557             : 
     558             :   /// Minimum number of processors to give to each app
     559             :   processor_id_type _min_procs_per_app;
     560             : 
     561             :   /// Whether or not to move the output of the MultiApp into position
     562             :   bool _output_in_position;
     563             : 
     564             :   /// The offset time so the MultiApp local time relative to the global time
     565             :   const Real _global_time_offset;
     566             : 
     567             :   /// The times at which to reset apps
     568             :   std::vector<Real> _reset_times;
     569             : 
     570             :   /// The apps to be reset
     571             :   std::vector<unsigned int> _reset_apps;
     572             : 
     573             :   /// Whether or not apps have been reset at each time
     574             :   std::vector<bool> _reset_happened;
     575             : 
     576             :   /// The time at which to move apps
     577             :   Real _move_time;
     578             : 
     579             :   /// The apps to be moved
     580             :   std::vector<unsigned int> _move_apps;
     581             : 
     582             :   /// The new positions for the apps to be moved
     583             :   std::vector<Point> _move_positions;
     584             : 
     585             :   /// Whether or not the move has happened
     586             :   bool _move_happened;
     587             : 
     588             :   /// Whether or not this processor as an App _at all_
     589             :   bool _has_an_app;
     590             : 
     591             :   /// CommandLine arguments (controllable!)
     592             :   const std::vector<CLIArgString> & _cli_args;
     593             : 
     594             :   /// CommandLine arguments from files
     595             :   std::vector<std::string> _cli_args_from_file;
     596             : 
     597             :   /// Flag indicates if or not restart from the latest solution
     598             :   bool _keep_solution_during_restore;
     599             : 
     600             :   /// Flag indicates if or not restart the auxiliary system from the latest auxiliary solution
     601             :   bool _keep_aux_solution_during_restore;
     602             : 
     603             :   /// Whether or not to skip restoring completely
     604             :   const bool _no_restore;
     605             : 
     606             :   /// The solution from the end of the previous solve, this is cloned from the Nonlinear solution during restore
     607             :   std::vector<std::unique_ptr<libMesh::NumericVector<Real>>> _end_solutions;
     608             : 
     609             :   /// The auxiliary solution from the end of the previous solve, this is cloned from the auxiliary solution during restore
     610             :   std::vector<std::unique_ptr<NumericVector<Real>>> _end_aux_solutions;
     611             : 
     612             :   /// The app configuration resulting from calling init
     613             :   LocalRankConfig _rank_config;
     614             : 
     615             :   /// Transfers associated with this multiapp
     616             :   std::vector<MultiAppTransfer *> _associated_transfers;
     617             : 
     618             :   /// Whether to run the child apps with their meshes transformed with the coordinate transforms
     619             :   const bool _run_in_position;
     620             : 
     621             :   /// The cached subapp backups (passed from the parent app)
     622             :   SubAppBackups & _sub_app_backups;
     623             : 
     624             :   /// Timers
     625             :   const PerfID _solve_step_timer;
     626             :   const PerfID _init_timer;
     627             :   const PerfID _backup_timer;
     628             :   const PerfID _restore_timer;
     629             :   const PerfID _reset_timer;
     630             : 
     631             : private:
     632             :   /// The parameter that was used to set the command line args, if any
     633             :   mutable std::optional<std::string> _cli_args_param;
     634             : };
     635             : 
     636             : void dataStore(std::ostream & stream, SubAppBackups & backups, void * context);
     637             : void dataLoad(std::istream & stream, SubAppBackups & backups, void * context);