https://mooseframework.inl.gov
MultiApp.h
Go to the documentation of this file.
1 //* This file is part of the MOOSE framework
2 //* https://mooseframework.inl.gov
3 //*
4 //* All rights reserved, see COPYRIGHT for full restrictions
5 //* https://github.com/idaholab/moose/blob/master/COPYRIGHT
6 //*
7 //* Licensed under LGPL 2.1, please see LICENSE for details
8 //* https://www.gnu.org/licenses/lgpl-2.1.html
9 
10 #pragma once
11 
12 #include "MooseObject.h"
13 #include "SetupInterface.h"
14 #include "Restartable.h"
15 #include "PerfGraphInterface.h"
16 #include "Backup.h"
17 
18 #include "libmesh/communicator.h"
19 #include "libmesh/point.h"
20 
21 #include "nlohmann/json.h"
22 
23 class UserObject;
24 class FEProblemBase;
25 class FEProblem;
26 class Executioner;
27 class MooseApp;
28 class MultiAppTransfer;
30 class Positions;
31 
32 // libMesh forward declarations
33 namespace libMesh
34 {
35 class BoundingBox;
36 namespace MeshTools
37 {
38 class BoundingBox;
39 }
40 template <typename T>
41 class NumericVector;
42 } // namespace libMesh
43 
47 {
75 };
76 
89  processor_id_type nprocs,
90  dof_id_type napps,
91  processor_id_type min_app_procs,
92  processor_id_type max_app_procs,
93  bool batch_mode = false);
94 
100 class SubAppBackups : public std::vector<std::unique_ptr<Backup>>
101 {
102 };
103 
112 class MultiApp : public MooseObject,
113  public SetupInterface,
114  public Restartable,
115  public PerfGraphInterface
116 {
117 public:
118  static InputParameters validParams();
119 
121 
122  virtual void preExecute() {}
123 
127  virtual void finalize();
128 
132  virtual void postExecute();
133 
138  void setupPositions();
139 
144  virtual void createLocalApp(const unsigned int i);
145 
149  virtual void initialSetup() override;
150 
155  virtual void preTransfer(Real dt, Real target_time);
156 
170  virtual bool solveStep(Real dt, Real target_time, bool auto_advance = true) = 0;
171 
178  virtual void incrementTStep(Real /*target_time*/) {}
179 
188  virtual void finishStep(bool /*recurse_through_multiapp_levels*/ = false) {}
189 
195  virtual void backup();
196 
204  virtual void restore(bool force = true);
205 
210  bool needsRestoration() { return !_no_restore; }
211 
216  virtual Executioner * getExecutioner(unsigned int app);
217 
229  virtual libMesh::BoundingBox
230  getBoundingBox(unsigned int app,
231  bool displaced_mesh,
232  const MultiAppCoordTransform * coord_transform = nullptr);
233 
238 
243  FEProblemBase & appProblemBase(unsigned int app);
244 
249  FEProblem & appProblem(unsigned int app);
250 
256  const UserObject & appUserObjectBase(unsigned int app, const std::string & name);
257 
263  Real appPostprocessorValue(unsigned int app, const std::string & name);
264 
273  std::string var_name);
274 
278  unsigned int numGlobalApps() const { return _total_num_apps; }
279 
283  unsigned int numLocalApps() { return _apps.size(); }
284 
288  unsigned int firstLocalApp() { return _first_local_app; }
289 
293  bool isFirstLocalRank() const;
294 
298  bool hasApp() { return _has_an_app; }
299 
305  bool hasLocalApp(unsigned int global_app) const;
306 
311  MooseApp * localApp(unsigned int local_app);
312 
318  const Point & position(unsigned int app) const;
319 
332  virtual void resetApp(unsigned int global_app, Real time = 0.0);
333 
340  virtual void moveApp(unsigned int global_app, Point p);
341 
346  virtual void parentOutputPositionChanged();
347 
352  MPI_Comm & comm() { return _my_comm; }
353 
358  bool isRootProcessor() { return _my_rank == 0; }
359 
363  bool usingPositions() const { return _use_positions; }
364 
369  bool runningInPosition() const { return _run_in_position; }
370 
374  void addAssociatedTransfer(MultiAppTransfer & transfer);
375 
380  static void transformBoundingBox(libMesh::BoundingBox & box,
381  const MultiAppCoordTransform & transform);
382 
386  void setAppOutputFileBase();
387 
388 protected:
390  virtual std::vector<std::string> cliArgs() const;
391 
395  virtual void fillPositions();
396 
401 
408  void createApp(unsigned int i, Real start_time);
409 
415  void buildComm();
416 
424  unsigned int globalAppToLocal(unsigned int global_app);
425 
427  virtual void preRunInputFile();
428 
437  virtual std::vector<std::string> getCommandLineArgs(const unsigned int local_app);
438 
442  void init(unsigned int num_apps, bool batch_mode = false);
443 
447  void init(unsigned int num_apps, const LocalRankConfig & config);
448 
454  void createApps();
455 
460  void keepSolutionDuringRestore(bool keep_solution_during_restore);
461 
468  void setAppOutputFileBase(unsigned int index);
469 
478  static std::string
479  getMultiAppName(const std::string & base_name, dof_id_type index, dof_id_type total);
480 
483 
485  std::string _app_type;
486 
488  std::vector<Point> _positions;
490  std::vector<const Positions *> _positions_objs;
492  std::vector<unsigned int> _positions_index_offsets;
493 
496  const bool _use_positions;
497 
499  std::vector<FileName> _input_files;
500 
503 
505  std::vector<unsigned int> _npositions_inputfile;
506 
508  std::string _output_base;
509 
511  unsigned int _total_num_apps;
512 
514  unsigned int _my_num_apps;
515 
517  unsigned int _first_local_app;
518 
520  const MPI_Comm & _orig_comm;
521 
524 
526  MPI_Comm & _my_comm;
527 
530 
533 
535  std::string _node_name;
536 
538  int _my_rank;
539 
541  std::vector<std::shared_ptr<MooseApp>> _apps;
542 
544  std::vector<bool> _has_bounding_box;
545 
547  std::vector<libMesh::BoundingBox> _bounding_box;
548 
551 
554 
557 
560 
563 
566 
568  std::vector<Real> _reset_times;
569 
571  std::vector<unsigned int> _reset_apps;
572 
574  std::vector<bool> _reset_happened;
575 
578 
580  std::vector<unsigned int> _move_apps;
581 
583  std::vector<Point> _move_positions;
584 
587 
590 
592  const std::vector<CLIArgString> & _cli_args;
593 
595  std::vector<std::string> _cli_args_from_file;
596 
599 
602 
604  const bool _no_restore;
605 
607  std::vector<std::unique_ptr<libMesh::NumericVector<Real>>> _end_solutions;
608 
610  std::vector<std::unique_ptr<NumericVector<Real>>> _end_aux_solutions;
611 
614 
616  std::vector<MultiAppTransfer *> _associated_transfers;
617 
619  const bool _run_in_position;
620 
623 
630 
631 private:
633  mutable std::optional<std::string> _cli_args_param;
634 };
635 
636 void dataStore(std::ostream & stream, SubAppBackups & backups, void * context);
637 void dataLoad(std::istream & stream, SubAppBackups & backups, void * context);
void keepSolutionDuringRestore(bool keep_solution_during_restore)
Preserve the solution from the previous simulation, and it is used as an initial guess for the next r...
Definition: MultiApp.C:861
bool hasLocalApp(unsigned int global_app) const
Whether or not the given global app number is on this processor.
Definition: MultiApp.C:1039
std::vector< unsigned int > _reset_apps
The apps to be reset.
Definition: MultiApp.h:571
bool needsRestoration()
Whether or not this MultiApp should be restored at the beginning of each Picard iteration.
Definition: MultiApp.h:210
A class for creating restricted objects.
Definition: Restartable.h:28
MultiApp(const InputParameters &parameters)
Definition: MultiApp.C:273
std::vector< libMesh::BoundingBox > _bounding_box
This multi-app&#39;s bounding box.
Definition: MultiApp.h:547
std::vector< const Positions * > _positions_objs
The positions of all of the apps, using the Positions system.
Definition: MultiApp.h:490
virtual void restore(bool force=true)
Restore the state of every Sub App.
Definition: MultiApp.C:762
virtual void backup()
Save off the state of every Sub App.
Definition: MultiApp.C:747
const bool _use_positions
Toggle use of "positions".
Definition: MultiApp.h:496
Specialization of SubProblem for solving nonlinear equations plus auxiliary equations.
Definition: FEProblem.h:20
const std::vector< CLIArgString > & _cli_args
CommandLine arguments (controllable!)
Definition: MultiApp.h:592
std::vector< unsigned int > _npositions_inputfile
Number of positions for each input file.
Definition: MultiApp.h:505
void setupPositions()
Called just after construction to allow derived classes to set _positions and create sub-apps accordi...
Definition: MultiApp.C:377
std::vector< unsigned int > _positions_index_offsets
The offsets, in case multiple Positions objects are specified.
Definition: MultiApp.h:492
Real _move_time
The time at which to move apps.
Definition: MultiApp.h:577
LocalRankConfig rankConfig(processor_id_type rank, processor_id_type nprocs, dof_id_type napps, processor_id_type min_app_procs, processor_id_type max_app_procs, bool batch_mode=false)
Returns app partitioning information relevant to the given rank for a multiapp scenario with the give...
Definition: MultiApp.C:1292
unsigned int numLocalApps()
Definition: MultiApp.h:283
Real appPostprocessorValue(unsigned int app, const std::string &name)
Get a Postprocessor value for a specified global app.
Definition: MultiApp.C:1018
int _orig_rank
The mpi "rank" of this processor in the original communicator.
Definition: MultiApp.h:532
LocalRankConfig _rank_config
The app configuration resulting from calling init.
Definition: MultiApp.h:613
const PerfID _solve_step_timer
Timers.
Definition: MultiApp.h:625
std::optional< std::string > _cli_args_param
The parameter that was used to set the command line args, if any.
Definition: MultiApp.h:633
std::vector< std::shared_ptr< MooseApp > > _apps
Pointers to each of the Apps.
Definition: MultiApp.h:541
Positions objects are under the hood Reporters.
Definition: Positions.h:20
dof_id_type first_local_app_index
The (global) index of the first local app for this rank.
Definition: MultiApp.h:64
virtual void parentOutputPositionChanged()
For apps outputting in position we need to change their output positions if their parent app moves...
Definition: MultiApp.C:1096
std::vector< Point > _move_positions
The new positions for the apps to be moved.
Definition: MultiApp.h:583
Base class for MOOSE-based applications.
Definition: MooseApp.h:85
The main MOOSE class responsible for handling user-defined parameters in almost every MOOSE system...
const PerfID _restore_timer
Definition: MultiApp.h:628
The following methods are specializations for using the libMesh::Parallel::packed_range_* routines fo...
Real _inflation
Relative bounding box inflation.
Definition: MultiApp.h:550
FEProblemBase & _fe_problem
The FEProblemBase this MultiApp is part of.
Definition: MultiApp.h:482
virtual std::vector< std::string > getCommandLineArgs(const unsigned int local_app)
Definition: MultiApp.C:1239
Specialization of SubProblem for solving nonlinear equations plus auxiliary equations.
processor_id_type _min_procs_per_app
Minimum number of processors to give to each app.
Definition: MultiApp.h:559
virtual const std::string & name() const
Get the name of the class.
Definition: MooseBase.h:57
int _orig_num_procs
The number of processors in the original comm.
Definition: MultiApp.h:529
const PerfID _backup_timer
Definition: MultiApp.h:627
unsigned int PerfID
Definition: MooseTypes.h:212
virtual void finishStep(bool=false)
Calls multi-apps executioners&#39; endStep and postStep methods which creates output and advances time (n...
Definition: MultiApp.h:188
std::vector< Real > _reset_times
The times at which to reset apps.
Definition: MultiApp.h:568
virtual void preTransfer(Real dt, Real target_time)
Gets called just before transfers are done to the MultiApp (Which is just before the MultiApp is solv...
Definition: MultiApp.C:657
void createApps()
Create the provided number of apps.
Definition: MultiApp.C:388
virtual libMesh::BoundingBox getBoundingBox(unsigned int app, bool displaced_mesh, const MultiAppCoordTransform *coord_transform=nullptr)
Get the BoundingBox for the mesh associated with app The bounding box will be shifted to be in the co...
Definition: MultiApp.C:911
uint8_t processor_id_type
const bool & _wait_for_first_app_init
Whether to create the first app on rank 0 while all other MPI ranks are idle.
Definition: MultiApp.h:502
static void transformBoundingBox(libMesh::BoundingBox &box, const MultiAppCoordTransform &transform)
Transform a bounding box according to the transformations in the provided coordinate transformation o...
Definition: MultiApp.C:871
virtual libMesh::NumericVector< libMesh::Number > & appTransferVector(unsigned int app, std::string var_name)
Get the vector to transfer to for this MultiApp.
Definition: MultiApp.C:1027
unsigned int numGlobalApps() const
Definition: MultiApp.h:278
bool _keep_solution_during_restore
Flag indicates if or not restart from the latest solution.
Definition: MultiApp.h:598
Every object that can be built by the factory should be derived from this class.
Definition: MooseObject.h:28
processor_id_type _max_procs_per_app
Maximum number of processors to give to each app.
Definition: MultiApp.h:556
virtual std::vector< std::string > cliArgs() const
function that provides cli_args to subapps
Definition: MultiApp.C:1449
virtual void fillPositions()
must fill in _positions with the positions of the sub-aps
Definition: MultiApp.C:544
bool _move_happened
Whether or not the move has happened.
Definition: MultiApp.h:586
std::vector< MultiAppTransfer * > _associated_transfers
Transfers associated with this multiapp.
Definition: MultiApp.h:616
Helper class for holding Sub-app backups.
Definition: MultiApp.h:100
bool hasApp()
Whether or not this MultiApp has an app on this processor.
Definition: MultiApp.h:298
const PerfID _reset_timer
Definition: MultiApp.h:629
bool runningInPosition() const
Whether or not this MultiApp is being run in position, eg with the coordinate transform already appli...
Definition: MultiApp.h:369
std::string _output_base
The output file basename for each multiapp.
Definition: MultiApp.h:508
std::vector< std::unique_ptr< NumericVector< Real > > > _end_aux_solutions
The auxiliary solution from the end of the previous solve, this is cloned from the auxiliary solution...
Definition: MultiApp.h:610
std::string _app_type
The type of application to build.
Definition: MultiApp.h:485
bool isRootProcessor()
Whether or not this processor is the "root" processor for the sub communicator.
Definition: MultiApp.h:358
FEProblemBase & appProblemBase(unsigned int app)
Get the FEProblemBase for the global app desired.
Definition: MultiApp.C:985
std::vector< unsigned int > _move_apps
The apps to be moved.
Definition: MultiApp.h:580
Executioners are objects that do the actual work of solving your problem.
Definition: Executioner.h:30
void buildComm()
Create an MPI communicator suitable for each app.
Definition: MultiApp.C:1370
virtual void resetApp(unsigned int global_app, Real time=0.0)
"Reset" the App corresponding to the global App number passed in.
Definition: MultiApp.C:1056
bool isFirstLocalRank() const
Definition: MultiApp.C:1033
Interface for objects interacting with the PerfGraph.
unsigned int _total_num_apps
The total number of apps to simulate.
Definition: MultiApp.h:511
dof_id_type num_local_sims
The number of simulations that should/will be run locally on this rank.
Definition: MultiApp.h:49
virtual bool solveStep(Real dt, Real target_time, bool auto_advance=true)=0
Re-solve all of the Apps.
unsigned int _my_num_apps
The number of apps this object is involved in simulating.
Definition: MultiApp.h:514
static std::string getMultiAppName(const std::string &base_name, dof_id_type index, dof_id_type total)
Helper for constructing the name of the multiapp.
Definition: MultiApp.C:1465
std::string _node_name
Node Name.
Definition: MultiApp.h:535
bool _has_an_app
Whether or not this processor as an App at all
Definition: MultiApp.h:589
std::vector< std::unique_ptr< libMesh::NumericVector< Real > > > _end_solutions
The solution from the end of the previous solve, this is cloned from the Nonlinear solution during re...
Definition: MultiApp.h:607
libMesh::Parallel::Communicator _my_communicator
The communicator object that holds the MPI_Comm that we&#39;re going to use.
Definition: MultiApp.h:523
void createApp(unsigned int i, Real start_time)
Helper function for creating an App instance.
Definition: MultiApp.C:1104
bool usingPositions() const
Whether or not this MultiApp is using positions or its own way for constructing sub-apps.
Definition: MultiApp.h:363
const PerfID _init_timer
Definition: MultiApp.h:626
unsigned int _first_local_app
The number of the first app on this processor.
Definition: MultiApp.h:517
void addAssociatedTransfer(MultiAppTransfer &transfer)
Add a transfer that is associated with this multiapp.
Definition: MultiApp.C:1436
void dataStore(std::ostream &stream, SubAppBackups &backups, void *context)
Definition: MultiApp.C:1485
virtual void preExecute()
Definition: MultiApp.h:122
const bool _no_restore
Whether or not to skip restoring completely.
Definition: MultiApp.h:604
unsigned int firstLocalApp()
Definition: MultiApp.h:288
DIE A HORRIBLE DEATH HERE typedef LIBMESH_DEFAULT_SCALAR_TYPE Real
MooseApp * localApp(unsigned int local_app)
Get the local MooseApp object.
Definition: MultiApp.C:1049
virtual void initialSetup() override
Method to be called in main-app initial setup for create sub-apps if using positions is false...
Definition: MultiApp.C:438
Base class for all MultiAppTransfer objects.
MPI_Comm & comm()
Get the MPI communicator this MultiApp is operating on.
Definition: MultiApp.h:352
bool is_first_local_rank
This is true if this rank is the primary/zero rank for a (sub)app slot.
Definition: MultiApp.h:71
FEProblem & appProblem(unsigned int app)
Get the FEProblem for the global app is part of.
Definition: MultiApp.C:996
SubAppBackups & _sub_app_backups
The cached subapp backups (passed from the parent app)
Definition: MultiApp.h:622
virtual void postExecute()
Method called at the end of the simulation (after finalize).
Definition: MultiApp.C:735
bool _keep_aux_solution_during_restore
Flag indicates if or not restart the auxiliary system from the latest auxiliary solution.
Definition: MultiApp.h:601
bool _output_in_position
Whether or not to move the output of the MultiApp into position.
Definition: MultiApp.h:562
void dataLoad(std::istream &stream, SubAppBackups &backups, void *context)
Definition: MultiApp.C:1496
std::vector< Point > _positions
The positions of all of the apps, using input constant vectors (to be deprecated) ...
Definition: MultiApp.h:488
const InputParameters & parameters() const
Get the parameters of the object.
virtual void finalize()
Method called towards the end of the simulation to execute on final.
Definition: MultiApp.C:722
void readCommandLineArguments()
Fill command line arguments for sub apps.
Definition: MultiApp.C:447
int _my_rank
The mpi "rank" of this processor in the sub communicator.
Definition: MultiApp.h:538
dof_id_type num_local_apps
The number of (sub)apps that should/will be run locally on this rank.
Definition: MultiApp.h:58
virtual void createLocalApp(const unsigned int i)
Create the i-th local app.
Definition: MultiApp.C:432
std::vector< FileName > _input_files
The input file for each app&#39;s simulation.
Definition: MultiApp.h:499
Holds app partitioning information relevant to the a particular rank for a multiapp scenario...
Definition: MultiApp.h:46
dof_id_type first_local_sim_index
The (global) index of the first local simulation for this rank.
Definition: MultiApp.h:53
virtual void preRunInputFile()
call back executed right before app->runInputFile()
Definition: MultiApp.C:1431
Point _bounding_box_padding
Additional padding added to the bounding box, useful for 1D meshes.
Definition: MultiApp.h:553
const Real _global_time_offset
The offset time so the MultiApp local time relative to the global time.
Definition: MultiApp.h:565
void init(unsigned int num_apps, bool batch_mode=false)
Build communicators and reserve backups.
Definition: MultiApp.C:340
const MPI_Comm & _orig_comm
The original comm handle.
Definition: MultiApp.h:520
virtual void incrementTStep(Real)
Advances the multi-apps time step which is important for dt selection.
Definition: MultiApp.h:178
static InputParameters validParams()
Definition: MultiApp.C:50
A MultiApp represents one or more MOOSE applications that are running simultaneously.
Definition: MultiApp.h:112
void setAppOutputFileBase()
Sets all the app&#39;s output file bases.
Definition: MultiApp.C:1442
processor_id_type my_first_rank
For every rank working on a subapp, we store the first rank on each process to make the communication...
Definition: MultiApp.h:74
FEProblemBase & problemBase()
Get the FEProblemBase this MultiApp is part of.
Definition: MultiApp.h:237
MPI_Comm & _my_comm
The MPI communicator this object is going to use.
Definition: MultiApp.h:526
const bool _run_in_position
Whether to run the child apps with their meshes transformed with the coordinate transforms.
Definition: MultiApp.h:619
virtual void moveApp(unsigned int global_app, Point p)
Move the global_app to Point p.
Definition: MultiApp.C:1077
virtual Executioner * getExecutioner(unsigned int app)
Definition: MultiApp.C:713
Base class for user-specific data.
Definition: UserObject.h:40
const Point & position(unsigned int app) const
The physical position of a global App number.
Definition: MultiApp.C:1474
std::vector< std::string > _cli_args_from_file
CommandLine arguments from files.
Definition: MultiApp.h:595
const UserObject & appUserObjectBase(unsigned int app, const std::string &name)
Get a UserObject base for a specific global app.
Definition: MultiApp.C:1009
uint8_t dof_id_type
unsigned int globalAppToLocal(unsigned int global_app)
Map a global App number to the local number.
Definition: MultiApp.C:1412
This class contains transformation information that only exists in a context in which there are multi...
std::vector< bool > _reset_happened
Whether or not apps have been reset at each time.
Definition: MultiApp.h:574
std::vector< bool > _has_bounding_box
Flag if this multi-app computed its bounding box (valid only for non-displaced meshes) ...
Definition: MultiApp.h:544