MultiApp.h

Go to the documentation of this file.

//* This file is part of the MOOSE framework
//* https://mooseframework.inl.gov
//*
//* All rights reserved, see COPYRIGHT for full restrictions
//* https://github.com/idaholab/moose/blob/master/COPYRIGHT
//*
//* Licensed under LGPL 2.1, please see LICENSE for details
//* https://www.gnu.org/licenses/lgpl-2.1.html
 
#pragma once
 
#include "MooseObject.h"
#include "SetupInterface.h"
#include "Restartable.h"
#include "PerfGraphInterface.h"
#include "Backup.h"
 
#include "libmesh/communicator.h"
#include "libmesh/point.h"
 
#include "nlohmann/json.h"
 
class UserObject;
class FEProblemBase;
class FEProblem;
class Executioner;
class MooseApp;
class MultiAppTransfer;
class MultiAppCoordTransform;
class Positions;
 
// libMesh forward declarations
namespace libMesh
{
class BoundingBox;
namespace MeshTools
{
class BoundingBox;
}
template <typename T>
class NumericVector;
} // namespace libMesh
 
struct LocalRankConfig
{
  dof_id_type num_local_sims;
  dof_id_type first_local_sim_index;
  dof_id_type num_local_apps;
  dof_id_type first_local_app_index;
  bool is_first_local_rank;
  processor_id_type my_first_rank;
};
 
template <>
inline void
dataStore(std::ostream & stream, LocalRankConfig & v, void * context)
{
  dataStore(stream, v.num_local_sims, context);
  dataStore(stream, v.first_local_sim_index, context);
  dataStore(stream, v.num_local_apps, context);
  dataStore(stream, v.first_local_app_index, context);
  dataStore(stream, v.is_first_local_rank, context);
  dataStore(stream, v.my_first_rank, context);
}
 
template <>
inline void
dataLoad(std::istream & stream, LocalRankConfig & v, void * context)
{
  dataLoad(stream, v.num_local_sims, context);
  dataLoad(stream, v.first_local_sim_index, context);
  dataLoad(stream, v.num_local_apps, context);
  dataLoad(stream, v.first_local_app_index, context);
  dataLoad(stream, v.is_first_local_rank, context);
  dataLoad(stream, v.my_first_rank, context);
}
 
LocalRankConfig rankConfig(processor_id_type rank,
                           processor_id_type nprocs,
                           dof_id_type napps,
                           processor_id_type min_app_procs,
                           processor_id_type max_app_procs,
                           bool batch_mode = false);
 
class SubAppBackups : public std::vector<std::unique_ptr<Backup>>
{
};
 
class MultiApp : public MooseObject,
                 public SetupInterface,
                 public Restartable,
                 public PerfGraphInterface
{
public:
  static InputParameters validParams();
 
  MultiApp(const InputParameters & parameters);
 
  virtual void preExecute() {}
 
  virtual void finalize();
 
  virtual void postExecute();
 
  void possiblyCreateChildApplications();
 
  void init(unsigned int num_apps, const LocalRankConfig & config);
 
  virtual void createLocalApp(const unsigned int i);
 
  virtual void initialSetup() override;
 
  virtual void preTransfer(Real dt, Real target_time);
 
  virtual bool solveStep(Real dt, Real target_time, bool auto_advance = true) = 0;
 
  virtual void incrementTStep(Real /*target_time*/) {}
 
  virtual void finishStep(bool /*recurse_through_multiapp_levels*/ = false) {}
 
  virtual void backup();
 
  virtual void restore(bool force = true);
 
  bool needsRestoration() { return !_no_restore; }
 
  virtual Executioner * getExecutioner(unsigned int app);
 
  virtual libMesh::BoundingBox
  getBoundingBox(unsigned int app,
                 bool displaced_mesh,
                 const MultiAppCoordTransform * coord_transform = nullptr);
 
  FEProblemBase & problemBase() { return _fe_problem; }
 
  FEProblemBase & appProblemBase(unsigned int app);
 
  FEProblem & appProblem(unsigned int app);
 
  const UserObject & appUserObjectBase(unsigned int app, const std::string & name);
 
  Real appPostprocessorValue(unsigned int app, const std::string & name);
 
  virtual libMesh::NumericVector<libMesh::Number> & appTransferVector(unsigned int app,
                                                                      std::string var_name);
 
  unsigned int numGlobalApps() const { return _total_num_apps; }
 
  unsigned int numLocalApps() { return _apps.size(); }
 
  unsigned int firstLocalApp() { return _first_local_app; }
 
  bool isFirstLocalRank() const;
 
  bool hasApp() { return _has_an_app; }
 
  bool hasLocalApp(unsigned int global_app) const;
 
  MooseApp * localApp(unsigned int local_app);
 
  const Point & position(unsigned int app) const;
 
  virtual void resetApp(unsigned int global_app, Real time = 0.0);
 
  virtual void moveApp(unsigned int global_app, Point p);
 
  virtual void parentOutputPositionChanged();
 
  MPI_Comm & comm() { return _my_comm; }
 
  bool isRootProcessor() { return _my_rank == 0; }
 
  bool usingPositions() const { return _use_positions; }
 
  bool runningInPosition() const { return _run_in_position; }
 
  void addAssociatedTransfer(MultiAppTransfer & transfer);
 
  static void transformBoundingBox(libMesh::BoundingBox & box,
                                   const MultiAppCoordTransform & transform);
 
  void setAppOutputFileBase();
 
protected:
  virtual std::vector<std::string> cliArgs() const;
 
  virtual void fillPositions();
 
  void readCommandLineArguments();
 
  void createApp(unsigned int i, Real start_time);
 
  virtual bool propagateRecoverToSubApps() const { return true; }
 
  void buildComm();
 
  unsigned int globalAppToLocal(unsigned int global_app);
 
  virtual void preRunInputFile();
 
  virtual std::vector<std::string> getCommandLineArgs(const unsigned int local_app);
 
  void init(unsigned int num_apps, bool batch_mode = false);
 
  void createApps();
 
  void keepSolutionDuringRestore(bool keep_solution_during_restore);
 
  void setAppOutputFileBase(unsigned int index);
 
  static std::string
  getMultiAppName(const std::string & base_name, dof_id_type index, dof_id_type total);
 
  FEProblemBase & _fe_problem;
 
  std::string _app_type;
 
  std::vector<Point> _positions;
  std::vector<const Positions *> _positions_objs;
  std::vector<unsigned int> _positions_index_offsets;
 
  const bool _use_positions;
 
  const bool _create_child_apps_on_initial_setup;
 
  std::vector<FileName> _input_files;
 
  const bool & _wait_for_first_app_init;
 
  std::vector<unsigned int> _npositions_inputfile;
 
  std::string _output_base;
 
  unsigned int _total_num_apps;
 
  unsigned int _my_num_apps;
 
  unsigned int _first_local_app;
 
  const MPI_Comm & _orig_comm;
 
  libMesh::Parallel::Communicator _my_communicator;
 
  MPI_Comm & _my_comm;
 
  int _orig_num_procs;
 
  int _orig_rank;
 
  std::string _node_name;
 
  int _my_rank;
 
  std::vector<std::shared_ptr<MooseApp>> _apps;
 
  std::vector<bool> _has_bounding_box;
 
  std::vector<libMesh::BoundingBox> _bounding_box;
 
  Real _inflation;
 
  Point _bounding_box_padding;
 
  processor_id_type _max_procs_per_app;
 
  processor_id_type _min_procs_per_app;
 
  bool _output_in_position;
 
  const Real _global_time_offset;
 
  std::vector<Real> _reset_times;
 
  std::vector<unsigned int> _reset_apps;
 
  std::vector<bool> _reset_happened;
 
  Real _move_time;
 
  std::vector<unsigned int> _move_apps;
 
  std::vector<Point> _move_positions;
 
  bool _move_happened;
 
  bool _has_an_app;
 
  const std::vector<CLIArgString> & _cli_args;
 
  std::vector<std::string> _cli_args_from_file;
 
  bool _keep_solution_during_restore;
 
  bool _keep_aux_solution_during_restore;
 
  const bool _no_restore;
 
  std::vector<std::vector<std::unique_ptr<libMesh::NumericVector<Real>>>> _end_solutions;
 
  std::vector<std::unique_ptr<NumericVector<Real>>> _end_aux_solutions;
 
  LocalRankConfig _rank_config;
 
  std::vector<MultiAppTransfer *> _associated_transfers;
 
  const bool _run_in_position;
 
  SubAppBackups & _sub_app_backups;
 
  const PerfID _solve_step_timer;
  const PerfID _init_timer;
  const PerfID _backup_timer;
  const PerfID _restore_timer;
  const PerfID _reset_timer;
 
private:
  mutable std::optional<std::string> _cli_args_param;
};
 
void dataStore(std::ostream & stream, SubAppBackups & backups, void * context);
void dataLoad(std::istream & stream, SubAppBackups & backups, void * context);