MultiApp.h

Go to the documentation of this file.

//* This file is part of the MOOSE framework
//* https://mooseframework.inl.gov
//*
//* All rights reserved, see COPYRIGHT for full restrictions
//* https://github.com/idaholab/moose/blob/master/COPYRIGHT
//*
//* Licensed under LGPL 2.1, please see LICENSE for details
//* https://www.gnu.org/licenses/lgpl-2.1.html

#pragma once

#include "MooseObject.h"
#include "SetupInterface.h"
#include "Restartable.h"
#include "PerfGraphInterface.h"
#include "Backup.h"

#include "libmesh/communicator.h"
#include "libmesh/point.h"

#include "nlohmann/json.h"

class UserObject;
class FEProblemBase;
class FEProblem;
class Executioner;
class MooseApp;
class MultiAppTransfer;
class MultiAppCoordTransform;
class Positions;

// libMesh forward declarations
namespace libMesh
{
class BoundingBox;
namespace MeshTools
{
class BoundingBox;
}
template <typename T>
class NumericVector;
} // namespace libMesh

struct LocalRankConfig
{
  dof_id_type num_local_sims;
  dof_id_type first_local_sim_index;
  dof_id_type num_local_apps;
  dof_id_type first_local_app_index;
  bool is_first_local_rank;
  processor_id_type my_first_rank;
};

LocalRankConfig rankConfig(processor_id_type rank,
                           processor_id_type nprocs,
                           dof_id_type napps,
                           processor_id_type min_app_procs,
                           processor_id_type max_app_procs,
                           bool batch_mode = false);

class SubAppBackups : public std::vector<std::unique_ptr<Backup>>
{
};

class MultiApp : public MooseObject,
                 public SetupInterface,
                 public Restartable,
                 public PerfGraphInterface
{
public:
  static InputParameters validParams();

  MultiApp(const InputParameters & parameters);

  virtual void preExecute() {}

  virtual void finalize();

  virtual void postExecute();

  void setupPositions();

  virtual void createLocalApp(const unsigned int i);

  virtual void initialSetup() override;

  virtual void preTransfer(Real dt, Real target_time);

  virtual bool solveStep(Real dt, Real target_time, bool auto_advance = true) = 0;

  virtual void incrementTStep(Real /*target_time*/) {}

  virtual void finishStep(bool /*recurse_through_multiapp_levels*/ = false) {}

  virtual void backup();

  virtual void restore(bool force = true);

  bool needsRestoration() { return !_no_restore; }

  virtual Executioner * getExecutioner(unsigned int app);

  virtual libMesh::BoundingBox
  getBoundingBox(unsigned int app,
                 bool displaced_mesh,
                 const MultiAppCoordTransform * coord_transform = nullptr);

  FEProblemBase & problemBase() { return _fe_problem; }

  FEProblemBase & appProblemBase(unsigned int app);

  FEProblem & appProblem(unsigned int app);

  const UserObject & appUserObjectBase(unsigned int app, const std::string & name);

  Real appPostprocessorValue(unsigned int app, const std::string & name);

  virtual libMesh::NumericVector<libMesh::Number> & appTransferVector(unsigned int app,
                                                                      std::string var_name);

  unsigned int numGlobalApps() const { return _total_num_apps; }

  unsigned int numLocalApps() { return _apps.size(); }

  unsigned int firstLocalApp() { return _first_local_app; }

  bool isFirstLocalRank() const;

  bool hasApp() { return _has_an_app; }

  bool hasLocalApp(unsigned int global_app) const;

  MooseApp * localApp(unsigned int local_app);

  const Point & position(unsigned int app) const;

  virtual void resetApp(unsigned int global_app, Real time = 0.0);

  virtual void moveApp(unsigned int global_app, Point p);

  virtual void parentOutputPositionChanged();

  MPI_Comm & comm() { return _my_comm; }

  bool isRootProcessor() { return _my_rank == 0; }

  bool usingPositions() const { return _use_positions; }

  bool runningInPosition() const { return _run_in_position; }

  void addAssociatedTransfer(MultiAppTransfer & transfer);

  static void transformBoundingBox(libMesh::BoundingBox & box,
                                   const MultiAppCoordTransform & transform);

  void setAppOutputFileBase();

protected:
  virtual std::vector<std::string> cliArgs() const;

  virtual void fillPositions();

  void readCommandLineArguments();

  void createApp(unsigned int i, Real start_time);

  void buildComm();

  unsigned int globalAppToLocal(unsigned int global_app);

  virtual void preRunInputFile();

  virtual std::vector<std::string> getCommandLineArgs(const unsigned int local_app);

  void init(unsigned int num_apps, bool batch_mode = false);

  void init(unsigned int num_apps, const LocalRankConfig & config);

  void createApps();

  void keepSolutionDuringRestore(bool keep_solution_during_restore);

  void setAppOutputFileBase(unsigned int index);

  static std::string
  getMultiAppName(const std::string & base_name, dof_id_type index, dof_id_type total);

  FEProblemBase & _fe_problem;

  std::string _app_type;

  std::vector<Point> _positions;
  std::vector<const Positions *> _positions_objs;
  std::vector<unsigned int> _positions_index_offsets;

  const bool _use_positions;

  std::vector<FileName> _input_files;

  const bool & _wait_for_first_app_init;

  std::vector<unsigned int> _npositions_inputfile;

  std::string _output_base;

  unsigned int _total_num_apps;

  unsigned int _my_num_apps;

  unsigned int _first_local_app;

  const MPI_Comm & _orig_comm;

  libMesh::Parallel::Communicator _my_communicator;

  MPI_Comm & _my_comm;

  int _orig_num_procs;

  int _orig_rank;

  std::string _node_name;

  int _my_rank;

  std::vector<std::shared_ptr<MooseApp>> _apps;

  std::vector<bool> _has_bounding_box;

  std::vector<libMesh::BoundingBox> _bounding_box;

  Real _inflation;

  Point _bounding_box_padding;

  processor_id_type _max_procs_per_app;

  processor_id_type _min_procs_per_app;

  bool _output_in_position;

  const Real _global_time_offset;

  std::vector<Real> _reset_times;

  std::vector<unsigned int> _reset_apps;

  std::vector<bool> _reset_happened;

  Real _move_time;

  std::vector<unsigned int> _move_apps;

  std::vector<Point> _move_positions;

  bool _move_happened;

  bool _has_an_app;

  const std::vector<CLIArgString> & _cli_args;

  std::vector<std::string> _cli_args_from_file;

  bool _keep_solution_during_restore;

  bool _keep_aux_solution_during_restore;

  const bool _no_restore;

  std::vector<std::unique_ptr<libMesh::NumericVector<Real>>> _end_solutions;

  std::vector<std::unique_ptr<NumericVector<Real>>> _end_aux_solutions;

  LocalRankConfig _rank_config;

  std::vector<MultiAppTransfer *> _associated_transfers;

  const bool _run_in_position;

  SubAppBackups & _sub_app_backups;

  const PerfID _solve_step_timer;
  const PerfID _init_timer;
  const PerfID _backup_timer;
  const PerfID _restore_timer;
  const PerfID _reset_timer;

private:
  mutable std::optional<std::string> _cli_args_param;
};

void dataStore(std::ostream & stream, SubAppBackups & backups, void * context);
void dataLoad(std::istream & stream, SubAppBackups & backups, void * context);