www.mooseframework.org
MultiApp.C
Go to the documentation of this file.
1 //* This file is part of the MOOSE framework
2 //* https://www.mooseframework.org
3 //*
4 //* All rights reserved, see COPYRIGHT for full restrictions
5 //* https://github.com/idaholab/moose/blob/master/COPYRIGHT
6 //*
7 //* Licensed under LGPL 2.1, please see LICENSE for details
8 //* https://www.gnu.org/licenses/lgpl-2.1.html
9 
10 // MOOSE includes
11 #include "MultiApp.h"
12 
13 #include "AppFactory.h"
14 #include "AuxiliarySystem.h"
15 #include "DisplacedProblem.h"
16 #include "Console.h"
17 #include "Executioner.h"
18 #include "FEProblem.h"
19 #include "MooseMesh.h"
20 #include "MooseUtils.h"
21 #include "OutputWarehouse.h"
22 #include "RestartableDataIO.h"
23 #include "SetupInterface.h"
24 #include "UserObject.h"
25 #include "CommandLine.h"
26 #include "Conversion.h"
27 #include "NonlinearSystemBase.h"
28 
29 #include "libmesh/mesh_tools.h"
30 #include "libmesh/numeric_vector.h"
31 
32 // C++ includes
33 #include <fstream>
34 #include <iomanip>
35 #include <iterator>
36 #include <algorithm>
37 
38 // Call to "uname"
39 #include <sys/utsname.h>
40 
41 template <>
44 {
46  params += validParams<SetupInterface>();
47 
48  params.addParam<bool>("use_displaced_mesh",
49  false,
50  "Whether or not this object should use the "
51  "displaced mesh for computation. Note that "
52  "in the case this is true but no "
53  "displacements are provided in the Mesh block "
54  "the undisplaced mesh will still be used.");
55  params.addParamNamesToGroup("use_displaced_mesh", "Advanced");
56 
57  std::ostringstream app_types_strings;
59  for (; it != AppFactory::instance().registeredObjectsEnd(); ++it)
60  app_types_strings << it->first << " ";
61  MooseEnum app_types_options(app_types_strings.str(), "", true);
62 
63  params.addParam<MooseEnum>("app_type",
64  app_types_options,
65  "The type of application to build (applications not "
66  "registered can be loaded with dynamic libraries. Master "
67  "application type will be used if not provided.");
68  params.addParam<std::string>("library_path",
69  "",
70  "Path to search for dynamic libraries (please "
71  "avoid committing absolute paths in addition to "
72  "MOOSE_LIBRARY_PATH)");
73  params.addParam<std::string>(
74  "library_name",
75  "",
76  "The file name of the library (*.la file) that will be dynamically loaded.");
77  params.addParam<std::vector<Point>>(
78  "positions",
79  "The positions of the App locations. Each set of 3 values will represent a "
80  "Point. This and 'positions_file' cannot be both supplied. If this and "
81  "'positions_file' are not supplied, a single position (0,0,0) will be used");
82  params.addParam<std::vector<FileName>>("positions_file",
83  "A filename that should be looked in for positions. Each "
84  "set of 3 values in that file will represent a Point. "
85  "This and 'positions' cannot be both supplied");
86 
87  params.addRequiredParam<std::vector<FileName>>(
88  "input_files",
89  "The input file for each App. If this parameter only contains one input file "
90  "it will be used for all of the Apps. When using 'positions_from_file' it is "
91  "also admissable to provide one input_file per file.");
92  params.addParam<Real>("bounding_box_inflation",
93  0.01,
94  "Relative amount to 'inflate' the bounding box of this MultiApp.");
95  params.addParam<Point>("bounding_box_padding",
97  "Additional padding added to the dimensions of the bounding box. The "
98  "values are added to the x, y and z dimension respectively.");
99 
100  params.addPrivateParam<MPI_Comm>("_mpi_comm");
101 
102  // Set the default execution time
103  params.set<ExecFlagEnum>("execute_on", true) = EXEC_TIMESTEP_BEGIN;
104 
105  params.addParam<unsigned int>("max_procs_per_app",
106  std::numeric_limits<unsigned int>::max(),
107  "Maximum number of processors to give to each App in this "
108  "MultiApp. Useful for restricting small solves to just a few "
109  "procs so they don't get spread out");
110 
111  params.addParam<bool>(
112  "output_in_position",
113  false,
114  "If true this will cause the output from the MultiApp to be 'moved' by its position vector");
115 
116  params.addParam<Real>("global_time_offset",
117  0,
118  "The time offset relative to the master application for the purpose of "
119  "starting a subapp at different time from the master application. The "
120  "global time will be ahead by the offset specified here.");
121  params.addParam<Real>("reset_time",
122  std::numeric_limits<Real>::max(),
123  "The time at which to reset Apps given by the 'reset_apps' parameter. "
124  "Resetting an App means that it is destroyed and recreated, possibly "
125  "modeling the insertion of 'new' material for that app.");
126 
127  params.addParam<std::vector<unsigned int>>(
128  "reset_apps",
129  "The Apps that will be reset when 'reset_time' is hit. These are the App "
130  "'numbers' starting with 0 corresponding to the order of the App positions. "
131  "Resetting an App means that it is destroyed and recreated, possibly modeling "
132  "the insertion of 'new' material for that app.");
133 
134  params.addParam<Real>(
135  "move_time",
136  std::numeric_limits<Real>::max(),
137  "The time at which Apps designated by move_apps are moved to move_positions.");
138 
139  params.addParam<std::vector<unsigned int>>(
140  "move_apps",
141  "Apps, designated by their 'numbers' starting with 0 corresponding to the order "
142  "of the App positions, to be moved at move_time to move_positions");
143 
144  params.addParam<std::vector<Point>>("move_positions",
145  "The positions corresponding to each move_app.");
146 
147  params.addParam<std::vector<std::string>>(
148  "cli_args",
149  std::vector<std::string>(),
150  "Additional command line arguments to pass to the sub apps. If one set is provided the "
151  "arguments are applied to all, otherwise there must be a set for each sub app.");
152 
153  params.addRangeCheckedParam<Real>("relaxation_factor",
154  1.0,
155  "relaxation_factor>0 & relaxation_factor<2",
156  "Fraction of newly computed value to keep."
157  "Set between 0 and 2.");
158  params.addParam<std::vector<std::string>>("relaxed_variables",
159  std::vector<std::string>(),
160  "List of variables to relax during Picard Iteration");
161 
162  params.addPrivateParam<std::shared_ptr<CommandLine>>("_command_line");
163  params.addPrivateParam<bool>("use_positions", true);
164  params.declareControllable("enable");
165  params.declareControllable("cli_args", {EXEC_PRE_MULTIAPP_SETUP});
166  params.registerBase("MultiApp");
167 
168  return params;
169 }
170 
172  : MooseObject(parameters),
173  SetupInterface(this),
174  Restartable(this, "MultiApps"),
175  _fe_problem(*getCheckedPointerParam<FEProblemBase *>("_fe_problem_base")),
176  _app_type(isParamValid("app_type") ? std::string(getParam<MooseEnum>("app_type"))
177  : _fe_problem.getMooseApp().type()),
178  _use_positions(getParam<bool>("use_positions")),
179  _input_files(getParam<std::vector<FileName>>("input_files")),
180  _total_num_apps(0),
181  _my_num_apps(0),
182  _first_local_app(0),
183  _orig_comm(_communicator.get()),
184  _my_communicator(),
185  _my_comm(_my_communicator.get()),
186  _my_rank(0),
187  _inflation(getParam<Real>("bounding_box_inflation")),
188  _bounding_box_padding(getParam<Point>("bounding_box_padding")),
189  _max_procs_per_app(getParam<unsigned int>("max_procs_per_app")),
190  _output_in_position(getParam<bool>("output_in_position")),
191  _global_time_offset(getParam<Real>("global_time_offset")),
192  _reset_time(getParam<Real>("reset_time")),
193  _reset_apps(getParam<std::vector<unsigned int>>("reset_apps")),
194  _reset_happened(false),
195  _move_time(getParam<Real>("move_time")),
196  _move_apps(getParam<std::vector<unsigned int>>("move_apps")),
197  _move_positions(getParam<std::vector<Point>>("move_positions")),
198  _move_happened(false),
199  _has_an_app(true),
200  _backups(declareRestartableDataWithContext<SubAppBackups>("backups", this)),
201  _cli_args(getParam<std::vector<std::string>>("cli_args"))
202 {
203 }
204 
205 void
206 MultiApp::init(unsigned int num)
207 {
208  _total_num_apps = num;
209  buildComm();
210  _backups.reserve(_my_num_apps);
211  for (unsigned int i = 0; i < _my_num_apps; i++)
212  _backups.emplace_back(std::make_shared<Backup>());
213 
214  _has_bounding_box.resize(_my_num_apps, false);
215  _bounding_box.resize(_my_num_apps);
216 
217  if ((_cli_args.size() > 1) && (_total_num_apps != _cli_args.size()))
218  paramError("cli_args",
219  "The number of items supplied must be 1 or equal to the number of sub apps.");
220 }
221 
222 void
224 {
225  if (_use_positions)
226  {
227  fillPositions();
228  init(_positions.size());
229  }
230 }
231 
232 void
234 {
235  if (!_has_an_app)
236  return;
237 
239 
240  _apps.resize(_my_num_apps);
241 
242  // If the user provided an unregistered app type, see if we can load it dynamically
243  if (!AppFactory::instance().isRegistered(_app_type))
245  _app_type, getParam<std::string>("library_path"), getParam<std::string>("library_name"));
246 
247  for (unsigned int i = 0; i < _my_num_apps; i++)
249 }
250 
251 void
253 {
254  if (_move_apps.size() != _move_positions.size())
255  mooseError("The number of apps to move and the positions to move them to must be the same for "
256  "MultiApp ",
257  _name);
258 
259  if (isParamValid("positions") && isParamValid("positions_file"))
260  mooseError(
261  "Both 'positions' and 'positions_file' cannot be specified simultaneously in MultiApp ",
262  name());
263 
264  if (isParamValid("positions"))
265  {
266  _positions = getParam<std::vector<Point>>("positions");
267 
268  if (_positions.size() < _input_files.size())
269  mooseError("Not enough positions for the number of input files provided in MultiApp ",
270  name());
271  }
272  else if (isParamValid("positions_file"))
273  {
274  std::vector<FileName> positions_files = getParam<std::vector<FileName>>("positions_file");
275  std::vector<FileName> input_files = getParam<std::vector<FileName>>("input_files");
276 
277  if (input_files.size() != 1 && positions_files.size() != input_files.size())
278  mooseError("Number of input_files for MultiApp ",
279  name(),
280  " must either be only one or match the number of positions_file files");
281 
282  // Clear out the _input_files because we're going to rebuild it
283  if (input_files.size() != 1)
284  _input_files.clear();
285 
286  for (unsigned int p_file_it = 0; p_file_it < positions_files.size(); p_file_it++)
287  {
288  std::string positions_file = positions_files[p_file_it];
289 
290  std::vector<Real> positions_vec;
291 
292  // Read the file on the root processor then broadcast it
293  if (processor_id() == 0)
294  {
295  MooseUtils::checkFileReadable(positions_file);
296 
297  std::ifstream is(positions_file.c_str());
298  std::istream_iterator<Real> begin(is), end;
299  positions_vec.insert(positions_vec.begin(), begin, end);
300 
301  if (positions_vec.size() % LIBMESH_DIM != 0)
302  mooseError("Number of entries in 'positions_file' ",
303  positions_file,
304  " must be divisible by ",
305  LIBMESH_DIM,
306  " in MultiApp ",
307  name());
308  }
309 
310  // Bradcast the vector to all processors
311  std::size_t num_positions = positions_vec.size();
312  _communicator.broadcast(num_positions);
313  positions_vec.resize(num_positions);
314  _communicator.broadcast(positions_vec);
315 
316  for (unsigned int i = 0; i < positions_vec.size(); i += LIBMESH_DIM)
317  {
318  if (input_files.size() != 1)
319  _input_files.push_back(input_files[p_file_it]);
320 
321  Point position;
322 
323  // This is here so it will theoretically work with LIBMESH_DIM=1 or 2. That is completely
324  // untested!
325  for (unsigned int j = 0; j < LIBMESH_DIM; j++)
326  position(j) = positions_vec[i + j];
327 
328  _positions.push_back(position);
329  }
330  }
331  }
332  else
333  {
334  _positions = {Point()};
335 
336  if (_positions.size() < _input_files.size())
337  mooseError("Not enough positions for the number of input files provided in MultiApp ",
338  name());
339  }
340 
341  mooseAssert(_input_files.size() == 1 || _positions.size() == _input_files.size(),
342  "Number of positions and input files are not the same!");
343 }
344 
345 void
346 MultiApp::preTransfer(Real /*dt*/, Real target_time)
347 {
348  // First, see if any Apps need to be Reset
349  if (!_reset_happened && target_time + 1e-14 >= _reset_time)
350  {
351  _reset_happened = true;
352  for (auto & app : _reset_apps)
353  resetApp(app);
354  }
355 
356  // Now move any apps that should be moved
357  if (_use_positions && !_move_happened && target_time + 1e-14 >= _move_time)
358  {
359  _move_happened = true;
360  for (unsigned int i = 0; i < _move_apps.size(); i++)
362  }
363 }
364 
365 Executioner *
366 MultiApp::getExecutioner(unsigned int app)
367 {
368  if (!_has_an_app)
369  mooseError("No app for ", name(), " on processor ", _orig_rank);
370 
371  return _apps[globalAppToLocal(app)]->getExecutioner();
372 }
373 
374 void
376 {
377  for (const auto & app_ptr : _apps)
378  {
379  auto * executioner = app_ptr->getExecutioner();
380  mooseAssert(executioner, "Executioner is nullptr");
381 
382  executioner->feProblem().execute(EXEC_FINAL);
383  executioner->feProblem().outputStep(EXEC_FINAL);
384  }
385 }
386 
387 void
389 {
390  for (const auto & app_ptr : _apps)
391  {
392  auto * executioner = app_ptr->getExecutioner();
393  mooseAssert(executioner, "Executioner is nullptr");
394 
395  executioner->postExecute();
396  }
397 }
398 
399 void
401 {
402  _console << "Beginning backing up MultiApp " << name() << std::endl;
403  for (unsigned int i = 0; i < _my_num_apps; i++)
404  _backups[i] = _apps[i]->backup();
405  _console << "Finished backing up MultiApp " << name() << std::endl;
406 }
407 
408 void
410 {
411  // Must be restarting / recovering so hold off on restoring
412  // Instead - the restore will happen in createApp()
413  // Note that _backups was already populated by dataLoad()
414  if (_apps.empty())
415  return;
416 
417  _console << "Begining restoring MultiApp " << name() << std::endl;
418  for (unsigned int i = 0; i < _my_num_apps; i++)
419  _apps[i]->restore(_backups[i]);
420  _console << "Finished restoring MultiApp " << name() << std::endl;
421 }
422 
423 BoundingBox
424 MultiApp::getBoundingBox(unsigned int app, bool displaced_mesh)
425 {
426  if (!_has_an_app)
427  mooseError("No app for ", name(), " on processor ", _orig_rank);
428 
429  unsigned int local_app = globalAppToLocal(app);
430  FEProblemBase & fe_problem_base = _apps[local_app]->getExecutioner()->feProblem();
431  MooseMesh & mesh = (displaced_mesh && fe_problem_base.getDisplacedProblem().get() != NULL)
432  ? fe_problem_base.getDisplacedProblem()->mesh()
433  : fe_problem_base.mesh();
434 
435  {
437  if (displaced_mesh)
438  _bounding_box[local_app] = MeshTools::create_bounding_box(mesh);
439  else
440  {
441  if (!_has_bounding_box[local_app])
442  {
443  _bounding_box[local_app] = MeshTools::create_bounding_box(mesh);
444  _has_bounding_box[local_app] = true;
445  }
446  }
447  }
448  BoundingBox bbox = _bounding_box[local_app];
449 
450  Point min = bbox.min();
451  min -= _bounding_box_padding;
452  Point max = bbox.max();
453  max += _bounding_box_padding;
454 
455  Point inflation_amount = (max - min) * _inflation;
456 
457  Point inflated_min = min - inflation_amount;
458  Point inflated_max = max + inflation_amount;
459 
460  // This is where the app is located. We need to shift by this amount.
461  Point p = position(app);
462 
463  Point shifted_min = inflated_min;
464  Point shifted_max = inflated_max;
465 
466  // If the problem is RZ then we're going to invent a box that would cover the whole "3D" app
467  // FIXME: Assuming all subdomains are the same coordinate system type!
468  if (fe_problem_base.getCoordSystem(*(mesh.meshSubdomains().begin())) == Moose::COORD_RZ)
469  {
470  shifted_min(0) = -inflated_max(0);
471  shifted_min(1) = inflated_min(1);
472  shifted_min(2) = -inflated_max(0);
473 
474  shifted_max(0) = inflated_max(0);
475  shifted_max(1) = inflated_max(1);
476  shifted_max(2) = inflated_max(0);
477  }
478 
479  // Shift them to the position they're supposed to be
480  shifted_min += p;
481  shifted_max += p;
482 
483  return BoundingBox(shifted_min, shifted_max);
484 }
485 
487 MultiApp::appProblemBase(unsigned int app)
488 {
489  if (!_has_an_app)
490  mooseError("No app for ", name(), " on processor ", _orig_rank);
491 
492  unsigned int local_app = globalAppToLocal(app);
493 
494  return _apps[local_app]->getExecutioner()->feProblem();
495 }
496 
497 FEProblem &
498 MultiApp::appProblem(unsigned int app)
499 {
501  "MultiApp::appProblem() is deprecated, call MultiApp::appProblemBase() instead.\n");
502  if (!_has_an_app)
503  mooseError("No app for ", name(), " on processor ", _orig_rank);
504 
505  unsigned int local_app = globalAppToLocal(app);
506 
507  return dynamic_cast<FEProblem &>(_apps[local_app]->getExecutioner()->feProblem());
508 }
509 
510 const UserObject &
511 MultiApp::appUserObjectBase(unsigned int app, const std::string & name)
512 {
513  if (!_has_an_app)
514  mooseError("No app for ", MultiApp::name(), " on processor ", _orig_rank);
515 
517 }
518 
519 Real
520 MultiApp::appPostprocessorValue(unsigned int app, const std::string & name)
521 {
522  if (!_has_an_app)
523  mooseError("No app for ", MultiApp::name(), " on processor ", _orig_rank);
524 
526 }
527 
528 NumericVector<Number> &
529 MultiApp::appTransferVector(unsigned int app, std::string /*var_name*/)
530 {
532 }
533 
534 bool
535 MultiApp::hasLocalApp(unsigned int global_app)
536 {
537  if (_has_an_app && global_app >= _first_local_app &&
538  global_app <= _first_local_app + (_my_num_apps - 1))
539  return true;
540 
541  return false;
542 }
543 
544 MooseApp *
545 MultiApp::localApp(unsigned int local_app)
546 {
547  mooseAssert(local_app < _apps.size(), "Index out of range: " + Moose::stringify(local_app));
548  return _apps[local_app].get();
549 }
550 
551 void
552 MultiApp::resetApp(unsigned int global_app, Real time)
553 {
555 
556  if (hasLocalApp(global_app))
557  {
558  unsigned int local_app = globalAppToLocal(global_app);
559 
560  // Extract the file numbers from the output, so that the numbering is maintained after reset
561  std::map<std::string, unsigned int> m = _apps[local_app]->getOutputWarehouse().getFileNumbers();
562 
563  createApp(local_app, time);
564 
565  // Reset the file numbers of the newly reset apps
566  _apps[local_app]->getOutputWarehouse().setFileNumbers(m);
567  }
568 }
569 
570 void
571 MultiApp::moveApp(unsigned int global_app, Point p)
572 {
573  if (_use_positions)
574  {
575  _positions[global_app] = p;
576 
577  if (hasLocalApp(global_app))
578  {
579  unsigned int local_app = globalAppToLocal(global_app);
580 
582  _apps[local_app]->setOutputPosition(p);
583  }
584  }
585 }
586 
587 void
589 {
591  for (unsigned int i = 0; i < _apps.size(); i++)
592  _apps[i]->setOutputPosition(_app.getOutputPosition() + _positions[_first_local_app + i]);
593 }
594 
595 void
596 MultiApp::createApp(unsigned int i, Real start_time)
597 {
598  // Define the app name
599  std::ostringstream multiapp_name;
600  std::string full_name;
601  multiapp_name << name() << std::setw(std::ceil(std::log10(_total_num_apps)))
602  << std::setprecision(0) << std::setfill('0') << std::right << _first_local_app + i;
603 
604  // Only add parent name if it the parent is not the main app
605  if (_app.multiAppLevel() > 0)
606  full_name = _app.name() + "_" + multiapp_name.str();
607  else
608  full_name = multiapp_name.str();
609 
611  app_params.set<FEProblemBase *>("_parent_fep") = &_fe_problem;
612  app_params.set<std::shared_ptr<CommandLine>>("_command_line") = _app.commandLine();
613 
614  if (_cli_args.size() > 0)
615  {
616  for (const std::string & str : MooseUtils::split(getCommandLineArgsParamHelper(i), ";"))
617  {
618  std::ostringstream oss;
619  oss << full_name << ":" << str;
620  app_params.get<std::shared_ptr<CommandLine>>("_command_line")->addArgument(oss.str());
621  }
622  }
623 
624  _console << COLOR_CYAN << "Creating MultiApp " << name() << " of type " << _app_type
625  << " of level " << _app.multiAppLevel() + 1 << " and number " << _first_local_app + i
626  << ":" << COLOR_DEFAULT << std::endl;
627  app_params.set<unsigned int>("_multiapp_level") = _app.multiAppLevel() + 1;
628  app_params.set<unsigned int>("_multiapp_number") = _first_local_app + i;
629  _apps[i] = AppFactory::instance().createShared(_app_type, full_name, app_params, _my_comm);
630  auto & app = _apps[i];
631 
632  std::string input_file = "";
633  if (_input_files.size() == 1) // If only one input file was provided, use it for all the solves
634  input_file = _input_files[0];
635  else
636  input_file = _input_files[_first_local_app + i];
637 
638  std::ostringstream output_base;
639 
640  // Create an output base by taking the output base of the master problem and appending
641  // the name of the multiapp + a number to it
642  if (!_app.getOutputFileBase().empty())
643  output_base << _app.getOutputFileBase() + "_";
644  else
645  {
646  std::string base = _app.getFileName();
647  size_t pos = base.find_last_of('.');
648  output_base << base.substr(0, pos) + "_out_";
649  }
650 
651  // Append the sub app name to the output file base
652  output_base << multiapp_name.str();
653  app->setGlobalTimeOffset(start_time);
654  app->setInputFileName(input_file);
655  app->setOutputFileBase(output_base.str());
656  app->setOutputFileNumbers(_app.getOutputWarehouse().getFileNumbers());
657  app->setRestart(_app.isRestarting());
658  app->setRecover(_app.isRecovering());
659 
660  // This means we have a backup of this app that we need to give to it
661  // Note: This won't do the restoration immediately. The Backup
662  // will be cached by the MooseApp object so that it can be used
663  // during FEProblemBase::initialSetup() during runInputFile()
664  if (_app.isRestarting() || _app.isRecovering())
665  app->setBackupObject(_backups[i]);
666 
667  if (_use_positions && getParam<bool>("output_in_position"))
668  app->setOutputPosition(_app.getOutputPosition() + _positions[_first_local_app + i]);
669 
670  // Update the MultiApp level for the app that was just created
671  app->setupOptions();
672  preRunInputFile();
673  app->runInputFile();
674 
675  auto & picard_solve = _apps[i]->getExecutioner()->picardSolve();
676  picard_solve.setMultiAppRelaxationFactor(getParam<Real>("relaxation_factor"));
677  picard_solve.setMultiAppRelaxationVariables(
678  getParam<std::vector<std::string>>("relaxed_variables"));
679  if (getParam<Real>("relaxation_factor") != 1.0)
680  {
681  // Store a copy of the previous solution here
682  FEProblemBase & fe_problem_base = _apps[i]->getExecutioner()->feProblem();
683  fe_problem_base.getNonlinearSystemBase().addVector("self_relax_previous", false, PARALLEL);
684  }
685 }
686 
687 std::string
689 {
690 
691  // Single set of "cli_args" to be applied to all sub apps
692  if (_cli_args.size() == 1)
693  return _cli_args[0];
694 
695  // Unique set of "cli_args" to be applied to each sub apps
696  else
697  return _cli_args[local_app + _first_local_app];
698 }
699 
700 void
702 {
703  int ierr;
704 
705  ierr = MPI_Comm_size(_communicator.get(), &_orig_num_procs);
706  mooseCheckMPIErr(ierr);
707  ierr = MPI_Comm_rank(_communicator.get(), &_orig_rank);
708  mooseCheckMPIErr(ierr);
709 
710  struct utsname sysInfo;
711  uname(&sysInfo);
712 
713  _node_name = sysInfo.nodename;
714 
715  // If we have more apps than processors then we're just going to divide up the work
716  if (_total_num_apps >= (unsigned)_orig_num_procs)
717  {
718  _my_comm = MPI_COMM_SELF;
719  _my_rank = 0;
720 
722  unsigned int jobs_left = _total_num_apps - (_my_num_apps * _orig_num_procs);
723 
724  if (jobs_left != 0)
725  {
726  // Spread the remaining jobs out over the first set of processors
727  if ((unsigned)_orig_rank < jobs_left) // (these are the "jobs_left_pids" ie the pids that are
728  // snatching up extra jobs)
729  {
730  _my_num_apps += 1;
732  }
733  else
734  {
735  unsigned int num_apps_in_jobs_left_pids = (_my_num_apps + 1) * jobs_left;
736  unsigned int distance_to_jobs_left_pids = _orig_rank - jobs_left;
737 
738  _first_local_app = num_apps_in_jobs_left_pids + (_my_num_apps * distance_to_jobs_left_pids);
739  }
740  }
741  else
743 
744  return;
745  }
746 
747  // In this case we need to divide up the processors that are going to work on each app
748  int rank;
749  ierr = MPI_Comm_rank(_communicator.get(), &rank);
750  mooseCheckMPIErr(ierr);
751 
752  unsigned int procs_per_app = _orig_num_procs / _total_num_apps;
753 
754  if (_max_procs_per_app < procs_per_app)
755  procs_per_app = _max_procs_per_app;
756 
757  int my_app = rank / procs_per_app;
758  unsigned int procs_for_my_app = procs_per_app;
759 
760  if ((unsigned int)my_app > _total_num_apps - 1 && procs_for_my_app == _max_procs_per_app)
761  {
762  // If we've already hit the max number of procs per app then this processor
763  // won't have an app at all
764  _my_num_apps = 0;
765  _has_an_app = false;
766  }
767  else if ((unsigned int)my_app >=
768  _total_num_apps - 1) // The last app will gain any left-over procs
769  {
770  my_app = _total_num_apps - 1;
771  // procs_for_my_app += _orig_num_procs % _total_num_apps;
772  _first_local_app = my_app;
773  _my_num_apps = 1;
774  }
775  else
776  {
777  _first_local_app = my_app;
778  _my_num_apps = 1;
779  }
780 
781  if (_has_an_app)
782  {
783  _communicator.split(_first_local_app, rank, _my_communicator);
784 
785  ierr = MPI_Comm_rank(_my_comm, &_my_rank);
786  mooseCheckMPIErr(ierr);
787  }
788  else
789  {
790  _communicator.split(MPI_UNDEFINED, rank, _my_communicator);
791 
792  _my_rank = 0;
793  }
794 }
795 
796 unsigned int
797 MultiApp::globalAppToLocal(unsigned int global_app)
798 {
799  if (global_app >= _first_local_app && global_app <= _first_local_app + (_my_num_apps - 1))
800  return global_app - _first_local_app;
801 
802  _console << _first_local_app << " " << global_app << '\n';
803  mooseError("Invalid global_app!");
804 }
805 
806 void
808 {
809 }
registeredMooseAppIterator registeredObjectsBegin()
Returns iterators to the begin/end of the registered objects data structure: a name -> validParams fu...
Definition: AppFactory.h:114
bool hasLocalApp(unsigned int global_app)
Whether or not the given global app number is on this processor.
Definition: MultiApp.C:535
Point position(unsigned int app)
The physical position of a global App number.
Definition: MultiApp.h:235
std::string getFileName(bool stripLeadingPath=true) const
Return the filename that was parsed.
Definition: MooseApp.C:1049
const std::string & name() const
Get the name of the object.
Definition: MooseApp.h:73
std::vector< unsigned int > _reset_apps
The apps to be reset.
Definition: MultiApp.h:400
virtual void restore()
Restore the state of every Sub App.
Definition: MultiApp.C:409
virtual BoundingBox getBoundingBox(unsigned int app, bool displaced_mesh)
Get the BoundingBox for the mesh associated with app The bounding box will be shifted to be in the co...
Definition: MultiApp.C:424
A MultiMooseEnum object to hold "execute_on" flags.
Definition: ExecFlagEnum.h:24
A class for creating restricted objects.
Definition: Restartable.h:29
MultiApp(const InputParameters &parameters)
Definition: MultiApp.C:171
unsigned int _max_procs_per_app
Maximum number of processors to give to each app.
Definition: MultiApp.h:388
NonlinearSystemBase & getNonlinearSystemBase()
virtual void backup()
Save off the state of every Sub App.
Definition: MultiApp.C:400
const bool _use_positions
Toggle use of "positions".
Definition: MultiApp.h:334
Specialization of SubProblem for solving nonlinear equations plus auxiliary equations.
Definition: FEProblem.h:24
virtual NumericVector< Number > & addVector(const std::string &vector_name, const bool project, const ParallelType type)
Adds a solution length vector to the system.
Definition: SystemBase.C:542
VectorValue< Real > RealVectorValue
Definition: Assembly.h:31
void setupPositions()
Called just after construction to allow derived classes to set _positions;.
Definition: MultiApp.C:223
std::shared_ptr< CommandLine > commandLine() const
Get the command line.
Definition: MooseApp.h:300
const UserObject & getUserObjectBase(const std::string &name) const
Get the user object by its name.
Real _move_time
The time at which to move apps.
Definition: MultiApp.h:406
MooseAppPtr createShared(const std::string &app_type, const std::string &name, InputParameters parameters, MPI_Comm COMM_WORLD_IN)
Build an application object (must be registered)
Definition: AppFactory.C:51
Real appPostprocessorValue(unsigned int app, const std::string &name)
Get a Postprocessor value for a specified global app.
Definition: MultiApp.C:520
int _orig_rank
The mpi "rank" of this processor in the original communicator.
Definition: MultiApp.h:364
virtual std::string getCommandLineArgsParamHelper(unsigned int local_app)
Method to aid in getting the "cli_args" parameters.
Definition: MultiApp.C:688
std::vector< std::shared_ptr< MooseApp > > _apps
Pointers to each of the Apps.
Definition: MultiApp.h:373
T & set(const std::string &name, bool quiet_mode=false)
Returns a writable reference to the named parameters.
unsigned int multiAppLevel() const
The MultiApp Level.
Definition: MooseApp.h:517
virtual void parentOutputPositionChanged()
For apps outputting in position we need to change their output positions if their parent app moves...
Definition: MultiApp.C:588
std::vector< Point > _move_positions
The new positions for the apps to be moved.
Definition: MultiApp.h:412
Base class for MOOSE-based applications.
Definition: MooseApp.h:59
The main MOOSE class responsible for handling user-defined parameters in almost every MOOSE system...
std::string getOutputFileBase() const
Override the selection of the output file base name.
Definition: MooseApp.C:790
InputParameters validParams< SetupInterface >()
void mooseError(Args &&... args) const
Definition: MooseObject.h:147
Real _inflation
Relative bounding box inflation.
Definition: MultiApp.h:382
FEProblemBase & _fe_problem
The FEProblemBase this MultiApp is part of.
Definition: MultiApp.h:325
bool isRestarting() const
Whether or not this is a "restart" calculation.
Definition: MooseApp.C:865
bool _reset_happened
Whether or not apps have been reset.
Definition: MultiApp.h:403
Specialization of SubProblem for solving nonlinear equations plus auxiliary equations.
int _orig_num_procs
The number of processors in the original comm.
Definition: MultiApp.h:361
const T & getParam(const std::string &name) const
Retrieve a parameter for the object.
Definition: MooseObject.h:191
virtual void preTransfer(Real dt, Real target_time)
Gets called just before transfers are done to the MultiApp (Which is just before the MultiApp is solv...
Definition: MultiApp.C:346
std::vector< std::string > split(const std::string &str, const std::string &delimiter)
Python like split function for strings.
Definition: MooseUtils.C:736
bool checkFileReadable(const std::string &filename, bool check_line_endings=false, bool throw_on_unreadable=true)
Checks to see if a file is readable (exists and permissions)
Definition: MooseUtils.C:146
virtual NumericVector< Number > & appTransferVector(unsigned int app, std::string var_name)
Get the vector to transfer to for this MultiApp.
Definition: MultiApp.C:529
virtual std::shared_ptr< DisplacedProblem > getDisplacedProblem()
Every object that can be built by the factory should be derived from this class.
Definition: MooseObject.h:42
PetscInt m
std::map< std::string, unsigned int > getFileNumbers()
Extracts the file numbers from the output objects.
const ExecFlagType EXEC_TIMESTEP_BEGIN
virtual void fillPositions()
must fill in _positions with the positions of the sub-aps
Definition: MultiApp.C:252
const ExecFlagType EXEC_PRE_MULTIAPP_SETUP
bool _move_happened
Whether or not the move has happened.
Definition: MultiApp.h:415
Helper class for holding Sub-app backups.
Definition: MultiApp.h:45
Real _reset_time
The time at which to reset apps.
Definition: MultiApp.h:397
std::string _app_type
The type of application to build.
Definition: MultiApp.h:328
MooseMesh wraps a libMesh::Mesh object and enhances its capabilities by caching additional data and s...
Definition: MooseMesh.h:74
NumericVector< Number > & solution() override
void dynamicAppRegistration(const std::string &app_name, std::string library_path, const std::string &library_name)
Definition: MooseApp.C:1101
FEProblemBase & appProblemBase(unsigned int app)
Get the FEProblemBase for the global app is part of.
Definition: MultiApp.C:487
This is a "smart" enum class intended to replace many of the shortcomings in the C++ enum type It sho...
Definition: MooseEnum.h:31
std::vector< unsigned int > _move_apps
The apps to be moved.
Definition: MultiApp.h:409
Executioners are objects that do the actual work of solving your problem.
Definition: Executioner.h:32
void buildComm()
Create an MPI communicator suitable for each app.
Definition: MultiApp.C:701
virtual void resetApp(unsigned int global_app, Real time=0.0)
"Reset" the App corresponding to the global App number passed in.
Definition: MultiApp.C:552
std::string stringify(const T &t)
conversion to string
Definition: Conversion.h:60
AuxiliarySystem & getAuxiliarySystem()
unsigned int _total_num_apps
The total number of apps to simulate.
Definition: MultiApp.h:343
std::vector< BoundingBox > _bounding_box
This multi-app&#39;s bounding box.
Definition: MultiApp.h:379
Point getOutputPosition() const
Get the output position.
Definition: MooseApp.h:199
unsigned int _my_num_apps
The number of apps this object is involved in simulating.
Definition: MultiApp.h:346
std::string _node_name
Node Name.
Definition: MultiApp.h:367
InputParameters validParams< MooseObject >()
Definition: MooseObject.C:25
static AppFactory & instance()
Get the instance of the AppFactory.
Definition: AppFactory.C:17
bool _has_an_app
Whether or not this processor as an App at all
Definition: MultiApp.h:418
void paramError(const std::string &param, Args... args)
Emits an error prefixed with the file and line number of the given param (from the input file) along ...
Definition: MooseObject.h:108
SubAppBackups & _backups
Backups for each local App.
Definition: MultiApp.h:421
libMesh::Parallel::Communicator _my_communicator
The communicator object that holds the MPI_Comm that we&#39;re going to use.
Definition: MultiApp.h:355
InputParameters validParams< MultiApp >()
Definition: MultiApp.C:43
void createApp(unsigned int i, Real start_time)
Helper function for creating an App instance.
Definition: MultiApp.C:596
unsigned int _first_local_app
The number of the first app on this processor.
Definition: MultiApp.h:349
const std::string & _name
The name of this object, reference to value stored in InputParameters.
Definition: MooseObject.h:183
MatType type
MooseApp * localApp(unsigned int local_app)
Get the local MooseApp object.
Definition: MultiApp.C:545
virtual void initialSetup() override
Gets called at the beginning of the simulation before this object is asked to do its job...
Definition: MultiApp.C:233
void init(unsigned int num)
Initialize the MultiApp by creating the provided number of apps.
Definition: MultiApp.C:206
void mooseDeprecated(Args &&... args) const
Definition: MooseObject.h:161
MooseApp & _app
The MooseApp this object is associated with.
Definition: MooseObject.h:177
virtual Moose::CoordinateSystemType getCoordSystem(SubdomainID sid) override
FEProblem & appProblem(unsigned int app)
Get the FEProblem for the global app is part of.
Definition: MultiApp.C:498
virtual MooseMesh & mesh() override
virtual void postExecute()
Method called at the end of the simulation (after finalize)
Definition: MultiApp.C:388
bool _output_in_position
Whether or not to move the output of the MultiApp into position.
Definition: MultiApp.h:391
const std::string & name() const
Get the name of the object.
Definition: MooseObject.h:59
std::vector< Point > _positions
The positions of all of the apps.
Definition: MultiApp.h:331
virtual void finalize()
Method called towards the end of the simulation to execute on final.
Definition: MultiApp.C:375
int _my_rank
The mpi "rank" of this processor in the sub communicator.
Definition: MultiApp.h:370
PostprocessorValue & getPostprocessorValue(const PostprocessorName &name)
Get a reference to the value associated with the postprocessor.
void addParam(const std::string &name, const S &value, const std::string &doc_string)
These methods add an option parameter and a documentation string to the InputParameters object...
std::vector< FileName > _input_files
The input file for each app&#39;s simulation.
Definition: MultiApp.h:337
ierr
virtual void preRunInputFile()
call back executed right before app->runInputFile()
Definition: MultiApp.C:807
const ConsoleStream _console
An instance of helper class to write streams to the Console objects.
InputParameters getValidParams(const std::string &name)
Get valid parameters for the object.
Definition: AppFactory.C:25
Point _bounding_box_padding
Additional padding added to the bounding box, useful for 1D meshes.
Definition: MultiApp.h:385
const Real _global_time_offset
The offset time so the MultiApp local time relative to the global time.
Definition: MultiApp.h:394
registeredMooseAppIterator registeredObjectsEnd()
Definition: AppFactory.h:115
std::map< std::string, paramsPtr >::iterator registeredMooseAppIterator
alias for registered Object iterator
Definition: AppFactory.h:42
bool isParamValid(const std::string &name) const
Test if the supplied parameter is valid.
Definition: MooseObject.h:89
bool isRecovering() const
Whether or not this is a "recover" calculation.
Definition: MooseApp.C:859
MPI_Comm & _my_comm
The MPI communicator this object is going to use.
Definition: MultiApp.h:358
const ExecFlagType EXEC_FINAL
virtual void moveApp(unsigned int global_app, Point p)
Move the global_app to Point p.
Definition: MultiApp.C:571
virtual Executioner * getExecutioner(unsigned int app)
Definition: MultiApp.C:366
Base class for user-specific data.
Definition: UserObject.h:37
const std::vector< std::string > & _cli_args
Storage for command line arguments.
Definition: MultiApp.h:424
OutputWarehouse & getOutputWarehouse()
Get the OutputWarehouse objects.
Definition: MooseApp.C:1055
const UserObject & appUserObjectBase(unsigned int app, const std::string &name)
Get a UserObject base for a specific global app.
Definition: MultiApp.C:511
unsigned int globalAppToLocal(unsigned int global_app)
Map a global App number to the local number.
Definition: MultiApp.C:797
const std::set< SubdomainID > & meshSubdomains() const
Returns a read-only reference to the set of subdomains currently present in the Mesh.
Definition: MooseMesh.C:2316
std::vector< bool > _has_bounding_box
Flag if this multi-app computed its bounding box (valid only for non-displaced meshes) ...
Definition: MultiApp.h:376