Line data Source code
1 : //* This file is part of the MOOSE framework
2 : //* https://mooseframework.inl.gov
3 : //*
4 : //* All rights reserved, see COPYRIGHT for full restrictions
5 : //* https://github.com/idaholab/moose/blob/master/COPYRIGHT
6 : //*
7 : //* Licensed under LGPL 2.1, please see LICENSE for details
8 : //* https://www.gnu.org/licenses/lgpl-2.1.html
9 :
10 : // MOOSE includes
11 : #include "MultiApp.h"
12 :
13 : #include "AppFactory.h"
14 : #include "AuxiliarySystem.h"
15 : #include "DisplacedProblem.h"
16 : #include "Console.h"
17 : #include "Executioner.h"
18 : #include "FEProblem.h"
19 : #include "MooseMesh.h"
20 : #include "MooseUtils.h"
21 : #include "OutputWarehouse.h"
22 : #include "SetupInterface.h"
23 : #include "UserObject.h"
24 : #include "CommandLine.h"
25 : #include "Conversion.h"
26 : #include "NonlinearSystemBase.h"
27 : #include "DelimitedFileReader.h"
28 : #include "MooseAppCoordTransform.h"
29 : #include "MultiAppTransfer.h"
30 : #include "Positions.h"
31 : #include "Transient.h"
32 : #include "Backup.h"
33 : #include "Parser.h"
34 :
35 : #include "libmesh/mesh_tools.h"
36 : #include "libmesh/numeric_vector.h"
37 :
38 : // C++ includes
39 : #include <algorithm>
40 : #include <fstream>
41 : #include <iomanip>
42 : #include <iterator>
43 : #include <set>
44 :
45 : // Call to "uname"
46 : #ifdef LIBMESH_HAVE_SYS_UTSNAME_H
47 : #include <sys/utsname.h>
48 : #endif
49 :
50 : InputParameters
51 31568 : MultiApp::validParams()
52 : {
53 31568 : InputParameters params = MooseObject::validParams();
54 31568 : params += SetupInterface::validParams();
55 :
56 63136 : params.addParam<bool>("use_displaced_mesh",
57 63136 : false,
58 : "Whether or not this object should use the "
59 : "displaced mesh for computation. Note that "
60 : "in the case this is true but no "
61 : "displacements are provided in the Mesh block "
62 : "the undisplaced mesh will still be used.");
63 :
64 31568 : std::ostringstream app_types_strings;
65 63162 : for (const auto & name_bi_pair : AppFactory::instance().registeredObjects())
66 31594 : app_types_strings << name_bi_pair.first << " ";
67 63136 : MooseEnum app_types_options(app_types_strings.str(), "", true);
68 :
69 : // Dynamic loading
70 126272 : params.addParam<MooseEnum>("app_type",
71 : app_types_options,
72 : "The type of application to build (applications not "
73 : "registered can be loaded with dynamic libraries. Parent "
74 : "application type will be used if not provided.");
75 126272 : params.addParam<std::string>("library_path",
76 : "",
77 : "Path to search for dynamic libraries (please "
78 : "avoid committing absolute paths in addition to "
79 : "MOOSE_LIBRARY_PATH)");
80 126272 : params.addParam<std::string>(
81 : "library_name",
82 : "",
83 : "The file name of the library (*.la file) that will be dynamically loaded.");
84 94704 : params.addParam<bool>("library_load_dependencies",
85 63136 : false,
86 : "Tells MOOSE to manually load library dependencies. This should not be "
87 : "necessary and is here for debugging/troubleshooting.");
88 :
89 : // Subapp positions
90 126272 : params.addParam<std::vector<Point>>(
91 : "positions",
92 : "The positions of the App locations. Each set of 3 values will represent a "
93 : "Point. This and 'positions_file' cannot be both supplied. If this and "
94 : "'positions_file'/'_objects' are not supplied, a single position (0,0,0) will be used");
95 126272 : params.addParam<std::vector<FileName>>("positions_file",
96 : "Filename(s) that should be looked in for positions. Each"
97 : " set of 3 values in that file will represent a Point. "
98 : "This and 'positions(_objects)' cannot be both supplied");
99 126272 : params.addParam<std::vector<PositionsName>>("positions_objects",
100 : "The name of a Positions object that will contain "
101 : "the locations of the sub-apps created. This and "
102 : "'positions(_file)' cannot be both supplied");
103 94704 : params.addParam<bool>(
104 : "output_in_position",
105 63136 : false,
106 : "If true this will cause the output from the MultiApp to be 'moved' by its position vector");
107 94704 : params.addParam<bool>(
108 : "run_in_position",
109 63136 : false,
110 : "If true this will cause the mesh from the MultiApp to be 'moved' by its position vector");
111 :
112 126272 : params.addRequiredParam<std::vector<FileName>>(
113 : "input_files",
114 : "The input file for each App. If this parameter only contains one input file "
115 : "it will be used for all of the Apps. When using 'positions_from_file' it is "
116 : "also admissable to provide one input_file per file.");
117 94704 : params.addParam<Real>("bounding_box_inflation",
118 63136 : 0.01,
119 : "Relative amount to 'inflate' the bounding box of this MultiApp.");
120 94704 : params.addParam<Point>("bounding_box_padding",
121 63136 : RealVectorValue(),
122 : "Additional padding added to the dimensions of the bounding box. The "
123 : "values are added to the x, y and z dimension respectively.");
124 :
125 63136 : params.addPrivateParam<MPI_Comm>("_mpi_comm");
126 :
127 : // Set the default execution time
128 63136 : params.set<ExecFlagEnum>("execute_on", true) = EXEC_TIMESTEP_BEGIN;
129 : // Add the POST_ADAPTIVITY execution flag.
130 : #ifdef LIBMESH_ENABLE_AMR
131 31568 : ExecFlagEnum & exec_enum = params.set<ExecFlagEnum>("execute_on");
132 31568 : exec_enum.addAvailableFlags(EXEC_POST_ADAPTIVITY);
133 94704 : params.setDocString("execute_on", exec_enum.getDocString());
134 : #endif
135 :
136 94704 : params.addParam<processor_id_type>("max_procs_per_app",
137 63136 : std::numeric_limits<processor_id_type>::max(),
138 : "Maximum number of processors to give to each App in this "
139 : "MultiApp. Useful for restricting small solves to just a few "
140 : "procs so they don't get spread out");
141 94704 : params.addParam<processor_id_type>("min_procs_per_app",
142 63136 : 1,
143 : "Minimum number of processors to give to each App in this "
144 : "MultiApp. Useful for larger, distributed mesh solves.");
145 94704 : params.addParam<bool>(
146 : "wait_for_first_app_init",
147 63136 : false,
148 : "Create the first sub-application on rank 0, then MPI_Barrier before "
149 : "creating the next N-1 apps (on all ranks). "
150 : "This is only needed if your sub-application needs to perform some setup "
151 : "actions in quiet, without other sub-applications working at the same time.");
152 :
153 94704 : params.addParam<Real>("global_time_offset",
154 63136 : 0,
155 : "The time offset relative to the parent application for the purpose of "
156 : "starting a subapp at a different time from the parent application. The "
157 : "global time will be ahead by the offset specified here.");
158 :
159 : // Resetting subapps
160 126272 : params.addParam<std::vector<Real>>(
161 : "reset_time",
162 : {},
163 : "The time(s) at which to reset Apps given by the 'reset_apps' parameter. "
164 : "Resetting an App means that it is destroyed and recreated, possibly "
165 : "modeling the insertion of 'new' material for that app.");
166 126272 : params.addParam<std::vector<unsigned int>>(
167 : "reset_apps",
168 : {},
169 : "The Apps that will be reset when 'reset_time' is hit. These are the App "
170 : "'numbers' starting with 0 corresponding to the order of the App positions. "
171 : "Resetting an App means that it is destroyed and recreated, possibly modeling "
172 : "the insertion of 'new' material for that app.");
173 :
174 : // Moving subapps
175 94704 : params.addParam<Real>(
176 : "move_time",
177 63136 : std::numeric_limits<Real>::max(),
178 : "The time at which Apps designated by move_apps are moved to move_positions.");
179 :
180 126272 : params.addParam<std::vector<unsigned int>>(
181 : "move_apps",
182 : {},
183 : "Apps, designated by their 'numbers' starting with 0 corresponding to the order "
184 : "of the App positions, to be moved at move_time to move_positions");
185 126272 : params.addParam<std::vector<Point>>(
186 : "move_positions", {}, "The positions corresponding to each move_app.");
187 :
188 126272 : params.addParam<std::vector<CLIArgString>>(
189 : "cli_args",
190 : {},
191 : "Additional command line arguments to pass to the sub apps. If one set is provided the "
192 : "arguments are applied to all, otherwise there must be a set for each sub app.");
193 :
194 126272 : params.addParam<std::vector<FileName>>(
195 : "cli_args_files",
196 : "File names that should be looked in for additional command line arguments "
197 : "to pass to the sub apps. Each line of a file is set to each sub app. If only "
198 : "one line is provided, it will be applied to all sub apps.");
199 :
200 : // Fixed point iterations
201 157840 : params.addRangeCheckedParam<Real>("relaxation_factor",
202 63136 : 1.0,
203 : "relaxation_factor>0 & relaxation_factor<2",
204 : "Fraction of newly computed value to keep."
205 : "Set between 0 and 2.");
206 189408 : params.addDeprecatedParam<std::vector<std::string>>(
207 : "relaxed_variables",
208 : {},
209 : "Use transformed_variables.",
210 : "List of subapp variables to relax during Multiapp coupling iterations");
211 126272 : params.addParam<std::vector<std::string>>(
212 : "transformed_variables",
213 : {},
214 : "List of subapp variables to use coupling algorithm on during Multiapp coupling iterations");
215 126272 : params.addParam<std::vector<PostprocessorName>>(
216 : "transformed_postprocessors",
217 : {},
218 : "List of subapp postprocessors to use coupling "
219 : "algorithm on during Multiapp coupling iterations");
220 94704 : params.addParam<bool>("keep_solution_during_restore",
221 63136 : false,
222 : "This is useful when doing MultiApp coupling iterations. It takes the "
223 : "final solution from the previous coupling iteration"
224 : "and re-uses it as the initial guess for the next coupling iteration");
225 94704 : params.addParam<bool>("keep_aux_solution_during_restore",
226 63136 : false,
227 : "This is useful when doing MultiApp coupling iterations. It takes the "
228 : "final auxiliary solution from the previous coupling iteration"
229 : "and re-uses it as the initial guess for the next coupling iteration");
230 94704 : params.addParam<bool>(
231 : "no_restore",
232 63136 : false,
233 : "True to turn off restore for this multiapp. This is useful when doing steady-state "
234 : "Picard iterations where we want to use the solution of previous Picard iteration as the "
235 : "initial guess of the current Picard iteration.");
236 94704 : params.addParam<unsigned int>(
237 : "max_multiapp_level",
238 63136 : 10,
239 : "Integer set by user that will stop the simulation if the multiapp level "
240 : "exceeds it. Useful for preventing infinite loops with multiapp simulations");
241 :
242 157840 : params.addDeprecatedParam<bool>("clone_master_mesh",
243 63136 : false,
244 : "True to clone parent app mesh and use it for this MultiApp.",
245 : "clone_master_mesh is deprecated, use clone_parent_mesh instead");
246 63136 : params.addParam<bool>(
247 63136 : "clone_parent_mesh", false, "True to clone parent app mesh and use it for this MultiApp.");
248 :
249 63136 : params.addPrivateParam<bool>("use_positions", true);
250 94704 : params.declareControllable("enable");
251 157840 : params.declareControllable("cli_args", {EXEC_PRE_MULTIAPP_SETUP});
252 63136 : params.registerBase("MultiApp");
253 :
254 126272 : params.addParamNamesToGroup("use_displaced_mesh wait_for_first_app_init max_multiapp_level",
255 : "Advanced");
256 126272 : params.addParamNamesToGroup("positions positions_file positions_objects run_in_position "
257 : "output_in_position",
258 : "Positions / transformations of the MultiApp frame of reference");
259 126272 : params.addParamNamesToGroup("min_procs_per_app max_procs_per_app", "Parallelism");
260 126272 : params.addParamNamesToGroup("reset_time reset_apps", "Reset MultiApp");
261 126272 : params.addParamNamesToGroup("move_time move_apps move_positions", "Timed move of MultiApps");
262 126272 : params.addParamNamesToGroup("relaxation_factor transformed_variables transformed_postprocessors "
263 : "keep_solution_during_restore keep_aux_solution_during_restore "
264 : "no_restore",
265 : "Fixed point iteration");
266 126272 : params.addParamNamesToGroup("library_name library_path library_load_dependencies",
267 : "Dynamic loading");
268 94704 : params.addParamNamesToGroup("cli_args cli_args_files", "Passing command line argument");
269 63136 : return params;
270 63136 : }
271 :
272 8129 : MultiApp::MultiApp(const InputParameters & parameters)
273 : : MooseObject(parameters),
274 : SetupInterface(this),
275 : Restartable(this, "MultiApps"),
276 16258 : PerfGraphInterface(this, std::string("MultiApp::") + _name),
277 24387 : _fe_problem(*getCheckedPointerParam<FEProblemBase *>("_fe_problem_base")),
278 28730 : _app_type(isParamValid("app_type") ? std::string(getParam<MooseEnum>("app_type"))
279 3786 : : _fe_problem.getMooseApp().type()),
280 16258 : _use_positions(getParam<bool>("use_positions")),
281 16258 : _input_files(getParam<std::vector<FileName>>("input_files")),
282 16258 : _wait_for_first_app_init(getParam<bool>("wait_for_first_app_init")),
283 8129 : _total_num_apps(0),
284 8129 : _my_num_apps(0),
285 8129 : _first_local_app(0),
286 8129 : _orig_comm(_communicator.get()),
287 8129 : _my_communicator(),
288 8129 : _my_comm(_my_communicator.get()),
289 8129 : _my_rank(0),
290 16258 : _inflation(getParam<Real>("bounding_box_inflation")),
291 16258 : _bounding_box_padding(getParam<Point>("bounding_box_padding")),
292 16258 : _max_procs_per_app(getParam<processor_id_type>("max_procs_per_app")),
293 16258 : _min_procs_per_app(getParam<processor_id_type>("min_procs_per_app")),
294 16258 : _output_in_position(getParam<bool>("output_in_position")),
295 16258 : _global_time_offset(getParam<Real>("global_time_offset")),
296 16258 : _reset_times(getParam<std::vector<Real>>("reset_time")),
297 16258 : _reset_apps(getParam<std::vector<unsigned int>>("reset_apps")),
298 16258 : _reset_happened(false),
299 16258 : _move_time(getParam<Real>("move_time")),
300 16258 : _move_apps(getParam<std::vector<unsigned int>>("move_apps")),
301 16258 : _move_positions(getParam<std::vector<Point>>("move_positions")),
302 8129 : _move_happened(false),
303 8129 : _has_an_app(true),
304 16258 : _cli_args(getParam<std::vector<CLIArgString>>("cli_args")),
305 16258 : _keep_solution_during_restore(getParam<bool>("keep_solution_during_restore")),
306 16258 : _keep_aux_solution_during_restore(getParam<bool>("keep_aux_solution_during_restore")),
307 16258 : _no_restore(getParam<bool>("no_restore")),
308 16258 : _run_in_position(getParam<bool>("run_in_position")),
309 16258 : _sub_app_backups(declareRestartableDataWithContext<SubAppBackups>("sub_app_backups", this)),
310 32516 : _solve_step_timer(registerTimedSection("solveStep", 3, "Executing MultiApps", false)),
311 32516 : _init_timer(registerTimedSection("init", 3, "Initializing MultiApp")),
312 32516 : _backup_timer(registerTimedSection("backup", 3, "Backing Up MultiApp")),
313 32516 : _restore_timer(registerTimedSection("restore", 3, "Restoring MultiApp")),
314 121935 : _reset_timer(registerTimedSection("resetApp", 3, "Resetting MultiApp"))
315 : {
316 38252 : if (parameters.isParamSetByUser("cli_args") && parameters.isParamValid("cli_args") &&
317 11953 : parameters.isParamValid("cli_args_files"))
318 6 : paramError("cli_args",
319 : "'cli_args' and 'cli_args_files' cannot be specified simultaneously in MultiApp ");
320 :
321 16252 : if (!_use_positions && (isParamValid("positions") || isParamValid("positions_file") ||
322 8126 : isParamValid("positions_objects")))
323 0 : paramError("use_positions",
324 : "This MultiApps has been set to not use positions, "
325 : "but a 'positions' parameter has been set.");
326 :
327 24316 : if ((_reset_apps.size() > 0 && _reset_times.size() == 0) ||
328 16190 : (_reset_apps.size() == 0 && _reset_times.size() > 0))
329 0 : mooseError("reset_time and reset_apps may only be specified together");
330 :
331 : // Check that the reset times are sorted by the user
332 8126 : auto sorted_times = _reset_times;
333 8126 : std::sort(sorted_times.begin(), sorted_times.end());
334 8126 : if (_reset_times.size() && _reset_times != sorted_times)
335 6 : paramError("reset_time", "List of reset times must be sorted in increasing order");
336 8123 : }
337 :
338 : void
339 8102 : MultiApp::init(unsigned int num_apps, bool batch_mode)
340 : {
341 8102 : auto config = rankConfig(
342 : processor_id(), n_processors(), num_apps, _min_procs_per_app, _max_procs_per_app, batch_mode);
343 8102 : init(num_apps, config);
344 8099 : }
345 :
346 : void
347 8102 : MultiApp::init(unsigned int num_apps, const LocalRankConfig & config)
348 : {
349 8102 : TIME_SECTION(_init_timer);
350 :
351 8102 : _total_num_apps = num_apps;
352 8102 : _rank_config = config;
353 8102 : buildComm();
354 8102 : _sub_app_backups.resize(_my_num_apps);
355 :
356 8102 : _has_bounding_box.resize(_my_num_apps, false);
357 8102 : _reset_happened.resize(_reset_times.size(), false);
358 8102 : _bounding_box.resize(_my_num_apps);
359 :
360 8102 : if ((_cli_args.size() > 1) && (_total_num_apps != _cli_args.size()))
361 6 : paramError("cli_args",
362 : "The number of items supplied must be 1 or equal to the number of sub apps.");
363 :
364 : // if cliArgs() != _cli_args, then cliArgs() was overridden and we need to check it
365 8099 : auto cla = cliArgs();
366 16198 : if (cla != std::vector<std::string>(_cli_args.begin(), _cli_args.end()))
367 : {
368 12 : if ((cla.size() > 1) && (_total_num_apps != cla.size()))
369 0 : mooseError("The number of items supplied as command line argument to subapps must be 1 or "
370 : "equal to the number of sub apps. Note: you use a multiapp that provides its own "
371 : "command line parameters so the error is not in cli_args");
372 : }
373 8099 : }
374 :
375 : void
376 8120 : MultiApp::setupPositions()
377 : {
378 8120 : if (_use_positions)
379 : {
380 8120 : fillPositions();
381 8102 : init(_positions.size());
382 8099 : createApps();
383 : }
384 8033 : }
385 :
386 : void
387 8099 : MultiApp::createApps()
388 : {
389 8099 : if (!_has_an_app)
390 52 : return;
391 :
392 40235 : TIME_SECTION("createApps", 2, "Instantiating Sub-Apps", false);
393 :
394 : // Read commandLine arguments that will be used when creating apps
395 8047 : readCommandLineArguments();
396 :
397 8032 : Moose::ScopedCommSwapper swapper(_my_comm);
398 :
399 8032 : _apps.resize(_my_num_apps);
400 :
401 : // If the user provided an unregistered app type, see if we can load it dynamically
402 8032 : if (!AppFactory::instance().isRegistered(_app_type))
403 5 : _app.dynamicAppRegistration(_app_type,
404 : getParam<std::string>("library_path"),
405 7 : getParam<std::string>("library_name"),
406 14 : getParam<bool>("library_load_dependencies"));
407 :
408 8029 : bool rank_did_quiet_init = false;
409 8029 : unsigned int local_app = libMesh::invalid_uint;
410 8029 : if (_wait_for_first_app_init)
411 : {
412 14 : if (hasLocalApp(0))
413 : {
414 7 : rank_did_quiet_init = true;
415 7 : local_app = globalAppToLocal(0);
416 7 : createLocalApp(local_app);
417 : }
418 :
419 14 : MPI_Barrier(_orig_comm);
420 : }
421 :
422 20115 : for (unsigned int i = 0; i < _my_num_apps; i++)
423 : {
424 12134 : if (rank_did_quiet_init && i == local_app)
425 7 : continue;
426 12127 : createLocalApp(i);
427 : }
428 7981 : }
429 :
430 : void
431 12134 : MultiApp::createLocalApp(const unsigned int i)
432 : {
433 12134 : createApp(i, _global_time_offset);
434 12086 : }
435 :
436 : void
437 7961 : MultiApp::initialSetup()
438 : {
439 7961 : if (!_use_positions)
440 : // if not using positions, we create the sub-apps in initialSetup instead of right after
441 : // construction of MultiApp
442 0 : createApps();
443 7961 : }
444 :
445 : void
446 8047 : MultiApp::readCommandLineArguments()
447 : {
448 24141 : if (isParamValid("cli_args_files"))
449 : {
450 51 : _cli_args_from_file.clear();
451 :
452 102 : std::vector<FileName> cli_args_files = getParam<std::vector<FileName>>("cli_args_files");
453 102 : std::vector<FileName> input_files = getParam<std::vector<FileName>>("input_files");
454 :
455 : // If we use parameter "cli_args_files", at least one file should be provided
456 51 : if (!cli_args_files.size())
457 6 : paramError("cli_args_files", "You need to provide at least one commandLine argument file ");
458 :
459 : // If we multiple input files, then we need to check if the number of input files
460 : // match with the number of argument files
461 48 : if (cli_args_files.size() != 1 && cli_args_files.size() != input_files.size())
462 9 : paramError("cli_args_files",
463 : "The number of commandLine argument files ",
464 : cli_args_files.size(),
465 : " for MultiApp ",
466 3 : name(),
467 : " must either be only one or match the number of input files ",
468 : input_files.size());
469 :
470 : // Go through all argument files
471 45 : std::vector<std::string> cli_args;
472 96 : for (unsigned int p_file_it = 0; p_file_it < cli_args_files.size(); p_file_it++)
473 : {
474 57 : std::string cli_args_file = cli_args_files[p_file_it];
475 : // Clear up
476 57 : cli_args.clear();
477 : // Read the file on the root processor then broadcast it
478 57 : if (processor_id() == 0)
479 : {
480 45 : MooseUtils::checkFileReadable(cli_args_file);
481 :
482 45 : std::ifstream is(cli_args_file.c_str());
483 : // Read by line rather than space separated like the cli_args parameter
484 45 : std::string line;
485 183 : while (std::getline(is, line))
486 138 : cli_args.push_back(line);
487 :
488 : // We do not allow empty files
489 45 : if (!cli_args.size())
490 6 : paramError("cli_args_files",
491 : "There is no commandLine argument in the commandLine argument file ",
492 : cli_args_file);
493 :
494 : // If we have position files, we need to
495 : // make sure the number of commandLine argument strings
496 : // match with the number of positions
497 42 : if (_npositions_inputfile.size())
498 : {
499 39 : auto num_positions = _npositions_inputfile[p_file_it];
500 : // Check if the number of commandLine argument strings equal to
501 : // the number of positions
502 39 : if (cli_args.size() == 1)
503 0 : for (MooseIndex(num_positions) num = 0; num < num_positions; num++)
504 0 : _cli_args_from_file.push_back(cli_args.front());
505 39 : else if (cli_args.size() == num_positions)
506 144 : for (auto && cli_arg : cli_args)
507 108 : _cli_args_from_file.push_back(cli_arg);
508 3 : else if (cli_args.size() != num_positions)
509 6 : paramError("cli_args_files",
510 : "The number of commandLine argument strings ",
511 : cli_args.size(),
512 : " in the file ",
513 : cli_args_file,
514 : " must either be only one or match the number of positions ",
515 : num_positions);
516 : }
517 : else
518 : {
519 : // If we do not have position files, we will check if the number of
520 : // commandLine argument strings match with the total number of subapps
521 18 : for (auto && cli_arg : cli_args)
522 15 : _cli_args_from_file.push_back(cli_arg);
523 : }
524 39 : }
525 51 : }
526 :
527 : // Broad cast all arguments to everyone
528 39 : _communicator.broadcast(_cli_args_from_file);
529 39 : }
530 :
531 8074 : if (_cli_args_from_file.size() && _cli_args_from_file.size() != 1 &&
532 39 : _cli_args_from_file.size() != _total_num_apps)
533 6 : mooseError(" The number of commandLine argument strings ",
534 3 : _cli_args_from_file.size(),
535 : " must either be only one or match the total "
536 : "number of sub apps ",
537 3 : _total_num_apps);
538 :
539 8032 : if (_cli_args_from_file.size() && cliArgs().size())
540 0 : mooseError("Cannot set commandLine arguments from both input_file and external files");
541 8032 : }
542 :
543 : void
544 8013 : MultiApp::fillPositions()
545 : {
546 8013 : if (_move_apps.size() != _move_positions.size())
547 0 : mooseError("The number of apps to move and the positions to move them to must be the same for "
548 : "MultiApp ",
549 0 : _name);
550 :
551 40065 : if (isParamValid("positions") + isParamValid("positions_file") +
552 24039 : isParamValid("positions_objects") >
553 : 1)
554 3 : mooseError("Only one 'positions' parameter may be specified");
555 :
556 24030 : if (isParamValid("positions"))
557 : {
558 10054 : _positions = getParam<std::vector<Point>>("positions");
559 :
560 5027 : if (_positions.size() < _input_files.size())
561 3 : mooseError("Not enough positions for the number of input files provided in MultiApp ",
562 3 : name());
563 : }
564 8949 : else if (isParamValid("positions_file"))
565 : {
566 168 : std::vector<FileName> positions_files = getParam<std::vector<FileName>>("positions_file");
567 168 : std::vector<FileName> input_files = getParam<std::vector<FileName>>("input_files");
568 :
569 84 : if (input_files.size() != 1 && positions_files.size() != input_files.size())
570 3 : mooseError("Number of input_files for MultiApp ",
571 3 : name(),
572 : " must either be only one or match the number of positions_file files");
573 :
574 : // Clear out the _input_files because we're going to rebuild it
575 81 : if (input_files.size() != 1)
576 27 : _input_files.clear();
577 :
578 186 : for (unsigned int p_file_it = 0; p_file_it < positions_files.size(); p_file_it++)
579 : {
580 108 : std::string positions_file = positions_files[p_file_it];
581 108 : MooseUtils::DelimitedFileReader file(positions_file, &_communicator);
582 108 : file.setFormatFlag(MooseUtils::DelimitedFileReader::FormatFlag::ROWS);
583 108 : file.read();
584 :
585 108 : const std::vector<Point> & data = file.getDataAsPoints();
586 513 : for (const auto & d : data)
587 408 : _positions.push_back(d);
588 :
589 : // Save the number of positions for this input file
590 105 : _npositions_inputfile.push_back(data.size());
591 :
592 513 : for (unsigned int i = 0; i < data.size(); ++i)
593 408 : if (input_files.size() != 1)
594 108 : _input_files.push_back(input_files[p_file_it]);
595 105 : }
596 78 : }
597 8697 : else if (isParamValid("positions_objects"))
598 : {
599 930 : const auto & positions_param_objs = getParam<std::vector<PositionsName>>("positions_objects");
600 930 : const auto & input_files = getParam<std::vector<FileName>>("input_files");
601 :
602 465 : if (input_files.size() != 1 && positions_param_objs.size() != input_files.size())
603 0 : mooseError("Number of input_files for MultiApp ",
604 0 : name(),
605 : " must either be only one or match the number of positions_objects specified");
606 :
607 : // Clear out the _input_files because we're going to rebuild it
608 465 : if (input_files.size() != 1)
609 0 : _input_files.clear();
610 :
611 : // Keeps track of where each positions object start in terms of subapp numbers
612 465 : unsigned int offset = 0;
613 :
614 966 : for (const auto p_obj_it : index_range(positions_param_objs))
615 : {
616 504 : const std::string & positions_name = positions_param_objs[p_obj_it];
617 504 : auto positions_obj = &_fe_problem.getPositionsObject(positions_name);
618 :
619 504 : const auto & data = positions_obj->getPositions(true);
620 :
621 : // Append all positions from this object
622 2151 : for (const auto & d : data)
623 1650 : _positions.push_back(d);
624 :
625 : // Save the number of positions for this input file
626 501 : _npositions_inputfile.push_back(data.size());
627 :
628 501 : if (!positions_obj)
629 0 : paramError("positions_objects",
630 0 : "'" + positions_name + "' is not of the expected type. Should be a Positions");
631 :
632 : // Keep track of which positions is tied to what subapp
633 2151 : for (unsigned int i = 0; i < data.size(); ++i)
634 : {
635 1650 : if (input_files.size() != 1)
636 0 : _input_files.push_back(input_files[p_obj_it]);
637 1650 : _positions_objs.push_back(positions_obj);
638 1650 : _positions_index_offsets.push_back(offset);
639 : }
640 501 : offset += data.size();
641 : }
642 : }
643 : else
644 : {
645 2434 : _positions = {Point()};
646 :
647 2434 : if (_positions.size() < _input_files.size())
648 3 : mooseError("Not enough positions for the number of input files provided in MultiApp ",
649 3 : name());
650 : }
651 :
652 : mooseAssert(_input_files.size() == 1 || _positions.size() == _input_files.size(),
653 : "Number of positions and input files are not the same!");
654 7995 : }
655 :
656 : void
657 65139 : MultiApp::preTransfer(Real /*dt*/, Real target_time)
658 : {
659 : // Get a transient executioner to get a user-set tolerance
660 65139 : Real timestep_tol = 1e-13;
661 65139 : if (dynamic_cast<TransientBase *>(_fe_problem.getMooseApp().getExecutioner()))
662 61858 : timestep_tol =
663 61858 : dynamic_cast<TransientBase *>(_fe_problem.getMooseApp().getExecutioner())->timestepTol();
664 :
665 : // Determination on whether we need to backup the app due to changes below
666 65139 : bool backup_apps = false;
667 :
668 : // First, see if any Apps need to be reset
669 65847 : for (unsigned int i = 0; i < _reset_times.size(); i++)
670 : {
671 777 : if (!_reset_happened[i] && (target_time + timestep_tol >= _reset_times[i]))
672 : {
673 69 : _reset_happened[i] = true;
674 69 : if (_reset_apps.size() > 0)
675 138 : for (auto & app : _reset_apps)
676 69 : resetApp(app);
677 :
678 : // If we reset an application, then we delete the old objects, including the coordinate
679 : // transformation classes. Consequently we need to reset the coordinate transformation classes
680 : // in the associated transfer classes
681 99 : for (auto * const transfer : _associated_transfers)
682 30 : transfer->getAppInfo();
683 :
684 : // Similarly we need to transform the mesh again
685 69 : if (_run_in_position)
686 0 : for (const auto i : make_range(_my_num_apps))
687 : {
688 0 : auto app_ptr = _apps[i];
689 0 : if (usingPositions())
690 0 : app_ptr->getExecutioner()->feProblem().coordTransform().transformMesh(
691 0 : app_ptr->getExecutioner()->feProblem().mesh(), _positions[_first_local_app + i]);
692 : else
693 0 : app_ptr->getExecutioner()->feProblem().coordTransform().transformMesh(
694 0 : app_ptr->getExecutioner()->feProblem().mesh(), Point(0, 0, 0));
695 0 : }
696 :
697 : // If the time step covers multiple reset times, set them all as having 'happened'
698 158 : for (unsigned int j = i; j < _reset_times.size(); j++)
699 89 : if (target_time + timestep_tol >= _reset_times[j])
700 79 : _reset_happened[j] = true;
701 :
702 : // Backup in case the next solve fails
703 69 : backup_apps = true;
704 :
705 69 : break;
706 : }
707 : }
708 :
709 : // Now move any apps that should be moved
710 65139 : if (_use_positions && !_move_happened && target_time + timestep_tol >= _move_time)
711 : {
712 54 : _move_happened = true;
713 105 : for (unsigned int i = 0; i < _move_apps.size(); i++)
714 54 : moveApp(_move_apps[i], _move_positions[i]);
715 :
716 : // Backup in case the next solve fails
717 51 : backup_apps = true;
718 : }
719 :
720 65136 : if (backup_apps)
721 101 : backup();
722 65136 : }
723 :
724 : Executioner *
725 0 : MultiApp::getExecutioner(unsigned int app)
726 : {
727 0 : if (!_has_an_app)
728 0 : mooseError("No app for ", name(), " on processor ", _orig_rank);
729 :
730 0 : return _apps[globalAppToLocal(app)]->getExecutioner();
731 : }
732 :
733 : void
734 4640 : MultiApp::finalize()
735 : {
736 11298 : for (const auto & app_ptr : _apps)
737 : {
738 6658 : auto * executioner = app_ptr->getExecutioner();
739 : mooseAssert(executioner, "Executioner is nullptr");
740 :
741 6658 : executioner->feProblem().execute(EXEC_FINAL);
742 6658 : executioner->feProblem().outputStep(EXEC_FINAL);
743 : }
744 4640 : }
745 :
746 : void
747 5004 : MultiApp::postExecute()
748 : {
749 12326 : for (const auto & app_ptr : _apps)
750 : {
751 7322 : auto * executioner = app_ptr->getExecutioner();
752 : mooseAssert(executioner, "Executioner is nullptr");
753 :
754 7322 : executioner->postExecute();
755 : }
756 5004 : }
757 :
758 : void
759 25358 : MultiApp::backup()
760 : {
761 25358 : TIME_SECTION(_backup_timer);
762 :
763 25358 : if (_fe_problem.verboseMultiApps())
764 926 : _console << "Backed up MultiApp ... ";
765 :
766 59212 : for (unsigned int i = 0; i < _my_num_apps; i++)
767 33854 : _sub_app_backups[i] = _apps[i]->backup();
768 :
769 25358 : if (_fe_problem.verboseMultiApps())
770 926 : _console << name() << std::endl;
771 25358 : }
772 :
773 : void
774 43780 : MultiApp::restore(bool force)
775 : {
776 43780 : TIME_SECTION(_restore_timer);
777 :
778 43780 : if (force || needsRestoration())
779 : {
780 : // Must be restarting / recovering from main app so hold off on restoring
781 : // Instead - the restore will happen in sub-apps' initialSetup()
782 : // Note that _backups was already populated by dataLoad() in the main app
783 9336 : if (_fe_problem.getCurrentExecuteOnFlag() == EXEC_INITIAL)
784 580 : return;
785 :
786 : // We temporarily copy and store solutions for all subapps
787 8756 : if (_keep_solution_during_restore)
788 : {
789 324 : _end_solutions.resize(_my_num_apps);
790 :
791 645 : for (unsigned int i = 0; i < _my_num_apps; i++)
792 : {
793 324 : _end_solutions[i].resize(_apps[i]->getExecutioner()->feProblem().numSolverSystems());
794 658 : for (unsigned int j = 0; j < _apps[i]->getExecutioner()->feProblem().numSolverSystems();
795 : j++)
796 : {
797 668 : _end_solutions[i][j] = _apps[i]
798 : ->getExecutioner()
799 334 : ->feProblem()
800 334 : .getSolverSystem(/*solver_sys=*/j)
801 334 : .solution()
802 668 : .clone();
803 : }
804 : auto & sub_multiapps =
805 324 : _apps[i]->getExecutioner()->feProblem().getMultiAppWarehouse().getObjects();
806 :
807 : // multiapps of each subapp should do the same things
808 : // It is implemented recursively
809 334 : for (auto & multi_app : sub_multiapps)
810 13 : multi_app->keepSolutionDuringRestore(_keep_solution_during_restore);
811 : }
812 : }
813 :
814 : // We temporarily copy and store solutions for all subapps
815 8753 : if (_keep_aux_solution_during_restore)
816 : {
817 120 : _end_aux_solutions.resize(_my_num_apps);
818 :
819 240 : for (unsigned int i = 0; i < _my_num_apps; i++)
820 120 : _end_aux_solutions[i] =
821 240 : _apps[i]->getExecutioner()->feProblem().getAuxiliarySystem().solution().clone();
822 : }
823 :
824 8753 : if (_fe_problem.verboseMultiApps())
825 0 : _console << "Restoring MultiApp ... ";
826 :
827 17692 : for (unsigned int i = 0; i < _my_num_apps; i++)
828 : {
829 8939 : _apps[i]->restore(std::move(_sub_app_backups[i]), false);
830 8939 : _sub_app_backups[i] = _apps[i]->finalizeRestore();
831 : mooseAssert(_sub_app_backups[i], "Should have a backup");
832 : }
833 :
834 8753 : if (_fe_problem.verboseMultiApps())
835 0 : _console << name() << std::endl;
836 :
837 : // Now copy the latest solutions back for each subapp
838 8753 : if (_keep_solution_during_restore)
839 : {
840 642 : for (unsigned int i = 0; i < _my_num_apps; i++)
841 : {
842 652 : for (unsigned int j = 0; j < _apps[i]->getExecutioner()->feProblem().numSolverSystems();
843 : j++)
844 : {
845 331 : _apps[i]->getExecutioner()->feProblem().getSolverSystem(/*solver_sys=*/j).solution() =
846 662 : *_end_solutions[i][j];
847 :
848 : // We need to synchronize solution so that local_solution has the right values
849 331 : _apps[i]->getExecutioner()->feProblem().getSolverSystem(/*solver_sys=*/j).update();
850 : }
851 : }
852 :
853 321 : _end_solutions.clear();
854 : }
855 : // Now copy the latest auxiliary solutions back for each subapp
856 8753 : if (_keep_aux_solution_during_restore)
857 : {
858 240 : for (unsigned int i = 0; i < _my_num_apps; i++)
859 : {
860 120 : _apps[i]->getExecutioner()->feProblem().getAuxiliarySystem().solution() =
861 240 : *_end_aux_solutions[i];
862 :
863 : // We need to synchronize solution so that local_solution has the right values
864 120 : _apps[i]->getExecutioner()->feProblem().getAuxiliarySystem().update();
865 : }
866 :
867 120 : _end_aux_solutions.clear();
868 : }
869 :
870 : // Make sure the displaced mesh on the multiapp is up-to-date with displacement variables
871 17692 : for (const auto & app_ptr : _apps)
872 8939 : if (app_ptr->feProblem().getDisplacedProblem())
873 48 : app_ptr->feProblem().getDisplacedProblem()->updateMesh();
874 :
875 : // If we are restoring due to a failed solve, make sure reset the solved state in the sub-apps
876 8753 : if (!getMooseApp().getExecutioner()->lastSolveConverged())
877 1430 : for (auto & app_ptr : _apps)
878 802 : app_ptr->getExecutioner()->fixedPointSolve().clearFixedPointStatus();
879 : }
880 : else
881 : {
882 68888 : for (unsigned int i = 0; i < _my_num_apps; i++)
883 : {
884 34444 : for (auto & sub_app :
885 69320 : _apps[i]->getExecutioner()->feProblem().getMultiAppWarehouse().getObjects())
886 432 : sub_app->restore(false);
887 : }
888 : }
889 43777 : }
890 :
891 : void
892 13 : MultiApp::keepSolutionDuringRestore(bool keep_solution_during_restore)
893 : {
894 39 : if (_pars.isParamSetByUser("keep_solution_during_restore"))
895 6 : paramError("keep_solution_during_restore",
896 : "This parameter should only be provided in parent app");
897 :
898 10 : _keep_solution_during_restore = keep_solution_during_restore;
899 10 : }
900 :
901 : void
902 376520 : MultiApp::transformBoundingBox(BoundingBox & box, const MultiAppCoordTransform & transform)
903 : {
904 376520 : const Real min_x = box.first(0);
905 376520 : const Real max_x = box.second(0);
906 376520 : const Real min_y = box.first(1);
907 376520 : const Real max_y = box.second(1);
908 376520 : const Real min_z = box.first(2);
909 376520 : const Real max_z = box.second(2);
910 :
911 : std::array<Point, 8> box_corners = {{Point(min_x, min_y, min_z),
912 : Point(max_x, min_y, min_z),
913 : Point(min_x, max_y, min_z),
914 : Point(max_x, max_y, min_z),
915 : Point(min_x, min_y, max_z),
916 : Point(max_x, min_y, max_z),
917 : Point(min_x, max_y, max_z),
918 376520 : Point(max_x, max_y, max_z)}};
919 :
920 : // transform each corner
921 3388680 : for (auto & corner : box_corners)
922 3012160 : corner = transform(corner);
923 :
924 : // Create new bounding box
925 376520 : Point new_box_min = box_corners[0];
926 376520 : Point new_box_max = new_box_min;
927 3012160 : for (const auto p : make_range(1, 8))
928 10542560 : for (const auto d : make_range(Moose::dim))
929 : {
930 7906920 : const Point & pt = box_corners[p];
931 7906920 : if (new_box_min(d) > pt(d))
932 2292 : new_box_min(d) = pt(d);
933 :
934 7906920 : if (new_box_max(d) < pt(d))
935 1065095 : new_box_max(d) = pt(d);
936 : }
937 376520 : box.first = new_box_min;
938 376520 : box.second = new_box_max;
939 376520 : }
940 :
941 : BoundingBox
942 318895 : MultiApp::getBoundingBox(unsigned int app,
943 : bool displaced_mesh,
944 : const MultiAppCoordTransform * const coord_transform)
945 : {
946 318895 : if (!_has_an_app)
947 0 : mooseError("No app for ", name(), " on processor ", _orig_rank);
948 :
949 318895 : unsigned int local_app = globalAppToLocal(app);
950 318895 : FEProblemBase & fe_problem_base = _apps[local_app]->getExecutioner()->feProblem();
951 8256 : MooseMesh & mesh = (displaced_mesh && fe_problem_base.getDisplacedProblem().get() != NULL)
952 641153 : ? fe_problem_base.getDisplacedProblem()->mesh()
953 318895 : : fe_problem_base.mesh();
954 :
955 : {
956 318895 : Moose::ScopedCommSwapper swapper(_my_comm);
957 318895 : if (displaced_mesh)
958 4128 : _bounding_box[local_app] = MeshTools::create_bounding_box(mesh);
959 : else
960 : {
961 314767 : if (!_has_bounding_box[local_app])
962 : {
963 92 : _bounding_box[local_app] = MeshTools::create_bounding_box(mesh);
964 92 : _has_bounding_box[local_app] = true;
965 : }
966 : }
967 318895 : }
968 318895 : BoundingBox bbox = _bounding_box[local_app];
969 :
970 318895 : Point min = bbox.min();
971 318895 : min -= _bounding_box_padding;
972 318895 : Point max = bbox.max();
973 318895 : max += _bounding_box_padding;
974 :
975 318895 : Point inflation_amount = (max - min) * _inflation;
976 :
977 318895 : Point inflated_min = min - inflation_amount;
978 318895 : Point inflated_max = max + inflation_amount;
979 :
980 318895 : Point shifted_min = inflated_min;
981 318895 : Point shifted_max = inflated_max;
982 :
983 637790 : if ((!coord_transform || coord_transform->skipCoordinateCollapsing()) &&
984 637790 : fe_problem_base.getCoordSystem(*(mesh.meshSubdomains().begin())) == Moose::COORD_RZ)
985 : {
986 : // If the problem is RZ then we're going to invent a box that would cover the whole "3D" app
987 : // FIXME: Assuming all subdomains are the same coordinate system type!
988 308088 : shifted_min(0) = -inflated_max(0);
989 308088 : shifted_min(1) = inflated_min(1);
990 308088 : shifted_min(2) = -inflated_max(0);
991 :
992 308088 : shifted_max(0) = inflated_max(0);
993 308088 : shifted_max(1) = inflated_max(1);
994 308088 : shifted_max(2) = inflated_max(0);
995 : }
996 :
997 318895 : if (coord_transform)
998 : {
999 318895 : BoundingBox transformed_bbox(shifted_min, shifted_max);
1000 318895 : transformBoundingBox(transformed_bbox, *coord_transform);
1001 318895 : return transformed_bbox;
1002 : }
1003 : else
1004 : {
1005 : // This is where the app is located. We need to shift by this amount.
1006 0 : Point p = position(app);
1007 :
1008 : // Shift them to the position they're supposed to be
1009 0 : shifted_min += p;
1010 0 : shifted_max += p;
1011 0 : return BoundingBox(shifted_min, shifted_max);
1012 : }
1013 : }
1014 :
1015 : FEProblemBase &
1016 485331 : MultiApp::appProblemBase(unsigned int app)
1017 : {
1018 485331 : if (!_has_an_app)
1019 0 : mooseError("No app for ", name(), " on processor ", _orig_rank);
1020 :
1021 485331 : unsigned int local_app = globalAppToLocal(app);
1022 :
1023 485331 : return _apps[local_app]->getExecutioner()->feProblem();
1024 : }
1025 :
1026 : FEProblem &
1027 0 : MultiApp::appProblem(unsigned int app)
1028 : {
1029 0 : mooseDeprecated(
1030 : "MultiApp::appProblem() is deprecated, call MultiApp::appProblemBase() instead.\n");
1031 0 : if (!_has_an_app)
1032 0 : mooseError("No app for ", name(), " on processor ", _orig_rank);
1033 :
1034 0 : unsigned int local_app = globalAppToLocal(app);
1035 :
1036 0 : return dynamic_cast<FEProblem &>(_apps[local_app]->getExecutioner()->feProblem());
1037 : }
1038 :
1039 : const UserObject &
1040 167775 : MultiApp::appUserObjectBase(unsigned int app, const std::string & name)
1041 : {
1042 167775 : if (!_has_an_app)
1043 0 : mooseError("No app for ", MultiApp::name(), " on processor ", _orig_rank);
1044 :
1045 167775 : return appProblemBase(app).getUserObjectBase(name);
1046 : }
1047 :
1048 : Real
1049 3700 : MultiApp::appPostprocessorValue(unsigned int app, const std::string & name)
1050 : {
1051 3700 : if (!_has_an_app)
1052 0 : mooseError("No app for ", MultiApp::name(), " on processor ", _orig_rank);
1053 :
1054 3700 : return appProblemBase(app).getPostprocessorValueByName(name);
1055 : }
1056 :
1057 : NumericVector<Number> &
1058 849 : MultiApp::appTransferVector(unsigned int app, std::string var_name)
1059 : {
1060 849 : return *(appProblemBase(app).getSystem(var_name).solution);
1061 : }
1062 :
1063 : bool
1064 136 : MultiApp::isFirstLocalRank() const
1065 : {
1066 136 : return _rank_config.is_first_local_rank;
1067 : }
1068 :
1069 : bool
1070 814258 : MultiApp::hasLocalApp(unsigned int global_app) const
1071 : {
1072 814258 : if (_has_an_app && global_app >= _first_local_app &&
1073 726817 : global_app <= _first_local_app + (_my_num_apps - 1))
1074 641514 : return true;
1075 :
1076 172744 : return false;
1077 : }
1078 :
1079 : MooseApp *
1080 11521 : MultiApp::localApp(unsigned int local_app)
1081 : {
1082 : mooseAssert(local_app < _apps.size(), "Index out of range: " + Moose::stringify(local_app));
1083 11521 : return _apps[local_app].get();
1084 : }
1085 :
1086 : void
1087 69 : MultiApp::resetApp(unsigned int global_app, Real time)
1088 : {
1089 69 : TIME_SECTION(_reset_timer);
1090 :
1091 69 : Moose::ScopedCommSwapper swapper(_my_comm);
1092 :
1093 69 : if (hasLocalApp(global_app))
1094 : {
1095 69 : unsigned int local_app = globalAppToLocal(global_app);
1096 :
1097 : // Extract the file numbers from the output, so that the numbering is maintained after reset
1098 69 : std::map<std::string, unsigned int> m = _apps[local_app]->getOutputWarehouse().getFileNumbers();
1099 :
1100 69 : createApp(local_app, time);
1101 :
1102 : // Reset the file numbers of the newly reset apps
1103 69 : _apps[local_app]->getOutputWarehouse().setFileNumbers(m);
1104 69 : }
1105 69 : }
1106 :
1107 : void
1108 54 : MultiApp::moveApp(unsigned int global_app, Point p)
1109 : {
1110 54 : if (_use_positions)
1111 : {
1112 54 : _positions[global_app] = p;
1113 :
1114 54 : if (hasLocalApp(global_app))
1115 : {
1116 54 : unsigned int local_app = globalAppToLocal(global_app);
1117 :
1118 54 : if (_output_in_position)
1119 51 : _apps[local_app]->setOutputPosition(p);
1120 54 : if (_run_in_position)
1121 6 : paramError("run_in_position", "Moving apps and running apps in position is not supported");
1122 : }
1123 : }
1124 51 : }
1125 :
1126 : void
1127 19 : MultiApp::parentOutputPositionChanged()
1128 : {
1129 19 : if (_use_positions && _output_in_position)
1130 38 : for (unsigned int i = 0; i < _apps.size(); i++)
1131 19 : _apps[i]->setOutputPosition(_app.getOutputPosition() + _positions[_first_local_app + i]);
1132 19 : }
1133 :
1134 : void
1135 12203 : MultiApp::createApp(unsigned int i, Real start_time)
1136 : {
1137 : // Delete the old app if we're resetting
1138 12203 : if (_apps[i])
1139 69 : _apps[i].reset();
1140 :
1141 : // Define the app name
1142 12203 : const std::string multiapp_name = getMultiAppName(name(), _first_local_app + i, _total_num_apps);
1143 12203 : std::string full_name;
1144 :
1145 : // Only add parent name if the parent is not the main app
1146 12203 : if (_app.multiAppLevel() > 0)
1147 504 : full_name = _app.name() + "_" + multiapp_name;
1148 : else
1149 11699 : full_name = multiapp_name;
1150 :
1151 12203 : InputParameters app_params = AppFactory::instance().getValidParams(_app_type);
1152 24406 : app_params.set<FEProblemBase *>("_parent_fep") = &_fe_problem;
1153 24406 : app_params.set<std::unique_ptr<Backup> *>("_initial_backup") = &_sub_app_backups[i];
1154 :
1155 : // Build the CommandLine with the relevant options for this subapp and add the
1156 : // cli args from the input file
1157 12203 : std::vector<std::string> input_cli_args;
1158 12203 : if (cliArgs().size() > 0 || _cli_args_from_file.size() > 0)
1159 3177 : input_cli_args = getCommandLineArgs(i);
1160 : // FullSolveMultiApp (and its derived classes) always performs a complete fresh
1161 : // solve on every execution, so the parent's recovery-related global CLI params
1162 : // must not be propagated to those sub-apps.
1163 : const std::set<std::string> recover_exclude =
1164 12203 : propagateRecoverToSubApps()
1165 12203 : ? std::set<std::string>{}
1166 29019 : : std::set<std::string>{"recover", "test_checkpoint_half_transient"};
1167 : // This will mark all hit CLI command line parameters that are passed to subapps
1168 : // as used within the parent app (_app)
1169 12203 : auto app_cli = _app.commandLine()->initSubAppCommandLine(
1170 12203 : name(), multiapp_name, input_cli_args, recover_exclude);
1171 12203 : app_cli->parse();
1172 :
1173 12203 : if (_fe_problem.verboseMultiApps())
1174 849 : _console << COLOR_CYAN << "Creating MultiApp " << name() << " of type " << _app_type
1175 849 : << " of level " << _app.multiAppLevel() + 1 << " and number " << _first_local_app + i
1176 849 : << " on processor " << processor_id() << " with full name " << full_name
1177 849 : << COLOR_DEFAULT << std::endl;
1178 24406 : app_params.set<unsigned int>("_multiapp_level") = _app.multiAppLevel() + 1;
1179 24406 : app_params.set<unsigned int>("_multiapp_number") = _first_local_app + i;
1180 24406 : app_params.set<const MooseMesh *>("_master_mesh") = &_fe_problem.mesh();
1181 : #ifdef MOOSE_MFEM_ENABLED
1182 : // MFEM device must only be set once across all apps
1183 : // FIXME: this required that the base app is an MFEM app; itwill
1184 : // still fail if multiple MFEM sub-apps are launched from a libMesh base app
1185 9417 : if (i == 0)
1186 : {
1187 12404 : app_params.set<std::shared_ptr<mfem::Device>>("_mfem_device") =
1188 18606 : _app.getMFEMDevice(Moose::PassKey<MultiApp>());
1189 6202 : const auto & mfem_device_set = _app.getMFEMDevices(Moose::PassKey<MultiApp>());
1190 12404 : app_params.set<std::set<std::string>>("_mfem_devices") = mfem_device_set;
1191 : }
1192 : else
1193 : {
1194 6430 : app_params.set<std::shared_ptr<mfem::Device>>("_mfem_device") =
1195 9645 : _apps[0]->getMFEMDevice(Moose::PassKey<MultiApp>());
1196 3215 : const auto & mfem_device_set = _apps[0]->getMFEMDevices(Moose::PassKey<MultiApp>());
1197 6430 : app_params.set<std::set<std::string>>("_mfem_devices") = mfem_device_set;
1198 : }
1199 : #endif
1200 61015 : if (getParam<bool>("clone_master_mesh") || getParam<bool>("clone_parent_mesh"))
1201 : {
1202 781 : if (_fe_problem.verboseMultiApps())
1203 0 : _console << COLOR_CYAN << "Cloned parent app mesh will be used for MultiApp " << name()
1204 0 : << COLOR_DEFAULT << std::endl;
1205 781 : app_params.set<bool>("_use_master_mesh") = true;
1206 781 : auto displaced_problem = _fe_problem.getDisplacedProblem();
1207 781 : if (displaced_problem)
1208 0 : app_params.set<const MooseMesh *>("_master_displaced_mesh") = &displaced_problem->mesh();
1209 781 : }
1210 :
1211 : // If only one input file was provided, use it for all the solves
1212 12203 : const auto input_index = _input_files.size() == 1 ? 0 : _first_local_app + i;
1213 12203 : const auto & input_file = _input_files[input_index];
1214 :
1215 : // create new parser tree for the application and parse
1216 12203 : auto parser = std::make_unique<Parser>(input_file);
1217 12203 : parser->setCommandLineParams(app_cli->buildHitParams());
1218 12203 : parser->parse();
1219 :
1220 : // Checks on app type
1221 12203 : const auto & app_type = parser->getAppType();
1222 12203 : if (app_type.empty() && _app_type.empty())
1223 0 : mooseWarning("The application type is not specified for ",
1224 : full_name,
1225 : ". Please use [Application] block to specify the application type.");
1226 12203 : if (!app_type.empty() && app_type != _app_type && !AppFactory::instance().isRegistered(app_type))
1227 3 : mooseError("In the ",
1228 : full_name,
1229 : ", '",
1230 : app_type,
1231 : "' is not a registered application. The registered application is named: '",
1232 3 : _app_type,
1233 : "'. Please double check the [Application] block to make sure the correct "
1234 : "application is provided. \n");
1235 :
1236 12200 : if (parser->getAppType().empty())
1237 12188 : parser->setAppType(_app_type);
1238 :
1239 24400 : app_params.set<std::shared_ptr<Parser>>("_parser") = std::move(parser);
1240 24400 : app_params.set<std::shared_ptr<CommandLine>>("_command_line") = std::move(app_cli);
1241 12200 : _apps[i] = AppFactory::instance().create(_app_type, full_name, app_params, _my_comm);
1242 12200 : auto & app = _apps[i];
1243 :
1244 12200 : app->setGlobalTimeOffset(start_time);
1245 12200 : app->setOutputFileNumbers(_app.getOutputWarehouse().getFileNumbers());
1246 12200 : app->setRestart(_app.isRestarting());
1247 12200 : app->setRecover(propagateRecoverToSubApps() && _app.isRecovering());
1248 :
1249 36600 : if (_use_positions && getParam<bool>("output_in_position"))
1250 3359 : app->setOutputPosition(_app.getOutputPosition() + _positions[_first_local_app + i]);
1251 12200 : if (_output_in_position && _run_in_position)
1252 6 : paramError("run_in_position",
1253 : "Sub-apps are already displaced, so they are already output in position");
1254 :
1255 : // Update the MultiApp level for the app that was just created
1256 12197 : app->setupOptions();
1257 : // if multiapp does not have file base in Outputs input block, output file base will
1258 : // be empty here since setupOptions() does not set the default file base with the multiapp
1259 : // input file name. Parent app will create the default file base for multiapp by taking the
1260 : // output base of the parent app problem and appending the name of the multiapp plus a number to
1261 : // it
1262 12197 : if (app->getOutputFileBase().empty())
1263 12161 : setAppOutputFileBase(i);
1264 12197 : preRunInputFile();
1265 36591 : if (_app.multiAppLevel() > getParam<unsigned int>("max_multiapp_level"))
1266 3 : mooseError("Maximum multiapp level has been reached. This is likely caused by an infinite loop "
1267 : "in your multiapp system. If additional multiapp levels are needed, "
1268 : "max_multiapp_level can be specified in the MuliApps block.");
1269 :
1270 : // Transfer coupling relaxation information to the subapps
1271 24388 : _apps[i]->fixedPointConfig().sub_relaxation_factor = getParam<Real>("relaxation_factor");
1272 12194 : _apps[i]->fixedPointConfig().sub_transformed_vars =
1273 36582 : getParam<std::vector<std::string>>("transformed_variables");
1274 : // Handle deprecated parameter
1275 36582 : if (!parameters().isParamSetByAddParam("relaxed_variables"))
1276 0 : _apps[i]->fixedPointConfig().sub_transformed_vars =
1277 0 : getParam<std::vector<std::string>>("relaxed_variables");
1278 12194 : _apps[i]->fixedPointConfig().sub_transformed_pps =
1279 36582 : getParam<std::vector<PostprocessorName>>("transformed_postprocessors");
1280 :
1281 12194 : app->runInputFile();
1282 12158 : auto fixed_point_solve = &(_apps[i]->getExecutioner()->fixedPointSolve());
1283 12158 : if (fixed_point_solve)
1284 12158 : fixed_point_solve->allocateStorage(false);
1285 :
1286 : // Transform the app mesh if requested
1287 12158 : if (_run_in_position)
1288 : {
1289 246 : if (usingPositions())
1290 492 : app->getExecutioner()->feProblem().coordTransform().transformMesh(
1291 246 : app->getExecutioner()->feProblem().mesh(), _positions[_first_local_app + i]);
1292 : else
1293 0 : app->getExecutioner()->feProblem().coordTransform().transformMesh(
1294 0 : app->getExecutioner()->feProblem().mesh(), Point(0, 0, 0));
1295 : }
1296 36970 : }
1297 :
1298 : std::vector<std::string>
1299 3177 : MultiApp::getCommandLineArgs(const unsigned int local_app)
1300 : {
1301 3177 : const auto cla = cliArgs();
1302 3177 : auto cli_args_param = _cli_args_param;
1303 3177 : std::string combined_args;
1304 :
1305 : // Single set of args from cliArgs() to be provided to all apps
1306 3177 : if (cla.size() == 1)
1307 2383 : combined_args = cla[0];
1308 : // Single "cli_args_files" file to be provided to all apps
1309 794 : else if (_cli_args_from_file.size() == 1)
1310 : {
1311 0 : cli_args_param = "cli_args_files";
1312 0 : combined_args = _cli_args_from_file[0];
1313 : }
1314 : // Unique set of args from cliArgs() to be provided to each app
1315 794 : else if (cla.size())
1316 686 : combined_args = cla[local_app + _first_local_app];
1317 : // Unique set of args from "cli_args_files" to be provided to all apps
1318 : else
1319 : {
1320 108 : cli_args_param = "cli_args_files";
1321 108 : combined_args = _cli_args_from_file[local_app + _first_local_app];
1322 : }
1323 :
1324 : // Remove all of the beginning and end whitespace so we can recognize truly empty
1325 3177 : combined_args = MooseUtils::trim(combined_args);
1326 :
1327 : // MooseUtils::split will return a single empty entry if there is nothing,
1328 : // so exit early if we have nothing
1329 3177 : if (combined_args.empty())
1330 0 : return {};
1331 :
1332 : // Split the argument into a vector of arguments, and make sure
1333 : // that we don't have any empty arguments
1334 6354 : const auto args = MooseUtils::split(combined_args, ";");
1335 10545 : for (const auto & arg : args)
1336 : {
1337 7368 : if (arg.empty())
1338 : {
1339 0 : const auto error = "An empty MultiApp command line argument was provided. Your "
1340 : "combined command line string has a ';' with no argument after it.";
1341 0 : if (cli_args_param)
1342 0 : paramError(*cli_args_param, error);
1343 : else
1344 0 : mooseError(error);
1345 : }
1346 : }
1347 :
1348 3177 : return args;
1349 3177 : }
1350 :
1351 : LocalRankConfig
1352 8950 : rankConfig(processor_id_type rank,
1353 : processor_id_type nprocs,
1354 : dof_id_type napps,
1355 : processor_id_type min_app_procs,
1356 : processor_id_type max_app_procs,
1357 : bool batch_mode)
1358 : {
1359 8950 : if (min_app_procs > nprocs)
1360 0 : mooseError("minimum number of procs per app is higher than the available number of procs");
1361 8950 : else if (min_app_procs > max_app_procs)
1362 0 : mooseError("minimum number of procs per app must be lower than the max procs per app");
1363 :
1364 : mooseAssert(rank < nprocs, "rank must be smaller than the number of procs");
1365 :
1366 : // A "slot" is a group of procs/ranks that are grouped together to run a
1367 : // single (sub)app/sim in parallel.
1368 :
1369 : const processor_id_type slot_size =
1370 8950 : std::max(std::min(cast_int<processor_id_type>(nprocs / napps), max_app_procs), min_app_procs);
1371 8950 : const processor_id_type nslots = std::min(
1372 17900 : nprocs / slot_size,
1373 8950 : cast_int<processor_id_type>(std::min(
1374 8950 : static_cast<dof_id_type>(std::numeric_limits<processor_id_type>::max()), napps)));
1375 : mooseAssert(nprocs >= (nslots * slot_size),
1376 : "Ensure that leftover procs is represented by an unsigned type");
1377 8950 : const processor_id_type leftover_procs = nprocs - nslots * slot_size;
1378 8950 : const dof_id_type apps_per_slot = napps / nslots;
1379 8950 : const dof_id_type leftover_apps = napps % nslots;
1380 :
1381 8950 : std::vector<int> slot_for_rank(nprocs);
1382 8950 : processor_id_type slot = 0;
1383 8950 : processor_id_type procs_in_slot = 0;
1384 22483 : for (processor_id_type rankiter = 0; rankiter <= rank; rankiter++)
1385 : {
1386 13533 : if (slot < nslots)
1387 13313 : slot_for_rank[rankiter] = cast_int<int>(slot);
1388 : else
1389 220 : slot_for_rank[rankiter] = -1;
1390 13533 : procs_in_slot++;
1391 : // this slot keeps growing until we reach slot size plus possibly an extra
1392 : // proc if there were any leftover from the slotization of nprocs - this
1393 : // must also make sure we don't go over max app procs.
1394 13533 : if (procs_in_slot == slot_size + 1 * (slot < leftover_procs && slot_size < max_app_procs))
1395 : {
1396 9289 : procs_in_slot = 0;
1397 9289 : slot++;
1398 : }
1399 : }
1400 :
1401 8950 : if (slot_for_rank[rank] < 0)
1402 : // ranks assigned a negative slot don't have any apps running on them.
1403 88 : return {0, 0, 0, 0, false, 0};
1404 8862 : const processor_id_type slot_num = cast_int<processor_id_type>(slot_for_rank[rank]);
1405 :
1406 8862 : const bool is_first_local_rank = rank == 0 || (slot_for_rank[rank - 1] != slot_for_rank[rank]);
1407 8862 : const dof_id_type n_local_apps = apps_per_slot + 1 * (slot_num < leftover_apps);
1408 :
1409 8862 : processor_id_type my_first_rank = 0;
1410 11782 : for (processor_id_type rankiter = rank; rankiter > 0; rankiter--)
1411 3238 : if (slot_for_rank[rank] != slot_for_rank[rankiter])
1412 : {
1413 318 : my_first_rank = cast_int<processor_id_type>(slot_for_rank[rankiter + 1]);
1414 318 : break;
1415 : }
1416 :
1417 8862 : dof_id_type app_index = 0;
1418 10485 : for (processor_id_type slot = 0; slot < slot_num; slot++)
1419 : {
1420 1623 : const dof_id_type num_slot_apps = apps_per_slot + 1 * (slot < leftover_apps);
1421 1623 : app_index += num_slot_apps;
1422 : }
1423 :
1424 8862 : if (batch_mode)
1425 347 : return {n_local_apps, app_index, 1, slot_num, is_first_local_rank, my_first_rank};
1426 8515 : return {n_local_apps, app_index, n_local_apps, app_index, is_first_local_rank, my_first_rank};
1427 8950 : }
1428 :
1429 : void
1430 8102 : MultiApp::buildComm()
1431 : {
1432 : int ierr;
1433 :
1434 8102 : ierr = MPI_Comm_size(_communicator.get(), &_orig_num_procs);
1435 8102 : mooseCheckMPIErr(ierr);
1436 8102 : ierr = MPI_Comm_rank(_communicator.get(), &_orig_rank);
1437 8102 : mooseCheckMPIErr(ierr);
1438 :
1439 : #ifdef LIBMESH_HAVE_SYS_UTSNAME_H
1440 : struct utsname sysInfo;
1441 8102 : uname(&sysInfo);
1442 8102 : _node_name = sysInfo.nodename;
1443 : #else
1444 : _node_name = "Unknown";
1445 : #endif
1446 :
1447 : int rank;
1448 8102 : ierr = MPI_Comm_rank(_communicator.get(), &rank);
1449 8102 : mooseCheckMPIErr(ierr);
1450 :
1451 8102 : _my_num_apps = _rank_config.num_local_apps;
1452 8102 : _first_local_app = _rank_config.first_local_app_index;
1453 :
1454 8102 : _has_an_app = _rank_config.num_local_apps > 0;
1455 8102 : if (_rank_config.first_local_app_index >= _total_num_apps)
1456 0 : mooseError("Internal error, a processor has an undefined app.");
1457 :
1458 8102 : if (_has_an_app)
1459 : {
1460 8050 : _communicator.split(_rank_config.first_local_app_index, rank, _my_communicator);
1461 8050 : ierr = MPI_Comm_rank(_my_comm, &_my_rank);
1462 8050 : mooseCheckMPIErr(ierr);
1463 : }
1464 : else
1465 : {
1466 52 : _communicator.split(MPI_UNDEFINED, rank, _my_communicator);
1467 52 : _my_rank = 0;
1468 : }
1469 8102 : }
1470 :
1471 : unsigned int
1472 804425 : MultiApp::globalAppToLocal(unsigned int global_app)
1473 : {
1474 804425 : if (global_app >= _first_local_app && global_app <= _first_local_app + (_my_num_apps - 1))
1475 804425 : return global_app - _first_local_app;
1476 :
1477 0 : std::stringstream ss;
1478 0 : ss << "Requesting app " << global_app << ", but processor " << processor_id() << " ";
1479 0 : if (_my_num_apps == 0)
1480 0 : ss << "does not own any apps";
1481 0 : else if (_my_num_apps == 1)
1482 0 : ss << "owns app " << _first_local_app;
1483 : else
1484 0 : ss << "owns apps " << _first_local_app << "-" << _first_local_app + (_my_num_apps - 1);
1485 0 : ss << ".";
1486 0 : mooseError("Invalid global_app!\n", ss.str());
1487 : return 0;
1488 0 : }
1489 :
1490 : void
1491 12197 : MultiApp::preRunInputFile()
1492 : {
1493 12197 : }
1494 :
1495 : void
1496 14383 : MultiApp::addAssociatedTransfer(MultiAppTransfer & transfer)
1497 : {
1498 14383 : _associated_transfers.push_back(&transfer);
1499 14383 : }
1500 :
1501 : void
1502 0 : MultiApp::setAppOutputFileBase()
1503 : {
1504 0 : for (unsigned int i = 0; i < _my_num_apps; ++i)
1505 0 : setAppOutputFileBase(i);
1506 0 : }
1507 :
1508 : std::vector<std::string>
1509 23479 : MultiApp::cliArgs() const
1510 : {
1511 : // So that we can error out with paramError("cli_args", ...);
1512 23479 : _cli_args_param = "cli_args";
1513 46958 : return std::vector<std::string>(_cli_args.begin(), _cli_args.end());
1514 : }
1515 :
1516 : void
1517 12161 : MultiApp::setAppOutputFileBase(unsigned int index)
1518 : {
1519 : const std::string multiapp_name =
1520 12161 : getMultiAppName(name(), _first_local_app + index, _total_num_apps);
1521 12161 : _apps[index]->setOutputFileBase(_app.getOutputFileBase() + "_" + multiapp_name);
1522 12161 : }
1523 :
1524 : std::string
1525 24364 : MultiApp::getMultiAppName(const std::string & base_name, dof_id_type index, dof_id_type total)
1526 : {
1527 24364 : std::ostringstream multiapp_name;
1528 48728 : multiapp_name << base_name << std::setw(std::ceil(std::log10(total))) << std::setprecision(0)
1529 24364 : << std::setfill('0') << std::right << index;
1530 48728 : return multiapp_name.str();
1531 24364 : }
1532 :
1533 : const Point &
1534 199558 : MultiApp::position(unsigned int app) const
1535 : {
1536 : // If we're not using positions, it won't have changed
1537 199558 : if (_positions_objs.empty())
1538 184181 : return _positions[app];
1539 : else
1540 : // Find which Positions object is specifying it, and query a potentially updated value
1541 15377 : return _positions_objs[app]->getPosition(app - _positions_index_offsets[app], false);
1542 : }
1543 :
1544 : void
1545 2742 : dataStore(std::ostream & stream, SubAppBackups & backups, void * context)
1546 : {
1547 2742 : MultiApp * multi_app = static_cast<MultiApp *>(context);
1548 : mooseAssert(multi_app, "Not set");
1549 :
1550 2742 : multi_app->backup();
1551 :
1552 2742 : dataStore(stream, static_cast<std::vector<std::unique_ptr<Backup>> &>(backups), nullptr);
1553 2742 : }
1554 :
1555 : void
1556 665 : dataLoad(std::istream & stream, SubAppBackups & backups, void * context)
1557 : {
1558 665 : MultiApp * multi_app = static_cast<MultiApp *>(context);
1559 : mooseAssert(multi_app, "Not set");
1560 :
1561 665 : dataLoad(stream, static_cast<std::vector<std::unique_ptr<Backup>> &>(backups), nullptr);
1562 :
1563 665 : multi_app->restore();
1564 665 : }
|
