Line data Source code
1 : //* This file is part of the MOOSE framework
2 : //* https://mooseframework.inl.gov
3 : //*
4 : //* All rights reserved, see COPYRIGHT for full restrictions
5 : //* https://github.com/idaholab/moose/blob/master/COPYRIGHT
6 : //*
7 : //* Licensed under LGPL 2.1, please see LICENSE for details
8 : //* https://www.gnu.org/licenses/lgpl-2.1.html
9 :
10 : // MOOSE includes
11 : #include "MultiApp.h"
12 :
13 : #include "AppFactory.h"
14 : #include "AuxiliarySystem.h"
15 : #include "DisplacedProblem.h"
16 : #include "Console.h"
17 : #include "Executioner.h"
18 : #include "FEProblem.h"
19 : #include "MooseMesh.h"
20 : #include "MooseUtils.h"
21 : #include "OutputWarehouse.h"
22 : #include "SetupInterface.h"
23 : #include "UserObject.h"
24 : #include "CommandLine.h"
25 : #include "Conversion.h"
26 : #include "NonlinearSystemBase.h"
27 : #include "DelimitedFileReader.h"
28 : #include "MooseAppCoordTransform.h"
29 : #include "MultiAppTransfer.h"
30 : #include "Positions.h"
31 : #include "Transient.h"
32 : #include "Backup.h"
33 : #include "Parser.h"
34 :
35 : #include "libmesh/mesh_tools.h"
36 : #include "libmesh/numeric_vector.h"
37 :
38 : // C++ includes
39 : #include <fstream>
40 : #include <iomanip>
41 : #include <iterator>
42 : #include <algorithm>
43 :
44 : // Call to "uname"
45 : #ifdef LIBMESH_HAVE_SYS_UTSNAME_H
46 : #include <sys/utsname.h>
47 : #endif
48 :
49 : InputParameters
50 86609 : MultiApp::validParams()
51 : {
52 86609 : InputParameters params = MooseObject::validParams();
53 86609 : params += SetupInterface::validParams();
54 :
55 173218 : params.addParam<bool>("use_displaced_mesh",
56 173218 : false,
57 : "Whether or not this object should use the "
58 : "displaced mesh for computation. Note that "
59 : "in the case this is true but no "
60 : "displacements are provided in the Mesh block "
61 : "the undisplaced mesh will still be used.");
62 :
63 86609 : std::ostringstream app_types_strings;
64 173236 : for (const auto & name_bi_pair : AppFactory::instance().registeredObjects())
65 86627 : app_types_strings << name_bi_pair.first << " ";
66 173218 : MooseEnum app_types_options(app_types_strings.str(), "", true);
67 :
68 : // Dynamic loading
69 346436 : params.addParam<MooseEnum>("app_type",
70 : app_types_options,
71 : "The type of application to build (applications not "
72 : "registered can be loaded with dynamic libraries. Parent "
73 : "application type will be used if not provided.");
74 346436 : params.addParam<std::string>("library_path",
75 : "",
76 : "Path to search for dynamic libraries (please "
77 : "avoid committing absolute paths in addition to "
78 : "MOOSE_LIBRARY_PATH)");
79 346436 : params.addParam<std::string>(
80 : "library_name",
81 : "",
82 : "The file name of the library (*.la file) that will be dynamically loaded.");
83 259827 : params.addParam<bool>("library_load_dependencies",
84 173218 : false,
85 : "Tells MOOSE to manually load library dependencies. This should not be "
86 : "necessary and is here for debugging/troubleshooting.");
87 :
88 : // Subapp positions
89 346436 : params.addParam<std::vector<Point>>(
90 : "positions",
91 : "The positions of the App locations. Each set of 3 values will represent a "
92 : "Point. This and 'positions_file' cannot be both supplied. If this and "
93 : "'positions_file'/'_objects' are not supplied, a single position (0,0,0) will be used");
94 346436 : params.addParam<std::vector<FileName>>("positions_file",
95 : "Filename(s) that should be looked in for positions. Each"
96 : " set of 3 values in that file will represent a Point. "
97 : "This and 'positions(_objects)' cannot be both supplied");
98 346436 : params.addParam<std::vector<PositionsName>>("positions_objects",
99 : "The name of a Positions object that will contain "
100 : "the locations of the sub-apps created. This and "
101 : "'positions(_file)' cannot be both supplied");
102 259827 : params.addParam<bool>(
103 : "output_in_position",
104 173218 : false,
105 : "If true this will cause the output from the MultiApp to be 'moved' by its position vector");
106 259827 : params.addParam<bool>(
107 : "run_in_position",
108 173218 : false,
109 : "If true this will cause the mesh from the MultiApp to be 'moved' by its position vector");
110 :
111 346436 : params.addRequiredParam<std::vector<FileName>>(
112 : "input_files",
113 : "The input file for each App. If this parameter only contains one input file "
114 : "it will be used for all of the Apps. When using 'positions_from_file' it is "
115 : "also admissable to provide one input_file per file.");
116 259827 : params.addParam<Real>("bounding_box_inflation",
117 173218 : 0.01,
118 : "Relative amount to 'inflate' the bounding box of this MultiApp.");
119 259827 : params.addParam<Point>("bounding_box_padding",
120 173218 : RealVectorValue(),
121 : "Additional padding added to the dimensions of the bounding box. The "
122 : "values are added to the x, y and z dimension respectively.");
123 :
124 173218 : params.addPrivateParam<MPI_Comm>("_mpi_comm");
125 :
126 : // Set the default execution time
127 173218 : params.set<ExecFlagEnum>("execute_on", true) = EXEC_TIMESTEP_BEGIN;
128 : // Add the POST_ADAPTIVITY execution flag.
129 : #ifdef LIBMESH_ENABLE_AMR
130 86609 : ExecFlagEnum & exec_enum = params.set<ExecFlagEnum>("execute_on");
131 86609 : exec_enum.addAvailableFlags(EXEC_POST_ADAPTIVITY);
132 259827 : params.setDocString("execute_on", exec_enum.getDocString());
133 : #endif
134 :
135 259827 : params.addParam<processor_id_type>("max_procs_per_app",
136 173218 : std::numeric_limits<processor_id_type>::max(),
137 : "Maximum number of processors to give to each App in this "
138 : "MultiApp. Useful for restricting small solves to just a few "
139 : "procs so they don't get spread out");
140 259827 : params.addParam<processor_id_type>("min_procs_per_app",
141 173218 : 1,
142 : "Minimum number of processors to give to each App in this "
143 : "MultiApp. Useful for larger, distributed mesh solves.");
144 259827 : params.addParam<bool>(
145 : "wait_for_first_app_init",
146 173218 : false,
147 : "Create the first sub-application on rank 0, then MPI_Barrier before "
148 : "creating the next N-1 apps (on all ranks). "
149 : "This is only needed if your sub-application needs to perform some setup "
150 : "actions in quiet, without other sub-applications working at the same time.");
151 :
152 259827 : params.addParam<Real>("global_time_offset",
153 173218 : 0,
154 : "The time offset relative to the parent application for the purpose of "
155 : "starting a subapp at a different time from the parent application. The "
156 : "global time will be ahead by the offset specified here.");
157 :
158 : // Resetting subapps
159 346436 : params.addParam<std::vector<Real>>(
160 : "reset_time",
161 : {},
162 : "The time(s) at which to reset Apps given by the 'reset_apps' parameter. "
163 : "Resetting an App means that it is destroyed and recreated, possibly "
164 : "modeling the insertion of 'new' material for that app.");
165 346436 : params.addParam<std::vector<unsigned int>>(
166 : "reset_apps",
167 : {},
168 : "The Apps that will be reset when 'reset_time' is hit. These are the App "
169 : "'numbers' starting with 0 corresponding to the order of the App positions. "
170 : "Resetting an App means that it is destroyed and recreated, possibly modeling "
171 : "the insertion of 'new' material for that app.");
172 :
173 : // Moving subapps
174 259827 : params.addParam<Real>(
175 : "move_time",
176 173218 : std::numeric_limits<Real>::max(),
177 : "The time at which Apps designated by move_apps are moved to move_positions.");
178 :
179 346436 : params.addParam<std::vector<unsigned int>>(
180 : "move_apps",
181 : {},
182 : "Apps, designated by their 'numbers' starting with 0 corresponding to the order "
183 : "of the App positions, to be moved at move_time to move_positions");
184 346436 : params.addParam<std::vector<Point>>(
185 : "move_positions", {}, "The positions corresponding to each move_app.");
186 :
187 346436 : params.addParam<std::vector<CLIArgString>>(
188 : "cli_args",
189 : {},
190 : "Additional command line arguments to pass to the sub apps. If one set is provided the "
191 : "arguments are applied to all, otherwise there must be a set for each sub app.");
192 :
193 346436 : params.addParam<std::vector<FileName>>(
194 : "cli_args_files",
195 : "File names that should be looked in for additional command line arguments "
196 : "to pass to the sub apps. Each line of a file is set to each sub app. If only "
197 : "one line is provided, it will be applied to all sub apps.");
198 :
199 : // Fixed point iterations
200 433045 : params.addRangeCheckedParam<Real>("relaxation_factor",
201 173218 : 1.0,
202 : "relaxation_factor>0 & relaxation_factor<2",
203 : "Fraction of newly computed value to keep."
204 : "Set between 0 and 2.");
205 519654 : params.addDeprecatedParam<std::vector<std::string>>(
206 : "relaxed_variables",
207 : {},
208 : "Use transformed_variables.",
209 : "List of subapp variables to relax during Multiapp coupling iterations");
210 346436 : params.addParam<std::vector<std::string>>(
211 : "transformed_variables",
212 : {},
213 : "List of subapp variables to use coupling algorithm on during Multiapp coupling iterations");
214 346436 : params.addParam<std::vector<PostprocessorName>>(
215 : "transformed_postprocessors",
216 : {},
217 : "List of subapp postprocessors to use coupling "
218 : "algorithm on during Multiapp coupling iterations");
219 259827 : params.addParam<bool>("keep_solution_during_restore",
220 173218 : false,
221 : "This is useful when doing MultiApp coupling iterations. It takes the "
222 : "final solution from the previous coupling iteration"
223 : "and re-uses it as the initial guess for the next coupling iteration");
224 259827 : params.addParam<bool>("keep_aux_solution_during_restore",
225 173218 : false,
226 : "This is useful when doing MultiApp coupling iterations. It takes the "
227 : "final auxiliary solution from the previous coupling iteration"
228 : "and re-uses it as the initial guess for the next coupling iteration");
229 259827 : params.addParam<bool>(
230 : "no_backup_and_restore",
231 173218 : false,
232 : "True to turn off restore for this multiapp. This is useful when doing steady-state "
233 : "Picard iterations where we want to use the solution of previous Picard iteration as the "
234 : "initial guess of the current Picard iteration.");
235 259827 : params.addParam<unsigned int>(
236 : "max_multiapp_level",
237 173218 : 10,
238 : "Integer set by user that will stop the simulation if the multiapp level "
239 : "exceeds it. Useful for preventing infinite loops with multiapp simulations");
240 519654 : params.deprecateParam("no_backup_and_restore", "no_restore", "01/01/2025");
241 :
242 433045 : params.addDeprecatedParam<bool>("clone_master_mesh",
243 173218 : false,
244 : "True to clone parent app mesh and use it for this MultiApp.",
245 : "clone_master_mesh is deprecated, use clone_parent_mesh instead");
246 173218 : params.addParam<bool>(
247 173218 : "clone_parent_mesh", false, "True to clone parent app mesh and use it for this MultiApp.");
248 :
249 173218 : params.addPrivateParam<bool>("use_positions", true);
250 259827 : params.declareControllable("enable");
251 433045 : params.declareControllable("cli_args", {EXEC_PRE_MULTIAPP_SETUP});
252 173218 : params.registerBase("MultiApp");
253 :
254 346436 : params.addParamNamesToGroup("use_displaced_mesh wait_for_first_app_init max_multiapp_level",
255 : "Advanced");
256 346436 : params.addParamNamesToGroup("positions positions_file positions_objects run_in_position "
257 : "output_in_position",
258 : "Positions / transformations of the MultiApp frame of reference");
259 346436 : params.addParamNamesToGroup("min_procs_per_app max_procs_per_app", "Parallelism");
260 346436 : params.addParamNamesToGroup("reset_time reset_apps", "Reset MultiApp");
261 346436 : params.addParamNamesToGroup("move_time move_apps move_positions", "Timed move of MultiApps");
262 346436 : params.addParamNamesToGroup("relaxation_factor transformed_variables transformed_postprocessors "
263 : "keep_solution_during_restore keep_aux_solution_during_restore "
264 : "no_restore",
265 : "Fixed point iteration");
266 346436 : params.addParamNamesToGroup("library_name library_path library_load_dependencies",
267 : "Dynamic loading");
268 259827 : params.addParamNamesToGroup("cli_args cli_args_files", "Passing command line argument");
269 173218 : return params;
270 173218 : }
271 :
272 7637 : MultiApp::MultiApp(const InputParameters & parameters)
273 : : MooseObject(parameters),
274 : SetupInterface(this),
275 : Restartable(this, "MultiApps"),
276 15274 : PerfGraphInterface(this, std::string("MultiApp::") + _name),
277 22911 : _fe_problem(*getCheckedPointerParam<FEProblemBase *>("_fe_problem_base")),
278 27551 : _app_type(isParamValid("app_type") ? std::string(getParam<MooseEnum>("app_type"))
279 2997 : : _fe_problem.getMooseApp().type()),
280 15274 : _use_positions(getParam<bool>("use_positions")),
281 15274 : _input_files(getParam<std::vector<FileName>>("input_files")),
282 15274 : _wait_for_first_app_init(getParam<bool>("wait_for_first_app_init")),
283 7637 : _total_num_apps(0),
284 7637 : _my_num_apps(0),
285 7637 : _first_local_app(0),
286 7637 : _orig_comm(_communicator.get()),
287 7637 : _my_communicator(),
288 7637 : _my_comm(_my_communicator.get()),
289 7637 : _my_rank(0),
290 15274 : _inflation(getParam<Real>("bounding_box_inflation")),
291 15274 : _bounding_box_padding(getParam<Point>("bounding_box_padding")),
292 15274 : _max_procs_per_app(getParam<processor_id_type>("max_procs_per_app")),
293 15274 : _min_procs_per_app(getParam<processor_id_type>("min_procs_per_app")),
294 15274 : _output_in_position(getParam<bool>("output_in_position")),
295 15274 : _global_time_offset(getParam<Real>("global_time_offset")),
296 15274 : _reset_times(getParam<std::vector<Real>>("reset_time")),
297 15274 : _reset_apps(getParam<std::vector<unsigned int>>("reset_apps")),
298 15274 : _reset_happened(false),
299 15274 : _move_time(getParam<Real>("move_time")),
300 15274 : _move_apps(getParam<std::vector<unsigned int>>("move_apps")),
301 15274 : _move_positions(getParam<std::vector<Point>>("move_positions")),
302 7637 : _move_happened(false),
303 7637 : _has_an_app(true),
304 15274 : _cli_args(getParam<std::vector<CLIArgString>>("cli_args")),
305 15274 : _keep_solution_during_restore(getParam<bool>("keep_solution_during_restore")),
306 15274 : _keep_aux_solution_during_restore(getParam<bool>("keep_aux_solution_during_restore")),
307 15274 : _no_restore(getParam<bool>("no_restore")),
308 15274 : _run_in_position(getParam<bool>("run_in_position")),
309 15274 : _sub_app_backups(declareRestartableDataWithContext<SubAppBackups>("sub_app_backups", this)),
310 30548 : _solve_step_timer(registerTimedSection("solveStep", 3, "Executing MultiApps", false)),
311 30548 : _init_timer(registerTimedSection("init", 3, "Initializing MultiApp")),
312 30548 : _backup_timer(registerTimedSection("backup", 3, "Backing Up MultiApp")),
313 30548 : _restore_timer(registerTimedSection("restore", 3, "Restoring MultiApp")),
314 114555 : _reset_timer(registerTimedSection("resetApp", 3, "Resetting MultiApp"))
315 : {
316 36143 : if (parameters.isParamSetByUser("cli_args") && parameters.isParamValid("cli_args") &&
317 11367 : parameters.isParamValid("cli_args_files"))
318 8 : paramError("cli_args",
319 : "'cli_args' and 'cli_args_files' cannot be specified simultaneously in MultiApp ");
320 :
321 15266 : if (!_use_positions && (isParamValid("positions") || isParamValid("positions_file") ||
322 7633 : isParamValid("positions_objects")))
323 0 : paramError("use_positions",
324 : "This MultiApps has been set to not use positions, "
325 : "but a 'positions' parameter has been set.");
326 :
327 22830 : if ((_reset_apps.size() > 0 && _reset_times.size() == 0) ||
328 15197 : (_reset_apps.size() == 0 && _reset_times.size() > 0))
329 0 : mooseError("reset_time and reset_apps may only be specified together");
330 :
331 : // Check that the reset times are sorted by the user
332 7633 : auto sorted_times = _reset_times;
333 7633 : std::sort(sorted_times.begin(), sorted_times.end());
334 7633 : if (_reset_times.size() && _reset_times != sorted_times)
335 8 : paramError("reset_time", "List of reset times must be sorted in increasing order");
336 7629 : }
337 :
338 : void
339 7601 : MultiApp::init(unsigned int num_apps, bool batch_mode)
340 : {
341 7601 : auto config = rankConfig(
342 : processor_id(), n_processors(), num_apps, _min_procs_per_app, _max_procs_per_app, batch_mode);
343 7601 : init(num_apps, config);
344 7597 : }
345 :
346 : void
347 7601 : MultiApp::init(unsigned int num_apps, const LocalRankConfig & config)
348 : {
349 7601 : TIME_SECTION(_init_timer);
350 :
351 7601 : _total_num_apps = num_apps;
352 7601 : _rank_config = config;
353 7601 : buildComm();
354 7601 : _sub_app_backups.resize(_my_num_apps);
355 :
356 7601 : _has_bounding_box.resize(_my_num_apps, false);
357 7601 : _reset_happened.resize(_reset_times.size(), false);
358 7601 : _bounding_box.resize(_my_num_apps);
359 :
360 7601 : if ((_cli_args.size() > 1) && (_total_num_apps != _cli_args.size()))
361 8 : paramError("cli_args",
362 : "The number of items supplied must be 1 or equal to the number of sub apps.");
363 :
364 : // if cliArgs() != _cli_args, then cliArgs() was overridden and we need to check it
365 7597 : auto cla = cliArgs();
366 15194 : if (cla != std::vector<std::string>(_cli_args.begin(), _cli_args.end()))
367 : {
368 13 : if ((cla.size() > 1) && (_total_num_apps != cla.size()))
369 0 : mooseError("The number of items supplied as command line argument to subapps must be 1 or "
370 : "equal to the number of sub apps. Note: you use a multiapp that provides its own "
371 : "command line parameters so the error is not in cli_args");
372 : }
373 7597 : }
374 :
375 : void
376 7625 : MultiApp::setupPositions()
377 : {
378 7625 : if (_use_positions)
379 : {
380 7625 : fillPositions();
381 7601 : init(_positions.size());
382 7597 : createApps();
383 : }
384 7510 : }
385 :
386 : void
387 7597 : MultiApp::createApps()
388 : {
389 7597 : if (!_has_an_app)
390 54 : return;
391 :
392 37715 : TIME_SECTION("createApps", 2, "Instantiating Sub-Apps", false);
393 :
394 : // Read commandLine arguments that will be used when creating apps
395 7543 : readCommandLineArguments();
396 :
397 7523 : Moose::ScopedCommSwapper swapper(_my_comm);
398 :
399 7523 : _apps.resize(_my_num_apps);
400 :
401 : // If the user provided an unregistered app type, see if we can load it dynamically
402 7523 : if (!AppFactory::instance().isRegistered(_app_type))
403 5 : _app.dynamicAppRegistration(_app_type,
404 : getParam<std::string>("library_path"),
405 7 : getParam<std::string>("library_name"),
406 14 : getParam<bool>("library_load_dependencies"));
407 :
408 7520 : bool rank_did_quiet_init = false;
409 7520 : unsigned int local_app = libMesh::invalid_uint;
410 7520 : if (_wait_for_first_app_init)
411 : {
412 16 : if (hasLocalApp(0))
413 : {
414 8 : rank_did_quiet_init = true;
415 8 : local_app = globalAppToLocal(0);
416 8 : createLocalApp(local_app);
417 : }
418 :
419 16 : MPI_Barrier(_orig_comm);
420 : }
421 :
422 19800 : for (unsigned int i = 0; i < _my_num_apps; i++)
423 : {
424 12344 : if (rank_did_quiet_init && i == local_app)
425 8 : continue;
426 12336 : createLocalApp(i);
427 : }
428 7456 : }
429 :
430 : void
431 12344 : MultiApp::createLocalApp(const unsigned int i)
432 : {
433 12344 : createApp(i, _global_time_offset);
434 12280 : }
435 :
436 : void
437 7414 : MultiApp::initialSetup()
438 : {
439 7414 : if (!_use_positions)
440 : // if not using positions, we create the sub-apps in initialSetup instead of right after
441 : // construction of MultiApp
442 0 : createApps();
443 7414 : }
444 :
445 : void
446 7543 : MultiApp::readCommandLineArguments()
447 : {
448 22629 : if (isParamValid("cli_args_files"))
449 : {
450 59 : _cli_args_from_file.clear();
451 :
452 118 : std::vector<FileName> cli_args_files = getParam<std::vector<FileName>>("cli_args_files");
453 118 : std::vector<FileName> input_files = getParam<std::vector<FileName>>("input_files");
454 :
455 : // If we use parameter "cli_args_files", at least one file should be provided
456 59 : if (!cli_args_files.size())
457 8 : paramError("cli_args_files", "You need to provide at least one commandLine argument file ");
458 :
459 : // If we multiple input files, then we need to check if the number of input files
460 : // match with the number of argument files
461 55 : if (cli_args_files.size() != 1 && cli_args_files.size() != input_files.size())
462 12 : paramError("cli_args_files",
463 : "The number of commandLine argument files ",
464 : cli_args_files.size(),
465 : " for MultiApp ",
466 4 : name(),
467 : " must either be only one or match the number of input files ",
468 : input_files.size());
469 :
470 : // Go through all argument files
471 51 : std::vector<std::string> cli_args;
472 107 : for (unsigned int p_file_it = 0; p_file_it < cli_args_files.size(); p_file_it++)
473 : {
474 64 : std::string cli_args_file = cli_args_files[p_file_it];
475 : // Clear up
476 64 : cli_args.clear();
477 : // Read the file on the root processor then broadcast it
478 64 : if (processor_id() == 0)
479 : {
480 52 : MooseUtils::checkFileReadable(cli_args_file);
481 :
482 52 : std::ifstream is(cli_args_file.c_str());
483 : // Read by line rather than space separated like the cli_args parameter
484 52 : std::string line;
485 212 : while (std::getline(is, line))
486 160 : cli_args.push_back(line);
487 :
488 : // We do not allow empty files
489 52 : if (!cli_args.size())
490 8 : paramError("cli_args_files",
491 : "There is no commandLine argument in the commandLine argument file ",
492 : cli_args_file);
493 :
494 : // If we have position files, we need to
495 : // make sure the number of commandLine argument strings
496 : // match with the number of positions
497 48 : if (_npositions_inputfile.size())
498 : {
499 44 : auto num_positions = _npositions_inputfile[p_file_it];
500 : // Check if the number of commandLine argument strings equal to
501 : // the number of positions
502 44 : if (cli_args.size() == 1)
503 0 : for (MooseIndex(num_positions) num = 0; num < num_positions; num++)
504 0 : _cli_args_from_file.push_back(cli_args.front());
505 44 : else if (cli_args.size() == num_positions)
506 160 : for (auto && cli_arg : cli_args)
507 120 : _cli_args_from_file.push_back(cli_arg);
508 4 : else if (cli_args.size() != num_positions)
509 8 : paramError("cli_args_files",
510 : "The number of commandLine argument strings ",
511 : cli_args.size(),
512 : " in the file ",
513 : cli_args_file,
514 : " must either be only one or match the number of positions ",
515 : num_positions);
516 : }
517 : else
518 : {
519 : // If we do not have position files, we will check if the number of
520 : // commandLine argument strings match with the total number of subapps
521 24 : for (auto && cli_arg : cli_args)
522 20 : _cli_args_from_file.push_back(cli_arg);
523 : }
524 44 : }
525 56 : }
526 :
527 : // Broad cast all arguments to everyone
528 43 : _communicator.broadcast(_cli_args_from_file);
529 43 : }
530 :
531 7570 : if (_cli_args_from_file.size() && _cli_args_from_file.size() != 1 &&
532 43 : _cli_args_from_file.size() != _total_num_apps)
533 8 : mooseError(" The number of commandLine argument strings ",
534 4 : _cli_args_from_file.size(),
535 : " must either be only one or match the total "
536 : "number of sub apps ",
537 4 : _total_num_apps);
538 :
539 7523 : if (_cli_args_from_file.size() && cliArgs().size())
540 0 : mooseError("Cannot set commandLine arguments from both input_file and external files");
541 7523 : }
542 :
543 : void
544 7506 : MultiApp::fillPositions()
545 : {
546 7506 : if (_move_apps.size() != _move_positions.size())
547 0 : mooseError("The number of apps to move and the positions to move them to must be the same for "
548 : "MultiApp ",
549 0 : _name);
550 :
551 37530 : if (isParamValid("positions") + isParamValid("positions_file") +
552 22518 : isParamValid("positions_objects") >
553 : 1)
554 4 : mooseError("Only one 'positions' parameter may be specified");
555 :
556 22506 : if (isParamValid("positions"))
557 : {
558 10572 : _positions = getParam<std::vector<Point>>("positions");
559 :
560 5286 : if (_positions.size() < _input_files.size())
561 4 : mooseError("Not enough positions for the number of input files provided in MultiApp ",
562 4 : name());
563 : }
564 6648 : else if (isParamValid("positions_file"))
565 : {
566 188 : std::vector<FileName> positions_files = getParam<std::vector<FileName>>("positions_file");
567 188 : std::vector<FileName> input_files = getParam<std::vector<FileName>>("input_files");
568 :
569 94 : if (input_files.size() != 1 && positions_files.size() != input_files.size())
570 4 : mooseError("Number of input_files for MultiApp ",
571 4 : name(),
572 : " must either be only one or match the number of positions_file files");
573 :
574 : // Clear out the _input_files because we're going to rebuild it
575 90 : if (input_files.size() != 1)
576 30 : _input_files.clear();
577 :
578 206 : for (unsigned int p_file_it = 0; p_file_it < positions_files.size(); p_file_it++)
579 : {
580 120 : std::string positions_file = positions_files[p_file_it];
581 120 : MooseUtils::DelimitedFileReader file(positions_file, &_communicator);
582 120 : file.setFormatFlag(MooseUtils::DelimitedFileReader::FormatFlag::ROWS);
583 120 : file.read();
584 :
585 120 : const std::vector<Point> & data = file.getDataAsPoints();
586 564 : for (const auto & d : data)
587 448 : _positions.push_back(d);
588 :
589 : // Save the number of positions for this input file
590 116 : _npositions_inputfile.push_back(data.size());
591 :
592 564 : for (unsigned int i = 0; i < data.size(); ++i)
593 448 : if (input_files.size() != 1)
594 120 : _input_files.push_back(input_files[p_file_it]);
595 116 : }
596 86 : }
597 6366 : else if (isParamValid("positions_objects"))
598 : {
599 928 : const auto & positions_param_objs = getParam<std::vector<PositionsName>>("positions_objects");
600 928 : const auto & input_files = getParam<std::vector<FileName>>("input_files");
601 :
602 464 : if (input_files.size() != 1 && positions_param_objs.size() != input_files.size())
603 0 : mooseError("Number of input_files for MultiApp ",
604 0 : name(),
605 : " must either be only one or match the number of positions_objects specified");
606 :
607 : // Clear out the _input_files because we're going to rebuild it
608 464 : if (input_files.size() != 1)
609 0 : _input_files.clear();
610 :
611 : // Keeps track of where each positions object start in terms of subapp numbers
612 464 : unsigned int offset = 0;
613 :
614 967 : for (const auto p_obj_it : index_range(positions_param_objs))
615 : {
616 507 : const std::string & positions_name = positions_param_objs[p_obj_it];
617 507 : auto positions_obj = &_fe_problem.getPositionsObject(positions_name);
618 :
619 507 : const auto & data = positions_obj->getPositions(true);
620 :
621 : // Append all positions from this object
622 2200 : for (const auto & d : data)
623 1697 : _positions.push_back(d);
624 :
625 : // Save the number of positions for this input file
626 503 : _npositions_inputfile.push_back(data.size());
627 :
628 503 : if (!positions_obj)
629 0 : paramError("positions_objects",
630 0 : "'" + positions_name + "' is not of the expected type. Should be a Positions");
631 :
632 : // Keep track of which positions is tied to what subapp
633 2200 : for (unsigned int i = 0; i < data.size(); ++i)
634 : {
635 1697 : if (input_files.size() != 1)
636 0 : _input_files.push_back(input_files[p_obj_it]);
637 1697 : _positions_objs.push_back(positions_obj);
638 1697 : _positions_index_offsets.push_back(offset);
639 : }
640 503 : offset += data.size();
641 : }
642 : }
643 : else
644 : {
645 1658 : _positions = {Point()};
646 :
647 1658 : if (_positions.size() < _input_files.size())
648 4 : mooseError("Not enough positions for the number of input files provided in MultiApp ",
649 4 : name());
650 : }
651 :
652 : mooseAssert(_input_files.size() == 1 || _positions.size() == _input_files.size(),
653 : "Number of positions and input files are not the same!");
654 7482 : }
655 :
656 : void
657 67935 : MultiApp::preTransfer(Real /*dt*/, Real target_time)
658 : {
659 : // Get a transient executioner to get a user-set tolerance
660 67935 : Real timestep_tol = 1e-13;
661 67935 : if (dynamic_cast<TransientBase *>(_fe_problem.getMooseApp().getExecutioner()))
662 65028 : timestep_tol =
663 65028 : dynamic_cast<TransientBase *>(_fe_problem.getMooseApp().getExecutioner())->timestepTol();
664 :
665 : // Determination on whether we need to backup the app due to changes below
666 67935 : bool backup_apps = false;
667 :
668 : // First, see if any Apps need to be reset
669 68714 : for (unsigned int i = 0; i < _reset_times.size(); i++)
670 : {
671 855 : if (!_reset_happened[i] && (target_time + timestep_tol >= _reset_times[i]))
672 : {
673 76 : _reset_happened[i] = true;
674 76 : if (_reset_apps.size() > 0)
675 152 : for (auto & app : _reset_apps)
676 76 : resetApp(app);
677 :
678 : // If we reset an application, then we delete the old objects, including the coordinate
679 : // transformation classes. Consequently we need to reset the coordinate transformation classes
680 : // in the associated transfer classes
681 109 : for (auto * const transfer : _associated_transfers)
682 33 : transfer->getAppInfo();
683 :
684 : // Similarly we need to transform the mesh again
685 76 : if (_run_in_position)
686 0 : for (const auto i : make_range(_my_num_apps))
687 : {
688 0 : auto app_ptr = _apps[i];
689 0 : if (usingPositions())
690 0 : app_ptr->getExecutioner()->feProblem().coordTransform().transformMesh(
691 0 : app_ptr->getExecutioner()->feProblem().mesh(), _positions[_first_local_app + i]);
692 : else
693 0 : app_ptr->getExecutioner()->feProblem().coordTransform().transformMesh(
694 0 : app_ptr->getExecutioner()->feProblem().mesh(), Point(0, 0, 0));
695 0 : }
696 :
697 : // If the time step covers multiple reset times, set them all as having 'happened'
698 174 : for (unsigned int j = i; j < _reset_times.size(); j++)
699 98 : if (target_time + timestep_tol >= _reset_times[j])
700 87 : _reset_happened[j] = true;
701 :
702 : // Backup in case the next solve fails
703 76 : backup_apps = true;
704 :
705 76 : break;
706 : }
707 : }
708 :
709 : // Now move any apps that should be moved
710 67935 : if (_use_positions && !_move_happened && target_time + timestep_tol >= _move_time)
711 : {
712 60 : _move_happened = true;
713 116 : for (unsigned int i = 0; i < _move_apps.size(); i++)
714 60 : moveApp(_move_apps[i], _move_positions[i]);
715 :
716 : // Backup in case the next solve fails
717 56 : backup_apps = true;
718 : }
719 :
720 67931 : if (backup_apps)
721 111 : backup();
722 67931 : }
723 :
724 : Executioner *
725 0 : MultiApp::getExecutioner(unsigned int app)
726 : {
727 0 : if (!_has_an_app)
728 0 : mooseError("No app for ", name(), " on processor ", _orig_rank);
729 :
730 0 : return _apps[globalAppToLocal(app)]->getExecutioner();
731 : }
732 :
733 : void
734 4571 : MultiApp::finalize()
735 : {
736 11325 : for (const auto & app_ptr : _apps)
737 : {
738 6754 : auto * executioner = app_ptr->getExecutioner();
739 : mooseAssert(executioner, "Executioner is nullptr");
740 :
741 6754 : executioner->feProblem().execute(EXEC_FINAL);
742 6754 : executioner->feProblem().outputStep(EXEC_FINAL);
743 : }
744 4571 : }
745 :
746 : void
747 4909 : MultiApp::postExecute()
748 : {
749 12281 : for (const auto & app_ptr : _apps)
750 : {
751 7372 : auto * executioner = app_ptr->getExecutioner();
752 : mooseAssert(executioner, "Executioner is nullptr");
753 :
754 7372 : executioner->postExecute();
755 : }
756 4909 : }
757 :
758 : void
759 25114 : MultiApp::backup()
760 : {
761 25114 : TIME_SECTION(_backup_timer);
762 :
763 25114 : if (_fe_problem.verboseMultiApps())
764 1059 : _console << "Backed up MultiApp ... ";
765 :
766 59581 : for (unsigned int i = 0; i < _my_num_apps; i++)
767 34467 : _sub_app_backups[i] = _apps[i]->backup();
768 :
769 25114 : if (_fe_problem.verboseMultiApps())
770 1059 : _console << name() << std::endl;
771 25114 : }
772 :
773 : void
774 46627 : MultiApp::restore(bool force)
775 : {
776 46627 : TIME_SECTION(_restore_timer);
777 :
778 46627 : if (force || needsRestoration())
779 : {
780 : // Must be restarting / recovering from main app so hold off on restoring
781 : // Instead - the restore will happen in sub-apps' initialSetup()
782 : // Note that _backups was already populated by dataLoad() in the main app
783 10346 : if (_fe_problem.getCurrentExecuteOnFlag() == EXEC_INITIAL)
784 552 : return;
785 :
786 : // We temporarily copy and store solutions for all subapps
787 9794 : if (_keep_solution_during_restore)
788 : {
789 355 : _end_solutions.resize(_my_num_apps);
790 :
791 706 : for (unsigned int i = 0; i < _my_num_apps; i++)
792 : {
793 355 : _end_solutions[i].resize(_apps[i]->getExecutioner()->feProblem().numSolverSystems());
794 722 : for (unsigned int j = 0; j < _apps[i]->getExecutioner()->feProblem().numSolverSystems();
795 : j++)
796 : {
797 734 : _end_solutions[i][j] = _apps[i]
798 : ->getExecutioner()
799 367 : ->feProblem()
800 367 : .getSolverSystem(/*solver_sys=*/j)
801 367 : .solution()
802 734 : .clone();
803 : }
804 : auto & sub_multiapps =
805 355 : _apps[i]->getExecutioner()->feProblem().getMultiAppWarehouse().getObjects();
806 :
807 : // multiapps of each subapp should do the same things
808 : // It is implemented recursively
809 366 : for (auto & multi_app : sub_multiapps)
810 15 : multi_app->keepSolutionDuringRestore(_keep_solution_during_restore);
811 : }
812 : }
813 :
814 : // We temporarily copy and store solutions for all subapps
815 9790 : if (_keep_aux_solution_during_restore)
816 : {
817 132 : _end_aux_solutions.resize(_my_num_apps);
818 :
819 264 : for (unsigned int i = 0; i < _my_num_apps; i++)
820 132 : _end_aux_solutions[i] =
821 264 : _apps[i]->getExecutioner()->feProblem().getAuxiliarySystem().solution().clone();
822 : }
823 :
824 9790 : if (_fe_problem.verboseMultiApps())
825 516 : _console << "Restoring MultiApp ... ";
826 :
827 19765 : for (unsigned int i = 0; i < _my_num_apps; i++)
828 : {
829 9975 : _apps[i]->restore(std::move(_sub_app_backups[i]), false);
830 9975 : _sub_app_backups[i] = _apps[i]->finalizeRestore();
831 : mooseAssert(_sub_app_backups[i], "Should have a backup");
832 : }
833 :
834 9790 : if (_fe_problem.verboseMultiApps())
835 516 : _console << name() << std::endl;
836 :
837 : // Now copy the latest solutions back for each subapp
838 9790 : if (_keep_solution_during_restore)
839 : {
840 702 : for (unsigned int i = 0; i < _my_num_apps; i++)
841 : {
842 714 : for (unsigned int j = 0; j < _apps[i]->getExecutioner()->feProblem().numSolverSystems();
843 : j++)
844 : {
845 363 : _apps[i]->getExecutioner()->feProblem().getSolverSystem(/*solver_sys=*/j).solution() =
846 726 : *_end_solutions[i][j];
847 :
848 : // We need to synchronize solution so that local_solution has the right values
849 363 : _apps[i]->getExecutioner()->feProblem().getSolverSystem(/*solver_sys=*/j).update();
850 : }
851 : }
852 :
853 351 : _end_solutions.clear();
854 : }
855 : // Now copy the latest auxiliary solutions back for each subapp
856 9790 : if (_keep_aux_solution_during_restore)
857 : {
858 264 : for (unsigned int i = 0; i < _my_num_apps; i++)
859 : {
860 132 : _apps[i]->getExecutioner()->feProblem().getAuxiliarySystem().solution() =
861 264 : *_end_aux_solutions[i];
862 :
863 : // We need to synchronize solution so that local_solution has the right values
864 132 : _apps[i]->getExecutioner()->feProblem().getAuxiliarySystem().update();
865 : }
866 :
867 132 : _end_aux_solutions.clear();
868 : }
869 :
870 : // Make sure the displaced mesh on the multiapp is up-to-date with displacement variables
871 19765 : for (const auto & app_ptr : _apps)
872 9975 : if (app_ptr->feProblem().getDisplacedProblem())
873 48 : app_ptr->feProblem().getDisplacedProblem()->updateMesh();
874 :
875 : // If we are restoring due to a failed solve, make sure reset the solved state in the sub-apps
876 9790 : if (!getMooseApp().getExecutioner()->lastSolveConverged())
877 1663 : for (auto & app_ptr : _apps)
878 918 : app_ptr->getExecutioner()->fixedPointSolve().clearFixedPointStatus();
879 : }
880 : else
881 : {
882 72562 : for (unsigned int i = 0; i < _my_num_apps; i++)
883 : {
884 36281 : for (auto & sub_app :
885 73033 : _apps[i]->getExecutioner()->feProblem().getMultiAppWarehouse().getObjects())
886 471 : sub_app->restore(false);
887 : }
888 : }
889 46623 : }
890 :
891 : void
892 15 : MultiApp::keepSolutionDuringRestore(bool keep_solution_during_restore)
893 : {
894 45 : if (_pars.isParamSetByUser("keep_solution_during_restore"))
895 8 : paramError("keep_solution_during_restore",
896 : "This parameter should only be provided in parent app");
897 :
898 11 : _keep_solution_during_restore = keep_solution_during_restore;
899 11 : }
900 :
901 : void
902 416465 : MultiApp::transformBoundingBox(BoundingBox & box, const MultiAppCoordTransform & transform)
903 : {
904 416465 : const Real min_x = box.first(0);
905 416465 : const Real max_x = box.second(0);
906 416465 : const Real min_y = box.first(1);
907 416465 : const Real max_y = box.second(1);
908 416465 : const Real min_z = box.first(2);
909 416465 : const Real max_z = box.second(2);
910 :
911 : std::array<Point, 8> box_corners = {{Point(min_x, min_y, min_z),
912 : Point(max_x, min_y, min_z),
913 : Point(min_x, max_y, min_z),
914 : Point(max_x, max_y, min_z),
915 : Point(min_x, min_y, max_z),
916 : Point(max_x, min_y, max_z),
917 : Point(min_x, max_y, max_z),
918 416465 : Point(max_x, max_y, max_z)}};
919 :
920 : // transform each corner
921 3748185 : for (auto & corner : box_corners)
922 3331720 : corner = transform(corner);
923 :
924 : // Create new bounding box
925 416465 : Point new_box_min = box_corners[0];
926 416465 : Point new_box_max = new_box_min;
927 3331720 : for (const auto p : make_range(1, 8))
928 11661020 : for (const auto d : make_range(Moose::dim))
929 : {
930 8745765 : const Point & pt = box_corners[p];
931 8745765 : if (new_box_min(d) > pt(d))
932 2540 : new_box_min(d) = pt(d);
933 :
934 8745765 : if (new_box_max(d) < pt(d))
935 1178964 : new_box_max(d) = pt(d);
936 : }
937 416465 : box.first = new_box_min;
938 416465 : box.second = new_box_max;
939 416465 : }
940 :
941 : BoundingBox
942 354231 : MultiApp::getBoundingBox(unsigned int app,
943 : bool displaced_mesh,
944 : const MultiAppCoordTransform * const coord_transform)
945 : {
946 354231 : if (!_has_an_app)
947 0 : mooseError("No app for ", name(), " on processor ", _orig_rank);
948 :
949 354231 : unsigned int local_app = globalAppToLocal(app);
950 354231 : FEProblemBase & fe_problem_base = _apps[local_app]->getExecutioner()->feProblem();
951 9078 : MooseMesh & mesh = (displaced_mesh && fe_problem_base.getDisplacedProblem().get() != NULL)
952 712146 : ? fe_problem_base.getDisplacedProblem()->mesh()
953 354231 : : fe_problem_base.mesh();
954 :
955 : {
956 354231 : Moose::ScopedCommSwapper swapper(_my_comm);
957 354231 : if (displaced_mesh)
958 4539 : _bounding_box[local_app] = MeshTools::create_bounding_box(mesh);
959 : else
960 : {
961 349692 : if (!_has_bounding_box[local_app])
962 : {
963 102 : _bounding_box[local_app] = MeshTools::create_bounding_box(mesh);
964 102 : _has_bounding_box[local_app] = true;
965 : }
966 : }
967 354231 : }
968 354231 : BoundingBox bbox = _bounding_box[local_app];
969 :
970 354231 : Point min = bbox.min();
971 354231 : min -= _bounding_box_padding;
972 354231 : Point max = bbox.max();
973 354231 : max += _bounding_box_padding;
974 :
975 354231 : Point inflation_amount = (max - min) * _inflation;
976 :
977 354231 : Point inflated_min = min - inflation_amount;
978 354231 : Point inflated_max = max + inflation_amount;
979 :
980 354231 : Point shifted_min = inflated_min;
981 354231 : Point shifted_max = inflated_max;
982 :
983 708462 : if ((!coord_transform || coord_transform->skipCoordinateCollapsing()) &&
984 708462 : fe_problem_base.getCoordSystem(*(mesh.meshSubdomains().begin())) == Moose::COORD_RZ)
985 : {
986 : // If the problem is RZ then we're going to invent a box that would cover the whole "3D" app
987 : // FIXME: Assuming all subdomains are the same coordinate system type!
988 342175 : shifted_min(0) = -inflated_max(0);
989 342175 : shifted_min(1) = inflated_min(1);
990 342175 : shifted_min(2) = -inflated_max(0);
991 :
992 342175 : shifted_max(0) = inflated_max(0);
993 342175 : shifted_max(1) = inflated_max(1);
994 342175 : shifted_max(2) = inflated_max(0);
995 : }
996 :
997 354231 : if (coord_transform)
998 : {
999 354231 : BoundingBox transformed_bbox(shifted_min, shifted_max);
1000 354231 : transformBoundingBox(transformed_bbox, *coord_transform);
1001 354231 : return transformed_bbox;
1002 : }
1003 : else
1004 : {
1005 : // This is where the app is located. We need to shift by this amount.
1006 0 : Point p = position(app);
1007 :
1008 : // Shift them to the position they're supposed to be
1009 0 : shifted_min += p;
1010 0 : shifted_max += p;
1011 0 : return BoundingBox(shifted_min, shifted_max);
1012 : }
1013 : }
1014 :
1015 : FEProblemBase &
1016 518606 : MultiApp::appProblemBase(unsigned int app)
1017 : {
1018 518606 : if (!_has_an_app)
1019 0 : mooseError("No app for ", name(), " on processor ", _orig_rank);
1020 :
1021 518606 : unsigned int local_app = globalAppToLocal(app);
1022 :
1023 518606 : return _apps[local_app]->getExecutioner()->feProblem();
1024 : }
1025 :
1026 : FEProblem &
1027 0 : MultiApp::appProblem(unsigned int app)
1028 : {
1029 0 : mooseDeprecated(
1030 : "MultiApp::appProblem() is deprecated, call MultiApp::appProblemBase() instead.\n");
1031 0 : if (!_has_an_app)
1032 0 : mooseError("No app for ", name(), " on processor ", _orig_rank);
1033 :
1034 0 : unsigned int local_app = globalAppToLocal(app);
1035 :
1036 0 : return dynamic_cast<FEProblem &>(_apps[local_app]->getExecutioner()->feProblem());
1037 : }
1038 :
1039 : const UserObject &
1040 186397 : MultiApp::appUserObjectBase(unsigned int app, const std::string & name)
1041 : {
1042 186397 : if (!_has_an_app)
1043 0 : mooseError("No app for ", MultiApp::name(), " on processor ", _orig_rank);
1044 :
1045 186397 : return appProblemBase(app).getUserObjectBase(name);
1046 : }
1047 :
1048 : Real
1049 4152 : MultiApp::appPostprocessorValue(unsigned int app, const std::string & name)
1050 : {
1051 4152 : if (!_has_an_app)
1052 0 : mooseError("No app for ", MultiApp::name(), " on processor ", _orig_rank);
1053 :
1054 4152 : return appProblemBase(app).getPostprocessorValueByName(name);
1055 : }
1056 :
1057 : NumericVector<Number> &
1058 942 : MultiApp::appTransferVector(unsigned int app, std::string var_name)
1059 : {
1060 942 : return *(appProblemBase(app).getSystem(var_name).solution);
1061 : }
1062 :
1063 : bool
1064 150 : MultiApp::isFirstLocalRank() const
1065 : {
1066 150 : return _rank_config.is_first_local_rank;
1067 : }
1068 :
1069 : bool
1070 870466 : MultiApp::hasLocalApp(unsigned int global_app) const
1071 : {
1072 870466 : if (_has_an_app && global_app >= _first_local_app &&
1073 783743 : global_app <= _first_local_app + (_my_num_apps - 1))
1074 698825 : return true;
1075 :
1076 171641 : return false;
1077 : }
1078 :
1079 : MooseApp *
1080 11187 : MultiApp::localApp(unsigned int local_app)
1081 : {
1082 : mooseAssert(local_app < _apps.size(), "Index out of range: " + Moose::stringify(local_app));
1083 11187 : return _apps[local_app].get();
1084 : }
1085 :
1086 : void
1087 76 : MultiApp::resetApp(unsigned int global_app, Real time)
1088 : {
1089 76 : TIME_SECTION(_reset_timer);
1090 :
1091 76 : Moose::ScopedCommSwapper swapper(_my_comm);
1092 :
1093 76 : if (hasLocalApp(global_app))
1094 : {
1095 76 : unsigned int local_app = globalAppToLocal(global_app);
1096 :
1097 : // Extract the file numbers from the output, so that the numbering is maintained after reset
1098 76 : std::map<std::string, unsigned int> m = _apps[local_app]->getOutputWarehouse().getFileNumbers();
1099 :
1100 76 : createApp(local_app, time);
1101 :
1102 : // Reset the file numbers of the newly reset apps
1103 76 : _apps[local_app]->getOutputWarehouse().setFileNumbers(m);
1104 76 : }
1105 76 : }
1106 :
1107 : void
1108 60 : MultiApp::moveApp(unsigned int global_app, Point p)
1109 : {
1110 60 : if (_use_positions)
1111 : {
1112 60 : _positions[global_app] = p;
1113 :
1114 60 : if (hasLocalApp(global_app))
1115 : {
1116 60 : unsigned int local_app = globalAppToLocal(global_app);
1117 :
1118 60 : if (_output_in_position)
1119 56 : _apps[local_app]->setOutputPosition(p);
1120 60 : if (_run_in_position)
1121 8 : paramError("run_in_position", "Moving apps and running apps in position is not supported");
1122 : }
1123 : }
1124 56 : }
1125 :
1126 : void
1127 21 : MultiApp::parentOutputPositionChanged()
1128 : {
1129 21 : if (_use_positions && _output_in_position)
1130 42 : for (unsigned int i = 0; i < _apps.size(); i++)
1131 21 : _apps[i]->setOutputPosition(_app.getOutputPosition() + _positions[_first_local_app + i]);
1132 21 : }
1133 :
1134 : void
1135 12420 : MultiApp::createApp(unsigned int i, Real start_time)
1136 : {
1137 : // Delete the old app if we're resetting
1138 12420 : if (_apps[i])
1139 76 : _apps[i].reset();
1140 :
1141 : // Define the app name
1142 12420 : const std::string multiapp_name = getMultiAppName(name(), _first_local_app + i, _total_num_apps);
1143 12420 : std::string full_name;
1144 :
1145 : // Only add parent name if the parent is not the main app
1146 12420 : if (_app.multiAppLevel() > 0)
1147 525 : full_name = _app.name() + "_" + multiapp_name;
1148 : else
1149 11895 : full_name = multiapp_name;
1150 :
1151 12420 : InputParameters app_params = AppFactory::instance().getValidParams(_app_type);
1152 24840 : app_params.set<FEProblemBase *>("_parent_fep") = &_fe_problem;
1153 24840 : app_params.set<std::unique_ptr<Backup> *>("_initial_backup") = &_sub_app_backups[i];
1154 :
1155 : // Build the CommandLine with the relevant options for this subapp and add the
1156 : // cli args from the input file
1157 12420 : std::vector<std::string> input_cli_args;
1158 12420 : if (cliArgs().size() > 0 || _cli_args_from_file.size() > 0)
1159 3247 : input_cli_args = getCommandLineArgs(i);
1160 : // This will mark all hit CLI command line parameters that are passed to subapps
1161 : // as used within the parent app (_app)
1162 12420 : auto app_cli = _app.commandLine()->initSubAppCommandLine(name(), multiapp_name, input_cli_args);
1163 12420 : app_cli->parse();
1164 :
1165 12420 : if (_fe_problem.verboseMultiApps())
1166 1339 : _console << COLOR_CYAN << "Creating MultiApp " << name() << " of type " << _app_type
1167 1339 : << " of level " << _app.multiAppLevel() + 1 << " and number " << _first_local_app + i
1168 1339 : << " on processor " << processor_id() << " with full name " << full_name
1169 1339 : << COLOR_DEFAULT << std::endl;
1170 24840 : app_params.set<unsigned int>("_multiapp_level") = _app.multiAppLevel() + 1;
1171 24840 : app_params.set<unsigned int>("_multiapp_number") = _first_local_app + i;
1172 24840 : app_params.set<const MooseMesh *>("_master_mesh") = &_fe_problem.mesh();
1173 : #ifdef MOOSE_MFEM_ENABLED
1174 16630 : app_params.set<std::shared_ptr<mfem::Device>>("_mfem_device") =
1175 24945 : _app.getMFEMDevice(Moose::PassKey<MultiApp>());
1176 8315 : const auto & mfem_device_set = _app.getMFEMDevices(Moose::PassKey<MultiApp>());
1177 8315 : app_params.set<std::set<std::string>>("_mfem_devices") = mfem_device_set;
1178 : #endif
1179 62100 : if (getParam<bool>("clone_master_mesh") || getParam<bool>("clone_parent_mesh"))
1180 : {
1181 849 : if (_fe_problem.verboseMultiApps())
1182 0 : _console << COLOR_CYAN << "Cloned parent app mesh will be used for MultiApp " << name()
1183 0 : << COLOR_DEFAULT << std::endl;
1184 849 : app_params.set<bool>("_use_master_mesh") = true;
1185 849 : auto displaced_problem = _fe_problem.getDisplacedProblem();
1186 849 : if (displaced_problem)
1187 0 : app_params.set<const MooseMesh *>("_master_displaced_mesh") = &displaced_problem->mesh();
1188 849 : }
1189 :
1190 : // If only one input file was provided, use it for all the solves
1191 12420 : const auto input_index = _input_files.size() == 1 ? 0 : _first_local_app + i;
1192 12420 : const auto & input_file = _input_files[input_index];
1193 :
1194 : // create new parser tree for the application and parse
1195 12420 : auto parser = std::make_unique<Parser>(input_file);
1196 12420 : parser->setCommandLineParams(app_cli->buildHitParams());
1197 12420 : parser->parse();
1198 :
1199 : // Checks on app type
1200 12420 : const auto & app_type = parser->getAppType();
1201 12420 : if (app_type.empty() && _app_type.empty())
1202 0 : mooseWarning("The application type is not specified for ",
1203 : full_name,
1204 : ". Please use [Application] block to specify the application type.");
1205 12420 : if (!app_type.empty() && app_type != _app_type && !AppFactory::instance().isRegistered(app_type))
1206 4 : mooseError("In the ",
1207 : full_name,
1208 : ", '",
1209 : app_type,
1210 : "' is not a registered application. The registered application is named: '",
1211 4 : _app_type,
1212 : "'. Please double check the [Application] block to make sure the correct "
1213 : "application is provided. \n");
1214 :
1215 12416 : if (parser->getAppType().empty())
1216 12403 : parser->setAppType(_app_type);
1217 :
1218 24832 : app_params.set<std::shared_ptr<Parser>>("_parser") = std::move(parser);
1219 24832 : app_params.set<std::shared_ptr<CommandLine>>("_command_line") = std::move(app_cli);
1220 12416 : _apps[i] = AppFactory::instance().create(_app_type, full_name, app_params, _my_comm);
1221 12416 : auto & app = _apps[i];
1222 :
1223 12416 : app->setGlobalTimeOffset(start_time);
1224 12416 : app->setOutputFileNumbers(_app.getOutputWarehouse().getFileNumbers());
1225 12416 : app->setRestart(_app.isRestarting());
1226 12416 : app->setRecover(_app.isRecovering());
1227 :
1228 37248 : if (_use_positions && getParam<bool>("output_in_position"))
1229 3286 : app->setOutputPosition(_app.getOutputPosition() + _positions[_first_local_app + i]);
1230 12416 : if (_output_in_position && _run_in_position)
1231 8 : paramError("run_in_position",
1232 : "Sub-apps are already displaced, so they are already output in position");
1233 :
1234 : // Update the MultiApp level for the app that was just created
1235 12412 : app->setupOptions();
1236 : // if multiapp does not have file base in Outputs input block, output file base will
1237 : // be empty here since setupOptions() does not set the default file base with the multiapp
1238 : // input file name. Parent app will create the default file base for multiapp by taking the
1239 : // output base of the parent app problem and appending the name of the multiapp plus a number to
1240 : // it
1241 12412 : if (app->getOutputFileBase().empty())
1242 12390 : setAppOutputFileBase(i);
1243 12412 : preRunInputFile();
1244 37236 : if (_app.multiAppLevel() > getParam<unsigned int>("max_multiapp_level"))
1245 4 : mooseError("Maximum multiapp level has been reached. This is likely caused by an infinite loop "
1246 : "in your multiapp system. If additional multiapp levels are needed, "
1247 : "max_multiapp_level can be specified in the MuliApps block.");
1248 :
1249 : // Transfer coupling relaxation information to the subapps
1250 24816 : _apps[i]->fixedPointConfig().sub_relaxation_factor = getParam<Real>("relaxation_factor");
1251 12408 : _apps[i]->fixedPointConfig().sub_transformed_vars =
1252 37224 : getParam<std::vector<std::string>>("transformed_variables");
1253 : // Handle deprecated parameter
1254 37224 : if (!parameters().isParamSetByAddParam("relaxed_variables"))
1255 0 : _apps[i]->fixedPointConfig().sub_transformed_vars =
1256 0 : getParam<std::vector<std::string>>("relaxed_variables");
1257 12408 : _apps[i]->fixedPointConfig().sub_transformed_pps =
1258 37224 : getParam<std::vector<PostprocessorName>>("transformed_postprocessors");
1259 :
1260 12408 : app->runInputFile();
1261 12360 : auto fixed_point_solve = &(_apps[i]->getExecutioner()->fixedPointSolve());
1262 12360 : if (fixed_point_solve)
1263 12360 : fixed_point_solve->allocateStorage(false);
1264 :
1265 : // Transform the app mesh if requested
1266 12360 : if (_run_in_position)
1267 : {
1268 274 : if (usingPositions())
1269 548 : app->getExecutioner()->feProblem().coordTransform().transformMesh(
1270 274 : app->getExecutioner()->feProblem().mesh(), _positions[_first_local_app + i]);
1271 : else
1272 0 : app->getExecutioner()->feProblem().coordTransform().transformMesh(
1273 0 : app->getExecutioner()->feProblem().mesh(), Point(0, 0, 0));
1274 : }
1275 12356 : }
1276 :
1277 : std::vector<std::string>
1278 3247 : MultiApp::getCommandLineArgs(const unsigned int local_app)
1279 : {
1280 3247 : const auto cla = cliArgs();
1281 3247 : auto cli_args_param = _cli_args_param;
1282 3247 : std::string combined_args;
1283 :
1284 : // Single set of args from cliArgs() to be provided to all apps
1285 3247 : if (cla.size() == 1)
1286 2323 : combined_args = cla[0];
1287 : // Single "cli_args_files" file to be provided to all apps
1288 924 : else if (_cli_args_from_file.size() == 1)
1289 : {
1290 0 : cli_args_param = "cli_args_files";
1291 0 : combined_args = _cli_args_from_file[0];
1292 : }
1293 : // Unique set of args from cliArgs() to be provided to each app
1294 924 : else if (cla.size())
1295 804 : combined_args = cla[local_app + _first_local_app];
1296 : // Unique set of args from "cli_args_files" to be provided to all apps
1297 : else
1298 : {
1299 120 : cli_args_param = "cli_args_files";
1300 120 : combined_args = _cli_args_from_file[local_app + _first_local_app];
1301 : }
1302 :
1303 : // Remove all of the beginning and end whitespace so we can recognize truly empty
1304 3247 : combined_args = MooseUtils::trim(combined_args);
1305 :
1306 : // MooseUtils::split will return a single empty entry if there is nothing,
1307 : // so exit early if we have nothing
1308 3247 : if (combined_args.empty())
1309 0 : return {};
1310 :
1311 : // Split the argument into a vector of arguments, and make sure
1312 : // that we don't have any empty arguments
1313 6494 : const auto args = MooseUtils::split(combined_args, ";");
1314 10687 : for (const auto & arg : args)
1315 : {
1316 7440 : if (arg.empty())
1317 : {
1318 0 : const auto error = "An empty MultiApp command line argument was provided. Your "
1319 : "combined command line string has a ';' with no argument after it.";
1320 0 : if (cli_args_param)
1321 0 : paramError(*cli_args_param, error);
1322 : else
1323 0 : mooseError(error);
1324 : }
1325 : }
1326 :
1327 3247 : return args;
1328 3247 : }
1329 :
1330 : LocalRankConfig
1331 8488 : rankConfig(processor_id_type rank,
1332 : processor_id_type nprocs,
1333 : dof_id_type napps,
1334 : processor_id_type min_app_procs,
1335 : processor_id_type max_app_procs,
1336 : bool batch_mode)
1337 : {
1338 8488 : if (min_app_procs > nprocs)
1339 0 : mooseError("minimum number of procs per app is higher than the available number of procs");
1340 8488 : else if (min_app_procs > max_app_procs)
1341 0 : mooseError("minimum number of procs per app must be lower than the max procs per app");
1342 :
1343 : mooseAssert(rank < nprocs, "rank must be smaller than the number of procs");
1344 :
1345 : // A "slot" is a group of procs/ranks that are grouped together to run a
1346 : // single (sub)app/sim in parallel.
1347 :
1348 : const processor_id_type slot_size =
1349 8488 : std::max(std::min(cast_int<processor_id_type>(nprocs / napps), max_app_procs), min_app_procs);
1350 8488 : const processor_id_type nslots = std::min(
1351 16976 : nprocs / slot_size,
1352 8488 : cast_int<processor_id_type>(std::min(
1353 8488 : static_cast<dof_id_type>(std::numeric_limits<processor_id_type>::max()), napps)));
1354 : mooseAssert(nprocs >= (nslots * slot_size),
1355 : "Ensure that leftover procs is represented by an unsigned type");
1356 8488 : const processor_id_type leftover_procs = nprocs - nslots * slot_size;
1357 8488 : const dof_id_type apps_per_slot = napps / nslots;
1358 8488 : const dof_id_type leftover_apps = napps % nslots;
1359 :
1360 8488 : std::vector<int> slot_for_rank(nprocs);
1361 8488 : processor_id_type slot = 0;
1362 8488 : processor_id_type procs_in_slot = 0;
1363 20577 : for (processor_id_type rankiter = 0; rankiter <= rank; rankiter++)
1364 : {
1365 12089 : if (slot < nslots)
1366 11941 : slot_for_rank[rankiter] = cast_int<int>(slot);
1367 : else
1368 148 : slot_for_rank[rankiter] = -1;
1369 12089 : procs_in_slot++;
1370 : // this slot keeps growing until we reach slot size plus possibly an extra
1371 : // proc if there were any leftover from the slotization of nprocs - this
1372 : // must also make sure we don't go over max app procs.
1373 12089 : if (procs_in_slot == slot_size + 1 * (slot < leftover_procs && slot_size < max_app_procs))
1374 : {
1375 8942 : procs_in_slot = 0;
1376 8942 : slot++;
1377 : }
1378 : }
1379 :
1380 8488 : if (slot_for_rank[rank] < 0)
1381 : // ranks assigned a negative slot don't have any apps running on them.
1382 72 : return {0, 0, 0, 0, false, 0};
1383 8416 : const processor_id_type slot_num = cast_int<processor_id_type>(slot_for_rank[rank]);
1384 :
1385 8416 : const bool is_first_local_rank = rank == 0 || (slot_for_rank[rank - 1] != slot_for_rank[rank]);
1386 8416 : const dof_id_type n_local_apps = apps_per_slot + 1 * (slot_num < leftover_apps);
1387 :
1388 8416 : processor_id_type my_first_rank = 0;
1389 10889 : for (processor_id_type rankiter = rank; rankiter > 0; rankiter--)
1390 2767 : if (slot_for_rank[rank] != slot_for_rank[rankiter])
1391 : {
1392 294 : my_first_rank = cast_int<processor_id_type>(slot_for_rank[rankiter + 1]);
1393 294 : break;
1394 : }
1395 :
1396 8416 : dof_id_type app_index = 0;
1397 9878 : for (processor_id_type slot = 0; slot < slot_num; slot++)
1398 : {
1399 1462 : const dof_id_type num_slot_apps = apps_per_slot + 1 * (slot < leftover_apps);
1400 1462 : app_index += num_slot_apps;
1401 : }
1402 :
1403 8416 : if (batch_mode)
1404 405 : return {n_local_apps, app_index, 1, slot_num, is_first_local_rank, my_first_rank};
1405 8011 : return {n_local_apps, app_index, n_local_apps, app_index, is_first_local_rank, my_first_rank};
1406 8488 : }
1407 :
1408 : void
1409 7601 : MultiApp::buildComm()
1410 : {
1411 : int ierr;
1412 :
1413 7601 : ierr = MPI_Comm_size(_communicator.get(), &_orig_num_procs);
1414 7601 : mooseCheckMPIErr(ierr);
1415 7601 : ierr = MPI_Comm_rank(_communicator.get(), &_orig_rank);
1416 7601 : mooseCheckMPIErr(ierr);
1417 :
1418 : #ifdef LIBMESH_HAVE_SYS_UTSNAME_H
1419 : struct utsname sysInfo;
1420 7601 : uname(&sysInfo);
1421 7601 : _node_name = sysInfo.nodename;
1422 : #else
1423 : _node_name = "Unknown";
1424 : #endif
1425 :
1426 : int rank;
1427 7601 : ierr = MPI_Comm_rank(_communicator.get(), &rank);
1428 7601 : mooseCheckMPIErr(ierr);
1429 :
1430 7601 : _my_num_apps = _rank_config.num_local_apps;
1431 7601 : _first_local_app = _rank_config.first_local_app_index;
1432 :
1433 7601 : _has_an_app = _rank_config.num_local_apps > 0;
1434 7601 : if (_rank_config.first_local_app_index >= _total_num_apps)
1435 0 : mooseError("Internal error, a processor has an undefined app.");
1436 :
1437 7601 : if (_has_an_app)
1438 : {
1439 7547 : _communicator.split(_rank_config.first_local_app_index, rank, _my_communicator);
1440 7547 : ierr = MPI_Comm_rank(_my_comm, &_my_rank);
1441 7547 : mooseCheckMPIErr(ierr);
1442 : }
1443 : else
1444 : {
1445 54 : _communicator.split(MPI_UNDEFINED, rank, _my_communicator);
1446 54 : _my_rank = 0;
1447 : }
1448 7601 : }
1449 :
1450 : unsigned int
1451 873057 : MultiApp::globalAppToLocal(unsigned int global_app)
1452 : {
1453 873057 : if (global_app >= _first_local_app && global_app <= _first_local_app + (_my_num_apps - 1))
1454 873057 : return global_app - _first_local_app;
1455 :
1456 0 : std::stringstream ss;
1457 0 : ss << "Requesting app " << global_app << ", but processor " << processor_id() << " ";
1458 0 : if (_my_num_apps == 0)
1459 0 : ss << "does not own any apps";
1460 0 : else if (_my_num_apps == 1)
1461 0 : ss << "owns app " << _first_local_app;
1462 : else
1463 0 : ss << "owns apps " << _first_local_app << "-" << _first_local_app + (_my_num_apps - 1);
1464 0 : ss << ".";
1465 0 : mooseError("Invalid global_app!\n", ss.str());
1466 : return 0;
1467 0 : }
1468 :
1469 : void
1470 12412 : MultiApp::preRunInputFile()
1471 : {
1472 12412 : }
1473 :
1474 : void
1475 13372 : MultiApp::addAssociatedTransfer(MultiAppTransfer & transfer)
1476 : {
1477 13372 : _associated_transfers.push_back(&transfer);
1478 13372 : }
1479 :
1480 : void
1481 0 : MultiApp::setAppOutputFileBase()
1482 : {
1483 0 : for (unsigned int i = 0; i < _my_num_apps; ++i)
1484 0 : setAppOutputFileBase(i);
1485 0 : }
1486 :
1487 : std::vector<std::string>
1488 23264 : MultiApp::cliArgs() const
1489 : {
1490 : // So that we can error out with paramError("cli_args", ...);
1491 23264 : _cli_args_param = "cli_args";
1492 46528 : return std::vector<std::string>(_cli_args.begin(), _cli_args.end());
1493 : }
1494 :
1495 : void
1496 12390 : MultiApp::setAppOutputFileBase(unsigned int index)
1497 : {
1498 : const std::string multiapp_name =
1499 12390 : getMultiAppName(name(), _first_local_app + index, _total_num_apps);
1500 12390 : _apps[index]->setOutputFileBase(_app.getOutputFileBase() + "_" + multiapp_name);
1501 12390 : }
1502 :
1503 : std::string
1504 24810 : MultiApp::getMultiAppName(const std::string & base_name, dof_id_type index, dof_id_type total)
1505 : {
1506 24810 : std::ostringstream multiapp_name;
1507 49620 : multiapp_name << base_name << std::setw(std::ceil(std::log10(total))) << std::setprecision(0)
1508 24810 : << std::setfill('0') << std::right << index;
1509 49620 : return multiapp_name.str();
1510 24810 : }
1511 :
1512 : const Point &
1513 208819 : MultiApp::position(unsigned int app) const
1514 : {
1515 : // If we're not using positions, it won't have changed
1516 208819 : if (_positions_objs.empty())
1517 192691 : return _positions[app];
1518 : else
1519 : // Find which Positions object is specifying it, and query a potentially updated value
1520 16128 : return _positions_objs[app]->getPosition(app - _positions_index_offsets[app], false);
1521 : }
1522 :
1523 : void
1524 2500 : dataStore(std::ostream & stream, SubAppBackups & backups, void * context)
1525 : {
1526 2500 : MultiApp * multi_app = static_cast<MultiApp *>(context);
1527 : mooseAssert(multi_app, "Not set");
1528 :
1529 2500 : multi_app->backup();
1530 :
1531 2500 : dataStore(stream, static_cast<std::vector<std::unique_ptr<Backup>> &>(backups), nullptr);
1532 2500 : }
1533 :
1534 : void
1535 651 : dataLoad(std::istream & stream, SubAppBackups & backups, void * context)
1536 : {
1537 651 : MultiApp * multi_app = static_cast<MultiApp *>(context);
1538 : mooseAssert(multi_app, "Not set");
1539 :
1540 651 : dataLoad(stream, static_cast<std::vector<std::unique_ptr<Backup>> &>(backups), nullptr);
1541 :
1542 651 : multi_app->restore();
1543 651 : }
|
