Line data Source code
1 : //* This file is part of the MOOSE framework
2 : //* https://mooseframework.inl.gov
3 : //*
4 : //* All rights reserved, see COPYRIGHT for full restrictions
5 : //* https://github.com/idaholab/moose/blob/master/COPYRIGHT
6 : //*
7 : //* Licensed under LGPL 2.1, please see LICENSE for details
8 : //* https://www.gnu.org/licenses/lgpl-2.1.html
9 :
10 : // MOOSE includes
11 : #include "MultiApp.h"
12 :
13 : #include "AppFactory.h"
14 : #include "AuxiliarySystem.h"
15 : #include "DisplacedProblem.h"
16 : #include "Console.h"
17 : #include "Executioner.h"
18 : #include "FEProblem.h"
19 : #include "MooseMesh.h"
20 : #include "MooseUtils.h"
21 : #include "OutputWarehouse.h"
22 : #include "SetupInterface.h"
23 : #include "UserObject.h"
24 : #include "CommandLine.h"
25 : #include "Conversion.h"
26 : #include "NonlinearSystemBase.h"
27 : #include "DelimitedFileReader.h"
28 : #include "MooseAppCoordTransform.h"
29 : #include "MultiAppTransfer.h"
30 : #include "Positions.h"
31 : #include "Transient.h"
32 : #include "Backup.h"
33 : #include "Parser.h"
34 :
35 : #include "libmesh/mesh_tools.h"
36 : #include "libmesh/numeric_vector.h"
37 :
38 : // C++ includes
39 : #include <fstream>
40 : #include <iomanip>
41 : #include <iterator>
42 : #include <algorithm>
43 :
44 : // Call to "uname"
45 : #ifdef LIBMESH_HAVE_SYS_UTSNAME_H
46 : #include <sys/utsname.h>
47 : #endif
48 :
49 : InputParameters
50 89679 : MultiApp::validParams()
51 : {
52 89679 : InputParameters params = MooseObject::validParams();
53 89679 : params += SetupInterface::validParams();
54 :
55 179358 : params.addParam<bool>("use_displaced_mesh",
56 179358 : false,
57 : "Whether or not this object should use the "
58 : "displaced mesh for computation. Note that "
59 : "in the case this is true but no "
60 : "displacements are provided in the Mesh block "
61 : "the undisplaced mesh will still be used.");
62 :
63 89679 : std::ostringstream app_types_strings;
64 179384 : for (const auto & name_bi_pair : AppFactory::instance().registeredObjects())
65 89705 : app_types_strings << name_bi_pair.first << " ";
66 179358 : MooseEnum app_types_options(app_types_strings.str(), "", true);
67 :
68 : // Dynamic loading
69 358716 : params.addParam<MooseEnum>("app_type",
70 : app_types_options,
71 : "The type of application to build (applications not "
72 : "registered can be loaded with dynamic libraries. Parent "
73 : "application type will be used if not provided.");
74 358716 : params.addParam<std::string>("library_path",
75 : "",
76 : "Path to search for dynamic libraries (please "
77 : "avoid committing absolute paths in addition to "
78 : "MOOSE_LIBRARY_PATH)");
79 358716 : params.addParam<std::string>(
80 : "library_name",
81 : "",
82 : "The file name of the library (*.la file) that will be dynamically loaded.");
83 269037 : params.addParam<bool>("library_load_dependencies",
84 179358 : false,
85 : "Tells MOOSE to manually load library dependencies. This should not be "
86 : "necessary and is here for debugging/troubleshooting.");
87 :
88 : // Subapp positions
89 358716 : params.addParam<std::vector<Point>>(
90 : "positions",
91 : "The positions of the App locations. Each set of 3 values will represent a "
92 : "Point. This and 'positions_file' cannot be both supplied. If this and "
93 : "'positions_file'/'_objects' are not supplied, a single position (0,0,0) will be used");
94 358716 : params.addParam<std::vector<FileName>>("positions_file",
95 : "Filename(s) that should be looked in for positions. Each"
96 : " set of 3 values in that file will represent a Point. "
97 : "This and 'positions(_objects)' cannot be both supplied");
98 358716 : params.addParam<std::vector<PositionsName>>("positions_objects",
99 : "The name of a Positions object that will contain "
100 : "the locations of the sub-apps created. This and "
101 : "'positions(_file)' cannot be both supplied");
102 269037 : params.addParam<bool>(
103 : "output_in_position",
104 179358 : false,
105 : "If true this will cause the output from the MultiApp to be 'moved' by its position vector");
106 269037 : params.addParam<bool>(
107 : "run_in_position",
108 179358 : false,
109 : "If true this will cause the mesh from the MultiApp to be 'moved' by its position vector");
110 :
111 358716 : params.addRequiredParam<std::vector<FileName>>(
112 : "input_files",
113 : "The input file for each App. If this parameter only contains one input file "
114 : "it will be used for all of the Apps. When using 'positions_from_file' it is "
115 : "also admissable to provide one input_file per file.");
116 269037 : params.addParam<Real>("bounding_box_inflation",
117 179358 : 0.01,
118 : "Relative amount to 'inflate' the bounding box of this MultiApp.");
119 269037 : params.addParam<Point>("bounding_box_padding",
120 179358 : RealVectorValue(),
121 : "Additional padding added to the dimensions of the bounding box. The "
122 : "values are added to the x, y and z dimension respectively.");
123 :
124 179358 : params.addPrivateParam<MPI_Comm>("_mpi_comm");
125 :
126 : // Set the default execution time
127 179358 : params.set<ExecFlagEnum>("execute_on", true) = EXEC_TIMESTEP_BEGIN;
128 : // Add the POST_ADAPTIVITY execution flag.
129 : #ifdef LIBMESH_ENABLE_AMR
130 89679 : ExecFlagEnum & exec_enum = params.set<ExecFlagEnum>("execute_on");
131 89679 : exec_enum.addAvailableFlags(EXEC_POST_ADAPTIVITY);
132 269037 : params.setDocString("execute_on", exec_enum.getDocString());
133 : #endif
134 :
135 269037 : params.addParam<processor_id_type>("max_procs_per_app",
136 179358 : std::numeric_limits<processor_id_type>::max(),
137 : "Maximum number of processors to give to each App in this "
138 : "MultiApp. Useful for restricting small solves to just a few "
139 : "procs so they don't get spread out");
140 269037 : params.addParam<processor_id_type>("min_procs_per_app",
141 179358 : 1,
142 : "Minimum number of processors to give to each App in this "
143 : "MultiApp. Useful for larger, distributed mesh solves.");
144 269037 : params.addParam<bool>(
145 : "wait_for_first_app_init",
146 179358 : false,
147 : "Create the first sub-application on rank 0, then MPI_Barrier before "
148 : "creating the next N-1 apps (on all ranks). "
149 : "This is only needed if your sub-application needs to perform some setup "
150 : "actions in quiet, without other sub-applications working at the same time.");
151 :
152 269037 : params.addParam<Real>("global_time_offset",
153 179358 : 0,
154 : "The time offset relative to the parent application for the purpose of "
155 : "starting a subapp at a different time from the parent application. The "
156 : "global time will be ahead by the offset specified here.");
157 :
158 : // Resetting subapps
159 358716 : params.addParam<std::vector<Real>>(
160 : "reset_time",
161 : {},
162 : "The time(s) at which to reset Apps given by the 'reset_apps' parameter. "
163 : "Resetting an App means that it is destroyed and recreated, possibly "
164 : "modeling the insertion of 'new' material for that app.");
165 358716 : params.addParam<std::vector<unsigned int>>(
166 : "reset_apps",
167 : {},
168 : "The Apps that will be reset when 'reset_time' is hit. These are the App "
169 : "'numbers' starting with 0 corresponding to the order of the App positions. "
170 : "Resetting an App means that it is destroyed and recreated, possibly modeling "
171 : "the insertion of 'new' material for that app.");
172 :
173 : // Moving subapps
174 269037 : params.addParam<Real>(
175 : "move_time",
176 179358 : std::numeric_limits<Real>::max(),
177 : "The time at which Apps designated by move_apps are moved to move_positions.");
178 :
179 358716 : params.addParam<std::vector<unsigned int>>(
180 : "move_apps",
181 : {},
182 : "Apps, designated by their 'numbers' starting with 0 corresponding to the order "
183 : "of the App positions, to be moved at move_time to move_positions");
184 358716 : params.addParam<std::vector<Point>>(
185 : "move_positions", {}, "The positions corresponding to each move_app.");
186 :
187 358716 : params.addParam<std::vector<CLIArgString>>(
188 : "cli_args",
189 : {},
190 : "Additional command line arguments to pass to the sub apps. If one set is provided the "
191 : "arguments are applied to all, otherwise there must be a set for each sub app.");
192 :
193 358716 : params.addParam<std::vector<FileName>>(
194 : "cli_args_files",
195 : "File names that should be looked in for additional command line arguments "
196 : "to pass to the sub apps. Each line of a file is set to each sub app. If only "
197 : "one line is provided, it will be applied to all sub apps.");
198 :
199 : // Fixed point iterations
200 448395 : params.addRangeCheckedParam<Real>("relaxation_factor",
201 179358 : 1.0,
202 : "relaxation_factor>0 & relaxation_factor<2",
203 : "Fraction of newly computed value to keep."
204 : "Set between 0 and 2.");
205 538074 : params.addDeprecatedParam<std::vector<std::string>>(
206 : "relaxed_variables",
207 : {},
208 : "Use transformed_variables.",
209 : "List of subapp variables to relax during Multiapp coupling iterations");
210 358716 : params.addParam<std::vector<std::string>>(
211 : "transformed_variables",
212 : {},
213 : "List of subapp variables to use coupling algorithm on during Multiapp coupling iterations");
214 358716 : params.addParam<std::vector<PostprocessorName>>(
215 : "transformed_postprocessors",
216 : {},
217 : "List of subapp postprocessors to use coupling "
218 : "algorithm on during Multiapp coupling iterations");
219 269037 : params.addParam<bool>("keep_solution_during_restore",
220 179358 : false,
221 : "This is useful when doing MultiApp coupling iterations. It takes the "
222 : "final solution from the previous coupling iteration"
223 : "and re-uses it as the initial guess for the next coupling iteration");
224 269037 : params.addParam<bool>("keep_aux_solution_during_restore",
225 179358 : false,
226 : "This is useful when doing MultiApp coupling iterations. It takes the "
227 : "final auxiliary solution from the previous coupling iteration"
228 : "and re-uses it as the initial guess for the next coupling iteration");
229 269037 : params.addParam<bool>(
230 : "no_restore",
231 179358 : false,
232 : "True to turn off restore for this multiapp. This is useful when doing steady-state "
233 : "Picard iterations where we want to use the solution of previous Picard iteration as the "
234 : "initial guess of the current Picard iteration.");
235 269037 : params.addParam<unsigned int>(
236 : "max_multiapp_level",
237 179358 : 10,
238 : "Integer set by user that will stop the simulation if the multiapp level "
239 : "exceeds it. Useful for preventing infinite loops with multiapp simulations");
240 :
241 448395 : params.addDeprecatedParam<bool>("clone_master_mesh",
242 179358 : false,
243 : "True to clone parent app mesh and use it for this MultiApp.",
244 : "clone_master_mesh is deprecated, use clone_parent_mesh instead");
245 179358 : params.addParam<bool>(
246 179358 : "clone_parent_mesh", false, "True to clone parent app mesh and use it for this MultiApp.");
247 :
248 179358 : params.addPrivateParam<bool>("use_positions", true);
249 269037 : params.declareControllable("enable");
250 448395 : params.declareControllable("cli_args", {EXEC_PRE_MULTIAPP_SETUP});
251 179358 : params.registerBase("MultiApp");
252 :
253 358716 : params.addParamNamesToGroup("use_displaced_mesh wait_for_first_app_init max_multiapp_level",
254 : "Advanced");
255 358716 : params.addParamNamesToGroup("positions positions_file positions_objects run_in_position "
256 : "output_in_position",
257 : "Positions / transformations of the MultiApp frame of reference");
258 358716 : params.addParamNamesToGroup("min_procs_per_app max_procs_per_app", "Parallelism");
259 358716 : params.addParamNamesToGroup("reset_time reset_apps", "Reset MultiApp");
260 358716 : params.addParamNamesToGroup("move_time move_apps move_positions", "Timed move of MultiApps");
261 358716 : params.addParamNamesToGroup("relaxation_factor transformed_variables transformed_postprocessors "
262 : "keep_solution_during_restore keep_aux_solution_during_restore "
263 : "no_restore",
264 : "Fixed point iteration");
265 358716 : params.addParamNamesToGroup("library_name library_path library_load_dependencies",
266 : "Dynamic loading");
267 269037 : params.addParamNamesToGroup("cli_args cli_args_files", "Passing command line argument");
268 179358 : return params;
269 179358 : }
270 :
271 7668 : MultiApp::MultiApp(const InputParameters & parameters)
272 : : MooseObject(parameters),
273 : SetupInterface(this),
274 : Restartable(this, "MultiApps"),
275 15336 : PerfGraphInterface(this, std::string("MultiApp::") + _name),
276 23004 : _fe_problem(*getCheckedPointerParam<FEProblemBase *>("_fe_problem_base")),
277 27660 : _app_type(isParamValid("app_type") ? std::string(getParam<MooseEnum>("app_type"))
278 3012 : : _fe_problem.getMooseApp().type()),
279 15336 : _use_positions(getParam<bool>("use_positions")),
280 15336 : _input_files(getParam<std::vector<FileName>>("input_files")),
281 15336 : _wait_for_first_app_init(getParam<bool>("wait_for_first_app_init")),
282 7668 : _total_num_apps(0),
283 7668 : _my_num_apps(0),
284 7668 : _first_local_app(0),
285 7668 : _orig_comm(_communicator.get()),
286 7668 : _my_communicator(),
287 7668 : _my_comm(_my_communicator.get()),
288 7668 : _my_rank(0),
289 15336 : _inflation(getParam<Real>("bounding_box_inflation")),
290 15336 : _bounding_box_padding(getParam<Point>("bounding_box_padding")),
291 15336 : _max_procs_per_app(getParam<processor_id_type>("max_procs_per_app")),
292 15336 : _min_procs_per_app(getParam<processor_id_type>("min_procs_per_app")),
293 15336 : _output_in_position(getParam<bool>("output_in_position")),
294 15336 : _global_time_offset(getParam<Real>("global_time_offset")),
295 15336 : _reset_times(getParam<std::vector<Real>>("reset_time")),
296 15336 : _reset_apps(getParam<std::vector<unsigned int>>("reset_apps")),
297 15336 : _reset_happened(false),
298 15336 : _move_time(getParam<Real>("move_time")),
299 15336 : _move_apps(getParam<std::vector<unsigned int>>("move_apps")),
300 15336 : _move_positions(getParam<std::vector<Point>>("move_positions")),
301 7668 : _move_happened(false),
302 7668 : _has_an_app(true),
303 15336 : _cli_args(getParam<std::vector<CLIArgString>>("cli_args")),
304 15336 : _keep_solution_during_restore(getParam<bool>("keep_solution_during_restore")),
305 15336 : _keep_aux_solution_during_restore(getParam<bool>("keep_aux_solution_during_restore")),
306 15336 : _no_restore(getParam<bool>("no_restore")),
307 15336 : _run_in_position(getParam<bool>("run_in_position")),
308 15336 : _sub_app_backups(declareRestartableDataWithContext<SubAppBackups>("sub_app_backups", this)),
309 30672 : _solve_step_timer(registerTimedSection("solveStep", 3, "Executing MultiApps", false)),
310 30672 : _init_timer(registerTimedSection("init", 3, "Initializing MultiApp")),
311 30672 : _backup_timer(registerTimedSection("backup", 3, "Backing Up MultiApp")),
312 30672 : _restore_timer(registerTimedSection("restore", 3, "Restoring MultiApp")),
313 115020 : _reset_timer(registerTimedSection("resetApp", 3, "Resetting MultiApp"))
314 : {
315 36267 : if (parameters.isParamSetByUser("cli_args") && parameters.isParamValid("cli_args") &&
316 11398 : parameters.isParamValid("cli_args_files"))
317 8 : paramError("cli_args",
318 : "'cli_args' and 'cli_args_files' cannot be specified simultaneously in MultiApp ");
319 :
320 15328 : if (!_use_positions && (isParamValid("positions") || isParamValid("positions_file") ||
321 7664 : isParamValid("positions_objects")))
322 0 : paramError("use_positions",
323 : "This MultiApps has been set to not use positions, "
324 : "but a 'positions' parameter has been set.");
325 :
326 22923 : if ((_reset_apps.size() > 0 && _reset_times.size() == 0) ||
327 15259 : (_reset_apps.size() == 0 && _reset_times.size() > 0))
328 0 : mooseError("reset_time and reset_apps may only be specified together");
329 :
330 : // Check that the reset times are sorted by the user
331 7664 : auto sorted_times = _reset_times;
332 7664 : std::sort(sorted_times.begin(), sorted_times.end());
333 7664 : if (_reset_times.size() && _reset_times != sorted_times)
334 8 : paramError("reset_time", "List of reset times must be sorted in increasing order");
335 7660 : }
336 :
337 : void
338 7632 : MultiApp::init(unsigned int num_apps, bool batch_mode)
339 : {
340 7632 : auto config = rankConfig(
341 : processor_id(), n_processors(), num_apps, _min_procs_per_app, _max_procs_per_app, batch_mode);
342 7632 : init(num_apps, config);
343 7628 : }
344 :
345 : void
346 7632 : MultiApp::init(unsigned int num_apps, const LocalRankConfig & config)
347 : {
348 7632 : TIME_SECTION(_init_timer);
349 :
350 7632 : _total_num_apps = num_apps;
351 7632 : _rank_config = config;
352 7632 : buildComm();
353 7632 : _sub_app_backups.resize(_my_num_apps);
354 :
355 7632 : _has_bounding_box.resize(_my_num_apps, false);
356 7632 : _reset_happened.resize(_reset_times.size(), false);
357 7632 : _bounding_box.resize(_my_num_apps);
358 :
359 7632 : if ((_cli_args.size() > 1) && (_total_num_apps != _cli_args.size()))
360 8 : paramError("cli_args",
361 : "The number of items supplied must be 1 or equal to the number of sub apps.");
362 :
363 : // if cliArgs() != _cli_args, then cliArgs() was overridden and we need to check it
364 7628 : auto cla = cliArgs();
365 15256 : if (cla != std::vector<std::string>(_cli_args.begin(), _cli_args.end()))
366 : {
367 13 : if ((cla.size() > 1) && (_total_num_apps != cla.size()))
368 0 : mooseError("The number of items supplied as command line argument to subapps must be 1 or "
369 : "equal to the number of sub apps. Note: you use a multiapp that provides its own "
370 : "command line parameters so the error is not in cli_args");
371 : }
372 7628 : }
373 :
374 : void
375 7656 : MultiApp::setupPositions()
376 : {
377 7656 : if (_use_positions)
378 : {
379 7656 : fillPositions();
380 7632 : init(_positions.size());
381 7628 : createApps();
382 : }
383 7541 : }
384 :
385 : void
386 7628 : MultiApp::createApps()
387 : {
388 7628 : if (!_has_an_app)
389 54 : return;
390 :
391 37870 : TIME_SECTION("createApps", 2, "Instantiating Sub-Apps", false);
392 :
393 : // Read commandLine arguments that will be used when creating apps
394 7574 : readCommandLineArguments();
395 :
396 7554 : Moose::ScopedCommSwapper swapper(_my_comm);
397 :
398 7554 : _apps.resize(_my_num_apps);
399 :
400 : // If the user provided an unregistered app type, see if we can load it dynamically
401 7554 : if (!AppFactory::instance().isRegistered(_app_type))
402 5 : _app.dynamicAppRegistration(_app_type,
403 : getParam<std::string>("library_path"),
404 7 : getParam<std::string>("library_name"),
405 14 : getParam<bool>("library_load_dependencies"));
406 :
407 7551 : bool rank_did_quiet_init = false;
408 7551 : unsigned int local_app = libMesh::invalid_uint;
409 7551 : if (_wait_for_first_app_init)
410 : {
411 16 : if (hasLocalApp(0))
412 : {
413 8 : rank_did_quiet_init = true;
414 8 : local_app = globalAppToLocal(0);
415 8 : createLocalApp(local_app);
416 : }
417 :
418 16 : MPI_Barrier(_orig_comm);
419 : }
420 :
421 19894 : for (unsigned int i = 0; i < _my_num_apps; i++)
422 : {
423 12407 : if (rank_did_quiet_init && i == local_app)
424 8 : continue;
425 12399 : createLocalApp(i);
426 : }
427 7487 : }
428 :
429 : void
430 12407 : MultiApp::createLocalApp(const unsigned int i)
431 : {
432 12407 : createApp(i, _global_time_offset);
433 12343 : }
434 :
435 : void
436 7445 : MultiApp::initialSetup()
437 : {
438 7445 : if (!_use_positions)
439 : // if not using positions, we create the sub-apps in initialSetup instead of right after
440 : // construction of MultiApp
441 0 : createApps();
442 7445 : }
443 :
444 : void
445 7574 : MultiApp::readCommandLineArguments()
446 : {
447 22722 : if (isParamValid("cli_args_files"))
448 : {
449 59 : _cli_args_from_file.clear();
450 :
451 118 : std::vector<FileName> cli_args_files = getParam<std::vector<FileName>>("cli_args_files");
452 118 : std::vector<FileName> input_files = getParam<std::vector<FileName>>("input_files");
453 :
454 : // If we use parameter "cli_args_files", at least one file should be provided
455 59 : if (!cli_args_files.size())
456 8 : paramError("cli_args_files", "You need to provide at least one commandLine argument file ");
457 :
458 : // If we multiple input files, then we need to check if the number of input files
459 : // match with the number of argument files
460 55 : if (cli_args_files.size() != 1 && cli_args_files.size() != input_files.size())
461 12 : paramError("cli_args_files",
462 : "The number of commandLine argument files ",
463 : cli_args_files.size(),
464 : " for MultiApp ",
465 4 : name(),
466 : " must either be only one or match the number of input files ",
467 : input_files.size());
468 :
469 : // Go through all argument files
470 51 : std::vector<std::string> cli_args;
471 107 : for (unsigned int p_file_it = 0; p_file_it < cli_args_files.size(); p_file_it++)
472 : {
473 64 : std::string cli_args_file = cli_args_files[p_file_it];
474 : // Clear up
475 64 : cli_args.clear();
476 : // Read the file on the root processor then broadcast it
477 64 : if (processor_id() == 0)
478 : {
479 52 : MooseUtils::checkFileReadable(cli_args_file);
480 :
481 52 : std::ifstream is(cli_args_file.c_str());
482 : // Read by line rather than space separated like the cli_args parameter
483 52 : std::string line;
484 212 : while (std::getline(is, line))
485 160 : cli_args.push_back(line);
486 :
487 : // We do not allow empty files
488 52 : if (!cli_args.size())
489 8 : paramError("cli_args_files",
490 : "There is no commandLine argument in the commandLine argument file ",
491 : cli_args_file);
492 :
493 : // If we have position files, we need to
494 : // make sure the number of commandLine argument strings
495 : // match with the number of positions
496 48 : if (_npositions_inputfile.size())
497 : {
498 44 : auto num_positions = _npositions_inputfile[p_file_it];
499 : // Check if the number of commandLine argument strings equal to
500 : // the number of positions
501 44 : if (cli_args.size() == 1)
502 0 : for (MooseIndex(num_positions) num = 0; num < num_positions; num++)
503 0 : _cli_args_from_file.push_back(cli_args.front());
504 44 : else if (cli_args.size() == num_positions)
505 160 : for (auto && cli_arg : cli_args)
506 120 : _cli_args_from_file.push_back(cli_arg);
507 4 : else if (cli_args.size() != num_positions)
508 8 : paramError("cli_args_files",
509 : "The number of commandLine argument strings ",
510 : cli_args.size(),
511 : " in the file ",
512 : cli_args_file,
513 : " must either be only one or match the number of positions ",
514 : num_positions);
515 : }
516 : else
517 : {
518 : // If we do not have position files, we will check if the number of
519 : // commandLine argument strings match with the total number of subapps
520 24 : for (auto && cli_arg : cli_args)
521 20 : _cli_args_from_file.push_back(cli_arg);
522 : }
523 44 : }
524 56 : }
525 :
526 : // Broad cast all arguments to everyone
527 43 : _communicator.broadcast(_cli_args_from_file);
528 43 : }
529 :
530 7601 : if (_cli_args_from_file.size() && _cli_args_from_file.size() != 1 &&
531 43 : _cli_args_from_file.size() != _total_num_apps)
532 8 : mooseError(" The number of commandLine argument strings ",
533 4 : _cli_args_from_file.size(),
534 : " must either be only one or match the total "
535 : "number of sub apps ",
536 4 : _total_num_apps);
537 :
538 7554 : if (_cli_args_from_file.size() && cliArgs().size())
539 0 : mooseError("Cannot set commandLine arguments from both input_file and external files");
540 7554 : }
541 :
542 : void
543 7537 : MultiApp::fillPositions()
544 : {
545 7537 : if (_move_apps.size() != _move_positions.size())
546 0 : mooseError("The number of apps to move and the positions to move them to must be the same for "
547 : "MultiApp ",
548 0 : _name);
549 :
550 37685 : if (isParamValid("positions") + isParamValid("positions_file") +
551 22611 : isParamValid("positions_objects") >
552 : 1)
553 4 : mooseError("Only one 'positions' parameter may be specified");
554 :
555 22599 : if (isParamValid("positions"))
556 : {
557 10604 : _positions = getParam<std::vector<Point>>("positions");
558 :
559 5302 : if (_positions.size() < _input_files.size())
560 4 : mooseError("Not enough positions for the number of input files provided in MultiApp ",
561 4 : name());
562 : }
563 6693 : else if (isParamValid("positions_file"))
564 : {
565 188 : std::vector<FileName> positions_files = getParam<std::vector<FileName>>("positions_file");
566 188 : std::vector<FileName> input_files = getParam<std::vector<FileName>>("input_files");
567 :
568 94 : if (input_files.size() != 1 && positions_files.size() != input_files.size())
569 4 : mooseError("Number of input_files for MultiApp ",
570 4 : name(),
571 : " must either be only one or match the number of positions_file files");
572 :
573 : // Clear out the _input_files because we're going to rebuild it
574 90 : if (input_files.size() != 1)
575 30 : _input_files.clear();
576 :
577 206 : for (unsigned int p_file_it = 0; p_file_it < positions_files.size(); p_file_it++)
578 : {
579 120 : std::string positions_file = positions_files[p_file_it];
580 120 : MooseUtils::DelimitedFileReader file(positions_file, &_communicator);
581 120 : file.setFormatFlag(MooseUtils::DelimitedFileReader::FormatFlag::ROWS);
582 120 : file.read();
583 :
584 120 : const std::vector<Point> & data = file.getDataAsPoints();
585 564 : for (const auto & d : data)
586 448 : _positions.push_back(d);
587 :
588 : // Save the number of positions for this input file
589 116 : _npositions_inputfile.push_back(data.size());
590 :
591 564 : for (unsigned int i = 0; i < data.size(); ++i)
592 448 : if (input_files.size() != 1)
593 120 : _input_files.push_back(input_files[p_file_it]);
594 116 : }
595 86 : }
596 6411 : else if (isParamValid("positions_objects"))
597 : {
598 928 : const auto & positions_param_objs = getParam<std::vector<PositionsName>>("positions_objects");
599 928 : const auto & input_files = getParam<std::vector<FileName>>("input_files");
600 :
601 464 : if (input_files.size() != 1 && positions_param_objs.size() != input_files.size())
602 0 : mooseError("Number of input_files for MultiApp ",
603 0 : name(),
604 : " must either be only one or match the number of positions_objects specified");
605 :
606 : // Clear out the _input_files because we're going to rebuild it
607 464 : if (input_files.size() != 1)
608 0 : _input_files.clear();
609 :
610 : // Keeps track of where each positions object start in terms of subapp numbers
611 464 : unsigned int offset = 0;
612 :
613 967 : for (const auto p_obj_it : index_range(positions_param_objs))
614 : {
615 507 : const std::string & positions_name = positions_param_objs[p_obj_it];
616 507 : auto positions_obj = &_fe_problem.getPositionsObject(positions_name);
617 :
618 507 : const auto & data = positions_obj->getPositions(true);
619 :
620 : // Append all positions from this object
621 2200 : for (const auto & d : data)
622 1697 : _positions.push_back(d);
623 :
624 : // Save the number of positions for this input file
625 503 : _npositions_inputfile.push_back(data.size());
626 :
627 503 : if (!positions_obj)
628 0 : paramError("positions_objects",
629 0 : "'" + positions_name + "' is not of the expected type. Should be a Positions");
630 :
631 : // Keep track of which positions is tied to what subapp
632 2200 : for (unsigned int i = 0; i < data.size(); ++i)
633 : {
634 1697 : if (input_files.size() != 1)
635 0 : _input_files.push_back(input_files[p_obj_it]);
636 1697 : _positions_objs.push_back(positions_obj);
637 1697 : _positions_index_offsets.push_back(offset);
638 : }
639 503 : offset += data.size();
640 : }
641 : }
642 : else
643 : {
644 1673 : _positions = {Point()};
645 :
646 1673 : if (_positions.size() < _input_files.size())
647 4 : mooseError("Not enough positions for the number of input files provided in MultiApp ",
648 4 : name());
649 : }
650 :
651 : mooseAssert(_input_files.size() == 1 || _positions.size() == _input_files.size(),
652 : "Number of positions and input files are not the same!");
653 7513 : }
654 :
655 : void
656 67950 : MultiApp::preTransfer(Real /*dt*/, Real target_time)
657 : {
658 : // Get a transient executioner to get a user-set tolerance
659 67950 : Real timestep_tol = 1e-13;
660 67950 : if (dynamic_cast<TransientBase *>(_fe_problem.getMooseApp().getExecutioner()))
661 65035 : timestep_tol =
662 65035 : dynamic_cast<TransientBase *>(_fe_problem.getMooseApp().getExecutioner())->timestepTol();
663 :
664 : // Determination on whether we need to backup the app due to changes below
665 67950 : bool backup_apps = false;
666 :
667 : // First, see if any Apps need to be reset
668 68729 : for (unsigned int i = 0; i < _reset_times.size(); i++)
669 : {
670 855 : if (!_reset_happened[i] && (target_time + timestep_tol >= _reset_times[i]))
671 : {
672 76 : _reset_happened[i] = true;
673 76 : if (_reset_apps.size() > 0)
674 152 : for (auto & app : _reset_apps)
675 76 : resetApp(app);
676 :
677 : // If we reset an application, then we delete the old objects, including the coordinate
678 : // transformation classes. Consequently we need to reset the coordinate transformation classes
679 : // in the associated transfer classes
680 109 : for (auto * const transfer : _associated_transfers)
681 33 : transfer->getAppInfo();
682 :
683 : // Similarly we need to transform the mesh again
684 76 : if (_run_in_position)
685 0 : for (const auto i : make_range(_my_num_apps))
686 : {
687 0 : auto app_ptr = _apps[i];
688 0 : if (usingPositions())
689 0 : app_ptr->getExecutioner()->feProblem().coordTransform().transformMesh(
690 0 : app_ptr->getExecutioner()->feProblem().mesh(), _positions[_first_local_app + i]);
691 : else
692 0 : app_ptr->getExecutioner()->feProblem().coordTransform().transformMesh(
693 0 : app_ptr->getExecutioner()->feProblem().mesh(), Point(0, 0, 0));
694 0 : }
695 :
696 : // If the time step covers multiple reset times, set them all as having 'happened'
697 174 : for (unsigned int j = i; j < _reset_times.size(); j++)
698 98 : if (target_time + timestep_tol >= _reset_times[j])
699 87 : _reset_happened[j] = true;
700 :
701 : // Backup in case the next solve fails
702 76 : backup_apps = true;
703 :
704 76 : break;
705 : }
706 : }
707 :
708 : // Now move any apps that should be moved
709 67950 : if (_use_positions && !_move_happened && target_time + timestep_tol >= _move_time)
710 : {
711 60 : _move_happened = true;
712 116 : for (unsigned int i = 0; i < _move_apps.size(); i++)
713 60 : moveApp(_move_apps[i], _move_positions[i]);
714 :
715 : // Backup in case the next solve fails
716 56 : backup_apps = true;
717 : }
718 :
719 67946 : if (backup_apps)
720 111 : backup();
721 67946 : }
722 :
723 : Executioner *
724 0 : MultiApp::getExecutioner(unsigned int app)
725 : {
726 0 : if (!_has_an_app)
727 0 : mooseError("No app for ", name(), " on processor ", _orig_rank);
728 :
729 0 : return _apps[globalAppToLocal(app)]->getExecutioner();
730 : }
731 :
732 : void
733 4571 : MultiApp::finalize()
734 : {
735 11325 : for (const auto & app_ptr : _apps)
736 : {
737 6754 : auto * executioner = app_ptr->getExecutioner();
738 : mooseAssert(executioner, "Executioner is nullptr");
739 :
740 6754 : executioner->feProblem().execute(EXEC_FINAL);
741 6754 : executioner->feProblem().outputStep(EXEC_FINAL);
742 : }
743 4571 : }
744 :
745 : void
746 4909 : MultiApp::postExecute()
747 : {
748 12281 : for (const auto & app_ptr : _apps)
749 : {
750 7372 : auto * executioner = app_ptr->getExecutioner();
751 : mooseAssert(executioner, "Executioner is nullptr");
752 :
753 7372 : executioner->postExecute();
754 : }
755 4909 : }
756 :
757 : void
758 25129 : MultiApp::backup()
759 : {
760 25129 : TIME_SECTION(_backup_timer);
761 :
762 25129 : if (_fe_problem.verboseMultiApps())
763 1059 : _console << "Backed up MultiApp ... ";
764 :
765 59611 : for (unsigned int i = 0; i < _my_num_apps; i++)
766 34482 : _sub_app_backups[i] = _apps[i]->backup();
767 :
768 25129 : if (_fe_problem.verboseMultiApps())
769 1059 : _console << name() << std::endl;
770 25129 : }
771 :
772 : void
773 46627 : MultiApp::restore(bool force)
774 : {
775 46627 : TIME_SECTION(_restore_timer);
776 :
777 46627 : if (force || needsRestoration())
778 : {
779 : // Must be restarting / recovering from main app so hold off on restoring
780 : // Instead - the restore will happen in sub-apps' initialSetup()
781 : // Note that _backups was already populated by dataLoad() in the main app
782 10346 : if (_fe_problem.getCurrentExecuteOnFlag() == EXEC_INITIAL)
783 552 : return;
784 :
785 : // We temporarily copy and store solutions for all subapps
786 9794 : if (_keep_solution_during_restore)
787 : {
788 355 : _end_solutions.resize(_my_num_apps);
789 :
790 706 : for (unsigned int i = 0; i < _my_num_apps; i++)
791 : {
792 355 : _end_solutions[i].resize(_apps[i]->getExecutioner()->feProblem().numSolverSystems());
793 722 : for (unsigned int j = 0; j < _apps[i]->getExecutioner()->feProblem().numSolverSystems();
794 : j++)
795 : {
796 734 : _end_solutions[i][j] = _apps[i]
797 : ->getExecutioner()
798 367 : ->feProblem()
799 367 : .getSolverSystem(/*solver_sys=*/j)
800 367 : .solution()
801 734 : .clone();
802 : }
803 : auto & sub_multiapps =
804 355 : _apps[i]->getExecutioner()->feProblem().getMultiAppWarehouse().getObjects();
805 :
806 : // multiapps of each subapp should do the same things
807 : // It is implemented recursively
808 366 : for (auto & multi_app : sub_multiapps)
809 15 : multi_app->keepSolutionDuringRestore(_keep_solution_during_restore);
810 : }
811 : }
812 :
813 : // We temporarily copy and store solutions for all subapps
814 9790 : if (_keep_aux_solution_during_restore)
815 : {
816 132 : _end_aux_solutions.resize(_my_num_apps);
817 :
818 264 : for (unsigned int i = 0; i < _my_num_apps; i++)
819 132 : _end_aux_solutions[i] =
820 264 : _apps[i]->getExecutioner()->feProblem().getAuxiliarySystem().solution().clone();
821 : }
822 :
823 9790 : if (_fe_problem.verboseMultiApps())
824 516 : _console << "Restoring MultiApp ... ";
825 :
826 19765 : for (unsigned int i = 0; i < _my_num_apps; i++)
827 : {
828 9975 : _apps[i]->restore(std::move(_sub_app_backups[i]), false);
829 9975 : _sub_app_backups[i] = _apps[i]->finalizeRestore();
830 : mooseAssert(_sub_app_backups[i], "Should have a backup");
831 : }
832 :
833 9790 : if (_fe_problem.verboseMultiApps())
834 516 : _console << name() << std::endl;
835 :
836 : // Now copy the latest solutions back for each subapp
837 9790 : if (_keep_solution_during_restore)
838 : {
839 702 : for (unsigned int i = 0; i < _my_num_apps; i++)
840 : {
841 714 : for (unsigned int j = 0; j < _apps[i]->getExecutioner()->feProblem().numSolverSystems();
842 : j++)
843 : {
844 363 : _apps[i]->getExecutioner()->feProblem().getSolverSystem(/*solver_sys=*/j).solution() =
845 726 : *_end_solutions[i][j];
846 :
847 : // We need to synchronize solution so that local_solution has the right values
848 363 : _apps[i]->getExecutioner()->feProblem().getSolverSystem(/*solver_sys=*/j).update();
849 : }
850 : }
851 :
852 351 : _end_solutions.clear();
853 : }
854 : // Now copy the latest auxiliary solutions back for each subapp
855 9790 : if (_keep_aux_solution_during_restore)
856 : {
857 264 : for (unsigned int i = 0; i < _my_num_apps; i++)
858 : {
859 132 : _apps[i]->getExecutioner()->feProblem().getAuxiliarySystem().solution() =
860 264 : *_end_aux_solutions[i];
861 :
862 : // We need to synchronize solution so that local_solution has the right values
863 132 : _apps[i]->getExecutioner()->feProblem().getAuxiliarySystem().update();
864 : }
865 :
866 132 : _end_aux_solutions.clear();
867 : }
868 :
869 : // Make sure the displaced mesh on the multiapp is up-to-date with displacement variables
870 19765 : for (const auto & app_ptr : _apps)
871 9975 : if (app_ptr->feProblem().getDisplacedProblem())
872 48 : app_ptr->feProblem().getDisplacedProblem()->updateMesh();
873 :
874 : // If we are restoring due to a failed solve, make sure reset the solved state in the sub-apps
875 9790 : if (!getMooseApp().getExecutioner()->lastSolveConverged())
876 1663 : for (auto & app_ptr : _apps)
877 918 : app_ptr->getExecutioner()->fixedPointSolve().clearFixedPointStatus();
878 : }
879 : else
880 : {
881 72562 : for (unsigned int i = 0; i < _my_num_apps; i++)
882 : {
883 36281 : for (auto & sub_app :
884 73033 : _apps[i]->getExecutioner()->feProblem().getMultiAppWarehouse().getObjects())
885 471 : sub_app->restore(false);
886 : }
887 : }
888 46623 : }
889 :
890 : void
891 15 : MultiApp::keepSolutionDuringRestore(bool keep_solution_during_restore)
892 : {
893 45 : if (_pars.isParamSetByUser("keep_solution_during_restore"))
894 8 : paramError("keep_solution_during_restore",
895 : "This parameter should only be provided in parent app");
896 :
897 11 : _keep_solution_during_restore = keep_solution_during_restore;
898 11 : }
899 :
900 : void
901 416465 : MultiApp::transformBoundingBox(BoundingBox & box, const MultiAppCoordTransform & transform)
902 : {
903 416465 : const Real min_x = box.first(0);
904 416465 : const Real max_x = box.second(0);
905 416465 : const Real min_y = box.first(1);
906 416465 : const Real max_y = box.second(1);
907 416465 : const Real min_z = box.first(2);
908 416465 : const Real max_z = box.second(2);
909 :
910 : std::array<Point, 8> box_corners = {{Point(min_x, min_y, min_z),
911 : Point(max_x, min_y, min_z),
912 : Point(min_x, max_y, min_z),
913 : Point(max_x, max_y, min_z),
914 : Point(min_x, min_y, max_z),
915 : Point(max_x, min_y, max_z),
916 : Point(min_x, max_y, max_z),
917 416465 : Point(max_x, max_y, max_z)}};
918 :
919 : // transform each corner
920 3748185 : for (auto & corner : box_corners)
921 3331720 : corner = transform(corner);
922 :
923 : // Create new bounding box
924 416465 : Point new_box_min = box_corners[0];
925 416465 : Point new_box_max = new_box_min;
926 3331720 : for (const auto p : make_range(1, 8))
927 11661020 : for (const auto d : make_range(Moose::dim))
928 : {
929 8745765 : const Point & pt = box_corners[p];
930 8745765 : if (new_box_min(d) > pt(d))
931 2540 : new_box_min(d) = pt(d);
932 :
933 8745765 : if (new_box_max(d) < pt(d))
934 1178964 : new_box_max(d) = pt(d);
935 : }
936 416465 : box.first = new_box_min;
937 416465 : box.second = new_box_max;
938 416465 : }
939 :
940 : BoundingBox
941 354231 : MultiApp::getBoundingBox(unsigned int app,
942 : bool displaced_mesh,
943 : const MultiAppCoordTransform * const coord_transform)
944 : {
945 354231 : if (!_has_an_app)
946 0 : mooseError("No app for ", name(), " on processor ", _orig_rank);
947 :
948 354231 : unsigned int local_app = globalAppToLocal(app);
949 354231 : FEProblemBase & fe_problem_base = _apps[local_app]->getExecutioner()->feProblem();
950 9078 : MooseMesh & mesh = (displaced_mesh && fe_problem_base.getDisplacedProblem().get() != NULL)
951 712146 : ? fe_problem_base.getDisplacedProblem()->mesh()
952 354231 : : fe_problem_base.mesh();
953 :
954 : {
955 354231 : Moose::ScopedCommSwapper swapper(_my_comm);
956 354231 : if (displaced_mesh)
957 4539 : _bounding_box[local_app] = MeshTools::create_bounding_box(mesh);
958 : else
959 : {
960 349692 : if (!_has_bounding_box[local_app])
961 : {
962 102 : _bounding_box[local_app] = MeshTools::create_bounding_box(mesh);
963 102 : _has_bounding_box[local_app] = true;
964 : }
965 : }
966 354231 : }
967 354231 : BoundingBox bbox = _bounding_box[local_app];
968 :
969 354231 : Point min = bbox.min();
970 354231 : min -= _bounding_box_padding;
971 354231 : Point max = bbox.max();
972 354231 : max += _bounding_box_padding;
973 :
974 354231 : Point inflation_amount = (max - min) * _inflation;
975 :
976 354231 : Point inflated_min = min - inflation_amount;
977 354231 : Point inflated_max = max + inflation_amount;
978 :
979 354231 : Point shifted_min = inflated_min;
980 354231 : Point shifted_max = inflated_max;
981 :
982 708462 : if ((!coord_transform || coord_transform->skipCoordinateCollapsing()) &&
983 708462 : fe_problem_base.getCoordSystem(*(mesh.meshSubdomains().begin())) == Moose::COORD_RZ)
984 : {
985 : // If the problem is RZ then we're going to invent a box that would cover the whole "3D" app
986 : // FIXME: Assuming all subdomains are the same coordinate system type!
987 342175 : shifted_min(0) = -inflated_max(0);
988 342175 : shifted_min(1) = inflated_min(1);
989 342175 : shifted_min(2) = -inflated_max(0);
990 :
991 342175 : shifted_max(0) = inflated_max(0);
992 342175 : shifted_max(1) = inflated_max(1);
993 342175 : shifted_max(2) = inflated_max(0);
994 : }
995 :
996 354231 : if (coord_transform)
997 : {
998 354231 : BoundingBox transformed_bbox(shifted_min, shifted_max);
999 354231 : transformBoundingBox(transformed_bbox, *coord_transform);
1000 354231 : return transformed_bbox;
1001 : }
1002 : else
1003 : {
1004 : // This is where the app is located. We need to shift by this amount.
1005 0 : Point p = position(app);
1006 :
1007 : // Shift them to the position they're supposed to be
1008 0 : shifted_min += p;
1009 0 : shifted_max += p;
1010 0 : return BoundingBox(shifted_min, shifted_max);
1011 : }
1012 : }
1013 :
1014 : FEProblemBase &
1015 518774 : MultiApp::appProblemBase(unsigned int app)
1016 : {
1017 518774 : if (!_has_an_app)
1018 0 : mooseError("No app for ", name(), " on processor ", _orig_rank);
1019 :
1020 518774 : unsigned int local_app = globalAppToLocal(app);
1021 :
1022 518774 : return _apps[local_app]->getExecutioner()->feProblem();
1023 : }
1024 :
1025 : FEProblem &
1026 0 : MultiApp::appProblem(unsigned int app)
1027 : {
1028 0 : mooseDeprecated(
1029 : "MultiApp::appProblem() is deprecated, call MultiApp::appProblemBase() instead.\n");
1030 0 : if (!_has_an_app)
1031 0 : mooseError("No app for ", name(), " on processor ", _orig_rank);
1032 :
1033 0 : unsigned int local_app = globalAppToLocal(app);
1034 :
1035 0 : return dynamic_cast<FEProblem &>(_apps[local_app]->getExecutioner()->feProblem());
1036 : }
1037 :
1038 : const UserObject &
1039 186397 : MultiApp::appUserObjectBase(unsigned int app, const std::string & name)
1040 : {
1041 186397 : if (!_has_an_app)
1042 0 : mooseError("No app for ", MultiApp::name(), " on processor ", _orig_rank);
1043 :
1044 186397 : return appProblemBase(app).getUserObjectBase(name);
1045 : }
1046 :
1047 : Real
1048 4152 : MultiApp::appPostprocessorValue(unsigned int app, const std::string & name)
1049 : {
1050 4152 : if (!_has_an_app)
1051 0 : mooseError("No app for ", MultiApp::name(), " on processor ", _orig_rank);
1052 :
1053 4152 : return appProblemBase(app).getPostprocessorValueByName(name);
1054 : }
1055 :
1056 : NumericVector<Number> &
1057 942 : MultiApp::appTransferVector(unsigned int app, std::string var_name)
1058 : {
1059 942 : return *(appProblemBase(app).getSystem(var_name).solution);
1060 : }
1061 :
1062 : bool
1063 150 : MultiApp::isFirstLocalRank() const
1064 : {
1065 150 : return _rank_config.is_first_local_rank;
1066 : }
1067 :
1068 : bool
1069 870607 : MultiApp::hasLocalApp(unsigned int global_app) const
1070 : {
1071 870607 : if (_has_an_app && global_app >= _first_local_app &&
1072 783884 : global_app <= _first_local_app + (_my_num_apps - 1))
1073 698966 : return true;
1074 :
1075 171641 : return false;
1076 : }
1077 :
1078 : MooseApp *
1079 11202 : MultiApp::localApp(unsigned int local_app)
1080 : {
1081 : mooseAssert(local_app < _apps.size(), "Index out of range: " + Moose::stringify(local_app));
1082 11202 : return _apps[local_app].get();
1083 : }
1084 :
1085 : void
1086 76 : MultiApp::resetApp(unsigned int global_app, Real time)
1087 : {
1088 76 : TIME_SECTION(_reset_timer);
1089 :
1090 76 : Moose::ScopedCommSwapper swapper(_my_comm);
1091 :
1092 76 : if (hasLocalApp(global_app))
1093 : {
1094 76 : unsigned int local_app = globalAppToLocal(global_app);
1095 :
1096 : // Extract the file numbers from the output, so that the numbering is maintained after reset
1097 76 : std::map<std::string, unsigned int> m = _apps[local_app]->getOutputWarehouse().getFileNumbers();
1098 :
1099 76 : createApp(local_app, time);
1100 :
1101 : // Reset the file numbers of the newly reset apps
1102 76 : _apps[local_app]->getOutputWarehouse().setFileNumbers(m);
1103 76 : }
1104 76 : }
1105 :
1106 : void
1107 60 : MultiApp::moveApp(unsigned int global_app, Point p)
1108 : {
1109 60 : if (_use_positions)
1110 : {
1111 60 : _positions[global_app] = p;
1112 :
1113 60 : if (hasLocalApp(global_app))
1114 : {
1115 60 : unsigned int local_app = globalAppToLocal(global_app);
1116 :
1117 60 : if (_output_in_position)
1118 56 : _apps[local_app]->setOutputPosition(p);
1119 60 : if (_run_in_position)
1120 8 : paramError("run_in_position", "Moving apps and running apps in position is not supported");
1121 : }
1122 : }
1123 56 : }
1124 :
1125 : void
1126 21 : MultiApp::parentOutputPositionChanged()
1127 : {
1128 21 : if (_use_positions && _output_in_position)
1129 42 : for (unsigned int i = 0; i < _apps.size(); i++)
1130 21 : _apps[i]->setOutputPosition(_app.getOutputPosition() + _positions[_first_local_app + i]);
1131 21 : }
1132 :
1133 : void
1134 12483 : MultiApp::createApp(unsigned int i, Real start_time)
1135 : {
1136 : // Delete the old app if we're resetting
1137 12483 : if (_apps[i])
1138 76 : _apps[i].reset();
1139 :
1140 : // Define the app name
1141 12483 : const std::string multiapp_name = getMultiAppName(name(), _first_local_app + i, _total_num_apps);
1142 12483 : std::string full_name;
1143 :
1144 : // Only add parent name if the parent is not the main app
1145 12483 : if (_app.multiAppLevel() > 0)
1146 525 : full_name = _app.name() + "_" + multiapp_name;
1147 : else
1148 11958 : full_name = multiapp_name;
1149 :
1150 12483 : InputParameters app_params = AppFactory::instance().getValidParams(_app_type);
1151 24966 : app_params.set<FEProblemBase *>("_parent_fep") = &_fe_problem;
1152 24966 : app_params.set<std::unique_ptr<Backup> *>("_initial_backup") = &_sub_app_backups[i];
1153 :
1154 : // Build the CommandLine with the relevant options for this subapp and add the
1155 : // cli args from the input file
1156 12483 : std::vector<std::string> input_cli_args;
1157 12483 : if (cliArgs().size() > 0 || _cli_args_from_file.size() > 0)
1158 3247 : input_cli_args = getCommandLineArgs(i);
1159 : // This will mark all hit CLI command line parameters that are passed to subapps
1160 : // as used within the parent app (_app)
1161 12483 : auto app_cli = _app.commandLine()->initSubAppCommandLine(name(), multiapp_name, input_cli_args);
1162 12483 : app_cli->parse();
1163 :
1164 12483 : if (_fe_problem.verboseMultiApps())
1165 1339 : _console << COLOR_CYAN << "Creating MultiApp " << name() << " of type " << _app_type
1166 1339 : << " of level " << _app.multiAppLevel() + 1 << " and number " << _first_local_app + i
1167 1339 : << " on processor " << processor_id() << " with full name " << full_name
1168 1339 : << COLOR_DEFAULT << std::endl;
1169 24966 : app_params.set<unsigned int>("_multiapp_level") = _app.multiAppLevel() + 1;
1170 24966 : app_params.set<unsigned int>("_multiapp_number") = _first_local_app + i;
1171 24966 : app_params.set<const MooseMesh *>("_master_mesh") = &_fe_problem.mesh();
1172 : #ifdef MOOSE_MFEM_ENABLED
1173 16708 : app_params.set<std::shared_ptr<mfem::Device>>("_mfem_device") =
1174 25062 : _app.getMFEMDevice(Moose::PassKey<MultiApp>());
1175 8354 : const auto & mfem_device_set = _app.getMFEMDevices(Moose::PassKey<MultiApp>());
1176 8354 : app_params.set<std::set<std::string>>("_mfem_devices") = mfem_device_set;
1177 : #endif
1178 62415 : if (getParam<bool>("clone_master_mesh") || getParam<bool>("clone_parent_mesh"))
1179 : {
1180 849 : if (_fe_problem.verboseMultiApps())
1181 0 : _console << COLOR_CYAN << "Cloned parent app mesh will be used for MultiApp " << name()
1182 0 : << COLOR_DEFAULT << std::endl;
1183 849 : app_params.set<bool>("_use_master_mesh") = true;
1184 849 : auto displaced_problem = _fe_problem.getDisplacedProblem();
1185 849 : if (displaced_problem)
1186 0 : app_params.set<const MooseMesh *>("_master_displaced_mesh") = &displaced_problem->mesh();
1187 849 : }
1188 :
1189 : // If only one input file was provided, use it for all the solves
1190 12483 : const auto input_index = _input_files.size() == 1 ? 0 : _first_local_app + i;
1191 12483 : const auto & input_file = _input_files[input_index];
1192 :
1193 : // create new parser tree for the application and parse
1194 12483 : auto parser = std::make_unique<Parser>(input_file);
1195 12483 : parser->setCommandLineParams(app_cli->buildHitParams());
1196 12483 : parser->parse();
1197 :
1198 : // Checks on app type
1199 12483 : const auto & app_type = parser->getAppType();
1200 12483 : if (app_type.empty() && _app_type.empty())
1201 0 : mooseWarning("The application type is not specified for ",
1202 : full_name,
1203 : ". Please use [Application] block to specify the application type.");
1204 12483 : if (!app_type.empty() && app_type != _app_type && !AppFactory::instance().isRegistered(app_type))
1205 4 : mooseError("In the ",
1206 : full_name,
1207 : ", '",
1208 : app_type,
1209 : "' is not a registered application. The registered application is named: '",
1210 4 : _app_type,
1211 : "'. Please double check the [Application] block to make sure the correct "
1212 : "application is provided. \n");
1213 :
1214 12479 : if (parser->getAppType().empty())
1215 12466 : parser->setAppType(_app_type);
1216 :
1217 24958 : app_params.set<std::shared_ptr<Parser>>("_parser") = std::move(parser);
1218 24958 : app_params.set<std::shared_ptr<CommandLine>>("_command_line") = std::move(app_cli);
1219 12479 : _apps[i] = AppFactory::instance().create(_app_type, full_name, app_params, _my_comm);
1220 12479 : auto & app = _apps[i];
1221 :
1222 12479 : app->setGlobalTimeOffset(start_time);
1223 12479 : app->setOutputFileNumbers(_app.getOutputWarehouse().getFileNumbers());
1224 12479 : app->setRestart(_app.isRestarting());
1225 12479 : app->setRecover(_app.isRecovering());
1226 :
1227 37437 : if (_use_positions && getParam<bool>("output_in_position"))
1228 3334 : app->setOutputPosition(_app.getOutputPosition() + _positions[_first_local_app + i]);
1229 12479 : if (_output_in_position && _run_in_position)
1230 8 : paramError("run_in_position",
1231 : "Sub-apps are already displaced, so they are already output in position");
1232 :
1233 : // Update the MultiApp level for the app that was just created
1234 12475 : app->setupOptions();
1235 : // if multiapp does not have file base in Outputs input block, output file base will
1236 : // be empty here since setupOptions() does not set the default file base with the multiapp
1237 : // input file name. Parent app will create the default file base for multiapp by taking the
1238 : // output base of the parent app problem and appending the name of the multiapp plus a number to
1239 : // it
1240 12475 : if (app->getOutputFileBase().empty())
1241 12453 : setAppOutputFileBase(i);
1242 12475 : preRunInputFile();
1243 37425 : if (_app.multiAppLevel() > getParam<unsigned int>("max_multiapp_level"))
1244 4 : mooseError("Maximum multiapp level has been reached. This is likely caused by an infinite loop "
1245 : "in your multiapp system. If additional multiapp levels are needed, "
1246 : "max_multiapp_level can be specified in the MuliApps block.");
1247 :
1248 : // Transfer coupling relaxation information to the subapps
1249 24942 : _apps[i]->fixedPointConfig().sub_relaxation_factor = getParam<Real>("relaxation_factor");
1250 12471 : _apps[i]->fixedPointConfig().sub_transformed_vars =
1251 37413 : getParam<std::vector<std::string>>("transformed_variables");
1252 : // Handle deprecated parameter
1253 37413 : if (!parameters().isParamSetByAddParam("relaxed_variables"))
1254 0 : _apps[i]->fixedPointConfig().sub_transformed_vars =
1255 0 : getParam<std::vector<std::string>>("relaxed_variables");
1256 12471 : _apps[i]->fixedPointConfig().sub_transformed_pps =
1257 37413 : getParam<std::vector<PostprocessorName>>("transformed_postprocessors");
1258 :
1259 12471 : app->runInputFile();
1260 12423 : auto fixed_point_solve = &(_apps[i]->getExecutioner()->fixedPointSolve());
1261 12423 : if (fixed_point_solve)
1262 12423 : fixed_point_solve->allocateStorage(false);
1263 :
1264 : // Transform the app mesh if requested
1265 12423 : if (_run_in_position)
1266 : {
1267 274 : if (usingPositions())
1268 548 : app->getExecutioner()->feProblem().coordTransform().transformMesh(
1269 274 : app->getExecutioner()->feProblem().mesh(), _positions[_first_local_app + i]);
1270 : else
1271 0 : app->getExecutioner()->feProblem().coordTransform().transformMesh(
1272 0 : app->getExecutioner()->feProblem().mesh(), Point(0, 0, 0));
1273 : }
1274 12419 : }
1275 :
1276 : std::vector<std::string>
1277 3247 : MultiApp::getCommandLineArgs(const unsigned int local_app)
1278 : {
1279 3247 : const auto cla = cliArgs();
1280 3247 : auto cli_args_param = _cli_args_param;
1281 3247 : std::string combined_args;
1282 :
1283 : // Single set of args from cliArgs() to be provided to all apps
1284 3247 : if (cla.size() == 1)
1285 2323 : combined_args = cla[0];
1286 : // Single "cli_args_files" file to be provided to all apps
1287 924 : else if (_cli_args_from_file.size() == 1)
1288 : {
1289 0 : cli_args_param = "cli_args_files";
1290 0 : combined_args = _cli_args_from_file[0];
1291 : }
1292 : // Unique set of args from cliArgs() to be provided to each app
1293 924 : else if (cla.size())
1294 804 : combined_args = cla[local_app + _first_local_app];
1295 : // Unique set of args from "cli_args_files" to be provided to all apps
1296 : else
1297 : {
1298 120 : cli_args_param = "cli_args_files";
1299 120 : combined_args = _cli_args_from_file[local_app + _first_local_app];
1300 : }
1301 :
1302 : // Remove all of the beginning and end whitespace so we can recognize truly empty
1303 3247 : combined_args = MooseUtils::trim(combined_args);
1304 :
1305 : // MooseUtils::split will return a single empty entry if there is nothing,
1306 : // so exit early if we have nothing
1307 3247 : if (combined_args.empty())
1308 0 : return {};
1309 :
1310 : // Split the argument into a vector of arguments, and make sure
1311 : // that we don't have any empty arguments
1312 6494 : const auto args = MooseUtils::split(combined_args, ";");
1313 10687 : for (const auto & arg : args)
1314 : {
1315 7440 : if (arg.empty())
1316 : {
1317 0 : const auto error = "An empty MultiApp command line argument was provided. Your "
1318 : "combined command line string has a ';' with no argument after it.";
1319 0 : if (cli_args_param)
1320 0 : paramError(*cli_args_param, error);
1321 : else
1322 0 : mooseError(error);
1323 : }
1324 : }
1325 :
1326 3247 : return args;
1327 3247 : }
1328 :
1329 : LocalRankConfig
1330 8636 : rankConfig(processor_id_type rank,
1331 : processor_id_type nprocs,
1332 : dof_id_type napps,
1333 : processor_id_type min_app_procs,
1334 : processor_id_type max_app_procs,
1335 : bool batch_mode)
1336 : {
1337 8636 : if (min_app_procs > nprocs)
1338 0 : mooseError("minimum number of procs per app is higher than the available number of procs");
1339 8636 : else if (min_app_procs > max_app_procs)
1340 0 : mooseError("minimum number of procs per app must be lower than the max procs per app");
1341 :
1342 : mooseAssert(rank < nprocs, "rank must be smaller than the number of procs");
1343 :
1344 : // A "slot" is a group of procs/ranks that are grouped together to run a
1345 : // single (sub)app/sim in parallel.
1346 :
1347 : const processor_id_type slot_size =
1348 8636 : std::max(std::min(cast_int<processor_id_type>(nprocs / napps), max_app_procs), min_app_procs);
1349 8636 : const processor_id_type nslots = std::min(
1350 17272 : nprocs / slot_size,
1351 8636 : cast_int<processor_id_type>(std::min(
1352 8636 : static_cast<dof_id_type>(std::numeric_limits<processor_id_type>::max()), napps)));
1353 : mooseAssert(nprocs >= (nslots * slot_size),
1354 : "Ensure that leftover procs is represented by an unsigned type");
1355 8636 : const processor_id_type leftover_procs = nprocs - nslots * slot_size;
1356 8636 : const dof_id_type apps_per_slot = napps / nslots;
1357 8636 : const dof_id_type leftover_apps = napps % nslots;
1358 :
1359 8636 : std::vector<int> slot_for_rank(nprocs);
1360 8636 : processor_id_type slot = 0;
1361 8636 : processor_id_type procs_in_slot = 0;
1362 21698 : for (processor_id_type rankiter = 0; rankiter <= rank; rankiter++)
1363 : {
1364 13062 : if (slot < nslots)
1365 12838 : slot_for_rank[rankiter] = cast_int<int>(slot);
1366 : else
1367 224 : slot_for_rank[rankiter] = -1;
1368 13062 : procs_in_slot++;
1369 : // this slot keeps growing until we reach slot size plus possibly an extra
1370 : // proc if there were any leftover from the slotization of nprocs - this
1371 : // must also make sure we don't go over max app procs.
1372 13062 : if (procs_in_slot == slot_size + 1 * (slot < leftover_procs && slot_size < max_app_procs))
1373 : {
1374 9308 : procs_in_slot = 0;
1375 9308 : slot++;
1376 : }
1377 : }
1378 :
1379 8636 : if (slot_for_rank[rank] < 0)
1380 : // ranks assigned a negative slot don't have any apps running on them.
1381 90 : return {0, 0, 0, 0, false, 0};
1382 8546 : const processor_id_type slot_num = cast_int<processor_id_type>(slot_for_rank[rank]);
1383 :
1384 8546 : const bool is_first_local_rank = rank == 0 || (slot_for_rank[rank - 1] != slot_for_rank[rank]);
1385 8546 : const dof_id_type n_local_apps = apps_per_slot + 1 * (slot_num < leftover_apps);
1386 :
1387 8546 : processor_id_type my_first_rank = 0;
1388 11238 : for (processor_id_type rankiter = rank; rankiter > 0; rankiter--)
1389 3048 : if (slot_for_rank[rank] != slot_for_rank[rankiter])
1390 : {
1391 356 : my_first_rank = cast_int<processor_id_type>(slot_for_rank[rankiter + 1]);
1392 356 : break;
1393 : }
1394 :
1395 8546 : dof_id_type app_index = 0;
1396 10230 : for (processor_id_type slot = 0; slot < slot_num; slot++)
1397 : {
1398 1684 : const dof_id_type num_slot_apps = apps_per_slot + 1 * (slot < leftover_apps);
1399 1684 : app_index += num_slot_apps;
1400 : }
1401 :
1402 8546 : if (batch_mode)
1403 425 : return {n_local_apps, app_index, 1, slot_num, is_first_local_rank, my_first_rank};
1404 8121 : return {n_local_apps, app_index, n_local_apps, app_index, is_first_local_rank, my_first_rank};
1405 8636 : }
1406 :
1407 : void
1408 7632 : MultiApp::buildComm()
1409 : {
1410 : int ierr;
1411 :
1412 7632 : ierr = MPI_Comm_size(_communicator.get(), &_orig_num_procs);
1413 7632 : mooseCheckMPIErr(ierr);
1414 7632 : ierr = MPI_Comm_rank(_communicator.get(), &_orig_rank);
1415 7632 : mooseCheckMPIErr(ierr);
1416 :
1417 : #ifdef LIBMESH_HAVE_SYS_UTSNAME_H
1418 : struct utsname sysInfo;
1419 7632 : uname(&sysInfo);
1420 7632 : _node_name = sysInfo.nodename;
1421 : #else
1422 : _node_name = "Unknown";
1423 : #endif
1424 :
1425 : int rank;
1426 7632 : ierr = MPI_Comm_rank(_communicator.get(), &rank);
1427 7632 : mooseCheckMPIErr(ierr);
1428 :
1429 7632 : _my_num_apps = _rank_config.num_local_apps;
1430 7632 : _first_local_app = _rank_config.first_local_app_index;
1431 :
1432 7632 : _has_an_app = _rank_config.num_local_apps > 0;
1433 7632 : if (_rank_config.first_local_app_index >= _total_num_apps)
1434 0 : mooseError("Internal error, a processor has an undefined app.");
1435 :
1436 7632 : if (_has_an_app)
1437 : {
1438 7578 : _communicator.split(_rank_config.first_local_app_index, rank, _my_communicator);
1439 7578 : ierr = MPI_Comm_rank(_my_comm, &_my_rank);
1440 7578 : mooseCheckMPIErr(ierr);
1441 : }
1442 : else
1443 : {
1444 54 : _communicator.split(MPI_UNDEFINED, rank, _my_communicator);
1445 54 : _my_rank = 0;
1446 : }
1447 7632 : }
1448 :
1449 : unsigned int
1450 873225 : MultiApp::globalAppToLocal(unsigned int global_app)
1451 : {
1452 873225 : if (global_app >= _first_local_app && global_app <= _first_local_app + (_my_num_apps - 1))
1453 873225 : return global_app - _first_local_app;
1454 :
1455 0 : std::stringstream ss;
1456 0 : ss << "Requesting app " << global_app << ", but processor " << processor_id() << " ";
1457 0 : if (_my_num_apps == 0)
1458 0 : ss << "does not own any apps";
1459 0 : else if (_my_num_apps == 1)
1460 0 : ss << "owns app " << _first_local_app;
1461 : else
1462 0 : ss << "owns apps " << _first_local_app << "-" << _first_local_app + (_my_num_apps - 1);
1463 0 : ss << ".";
1464 0 : mooseError("Invalid global_app!\n", ss.str());
1465 : return 0;
1466 0 : }
1467 :
1468 : void
1469 12475 : MultiApp::preRunInputFile()
1470 : {
1471 12475 : }
1472 :
1473 : void
1474 13403 : MultiApp::addAssociatedTransfer(MultiAppTransfer & transfer)
1475 : {
1476 13403 : _associated_transfers.push_back(&transfer);
1477 13403 : }
1478 :
1479 : void
1480 0 : MultiApp::setAppOutputFileBase()
1481 : {
1482 0 : for (unsigned int i = 0; i < _my_num_apps; ++i)
1483 0 : setAppOutputFileBase(i);
1484 0 : }
1485 :
1486 : std::vector<std::string>
1487 23358 : MultiApp::cliArgs() const
1488 : {
1489 : // So that we can error out with paramError("cli_args", ...);
1490 23358 : _cli_args_param = "cli_args";
1491 46716 : return std::vector<std::string>(_cli_args.begin(), _cli_args.end());
1492 : }
1493 :
1494 : void
1495 12453 : MultiApp::setAppOutputFileBase(unsigned int index)
1496 : {
1497 : const std::string multiapp_name =
1498 12453 : getMultiAppName(name(), _first_local_app + index, _total_num_apps);
1499 12453 : _apps[index]->setOutputFileBase(_app.getOutputFileBase() + "_" + multiapp_name);
1500 12453 : }
1501 :
1502 : std::string
1503 24936 : MultiApp::getMultiAppName(const std::string & base_name, dof_id_type index, dof_id_type total)
1504 : {
1505 24936 : std::ostringstream multiapp_name;
1506 49872 : multiapp_name << base_name << std::setw(std::ceil(std::log10(total))) << std::setprecision(0)
1507 24936 : << std::setfill('0') << std::right << index;
1508 49872 : return multiapp_name.str();
1509 24936 : }
1510 :
1511 : const Point &
1512 208945 : MultiApp::position(unsigned int app) const
1513 : {
1514 : // If we're not using positions, it won't have changed
1515 208945 : if (_positions_objs.empty())
1516 192817 : return _positions[app];
1517 : else
1518 : // Find which Positions object is specifying it, and query a potentially updated value
1519 16128 : return _positions_objs[app]->getPosition(app - _positions_index_offsets[app], false);
1520 : }
1521 :
1522 : void
1523 2500 : dataStore(std::ostream & stream, SubAppBackups & backups, void * context)
1524 : {
1525 2500 : MultiApp * multi_app = static_cast<MultiApp *>(context);
1526 : mooseAssert(multi_app, "Not set");
1527 :
1528 2500 : multi_app->backup();
1529 :
1530 2500 : dataStore(stream, static_cast<std::vector<std::unique_ptr<Backup>> &>(backups), nullptr);
1531 2500 : }
1532 :
1533 : void
1534 651 : dataLoad(std::istream & stream, SubAppBackups & backups, void * context)
1535 : {
1536 651 : MultiApp * multi_app = static_cast<MultiApp *>(context);
1537 : mooseAssert(multi_app, "Not set");
1538 :
1539 651 : dataLoad(stream, static_cast<std::vector<std::unique_ptr<Backup>> &>(backups), nullptr);
1540 :
1541 651 : multi_app->restore();
1542 651 : }
|
